From a620add3b7353e1a7ec723927afa3ebcd7378201 Mon Sep 17 00:00:00 2001 From: Lumpiasty Date: Fri, 9 Oct 2026 00:34:09 +0200 Subject: [PATCH] llama: release/restore the MoE cache with the device weights The MoE cache (--moe-cache-mib, #29887) keeps the most recently used host experts in device banks. The on-demand VRAM path (release_device / restore_device, used by the server VRAM arbiter and the on-demand mmproj path) did not know about it, so its banks were never freed on handoff. Restoring them by reallocation alone would also be wrong: the banks come back zeroed while the slot maps still mark experts as cached, so the model would silently read zero weights. Free the banks on release and empty the cache instead of shadowing it: re-init every LRU, re-point its slot-map pointers at the host slot maps and clear those to -1. The banks are reallocated (which also re-inits their views) and cleared on restore, before the scheduler is rebuilt, and the cache refills from the host experts as before. Release runs in both modes, weights-only and evict_kv. device_resident() now covers the cache too, for the early returns in release/restore, the auto-restore in decode() and llama_context_weights_resident(). Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01PZz44SLQvTXMyWGio6t9DZ --- src/llama-context.cpp | 19 ++++++-- src/llama-context.h | 3 ++ src/llama-moe-cache.cpp | 99 ++++++++++++++++++++++++++++++++++++++++- src/llama-moe-cache.h | 6 +++ 4 files changed, 122 insertions(+), 5 deletions(-) diff --git a/src/llama-context.cpp b/src/llama-context.cpp index f6f8e7528..9d2d57bcd 100644 --- a/src/llama-context.cpp +++ b/src/llama-context.cpp @@ -981,13 +981,21 @@ void llama_context::synchronize() { t_compute_start_us = 0; } +bool llama_context::device_resident() const { + return model.weights_resident() && (!moe_cache || moe_cache->resident()); +} + void llama_context::release_device(bool evict_kv) { - if (!model.weights_resident() && !(evict_kv && !kv_device_evicted)) { + if (!device_resident() && !(evict_kv && !kv_device_evicted)) { return; } // ensure no compute is in flight before freeing the device buffers synchronize(); model.release_device_weights(); + // the banks are emptied, not shadowed: restored slot maps must not point at zeroed experts + if (moe_cache) { + moe_cache->release_device(); + } if (evict_kv && memory && !kv_device_evicted) { memory->release_device_buffers(); kv_device_evicted = true; @@ -1004,10 +1012,13 @@ void llama_context::release_device(bool evict_kv) { } void llama_context::restore_device() { - if (model.weights_resident() && !kv_device_evicted && sched) { + if (device_resident() && !kv_device_evicted && sched) { return; } model.restore_device_weights(); + if (moe_cache) { + moe_cache->restore_device(); + } if (kv_device_evicted && memory) { memory->restore_device_buffers(); kv_device_evicted = false; @@ -2208,7 +2219,7 @@ int llama_context::decode(const llama_batch_ext & batch_inp) { // on-demand VRAM: if the weights were released while this model was idle, bring them back // to the device before building the compute graph - if (!model.weights_resident()) { + if (!device_resident()) { restore_device(); } @@ -4727,7 +4738,7 @@ void llama_context_restore_device(llama_context * ctx) { } bool llama_context_weights_resident(const llama_context * ctx) { - return ctx->get_model().weights_resident(); + return ctx->device_resident(); } float * llama_get_logits(llama_context * ctx) { diff --git a/src/llama-context.h b/src/llama-context.h index a78bff50b..81b935096 100644 --- a/src/llama-context.h +++ b/src/llama-context.h @@ -68,6 +68,9 @@ struct llama_context { void release_device(bool evict_kv = false); void restore_device(); + // the model weights and the MoE cache are on the device + bool device_resident() const; + const llama_model & get_model() const; const llama_cparams & get_cparams() const; diff --git a/src/llama-moe-cache.cpp b/src/llama-moe-cache.cpp index ba35af021..7fffd4ff4 100644 --- a/src/llama-moe-cache.cpp +++ b/src/llama-moe-cache.cpp @@ -369,6 +369,7 @@ struct llama_moe_cache::impl { static constexpr int64_t max_batch = 32; int32_t n_expert_used; + bool no_alloc; stats stats_small; // up to 8 tokens per ubatch stats stats_large; @@ -400,7 +401,7 @@ struct llama_moe_cache::impl { ggml_context_ptr ctx_views; impl(const llama_model & model, bool no_alloc, const std::vector & backends, const std::vector & bufts, size_t size, float cpu_share) : - n_expert_used(model.hparams.n_expert_used_max()), layers(model.layers.size()), cpu_share(std::clamp(cpu_share, 0.0f, 1.0f)) { + n_expert_used(model.hparams.n_expert_used_max()), no_alloc(no_alloc), layers(model.layers.size()), cpu_share(std::clamp(cpu_share, 0.0f, 1.0f)) { for (size_t i = 0; i < backends.size(); ++i) { const auto dev_type = ggml_backend_dev_type(ggml_backend_get_device(backends[i])); if (dev_type == GGML_BACKEND_DEVICE_TYPE_GPU || dev_type == GGML_BACKEND_DEVICE_TYPE_IGPU) { @@ -861,6 +862,90 @@ struct llama_moe_cache::impl { return true; } + // on-demand VRAM: free the banks and forget the cached experts, the slot maps all read -1 again + void release_device() { + if (no_alloc) { + return; + } + size_t freed = 0; + for (device & d : devices) { + if (!d.buf) { + continue; + } + freed += ggml_backend_buffer_get_size(d.buf.get()); + d.buf.reset(); + for (ggml_tensor * t = ggml_get_first_tensor(d.ctx.get()); t != nullptr; t = ggml_get_next_tensor(d.ctx.get(), t)) { + t->buffer = nullptr; + t->data = nullptr; + } + } + invalidate(); + if (freed > 0) { + LLAMA_LOG_INFO("%s: released %.2f MiB of MoE cache\n", __func__, freed/1024.0/1024.0); + } + } + + // the banks come back empty, so the slot maps stay invalidated + bool restore_device() { + if (no_alloc) { + return true; + } + size_t restored = 0; + for (device & d : devices) { + if (!d.ctx || d.buf) { + continue; + } + // also re-inits the views of the banks + d.buf.reset(ggml_backend_alloc_ctx_tensors_from_buft(d.ctx.get(), d.buft)); + if (!d.buf) { + LLAMA_LOG_ERROR("%s: failed to reallocate the MoE cache buffer (out of VRAM?)\n", __func__); + return false; + } + ggml_backend_buffer_clear(d.buf.get(), 0); + ggml_backend_buffer_set_usage(d.buf.get(), GGML_BACKEND_BUFFER_USAGE_WEIGHTS); + d.buf_size = ggml_backend_buffer_get_size(d.buf.get()); + restored += d.buf_size; + } + if (restored > 0) { + LLAMA_LOG_INFO("%s: restored %.2f MiB of MoE cache\n", __func__, restored/1024.0/1024.0); + } + return true; + } + + bool resident() const { + for (const device & d : devices) { + if (d.ctx && !d.buf) { + return false; + } + } + return true; + } + + // empty every LRU and mark all experts as not cached + void invalidate() { + if (buf_host) { + ggml_backend_buffer_clear(buf_host.get(), 0xff); // the CPU maps and the unused slot maps are -1 + } + for (group & g : groups) { + if (g.n_slots == 0) { + continue; + } + // start from a fresh LRU: init() does not reset all of its state (n_filled, the scratch vectors) + g.lru = moe_cache_lru(); + g.lru.init((int32_t) layers.size(), (int32_t) g.ref[0]->ne[2], g.n_slots); + for (int32_t il : g.layers) { + layer & l = layers[il]; + if (l.slot_map != nullptr) { + int32_t * slots = (int32_t *) l.slot_map->data; + std::fill(slots, slots + l.slot_map->ne[1], g.lru.empty); // not cached + g.lru.slot_map[il] = slots; + } + l.cpu_experts.clear(); + l.cpu_acc = 0; + } + } + } + void log_stats() const { auto log = [](const char * name, const stats & st) { const size_t n = st.hits + st.misses; @@ -970,6 +1055,18 @@ int64_t llama_moe_cache::copy_experts(ggml_backend_t backend, const ggml_tensor return pimpl->copy_experts(backend, w, dst, e, last); } +void llama_moe_cache::release_device() { + pimpl->release_device(); +} + +bool llama_moe_cache::restore_device() { + return pimpl->restore_device(); +} + +bool llama_moe_cache::resident() const { + return pimpl->resident(); +} + std::map llama_moe_cache::memory_breakdown() const { std::map res; for (const auto & d : pimpl->devices) { diff --git a/src/llama-moe-cache.h b/src/llama-moe-cache.h index 833eac360..ed11fb81e 100644 --- a/src/llama-moe-cache.h +++ b/src/llama-moe-cache.h @@ -60,6 +60,12 @@ public: // returns the number of experts copied, 0 if expert e is not in the cache int64_t copy_experts(ggml_backend_t backend, const ggml_tensor * w, ggml_tensor * dst, int64_t e, int64_t last); + // on-demand VRAM: free the device banks and empty the cache, the caller must synchronize first + void release_device(); + // reallocate the banks empty, returns false if out of memory + bool restore_device(); + bool resident() const; + std::map memory_breakdown() const; private: