llama: release/restore the MoE cache with the device weights

The MoE cache (--moe-cache-mib, #29887) keeps the most recently used host
experts in device banks. The on-demand VRAM path (release_device /
restore_device, used by the server VRAM arbiter and the on-demand mmproj path)
did not know about it, so its banks were never freed on handoff. Restoring them
by reallocation alone would also be wrong: the banks come back zeroed while the
slot maps still mark experts as cached, so the model would silently read zero
weights.

Free the banks on release and empty the cache instead of shadowing it: re-init
every LRU, re-point its slot-map pointers at the host slot maps and clear those
to -1. The banks are reallocated (which also re-inits their views) and cleared
on restore, before the scheduler is rebuilt, and the cache refills from the
host experts as before. Release runs in both modes, weights-only and evict_kv.

device_resident() now covers the cache too, for the early returns in
release/restore, the auto-restore in decode() and
llama_context_weights_resident().

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01PZz44SLQvTXMyWGio6t9DZ
This commit is contained in:
LumpiastyandClaude Opus 5.5 committed 2026-10-24 18:33:19 +02:00
1 parent 53f9077719
commit 7ac2014a80
4 files changed
+117 -5

No files matched your search

+15 -4
View File
@@ -919,13 +919,21 @@ void llama_context::synchronize() {
t_compute_start_us = 0;
}
bool llama_context::device_resident() const {
return model.weights_resident() && (!moe_cache || moe_cache->resident());
}
void llama_context::release_device(bool evict_kv) {
if (!model.weights_resident() && !(evict_kv && !kv_device_evicted)) {
if (!device_resident() && !(evict_kv && !kv_device_evicted)) {
return;
}
// ensure no compute is in flight before freeing the device buffers
synchronize();
model.release_device_weights();
// the banks are emptied, not shadowed: restored slot maps must not point at zeroed experts
if (moe_cache) {
moe_cache->release_device();
}
if (evict_kv && memory && !kv_device_evicted) {
memory->release_device_buffers();
kv_device_evicted = true;
@@ -942,10 +950,13 @@ void llama_context::release_device(bool evict_kv) {
}
void llama_context::restore_device() {
if (model.weights_resident() && !kv_device_evicted && sched) {
if (device_resident() && !kv_device_evicted && sched) {
return;
}
model.restore_device_weights();
if (moe_cache) {
moe_cache->restore_device();
}
if (kv_device_evicted && memory) {
memory->restore_device_buffers();
kv_device_evicted = false;
@@ -1895,7 +1906,7 @@ int llama_context::decode(const llama_batch_ext & batch_inp) {
// on-demand VRAM: if the weights were released while this model was idle, bring them back
// to the device before building the compute graph
if (!model.weights_resident()) {
if (!device_resident()) {
restore_device();
}
@@ -4306,7 +4317,7 @@ void llama_context_restore_device(llama_context * ctx) {
}
bool llama_context_weights_resident(const llama_context * ctx) {
return ctx->get_model().weights_resident();
return ctx->device_resident();
}
float * llama_get_logits(llama_context * ctx) {
+3
View File
@@ -68,6 +68,9 @@ struct llama_context {
void release_device(bool evict_kv = false);
void restore_device();
// the model weights and the MoE cache are on the device
bool device_resident() const;
const llama_model & get_model() const;
const llama_cparams & get_cparams() const;
+93 -1
View File
@@ -230,6 +230,7 @@ struct llama_moe_cache::impl {
static constexpr int64_t max_batch = 32;
int32_t n_expert_used;
bool no_alloc;
stats stats_small; // up to 8 tokens per ubatch
stats stats_large;
@@ -253,7 +254,7 @@ struct llama_moe_cache::impl {
ggml_context_ptr ctx_views;
impl(const llama_model & model, const std::vector<ggml_backend_t> & backends, const std::vector<ggml_backend_buffer_type_t> & bufts, size_t size) :
n_expert_used(model.hparams.n_expert_used_max()), layers(model.layers.size()) {
n_expert_used(model.hparams.n_expert_used_max()), no_alloc(model.hparams.no_alloc), layers(model.layers.size()) {
for (size_t i = 0; i < backends.size(); ++i) {
const auto dev_type = ggml_backend_dev_type(ggml_backend_get_device(backends[i]));
if (dev_type == GGML_BACKEND_DEVICE_TYPE_GPU || dev_type == GGML_BACKEND_DEVICE_TYPE_IGPU) {
@@ -616,6 +617,85 @@ struct llama_moe_cache::impl {
return true;
}
// on-demand VRAM: free the banks and forget the cached experts, the slot maps all read -1 again
void release_device() {
if (no_alloc) {
return;
}
size_t freed = 0;
for (device & d : devices) {
if (!d.buf) {
continue;
}
freed += ggml_backend_buffer_get_size(d.buf.get());
d.buf.reset();
for (ggml_tensor * t = ggml_get_first_tensor(d.ctx.get()); t != nullptr; t = ggml_get_next_tensor(d.ctx.get(), t)) {
t->buffer = nullptr;
t->data = nullptr;
}
}
invalidate();
if (freed > 0) {
LLAMA_LOG_INFO("%s: released %.2f MiB of MoE cache\n", __func__, freed/1024.0/1024.0);
}
}
// the banks come back empty, so the slot maps stay invalidated
bool restore_device() {
if (no_alloc) {
return true;
}
size_t restored = 0;
for (device & d : devices) {
if (!d.ctx || d.buf) {
continue;
}
// also re-inits the views of the banks
d.buf.reset(ggml_backend_alloc_ctx_tensors_from_buft(d.ctx.get(), d.buft));
if (!d.buf) {
LLAMA_LOG_ERROR("%s: failed to reallocate the MoE cache buffer (out of VRAM?)\n", __func__);
return false;
}
ggml_backend_buffer_clear(d.buf.get(), 0);
ggml_backend_buffer_set_usage(d.buf.get(), GGML_BACKEND_BUFFER_USAGE_WEIGHTS);
d.buf_size = ggml_backend_buffer_get_size(d.buf.get());
restored += d.buf_size;
}
if (restored > 0) {
LLAMA_LOG_INFO("%s: restored %.2f MiB of MoE cache\n", __func__, restored/1024.0/1024.0);
}
return true;
}
bool resident() const {
for (const device & d : devices) {
if (d.ctx && !d.buf) {
return false;
}
}
return true;
}
// empty every SLRU and mark all experts as not cached
void invalidate() {
for (group & g : groups) {
if (g.n_slots == 0) {
continue;
}
// start from a fresh SLRU: init() does not reset the protected segment
g.slru = moe_cache_slru();
g.slru.init((int32_t) layers.size(), (int32_t) g.ref[0]->ne[2], g.n_slots);
for (int32_t il : g.layers) {
if (layers[il].slot_map != nullptr) {
g.slru.slot_map[il] = (int32_t *) layers[il].slot_map->data;
}
}
}
if (buf_host) {
ggml_backend_buffer_clear(buf_host.get(), 0xff); // all slots are -1
}
}
void log_stats() const {
auto log = [](const char * name, const stats & st) {
const size_t n = st.hits + st.misses;
@@ -659,6 +739,18 @@ int64_t llama_moe_cache::copy_experts(ggml_backend_t backend, const ggml_tensor
return pimpl->copy_experts(backend, w, dst, e, last);
}
void llama_moe_cache::release_device() {
pimpl->release_device();
}
bool llama_moe_cache::restore_device() {
return pimpl->restore_device();
}
bool llama_moe_cache::resident() const {
return pimpl->resident();
}
std::map<ggml_backend_buffer_type_t, size_t> llama_moe_cache::memory_breakdown() const {
std::map<ggml_backend_buffer_type_t, size_t> res;
for (const auto & d : pimpl->devices) {
+6
View File
@@ -32,6 +32,12 @@ public:
// returns the number of experts copied, 0 if expert e is not in the cache
int64_t copy_experts(ggml_backend_t backend, const ggml_tensor * w, ggml_tensor * dst, int64_t e, int64_t last);
// on-demand VRAM: free the device banks and empty the cache, the caller must synchronize first
void release_device();
// reallocate the banks empty, returns false if out of memory
bool restore_device();
bool resident() const;
std::map<ggml_backend_buffer_type_t, size_t> memory_breakdown() const;
private: