llama: release/restore the MoE cache with the device weights
The MoE cache (--moe-cache-mib, #29887) keeps the most recently used host experts in device banks. The on-demand VRAM path (release_device / restore_device, used by the server VRAM arbiter and the on-demand mmproj path) did not know about it, so its banks were never freed on handoff. Restoring them by reallocation alone would also be wrong: the banks come back zeroed while the slot maps still mark experts as cached, so the model would silently read zero weights. Free the banks on release and empty the cache instead of shadowing it: re-init every LRU, re-point its slot-map pointers at the host slot maps and clear those to -1. The banks are reallocated (which also re-inits their views) and cleared on restore, before the scheduler is rebuilt, and the cache refills from the host experts as before. Release runs in both modes, weights-only and evict_kv. device_resident() now covers the cache too, for the early returns in release/restore, the auto-restore in decode() and llama_context_weights_resident(). Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01PZz44SLQvTXMyWGio6t9DZ
This commit is contained in:
1 parent
85ebb674e0
commit
3d066892a0
4 files changed
+115
-5
No files matched your search
+15
-4
@@ -812,13 +812,21 @@ void llama_context::synchronize() {
|
||||
t_compute_start_us = 0;
|
||||
}
|
||||
|
||||
bool llama_context::device_resident() const {
|
||||
return model.weights_resident() && (!moe_cache || moe_cache->resident());
|
||||
}
|
||||
|
||||
void llama_context::release_device(bool evict_kv) {
|
||||
if (!model.weights_resident() && !(evict_kv && !kv_device_evicted)) {
|
||||
if (!device_resident() && !(evict_kv && !kv_device_evicted)) {
|
||||
return;
|
||||
}
|
||||
// ensure no compute is in flight before freeing the device buffers
|
||||
synchronize();
|
||||
model.release_device_weights();
|
||||
// the banks are emptied, not shadowed: restored slot maps must not point at zeroed experts
|
||||
if (moe_cache) {
|
||||
moe_cache->release_device();
|
||||
}
|
||||
if (evict_kv && memory && !kv_device_evicted) {
|
||||
memory->release_device_buffers();
|
||||
kv_device_evicted = true;
|
||||
@@ -835,10 +843,13 @@ void llama_context::release_device(bool evict_kv) {
|
||||
}
|
||||
|
||||
void llama_context::restore_device() {
|
||||
if (model.weights_resident() && !kv_device_evicted && sched) {
|
||||
if (device_resident() && !kv_device_evicted && sched) {
|
||||
return;
|
||||
}
|
||||
model.restore_device_weights();
|
||||
if (moe_cache) {
|
||||
moe_cache->restore_device();
|
||||
}
|
||||
if (kv_device_evicted && memory) {
|
||||
memory->restore_device_buffers();
|
||||
kv_device_evicted = false;
|
||||
@@ -1786,7 +1797,7 @@ int llama_context::decode(const llama_batch_ext & batch_inp) {
|
||||
|
||||
// on-demand VRAM: if the weights were released while this model was idle, bring them back
|
||||
// to the device before building the compute graph
|
||||
if (!model.weights_resident()) {
|
||||
if (!device_resident()) {
|
||||
restore_device();
|
||||
}
|
||||
|
||||
@@ -4178,7 +4189,7 @@ void llama_context_restore_device(llama_context * ctx) {
|
||||
}
|
||||
|
||||
bool llama_context_weights_resident(const llama_context * ctx) {
|
||||
return ctx->get_model().weights_resident();
|
||||
return ctx->device_resident();
|
||||
}
|
||||
|
||||
float * llama_get_logits(llama_context * ctx) {
|
||||
|
||||
@@ -68,6 +68,9 @@ struct llama_context {
|
||||
void release_device(bool evict_kv = false);
|
||||
void restore_device();
|
||||
|
||||
// the model weights and the MoE cache are on the device
|
||||
bool device_resident() const;
|
||||
|
||||
const llama_model & get_model() const;
|
||||
const llama_cparams & get_cparams() const;
|
||||
|
||||
|
||||
+91
-1
@@ -196,6 +196,7 @@ struct llama_moe_cache::impl {
|
||||
static constexpr int64_t max_batch = 32;
|
||||
|
||||
int32_t n_expert_used;
|
||||
bool no_alloc;
|
||||
|
||||
stats stats_small; // up to 8 tokens per ubatch
|
||||
stats stats_large;
|
||||
@@ -219,7 +220,7 @@ struct llama_moe_cache::impl {
|
||||
ggml_context_ptr ctx_views;
|
||||
|
||||
impl(const llama_model & model, const std::vector<ggml_backend_t> & backends, const std::vector<ggml_backend_buffer_type_t> & bufts, size_t size) :
|
||||
n_expert_used(model.hparams.n_expert_used_max()), layers(model.layers.size()) {
|
||||
n_expert_used(model.hparams.n_expert_used_max()), no_alloc(model.hparams.no_alloc), layers(model.layers.size()) {
|
||||
for (size_t i = 0; i < backends.size(); ++i) {
|
||||
const auto dev_type = ggml_backend_dev_type(ggml_backend_get_device(backends[i]));
|
||||
if (dev_type == GGML_BACKEND_DEVICE_TYPE_GPU || dev_type == GGML_BACKEND_DEVICE_TYPE_IGPU) {
|
||||
@@ -582,6 +583,83 @@ struct llama_moe_cache::impl {
|
||||
return true;
|
||||
}
|
||||
|
||||
// on-demand VRAM: free the banks and forget the cached experts, the slot maps all read -1 again
|
||||
void release_device() {
|
||||
if (no_alloc) {
|
||||
return;
|
||||
}
|
||||
size_t freed = 0;
|
||||
for (device & d : devices) {
|
||||
if (!d.buf) {
|
||||
continue;
|
||||
}
|
||||
freed += ggml_backend_buffer_get_size(d.buf.get());
|
||||
d.buf.reset();
|
||||
for (ggml_tensor * t = ggml_get_first_tensor(d.ctx.get()); t != nullptr; t = ggml_get_next_tensor(d.ctx.get(), t)) {
|
||||
t->buffer = nullptr;
|
||||
t->data = nullptr;
|
||||
}
|
||||
}
|
||||
invalidate();
|
||||
if (freed > 0) {
|
||||
LLAMA_LOG_INFO("%s: released %.2f MiB of MoE cache\n", __func__, freed/1024.0/1024.0);
|
||||
}
|
||||
}
|
||||
|
||||
// the banks come back empty, so the slot maps stay invalidated
|
||||
bool restore_device() {
|
||||
if (no_alloc) {
|
||||
return true;
|
||||
}
|
||||
size_t restored = 0;
|
||||
for (device & d : devices) {
|
||||
if (!d.ctx || d.buf) {
|
||||
continue;
|
||||
}
|
||||
// also re-inits the views of the banks
|
||||
d.buf.reset(ggml_backend_alloc_ctx_tensors_from_buft(d.ctx.get(), d.buft));
|
||||
if (!d.buf) {
|
||||
LLAMA_LOG_ERROR("%s: failed to reallocate the MoE cache buffer (out of VRAM?)\n", __func__);
|
||||
return false;
|
||||
}
|
||||
ggml_backend_buffer_clear(d.buf.get(), 0);
|
||||
ggml_backend_buffer_set_usage(d.buf.get(), GGML_BACKEND_BUFFER_USAGE_WEIGHTS);
|
||||
d.buf_size = ggml_backend_buffer_get_size(d.buf.get());
|
||||
restored += d.buf_size;
|
||||
}
|
||||
if (restored > 0) {
|
||||
LLAMA_LOG_INFO("%s: restored %.2f MiB of MoE cache\n", __func__, restored/1024.0/1024.0);
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
bool resident() const {
|
||||
for (const device & d : devices) {
|
||||
if (d.ctx && !d.buf) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
// empty every LRU and mark all experts as not cached
|
||||
void invalidate() {
|
||||
for (group & g : groups) {
|
||||
if (g.n_slots == 0) {
|
||||
continue;
|
||||
}
|
||||
g.lru.init((int32_t) layers.size(), (int32_t) g.ref[0]->ne[2], g.n_slots);
|
||||
for (int32_t il : g.layers) {
|
||||
if (layers[il].slot_map != nullptr) {
|
||||
g.lru.slot_map[il] = (int32_t *) layers[il].slot_map->data;
|
||||
}
|
||||
}
|
||||
}
|
||||
if (buf_host) {
|
||||
ggml_backend_buffer_clear(buf_host.get(), 0xff); // all slots are -1
|
||||
}
|
||||
}
|
||||
|
||||
void log_stats() const {
|
||||
auto log = [](const char * name, const stats & st) {
|
||||
const size_t n = st.hits + st.misses;
|
||||
@@ -625,6 +703,18 @@ int64_t llama_moe_cache::copy_experts(ggml_backend_t backend, const ggml_tensor
|
||||
return pimpl->copy_experts(backend, w, dst, e, last);
|
||||
}
|
||||
|
||||
void llama_moe_cache::release_device() {
|
||||
pimpl->release_device();
|
||||
}
|
||||
|
||||
bool llama_moe_cache::restore_device() {
|
||||
return pimpl->restore_device();
|
||||
}
|
||||
|
||||
bool llama_moe_cache::resident() const {
|
||||
return pimpl->resident();
|
||||
}
|
||||
|
||||
std::map<ggml_backend_buffer_type_t, size_t> llama_moe_cache::memory_breakdown() const {
|
||||
std::map<ggml_backend_buffer_type_t, size_t> res;
|
||||
for (const auto & d : pimpl->devices) {
|
||||
|
||||
@@ -32,6 +32,12 @@ public:
|
||||
// returns the number of experts copied, 0 if expert e is not in the cache
|
||||
int64_t copy_experts(ggml_backend_t backend, const ggml_tensor * w, ggml_tensor * dst, int64_t e, int64_t last);
|
||||
|
||||
// on-demand VRAM: free the device banks and empty the cache, the caller must synchronize first
|
||||
void release_device();
|
||||
// reallocate the banks empty, returns false if out of memory
|
||||
bool restore_device();
|
||||
bool resident() const;
|
||||
|
||||
std::map<ggml_backend_buffer_type_t, size_t> memory_breakdown() const;
|
||||
|
||||
private:
|
||||
|
||||
Reference in new issue
Block a user