diff --git a/src/llama-context.cpp b/src/llama-context.cpp index ccc5ed431..cb9f321a7 100644 --- a/src/llama-context.cpp +++ b/src/llama-context.cpp @@ -919,13 +919,21 @@ void llama_context::synchronize() { t_compute_start_us = 0; } +bool llama_context::device_resident() const { + return model.weights_resident() && (!moe_cache || moe_cache->resident()); +} + void llama_context::release_device(bool evict_kv) { - if (!model.weights_resident() && !(evict_kv && !kv_device_evicted)) { + if (!device_resident() && !(evict_kv && !kv_device_evicted)) { return; } // ensure no compute is in flight before freeing the device buffers synchronize(); model.release_device_weights(); + // the banks are emptied, not shadowed: restored slot maps must not point at zeroed experts + if (moe_cache) { + moe_cache->release_device(); + } if (evict_kv && memory && !kv_device_evicted) { memory->release_device_buffers(); kv_device_evicted = true; @@ -942,10 +950,13 @@ void llama_context::release_device(bool evict_kv) { } void llama_context::restore_device() { - if (model.weights_resident() && !kv_device_evicted && sched) { + if (device_resident() && !kv_device_evicted && sched) { return; } model.restore_device_weights(); + if (moe_cache) { + moe_cache->restore_device(); + } if (kv_device_evicted && memory) { memory->restore_device_buffers(); kv_device_evicted = false; @@ -1895,7 +1906,7 @@ int llama_context::decode(const llama_batch_ext & batch_inp) { // on-demand VRAM: if the weights were released while this model was idle, bring them back // to the device before building the compute graph - if (!model.weights_resident()) { + if (!device_resident()) { restore_device(); } @@ -4306,7 +4317,7 @@ void llama_context_restore_device(llama_context * ctx) { } bool llama_context_weights_resident(const llama_context * ctx) { - return ctx->get_model().weights_resident(); + return ctx->device_resident(); } float * llama_get_logits(llama_context * ctx) { diff --git a/src/llama-context.h b/src/llama-context.h index 401dd53ec..c3a437415 100644 --- a/src/llama-context.h +++ b/src/llama-context.h @@ -68,6 +68,9 @@ struct llama_context { void release_device(bool evict_kv = false); void restore_device(); + // the model weights and the MoE cache are on the device + bool device_resident() const; + const llama_model & get_model() const; const llama_cparams & get_cparams() const; diff --git a/src/llama-moe-cache.cpp b/src/llama-moe-cache.cpp index 4d3596b24..5f3a7dec3 100644 --- a/src/llama-moe-cache.cpp +++ b/src/llama-moe-cache.cpp @@ -230,6 +230,7 @@ struct llama_moe_cache::impl { static constexpr int64_t max_batch = 32; int32_t n_expert_used; + bool no_alloc; stats stats_small; // up to 8 tokens per ubatch stats stats_large; @@ -253,7 +254,7 @@ struct llama_moe_cache::impl { ggml_context_ptr ctx_views; impl(const llama_model & model, const std::vector & backends, const std::vector & bufts, size_t size) : - n_expert_used(model.hparams.n_expert_used_max()), layers(model.layers.size()) { + n_expert_used(model.hparams.n_expert_used_max()), no_alloc(model.hparams.no_alloc), layers(model.layers.size()) { for (size_t i = 0; i < backends.size(); ++i) { const auto dev_type = ggml_backend_dev_type(ggml_backend_get_device(backends[i])); if (dev_type == GGML_BACKEND_DEVICE_TYPE_GPU || dev_type == GGML_BACKEND_DEVICE_TYPE_IGPU) { @@ -616,6 +617,85 @@ struct llama_moe_cache::impl { return true; } + // on-demand VRAM: free the banks and forget the cached experts, the slot maps all read -1 again + void release_device() { + if (no_alloc) { + return; + } + size_t freed = 0; + for (device & d : devices) { + if (!d.buf) { + continue; + } + freed += ggml_backend_buffer_get_size(d.buf.get()); + d.buf.reset(); + for (ggml_tensor * t = ggml_get_first_tensor(d.ctx.get()); t != nullptr; t = ggml_get_next_tensor(d.ctx.get(), t)) { + t->buffer = nullptr; + t->data = nullptr; + } + } + invalidate(); + if (freed > 0) { + LLAMA_LOG_INFO("%s: released %.2f MiB of MoE cache\n", __func__, freed/1024.0/1024.0); + } + } + + // the banks come back empty, so the slot maps stay invalidated + bool restore_device() { + if (no_alloc) { + return true; + } + size_t restored = 0; + for (device & d : devices) { + if (!d.ctx || d.buf) { + continue; + } + // also re-inits the views of the banks + d.buf.reset(ggml_backend_alloc_ctx_tensors_from_buft(d.ctx.get(), d.buft)); + if (!d.buf) { + LLAMA_LOG_ERROR("%s: failed to reallocate the MoE cache buffer (out of VRAM?)\n", __func__); + return false; + } + ggml_backend_buffer_clear(d.buf.get(), 0); + ggml_backend_buffer_set_usage(d.buf.get(), GGML_BACKEND_BUFFER_USAGE_WEIGHTS); + d.buf_size = ggml_backend_buffer_get_size(d.buf.get()); + restored += d.buf_size; + } + if (restored > 0) { + LLAMA_LOG_INFO("%s: restored %.2f MiB of MoE cache\n", __func__, restored/1024.0/1024.0); + } + return true; + } + + bool resident() const { + for (const device & d : devices) { + if (d.ctx && !d.buf) { + return false; + } + } + return true; + } + + // empty every SLRU and mark all experts as not cached + void invalidate() { + for (group & g : groups) { + if (g.n_slots == 0) { + continue; + } + // start from a fresh SLRU: init() does not reset the protected segment + g.slru = moe_cache_slru(); + g.slru.init((int32_t) layers.size(), (int32_t) g.ref[0]->ne[2], g.n_slots); + for (int32_t il : g.layers) { + if (layers[il].slot_map != nullptr) { + g.slru.slot_map[il] = (int32_t *) layers[il].slot_map->data; + } + } + } + if (buf_host) { + ggml_backend_buffer_clear(buf_host.get(), 0xff); // all slots are -1 + } + } + void log_stats() const { auto log = [](const char * name, const stats & st) { const size_t n = st.hits + st.misses; @@ -659,6 +739,18 @@ int64_t llama_moe_cache::copy_experts(ggml_backend_t backend, const ggml_tensor return pimpl->copy_experts(backend, w, dst, e, last); } +void llama_moe_cache::release_device() { + pimpl->release_device(); +} + +bool llama_moe_cache::restore_device() { + return pimpl->restore_device(); +} + +bool llama_moe_cache::resident() const { + return pimpl->resident(); +} + std::map llama_moe_cache::memory_breakdown() const { std::map res; for (const auto & d : pimpl->devices) { diff --git a/src/llama-moe-cache.h b/src/llama-moe-cache.h index 9baf06390..4c3b99c66 100644 --- a/src/llama-moe-cache.h +++ b/src/llama-moe-cache.h @@ -32,6 +32,12 @@ public: // returns the number of experts copied, 0 if expert e is not in the cache int64_t copy_experts(ggml_backend_t backend, const ggml_tensor * w, ggml_tensor * dst, int64_t e, int64_t last); + // on-demand VRAM: free the device banks and empty the cache, the caller must synchronize first + void release_device(); + // reallocate the banks empty, returns false if out of memory + bool restore_device(); + bool resident() const; + std::map memory_breakdown() const; private: