diff --git a/include/llama.h b/include/llama.h index 894b707b4..a99addbbe 100644 --- a/include/llama.h +++ b/include/llama.h @@ -600,6 +600,9 @@ extern "C" { // Intended for time-sharing a single GPU between several always-loaded models. LLAMA_API void llama_context_release_device(struct llama_context * ctx, bool evict_kv); LLAMA_API void llama_context_restore_device(struct llama_context * ctx); + // Free all device buffers (weights, KV, compute) for good, without host copies, e.g. to hand + // the VRAM to another process before exit. The context must not be used for compute afterwards. + LLAMA_API void llama_context_discard_device(struct llama_context * ctx); LLAMA_API bool llama_context_weights_resident(const struct llama_context * ctx); LLAMA_API enum llama_pooling_type llama_pooling_type(const struct llama_context * ctx); // TODO: rename to llama_get_pooling_type diff --git a/src/llama-context.cpp b/src/llama-context.cpp index 60cb7872e..209ee666b 100644 --- a/src/llama-context.cpp +++ b/src/llama-context.cpp @@ -842,7 +842,27 @@ void llama_context::release_device(bool evict_kv) { } } +void llama_context::discard_device() { + // same as release_device(true), minus the host copies of the weights and the KV + synchronize(); + model.release_device_weights(/*keep_shadow =*/ false); + if (moe_cache) { + moe_cache->release_device(); + } + if (memory) { + memory->release_device_buffers(/*keep_shadow =*/ false); + } + kv_device_evicted = true; + device_discarded = true; + sched.reset(); + sched_need_reserve = true; + for (auto & backend : backends) { + ggml_backend_free_scratch(backend.get()); + } +} + void llama_context::restore_device() { + GGML_ASSERT(!device_discarded && "device buffers were discarded"); if (device_resident() && !kv_device_evicted && sched) { return; } @@ -4188,6 +4208,10 @@ void llama_context_restore_device(llama_context * ctx) { ctx->restore_device(); } +void llama_context_discard_device(llama_context * ctx) { + ctx->discard_device(); +} + bool llama_context_weights_resident(const llama_context * ctx) { return ctx->device_resident(); } diff --git a/src/llama-context.h b/src/llama-context.h index a3f7efc18..f1b04b3e2 100644 --- a/src/llama-context.h +++ b/src/llama-context.h @@ -67,6 +67,9 @@ struct llama_context { // other model) - this adds a D2H/H2D copy of the live KV on each cycle. void release_device(bool evict_kv = false); void restore_device(); + // free every device buffer for good, without host copies (before exit). The context must not + // compute again: restore_device() aborts. + void discard_device(); // the model weights and the MoE cache are on the device bool device_resident() const; @@ -390,6 +393,8 @@ private: // on-demand device residency: true while the KV cache device buffers have been evicted to host bool kv_device_evicted = false; + // set by discard_device(): the device buffers are gone and cannot be restored + bool device_discarded = false; ggml_backend_t backend_cpu = nullptr; std::vector backends; diff --git a/src/llama-kv-cache-iswa.cpp b/src/llama-kv-cache-iswa.cpp index 56fdfe5c1..7a1588d5c 100644 --- a/src/llama-kv-cache-iswa.cpp +++ b/src/llama-kv-cache-iswa.cpp @@ -272,9 +272,9 @@ void llama_kv_cache_iswa::state_read(llama_io_read_i & io, llama_seq_id seq_id, kv_swa->state_read(io, seq_id, flags); } -void llama_kv_cache_iswa::release_device_buffers() { - kv_base->release_device_buffers(); - kv_swa->release_device_buffers(); +void llama_kv_cache_iswa::release_device_buffers(bool keep_shadow) { + kv_base->release_device_buffers(keep_shadow); + kv_swa->release_device_buffers(keep_shadow); } bool llama_kv_cache_iswa::restore_device_buffers() { diff --git a/src/llama-kv-cache-iswa.h b/src/llama-kv-cache-iswa.h index 653bcc4e5..68c2c8969 100644 --- a/src/llama-kv-cache-iswa.h +++ b/src/llama-kv-cache-iswa.h @@ -83,7 +83,7 @@ public: void state_write(llama_io_write_i & io, llama_seq_id seq_id = -1, llama_state_seq_flags flags = 0) const override; void state_read (llama_io_read_i & io, llama_seq_id seq_id = -1, llama_state_seq_flags flags = 0) override; - void release_device_buffers() override; + void release_device_buffers(bool keep_shadow = true) override; bool restore_device_buffers() override; // diff --git a/src/llama-kv-cache.cpp b/src/llama-kv-cache.cpp index c1c8041a8..f21ddd810 100644 --- a/src/llama-kv-cache.cpp +++ b/src/llama-kv-cache.cpp @@ -1924,7 +1924,7 @@ size_t llama_kv_cache::total_size() const { return size; } -void llama_kv_cache::release_device_buffers() { +void llama_kv_cache::release_device_buffers(bool keep_shadow) { // NOTE: the caller must have synchronized the backend so nothing references these buffers. // The KV cache is read-write, so its host shadow is (re)captured fresh on every release. if (dev_released) { @@ -1943,17 +1943,19 @@ void llama_kv_cache::release_device_buffers() { sh.buft = ggml_backend_buffer_get_type(buf); // capture live contents compactly in stable iteration order (skip views, which alias a base) - size_t total = 0; - for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != nullptr; t = ggml_get_next_tensor(ctx, t)) { - if (t->view_src == nullptr) { total += ggml_nbytes(t); } - } - sh.data.resize(total); - size_t off = 0; - for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != nullptr; t = ggml_get_next_tensor(ctx, t)) { - if (t->view_src != nullptr) { continue; } - const size_t n = ggml_nbytes(t); - ggml_backend_tensor_get(t, sh.data.data() + off, 0, n); - off += n; + if (keep_shadow) { + size_t total = 0; + for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != nullptr; t = ggml_get_next_tensor(ctx, t)) { + if (t->view_src == nullptr) { total += ggml_nbytes(t); } + } + sh.data.resize(total); + size_t off = 0; + for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != nullptr; t = ggml_get_next_tensor(ctx, t)) { + if (t->view_src != nullptr) { continue; } + const size_t n = ggml_nbytes(t); + ggml_backend_tensor_get(t, sh.data.data() + off, 0, n); + off += n; + } } freed += ggml_backend_buffer_get_size(buf); diff --git a/src/llama-kv-cache.h b/src/llama-kv-cache.h index a933a149e..46e5f345f 100644 --- a/src/llama-kv-cache.h +++ b/src/llama-kv-cache.h @@ -152,7 +152,7 @@ public: void state_read (llama_io_read_i & io, llama_seq_id seq_id = -1, llama_state_seq_flags flags = 0) override; // on-demand device (VRAM) residency (see llama_memory_i) - void release_device_buffers() override; + void release_device_buffers(bool keep_shadow = true) override; bool restore_device_buffers() override; // diff --git a/src/llama-memory-hybrid.cpp b/src/llama-memory-hybrid.cpp index 8f107e471..c8e0bdc47 100644 --- a/src/llama-memory-hybrid.cpp +++ b/src/llama-memory-hybrid.cpp @@ -213,10 +213,10 @@ void llama_memory_hybrid::state_read(llama_io_read_i & io, llama_seq_id seq_id, } } -void llama_memory_hybrid::release_device_buffers() { +void llama_memory_hybrid::release_device_buffers(bool keep_shadow) { // evict both the attention KV (grows with context) and the recurrent/SSM state - mem_attn->release_device_buffers(); - mem_recr->release_device_buffers(); + mem_attn->release_device_buffers(keep_shadow); + mem_recr->release_device_buffers(keep_shadow); } bool llama_memory_hybrid::restore_device_buffers() { diff --git a/src/llama-memory-hybrid.h b/src/llama-memory-hybrid.h index 318076def..91a6fe196 100644 --- a/src/llama-memory-hybrid.h +++ b/src/llama-memory-hybrid.h @@ -76,7 +76,7 @@ public: void state_write(llama_io_write_i & io, llama_seq_id seq_id = -1, llama_state_seq_flags flags = 0) const override; void state_read (llama_io_read_i & io, llama_seq_id seq_id = -1, llama_state_seq_flags flags = 0) override; - void release_device_buffers() override; + void release_device_buffers(bool keep_shadow = true) override; bool restore_device_buffers() override; // diff --git a/src/llama-memory-recurrent.cpp b/src/llama-memory-recurrent.cpp index 004ec38e4..92cbde45f 100644 --- a/src/llama-memory-recurrent.cpp +++ b/src/llama-memory-recurrent.cpp @@ -754,7 +754,7 @@ size_t llama_memory_recurrent::total_size() const { return size; } -void llama_memory_recurrent::release_device_buffers() { +void llama_memory_recurrent::release_device_buffers(bool keep_shadow) { // Same mechanism as llama_kv_cache: the recurrent (SSM/conv) state is read-write, so its host // shadow is (re)captured on every release. The caller must have synchronized the backend. if (dev_released) { @@ -772,17 +772,19 @@ void llama_memory_recurrent::release_device_buffers() { sh.releasable = true; sh.buft = ggml_backend_buffer_get_type(buf); - size_t total = 0; - for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != nullptr; t = ggml_get_next_tensor(ctx, t)) { - if (t->view_src == nullptr) { total += ggml_nbytes(t); } - } - sh.data.resize(total); - size_t off = 0; - for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != nullptr; t = ggml_get_next_tensor(ctx, t)) { - if (t->view_src != nullptr) { continue; } - const size_t n = ggml_nbytes(t); - ggml_backend_tensor_get(t, sh.data.data() + off, 0, n); - off += n; + if (keep_shadow) { + size_t total = 0; + for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != nullptr; t = ggml_get_next_tensor(ctx, t)) { + if (t->view_src == nullptr) { total += ggml_nbytes(t); } + } + sh.data.resize(total); + size_t off = 0; + for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != nullptr; t = ggml_get_next_tensor(ctx, t)) { + if (t->view_src != nullptr) { continue; } + const size_t n = ggml_nbytes(t); + ggml_backend_tensor_get(t, sh.data.data() + off, 0, n); + off += n; + } } freed += ggml_backend_buffer_get_size(buf); diff --git a/src/llama-memory-recurrent.h b/src/llama-memory-recurrent.h index 3043a5bac..3dfd209bd 100644 --- a/src/llama-memory-recurrent.h +++ b/src/llama-memory-recurrent.h @@ -67,7 +67,7 @@ public: void state_read (llama_io_read_i & io, llama_seq_id seq_id = -1, llama_state_seq_flags flags = 0) override; // on-demand device (VRAM) residency (see llama_memory_i) - void release_device_buffers() override; + void release_device_buffers(bool keep_shadow = true) override; bool restore_device_buffers() override; uint32_t head = 0; // the location where the batch will be placed in the cache (see find_slot()) diff --git a/src/llama-memory.h b/src/llama-memory.h index e6f3cc0ba..6a390401e 100644 --- a/src/llama-memory.h +++ b/src/llama-memory.h @@ -132,7 +132,8 @@ struct llama_memory_i { // Free this memory's device (VRAM) buffers to a host shadow and rebuild them on demand, to // hand VRAM to another model while keeping the cached data (no re-prefill). The caller must // have synchronized the backend first. Default: no-op (the memory stays resident). - virtual void release_device_buffers() {} + // keep_shadow = false frees the buffers for good (before exit): no host copy, no restore. + virtual void release_device_buffers(bool /*keep_shadow*/ = true) {} virtual bool restore_device_buffers() { return true; } }; diff --git a/src/llama-model.cpp b/src/llama-model.cpp index 56bd27aab..c95ff5499 100644 --- a/src/llama-model.cpp +++ b/src/llama-model.cpp @@ -2024,7 +2024,7 @@ ggml_tensor * llama_model_base::create_tensor(llama_model_loader & ml, const LLM tn, ne, flags); } -void llama_model::release_device_weights() const { +void llama_model::release_device_weights(bool keep_shadow) const { // NOTE: the caller is responsible for synchronizing the backend scheduler first, so that no // compute is in flight referencing these buffers when they are freed. size_t freed = 0; @@ -2041,7 +2041,7 @@ void llama_model::release_device_weights() const { // Store tensor bytes compactly in stable iteration order (independent of buffer layout / // alignment padding) so restore does not depend on the re-allocated offsets matching. // View tensors alias their base, so they are skipped (restored implicitly via their base). - if (slot.shadow.empty()) { + if (keep_shadow && slot.shadow.empty()) { size_t total = 0; for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != nullptr; t = ggml_get_next_tensor(ctx, t)) { if (t->view_src != nullptr) continue; diff --git a/src/llama-model.h b/src/llama-model.h index cb76e73cc..2ef0a088d 100644 --- a/src/llama-model.h +++ b/src/llama-model.h @@ -791,7 +791,7 @@ struct llama_model { // shadow) and rebuild them on demand. Used to time-share VRAM between models without // reloading or losing the KV cache. release_device_weights() requires the caller to have // synchronized the backend scheduler first. - void release_device_weights() const; + void release_device_weights(bool keep_shadow = true) const; bool restore_device_weights() const; bool weights_resident() const; diff --git a/tools/server/server-context.cpp b/tools/server/server-context.cpp index 68ed14a16..7f4a99569 100644 --- a/tools/server/server-context.cpp +++ b/tools/server/server-context.cpp @@ -999,13 +999,13 @@ public: } ~server_context_impl() { - // release the VRAM token before tearing anything down - vram_share_shutdown(); if (!sleeping) { // destroy() is already called when entering sleeping state // we don't call it again here to avoid double free destroy(); } + // only once our VRAM is freed (normally vram_share_exit() has done both already) + vram_share_shutdown(); } server_metrics get_metrics() const { @@ -1168,7 +1168,8 @@ private: // take the shared VRAM token: register on want.lock, then wait for token.lock. want.lock is // level-triggered state, so the holder still learns of us if it is loading or restoring when we // start to wait. No timeout: the holder can be mid-generation, and decoding without the token - // risks an OOM that kills both processes. + // risks an OOM that kills both processes. Returns false without the token if the server shuts + // down meanwhile (the caller checks queue_tasks.is_stopping()). bool vram_take_token() { #if !defined(_WIN32) if (!vram_flock) { @@ -1182,8 +1183,15 @@ private: SRV_INF("%s", "VRAM arbiter: waiting for the GPU token\n"); logged = true; } - flock(vram_lock_fd, LOCK_EX); // blocking: the kernel grants it the moment the holder unlocks - continue; // re-check with LOCK_NB, which also succeeds on a lock we already own + // poll, not a blocking flock: terminate() must be able to stop this wait, and it can + // only signal us through an atomic (we hold mutex_tasks here) + if (queue_tasks.is_stopping()) { + SRV_INF("%s", "VRAM arbiter: shutting down, no longer waiting for the GPU token\n"); + vram_want_release(); + return false; + } + std::this_thread::sleep_for(std::chrono::milliseconds(2)); + continue; } if (errno == EINTR) { continue; // our signal handlers run without SA_RESTART @@ -1200,12 +1208,19 @@ private: } // acquire the VRAM token (blocking) and bring weights back to the device. Called on the loop - // thread right before a decode, so it never races compute. - void vram_ensure_warm() { + // thread right before a decode, so it never races compute. Returns false, still cold, if the + // server shuts down first: the caller must not touch the device then. + bool vram_ensure_warm() { if (!vram_only || !vram_cold) { - return; + return true; + } + if (queue_tasks.is_stopping()) { + return false; // no restore just to exit } if (!vram_take_token()) { + if (queue_tasks.is_stopping()) { + return false; + } SRV_WRN("%s", "VRAM arbiter: restoring weights without the GPU token\n"); } llama_context_restore_device(ctx_tgt); @@ -1221,6 +1236,25 @@ private: // bring the weights up. We dropped want.lock when we took the token, so the next probe // reports only the waiters that are still there. vram_cold = false; + return true; + } + + // the loop has stopped: free our VRAM and hand the token over now, without waiting for the HTTP + // threads and the teardown. Discarded, not released: no host copy, and the context never decodes + // again. Runs on the loop thread after the loop returned, so nothing else uses the device. + void vram_share_exit() { + if (vram_only && !vram_cold && ctx_tgt != nullptr) { + llama_context_discard_device(ctx_tgt); + if (ctx_dft != nullptr) { + llama_context_discard_device(ctx_dft); + } + if (mctx != nullptr) { + mtmd_release_device(mctx); + } + vram_cold = true; + SRV_INF("%s", "VRAM arbiter: loop stopped, device memory freed, releasing the GPU token\n"); + } + vram_share_shutdown(); } void vram_share_shutdown() { @@ -1337,6 +1371,7 @@ private: // (e.g. SWA checkpoint creation reads it via ggml_backend_tensor_get) before the // decode-time vram_ensure_warm(), so with KV eviction the KV must already be resident // here or those reads hit a freed (null) buffer. + // false (still cold) only on shutdown: the loop then exits before update_slots vram_ensure_warm(); } sleeping = new_state; @@ -4469,7 +4504,9 @@ private: // VRAM arbiter: make sure our weights are resident (acquiring the shared GPU token first) // before we decode. Runs on the loop thread before entering yield_to_queue(), so it never // races an in-flight decode. - vram_ensure_warm(); + if (!vram_ensure_warm()) { + throw std::runtime_error("server is shutting down"); + } // yield to the queue, so we can still handle metrics tasks while decoding // note: the sync is done here too, so that the wait is also covered by the yield @@ -4982,9 +5019,12 @@ bool server_context::load_model(common_params & params) { void server_context::start_loop() { auto & params = impl->params_base; impl->queue_tasks.start_loop(params.sleep_idle_seconds * 1000); + impl->vram_share_exit(); } void server_context::terminate() { + // wake the HTTP threads waiting for results first: the loop will not produce them any more + impl->queue_results.terminate(); impl->queue_tasks.terminate(); } diff --git a/tools/server/server-queue.cpp b/tools/server/server-queue.cpp index b7ce86433..afdd1ed4c 100644 --- a/tools/server/server-queue.cpp +++ b/tools/server/server-queue.cpp @@ -116,7 +116,8 @@ void server_queue::wait_until_no_sleep() { std::unique_lock lock(mutex_tasks); // re-ask on every wake: the loop clears req_stop_sleeping on the way into sleep, so a loop that // goes back to sleep before we run would strand us here forever - while (sleeping) { + // a loop that terminates while asleep never wakes: give up then, the reader reports the shutdown + while (sleeping && !stopping) { if (!req_stop_sleeping) { QUE_DBG("%s", "requesting to stop sleeping\n"); req_stop_sleeping = true; @@ -128,6 +129,7 @@ void server_queue::wait_until_no_sleep() { } void server_queue::terminate() { + stopping = true; // before the lock: the loop may hold it while it waits for the VRAM token std::unique_lock lock(mutex_tasks); running = false; condition_tasks.notify_all(); @@ -136,7 +138,7 @@ void server_queue::terminate() { bool server_queue::process_new_tasks(bool is_yielding) { while (true) { std::unique_lock lock(mutex_tasks); - if (!running) { + if (!running || stopping) { QUE_DBG("%s", "terminate\n"); return true; } @@ -472,11 +474,12 @@ server_task_result_ptr server_response::recv_with_timeout(const std::unordered_s } } - std::cv_status cr_res = condition_results.wait_for(lock, std::chrono::seconds(timeout)); if (!running) { RES_DBG("%s : queue result stop\n", __func__); - std::terminate(); // we cannot return here since the caller is HTTP code + return nullptr; // the caller checks is_running() } + + std::cv_status cr_res = condition_results.wait_for(lock, std::chrono::seconds(timeout)); if (cr_res == std::cv_status::timeout) { return nullptr; } @@ -517,6 +520,7 @@ void server_response::broadcast(server_task_result_ptr && result) { } void server_response::terminate() { + std::unique_lock lock(mutex_results); running = false; condition_results.notify_all(); } @@ -564,6 +568,14 @@ server_task_result_ptr server_response_reader::next(const std::function while (true) { server_task_result_ptr result = queue_results.recv_with_timeout(id_tasks, polling_interval_seconds); if (result == nullptr) { + if (!queue_results.is_running()) { + // the server is shutting down: no result will ever come + stop(); + auto res = std::make_unique(); + res->err_type = ERROR_TYPE_UNAVAILABLE; + res->err_msg = "server is shutting down"; + return res; + } // timeout, check stop condition if (should_stop()) { return nullptr; diff --git a/tools/server/server-queue.h b/tools/server/server-queue.h index 7939cb299..51094fb0b 100644 --- a/tools/server/server-queue.h +++ b/tools/server/server-queue.h @@ -2,6 +2,7 @@ #include "server-task.h" +#include #include #include #include @@ -16,6 +17,9 @@ struct server_queue { private: int id = 0; bool running = false; + // set by terminate() before it takes mutex_tasks, so a loop blocked under that mutex (e.g. in a + // VRAM arbiter token wait) can see the shutdown and bail out + std::atomic stopping{false}; bool sleeping = false; bool req_stop_sleeping = false; int64_t time_last_task = 0; @@ -78,6 +82,11 @@ public: // end the start_loop routine void terminate(); + // true once terminate() was called; safe from any thread, including under mutex_tasks + bool is_stopping() const { + return stopping.load(); + } + /** * Main loop consists of these steps: * - Wait until a new task arrives @@ -202,8 +211,13 @@ public: // (used by router mode) void broadcast(server_task_result_ptr && result); - // terminate the waiting loop + // terminate the waiting loop: pending and later readers get an error instead of a result void terminate(); + + bool is_running() { + std::unique_lock lock(mutex_results); + return running; + } }; // RAII wrapper to make working with server_queue and server_response easier