From f5c0d8f28a0643a5e2c9238668d9947b6ea64388 Mon Sep 17 00:00:00 2001 From: Lumpiasty Date: Sun, 25 Oct 2026 00:30:20 +0200 Subject: [PATCH] server: fix stale VRAM-arbiter comments and the sleep log line Comments only, plus one log string: - the arbiter block comment said the KV cache is never evicted; with LLAMA_SLEEP_EVICT_KV it is (and the scheduler and scratch with it) - vram_go_cold said vram_ensure_warm() rebuilds the encoder; only encode_mtmd_chunk() restores it, and only for an encode - vram_ensure_warm and the mtmd init block named process_mtmd_chunk; the restore lives in encode_mtmd_chunk - handle_sleeping_state said waking happens lazily before decode; it restores eagerly on the wake transition, before update_slots() - the vram_cold ordering note now says why it still holds with the inline want-lock probe - the sleep log line said "keeping KV cache" even when LLAMA_SLEEP_EVICT_KV evicts it; it now names what actually happens (the prefix "server entering sleeping state" that fork-validate.sh counts is kept) Co-Authored-By: Claude Opus 5.5 --- tools/server/server-context.cpp | 54 +++++++++++++++++++++------------ 1 file changed, 34 insertions(+), 20 deletions(-) diff --git a/tools/server/server-context.cpp b/tools/server/server-context.cpp index 12f103bae..5eed6ffcd 100644 --- a/tools/server/server-context.cpp +++ b/tools/server/server-context.cpp @@ -1115,9 +1115,12 @@ private: // Cross-process VRAM arbiter: when LLAMA_SLEEP_VRAM_ONLY is set, several always-loaded // llama-server processes time-share one GPU. A single flock() on /token.lock is the - // baton: "resident (weights in VRAM) iff I hold the token". A model warms up (locks + restores - // weights) right before it decodes, and goes cold (releases weights + unlocks) when it idles, - // so only one model holds VRAM at a time and the KV cache is never evicted (no re-prefill). + // baton: "resident (device buffers in VRAM) iff I hold the token". A model warms up (locks + + // restores) when it wakes for a request, and goes cold (releases + unlocks) when it is idle and + // another process waits for the token, so only one model holds VRAM at a time. The process, its + // context and its prompt cache stay alive, so nothing is re-prefilled. Going cold releases the + // weights; with LLAMA_SLEEP_EVICT_KV it also evicts the KV/recurrent state to a host shadow and + // frees the scheduler and backend scratch. All of it is restored bit-exact on wake. bool vram_only = false; // release-mode sleep active (LLAMA_SLEEP_VRAM_ONLY) bool vram_evict_kv = false; // also evict the KV cache to host on release (LLAMA_SLEEP_EVICT_KV) bool vram_flock = false; // cross-process flock coordination available @@ -1288,9 +1291,11 @@ private: #endif } - // acquire the VRAM token (blocking) and bring weights back to the device. Called on the loop - // thread right before a decode, so it never races compute. Returns false, still cold, if the - // server shuts down first: the caller must not touch the device then. + // acquire the VRAM token (blocking) and bring the device buffers back (weights; with + // LLAMA_SLEEP_EVICT_KV also the KV/recurrent state and the scheduler). Called on the loop thread: + // on wake from handle_sleeping_state(false), before update_slots() touches the KV, and again + // right before each decode (a no-op when already warm). It never races compute. Returns false, + // still cold, if the server shuts down first: the caller must not touch the device then. bool vram_ensure_warm() { if (!vram_only || !vram_cold) { return true; @@ -1309,13 +1314,16 @@ private: llama_context_restore_device(ctx_dft); } // NOTE: the multimodal (vision/audio) encoder is intentionally NOT restored here. It is - // brought into VRAM just-in-time before an image/audio encode (process_mtmd_chunk) and, in + // brought into VRAM just-in-time before an image/audio encode (encode_mtmd_chunk) and, in // on-demand mode, released again right after - so a warm text-only model holds no encoder // VRAM (that space is free for KV / expert cache). A cold->warm wake for a *text* request // therefore leaves the encoder in RAM; an image request restores it at encode time. - // NOTE: keep this AFTER the restore, so the loop cannot give away the token while we still - // bring the weights up. We dropped want.lock when we took the token, so the next probe - // reports only the waiters that are still there. + // NOTE: keep this AFTER the restore: vram_cold == false means "holding the token AND + // resident". Today the only reader that could give the token away, vram_should_yield(), runs + // on this same loop thread and so cannot run before we return; the order is what kept the + // retired cross-thread doorbell design from yielding mid-restore (FORK.md F6, attempt 1). + // Do not move it without an A/B of ordinary handoff latency. We dropped want.lock when we + // took the token, so the next probe reports only the waiters that are still there. vram_cold = false; return true; } @@ -1347,8 +1355,10 @@ private: #endif } - // release the device weights (keeping KV + host shadow) and drop the VRAM token so another - // model can warm up. Called on the loop thread when the server goes idle. + // release the device weights (their host shadow is kept; with LLAMA_SLEEP_EVICT_KV also the + // KV/recurrent state, scheduler and backend scratch) and drop the VRAM token so another model + // can warm up. Called on the loop thread from handle_sleeping_state(true): the queue is idle and + // another process waits for the token. void vram_go_cold() { if (!vram_only || vram_cold) { return; @@ -1357,8 +1367,10 @@ private: if (ctx_dft != nullptr) { llama_context_release_device(ctx_dft, vram_evict_kv); } - // also release the multimodal (vision/audio) encoder weights (dead weight while cold); - // its read-only host shadow is captured once and rebuilt on wake by vram_ensure_warm() + // also release the multimodal (vision/audio) encoder weights (dead weight while cold; a + // no-op when LLAMA_MMPROJ_ONDEMAND already keeps them out). Their read-only host shadow is + // captured once. vram_ensure_warm() does not bring them back: encode_mtmd_chunk() restores + // the encoder only for the duration of an encode. if (mctx != nullptr) { mtmd_release_device(mctx); } @@ -1437,14 +1449,16 @@ private: GGML_ASSERT(sleeping != new_state); // Lightweight VRAM-only sleep: instead of a full unload/reload (which frees RAM and the - // KV cache and pays a full reload on wake), just release the model's device (VRAM) weight - // buffers to a host shadow (keeping the context, KV cache and host weights) and drop the - // shared VRAM token. Waking is handled lazily by vram_ensure_warm() right before the next - // decode (which re-acquires the token first), so a single GPU is time-shared between models + // KV cache and pays a full reload on wake), just release the model's device (VRAM) buffers + // to a host shadow (keeping the context, the prompt cache and the host weights; the KV stays + // on the device unless LLAMA_SLEEP_EVICT_KV is set) and drop the shared VRAM token. Waking + // restores eagerly, on the wake transition below: vram_ensure_warm() re-acquires the token + // and restores before update_slots() runs. A single GPU is so time-shared between models // with no re-prefill. if (vram_only && ctx_tgt != nullptr) { if (new_state) { - SRV_INF("%s", "server entering sleeping state (VRAM-only: releasing device weights, keeping KV cache)\n"); + SRV_INF("server entering sleeping state (VRAM-only: releasing device weights, %s)\n", + vram_evict_kv ? "evicting KV cache to host" : "keeping KV cache in VRAM"); vram_go_cold(); } else { SRV_INF("%s", "server exiting sleeping state (VRAM-only: restoring device weights/KV)\n"); @@ -1717,7 +1731,7 @@ private: // on-demand encoder: keep the vision/audio encoder weights in RAM (out of VRAM) until an // image/audio actually needs encoding, freeing that VRAM for KV / expert cache on the - // common text-only path. process_mtmd_chunk() brings the encoder in just-in-time. + // common text-only path. encode_mtmd_chunk() brings the encoder in just-in-time. if (getenv("LLAMA_MMPROJ_ONDEMAND") != nullptr) { mtmd_release_device(mctx); }