From fd7cbd5f4fb9704ea9cefb6cf89e8b42544c4f4e Mon Sep 17 00:00:00 2001 From: Lumpiasty Date: Sun, 26 Jul 2026 00:32:33 +0200 Subject: [PATCH] server: also free the compute-graph scheduler on KV eviction Under LLAMA_SLEEP_EVICT_KV a cold model still held the scheduler's worst-case compute buffer (hundreds of MiB, e.g. ~700 MiB for gemma-26B) plus the resident experts. release_device(evict_kv) now also frees the sched (sched.reset() + sched_need_reserve), and restore_device() rebuilds it via sched_reserve(), so a fully-evicted model holds essentially no VRAM (gemma cold: 7147 -> 51 MiB). Trades a heavier re-warm (weights+KV H2D + sched reserve) for the freed VRAM; weights-only mode is unchanged (fast switch, keeps KV+compute). Assisted-by: Claude --- src/llama-context.cpp | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/llama-context.cpp b/src/llama-context.cpp index 0aec2d700..559282a9a 100644 --- a/src/llama-context.cpp +++ b/src/llama-context.cpp @@ -738,11 +738,15 @@ void llama_context::release_device(bool evict_kv) { if (evict_kv && memory && !kv_device_evicted) { memory->release_device_buffers(); kv_device_evicted = true; + // also free the scheduler and its worst-case compute buffer (hundreds of MiB) so a cold + // model holds essentially no VRAM. Rebuilt lazily by sched_reserve() on restore. + sched.reset(); + sched_need_reserve = true; } } void llama_context::restore_device() { - if (model.weights_resident() && !kv_device_evicted) { + if (model.weights_resident() && !kv_device_evicted && sched) { return; } model.restore_device_weights(); @@ -750,6 +754,10 @@ void llama_context::restore_device() { memory->restore_device_buffers(); kv_device_evicted = false; } + // rebuild the scheduler + compute buffer if they were freed on release (evict_kv mode) + if (!sched) { + sched_reserve(); + } // make sure all weight/KV uploads have completed before any compute reads them if (sched) { ggml_backend_sched_synchronize(sched.get());