From 4d576de9dd07be4f9dfa3c51d0844cda1f3263c0 Mon Sep 17 00:00:00 2001 From: Lumpiasty Date: Sun, 26 Jul 2026 00:07:01 +0200 Subject: [PATCH] server: restore weights/KV on wake, not at decode, so KV eviction is safe With LLAMA_SLEEP_EVICT_KV the KV cache device buffers are freed when a model goes cold. update_slots() touches the KV before the decode-time vram_ensure_warm (e.g. SWA models create a checkpoint that reads the KV via ggml_backend_tensor_get), so the KV must already be resident by then. Move the restore to the sleep-wake handler (handle_sleeping_state(false)), which runs before update_slots, fixing a GGML_ASSERT(buffer) crash on iSWA models (gemma) when KV eviction is enabled. Assisted-by: Claude --- tools/server/server-context.cpp | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/tools/server/server-context.cpp b/tools/server/server-context.cpp index 770e5636e..dbc6bcda4 100644 --- a/tools/server/server-context.cpp +++ b/tools/server/server-context.cpp @@ -1159,8 +1159,12 @@ private: SRV_INF("%s", "server entering sleeping state (VRAM-only: releasing device weights, keeping KV cache)\n"); vram_go_cold(); } else { - SRV_INF("%s", "server exiting sleeping state (VRAM-only: weights restored on next decode)\n"); - // token is acquired and weights restored by vram_ensure_warm() before decode + SRV_INF("%s", "server exiting sleeping state (VRAM-only: restoring device weights/KV)\n"); + // Restore NOW, on wake, before update_slots runs. update_slots touches the KV cache + // (e.g. SWA checkpoint creation reads it via ggml_backend_tensor_get) before the + // decode-time vram_ensure_warm(), so with KV eviction the KV must already be resident + // here or those reads hit a freed (null) buffer. + vram_ensure_warm(); } sleeping = new_state; return;