From dd851fd0c9d2e8e18d068767ff40e641796ff73d Mon Sep 17 00:00:00 2001 From: Lumpiasty Date: Sun, 25 Oct 2026 14:27:50 +0100 Subject: [PATCH] llama: read a released KV/recurrent state from its host shadow After llama_context_release_device(ctx, true) the KV cache and the recurrent state live only in host shadows, and their tensors have no buffer, so llama_state_seq_get_data cannot read them. The disk prompt cache must save a cold server's slots at exit, and a shutting-down server must not restore to the device (it may not hold the GPU token, F7). llama_context_restore_memory_host() makes the state readable without the device. restore_device_buffers(to_host = true) wraps each shadow as a CPU_Mapped buffer (ggml_backend_cpu_buffer_from_ptr) and points the tensors into it: there is no copy and no allocation. The first version copied into a CPU buffer instead, which needs the full KV allocation a second time, about 2.7 GB for the 35B at ctx 131072; the node has had an OOM-controller kill before. The buffer base is rounded down to the 32-byte alignment that cpu_buffer_from_ptr asserts (a large std::vector is only 16-byte aligned); those leading bytes are never accessed. With to_host the shadows are kept, because the buffers point into them. It sets device_discarded, so restore_device() aborts afterwards. Exit only. Co-Authored-By: Claude Opus 5.5 --- include/llama.h | 4 ++++ src/llama-context.cpp | 12 ++++++++++++ src/llama-context.h | 3 +++ src/llama-kv-cache-iswa.cpp | 6 +++--- src/llama-kv-cache-iswa.h | 2 +- src/llama-kv-cache.cpp | 10 ++++++++-- src/llama-kv-cache.h | 2 +- src/llama-memory-hybrid.cpp | 6 +++--- src/llama-memory-hybrid.h | 2 +- src/llama-memory-recurrent.cpp | 10 ++++++++-- src/llama-memory-recurrent.h | 2 +- src/llama-memory.cpp | 20 ++++++++++++++++++++ src/llama-memory.h | 8 +++++++- 13 files changed, 72 insertions(+), 15 deletions(-) diff --git a/include/llama.h b/include/llama.h index c02cc626d..e880f1535 100644 --- a/include/llama.h +++ b/include/llama.h @@ -603,6 +603,10 @@ extern "C" { // Free all device buffers (weights, KV, compute) for good, without host copies, e.g. to hand // the VRAM to another process before exit. The context must not be used for compute afterwards. LLAMA_API void llama_context_discard_device(struct llama_context * ctx); + // Before exit, after llama_context_release_device(ctx, true): point the evicted KV/recurrent state + // at its host copy (no device use, no extra copy) so llama_state_seq_get_data can read it. The + // context must not be used for compute afterwards. + LLAMA_API bool llama_context_restore_memory_host(struct llama_context * ctx); LLAMA_API bool llama_context_weights_resident(const struct llama_context * ctx); LLAMA_API enum llama_pooling_type llama_pooling_type(const struct llama_context * ctx); // TODO: rename to llama_get_pooling_type diff --git a/src/llama-context.cpp b/src/llama-context.cpp index b0d6bfdd2..c0264c0db 100644 --- a/src/llama-context.cpp +++ b/src/llama-context.cpp @@ -968,6 +968,14 @@ void llama_context::discard_device() { } } +bool llama_context::restore_memory_host() { + device_discarded = true; + if (!kv_device_evicted || !memory) { + return true; + } + return memory->restore_device_buffers(/*to_host =*/ true); +} + void llama_context::restore_device() { GGML_ASSERT(!device_discarded && "device buffers were discarded"); if (device_resident() && !kv_device_evicted && sched) { @@ -4350,6 +4358,10 @@ void llama_context_discard_device(llama_context * ctx) { ctx->discard_device(); } +bool llama_context_restore_memory_host(llama_context * ctx) { + return ctx->restore_memory_host(); +} + bool llama_context_weights_resident(const llama_context * ctx) { return ctx->device_resident(); } diff --git a/src/llama-context.h b/src/llama-context.h index 1a90c2005..d6fe6dc07 100644 --- a/src/llama-context.h +++ b/src/llama-context.h @@ -70,6 +70,9 @@ struct llama_context { // free every device buffer for good, without host copies (before exit). The context must not // compute again: restore_device() aborts. void discard_device(); + // before exit: bring an evicted KV/recurrent state back in host memory so the state API can read + // it without the device. The context must not compute again: restore_device() aborts. + bool restore_memory_host(); // the model weights and the MoE cache are on the device bool device_resident() const; diff --git a/src/llama-kv-cache-iswa.cpp b/src/llama-kv-cache-iswa.cpp index 0d3d5f822..d10f856db 100644 --- a/src/llama-kv-cache-iswa.cpp +++ b/src/llama-kv-cache-iswa.cpp @@ -269,9 +269,9 @@ void llama_kv_cache_iswa::release_device_buffers(bool keep_shadow) { kv_swa->release_device_buffers(keep_shadow); } -bool llama_kv_cache_iswa::restore_device_buffers() { - bool ok = kv_base->restore_device_buffers(); - ok = kv_swa->restore_device_buffers() && ok; +bool llama_kv_cache_iswa::restore_device_buffers(bool to_host) { + bool ok = kv_base->restore_device_buffers(to_host); + ok = kv_swa->restore_device_buffers(to_host) && ok; return ok; } diff --git a/src/llama-kv-cache-iswa.h b/src/llama-kv-cache-iswa.h index 68c2c8969..2749a0039 100644 --- a/src/llama-kv-cache-iswa.h +++ b/src/llama-kv-cache-iswa.h @@ -84,7 +84,7 @@ public: void state_read (llama_io_read_i & io, llama_seq_id seq_id = -1, llama_state_seq_flags flags = 0) override; void release_device_buffers(bool keep_shadow = true) override; - bool restore_device_buffers() override; + bool restore_device_buffers(bool to_host = false) override; // // llama_kv_cache_iswa specific API diff --git a/src/llama-kv-cache.cpp b/src/llama-kv-cache.cpp index 00ed954c6..7ea713f16 100644 --- a/src/llama-kv-cache.cpp +++ b/src/llama-kv-cache.cpp @@ -1976,7 +1976,7 @@ void llama_kv_cache::release_device_buffers(bool keep_shadow) { } } -bool llama_kv_cache::restore_device_buffers() { +bool llama_kv_cache::restore_device_buffers(bool to_host) { if (!dev_released) { return true; } @@ -1986,6 +1986,10 @@ bool llama_kv_cache::restore_device_buffers() { continue; } ggml_context * ctx = ctxs_bufs[i].first.get(); + if (to_host) { + ctxs_bufs[i].second.reset(llama_memory_wrap_host(ctx, sh.data)); + continue; + } ggml_backend_buffer_t buf = ggml_backend_alloc_ctx_tensors_from_buft(ctx, sh.buft); if (buf == nullptr) { LLAMA_LOG_ERROR("%s: failed to reallocate KV cache device buffer (out of VRAM?)\n", __func__); @@ -2001,7 +2005,9 @@ bool llama_kv_cache::restore_device_buffers() { ctxs_bufs[i].second.reset(buf); } dev_released = false; - dev_shadows.clear(); // recaptured on next release + if (!to_host) { + dev_shadows.clear(); // recaptured on next release; to_host buffers point into the shadows + } return true; } diff --git a/src/llama-kv-cache.h b/src/llama-kv-cache.h index 46e5f345f..5be467be5 100644 --- a/src/llama-kv-cache.h +++ b/src/llama-kv-cache.h @@ -153,7 +153,7 @@ public: // on-demand device (VRAM) residency (see llama_memory_i) void release_device_buffers(bool keep_shadow = true) override; - bool restore_device_buffers() override; + bool restore_device_buffers(bool to_host = false) override; // // llama_kv_cache specific API diff --git a/src/llama-memory-hybrid.cpp b/src/llama-memory-hybrid.cpp index 6891aee20..daf76fcfe 100644 --- a/src/llama-memory-hybrid.cpp +++ b/src/llama-memory-hybrid.cpp @@ -211,9 +211,9 @@ void llama_memory_hybrid::release_device_buffers(bool keep_shadow) { mem_recr->release_device_buffers(keep_shadow); } -bool llama_memory_hybrid::restore_device_buffers() { - bool ok = mem_attn->restore_device_buffers(); - ok = mem_recr->restore_device_buffers() && ok; +bool llama_memory_hybrid::restore_device_buffers(bool to_host) { + bool ok = mem_attn->restore_device_buffers(to_host); + ok = mem_recr->restore_device_buffers(to_host) && ok; return ok; } diff --git a/src/llama-memory-hybrid.h b/src/llama-memory-hybrid.h index 91a6fe196..7a591f135 100644 --- a/src/llama-memory-hybrid.h +++ b/src/llama-memory-hybrid.h @@ -77,7 +77,7 @@ public: void state_read (llama_io_read_i & io, llama_seq_id seq_id = -1, llama_state_seq_flags flags = 0) override; void release_device_buffers(bool keep_shadow = true) override; - bool restore_device_buffers() override; + bool restore_device_buffers(bool to_host = false) override; // // llama_memory_hybrid specific API diff --git a/src/llama-memory-recurrent.cpp b/src/llama-memory-recurrent.cpp index 92cbde45f..e352ba104 100644 --- a/src/llama-memory-recurrent.cpp +++ b/src/llama-memory-recurrent.cpp @@ -800,7 +800,7 @@ void llama_memory_recurrent::release_device_buffers(bool keep_shadow) { } } -bool llama_memory_recurrent::restore_device_buffers() { +bool llama_memory_recurrent::restore_device_buffers(bool to_host) { if (!dev_released) { return true; } @@ -810,6 +810,10 @@ bool llama_memory_recurrent::restore_device_buffers() { continue; } ggml_context * ctx = ctxs_bufs[i].first.get(); + if (to_host) { + ctxs_bufs[i].second.reset(llama_memory_wrap_host(ctx, sh.data)); + continue; + } ggml_backend_buffer_t buf = ggml_backend_alloc_ctx_tensors_from_buft(ctx, sh.buft); if (buf == nullptr) { LLAMA_LOG_ERROR("%s: failed to reallocate recurrent device buffer (out of VRAM?)\n", __func__); @@ -825,7 +829,9 @@ bool llama_memory_recurrent::restore_device_buffers() { ctxs_bufs[i].second.reset(buf); } dev_released = false; - dev_shadows.clear(); + if (!to_host) { + dev_shadows.clear(); // to_host buffers point into the shadows + } return true; } diff --git a/src/llama-memory-recurrent.h b/src/llama-memory-recurrent.h index 3dfd209bd..b93bd972a 100644 --- a/src/llama-memory-recurrent.h +++ b/src/llama-memory-recurrent.h @@ -68,7 +68,7 @@ public: // on-demand device (VRAM) residency (see llama_memory_i) void release_device_buffers(bool keep_shadow = true) override; - bool restore_device_buffers() override; + bool restore_device_buffers(bool to_host = false) override; uint32_t head = 0; // the location where the batch will be placed in the cache (see find_slot()) uint32_t size = 0; // total number of cells, shared across all sequences diff --git a/src/llama-memory.cpp b/src/llama-memory.cpp index 961dd74ef..b3834d8c9 100644 --- a/src/llama-memory.cpp +++ b/src/llama-memory.cpp @@ -3,6 +3,7 @@ #include "llama-ext.h" #include "llama-io.h" +#include #include #include @@ -90,3 +91,22 @@ std::vector llama_memory_get_buffers(const llama_memory_t return std::vector(writer.buffers.begin(), writer.buffers.end()); } + +ggml_backend_buffer_t llama_memory_wrap_host(ggml_context * ctx, std::vector & data) { + // the buffer base must be aligned to 32 bytes; the bytes before data.data() are never accessed + uint8_t * base = (uint8_t *) ((uintptr_t) data.data() & ~(uintptr_t) 31); + ggml_backend_buffer_t buf = ggml_backend_cpu_buffer_from_ptr(base, data.size() + (data.data() - base)); + size_t off = 0; + for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != nullptr; t = ggml_get_next_tensor(ctx, t)) { + if (t->view_src == nullptr) { + ggml_backend_tensor_alloc(buf, t, data.data() + off); + off += ggml_nbytes(t); + } + } + for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != nullptr; t = ggml_get_next_tensor(ctx, t)) { + if (t->view_src != nullptr) { + ggml_backend_view_init(t); + } + } + return buf; +} diff --git a/src/llama-memory.h b/src/llama-memory.h index 488a0038a..0366a1e07 100644 --- a/src/llama-memory.h +++ b/src/llama-memory.h @@ -6,6 +6,7 @@ #include #include #include +#include struct llama_ubatch; @@ -134,8 +135,13 @@ struct llama_memory_i { // hand VRAM to another model while keeping the cached data (no re-prefill). The caller must // have synchronized the backend first. Default: no-op (the memory stays resident). // keep_shadow = false frees the buffers for good (before exit): no host copy, no restore. + // to_host = true points them at the host shadow instead, without a copy (before exit, to read a released state). virtual void release_device_buffers(bool /*keep_shadow*/ = true) {} - virtual bool restore_device_buffers() { return true; } + virtual bool restore_device_buffers(bool /*to_host*/ = false) { return true; } }; +// wrap a compact host shadow (tensor bytes in iteration order, views skipped) as the buffer of the tensors of ctx, without a copy +// the shadow must outlive the buffer +ggml_backend_buffer_t llama_memory_wrap_host(ggml_context * ctx, std::vector & data); + using llama_memory_ptr = std::unique_ptr;