llama: read a released KV/recurrent state from its host shadow

After llama_context_release_device(ctx, true) the KV cache and the recurrent
state live only in host shadows, and their tensors have no buffer, so
llama_state_seq_get_data cannot read them. The disk prompt cache must save
a cold server's slots at exit, and a shutting-down server must not restore
to the device (it may not hold the GPU token, F7).

llama_context_restore_memory_host() makes the state readable without the
device. restore_device_buffers(to_host = true) wraps each shadow as a
CPU_Mapped buffer (ggml_backend_cpu_buffer_from_ptr) and points the tensors
into it: there is no copy and no allocation. The first version copied into a
CPU buffer instead, which needs the full KV allocation a second time, about
2.7 GB for the 35B at ctx 131072; the node has had an OOM-controller kill
before. The buffer base is rounded down to the 32-byte alignment that
cpu_buffer_from_ptr asserts (a large std::vector is only 16-byte aligned);
those leading bytes are never accessed. With to_host the shadows are kept,
because the buffers point into them.

It sets device_discarded, so restore_device() aborts afterwards. Exit only.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
LumpiastyandClaude Opus 5.5 committed 2026-10-25 18:55:57 +01:00
1 parent f24c3582c2
commit dd851fd0c9
13 files changed
+72 -15

No files matched your search

+4
View File
@@ -603,6 +603,10 @@ extern "C" {
// Free all device buffers (weights, KV, compute) for good, without host copies, e.g. to hand
// the VRAM to another process before exit. The context must not be used for compute afterwards.
LLAMA_API void llama_context_discard_device(struct llama_context * ctx);
// Before exit, after llama_context_release_device(ctx, true): point the evicted KV/recurrent state
// at its host copy (no device use, no extra copy) so llama_state_seq_get_data can read it. The
// context must not be used for compute afterwards.
LLAMA_API bool llama_context_restore_memory_host(struct llama_context * ctx);
LLAMA_API bool llama_context_weights_resident(const struct llama_context * ctx);
LLAMA_API enum llama_pooling_type llama_pooling_type(const struct llama_context * ctx); // TODO: rename to llama_get_pooling_type
+12
View File
@@ -968,6 +968,14 @@ void llama_context::discard_device() {
}
}
bool llama_context::restore_memory_host() {
device_discarded = true;
if (!kv_device_evicted || !memory) {
return true;
}
return memory->restore_device_buffers(/*to_host =*/ true);
}
void llama_context::restore_device() {
GGML_ASSERT(!device_discarded && "device buffers were discarded");
if (device_resident() && !kv_device_evicted && sched) {
@@ -4350,6 +4358,10 @@ void llama_context_discard_device(llama_context * ctx) {
ctx->discard_device();
}
bool llama_context_restore_memory_host(llama_context * ctx) {
return ctx->restore_memory_host();
}
bool llama_context_weights_resident(const llama_context * ctx) {
return ctx->device_resident();
}
+3
View File
@@ -70,6 +70,9 @@ struct llama_context {
// free every device buffer for good, without host copies (before exit). The context must not
// compute again: restore_device() aborts.
void discard_device();
// before exit: bring an evicted KV/recurrent state back in host memory so the state API can read
// it without the device. The context must not compute again: restore_device() aborts.
bool restore_memory_host();
// the model weights and the MoE cache are on the device
bool device_resident() const;
+3 -3
View File
@@ -269,9 +269,9 @@ void llama_kv_cache_iswa::release_device_buffers(bool keep_shadow) {
kv_swa->release_device_buffers(keep_shadow);
}
bool llama_kv_cache_iswa::restore_device_buffers() {
bool ok = kv_base->restore_device_buffers();
ok = kv_swa->restore_device_buffers() && ok;
bool llama_kv_cache_iswa::restore_device_buffers(bool to_host) {
bool ok = kv_base->restore_device_buffers(to_host);
ok = kv_swa->restore_device_buffers(to_host) && ok;
return ok;
}
+1 -1
View File
@@ -84,7 +84,7 @@ public:
void state_read (llama_io_read_i & io, llama_seq_id seq_id = -1, llama_state_seq_flags flags = 0) override;
void release_device_buffers(bool keep_shadow = true) override;
bool restore_device_buffers() override;
bool restore_device_buffers(bool to_host = false) override;
//
// llama_kv_cache_iswa specific API
+8 -2
View File
@@ -1976,7 +1976,7 @@ void llama_kv_cache::release_device_buffers(bool keep_shadow) {
}
}
bool llama_kv_cache::restore_device_buffers() {
bool llama_kv_cache::restore_device_buffers(bool to_host) {
if (!dev_released) {
return true;
}
@@ -1986,6 +1986,10 @@ bool llama_kv_cache::restore_device_buffers() {
continue;
}
ggml_context * ctx = ctxs_bufs[i].first.get();
if (to_host) {
ctxs_bufs[i].second.reset(llama_memory_wrap_host(ctx, sh.data));
continue;
}
ggml_backend_buffer_t buf = ggml_backend_alloc_ctx_tensors_from_buft(ctx, sh.buft);
if (buf == nullptr) {
LLAMA_LOG_ERROR("%s: failed to reallocate KV cache device buffer (out of VRAM?)\n", __func__);
@@ -2001,7 +2005,9 @@ bool llama_kv_cache::restore_device_buffers() {
ctxs_bufs[i].second.reset(buf);
}
dev_released = false;
dev_shadows.clear(); // recaptured on next release
if (!to_host) {
dev_shadows.clear(); // recaptured on next release; to_host buffers point into the shadows
}
return true;
}
+1 -1
View File
@@ -153,7 +153,7 @@ public:
// on-demand device (VRAM) residency (see llama_memory_i)
void release_device_buffers(bool keep_shadow = true) override;
bool restore_device_buffers() override;
bool restore_device_buffers(bool to_host = false) override;
//
// llama_kv_cache specific API
+3 -3
View File
@@ -211,9 +211,9 @@ void llama_memory_hybrid::release_device_buffers(bool keep_shadow) {
mem_recr->release_device_buffers(keep_shadow);
}
bool llama_memory_hybrid::restore_device_buffers() {
bool ok = mem_attn->restore_device_buffers();
ok = mem_recr->restore_device_buffers() && ok;
bool llama_memory_hybrid::restore_device_buffers(bool to_host) {
bool ok = mem_attn->restore_device_buffers(to_host);
ok = mem_recr->restore_device_buffers(to_host) && ok;
return ok;
}
+1 -1
View File
@@ -77,7 +77,7 @@ public:
void state_read (llama_io_read_i & io, llama_seq_id seq_id = -1, llama_state_seq_flags flags = 0) override;
void release_device_buffers(bool keep_shadow = true) override;
bool restore_device_buffers() override;
bool restore_device_buffers(bool to_host = false) override;
//
// llama_memory_hybrid specific API
+8 -2
View File
@@ -800,7 +800,7 @@ void llama_memory_recurrent::release_device_buffers(bool keep_shadow) {
}
}
bool llama_memory_recurrent::restore_device_buffers() {
bool llama_memory_recurrent::restore_device_buffers(bool to_host) {
if (!dev_released) {
return true;
}
@@ -810,6 +810,10 @@ bool llama_memory_recurrent::restore_device_buffers() {
continue;
}
ggml_context * ctx = ctxs_bufs[i].first.get();
if (to_host) {
ctxs_bufs[i].second.reset(llama_memory_wrap_host(ctx, sh.data));
continue;
}
ggml_backend_buffer_t buf = ggml_backend_alloc_ctx_tensors_from_buft(ctx, sh.buft);
if (buf == nullptr) {
LLAMA_LOG_ERROR("%s: failed to reallocate recurrent device buffer (out of VRAM?)\n", __func__);
@@ -825,7 +829,9 @@ bool llama_memory_recurrent::restore_device_buffers() {
ctxs_bufs[i].second.reset(buf);
}
dev_released = false;
dev_shadows.clear();
if (!to_host) {
dev_shadows.clear(); // to_host buffers point into the shadows
}
return true;
}
+1 -1
View File
@@ -68,7 +68,7 @@ public:
// on-demand device (VRAM) residency (see llama_memory_i)
void release_device_buffers(bool keep_shadow = true) override;
bool restore_device_buffers() override;
bool restore_device_buffers(bool to_host = false) override;
uint32_t head = 0; // the location where the batch will be placed in the cache (see find_slot())
uint32_t size = 0; // total number of cells, shared across all sequences
+20
View File
@@ -3,6 +3,7 @@
#include "llama-ext.h"
#include "llama-io.h"
#include <cstdint>
#include <set>
#include <vector>
@@ -90,3 +91,22 @@ std::vector<ggml_backend_buffer_t> llama_memory_get_buffers(const llama_memory_t
return std::vector<ggml_backend_buffer_t>(writer.buffers.begin(), writer.buffers.end());
}
ggml_backend_buffer_t llama_memory_wrap_host(ggml_context * ctx, std::vector<uint8_t> & data) {
// the buffer base must be aligned to 32 bytes; the bytes before data.data() are never accessed
uint8_t * base = (uint8_t *) ((uintptr_t) data.data() & ~(uintptr_t) 31);
ggml_backend_buffer_t buf = ggml_backend_cpu_buffer_from_ptr(base, data.size() + (data.data() - base));
size_t off = 0;
for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != nullptr; t = ggml_get_next_tensor(ctx, t)) {
if (t->view_src == nullptr) {
ggml_backend_tensor_alloc(buf, t, data.data() + off);
off += ggml_nbytes(t);
}
}
for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != nullptr; t = ggml_get_next_tensor(ctx, t)) {
if (t->view_src != nullptr) {
ggml_backend_view_init(t);
}
}
return buf;
}
+7 -1
View File
@@ -6,6 +6,7 @@
#include <map>
#include <memory>
#include <functional>
#include <vector>
struct llama_ubatch;
@@ -134,8 +135,13 @@ struct llama_memory_i {
// hand VRAM to another model while keeping the cached data (no re-prefill). The caller must
// have synchronized the backend first. Default: no-op (the memory stays resident).
// keep_shadow = false frees the buffers for good (before exit): no host copy, no restore.
// to_host = true points them at the host shadow instead, without a copy (before exit, to read a released state).
virtual void release_device_buffers(bool /*keep_shadow*/ = true) {}
virtual bool restore_device_buffers() { return true; }
virtual bool restore_device_buffers(bool /*to_host*/ = false) { return true; }
};
// wrap a compact host shadow (tensor bytes in iteration order, views skipped) as the buffer of the tensors of ctx, without a copy
// the shadow must outlive the buffer
ggml_backend_buffer_t llama_memory_wrap_host(ggml_context * ctx, std::vector<uint8_t> & data);
using llama_memory_ptr = std::unique_ptr<llama_memory_i>;