llama: read a released KV/recurrent state from its host shadow
After llama_context_release_device(ctx, true) the KV cache and the recurrent state live only in host shadows, and their tensors have no buffer, so llama_state_seq_get_data cannot read them. The disk prompt cache must save a cold server's slots at exit, and a shutting-down server must not restore to the device (it may not hold the GPU token, F7). llama_context_restore_memory_host() makes the state readable without the device. restore_device_buffers(to_host = true) wraps each shadow as a CPU_Mapped buffer (ggml_backend_cpu_buffer_from_ptr) and points the tensors into it: there is no copy and no allocation. The first version copied into a CPU buffer instead, which needs the full KV allocation a second time, about 2.7 GB for the 35B at ctx 131072; the node has had an OOM-controller kill before. The buffer base is rounded down to the 32-byte alignment that cpu_buffer_from_ptr asserts (a large std::vector is only 16-byte aligned); those leading bytes are never accessed. With to_host the shadows are kept, because the buffers point into them. It sets device_discarded, so restore_device() aborts afterwards. Exit only. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
f24c3582c2
commit
dd851fd0c9
13 files changed
+72
-15
No files matched your search
@@ -603,6 +603,10 @@ extern "C" {
|
||||
// Free all device buffers (weights, KV, compute) for good, without host copies, e.g. to hand
|
||||
// the VRAM to another process before exit. The context must not be used for compute afterwards.
|
||||
LLAMA_API void llama_context_discard_device(struct llama_context * ctx);
|
||||
// Before exit, after llama_context_release_device(ctx, true): point the evicted KV/recurrent state
|
||||
// at its host copy (no device use, no extra copy) so llama_state_seq_get_data can read it. The
|
||||
// context must not be used for compute afterwards.
|
||||
LLAMA_API bool llama_context_restore_memory_host(struct llama_context * ctx);
|
||||
LLAMA_API bool llama_context_weights_resident(const struct llama_context * ctx);
|
||||
LLAMA_API enum llama_pooling_type llama_pooling_type(const struct llama_context * ctx); // TODO: rename to llama_get_pooling_type
|
||||
|
||||
|
||||
@@ -968,6 +968,14 @@ void llama_context::discard_device() {
|
||||
}
|
||||
}
|
||||
|
||||
bool llama_context::restore_memory_host() {
|
||||
device_discarded = true;
|
||||
if (!kv_device_evicted || !memory) {
|
||||
return true;
|
||||
}
|
||||
return memory->restore_device_buffers(/*to_host =*/ true);
|
||||
}
|
||||
|
||||
void llama_context::restore_device() {
|
||||
GGML_ASSERT(!device_discarded && "device buffers were discarded");
|
||||
if (device_resident() && !kv_device_evicted && sched) {
|
||||
@@ -4350,6 +4358,10 @@ void llama_context_discard_device(llama_context * ctx) {
|
||||
ctx->discard_device();
|
||||
}
|
||||
|
||||
bool llama_context_restore_memory_host(llama_context * ctx) {
|
||||
return ctx->restore_memory_host();
|
||||
}
|
||||
|
||||
bool llama_context_weights_resident(const llama_context * ctx) {
|
||||
return ctx->device_resident();
|
||||
}
|
||||
|
||||
@@ -70,6 +70,9 @@ struct llama_context {
|
||||
// free every device buffer for good, without host copies (before exit). The context must not
|
||||
// compute again: restore_device() aborts.
|
||||
void discard_device();
|
||||
// before exit: bring an evicted KV/recurrent state back in host memory so the state API can read
|
||||
// it without the device. The context must not compute again: restore_device() aborts.
|
||||
bool restore_memory_host();
|
||||
|
||||
// the model weights and the MoE cache are on the device
|
||||
bool device_resident() const;
|
||||
|
||||
@@ -269,9 +269,9 @@ void llama_kv_cache_iswa::release_device_buffers(bool keep_shadow) {
|
||||
kv_swa->release_device_buffers(keep_shadow);
|
||||
}
|
||||
|
||||
bool llama_kv_cache_iswa::restore_device_buffers() {
|
||||
bool ok = kv_base->restore_device_buffers();
|
||||
ok = kv_swa->restore_device_buffers() && ok;
|
||||
bool llama_kv_cache_iswa::restore_device_buffers(bool to_host) {
|
||||
bool ok = kv_base->restore_device_buffers(to_host);
|
||||
ok = kv_swa->restore_device_buffers(to_host) && ok;
|
||||
return ok;
|
||||
}
|
||||
|
||||
|
||||
@@ -84,7 +84,7 @@ public:
|
||||
void state_read (llama_io_read_i & io, llama_seq_id seq_id = -1, llama_state_seq_flags flags = 0) override;
|
||||
|
||||
void release_device_buffers(bool keep_shadow = true) override;
|
||||
bool restore_device_buffers() override;
|
||||
bool restore_device_buffers(bool to_host = false) override;
|
||||
|
||||
//
|
||||
// llama_kv_cache_iswa specific API
|
||||
|
||||
@@ -1976,7 +1976,7 @@ void llama_kv_cache::release_device_buffers(bool keep_shadow) {
|
||||
}
|
||||
}
|
||||
|
||||
bool llama_kv_cache::restore_device_buffers() {
|
||||
bool llama_kv_cache::restore_device_buffers(bool to_host) {
|
||||
if (!dev_released) {
|
||||
return true;
|
||||
}
|
||||
@@ -1986,6 +1986,10 @@ bool llama_kv_cache::restore_device_buffers() {
|
||||
continue;
|
||||
}
|
||||
ggml_context * ctx = ctxs_bufs[i].first.get();
|
||||
if (to_host) {
|
||||
ctxs_bufs[i].second.reset(llama_memory_wrap_host(ctx, sh.data));
|
||||
continue;
|
||||
}
|
||||
ggml_backend_buffer_t buf = ggml_backend_alloc_ctx_tensors_from_buft(ctx, sh.buft);
|
||||
if (buf == nullptr) {
|
||||
LLAMA_LOG_ERROR("%s: failed to reallocate KV cache device buffer (out of VRAM?)\n", __func__);
|
||||
@@ -2001,7 +2005,9 @@ bool llama_kv_cache::restore_device_buffers() {
|
||||
ctxs_bufs[i].second.reset(buf);
|
||||
}
|
||||
dev_released = false;
|
||||
dev_shadows.clear(); // recaptured on next release
|
||||
if (!to_host) {
|
||||
dev_shadows.clear(); // recaptured on next release; to_host buffers point into the shadows
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
|
||||
@@ -153,7 +153,7 @@ public:
|
||||
|
||||
// on-demand device (VRAM) residency (see llama_memory_i)
|
||||
void release_device_buffers(bool keep_shadow = true) override;
|
||||
bool restore_device_buffers() override;
|
||||
bool restore_device_buffers(bool to_host = false) override;
|
||||
|
||||
//
|
||||
// llama_kv_cache specific API
|
||||
|
||||
@@ -211,9 +211,9 @@ void llama_memory_hybrid::release_device_buffers(bool keep_shadow) {
|
||||
mem_recr->release_device_buffers(keep_shadow);
|
||||
}
|
||||
|
||||
bool llama_memory_hybrid::restore_device_buffers() {
|
||||
bool ok = mem_attn->restore_device_buffers();
|
||||
ok = mem_recr->restore_device_buffers() && ok;
|
||||
bool llama_memory_hybrid::restore_device_buffers(bool to_host) {
|
||||
bool ok = mem_attn->restore_device_buffers(to_host);
|
||||
ok = mem_recr->restore_device_buffers(to_host) && ok;
|
||||
return ok;
|
||||
}
|
||||
|
||||
|
||||
@@ -77,7 +77,7 @@ public:
|
||||
void state_read (llama_io_read_i & io, llama_seq_id seq_id = -1, llama_state_seq_flags flags = 0) override;
|
||||
|
||||
void release_device_buffers(bool keep_shadow = true) override;
|
||||
bool restore_device_buffers() override;
|
||||
bool restore_device_buffers(bool to_host = false) override;
|
||||
|
||||
//
|
||||
// llama_memory_hybrid specific API
|
||||
|
||||
@@ -800,7 +800,7 @@ void llama_memory_recurrent::release_device_buffers(bool keep_shadow) {
|
||||
}
|
||||
}
|
||||
|
||||
bool llama_memory_recurrent::restore_device_buffers() {
|
||||
bool llama_memory_recurrent::restore_device_buffers(bool to_host) {
|
||||
if (!dev_released) {
|
||||
return true;
|
||||
}
|
||||
@@ -810,6 +810,10 @@ bool llama_memory_recurrent::restore_device_buffers() {
|
||||
continue;
|
||||
}
|
||||
ggml_context * ctx = ctxs_bufs[i].first.get();
|
||||
if (to_host) {
|
||||
ctxs_bufs[i].second.reset(llama_memory_wrap_host(ctx, sh.data));
|
||||
continue;
|
||||
}
|
||||
ggml_backend_buffer_t buf = ggml_backend_alloc_ctx_tensors_from_buft(ctx, sh.buft);
|
||||
if (buf == nullptr) {
|
||||
LLAMA_LOG_ERROR("%s: failed to reallocate recurrent device buffer (out of VRAM?)\n", __func__);
|
||||
@@ -825,7 +829,9 @@ bool llama_memory_recurrent::restore_device_buffers() {
|
||||
ctxs_bufs[i].second.reset(buf);
|
||||
}
|
||||
dev_released = false;
|
||||
dev_shadows.clear();
|
||||
if (!to_host) {
|
||||
dev_shadows.clear(); // to_host buffers point into the shadows
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
|
||||
@@ -68,7 +68,7 @@ public:
|
||||
|
||||
// on-demand device (VRAM) residency (see llama_memory_i)
|
||||
void release_device_buffers(bool keep_shadow = true) override;
|
||||
bool restore_device_buffers() override;
|
||||
bool restore_device_buffers(bool to_host = false) override;
|
||||
|
||||
uint32_t head = 0; // the location where the batch will be placed in the cache (see find_slot())
|
||||
uint32_t size = 0; // total number of cells, shared across all sequences
|
||||
|
||||
@@ -3,6 +3,7 @@
|
||||
#include "llama-ext.h"
|
||||
#include "llama-io.h"
|
||||
|
||||
#include <cstdint>
|
||||
#include <set>
|
||||
#include <vector>
|
||||
|
||||
@@ -90,3 +91,22 @@ std::vector<ggml_backend_buffer_t> llama_memory_get_buffers(const llama_memory_t
|
||||
|
||||
return std::vector<ggml_backend_buffer_t>(writer.buffers.begin(), writer.buffers.end());
|
||||
}
|
||||
|
||||
ggml_backend_buffer_t llama_memory_wrap_host(ggml_context * ctx, std::vector<uint8_t> & data) {
|
||||
// the buffer base must be aligned to 32 bytes; the bytes before data.data() are never accessed
|
||||
uint8_t * base = (uint8_t *) ((uintptr_t) data.data() & ~(uintptr_t) 31);
|
||||
ggml_backend_buffer_t buf = ggml_backend_cpu_buffer_from_ptr(base, data.size() + (data.data() - base));
|
||||
size_t off = 0;
|
||||
for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != nullptr; t = ggml_get_next_tensor(ctx, t)) {
|
||||
if (t->view_src == nullptr) {
|
||||
ggml_backend_tensor_alloc(buf, t, data.data() + off);
|
||||
off += ggml_nbytes(t);
|
||||
}
|
||||
}
|
||||
for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != nullptr; t = ggml_get_next_tensor(ctx, t)) {
|
||||
if (t->view_src != nullptr) {
|
||||
ggml_backend_view_init(t);
|
||||
}
|
||||
}
|
||||
return buf;
|
||||
}
|
||||
+7
-1
@@ -6,6 +6,7 @@
|
||||
#include <map>
|
||||
#include <memory>
|
||||
#include <functional>
|
||||
#include <vector>
|
||||
|
||||
struct llama_ubatch;
|
||||
|
||||
@@ -134,8 +135,13 @@ struct llama_memory_i {
|
||||
// hand VRAM to another model while keeping the cached data (no re-prefill). The caller must
|
||||
// have synchronized the backend first. Default: no-op (the memory stays resident).
|
||||
// keep_shadow = false frees the buffers for good (before exit): no host copy, no restore.
|
||||
// to_host = true points them at the host shadow instead, without a copy (before exit, to read a released state).
|
||||
virtual void release_device_buffers(bool /*keep_shadow*/ = true) {}
|
||||
virtual bool restore_device_buffers() { return true; }
|
||||
virtual bool restore_device_buffers(bool /*to_host*/ = false) { return true; }
|
||||
};
|
||||
|
||||
// wrap a compact host shadow (tensor bytes in iteration order, views skipped) as the buffer of the tensors of ctx, without a copy
|
||||
// the shadow must outlive the buffer
|
||||
ggml_backend_buffer_t llama_memory_wrap_host(ggml_context * ctx, std::vector<uint8_t> & data);
|
||||
|
||||
using llama_memory_ptr = std::unique_ptr<llama_memory_i>;
|
||||
Reference in new issue
Block a user