Files
llama.cpp/src/llama-memory.cpp
T
LumpiastyandClaude Opus 5.5 7dbe7fdafa llama: read a released KV/recurrent state from its host shadow
After llama_context_release_device(ctx, true) the KV cache and the recurrent
state live only in host shadows, and their tensors have no buffer, so
llama_state_seq_get_data cannot read them. The disk prompt cache must save
a cold server's slots at exit, and a shutting-down server must not restore
to the device (it may not hold the GPU token, F7).

llama_context_restore_memory_host() makes the state readable without the
device. restore_device_buffers(to_host = true) wraps each shadow as a
CPU_Mapped buffer (ggml_backend_cpu_buffer_from_ptr) and points the tensors
into it: there is no copy and no allocation. The first version copied into a
CPU buffer instead, which needs the full KV allocation a second time, about
2.7 GB for the 35B at ctx 131072; the node has had an OOM-controller kill
before. The buffer base is rounded down to the 32-byte alignment that
cpu_buffer_from_ptr asserts (a large std::vector is only 16-byte aligned);
those leading bytes are never accessed. With to_host the shadows are kept,
because the buffers point into them.

It sets device_discarded, so restore_device() aborts afterwards. Exit only.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-21 19:39:56 +01:00

177 lines
5.4 KiB
C++

#include "llama-memory.h"
#include "llama-ext.h"
#include "llama-io.h"
#include <cstdint>
#include <set>
#include <stdexcept>
#include <string>
#include <vector>
llama_memory_status llama_memory_status_combine(llama_memory_status s0, llama_memory_status s1) {
bool has_update = false;
switch (s0) {
case LLAMA_MEMORY_STATUS_SUCCESS:
{
has_update = true;
break;
}
case LLAMA_MEMORY_STATUS_NO_UPDATE:
{
break;
}
case LLAMA_MEMORY_STATUS_FAILED_PREPARE:
case LLAMA_MEMORY_STATUS_FAILED_COMPUTE:
{
return s0;
}
}
switch (s1) {
case LLAMA_MEMORY_STATUS_SUCCESS:
{
has_update = true;
break;
}
case LLAMA_MEMORY_STATUS_NO_UPDATE:
{
break;
}
case LLAMA_MEMORY_STATUS_FAILED_PREPARE:
case LLAMA_MEMORY_STATUS_FAILED_COMPUTE:
{
return s1;
}
}
// if either status has an update, then the combined status has an update
return has_update ? LLAMA_MEMORY_STATUS_SUCCESS : LLAMA_MEMORY_STATUS_NO_UPDATE;
}
bool llama_memory_status_is_fail(llama_memory_status status) {
switch (status) {
case LLAMA_MEMORY_STATUS_SUCCESS:
case LLAMA_MEMORY_STATUS_NO_UPDATE:
{
return false;
}
case LLAMA_MEMORY_STATUS_FAILED_PREPARE:
case LLAMA_MEMORY_STATUS_FAILED_COMPUTE:
{
return true;
}
}
return false;
}
//
// llama_memory_bufs
//
ggml_backend_buffer_ptr llama_memory_bufs::alloc_buf(ggml_context * ctx, ggml_backend_buffer_type_t buft, bool no_alloc, const char * name) {
ggml_backend_buffer_ptr buf;
if (no_alloc) {
buf.reset(ggml_backend_buft_alloc_buffer(buft, /*size =*/ 0)); // dummy buffer
for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != nullptr; t = ggml_get_next_tensor(ctx, t)) {
t->buffer = buf.get(); // set dummy buffer so that the backend scheduler won't try to allocate the tensors
}
} else {
buf.reset(ggml_backend_alloc_ctx_tensors_from_buft(ctx, buft)); // real buffer
}
if (!buf) {
throw std::runtime_error(std::string("failed to allocate buffer for ") + name);
}
return buf;
}
ggml_backend_buffer_t llama_memory_bufs::emplace(ggml_context_ptr ctx, ggml_backend_buffer_type_t buft, const char * name) {
ggml_backend_buffer_ptr buf = alloc_buf(ctx.get(), buft, no_alloc, name);
// initialize the buffer to avoid NaNs in the padding
ggml_backend_buffer_clear(buf.get(), 0);
bufs.emplace_back(std::move(ctx), std::move(buf));
return bufs.back().second.get();
}
std::map<ggml_backend_buffer_type_t, size_t> llama_memory_bufs::memory_breakdown() const {
std::map<ggml_backend_buffer_type_t, size_t> ret;
for (const auto & [ctx, buf] : bufs) {
if (!buf) { // may be null if evicted for on-demand VRAM sharing
continue;
}
ggml_backend_buffer_type_t buft = ggml_backend_buffer_get_type(buf.get());
if (no_alloc) {
GGML_ASSERT(ggml_backend_buffer_get_base(buf.get()) == nullptr);
ret[buft] += ggml_backend_alloc_ctx_tensors_from_buft_size(ctx.get(), buft);
} else {
ret[buft] += ggml_backend_buffer_get_size(buf.get());
}
}
return ret;
}
// collects the tensors that a state write visits, without keeping the data
struct llama_memory_tensors_writer : public llama_io_write_i {
std::set<ggml_tensor *> tensors;
size_t size = 0;
void write(const void *, size_t n) override {
size += n;
}
void write_tensor(ggml_tensor * tensor, size_t, size_t n) override {
tensors.insert(tensor);
size += n;
}
size_t n_bytes() override {
return size;
}
};
std::vector<ggml_backend_buffer_t> llama_memory_get_buffers(const llama_memory_t mem) {
std::set<ggml_backend_buffer_t> buffers;
for (ggml_tensor * tensor : llama_memory_get_tensors(mem)) {
buffers.insert(tensor->buffer);
}
return std::vector<ggml_backend_buffer_t>(buffers.begin(), buffers.end());
}
std::vector<ggml_tensor *> llama_memory_get_tensors(const llama_memory_t mem) {
if (mem == nullptr) {
return {};
}
llama_memory_tensors_writer writer;
mem->state_write(writer);
return std::vector<ggml_tensor *>(writer.tensors.begin(), writer.tensors.end());
}
ggml_backend_buffer_t llama_memory_wrap_host(ggml_context * ctx, std::vector<uint8_t> & data) {
// the buffer base must be aligned to 32 bytes; the bytes before data.data() are never accessed
uint8_t * base = (uint8_t *) ((uintptr_t) data.data() & ~(uintptr_t) 31);
ggml_backend_buffer_t buf = ggml_backend_cpu_buffer_from_ptr(base, data.size() + (data.data() - base));
size_t off = 0;
for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != nullptr; t = ggml_get_next_tensor(ctx, t)) {
if (t->view_src == nullptr) {
ggml_backend_tensor_alloc(buf, t, data.data() + off);
off += ggml_nbytes(t);
}
}
for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != nullptr; t = ggml_get_next_tensor(ctx, t)) {
if (t->view_src != nullptr) {
ggml_backend_view_init(t);
}
}
return buf;
}