After llama_context_release_device(ctx, true) the KV cache and the recurrent state live only in host shadows, and their tensors have no buffer, so llama_state_seq_get_data cannot read them. The disk prompt cache must save a cold server's slots at exit, and a shutting-down server must not restore to the device (it may not hold the GPU token, F7). llama_context_restore_memory_host() makes the state readable without the device. restore_device_buffers(to_host = true) wraps each shadow as a CPU_Mapped buffer (ggml_backend_cpu_buffer_from_ptr) and points the tensors into it: there is no copy and no allocation. The first version copied into a CPU buffer instead, which needs the full KV allocation a second time, about 2.7 GB for the 35B at ctx 131072; the node has had an OOM-controller kill before. The buffer base is rounded down to the 32-byte alignment that cpu_buffer_from_ptr asserts (a large std::vector is only 16-byte aligned); those leading bytes are never accessed. With to_host the shadows are kept, because the buffers point into them. It sets device_discarded, so restore_device() aborts afterwards. Exit only. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
113 lines
3.2 KiB
C++
113 lines
3.2 KiB
C++
#include "llama-memory.h"
|
|
|
|
#include "llama-ext.h"
|
|
#include "llama-io.h"
|
|
|
|
#include <cstdint>
|
|
#include <set>
|
|
#include <vector>
|
|
|
|
llama_memory_status llama_memory_status_combine(llama_memory_status s0, llama_memory_status s1) {
|
|
bool has_update = false;
|
|
|
|
switch (s0) {
|
|
case LLAMA_MEMORY_STATUS_SUCCESS:
|
|
{
|
|
has_update = true;
|
|
break;
|
|
}
|
|
case LLAMA_MEMORY_STATUS_NO_UPDATE:
|
|
{
|
|
break;
|
|
}
|
|
case LLAMA_MEMORY_STATUS_FAILED_PREPARE:
|
|
case LLAMA_MEMORY_STATUS_FAILED_COMPUTE:
|
|
{
|
|
return s0;
|
|
}
|
|
}
|
|
|
|
switch (s1) {
|
|
case LLAMA_MEMORY_STATUS_SUCCESS:
|
|
{
|
|
has_update = true;
|
|
break;
|
|
}
|
|
case LLAMA_MEMORY_STATUS_NO_UPDATE:
|
|
{
|
|
break;
|
|
}
|
|
case LLAMA_MEMORY_STATUS_FAILED_PREPARE:
|
|
case LLAMA_MEMORY_STATUS_FAILED_COMPUTE:
|
|
{
|
|
return s1;
|
|
}
|
|
}
|
|
|
|
// if either status has an update, then the combined status has an update
|
|
return has_update ? LLAMA_MEMORY_STATUS_SUCCESS : LLAMA_MEMORY_STATUS_NO_UPDATE;
|
|
}
|
|
|
|
bool llama_memory_status_is_fail(llama_memory_status status) {
|
|
switch (status) {
|
|
case LLAMA_MEMORY_STATUS_SUCCESS:
|
|
case LLAMA_MEMORY_STATUS_NO_UPDATE:
|
|
{
|
|
return false;
|
|
}
|
|
case LLAMA_MEMORY_STATUS_FAILED_PREPARE:
|
|
case LLAMA_MEMORY_STATUS_FAILED_COMPUTE:
|
|
{
|
|
return true;
|
|
}
|
|
}
|
|
|
|
return false;
|
|
}
|
|
|
|
// collects the buffers of the tensors that a state write visits, without keeping the data
|
|
struct llama_memory_buffers_writer : public llama_io_write_i {
|
|
std::set<ggml_backend_buffer_t> buffers;
|
|
size_t size = 0;
|
|
|
|
void write(const void *, size_t n) override {
|
|
size += n;
|
|
}
|
|
|
|
void write_tensor(ggml_tensor * tensor, size_t, size_t n) override {
|
|
buffers.insert(tensor->buffer);
|
|
size += n;
|
|
}
|
|
|
|
size_t n_bytes() override {
|
|
return size;
|
|
}
|
|
};
|
|
|
|
std::vector<ggml_backend_buffer_t> llama_memory_get_buffers(const llama_memory_t mem) {
|
|
llama_memory_buffers_writer writer;
|
|
|
|
mem->state_write(writer);
|
|
|
|
return std::vector<ggml_backend_buffer_t>(writer.buffers.begin(), writer.buffers.end());
|
|
}
|
|
|
|
ggml_backend_buffer_t llama_memory_wrap_host(ggml_context * ctx, std::vector<uint8_t> & data) {
|
|
// the buffer base must be aligned to 32 bytes; the bytes before data.data() are never accessed
|
|
uint8_t * base = (uint8_t *) ((uintptr_t) data.data() & ~(uintptr_t) 31);
|
|
ggml_backend_buffer_t buf = ggml_backend_cpu_buffer_from_ptr(base, data.size() + (data.data() - base));
|
|
size_t off = 0;
|
|
for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != nullptr; t = ggml_get_next_tensor(ctx, t)) {
|
|
if (t->view_src == nullptr) {
|
|
ggml_backend_tensor_alloc(buf, t, data.data() + off);
|
|
off += ggml_nbytes(t);
|
|
}
|
|
}
|
|
for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != nullptr; t = ggml_get_next_tensor(ctx, t)) {
|
|
if (t->view_src != nullptr) {
|
|
ggml_backend_view_init(t);
|
|
}
|
|
}
|
|
return buf;
|
|
}
|