server: on-demand VRAM sharing to time-share one GPU between models

Add release/restore of a model's GPU weight buffers (keeping a host shadow
and the KV cache) so several always-loaded llama-server processes can
time-share a single GPU without reloading or losing the prompt cache.

- llama-model: release_device_weights()/restore_device_weights() capture a
  compact host shadow (stable iteration order, view-skipping) and free then
  realloc the device weight buffers; weights_resident() query.
- llama-context: release_device()/restore_device() wrappers; decode() auto-
  restores; public C API llama_context_release_device/restore_device.
- server: LLAMA_SLEEP_VRAM_ONLY makes idle-sleep release only the VRAM weights
  (not a full unload/reload). A cross-process flock token in LLAMA_VRAM_ARENA
  enforces "resident iff holds token"; an inotify doorbell forces the holder
  to release on contention. The warden thread only touches the task queue, so
  releases run on the loop thread and never race a decode.

Validated on RX 580 (Vulkan): two models share 8GB, never both resident,
correct output under contention, KV cache preserved (no re-prefill).

Assisted-by: Claude
This commit is contained in:
Lumpiasty committed 2026-11-21 19:32:30 +01:00
1 parent cfda297cbb
commit 56a1010bbc
8 files changed
+400 -6

No files matched your search

+122 -1
View File
@@ -1388,6 +1388,16 @@ struct llama_model::impl {
// contexts where the model tensors metadata is stored as well as the corresponding buffers:
std::vector<std::pair<ggml_context_ptr, std::vector<ggml_backend_buffer_ptr>>> ctxs_bufs;
// on-demand device (VRAM) weight residency: per-ctxs_bufs slot describing whether the
// device weight buffer can be freed and rebuilt from a host shadow (see release/restore_device_weights)
struct weight_release_slot {
ggml_backend_buffer_type_t buft = nullptr; // buffer type to reallocate on restore
bool releasable = false; // single-buffer alloc-path device (VRAM) buffer
bool resident = true; // currently allocated on device
std::vector<uint8_t> shadow; // host copy, captured lazily on first release
};
std::vector<weight_release_slot> weight_release; // parallel to ctxs_bufs
buft_list_t cpu_buft_list;
std::map<ggml_backend_dev_t, buft_list_t> gpu_buft_list;
@@ -2129,8 +2139,9 @@ bool llama_model_base::load_tensors(llama_model_loader & ml) {
// a lazy context is mapped whatever the load mode, but the memory-fit pass maps nothing
const bool is_lazy_mapped = ctx_key.lazy && !ml.no_alloc;
const bool host_ptr_path = (ml.use_mmap || is_lazy_mapped) && use_mmap_buffer && buffer_from_host_ptr_supported && is_default_buft;
if ((ml.use_mmap || is_lazy_mapped) && use_mmap_buffer && buffer_from_host_ptr_supported && is_default_buft) {
if (host_ptr_path) {
GGML_ASSERT(!ml.no_alloc);
const size_t max_size = ggml_get_max_tensor_size(ctx);
@@ -2195,6 +2206,23 @@ bool llama_model_base::load_tensors(llama_model_loader & ml) {
pimpl->ctxs_bufs.emplace_back(std::move(ctx_ptr), std::move(bufs));
// record on-demand release metadata (parallel to ctxs_bufs). Only the single-buffer
// alloc path on a device (non-host) buffer can be freed and rebuilt from a host shadow;
// the host_ptr path (mmap / lazy), the no_alloc dummy buffers and host (CPU) buffers are
// left resident.
{
llama_model::impl::weight_release_slot slot;
auto & last_bufs = pimpl->ctxs_bufs.back().second;
if (!host_ptr_path && !ml.no_alloc && last_bufs.size() == 1) {
ggml_backend_buffer_t b = last_bufs[0].get();
if (b != nullptr && !ggml_backend_buffer_is_host(b)) {
slot.releasable = true;
slot.buft = buft;
}
}
pimpl->weight_release.push_back(std::move(slot));
}
ctx_bufs.emplace_back(ctx, std::move(buf_ptrs));
}
@@ -2523,6 +2551,7 @@ void llama_model_base::load_tensors_hadamard() {
std::vector<ggml_backend_buffer_ptr> buffers;
buffers.emplace_back(std::move(buffer));
pimpl->ctxs_bufs.emplace_back(std::move(ctx), std::move(buffers));
pimpl->weight_release.emplace_back(); // small, left resident
rotations.push_back({ block_size, buft, rotation });
it = std::prev(rotations.end());
}
@@ -2568,6 +2597,7 @@ void llama_model_base::load_tensors_hadamard() {
std::vector<ggml_backend_buffer_ptr> buffers;
buffers.emplace_back(std::move(buffer));
pimpl->ctxs_bufs.emplace_back(std::move(ctx), std::move(buffers));
pimpl->weight_release.emplace_back(); // small, left resident
st = sign_tensors.emplace(key, signs).first;
}
sign_tensor = st->second;
@@ -2598,6 +2628,97 @@ ggml_tensor * llama_model_base::create_tensor(llama_model_loader & ml, const LLM
tn, ne, flags);
}
void llama_model::release_device_weights() const {
// NOTE: the caller is responsible for synchronizing the backend scheduler first, so that no
// compute is in flight referencing these buffers when they are freed.
size_t freed = 0;
for (size_t i = 0; i < pimpl->ctxs_bufs.size(); ++i) {
auto & slot = pimpl->weight_release[i];
if (!slot.releasable || !slot.resident) {
continue;
}
ggml_context * ctx = pimpl->ctxs_bufs[i].first.get();
ggml_backend_buffer_t buf = pimpl->ctxs_bufs[i].second[0].get();
const size_t sz = ggml_backend_buffer_get_size(buf);
// lazily capture a host shadow of the weights (read-only, so captured only once).
// Store tensor bytes compactly in stable iteration order (independent of buffer layout /
// alignment padding) so restore does not depend on the re-allocated offsets matching.
// View tensors alias their base, so they are skipped (restored implicitly via their base).
if (slot.shadow.empty()) {
size_t total = 0;
for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != nullptr; t = ggml_get_next_tensor(ctx, t)) {
if (t->view_src != nullptr) continue;
total += ggml_nbytes(t);
}
slot.shadow.resize(total);
size_t off = 0;
for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != nullptr; t = ggml_get_next_tensor(ctx, t)) {
if (t->view_src != nullptr) continue;
const size_t n = ggml_nbytes(t);
ggml_backend_tensor_get(t, slot.shadow.data() + off, 0, n);
off += n;
}
}
// free the device (VRAM) buffer and clear the now-dangling tensor pointers so that
// ggml_backend_alloc_ctx_tensors_from_buft reallocates them cleanly on restore
pimpl->ctxs_bufs[i].second[0].reset();
for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != nullptr; t = ggml_get_next_tensor(ctx, t)) {
t->buffer = nullptr;
t->data = nullptr;
}
slot.resident = false;
freed += sz;
}
if (freed > 0) {
LLAMA_LOG_INFO("%s: released %.2f MiB of device weights\n", __func__, freed / 1024.0 / 1024.0);
}
}
bool llama_model::restore_device_weights() const {
size_t restored = 0;
for (size_t i = 0; i < pimpl->ctxs_bufs.size(); ++i) {
auto & slot = pimpl->weight_release[i];
if (!slot.releasable || slot.resident) {
continue;
}
ggml_context * ctx = pimpl->ctxs_bufs[i].first.get();
ggml_backend_buffer_t buf = ggml_backend_alloc_ctx_tensors_from_buft(ctx, slot.buft);
if (buf == nullptr) {
LLAMA_LOG_ERROR("%s: failed to reallocate device weight buffer (out of VRAM?)\n", __func__);
return false;
}
ggml_backend_buffer_set_usage(buf, GGML_BACKEND_BUFFER_USAGE_WEIGHTS);
// re-upload weights from the compact host shadow, using the same stable iteration order
// and view-skipping as the capture above
size_t off = 0;
for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != nullptr; t = ggml_get_next_tensor(ctx, t)) {
if (t->view_src != nullptr) continue;
const size_t n = ggml_nbytes(t);
ggml_backend_tensor_set(t, slot.shadow.data() + off, 0, n);
off += n;
}
pimpl->ctxs_bufs[i].second[0].reset(buf);
slot.resident = true;
restored += ggml_backend_buffer_get_size(buf);
}
if (restored > 0) {
LLAMA_LOG_INFO("%s: restored %.2f MiB of device weights\n", __func__, restored / 1024.0 / 1024.0);
}
return true;
}
bool llama_model::weights_resident() const {
for (const auto & slot : pimpl->weight_release) {
if (slot.releasable && !slot.resident) {
return false;
}
}
return true;
}
std::string llama_model::arch_name() const {
return llm_arch_name(arch);
}