server: on-demand VRAM sharing to time-share one GPU between models

Add release/restore of a model's GPU weight buffers (keeping a host shadow
and the KV cache) so several always-loaded llama-server processes can
time-share a single GPU without reloading or losing the prompt cache.

- llama-model: release_device_weights()/restore_device_weights() capture a
  compact host shadow (stable iteration order, view-skipping) and free then
  realloc the device weight buffers; weights_resident() query.
- llama-context: release_device()/restore_device() wrappers; decode() auto-
  restores; public C API llama_context_release_device/restore_device.
- server: LLAMA_SLEEP_VRAM_ONLY makes idle-sleep release only the VRAM weights
  (not a full unload/reload). A cross-process flock token in LLAMA_VRAM_ARENA
  enforces "resident iff holds token"; an inotify doorbell forces the holder
  to release on contention. The warden thread only touches the task queue, so
  releases run on the loop thread and never race a decode.

Validated on RX 580 (Vulkan): two models share 8GB, never both resident,
correct output under contention, KV cache preserved (no re-prefill).

Assisted-by: Claude

src/llama-model.cpp

@@ -1019,6 +1019,16 @@ struct llama_model::impl {
     // contexts where the model tensors metadata is stored as well as the corresponding buffers:
     std::vector<std::pair<ggml_context_ptr, std::vector<ggml_backend_buffer_ptr>>> ctxs_bufs;
 
+    // on-demand device (VRAM) weight residency: per-ctxs_bufs slot describing whether the
+    // device weight buffer can be freed and rebuilt from a host shadow (see release/restore_device_weights)
+    struct weight_release_slot {
+        ggml_backend_buffer_type_t buft = nullptr; // buffer type to reallocate on restore
+        bool releasable = false;                   // single-buffer alloc-path device (VRAM) buffer
+        bool resident   = true;                    // currently allocated on device
+        std::vector<uint8_t> shadow;               // host copy, captured lazily on first release
+    };
+    std::vector<weight_release_slot> weight_release; // parallel to ctxs_bufs
+
     buft_list_t cpu_buft_list;
     std::map<ggml_backend_dev_t, buft_list_t> gpu_buft_list;
 
@@ -1614,6 +1624,23 @@ bool llama_model_base::load_tensors(llama_model_loader & ml) {
 
         pimpl->ctxs_bufs.emplace_back(std::move(ctx_ptr), std::move(bufs));
 
+        // record on-demand release metadata (parallel to ctxs_bufs). Only the single-buffer
+        // alloc path on a device (non-host) buffer can be freed and rebuilt from a host shadow;
+        // the mmap buffer_from_host_ptr path and host (CPU) buffers are left resident.
+        {
+            llama_model::impl::weight_release_slot slot;
+            const bool mmap_host_ptr_path = ml.use_mmap && use_mmap_buffer && buffer_from_host_ptr_supported && is_default_buft;
+            auto & last_bufs = pimpl->ctxs_bufs.back().second;
+            if (!mmap_host_ptr_path && last_bufs.size() == 1) {
+                ggml_backend_buffer_t b = last_bufs[0].get();
+                if (b != nullptr && !ggml_backend_buffer_is_host(b)) {
+                    slot.releasable = true;
+                    slot.buft = buft;
+                }
+            }
+            pimpl->weight_release.push_back(std::move(slot));
+        }
+
         ctx_buf_maps.emplace_back(ctx, buf_map);
     }
 
@@ -1668,6 +1695,97 @@ ggml_tensor * llama_model_base::create_tensor(llama_model_loader & ml, const LLM
         tn, ne, flags);
 }
 
+void llama_model::release_device_weights() const {
+    // NOTE: the caller is responsible for synchronizing the backend scheduler first, so that no
+    // compute is in flight referencing these buffers when they are freed.
+    size_t freed = 0;
+    for (size_t i = 0; i < pimpl->ctxs_bufs.size(); ++i) {
+        auto & slot = pimpl->weight_release[i];
+        if (!slot.releasable || !slot.resident) {
+            continue;
+        }
+        ggml_context * ctx = pimpl->ctxs_bufs[i].first.get();
+        ggml_backend_buffer_t buf = pimpl->ctxs_bufs[i].second[0].get();
+        const size_t sz = ggml_backend_buffer_get_size(buf);
+
+        // lazily capture a host shadow of the weights (read-only, so captured only once).
+        // Store tensor bytes compactly in stable iteration order (independent of buffer layout /
+        // alignment padding) so restore does not depend on the re-allocated offsets matching.
+        // View tensors alias their base, so they are skipped (restored implicitly via their base).
+        if (slot.shadow.empty()) {
+            size_t total = 0;
+            for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != nullptr; t = ggml_get_next_tensor(ctx, t)) {
+                if (t->view_src != nullptr) continue;
+                total += ggml_nbytes(t);
+            }
+            slot.shadow.resize(total);
+            size_t off = 0;
+            for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != nullptr; t = ggml_get_next_tensor(ctx, t)) {
+                if (t->view_src != nullptr) continue;
+                const size_t n = ggml_nbytes(t);
+                ggml_backend_tensor_get(t, slot.shadow.data() + off, 0, n);
+                off += n;
+            }
+        }
+
+        // free the device (VRAM) buffer and clear the now-dangling tensor pointers so that
+        // ggml_backend_alloc_ctx_tensors_from_buft reallocates them cleanly on restore
+        pimpl->ctxs_bufs[i].second[0].reset();
+        for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != nullptr; t = ggml_get_next_tensor(ctx, t)) {
+            t->buffer = nullptr;
+            t->data   = nullptr;
+        }
+        slot.resident = false;
+        freed += sz;
+    }
+    if (freed > 0) {
+        LLAMA_LOG_INFO("%s: released %.2f MiB of device weights\n", __func__, freed / 1024.0 / 1024.0);
+    }
+}
+
+bool llama_model::restore_device_weights() const {
+    size_t restored = 0;
+    for (size_t i = 0; i < pimpl->ctxs_bufs.size(); ++i) {
+        auto & slot = pimpl->weight_release[i];
+        if (!slot.releasable || slot.resident) {
+            continue;
+        }
+        ggml_context * ctx = pimpl->ctxs_bufs[i].first.get();
+        ggml_backend_buffer_t buf = ggml_backend_alloc_ctx_tensors_from_buft(ctx, slot.buft);
+        if (buf == nullptr) {
+            LLAMA_LOG_ERROR("%s: failed to reallocate device weight buffer (out of VRAM?)\n", __func__);
+            return false;
+        }
+        ggml_backend_buffer_set_usage(buf, GGML_BACKEND_BUFFER_USAGE_WEIGHTS);
+
+        // re-upload weights from the compact host shadow, using the same stable iteration order
+        // and view-skipping as the capture above
+        size_t off = 0;
+        for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != nullptr; t = ggml_get_next_tensor(ctx, t)) {
+            if (t->view_src != nullptr) continue;
+            const size_t n = ggml_nbytes(t);
+            ggml_backend_tensor_set(t, slot.shadow.data() + off, 0, n);
+            off += n;
+        }
+        pimpl->ctxs_bufs[i].second[0].reset(buf);
+        slot.resident = true;
+        restored += ggml_backend_buffer_get_size(buf);
+    }
+    if (restored > 0) {
+        LLAMA_LOG_INFO("%s: restored %.2f MiB of device weights\n", __func__, restored / 1024.0 / 1024.0);
+    }
+    return true;
+}
+
+bool llama_model::weights_resident() const {
+    for (const auto & slot : pimpl->weight_release) {
+        if (slot.releasable && !slot.resident) {
+            return false;
+        }
+    }
+    return true;
+}
+
 std::string llama_model::arch_name() const {
     return llm_arch_name(arch);
 }