From f707a2430dbd55ae57512ae9bc4a43586734ffd3 Mon Sep 17 00:00:00 2001
From: Lumpiasty <arek.dzski@gmail.com>
Date: Sat, 25 Jul 2026 14:36:17 +0200
Subject: [PATCH] llama: guard buffer iteration against released device buffers

release_device_weights()/release_device_buffers() leave a null buffer in
ctxs_bufs while the device memory is released for on-demand VRAM sharing.
The memory-breakdown / total_size / clear paths iterated these and called
ggml_backend_buffer_get_size()/get_type()/clear() on the null buffer, tripping
GGML_ASSERT(buffer) and aborting on shutdown of a cold model. Skip null buffers
in llama_model::memory_breakdown and llama_kv_cache::{memory_breakdown,
total_size,clear}.

Assisted-by: Claude
---
 src/llama-kv-cache.cpp | 11 +++++++++--
 src/llama-model.cpp    |  5 +++++
 2 files changed, 14 insertions(+), 2 deletions(-)

diff --git a/src/llama-kv-cache.cpp b/src/llama-kv-cache.cpp
index a24a9cd2e..c730a37e0 100644
--- a/src/llama-kv-cache.cpp
+++ b/src/llama-kv-cache.cpp
@@ -398,7 +398,9 @@ void llama_kv_cache::clear(bool data) {
 
     if (data) {
         for (auto & [_, buf] : ctxs_bufs) {
-            ggml_backend_buffer_clear(buf.get(), 0);
+            if (buf) { // may be null if evicted for on-demand VRAM sharing
+                ggml_backend_buffer_clear(buf.get(), 0);
+            }
         }
     }
 }
@@ -741,6 +743,9 @@ llama_pos llama_kv_cache::seq_pos_max(llama_seq_id seq_id) const {
 std::map<ggml_backend_buffer_type_t, size_t> llama_kv_cache::memory_breakdown() const {
     std::map<ggml_backend_buffer_type_t, size_t> ret;
     for (const auto & [ctx, buf] : ctxs_bufs) {
+        if (!buf) { // may be null if evicted for on-demand VRAM sharing
+            continue;
+        }
         ggml_backend_buffer_type_t buft = ggml_backend_buffer_get_type(buf.get());
 
         if (hparams.no_alloc) {
@@ -1953,7 +1958,9 @@ size_t llama_kv_cache::total_size() const {
     size_t size = 0;
 
     for (const auto & [_, buf] : ctxs_bufs) {
-        size += ggml_backend_buffer_get_size(buf.get());
+        if (buf) { // may be null if evicted for on-demand VRAM sharing
+            size += ggml_backend_buffer_get_size(buf.get());
+        }
     }
 
     return size;
diff --git a/src/llama-model.cpp b/src/llama-model.cpp
index d2bf978c5..cbb756cd4 100644
--- a/src/llama-model.cpp
+++ b/src/llama-model.cpp
@@ -1838,6 +1838,11 @@ std::map<ggml_backend_buffer_type_t, size_t> llama_model::memory_breakdown() con
             ret[buft] += ggml_backend_alloc_ctx_tensors_from_buft_size(ctx.get(), buft);
         } else {
             for (const auto & buf : bufs) {
+                // buf may be null if the device weights were released for on-demand VRAM sharing
+                // (see release_device_weights); skip it in the breakdown
+                if (!buf) {
+                    continue;
+                }
                 // GGML_ASSERT(ggml_backend_buffer_get_base(buf.get()) != nullptr); // multi_buffer does not have a defined base
                 ret[ggml_backend_buffer_get_type(buf.get())] += ggml_backend_buffer_get_size(buf.get());
             }