diff --git a/src/llama-kv-cache.cpp b/src/llama-kv-cache.cpp index a24a9cd2e..c730a37e0 100644 --- a/src/llama-kv-cache.cpp +++ b/src/llama-kv-cache.cpp @@ -398,7 +398,9 @@ void llama_kv_cache::clear(bool data) { if (data) { for (auto & [_, buf] : ctxs_bufs) { - ggml_backend_buffer_clear(buf.get(), 0); + if (buf) { // may be null if evicted for on-demand VRAM sharing + ggml_backend_buffer_clear(buf.get(), 0); + } } } } @@ -741,6 +743,9 @@ llama_pos llama_kv_cache::seq_pos_max(llama_seq_id seq_id) const { std::map llama_kv_cache::memory_breakdown() const { std::map ret; for (const auto & [ctx, buf] : ctxs_bufs) { + if (!buf) { // may be null if evicted for on-demand VRAM sharing + continue; + } ggml_backend_buffer_type_t buft = ggml_backend_buffer_get_type(buf.get()); if (hparams.no_alloc) { @@ -1953,7 +1958,9 @@ size_t llama_kv_cache::total_size() const { size_t size = 0; for (const auto & [_, buf] : ctxs_bufs) { - size += ggml_backend_buffer_get_size(buf.get()); + if (buf) { // may be null if evicted for on-demand VRAM sharing + size += ggml_backend_buffer_get_size(buf.get()); + } } return size; diff --git a/src/llama-model.cpp b/src/llama-model.cpp index d2bf978c5..cbb756cd4 100644 --- a/src/llama-model.cpp +++ b/src/llama-model.cpp @@ -1838,6 +1838,11 @@ std::map llama_model::memory_breakdown() con ret[buft] += ggml_backend_alloc_ctx_tensors_from_buft_size(ctx.get(), buft); } else { for (const auto & buf : bufs) { + // buf may be null if the device weights were released for on-demand VRAM sharing + // (see release_device_weights); skip it in the breakdown + if (!buf) { + continue; + } // GGML_ASSERT(ggml_backend_buffer_get_base(buf.get()) != nullptr); // multi_buffer does not have a defined base ret[ggml_backend_buffer_get_type(buf.get())] += ggml_backend_buffer_get_size(buf.get()); }