diff --git a/src/llama-kv-cache.cpp b/src/llama-kv-cache.cpp index 8ca118366..a0cbb0767 100644 --- a/src/llama-kv-cache.cpp +++ b/src/llama-kv-cache.cpp @@ -371,7 +371,9 @@ void llama_kv_cache::clear(bool data) { if (data) { for (auto & [_, buf] : ctxs_bufs) { - ggml_backend_buffer_clear(buf.get(), 0); + if (buf) { // may be null if evicted for on-demand VRAM sharing + ggml_backend_buffer_clear(buf.get(), 0); + } } } } @@ -681,6 +683,9 @@ llama_pos llama_kv_cache::seq_pos_max(llama_seq_id seq_id) const { std::map llama_kv_cache::memory_breakdown() const { std::map ret; for (const auto & [ctx, buf] : ctxs_bufs) { + if (!buf) { // may be null if evicted for on-demand VRAM sharing + continue; + } ggml_backend_buffer_type_t buft = ggml_backend_buffer_get_type(buf.get()); if (hparams.no_alloc) { @@ -1801,7 +1806,9 @@ size_t llama_kv_cache::total_size() const { size_t size = 0; for (const auto & [_, buf] : ctxs_bufs) { - size += ggml_backend_buffer_get_size(buf.get()); + if (buf) { // may be null if evicted for on-demand VRAM sharing + size += ggml_backend_buffer_get_size(buf.get()); + } } return size; diff --git a/src/llama-model.cpp b/src/llama-model.cpp index 962d17b37..f643a8087 100644 --- a/src/llama-model.cpp +++ b/src/llama-model.cpp @@ -1832,6 +1832,11 @@ std::map llama_model::memory_breakdown() con ret[buft] += ggml_backend_alloc_ctx_tensors_from_buft_size(ctx.get(), buft); } else { for (const auto & buf : bufs) { + // buf may be null if the device weights were released for on-demand VRAM sharing + // (see release_device_weights); skip it in the breakdown + if (!buf) { + continue; + } // GGML_ASSERT(ggml_backend_buffer_get_base(buf.get()) != nullptr); // multi_buffer does not have a defined base ret[ggml_backend_buffer_get_type(buf.get())] += ggml_backend_buffer_get_size(buf.get()); }