diff --git a/src/llama-context.cpp b/src/llama-context.cpp index 16e72cda8..fea2c4380 100644 --- a/src/llama-context.cpp +++ b/src/llama-context.cpp @@ -742,6 +742,11 @@ void llama_context::release_device(bool evict_kv) { // model holds essentially no VRAM. Rebuilt lazily by sched_reserve() on restore. sched.reset(); sched_need_reserve = true; + // finally, free each backend's own compute-scratch (Vulkan prealloc/staging buffers, which + // are owned by the backend and survive sched.reset()). Reallocated lazily on next compute. + for (auto & backend : backends) { + ggml_backend_free_scratch(backend.get()); + } } } @@ -3265,17 +3270,21 @@ llama_memory_breakdown llama_context::memory_breakdown() const { ret[buft].context += size; } } - if (model.hparams.no_alloc) { - for (size_t i = 0; i < backends.size(); ++i) { - ggml_backend_t backend = backends[i].get(); - ggml_backend_buffer_type_t buft = ggml_backend_sched_get_buffer_type(sched.get(), backend); - ret[buft].compute += backend_buf_exp_size[i]; - } - } else { - for (const auto & backend_ptr : backends) { - ggml_backend_t backend = backend_ptr.get(); - ggml_backend_buffer_type_t buft = ggml_backend_sched_get_buffer_type(sched.get(), backend); - ret[buft].compute += ggml_backend_sched_get_buffer_size(sched.get(), backend); + // the scheduler (and its compute buffers) may have been freed while the model is cold + // (on-demand VRAM eviction, see release_device); it contributes no compute VRAM then. + if (sched) { + if (model.hparams.no_alloc) { + for (size_t i = 0; i < backends.size(); ++i) { + ggml_backend_t backend = backends[i].get(); + ggml_backend_buffer_type_t buft = ggml_backend_sched_get_buffer_type(sched.get(), backend); + ret[buft].compute += backend_buf_exp_size[i]; + } + } else { + for (const auto & backend_ptr : backends) { + ggml_backend_t backend = backend_ptr.get(); + ggml_backend_buffer_type_t buft = ggml_backend_sched_get_buffer_type(sched.get(), backend); + ret[buft].compute += ggml_backend_sched_get_buffer_size(sched.get(), backend); + } } } return ret;