diff --git a/src/llama-context.cpp b/src/llama-context.cpp index eb47e3516..fc4253422 100644 --- a/src/llama-context.cpp +++ b/src/llama-context.cpp @@ -928,11 +928,15 @@ void llama_context::release_device(bool evict_kv) { if (evict_kv && memory && !kv_device_evicted) { memory->release_device_buffers(); kv_device_evicted = true; + // also free the scheduler and its worst-case compute buffer (hundreds of MiB) so a cold + // model holds essentially no VRAM. Rebuilt lazily by sched_reserve() on restore. + sched.reset(); + sched_need_reserve = true; } } void llama_context::restore_device() { - if (model.weights_resident() && !kv_device_evicted) { + if (model.weights_resident() && !kv_device_evicted && sched) { return; } model.restore_device_weights(); @@ -940,6 +944,10 @@ void llama_context::restore_device() { memory->restore_device_buffers(); kv_device_evicted = false; } + // rebuild the scheduler + compute buffer if they were freed on release (evict_kv mode) + if (!sched) { + sched_reserve(); + } // make sure all weight/KV uploads have completed before any compute reads them if (sched) { ggml_backend_sched_synchronize(sched.get());