server: also free the compute-graph scheduler on KV eviction
Under LLAMA_SLEEP_EVICT_KV a cold model still held the scheduler's worst-case compute buffer (hundreds of MiB, e.g. ~700 MiB for gemma-26B) plus the resident experts. release_device(evict_kv) now also frees the sched (sched.reset() + sched_need_reserve), and restore_device() rebuilds it via sched_reserve(), so a fully-evicted model holds essentially no VRAM (gemma cold: 7147 -> 51 MiB). Trades a heavier re-warm (weights+KV H2D + sched reserve) for the freed VRAM; weights-only mode is unchanged (fast switch, keeps KV+compute). Assisted-by: Claude
This commit is contained in:
@@ -738,11 +738,15 @@ void llama_context::release_device(bool evict_kv) {
|
||||
if (evict_kv && memory && !kv_device_evicted) {
|
||||
memory->release_device_buffers();
|
||||
kv_device_evicted = true;
|
||||
// also free the scheduler and its worst-case compute buffer (hundreds of MiB) so a cold
|
||||
// model holds essentially no VRAM. Rebuilt lazily by sched_reserve() on restore.
|
||||
sched.reset();
|
||||
sched_need_reserve = true;
|
||||
}
|
||||
}
|
||||
|
||||
void llama_context::restore_device() {
|
||||
if (model.weights_resident() && !kv_device_evicted) {
|
||||
if (model.weights_resident() && !kv_device_evicted && sched) {
|
||||
return;
|
||||
}
|
||||
model.restore_device_weights();
|
||||
@@ -750,6 +754,10 @@ void llama_context::restore_device() {
|
||||
memory->restore_device_buffers();
|
||||
kv_device_evicted = false;
|
||||
}
|
||||
// rebuild the scheduler + compute buffer if they were freed on release (evict_kv mode)
|
||||
if (!sched) {
|
||||
sched_reserve();
|
||||
}
|
||||
// make sure all weight/KV uploads have completed before any compute reads them
|
||||
if (sched) {
|
||||
ggml_backend_sched_synchronize(sched.get());
|
||||
|
||||
Reference in New Issue
Block a user