diff --git a/src/llama-context.cpp b/src/llama-context.cpp
index f4c094271..94999da78 100644
--- a/src/llama-context.cpp
+++ b/src/llama-context.cpp
@@ -483,7 +483,8 @@ llama_context::~llama_context() {
     synchronize();
 
     // when training, ggml_opt allocates extra buffers through the scheduler, so the sizes no longer match the expectation
-    if (!model.hparams.no_alloc && !opt_ctx) {
+    // the scheduler is also gone if the context is destroyed while cold (on-demand VRAM eviction, see release_device)
+    if (sched && !model.hparams.no_alloc && !opt_ctx) {
         for (size_t i = 0; i < backend_ptrs.size(); ++i) {
             ggml_backend_t             backend = backend_ptrs[i];
             ggml_backend_buffer_type_t buft    = backend_buft[i];