diff --git a/tools/server/server-context.cpp b/tools/server/server-context.cpp
index deebbfa6a..a6a3389e5 100644
--- a/tools/server/server-context.cpp
+++ b/tools/server/server-context.cpp
@@ -4199,8 +4199,12 @@ struct server_res_generator : server_res_spipe {
     server_response_reader rd;
     server_res_generator(server_queue & queue_tasks, server_response & queue_results, int sleep_idle_seconds, bool bypass_sleep = false)
             : rd(queue_tasks, queue_results, HTTP_POLLING_SECONDS) {
-        // fast path in case sleeping is disabled
-        bypass_sleep |= sleep_idle_seconds < 0;
+        // fast path in case sleeping is disabled. Note: the VRAM arbiter (LLAMA_SLEEP_VRAM_ONLY)
+        // can put the server to sleep via the cross-process doorbell even when idle-sleep is
+        // disabled (sleep_idle_seconds < 0), so in that case requests must still wake it - otherwise
+        // a doorbell-slept server would hang, never returning from a request.
+        static const bool vram_arbiter = getenv("LLAMA_SLEEP_VRAM_ONLY") != nullptr;
+        bypass_sleep |= (sleep_idle_seconds < 0 && !vram_arbiter);
         if (!bypass_sleep) {
             queue_tasks.wait_until_no_sleep();
         }