server: re-ring the VRAM doorbell until the token is granted

The doorbell is edge-triggered and only honoured by a process that is already
warm and already listening, so a ring is silently lost in three windows: while
the holder is still loading its model (the inotify watch does not exist yet, and
the kernel does not queue events for a watch that is not there), while it is
inside restore_device (vram_cold is still set, so its own warden discards the
ring), and when a cold waiter's warden consumes a third process's ring and then
wins the flock. Nothing re-sends it, flock(LOCK_EX) never times out, and with no
--sleep-idle-seconds the holder has no other reason to release - so a lost ring
wedges the waiter permanently. It blocks under mutex_tasks, so the whole server
stops answering while /health still returns 200.

Take the token through one helper that retries flock with LOCK_NB and re-rings
about once a second, warning every five. Also retry on EINTR: the return value
was previously discarded and the signal handlers run without SA_RESTART, so a
signal made the caller restore weights believing it held the token - two models
uploading into 8 GB, the OOM the arbiter exists to prevent.

Make vram_cold mean "does not hold the token" rather than "weights are not
resident", so the warden stops dropping rings during the restore, and unlink our
own doorbell file on shutdown.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01PZz44SLQvTXMyWGio6t9DZ

tools/server/server-context.cpp

@@ -46,6 +46,7 @@
 #include <fcntl.h>
 #include <unistd.h>
 #include <poll.h>
+#include <cerrno>
 #endif
 
 constexpr int HTTP_POLLING_SECONDS = 1;
@@ -988,12 +989,9 @@ private:
         if (!vram_only) {
             return;
         }
-#if !defined(_WIN32)
-        if (vram_flock) {
-            vram_ring_doorbell();         // nudge whoever is resident to release
-            flock(vram_lock_fd, LOCK_EX); // block until the GPU is free
+        if (!vram_take_token()) {
+            SRV_WRN("%s", "VRAM arbiter: loading weights without the GPU token\n");
         }
-#endif
         vram_cold = false; // we hold the token; weights will be resident after the upload
     }
 
@@ -1068,18 +1066,53 @@ private:
 #endif
     }
 
+    // take the shared VRAM token: ring the doorbell, then block on the flock. The doorbell is
+    // edge-triggered and only heard by a process that is already warm and listening, so a ring sent
+    // while the holder still loads or restores is lost - retry the lock and re-ring about once a
+    // second until we get it. No timeout: the holder can be mid-generation, and decoding without
+    // the token risks an OOM that kills both processes.
+    bool vram_take_token() {
+#if !defined(_WIN32)
+        if (!vram_flock) {
+            return true; // no cross-process lock: there is no token to take
+        }
+        vram_ring_doorbell();
+        for (int tries = 1; ; tries++) {
+            if (flock(vram_lock_fd, LOCK_EX | LOCK_NB) == 0) {
+                return true;
+            }
+            if (errno == EINTR) {
+                continue; // our signal handlers run without SA_RESTART
+            }
+            if (errno != EWOULDBLOCK && errno != EAGAIN) {
+                SRV_WRN("VRAM arbiter: flock failed (errno %d)\n", errno);
+                return false;
+            }
+            poll(nullptr, 0, 200);
+            if (tries % 5 == 0) {
+                vram_ring_doorbell(); // the holder may have missed the ring while it was not listening
+            }
+            if (tries % 25 == 0) {
+                SRV_WRN("VRAM arbiter: still waiting for the GPU token after %d s\n", tries / 5);
+            }
+        }
+#else
+        return true;
+#endif
+    }
+
     // acquire the VRAM token (blocking) and bring weights back to the device. Called on the loop
     // thread right before a decode, so it never races compute.
     void vram_ensure_warm() {
         if (!vram_only || !vram_cold) {
             return;
         }
-#if !defined(_WIN32)
-        if (vram_flock) {
-            vram_ring_doorbell();         // nudge the current holder to release
-            flock(vram_lock_fd, LOCK_EX); // blocks until the current holder goes cold
+        if (!vram_take_token()) {
+            SRV_WRN("%s", "VRAM arbiter: restoring weights without the GPU token\n");
         }
-#endif
+        // cold means "does not hold the token", so clear it before the slow restore - while it is
+        // set our own warden drops foreign rings, and a dropped ring is never re-sent
+        vram_cold = false;
         llama_context_restore_device(ctx_tgt);
         if (ctx_dft != nullptr) {
             llama_context_restore_device(ctx_dft);
@@ -1089,7 +1122,6 @@ private:
         // on-demand mode, released again right after - so a warm text-only model holds no encoder
         // VRAM (that space is free for KV / expert cache). A cold->warm wake for a *text* request
         // therefore leaves the encoder in RAM; an image request restores it at encode time.
-        vram_cold = false;
     }
 
     void vram_share_shutdown() {
@@ -1098,6 +1130,10 @@ private:
         if (vram_warden.joinable()) {
             vram_warden.join();
         }
+        // drop our own ring only - a foreign file can still be a live pending request
+        if (!vram_doorbell_dir.empty() && !vram_pid_str.empty()) {
+            unlink((vram_doorbell_dir + "/" + vram_pid_str).c_str());
+        }
         if (vram_inotify_fd >= 0) { close(vram_inotify_fd); vram_inotify_fd = -1; }
         if (vram_lock_fd    >= 0) { flock(vram_lock_fd, LOCK_UN); close(vram_lock_fd); vram_lock_fd = -1; }
 #endif
@@ -1118,12 +1154,12 @@ private:
         if (mctx != nullptr) {
             mtmd_release_device(mctx);
         }
+        vram_cold = true; // drop the token flag first: a waiter can lock the moment we unlock
 #if !defined(_WIN32)
         if (vram_flock) {
             flock(vram_lock_fd, LOCK_UN);
         }
 #endif
-        vram_cold = true;
     }
 
     common_speculative_init_result_ptr spec_init;