From 64543a435b092ec170ef843f889d486f51a49996 Mon Sep 17 00:00:00 2001 From: Lumpiasty Date: Thu, 22 Oct 2026 14:26:30 +0200 Subject: [PATCH] server: run the shutdown handler off the signal-interrupted thread A SIGTERM that lands while the start_loop() thread is inside a sleeping callback deadlocks the server. The callbacks run under server_queue::mutex_tasks, and the kernel delivers the signal to that same thread; the handler calls terminate(), which locks mutex_tasks again, and the thread waits on itself forever. With the VRAM arbiter that window is wide - a whole device release, or a blocking wait for the GPU token - and llama-swap sends SIGTERM to evict models, so it hit production: an evicted model wedged mid-release, never released token.lock, and every other model queued behind it. SIGTERM could not stop it either; only SIGKILL did. Make the handler only write the signal number to a self-pipe, which is async-signal-safe, and run shutdown_handler on a dedicated thread. terminate() there simply waits until the loop thread drops mutex_tasks, after which the loop exits normally. The thread is joined before the objects the handler uses are destroyed. If pipe() fails the old direct call is kept. Reproduced 6/6 off production (SIGTERM during release and during the token wait); with the fix 8/8 clean exits, and handoffs, image requests and greedy output are unchanged. Co-Authored-By: Claude Opus 5.5 --- tools/server/server.cpp | 55 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 55 insertions(+) diff --git a/tools/server/server.cpp b/tools/server/server.cpp index 6e2d8ff9d..986eff7f5 100644 --- a/tools/server/server.cpp +++ b/tools/server/server.cpp @@ -13,6 +13,7 @@ #include "log.h" #include +#include #include #include #include @@ -20,11 +21,22 @@ #if defined(_WIN32) #include +#else +#include +#include #endif static std::function shutdown_handler; static std::atomic_flag is_terminating = ATOMIC_FLAG_INIT; +#if !defined(_WIN32) +// The handler can interrupt the start_loop() thread while it holds server_queue::mutex_tasks (e.g. +// in the sleeping callbacks, during a VRAM arbiter release or token wait), so it must not run +// shutdown_handler itself: terminate() takes that mutex and the thread would deadlock on itself. +// It only writes the signal number here; shutdown_handler runs on a dedicated thread. +static int shutdown_pipe[2] = { -1, -1 }; +#endif + static inline void signal_handler(int signal) { if (is_terminating.test_and_set()) { // in case it hangs, we can force terminate the server by hitting Ctrl+C twice @@ -33,9 +45,48 @@ static inline void signal_handler(int signal) { exit(1); } +#if !defined(_WIN32) + if (shutdown_pipe[1] >= 0) { + const char sig = (char) signal; + const ssize_t res = write(shutdown_pipe[1], &sig, 1); // async-signal-safe + (void) res; + return; + } +#endif shutdown_handler(signal); } +#if !defined(_WIN32) +// runs shutdown_handler for signal_handler; joined (after a 0 byte) when llama_server() returns +struct shutdown_thread { + std::thread thread; + + void start() { + if (pipe(shutdown_pipe) != 0) { + return; // fall back to calling shutdown_handler from the signal handler + } + fcntl(shutdown_pipe[0], F_SETFD, FD_CLOEXEC); // not inherited by spawned tools / MCP servers + fcntl(shutdown_pipe[1], F_SETFD, FD_CLOEXEC); + thread = std::thread([]() { + char sig = 0; + while (read(shutdown_pipe[0], &sig, 1) < 0 && errno == EINTR) {} + if (sig != 0) { + shutdown_handler(sig); + } + }); + } + + ~shutdown_thread() { + if (thread.joinable()) { + const char sig = 0; + const ssize_t res = write(shutdown_pipe[1], &sig, 1); + (void) res; + thread.join(); + } + } +}; +#endif + // satisfies -Wmissing-declarations (used by llama command) int llama_server(int argc, char ** argv); @@ -505,8 +556,12 @@ int llama_server(common_params & params, int argc, char ** argv, server_child & } // register signal handler if not running by CLI +#if !defined(_WIN32) + shutdown_thread sig_thread; // declared after ctx_server & co, so joined before they are destroyed +#endif if (!is_run_by_cli) { #if defined (__unix__) || (defined (__APPLE__) && defined (__MACH__)) + sig_thread.start(); struct sigaction sigint_action; sigint_action.sa_handler = signal_handler; sigemptyset (&sigint_action.sa_mask);