From 3b068bdb3a1e1853910d15bf1e4ea5e1e69caf8f Mon Sep 17 00:00:00 2001 From: Lumpiasty Date: Thu, 22 Oct 2026 14:26:30 +0200 Subject: [PATCH] server: run the shutdown handler off the signal-interrupted thread A SIGTERM that lands while the start_loop() thread is inside a sleeping callback deadlocks the server. The callbacks run under server_queue::mutex_tasks, and the kernel delivers the signal to that same thread; the handler calls terminate(), which locks mutex_tasks again, and the thread waits on itself forever. With the VRAM arbiter that window is wide - a whole device release, or a blocking wait for the GPU token - and llama-swap sends SIGTERM to evict models, so it hit production: an evicted model wedged mid-release, never released token.lock, and every other model queued behind it. SIGTERM could not stop it either; only SIGKILL did. Make the handler only write the signal number to a self-pipe, which is async-signal-safe, and run shutdown_handler on a dedicated thread. terminate() there simply waits until the loop thread drops mutex_tasks, after which the loop exits normally. The thread is joined before the objects the handler uses are destroyed. If pipe() fails the old direct call is kept. Reproduced 6/6 off production (SIGTERM during release and during the token wait); with the fix 8/8 clean exits, and handoffs, image requests and greedy output are unchanged. Co-Authored-By: Claude Opus 5.5 --- tools/server/server.cpp | 57 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 57 insertions(+) diff --git a/tools/server/server.cpp b/tools/server/server.cpp index fabf0a041..1dd23db78 100644 --- a/tools/server/server.cpp +++ b/tools/server/server.cpp @@ -15,6 +15,7 @@ #include #include +#include #include #include #include @@ -24,11 +25,22 @@ #if defined(_WIN32) #include +#else +#include +#include #endif static std::function shutdown_handler; static std::atomic_flag is_terminating = ATOMIC_FLAG_INIT; +#if !defined(_WIN32) +// The handler can interrupt the start_loop() thread while it holds server_queue::mutex_tasks (e.g. +// in the sleeping callbacks, during a VRAM arbiter release or token wait), so it must not run +// shutdown_handler itself: terminate() takes that mutex and the thread would deadlock on itself. +// It only writes the signal number here; shutdown_handler runs on a dedicated thread. +static int shutdown_pipe[2] = { -1, -1 }; +#endif + static inline void signal_handler(int signal) { if (is_terminating.test_and_set()) { // in case it hangs, we can force terminate the server by hitting Ctrl+C twice @@ -37,9 +49,48 @@ static inline void signal_handler(int signal) { exit(1); } +#if !defined(_WIN32) + if (shutdown_pipe[1] >= 0) { + const char sig = (char) signal; + const ssize_t res = write(shutdown_pipe[1], &sig, 1); // async-signal-safe + (void) res; + return; + } +#endif shutdown_handler(signal); } +#if !defined(_WIN32) +// runs shutdown_handler for signal_handler; joined (after a 0 byte) when the instance returns +struct shutdown_thread { + std::thread thread; + + void start() { + if (pipe(shutdown_pipe) != 0) { + return; // fall back to calling shutdown_handler from the signal handler + } + fcntl(shutdown_pipe[0], F_SETFD, FD_CLOEXEC); // not inherited by spawned tools / MCP servers + fcntl(shutdown_pipe[1], F_SETFD, FD_CLOEXEC); + thread = std::thread([]() { + char sig = 0; + while (read(shutdown_pipe[0], &sig, 1) < 0 && errno == EINTR) {} + if (sig != 0) { + shutdown_handler(sig); + } + }); + } + + ~shutdown_thread() { + if (thread.joinable()) { + const char sig = 0; + const ssize_t res = write(shutdown_pipe[1], &sig, 1); + (void) res; + thread.join(); + } + } +}; +#endif + // how the server is run; everything is on when it is the llama-server binary struct server_run_opts { bool install_signal_handlers = true; // off when embedded in llama-cli, which handles them itself @@ -715,7 +766,13 @@ static int server_run_instance(common_params & params, server_role & role, commo ctx_server.terminate(); }; +#if !defined(_WIN32) + shutdown_thread sig_thread; // declared after ctx_server & co, so joined before they are destroyed +#endif if (opts.install_signal_handlers) { +#if !defined(_WIN32) + sig_thread.start(); +#endif server_install_signal_handlers(); }