server: run the shutdown handler off the signal-interrupted thread
A SIGTERM that lands while the start_loop() thread is inside a sleeping callback deadlocks the server. The callbacks run under server_queue::mutex_tasks, and the kernel delivers the signal to that same thread; the handler calls terminate(), which locks mutex_tasks again, and the thread waits on itself forever. With the VRAM arbiter that window is wide - a whole device release, or a blocking wait for the GPU token - and llama-swap sends SIGTERM to evict models, so it hit production: an evicted model wedged mid-release, never released token.lock, and every other model queued behind it. SIGTERM could not stop it either; only SIGKILL did. Make the handler only write the signal number to a self-pipe, which is async-signal-safe, and run shutdown_handler on a dedicated thread. terminate() there simply waits until the loop thread drops mutex_tasks, after which the loop exits normally. The thread is joined before the objects the handler uses are destroyed. If pipe() fails the old direct call is kept. Reproduced 6/6 off production (SIGTERM during release and during the token wait); with the fix 8/8 clean exits, and handoffs, image requests and greedy output are unchanged. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
54a7f534b6
commit
11044b35e3
1 file changed
+55
@@ -14,6 +14,7 @@
|
||||
#include "log.h"
|
||||
|
||||
#include <atomic>
|
||||
#include <cerrno>
|
||||
#include <clocale>
|
||||
#include <exception>
|
||||
#include <signal.h>
|
||||
@@ -21,11 +22,22 @@
|
||||
|
||||
#if defined(_WIN32)
|
||||
#include <windows.h>
|
||||
#else
|
||||
#include <fcntl.h>
|
||||
#include <unistd.h>
|
||||
#endif
|
||||
|
||||
static std::function<void(int)> shutdown_handler;
|
||||
static std::atomic_flag is_terminating = ATOMIC_FLAG_INIT;
|
||||
|
||||
#if !defined(_WIN32)
|
||||
// The handler can interrupt the start_loop() thread while it holds server_queue::mutex_tasks (e.g.
|
||||
// in the sleeping callbacks, during a VRAM arbiter release or token wait), so it must not run
|
||||
// shutdown_handler itself: terminate() takes that mutex and the thread would deadlock on itself.
|
||||
// It only writes the signal number here; shutdown_handler runs on a dedicated thread.
|
||||
static int shutdown_pipe[2] = { -1, -1 };
|
||||
#endif
|
||||
|
||||
static inline void signal_handler(int signal) {
|
||||
if (is_terminating.test_and_set()) {
|
||||
// in case it hangs, we can force terminate the server by hitting Ctrl+C twice
|
||||
@@ -34,9 +46,48 @@ static inline void signal_handler(int signal) {
|
||||
exit(1);
|
||||
}
|
||||
|
||||
#if !defined(_WIN32)
|
||||
if (shutdown_pipe[1] >= 0) {
|
||||
const char sig = (char) signal;
|
||||
const ssize_t res = write(shutdown_pipe[1], &sig, 1); // async-signal-safe
|
||||
(void) res;
|
||||
return;
|
||||
}
|
||||
#endif
|
||||
shutdown_handler(signal);
|
||||
}
|
||||
|
||||
#if !defined(_WIN32)
|
||||
// runs shutdown_handler for signal_handler; joined (after a 0 byte) when llama_server() returns
|
||||
struct shutdown_thread {
|
||||
std::thread thread;
|
||||
|
||||
void start() {
|
||||
if (pipe(shutdown_pipe) != 0) {
|
||||
return; // fall back to calling shutdown_handler from the signal handler
|
||||
}
|
||||
fcntl(shutdown_pipe[0], F_SETFD, FD_CLOEXEC); // not inherited by spawned tools / MCP servers
|
||||
fcntl(shutdown_pipe[1], F_SETFD, FD_CLOEXEC);
|
||||
thread = std::thread([]() {
|
||||
char sig = 0;
|
||||
while (read(shutdown_pipe[0], &sig, 1) < 0 && errno == EINTR) {}
|
||||
if (sig != 0) {
|
||||
shutdown_handler(sig);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
~shutdown_thread() {
|
||||
if (thread.joinable()) {
|
||||
const char sig = 0;
|
||||
const ssize_t res = write(shutdown_pipe[1], &sig, 1);
|
||||
(void) res;
|
||||
thread.join();
|
||||
}
|
||||
}
|
||||
};
|
||||
#endif
|
||||
|
||||
// satisfies -Wmissing-declarations (used by llama command)
|
||||
int llama_server(int argc, char ** argv);
|
||||
|
||||
@@ -528,8 +579,12 @@ int llama_server(common_params & params, int argc, char ** argv, server_child &
|
||||
}
|
||||
|
||||
// register signal handler if not running by CLI
|
||||
#if !defined(_WIN32)
|
||||
shutdown_thread sig_thread; // declared after ctx_server & co, so joined before they are destroyed
|
||||
#endif
|
||||
if (!is_run_by_cli) {
|
||||
#if defined (__unix__) || (defined (__APPLE__) && defined (__MACH__))
|
||||
sig_thread.start();
|
||||
struct sigaction sigint_action;
|
||||
sigint_action.sa_handler = signal_handler;
|
||||
sigemptyset (&sigint_action.sa_mask);
|
||||
|
||||
Reference in new issue
Block a user