server: run the shutdown handler off the signal-interrupted thread
A SIGTERM that lands while the start_loop() thread is inside a sleeping callback deadlocks the server. The callbacks run under server_queue::mutex_tasks, and the kernel delivers the signal to that same thread; the handler calls terminate(), which locks mutex_tasks again, and the thread waits on itself forever. With the VRAM arbiter that window is wide - a whole device release, or a blocking wait for the GPU token - and llama-swap sends SIGTERM to evict models, so it hit production: an evicted model wedged mid-release, never released token.lock, and every other model queued behind it. SIGTERM could not stop it either; only SIGKILL did. Make the handler only write the signal number to a self-pipe, which is async-signal-safe, and run shutdown_handler on a dedicated thread. terminate() there simply waits until the loop thread drops mutex_tasks, after which the loop exits normally. The thread is joined before the objects the handler uses are destroyed. If pipe() fails the old direct call is kept. Reproduced 6/6 off production (SIGTERM during release and during the token wait); with the fix 8/8 clean exits, and handoffs, image requests and greedy output are unchanged. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
ea144c8a9e
commit
33f8b89f9b
1 file changed
+57
@@ -15,6 +15,7 @@
|
||||
|
||||
#include <algorithm>
|
||||
#include <atomic>
|
||||
#include <cerrno>
|
||||
#include <clocale>
|
||||
#include <exception>
|
||||
#include <mutex>
|
||||
@@ -24,11 +25,22 @@
|
||||
|
||||
#if defined(_WIN32)
|
||||
#include <windows.h>
|
||||
#else
|
||||
#include <fcntl.h>
|
||||
#include <unistd.h>
|
||||
#endif
|
||||
|
||||
static std::function<void(int)> shutdown_handler;
|
||||
static std::atomic_flag is_terminating = ATOMIC_FLAG_INIT;
|
||||
|
||||
#if !defined(_WIN32)
|
||||
// The handler can interrupt the start_loop() thread while it holds server_queue::mutex_tasks (e.g.
|
||||
// in the sleeping callbacks, during a VRAM arbiter release or token wait), so it must not run
|
||||
// shutdown_handler itself: terminate() takes that mutex and the thread would deadlock on itself.
|
||||
// It only writes the signal number here; shutdown_handler runs on a dedicated thread.
|
||||
static int shutdown_pipe[2] = { -1, -1 };
|
||||
#endif
|
||||
|
||||
static inline void signal_handler(int signal) {
|
||||
if (is_terminating.test_and_set()) {
|
||||
// in case it hangs, we can force terminate the server by hitting Ctrl+C twice
|
||||
@@ -37,9 +49,48 @@ static inline void signal_handler(int signal) {
|
||||
exit(1);
|
||||
}
|
||||
|
||||
#if !defined(_WIN32)
|
||||
if (shutdown_pipe[1] >= 0) {
|
||||
const char sig = (char) signal;
|
||||
const ssize_t res = write(shutdown_pipe[1], &sig, 1); // async-signal-safe
|
||||
(void) res;
|
||||
return;
|
||||
}
|
||||
#endif
|
||||
shutdown_handler(signal);
|
||||
}
|
||||
|
||||
#if !defined(_WIN32)
|
||||
// runs shutdown_handler for signal_handler; joined (after a 0 byte) when the instance returns
|
||||
struct shutdown_thread {
|
||||
std::thread thread;
|
||||
|
||||
void start() {
|
||||
if (pipe(shutdown_pipe) != 0) {
|
||||
return; // fall back to calling shutdown_handler from the signal handler
|
||||
}
|
||||
fcntl(shutdown_pipe[0], F_SETFD, FD_CLOEXEC); // not inherited by spawned tools / MCP servers
|
||||
fcntl(shutdown_pipe[1], F_SETFD, FD_CLOEXEC);
|
||||
thread = std::thread([]() {
|
||||
char sig = 0;
|
||||
while (read(shutdown_pipe[0], &sig, 1) < 0 && errno == EINTR) {}
|
||||
if (sig != 0) {
|
||||
shutdown_handler(sig);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
~shutdown_thread() {
|
||||
if (thread.joinable()) {
|
||||
const char sig = 0;
|
||||
const ssize_t res = write(shutdown_pipe[1], &sig, 1);
|
||||
(void) res;
|
||||
thread.join();
|
||||
}
|
||||
}
|
||||
};
|
||||
#endif
|
||||
|
||||
// how the server is run; everything is on when it is the llama-server binary
|
||||
struct server_run_opts {
|
||||
bool install_signal_handlers = true; // off when embedded in llama-cli, which handles them itself
|
||||
@@ -776,7 +827,13 @@ static int server_run_instance(common_params & params, server_role & role, commo
|
||||
ctx_server.terminate();
|
||||
};
|
||||
|
||||
#if !defined(_WIN32)
|
||||
shutdown_thread sig_thread; // declared after ctx_server & co, so joined before they are destroyed
|
||||
#endif
|
||||
if (opts.install_signal_handlers) {
|
||||
#if !defined(_WIN32)
|
||||
sig_thread.start();
|
||||
#endif
|
||||
server_install_signal_handlers();
|
||||
}
|
||||
|
||||
|
||||
Reference in new issue
Block a user