log, server: self contained colors, split child commands from logs in router mode (#29895)
* log, server: make router child lines carry their own colors The logger writes the color reset after the trailing newline, so the reset opens the next line. On the shared pipe of a router child it lands in front of the next state command, which the router then misses, and the line break that works around it shows up as an empty log line on every progress update. The reset now goes before the trailing newlines, so every line is self contained and the command goes back to its plain framing. The router passes its effective color setting to its children, whose output ends up in its terminal, and leaves that option out when comparing presets on reload. * log: enable ANSI colors on the Windows console A Windows console renders ANSI sequences only in virtual terminal mode, which nothing turns on for the logger, so llama-server prints raw escape codes on the Windows 10 console while llama-cli, whose console code enables it, shows colors. The logger now enables virtual terminal mode on stdout and stderr when it turns colors on, and keeps colors off when a console cannot render them. Pipes and files take the sequences as is. * server: separate the router child commands from its logs The child sent its state commands on the same pipe as its logs, so the router had to pick them out of the log stream by a line prefix, and any unterminated write in front of a command made the router miss it. This resolves the TODO at the spawn that called for splitting stdout and stderr. The child now keeps stdout for the commands and points everything else written to stdout at stderr, before anything is written. The router reads both pipes, handles the commands from stdout and forwards stderr as the log, and warns about any other line on the command pipe. * server: address review from ngxson The single server_child is now created first in the entry point and its constructor keeps stdout for the commands, so the stream is a member of the instance instead of a static, and init() is gone. The instance is passed down to the server, while the CLI entry point creates its own. * Update tools/server/server.cpp --------- Co-authored-by: Xuan-Son Nguyen <thichthat@gmail.com>
This commit is contained in:
1 parent
0bb496dbd3
commit
a7fb71fab8
9 files changed
+196
-75
No files matched your search
@@ -1885,38 +1885,71 @@ server_tokens format_prompt_rerank(
|
||||
// server_subproc
|
||||
//
|
||||
|
||||
bool server_subproc::has_output() {
|
||||
if (out_handle >= 0) {
|
||||
return true;
|
||||
}
|
||||
FILE * f = sproc.stdout_file(); // combined stdout/stderr
|
||||
if (!f) {
|
||||
return false;
|
||||
}
|
||||
FILE * server_reserve_stdout() {
|
||||
fflush(stdout);
|
||||
// the reserved stream is not inherited, grandchildren get stdout and stderr of their own
|
||||
#ifdef _WIN32
|
||||
HANDLE h = (HANDLE) _get_osfhandle(_fileno(f));
|
||||
if (h != INVALID_HANDLE_VALUE) {
|
||||
out_handle = (intptr_t) h;
|
||||
}
|
||||
int fd = _dup(_fileno(stdout));
|
||||
GGML_ASSERT(fd >= 0);
|
||||
SetHandleInformation((HANDLE) _get_osfhandle(fd), HANDLE_FLAG_INHERIT, 0);
|
||||
_dup2(_fileno(stderr), _fileno(stdout));
|
||||
SetStdHandle(STD_OUTPUT_HANDLE, GetStdHandle(STD_ERROR_HANDLE));
|
||||
FILE * f = _fdopen(fd, "w");
|
||||
#else
|
||||
int fd = fileno(f);
|
||||
if (fd >= 0) {
|
||||
fcntl(fd, F_SETFL, fcntl(fd, F_GETFL, 0) | O_NONBLOCK);
|
||||
out_handle = fd;
|
||||
}
|
||||
int fd = fcntl(fileno(stdout), F_DUPFD_CLOEXEC, 0);
|
||||
GGML_ASSERT(fd >= 0);
|
||||
dup2(fileno(stderr), fileno(stdout));
|
||||
FILE * f = fdopen(fd, "w");
|
||||
#endif
|
||||
return out_handle >= 0;
|
||||
GGML_ASSERT(f);
|
||||
return f;
|
||||
}
|
||||
|
||||
int server_subproc::read_output(char * buf, size_t len) {
|
||||
bool server_subproc::has_output() {
|
||||
FILE * files[SERVER_SUBPROC_STREAMS] = { sproc.stdout_file(), sproc.stderr_file() };
|
||||
for (int i = 0; i < SERVER_SUBPROC_STREAMS; i++) {
|
||||
if (out_handles[i] >= 0) {
|
||||
continue;
|
||||
}
|
||||
if (!files[i]) {
|
||||
return false;
|
||||
}
|
||||
#ifdef _WIN32
|
||||
HANDLE h = (HANDLE) _get_osfhandle(_fileno(files[i]));
|
||||
if (h == INVALID_HANDLE_VALUE) {
|
||||
return false;
|
||||
}
|
||||
out_handles[i] = (intptr_t) h;
|
||||
#else
|
||||
int fd = fileno(files[i]);
|
||||
if (fd < 0) {
|
||||
return false;
|
||||
}
|
||||
fcntl(fd, F_SETFL, fcntl(fd, F_GETFL, 0) | O_NONBLOCK);
|
||||
out_handles[i] = fd;
|
||||
#endif
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
bool server_subproc::output_closed() const {
|
||||
return out_closed[SERVER_SUBPROC_STDOUT] && out_closed[SERVER_SUBPROC_STDERR];
|
||||
}
|
||||
|
||||
int server_subproc::read_output(server_subproc_stream stream, char * buf, size_t len) {
|
||||
if (out_closed[stream]) {
|
||||
return -1;
|
||||
}
|
||||
if (!has_output()) {
|
||||
out_closed[stream] = true;
|
||||
return -1;
|
||||
}
|
||||
#ifdef _WIN32
|
||||
HANDLE h = (HANDLE) out_handle;
|
||||
HANDLE h = (HANDLE) out_handles[stream];
|
||||
DWORD avail = 0;
|
||||
if (!PeekNamedPipe(h, NULL, 0, NULL, &avail, NULL)) {
|
||||
return -1; // pipe broken, child gone
|
||||
out_closed[stream] = true; // pipe broken, child gone
|
||||
return -1;
|
||||
}
|
||||
if (avail == 0) {
|
||||
return 0;
|
||||
@@ -1924,24 +1957,23 @@ int server_subproc::read_output(char * buf, size_t len) {
|
||||
DWORD to_read = avail < (DWORD) len ? avail : (DWORD) len;
|
||||
DWORD got = 0;
|
||||
if (!ReadFile(h, buf, to_read, &got, NULL) || got == 0) {
|
||||
out_closed[stream] = true;
|
||||
return -1;
|
||||
}
|
||||
return (int) got;
|
||||
#else
|
||||
while (true) {
|
||||
ssize_t r = read((int) out_handle, buf, len);
|
||||
ssize_t r = read((int) out_handles[stream], buf, len);
|
||||
if (r > 0) {
|
||||
return (int) r;
|
||||
}
|
||||
if (r == 0) {
|
||||
return -1; // EOF
|
||||
}
|
||||
if (errno == EINTR) {
|
||||
if (r < 0 && errno == EINTR) {
|
||||
continue;
|
||||
}
|
||||
if (errno == EAGAIN || errno == EWOULDBLOCK) {
|
||||
if (r < 0 && (errno == EAGAIN || errno == EWOULDBLOCK)) {
|
||||
return 0;
|
||||
}
|
||||
out_closed[stream] = true; // EOF or error
|
||||
return -1;
|
||||
}
|
||||
#endif
|
||||
@@ -1979,11 +2011,15 @@ void server_subproc::waiter::wait(const std::vector<server_subproc *> & procs, s
|
||||
// no waitable wait exists for anonymous pipes, so poll them in 50 ms steps
|
||||
bool any = false;
|
||||
for (size_t i = 0; i < procs.size(); i++) {
|
||||
DWORD avail = 0;
|
||||
if (!procs[i]->has_output() || !PeekNamedPipe((HANDLE) procs[i]->out_handle, NULL, 0, NULL, &avail, NULL) || avail > 0) {
|
||||
ready[i] = true; // data or broken pipe, read_output() tells which
|
||||
any = true;
|
||||
server_subproc * p = procs[i];
|
||||
ready[i] = !p->has_output();
|
||||
for (int s = 0; s < SERVER_SUBPROC_STREAMS && !ready[i]; s++) {
|
||||
DWORD avail = 0;
|
||||
if (!p->out_closed[s] && (!PeekNamedPipe((HANDLE) p->out_handles[s], NULL, 0, NULL, &avail, NULL) || avail > 0)) {
|
||||
ready[i] = true; // data or broken pipe, read_output() tells which
|
||||
}
|
||||
}
|
||||
any = any || ready[i];
|
||||
}
|
||||
if (!any) {
|
||||
int64_t step = timeout_ms < 0 ? 50 : std::min<int64_t>(timeout_ms, 50);
|
||||
@@ -1991,10 +2027,13 @@ void server_subproc::waiter::wait(const std::vector<server_subproc *> & procs, s
|
||||
}
|
||||
#else
|
||||
std::vector<pollfd> pfds;
|
||||
pfds.reserve(procs.size() + 1);
|
||||
pfds.reserve(procs.size() * SERVER_SUBPROC_STREAMS + 1);
|
||||
pfds.push_back({ (int) wake_fd[0], POLLIN, 0 });
|
||||
for (auto * p : procs) {
|
||||
pfds.push_back({ p->has_output() ? (int) p->out_handle : -1, POLLIN, 0 }); // poll() skips negative fds
|
||||
const bool open = p->has_output();
|
||||
for (int s = 0; s < SERVER_SUBPROC_STREAMS; s++) {
|
||||
pfds.push_back({ open && !p->out_closed[s] ? (int) p->out_handles[s] : -1, POLLIN, 0 }); // poll() skips negative fds
|
||||
}
|
||||
}
|
||||
int timeout = timeout_ms < 0 ? -1 : (int) std::min<int64_t>(timeout_ms, std::numeric_limits<int>::max());
|
||||
int r = poll(pfds.data(), pfds.size(), timeout);
|
||||
@@ -2006,7 +2045,10 @@ void server_subproc::waiter::wait(const std::vector<server_subproc *> & procs, s
|
||||
while (read((int) wake_fd[0], buf, sizeof(buf)) > 0) {}
|
||||
}
|
||||
for (size_t i = 0; i < procs.size(); i++) {
|
||||
ready[i] = pfds[i + 1].fd < 0 || pfds[i + 1].revents != 0;
|
||||
ready[i] = !procs[i]->has_output();
|
||||
for (int s = 0; s < SERVER_SUBPROC_STREAMS; s++) {
|
||||
ready[i] = ready[i] || pfds[1 + i * SERVER_SUBPROC_STREAMS + s].revents != 0;
|
||||
}
|
||||
}
|
||||
#endif
|
||||
}
|
||||
Reference in new issue
Block a user