log, server: self contained colors, split child commands from logs in router mode (#29895)

* log, server: make router child lines carry their own colors

The logger writes the color reset after the trailing newline, so the
reset opens the next line. On the shared pipe of a router child it lands
in front of the next state command, which the router then misses, and
the line break that works around it shows up as an empty log line on
every progress update.

The reset now goes before the trailing newlines, so every line is self
contained and the command goes back to its plain framing. The router
passes its effective color setting to its children, whose output ends
up in its terminal, and leaves that option out when comparing presets
on reload.

* log: enable ANSI colors on the Windows console

A Windows console renders ANSI sequences only in virtual terminal mode,
which nothing turns on for the logger, so llama-server prints raw escape
codes on the Windows 10 console while llama-cli, whose console code
enables it, shows colors. The logger now enables virtual terminal mode
on stdout and stderr when it turns colors on, and keeps colors off when
a console cannot render them. Pipes and files take the sequences as is.

* server: separate the router child commands from its logs

The child sent its state commands on the same pipe as its logs, so the
router had to pick them out of the log stream by a line prefix, and any
unterminated write in front of a command made the router miss it. This
resolves the TODO at the spawn that called for splitting stdout and
stderr.

The child now keeps stdout for the commands and points everything else
written to stdout at stderr, before anything is written. The router
reads both pipes, handles the commands from stdout and forwards stderr
as the log, and warns about any other line on the command pipe.

* server: address review from ngxson

The single server_child is now created first in the entry point and its
constructor keeps stdout for the commands, so the stream is a member of
the instance instead of a static, and init() is gone. The instance is
passed down to the server, while the CLI entry point creates its own.

* Update tools/server/server.cpp

---------

Co-authored-by: Xuan-Son Nguyen <thichthat@gmail.com>
This commit is contained in:
PascalandXuan-Son Nguyen authored and GitHub committed 2026-10-05 01:59:37 +02:00
1 parent 0bb496dbd3
commit a7fb71fab8
9 files changed
+196 -75

No files matched your search

+48 -30
View File
@@ -94,7 +94,7 @@ private:
std::shared_ptr<server_subproc> proc;
server_child_mode mode = SERVER_CHILD_MODE_NORMAL;
int port = 0;
std::string buf; // partial line
std::string buf[SERVER_SUBPROC_STREAMS]; // partial line of each pipe
bool eof = false; // output closed, waiting for the process to be reaped
int64_t deadline = 0; // force-kill time in ms, 0 when no stop is pending
};
@@ -147,46 +147,56 @@ private:
return false;
}
// read what the child wrote, forward complete lines
// read what the child wrote, handle its commands and forward its logs, line by line
void read_output(child_t & c) {
for (int i = 0; i < SERVER_SUBPROC_STREAMS; i++) {
read_stream(c, (server_subproc_stream) i);
}
c.eof = c.proc->output_closed();
}
void read_stream(child_t & c, server_subproc_stream stream) {
char chunk[4096];
while (!c.eof) {
int n = c.proc->read_output(chunk, sizeof(chunk));
std::string & buf = c.buf[stream];
bool closed = false;
while (true) {
int n = c.proc->read_output(stream, chunk, sizeof(chunk));
if (n < 0) {
c.eof = true;
closed = true;
break;
}
if (n == 0) {
break;
}
c.buf.append(chunk, (size_t) n);
buf.append(chunk, (size_t) n);
size_t start = 0;
while (true) {
size_t nl = c.buf.find('\n', start);
size_t nl = buf.find('\n', start);
if (nl == std::string::npos) {
break;
}
std::string line = c.buf.substr(start, nl + 1 - start);
on_line(c, stream, buf.substr(start, nl + 1 - start));
start = nl + 1;
on_line(c, line);
}
c.buf.erase(0, start);
if (c.buf.size() > max_line) {
c.buf.clear(); // a child that never writes a newline must not grow this without bound
buf.erase(0, start);
if (buf.size() > max_line) {
buf.clear(); // a child that never writes a newline must not grow this without bound
}
}
if (c.eof && !c.buf.empty()) {
on_line(c, c.buf);
c.buf.clear();
if (closed && !buf.empty()) {
on_line(c, stream, buf);
buf.clear();
}
}
void on_line(child_t & c, const std::string & line) {
if (string_starts_with(line, CMD_CHILD_TO_ROUTER_STATE)) {
void on_line(child_t & c, server_subproc_stream stream, const std::string & line) {
if (stream == SERVER_SUBPROC_STDERR) {
LOG("[%5d] %s", c.port, line.c_str()); // forward log
} else if (string_starts_with(line, CMD_CHILD_TO_ROUTER_STATE)) {
LOG_DBG("[%5d] %s", c.port, line.c_str()); // prevent spamming the log
models.handle_child_state(c.name, line);
} else {
LOG("[%5d] %s", c.port, line.c_str()); // forward log
SRV_WRN("[%5d] unexpected output on the command pipe: %s", c.port, line.c_str());
}
}
@@ -513,6 +523,8 @@ void server_model_meta::update_args(common_preset_context & ctx_preset, std::str
preset.set_option(ctx_preset, "LLAMA_ARG_HOST", CHILD_ADDR);
preset.set_option(ctx_preset, "LLAMA_ARG_PORT", std::to_string(port));
preset.set_option(ctx_preset, "LLAMA_ARG_ALIAS", name);
// the child output goes through the router to its terminal, so it follows the router colors
preset.set_option(ctx_preset, "LLAMA_ARG_LOG_COLORS", common_log_get_colors(common_log_main()) ? "on" : "off");
// TODO: maybe validate preset before rendering ?
// render args
args = preset.to_args(bin_path);
@@ -796,11 +808,12 @@ void server_models::load_models() {
inst.meta.hidden = hidden_models.count(name) > 0;
}
};
// update_args() injects HOST/PORT/ALIAS, so strip them before comparing presets
// update_args() injects HOST/PORT/ALIAS/LOG_COLORS, so strip them before comparing presets
auto preset_options_for_compare = [](common_preset p) {
p.unset_option("LLAMA_ARG_HOST");
p.unset_option("LLAMA_ARG_PORT");
p.unset_option("LLAMA_ARG_ALIAS");
p.unset_option("LLAMA_ARG_LOG_COLORS");
return p.options;
};
@@ -1171,9 +1184,8 @@ void server_models::load(const std::string & name, const load_options & opts) {
}
inst.meta.args = child_args; // save for debugging
// TODO @ngxson : maybe separate stdout and stderr in the future
// so that we can use stdout for commands and stderr for logging
int options = subprocess_option_no_window | subprocess_option_combined_stdout_stderr;
// the child writes its commands to stdout and its logs to stderr
int options = subprocess_option_no_window;
if (!inst.subproc->sproc.create(child_args, options, child_env)) {
throw std::runtime_error("failed to spawn server instance");
}
@@ -1673,6 +1685,18 @@ void server_models::handle_child_state(const std::string & name, const std::stri
// server_child
//
server_child::server_child() {
if (is_child()) {
cmd_out = server_reserve_stdout();
}
}
server_child::~server_child() {
if (cmd_out) {
fclose(cmd_out);
}
}
bool server_child::is_child() {
const char * router_port = std::getenv("LLAMA_SERVER_ROUTER_PORT");
return router_port != nullptr;
@@ -1795,14 +1819,8 @@ void server_child::notify_to_router(const std::string & state, const json & payl
{"payload", payload},
};
std::lock_guard<std::mutex> lk(mtx_stdout);
common_log_pause(common_log_main());
fflush(stdout);
// the router matches the command on a line prefix, so the leading newline
// closes whatever the logger left open on the shared pipe, down to the
// trailing color reset that carries no newline of its own
fprintf(stdout, "\n%s%s\n", CMD_CHILD_TO_ROUTER_STATE, safe_json_to_str(data).c_str());
fflush(stdout);
common_log_resume(common_log_main());
fprintf(cmd_out, "%s%s\n", CMD_CHILD_TO_ROUTER_STATE, safe_json_to_str(data).c_str());
fflush(cmd_out);
}