1 parent
3f03b17724
commit
f3269ef96a
7 files changed
+651
-527
No files matched your search
+15
-165
@@ -37,9 +37,6 @@ extern char **environ;
|
||||
|
||||
#define DEFAULT_STOP_TIMEOUT 10 // seconds
|
||||
|
||||
#define CMD_ROUTER_TO_CHILD_EXIT "cmd_router_to_child:exit"
|
||||
#define CMD_CHILD_TO_ROUTER_STATE "cmd_child_to_router:state:" // followed by json string
|
||||
|
||||
// note: SIGPIPE is ignored by the server
|
||||
static void request_child_exit(server_subproc & proc) {
|
||||
FILE * stdin_file = proc.sproc.stdin_file();
|
||||
@@ -49,10 +46,6 @@ static void request_child_exit(server_subproc & proc) {
|
||||
}
|
||||
}
|
||||
|
||||
// address for child process, this is needed because router may run on 0.0.0.0
|
||||
// ref: https://github.com/ggml-org/llama.cpp/issues/17862
|
||||
#define CHILD_ADDR "127.0.0.1"
|
||||
|
||||
// single-threaded, watching all child processes at once
|
||||
struct server_monitor {
|
||||
server_monitor(server_models & models) : models(models) {
|
||||
@@ -65,11 +58,11 @@ struct server_monitor {
|
||||
}
|
||||
|
||||
// thread-safe
|
||||
void watch(const std::string & name, std::shared_ptr<server_subproc> proc, server_child_mode mode, int port) {
|
||||
void watch(const std::string & name, std::shared_ptr<server_subproc> proc, server_role_type role, int port) {
|
||||
child_t c;
|
||||
c.name = name;
|
||||
c.proc = std::move(proc);
|
||||
c.mode = mode;
|
||||
c.role = role;
|
||||
c.port = port;
|
||||
if (!c.proc->has_output()) {
|
||||
SRV_ERR("failed to get stdout/stderr of child process for name=%s\n", name.c_str());
|
||||
@@ -87,7 +80,7 @@ private:
|
||||
struct child_t {
|
||||
std::string name;
|
||||
std::shared_ptr<server_subproc> proc;
|
||||
server_child_mode mode = SERVER_CHILD_MODE_NORMAL;
|
||||
server_role_type role = SERVER_ROLE_INSTANCE;
|
||||
int port = 0;
|
||||
std::string buf[SERVER_SUBPROC_STREAMS]; // partial line of each pipe
|
||||
bool eof = false; // output closed, waiting for the process to be reaped
|
||||
@@ -241,7 +234,7 @@ private:
|
||||
if (it->eof && !it->proc->is_alive()) {
|
||||
int exit_code = it->proc->join();
|
||||
it->proc->stopped.store(true, std::memory_order_release);
|
||||
models.on_child_exit(it->name, it->proc, it->mode, exit_code);
|
||||
models.on_child_exit(it->name, it->proc, it->role, exit_code);
|
||||
SRV_INF("instance name=%s exited with status %d\n", it->name.c_str(), exit_code);
|
||||
it = children.erase(it);
|
||||
} else {
|
||||
@@ -1324,7 +1317,7 @@ void server_models::load(const std::string & name) {
|
||||
|
||||
void server_models::load(const std::string & name, const load_options & opts) {
|
||||
// the fake load time of a model; a download worker loads none
|
||||
if (debug_fake_timing && opts.mode == SERVER_CHILD_MODE_NORMAL) {
|
||||
if (debug_fake_timing && opts.role == SERVER_ROLE_INSTANCE) {
|
||||
// do not hold the mutex here, other requests must keep making progress
|
||||
std::this_thread::sleep_for(std::chrono::seconds(2));
|
||||
}
|
||||
@@ -1345,7 +1338,7 @@ void server_models::load(const std::string & name, const load_options & opts) {
|
||||
// same model cannot start in between: a paused download or an idle cached model
|
||||
// is downloaded again with its files kept, a download on its way and a running
|
||||
// model are refused, and a preset or local model is not a download
|
||||
if (opts.mode == SERVER_CHILD_MODE_DOWNLOAD) {
|
||||
if (opts.role == SERVER_ROLE_DOWNLOAD) {
|
||||
auto it = mapping.find(name);
|
||||
if (it != mapping.end()) {
|
||||
const auto & cur = it->second.meta;
|
||||
@@ -1389,7 +1382,7 @@ void server_models::load(const std::string & name, const load_options & opts) {
|
||||
// releasing its lock and this lock_guard acquiring allows multiple
|
||||
// threads to each observe capacity and all proceed to load.
|
||||
// Download workers do not use models_max slots.
|
||||
if (opts.mode == SERVER_CHILD_MODE_NORMAL && base_params.models_max > 0) {
|
||||
if (opts.role == SERVER_ROLE_INSTANCE && base_params.models_max > 0) {
|
||||
size_t count_active = 0;
|
||||
for (const auto & m : mapping) {
|
||||
if (m.second.meta.is_running()) {
|
||||
@@ -1421,11 +1414,11 @@ void server_models::load(const std::string & name, const load_options & opts) {
|
||||
|
||||
std::vector<std::string> child_args = inst.meta.args; // copy
|
||||
std::vector<std::string> child_env = base_env; // copy
|
||||
child_env.push_back("LLAMA_SERVER_ROUTER_PORT=" + std::to_string(base_params.port));
|
||||
child_env.push_back(std::string(SERVER_ENV_ROUTER_PORT) + "=" + std::to_string(base_params.port));
|
||||
|
||||
if (opts.mode == SERVER_CHILD_MODE_DOWNLOAD) {
|
||||
if (opts.role == SERVER_ROLE_DOWNLOAD) {
|
||||
inst.meta.status = SERVER_MODEL_STATUS_DOWNLOADING;
|
||||
child_env.push_back("LLAMA_SERVER_CHILD_MODE=download");
|
||||
child_env.push_back(std::string(SERVER_ENV_CHILD_ROLE) + "=" + server_role_type_to_str(SERVER_ROLE_DOWNLOAD));
|
||||
child_env.push_back("LLAMA_ARG_HF_REPO=" + name);
|
||||
|
||||
// the sidecars as -hf would pull them: the draft types, and the mmproj switch
|
||||
@@ -1460,7 +1453,7 @@ void server_models::load(const std::string & name, const load_options & opts) {
|
||||
}
|
||||
|
||||
// remember the download: the partial files carry a content hash, not the name
|
||||
if (opts.mode == SERVER_CHILD_MODE_DOWNLOAD) {
|
||||
if (opts.role == SERVER_ROLE_DOWNLOAD) {
|
||||
unfinished_downloads.add({ name, opts.sidecars, opts.partials });
|
||||
}
|
||||
|
||||
@@ -1480,7 +1473,7 @@ void server_models::load(const std::string & name, const load_options & opts) {
|
||||
auto proc = inst.subproc;
|
||||
int port = inst.meta.port;
|
||||
mapping[name] = std::move(inst);
|
||||
monitor->watch(name, proc, opts.mode, port);
|
||||
monitor->watch(name, proc, opts.role, port);
|
||||
cv.notify_all();
|
||||
}
|
||||
|
||||
@@ -1493,7 +1486,7 @@ void server_models::request_stop(const std::string & name, bool send_exit) {
|
||||
monitor->stop(name, it->second.meta.stop_timeout, send_exit);
|
||||
}
|
||||
|
||||
void server_models::on_child_exit(const std::string & name, const std::shared_ptr<server_subproc> & proc, server_child_mode mode, int exit_code) {
|
||||
void server_models::on_child_exit(const std::string & name, const std::shared_ptr<server_subproc> & proc, server_role_type role, int exit_code) {
|
||||
{
|
||||
std::lock_guard<std::mutex> lk(mutex);
|
||||
auto it = mapping.find(name);
|
||||
@@ -1502,7 +1495,7 @@ void server_models::on_child_exit(const std::string & name, const std::shared_pt
|
||||
return; // entry erased, or a newer instance took the name
|
||||
}
|
||||
}
|
||||
if (mode == SERVER_CHILD_MODE_DOWNLOAD) {
|
||||
if (role == SERVER_ROLE_DOWNLOAD) {
|
||||
// instance will be cleaned up on next load_models() call
|
||||
std::lock_guard<std::mutex> lk(mutex);
|
||||
stopping_models.erase(name);
|
||||
@@ -2046,149 +2039,6 @@ void server_models::handle_child_state(const std::string & name, const std::stri
|
||||
}
|
||||
}
|
||||
|
||||
//
|
||||
// server_child
|
||||
//
|
||||
|
||||
server_child::server_child() {
|
||||
if (is_child()) {
|
||||
cmd_out = server_reserve_stdout();
|
||||
}
|
||||
}
|
||||
|
||||
server_child::~server_child() {
|
||||
if (cmd_out) {
|
||||
fclose(cmd_out);
|
||||
}
|
||||
}
|
||||
|
||||
bool server_child::is_child() {
|
||||
const char * router_port = std::getenv("LLAMA_SERVER_ROUTER_PORT");
|
||||
return router_port != nullptr;
|
||||
}
|
||||
|
||||
server_child_mode server_child::get_mode() {
|
||||
const char * mode = std::getenv("LLAMA_SERVER_CHILD_MODE");
|
||||
std::string mode_str(mode ? mode : "");
|
||||
if (mode_str == "download") {
|
||||
return SERVER_CHILD_MODE_DOWNLOAD;
|
||||
} else {
|
||||
return SERVER_CHILD_MODE_NORMAL;
|
||||
}
|
||||
}
|
||||
|
||||
struct server_download_state : public common_download_callback {
|
||||
server_child * self;
|
||||
std::function<bool()> should_stop;
|
||||
std::atomic<int64_t> last_progress_time{0}; // multiple files downloading in different threads
|
||||
bool is_ok = false;
|
||||
|
||||
server_download_state(server_child * s) : self(s) {}
|
||||
|
||||
bool run(common_params & params) {
|
||||
try {
|
||||
common_models_handler handler = common_models_handler_init(params, LLAMA_EXAMPLE_SERVER);
|
||||
common_models_handler_apply(handler, params, this);
|
||||
is_ok = true;
|
||||
} catch (const std::exception & e) {
|
||||
auto model_name = params.model.get_name();
|
||||
SRV_ERR("download failed for model name=%s: %s\n", model_name.c_str(), e.what());
|
||||
is_ok = false;
|
||||
}
|
||||
return is_ok;
|
||||
}
|
||||
void on_progress(const common_download_progress & p) {
|
||||
json data = {
|
||||
{"url", p.url},
|
||||
{"downloaded", p.downloaded},
|
||||
{"total", p.total},
|
||||
};
|
||||
self->notify_to_router(server_state_to_str(SERVER_STATE_DOWNLOADING), data);
|
||||
}
|
||||
void on_start(const common_download_progress & p) override {
|
||||
on_progress(p);
|
||||
}
|
||||
void on_update(const common_download_progress & p) override {
|
||||
int64_t now = ggml_time_ms();
|
||||
// throttle progress updates to avoid flooding logs
|
||||
if (now - last_progress_time.load(std::memory_order_relaxed) >= 100) {
|
||||
on_progress(p);
|
||||
last_progress_time.store(now, std::memory_order_relaxed);
|
||||
}
|
||||
}
|
||||
void on_done(const common_download_progress & p, bool) override {
|
||||
on_progress(p);
|
||||
}
|
||||
bool is_cancelled() const override {
|
||||
return should_stop ? should_stop() : false;
|
||||
}
|
||||
};
|
||||
|
||||
int server_child::run_download(common_params & params) {
|
||||
auto cancelled = std::make_shared<std::atomic<bool>>(false);
|
||||
|
||||
// monitor stdin for cancellation command from the router
|
||||
std::thread signal_thread = setup([cancelled](int) {
|
||||
cancelled->store(true, std::memory_order_relaxed);
|
||||
});
|
||||
|
||||
server_download_state dl(this);
|
||||
dl.should_stop = [cancelled]() {
|
||||
return cancelled->load(std::memory_order_relaxed);
|
||||
};
|
||||
|
||||
bool ok = dl.run(params);
|
||||
|
||||
notify_to_router(server_state_to_str(SERVER_STATE_DOWNLOADING), {
|
||||
{"result", ok ? "download_finished" : "download_failed"},
|
||||
});
|
||||
|
||||
// router should send CMD_ROUTER_TO_CHILD_EXIT after receiving the result
|
||||
if (signal_thread.joinable()) {
|
||||
signal_thread.join();
|
||||
}
|
||||
|
||||
SRV_INF("download completed %s\n", ok ? "successfully" : "with errors");
|
||||
return 0;
|
||||
}
|
||||
|
||||
std::thread server_child::setup(const std::function<void(int)> & shutdown_handler) {
|
||||
// setup thread for monitoring stdin
|
||||
return std::thread([shutdown_handler]() {
|
||||
// wait for EOF on stdin
|
||||
SRV_INF("%s", "child server monitoring thread started, waiting for EOF on stdin...\n");
|
||||
bool eof = false;
|
||||
while (true) {
|
||||
std::string line;
|
||||
if (!std::getline(std::cin, line)) {
|
||||
// EOF detected, that means the router server is unexpectedly exit or killed
|
||||
eof = true;
|
||||
break;
|
||||
}
|
||||
if (line.find(CMD_ROUTER_TO_CHILD_EXIT) != std::string::npos) {
|
||||
SRV_INF("%s", "exit command received, exiting...\n");
|
||||
shutdown_handler(0);
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (eof) {
|
||||
SRV_INF("%s", "EOF on stdin detected, forcing shutdown...\n");
|
||||
exit(1);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
void server_child::notify_to_router(const std::string & state, const json & payload) {
|
||||
json data = {
|
||||
{"state", state},
|
||||
{"payload", payload},
|
||||
};
|
||||
std::lock_guard<std::mutex> lk(mtx_stdout);
|
||||
fprintf(cmd_out, "%s%s\n", CMD_CHILD_TO_ROUTER_STATE, safe_json_to_str(data).c_str());
|
||||
fflush(cmd_out);
|
||||
}
|
||||
|
||||
|
||||
//
|
||||
// server_models_routes
|
||||
//
|
||||
@@ -2617,7 +2467,7 @@ void server_models_routes::init_routes() {
|
||||
SRV_INF("starting download for model '%s'\n", name.c_str());
|
||||
{
|
||||
server_models::load_options load_opts;
|
||||
load_opts.mode = SERVER_CHILD_MODE_DOWNLOAD;
|
||||
load_opts.role = SERVER_ROLE_DOWNLOAD;
|
||||
load_opts.custom_meta = server_model_meta{};
|
||||
load_opts.custom_meta->source = SERVER_MODEL_SOURCE_CACHE;
|
||||
load_opts.custom_meta->name = name;
|
||||
|
||||
Reference in new issue
Block a user