diff --git a/common/arg.cpp b/common/arg.cpp index a63823767..e99b48c5b 100644 --- a/common/arg.cpp +++ b/common/arg.cpp @@ -326,7 +326,7 @@ static bool parse_bool_value(const std::string & value) { // common_models_handler // -static std::string get_default_local_path(const std::string & url) { +std::string common_get_default_local_path(const std::string & url) { auto f = string_split(url, '#').front(); f = string_split(f, '?').front(); return fs_get_cache_file(string_split(f, '/').back()).string(); @@ -436,7 +436,7 @@ static std::vector build_url_tasks(const common_params_mod if (parts.size() == 1) { common_download_task task; task.url = parts[0]; - task.local_path = model.path.empty() ? get_default_local_path(parts[0]) : model.path; + task.local_path = model.path.empty() ? common_get_default_local_path(parts[0]) : model.path; task.opts = opts; tasks.push_back(std::move(task)); return tasks; @@ -454,7 +454,7 @@ static std::vector build_url_tasks(const common_params_mod task.url = part; task.opts = opts; - std::string local = get_default_local_path(part); + std::string local = common_get_default_local_path(part); if (!base_dir.empty()) { auto pos = local.rfind('/'); std::string name = pos == std::string::npos ? local : local.substr(pos + 1); @@ -487,7 +487,7 @@ void common_models_handler_apply(common_models_handler & handler, common_params auto handle_url = [&](common_params_model & model) { if (!model.url.empty()) { if (model.path.empty()) { - model.path = get_default_local_path(model.url); + model.path = common_get_default_local_path(model.url); } } }; @@ -498,7 +498,7 @@ void common_models_handler_apply(common_models_handler & handler, common_params // optionally, if docker repo is set, resolve it if (!params.model.docker_repo.empty()) { params.model.url = common_docker_resolve_model(params.model.docker_repo); - params.model.path = get_default_local_path(params.model.url); + params.model.path = common_get_default_local_path(params.model.url); } // handle plain "url" tasks (non-hf) diff --git a/common/arg.h b/common/arg.h index 203d1b4e1..a7cf30be4 100644 --- a/common/arg.h +++ b/common/arg.h @@ -149,6 +149,9 @@ common_models_handler common_models_handler_init(const common_params & params, l // check if the model is a preset repo (i.e. has a preset file) bool common_models_handler_is_preset_repo(const common_models_handler & handler); +// the cache path a model downloaded from a plain url is stored at +std::string common_get_default_local_path(const std::string & url); + // download and update params with the downloaded model path void common_models_handler_apply(common_models_handler & handler, common_params & params, common_download_callback * callback = nullptr); diff --git a/common/download.cpp b/common/download.cpp index 682fd6e3e..4dabaedb5 100644 --- a/common/download.cpp +++ b/common/download.cpp @@ -98,6 +98,9 @@ class ProgressBar : public common_download_callback { std::string filename; size_t len = 0; + // the caller's callback, if any, still receives every event + common_download_callback * next; + static void cleanup(const ProgressBar * line) { lines.erase(line); if (lines.empty()) { @@ -110,9 +113,12 @@ class ProgressBar : public common_download_callback { } public: - ProgressBar() = default; + ProgressBar(common_download_callback * next) : next(next) {} void on_start(const common_download_progress & p) override { + if (next) { + next->on_start(p); + } filename = url_get_filename(p.url); for (size_t i = 0; i < filename.size(); ++i) { @@ -126,12 +132,22 @@ public: } } - void on_done(const common_download_progress &, bool) override { + void on_done(const common_download_progress & p, bool ok) override { + if (next) { + next->on_done(p, ok); + } std::lock_guard lock(mutex); cleanup(this); } + bool is_cancelled() const override { + return next && next->is_cancelled(); + } + void on_update(const common_download_progress & p) override { + if (next) { + next->on_update(p); + } if (!p.total || !is_output_a_tty()) { return; } @@ -519,11 +535,9 @@ int common_download_file_single(const std::string & url, const std::string & lock_path, const std::string & alt_path) { if (!opts.offline) { - ProgressBar tty_cb; + ProgressBar tty_cb(opts.callback); common_download_opts online_opts = opts; - if (!online_opts.callback) { - online_opts.callback = &tty_cb; - } + online_opts.callback = &tty_cb; return common_download_file_single_online(url, path, online_opts, skip_etag, lock_path, alt_path); } diff --git a/tools/server/README.md b/tools/server/README.md index b0168b6d1..b995e7403 100644 --- a/tools/server/README.md +++ b/tools/server/README.md @@ -2435,6 +2435,8 @@ Response: ### GET `/models/sse`: Real-time events +This endpoint is also available in single-model mode (`-m`, `-hf`, ...), where it is served before the model is loaded, to follow the download and loading progress. As `GET /models` is unavailable until the model is loaded, each new client first receives the last event sent. If the download or the loading fails, a `status_change` event with status `unloaded` is sent before the server exits. + Example events: ```js diff --git a/tools/server/server-common.cpp b/tools/server/server-common.cpp index 8d4c022e9..95c57f877 100644 --- a/tools/server/server-common.cpp +++ b/tools/server/server-common.cpp @@ -1,3 +1,4 @@ +#include "arg.h" #include "common.h" #include "download.h" #include "log.h" @@ -2386,6 +2387,22 @@ server_role::~server_role() { } } +std::string server_get_model_name(const common_params & params) { + if (!params.model_alias.empty()) { + // backward compat: use first alias as model name + return *params.model_alias.begin(); + } + if (!params.model.get_name().empty()) { + return params.model.get_name(); + } + if (!params.model.url.empty()) { + // -mu before the download: the path the model will be stored at + return common_get_default_local_path(params.model.url); + } + // fallback: derive model name from file name + return std::filesystem::path(params.model.path).filename().string(); +} + bool server_role::is_managed_by_router() { return std::getenv(SERVER_ENV_ROUTER_PORT) != nullptr; } diff --git a/tools/server/server-common.h b/tools/server/server-common.h index 911bb31d7..c1a0f4dca 100644 --- a/tools/server/server-common.h +++ b/tools/server/server-common.h @@ -837,6 +837,9 @@ static inline server_state server_state_from_str(const std::string & str) { GGML_ASSERT(false && "invalid server_state string"); } +// the model name reported by the server: the first alias, else the model's repo or path +std::string server_get_model_name(const common_params & params); + // the role of this process and, when it is a child of a router, its side of the protocol struct server_role { // serializes the notify_to_router writes diff --git a/tools/server/server-context.cpp b/tools/server/server-context.cpp index b458a17c4..89f884e65 100644 --- a/tools/server/server-context.cpp +++ b/tools/server/server-context.cpp @@ -1642,16 +1642,7 @@ private: SRV_TRC("%s", "context checkpoints disabled\n"); } - if (!params_base.model_alias.empty()) { - // backward compat: use first alias as model name - model_name = *params_base.model_alias.begin(); - } else if (!params_base.model.get_name().empty()) { - model_name = params_base.model.get_name(); - } else { - // fallback: derive model name from file name - auto model_path = std::filesystem::path(params_base.model.path); - model_name = model_path.filename().string(); - } + model_name = server_get_model_name(params_base); model_aliases = params_base.model_alias; model_tags = params_base.model_tags; diff --git a/tools/server/server-http.cpp b/tools/server/server-http.cpp index 0244d4a72..d1f31e3ae 100644 --- a/tools/server/server-http.cpp +++ b/tools/server/server-http.cpp @@ -396,6 +396,9 @@ bool server_http_context::init_listener(const common_params & params) { if (frontend_paths.count(req.path)) { return true; // frontend asset, allow it to load and show "loading" } + if (req.path == "/models/sse") { + return true; // reports the download and loading progress + } // no endpoints are allowed to be accessed when the server is not ready // this is to prevent any data races or inconsistent states res.status = 503; diff --git a/tools/server/server-models.cpp b/tools/server/server-models.cpp index 5d5992c08..8e3bb4982 100644 --- a/tools/server/server-models.cpp +++ b/tools/server/server-models.cpp @@ -734,16 +734,7 @@ bool server_models::remove_sidecar(const std::string & name, const std::string & } void server_models::notify_sse(const std::string & event, const std::string & model_id, const json & data) { - std::unique_ptr result = std::make_unique(); - result->data = { - {"model", model_id}, - {"event", event}, - }; - if (!data.is_null()) { - result->data["data"] = data; - } - SRV_DBG("notifying SSE clients about event '%s' for model '%s': %s\n", event.c_str(), model_id.c_str(), safe_json_to_str(result->data).c_str()); - sse.broadcast(std::move(result)); + sse.notify(event, model_id, data); } void server_models::load_models() { @@ -1931,7 +1922,7 @@ void server_models::handle_child_state(const std::string & name, const std::stri } // -// server_models_routes +// server_models_sse // // RAII wrapper similar to server_response_reader, but doesn't use server_queue @@ -1970,6 +1961,61 @@ struct server_models_sse_client { } }; +void server_models_sse::notify(const std::string & event, const std::string & model_id, const json & data) { + std::unique_ptr result = std::make_unique(); + result->data = { + {"model", model_id}, + {"event", event}, + }; + if (!data.is_null()) { + result->data["data"] = data; + } + SRV_DBG("notifying SSE clients about event '%s' for model '%s': %s\n", event.c_str(), model_id.c_str(), safe_json_to_str(result->data).c_str()); + // under the lock, so that a client connecting meanwhile gets this event either replayed or queued, not both + std::lock_guard lk(mutex); + if (replay_last) { + last_event = result->data; + } + queue.broadcast(std::move(result)); +} + +server_http_context::handler_t server_models_sse::make_handler(std::function should_stop) { + return [this, should_stop = std::move(should_stop)](const server_http_req & req) { + auto res = std::make_unique(); + res->status = 200; + res->content_type = "text/event-stream"; + std::shared_ptr sse_client; + std::string pending; // the replayed event, sent first + { + std::lock_guard lk(mutex); + sse_client = std::make_shared(queue); + if (replay_last && !last_event.is_null()) { + pending = "data: " + safe_json_to_str(last_event) + "\n\n"; + } + } + res->next = [should_stop, sse_client, pending = std::move(pending), &req](std::string & output) mutable -> bool { + if (!pending.empty()) { + output = std::move(pending); + pending.clear(); + return true; + } + auto result = sse_client->next([&]() { + return should_stop() || req.should_stop(); + }); + if (result == nullptr) { + return false; // client disconnected or should_stop + } + output = "data: " + safe_json_to_str(result->to_json()) + "\n\n"; + return true; // listen for the next event + }; + return res; + }; +} + +// +// server_models_routes +// + static void res_ok(std::unique_ptr & res, const json & response_data) { res->status = 200; res->data = safe_json_to_str(response_data); @@ -2312,23 +2358,9 @@ void server_models_routes::init_routes() { return res; }; - this->get_router_models_sse = [this](const server_http_req & req) { - auto res = std::make_unique(); - res->status = 200; - res->content_type = "text/event-stream"; - auto sse_client = std::make_shared(models.sse); - res->next = [this, sse_client, &req](std::string & output) -> bool { - auto result = sse_client->next([&]() { - return stopping.load(std::memory_order_relaxed) || req.should_stop(); - }); - if (result == nullptr) { - return false; // client disconnected or should_stop - } - output = "data: " + safe_json_to_str(result->to_json()) + "\n\n"; - return true; // listen for the next event - }; - return res; - }; + this->get_router_models_sse = models.sse.make_handler([this]() { + return stopping.load(std::memory_order_relaxed); + }); this->post_router_models = [this](const server_http_req & req) { auto res = std::make_unique(); diff --git a/tools/server/server-models.h b/tools/server/server-models.h index e18ae3a31..fa5322ad8 100644 --- a/tools/server/server-models.h +++ b/tools/server/server-models.h @@ -157,6 +157,25 @@ private: void delete_partials(const std::vector & parts); }; +// broadcasts model events to the clients of GET /models/sse, used by both router and single-model mode +struct server_models_sse { + // send the last event to each new client, for single-model mode where GET /models is unavailable while loading + const bool replay_last; + + server_models_sse(bool replay_last = false) : replay_last(replay_last) {} + + // thread-safe + void notify(const std::string & event, const std::string & model_id, const json & data = nullptr); + + // handler for GET /models/sse, a client is disconnected once should_stop() returns true + server_http_context::handler_t make_handler(std::function should_stop); + +private: + std::mutex mutex; + json last_event; + server_response queue; +}; + struct server_models_routes; struct server_monitor; // defined in server-models.cpp @@ -289,7 +308,7 @@ public: server_models(const common_params & params, int argc, char ** argv); ~server_models(); - server_response sse; // for real-time updates via SSE endpoint + server_models_sse sse; // for real-time updates via SSE endpoint // (re-)load the list of models from various sources and prepare the metadata mapping // - if this is called the first time, simply populate the metadata diff --git a/tools/server/server.cpp b/tools/server/server.cpp index fabf0a041..e022355dc 100644 --- a/tools/server/server.cpp +++ b/tools/server/server.cpp @@ -109,6 +109,7 @@ static server_role_type server_role_detect(const common_params & params, server_ } // a server given no model is a router const bool has_model = !params.model.path.empty() + || !params.model.url.empty() || !params.model.hf_repo.empty() || !params.model.docker_repo.empty(); return has_model ? SERVER_ROLE_INSTANCE : SERVER_ROLE_ROUTER; @@ -363,66 +364,45 @@ static bool server_on_listening(server_http_context & ctx_http, const common_par return true; } -// -// download worker: a child of the router that pulls a model into the cache and reports the progress -// - -struct server_download_state : public common_download_callback { - server_role * role; +// throttled download progress, shared by the download worker and single-model mode +struct server_download_progress : public common_download_callback { + // called under the lock, the files are downloaded in parallel threads + std::function on_progress; std::function should_stop; - // files download in parallel threads: throttle each file on its own, a shared - // timer lets a small, fast file (e.g. a draft) starve the updates of the others - std::mutex progress_mutex; + // throttle each file on its own, a shared timer lets a small, fast file (e.g. a draft) starve the updates of the others + std::mutex mutex; std::unordered_map last_progress_time; - bool is_ok = false; - server_download_state(server_role * r) : role(r) {} - - bool run(common_params & params) { - try { - common_models_handler handler = common_models_handler_init(params, LLAMA_EXAMPLE_SERVER); - common_models_handler_apply(handler, params, this); - is_ok = true; - } catch (const std::exception & e) { - auto model_name = params.model.get_name(); - SRV_ERR("download failed for model name=%s: %s\n", model_name.c_str(), e.what()); - is_ok = false; - } - return is_ok; - } - void on_progress(const common_download_progress & p) { - json data = { - {"url", p.url}, - {"downloaded", p.downloaded}, - {"total", p.total}, - }; - role->notify_to_router(server_state_to_str(SERVER_STATE_DOWNLOADING), data); + void notify(const common_download_progress & p) { + std::lock_guard lk(mutex); + on_progress(p); } void on_start(const common_download_progress & p) override { - on_progress(p); + notify(p); } void on_update(const common_download_progress & p) override { int64_t now = ggml_time_ms(); - // throttle progress updates to avoid flooding logs; the first update of a file - // always passes, it carries the file's total size - { - std::lock_guard lk(progress_mutex); - auto it = last_progress_time.find(p.url); - if (it != last_progress_time.end() && now - it->second < 100) { - return; - } - last_progress_time[p.url] = now; + // the first update of a file always passes, it carries the file's total size + std::lock_guard lk(mutex); + auto it = last_progress_time.find(p.url); + if (it != last_progress_time.end() && now - it->second < 100) { + return; } + last_progress_time[p.url] = now; on_progress(p); } void on_done(const common_download_progress & p, bool) override { - on_progress(p); + notify(p); } bool is_cancelled() const override { return should_stop ? should_stop() : false; } }; +// +// download worker: a child of the router that pulls a model into the cache and reports the progress +// + static int server_run_download(common_params & params, server_role & role) { auto cancelled = std::make_shared>(false); @@ -431,12 +411,27 @@ static int server_run_download(common_params & params, server_role & role) { cancelled->store(true, std::memory_order_relaxed); }); - server_download_state dl(&role); + server_download_progress dl; + dl.on_progress = [&role](const common_download_progress & p) { + role.notify_to_router(server_state_to_str(SERVER_STATE_DOWNLOADING), { + {"url", p.url}, + {"downloaded", p.downloaded}, + {"total", p.total}, + }); + }; dl.should_stop = [cancelled]() { return cancelled->load(std::memory_order_relaxed); }; - bool ok = dl.run(params); + bool ok = true; + try { + common_models_handler handler = common_models_handler_init(params, LLAMA_EXAMPLE_SERVER); + common_models_handler_apply(handler, params, &dl); + } catch (const std::exception & e) { + auto model_name = params.model.get_name(); + SRV_ERR("download failed for model name=%s: %s\n", model_name.c_str(), e.what()); + ok = false; + } role.notify_to_router(server_state_to_str(SERVER_STATE_DOWNLOADING), { {"result", ok ? "download_finished" : "download_failed"}, @@ -638,15 +633,21 @@ static int server_run_router(common_params & params, int argc, char ** argv, con // static int server_run_instance(common_params & params, server_role & role, common_models_handler & models_handler, const server_run_opts & opts) { - // -hf / docker: download if needed and resolve the local path. a child resolves what the router downloaded - if (opts.resolve_model) { - try { - common_models_handler_apply(models_handler, params); - } catch (const std::exception & e) { - SRV_ERR("failed to download model: %s\n", e.what()); - return 1; + // a router child reports its state to the router, a single-model server to its own /models/sse clients + const bool managed = role.is_managed_by_router(); + const std::string model_name = server_get_model_name(params); + server_models_sse sse(/* replay_last */ true); + std::atomic sse_stopping = false; + + auto notify = [&](const std::string & event, const json & data = nullptr) { + if (!managed) { + sse.notify(event, model_name, data); } - } + }; + auto notify_status = [&](server_model_status status, json data = json::object()) { + data["status"] = server_model_status_to_string(status); + notify("status_change", data); + }; // note: this is guaranteed to out-live ctx_http and tools server_mcp mcp_mgr; @@ -667,37 +668,82 @@ static int server_run_instance(common_params & params, server_role & role, commo server_stream_make_get_handler(), server_stream_make_lookup_handler(), server_stream_make_delete_handler()); + if (!managed) { + ctx_http.get("/models/sse", ex_wrapper(sse.make_handler([&sse_stopping]() { + return sse_stopping.load(std::memory_order_relaxed); + }))); + } if (!server_setup_features(ctx_http, params, mcp_mgr, tools, SERVER_ROLE_INSTANCE)) { return 1; } // setup clean up function, to be called before exit - auto clean_up = [&ctx_http, &ctx_server, &mcp_mgr]() { + auto clean_up = [&ctx_http, &ctx_server, &mcp_mgr, &sse_stopping]() { SRV_INF("%s: cleaning up before exit...\n", __func__); // stop the session GC first, it finalizes live sessions and wakes pending readers server_stream_session_manager_stop(); + sse_stopping.store(true); // disconnect the SSE clients ctx_http.stop(); ctx_server.terminate(); mcp_mgr.shutdown(); llama_backend_free(); }; - // start the HTTP server before loading the model to be able to serve /health requests + // start the HTTP server before downloading and loading the model to be able to serve /health and /models/sse requests if (!ctx_http.start()) { clean_up(); SRV_ERR("%s", "exiting due to HTTP server error\n"); return 1; } - // setup communication child --> router if necessary - if (role.is_managed_by_router()) { - ctx_server.set_state_callback([&](server_state state, json payload) { - role.notify_to_router(server_state_to_str(state), payload); - }); + // -hf / docker: download if needed and resolve the local path. a child resolves what the router downloaded + if (opts.resolve_model) { + json progress = json::object(); // url -> {done, total}, the same shape as the router sends + server_download_progress dl; + dl.on_progress = [&](const common_download_progress & p) { + if (p.total == 0) { + return; // not started yet, or found in the cache + } + progress[p.url] = { + {"done", p.downloaded}, + {"total", p.total}, + }; + notify("download_progress", {{"progress", progress}}); + }; + try { + common_models_handler_apply(models_handler, params, managed ? nullptr : &dl); + if (!progress.empty()) { + notify("download_finished"); + } + } catch (const std::exception & e) { + SRV_ERR("failed to download model: %s\n", e.what()); + notify("download_failed"); + notify_status(SERVER_MODEL_STATUS_UNLOADED, {{"exit_code", 1}}); + clean_up(); + ctx_http.join(); + return 1; + } } + // setup communication child --> router, or to the SSE clients + ctx_server.set_state_callback([&](server_state state, json payload) { + if (managed) { + role.notify_to_router(server_state_to_str(state), payload); + return; + } + switch (state) { + case SERVER_STATE_LOADING: notify_status(SERVER_MODEL_STATUS_LOADING, {{"progress", payload}}); break; + case SERVER_STATE_READY: notify_status(SERVER_MODEL_STATUS_LOADED); break; // wakeup from sleeping + case SERVER_STATE_SLEEPING: notify_status(SERVER_MODEL_STATUS_SLEEPING); break; + default: break; + } + }); + + notify_status(SERVER_MODEL_STATUS_LOADING); + if (!ctx_server.load_model(params)) { + notify_status(SERVER_MODEL_STATUS_UNLOADED, {{"exit_code", 1}}); clean_up(); ctx_http.join(); SRV_ERR("%s", "exiting due to model loading error\n"); @@ -709,7 +755,22 @@ static int server_run_instance(common_params & params, server_role & role, commo SRV_INF("%s", "model loaded\n"); + json info = routes.get_model_info(); + json memory; + if (const auto * ll_ctx = ctx_server.get_llama_context()) { + memory = server_model_memory(llama_n_ctx(ll_ctx), common_memory_breakdown_rows(ll_ctx)).to_json(); + } + + { + json data = {{"info", info}}; + if (!memory.is_null()) { + data["memory"] = memory; + } + notify_status(SERVER_MODEL_STATUS_LOADED, data); + } + shutdown_handler = [&](int) { + sse_stopping.store(true); mcp_mgr.shutdown(); // this will unblock start_loop() ctx_server.terminate(); @@ -729,12 +790,11 @@ static int server_run_instance(common_params & params, server_role & role, commo // optionally, notify router server that this instance is ready std::thread monitor_thread; - if (role.is_managed_by_router()) { + if (managed) { monitor_thread = role.setup(shutdown_handler); - json info = routes.get_model_info(); // the router tracks device memory from these reports - if (const auto * ll_ctx = ctx_server.get_llama_context()) { - info["memory"] = server_model_memory(llama_n_ctx(ll_ctx), common_memory_breakdown_rows(ll_ctx)).to_json(); + if (!memory.is_null()) { + info["memory"] = memory; } role.notify_to_router(server_state_to_str(SERVER_STATE_READY), info); } diff --git a/tools/server/tests/unit/test_basic.py b/tools/server/tests/unit/test_basic.py index 423a7c326..ad3cea679 100644 --- a/tools/server/tests/unit/test_basic.py +++ b/tools/server/tests/unit/test_basic.py @@ -143,3 +143,19 @@ def test_server_model_aliases_and_tags(): assert set(model["tags"]) == {"chat", "fim", "small"} # id is derived from first alias (alphabetical order from std::set) assert model["id"] == "code" + + +def test_server_models_sse_single_model(): + global server + server.start() + # a client connecting after the load gets the last status replayed + with requests.get(server.make_url("/models/sse"), stream=True, timeout=10) as res: + assert res.status_code == 200 + assert res.headers["Content-Type"].startswith("text/event-stream") + line = next(x for x in res.iter_lines() if x) + event = json.loads(line.decode("utf-8").removeprefix("data: ")) + model_id = server.make_request("GET", "/models").body["data"][0]["id"] + assert event["model"] == model_id + assert event["event"] == "status_change" + assert event["data"]["status"] == "loaded" + assert event["data"]["info"]["id"] == model_id