Files
llama.cpp/tools/server/server.cpp
T
Aleksander Grygier 1d81992784 ui: keep downloads in Discover during search and fix their progress with a draft (#31467)
* ui : keep downloads in Discover sidebar during search, sort base-org repos first

- Show the sticky Downloading section regardless of an active search query
- Order search results so repos published by their base model's org lead

Assisted-by: Claude (Opus 5.5 Medium)

* server : throttle download progress per file

Files of a download run in parallel threads, but progress updates were
throttled with one shared timer. A small draft sidecar reports far more
often than the main weights, so it took nearly every slot and the main
file's updates, including the first one carrying its total size, were
dropped until the draft finished. The UI then showed the draft's progress
first and restarted from 0% for the model.

Throttle each file on its own and always pass a file's first update.

Assisted-by: Claude (Opus 5.5 Medium)

* ui: list a downloading repo once and match base orgs case-insensitively

A search hit that is downloading already shows under Downloading, so the
results leave it out. Base model orgs compare without case, as the hub
treats them.

Assisted-by: Claude (Opus 5.5 Medium)

* server : tell clients a download stopped on request

When a client paused or cancelled a download, every other client following
/models/sse got a bare download_failed and could not tell it from an error,
so the web UI showed "Download failed" for a pause made in the Llama app.
A stop asked for through unload or delete now fails with {"stopped": true}.

Deleting a paused download, which has no instance left, now sends
model_remove as deleting a running one does, so other clients drop its row.

Assisted-by: Claude (Opus 5.5 Medium)

* ui : follow downloads other clients start, pause or cancel

A download another client started (e.g. the Llama app) already showed up
from the feed; it now also picks up the sidecars the list reports for it.
A stop another client asked for reads as a pause instead of a failure, and
a removal clears the download row even when the list does not carry it yet.

Assisted-by: Claude (Opus 5.5 Medium)
2026-11-11 23:26:39 +01:00

828 lines
32 KiB
C++

#include "server-connect.h"
#include "server-context.h"
#include "server-http.h"
#include "server-models.h"
#include "server-cors-proxy.h"
#include "server-stream.h"
#include "server-tools.h"
#include "arg.h"
#include "build-info.h"
#include "common.h"
#include "fit.h"
#include "llama.h"
#include "log.h"
#include <atomic>
#include <clocale>
#include <exception>
#include <mutex>
#include <signal.h>
#include <thread> // for std::thread::hardware_concurrency
#include <unordered_map>
#if defined(_WIN32)
#include <windows.h>
#endif
static std::function<void(int)> shutdown_handler;
static std::atomic_flag is_terminating = ATOMIC_FLAG_INIT;
static inline void signal_handler(int signal) {
if (is_terminating.test_and_set()) {
// in case it hangs, we can force terminate the server by hitting Ctrl+C twice
// this is for better developer experience, we can remove when the server is stable enough
fprintf(stderr, "Received second interrupt, terminating immediately.\n");
exit(1);
}
shutdown_handler(signal);
}
// how the server is run; everything is on when it is the llama-server binary
struct server_run_opts {
bool install_signal_handlers = true; // off when embedded in llama-cli, which handles them itself
bool resolve_model = true; // download / resolve -hf and docker models; off when the caller did it already
};
// satisfies -Wmissing-declarations (used by llama command)
int llama_server(int argc, char ** argv);
// to be used via CLI (argc / argv are used by router mode only)
int llama_server(common_params & params, int argc, char ** argv);
void llama_server_terminate();
void llama_server_terminate() {
if (shutdown_handler) {
shutdown_handler(0);
}
}
// wrapper function that handles exceptions and logs errors
// this is to make sure handler_t never throws exceptions; instead, it returns an error response
static server_http_context::handler_t ex_wrapper(server_http_context::handler_t func) {
return [func = std::move(func)](const server_http_req & req) -> server_http_res_ptr {
std::string message;
error_type error;
try {
return func(req);
} catch (const std::invalid_argument & e) {
// treat invalid_argument as invalid request (400)
error = ERROR_TYPE_INVALID_REQUEST;
message = e.what();
} catch (const common_json_error & e) {
// JSON parse and type errors are invalid requests (400)
error = ERROR_TYPE_INVALID_REQUEST;
message = e.what();
} catch (const std::exception & e) {
// treat other exceptions as server error (500)
error = ERROR_TYPE_SERVER;
message = e.what();
} catch (...) {
error = ERROR_TYPE_SERVER;
message = "unknown error";
}
auto res = std::make_unique<server_http_res>();
res->status = 500;
try {
json error_data = format_error_response(message, error);
res->status = json_value(error_data, "code", 500);
res->data = safe_json_to_str({{ "error", error_data }});
SRV_WRN("got exception: %s\n", res->data.c_str());
} catch (const std::exception & e) {
SRV_ERR("got another exception: %s | while handling exception: %s\n", e.what(), message.c_str());
res->data = "Internal Server Error";
}
return res;
};
}
//
// shared by every role
//
static server_role_type server_role_detect(const common_params & params, server_role & role) {
if (role.is_managed_by_router()) {
return role.get_role();
}
// a server given no model is a router
const bool has_model = !params.model.path.empty()
|| !params.model.hf_repo.empty()
|| !params.model.docker_repo.empty();
return has_model ? SERVER_ROLE_INSTANCE : SERVER_ROLE_ROUTER;
}
// adjust the params for the role; returns false on a configuration error
static bool server_prepare_params(common_params & params, server_role_type role_type) {
if (role_type != SERVER_ROLE_ROUTER) {
// validate batch size for embeddings
// embeddings require all tokens to be processed in a single ubatch
// see https://github.com/ggml-org/llama.cpp/issues/12836
if (params.embedding && params.n_batch > params.n_ubatch) {
SRV_WRN("embeddings enabled with n_batch (%d) > n_ubatch (%d)\n", params.n_batch, params.n_ubatch);
SRV_WRN("setting n_batch = n_ubatch = %d to avoid assertion failure\n", params.n_ubatch);
params.n_batch = params.n_ubatch;
}
if (params.n_parallel < 0) {
SRV_TRC("%s", "n_parallel is set to auto, using n_parallel = 4 and kv_unified = true\n");
params.n_parallel = 4;
params.kv_unified = true;
}
}
// size the KV pool from --kv-unified-per-slot, unless the user pinned it with -c
// or with -c 0 for max context
const bool ctx_pool_auto_sized = params.kv_unified_per_slot > 0 &&
params.n_ctx == 0 &&
(uint32_t) params.fit_params_min_ctx != UINT32_MAX;
if (ctx_pool_auto_sized) {
params.n_ctx = params.n_parallel * params.kv_unified_per_slot;
SRV_INF("--kv-unified-per-slot: sizing KV pool to n_parallel * kv_unified_per_slot = %d * %d = %d\n", params.n_parallel,
params.kv_unified_per_slot, params.n_ctx);
}
// for consistency between server router mode and single-model mode, we set the same model name as alias
auto model_name = params.model.get_name();
if (params.model_alias.empty() && !model_name.empty()) {
params.model_alias.insert(model_name);
}
// check early, so a missing llama-connect fails-fast
if (params.server_connect) {
const std::string reason = server_connect::unavailable_reason(params);
if (!reason.empty()) {
SRV_ERR("--connect is not available: %s\n", reason.c_str());
return false;
}
params.server_connect_code = server_connect::resolve_code(params.server_connect_code);
} else if (!params.server_connect_code.empty()) {
// the code is only meaningful while the tunnel runs, and /props keys off it being set
SRV_WRN("%s", "--connect-code is ignored without --connect\n");
params.server_connect_code.clear();
}
return true;
}
// the API of every server; a router points the handlers in routes at its proxies before calling this
static void server_register_routes(server_http_context & ctx_http, server_routes & routes) {
ctx_http.get ("/health", ex_wrapper(routes.get_health)); // public endpoint (no API key check)
ctx_http.get ("/v1/health", ex_wrapper(routes.get_health)); // public endpoint (no API key check)
ctx_http.get ("/metrics", ex_wrapper(routes.get_metrics));
ctx_http.get ("/props", ex_wrapper(routes.get_props));
ctx_http.post("/props", ex_wrapper(routes.post_props));
ctx_http.get ("/models", ex_wrapper(routes.get_models));
ctx_http.get ("/v1/models", ex_wrapper(routes.get_models));
ctx_http.post("/completion", ex_wrapper(routes.post_completions)); // legacy
ctx_http.post("/completions", ex_wrapper(routes.post_completions));
ctx_http.post("/v1/completions", ex_wrapper(routes.post_completions_oai));
ctx_http.post("/chat/completions", ex_wrapper(routes.post_chat_completions));
ctx_http.post("/v1/chat/completions", ex_wrapper(routes.post_chat_completions));
ctx_http.post("/v1/chat/completions/control", ex_wrapper(routes.post_control));
ctx_http.post("/v1/responses", ex_wrapper(routes.post_responses_oai));
ctx_http.post("/responses", ex_wrapper(routes.post_responses_oai));
ctx_http.post("/v1/audio/transcriptions", ex_wrapper(routes.post_transcriptions_oai));
ctx_http.post("/audio/transcriptions", ex_wrapper(routes.post_transcriptions_oai));
ctx_http.post("/tts", ex_wrapper(routes.post_tts));
ctx_http.post("/v1/audio/speech", ex_wrapper(routes.post_speech_oai));
ctx_http.post("/audio/speech", ex_wrapper(routes.post_speech_oai));
ctx_http.ws ("/v1/realtime", routes.ws_realtime);
ctx_http.post("/v1/messages", ex_wrapper(routes.post_anthropic_messages)); // anthropic messages API
ctx_http.post("/infill", ex_wrapper(routes.post_infill));
ctx_http.post("/embedding", ex_wrapper(routes.post_embeddings)); // legacy
ctx_http.post("/embeddings", ex_wrapper(routes.post_embeddings));
ctx_http.post("/v1/embeddings", ex_wrapper(routes.post_embeddings_oai));
ctx_http.post("/rerank", ex_wrapper(routes.post_rerank));
ctx_http.post("/reranking", ex_wrapper(routes.post_rerank));
ctx_http.post("/v1/rerank", ex_wrapper(routes.post_rerank));
ctx_http.post("/v1/reranking", ex_wrapper(routes.post_rerank));
ctx_http.post("/v1/systemone", ex_wrapper(routes.post_systemone));
ctx_http.post("/tokenize", ex_wrapper(routes.post_tokenize));
ctx_http.post("/detokenize", ex_wrapper(routes.post_detokenize));
ctx_http.post("/apply-template", ex_wrapper(routes.post_apply_template));
// token counting
ctx_http.post("/chat/completions/input_tokens", ex_wrapper(routes.post_chat_completions_tok));
ctx_http.post("/v1/chat/completions/input_tokens", ex_wrapper(routes.post_chat_completions_tok));
ctx_http.post("/responses/input_tokens", ex_wrapper(routes.post_responses_tok_oai));
ctx_http.post("/v1/responses/input_tokens", ex_wrapper(routes.post_responses_tok_oai));
ctx_http.post("/v1/messages/count_tokens", ex_wrapper(routes.post_anthropic_count_tokens)); // anthropic token counting
// LoRA adapters hotswap
ctx_http.get ("/lora-adapters", ex_wrapper(routes.get_lora_adapters));
ctx_http.post("/lora-adapters", ex_wrapper(routes.post_lora_adapters));
// Save & load slots
ctx_http.get ("/slots", ex_wrapper(routes.get_slots));
ctx_http.post("/slots/:id_slot", ex_wrapper(routes.post_slots));
// Google Cloud Platform (Vertex AI) compat
ctx_http.register_gcp_compat();
}
// resumable streaming, see server-stream.h: an instance binds the local session factories,
// a router binds proxies that resolve the owning child
static void server_register_stream_routes(
server_http_context & ctx_http,
const server_http_context::handler_t & stream_get,
const server_http_context::handler_t & streams_lookup,
const server_http_context::handler_t & stream_delete) {
ctx_http.get ("/v1/stream", ex_wrapper(stream_get));
ctx_http.post("/v1/streams/lookup", ex_wrapper(streams_lookup));
ctx_http.del ("/v1/stream", ex_wrapper(stream_delete));
}
// the features behind a flag, and the warning about the ones that must not face untrusted clients
// returns false when one of them fails to start
static bool server_setup_features(
server_http_context & ctx_http,
const common_params & params,
server_mcp & mcp_mgr,
server_tools & tools,
server_role_type role_type) {
// return 403 for disabled features
server_http_context::handler_t res_403 = [](const server_http_req &) {
auto res = std::make_unique<server_http_res>();
res->status = 403;
res->data = safe_json_to_str({
{"error", {
{"message", "this feature is disabled"},
{"type", "feature_disabled"},
}}
});
return res;
};
if (params.cors_origins == "*" && params.api_keys.empty()) {
SRV_WRN("%s", "security: no API key is set and CORS allows all origins (see https://github.com/ggml-org/llama.cpp/pull/25655)\n");
}
std::vector<std::string> warn_names;
if (role_type == SERVER_ROLE_ROUTER) {
warn_names.push_back("router mode");
}
if (params.server_connect) {
warn_names.push_back("peer-to-peer tunnel (llama-connect, experimental)");
}
// CORS proxy (EXPERIMENTAL, only used by the Web UI for MCP)
if (params.ui_mcp_proxy) {
ctx_http.get ("/cors-proxy", ex_wrapper(proxy_handler_get));
ctx_http.post("/cors-proxy", ex_wrapper(proxy_handler_post));
warn_names.push_back("MCP proxy (experimental)");
} else {
ctx_http.get ("/cors-proxy", ex_wrapper(res_403));
ctx_http.post("/cors-proxy", ex_wrapper(res_403));
}
try {
mcp_mgr.start(params);
} catch (const std::exception & e) {
SRV_ERR("MCP starting failed: %s\n", e.what());
return false;
}
if (!params.server_tools.empty() || !mcp_mgr.empty()) {
try {
tools.setup(params.server_tools, mcp_mgr, params.server_tools_runtime);
} catch (const std::exception & e) {
SRV_ERR("tools setup failed: %s\n", e.what());
return false;
}
ctx_http.get ("/tools", ex_wrapper(tools.handle_get));
ctx_http.post("/tools", ex_wrapper(tools.handle_post));
if (!params.server_tools.empty()) {
warn_names.push_back("server tools (experimental)");
}
if (!params.server_tools_runtime.empty()) {
warn_names.push_back("tools runtime (experimental)");
}
if (!mcp_mgr.empty()) {
warn_names.push_back("MCP servers (experimental)");
}
} else {
ctx_http.get ("/tools", ex_wrapper(res_403));
ctx_http.post("/tools", ex_wrapper(res_403));
}
if (!warn_names.empty()) {
std::string features;
for (const auto & name : warn_names) {
if (!features.empty()) features += ", ";
features += name;
}
SRV_WRN("security: %s enabled - do not expose to untrusted environments\n", features.c_str());
}
return true;
}
static void server_install_signal_handlers() {
#if defined (__unix__) || (defined (__APPLE__) && defined (__MACH__))
struct sigaction sigint_action;
sigint_action.sa_handler = signal_handler;
sigemptyset (&sigint_action.sa_mask);
sigint_action.sa_flags = 0;
sigaction(SIGINT, &sigint_action, NULL);
sigaction(SIGTERM, &sigint_action, NULL);
#elif defined (_WIN32)
auto console_ctrl_handler = +[](DWORD ctrl_type) -> BOOL {
return (ctrl_type == CTRL_C_EVENT) ? (signal_handler(SIGINT), true) : false;
};
SetConsoleCtrlHandler(reinterpret_cast<PHANDLER_ROUTINE>(console_ctrl_handler), true);
#endif
}
// once the HTTP server listens: log the addresses and spawn llama-connect if asked, so the port is
// bound and the child startup health check reaches us. the destructor of connect_proc stops it
static bool server_on_listening(server_http_context & ctx_http, const common_params & params, server_connect & connect_proc) {
for (const auto & address : ctx_http.listening_addresses) {
SRV_INF("listening on %s\n", address.c_str());
}
if (params.server_connect && !connect_proc.start(params, ctx_http.port)) {
SRV_ERR("%s", "exiting due to llama-connect error\n");
return false;
}
return true;
}
//
// download worker: a child of the router that pulls a model into the cache and reports the progress
//
struct server_download_state : public common_download_callback {
server_role * role;
std::function<bool()> should_stop;
// files download in parallel threads: throttle each file on its own, a shared
// timer lets a small, fast file (e.g. a draft) starve the updates of the others
std::mutex progress_mutex;
std::unordered_map<std::string, int64_t> last_progress_time;
bool is_ok = false;
server_download_state(server_role * r) : role(r) {}
bool run(common_params & params) {
try {
common_models_handler handler = common_models_handler_init(params, LLAMA_EXAMPLE_SERVER);
common_models_handler_apply(handler, params, this);
is_ok = true;
} catch (const std::exception & e) {
auto model_name = params.model.get_name();
SRV_ERR("download failed for model name=%s: %s\n", model_name.c_str(), e.what());
is_ok = false;
}
return is_ok;
}
void on_progress(const common_download_progress & p) {
json data = {
{"url", p.url},
{"downloaded", p.downloaded},
{"total", p.total},
};
role->notify_to_router(server_state_to_str(SERVER_STATE_DOWNLOADING), data);
}
void on_start(const common_download_progress & p) override {
on_progress(p);
}
void on_update(const common_download_progress & p) override {
int64_t now = ggml_time_ms();
// throttle progress updates to avoid flooding logs; the first update of a file
// always passes, it carries the file's total size
{
std::lock_guard<std::mutex> lk(progress_mutex);
auto it = last_progress_time.find(p.url);
if (it != last_progress_time.end() && now - it->second < 100) {
return;
}
last_progress_time[p.url] = now;
}
on_progress(p);
}
void on_done(const common_download_progress & p, bool) override {
on_progress(p);
}
bool is_cancelled() const override {
return should_stop ? should_stop() : false;
}
};
static int server_run_download(common_params & params, server_role & role) {
auto cancelled = std::make_shared<std::atomic<bool>>(false);
// monitor stdin for cancellation command from the router
std::thread signal_thread = role.setup([cancelled](int) {
cancelled->store(true, std::memory_order_relaxed);
});
server_download_state dl(&role);
dl.should_stop = [cancelled]() {
return cancelled->load(std::memory_order_relaxed);
};
bool ok = dl.run(params);
role.notify_to_router(server_state_to_str(SERVER_STATE_DOWNLOADING), {
{"result", ok ? "download_finished" : "download_failed"},
});
// router should send CMD_ROUTER_TO_CHILD_EXIT after receiving the result
if (signal_thread.joinable()) {
signal_thread.join();
}
SRV_INF("download completed %s\n", ok ? "successfully" : "with errors");
return 0;
}
//
// measure worker: a child of the router, reports the memory a model needs without loading it
//
static int server_run_measure(common_params & params, server_role & role, common_models_handler & models_handler) {
// the router sets --offline: only resolve the local path, never download
try {
common_models_handler_apply(models_handler, params);
} catch (const std::exception & e) {
SRV_ERR("failed to resolve model: %s\n", e.what());
return 1;
}
const std::string model_name = params.model.get_name();
// TODO: mmproj is not in llama_get_memory_breakdown, a multimodal model is underestimated
json payload;
try {
common_memory_reports m = common_measure_memory(params);
payload = {
{"result", "measured"},
{"n_ctx_train", m.n_ctx_train},
{"min", server_model_memory(m.n_ctx_min, m.info_min).to_json()},
{"max", server_model_memory(m.n_ctx_max, m.info_max).to_json()},
};
SRV_INF("memory measured for model name=%s at n_ctx=%u (min) and n_ctx=%u (max)\n",
model_name.c_str(), m.n_ctx_min, m.n_ctx_max);
} catch (const std::exception & e) {
SRV_ERR("failed to measure memory for model name=%s: %s\n", model_name.c_str(), e.what());
payload = {
{"result", "failed"},
{"error", e.what()},
};
}
role.notify_to_router(server_state_to_str(SERVER_STATE_MEASURING), payload);
return 0;
}
//
// router
//
static int server_run_router(common_params & params, int argc, char ** argv, const server_run_opts & opts) {
SRV_INF("%s", "starting server in router mode. models will be automatically loaded on-demand\n");
// note: this is guaranteed to out-live ctx_http and tools
server_mcp mcp_mgr;
// the router loads no model, its context stays empty; the routes it does not proxy still need one
server_context ctx_server;
server_routes routes(params, ctx_server);
server_tools tools;
server_http_context ctx_http;
if (!ctx_http.init(params)) {
SRV_ERR("%s", "failed to initialize HTTP server\n");
return 1;
}
// setup server instances manager
std::optional<server_models_routes> models_routes;
try {
models_routes.emplace(params, argc, argv);
} catch (const std::exception & e) {
SRV_ERR("failed to initialize router models: %s\n", e.what());
return 1;
}
// proxy handlers
// note: routes.get_health stays the same
routes.get_metrics = models_routes->proxy_get;
routes.post_props = models_routes->proxy_post;
routes.post_completions = models_routes->proxy_post;
routes.post_completions_oai = models_routes->proxy_post;
routes.post_chat_completions = models_routes->proxy_post;
routes.post_control = models_routes->proxy_post;
routes.post_responses_oai = models_routes->proxy_post;
routes.post_transcriptions_oai = models_routes->proxy_post;
routes.post_tts = models_routes->proxy_post;
routes.post_speech_oai = models_routes->proxy_post;
routes.ws_realtime = models_routes->proxy_ws;
routes.post_anthropic_messages = models_routes->proxy_post;
routes.post_anthropic_count_tokens = models_routes->proxy_post;
routes.post_infill = models_routes->proxy_post;
routes.post_embeddings = models_routes->proxy_post;
routes.post_embeddings_oai = models_routes->proxy_post;
routes.post_rerank = models_routes->proxy_post;
routes.post_systemone = models_routes->proxy_post;
routes.post_tokenize = models_routes->proxy_post;
routes.post_detokenize = models_routes->proxy_post;
routes.post_apply_template = models_routes->proxy_post;
routes.post_chat_completions_tok = models_routes->proxy_post;
routes.post_responses_tok_oai = models_routes->proxy_post;
routes.get_lora_adapters = models_routes->proxy_get;
routes.post_lora_adapters = models_routes->proxy_post;
routes.get_slots = models_routes->proxy_get;
routes.post_slots = models_routes->proxy_post;
// custom routes for router
routes.get_props = models_routes->get_router_props;
routes.get_models = models_routes->get_router_models;
ctx_http.post("/models", ex_wrapper(models_routes->post_router_models));
ctx_http.post("/models/load", ex_wrapper(models_routes->post_router_models_load));
ctx_http.post("/models/unload", ex_wrapper(models_routes->post_router_models_unload));
ctx_http.get ("/models/sse", ex_wrapper(models_routes->get_router_models_sse));
ctx_http.del ("/models", ex_wrapper(models_routes->del_router_models));
server_register_routes(ctx_http, routes);
server_register_stream_routes(ctx_http,
models_routes->router_stream_get,
models_routes->router_streams_lookup,
models_routes->router_stream_delete);
if (!server_setup_features(ctx_http, params, mcp_mgr, tools, SERVER_ROLE_ROUTER)) {
return 1;
}
auto clean_up = [&models_routes, &mcp_mgr]() {
SRV_INF("%s: cleaning up before exit...\n", __func__);
// stop the session GC first, it finalizes live sessions and wakes pending readers
server_stream_session_manager_stop();
if (models_routes.has_value()) {
models_routes->stopping.store(true); // maybe redundant, but just to be safe
models_routes->models.unload_all();
}
mcp_mgr.shutdown();
llama_backend_free();
};
if (!ctx_http.start()) {
clean_up();
SRV_ERR("%s", "exiting due to HTTP server error\n");
return 1;
}
ctx_http.is_ready.store(true);
shutdown_handler = [&](int) {
if (models_routes.has_value()) {
// important to disconnect any SSE clients
models_routes->stopping.store(true);
}
mcp_mgr.shutdown();
ctx_http.stop();
};
try {
models_routes->models.load_startup_models();
} catch (const std::exception & e) {
SRV_ERR("failed to load models on startup: %s\n", e.what());
ctx_http.stop();
ctx_http.join();
clean_up();
return 1;
}
if (opts.install_signal_handlers) {
server_install_signal_handlers();
}
server_connect connect_proc;
if (!server_on_listening(ctx_http, params, connect_proc)) {
ctx_http.stop();
ctx_http.join();
clean_up();
return 1;
}
if (!params.models_preset_hf.empty()) {
SRV_WRN( "NOTE: using preset.ini from HF repo '%s'\n", params.models_preset_hf.c_str());
SRV_WRN("%s", " please only use presets that you can trust! Unknown presets may be unsafe\n");
}
ctx_http.join(); // keep the main thread alive
// when the HTTP server stops, clean up and exit
clean_up();
return 0;
}
//
// instance
//
static int server_run_instance(common_params & params, server_role & role, common_models_handler & models_handler, const server_run_opts & opts) {
// -hf / docker: download if needed and resolve the local path. a child resolves what the router downloaded
if (opts.resolve_model) {
try {
common_models_handler_apply(models_handler, params);
} catch (const std::exception & e) {
SRV_ERR("failed to download model: %s\n", e.what());
return 1;
}
}
// note: this is guaranteed to out-live ctx_http and tools
server_mcp mcp_mgr;
// struct that contains llama context and inference
server_context ctx_server;
server_routes routes(params, ctx_server);
server_tools tools;
server_http_context ctx_http;
if (!ctx_http.init(params)) {
SRV_ERR("%s", "failed to initialize HTTP server\n");
return 1;
}
server_register_routes(ctx_http, routes);
server_register_stream_routes(ctx_http,
server_stream_make_get_handler(),
server_stream_make_lookup_handler(),
server_stream_make_delete_handler());
if (!server_setup_features(ctx_http, params, mcp_mgr, tools, SERVER_ROLE_INSTANCE)) {
return 1;
}
// setup clean up function, to be called before exit
auto clean_up = [&ctx_http, &ctx_server, &mcp_mgr]() {
SRV_INF("%s: cleaning up before exit...\n", __func__);
// stop the session GC first, it finalizes live sessions and wakes pending readers
server_stream_session_manager_stop();
ctx_http.stop();
ctx_server.terminate();
mcp_mgr.shutdown();
llama_backend_free();
};
// start the HTTP server before loading the model to be able to serve /health requests
if (!ctx_http.start()) {
clean_up();
SRV_ERR("%s", "exiting due to HTTP server error\n");
return 1;
}
// setup communication child --> router if necessary
if (role.is_managed_by_router()) {
ctx_server.set_state_callback([&](server_state state, json payload) {
role.notify_to_router(server_state_to_str(state), payload);
});
}
if (!ctx_server.load_model(params)) {
clean_up();
ctx_http.join();
SRV_ERR("%s", "exiting due to model loading error\n");
return 1;
}
routes.update_meta(ctx_server);
ctx_http.is_ready.store(true);
SRV_INF("%s", "model loaded\n");
shutdown_handler = [&](int) {
mcp_mgr.shutdown();
// this will unblock start_loop()
ctx_server.terminate();
};
if (opts.install_signal_handlers) {
server_install_signal_handlers();
}
server_connect connect_proc;
if (!server_on_listening(ctx_http, params, connect_proc)) {
ctx_http.stop();
ctx_http.join();
clean_up();
return 1;
}
// optionally, notify router server that this instance is ready
std::thread monitor_thread;
if (role.is_managed_by_router()) {
monitor_thread = role.setup(shutdown_handler);
json info = routes.get_model_info();
// the router tracks device memory from these reports
if (const auto * ll_ctx = ctx_server.get_llama_context()) {
info["memory"] = server_model_memory(llama_n_ctx(ll_ctx), common_memory_breakdown_rows(ll_ctx)).to_json();
}
role.notify_to_router(server_state_to_str(SERVER_STATE_READY), info);
}
// this call blocks the main thread until queue_tasks.terminate() is called
ctx_server.start_loop();
clean_up();
ctx_http.join();
if (monitor_thread.joinable()) {
monitor_thread.join();
}
auto * ll_ctx = ctx_server.get_llama_context();
if (ll_ctx != nullptr) {
common_memory_breakdown_print(ll_ctx);
}
return 0;
}
//
// entry points
//
static int server_run(common_params & params, int argc, char ** argv, server_role & role, const server_run_opts & opts) {
common_models_handler models_handler;
// -hf may name a preset repo, which makes this a router: settle that before deciding the role
if (opts.resolve_model && !params.model.hf_repo.empty()) {
try {
models_handler = common_models_handler_init(params, LLAMA_EXAMPLE_SERVER);
if (common_models_handler_is_preset_repo(models_handler)) {
// apply the preset and start the server in router mode
common_models_handler_apply(models_handler, params);
}
} catch (const std::exception & e) {
SRV_ERR("failed to fetch model metadata: %s\n", e.what());
return 1;
}
}
const server_role_type role_type = server_role_detect(params, role);
// only an instance loads a model; the others must not touch the GPU, so skip the
// device enumeration for them and the CUDA primary context stays uncreated
common_params_print_info(params, role_type == SERVER_ROLE_INSTANCE);
if (!server_prepare_params(params, role_type)) {
return 1;
}
switch (role_type) {
case SERVER_ROLE_DOWNLOAD: return server_run_download(params, role);
case SERVER_ROLE_MEASURE: return server_run_measure(params, role, models_handler);
case SERVER_ROLE_ROUTER: return server_run_router(params, argc, argv, opts);
case SERVER_ROLE_INSTANCE: return server_run_instance(params, role, models_handler, opts);
}
GGML_ABORT("invalid server role");
}
int llama_server(int argc, char ** argv) {
server_role role;
std::setlocale(LC_NUMERIC, "C");
#ifndef _WIN32
// Ignore SIGPIPE so the server does not crash if a child (MCP server, tools runtime) exits while we are writing to its stdin
signal(SIGPIPE, SIG_IGN);
#endif
// own arguments required by this example
common_params params;
common_init();
// start the stream session manager GC right after common init, before any HTTP route can
// touch it. lifecycle is symmetric, stop_gc() runs in clean_up() before backend free
server_stream_session_manager_start();
if (!common_params_parse(argc, argv, params, LLAMA_EXAMPLE_SERVER)) {
return 1;
}
SRV_INF("%s", "initializing ...\n");
llama_backend_init();
llama_numa_init(params.numa);
const int result = server_run(params, argc, argv, role, server_run_opts{});
common_log_flush(common_log_main());
return result;
}
int llama_server(common_params & params, int argc, char ** argv) {
// embedded in llama-cli: it handles the signals, and the model is resolved already
server_run_opts opts;
opts.install_signal_handlers = false;
opts.resolve_model = false;
server_role role;
return server_run(params, argc, argv, role, opts);
}