Files
llama.cpp/tools/server/server-models.cpp
T
Aleksander Grygier 1d81992784 ui: keep downloads in Discover during search and fix their progress with a draft (#31467)
* ui : keep downloads in Discover sidebar during search, sort base-org repos first

- Show the sticky Downloading section regardless of an active search query
- Order search results so repos published by their base model's org lead

Assisted-by: Claude (Opus 5.5 Medium)

* server : throttle download progress per file

Files of a download run in parallel threads, but progress updates were
throttled with one shared timer. A small draft sidecar reports far more
often than the main weights, so it took nearly every slot and the main
file's updates, including the first one carrying its total size, were
dropped until the draft finished. The UI then showed the draft's progress
first and restarted from 0% for the model.

Throttle each file on its own and always pass a file's first update.

Assisted-by: Claude (Opus 5.5 Medium)

* ui: list a downloading repo once and match base orgs case-insensitively

A search hit that is downloading already shows under Downloading, so the
results leave it out. Base model orgs compare without case, as the hub
treats them.

Assisted-by: Claude (Opus 5.5 Medium)

* server : tell clients a download stopped on request

When a client paused or cancelled a download, every other client following
/models/sse got a bare download_failed and could not tell it from an error,
so the web UI showed "Download failed" for a pause made in the Llama app.
A stop asked for through unload or delete now fails with {"stopped": true}.

Deleting a paused download, which has no instance left, now sends
model_remove as deleting a running one does, so other clients drop its row.

Assisted-by: Claude (Opus 5.5 Medium)

* ui : follow downloads other clients start, pause or cancel

A download another client started (e.g. the Llama app) already showed up
from the feed; it now also picks up the sidecars the list reports for it.
A stop another client asked for reads as a pause instead of a failure, and
a removal clears the download row even when the list does not carry it yet.

Assisted-by: Claude (Opus 5.5 Medium)
2026-11-11 23:26:39 +01:00

2825 lines
110 KiB
C++

#include "server-common.h"
#include "http.h"
#include "server-models.h"
#include "server-context.h"
#include "server-stream.h"
#include "build-info.h"
#include "preset.h"
#include "download.h"
#include "fit.h"
#include "hf-cache.h"
#include "http.h"
#include "subproc.h"
#include <cpp-httplib/httplib.h> // TODO: remove this once we use HTTP client from download.h
#include <optional>
#include <functional>
#include <optional>
#include <algorithm>
#include <thread>
#include <mutex>
#include <condition_variable>
#include <cstring>
#include <cstdlib>
#include <atomic>
#include <chrono>
#include <deque>
#include <queue>
#include <filesystem>
#include <fstream>
#include <random>
#include <sstream>
#include <cstring>
#ifndef _WIN32
extern char **environ;
#endif
#define DEFAULT_STOP_TIMEOUT 10 // seconds
// note: SIGPIPE is ignored by the server
static void request_child_exit(server_subproc & proc) {
FILE * stdin_file = proc.sproc.stdin_file();
if (stdin_file) {
fprintf(stdin_file, "%s\n", CMD_ROUTER_TO_CHILD_EXIT);
fflush(stdin_file);
}
}
// single-threaded, watching all child processes at once
struct server_monitor {
server_monitor(server_models & models) : models(models) {
th = std::thread([this]() { run(); });
}
~server_monitor() {
push({ cmd_t::QUIT, {}, "", 0, false });
th.join();
}
// thread-safe
// kill_after > 0: force-kill the child after that many seconds
void watch(const std::string & name, std::shared_ptr<server_subproc> proc, server_role_type role, int port, int kill_after = 0) {
child_t c;
c.name = name;
c.proc = std::move(proc);
c.role = role;
c.port = port;
if (kill_after > 0) {
c.deadline = ggml_time_ms() + (int64_t) kill_after * 1000;
}
if (!c.proc->has_output()) {
SRV_ERR("failed to get stdout/stderr of child process for name=%s\n", name.c_str());
c.eof = true;
}
push({ cmd_t::WATCH, std::move(c), "", 0, false });
}
// thread-safe
void stop(const std::string & name, int stop_timeout, bool send_exit) {
push({ cmd_t::STOP, {}, name, stop_timeout, send_exit });
}
private:
struct child_t {
std::string name;
std::shared_ptr<server_subproc> proc;
server_role_type role = SERVER_ROLE_INSTANCE;
int port = 0;
std::string buf[SERVER_SUBPROC_STREAMS]; // partial line of each pipe
bool eof = false; // output closed, waiting for the process to be reaped
int64_t deadline = 0; // force-kill time in ms, 0 when no stop is pending
};
struct cmd_t {
enum { WATCH, STOP, QUIT } type;
child_t child;
std::string name;
int stop_timeout;
bool send_exit;
};
void push(cmd_t && cmd) {
{
std::lock_guard<std::mutex> lk(mu);
cmds.push_back(std::move(cmd));
}
waiter.wake();
}
// returns true if the loop should exit
bool handle_commands() {
std::deque<cmd_t> batch;
{
std::lock_guard<std::mutex> lk(mu);
batch.swap(cmds);
}
for (auto & cmd : batch) {
switch (cmd.type) {
case cmd_t::WATCH:
children.push_back(std::move(cmd.child));
break;
case cmd_t::STOP:
// the newest child with this name is the one the registry knows
for (auto it = children.rbegin(); it != children.rend(); ++it) {
// a measure child has the same name, skip it
if (it->name != cmd.name || it->role == SERVER_ROLE_MEASURE) {
continue;
}
if (cmd.send_exit && !it->eof) {
request_child_exit(*it->proc);
}
it->deadline = ggml_time_ms() + (int64_t) cmd.stop_timeout * 1000;
break;
}
break;
case cmd_t::QUIT:
return true;
}
}
return false;
}
// read what the child wrote, handle its commands and forward its logs, line by line
void read_output(child_t & c) {
for (int i = 0; i < SERVER_SUBPROC_STREAMS; i++) {
read_stream(c, (server_subproc_stream) i);
}
c.eof = c.proc->output_closed();
}
void read_stream(child_t & c, server_subproc_stream stream) {
char chunk[4096];
std::string & buf = c.buf[stream];
bool closed = false;
while (true) {
int n = c.proc->read_output(stream, chunk, sizeof(chunk));
if (n < 0) {
closed = true;
break;
}
if (n == 0) {
break;
}
buf.append(chunk, (size_t) n);
size_t start = 0;
while (true) {
size_t nl = buf.find('\n', start);
if (nl == std::string::npos) {
break;
}
on_line(c, stream, buf.substr(start, nl + 1 - start));
start = nl + 1;
}
buf.erase(0, start);
if (buf.size() > max_line) {
buf.clear(); // a child that never writes a newline must not grow this without bound
}
}
if (closed && !buf.empty()) {
on_line(c, stream, buf);
buf.clear();
}
}
void on_line(child_t & c, server_subproc_stream stream, const std::string & line) {
if (stream == SERVER_SUBPROC_STDERR) {
LOG("[%5d] %s", c.port, line.c_str()); // forward log
} else if (string_starts_with(line, CMD_CHILD_TO_ROUTER_STATE)) {
LOG_DBG("[%5d] %s", c.port, line.c_str()); // prevent spamming the log
models.handle_child_state(c.name, line);
} else {
SRV_WRN("[%5d] unexpected output on the command pipe: %s", c.port, line.c_str());
}
}
void run() {
while (true) {
if (handle_commands()) {
return;
}
// wait for output, a wakeup, or the next deadline;
// a child whose output closed is polled for its exit every 50 ms
int64_t now = ggml_time_ms();
int64_t timeout = -1;
for (const auto & c : children) {
if (c.eof) {
timeout = timeout < 0 ? 50 : std::min<int64_t>(timeout, 50);
}
if (c.deadline) {
int64_t d = std::max<int64_t>(0, c.deadline - now);
timeout = timeout < 0 ? d : std::min(timeout, d);
}
}
std::vector<server_subproc *> procs;
std::vector<child_t *> owners;
for (auto & c : children) {
if (!c.eof) {
procs.push_back(c.proc.get());
owners.push_back(&c);
}
}
std::vector<bool> ready;
waiter.wait(procs, ready, timeout);
for (size_t i = 0; i < owners.size(); i++) {
if (ready[i]) {
read_output(*owners[i]);
}
}
// deadlines and exits
now = ggml_time_ms();
for (auto it = children.begin(); it != children.end();) {
if (it->deadline && now >= it->deadline && !it->proc->stopped.load(std::memory_order_acquire)) {
SRV_WRN("force-killing child process for name=%s after timeout\n", it->name.c_str());
it->proc->terminate();
it->deadline = 0;
}
if (it->eof && !it->proc->is_alive()) {
int exit_code = it->proc->join();
it->proc->stopped.store(true, std::memory_order_release);
models.on_child_exit(it->name, it->proc, it->role, exit_code);
SRV_INF("instance name=%s exited with status %d\n", it->name.c_str(), exit_code);
it = children.erase(it);
} else {
++it;
}
}
}
}
static constexpr size_t max_line = 1024 * 1024;
server_models & models;
std::mutex mu;
std::deque<cmd_t> cmds;
std::vector<child_t> children; // monitor thread only
server_subproc::waiter waiter;
std::thread th;
};
// short loopback budget for the resumable stream router to child JSON calls (probe, lookup,
// delete). distinct from params.timeout_read/write which only applies to the generation proxy
static constexpr int STREAM_LOOKUP_TIMEOUT_MS = 250;
// router already checked the origin; child sees a loopback Host, so do not forward origin headers
static bool is_browser_origin_header(const std::string & key) {
std::string lower = key;
std::transform(lower.begin(), lower.end(), lower.begin(), ::tolower);
return lower == "origin" || lower.compare(0, 10, "sec-fetch-") == 0;
}
static std::map<std::string, std::string> child_request_headers(const std::map<std::string, std::string> & headers) {
std::map<std::string, std::string> out;
for (const auto & [key, value] : headers) {
if (!is_browser_origin_header(key)) {
out.emplace(key, value);
}
}
return out;
}
static void unset_reserved_args(common_preset & preset, bool unset_model_args) {
preset.unset_option("LLAMA_ARG_SSL_KEY_FILE");
preset.unset_option("LLAMA_ARG_SSL_CERT_FILE");
preset.unset_option("LLAMA_API_KEY");
preset.unset_option("LLAMA_ARG_API_KEY_FILE");
preset.unset_option("LLAMA_ARG_MODELS_DIR");
preset.unset_option("LLAMA_ARG_MODELS_MAX");
preset.unset_option("LLAMA_ARG_MODELS_MEMORY_MARGIN");
preset.unset_option("LLAMA_ARG_MODELS_PRESET");
preset.unset_option("LLAMA_ARG_MODELS_AUTOLOAD");
preset.unset_option("LLAMA_ARG_CONNECT");
preset.unset_option("LLAMA_ARG_CONNECT_CODE");
if (unset_model_args) {
preset.unset_option("LLAMA_ARG_MODEL");
preset.unset_option("LLAMA_ARG_SHARD");
preset.unset_option("LLAMA_ARG_MMPROJ");
preset.unset_option("LLAMA_ARG_ALIAS");
preset.unset_option("LLAMA_ARG_HF_REPO");
}
}
static std::vector<std::string> get_environment() {
std::vector<std::string> env;
#ifdef _WIN32
LPWCH env_block = GetEnvironmentStringsW();
if (!env_block) {
return env;
}
for (LPWCH e = env_block; *e; e += wcslen(e) + 1) {
env.emplace_back(wstring_to_utf8(e));
}
FreeEnvironmentStringsW(env_block);
#else
if (environ == nullptr) {
return env;
}
for (char ** e = environ; *e != nullptr; e++) {
env.emplace_back(*e);
}
#endif
return env;
}
void server_model_meta::update_args(common_preset_context & ctx_preset, std::string bin_path) {
// update params
unset_reserved_args(preset, false);
preset.set_option(ctx_preset, "LLAMA_ARG_HOST", CHILD_ADDR);
preset.set_option(ctx_preset, "LLAMA_ARG_PORT", std::to_string(port));
preset.set_option(ctx_preset, "LLAMA_ARG_ALIAS", name);
// the child output goes through the router to its terminal, so it follows the router colors
preset.set_option(ctx_preset, "LLAMA_ARG_LOG_COLORS", common_log_get_colors(common_log_main()) ? "on" : "off");
// TODO: maybe validate preset before rendering ?
// render args
args = preset.to_args(bin_path);
// unified binary dispatches by subcommand, re-inject it right after the
// binary path so the child starts as 'llama serve ...' not 'llama ...'
const char * app_cmd = std::getenv("LLAMA_APP_CMD");
if (app_cmd != nullptr && app_cmd[0] != '\0' && !bin_path.empty()) {
args.insert(args.begin() + 1, app_cmd);
}
}
void server_model_meta::update_caps(const common_params & base) {
// reset to the default so a failed refresh cannot keep old values
architecture = server_model_architecture_json(false, false, false, {"text"});
n_ctx_train = 0;
// resolve the model file offline; do not download
common_params params;
params.model = base.model;
// --no-mmproj applies to child models and blocks auto-attached projectors
params.no_mmproj = base.no_mmproj;
try {
preset.apply_to_params(params, {
"LLAMA_ARG_MODEL",
"LLAMA_ARG_MODEL_URL",
"LLAMA_ARG_MMPROJ",
"LLAMA_ARG_MMPROJ_URL",
"LLAMA_ARG_MMPROJ_AUTO",
"LLAMA_ARG_HF_REPO",
"LLAMA_ARG_HF_FILE",
});
params.offline = true;
common_models_handler handler = common_models_handler_init(params, LLAMA_EXAMPLE_SERVER);
common_models_handler_apply(handler, params);
} catch (const std::exception & e) {
LOG_WRN("failed to resolve the model of '%s': %s\n", name.c_str(), e.what());
return;
}
bool inp_image = false;
bool inp_audio = false;
bool gen_audio = false;
try {
if (!params.no_mmproj && !params.mmproj.path.empty()) {
mtmd_caps caps = mtmd_get_cap_from_file(params.mmproj.path.c_str());
inp_image = caps.inp_vision;
inp_audio = caps.inp_audio;
gen_audio = caps.gen_audio;
}
} catch (const std::exception & e) {
LOG_WRN("failed to read the multimodal capabilities of '%s': %s\n", name.c_str(), e.what());
// keep the output modalities from the GGUF metadata
}
// read the output modalities and the trained context from the GGUF metadata
std::vector<std::string> output_modalities = {"text"};
if (!params.model.path.empty()) {
const common_gguf_info info = common_get_gguf_info(params.model.path);
// same rule as mtmd_helper_get_capabilities()
const bool can_chat = !gen_audio || info.has_chat_tmpl;
output_modalities = server_model_output_modalities(info.decision_type, can_chat, gen_audio);
n_ctx_train = info.n_ctx_train;
}
// offline discovery cannot see video; a loaded model reports it
architecture = server_model_architecture_json(inp_image, inp_audio, false, output_modalities);
}
//
// server_models
//
server_models::server_models(
const common_params & params,
int argc,
char ** argv)
: ctx_preset(LLAMA_EXAMPLE_SERVER),
base_params(params),
base_env(get_environment()),
base_preset(ctx_preset.load_from_args(argc, argv)),
sched(std::make_unique<server_models_sched>(*this)),
monitor(std::make_unique<server_monitor>(*this)) {
// propagate base params to child
unset_reserved_args(base_preset, true);
// do not propagate these options, but allow preset to explicitly set them
base_preset.unset_option("LLAMA_ARG_LOG_FILE");
// set binary path
try {
bin_path = get_server_exec_path().string();
} catch (const std::exception & e) {
bin_path = argv[0];
SRV_WRN("failed to get server executable path: %s\n", e.what());
SRV_WRN("using original argv[0] as fallback: %s\n", argv[0]);
}
unfinished_downloads.load();
load_models();
debug_fake_timing = !common_get_env("LLAMA_SERVER_DEBUG_FAKE_TIMING").empty();
}
server_models::~server_models() = default;
void server_models::instance_t::request_exit() const {
request_child_exit(*subproc);
}
void server_models::add_model(server_model_meta && meta) {
if (mapping.find(meta.name) != mapping.end()) {
throw std::runtime_error(string_format("model '%s' appears multiple times", meta.name.c_str()));
}
// check model name does not conflict with existing aliases
for (const auto & [key, inst] : mapping) {
if (inst.meta.aliases.count(meta.name)) {
throw std::runtime_error(string_format("model name '%s' conflicts with alias of model '%s'",
meta.name.c_str(), key.c_str()));
}
}
// parse aliases from preset's --alias option (comma-separated)
std::string alias_str;
if (meta.preset.get_option("LLAMA_ARG_ALIAS", alias_str) && !alias_str.empty()) {
for (auto & alias : string_split<std::string>(alias_str, ',')) {
alias = string_strip(alias);
if (!alias.empty()) {
meta.aliases.insert(alias);
}
}
}
// parse tags from preset's --tags option (comma-separated)
std::string tags_str;
if (meta.preset.get_option("LLAMA_ARG_TAGS", tags_str) && !tags_str.empty()) {
for (auto & tag : string_split<std::string>(tags_str, ',')) {
tag = string_strip(tag);
if (!tag.empty()) {
meta.tags.insert(tag);
}
}
}
// validate aliases do not conflict with existing names or aliases
for (const auto & alias : meta.aliases) {
if (mapping.find(alias) != mapping.end()) {
throw std::runtime_error(string_format("alias '%s' for model '%s' conflicts with existing model name",
alias.c_str(), meta.name.c_str()));
}
for (const auto & [key, inst] : mapping) {
if (inst.meta.aliases.count(alias)) {
throw std::runtime_error(string_format("alias '%s' for model '%s' conflicts with alias of model '%s'",
alias.c_str(), meta.name.c_str(), key.c_str()));
}
}
}
meta.update_args(ctx_preset, bin_path); // render args
meta.update_caps(base_params);
std::string name = meta.name;
mapping[name] = instance_t{
/* subproc */ std::make_shared<server_subproc>(),
/* meta */ std::move(meta)
};
}
static bool has_sidecar(const std::vector<std::string> & sidecars, const std::string & kind) {
return std::find(sidecars.begin(), sidecars.end(), kind) != sidecars.end();
}
// reject a sidecar kind no download can pull
static void check_sidecar(const std::string & kind) {
std::vector<std::string> kinds;
for (const auto & s : common_download_sidecars()) {
kinds.emplace_back(s.name);
}
if (!has_sidecar(kinds, kind)) {
throw std::invalid_argument("unknown sidecar '" + kind + "', expected one of " + string_join(kinds, ", "));
}
}
// the sidecars a download asks for; by default the ones -hf pulls without a draft type
static std::vector<std::string> parse_sidecars(const json & body) {
std::vector<std::string> defaults;
for (const auto & s : common_download_sidecars()) {
if (!s.draft) {
defaults.emplace_back(s.name);
}
}
std::vector<std::string> sidecars = json_value(body, "sidecars", defaults);
for (const auto & kind : sidecars) {
check_sidecar(kind);
}
return sidecars;
}
// the download options of a model with its sidecars, the plan a removal purges partials from
static common_download_opts sidecar_download_opts(const common_params & params, const std::vector<std::string> & sidecars) {
common_download_opts opts;
opts.bearer_token = params.hf_token;
for (const auto & kind : sidecars) {
common_download_opts_add_sidecar(opts, kind);
}
return opts;
}
// the HF repo a model resolves from: a preset names its own, a cached model is
// listed under it, and a local file or a models dir entry has none
static std::string model_hf_repo(const server_model_meta & meta) {
std::string hf_repo;
if (meta.preset.get_option("LLAMA_ARG_HF_REPO", hf_repo) && !hf_repo.empty()) {
return hf_repo;
}
return meta.source == SERVER_MODEL_SOURCE_CACHE ? meta.name : "";
}
// the sidecar kinds of a HF repo present on disk, as the offline plan resolves them
static json cached_sidecars(const std::string & hf_repo) {
common_params_model model;
model.hf_repo = hf_repo;
common_download_opts opts;
opts.offline = true;
for (const auto & s : common_download_sidecars()) {
opts.*s.opt = true;
}
return common_download_plan_sidecars(common_download_get_hf_plan(model, opts));
}
// replace any inherited value of `key` in a child environment
static void set_child_env(std::vector<std::string> & env, const std::string & key, const std::string & value) {
env.erase(std::remove_if(env.begin(), env.end(), [&](const std::string & e) {
return e.rfind(key + "=", 0) == 0;
}), env.end());
env.push_back(key + "=" + value);
}
// where the unfinished downloads are remembered, inside the model cache
static std::string unfinished_downloads_path() {
return hf_cache::get_cache_path() + "/unfinished-downloads.json";
}
void server_unfinished_downloads::load() {
std::ifstream in(unfinished_downloads_path());
if (!in) {
return;
}
std::string raw((std::istreambuf_iterator<char>(in)), std::istreambuf_iterator<char>());
std::lock_guard<std::mutex> lk(mutex);
try {
json data = json::parse(raw);
for (const auto & d : data.value("downloads", json::array())) {
server_unfinished_download download;
download.model = json_value(d, "model", std::string());
download.sidecars = json_value(d, "sidecars", std::vector<std::string>());
for (const auto & p : d.value("partials", json::array())) {
download.partials.push_back({ json_value(p, "kind", std::string()), json_value(p, "path", std::string()) });
}
if (!download.model.empty()) {
downloads.push_back(std::move(download));
}
}
} catch (const std::exception & e) {
LOG_WRN("failed to read unfinished downloads: %s\n", e.what());
}
}
void server_unfinished_downloads::save() {
json data;
data["downloads"] = json::array();
for (const auto & d : downloads) {
json partials = json::array();
for (const auto & p : d.partials) {
partials.push_back({ {"kind", p.kind}, {"path", p.path} });
}
data["downloads"].push_back({ {"model", d.model}, {"sidecars", d.sidecars}, {"partials", partials} });
}
try {
fs_write_atomic(unfinished_downloads_path(), data.dump());
} catch (const std::exception & e) {
LOG_WRN("failed to record unfinished downloads: %s\n", e.what());
}
}
void server_unfinished_downloads::delete_partials(const std::vector<common_download_partial> & parts) {
for (const auto & p : parts) {
const bool shared = std::any_of(downloads.begin(), downloads.end(), [&](const server_unfinished_download & d) {
return std::any_of(d.partials.begin(), d.partials.end(), [&](const common_download_partial & q) {
return q.path == p.path;
});
});
if (shared) {
continue;
}
std::error_code ec;
std::filesystem::remove(p.path, ec);
}
}
void server_unfinished_downloads::add(const server_unfinished_download & download) {
std::lock_guard<std::mutex> lk(mutex);
std::vector<common_download_partial> stale;
auto it = std::find_if(downloads.begin(), downloads.end(), [&](const auto & d) { return d.model == download.model; });
if (it != downloads.end()) {
stale = std::move(it->partials);
*it = download;
} else {
downloads.push_back(download);
}
save();
delete_partials(stale);
}
void server_unfinished_downloads::forget(const std::string & model) {
std::lock_guard<std::mutex> lk(mutex);
auto it = std::find_if(downloads.begin(), downloads.end(), [&](const auto & d) { return d.model == model; });
if (it != downloads.end()) {
downloads.erase(it);
save();
}
}
bool server_unfinished_downloads::remove(const std::string & model) {
std::lock_guard<std::mutex> lk(mutex);
auto it = std::find_if(downloads.begin(), downloads.end(), [&](const auto & d) { return d.model == model; });
if (it == downloads.end()) {
return false;
}
auto partials = std::move(it->partials);
downloads.erase(it);
save();
delete_partials(partials);
return true;
}
bool server_unfinished_downloads::remove_sidecar(const std::string & model, const std::string & kind) {
std::lock_guard<std::mutex> lk(mutex);
auto it = std::find_if(downloads.begin(), downloads.end(), [&](const auto & d) { return d.model == model; });
if (it == downloads.end() || !has_sidecar(it->sidecars, kind)) {
return false;
}
it->sidecars.erase(std::remove(it->sidecars.begin(), it->sidecars.end(), kind), it->sidecars.end());
std::vector<common_download_partial> dropped;
auto & partials = it->partials;
auto pos = std::stable_partition(partials.begin(), partials.end(), [&](const common_download_partial & p) {
return p.kind != kind;
});
dropped.assign(pos, partials.end());
partials.erase(pos, partials.end());
save();
delete_partials(dropped);
return true;
}
std::vector<server_unfinished_download> server_unfinished_downloads::get_all() {
std::lock_guard<std::mutex> lk(mutex);
return downloads;
}
std::vector<server_unfinished_download> server_models::unfinished_downloads_get() {
return unfinished_downloads.get_all();
}
void server_models::unfinished_downloads_remove(const std::string & name) {
// under the lock, so a download of the same tag cannot start while its files go
std::lock_guard<std::mutex> lk(mutex);
if (mapping.find(name) != mapping.end()) {
throw std::invalid_argument("model '" + name + "' is downloading");
}
// a name with no record is no model: its files belong to the cached tags of the repo
if (!unfinished_downloads.remove(name)) {
throw std::runtime_error("model name=" + name + " is not found");
}
bool ok = common_download_remove(name);
SRV_INF("removing unfinished download name=%s (%s)\n", name.c_str(), ok ? "succeeded" : "nothing cached");
// other clients still list it as a paused download
notify_sse("model_remove", name, {});
}
bool server_models::remove_sidecar(const std::string & name, const std::string & kind) {
std::lock_guard<std::mutex> lk(mutex);
auto it = mapping.find(name);
if (it != mapping.end() && (it->second.meta.is_running() || it->second.meta.status == SERVER_MODEL_STATUS_DOWNLOADING)) {
throw std::invalid_argument("model '" + name + "' is in use, unload it first");
}
// a sidecar that only ever landed as a partial is still something removed
bool ok = unfinished_downloads.remove_sidecar(name, kind);
ok = common_download_remove_sidecars(name, sidecar_download_opts(base_params, { kind })) || ok;
SRV_INF("removing sidecar %s of model name=%s (%s)\n", kind.c_str(), name.c_str(), ok ? "succeeded" : "nothing cached");
return ok;
}
void server_models::notify_sse(const std::string & event, const std::string & model_id, const json & data) {
std::unique_ptr<server_task_result_router> result = std::make_unique<server_task_result_router>();
result->data = {
{"model", model_id},
{"event", event},
};
if (!data.is_null()) {
result->data["data"] = data;
}
SRV_DBG("notifying SSE clients about event '%s' for model '%s': %s\n", event.c_str(), model_id.c_str(), safe_json_to_str(result->data).c_str());
sse.broadcast(std::move(result));
}
void server_models::load_models() {
// Phase 1: load presets from all sources - pure I/O, no lock needed
// 1. cached models
common_presets cached_models = ctx_preset.load_from_cache();
SRV_TRC("Loaded %zu cached model presets from %s\n", cached_models.size(), hf_cache::get_cache_path().c_str());
// 2. local models from --models-dir
common_presets local_models;
if (!base_params.models_dir.empty()) {
local_models = ctx_preset.load_from_models_dir(base_params.models_dir);
SRV_TRC("Loaded %zu local model presets from %s\n", local_models.size(), base_params.models_dir.c_str());
}
// 3. custom-path models from presets
common_preset global = {};
common_presets custom_presets = {};
if (!base_params.models_preset.empty()) {
custom_presets = ctx_preset.load_from_ini(base_params.models_preset, global);
SRV_TRC("Loaded %zu custom model presets from %s\n", custom_presets.size(), base_params.models_preset.c_str());
}
// cascade, apply global preset first
cached_models = ctx_preset.cascade(global, cached_models);
local_models = ctx_preset.cascade(global, local_models);
custom_presets = ctx_preset.cascade(global, custom_presets);
// note: if a model exists in both cached and local, local takes precedence
common_presets final_presets;
std::unordered_map<std::string, server_model_source> source_map;
for (const auto & [name, preset] : cached_models) {
final_presets[name] = preset;
source_map[name] = SERVER_MODEL_SOURCE_CACHE;
}
for (const auto & [name, preset] : local_models) {
final_presets[name] = preset;
source_map[name] = SERVER_MODEL_SOURCE_MODELS_DIR;
}
for (const auto & [name, custom] : custom_presets) {
if (final_presets.find(name) != final_presets.end()) {
final_presets[name].merge(custom);
} else {
final_presets[name] = custom;
}
source_map[name] = SERVER_MODEL_SOURCE_PRESET;
}
// overlay router's own CLI args on top of every model preset so that
// e.g. `llama-server --temp 0` is honoured by all child processes
for (auto & [name, preset] : final_presets) {
preset.merge(base_preset);
}
auto get_source = [&](const std::string & name) {
return source_map.count(name) ? source_map.at(name) : SERVER_MODEL_SOURCE_PRESET;
};
// hide cache models whose resolved file is already used by a preset with dedup-cache-models enabled
std::set<std::string> hidden_models;
{
std::set<std::string> preset_paths;
auto add_hf_path = [&preset_paths](const common_preset & preset, const char * repo_key, const char * file_key) {
std::string hf_repo;
if (!preset.get_option(repo_key, hf_repo) || hf_repo.empty()) {
return;
}
std::string hf_file;
preset.get_option(file_key, hf_file);
std::string path = common_download_resolve_path(hf_repo, hf_file);
if (!path.empty()) {
preset_paths.insert(path);
}
};
for (const auto & [name, preset] : custom_presets) {
std::string val;
if (!preset.get_option(COMMON_ARG_PRESET_DEDUP_CACHE_MODELS, val) || !common_arg_utils::is_truthy(val)) {
continue;
}
add_hf_path(preset, "LLAMA_ARG_HF_REPO", "LLAMA_ARG_HF_FILE");
add_hf_path(preset, "LLAMA_ARG_SPEC_DRAFT_HF_REPO", "LLAMA_ARG_SPEC_DRAFT_MODEL");
}
if (!preset_paths.empty()) {
for (const auto & [name, preset] : cached_models) {
if (get_source(name) != SERVER_MODEL_SOURCE_CACHE) {
continue; // merged with another source, not a pure cache entry
}
std::string path = common_download_resolve_path(name);
if (!path.empty() && preset_paths.count(path)) {
SRV_INF("hiding cache model name=%s (deduplicated by a preset)\n", name.c_str());
hidden_models.insert(name);
}
}
}
}
// Helpers that read `mapping` - must be called while holding the lock.
auto join_set = [](const std::set<std::string> & s) {
std::string result;
for (const auto & v : s) {
if (!result.empty()) result += ", ";
result += v;
}
return result;
};
auto log_available_models = [&]() {
SRV_INF("Available models (%zu):\n", mapping.size());
if (mapping.empty()) {
SRV_INF("%s", " no models found on the system (visit https://llama.app/models for suggestions)\n");
} else {
for (const auto & [name, inst] : mapping) {
const std::string source = server_model_source_to_string(inst.meta.source);
std::string info;
if (!inst.meta.aliases.empty()) info += " (aliases: " + join_set(inst.meta.aliases) + ")";
if (!inst.meta.tags.empty()) info += " [tags: " + join_set(inst.meta.tags) + "]";
SRV_INF(" [%10s] %s%s\n", source.c_str(), name.c_str(), info.c_str());
}
}
};
auto apply_stop_timeout = [&]() {
for (auto & [name, inst] : mapping) {
std::string val;
if (inst.meta.preset.get_option(COMMON_ARG_PRESET_STOP_TIMEOUT, val)) {
try {
inst.meta.stop_timeout = std::stoi(val);
} catch (...) {
SRV_WRN("invalid stop-timeout value '%s' for model '%s', using default %d seconds\n",
val.c_str(), name.c_str(), DEFAULT_STOP_TIMEOUT);
inst.meta.stop_timeout = DEFAULT_STOP_TIMEOUT;
}
}
}
};
auto apply_hidden = [&]() {
for (auto & [name, inst] : mapping) {
inst.meta.hidden = hidden_models.count(name) > 0;
}
};
// update_args() injects HOST/PORT/ALIAS/LOG_COLORS, so strip them before comparing presets
auto preset_options_for_compare = [](common_preset p) {
p.unset_option("LLAMA_ARG_HOST");
p.unset_option("LLAMA_ARG_PORT");
p.unset_option("LLAMA_ARG_ALIAS");
p.unset_option("LLAMA_ARG_LOG_COLORS");
return p.options;
};
// Phase 2: acquire the lock once for all mapping mutations.
// We temporarily release it only when calling functions that acquire it internally (unload)
std::unique_lock<std::mutex> lk(mutex);
need_reload = false;
bool is_first_load = mapping.empty();
if (is_first_load) {
// FIRST LOAD: add all models, then unlock for autoloading
for (const auto & [name, preset] : final_presets) {
server_model_meta meta{
/* source */ get_source(name),
/* preset */ preset,
/* name */ name,
/* aliases */ {},
/* tags */ {},
/* port */ 0,
/* status */ SERVER_MODEL_STATUS_UNLOADED,
/* last_used */ 0,
/* args */ std::vector<std::string>(),
/* loaded_info */ {},
/* progress */ {},
/* exit_code */ 0,
/* stop_timeout */ DEFAULT_STOP_TIMEOUT,
// /* need_download */ false,
};
add_model(std::move(meta));
}
apply_stop_timeout();
apply_hidden();
log_available_models();
sched->measure_unmeasured(lk);
// skipped on reload, see startup_models
if (startup_models.has_value()) {
std::vector<std::string> models_to_load;
for (const auto & [name, inst] : mapping) {
std::string val;
if (inst.meta.preset.get_option(COMMON_ARG_PRESET_LOAD_ON_STARTUP, val) && common_arg_utils::is_truthy(val)) {
models_to_load.push_back(name);
}
}
if ((int)models_to_load.size() > base_params.models_max) {
throw std::runtime_error(string_format(
"number of models to load on startup (%zu) exceeds models_max (%d)",
models_to_load.size(), base_params.models_max));
}
// to be lazy-loaded after main() setup phase is completed
startup_models = std::move(models_to_load);
}
lk.unlock();
} else {
// RELOAD: diff the new preset list against the current mapping and reconcile
is_reloading = true;
// find running models whose source was removed or whose preset changed
std::vector<std::string> to_unload;
for (const auto & [name, inst] : mapping) {
if (!inst.meta.is_running()) continue;
auto it = final_presets.find(name);
if (it == final_presets.end()) {
to_unload.push_back(name); // removed from source
} else if (preset_options_for_compare(inst.meta.preset) != preset_options_for_compare(it->second)) {
to_unload.push_back(name); // preset changed
}
}
// unload() acquires the lock internally, so release before each call
for (const auto & name : to_unload) {
SRV_INF("(reload) unloading model name=%s (source updated or removed)\n", name.c_str());
lk.unlock();
unload(name);
lk.lock();
}
// wait for all targeted models to reach UNLOADED; cv.wait handles unlock/relock
cv.wait(lk, [&]() {
for (const auto & name : to_unload) {
auto it = mapping.find(name);
if (it != mapping.end() && it->second.meta.is_running()) return false;
}
return true;
});
// erase models no longer in any source
for (auto it = mapping.begin(); it != mapping.end(); ) {
if (it->second.meta.status == SERVER_MODEL_STATUS_DOWNLOADING) {
++it; // download thread is still busy, skip
} else if (it->second.meta.status == SERVER_MODEL_STATUS_DOWNLOADED) {
// download finished, safe to erase
it = mapping.erase(it);
} else if (final_presets.find(it->first) == final_presets.end()) {
SRV_INF("(reload) removing model name=%s (no longer in source)\n", it->first.c_str());
it = mapping.erase(it);
} else {
++it;
}
}
// update presets for non-running models still in source
for (auto & [name, inst] : mapping) {
if (inst.meta.is_running()) continue;
auto it = final_presets.find(name);
if (it == final_presets.end()) continue; // erased above
if (preset_options_for_compare(inst.meta.preset) != preset_options_for_compare(it->second)) {
// different args, different memory: measure again
inst.meta.memory = {};
}
inst.meta.preset = it->second;
// re-parse aliases, then validate against other models
std::set<std::string> new_aliases;
std::string alias_str;
if (inst.meta.preset.get_option("LLAMA_ARG_ALIAS", alias_str) && !alias_str.empty()) {
for (auto & alias : string_split<std::string>(alias_str, ',')) {
alias = string_strip(alias);
if (!alias.empty()) new_aliases.insert(alias);
}
}
inst.meta.aliases.clear();
for (const auto & alias : new_aliases) {
bool conflict = false;
for (const auto & [other_name, other_inst] : mapping) {
if (other_name == name) continue;
if (other_name == alias || other_inst.meta.aliases.count(alias)) {
SRV_WRN("(reload) alias '%s' for model '%s' conflicts with model '%s', skipping\n",
alias.c_str(), name.c_str(), other_name.c_str());
conflict = true;
break;
}
}
if (!conflict) inst.meta.aliases.insert(alias);
}
// re-parse tags
inst.meta.tags.clear();
std::string tags_str;
if (inst.meta.preset.get_option("LLAMA_ARG_TAGS", tags_str) && !tags_str.empty()) {
for (auto & tag : string_split<std::string>(tags_str, ',')) {
tag = string_strip(tag);
if (!tag.empty()) inst.meta.tags.insert(tag);
}
}
inst.meta.exit_code = 0; // clear failed state so the model can be reloaded
inst.meta.update_args(ctx_preset, bin_path);
inst.meta.update_caps(base_params);
}
// add models that are new in this reload, load-on-startup is not honored here since a
// reload never spawns an instance
for (const auto & [name, preset] : final_presets) {
if (mapping.find(name) == mapping.end()) {
server_model_meta meta{
/* source */ get_source(name),
/* preset */ preset,
/* name */ name,
/* aliases */ {},
/* tags */ {},
/* port */ 0,
/* status */ SERVER_MODEL_STATUS_UNLOADED,
/* last_used */ 0,
/* args */ std::vector<std::string>(),
/* loaded_info */ {},
/* progress */ {},
/* exit_code */ 0,
/* stop_timeout */ DEFAULT_STOP_TIMEOUT,
// /* need_download */ false,
};
add_model(std::move(meta));
}
}
apply_stop_timeout();
apply_hidden();
sched->measure_unmeasured(lk);
// clear reload flag under the lock, this releases the load() calls waiting on !is_reloading
is_reloading = false;
cv.notify_all();
log_available_models();
lk.unlock();
notify_sse("models_reload", "*");
}
}
void server_models::load_startup_models() {
std::vector<std::string> to_load;
{
std::lock_guard<std::mutex> lk(mutex);
if (!startup_models.has_value()) {
return; // already drained
}
to_load = std::move(*startup_models);
startup_models.reset();
}
for (const auto & name : to_load) {
SRV_INF("(startup) loading model %s\n", name.c_str());
load(name);
}
}
void server_models::update_meta(const std::string & name, const server_model_meta & meta) {
std::lock_guard<std::mutex> lk(mutex);
auto it = mapping.find(name);
if (it != mapping.end()) {
it->second.meta = meta;
}
cv.notify_all(); // notify wait_until_loading_finished
}
bool server_models::has_model(const std::string & name) {
std::lock_guard<std::mutex> lk(mutex);
if (mapping.find(name) != mapping.end()) {
return true;
}
for (const auto & [key, inst] : mapping) {
if (inst.meta.aliases.count(name)) {
return true;
}
}
return false;
}
std::optional<server_model_meta> server_models::get_meta(const std::string & name) {
std::unique_lock<std::mutex> lk(mutex);
if (need_reload) {
lk.unlock();
load_models();
lk.lock();
}
auto it = mapping.find(name);
if (it != mapping.end()) {
return it->second.meta;
}
for (const auto & [key, inst] : mapping) {
if (inst.meta.aliases.count(name)) {
return inst.meta;
}
}
return std::nullopt;
}
std::vector<server_model_meta> server_models::get_all_meta() {
std::unique_lock<std::mutex> lk(mutex);
if (need_reload) {
lk.unlock();
load_models();
lk.lock();
}
std::vector<server_model_meta> result;
result.reserve(mapping.size());
for (const auto & [name, inst] : mapping) {
result.push_back(inst.meta);
}
return result;
}
void server_models::load(const std::string & name) {
load(name, load_options{});
}
void server_models::load(const std::string & name, const load_options & opts) {
// the fake load time of a model; a download worker loads none
if (debug_fake_timing && opts.role == SERVER_ROLE_INSTANCE) {
// do not hold the mutex here, other requests must keep making progress
std::this_thread::sleep_for(std::chrono::seconds(2));
}
if (!opts.custom_meta.has_value()) {
if (!has_model(name)) {
throw std::runtime_error("model name=" + name + " is not found");
}
if (opts.role == SERVER_ROLE_INSTANCE) {
sched->make_room(name);
}
}
std::unique_lock<std::mutex> lk(mutex);
// edge case: block until any in-progress reload has finished so we always load
// against the freshest preset and a consistent mapping state
cv.wait(lk, [this]() { return !is_reloading; });
// a download over an existing entry is decided under the lock, so a load of the
// same model cannot start in between: a paused download or an idle cached model
// is downloaded again with its files kept, a download on its way and a running
// model are refused, and a preset or local model is not a download
if (opts.role == SERVER_ROLE_DOWNLOAD) {
auto it = mapping.find(name);
if (it != mapping.end()) {
const auto & cur = it->second.meta;
if (cur.status == SERVER_MODEL_STATUS_DOWNLOADING) {
throw std::invalid_argument("model '" + name + "' is already downloading");
}
if (cur.is_running()) {
throw std::invalid_argument("model '" + name + "' is running, unload it first");
}
if (cur.source != SERVER_MODEL_SOURCE_CACHE) {
throw std::invalid_argument("model '" + name + "' already exists");
}
SRV_INF("resuming download for model '%s'\n", name.c_str());
}
// two downloads writing the same partial would interleave their bytes
for (const auto & d : unfinished_downloads.get_all()) {
auto other = mapping.find(d.model);
if (d.model == name || other == mapping.end() || other->second.meta.status != SERVER_MODEL_STATUS_DOWNLOADING) {
continue;
}
for (const auto & p : d.partials) {
const bool shared = std::any_of(opts.partials.begin(), opts.partials.end(), [&](const common_download_partial & q) {
return q.path == p.path;
});
if (shared) {
throw std::invalid_argument("model '" + name + "' shares files with the download of '" + d.model + "' on its way, retry once it is done");
}
}
}
}
auto meta = opts.custom_meta.has_value() ? *opts.custom_meta : mapping[name].meta;
if (meta.status != SERVER_MODEL_STATUS_UNLOADED) {
SRV_INF("model %s is not ready\n", name.c_str());
return;
}
// Re-check capacity under the lock to prevent concurrent loads from
// exceeding models_max. Without this, the window between make_room()
// releasing its lock and this lock_guard acquiring allows multiple
// threads to each observe capacity and all proceed to load.
// Download workers do not use models_max slots.
if (opts.role == SERVER_ROLE_INSTANCE && sched->limits_exceeded(lk, name, /* verbose */ false)) {
throw std::runtime_error("model limit reached, try again later");
}
// prepare new instance info
instance_t inst;
inst.meta = meta;
inst.meta.port = common_http_get_free_port();
inst.meta.status = SERVER_MODEL_STATUS_LOADING;
inst.meta.loaded_info = json{};
inst.meta.last_used = ggml_time_ms();
if (inst.meta.port <= 0) {
throw std::runtime_error("failed to get a port number");
}
inst.subproc = std::make_shared<server_subproc>();
{
SRV_INF("spawning server instance with name=%s on port %d\n", inst.meta.name.c_str(), inst.meta.port);
inst.meta.update_args(ctx_preset, bin_path); // render args
std::vector<std::string> child_args = inst.meta.args; // copy
std::vector<std::string> child_env = base_env; // copy
child_env.push_back(std::string(SERVER_ENV_ROUTER_PORT) + "=" + std::to_string(base_params.port));
if (opts.role == SERVER_ROLE_DOWNLOAD) {
inst.meta.status = SERVER_MODEL_STATUS_DOWNLOADING;
child_env.push_back(std::string(SERVER_ENV_CHILD_ROLE) + "=" + server_role_type_to_str(SERVER_ROLE_DOWNLOAD));
child_env.push_back("LLAMA_ARG_HF_REPO=" + name);
// the sidecars as -hf would pull them: the draft types, and the mmproj switch
// for the one that is not a draft
bool mmproj = false;
std::string spec_types;
for (const auto & s : common_download_sidecars()) {
if (!has_sidecar(opts.sidecars, s.name)) {
continue;
}
if (s.draft) {
spec_types += (spec_types.empty() ? "" : ",") + std::string("draft-") + s.name;
} else {
mmproj = true;
}
}
set_child_env(child_env, "LLAMA_ARG_MMPROJ_AUTO", mmproj ? "1" : "0");
set_child_env(child_env, "LLAMA_ARG_SPEC_TYPE", spec_types.empty() ? "none" : spec_types);
}
SRV_INF("%s", "spawning server instance with args:\n");
for (const auto & arg : child_args) {
SRV_INF(" %s\n", arg.c_str());
}
inst.meta.args = child_args; // save for debugging
// the child writes its commands to stdout and its logs to stderr
int options = subprocess_option_no_window;
if (!inst.subproc->sproc.create(child_args, options, child_env)) {
throw std::runtime_error("failed to spawn server instance");
}
}
// remember the download: the partial files carry a content hash, not the name
if (opts.role == SERVER_ROLE_DOWNLOAD) {
unfinished_downloads.add({ name, opts.sidecars, opts.partials });
}
// old process should have exited already, but just in case, we clean it up here
{
auto it = mapping.find(name);
if (it != mapping.end() && it->second.subproc && it->second.subproc->is_alive()) {
SRV_WRN("old process for model name=%s is still alive, this is unexpected\n", name.c_str());
it->second.subproc->terminate(); // force kill
}
}
notify_sse("model_status", name, {
{"status", server_model_status_to_string(inst.meta.status)},
});
auto proc = inst.subproc;
int port = inst.meta.port;
mapping[name] = std::move(inst);
if (opts.role == SERVER_ROLE_INSTANCE) {
sched->on_spawned(lk, name);
}
monitor->watch(name, proc, opts.role, port);
cv.notify_all();
}
void server_models::request_stop(const std::string & name, bool send_exit) {
auto it = mapping.find(name);
if (it == mapping.end() || stopping_models.count(name)) {
return;
}
stopping_models.insert(name);
monitor->stop(name, it->second.meta.stop_timeout, send_exit);
}
void server_models::watch_child(const std::string & name, std::shared_ptr<server_subproc> proc, server_role_type role, int port, int kill_after) {
monitor->watch(name, std::move(proc), role, port, kill_after);
}
void server_models::on_child_exit(const std::string & name, const std::shared_ptr<server_subproc> & proc, server_role_type role, int exit_code) {
if (role == SERVER_ROLE_MEASURE) {
sched->on_measure_exit(name, proc, exit_code);
return;
}
{
std::lock_guard<std::mutex> lk(mutex);
auto it = mapping.find(name);
if (it == mapping.end() || it->second.subproc != proc) {
stopping_models.erase(name);
return; // entry erased, or a newer instance took the name
}
}
if (role == SERVER_ROLE_DOWNLOAD) {
// instance will be cleaned up on next load_models() call
std::lock_guard<std::mutex> lk(mutex);
stopping_models.erase(name);
cv.notify_all();
} else {
update_status(name, {
SERVER_MODEL_STATUS_UNLOADED,
exit_code
});
}
}
void server_models::unload(const std::string & name) {
std::unique_lock<std::mutex> lk(mutex);
auto it = mapping.find(name);
if (it != mapping.end()) {
if (it->second.meta.status == SERVER_MODEL_STATUS_DOWNLOADING) {
SRV_INF("cancelling download for model name=%s\n", name.c_str());
it->second.download_stop_requested = true;
it->second.request_exit();
// for convenience, we wait the status change here
wait(lk, name, [](const server_model_meta & new_meta) {
return new_meta.status != SERVER_MODEL_STATUS_DOWNLOADING;
});
} else if (it->second.meta.is_running()) {
SRV_INF("stopping model instance name=%s\n", name.c_str());
bool loading = it->second.meta.status == SERVER_MODEL_STATUS_LOADING;
if (loading) {
// special case: if model is in loading state, unloading means force-killing it
SRV_WRN("model name=%s is still loading, force-killing\n", name.c_str());
it->second.subproc->terminate();
}
request_stop(name, !loading);
// status change will be handled by the monitor
} else {
SRV_WRN("model instance name=%s is not running\n", name.c_str());
}
}
}
void server_models::unload_all() {
std::unique_lock<std::mutex> lk(mutex);
sched->cancel_measurements(lk);
for (auto & [name, inst] : mapping) {
if (inst.meta.status == SERVER_MODEL_STATUS_DOWNLOADING) {
SRV_INF("cancelling download for model name=%s\n", name.c_str());
inst.download_stop_requested = true;
inst.request_exit();
} else if (inst.meta.is_running()) {
SRV_INF("stopping model instance name=%s\n", name.c_str());
bool loading = inst.meta.status == SERVER_MODEL_STATUS_LOADING;
if (loading) {
inst.subproc->terminate();
}
request_stop(name, !loading);
}
}
// wait for every child to exit, the monitor force-kills the ones that ignore the exit command
cv.wait(lk, [this, &lk]() {
if (sched->is_measuring(lk)) {
return false;
}
for (const auto & [name, inst] : mapping) {
if (inst.meta.is_running() || inst.meta.status == SERVER_MODEL_STATUS_DOWNLOADING) {
return false;
}
}
return true;
});
}
void server_models::update_status(const std::string & name, const update_status_args & args) {
std::unique_lock<std::mutex> lk(mutex);
auto it = mapping.find(name);
if (it != mapping.end()) {
auto & meta = it->second.meta;
meta.status = args.status;
meta.exit_code = args.exit_code;
if (args.status == SERVER_MODEL_STATUS_UNLOADED) {
stopping_models.erase(name);
}
if (!args.loaded_info.is_null()) {
meta.loaded_info = args.loaded_info;
// the child replaces both arrays in full; a bad or missing value changes nothing
if (args.loaded_info.contains("architecture") && args.loaded_info.at("architecture").is_object()) {
const json & child_arch = args.loaded_info.at("architecture");
for (const char * key : { "input_modalities", "output_modalities" }) {
if (!child_arch.contains(key) || !child_arch.at(key).is_array()) {
continue;
}
std::vector<std::string> modalities;
bool valid = true;
for (const auto & m : child_arch.at(key)) {
if (!m.is_string()) {
valid = false;
break;
}
modalities.push_back(m.get<std::string>());
}
if (valid) {
meta.architecture[key] = std::move(modalities);
}
}
}
}
if (!args.progress.is_null()) {
meta.progress = args.progress;
}
switch (args.status) {
case SERVER_MODEL_STATUS_LOADED: sched->on_ready(lk, name, args.memory); break;
case SERVER_MODEL_STATUS_SLEEPING: sched->on_sleeping(lk, name); break;
case SERVER_MODEL_STATUS_UNLOADED: sched->on_unloaded(lk, name); break;
default: break;
}
// a model that comes up idle or goes down changes the slot count for queued requests
sched->tick(lk);
}
// broadcast status change to SSE
{
json data = {
{"status", server_model_status_to_string(args.status)},
};
if (args.status == SERVER_MODEL_STATUS_UNLOADED) {
data["exit_code"] = args.exit_code;
}
if (!args.loaded_info.is_null()) {
data["info"] = args.loaded_info;
}
if (!args.progress.is_null()) {
data["progress"] = args.progress;
}
if (args.memory.has_value()) {
data["memory"] = args.memory->to_json();
}
// note: notify_sse doesn't acquire the lock, so no deadlock here
notify_sse("status_change", name, data);
}
cv.notify_all();
}
void server_models::update_download_progress(const std::string & name, const common_download_progress & progress, bool done, bool ok) {
json curr;
bool stopped = false;
{
std::lock_guard<std::mutex> lk(mutex);
auto it = mapping.find(name);
if (it != mapping.end()) {
if (done) {
// mark the instance to be erased on next load_models() call
it->second.meta.status = SERVER_MODEL_STATUS_DOWNLOADED;
need_reload = true;
stopped = it->second.download_stop_requested;
// a download that failed keeps its tag: its partial files are still
// there and re-posting the tag resumes them
if (ok) {
unfinished_downloads.forget(name);
}
} else {
json & info = it->second.meta.loaded_info;
if (!info.contains("progress")) {
info["progress"] = json{};
}
info["progress"][progress.url] = {
{"done", progress.downloaded},
{"total", progress.total},
};
curr = it->second.meta.loaded_info; // copy
}
}
}
if (done) {
cv.notify_all(); // notify in case unload() is waiting for download to be cancelled
// a stop a client asked for (pause or cancel) is no error: say so, so that every
// client, not only the one that asked, can tell it from a failed download
if (ok) {
notify_sse("download_finished", name, {});
} else {
notify_sse("download_failed", name, stopped ? json{{"stopped", true}} : json{});
}
} else {
notify_sse("download_progress", name, curr);
}
}
bool server_models::remove(const std::string & name) {
// do everything under one lock acquisition; avoid get_meta() /
// unload() because they can trigger load_models() which erases
// transient DOWNLOADING / DOWNLOADED entries as a side-effect
std::unique_lock<std::mutex> lk(mutex);
auto it = mapping.find(name);
if (it == mapping.end()) {
throw std::runtime_error("model name=" + name + " is not found");
}
if (it->second.meta.source != SERVER_MODEL_SOURCE_CACHE) {
throw std::runtime_error("model name=" + name + " is not removable (not from cache)");
}
if (it->second.meta.status == SERVER_MODEL_STATUS_DOWNLOADING) {
// cancel in-flight download
SRV_INF("cancelling download for model name=%s\n", name.c_str());
it->second.download_stop_requested = true;
it->second.request_exit();
} else if (it->second.meta.is_running()) {
// stop running instance
SRV_INF("stopping model instance name=%s\n", name.c_str());
bool loading = it->second.meta.status == SERVER_MODEL_STATUS_LOADING;
if (loading) {
it->second.subproc->terminate();
}
request_stop(name, !loading);
}
// wait until the child is gone
wait(lk, name, [](const server_model_meta & meta) {
return meta.status == SERVER_MODEL_STATUS_UNLOADED
|| meta.status == SERVER_MODEL_STATUS_DOWNLOADED;
});
// load_models() may have erased the entry during the wait; the files go under the
// lock, so a download of the same tag cannot start meanwhile
mapping.erase(name);
unfinished_downloads.remove(name);
// best-effort: a cancelled download may have no cached files
bool ok = common_download_remove(name);
SRV_INF("removing model name=%s from cache (%s)\n", name.c_str(), ok ? "succeeded" : "nothing cached");
notify_sse("model_remove", name, {});
return true;
}
void server_models::wait(const std::string & name, std::function<bool(const server_model_meta &)> predicate) {
std::unique_lock<std::mutex> lk(mutex);
wait(lk, name, predicate);
}
void server_models::wait(std::unique_lock<std::mutex> & lk, const std::string & name, std::function<bool(const server_model_meta &)> predicate) {
cv.wait(lk, [this, &name, &predicate]() {
auto it = mapping.find(name);
if (it != mapping.end()) {
return predicate(it->second.meta);
}
// model was removed from mapping by another code path (e.g. load_models()).
// nothing left to wait for - tell the caller to proceed.
return true;
});
}
bool server_models::ensure_model_ready(const std::string & name, const std::function<bool()> & should_stop) {
auto meta = get_meta(name);
if (!meta.has_value()) {
throw std::runtime_error("model name=" + name + " is not found");
}
bool stopping;
{
std::lock_guard<std::mutex> lk(mutex);
stopping = stopping_models.count(name) > 0;
}
if (!stopping && meta->is_ready()) {
return false; // ready for taking requests
}
if (!stopping && meta->status == SERVER_MODEL_STATUS_SLEEPING) {
return false; // child is sleeping but still running; new request will wake it up
}
bool queued = false;
bool did_load = false;
{
std::unique_lock<std::mutex> lk(mutex);
auto it = mapping.find(name);
if (it != mapping.end() && it->second.meta.status == SERVER_MODEL_STATUS_UNLOADED) {
// the queue entry protects the model from eviction until its waiters leave
sched->join(lk, name);
sched->tick(lk);
queued = true;
}
}
// while queued, this is also where the load happens: the head of the queue does it
SRV_INF("waiting until model name=%s is fully loaded...\n", name.c_str());
std::unique_lock<std::mutex> lk(mutex);
auto leave_queue = [this, &queued, &lk, &name]() {
if (queued) {
sched->leave(lk, name);
queued = false;
}
};
try {
bool saw_loading = false;
while (true) {
auto it = mapping.find(name);
if (it == mapping.end()) {
break; // removed by another code path, nothing to wait for
}
if (stopping_models.count(name)) {
// a stopping instance takes no new request, the next instance serves it
if (!queued) {
sched->join(lk, name);
sched->tick(lk);
queued = true;
}
if (should_stop && should_stop()) {
throw std::runtime_error("request cancelled while waiting for model name=" + name);
}
cv.wait_for(lk, std::chrono::milliseconds(200));
continue;
}
const server_model_status status = it->second.meta.status;
if (status == SERVER_MODEL_STATUS_LOADED || status == SERVER_MODEL_STATUS_SLEEPING) {
break;
}
if (status == SERVER_MODEL_STATUS_DOWNLOADING || status == SERVER_MODEL_STATUS_DOWNLOADED) {
break; // do not wait on a download child
}
if (status == SERVER_MODEL_STATUS_LOADING) {
saw_loading = true;
} else if (status == SERVER_MODEL_STATUS_UNLOADED) {
if (did_load || saw_loading) {
// a spawn happened and the instance came back down
if (it->second.meta.is_failed()) {
throw std::runtime_error("model name=" + name + " failed to load");
}
break; // unloaded by another code path, caller reports "not running"
}
if (!queued) {
break; // not queued, and the load someone else started fell over
}
}
if (should_stop && should_stop()) {
// if a model was evicted for us, the free slot goes to the next waiter
throw std::runtime_error("request cancelled while waiting for model name=" + name);
}
// our turn: our model is at the head, and a slot really did free up
if (status == SERVER_MODEL_STATUS_UNLOADED && sched->try_claim(lk, name)) {
lk.unlock();
bool ok = true;
try {
SRV_INF("slot available, loading queued model name=%s\n", name.c_str());
load(name);
did_load = true;
} catch (const std::exception & e) {
// lost a race for the slot, stay in line and retry
SRV_WRN("queued load of name=%s did not go through: %s\n", name.c_str(), e.what());
ok = false;
}
lk.lock();
sched->claim_done(lk, name, ok);
sched->tick(lk);
continue;
}
cv.wait_for(lk, std::chrono::milliseconds(200));
}
} catch (...) {
leave_queue();
sched->tick(lk); // a slot freed for this waiter goes to the next one
throw;
}
leave_queue();
return true;
}
server_http_res_ptr server_models::proxy_request(const server_http_req & req, const std::string & method, const std::string & name, bool update_last_used, bool detached) {
auto meta = get_meta(name);
if (!meta.has_value()) {
throw std::runtime_error("model name=" + name + " is not found");
}
if (!meta->is_running()) {
throw std::invalid_argument("model name=" + name + " is not running");
}
{
std::unique_lock<std::mutex> lk(mutex);
if (update_last_used) {
mapping[name].meta.last_used = ggml_time_ms();
}
mapping[name].req_count++;
}
if (debug_fake_timing) {
// sleep after req_count++, so the model counts as busy while we wait here
std::this_thread::sleep_for(std::chrono::seconds(2));
}
SRV_TRC("proxying request to model %s on port %d\n", name.c_str(), meta->port);
std::string proxy_path = req.path;
if (!req.query_string.empty()) {
proxy_path += '?' + req.query_string;
}
auto proxy = std::make_unique<server_http_proxy>(
method,
"http",
CHILD_ADDR,
meta->port,
proxy_path,
child_request_headers(req.headers),
req.body,
req.files,
// a detached request belongs to a replay session
detached
? std::function<bool()>([]() { return false; })
: req.should_stop,
base_params.timeout_read,
base_params.timeout_write
);
proxy->cleanup = [this, name]() {
std::unique_lock<std::mutex> lk(mutex);
auto it = mapping.find(name);
if (it != mapping.end() && it->second.req_count > 0) {
it->second.req_count--;
if (it->second.req_count == 0) {
sched->tick(lk);
}
}
};
return proxy;
}
void server_models::proxy_ws(const server_http_req & req, server_http_context::ws_conn & conn, const std::string & name) {
auto send_error = [&conn](const std::string & message) {
conn.send(safe_json_to_str(json {
{"type", "error"},
{"error", {{"type", "server_error"}, {"code", nullptr}, {"message", message}, {"param", nullptr}}},
}));
conn.close();
};
auto meta = get_meta(name);
if (!meta.has_value() || !meta->is_running()) {
send_error("model name=" + name + " is not running");
return;
}
{
std::unique_lock<std::mutex> lk(mutex);
mapping[name].meta.last_used = ggml_time_ms();
mapping[name].req_count++;
}
SRV_TRC("proxying websocket to model %s on port %d\n", name.c_str(), meta->port);
std::string url = std::string("ws://") + CHILD_ADDR + ":" + std::to_string(meta->port) + req.path;
if (!req.query_string.empty()) {
url += '?' + req.query_string;
}
// the client library adds its own upgrade headers
httplib::Headers headers;
for (const auto & [key, value] : req.headers) {
std::string lower = key;
std::transform(lower.begin(), lower.end(), lower.begin(), ::tolower);
if (lower == "host" || lower == "connection" || lower == "upgrade" || lower.compare(0, 14, "sec-websocket-") == 0
|| is_browser_origin_header(key)) {
continue;
}
headers.emplace(key, value);
}
{
httplib::ws::WebSocketClient upstream(url, headers);
if (!upstream.connect()) {
send_error("failed to connect to model name=" + name);
} else {
// client to child
std::thread pump([&]() {
std::string msg;
while (true) {
const int res = conn.read(msg);
if (res == 0 || !(res == 1 ? upstream.send(msg) : upstream.send(msg.data(), msg.size()))) {
break;
}
}
upstream.close();
});
// child to client
std::string msg;
while (true) {
const auto res = upstream.read(msg);
if (res == httplib::ws::Fail || !(res == httplib::ws::Text ? conn.send(msg) : conn.send_binary(msg))) {
break;
}
}
conn.close();
pump.join();
}
}
std::unique_lock<std::mutex> lk(mutex);
auto it = mapping.find(name);
if (it != mapping.end() && it->second.req_count > 0) {
it->second.req_count--;
if (it->second.req_count == 0) {
sched->tick(lk);
}
}
}
void server_models::handle_child_state(const std::string & name, const std::string & raw_input) {
server_state state;
json payload;
try {
json data = json::parse(raw_input.substr(strlen(CMD_CHILD_TO_ROUTER_STATE)));
state = server_state_from_str(json_value(data, "state", std::string()));
payload = json_value(data, "payload", json{});
} catch (const std::exception & e) {
SRV_ERR("failed to parse child state update for name=%s: %s\n", name.c_str(), e.what());
return;
}
switch (state) {
case SERVER_STATE_DOWNLOADING:
{
std::string result = json_value(payload, "result", std::string());
std::string url = json_value(payload, "url", std::string());
auto request_exit = [&]() {
std::lock_guard<std::mutex> lk(mutex);
auto it = mapping.find(name);
if (it != mapping.end()) {
return it->second.request_exit();
}
};
if (result == "download_finished") {
update_download_progress(name, {}, true, true);
request_exit();
} else if (result == "download_failed") {
update_download_progress(name, {}, true, false);
request_exit();
} else if (!url.empty()) {
common_download_progress p;
p.url = url;
p.downloaded = json_value(payload, "downloaded", (size_t)0);
p.total = json_value(payload, "total", (size_t)0);
update_download_progress(name, p, false);
}
} break;
case SERVER_STATE_LOADING:
{
update_status(name, {
SERVER_MODEL_STATUS_LOADING,
0,
nullptr, // no loaded_info yet
payload,
});
} break;
case SERVER_STATE_MEASURING:
{
sched->handle_measured(name, payload);
} break;
case SERVER_STATE_READY:
{
std::optional<server_model_memory> memory;
json info = payload;
if (info.contains("memory")) {
try {
memory = server_model_memory::from_json(info.at("memory"));
} catch (const std::exception & e) {
SRV_WRN("malformed memory report from model name=%s: %s\n", name.c_str(), e.what());
}
info.erase("memory");
}
update_status(name, {
SERVER_MODEL_STATUS_LOADED,
0,
// note: payload can be empty if this is a wakeup from sleep
info.size() > 0 ? info : nullptr,
{}, // reset progress info
memory,
});
} break;
case SERVER_STATE_SLEEPING:
{
update_status(name, { SERVER_MODEL_STATUS_SLEEPING });
} break;
default:
// should never happen, but just in case
GGML_ASSERT(false && "unexpected state from child server");
}
}
//
// server_models_routes
//
// RAII wrapper similar to server_response_reader, but doesn't use server_queue
static std::atomic<int> sse_client_id_counter = 0;
struct server_models_sse_client {
server_response & queue_results;
int client_id;
server_models_sse_client(server_response & q)
: queue_results(q), client_id(sse_client_id_counter.fetch_add(1, std::memory_order_relaxed)) {
SRV_DBG("new SSE client connected, assigned client_id=%d\n", client_id);
queue_results.add_waiting_task_id(client_id);
}
~server_models_sse_client() {
SRV_DBG("SSE client disconnected, removing client_id=%d\n", client_id);
queue_results.remove_waiting_task_id(client_id);
}
// return nullptr if should_stop() is true before receiving a result
// note: if one error is received, it will stop further processing and return error result
server_task_result_ptr next(const std::function<bool()> & should_stop) {
while (true) {
static const int http_polling_seconds = 1; // check should_stop every 1 second
server_task_result_ptr result = queue_results.recv_with_timeout({client_id}, http_polling_seconds);
if (result == nullptr) {
// timeout, check stop condition
if (should_stop()) {
return nullptr;
}
// continue waiting otherwise
} else {
SRV_DBG("recv result for client_id=%d: %s\n", client_id, safe_json_to_str(result->to_json()).c_str());
return result;
}
}
// should not reach here
}
};
static void res_ok(std::unique_ptr<server_http_res> & res, const json & response_data) {
res->status = 200;
res->data = safe_json_to_str(response_data);
}
static void res_err(std::unique_ptr<server_http_res> & res, const json & error_data) {
res->status = json_value(error_data, "code", 500);
res->data = safe_json_to_str({{ "error", error_data }});
}
static bool router_validate_model(std::string & name, server_models & models, bool models_autoload, std::unique_ptr<server_http_res> & res) {
if (name.empty()) {
res_err(res, format_error_response("model name is missing from the request", ERROR_TYPE_INVALID_REQUEST));
return false;
}
auto meta = models.get_meta(name);
if (!meta.has_value()) {
res_err(res, format_error_response(string_format("model '%s' not found", name.c_str()), ERROR_TYPE_INVALID_REQUEST));
return false;
}
// resolve alias to canonical model name
name = meta->name;
if (!models_autoload && !meta->is_running()) {
res_err(res, format_error_response("model is not loaded", ERROR_TYPE_INVALID_REQUEST));
return false;
}
return true;
}
static bool is_autoload(const common_params & params, const server_http_req & req) {
std::string autoload = req.get_param("autoload");
if (autoload.empty()) {
return params.models_autoload;
} else {
return autoload == "true" || autoload == "1";
}
}
// percent encode one query or path component, covers reserved chars without pulling in
// httplib::detail. used by the stream routes to forward conversation_id to children safely
static std::string encode_qs(const std::string & in) {
std::string out;
out.reserve(in.size() * 3);
for (unsigned char c : in) {
bool safe = (c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z') || (c >= '0' && c <= '9')
|| c == '-' || c == '_' || c == '.' || c == '~';
if (safe) {
out.push_back(char(c));
} else {
char buf[4];
std::snprintf(buf, sizeof(buf), "%%%02X", c);
out.append(buf, 3);
}
}
return out;
}
// resolve the child that owns a conversation's stream session via the conv_id -> model map
// populated when the POST was routed. single map lookup then a meta lookup, no polling, no
// parsing of the conv id. returns nullopt when nothing maps, the caller answers not found and
// the client recovers
static std::optional<server_model_meta> resolve_child_for_conv(
server_models & models, const std::string & conversation_id) {
if (conversation_id.empty()) {
return std::nullopt;
}
auto tracked = models.conv_models.lookup(conversation_id);
if (!tracked.has_value()) {
return std::nullopt;
}
auto meta = models.get_meta(*tracked);
if (meta.has_value() && meta->is_ready()) {
return meta;
}
return std::nullopt;
}
void server_models_routes::init_routes() {
if (!common_subproc::is_supported()) {
throw std::runtime_error("subprocess is not enabled on this build");
}
this->get_router_props = [this](const server_http_req & req) {
std::string name = req.get_param("model");
if (name.empty()) {
// main instance
auto res = std::make_unique<server_http_res>();
json props = {
// TODO: add support for this on web UI
{"role", "router"},
{"max_instances", params.models_max},
{"models_autoload", params.models_autoload},
// this is a dummy response to make sure the UI doesn't break
{"model_alias", "llama-server"},
{"model_path", "none"},
{"default_generation_settings", {
{"params", json{}},
{"n_ctx", 0},
}},
// New key
{"ui_settings", ui_settings},
{"build_info", std::string(llama_build_info())},
{"cors_proxy_enabled", params.ui_mcp_proxy},
};
if (!params.server_connect_code.empty()) {
props["connect_code"] = params.server_connect_code;
}
res_ok(res, props);
return res;
}
return proxy_get(req);
};
this->proxy_get = [this](const server_http_req & req) {
std::string method = "GET";
std::string name = req.get_param("model");
bool autoload = is_autoload(params, req);
auto error_res = std::make_unique<server_http_res>();
if (!router_validate_model(name, models, autoload, error_res)) {
return error_res;
}
if (autoload) {
models.ensure_model_ready(name, req.should_stop);
}
return models.proxy_request(req, method, name, false);
};
this->proxy_ws = [this](const server_http_req & req, server_http_context::ws_conn & conn) {
std::string name = req.get_param("model");
bool autoload = is_autoload(params, req);
auto error_res = std::make_unique<server_http_res>();
if (!router_validate_model(name, models, autoload, error_res)) {
// error_res holds {"error": {...}}, send it as a websocket error event
json err = json::parse(error_res->data);
err["type"] = "error";
conn.send(safe_json_to_str(err));
conn.close();
return;
}
if (autoload) {
models.ensure_model_ready(name, req.should_stop);
}
models.proxy_ws(req, conn, name);
};
this->proxy_post = [this](const server_http_req & req) {
std::string method = "POST";
json body = json::parse(req.body);
std::string name = json_value(body, "model", std::string());
bool autoload = is_autoload(params, req);
auto error_res = std::make_unique<server_http_res>();
if (!router_validate_model(name, models, autoload, error_res)) {
return error_res;
}
// remember which child serves this conversation so the stream routes can route straight
// to it without polling, keyed on the exact conv id from the header. registered before
// the load wait so a stop issued while the model loads can erase the entry and cancel
// this request instead of leaving an orphan generation
std::string conv_id = server_stream_conv_id_from_headers(req.headers);
uint64_t ticket = models.conv_models.remember(conv_id, name);
// a dead socket must not cancel a session request, only a stop does (checked right below)
auto should_stop = ticket == 0 ? req.should_stop : nullptr;
bool waited = autoload && models.ensure_model_ready(name, should_stop);
if (ticket != 0 && !models.conv_models.alive(conv_id, ticket)) {
SRV_INF("request for conv_id=%s cancelled while model name=%s was loading\n",
conv_id.c_str(), name.c_str());
res_err(error_res, format_error_response(
"request cancelled by a stop while the model was loading", ERROR_TYPE_INVALID_REQUEST));
return error_res;
}
// a session request that waited for a load detaches from the client socket: the
// client may have dropped during the wait (page reload) and the session buffer must
// still receive the generation for a later resume
return models.proxy_request(req, method, name, true, waited && ticket != 0); // update last usage for POST request only
};
this->post_router_models_load = [this](const server_http_req & req) {
auto res = std::make_unique<server_http_res>();
json body = json::parse(req.body);
std::string name = json_value(body, "model", std::string());
auto meta = models.get_meta(name);
if (!meta.has_value()) {
res_err(res, format_error_response("model is not found", ERROR_TYPE_NOT_FOUND));
return res;
}
if (meta->is_running()) {
res_err(res, format_error_response("model is already running", ERROR_TYPE_INVALID_REQUEST));
return res;
}
models.load(meta->name);
res_ok(res, {{"success", true}});
return res;
};
this->get_router_models = [this](const server_http_req & req) {
bool reload = !req.get_param("reload", "").empty();
if (reload) {
models.load_models();
}
auto res = std::make_unique<server_http_res>();
json models_json = json::array();
auto all_models = models.get_all_meta();
std::time_t t = std::time(0);
// a download on its way has no files to resolve yet, only the kinds it pulls
const auto unfinished = models.unfinished_downloads_get();
for (const auto & meta : all_models) {
if (meta.hidden) {
continue; // cache model deduplicated by a preset
}
json status {
{"value", server_model_status_to_string(meta.status)},
{"args", meta.args},
};
if (!meta.preset.name.empty()) {
common_preset preset_copy = meta.preset;
unset_reserved_args(preset_copy, false);
preset_copy.unset_option("LLAMA_ARG_HOST");
preset_copy.unset_option("LLAMA_ARG_PORT");
preset_copy.unset_option("LLAMA_ARG_ALIAS");
preset_copy.unset_option("LLAMA_ARG_TAGS");
status["preset"] = preset_copy.to_ini();
}
if (meta.is_failed()) {
status["exit_code"] = meta.exit_code;
status["failed"] = true;
}
json model_info = json {
{"id", meta.name},
{"aliases", meta.aliases},
{"tags", meta.tags},
{"object", "model"}, // for OAI-compat
{"owned_by", "llamacpp"}, // for OAI-compat
{"created", t}, // for OAI-compat
{"status", status},
{"architecture", meta.architecture},
{"source", server_model_source_to_string(meta.source)},
{"can_remove", meta.source == SERVER_MODEL_SOURCE_CACHE},
// {"need_download", meta.need_download},
// TODO: add other fields from the GGUF metadata
};
if (meta.n_ctx_train > 0) {
model_info["context_length"] = meta.n_ctx_train;
}
if (models.sched->memory_enabled()) {
model_info["memory"] = meta.memory.to_json();
}
// the sidecars a model has on disk, resolved as -hf would load them, for a
// model that resolves from a HF repo. A download on its way reports the
// kinds it pulls, as an unfinished one does
const auto it_unfinished = std::find_if(unfinished.begin(), unfinished.end(),
[&](const server_unfinished_download & d) { return d.model == meta.name; });
const std::string hf_repo = model_hf_repo(meta);
if (it_unfinished != unfinished.end() && meta.status == SERVER_MODEL_STATUS_DOWNLOADING) {
model_info["sidecars"] = it_unfinished->sidecars;
} else if (!hf_repo.empty()) {
const json cached = cached_sidecars(hf_repo);
model_info["sidecars"] = cached;
// the sidecars of an unfinished download of this model that are not on
// disk yet: a POST with the kind resumes one, a DELETE with it drops it
if (it_unfinished != unfinished.end()) {
std::vector<std::string> incomplete;
for (const auto & p : it_unfinished->partials) {
if (!p.kind.empty() && !has_sidecar(incomplete, p.kind) &&
std::find(cached.begin(), cached.end(), p.kind) == cached.end()) {
incomplete.push_back(p.kind);
}
}
if (!incomplete.empty()) {
model_info["incomplete_sidecars"] = incomplete;
}
}
}
// merge with loaded_info from the child process if available
if (meta.is_running()) {
for (auto it = meta.loaded_info.begin(); it != meta.loaded_info.end(); ++it) {
if (!model_info.contains(it.key())) {
model_info[it.key()] = it.value();
}
}
}
models_json.push_back(model_info);
}
const json architecture_default = json {
{"input_modalities", json::array({"text"})},
{"output_modalities", json::array({"text"})},
};
// a download that never finished has no instance: report it as incomplete so
// the UI can offer to resume it, without starting anything
for (const auto & d : unfinished) {
if (models.has_model(d.model)) {
continue; // complete, or downloading right now
}
models_json.push_back(json {
{"id", d.model},
{"aliases", json::array()},
{"tags", json::array()},
{"object", "model"},
{"owned_by", "llamacpp"},
{"created", t},
{"status", json { {"value", server_model_status_to_string(SERVER_MODEL_STATUS_INCOMPLETE)}, {"args", json::array()} }},
{"architecture", architecture_default},
{"source", server_model_source_to_string(SERVER_MODEL_SOURCE_CACHE)},
{"can_remove", true},
{"sidecars", d.sidecars},
});
}
res_ok(res, {
{"data", models_json},
{"object", "list"},
});
return res;
};
this->post_router_models_unload = [this](const server_http_req & req) {
auto res = std::make_unique<server_http_res>();
json body = json::parse(req.body);
std::string name = json_value(body, "model", std::string());
auto model = models.get_meta(name);
if (!model.has_value()) {
res_err(res, format_error_response("model is not found", ERROR_TYPE_INVALID_REQUEST));
return res;
}
if (!model->is_running() && model->status != SERVER_MODEL_STATUS_DOWNLOADING) {
res_err(res, format_error_response("model is not running", ERROR_TYPE_INVALID_REQUEST));
return res;
}
models.unload(model->name);
res_ok(res, {{"success", true}});
return res;
};
this->get_router_models_sse = [this](const server_http_req & req) {
auto res = std::make_unique<server_http_res>();
res->status = 200;
res->content_type = "text/event-stream";
auto sse_client = std::make_shared<server_models_sse_client>(models.sse);
res->next = [this, sse_client, &req](std::string & output) -> bool {
auto result = sse_client->next([&]() {
return stopping.load(std::memory_order_relaxed) || req.should_stop();
});
if (result == nullptr) {
return false; // client disconnected or should_stop
}
output = "data: " + safe_json_to_str(result->to_json()) + "\n\n";
return true; // listen for the next event
};
return res;
};
this->post_router_models = [this](const server_http_req & req) {
auto res = std::make_unique<server_http_res>();
json body = json::parse(req.body);
std::string name = json_value(body, "model", std::string());
if (name.empty()) {
throw std::invalid_argument("model must be a non-empty string");
}
const auto sidecars = parse_sidecars(body);
common_params_model model;
model.hf_repo = name;
// validate by fetching metadata, through the plan the download follows
common_download_hf_plan plan;
try {
plan = common_download_get_hf_plan(model, sidecar_download_opts(params, sidecars));
} catch (...) {
SRV_ERR("unknown error while validating model '%s'\n", name.c_str());
// other exceptions will be handled by the outer ex_wrapper()
throw;
}
// a tag that resolves no file has nothing to download
if (plan.preset.path.empty() && plan.primary.path.empty() && common_download_plan_sidecars(plan).empty()) {
throw std::invalid_argument("model '" + name + "' resolves no file to download");
}
// then, proceed with the actual download; load() decides over an existing entry
SRV_INF("starting download for model '%s'\n", name.c_str());
{
server_models::load_options load_opts;
load_opts.role = SERVER_ROLE_DOWNLOAD;
load_opts.custom_meta = server_model_meta{};
load_opts.custom_meta->source = SERVER_MODEL_SOURCE_CACHE;
load_opts.custom_meta->name = name;
load_opts.sidecars = sidecars;
load_opts.partials = common_download_plan_partials(plan);
models.load(name, load_opts);
}
res_ok(res, {{"success", true}});
return res;
};
this->del_router_models = [this](const server_http_req & req) {
auto res = std::make_unique<server_http_res>();
std::string name = req.get_param("model");
if (name.empty()) {
throw std::invalid_argument("model must be a non-empty string");
}
// a sidecar alone goes, the model keeps its files
std::string sidecar = req.get_param("sidecar");
if (!sidecar.empty()) {
check_sidecar(sidecar);
res_ok(res, {{"success", models.remove_sidecar(name, sidecar)}});
return res;
}
// a download listed from an earlier run has no instance, only its tag and
// its partial files
if (!models.has_model(name)) {
models.unfinished_downloads_remove(name);
res_ok(res, {{"success", true}});
return res;
}
models.remove(name); // throws on error
res_ok(res, {{"success", true}});
return res;
};
this->router_stream_get = [this](const server_http_req & req) {
// GET /v1/stream?conv_id=<id>&from=N. resolve the owning child from the conv_id -> model
// map, 404 when nothing maps
auto res = std::make_unique<server_http_res>();
std::string conv_id = req.get_param("conv_id");
if (conv_id.empty()) {
res_err(res, format_error_response("Missing conversation id in path", ERROR_TYPE_INVALID_REQUEST));
return res;
}
std::optional<server_model_meta> owner = resolve_child_for_conv(models, conv_id);
if (!owner.has_value()) {
// a registered conv whose model is still loading earns a retry: the session appears
// once the load ends and the pending request reaches the child
auto tracked = models.conv_models.lookup(conv_id);
auto meta = tracked.has_value() ? models.get_meta(*tracked) : std::nullopt;
bool transient = meta.has_value() && (meta->status == SERVER_MODEL_STATUS_LOADING ||
meta->status == SERVER_MODEL_STATUS_DOWNLOADING ||
meta->status == SERVER_MODEL_STATUS_DOWNLOADED);
if (transient) {
res_err(res, format_error_response("Stream owner model is loading, retry later", ERROR_TYPE_UNAVAILABLE));
} else {
res_err(res, format_error_response("Stream not found or expired", ERROR_TYPE_NOT_FOUND));
}
return res;
}
std::string from = req.get_param("from");
std::string child_path = "/v1/stream?conv_id=" + encode_qs(conv_id);
if (!from.empty()) {
child_path += "&from=" + from;
}
SRV_TRC("proxying stream resume to model %s on port %d, path=%s\n",
owner->name.c_str(), owner->port, child_path.c_str());
auto proxy = std::make_unique<server_http_proxy>(
"GET",
"http",
CHILD_ADDR,
owner->port,
child_path,
child_request_headers(req.headers),
req.body,
req.files,
req.should_stop,
params.timeout_read,
params.timeout_write);
return std::unique_ptr<server_http_res>(std::move(proxy));
};
this->router_streams_lookup = [this](const server_http_req & req) {
// POST /v1/streams/lookup. resolve each requested conv id to its owning child via the
// map, group the ids per child, and query only the children that actually own some of
// them instead of fanning out to every ready child. a child only answers for the ids
// it owns, never lists anything else
auto res = std::make_unique<server_http_res>();
std::vector<std::string> requested;
try {
json body = json::parse(req.body);
if (body.contains("conversation_ids") && body["conversation_ids"].is_array()) {
for (const auto & v : body["conversation_ids"]) {
if (v.is_string() && !v.get<std::string>().empty()) {
requested.push_back(v.get<std::string>());
}
}
}
} catch (const std::exception &) {
res_ok(res, json::array());
return res;
}
// group requested ids by the child port that owns them, drop ids that map to nothing
std::unordered_map<int, json> per_child;
for (const auto & cid : requested) {
auto owner = resolve_child_for_conv(models, cid);
if (!owner.has_value()) {
continue;
}
per_child[owner->port].push_back(cid);
}
json aggregated = json::array();
for (auto & [port, ids] : per_child) {
json child_body = {{"conversation_ids", ids}};
httplib::Client cli(CHILD_ADDR, port);
cli.set_connection_timeout(0, STREAM_LOOKUP_TIMEOUT_MS * 1000);
cli.set_read_timeout(0, STREAM_LOOKUP_TIMEOUT_MS * 1000);
cli.set_write_timeout(0, STREAM_LOOKUP_TIMEOUT_MS * 1000);
auto resp = cli.Post("/v1/streams/lookup", child_body.dump(), "application/json");
if (!resp || resp->status != 200) {
continue;
}
try {
json child_arr = json::parse(resp->body);
if (!child_arr.is_array()) {
continue;
}
for (auto & entry : child_arr) {
if (entry.is_object()) {
aggregated.push_back(entry);
}
}
} catch (const std::exception &) {
continue;
}
}
res_ok(res, aggregated);
return res;
};
this->router_stream_delete = [this](const server_http_req & req) {
// DELETE /v1/stream?conv_id=<id>. resolve the owning child via the map and forward only to
// it, evict_and_cancel is idempotent on the child
auto res = std::make_unique<server_http_res>();
std::string conv_id = req.get_param("conv_id");
if (conv_id.empty()) {
res_err(res, format_error_response("Missing conversation id in path", ERROR_TYPE_INVALID_REQUEST));
return res;
}
std::string child_path = "/v1/stream?conv_id=" + encode_qs(conv_id);
auto owner = resolve_child_for_conv(models, conv_id);
if (owner.has_value()) {
httplib::Client cli(CHILD_ADDR, owner->port);
cli.set_connection_timeout(0, STREAM_LOOKUP_TIMEOUT_MS * 1000);
cli.set_read_timeout(0, STREAM_LOOKUP_TIMEOUT_MS * 1000);
cli.set_write_timeout(0, STREAM_LOOKUP_TIMEOUT_MS * 1000);
auto resp = cli.Delete(child_path.c_str());
(void) resp; // the child logs its own miss when the session is unknown there
} else if (auto tracked = models.conv_models.lookup(conv_id); tracked.has_value()) {
// the entry exists but its model is still loading: the forget below erases it,
// which cancels the request parked in proxy_post before the generation starts
SRV_INF("router stop for conv_id=%s while model name=%s is loading, cancelling the pending request\n",
conv_id.c_str(), tracked->c_str());
} else {
SRV_WRN("router stop for unknown conv_id=%s, no owning child in the conv map\n",
conv_id.c_str());
}
// drop the tracking entry, the session is being torn down
models.conv_models.forget(conv_id);
res->status = 204;
res->content_type = "application/json";
return res;
};
}
//
// server_http_proxy
//
static std::string to_lower_copy(const std::string & value) {
std::string lowered(value.size(), '\0');
std::transform(value.begin(), value.end(), lowered.begin(), [](unsigned char c) { return std::tolower(c); });
return lowered;
}
static bool should_strip_proxy_header(const std::string & header_name) {
// Headers that get duplicated when router forwards child responses
if (header_name == "server" ||
header_name == "transfer-encoding" ||
header_name == "content-length" || // quick fix for https://github.com/ggml-org/llama.cpp/issues/17710
header_name == "keep-alive") {
return true;
}
// Router injects CORS, child also sends them: duplicate
if (header_name.rfind("access-control-", 0) == 0) {
return true;
}
return false;
}
static std::string generate_multipart_boundary() {
thread_local std::mt19937 gen(std::random_device{}());
static const char chars[] = "0123456789abcdefghijklmnopqrstuvwxyz";
std::uniform_int_distribution<> dis(0, sizeof(chars) - 2);
std::string boundary = "----llama-cpp-proxy-";
for (int i = 0; i < 16; i++) {
boundary += chars[dis(gen)];
}
return boundary;
}
static std::string build_multipart_body(
const json & form_fields,
const std::map<std::string, uploaded_file> & files,
const std::string & boundary) {
static auto sanitize_field = [](const std::string & text) {
std::string result;
result.reserve(text.size());
for (char c : text) {
if (c != '\n' && c != '\r' && c != '"') {
result += c;
}
}
return result;
};
std::ostringstream body;
for (const auto & [key, value] : form_fields.items()) {
if (value.is_array()) {
for (const auto & item : value) {
body << "--" << boundary << "\r\n";
body << "Content-Disposition: form-data; name=\"" << sanitize_field(key) << "\"\r\n";
body << "\r\n";
if (!item.is_string()) {
throw std::invalid_argument("expected string");
}
body << item.get<std::string>() << "\r\n";
}
} else {
body << "--" << boundary << "\r\n";
body << "Content-Disposition: form-data; name=\"" << sanitize_field(key) << "\"\r\n";
body << "\r\n";
if (!value.is_string()) {
throw std::invalid_argument("expected string");
}
body << value.get<std::string>() << "\r\n";
}
}
for (const auto & [key, file] : files) {
body << "--" << boundary << "\r\n";
body << "Content-Disposition: form-data; name=\"" << sanitize_field(key) << "\"";
if (!file.filename.empty()) {
body << "; filename=\"" << sanitize_field(file.filename) << "\"";
}
body << "\r\n";
if (!file.content_type.empty()) {
body << "Content-Type: " << sanitize_field(file.content_type) << "\r\n";
} else {
body << "Content-Type: application/octet-stream\r\n";
}
body << "\r\n";
body.write(reinterpret_cast<const char*>(file.data.data()), file.data.size());
body << "\r\n";
}
body << "--" << boundary << "--\r\n";
return body.str();
}
server_http_proxy::server_http_proxy(
const std::string & method,
const std::string & scheme,
const std::string & host,
int port,
const std::string & path,
const std::map<std::string, std::string> & headers,
const std::string & body,
const std::map<std::string, uploaded_file> & files,
const std::function<bool()> should_stop,
int32_t timeout_read,
int32_t timeout_write
) {
// shared between reader and writer threads
auto cli = std::make_shared<httplib::ClientImpl>(host, port);
auto pipe = std::make_shared<server_pipe<msg_t>>();
if (scheme == "https") {
#ifdef CPPHTTPLIB_OPENSSL_SUPPORT
cli.reset(new httplib::SSLClient(host, port));
#else
throw std::runtime_error("HTTPS requested but CPPHTTPLIB_OPENSSL_SUPPORT is not defined");
#endif
}
// setup Client
cli->set_follow_location(true);
cli->set_connection_timeout(timeout_read, 0); // use --timeout value instead of hardcoded 5 s
cli->set_write_timeout(timeout_read, 0); // reversed for cli (client) vs srv (server)
cli->set_read_timeout(timeout_write, 0);
this->status = 500; // to be overwritten upon response
this->cleanup_pipes = [pipe, cli]() {
pipe->close_read();
pipe->close_write();
cli->stop();
};
// wire up the receive end of the pipe
this->next = [pipe, should_stop](std::string & out) -> bool {
msg_t msg;
bool has_next = pipe->read(msg, should_stop);
if (!msg.data.empty()) {
out = std::move(msg.data);
}
return has_next; // false if EOF or pipe broken
};
// build the header message forwarded to the reader thread, stripping internal proxy headers
auto make_header_msg = [](const httplib::Response & response) {
msg_t msg;
msg.status = response.status;
for (const auto & [key, value] : response.headers) {
const auto lowered = to_lower_copy(key);
if (should_strip_proxy_header(lowered)) {
continue;
}
if (lowered == "content-type") {
msg.content_type = value;
continue;
}
msg.headers[key] = value;
}
return msg;
};
// true once response_handler has already forwarded the headers
auto headers_sent = std::make_shared<std::atomic<bool>>(false);
// wire up the HTTP client
// note: do NOT capture `this` pointer, as it may be destroyed before the thread ends
httplib::ResponseHandler response_handler = [pipe, headers_sent, make_header_msg](const httplib::Response & response) {
headers_sent->store(true);
return pipe->write(make_header_msg(response)); // send headers first
};
httplib::ContentReceiverWithProgress content_receiver = [pipe](const char * data, size_t data_length, size_t, size_t) {
// send data chunks
// returns false if pipe is closed / broken (signal to stop receiving)
return pipe->write({{}, 0, std::string(data, data_length), ""});
};
// when files are present, the body was converted from multipart form data to JSON
// we need to reconstruct the multipart body for the downstream server
std::string effective_body = body;
std::string override_content_type;
bool has_files = !files.empty();
if (has_files) {
json form_fields = json::parse_no_throw(body);
if (!form_fields.is_discarded()) {
auto boundary = generate_multipart_boundary();
effective_body = build_multipart_body(form_fields, files, boundary);
override_content_type = "multipart/form-data; boundary=" + boundary;
} else {
throw std::runtime_error("failed to parse multipart form fields JSON");
}
}
// prepare the request to destination server
httplib::Request req;
{
req.method = method;
req.path = path;
for (const auto & [key, value] : headers) {
const auto lowered = to_lower_copy(key);
if (lowered == "accept-encoding") {
// disable Accept-Encoding to avoid compressed responses
continue;
}
if (lowered == "transfer-encoding") {
// the body is already decoded
continue;
}
if (lowered == "content-length") {
// let httplib calculate Content-Length from the actual body
continue;
}
if (lowered == "content-type") {
if (has_files) {
// we set our own Content-Type with the new boundary
continue;
}
// when no files but the original request was multipart,
// the body is now JSON, so correct the Content-Type
if (value.find("multipart/form-data") != std::string::npos) {
override_content_type = "application/json; charset=utf-8";
continue;
}
}
if (lowered == "host") {
bool is_default_port = (scheme == "https" && port == 443) || (scheme == "http" && port == 80);
const std::string url_host = common_http_format_host(host);
req.set_header(key, is_default_port ? url_host : url_host + ":" + std::to_string(port));
} else {
req.set_header(key, value);
}
}
req.body = effective_body;
if (!override_content_type.empty()) {
req.set_header("Content-Type", override_content_type);
}
req.response_handler = response_handler;
req.content_receiver = content_receiver;
}
// start the proxy thread
SRV_DBG("start proxy thread %s %s\n", req.method.c_str(), req.path.c_str());
this->thread = std::thread([cli, pipe, req, headers_sent, make_header_msg]() {
auto result = cli->send(std::move(req));
if (result.error() != httplib::Error::Success) {
auto err_str = httplib::to_string(result.error());
SRV_ERR("http client error: %s\n", err_str.c_str());
pipe->write({{}, 500, "", ""}); // header
pipe->write({{}, 0, "proxy error: " + err_str, ""}); // body
} else if (!headers_sent->load()) {
// httplib skips response_handler for bodyless statuses like 204, send headers here instead
pipe->write(make_header_msg(*result));
}
pipe->close_write(); // signal EOF to reader
SRV_DBG("%s", "client request thread ended\n");
});
this->thread.detach();
// wait for the first chunk (headers)
{
msg_t header;
if (pipe->read(header, should_stop)) {
SRV_DBG("%s", "received response headers\n");
this->status = header.status;
this->headers = std::move(header.headers);
if (!header.content_type.empty()) {
this->content_type = std::move(header.content_type);
}
} else {
SRV_DBG("%s", "no response headers received (request cancelled?)\n");
}
}
}