diff --git a/common/arg.cpp b/common/arg.cpp index 0766087c3..6f5fe377d 100644 --- a/common/arg.cpp +++ b/common/arg.cpp @@ -4658,6 +4658,12 @@ void common_params_add_preset_options(std::vector & args) { [](common_params &, int) { /* unused */ } ).set_env(COMMON_ARG_PRESET_STOP_TIMEOUT).set_preset_only()); + args.push_back(common_arg( + {"dedup-cache-models"}, "0|1", + "in server router mode, hide a cached model from the model list when this preset resolves to the same model file", + [](common_params &, const std::string &) { /* unused */ } + ).set_env(COMMON_ARG_PRESET_DEDUP_CACHE_MODELS).set_preset_only()); + // args.push_back(common_arg( // {"pin"}, // "in server router mode, do not unload this model if models_max is exceeded", diff --git a/common/arg.h b/common/arg.h index 44b9e887c..421bc295f 100644 --- a/common/arg.h +++ b/common/arg.h @@ -11,8 +11,9 @@ #include // pseudo-env variable to identify preset-only arguments -#define COMMON_ARG_PRESET_LOAD_ON_STARTUP "__PRESET_LOAD_ON_STARTUP" -#define COMMON_ARG_PRESET_STOP_TIMEOUT "__PRESET_STOP_TIMEOUT" +#define COMMON_ARG_PRESET_LOAD_ON_STARTUP "__PRESET_LOAD_ON_STARTUP" +#define COMMON_ARG_PRESET_STOP_TIMEOUT "__PRESET_STOP_TIMEOUT" +#define COMMON_ARG_PRESET_DEDUP_CACHE_MODELS "__PRESET_DEDUP_CACHE_MODELS" // // CLI argument parsing diff --git a/common/download.cpp b/common/download.cpp index 44c6cea42..2509f75ab 100644 --- a/common/download.cpp +++ b/common/download.cpp @@ -989,6 +989,26 @@ std::vector common_list_cached_models() { return result; } +std::string common_download_resolve_path(const std::string & hf_repo_with_tag, const std::string & hf_file) { + auto [repo, tag] = common_download_split_repo_tag(hf_repo_with_tag); + + auto files = hf_cache::get_cached_files(repo); + if (files.empty()) { + return ""; + } + + if (!hf_file.empty()) { + for (const auto & f : files) { + if (f.path == hf_file) { + return f.local_path; + } + } + return ""; + } + + return find_best_model(files, tag).local_path; +} + bool common_download_remove(const std::string & hf_repo_with_tag) { namespace fs = std::filesystem; diff --git a/common/download.h b/common/download.h index 9a03f5e91..8c30cfc3e 100644 --- a/common/download.h +++ b/common/download.h @@ -85,6 +85,10 @@ std::vector common_download_get_all_parts(const std::string & url); // returns list of cached models std::vector common_list_cached_models(); +// resolve the local cached file path for a HF repo without network access (hf_file, if given, must match exactly) +// returns an empty string if the model is not present in the cache +std::string common_download_resolve_path(const std::string & hf_repo_with_tag, const std::string & hf_file = ""); + // download single file from url to local path // returns status code or -1 on error // skip_etag: if true, don't read/write .etag files (for HF cache where filename is the hash) diff --git a/tools/server/README.md b/tools/server/README.md index 67e52b1db..78274967d 100644 --- a/tools/server/README.md +++ b/tools/server/README.md @@ -1759,6 +1759,7 @@ The precedence rule for preset options is as follows: We also offer additional options that are exclusive to presets (these aren't treated as command-line arguments): - `load-on-startup` (boolean): Controls whether the model loads automatically when the server starts - `stop-timeout` (int, seconds): After requested unload, wait for this many seconds before forcing termination (default: 10) +- `dedup-cache-models` (boolean): When the preset uses `hf-repo` pointing to a model that is already downloaded, hide the corresponding cached model entry from `GET /models` (the preset entry remains visible). Set it in the `[*]` section to apply to all presets. ### Routing requests diff --git a/tools/server/server-models.cpp b/tools/server/server-models.cpp index df34adc29..35b935570 100644 --- a/tools/server/server-models.cpp +++ b/tools/server/server-models.cpp @@ -555,6 +555,40 @@ void server_models::load_models() { return source_map.count(name) ? source_map.at(name) : SERVER_MODEL_SOURCE_PRESET; }; + // hide cache models whose resolved file is already used by a preset with dedup-cache-models enabled + std::set hidden_models; + { + std::set preset_paths; + for (const auto & [name, preset] : custom_presets) { + std::string val; + if (!preset.get_option(COMMON_ARG_PRESET_DEDUP_CACHE_MODELS, val) || !common_arg_utils::is_truthy(val)) { + continue; + } + std::string hf_repo; + if (!preset.get_option("LLAMA_ARG_HF_REPO", hf_repo) || hf_repo.empty()) { + continue; + } + std::string hf_file; + preset.get_option("LLAMA_ARG_HF_FILE", hf_file); + std::string path = common_download_resolve_path(hf_repo, hf_file); + if (!path.empty()) { + preset_paths.insert(path); + } + } + if (!preset_paths.empty()) { + for (const auto & [name, preset] : cached_models) { + if (get_source(name) != SERVER_MODEL_SOURCE_CACHE) { + continue; // merged with another source, not a pure cache entry + } + std::string path = common_download_resolve_path(name); + if (!path.empty() && preset_paths.count(path)) { + SRV_INF("hiding cache model name=%s (deduplicated by a preset)\n", name.c_str()); + hidden_models.insert(name); + } + } + } + } + // Helpers that read `mapping` - must be called while holding the lock. std::unordered_set custom_names; for (const auto & [name, preset] : custom_presets) custom_names.insert(name); @@ -590,6 +624,11 @@ void server_models::load_models() { } } }; + auto apply_hidden = [&]() { + for (auto & [name, inst] : mapping) { + inst.meta.hidden = hidden_models.count(name) > 0; + } + }; // update_args() injects HOST/PORT/ALIAS, so strip them before comparing presets auto preset_options_for_compare = [](common_preset p) { p.unset_option("LLAMA_ARG_HOST"); @@ -630,6 +669,7 @@ void server_models::load_models() { add_model(std::move(meta)); } apply_stop_timeout(); + apply_hidden(); log_available_models(); std::vector models_to_load; @@ -806,6 +846,7 @@ void server_models::load_models() { } apply_stop_timeout(); + apply_hidden(); // clear reload flag before unlocking for autoload - load() blocks on !is_reloading, // so clearing it here (while still locked) prevents a deadlock in the autoload calls below @@ -1929,6 +1970,9 @@ void server_models_routes::init_routes() { auto all_models = models.get_all_meta(); std::time_t t = std::time(0); for (const auto & meta : all_models) { + if (meta.hidden) { + continue; // cache model deduplicated by a preset + } json status { {"value", server_model_status_to_string(meta.status)}, {"args", meta.args}, diff --git a/tools/server/server-models.h b/tools/server/server-models.h index 615acb577..79b231cba 100644 --- a/tools/server/server-models.h +++ b/tools/server/server-models.h @@ -84,6 +84,7 @@ struct server_model_meta { int exit_code = 0; // exit code of the model instance process (only valid if status == FAILED) int stop_timeout = 0; // seconds to wait before force-killing the model instance during shutdown mtmd_caps multimodal; // multimodal capabilities + bool hidden = false; // hidden from GET /models, but still accept if requested bool is_ready() const { return status == SERVER_MODEL_STATUS_LOADED; diff --git a/tools/server/tests/unit/test_router.py b/tools/server/tests/unit/test_router.py index 5ab62666c..0e1467de3 100644 --- a/tools/server/tests/unit/test_router.py +++ b/tools/server/tests/unit/test_router.py @@ -406,6 +406,59 @@ def test_router_reload_models(): os.remove(preset_path) +def test_router_dedup_cache_models(): + """dedup-cache-models hides the cache entry backing a preset from GET /models""" + global server + + preset_path = os.path.join(TMP_DIR, "test_dedup.ini") + cache_id = "ggml-org/test-model-stories260K:F32" + + with open(preset_path, "w") as f: + f.write( + "[model-dedup]\n" + "hf-repo = ggml-org/test-model-stories260K\n" + "dedup-cache-models = 1\n" + ) + + server.models_preset = preset_path + server.start() + + try: + ids = _get_model_ids(is_reload=False) + assert "model-dedup" in ids + assert cache_id not in ids, "cache model should be hidden by dedup" + # other cache models are unaffected + assert "ggml-org/tinygemma3-GGUF:Q8_0" in ids + + # the hidden model is only hidden from the listing, it can still be used + res = server.make_request("POST", "/tokenize", data={"model": cache_id, "content": "hello"}) + assert res.status_code == 200 + + # disabling the flag brings the cache entry back on reload + with open(preset_path, "w") as f: + f.write( + "[model-dedup]\n" + "hf-repo = ggml-org/test-model-stories260K\n" + ) + ids = _get_model_ids(is_reload=True) + assert cache_id in ids + + # the flag also works from the global section + with open(preset_path, "w") as f: + f.write( + "[*]\n" + "dedup-cache-models = 1\n" + "\n" + "[model-dedup]\n" + "hf-repo = ggml-org/test-model-stories260K\n" + ) + ids = _get_model_ids(is_reload=True) + assert "model-dedup" in ids + assert cache_id not in ids, "cache model should be hidden by global dedup" + finally: + os.remove(preset_path) + + def test_router_remote_preset(): global server server.model_hf_repo = "ggml-org/test-preset-ci"