diff --git a/common/download.cpp b/common/download.cpp index ae0f7ffed..38f26e1b4 100644 --- a/common/download.cpp +++ b/common/download.cpp @@ -39,6 +39,9 @@ // downloader // +// the suffix of the file a download writes until it completes +static const std::string PARTIAL_SUFFIX = ".downloadInProgress"; + static void write_etag(const std::string & path, const std::string & etag) { const std::string etag_path = path + ".etag"; fs_write_atomic(std::filesystem::u8path(etag_path), etag); @@ -186,6 +189,13 @@ static bool common_pull_file(httplib::Client & cli, [&](const httplib::Response &response) { if (p.downloaded > 0 && response.status != 206) { LOG_WRN("%s: server did not respond with 206 Partial Content for a resume request. Status: %d\n", func, response.status); + if (response.status == 416 || response.status == 200) { + // the kept partial cannot be resumed: 416 for a complete or stale one, + // 200 from a server that ignores the range; drop it, so the next + // attempt starts over instead of wedging on the same answer + ofs.close(); + remove(path_tmp.c_str()); + } return false; } if (p.downloaded == 0 && response.status != 200) { @@ -214,7 +224,7 @@ static bool common_pull_file(httplib::Client & cli, if (progress_step >= p.total / 1000 || p.downloaded == p.total) { if (callback) { callback->on_update(p); - if (callback->is_cancelled()) { + if (callback->is_cancelled() && (p.total == 0 || p.downloaded != p.total)) { return false; } } @@ -331,10 +341,24 @@ static int common_download_file_single_online(const std::string & url, } bool success = false; - const std::string path_temporary = path + ".downloadInProgress"; + const std::string path_temporary = path + PARTIAL_SUFFIX; int delay = retry_delay_seconds; + if (!skip_etag && !etag.empty()) { + if (std::filesystem::exists(path_temporary) && read_etag(path_temporary) != etag) { + LOG_WRN("%s: partial file does not match the server etag, discarding it: %s\n", __func__, path_temporary.c_str()); + if (remove(path_temporary.c_str()) != 0) { + LOG_ERR("%s: unable to delete file: %s\n", __func__, path_temporary.c_str()); + return -1; + } + } + write_etag(path_temporary, etag); + } + if (opts.callback) { + // report the resume offset, so a resumed file does not start at 0 + p.downloaded = supports_ranges && std::filesystem::exists(path_temporary) ? + std::filesystem::file_size(path_temporary) : 0; opts.callback->on_start(p); } @@ -372,6 +396,7 @@ static int common_download_file_single_online(const std::string & url, } if (!etag.empty() && !skip_etag) { write_etag(path, etag); + remove((path_temporary + ".etag").c_str()); } success = true; break; @@ -381,12 +406,7 @@ static int common_download_file_single_online(const std::string & url, if (opts.callback) { opts.callback->on_done(p, success); } - if (opts.callback && opts.callback->is_cancelled() && - std::filesystem::exists(path_temporary)) { - if (remove(path_temporary.c_str()) != 0) { - LOG_ERR("%s: unable to delete temporary file: %s\n", __func__, path_temporary.c_str()); - } - } + // a stopped download keeps path_temporary, so the next attempt resumes from it if (!success) { LOG_ERR("%s: download failed after %d attempts\n", __func__, max_attempts); return -1; // max attempts reached @@ -594,33 +614,65 @@ static hf_cache::hf_file find_best_sibling(const hf_cache::hf_files & files, return best; } -static hf_cache::hf_file find_best_mmproj(const hf_cache::hf_files & files, - const std::string & model) { - return find_best_sibling(files, model, "mmproj"); +static const std::vector sidecars = { + { "mmproj", "mmproj", false, &common_download_opts::download_mmproj, &common_download_hf_plan::mmproj }, + { "mtp", "mtp-", true, &common_download_opts::download_mtp, &common_download_hf_plan::mtp }, + { "dflash", "dflash-", true, &common_download_opts::download_dflash, &common_download_hf_plan::dflash }, + { "eagle3", "eagle3-", true, &common_download_opts::download_eagle3, &common_download_hf_plan::eagle3 }, + { "dspark", "dspark-", true, &common_download_opts::download_dspark, &common_download_hf_plan::dspark }, +}; + +const std::vector & common_download_sidecars() { + return sidecars; } -static hf_cache::hf_file find_best_mtp(const hf_cache::hf_files & files, - const std::string & model, - const std::string & tag = "") { - return find_best_sibling(files, model, "mtp-", tag); +bool common_download_opts_add_sidecar(common_download_opts & opts, const std::string & kind) { + for (const auto & s : sidecars) { + if (kind == s.name) { + opts.*s.opt = true; + return true; + } + } + return false; } -static hf_cache::hf_file find_best_eagle3(const hf_cache::hf_files & files, - const std::string & model, - const std::string & tag = "") { - return find_best_sibling(files, model, "eagle3-", tag); +std::vector common_download_plan_sidecars(const common_download_hf_plan & plan) { + std::vector out; + for (const auto & s : sidecars) { + if (!(plan.*s.file).path.empty()) { + out.emplace_back(s.name); + } + } + return out; } -static hf_cache::hf_file find_best_dflash(const hf_cache::hf_files & files, - const std::string & model, - const std::string & tag = "") { - return find_best_sibling(files, model, "dflash-", tag); +std::vector common_download_plan_partials(const common_download_hf_plan & plan) { + std::vector out; + + // a file in the cache resolves to its snapshot entry, a file still to download to its blob + auto add = [&](const std::string & kind, const hf_cache::hf_file & f) { + if (!f.local_path.empty() && f.local_path != f.final_path) { + out.push_back({ kind, f.local_path + PARTIAL_SUFFIX }); + } + }; + + for (const auto & f : plan.model_files) { + add("", f); + } + for (const auto & s : sidecars) { + add(s.name, plan.*s.file); + } + return out; } -static hf_cache::hf_file find_best_dspark(const hf_cache::hf_files & files, - const std::string & model, - const std::string & tag = "") { - return find_best_sibling(files, model, "dspark-", tag); +// true when `name` carries the keyword of a sidecar kind +static bool name_is_sidecar(const std::string & name) { + for (const auto & s : sidecars) { + if (name.find(s.keyword) != std::string::npos) { + return true; + } + } + return false; } static bool gguf_filename_is_model(const std::string & filepath) { @@ -633,12 +685,7 @@ static bool gguf_filename_is_model(const std::string & filepath) { filename = filename.substr(pos + 1); } - return filename.find("mmproj") == std::string::npos && - filename.find("imatrix") == std::string::npos && - filename.find("mtp-") == std::string::npos && - filename.find("eagle3-") == std::string::npos && - filename.find("dflash-") == std::string::npos && - filename.find("dspark-") == std::string::npos; + return filename.find("imatrix") == std::string::npos && !name_is_sidecar(filename); } static hf_cache::hf_file find_best_model(const hf_cache::hf_files & files, @@ -730,8 +777,11 @@ common_download_hf_plan common_download_get_hf_plan(const common_params_model & } } else { primary = find_best_model(all, tag); - // a requested sidecar can resolve on its own, without a full model of the same tag - if (primary.path.empty() && !opts.download_mtp && !opts.download_dflash && !opts.download_eagle3 && !opts.download_dspark) { + // a requested draft can resolve on its own, without a full model of the same tag + const bool want_draft = std::any_of(sidecars.begin(), sidecars.end(), [&](const common_download_sidecar & s) { + return s.draft && opts.*s.opt; + }); + if (primary.path.empty() && !want_draft) { LOG_ERR("%s: no GGUF files found in repository %s\n", __func__, repo.c_str()); list_available_gguf_files(all); return plan; @@ -743,24 +793,22 @@ common_download_hf_plan common_download_get_hf_plan(const common_params_model & plan.model_files = get_split_files(all, primary); } - if (opts.download_mmproj && !primary.path.empty()) { - plan.mmproj = find_best_mmproj(all, primary.path); - } - if (opts.download_mtp) { - plan.mtp = find_best_mtp(all, primary.path, tag); - } - if (opts.download_dflash) { - plan.dflash = find_best_dflash(all, primary.path, tag); - } - if (opts.download_eagle3) { - plan.eagle3 = find_best_eagle3(all, primary.path, tag); - } - if (opts.download_dspark) { - plan.dspark = find_best_dspark(all, primary.path, tag); + // a draft resolves at the tag, the mmproj pairs with the model + for (const auto & s : sidecars) { + if (!(opts.*s.opt)) { + continue; + } + if (s.draft) { + plan.*s.file = find_best_sibling(all, primary.path, s.keyword, tag); + } else if (!primary.path.empty()) { + plan.*s.file = find_best_sibling(all, primary.path, s.keyword); + } } - if (primary.path.empty() && - plan.mtp.local_path.empty() && plan.dflash.local_path.empty() && plan.eagle3.local_path.empty() && plan.dspark.local_path.empty()) { + const bool has_draft = std::any_of(sidecars.begin(), sidecars.end(), [&](const common_download_sidecar & s) { + return s.draft && !(plan.*s.file).local_path.empty(); + }); + if (primary.path.empty() && !has_draft) { LOG_ERR("%s: no GGUF files found in repository %s\n", __func__, repo.c_str()); list_available_gguf_files(all); } @@ -926,6 +974,11 @@ std::string common_docker_resolve_model(const std::string & docker) { } } +// a cached file that lists a model under its tag: the first split of a model, not a sidecar +static bool is_listed_model(const gguf_split_info & split) { + return split.index == 1 && !split.tag.empty() && !name_is_sidecar(split.prefix); +} + std::vector common_list_cached_models() { std::unordered_set seen; std::vector result; @@ -934,12 +987,7 @@ std::vector common_list_cached_models() { for (const auto & f : files) { auto split = get_gguf_split_info(f.path); - if (split.index != 1 || split.tag.empty() || - split.prefix.find("mmproj") != std::string::npos || - split.prefix.find("mtp-") != std::string::npos || - split.prefix.find("eagle3-") != std::string::npos || - split.prefix.find("dflash-") != std::string::npos || - split.prefix.find("dspark-") != std::string::npos) { + if (!is_listed_model(split)) { continue; } if (seen.insert(f.repo_id + ":" + split.tag).second) { @@ -970,6 +1018,44 @@ std::string common_download_resolve_path(const std::string & hf_repo_with_tag, c return find_best_model(files, tag).local_path; } +// the files -hf loads from the cache, the model with every sidecar, for each cached +// tag of `repo_id` other than `tag` +static std::unordered_set files_in_use(const std::string & repo_id, const std::string & tag) { + common_download_opts opts; + opts.offline = true; + for (const auto & s : sidecars) { + opts.*s.opt = true; + } + + std::unordered_set tags; + std::unordered_set used; + + for (const auto & f : hf_cache::get_cached_files(repo_id)) { + auto split = get_gguf_split_info(f.path); + if (!is_listed_model(split) || split.tag == tag || !tags.insert(split.tag).second) { + continue; + } + + common_params_model model; + model.hf_repo = repo_id + ":" + split.tag; + + auto plan = common_download_get_hf_plan(model, opts); + + for (const auto & m : plan.model_files) { + used.insert(m.local_path); + } + for (const auto & s : sidecars) { + const auto & sf = plan.*s.file; + if (!sf.local_path.empty()) { + used.insert(sf.local_path); + } + } + } + return used; +} + +static bool remove_snapshot_entries(const std::string & repo_id, const std::vector & to_remove); + bool common_download_remove(const std::string & hf_repo_with_tag) { namespace fs = std::filesystem; @@ -989,11 +1075,13 @@ bool common_download_remove(const std::string & hf_repo_with_tag) { return false; } - // collect snapshot entries whose tag matches + // collect snapshot entries whose tag matches, but a file another tag loads stays + const auto used = files_in_use(repo_id, tag_upper); + std::vector to_remove; for (const auto & f : files) { auto split = get_gguf_split_info(f.path); - if (split.tag == tag_upper) { + if (split.tag == tag_upper && used.find(f.local_path) == used.end()) { to_remove.emplace_back(f.local_path); } } @@ -1002,6 +1090,13 @@ bool common_download_remove(const std::string & hf_repo_with_tag) { return false; } + return remove_snapshot_entries(repo_id, to_remove); +} + +// remove snapshot entries of `repo_id`, then the blobs no remaining entry references +static bool remove_snapshot_entries(const std::string & repo_id, const std::vector & to_remove) { + namespace fs = std::filesystem; + // resolve blob paths from symlinks before deleting snapshot entries std::vector blobs_to_check; for (const auto & p : to_remove) { @@ -1053,3 +1148,26 @@ bool common_download_remove(const std::string & hf_repo_with_tag) { return true; } + +bool common_download_remove_sidecars(const std::string & hf_repo_with_tag, const common_download_opts & opts) { + common_params_model model; + model.hf_repo = hf_repo_with_tag; + + common_download_opts cached = opts; + cached.offline = true; + + auto plan = common_download_get_hf_plan(model, cached); + + std::vector to_remove; + for (const auto & s : sidecars) { + const auto & f = plan.*s.file; + if (!f.local_path.empty()) { + to_remove.emplace_back(std::filesystem::u8path(f.local_path)); + } + } + if (to_remove.empty()) { + return false; + } + + return remove_snapshot_entries(common_download_split_repo_tag(hf_repo_with_tag).first, to_remove); +} diff --git a/common/download.h b/common/download.h index 8c30cfc3e..4dfd0c983 100644 --- a/common/download.h +++ b/common/download.h @@ -104,10 +104,17 @@ std::string common_docker_resolve_model(const std::string & docker); // Remove a cached model from disk // input format: "user/model" or "user/model:tag" // - if tag is omitted, removes the entire repo cache directory -// - if tag is present, removes only files matching that tag (and orphaned blobs) +// - if tag is present, removes only files matching that tag (and orphaned blobs), +// except the files another cached tag of the repo still loads // returns true if anything was removed bool common_download_remove(const std::string & hf_repo_with_tag); +// remove the cached sidecar files `opts` selects for "user/model:tag", as the cached +// plan resolves them, and leave the model files in place; a sidecar file another tag +// shares goes for every tag that resolves it +// returns true if anything was removed +bool common_download_remove_sidecars(const std::string & hf_repo_with_tag, const common_download_opts & opts); + struct common_download_hf_plan { hf_cache::hf_file primary; hf_cache::hf_files model_files; @@ -119,3 +126,33 @@ struct common_download_hf_plan { hf_cache::hf_file preset; // if set, only this file is downloaded }; common_download_hf_plan common_download_get_hf_plan(const common_params_model & model, const common_download_opts & opts); + +// a sidecar kind a download can pull next to its model: mmproj, and the draft heads +// of --spec-type draft- +struct common_download_sidecar { + const char * name; // the kind, as the API names it + const char * keyword; // the part of a file name that marks a file of this kind + bool draft; // a draft head, resolved at the tag even without a model + + bool common_download_opts::* opt; // the option that requests it + hf_cache::hf_file common_download_hf_plan::* file; // the plan file it resolves to +}; + +// every sidecar kind, the single list the resolution, the listing and the removal follow +const std::vector & common_download_sidecars(); + +// select the sidecar `kind` in `opts`; returns false for an unknown kind +bool common_download_opts_add_sidecar(common_download_opts & opts, const std::string & kind); + +// the sidecar kinds `plan` resolves a file for +std::vector common_download_plan_sidecars(const common_download_hf_plan & plan); + +// a file `plan` still has to download, at the path its partial takes while it is +// incomplete; kind is the sidecar it belongs to, empty for the model files +struct common_download_partial { + std::string kind; + std::string path; +}; + +// the partials the download of `plan` writes, for the files not in the cache yet +std::vector common_download_plan_partials(const common_download_hf_plan & plan); diff --git a/tests/test-model-resolution.cpp b/tests/test-model-resolution.cpp index 5c793e24a..eb7b30936 100644 --- a/tests/test-model-resolution.cpp +++ b/tests/test-model-resolution.cpp @@ -15,6 +15,7 @@ #include #include #include +#include #include #include #include @@ -339,6 +340,124 @@ static void test_plan_resolution() { g_repos.clear(); } +// put a file in the cache the way a finished download leaves it, refs/main included +static void cache_put(const std::string & repo_id, const std::string & path) { + namespace fs = std::filesystem; + + const fs::path local = cached(repo_id, path); + fs::create_directories(local.parent_path()); + { std::ofstream(local) << "gguf"; } + + const fs::path ref = local.parent_path().parent_path().parent_path() / "refs" / "main"; + fs::create_directories(ref.parent_path()); + if (!fs::exists(ref)) { + std::ofstream(ref) << COMMIT << "\n"; + } +} + +// +// partials of a download: the files of its plan not in the cache yet, next to +// the blobs, named after the oids the listing serves +// + +static void test_plan_partials() { + namespace fs = std::filesystem; + + printf("test-model-resolution: plan partials\n"); + + g_context = "plan partials"; + g_repos.clear(); + g_repos["test/partial"] = { "model-Q4_0.gguf", "mmproj-model-F16.gguf", "mtp-model-Q4_0.gguf" }; + + // the mmproj is complete, the model and its mtp are still to download + cache_put("test/partial", "mmproj-model-F16.gguf"); + + common_download_opts opts; + opts.download_mmproj = true; + opts.download_mtp = true; + + const auto parts = common_download_plan_partials(common_download_get_hf_plan(model_ref("test/partial:Q4_0"), opts)); + + const fs::path blobs = cache_dir / "models--test--partial" / "blobs"; + REQUIRE(parts.size() == 2); + REQUIRE(parts[0].kind.empty()); + REQUIRE(fs::path(parts[0].path).lexically_normal() == (blobs / "0000000000000000000000000000000000000001.downloadInProgress").lexically_normal()); + REQUIRE_EQ(parts[1].kind, "mtp"); + REQUIRE(fs::path(parts[1].path).lexically_normal() == (blobs / "0000000000000000000000000000000000000003.downloadInProgress").lexically_normal()); + + fs::remove_all(cache_dir / "models--test--partial"); + g_repos.clear(); +} + +// +// removal of a sidecar alone: its snapshot entry goes, the model keeps its files +// + +static void test_sidecar_removal() { + namespace fs = std::filesystem; + + printf("test-model-resolution: sidecar removal\n"); + + g_context = "sidecar removal"; + g_repos.clear(); + + cache_put("test/sidecar", "model-Q4_0.gguf"); + cache_put("test/sidecar", "dflash-model-Q4_0.gguf"); + cache_put("test/sidecar", "mtp-model-Q4_0.gguf"); + + common_download_opts opts; + opts.download_dflash = true; + + // the selected sidecar goes, the other one and the model stay + REQUIRE(common_download_remove_sidecars("test/sidecar:Q4_0", opts)); + REQUIRE(!fs::exists(cached("test/sidecar", "dflash-model-Q4_0.gguf"))); + REQUIRE(fs::exists(cached("test/sidecar", "mtp-model-Q4_0.gguf"))); + REQUIRE(fs::exists(cached("test/sidecar", "model-Q4_0.gguf"))); + REQUIRE(!common_download_remove_sidecars("test/sidecar:Q4_0", opts)); + + fs::remove_all(cache_dir / "models--test--sidecar"); +} + +// +// removal of a tag keeps the files another cached tag still loads: a sidecar +// named after the removed tag that another quant resolves stays on disk +// + +static void test_shared_removal() { + namespace fs = std::filesystem; + + printf("test-model-resolution: shared removal\n"); + + g_context = "shared removal"; + g_repos.clear(); + + for (const char * f : { "model-Q8_0.gguf", "model-BF16.gguf", "mmproj-model-BF16.gguf", "mtp-model-BF16.gguf" }) { + cache_put("test/shared", f); + } + + // the BF16 model goes, the mmproj and the mtp Q8_0 resolves stay + REQUIRE(common_download_remove("test/shared:BF16")); + REQUIRE(!fs::exists(cached("test/shared", "model-BF16.gguf"))); + REQUIRE(fs::exists(cached("test/shared", "mmproj-model-BF16.gguf"))); + REQUIRE(fs::exists(cached("test/shared", "mtp-model-BF16.gguf"))); + REQUIRE(fs::exists(cached("test/shared", "model-Q8_0.gguf"))); + + fs::remove_all(cache_dir / "models--test--shared"); + + // with no other tag, the files of the removed tag all go, sidecars included + for (const char * f : { "model-BF16.gguf", "mmproj-model-BF16.gguf", "mtp-model-BF16.gguf" }) { + cache_put("test/alone", f); + } + + REQUIRE(common_download_remove("test/alone:BF16")); + REQUIRE(!fs::exists(cached("test/alone", "model-BF16.gguf"))); + REQUIRE(!fs::exists(cached("test/alone", "mmproj-model-BF16.gguf"))); + REQUIRE(!fs::exists(cached("test/alone", "mtp-model-BF16.gguf"))); + REQUIRE(!common_download_remove("test/alone:BF16")); + + fs::remove_all(cache_dir / "models--test--alone"); +} + // // end-to-end assembly: real CLI parsing, real handler init resolving over the // loopback, downloads skipped by flipping offline before apply @@ -509,6 +628,9 @@ int main(void) { common_set_env("MODEL_ENDPOINT", "http://127.0.0.1:" + std::to_string(port) + "/"); test_plan_resolution(); + test_plan_partials(); + test_sidecar_removal(); + test_shared_removal(); test_task_assembly(); server.stop(); diff --git a/tools/server/README.md b/tools/server/README.md index c42336c7f..ec82db007 100644 --- a/tools/server/README.md +++ b/tools/server/README.md @@ -2106,6 +2106,32 @@ Note: for "downloading" state, there can be multiple files be downloading in par } ``` +A download that never finished (paused or interrupted) is listed under a separate +"incomplete" status, without an instance: posting the model to `/models` again resumes it +from the files already on disk, and `DELETE /models` drops its partial files. + +```json +"status": { + "value": "incomplete", + "args": [] +} +``` + +Cached models and incomplete downloads carry the sidecar kinds they have on disk +(`mmproj`, `mtp`, `dflash`, `eagle3`, `dspark`), resolved the way `-hf` loads them: + +```json +"sidecars": ["mmproj", "mtp"] +``` + +A cached model whose download of more sidecars never finished also lists the kinds not on +disk yet: posting the model with them resumes them, and `DELETE /models` with `&sidecar={kind}` +drops their partial files. + +```json +"incomplete_sidecars": ["mtp"] +``` + ### POST `/models/load`: Load a model Load a model @@ -2225,7 +2251,7 @@ Example events: Trigger a new download (non-blocking), the progress can be tracked via SSE endpoint `/models/sse` -To cancel model downloading, send an event to `/models/unload` +To pause a download, send an event to `/models/unload`: it keeps its partial files and is listed as "incomplete" until it is posted again. To cancel it and drop its files, use `DELETE /models`. Download procedure: - Send POST request to `/models` @@ -2238,9 +2264,14 @@ Payload: ```json { "model": "ggml-org/gemma-3-4b-it-GGUF:Q4_K_M", + "sidecars": ["mmproj", "mtp"] } ``` +`sidecars` lists the sidecar kinds pulled with the model, picked by the server the way `-hf` picks them: `mmproj`, and the draft heads `mtp`, `dflash`, `eagle3` and `dspark`. It defaults to `["mmproj"]`, as `-hf` does. + +Posting a model that is already in the cache and not running downloads the sidecars it lacks and keeps the files already on disk. + Response (download is started in the background): ```json @@ -2267,6 +2298,10 @@ IMPORTANT: only model stored in cache can be deleted. You cannot delete models i Model name must be passed via query param: `?model={name}` +To remove one sidecar of a cached model and keep the model, add `&sidecar={kind}` (`mmproj`, `mtp`, `dflash`, `eagle3` or `dspark`); the model must not be running. + +Deleting a model keeps the files another cached quant of the repo still loads, such as an mmproj named after the deleted quant, and the partial files another download still writes. A sidecar deleted on its own goes for every quant that shares it. + If delete success, it will send an SSE event of type `model_remove` Response: diff --git a/tools/server/server-models.cpp b/tools/server/server-models.cpp index 60353240a..d77837c86 100644 --- a/tools/server/server-models.cpp +++ b/tools/server/server-models.cpp @@ -26,6 +26,7 @@ #include #include #include +#include #include #include #include @@ -618,6 +619,7 @@ server_models::server_models( LOG_WRN("failed to get server executable path: %s\n", e.what()); LOG_WRN("using original argv[0] as fallback: %s\n", argv[0]); } + unfinished_downloads.load(); load_models(); debug_fake_timing = !common_get_env("LLAMA_SERVER_DEBUG_FAKE_TIMING").empty(); } @@ -686,6 +688,233 @@ void server_models::add_model(server_model_meta && meta) { }; } +static bool has_sidecar(const std::vector & sidecars, const std::string & kind) { + return std::find(sidecars.begin(), sidecars.end(), kind) != sidecars.end(); +} + +// reject a sidecar kind no download can pull +static void check_sidecar(const std::string & kind) { + std::vector kinds; + for (const auto & s : common_download_sidecars()) { + kinds.emplace_back(s.name); + } + if (!has_sidecar(kinds, kind)) { + throw std::invalid_argument("unknown sidecar '" + kind + "', expected one of " + string_join(kinds, ", ")); + } +} + +// the sidecars a download asks for; by default the ones -hf pulls without a draft type +static std::vector parse_sidecars(const json & body) { + std::vector defaults; + for (const auto & s : common_download_sidecars()) { + if (!s.draft) { + defaults.emplace_back(s.name); + } + } + std::vector sidecars = json_value(body, "sidecars", defaults); + for (const auto & kind : sidecars) { + check_sidecar(kind); + } + return sidecars; +} + +// the download options of a model with its sidecars, the plan a removal purges partials from +static common_download_opts sidecar_download_opts(const common_params & params, const std::vector & sidecars) { + common_download_opts opts; + opts.bearer_token = params.hf_token; + for (const auto & kind : sidecars) { + common_download_opts_add_sidecar(opts, kind); + } + return opts; +} + +// the HF repo a model resolves from: a preset names its own, a cached model is +// listed under it, and a local file or a models dir entry has none +static std::string model_hf_repo(const server_model_meta & meta) { + std::string hf_repo; + if (meta.preset.get_option("LLAMA_ARG_HF_REPO", hf_repo) && !hf_repo.empty()) { + return hf_repo; + } + return meta.source == SERVER_MODEL_SOURCE_CACHE ? meta.name : ""; +} + +// the sidecar kinds of a HF repo present on disk, as the offline plan resolves them +static json cached_sidecars(const std::string & hf_repo) { + common_params_model model; + model.hf_repo = hf_repo; + + common_download_opts opts; + opts.offline = true; + for (const auto & s : common_download_sidecars()) { + opts.*s.opt = true; + } + + return common_download_plan_sidecars(common_download_get_hf_plan(model, opts)); +} + +// replace any inherited value of `key` in a child environment +static void set_child_env(std::vector & env, const std::string & key, const std::string & value) { + env.erase(std::remove_if(env.begin(), env.end(), [&](const std::string & e) { + return e.rfind(key + "=", 0) == 0; + }), env.end()); + env.push_back(key + "=" + value); +} + +// where the unfinished downloads are remembered, inside the model cache +static std::string unfinished_downloads_path() { + return hf_cache::get_cache_path() + "/unfinished-downloads.json"; +} + +void server_unfinished_downloads::load() { + std::ifstream in(unfinished_downloads_path()); + if (!in) { + return; + } + std::string raw((std::istreambuf_iterator(in)), std::istreambuf_iterator()); + std::lock_guard lk(mutex); + try { + json data = json::parse(raw); + for (const auto & d : data.value("downloads", json::array())) { + server_unfinished_download download; + download.model = json_value(d, "model", std::string()); + download.sidecars = json_value(d, "sidecars", std::vector()); + for (const auto & p : d.value("partials", json::array())) { + download.partials.push_back({ json_value(p, "kind", std::string()), json_value(p, "path", std::string()) }); + } + if (!download.model.empty()) { + downloads.push_back(std::move(download)); + } + } + } catch (const std::exception & e) { + LOG_WRN("failed to read unfinished downloads: %s\n", e.what()); + } +} + +void server_unfinished_downloads::save() { + json data; + data["downloads"] = json::array(); + for (const auto & d : downloads) { + json partials = json::array(); + for (const auto & p : d.partials) { + partials.push_back({ {"kind", p.kind}, {"path", p.path} }); + } + data["downloads"].push_back({ {"model", d.model}, {"sidecars", d.sidecars}, {"partials", partials} }); + } + try { + fs_write_atomic(std::filesystem::u8path(unfinished_downloads_path()), data.dump()); + } catch (const std::exception & e) { + LOG_WRN("failed to record unfinished downloads: %s\n", e.what()); + } +} + +void server_unfinished_downloads::delete_partials(const std::vector & parts) { + for (const auto & p : parts) { + const bool shared = std::any_of(downloads.begin(), downloads.end(), [&](const server_unfinished_download & d) { + return std::any_of(d.partials.begin(), d.partials.end(), [&](const common_download_partial & q) { + return q.path == p.path; + }); + }); + if (shared) { + continue; + } + std::error_code ec; + std::filesystem::remove(std::filesystem::u8path(p.path), ec); + } +} + +void server_unfinished_downloads::add(const server_unfinished_download & download) { + std::lock_guard lk(mutex); + std::vector stale; + auto it = std::find_if(downloads.begin(), downloads.end(), [&](const auto & d) { return d.model == download.model; }); + if (it != downloads.end()) { + stale = std::move(it->partials); + *it = download; + } else { + downloads.push_back(download); + } + save(); + delete_partials(stale); +} + +void server_unfinished_downloads::forget(const std::string & model) { + std::lock_guard lk(mutex); + auto it = std::find_if(downloads.begin(), downloads.end(), [&](const auto & d) { return d.model == model; }); + if (it != downloads.end()) { + downloads.erase(it); + save(); + } +} + +bool server_unfinished_downloads::remove(const std::string & model) { + std::lock_guard lk(mutex); + auto it = std::find_if(downloads.begin(), downloads.end(), [&](const auto & d) { return d.model == model; }); + if (it == downloads.end()) { + return false; + } + auto partials = std::move(it->partials); + downloads.erase(it); + save(); + delete_partials(partials); + return true; +} + +bool server_unfinished_downloads::remove_sidecar(const std::string & model, const std::string & kind) { + std::lock_guard lk(mutex); + auto it = std::find_if(downloads.begin(), downloads.end(), [&](const auto & d) { return d.model == model; }); + if (it == downloads.end() || !has_sidecar(it->sidecars, kind)) { + return false; + } + it->sidecars.erase(std::remove(it->sidecars.begin(), it->sidecars.end(), kind), it->sidecars.end()); + + std::vector dropped; + auto & partials = it->partials; + auto pos = std::stable_partition(partials.begin(), partials.end(), [&](const common_download_partial & p) { + return p.kind != kind; + }); + dropped.assign(pos, partials.end()); + partials.erase(pos, partials.end()); + + save(); + delete_partials(dropped); + return true; +} + +std::vector server_unfinished_downloads::get_all() { + std::lock_guard lk(mutex); + return downloads; +} + +std::vector server_models::unfinished_downloads_get() { + return unfinished_downloads.get_all(); +} + +void server_models::unfinished_downloads_remove(const std::string & name) { + // under the lock, so a download of the same tag cannot start while its files go + std::lock_guard lk(mutex); + if (mapping.find(name) != mapping.end()) { + throw std::invalid_argument("model '" + name + "' is downloading"); + } + // a name with no record is no model: its files belong to the cached tags of the repo + if (!unfinished_downloads.remove(name)) { + throw std::runtime_error("model name=" + name + " is not found"); + } + bool ok = common_download_remove(name); + SRV_INF("removing unfinished download name=%s (%s)\n", name.c_str(), ok ? "succeeded" : "nothing cached"); +} + +bool server_models::remove_sidecar(const std::string & name, const std::string & kind) { + std::lock_guard lk(mutex); + auto it = mapping.find(name); + if (it != mapping.end() && (it->second.meta.is_running() || it->second.meta.status == SERVER_MODEL_STATUS_DOWNLOADING)) { + throw std::invalid_argument("model '" + name + "' is in use, unload it first"); + } + // a sidecar that only ever landed as a partial is still something removed + bool ok = unfinished_downloads.remove_sidecar(name, kind); + ok = common_download_remove_sidecars(name, sidecar_download_opts(base_params, { kind })) || ok; + SRV_INF("removing sidecar %s of model name=%s (%s)\n", kind.c_str(), name.c_str(), ok ? "succeeded" : "nothing cached"); + return ok; +} + void server_models::notify_sse(const std::string & event, const std::string & model_id, const json & data) { std::unique_ptr result = std::make_unique(); result->data = { @@ -1135,7 +1364,8 @@ void server_models::load(const std::string & name) { } void server_models::load(const std::string & name, const load_options & opts) { - if (debug_fake_timing) { + // the fake load time of a model; a download worker loads none + if (debug_fake_timing && opts.mode == SERVER_CHILD_MODE_NORMAL) { // do not hold the mutex here, other requests must keep making progress std::this_thread::sleep_for(std::chrono::seconds(2)); } @@ -1152,6 +1382,43 @@ void server_models::load(const std::string & name, const load_options & opts) { // against the freshest preset and a consistent mapping state cv.wait(lk, [this]() { return !is_reloading; }); + // a download over an existing entry is decided under the lock, so a load of the + // same model cannot start in between: a paused download or an idle cached model + // is downloaded again with its files kept, a download on its way and a running + // model are refused, and a preset or local model is not a download + if (opts.mode == SERVER_CHILD_MODE_DOWNLOAD) { + auto it = mapping.find(name); + if (it != mapping.end()) { + const auto & cur = it->second.meta; + if (cur.status == SERVER_MODEL_STATUS_DOWNLOADING) { + throw std::invalid_argument("model '" + name + "' is already downloading"); + } + if (cur.is_running()) { + throw std::invalid_argument("model '" + name + "' is running, unload it first"); + } + if (cur.source != SERVER_MODEL_SOURCE_CACHE) { + throw std::invalid_argument("model '" + name + "' already exists"); + } + SRV_INF("resuming download for model '%s'\n", name.c_str()); + } + + // two downloads writing the same partial would interleave their bytes + for (const auto & d : unfinished_downloads.get_all()) { + auto other = mapping.find(d.model); + if (d.model == name || other == mapping.end() || other->second.meta.status != SERVER_MODEL_STATUS_DOWNLOADING) { + continue; + } + for (const auto & p : d.partials) { + const bool shared = std::any_of(opts.partials.begin(), opts.partials.end(), [&](const common_download_partial & q) { + return q.path == p.path; + }); + if (shared) { + throw std::invalid_argument("model '" + name + "' shares files with the download of '" + d.model + "' on its way, retry once it is done"); + } + } + } + } + auto meta = opts.custom_meta.has_value() ? *opts.custom_meta : mapping[name].meta; if (meta.status != SERVER_MODEL_STATUS_UNLOADED) { SRV_INF("model %s is not ready\n", name.c_str()); @@ -1201,6 +1468,23 @@ void server_models::load(const std::string & name, const load_options & opts) { inst.meta.status = SERVER_MODEL_STATUS_DOWNLOADING; child_env.push_back("LLAMA_SERVER_CHILD_MODE=download"); child_env.push_back("LLAMA_ARG_HF_REPO=" + name); + + // the sidecars as -hf would pull them: the draft types, and the mmproj switch + // for the one that is not a draft + bool mmproj = false; + std::string spec_types; + for (const auto & s : common_download_sidecars()) { + if (!has_sidecar(opts.sidecars, s.name)) { + continue; + } + if (s.draft) { + spec_types += (spec_types.empty() ? "" : ",") + std::string("draft-") + s.name; + } else { + mmproj = true; + } + } + set_child_env(child_env, "LLAMA_ARG_MMPROJ_AUTO", mmproj ? "1" : "0"); + set_child_env(child_env, "LLAMA_ARG_SPEC_TYPE", spec_types.empty() ? "none" : spec_types); } SRV_INF("%s", "spawning server instance with args:\n"); @@ -1216,6 +1500,11 @@ void server_models::load(const std::string & name, const load_options & opts) { } } + // remember the download: the partial files carry a content hash, not the name + if (opts.mode == SERVER_CHILD_MODE_DOWNLOAD) { + unfinished_downloads.add({ name, opts.sidecars, opts.partials }); + } + // old process should have exited already, but just in case, we clean it up here { auto it = mapping.find(name); @@ -1390,6 +1679,12 @@ void server_models::update_download_progress(const std::string & name, const com // mark the instance to be erased on next load_models() call it->second.meta.status = SERVER_MODEL_STATUS_DOWNLOADED; need_reload = true; + + // a download that failed keeps its tag: its partial files are still + // there and re-posting the tag resumes them + if (ok) { + unfinished_downloads.forget(name); + } } else { json & info = it->second.meta.loaded_info; if (!info.contains("progress")) { @@ -1445,24 +1740,14 @@ bool server_models::remove(const std::string & name) { || meta.status == SERVER_MODEL_STATUS_DOWNLOADED; }); - // re-find after wait - load_models() may have erased the entry during the wait - it = mapping.find(name); - if (it == mapping.end()) { - // load_models() already erased the entry; we just need to clean up the cached files on disk - lk.unlock(); - bool ok = common_download_remove(name); - SRV_INF("removing model name=%s from cache (%s)\n", name.c_str(), ok ? "succeeded" : "partial"); - notify_sse("model_remove", name, {}); - return true; - } - - // remove from disk (best-effort: cancelled downloads may have no cached files) - bool ok = common_download_remove(name); + // load_models() may have erased the entry during the wait; the files go under the + // lock, so a download of the same tag cannot start meanwhile mapping.erase(name); - if (!ok) { - SRV_WRN("removing model name=%s from disk returned false (no cached files?)\n", name.c_str()); - } - SRV_INF("removing model name=%s from cache (%s)\n", name.c_str(), ok ? "succeeded" : "partial"); + unfinished_downloads.remove(name); + + // best-effort: a cancelled download may have no cached files + bool ok = common_download_remove(name); + SRV_INF("removing model name=%s from cache (%s)\n", name.c_str(), ok ? "succeeded" : "nothing cached"); notify_sse("model_remove", name, {}); return true; } @@ -2091,6 +2376,9 @@ void server_models_routes::init_routes() { json models_json = json::array(); auto all_models = models.get_all_meta(); std::time_t t = std::time(0); + // a download on its way has no files to resolve yet, only the kinds it pulls + const auto unfinished = models.unfinished_downloads_get(); + for (const auto & meta : all_models) { if (meta.hidden) { continue; // cache model deduplicated by a preset @@ -2132,6 +2420,35 @@ void server_models_routes::init_routes() { model_info["context_length"] = meta.n_ctx_train; } + // the sidecars a model has on disk, resolved as -hf would load them, for a + // model that resolves from a HF repo. A download on its way reports the + // kinds it pulls, as an unfinished one does + const auto it_unfinished = std::find_if(unfinished.begin(), unfinished.end(), + [&](const server_unfinished_download & d) { return d.model == meta.name; }); + const std::string hf_repo = model_hf_repo(meta); + + if (it_unfinished != unfinished.end() && meta.status == SERVER_MODEL_STATUS_DOWNLOADING) { + model_info["sidecars"] = it_unfinished->sidecars; + } else if (!hf_repo.empty()) { + const json cached = cached_sidecars(hf_repo); + model_info["sidecars"] = cached; + + // the sidecars of an unfinished download of this model that are not on + // disk yet: a POST with the kind resumes one, a DELETE with it drops it + if (it_unfinished != unfinished.end()) { + std::vector incomplete; + for (const auto & p : it_unfinished->partials) { + if (!p.kind.empty() && !has_sidecar(incomplete, p.kind) && + std::find(cached.begin(), cached.end(), p.kind) == cached.end()) { + incomplete.push_back(p.kind); + } + } + if (!incomplete.empty()) { + model_info["incomplete_sidecars"] = incomplete; + } + } + } + // merge with loaded_info from the child process if available if (meta.is_running()) { for (auto it = meta.loaded_info.begin(); it != meta.loaded_info.end(); ++it) { @@ -2142,6 +2459,33 @@ void server_models_routes::init_routes() { } models_json.push_back(model_info); } + + const json architecture_default = json { + {"input_modalities", json::array({"text"})}, + {"output_modalities", json::array({"text"})}, + }; + + // a download that never finished has no instance: report it as incomplete so + // the UI can offer to resume it, without starting anything + for (const auto & d : unfinished) { + if (models.has_model(d.model)) { + continue; // complete, or downloading right now + } + models_json.push_back(json { + {"id", d.model}, + {"aliases", json::array()}, + {"tags", json::array()}, + {"object", "model"}, + {"owned_by", "llamacpp"}, + {"created", t}, + {"status", json { {"value", server_model_status_to_string(SERVER_MODEL_STATUS_INCOMPLETE)}, {"args", json::array()} }}, + {"architecture", architecture_default}, + {"source", server_model_source_to_string(SERVER_MODEL_SOURCE_CACHE)}, + {"can_remove", true}, + {"sidecars", d.sidecars}, + }); + } + res_ok(res, { {"data", models_json}, {"object", "list"}, @@ -2193,32 +2537,27 @@ void server_models_routes::init_routes() { if (name.empty()) { throw std::invalid_argument("model must be a non-empty string"); } + const auto sidecars = parse_sidecars(body); - common_params p; - p.model.hf_repo = name; - p.hf_token = params.hf_token; + common_params_model model; + model.hf_repo = name; - // validate by fetching metadata - bool ok = false; + // validate by fetching metadata, through the plan the download follows + common_download_hf_plan plan; try { - common_models_handler_init(p, LLAMA_EXAMPLE_SERVER); - ok = true; + plan = common_download_get_hf_plan(model, sidecar_download_opts(params, sidecars)); } catch (...) { SRV_ERR("unknown error while validating model '%s'\n", name.c_str()); // other exceptions will be handled by the outer ex_wrapper() throw; } - if (!ok) { - throw std::invalid_argument("model validation failed, unable to download"); + // a tag that resolves no file has nothing to download + if (plan.preset.path.empty() && plan.primary.path.empty() && common_download_plan_sidecars(plan).empty()) { + throw std::invalid_argument("model '" + name + "' resolves no file to download"); } - // reject if model already exists - if (models.has_model(name)) { - throw std::invalid_argument("model '" + name + "' already exists"); - } - - // then, proceed with the actual download + // then, proceed with the actual download; load() decides over an existing entry SRV_INF("starting download for model '%s'\n", name.c_str()); { server_models::load_options load_opts; @@ -2226,6 +2565,8 @@ void server_models_routes::init_routes() { load_opts.custom_meta = server_model_meta{}; load_opts.custom_meta->source = SERVER_MODEL_SOURCE_CACHE; load_opts.custom_meta->name = name; + load_opts.sidecars = sidecars; + load_opts.partials = common_download_plan_partials(plan); models.load(name, load_opts); } @@ -2241,6 +2582,22 @@ void server_models_routes::init_routes() { throw std::invalid_argument("model must be a non-empty string"); } + // a sidecar alone goes, the model keeps its files + std::string sidecar = req.get_param("sidecar"); + if (!sidecar.empty()) { + check_sidecar(sidecar); + res_ok(res, {{"success", models.remove_sidecar(name, sidecar)}}); + return res; + } + + // a download listed from an earlier run has no instance, only its tag and + // its partial files + if (!models.has_model(name)) { + models.unfinished_downloads_remove(name); + res_ok(res, {{"success", true}}); + return res; + } + models.remove(name); // throws on error res_ok(res, {{"success", true}}); diff --git a/tools/server/server-models.h b/tools/server/server-models.h index f415b7c5f..b04fc7384 100644 --- a/tools/server/server-models.h +++ b/tools/server/server-models.h @@ -16,6 +16,7 @@ #include #include #include +#include /** * state diagram: @@ -31,6 +32,7 @@ enum server_model_status { // TODO: also add downloading state when the logic is added SERVER_MODEL_STATUS_DOWNLOADING, + SERVER_MODEL_STATUS_INCOMPLETE, // a download that never finished, listed without an instance SERVER_MODEL_STATUS_DOWNLOADED, SERVER_MODEL_STATUS_UNLOADED, SERVER_MODEL_STATUS_LOADING, @@ -52,6 +54,7 @@ enum server_child_mode { static std::string server_model_status_to_string(server_model_status status) { switch (status) { case SERVER_MODEL_STATUS_DOWNLOADING: return "downloading"; + case SERVER_MODEL_STATUS_INCOMPLETE: return "incomplete"; case SERVER_MODEL_STATUS_DOWNLOADED: return "downloaded"; case SERVER_MODEL_STATUS_UNLOADED: return "unloaded"; case SERVER_MODEL_STATUS_LOADING: return "loading"; @@ -108,6 +111,51 @@ struct server_model_meta { void update_caps(const common_params & base); }; +// a download that never finished: the tag it was asked under, the sidecar kinds it +// pulls and the partial files its plan writes +struct server_unfinished_download { + std::string model; + std::vector sidecars; + std::vector partials; +}; + +// the downloads that never finished, recorded next to the model cache. The cache names +// partial files after their content hash, so this record is the only trace of what is +// still incomplete: it lets a restarted router list, resume or remove them +// a partial another recorded download also writes is never deleted +// these functions are thread-safe +struct server_unfinished_downloads { + // read the record an earlier run left + void load(); + + // record a download that starts, replacing an earlier record of the same model, + // and delete the partials of that record the new plan no longer writes + void add(const server_unfinished_download & download); + + // forget a download whose files are complete + void forget(const std::string & model); + + // forget a download and delete its partials; returns false when it is not recorded + bool remove(const std::string & model); + + // drop the sidecar `kind` from a download and delete its partials; returns false + // when the download is not recorded with that kind + bool remove_sidecar(const std::string & model, const std::string & kind); + + // every recorded download + std::vector get_all(); + +private: + std::mutex mutex; + std::vector downloads; + + // write the record back; callers hold the mutex + void save(); + + // delete the partials of `parts` no recorded download writes; callers hold the mutex + void delete_partials(const std::vector & parts); +}; + struct server_models_routes; struct server_lru_sched; // defined in server-models.cpp struct server_monitor; // defined in server-models.cpp @@ -134,6 +182,9 @@ private: // models asked to stop, still counted as running until the monitor records their exit std::set stopping_models; + // downloads that never finished, with the sidecar kinds they pull + server_unfinished_downloads unfinished_downloads; + // set to true while load_models() is executing a reload; load() will wait until clear bool is_reloading = false; @@ -252,6 +303,16 @@ public: // check if a model instance exists (thread-safe) bool has_model(const std::string & name); + // downloads that never finished, with their sidecar kinds (thread-safe) + std::vector unfinished_downloads_get(); + + // drop a download that will never be resumed, its record and its partial files; + // throws for a name that is not recorded (thread-safe) + void unfinished_downloads_remove(const std::string & name); + + // remove the sidecar `kind` of a model that is not in use, its files and partials (thread-safe) + bool remove_sidecar(const std::string & name, const std::string & kind); + // return a copy of model metadata (thread-safe) std::optional get_meta(const std::string & name); @@ -262,6 +323,10 @@ public: server_child_mode mode = SERVER_CHILD_MODE_NORMAL; // used for spawning a downloading child process std::optional custom_meta = std::nullopt; + // download mode: the sidecar kinds pulled with the model, as -hf would, and + // the partial files the download plan writes + std::vector sidecars; + std::vector partials; }; // load and unload model instances diff --git a/tools/server/tests/unit/test_router.py b/tools/server/tests/unit/test_router.py index 5355ab736..6df41f8d6 100644 --- a/tools/server/tests/unit/test_router.py +++ b/tools/server/tests/unit/test_router.py @@ -1,3 +1,4 @@ +import shutil import threading import pytest from utils import * @@ -546,6 +547,15 @@ def _wait_for_sse_event(collected: list, event_type: str, model: str, timeout: i return False +def _own_cache(name: str) -> str: + """An empty model cache for one test: the workers run in parallel, and a download + test must not see the files or the record another test writes in the shared cache.""" + path = os.path.join(TMP_DIR, name) + shutil.rmtree(path, ignore_errors=True) + os.makedirs(path) + return path + + def test_router_download_model(): """Case 1: download a model at the model limit, verify SSE events and GET /models.""" global server @@ -626,3 +636,230 @@ def test_router_delete_model(): # Model should no longer appear in GET /models ids = _get_model_ids(is_reload=False) assert MODEL_DOWNLOAD_ID not in ids, f"{MODEL_DOWNLOAD_ID} still present after deletion" + + +def test_router_download_sidecars(): + """Case 3: sidecars are a parameter of the download, never models of their own.""" + global server + server.llama_cache = _own_cache("cache_download_sidecars") + server.start() + + server.make_request("DELETE", f"/models?model={MODEL_DOWNLOAD_ID}") + + # an unknown sidecar kind is rejected before anything starts + res = server.make_request("POST", "/models", data={"model": MODEL_DOWNLOAD_ID, "sidecars": ["lora"]}) + assert res.status_code == 400 + assert MODEL_DOWNLOAD_ID not in _get_model_ids(is_reload=False) + + sse_events: list = [] + stop = threading.Event() + sse_ready = threading.Event() + threading.Thread( + target=_listen_sse, args=(server, sse_events, stop, sse_ready), daemon=True + ).start() + assert sse_ready.wait(10), "SSE client failed to connect" + + res = server.make_request("POST", "/models", data={"model": MODEL_DOWNLOAD_ID, "sidecars": []}) + assert res.status_code == 200 + finished = _wait_for_sse_event( + sse_events, "download_finished", MODEL_DOWNLOAD_ID, MODEL_DOWNLOAD_TIMEOUT + ) + stop.set() + assert finished, "Never received download_finished SSE event" + + # the model lists the sidecars it has on disk, and no sidecar is listed as a model + res = server.make_request("GET", "/models") + assert res.status_code == 200 + entries = {item["id"]: item for item in res.body.get("data", [])} + assert entries[MODEL_DOWNLOAD_ID]["sidecars"] == [] + for kind in ("mmproj", "mtp", "dflash", "eagle3", "dspark"): + assert not any(i.endswith("-" + kind) for i in entries), f"a {kind} sidecar is listed as a model" + + server.make_request("DELETE", f"/models?model={MODEL_DOWNLOAD_ID}") + + +def test_router_download_conflicts(): + """A download is refused over one on its way, and over a model that is running.""" + global server + server.models_max = 2 + server.llama_cache = _own_cache("cache_download_conflicts") + server.start() + + server.make_request("DELETE", f"/models?model={MODEL_DOWNLOAD_ID}") + + # a download on its way is not started again; the entry reports the state before + # the child fetches anything, so the probe lands inside the download + res = server.make_request("POST", "/models", data={"model": MODEL_DOWNLOAD_ID}) + assert res.status_code == 200 + _wait_for_model_status(MODEL_DOWNLOAD_ID, {"downloading"}, timeout=30) + + res = server.make_request("POST", "/models", data={"model": MODEL_DOWNLOAD_ID}) + assert res.status_code == 400 + assert "is already downloading" in res.body["error"]["message"] + + # a loaded model holds its files, so it goes only once it is unloaded; the + # reload that follows the download parks the entry as unloaded + _wait_for_model_status(MODEL_DOWNLOAD_ID, {"downloaded", "unloaded"}, timeout=MODEL_DOWNLOAD_TIMEOUT) + _load_model_and_wait(MODEL_DOWNLOAD_ID, timeout=120) + + res = server.make_request("POST", "/models", data={"model": MODEL_DOWNLOAD_ID}) + assert res.status_code == 400 + assert "is running, unload it first" in res.body["error"]["message"] + + +def test_router_delete_unknown_name(): + """A name that is neither a model nor a recorded download is not found, and the + cached tags of its repo keep their files.""" + global server + server.llama_cache = _own_cache("cache_delete_unknown_name") + server.start() + + res = server.make_request("POST", "/models", data={"model": MODEL_DOWNLOAD_ID, "sidecars": []}) + assert res.status_code == 200 + _wait_for_model_status(MODEL_DOWNLOAD_ID, {"downloaded", "unloaded"}, timeout=MODEL_DOWNLOAD_TIMEOUT) + + repo = MODEL_DOWNLOAD_ID.split(":")[0] + res = server.make_request("DELETE", f"/models?model={repo}") + assert res.status_code != 200 + assert "is not found" in res.body["error"]["message"] + assert MODEL_DOWNLOAD_ID in _get_model_ids(is_reload=True) + + server.make_request("DELETE", f"/models?model={MODEL_DOWNLOAD_ID}") + + +def test_router_incomplete_sidecars(): + """A cached model lists the sidecars its unfinished download has not written yet, until + they are dropped.""" + global server + cache_dir = _own_cache("cache_incomplete_sidecars") + server.llama_cache = cache_dir + server.start() + + res = server.make_request("POST", "/models", data={"model": MODEL_DOWNLOAD_ID, "sidecars": []}) + assert res.status_code == 200 + _wait_for_model_status(MODEL_DOWNLOAD_ID, {"downloaded", "unloaded"}, timeout=MODEL_DOWNLOAD_TIMEOUT) + server.stop() + + # a paused mtp of the cached model, as the record of an earlier run holds it + partial_path = os.path.abspath(os.path.join(cache_dir, "test-mtp.gguf.downloadInProgress")) + with open(partial_path, "w") as f: + f.write("partial") + with open(os.path.join(cache_dir, "unfinished-downloads.json"), "w") as f: + json.dump({"downloads": [{ + "model": MODEL_DOWNLOAD_ID, + "sidecars": ["mtp"], + "partials": [{"kind": "mtp", "path": partial_path}], + }]}, f) + + server.start() + + def entry() -> dict: + res = server.make_request("GET", "/models") + assert res.status_code == 200 + return {item["id"]: item for item in res.body.get("data", [])}[MODEL_DOWNLOAD_ID] + + assert entry()["sidecars"] == [] + assert entry()["incomplete_sidecars"] == ["mtp"] + + res = server.make_request("DELETE", f"/models?model={MODEL_DOWNLOAD_ID}&sidecar=mtp") + assert res.status_code == 200 + assert res.body.get("success") is True + assert not os.path.exists(partial_path) + assert "incomplete_sidecars" not in entry() + + server.make_request("DELETE", f"/models?model={MODEL_DOWNLOAD_ID}") + + +def test_router_preset_model_sidecars(): + """A preset model reports the sidecars its files have on disk, like a cached one.""" + global server + + preset_path = os.path.join(TMP_DIR, "test_preset_sidecars.ini") + + with open(preset_path, "w") as f: + f.write("[preset-sidecars]\nhf-repo = ggml-org/tinygemma3-GGUF:Q8_0\n") + + server.models_preset = preset_path + server.start() + + try: + # the mmproj of that repo lands next to the model, as -hf would pull it + res = server.make_request("POST", "/models", data={"model": MODEL_A, "sidecars": ["mmproj"]}) + assert res.status_code == 200 + _wait_for_model_status(MODEL_A, {"downloaded", "unloaded"}, timeout=MODEL_DOWNLOAD_TIMEOUT) + + res = server.make_request("GET", "/models") + assert res.status_code == 200 + entries = {item["id"]: item for item in res.body.get("data", [])} + + assert entries["preset-sidecars"]["source"] == "preset" + assert entries["preset-sidecars"]["sidecars"] == ["mmproj"] + finally: + os.remove(preset_path) + + +def test_router_local_preset_listing(): + """A preset on a local file resolves from no HF repo: it lists without sidecars.""" + global server + + model_path = os.path.join(TMP_DIR, "test_local_preset.gguf") + preset_path = os.path.join(TMP_DIR, "test_local_preset.ini") + + with open(model_path, "w") as f: + f.write("GGUF") + with open(preset_path, "w") as f: + f.write(f"[local]\nmodel = {model_path}\n") + + server.models_preset = preset_path + server.start() + + try: + res = server.make_request("GET", "/models") + assert res.status_code == 200 + entries = {item["id"]: item for item in res.body.get("data", [])} + + assert entries["local"]["source"] == "preset" + assert "sidecars" not in entries["local"] + finally: + os.remove(preset_path) + os.remove(model_path) + + +def test_router_download_unresolved_tag(): + """A tag that resolves no file of the repo is refused before anything starts.""" + global server + server.start() + + model = MODEL_DOWNLOAD_ID.split(":")[0] + ":Q9_Z" + res = server.make_request("POST", "/models", data={"model": model}) + assert res.status_code == 400 + assert "resolves no file to download" in res.body["error"]["message"] + assert model not in _get_model_ids(is_reload=False) + + +def test_router_download_drops_stale_partials(): + """Posting a recorded download again deletes the partials its new plan no longer writes.""" + global server + + cache_dir = _own_cache("cache_download_drops_stale_partials") + stale_path = os.path.abspath(os.path.join(cache_dir, "test-stale.gguf.downloadInProgress")) + record_path = os.path.join(cache_dir, "unfinished-downloads.json") + + with open(stale_path, "w") as f: + f.write("stale") + with open(record_path, "w") as f: + json.dump({"downloads": [{ + "model": MODEL_DOWNLOAD_ID, + "sidecars": ["mtp"], + "partials": [{"kind": "mtp", "path": stale_path}], + }]}, f) + + server.llama_cache = cache_dir + server.start() + + res = server.make_request("POST", "/models", data={"model": MODEL_DOWNLOAD_ID, "sidecars": []}) + assert res.status_code == 200 + assert not os.path.exists(stale_path), "the stale partial survived the new record" + + _wait_for_model_status(MODEL_DOWNLOAD_ID, {"downloaded", "unloaded"}, timeout=MODEL_DOWNLOAD_TIMEOUT) + server.make_request("DELETE", f"/models?model={MODEL_DOWNLOAD_ID}") diff --git a/tools/server/tests/utils.py b/tools/server/tests/utils.py index 1c84f5121..288452808 100644 --- a/tools/server/tests/utils.py +++ b/tools/server/tests/utils.py @@ -127,6 +127,7 @@ class ServerProcess: mcp_servers_config: str | None = None mcp_servers_json: str | None = None cors_origins: str | None = None + llama_cache: str | None = None # session variables process: subprocess.Popen | None = None @@ -145,7 +146,9 @@ class ServerProcess: **os.environ, "LLAMA_SERVER_DEBUG_FAKE_TIMING": "1", } - if "LLAMA_CACHE" not in os.environ: + if self.llama_cache is not None: + env["LLAMA_CACHE"] = self.llama_cache + elif "LLAMA_CACHE" not in os.environ: env["LLAMA_CACHE"] = "tmp" if self.external_server: print(f"[external_server]: Assuming external server running on {self.server_host}:{self.server_port}") diff --git a/tools/ui/src/lib/components/app/content/MarkdownContent/MarkdownContent.svelte b/tools/ui/src/lib/components/app/content/MarkdownContent/MarkdownContent.svelte index c217a769a..895e9a80a 100644 --- a/tools/ui/src/lib/components/app/content/MarkdownContent/MarkdownContent.svelte +++ b/tools/ui/src/lib/components/app/content/MarkdownContent/MarkdownContent.svelte @@ -7,6 +7,7 @@ getMdastNodeHash, isAppendMode } from './markdown-utils'; + import { SAFE_HTML_CONFIG } from './safe-html-config'; import { ActionIconCopyToClipboard, CodeBlockActions, @@ -44,6 +45,7 @@ import { detectIncompleteCodeBlock, highlightCode, type IncompleteCodeBlock } from '$lib/utils'; import { sanitizeSvg } from '$lib/utils/sanitize-svg'; import { mountSvgShadow } from '$lib/utils/svg-shadow'; + import DOMPurify from 'dompurify'; import type { Root as HastRoot, RootContent as HastRootContent } from 'hast'; import githubLightCss from 'highlight.js/styles/github.css?inline'; import githubDarkCss from 'highlight.js/styles/github-dark.css?inline'; @@ -57,6 +59,8 @@ content: string; class?: string; disableMath?: boolean; + /** Render raw HTML found in the markdown (sanitized) instead of escaping it. */ + allowHtml?: boolean; } interface MarkdownBlock { @@ -65,7 +69,13 @@ contentHash?: string; } - let { attachments, class: className = '', content, disableMath = false }: Props = $props(); + let { + allowHtml = false, + attachments, + class: className = '', + content, + disableMath = false + }: Props = $props(); let containerRef = $state(); let renderedBlocks = $state([]); @@ -121,6 +131,8 @@ // Per-instance transform cache, avoids re-transforming stable blocks during streaming // Garbage collected when component is destroyed (on conversation change) + const CACHE_KEY_PLAIN = 'plain'; + const CACHE_KEY_ALLOW_HTML = 'allow-html'; const transformCache = new SvelteMap(); let previousContent = ''; @@ -183,7 +195,10 @@ index: number ): Promise<{ html: string; hash: string }> { const hash = getMdastNodeHash(node, index); - const cached = transformCache.get(hash); + // the rendered HTML also depends on allowHtml (sanitized raw vs escaped), + // so the cache is keyed per mode + const cacheKey = `${allowHtml ? CACHE_KEY_ALLOW_HTML : CACHE_KEY_PLAIN}:${hash}`; + const cached = transformCache.get(cacheKey); if (cached) { return { hash, html: cached }; @@ -192,10 +207,13 @@ const singleNodeRoot = { children: [node], type: 'root' }; const transformedRoot = (await processorInstance.run(singleNodeRoot as MdastRoot)) as HastRoot; const html = processorInstance.stringify(transformedRoot); + const safeHtml = allowHtml + ? (DOMPurify.sanitize(html, SAFE_HTML_CONFIG) as unknown as string) + : html; - transformCache.set(hash, html); + transformCache.set(cacheKey, safeHtml); - return { hash, html }; + return { hash, html: safeHtml }; } /** @@ -300,7 +318,7 @@ if (prefixMarkdown.trim()) { const normalizedPrefix = preprocessLaTeX(prefixMarkdown); - const processorInstance = getMarkdownProcessor({ attachments, disableMath }); + const processorInstance = getMarkdownProcessor({ allowHtml, attachments, disableMath }); const ast = processorInstance.parse(normalizedPrefix) as MdastRoot; const mdastChildren = (ast as { children?: unknown[] }).children ?? []; const nextBlocks: MarkdownBlock[] = []; @@ -350,7 +368,7 @@ incompleteCodeBlock = null; const normalized = preprocessLaTeX(markdown); - const processorInstance = getMarkdownProcessor({ attachments, disableMath }); + const processorInstance = getMarkdownProcessor({ allowHtml, attachments, disableMath }); const ast = processorInstance.parse(normalized) as MdastRoot; const mdastChildren = (ast as { children?: unknown[] }).children ?? []; const stableCount = Math.max(mdastChildren.length - 1, 0); @@ -394,6 +412,10 @@ )) as HastRoot; unstableHtml = processorInstance.stringify(transformedRoot); + + if (allowHtml) { + unstableHtml = DOMPurify.sanitize(unstableHtml, SAFE_HTML_CONFIG) as unknown as string; + } } renderedBlocks = nextBlocks; diff --git a/tools/ui/src/lib/components/app/content/MarkdownContent/markdown-processor.ts b/tools/ui/src/lib/components/app/content/MarkdownContent/markdown-processor.ts index 57ded3e99..26cae5b5c 100644 --- a/tools/ui/src/lib/components/app/content/MarkdownContent/markdown-processor.ts +++ b/tools/ui/src/lib/components/app/content/MarkdownContent/markdown-processor.ts @@ -42,12 +42,15 @@ export interface MarkdownProcessor { export interface MarkdownProcessorOptions { attachments?: DatabaseMessageExtra[]; disableMath?: boolean; + /** Render raw HTML found in the markdown instead of escaping it. */ + allowHtml?: boolean; } const sharedPipelines = new Map(); -const attachmentPipelines = new WeakMap(); +const attachmentPipelines = new WeakMap>(); function buildPipeline({ + allowHtml = false, attachments, disableMath = false }: MarkdownProcessorOptions): MarkdownProcessor { @@ -58,11 +61,14 @@ function buildPipeline({ proc = proc.use(remarkMath); // Parse $inline$ and $$block$$ math } - proc = proc - .use(remarkBreaks) // Convert line breaks to
+ proc = proc.use(remarkBreaks); // Convert line breaks to
+ + if (!allowHtml) { // Treat raw HTML as literal text with preserved indentation - .use(remarkLiteralHtml) - .use(remarkRehype); // Convert Markdown AST to rehype + proc = proc.use(remarkLiteralHtml); + } + + proc = proc.use(remarkRehype, allowHtml ? { allowDangerousHtml: true } : undefined); if (!disableMath) { proc = proc.use(rehypeKatex); // Render math using KaTeX @@ -91,17 +97,26 @@ function buildPipeline({ export function getMarkdownProcessor(options: MarkdownProcessorOptions): MarkdownProcessor { if (options.attachments && options.attachments.length > 0) { - let cached = attachmentPipelines.get(options.attachments); + let byOptions = attachmentPipelines.get(options.attachments); + + if (!byOptions) { + byOptions = new Map(); + attachmentPipelines.set(options.attachments, byOptions); + } + + const key = `${Boolean(options.disableMath)}:${Boolean(options.allowHtml)}`; + + let cached = byOptions.get(key); if (!cached) { cached = buildPipeline(options); - attachmentPipelines.set(options.attachments, cached); + byOptions.set(key, cached); } return cached; } - const key = String(Boolean(options.disableMath)); + const key = `${Boolean(options.disableMath)}:${Boolean(options.allowHtml)}`; let cached = sharedPipelines.get(key); diff --git a/tools/ui/src/lib/components/app/content/MarkdownContent/safe-html-config.ts b/tools/ui/src/lib/components/app/content/MarkdownContent/safe-html-config.ts new file mode 100644 index 000000000..d8dff07dd --- /dev/null +++ b/tools/ui/src/lib/components/app/content/MarkdownContent/safe-html-config.ts @@ -0,0 +1,138 @@ +// Sanitization rules for raw HTML in model cards (MarkdownContent's +// allowHtml mode). Kept as its own module so the adversarial test in +// tests/client can import the exact shipped configuration. +import DOMPurify from 'dompurify'; + +// style is allowed for the KaTeX output, which lays its spans out with inline +// styles; a third party card can use the same attribute to draw a fixed +// overlay over the app, so positioned values are scrubbed +DOMPurify.addHook('afterSanitizeAttributes', (node) => { + if (typeof (node as Element).getAttribute !== 'function') return; + + const style = (node as Element).getAttribute('style'); + + if (!style || !/position\s*:\s*(fixed|absolute)/i.test(style)) return; + + (node as Element).setAttribute( + 'style', + style.replace(/position\s*:\s*(fixed|absolute)\s*;?/gi, '') + ); +}); + +/** + * Explicit DOMPurify allow list: the tags and attributes model cards may use + * plus the KaTeX output (spans with inline styles and MathML). Everything + * else - scripts, event handlers, forms, iframes, style tags - is stripped. + */ +export const SAFE_HTML_CONFIG = { + ALLOWED_ATTR: [ + 'class', + 'style', + 'id', + 'title', + 'role', + 'aria-hidden', + 'aria-label', + 'aria-describedby', + 'aria-expanded', + 'href', + 'target', + 'rel', + 'src', + 'srcset', + 'alt', + 'width', + 'height', + 'align', + 'valign', + 'colspan', + 'rowspan', + 'controls', + 'type', + // MathML + 'xmlns', + 'display', + 'encoding', + 'mathvariant' + ], + ALLOWED_TAGS: [ + // text and structure + 'a', + 'b', + 'i', + 'u', + 's', + 'em', + 'strong', + 'code', + 'pre', + 'kbd', + 'sub', + 'sup', + 'br', + 'hr', + 'p', + 'span', + 'div', + 'blockquote', + 'center', + 'ul', + 'ol', + 'li', + 'dl', + 'dt', + 'dd', + 'h1', + 'h2', + 'h3', + 'h4', + 'h5', + 'h6', + 'table', + 'thead', + 'tbody', + 'tr', + 'th', + 'td', + 'colgroup', + 'col', + 'caption', + // media + 'img', + 'picture', + 'source', + 'figure', + 'figcaption', + 'video', + 'audio', + // MathML, for the KaTeX output + 'math', + 'mrow', + 'mi', + 'mo', + 'mn', + 'ms', + 'mtext', + 'mfrac', + 'mroot', + 'msqrt', + 'msub', + 'msup', + 'msubsup', + 'munder', + 'mover', + 'mmultiscripts', + 'mtable', + 'mtr', + 'mtd', + 'mspace', + 'mglyph', + 'maligngroup', + 'malignmark', + 'mpadded', + 'mphantom', + 'mstyle', + 'semantics', + 'annotation' + ] +}; diff --git a/tools/ui/src/lib/components/app/dialogs/DialogManageModels.svelte b/tools/ui/src/lib/components/app/dialogs/DialogManageModels.svelte index 45b73dc72..fbe711c25 100644 --- a/tools/ui/src/lib/components/app/dialogs/DialogManageModels.svelte +++ b/tools/ui/src/lib/components/app/dialogs/DialogManageModels.svelte @@ -1,7 +1,18 @@ {#each badges as badge (badge.kind + (badge.quant ?? ''))} - - + import { Check } from '@lucide/svelte'; + import { MODEL_BADGE_CLASS, MODEL_VARIANT_BADGE_CLASS } from '$lib/constants'; + import { ModelRowDownloadState } from '$lib/enums'; + + interface Props { + /** + * Left inset of the tree connector, tuned per surface so the elbow hangs + * under the main line's avatar and "Draft model" aligns with its id. + */ + connectorClass?: string; + /** Draft sidecar kind, e.g. dflash. */ + kind: string; + /** Quant of the draft, when its tag carries one. */ + quant?: string | null; + /** Live state of the draft download. */ + state: ModelRowDownloadState; + /** Rounded download percent, when the feed reports progress. */ + percent?: number | null; + } + + let { connectorClass = 'ml-6', kind, percent = null, quant = null, state }: Props = $props(); + + + +
+ + + Draft model + + {kind} + + {#if quant} + {quant} + {/if} + + {#if state === ModelRowDownloadState.DOWNLOADED} + + {:else} + + {#if percent !== null} + {percent}% + {:else if state === ModelRowDownloadState.PAUSED} + Paused + {/if} + + {/if} +
diff --git a/tools/ui/src/lib/components/app/models/ModelId.svelte b/tools/ui/src/lib/components/app/models/ModelId.svelte index 26d72f662..9a3464ab8 100644 --- a/tools/ui/src/lib/components/app/models/ModelId.svelte +++ b/tools/ui/src/lib/components/app/models/ModelId.svelte @@ -5,6 +5,7 @@ import { TruncatedText } from '$lib/components/app'; import * as Tooltip from '$lib/components/ui/tooltip'; import { MODEL_BADGE_CLASS, MODEL_VARIANT_BADGE_CLASS } from '$lib/constants'; + import type { ModelDraftSidecar } from '$lib/enums'; import { HuggingFaceService } from '$lib/services'; import { ModelsService } from '$lib/services/models.service'; import { settingsStore } from '$lib/stores'; @@ -39,6 +40,8 @@ sizeRange?: { min: number; max: number } | null; /** Draft sidecars available for the model, badged with their own quant. */ draftSidecars?: ModelSidecarBadge[]; + /** Draft sidecar kinds the repo offers, one badge per kind and no quant. */ + draftKinds?: ModelDraftSidecar[]; /** Give the id a row of its own, with its badges and icons together under it. */ stackId?: boolean; /** Allow badges to wrap onto new lines instead of truncating. */ @@ -50,6 +53,7 @@ aliases, class: className = '', contextLength, + draftKinds = [], draftSidecars = [], hideCapabilities = false, hideModalities = false, @@ -93,12 +97,17 @@ let uniqueAliases = $derived([...new Set(aliases ?? [])]); let uniqueTags = $derived([...new Set([...(parsed.tags ?? []), ...(tags ?? [])])]); let uniqueDraftSidecars = $derived(draftSidecars.filter((badge) => !isAuxSidecar(badge.kind))); + // the model id can already name one of the kinds + let uniqueDraftKinds = $derived([ + ...new Set(draftKinds.filter((kind) => kind !== parsed.sidecar)) + ]); let primaryAlias = $derived(uniqueAliases.length === 1 ? uniqueAliases[0] : null); let displayName = $derived(primaryAlias ?? parsed.modelName ?? modelId); let hasBadges = $derived( parsed.sidecar || + uniqueDraftKinds.length > 0 || uniqueDraftSidecars.length > 0 || (parsed.params && !hideParameters) || (parsed.quantization && !resolvedHideQuantization) || @@ -144,6 +153,12 @@
{/if} + {#each uniqueDraftKinds as kind (kind)} + + {kind} + + {/each} + {#if primaryAlias} diff --git a/tools/ui/src/lib/components/app/models/ModelsManager/ModelsManager.svelte b/tools/ui/src/lib/components/app/models/ModelsManager/ModelsManager.svelte index 36fd3e22a..9853bafe5 100644 --- a/tools/ui/src/lib/components/app/models/ModelsManager/ModelsManager.svelte +++ b/tools/ui/src/lib/components/app/models/ModelsManager/ModelsManager.svelte @@ -32,9 +32,22 @@ let capabilityFilter = $state([]); let selectedId = $state(null); + // tracked downloads get their own section, listed like any other model + let downloads = $derived( + downloadGroups(modelsStore.status.getDownloadEntries(), modelsStore.models, (repo) => + modelsStore.status.getDraftSidecars(repo) + ) + ); + let allModels = $derived(modelsStore.models); - let selected = $derived(allModels.find((option) => option.id === selectedId) ?? null); + // the download groups stand in for rows the listing does not carry yet, so a + // click on them still finds its option and opens the pane + let selected = $derived( + allModels.find((option) => option.id === selectedId) ?? + downloads.find((group) => group.base.id === selectedId)?.base ?? + null + ); // The pane is laid out before it is ever opened, so the first open only slides a // finished panel in. It renders the selection, else the model it last showed. let lastPicked = $state(null); @@ -42,7 +55,11 @@ let shownId = $state(null); let isSwapping = $state(false); let fade = $state<'open' | 'swap'>('open'); - let shownOption = $derived(allModels.find((option) => option.id === shownId) ?? null); + let shownOption = $derived( + allModels.find((option) => option.id === shownId) ?? + downloads.find((group) => group.base.id === shownId)?.base ?? + null + ); $effect(() => { const id = selectedId; @@ -144,10 +161,6 @@ rank.size === 0 ? list : [...list].sort((a, b) => rankOf(a) - rankOf(b)); // one entry per repo, so a model with several quants takes a single table row let entries = $derived(byRecency(groupModelQuants(matching))); - // tracked downloads get their own section, listed like any other model - let downloads = $derived( - downloadGroups(modelsStore.status.getDownloadEntries(), modelsStore.models) - ); let groups = $derived.by(() => { // a loaded quant is a model of its own: its repo keeps the quants left behind const isLoaded = (option: ModelOption) => modelsStore.isModelLoaded(option.model); @@ -216,13 +229,16 @@ return ordered; }); - // a caller can ask for one model to be revealed, the download rows do + // a caller can ask for one model to be revealed, the download rows do; a + // tracked download is not in the listing yet, so its own rows are searched too $effect(() => { const focus = uiStore.manageModelFocus; if (!focus) return; - const option = allModels.find((model) => model.id === focus || model.model === focus); + const matches = (candidate: ModelOption) => candidate.id === focus || candidate.model === focus; + const option = + allModels.find(matches) ?? downloads.find((group) => matches(group.base))?.base ?? null; if (option) selectedId = option.id; diff --git a/tools/ui/src/lib/components/app/models/ModelsManager/ModelsManagerDownloadControl.svelte b/tools/ui/src/lib/components/app/models/ModelsManager/ModelsManagerDownloadControl.svelte index 0e81500f2..5e77caed0 100644 --- a/tools/ui/src/lib/components/app/models/ModelsManager/ModelsManagerDownloadControl.svelte +++ b/tools/ui/src/lib/components/app/models/ModelsManager/ModelsManagerDownloadControl.svelte @@ -38,10 +38,7 @@ class="h-5 w-5 hover:text-foreground" icon={isPaused ? MODEL_DOWNLOAD_ICONS.resume : MODEL_DOWNLOAD_ICONS.pause} iconSize="h-4 w-4" - onclick={() => - void (isPaused - ? modelsStore.status.downloadModel(option.model) - : modelsStore.status.pauseDownload(option.model))} + onclick={() => void modelsStore.status.toggleDownload(option.model)} stopPropagationOnClick tooltip={isPaused ? 'Resume downloading' : 'Pause downloading'} tooltipAsTitle diff --git a/tools/ui/src/lib/components/app/models/ModelsManager/ModelsManagerModelConfiguration/ModelsManagerModelConfiguration.svelte b/tools/ui/src/lib/components/app/models/ModelsManager/ModelsManagerModelConfiguration/ModelsManagerModelConfiguration.svelte index 8a109a8db..40a7565ed 100644 --- a/tools/ui/src/lib/components/app/models/ModelsManager/ModelsManagerModelConfiguration/ModelsManagerModelConfiguration.svelte +++ b/tools/ui/src/lib/components/app/models/ModelsManager/ModelsManagerModelConfiguration/ModelsManagerModelConfiguration.svelte @@ -1,4 +1,6 @@
- {#if isMobile} - - - {/if} + Pause downloading + {/if} + -
- {#if !isMobile} - +
+ + {#if onSeeInDiscover && discoverEnabled} + {/if} + {:else} + {#if isMobile} + + {/if} -
+ +
+ {#if !isMobile} + + {/if} + + {#if canToggleLoad} + + {/if} + + {#if onSeeInDiscover && discoverEnabled} + + {/if} +
+ {/if} diff --git a/tools/ui/src/lib/components/app/models/ModelsManager/ModelsManagerModelConfiguration/ModelsManagerModelConfigurationHeader.svelte b/tools/ui/src/lib/components/app/models/ModelsManager/ModelsManagerModelConfiguration/ModelsManagerModelConfigurationHeader.svelte index 40f5d0d54..d2cc2d5b8 100644 --- a/tools/ui/src/lib/components/app/models/ModelsManager/ModelsManagerModelConfiguration/ModelsManagerModelConfigurationHeader.svelte +++ b/tools/ui/src/lib/components/app/models/ModelsManager/ModelsManagerModelConfiguration/ModelsManagerModelConfigurationHeader.svelte @@ -8,8 +8,14 @@ import type { ModelOption } from '$lib/types/models'; interface Props { + isDownload?: boolean; + isDownloadPaused?: boolean; isLoaded: boolean; onClose: () => void; + /** Opens the manager dialog on the Discover view, focused on this model's repo. */ + onSeeInDiscover?: () => void; + onRequestCancelDownload?: () => void; + onToggleDownload?: () => void; onToggleLoad: () => void; onUseInChat: () => void; onUseInNewChat: () => void; @@ -18,13 +24,27 @@ status: ServerModelStatus | null; } - let { isLoaded, onClose, onToggleLoad, onUseInChat, onUseInNewChat, option, status }: Props = - $props(); + let { + isDownload = false, + isDownloadPaused = false, + isLoaded, + onClose, + onRequestCancelDownload, + onSeeInDiscover, + onToggleDownload, + onToggleLoad, + onUseInChat, + onUseInNewChat, + option, + status + }: Props = $props(); let isMobile = $derived(deviceStore.isMobile); let isLoading = $derived(status === ServerModelStatus.LOADING); let statusLabel = $derived.by(() => { + if (isDownload) return isDownloadPaused ? 'Paused' : 'Downloading'; + if (status === ServerModelStatus.LOADING) return 'Loading'; if (status === ServerModelStatus.FAILED) return 'Failed to load'; @@ -37,6 +57,8 @@ }); let statusDot = $derived.by(() => { + if (isDownload) return isDownloadPaused ? 'bg-yellow-500' : 'bg-blue-500 animate-pulse'; + if (status === ServerModelStatus.FAILED) return 'bg-red-500'; if (status === ServerModelStatus.LOADING) return 'bg-muted-foreground/50 animate-pulse'; @@ -100,8 +122,13 @@ {#if !isMobile} boolean; @@ -20,11 +21,21 @@ onDelete: (option: ModelOption) => void; onSelect: (option: ModelOption) => void; selected: boolean; + /** Draft sidecars already on disk, when the row stands for a download. */ + sidecarBadges?: ModelSidecarBadge[]; /** Left padding in px, from the nesting depth. */ indent?: number; } - let { indent = 0, isFavorite, onDelete, onSelect, option, selected }: Props = $props(); + let { + indent = 0, + isFavorite, + onDelete, + onSelect, + option, + selected, + sidecarBadges + }: Props = $props(); let favorite = $derived(isFavorite(option)); let isHidden = $derived(modelsStore.isHidden(option.id)); @@ -38,6 +49,10 @@ ? ModelRowDownloadState.DOWNLOADING : null ); + + function activate(): void { + onSelect(option); + }
- - + favorited + {/if} + + + + {#if sidecarBadges?.length} + +
+ {#each sidecarBadges as badge (badge.repo + ':' + badge.kind + ':' + (badge.quant ?? ''))} + + {/each} +
+ {/if} +
(); @@ -129,6 +130,10 @@ // cancel is confirmed once for the whole list, so one dialog serves every row function requestDelete(option: ModelOption): void { pendingDelete = option.model; + // an in-flight row cancels (and drops the partials), a paused one deletes + pendingAction = modelsStore.status.isDownloadInProgress(option.model) + ? ModelDownloadConfirmAction.CANCEL + : ModelDownloadConfirmAction.DELETE; deleteOpen = true; } @@ -248,6 +253,7 @@ {onSelect} option={entry.base} selected={selectedId === entry.base.id} + sidecarBadges={entry.sidecarBadges} /> {/if} {/snippet} @@ -328,7 +334,7 @@ {/snippet} (deleteOpen = false)} open={deleteOpen} repoWithTag={pendingDelete} diff --git a/tools/ui/src/lib/components/app/models/ModelsManager/row-actions.ts b/tools/ui/src/lib/components/app/models/ModelsManager/row-actions.ts index 9931dd3f4..3407634ec 100644 --- a/tools/ui/src/lib/components/app/models/ModelsManager/row-actions.ts +++ b/tools/ui/src/lib/components/app/models/ModelsManager/row-actions.ts @@ -1,10 +1,10 @@ -import { Eye, EyeOff, Heart, HeartOff, Trash2 } from '@lucide/svelte'; +import { Compass, Eye, EyeOff, Heart, HeartOff, Trash2 } from '@lucide/svelte'; import { MODEL_DOWNLOAD_ICONS } from '$lib/constants'; import { ModelRowDownloadState } from '$lib/enums'; -import { modelsStore } from '$lib/stores'; +import { modelsStore, uiStore } from '$lib/stores'; import type { ModelOption } from '$lib/types/models'; +import { repoOf } from '$lib/utils'; -/** Row actions follow the app's dropdown pattern: icon, label, separators, variants. */ export function modelRowActions( option: ModelOption, favorite: boolean, @@ -13,9 +13,17 @@ export function modelRowActions( /** Download state, when the row stands for a tracked download. */ download?: ModelRowDownloadState | null ) { + const viewInDiscover = { + icon: Compass, + label: 'See in Discover', + // the details pane is keyed by repo, not by `:` + onclick: () => uiStore.openModelsDiscover(repoOf(option.model)), + separator: true + }; + // a tracked download is paused, resumed or dropped: it is not loaded, hidden or - // drafted. Deleting it stops the download and removes what is on disk, which is - // what the same wording offers on a cached model + // drafted. An in-flight row cancels, which stops the download and removes the + // partial files; a paused one deletes, which removes the partials from disk if (download) { const isPaused = download === ModelRowDownloadState.PAUSED; @@ -23,18 +31,16 @@ export function modelRowActions( { icon: isPaused ? MODEL_DOWNLOAD_ICONS.resume : MODEL_DOWNLOAD_ICONS.pause, label: isPaused ? 'Resume downloading' : 'Pause downloading', - onclick: () => - void (isPaused - ? modelsStore.status.downloadModel(option.model) - : modelsStore.status.pauseDownload(option.model)) + onclick: () => void modelsStore.status.toggleDownload(option.model) }, { icon: Trash2, - label: 'Delete from disk', + label: isPaused ? 'Delete from disk' : 'Cancel download', onclick: () => onDelete(option), separator: true, variant: 'destructive' as const - } + }, + viewInDiscover ]; } @@ -56,6 +62,7 @@ export function modelRowActions( label: isHidden ? 'Unhide model' : 'Hide model', onclick: () => modelsStore.toggleHidden(option.id), separator: true - } + }, + viewInDiscover ]; } diff --git a/tools/ui/src/lib/components/app/models/ModelsManager/utils.ts b/tools/ui/src/lib/components/app/models/ModelsManager/utils.ts index 208343664..435700a10 100644 --- a/tools/ui/src/lib/components/app/models/ModelsManager/utils.ts +++ b/tools/ui/src/lib/components/app/models/ModelsManager/utils.ts @@ -2,7 +2,12 @@ import { LOCAL_BACKEND_ID, type ModalityKey } from '$lib/constants'; import { ModelCapability, ModelGroupKind, ModelsTableGroupKind } from '$lib/enums'; import { HuggingFaceService, ModelsService } from '$lib/services'; import { modelsStore } from '$lib/stores'; -import type { ModelDownloadEntry, ModelDownloadProgress, ModelOption } from '$lib/types/models'; +import type { + ModelDownloadEntry, + ModelDownloadProgress, + ModelOption, + ModelSidecarBadge +} from '$lib/types/models'; import { detectThinkingSupport, detectToolUseSupport, repoOf } from '$lib/utils'; /** One repo of the table, with the rows it ships as. */ @@ -11,6 +16,8 @@ export interface ModelQuantGroup { key: string; kind: ModelGroupKind; quants: ModelOption[]; + /** Draft sidecars already on disk, shown as checked sub-lines. */ + sidecarBadges?: ModelSidecarBadge[]; } /** True when any of the manager's filters narrows the table. */ @@ -31,28 +38,36 @@ export function downloadProgressFor(repoWithTag: string): ModelDownloadProgress } /** - * One table row per tracked download, so the table lists them the way it lists - * any other model. A paused download is already in the router's listing, so its - * own option is reused; a fresh one stands for its tag alone. + * One table row per tracked download, listed like any other model. A paused download + * is already in the router's listing, so its own option is reused; a fresh one is + * synthesized for its tag alone. */ export function downloadGroups( entries: ModelDownloadEntry[], - models: ModelOption[] + models: ModelOption[], + sidecarsFor: (repo: string) => ModelSidecarBadge[] ): ModelQuantGroup[] { return entries.map((entry) => { - const option = models.find((candidate) => candidate.model === entry.repoWithTag) ?? { + const listed = models.find((candidate) => candidate.model === entry.repoWithTag); + // the synthesized option carries backendId, which ModelOption drops on + // assignment, so the callers spread it instead of typing this as one + const option = listed ?? { backendId: LOCAL_BACKEND_ID, capabilities: [], id: entry.repoWithTag, model: entry.repoWithTag, name: entry.repoWithTag }; + // a draft still coming down badges the id, an on-disk one keeps its sub-line + const sidecars = sidecarsFor(repoOf(entry.repoWithTag)); + const pending = sidecars.filter((badge) => badge.pending); return { - base: option, + base: { ...option, draftSidecars: pending }, key: `download::${entry.repoWithTag}`, kind: ModelGroupKind.QUANTS, - quants: [option] + quants: [{ ...option, draftSidecars: pending }], + sidecarBadges: sidecars.filter((badge) => !badge.pending) }; }); } diff --git a/tools/ui/src/lib/components/app/models/ModelsSelector/ModelsSelectorDownloadItem.svelte b/tools/ui/src/lib/components/app/models/ModelsSelector/ModelsSelectorDownloadItem.svelte index 6ef1433b9..7cc286f89 100644 --- a/tools/ui/src/lib/components/app/models/ModelsSelector/ModelsSelectorDownloadItem.svelte +++ b/tools/ui/src/lib/components/app/models/ModelsSelector/ModelsSelectorDownloadItem.svelte @@ -1,22 +1,38 @@ - -
- - + when paused; opacity only for the spinner, the spin owns the transform --> + + - - onRequestCancel?.(entry.repoWithTag)} - /> + + + // a phone has no manager: its information dialog takes the icon + deviceStore.isMobile + ? uiStore.openModelInformation(option) + : uiStore.openModelsManager(entry.repoWithTag)} + tooltip="Manage model" + tooltipAsTitle + /> + + onRequestCancel?.(entry.repoWithTag)} + /> +
+ + + {#if onDiskDrafts.length} +
+ {#each onDiskDrafts as badge (badge.repo + ':' + badge.kind + ':' + (badge.quant ?? ''))} + + {/each} +
+ {/if} {#if entry.progress && entry.progress.totalBytes > 0} Downloading

{#each downloadEntries as entry (entry.repoWithTag)} - + {/each} {/if} @@ -230,6 +236,7 @@ (cancelOpen = false)} + onConfirm={() => void modelsStore.status.cancelDownload(pendingCancel)} open={cancelOpen} repoWithTag={pendingCancel} /> diff --git a/tools/ui/src/lib/components/app/models/discover/ModelsDiscover.svelte b/tools/ui/src/lib/components/app/models/discover/ModelsDiscover.svelte new file mode 100644 index 000000000..292938d59 --- /dev/null +++ b/tools/ui/src/lib/components/app/models/discover/ModelsDiscover.svelte @@ -0,0 +1,210 @@ + + +{#snippet catalog()} + (selectedId = id)} + showBaseModelAvatar + /> +{/snippet} + + + +
+ {#if selectedId} + + {/if} +
+ + (cancelOpen = false)} + onConfirm={() => void modelsStore.status.cancelDownload(pendingCancel)} + open={cancelOpen} + repoWithTag={pendingCancel} +/> diff --git a/tools/ui/src/lib/components/app/models/discover/ModelsDiscoverDetails/ModelsDiscoverChatTemplateDialog.svelte b/tools/ui/src/lib/components/app/models/discover/ModelsDiscoverDetails/ModelsDiscoverChatTemplateDialog.svelte new file mode 100644 index 000000000..11240a4a9 --- /dev/null +++ b/tools/ui/src/lib/components/app/models/discover/ModelsDiscoverDetails/ModelsDiscoverChatTemplateDialog.svelte @@ -0,0 +1,75 @@ + + + + + + + Chat template + + + + + + + + + + + diff --git a/tools/ui/src/lib/components/app/models/discover/ModelsDiscoverDetails/ModelsDiscoverDetails.svelte b/tools/ui/src/lib/components/app/models/discover/ModelsDiscoverDetails/ModelsDiscoverDetails.svelte new file mode 100644 index 000000000..8f7e50da6 --- /dev/null +++ b/tools/ui/src/lib/components/app/models/discover/ModelsDiscoverDetails/ModelsDiscoverDetails.svelte @@ -0,0 +1,96 @@ + + +{#if loading} + +{:else if error} +
+

{error}

+
+{:else if details} +
+ + + + + +
+{/if} diff --git a/tools/ui/src/lib/components/app/models/discover/ModelsDiscoverDetails/ModelsDiscoverDetailsContainer.svelte b/tools/ui/src/lib/components/app/models/discover/ModelsDiscoverDetails/ModelsDiscoverDetailsContainer.svelte new file mode 100644 index 000000000..7d81ea8ab --- /dev/null +++ b/tools/ui/src/lib/components/app/models/discover/ModelsDiscoverDetails/ModelsDiscoverDetailsContainer.svelte @@ -0,0 +1,66 @@ + + + diff --git a/tools/ui/src/lib/components/app/models/discover/ModelsDiscoverDetails/ModelsDiscoverDetailsDownloadOptions/ModelsDiscoverDetailsDownloadOptions.svelte b/tools/ui/src/lib/components/app/models/discover/ModelsDiscoverDetails/ModelsDiscoverDetailsDownloadOptions/ModelsDiscoverDetailsDownloadOptions.svelte new file mode 100644 index 000000000..a1bd05a37 --- /dev/null +++ b/tools/ui/src/lib/components/app/models/discover/ModelsDiscoverDetails/ModelsDiscoverDetailsDownloadOptions/ModelsDiscoverDetailsDownloadOptions.svelte @@ -0,0 +1,258 @@ + + +{#if bitDepthRows.length} +
+ +
+ {#each rows as row (row.bitDepth)} + (draftPick = kind)} + onRequestCancel={requestCancel} + onRequestDelete={requestDelete} + /> + {/each} +
+ + + {#if draftKinds.length} +
+ + + value && (draftPick = value as ModelDraftSidecar)} + type="single" + value={draftKind} + > + + {draftKindLabel} + + + + {#each draftKinds as kind (kind)} + + {kind.toUpperCase()} + + {/each} + + + + +
+ {/if} +
+{/if} + + (confirmOpen = false)} + open={confirmOpen} + repoWithTag={pending.repoWithTag} +/> diff --git a/tools/ui/src/lib/components/app/models/discover/ModelsDiscoverDetails/ModelsDiscoverDetailsDownloadOptions/ModelsDiscoverDetailsDownloadOptionsDownloadCommand.svelte b/tools/ui/src/lib/components/app/models/discover/ModelsDiscoverDetails/ModelsDiscoverDetailsDownloadOptions/ModelsDiscoverDetailsDownloadOptionsDownloadCommand.svelte new file mode 100644 index 000000000..eb7f4e55f --- /dev/null +++ b/tools/ui/src/lib/components/app/models/discover/ModelsDiscoverDetails/ModelsDiscoverDetailsDownloadOptions/ModelsDiscoverDetailsDownloadOptionsDownloadCommand.svelte @@ -0,0 +1,131 @@ + + +
+ +
+ {SERVE_COMMAND.BIN} + + {SERVE_COMMAND.SUBCOMMAND} + + {SERVE_COMMAND.MODEL_FLAG} + + + {modelId}{quant ? MODEL_ID.QUANTIZATION_SEPARATOR : ''} + + + + {#if quant} + + {quant} + + {/if} + + + {#if draftKind} + {SERVE_COMMAND.SPEC_TYPE_FLAG} + + + {#if draftKinds.length > 1} + value && onDraftKindChange?.(value as ModelDraftSidecar)} + type="single" + value={draftKind} + > + + {SPEC_TYPE[draftKind]} + + + + {#each draftKinds as kind (kind)} + + {SPEC_TYPE[kind]} + + {/each} + + + {:else} + + {SPEC_TYPE[draftKind]} + + {/if} + {/if} +
+ + +
diff --git a/tools/ui/src/lib/components/app/models/discover/ModelsDiscoverDetails/ModelsDiscoverDetailsDownloadOptions/ModelsDiscoverDetailsDownloadOptionsDraftDownloadButton.svelte b/tools/ui/src/lib/components/app/models/discover/ModelsDiscoverDetails/ModelsDiscoverDetailsDownloadOptions/ModelsDiscoverDetailsDownloadOptionsDraftDownloadButton.svelte new file mode 100644 index 000000000..d7cbd8bec --- /dev/null +++ b/tools/ui/src/lib/components/app/models/discover/ModelsDiscoverDetails/ModelsDiscoverDetailsDownloadOptions/ModelsDiscoverDetailsDownloadOptionsDraftDownloadButton.svelte @@ -0,0 +1,61 @@ + + + + + + {#snippet child({ props })} + + {/snippet} + + + +

{tooltipText}

+
+
diff --git a/tools/ui/src/lib/components/app/models/discover/ModelsDiscoverDetails/ModelsDiscoverDetailsDownloadOptions/ModelsDiscoverDetailsDownloadOptionsQuantDownloadButton.svelte b/tools/ui/src/lib/components/app/models/discover/ModelsDiscoverDetails/ModelsDiscoverDetailsDownloadOptions/ModelsDiscoverDetailsDownloadOptionsQuantDownloadButton.svelte new file mode 100644 index 000000000..5ccbd31d0 --- /dev/null +++ b/tools/ui/src/lib/components/app/models/discover/ModelsDiscoverDetails/ModelsDiscoverDetailsDownloadOptions/ModelsDiscoverDetailsDownloadOptionsQuantDownloadButton.svelte @@ -0,0 +1,272 @@ + + +{#snippet chipBody(dividerClass: string)} + + {label} + + {#if showsCompanion && companion} + + + + + + {companion.kind}{companion.quant ? `:${companion.quant}` : ''} + + {#if companion.onDisk} + + {/if} + + {/if} + + +{/snippet} + + +{#snippet tooltipTrigger(tip: string, button: Snippet<[Record]>)} + + + {#snippet child({ props })} + {@render button(props)} + {/snippet} + + + +

{tip}

+
+
+{/snippet} + +{#snippet deleteChip(props: Record)} + + +{/snippet} + +{#snippet pauseResumeChip(props: Record)} + +{/snippet} + +{#snippet cancelChip(props: Record)} + + +{/snippet} + +{#snippet downloadChip(props: Record)} + + +{/snippet} + +{#if entry.isDownloaded} + {@render tooltipTrigger(tooltipText, deleteChip)} +{:else if entry.isDownloading || entry.isPaused} + +
+ {@render tooltipTrigger(tooltipText, pauseResumeChip)} + + {@render tooltipTrigger('Cancel downloading', cancelChip)} + + {#if percent !== null} + + {/if} +
+{:else} + {@render tooltipTrigger(tooltipText, downloadChip)} +{/if} diff --git a/tools/ui/src/lib/components/app/models/discover/ModelsDiscoverDetails/ModelsDiscoverDetailsDownloadOptions/ModelsDiscoverDetailsDownloadOptionsRow.svelte b/tools/ui/src/lib/components/app/models/discover/ModelsDiscoverDetails/ModelsDiscoverDetailsDownloadOptions/ModelsDiscoverDetailsDownloadOptionsRow.svelte new file mode 100644 index 000000000..4527014fc --- /dev/null +++ b/tools/ui/src/lib/components/app/models/discover/ModelsDiscoverDetails/ModelsDiscoverDetailsDownloadOptions/ModelsDiscoverDetailsDownloadOptionsRow.svelte @@ -0,0 +1,134 @@ + + +
+
+ + + +
+ {#each files as file (file.path)} + + {/each} + + {#each missingDrafts as draft (draft.repoWithTag + ':' + draft.companion.kind)} + + {/each} +
+
+ + {#if commandOpen} +
+ +
+ {/if} +
diff --git a/tools/ui/src/lib/components/app/models/discover/ModelsDiscoverDetails/ModelsDiscoverDetailsDownloadOptions/download-options.utils.ts b/tools/ui/src/lib/components/app/models/discover/ModelsDiscoverDetails/ModelsDiscoverDetailsDownloadOptions/download-options.utils.ts new file mode 100644 index 000000000..3c07cbafd --- /dev/null +++ b/tools/ui/src/lib/components/app/models/discover/ModelsDiscoverDetails/ModelsDiscoverDetailsDownloadOptions/download-options.utils.ts @@ -0,0 +1,57 @@ +import { DRAFT_FILE_LABEL, MODEL_ID, PATH_SEPARATOR } from '$lib/constants'; +import { ModelDraftSidecar, ModelSelectableFileKind } from '$lib/enums'; +import { HuggingFaceService } from '$lib/services'; +import type { ModelSelectableFile } from '$lib/types'; +import { isAuxSidecar, isDraftSidecar } from '$lib/utils'; + +/** + * Draft kinds in the order a chip lists them. DFlash wins: on the models that + * ship both it drafts a block at a time and stays accurate further out than + * MTP's few tokens; a kind not listed here sorts last. + */ +export const DRAFT_PREFERENCE_ORDER: readonly ModelDraftSidecar[] = [ + ModelDraftSidecar.DFLASH, + ModelDraftSidecar.MTP +]; + +/** Draft sidecar of a file, typed for files already known to be drafts; null when it carries none. */ +export function draftBadgeOf(file: Pick): ModelDraftSidecar | null { + const sidecar = HuggingFaceService.extractQuantMeta(file.path)?.sidecar; + + return sidecar && isDraftSidecar(sidecar) ? sidecar : null; +} + +/** Kind of a file path: the main weights, a draft sidecar, or an aux sidecar (mmproj). */ +export function classify(path: string): ModelSelectableFileKind { + const sidecar = HuggingFaceService.extractQuantMeta(path)?.sidecar; + + if (!sidecar) return ModelSelectableFileKind.MAIN; + + return isAuxSidecar(sidecar) ? ModelSelectableFileKind.AUX : ModelSelectableFileKind.DRAFT; +} + +/** Draft kinds best first, so a chip lists its joined badges in the pull order. */ +export function sortDraftKinds(kinds: readonly ModelDraftSidecar[]): ModelDraftSidecar[] { + const rank = (kind: ModelDraftSidecar) => { + const index = DRAFT_PREFERENCE_ORDER.indexOf(kind); + + return index === -1 ? DRAFT_PREFERENCE_ORDER.length : index; + }; + + return [...kinds].sort((a, b) => rank(a) - rank(b)); +} + +/** Display label of a file: its quant, else the file name without the extension. */ +export function labelFor(path: string): string { + const meta = HuggingFaceService.extractQuantMeta(path); + + if (meta?.quant) return meta.quant; + + // Quantless sidecar files: the chip badge already carries the sidecar type, + // so the label only marks draft files; aux sidecars badge alone. + if (meta?.sidecar) return isDraftSidecar(meta.sidecar) ? DRAFT_FILE_LABEL : ''; + + const basename = path.split(PATH_SEPARATOR).pop() ?? path; + + return basename.replace(MODEL_ID.WEIGHT_EXTENSION_REGEX, ''); +} diff --git a/tools/ui/src/lib/components/app/models/discover/ModelsDiscoverDetails/ModelsDiscoverDetailsDownloadOptions/index.ts b/tools/ui/src/lib/components/app/models/discover/ModelsDiscoverDetails/ModelsDiscoverDetailsDownloadOptions/index.ts new file mode 100644 index 000000000..bb82218cc --- /dev/null +++ b/tools/ui/src/lib/components/app/models/discover/ModelsDiscoverDetails/ModelsDiscoverDetailsDownloadOptions/index.ts @@ -0,0 +1,24 @@ +/** + * + * MODELS DISCOVER - DETAILS - DOWNLOAD OPTIONS + * + * The download area of the detail pane: one chip per main quant, the drafts it rides with + * badged on it, a draft checkbox and type pick at the bottom, and a terminal command + * preview inside every row. + * + */ + +/** One download action per main quant, the sidecars it pulls, and the row's command. */ +export { default as ModelsDiscoverDetailsDownloadOptions } from './ModelsDiscoverDetailsDownloadOptions.svelte'; + +/** One bit-depth row: the depth label opens its command, the quants of that depth download beside it. */ +export { default as ModelsDiscoverDetailsDownloadOptionsRow } from './ModelsDiscoverDetailsDownloadOptionsRow.svelte'; + +/** A single GGUF file as an independent action chip. */ +export { default as ModelsDiscoverDetailsDownloadOptionsQuantDownloadButton } from './ModelsDiscoverDetailsDownloadOptionsQuantDownloadButton.svelte'; + +/** Chip for a draft a downloaded model is missing; posts the model with that kind, so the server keeps what is on disk and pulls the draft. */ +export { default as ModelsDiscoverDetailsDownloadOptionsDraftDownloadButton } from './ModelsDiscoverDetailsDownloadOptionsDraftDownloadButton.svelte'; + +/** The `llama serve -hf ...` command box of one row, with a draft type pick and a copy button. */ +export { default as ModelsDiscoverDetailsDownloadOptionsDownloadCommand } from './ModelsDiscoverDetailsDownloadOptionsDownloadCommand.svelte'; diff --git a/tools/ui/src/lib/components/app/models/discover/ModelsDiscoverDetails/ModelsDiscoverDetailsHeader.svelte b/tools/ui/src/lib/components/app/models/discover/ModelsDiscoverDetails/ModelsDiscoverDetailsHeader.svelte new file mode 100644 index 000000000..868765079 --- /dev/null +++ b/tools/ui/src/lib/components/app/models/discover/ModelsDiscoverDetails/ModelsDiscoverDetailsHeader.svelte @@ -0,0 +1,96 @@ + + +
+
+
+ + +
+
+

{details.id ?? modelId}

+ + +
+ + {#if baseModels.length} +
+ {baseModels.join(', ')} + + + + +
+ {/if} +
+
+ + +
+ + + + +
diff --git a/tools/ui/src/lib/components/app/models/discover/ModelsDiscoverDetails/ModelsDiscoverDetailsHfHubStats.svelte b/tools/ui/src/lib/components/app/models/discover/ModelsDiscoverDetails/ModelsDiscoverDetailsHfHubStats.svelte new file mode 100644 index 000000000..b848a9bfe --- /dev/null +++ b/tools/ui/src/lib/components/app/models/discover/ModelsDiscoverDetails/ModelsDiscoverDetailsHfHubStats.svelte @@ -0,0 +1,31 @@ + + +
+ {#if typeof details.downloads === 'number'} + + + {HuggingFaceService.formatDownloads(details.downloads)} + + {/if} + + {#if typeof details.likes === 'number'} + + + {HuggingFaceService.formatLikes(details.likes)} + + {/if} + + {#if details.lastModified} + Updated {HuggingFaceService.formatRelativeTime(details.lastModified)} + {/if} +
diff --git a/tools/ui/src/lib/components/app/models/discover/ModelsDiscoverDetails/ModelsDiscoverDetailsMetadata.svelte b/tools/ui/src/lib/components/app/models/discover/ModelsDiscoverDetails/ModelsDiscoverDetailsMetadata.svelte new file mode 100644 index 000000000..af7a38089 --- /dev/null +++ b/tools/ui/src/lib/components/app/models/discover/ModelsDiscoverDetails/ModelsDiscoverDetailsMetadata.svelte @@ -0,0 +1,82 @@ + + +{#if description} +

{description}

+{/if} + + +
+ {#if gguf?.total} + + {/if} + + {#if gguf?.context_length} + + {/if} + + {#if gguf?.architecture} + + {/if} + + {#if gguf?.chat_template} + + {/if} + + {#if licenseTag} + + {/if} + + {#if details.gated === true} + + gated + + {/if} +
+ +{#if gguf?.chat_template} + +{/if} diff --git a/tools/ui/src/lib/components/app/models/discover/ModelsDiscoverDetails/ModelsDiscoverDetailsMetadataItem.svelte b/tools/ui/src/lib/components/app/models/discover/ModelsDiscoverDetails/ModelsDiscoverDetailsMetadataItem.svelte new file mode 100644 index 000000000..4bfc027d8 --- /dev/null +++ b/tools/ui/src/lib/components/app/models/discover/ModelsDiscoverDetails/ModelsDiscoverDetailsMetadataItem.svelte @@ -0,0 +1,18 @@ + + + + + {label} + + {value} + diff --git a/tools/ui/src/lib/components/app/models/discover/ModelsDiscoverDetails/ModelsDiscoverDetailsReadme.svelte b/tools/ui/src/lib/components/app/models/discover/ModelsDiscoverDetails/ModelsDiscoverDetailsReadme.svelte new file mode 100644 index 000000000..fbf043b79 --- /dev/null +++ b/tools/ui/src/lib/components/app/models/discover/ModelsDiscoverDetails/ModelsDiscoverDetailsReadme.svelte @@ -0,0 +1,17 @@ + + +{#if readme} +
+ +
+{/if} diff --git a/tools/ui/src/lib/components/app/models/discover/ModelsDiscoverDetails/ModelsDiscoverDetailsSkeleton.svelte b/tools/ui/src/lib/components/app/models/discover/ModelsDiscoverDetails/ModelsDiscoverDetailsSkeleton.svelte new file mode 100644 index 000000000..8438a3c2b --- /dev/null +++ b/tools/ui/src/lib/components/app/models/discover/ModelsDiscoverDetails/ModelsDiscoverDetailsSkeleton.svelte @@ -0,0 +1,98 @@ + + + +
+ +
+
+
+ + +
+ + + +
+
+ + +
+ + +
+ + + + + +
+ + + + + +
+
+ + + + + + + +
+
+
+ + +
+
+ {#each [0, 1, 2] as _, index (index)} +
+
+ + + +
+ +
+ + + +
+
+ {/each} +
+ +
+ + +
+ + + + + +
+ + +
+
+ + +
+
+ + + + + + + +
+
+
diff --git a/tools/ui/src/lib/components/app/models/discover/ModelsDiscoverDetails/index.ts b/tools/ui/src/lib/components/app/models/discover/ModelsDiscoverDetails/index.ts new file mode 100644 index 000000000..91d5c8906 --- /dev/null +++ b/tools/ui/src/lib/components/app/models/discover/ModelsDiscoverDetails/index.ts @@ -0,0 +1,37 @@ +/** + * + * MODELS DISCOVER - DETAILS + * + * The right-hand detail pane of the discover view: header (avatar, name, stats, metadata + * chips, capability badges), the download options area and the model-card README. + * + */ + +/** Detail pane of the selected model; fetches its own details and GGUF file list. */ +export { default as ModelsDiscoverDetails } from './ModelsDiscoverDetails.svelte'; + +/** Fetches one repo's details, files and readme for hosts outside Discover. */ +export { default as ModelsDiscoverDetailsContainer } from './ModelsDiscoverDetailsContainer.svelte'; + +/** Header of the detail pane. */ +export { default as ModelsDiscoverDetailsHeader } from './ModelsDiscoverDetailsHeader.svelte'; + +/** Downloads, likes and last updated, formatted the way the Hub shows them. */ +export { default as ModelsDiscoverDetailsHfHubStats } from './ModelsDiscoverDetailsHfHubStats.svelte'; + +/** Static placeholder matching the detail layout. */ +export { default as ModelsDiscoverDetailsSkeleton } from './ModelsDiscoverDetailsSkeleton.svelte'; + +/** Model card README as markdown. */ +export { default as ModelsDiscoverDetailsReadme } from './ModelsDiscoverDetailsReadme.svelte'; + +/** Metadata chips: size, context, architecture, license, chat template. */ +export { default as ModelsDiscoverDetailsMetadata } from './ModelsDiscoverDetailsMetadata.svelte'; + +/** One label | value chip of the metadata row. */ +export { default as ModelsDiscoverDetailsMetadataItem } from './ModelsDiscoverDetailsMetadataItem.svelte'; + +/** Chat template in a scrollable dialog with a copy button. */ +export { default as ModelsDiscoverChatTemplateDialog } from './ModelsDiscoverChatTemplateDialog.svelte'; + +export * from './ModelsDiscoverDetailsDownloadOptions'; diff --git a/tools/ui/src/lib/components/app/models/discover/ModelsDiscoverList/ModelsDiscoverList.svelte b/tools/ui/src/lib/components/app/models/discover/ModelsDiscoverList/ModelsDiscoverList.svelte new file mode 100644 index 000000000..5ebff7230 --- /dev/null +++ b/tools/ui/src/lib/components/app/models/discover/ModelsDiscoverList/ModelsDiscoverList.svelte @@ -0,0 +1,41 @@ + + +
    + {#if loading} + {#each Array(loadingSkeletonRowsCount) as _, index (index)} + + {/each} + {:else} + {#each models as model (model.id)} + + {/each} + {/if} +
diff --git a/tools/ui/src/lib/components/app/models/discover/ModelsDiscoverList/ModelsDiscoverListItem.svelte b/tools/ui/src/lib/components/app/models/discover/ModelsDiscoverList/ModelsDiscoverListItem.svelte new file mode 100644 index 000000000..1eee85ea0 --- /dev/null +++ b/tools/ui/src/lib/components/app/models/discover/ModelsDiscoverList/ModelsDiscoverListItem.svelte @@ -0,0 +1,256 @@ + + +
  • +
    + + + {#if download} + {@const state = download} + + + + + {percent !== null ? `${percent}%` : state.isPaused ? 'Paused' : 'Downloading'} + + + + + {/if} +
    + + + {#if onDiskDrafts.length} +
    + {#each onDiskDrafts as badge (badge.repo + ':' + badge.kind + ':' + (badge.quant ?? ''))} + + {/each} +
    + {/if} + + {#if download?.progress && download.progress.totalBytes > 0} + + {/if} +
  • diff --git a/tools/ui/src/lib/components/app/models/discover/ModelsDiscoverList/ModelsDiscoverListItemSkeleton.svelte b/tools/ui/src/lib/components/app/models/discover/ModelsDiscoverList/ModelsDiscoverListItemSkeleton.svelte new file mode 100644 index 000000000..851abc523 --- /dev/null +++ b/tools/ui/src/lib/components/app/models/discover/ModelsDiscoverList/ModelsDiscoverListItemSkeleton.svelte @@ -0,0 +1,39 @@ + + + +
  • +
    + + +
    + + +
    + {#each badgeWidths as width, i (i)} + + {/each} +
    +
    +
    +
  • diff --git a/tools/ui/src/lib/components/app/models/discover/ModelsDiscoverList/ModelsDiscoverListSearch.svelte b/tools/ui/src/lib/components/app/models/discover/ModelsDiscoverList/ModelsDiscoverListSearch.svelte new file mode 100644 index 000000000..458c31648 --- /dev/null +++ b/tools/ui/src/lib/components/app/models/discover/ModelsDiscoverList/ModelsDiscoverListSearch.svelte @@ -0,0 +1,30 @@ + + + +
    + handleInput(v)} {placeholder} size="sm" /> +
    diff --git a/tools/ui/src/lib/components/app/models/discover/ModelsDiscoverList/index.ts b/tools/ui/src/lib/components/app/models/discover/ModelsDiscoverList/index.ts new file mode 100644 index 000000000..c3025c0b2 --- /dev/null +++ b/tools/ui/src/lib/components/app/models/discover/ModelsDiscoverList/index.ts @@ -0,0 +1,16 @@ +/** + * + * MODELS DISCOVER - LIST + * + * The discover sidebar column: a debounced search field above a navigable list of + * HuggingFace GGUF models, with skeleton rows while loading. + * + */ + +export { default as ModelsDiscoverList } from './ModelsDiscoverList.svelte'; +export { default as ModelsDiscoverListSearch } from './ModelsDiscoverListSearch.svelte'; + +/** One model entry; also the row of a tracked download, with its own controls. */ +export { default as ModelsDiscoverListItem } from './ModelsDiscoverListItem.svelte'; + +export { default as ModelsDiscoverListItemSkeleton } from './ModelsDiscoverListItemSkeleton.svelte'; diff --git a/tools/ui/src/lib/components/app/models/discover/index.ts b/tools/ui/src/lib/components/app/models/discover/index.ts new file mode 100644 index 000000000..bb353c72d --- /dev/null +++ b/tools/ui/src/lib/components/app/models/discover/index.ts @@ -0,0 +1,16 @@ +/** + * + * MODELS DISCOVER + * + * Components for the Models Discover view: a sidebar search + list of HuggingFace GGUF + * models and a detail view for the selected model, used as the body of the discovery + * dialog. The list and detail trees live in their own subfolders; this barrel re-exports + * them alongside the shared leaves. + * + */ + +/** The whole view: sidebar list on the left, detail pane on the right. */ +export { default as ModelsDiscover } from './ModelsDiscover.svelte'; + +export * from './ModelsDiscoverList'; +export * from './ModelsDiscoverDetails'; diff --git a/tools/ui/src/lib/components/app/models/index.ts b/tools/ui/src/lib/components/app/models/index.ts index dbfb5c60e..83f076fd1 100644 --- a/tools/ui/src/lib/components/app/models/index.ts +++ b/tools/ui/src/lib/components/app/models/index.ts @@ -59,4 +59,7 @@ export { default as ModelCapabilityIcons } from './ModelCapabilityIcons.svelte'; /** Draft sidecar badges, rendered as `+ [KIND] [QUANT]` per sidecar. */ export { default as ModelDraftSidecars } from './ModelDraftSidecars.svelte'; + +/** Compact draft download line, embedded in the item it belongs to. */ +export { default as ModelDraftSubline } from './ModelDraftSubline.svelte'; export * from './ModelsSelector'; diff --git a/tools/ui/src/lib/components/app/settings/SettingsChat/SettingsChat.svelte b/tools/ui/src/lib/components/app/settings/SettingsChat/SettingsChat.svelte index 879390075..aebf37251 100644 --- a/tools/ui/src/lib/components/app/settings/SettingsChat/SettingsChat.svelte +++ b/tools/ui/src/lib/components/app/settings/SettingsChat/SettingsChat.svelte @@ -13,6 +13,7 @@ NUMERIC_FIELDS, POSITIVE_INTEGER_FIELDS, SETTINGS_CHAT_SECTIONS, + SETTINGS_KEYS, SETTINGS_SECTION_SLUGS } from '$lib/constants'; import { ColorMode } from '$lib/enums/ui.enums'; @@ -70,6 +71,12 @@ function handleConfigChange(key: string, value: string | boolean) { localConfig[key] = value; + + // discover needs the hub metadata reads: checking it turns the hub on, + // unchecking it turns the hub back off, so the checkbox never lies + if (key === SETTINGS_KEYS.ENABLE_DISCOVER_MODELS) { + localConfig[SETTINGS_KEYS.USE_HUGGING_FACE_HUB] = value === true; + } } function handleReset() { diff --git a/tools/ui/src/lib/components/app/settings/SettingsChat/SettingsChatFields.svelte b/tools/ui/src/lib/components/app/settings/SettingsChat/SettingsChatFields.svelte index e8795d7c5..5dfba6bb3 100644 --- a/tools/ui/src/lib/components/app/settings/SettingsChat/SettingsChatFields.svelte +++ b/tools/ui/src/lib/components/app/settings/SettingsChat/SettingsChatFields.svelte @@ -22,6 +22,13 @@ let { fields, localConfig, onConfigChange, onThemeChange }: Props = $props(); + // discover needs the hub metadata reads, so its checkbox is read-only while discover is on + let hubLocked = $derived(Boolean(localConfig[SETTINGS_KEYS.ENABLE_DISCOVER_MODELS])); + + function isLockedField(key: string): boolean { + return hubLocked && key === SETTINGS_KEYS.USE_HUGGING_FACE_HUB; + } + let currentModelParams = $derived.by(() => { void modelsStore.props.cacheVersion; @@ -301,13 +308,18 @@ onConfigChange(field.key, checked)} />