Lumpiasty/llama.cpp
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

common : auto-download dflash- and eagle3- HF sidecars (#25811)

Browse Source 
* common: auto-download dflash- and eagle3- HF sidecars

Mirror the existing mtp- sidecar logic to support auto-discovery and
download of DFlash (dflash-) and Eagle3 (eagle3-) speculative decoding
sidecars from Hugging Face repos.

Changes:
- Add --dflash and --eagle3 CLI flags to trigger sidecar download
- Add find_best_dflash() and find_best_eagle3() using find_best_sibling
- Exclude dflash- and eagle3- filenames from primary model selection
- Filter dflash- and eagle3- from cached model listings
- Wire download tasks that set speculative.draft.mparams as fallback

Assisted-by: pi:llama.cpp/Qwen3.6-27B

* docs : regen
This commit is contained in:
Georgi Gerganov
2026-07-17 12:15:30 +03:00
committed by GitHub
parent 11fd0a6fb7
commit 635cdd5fcc
6 changed files with 91 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files
common/arg.cpp
+44
View File
@@ -361,6 +361,14 @@ common_models_handler common_models_handler_init(const common_params & params, l
params.speculative.types.end(),
COMMON_SPECULATIVE_TYPE_DRAFT_MTP) != params.speculative.types.end();
const bool spec_type_draft_dflash = std::find(params.speculative.types.begin(),
params.speculative.types.end(),
COMMON_SPECULATIVE_TYPE_DRAFT_DFLASH) != params.speculative.types.end();
const bool spec_type_draft_eagle3 = std::find(params.speculative.types.begin(),
params.speculative.types.end(),
COMMON_SPECULATIVE_TYPE_DRAFT_EAGLE3) != params.speculative.types.end();
// only download mmproj if the current example is using it
bool use_mmproj = false;
for (const auto & ex : mmproj_examples) {
@@ -373,6 +381,8 @@ common_models_handler common_models_handler_init(const common_params & params, l
opts.bearer_token = params.hf_token;
opts.offline = params.offline;
opts.download_mtp = spec_type_draft_mtp;
opts.download_eagle3 = spec_type_draft_eagle3;
opts.download_dflash = spec_type_draft_dflash;
opts.download_mmproj = use_mmproj && !params.no_mmproj
&& params.mmproj.path.empty() && params.mmproj.url.empty();
@@ -546,6 +556,26 @@ void common_models_handler_apply(common_models_handler & handler, common_params
}
});
}
if (!plan.dflash.local_path.empty() && !had_spec_url) {
tasks.emplace_back(plan.dflash, opts, [&]() {
// only fall back to the discovered DFlash sidecar when no draft was explicitly provided
if (params.speculative.draft.mparams.empty()) {
params.speculative.draft.mparams.path = hf_cache::finalize_file(plan.dflash);
} else {
hf_cache::finalize_file(plan.dflash);
}
});
}
if (!plan.eagle3.local_path.empty() && !had_spec_url) {
tasks.emplace_back(plan.eagle3, opts, [&]() {
// only fall back to the discovered Eagle3 sidecar when no draft was explicitly provided
if (params.speculative.draft.mparams.empty()) {
params.speculative.draft.mparams.path = hf_cache::finalize_file(plan.eagle3);
} else {
hf_cache::finalize_file(plan.eagle3);
}
});
}
if (!plan.preset.local_path.empty()) {
tasks.emplace_back(plan.preset, opts, [&]() {
// if HF repo is a preset repo, we simply run server in router mode with the preset.ini file
@@ -2815,6 +2845,20 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
params.speculative.types.push_back(COMMON_SPECULATIVE_TYPE_DRAFT_MTP);
}
).set_examples({LLAMA_EXAMPLE_DOWNLOAD}));
add_opt(common_arg(
{"--dflash"},
"also download the DFlash sidecar, if available (default: unused)",
[](common_params & params) {
params.speculative.types.push_back(COMMON_SPECULATIVE_TYPE_DRAFT_DFLASH);
}
).set_examples({LLAMA_EXAMPLE_DOWNLOAD}));
add_opt(common_arg(
{"--eagle3"},
"also download the Eagle3 sidecar, if available (default: unused)",
[](common_params & params) {
params.speculative.types.push_back(COMMON_SPECULATIVE_TYPE_DRAFT_EAGLE3);
}
).set_examples({LLAMA_EXAMPLE_DOWNLOAD}));
add_opt(common_arg(
{"--context-file"}, "FNAME",
"file to load context from (use comma-separated values to specify multiple files)",
common/download.cpp
+23 -3
View File
@@ -620,6 +620,16 @@ static hf_cache::hf_file find_best_mtp(const hf_cache::hf_files & files,
return find_best_sibling(files, model, "mtp-");
}
static hf_cache::hf_file find_best_eagle3(const hf_cache::hf_files & files,
const std::string & model) {
return find_best_sibling(files, model, "eagle3-");
}
static hf_cache::hf_file find_best_dflash(const hf_cache::hf_files & files,
const std::string & model) {
return find_best_sibling(files, model, "dflash-");
}
static bool gguf_filename_is_model(const std::string & filepath) {
if (!string_ends_with(filepath, ".gguf")) {
return false;
@@ -632,7 +642,9 @@ static bool gguf_filename_is_model(const std::string & filepath) {
return filename.find("mmproj") == std::string::npos &&
filename.find("imatrix") == std::string::npos &&
filename.find("mtp-") == std::string::npos;
filename.find("mtp-") == std::string::npos &&
filename.find("eagle3-") == std::string::npos &&
filename.find("dflash-") == std::string::npos;
}
static hf_cache::hf_file find_best_model(const hf_cache::hf_files & files,
@@ -740,6 +752,12 @@ common_download_hf_plan common_download_get_hf_plan(const common_params_model &
if (opts.download_mtp) {
plan.mtp = find_best_mtp(all, primary.path);
}
if (opts.download_dflash) {
plan.dflash = find_best_dflash(all, primary.path);
}
if (opts.download_eagle3) {
plan.eagle3 = find_best_eagle3(all, primary.path);
}
return plan;
}
@@ -911,8 +929,10 @@ std::vector<common_cached_model_info> common_list_cached_models() {
for (const auto & f : files) {
auto split = get_gguf_split_info(f.path);
if (split.index != 1 || split.tag.empty() ||
split.prefix.find("mmproj") != std::string::npos ||
split.prefix.find("mtp-") != std::string::npos) {
split.prefix.find("mmproj") != std::string::npos ||
split.prefix.find("mtp-") != std::string::npos ||
split.prefix.find("eagle3-") != std::string::npos ||
split.prefix.find("dflash-") != std::string::npos) {
continue;
}
if (seen.insert(f.repo_id + ":" + split.tag).second) {
common/download.h
+6 -2
View File
@@ -55,8 +55,10 @@ struct common_download_opts {
std::string bearer_token;
common_header_list headers;
bool offline = false;
bool download_mmproj = false;
bool download_mtp = false;
bool download_mmproj = false;
bool download_mtp = false;
bool download_eagle3 = false;
bool download_dflash = false;
common_download_callback * callback = nullptr;
};
@@ -106,6 +108,8 @@ struct common_download_hf_plan {
hf_cache::hf_files model_files;
hf_cache::hf_file mmproj;
hf_cache::hf_file mtp;
hf_cache::hf_file eagle3;
hf_cache::hf_file dflash;
hf_cache::hf_file preset; // if set, only this file is downloaded
};
common_download_hf_plan common_download_get_hf_plan(const common_params_model & model, const common_download_opts & opts);