Lumpiasty/llama.cpp
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

spec: single device drafter should create meta backend wrapper (#28390)

Browse Source
This commit is contained in:
Aman Gupta
2026-09-08 20:44:33 +08:00
committed by GitHub
parent 03fa73cb27
commit 415e909d84
5 changed files with 27 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files
common/arg.cpp
+8 -2
View File
@@ -894,6 +894,12 @@ static bool common_params_parse_ex(int argc, char ** argv, common_params_context
postprocess_cpu_params(params.speculative.draft.cpuparams, &params.cpuparams);
postprocess_cpu_params(params.speculative.draft.cpuparams_batch, &params.cpuparams_batch);
// default the mmproj device to the global device selection if not set explicitly with -mmdev
if (params.mmproj_use_gpu && params.mmproj_device == nullptr && !params.devices.empty()) {
params.mmproj_device = params.devices.front();
params.mmproj_use_gpu = params.mmproj_device != nullptr;
}
if (params.prompt_cache_all && (params.interactive || params.interactive_first)) {
throw std::invalid_argument("error: --prompt-cache-all not supported in interactive mode yet\n");
}
@@ -2610,7 +2616,7 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
add_opt(common_arg(
// note: "-mmdev" must sort after "--rpc" in the preset map, else RPC devices are not registered yet
{"-mmdev", "--mmproj-device"}, "DEVICE",
"device to use for multimodal projector (none = don't offload, default: auto)\n"
"device to use for multimodal projector (none = don't offload, default: follows --device)\n"
"use --list-devices to see a list of available devices",
[](common_params & params, const std::string & value) {
if (value == "none") {
@@ -4229,7 +4235,7 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
).set_spec().set_examples({LLAMA_EXAMPLE_SPECULATIVE, LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_CLI}).set_env("LLAMA_ARG_SPEC_DRAFT_BACKEND_SAMPLING"));
add_opt(common_arg(
{"--spec-draft-device", "-devd", "--device-draft"}, "<dev1,dev2,..>",
"comma-separated list of devices to use for offloading the draft model (none = don't offload)\n"
"comma-separated list of devices to use for offloading the draft model (none = don't offload, default: follows --device)\n"
"use --list-devices to see a list of available devices",
[](common_params & params, const std::string & value) {
params.speculative.draft.devices = parse_device_list(value);