Lumpiasty/llama.cpp
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

mtmd: add --mmproj-device argument (#23255)

Browse Source 
* feat: add --mmproj-device arg & backwards compatible MTMD_BACKEND_DEVICE env var

* feat: load mmproj device backend immediately, add -mmdev shortflag

* fix: its a pointer now get the name

* clean up

* gen docs

* nits

---------

Co-authored-by: Xuan Son Nguyen <son@huggingface.co>
This commit is contained in:
John-Henry Lim
2026-08-20 18:45:37 +02:00
committed by GitHub
co-authored by Xuan Son Nguyen
parent 07822bddf8
commit 681c29d36a
12 changed files with 41 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files
common/arg.cpp
+20
View File
@@ -2595,6 +2595,26 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
params.mmproj_use_gpu = value;
}
).set_examples(mmproj_examples).set_env("LLAMA_ARG_MMPROJ_OFFLOAD"));
add_opt(common_arg(
// note: "-mmdev" must sort after "--rpc" in the preset map, else RPC devices are not registered yet
{"-mmdev", "--mmproj-device"}, "DEVICE",
"device to use for multimodal projector (none = don't offload, default: auto)\n"
"use --list-devices to see a list of available devices",
[](common_params & params, const std::string & value) {
if (value == "none") {
params.mmproj_use_gpu = false;
params.mmproj_device = nullptr;
return;
}
auto devices = parse_device_list(value);
// parse_device_list pushes nullptr at back so devices is length 2 for single device.
if (devices.size() > 2) {
throw std::invalid_argument("only one device may be specified for mmproj");
}
params.mmproj_use_gpu = true;
params.mmproj_device = devices.front();
}
).set_examples(mmproj_examples).set_env("MTMD_BACKEND_DEVICE")); // no LLAMA_ARG_ prefix for backward compatibility reason
add_opt(common_arg(
{"--image", "--audio", "--video"}, "FILE",
"path to an image, audio, or video file. use with multimodal models, use comma-separated values for multiple files\n",
common/common.h
+4 -3
View File
@@ -581,9 +581,10 @@ struct common_params {
// multimodal models (see tools/mtmd)
struct common_params_model mmproj;
bool mmproj_use_gpu = true; // use GPU for multimodal model
bool no_mmproj = false; // explicitly disable multimodal model
std::vector<std::string> image; // path to image file(s) ; TODO: change the name to "media"
bool mmproj_use_gpu = true; // use GPU for multimodal model
ggml_backend_dev_t mmproj_device = nullptr; // GPU device to use for multimodal model
bool no_mmproj = false; // explicitly disable multimodal model
std::vector<std::string> image; // path to image file(s) ; TODO: change the name to "media"
int image_min_tokens = -1;
int image_max_tokens = -1;
int mtmd_batch_max_tokens = 1024;