Lumpiasty/llama.cpp
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

llama: add default load-mode auto, which avoids mmap on iGPUs (#26081)

Browse Source 
* llama: add new default load-mode auto which picks mmap unless a non-Metal iGPU is used

* Update ggml/src/ggml-hexagon/ggml-hexagon.cpp

Co-authored-by: Max Krasnyansky <maxk@qti.qualcomm.com>

* set mmap_support to false on OpenCL backend

* fix order of load modes

* use -1 for auto

* resolve load mode auto earlier to correctly pick gpu host or cpu memory

* add load mode auto to llama-bench

* bump virtgpu api version, regenerate docs

---------

Co-authored-by: Piotr Wilkin (ilintar) <piotr.wilkin@syndatis.com>
Co-authored-by: Max Krasnyansky <maxk@qti.qualcomm.com>
Co-authored-by: Georgi Gerganov <ggerganov@gmail.com>
This commit is contained in:
Ruben Ortlam
2026-08-11 09:20:46 +03:00
committed by GitHub
co-authored by Max Krasnyansky Piotr Wilkin Georgi Gerganov
parent b3df57286c
commit 153d324bcf
33 changed files with 75 additions and 29 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tools/llama-bench/llama-bench.cpp
+5 -3
View File
@@ -384,7 +384,7 @@ static const cmd_params cmd_params_defaults = {
/* n_gpu_layers */ { -1 },
/* n_cpu_moe */ { 0 },
/* split_mode */ { LLAMA_SPLIT_MODE_LAYER },
/* load_mode */ { LLAMA_LOAD_MODE_MMAP },
/* load_mode */ { LLAMA_LOAD_MODE_AUTO },
/* main_gpu */ { 0 },
/* no_kv_offload */ { false },
/* flash_attn */ { LLAMA_FLASH_ATTN_TYPE_AUTO },
@@ -459,7 +459,7 @@ static void print_usage(int /* argc */, char ** argv) {
printf(" -nkvo, --no-kv-offload <0|1> (default: %s)\n", join(cmd_params_defaults.no_kv_offload, ",").c_str());
printf(" -fa, --flash-attn <on|off|auto> (default: %s)\n", join(transform_to_str(cmd_params_defaults.flash_attn, llama_flash_attn_type_name), ",").c_str());
printf(" -dev, --device <dev0/dev1/...> (default: auto)\n");
printf(" -lm, --load-mode <none|mmap|mlock|mmap+mlock|dio> (default: %s)\n", join(transform_to_str(cmd_params_defaults.load_mode, llama_load_mode_name), ",").c_str());
printf(" -lm, --load-mode <auto|none|mmap|mlock|mmap+mlock|dio> (default: %s)\n", join(transform_to_str(cmd_params_defaults.load_mode, llama_load_mode_name), ",").c_str());
printf(" -mmp, --mmap <0|1> (DEPRECATED IN FAVOUR OF --load-mode)\n");
printf(" -dio, --direct-io <0|1> (DEPRECATED IN FAVOUR OF --load-mode)\n");
printf(" -embd, --embeddings <0|1> (default: %s)\n", join(cmd_params_defaults.embeddings, ",").c_str());
@@ -764,7 +764,9 @@ static cmd_params parse_cmd_params(int argc, char ** argv) {
std::vector<llama_load_mode> modes;
for (const auto & m : p) {
llama_load_mode mode;
if (m == "none") {
if (m == "auto") {
mode = LLAMA_LOAD_MODE_AUTO;
} else if (m == "none") {
mode = LLAMA_LOAD_MODE_NONE;
} else if (m == "mmap") {
mode = LLAMA_LOAD_MODE_MMAP;