Lumpiasty/llama.cpp
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

args: refactor mlock/mmap/directio into load-mode (#20834)

Browse Source 
* args: overhaul mmap/mlock/dio into single arg

Signed-off-by: Aaron Teo <aaron.teo1@ibm.com>

* docs: update docs with llama-gen-docs

Signed-off-by: Aaron Teo <aaron.teo1@ibm.com>

* chore: satisfy code quality

Signed-off-by: Aaron Teo <aaron.teo1@ibm.com>

* args: make the `+` sign an actual modifier now

Signed-off-by: Aaron Teo <aaron.teo1@ibm.com>

* chore: general code clean up + comments

Signed-off-by: Aaron Teo <aaron.teo1@ibm.com>

* arg: fix deprecated flags support

Signed-off-by: Aaron Teo <aaron.teo1@ibm.com>

* arg: quick sanity check

Signed-off-by: Aaron Teo <aaron.teo1@ibm.com>

* bench: sync llama-bench argument parsing

Signed-off-by: Aaron Teo <aaron.teo1@ibm.com>

* fix: bugfix variable behaviour + llama-bench lm column size

Signed-off-by: Aaron Teo <aaron.teo1@ibm.com>

* arg: inverse commands should do the opposite instead of doing nothing

Signed-off-by: Aaron Teo <aaron.teo1@ibm.com>

* bench: fix incorrect dash

Signed-off-by: Aaron Teo <aaron.teo1@ibm.com>

* bench: fix missing modifiers for deprecated flags

Signed-off-by: Aaron Teo <aaron.teo1@ibm.com>

* llama: switch back to thread_local

Signed-off-by: Aaron Teo <aaron.teo1@ibm.com>

* arg: switch back to single enum

Signed-off-by: Aaron Teo <aaron.teo1@ibm.com>

* docs: update arg docs

Signed-off-by: Aaron Teo <aaron.teo1@ibm.com>

* chore: fix missing `mlock` from llama_load_mode_from_str + cleanup llama-bench

Signed-off-by: Aaron Teo <aaron.teo1@ibm.com>

* llama: fix mlock not activating

Signed-off-by: Aaron Teo <aaron.teo1@ibm.com>

* arg: add deprecation warning when old and new flags are combined

Signed-off-by: Aaron Teo <aaron.teo1@ibm.com>

* arg: cont add comment for todo in the future

Signed-off-by: Aaron Teo <aaron.teo1@ibm.com>

* docs: sync with upstream

Signed-off-by: Aaron Teo <aaron.teo1@ibm.com>

* docs: re-sync with upstream again

Signed-off-by: Aaron Teo <aaron.teo1@ibm.com>

---------

Signed-off-by: Aaron Teo <aaron.teo1@ibm.com>
This commit is contained in:
Aaron Teo
2026-07-23 20:32:56 +08:00
committed by GitHub
parent da296d6e72
commit e6dd0e29a6
20 changed files with 315 additions and 207 deletions
Download Patch File Download Diff File Expand all files Collapse all files
common/arg.cpp
+36 -6
View File
@@ -5,6 +5,7 @@
#include "common.h"
#include "download.h"
#include "json-schema-to-grammar.h"
#include "llama.h"
#include "log.h"
#include "sampling.h"
#include "speculative.h"
@@ -785,6 +786,17 @@ static bool common_params_parse_ex(int argc, char ** argv, common_params_context
arg.c_str(), e.what(), opt.to_string().c_str()));
}
}
// TODO: remove this check after deprecating --mmap|mlock|dio
auto has_arg = [&](std::initializer_list<const char *> names) {
return std::any_of(names.begin(), names.end(), [&](const char * name) {
return seen_args.count(name);
});
};
if (has_arg({"-lm", "--load-mode"}) &&
has_arg({"--mlock", "--mmap", "--no-mmap", "-dio", "--direct-io", "-ndio", "--no-direct-io"})) {
LOG_WRN("DEPRECATED: `--load-mode` and `--mlock`/`--mmap`/`--direct-io` should not be combined; only the last flag on the command line will take effect\n");
}
};
// parse all CLI args now, so that -hf is available below for remote preset resolution
@@ -2495,27 +2507,45 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
}
add_opt(common_arg(
{"--mlock"},
"force system to keep model in RAM rather than swapping or compressing",
"DEPRECATED in favor of `--load-mode`: mmap + force system to keep model in RAM rather than swapping or compressing",
[](common_params & params) {
params.use_mlock = true;
LOG_WRN("DEPRECATED: --mlock is deprecated. use --load-mode mlock instead\n");
params.load_mode = LLAMA_LOAD_MODE_MLOCK;
}
).set_env("LLAMA_ARG_MLOCK"));
add_opt(common_arg(
{"--mmap"},
{"--no-mmap"},
string_format("whether to memory-map model. (if mmap disabled, slower load but may reduce pageouts if not using mlock) (default: %s)", params.use_mmap ? "enabled" : "disabled"),
"DEPRECATED in favor of `--load-mode`: whether to memory-map model. (if mmap disabled, slower load but may reduce pageouts if not using mlock)",
[](common_params & params, bool value) {
params.use_mmap = value;
LOG_WRN("DEPRECATED: --mmap and --no-mmap are deprecated. use --load-mode mmap instead\n");
params.load_mode = value ? LLAMA_LOAD_MODE_MMAP : LLAMA_LOAD_MODE_NONE;
}
).set_env("LLAMA_ARG_MMAP"));
add_opt(common_arg(
{"-dio", "--direct-io"},
{"-ndio", "--no-direct-io"},
string_format("use DirectIO if available. (default: %s)", params.use_direct_io ? "enabled" : "disabled"),
"DEPRECATED in favor of `--load-mode`: use DirectIO if available",
[](common_params & params, bool value) {
params.use_direct_io = value;
LOG_WRN("DEPRECATED: --direct-io and --no-direct-io are deprecated. use --load-mode dio instead\n");
params.load_mode = value ? LLAMA_LOAD_MODE_DIRECT_IO : LLAMA_LOAD_MODE_NONE;
}
).set_env("LLAMA_ARG_DIO"));
add_opt(common_arg(
{"-lm", "--load-mode"}, "MODE",
"model loading mode (default: mmap)\n"
"- none: no special loading mode\n"
"- mmap: memory-map model (if mmap disabled, slower load but may reduce pageouts if not using mlock)\n"
"- mlock: mmap + force system to keep model in RAM rather than swapping or compressing\n"
"- dio: use DirectIO if available\n",
[](common_params & params, const std::string & value) {
/**/ if (value == "none") { params.load_mode = LLAMA_LOAD_MODE_NONE; }
else if (value == "mmap") { params.load_mode = LLAMA_LOAD_MODE_MMAP; }
else if (value == "mlock") { params.load_mode = LLAMA_LOAD_MODE_MLOCK; }
else if (value == "dio") { params.load_mode = LLAMA_LOAD_MODE_DIRECT_IO; }
else { throw std::invalid_argument("invalid value"); }
}
).set_env("LLAMA_ARG_LOAD_MODE"));
add_opt(common_arg(
{"--numa"}, "TYPE",
"attempt optimizations that help on some NUMA systems\n"
common/common.cpp
+1 -3
View File
@@ -1558,10 +1558,8 @@ struct llama_model_params common_model_params_to_llama(common_params & params) {
mparams.n_gpu_layers = params.n_gpu_layers;
mparams.main_gpu = params.main_gpu;
mparams.split_mode = params.split_mode;
mparams.load_mode = params.load_mode;
mparams.tensor_split = params.tensor_split;
mparams.use_mmap = params.use_mmap;
mparams.use_direct_io = params.use_direct_io;
mparams.use_mlock = params.use_mlock;
mparams.check_tensors = params.check_tensors;
mparams.use_extra_bufts = !params.no_extra_bufts;
mparams.no_host = params.no_host;
common/common.h
+2 -3
View File
@@ -6,6 +6,7 @@
#include "ggml-opt.h"
#include "ggml.h"
#include "llama.h"
#include <set>
#include <sstream>
@@ -482,6 +483,7 @@ struct common_params {
std::vector<size_t> fit_params_target = std::vector<size_t>(llama_max_devices(), 1024 * 1024*1024);
enum llama_split_mode split_mode = LLAMA_SPLIT_MODE_LAYER; // how to split the model across GPUs
enum llama_load_mode load_mode = LLAMA_LOAD_MODE_MMAP; // how to load the model
common_cpu_params cpuparams;
common_cpu_params cpuparams_batch;
@@ -572,9 +574,6 @@ struct common_params {
bool kv_unified = false; // enable unified KV cache
bool input_prefix_bos = false; // prefix BOS to user inputs, preceding input_prefix
bool use_mmap = true; // enable mmap to use filesystem cache
bool use_direct_io = false; // read from disk without buffering
bool use_mlock = false; // use mlock to keep model in memory
bool verbose_prompt = false; // print prompt tokens before generation
bool display_prompt = true; // print prompt before generation
bool no_kv_offload = false; // disable KV offloading
common/fit.cpp
+1 -2
View File
@@ -54,8 +54,7 @@ static std::vector<llama_device_memory_data> common_get_device_memory_data_impl(
llama_model_params mparams_copy = *mparams;
mparams_copy.no_alloc = true;
mparams_copy.use_mmap = false;
mparams_copy.use_mlock = false;
mparams_copy.load_mode = LLAMA_LOAD_MODE_NONE;
llama_model * model = llama_model_load_from_file(path_model, mparams_copy);
if (model == nullptr) {