Lumpiasty/llama.cpp
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

bench: add --tensor-read-lazy (#27881)

Browse Source 
* bench: add --tensor-read-lazy

* rm the alias

* rename to LLAMA_LAZY_MODE_*
This commit is contained in:
Xuan-Son Nguyen
2026-08-28 20:51:05 +02:00
committed by GitHub
parent 6fe7498016
commit 50f068ffff
10 changed files with 78 additions and 18 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tools/llama-bench/README.md
+1
View File
@@ -67,6 +67,7 @@ test parameters:
-nkvo, --no-kv-offload <0|1> (default: 0)
-fa, --flash-attn <on|off|auto> (default: auto)
-dev, --device <dev0/dev1/...> (default: auto)
--tensor-read-lazy <on|auto|off> (default: auto)
-mmp, --mmap <0|1> (DEPRECATED IN FAVOUR OF --load-mode)
-dio, --direct-io <0|1> (DEPRECATED IN FAVOUR OF --load-mode)
-embd, --embeddings <0|1> (default: 0)
tools/llama-bench/llama-bench.cpp
+62 -3
View File
@@ -271,6 +271,19 @@ static const char * split_mode_str(llama_split_mode mode) {
}
}
static const char * lazy_mode_str(llama_lazy_mode mode) {
switch (mode) {
case LLAMA_LAZY_MODE_OFF:
return "off";
case LLAMA_LAZY_MODE_AUTO:
return "auto";
case LLAMA_LAZY_MODE_ON:
return "on";
default:
GGML_ABORT("invalid tensor read lazy mode");
}
}
static std::string pair_str(const std::pair<int, int> & p) {
static char buf[32];
snprintf(buf, sizeof(buf), "%d,%d", p.first, p.second);
@@ -341,6 +354,7 @@ struct cmd_params {
std::vector<int> n_cpu_moe;
std::vector<llama_split_mode> split_mode;
std::vector<llama_load_mode> load_mode;
std::vector<llama_lazy_mode> lazy_mode;
std::vector<int> main_gpu;
std::vector<bool> no_kv_offload;
std::vector<llama_flash_attn_type> flash_attn;
@@ -385,6 +399,7 @@ static const cmd_params cmd_params_defaults = {
/* n_cpu_moe */ { 0 },
/* split_mode */ { LLAMA_SPLIT_MODE_LAYER },
/* load_mode */ { LLAMA_LOAD_MODE_AUTO },
/* lazy_mode */ { LLAMA_LAZY_MODE_AUTO },
/* main_gpu */ { 0 },
/* no_kv_offload */ { false },
/* flash_attn */ { LLAMA_FLASH_ATTN_TYPE_AUTO },
@@ -460,6 +475,7 @@ static void print_usage(int /* argc */, char ** argv) {
printf(" -fa, --flash-attn <on|off|auto> (default: %s)\n", join(transform_to_str(cmd_params_defaults.flash_attn, llama_flash_attn_type_name), ",").c_str());
printf(" -dev, --device <dev0/dev1/...> (default: auto)\n");
printf(" -lm, --load-mode <auto|none|mmap|mlock|mmap+mlock|dio> (default: %s)\n", join(transform_to_str(cmd_params_defaults.load_mode, llama_load_mode_name), ",").c_str());
printf(" --tensor-read-lazy <on|auto|off> (default: %s)\n", join(transform_to_str(cmd_params_defaults.lazy_mode, lazy_mode_str), ",").c_str());
printf(" -mmp, --mmap <0|1> (DEPRECATED IN FAVOUR OF --load-mode)\n");
printf(" -dio, --direct-io <0|1> (DEPRECATED IN FAVOUR OF --load-mode)\n");
printf(" -embd, --embeddings <0|1> (default: %s)\n", join(cmd_params_defaults.embeddings, ",").c_str());
@@ -786,6 +802,32 @@ static cmd_params parse_cmd_params(int argc, char ** argv) {
break;
}
params.load_mode.insert(params.load_mode.end(), modes.begin(), modes.end());
} else if (arg == "--tensor-read-lazy") {
if (++i >= argc) {
invalid_param = true;
break;
}
auto p = string_split<std::string>(argv[i], split_delim);
std::vector<llama_lazy_mode> modes;
for (const auto & m : p) {
llama_lazy_mode mode;
if (m == "on") {
mode = LLAMA_LAZY_MODE_ON;
} else if (m == "auto") {
mode = LLAMA_LAZY_MODE_AUTO;
} else if (m == "off") {
mode = LLAMA_LAZY_MODE_OFF;
} else {
invalid_param = true;
break;
}
modes.push_back(mode);
}
if (invalid_param) {
break;
}
params.lazy_mode.insert(params.lazy_mode.end(), modes.begin(), modes.end());
} else if (arg == "-mg" || arg == "--main-gpu") {
if (++i >= argc) {
invalid_param = true;
@@ -1137,6 +1179,9 @@ static cmd_params parse_cmd_params(int argc, char ** argv) {
if (params.load_mode.empty()) {
params.load_mode = cmd_params_defaults.load_mode;
}
if (params.lazy_mode.empty()) {
params.lazy_mode = cmd_params_defaults.lazy_mode;
}
if (params.main_gpu.empty()) {
params.main_gpu = cmd_params_defaults.main_gpu;
}
@@ -1203,6 +1248,7 @@ struct cmd_params_instance {
int n_cpu_moe;
llama_split_mode split_mode;
llama_load_mode load_mode;
llama_lazy_mode lazy_mode;
int main_gpu;
bool no_kv_offload;
llama_flash_attn_type flash_attn;
@@ -1224,6 +1270,7 @@ struct cmd_params_instance {
}
mparams.split_mode = split_mode;
mparams.load_mode = load_mode;
mparams.lazy_mode = lazy_mode;
mparams.main_gpu = main_gpu;
mparams.tensor_split = tensor_split.data();
mparams.no_host = no_host;
@@ -1271,7 +1318,8 @@ struct cmd_params_instance {
return model == other.model && n_gpu_layers == other.n_gpu_layers && n_cpu_moe == other.n_cpu_moe &&
split_mode == other.split_mode &&
main_gpu == other.main_gpu && tensor_split == other.tensor_split &&
load_mode == other.load_mode && devices == other.devices && no_host == other.no_host &&
load_mode == other.load_mode && lazy_mode == other.lazy_mode &&
devices == other.devices && no_host == other.no_host &&
vec_tensor_buft_override_equal(tensor_buft_overrides, other.tensor_buft_overrides);
}
@@ -1305,6 +1353,7 @@ static std::vector<cmd_params_instance> get_cmd_params_instances(const cmd_param
for (const auto & ncmoe : params.n_cpu_moe)
for (const auto & sm : params.split_mode)
for (const auto & lm : params.load_mode)
for (const auto & lzm : params.lazy_mode)
for (const auto & mg : params.main_gpu)
for (const auto & devs : params.devices)
for (const auto & ts : params.tensor_split)
@@ -1344,6 +1393,7 @@ static std::vector<cmd_params_instance> get_cmd_params_instances(const cmd_param
/* .n_cpu_moe = */ ncmoe,
/* .split_mode = */ sm,
/* .load_mode = */ lm,
/* .lazy_mode = */ lzm,
/* .main_gpu = */ mg,
/* .no_kv_offload = */ nkvo,
/* .flash_attn = */ fa,
@@ -1380,6 +1430,7 @@ static std::vector<cmd_params_instance> get_cmd_params_instances(const cmd_param
/* .n_cpu_moe = */ ncmoe,
/* .split_mode = */ sm,
/* .load_mode = */ lm,
/* .lazy_mode = */ lzm,
/* .main_gpu = */ mg,
/* .no_kv_offload = */ nkvo,
/* .flash_attn = */ fa,
@@ -1416,6 +1467,7 @@ static std::vector<cmd_params_instance> get_cmd_params_instances(const cmd_param
/* .n_cpu_moe = */ ncmoe,
/* .split_mode = */ sm,
/* .load_mode = */ lm,
/* .lazy_mode = */ lzm,
/* .main_gpu = */ mg,
/* .no_kv_offload = */ nkvo,
/* .flash_attn = */ fa,
@@ -1457,6 +1509,7 @@ struct test {
int n_cpu_moe;
llama_split_mode split_mode;
llama_load_mode load_mode;
llama_lazy_mode lazy_mode;
int main_gpu;
bool no_kv_offload;
llama_flash_attn_type flash_attn;
@@ -1496,6 +1549,7 @@ struct test {
n_cpu_moe = inst.n_cpu_moe;
split_mode = inst.split_mode;
load_mode = inst.load_mode;
lazy_mode = inst.lazy_mode;
main_gpu = inst.main_gpu;
no_kv_offload = inst.no_kv_offload;
flash_attn = inst.flash_attn;
@@ -1563,7 +1617,8 @@ struct test {
"n_ubatch", "n_threads", "cpu_mask", "cpu_strict", "poll",
"type_k", "type_v", "n_gpu_layers", "n_cpu_moe", "split_mode",
"main_gpu", "no_kv_offload", "flash_attn", "devices", "tensor_split",
"tensor_buft_overrides", "load_mode", "embeddings",
"tensor_buft_overrides", "load_mode", "lazy_mode",
"embeddings",
"no_op_offload", "no_host", "fit_target", "fit_min_ctx",
"n_prompt", "n_gen", "n_depth",
"test_time", "avg_ns", "stddev_ns", "avg_ts", "stddev_ts"
@@ -1588,7 +1643,7 @@ struct test {
if (field == "avg_ts" || field == "stddev_ts") {
return FLOAT;
}
if (field == "load_mode") {
if (field == "load_mode" || field == "lazy_mode") {
return STRING;
}
return STRING;
@@ -1658,6 +1713,7 @@ struct test {
tensor_split_str,
tensor_buft_overrides_str,
llama_load_mode_name(load_mode),
lazy_mode_str(lazy_mode),
std::to_string(embeddings),
std::to_string(no_op_offload),
std::to_string(no_host),
@@ -1972,6 +2028,9 @@ struct markdown_printer : public printer {
if (params.load_mode.size() > 1 || params.load_mode != cmd_params_defaults.load_mode) {
fields.emplace_back("load_mode");
}
if (params.lazy_mode.size() > 1 || params.lazy_mode != cmd_params_defaults.lazy_mode) {
fields.emplace_back("lazy_mode");
}
if (params.embeddings.size() > 1 || params.embeddings != cmd_params_defaults.embeddings) {
fields.emplace_back("embeddings");
}