Lumpiasty/llama.cpp
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

bench: add --tensor-read-lazy (#27881)

Browse Source 
* bench: add --tensor-read-lazy

* rm the alias

* rename to LLAMA_LAZY_MODE_*
This commit is contained in:
Xuan-Son Nguyen
2026-08-28 20:51:05 +02:00
committed by GitHub
parent 6fe7498016
commit 50f068ffff
10 changed files with 78 additions and 18 deletions
Download Patch File Download Diff File Expand all files Collapse all files
common/arg.cpp
+3 -3
View File
@@ -2735,9 +2735,9 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
"- auto: on, but only for tensors larger than 4 GiB\n"
"- off: always keep them resident",
[](common_params & params, const std::string & value) {
/**/ if (value == "on") { params.tensor_read_lazy = LLAMA_TENSOR_READ_LAZY_ON; }
else if (value == "auto") { params.tensor_read_lazy = LLAMA_TENSOR_READ_LAZY_AUTO; }
else if (value == "off") { params.tensor_read_lazy = LLAMA_TENSOR_READ_LAZY_OFF; }
/**/ if (value == "on") { params.lazy_mode = LLAMA_LAZY_MODE_ON; }
else if (value == "auto") { params.lazy_mode = LLAMA_LAZY_MODE_AUTO; }
else if (value == "off") { params.lazy_mode = LLAMA_LAZY_MODE_OFF; }
else { throw std::invalid_argument("invalid value"); }
}
).set_env("LLAMA_ARG_TENSOR_READ_LAZY"));
common/common.cpp
+1 -1
View File
@@ -1688,7 +1688,7 @@ struct llama_model_params common_model_params_to_llama(common_params & params) {
mparams.main_gpu = params.main_gpu;
mparams.split_mode = params.split_mode;
mparams.load_mode = params.load_mode;
mparams.tensor_read_lazy = params.tensor_read_lazy;
mparams.lazy_mode = params.lazy_mode;
mparams.tensor_split = params.tensor_split;
mparams.check_tensors = params.check_tensors;
mparams.use_extra_bufts = !params.no_extra_bufts;
common/common.h
+1 -1
View File
@@ -483,7 +483,7 @@ struct common_params {
enum llama_split_mode split_mode = LLAMA_SPLIT_MODE_LAYER; // how to split the model across GPUs
enum llama_load_mode load_mode = LLAMA_LOAD_MODE_AUTO; // how to load the model
enum llama_tensor_read_lazy tensor_read_lazy = LLAMA_TENSOR_READ_LAZY_AUTO; // on-demand reading of tensors marked by the arch
enum llama_lazy_mode lazy_mode = LLAMA_LAZY_MODE_AUTO; // on-demand reading of tensors marked by the arch
common_cpu_params cpuparams;
common_cpu_params cpuparams_batch;
include/llama.h
+5 -5
View File
@@ -214,10 +214,10 @@ extern "C" {
LLAMA_API const char * llama_load_mode_name(enum llama_load_mode load_mode);
LLAMA_API enum llama_load_mode llama_load_mode_from_str(const char * str);
enum llama_tensor_read_lazy {
LLAMA_TENSOR_READ_LAZY_OFF = 0, // always read the whole tensor up front
LLAMA_TENSOR_READ_LAZY_AUTO = 1, // lazy only for marked tensors larger than 4 GiB (requires mmap)
LLAMA_TENSOR_READ_LAZY_ON = 2, // read the rows of tensors marked by the arch on demand (requires mmap)
enum llama_lazy_mode {
LLAMA_LAZY_MODE_OFF = 0, // always read the whole tensor up front
LLAMA_LAZY_MODE_AUTO = 1, // lazy only for marked tensors larger than 4 GiB (requires mmap)
LLAMA_LAZY_MODE_ON = 2, // read the rows of tensors marked by the arch on demand (requires mmap)
};
enum llama_context_type {
@@ -321,7 +321,7 @@ extern "C" {
enum llama_split_mode split_mode; // how to split the model across multiple GPUs
enum llama_load_mode load_mode; // how to load the model
enum llama_tensor_read_lazy tensor_read_lazy; // on-demand reading of tensors marked by the arch
enum llama_lazy_mode lazy_mode; // on-demand reading of tensors marked by the arch
// the GPU that is used for the entire model when split_mode is LLAMA_SPLIT_MODE_NONE
int32_t main_gpu;
src/llama-model-loader.cpp
+2 -2
View File
@@ -1287,10 +1287,10 @@ struct ggml_tensor * llama_model_loader::create_tensor(
return NULL;
}
if ((flags & TENSOR_READ_LAZY) && use_mmap && tensor_read_lazy != LLAMA_TENSOR_READ_LAZY_OFF) {
if ((flags & TENSOR_READ_LAZY) && use_mmap && lazy_mode != LLAMA_LAZY_MODE_OFF) {
// in auto mode, small tensors are cheap enough to keep resident
constexpr size_t auto_lazy_min_size = 4ull * 1024 * 1024 * 1024;
if (tensor_read_lazy == LLAMA_TENSOR_READ_LAZY_ON || ggml_nbytes(cur) > auto_lazy_min_size) {
if (lazy_mode == LLAMA_LAZY_MODE_ON || ggml_nbytes(cur) > auto_lazy_min_size) {
const auto & w = require_weight(tn.str().c_str());
lazy_tensor_ranges[w.idx].emplace_back(w.offs, w.offs + ggml_nbytes(cur));
src/llama-model-loader.h
+1 -1
View File
@@ -84,7 +84,7 @@ struct llama_model_loader {
bool load_mtp;
// set by the caller before the create_tensor() calls
enum llama_tensor_read_lazy tensor_read_lazy = LLAMA_TENSOR_READ_LAZY_OFF;
enum llama_lazy_mode lazy_mode = LLAMA_LAZY_MODE_OFF;
llama_files files;
llama_ftype ftype;
src/llama-model.cpp
+1 -1
View File
@@ -2681,7 +2681,7 @@ llama_model_params llama_model_default_params() {
/*.n_gpu_layers =*/ -1,
/*.split_mode =*/ LLAMA_SPLIT_MODE_LAYER,
/*.load_mode =*/ LLAMA_LOAD_MODE_AUTO,
/*.tensor_read_lazy =*/ LLAMA_TENSOR_READ_LAZY_AUTO,
/*.lazy_mode =*/ LLAMA_LAZY_MODE_AUTO,
/*.main_gpu =*/ 0,
/*.tensor_split =*/ nullptr,
/*.progress_callback =*/ nullptr,
src/llama.cpp
+1 -1
View File
@@ -318,7 +318,7 @@ static std::pair<int, llama_model *> llama_model_load(struct gguf_context * meta
llama_model_loader ml(metadata, set_tensor_data, set_tensor_data_ud, fname, splits, file, params.load_mode,
params.check_tensors, params.no_alloc, params.load_mtp, params.kv_overrides, params.tensor_buft_overrides);
ml.tensor_read_lazy = params.tensor_read_lazy;
ml.lazy_mode = params.lazy_mode;
ml.print_info();
std::unique_ptr<llama_model> model_ptr(llama_model_create(ml, params));
tools/llama-bench/README.md
+1
View File
@@ -67,6 +67,7 @@ test parameters:
-nkvo, --no-kv-offload <0|1> (default: 0)
-fa, --flash-attn <on|off|auto> (default: auto)
-dev, --device <dev0/dev1/...> (default: auto)
--tensor-read-lazy <on|auto|off> (default: auto)
-mmp, --mmap <0|1> (DEPRECATED IN FAVOUR OF --load-mode)
-dio, --direct-io <0|1> (DEPRECATED IN FAVOUR OF --load-mode)
-embd, --embeddings <0|1> (default: 0)
tools/llama-bench/llama-bench.cpp
+62 -3
View File
@@ -271,6 +271,19 @@ static const char * split_mode_str(llama_split_mode mode) {
}
}
static const char * lazy_mode_str(llama_lazy_mode mode) {
switch (mode) {
case LLAMA_LAZY_MODE_OFF:
return "off";
case LLAMA_LAZY_MODE_AUTO:
return "auto";
case LLAMA_LAZY_MODE_ON:
return "on";
default:
GGML_ABORT("invalid tensor read lazy mode");
}
}
static std::string pair_str(const std::pair<int, int> & p) {
static char buf[32];
snprintf(buf, sizeof(buf), "%d,%d", p.first, p.second);
@@ -341,6 +354,7 @@ struct cmd_params {
std::vector<int> n_cpu_moe;
std::vector<llama_split_mode> split_mode;
std::vector<llama_load_mode> load_mode;
std::vector<llama_lazy_mode> lazy_mode;
std::vector<int> main_gpu;
std::vector<bool> no_kv_offload;
std::vector<llama_flash_attn_type> flash_attn;
@@ -385,6 +399,7 @@ static const cmd_params cmd_params_defaults = {
/* n_cpu_moe */ { 0 },
/* split_mode */ { LLAMA_SPLIT_MODE_LAYER },
/* load_mode */ { LLAMA_LOAD_MODE_AUTO },
/* lazy_mode */ { LLAMA_LAZY_MODE_AUTO },
/* main_gpu */ { 0 },
/* no_kv_offload */ { false },
/* flash_attn */ { LLAMA_FLASH_ATTN_TYPE_AUTO },
@@ -460,6 +475,7 @@ static void print_usage(int /* argc */, char ** argv) {
printf(" -fa, --flash-attn <on|off|auto> (default: %s)\n", join(transform_to_str(cmd_params_defaults.flash_attn, llama_flash_attn_type_name), ",").c_str());
printf(" -dev, --device <dev0/dev1/...> (default: auto)\n");
printf(" -lm, --load-mode <auto|none|mmap|mlock|mmap+mlock|dio> (default: %s)\n", join(transform_to_str(cmd_params_defaults.load_mode, llama_load_mode_name), ",").c_str());
printf(" --tensor-read-lazy <on|auto|off> (default: %s)\n", join(transform_to_str(cmd_params_defaults.lazy_mode, lazy_mode_str), ",").c_str());
printf(" -mmp, --mmap <0|1> (DEPRECATED IN FAVOUR OF --load-mode)\n");
printf(" -dio, --direct-io <0|1> (DEPRECATED IN FAVOUR OF --load-mode)\n");
printf(" -embd, --embeddings <0|1> (default: %s)\n", join(cmd_params_defaults.embeddings, ",").c_str());
@@ -786,6 +802,32 @@ static cmd_params parse_cmd_params(int argc, char ** argv) {
break;
}
params.load_mode.insert(params.load_mode.end(), modes.begin(), modes.end());
} else if (arg == "--tensor-read-lazy") {
if (++i >= argc) {
invalid_param = true;
break;
}
auto p = string_split<std::string>(argv[i], split_delim);
std::vector<llama_lazy_mode> modes;
for (const auto & m : p) {
llama_lazy_mode mode;
if (m == "on") {
mode = LLAMA_LAZY_MODE_ON;
} else if (m == "auto") {
mode = LLAMA_LAZY_MODE_AUTO;
} else if (m == "off") {
mode = LLAMA_LAZY_MODE_OFF;
} else {
invalid_param = true;
break;
}
modes.push_back(mode);
}
if (invalid_param) {
break;
}
params.lazy_mode.insert(params.lazy_mode.end(), modes.begin(), modes.end());
} else if (arg == "-mg" || arg == "--main-gpu") {
if (++i >= argc) {
invalid_param = true;
@@ -1137,6 +1179,9 @@ static cmd_params parse_cmd_params(int argc, char ** argv) {
if (params.load_mode.empty()) {
params.load_mode = cmd_params_defaults.load_mode;
}
if (params.lazy_mode.empty()) {
params.lazy_mode = cmd_params_defaults.lazy_mode;
}
if (params.main_gpu.empty()) {
params.main_gpu = cmd_params_defaults.main_gpu;
}
@@ -1203,6 +1248,7 @@ struct cmd_params_instance {
int n_cpu_moe;
llama_split_mode split_mode;
llama_load_mode load_mode;
llama_lazy_mode lazy_mode;
int main_gpu;
bool no_kv_offload;
llama_flash_attn_type flash_attn;
@@ -1224,6 +1270,7 @@ struct cmd_params_instance {
}
mparams.split_mode = split_mode;
mparams.load_mode = load_mode;
mparams.lazy_mode = lazy_mode;
mparams.main_gpu = main_gpu;
mparams.tensor_split = tensor_split.data();
mparams.no_host = no_host;
@@ -1271,7 +1318,8 @@ struct cmd_params_instance {
return model == other.model && n_gpu_layers == other.n_gpu_layers && n_cpu_moe == other.n_cpu_moe &&
split_mode == other.split_mode &&
main_gpu == other.main_gpu && tensor_split == other.tensor_split &&
load_mode == other.load_mode && devices == other.devices && no_host == other.no_host &&
load_mode == other.load_mode && lazy_mode == other.lazy_mode &&
devices == other.devices && no_host == other.no_host &&
vec_tensor_buft_override_equal(tensor_buft_overrides, other.tensor_buft_overrides);
}
@@ -1305,6 +1353,7 @@ static std::vector<cmd_params_instance> get_cmd_params_instances(const cmd_param
for (const auto & ncmoe : params.n_cpu_moe)
for (const auto & sm : params.split_mode)
for (const auto & lm : params.load_mode)
for (const auto & lzm : params.lazy_mode)
for (const auto & mg : params.main_gpu)
for (const auto & devs : params.devices)
for (const auto & ts : params.tensor_split)
@@ -1344,6 +1393,7 @@ static std::vector<cmd_params_instance> get_cmd_params_instances(const cmd_param
/* .n_cpu_moe = */ ncmoe,
/* .split_mode = */ sm,
/* .load_mode = */ lm,
/* .lazy_mode = */ lzm,
/* .main_gpu = */ mg,
/* .no_kv_offload = */ nkvo,
/* .flash_attn = */ fa,
@@ -1380,6 +1430,7 @@ static std::vector<cmd_params_instance> get_cmd_params_instances(const cmd_param
/* .n_cpu_moe = */ ncmoe,
/* .split_mode = */ sm,
/* .load_mode = */ lm,
/* .lazy_mode = */ lzm,
/* .main_gpu = */ mg,
/* .no_kv_offload = */ nkvo,
/* .flash_attn = */ fa,
@@ -1416,6 +1467,7 @@ static std::vector<cmd_params_instance> get_cmd_params_instances(const cmd_param
/* .n_cpu_moe = */ ncmoe,
/* .split_mode = */ sm,
/* .load_mode = */ lm,
/* .lazy_mode = */ lzm,
/* .main_gpu = */ mg,
/* .no_kv_offload = */ nkvo,
/* .flash_attn = */ fa,
@@ -1457,6 +1509,7 @@ struct test {
int n_cpu_moe;
llama_split_mode split_mode;
llama_load_mode load_mode;
llama_lazy_mode lazy_mode;
int main_gpu;
bool no_kv_offload;
llama_flash_attn_type flash_attn;
@@ -1496,6 +1549,7 @@ struct test {
n_cpu_moe = inst.n_cpu_moe;
split_mode = inst.split_mode;
load_mode = inst.load_mode;
lazy_mode = inst.lazy_mode;
main_gpu = inst.main_gpu;
no_kv_offload = inst.no_kv_offload;
flash_attn = inst.flash_attn;
@@ -1563,7 +1617,8 @@ struct test {
"n_ubatch", "n_threads", "cpu_mask", "cpu_strict", "poll",
"type_k", "type_v", "n_gpu_layers", "n_cpu_moe", "split_mode",
"main_gpu", "no_kv_offload", "flash_attn", "devices", "tensor_split",
"tensor_buft_overrides", "load_mode", "embeddings",
"tensor_buft_overrides", "load_mode", "lazy_mode",
"embeddings",
"no_op_offload", "no_host", "fit_target", "fit_min_ctx",
"n_prompt", "n_gen", "n_depth",
"test_time", "avg_ns", "stddev_ns", "avg_ts", "stddev_ts"
@@ -1588,7 +1643,7 @@ struct test {
if (field == "avg_ts" || field == "stddev_ts") {
return FLOAT;
}
if (field == "load_mode") {
if (field == "load_mode" || field == "lazy_mode") {
return STRING;
}
return STRING;
@@ -1658,6 +1713,7 @@ struct test {
tensor_split_str,
tensor_buft_overrides_str,
llama_load_mode_name(load_mode),
lazy_mode_str(lazy_mode),
std::to_string(embeddings),
std::to_string(no_op_offload),
std::to_string(no_host),
@@ -1972,6 +2028,9 @@ struct markdown_printer : public printer {
if (params.load_mode.size() > 1 || params.load_mode != cmd_params_defaults.load_mode) {
fields.emplace_back("load_mode");
}
if (params.lazy_mode.size() > 1 || params.lazy_mode != cmd_params_defaults.lazy_mode) {
fields.emplace_back("lazy_mode");
}
if (params.embeddings.size() > 1 || params.embeddings != cmd_params_defaults.embeddings) {
fields.emplace_back("embeddings");
}