Lumpiasty/llama.cpp
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

llama: model_loader: add TENSOR_READ_LAZY (#27794)

Browse Source 
* llama: model_loader: add TENSOR_GET_ROW_LAZY

* add --tensor-read-lazy

* rename to TENSOR_READ_LAZY

* gen docs

* address comments
This commit is contained in:
Xuan-Son Nguyen
2026-08-27 15:14:34 +02:00
committed by GitHub
parent cae63579b6
commit fac889fb38
15 changed files with 129 additions and 19 deletions
Download Patch File Download Diff File Expand all files Collapse all files
common/arg.cpp
+13
View File
@@ -2720,6 +2720,19 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
else { throw std::invalid_argument("invalid value"); }
}
).set_env("LLAMA_ARG_LOAD_MODE"));
add_opt(common_arg(
{"--tensor-read-lazy"}, "MODE",
"on-demand reading of certain tensors, for example per-layer embeddings (default: auto)\n"
"- on: read the rows of such tensors from disk on demand instead of keeping them resident (requires mmap)\n"
"- auto: on, but only for tensors larger than 4 GiB\n"
"- off: always keep them resident",
[](common_params & params, const std::string & value) {
/**/ if (value == "on") { params.tensor_read_lazy = LLAMA_TENSOR_READ_LAZY_ON; }
else if (value == "auto") { params.tensor_read_lazy = LLAMA_TENSOR_READ_LAZY_AUTO; }
else if (value == "off") { params.tensor_read_lazy = LLAMA_TENSOR_READ_LAZY_OFF; }
else { throw std::invalid_argument("invalid value"); }
}
).set_env("LLAMA_ARG_TENSOR_READ_LAZY"));
add_opt(common_arg(
{"--numa"}, "TYPE",
"attempt optimizations that help on some NUMA systems\n"