Lumpiasty/llama.cpp
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

llama: model_loader: add TENSOR_READ_LAZY (#27794)

Browse Source 
* llama: model_loader: add TENSOR_GET_ROW_LAZY

* add --tensor-read-lazy

* rename to TENSOR_READ_LAZY

* gen docs

* address comments
This commit is contained in:
Xuan-Son Nguyen
2026-08-27 15:14:34 +02:00
committed by GitHub
parent cae63579b6
commit fac889fb38
15 changed files with 129 additions and 19 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/llama-model-loader.h
+7
View File
@@ -68,6 +68,7 @@ struct llama_model_loader {
static const int TENSOR_SKIP = 1 << 2;
static const int TENSOR_SKIP_IF_VIRTUAL = 1 << 3;
static const int TENSOR_ALLOW_RESHAPE = 1 << 4;
static const int TENSOR_READ_LAZY = 1 << 5; // read rows on demand instead of loading whole tensor; requires mmap for now
int n_kv = 0;
int n_tensors = 0;
@@ -82,12 +83,18 @@ struct llama_model_loader {
bool no_alloc;
bool load_mtp;
// set by the caller before the create_tensor() calls
enum llama_tensor_read_lazy tensor_read_lazy = LLAMA_TENSOR_READ_LAZY_OFF;
llama_files files;
llama_ftype ftype;
llama_fver fver;
llama_mmaps mappings;
// byte ranges of TENSOR_READ_LAZY tensors, per file index
std::map<uint32_t, llama_mmap::ranges> lazy_tensor_ranges;
std::map<std::string, llama_tensor_weight, weight_name_comparer> weights_map;
std::unordered_map<std::string, llama_model_kv_override> kv_overrides;
const llama_model_tensor_buft_override * tensor_buft_overrides;