Lumpiasty/llama.cpp
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

llama: model_loader: add TENSOR_READ_LAZY (#27794)

Browse Source 
* llama: model_loader: add TENSOR_GET_ROW_LAZY

* add --tensor-read-lazy

* rename to TENSOR_READ_LAZY

* gen docs

* address comments
This commit is contained in:
Xuan-Son Nguyen
2026-08-27 15:14:34 +02:00
committed by GitHub
parent cae63579b6
commit fac889fb38
15 changed files with 129 additions and 19 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/llama-model-loader.cpp
+20 -2
View File
@@ -1282,6 +1282,18 @@ struct ggml_tensor * llama_model_loader::create_tensor(
return NULL;
}
if ((flags & TENSOR_READ_LAZY) && use_mmap && tensor_read_lazy != LLAMA_TENSOR_READ_LAZY_OFF) {
// in auto mode, small tensors are cheap enough to keep resident
constexpr size_t auto_lazy_min_size = 4ull * 1024 * 1024 * 1024;
if (tensor_read_lazy == LLAMA_TENSOR_READ_LAZY_ON || ggml_nbytes(cur) > auto_lazy_min_size) {
const auto & w = require_weight(tn.str().c_str());
lazy_tensor_ranges[w.idx].emplace_back(w.offs, w.offs + ggml_nbytes(cur));
LLAMA_LOG_INFO("%s: tensor %s (size = %zu MiB) lazy read enabled\n",
__func__, tn.str().c_str(), ggml_nbytes(cur)/1024/1024);
}
}
ggml_tensor t_meta = *cur;
if (flags & TENSOR_ALLOW_RESHAPE) {
for (size_t dim = 0; dim < GGML_MAX_DIMS; dim++) {
@@ -1349,7 +1361,9 @@ void llama_model_loader::init_mappings(bool prefetch, llama_mlocks * mlock_mmaps
if (use_mmap) {
mappings.reserve(files.size());
mmaps_used.reserve(files.size());
for (const auto & file : files) {
for (uint32_t idx = 0; idx < files.size(); idx++) {
const auto & file = files[idx];
bool is_numa = false;
auto * dev = ggml_backend_dev_by_type(GGML_BACKEND_DEVICE_TYPE_CPU);
@@ -1361,7 +1375,11 @@ void llama_model_loader::init_mappings(bool prefetch, llama_mlocks * mlock_mmaps
}
}
std::unique_ptr<llama_mmap> mapping = std::make_unique<llama_mmap>(file.get(), prefetch ? -1 : 0, is_numa);
const auto it_lazy = lazy_tensor_ranges.find(idx);
static const llama_mmap::ranges no_lazy_ranges;
std::unique_ptr<llama_mmap> mapping = std::make_unique<llama_mmap>(file.get(), prefetch ? -1 : 0, is_numa,
it_lazy != lazy_tensor_ranges.end() ? it_lazy->second : no_lazy_ranges);
mmaps_used.emplace_back(mapping->size(), 0);
if (mlock_mmaps) {
std::unique_ptr<llama_mlock> mlock_mmap(new llama_mlock());