Lumpiasty/llama.cpp
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

llama: improve TENSOR_READ_LAZY handling (#27837)

Browse Source 
* force lazy tensor on cpu if lazy is on

* llama: improve TENSOR_READ_LAZY handling
This commit is contained in:
Xuan-Son Nguyen
2026-08-30 16:59:48 +02:00
committed by GitHub
parent f1793c1c4e
commit 2578138397
4 changed files with 118 additions and 28 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/llama-model.cpp
+7 -2
View File
@@ -1689,7 +1689,8 @@ bool llama_model_base::load_tensors(llama_model_loader & ml) {
const size_t n_max_backend_buffer = ml.ctx_map.size() * ml.files.size();
pimpl->ctxs_bufs.reserve(n_max_backend_buffer);
for (auto & [buft, ctx_ptr] : ml.ctx_map) {
for (auto & [ctx_key, ctx_ptr] : ml.ctx_map) {
ggml_backend_buffer_type_t buft = ctx_key.buft;
ggml_context * ctx = ctx_ptr.get();
// skip contexts without tensors
@@ -1715,7 +1716,11 @@ bool llama_model_base::load_tensors(llama_model_loader & ml) {
bool is_default_buft = buft == ggml_backend_dev_buffer_type(dev);
std::vector<ggml_backend_buffer_ptr> bufs;
if (ml.use_mmap && use_mmap_buffer && buffer_from_host_ptr_supported && is_default_buft) {
// a lazy context is mapped whatever the load mode, but the memory-fit pass maps nothing
const bool is_lazy_mapped = ctx_key.lazy && !ml.no_alloc;
if ((ml.use_mmap || is_lazy_mapped) && use_mmap_buffer && buffer_from_host_ptr_supported && is_default_buft) {
GGML_ASSERT(!ml.no_alloc);
for (uint32_t idx = 0; idx < ml.files.size(); idx++) {
// only the mmap region containing the tensors in the model is mapped to the backend buffer