llama : do not map foreign ranges into a backend buffer (#30405)

* llama : do not map foreign ranges into a backend buffer

A buffer created from a host pointer pins everything that it wraps. When
several contexts share the same file (lazy tensors, MoE tensors kept on the
CPU with -cmoe), the single buffer that spans the first and the last tensor
of a context also wraps the ranges owned by the other contexts, pinning data
that the backend never reads.

For models with per-layer token embeddings this pins the whole lazy tensor
into the GPU buffer (27 GiB for Qwen3.8-Flash-Next IQ4_NL), defeating the
point of reading it lazily.

Instead, compute the file ranges of each context and subtract the ranges
owned by the other contexts, then create one buffer per range. Weights that
are mapped in more than one context (e.g. token_embd.weight) keep being part
of the range of every context that uses them.

Assisted-by: pi:llama.cpp/Qwen3.8-Flash-Next

* llama : enable prefetch on macOS

The prefetch hint of the tensors marked as prefetchable is a no-op on macOS,
so their pages are faulted in one by one while the graph runs. madvise
(MADV_WILLNEED) behaves as expected here, so add the platform to the guard.

With the lazy per-layer token embeddings of Qwen3.8-Flash-Next IQ4_NL read
from a cold file cache, prompt processing goes from 356 / 375 / 475 t/s to
735 / 749 / 788 t/s for pp512 / pp2048 / pp4096.

Assisted-by: pi:llama.cpp/Qwen3.8-Flash-Next

* llama : use a plain buffer list per context

Assisted-by: pi:llama.cpp/Qwen3.8-Flash-Next
This commit is contained in:
Georgi Gerganov authored and GitHub committed 2026-10-13 17:28:15 +03:00
1 parent f441d2d9db
commit b9f3599fb1
4 files changed
+127 -53

No files matched your search

+81 -15
View File
@@ -13,6 +13,7 @@
#include <cstring>
#include <future>
#include <regex>
#include <unordered_set>
static const size_t kiB = 1024;
static const size_t MiB = 1024*kiB;
@@ -1446,21 +1447,80 @@ void llama_model_loader::init_mappings(bool prefetch, llama_mlocks * mlock_mmaps
}
}
void llama_model_loader::get_mapping_range(size_t * first, size_t * last, void ** addr, int idx, ggml_context * ctx) const {
GGML_ASSERT(!mappings.empty());
const auto & mapping = mappings.at(idx);
using mapping_ranges = std::vector<llama_model_loader::mapping_range>;
// merge a list of [first, last) ranges into a sorted, non-overlapping list
static mapping_ranges merge_ranges(mapping_ranges ranges) {
std::sort(ranges.begin(), ranges.end(), [](const auto & a, const auto & b) { return a.first < b.first; });
mapping_ranges res;
for (const auto & range : ranges) {
if (!res.empty() && range.first <= res.back().last) {
res.back().last = std::max(res.back().last, range.last);
} else {
res.push_back(range);
}
}
return res;
}
// subtract the sorted, non-overlapping ranges 'sub' from 'a'
static mapping_ranges sub_ranges(const mapping_ranges & a, const mapping_ranges & sub) {
mapping_ranges res;
for (const auto & x : a) {
size_t cur = x.first;
for (const auto & y : sub) {
if (y.last <= cur || y.first >= x.last) {
continue;
}
if (y.first > cur) {
res.push_back({ cur, y.first });
}
cur = std::max(cur, y.last);
}
if (cur < x.last) {
res.push_back({ cur, x.last });
}
}
return res;
}
std::vector<llama_model_loader::mapping_range> llama_model_loader::get_mapping_ranges(uint32_t idx, ggml_context * ctx) const {
GGML_ASSERT(idx < mappings.size());
// data of this context, and data of the other contexts that share the same file
// a weight that is mapped in several contexts (e.g. the token embedding) is not foreign data
std::unordered_set<const llama_tensor_weight *> own;
mapping_ranges own_ranges;
mapping_ranges other;
*first = mapping->size();
*last = 0;
*addr = mapping->addr();
for (ggml_tensor * tensor = ggml_get_first_tensor(ctx); tensor; tensor = ggml_get_next_tensor(ctx, tensor)) {
const auto * weight = get_weight(ggml_get_name(tensor));
if (!weight || weight->idx != idx) {
continue;
}
*first = std::min(*first, weight->offs);
*last = std::max(*last, weight->offs + ggml_nbytes(tensor));
own.insert(weight);
own_ranges.push_back({ weight->offs, weight->offs + ggml_nbytes(tensor) });
}
for (const auto & it : ctx_map) {
if (!it.second || it.second.get() == ctx) {
continue;
}
for (ggml_tensor * tensor = ggml_get_first_tensor(it.second.get()); tensor; tensor = ggml_get_next_tensor(it.second.get(), tensor)) {
const auto * weight = get_weight(ggml_get_name(tensor));
if (!weight || weight->idx != idx || own.count(weight)) {
continue;
}
other.push_back({ weight->offs, weight->offs + ggml_nbytes(tensor) });
}
}
return sub_ranges(merge_ranges(std::move(own_ranges)), merge_ranges(std::move(other)));
}
void llama_model_loader::unmap_weight(const llama_tensor_weight & w) const {
@@ -1492,7 +1552,7 @@ const void * llama_model_loader::load_data_range(const llama_tensor_weight & w,
bool llama_model_loader::load_all_data(
struct ggml_context * ctx,
llama_buf_map & bufs,
std::vector<ggml_backend_buffer_t> & bufs,
llama_mlocks * lmlocks,
llama_progress_callback progress_callback,
void * progress_callback_user_data) {
@@ -1528,7 +1588,7 @@ bool llama_model_loader::load_all_data(
}
// When not using mmaped io use async uploads from pinned memory to GPU memory.
// First determine if the backend supports the necessary features for async uploads.
auto * buf = bufs.count(0) ? bufs.at(0) : nullptr;
auto * buf = bufs.empty() ? nullptr : bufs[0];
if (!buf) {
LLAMA_LOG_DEBUG("%s: no buffer found for async uploads\n", func);
return nullptr;
@@ -1599,7 +1659,7 @@ bool llama_model_loader::load_all_data(
if (upload_backend) {
LLAMA_LOG_DEBUG("%s: using async uploads for device %s, buffer type %s, backend %s\n", __func__,
ggml_backend_dev_name(ggml_backend_get_device(upload_backend)),
ggml_backend_buft_name(ggml_backend_buffer_get_type(bufs.at(0))),
ggml_backend_buft_name(ggml_backend_buffer_get_type(bufs[0])),
ggml_backend_name(upload_backend));
}
@@ -1640,12 +1700,18 @@ bool llama_model_loader::load_all_data(
if (from_mapping) {
const auto & mapping = mappings.at(weight->idx);
ggml_backend_buffer_t buf_mmap = nullptr;
if (bufs.count(weight->idx)) {
buf_mmap = bufs.at(weight->idx);
}
uint8_t * data = (uint8_t *) mapping->addr() + weight->offs;
// the tensors of a file can be spread across several buffers - find the one containing this tensor's data
ggml_backend_buffer_t buf_mmap = nullptr;
for (const auto & buf : bufs) {
const uint8_t * base = (const uint8_t *) ggml_backend_buffer_get_base(buf);
if (base && base <= data && data + n_size <= base + ggml_backend_buffer_get_size(buf)) {
buf_mmap = buf;
break;
}
}
if (check_tensors) {
validation_result.emplace_back(std::async(std::launch::async, [cur, data, n_size] {
return std::make_pair(cur, ggml_validate_row_data(cur->type, data, n_size));