llama: drop the model file's page cache after a copying load
With -lm none the GGUF is read() into our own buffers, but the kernel keeps the file in page cache afterwards, charged to the pod's cgroup: about 20 GB after a 35B load. On 2026-10-24 the Talos OOM controller killed the pod right after a load under exactly that pressure. Those pages are pure waste once the data is copied. llama_file::drop_cache() wraps posix_fadvise(POSIX_FADV_DONTNEED) (Linux only, errors logged at debug). load_all_data() drops each copied tensor's range right after reading it, so the cache never grows past about one tensor, and drops the whole file at the final cleanup to catch the metadata and partial pages at tensor edges. Both only when the loader holds no mapping of the files (mappings.empty(): not mmap, not mmap+mlock, no lazy tensors), so nothing that is mapped is ever advised away. Split GGUFs are covered (every file in files). clip.cpp always copies the mmproj via ifstream, so it drops the whole file after loading it. Measured on the pod, gemma-4-E2B UD-Q4_K_XL (2.5 GiB) + mmproj BF16 (0.9 GiB), CPU only, -lm none, cold cache, same binary with the drops disabled as the control (4 rounds, rotated order): - file pages resident after load (mincore): 2431-2444 / 940 MiB -> 0 / 0 - peak model-file residency during load: 2.5 GiB -> 1.2-1.5 GiB (largest tensor) - pod cgroup file after load: +3.37 GiB -> +0; host MemFree after load 0.6 GB -> 2.5-2.8 GB (MemAvailable counts clean cache, so it cannot show this) - load time: 5.87 s -> 6.82 s median (+0.95 s); a post-load-only drop is 6.56 s (+0.68 s). The cost is synchronous page freeing that reclaim would otherwise do later; -lm dio (direct IO) costs the same (+0.9 s) and does not cover metadata or the mmproj. - greedy output byte-identical; -lm mmap keeps the model file fully resident and mapped (2498 MiB, no refaults); -lzm on skips the drop (file mapped). Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
d7b3510499
commit
cce0ac1f09
4 files changed
+40
No files matched your search
@@ -422,6 +422,19 @@ size_t llama_file::size() const { return pimpl->size; }
|
||||
size_t llama_file::read_alignment() const { return pimpl->read_alignment(); }
|
||||
bool llama_file::has_direct_io() const { return pimpl->has_direct_io(); }
|
||||
|
||||
void llama_file::drop_cache(size_t offset, size_t len) const {
|
||||
#ifdef __linux__
|
||||
// only a hint: the pages are clean, a failure just leaves them cached
|
||||
const int err = posix_fadvise(file_id(), (off_t) offset, (off_t) len, POSIX_FADV_DONTNEED);
|
||||
if (err != 0) {
|
||||
LLAMA_LOG_DEBUG("%s: posix_fadvise(.., POSIX_FADV_DONTNEED) failed: %s\n", __func__, strerror(err));
|
||||
}
|
||||
#else
|
||||
GGML_UNUSED(offset);
|
||||
GGML_UNUSED(len);
|
||||
#endif
|
||||
}
|
||||
|
||||
int llama_file::file_id() const {
|
||||
#ifdef _WIN32
|
||||
return _fileno(pimpl->fp);
|
||||
|
||||
@@ -39,6 +39,9 @@ struct llama_file {
|
||||
|
||||
size_t read_alignment() const;
|
||||
bool has_direct_io() const;
|
||||
|
||||
// hint the kernel to drop the file's page cache in [offset, offset + len), len 0 = to the end (Linux only)
|
||||
void drop_cache(size_t offset = 0, size_t len = 0) const;
|
||||
private:
|
||||
struct impl;
|
||||
std::unique_ptr<impl> pimpl;
|
||||
|
||||
@@ -1817,6 +1817,11 @@ bool llama_model_loader::load_all_data(
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// the tensor is copied: drop its page cache as we go, unless the file is mapped (lazy tensors)
|
||||
if (mappings.empty()) {
|
||||
file->drop_cache(weight->offs, n_size);
|
||||
}
|
||||
}
|
||||
|
||||
size_done += n_size;
|
||||
@@ -1858,6 +1863,12 @@ bool llama_model_loader::load_all_data(
|
||||
}
|
||||
}
|
||||
}
|
||||
// copied files: also drop what the per-tensor drops leave (metadata, partial pages at tensor edges)
|
||||
if (mappings.empty()) {
|
||||
for (const auto & file : files) {
|
||||
file->drop_cache();
|
||||
}
|
||||
}
|
||||
if (progress_callback) {
|
||||
// Even though the model is done loading, we still honor
|
||||
// cancellation since we need to free allocations.
|
||||
|
||||
@@ -27,6 +27,11 @@
|
||||
#include <functional>
|
||||
#include <float.h>
|
||||
|
||||
#if defined(__linux__)
|
||||
#include <fcntl.h>
|
||||
#include <unistd.h>
|
||||
#endif
|
||||
|
||||
struct clip_logger_state g_logger_state = {clip_log_callback_default, NULL};
|
||||
|
||||
//#define CLIP_DEBUG_FUNCTIONS
|
||||
@@ -3994,6 +3999,14 @@ struct clip_model_loader {
|
||||
}
|
||||
}
|
||||
LOG_DBG("%s: loaded %zu tensors from %s\n", __func__, tensors_to_load.size(), fname.c_str());
|
||||
#if defined(__linux__)
|
||||
// the weights are copied into our buffer: drop the file's page cache (only a hint)
|
||||
const int fd = ::open(fname.c_str(), O_RDONLY);
|
||||
if (fd != -1) {
|
||||
posix_fadvise(fd, 0, 0, POSIX_FADV_DONTNEED);
|
||||
::close(fd);
|
||||
}
|
||||
#endif
|
||||
} else {
|
||||
LOG_DBG("%s: no_alloc is set, skipping tensor data loading (%zu tensors)\n", __func__, tensors_to_load.size());
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user