diff --git a/src/llama-mmap.cpp b/src/llama-mmap.cpp index 4d1cd31e9..81eb5d573 100644 --- a/src/llama-mmap.cpp +++ b/src/llama-mmap.cpp @@ -422,6 +422,19 @@ size_t llama_file::size() const { return pimpl->size; } size_t llama_file::read_alignment() const { return pimpl->read_alignment(); } bool llama_file::has_direct_io() const { return pimpl->has_direct_io(); } +void llama_file::drop_cache(size_t offset, size_t len) const { +#ifdef __linux__ + // only a hint: the pages are clean, a failure just leaves them cached + const int err = posix_fadvise(file_id(), (off_t) offset, (off_t) len, POSIX_FADV_DONTNEED); + if (err != 0) { + LLAMA_LOG_DEBUG("%s: posix_fadvise(.., POSIX_FADV_DONTNEED) failed: %s\n", __func__, strerror(err)); + } +#else + GGML_UNUSED(offset); + GGML_UNUSED(len); +#endif +} + int llama_file::file_id() const { #ifdef _WIN32 return _fileno(pimpl->fp); diff --git a/src/llama-mmap.h b/src/llama-mmap.h index 6c6393d57..cc402f089 100644 --- a/src/llama-mmap.h +++ b/src/llama-mmap.h @@ -39,6 +39,9 @@ struct llama_file { size_t read_alignment() const; bool has_direct_io() const; + + // hint the kernel to drop the file's page cache in [offset, offset + len), len 0 = to the end (Linux only) + void drop_cache(size_t offset = 0, size_t len = 0) const; private: struct impl; std::unique_ptr pimpl; diff --git a/src/llama-model-loader.cpp b/src/llama-model-loader.cpp index 066e69e91..c6cdb5049 100644 --- a/src/llama-model-loader.cpp +++ b/src/llama-model-loader.cpp @@ -1817,6 +1817,11 @@ bool llama_model_loader::load_all_data( } } } + + // the tensor is copied: drop its page cache as we go, unless the file is mapped (lazy tensors) + if (mappings.empty()) { + file->drop_cache(weight->offs, n_size); + } } size_done += n_size; @@ -1858,6 +1863,12 @@ bool llama_model_loader::load_all_data( } } } + // copied files: also drop what the per-tensor drops leave (metadata, partial pages at tensor edges) + if (mappings.empty()) { + for (const auto & file : files) { + file->drop_cache(); + } + } if (progress_callback) { // Even though the model is done loading, we still honor // cancellation since we need to free allocations. diff --git a/tools/mtmd/clip.cpp b/tools/mtmd/clip.cpp index e4e04e008..2d07ea647 100644 --- a/tools/mtmd/clip.cpp +++ b/tools/mtmd/clip.cpp @@ -27,6 +27,11 @@ #include #include +#if defined(__linux__) +#include +#include +#endif + struct clip_logger_state g_logger_state = {clip_log_callback_default, NULL}; //#define CLIP_DEBUG_FUNCTIONS @@ -3994,6 +3999,14 @@ struct clip_model_loader { } } LOG_DBG("%s: loaded %zu tensors from %s\n", __func__, tensors_to_load.size(), fname.c_str()); +#if defined(__linux__) + // the weights are copied into our buffer: drop the file's page cache (only a hint) + const int fd = ::open(fname.c_str(), O_RDONLY); + if (fd != -1) { + posix_fadvise(fd, 0, 0, POSIX_FADV_DONTNEED); + ::close(fd); + } +#endif } else { LOG_DBG("%s: no_alloc is set, skipping tensor data loading (%zu tensors)\n", __func__, tensors_to_load.size()); }