llama : pin mmap-backed CPU weights for faster H2D uploads

Wire the existing GGML_CUDA_REGISTER_HOST path back up: after model load,
cudaHostRegister the mmap pages backing weights kept in system memory.
Recovers pageable-copy losses when MoE experts are streamed to the GPU
during prefill (n-cpu-moe): Qwen3.6-35B-A3B pp2048 1144 -> 1385 t/s on
RTX 3060. Opt-in via GGML_CUDA_REGISTER_HOST=1, unchanged otherwise.
This commit is contained in:
thecodacus
2026-07-02 09:44:53 +00:00
parent 4fc4ec5541
commit 20f5994bfe
3 changed files with 59 additions and 1 deletions
+35 -1
View File
@@ -618,13 +618,47 @@ struct llama_mmap::impl {
};
llama_mmap::llama_mmap(struct llama_file * file, size_t prefetch, bool numa) : pimpl(std::make_unique<impl>(file, prefetch, numa)) {}
llama_mmap::~llama_mmap() = default;
llama_mmap::~llama_mmap() {
// unpin before the pages are unmapped by the impl destructor
if (host_reg_addr && host_unreg_fn) {
host_unreg_fn(host_reg_addr);
}
}
size_t llama_mmap::size() const { return pimpl->size; }
void * llama_mmap::addr() const { return pimpl->addr; }
void llama_mmap::unmap_fragment(size_t first, size_t last) { pimpl->unmap_fragment(first, last); }
size_t llama_mmap::register_host(size_t first, size_t last, bool (*reg_fn)(void *, size_t), void (*unreg_fn)(void *)) {
#ifdef _POSIX_MAPPED_FILES
if (host_reg_addr || !reg_fn || !unreg_fn || last <= first) {
return 0;
}
// expand outward to the page boundaries retained by unmap_fragment
const size_t page_size = sysconf(_SC_PAGESIZE);
first = first & ~(page_size - 1);
last = (last + page_size - 1) & ~(page_size - 1);
void * reg_addr = (uint8_t *) pimpl->addr + first;
if (!reg_fn(reg_addr, last - first)) {
return 0;
}
host_reg_addr = reg_addr;
host_unreg_fn = unreg_fn;
return last - first;
#else
GGML_UNUSED(first);
GGML_UNUSED(last);
GGML_UNUSED(reg_fn);
GGML_UNUSED(unreg_fn);
return 0;
#endif
}
#if defined(_POSIX_MEMLOCK_RANGE) || defined(_WIN32)
const bool llama_mmap::SUPPORTED = true;
#else