llama : pin mmap-backed CPU weights for faster H2D uploads
Wire the existing GGML_CUDA_REGISTER_HOST path back up: after model load, cudaHostRegister the mmap pages backing weights kept in system memory. Recovers pageable-copy losses when MoE experts are streamed to the GPU during prefill (n-cpu-moe): Qwen3.6-35B-A3B pp2048 1144 -> 1385 t/s on RTX 3060. Opt-in via GGML_CUDA_REGISTER_HOST=1, unchanged otherwise.
This commit is contained in:
@@ -50,11 +50,19 @@ struct llama_mmap {
|
||||
|
||||
void unmap_fragment(size_t first, size_t last);
|
||||
|
||||
// pin the pages backing [first, last) with a backend allocator for faster H2D copies,
|
||||
// unpinned in the destructor before the pages are unmapped
|
||||
// returns the number of bytes registered, 0 on failure
|
||||
size_t register_host(size_t first, size_t last, bool (*reg_fn)(void *, size_t), void (*unreg_fn)(void *));
|
||||
|
||||
static const bool SUPPORTED;
|
||||
|
||||
private:
|
||||
struct impl;
|
||||
std::unique_ptr<impl> pimpl;
|
||||
|
||||
void * host_reg_addr = nullptr;
|
||||
void (*host_unreg_fn)(void *) = nullptr;
|
||||
};
|
||||
|
||||
struct llama_mlock {
|
||||
|
||||
Reference in New Issue
Block a user