thecodacus
20f5994bfe
Wire the existing GGML_CUDA_REGISTER_HOST path back up: after model load, cudaHostRegister the mmap pages backing weights kept in system memory. Recovers pageable-copy losses when MoE experts are streamed to the GPU during prefill (n-cpu-moe): Qwen3.6-35B-A3B pp2048 1144 -> 1385 t/s on RTX 3060. Opt-in via GGML_CUDA_REGISTER_HOST=1, unchanged otherwise.
83 lines
2.0 KiB
C++
83 lines
2.0 KiB
C++
#pragma once
|
|
|
|
#include <cstdint>
|
|
#include <memory>
|
|
#include <vector>
|
|
#include <cstdio>
|
|
|
|
struct llama_file;
|
|
struct llama_mmap;
|
|
struct llama_mlock;
|
|
|
|
using llama_files = std::vector<std::unique_ptr<llama_file>>;
|
|
using llama_mmaps = std::vector<std::unique_ptr<llama_mmap>>;
|
|
using llama_mlocks = std::vector<std::unique_ptr<llama_mlock>>;
|
|
|
|
struct llama_file {
|
|
llama_file(const char * fname, const char * mode, bool use_direct_io = false);
|
|
llama_file(FILE * file);
|
|
~llama_file();
|
|
|
|
size_t tell() const;
|
|
size_t size() const;
|
|
|
|
int file_id() const; // fileno overload
|
|
|
|
void seek(size_t offset, int whence) const;
|
|
|
|
void read_raw(void * ptr, size_t len);
|
|
void read_raw_unsafe(void * ptr, size_t len);
|
|
void read_aligned_chunk(void * dest, size_t size);
|
|
uint32_t read_u32();
|
|
|
|
void write_raw(const void * ptr, size_t len) const;
|
|
void write_u32(uint32_t val) const;
|
|
|
|
size_t read_alignment() const;
|
|
bool has_direct_io() const;
|
|
private:
|
|
struct impl;
|
|
std::unique_ptr<impl> pimpl;
|
|
};
|
|
|
|
struct llama_mmap {
|
|
llama_mmap(const llama_mmap &) = delete;
|
|
llama_mmap(struct llama_file * file, size_t prefetch = (size_t) -1, bool numa = false);
|
|
~llama_mmap();
|
|
|
|
size_t size() const;
|
|
void * addr() const;
|
|
|
|
void unmap_fragment(size_t first, size_t last);
|
|
|
|
// pin the pages backing [first, last) with a backend allocator for faster H2D copies,
|
|
// unpinned in the destructor before the pages are unmapped
|
|
// returns the number of bytes registered, 0 on failure
|
|
size_t register_host(size_t first, size_t last, bool (*reg_fn)(void *, size_t), void (*unreg_fn)(void *));
|
|
|
|
static const bool SUPPORTED;
|
|
|
|
private:
|
|
struct impl;
|
|
std::unique_ptr<impl> pimpl;
|
|
|
|
void * host_reg_addr = nullptr;
|
|
void (*host_unreg_fn)(void *) = nullptr;
|
|
};
|
|
|
|
struct llama_mlock {
|
|
llama_mlock();
|
|
~llama_mlock();
|
|
|
|
void init(void * ptr);
|
|
void grow_to(size_t target_size);
|
|
|
|
static const bool SUPPORTED;
|
|
|
|
private:
|
|
struct impl;
|
|
std::unique_ptr<impl> pimpl;
|
|
};
|
|
|
|
size_t llama_path_max();
|