* moe-cache : compute part of the misses on the CPU --moe-cache-cpu-share F computes a share F of the MoE cache misses on the CPU from the host experts instead of uploading them. The graph builds the expert FFN twice: on the device from the cache slots, where the misses left to the CPU read zero slots, and on the CPU from the host experts, where the experts computed on the device have id -1. The two results are added before the down bias. - ggml-cpu: MUL_MAT_ID skips negative ids and writes zero rows - sched: the copy callback is called for all the inputs of a split, and before the host sync when there are no events - the slot lookup callback reads the inputs of the CPU part together with the selected experts, so the CPU part does not wait for the device, and its result is copied back asynchronously - each token uses distinct zero slots, mul_mat_id does not support an expert selected twice by a token - LLAMA_MOE_CACHE_TRACE writes the selected experts of each lookup, for a simulator of the cache * scripts : add moe-cache-tune.py to pick the MoE cache arguments The script wraps a llama-server command: it starts the server at two cache sizes and three CPU shares, fits the upload and CPU speeds of the machine, and replays the routing of the runs through a copy of the cache logic to predict other sizes and shares. It prints the recommended --moe-cache-mib and --moe-cache-cpu-share. docs/moe-cache.md describes the MoE cache arguments and the script. * add am17an as codeowner for MoE cache * moe-cache : upload all the misses until the cache is full With a high --moe-cache-cpu-share, few misses are uploaded, so an empty cache fills slowly and more misses than needed go to the CPU. The CPU share now starts once all the slots of a cache group hold an expert. moe-cache-tune.py replays the same rule. * add backend sync for async transfer queue * moe-cache : let the scheduler copy the results of the CPU part * Fix MSVC compilation * remove sync change * llama-graph : extract the CPU share condition into can_cpu_share * llama-graph : make llm_graph_params::moe_cache const The graph construction no longer updates the MoE cache directly: the CPU parts of the layers are accumulated in the graph result and applied to the cache in llama_context::process_ubatch, after the graph inputs are set and before the graph is computed. Assisted-by: dsh:llama.cpp/ggml-org/GLM-5.3-Flash-GGUF (Mac Studio, M5 Ultra, 256GB) * scripts : replay the ClassLRU policy in moe-cache-tune.py * moe-cache : pick CPU misses by CRF instead of random (#31853) * moe-cache : remove the backend logic of the CPU part from the graph The CPU part reads the host experts, so the scheduler places it on the CPU without set_tensor_backend. The cache gives no CPU map for batches that a device would offload. Graphs with a CPU part look up their own slot map, so the graph does not register the part with the cache. ggml-backend : the inputs of the next split that are already computed are copied to host memory before the current split runs, so a CPU split does not wait for the device split before it. This replaces the stash of the cache, and the copy callback is again called for host weights only. * llama-graph : return the MoE cache slots and CPU ids in a struct The scheduler prefetch no longer checks that the next split runs on another backend: the inputs of a split are never on its own backend, so the input loop already skips that case. --------- Co-authored-by: Gaurav Garg <gaugarg@nvidia.com> Co-authored-by: Georgi Gerganov <ggerganov@gmail.com> Co-authored-by: Alexey Kopytko <alexey@kopytko.com>
917 lines
38 KiB
C++
917 lines
38 KiB
C++
#include "llama-moe-cache.h"
|
|
|
|
#include "llama-impl.h"
|
|
#include "llama-memory.h"
|
|
#include "llama-model.h"
|
|
|
|
#include "ggml-cpp.h"
|
|
|
|
#include <algorithm>
|
|
#include <cmath>
|
|
#include <cstdio>
|
|
#include <cstdlib>
|
|
#include <cstring>
|
|
#include <stdexcept>
|
|
#include <unordered_map>
|
|
#include <vector>
|
|
|
|
namespace {
|
|
|
|
// ClassLRU: one LRU list per frequency class, over the experts of a group of layers
|
|
// slot_map[il][e] is the slot of expert e of layer il, n_slots or later if not cached, the host slot map tensor
|
|
// CRF (combined recency and frequency) = uses of an expert, decayed with a half-life of N tokens, retained across evictions
|
|
// class = floor(log2(CRF)), at most n_class - 1; each slot is in the LRU list of the class, as of its last use
|
|
// victim: an empty slot, or the least recently used slot of each list not in the current plan() with the max age/2^class
|
|
struct moe_cache_lru {
|
|
static constexpr int n_class = 8; // floor(log2(1/(1 - 2^(-1/crf_half_life)))) = 7, class of CRF_max for half-life of 128
|
|
static constexpr float crf_half_life = 128.0f; // tokens
|
|
|
|
int32_t n_expert = 0;
|
|
int32_t n_slots = 0;
|
|
int32_t empty = 0; // slot of the experts that are not cached
|
|
|
|
// [n_layer] data of the slot maps, the experts at slot n_slots or later are not cached
|
|
// the experts computed on the CPU read zero slots after the cached ones on the device
|
|
std::vector<int32_t *> slot_map;
|
|
std::vector<int32_t> key_of; // [n_slots] key of the expert in the slot, -1 if empty
|
|
|
|
// one LRU list per class, the head is the least recently used end
|
|
std::vector<int32_t> prev; // [n_slots] -1 at the ends
|
|
std::vector<int32_t> next; // [n_slots]
|
|
int32_t head[n_class];
|
|
int32_t tail[n_class];
|
|
std::vector<int8_t> list_of; // [n_slots] class list of the slot
|
|
int32_t n_filled = 0; // slots that hold an expert
|
|
|
|
std::vector<float> crf; // [n_layer*n_expert]
|
|
std::vector<int64_t> last_use; // [n_layer*n_expert] layer clock at the last use, -1 if not used
|
|
std::vector<int64_t> clock; // [n_layer] tokens seen by the layer (as added by plan() for each ubatch)
|
|
|
|
std::vector<uint32_t> seen; // [n_expert] seen[e] == seen_gen if expert e is selected by the current plan()
|
|
uint32_t seen_gen = 0;
|
|
std::vector<int32_t> uniq; // the distinct selected experts
|
|
std::vector<int32_t> misses;
|
|
std::vector<uint64_t> token_zero; // [n_tokens] zero slots used by each token
|
|
|
|
void init(int32_t n_layer, int32_t n_expert, int32_t n_slots) {
|
|
GGML_ASSERT(n_class == (int) std::log2(1.0f/(1.0f - std::exp2(-1.0f/crf_half_life))) + 1);
|
|
this->n_expert = n_expert;
|
|
this->n_slots = n_slots;
|
|
this->empty = n_slots;
|
|
slot_map.assign(n_layer, nullptr);
|
|
key_of.assign(n_slots, -1);
|
|
prev.resize(n_slots);
|
|
next.resize(n_slots);
|
|
for (int32_t s = 0; s < n_slots; ++s) {
|
|
prev[s] = s - 1;
|
|
next[s] = s + 1 < n_slots ? s + 1 : -1;
|
|
}
|
|
for (int c = 0; c < n_class; ++c) {
|
|
head[c] = -1;
|
|
tail[c] = -1;
|
|
}
|
|
head[0] = 0;
|
|
tail[0] = n_slots - 1;
|
|
list_of.assign(n_slots, 0);
|
|
crf.assign((size_t) n_layer*n_expert, 0.0f);
|
|
last_use.assign((size_t) n_layer*n_expert, -1);
|
|
clock.assign(n_layer, 0);
|
|
seen.assign(n_expert, 0);
|
|
}
|
|
|
|
// move slot s to the most recently used end of list c
|
|
void move(int32_t s, int c) {
|
|
const int from = list_of[s];
|
|
if (from == c && s == tail[from]) {
|
|
return;
|
|
}
|
|
if (prev[s] >= 0) {
|
|
next[prev[s]] = next[s];
|
|
} else {
|
|
head[from] = next[s];
|
|
}
|
|
if (next[s] >= 0) {
|
|
prev[next[s]] = prev[s];
|
|
} else {
|
|
tail[from] = prev[s];
|
|
}
|
|
|
|
prev[s] = tail[c];
|
|
next[s] = -1;
|
|
if (tail[c] >= 0) {
|
|
next[tail[c]] = s;
|
|
} else {
|
|
head[c] = s;
|
|
}
|
|
tail[c] = s;
|
|
list_of[s] = c;
|
|
}
|
|
|
|
// key = index of an expert of a layer in crf and last_use
|
|
int32_t make_key(int32_t il, int32_t e) const { return il*n_expert + e; }
|
|
int32_t key_layer (int32_t key) const { return key / n_expert; }
|
|
int32_t key_expert(int32_t key) const { return key % n_expert; }
|
|
|
|
// for a key after use(): its CRF is >= 1, so log2 is >= 0 and the cast is the floor
|
|
int class_of(int32_t key) const {
|
|
return std::min(n_class - 1, (int) std::log2(crf[key]));
|
|
}
|
|
|
|
// the CRF of a key, decayed by the time since its last use
|
|
float decayed_crf(int32_t il, int32_t key) const {
|
|
return crf[key]*std::exp2(-(clock[il] - last_use[key])/crf_half_life);
|
|
}
|
|
|
|
// an unused key gets 1 because crf[key] is 0 prior to first use
|
|
void use(int32_t il, int32_t key) {
|
|
crf[key] = decayed_crf(il, key) + 1.0f;
|
|
last_use[key] = clock[il];
|
|
}
|
|
|
|
// the experts selected by this plan() must not be evicted, nor the slots this plan() filled already
|
|
bool is_selected(int32_t il, int32_t key) const {
|
|
if (key < 0) {
|
|
return false; // unused slot
|
|
}
|
|
if (key_layer(key) != il) {
|
|
return false; // seen is per expert index, an expert of another layer can have the same index
|
|
}
|
|
return seen[key_expert(key)] == seen_gen;
|
|
}
|
|
|
|
// the least recently used slot of list c that this plan() does not select, -1 if none
|
|
int32_t candidate(int c, int32_t il) const {
|
|
for (int32_t s = head[c]; s >= 0; s = next[s]) {
|
|
if (!is_selected(il, key_of[s])) {
|
|
return s;
|
|
}
|
|
}
|
|
return -1;
|
|
}
|
|
|
|
int32_t victim(int32_t il) const {
|
|
int32_t res = -1;
|
|
double res_score = -1.0;
|
|
for (int c = 0; c < n_class; ++c) {
|
|
const int32_t s = candidate(c, il);
|
|
if (s < 0) {
|
|
continue;
|
|
}
|
|
const int32_t key = key_of[s];
|
|
if (key < 0) {
|
|
return s; // unused slot
|
|
}
|
|
const int64_t age = clock[key_layer(key)] - last_use[key];
|
|
const double score = std::ldexp((double) age, -c); // age/2^class
|
|
if (score > res_score) {
|
|
res = s;
|
|
res_score = score;
|
|
}
|
|
}
|
|
if (res < 0) {
|
|
GGML_ABORT("no MoE cache victim");
|
|
}
|
|
return res;
|
|
}
|
|
|
|
struct fill {
|
|
int32_t expert;
|
|
int32_t slot;
|
|
};
|
|
|
|
// give a slot to each expert selected by ids (n_tokens tokens) in layer il
|
|
// n_cpu(n) of the n misses are left to the CPU and read a zero slot on the device, they are not cached
|
|
// a zero slot is used at most once per token, a miss that does not get one is cached instead
|
|
// returns false if the ids select more distinct experts than there are slots
|
|
template <typename F>
|
|
bool plan(int32_t il, const int32_t * ids, size_t n_ids, int64_t n_tokens, int32_t n_used, int32_t n_zero, F && n_cpu,
|
|
std::vector<fill> & fills, std::vector<fill> & cpu, size_t & n_hit) {
|
|
fills.clear();
|
|
cpu.clear();
|
|
n_hit = 0;
|
|
|
|
if (++seen_gen == 0) {
|
|
std::fill(seen.begin(), seen.end(), 0);
|
|
seen_gen = 1;
|
|
}
|
|
uniq.clear();
|
|
for (size_t i = 0; i < n_ids; ++i) {
|
|
GGML_ASSERT(ids[i] >= 0 && ids[i] < n_expert);
|
|
if (seen[ids[i]] != seen_gen) {
|
|
seen[ids[i]] = seen_gen;
|
|
uniq.push_back(ids[i]);
|
|
}
|
|
}
|
|
if (uniq.size() > (size_t) n_slots) {
|
|
return false;
|
|
}
|
|
|
|
int32_t * slots = slot_map[il];
|
|
|
|
// update the CRF of all selected experts and promote hits before choosing victims for misses
|
|
misses.clear();
|
|
for (int32_t e : uniq) {
|
|
const int32_t key = make_key(il, e);
|
|
use(il, key);
|
|
if (slots[e] < n_slots) {
|
|
move(slots[e], class_of(key));
|
|
n_hit++;
|
|
} else {
|
|
misses.push_back(e);
|
|
}
|
|
}
|
|
|
|
// the misses with the lowest CRF are left to the CPU, a miss that keeps being used gains CRF and enters the cache
|
|
// all misses are used at the same clock, so their CRF compare without decay
|
|
const size_t n_c = std::min(n_cpu(misses.size()), misses.size());
|
|
if (n_c > 0) {
|
|
std::sort(misses.begin(), misses.end(), [&](int32_t a, int32_t b) {
|
|
const float crf_a = crf[make_key(il, a)];
|
|
const float crf_b = crf[make_key(il, b)];
|
|
return crf_a != crf_b ? crf_a < crf_b : a < b;
|
|
});
|
|
GGML_ASSERT((size_t) n_tokens*n_used == n_ids);
|
|
token_zero.assign(n_tokens, 0);
|
|
size_t n_keep = n_c;
|
|
for (size_t i = 0; i < n_keep; ) {
|
|
const int32_t e = misses[i];
|
|
uint64_t used = 0;
|
|
for (int64_t it = 0; it < n_tokens; ++it) {
|
|
for (int32_t k = 0; k < n_used; ++k) {
|
|
if (ids[it*n_used + k] == e) {
|
|
used |= token_zero[it];
|
|
}
|
|
}
|
|
}
|
|
int32_t z = 0;
|
|
while (z < 64 && (used >> z & 1)) {
|
|
z++;
|
|
}
|
|
if (z >= n_zero) {
|
|
std::swap(misses[i], misses[--n_keep]);
|
|
continue;
|
|
}
|
|
for (int64_t it = 0; it < n_tokens; ++it) {
|
|
for (int32_t k = 0; k < n_used; ++k) {
|
|
if (ids[it*n_used + k] == e) {
|
|
token_zero[it] |= 1ull << z;
|
|
}
|
|
}
|
|
}
|
|
slots[e] = n_slots + z;
|
|
cpu.push_back({ e, n_slots + z });
|
|
i++;
|
|
}
|
|
misses.erase(misses.begin(), misses.begin() + n_keep);
|
|
}
|
|
|
|
// fill the misses in expert order: victim() takes unused slots, and slots filled by one plan(), in list order,
|
|
// so consecutive experts often get consecutive slots and one upload
|
|
std::sort(misses.begin(), misses.end());
|
|
for (int32_t e : misses) {
|
|
const int32_t s = victim(il);
|
|
if (key_of[s] >= 0) {
|
|
slot_map[key_layer(key_of[s])][key_expert(key_of[s])] = empty;
|
|
} else {
|
|
n_filled++;
|
|
}
|
|
const int32_t key = make_key(il, e);
|
|
key_of[s] = key;
|
|
slots[e] = s;
|
|
move(s, class_of(key));
|
|
fills.push_back({ e, s });
|
|
}
|
|
clock[il] += n_tokens;
|
|
return true;
|
|
}
|
|
};
|
|
|
|
// gate, up, down or gate_up, down
|
|
static std::vector<ggml_tensor *> llama_moe_cache_layer_experts(const llama_layer & layer) {
|
|
std::vector<ggml_tensor *> res;
|
|
for (ggml_tensor * t : { layer.ffn_gate_up_exps, layer.ffn_gate_exps, layer.ffn_up_exps, layer.ffn_down_exps }) {
|
|
if (t != nullptr) {
|
|
res.push_back(t);
|
|
}
|
|
}
|
|
return res;
|
|
}
|
|
|
|
static bool llama_moe_cache_same_layout(const std::vector<ggml_tensor *> & a, const std::vector<ggml_tensor *> & b) {
|
|
if (a.size() != b.size()) {
|
|
return false;
|
|
}
|
|
for (size_t i = 0; i < a.size(); ++i) {
|
|
if (a[i]->type != b[i]->type || !ggml_are_same_shape(a[i], b[i]) || a[i]->nb[2] != b[i]->nb[2]) {
|
|
return false;
|
|
}
|
|
}
|
|
return true;
|
|
}
|
|
|
|
static bool llama_moe_cache_is_host_weight(const ggml_tensor * t) {
|
|
return t->buffer != nullptr &&
|
|
ggml_backend_buffer_get_usage(t->buffer) == GGML_BACKEND_BUFFER_USAGE_WEIGHTS &&
|
|
ggml_backend_buffer_is_host(t->buffer);
|
|
}
|
|
|
|
}
|
|
|
|
struct llama_moe_cache::impl {
|
|
// a GPU with its own budget and banks, it caches the layers assigned to it
|
|
struct device {
|
|
ggml_backend_t backend;
|
|
ggml_backend_buffer_type_t buft;
|
|
size_t host_bytes = 0; // host experts of the layers it caches
|
|
double split = 0.0; // share of the budget
|
|
|
|
// banks and their views
|
|
ggml_context_ptr ctx;
|
|
ggml_backend_buffer_ptr buf;
|
|
size_t buf_size = 0;
|
|
};
|
|
|
|
// layers of the same device with the same expert tensor layout share the banks and the LRU of a group
|
|
struct group {
|
|
int32_t id; // device
|
|
std::vector<ggml_tensor *> ref; // expert tensors of the first layer
|
|
std::vector<int32_t> layers;
|
|
std::vector<ggml_tensor *> banks; // device storage of all slots, one per expert tensor
|
|
size_t host_bytes = 0;
|
|
int32_t n_slots = 0;
|
|
moe_cache_lru lru;
|
|
};
|
|
|
|
struct layer {
|
|
int32_t ig = -1; // -1 if the layer is not cached
|
|
ggml_tensor * slot_map = nullptr; // I32 [1, n_expert] in host memory
|
|
ggml_tensor * slot_map_cpu = nullptr; // looked up in place of slot_map by the graphs with a CPU part, if cpu_share > 0, its data is not used
|
|
ggml_tensor * cpu_map = nullptr; // I32 [1, n_expert] in host memory, if cpu_share > 0
|
|
std::vector<int32_t> cpu_experts; // experts set in cpu_map
|
|
uint32_t cpu_acc = 0; // fraction of a miss carried over to the next plan, in 1/65536
|
|
std::vector<ggml_tensor *> experts; // host expert tensors, in the order of the banks
|
|
};
|
|
|
|
struct binding {
|
|
int32_t il;
|
|
int32_t ip; // index of the bank
|
|
ggml_tensor * cached; // view of the bank used in place of the host experts
|
|
};
|
|
|
|
struct stats {
|
|
size_t hits = 0;
|
|
size_t misses = 0;
|
|
size_t cpu = 0; // misses computed on the CPU
|
|
size_t bytes = 0;
|
|
};
|
|
|
|
static constexpr int64_t max_batch = 32;
|
|
|
|
int32_t n_expert_used;
|
|
|
|
stats stats_small; // up to 8 tokens per ubatch
|
|
stats stats_large;
|
|
stats stats_copy; // experts copied from the cache for large batches
|
|
|
|
std::vector<device> devices;
|
|
std::vector<group> groups;
|
|
std::vector<layer> layers;
|
|
std::unordered_map<const ggml_tensor *, binding> bindings; // host experts -> cached experts
|
|
std::unordered_map<const ggml_tensor *, std::pair<int32_t, bool>> layer_of; // slot map -> layer, and if the graph that looks it up has a CPU part
|
|
|
|
std::vector<int32_t> ids;
|
|
std::vector<moe_cache_lru::fill> fills;
|
|
std::vector<moe_cache_lru::fill> cpu;
|
|
|
|
float cpu_share = 0.0f;
|
|
uint32_t cpu_share_q = 0; // cpu_share in 1/65536, integer so that a simulator can reproduce the rounding
|
|
int32_t n_zero = 0; // zero slots per group
|
|
|
|
// LLAMA_MOE_CACHE_TRACE: the selected experts of each slot lookup, for a simulator of the cache
|
|
FILE * trace = nullptr;
|
|
|
|
// slot maps in host memory
|
|
ggml_context_ptr ctx_host;
|
|
ggml_backend_buffer_ptr buf_host;
|
|
size_t buf_host_size = 0;
|
|
|
|
// views used by copy_experts
|
|
ggml_context_ptr ctx_views;
|
|
|
|
impl(const llama_model & model, bool no_alloc, const std::vector<ggml_backend_t> & backends, const std::vector<ggml_backend_buffer_type_t> & bufts, size_t size, float cpu_share) :
|
|
n_expert_used(model.hparams.n_expert_used_max()), layers(model.layers.size()), cpu_share(std::clamp(cpu_share, 0.0f, 1.0f)) {
|
|
for (size_t i = 0; i < backends.size(); ++i) {
|
|
const auto dev_type = ggml_backend_dev_type(ggml_backend_get_device(backends[i]));
|
|
if (dev_type == GGML_BACKEND_DEVICE_TYPE_GPU || dev_type == GGML_BACKEND_DEVICE_TYPE_IGPU) {
|
|
auto & d = devices.emplace_back();
|
|
d.backend = backends[i];
|
|
d.buft = bufts[i];
|
|
}
|
|
}
|
|
if (devices.empty()) {
|
|
throw std::runtime_error("MoE cache requires a GPU backend");
|
|
}
|
|
if (model.split_mode() == LLAMA_SPLIT_MODE_TENSOR) {
|
|
throw std::runtime_error("MoE cache does not support tensor parallelism");
|
|
}
|
|
if (model.hparams.n_expert == 0 || n_expert_used == 0) {
|
|
throw std::runtime_error("MoE cache requires a MoE model");
|
|
}
|
|
|
|
// only cache layers that keep all of their experts in host memory, on the device the layer is assigned to
|
|
for (size_t il = 0; il < model.layers.size(); ++il) {
|
|
auto experts = llama_moe_cache_layer_experts(model.layers[il]);
|
|
if (experts.empty() || !std::all_of(experts.begin(), experts.end(), llama_moe_cache_is_host_weight)) {
|
|
continue;
|
|
}
|
|
const auto it_dev = std::find_if(devices.begin(), devices.end(), [&](const device & d) { return ggml_backend_get_device(d.backend) == model.dev_layer(il); });
|
|
if (it_dev == devices.end()) {
|
|
continue;
|
|
}
|
|
const int32_t id = (int32_t) (it_dev - devices.begin());
|
|
auto it = std::find_if(groups.begin(), groups.end(), [&](const group & g) { return g.id == id && llama_moe_cache_same_layout(g.ref, experts); });
|
|
if (it == groups.end()) {
|
|
groups.emplace_back();
|
|
it = groups.end() - 1;
|
|
it->id = id;
|
|
it->ref = experts;
|
|
}
|
|
it->layers.push_back(il);
|
|
for (const ggml_tensor * t : experts) {
|
|
it->host_bytes += ggml_nbytes(t);
|
|
devices[id].host_bytes += ggml_nbytes(t);
|
|
}
|
|
}
|
|
if (groups.empty()) {
|
|
LLAMA_LOG_WARN("%s: no layer has all of its experts in host memory, MoE cache is disabled\n", __func__);
|
|
return;
|
|
}
|
|
|
|
// with misses on the CPU, the n_zero slots after the cached ones stay zero: the device computes these misses as zero
|
|
// one extra slot at the end, CUDA MMQ can read past the last expert
|
|
n_zero = cpu_share > 0.0f ? std::min<int32_t>(n_expert_used, 64) : 0;
|
|
auto alloc_size = [&](const group & g, int32_t n_slots) {
|
|
const size_t alignment = ggml_backend_buft_get_alignment(devices[g.id].buft);
|
|
size_t res = 0;
|
|
for (const ggml_tensor * t : g.ref) {
|
|
res += GGML_PAD(t->nb[2]*(n_slots + n_zero + 1), alignment);
|
|
}
|
|
return res;
|
|
};
|
|
|
|
// the budget is split among the devices with host experts like the layers, by the tensor split or by default by free memory
|
|
const float * tensor_split = model.tensor_split();
|
|
const bool split_by_free = tensor_split == nullptr ||
|
|
std::all_of(tensor_split, tensor_split + model.n_devices(), [](float x) { return x == 0.0f; });
|
|
double split_sum = 0.0;
|
|
for (device & d : devices) {
|
|
if (d.host_bytes == 0) {
|
|
continue;
|
|
}
|
|
ggml_backend_dev_t dev = ggml_backend_get_device(d.backend);
|
|
if (split_by_free) {
|
|
size_t free;
|
|
size_t total;
|
|
ggml_backend_dev_memory(dev, &free, &total);
|
|
d.split = (double) free;
|
|
} else {
|
|
const auto it = std::find_if(model.devices.begin(), model.devices.end(), [&](const llama_device & ld) { return ld.dev == dev; });
|
|
GGML_ASSERT(it != model.devices.end());
|
|
d.split = (double) tensor_split[it - model.devices.begin()];
|
|
}
|
|
split_sum += d.split;
|
|
}
|
|
if (split_sum == 0.0) {
|
|
// the devices do not report their free memory
|
|
for (device & d : devices) {
|
|
d.split = d.host_bytes > 0 ? 1.0 : 0.0;
|
|
split_sum += d.split;
|
|
}
|
|
}
|
|
|
|
// within a device the budget is split by the size of the experts, so each group caches the same fraction of its experts
|
|
cpu_share_q = (uint32_t) std::lround(cpu_share*65536.0f);
|
|
if (const char * path = getenv("LLAMA_MOE_CACHE_TRACE"); path != nullptr && *path != '\0') {
|
|
trace = fopen(path, "wb");
|
|
if (trace == nullptr) {
|
|
throw std::runtime_error(std::string("failed to open the MoE cache trace ") + path);
|
|
}
|
|
write_trace_header(split_sum);
|
|
}
|
|
std::vector<size_t> n_tensors(devices.size(), 0);
|
|
size_t n_tensors_host = 0;
|
|
for (group & g : groups) {
|
|
const device & d = devices[g.id];
|
|
const int32_t n_expert = g.ref[0]->ne[2];
|
|
const size_t budget = (size_t) ((double) size*d.split/split_sum*g.host_bytes/d.host_bytes);
|
|
const int32_t max_slots = g.layers.size()*n_expert;
|
|
while (g.n_slots < max_slots && alloc_size(g, g.n_slots + 1) <= budget) {
|
|
g.n_slots++;
|
|
}
|
|
if (g.n_slots < n_expert_used) {
|
|
LLAMA_LOG_WARN("%s: MoE cache budget is too small for %zu layers, they are not cached\n", __func__, g.layers.size());
|
|
g.n_slots = 0;
|
|
continue;
|
|
}
|
|
g.lru.init(model.layers.size(), n_expert, g.n_slots);
|
|
n_tensors[g.id] += g.ref.size()*(1 + g.layers.size());
|
|
n_tensors_host += g.layers.size()*(n_zero > 0 ? 3 : 1); // slot maps and CPU map
|
|
}
|
|
if (n_tensors_host == 0) {
|
|
throw std::runtime_error("MoE cache is too small to hold the experts of one token");
|
|
}
|
|
|
|
auto init_ctx = [](size_t n_tensors) {
|
|
ggml_init_params params = {
|
|
/*.mem_size =*/ n_tensors*ggml_tensor_overhead(),
|
|
/*.mem_buffer =*/ nullptr,
|
|
/*.no_alloc =*/ true,
|
|
};
|
|
ggml_context_ptr res(ggml_init(params));
|
|
if (!res) {
|
|
throw std::runtime_error("failed to create the MoE cache context");
|
|
}
|
|
return res;
|
|
};
|
|
for (size_t id = 0; id < devices.size(); ++id) {
|
|
if (n_tensors[id] > 0) {
|
|
devices[id].ctx = init_ctx(n_tensors[id]);
|
|
}
|
|
}
|
|
ctx_host = init_ctx(n_tensors_host);
|
|
ctx_views = init_ctx(2);
|
|
|
|
ggml_backend_buffer_type_t buft_host = ggml_backend_cpu_buffer_type();
|
|
const size_t alignment_host = ggml_backend_buft_get_alignment(buft_host);
|
|
|
|
for (size_t ig = 0; ig < groups.size(); ++ig) {
|
|
group & g = groups[ig];
|
|
if (g.n_slots == 0) {
|
|
continue;
|
|
}
|
|
ggml_context * ctx = devices[g.id].ctx.get();
|
|
for (const ggml_tensor * t : g.ref) {
|
|
ggml_tensor * bank = ggml_new_tensor_3d(ctx, t->type, t->ne[0], t->ne[1], g.n_slots + n_zero + 1);
|
|
GGML_ASSERT(bank->nb[2] == t->nb[2]);
|
|
ggml_format_name(bank, "moe_cache.%zu.%s", ig, t->name);
|
|
g.banks.push_back(bank);
|
|
}
|
|
for (int32_t il : g.layers) {
|
|
layer & l = layers[il];
|
|
l.ig = (int32_t) ig;
|
|
l.experts = llama_moe_cache_layer_experts(model.layers[il]);
|
|
for (size_t ip = 0; ip < l.experts.size(); ++ip) {
|
|
ggml_tensor * bank = g.banks[ip];
|
|
ggml_tensor * cached = ggml_view_3d(ctx, bank, bank->ne[0], bank->ne[1], g.n_slots + n_zero, bank->nb[1], bank->nb[2], 0);
|
|
ggml_format_name(cached, "moe_cache.%s", l.experts[ip]->name);
|
|
bindings[l.experts[ip]] = { il, (int32_t) ip, cached };
|
|
}
|
|
l.slot_map = ggml_new_tensor_2d(ctx_host.get(), GGML_TYPE_I32, 1, g.ref[0]->ne[2]);
|
|
ggml_format_name(l.slot_map, "moe_cache.slot_map-%d", il);
|
|
layer_of[l.slot_map] = { il, false };
|
|
buf_host_size += GGML_PAD(ggml_nbytes(l.slot_map), alignment_host);
|
|
if (n_zero > 0) {
|
|
l.slot_map_cpu = ggml_new_tensor_2d(ctx_host.get(), GGML_TYPE_I32, 1, g.ref[0]->ne[2]);
|
|
ggml_format_name(l.slot_map_cpu, "moe_cache.slot_map_cpu-%d", il);
|
|
layer_of[l.slot_map_cpu] = { il, true };
|
|
buf_host_size += GGML_PAD(ggml_nbytes(l.slot_map_cpu), alignment_host);
|
|
l.cpu_map = ggml_new_tensor_2d(ctx_host.get(), GGML_TYPE_I32, 1, g.ref[0]->ne[2]);
|
|
ggml_format_name(l.cpu_map, "moe_cache.cpu_map-%d", il);
|
|
buf_host_size += GGML_PAD(ggml_nbytes(l.cpu_map), alignment_host);
|
|
}
|
|
}
|
|
devices[g.id].buf_size += alloc_size(g, g.n_slots);
|
|
}
|
|
|
|
// with no_alloc, dummy buffers are attached so the memory use can still be measured, see llama_context::memory_breakdown
|
|
for (device & d : devices) {
|
|
if (!d.ctx) {
|
|
continue;
|
|
}
|
|
d.buf = llama_memory_bufs::alloc_buf(d.ctx.get(), d.buft, no_alloc, "MoE cache");
|
|
if (!no_alloc) {
|
|
ggml_backend_buffer_clear(d.buf.get(), 0);
|
|
d.buf_size = ggml_backend_buffer_get_size(d.buf.get());
|
|
}
|
|
}
|
|
buf_host = llama_memory_bufs::alloc_buf(ctx_host.get(), buft_host, no_alloc, "MoE cache slot maps");
|
|
if (!no_alloc) {
|
|
ggml_backend_buffer_clear(buf_host.get(), 0xff); // the CPU maps are -1
|
|
buf_host_size = ggml_backend_buffer_get_size(buf_host.get());
|
|
|
|
for (group & g : groups) {
|
|
for (int32_t il : g.layers) {
|
|
if (layers[il].slot_map != nullptr) {
|
|
int32_t * slots = (int32_t *) layers[il].slot_map->data;
|
|
std::fill(slots, slots + layers[il].slot_map->ne[1], g.lru.empty);
|
|
g.lru.slot_map[il] = slots;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
// as weights, the ops that read the banks run on the device and the slot maps are copied with the copy callback
|
|
for (device & d : devices) {
|
|
if (d.buf) {
|
|
ggml_backend_buffer_set_usage(d.buf.get(), GGML_BACKEND_BUFFER_USAGE_WEIGHTS);
|
|
}
|
|
}
|
|
ggml_backend_buffer_set_usage(buf_host.get(), GGML_BACKEND_BUFFER_USAGE_WEIGHTS);
|
|
|
|
for (size_t id = 0; id < devices.size(); ++id) {
|
|
const device & d = devices[id];
|
|
if (d.host_bytes == 0) {
|
|
continue;
|
|
}
|
|
LLAMA_LOG_INFO("%s: %10s MoE cache size = %8.2f MiB for %.2f MiB of host experts, %.0f%% of the misses on the CPU\n", __func__,
|
|
ggml_backend_buft_name(d.buft), d.buf_size/1024.0/1024.0, d.host_bytes/1024.0/1024.0, 100.0*cpu_share);
|
|
for (const group & g : groups) {
|
|
if (g.id == (int32_t) id) {
|
|
LLAMA_LOG_INFO("%s: %2zu layers, %s: %5d slots (%.1f%%, %.1f tokens worth)\n", __func__,
|
|
g.layers.size(), ggml_type_name(g.ref.back()->type), g.n_slots, 100.0*g.n_slots/(g.layers.size()*g.ref[0]->ne[2]),
|
|
(double) g.n_slots/(g.layers.size()*n_expert_used));
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
ggml_backend_t backend(int32_t il) const {
|
|
return devices[groups[layers[il].ig].id].backend;
|
|
}
|
|
|
|
// trace format, little endian: "MCT1", then u32 n_layer, n_expert_used, n_group
|
|
// per group: u32 device, n_expert, n_layers, the layers (u32), f64 split, split_sum, u64 group host bytes, device host bytes, alignment,
|
|
// u32 n_tensors, the bytes of one expert of each tensor (u64)
|
|
// then one record per slot lookup: i32 layer, experts per token, n_ids, the ids (i32)
|
|
void write_trace_header(double split_sum) {
|
|
auto u32 = [&](uint32_t v) { fwrite(&v, sizeof(v), 1, trace); };
|
|
auto u64 = [&](uint64_t v) { fwrite(&v, sizeof(v), 1, trace); };
|
|
auto f64 = [&](double v) { fwrite(&v, sizeof(v), 1, trace); };
|
|
fwrite("MCT1", 1, 4, trace);
|
|
u32(layers.size());
|
|
u32(n_expert_used);
|
|
u32(groups.size());
|
|
for (const group & g : groups) {
|
|
const device & d = devices[g.id];
|
|
u32(g.id);
|
|
u32(g.ref[0]->ne[2]);
|
|
u32(g.layers.size());
|
|
for (int32_t il : g.layers) {
|
|
u32(il);
|
|
}
|
|
f64(d.split);
|
|
f64(split_sum);
|
|
u64(g.host_bytes);
|
|
u64(d.host_bytes);
|
|
u64(ggml_backend_buft_get_alignment(d.buft));
|
|
u32(g.ref.size());
|
|
for (const ggml_tensor * t : g.ref) {
|
|
u64(t->nb[2]);
|
|
}
|
|
}
|
|
}
|
|
|
|
void write_trace(int32_t il, int32_t n_used) {
|
|
const int32_t rec[3] = { il, n_used, (int32_t) ids.size() };
|
|
fwrite(rec, sizeof(int32_t), 3, trace);
|
|
fwrite(ids.data(), sizeof(int32_t), ids.size(), trace);
|
|
}
|
|
|
|
~impl() {
|
|
log_stats();
|
|
if (trace != nullptr) {
|
|
fclose(trace);
|
|
}
|
|
}
|
|
|
|
ggml_tensor * get_slot_map(int32_t il, int64_t n_tokens, int64_t n_expert_used, bool cpu_part) const {
|
|
if (il < 0 || il >= (int32_t) layers.size() || layers[il].ig < 0) {
|
|
return nullptr;
|
|
}
|
|
const layer & l = layers[il];
|
|
|
|
// large batches use most experts of a layer, so they gain little from the cache and would evict the experts used in generation
|
|
if (n_tokens == 0 || n_tokens > max_batch || std::min(n_tokens*n_expert_used, l.slot_map->ne[1]) > groups[l.ig].n_slots) {
|
|
return nullptr;
|
|
}
|
|
GGML_ASSERT(!cpu_part || l.slot_map_cpu != nullptr);
|
|
return cpu_part ? l.slot_map_cpu : l.slot_map;
|
|
}
|
|
|
|
ggml_tensor * get_cpu_map(int32_t il, int64_t n_tokens) const {
|
|
if (il < 0 || il >= (int32_t) layers.size() || layers[il].cpu_map == nullptr) {
|
|
return nullptr;
|
|
}
|
|
// the scheduler would run the CPU part on a device that offloads such batches from host weights
|
|
ggml_tensor probe = {};
|
|
probe.op = GGML_OP_MUL_MAT_ID;
|
|
probe.type = GGML_TYPE_F32;
|
|
probe.ne[0] = 1;
|
|
probe.ne[1] = n_expert_used;
|
|
probe.ne[2] = n_tokens;
|
|
probe.ne[3] = 1;
|
|
for (const device & d : devices) {
|
|
if (ggml_backend_offload_op(d.backend, &probe)) {
|
|
return nullptr;
|
|
}
|
|
}
|
|
return layers[il].cpu_map;
|
|
}
|
|
|
|
ggml_tensor * get_experts(const ggml_tensor * w) const {
|
|
const auto it = bindings.find(w);
|
|
return it != bindings.end() ? it->second.cached : nullptr;
|
|
}
|
|
|
|
int64_t copy_experts(ggml_backend_t backend, const ggml_tensor * w, ggml_tensor * dst, int64_t e, int64_t last) {
|
|
const auto it = bindings.find(w);
|
|
if (it == bindings.end()) {
|
|
return 0;
|
|
}
|
|
const binding & b = it->second;
|
|
const group & g = groups[layers[b.il].ig];
|
|
if (backend != devices[g.id].backend) {
|
|
return 0;
|
|
}
|
|
|
|
// large batches only read the cache, so the experts used in generation stay in it
|
|
const int32_t * slots = g.lru.slot_map[b.il];
|
|
if (slots == nullptr || slots[e] >= g.n_slots) {
|
|
return 0;
|
|
}
|
|
int64_t n = 1;
|
|
while (e + n <= last && slots[e + n] < g.n_slots && slots[e + n] == slots[e] + n) {
|
|
n++;
|
|
}
|
|
|
|
ggml_tensor * bank = g.banks[b.ip];
|
|
ggml_reset(ctx_views.get());
|
|
ggml_tensor * src_view = ggml_view_3d(ctx_views.get(), bank, bank->ne[0], bank->ne[1], n, bank->nb[1], bank->nb[2], slots[e]*bank->nb[2]);
|
|
ggml_tensor * dst_view = ggml_view_3d(ctx_views.get(), dst, dst->ne[0], dst->ne[1], n, dst->nb[1], dst->nb[2], e*dst->nb[2]);
|
|
ggml_backend_view_init(src_view);
|
|
ggml_backend_view_init(dst_view);
|
|
ggml_backend_tensor_copy_async(backend, backend, src_view, dst_view);
|
|
|
|
stats_copy.hits += n;
|
|
stats_copy.bytes += ggml_nbytes(src_view);
|
|
|
|
return n;
|
|
}
|
|
|
|
bool copy(ggml_backend_t backend, const ggml_tensor * src, ggml_tensor * dst, ggml_cgraph * graph) {
|
|
const auto it = layer_of.find(src);
|
|
if (it == layer_of.end()) {
|
|
return false;
|
|
}
|
|
const int32_t il = it->second.first;
|
|
const bool has_cpu = it->second.second;
|
|
layer & l = layers[il];
|
|
group & g = groups[l.ig];
|
|
|
|
GGML_ASSERT(backend == devices[g.id].backend);
|
|
|
|
// the get_rows that looks up the slots of the selected experts
|
|
const int n_nodes = ggml_graph_n_nodes(graph);
|
|
const ggml_tensor * lookup = nullptr;
|
|
for (int i = 0; i < n_nodes && lookup == nullptr; ++i) {
|
|
const ggml_tensor * node = ggml_graph_node(graph, i);
|
|
if (node->op == GGML_OP_GET_ROWS && node->src[0] == dst) {
|
|
lookup = node;
|
|
}
|
|
}
|
|
GGML_ASSERT(lookup != nullptr);
|
|
|
|
// the selected experts must be computed in an earlier split
|
|
// the scheduler starts a new split at the lookup because it reads a host weight, but only if the split already has inputs
|
|
const ggml_tensor * sel = lookup->src[1];
|
|
for (int i = 0; i < n_nodes; ++i) {
|
|
const ggml_tensor * node = ggml_graph_node(graph, i);
|
|
if (node == sel || node == sel->view_src) {
|
|
GGML_ABORT("the experts of layer %d are selected in the same split as their MoE cache lookup", il);
|
|
}
|
|
}
|
|
GGML_ASSERT(ggml_is_contiguous(sel));
|
|
GGML_ASSERT(sel->op == GGML_OP_RESHAPE); // of the [n_expert_used, n_tokens] ids, see build_moe_cache_slots
|
|
|
|
const int32_t n_used = (int32_t) sel->src[0]->ne[0];
|
|
|
|
ids.resize(ggml_nelements(sel));
|
|
ggml_backend_tensor_get_async(backend, sel, ids.data(), 0, ggml_nbytes(sel));
|
|
ggml_backend_synchronize(backend);
|
|
|
|
if (trace != nullptr) {
|
|
write_trace(il, n_used);
|
|
}
|
|
|
|
// part of the misses is computed on the CPU once the cache is full, the carry keeps the share right on average
|
|
const uint32_t share_q = has_cpu && g.lru.n_filled == g.lru.n_slots ? cpu_share_q : 0;
|
|
auto n_cpu = [&](size_t n_miss) {
|
|
const uint64_t acc = (uint64_t) n_miss*share_q + l.cpu_acc;
|
|
l.cpu_acc = acc & 0xffff;
|
|
return (size_t) (acc >> 16);
|
|
};
|
|
size_t n_hit = 0;
|
|
if (!g.lru.plan(il, ids.data(), ids.size(), sel->src[0]->ne[1], n_used, n_zero, n_cpu, fills, cpu, n_hit)) {
|
|
GGML_ABORT("the MoE cache is too small for the experts selected in layer %d", il);
|
|
}
|
|
if (has_cpu) {
|
|
int32_t * cpu_map = (int32_t *) l.cpu_map->data;
|
|
for (int32_t e : l.cpu_experts) {
|
|
cpu_map[e] = -1;
|
|
}
|
|
l.cpu_experts.clear();
|
|
for (const auto & f : cpu) {
|
|
cpu_map[f.expert] = f.expert;
|
|
l.cpu_experts.push_back(f.expert);
|
|
}
|
|
}
|
|
|
|
// upload the missing experts, consecutive experts going to consecutive slots are uploaded together
|
|
size_t bytes = 0;
|
|
for (size_t ip = 0; ip < l.experts.size(); ++ip) {
|
|
const ggml_tensor * w = l.experts[ip];
|
|
ggml_tensor * bank = g.banks[ip];
|
|
const size_t expert_size = w->nb[2];
|
|
for (size_t i = 0; i < fills.size();) {
|
|
size_t n = 1;
|
|
while (i + n < fills.size() && fills[i + n].expert == fills[i].expert + (int32_t) n && fills[i + n].slot == fills[i].slot + (int32_t) n) {
|
|
n++;
|
|
}
|
|
ggml_backend_tensor_set_async(backend, bank, (const uint8_t *) w->data + fills[i].expert*expert_size, fills[i].slot*expert_size, n*expert_size);
|
|
bytes += n*expert_size;
|
|
i += n;
|
|
}
|
|
}
|
|
|
|
stats & st = ids.size() <= (size_t) 8*n_expert_used ? stats_small : stats_large;
|
|
st.hits += n_hit;
|
|
st.misses += fills.size() + cpu.size();
|
|
st.cpu += cpu.size();
|
|
st.bytes += bytes;
|
|
|
|
// the next copy synchronizes the backend before it changes the slot map again
|
|
ggml_backend_tensor_set_async(backend, dst, l.slot_map->data, 0, ggml_nbytes(l.slot_map));
|
|
|
|
return true;
|
|
}
|
|
|
|
void log_stats() const {
|
|
auto log = [](const char * name, const stats & st) {
|
|
const size_t n = st.hits + st.misses;
|
|
if (n == 0) {
|
|
return;
|
|
}
|
|
LLAMA_LOG_INFO("llama_moe_cache: %s: hits = %zu, misses = %zu (%zu on the CPU), hit rate = %.2f%%, uploaded = %.2f MiB\n",
|
|
name, st.hits, st.misses, st.cpu, 100.0*st.hits/n, st.bytes/1024.0/1024.0);
|
|
};
|
|
log("ubatch <= 8", stats_small);
|
|
log("ubatch > 8", stats_large);
|
|
if (stats_copy.hits > 0) {
|
|
LLAMA_LOG_INFO("llama_moe_cache: large batches: %zu experts copied from the cache, %.2f MiB\n", stats_copy.hits, stats_copy.bytes/1024.0/1024.0);
|
|
}
|
|
}
|
|
};
|
|
|
|
llama_moe_cache::llama_moe_cache(const llama_model & model, bool no_alloc, const std::vector<ggml_backend_t> & backends, const std::vector<ggml_backend_buffer_type_t> & bufts, size_t size, float cpu_share) :
|
|
pimpl(new impl(model, no_alloc, backends, bufts, size, cpu_share)) {
|
|
}
|
|
|
|
llama_moe_cache::~llama_moe_cache() = default;
|
|
|
|
ggml_backend_t llama_moe_cache::backend(int32_t il) const {
|
|
return pimpl->backend(il);
|
|
}
|
|
|
|
ggml_tensor * llama_moe_cache::get_slot_map(int32_t il, int64_t n_tokens, int64_t n_expert_used, bool cpu_part) const {
|
|
return pimpl->get_slot_map(il, n_tokens, n_expert_used, cpu_part);
|
|
}
|
|
|
|
ggml_tensor * llama_moe_cache::get_experts(const ggml_tensor * w) const {
|
|
return pimpl->get_experts(w);
|
|
}
|
|
|
|
ggml_tensor * llama_moe_cache::get_cpu_map(int32_t il, int64_t n_tokens) const {
|
|
return pimpl->get_cpu_map(il, n_tokens);
|
|
}
|
|
|
|
bool llama_moe_cache::copy(ggml_backend_t backend, const ggml_tensor * src, ggml_tensor * dst, ggml_cgraph * graph) {
|
|
return pimpl->copy(backend, src, dst, graph);
|
|
}
|
|
|
|
int64_t llama_moe_cache::copy_experts(ggml_backend_t backend, const ggml_tensor * w, ggml_tensor * dst, int64_t e, int64_t last) {
|
|
return pimpl->copy_experts(backend, w, dst, e, last);
|
|
}
|
|
|
|
std::map<ggml_backend_buffer_type_t, size_t> llama_moe_cache::memory_breakdown() const {
|
|
std::map<ggml_backend_buffer_type_t, size_t> res;
|
|
for (const auto & d : pimpl->devices) {
|
|
if (d.buf) {
|
|
res[ggml_backend_buffer_get_type(d.buf.get())] += d.buf_size;
|
|
}
|
|
}
|
|
if (pimpl->buf_host) {
|
|
res[ggml_backend_buffer_get_type(pimpl->buf_host.get())] += pimpl->buf_host_size;
|
|
}
|
|
return res;
|
|
}
|