ggml: refactor selective expert copying to user code (#29943)
* ggml: refactor selective expert copying to user code * tests: enroll two models into selective expert copy test * tests: use deepseek2 as test model * improve comment in ggml-backend.h Co-authored-by: Georgi Gerganov <ggerganov@gmail.com> * cont: fix whitespace * cont : better comments Co-authored-by: Georgi Gerganov <ggerganov@gmail.com> --------- Co-authored-by: Georgi Gerganov <ggerganov@gmail.com>
This commit is contained in:
1 parent
cbb7d52ecb
commit
6753a033f0
5 files changed
+195
-132
No files matched your search
@@ -317,6 +317,15 @@ extern "C" {
|
||||
//
|
||||
typedef bool (*ggml_backend_sched_eval_callback)(struct ggml_tensor * t, bool ask, void * user_data);
|
||||
|
||||
// Callback while copying input weights of a split
|
||||
// if the user returns false the scheduler simply copies the entire weight
|
||||
// the callback is called only for input weights in host buffers
|
||||
// the callback is called after all non-weight inputs of the split have been copied
|
||||
// `src` is the tensor in the previous split
|
||||
// `dst` is the copy of `src` in the split
|
||||
// `graph` is the compute graph of the split
|
||||
typedef bool (*ggml_backend_sched_copy_callback)(ggml_backend_t backend, const struct ggml_tensor * src, struct ggml_tensor * dst, struct ggml_cgraph * graph, void * user_data);
|
||||
|
||||
// Initialize a backend scheduler, backends with low index are given priority over backends with high index
|
||||
GGML_API ggml_backend_sched_t ggml_backend_sched_new(ggml_backend_t * backends, ggml_backend_buffer_type_t * bufts, int n_backends, size_t graph_size, bool parallel, bool op_offload);
|
||||
GGML_API void ggml_backend_sched_free(ggml_backend_sched_t sched);
|
||||
@@ -355,6 +364,9 @@ extern "C" {
|
||||
// Set a callback to be called for each resulting node during graph compute
|
||||
GGML_API void ggml_backend_sched_set_eval_callback(ggml_backend_sched_t sched, ggml_backend_sched_eval_callback callback, void * user_data);
|
||||
|
||||
// Set a callback to be called when the inputs weights of a split are being copied
|
||||
GGML_API void ggml_backend_sched_set_copy_callback(ggml_backend_sched_t sched, ggml_backend_sched_copy_callback callback, void * user_data);
|
||||
|
||||
//
|
||||
// Meta backend
|
||||
//
|
||||
|
||||
+65
-126
@@ -966,6 +966,9 @@ struct ggml_backend_sched {
|
||||
ggml_backend_sched_eval_callback callback_eval;
|
||||
void * callback_eval_user_data;
|
||||
|
||||
ggml_backend_sched_copy_callback callback_copy;
|
||||
void * callback_copy_user_data;
|
||||
|
||||
char * context_buffer;
|
||||
size_t context_buffer_size;
|
||||
|
||||
@@ -1799,14 +1802,58 @@ static bool ggml_backend_sched_alloc_splits(ggml_backend_sched_t sched) {
|
||||
return true;
|
||||
}
|
||||
|
||||
static bool ggml_backend_sched_is_host_weight(const struct ggml_tensor * t) {
|
||||
return t->buffer != NULL &&
|
||||
ggml_backend_buffer_get_usage(t->buffer) == GGML_BACKEND_BUFFER_USAGE_WEIGHTS &&
|
||||
ggml_backend_buffer_is_host(t->buffer);
|
||||
}
|
||||
|
||||
static void ggml_backend_sched_copy_input(ggml_backend_sched_t sched, struct ggml_backend_sched_split * split, struct ggml_tensor * input) {
|
||||
const int split_backend_id = split->backend_id;
|
||||
ggml_backend_t split_backend = sched->backends[split_backend_id];
|
||||
ggml_backend_t input_backend = ggml_backend_sched_get_tensor_backend(sched, input);
|
||||
struct ggml_tensor * input_cpy = tensor_copy(input, split_backend_id, sched->cur_copy);
|
||||
|
||||
if (input->flags & GGML_TENSOR_FLAG_INPUT) {
|
||||
// inputs from the user must be copied immediately to prevent the user overwriting the data before the copy is done
|
||||
if (sched->events[split_backend_id][sched->cur_copy] != NULL) {
|
||||
ggml_backend_event_synchronize(sched->events[split_backend_id][sched->cur_copy]);
|
||||
} else {
|
||||
ggml_backend_synchronize(split_backend);
|
||||
}
|
||||
ggml_backend_tensor_copy(input, input_cpy);
|
||||
return;
|
||||
}
|
||||
|
||||
// wait for the split backend to finish using the input before overwriting it
|
||||
if (sched->events[split_backend_id][sched->cur_copy] != NULL) {
|
||||
ggml_backend_event_wait(split_backend, sched->events[split_backend_id][sched->cur_copy]);
|
||||
} else {
|
||||
ggml_backend_synchronize(split_backend);
|
||||
}
|
||||
|
||||
if (sched->callback_copy != NULL && ggml_backend_sched_is_host_weight(input) &&
|
||||
sched->callback_copy(split_backend, input, input_cpy, &split->graph, sched->callback_copy_user_data)) {
|
||||
return;
|
||||
}
|
||||
|
||||
// try async copy, but if not possible, we can still use a sync copy without synchronizing the dst backend, since we handle the synchronization here with multiple copies and events
|
||||
// TODO: add public function to facilitate this, since applications do not have direct access to the backend interface
|
||||
if (!split_backend->iface.cpy_tensor_async || !split_backend->iface.cpy_tensor_async(input_backend, split_backend, input, input_cpy)) {
|
||||
ggml_backend_synchronize(input_backend);
|
||||
if (sched->events[split_backend_id][sched->cur_copy] != NULL) {
|
||||
ggml_backend_event_synchronize(sched->events[split_backend_id][sched->cur_copy]);
|
||||
} else {
|
||||
ggml_backend_synchronize(split_backend);
|
||||
}
|
||||
ggml_backend_tensor_copy(input, input_cpy);
|
||||
}
|
||||
}
|
||||
|
||||
static enum ggml_status ggml_backend_sched_compute_splits(ggml_backend_sched_t sched) {
|
||||
GGML_ASSERT(sched);
|
||||
struct ggml_backend_sched_split * splits = sched->splits;
|
||||
|
||||
ggml_tensor * prev_ids_tensor = nullptr;
|
||||
std::vector<int32_t> ids;
|
||||
std::vector<ggml_bitset_t> used_ids;
|
||||
|
||||
int prev_backend_id = -1;
|
||||
|
||||
for (int split_id = 0; split_id < sched->n_splits; split_id++) {
|
||||
@@ -1825,129 +1872,15 @@ static enum ggml_status ggml_backend_sched_compute_splits(ggml_backend_sched_t s
|
||||
}
|
||||
|
||||
// copy the input tensors to the split backend
|
||||
// the weights in host memory are copied last, so that the copy callback can read the other inputs of the split
|
||||
for (int input_id = 0; input_id < split->n_inputs; input_id++) {
|
||||
ggml_backend_t input_backend = ggml_backend_sched_get_tensor_backend(sched, split->inputs[input_id]);
|
||||
struct ggml_tensor * input = split->inputs[input_id];
|
||||
struct ggml_tensor * input_cpy = tensor_copy(input, split_backend_id, sched->cur_copy);
|
||||
|
||||
if (input->flags & GGML_TENSOR_FLAG_INPUT) {
|
||||
// inputs from the user must be copied immediately to prevent the user overwriting the data before the copy is done
|
||||
if (sched->events[split_backend_id][sched->cur_copy] != NULL) {
|
||||
ggml_backend_event_synchronize(sched->events[split_backend_id][sched->cur_copy]);
|
||||
} else {
|
||||
ggml_backend_synchronize(split_backend);
|
||||
}
|
||||
ggml_backend_tensor_copy(input, input_cpy);
|
||||
} else {
|
||||
// wait for the split backend to finish using the input before overwriting it
|
||||
if (sched->events[split_backend_id][sched->cur_copy] != NULL) {
|
||||
ggml_backend_event_wait(split_backend, sched->events[split_backend_id][sched->cur_copy]);
|
||||
} else {
|
||||
ggml_backend_synchronize(split_backend);
|
||||
}
|
||||
|
||||
// when offloading MoE weights, we can reduce the amount of data copied by copying only the experts that are used
|
||||
ggml_tensor * node = split->graph.nodes[0];
|
||||
if (split->graph.n_nodes > 0 &&
|
||||
ggml_backend_buffer_get_usage(input->buffer) == GGML_BACKEND_BUFFER_USAGE_WEIGHTS &&
|
||||
ggml_backend_buffer_is_host(input->buffer) && (
|
||||
(node->src[0] == input_cpy && node->op == GGML_OP_MUL_MAT_ID)
|
||||
//|| (node->src[1] == input_cpy && node->op == GGML_OP_ADD_ID) /* GGML_OP_ADD_ID weights are small and not worth splitting */
|
||||
)) {
|
||||
|
||||
const int64_t n_expert = node->op == GGML_OP_MUL_MAT_ID ? input->ne[2] : input->ne[1];
|
||||
const size_t expert_size = node->op == GGML_OP_MUL_MAT_ID ? input->nb[2] : input->nb[1];
|
||||
|
||||
ggml_backend_synchronize(input_backend);
|
||||
|
||||
// get the ids
|
||||
ggml_tensor * ids_tensor = node->src[2];
|
||||
ggml_backend_t ids_backend = split_backend;
|
||||
|
||||
if (ggml_nelements(ids_tensor) == 0) {
|
||||
continue;
|
||||
}
|
||||
|
||||
// if the ids tensor is also an input of the split, it may not have been copied yet to the split backend
|
||||
// in that case, we use the original ids tensor
|
||||
for (int i = input_id + 1; i < split->n_inputs; i++) {
|
||||
if (ids_tensor == tensor_copy(split->inputs[i], split_backend_id, sched->cur_copy)) {
|
||||
ids_tensor = split->inputs[i];
|
||||
ids_backend = ggml_backend_sched_get_tensor_backend(sched, split->inputs[i]);
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (ids_tensor != prev_ids_tensor) {
|
||||
ids.resize(ggml_nbytes(ids_tensor) / sizeof(int32_t));
|
||||
ggml_backend_tensor_get_async(ids_backend, ids_tensor, ids.data(), 0, ggml_nbytes(ids_tensor));
|
||||
ggml_backend_synchronize(ids_backend);
|
||||
|
||||
// find the used experts
|
||||
used_ids.clear();
|
||||
used_ids.resize(ggml_bitset_size(n_expert));
|
||||
for (int64_t i1 = 0; i1 < ids_tensor->ne[1]; i1++) {
|
||||
for (int64_t i0 = 0; i0 < ids_tensor->ne[0]; i0++) {
|
||||
int32_t id = ids[i1 * ids_tensor->nb[1]/sizeof(int32_t) + i0 * ids_tensor->nb[0]/sizeof(int32_t)];
|
||||
GGML_ASSERT(id >= 0 && id < n_expert);
|
||||
ggml_bitset_set(used_ids.data(), id);
|
||||
}
|
||||
}
|
||||
|
||||
prev_ids_tensor = ids_tensor;
|
||||
}
|
||||
|
||||
// group consecutive experts and copy them together
|
||||
auto copy_experts = [&](int32_t first_id, int32_t last_id) {
|
||||
const size_t expert_offset = first_id * expert_size;
|
||||
const size_t expert_size_copy = (last_id - first_id + 1) * expert_size;
|
||||
const size_t padding = std::min<size_t>(expert_size, 512);
|
||||
const size_t padding_end = last_id < n_expert - 1 ? padding : 0;
|
||||
|
||||
ggml_backend_tensor_set_async(split_backend,
|
||||
input_cpy,
|
||||
(const uint8_t *)input->data + expert_offset, expert_offset,
|
||||
// copy a bit extra at the to ensure there are no NaNs in the padding of the last expert
|
||||
// this is necessary for MMQ in the CUDA backend
|
||||
expert_size_copy + padding_end);
|
||||
};
|
||||
|
||||
int id = 0;
|
||||
while (!ggml_bitset_get(used_ids.data(), id)) {
|
||||
id++;
|
||||
}
|
||||
int32_t first_id = id;
|
||||
int32_t last_id = first_id;
|
||||
|
||||
for (++id; id < n_expert; ++id) {
|
||||
if (!ggml_bitset_get(used_ids.data(), id)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
if (id == last_id + 1) {
|
||||
last_id = id;
|
||||
continue;
|
||||
}
|
||||
|
||||
copy_experts(first_id, last_id);
|
||||
|
||||
first_id = id;
|
||||
last_id = id;
|
||||
}
|
||||
copy_experts(first_id, last_id);
|
||||
} else {
|
||||
// try async copy, but if not possible, we can still use a sync copy without synchronizing the dst backend, since we handle the synchronization here with multiple copies and events
|
||||
// TODO: add public function to facilitate this, since applications do not have direct access to the backend interface
|
||||
if (!split_backend->iface.cpy_tensor_async || !split_backend->iface.cpy_tensor_async(input_backend, split_backend, input, input_cpy)) {
|
||||
ggml_backend_synchronize(input_backend);
|
||||
if (sched->events[split_backend_id][sched->cur_copy] != NULL) {
|
||||
ggml_backend_event_synchronize(sched->events[split_backend_id][sched->cur_copy]);
|
||||
} else {
|
||||
ggml_backend_synchronize(split_backend);
|
||||
}
|
||||
ggml_backend_tensor_copy(input, input_cpy);
|
||||
}
|
||||
}
|
||||
if (!ggml_backend_sched_is_host_weight(split->inputs[input_id])) {
|
||||
ggml_backend_sched_copy_input(sched, split, split->inputs[input_id]);
|
||||
}
|
||||
}
|
||||
for (int input_id = 0; input_id < split->n_inputs; input_id++) {
|
||||
if (ggml_backend_sched_is_host_weight(split->inputs[input_id])) {
|
||||
ggml_backend_sched_copy_input(sched, split, split->inputs[input_id]);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2204,6 +2137,12 @@ void ggml_backend_sched_set_eval_callback(ggml_backend_sched_t sched, ggml_backe
|
||||
sched->callback_eval_user_data = user_data;
|
||||
}
|
||||
|
||||
void ggml_backend_sched_set_copy_callback(ggml_backend_sched_t sched, ggml_backend_sched_copy_callback callback, void * user_data) {
|
||||
GGML_ASSERT(sched);
|
||||
sched->callback_copy = callback;
|
||||
sched->callback_copy_user_data = user_data;
|
||||
}
|
||||
|
||||
int ggml_backend_sched_get_n_splits(ggml_backend_sched_t sched) {
|
||||
GGML_ASSERT(sched);
|
||||
return sched->n_splits;
|
||||
|
||||
@@ -641,6 +641,7 @@ void llama_context::sched_reserve() {
|
||||
gf_res_prev_active = nullptr;
|
||||
|
||||
sched.reset(ggml_backend_sched_new(backend_ptrs.data(), backend_buft.data(), backend_ptrs.size(), max_nodes, cparams.pipeline_parallel, cparams.op_offload));
|
||||
ggml_backend_sched_set_copy_callback(sched.get(), sched_copy_experts, this);
|
||||
|
||||
llama_memory_context_ptr mctx;
|
||||
if (memory) {
|
||||
@@ -680,6 +681,7 @@ void llama_context::sched_reserve() {
|
||||
LLAMA_LOG_WARN("%s: compute buffer allocation failed, retrying without pipeline parallelism\n", __func__);
|
||||
cparams.pipeline_parallel = false;
|
||||
sched.reset(ggml_backend_sched_new(backend_ptrs.data(), backend_buft.data(), backend_ptrs.size(), max_nodes, false, cparams.op_offload));
|
||||
ggml_backend_sched_set_copy_callback(sched.get(), sched_copy_experts, this);
|
||||
gf = graph_reserve(n_tokens, n_seqs, n_outputs_pp, mctx.get());
|
||||
}
|
||||
if (!gf) {
|
||||
@@ -2625,6 +2627,8 @@ ggml_status llama_context::graph_compute(
|
||||
set_n_threads_fn.second(set_n_threads_fn.first, n_threads);
|
||||
}
|
||||
|
||||
copy_experts.reset();
|
||||
|
||||
auto status = ggml_backend_sched_graph_compute_async(sched.get(), gf);
|
||||
if (status != GGML_STATUS_SUCCESS) {
|
||||
LLAMA_LOG_ERROR("%s: ggml_backend_sched_graph_compute_async failed with error %d\n", __func__, status);
|
||||
@@ -2635,6 +2639,65 @@ ggml_status llama_context::graph_compute(
|
||||
return status;
|
||||
}
|
||||
|
||||
bool llama_context::sched_copy_experts(ggml_backend_t backend, const ggml_tensor * src, ggml_tensor * dst, ggml_cgraph * graph, void * user_data) {
|
||||
auto & st = static_cast<llama_context *>(user_data)->copy_experts;
|
||||
|
||||
// the ids must be computed before the split starts, so only the first node of the split is considered
|
||||
if (ggml_graph_n_nodes(graph) == 0) {
|
||||
return false;
|
||||
}
|
||||
const ggml_tensor * node = ggml_graph_node(graph, 0);
|
||||
if (node->op != GGML_OP_MUL_MAT_ID || node->src[0] != dst) {
|
||||
return false;
|
||||
}
|
||||
|
||||
const ggml_tensor * ids = node->src[2];
|
||||
if (ggml_nelements(ids) == 0) {
|
||||
return true;
|
||||
}
|
||||
|
||||
const int64_t n_expert = src->ne[2];
|
||||
const size_t expert_size = src->nb[2];
|
||||
|
||||
if (ids != st.ids || (int64_t) st.used.size() != n_expert) {
|
||||
st.ids_data.resize(ggml_nbytes(ids)/sizeof(int32_t));
|
||||
ggml_backend_tensor_get_async(backend, ids, st.ids_data.data(), 0, ggml_nbytes(ids));
|
||||
ggml_backend_synchronize(backend);
|
||||
|
||||
st.used.assign(n_expert, false);
|
||||
for (int64_t i1 = 0; i1 < ids->ne[1]; i1++) {
|
||||
for (int64_t i0 = 0; i0 < ids->ne[0]; i0++) {
|
||||
const int32_t id = st.ids_data[i1*ids->nb[1]/sizeof(int32_t) + i0*ids->nb[0]/sizeof(int32_t)];
|
||||
GGML_ASSERT(id >= 0 && id < n_expert);
|
||||
st.used[id] = true;
|
||||
}
|
||||
}
|
||||
|
||||
st.ids = ids;
|
||||
}
|
||||
|
||||
// group consecutive experts and copy them together
|
||||
for (int64_t first = 0; first < n_expert; ) {
|
||||
if (!st.used[first]) {
|
||||
first++;
|
||||
continue;
|
||||
}
|
||||
int64_t last = first;
|
||||
while (last + 1 < n_expert && st.used[last + 1]) {
|
||||
last++;
|
||||
}
|
||||
|
||||
// copy a bit extra to ensure there are no NaNs in the padding of the last expert, this is necessary for MMQ in the CUDA backend
|
||||
const size_t offset = first*expert_size;
|
||||
const size_t padding = last < n_expert - 1 ? std::min<size_t>(expert_size, 512) : 0;
|
||||
ggml_backend_tensor_set_async(backend, dst, (const uint8_t *) src->data + offset, offset, (last - first + 1)*expert_size + padding);
|
||||
|
||||
first = last + 1;
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
llm_graph_cb llama_context::graph_get_cb() const {
|
||||
return [&](const llama_ubatch & ubatch, ggml_tensor * cur, const char * name, int il) {
|
||||
if (il >= 0) {
|
||||
@@ -3691,6 +3754,7 @@ void llama_context::opt_epoch_iter(
|
||||
ggml_backend_tensor_set(labels, &onef, (pos_ubatch*labels->ne[0] + labels_sparse[ilabel])*sizeof(float), sizeof(float));
|
||||
}
|
||||
}
|
||||
copy_experts.reset();
|
||||
ggml_opt_eval(opt_ctx, result);
|
||||
if (callback) {
|
||||
callback(train, opt_ctx, dataset, result, idata_in_loop + (pos_ctx + pos_batch)/n_ubatch + 1, ndata_in_loop, t_loop_start);
|
||||
|
||||
@@ -271,6 +271,9 @@ private:
|
||||
|
||||
llm_graph_cb graph_get_cb() const;
|
||||
|
||||
// ggml_backend_sched copy callback, copies only the experts used by MUL_MAT_ID
|
||||
static bool sched_copy_experts(ggml_backend_t backend, const ggml_tensor * src, ggml_tensor * dst, ggml_cgraph * graph, void * user_data);
|
||||
|
||||
// disable auto fused ops (Flash Attention, Gated Delta Net) whose op lands on a device
|
||||
// that differs from the layer it belongs to (usually due to missing backend support)
|
||||
void resolve_fused_ops(const llama_memory_context_i * mctx, uint32_t n_seqs);
|
||||
@@ -356,6 +359,21 @@ private:
|
||||
|
||||
bool sched_need_reserve = true;
|
||||
|
||||
// state of sched_copy_experts, reset before each graph compute
|
||||
struct copy_experts_info {
|
||||
const ggml_tensor * ids = nullptr;
|
||||
std::vector<int32_t> ids_data;
|
||||
std::vector<bool> used;
|
||||
|
||||
void reset() {
|
||||
ids = nullptr;
|
||||
ids_data.clear();
|
||||
used.clear();
|
||||
}
|
||||
};
|
||||
|
||||
copy_experts_info copy_experts;
|
||||
|
||||
ggml_backend_t backend_cpu = nullptr;
|
||||
std::vector<ggml_backend_ptr> backends;
|
||||
|
||||
|
||||
@@ -113,6 +113,16 @@ static std::vector<llama_token> get_tokens(const uint32_t n_tokens, const uint32
|
||||
return ret;
|
||||
}
|
||||
|
||||
// MoE archs that are also tested with the experts in host memory
|
||||
static bool host_experts_test(const llm_arch arch) {
|
||||
switch (arch) {
|
||||
case LLM_ARCH_DEEPSEEK2:
|
||||
return true;
|
||||
default:
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
static gguf_context_ptr get_gguf_ctx(const llm_arch arch, const bool moe) {
|
||||
gguf_context_ptr ret(gguf_init_empty());
|
||||
llama_model_saver ms(arch, ret.get());
|
||||
@@ -415,7 +425,8 @@ static gguf_context_ptr get_gguf_ctx(const llm_arch arch, const bool moe) {
|
||||
ms.add_kv(LLM_KV_EXPERT_SHARED_FEED_FORWARD_LENGTH, n_ff / 2); // distinct from n_ff so a saver key-clobber surfaces on reload
|
||||
ms.add_kv(LLM_KV_EXPERT_LATENT_LENGTH, n_ff);
|
||||
ms.add_kv(LLM_KV_INTERLEAVE_MOE_LAYER_STEP, uint32_t(2));
|
||||
ms.add_kv(LLM_KV_EXPERT_COUNT, uint32_t(2));
|
||||
// with more experts than a ubatch uses, the copy of the used experts in host memory skips some of them
|
||||
ms.add_kv(LLM_KV_EXPERT_COUNT, uint32_t(host_experts_test(arch) ? 64 : 2));
|
||||
ms.add_kv(LLM_KV_EXPERT_USED_COUNT, uint32_t(2));
|
||||
ms.add_kv(LLM_KV_EXPERT_SHARED_COUNT, uint32_t(1));
|
||||
ms.add_kv(LLM_KV_EXPERT_GATING_FUNC, arch == LLM_ARCH_DEEPSEEK4 ? uint32_t(4) : uint32_t(2)); // sqrtsoftplus : sigmoid
|
||||
@@ -474,7 +485,8 @@ static bool silent_model_load_progress(float /*progress*/, void * /*user_data*/)
|
||||
static std::pair<llama_model_ptr, llama_context_ptr> get_model_and_ctx(
|
||||
struct gguf_context * gguf_ctx, FILE * file, const size_t seed, const float stdev,
|
||||
const std::vector<ggml_backend_dev_t> & devs,
|
||||
const llama_split_mode split_mode = LLAMA_SPLIT_MODE_LAYER, bool encode = false) {
|
||||
const llama_split_mode split_mode = LLAMA_SPLIT_MODE_LAYER, bool encode = false,
|
||||
const llama_model_tensor_buft_override * tensor_buft_overrides = nullptr) {
|
||||
GGML_ASSERT((gguf_ctx == nullptr) != (file == nullptr));
|
||||
llama_model_params model_params = llama_model_default_params();
|
||||
model_params.progress_callback = silent_model_load_progress;
|
||||
@@ -482,6 +494,7 @@ static std::pair<llama_model_ptr, llama_context_ptr> get_model_and_ctx(
|
||||
devs_copy.push_back(nullptr);
|
||||
model_params.devices = devs_copy.data();
|
||||
model_params.split_mode = split_mode;
|
||||
model_params.tensor_buft_overrides = tensor_buft_overrides;
|
||||
|
||||
llama_context_params ctx_params = llama_context_default_params();
|
||||
ctx_params.n_ctx = 0;
|
||||
@@ -841,9 +854,15 @@ static int test_backends(const std::string & arch_filter, const size_t seed, con
|
||||
std::vector<ggml_backend_dev_t> devs;
|
||||
std::string label;
|
||||
llama_split_mode split_mode;
|
||||
bool host_experts; // keep the experts in host memory, see host_experts_test
|
||||
|
||||
device_config(std::vector<ggml_backend_dev_t> devs, std::string name, llama_split_mode split_mode)
|
||||
: devs(std::move(devs)), label(std::move(name)), split_mode(split_mode) {}
|
||||
device_config(std::vector<ggml_backend_dev_t> devs, std::string name, llama_split_mode split_mode, bool host_experts = false)
|
||||
: devs(std::move(devs)), label(std::move(name)), split_mode(split_mode), host_experts(host_experts) {}
|
||||
};
|
||||
|
||||
const llama_model_tensor_buft_override host_experts_overrides[] = {
|
||||
{ LLM_FFN_EXPS_REGEX, ggml_backend_cpu_buffer_type() },
|
||||
{ nullptr, nullptr },
|
||||
};
|
||||
|
||||
std::vector<device_config> dev_configs;
|
||||
@@ -870,6 +889,12 @@ static int test_backends(const std::string & arch_filter, const size_t seed, con
|
||||
if (target_backend == nullptr) {
|
||||
dev_configs.emplace_back(devices_meta, "Meta", LLAMA_SPLIT_MODE_TENSOR);
|
||||
}
|
||||
|
||||
// the ops that use the experts are offloaded to the first device and the scheduler copies the used experts
|
||||
if (!devices_meta.empty()) {
|
||||
dev_configs.emplace_back(devices_meta, "Host experts", LLAMA_SPLIT_MODE_LAYER, true);
|
||||
max_device_label_length = std::max(max_device_label_length, dev_configs.back().label.length());
|
||||
}
|
||||
}
|
||||
|
||||
size_t max_arch_name_length = 0;
|
||||
@@ -925,6 +950,11 @@ static int test_backends(const std::string & arch_filter, const size_t seed, con
|
||||
std::pair<llama_model_ptr, llama_context_ptr> model_and_ctx_cpu;
|
||||
std::vector<float> logits_cpu;
|
||||
for (device_config & dc : dev_configs) {
|
||||
if (dc.host_experts && (!moe || !host_experts_test(arch))) {
|
||||
continue;
|
||||
}
|
||||
const llama_model_tensor_buft_override * overrides = dc.host_experts ? host_experts_overrides : nullptr;
|
||||
|
||||
// print test config first; should anything fail during model loading or inference, at least we know which test case caused it
|
||||
LOG(template_row_cfg.c_str(), llm_arch_name(arch), dc.label.c_str(), config_name.c_str());
|
||||
fflush(stdout);
|
||||
@@ -947,7 +977,7 @@ static int test_backends(const std::string & arch_filter, const size_t seed, con
|
||||
}
|
||||
if (dc.split_mode != LLAMA_SPLIT_MODE_TENSOR || llm_arch_supports_sm_tensor(arch)) {
|
||||
test_executed = true;
|
||||
model_and_ctx_dev = get_model_and_ctx(gguf_ctx.get(), nullptr, seed, stdev, dc.devs, dc.split_mode, encode);
|
||||
model_and_ctx_dev = get_model_and_ctx(gguf_ctx.get(), nullptr, seed, stdev, dc.devs, dc.split_mode, encode, overrides);
|
||||
logits_dev = get_logits(model_and_ctx_dev.first.get(), model_and_ctx_dev.second.get(), tokens, encode);
|
||||
const double nmse_val = nmse(logits_cpu, logits_dev);
|
||||
snprintf(nmse_str, sizeof(nmse_str), "(%.2e)", nmse_val);
|
||||
@@ -1013,7 +1043,7 @@ static int test_backends(const std::string & arch_filter, const size_t seed, con
|
||||
ms.save(file);
|
||||
rewind(file);
|
||||
|
||||
auto model_and_ctx_roundtrip = get_model_and_ctx(nullptr, file, seed, stdev, dc.devs, dc.split_mode, encode);
|
||||
auto model_and_ctx_roundtrip = get_model_and_ctx(nullptr, file, seed, stdev, dc.devs, dc.split_mode, encode, overrides);
|
||||
const std::vector<float> logits_roundtrip = get_logits(
|
||||
model_and_ctx_roundtrip.first.get(), model_and_ctx_roundtrip.second.get(), tokens, encode);
|
||||
status_roundtrip = "\033[1;32mOK\033[0m";
|
||||
|
||||
Reference in new issue
Block a user