llama : reuse backend scheduler when sched_params are compatible (#32009)
This commit is contained in:
1 parent
494d0f47bd
commit
3e86ce2c2d
2 files changed
+39
-6
No files matched your search
+21
-4
@@ -783,12 +783,21 @@ static int llama_graph_n_input_tensors(ggml_cgraph * gf) {
|
||||
return (int) users.size();
|
||||
}
|
||||
|
||||
void llama_context::sched_new(size_t max_nodes, bool parallel) {
|
||||
void llama_context::sched_new(const sched_params & params) {
|
||||
// reusing the scheduler avoids reallocating its buffers (incl. the compute buffers)
|
||||
if (sched && params.can_reuse(sched_params_cur)) {
|
||||
LLAMA_LOG_DEBUG("%s: reusing scheduler: max_nodes = %zu (current max_nodes = %zu)\n",
|
||||
__func__, params.max_nodes, sched_params_cur.max_nodes);
|
||||
return;
|
||||
}
|
||||
|
||||
sched.reset(ggml_backend_sched_new(
|
||||
backend_ptrs.data(), backend_buft.data(), backend_ptrs.size(), max_nodes, parallel, cparams.op_offload));
|
||||
backend_ptrs.data(), backend_buft.data(), backend_ptrs.size(), params.max_nodes, params.parallel, params.op_offload));
|
||||
|
||||
ggml_backend_sched_set_copy_callback(sched.get(), sched_copy_experts, this);
|
||||
ggml_backend_sched_set_realloc_callback(sched.get(), sched_graph_realloc, this);
|
||||
|
||||
sched_params_cur = params;
|
||||
}
|
||||
|
||||
void llama_context::sched_reserve() {
|
||||
@@ -817,7 +826,11 @@ void llama_context::sched_reserve() {
|
||||
gf_res_reserve.reset(new llm_graph_result(max_nodes));
|
||||
gf_res_prev_active = nullptr;
|
||||
|
||||
sched_new(max_nodes, cparams.pipeline_parallel);
|
||||
sched_new({
|
||||
/*.max_nodes =*/ max_nodes,
|
||||
/*.parallel =*/ cparams.pipeline_parallel,
|
||||
/*.op_offload =*/ cparams.op_offload,
|
||||
});
|
||||
|
||||
llama_memory_context_ptr mctx;
|
||||
if (memory) {
|
||||
@@ -856,7 +869,11 @@ void llama_context::sched_reserve() {
|
||||
if (cparams.pipeline_parallel) {
|
||||
LLAMA_LOG_WARN("%s: compute buffer allocation failed, retrying without pipeline parallelism\n", __func__);
|
||||
cparams.pipeline_parallel = false;
|
||||
sched_new(max_nodes, false);
|
||||
sched_new({
|
||||
/*.max_nodes =*/ max_nodes,
|
||||
/*.parallel =*/ false,
|
||||
/*.op_offload =*/ cparams.op_offload,
|
||||
});
|
||||
gf = graph_reserve(n_tokens, n_seqs, n_outputs_pp, mctx.get());
|
||||
}
|
||||
if (!gf) {
|
||||
|
||||
+18
-2
@@ -297,8 +297,22 @@ private:
|
||||
|
||||
llm_graph_cb graph_get_cb() const;
|
||||
|
||||
// create a new backend scheduler for a graph of the specified size and register the callbacks below
|
||||
void sched_new(size_t max_nodes, bool parallel);
|
||||
// parameters defining the backend scheduler instance
|
||||
struct sched_params {
|
||||
size_t max_nodes = 0;
|
||||
bool parallel = false;
|
||||
bool op_offload = false;
|
||||
|
||||
// a scheduler can serve any graph of up to max_nodes nodes
|
||||
bool can_reuse(const sched_params & other) const {
|
||||
return parallel == other.parallel
|
||||
&& op_offload == other.op_offload
|
||||
&& max_nodes <= other.max_nodes;
|
||||
}
|
||||
};
|
||||
|
||||
// create a new backend scheduler with the given parameters, unless the current one can be reused
|
||||
void sched_new(const sched_params & params);
|
||||
|
||||
// ggml_backend_sched copy callback, copies only the experts used by MUL_MAT_ID and updates the MoE cache
|
||||
static bool sched_copy_experts(ggml_backend_t backend, const ggml_tensor * src, ggml_tensor * dst, ggml_cgraph * graph, void * user_data);
|
||||
@@ -390,6 +404,8 @@ private:
|
||||
|
||||
ggml_backend_sched_ptr sched;
|
||||
|
||||
sched_params sched_params_cur;
|
||||
|
||||
bool sched_need_reserve = true;
|
||||
|
||||
// whether the reserve graph has a logits tensor, set in sched_reserve()
|
||||
|
||||
Reference in new issue
Block a user