llama : reuse backend scheduler when sched_params are compatible (#32009)

This commit is contained in:
Georgi Gerganov authored and GitHub committed 2026-11-10 17:10:11 +02:00
1 parent 494d0f47bd
commit 3e86ce2c2d
2 files changed
+39 -6

No files matched your search

+21 -4
View File
@@ -783,12 +783,21 @@ static int llama_graph_n_input_tensors(ggml_cgraph * gf) {
return (int) users.size();
}
void llama_context::sched_new(size_t max_nodes, bool parallel) {
void llama_context::sched_new(const sched_params & params) {
// reusing the scheduler avoids reallocating its buffers (incl. the compute buffers)
if (sched && params.can_reuse(sched_params_cur)) {
LLAMA_LOG_DEBUG("%s: reusing scheduler: max_nodes = %zu (current max_nodes = %zu)\n",
__func__, params.max_nodes, sched_params_cur.max_nodes);
return;
}
sched.reset(ggml_backend_sched_new(
backend_ptrs.data(), backend_buft.data(), backend_ptrs.size(), max_nodes, parallel, cparams.op_offload));
backend_ptrs.data(), backend_buft.data(), backend_ptrs.size(), params.max_nodes, params.parallel, params.op_offload));
ggml_backend_sched_set_copy_callback(sched.get(), sched_copy_experts, this);
ggml_backend_sched_set_realloc_callback(sched.get(), sched_graph_realloc, this);
sched_params_cur = params;
}
void llama_context::sched_reserve() {
@@ -817,7 +826,11 @@ void llama_context::sched_reserve() {
gf_res_reserve.reset(new llm_graph_result(max_nodes));
gf_res_prev_active = nullptr;
sched_new(max_nodes, cparams.pipeline_parallel);
sched_new({
/*.max_nodes =*/ max_nodes,
/*.parallel =*/ cparams.pipeline_parallel,
/*.op_offload =*/ cparams.op_offload,
});
llama_memory_context_ptr mctx;
if (memory) {
@@ -856,7 +869,11 @@ void llama_context::sched_reserve() {
if (cparams.pipeline_parallel) {
LLAMA_LOG_WARN("%s: compute buffer allocation failed, retrying without pipeline parallelism\n", __func__);
cparams.pipeline_parallel = false;
sched_new(max_nodes, false);
sched_new({
/*.max_nodes =*/ max_nodes,
/*.parallel =*/ false,
/*.op_offload =*/ cparams.op_offload,
});
gf = graph_reserve(n_tokens, n_seqs, n_outputs_pp, mctx.get());
}
if (!gf) {