From 3e86ce2c2dd11ae084fa435c11ea99cfd035988e Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Tue, 10 Nov 2026 17:10:11 +0200 Subject: [PATCH] llama : reuse backend scheduler when sched_params are compatible (#32009) --- src/llama-context.cpp | 25 +++++++++++++++++++++---- src/llama-context.h | 20 ++++++++++++++++++-- 2 files changed, 39 insertions(+), 6 deletions(-) diff --git a/src/llama-context.cpp b/src/llama-context.cpp index 394e39d85..5bb886454 100644 --- a/src/llama-context.cpp +++ b/src/llama-context.cpp @@ -783,12 +783,21 @@ static int llama_graph_n_input_tensors(ggml_cgraph * gf) { return (int) users.size(); } -void llama_context::sched_new(size_t max_nodes, bool parallel) { +void llama_context::sched_new(const sched_params & params) { + // reusing the scheduler avoids reallocating its buffers (incl. the compute buffers) + if (sched && params.can_reuse(sched_params_cur)) { + LLAMA_LOG_DEBUG("%s: reusing scheduler: max_nodes = %zu (current max_nodes = %zu)\n", + __func__, params.max_nodes, sched_params_cur.max_nodes); + return; + } + sched.reset(ggml_backend_sched_new( - backend_ptrs.data(), backend_buft.data(), backend_ptrs.size(), max_nodes, parallel, cparams.op_offload)); + backend_ptrs.data(), backend_buft.data(), backend_ptrs.size(), params.max_nodes, params.parallel, params.op_offload)); ggml_backend_sched_set_copy_callback(sched.get(), sched_copy_experts, this); ggml_backend_sched_set_realloc_callback(sched.get(), sched_graph_realloc, this); + + sched_params_cur = params; } void llama_context::sched_reserve() { @@ -817,7 +826,11 @@ void llama_context::sched_reserve() { gf_res_reserve.reset(new llm_graph_result(max_nodes)); gf_res_prev_active = nullptr; - sched_new(max_nodes, cparams.pipeline_parallel); + sched_new({ + /*.max_nodes =*/ max_nodes, + /*.parallel =*/ cparams.pipeline_parallel, + /*.op_offload =*/ cparams.op_offload, + }); llama_memory_context_ptr mctx; if (memory) { @@ -856,7 +869,11 @@ void llama_context::sched_reserve() { if (cparams.pipeline_parallel) { LLAMA_LOG_WARN("%s: compute buffer allocation failed, retrying without pipeline parallelism\n", __func__); cparams.pipeline_parallel = false; - sched_new(max_nodes, false); + sched_new({ + /*.max_nodes =*/ max_nodes, + /*.parallel =*/ false, + /*.op_offload =*/ cparams.op_offload, + }); gf = graph_reserve(n_tokens, n_seqs, n_outputs_pp, mctx.get()); } if (!gf) { diff --git a/src/llama-context.h b/src/llama-context.h index ee9c83752..f4061432f 100644 --- a/src/llama-context.h +++ b/src/llama-context.h @@ -297,8 +297,22 @@ private: llm_graph_cb graph_get_cb() const; - // create a new backend scheduler for a graph of the specified size and register the callbacks below - void sched_new(size_t max_nodes, bool parallel); + // parameters defining the backend scheduler instance + struct sched_params { + size_t max_nodes = 0; + bool parallel = false; + bool op_offload = false; + + // a scheduler can serve any graph of up to max_nodes nodes + bool can_reuse(const sched_params & other) const { + return parallel == other.parallel + && op_offload == other.op_offload + && max_nodes <= other.max_nodes; + } + }; + + // create a new backend scheduler with the given parameters, unless the current one can be reused + void sched_new(const sched_params & params); // ggml_backend_sched copy callback, copies only the experts used by MUL_MAT_ID and updates the MoE cache static bool sched_copy_experts(ggml_backend_t backend, const ggml_tensor * src, ggml_tensor * dst, ggml_cgraph * graph, void * user_data); @@ -390,6 +404,8 @@ private: ggml_backend_sched_ptr sched; + sched_params sched_params_cur; + bool sched_need_reserve = true; // whether the reserve graph has a logits tensor, set in sched_reserve()