llama : report compute graph reallocations during inference (#30329)
Add an optional ggml_backend_sched_realloc_callback that user code registers with ggml_backend_sched_set_realloc_callback to be notified when the buffers reserved for a graph are not sufficient and have to be reserved again. The callback reports whether the backend assignments and the graph size changed since the previous scheduling. llama_context registers the callback when creating the scheduler (wrapped into the new llama_context::sched_new helper) and treats the reallocations that follow a change of the backend assignments as expected, counting the rest by graph size - unchanged or changed from the previously scheduled graph. Both counters are reported on one line in the perf stats. Assisted-by: pi:llama.cpp/Qwen3.8-Flash-Next
This commit is contained in:
1 parent
2f160ba59a
commit
972b5fcc19
7 files changed
+92
-14
No files matched your search
+35
-4
@@ -704,6 +704,14 @@ static int llama_graph_n_input_tensors(ggml_cgraph * gf) {
|
||||
return (int) users.size();
|
||||
}
|
||||
|
||||
void llama_context::sched_new(size_t max_nodes, bool parallel) {
|
||||
sched.reset(ggml_backend_sched_new(
|
||||
backend_ptrs.data(), backend_buft.data(), backend_ptrs.size(), max_nodes, parallel, cparams.op_offload));
|
||||
|
||||
ggml_backend_sched_set_copy_callback(sched.get(), sched_copy_experts, this);
|
||||
ggml_backend_sched_set_realloc_callback(sched.get(), sched_graph_realloc, this);
|
||||
}
|
||||
|
||||
void llama_context::sched_reserve() {
|
||||
if (!sched_need_reserve) {
|
||||
return;
|
||||
@@ -730,8 +738,7 @@ void llama_context::sched_reserve() {
|
||||
gf_res_reserve.reset(new llm_graph_result(max_nodes));
|
||||
gf_res_prev_active = nullptr;
|
||||
|
||||
sched.reset(ggml_backend_sched_new(backend_ptrs.data(), backend_buft.data(), backend_ptrs.size(), max_nodes, cparams.pipeline_parallel, cparams.op_offload));
|
||||
ggml_backend_sched_set_copy_callback(sched.get(), sched_copy_experts, this);
|
||||
sched_new(max_nodes, cparams.pipeline_parallel);
|
||||
|
||||
llama_memory_context_ptr mctx;
|
||||
if (memory) {
|
||||
@@ -770,8 +777,7 @@ void llama_context::sched_reserve() {
|
||||
if (cparams.pipeline_parallel) {
|
||||
LLAMA_LOG_WARN("%s: compute buffer allocation failed, retrying without pipeline parallelism\n", __func__);
|
||||
cparams.pipeline_parallel = false;
|
||||
sched.reset(ggml_backend_sched_new(backend_ptrs.data(), backend_buft.data(), backend_ptrs.size(), max_nodes, false, cparams.op_offload));
|
||||
ggml_backend_sched_set_copy_callback(sched.get(), sched_copy_experts, this);
|
||||
sched_new(max_nodes, false);
|
||||
gf = graph_reserve(n_tokens, n_seqs, n_outputs_pp, mctx.get());
|
||||
}
|
||||
if (!gf) {
|
||||
@@ -2697,6 +2703,25 @@ llm_graph_params llama_context::graph_params(
|
||||
};
|
||||
}
|
||||
|
||||
void llama_context::sched_graph_realloc(ggml_backend_sched_t sched, ggml_cgraph * gf, bool backend_ids_changed, bool graph_size_changed, void * user_data) {
|
||||
// reallocating after the backend assignments changed is expected, trace only the unexpected ones
|
||||
if (backend_ids_changed) {
|
||||
return;
|
||||
}
|
||||
|
||||
auto * lctx = static_cast<llama_context *>(user_data);
|
||||
|
||||
if (graph_size_changed) {
|
||||
lctx->n_reallocs_diff++;
|
||||
} else {
|
||||
lctx->n_reallocs_same++;
|
||||
}
|
||||
|
||||
LLAMA_LOG_INFO("%s: graph reallocation: nodes = %d, splits = %d, n_same = %d, n_diff = %d\n",
|
||||
__func__, ggml_graph_n_nodes(gf), ggml_backend_sched_get_n_splits(sched),
|
||||
lctx->n_reallocs_same, lctx->n_reallocs_diff);
|
||||
}
|
||||
|
||||
ggml_status llama_context::graph_compute(
|
||||
ggml_cgraph * gf,
|
||||
bool batched) {
|
||||
@@ -3651,6 +3676,9 @@ llama_perf_context_data llama_context::perf_get_data() const {
|
||||
data.n_eval = std::max(1, n_eval);
|
||||
data.n_reused = std::max(0, n_reused);
|
||||
|
||||
data.n_reallocs_same = n_reallocs_same;
|
||||
data.n_reallocs_diff = n_reallocs_diff;
|
||||
|
||||
return data;
|
||||
}
|
||||
|
||||
@@ -3659,6 +3687,8 @@ void llama_context::perf_reset() {
|
||||
t_eval_us = n_eval = 0;
|
||||
t_p_eval_us = n_p_eval = 0;
|
||||
n_reused = 0;
|
||||
|
||||
// note: the reallocation counters are not reset - they track the lifetime totals for this context
|
||||
}
|
||||
|
||||
llama_memory_breakdown llama_context::memory_breakdown() const {
|
||||
@@ -4624,6 +4654,7 @@ void llama_perf_context_print(const llama_context * ctx) {
|
||||
__func__, data.t_eval_ms, data.n_eval, data.t_eval_ms / data.n_eval, 1e3 / data.t_eval_ms * data.n_eval);
|
||||
LLAMA_LOG_INFO("%s: total time = %10.2f ms / %5d tokens\n", __func__, (t_end_ms - data.t_start_ms), (data.n_p_eval + data.n_eval));
|
||||
LLAMA_LOG_INFO("%s: graphs reused = %10d\n", __func__, data.n_reused);
|
||||
LLAMA_LOG_INFO("%s: graph reallocs = %10d / %5d (same size) / (changed size)\n", __func__, data.n_reallocs_same, data.n_reallocs_diff);
|
||||
}
|
||||
|
||||
void llama_perf_context_reset(llama_context * ctx) {
|
||||
|
||||
Reference in new issue
Block a user