llama : report compute graph reallocations during inference (#30329)
Add an optional ggml_backend_sched_realloc_callback that user code registers with ggml_backend_sched_set_realloc_callback to be notified when the buffers reserved for a graph are not sufficient and have to be reserved again. The callback reports whether the backend assignments and the graph size changed since the previous scheduling. llama_context registers the callback when creating the scheduler (wrapped into the new llama_context::sched_new helper) and treats the reallocations that follow a change of the backend assignments as expected, counting the rest by graph size - unchanged or changed from the previously scheduled graph. Both counters are reported on one line in the perf stats. Assisted-by: pi:llama.cpp/Qwen3.8-Flash-Next
This commit is contained in:
1 parent
2f160ba59a
commit
972b5fcc19
7 files changed
+92
-14
No files matched your search
@@ -589,6 +589,7 @@ void common_perf_print(const struct llama_context * ctx, const struct common_sam
|
||||
LOG_INF("%s: total time = %10.2f ms / %5d tokens\n", __func__, (t_end_ms - data.t_start_ms), (data.n_p_eval + data.n_eval));
|
||||
LOG_INF("%s: unaccounted time = %10.2f ms / %5.1f %% (total - sampling - prompt eval - eval) / (total)\n", __func__, t_unacc_ms, t_unacc_pc);
|
||||
LOG_INF("%s: graphs reused = %10d\n", __func__, data.n_reused);
|
||||
LOG_INF("%s: graph reallocs = %10d / %5d (same size) / (changed size)\n", __func__, data.n_reallocs_same, data.n_reallocs_diff);
|
||||
|
||||
common_memory_breakdown_print(ctx);
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user