llama : reuse graph result instances across scheduler reserves (#32089)
Allocate gf_res_reserve and both gf_res_prev entries once in the llama_context constructor. Add llm_graph_result::set_max_nodes() and reset the existing instances in sched_reserve() instead of recreating them. get_gf_res_prev() now only selects the arena for the current batch. Assisted-by: dsh:llama.cpp/ggml-org/DeepSeek-V4.1-Flash-1017-GGUF (Mac Studio, M5 Ultra, 256GB)
This commit is contained in:
1 parent
cd2db800db
commit
cb44fdb189
3 files changed
+26
-16
No files matched your search
+19
-15
@@ -449,6 +449,15 @@ llama_context::llama_context(
|
||||
__func__, cparams.n_ctx_seq, hparams.n_ctx_train);
|
||||
}
|
||||
|
||||
{
|
||||
const int64_t max_nodes = graph_max_nodes();
|
||||
|
||||
gf_res_reserve = std::make_unique<llm_graph_result>(max_nodes);
|
||||
for (auto & res : gf_res_prev) {
|
||||
res = std::make_unique<llm_graph_result>(max_nodes);
|
||||
}
|
||||
}
|
||||
|
||||
if (!hparams.vocab_only) {
|
||||
// GPU backends
|
||||
for (const auto & dev : model.devices) {
|
||||
@@ -797,12 +806,15 @@ void llama_context::sched_reserve() {
|
||||
|
||||
LLAMA_LOG_DEBUG("%s: max_nodes = %zu\n", __func__, max_nodes);
|
||||
|
||||
for (auto & res : gf_res_prev) {
|
||||
res.reset();
|
||||
}
|
||||
gf_res_reserve.reset(new llm_graph_result(max_nodes));
|
||||
gf_res_prev_active = nullptr;
|
||||
|
||||
gf_res_reserve->set_max_nodes(max_nodes);
|
||||
gf_res_reserve->reset();
|
||||
for (auto & res : gf_res_prev) {
|
||||
res->set_max_nodes(max_nodes);
|
||||
res->reset();
|
||||
}
|
||||
|
||||
sched_new({
|
||||
/*.max_nodes =*/ max_nodes,
|
||||
/*.parallel =*/ cparams.pipeline_parallel,
|
||||
@@ -1047,9 +1059,7 @@ bool llama_context::memory_update(bool optimize) {
|
||||
// reset the previous graph results to make sure that they won't be reused
|
||||
// TODO: make mctx->apply() report if a graph reserve is needed, then reset graph results only if the memory module reset the scheduler
|
||||
for (auto & res : gf_res_prev) {
|
||||
if (res) {
|
||||
res->reset();
|
||||
}
|
||||
res->reset();
|
||||
}
|
||||
gf_res_prev_active = nullptr;
|
||||
|
||||
@@ -2947,11 +2957,7 @@ llm_graph_result * llama_context::get_gf_res_reserve() const {
|
||||
}
|
||||
|
||||
llm_graph_result * llama_context::get_gf_res_prev() {
|
||||
auto & res = gf_res_prev[n_outputs > 0];
|
||||
if (!res) {
|
||||
res.reset(new llm_graph_result(gf_res_reserve->get_max_nodes()));
|
||||
}
|
||||
return res.get();
|
||||
return gf_res_prev[n_outputs > 0].get();
|
||||
}
|
||||
|
||||
// pack sampler outputs into as few sequences as possible before using sequences without samplers
|
||||
@@ -3025,9 +3031,7 @@ ggml_cgraph * llama_context::graph_reserve(
|
||||
|
||||
// when the scheduler is reset, we cannot reuse old graphs, so we reset the previous graph results
|
||||
for (auto & res : gf_res_prev) {
|
||||
if (res) {
|
||||
res->reset();
|
||||
}
|
||||
res->reset();
|
||||
}
|
||||
gf_res_prev_active = nullptr;
|
||||
|
||||
|
||||
Reference in new issue
Block a user