From 40f4f4ac9d91f3269b7a35b6ca6cd28587e9d113 Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Tue, 10 Nov 2026 14:28:36 +0200 Subject: [PATCH] graph: mark every sampling chain as an output to prevent aliasing The sampling chains are read on the host after the graph compute, and the graph addresses are reused across builds with different selected rows: the token-generation reserve graph has one output row per sequence, while the actual block draft has n_max rows per sequence. The reserve allocates the unselected chains without keeping them, so their buffers alias, and the actual graph then reuses those addresses by node index. set_outputs only marked the tensors stored for the selected rows, so the unselected per-(sequence, row) chains were freed and reused. Collect every chain tensor in t_sampling_all and mark all of them as outputs there. Reproduces with -np 2 (non-unified KV) and 2 concurrent requests for draft-dspark, draft-dflash and draft-eagle3; with different prompts one sequence's acceptance collapses (eagle3 0.15 vs 0.40). After the fix: dspark 0.71/0.78, dflash 0.56/0.44, eagle3 0.35/0.42. Assisted-by: dsh:llama.cpp/ggml-org/DeepSeek-V4.1-Flash-1017-GGUF (Mac Studio, M5 Ultra, 256GB) --- src/llama-graph.cpp | 24 ++++++++---------------- src/llama-graph.h | 2 ++ 2 files changed, 10 insertions(+), 16 deletions(-) diff --git a/src/llama-graph.cpp b/src/llama-graph.cpp index b5d4642a2..f9dd3e0f6 100644 --- a/src/llama-graph.cpp +++ b/src/llama-graph.cpp @@ -1418,6 +1418,7 @@ void llm_graph_result::reset() { t_sampled_probs.clear(); t_sampled_logits.clear(); t_candidates.clear(); + t_sampling_all.clear(); params = {}; @@ -1466,22 +1467,8 @@ void llm_graph_result::set_outputs(const llm_graph_params & params) { } } } - for (auto * tensor : t_sampled) { - if (tensor != nullptr) { - ggml_set_output(tensor); - } - } - for (auto * tensor : t_sampled_probs) { - if (tensor != nullptr) { - ggml_set_output(tensor); - } - } - for (auto * tensor : t_sampled_logits) { - if (tensor != nullptr) { - ggml_set_output(tensor); - } - } - for (auto * tensor : t_candidates) { + // the chains are read on the host after compute, so keep every one alive + for (auto * tensor : t_sampling_all) { if (tensor != nullptr) { ggml_set_output(tensor); } @@ -4455,6 +4442,11 @@ void llm_graph_context::build_sampling() const { assert(sampler->iface->backend_apply); sampler->iface->backend_apply(sampler, ctx0, gf, &data); + if (data.sampled != nullptr) { res->t_sampling_all.push_back(data.sampled); } + if (data.probs != nullptr) { res->t_sampling_all.push_back(data.probs); } + if (data.logits != nullptr) { res->t_sampling_all.push_back(data.logits); } + if (data.candidates != nullptr) { res->t_sampling_all.push_back(data.candidates); } + if (data.sampled != nullptr) { if (active) { res->t_sampled[row] = data.sampled; diff --git a/src/llama-graph.h b/src/llama-graph.h index 9867c9540..da473f043 100644 --- a/src/llama-graph.h +++ b/src/llama-graph.h @@ -979,6 +979,8 @@ public: std::vector t_sampled_logits; std::vector t_candidates; + std::vector t_sampling_all; + std::vector inputs; std::vector fused_nodes;