graph: mark every sampling chain as an output to prevent aliasing
The sampling chains are read on the host after the graph compute, and the graph addresses are reused across builds with different selected rows: the token-generation reserve graph has one output row per sequence, while the actual block draft has n_max rows per sequence. The reserve allocates the unselected chains without keeping them, so their buffers alias, and the actual graph then reuses those addresses by node index. set_outputs only marked the tensors stored for the selected rows, so the unselected per-(sequence, row) chains were freed and reused. Collect every chain tensor in t_sampling_all and mark all of them as outputs there. Reproduces with -np 2 (non-unified KV) and 2 concurrent requests for draft-dspark, draft-dflash and draft-eagle3; with different prompts one sequence's acceptance collapses (eagle3 0.15 vs 0.40). After the fix: dspark 0.71/0.78, dflash 0.56/0.44, eagle3 0.35/0.42. Assisted-by: dsh:llama.cpp/ggml-org/DeepSeek-V4.1-Flash-1017-GGUF (Mac Studio, M5 Ultra, 256GB)
This commit is contained in:
1 parent
a691b23458
commit
40f4f4ac9d
2 files changed
+10
-16
No files matched your search
+8
-16
@@ -1418,6 +1418,7 @@ void llm_graph_result::reset() {
|
||||
t_sampled_probs.clear();
|
||||
t_sampled_logits.clear();
|
||||
t_candidates.clear();
|
||||
t_sampling_all.clear();
|
||||
|
||||
params = {};
|
||||
|
||||
@@ -1466,22 +1467,8 @@ void llm_graph_result::set_outputs(const llm_graph_params & params) {
|
||||
}
|
||||
}
|
||||
}
|
||||
for (auto * tensor : t_sampled) {
|
||||
if (tensor != nullptr) {
|
||||
ggml_set_output(tensor);
|
||||
}
|
||||
}
|
||||
for (auto * tensor : t_sampled_probs) {
|
||||
if (tensor != nullptr) {
|
||||
ggml_set_output(tensor);
|
||||
}
|
||||
}
|
||||
for (auto * tensor : t_sampled_logits) {
|
||||
if (tensor != nullptr) {
|
||||
ggml_set_output(tensor);
|
||||
}
|
||||
}
|
||||
for (auto * tensor : t_candidates) {
|
||||
// the chains are read on the host after compute, so keep every one alive
|
||||
for (auto * tensor : t_sampling_all) {
|
||||
if (tensor != nullptr) {
|
||||
ggml_set_output(tensor);
|
||||
}
|
||||
@@ -4455,6 +4442,11 @@ void llm_graph_context::build_sampling() const {
|
||||
assert(sampler->iface->backend_apply);
|
||||
sampler->iface->backend_apply(sampler, ctx0, gf, &data);
|
||||
|
||||
if (data.sampled != nullptr) { res->t_sampling_all.push_back(data.sampled); }
|
||||
if (data.probs != nullptr) { res->t_sampling_all.push_back(data.probs); }
|
||||
if (data.logits != nullptr) { res->t_sampling_all.push_back(data.logits); }
|
||||
if (data.candidates != nullptr) { res->t_sampling_all.push_back(data.candidates); }
|
||||
|
||||
if (data.sampled != nullptr) {
|
||||
if (active) {
|
||||
res->t_sampled[row] = data.sampled;
|
||||
|
||||
Reference in new issue
Block a user