From 225a5345bbb0e0c70859af5441ba922beb666fcf Mon Sep 17 00:00:00 2001 From: Aman Gupta Date: Sun, 22 Nov 2026 11:25:13 +0800 Subject: [PATCH] server : fit non-causal batches in one ubatch (#32713) --- tools/server/server-context.cpp | 4 +++- tools/server/tests/unit/test_systemone.py | 24 +++++++++++++++++++++++ 2 files changed, 27 insertions(+), 1 deletion(-) diff --git a/tools/server/server-context.cpp b/tools/server/server-context.cpp index e37661a9a..b458a17c4 100644 --- a/tools/server/server-context.cpp +++ b/tools/server/server-context.cpp @@ -4285,7 +4285,9 @@ private: if (!slot.can_split()) { // cannot fit the prompt in the current batch - will try next iter - if (batch.size() + slot.task->n_tokens() > n_batch) { + // with non-causal attention the whole batch must fit in one ubatch, see llama_decode + const int32_t n_batch_max = llama_get_causal_attn(ctx_tgt) ? n_batch : std::min(n_batch, n_ubatch); + if (batch.size() + slot.task->n_tokens() > n_batch_max) { return; } } diff --git a/tools/server/tests/unit/test_systemone.py b/tools/server/tests/unit/test_systemone.py index b0b5715e9..96f70632b 100644 --- a/tools/server/tests/unit/test_systemone.py +++ b/tools/server/tests/unit/test_systemone.py @@ -375,6 +375,30 @@ def test_systemone_d1_parallel(): assert_answers_close(a[qid], b[qid], 1e-6) +def test_systemone_noncausal_parallel(): + global server + server = ServerPreset.tinydecision("vllm-d3") + # n_batch > n_ubatch: each prompt fits in one ubatch, but several of them do not + server.n_batch = 2048 + server.n_ubatch = 512 + server.n_slots = 4 + server.start() + state = " ".join(f"The order {i} arrived damaged and the customer wants a replacement." for i in range(14)) + questions = {"angry": TEST_QUESTIONS["angry"]} + + def decide(): + res = server.make_request("POST", "/v1/systemone", data={"state": state, "questions": questions}) + assert res.status_code == 200 + assert res.body["usage"]["input_tokens"] > 512 // 2 + return res.body["answers"] + + # a non-causal batch is evaluated in one ubatch, the concurrent prompts are spread over several batches + single = decide() + parallel = parallel_function_calls([(decide, ()) for _ in range(4)]) + for answers in parallel: + assert_answers_close(single["angry"], answers["angry"], 1e-6) + + def test_systemone_d1_special_tokens(): global server server = ServerPreset.tinydecision("lfm3-d1-omni")