From e7e907b7c3230b123935d1967efabea5a01dcc4d Mon Sep 17 00:00:00 2001 From: Lumpiasty Date: Sun, 22 Nov 2026 20:40:50 +0100 Subject: [PATCH] server: F14 - keep the batch break at the first user message (system-prompt checkpoint) With LLAMA_USERSTART_MERGE on (the default), F14 skipped every user-message batch break when the rest of the prompt fits in one ubatch. On a fresh prompt that also dropped the break at the first user message, so no context checkpoint was left at the end of the system prompt, and a new chat with the same system prompt re-processed the system prompt from scratch (a recurrent/hybrid state cannot be cut back without a checkpoint). Keep that one break in the merged path: the batch ends right before the first user message, and the next batch starts there and gets the checkpoint (is_user_start, near-end rule). The batch only reaches the first user message when the prompt is processed from before it: checkpoints past the restore point are erased on reuse, so this is a fresh prompt or one that diverged inside the system prompt. Follow-up turns start past it and keep the merge (no break at the last user message). Boundaries below 8 tokens (only a BOS before the first user message) are skipped. Cost: on a fresh prompt the system prompt runs as its own batch again (the upstream layout for turn 1), measured before as part of turn 1's 6.08 vs 3.99 s TTFT (FORK.md F14). The unmerged path is unchanged. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01PZz44SLQvTXMyWGio6t9DZ --- tools/server/server-context.cpp | 25 +++++++++++++++++++++++-- 1 file changed, 23 insertions(+), 2 deletions(-) diff --git a/tools/server/server-context.cpp b/tools/server/server-context.cpp index d1fc2fd93..6918f10a8 100644 --- a/tools/server/server-context.cpp +++ b/tools/server/server-context.cpp @@ -4948,6 +4948,23 @@ private: const bool merge_user_starts = userstart_merge && !has_mtmd && slot.task->n_tokens() - slot.prompt.n_tokens() <= n_ubatch; + // [fork F14] merged or not, keep the break at the first user message, the end of + // the system prompt: the checkpoint there is what a new chat with the same system + // prompt restores. The batch only reaches it when the prompt is processed from + // before it (a fresh prompt; checkpoints past the restore point are erased), so + // follow-up turns are not affected. Below 8 tokens there is no system prompt + // worth a checkpoint (only a BOS before the first user message). + int32_t first_user_pos = -1; + for (const auto & s : spans.spans) { + if (s.role == COMMON_CHAT_ROLE_USER) { + first_user_pos = (int32_t) s.pos; + break; + } + } + if (first_user_pos < 8) { + first_user_pos = -1; + } + // add prompt tokens for processing in the current batch while (slot.prompt.n_tokens() < slot.task->n_tokens() && batch.size() < n_batch) { // get next token to process @@ -4991,11 +5008,15 @@ private: slot.prompt.tokens.push_back(cur_tok); // break at the last user message, or at user messages at least min step past the last checkpoint - if (do_checkpoint && !merge_user_starts && spans.is_user_start(slot.prompt.n_tokens())) { + if (do_checkpoint && spans.is_user_start(slot.prompt.n_tokens())) { const auto pos = slot.prompt.n_tokens(); const auto & checkpoints = slot.prompt.checkpoints; - if (pos == last_user_pos || checkpoints.empty() || pos > checkpoints.back()->n_tokens + params_base.checkpoint_min_step) { + if (merge_user_starts) { + if (pos == first_user_pos) { + break; + } + } else if (pos == last_user_pos || checkpoints.empty() || pos > checkpoints.back()->n_tokens + params_base.checkpoint_min_step) { break; } }