From 61ad80bf93de0ac03a29f83f5f3e9f596bf40c2d Mon Sep 17 00:00:00 2001 From: Pascal Date: Thu, 15 Oct 2026 15:25:19 +0200 Subject: [PATCH] server: fix the off-by-two in the SWA reuse threshold (#30580) The first evaluated token attends to the positions above pos - n_swa, so a SWA state starting at pos - n_swa + 1 holds the whole window, but the reuse threshold asked for two more cells. A saved SWA state keeps only the cells inside its window, so a state read back from a slot file or from the RAM prompt cache was always rejected and the slot rolled back to a checkpoint. The SWA threshold now matches the window, while a state without SWA keeps the exact bound. The test harness gains override_kv, used to give the SWA test model a window smaller than the test context. [no release] --- tools/server/server-context.cpp | 8 ++- tools/server/tests/unit/test_slot_save.py | 86 +++++++++++++++++++++++ tools/server/tests/utils.py | 3 + 3 files changed, 95 insertions(+), 2 deletions(-) diff --git a/tools/server/server-context.cpp b/tools/server/server-context.cpp index c20167b90..a18b7542c 100644 --- a/tools/server/server-context.cpp +++ b/tools/server/server-context.cpp @@ -3784,8 +3784,12 @@ private: // ref: https://github.com/ggml-org/llama.cpp/pull/24110 const bool has_new_tokens = (n_past < slot.task->n_tokens()); - // the largest pos_min required for a checkpoint to be useful - const auto pos_min_thold = std::max(0, pos_next - n_swa - (has_new_tokens ? 0 : 1)); + // the first evaluated token attends to the positions > pos_eval - n_swa, so a SWA state + // starting at pos_eval - n_swa + 1 is enough, while a state without SWA must end before pos_eval + const llama_pos pos_eval = pos_next - (has_new_tokens ? 0 : 1); + + // a state or a checkpoint is useful when its pos_min is below this threshold + const auto pos_min_thold = std::max(0, n_swa > 0 ? pos_eval - n_swa + 2 : pos_eval); if (n_past > 0 && n_past <= slot.prompt.n_tokens()) { const auto pos_min = llama_memory_seq_pos_min(llama_get_memory(ctx_tgt), slot.id); diff --git a/tools/server/tests/unit/test_slot_save.py b/tools/server/tests/unit/test_slot_save.py index 793f1a0a7..488b63701 100644 --- a/tools/server/tests/unit/test_slot_save.py +++ b/tools/server/tests/unit/test_slot_save.py @@ -838,3 +838,89 @@ def test_slot_restore_checkpoints_draft_kv_type_change(swa_server, ctkd_restore) }) assert res.status_code == 200 assert res.body["timings"]["prompt_n"] == n_live + + +# a SWA window smaller than the prompt, so a saved state holds only the cells inside the window +@pytest.fixture +def swa_window_server(tmp_path): + swa = ServerPreset.tinygemma3() + swa.override_kv = "gemma3.attention.sliding_window=int:64" + swa.slot_save_path = str(tmp_path) + swa.temperature = 0.0 + swa.n_predict = 8 + return swa + + +SWA_PROMPT = "The quick brown fox jumps over the lazy dog. " * 30 + + +def tokenize(server, text): + res = server.make_request("POST", "/tokenize", data={"content": text}) + assert res.status_code == 200 + return res.body["tokens"] + + +# the first turn and the exact token continuation of it, so the next turn extends the saved state +def swa_turns(server, id_slot): + res = server.make_request("POST", "/completion", data={ + "prompt": tokenize(server, SWA_PROMPT), + "id_slot": id_slot, + "cache_prompt": True, + "return_tokens": True, + }) + assert res.status_code == 200 + return tokenize(server, SWA_PROMPT) + res.body["tokens"] + tokenize(server, " The end.") + + +def swa_next(server, id_slot, prompt): + res = server.make_request("POST", "/completion", data={ + "prompt": prompt, + "id_slot": id_slot, + "cache_prompt": True, + }) + assert res.status_code == 200 + return res.body["timings"] + + +# a restored SWA state holds the window the next token needs, so it is reused as is +def test_slot_restore_swa_window(swa_window_server): + server = swa_window_server + server.cache_ram = 0 + server.start() + + prompt = {id_slot: swa_turns(server, id_slot) for id_slot in [0, 1]} + assert prompt[0] == prompt[1] + + res = server.make_request("POST", "/slots/0?action=save", data={"filename": "swa_slot0.bin"}) + assert res.status_code == 200 + res = server.make_request("POST", "/slots/0?action=erase") + assert res.status_code == 200 + res = server.make_request("POST", "/slots/0?action=restore", data={"filename": "swa_slot0.bin"}) + assert res.status_code == 200 + + timings = {id_slot: swa_next(server, id_slot, prompt[id_slot]) for id_slot in [0, 1]} + + assert timings[0]["cache_n"] == timings[1]["cache_n"] + assert timings[0]["prompt_n"] == timings[1]["prompt_n"] + + +# a SWA state loaded back from the RAM prompt cache is reused as is +def test_prompt_cache_swa_window(swa_window_server): + server = swa_window_server + server.n_slots = 1 + + def run(other_prompt_first): + server.start() + prompt = swa_turns(server, 0) + if other_prompt_first: + res = server.make_request("POST", "/completion", data={"prompt": "Once upon a time " * 60, "cache_prompt": True}) + assert res.status_code == 200 + timings = swa_next(server, 0, prompt) + server.stop() + return timings + + direct = run(False) + cached = run(True) + + assert cached["cache_n"] == direct["cache_n"] + assert cached["prompt_n"] == direct["prompt_n"] diff --git a/tools/server/tests/utils.py b/tools/server/tests/utils.py index b3207d387..1c84f5121 100644 --- a/tools/server/tests/utils.py +++ b/tools/server/tests/utils.py @@ -116,6 +116,7 @@ class ServerProcess: media_path: str | None = None sleep_idle_seconds: int | None = None cache_ram: int | None = None + override_kv: str | None = None no_cache_idle_slots: bool = False log_path: str | None = None ui_mcp_proxy: bool = False @@ -286,6 +287,8 @@ class ServerProcess: server_args.extend(["--sleep-idle-seconds", self.sleep_idle_seconds]) if self.cache_ram is not None: server_args.extend(["--cache-ram", self.cache_ram]) + if self.override_kv: + server_args.extend(["--override-kv", self.override_kv]) if self.no_cache_idle_slots: server_args.append("--no-cache-idle-slots") if self.ui_mcp_proxy: