server: fix the off-by-two in the SWA reuse threshold (#30580)

The first evaluated token attends to the positions above
pos - n_swa, so a SWA state starting at pos - n_swa + 1 holds the
whole window, but the reuse threshold asked for two more cells.
A saved SWA state keeps only the cells inside its window, so a
state read back from a slot file or from the RAM prompt cache was
always rejected and the slot rolled back to a checkpoint.

The SWA threshold now matches the window, while a state without SWA
keeps the exact bound. The test harness gains override_kv, used to
give the SWA test model a window smaller than the test context.

[no release]
This commit is contained in:
Pascal authored and GitHub committed 2026-10-15 16:25:19 +03:00
1 parent a6b91d8c5e
commit 61ad80bf93
3 files changed
+95 -2

No files matched your search

+6 -2
View File
@@ -3784,8 +3784,12 @@ private:
// ref: https://github.com/ggml-org/llama.cpp/pull/24110
const bool has_new_tokens = (n_past < slot.task->n_tokens());
// the largest pos_min required for a checkpoint to be useful
const auto pos_min_thold = std::max(0, pos_next - n_swa - (has_new_tokens ? 0 : 1));
// the first evaluated token attends to the positions > pos_eval - n_swa, so a SWA state
// starting at pos_eval - n_swa + 1 is enough, while a state without SWA must end before pos_eval
const llama_pos pos_eval = pos_next - (has_new_tokens ? 0 : 1);
// a state or a checkpoint is useful when its pos_min is below this threshold
const auto pos_min_thold = std::max(0, n_swa > 0 ? pos_eval - n_swa + 2 : pos_eval);
if (n_past > 0 && n_past <= slot.prompt.n_tokens()) {
const auto pos_min = llama_memory_seq_pos_min(llama_get_memory(ctx_tgt), slot.id);
+86
View File
@@ -838,3 +838,89 @@ def test_slot_restore_checkpoints_draft_kv_type_change(swa_server, ctkd_restore)
})
assert res.status_code == 200
assert res.body["timings"]["prompt_n"] == n_live
# a SWA window smaller than the prompt, so a saved state holds only the cells inside the window
@pytest.fixture
def swa_window_server(tmp_path):
swa = ServerPreset.tinygemma3()
swa.override_kv = "gemma3.attention.sliding_window=int:64"
swa.slot_save_path = str(tmp_path)
swa.temperature = 0.0
swa.n_predict = 8
return swa
SWA_PROMPT = "The quick brown fox jumps over the lazy dog. " * 30
def tokenize(server, text):
res = server.make_request("POST", "/tokenize", data={"content": text})
assert res.status_code == 200
return res.body["tokens"]
# the first turn and the exact token continuation of it, so the next turn extends the saved state
def swa_turns(server, id_slot):
res = server.make_request("POST", "/completion", data={
"prompt": tokenize(server, SWA_PROMPT),
"id_slot": id_slot,
"cache_prompt": True,
"return_tokens": True,
})
assert res.status_code == 200
return tokenize(server, SWA_PROMPT) + res.body["tokens"] + tokenize(server, " The end.")
def swa_next(server, id_slot, prompt):
res = server.make_request("POST", "/completion", data={
"prompt": prompt,
"id_slot": id_slot,
"cache_prompt": True,
})
assert res.status_code == 200
return res.body["timings"]
# a restored SWA state holds the window the next token needs, so it is reused as is
def test_slot_restore_swa_window(swa_window_server):
server = swa_window_server
server.cache_ram = 0
server.start()
prompt = {id_slot: swa_turns(server, id_slot) for id_slot in [0, 1]}
assert prompt[0] == prompt[1]
res = server.make_request("POST", "/slots/0?action=save", data={"filename": "swa_slot0.bin"})
assert res.status_code == 200
res = server.make_request("POST", "/slots/0?action=erase")
assert res.status_code == 200
res = server.make_request("POST", "/slots/0?action=restore", data={"filename": "swa_slot0.bin"})
assert res.status_code == 200
timings = {id_slot: swa_next(server, id_slot, prompt[id_slot]) for id_slot in [0, 1]}
assert timings[0]["cache_n"] == timings[1]["cache_n"]
assert timings[0]["prompt_n"] == timings[1]["prompt_n"]
# a SWA state loaded back from the RAM prompt cache is reused as is
def test_prompt_cache_swa_window(swa_window_server):
server = swa_window_server
server.n_slots = 1
def run(other_prompt_first):
server.start()
prompt = swa_turns(server, 0)
if other_prompt_first:
res = server.make_request("POST", "/completion", data={"prompt": "Once upon a time " * 60, "cache_prompt": True})
assert res.status_code == 200
timings = swa_next(server, 0, prompt)
server.stop()
return timings
direct = run(False)
cached = run(True)
assert cached["cache_n"] == direct["cache_n"]
assert cached["prompt_n"] == direct["prompt_n"]
+3
View File
@@ -116,6 +116,7 @@ class ServerProcess:
media_path: str | None = None
sleep_idle_seconds: int | None = None
cache_ram: int | None = None
override_kv: str | None = None
no_cache_idle_slots: bool = False
log_path: str | None = None
ui_mcp_proxy: bool = False
@@ -286,6 +287,8 @@ class ServerProcess:
server_args.extend(["--sleep-idle-seconds", self.sleep_idle_seconds])
if self.cache_ram is not None:
server_args.extend(["--cache-ram", self.cache_ram])
if self.override_kv:
server_args.extend(["--override-kv", self.override_kv])
if self.no_cache_idle_slots:
server_args.append("--no-cache-idle-slots")
if self.ui_mcp_proxy: