server: fix the off-by-two in the SWA reuse threshold (#30580)
The first evaluated token attends to the positions above pos - n_swa, so a SWA state starting at pos - n_swa + 1 holds the whole window, but the reuse threshold asked for two more cells. A saved SWA state keeps only the cells inside its window, so a state read back from a slot file or from the RAM prompt cache was always rejected and the slot rolled back to a checkpoint. The SWA threshold now matches the window, while a state without SWA keeps the exact bound. The test harness gains override_kv, used to give the SWA test model a window smaller than the test context. [no release]
This commit is contained in:
1 parent
a6b91d8c5e
commit
61ad80bf93
3 files changed
+95
-2
No files matched your search
@@ -3784,8 +3784,12 @@ private:
|
||||
// ref: https://github.com/ggml-org/llama.cpp/pull/24110
|
||||
const bool has_new_tokens = (n_past < slot.task->n_tokens());
|
||||
|
||||
// the largest pos_min required for a checkpoint to be useful
|
||||
const auto pos_min_thold = std::max(0, pos_next - n_swa - (has_new_tokens ? 0 : 1));
|
||||
// the first evaluated token attends to the positions > pos_eval - n_swa, so a SWA state
|
||||
// starting at pos_eval - n_swa + 1 is enough, while a state without SWA must end before pos_eval
|
||||
const llama_pos pos_eval = pos_next - (has_new_tokens ? 0 : 1);
|
||||
|
||||
// a state or a checkpoint is useful when its pos_min is below this threshold
|
||||
const auto pos_min_thold = std::max(0, n_swa > 0 ? pos_eval - n_swa + 2 : pos_eval);
|
||||
|
||||
if (n_past > 0 && n_past <= slot.prompt.n_tokens()) {
|
||||
const auto pos_min = llama_memory_seq_pos_min(llama_get_memory(ctx_tgt), slot.id);
|
||||
|
||||
@@ -838,3 +838,89 @@ def test_slot_restore_checkpoints_draft_kv_type_change(swa_server, ctkd_restore)
|
||||
})
|
||||
assert res.status_code == 200
|
||||
assert res.body["timings"]["prompt_n"] == n_live
|
||||
|
||||
|
||||
# a SWA window smaller than the prompt, so a saved state holds only the cells inside the window
|
||||
@pytest.fixture
|
||||
def swa_window_server(tmp_path):
|
||||
swa = ServerPreset.tinygemma3()
|
||||
swa.override_kv = "gemma3.attention.sliding_window=int:64"
|
||||
swa.slot_save_path = str(tmp_path)
|
||||
swa.temperature = 0.0
|
||||
swa.n_predict = 8
|
||||
return swa
|
||||
|
||||
|
||||
SWA_PROMPT = "The quick brown fox jumps over the lazy dog. " * 30
|
||||
|
||||
|
||||
def tokenize(server, text):
|
||||
res = server.make_request("POST", "/tokenize", data={"content": text})
|
||||
assert res.status_code == 200
|
||||
return res.body["tokens"]
|
||||
|
||||
|
||||
# the first turn and the exact token continuation of it, so the next turn extends the saved state
|
||||
def swa_turns(server, id_slot):
|
||||
res = server.make_request("POST", "/completion", data={
|
||||
"prompt": tokenize(server, SWA_PROMPT),
|
||||
"id_slot": id_slot,
|
||||
"cache_prompt": True,
|
||||
"return_tokens": True,
|
||||
})
|
||||
assert res.status_code == 200
|
||||
return tokenize(server, SWA_PROMPT) + res.body["tokens"] + tokenize(server, " The end.")
|
||||
|
||||
|
||||
def swa_next(server, id_slot, prompt):
|
||||
res = server.make_request("POST", "/completion", data={
|
||||
"prompt": prompt,
|
||||
"id_slot": id_slot,
|
||||
"cache_prompt": True,
|
||||
})
|
||||
assert res.status_code == 200
|
||||
return res.body["timings"]
|
||||
|
||||
|
||||
# a restored SWA state holds the window the next token needs, so it is reused as is
|
||||
def test_slot_restore_swa_window(swa_window_server):
|
||||
server = swa_window_server
|
||||
server.cache_ram = 0
|
||||
server.start()
|
||||
|
||||
prompt = {id_slot: swa_turns(server, id_slot) for id_slot in [0, 1]}
|
||||
assert prompt[0] == prompt[1]
|
||||
|
||||
res = server.make_request("POST", "/slots/0?action=save", data={"filename": "swa_slot0.bin"})
|
||||
assert res.status_code == 200
|
||||
res = server.make_request("POST", "/slots/0?action=erase")
|
||||
assert res.status_code == 200
|
||||
res = server.make_request("POST", "/slots/0?action=restore", data={"filename": "swa_slot0.bin"})
|
||||
assert res.status_code == 200
|
||||
|
||||
timings = {id_slot: swa_next(server, id_slot, prompt[id_slot]) for id_slot in [0, 1]}
|
||||
|
||||
assert timings[0]["cache_n"] == timings[1]["cache_n"]
|
||||
assert timings[0]["prompt_n"] == timings[1]["prompt_n"]
|
||||
|
||||
|
||||
# a SWA state loaded back from the RAM prompt cache is reused as is
|
||||
def test_prompt_cache_swa_window(swa_window_server):
|
||||
server = swa_window_server
|
||||
server.n_slots = 1
|
||||
|
||||
def run(other_prompt_first):
|
||||
server.start()
|
||||
prompt = swa_turns(server, 0)
|
||||
if other_prompt_first:
|
||||
res = server.make_request("POST", "/completion", data={"prompt": "Once upon a time " * 60, "cache_prompt": True})
|
||||
assert res.status_code == 200
|
||||
timings = swa_next(server, 0, prompt)
|
||||
server.stop()
|
||||
return timings
|
||||
|
||||
direct = run(False)
|
||||
cached = run(True)
|
||||
|
||||
assert cached["cache_n"] == direct["cache_n"]
|
||||
assert cached["prompt_n"] == direct["prompt_n"]
|
||||
@@ -116,6 +116,7 @@ class ServerProcess:
|
||||
media_path: str | None = None
|
||||
sleep_idle_seconds: int | None = None
|
||||
cache_ram: int | None = None
|
||||
override_kv: str | None = None
|
||||
no_cache_idle_slots: bool = False
|
||||
log_path: str | None = None
|
||||
ui_mcp_proxy: bool = False
|
||||
@@ -286,6 +287,8 @@ class ServerProcess:
|
||||
server_args.extend(["--sleep-idle-seconds", self.sleep_idle_seconds])
|
||||
if self.cache_ram is not None:
|
||||
server_args.extend(["--cache-ram", self.cache_ram])
|
||||
if self.override_kv:
|
||||
server_args.extend(["--override-kv", self.override_kv])
|
||||
if self.no_cache_idle_slots:
|
||||
server_args.append("--no-cache-idle-slots")
|
||||
if self.ui_mcp_proxy:
|
||||
|
||||
Reference in new issue
Block a user