server : fix swa checkpoint invalidation logic (#30666)
* server : fix swa checkpoint invalidation logic Assisted-by: Claude * add testcase that fails with > n_past, but works with > n_prefix
This commit is contained in:
1 parent
7951f601e8
commit
f174553e16
2 files changed
+83
-2
No files matched your search
@@ -3781,6 +3781,9 @@ private:
|
||||
|
||||
llama_pos pos_next = slot.prompt.tokens.pos_next(n_past);
|
||||
|
||||
// number of cached tokens shared with the new prompt, before any checkpoint is restored
|
||||
const int n_prefix = n_past;
|
||||
|
||||
// ref: https://github.com/ggml-org/llama.cpp/pull/24110
|
||||
const bool has_new_tokens = (n_past < slot.task->n_tokens());
|
||||
|
||||
@@ -3891,10 +3894,10 @@ private:
|
||||
}
|
||||
|
||||
{
|
||||
// erase any checkpoints with pos_max > pos_next
|
||||
// erase any checkpoints with pos_max > pos_next, or that cover tokens not shared with the new prompt
|
||||
for (auto it = slot.prompt.checkpoints.begin(); it != slot.prompt.checkpoints.end();) {
|
||||
const auto & cur = *it;
|
||||
if (cur.pos_max > pos_next) {
|
||||
if (cur.pos_max > pos_next || cur.n_tokens > n_prefix) {
|
||||
SLT_TRC(slot, "erased invalidated context checkpoint (pos_min = %d, pos_max = %d, n_tokens = %" PRId64 ", n_swa = %d, pos_next = %d, size = %.3f MiB)\n", cur.pos_min, cur.pos_max, cur.n_tokens, n_swa, pos_next, (float) cur.size() / 1024 / 1024);
|
||||
it = slot.prompt.checkpoints.erase(it);
|
||||
} else {
|
||||
|
||||
Reference in new issue
Block a user