diff --git a/src/llama-kv-cache.cpp b/src/llama-kv-cache.cpp index 3d6fd2a5f..5dd0e5f02 100644 --- a/src/llama-kv-cache.cpp +++ b/src/llama-kv-cache.cpp @@ -458,19 +458,7 @@ bool llama_kv_cache::seq_rm(llama_seq_id seq_id, llama_pos p0, llama_pos p1) { auto & cells = v_cells[seq_to_stream[seq_id]]; auto & head = v_heads[seq_to_stream[seq_id]]; - uint32_t new_head = cells.size(); - - for (uint32_t i = 0; i < cells.size(); ++i) { - if (!cells.pos_in(i, p0, p1)) { - continue; - } - - if (cells.seq_has(i, seq_id) && cells.seq_rm(i, seq_id)) { - if (new_head == cells.size()) { - new_head = i; - } - } - } + const uint32_t new_head = cells.seq_rm(seq_id, p0, p1); // If we freed up a slot, set head to it so searching can start there. if (new_head != cells.size() && new_head < head) { diff --git a/src/llama-kv-cells.h b/src/llama-kv-cells.h index b0202cb28..1187cc736 100644 --- a/src/llama-kv-cells.h +++ b/src/llama-kv-cells.h @@ -308,6 +308,27 @@ public: return false; } + // remove seq_id from its cells at positions in [p0, p1) + // return the lowest index of a cell that becomes empty, or size() if none does + uint32_t seq_rm(llama_seq_id seq_id, llama_pos p0, llama_pos p1) { + assert(seq_id >= 0); + assert(seq_id < LLAMA_MAX_SEQ); + + uint32_t res = size(); + + const auto & sp = seq_pos[seq_id]; + for (auto it = sp.lower_bound({ p0, 0 }); it != sp.end() && it->first < p1; ) { + const uint32_t i = it->second; + ++it; // advance first: seq_rm(i, seq_id) erases this entry from sp + + if (seq_rm(i, seq_id)) { + res = std::min(res, i); + } + } + + return res; + } + // return true if the cell becomes empty (i.e. it did not contain seq_id before the call) bool seq_keep(uint32_t i, llama_seq_id seq_id) { assert(i < pos.size()); diff --git a/tests/test-kv-cache.cpp b/tests/test-kv-cache.cpp index c8255f572..4b48387b9 100644 --- a/tests/test-kv-cache.cpp +++ b/tests/test-kv-cache.cpp @@ -3,6 +3,7 @@ #include #include +#include static void init_model(llama_model & model, uint32_t n_layer) { model.hparams = {}; @@ -105,6 +106,75 @@ int main() { } } + // seq_rm removes [p0, p1) of one sequence, a negative bound is open + { + std::unique_ptr model(llama_model_create(LLM_ARCH_LLAMA, llama_model_default_params())); + init_model(*model, 1); + + llama_kv_cache kv(*model, false, model->hparams, GGML_TYPE_F16, GGML_TYPE_F16, + false, false, true, 32, 2, 0, LLAMA_SWA_TYPE_NONE, nullptr, nullptr, nullptr, nullptr); + + // returns the first cell that takes the tokens + auto add = [&](llama_seq_id seq_id, llama_pos p0, uint32_t n) { + std::vector token(n, 1); + std::vector pos(n); + std::vector n_seq_id(n, 1); + std::vector seq_ids(n, &seq_id); + std::iota(pos.begin(), pos.end(), p0); + + llama_ubatch ubatch = {}; + ubatch.n_tokens = n; + ubatch.n_seq_tokens = n; + ubatch.n_seqs = 1; + ubatch.n_seqs_unq = 1; + ubatch.n_pos = 1; + ubatch.token = token.data(); + ubatch.pos = pos.data(); + ubatch.n_seq_id = n_seq_id.data(); + ubatch.seq_id = seq_ids.data(); + ubatch.seq_id_unq = &seq_id; + + const auto sinfo = kv.find_slot(ubatch, false); + GGML_ASSERT(!sinfo.empty()); + kv.apply_ubatch(sinfo, ubatch); + + return sinfo.head(); + }; + + GGML_ASSERT(add(0, 0, 8) == 0); + GGML_ASSERT(add(1, 0, 4) == 8); + + // the tail of sequence 0, the other sequence keeps its cells + GGML_ASSERT(kv.seq_rm(0, 5, -1)); + GGML_ASSERT(kv.seq_pos_min(0) == 0 && kv.seq_pos_max(0) == 4); + GGML_ASSERT(kv.seq_pos_min(1) == 0 && kv.seq_pos_max(1) == 3); + + // the next tokens go to the lowest freed cell + GGML_ASSERT(add(0, 5, 2) == 5); + + // p1 is not part of the range + GGML_ASSERT(kv.seq_rm(0, -1, 2)); + GGML_ASSERT(kv.seq_pos_min(0) == 2 && kv.seq_pos_max(0) == 6); + GGML_ASSERT(kv.seq_rm(0, 6, 6)); + GGML_ASSERT(kv.seq_pos_max(0) == 6); + + // freed cells are taken in cell order, so a higher position can sit in a lower cell + GGML_ASSERT(add(0, 7, 3) == 0); + GGML_ASSERT(kv.seq_rm(0, 6, -1)); + GGML_ASSERT(add(0, 6, 1) == 0); + + // a cell shared with another sequence stays for that sequence + kv.seq_cp(0, 1, 5, -1); + GGML_ASSERT(kv.seq_rm(0, 5, -1)); + GGML_ASSERT(kv.seq_pos_max(0) == 4 && kv.seq_pos_max(1) == 6); + + // every cell of a repeated position is removed + add(0, 5, 1); + add(0, 5, 1); + GGML_ASSERT(kv.seq_rm(0, 5, 6)); + GGML_ASSERT(kv.seq_pos_max(0) == 4); + } + llama_backend_free(); return 0; }