kv-cells: avoid an O(n_cells) scan per sequence removal (#32380)
This commit is contained in:
1 parent
a946eb541a
commit
21fa9c043f
3 files changed
+92
-13
No files matched your search
+1
-13
@@ -458,19 +458,7 @@ bool llama_kv_cache::seq_rm(llama_seq_id seq_id, llama_pos p0, llama_pos p1) {
|
||||
auto & cells = v_cells[seq_to_stream[seq_id]];
|
||||
auto & head = v_heads[seq_to_stream[seq_id]];
|
||||
|
||||
uint32_t new_head = cells.size();
|
||||
|
||||
for (uint32_t i = 0; i < cells.size(); ++i) {
|
||||
if (!cells.pos_in(i, p0, p1)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
if (cells.seq_has(i, seq_id) && cells.seq_rm(i, seq_id)) {
|
||||
if (new_head == cells.size()) {
|
||||
new_head = i;
|
||||
}
|
||||
}
|
||||
}
|
||||
const uint32_t new_head = cells.seq_rm(seq_id, p0, p1);
|
||||
|
||||
// If we freed up a slot, set head to it so searching can start there.
|
||||
if (new_head != cells.size() && new_head < head) {
|
||||
|
||||
Reference in new issue
Block a user