llama : fix K/V and recurrent state cleanup after failed restores (#27530)

* llama : add discard for deferred state writes

* llama : add tensor zeroing helper for backends without tensor memset

* llama : clear K/V data after failed sequence restore

* llama : clear recurrent state data after failed sequence restore

* llama : simplify discard and restore cleanup

* llama : report error when abnormal cell count is found in state_read_meta

* llama : clear attention state on hybrid restore failure

* tests : cover failed state restore cleanup

* llama : clear MLA state on dsa restore failure

* tests : update test for rebased test suite

* llama : clarify comment in llama_memory_recurrent::state_read
This commit is contained in:
Chipmunk authored and GitHub committed 2026-09-26 10:23:03 +03:00
1 parent a1de614ba3
commit 08618ff8e7
11 files changed
+383 -17

No files matched your search

+112 -5
View File
@@ -2188,11 +2188,7 @@ const slot_info_vec_t * sinfos_in) {
}
if (!res) {
if (seq_id == -1) {
clear(true);
} else {
seq_rm(seq_id, -1, -1);
}
state_clear(seq_id, strm, sinfo);
throw std::runtime_error("failed to restore kv cache");
}
@@ -2340,6 +2336,11 @@ bool llama_kv_cache::state_read_meta(llama_io_read_i & io, uint32_t strm, uint32
if (dest_seq_id != -1) {
// single sequence
if (cell_count > cells.size()) {
LLAMA_LOG_ERROR("%s: not enough cells in kv cache\n", __func__);
return false;
}
seq_rm(dest_seq_id, -1, -1);
llama_batch_allocr balloc(hparams.n_pos_per_embd());
@@ -2663,6 +2664,112 @@ bool llama_kv_cache::state_read_data(llama_io_read_i & io, uint32_t strm, uint32
return true;
}
void llama_kv_cache::state_clear(llama_seq_id seq_id) {
if (seq_id == -1) {
clear(true);
return;
}
GGML_ASSERT(seq_id >= 0 && (size_t) seq_id < seq_to_stream.size());
const uint32_t strm = seq_to_stream[seq_id];
const auto & cells = v_cells[strm];
slot_info sinfo;
sinfo.s0 = strm;
sinfo.s1 = strm;
sinfo.resize(1);
sinfo.strm[0] = strm;
// a cell that another sequence still uses keeps its data
for (uint32_t i = 0; i < cells.size(); ++i) {
if (cells.seq_has(i, seq_id) && cells.seq_count(i) == 1) {
sinfo.idxs[0].push_back(i);
}
}
state_clear(seq_id, strm, sinfo);
}
// the cleared ranges mirror the write pattern of state_read_data() - keep both in sync
void llama_kv_cache::state_clear(llama_seq_id seq_id, uint32_t strm, const slot_info & sinfo) {
if (seq_id == -1) {
clear(true);
return;
}
seq_rm(seq_id, -1, -1);
// zero the K/V data of the failed restore attempt - the attention can still read the data of free cells
if (sinfo.empty() || sinfo.size() == 0) {
return;
}
const auto & cells = v_cells[strm];
const uint32_t cell_count = sinfo.size();
const bool is_contiguous = sinfo.is_contiguous();
for (const auto & layer : layers) {
const uint32_t il = layer.il;
const uint32_t n_embd_k_gqa = hparams.n_embd_k_gqa(il);
auto * k = layer.k_stream[strm];
const size_t k_size_row = ggml_row_size(k->type, n_embd_k_gqa);
if (is_contiguous) {
llama_clear_tensor_data(k, sinfo.head() * k_size_row, cell_count * k_size_row);
} else {
for (uint32_t i = 0; i < cell_count; ++i) {
llama_clear_tensor_data(k, sinfo.idxs[0][i] * k_size_row, k_size_row);
}
}
}
for (const auto & layer : layers) {
const uint32_t il = layer.il;
const uint32_t n_embd_v_gqa = hparams.n_embd_v_gqa(il);
auto * v = layer.v_stream[strm];
if (!v) {
continue;
}
if (!v_trans) {
const size_t v_size_row = ggml_row_size(v->type, n_embd_v_gqa);
if (is_contiguous) {
llama_clear_tensor_data(v, sinfo.head() * v_size_row, cell_count * v_size_row);
} else {
for (uint32_t i = 0; i < cell_count; ++i) {
llama_clear_tensor_data(v, sinfo.idxs[0][i] * v_size_row, v_size_row);
}
}
} else {
const size_t v_size_el = ggml_type_size(v->type);
if (is_contiguous) {
const uint32_t h = sinfo.head();
for (uint32_t j = 0; j < n_embd_v_gqa; ++j) {
llama_clear_tensor_data(v, (h + j * cells.size()) * v_size_el, cell_count * v_size_el);
}
} else {
for (uint32_t j = 0; j < n_embd_v_gqa; ++j) {
for (uint32_t i = 0; i < cell_count; ++i) {
llama_clear_tensor_data(v, (sinfo.idxs[0][i] + j * cells.size()) * v_size_el, v_size_el);
}
}
}
}
}
}
//
// llama_kv_cache_context
//