llama : fix K/V and recurrent state cleanup after failed restores (#27530)
* llama : add discard for deferred state writes * llama : add tensor zeroing helper for backends without tensor memset * llama : clear K/V data after failed sequence restore * llama : clear recurrent state data after failed sequence restore * llama : simplify discard and restore cleanup * llama : report error when abnormal cell count is found in state_read_meta * llama : clear attention state on hybrid restore failure * tests : cover failed state restore cleanup * llama : clear MLA state on dsa restore failure * tests : update test for rebased test suite * llama : clarify comment in llama_memory_recurrent::state_read
This commit is contained in:
1 parent
a1de614ba3
commit
08618ff8e7
11 files changed
+383
-17
No files matched your search
@@ -852,7 +852,12 @@ void llama_memory_recurrent::state_read(llama_io_read_i & io, llama_seq_id seq_i
|
||||
|
||||
bool res = true;
|
||||
|
||||
res = res && state_read_meta(io, cell_count, seq_id);
|
||||
// save the head of the restored cells - could be needed to clear the state
|
||||
// the head is valid only when state_read_meta() succeeded
|
||||
const bool meta_read = state_read_meta(io, cell_count, seq_id);
|
||||
const uint32_t cell_head = head;
|
||||
|
||||
res = res && meta_read;
|
||||
|
||||
try {
|
||||
res = res && state_read_data(io, cell_count);
|
||||
@@ -861,12 +866,7 @@ void llama_memory_recurrent::state_read(llama_io_read_i & io, llama_seq_id seq_i
|
||||
}
|
||||
|
||||
if (!res) {
|
||||
// TODO: fix incosistent handling of `seq_id < 0` and `seq_id == -1` in the codebase [TAG_LLAMA_SEQ_ID_NEG]
|
||||
if (seq_id == -1) {
|
||||
clear(true);
|
||||
} else {
|
||||
seq_rm(seq_id, -1, -1);
|
||||
}
|
||||
state_clear(seq_id, cell_head, meta_read ? cell_count : 0);
|
||||
throw std::runtime_error("failed to restore kv cache");
|
||||
}
|
||||
|
||||
@@ -992,6 +992,11 @@ void llama_memory_recurrent::state_write_data(llama_io_write_i & io, const std::
|
||||
bool llama_memory_recurrent::state_read_meta(llama_io_read_i & io, uint32_t cell_count, llama_seq_id dest_seq_id) {
|
||||
if (dest_seq_id != -1) {
|
||||
// single sequence
|
||||
if (cell_count > size) {
|
||||
LLAMA_LOG_ERROR("%s: not enough cells in kv cache\n", __func__);
|
||||
return false;
|
||||
}
|
||||
|
||||
seq_rm(dest_seq_id, -1, -1);
|
||||
|
||||
if (cell_count == 0) {
|
||||
@@ -1223,6 +1228,41 @@ bool llama_memory_recurrent::state_read_data(llama_io_read_i & io, uint32_t cell
|
||||
return true;
|
||||
}
|
||||
|
||||
// the cleared ranges mirror the write pattern of state_read_data() - keep both in sync
|
||||
// the transposed s layout is not handled - state_read_data() rejects it before any write
|
||||
void llama_memory_recurrent::state_clear(llama_seq_id seq_id, uint32_t cell_head, uint32_t cell_count) {
|
||||
// TODO: fix incosistent handling of `seq_id < 0` and `seq_id == -1` in the codebase [TAG_LLAMA_SEQ_ID_NEG]
|
||||
if (seq_id == -1) {
|
||||
clear(true);
|
||||
return;
|
||||
}
|
||||
|
||||
seq_rm(seq_id, -1, -1);
|
||||
|
||||
if (cell_count == 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
const uint32_t n_layer = hparams.n_layer();
|
||||
|
||||
for (uint32_t il = 0; il < n_layer; ++il) {
|
||||
if (r_l[il] != nullptr) {
|
||||
const size_t r_size_row = ggml_row_size(r_l[il]->type, hparams.n_embd_r());
|
||||
llama_clear_tensor_data(r_l[il], cell_head * r_size_row, cell_count * r_size_row);
|
||||
}
|
||||
|
||||
if (s_l[il] != nullptr) {
|
||||
const size_t s_size_row = ggml_row_size(s_l[il]->type, hparams.n_embd_s());
|
||||
llama_clear_tensor_data(s_l[il], cell_head * s_size_row, cell_count * s_size_row);
|
||||
}
|
||||
|
||||
if (p_l[il] != nullptr) {
|
||||
const size_t p_size_row = ggml_row_size(p_l[il]->type, hparams.ple_conv_state());
|
||||
llama_clear_tensor_data(p_l[il], cell_head * p_size_row, cell_count * p_size_row);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
//
|
||||
// llama_memory_recurrent_context
|
||||
//
|
||||
|
||||
Reference in new issue
Block a user