llama : fix K/V and recurrent state cleanup after failed restores (#27530)
* llama : add discard for deferred state writes * llama : add tensor zeroing helper for backends without tensor memset * llama : clear K/V data after failed sequence restore * llama : clear recurrent state data after failed sequence restore * llama : simplify discard and restore cleanup * llama : report error when abnormal cell count is found in state_read_meta * llama : clear attention state on hybrid restore failure * tests : cover failed state restore cleanup * llama : clear MLA state on dsa restore failure * tests : update test for rebased test suite * llama : clarify comment in llama_memory_recurrent::state_read
This commit is contained in:
1 parent
a1de614ba3
commit
08618ff8e7
11 files changed
+383
-17
No files matched your search
+112
-5
@@ -2188,11 +2188,7 @@ const slot_info_vec_t * sinfos_in) {
|
||||
}
|
||||
|
||||
if (!res) {
|
||||
if (seq_id == -1) {
|
||||
clear(true);
|
||||
} else {
|
||||
seq_rm(seq_id, -1, -1);
|
||||
}
|
||||
state_clear(seq_id, strm, sinfo);
|
||||
throw std::runtime_error("failed to restore kv cache");
|
||||
}
|
||||
|
||||
@@ -2340,6 +2336,11 @@ bool llama_kv_cache::state_read_meta(llama_io_read_i & io, uint32_t strm, uint32
|
||||
|
||||
if (dest_seq_id != -1) {
|
||||
// single sequence
|
||||
if (cell_count > cells.size()) {
|
||||
LLAMA_LOG_ERROR("%s: not enough cells in kv cache\n", __func__);
|
||||
return false;
|
||||
}
|
||||
|
||||
seq_rm(dest_seq_id, -1, -1);
|
||||
|
||||
llama_batch_allocr balloc(hparams.n_pos_per_embd());
|
||||
@@ -2663,6 +2664,112 @@ bool llama_kv_cache::state_read_data(llama_io_read_i & io, uint32_t strm, uint32
|
||||
return true;
|
||||
}
|
||||
|
||||
void llama_kv_cache::state_clear(llama_seq_id seq_id) {
|
||||
if (seq_id == -1) {
|
||||
clear(true);
|
||||
return;
|
||||
}
|
||||
|
||||
GGML_ASSERT(seq_id >= 0 && (size_t) seq_id < seq_to_stream.size());
|
||||
|
||||
const uint32_t strm = seq_to_stream[seq_id];
|
||||
|
||||
const auto & cells = v_cells[strm];
|
||||
|
||||
slot_info sinfo;
|
||||
sinfo.s0 = strm;
|
||||
sinfo.s1 = strm;
|
||||
sinfo.resize(1);
|
||||
sinfo.strm[0] = strm;
|
||||
|
||||
// a cell that another sequence still uses keeps its data
|
||||
for (uint32_t i = 0; i < cells.size(); ++i) {
|
||||
if (cells.seq_has(i, seq_id) && cells.seq_count(i) == 1) {
|
||||
sinfo.idxs[0].push_back(i);
|
||||
}
|
||||
}
|
||||
|
||||
state_clear(seq_id, strm, sinfo);
|
||||
}
|
||||
|
||||
// the cleared ranges mirror the write pattern of state_read_data() - keep both in sync
|
||||
void llama_kv_cache::state_clear(llama_seq_id seq_id, uint32_t strm, const slot_info & sinfo) {
|
||||
if (seq_id == -1) {
|
||||
clear(true);
|
||||
return;
|
||||
}
|
||||
|
||||
seq_rm(seq_id, -1, -1);
|
||||
|
||||
// zero the K/V data of the failed restore attempt - the attention can still read the data of free cells
|
||||
if (sinfo.empty() || sinfo.size() == 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
const auto & cells = v_cells[strm];
|
||||
|
||||
const uint32_t cell_count = sinfo.size();
|
||||
|
||||
const bool is_contiguous = sinfo.is_contiguous();
|
||||
|
||||
for (const auto & layer : layers) {
|
||||
const uint32_t il = layer.il;
|
||||
|
||||
const uint32_t n_embd_k_gqa = hparams.n_embd_k_gqa(il);
|
||||
|
||||
auto * k = layer.k_stream[strm];
|
||||
|
||||
const size_t k_size_row = ggml_row_size(k->type, n_embd_k_gqa);
|
||||
|
||||
if (is_contiguous) {
|
||||
llama_clear_tensor_data(k, sinfo.head() * k_size_row, cell_count * k_size_row);
|
||||
} else {
|
||||
for (uint32_t i = 0; i < cell_count; ++i) {
|
||||
llama_clear_tensor_data(k, sinfo.idxs[0][i] * k_size_row, k_size_row);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for (const auto & layer : layers) {
|
||||
const uint32_t il = layer.il;
|
||||
|
||||
const uint32_t n_embd_v_gqa = hparams.n_embd_v_gqa(il);
|
||||
|
||||
auto * v = layer.v_stream[strm];
|
||||
if (!v) {
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!v_trans) {
|
||||
const size_t v_size_row = ggml_row_size(v->type, n_embd_v_gqa);
|
||||
|
||||
if (is_contiguous) {
|
||||
llama_clear_tensor_data(v, sinfo.head() * v_size_row, cell_count * v_size_row);
|
||||
} else {
|
||||
for (uint32_t i = 0; i < cell_count; ++i) {
|
||||
llama_clear_tensor_data(v, sinfo.idxs[0][i] * v_size_row, v_size_row);
|
||||
}
|
||||
}
|
||||
} else {
|
||||
const size_t v_size_el = ggml_type_size(v->type);
|
||||
|
||||
if (is_contiguous) {
|
||||
const uint32_t h = sinfo.head();
|
||||
|
||||
for (uint32_t j = 0; j < n_embd_v_gqa; ++j) {
|
||||
llama_clear_tensor_data(v, (h + j * cells.size()) * v_size_el, cell_count * v_size_el);
|
||||
}
|
||||
} else {
|
||||
for (uint32_t j = 0; j < n_embd_v_gqa; ++j) {
|
||||
for (uint32_t i = 0; i < cell_count; ++i) {
|
||||
llama_clear_tensor_data(v, (sinfo.idxs[0][i] + j * cells.size()) * v_size_el, v_size_el);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
//
|
||||
// llama_kv_cache_context
|
||||
//
|
||||
|
||||
Reference in new issue
Block a user