llama : flash_attn_ext_rows - more efficient unified kv for multi-seq (#29510)
* llama : use llm_attn_mask and kv_rows for unified attention Assisted-by: dsh:llama.cpp/ggml-org/DeepSeek-V4-Flash-Vision-Exp-GGUF (Mac Studio, M2 Ultra, 192GB) Co-authored-by: Georgi Gerganov <ggerganov@gmail.com> * cont : optimize `get_seq_cells` * llama : build the kv_rows lists from per-sequence cell bitmaps Keep a bitmap over the cells for each sequence (the transpose of `seq`), updated in seq_pos_inc/dec, and build the cell lists by walking its set bits. The lists stay in cell order, cells shared by several sequences go to every list, and nothing has to be rebuilt when cells are removed. Also leave out of the lists the cells that the SWA hides from every token of the slice (standard and chunked SWA). * cont : make get_kv_rows_p_min private * cont : minor metal * llama : use kv_rows for every ubatch of a unified cache A ubatch made of equal-length runs with a single sequence each (f.ex. the drafts of several slots) now splits into per-sequence slices. Any other ubatch (interleaved, uneven or coupled sequences) becomes a single slice over the cells of all its sequences instead of falling back to the dense KQ mask, so the graph keeps the topology of the reserved graph. * llama : merge get_kv_rows_seq_ids and get_kv_rows_p_min into get_kv_rows_slices Assisted-by: dsh:llama.cpp/ggml-org/GLM-5.3-Flash-GGUF (Mac Studio, M5 Ultra, 256GB) --------- Co-authored-by: Georgi Gerganov <ggerganov@gmail.com>
This commit is contained in:
1 parent
3f8af15a76
commit
1b6935b227
49 files changed
+1207
-333
No files matched your search
+9
-9
@@ -321,7 +321,7 @@ public:
|
||||
using llm_graph_input_attn_no_cache::llm_graph_input_attn_no_cache;
|
||||
|
||||
void set_input(const llama_ubatch * ubatch) override {
|
||||
const int64_t n_kv = self_kq_mask->ne[0];
|
||||
const int64_t n_kv = self_attn_mask.raw->ne[0];
|
||||
const int64_t n_tokens = ubatch->n_tokens;
|
||||
|
||||
const auto fill_mask = [&](auto * data, auto zero, auto ninf) {
|
||||
@@ -333,11 +333,11 @@ public:
|
||||
}
|
||||
};
|
||||
|
||||
GGML_ASSERT(ggml_backend_buffer_is_host(self_kq_mask->buffer));
|
||||
if (self_kq_mask->type == GGML_TYPE_F16) {
|
||||
fill_mask((ggml_fp16_t *) self_kq_mask->data, ggml_fp32_to_fp16(0.0f), ggml_fp32_to_fp16(-INFINITY));
|
||||
GGML_ASSERT(ggml_backend_buffer_is_host(self_attn_mask.raw->buffer));
|
||||
if (self_attn_mask.raw->type == GGML_TYPE_F16) {
|
||||
fill_mask((ggml_fp16_t *) self_attn_mask.raw->data, ggml_fp32_to_fp16(0.0f), ggml_fp32_to_fp16(-INFINITY));
|
||||
} else {
|
||||
fill_mask((float *) self_kq_mask->data, 0.0f, -INFINITY);
|
||||
fill_mask((float *) self_attn_mask.raw->data, 0.0f, -INFINITY);
|
||||
}
|
||||
}
|
||||
};
|
||||
@@ -348,11 +348,11 @@ llm_graph_input_attn_no_cache * llama_model_clef::graph::build_attn_inp_causal()
|
||||
|
||||
const auto type_mask = cparams.flash_attn ? GGML_TYPE_F16 : GGML_TYPE_F32;
|
||||
|
||||
inp->self_kq_mask = ggml_new_tensor_4d(ctx0, type_mask, GGML_PAD(n_tokens, 256), n_tokens, 1, 1);
|
||||
ggml_set_input(inp->self_kq_mask);
|
||||
cb(inp->self_kq_mask, "self_kq_mask", -1);
|
||||
inp->self_attn_mask.raw = ggml_new_tensor_4d(ctx0, type_mask, GGML_PAD(n_tokens, 256), n_tokens, 1, 1);
|
||||
ggml_set_input(inp->self_attn_mask.raw);
|
||||
cb(inp->self_attn_mask.raw, "self_kq_mask", -1);
|
||||
|
||||
inp->self_kq_mask_cnv = inp->self_kq_mask;
|
||||
inp->self_attn_mask.cnv = inp->self_attn_mask.raw;
|
||||
|
||||
return (llm_graph_input_attn_no_cache *) res->add_input(std::move(inp));
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user