model: add Kimi-K3.1-Pro text model (#31779)
* model: add Kimi-K3.1-Pro text model
K3 with the full-attention MLA layers replaced by DSA: on this checkpoint 24
of the 93 layers carry a lightning indexer (32 heads x 128, top_k 2048, no
rope). The KDA (linear-attention) layers and the rest of the K3 skeleton -
cross-layer residuals, latent MoE, situ, MLA output gate, full-rank KDA gate -
are unchanged, so this reuses the K3 graph and converter and adds the indexer.
Indexer:
- the selection is an additive mask over the whole cache, not a gather of
the top-k rows, so it reuses the existing sparse flash attention path;
off that path the mask stays correct but still costs a kq over the cache
- the indexer key LayerNorm takes its eps from the model's rms_norm_eps,
because dsa_config declares no eps of its own
Graph and cache:
- new arch with four indexer tensors per DSA layer, and a hybrid memory
that keeps a K-only MLA cache for those layers, recurrent state for the
KDA layers, and the indexer key cache
- the graph node budget shares the K3 entry
Conversion:
- KimiK31Model(KimiK3Model), picked by KimiK31ForConditionalGeneration
- dsa_config becomes attention.indexer.{head_count,key_length,top_k,types}
plus the indexer LayerNorm eps; the layer list is 1-indexed like
linear_attn_config's, and the converter asserts that it covers exactly
the full-attention layers
- the routed experts stay MXFP4 and are repacked as in K3, not dequantized
- the vision tower and mm_projector are skipped: text only for now
- the arch key follows the checkpoint's own model_type ("kimi_k31") and the
loader registers KimiK31ForConditionalGeneration; the release name carries
a -Pro suffix that nothing in the config or the tensor names uses
Quantisation: the indexer's q_b, k and proj are kept out of the quant
overrides, since the top-k selection is sensitive to their precision.
Verified on an 8-layer slice against the HF checkpoint: 38 tensors checked
exactly, the indexer metadata against the config, and the selection against
full attention - identical PPL at top_k beyond the context length, divergent
once top_k is forced below it. Arch test fixture and the full arch suite pass.
* converter: avoid clutter in the k3 base
* narrow comment to match model
* update metal fusion expectations
* restore
* sync with PR 31733 refactoring
This commit is contained in:
1 parent
669204ee2b
commit
bcb0152876
15 files changed
+919
-35
No files matched your search
+36
-30
@@ -3456,6 +3456,41 @@ ggml_tensor * llm_graph_context::build_fused_top_k_mask(
|
||||
return cur;
|
||||
}
|
||||
|
||||
ggml_tensor * llm_graph_context::build_top_k_mask(
|
||||
ggml_tensor * kq_mask,
|
||||
ggml_tensor * top_k,
|
||||
int il) const {
|
||||
if (cparams.fused_top_k_mask) {
|
||||
return build_fused_top_k_mask(kq_mask, top_k, il);
|
||||
}
|
||||
|
||||
// prepare new kq mask - starts filled with -INFINITY
|
||||
ggml_tensor * kq_mask_all = ggml_fill(ctx0, kq_mask, -INFINITY);
|
||||
|
||||
// reshape KQ mask into tensor with rows of size 1:
|
||||
// [n_kv, n_batch, 1, n_stream] -> [1, n_kv, n_batch, n_stream]
|
||||
kq_mask_all = ggml_view_4d(ctx0, kq_mask_all, 1, kq_mask_all->ne[0], kq_mask_all->ne[1], kq_mask_all->ne[3], kq_mask_all->nb[0], kq_mask_all->nb[1], kq_mask_all->nb[2], 0);
|
||||
|
||||
// reshape top_k indices: [n_top_k, n_batch, 1, n_stream] -> [n_top_k, n_batch, n_stream, 1]
|
||||
ggml_tensor * top_k_3d = ggml_view_4d(ctx0, top_k, top_k->ne[0], top_k->ne[1], top_k->ne[3], 1, top_k->nb[1], top_k->nb[2], top_k->ne[3]*top_k->nb[3], 0);
|
||||
|
||||
// prepare zero-filled tensor with rows of size 1: [1, n_top_k, n_batch, n_stream]
|
||||
// this will be our source of zero values for unmasking top k mask elements
|
||||
ggml_tensor * zeros = ggml_new_tensor_4d(ctx0, GGML_TYPE_F32, 1, top_k_3d->ne[0], top_k_3d->ne[1], top_k_3d->ne[2]);
|
||||
zeros = ggml_fill(ctx0, zeros, 0.0f);
|
||||
|
||||
// modify KQ mask by unmasking elements that are in top_k indices
|
||||
// ggml_set_rows([1, n_kv, n_batch, n_stream], [1, n_top_k, n_batch, n_stream], [n_top_k, n_batch, n_stream, 1])
|
||||
ggml_tensor * kq_mask_top_k = ggml_set_rows(ctx0, kq_mask_all, zeros, top_k_3d);
|
||||
|
||||
// reshape to restore the original shape of KQ mask:
|
||||
// [1, n_kv, n_batch, n_stream] -> [n_kv, n_batch, 1, n_stream]
|
||||
kq_mask_top_k = ggml_view_4d(ctx0, kq_mask_top_k, kq_mask_top_k->ne[1], kq_mask_top_k->ne[2], 1, kq_mask_top_k->ne[3], kq_mask_top_k->nb[2], kq_mask_top_k->nb[3], kq_mask_top_k->nb[3], 0);
|
||||
|
||||
// combine with the original kq mask
|
||||
return ggml_add(ctx0, kq_mask_top_k, kq_mask);
|
||||
}
|
||||
|
||||
ggml_tensor * llm_graph_context::build_attn(
|
||||
llm_graph_input_attn_k_dsa * inp,
|
||||
ggml_tensor * wo,
|
||||
@@ -3500,36 +3535,7 @@ ggml_tensor * llm_graph_context::build_attn(
|
||||
kq_mask = inp->self_attn_mask_lid.raw;
|
||||
}
|
||||
|
||||
ggml_tensor * kq_mask_top_k = nullptr;
|
||||
if (cparams.fused_top_k_mask) {
|
||||
kq_mask_top_k = build_fused_top_k_mask(kq_mask, top_k, il);
|
||||
} else {
|
||||
// prepare new kq mask - starts filled with -INFINITY
|
||||
ggml_tensor * kq_mask_all = ggml_fill(ctx0, kq_mask, -INFINITY);
|
||||
|
||||
// reshape KQ mask into tensor with rows of size 1:
|
||||
// [n_kv, n_batch, 1, n_stream] -> [1, n_kv, n_batch, n_stream]
|
||||
kq_mask_all = ggml_view_4d(ctx0, kq_mask_all, 1, kq_mask_all->ne[0], kq_mask_all->ne[1], kq_mask_all->ne[3], kq_mask_all->nb[0], kq_mask_all->nb[1], kq_mask_all->nb[2], 0);
|
||||
|
||||
// reshape top_k indices: [n_top_k, n_batch, 1, n_stream] -> [n_top_k, n_batch, n_stream, 1]
|
||||
ggml_tensor * top_k_3d = ggml_view_4d(ctx0, top_k, top_k->ne[0], top_k->ne[1], top_k->ne[3], 1, top_k->nb[1], top_k->nb[2], top_k->ne[3]*top_k->nb[3], 0);
|
||||
|
||||
// prepare zero-filled tensor with rows of size 1: [1, n_top_k, n_batch, n_stream]
|
||||
// this will be our source of zero values for unmasking top k mask elements
|
||||
ggml_tensor * zeros = ggml_new_tensor_4d(ctx0, GGML_TYPE_F32, 1, top_k_3d->ne[0], top_k_3d->ne[1], top_k_3d->ne[2]);
|
||||
zeros = ggml_fill(ctx0, zeros, 0.0f);
|
||||
|
||||
// modify KQ mask by unmasking elements that are in top_k indices
|
||||
// ggml_set_rows([1, n_kv, n_batch, n_stream], [1, n_top_k, n_batch, n_stream], [n_top_k, n_batch, n_stream, 1])
|
||||
kq_mask_top_k = ggml_set_rows(ctx0, kq_mask_all, zeros, top_k_3d);
|
||||
|
||||
// reshape to restore the original shape of KQ mask:
|
||||
// [1, n_kv, n_batch, n_stream] -> [n_kv, n_batch, 1, n_stream]
|
||||
kq_mask_top_k = ggml_view_4d(ctx0, kq_mask_top_k, kq_mask_top_k->ne[1], kq_mask_top_k->ne[2], 1, kq_mask_top_k->ne[3], kq_mask_top_k->nb[2], kq_mask_top_k->nb[3], kq_mask_top_k->nb[3], 0);
|
||||
|
||||
// combine with the original kq mask
|
||||
kq_mask_top_k = ggml_add(ctx0, kq_mask_top_k, kq_mask);
|
||||
}
|
||||
ggml_tensor * kq_mask_top_k = build_top_k_mask(kq_mask, top_k, il);
|
||||
|
||||
ggml_tensor * q = q_cur;
|
||||
ggml_tensor * k = mctx_cur->get_k(ctx0, il);
|
||||
|
||||
Reference in new issue
Block a user