graph: store the MTP conv snapshots of every recurrent model in one set_rows (#31660)

* graph: store the conv state of every rollback arch in one set_rows

The rollback slots of the causal conv history were written with one
copy per slot in qwen4exp, glm5-next, kimi-k3, bailingmoe3, lfm and
mamba2, while delta_net already wrote them with a single set_rows.
build_conv_state_store now does this for all of them: one copy without
rollback, otherwise one set_rows of the overlapping windows into the
min(n, n_rs_seq + 1) slots a rollback can read.

The slot order {n_rs_seq, ..., 0} is computed once per graph and each
conv layer takes a view of its tail. Its size does not depend on the
ubatch, so the reserved graph covers every ubatch shape, and the graph
input that held it is gone.

A model that is not causal never rolls back, so the context clamps
n_rs_seq to 0 for it, like for an arch without rollback support.

* nits: compute the conv slot order in the graph and keep the n_head name of bailingmoe3

* graph: build the conv slot order in each conv layer

The slot order {K-1, ..., 0} of the conv state store was one tensor
shared by every conv layer of the graph, built on the device of the
first one, so each layer on another device copied the view of its
tail. Each conv layer now builds its own order next to its set_rows,
and llm_graph_input_rs no longer holds it.
This commit is contained in:
Pascal authored and GitHub committed 2026-11-05 09:28:03 +01:00
1 parent d4c345d6c4
commit 040ce199a1
11 files changed
+127 -238

No files matched your search

+15 -41
View File
@@ -160,24 +160,10 @@ std::unique_ptr<llm_graph_context> llama_model_bailingmoe3::build_arch_graph(con
return std::make_unique<graph>(*this, params);
}
static ggml_tensor * bailingmoe3_causal_conv1d(
ggml_cgraph * gf,
ggml_context * ctx0,
ggml_tensor * conv_states_all,
ggml_tensor * conv_state_all,
int64_t qkv,
ggml_tensor * x,
ggml_tensor * proj_w,
ggml_tensor * conv_w,
int64_t d_conv,
int64_t head_dim,
int64_t n_head,
int64_t n_seq_tokens,
int64_t n_seqs,
int64_t n_tokens,
int64_t cache_head,
uint32_t mem_size,
uint32_t n_rs_seq) {
ggml_tensor * llama_model_bailingmoe3::graph::build_causal_conv1d(
llm_graph_input_rs * inp_rs, ggml_tensor * conv_states_all, ggml_tensor * conv_state_all,
int64_t qkv, ggml_tensor * x, ggml_tensor * proj_w, ggml_tensor * conv_w,
int64_t d_conv, int64_t head_dim, int64_t n_head, int64_t n_seq_tokens, int64_t n_seqs) {
const int64_t d_inner = head_dim * n_head;
const int64_t conv_state_size = (d_conv - 1) * d_inner;
const int64_t total_state_size = 3 * conv_state_size;
@@ -191,18 +177,8 @@ static ggml_tensor * bailingmoe3_causal_conv1d(
x_proj = ggml_reshape_3d(ctx0, x_proj, d_inner, n_seq_tokens, n_seqs);
ggml_tensor * conv_x = ggml_concat(ctx0, conv_state, ggml_transpose(ctx0, x_proj), 0);
const int64_t K = (int64_t) n_rs_seq + 1;
const int64_t n_written = std::min<int64_t>(n_seq_tokens, K);
for (int64_t slot = 0; slot < n_written; ++slot) {
ggml_tensor * conv_snap = ggml_view_3d(ctx0, conv_x, d_conv - 1, d_inner, n_seqs,
conv_x->nb[1], conv_x->nb[2], (conv_x->ne[0] - (d_conv - 1) - slot) * conv_x->nb[0]);
ggml_build_forward_expand(gf, ggml_cpy(ctx0, conv_snap,
ggml_view_3d(ctx0, conv_states_all, d_conv - 1, d_inner, n_seqs,
(d_conv - 1) * ggml_element_size(conv_states_all),
total_state_size * ggml_element_size(conv_states_all),
((slot * mem_size + cache_head) * total_state_size + qkv * conv_state_size) * ggml_element_size(conv_states_all))));
}
build_conv_state_store(inp_rs, conv_states_all, conv_x, d_conv - 1,
ggml_row_size(conv_states_all->type, qkv * conv_state_size));
ggml_tensor * conv_weight = ggml_reshape_2d(ctx0, conv_w, d_conv, d_inner);
ggml_tensor * out = ggml_ssm_conv(ctx0, conv_x, conv_weight);
@@ -253,20 +229,18 @@ llama_model_bailingmoe3::graph::graph(const llama_model & model, const llm_graph
if (hparams.is_recr(il)) {
const auto * mctx_cur = inp_rs->mctx;
const auto cache_head = mctx_cur->get_head();
const auto mem_size = mctx_cur->get_size();
ggml_tensor * conv_states_all = mctx_cur->get_r_l(il);
ggml_tensor * conv_state_all = build_rs(inp_rs, conv_states_all, hparams.n_embd_r(), n_seqs);
ggml_tensor * q = bailingmoe3_causal_conv1d(
gf, ctx0, conv_states_all, conv_state_all, 0, cur, layer.wq, layer.ssm_q_conv,
d_conv, head_dim, n_head, n_seq_tokens, n_seqs, n_tokens, cache_head, mem_size, cparams.n_rs_seq);
ggml_tensor * k = bailingmoe3_causal_conv1d(
gf, ctx0, conv_states_all, conv_state_all, 1, cur, layer.wk, layer.ssm_k_conv,
d_conv, head_dim, n_head, n_seq_tokens, n_seqs, n_tokens, cache_head, mem_size, cparams.n_rs_seq);
ggml_tensor * v = bailingmoe3_causal_conv1d(
gf, ctx0, conv_states_all, conv_state_all, 2, cur, layer.wv, layer.ssm_v_conv,
d_conv, head_dim, n_head, n_seq_tokens, n_seqs, n_tokens, cache_head, mem_size, cparams.n_rs_seq);
ggml_tensor * q = build_causal_conv1d(
inp_rs, conv_states_all, conv_state_all, 0, cur, layer.wq, layer.ssm_q_conv,
d_conv, head_dim, n_head, n_seq_tokens, n_seqs);
ggml_tensor * k = build_causal_conv1d(
inp_rs, conv_states_all, conv_state_all, 1, cur, layer.wk, layer.ssm_k_conv,
d_conv, head_dim, n_head, n_seq_tokens, n_seqs);
ggml_tensor * v = build_causal_conv1d(
inp_rs, conv_states_all, conv_state_all, 2, cur, layer.wv, layer.ssm_v_conv,
d_conv, head_dim, n_head, n_seq_tokens, n_seqs);
ggml_tensor * gate = ggml_mul_mat(ctx0, layer.ssm_f_a, cur);
gate = ggml_add(ctx0, gate, layer.ssm_dt_b);