- remove build_delta_net_chunking and build_delta_net_autoregressive - always use the fused gated delta net - make graph_max_nodes independent of the ubatch size - drop the fused GDN probes and fused node tags Assisted-by: dsh:llama.cpp/ggml-org/DeepSeek-V4.1-Flash-1017-GGUF (Mac Studio, M5 Ultra, 256GB)
153 lines
5.8 KiB
C++
153 lines
5.8 KiB
C++
#include "models.h"
|
|
|
|
#include "llama-impl.h"
|
|
#include "llama-memory-recurrent.h"
|
|
|
|
llm_build_delta_net_base::llm_build_delta_net_base(const llm_graph_params & params) : llm_graph_context(params) {}
|
|
|
|
std::pair<ggml_tensor *, ggml_tensor *> llm_build_delta_net_base::build_delta_net(
|
|
ggml_tensor * q,
|
|
ggml_tensor * k,
|
|
ggml_tensor * v,
|
|
ggml_tensor * g,
|
|
ggml_tensor * b,
|
|
ggml_tensor * s) {
|
|
const int64_t S_k = q->ne[0];
|
|
const int64_t H_k = q->ne[1];
|
|
const int64_t n_tokens = q->ne[2];
|
|
const int64_t n_seqs = q->ne[3];
|
|
|
|
const int64_t S_v = v->ne[0];
|
|
const int64_t H_v = v->ne[1];
|
|
|
|
GGML_ASSERT(S_k == S_v);
|
|
GGML_ASSERT(H_v % H_k == 0);
|
|
|
|
GGML_ASSERT(q->ne[0] == S_k && q->ne[1] == H_k && q->ne[2] == n_tokens && q->ne[3] == n_seqs);
|
|
GGML_ASSERT(k->ne[0] == S_k && k->ne[1] == H_k && k->ne[2] == n_tokens && k->ne[3] == n_seqs);
|
|
GGML_ASSERT(v->ne[0] == S_v && v->ne[1] == H_v && v->ne[2] == n_tokens && v->ne[3] == n_seqs);
|
|
|
|
GGML_ASSERT(g->ne[0] == 1 || g->ne[0] == S_v);
|
|
GGML_ASSERT( g->ne[1] == H_v && g->ne[2] == n_tokens && g->ne[3] == n_seqs);
|
|
GGML_ASSERT(b->ne[0] == 1 && b->ne[1] == H_v && b->ne[2] == n_tokens && b->ne[3] == n_seqs);
|
|
GGML_ASSERT(s->ne[0] == S_v && s->ne[1] == S_v && s->ne[2] == H_v && s->ne[3] == n_seqs);
|
|
|
|
// K=1: output carries the final state only. state s is 4D [S_v, S_v, H_v, n_seqs].
|
|
ggml_tensor * result = ggml_gated_delta_net(ctx0, q, k, v, g, b, s, /*K=*/1);
|
|
ggml_tensor * output = ggml_view_4d(ctx0, result,
|
|
S_v, H_v, n_tokens, n_seqs,
|
|
ggml_row_size(result->type, S_v),
|
|
ggml_row_size(result->type, S_v * H_v),
|
|
ggml_row_size(result->type, S_v * H_v * n_tokens), 0);
|
|
|
|
ggml_tensor * new_state = ggml_view_4d(ctx0, result,
|
|
S_v, S_v, H_v, n_seqs,
|
|
ggml_row_size(result->type, S_v),
|
|
ggml_row_size(result->type, S_v * S_v),
|
|
ggml_row_size(result->type, S_v * S_v * H_v),
|
|
ggml_row_size(result->type, S_v * H_v * n_tokens * n_seqs));
|
|
|
|
return {output, new_state};
|
|
}
|
|
|
|
ggml_tensor * llm_build_delta_net_base::build_conv_state(
|
|
llm_graph_input_rs * inp,
|
|
ggml_tensor * conv_states_all,
|
|
ggml_tensor * qkv_mixed,
|
|
int64_t conv_kernel_size,
|
|
int64_t conv_channels,
|
|
int il) {
|
|
const int64_t n_seqs = ubatch.n_seqs;
|
|
|
|
ggml_tensor * conv_states = build_rs(inp, conv_states_all, hparams.n_embd_r(), n_seqs);
|
|
cb(conv_states, "conv_states", il);
|
|
|
|
conv_states = ggml_reshape_3d(ctx0, conv_states, conv_kernel_size - 1, conv_channels, n_seqs);
|
|
cb(conv_states, "conv_states_reshaped", il);
|
|
|
|
qkv_mixed = ggml_transpose(ctx0, qkv_mixed);
|
|
cb(qkv_mixed, "qkv_mixed_transposed", il);
|
|
|
|
ggml_tensor * conv_input = ggml_concat(ctx0, conv_states, qkv_mixed, 0);
|
|
cb(conv_input, "conv_input", il);
|
|
|
|
build_conv_state_store(inp, conv_states_all, conv_input, conv_kernel_size - 1, 0);
|
|
|
|
return conv_input;
|
|
}
|
|
|
|
ggml_tensor * llm_build_delta_net_base::build_recurrent_attn(
|
|
llm_graph_input_rs * inp,
|
|
ggml_tensor * ssm_states_all,
|
|
ggml_tensor * q,
|
|
ggml_tensor * k,
|
|
ggml_tensor * v,
|
|
ggml_tensor * g,
|
|
ggml_tensor * b,
|
|
ggml_tensor * s,
|
|
int il) {
|
|
const auto * mctx_cur = inp->mctx;
|
|
const auto kv_head = mctx_cur->get_head();
|
|
const uint32_t mem_size = mctx_cur->get_size();
|
|
|
|
const int64_t S_v = s->ne[0];
|
|
const int64_t H_v = s->ne[2];
|
|
const int64_t n_seqs = s->ne[3];
|
|
const int64_t n_seq_tokens = q->ne[2];
|
|
|
|
const bool keep = cparams.n_rs_seq > 0;
|
|
|
|
if (!keep) {
|
|
auto attn_out = build_delta_net(q, k, v, g, b, s);
|
|
ggml_tensor * output = attn_out.first;
|
|
ggml_tensor * new_state = attn_out.second;
|
|
cb(output, "attn_output", il);
|
|
cb(new_state, "new_state", il);
|
|
|
|
ggml_build_forward_expand(gf,
|
|
ggml_cpy(ctx0, new_state,
|
|
ggml_view_2d(ctx0, ssm_states_all, hparams.n_embd_s(), n_seqs, ssm_states_all->nb[1],
|
|
kv_head * hparams.n_embd_s() * ggml_element_size(ssm_states_all))));
|
|
|
|
return output;
|
|
}
|
|
|
|
const int64_t D = S_v * S_v * H_v;
|
|
const int64_t K = cparams.n_rs_seq + 1;
|
|
|
|
// state s is 4D [S_v, S_v, H_v, n_seqs]; K snapshot slots are written into the output.
|
|
ggml_tensor * gdn_out = ggml_gated_delta_net(ctx0, q, k, v, g, b, s, K);
|
|
const int64_t attn_score_elems = S_v * H_v * n_seq_tokens * n_seqs;
|
|
const int64_t state_size_per_snap = S_v * S_v * H_v * n_seqs;
|
|
|
|
ggml_tensor * output = ggml_view_4d(ctx0, gdn_out,
|
|
S_v, H_v, n_seq_tokens, n_seqs,
|
|
ggml_row_size(gdn_out->type, S_v),
|
|
ggml_row_size(gdn_out->type, S_v * H_v),
|
|
ggml_row_size(gdn_out->type, S_v * H_v * n_seq_tokens),
|
|
0);
|
|
cb(output, "attn_output", il);
|
|
|
|
const size_t row_size = hparams.n_embd_s() * ggml_element_size(ssm_states_all);
|
|
|
|
// op writes the last min(n_seq_tokens, K) snapshots; trailing slots are left unwritten
|
|
const int64_t n_written = std::min<int64_t>(n_seq_tokens, K);
|
|
|
|
// write the produced snapshots into the recurrent cache (snapshot slot i -> rollback group i)
|
|
ggml_tensor * src = ggml_view_3d(ctx0, gdn_out,
|
|
D, n_seqs, n_written,
|
|
ggml_row_size(gdn_out->type, D),
|
|
ggml_row_size(gdn_out->type, state_size_per_snap),
|
|
ggml_row_size(gdn_out->type, attn_score_elems));
|
|
|
|
ggml_tensor * dst = ggml_view_3d(ctx0, ssm_states_all,
|
|
D, n_seqs, n_written,
|
|
ssm_states_all->nb[1],
|
|
(size_t) mem_size * row_size,
|
|
(size_t) kv_head * row_size);
|
|
|
|
ggml_build_forward_expand(gf, ggml_cpy(ctx0, src, dst));
|
|
|
|
return output;
|
|
}
|