spec: properly handle mtmd input for mtp (#30257)

* spec: properly handle mtmd input for mtp

* nits
This commit is contained in:
Xuan-Son Nguyen authored and GitHub committed 2026-10-10 11:22:38 +02:00
1 parent 0fd868cbca
commit 781dbc5ac9
27 files changed
+248 -99

No files matched your search

+8 -5
View File
@@ -282,18 +282,21 @@ llama_model_mimo2::graph_mtp::graph_mtp(const llama_model & model, const llm_gra
const float freq_scale_l = model.get_rope_freq_scale(cparams, il);
const float v_scale = hparams.f_attn_value_scale;
auto inp = std::make_unique<llm_graph_input_embd>(hparams.n_embd);
auto inp = std::make_unique<llm_graph_input_embd_h>(hparams.n_embd_inp(), hparams.n_embd);
inp->tokens = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_tokens);
ggml_set_input(inp->tokens);
inp->embd = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, hparams.n_embd, n_tokens);
inp->embd = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, hparams.n_embd_inp(), n_tokens);
ggml_set_input(inp->embd);
ggml_set_name(inp->embd, "mtp_h_input");
inp->h = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, hparams.n_embd, n_tokens);
ggml_set_input(inp->h);
ggml_set_name(inp->h, "mtp_h_input");
ggml_tensor * tok_embd_w = layer.nextn.embed_tokens ? layer.nextn.embed_tokens : model.tok_embd;
ggml_tensor * h_input = inp->embd;
ggml_tensor * tok_embd = ggml_get_rows(ctx0, tok_embd_w, inp->tokens);
ggml_tensor * h_input = inp->h;
ggml_tensor * tok_embd = ubatch.token ? ggml_get_rows(ctx0, tok_embd_w, inp->tokens) : inp->embd;
cb(tok_embd, "mtp_tok_embd", il);
res->add_input(std::move(inp));