mtp: read the target hidden states from a device buffer (#31927)

This commit is contained in:
Aman Gupta authored and GitHub committed 2026-11-09 20:52:32 +08:00
1 parent b39680c41a
commit a8fad21e99
31 files changed
+453 -61

No files matched your search

+1 -3
View File
@@ -275,9 +275,7 @@ llama_model_mimo2::graph_mtp::graph_mtp(const llm_graph_params & params)
inp->embd = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, hparams.n_embd_inp(), n_tokens);
ggml_set_input(inp->embd);
inp->h = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, hparams.n_embd, n_tokens);
ggml_set_input(inp->h);
ggml_set_name(inp->h, "mtp_h_input");
inp->h = build_inp_embd_state(inp.get(), hparams.n_embd);
ggml_tensor * tok_embd_w = layer.nextn.embed_tokens ? layer.nextn.embed_tokens : model.tok_embd;
ggml_tensor * h_input = inp->h;