mtp: read the target hidden states from a device buffer (#31927)
This commit is contained in:
1 parent
b39680c41a
commit
a8fad21e99
31 files changed
+453
-61
No files matched your search
@@ -379,9 +379,7 @@ llama_model_step35::graph_mtp::graph_mtp(const llm_graph_params & params)
|
||||
inp->embd = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, hparams.n_embd_inp(), n_tokens);
|
||||
ggml_set_input(inp->embd);
|
||||
|
||||
inp->h = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, hparams.n_embd, n_tokens);
|
||||
ggml_set_input(inp->h);
|
||||
ggml_set_name(inp->h, "mtp_h_input");
|
||||
inp->h = build_inp_embd_state(inp.get(), hparams.n_embd);
|
||||
|
||||
ggml_tensor * tok_embd_w = layer.nextn.embed_tokens ? layer.nextn.embed_tokens : model.tok_embd;
|
||||
|
||||
|
||||
Reference in new issue
Block a user