graph : reorder get_rows for embeddings (#30160)
* graph : reorder get_rows for embeddings * cont : fix gemma4 and improve input embedding construction logic * cont : add TODO for lora * cont : fix raw embeddings path * gemma4 : avoid ple cast in embeddings path
This commit is contained in:
1 parent
baef3ed9a1
commit
8e2d31e0eb
3 files changed
+95
-54
No files matched your search
+11
-11
@@ -405,10 +405,6 @@ llama_model_qwen4exp::graph::graph(const llama_model & model, const llm_graph_pa
|
||||
int sections[4];
|
||||
std::copy(std::begin(hparams.rope_sections), std::begin(hparams.rope_sections) + 4, sections);
|
||||
|
||||
ggml_tensor * inpL = build_inp_embd(model.tok_embd);
|
||||
cb(inpL, "model.input_embed", -1);
|
||||
ggml_build_forward_expand(gf, inpL);
|
||||
|
||||
auto * inp = build_inp_mem_hybrid();
|
||||
|
||||
// qwen4exp always builds llama_memory_hybrid_idx, so this downcast is safe
|
||||
@@ -421,6 +417,17 @@ llama_model_qwen4exp::graph::graph(const llama_model & model, const llm_graph_pa
|
||||
"the indexer cache must track the attention cache cell for cell");
|
||||
}
|
||||
|
||||
ggml_tensor * ple_emb = nullptr;
|
||||
if (hparams.ple_n_heads > 0) {
|
||||
ple_emb = build_inp_ple(mctx_hyb);
|
||||
// make sure ple_emb and build_inp_embd are in the same graph split
|
||||
ggml_build_forward_expand(gf, ple_emb);
|
||||
}
|
||||
|
||||
ggml_tensor * inpL = build_inp_embd(model.tok_embd);
|
||||
cb(inpL, "model.input_embed", -1);
|
||||
ggml_build_forward_expand(gf, inpL);
|
||||
|
||||
// the QSA layers share one set of k-pool inputs
|
||||
// the CUDA lightning indexer takes 32 or 64 heads, QSA has a few, so it scores with plain ops
|
||||
llm_graph_input_kpool * inp_kpool = nullptr;
|
||||
@@ -431,13 +438,6 @@ llama_model_qwen4exp::graph::graph(const llama_model & model, const llm_graph_pa
|
||||
ggml_tensor * inp_pos = build_inp_pos();
|
||||
ggml_tensor * inp_out_ids = build_inp_out_ids();
|
||||
|
||||
ggml_tensor * ple_emb = nullptr;
|
||||
if (hparams.ple_n_heads > 0) {
|
||||
ple_emb = build_inp_ple(mctx_hyb);
|
||||
// make sure ple_emb and build_inp_embd are in the same graph split
|
||||
ggml_build_forward_expand(gf, ple_emb);
|
||||
}
|
||||
|
||||
// the wide residual starts as hc identical copies of the embedding
|
||||
ggml_tensor * res_hc = ggml_repeat_4d(ctx0,
|
||||
ggml_reshape_3d(ctx0, inpL, n_embd, 1, n_tokens),
|
||||
|
||||
Reference in new issue
Block a user