graph : reorder get_rows for embeddings (#30160)

* graph : reorder get_rows for embeddings

* cont : fix gemma4 and improve input embedding construction logic

* cont : add TODO for lora

* cont : fix raw embeddings path

* gemma4 : avoid ple cast in embeddings path
This commit is contained in:
Georgi Gerganov authored and GitHub committed 2026-10-09 20:43:05 +03:00
1 parent baef3ed9a1
commit 8e2d31e0eb
3 files changed
+95 -54

No files matched your search

+58 -25
View File
@@ -2471,19 +2471,57 @@ ggml_tensor * llm_graph_context::build_inp_embd(ggml_tensor * tok_embd, float to
auto inp = std::make_unique<llm_graph_input_embd>(n_embd_inp);
inp->tokens = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, ubatch.n_tokens);
cb(inp->tokens, "inp_tokens", -1);
ggml_set_input(inp->tokens);
res->t_inp_tokens = inp->tokens;
// mixed path (ubatch.is_mixed()): set_rows the token rows into a copy of the embd rows, with its own inputs as select branches must not share tensors
// TODO: use inp->tokens and inp->embd once ggml_build_forward_select allows it
const bool has_mixed = llm_arch_supports_mixed_batch(arch) && cparams.ctx_type == LLAMA_CONTEXT_TYPE_DEFAULT;
inp->embd = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_embd_inp, ubatch.n_tokens);
cb(inp->embd, "inp_embd", -1);
ggml_set_input(inp->embd);
const int64_t n_tok_rows = has_mixed ? llm_graph_n_tok_rows(ubatch) : 0;
// token embeddings with lora and padding
auto build_tok = [&](ggml_tensor * ids) {
ggml_tensor * cur = ggml_get_rows(ctx0, tok_embd, ids);
// we have 3 standard paths to produce the input embeddings for the first layer:
// - embd0: extract from the token embeddings weight (`tok_embd`) using the input token ids
// - embd1: pass raw embeddings, skipping the `tok_embd`
// - embd2: mixed path of both tokens ids + raw embeddings (if supported)
ggml_tensor * embd0 = nullptr;
ggml_tensor * embd1 = nullptr;
ggml_tensor * embd2 = nullptr;
// construct the input tensors
{
inp->tokens = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, ubatch.n_tokens);
cb(inp->tokens, "inp_tokens", -1);
ggml_set_input(inp->tokens);
res->t_inp_tokens = inp->tokens;
inp->embd = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_embd_inp, ubatch.n_tokens);
cb(inp->embd, "inp_embd", -1);
ggml_set_input(inp->embd);
if (has_mixed) {
inp->mixed_tokens = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_tok_rows);
cb(inp->mixed_tokens, "inp_mixed_tokens", -1);
ggml_set_input(inp->mixed_tokens);
}
}
// the embeddings placeholders for the 3 paths
// we use ggml_build_forward_order to make the GET_ROWS ops stick at the beginning of the compute graph
// this way the embeddings remain in the host buffer, and the GET_ROWS run before any other computations
{
embd0 = ggml_get_rows(ctx0, tok_embd, inp->tokens);
ggml_build_forward_order(gf, embd0);
embd1 = inp->embd;
if (has_mixed) {
embd2 = ggml_get_rows(ctx0, tok_embd, inp->mixed_tokens);
ggml_build_forward_order(gf, embd2);
}
}
// helper for extracting token embeddings with lora and padding
// TODO: when lora is active, this is likely going to cause issues similar to https://github.com/ggml-org/llama.cpp/pull/30160
// need to add lora tests and refactor the logic to make the lora GET_ROWS go at the front of the graph
auto build_tok = [&](ggml_tensor * cur, ggml_tensor * ids) {
// apply lora for embedding tokens if needed
for (const auto & lora : *loras) {
llama_adapter_lora_weight * lw = lora.first->get_weight(tok_embd);
@@ -2514,21 +2552,15 @@ ggml_tensor * llm_graph_context::build_inp_embd(ggml_tensor * tok_embd, float to
std::array<ggml_tensor *, 3> inps = {};
// token embeddings path (ubatch.token != nullptr)
inps[0] = build_tok(inp->tokens);
inps[0] = build_tok(embd0, inp->tokens);
// vector embeddings path (ubatch.embd != nullptr)
inps[1] = inp->embd;
inps[1] = embd1;
assert(ggml_are_same_shape (inps[0], inps[1]));
assert(ggml_are_same_stride(inps[0], inps[1]));
// mixed path (ubatch.is_mixed()): set_rows the token rows into a copy of the embd rows, with its own inputs as select branches must not share tensors
// TODO: use inp->tokens and inp->embd once ggml_build_forward_select allows it
const bool has_mixed = llm_arch_supports_mixed_batch(arch) && cparams.ctx_type == LLAMA_CONTEXT_TYPE_DEFAULT;
if (has_mixed) {
const int64_t n_tok_rows = llm_graph_n_tok_rows(ubatch);
inp->mixed_tokens = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_tok_rows);
cb(inp->mixed_tokens, "inp_mixed_tokens", -1);
ggml_set_input(inp->mixed_tokens);
inp->mixed_slots = ggml_new_tensor_1d(ctx0, GGML_TYPE_I64, n_tok_rows);
cb(inp->mixed_slots, "inp_mixed_slots", -1);
ggml_set_input(inp->mixed_slots);
@@ -2538,11 +2570,12 @@ ggml_tensor * llm_graph_context::build_inp_embd(ggml_tensor * tok_embd, float to
ggml_set_input(inp->mixed_embd);
// note: set_rows writes into its destination, so it gets a copy of the input
inps[2] = ggml_set_rows(ctx0, ggml_dup(ctx0, inp->mixed_embd), build_tok(inp->mixed_tokens), inp->mixed_slots);
}
ggml_tensor * embd_mixed = build_tok(embd2, inp->mixed_tokens);
inps[2] = ggml_set_rows(ctx0, ggml_dup(ctx0, inp->mixed_embd), embd_mixed, inp->mixed_slots);
assert(ggml_are_same_shape (inps[0], inps[1]));
assert(ggml_are_same_stride(inps[0], inps[1]));
assert(ggml_are_same_shape (inps[0], inps[2]));
assert(ggml_are_same_stride(inps[0], inps[2]));
}
const int idx = ubatch.is_mixed() ? 2 : ubatch.token ? 0 : 1;