graph : reorder get_rows for embeddings (#30160)
* graph : reorder get_rows for embeddings * cont : fix gemma4 and improve input embedding construction logic * cont : add TODO for lora * cont : fix raw embeddings path * gemma4 : avoid ple cast in embeddings path
This commit is contained in:
1 parent
baef3ed9a1
commit
8e2d31e0eb
3 files changed
+95
-54
No files matched your search
+58
-25
@@ -2471,19 +2471,57 @@ ggml_tensor * llm_graph_context::build_inp_embd(ggml_tensor * tok_embd, float to
|
||||
|
||||
auto inp = std::make_unique<llm_graph_input_embd>(n_embd_inp);
|
||||
|
||||
inp->tokens = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, ubatch.n_tokens);
|
||||
cb(inp->tokens, "inp_tokens", -1);
|
||||
ggml_set_input(inp->tokens);
|
||||
res->t_inp_tokens = inp->tokens;
|
||||
// mixed path (ubatch.is_mixed()): set_rows the token rows into a copy of the embd rows, with its own inputs as select branches must not share tensors
|
||||
// TODO: use inp->tokens and inp->embd once ggml_build_forward_select allows it
|
||||
const bool has_mixed = llm_arch_supports_mixed_batch(arch) && cparams.ctx_type == LLAMA_CONTEXT_TYPE_DEFAULT;
|
||||
|
||||
inp->embd = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_embd_inp, ubatch.n_tokens);
|
||||
cb(inp->embd, "inp_embd", -1);
|
||||
ggml_set_input(inp->embd);
|
||||
const int64_t n_tok_rows = has_mixed ? llm_graph_n_tok_rows(ubatch) : 0;
|
||||
|
||||
// token embeddings with lora and padding
|
||||
auto build_tok = [&](ggml_tensor * ids) {
|
||||
ggml_tensor * cur = ggml_get_rows(ctx0, tok_embd, ids);
|
||||
// we have 3 standard paths to produce the input embeddings for the first layer:
|
||||
// - embd0: extract from the token embeddings weight (`tok_embd`) using the input token ids
|
||||
// - embd1: pass raw embeddings, skipping the `tok_embd`
|
||||
// - embd2: mixed path of both tokens ids + raw embeddings (if supported)
|
||||
ggml_tensor * embd0 = nullptr;
|
||||
ggml_tensor * embd1 = nullptr;
|
||||
ggml_tensor * embd2 = nullptr;
|
||||
|
||||
// construct the input tensors
|
||||
{
|
||||
inp->tokens = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, ubatch.n_tokens);
|
||||
cb(inp->tokens, "inp_tokens", -1);
|
||||
ggml_set_input(inp->tokens);
|
||||
res->t_inp_tokens = inp->tokens;
|
||||
|
||||
inp->embd = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_embd_inp, ubatch.n_tokens);
|
||||
cb(inp->embd, "inp_embd", -1);
|
||||
ggml_set_input(inp->embd);
|
||||
|
||||
if (has_mixed) {
|
||||
inp->mixed_tokens = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_tok_rows);
|
||||
cb(inp->mixed_tokens, "inp_mixed_tokens", -1);
|
||||
ggml_set_input(inp->mixed_tokens);
|
||||
}
|
||||
}
|
||||
|
||||
// the embeddings placeholders for the 3 paths
|
||||
// we use ggml_build_forward_order to make the GET_ROWS ops stick at the beginning of the compute graph
|
||||
// this way the embeddings remain in the host buffer, and the GET_ROWS run before any other computations
|
||||
{
|
||||
embd0 = ggml_get_rows(ctx0, tok_embd, inp->tokens);
|
||||
ggml_build_forward_order(gf, embd0);
|
||||
|
||||
embd1 = inp->embd;
|
||||
|
||||
if (has_mixed) {
|
||||
embd2 = ggml_get_rows(ctx0, tok_embd, inp->mixed_tokens);
|
||||
ggml_build_forward_order(gf, embd2);
|
||||
}
|
||||
}
|
||||
|
||||
// helper for extracting token embeddings with lora and padding
|
||||
// TODO: when lora is active, this is likely going to cause issues similar to https://github.com/ggml-org/llama.cpp/pull/30160
|
||||
// need to add lora tests and refactor the logic to make the lora GET_ROWS go at the front of the graph
|
||||
auto build_tok = [&](ggml_tensor * cur, ggml_tensor * ids) {
|
||||
// apply lora for embedding tokens if needed
|
||||
for (const auto & lora : *loras) {
|
||||
llama_adapter_lora_weight * lw = lora.first->get_weight(tok_embd);
|
||||
@@ -2514,21 +2552,15 @@ ggml_tensor * llm_graph_context::build_inp_embd(ggml_tensor * tok_embd, float to
|
||||
std::array<ggml_tensor *, 3> inps = {};
|
||||
|
||||
// token embeddings path (ubatch.token != nullptr)
|
||||
inps[0] = build_tok(inp->tokens);
|
||||
inps[0] = build_tok(embd0, inp->tokens);
|
||||
|
||||
// vector embeddings path (ubatch.embd != nullptr)
|
||||
inps[1] = inp->embd;
|
||||
inps[1] = embd1;
|
||||
|
||||
assert(ggml_are_same_shape (inps[0], inps[1]));
|
||||
assert(ggml_are_same_stride(inps[0], inps[1]));
|
||||
|
||||
// mixed path (ubatch.is_mixed()): set_rows the token rows into a copy of the embd rows, with its own inputs as select branches must not share tensors
|
||||
// TODO: use inp->tokens and inp->embd once ggml_build_forward_select allows it
|
||||
const bool has_mixed = llm_arch_supports_mixed_batch(arch) && cparams.ctx_type == LLAMA_CONTEXT_TYPE_DEFAULT;
|
||||
if (has_mixed) {
|
||||
const int64_t n_tok_rows = llm_graph_n_tok_rows(ubatch);
|
||||
|
||||
inp->mixed_tokens = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_tok_rows);
|
||||
cb(inp->mixed_tokens, "inp_mixed_tokens", -1);
|
||||
ggml_set_input(inp->mixed_tokens);
|
||||
|
||||
inp->mixed_slots = ggml_new_tensor_1d(ctx0, GGML_TYPE_I64, n_tok_rows);
|
||||
cb(inp->mixed_slots, "inp_mixed_slots", -1);
|
||||
ggml_set_input(inp->mixed_slots);
|
||||
@@ -2538,11 +2570,12 @@ ggml_tensor * llm_graph_context::build_inp_embd(ggml_tensor * tok_embd, float to
|
||||
ggml_set_input(inp->mixed_embd);
|
||||
|
||||
// note: set_rows writes into its destination, so it gets a copy of the input
|
||||
inps[2] = ggml_set_rows(ctx0, ggml_dup(ctx0, inp->mixed_embd), build_tok(inp->mixed_tokens), inp->mixed_slots);
|
||||
}
|
||||
ggml_tensor * embd_mixed = build_tok(embd2, inp->mixed_tokens);
|
||||
inps[2] = ggml_set_rows(ctx0, ggml_dup(ctx0, inp->mixed_embd), embd_mixed, inp->mixed_slots);
|
||||
|
||||
assert(ggml_are_same_shape (inps[0], inps[1]));
|
||||
assert(ggml_are_same_stride(inps[0], inps[1]));
|
||||
assert(ggml_are_same_shape (inps[0], inps[2]));
|
||||
assert(ggml_are_same_stride(inps[0], inps[2]));
|
||||
}
|
||||
|
||||
const int idx = ubatch.is_mixed() ? 2 : ubatch.token ? 0 : 1;
|
||||
|
||||
|
||||
Reference in new issue
Block a user