graph : reorder get_rows for embeddings (#30160)
* graph : reorder get_rows for embeddings * cont : fix gemma4 and improve input embedding construction logic * cont : add TODO for lora * cont : fix raw embeddings path * gemma4 : avoid ple cast in embeddings path
This commit is contained in:
1 parent
baef3ed9a1
commit
8e2d31e0eb
3 files changed
+95
-54
No files matched your search
+26
-18
@@ -159,6 +159,14 @@ llama_model_gemma4::graph::graph(const llama_model & model, const llm_graph_para
|
||||
ggml_tensor * cur;
|
||||
ggml_tensor * inpL;
|
||||
|
||||
// do the PLE first to guarantee it is done in the host buffer
|
||||
// ref: https://github.com/ggml-org/llama.cpp/pull/30160
|
||||
ggml_tensor * inp_per_layer = nullptr;
|
||||
if (model.per_layer_tok_embd) {
|
||||
inp_per_layer = build_inp_per_layer();
|
||||
ggml_build_forward_expand(gf, inp_per_layer);
|
||||
}
|
||||
|
||||
// important: do not normalize weights for raw embeddings input (i.e. encoded image emdeddings)
|
||||
inpL = build_inp_embd(model.tok_embd, sqrtf(n_embd));
|
||||
cb(inpL, "inp_scaled", -1);
|
||||
@@ -171,10 +179,11 @@ llama_model_gemma4::graph::graph(const llama_model & model, const llm_graph_para
|
||||
|
||||
ggml_tensor * inp_out_ids = build_inp_out_ids();
|
||||
|
||||
ggml_tensor * inp_per_layer = nullptr;
|
||||
if (model.per_layer_tok_embd) {
|
||||
inp_per_layer = build_inp_per_layer();
|
||||
ggml_build_forward_expand(gf, inp_per_layer);
|
||||
const float tok_embd_scale = sqrtf((float) n_embd_per_layer);
|
||||
|
||||
inp_per_layer = ggml_scale (ctx0, inp_per_layer, tok_embd_scale);
|
||||
inp_per_layer = ggml_reshape_3d(ctx0, inp_per_layer, n_embd_per_layer, n_layer, inp_per_layer->ne[1]);
|
||||
|
||||
// inp_per_layer shape: [n_embd_per_layer, n_tokens, n_layer]
|
||||
inp_per_layer = project_per_layer_inputs(inpL, inp_per_layer);
|
||||
@@ -448,10 +457,13 @@ public:
|
||||
llama_prefetch_rows(ple, ubatch->token, ubatch->n_tokens);
|
||||
}
|
||||
ggml_backend_tensor_set(tokens, ubatch->token, 0, ubatch->n_tokens * ggml_element_size(tokens));
|
||||
} else if (prefetch) {
|
||||
// [TAG_GEMMA4_IMG_PADDING]
|
||||
} else {
|
||||
const int32_t padding = 0;
|
||||
llama_prefetch_rows(ple, &padding, 1);
|
||||
if (prefetch) {
|
||||
// [TAG_GEMMA4_IMG_PADDING]
|
||||
llama_prefetch_rows(ple, &padding, 1);
|
||||
}
|
||||
ggml_backend_tensor_set(token0, &padding, 0, ggml_element_size(token0));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -460,6 +472,7 @@ public:
|
||||
}
|
||||
|
||||
ggml_tensor * tokens = nullptr;
|
||||
ggml_tensor * token0 = nullptr;
|
||||
|
||||
const llama_model & model;
|
||||
};
|
||||
@@ -470,30 +483,25 @@ ggml_tensor * llama_model_gemma4::graph::build_inp_per_layer() {
|
||||
auto inp = std::make_unique<llm_graph_input_gemma4_ple>(model);
|
||||
|
||||
ggml_tensor * inp_per_layer;
|
||||
float tok_embd_scale = sqrtf((float) n_embd_per_layer);
|
||||
// mixed ubatch: embd rows have token id 0, same padding row as below
|
||||
// TODO: use ggml_build_forward_select
|
||||
if (ubatch.token) {
|
||||
inp->tokens = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, ubatch.n_tokens);
|
||||
ggml_set_input(inp->tokens);
|
||||
res->t_inp_tokens = inp->tokens;
|
||||
|
||||
inp_per_layer = ggml_get_rows (ctx0, model.per_layer_tok_embd, inp->tokens);
|
||||
inp_per_layer = ggml_reshape_3d(ctx0, inp_per_layer, n_embd_per_layer, n_layer, n_tokens);
|
||||
inp_per_layer = ggml_scale (ctx0, inp_per_layer, tok_embd_scale);
|
||||
inp_per_layer = ggml_get_rows(ctx0, model.per_layer_tok_embd, inp->tokens);
|
||||
cb(inp_per_layer, "inp_per_layer_selected", -1);
|
||||
} else {
|
||||
// [TAG_GEMMA4_IMG_PADDING]
|
||||
// Multimodal embedding path: use padding token (ID=0) embedding
|
||||
// TODO: verify if this is the correct behavior in transformers implementation
|
||||
const int64_t embd_size = model.per_layer_tok_embd->ne[0]; // n_embd_per_layer * n_layer
|
||||
|
||||
// Extract and dequantize padding token embedding (row 0)
|
||||
ggml_tensor * padding = ggml_view_1d(ctx0, model.per_layer_tok_embd, embd_size, 0);
|
||||
inp_per_layer = ggml_cast (ctx0, padding, GGML_TYPE_F32);
|
||||
inp_per_layer = ggml_scale(ctx0, inp_per_layer, tok_embd_scale);
|
||||
// [TAG_GEMMA4_IMG_PADDING]
|
||||
inp->token0 = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, 1);
|
||||
ggml_set_input(inp->token0);
|
||||
res->t_inp_tokens = inp->token0;
|
||||
|
||||
// Reshape to [n_embd_per_layer, n_layer, 1]
|
||||
inp_per_layer = ggml_reshape_3d(ctx0, inp_per_layer, n_embd_per_layer, n_layer, 1);
|
||||
inp_per_layer = ggml_get_rows(ctx0, model.per_layer_tok_embd, inp->token0);
|
||||
cb(inp_per_layer, "inp_per_layer_multimodal", -1);
|
||||
}
|
||||
res->add_input(std::move(inp));
|
||||
|
||||
Reference in new issue
Block a user