llama: support both embd + raw tokens in batch (#29622)
* llama: support both embd + raw tokens in batch * add to test-llama-archs * also check case llm_arch_supports_mixed_batch = false * constant graph topology * have dedicated input for mixed case * rm set_tensor_backend * is_embd --> type * consolidate m-rope pos handling into one place * nits
This commit is contained in:
1 parent
2ca15f5404
commit
0bb496dbd3
29 files changed
+506
-99
No files matched your search
@@ -159,10 +159,8 @@ llama_model_gemma4::graph::graph(const llama_model & model, const llm_graph_para
|
||||
ggml_tensor * cur;
|
||||
ggml_tensor * inpL;
|
||||
|
||||
inpL = build_inp_embd(model.tok_embd);
|
||||
|
||||
// important: do not normalize weights for raw embeddings input (i.e. encoded image emdeddings)
|
||||
inpL = ggml_scale(ctx0, inpL, ubatch.token ? sqrtf(n_embd) : 1.0f);
|
||||
inpL = build_inp_embd(model.tok_embd, sqrtf(n_embd));
|
||||
cb(inpL, "inp_scaled", -1);
|
||||
|
||||
// inp_pos - contains the positions
|
||||
@@ -473,6 +471,7 @@ ggml_tensor * llama_model_gemma4::graph::build_inp_per_layer() {
|
||||
|
||||
ggml_tensor * inp_per_layer;
|
||||
float tok_embd_scale = sqrtf((float) n_embd_per_layer);
|
||||
// mixed ubatch: embd rows have token id 0, same padding row as below
|
||||
if (ubatch.token) {
|
||||
inp->tokens = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, ubatch.n_tokens);
|
||||
ggml_set_input(inp->tokens);
|
||||
|
||||
Reference in new issue
Block a user