llama: support both embd + raw tokens in batch (#29622)
* llama: support both embd + raw tokens in batch * add to test-llama-archs * also check case llm_arch_supports_mixed_batch = false * constant graph topology * have dedicated input for mixed case * rm set_tensor_backend * is_embd --> type * consolidate m-rope pos handling into one place * nits
This commit is contained in:
1 parent
2ca15f5404
commit
0bb496dbd3
29 files changed
+506
-99
No files matched your search
@@ -317,6 +317,7 @@ llama_model_cohere2moe::graph_mtp::graph_mtp(const llama_model & model, const ll
|
||||
// TODO: make static using `ggml_build_forward_select()`
|
||||
// see llm_graph_context::build_inp_embd() for reference
|
||||
ggml_tensor * tok_embd;
|
||||
ASSERT_EMBD_OR_TOKEN(ubatch);
|
||||
if (ubatch.token) {
|
||||
ggml_tensor * tok_embd_w = layer.nextn.embed_tokens ? layer.nextn.embed_tokens : model.tok_embd;
|
||||
tok_embd = ggml_get_rows(ctx0, tok_embd_w, inp->tokens);
|
||||
|
||||
Reference in new issue
Block a user