llama: support both embd + raw tokens in batch (#29622)

* llama: support both embd + raw tokens in batch

* add to test-llama-archs

* also check case llm_arch_supports_mixed_batch = false

* constant graph topology

* have dedicated input for mixed case

* rm set_tensor_backend

* is_embd --> type

* consolidate m-rope pos handling into one place

* nits
This commit is contained in:
Xuan-Son Nguyen authored and GitHub committed 2026-10-05 01:35:49 +02:00
1 parent 2ca15f5404
commit 0bb496dbd3
29 files changed
+506 -99

No files matched your search

+2
View File
@@ -1281,6 +1281,7 @@ llama_model_deepseek4::graph::graph(const llama_model & model, const llm_graph_p
ggml_tensor * exp_probs_b = layer.ffn_exp_probs_b;
// may apply exp_probs_b_vl is input is from mtmd
ASSERT_EMBD_OR_TOKEN(ubatch);
const bool is_media = ubatch.embd != nullptr;
if (is_media) {
if (layer.ffn_exp_probs_b_vl) {
@@ -1366,6 +1367,7 @@ llama_model_deepseek4::graph_mtp::graph_mtp(const llama_model & model, const llm
GGML_ASSERT(cparams.nextn_layer_offset >= 0 &&
cparams.nextn_layer_offset < (int) hparams.n_layer_nextn &&
"nextn_layer_offset out of range [0, n_layer_nextn)");
ASSERT_EMBD_OR_TOKEN(ubatch);
GGML_ASSERT(ubatch.token && "DEEPSEEK4 MTP requires token input");
const int64_t hc = hparams.dsv4_hc_mult;