llama: support both embd + raw tokens in batch (#29622)

* llama: support both embd + raw tokens in batch

* add to test-llama-archs

* also check case llm_arch_supports_mixed_batch = false

* constant graph topology

* have dedicated input for mixed case

* rm set_tensor_backend

* is_embd --> type

* consolidate m-rope pos handling into one place

* nits
This commit is contained in:
Xuan-Son Nguyen authored and GitHub committed 2026-10-05 01:35:49 +02:00
1 parent 2ca15f5404
commit 0bb496dbd3
29 files changed
+506 -99

No files matched your search

+2
View File
@@ -149,6 +149,7 @@ public:
// K dim-0 is +gain for an adapter token, -gain otherwise; the causal softmax then
// lets a single visible adapter token dominate so the readback recovers its slot.
void llm_graph_input_switch::set_input(const llama_ubatch * ubatch) {
ASSERT_EMBD_OR_TOKEN(*ubatch);
if (!ubatch->token) {
return;
}
@@ -226,6 +227,7 @@ llama_model_granite_switch::graph::graph(
const auto & smodel = static_cast<const llama_model_granite_switch &>(model);
// TODO: support raw embedding input (multimodal / pre-embedded tokens) when needed
ASSERT_EMBD_OR_TOKEN(ubatch);
GGML_ASSERT(ubatch.token && "granite-switch requires token input");
const int64_t n_embd_head = hparams.n_embd_head_v();