llama: support both embd + raw tokens in batch (#29622)
* llama: support both embd + raw tokens in batch * add to test-llama-archs * also check case llm_arch_supports_mixed_batch = false * constant graph topology * have dedicated input for mixed case * rm set_tensor_backend * is_embd --> type * consolidate m-rope pos handling into one place * nits
This commit is contained in:
1 parent
2ca15f5404
commit
0bb496dbd3
29 files changed
+506
-99
No files matched your search
@@ -149,6 +149,7 @@ public:
|
||||
// K dim-0 is +gain for an adapter token, -gain otherwise; the causal softmax then
|
||||
// lets a single visible adapter token dominate so the readback recovers its slot.
|
||||
void llm_graph_input_switch::set_input(const llama_ubatch * ubatch) {
|
||||
ASSERT_EMBD_OR_TOKEN(*ubatch);
|
||||
if (!ubatch->token) {
|
||||
return;
|
||||
}
|
||||
@@ -226,6 +227,7 @@ llama_model_granite_switch::graph::graph(
|
||||
const auto & smodel = static_cast<const llama_model_granite_switch &>(model);
|
||||
|
||||
// TODO: support raw embedding input (multimodal / pre-embedded tokens) when needed
|
||||
ASSERT_EMBD_OR_TOKEN(ubatch);
|
||||
GGML_ASSERT(ubatch.token && "granite-switch requires token input");
|
||||
|
||||
const int64_t n_embd_head = hparams.n_embd_head_v();
|
||||
|
||||
Reference in new issue
Block a user