llama: support both embd + raw tokens in batch (#29622)
* llama: support both embd + raw tokens in batch * add to test-llama-archs * also check case llm_arch_supports_mixed_batch = false * constant graph topology * have dedicated input for mixed case * rm set_tensor_backend * is_embd --> type * consolidate m-rope pos handling into one place * nits
This commit is contained in:
1 parent
2ca15f5404
commit
0bb496dbd3
29 files changed
+506
-99
No files matched your search
@@ -1281,6 +1281,7 @@ llama_model_deepseek4::graph::graph(const llama_model & model, const llm_graph_p
|
||||
ggml_tensor * exp_probs_b = layer.ffn_exp_probs_b;
|
||||
|
||||
// may apply exp_probs_b_vl is input is from mtmd
|
||||
ASSERT_EMBD_OR_TOKEN(ubatch);
|
||||
const bool is_media = ubatch.embd != nullptr;
|
||||
if (is_media) {
|
||||
if (layer.ffn_exp_probs_b_vl) {
|
||||
@@ -1366,6 +1367,7 @@ llama_model_deepseek4::graph_mtp::graph_mtp(const llama_model & model, const llm
|
||||
GGML_ASSERT(cparams.nextn_layer_offset >= 0 &&
|
||||
cparams.nextn_layer_offset < (int) hparams.n_layer_nextn &&
|
||||
"nextn_layer_offset out of range [0, n_layer_nextn)");
|
||||
ASSERT_EMBD_OR_TOKEN(ubatch);
|
||||
GGML_ASSERT(ubatch.token && "DEEPSEEK4 MTP requires token input");
|
||||
|
||||
const int64_t hc = hparams.dsv4_hc_mult;
|
||||
|
||||
Reference in new issue
Block a user