mtmd: add cohere2 vision support (#30062)

* cohere2 vision model

* address comments

* remove redundant mapping

* follow existing patterns

* fused linear_1
This commit is contained in:
Terrence Zhao authored and GitHub committed 2026-10-07 20:07:50 +02:00
1 parent d6cf9acb25
commit 50a6c5cf7c
10 files changed
+146 -2

No files matched your search

+10
View File
@@ -45,6 +45,16 @@ ggml_cgraph * clip_graph_siglip::build() {
cur = build_patch_merge_permute(cur, scale_factor);
cur = build_mm(model.mm_fc_w, cur);
} else if (proj_type == PROJECTOR_TYPE_COHERE2V) {
// tiles are square, so the pixel shuffle is the same as Idefics3
cur = build_patch_merge_permute(cur, model.hparams.n_merge);
cur = build_mm(model.mm_1_w, cur);
cur = ggml_add(ctx0, cur, model.mm_1_b);
// linear_1 output is [x, gate], HF computes silu(gate) * x
cur = ggml_swiglu_swapped(ctx0, cur);
cur = build_mm(model.mm_2_w, cur);
cur = ggml_add(ctx0, cur, model.mm_2_b);
} else if (proj_type == PROJECTOR_TYPE_LFM2) {
// pixel unshuffle block
const int scale_factor = model.hparams.n_merge;