Lumpiasty/llama.cpp
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

model, mtmd: fix gemma4 vision handling (#28335)

Browse Source 
* model, mtmd: fix gemma4 vision handling

* nits
This commit is contained in:
Xuan-Son Nguyen
2026-09-04 12:23:27 +02:00
committed by GitHub
parent 8f83678fd8
commit 163a40796f
7 changed files with 30 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tools/mtmd/clip.cpp
+1 -2
View File
@@ -1636,8 +1636,7 @@ struct clip_model_loader {
hparams.patch_size = hparams.patch_size * hparams.n_merge;
hparams.n_merge = 1;
}
// @ngxson : the model performs quite poor with small images, we need to bump minimum image tokens to 40 to avoid that
hparams.set_limit_image_tokens(40, 280);
hparams.set_limit_image_tokens(70, 1120);
hparams.set_warmup_n_tokens(256); // avoid OOM on warmup
} break;
tools/mtmd/mtmd.cpp
+3 -1
View File
@@ -2173,9 +2173,11 @@ bool mtmd_decode_use_non_causal(const mtmd_context * ctx, const mtmd_input_chunk
proj_type = ctx->proj_type_a();
}
switch (proj_type) {
case PROJECTOR_TYPE_GEMMA3:
case PROJECTOR_TYPE_GEMMA4V:
// E2B (n_embd = 1536) and E4B (n_embd = 2560) always use causal
return ctx->n_embd_text != 1536 && ctx->n_embd_text != 2560;
case PROJECTOR_TYPE_GEMMA4UV:
case PROJECTOR_TYPE_GEMMA3:
case PROJECTOR_TYPE_DEEPSEEK4V:
return true;
default: