mtmd: add minicpmv46 downsample (#25993)

* add minicpmv46 downsample

Signed-off-by: tc-mb <tianchi_cai@icloud.com>

* put downsample mode inside gguf.

Signed-off-by: tc-mb <tianchi_cai@icloud.com>

* build mtmd_image_preprocessor_llava_uhd

Signed-off-by: tc-mb <tianchi_cai@icloud.com>

* fix code

Signed-off-by: tc-mb <tianchi_cai@icloud.com>

* add convert

Signed-off-by: tc-mb <tianchi_cai@icloud.com>

* add 4x ignore vit merger

Signed-off-by: tc-mb <tianchi_cai@icloud.com>

---------

Signed-off-by: tc-mb <tianchi_cai@icloud.com>
This commit is contained in:
tc-mb
2026-08-01 13:38:36 +02:00
committed by GitHub
parent ddd4ec1428
commit de699957b9
6 changed files with 251 additions and 252 deletions
+11 -2
View File
@@ -137,6 +137,15 @@ class MiniCPMV4_6TextModel(Qwen3_5TextModel):
class MiniCPMV4_6VisionModel(MmprojModel):
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self.downsample_mode = self.preprocessor_config.get("downsample_mode", "16x")
if self.downsample_mode not in {"4x", "16x"}:
raise ValueError(f"Unsupported downsample mode: {self.downsample_mode}")
if self.downsample_mode == "4x":
self.model_tensors = {
name: tensor for name, tensor in self.model_tensors.items()
if ".vit_merger." not in name
}
if self.hparams_vision is not None:
# In MiniCPM-V 4.6 `vision_config.image_size` (980) describes the SigLIP
# positional embedding bucket grid (70 x 70), while the per-slice processing
@@ -156,8 +165,8 @@ class MiniCPMV4_6VisionModel(MmprojModel):
# (mapped to PROJECTOR_TYPE_MINICPMV4_6).
self.gguf_writer.add_clip_projector_type(gguf.VisionProjectorType.MINICPMV4_6)
# ViT merger 2x2 + final merger 2x2 = 4x spatial merge per dimension; used for slice alignment
self.gguf_writer.add_vision_projector_scale_factor(4)
self.gguf_writer.add_vision_projector_scale_factor(
2 if self.downsample_mode == "4x" else 4)
# borrow wa_layer_indexes for vit_merger insertion point
insert_layer_id = int(self.global_config.get(