Lumpiasty/llama.cpp
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

mtmd: support DeepSeek-V4-Flash-Vision-Exp (#28133)

Browse Source 
* mtmd: support DeepSeek-V4-Flash-Vision-Exp

* handle min/max token counts from CLI

* rm debugging

* use GGML_ROPE_TYPE_VISION

* nits

* apply review comments

* correct token count
This commit is contained in:
Xuan-Son Nguyen
2026-09-02 16:43:43 +02:00
committed by GitHub
parent 8e93a9773b
commit 7798007a29
13 changed files with 504 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files
gguf-py/gguf/tensor_mapping.py
+23
View File
@@ -1476,6 +1476,7 @@ class TensorNameMap:
## Vision encoder
MODEL_TENSOR.V_MMPROJ: (
"aligner.w{bid}", # deepseek4v (w1 -> mm.1, w2 -> mm.2)
"multi_modal_projector.linear_{bid}",
"mm_projector.proj.linear_{bid}", # Kimi-K2.5
"visual.merger.mlp.{bid}", # qwen2vl
@@ -1515,6 +1516,7 @@ class TensorNameMap:
),
MODEL_TENSOR.V_ENC_EMBD_PATCH: (
"vision.patch_embed.proj", # deepseek4v
"model.vision_tower.vision_model.embeddings.patch_embedding", # Granite4Vision
"vision_tower.vision_model.embeddings.patch_embedding",
"model.vision_tower.embeddings.patch_embedding", # minicpmv4_6
@@ -1570,6 +1572,7 @@ class TensorNameMap:
# TODO: I think these should all be moved to mapping_cfg?
MODEL_TENSOR.V_ENC_EMBD_IMGNL: (
"image_newline", # deepseek4v
"model.image_newline", # Deepseek-OCR, Granite4Vision
"vit.perceive.image_newline", # HunyuanVL
),
@@ -1580,6 +1583,7 @@ class TensorNameMap:
),
MODEL_TENSOR.V_ENC_ATTN_QKV: (
"vision.blocks.{bid}.attn.wqkv", # deepseek4v
"visual.blocks.{bid}.attn.qkv", # qwen3vl
"vision_tower.blocks.{bid}.attn.qkv", # dots.ocr
"vision_encoder.blocks.{bid}.attn.qkv", # dots3note
@@ -1667,6 +1671,7 @@ class TensorNameMap:
),
MODEL_TENSOR.V_ENC_INPUT_NORM: (
"vision.blocks.{bid}.norm1", # deepseek4v
"model.vision_tower.vision_model.encoder.layers.{bid}.layer_norm1", # Granite4Vision
"vision_tower.vision_model.encoder.layers.{bid}.layer_norm1",
"model.vision_tower.encoder.layers.{bid}.layer_norm1", # minicpmv4_6
@@ -1692,6 +1697,7 @@ class TensorNameMap:
),
MODEL_TENSOR.V_ENC_ATTN_O: (
"vision.blocks.{bid}.attn.wo", # deepseek4v
"model.vision_tower.vision_model.encoder.layers.{bid}.self_attn.out_proj", # Granite4Vision
"vision_tower.vision_model.encoder.layers.{bid}.self_attn.out_proj",
"model.vision_tower.encoder.layers.{bid}.self_attn.out_proj", # minicpmv4_6
@@ -1723,6 +1729,7 @@ class TensorNameMap:
),
MODEL_TENSOR.V_ENC_POST_ATTN_NORM: (
"vision.blocks.{bid}.norm2", # deepseek4v
"model.vision_tower.vision_model.encoder.layers.{bid}.layer_norm2", # Granite4Vision
"vision_tower.vision_model.encoder.layers.{bid}.layer_norm2",
"model.vision_tower.encoder.layers.{bid}.layer_norm2", # minicpmv4_6
@@ -1749,6 +1756,7 @@ class TensorNameMap:
),
MODEL_TENSOR.V_ENC_FFN_UP: (
"vision.blocks.{bid}.mlp.w1_up", # deepseek4v (split from fused w1)
"vision_encoder.blocks.{bid}.mlp.fc3", # dots3note
"model.vision_tower.vision_model.encoder.layers.{bid}.mlp.fc1", # Granite4Vision
"vision_tower.vision_model.encoder.layers.{bid}.mlp.fc1",
@@ -1775,6 +1783,7 @@ class TensorNameMap:
),
MODEL_TENSOR.V_ENC_FFN_GATE: (
"vision.blocks.{bid}.mlp.w1_gate", # deepseek4v (split from fused w1)
"vision_encoder.blocks.{bid}.mlp.fc1", # dots3note
"vision_tower.transformer.layers.{bid}.feed_forward.gate_proj", # pixtral-hf
"vision_encoder.transformer.layers.{bid}.feed_forward.w1", # pixtral
@@ -1784,6 +1793,7 @@ class TensorNameMap:
),
MODEL_TENSOR.V_ENC_FFN_DOWN: (
"vision.blocks.{bid}.mlp.w2", # deepseek4v
"vision_encoder.blocks.{bid}.mlp.fc2", # dots3note
"model.vision_tower.vision_model.encoder.layers.{bid}.mlp.fc2", # Granite4Vision
"vision_tower.vision_model.encoder.layers.{bid}.mlp.fc2",
@@ -1869,6 +1879,7 @@ class TensorNameMap:
),
MODEL_TENSOR.V_POST_NORM: (
"vision.norm", # deepseek4v
"model.vision_tower.vision_model.post_layernorm", # Granite4Vision
"vision_tower.vision_model.post_layernorm",
"model.vision_tower.post_layernorm", # minicpmv4_6
@@ -1960,6 +1971,18 @@ class TensorNameMap:
"v.token_embd.img_break", # for pixtral, this is a generated vector
),
MODEL_TENSOR.V_TOK_EMBD_IMG_START: (
"image_start", # deepseek4v
),
MODEL_TENSOR.V_TOK_EMBD_IMG_END: (
"image_end", # deepseek4v
),
MODEL_TENSOR.V_TOK_EMBD_IMG_PAD: (
"image_pad", # deepseek4v
),
MODEL_TENSOR.V_MM_PATCH_MERGER: (
"multi_modal_projector.patch_merger.merging_layer", # mistral small 3.1 - hf
"patch_merger.merging_layer", # mistral