Lumpiasty/llama.cpp
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

mtmd: support DeepSeek-V4-Flash-Vision-Exp (#28133)

Browse Source 
* mtmd: support DeepSeek-V4-Flash-Vision-Exp

* handle min/max token counts from CLI

* rm debugging

* use GGML_ROPE_TYPE_VISION

* nits

* apply review comments

* correct token count
This commit is contained in:
Xuan-Son Nguyen
2026-09-02 16:43:43 +02:00
committed by GitHub
parent 8e93a9773b
commit 7798007a29
13 changed files with 504 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files
gguf-py/gguf/constants.py
+10
View File
@@ -950,6 +950,9 @@ class MODEL_TENSOR(IntEnum):
V_RESMPL_PROJ = auto() # minicpmv
V_RESMPL_QUERY = auto() # minicpmv
V_TOK_EMBD_IMG_BREAK = auto() # pixtral
V_TOK_EMBD_IMG_START = auto() # deepseek4v
V_TOK_EMBD_IMG_END = auto() # deepseek4v
V_TOK_EMBD_IMG_PAD = auto() # deepseek4v
V_MM_PATCH_MERGER = auto() # mistral small 3.1
V_DS_NORM = auto() # qwen3vl
V_DS_FC1 = auto() # qwen3vl
@@ -1696,6 +1699,9 @@ TENSOR_NAMES: dict[MODEL_TENSOR, str] = {
MODEL_TENSOR.V_RESMPL_PROJ: "resampler.proj",
MODEL_TENSOR.V_RESMPL_QUERY: "resampler.query",
MODEL_TENSOR.V_TOK_EMBD_IMG_BREAK: "v.token_embd.img_break", # pixtral
MODEL_TENSOR.V_TOK_EMBD_IMG_START: "v.token_embd.img_start", # deepseek4v
MODEL_TENSOR.V_TOK_EMBD_IMG_END: "v.token_embd.img_end", # deepseek4v
MODEL_TENSOR.V_TOK_EMBD_IMG_PAD: "v.token_embd.img_pad", # deepseek4v
MODEL_TENSOR.V_MM_PATCH_MERGER: "mm.patch_merger", # mistral small 3.1
MODEL_TENSOR.V_DS_NORM: "v.deepstack.{bid}.norm",
MODEL_TENSOR.V_DS_FC1: "v.deepstack.{bid}.fc1",
@@ -2030,6 +2036,9 @@ MODEL_TENSORS: dict[MODEL_ARCH, list[MODEL_TENSOR]] = {
MODEL_TENSOR.V_RESMPL_PROJ,
MODEL_TENSOR.V_RESMPL_QUERY,
MODEL_TENSOR.V_TOK_EMBD_IMG_BREAK,
MODEL_TENSOR.V_TOK_EMBD_IMG_START,
MODEL_TENSOR.V_TOK_EMBD_IMG_END,
MODEL_TENSOR.V_TOK_EMBD_IMG_PAD,
MODEL_TENSOR.V_MM_PATCH_MERGER,
MODEL_TENSOR.V_MM_MERGER_FC1,
MODEL_TENSOR.V_MM_MERGER_FC2,
@@ -5645,6 +5654,7 @@ class VisionProjectorType:
DOTS3NOTE_A = "dots3note_a" # audio
DEEPSEEKOCR = "deepseekocr"
DEEPSEEKOCR2 = "deepseekocr2"
DEEPSEEK4V = "deepseek4v"
LFM2A = "lfm2a" # audio
MUSIC_FLAMINGO = "musicflamingo" # audio
GLM4V = "glm4v"
gguf-py/gguf/tensor_mapping.py
+23
View File
@@ -1476,6 +1476,7 @@ class TensorNameMap:
## Vision encoder
MODEL_TENSOR.V_MMPROJ: (
"aligner.w{bid}", # deepseek4v (w1 -> mm.1, w2 -> mm.2)
"multi_modal_projector.linear_{bid}",
"mm_projector.proj.linear_{bid}", # Kimi-K2.5
"visual.merger.mlp.{bid}", # qwen2vl
@@ -1515,6 +1516,7 @@ class TensorNameMap:
),
MODEL_TENSOR.V_ENC_EMBD_PATCH: (
"vision.patch_embed.proj", # deepseek4v
"model.vision_tower.vision_model.embeddings.patch_embedding", # Granite4Vision
"vision_tower.vision_model.embeddings.patch_embedding",
"model.vision_tower.embeddings.patch_embedding", # minicpmv4_6
@@ -1570,6 +1572,7 @@ class TensorNameMap:
# TODO: I think these should all be moved to mapping_cfg?
MODEL_TENSOR.V_ENC_EMBD_IMGNL: (
"image_newline", # deepseek4v
"model.image_newline", # Deepseek-OCR, Granite4Vision
"vit.perceive.image_newline", # HunyuanVL
),
@@ -1580,6 +1583,7 @@ class TensorNameMap:
),
MODEL_TENSOR.V_ENC_ATTN_QKV: (
"vision.blocks.{bid}.attn.wqkv", # deepseek4v
"visual.blocks.{bid}.attn.qkv", # qwen3vl
"vision_tower.blocks.{bid}.attn.qkv", # dots.ocr
"vision_encoder.blocks.{bid}.attn.qkv", # dots3note
@@ -1667,6 +1671,7 @@ class TensorNameMap:
),
MODEL_TENSOR.V_ENC_INPUT_NORM: (
"vision.blocks.{bid}.norm1", # deepseek4v
"model.vision_tower.vision_model.encoder.layers.{bid}.layer_norm1", # Granite4Vision
"vision_tower.vision_model.encoder.layers.{bid}.layer_norm1",
"model.vision_tower.encoder.layers.{bid}.layer_norm1", # minicpmv4_6
@@ -1692,6 +1697,7 @@ class TensorNameMap:
),
MODEL_TENSOR.V_ENC_ATTN_O: (
"vision.blocks.{bid}.attn.wo", # deepseek4v
"model.vision_tower.vision_model.encoder.layers.{bid}.self_attn.out_proj", # Granite4Vision
"vision_tower.vision_model.encoder.layers.{bid}.self_attn.out_proj",
"model.vision_tower.encoder.layers.{bid}.self_attn.out_proj", # minicpmv4_6
@@ -1723,6 +1729,7 @@ class TensorNameMap:
),
MODEL_TENSOR.V_ENC_POST_ATTN_NORM: (
"vision.blocks.{bid}.norm2", # deepseek4v
"model.vision_tower.vision_model.encoder.layers.{bid}.layer_norm2", # Granite4Vision
"vision_tower.vision_model.encoder.layers.{bid}.layer_norm2",
"model.vision_tower.encoder.layers.{bid}.layer_norm2", # minicpmv4_6
@@ -1749,6 +1756,7 @@ class TensorNameMap:
),
MODEL_TENSOR.V_ENC_FFN_UP: (
"vision.blocks.{bid}.mlp.w1_up", # deepseek4v (split from fused w1)
"vision_encoder.blocks.{bid}.mlp.fc3", # dots3note
"model.vision_tower.vision_model.encoder.layers.{bid}.mlp.fc1", # Granite4Vision
"vision_tower.vision_model.encoder.layers.{bid}.mlp.fc1",
@@ -1775,6 +1783,7 @@ class TensorNameMap:
),
MODEL_TENSOR.V_ENC_FFN_GATE: (
"vision.blocks.{bid}.mlp.w1_gate", # deepseek4v (split from fused w1)
"vision_encoder.blocks.{bid}.mlp.fc1", # dots3note
"vision_tower.transformer.layers.{bid}.feed_forward.gate_proj", # pixtral-hf
"vision_encoder.transformer.layers.{bid}.feed_forward.w1", # pixtral
@@ -1784,6 +1793,7 @@ class TensorNameMap:
),
MODEL_TENSOR.V_ENC_FFN_DOWN: (
"vision.blocks.{bid}.mlp.w2", # deepseek4v
"vision_encoder.blocks.{bid}.mlp.fc2", # dots3note
"model.vision_tower.vision_model.encoder.layers.{bid}.mlp.fc2", # Granite4Vision
"vision_tower.vision_model.encoder.layers.{bid}.mlp.fc2",
@@ -1869,6 +1879,7 @@ class TensorNameMap:
),
MODEL_TENSOR.V_POST_NORM: (
"vision.norm", # deepseek4v
"model.vision_tower.vision_model.post_layernorm", # Granite4Vision
"vision_tower.vision_model.post_layernorm",
"model.vision_tower.post_layernorm", # minicpmv4_6
@@ -1960,6 +1971,18 @@ class TensorNameMap:
"v.token_embd.img_break", # for pixtral, this is a generated vector
),
MODEL_TENSOR.V_TOK_EMBD_IMG_START: (
"image_start", # deepseek4v
),
MODEL_TENSOR.V_TOK_EMBD_IMG_END: (
"image_end", # deepseek4v
),
MODEL_TENSOR.V_TOK_EMBD_IMG_PAD: (
"image_pad", # deepseek4v
),
MODEL_TENSOR.V_MM_PATCH_MERGER: (
"multi_modal_projector.patch_merger.merging_layer", # mistral small 3.1 - hf
"patch_merger.merging_layer", # mistral