mtmd: support dots3-note vision+audio (#27524)
* text: conversion * init impl * mtmd: conversion * impl mtmd cpp * Update gguf-py/gguf/tensor_mapping.py Co-authored-by: Sigbjørn Skjæret <sigbjorn.skjaeret@huggingface.co> --------- Co-authored-by: Sigbjørn Skjæret <sigbjorn.skjaeret@huggingface.co>
This commit is contained in:
Xuan-Son Nguyen
co-authored by
Sigbjørn Skjæret
Sigbjørn Skjæret
parent
3a653fea93
commit
54ee5ee643
@@ -1454,6 +1454,7 @@ class TensorNameMap:
|
||||
"mlp_AR.linear_{bid}", # PaddleOCR-VL
|
||||
"merger.mlp.{bid}",
|
||||
"vision_tower.merger.mlp.{bid}", # dots.ocr
|
||||
"vision_encoder.adapter.mlp.{bid}", # dots3note
|
||||
"vit.perceive.proj.{bid}", # HunyuanVL (proj.0 = conv1, proj.2 = conv2)
|
||||
),
|
||||
|
||||
@@ -1504,6 +1505,7 @@ class TensorNameMap:
|
||||
"vision_model.radio_model.model.patch_generator.embedder", # Nemotron Nano v2 VL
|
||||
"model.vision_tower.patch_embedder.input_proj", # gemma4
|
||||
"vision_tower.patch_embed.patchifier.proj", # dots.ocr
|
||||
"vision_encoder.patch_embed.proj", # dots3note
|
||||
"vision_model.conv1", # Step3-VL
|
||||
"model.vision_embedder.patch_dense", # gemma4 unified
|
||||
"model.vision_tower.patch_embedder.patch_embedding", # muse-glimmer
|
||||
@@ -1512,6 +1514,7 @@ class TensorNameMap:
|
||||
MODEL_TENSOR.V_ENC_EMBD_NORM: (
|
||||
"visual.post_conv_layernorm", # glm4v
|
||||
"vision_tower.patch_embed.patchifier.norm", # dots.ocr
|
||||
"vision_encoder.patch_embed.norm", # dots3note
|
||||
),
|
||||
|
||||
MODEL_TENSOR.V_ENC_EMBD_PATCH_NORM: (
|
||||
@@ -1551,6 +1554,7 @@ class TensorNameMap:
|
||||
MODEL_TENSOR.V_ENC_ATTN_QKV: (
|
||||
"visual.blocks.{bid}.attn.qkv", # qwen3vl
|
||||
"vision_tower.blocks.{bid}.attn.qkv", # dots.ocr
|
||||
"vision_encoder.blocks.{bid}.attn.qkv", # dots3note
|
||||
"model.vision.transformer.layers.{bid}.attention.query_key_value", # cogvlm
|
||||
"model.vision_model.transformer.layers.{bid}.self_attn.qkv_proj", # Deepseek-OCR CLIP
|
||||
"vision_tower.encoder.blocks.{bid}.wqkv", # Kimi-K2.5
|
||||
@@ -1579,6 +1583,7 @@ class TensorNameMap:
|
||||
),
|
||||
|
||||
MODEL_TENSOR.V_ENC_ATTN_Q_NORM: (
|
||||
"vision_encoder.blocks.{bid}.attn.q_norm", # dots3note
|
||||
"vision_tower.vision_model.encoder.layers.{bid}.attn.q_norm", # InternVL
|
||||
"model.vision_tower.encoder.layer.{bid}.attention.q_norm", # Intern-S1
|
||||
"visual.blocks.{bid}.attn.q_norm", # GLM-OCR
|
||||
@@ -1606,6 +1611,7 @@ class TensorNameMap:
|
||||
),
|
||||
|
||||
MODEL_TENSOR.V_ENC_ATTN_K_NORM: (
|
||||
"vision_encoder.blocks.{bid}.attn.k_norm", # dots3note
|
||||
"vision_tower.vision_model.encoder.layers.{bid}.attn.k_norm", # InternVL
|
||||
"model.vision_tower.encoder.layer.{bid}.attention.k_norm", # Intern-S1
|
||||
"visual.blocks.{bid}.attn.k_norm", # GLM-OCR
|
||||
@@ -1651,6 +1657,7 @@ class TensorNameMap:
|
||||
"siglip2.vision_model.encoder.layers.{bid}.layer_norm1",
|
||||
"vision_model.radio_model.model.blocks.{bid}.norm1", # Nemotron Nano v2 VL
|
||||
"vision_tower.blocks.{bid}.norm1", # dots.ocr
|
||||
"vision_encoder.blocks.{bid}.norm_1", # dots3note
|
||||
"vision_model.transformer.resblocks.{bid}.ln_1", # Step3-VL
|
||||
"model.qwen2_model.model.model.layers.{bid}.input_layernorm", # Deepseek-OCR-2 qwen2
|
||||
"model.vision_tower.layers.{bid}.norm1", # muse-glimmer
|
||||
@@ -1678,6 +1685,7 @@ class TensorNameMap:
|
||||
"model.qwen2_model.model.model.layers.{bid}.self_attn.o_proj", # Deepseek-OCR-2 qwen2
|
||||
"vision_model.model.layers.{bid}.self_attn.o_proj.linear", # gemma4
|
||||
"vision_tower.blocks.{bid}.attn.proj", # dots.ocr
|
||||
"vision_encoder.blocks.{bid}.attn.proj", # dots3note
|
||||
"vision_model.transformer.resblocks.{bid}.attn.out_proj", # Step3-VL
|
||||
"model.vision_tower.layers.{bid}.attn.proj", # muse-glimmer
|
||||
),
|
||||
@@ -1706,12 +1714,14 @@ class TensorNameMap:
|
||||
"vision_model.radio_model.model.blocks.{bid}.norm2", # Nemotron Nano v2 VL
|
||||
"vision_model.model.layers.{bid}.pre_feedforward_layernorm", # gemma4
|
||||
"vision_tower.blocks.{bid}.norm2", # dots.ocr
|
||||
"vision_encoder.blocks.{bid}.norm_2", # dots3note
|
||||
"vision_model.transformer.resblocks.{bid}.ln_2", # Step3-VL
|
||||
"model.qwen2_model.model.model.layers.{bid}.post_attention_layernorm", # Deepseek-OCR-2 qwen2
|
||||
"model.vision_tower.layers.{bid}.norm2", # muse-glimmer
|
||||
),
|
||||
|
||||
MODEL_TENSOR.V_ENC_FFN_UP: (
|
||||
"vision_encoder.blocks.{bid}.mlp.fc3", # dots3note
|
||||
"model.vision_tower.vision_model.encoder.layers.{bid}.mlp.fc1", # Granite4Vision
|
||||
"vision_tower.vision_model.encoder.layers.{bid}.mlp.fc1",
|
||||
"model.vision_tower.encoder.layers.{bid}.mlp.fc1", # minicpmv4_6
|
||||
@@ -1737,6 +1747,7 @@ class TensorNameMap:
|
||||
),
|
||||
|
||||
MODEL_TENSOR.V_ENC_FFN_GATE: (
|
||||
"vision_encoder.blocks.{bid}.mlp.fc1", # dots3note
|
||||
"vision_tower.transformer.layers.{bid}.feed_forward.gate_proj", # pixtral-hf
|
||||
"vision_encoder.transformer.layers.{bid}.feed_forward.w1", # pixtral
|
||||
"visual.blocks.{bid}.mlp.gate_proj", # qwen2.5vl
|
||||
@@ -1745,6 +1756,7 @@ class TensorNameMap:
|
||||
),
|
||||
|
||||
MODEL_TENSOR.V_ENC_FFN_DOWN: (
|
||||
"vision_encoder.blocks.{bid}.mlp.fc2", # dots3note
|
||||
"model.vision_tower.vision_model.encoder.layers.{bid}.mlp.fc2", # Granite4Vision
|
||||
"vision_tower.vision_model.encoder.layers.{bid}.mlp.fc2",
|
||||
"model.vision_tower.encoder.layers.{bid}.mlp.fc2", # minicpmv4_6
|
||||
@@ -1769,6 +1781,29 @@ class TensorNameMap:
|
||||
"model.vision_tower.layers.{bid}.mlp.fc2", # muse-glimmer
|
||||
),
|
||||
|
||||
|
||||
MODEL_TENSOR.V_ENC_FFN_GATE_INP: (
|
||||
"vision_encoder.blocks.{bid}.mlp.gate_weight", # dots3note
|
||||
),
|
||||
|
||||
MODEL_TENSOR.V_ENC_FFN_EXP_PROBS_B: (
|
||||
"vision_encoder.blocks.{bid}.mlp.router_bias", # dots3note
|
||||
),
|
||||
|
||||
# note: expert weights are stacked into a single 3D tensor in conversion code,
|
||||
# which emits the pseudo-names below
|
||||
MODEL_TENSOR.V_ENC_FFN_GATE_EXPS: (
|
||||
"vision_encoder.blocks.{bid}.mlp.experts.fc1", # dots3note
|
||||
),
|
||||
|
||||
MODEL_TENSOR.V_ENC_FFN_UP_EXPS: (
|
||||
"vision_encoder.blocks.{bid}.mlp.experts.fc3", # dots3note
|
||||
),
|
||||
|
||||
MODEL_TENSOR.V_ENC_FFN_DOWN_EXPS: (
|
||||
"vision_encoder.blocks.{bid}.mlp.experts.fc2", # dots3note
|
||||
),
|
||||
|
||||
MODEL_TENSOR.V_ENC_ATTN_POST_NORM: (
|
||||
"vision_model.model.layers.{bid}.post_attention_layernorm", # gemma4
|
||||
),
|
||||
@@ -1800,6 +1835,7 @@ class TensorNameMap:
|
||||
"vision_model.layernorm_pre", # llama4
|
||||
"model.vision_model.pre_layrnorm", # Deepseek-OCR CLIP
|
||||
"vision_tower.patch_embed.patchifier.norm", # dots.ocr
|
||||
"vision_encoder.patch_embed.norm", # dots3note
|
||||
"vision_model.ln_pre", # Step3-VL
|
||||
"model.vision_tower.ln_pre", # muse-glimmer
|
||||
),
|
||||
@@ -1821,6 +1857,7 @@ class TensorNameMap:
|
||||
MODEL_TENSOR.V_MM_POST_NORM: (
|
||||
"visual.merger.post_projection_norm", # glm4v
|
||||
"vision_tower.post_trunk_norm", # dots.ocr
|
||||
"vision_encoder.post_trunk_norm", # dots3note
|
||||
"vit.perceive.after_rms", # HunyuanVL
|
||||
),
|
||||
|
||||
@@ -1838,6 +1875,7 @@ class TensorNameMap:
|
||||
"mlp_AR.pre_norm", # PaddleOCR-VL
|
||||
"merger.ln_q",
|
||||
"vision_tower.merger.ln_q", # dots.ocr
|
||||
"vision_encoder.adapter.ln_q", # dots3note
|
||||
"model.merger.mlp.0.pre_norm", # minicpmv4_6
|
||||
),
|
||||
|
||||
@@ -2173,10 +2211,12 @@ class TensorNameMap:
|
||||
|
||||
MODEL_TENSOR.A_ENC_CONV2D: (
|
||||
"audio_tower.conv2d{bid}", # qwen3omni
|
||||
"audio_encoder.dots_encoder.speech_encoder.conv2d{bid}", # dots3note
|
||||
),
|
||||
|
||||
MODEL_TENSOR.A_ENC_CONV_OUT: (
|
||||
"audio_tower.conv_out", # qwen3omni
|
||||
"audio_encoder.dots_encoder.speech_encoder.conv_out", # dots3note
|
||||
"speaker_encoder.mfa.conv", # qwen3tts speaker encoder: multi-layer feature aggregation
|
||||
),
|
||||
|
||||
@@ -2184,12 +2224,14 @@ class TensorNameMap:
|
||||
|
||||
MODEL_TENSOR.A_POST_NORM: (
|
||||
"audio_tower.layer_norm", # ultravox
|
||||
"audio_encoder.dots_encoder.speech_encoder.layer_norm", # dots3note
|
||||
"audio_tower.ln_post", # qwen2omni
|
||||
"encoder.layer_norm", # mimo-audio-tokenizer
|
||||
),
|
||||
|
||||
MODEL_TENSOR.A_ENC_ATTN_Q: (
|
||||
"audio_tower.layers.{bid}.self_attn.q_proj", # ultravox
|
||||
"audio_encoder.dots_encoder.speech_encoder.layers.{bid}.self_attn.q_proj", # dots3note
|
||||
"conformer.layers.{bid}.self_attn.linear_q", # lfm2
|
||||
"conformer.layers.{bid}.attention.attn.q_proj", # gemma3n
|
||||
"conformer.layers.{bid}.self_attn.q_proj", # gemma4
|
||||
@@ -2200,6 +2242,7 @@ class TensorNameMap:
|
||||
|
||||
MODEL_TENSOR.A_ENC_ATTN_K: (
|
||||
"audio_tower.layers.{bid}.self_attn.k_proj", # ultravox
|
||||
"audio_encoder.dots_encoder.speech_encoder.layers.{bid}.self_attn.k_proj", # dots3note
|
||||
"conformer.layers.{bid}.self_attn.linear_k", # lfm2
|
||||
"conformer.layers.{bid}.attention.attn.k_proj", # gemma3n
|
||||
"conformer.layers.{bid}.self_attn.k_proj", # gemma4
|
||||
@@ -2210,6 +2253,7 @@ class TensorNameMap:
|
||||
|
||||
MODEL_TENSOR.A_ENC_ATTN_V: (
|
||||
"audio_tower.layers.{bid}.self_attn.v_proj", # ultravox
|
||||
"audio_encoder.dots_encoder.speech_encoder.layers.{bid}.self_attn.v_proj", # dots3note
|
||||
"conformer.layers.{bid}.self_attn.linear_v", # lfm2
|
||||
"conformer.layers.{bid}.attention.attn.v_proj", # gemma3n
|
||||
"conformer.layers.{bid}.self_attn.v_proj", # gemma4
|
||||
@@ -2241,6 +2285,7 @@ class TensorNameMap:
|
||||
|
||||
MODEL_TENSOR.A_ENC_INPUT_NORM: (
|
||||
"audio_tower.layers.{bid}.self_attn_layer_norm", # ultravox
|
||||
"audio_encoder.dots_encoder.speech_encoder.layers.{bid}.self_attn_layer_norm", # dots3note
|
||||
"conformer.layers.{bid}.norm_self_att", # lfm2
|
||||
"conformer.layers.{bid}.attention.pre_attn_norm", # gemma3n
|
||||
"sound_encoder.encoder.layers.{bid}.norm_self_att", # parakeet
|
||||
@@ -2250,6 +2295,7 @@ class TensorNameMap:
|
||||
|
||||
MODEL_TENSOR.A_ENC_OUTPUT: (
|
||||
"audio_tower.layers.{bid}.self_attn.out_proj", # ultravox
|
||||
"audio_encoder.dots_encoder.speech_encoder.layers.{bid}.self_attn.out_proj", # dots3note
|
||||
"conformer.layers.{bid}.self_attn.linear_out", # lfm2
|
||||
"conformer.layers.{bid}.attention.post", # gemma3n
|
||||
"conformer.layers.{bid}.self_attn.post", # gemma4
|
||||
@@ -2260,6 +2306,7 @@ class TensorNameMap:
|
||||
|
||||
MODEL_TENSOR.A_ENC_OUTPUT_NORM: (
|
||||
"audio_tower.layers.{bid}.final_layer_norm", # ultravox
|
||||
"audio_encoder.dots_encoder.speech_encoder.layers.{bid}.final_layer_norm", # dots3note
|
||||
"conformer.layers.{bid}.norm_out", # lfm2
|
||||
"conformer.layers.{bid}.attention.post_norm", # gemma3n
|
||||
"sound_encoder.encoder.layers.{bid}.norm_out", # parakeet
|
||||
@@ -2285,6 +2332,7 @@ class TensorNameMap:
|
||||
),
|
||||
|
||||
MODEL_TENSOR.A_ENC_FFN_UP: (
|
||||
"audio_encoder.dots_encoder.speech_encoder.layers.{bid}.fc1_up", # dots3note (split from fc1 in conversion code)
|
||||
"audio_tower.layers.{bid}.fc1", # ultravox
|
||||
"conformer.layers.{bid}.feed_forward1.linear1", # lfm2
|
||||
"conformer.layers.{bid}.ffw_layer_start.ffw_layer_1", # gemma3n
|
||||
@@ -2294,9 +2342,12 @@ class TensorNameMap:
|
||||
"encoder.layers.{bid}.fc1", # mimo-audio-tokenizer
|
||||
),
|
||||
|
||||
MODEL_TENSOR.A_ENC_FFN_GATE: (),
|
||||
MODEL_TENSOR.A_ENC_FFN_GATE: (
|
||||
"audio_encoder.dots_encoder.speech_encoder.layers.{bid}.fc1_gate", # dots3note (split from fc1 in conversion code)
|
||||
),
|
||||
|
||||
MODEL_TENSOR.A_ENC_FFN_DOWN: (
|
||||
"audio_encoder.dots_encoder.speech_encoder.layers.{bid}.fc2", # dots3note
|
||||
"audio_tower.layers.{bid}.fc2", # ultravox
|
||||
"conformer.layers.{bid}.feed_forward1.linear2", # lfm2
|
||||
"conformer.layers.{bid}.ffw_layer_start.ffw_layer_2", # gemma3n
|
||||
@@ -2380,6 +2431,7 @@ class TensorNameMap:
|
||||
|
||||
MODEL_TENSOR.A_MMPROJ: (
|
||||
"audio.multi_modal_projector.linear_{bid}", # ultravox, meralion
|
||||
"audio_encoder.audio_adapter.proj.{bid}", # dots3note (proj.1, proj.3)
|
||||
"audio_adapter.model.{bid}", # lfm2
|
||||
"audio_tower.proj{bid}", # qwen3omni
|
||||
"sound_projection.linear{bid}", # parakeet (linear1, linear2)
|
||||
@@ -2394,6 +2446,7 @@ class TensorNameMap:
|
||||
|
||||
MODEL_TENSOR.A_MM_NORM_PRE: (
|
||||
"audio.multi_modal_projector.ln_pre", # ultravox
|
||||
"audio_encoder.audio_adapter.proj.0", # dots3note
|
||||
"sound_projection.norm", # parakeet
|
||||
),
|
||||
|
||||
|
||||
Reference in New Issue
Block a user