Lumpiasty/llama.cpp
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

mtmd: support MiMo-V2.5 audio input (RVQ-based model) (#26190)

Browse Source 
* gguf converter for mimo audio

* fix conv

* cpp impl

* nits

* nits 2
This commit is contained in:
Xuan-Son Nguyen
2026-07-27 23:17:09 +02:00
committed by GitHub
parent 0e4a036223
commit 1cbfd19883
15 changed files with 770 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files
gguf-py/gguf/tensor_mapping.py
+59
View File
@@ -2095,6 +2095,7 @@ class TensorNameMap:
"conformer.pre_encode.conv.{bid}", # lfm2
"model.audio_tower.subsample_conv_projection.conv_{bid}.conv", # gemma3n
"conformer.subsample_conv_projection.layer{bid}.conv", # gemma4
"encoder.conv{bid}", # mimo-audio-tokenizer
),
MODEL_TENSOR.A_ENC_CONV1D_NORM: (
@@ -2119,6 +2120,7 @@ class TensorNameMap:
MODEL_TENSOR.A_POST_NORM: (
"audio_tower.layer_norm", # ultravox
"audio_tower.ln_post", # qwen2omni
"encoder.layer_norm", # mimo-audio-tokenizer
),
MODEL_TENSOR.A_ENC_ATTN_Q: (
@@ -2127,6 +2129,7 @@ class TensorNameMap:
"conformer.layers.{bid}.attention.attn.q_proj", # gemma3n
"conformer.layers.{bid}.self_attn.q_proj", # gemma4
"encoder.layers.{bid}.attn.to_q", # granite_speech
"encoder.layers.{bid}.self_attn.q_proj", # mimo-audio-tokenizer
),
MODEL_TENSOR.A_ENC_ATTN_K: (
@@ -2135,6 +2138,7 @@ class TensorNameMap:
"conformer.layers.{bid}.attention.attn.k_proj", # gemma3n
"conformer.layers.{bid}.self_attn.k_proj", # gemma4
"encoder.layers.{bid}.attn.to_k", # granite_speech (split from to_kv)
"encoder.layers.{bid}.self_attn.k_proj", # mimo-audio-tokenizer
),
MODEL_TENSOR.A_ENC_ATTN_V: (
@@ -2143,6 +2147,7 @@ class TensorNameMap:
"conformer.layers.{bid}.attention.attn.v_proj", # gemma3n
"conformer.layers.{bid}.self_attn.v_proj", # gemma4
"encoder.layers.{bid}.attn.to_v", # granite_speech (split from to_kv)
"encoder.layers.{bid}.self_attn.v_proj", # mimo-audio-tokenizer
),
MODEL_TENSOR.A_ENC_ATTN_K_REL: (
@@ -2171,6 +2176,7 @@ class TensorNameMap:
"conformer.layers.{bid}.norm_self_att", # lfm2
"conformer.layers.{bid}.attention.pre_attn_norm", # gemma3n
"encoder.layers.{bid}.attn.pre_norm", # granite_speech
"encoder.layers.{bid}.self_attn_layer_norm", # mimo-audio-tokenizer
),
MODEL_TENSOR.A_ENC_OUTPUT: (
@@ -2179,6 +2185,7 @@ class TensorNameMap:
"conformer.layers.{bid}.attention.post", # gemma3n
"conformer.layers.{bid}.self_attn.post", # gemma4
"encoder.layers.{bid}.attn.to_out", # granite_speech
"encoder.layers.{bid}.self_attn.out_proj", # mimo-audio-tokenizer
),
MODEL_TENSOR.A_ENC_OUTPUT_NORM: (
@@ -2186,6 +2193,7 @@ class TensorNameMap:
"conformer.layers.{bid}.norm_out", # lfm2
"conformer.layers.{bid}.attention.post_norm", # gemma3n
"encoder.layers.{bid}.post_norm", # granite_speech
"encoder.layers.{bid}.final_layer_norm", # mimo-audio-tokenizer
),
MODEL_TENSOR.A_ENC_FFN_NORM: (
@@ -2210,6 +2218,7 @@ class TensorNameMap:
"conformer.layers.{bid}.ffw_layer_start.ffw_layer_1", # gemma3n
"conformer.layers.{bid}.feed_forward1.ffw_layer_1", # gemma4
"encoder.layers.{bid}.ff1.up_proj", # granite_speech
"encoder.layers.{bid}.fc1", # mimo-audio-tokenizer
),
MODEL_TENSOR.A_ENC_FFN_GATE: (),
@@ -2220,6 +2229,7 @@ class TensorNameMap:
"conformer.layers.{bid}.ffw_layer_start.ffw_layer_2", # gemma3n
"conformer.layers.{bid}.feed_forward1.ffw_layer_2", # gemma4
"encoder.layers.{bid}.ff1.down_proj", # granite_speech
"encoder.layers.{bid}.fc2", # mimo-audio-tokenizer
),
MODEL_TENSOR.A_ENC_FFN_UP_1: (
@@ -2243,6 +2253,19 @@ class TensorNameMap:
"encoder.layers.{bid}.ff2.pre_norm", # granite_speech
),
MODEL_TENSOR.A_ENC_DOWNSAMPLE_CONV: (
"encoder.down_sample_layer.0", # mimo-audio-tokenizer
),
MODEL_TENSOR.A_ENC_DOWNSAMPLE_NORM: (
"encoder.down_sample_norm", # mimo-audio-tokenizer
),
# note: the raw per-quantizer "encoder.quantizer.vq.layers.{i}._codebook.embed"
# tensors are merged (padded + stacked, like MoE experts) into this single 3D
# tensor in conversion code, so no raw-name mapping is registered here.
MODEL_TENSOR.A_ENC_RVQ_CODEBOOK: (),
MODEL_TENSOR.A_ENC_FFN_POST_NORM_1: (
"conformer.layers.{bid}.ffw_layer_end.post_layer_norm", # gemma3n
"conformer.layers.{bid}.feed_forward2.post_layer_norm", # gemma4
@@ -2294,6 +2317,42 @@ class TensorNameMap:
"audio.multi_modal_projector.ln_mid", # ultravox
),
# note: the raw per-channel "speech_embeddings.{i}" tensors are merged
# (stacked, like MoE experts) into this single 3D tensor in conversion
# code, so no raw-name mapping is registered here.
MODEL_TENSOR.A_MM_CODE_EMBD: (),
MODEL_TENSOR.A_MM_LOCAL_ATTN_Q: (
"audio_encoder.input_local_transformer.layers.{bid}.self_attn.q_proj", # mimo-v2.5
),
MODEL_TENSOR.A_MM_LOCAL_ATTN_K: (
"audio_encoder.input_local_transformer.layers.{bid}.self_attn.k_proj", # mimo-v2.5
),
MODEL_TENSOR.A_MM_LOCAL_ATTN_V: (
"audio_encoder.input_local_transformer.layers.{bid}.self_attn.v_proj", # mimo-v2.5
),
MODEL_TENSOR.A_MM_LOCAL_ATTN_OUT: (
"audio_encoder.input_local_transformer.layers.{bid}.self_attn.o_proj", # mimo-v2.5
),
MODEL_TENSOR.A_MM_LOCAL_FFN_GATE: (
"audio_encoder.input_local_transformer.layers.{bid}.mlp.gate_proj", # mimo-v2.5
),
MODEL_TENSOR.A_MM_LOCAL_FFN_UP: (
"audio_encoder.input_local_transformer.layers.{bid}.mlp.up_proj", # mimo-v2.5
),
MODEL_TENSOR.A_MM_LOCAL_FFN_DOWN: (
"audio_encoder.input_local_transformer.layers.{bid}.mlp.down_proj", # mimo-v2.5
),
MODEL_TENSOR.A_MM_LOCAL_LN1: (
"audio_encoder.input_local_transformer.layers.{bid}.input_layernorm", # mimo-v2.5
),
MODEL_TENSOR.A_MM_LOCAL_LN2: (
"audio_encoder.input_local_transformer.layers.{bid}.post_attention_layernorm", # mimo-v2.5
),
MODEL_TENSOR.A_MM_LOCAL_NORM: (
"audio_encoder.input_local_transformer.norm", # mimo-v2.5
),
MODEL_TENSOR.A_ENC_CONV_DW: (
"conformer.layers.{bid}.conv.depthwise_conv", # lfm2
"conformer.layers.{bid}.lconv1d.depthwise_conv1d", # gemma3n