Lumpiasty/llama.cpp
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

mtmd: support MiMo-V2.5 audio input (RVQ-based model) (#26190)

Browse Source 
* gguf converter for mimo audio

* fix conv

* cpp impl

* nits

* nits 2
This commit is contained in:
Xuan-Son Nguyen
2026-07-27 23:17:09 +02:00
committed by GitHub
parent 0e4a036223
commit 1cbfd19883
15 changed files with 770 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files
gguf-py/gguf/constants.py
+49
View File
@@ -376,6 +376,12 @@ class Keys:
CONV_KERNEL_SIZE = "clip.audio.conv_kernel_size"
MAX_POS_EMB = "clip.audio.max_pos_emb"
FEATURE_LAYERS = "clip.audio.feature_layer" # Granite Speech Plus
RVQ_NUM_QUANTIZERS = "clip.audio.rvq.num_quantizers"
RVQ_CODEBOOK_SIZE = "clip.audio.rvq.codebook_size"
WA_PATTERN_MODE = "clip.audio.wa_pattern_mode" # per-layer -1 (full) / 0 (windowed)
WINDOW_SIZE = "clip.audio.window_size"
LOCAL_BLOCK_COUNT = "clip.audio.local_block_count" # mimo-v2.5: input_local_transformer layer count
LOCAL_GROUP_SIZE = "clip.audio.local_group_size" # mimo-v2.5: input_local_transformer grouping size
class Attention:
HEAD_COUNT = "clip.audio.attention.head_count"
@@ -945,6 +951,9 @@ class MODEL_TENSOR(IntEnum):
A_ENC_FFN_SCALE_1 = auto() # gemma3n
A_ENC_FFN_GATE_1 = auto() # lfm2, gemma3n
A_ENC_FFN_DOWN_1 = auto() # lfm2, gemma3n
A_ENC_DOWNSAMPLE_CONV = auto() # mimo-audio-tokenizer: post-transformer downsample conv
A_ENC_DOWNSAMPLE_NORM = auto() # mimo-audio-tokenizer: post-transformer downsample norm
A_ENC_RVQ_CODEBOOK = auto() # mimo-audio-tokenizer: residual vector quantizer codebook, per quantizer index
A_MMPROJ = auto()
A_MMPROJ_FC = auto()
A_MM_NORM_PRE = auto()
@@ -953,6 +962,17 @@ class MODEL_TENSOR(IntEnum):
A_MM_HARD_EMB_NORM = auto() # gemma3n
A_MM_SOFT_EMB_NORM = auto() # gemma3n
A_MM_INP_PROJ = auto() # gemma3n
A_MM_CODE_EMBD = auto() # mimo: text-side RVQ code embedding table ("text codebook"), merged 3D [n_channels, vocab, dim]
A_MM_LOCAL_ATTN_Q = auto() # mimo: input_local_transformer (LLM-side connector)
A_MM_LOCAL_ATTN_K = auto()
A_MM_LOCAL_ATTN_V = auto()
A_MM_LOCAL_ATTN_OUT = auto()
A_MM_LOCAL_FFN_GATE = auto()
A_MM_LOCAL_FFN_UP = auto()
A_MM_LOCAL_FFN_DOWN = auto()
A_MM_LOCAL_LN1 = auto()
A_MM_LOCAL_LN2 = auto()
A_MM_LOCAL_NORM = auto() # final norm after all input_local_transformer layers
A_PER_DIM_K_SCALE = auto() # gemma4
A_PER_DIM_SCALE = auto() # gemma4
# nextn/mtp
@@ -1532,6 +1552,9 @@ TENSOR_NAMES: dict[MODEL_TENSOR, str] = {
MODEL_TENSOR.A_ENC_FFN_UP_1: "a.blk.{bid}.ffn_up_1",
MODEL_TENSOR.A_ENC_FFN_GATE_1: "a.blk.{bid}.ffn_gate_1",
MODEL_TENSOR.A_ENC_FFN_DOWN_1: "a.blk.{bid}.ffn_down_1",
MODEL_TENSOR.A_ENC_DOWNSAMPLE_CONV: "a.downsample.conv",
MODEL_TENSOR.A_ENC_DOWNSAMPLE_NORM: "a.downsample.norm",
MODEL_TENSOR.A_ENC_RVQ_CODEBOOK: "a.rvq.codebook",
MODEL_TENSOR.A_MMPROJ: "mm.a.mlp.{bid}",
MODEL_TENSOR.A_MMPROJ_FC: "mm.a.fc",
MODEL_TENSOR.A_MM_NORM_PRE: "mm.a.norm_pre",
@@ -1540,6 +1563,17 @@ TENSOR_NAMES: dict[MODEL_TENSOR, str] = {
MODEL_TENSOR.A_MM_SOFT_EMB_NORM: "mm.a.soft_emb_norm", # gemma3n
MODEL_TENSOR.A_MM_EMBEDDING: "mm.a.embedding", # gemma3n
MODEL_TENSOR.A_MM_HARD_EMB_NORM: "mm.a.hard_emb_norm", # gemma3n
MODEL_TENSOR.A_MM_CODE_EMBD: "mm.a.code_embd",
MODEL_TENSOR.A_MM_LOCAL_ATTN_Q: "mm.a.local_blk.{bid}.attn_q",
MODEL_TENSOR.A_MM_LOCAL_ATTN_K: "mm.a.local_blk.{bid}.attn_k",
MODEL_TENSOR.A_MM_LOCAL_ATTN_V: "mm.a.local_blk.{bid}.attn_v",
MODEL_TENSOR.A_MM_LOCAL_ATTN_OUT: "mm.a.local_blk.{bid}.attn_out",
MODEL_TENSOR.A_MM_LOCAL_FFN_GATE: "mm.a.local_blk.{bid}.ffn_gate",
MODEL_TENSOR.A_MM_LOCAL_FFN_UP: "mm.a.local_blk.{bid}.ffn_up",
MODEL_TENSOR.A_MM_LOCAL_FFN_DOWN: "mm.a.local_blk.{bid}.ffn_down",
MODEL_TENSOR.A_MM_LOCAL_LN1: "mm.a.local_blk.{bid}.ln1",
MODEL_TENSOR.A_MM_LOCAL_LN2: "mm.a.local_blk.{bid}.ln2",
MODEL_TENSOR.A_MM_LOCAL_NORM: "mm.a.local_norm",
MODEL_TENSOR.A_PER_DIM_K_SCALE: "a.blk.{bid}.per_dim_k_scale", # gemma4
MODEL_TENSOR.A_PER_DIM_SCALE: "a.blk.{bid}.per_dim_scale", # gemma4
# lfm2 audio
@@ -1741,10 +1775,24 @@ MODEL_TENSORS: dict[MODEL_ARCH, list[MODEL_TENSOR]] = {
MODEL_TENSOR.A_ENC_FFN_UP_1,
MODEL_TENSOR.A_ENC_FFN_GATE_1,
MODEL_TENSOR.A_ENC_FFN_DOWN_1,
MODEL_TENSOR.A_ENC_DOWNSAMPLE_CONV,
MODEL_TENSOR.A_ENC_DOWNSAMPLE_NORM,
MODEL_TENSOR.A_ENC_RVQ_CODEBOOK,
MODEL_TENSOR.A_MMPROJ,
MODEL_TENSOR.A_MMPROJ_FC,
MODEL_TENSOR.A_MM_NORM_PRE,
MODEL_TENSOR.A_MM_NORM_MID,
MODEL_TENSOR.A_MM_CODE_EMBD,
MODEL_TENSOR.A_MM_LOCAL_ATTN_Q,
MODEL_TENSOR.A_MM_LOCAL_ATTN_K,
MODEL_TENSOR.A_MM_LOCAL_ATTN_V,
MODEL_TENSOR.A_MM_LOCAL_ATTN_OUT,
MODEL_TENSOR.A_MM_LOCAL_FFN_GATE,
MODEL_TENSOR.A_MM_LOCAL_FFN_UP,
MODEL_TENSOR.A_MM_LOCAL_FFN_DOWN,
MODEL_TENSOR.A_MM_LOCAL_LN1,
MODEL_TENSOR.A_MM_LOCAL_LN2,
MODEL_TENSOR.A_MM_LOCAL_NORM,
MODEL_TENSOR.A_ENC_NORM_CONV,
MODEL_TENSOR.A_ENC_LINEAR_POS,
MODEL_TENSOR.A_ENC_POS_BIAS_U,
@@ -4804,6 +4852,7 @@ class VisionProjectorType:
MINICPMV4_6 = "minicpmv4_6"
GRANITE_SPEECH = "granite_speech" # audio
MIMOVL = "mimovl"
MIMO_AUDIO = "mimo_audio"
GRANITE4_VISION = "granite4_vision"
gguf-py/gguf/gguf_writer.py
+18
View File
@@ -1350,6 +1350,24 @@ class GGUFWriter:
def add_audio_num_mel_bins(self, value: int) -> None:
self.add_uint32(Keys.ClipAudio.NUM_MEL_BINS, value)
def add_audio_rvq_num_quantizers(self, value: int) -> None:
self.add_uint32(Keys.ClipAudio.RVQ_NUM_QUANTIZERS, value)
def add_audio_rvq_codebook_size(self, values: Sequence[int]) -> None:
self.add_array(Keys.ClipAudio.RVQ_CODEBOOK_SIZE, values)
def add_audio_wa_pattern_mode(self, modes: Sequence[int]) -> None:
self.add_array(Keys.ClipAudio.WA_PATTERN_MODE, modes)
def add_audio_window_size(self, value: int) -> None:
self.add_uint32(Keys.ClipAudio.WINDOW_SIZE, value)
def add_audio_local_block_count(self, value: int) -> None:
self.add_uint32(Keys.ClipAudio.LOCAL_BLOCK_COUNT, value)
def add_audio_local_group_size(self, value: int) -> None:
self.add_uint32(Keys.ClipAudio.LOCAL_GROUP_SIZE, value)
def add_audio_stack_factor(self, value: int) -> None:
self.add_uint32(Keys.ClipAudio.Projector.STACK_FACTOR, value)
gguf-py/gguf/tensor_mapping.py
+59
View File
@@ -2095,6 +2095,7 @@ class TensorNameMap:
"conformer.pre_encode.conv.{bid}", # lfm2
"model.audio_tower.subsample_conv_projection.conv_{bid}.conv", # gemma3n
"conformer.subsample_conv_projection.layer{bid}.conv", # gemma4
"encoder.conv{bid}", # mimo-audio-tokenizer
),
MODEL_TENSOR.A_ENC_CONV1D_NORM: (
@@ -2119,6 +2120,7 @@ class TensorNameMap:
MODEL_TENSOR.A_POST_NORM: (
"audio_tower.layer_norm", # ultravox
"audio_tower.ln_post", # qwen2omni
"encoder.layer_norm", # mimo-audio-tokenizer
),
MODEL_TENSOR.A_ENC_ATTN_Q: (
@@ -2127,6 +2129,7 @@ class TensorNameMap:
"conformer.layers.{bid}.attention.attn.q_proj", # gemma3n
"conformer.layers.{bid}.self_attn.q_proj", # gemma4
"encoder.layers.{bid}.attn.to_q", # granite_speech
"encoder.layers.{bid}.self_attn.q_proj", # mimo-audio-tokenizer
),
MODEL_TENSOR.A_ENC_ATTN_K: (
@@ -2135,6 +2138,7 @@ class TensorNameMap:
"conformer.layers.{bid}.attention.attn.k_proj", # gemma3n
"conformer.layers.{bid}.self_attn.k_proj", # gemma4
"encoder.layers.{bid}.attn.to_k", # granite_speech (split from to_kv)
"encoder.layers.{bid}.self_attn.k_proj", # mimo-audio-tokenizer
),
MODEL_TENSOR.A_ENC_ATTN_V: (
@@ -2143,6 +2147,7 @@ class TensorNameMap:
"conformer.layers.{bid}.attention.attn.v_proj", # gemma3n
"conformer.layers.{bid}.self_attn.v_proj", # gemma4
"encoder.layers.{bid}.attn.to_v", # granite_speech (split from to_kv)
"encoder.layers.{bid}.self_attn.v_proj", # mimo-audio-tokenizer
),
MODEL_TENSOR.A_ENC_ATTN_K_REL: (
@@ -2171,6 +2176,7 @@ class TensorNameMap:
"conformer.layers.{bid}.norm_self_att", # lfm2
"conformer.layers.{bid}.attention.pre_attn_norm", # gemma3n
"encoder.layers.{bid}.attn.pre_norm", # granite_speech
"encoder.layers.{bid}.self_attn_layer_norm", # mimo-audio-tokenizer
),
MODEL_TENSOR.A_ENC_OUTPUT: (
@@ -2179,6 +2185,7 @@ class TensorNameMap:
"conformer.layers.{bid}.attention.post", # gemma3n
"conformer.layers.{bid}.self_attn.post", # gemma4
"encoder.layers.{bid}.attn.to_out", # granite_speech
"encoder.layers.{bid}.self_attn.out_proj", # mimo-audio-tokenizer
),
MODEL_TENSOR.A_ENC_OUTPUT_NORM: (
@@ -2186,6 +2193,7 @@ class TensorNameMap:
"conformer.layers.{bid}.norm_out", # lfm2
"conformer.layers.{bid}.attention.post_norm", # gemma3n
"encoder.layers.{bid}.post_norm", # granite_speech
"encoder.layers.{bid}.final_layer_norm", # mimo-audio-tokenizer
),
MODEL_TENSOR.A_ENC_FFN_NORM: (
@@ -2210,6 +2218,7 @@ class TensorNameMap:
"conformer.layers.{bid}.ffw_layer_start.ffw_layer_1", # gemma3n
"conformer.layers.{bid}.feed_forward1.ffw_layer_1", # gemma4
"encoder.layers.{bid}.ff1.up_proj", # granite_speech
"encoder.layers.{bid}.fc1", # mimo-audio-tokenizer
),
MODEL_TENSOR.A_ENC_FFN_GATE: (),
@@ -2220,6 +2229,7 @@ class TensorNameMap:
"conformer.layers.{bid}.ffw_layer_start.ffw_layer_2", # gemma3n
"conformer.layers.{bid}.feed_forward1.ffw_layer_2", # gemma4
"encoder.layers.{bid}.ff1.down_proj", # granite_speech
"encoder.layers.{bid}.fc2", # mimo-audio-tokenizer
),
MODEL_TENSOR.A_ENC_FFN_UP_1: (
@@ -2243,6 +2253,19 @@ class TensorNameMap:
"encoder.layers.{bid}.ff2.pre_norm", # granite_speech
),
MODEL_TENSOR.A_ENC_DOWNSAMPLE_CONV: (
"encoder.down_sample_layer.0", # mimo-audio-tokenizer
),
MODEL_TENSOR.A_ENC_DOWNSAMPLE_NORM: (
"encoder.down_sample_norm", # mimo-audio-tokenizer
),
# note: the raw per-quantizer "encoder.quantizer.vq.layers.{i}._codebook.embed"
# tensors are merged (padded + stacked, like MoE experts) into this single 3D
# tensor in conversion code, so no raw-name mapping is registered here.
MODEL_TENSOR.A_ENC_RVQ_CODEBOOK: (),
MODEL_TENSOR.A_ENC_FFN_POST_NORM_1: (
"conformer.layers.{bid}.ffw_layer_end.post_layer_norm", # gemma3n
"conformer.layers.{bid}.feed_forward2.post_layer_norm", # gemma4
@@ -2294,6 +2317,42 @@ class TensorNameMap:
"audio.multi_modal_projector.ln_mid", # ultravox
),
# note: the raw per-channel "speech_embeddings.{i}" tensors are merged
# (stacked, like MoE experts) into this single 3D tensor in conversion
# code, so no raw-name mapping is registered here.
MODEL_TENSOR.A_MM_CODE_EMBD: (),
MODEL_TENSOR.A_MM_LOCAL_ATTN_Q: (
"audio_encoder.input_local_transformer.layers.{bid}.self_attn.q_proj", # mimo-v2.5
),
MODEL_TENSOR.A_MM_LOCAL_ATTN_K: (
"audio_encoder.input_local_transformer.layers.{bid}.self_attn.k_proj", # mimo-v2.5
),
MODEL_TENSOR.A_MM_LOCAL_ATTN_V: (
"audio_encoder.input_local_transformer.layers.{bid}.self_attn.v_proj", # mimo-v2.5
),
MODEL_TENSOR.A_MM_LOCAL_ATTN_OUT: (
"audio_encoder.input_local_transformer.layers.{bid}.self_attn.o_proj", # mimo-v2.5
),
MODEL_TENSOR.A_MM_LOCAL_FFN_GATE: (
"audio_encoder.input_local_transformer.layers.{bid}.mlp.gate_proj", # mimo-v2.5
),
MODEL_TENSOR.A_MM_LOCAL_FFN_UP: (
"audio_encoder.input_local_transformer.layers.{bid}.mlp.up_proj", # mimo-v2.5
),
MODEL_TENSOR.A_MM_LOCAL_FFN_DOWN: (
"audio_encoder.input_local_transformer.layers.{bid}.mlp.down_proj", # mimo-v2.5
),
MODEL_TENSOR.A_MM_LOCAL_LN1: (
"audio_encoder.input_local_transformer.layers.{bid}.input_layernorm", # mimo-v2.5
),
MODEL_TENSOR.A_MM_LOCAL_LN2: (
"audio_encoder.input_local_transformer.layers.{bid}.post_attention_layernorm", # mimo-v2.5
),
MODEL_TENSOR.A_MM_LOCAL_NORM: (
"audio_encoder.input_local_transformer.norm", # mimo-v2.5
),
MODEL_TENSOR.A_ENC_CONV_DW: (
"conformer.layers.{bid}.conv.depthwise_conv", # lfm2
"conformer.layers.{bid}.lconv1d.depthwise_conv1d", # gemma3n