Lumpiasty/llama.cpp
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

mtmd: support MiMo-V2.5 audio input (RVQ-based model) (#26190)

Browse Source 
* gguf converter for mimo audio

* fix conv

* cpp impl

* nits

* nits 2
This commit is contained in:
Xuan-Son Nguyen
2026-07-27 23:17:09 +02:00
committed by GitHub
parent 0e4a036223
commit 1cbfd19883
15 changed files with 770 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tools/mtmd/clip-model.h
+22
View File
@@ -124,6 +124,14 @@ struct clip_hparams {
int32_t audio_window_len = -1;
int32_t audio_hop_len = -1;
// mimo-audio-tokenizer: residual vector quantizer
int32_t rvq_num_quantizers = 0;
std::vector<int32_t> rvq_codebook_size; // per-quantizer bin count (ragged, e.g. 1024/1024/256/128x17)
// mimo-v2.5: LLM-side connector (input_local_transformer)
int32_t audio_local_n_layer = 0;
int32_t audio_local_group_size = 0;
// legacy
bool has_llava_projector = false;
int minicpmv_version = 0;
@@ -537,6 +545,20 @@ struct clip_model {
ggml_tensor * mm_norm_pre_b = nullptr;
ggml_tensor * mm_norm_mid_w = nullptr;
// mimo-audio-tokenizer: post-transformer downsample + RVQ codebook
ggml_tensor * downsample_conv_w = nullptr; // no bias
ggml_tensor * downsample_norm_w = nullptr;
ggml_tensor * downsample_norm_b = nullptr;
ggml_tensor * rvq_codebook = nullptr; // merged 3D [n_q, max_bins, dim]
// mimo-v2.5: text-side RVQ code embedding ("text codebook")
ggml_tensor * mm_a_code_embd = nullptr; // merged 3D [n_channels, vocab, dim]
// mimo-v2.5: LLM-side connector (input_local_transformer, separate from the
// audio_tokenizer's own encoder `layers`)
std::vector<clip_layer> mm_a_local_layers;
ggml_tensor * mm_a_local_norm_w = nullptr;
// qwen3a
ggml_tensor * conv2d_1_w = nullptr;
ggml_tensor * conv2d_1_b = nullptr;