mtmd: support higgs v3 STT and vibevoice-ASR (#32329)

* mtmd: support higgs v3 and vibevoice-ASR

* use proper mapping

* address review comment

* nits

* rm hacky get_audio_length, add mtmd_helper_get_media_meta
This commit is contained in:
Xuan-Son Nguyen authored and GitHub committed 2026-11-15 21:25:34 +01:00
1 parent 18b74fa41d
commit fc504f91d8
24 files changed
+1101 -18

No files matched your search

+122
View File
@@ -785,6 +785,46 @@ struct mtmd_model_loader {
}
hparams.warmup_audio_size = hparams.audio_samples_per_token();
} break;
case PROJECTOR_TYPE_VIBEVOICE_ASR:
{
hparams.audio_sample_rate = 24000; // mono waveform, no frontend
hparams.audio_chunk_len = 30; // in seconds, any length works as the convs are causal
get_u32(KEY_A_CONV_KERNEL_SIZE, hparams.audio_conv_kernel_size, true);
get_arr_int(KEY_A_PATCH_SIZES, hparams.audio_patch_sizes, true);
get_arr_int(KEY_A_DEPTHS, hparams.audio_depths, true);
// bounds keep the receptive field below from overflowing
const size_t n_stage = hparams.audio_patch_sizes.size();
if (n_stage == 0 || n_stage > 16 || hparams.audio_depths.size() != n_stage || hparams.n_mel_bins != 1 ||
hparams.audio_conv_kernel_size < 1 || hparams.audio_conv_kernel_size > 64) {
throw std::runtime_error(mtmd_format("%s: invalid vibevoice audio hparams\n", __func__));
}
// receptive field: stem conv (kernel), downsample convs (2 * stride), blocks (kernel), head conv (kernel)
const int64_t k = hparams.audio_conv_kernel_size;
int64_t hop = 1;
int64_t rf = 0;
int64_t n_blocks = 0;
for (size_t i = 0; i < n_stage; ++i) {
const int64_t stride = hparams.audio_patch_sizes[i];
if (stride < 1 || stride > 64 || hop * stride > (1 << 20) || hparams.audio_depths[i] < 0 || hparams.audio_depths[i] > 64) {
throw std::runtime_error(mtmd_format("%s: invalid vibevoice audio hparams\n", __func__));
}
rf += ((i == 0 ? k : 2 * stride) - 1) * hop;
hop *= stride;
rf += hparams.audio_depths[i] * (k - 1) * hop;
n_blocks += hparams.audio_depths[i];
}
rf += (k - 1) * hop;
if (rf > 60 * hparams.audio_sample_rate) {
throw std::runtime_error(mtmd_format("%s: vibevoice receptive field of %" PRId64 " samples is too long\n", __func__, rf));
}
if (n_blocks != hparams.n_layer) {
throw std::runtime_error(mtmd_format("%s: sum of audio depths (%" PRId64 ") does not match n_layer (%d)\n", __func__, n_blocks, hparams.n_layer));
}
hparams.audio_n_ctx = (int32_t) ((rf + hop - 1) / hop * hop);
hparams.warmup_audio_size = (int32_t) ((int64_t) hparams.audio_chunk_len * hparams.audio_sample_rate / hop * hop) + hparams.audio_n_ctx;
} break;
case PROJECTOR_TYPE_INKLING2V:
{
hparams.ffn_op = FFN_GELU_ERF;
@@ -892,6 +932,7 @@ struct mtmd_model_loader {
} break;
case PROJECTOR_TYPE_ULTRAVOX:
case PROJECTOR_TYPE_QWEN2A:
case PROJECTOR_TYPE_HIGGS_AUDIO:
case PROJECTOR_TYPE_QWEN3A:
case PROJECTOR_TYPE_GLMA:
case PROJECTOR_TYPE_VOXTRAL:
@@ -912,6 +953,12 @@ struct mtmd_model_loader {
hparams.audio_n_fft = 400;
hparams.audio_window_len = 400;
hparams.audio_hop_len = 160;
if (model.proj_type == PROJECTOR_TYPE_HIGGS_AUDIO) {
// each 4s chunk is encoded on its own
hparams.audio_chunk_len = 4;
hparams.warmup_audio_size = 400;
}
} break;
case PROJECTOR_TYPE_VOXTRAL_RT:
{
@@ -1324,6 +1371,7 @@ struct mtmd_model_loader {
// Muse Spark is encoder-free in the same sense: its conv pyramid reads the raw waveform, one channel takes the place of the mel bins
const bool fft_based = model.proj_type != PROJECTOR_TYPE_GEMMA4UA &&
model.proj_type != PROJECTOR_TYPE_MUSE_SPARK_AUDIO &&
model.proj_type != PROJECTOR_TYPE_VIBEVOICE_ASR &&
model.proj_type != PROJECTOR_TYPE_POCKETTTS_SPKENC;
// Validate audio hparams loaded from GGUF metadata
@@ -1497,6 +1545,7 @@ struct mtmd_model_loader {
model.proj_type != PROJECTOR_TYPE_POCKETTTS_GEN &&
model.proj_type != PROJECTOR_TYPE_SOPRANO &&
model.proj_type != PROJECTOR_TYPE_MUSE_SPARK_AUDIO &&
model.proj_type != PROJECTOR_TYPE_VIBEVOICE_ASR &&
model.proj_type != PROJECTOR_TYPE_CHATTERBOX &&
model.proj_type != PROJECTOR_TYPE_CHATTERBOX_SPKENC);
@@ -1928,6 +1977,52 @@ struct mtmd_model_loader {
model.mm_fc_w = get_tensor(mtmd_format(TN_MM_AUDIO_FC, "weight"));
model.mm_fc_b = get_tensor(mtmd_format(TN_MM_AUDIO_FC, "bias"));
} break;
case PROJECTOR_TYPE_VIBEVOICE_ASR:
{
const int n_stage = (int) hparams.audio_patch_sizes.size();
for (int e = 0; e < 2; ++e) {
auto & enc = model.cnx_enc[e];
enc.conv_w.resize(n_stage + 1);
enc.conv_b.resize(n_stage + 1);
for (int i = 0; i <= n_stage; ++i) {
enc.conv_w[i] = get_tensor(mtmd_format(TN_A_CNX_CONV, e, i, "weight"));
enc.conv_b[i] = get_tensor(mtmd_format(TN_A_CNX_CONV, e, i, "bias"));
// the receptive field in load_hparams assumes these kernel sizes
const int64_t k_expected = (i == 0 || i == n_stage) ? hparams.audio_conv_kernel_size : 2 * hparams.audio_patch_sizes[i];
if (enc.conv_w[i]->ne[0] != k_expected) {
throw std::runtime_error(mtmd_format("%s: %s has kernel %" PRId64 ", expected %" PRId64 "\n", __func__, ggml_get_name(enc.conv_w[i]), enc.conv_w[i]->ne[0], k_expected));
}
}
enc.blocks.resize(hparams.n_layer);
for (int il = 0; il < hparams.n_layer; ++il) {
auto & layer = enc.blocks[il];
layer.ln_1_w = get_tensor(mtmd_format(TN_A_CNX_NORM, e, il, "weight"));
layer.conv_w = get_tensor(mtmd_format(TN_A_CNX_DW, e, il, "weight"));
layer.conv_b = get_tensor(mtmd_format(TN_A_CNX_DW, e, il, "bias"));
layer.ls_1_w = get_tensor(mtmd_format(TN_A_CNX_LS1, e, il, "weight"));
layer.ln_2_w = get_tensor(mtmd_format(TN_A_CNX_FFN_NORM, e, il, "weight"));
layer.ff_up_w = get_tensor(mtmd_format(TN_A_CNX_FFN_UP, e, il, "weight"));
layer.ff_up_b = get_tensor(mtmd_format(TN_A_CNX_FFN_UP, e, il, "bias"));
layer.ff_down_w = get_tensor(mtmd_format(TN_A_CNX_FFN_DOWN, e, il, "weight"));
layer.ff_down_b = get_tensor(mtmd_format(TN_A_CNX_FFN_DOWN, e, il, "bias"));
layer.ls_2_w = get_tensor(mtmd_format(TN_A_CNX_LS2, e, il, "weight"));
if (layer.conv_w->ne[0] != hparams.audio_conv_kernel_size || layer.conv_w->type != GGML_TYPE_F32) {
throw std::runtime_error(mtmd_format("%s: %s must be F32 with kernel %d\n", __func__, ggml_get_name(layer.conv_w), hparams.audio_conv_kernel_size));
}
}
}
// acoustic connector: mlp 0 -> norm -> mlp 1, semantic connector: mlp 2 -> norm -> mlp 3
model.mm_0_w = get_tensor(mtmd_format(TN_MM_AUDIO_MLP, 0, "weight"));
model.mm_0_b = get_tensor(mtmd_format(TN_MM_AUDIO_MLP, 0, "bias"));
model.mm_norm_pre_w = get_tensor(mtmd_format(TN_MM_AUDIO_MLP_NORM, 0, "weight"));
model.mm_1_w = get_tensor(mtmd_format(TN_MM_AUDIO_MLP, 1, "weight"));
model.mm_1_b = get_tensor(mtmd_format(TN_MM_AUDIO_MLP, 1, "bias"));
model.mm_2_w = get_tensor(mtmd_format(TN_MM_AUDIO_MLP, 2, "weight"));
model.mm_2_b = get_tensor(mtmd_format(TN_MM_AUDIO_MLP, 2, "bias"));
model.mm_norm_mid_w = get_tensor(mtmd_format(TN_MM_AUDIO_MLP_NORM, 2, "weight"));
model.mm_3_w = get_tensor(mtmd_format(TN_MM_AUDIO_MLP, 3, "weight"));
model.mm_3_b = get_tensor(mtmd_format(TN_MM_AUDIO_MLP, 3, "bias"));
} break;
case PROJECTOR_TYPE_INKLING2V:
{
// these come from the generic path above, where they are optional
@@ -2315,6 +2410,19 @@ struct mtmd_model_loader {
model.mm_fc_w = get_tensor(mtmd_format(TN_MM_AUDIO_FC, "weight"));
model.mm_fc_b = get_tensor(mtmd_format(TN_MM_AUDIO_FC, "bias"));
} break;
case PROJECTOR_TYPE_HIGGS_AUDIO:
{
model.conv1d_1_w = get_tensor(mtmd_format(TN_CONV1D, 1, "weight"));
model.conv1d_1_b = get_tensor(mtmd_format(TN_CONV1D, 1, "bias"));
model.conv1d_2_w = get_tensor(mtmd_format(TN_CONV1D, 2, "weight"));
model.conv1d_2_b = get_tensor(mtmd_format(TN_CONV1D, 2, "bias"));
model.downsample_conv_w = get_tensor(mtmd_format(TN_A_DOWNSAMPLE_CONV, "weight"));
model.downsample_conv_b = get_tensor(mtmd_format(TN_A_DOWNSAMPLE_CONV, "bias"));
model.mm_1_w = get_tensor(mtmd_format(TN_MM_AUDIO_MLP, 1, "weight"));
model.mm_1_b = get_tensor(mtmd_format(TN_MM_AUDIO_MLP, 1, "bias"));
model.mm_2_w = get_tensor(mtmd_format(TN_MM_AUDIO_MLP, 2, "weight"));
model.mm_2_b = get_tensor(mtmd_format(TN_MM_AUDIO_MLP, 2, "bias"));
} break;
case PROJECTOR_TYPE_QWEN3A:
{
model.conv2d_1_w = get_tensor(mtmd_format(TN_CONV2D, 1, "weight"));
@@ -4187,6 +4295,10 @@ int mtmd_model::n_output_tokens(const mtmd_image_f32 * img) const {
const int64_t fold = hparams.audio_samples_per_token();
n_patches = (int) ((img->nx() + fold - 1) / fold);
} break;
case PROJECTOR_TYPE_VIBEVOICE_ASR:
{
n_patches = (img->nx() - img->audio_n_ctx) / hparams.audio_samples_per_token();
} break;
case PROJECTOR_TYPE_VOXTRAL:
case PROJECTOR_TYPE_ULTRAVOX:
case PROJECTOR_TYPE_QWEN2A:
@@ -4210,6 +4322,13 @@ int mtmd_model::n_output_tokens(const mtmd_image_f32 * img) const {
n_patches /= 2;
}
} break;
case PROJECTOR_TYPE_HIGGS_AUDIO:
{
// stride-2 conv1d, avgpool(2), stride-2 depthwise conv1d
n_patches = (img->nx() + 1) / 2;
n_patches = n_patches / 2;
n_patches = (n_patches + 1) / 2;
} break;
case PROJECTOR_TYPE_QWEN3A:
{
// chunk_size=100 frames --> 3x stride-2 conv2d --> 13 tokens per chunk
@@ -4449,6 +4568,8 @@ int mtmd_model::n_mmproj_embd() const {
case PROJECTOR_TYPE_IDEFICS3:
case PROJECTOR_TYPE_MUSE_SPARK_AUDIO:
return mm_fc_w->ne[1];
case PROJECTOR_TYPE_VIBEVOICE_ASR:
return mm_1_w->ne[1];
case PROJECTOR_TYPE_ULTRAVOX:
case PROJECTOR_TYPE_VOXTRAL:
case PROJECTOR_TYPE_VOXTRAL_RT:
@@ -4466,6 +4587,7 @@ int mtmd_model::n_mmproj_embd() const {
case PROJECTOR_TYPE_NEMOTRON_TRANSCRIBE:
return mm_fc_w->ne[1];
case PROJECTOR_TYPE_QWEN3A:
case PROJECTOR_TYPE_HIGGS_AUDIO:
return mm_2_w->ne[1];
case PROJECTOR_TYPE_GLMA:
case PROJECTOR_TYPE_LFM2: