Files
llama.cpp/tools/mtmd/models/vibevoice-asr.cpp
T
Xuan-Son Nguyen fc504f91d8 mtmd: support higgs v3 STT and vibevoice-ASR (#32329)
* mtmd: support higgs v3 and vibevoice-ASR

* use proper mapping

* address review comment

* nits

* rm hacky get_audio_length, add mtmd_helper_get_media_meta
2026-11-15 21:25:34 +01:00

139 lines
6.2 KiB
C++

#include "models.h"
// VibeVoice-ASR audio tower: two causal ConvNeXt pyramids (acoustic and semantic) over the raw 24 kHz waveform
// each one is a VAE tokenizer encoder, its latents go through their own connector and the two are summed
// see modeling_vibevoice_acoustic_tokenizer.py and modeling_vibevoice_asr.py in transformers
//
// tensors are T-first [T, C] around the convs and C-first [C, T] in the blocks
static std::string cnx_slot(int e, const char * kind, int i) {
return "e" + std::to_string(e) + "_" + kind + "_" + std::to_string(i);
}
std::vector<c2w_state_slot> list_vibevoice_asr_state_slots(const mtmd_hparams & hparams, const mtmd_model & model) {
std::vector<c2w_state_slot> slots;
slots.push_back({"pos", 1, 1}); // unused, set_enc_state_in() expects it
const int n_stage = (int) hparams.audio_patch_sizes.size();
for (int e = 0; e < 2; ++e) {
const auto & enc = model.cnx_enc[e];
for (int i = 0; i <= n_stage; ++i) {
const int64_t stride = i < n_stage ? hparams.audio_patch_sizes[i] : 1;
slots.push_back({cnx_slot(e, "conv", i), enc.conv_w[i]->ne[0] - stride, enc.conv_w[i]->ne[1]});
}
for (int il = 0; il < (int) enc.blocks.size(); ++il) {
const auto * w = enc.blocks[il].conv_w;
slots.push_back({cnx_slot(e, "dw", il), w->ne[0] - 1, w->ne[1]});
}
}
return slots;
}
// x: [T, C] -> [left + T, C], the left context comes from the state, its new value is the tail
ggml_tensor * mtmd_graph_vibevoice_asr::with_left_ctx(ggml_tensor * x, const std::string & name) const {
ggml_tensor * left = state_in.at(name);
x = ggml_concat(ctx0, left, x, 0);
state_out.push_back({name, ggml_cont(ctx0, ggml_view_2d(ctx0, x, left->ne[0], x->ne[1], x->nb[1], (x->ne[0] - left->ne[0]) * x->nb[0]))});
return x;
}
// x: [T, IC], w: [K, IC, OC] -> [T / stride, OC]
ggml_tensor * mtmd_graph_vibevoice_asr::causal_conv(ggml_tensor * x, ggml_tensor * w, ggml_tensor * b, int stride, const std::string & name) const {
x = with_left_ctx(x, name);
ggml_tensor * y = ggml_conv_1d(ctx0, w, x, stride, 0, 1);
y = ggml_reshape_2d(ctx0, y, y->ne[0], y->ne[1]);
return ggml_add(ctx0, y, ggml_reshape_2d(ctx0, b, 1, b->ne[0]));
}
// x: [T, 1] -> [C_latent, T / hop]
ggml_tensor * mtmd_graph_vibevoice_asr::encode(int e, ggml_tensor * x) const {
const auto & enc = model.cnx_enc[e];
const int n_stage = (int) hparams.audio_patch_sizes.size();
ggml_tensor * cur = x;
int bid = 0;
for (int stage = 0; stage < n_stage; ++stage) {
cur = causal_conv(cur, enc.conv_w[stage], enc.conv_b[stage], hparams.audio_patch_sizes[stage], cnx_slot(e, "conv", stage));
cur = ggml_cont(ctx0, ggml_transpose(ctx0, cur));
cb(cur, "stage_in", stage);
const int64_t n_pos = cur->ne[1];
for (int i = 0; i < hparams.audio_depths[stage]; ++i, ++bid) {
const auto & layer = enc.blocks[bid];
// causal depthwise conv mixer
ggml_tensor * h = build_norm(cur, layer.ln_1_w, nullptr, NORM_TYPE_RMS, hparams.eps, bid);
h = ggml_cont(ctx0, ggml_transpose(ctx0, h));
h = with_left_ctx(h, cnx_slot(e, "dw", bid));
h = ggml_reshape_3d(ctx0, h, h->ne[0], h->ne[1], 1);
h = ggml_ssm_conv(ctx0, h, layer.conv_w);
h = ggml_reshape_2d(ctx0, h, h->ne[0], n_pos);
h = ggml_add(ctx0, h, layer.conv_b);
cur = ggml_add(ctx0, cur, ggml_mul(ctx0, h, layer.ls_1_w));
h = build_norm(cur, layer.ln_2_w, nullptr, NORM_TYPE_RMS, hparams.eps, bid);
h = build_ffn(h,
layer.ff_up_w, layer.ff_up_b,
nullptr, nullptr,
layer.ff_down_w, layer.ff_down_b,
FFN_GELU_ERF, bid);
cur = ggml_add(ctx0, cur, ggml_mul(ctx0, h, layer.ls_2_w));
cb(cur, "block_out", bid);
}
cur = ggml_cont(ctx0, ggml_transpose(ctx0, cur));
}
cur = causal_conv(cur, enc.conv_w[n_stage], enc.conv_b[n_stage], 1, cnx_slot(e, "conv", n_stage));
return ggml_cont(ctx0, ggml_transpose(ctx0, cur));
}
ggml_cgraph * mtmd_graph_vibevoice_asr::build() {
const int64_t hop = hparams.audio_samples_per_token();
const int64_t n_frames = img.nx() / hop;
const int64_t n_ctx = img.audio_n_ctx / hop;
GGML_ASSERT(img.nx() % hop == 0 && img.audio_n_ctx % hop == 0 && n_ctx < n_frames);
for (const auto & slot : list_vibevoice_asr_state_slots(hparams, model)) {
ggml_tensor * t = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, slot.ne0, slot.ne1);
ggml_set_name(t, ("state_in_" + slot.name).c_str());
ggml_set_input(t);
state_in[slot.name] = t;
}
state_out.push_back({"pos", ggml_scale_bias(ctx0, state_in.at("pos"), 1.0f, (float) n_frames)});
ggml_tensor * inp = build_inp_raw(1);
inp = ggml_reshape_2d(ctx0, inp, img.nx(), 1);
// the leading context frames only warm up the causal convs
auto drop_ctx = [&](ggml_tensor * t) {
return ggml_view_2d(ctx0, t, t->ne[0], n_frames - n_ctx, t->nb[1], n_ctx * t->nb[1]);
};
// the reference adds VAE noise to the acoustic latents, skipped here to keep the output deterministic
ggml_tensor * acoustic = drop_ctx(encode(0, inp));
ggml_tensor * semantic = drop_ctx(encode(1, inp));
cb(acoustic, "acoustic_latents", -1);
cb(semantic, "semantic_latents", -1);
acoustic = ggml_add(ctx0, build_mm(model.mm_0_w, acoustic), model.mm_0_b);
acoustic = build_norm(acoustic, model.mm_norm_pre_w, nullptr, NORM_TYPE_RMS, 1e-6f, -1);
acoustic = ggml_add(ctx0, build_mm(model.mm_1_w, acoustic), model.mm_1_b);
semantic = ggml_add(ctx0, build_mm(model.mm_2_w, semantic), model.mm_2_b);
semantic = build_norm(semantic, model.mm_norm_mid_w, nullptr, NORM_TYPE_RMS, 1e-6f, -1);
semantic = ggml_add(ctx0, build_mm(model.mm_3_w, semantic), model.mm_3_b);
ggml_tensor * cur = ggml_add(ctx0, acoustic, semantic);
cb(cur, "projected", -1);
// the embeddings must stay the last node
for (const auto & s : state_out) {
ggml_set_name(s.second, ("state_out_" + s.first).c_str());
ggml_set_output(s.second);
ggml_build_forward_expand(gf, s.second);
}
ggml_build_forward_expand(gf, cur);
return gf;
}