mtmd: support higgs v3 STT and vibevoice-ASR (#32329)

* mtmd: support higgs v3 and vibevoice-ASR

* use proper mapping

* address review comment

* nits

* rm hacky get_audio_length, add mtmd_helper_get_media_meta
This commit is contained in:
Xuan-Son Nguyen authored and GitHub committed 2026-11-15 21:25:34 +01:00
1 parent 18b74fa41d
commit fc504f91d8
24 files changed
+1101 -18

No files matched your search

+274 -2
View File
@@ -48,7 +48,7 @@ std::string mtmd_format(const char * fmt, ...) {
}
// remember to bump this if the serialization format changes
#define MTMD_SERIALIZATION_VERSION 4
#define MTMD_SERIALIZATION_VERSION 5
// oldest compat version that can be loaded
#define MTMD_SERIALIZATION_VERSION_MIN 2
@@ -137,6 +137,7 @@ void mtmd_image_f32::serialize(mtmd_serialization & ser) const {
ser.write((int32_t)anyres.orig_ny);
ser.write((int32_t)nx_);
ser.write((int32_t)ny_);
ser.write(audio_n_ctx);
}
void mtmd_image_f32::deserialize(mtmd_serialization & ser) {
add_viewsep = ser.read<bool>();
@@ -155,6 +156,12 @@ void mtmd_image_f32::deserialize(mtmd_serialization & ser) {
}
nx_ = ser.read<int32_t>();
ny_ = ser.read<int32_t>();
if (ser.version >= 5) {
audio_n_ctx = ser.read<int32_t>();
if (audio_n_ctx < 0 || audio_n_ctx > nx_) {
throw std::runtime_error("invalid audio_n_ctx");
}
}
buf.clear(); // always a placeholder after loading
}
@@ -1194,6 +1201,19 @@ struct mtmd_context {
aud_end = "<|audio_eos|>";
audio_preproc = std::make_unique<mtmd_audio_preprocessor_whisper>(models.a.get());
} break;
case PROJECTOR_TYPE_HIGGS_AUDIO:
{
// the 4s chunks are joined in one <|audio_bos|> ... <|audio_eos|> span
aud_beg = "<|audio_bos|>";
aud_end = "<|audio_eos|>";
audio_preproc = std::make_unique<mtmd_audio_preprocessor_higgs_audio>(models.a.get());
} break;
case PROJECTOR_TYPE_VIBEVOICE_ASR:
{
aud_beg = "<|object_ref_start|>";
aud_end = "<|object_ref_end|>";
audio_preproc = std::make_unique<mtmd_audio_preprocessor_vibevoice_asr>(models.a.get());
} break;
case PROJECTOR_TYPE_MUSE_SPARK_AUDIO:
{
aud_beg = "<|soft_audio_start|>";
@@ -4270,12 +4290,58 @@ struct mtmd_audio_stream_gemma4a : mtmd_audio_stream {
bool encode(int n);
};
// Higgs Audio: each 4 s chunk is encoded on its own, so a chunk goes out once it is complete
// a flush encodes the partial chunk as a chunk of its own, the reference also cuts chunks at VAD boundaries
struct mtmd_audio_stream_higgs_audio : mtmd_audio_stream {
int64_t chunk_len = 0; // samples per complete chunk
std::vector<float> beg_rows;
std::vector<float> end_rows;
bool beg_done = false;
bool end_done = false;
bool init(const mtmd_audio_stream_params & params) override;
int32_t flush(bool end) override;
int32_t step(llama_token last_token) override;
int64_t n_chunk_rows(int64_t n) const;
bool encode(int64_t n);
};
// VibeVoice-ASR: the encoder is causal, its convs keep their left context in the state
// the reference scales the whole clip to -25 dBFS, here the gain comes from the audio pushed so far (see mtmd_audio_vibevoice_gain)
struct mtmd_audio_stream_vibevoice_asr : mtmd_audio_stream {
mtmd_subctx_enc_state * enc_state = nullptr;
int64_t n_step = 0; // samples per encoder call
int64_t n_step_max = 0; // largest call, the warmup size
int64_t n_stat = 0; // pushed samples counted in sum_sq and peak
double sum_sq = 0.0;
double peak = 0.0;
std::vector<float> beg_rows;
std::vector<float> end_rows;
bool beg_done = false;
bool end_done = false;
~mtmd_audio_stream_vibevoice_asr() override {
delete enc_state;
}
bool init(const mtmd_audio_stream_params & params) override;
int32_t flush(bool end) override;
int32_t step(llama_token last_token) override;
bool encode(int64_t n);
};
bool mtmd_support_audio_stream(const mtmd_context * ctx) {
if (!ctx->ctx_a || !ctx->text_model) {
return false;
}
const projector_type proj = ctx->ctx_a->model.proj_type;
return proj == PROJECTOR_TYPE_VOXTRAL_RT || proj == PROJECTOR_TYPE_LFM3A || proj == PROJECTOR_TYPE_GEMMA4A;
return proj == PROJECTOR_TYPE_VOXTRAL_RT || proj == PROJECTOR_TYPE_LFM3A || proj == PROJECTOR_TYPE_GEMMA4A ||
proj == PROJECTOR_TYPE_HIGGS_AUDIO || proj == PROJECTOR_TYPE_VIBEVOICE_ASR;
}
mtmd_audio_stream_params mtmd_audio_stream_params_default(void) {
@@ -4298,6 +4364,8 @@ mtmd_audio_stream * mtmd_audio_stream_init(mtmd_context * ctx, mtmd_audio_stream
case PROJECTOR_TYPE_VOXTRAL_RT: s = std::make_unique<mtmd_audio_stream_voxtral_rt>(); break;
case PROJECTOR_TYPE_LFM3A: s = std::make_unique<mtmd_audio_stream_lfm3a>(); break;
case PROJECTOR_TYPE_GEMMA4A: s = std::make_unique<mtmd_audio_stream_gemma4a>(); break;
case PROJECTOR_TYPE_HIGGS_AUDIO: s = std::make_unique<mtmd_audio_stream_higgs_audio>(); break;
case PROJECTOR_TYPE_VIBEVOICE_ASR: s = std::make_unique<mtmd_audio_stream_vibevoice_asr>(); break;
default: GGML_ABORT("unsupported projector");
}
s->ctx = ctx;
@@ -5036,6 +5104,210 @@ int32_t mtmd_audio_stream_gemma4a::step(llama_token /* last_token */) {
return (int32_t) n_out;
}
// encodes one audio entry, the rows are appended to embd
static bool mtmd_audio_stream_encode(mtmd_audio_stream & s, mtmd_image_f32 && img, mtmd_subctx_enc_state * enc_state) {
const int n_out = s.sctx->model.n_output_tokens(&img);
mtmd_image_f32_batch batch;
batch.is_audio = true;
batch.entries.push_back(std::move(img));
std::vector<float> res((size_t) s.n_embd * n_out);
mtmd_subctx_encode_params params;
params.n_threads = s.ctx->n_threads;
params.imgs = &batch;
params.out_embd = &res;
params.enc_state = enc_state;
if (!s.sctx->encode(&params)) {
LOG_ERR("%s: failed to encode audio\n", __func__);
return false;
}
s.embd.insert(s.embd.end(), res.begin(), res.end());
return true;
}
// rows carry no text: all of them go out at once between the audio markers, last_token is not used
static int32_t mtmd_audio_stream_emit_rows(mtmd_audio_stream & s, const std::vector<float> & beg_rows, const std::vector<float> & end_rows,
bool & beg_done, bool & end_done) {
const int64_t n_avail = (int64_t) (s.embd.size() / s.n_embd);
s.out.clear();
if (n_avail > 0 || s.ended) {
if (!beg_done) {
beg_done = true;
s.out = beg_rows;
}
s.out.insert(s.out.end(), s.embd.begin(), s.embd.end());
s.embd.clear();
if (s.ended && !end_done) {
end_done = true;
s.out.insert(s.out.end(), end_rows.begin(), end_rows.end());
}
}
const int64_t n_out = (int64_t) (s.out.size() / s.n_embd);
if (n_out == 0) {
return s.ended ? -1 : 0;
}
s.n_rows += n_out;
s.n_rows_done += n_avail; // the markers are not audio rows
return (int32_t) n_out;
}
//
// Higgs Audio
//
bool mtmd_audio_stream_higgs_audio::init(const mtmd_audio_stream_params & params) {
const mtmd_hparams * hparams = &sctx->model.hparams;
if (params.n_prompt > 0) {
LOG_ERR("%s: Higgs Audio takes no prompt, its audio is a chat turn\n", __func__);
return false;
}
if (!ctx->audio_preproc) {
LOG_ERR("%s: the audio preprocessor is not initialized\n", __func__);
return false;
}
chunk_len = (int64_t) hparams->audio_chunk_len * hparams->audio_sample_rate;
frames_per_tok = 8; // stride-2 conv, avgpool, stride-2 projector conv
samples_per_tok = frames_per_tok * hop;
return mtmd_audio_stream_marker_rows(ctx, ctx->aud_beg, beg_rows) && mtmd_audio_stream_marker_rows(ctx, ctx->aud_end, end_rows);
}
// rows of a chunk with n samples, see mtmd_audio_preprocessor_higgs_audio
int64_t mtmd_audio_stream_higgs_audio::n_chunk_rows(int64_t n) const {
const int64_t n_frames = n / hop;
if (n_frames < 3) {
return 0;
}
return ((n_frames + 1) / 2 / 2 + 1) / 2;
}
bool mtmd_audio_stream_higgs_audio::encode(int64_t n) {
mtmd_audio_preproc_out pre = ctx->audio_preproc->preprocess(buf.data(), (size_t) n);
if (pre.entries.empty() || !ctx->audio_preproc->compute(pre)) {
LOG_ERR("%s: unable to preprocess the audio\n", __func__);
return false;
}
for (auto & entry : pre.entries) {
if (!mtmd_audio_stream_encode(*this, std::move(entry), nullptr)) {
return false;
}
}
buf.erase(buf.begin(), buf.begin() + n);
buf_pos += n;
return true;
}
int32_t mtmd_audio_stream_higgs_audio::flush(bool end) {
const int64_t n_pending = n_samples - buf_pos;
n_rows_flushed = n_rows_done + (int64_t) (embd.size() / n_embd) + (n_pending / chunk_len) * n_chunk_rows(chunk_len) + n_chunk_rows(n_pending % chunk_len);
pending_flush = true;
ended = end;
return 0;
}
int32_t mtmd_audio_stream_higgs_audio::step(llama_token /* last_token */) {
const bool flush = pending_flush;
pending_flush = false;
while ((int64_t) buf.size() >= chunk_len) {
if (!encode(chunk_len)) {
return -2;
}
}
// a tail too short for a row is kept until the next flush, or dropped at the end
if ((flush || ended) && !buf.empty()) {
if (n_chunk_rows((int64_t) buf.size()) > 0) {
if (!encode((int64_t) buf.size())) {
return -2;
}
} else if (ended) {
buf_pos += (int64_t) buf.size();
buf.clear();
}
}
return mtmd_audio_stream_emit_rows(*this, beg_rows, end_rows, beg_done, end_done);
}
//
// VibeVoice-ASR
//
bool mtmd_audio_stream_vibevoice_asr::init(const mtmd_audio_stream_params & params) {
const mtmd_hparams * hparams = &sctx->model.hparams;
if (params.n_prompt > 0) {
LOG_ERR("%s: VibeVoice-ASR takes no prompt, its audio is a chat turn\n", __func__);
return false;
}
hop = hparams->audio_samples_per_token();
frames_per_tok = 1;
samples_per_tok = hop;
n_step = std::max<int64_t>(1, ((int64_t) params.step_ms * hparams->audio_sample_rate / 1000 + hop - 1) / hop) * hop;
n_step_max = std::max(n_step, (int64_t) hparams->audio_chunk_len * hparams->audio_sample_rate / hop * hop);
enc_state = sctx->enc_state_init(0);
if (!enc_state) {
return false;
}
return mtmd_audio_stream_marker_rows(ctx, ctx->aud_beg, beg_rows) && mtmd_audio_stream_marker_rows(ctx, ctx->aud_end, end_rows);
}
// n is a multiple of hop, see mtmd_audio_preprocessor_vibevoice_asr for the gain
bool mtmd_audio_stream_vibevoice_asr::encode(int64_t n) {
const double gain = mtmd_audio_vibevoice_gain(sum_sq, n_stat, peak);
std::vector<float> data((size_t) n);
for (int64_t i = 0; i < n; i++) {
data[i] = (float) (buf[i] * gain);
}
mtmd_image_f32 img;
img.set_size({(int) n, 1}, false, /* is_audio */ true);
img.cpy_buf(data);
if (!mtmd_audio_stream_encode(*this, std::move(img), enc_state)) {
return false;
}
buf.erase(buf.begin(), buf.begin() + n);
buf_pos += n;
return true;
}
int32_t mtmd_audio_stream_vibevoice_asr::flush(bool end) {
n_rows_flushed = end ? (n_samples + hop - 1) / hop : n_samples / hop;
pending_flush = true;
ended = end;
return 0;
}
int32_t mtmd_audio_stream_vibevoice_asr::step(llama_token /* last_token */) {
const bool flush = pending_flush;
pending_flush = false;
// the padding added at the end does not count in the gain
for (int64_t i = n_stat - buf_pos; i < (int64_t) buf.size(); i++) {
sum_sq += (double) buf[i] * buf[i];
peak = std::max(peak, (double) std::fabs(buf[i]));
}
n_stat = n_samples;
if (ended && n_samples % hop != 0) {
add_silence(hop - n_samples % hop);
n_stat = n_samples;
}
// whole rows only: a partial row would put its zero padding into the state
int64_t n = (int64_t) buf.size();
n -= n % hop;
if (!flush && !ended) {
n -= n % n_step;
}
while (n > 0) {
const int64_t m = std::min(n, n_step_max);
if (!encode(m)) {
return -2;
}
n -= m;
}
return mtmd_audio_stream_emit_rows(*this, beg_rows, end_rows, beg_done, end_done);
}
int64_t mtmd_audio_stream_get_n_rows(const mtmd_audio_stream * s) {
return s->n_rows_done;
}