Files
llama.cpp/tools/mtmd/mtmd-audio.h
T
Xuan-Son Nguyen e4b620af19 mtmd: support voxcpm2 (#32443)
* mtmd: support voxcpm

* speed up a bit

* various fixes
2026-11-19 02:39:05 +01:00

385 lines
16 KiB
C++
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
#pragma once
#include "ggml.h"
#include "ggml-backend.h"
#include "mtmd-model.h"
#include <cstdint>
#include <vector>
#include <string>
#define MTMD_INTERNAL_HEADER
struct mtmd_audio_mel {
int64_t n_len;
int64_t n_len_org;
int64_t n_mel;
int64_t n_ctx = 0; // see mtmd_image_f32::audio_n_ctx
std::vector<float> data;
};
struct mtmd_audio_mel_filters {
int64_t n_mel;
int64_t n_fft;
std::vector<float> data;
};
// cache for audio processing, each processor instance owns its own cache
struct mtmd_audio_cache {
std::vector<float> sin_vals;
std::vector<float> cos_vals;
std::vector<float> hann_window;
mtmd_audio_mel_filters filters;
void fill_sin_cos_table(uint32_t n);
void fill_hann_window(uint32_t length, bool periodic);
// Build mel filterbank matrix [n_mel × n_fft_bins] at runtime.
// n_fft_bins must be (N_fft / 2 + 1). Example: if N_fft=512 -> n_fft_bins=257.
void fill_mel_filterbank_matrix(int64_t n_mel,
int64_t n_fft,
int sample_rate, // e.g. 16000
float fmin = 0.0f, // e.g. 0.0
float fmax = -1.0f, // e.g. sr/2; pass -1 for auto
bool slaney_area_norm = true,
float scale = 1.0f,
bool use_htk = false
);
};
// FFT of N real samples -> N complex values (interleaved), in needs 2 * N floats and out 8 * N floats (scratch)
void mtmd_audio_fft(const mtmd_audio_cache & cache, float * in, int N, float * out);
// inverse FFT of N complex values (interleaved), scaled by 1 / N; in and out need 8 * N floats (scratch)
void mtmd_audio_ifft(const mtmd_audio_cache & cache, float * in, int N, float * out);
// compute one log10-mel frame from n_fft windowed samples, cache must be initialized for n_fft
void mtmd_audio_log_mel_frame(const mtmd_audio_cache & cache, const float * samples, int n_fft, int n_mel, float * out);
struct mtmd_audio_mel_config {
int64_t n_mel = 0;
int64_t n_fft_bins = 0;
int32_t hann_window_size = 0;
int32_t hop_length = 0;
int32_t sample_rate = 0;
bool no_padding = false;
bool center_padding = false;
float preemph = 0.f;
bool use_natural_log = false;
bool norm_per_feature = false;
bool use_magnitude = false; // |X| instead of |X|^2
float mel_floor = 5.960464477539063e-08f;
bool mel_floor_add = false; // log(x + floor) instead of log(max(x, floor))
bool std_eps_after_sqrt = false; // std + eps instead of sqrt(var + eps)
int n_threads = 1;
const mtmd_audio_cache * cache = nullptr;
ggml_backend_sched_t sched = nullptr;
};
struct mtmd_audio_preproc_out {
std::vector<mtmd_image_f32> entries; // one per chunk, empty on failure
std::vector<float> samples; // input of compute(), empty if entries already hold the data
bool needs_compute() const {
return !samples.empty() && (entries.empty() || entries[0].is_placeholder());
}
};
struct mtmd_audio_preprocessor {
const mtmd_hparams & hparams;
mtmd_audio_mel_config mel_config; // filled by initialize() of the preprocessors that use the log-mel graph
// mel_config.sched must be set before the first compute(), see mtmd_context::init_audio()
mtmd_audio_preprocessor(const mtmd_model * model): hparams(model->hparams) {}
mtmd_audio_preprocessor(const mtmd_hparams & hparams, ggml_backend_sched_t sched): hparams(hparams) {
mel_config.sched = sched;
}
// mel_config points to the cache of this instance
mtmd_audio_preprocessor(const mtmd_audio_preprocessor &) = delete;
mtmd_audio_preprocessor & operator=(const mtmd_audio_preprocessor &) = delete;
virtual ~mtmd_audio_preprocessor() = default;
virtual void initialize() = 0; // NOT thread-safe
// entries have the final shapes, but no data if needs_compute()
mtmd_audio_preproc_out preprocess(const float * samples, size_t n_samples) const;
// run it on the thread that encodes, it uses mel_config.sched
bool compute(mtmd_audio_preproc_out & out) const;
protected:
// shape_only skips the log-mel graph: the shapes are right, the data is zero-filled
virtual bool process(const float * samples, size_t n_samples, std::vector<mtmd_audio_mel> & output, bool shape_only) const = 0;
};
struct mtmd_audio_preprocessor_whisper : mtmd_audio_preprocessor {
mtmd_audio_preprocessor_whisper(const mtmd_model * model) : mtmd_audio_preprocessor(model) {}
mtmd_audio_preprocessor_whisper(const mtmd_hparams & hparams, ggml_backend_sched_t sched) : mtmd_audio_preprocessor(hparams, sched) {}
void initialize() override;
bool process(const float * samples, size_t n_samples, std::vector<mtmd_audio_mel> & output, bool shape_only) const override;
private:
mtmd_audio_cache cache;
};
struct mtmd_audio_preprocessor_muse_spark_audio : mtmd_audio_preprocessor {
mtmd_audio_preprocessor_muse_spark_audio(const mtmd_model * model) : mtmd_audio_preprocessor(model) {}
void initialize() override {}
bool process(const float * samples, size_t n_samples, std::vector<mtmd_audio_mel> & output, bool shape_only) const override;
};
// speaker reference (fish_s2, voxcpm2): same as muse spark (raw waveform, whole frames), truncated to max_seconds
struct mtmd_audio_preprocessor_spk_ref : mtmd_audio_preprocessor_muse_spark_audio {
mtmd_audio_preprocessor_spk_ref(const mtmd_model * model, int32_t max_seconds)
: mtmd_audio_preprocessor_muse_spark_audio(model), max_seconds(max_seconds) {}
bool process(const float * samples, size_t n_samples, std::vector<mtmd_audio_mel> & output, bool shape_only) const override;
const int32_t max_seconds;
};
// vibevoice: gain to -25 dBFS RMS, without clipping the peak
// the RMS is floored at -50 dBFS, so that silence or noise alone is not boosted to full scale
double mtmd_audio_vibevoice_gain(double sum_sq, int64_t n_samples, double peak);
// raw 24 kHz waveform, cut in chunks that carry their causal context, see mtmd_hparams::audio_n_ctx
struct mtmd_audio_preprocessor_vibevoice_asr : mtmd_audio_preprocessor {
mtmd_audio_preprocessor_vibevoice_asr(const mtmd_model * model) : mtmd_audio_preprocessor(model) {}
void initialize() override {}
bool process(const float * samples, size_t n_samples, std::vector<mtmd_audio_mel> & output, bool shape_only) const override;
};
struct mtmd_audio_preprocessor_conformer : mtmd_audio_preprocessor {
mtmd_audio_preprocessor_conformer(const mtmd_model * model) : mtmd_audio_preprocessor(model) {}
mtmd_audio_preprocessor_conformer(const mtmd_hparams & hparams, ggml_backend_sched_t sched) : mtmd_audio_preprocessor(hparams, sched) {}
void initialize() override;
bool process(const float * samples, size_t n_samples, std::vector<mtmd_audio_mel> & output, bool shape_only) const override;
private:
mtmd_audio_cache cache;
};
// same as conformer, the audio is cut to 30 s and padded to 0.5 s (d1-omni audio.py)
struct mtmd_audio_preprocessor_d1omni : mtmd_audio_preprocessor_conformer {
using mtmd_audio_preprocessor_conformer::mtmd_audio_preprocessor_conformer;
bool process(const float * samples, size_t n_samples, std::vector<mtmd_audio_mel> & output, bool shape_only) const override;
};
struct mtmd_audio_preprocessor_lfm3a : mtmd_audio_preprocessor {
mtmd_audio_preprocessor_lfm3a(const mtmd_model * model) : mtmd_audio_preprocessor(model) {}
void initialize() override;
bool process(const float * samples, size_t n_samples, std::vector<mtmd_audio_mel> & output, bool shape_only) const override;
private:
mtmd_audio_cache cache;
};
struct mtmd_audio_preprocessor_granite_speech : mtmd_audio_preprocessor {
mtmd_audio_preprocessor_granite_speech(const mtmd_model * model) : mtmd_audio_preprocessor(model) {}
void initialize() override;
bool process(const float * samples, size_t n_samples, std::vector<mtmd_audio_mel> & output, bool shape_only) const override;
private:
mtmd_audio_cache cache;
};
struct mtmd_audio_preprocessor_gemma4a : mtmd_audio_preprocessor {
// each segment is padded and encoded on its own
static constexpr int segment_seconds = 30;
mtmd_audio_preprocessor_gemma4a(const mtmd_model * model) : mtmd_audio_preprocessor(model) {}
void initialize() override;
bool process(const float * samples, size_t n_samples, std::vector<mtmd_audio_mel> & output, bool shape_only) const override;
private:
mtmd_audio_cache cache;
};
struct mtmd_audio_preprocessor_gemma4ua : mtmd_audio_preprocessor {
mtmd_audio_preprocessor_gemma4ua(const mtmd_model * model) : mtmd_audio_preprocessor(model) {}
void initialize() override;
bool process(const float * samples, size_t n_samples, std::vector<mtmd_audio_mel> & output, bool shape_only) const override;
};
struct mtmd_audio_preprocessor_qwen3a : mtmd_audio_preprocessor {
mtmd_audio_preprocessor_qwen3a(const mtmd_model * model) : mtmd_audio_preprocessor(model) {}
void initialize() override;
bool process(const float * samples, size_t n_samples, std::vector<mtmd_audio_mel> & output, bool shape_only) const override;
private:
mtmd_audio_cache cache;
};
struct mtmd_audio_preprocessor_dots3note : mtmd_audio_preprocessor {
mtmd_audio_preprocessor_dots3note(const mtmd_model * model) : mtmd_audio_preprocessor(model) {}
void initialize() override;
bool process(const float * samples, size_t n_samples, std::vector<mtmd_audio_mel> & output, bool shape_only) const override;
private:
mtmd_audio_cache cache;
};
// same per-chunk log-mel as dots3note, with 4s chunks
struct mtmd_audio_preprocessor_higgs_audio : mtmd_audio_preprocessor_dots3note {
mtmd_audio_preprocessor_higgs_audio(const mtmd_model * model) : mtmd_audio_preprocessor_dots3note(model) {}
bool process(const float * samples, size_t n_samples, std::vector<mtmd_audio_mel> & output, bool shape_only) const override;
};
struct mtmd_audio_preprocessor_mimo_audio : mtmd_audio_preprocessor {
mtmd_audio_preprocessor_mimo_audio(const mtmd_model * model) : mtmd_audio_preprocessor(model) {}
void initialize() override;
bool process(const float * samples, size_t n_samples, std::vector<mtmd_audio_mel> & output, bool shape_only) const override;
private:
mtmd_audio_cache cache;
};
struct mtmd_audio_preprocessor_qwen3tts_spk : mtmd_audio_preprocessor {
mtmd_audio_preprocessor_qwen3tts_spk(const mtmd_model * model) : mtmd_audio_preprocessor(model) {}
void initialize() override;
bool process(const float * samples, size_t n_samples, std::vector<mtmd_audio_mel> & output, bool shape_only) const override;
private:
mtmd_audio_cache cache;
};
// mimi convolves the waveform directly, so this only pads it to a whole number of frames
struct mtmd_audio_preprocessor_pockettts : mtmd_audio_preprocessor {
mtmd_audio_preprocessor_pockettts(const mtmd_model * model) : mtmd_audio_preprocessor(model) {}
void initialize() override {}
bool process(const float * samples, size_t n_samples, std::vector<mtmd_audio_mel> & output, bool shape_only) const override;
};
// chatterbox reference front-end: one clip gives five entries, used by position in mtmd.cpp
// 0: CAMPPlus kaldi fbank [80 x n_frames] (mel major)
// 1: s3 tokenizer log-mel, flow cap [n_mel x n_frames] (mel major)
// 2: s3 tokenizer log-mel, t3 cap [n_mel x n_frames] (mel major)
// 3: s3gen prompt features [n_frames x 80] (frame major, 24 kHz mel rate)
// 4: voice encoder power mel [n_frames x 40] (frame major, trimmed, partial grid)
// turbo loudness-normalizes the clip to -27 LUFS and caps the t3 clip at 15 s instead of 6 s
struct mtmd_audio_preprocessor_chatterbox_ref : mtmd_audio_preprocessor {
mtmd_audio_preprocessor_chatterbox_ref(const mtmd_model * model, bool is_mtl) : mtmd_audio_preprocessor(model), is_mtl(is_mtl) {}
void initialize() override;
bool process(const float * samples, size_t n_samples, std::vector<mtmd_audio_mel> & output, bool shape_only) const override;
private:
bool is_mtl;
mtmd_audio_cache fbank_cache; // entry 0, the window slot holds the povey window
mtmd_audio_cache s3tok_cache; // entries 1 and 2
mtmd_audio_cache matcha_cache; // entry 3
mtmd_audio_cache ve_cache; // entry 4
};
// zonos3 speaker encoder front-end: kaldi fbank with a hamming window [80 x n_frames] (mel major)
struct mtmd_audio_preprocessor_zonos3_spk : mtmd_audio_preprocessor {
mtmd_audio_preprocessor_zonos3_spk(const mtmd_model * model) : mtmd_audio_preprocessor(model) {}
void initialize() override;
bool process(const float * samples, size_t n_samples, std::vector<mtmd_audio_mel> & output, bool shape_only) const override;
private:
mtmd_audio_cache cache;
};
struct mtmd_audio_preprocessor_parakeet : mtmd_audio_preprocessor {
mtmd_audio_preprocessor_parakeet(const mtmd_model * model) : mtmd_audio_preprocessor(model) { }
void initialize() override;
bool process(const float * samples, size_t n_samples, std::vector<mtmd_audio_mel> & output, bool shape_only) const override;
protected:
bool log_mel(const float * samples, size_t n_samples, mtmd_audio_mel & out, bool shape_only) const;
private:
mtmd_audio_cache cache;
};
// one chunk per recording: the raw log-mel of its valid frames
struct mtmd_audio_preprocessor_nemotron_transcribe : mtmd_audio_preprocessor_parakeet {
mtmd_audio_preprocessor_nemotron_transcribe(const mtmd_model * model) : mtmd_audio_preprocessor_parakeet(model) { }
bool process(const float * samples, size_t n_samples, std::vector<mtmd_audio_mel> & output, bool shape_only) const override;
};
//
// zonos3 iSTFT head, streaming: 2 iSTFT branches, FIR mixer, 44160 -> 44100 Hz resampler
// ported from the reference ggml runtime (zac.cpp)
//
struct mtmd_audio_zonos3_synth {
// mixer [2, fir_kernel], resample [rs_new, rs_orig + 2 * width]
mtmd_audio_zonos3_synth(const std::vector<float> & mixer, const std::vector<float> & resample,
int n_fft0, int n_fft1, int hop, int rs_orig, int rs_new);
// spec: n_cols frames of [log magnitude, phase] of each branch
void push(const float * spec, int64_t n_cols, std::vector<float> & out);
void flush(std::vector<float> & out);
private:
struct branch {
int n_fft;
mtmd_audio_cache cache;
std::vector<float> ola, env; // from ola_base on
int64_t ola_base = 0;
std::vector<float> mix_in; // from mix_base on
int64_t mix_avail = 0;
std::vector<float> ifft_in, ifft_out;
};
void ola_finalize(branch & b, int64_t pf);
void mix_emit(std::vector<float> & out);
void resample(bool flush, std::vector<float> & out);
branch br[2];
std::vector<float> mixer;
int fir_kernel;
int hop;
int64_t n_frames = 0;
int64_t mix_base = 0;
int64_t mix_next = 0;
bool done = false;
std::vector<float> rs_kernel;
int rs_orig, rs_new, rs_width;
std::vector<float> rs_buf; // from rs_base on
int64_t rs_base = 0;
int64_t rs_in = 0;
int64_t rs_block = 0;
};
//
// streaming ISTFT - converts spectrogram frames back to audio one frame at a time
//
struct mtmd_audio_streaming_istft {
mtmd_audio_streaming_istft(int n_fft, int hop_length);
// reset streaming state
void reset();
// process a single STFT frame (streaming)
// frame_spectrum: [n_fft_bins x 2] interleaved real/imag
// returns: up to hop_length samples
std::vector<float> process_frame(const float * frame_spectrum);
// flush remaining samples at end of stream
std::vector<float> flush();
private:
int n_fft;
int hop_length;
int n_fft_bins;
// Own cache for output processing
mtmd_audio_cache cache;
// Streaming state
std::vector<float> overlap_buffer;
std::vector<float> window_sum_buffer;
int padding_to_remove;
// Working buffers for IFFT
std::vector<float> ifft_in;
std::vector<float> ifft_out;
};