* add chatterbox TTS support Co-authored-by: Pascal <admin@serveurperso.com> * add built-in voice support * rm redundant code * clean up * clean up 2 * Apply batched suggestions from code review Co-authored-by: Sigbjørn Skjæret <sigbjorn.skjaeret@huggingface.co> * improve --------- Co-authored-by: Pascal <admin@serveurperso.com> Co-authored-by: Sigbjørn Skjæret <sigbjorn.skjaeret@huggingface.co>
252 lines
9.9 KiB
C++
252 lines
9.9 KiB
C++
#pragma once
|
||
|
||
#include "ggml.h"
|
||
#include "clip-model.h"
|
||
|
||
#include <cstdint>
|
||
#include <vector>
|
||
#include <string>
|
||
|
||
#define MTMD_INTERNAL_HEADER
|
||
|
||
struct mtmd_audio_mel {
|
||
int64_t n_len;
|
||
int64_t n_len_org;
|
||
int64_t n_mel;
|
||
|
||
std::vector<float> data;
|
||
};
|
||
|
||
struct mtmd_audio_mel_filters {
|
||
int64_t n_mel;
|
||
int64_t n_fft;
|
||
|
||
std::vector<float> data;
|
||
};
|
||
|
||
// cache for audio processing, each processor instance owns its own cache
|
||
struct mtmd_audio_cache {
|
||
std::vector<float> sin_vals;
|
||
std::vector<float> cos_vals;
|
||
|
||
std::vector<float> hann_window;
|
||
|
||
mtmd_audio_mel_filters filters;
|
||
|
||
void fill_sin_cos_table(uint32_t n);
|
||
|
||
void fill_hann_window(uint32_t length, bool periodic);
|
||
|
||
// Build mel filterbank matrix [n_mel × n_fft_bins] at runtime.
|
||
// n_fft_bins must be (N_fft / 2 + 1). Example: if N_fft=512 -> n_fft_bins=257.
|
||
void fill_mel_filterbank_matrix(int64_t n_mel,
|
||
int64_t n_fft,
|
||
int sample_rate, // e.g. 16000
|
||
float fmin = 0.0f, // e.g. 0.0
|
||
float fmax = -1.0f, // e.g. sr/2; pass -1 for auto
|
||
bool slaney_area_norm = true,
|
||
float scale = 1.0f,
|
||
bool use_htk = false
|
||
);
|
||
};
|
||
|
||
// compute one log10-mel frame from n_fft windowed samples, cache must be initialized for n_fft
|
||
void mtmd_audio_log_mel_frame(const mtmd_audio_cache & cache, const float * samples, int n_fft, int n_mel, float * out);
|
||
|
||
struct mtmd_audio_preprocessor {
|
||
const clip_hparams & hparams;
|
||
|
||
mtmd_audio_preprocessor(const clip_ctx * ctx): hparams(*clip_get_hparams(ctx)) {}
|
||
mtmd_audio_preprocessor(const clip_hparams & hparams): hparams(hparams) {}
|
||
|
||
virtual ~mtmd_audio_preprocessor() = default;
|
||
virtual void initialize() = 0; // NOT thread-safe
|
||
virtual bool preprocess(const float * samples, size_t n_samples, std::vector<mtmd_audio_mel> & output) const = 0;
|
||
};
|
||
|
||
struct mtmd_audio_preprocessor_whisper : mtmd_audio_preprocessor {
|
||
mtmd_audio_preprocessor_whisper(const clip_ctx * ctx) : mtmd_audio_preprocessor(ctx) {}
|
||
void initialize() override;
|
||
bool preprocess(const float * samples, size_t n_samples, std::vector<mtmd_audio_mel> & output) const override;
|
||
|
||
private:
|
||
mtmd_audio_cache cache;
|
||
};
|
||
|
||
struct mtmd_audio_preprocessor_muse_spark_audio : mtmd_audio_preprocessor {
|
||
mtmd_audio_preprocessor_muse_spark_audio(const clip_ctx * ctx) : mtmd_audio_preprocessor(ctx) {}
|
||
void initialize() override {}
|
||
bool preprocess(const float * samples, size_t n_samples, std::vector<mtmd_audio_mel> & output) const override;
|
||
};
|
||
|
||
struct mtmd_audio_preprocessor_conformer : mtmd_audio_preprocessor {
|
||
mtmd_audio_preprocessor_conformer(const clip_ctx * ctx) : mtmd_audio_preprocessor(ctx) {}
|
||
mtmd_audio_preprocessor_conformer(const clip_hparams & hparams) : mtmd_audio_preprocessor(hparams) {}
|
||
void initialize() override;
|
||
bool preprocess(const float * samples, size_t n_samples, std::vector<mtmd_audio_mel> & output) const override;
|
||
|
||
private:
|
||
mtmd_audio_cache cache;
|
||
};
|
||
|
||
// same as conformer, the audio is cut to 30 s and padded to 0.5 s (d1-omni audio.py)
|
||
struct mtmd_audio_preprocessor_d1omni : mtmd_audio_preprocessor_conformer {
|
||
using mtmd_audio_preprocessor_conformer::mtmd_audio_preprocessor_conformer;
|
||
bool preprocess(const float * samples, size_t n_samples, std::vector<mtmd_audio_mel> & output) const override;
|
||
};
|
||
|
||
struct mtmd_audio_preprocessor_lfm3a : mtmd_audio_preprocessor {
|
||
mtmd_audio_preprocessor_lfm3a(const clip_ctx * ctx) : mtmd_audio_preprocessor(ctx) {}
|
||
void initialize() override;
|
||
bool preprocess(const float * samples, size_t n_samples, std::vector<mtmd_audio_mel> & output) const override;
|
||
|
||
private:
|
||
mtmd_audio_cache cache;
|
||
};
|
||
|
||
struct mtmd_audio_preprocessor_granite_speech : mtmd_audio_preprocessor {
|
||
mtmd_audio_preprocessor_granite_speech(const clip_ctx * ctx) : mtmd_audio_preprocessor(ctx) {}
|
||
void initialize() override;
|
||
bool preprocess(const float * samples, size_t n_samples, std::vector<mtmd_audio_mel> & output) const override;
|
||
|
||
private:
|
||
mtmd_audio_cache cache;
|
||
};
|
||
|
||
struct mtmd_audio_preprocessor_gemma4a : mtmd_audio_preprocessor {
|
||
mtmd_audio_preprocessor_gemma4a(const clip_ctx * ctx) : mtmd_audio_preprocessor(ctx) {}
|
||
void initialize() override;
|
||
bool preprocess(const float * samples, size_t n_samples, std::vector<mtmd_audio_mel> & output) const override;
|
||
|
||
private:
|
||
mtmd_audio_cache cache;
|
||
};
|
||
|
||
struct mtmd_audio_preprocessor_gemma4ua : mtmd_audio_preprocessor {
|
||
mtmd_audio_preprocessor_gemma4ua(const clip_ctx * ctx) : mtmd_audio_preprocessor(ctx) {}
|
||
void initialize() override;
|
||
bool preprocess(const float * samples, size_t n_samples, std::vector<mtmd_audio_mel> & output) const override;
|
||
};
|
||
|
||
struct mtmd_audio_preprocessor_qwen3a : mtmd_audio_preprocessor {
|
||
mtmd_audio_preprocessor_qwen3a(const clip_ctx * ctx) : mtmd_audio_preprocessor(ctx) {}
|
||
void initialize() override;
|
||
bool preprocess(const float * samples, size_t n_samples, std::vector<mtmd_audio_mel> & output) const override;
|
||
|
||
private:
|
||
mtmd_audio_cache cache;
|
||
};
|
||
|
||
struct mtmd_audio_preprocessor_dots3note : mtmd_audio_preprocessor {
|
||
mtmd_audio_preprocessor_dots3note(const clip_ctx * ctx) : mtmd_audio_preprocessor(ctx) {}
|
||
void initialize() override;
|
||
bool preprocess(const float * samples, size_t n_samples, std::vector<mtmd_audio_mel> & output) const override;
|
||
|
||
private:
|
||
mtmd_audio_cache cache;
|
||
};
|
||
|
||
struct mtmd_audio_preprocessor_mimo_audio : mtmd_audio_preprocessor {
|
||
mtmd_audio_preprocessor_mimo_audio(const clip_ctx * ctx) : mtmd_audio_preprocessor(ctx) {}
|
||
void initialize() override;
|
||
bool preprocess(const float * samples, size_t n_samples, std::vector<mtmd_audio_mel> & output) const override;
|
||
|
||
private:
|
||
mtmd_audio_cache cache;
|
||
};
|
||
|
||
struct mtmd_audio_preprocessor_qwen3tts_spk : mtmd_audio_preprocessor {
|
||
mtmd_audio_preprocessor_qwen3tts_spk(const clip_ctx * ctx) : mtmd_audio_preprocessor(ctx) {}
|
||
void initialize() override;
|
||
bool preprocess(const float * samples, size_t n_samples, std::vector<mtmd_audio_mel> & output) const override;
|
||
|
||
private:
|
||
mtmd_audio_cache cache;
|
||
};
|
||
|
||
// mimi convolves the waveform directly, so this only pads it to a whole number of frames
|
||
struct mtmd_audio_preprocessor_pockettts : mtmd_audio_preprocessor {
|
||
mtmd_audio_preprocessor_pockettts(const clip_ctx * ctx) : mtmd_audio_preprocessor(ctx) {}
|
||
void initialize() override {}
|
||
bool preprocess(const float * samples, size_t n_samples, std::vector<mtmd_audio_mel> & output) const override;
|
||
};
|
||
|
||
// chatterbox reference front-end: one clip gives five entries, used by position in mtmd.cpp
|
||
// 0: CAMPPlus kaldi fbank [80 x n_frames] (mel major)
|
||
// 1: s3 tokenizer log-mel, flow cap [n_mel x n_frames] (mel major)
|
||
// 2: s3 tokenizer log-mel, t3 cap [n_mel x n_frames] (mel major)
|
||
// 3: s3gen prompt features [n_frames x 80] (frame major, 24 kHz mel rate)
|
||
// 4: voice encoder power mel [n_frames x 40] (frame major, trimmed, partial grid)
|
||
// turbo loudness-normalizes the clip to -27 LUFS and caps the t3 clip at 15 s instead of 6 s
|
||
struct mtmd_audio_preprocessor_chatterbox_ref : mtmd_audio_preprocessor {
|
||
mtmd_audio_preprocessor_chatterbox_ref(const clip_ctx * ctx, bool is_mtl) : mtmd_audio_preprocessor(ctx), is_mtl(is_mtl) {}
|
||
void initialize() override;
|
||
bool preprocess(const float * samples, size_t n_samples, std::vector<mtmd_audio_mel> & output) const override;
|
||
|
||
private:
|
||
bool is_mtl;
|
||
mtmd_audio_cache fbank_cache; // entry 0, the window slot holds the povey window
|
||
mtmd_audio_cache s3tok_cache; // entries 1 and 2
|
||
mtmd_audio_cache matcha_cache; // entry 3
|
||
mtmd_audio_cache ve_cache; // entry 4
|
||
};
|
||
|
||
struct mtmd_audio_preprocessor_parakeet : mtmd_audio_preprocessor {
|
||
mtmd_audio_preprocessor_parakeet(clip_ctx * ctx) : mtmd_audio_preprocessor(ctx) { }
|
||
void initialize() override;
|
||
bool preprocess(const float * samples, size_t n_samples, std::vector<mtmd_audio_mel> & output) const override;
|
||
|
||
protected:
|
||
void log_mel(const float * samples, size_t n_samples, mtmd_audio_mel & out) const;
|
||
|
||
private:
|
||
mtmd_audio_cache cache;
|
||
|
||
static void worker_thread(int ith, const float * window_func, int window_size,
|
||
const std::vector<float> & samples, int n_samples,
|
||
int frame_size, int frame_step, int n_threads,
|
||
int n_fft_bins,
|
||
const mtmd_audio_cache & cache, mtmd_audio_mel & mel);
|
||
};
|
||
|
||
// one chunk per recording: the raw log-mel of its valid frames
|
||
struct mtmd_audio_preprocessor_nemotron_transcribe : mtmd_audio_preprocessor_parakeet {
|
||
mtmd_audio_preprocessor_nemotron_transcribe(clip_ctx * ctx) : mtmd_audio_preprocessor_parakeet(ctx) { }
|
||
bool preprocess(const float * samples, size_t n_samples, std::vector<mtmd_audio_mel> & output) const override;
|
||
};
|
||
|
||
//
|
||
// streaming ISTFT - converts spectrogram frames back to audio one frame at a time
|
||
//
|
||
struct mtmd_audio_streaming_istft {
|
||
mtmd_audio_streaming_istft(int n_fft, int hop_length);
|
||
|
||
// reset streaming state
|
||
void reset();
|
||
|
||
// process a single STFT frame (streaming)
|
||
// frame_spectrum: [n_fft_bins x 2] interleaved real/imag
|
||
// returns: up to hop_length samples
|
||
std::vector<float> process_frame(const float * frame_spectrum);
|
||
|
||
// flush remaining samples at end of stream
|
||
std::vector<float> flush();
|
||
|
||
private:
|
||
int n_fft;
|
||
int hop_length;
|
||
int n_fft_bins;
|
||
|
||
// Own cache for output processing
|
||
mtmd_audio_cache cache;
|
||
|
||
// Streaming state
|
||
std::vector<float> overlap_buffer;
|
||
std::vector<float> window_sum_buffer;
|
||
int padding_to_remove;
|
||
|
||
// Working buffers for IFFT
|
||
std::vector<float> ifft_in;
|
||
std::vector<float> ifft_out;
|
||
};
|