mtmd : avoid padding-only Whisper audio chunks (#30403)

Assisted-by: Claude Opus 5
This commit is contained in:
Hera authored and GitHub committed 2026-11-14 13:33:59 +01:00
1 parent 32bdc3a89e
commit 0cce7580b8
3 files changed
+56 -1

No files matched your search

+52
View File
@@ -462,6 +462,58 @@ MAKE_TEST(test_video_sequence_lfm3_omni) {
// mtmd_audio
//
MAKE_TEST(test_audio_preprocessor_whisper_chunks) {
mtmd_hparams hparams;
hparams.n_mel_bins = 128;
hparams.audio_sample_rate = 16000;
hparams.audio_n_fft = 400;
hparams.audio_window_len = 400;
hparams.audio_hop_len = 160;
ggml_backend_ptr backend(ggml_backend_init_by_type(GGML_BACKEND_DEVICE_TYPE_CPU, nullptr));
ggml_backend_t backends[] = { backend.get() };
ggml_backend_sched_ptr sched(ggml_backend_sched_new(backends, nullptr, 1, 256, false, true));
mtmd_audio_preprocessor_whisper preproc(hparams, sched.get());
preproc.initialize();
// { sample count, expected chunks }, one chunk of 3000 frames covers 480000 samples
const std::vector<std::pair<size_t, size_t>> cases = {
{ 0, 0 },
{ 1, 1 },
{ 200, 1 },
{ 201, 1 },
{ 32000, 1 },
{ 479834, 1 },
{ 479839, 1 },
{ 479840, 1 },
{ 479999, 1 },
{ 480000, 1 },
{ 480001, 2 },
{ 480159, 2 },
{ 480160, 2 },
{ 959840, 2 },
{ 959999, 2 },
{ 960000, 2 },
{ 960001, 3 },
{ 1440000, 3 },
};
for (const auto & [n_samples, expected] : cases) {
const std::string name = std::to_string(n_samples) + " samples";
std::vector<float> samples(n_samples, 0.0f);
if (n_samples > 0) {
samples.back() = 0.5f;
}
mtmd_audio_preproc_out out = preproc.preprocess(samples.data(), samples.size());
t.assert_equal("chunks for " + name, expected, out.entries.size());
for (const auto & mel : out.entries) {
t.assert_equal("frames for " + name, 3000, mel.nx());
t.assert_equal("mel bins for " + name, 128, mel.ny());
}
}
}
MAKE_TEST(test_audio_preprocessor_conformer) {
mtmd_hparams hparams;
hparams.n_mel_bins = 128;