From 0cce7580b8169c044def5a01d6cbe61fff7c2b45 Mon Sep 17 00:00:00 2001 From: Hera <58063257+hera2019@users.noreply.github.com> Date: Sat, 14 Nov 2026 21:33:59 +0900 Subject: [PATCH] mtmd : avoid padding-only Whisper audio chunks (#30403) Assisted-by: Claude Opus 5 --- tests/test-mtmd-impl.cpp | 52 +++++++++++++++++++++++++++++++++++++++ tools/mtmd/mtmd-audio.cpp | 4 ++- tools/mtmd/mtmd-audio.h | 1 + 3 files changed, 56 insertions(+), 1 deletion(-) diff --git a/tests/test-mtmd-impl.cpp b/tests/test-mtmd-impl.cpp index 3fd720088..45c15f8c3 100644 --- a/tests/test-mtmd-impl.cpp +++ b/tests/test-mtmd-impl.cpp @@ -462,6 +462,58 @@ MAKE_TEST(test_video_sequence_lfm3_omni) { // mtmd_audio // +MAKE_TEST(test_audio_preprocessor_whisper_chunks) { + mtmd_hparams hparams; + hparams.n_mel_bins = 128; + hparams.audio_sample_rate = 16000; + hparams.audio_n_fft = 400; + hparams.audio_window_len = 400; + hparams.audio_hop_len = 160; + + ggml_backend_ptr backend(ggml_backend_init_by_type(GGML_BACKEND_DEVICE_TYPE_CPU, nullptr)); + ggml_backend_t backends[] = { backend.get() }; + ggml_backend_sched_ptr sched(ggml_backend_sched_new(backends, nullptr, 1, 256, false, true)); + + mtmd_audio_preprocessor_whisper preproc(hparams, sched.get()); + preproc.initialize(); + + // { sample count, expected chunks }, one chunk of 3000 frames covers 480000 samples + const std::vector> cases = { + { 0, 0 }, + { 1, 1 }, + { 200, 1 }, + { 201, 1 }, + { 32000, 1 }, + { 479834, 1 }, + { 479839, 1 }, + { 479840, 1 }, + { 479999, 1 }, + { 480000, 1 }, + { 480001, 2 }, + { 480159, 2 }, + { 480160, 2 }, + { 959840, 2 }, + { 959999, 2 }, + { 960000, 2 }, + { 960001, 3 }, + { 1440000, 3 }, + }; + + for (const auto & [n_samples, expected] : cases) { + const std::string name = std::to_string(n_samples) + " samples"; + std::vector samples(n_samples, 0.0f); + if (n_samples > 0) { + samples.back() = 0.5f; + } + mtmd_audio_preproc_out out = preproc.preprocess(samples.data(), samples.size()); + t.assert_equal("chunks for " + name, expected, out.entries.size()); + for (const auto & mel : out.entries) { + t.assert_equal("frames for " + name, 3000, mel.nx()); + t.assert_equal("mel bins for " + name, 128, mel.ny()); + } + } +} + MAKE_TEST(test_audio_preprocessor_conformer) { mtmd_hparams hparams; hparams.n_mel_bins = 128; diff --git a/tools/mtmd/mtmd-audio.cpp b/tools/mtmd/mtmd-audio.cpp index aceaf5391..33c26c31f 100644 --- a/tools/mtmd/mtmd-audio.cpp +++ b/tools/mtmd/mtmd-audio.cpp @@ -555,7 +555,9 @@ bool mtmd_audio_preprocessor_whisper::process(const float * samp } const size_t frames_per_chunk = 3000; GGML_ASSERT((size_t) out_full.n_len > frames_per_chunk); - for (size_t off = 0; off < (size_t) out_full.n_len; off += frames_per_chunk) { + // round up, so a partial audio frame is not dropped with the padding + const size_t n_audio_frames = (n_samples - 1) / (size_t) mel_config.hop_length + 1; + for (size_t off = 0; off < n_audio_frames; off += frames_per_chunk) { int64_t n_len = std::min((int64_t)frames_per_chunk, out_full.n_len - (int64_t)off); if (n_len < (int64_t)frames_per_chunk) { break; // last incomplete chunk will always be a padded chunk, safe to ignore diff --git a/tools/mtmd/mtmd-audio.h b/tools/mtmd/mtmd-audio.h index 1175ad624..7251f82db 100644 --- a/tools/mtmd/mtmd-audio.h +++ b/tools/mtmd/mtmd-audio.h @@ -114,6 +114,7 @@ struct mtmd_audio_preprocessor { struct mtmd_audio_preprocessor_whisper : mtmd_audio_preprocessor { mtmd_audio_preprocessor_whisper(const mtmd_model * model) : mtmd_audio_preprocessor(model) {} + mtmd_audio_preprocessor_whisper(const mtmd_hparams & hparams, ggml_backend_sched_t sched) : mtmd_audio_preprocessor(hparams, sched) {} void initialize() override; bool process(const float * samples, size_t n_samples, std::vector & output, bool shape_only) const override;