mtmd : avoid padding-only Whisper audio chunks (#30403)
Assisted-by: Claude Opus 5
This commit is contained in:
3 files changed
+56
-1
No files matched your search
@@ -462,6 +462,58 @@ MAKE_TEST(test_video_sequence_lfm3_omni) {
|
||||
// mtmd_audio
|
||||
//
|
||||
|
||||
MAKE_TEST(test_audio_preprocessor_whisper_chunks) {
|
||||
mtmd_hparams hparams;
|
||||
hparams.n_mel_bins = 128;
|
||||
hparams.audio_sample_rate = 16000;
|
||||
hparams.audio_n_fft = 400;
|
||||
hparams.audio_window_len = 400;
|
||||
hparams.audio_hop_len = 160;
|
||||
|
||||
ggml_backend_ptr backend(ggml_backend_init_by_type(GGML_BACKEND_DEVICE_TYPE_CPU, nullptr));
|
||||
ggml_backend_t backends[] = { backend.get() };
|
||||
ggml_backend_sched_ptr sched(ggml_backend_sched_new(backends, nullptr, 1, 256, false, true));
|
||||
|
||||
mtmd_audio_preprocessor_whisper preproc(hparams, sched.get());
|
||||
preproc.initialize();
|
||||
|
||||
// { sample count, expected chunks }, one chunk of 3000 frames covers 480000 samples
|
||||
const std::vector<std::pair<size_t, size_t>> cases = {
|
||||
{ 0, 0 },
|
||||
{ 1, 1 },
|
||||
{ 200, 1 },
|
||||
{ 201, 1 },
|
||||
{ 32000, 1 },
|
||||
{ 479834, 1 },
|
||||
{ 479839, 1 },
|
||||
{ 479840, 1 },
|
||||
{ 479999, 1 },
|
||||
{ 480000, 1 },
|
||||
{ 480001, 2 },
|
||||
{ 480159, 2 },
|
||||
{ 480160, 2 },
|
||||
{ 959840, 2 },
|
||||
{ 959999, 2 },
|
||||
{ 960000, 2 },
|
||||
{ 960001, 3 },
|
||||
{ 1440000, 3 },
|
||||
};
|
||||
|
||||
for (const auto & [n_samples, expected] : cases) {
|
||||
const std::string name = std::to_string(n_samples) + " samples";
|
||||
std::vector<float> samples(n_samples, 0.0f);
|
||||
if (n_samples > 0) {
|
||||
samples.back() = 0.5f;
|
||||
}
|
||||
mtmd_audio_preproc_out out = preproc.preprocess(samples.data(), samples.size());
|
||||
t.assert_equal("chunks for " + name, expected, out.entries.size());
|
||||
for (const auto & mel : out.entries) {
|
||||
t.assert_equal("frames for " + name, 3000, mel.nx());
|
||||
t.assert_equal("mel bins for " + name, 128, mel.ny());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
MAKE_TEST(test_audio_preprocessor_conformer) {
|
||||
mtmd_hparams hparams;
|
||||
hparams.n_mel_bins = 128;
|
||||
|
||||
Reference in new issue
Block a user