mtmd: add mtmd_helper_resampler (#31964)

* mtmd: add mtmd_helper_resampler

* update docs

* add test

* MA_MAX_FILTER_ORDER
This commit is contained in:
Xuan-Son Nguyen authored and GitHub committed 2026-11-09 23:04:39 +01:00
1 parent 0a94a3f289
commit 3d94ffe452
6 files changed
+244 -9

No files matched your search

+35 -3
View File
@@ -156,6 +156,7 @@ struct realtime_conversation {
bool tool_results = false; // tool results went in after the last response, the next one answers them
int n_turns = 0;
std::string next_header; // text that goes in before the next user turn
mtmd_helper::resampler_ptr resampler; // input_rate -> model rate
// the item or response the results belong to, shared with the result loop
std::mutex mutex;
@@ -362,7 +363,10 @@ struct realtime_conversation {
memcpy(&v, pcm.data() + 2 * i, sizeof(v));
samples[i] = v / 32768.0f;
}
if (samples.empty() || !ensure_turn()) {
if (samples.empty()) {
return;
}
if (!resample(samples) || samples.empty() || !ensure_turn()) {
return;
}
if (control(SERVER_CONTROL_ACTION_AUDIO_APPEND, {}, std::move(samples))) {
@@ -370,6 +374,26 @@ struct realtime_conversation {
}
}
bool resample(std::vector<float> & samples) {
if (!resampler) {
resampler.reset(mtmd_helper_resampler_init(input_rate, mtmd_get_audio_sample_rate(deps.mctx)));
if (!resampler) {
send_error("failed to initialize the audio resampler", "server_error");
return false;
}
}
const float * out = nullptr;
const int64_t n = mtmd_helper_resampler_process(resampler.get(), samples.data(), samples.size(), &out);
if (n < 0) {
send_error("failed to resample the audio", "server_error");
return false;
}
if (out != samples.data()) {
samples.assign(out, out + n);
}
return true;
}
// ends the audio of the open turn, it becomes a user item
bool commit() {
if (!has_audio) {
@@ -623,8 +647,16 @@ struct realtime_conversation {
}
const json audio = session.value("audio", json::object());
const json in = audio.value("input", json::object());
if (in.contains("format") && in["format"].is_object() && json_value(in["format"], "rate", input_rate) != input_rate) {
send_error("session.update: the input audio must be PCM16 at " + std::to_string(input_rate) + " Hz");
if (in.contains("format") && in["format"].is_object()) {
const int rate = json_value(in["format"], "rate", input_rate);
if (rate < 8000 || rate > 192000) {
send_error("session.update: the input audio rate must be between 8000 and 192000 Hz");
return;
}
if (rate != input_rate) {
input_rate = rate;
resampler.reset();
}
}
if (in.contains("transcription") && in["transcription"].is_object()) {
language = json_value(in["transcription"], "language", language);