mtmd: add mtmd_helper_resampler (#31964)

* mtmd: add mtmd_helper_resampler

* update docs

* add test

* MA_MAX_FILTER_ORDER
This commit is contained in:
Xuan-Son Nguyen authored and GitHub committed 2026-11-09 23:04:39 +01:00
1 parent 0a94a3f289
commit 3d94ffe452
6 files changed
+244 -9

No files matched your search

+39 -2
View File
@@ -6178,12 +6178,13 @@ void server_routes::init_routes() {
}
const std::string session_id = "sess_" + random_string();
const json session_config = json {
const int model_rate = mtmd_get_audio_sample_rate(ctx_server.mctx);
json session_config = json {
{"id", session_id},
{"object", "realtime.transcription_session"},
{"type", "transcription"},
{"audio", {{"input", {
{"format", {{"type", "audio/pcm"}, {"rate", mtmd_get_audio_sample_rate(ctx_server.mctx)}}},
{"format", {{"type", "audio/pcm"}, {"rate", model_rate}}},
{"transcription", {{"model", meta->model_name}}},
{"turn_detection", nullptr},
{"noise_reduction", nullptr},
@@ -6248,6 +6249,8 @@ void server_routes::init_routes() {
// client events are read on this thread, the transcript is sent from the thread below
std::atomic<bool> input_ended{false};
std::thread reader([&]() {
int input_rate = model_rate;
mtmd_helper::resampler_ptr resampler; // input_rate -> model_rate
std::string msg;
while (conn.read(msg) != 0) {
json ev;
@@ -6259,6 +6262,21 @@ void server_routes::init_routes() {
}
const std::string type = json_value(ev, "type", std::string());
if (type == "session.update" || type == "transcription_session.update") {
const json session = json_value(ev, "session", json::object());
const json audio = session.is_object() ? session.value("audio", json::object()) : json::object();
const json in = audio.is_object() ? audio.value("input", json::object()) : json::object();
if (in.is_object() && in.contains("format") && in["format"].is_object()) {
const int rate = json_value(in["format"], "rate", input_rate);
if (rate < 8000 || rate > 192000) {
send_error_event("session.update: the input audio rate must be between 8000 and 192000 Hz", "invalid_request_error");
continue;
}
if (rate != input_rate) {
input_rate = rate;
resampler.reset();
session_config["audio"]["input"]["format"]["rate"] = rate;
}
}
send_event(json {{"type", "session.updated"}, {"session", session_config}});
} else if (type == "input_audio_buffer.append") {
std::string pcm;
@@ -6274,6 +6292,25 @@ void server_routes::init_routes() {
memcpy(&v, pcm.data() + 2 * i, sizeof(v));
samples[i] = v / 32768.0f;
}
if (!resampler) {
resampler.reset(mtmd_helper_resampler_init(input_rate, model_rate));
if (!resampler) {
send_error_event("failed to initialize the audio resampler", "server_error");
break;
}
}
const float * out = nullptr;
const int64_t n = mtmd_helper_resampler_process(resampler.get(), samples.data(), samples.size(), &out);
if (n < 0) {
send_error_event("failed to resample the audio", "server_error");
break;
}
if (n == 0) {
continue;
}
if (out != samples.data()) {
samples.assign(out, out + n);
}
if (!post_control(SERVER_CONTROL_ACTION_AUDIO_APPEND, std::move(samples))) {
break;
}