mtmd: add mtmd_helper_resampler (#31964)
* mtmd: add mtmd_helper_resampler * update docs * add test * MA_MAX_FILTER_ORDER
This commit is contained in:
1 parent
0a94a3f289
commit
3d94ffe452
6 files changed
+244
-9
No files matched your search
@@ -6178,12 +6178,13 @@ void server_routes::init_routes() {
|
||||
}
|
||||
|
||||
const std::string session_id = "sess_" + random_string();
|
||||
const json session_config = json {
|
||||
const int model_rate = mtmd_get_audio_sample_rate(ctx_server.mctx);
|
||||
json session_config = json {
|
||||
{"id", session_id},
|
||||
{"object", "realtime.transcription_session"},
|
||||
{"type", "transcription"},
|
||||
{"audio", {{"input", {
|
||||
{"format", {{"type", "audio/pcm"}, {"rate", mtmd_get_audio_sample_rate(ctx_server.mctx)}}},
|
||||
{"format", {{"type", "audio/pcm"}, {"rate", model_rate}}},
|
||||
{"transcription", {{"model", meta->model_name}}},
|
||||
{"turn_detection", nullptr},
|
||||
{"noise_reduction", nullptr},
|
||||
@@ -6248,6 +6249,8 @@ void server_routes::init_routes() {
|
||||
// client events are read on this thread, the transcript is sent from the thread below
|
||||
std::atomic<bool> input_ended{false};
|
||||
std::thread reader([&]() {
|
||||
int input_rate = model_rate;
|
||||
mtmd_helper::resampler_ptr resampler; // input_rate -> model_rate
|
||||
std::string msg;
|
||||
while (conn.read(msg) != 0) {
|
||||
json ev;
|
||||
@@ -6259,6 +6262,21 @@ void server_routes::init_routes() {
|
||||
}
|
||||
const std::string type = json_value(ev, "type", std::string());
|
||||
if (type == "session.update" || type == "transcription_session.update") {
|
||||
const json session = json_value(ev, "session", json::object());
|
||||
const json audio = session.is_object() ? session.value("audio", json::object()) : json::object();
|
||||
const json in = audio.is_object() ? audio.value("input", json::object()) : json::object();
|
||||
if (in.is_object() && in.contains("format") && in["format"].is_object()) {
|
||||
const int rate = json_value(in["format"], "rate", input_rate);
|
||||
if (rate < 8000 || rate > 192000) {
|
||||
send_error_event("session.update: the input audio rate must be between 8000 and 192000 Hz", "invalid_request_error");
|
||||
continue;
|
||||
}
|
||||
if (rate != input_rate) {
|
||||
input_rate = rate;
|
||||
resampler.reset();
|
||||
session_config["audio"]["input"]["format"]["rate"] = rate;
|
||||
}
|
||||
}
|
||||
send_event(json {{"type", "session.updated"}, {"session", session_config}});
|
||||
} else if (type == "input_audio_buffer.append") {
|
||||
std::string pcm;
|
||||
@@ -6274,6 +6292,25 @@ void server_routes::init_routes() {
|
||||
memcpy(&v, pcm.data() + 2 * i, sizeof(v));
|
||||
samples[i] = v / 32768.0f;
|
||||
}
|
||||
if (!resampler) {
|
||||
resampler.reset(mtmd_helper_resampler_init(input_rate, model_rate));
|
||||
if (!resampler) {
|
||||
send_error_event("failed to initialize the audio resampler", "server_error");
|
||||
break;
|
||||
}
|
||||
}
|
||||
const float * out = nullptr;
|
||||
const int64_t n = mtmd_helper_resampler_process(resampler.get(), samples.data(), samples.size(), &out);
|
||||
if (n < 0) {
|
||||
send_error_event("failed to resample the audio", "server_error");
|
||||
break;
|
||||
}
|
||||
if (n == 0) {
|
||||
continue;
|
||||
}
|
||||
if (out != samples.data()) {
|
||||
samples.assign(out, out + n);
|
||||
}
|
||||
if (!post_control(SERVER_CONTROL_ACTION_AUDIO_APPEND, std::move(samples))) {
|
||||
break;
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user