diff --git a/tests/test-mtmd-impl.cpp b/tests/test-mtmd-impl.cpp index a0201bdde..3fd720088 100644 --- a/tests/test-mtmd-impl.cpp +++ b/tests/test-mtmd-impl.cpp @@ -665,6 +665,85 @@ MAKE_TEST(test_audio_log_mel_blocks) { t.assert_true("max_diff = " + std::to_string(max_diff), max_diff < 1e-5f); } +// +// mtmd_helper_resampler +// + +static std::vector make_tone(double freq, int rate, size_t n) { + const double pi = 3.14159265358979323846; + std::vector x(n); + for (size_t i = 0; i < n; i++) { + x[i] = (float) (0.5 * std::sin(2 * pi * freq * i / rate)); + } + return x; +} + +// amplitude of the freq component, skipping the edges +static double tone_amplitude(const std::vector & x, double freq, int rate) { + const double pi = 3.14159265358979323846; + const size_t skip = rate / 10; + double re = 0.0, im = 0.0; + for (size_t i = skip; i < x.size() - skip; i++) { + re += x[i] * std::cos(2 * pi * freq * i / rate); + im += x[i] * std::sin(2 * pi * freq * i / rate); + } + return 2.0 * std::sqrt(re * re + im * im) / (x.size() - 2 * skip); +} + +static std::vector resample_chunked(mtmd_helper_resampler * rs, const std::vector & in, size_t chunk) { + std::vector res; + for (size_t off = 0; off < in.size(); off += chunk) { + const float * out = nullptr; + const int64_t n = mtmd_helper_resampler_process(rs, in.data() + off, std::min(chunk, in.size() - off), &out); + if (n < 0) { + return {}; + } + res.insert(res.end(), out, out + n); + } + return res; +} + +MAKE_TEST(test_helper_resampler) { + const int rate_out = 16000; + for (int rate_in : { 8000, 24000, 44100, 48000 }) { + const std::string name = std::to_string(rate_in) + " -> " + std::to_string(rate_out); + const std::vector in = make_tone(440, rate_in, (size_t) rate_in * 2); + + mtmd_helper::resampler_ptr rs_all(mtmd_helper_resampler_init(rate_in, rate_out)); + mtmd_helper::resampler_ptr rs_chk(mtmd_helper_resampler_init(rate_in, rate_out)); + if (!t.assert_true(name + " init", rs_all && rs_chk)) { + continue; + } + const std::vector all = resample_chunked(rs_all.get(), in, in.size()); + const std::vector chk = resample_chunked(rs_chk.get(), in, 1234); // odd size, not a multiple of the ratio + + t.assert_equal(name + " n_samples", (size_t) rate_out * 2, all.size()); + t.assert_true(name + " chunked == one-shot", all == chk); + + const double amp = tone_amplitude(all, 440, rate_out); + t.assert_true(name + " amplitude = " + std::to_string(amp), std::fabs(amp - 0.5) < 0.02); + } + + // above the output Nyquist: 10 kHz at 48 kHz aliases to 6 kHz at 16 kHz, the low-pass filter must cut it + { + mtmd_helper::resampler_ptr rs(mtmd_helper_resampler_init(48000, rate_out)); + const std::vector out = resample_chunked(rs.get(), make_tone(10000, 48000, 48000 * 2), 4800); + const double db = 20 * std::log10(tone_amplitude(out, 6000, rate_out) / 0.5); + t.assert_true("aliasing = " + std::to_string(db) + " dB", db < -15); // about -20 dB with lpfOrder = 8, -10 dB with the default + } + + // same rate: no copy + { + mtmd_helper::resampler_ptr rs(mtmd_helper_resampler_init(rate_out, rate_out)); + const std::vector in = make_tone(440, rate_out, 1000); + const float * out = nullptr; + const int64_t n = rs ? mtmd_helper_resampler_process(rs.get(), in.data(), in.size(), &out) : -1; + t.assert_true("passthrough", n == (int64_t) in.size() && out == in.data()); + } + + t.assert_true("invalid rate", mtmd_helper_resampler_init(0, rate_out) == nullptr); +} + // // main // diff --git a/tools/mtmd/mtmd-helper.cpp b/tools/mtmd/mtmd-helper.cpp index d86537e55..aefe504b2 100644 --- a/tools/mtmd/mtmd-helper.cpp +++ b/tools/mtmd/mtmd-helper.cpp @@ -360,6 +360,74 @@ static bool decode_audio_from_buf(const unsigned char * buf_in, size_t len, int } // namespace audio_helpers +struct mtmd_helper_resampler { + ma_resampler rs; + bool passthrough = false; + std::vector out; +}; + +mtmd_helper_resampler * mtmd_helper_resampler_init(int32_t rate_in, int32_t rate_out) { + if (rate_in <= 0 || rate_out <= 0) { + LOG_ERR("%s: invalid sample rate %d -> %d\n", __func__, rate_in, rate_out); + return nullptr; + } + auto * ctx = new mtmd_helper_resampler; + if (rate_in == rate_out) { + ctx->passthrough = true; + return ctx; + } + ma_resampler_config config = ma_resampler_config_init(ma_format_f32, 1, (ma_uint32) rate_in, (ma_uint32) rate_out, + ma_resample_algorithm_linear); + config.linear.lpfOrder = MA_MAX_FILTER_ORDER; + if (ma_resampler_init(&config, nullptr, &ctx->rs) != MA_SUCCESS) { + LOG_ERR("%s: failed to initialize the resampler %d -> %d\n", __func__, rate_in, rate_out); + delete ctx; + return nullptr; + } + return ctx; +} + +void mtmd_helper_resampler_free(mtmd_helper_resampler * ctx) { + if (!ctx) { + return; + } + if (!ctx->passthrough) { + ma_resampler_uninit(&ctx->rs, nullptr); + } + delete ctx; +} + +int64_t mtmd_helper_resampler_process(mtmd_helper_resampler * ctx, const float * in, size_t n_in, const float ** out) { + if (ctx->passthrough) { + *out = in; + return (int64_t) n_in; + } + ma_uint64 n_expected = 0; + if (ma_resampler_get_expected_output_frame_count(&ctx->rs, n_in, &n_expected) != MA_SUCCESS) { + return -1; + } + ctx->out.resize(n_expected + 16); + size_t n_done_in = 0; + size_t n_done_out = 0; + while (n_done_in < n_in) { + if (n_done_out == ctx->out.size()) { + ctx->out.resize(ctx->out.size() * 2); + } + ma_uint64 cnt_in = n_in - n_done_in; + ma_uint64 cnt_out = ctx->out.size() - n_done_out; + if (ma_resampler_process_pcm_frames(&ctx->rs, in + n_done_in, &cnt_in, ctx->out.data() + n_done_out, &cnt_out) != MA_SUCCESS) { + return -1; + } + if (cnt_in == 0 && cnt_out == 0) { + break; + } + n_done_in += cnt_in; + n_done_out += cnt_out; + } + *out = ctx->out.data(); + return (int64_t) n_done_out; +} + static bool is_webp_file(const unsigned char * buf, size_t len) { // WEBP ref: https://developers.google.com/speed/webp/docs/riff_container return len >= 12 && memcmp(buf, "RIFF", 4) == 0 && memcmp(buf + 8, "WEBP", 4) == 0; diff --git a/tools/mtmd/mtmd-helper.h b/tools/mtmd/mtmd-helper.h index 8a1d3aae1..7d72fdb99 100644 --- a/tools/mtmd/mtmd-helper.h +++ b/tools/mtmd/mtmd-helper.h @@ -78,6 +78,19 @@ MTMD_API struct mtmd_helper_bitmap_wrapper mtmd_helper_bitmap_init_from_buf( bool placeholder, struct mtmd_helper_init_opt opt); +// streaming resampler for raw mono F32 PCM (e.g. realtime input), to match mtmd_get_audio_sample_rate() +// it keeps state between calls, use one per stream +struct mtmd_helper_resampler; +typedef struct mtmd_helper_resampler mtmd_helper_resampler; + +// returns NULL on failure +MTMD_API mtmd_helper_resampler * mtmd_helper_resampler_init(int32_t rate_in, int32_t rate_out); +MTMD_API void mtmd_helper_resampler_free(mtmd_helper_resampler * ctx); + +// *out is valid until the next call +// returns the number of output samples (can be 0), < 0 on error +MTMD_API int64_t mtmd_helper_resampler_process(mtmd_helper_resampler * ctx, const float * in, size_t n_in, const float ** out); + // helper to count the total number of tokens from a list of chunks, useful to keep track of KV cache MTMD_API size_t mtmd_helper_get_n_tokens(const mtmd_input_chunks * chunks); @@ -334,6 +347,12 @@ struct mtmd_helper_video_deleter { }; using video_ptr = std::unique_ptr; +// audio input-related C++ wrappers +struct mtmd_helper_resampler_deleter { + void operator()(mtmd_helper_resampler * val) { mtmd_helper_resampler_free(val); } +}; +using resampler_ptr = std::unique_ptr; + // audio generation-related C++ wrappers struct mtmd_helper_gen_audio_deleter { void operator()(mtmd_helper_gen_audio * val) { mtmd_helper_gen_audio_free(val); } diff --git a/tools/server/README.md b/tools/server/README.md index dad14aaf1..712205dd5 100644 --- a/tools/server/README.md +++ b/tools/server/README.md @@ -1596,11 +1596,11 @@ Models that write the text while they listen use this endpoint: Voxtral Realtime Live ASR models can also hold a conversation, so they transcribe live only in sessions opened with `/v1/realtime?intent=transcription`, as OpenAI transcription sessions are; other sessions are a [realtime conversation](#realtime-conversation). -Audio must be PCM16, mono, at the model sample rate. The sample rate is returned in `session.created` as `audio.input.format.rate`. Each `input_audio_buffer.append` carries a base64-encoded chunk of PCM16 samples. +Audio must be PCM16, mono. By default it is at the model sample rate, returned in `session.created` as `audio.input.format.rate`. A client can send another rate (8000 to 192000 Hz) by setting `audio.input.format.rate` with `session.update`, the server then resamples it. Each `input_audio_buffer.append` carries a base64-encoded chunk of PCM16 samples. *Client events:* -- `session.update`: accepted for compatibility, the server replies with `session.updated` +- `session.update`: only `audio.input.format.rate` is used, the server replies with `session.updated` - `input_audio_buffer.append`: `audio` holds base64 PCM16 samples - `input_audio_buffer.commit`: flushes the audio received so far, so the words still held back by the model's delay are produced. The server replies with `input_audio_buffer.committed` and starts a new item. With `"final": true`, the input ends instead and the last item is completed once the stream drains. - `input_audio_buffer.clear`: accepted for compatibility, audio already sent cannot be discarded @@ -1619,7 +1619,7 @@ Note that words held back by the model's delay at the time of a commit are attri #### Realtime conversation -Models that answer after the audio, like LFM3-Audio and LFM3-Omni, hold a conversation on the same endpoint (with an `asr` template, unless the session is opened with `?intent=transcription`). The audio is prefilled as it arrives, so the response starts right after the last chunk. Audio input is PCM16 mono at the model sample rate (`audio.input.format.rate` of `session.created`), audio output is PCM16 mono at `audio.output.format.rate`. +Models that answer after the audio, like LFM3-Audio and LFM3-Omni, hold a conversation on the same endpoint (with an `asr` template, unless the session is opened with `?intent=transcription`). The audio is prefilled as it arrives, so the response starts right after the last chunk. Audio input is PCM16 mono, by default at the model sample rate (`audio.input.format.rate` of `session.created`); another rate set with `session.update` is resampled by the server. Audio output is PCM16 mono at `audio.output.format.rate`. `session.type` selects the session, it can be changed with `session.update` before the first audio or message: @@ -1628,7 +1628,7 @@ Models that answer after the audio, like LFM3-Audio and LFM3-Omni, hold a conver *Client events:* -- `session.update`: `type`, `instructions` (the system prompt, before the first audio or message), `output_modalities`, `max_output_tokens` (an integer or `"inf"`, the default), `tools` (functions with `name`, `description` and `parameters`, before the first audio or message) and `tool_choice` (`"auto"`, the default, `"none"` or `"required"`) +- `session.update`: `type`, `audio.input.format.rate` (8000 to 192000 Hz), `instructions` (the system prompt, before the first audio or message), `output_modalities`, `max_output_tokens` (an integer or `"inf"`, the default), `tools` (functions with `name`, `description` and `parameters`, before the first audio or message) and `tool_choice` (`"auto"`, the default, `"none"` or `"required"`) - `input_audio_buffer.append`: `audio` holds base64 PCM16 samples - `input_audio_buffer.commit`: ends the audio of the user turn, in a transcription session the transcript follows - `conversation.item.create`: a user message with `input_text` content, or a `function_call_output` with the `call_id` and `output` of a tool call of the last response diff --git a/tools/server/server-context.cpp b/tools/server/server-context.cpp index 407672d5e..be44c3e2e 100644 --- a/tools/server/server-context.cpp +++ b/tools/server/server-context.cpp @@ -6178,12 +6178,13 @@ void server_routes::init_routes() { } const std::string session_id = "sess_" + random_string(); - const json session_config = json { + const int model_rate = mtmd_get_audio_sample_rate(ctx_server.mctx); + json session_config = json { {"id", session_id}, {"object", "realtime.transcription_session"}, {"type", "transcription"}, {"audio", {{"input", { - {"format", {{"type", "audio/pcm"}, {"rate", mtmd_get_audio_sample_rate(ctx_server.mctx)}}}, + {"format", {{"type", "audio/pcm"}, {"rate", model_rate}}}, {"transcription", {{"model", meta->model_name}}}, {"turn_detection", nullptr}, {"noise_reduction", nullptr}, @@ -6248,6 +6249,8 @@ void server_routes::init_routes() { // client events are read on this thread, the transcript is sent from the thread below std::atomic input_ended{false}; std::thread reader([&]() { + int input_rate = model_rate; + mtmd_helper::resampler_ptr resampler; // input_rate -> model_rate std::string msg; while (conn.read(msg) != 0) { json ev; @@ -6259,6 +6262,21 @@ void server_routes::init_routes() { } const std::string type = json_value(ev, "type", std::string()); if (type == "session.update" || type == "transcription_session.update") { + const json session = json_value(ev, "session", json::object()); + const json audio = session.is_object() ? session.value("audio", json::object()) : json::object(); + const json in = audio.is_object() ? audio.value("input", json::object()) : json::object(); + if (in.is_object() && in.contains("format") && in["format"].is_object()) { + const int rate = json_value(in["format"], "rate", input_rate); + if (rate < 8000 || rate > 192000) { + send_error_event("session.update: the input audio rate must be between 8000 and 192000 Hz", "invalid_request_error"); + continue; + } + if (rate != input_rate) { + input_rate = rate; + resampler.reset(); + session_config["audio"]["input"]["format"]["rate"] = rate; + } + } send_event(json {{"type", "session.updated"}, {"session", session_config}}); } else if (type == "input_audio_buffer.append") { std::string pcm; @@ -6274,6 +6292,25 @@ void server_routes::init_routes() { memcpy(&v, pcm.data() + 2 * i, sizeof(v)); samples[i] = v / 32768.0f; } + if (!resampler) { + resampler.reset(mtmd_helper_resampler_init(input_rate, model_rate)); + if (!resampler) { + send_error_event("failed to initialize the audio resampler", "server_error"); + break; + } + } + const float * out = nullptr; + const int64_t n = mtmd_helper_resampler_process(resampler.get(), samples.data(), samples.size(), &out); + if (n < 0) { + send_error_event("failed to resample the audio", "server_error"); + break; + } + if (n == 0) { + continue; + } + if (out != samples.data()) { + samples.assign(out, out + n); + } if (!post_control(SERVER_CONTROL_ACTION_AUDIO_APPEND, std::move(samples))) { break; } diff --git a/tools/server/server-realtime.cpp b/tools/server/server-realtime.cpp index 5ef36a2b8..689626f5f 100644 --- a/tools/server/server-realtime.cpp +++ b/tools/server/server-realtime.cpp @@ -156,6 +156,7 @@ struct realtime_conversation { bool tool_results = false; // tool results went in after the last response, the next one answers them int n_turns = 0; std::string next_header; // text that goes in before the next user turn + mtmd_helper::resampler_ptr resampler; // input_rate -> model rate // the item or response the results belong to, shared with the result loop std::mutex mutex; @@ -362,7 +363,10 @@ struct realtime_conversation { memcpy(&v, pcm.data() + 2 * i, sizeof(v)); samples[i] = v / 32768.0f; } - if (samples.empty() || !ensure_turn()) { + if (samples.empty()) { + return; + } + if (!resample(samples) || samples.empty() || !ensure_turn()) { return; } if (control(SERVER_CONTROL_ACTION_AUDIO_APPEND, {}, std::move(samples))) { @@ -370,6 +374,26 @@ struct realtime_conversation { } } + bool resample(std::vector & samples) { + if (!resampler) { + resampler.reset(mtmd_helper_resampler_init(input_rate, mtmd_get_audio_sample_rate(deps.mctx))); + if (!resampler) { + send_error("failed to initialize the audio resampler", "server_error"); + return false; + } + } + const float * out = nullptr; + const int64_t n = mtmd_helper_resampler_process(resampler.get(), samples.data(), samples.size(), &out); + if (n < 0) { + send_error("failed to resample the audio", "server_error"); + return false; + } + if (out != samples.data()) { + samples.assign(out, out + n); + } + return true; + } + // ends the audio of the open turn, it becomes a user item bool commit() { if (!has_audio) { @@ -623,8 +647,16 @@ struct realtime_conversation { } const json audio = session.value("audio", json::object()); const json in = audio.value("input", json::object()); - if (in.contains("format") && in["format"].is_object() && json_value(in["format"], "rate", input_rate) != input_rate) { - send_error("session.update: the input audio must be PCM16 at " + std::to_string(input_rate) + " Hz"); + if (in.contains("format") && in["format"].is_object()) { + const int rate = json_value(in["format"], "rate", input_rate); + if (rate < 8000 || rate > 192000) { + send_error("session.update: the input audio rate must be between 8000 and 192000 Hz"); + return; + } + if (rate != input_rate) { + input_rate = rate; + resampler.reset(); + } } if (in.contains("transcription") && in["transcription"].is_object()) { language = json_value(in["transcription"], "language", language);