From 48a32194eefc4bbedd31774ead928f64ef8aede2 Mon Sep 17 00:00:00 2001 From: Xuan-Son Nguyen Date: Wed, 18 Nov 2026 12:09:22 +0100 Subject: [PATCH] server: fix transcription prompt for v1/realtime (#32507) --- tools/server/README.md | 2 +- tools/server/server-realtime.cpp | 32 +++++++++++++++++++++++++++----- 2 files changed, 28 insertions(+), 6 deletions(-) diff --git a/tools/server/README.md b/tools/server/README.md index 4643eb7a5..292fe598e 100644 --- a/tools/server/README.md +++ b/tools/server/README.md @@ -1640,7 +1640,7 @@ Models that answer after the audio, like LFM3-Audio and LFM3-Omni, hold a conver `session.type` selects the session, it can be changed with `session.update` before the first audio or message: - `realtime` (default): a conversation, every response stays in context for the next turns -- `transcription`: each committed audio buffer is transcribed on its own, `audio.input.transcription.language` picks the language +- `transcription`: each committed audio buffer is transcribed on its own, `audio.input.transcription.language` picks the language and `audio.input.transcription.prompt` replaces the default instruction (`Transcribe audio to text`) as the `prompt` of `/v1/audio/transcriptions` does; the prompt can change between utterances *Client events:* diff --git a/tools/server/server-realtime.cpp b/tools/server/server-realtime.cpp index 201740b2e..6f08120dd 100644 --- a/tools/server/server-realtime.cpp +++ b/tools/server/server-realtime.cpp @@ -130,6 +130,7 @@ struct realtime_conversation { std::string type = "realtime"; std::string instructions; std::string language; // transcription + std::string prompt; // transcription std::string out_language; std::string out_voice; mtmd_helper_gen_audio_out_mode out_mode = MTMD_HELPER_GEN_AUDIO_OUT_AUTO; @@ -201,7 +202,11 @@ struct realtime_conversation { json session_json() const { json audio_in = {{"format", {{"type", "audio/pcm"}, {"rate", input_rate}}}, {"turn_detection", nullptr}}; if (type == "transcription") { - audio_in["transcription"] = {{"model", deps.meta.model_name}, {"language", language.empty() ? json(nullptr) : json(language)}}; + audio_in["transcription"] = { + {"model", deps.meta.model_name}, + {"language", language.empty() ? json(nullptr) : json(language)}, + {"prompt", prompt}, + }; return { {"id", id}, {"object", "realtime.transcription_session"}, @@ -342,12 +347,28 @@ struct realtime_conversation { // conversation // + // the transcription instruction, built the same way as /v1/audio/transcriptions + std::string asr_user_text(const common_chat_prompt_preset & preset) const { + std::string text = prompt.empty() || preset.has_context ? preset.user : prompt; + if (!language.empty() && !preset.has_language) { + text += string_format(" (language: %s)", language.c_str()); + } + return text; + } + // the header of a user turn goes in before its first audio or text bool ensure_turn() { if (turn_open) { return true; } - const std::string header = n_turns == 0 ? first_header : next_header + user_header; + std::string header = n_turns == 0 ? first_header : next_header + user_header; + if (type == "transcription") { + // most models take the instruction before the audio, the others get it in on_commit + const auto preset = common_chat_get_asr_prompt(deps.meta.chat_params.tmpls.get(), language, 0.0, prompt); + if (!preset.media_first) { + header += asr_user_text(preset); + } + } if (!control(SERVER_CONTROL_ACTION_TEXT_APPEND, header)) { return false; } @@ -440,11 +461,11 @@ struct realtime_conversation { } // some models (VibeVoice-ASR) take an instruction after the audio, with its length const auto * tmpls = deps.meta.chat_params.tmpls.get(); - common_chat_prompt_preset preset = common_chat_get_asr_prompt(tmpls, language); + common_chat_prompt_preset preset = common_chat_get_asr_prompt(tmpls, language, 0.0, prompt); if (preset.needs_audio_length) { - preset = common_chat_get_asr_prompt(tmpls, language, (double) n_audio / mtmd_get_audio_sample_rate(deps.mctx)); + preset = common_chat_get_asr_prompt(tmpls, language, (double) n_audio / mtmd_get_audio_sample_rate(deps.mctx), prompt); } - const std::string after_audio = preset.media_first ? preset.user : ""; + const std::string after_audio = preset.media_first ? asr_user_text(preset) : ""; server_control_response response; response.n_predict = max_output_tokens; @@ -682,6 +703,7 @@ struct realtime_conversation { } if (in.contains("transcription") && in["transcription"].is_object()) { language = json_value(in["transcription"], "language", language); + prompt = json_value(in["transcription"], "prompt", prompt); } const json out = audio.value("output", json::object()); if (out.contains("voice") || out.contains("language")) {