server: fix transcription prompt for v1/realtime (#32507)

This commit is contained in:
Xuan-Son Nguyen authored and GitHub committed 2026-11-18 12:09:22 +01:00
1 parent 6da36139e8
commit 48a32194ee
2 files changed
+28 -6

No files matched your search

+1 -1
View File
@@ -1640,7 +1640,7 @@ Models that answer after the audio, like LFM3-Audio and LFM3-Omni, hold a conver
`session.type` selects the session, it can be changed with `session.update` before the first audio or message:
- `realtime` (default): a conversation, every response stays in context for the next turns
- `transcription`: each committed audio buffer is transcribed on its own, `audio.input.transcription.language` picks the language
- `transcription`: each committed audio buffer is transcribed on its own, `audio.input.transcription.language` picks the language and `audio.input.transcription.prompt` replaces the default instruction (`Transcribe audio to text`) as the `prompt` of `/v1/audio/transcriptions` does; the prompt can change between utterances
*Client events:*
+27 -5
View File
@@ -130,6 +130,7 @@ struct realtime_conversation {
std::string type = "realtime";
std::string instructions;
std::string language; // transcription
std::string prompt; // transcription
std::string out_language;
std::string out_voice;
mtmd_helper_gen_audio_out_mode out_mode = MTMD_HELPER_GEN_AUDIO_OUT_AUTO;
@@ -201,7 +202,11 @@ struct realtime_conversation {
json session_json() const {
json audio_in = {{"format", {{"type", "audio/pcm"}, {"rate", input_rate}}}, {"turn_detection", nullptr}};
if (type == "transcription") {
audio_in["transcription"] = {{"model", deps.meta.model_name}, {"language", language.empty() ? json(nullptr) : json(language)}};
audio_in["transcription"] = {
{"model", deps.meta.model_name},
{"language", language.empty() ? json(nullptr) : json(language)},
{"prompt", prompt},
};
return {
{"id", id},
{"object", "realtime.transcription_session"},
@@ -342,12 +347,28 @@ struct realtime_conversation {
// conversation
//
// the transcription instruction, built the same way as /v1/audio/transcriptions
std::string asr_user_text(const common_chat_prompt_preset & preset) const {
std::string text = prompt.empty() || preset.has_context ? preset.user : prompt;
if (!language.empty() && !preset.has_language) {
text += string_format(" (language: %s)", language.c_str());
}
return text;
}
// the header of a user turn goes in before its first audio or text
bool ensure_turn() {
if (turn_open) {
return true;
}
const std::string header = n_turns == 0 ? first_header : next_header + user_header;
std::string header = n_turns == 0 ? first_header : next_header + user_header;
if (type == "transcription") {
// most models take the instruction before the audio, the others get it in on_commit
const auto preset = common_chat_get_asr_prompt(deps.meta.chat_params.tmpls.get(), language, 0.0, prompt);
if (!preset.media_first) {
header += asr_user_text(preset);
}
}
if (!control(SERVER_CONTROL_ACTION_TEXT_APPEND, header)) {
return false;
}
@@ -440,11 +461,11 @@ struct realtime_conversation {
}
// some models (VibeVoice-ASR) take an instruction after the audio, with its length
const auto * tmpls = deps.meta.chat_params.tmpls.get();
common_chat_prompt_preset preset = common_chat_get_asr_prompt(tmpls, language);
common_chat_prompt_preset preset = common_chat_get_asr_prompt(tmpls, language, 0.0, prompt);
if (preset.needs_audio_length) {
preset = common_chat_get_asr_prompt(tmpls, language, (double) n_audio / mtmd_get_audio_sample_rate(deps.mctx));
preset = common_chat_get_asr_prompt(tmpls, language, (double) n_audio / mtmd_get_audio_sample_rate(deps.mctx), prompt);
}
const std::string after_audio = preset.media_first ? preset.user : "";
const std::string after_audio = preset.media_first ? asr_user_text(preset) : "";
server_control_response response;
response.n_predict = max_output_tokens;
@@ -682,6 +703,7 @@ struct realtime_conversation {
}
if (in.contains("transcription") && in["transcription"].is_object()) {
language = json_value(in["transcription"], "language", language);
prompt = json_value(in["transcription"], "prompt", prompt);
}
const json out = audio.value("output", json::object());
if (out.contains("voice") || out.contains("language")) {