server: fix transcription prompt for v1/realtime (#32507)
This commit is contained in:
1 parent
6da36139e8
commit
48a32194ee
2 files changed
+28
-6
No files matched your search
@@ -1640,7 +1640,7 @@ Models that answer after the audio, like LFM3-Audio and LFM3-Omni, hold a conver
|
||||
`session.type` selects the session, it can be changed with `session.update` before the first audio or message:
|
||||
|
||||
- `realtime` (default): a conversation, every response stays in context for the next turns
|
||||
- `transcription`: each committed audio buffer is transcribed on its own, `audio.input.transcription.language` picks the language
|
||||
- `transcription`: each committed audio buffer is transcribed on its own, `audio.input.transcription.language` picks the language and `audio.input.transcription.prompt` replaces the default instruction (`Transcribe audio to text`) as the `prompt` of `/v1/audio/transcriptions` does; the prompt can change between utterances
|
||||
|
||||
*Client events:*
|
||||
|
||||
|
||||
@@ -130,6 +130,7 @@ struct realtime_conversation {
|
||||
std::string type = "realtime";
|
||||
std::string instructions;
|
||||
std::string language; // transcription
|
||||
std::string prompt; // transcription
|
||||
std::string out_language;
|
||||
std::string out_voice;
|
||||
mtmd_helper_gen_audio_out_mode out_mode = MTMD_HELPER_GEN_AUDIO_OUT_AUTO;
|
||||
@@ -201,7 +202,11 @@ struct realtime_conversation {
|
||||
json session_json() const {
|
||||
json audio_in = {{"format", {{"type", "audio/pcm"}, {"rate", input_rate}}}, {"turn_detection", nullptr}};
|
||||
if (type == "transcription") {
|
||||
audio_in["transcription"] = {{"model", deps.meta.model_name}, {"language", language.empty() ? json(nullptr) : json(language)}};
|
||||
audio_in["transcription"] = {
|
||||
{"model", deps.meta.model_name},
|
||||
{"language", language.empty() ? json(nullptr) : json(language)},
|
||||
{"prompt", prompt},
|
||||
};
|
||||
return {
|
||||
{"id", id},
|
||||
{"object", "realtime.transcription_session"},
|
||||
@@ -342,12 +347,28 @@ struct realtime_conversation {
|
||||
// conversation
|
||||
//
|
||||
|
||||
// the transcription instruction, built the same way as /v1/audio/transcriptions
|
||||
std::string asr_user_text(const common_chat_prompt_preset & preset) const {
|
||||
std::string text = prompt.empty() || preset.has_context ? preset.user : prompt;
|
||||
if (!language.empty() && !preset.has_language) {
|
||||
text += string_format(" (language: %s)", language.c_str());
|
||||
}
|
||||
return text;
|
||||
}
|
||||
|
||||
// the header of a user turn goes in before its first audio or text
|
||||
bool ensure_turn() {
|
||||
if (turn_open) {
|
||||
return true;
|
||||
}
|
||||
const std::string header = n_turns == 0 ? first_header : next_header + user_header;
|
||||
std::string header = n_turns == 0 ? first_header : next_header + user_header;
|
||||
if (type == "transcription") {
|
||||
// most models take the instruction before the audio, the others get it in on_commit
|
||||
const auto preset = common_chat_get_asr_prompt(deps.meta.chat_params.tmpls.get(), language, 0.0, prompt);
|
||||
if (!preset.media_first) {
|
||||
header += asr_user_text(preset);
|
||||
}
|
||||
}
|
||||
if (!control(SERVER_CONTROL_ACTION_TEXT_APPEND, header)) {
|
||||
return false;
|
||||
}
|
||||
@@ -440,11 +461,11 @@ struct realtime_conversation {
|
||||
}
|
||||
// some models (VibeVoice-ASR) take an instruction after the audio, with its length
|
||||
const auto * tmpls = deps.meta.chat_params.tmpls.get();
|
||||
common_chat_prompt_preset preset = common_chat_get_asr_prompt(tmpls, language);
|
||||
common_chat_prompt_preset preset = common_chat_get_asr_prompt(tmpls, language, 0.0, prompt);
|
||||
if (preset.needs_audio_length) {
|
||||
preset = common_chat_get_asr_prompt(tmpls, language, (double) n_audio / mtmd_get_audio_sample_rate(deps.mctx));
|
||||
preset = common_chat_get_asr_prompt(tmpls, language, (double) n_audio / mtmd_get_audio_sample_rate(deps.mctx), prompt);
|
||||
}
|
||||
const std::string after_audio = preset.media_first ? preset.user : "";
|
||||
const std::string after_audio = preset.media_first ? asr_user_text(preset) : "";
|
||||
|
||||
server_control_response response;
|
||||
response.n_predict = max_output_tokens;
|
||||
@@ -682,6 +703,7 @@ struct realtime_conversation {
|
||||
}
|
||||
if (in.contains("transcription") && in["transcription"].is_object()) {
|
||||
language = json_value(in["transcription"], "language", language);
|
||||
prompt = json_value(in["transcription"], "prompt", prompt);
|
||||
}
|
||||
const json out = audio.value("output", json::object());
|
||||
if (out.contains("voice") || out.contains("language")) {
|
||||
|
||||
Reference in new issue
Block a user