server : add realtime conversation and transcription on /v1/realtime (#31269)

* server : add realtime conversation and transcription on /v1/realtime

Co-authored-by: Xuan Son Nguyen <son@huggingface.co>
Assisted-by: Claude Opus 5.5

* examples : default the realtime client to the server port 9931

Assisted-by: Claude Opus 5.5

* server : sample realtime responses right after their own decode

Slots respond in turns on one context: sampling at the start of the next step read the logits of whichever slot decoded last, so concurrent sessions mixed their outputs.

Assisted-by: Claude Opus 5.5

* move more stuff to server-realtime, making it self-contained

* examples : fix the type check of the realtime client

Assisted-by: Claude Opus 5.5

* examples : use the model card prompts in the realtime client

Assisted-by: Claude Opus 5.5

---------

Co-authored-by: Xuan Son Nguyen <son@huggingface.co>
This commit is contained in:
Tarek DakhranandXuan Son Nguyen authored and GitHub committed 2026-10-28 01:26:12 +01:00
1 parent d05706b4f3
commit 8ca4060b7e
10 files changed
+1423 -14

No files matched your search

+108 -14
View File
@@ -8,6 +8,7 @@
#include "server-http.h"
#include "server-task.h"
#include "server-queue.h"
#include "server-realtime.h"
#include "server-schema.h"
#include "server-stream.h"
@@ -402,6 +403,9 @@ struct server_slot {
bool audio_wait = false; // the stream needs more audio before it can produce rows
std::deque<int64_t> audio_commit_rows; // end row of each non-final commit not yet completed
// realtime conversation, see SERVER_TASK_TYPE_REALTIME
std::unique_ptr<server_realtime_slot> rt;
server_slot_stats stats;
// accepted tokens per draft position
@@ -460,6 +464,7 @@ struct server_slot {
audio_wait = false;
audio_commit_rows.clear();
inp_embd.clear();
rt.reset();
}
void init_sampler() const {
@@ -660,7 +665,7 @@ struct server_slot {
// do not keep context of the child slots - the parent's context is enough
// nor of the stateless slots - no task ever reuses it
if (task->is_child() || task->is_stateless() || audio_stream) {
if (task->is_child() || task->is_stateless() || audio_stream || task->type == SERVER_TASK_TYPE_REALTIME) {
prompt_clear();
}
@@ -1023,6 +1028,9 @@ public:
mtmd_helper_init_opt init_opt = mtmd_helper_init_opt_default();
const llama_vocab * vocab = nullptr;
// prompt of the "asr" chat template, for live ASR
llama_tokens asr_prompt;
bool has_cap_tts() const {
return mtmd_cap.audio_gen;
}
@@ -1705,6 +1713,13 @@ private:
return false;
}
try {
asr_prompt = server_chat_get_asr_prompt(model_tgt);
} catch (const std::exception & e) {
SRV_ERR("%s: \"asr\" chat template error: %s\n", __func__, e.what());
return false;
}
// IMPORTANT: chat_params is reused across sleeping / resuming states,
// never store llama_context/llama_model pointers in chat_params,
// as they may be invalidated after sleeping
@@ -2020,6 +2035,23 @@ private:
}
}
// realtime transcription: the rows come from the audio stream, the helper may bias some tokens
if (task.type == SERVER_TASK_TYPE_TRANSCRIBE_RT) {
auto sparams = mtmd_audio_stream_params_default();
sparams.prompt = asr_prompt.data();
sparams.n_prompt = asr_prompt.size();
slot.audio_stream.reset(mtmd_audio_stream_init(mctx, sparams));
if (!slot.audio_stream) {
send_error(task, "failed to initialize the audio stream", ERROR_TYPE_SERVER);
return false;
}
if (!asr_prompt.empty()) {
for (const auto & b : mtmd_helper::audio_stream_logit_bias(mctx, model_tgt)) {
task.params.sampling.logit_bias.push_back(b);
}
}
}
// initialize samplers
if (task.need_sampling()) {
try {
@@ -2075,12 +2107,6 @@ private:
// no prompt, the rows come from the audio stream, which audio controls reach from now on
if (slot.task->type == SERVER_TASK_TYPE_TRANSCRIBE_RT) {
slot.audio_stream.reset(mtmd_audio_stream_init(mctx, mtmd_audio_stream_params_default()));
if (!slot.audio_stream) {
send_error(slot, "failed to initialize the audio stream", ERROR_TYPE_SERVER);
slot.release();
return false;
}
slot.prompt_clear();
slot.init_sampler();
slot.sampled = LLAMA_TOKEN_NULL;
@@ -2090,6 +2116,17 @@ private:
send_partial_response(slot, {}, false, true);
}
// no prompt either, the session appends text and audio and asks for responses with its controls
if (slot.task->type == SERVER_TASK_TYPE_REALTIME) {
slot.rt = std::make_unique<server_realtime_slot>(ctx_tgt, mctx, slot.id, slot.n_ctx, slot.task->params.sampling,
queue_results, slot.task->id, slot.task->index);
slot.prompt_clear();
slot.stats.update_prompt_start();
slot.state = SLOT_STATE_GENERATING;
slot.audio_wait = true;
send_partial_response(slot, {}, false, true);
}
// reset server kill-switch counter
n_empty_consecutive = 0;
@@ -2972,6 +3009,7 @@ private:
case SERVER_TASK_TYPE_COMPLETION:
case SERVER_TASK_TYPE_INFILL:
case SERVER_TASK_TYPE_TRANSCRIBE_RT:
case SERVER_TASK_TYPE_REALTIME:
case SERVER_TASK_TYPE_EMBEDDING:
case SERVER_TASK_TYPE_RERANK:
case SERVER_TASK_TYPE_TTS:
@@ -3084,6 +3122,20 @@ private:
// act on the live slot mid generation, never defer
common_sampler_reasoning_budget_force(slot->smpl.get());
res->success = true;
} else if (slot->rt) {
// a cancel acts now, everything else runs after what is already queued
if (task.params.control_action == SERVER_CONTROL_ACTION_RESPONSE_CANCEL) {
slot->rt->cancel();
} else {
slot->rt->push({
task.params.control_action,
std::move(task.params.control_text),
std::move(task.params.control_audio),
task.params.control_response,
});
}
slot->audio_wait = false;
res->success = true;
} else if (task.params.control_action == SERVER_CONTROL_ACTION_AUDIO_APPEND
|| task.params.control_action == SERVER_CONTROL_ACTION_AUDIO_COMMIT
|| task.params.control_action == SERVER_CONTROL_ACTION_AUDIO_END) {
@@ -3449,6 +3501,18 @@ private:
}
if (n > 0) {
// new sequence (e.g. live ASR restart): drop the old one, the text goes on in a new line
if (mtmd_audio_stream_is_seq_start(slot.audio_stream.get())) {
slot.prompt_clear();
if (slot.n_sent_text > 0) {
completion_token_output sep;
sep.tok = LLAMA_TOKEN_NULL;
sep.text_to_send = "\n";
slot.generated.append(common_chat_input(sep.text_to_send));
slot.n_sent_text += sep.text_to_send.size();
send_partial_response(slot, sep, false);
}
}
const float * rows = mtmd_audio_stream_get_output_embd(slot.audio_stream.get());
slot.inp_embd.assign(rows, rows + (size_t) n * batch.n_embd);
slot.audio_wait = false;
@@ -3534,6 +3598,14 @@ private:
abort_all_slots("process_tts_slots() failed: " + std::string(e.what()));
}
// so do the realtime conversations
try {
process_realtime_slots();
} catch (const std::exception & e) {
SRV_ERR("process_realtime_slots() failed: %s\n", e.what());
abort_all_slots("process_realtime_slots() failed: " + std::string(e.what()));
}
GGML_ASSERT(batch.slot_batched || batch.size() == 0);
if (batch.slot_batched) {
@@ -3596,6 +3668,21 @@ private:
}
}
// realtime conversations (SERVER_TASK_TYPE_REALTIME) drive their own decode loop, see server_realtime_slot
void process_realtime_slots() {
iterate(slots, [&](server_slot & slot) {
if (!slot.is_processing() || !slot.rt) {
return;
}
if (!slot.rt->update()) {
send_error(slot, slot.rt->error, slot.rt->error_type);
slot.release();
return;
}
slot.audio_wait = slot.rt->is_idle();
});
}
void process_tts_slots() {
iterate(slots, [&](server_slot & slot) {
if (!slot.is_processing() || slot.task->type != SERVER_TASK_TYPE_TTS) {
@@ -3675,8 +3762,8 @@ private:
// apply context-shift if needed
// TODO: simplify and improve
iterate(slots, [&](server_slot & slot) {
if (slot.task && slot.task->type == SERVER_TASK_TYPE_TTS) {
// TTS slots drive their own decode loop in process_tts_slots(), never enter the shared batch
if (slot.task && (slot.task->type == SERVER_TASK_TYPE_TTS || slot.task->type == SERVER_TASK_TYPE_REALTIME)) {
// TTS and realtime conversation slots drive their own decode loop, never enter the shared batch
return;
}
if (slot.state == SLOT_STATE_GENERATING && slot.prompt.n_tokens() + 1 >= slot.n_ctx) {
@@ -3751,7 +3838,7 @@ private:
// determine which slots are generating and drafting
iterate(slots, [&](server_slot & slot) {
if (slot.state != SLOT_STATE_GENERATING || slot.task->type == SERVER_TASK_TYPE_TTS) {
if (slot.state != SLOT_STATE_GENERATING || slot.task->type == SERVER_TASK_TYPE_TTS || slot.task->type == SERVER_TASK_TYPE_REALTIME) {
return;
}
@@ -3897,7 +3984,7 @@ private:
return; // batch is full, skip remaining slots
}
if (!slot.is_processing() || slot.task->type == SERVER_TASK_TYPE_TTS) {
if (!slot.is_processing() || slot.task->type == SERVER_TASK_TYPE_TTS || slot.task->type == SERVER_TASK_TYPE_REALTIME) {
return;
}
@@ -5150,8 +5237,7 @@ server_context_meta server_context::get_meta() const {
/* has_inp_video */ impl->chat_params.allow_video,
/* has_cap_chat */ impl->has_cap_chat(),
/* has_cap_tts */ impl->has_cap_tts(),
// only realtime transcription models (no chat) stream without a prompt
/* has_audio_stream */ impl->mtmd_cap.audio_stream && !impl->mtmd_cap.chat,
/* has_audio_stream */ impl->mtmd_cap.audio_stream,
/* json_ui_settings */ impl->json_ui_settings,
/* slot_n_ctx */ impl->n_ctx_slot(),
/* pooling_type */ llama_pooling_type(impl->ctx_tgt),
@@ -6025,7 +6111,7 @@ void server_routes::init_routes() {
// realtime transcription over websocket, modeled after the OpenAI realtime API
// audio is PCM16 mono at the model sample rate, sent as base64 in input_audio_buffer.append events
// input_audio_buffer.commit with "final": true ends the input, closing the socket also ends it
this->ws_realtime = [this](const server_http_req &, server_http_context::ws_conn & conn) {
this->ws_realtime = [this](const server_http_req & req, server_http_context::ws_conn & conn) {
auto send_event = [&conn](json ev) {
ev["event_id"] = "event_" + random_string();
conn.send(safe_json_to_str(ev));
@@ -6042,6 +6128,14 @@ void server_routes::init_routes() {
});
};
// models that answer after the audio hold a conversation, see server-realtime.h
// live transcription: Voxtral Realtime (no chat), or an "asr" model in a session opened with ?intent=transcription
const bool live_asr = !ctx_server.asr_prompt.empty() && req.get_param("intent") == "transcription";
if (ctx_server.mtmd_cap.audio_stream && ctx_server.mtmd_cap.chat && !live_asr) {
server_realtime_conversation({queue_tasks, queue_results, *meta, params.sampling, ctx_server.mctx, ctx_server.mtmd_cap, ctx_server.vocab}, conn);
return;
}
if (!meta->has_audio_stream) {
send_error_event("The current model does not support realtime transcription.", "invalid_request_error");
conn.close();