server : add realtime conversation and transcription on /v1/realtime (#31269)
* server : add realtime conversation and transcription on /v1/realtime Co-authored-by: Xuan Son Nguyen <son@huggingface.co> Assisted-by: Claude Opus 5.5 * examples : default the realtime client to the server port 9931 Assisted-by: Claude Opus 5.5 * server : sample realtime responses right after their own decode Slots respond in turns on one context: sampling at the start of the next step read the logits of whichever slot decoded last, so concurrent sessions mixed their outputs. Assisted-by: Claude Opus 5.5 * move more stuff to server-realtime, making it self-contained * examples : fix the type check of the realtime client Assisted-by: Claude Opus 5.5 * examples : use the model card prompts in the realtime client Assisted-by: Claude Opus 5.5 --------- Co-authored-by: Xuan Son Nguyen <son@huggingface.co>
This commit is contained in:
1 parent
d05706b4f3
commit
8ca4060b7e
10 files changed
+1423
-14
No files matched your search
+108
-14
@@ -8,6 +8,7 @@
|
||||
#include "server-http.h"
|
||||
#include "server-task.h"
|
||||
#include "server-queue.h"
|
||||
#include "server-realtime.h"
|
||||
#include "server-schema.h"
|
||||
#include "server-stream.h"
|
||||
|
||||
@@ -402,6 +403,9 @@ struct server_slot {
|
||||
bool audio_wait = false; // the stream needs more audio before it can produce rows
|
||||
std::deque<int64_t> audio_commit_rows; // end row of each non-final commit not yet completed
|
||||
|
||||
// realtime conversation, see SERVER_TASK_TYPE_REALTIME
|
||||
std::unique_ptr<server_realtime_slot> rt;
|
||||
|
||||
server_slot_stats stats;
|
||||
|
||||
// accepted tokens per draft position
|
||||
@@ -460,6 +464,7 @@ struct server_slot {
|
||||
audio_wait = false;
|
||||
audio_commit_rows.clear();
|
||||
inp_embd.clear();
|
||||
rt.reset();
|
||||
}
|
||||
|
||||
void init_sampler() const {
|
||||
@@ -660,7 +665,7 @@ struct server_slot {
|
||||
|
||||
// do not keep context of the child slots - the parent's context is enough
|
||||
// nor of the stateless slots - no task ever reuses it
|
||||
if (task->is_child() || task->is_stateless() || audio_stream) {
|
||||
if (task->is_child() || task->is_stateless() || audio_stream || task->type == SERVER_TASK_TYPE_REALTIME) {
|
||||
prompt_clear();
|
||||
}
|
||||
|
||||
@@ -1023,6 +1028,9 @@ public:
|
||||
mtmd_helper_init_opt init_opt = mtmd_helper_init_opt_default();
|
||||
const llama_vocab * vocab = nullptr;
|
||||
|
||||
// prompt of the "asr" chat template, for live ASR
|
||||
llama_tokens asr_prompt;
|
||||
|
||||
bool has_cap_tts() const {
|
||||
return mtmd_cap.audio_gen;
|
||||
}
|
||||
@@ -1705,6 +1713,13 @@ private:
|
||||
return false;
|
||||
}
|
||||
|
||||
try {
|
||||
asr_prompt = server_chat_get_asr_prompt(model_tgt);
|
||||
} catch (const std::exception & e) {
|
||||
SRV_ERR("%s: \"asr\" chat template error: %s\n", __func__, e.what());
|
||||
return false;
|
||||
}
|
||||
|
||||
// IMPORTANT: chat_params is reused across sleeping / resuming states,
|
||||
// never store llama_context/llama_model pointers in chat_params,
|
||||
// as they may be invalidated after sleeping
|
||||
@@ -2020,6 +2035,23 @@ private:
|
||||
}
|
||||
}
|
||||
|
||||
// realtime transcription: the rows come from the audio stream, the helper may bias some tokens
|
||||
if (task.type == SERVER_TASK_TYPE_TRANSCRIBE_RT) {
|
||||
auto sparams = mtmd_audio_stream_params_default();
|
||||
sparams.prompt = asr_prompt.data();
|
||||
sparams.n_prompt = asr_prompt.size();
|
||||
slot.audio_stream.reset(mtmd_audio_stream_init(mctx, sparams));
|
||||
if (!slot.audio_stream) {
|
||||
send_error(task, "failed to initialize the audio stream", ERROR_TYPE_SERVER);
|
||||
return false;
|
||||
}
|
||||
if (!asr_prompt.empty()) {
|
||||
for (const auto & b : mtmd_helper::audio_stream_logit_bias(mctx, model_tgt)) {
|
||||
task.params.sampling.logit_bias.push_back(b);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// initialize samplers
|
||||
if (task.need_sampling()) {
|
||||
try {
|
||||
@@ -2075,12 +2107,6 @@ private:
|
||||
|
||||
// no prompt, the rows come from the audio stream, which audio controls reach from now on
|
||||
if (slot.task->type == SERVER_TASK_TYPE_TRANSCRIBE_RT) {
|
||||
slot.audio_stream.reset(mtmd_audio_stream_init(mctx, mtmd_audio_stream_params_default()));
|
||||
if (!slot.audio_stream) {
|
||||
send_error(slot, "failed to initialize the audio stream", ERROR_TYPE_SERVER);
|
||||
slot.release();
|
||||
return false;
|
||||
}
|
||||
slot.prompt_clear();
|
||||
slot.init_sampler();
|
||||
slot.sampled = LLAMA_TOKEN_NULL;
|
||||
@@ -2090,6 +2116,17 @@ private:
|
||||
send_partial_response(slot, {}, false, true);
|
||||
}
|
||||
|
||||
// no prompt either, the session appends text and audio and asks for responses with its controls
|
||||
if (slot.task->type == SERVER_TASK_TYPE_REALTIME) {
|
||||
slot.rt = std::make_unique<server_realtime_slot>(ctx_tgt, mctx, slot.id, slot.n_ctx, slot.task->params.sampling,
|
||||
queue_results, slot.task->id, slot.task->index);
|
||||
slot.prompt_clear();
|
||||
slot.stats.update_prompt_start();
|
||||
slot.state = SLOT_STATE_GENERATING;
|
||||
slot.audio_wait = true;
|
||||
send_partial_response(slot, {}, false, true);
|
||||
}
|
||||
|
||||
// reset server kill-switch counter
|
||||
n_empty_consecutive = 0;
|
||||
|
||||
@@ -2972,6 +3009,7 @@ private:
|
||||
case SERVER_TASK_TYPE_COMPLETION:
|
||||
case SERVER_TASK_TYPE_INFILL:
|
||||
case SERVER_TASK_TYPE_TRANSCRIBE_RT:
|
||||
case SERVER_TASK_TYPE_REALTIME:
|
||||
case SERVER_TASK_TYPE_EMBEDDING:
|
||||
case SERVER_TASK_TYPE_RERANK:
|
||||
case SERVER_TASK_TYPE_TTS:
|
||||
@@ -3084,6 +3122,20 @@ private:
|
||||
// act on the live slot mid generation, never defer
|
||||
common_sampler_reasoning_budget_force(slot->smpl.get());
|
||||
res->success = true;
|
||||
} else if (slot->rt) {
|
||||
// a cancel acts now, everything else runs after what is already queued
|
||||
if (task.params.control_action == SERVER_CONTROL_ACTION_RESPONSE_CANCEL) {
|
||||
slot->rt->cancel();
|
||||
} else {
|
||||
slot->rt->push({
|
||||
task.params.control_action,
|
||||
std::move(task.params.control_text),
|
||||
std::move(task.params.control_audio),
|
||||
task.params.control_response,
|
||||
});
|
||||
}
|
||||
slot->audio_wait = false;
|
||||
res->success = true;
|
||||
} else if (task.params.control_action == SERVER_CONTROL_ACTION_AUDIO_APPEND
|
||||
|| task.params.control_action == SERVER_CONTROL_ACTION_AUDIO_COMMIT
|
||||
|| task.params.control_action == SERVER_CONTROL_ACTION_AUDIO_END) {
|
||||
@@ -3449,6 +3501,18 @@ private:
|
||||
}
|
||||
|
||||
if (n > 0) {
|
||||
// new sequence (e.g. live ASR restart): drop the old one, the text goes on in a new line
|
||||
if (mtmd_audio_stream_is_seq_start(slot.audio_stream.get())) {
|
||||
slot.prompt_clear();
|
||||
if (slot.n_sent_text > 0) {
|
||||
completion_token_output sep;
|
||||
sep.tok = LLAMA_TOKEN_NULL;
|
||||
sep.text_to_send = "\n";
|
||||
slot.generated.append(common_chat_input(sep.text_to_send));
|
||||
slot.n_sent_text += sep.text_to_send.size();
|
||||
send_partial_response(slot, sep, false);
|
||||
}
|
||||
}
|
||||
const float * rows = mtmd_audio_stream_get_output_embd(slot.audio_stream.get());
|
||||
slot.inp_embd.assign(rows, rows + (size_t) n * batch.n_embd);
|
||||
slot.audio_wait = false;
|
||||
@@ -3534,6 +3598,14 @@ private:
|
||||
abort_all_slots("process_tts_slots() failed: " + std::string(e.what()));
|
||||
}
|
||||
|
||||
// so do the realtime conversations
|
||||
try {
|
||||
process_realtime_slots();
|
||||
} catch (const std::exception & e) {
|
||||
SRV_ERR("process_realtime_slots() failed: %s\n", e.what());
|
||||
abort_all_slots("process_realtime_slots() failed: " + std::string(e.what()));
|
||||
}
|
||||
|
||||
GGML_ASSERT(batch.slot_batched || batch.size() == 0);
|
||||
|
||||
if (batch.slot_batched) {
|
||||
@@ -3596,6 +3668,21 @@ private:
|
||||
}
|
||||
}
|
||||
|
||||
// realtime conversations (SERVER_TASK_TYPE_REALTIME) drive their own decode loop, see server_realtime_slot
|
||||
void process_realtime_slots() {
|
||||
iterate(slots, [&](server_slot & slot) {
|
||||
if (!slot.is_processing() || !slot.rt) {
|
||||
return;
|
||||
}
|
||||
if (!slot.rt->update()) {
|
||||
send_error(slot, slot.rt->error, slot.rt->error_type);
|
||||
slot.release();
|
||||
return;
|
||||
}
|
||||
slot.audio_wait = slot.rt->is_idle();
|
||||
});
|
||||
}
|
||||
|
||||
void process_tts_slots() {
|
||||
iterate(slots, [&](server_slot & slot) {
|
||||
if (!slot.is_processing() || slot.task->type != SERVER_TASK_TYPE_TTS) {
|
||||
@@ -3675,8 +3762,8 @@ private:
|
||||
// apply context-shift if needed
|
||||
// TODO: simplify and improve
|
||||
iterate(slots, [&](server_slot & slot) {
|
||||
if (slot.task && slot.task->type == SERVER_TASK_TYPE_TTS) {
|
||||
// TTS slots drive their own decode loop in process_tts_slots(), never enter the shared batch
|
||||
if (slot.task && (slot.task->type == SERVER_TASK_TYPE_TTS || slot.task->type == SERVER_TASK_TYPE_REALTIME)) {
|
||||
// TTS and realtime conversation slots drive their own decode loop, never enter the shared batch
|
||||
return;
|
||||
}
|
||||
if (slot.state == SLOT_STATE_GENERATING && slot.prompt.n_tokens() + 1 >= slot.n_ctx) {
|
||||
@@ -3751,7 +3838,7 @@ private:
|
||||
|
||||
// determine which slots are generating and drafting
|
||||
iterate(slots, [&](server_slot & slot) {
|
||||
if (slot.state != SLOT_STATE_GENERATING || slot.task->type == SERVER_TASK_TYPE_TTS) {
|
||||
if (slot.state != SLOT_STATE_GENERATING || slot.task->type == SERVER_TASK_TYPE_TTS || slot.task->type == SERVER_TASK_TYPE_REALTIME) {
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -3897,7 +3984,7 @@ private:
|
||||
return; // batch is full, skip remaining slots
|
||||
}
|
||||
|
||||
if (!slot.is_processing() || slot.task->type == SERVER_TASK_TYPE_TTS) {
|
||||
if (!slot.is_processing() || slot.task->type == SERVER_TASK_TYPE_TTS || slot.task->type == SERVER_TASK_TYPE_REALTIME) {
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -5150,8 +5237,7 @@ server_context_meta server_context::get_meta() const {
|
||||
/* has_inp_video */ impl->chat_params.allow_video,
|
||||
/* has_cap_chat */ impl->has_cap_chat(),
|
||||
/* has_cap_tts */ impl->has_cap_tts(),
|
||||
// only realtime transcription models (no chat) stream without a prompt
|
||||
/* has_audio_stream */ impl->mtmd_cap.audio_stream && !impl->mtmd_cap.chat,
|
||||
/* has_audio_stream */ impl->mtmd_cap.audio_stream,
|
||||
/* json_ui_settings */ impl->json_ui_settings,
|
||||
/* slot_n_ctx */ impl->n_ctx_slot(),
|
||||
/* pooling_type */ llama_pooling_type(impl->ctx_tgt),
|
||||
@@ -6025,7 +6111,7 @@ void server_routes::init_routes() {
|
||||
// realtime transcription over websocket, modeled after the OpenAI realtime API
|
||||
// audio is PCM16 mono at the model sample rate, sent as base64 in input_audio_buffer.append events
|
||||
// input_audio_buffer.commit with "final": true ends the input, closing the socket also ends it
|
||||
this->ws_realtime = [this](const server_http_req &, server_http_context::ws_conn & conn) {
|
||||
this->ws_realtime = [this](const server_http_req & req, server_http_context::ws_conn & conn) {
|
||||
auto send_event = [&conn](json ev) {
|
||||
ev["event_id"] = "event_" + random_string();
|
||||
conn.send(safe_json_to_str(ev));
|
||||
@@ -6042,6 +6128,14 @@ void server_routes::init_routes() {
|
||||
});
|
||||
};
|
||||
|
||||
// models that answer after the audio hold a conversation, see server-realtime.h
|
||||
// live transcription: Voxtral Realtime (no chat), or an "asr" model in a session opened with ?intent=transcription
|
||||
const bool live_asr = !ctx_server.asr_prompt.empty() && req.get_param("intent") == "transcription";
|
||||
if (ctx_server.mtmd_cap.audio_stream && ctx_server.mtmd_cap.chat && !live_asr) {
|
||||
server_realtime_conversation({queue_tasks, queue_results, *meta, params.sampling, ctx_server.mctx, ctx_server.mtmd_cap, ctx_server.vocab}, conn);
|
||||
return;
|
||||
}
|
||||
|
||||
if (!meta->has_audio_stream) {
|
||||
send_error_event("The current model does not support realtime transcription.", "invalid_request_error");
|
||||
conn.close();
|
||||
|
||||
Reference in new issue
Block a user