server : add realtime conversation and transcription on /v1/realtime (#31269)
* server : add realtime conversation and transcription on /v1/realtime Co-authored-by: Xuan Son Nguyen <son@huggingface.co> Assisted-by: Claude Opus 5.5 * examples : default the realtime client to the server port 9931 Assisted-by: Claude Opus 5.5 * server : sample realtime responses right after their own decode Slots respond in turns on one context: sampling at the start of the next step read the logits of whichever slot decoded last, so concurrent sessions mixed their outputs. Assisted-by: Claude Opus 5.5 * move more stuff to server-realtime, making it self-contained * examples : fix the type check of the realtime client Assisted-by: Claude Opus 5.5 * examples : use the model card prompts in the realtime client Assisted-by: Claude Opus 5.5 --------- Co-authored-by: Xuan Son Nguyen <son@huggingface.co>
This commit is contained in:
1 parent
d05706b4f3
commit
8ca4060b7e
10 files changed
+1423
-14
No files matched your search
@@ -721,3 +721,22 @@ json convert_transcriptions_to_chatcmpl(
|
||||
|
||||
return chatcmpl_body;
|
||||
}
|
||||
|
||||
llama_tokens server_chat_get_asr_prompt(const llama_model * model) {
|
||||
const char * src = llama_model_chat_template(model, "asr");
|
||||
if (src == nullptr) {
|
||||
return {};
|
||||
}
|
||||
const llama_vocab * vocab = llama_model_get_vocab(model);
|
||||
const llama_token bos = llama_vocab_bos(vocab);
|
||||
const common_chat_template tmpl(src, bos == LLAMA_TOKEN_NULL ? "" : common_token_to_piece(vocab, bos, true), "");
|
||||
jinja::context ctx(tmpl.source());
|
||||
jinja::global_from_json(ctx, json {
|
||||
{"bos_token", tmpl.bos_token()},
|
||||
{"messages", json::array()},
|
||||
{"add_generation_prompt", true},
|
||||
}, false);
|
||||
jinja::runtime runtime(ctx);
|
||||
const std::string prompt = jinja::runtime::gather_string_parts(runtime.execute(tmpl.prog))->as_string().str();
|
||||
return common_tokenize(vocab, prompt, false, true);
|
||||
}
|
||||
Reference in new issue
Block a user