model : add LiquidAI/LFM3-Audio-2B and LiquidAI/LFM3-Omni-2B (#31257)
* model : add LiquidAI/LFM3-Audio-2B and LiquidAI/LFM3-Omni-2B Co-authored-by: Xuan Son Nguyen <son@huggingface.co> Assisted-by: Claude Opus 5.5 * convert : read the LFM3-Audio config.json as released A transformers that knows lfm3_audio rewrites the config into another layout (no lfm/encoder keys). Assisted-by: Claude Opus 5.5 * mtmd : disable flash attention for the LFM3-Audio depthformer Assisted-by: Claude Opus 5.5 * tts : add a voice option for LFM3-Audio and LFM3-Omni The model card offers a female and a male voice; --tts-voice for llama-tts, "voice" for the server /tts. Assisted-by: Claude Opus 5.5 * clean up template handling * convert : detect LFM3-Audio and LFM3-Omni by model_type Assisted-by: Claude Opus 5.5 * move banned tokens --> logits bias --> mtmd_helper * leave mtmd-cli unchanged * no more template_asr_system * left over * llama : keep the LFM3-Audio encoder convs and codec unquantized Assisted-by: Claude Opus 5.5 * common, mtmd : shorten comments Assisted-by: Claude Opus 5.5 --------- Co-authored-by: Xuan Son Nguyen <son@huggingface.co>
This commit is contained in:
1 parent
eb7818c0b8
commit
19ff817fb2
31 files changed
+2727
-166
No files matched your search
@@ -3424,7 +3424,9 @@ private:
|
||||
return true; // rows from a previous step, not decoded yet
|
||||
}
|
||||
|
||||
// every produced row is decoded and its delta queued, the commits ending there are complete
|
||||
const int32_t n = mtmd_audio_stream_step(slot.audio_stream.get(), slot.sampled);
|
||||
|
||||
// complete the commits whose rows have their text sampled, before the new rows: their text goes to the next item
|
||||
const int64_t n_rows = mtmd_audio_stream_get_n_rows(slot.audio_stream.get());
|
||||
while (!slot.audio_commit_rows.empty() && slot.audio_commit_rows.front() <= n_rows) {
|
||||
slot.audio_commit_rows.pop_front();
|
||||
@@ -3440,7 +3442,6 @@ private:
|
||||
queue_results.send(std::move(res));
|
||||
}
|
||||
|
||||
const int32_t n = mtmd_audio_stream_step(slot.audio_stream.get(), slot.sampled);
|
||||
if (n > 0) {
|
||||
const float * rows = mtmd_audio_stream_get_output_embd(slot.audio_stream.get());
|
||||
slot.inp_embd.assign(rows, rows + (size_t) n * batch.n_embd);
|
||||
@@ -5142,7 +5143,8 @@ server_context_meta server_context::get_meta() const {
|
||||
/* has_inp_video */ impl->chat_params.allow_video,
|
||||
/* has_cap_chat */ impl->has_cap_chat(),
|
||||
/* has_cap_tts */ impl->has_cap_tts(),
|
||||
/* has_audio_stream */ impl->mtmd_cap.audio_stream,
|
||||
// only realtime transcription models (no chat) stream without a prompt
|
||||
/* has_audio_stream */ impl->mtmd_cap.audio_stream && !impl->mtmd_cap.chat,
|
||||
/* json_ui_settings */ impl->json_ui_settings,
|
||||
/* slot_n_ctx */ impl->n_ctx_slot(),
|
||||
/* pooling_type */ llama_pooling_type(impl->ctx_tgt),
|
||||
@@ -6367,8 +6369,13 @@ void server_routes::init_routes() {
|
||||
server_task task(SERVER_TASK_TYPE_TTS);
|
||||
server_schema::eval_tts_schema(params, task, body);
|
||||
const bool stream = task.params.stream;
|
||||
task.tts_inp.set_prompt(prompt);
|
||||
task.tts_inp.set_lang(json_value(body, "lang", std::string()));
|
||||
// a model with a "tts" chat template (e.g. LFM3-Audio) speaks the prompt rendered from it
|
||||
const std::string lang = json_value(body, "lang", std::string());
|
||||
const std::string voice = json_value(body, "voice", std::string());
|
||||
const std::string tts_prompt = common_chat_get_tts_prompt(meta->chat_params.tmpls.get(), prompt, lang, voice);
|
||||
task.tts_inp.set_prompt(tts_prompt.empty() ? prompt : tts_prompt);
|
||||
task.tts_inp.set_lang(lang);
|
||||
task.tts_inp.set_voice(voice);
|
||||
task.tts_inp.data.stream = stream;
|
||||
task.tts_inp.data.out_type = response_format == "pcm"
|
||||
? MTMD_HELPER_GEN_AUDIO_OUTTYPE_PCM
|
||||
|
||||
Reference in new issue
Block a user