mtmd: support kyutai/hibiki realtime full-duplex (#32491)

* mtmd: support hibiki realtime duplex

* fix and clean up

* fix long session quality

* fix ci

* tests : add the moshi fusion baseline for Metal

The Metal fusion check found 5 RMS_NORM+MUL fusions in the new moshi
arch and no baseline for them.

---------

Co-authored-by: Pascal <admin@serveurperso.com>
This commit is contained in:
Xuan-Son NguyenandPascal authored and GitHub committed 2026-11-21 14:25:19 +01:00
1 parent c661033349
commit c622f623fc
31 files changed
+1845 -231

No files matched your search

+24
View File
@@ -126,6 +126,8 @@ static llama_model * llama_model_mapping(llm_arch arch, const llama_model_params
return new llama_model_qwen3tts(params);
case LLM_ARCH_POCKETTTS:
return new llama_model_pockettts(params);
case LLM_ARCH_MOSHI:
return new llama_model_moshi(params);
case LLM_ARCH_ZONOS3:
return new llama_model_zonos3(params);
case LLM_ARCH_KOKORO:
@@ -3154,6 +3156,27 @@ llama_memory_i * llama_model::create_memory(const llama_memory_params & params,
/* filter_recr */ std::move(filter_recr),
/* filter_idx */ std::move(filter_idx));
} break;
case LLM_ARCH_MOSHI:
{
// every layer attends to the same sliding window: one cache that masks it, the cells past the window are reused
res = new llama_kv_cache(
*this,
params.no_alloc,
hparams,
params.type_k,
params.type_v,
!cparams.flash_attn,
cparams.offload_kqv,
cparams.kv_unified,
cparams.n_ctx_seq,
cparams.n_seq_max,
hparams.n_swa,
LLAMA_SWA_TYPE_STANDARD,
nullptr,
nullptr,
nullptr,
nullptr);
} break;
case LLM_ARCH_HY_V4:
{
if (hparams.indexer_top_k == 0) {
@@ -3789,6 +3812,7 @@ llama_rope_type llama_model_rope_type(const llama_model * model) {
case LLM_ARCH_DOTS3NOTE:
case LLM_ARCH_NANBEIGE:
case LLM_ARCH_POCKETTTS:
case LLM_ARCH_MOSHI:
case LLM_ARCH_ZONOS3:
case LLM_ARCH_LIMITE:
case LLM_ARCH_TINY_VOCA: