Lumpiasty/llama.cpp
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

mtmd: support Qwen3-TTS (note: breaking change to llama-tts binary) (#26254)

Browse Source 
* convert text model

* main model load ok

* convert encoder ok

* speaker encoder loading ok

* speaker enc graph

* adapt vocab for backbone (with some tricks)

* add suppress_tokens

* poc new mtmd gen api

* convert code_predictor to gguf

* load gen_code model ok

* add clip_encode

* wire up

* code gen cgraph init version

Co-authored-by: Pascal <admin@serveurperso.com>

* code2wav convert to gguf

* code2wav graph ok

* wire up in/out

* (wip) subgraph

* wire up

* wip, correct code2wav

* demo (to be removed)

* code2wav preserve kv between calls

* demo voice clone

* llama: add llama_model_get_tok_embd

* mtmd_helper_gen_audio API

* fix clamp cold prefix

Co-authored-by: Pascal <admin@serveurperso.com>

* fuse snake op

Co-authored-by: Pascal <admin@serveurperso.com>

* demo: use proper sampling

* update dev docs

* polymorphism helper

* revamp llama-tts binary

* update docs

* fix compile

* fix lint

* nits

* add guide + docs

* more timings info

* clean up code comments

* security fixes

* update docs

* use ggml_build_forward_select, clean up comments

* fix ci

* use ISO 639-1 language code

* rename CODE2WAV --> GEN_WAV, update docs

* clean up

* clean up tts.cpp

* add seq_id

* add step_prompt()

* mtmd_helper_model_can_chat

* clean up comments

---------

Co-authored-by: Pascal <admin@serveurperso.com>
This commit is contained in:
Xuan-Son Nguyen
2026-08-04 17:26:15 +02:00
committed by GitHub
co-authored by Pascal
parent 1c3c9674de
commit 0713275082
42 changed files with 3808 additions and 1895 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tools/mtmd/clip-impl.h
+49
View File
@@ -32,6 +32,7 @@
#define KEY_PROJ_TYPE "clip.projector_type"
#define KEY_HAS_AUDIO_ENC "clip.has_audio_encoder"
#define KEY_HAS_VISION_ENC "clip.has_vision_encoder"
#define KEY_HAS_GEN_AUDIO_ENC "clip.has_gen_audio_encoder"
#define KEY_USE_GELU "clip.use_gelu"
#define KEY_USE_SILU "clip.use_silu"
@@ -89,6 +90,8 @@
#define KEY_A_ATTN_WINDOW_SIZE "clip.audio.window_size" // mimo-audio-tokenizer: sliding-window radius
#define KEY_A_LOCAL_BLOCK_COUNT "clip.audio.local_block_count" // mimo-v2.5: input_local_transformer layer count
#define KEY_A_LOCAL_GROUP_SIZE "clip.audio.local_group_size" // mimo-v2.5: input_local_transformer grouping size
// audio generation (gen-audio)-specific
#define KEY_GEN_AUDIO_PROJ_TYPE "clip.gen.audio.projector_type" // for models with mixed modalities
#define KEY_AUDIO_SUBSAMPLING_FACTOR "clip.audio.subsampling_factor"
//
@@ -201,6 +204,48 @@
#define TN_MM_A_LOCAL_LN2 "mm.a.local_blk.%d.ln2.%s"
#define TN_MM_A_LOCAL_NORM "mm.a.local_norm.%s"
// qwen3tts speaker encoder (ECAPA-TDNN)
#define TN_A_SE_CONV1 "a.blk.%d.se_conv1.%s"
#define TN_A_SE_CONV2 "a.blk.%d.se_conv2.%s"
#define TN_A_CONV_RES2 "a.blk.%d.res2.%d.%s"
#define TN_A_ASP_ATTN "a.asp_attn.%s"
#define TN_A_ASP_TDNN "a.asp_tdnn.%s"
// qwen3tts code_predictor
#define TN_A_GEN_CODE_PROJ_IN "a.gen.code.proj_in.%s"
#define TN_A_GEN_CODE_EMBD "a.gen.code.embd.%s"
#define TN_A_GEN_CODE_HEAD "a.gen.code.head.%s"
#define TN_A_GEN_CODE_OUT_EMBD "a.gen.code.out_embd.%s"
#define TN_A_GEN_CODE_NORM "a.gen.code.output_norm.%s"
// qwen3tts code2wav (RVQ codes -> raw PCM)
// pre_transformer layers use the generic TN_ATTN_*/TN_FFN_*/TN_LN_*/TN_LS_* macros, prefix "a.gen.wav.tfm"
#define TN_A_GEN_WAV_QUANT_FIRST_IN "a.gen.wav.quant.first.in_proj.%s"
#define TN_A_GEN_WAV_QUANT_FIRST_OUT "a.gen.wav.quant.first.out_proj.%s"
#define TN_A_GEN_WAV_QUANT_FIRST_CB "a.gen.wav.quant.first.codebook.%s"
#define TN_A_GEN_WAV_QUANT_REST_IN "a.gen.wav.quant.rest.in_proj.%s"
#define TN_A_GEN_WAV_QUANT_REST_OUT "a.gen.wav.quant.rest.out_proj.%s"
#define TN_A_GEN_WAV_QUANT_REST_CB "a.gen.wav.quant.rest.codebook.%s"
#define TN_A_GEN_WAV_PRE_CONV "a.gen.wav.pre_conv.%s"
#define TN_A_GEN_WAV_TFM_IN_PROJ "a.gen.wav.tfm.in_proj.%s"
#define TN_A_GEN_WAV_TFM_OUT_PROJ "a.gen.wav.tfm.out_proj.%s"
#define TN_A_GEN_WAV_TFM_OUT_NORM "a.gen.wav.tfm.output_norm.%s"
#define TN_A_GEN_WAV_UP_CONV "a.gen.wav.up.blk.%d.conv.%s"
#define TN_A_GEN_WAV_UP_DWCONV "a.gen.wav.up.blk.%d.dwconv.%s"
#define TN_A_GEN_WAV_UP_NORM "a.gen.wav.up.blk.%d.norm.%s"
#define TN_A_GEN_WAV_UP_PW1 "a.gen.wav.up.blk.%d.pw1.%s"
#define TN_A_GEN_WAV_UP_PW2 "a.gen.wav.up.blk.%d.pw2.%s"
#define TN_A_GEN_WAV_UP_GAMMA "a.gen.wav.up.blk.%d.gamma"
#define TN_A_GEN_WAV_DAC_ENTRY "a.gen.wav.dac.entry.%s"
#define TN_A_GEN_WAV_DAC_SNAKE "a.gen.wav.dac.blk.%d.snake.%s"
#define TN_A_GEN_WAV_DAC_CONV "a.gen.wav.dac.blk.%d.conv.%s"
#define TN_A_GEN_WAV_DAC_RES_ACT1 "a.gen.wav.dac.blk.%d.res.%d.act1.%s"
#define TN_A_GEN_WAV_DAC_RES_CONV1 "a.gen.wav.dac.blk.%d.res.%d.conv1.%s"
#define TN_A_GEN_WAV_DAC_RES_ACT2 "a.gen.wav.dac.blk.%d.res.%d.act2.%s"
#define TN_A_GEN_WAV_DAC_RES_CONV2 "a.gen.wav.dac.blk.%d.res.%d.conv2.%s"
#define TN_A_GEN_WAV_DAC_POST_SNAKE "a.gen.wav.dac.post_snake.%s"
#define TN_A_GEN_WAV_DAC_POST_CONV "a.gen.wav.dac.post_conv.%s"
// cogvlm
#define TN_MM_POST_FC_NORM "mm.post_fc_norm.%s"
#define TN_MM_H_TO_4H "mm.up.%s"
@@ -408,6 +453,8 @@ enum projector_type {
PROJECTOR_TYPE_MINIMAX_M3,
PROJECTOR_TYPE_GRANITE4_VISION,
PROJECTOR_TYPE_MIMO_AUDIO,
PROJECTOR_TYPE_QWEN3TTS_SPKENC,
PROJECTOR_TYPE_QWEN3TTS_GEN,
PROJECTOR_TYPE_UNKNOWN,
};
@@ -465,6 +512,8 @@ static std::map<projector_type, std::string> PROJECTOR_TYPE_NAMES = {
{ PROJECTOR_TYPE_GRANITE4_VISION, "granite4_vision"},
{ PROJECTOR_TYPE_MIMO_AUDIO, "mimo_audio"},
{ PROJECTOR_TYPE_PARAKEET, "parakeet"},
{ PROJECTOR_TYPE_QWEN3TTS_SPKENC, "qwen3tts_spkenc"},
{ PROJECTOR_TYPE_QWEN3TTS_GEN, "qwen3tts_gen"},
};
static projector_type clip_projector_type_from_string(const std::string & str) {