Lumpiasty/llama.cpp
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

mtmd: support Qwen3-TTS (note: breaking change to llama-tts binary) (#26254)

Browse Source 
* convert text model

* main model load ok

* convert encoder ok

* speaker encoder loading ok

* speaker enc graph

* adapt vocab for backbone (with some tricks)

* add suppress_tokens

* poc new mtmd gen api

* convert code_predictor to gguf

* load gen_code model ok

* add clip_encode

* wire up

* code gen cgraph init version

Co-authored-by: Pascal <admin@serveurperso.com>

* code2wav convert to gguf

* code2wav graph ok

* wire up in/out

* (wip) subgraph

* wire up

* wip, correct code2wav

* demo (to be removed)

* code2wav preserve kv between calls

* demo voice clone

* llama: add llama_model_get_tok_embd

* mtmd_helper_gen_audio API

* fix clamp cold prefix

Co-authored-by: Pascal <admin@serveurperso.com>

* fuse snake op

Co-authored-by: Pascal <admin@serveurperso.com>

* demo: use proper sampling

* update dev docs

* polymorphism helper

* revamp llama-tts binary

* update docs

* fix compile

* fix lint

* nits

* add guide + docs

* more timings info

* clean up code comments

* security fixes

* update docs

* use ggml_build_forward_select, clean up comments

* fix ci

* use ISO 639-1 language code

* rename CODE2WAV --> GEN_WAV, update docs

* clean up

* clean up tts.cpp

* add seq_id

* add step_prompt()

* mtmd_helper_model_can_chat

* clean up comments

---------

Co-authored-by: Pascal <admin@serveurperso.com>
This commit is contained in:
Xuan-Son Nguyen
2026-08-04 17:26:15 +02:00
committed by GitHub
co-authored by Pascal
parent 1c3c9674de
commit 0713275082
42 changed files with 3808 additions and 1895 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tools/mtmd/clip.cpp
+416 -39
View File
@@ -17,6 +17,7 @@
#include <cstring>
#include <fstream>
#include <map>
#include <random>
#include <stdexcept>
#include <unordered_set>
#include <vector>
@@ -269,6 +270,29 @@ clip_graph::clip_graph(clip_ctx * ctx, const clip_image_f32 & img) :
gf = ggml_new_graph_custom(ctx0, ctx->max_nodes, false);
}
clip_graph::clip_graph(const clip_graph & parent) :
model(parent.model),
hparams(parent.hparams),
proj_type(parent.proj_type),
img(parent.img),
patch_size(parent.patch_size),
n_patches_x(parent.n_patches_x),
n_patches_y(parent.n_patches_y),
n_patches(parent.n_patches),
n_embd(parent.n_embd),
n_head(parent.n_head),
n_head_kv(parent.n_head_kv),
d_head(parent.d_head),
n_layer(parent.n_layer),
n_mmproj_embd(parent.n_mmproj_embd),
eps(parent.eps),
kq_scale(parent.kq_scale),
flash_attn_type(parent.flash_attn_type) {
// reuse from parent
ctx0 = parent.ctx0;
gf = parent.gf;
}
ggml_tensor * clip_graph::build_mm(ggml_tensor * w, ggml_tensor * x) const {
return ggml_mul_mat(ctx0, w, x);
}
@@ -873,7 +897,8 @@ ggml_tensor * clip_graph::build_patch_merge_permute(ggml_tensor * cur, int scale
return cur;
}
static std::unique_ptr<clip_graph> clip_get_graph_builder(clip_ctx * ctx, const clip_image_f32_batch & imgs) {
static std::unique_ptr<clip_graph> clip_get_graph_builder(clip_ctx * ctx, const clip_image_f32_batch & imgs,
const clip_encode_params * params = nullptr) {
const clip_image_f32 & img = imgs.entries[0];
std::unique_ptr<clip_graph> builder;
@@ -1025,6 +1050,17 @@ static std::unique_ptr<clip_graph> clip_get_graph_builder(clip_ctx * ctx, const
{
builder = std::make_unique<clip_graph_mimo_audio>(ctx, img);
} break;
case PROJECTOR_TYPE_QWEN3TTS_SPKENC:
{
builder = std::make_unique<clip_graph_qwen3tts_spkenc>(ctx, img);
} break;
case PROJECTOR_TYPE_QWEN3TTS_GEN:
{
const auto gen_process = params ? params->gen_process : CLIP_GEN_PROCESS_GEN_CODE;
const int top_k = params ? params->top_k : 50;
const float top_p = params ? params->top_p : 1.0f;
builder = std::make_unique<clip_graph_qwen3tts_gen>(ctx, img, gen_process, top_k, top_p);
} break;
case PROJECTOR_TYPE_YOUTUVL:
{
builder = std::make_unique<clip_graph_youtuvl>(ctx, img);
@@ -1065,8 +1101,9 @@ struct clip_model_loader {
size_t model_size = 0; // in bytes
bool has_vision = false;
bool has_audio = false;
bool has_vision = false;
bool has_audio = false;
bool has_gen_audio = false;
mtmd_progress_callback progress_callback = nullptr;
void * progress_callback_user_data = nullptr;
@@ -1112,8 +1149,9 @@ struct clip_model_loader {
// modalities
{
get_bool(KEY_HAS_VISION_ENC, has_vision, false);
get_bool(KEY_HAS_AUDIO_ENC, has_audio, false);
get_bool(KEY_HAS_VISION_ENC, has_vision, false);
get_bool(KEY_HAS_AUDIO_ENC, has_audio, false);
get_bool(KEY_HAS_GEN_AUDIO_ENC, has_gen_audio, false);
if (has_vision) {
LOG_INF("%s: has vision encoder\n", __func__);
@@ -1121,6 +1159,9 @@ struct clip_model_loader {
if (has_audio) {
LOG_INF("%s: has audio encoder\n", __func__);
}
if (has_gen_audio) {
LOG_INF("%s: has audio generation (gen) encoder\n", __func__);
}
}
// tensors
@@ -1147,6 +1188,8 @@ struct clip_model_loader {
GGML_ASSERT(has_vision);
} else if (modality == CLIP_MODALITY_AUDIO) {
GGML_ASSERT(has_audio);
} else if (modality == CLIP_MODALITY_GEN_AUDIO) {
GGML_ASSERT(has_gen_audio);
}
model.modality = modality;
@@ -1163,6 +1206,8 @@ struct clip_model_loader {
get_string(KEY_VISION_PROJ_TYPE, proj_type, false);
} else if (modality == CLIP_MODALITY_AUDIO) {
get_string(KEY_AUDIO_PROJ_TYPE, proj_type, false);
} else if (modality == CLIP_MODALITY_GEN_AUDIO) {
get_string(KEY_GEN_AUDIO_PROJ_TYPE, proj_type, false);
} else {
GGML_ABORT("unknown modality");
}
@@ -1182,12 +1227,13 @@ struct clip_model_loader {
}
}
const bool is_vision = model.modality == CLIP_MODALITY_VISION;
const bool is_audio = model.modality == CLIP_MODALITY_AUDIO;
const bool is_vision = model.modality == CLIP_MODALITY_VISION;
const bool is_audio = model.modality == CLIP_MODALITY_AUDIO;
const bool is_gen_audio = model.modality == CLIP_MODALITY_GEN_AUDIO;
// other hparams
{
const char * prefix = is_vision ? "vision" : "audio";
const char * prefix = is_vision ? "vision" : (is_audio ? "audio" : "gen.audio");
get_u32(string_format(KEY_N_EMBD, prefix), hparams.n_embd);
get_u32(string_format(KEY_N_HEAD, prefix), hparams.n_head);
get_u32(string_format(KEY_N_EMBD_HEAD, prefix), hparams.n_embd_head, false);
@@ -1198,6 +1244,7 @@ struct clip_model_loader {
// n_head_kv is optional (for GQA), default to n_head
hparams.n_head_kv = hparams.n_head;
get_u32(string_format(KEY_N_HEAD_KV, prefix), hparams.n_head_kv, false);
if (is_vision) {
get_u32(KEY_IMAGE_SIZE, hparams.image_size);
@@ -1226,6 +1273,11 @@ struct clip_model_loader {
hparams.image_size = 0;
hparams.patch_size = 1;
} else if (is_gen_audio) {
// these are unused, but still need to be set to avoid issues
hparams.image_size = 0;
hparams.patch_size = 1;
} else {
GGML_ASSERT(false && "unknown modality");
}
@@ -1647,6 +1699,33 @@ struct clip_model_loader {
"%s: mimo_audio: %s must be > 0\n", __func__, KEY_A_LOCAL_GROUP_SIZE));
}
} break;
case PROJECTOR_TYPE_QWEN3TTS_SPKENC:
{
// ECAPA-TDNN speaker encoder, mel front-end uses the Slaney default (fmin=0, fmax=sr/2)
hparams.audio_sample_rate = 24000;
hparams.audio_n_fft = 1024;
hparams.audio_window_len = 1024;
hparams.audio_hop_len = 256;
} break;
case PROJECTOR_TYPE_QWEN3TTS_GEN:
{
// TODO: hardcoded for now, read from code_predictor_config instead
hparams.rope_theta = 1000000.0f;
// code2wav params
hparams.wav_tfm_n_layer = 8;
hparams.wav_tfm_n_embd = 512;
hparams.wav_tfm_n_ff = 1024;
hparams.wav_tfm_n_head = 16;
hparams.wav_tfm_n_head_kv = 16;
hparams.wav_tfm_eps = 1e-5f;
hparams.wav_tfm_rope_theta = 10000.0f;
hparams.wav_upsample_n_block = 2;
hparams.wav_dac_n_block = 4;
hparams.wav_dac_n_res = 3;
// matches the reference decoder's sliding_window (speech_tokenizer/config.json)
hparams.wav_tfm_swa = 72;
} break;
case PROJECTOR_TYPE_PADDLEOCR:
{
hparams.n_merge = 2;
@@ -1871,7 +1950,9 @@ struct clip_model_loader {
}
// TODO @ngxson : support both audio and video in the future
const char * prefix = model.modality == CLIP_MODALITY_AUDIO ? "a" : "v";
const char * prefix = model.modality == CLIP_MODALITY_AUDIO ? "a"
: model.modality == CLIP_MODALITY_GEN_AUDIO ? "a.gen.code"
: "v";
// get offsets
for (int64_t i = 0; i < gguf_get_n_tensors(ctx_gguf.get()); ++i) {
@@ -1973,7 +2054,8 @@ struct clip_model_loader {
model.position_embeddings = get_tensor(string_format(TN_POS_EMBD, prefix), false);
const bool has_standard_layers = (
model.proj_type != PROJECTOR_TYPE_GEMMA3NV);
model.proj_type != PROJECTOR_TYPE_GEMMA3NV &&
model.proj_type != PROJECTOR_TYPE_QWEN3TTS_SPKENC);
// layers
const int n_layers_to_load = has_standard_layers ? hparams.n_layer : 0;
@@ -2599,6 +2681,144 @@ struct clip_model_loader {
model.mm_1_w = get_tensor(string_format(TN_MM_AUDIO_MLP, 1, "weight"));
model.mm_2_w = get_tensor(string_format(TN_MM_AUDIO_MLP, 2, "weight"));
} break;
case PROJECTOR_TYPE_QWEN3TTS_SPKENC:
{
// stem TDNN (block 0)
model.conv1d_1_w = get_tensor(string_format(TN_CONV1D, 0, "weight"));
model.conv1d_1_b = get_tensor(string_format(TN_CONV1D, 0, "bias"));
// SE-Res2Net blocks (GGUF bid 1..3, one per hparams.n_layer)
model.layers.resize(hparams.n_layer);
for (int il = 0; il < hparams.n_layer; il++) {
auto & layer = model.layers[il];
int bid = il + 1;
layer.conv_pw1_w = get_tensor(string_format(TN_CONV_PW1, prefix, bid, "weight"));
layer.conv_pw1_b = get_tensor(string_format(TN_CONV_PW1, prefix, bid, "bias"));
layer.conv_pw2_w = get_tensor(string_format(TN_CONV_PW2, prefix, bid, "weight"));
layer.conv_pw2_b = get_tensor(string_format(TN_CONV_PW2, prefix, bid, "bias"));
layer.se_conv1_w = get_tensor(string_format(TN_A_SE_CONV1, bid, "weight"));
layer.se_conv1_b = get_tensor(string_format(TN_A_SE_CONV1, bid, "bias"));
layer.se_conv2_w = get_tensor(string_format(TN_A_SE_CONV2, bid, "weight"));
layer.se_conv2_b = get_tensor(string_format(TN_A_SE_CONV2, bid, "bias"));
layer.res2_conv_w.resize(7);
layer.res2_conv_b.resize(7);
for (int xid = 0; xid < 7; xid++) {
layer.res2_conv_w[xid] = get_tensor(string_format(TN_A_CONV_RES2, bid, xid, "weight"));
layer.res2_conv_b[xid] = get_tensor(string_format(TN_A_CONV_RES2, bid, xid, "bias"));
}
}
// multi-layer feature aggregation
model.conv_out_w = get_tensor(string_format(TN_CONV_OUT, "weight"));
model.conv_out_b = get_tensor(string_format(TN_CONV_OUT, "bias"));
// attentive statistics pooling
model.spk_asp_attn_w = get_tensor(string_format(TN_A_ASP_ATTN, "weight"));
model.spk_asp_attn_b = get_tensor(string_format(TN_A_ASP_ATTN, "bias"));
model.spk_asp_tdnn_w = get_tensor(string_format(TN_A_ASP_TDNN, "weight"));
model.spk_asp_tdnn_b = get_tensor(string_format(TN_A_ASP_TDNN, "bias"));
// final speaker embedding projection
model.mm_fc_w = get_tensor(string_format(TN_MM_AUDIO_FC, "weight"));
model.mm_fc_b = get_tensor(string_format(TN_MM_AUDIO_FC, "bias"));
} break;
case PROJECTOR_TYPE_QWEN3TTS_GEN:
{
// code_predictor
model.gen_code_proj_in_w = get_tensor(string_format(TN_A_GEN_CODE_PROJ_IN, "weight"));
model.gen_code_proj_in_b = get_tensor(string_format(TN_A_GEN_CODE_PROJ_IN, "bias"));
model.gen_code_embd_w = get_tensor(string_format(TN_A_GEN_CODE_EMBD, "weight"));
model.gen_code_head_w = get_tensor(string_format(TN_A_GEN_CODE_HEAD, "weight"));
model.gen_code_out_embd_w = get_tensor(string_format(TN_A_GEN_CODE_OUT_EMBD, "weight"));
model.gen_code_norm_w = get_tensor(string_format(TN_A_GEN_CODE_NORM, "weight"));
// code2wav: RVQ codes -> raw PCM, lives in the same ctx as code_predictor
{
auto & c2w = model.c2w;
c2w.quant_first_in_w = get_tensor(string_format(TN_A_GEN_WAV_QUANT_FIRST_IN, "weight"));
c2w.quant_first_out_w = get_tensor(string_format(TN_A_GEN_WAV_QUANT_FIRST_OUT, "weight"));
c2w.quant_first_cb_w = get_tensor(string_format(TN_A_GEN_WAV_QUANT_FIRST_CB, "weight"));
c2w.quant_rest_in_w = get_tensor(string_format(TN_A_GEN_WAV_QUANT_REST_IN, "weight"));
c2w.quant_rest_out_w = get_tensor(string_format(TN_A_GEN_WAV_QUANT_REST_OUT, "weight"));
c2w.quant_rest_cb_w = get_tensor(string_format(TN_A_GEN_WAV_QUANT_REST_CB, "weight"));
c2w.pre_conv_w = get_tensor(string_format(TN_A_GEN_WAV_PRE_CONV, "weight"));
c2w.pre_conv_b = get_tensor(string_format(TN_A_GEN_WAV_PRE_CONV, "bias"));
c2w.tfm_in_proj_w = get_tensor(string_format(TN_A_GEN_WAV_TFM_IN_PROJ, "weight"));
c2w.tfm_in_proj_b = get_tensor(string_format(TN_A_GEN_WAV_TFM_IN_PROJ, "bias"));
c2w.tfm_out_proj_w = get_tensor(string_format(TN_A_GEN_WAV_TFM_OUT_PROJ, "weight"));
c2w.tfm_out_proj_b = get_tensor(string_format(TN_A_GEN_WAV_TFM_OUT_PROJ, "bias"));
c2w.tfm_output_norm_w = get_tensor(string_format(TN_A_GEN_WAV_TFM_OUT_NORM, "weight"));
// loaded manually, the generic model.layers loop is taken by code_predictor
c2w.tfm_layers.resize(hparams.wav_tfm_n_layer);
for (int il = 0; il < hparams.wav_tfm_n_layer; il++) {
auto & layer = c2w.tfm_layers[il];
const char * p = "a.gen.wav.tfm";
layer.q_w = get_tensor(string_format(TN_ATTN_Q, p, il, "weight"));
layer.k_w = get_tensor(string_format(TN_ATTN_K, p, il, "weight"));
layer.v_w = get_tensor(string_format(TN_ATTN_V, p, il, "weight"));
layer.o_w = get_tensor(string_format(TN_ATTN_OUTPUT, p, il, "weight"));
layer.ln_1_w = get_tensor(string_format(TN_LN_1, p, il, "weight"));
layer.ln_2_w = get_tensor(string_format(TN_LN_2, p, il, "weight"));
layer.ls_1_w = get_tensor(string_format(TN_LS_1, p, il, "weight"));
layer.ls_2_w = get_tensor(string_format(TN_LS_2, p, il, "weight"));
layer.ff_gate_w = get_tensor(string_format(TN_FFN_GATE, p, il, "weight"));
layer.ff_up_w = get_tensor(string_format(TN_FFN_UP, p, il, "weight"));
layer.ff_down_w = get_tensor(string_format(TN_FFN_DOWN, p, il, "weight"));
}
// upsample: 2x (causal ConvTranspose1d + ConvNeXt block)
c2w.upsample.resize(hparams.wav_upsample_n_block);
for (int il = 0; il < hparams.wav_upsample_n_block; il++) {
auto & up = c2w.upsample[il];
up.conv_w = get_tensor(string_format(TN_A_GEN_WAV_UP_CONV, il, "weight"));
up.conv_b = get_tensor(string_format(TN_A_GEN_WAV_UP_CONV, il, "bias"));
up.dwconv_w = get_tensor(string_format(TN_A_GEN_WAV_UP_DWCONV, il, "weight"));
up.dwconv_b = get_tensor(string_format(TN_A_GEN_WAV_UP_DWCONV, il, "bias"));
up.norm_w = get_tensor(string_format(TN_A_GEN_WAV_UP_NORM, il, "weight"));
up.norm_b = get_tensor(string_format(TN_A_GEN_WAV_UP_NORM, il, "bias"));
up.pw1_w = get_tensor(string_format(TN_A_GEN_WAV_UP_PW1, il, "weight"));
up.pw1_b = get_tensor(string_format(TN_A_GEN_WAV_UP_PW1, il, "bias"));
up.pw2_w = get_tensor(string_format(TN_A_GEN_WAV_UP_PW2, il, "weight"));
up.pw2_b = get_tensor(string_format(TN_A_GEN_WAV_UP_PW2, il, "bias"));
up.gamma = get_tensor(string_format(TN_A_GEN_WAV_UP_GAMMA, il));
}
// DAC decoder: conv_pre + n upsample blocks (each with n_res residual units) + conv_post
c2w.dac_entry_w = get_tensor(string_format(TN_A_GEN_WAV_DAC_ENTRY, "weight"));
c2w.dac_entry_b = get_tensor(string_format(TN_A_GEN_WAV_DAC_ENTRY, "bias"));
c2w.dac.resize(hparams.wav_dac_n_block);
for (int il = 0; il < hparams.wav_dac_n_block; il++) {
auto & blk = c2w.dac[il];
blk.snake_alpha = get_tensor(string_format(TN_A_GEN_WAV_DAC_SNAKE, il, "alpha"));
blk.snake_beta = get_tensor(string_format(TN_A_GEN_WAV_DAC_SNAKE, il, "beta"));
blk.conv_w = get_tensor(string_format(TN_A_GEN_WAV_DAC_CONV, il, "weight"));
blk.conv_b = get_tensor(string_format(TN_A_GEN_WAV_DAC_CONV, il, "bias"));
blk.res.resize(hparams.wav_dac_n_res);
for (int ir = 0; ir < hparams.wav_dac_n_res; ir++) {
auto & res = blk.res[ir];
res.act1_alpha = get_tensor(string_format(TN_A_GEN_WAV_DAC_RES_ACT1, il, ir, "alpha"));
res.act1_beta = get_tensor(string_format(TN_A_GEN_WAV_DAC_RES_ACT1, il, ir, "beta"));
res.conv1_w = get_tensor(string_format(TN_A_GEN_WAV_DAC_RES_CONV1, il, ir, "weight"));
res.conv1_b = get_tensor(string_format(TN_A_GEN_WAV_DAC_RES_CONV1, il, ir, "bias"));
res.act2_alpha = get_tensor(string_format(TN_A_GEN_WAV_DAC_RES_ACT2, il, ir, "alpha"));
res.act2_beta = get_tensor(string_format(TN_A_GEN_WAV_DAC_RES_ACT2, il, ir, "beta"));
res.conv2_w = get_tensor(string_format(TN_A_GEN_WAV_DAC_RES_CONV2, il, ir, "weight"));
res.conv2_b = get_tensor(string_format(TN_A_GEN_WAV_DAC_RES_CONV2, il, ir, "bias"));
}
}
c2w.dac_post_snake_alpha = get_tensor(string_format(TN_A_GEN_WAV_DAC_POST_SNAKE, "alpha"));
c2w.dac_post_snake_beta = get_tensor(string_format(TN_A_GEN_WAV_DAC_POST_SNAKE, "beta"));
c2w.dac_post_conv_w = get_tensor(string_format(TN_A_GEN_WAV_DAC_POST_CONV, "weight"));
c2w.dac_post_conv_b = get_tensor(string_format(TN_A_GEN_WAV_DAC_POST_CONV, "bias"));
}
} break;
case PROJECTOR_TYPE_VOXTRAL:
{
model.conv1d_1_w = get_tensor(string_format(TN_CONV1D, 1, "weight"));
@@ -3427,6 +3647,7 @@ struct clip_model_loader {
struct clip_init_result clip_init(const char * fname, struct clip_context_params ctx_params) {
clip_ctx * ctx_vision = nullptr;
clip_ctx * ctx_audio = nullptr;
clip_ctx * ctx_gen_audio = nullptr;
try {
clip_model_loader loader(fname,
@@ -3459,16 +3680,25 @@ struct clip_init_result clip_init(const char * fname, struct clip_context_params
}
}
if (loader.has_gen_audio) {
ctx_gen_audio = new clip_ctx(ctx_params);
loader.load_hparams(ctx_gen_audio->model, CLIP_MODALITY_GEN_AUDIO);
loader.load_tensors(*ctx_gen_audio);
// TODO: fix warmup
ctx_gen_audio->buf_compute_meta.resize(ctx_gen_audio->max_nodes * ggml_tensor_overhead() + ggml_graph_overhead());
}
} catch (const std::exception & e) {
LOG_ERR("%s: failed to load model '%s': %s\n", __func__, fname, e.what());
delete ctx_vision;
delete ctx_audio;
delete ctx_gen_audio;
return {nullptr, nullptr};
return {nullptr, nullptr, nullptr};
}
return {ctx_vision, ctx_audio};
return {ctx_vision, ctx_audio, ctx_gen_audio};
}
struct clip_cap clip_get_cap(const char * fname) {
@@ -3784,6 +4014,16 @@ int clip_n_output_tokens(const clip_ctx * ctx, const clip_image_f32 * img) {
const int ds = ctx->model.hparams.audio_proj_downsample_rate;
n_patches = ((img->nx() + ws - 1) / ws) * (ws / ds);
} break;
case PROJECTOR_TYPE_QWEN3TTS_SPKENC:
{
// pooling gives one speaker embedding, whatever the clip length is
n_patches = 1;
} break;
case PROJECTOR_TYPE_QWEN3TTS_GEN:
{
// one hidden-state vector fed back to the talker per call
n_patches = 1;
} break;
case PROJECTOR_TYPE_GRANITE4_VISION:
{
// Per-tile output token count: each projector block outputs
@@ -3817,7 +4057,16 @@ bool clip_image_encode(struct clip_ctx * ctx, int n_threads, const clip_image_f3
}
bool clip_image_batch_encode(clip_ctx * ctx, int n_threads, const clip_image_f32_batch * imgs_c_ptr, std::vector<float> & out_batch_embd) {
const clip_image_f32_batch & imgs = *imgs_c_ptr;
clip_encode_params params;
params.imgs = imgs_c_ptr;
params.n_threads = n_threads;
params.out_embd = &out_batch_embd;
return clip_encode(ctx, &params);
}
bool clip_encode(struct clip_ctx * ctx, struct clip_encode_params * params) {
const clip_image_f32_batch & imgs = *params->imgs;
int n_batch_cur = imgs.entries.size();
// [QWEN_VIDEO] for video models, the batch dimension is used as temporal dimension for merged frames
@@ -3828,12 +4077,12 @@ bool clip_image_batch_encode(clip_ctx * ctx, int n_threads, const clip_image_f32
// if buffers are not allocated, we need to do a warmup run to allocate them
if (!ctx->is_allocated) {
clip_model_loader::warmup(*ctx, *imgs_c_ptr);
clip_model_loader::warmup(*ctx, *params->imgs);
}
// build the inference graph
ggml_backend_sched_reset(ctx->sched.get());
ggml_cgraph * gf = clip_get_graph_builder(ctx, imgs)->build();
ggml_cgraph * gf = clip_get_graph_builder(ctx, imgs, params)->build();
ggml_backend_sched_alloc_graph(ctx->sched.get(), gf);
// set inputs
@@ -3918,8 +4167,8 @@ bool clip_image_batch_encode(clip_ctx * ctx, int n_threads, const clip_image_f32
}
set_input_f32("inp_raw", inp_raw);
} else {
// audio input
} else if (!(ctx->proj_type() == PROJECTOR_TYPE_QWEN3TTS_GEN && params->gen_process == CLIP_GEN_PROCESS_GEN_WAV)) {
// audio input, code2wav is not here: its only input is "inp_codes", set in the switch below
GGML_ASSERT(imgs.entries.size() == 1);
const auto & mel_inp = imgs.entries[0];
@@ -4475,9 +4724,77 @@ bool clip_image_batch_encode(clip_ctx * ctx, int n_threads, const clip_image_f32
case PROJECTOR_TYPE_COGVLM:
case PROJECTOR_TYPE_YASA2:
case PROJECTOR_TYPE_GEMMA4UA:
case PROJECTOR_TYPE_QWEN3TTS_SPKENC:
{
// do nothing
} break;
case PROJECTOR_TYPE_QWEN3TTS_GEN:
{
if (params->gen_process == CLIP_GEN_PROCESS_GEN_WAV) {
GGML_ASSERT(params->codes != nullptr);
// frame-major input to group-major, rear-padded with code 0 up to one window
const int64_t n_codes = model.gen_code_head_w->ne[2] + 1;
const int64_t n_frames_w = hparams.wav_tfm_swa;
const int64_t n_frames = (int64_t) params->codes->size() / n_codes;
GGML_ASSERT(n_frames > 0 && n_frames <= n_frames_w);
// codes are used as ggml_get_rows indices, so check them against the codebook vocab
const int64_t vocab_first = model.c2w.quant_first_cb_w->ne[1];
const int64_t vocab_rest = model.c2w.quant_rest_cb_w->ne[1];
for (int64_t f = 0; f < n_frames; f++) {
for (int64_t g = 0; g < n_codes; g++) {
const int32_t c = (*params->codes)[f * n_codes + g];
const int64_t vocab = (g == 0) ? vocab_first : vocab_rest;
if (c < 0 || (int64_t) c >= vocab) {
LOG_ERR("%s: code out of range (frame %lld, group %lld, code %d, vocab %lld)\n",
__func__, (long long) f, (long long) g, c, (long long) vocab);
return false;
}
}
}
std::vector<int32_t> codes(n_frames_w * n_codes, 0);
for (int64_t f = 0; f < n_frames; f++) {
for (int64_t g = 0; g < n_codes; g++) {
codes[g * n_frames_w + f] = (*params->codes)[f * n_codes + g];
}
}
set_input_i32("inp_codes", codes);
// upload the state from the previous call, or zero-fill on a cold start
size_t offset = 0;
for (const auto & slot : list_c2w_state_slots(hparams, model)) {
ggml_tensor * t = get_inp_tensor(("state_in_" + slot.name).c_str());
const size_t nb = ggml_nbytes(t);
if (params->state_in && params->state_in->size() >= offset + nb) {
ggml_backend_tensor_set(t, params->state_in->data() + offset, 0, nb);
} else {
std::vector<uint8_t> zeros(nb, 0);
ggml_backend_tensor_set(t, zeros.data(), 0, nb);
}
offset += nb;
}
} else {
// code0 indexes gen_code_out_embd_w via ggml_get_rows; bound it
const int64_t vocab0 = model.gen_code_out_embd_w->ne[1];
if (params->code0 < 0 || (int64_t) params->code0 >= vocab0) {
LOG_ERR("%s: code0 out of range (%d, vocab %lld)\n", __func__, params->code0, (long long) vocab0);
return false;
}
std::vector<int32_t> code0 = { params->code0 };
set_input_i32("inp_code0", code0);
// one uniform(0,1) draw per codebook, used by do_sampling()
static std::mt19937 rng{ std::random_device{}() };
std::uniform_real_distribution<float> dist(0.0f, 1.0f);
const int64_t n_acoustic = model.gen_code_head_w->ne[2];
for (int64_t g = 0; g < n_acoustic; g++) {
std::vector<float> r = { dist(rng) };
set_input_f32(("inp_rand_" + std::to_string(g)).c_str(), r);
}
}
} break;
case PROJECTOR_TYPE_HUNYUANVL:
{
// Compute the HunyuanVL 2D position embedding on CPU (with the
@@ -4883,7 +5200,7 @@ bool clip_image_batch_encode(clip_ctx * ctx, int n_threads, const clip_image_f32
if (reg) {
auto ggml_backend_set_n_threads_fn = (ggml_backend_set_n_threads_t) ggml_backend_reg_get_proc_address(reg, "ggml_backend_set_n_threads");
if (ggml_backend_set_n_threads_fn) {
ggml_backend_set_n_threads_fn(ctx->backend_cpu, n_threads);
ggml_backend_set_n_threads_fn(ctx->backend_cpu, params->n_threads);
}
}
@@ -4893,34 +5210,90 @@ bool clip_image_batch_encode(clip_ctx * ctx, int n_threads, const clip_image_f32
return false;
}
// the last node is the embedding tensor
ggml_tensor * embeddings = ggml_graph_node(gf, -1);
// the last node is the embedding tensor, code2wav has no out_embd
ggml_tensor * embeddings = params->out_embd ? ggml_graph_node(gf, -1) : nullptr;
// sanity check (assuming that all images in batch have the same number of tokens, so we only check the first one)
const int n_tokens_out = embeddings->ne[1];
const int expected_n_tokens_out = clip_n_output_tokens(ctx, &imgs.entries[0]);
if (n_tokens_out != expected_n_tokens_out) {
LOG_ERR("%s: expected output %d tokens, got %d\n", __func__, expected_n_tokens_out, n_tokens_out);
GGML_ABORT("Invalid number of output tokens");
}
if (embeddings != nullptr) {
// sanity check (assuming that all images in batch have the same number of tokens, so we only check the first one)
const int n_tokens_out = embeddings->ne[1];
const int expected_n_tokens_out = clip_n_output_tokens(ctx, &imgs.entries[0]);
if (n_tokens_out != expected_n_tokens_out) {
LOG_ERR("%s: expected output %d tokens, got %d\n", __func__, expected_n_tokens_out, n_tokens_out);
GGML_ABORT("Invalid number of output tokens");
}
LOG_DBG("%s: output embedding shape [%d, %d, %d]\n", __func__,
(int)embeddings->ne[0], (int)embeddings->ne[1], (int)embeddings->ne[2]);
LOG_DBG("%s: output embedding shape [%d, %d, %d]\n", __func__,
(int)embeddings->ne[0], (int)embeddings->ne[1], (int)embeddings->ne[2]);
// copy output to user buffer if provided
// if output is empty, skip the copy
if (!out_batch_embd.empty()) {
if (out_batch_embd.size() != (size_t)ggml_nelements(embeddings)) {
LOG_ERR("%s: output buffer has %zu elements but expected %zu\n", __func__, out_batch_embd.size(), (size_t)ggml_nelements(embeddings));
GGML_ABORT("Output buffer size mismatch");
// copy output to user buffer if provided
// if output is empty, skip the copy
auto & out_batch_embd = *params->out_embd;
if (!out_batch_embd.empty()) {
if (out_batch_embd.size() != (size_t)ggml_nelements(embeddings)) {
LOG_ERR("%s: output buffer has %zu elements but expected %zu\n", __func__, out_batch_embd.size(), (size_t)ggml_nelements(embeddings));
GGML_ABORT("Output buffer size mismatch");
}
ggml_backend_tensor_get(embeddings, out_batch_embd.data(), 0, ggml_nbytes(embeddings));
} else {
LOG_WRN("%s: output buffer is empty, skipping copy\n", __func__);
}
ggml_backend_tensor_get(embeddings, out_batch_embd.data(), 0, ggml_nbytes(embeddings));
} else {
LOG_WRN("%s: output buffer is empty, skipping copy\n", __func__);
}
//
// for audio gen models
//
if (params->out_codes != nullptr) {
ggml_tensor * codes = ggml_graph_get_tensor(gf, "out_codes");
if (codes == nullptr) {
GGML_ABORT("out_codes requested but graph has no \"out_codes\" tensor");
}
auto & out_codes = *params->out_codes;
out_codes.resize(ggml_nelements(codes));
ggml_backend_tensor_get(codes, out_codes.data(), 0, ggml_nbytes(codes));
}
if (params->out_audio != nullptr) {
ggml_tensor * audio = ggml_graph_get_tensor(gf, "out_audio");
if (audio == nullptr) {
GGML_ABORT("out_audio requested but graph has no \"out_audio\" tensor");
}
auto & out_audio = *params->out_audio;
out_audio.resize(ggml_nelements(audio));
ggml_backend_tensor_get(audio, out_audio.data(), 0, ggml_nbytes(audio));
// drop the tail audio that comes from the code-0 rear padding
const int64_t n_codes = model.gen_code_head_w->ne[2] + 1;
const int64_t n_frames_w = hparams.wav_tfm_swa;
const int64_t n_frames = (int64_t) params->codes->size() / n_codes;
if (n_frames < n_frames_w) {
const size_t hop = out_audio.size() / n_frames_w;
out_audio.resize((size_t) n_frames * hop);
}
}
if (params->state_out != nullptr) {
auto & state_out = *params->state_out;
size_t total = 0;
for (const auto & slot : list_c2w_state_slots(hparams, model)) {
total += (size_t) (slot.ne0 * slot.ne1) * sizeof(float);
}
state_out.resize(total);
size_t offset = 0;
for (const auto & slot : list_c2w_state_slots(hparams, model)) {
ggml_tensor * t = ggml_graph_get_tensor(gf, ("state_out_" + slot.name).c_str());
if (t == nullptr) {
GGML_ABORT("state_out requested but graph has no \"state_out_%s\" tensor", slot.name.c_str());
}
const size_t nb = ggml_nbytes(t);
ggml_backend_tensor_get(t, state_out.data() + offset, 0, nb);
offset += nb;
}
}
//
// Debug: dump final embeddings if MTMD_DEBUG_EMBEDDINGS is set
if (ctx->debug_output_embeddings) {
//
if (ctx->debug_output_embeddings && embeddings != nullptr) {
const int64_t n_embd = embeddings->ne[0];
const int64_t n_tokens = embeddings->ne[1];
std::vector<float> emb_data(ggml_nelements(embeddings));
@@ -5047,6 +5420,10 @@ int clip_n_mmproj_embd(const struct clip_ctx * ctx) {
return ctx->model.mm_ffn_down_w->ne[1];
case PROJECTOR_TYPE_MIMO_AUDIO:
return ctx->model.mm_2_w->ne[1];
case PROJECTOR_TYPE_QWEN3TTS_SPKENC:
return ctx->model.mm_fc_w->ne[2];
case PROJECTOR_TYPE_QWEN3TTS_GEN:
return ctx->model.gen_code_out_embd_w->ne[0];
case PROJECTOR_TYPE_PARAKEET:
return ctx->model.mm_1_w->ne[1];
default: