mtmd: support Qwen3-TTS (note: breaking change to llama-tts binary) (#26254)
* convert text model * main model load ok * convert encoder ok * speaker encoder loading ok * speaker enc graph * adapt vocab for backbone (with some tricks) * add suppress_tokens * poc new mtmd gen api * convert code_predictor to gguf * load gen_code model ok * add clip_encode * wire up * code gen cgraph init version Co-authored-by: Pascal <admin@serveurperso.com> * code2wav convert to gguf * code2wav graph ok * wire up in/out * (wip) subgraph * wire up * wip, correct code2wav * demo (to be removed) * code2wav preserve kv between calls * demo voice clone * llama: add llama_model_get_tok_embd * mtmd_helper_gen_audio API * fix clamp cold prefix Co-authored-by: Pascal <admin@serveurperso.com> * fuse snake op Co-authored-by: Pascal <admin@serveurperso.com> * demo: use proper sampling * update dev docs * polymorphism helper * revamp llama-tts binary * update docs * fix compile * fix lint * nits * add guide + docs * more timings info * clean up code comments * security fixes * update docs * use ggml_build_forward_select, clean up comments * fix ci * use ISO 639-1 language code * rename CODE2WAV --> GEN_WAV, update docs * clean up * clean up tts.cpp * add seq_id * add step_prompt() * mtmd_helper_model_can_chat * clean up comments --------- Co-authored-by: Pascal <admin@serveurperso.com>
This commit is contained in:
Xuan-Son Nguyen
co-authored by
Pascal
Pascal
parent
1c3c9674de
commit
0713275082
@@ -2,6 +2,11 @@
|
||||
|
||||
#include "../clip-graph.h"
|
||||
|
||||
#include <map>
|
||||
#include <string>
|
||||
#include <utility>
|
||||
#include <vector>
|
||||
|
||||
/*
|
||||
* IMPORTANT: The mtmd module does NOT accept pull requests that are fully or predominantly AI-generated.
|
||||
* We encourage human contributors to ensure the quality and reliability of the codebase.
|
||||
@@ -215,6 +220,111 @@ struct clip_graph_mimo_audio : clip_graph {
|
||||
ggml_cgraph * build() override;
|
||||
};
|
||||
|
||||
struct clip_graph_qwen3tts_spkenc : clip_graph {
|
||||
clip_graph_qwen3tts_spkenc(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {}
|
||||
ggml_cgraph * build() override;
|
||||
|
||||
ggml_tensor * conv1d_same(ggml_tensor * x, ggml_tensor * w, ggml_tensor * b, int dilation) const;
|
||||
ggml_tensor * res2net(ggml_tensor * x, const clip_layer & layer, int dilation, int scale) const;
|
||||
ggml_tensor * se_block(ggml_tensor * x, const clip_layer & layer) const;
|
||||
ggml_tensor * se_res2net_block(ggml_tensor * x, const clip_layer & layer, int dilation, int scale) const;
|
||||
ggml_tensor * attentive_stats_pool(ggml_tensor * x) const;
|
||||
};
|
||||
|
||||
struct clip_graph_qwen3tts_gen : clip_graph {
|
||||
clip_graph_qwen3tts_gen(clip_ctx * ctx, const clip_image_f32 & img, clip_gen_process_type gen_process, int top_k, float top_p)
|
||||
: clip_graph(ctx, img), gen_process(gen_process), top_k(top_k), top_p(top_p) {}
|
||||
ggml_cgraph * build() override;
|
||||
|
||||
// which sub-graph build() constructs, fixed at graph-build time
|
||||
clip_gen_process_type gen_process;
|
||||
|
||||
// sampling params, fixed at graph-build time (GEN_CODE only)
|
||||
int top_k;
|
||||
float top_p;
|
||||
|
||||
//
|
||||
// code_gen: backbone hidden state + sampled code0 -> 16 RVQ codes
|
||||
// MTP-style code predictor, one token per codebook
|
||||
//
|
||||
struct code_gen : clip_graph {
|
||||
code_gen(const clip_graph & parent, int top_k, float top_p)
|
||||
: clip_graph(parent), top_k(top_k), top_p(top_p) {}
|
||||
ggml_cgraph * build() override { GGML_ABORT("call prefill()/step() instead"); }
|
||||
|
||||
int top_k;
|
||||
float top_p;
|
||||
|
||||
ggml_tensor * cache_set(ggml_tensor * cache, int row_idx, ggml_tensor * value) const;
|
||||
ggml_tensor * do_sampling(ggml_tensor * logits, ggml_tensor * inp_rand) const;
|
||||
|
||||
ggml_tensor * const_i32(ggml_tensor * anchor, float value) const;
|
||||
ggml_tensor * causal_mask_row(int64_t n_kv_pad, int pos) const;
|
||||
ggml_tensor * project_in(ggml_tensor * cur) const;
|
||||
|
||||
ggml_tensor * layer_forward(
|
||||
ggml_tensor * cur,
|
||||
const clip_layer & layer,
|
||||
ggml_tensor * inp_pos,
|
||||
ggml_tensor * kq_mask,
|
||||
ggml_tensor *& k_cache_layer,
|
||||
ggml_tensor *& v_cache_layer,
|
||||
int64_t n_kv_pad,
|
||||
int pos,
|
||||
int il) const;
|
||||
|
||||
void prefill(
|
||||
std::vector<ggml_tensor *> & k_cache,
|
||||
std::vector<ggml_tensor *> & v_cache,
|
||||
ggml_tensor *& out_code_cache,
|
||||
ggml_tensor * h_state,
|
||||
ggml_tensor * code0_embd,
|
||||
ggml_tensor * inp_rand) const;
|
||||
|
||||
ggml_tensor * step(
|
||||
std::vector<ggml_tensor *> & k_cache,
|
||||
std::vector<ggml_tensor *> & v_cache,
|
||||
ggml_tensor * out_code_cache,
|
||||
ggml_tensor * inp_rand,
|
||||
int step_idx) const;
|
||||
};
|
||||
|
||||
//
|
||||
// code2wav: RVQ codes -> raw PCM (quantizer + pre_conv + pre_transformer + upsample + DAC).
|
||||
//
|
||||
struct code2wav : clip_graph {
|
||||
code2wav(const clip_graph & parent) : clip_graph(parent) {}
|
||||
ggml_cgraph * build() override { GGML_ABORT("call decode() instead"); }
|
||||
|
||||
// state_in: previous call's persisted state, by slot name (see list_c2w_state_slots())
|
||||
std::map<std::string, ggml_tensor *> state_in;
|
||||
// state_out: this call's state to persist, added to the graph outputs by build()
|
||||
mutable std::vector<std::pair<std::string, ggml_tensor *>> state_out;
|
||||
|
||||
// stateful conv ops: read/update their state via state_in/state_out[state_name]
|
||||
ggml_tensor * causal_conv1d(ggml_tensor * x, ggml_tensor * w, ggml_tensor * b, int dilation, const std::string & state_name) const;
|
||||
ggml_tensor * causal_conv1d_dw(ggml_tensor * x, ggml_tensor * w, ggml_tensor * b, const std::string & state_name) const;
|
||||
ggml_tensor * causal_conv_transpose1d(ggml_tensor * x, ggml_tensor * w, ggml_tensor * b, int stride, const std::string & state_name) const;
|
||||
ggml_tensor * snake(ggml_tensor * x, ggml_tensor * alpha, ggml_tensor * beta) const;
|
||||
|
||||
ggml_tensor * quant_decode(ggml_tensor * inp_codes) const;
|
||||
ggml_tensor * tfm_layer_forward(ggml_tensor * cur, const clip_layer & layer, int il) const;
|
||||
ggml_tensor * convnext_block(ggml_tensor * x, const clip_code2wav::upsample_block & blk, const std::string & state_prefix) const;
|
||||
ggml_tensor * dac_res_unit(ggml_tensor * x, const clip_code2wav::dac_res & res, int dilation, const std::string & state_name) const;
|
||||
|
||||
// inp_codes [1, n_codes] I32 -> this frame's audio samples [n_samples] F32, clamped to [-1, 1]
|
||||
ggml_tensor * decode(ggml_tensor * inp_codes) const;
|
||||
};
|
||||
};
|
||||
|
||||
// one persisted state buffer used by code2wav, see qwen3tts-gen.cpp
|
||||
struct c2w_state_slot {
|
||||
std::string name;
|
||||
int64_t ne0;
|
||||
int64_t ne1;
|
||||
};
|
||||
std::vector<c2w_state_slot> list_c2w_state_slots(const clip_hparams & hparams, const clip_model & model);
|
||||
|
||||
struct clip_graph_kimik25 : clip_graph {
|
||||
clip_graph_kimik25(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {}
|
||||
ggml_cgraph * build() override;
|
||||
|
||||
Reference in New Issue
Block a user