mtmd: support Qwen3-TTS (note: breaking change to llama-tts binary) (#26254)
* convert text model * main model load ok * convert encoder ok * speaker encoder loading ok * speaker enc graph * adapt vocab for backbone (with some tricks) * add suppress_tokens * poc new mtmd gen api * convert code_predictor to gguf * load gen_code model ok * add clip_encode * wire up * code gen cgraph init version Co-authored-by: Pascal <admin@serveurperso.com> * code2wav convert to gguf * code2wav graph ok * wire up in/out * (wip) subgraph * wire up * wip, correct code2wav * demo (to be removed) * code2wav preserve kv between calls * demo voice clone * llama: add llama_model_get_tok_embd * mtmd_helper_gen_audio API * fix clamp cold prefix Co-authored-by: Pascal <admin@serveurperso.com> * fuse snake op Co-authored-by: Pascal <admin@serveurperso.com> * demo: use proper sampling * update dev docs * polymorphism helper * revamp llama-tts binary * update docs * fix compile * fix lint * nits * add guide + docs * more timings info * clean up code comments * security fixes * update docs * use ggml_build_forward_select, clean up comments * fix ci * use ISO 639-1 language code * rename CODE2WAV --> GEN_WAV, update docs * clean up * clean up tts.cpp * add seq_id * add step_prompt() * mtmd_helper_model_can_chat * clean up comments --------- Co-authored-by: Pascal <admin@serveurperso.com>
This commit is contained in:
Xuan-Son Nguyen
co-authored by
Pascal
Pascal
parent
1c3c9674de
commit
0713275082
@@ -262,6 +262,13 @@ struct mtmd_context {
|
||||
struct clip_ctx * ctx_a; // audio
|
||||
std::vector<float> out_embd; // image embedding vector
|
||||
|
||||
// generation context
|
||||
struct clip_ctx * ctx_gen_a; // audio
|
||||
std::vector<int32_t> gen_out_codes; // this frame's 16 sampled codes (GEN_CODE)
|
||||
std::vector<float> gen_out_embd; // next-step hidden state fed back to backbone (GEN_CODE)
|
||||
std::vector<float> gen_out_audio; // decoded PCM samples for the current frame (GEN_WAV)
|
||||
std::vector<uint8_t> gen_out_state; // state to feed into the next GEN_WAV call
|
||||
|
||||
bool print_timings;
|
||||
int n_threads;
|
||||
std::string media_marker;
|
||||
@@ -354,6 +361,7 @@ struct mtmd_context {
|
||||
auto res = clip_init(mmproj_fname, ctx_clip_params);
|
||||
ctx_v = res.ctx_v;
|
||||
ctx_a = res.ctx_a;
|
||||
ctx_gen_a = res.ctx_gen_a;
|
||||
if (!ctx_v && !ctx_a) {
|
||||
throw std::runtime_error(string_format("Failed to load CLIP model from %s\n", mmproj_fname));
|
||||
}
|
||||
@@ -378,6 +386,15 @@ struct mtmd_context {
|
||||
"hint: you may be using wrong mmproj\n",
|
||||
n_embd_text, n_embd_clip));
|
||||
}
|
||||
if (ctx_gen_a) {
|
||||
int n_embd_gen = clip_n_mmproj_embd(ctx_gen_a);
|
||||
if (n_embd_text > 0 && n_embd_text != n_embd_gen) {
|
||||
throw std::runtime_error(string_format(
|
||||
"mismatch between text model (n_embd = %d) and gen-audio mmproj (n_embd = %d)\n"
|
||||
"hint: you may be using wrong mmproj\n",
|
||||
n_embd_text, n_embd_gen));
|
||||
}
|
||||
}
|
||||
if (ctx_v) {
|
||||
init_vision();
|
||||
}
|
||||
@@ -740,6 +757,10 @@ struct mtmd_context {
|
||||
aud_end = "<|mimo_audio_end|>";
|
||||
audio_preproc = std::make_unique<mtmd_audio_preprocessor_mimo_audio>(ctx_a);
|
||||
} break;
|
||||
case PROJECTOR_TYPE_QWEN3TTS_SPKENC:
|
||||
{
|
||||
audio_preproc = std::make_unique<mtmd_audio_preprocessor_qwen3tts_spk>(ctx_a);
|
||||
} break;
|
||||
default:
|
||||
throw std::runtime_error(string_format("%s: unexpected audio projector type %d\n", __func__, proj));
|
||||
}
|
||||
@@ -780,6 +801,7 @@ struct mtmd_context {
|
||||
~mtmd_context() {
|
||||
clip_free(ctx_a);
|
||||
clip_free(ctx_v);
|
||||
clip_free(ctx_gen_a);
|
||||
}
|
||||
|
||||
private:
|
||||
@@ -1553,6 +1575,125 @@ float * mtmd_get_output_embd(mtmd_context * ctx) {
|
||||
return ctx->out_embd.data();
|
||||
}
|
||||
|
||||
//
|
||||
// audio generation
|
||||
//
|
||||
|
||||
mtmd_gen_audio_info mtmd_gen_audio_get_info(const mtmd_context * ctx) {
|
||||
mtmd_gen_audio_info info;
|
||||
if (!ctx->ctx_gen_a) {
|
||||
info.type = MTMD_GEN_AUDIO_TYPE_NONE;
|
||||
return info;
|
||||
}
|
||||
switch (clip_get_projector_type(ctx->ctx_gen_a)) {
|
||||
case PROJECTOR_TYPE_QWEN3TTS_GEN:
|
||||
info.type = MTMD_GEN_AUDIO_TYPE_QWEN3TTS;
|
||||
info.sample_rate = 24000;
|
||||
break;
|
||||
default:
|
||||
info.type = MTMD_GEN_AUDIO_TYPE_NONE;
|
||||
break;
|
||||
}
|
||||
return info;
|
||||
}
|
||||
|
||||
static int32_t mtmd_gen_audio_process_impl(mtmd_context * ctx, const mtmd_gen_inp * inp, mtmd_gen_out * out) {
|
||||
clip_ctx * ctx_clip = ctx->ctx_gen_a;
|
||||
if (!ctx_clip) {
|
||||
LOG_ERR("%s: model does not support audio generation\n", __func__);
|
||||
return 1;
|
||||
}
|
||||
|
||||
if (inp->type == MTMD_GEN_PROCESS_TYPE_GEN_CODE) {
|
||||
const size_t n_embd = (size_t) clip_n_mmproj_embd(ctx_clip);
|
||||
|
||||
clip_image_f32 hidden_state;
|
||||
hidden_state.set_size({(int) n_embd, 1}, false, true);
|
||||
hidden_state.cpy_buf(std::vector<float>(inp->embd, inp->embd + n_embd));
|
||||
|
||||
clip_image_f32_batch batch;
|
||||
batch.is_audio = true;
|
||||
batch.entries.push_back(std::move(hidden_state));
|
||||
|
||||
std::vector<float> out_embd(n_embd);
|
||||
std::vector<int32_t> out_codes;
|
||||
|
||||
clip_encode_params params;
|
||||
params.imgs = &batch;
|
||||
params.n_threads = ctx->n_threads;
|
||||
params.gen_process = CLIP_GEN_PROCESS_GEN_CODE;
|
||||
params.out_embd = &out_embd;
|
||||
params.out_codes = &out_codes;
|
||||
params.code0 = inp->code0;
|
||||
params.top_k = inp->top_k;
|
||||
params.top_p = inp->top_p;
|
||||
|
||||
if (!clip_encode(ctx_clip, ¶ms)) {
|
||||
LOG_ERR("%s: clip_encode failed (gen_code)\n", __func__);
|
||||
return 1;
|
||||
}
|
||||
|
||||
ctx->gen_out_embd = std::move(out_embd);
|
||||
ctx->gen_out_codes = std::move(out_codes);
|
||||
|
||||
out->embd = ctx->gen_out_embd.data();
|
||||
out->codes = ctx->gen_out_codes.data();
|
||||
out->n_codes = ctx->gen_out_codes.size();
|
||||
return 0;
|
||||
}
|
||||
|
||||
// MTMD_GEN_PROCESS_TYPE_GEN_WAV
|
||||
if (!inp->codes || inp->n_codes == 0) {
|
||||
LOG_ERR("%s: codes required for gen_wav\n", __func__);
|
||||
return 1;
|
||||
}
|
||||
std::vector<int32_t> in_codes(inp->codes, inp->codes + inp->n_codes);
|
||||
std::vector<uint8_t> in_state;
|
||||
if (inp->state_data) {
|
||||
in_state.assign(inp->state_data, inp->state_data + inp->state_size);
|
||||
}
|
||||
|
||||
// gen_wav has no hidden-state input, the batch entry is an unused placeholder
|
||||
// TODO @ngxson : some models in the future may require hidden-state input, need to update this code later
|
||||
clip_image_f32 dummy;
|
||||
dummy.set_size({1, 1}, false, true);
|
||||
dummy.cpy_buf(std::vector<float>(1, 0.0f));
|
||||
|
||||
clip_image_f32_batch batch;
|
||||
batch.is_audio = true;
|
||||
batch.entries.push_back(std::move(dummy));
|
||||
|
||||
clip_encode_params params;
|
||||
params.imgs = &batch;
|
||||
params.n_threads = ctx->n_threads;
|
||||
params.gen_process = CLIP_GEN_PROCESS_GEN_WAV;
|
||||
params.codes = &in_codes;
|
||||
params.out_audio = &ctx->gen_out_audio;
|
||||
params.state_in = inp->state_data ? &in_state : nullptr;
|
||||
params.state_out = &ctx->gen_out_state;
|
||||
|
||||
if (!clip_encode(ctx_clip, ¶ms)) {
|
||||
LOG_ERR("%s: clip_encode failed (code2wav)\n", __func__);
|
||||
return 1;
|
||||
}
|
||||
|
||||
out->audio = ctx->gen_out_audio.data();
|
||||
out->n_samples = ctx->gen_out_audio.size();
|
||||
out->state_data = (const char *) ctx->gen_out_state.data();
|
||||
out->state_size = ctx->gen_out_state.size();
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
int32_t mtmd_gen_audio_process(mtmd_context * ctx, const struct mtmd_gen_inp * inp, struct mtmd_gen_out * out) {
|
||||
try {
|
||||
return mtmd_gen_audio_process_impl(ctx, inp, out);
|
||||
} catch (const std::exception & e) {
|
||||
LOG_ERR("%s: error: %s\n", __func__, e.what());
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
|
||||
mtmd_batch * mtmd_batch_init(mtmd_context * ctx) {
|
||||
return new mtmd_batch(ctx);
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user