Lumpiasty/llama.cpp
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

mtmd: support Qwen3-TTS (note: breaking change to llama-tts binary) (#26254)

Browse Source 
* convert text model

* main model load ok

* convert encoder ok

* speaker encoder loading ok

* speaker enc graph

* adapt vocab for backbone (with some tricks)

* add suppress_tokens

* poc new mtmd gen api

* convert code_predictor to gguf

* load gen_code model ok

* add clip_encode

* wire up

* code gen cgraph init version

Co-authored-by: Pascal <admin@serveurperso.com>

* code2wav convert to gguf

* code2wav graph ok

* wire up in/out

* (wip) subgraph

* wire up

* wip, correct code2wav

* demo (to be removed)

* code2wav preserve kv between calls

* demo voice clone

* llama: add llama_model_get_tok_embd

* mtmd_helper_gen_audio API

* fix clamp cold prefix

Co-authored-by: Pascal <admin@serveurperso.com>

* fuse snake op

Co-authored-by: Pascal <admin@serveurperso.com>

* demo: use proper sampling

* update dev docs

* polymorphism helper

* revamp llama-tts binary

* update docs

* fix compile

* fix lint

* nits

* add guide + docs

* more timings info

* clean up code comments

* security fixes

* update docs

* use ggml_build_forward_select, clean up comments

* fix ci

* use ISO 639-1 language code

* rename CODE2WAV --> GEN_WAV, update docs

* clean up

* clean up tts.cpp

* add seq_id

* add step_prompt()

* mtmd_helper_model_can_chat

* clean up comments

---------

Co-authored-by: Pascal <admin@serveurperso.com>
This commit is contained in:
Xuan-Son Nguyen
2026-08-04 17:26:15 +02:00
committed by GitHub
co-authored by Pascal
parent 1c3c9674de
commit 0713275082
42 changed files with 3808 additions and 1895 deletions
Download Patch File Download Diff File Expand all files Collapse all files
common/arg.cpp
+12 -57
View File
@@ -61,6 +61,7 @@ static std::initializer_list<enum llama_example> mmproj_examples = {
LLAMA_EXAMPLE_MTMD,
LLAMA_EXAMPLE_SERVER,
LLAMA_EXAMPLE_CLI,
LLAMA_EXAMPLE_TTS,
};
static std::string read_file(const std::string & fname) {
@@ -360,7 +361,6 @@ static bool spec_types_is_default(const common_params & params) {
common_models_handler common_models_handler_init(const common_params & params, llama_example curr_ex) {
common_download_hf_plan plan;
common_download_hf_plan plan_spec;
common_download_hf_plan plan_voc;
common_download_opts opts;
const bool spec_type_draft_mtp = std::find(params.speculative.types.begin(),
@@ -413,11 +413,7 @@ common_models_handler common_models_handler_init(const common_params & params, l
plan_spec = common_download_get_hf_plan(params.speculative.draft.mparams, opts_spec);
}
if (!params.vocoder.model.hf_repo.empty()) {
plan_voc = common_download_get_hf_plan(params.vocoder.model, opts);
}
return common_models_handler{plan, plan_spec, plan_voc, opts};
return common_models_handler{plan, plan_spec, opts};
}
bool common_models_handler_is_preset_repo(const common_models_handler & handler) {
@@ -467,7 +463,6 @@ void common_models_handler_apply(common_models_handler & handler, common_params
auto & plan = handler.plan;
auto & plan_spec = handler.plan_spec;
auto & plan_voc = handler.plan_voc;
auto opts = handler.opts; // copy
opts.callback = callback;
@@ -482,7 +477,6 @@ void common_models_handler_apply(common_models_handler & handler, common_params
};
handle_url(params.model);
handle_url(params.mmproj);
handle_url(params.vocoder.model);
handle_url(params.speculative.draft.mparams);
// optionally, if docker repo is set, resolve it
@@ -510,14 +504,6 @@ void common_models_handler_apply(common_models_handler & handler, common_params
task.opts = opts;
tasks.push_back(task);
}
if (!params.vocoder.model.url.empty()) {
common_download_task task;
task.url = params.vocoder.model.url;
task.local_path = params.vocoder.model.path;
task.opts = opts;
tasks.push_back(task);
}
bool had_spec_url = false;
if (!params.speculative.draft.mparams.url.empty()) {
common_download_task task;
@@ -631,11 +617,6 @@ void common_models_handler_apply(common_models_handler & handler, common_params
had_spec_url = true;
}
// handle vocoder plan (e.g. --hf-repo-v)
if (!plan_voc.model_files.empty()) {
add_tasks(plan_voc.model_files, plan_voc.primary, params.vocoder.model);
}
if (!plan.model_files.empty()) {
add_tasks(plan.model_files, plan.primary, params.model);
}
@@ -1361,6 +1342,10 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
params.n_parallel = -1; // auto by default
} else if (ex == LLAMA_EXAMPLE_TOKENIZE) {
params.parse_special = true; // parse special tokens by default, like the old tokenize tool
} else if (ex == LLAMA_EXAMPLE_TTS) {
params.out_file = "output.wav";
params.sampling.penalty_repeat = 1.05f;
params.sampling.penalty_last_n = -1;
}
params.use_color = tty_can_use_colors();
@@ -2983,20 +2968,6 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
params.model.hf_file = value;
}
).set_examples({LLAMA_EXAMPLE_COMMON, LLAMA_EXAMPLE_DOWNLOAD, LLAMA_EXAMPLE_TOKENIZE}).set_env("LLAMA_ARG_HF_FILE"));
add_opt(common_arg(
{"-hfv", "-hfrv", "--hf-repo-v"}, "<user>/<model>[:quant]",
"Hugging Face model repository for the vocoder model (default: unused)",
[](common_params & params, const std::string & value) {
params.vocoder.model.hf_repo = value;
}
).set_env("LLAMA_ARG_HF_REPO_V"));
add_opt(common_arg(
{"-hffv", "--hf-file-v"}, "FILE",
"Hugging Face model file for the vocoder model (default: unused)",
[](common_params & params, const std::string & value) {
params.vocoder.model.hf_file = value;
}
).set_env("LLAMA_ARG_HF_FILE_V"));
add_opt(common_arg(
{"-hft", "--hf-token"}, "TOKEN",
"Hugging Face access token (default: value from HF_TOKEN environment variable)",
@@ -4272,24 +4243,18 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
//
add_opt(common_arg(
{"-mv", "--model-vocoder"}, "FNAME",
"vocoder model for audio generation (default: unused)",
{"--tts-lang"}, "FNAME",
"language (ISO 639-1) for audio generation\n"
"see tts/README.md for per-model usage notes",
[](common_params & params, const std::string & value) {
params.vocoder.model.path = value;
params.tts_lang = value;
}
).set_examples({LLAMA_EXAMPLE_TTS, LLAMA_EXAMPLE_SERVER}));
add_opt(common_arg(
{"--tts-use-guide-tokens"},
"Use guide tokens to improve TTS word recall",
[](common_params & params) {
params.vocoder.use_guide_tokens = true;
}
).set_examples({LLAMA_EXAMPLE_TTS, LLAMA_EXAMPLE_SERVER}));
).set_examples({LLAMA_EXAMPLE_TTS}));
add_opt(common_arg(
{"--tts-speaker-file"}, "FNAME",
"speaker file path for audio generation",
[](common_params & params, const std::string & value) {
params.vocoder.speaker_file = value;
params.tts_speaker_file = value;
}
).set_examples({LLAMA_EXAMPLE_TTS}));
@@ -4409,16 +4374,6 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
).set_examples({LLAMA_EXAMPLE_DEBUG}));
// presets
add_opt(common_arg(
{"--tts-oute-default"},
string_format("use default OuteTTS models (note: can download weights from the internet)"),
[](common_params & params) {
params.model.hf_repo = "OuteAI/OuteTTS-0.2-500M-GGUF";
params.model.hf_file = "OuteTTS-0.2-500M-Q8_0.gguf";
params.vocoder.model.hf_repo = "ggml-org/WavTokenizer";
params.vocoder.model.hf_file = "WavTokenizer-Large-75-F16.gguf";
}
).set_examples({LLAMA_EXAMPLE_TTS}));
add_opt(common_arg(
{"--embd-gemma-default"},
common/arg.h
-1
View File
@@ -137,7 +137,6 @@ void common_params_add_preset_options(std::vector<common_arg> & args);
struct common_models_handler {
common_download_hf_plan plan;
common_download_hf_plan plan_spec;
common_download_hf_plan plan_voc;
common_download_opts opts;
};
common/common.h
+4 -9
View File
@@ -392,14 +392,6 @@ struct common_params_speculative {
}
};
struct common_params_vocoder {
struct common_params_model model;
std::string speaker_file; // speaker file path
bool use_guide_tokens = false; // enable guide tokens to improve TTS accuracy
};
struct common_params_diffusion {
int32_t steps = 128;
bool visual_mode = false;
@@ -497,7 +489,6 @@ struct common_params {
struct common_params_sampling sampling;
struct common_params_speculative speculative;
struct common_params_vocoder vocoder;
struct common_params_diffusion diffusion;
struct common_params_model model;
@@ -740,6 +731,10 @@ struct common_params {
void * load_progress_callback_user_data = NULL;
bool no_alloc = false; // Don't allocate model buffers
// TTS params
std::string tts_lang = "";
std::string tts_speaker_file = "";
bool is_gen_docs = false; // whether we are running inside llama-gen-docs
};