Files
llama.cpp/tools/mtmd/mtmd-helper-gen.cpp
T
Xuan-Son NguyenandPascal 92192d31e4 mtmd: support YuE2-3B (#32369)
* mtmd: support YuE2-3B

* improve speed

* add CFG

* fix streaming

* add cfg control from api

* check ctx size

* fix n_predict

* add flow_steps config

* nits

* fix batch api

* nits

* adapt

* mtmd: yue2 samples from its own saved logits

The context logits belong to the last decode of any slot, so with
several server slots the score and the unguided codes were sampled
from another sequence. Keep the logits of each own decode in
save_logits(), which also does the guidance mix.

* mtmd: let the yue2 snake chain fuse

The beta reshape landed between sqr and mul, so the mul, sin, sqr,
mul, add chain was not consecutive and the backends never fused it.
Expand both reshapes first, like qwen3tts does.

---------

Co-authored-by: Pascal <admin@serveurperso.com>
2026-11-16 20:22:23 +01:00

3987 lines
154 KiB
C++

#include "mtmd.h"
#include "mtmd-helper.h"
#include "mtmd-helper-common.h"
#include "llama.h"
#include "../src/llama-ext.h"
#include <algorithm>
#include <cctype>
#include <cmath>
#include <cstring>
#include <functional>
#include <memory>
#include <random>
#include <string>
#include <unordered_map>
#include <vector>
#ifdef MTMD_INTERNAL_HEADER
#error "mtmd-helper is a public library outside of mtmd. it must not include internal headers"
#endif
//
// Audio generation helpers
//
// --tts-lang codes -> language names used by the codec_language special tokens
static const std::unordered_map<std::string, std::string> tts_lang_codes = {
{ "zh", "chinese" },
{ "en", "english" },
{ "de", "german" },
{ "it", "italian" },
{ "pt", "portuguese" },
{ "es", "spanish" },
{ "ja", "japanese" },
{ "ko", "korean" },
{ "fr", "french" },
{ "ru", "russian" },
};
static std::string tts_resolve_lang(const std::string & lang) {
auto it = tts_lang_codes.find(lang);
return it != tts_lang_codes.end() ? it->second : lang;
}
static llama_token find_special_token(const llama_vocab * vocab, const std::string & piece) {
const int32_t n = llama_vocab_n_tokens(vocab);
for (llama_token t = 0; t < n; t++) {
if (piece == llama_vocab_get_text(vocab, t)) {
return t;
}
}
return LLAMA_TOKEN_NULL;
}
static bool batch_set_tokens(llama_batch_ext * batch, const llama_token * tokens, int32_t n, llama_pos pos_0, llama_seq_id seq_id) {
llama_batch_ext_clear(batch);
for (int32_t i = 0; i < n; i++) {
const int32_t idx = llama_batch_ext_add_token(batch, seq_id, tokens[i]);
const llama_pos pos = pos_0 + i;
if (idx < 0) {
LOG_ERR("mtmd_helper_gen_audio: failed to add token %d to batch (error %d)\n", tokens[i], idx);
return false;
}
llama_batch_ext_set_pos(batch, idx, &pos);
}
return true;
}
static void put_bytes(std::vector<char> & buf, const void * p, size_t n) {
const char * c = (const char *) p;
buf.insert(buf.end(), c, c + n);
}
// data_sz == UINT32_MAX writes the "unknown length" sentinel (streaming), same as ffmpeg does on a pipe
// multi-channel PCM is interleaved
static void write_wav16_header(std::vector<char> & buf, uint32_t data_sz, int32_t rate, uint16_t ch = 1) {
const uint32_t riff_sz = data_sz == UINT32_MAX ? UINT32_MAX : 36 + data_sz;
const uint32_t fmt_sz = 16, byte_rate = (uint32_t) rate * 2 * ch;
const uint16_t fmt = 1, align = 2 * ch, bits = 16;
const uint32_t rate32 = (uint32_t) rate;
put_bytes(buf, "RIFF", 4); put_bytes(buf, &riff_sz, 4); put_bytes(buf, "WAVE", 4);
put_bytes(buf, "fmt ", 4); put_bytes(buf, &fmt_sz, 4);
put_bytes(buf, &fmt, 2); put_bytes(buf, &ch, 2); put_bytes(buf, &rate32, 4);
put_bytes(buf, &byte_rate, 4); put_bytes(buf, &align, 2); put_bytes(buf, &bits, 2);
put_bytes(buf, "data", 4); put_bytes(buf, &data_sz, 4);
}
static void append_wav16_pcm(std::vector<char> & buf, const float * pcm, size_t n) {
for (size_t i = 0; i < n; i++) {
int16_t s = (int16_t) std::lround(std::max(-1.0f, std::min(1.0f, pcm[i])) * 32767.0f);
put_bytes(buf, &s, 2);
}
}
static bool write_wav16(std::vector<char> & buf, const std::vector<float> & pcm, int32_t rate, uint16_t ch = 1) {
// RIFF chunk sizes are 32-bit; refuse to emit a file with a truncated header
if (pcm.size() > ((size_t) UINT32_MAX - 36) / 2) {
return false;
}
write_wav16_header(buf, (uint32_t) (pcm.size() * 2), rate, ch);
append_wav16_pcm(buf, pcm.data(), pcm.size());
return true;
}
class mtmd_gen_audio_pipeline {
public:
mtmd_gen_audio_pipeline(llama_context * lctx, mtmd_context * mctx)
: lctx(lctx), mctx(mctx), model(llama_get_model(lctx)), vocab(llama_model_get_vocab(model)),
n_embd(llama_model_n_embd(model)), info(mtmd_gen_audio_get_info(mctx)) {}
virtual ~mtmd_gen_audio_pipeline() = default;
// false if the backbone context cannot run the pipeline, checked by mtmd_helper_gen_audio_init()
virtual bool check_context() const { return true; }
// see mtmd_helper_gen_audio_max_frames()
virtual int32_t max_frames() const { return 512; }
virtual void reset() = 0;
virtual int32_t set_input(const mtmd_helper_gen_audio_inp * inp) = 0;
// decodes at most n_batch prompt tokens; returns remaining count (0 = done), <0 on error
virtual int32_t step_prompt(int32_t n_batch) = 0;
// sampled can be LLAMA_TOKEN_NULL for pipelines with no discrete backbone token,
// those read what they need from h_state_in instead
// set out_stop on end-of-speech, h_state_out must be null if no frame is generated
virtual int32_t step_gen(llama_token sampled, const float * h_state_in, const float ** h_state_out, bool * out_stop) = 0;
virtual int32_t get_output(int32_t * out_sample_rate, const char ** out_data, size_t * out_data_len, int64_t * out_n_samples) = 0;
// forces any buffered codes through code2wav now, regardless of window_frames
virtual int32_t flush() = 0;
// continuing a sequence (inp->n_past > 0) with the model's own text/audio switching
virtual bool supports_continue() const { return false; }
// mtmd_helper_gen_audio_inp::speed and loudness other than the defaults
virtual bool supports_delivery() const { return false; }
// mtmd_helper_gen_audio_inp::cfg_scale other than the default
virtual bool supports_cfg() const { return false; }
// mtmd_helper_gen_audio_inp::flow_steps other than the default
virtual bool supports_flow_steps() const { return false; }
virtual bool next_is_text() const { return false; }
std::vector<llama_logit_bias> logit_bias; // see mtmd_helper_gen_audio_get_logit_bias()
protected:
// one state per generation, so a fixed seed always gives the same output
bool init_gen_state(uint32_t seed) {
mtmd_gen_state_params params = mtmd_gen_state_params_default();
params.seed = seed;
gen_state.reset(mtmd_gen_state_init(mctx, params));
if (!gen_state) {
LOG_ERR("mtmd_helper_gen_audio: failed to create the gen state\n");
return false;
}
return true;
}
// encodes a speaker clip or a voice, out receives the encoder embeddings
bool encode_speaker(mtmd_bitmap * bitmap, std::vector<float> & out) {
if (!mtmd_bitmap_is_voice(bitmap) && !mtmd_support_audio(mctx)) {
LOG_ERR("mtmd_helper_gen_audio: mmproj has no speaker/audio encoder\n");
return false;
}
const std::string marker = mtmd_get_marker(mctx);
mtmd_input_text text{ marker.c_str(), marker.size(), false, true };
mtmd_input_chunks * chunks = mtmd_input_chunks_init();
const mtmd_bitmap * bptr = bitmap;
bool ok = mtmd_tokenize(mctx, chunks, &text, &bptr, 1) == 0;
if (ok) {
ok = false;
for (size_t i = 0; i < mtmd_input_chunks_size(chunks); i++) {
const mtmd_input_chunk * chunk = mtmd_input_chunks_get(chunks, i);
if (mtmd_input_chunk_get_type(chunk) != MTMD_INPUT_CHUNK_TYPE_AUDIO) {
continue;
}
if (mtmd_encode_chunk(mctx, chunk) != 0) {
LOG_ERR("mtmd_helper_gen_audio: speaker encode failed\n");
break;
}
const float * embd = mtmd_get_output_embd(mctx);
const size_t n = (size_t) llama_model_n_embd_inp(model) * mtmd_input_chunk_get_n_tokens(chunk);
out.assign(embd, embd + n);
ok = true;
break;
}
}
mtmd_input_chunks_free(chunks);
return ok;
}
llama_context * lctx;
mtmd_context * mctx;
const llama_model * model;
const llama_vocab * vocab;
int n_embd;
mtmd_gen_audio_info info;
mtmd::gen_state_ptr gen_state;
};
// Qwen3-TTS: backbone samples codec_0, code_predictor gives the other 15 codebooks,
// then code2wav decodes them to PCM
class qwen3tts_gen_audio_pipeline : public mtmd_gen_audio_pipeline {
public:
using mtmd_gen_audio_pipeline::mtmd_gen_audio_pipeline;
void reset() override {
seq_id = 0;
pos = 0;
codes_buf.clear();
gen_state.reset();
audio_pcm.clear();
overlay.clear();
h_state_buf.clear();
out_buf.clear();
prompt_embd_buf.clear();
prompt_batch.reset();
n_prompt = 0;
prompt_pos = 0;
stream = false;
pcm_sent = 0;
wav_header_sent = false;
}
int32_t set_input(const mtmd_helper_gen_audio_inp * inp) override {
reset();
seq_id = inp->seq_id;
if (!ensure_cache()) {
return 1;
}
const std::string lang = tts_resolve_lang((inp->lang && inp->lang[0]) ? inp->lang : "english");
const llama_token c_lang = find_special_token(vocab, ("<|codec_language_" + lang + "|>").c_str());
if (c_lang == LLAMA_TOKEN_NULL) {
LOG_ERR("mtmd_helper_gen_audio: unknown language '%s'\n", lang.c_str());
return 1;
}
std::vector<float> speaker_embd;
if (inp->speaker_ref) {
if (!encode_speaker(inp->speaker_ref, speaker_embd)) {
return 1;
}
}
const int n_e = n_embd;
auto row = [&](llama_token t) {
return std::vector<float>(tok_embd.begin() + (size_t) t * n_e,
tok_embd.begin() + (size_t) (t + 1) * n_e);
};
auto sum_row = [&](llama_token a, llama_token b) {
std::vector<float> va = row(a), vb = row(b);
for (int i = 0; i < n_e; i++) va[(size_t) i] += vb[(size_t) i];
return va;
};
auto sum_vec = [&](llama_token a, const std::vector<float> & vb) {
std::vector<float> va = row(a);
for (int i = 0; i < n_e; i++) va[(size_t) i] += vb[(size_t) i];
return va;
};
// upstream chat wrap, then slices: [0:3] role, [3:-5] utterance body
const std::string full = "<|im_start|>assistant\n" + std::string(inp->prompt, inp->prompt_len) +
"<|im_end|>\n<|im_start|>assistant\n";
std::vector<llama_token> ids(full.size() + 16);
int n_ids = llama_tokenize(vocab, full.c_str(), (int32_t) full.size(), ids.data(), (int32_t) ids.size(),
false, true);
if (n_ids < 8) {
LOG_ERR("mtmd_helper_gen_audio: tokenization failed\n");
return 1;
}
ids.resize((size_t) n_ids);
std::vector<std::vector<float>> prompt;
for (int i = 0; i < 3; i++) prompt.push_back(row(ids[(size_t) i]));
prompt.push_back(sum_row(tts_pad, c_think));
prompt.push_back(sum_row(tts_pad, c_think_b));
prompt.push_back(sum_row(tts_pad, c_lang));
prompt.push_back(sum_row(tts_pad, c_think_e));
if (!speaker_embd.empty()) prompt.push_back(sum_vec(tts_pad, speaker_embd));
prompt.push_back(sum_row(tts_bos, codec_pad));
for (int i = 3; i < n_ids - 5; i++) prompt.push_back(sum_row(ids[(size_t) i], codec_pad));
prompt.push_back(sum_row(tts_eos, codec_pad));
prompt.push_back(sum_row(tts_pad, codec_bos));
n_prompt = (int) prompt.size();
// the talker uses the qwen3vl interleaved mrope, all sections are equal for a text/codec stream
mrope = llama_model_rope_type(model) == LLAMA_ROPE_TYPE_MROPE ||
llama_model_rope_type(model) == LLAMA_ROPE_TYPE_IMROPE;
const int n_pos_per_embd = mrope ? 4 : 1;
prompt_embd_buf.resize((size_t) n_prompt * (size_t) n_e);
for (int i = 0; i < n_prompt; i++) {
memcpy(prompt_embd_buf.data() + (size_t) i * n_e, prompt[(size_t) i].data(), (size_t) n_e * sizeof(float));
}
prompt_batch.reset(new decode_embd_batch(prompt_embd_buf.data(), n_prompt, n_pos_per_embd, n_e));
if (mrope) prompt_batch->set_position_mrope_1d(0, seq_id);
else prompt_batch->set_position_normal (0, seq_id);
prompt_pos = 0;
pos = 0;
const mtmd_gen_inp def = mtmd_gen_inp_default(mctx);
top_k = inp->top_k > 0 ? inp->top_k : def.top_k;
top_p = inp->top_p > 0 ? inp->top_p : def.top_p;
out_type = inp->out_type;
stream = inp->stream;
if (!init_gen_state(inp->seed)) {
return 1;
}
// the prompt above holds the whole text stream up to tts_eos, so every generated
// frame adds tts_pad on top of the codes embedding
overlay = row(tts_pad);
return 0;
}
int32_t step_prompt(int32_t n_batch) override {
GGML_ASSERT(n_batch > 0);
if (prompt_pos >= n_prompt) {
return 0;
}
const int32_t n_tokens_batch = std::min(n_batch, n_prompt - prompt_pos);
const bool is_last_batch = (prompt_pos + n_tokens_batch) == n_prompt;
if (is_last_batch) {
prompt_batch->logits[prompt_pos + n_tokens_batch - 1] = 1;
}
if (llama_process(lctx, LLAMA_PROCESS_TYPE_DECODE, prompt_batch->render(lctx, prompt_pos, n_tokens_batch)) != 0) {
LOG_ERR("mtmd_helper_gen_audio: prompt decode failed\n");
return -1;
}
pos += n_tokens_batch;
prompt_pos += n_tokens_batch;
if (prompt_pos >= n_prompt) {
// prompt fully processed, its embedding buffer is no longer needed
prompt_batch.reset();
prompt_embd_buf.clear();
return 0;
}
return n_prompt - prompt_pos;
}
int32_t step_gen(llama_token sampled, const float * h_state_in, const float ** h_state_out, bool * out_stop) override {
if (sampled == LLAMA_TOKEN_NULL) {
LOG_ERR("mtmd_helper_gen_audio: qwen3tts requires a token sampled from the backbone\n");
return 1;
}
// backbone signals end-of-speech with a token, no frame for this step
if (sampled == codec_eos || llama_vocab_is_eog(vocab, sampled)) {
*out_stop = true;
*h_state_out = nullptr;
return 0;
}
mtmd_gen_inp inp = mtmd_gen_inp_default(mctx);
inp.type = MTMD_GEN_PROCESS_TYPE_GEN_CODE;
inp.code0 = sampled - codec_0;
inp.embd = const_cast<float *>(h_state_in);
inp.top_k = top_k;
inp.top_p = top_p;
mtmd_gen_out out{};
if (mtmd_gen_audio_process(mctx, gen_state.get(), &inp, &out) != 0) {
LOG_ERR("mtmd_helper_gen_audio: gen_code process failed\n");
return 1;
}
codes_buf.insert(codes_buf.end(), out.codes, out.codes + out.n_codes);
if (out.n_codes > 0 && codes_buf.size() / out.n_codes >= window_frames) {
if (!flush_gen_wav()) {
return 1;
}
}
std::vector<float> fb(out.embd, out.embd + n_embd);
for (int i = 0; i < n_embd; i++) fb[(size_t) i] += overlay[(size_t) i];
const int n_pos_per_embd = mrope ? 4 : 1;
decode_embd_batch batch_embd(fb.data(), 1, n_pos_per_embd, n_embd);
if (mrope) batch_embd.set_position_mrope_1d(pos, seq_id);
else batch_embd.set_position_normal (pos, seq_id);
batch_embd.logits[0] = 1;
pos++;
if (llama_process(lctx, LLAMA_PROCESS_TYPE_DECODE, batch_embd.render(lctx, 0, 1)) != 0) {
LOG_ERR("mtmd_helper_gen_audio: decode failed\n");
return 1;
}
const float * he = llama_get_embeddings_ith(lctx, -1);
h_state_buf.assign(he, he + n_embd);
*h_state_out = h_state_buf.data();
return 0;
}
int32_t get_output(int32_t * out_sample_rate, const char ** out_data, size_t * out_data_len, int64_t * out_n_samples) override {
*out_sample_rate = info.sample_rate;
if (!stream) {
// one-shot call: force out whatever's left, regardless of window_frames
if (!flush_gen_wav()) {
return 1;
}
if (out_n_samples) {
*out_n_samples = (int64_t) audio_pcm.size();
}
if (out_type == MTMD_HELPER_GEN_AUDIO_OUTTYPE_PCM) {
*out_data = (const char *) audio_pcm.data();
*out_data_len = audio_pcm.size() * sizeof(float);
return 0;
}
out_buf.clear();
if (!write_wav16(out_buf, audio_pcm, info.sample_rate)) {
LOG_ERR("mtmd_helper_gen_audio: output too large for WAV\n");
return 1;
}
*out_data = out_buf.data();
*out_data_len = out_buf.size();
return 0;
}
// streaming: only return audio produced since the previous call
const size_t n_new = audio_pcm.size() - pcm_sent;
if (out_n_samples) {
*out_n_samples = (int64_t) n_new;
}
if (out_type == MTMD_HELPER_GEN_AUDIO_OUTTYPE_PCM) {
*out_data = (const char *) (audio_pcm.data() + pcm_sent);
*out_data_len = n_new * sizeof(float);
pcm_sent = audio_pcm.size();
return 0;
}
out_buf.clear();
if (!wav_header_sent) {
write_wav16_header(out_buf, UINT32_MAX, info.sample_rate);
wav_header_sent = true;
}
append_wav16_pcm(out_buf, audio_pcm.data() + pcm_sent, n_new);
pcm_sent = audio_pcm.size();
*out_data = out_buf.data();
*out_data_len = out_buf.size();
return 0;
}
int32_t flush() override {
return flush_gen_wav() ? 0 : 1;
}
private:
bool ensure_cache() {
if (specials_ok) {
return true;
}
codec_0 = find_special_token(vocab, "<|codec_0|>");
codec_bos = find_special_token(vocab, "<|codec_bos|>");
codec_eos = find_special_token(vocab, "<|codec_eos_token|>");
codec_pad = find_special_token(vocab, "<|codec_pad|>");
c_think = find_special_token(vocab, "<|codec_think|>");
c_think_b = find_special_token(vocab, "<|codec_think_bos|>");
c_think_e = find_special_token(vocab, "<|codec_think_eos|>");
tts_pad = find_special_token(vocab, "<tts_pad>");
tts_bos = find_special_token(vocab, "<tts_text_bos>");
tts_eos = find_special_token(vocab, "<tts_text_eod>");
for (llama_token t : { codec_0, codec_bos, codec_eos, codec_pad,
c_think, c_think_b, c_think_e,
tts_pad, tts_bos, tts_eos }) {
if (t == LLAMA_TOKEN_NULL) {
LOG_ERR("mtmd_helper_gen_audio: missing a required special token in vocab\n");
return false;
}
}
const uint32_t n_tok_embd = llama_model_get_tok_embd(model, nullptr);
if (n_tok_embd == 0) {
LOG_ERR("mtmd_helper_gen_audio: model has no token embeddings\n");
return false;
}
tok_embd.resize(n_tok_embd);
if (llama_model_get_tok_embd(model, tok_embd.data()) != n_tok_embd) {
LOG_ERR("mtmd_helper_gen_audio: token embedding copy failed\n");
return false;
}
specials_ok = true;
return true;
}
// one GEN_WAV process() call over the buffered codes, state is carried across batches
bool flush_gen_wav() {
if (codes_buf.empty()) {
return true;
}
mtmd_gen_inp inp = mtmd_gen_inp_default(mctx);
inp.type = MTMD_GEN_PROCESS_TYPE_GEN_WAV;
inp.codes = codes_buf.data();
inp.n_codes = codes_buf.size();
mtmd_gen_out out{};
if (mtmd_gen_audio_process(mctx, gen_state.get(), &inp, &out) != 0) {
LOG_ERR("mtmd_helper_gen_audio: gen_wav process failed\n");
return false;
}
audio_pcm.insert(audio_pcm.end(), out.audio, out.audio + out.n_samples);
codes_buf.clear();
return true;
}
// vocab specials fixed across the whole session, looked up once
bool specials_ok = false;
llama_token codec_0 = LLAMA_TOKEN_NULL;
llama_token codec_bos = LLAMA_TOKEN_NULL;
llama_token codec_eos = LLAMA_TOKEN_NULL;
llama_token codec_pad = LLAMA_TOKEN_NULL;
llama_token c_think = LLAMA_TOKEN_NULL;
llama_token c_think_b = LLAMA_TOKEN_NULL;
llama_token c_think_e = LLAMA_TOKEN_NULL;
llama_token tts_pad = LLAMA_TOKEN_NULL;
llama_token tts_bos = LLAMA_TOKEN_NULL;
llama_token tts_eos = LLAMA_TOKEN_NULL;
std::vector<float> tok_embd; // whole token embedding matrix, n_vocab * n_embd
// must match hparams.wav_tfm_swa hardcoded in mtmd-model.cpp
size_t window_frames = 72;
// per-generation state, cleared by reset()
llama_seq_id seq_id = 0;
bool mrope = false;
int pos = 0;
// prompt decode state, consumed batch-by-batch by step_prompt()
std::vector<float> prompt_embd_buf;
std::unique_ptr<decode_embd_batch> prompt_batch;
int n_prompt = 0;
int prompt_pos = 0;
int32_t top_k = 50;
float top_p = 1.0f;
std::vector<int32_t> codes_buf;
std::vector<float> audio_pcm;
std::vector<float> overlay;
std::vector<float> h_state_buf;
mtmd_helper_gen_audio_outtype out_type = MTMD_HELPER_GEN_AUDIO_OUTTYPE_WAV;
std::vector<char> out_buf;
bool stream = false;
size_t pcm_sent = 0; // samples already returned by get_output()
bool wav_header_sent = false;
};
// settings that only live in the reference's per-pack yaml, not in the checkpoint
// the english packs share the same shapes and tokenizer, but disagree on these
// both are 0 / false when the pack does not tune them, the model default is then used
struct pockettts_pack_settings {
int frames_after_eos = 0;
bool pad_short_text = false;
};
static pockettts_pack_settings pockettts_pack(const char * variant) {
static const std::unordered_map<std::string, pockettts_pack_settings> packs = {
{ "english", { 0, false } },
{ "english_2026-01", { 0, true } },
{ "english_2026-04", { 0, false } },
{ "french_24l", { 8, false } },
{ "italian", { 0, false } },
};
auto it = packs.find(variant ? variant : "");
if (it == packs.end()) {
LOG_WRN("mtmd_helper_gen_audio: no tuned settings for pocket-tts variant \"%s\"\n",
variant ? variant : "");
return {};
}
return it->second;
}
// pocket-tts: the backbone emits no token, the flow net turns each hidden state into a latent
// the end-of-speech head also lives in the mmproj
class pockettts_gen_audio_pipeline : public mtmd_gen_audio_pipeline {
public:
using mtmd_gen_audio_pipeline::mtmd_gen_audio_pipeline;
void reset() override {
seq_id = 0;
pos = 0;
feats_buf.clear();
gen_state.reset();
audio_pcm.clear();
h_state_buf.clear();
out_buf.clear();
prompt_embd_buf.clear();
prompt_batch.reset();
n_prompt = 0;
prompt_pos = 0;
step_idx = 0;
eos_step = -1;
chunks.clear();
chunk_idx = 0;
n_voice_pos = 0;
chunk_budget = 0;
stream = false;
pcm_sent = 0;
wav_header_sent = false;
}
int32_t set_input(const mtmd_helper_gen_audio_inp * inp) override {
reset();
seq_id = inp->seq_id;
if (!ensure_cache()) {
return 1;
}
std::vector<float> voice;
if (inp->speaker_ref) {
if (!encode_speaker(inp->speaker_ref, voice)) {
return 1;
}
}
pack = pockettts_pack(info.model_variant);
const std::string text = prepare_text(std::string(inp->prompt, inp->prompt_len),
pack.pad_short_text);
if (text.empty()) {
LOG_ERR("mtmd_helper_gen_audio: empty prompt\n");
return 1;
}
std::vector<llama_token> ids(text.size() + 16);
int n_ids = llama_tokenize(vocab, text.c_str(), (int32_t) text.size(), ids.data(),
(int32_t) ids.size(), false, false);
if (n_ids <= 0) {
LOG_ERR("mtmd_helper_gen_audio: tokenization failed\n");
return 1;
}
ids.resize((size_t) n_ids);
// long inputs degrade badly, so each chunk restarts from the voice conditioning
// see split_into_best_sentences() in the reference
chunks = split_chunks(ids);
chunk_idx = 0;
if (chunks.size() > 1) {
LOG_INF("mtmd_helper_gen_audio: %d tokens split into %zu chunks\n", n_ids, chunks.size());
}
const int n_e = n_embd;
// sequence order is voice, then text, then the audio BOS that starts generation
if (!voice.empty()) {
GGML_ASSERT(voice.size() % (size_t) n_e == 0);
if (bos_before_voice != LLAMA_TOKEN_NULL) {
push_embd_row(prompt_embd_buf, bos_before_voice);
}
prompt_embd_buf.insert(prompt_embd_buf.end(), voice.begin(), voice.end());
}
// every later chunk rewinds to here and re-prompts, so the voice stays primed
n_voice_pos = (int) (prompt_embd_buf.size() / (size_t) n_e);
for (llama_token t : chunks[0]) {
push_embd_row(prompt_embd_buf, t);
}
push_embd_row(prompt_embd_buf, audio_bos);
arm_chunk_budget(0);
n_prompt = (int) (prompt_embd_buf.size() / (size_t) n_e);
prompt_batch.reset(new decode_embd_batch(prompt_embd_buf.data(), n_prompt, 1, n_e));
prompt_batch->set_position_normal(0, seq_id);
prompt_pos = 0;
temp = inp->temp;
out_type = inp->out_type;
stream = inp->stream;
if (!init_gen_state(inp->seed)) {
return 1;
}
return 0;
}
int32_t step_prompt(int32_t n_batch) override {
GGML_ASSERT(n_batch > 0);
if (prompt_pos >= n_prompt) {
return 0;
}
const int32_t n_tokens_batch = std::min(n_batch, n_prompt - prompt_pos);
if ((prompt_pos + n_tokens_batch) == n_prompt) {
prompt_batch->logits[prompt_pos + n_tokens_batch - 1] = 1;
}
if (llama_process(lctx, LLAMA_PROCESS_TYPE_DECODE, prompt_batch->render(lctx, prompt_pos, n_tokens_batch)) != 0) {
LOG_ERR("mtmd_helper_gen_audio: prompt decode failed\n");
return -1;
}
pos += n_tokens_batch;
prompt_pos += n_tokens_batch;
if (prompt_pos >= n_prompt) {
prompt_batch.reset();
prompt_embd_buf.clear();
return 0;
}
return n_prompt - prompt_pos;
}
int32_t step_gen(llama_token sampled, const float * h_state_in, const float ** h_state_out, bool * out_stop) override {
(void) sampled; // the backbone output is continuous, there is no token to consume
mtmd_gen_inp inp = mtmd_gen_inp_default(mctx);
inp.type = MTMD_GEN_PROCESS_TYPE_GEN_CODE;
inp.embd = const_cast<float *>(h_state_in);
if (temp >= 0.0f) {
inp.temp = temp;
}
mtmd_gen_out out{};
if (mtmd_gen_audio_process(mctx, gen_state.get(), &inp, &out) != 0) {
LOG_ERR("mtmd_helper_gen_audio: flow decode failed\n");
return 1;
}
if (out.is_eos && eos_step < 0) {
eos_step = step_idx;
}
// the frame of the stopping step is discarded, matching _autoregressive_generation().
// the budget is the reference's fallback for a chunk whose eos head never fires
const bool chunk_done = (eos_step >= 0 && step_idx >= eos_step + frames_after_eos) ||
step_idx >= chunk_budget;
if (chunk_done) {
if (eos_step < 0) {
LOG_WRN("mtmd_helper_gen_audio: chunk %zu hit its budget without end-of-speech\n", chunk_idx);
}
return finish_chunk(h_state_out, out_stop);
}
feats_buf.insert(feats_buf.end(), out.feats, out.feats + out.n_feats);
step_idx++;
if (out.n_feats > 0 && feats_buf.size() / out.n_feats >= window_frames) {
if (!flush_gen_wav()) {
return 1;
}
}
decode_embd_batch batch_embd(out.embd, 1, 1, n_embd);
batch_embd.set_position_normal(pos, seq_id);
batch_embd.logits[0] = 1;
pos++;
if (llama_process(lctx, LLAMA_PROCESS_TYPE_DECODE, batch_embd.render(lctx, 0, 1)) != 0) {
LOG_ERR("mtmd_helper_gen_audio: decode failed\n");
return 1;
}
const float * he = llama_get_embeddings_ith(lctx, -1);
h_state_buf.assign(he, he + n_embd);
*h_state_out = h_state_buf.data();
return 0;
}
int32_t get_output(int32_t * out_sample_rate, const char ** out_data, size_t * out_data_len, int64_t * out_n_samples) override {
*out_sample_rate = info.sample_rate;
if (!stream) {
// one-shot call: force out whatever's left, regardless of window_frames
if (!flush_gen_wav()) {
return 1;
}
if (out_n_samples) {
*out_n_samples = (int64_t) audio_pcm.size();
}
if (out_type == MTMD_HELPER_GEN_AUDIO_OUTTYPE_PCM) {
*out_data = (const char *) audio_pcm.data();
*out_data_len = audio_pcm.size() * sizeof(float);
return 0;
}
out_buf.clear();
if (!write_wav16(out_buf, audio_pcm, info.sample_rate)) {
LOG_ERR("mtmd_helper_gen_audio: output too large for WAV\n");
return 1;
}
*out_data = out_buf.data();
*out_data_len = out_buf.size();
return 0;
}
// streaming: only return audio produced since the previous call
const size_t n_new = audio_pcm.size() - pcm_sent;
if (out_n_samples) {
*out_n_samples = (int64_t) n_new;
}
if (out_type == MTMD_HELPER_GEN_AUDIO_OUTTYPE_PCM) {
*out_data = (const char *) (audio_pcm.data() + pcm_sent);
*out_data_len = n_new * sizeof(float);
pcm_sent = audio_pcm.size();
return 0;
}
out_buf.clear();
if (!wav_header_sent) {
write_wav16_header(out_buf, UINT32_MAX, info.sample_rate);
wav_header_sent = true;
}
append_wav16_pcm(out_buf, audio_pcm.data() + pcm_sent, n_new);
pcm_sent = audio_pcm.size();
*out_data = out_buf.data();
*out_data_len = out_buf.size();
return 0;
}
int32_t flush() override {
return flush_gen_wav() ? 0 : 1;
}
private:
bool ensure_cache() {
if (specials_ok) {
return true;
}
// bos_before_voice is optional, some packs do not insert it
bos_before_voice = find_special_token(vocab, "<|bos_before_voice|>");
audio_bos = find_special_token(vocab, "<|audio_bos|>");
if (audio_bos == LLAMA_TOKEN_NULL) {
LOG_ERR("mtmd_helper_gen_audio: missing <|audio_bos|> in vocab\n");
return false;
}
const uint32_t n_tok_embd = llama_model_get_tok_embd(model, nullptr);
if (n_tok_embd == 0) {
LOG_ERR("mtmd_helper_gen_audio: model has no token embeddings\n");
return false;
}
tok_embd.resize(n_tok_embd);
if (llama_model_get_tok_embd(model, tok_embd.data()) != n_tok_embd) {
LOG_ERR("mtmd_helper_gen_audio: token embedding copy failed\n");
return false;
}
GGML_ASSERT(n_embd > 0 && n_tok_embd % (uint32_t) n_embd == 0);
specials_ok = true;
return true;
}
// the table can be shorter than the vocab, so bound the row lookup
void push_embd_row(std::vector<float> & dst, llama_token t) const {
const size_t n_rows = tok_embd.size() / (size_t) n_embd;
GGML_ASSERT(t >= 0 && (size_t) t < n_rows);
dst.insert(dst.end(),
tok_embd.begin() + (size_t) t * n_embd,
tok_embd.begin() + (size_t) (t + 1) * n_embd);
}
// token ids of the pieces the reference splits on, see split_into_best_sentences().
// the leading token is dropped, it is the tokenizer's dummy prefix
std::vector<llama_token> punct_ids(const char * s) const {
std::vector<llama_token> ids(16);
const int n = llama_tokenize(vocab, s, (int32_t) strlen(s), ids.data(), (int32_t) ids.size(), false, false);
if (n <= 1) {
return {};
}
return std::vector<llama_token>(ids.begin() + 1, ids.begin() + n);
}
// cut after runs of boundary tokens, so punctuation stays with the sentence it ends
static std::vector<std::vector<llama_token>> split_on(const std::vector<llama_token> & ids,
const std::vector<llama_token> & boundary) {
std::vector<std::vector<llama_token>> out;
size_t start = 0;
bool prev_was_boundary = false;
for (size_t i = 0; i < ids.size(); i++) {
const bool is_boundary = std::find(boundary.begin(), boundary.end(), ids[i]) != boundary.end();
if (!is_boundary && prev_was_boundary) {
out.emplace_back(ids.begin() + start, ids.begin() + i);
start = i;
}
prev_was_boundary = is_boundary;
}
out.emplace_back(ids.begin() + start, ids.end());
return out;
}
std::vector<std::vector<llama_token>> split_chunks(const std::vector<llama_token> & ids) const {
if ((int) ids.size() <= max_chunk_tokens) {
return { ids };
}
const std::vector<llama_token> eos_punct = punct_ids(".!...?");
const std::vector<llama_token> mid_punct = punct_ids(",;:");
// oversized sentences are split again on weaker punctuation, else words get skipped
std::vector<std::vector<llama_token>> segments;
for (auto & seg : split_on(ids, eos_punct)) {
if ((int) seg.size() <= max_chunk_tokens) {
segments.push_back(std::move(seg));
continue;
}
auto sub = split_on(seg, mid_punct);
if (sub.size() > 1) {
for (auto & s : sub) {
segments.push_back(std::move(s));
}
} else {
segments.push_back(std::move(seg));
}
}
std::vector<std::vector<llama_token>> out;
for (auto & seg : segments) {
if (seg.empty()) {
continue;
}
if (!out.empty() && (int) (out.back().size() + seg.size()) <= max_chunk_tokens) {
out.back().insert(out.back().end(), seg.begin(), seg.end());
} else {
out.push_back(std::move(seg));
}
}
if (out.empty()) {
out.push_back(ids);
}
for (const auto & c : out) {
if ((int) c.size() > max_chunk_tokens) {
LOG_WRN("mtmd_helper_gen_audio: chunk of %zu tokens exceeds the %d token budget, "
"generation may skip words\n", c.size(), max_chunk_tokens);
}
}
return out;
}
// _estimate_max_gen_len() plus the per-chunk tail guess, both in frames
void arm_chunk_budget(size_t idx) {
const int n_tok = (int) chunks[idx].size();
chunk_budget = (int) std::ceil((n_tok / 3.0 + 2.0) * frame_rate);
// the pack may pin the tail, else the reference guesses it from the word count
frames_after_eos = pack.frames_after_eos > 0 ? pack.frames_after_eos : (n_tok <= 6 ? 5 : 3);
step_idx = 0;
eos_step = -1;
}
// ends the current chunk and, if there is another, re-prompts it on top of the voice
int32_t finish_chunk(const float ** h_state_out, bool * out_stop) {
if (!flush_gen_wav()) {
return 1;
}
// the decoder restarts too, the next chunk's audio is not continuous with this one
mtmd_gen_state_reset(gen_state.get());
if (chunk_idx + 1 >= chunks.size()) {
*out_stop = true;
*h_state_out = nullptr;
return 0;
}
chunk_idx++;
// drop this chunk's text and audio, keep the voice conditioning
llama_memory_seq_rm(llama_get_memory(lctx), seq_id, n_voice_pos, -1);
pos = n_voice_pos;
const int n_e = n_embd;
prompt_embd_buf.clear();
for (llama_token t : chunks[chunk_idx]) {
push_embd_row(prompt_embd_buf, t);
}
push_embd_row(prompt_embd_buf, audio_bos);
arm_chunk_budget(chunk_idx);
const int n_rows = (int) (prompt_embd_buf.size() / (size_t) n_e);
GGML_ASSERT(n_rows > 0);
decode_embd_batch batch(prompt_embd_buf.data(), n_rows, 1, n_e);
batch.set_position_normal(pos, seq_id);
batch.logits[n_rows - 1] = 1;
if (llama_process(lctx, LLAMA_PROCESS_TYPE_DECODE, batch.render(lctx, 0, n_rows)) != 0) {
LOG_ERR("mtmd_helper_gen_audio: chunk prompt decode failed\n");
return 1;
}
pos += n_rows;
prompt_embd_buf.clear();
const float * he = llama_get_embeddings_ith(lctx, -1);
h_state_buf.assign(he, he + n_embd);
*h_state_out = h_state_buf.data();
*out_stop = false;
return 0;
}
// same normalization as prepare_text_prompt() in the reference, it affects quality
static std::string prepare_text(const std::string & in, bool pad_short) {
std::string s;
s.reserve(in.size() + 1);
for (char c : in) {
if (c == '\n' || c == '\r') {
s += ' ';
} else if (c == ';') {
s += ',';
} else {
s += c;
}
}
const size_t b = s.find_first_not_of(' ');
const size_t e = s.find_last_not_of(' ');
if (b == std::string::npos) {
return "";
}
s = s.substr(b, e - b + 1);
if (s[0] >= 'a' && s[0] <= 'z') {
s[0] = (char) (s[0] - 'a' + 'A');
}
const unsigned char last = (unsigned char) s.back();
if (std::isalnum(last)) {
s += '.';
}
if (pad_short && count_words(s) < 5) {
s = std::string(8, ' ') + s;
}
return s;
}
static int count_words(const std::string & s) {
int n = 0;
bool in_word = false;
for (char c : s) {
if (c == ' ') {
in_word = false;
} else if (!in_word) {
in_word = true;
n++;
}
}
return n;
}
// decodes the buffered latents, the mimi decoder state carries over between calls
bool flush_gen_wav() {
if (feats_buf.empty()) {
return true;
}
mtmd_gen_inp inp = mtmd_gen_inp_default(mctx);
inp.type = MTMD_GEN_PROCESS_TYPE_GEN_WAV;
inp.feats = feats_buf.data();
inp.n_feats = feats_buf.size();
mtmd_gen_out out{};
if (mtmd_gen_audio_process(mctx, gen_state.get(), &inp, &out) != 0) {
LOG_ERR("mtmd_helper_gen_audio: mimi decode failed\n");
return false;
}
audio_pcm.insert(audio_pcm.end(), out.audio, out.audio + out.n_samples);
feats_buf.clear();
return true;
}
pockettts_pack_settings pack;
float temp = -1.0f; // the caller's flow noise scale, < 0 for the model default
bool specials_ok = false;
llama_token bos_before_voice = LLAMA_TOKEN_NULL;
llama_token audio_bos = LLAMA_TOKEN_NULL;
std::vector<float> tok_embd;
llama_seq_id seq_id = 0;
int pos = 0;
std::vector<float> prompt_embd_buf;
std::unique_ptr<decode_embd_batch> prompt_batch;
int n_prompt = 0;
int prompt_pos = 0;
// end-of-speech is latched, then a few more frames are generated as tail padding
int step_idx = 0;
int eos_step = -1;
int frames_after_eos = 3;
static constexpr int max_chunk_tokens = 50; // MAX_TOKEN_PER_CHUNK in the reference
static constexpr double frame_rate = 12.5;
std::vector<std::vector<llama_token>> chunks;
size_t chunk_idx = 0;
int n_voice_pos = 0; // KV positions held by the voice conditioning
int chunk_budget = 0;
// latents are decoded a window at a time, the decoder state bridges the windows
size_t window_frames = 8;
std::vector<float> feats_buf;
std::vector<float> audio_pcm;
std::vector<float> h_state_buf;
mtmd_helper_gen_audio_outtype out_type = MTMD_HELPER_GEN_AUDIO_OUTTYPE_WAV;
std::vector<char> out_buf;
bool stream = false;
size_t pcm_sent = 0; // samples already returned by get_output()
bool wav_header_sent = false;
};
// Soprano uses the hidden state that predicts each non-EOS token, including the prompt's last state.
class soprano_gen_audio_pipeline : public mtmd_gen_audio_pipeline {
public:
using mtmd_gen_audio_pipeline::mtmd_gen_audio_pipeline;
void reset() override {
prompt.clear();
features.clear();
h_next.clear();
audio.clear();
wav.clear();
pos = 0;
prompt_pos = 0;
decoded = false;
sent = false;
}
int32_t set_input(const mtmd_helper_gen_audio_inp * inp) override {
reset();
if (inp->speaker_ref || !inp->prompt || inp->prompt_len == 0 || inp->prompt_len > INT32_MAX ||
inp->seq_id < 0 || (uint32_t) inp->seq_id >= llama_n_seq_max(lctx)) {
LOG_ERR("mtmd_helper_gen_audio: soprano requires text, without a speaker reference\n");
return 1;
}
seq_id = inp->seq_id;
out_type = inp->out_type;
stream = inp->stream;
const llama_token text = find_special_token(vocab, "[TEXT]");
const llama_token start = find_special_token(vocab, "[START]");
const llama_token stop = find_special_token(vocab, "[STOP]");
if (text == LLAMA_TOKEN_NULL || start == LLAMA_TOKEN_NULL || stop == LLAMA_TOKEN_NULL) {
LOG_ERR("mtmd_helper_gen_audio: incompatible soprano vocabulary\n");
return 1;
}
const int required = llama_tokenize(vocab, inp->prompt, inp->prompt_len, nullptr, 0, false, false);
if (required >= 0 || (uint32_t) (-required + 3) >= llama_n_ctx_seq(lctx)) {
LOG_ERR("mtmd_helper_gen_audio: soprano prompt does not fit in the context\n");
return 1;
}
const int n = -required;
prompt.resize(n + 3);
prompt[0] = stop;
prompt[1] = text;
if (llama_tokenize(vocab, inp->prompt, inp->prompt_len, prompt.data() + 2, n, false, false) != n) {
return 1;
}
prompt.back() = start;
if (!llama_memory_seq_rm(llama_get_memory(lctx), seq_id, 0, -1)) {
LOG_ERR("mtmd_helper_gen_audio: cannot reset soprano sequence\n");
return 1;
}
if (!init_gen_state(inp->seed)) {
return 1;
}
return 0;
}
int32_t step_prompt(int32_t n_batch) override {
if (n_batch <= 0 || prompt.empty()) {
return -1;
}
const int n = std::min(n_batch, (int) prompt.size() - prompt_pos);
if (n == 0) {
return 0;
}
if (!decode(prompt.data() + prompt_pos, n)) {
return -1;
}
prompt_pos += n;
return (int) prompt.size() - prompt_pos;
}
int32_t step_gen(llama_token sampled, const float * h_state_in, const float ** h_state_out, bool * out_stop) override {
*h_state_out = nullptr;
*out_stop = false;
if (sampled < 0 || sampled >= llama_vocab_n_tokens(vocab) || !h_state_in || prompt_pos != (int) prompt.size() || prompt.empty()) {
return 1;
}
if (llama_vocab_is_eog(vocab, sampled) || (uint32_t) pos >= llama_n_ctx_seq(lctx)) {
*out_stop = true;
return 0;
}
features.insert(features.end(), h_state_in, h_state_in + n_embd);
if (!decode(&sampled, 1)) {
features.resize(features.size() - n_embd);
return 1;
}
const float * h = llama_get_embeddings_ith(lctx, -1);
if (!h) {
return 1;
}
h_next.assign(h, h + n_embd);
*h_state_out = h_next.data();
return 0;
}
int32_t get_output(int32_t * sample_rate, const char ** data, size_t * data_len, int64_t * n_samples) override {
*sample_rate = info.sample_rate;
*data_len = 0;
if (n_samples) { *n_samples = 0; }
if (stream) {
// the decoder needs the whole utterance, so the audio comes out in one chunk after flush()
if (!decoded || sent) {
return 0;
}
sent = true;
} else if (!decode_audio()) {
return 1;
}
if (n_samples) { *n_samples = audio.size(); }
if (out_type == MTMD_HELPER_GEN_AUDIO_OUTTYPE_PCM) {
*data = (const char *) audio.data();
*data_len = audio.size() * sizeof(float);
} else {
wav.clear();
if (!write_wav16(wav, audio, info.sample_rate)) {
return 1;
}
*data = wav.data();
*data_len = wav.size();
}
return 0;
}
// the whole utterance is decoded at once
int32_t flush() override {
return decode_audio() ? 0 : 1;
}
private:
bool decode_audio() {
if (decoded) {
return true;
}
if (features.size() < 2 * (size_t) n_embd) {
LOG_ERR("mtmd_helper_gen_audio: soprano needs at least two generated frames\n");
return false;
}
mtmd_gen_inp inp = mtmd_gen_inp_default(mctx);
inp.type = MTMD_GEN_PROCESS_TYPE_GEN_WAV;
inp.feats = features.data();
inp.n_feats = features.size();
mtmd_gen_out out{};
if (mtmd_gen_audio_process(mctx, gen_state.get(), &inp, &out) != 0) {
return false;
}
audio.assign(out.audio, out.audio + out.n_samples);
decoded = true;
return true;
}
bool decode(llama_token * tokens, int n) {
llama_batch_ext_ptr batch(llama_batch_ext_init(lctx));
if (!batch_set_tokens(batch.get(), tokens, n, pos, seq_id)) {
return false;
}
llama_batch_ext_set_output_logits(batch.get(), n - 1, true);
if (llama_process(lctx, LLAMA_PROCESS_TYPE_DECODE, batch.get()) != 0) {
LOG_ERR("mtmd_helper_gen_audio: soprano backbone decode failed\n");
return false;
}
pos += n;
return true;
}
llama_seq_id seq_id = 0;
llama_pos pos = 0;
int prompt_pos = 0;
std::vector<llama_token> prompt;
std::vector<float> features;
std::vector<float> h_next;
std::vector<float> audio;
std::vector<char> wav;
mtmd_helper_gen_audio_outtype out_type = MTMD_HELPER_GEN_AUDIO_OUTTYPE_WAV;
bool stream = false;
bool decoded = false;
bool sent = false;
};
// LFM3-Audio TTS, see generate() in the reference: each step samples a frame of codes by MaskGIT
// and feeds its embedding back, the end-of-audio code stops
class lfm3a_gen_audio_pipeline : public mtmd_gen_audio_pipeline {
public:
using mtmd_gen_audio_pipeline::mtmd_gen_audio_pipeline;
void reset() override {
seq_id = 0;
pos = 0;
continuing = false;
mode = MODE_AUDIO;
slots_left = 0;
speaking = false;
text_done = false;
prompt_tokens.clear();
prompt_pos = 0;
hist.clear();
gen_state.reset();
pending_tail = false;
pcm_sent = 0;
wav_header_sent = false;
audio_pcm.clear();
h_state_buf.clear();
out_buf.clear();
}
int32_t set_input(const mtmd_helper_gen_audio_inp * inp) override {
reset();
seq_id = inp->seq_id;
// a voice is named in the prompt instead
if (inp->speaker_ref && !mtmd_bitmap_is_voice(inp->speaker_ref)) {
LOG_WRN("mtmd_helper_gen_audio: the model has no voice conditioning, the speaker file is ignored\n");
}
// the prompt is rendered by the caller: the "tts" chat template of the model, or the next turn of a conversation
continuing = inp->n_past > 0;
pos = continuing ? inp->n_past : 0;
const std::string full(inp->prompt, inp->prompt_len);
prompt_tokens.resize(full.size() + 16);
const int n_ids = llama_tokenize(vocab, full.c_str(), (int32_t) full.size(), prompt_tokens.data(),
(int32_t) prompt_tokens.size(), false, true);
if (n_ids <= 0) {
LOG_ERR("mtmd_helper_gen_audio: tokenization failed\n");
return 1;
}
prompt_tokens.resize((size_t) n_ids);
if (!find_tokens()) {
return 1;
}
// a continued response opens with the token of the asked mode, a text-only one cannot switch to audio
if (continuing) {
switch (inp->out_mode) {
case MTMD_HELPER_GEN_AUDIO_OUT_AUDIO: prompt_tokens.push_back(tok_audio_start); break;
case MTMD_HELPER_GEN_AUDIO_OUT_MIXED: prompt_tokens.push_back(tok_mixed_start); break;
case MTMD_HELPER_GEN_AUDIO_OUT_TEXT:
logit_bias = { { tok_audio_start, -INFINITY }, { tok_mixed_start, -INFINITY } };
break;
default: break;
}
}
// the last prompt token picks the mode, see generate() of the reference
const llama_token last = prompt_tokens.back();
mode = last == tok_audio_start ? MODE_AUDIO : last == tok_mixed_start ? MODE_MIXED : MODE_TEXT;
slots_left = n_text_block;
if (!continuing && mode != MODE_AUDIO) {
LOG_ERR("mtmd_helper_gen_audio: the prompt must come from the \"tts\" chat template of the model\n");
return 1;
}
// the sampling defaults of the model, the caller can only override the temperature
const mtmd_gen_inp def = mtmd_gen_inp_default(mctx);
temp = inp->temp >= 0.0f ? inp->temp : def.temp;
top_k = def.top_k;
rep_penalty = def.penalty_repeat;
rep_window = def.penalty_last_n;
out_type = inp->out_type;
stream = inp->stream;
rng.seed(inp->seed == UINT32_MAX ? std::random_device{}() : inp->seed);
return init_gen_state(inp->seed) ? 0 : 1;
}
int32_t step_prompt(int32_t n_batch) override {
GGML_ASSERT(n_batch > 0);
const int n_prompt = (int) prompt_tokens.size();
if (prompt_pos >= n_prompt) {
return 0;
}
const int n = std::min(n_batch, n_prompt - prompt_pos);
llama_batch_ext_ptr batch(llama_batch_ext_init(lctx));
if (!batch_set_tokens(batch.get(), prompt_tokens.data() + prompt_pos, n, pos, seq_id)) {
return -1;
}
if (prompt_pos + n == n_prompt) {
llama_batch_ext_set_output_logits(batch.get(), n - 1, true);
}
if (llama_process(lctx, LLAMA_PROCESS_TYPE_DECODE, batch.get()) != 0) {
LOG_ERR("mtmd_helper_gen_audio: prompt decode failed\n");
return -1;
}
pos += n;
prompt_pos += n;
return n_prompt - prompt_pos;
}
bool supports_continue() const override {
return true;
}
bool next_is_text() const override {
return mode == MODE_TEXT || (mode == MODE_MIXED && !speaking);
}
// one generate() step: a text token from the caller or an audio frame from the hidden state
int32_t step_gen(llama_token sampled, const float * h_state_in, const float ** h_state_out, bool * out_stop) override {
const bool interleaving = mode == MODE_MIXED;
if (next_is_text()) {
// <|im_end|> ends the response in any text slot, <|text_end|> outside an interleaved block too
if (sampled == tok_im_end || (!interleaving && sampled == tok_text_end)) {
*out_stop = true;
*h_state_out = nullptr;
return 0;
}
// the switching tokens are fed back like any text token
if (!interleaving) {
if (sampled == tok_audio_start) {
mode = MODE_AUDIO;
} else if (sampled == tok_mixed_start) {
mode = MODE_MIXED;
slots_left = n_text_block;
speaking = false;
text_done = false;
}
} else if (sampled == tok_text_end) {
text_done = true;
speaking = true;
}
if (!decode(sampled)) {
return 1;
}
} else {
std::vector<int32_t> codes;
const float * fb = nullptr;
bool is_eos = false;
if (!sample_frame(h_state_in, codes, &fb, &is_eos)) {
return 1;
}
if (is_eos) {
if (!continuing) {
// TTS: the end of audio ends the generation
*out_stop = true;
*h_state_out = nullptr;
return 0;
}
// the segment ends, the codec tail goes out and the response continues in text
if (flush() != 0) {
return 1;
}
mode = MODE_TEXT;
} else {
hist.insert(hist.end(), codes.begin(), codes.end());
if (!decode_wav(codes.data(), codes.size())) {
return 1;
}
}
decode_embd_batch batch_embd(const_cast<float *>(fb), 1, 1, n_embd);
batch_embd.set_position_normal(pos, seq_id);
batch_embd.logits[0] = 1;
if (llama_process(lctx, LLAMA_PROCESS_TYPE_DECODE, batch_embd.render(lctx, 0, 1)) != 0) {
LOG_ERR("mtmd_helper_gen_audio: decode failed\n");
return 1;
}
pos++;
}
// a slot emitted while interleaving counts down the block, the switching token itself does not
if (interleaving && !text_done && --slots_left == 0) {
speaking = !speaking;
slots_left = speaking ? n_audio_block : n_text_block;
}
const float * he = llama_get_embeddings_ith(lctx, -1);
h_state_buf.assign(he, he + n_embd);
*h_state_out = h_state_buf.data();
return 0;
}
int32_t get_output(int32_t * out_sample_rate, const char ** out_data, size_t * out_data_len, int64_t * out_n_samples) override {
*out_sample_rate = info.sample_rate;
if (!stream) {
if (flush() != 0) {
return 1;
}
if (out_n_samples) {
*out_n_samples = (int64_t) audio_pcm.size();
}
if (out_type == MTMD_HELPER_GEN_AUDIO_OUTTYPE_PCM) {
*out_data = (const char *) audio_pcm.data();
*out_data_len = audio_pcm.size() * sizeof(float);
return 0;
}
out_buf.clear();
if (!write_wav16(out_buf, audio_pcm, info.sample_rate)) {
LOG_ERR("mtmd_helper_gen_audio: output too large for WAV\n");
return 1;
}
*out_data = out_buf.data();
*out_data_len = out_buf.size();
return 0;
}
// streaming: only the audio produced since the previous call, every frame is decoded as it is sampled
const size_t n_new = audio_pcm.size() - pcm_sent;
if (out_n_samples) {
*out_n_samples = (int64_t) n_new;
}
if (out_type == MTMD_HELPER_GEN_AUDIO_OUTTYPE_PCM) {
*out_data = (const char *) (audio_pcm.data() + pcm_sent);
*out_data_len = n_new * sizeof(float);
pcm_sent = audio_pcm.size();
return 0;
}
out_buf.clear();
if (!wav_header_sent) {
write_wav16_header(out_buf, UINT32_MAX, info.sample_rate);
wav_header_sent = true;
}
append_wav16_pcm(out_buf, audio_pcm.data() + pcm_sent, n_new);
pcm_sent = audio_pcm.size();
*out_data = out_buf.data();
*out_data_len = out_buf.size();
return 0;
}
// the ISTFT holds back the overlap of the last frame
int32_t flush() override {
if (pending_tail) {
if (!decode_wav(nullptr, 0)) {
return 1;
}
pending_tail = false;
}
return 0;
}
private:
// checked by the converter
static constexpr int n_cb = 12;
static constexpr int schedule[] = { 4, 4, 4 }; // codebooks revealed per depthformer pass
// a random order of the codebooks, revealed by the schedule; fb is the feedback embedding of the codes.
// at the end of audio every code is the end code, its feedback embedding is only built when continuing
bool sample_frame(const float * h_state, std::vector<int32_t> & codes, const float ** fb, bool * is_eos) {
std::vector<int> order(n_cb);
for (int i = 0; i < n_cb; i++) {
order[(size_t) i] = i;
}
std::shuffle(order.begin(), order.end(), rng);
codes.assign(n_cb, -1);
mtmd_gen_inp inp = mtmd_gen_inp_default(mctx);
inp.type = MTMD_GEN_PROCESS_TYPE_GEN_CODE;
inp.embd = const_cast<float *>(h_state);
inp.codes = codes.data();
inp.n_codes = codes.size();
mtmd_gen_out out{};
int k = 0;
for (int n_pick : schedule) {
if (mtmd_gen_audio_process(mctx, gen_state.get(), &inp, &out) != 0 || out.n_feats == 0 || out.n_feats % n_cb != 0) {
LOG_ERR("mtmd_helper_gen_audio: depthformer pass failed\n");
return false;
}
const size_t n_vocab = out.n_feats / n_cb;
logits.assign(out.feats, out.feats + out.n_feats);
for (int j = 0; j < n_pick; j++, k++) {
const int i = order[(size_t) k];
float * l = logits.data() + (size_t) i * n_vocab;
apply_rep_penalty(l, n_vocab, i);
codes[(size_t) i] = sample(l, n_vocab);
}
}
// the last code of every codebook ends the audio
const int32_t eos = (int32_t) (logits.size() / n_cb) - 1;
*is_eos = std::find(codes.begin(), codes.end(), eos) != codes.end();
if (*is_eos) {
codes.assign(n_cb, eos);
if (!continuing) {
*fb = nullptr;
return true;
}
}
// all codes known: only the feedback embedding
if (mtmd_gen_audio_process(mctx, gen_state.get(), &inp, &out) != 0) {
LOG_ERR("mtmd_helper_gen_audio: codebook embedding failed\n");
return false;
}
fb_buf.assign(out.embd, out.embd + n_embd);
*fb = fb_buf.data();
return true;
}
// CTRL penalty over the codes codebook cb took in the last rep_window frames
void apply_rep_penalty(float * l, size_t n_vocab, int cb) const {
if (rep_penalty <= 1.0f || rep_window <= 0) {
return;
}
const size_t n_hist = hist.size() / n_cb;
const size_t n_win = std::min(n_hist, (size_t) rep_window);
std::vector<bool> seen(n_vocab, false);
for (size_t f = n_hist - n_win; f < n_hist; f++) {
const int32_t c = hist[f * n_cb + (size_t) cb];
if (c >= 0 && (size_t) c < n_vocab && !seen[(size_t) c]) {
seen[(size_t) c] = true;
l[c] = l[c] > 0.0f ? l[c] / rep_penalty : l[c] * rep_penalty;
}
}
}
// TODO: reuse the samplers of common (top-k, temperature, penalties) instead of this one
int32_t sample(const float * l, size_t n_vocab) {
if (temp <= 0.0f) {
return (int32_t) (std::max_element(l, l + n_vocab) - l);
}
// top-k keeps the codes scoring at least the k-th best, ties included
float lmin = -INFINITY;
if (top_k > 0 && (size_t) top_k < n_vocab) {
std::vector<float> sorted(l, l + n_vocab);
std::nth_element(sorted.begin(), sorted.begin() + (top_k - 1), sorted.end(), std::greater<float>());
lmin = sorted[(size_t) top_k - 1];
}
const float lmax = *std::max_element(l, l + n_vocab);
std::vector<double> prob(n_vocab);
double sum = 0.0;
for (size_t v = 0; v < n_vocab; v++) {
prob[v] = l[v] < lmin ? 0.0 : std::exp((double) (l[v] - lmax) / temp);
sum += prob[v];
}
double r = std::uniform_real_distribution<double>(0.0, 1.0)(rng) * sum;
for (size_t v = 0; v < n_vocab; v++) {
r -= prob[v];
if (r <= 0.0) {
return (int32_t) v;
}
}
return (int32_t) n_vocab - 1;
}
// n_codes == 0 flushes the ISTFT tail
bool decode_wav(const int32_t * codes, size_t n_codes) {
mtmd_gen_inp inp = mtmd_gen_inp_default(mctx);
inp.type = MTMD_GEN_PROCESS_TYPE_GEN_WAV;
inp.codes = const_cast<int32_t *>(codes);
inp.n_codes = n_codes;
mtmd_gen_out out{};
if (mtmd_gen_audio_process(mctx, gen_state.get(), &inp, &out) != 0) {
LOG_ERR("mtmd_helper_gen_audio: codec decode failed\n");
return false;
}
audio_pcm.insert(audio_pcm.end(), out.audio, out.audio + out.n_samples);
pending_tail = n_codes > 0;
return true;
}
bool find_tokens() {
tok_audio_start = find_special_token(vocab, "<|audio_start|>");
tok_mixed_start = find_special_token(vocab, "<|mixed_start|>");
tok_text_end = find_special_token(vocab, "<|text_end|>");
tok_im_end = find_special_token(vocab, "<|im_end|>");
if (tok_audio_start == LLAMA_TOKEN_NULL || tok_mixed_start == LLAMA_TOKEN_NULL ||
tok_text_end == LLAMA_TOKEN_NULL || tok_im_end == LLAMA_TOKEN_NULL) {
LOG_ERR("mtmd_helper_gen_audio: missing LFM3-Audio control tokens in vocab\n");
return false;
}
return true;
}
bool decode(llama_token token) {
llama_batch_ext_ptr batch(llama_batch_ext_init(lctx));
if (!batch_set_tokens(batch.get(), &token, 1, pos, seq_id)) {
return false;
}
llama_batch_ext_set_output_logits(batch.get(), 0, true);
if (llama_process(lctx, LLAMA_PROCESS_TYPE_DECODE, batch.get()) != 0) {
LOG_ERR("mtmd_helper_gen_audio: decode failed\n");
return false;
}
pos += 1;
return true;
}
// generate() modes: text, audio until end of audio, interleaved until <|text_end|>
enum gen_mode { MODE_TEXT, MODE_AUDIO, MODE_MIXED };
// checked by the converter
static constexpr int n_text_block = 6; // interleaved_n_text
static constexpr int n_audio_block = 12; // interleaved_n_audio
llama_token tok_audio_start = LLAMA_TOKEN_NULL;
llama_token tok_mixed_start = LLAMA_TOKEN_NULL;
llama_token tok_text_end = LLAMA_TOKEN_NULL;
llama_token tok_im_end = LLAMA_TOKEN_NULL;
bool continuing = false; // the prompt continues a sequence, the response follows the model's switching
gen_mode mode = MODE_AUDIO;
int slots_left = 0;
bool speaking = false; // interleaving: the current block is audio
bool text_done = false; // interleaving: <|text_end|> was emitted, audio until the end-of-audio frame
llama_seq_id seq_id = 0;
int pos = 0;
std::vector<llama_token> prompt_tokens;
int prompt_pos = 0;
float temp = 0.0f;
int32_t top_k = 0;
float rep_penalty = 1.0f;
int32_t rep_window = 0; // frames
std::mt19937 rng;
std::vector<float> logits;
std::vector<float> fb_buf;
std::vector<int32_t> hist; // frame-major codes of the frames so far
bool pending_tail = false; // the ISTFT holds back the overlap of the last frame
std::vector<float> audio_pcm;
std::vector<float> h_state_buf;
mtmd_helper_gen_audio_outtype out_type = MTMD_HELPER_GEN_AUDIO_OUTTYPE_WAV;
std::vector<char> out_buf;
bool stream = false;
size_t pcm_sent = 0; // samples already returned by get_output()
bool wav_header_sent = false;
};
// Chatterbox: the backbone emits s3 speech tokens, then one flow-matching mel decode and the NSF-iSTFT vocoder make the PCM
// the multilingual variant runs cfg with an uncond sequence (text rows zeroed), paired as seq_id and n_seq_max / 2 + seq_id
class chatterbox_gen_audio_pipeline : public mtmd_gen_audio_pipeline {
public:
using mtmd_gen_audio_pipeline::mtmd_gen_audio_pipeline;
void reset() override {
seq_id = 0;
seq_uncond = 0;
pos = 0;
ar_idx = 0;
codes_buf.clear();
audio_pcm.clear();
h_state_buf.clear();
out_buf.clear();
prompt_embd_buf.clear();
uncond_embd_buf.clear();
cond_logits.clear();
n_prompt = 0;
prompt_pos = 0;
decoded = false;
stream = false;
pcm_sent = 0;
wav_header_sent = false;
gen_state.reset();
}
int32_t set_input(const mtmd_helper_gen_audio_inp * inp) override {
reset();
seq_id = inp->seq_id;
if (!ensure_cache()) {
return 1;
}
out_type = inp->out_type;
stream = inp->stream;
if (!init_gen_state(inp->seed)) {
return 1;
}
if (mtl) {
// the uncond sequence lives half a seq space away from the cond one
const int32_t n_seq_max = (int32_t) llama_n_seq_max(lctx);
seq_uncond = (llama_seq_id) (n_seq_max / 2 + seq_id);
if (!llama_model_classifier_free_guidance(llama_get_model(lctx)) || seq_id >= n_seq_max / 2) {
LOG_ERR("mtmd_helper_gen_audio: multilingual chatterbox needs a backbone with classifier_free_guidance and a paired cfg sequence "
"(n_seq_max = %d, seq_id = %d)\n", n_seq_max, seq_id);
return 1;
}
}
llama_memory_t mem = llama_get_memory(lctx);
if (!llama_memory_seq_rm(mem, seq_id, 0, -1) || (mtl && !llama_memory_seq_rm(mem, seq_uncond, 0, -1))) {
LOG_ERR("mtmd_helper_gen_audio: cannot reset chatterbox sequences\n");
return 1;
}
// the encode gives the talker conditioning rows, set_ref keeps the flow decoder reference
if (!inp->speaker_ref) {
LOG_ERR("mtmd_helper_gen_audio: chatterbox requires a speaker reference or a voice\n");
return 1;
}
if (!encode_speaker(inp->speaker_ref, prompt_embd_buf) ||
mtmd_gen_state_set_ref(mctx, gen_state.get()) != 0) {
LOG_ERR("mtmd_helper_gen_audio: speaker reference encoding failed\n");
return 1;
}
const int n_e = n_embd;
auto push_row = [&](const float * src) {
prompt_embd_buf.insert(prompt_embd_buf.end(), src, src + n_e);
};
auto add_pos = [&](float * r, const std::vector<float> & tab, int idx) {
const float * p = tab.data() + (size_t) idx * n_e;
for (int j = 0; j < n_e; j++) {
r[j] += p[j];
}
};
// prompt rows: [conditioning, text, speech bos]
// no text preprocessing here, the [SPACE] substitution and the case folding are baked in at conversion
std::string txt(inp->prompt, inp->prompt_len);
if (mtl && inp->lang && inp->lang[0]) {
// the reference prefixes the language tag, e.g. "[fr]"
std::string lang = inp->lang;
std::transform(lang.begin(), lang.end(), lang.begin(), [](unsigned char c) { return (char) std::tolower(c); });
txt = "[" + lang + "]" + txt;
}
std::vector<llama_token> ids(txt.size() + 16);
int n_ids = llama_tokenize(vocab, txt.c_str(), (int32_t) txt.size(), ids.data(), (int32_t) ids.size(),
false, true);
if (n_ids < 1) {
LOG_ERR("mtmd_helper_gen_audio: tokenization failed\n");
return 1;
}
ids.resize((size_t) n_ids);
if (mtl) {
// only the multilingual prompt wraps the text in start/stop tokens
ids.insert(ids.begin(), text_start);
ids.push_back(text_stop);
if (ids.size() > text_pos.size() / (size_t) n_e) {
LOG_ERR("mtmd_helper_gen_audio: prompt too long (%zu tokens, max %zu)\n", ids.size(), text_pos.size() / (size_t) n_e);
return 1;
}
}
const size_t text0 = prompt_embd_buf.size() / (size_t) n_e;
for (size_t i = 0; i < ids.size(); i++) {
push_row(tok_embd.data() + (size_t) ids[i] * n_e);
if (mtl) {
add_pos(prompt_embd_buf.data() + (text0 + i) * n_e, text_pos, (int) i);
}
}
// speech bos opens the AR stream
push_row(tok_embd.data() + (size_t) llama_vocab_bos(vocab) * n_e);
if (mtl) {
add_pos(prompt_embd_buf.data() + prompt_embd_buf.size() - n_e, speech_pos, 0);
}
ar_idx = 1;
n_prompt = (int) (prompt_embd_buf.size() / (size_t) n_e);
if (mtl) {
// cfg uncond prompt: text rows are zeroed but keep their learned positions, as in the reference
uncond_embd_buf = prompt_embd_buf;
for (size_t i = 0; i < ids.size(); i++) {
float * r = uncond_embd_buf.data() + (text0 + i) * n_e;
std::fill(r, r + n_e, 0.0f);
add_pos(r, text_pos, (int) i);
}
}
prompt_pos = 0;
return 0;
}
int32_t step_prompt(int32_t n_batch) override {
GGML_ASSERT(n_batch > 0);
// the uncond prompt follows the cond one on mtl
const int n_total = mtl ? 2 * n_prompt : n_prompt;
if (prompt_pos >= n_total) {
return 0;
}
const bool uncond = prompt_pos >= n_prompt;
const int p0 = uncond ? prompt_pos - n_prompt : prompt_pos;
const int32_t n_tokens_batch = std::min(n_batch, n_prompt - p0);
const bool last = p0 + n_tokens_batch == n_prompt;
decode_embd_batch batch((uncond ? uncond_embd_buf : prompt_embd_buf).data(), n_prompt, 1, n_embd);
batch.set_position_normal(0, uncond ? seq_uncond : seq_id);
if (last) {
batch.logits[n_prompt - 1] = 1;
}
if (llama_process(lctx, LLAMA_PROCESS_TYPE_DECODE, batch.render(lctx, p0, n_tokens_batch)) != 0) {
LOG_ERR("mtmd_helper_gen_audio: prompt decode failed\n");
return -1;
}
prompt_pos += n_tokens_batch;
if (last && mtl) {
if (!uncond) {
cfg_read();
} else {
cfg_apply();
}
}
if (prompt_pos >= n_total) {
pos = n_prompt;
prompt_embd_buf.clear();
uncond_embd_buf.clear();
return 0;
}
return n_total - prompt_pos;
}
int32_t step_gen(llama_token sampled, const float * h_state_in, const float ** h_state_out, bool * out_stop) override {
GGML_UNUSED(h_state_in);
if (sampled < 0 || sampled >= llama_vocab_n_tokens(vocab)) {
LOG_ERR("mtmd_helper_gen_audio: chatterbox requires a token sampled from the backbone\n");
return 1;
}
// the speech stop token is the backbone eos, the multilingual speech positions end the stream too
if (llama_vocab_is_eog(vocab, sampled) || (mtl && (size_t) ar_idx >= speech_pos.size() / (size_t) n_embd)) {
*out_stop = true;
*h_state_out = nullptr;
return 0;
}
// keep only the 6561 s3gen codes, dropping start/stop and oov ids
if (sampled >= speech_base && sampled - speech_base < 6561) {
codes_buf.push_back(sampled - speech_base);
}
std::vector<float> e(tok_embd.begin() + (size_t) sampled * n_embd,
tok_embd.begin() + (size_t) (sampled + 1) * n_embd);
if (mtl) {
// the multilingual backbone takes the token row plus the learned speech position
const float * p = speech_pos.data() + (size_t) ar_idx * n_embd;
for (int j = 0; j < n_embd; j++) {
e[(size_t) j] += p[j];
}
ar_idx++;
}
if (!decode_one(e.data(), seq_id)) {
return 1;
}
if (mtl) {
// cfg: the sampled token feeds both branches
cfg_read();
if (!decode_one(e.data(), seq_uncond)) {
return 1;
}
cfg_apply();
}
pos++;
const float * h = llama_get_embeddings_ith(lctx, -1);
h_state_buf.assign(h, h + n_embd);
*h_state_out = h_state_buf.data();
return 0;
}
int32_t get_output(int32_t * out_sample_rate, const char ** out_data, size_t * out_data_len, int64_t * out_n_samples) override {
*out_sample_rate = info.sample_rate;
if (!stream) {
if (!flush_gen_wav()) {
return 1;
}
if (out_n_samples) {
*out_n_samples = (int64_t) audio_pcm.size();
}
if (out_type == MTMD_HELPER_GEN_AUDIO_OUTTYPE_PCM) {
*out_data = (const char *) audio_pcm.data();
*out_data_len = audio_pcm.size() * sizeof(float);
return 0;
}
out_buf.clear();
if (!write_wav16(out_buf, audio_pcm, info.sample_rate)) {
LOG_ERR("mtmd_helper_gen_audio: output too large for WAV\n");
return 1;
}
*out_data = out_buf.data();
*out_data_len = out_buf.size();
return 0;
}
// streaming: the flow decode is one-shot, so all the audio arrives after flush()
const size_t n_new = audio_pcm.size() - pcm_sent;
if (out_n_samples) {
*out_n_samples = (int64_t) n_new;
}
if (out_type == MTMD_HELPER_GEN_AUDIO_OUTTYPE_PCM) {
*out_data = (const char *) (audio_pcm.data() + pcm_sent);
*out_data_len = n_new * sizeof(float);
pcm_sent = audio_pcm.size();
return 0;
}
out_buf.clear();
if (!wav_header_sent) {
write_wav16_header(out_buf, UINT32_MAX, info.sample_rate);
wav_header_sent = true;
}
append_wav16_pcm(out_buf, audio_pcm.data() + pcm_sent, n_new);
pcm_sent = audio_pcm.size();
*out_data = out_buf.data();
*out_data_len = out_buf.size();
return 0;
}
int32_t flush() override {
return flush_gen_wav() ? 0 : 1;
}
private:
bool decode_one(const float * e, llama_seq_id seq) {
decode_embd_batch batch(e, 1, 1, n_embd);
batch.set_position_normal(pos, seq);
batch.logits[0] = 1;
if (llama_process(lctx, LLAMA_PROCESS_TYPE_DECODE, batch.render(lctx, 0, 1)) != 0) {
LOG_ERR("mtmd_helper_gen_audio: step decode failed\n");
return false;
}
return true;
}
// cfg of the reference: logits = cond + 0.5 * (cond - uncond)
// the mix goes into the last output row (the uncond one), the caller samples from it
void cfg_read() {
const float * c = llama_get_logits_ith(lctx, -1);
cond_logits.assign(c, c + llama_vocab_n_tokens(vocab));
}
void cfg_apply() {
float * u = llama_get_logits_ith(lctx, -1);
for (size_t i = 0; i < cond_logits.size(); i++) {
u[i] = cond_logits[i] + 0.5f * (cond_logits[i] - u[i]);
}
}
// one GEN_WAV call over the whole utterance, the flow decode is not streamed
bool flush_gen_wav() {
if (decoded || codes_buf.empty()) {
return true;
}
if (!mtl) {
// turbo appends a short silence tail before vocoding
codes_buf.insert(codes_buf.end(), 3, 4299);
}
mtmd_gen_inp inp = mtmd_gen_inp_default(mctx);
inp.type = MTMD_GEN_PROCESS_TYPE_GEN_WAV;
inp.codes = codes_buf.data();
inp.n_codes = codes_buf.size();
mtmd_gen_out out{};
if (mtmd_gen_audio_process(mctx, gen_state.get(), &inp, &out) != 0) {
LOG_ERR("mtmd_helper_gen_audio: gen_wav decode failed\n");
return false;
}
audio_pcm.assign(out.audio, out.audio + out.n_samples);
decoded = true;
return true;
}
bool ensure_cache() {
if (!tok_embd.empty()) {
return true;
}
// fused vocab layout: [text 0..speech_base) then the speech tokens
speech_base = find_special_token(vocab, "<|speech_0|>");
if (speech_base == LLAMA_TOKEN_NULL) {
LOG_ERR("mtmd_helper_gen_audio: fused speech tokens not found in vocab\n");
return false;
}
const uint32_t n_tok_embd = llama_model_get_tok_embd(model, nullptr);
if (n_tok_embd != (uint32_t) llama_vocab_n_tokens(vocab) * (uint32_t) n_embd) {
LOG_ERR("mtmd_helper_gen_audio: unexpected token embedding size\n");
return false;
}
tok_embd.resize(n_tok_embd);
llama_model_get_tok_embd(model, tok_embd.data());
auto read_table = [&](const char * name, std::vector<float> & dst) {
const size_t n = mtmd_gen_audio_read_tensor(mctx, name, nullptr, 0);
dst.resize(n);
if (n == 0 || mtmd_gen_audio_read_tensor(mctx, name, dst.data(), n) != n) {
LOG_ERR("mtmd_helper_gen_audio: %s read failed\n", name);
return false;
}
return true;
};
// multilingual: the learned position tables
mtl = mtmd_gen_audio_read_tensor(mctx, "a.gen.t3.text_pos_emb", nullptr, 0) > 0;
if (mtl && (!read_table("a.gen.t3.text_pos_emb", text_pos) || !read_table("a.gen.t3.speech_pos_emb", speech_pos))) {
return false;
}
return true;
}
// vocab and mmproj data fixed across the whole session, read once
std::vector<float> tok_embd;
bool mtl = false;
std::vector<float> text_pos;
std::vector<float> speech_pos;
llama_token speech_base = LLAMA_TOKEN_NULL;
// start_text_token and stop_text_token of the reference, only the multilingual prompt uses them
static constexpr llama_token text_start = 255;
static constexpr llama_token text_stop = 0;
// per-generation state, cleared by reset()
llama_seq_id seq_id = 0;
llama_seq_id seq_uncond = 0;
llama_pos pos = 0;
int ar_idx = 0;
std::vector<float> prompt_embd_buf;
std::vector<float> uncond_embd_buf;
int n_prompt = 0;
int prompt_pos = 0;
std::vector<float> cond_logits;
std::vector<int32_t> codes_buf;
bool decoded = false;
std::vector<float> audio_pcm;
std::vector<float> h_state_buf;
mtmd_helper_gen_audio_outtype out_type = MTMD_HELPER_GEN_AUDIO_OUTTYPE_WAV;
std::vector<char> out_buf;
bool stream = false;
size_t pcm_sent = 0; // samples already returned by get_output()
bool wav_header_sent = false;
};
// Drama 3: backbone samples the main token of each frame, codebook tower samples the other codes, Fish DAC makes the PCM
// the tower attends over the whole sequence, so it also runs on every prompt row
class drama3_gen_audio_pipeline : public mtmd_gen_audio_pipeline {
public:
using mtmd_gen_audio_pipeline::mtmd_gen_audio_pipeline;
void reset() override {
seq_id = 0;
pos = 0;
prompt_tokens.clear();
prompt_pos = 0;
frame.clear();
main_logits.clear();
hist.clear();
codes_buf.clear();
n_decoded = 0;
gen_state.reset();
pcm_sent = 0;
wav_header_sent = false;
audio_pcm.clear();
h_state_buf.clear();
out_buf.clear();
}
int32_t set_input(const mtmd_helper_gen_audio_inp * inp) override {
reset();
seq_id = inp->seq_id;
out_type = inp->out_type;
stream = inp->stream;
if (inp->speaker_ref) {
LOG_WRN("mtmd_helper_gen_audio: voice cloning is not supported yet, the speaker file is ignored\n");
}
sem_begin = find_special_token(vocab, "<|semantic:0|>");
eos = find_special_token(vocab, "<|im_end|>");
if (sem_begin == LLAMA_TOKEN_NULL || eos == LLAMA_TOKEN_NULL ||
sem_begin + n_cb_vocab > llama_vocab_n_tokens(vocab)) {
LOG_ERR("mtmd_helper_gen_audio: missing semantic tokens in vocab\n");
return 1;
}
if (!tokenize_segments(prompt_segments(std::string(inp->prompt, inp->prompt_len)), prompt_tokens)) {
return 1;
}
if (prompt_tokens.size() >= llama_n_ctx_seq(lctx) || prompt_tokens.size() > max_rows()) {
LOG_ERR("mtmd_helper_gen_audio: prompt does not fit in the context\n");
return 1;
}
if (!llama_memory_seq_rm(llama_get_memory(lctx), seq_id, 0, -1)) {
LOG_ERR("mtmd_helper_gen_audio: cannot reset the sequence\n");
return 1;
}
rng.seed(inp->seed == UINT32_MAX ? std::random_device{}() : inp->seed);
return init_gen_state(inp->seed) ? 0 : 1;
}
// run the tower on every prompt row to fill its KV
int32_t step_prompt(int32_t n_batch) override {
GGML_ASSERT(n_batch > 0);
const int n_prompt = (int) prompt_tokens.size();
if (prompt_pos >= n_prompt) {
return 0;
}
// small batches: every row also outputs logits
const int n = std::min({ n_batch, n_prompt - prompt_pos, 64 });
llama_batch_ext_ptr batch(llama_batch_ext_init(lctx));
if (!batch_set_tokens(batch.get(), prompt_tokens.data() + prompt_pos, n, pos, seq_id)) {
return -1;
}
for (int i = 0; i < n; i++) {
llama_batch_ext_set_output_logits(batch.get(), i, true);
}
if (llama_process(lctx, LLAMA_PROCESS_TYPE_DECODE, batch.get()) != 0) {
LOG_ERR("mtmd_helper_gen_audio: prompt decode failed\n");
return -1;
}
for (int i = 0; i < n; i++) {
const float * h = llama_get_embeddings_ith(lctx, i);
if (prompt_pos + i == n_prompt - 1) {
h_state_buf.assign(h, h + n_embd); // first frame comes from the last prompt row
save_main_logits(i);
break;
}
if (!run_tower(-1, h)) {
return -1;
}
}
pos += n;
prompt_pos += n;
return n_prompt - prompt_pos;
}
// sample the main token here, ignore the token of the caller
int32_t step_gen(llama_token sampled, const float * h_state_in, const float ** h_state_out, bool * out_stop) override {
GGML_UNUSED(sampled);
GGML_UNUSED(h_state_in);
*h_state_out = nullptr;
*out_stop = false;
const llama_token main = sample_main(main_logits.data());
const size_t n_rows = prompt_tokens.size() - 1 + codes_buf.size() / n_cb;
if (main == eos || (uint32_t) pos >= llama_n_ctx_seq(lctx) || n_rows >= max_rows()) {
*out_stop = true;
return 0;
}
hist.push_back(main);
if (hist.size() > ras_window) {
hist.erase(hist.begin());
}
const float * next_embd = run_tower(main - sem_begin, h_state_buf.data());
if (!next_embd) {
return 1;
}
codes_buf.insert(codes_buf.end(), frame.begin(), frame.end());
// text/speech stream: all mrope sections use the same position
const bool mrope = llama_model_rope_type(model) == LLAMA_ROPE_TYPE_MROPE ||
llama_model_rope_type(model) == LLAMA_ROPE_TYPE_IMROPE;
decode_embd_batch batch(next_embd, 1, mrope ? 4 : 1, n_embd);
if (mrope) {
batch.set_position_mrope_1d(pos, seq_id);
} else {
batch.set_position_normal(pos, seq_id);
}
batch.logits[0] = 1;
if (llama_process(lctx, LLAMA_PROCESS_TYPE_DECODE, batch.render(lctx, 0, 1)) != 0) {
LOG_ERR("mtmd_helper_gen_audio: decode failed\n");
return 1;
}
pos++;
const float * h = llama_get_embeddings_ith(lctx, -1);
h_state_buf.assign(h, h + n_embd);
save_main_logits(-1);
*h_state_out = h_state_buf.data();
if (stream && codes_buf.size() / n_cb - n_decoded >= stream_frames) {
return decode_wav() ? 0 : 1;
}
return 0;
}
int32_t get_output(int32_t * out_sample_rate, const char ** out_data, size_t * out_data_len, int64_t * out_n_samples) override {
*out_sample_rate = info.sample_rate;
if (!stream) {
if (flush() != 0) {
return 1;
}
if (out_n_samples) {
*out_n_samples = (int64_t) audio_pcm.size();
}
if (out_type == MTMD_HELPER_GEN_AUDIO_OUTTYPE_PCM) {
*out_data = (const char *) audio_pcm.data();
*out_data_len = audio_pcm.size() * sizeof(float);
return 0;
}
out_buf.clear();
if (!write_wav16(out_buf, audio_pcm, info.sample_rate)) {
LOG_ERR("mtmd_helper_gen_audio: output too large for WAV\n");
return 1;
}
*out_data = out_buf.data();
*out_data_len = out_buf.size();
return 0;
}
// streaming: only the audio since the previous call
const size_t n_new = audio_pcm.size() - pcm_sent;
if (out_n_samples) {
*out_n_samples = (int64_t) n_new;
}
if (out_type == MTMD_HELPER_GEN_AUDIO_OUTTYPE_PCM) {
*out_data = (const char *) (audio_pcm.data() + pcm_sent);
*out_data_len = n_new * sizeof(float);
pcm_sent = audio_pcm.size();
return 0;
}
out_buf.clear();
if (!wav_header_sent) {
write_wav16_header(out_buf, UINT32_MAX, info.sample_rate);
wav_header_sent = true;
}
append_wav16_pcm(out_buf, audio_pcm.data() + pcm_sent, n_new);
pcm_sent = audio_pcm.size();
*out_data = out_buf.data();
*out_data_len = out_buf.size();
return 0;
}
int32_t flush() override {
return decode_wav() ? 0 : 1;
}
protected:
// tokenize each segment alone as the reference does
bool tokenize_segments(const std::vector<std::string> & segments, std::vector<llama_token> & out) const {
for (const std::string & seg : segments) {
std::vector<llama_token> ids(seg.size() + 16);
const int32_t n = llama_tokenize(vocab, seg.c_str(), (int32_t) seg.size(), ids.data(), (int32_t) ids.size(), false, true);
if (n < 0) {
LOG_ERR("mtmd_helper_gen_audio: tokenization failed\n");
return false;
}
out.insert(out.end(), ids.begin(), ids.begin() + n);
}
return true;
}
// reference-bracket-v1 prompt without references
virtual std::vector<std::string> prompt_segments(std::string text) const {
if (text.rfind("<|speaker:", 0) != 0) {
text = "<|speaker:0|> " + text;
}
return {
"<|im_start|>system\n", "convert the provided text to speech", "<|im_end|>\n",
"<|im_start|>user\n", text, "<|im_end|>\n", "<|im_start|>assistant\n<|voice|>",
};
}
// prompt rows but the last + generated frames
virtual size_t max_rows() const { return n_tower_rows; }
// copy right after our own decode, other sequences may decode on lctx before the next step_gen
// layout: eos, then semantic codes 0..n_cb_vocab-1
void save_main_logits(int32_t i) {
const float * logits = llama_get_logits_ith(lctx, i);
main_logits.resize(n_cb_vocab + 1);
main_logits[0] = logits[eos];
std::copy(logits + sem_begin, logits + sem_begin + n_cb_vocab, main_logits.begin() + 1);
}
// repetition-aware sampling: if a semantic draw repeats one of the last ras_window main tokens, draw again with fixed settings
static constexpr size_t ras_window = 16;
static constexpr size_t n_cb = 10;
// codebook tower KV rows (mtmd_graph_drama3_gen::n_pos_max): one per prompt row but the last, one per frame
static constexpr size_t n_tower_rows = 4096;
// frames per codec call when streaming (0.33 s)
static constexpr size_t stream_frames = 7;
// top-p keeps the token that crosses the threshold, as the reference does
llama_token draw(const std::vector<float> & cand_logits, float temp, int32_t top_k, float top_p) {
std::vector<std::pair<float, int>> cand(cand_logits.size());
for (size_t i = 0; i < cand.size(); i++) {
cand[i] = { cand_logits[i] / temp, (int) i };
}
std::sort(cand.begin(), cand.end(), [](const auto & a, const auto & b) { return a.first > b.first; });
const float lmax = cand[0].first;
double sum = 0.0;
std::vector<double> prob(cand.size());
for (size_t i = 0; i < cand.size(); i++) {
prob[i] = std::exp((double) (cand[i].first - lmax));
sum += prob[i];
}
size_t n_keep = cand.size();
if (top_k > 0) {
n_keep = std::min(n_keep, (size_t) top_k);
}
double cum = 0.0;
for (size_t i = 0; i < n_keep; i++) {
if (cum > top_p * sum) {
n_keep = i;
break;
}
cum += prob[i];
}
double kept = 0.0;
for (size_t i = 0; i < n_keep; i++) {
kept += prob[i];
}
double r = std::uniform_real_distribution<double>(0.0, 1.0)(rng) * kept;
for (size_t i = 0; i < n_keep; i++) {
r -= prob[i];
if (r <= 0.0) {
return cand[i].second;
}
}
return cand[n_keep - 1].second;
}
// candidates: eos, then semantic codes 1.. (the reference never samples code 0)
// logits as saved by save_main_logits()
virtual llama_token sample_main(const float * logits) {
std::vector<float> cand(n_cb_vocab);
cand[0] = logits[0];
for (int i = 1; i < n_cb_vocab; i++) {
cand[i] = logits[1 + i];
}
auto to_token = [&](int i) { return i == 0 ? eos : sem_begin + i; };
const llama_token first = to_token(draw(cand, 1.0f, 0, 0.9f));
if (first != eos && std::find(hist.begin(), hist.end(), first) != hist.end()) {
return to_token(draw(cand, 1.0f, 30, 0.95f));
}
return first;
}
// code0 < 0 for a prompt row, else the main code of the next frame
// returns the next backbone input, the codes of the next frame go to frame
const float * run_tower(int32_t code0, const float * h) {
mtmd_gen_inp inp = mtmd_gen_inp_default(mctx);
inp.type = MTMD_GEN_PROCESS_TYPE_GEN_CODE;
inp.code0 = code0;
inp.embd = const_cast<float *>(h);
inp.codes = frame.empty() ? nullptr : frame.data();
inp.n_codes = frame.size();
mtmd_gen_out out{};
if (mtmd_gen_audio_process(mctx, gen_state.get(), &inp, &out) != 0 || (code0 >= 0 && out.n_codes != n_cb)) {
LOG_ERR("mtmd_helper_gen_audio: codebook tower failed\n");
return nullptr;
}
if (code0 >= 0) {
frame.assign(out.codes, out.codes + out.n_codes);
}
return out.embd;
}
// the codec is stateful, so frames can be decoded in pieces
bool decode_wav() {
const size_t n_frames = codes_buf.size() / n_cb;
while (n_decoded < n_frames) {
const size_t n = std::min(n_frames - n_decoded, (size_t) 64);
mtmd_gen_inp inp = mtmd_gen_inp_default(mctx);
inp.type = MTMD_GEN_PROCESS_TYPE_GEN_WAV;
inp.codes = codes_buf.data() + n_decoded * n_cb;
inp.n_codes = n * n_cb;
mtmd_gen_out out{};
if (mtmd_gen_audio_process(mctx, gen_state.get(), &inp, &out) != 0) {
LOG_ERR("mtmd_helper_gen_audio: codec decode failed\n");
return false;
}
audio_pcm.insert(audio_pcm.end(), out.audio, out.audio + out.n_samples);
n_decoded += n;
}
return true;
}
static constexpr int n_cb_vocab = 4096; // <|semantic:0..4095|>
llama_token sem_begin = LLAMA_TOKEN_NULL;
llama_token eos = LLAMA_TOKEN_NULL;
llama_seq_id seq_id = 0;
llama_pos pos = 0;
std::vector<llama_token> prompt_tokens;
int prompt_pos = 0;
std::mt19937 rng;
std::vector<float> main_logits; // see save_main_logits()
std::vector<int32_t> frame; // codes of the last frame, input of the next tower row
std::vector<llama_token> hist; // last main tokens, for repetition-aware sampling
std::vector<int32_t> codes_buf; // frame-major
size_t n_decoded = 0; // frames already through the codec
std::vector<float> audio_pcm;
std::vector<float> h_state_buf;
mtmd_helper_gen_audio_outtype out_type = MTMD_HELPER_GEN_AUDIO_OUTTYPE_WAV;
std::vector<char> out_buf;
bool stream = false;
size_t pcm_sent = 0; // samples already returned by get_output()
bool wav_header_sent = false;
};
// Fish Audio S2 Pro: same tokens, main-token loop and Fish DAC as drama3
// the fast AR only runs on generated frames, so the prompt is a plain decode
class fish_s2_gen_audio_pipeline : public drama3_gen_audio_pipeline {
public:
using drama3_gen_audio_pipeline::drama3_gen_audio_pipeline;
void reset() override {
drama3_gen_audio_pipeline::reset();
parts.clear();
part_idx = 0;
part_pos = 0;
}
// voice cloning (sglang-omni fishaudio_s2_pro): a system message with the transcript and the codec rows of the clip
int32_t set_input(const mtmd_helper_gen_audio_inp * inp) override {
std::vector<float> ref_rows;
if (inp->speaker_ref && !encode_speaker(inp->speaker_ref, ref_rows)) {
return 1;
}
mtmd_helper_gen_audio_inp text_inp = *inp;
text_inp.speaker_ref = nullptr;
if (drama3_gen_audio_pipeline::set_input(&text_inp) != 0) {
return 1;
}
size_t n_total = prompt_tokens.size();
if (!ref_rows.empty()) {
// optional, the reference leaves the text empty when it has none
std::string ref_text = inp->speaker_ref_text ? inp->speaker_ref_text : "";
if (!ref_text.empty() && ref_text.rfind("<|speaker:", 0) != 0) {
ref_text = "<|speaker:0|>" + ref_text;
}
prompt_part head, rows, tail;
if (!tokenize_segments({ "<|im_start|>system\n", "convert the provided text to speech reference to the following:\n\nText:\n",
ref_text, "\n\nSpeech:\n" }, head.tokens) ||
!tokenize_segments({ "<|im_end|>\n" }, tail.tokens)) {
return 1;
}
rows.rows = std::move(ref_rows);
for (auto * part : { &head, &rows, &tail }) {
n_total += part->size(n_embd);
parts.push_back(std::move(*part));
}
}
if (n_total >= llama_n_ctx_seq(lctx)) {
LOG_ERR("mtmd_helper_gen_audio: prompt does not fit in the context\n");
return 1;
}
prompt_part text;
text.tokens = prompt_tokens;
parts.push_back(std::move(text));
return 0;
}
// the first frame comes from the last prompt row
int32_t step_prompt(int32_t n_batch) override {
GGML_ASSERT(n_batch > 0);
if (part_idx >= parts.size()) {
return 0;
}
const prompt_part & part = parts[part_idx];
const int n_part = part.size(n_embd);
const int n = std::min(n_batch, n_part - part_pos);
const bool last = part_idx + 1 == parts.size() && part_pos + n == n_part;
int ret = 0;
if (part.rows.empty()) {
llama_batch_ext_ptr batch(llama_batch_ext_init(lctx));
if (!batch_set_tokens(batch.get(), part.tokens.data() + part_pos, n, pos, seq_id)) {
return -1;
}
if (last) {
llama_batch_ext_set_output_logits(batch.get(), n - 1, true);
}
ret = llama_process(lctx, LLAMA_PROCESS_TYPE_DECODE, batch.get());
} else {
decode_embd_batch batch(part.rows.data() + (size_t) part_pos * n_embd, n, 1, n_embd);
batch.set_position_normal(pos, seq_id);
ret = llama_process(lctx, LLAMA_PROCESS_TYPE_DECODE, batch.render(lctx, 0, n));
}
if (ret != 0) {
LOG_ERR("mtmd_helper_gen_audio: prompt decode failed\n");
return -1;
}
if (last) {
const float * h = llama_get_embeddings_ith(lctx, -1);
h_state_buf.assign(h, h + n_embd);
save_main_logits(-1);
}
pos += n;
part_pos += n;
if (part_pos == n_part) {
part_idx++;
part_pos = 0;
}
int32_t n_left = -part_pos;
for (size_t i = part_idx; i < parts.size(); i++) {
n_left += parts[i].size(n_embd);
}
return n_left;
}
protected:
// reference (sglang-omni fishaudio_s2_pro) has no system message
std::vector<std::string> prompt_segments(std::string text) const override {
if (text.rfind("<|speaker:", 0) != 0) {
text = "<|speaker:0|>" + text;
}
return {
"<|im_start|>user\n", text, "<|im_end|>\n", "<|im_start|>assistant\n<|voice|>",
};
}
// no codebook tower, only the context limits the length
size_t max_rows() const override { return SIZE_MAX; }
// token ids, or backbone input rows of n_embd floats
struct prompt_part {
std::vector<llama_token> tokens;
std::vector<float> rows;
int size(int n_embd) const { return rows.empty() ? (int) tokens.size() : (int) (rows.size() / n_embd); }
};
std::vector<prompt_part> parts;
size_t part_idx = 0;
int part_pos = 0;
// same as the reference: repetition penalty over the last ras_window main tokens, top-k, top-p on the
// untempered logits (the crossing token is dropped), then temperature
// a repeat within the last 4 main tokens switches to the RAS settings
llama_token sample_main(const float * logits) override {
static constexpr int32_t top_k = 30;
static constexpr float rep_penalty = 1.1f;
std::vector<std::pair<float, llama_token>> cand;
cand.reserve(n_cb_vocab + 1);
cand.push_back({ logits[0], eos });
for (int i = 0; i < n_cb_vocab; i++) {
cand.push_back({ logits[1 + i], sem_begin + i });
}
for (auto & c : cand) {
if (std::find(hist.begin(), hist.end(), c.second) != hist.end()) {
c.first = c.first < 0.0f ? c.first * rep_penalty : c.first / rep_penalty;
}
}
bool ras = false;
if (hist.size() >= 4) {
std::vector<llama_token> last4(hist.end() - 4, hist.end());
std::sort(last4.begin(), last4.end());
ras = std::adjacent_find(last4.begin(), last4.end()) != last4.end();
}
const float temp = ras ? 1.0f : 0.8f;
const float top_p = ras ? 0.9f : 0.8f;
const size_t k = std::min<size_t>(top_k, cand.size());
std::partial_sort(cand.begin(), cand.begin() + k, cand.end(), [](const auto & a, const auto & b) { return a.first > b.first; });
std::vector<double> prob(k);
double sum = 0.0;
for (size_t i = 0; i < k; i++) {
prob[i] = std::exp((double) (cand[i].first - cand[0].first));
sum += prob[i];
}
size_t n_keep = k;
double cum = 0.0;
for (size_t i = 0; i < k; i++) {
cum += prob[i];
if (i > 0 && cum > top_p * sum) {
n_keep = i;
break;
}
}
double kept = 0.0;
for (size_t i = 0; i < n_keep; i++) {
prob[i] = std::exp((double) (cand[i].first - cand[0].first) / temp);
kept += prob[i];
}
double r = std::uniform_real_distribution<double>(0.0, 1.0)(rng) * kept;
for (size_t i = 0; i < n_keep; i++) {
r -= prob[i];
if (r <= 0.0) {
return cand[i].second;
}
}
return cand[n_keep - 1].second;
}
};
// ZONOS3: one backbone row per frame of 10 codebooks, codebook j lags j frames
class zonos3_gen_audio_pipeline : public mtmd_gen_audio_pipeline {
public:
using mtmd_gen_audio_pipeline::mtmd_gen_audio_pipeline;
bool supports_delivery() const override { return true; }
void reset() override {
seq_id = 0;
pos = 0;
segments.clear();
seg_idx = 0;
seg_pos = 0;
rows.clear();
step = 0;
eos_row = -1;
countdown = 0;
done = false;
n_decoded = 0;
n_codec_samples = 0;
flushed = false;
gen_state.reset();
pcm_sent = 0;
wav_header_sent = false;
audio_pcm.clear();
h_state_buf.clear();
out_buf.clear();
}
int32_t set_input(const mtmd_helper_gen_audio_inp * inp) override {
reset();
seq_id = inp->seq_id;
out_type = inp->out_type;
stream = inp->stream;
// settings of the reference, only the temperature can be changed
const mtmd_gen_inp def = mtmd_gen_inp_default(mctx);
temp = inp->temp >= 0.0f ? inp->temp : def.temp;
top_k = def.top_k;
top_p = def.top_p;
rep_pen = def.penalty_repeat;
rep_win = def.penalty_last_n;
rng.seed(inp->seed == UINT32_MAX ? std::random_device{}() : inp->seed);
if (!init_gen_state(inp->seed)) {
return 1;
}
// prompt: [voice rows] [condition rows] BOS text EOS [silence rows]
// the speaker CFG of the reference is omitted: it adds too much code and maintenance cost
if (inp->speaker_ref) {
segment seg;
if (!encode_speaker(inp->speaker_ref, seg.embd)) {
return 1;
}
segments.push_back(std::move(seg));
}
// condition rows: the bucket of each value over the checkpoint edges, or unknown
auto bucket = [](const std::vector<float> & edges, float v) {
return std::to_string(std::upper_bound(edges.begin(), edges.end(), v) - edges.begin());
};
const std::string loudness = inp->loudness != 0.0f ? bucket(loudness_edges, inp->loudness) : "unk";
const std::string rate = inp->speed != 1.0f ? bucket(rate_edges, inp->speed * typical_rate) : "unk";
segment text;
for (const std::string & name : { "<|loudness_" + loudness + "|>", std::string("<|fmax_unk|>"), "<|speaking_rate_" + rate + "|>" }) {
const llama_token t = find_special_token(vocab, name);
if (t == LLAMA_TOKEN_NULL) {
LOG_ERR("mtmd_helper_gen_audio: missing ZONOS3 token %s in vocab\n", name.c_str());
return 1;
}
text.tokens.push_back(t);
}
const std::string prompt(inp->prompt, inp->prompt_len);
std::vector<llama_token> ids(prompt.size() + 16);
const int32_t n_ids = llama_tokenize(vocab, prompt.c_str(), (int32_t) prompt.size(), ids.data(), (int32_t) ids.size(), true, false);
if (n_ids <= 2) {
LOG_ERR("mtmd_helper_gen_audio: empty prompt or tokenization failed\n");
return 1;
}
text.tokens.insert(text.tokens.end(), ids.begin(), ids.begin() + n_ids);
segments.push_back(std::move(text));
// codec silence, delayed
segment silence;
for (int k = 0; k < n_prefill; k++) {
std::vector<int32_t> row(n_cb, audio_pad);
for (int cb = 0; cb <= k && cb < n_cb; cb++) {
row[cb] = silence_frame[cb];
}
const float * embd = embed_row(row);
if (!embd) {
return 1;
}
silence.embd.insert(silence.embd.end(), embd, embd + n_embd);
rows.insert(rows.end(), row.begin(), row.end());
}
segments.push_back(std::move(silence));
size_t n_prompt = 0;
for (const auto & seg : segments) {
n_prompt += seg.n_rows(n_embd);
}
if (n_prompt >= llama_n_ctx_seq(lctx)) {
LOG_ERR("mtmd_helper_gen_audio: prompt does not fit in the context\n");
return 1;
}
if (!llama_memory_seq_rm(llama_get_memory(lctx), seq_id, 0, -1)) {
LOG_ERR("mtmd_helper_gen_audio: cannot reset the sequence\n");
return 1;
}
return 0;
}
int32_t step_prompt(int32_t n_batch) override {
GGML_ASSERT(n_batch > 0);
if (seg_idx >= segments.size()) {
return 0;
}
const segment & seg = segments[seg_idx];
const int n_rows = (int) seg.n_rows(n_embd);
const int n = std::min(n_batch, n_rows - seg_pos);
const bool last = seg_idx + 1 == segments.size() && seg_pos + n == n_rows;
int ret;
if (!seg.tokens.empty()) {
llama_batch_ext_ptr batch(llama_batch_ext_init(lctx));
if (!batch_set_tokens(batch.get(), seg.tokens.data() + seg_pos, n, pos, seq_id)) {
return -1;
}
if (last) {
llama_batch_ext_set_output_logits(batch.get(), n - 1, true);
}
ret = llama_process(lctx, LLAMA_PROCESS_TYPE_DECODE, batch.get());
} else {
decode_embd_batch batch(seg.embd.data() + (size_t) seg_pos * n_embd, n, 1, n_embd);
batch.set_position_normal(pos, seq_id);
batch.logits[n - 1] = last;
ret = llama_process(lctx, LLAMA_PROCESS_TYPE_DECODE, batch.render(lctx, 0, n));
}
if (ret != 0) {
LOG_ERR("mtmd_helper_gen_audio: prompt decode failed\n");
return -1;
}
pos += n;
seg_pos += n;
if (seg_pos == n_rows) {
seg_idx++;
seg_pos = 0;
}
int32_t n_left = 0;
for (size_t i = seg_idx; i < segments.size(); i++) {
n_left += (int32_t) segments[i].n_rows(n_embd) - (i == seg_idx ? seg_pos : 0);
}
if (n_left == 0) {
const float * h = llama_get_embeddings_ith(lctx, -1);
h_state_buf.assign(h, h + n_embd);
}
return n_left;
}
int32_t step_gen(llama_token sampled, const float * h_state_in, const float ** h_state_out, bool * out_stop) override {
GGML_UNUSED(sampled);
GGML_UNUSED(h_state_in);
*h_state_out = nullptr;
*out_stop = false;
if (done) {
*out_stop = true;
return 0;
}
mtmd_gen_inp inp = mtmd_gen_inp_default(mctx);
inp.type = MTMD_GEN_PROCESS_TYPE_GEN_CODE;
inp.embd = h_state_buf.data();
mtmd_gen_out out{};
if (mtmd_gen_audio_process(mctx, gen_state.get(), &inp, &out) != 0 || out.n_feats != (size_t) n_cb * n_vocab) {
LOG_ERR("mtmd_helper_gen_audio: codebook heads failed\n");
return 1;
}
std::vector<int32_t> frame(n_cb);
for (int cb = 0; cb < n_cb; cb++) {
frame[cb] = sample(out.feats + (size_t) cb * n_vocab, cb);
}
// codebooks whose delayed stream has not started yet are pad
const int r = n_prefill + step;
for (int cb = r + 1; cb < n_cb; cb++) {
frame[cb] = audio_pad;
}
rows.insert(rows.end(), frame.begin(), frame.end());
step++;
// after EOA, n_cb + 1 more rows flush the delay
int max_eoa = -1;
for (int cb = 0; cb < n_cb; cb++) {
if (frame[cb] == audio_eoa) {
max_eoa = cb;
}
}
if (eos_row < 0 && max_eoa >= 0) {
eos_row = std::max(0, r - max_eoa);
countdown = n_cb + 1;
}
if ((countdown > 0 && --countdown == 0) || step >= max_frames || (uint32_t) pos >= llama_n_ctx_seq(lctx)) {
done = true;
*out_stop = true;
return 0;
}
const float * next = embed_row(frame);
if (!next) {
return 1;
}
decode_embd_batch batch(next, 1, 1, n_embd);
batch.set_position_normal(pos, seq_id);
batch.logits[0] = 1;
if (llama_process(lctx, LLAMA_PROCESS_TYPE_DECODE, batch.render(lctx, 0, 1)) != 0) {
LOG_ERR("mtmd_helper_gen_audio: decode failed\n");
return 1;
}
pos++;
const float * h = llama_get_embeddings_ith(lctx, -1);
h_state_buf.assign(h, h + n_embd);
*h_state_out = h_state_buf.data();
if (stream) {
return decode_stream() ? 0 : 1;
}
return 0;
}
int32_t get_output(int32_t * out_sample_rate, const char ** out_data, size_t * out_data_len, int64_t * out_n_samples) override {
*out_sample_rate = info.sample_rate;
if (!stream) {
if (flush() != 0) {
return 1;
}
if (out_n_samples) {
*out_n_samples = (int64_t) audio_pcm.size();
}
if (out_type == MTMD_HELPER_GEN_AUDIO_OUTTYPE_PCM) {
*out_data = (const char *) audio_pcm.data();
*out_data_len = audio_pcm.size() * sizeof(float);
return 0;
}
out_buf.clear();
if (!write_wav16(out_buf, audio_pcm, info.sample_rate)) {
LOG_ERR("mtmd_helper_gen_audio: output too large for WAV\n");
return 1;
}
*out_data = out_buf.data();
*out_data_len = out_buf.size();
return 0;
}
// streaming: only the audio since the previous call
const size_t n_new = audio_pcm.size() - pcm_sent;
if (out_n_samples) {
*out_n_samples = (int64_t) n_new;
}
if (out_type == MTMD_HELPER_GEN_AUDIO_OUTTYPE_PCM) {
*out_data = (const char *) (audio_pcm.data() + pcm_sent);
*out_data_len = n_new * sizeof(float);
pcm_sent = audio_pcm.size();
return 0;
}
out_buf.clear();
if (!wav_header_sent) {
write_wav16_header(out_buf, UINT32_MAX, info.sample_rate);
wav_header_sent = true;
}
append_wav16_pcm(out_buf, audio_pcm.data() + pcm_sent, n_new);
pcm_sent = audio_pcm.size();
*out_data = out_buf.data();
*out_data_len = out_buf.size();
return 0;
}
int32_t flush() override {
return decode_wav(true) ? 0 : 1;
}
private:
static constexpr int n_cb = 10;
static constexpr int n_vocab = 4098; // codes, EOA, pad
static constexpr int audio_eoa = 4096;
static constexpr int audio_pad = 4097;
// condition_edges of params.json; speed 1 is this many UTF-8 bytes per second, the middle of the rate buckets
const std::vector<float> loudness_edges = { -29.778f, -26.947f, -25.047f, -23.3f, -21.743f, -20.386f, -19.186f, -17.851f, -15.967f };
const std::vector<float> rate_edges = {
6.546f, 8.182f, 9.273f, 10.31f, 11.054f, 11.667f, 12.193f, 12.656f, 13.083f, 13.476f, 13.846f, 14.196f, 14.53f, 14.887f, 15.193f,
15.517f, 15.865f, 16.208f, 16.579f, 16.972f, 17.411f, 17.898f, 18.434f, 19.069f, 19.818f, 20.816f, 22.286f, 24.722f, 29.25f,
};
static constexpr float typical_rate = 15.0f;
// digital silence of the ZAC codec (registries/silence_frame_*.json)
static constexpr int32_t silence_frame[n_cb] = { 1918, 746, 3544, 234, 1267, 2914, 3944, 2849, 326, 1337 };
static constexpr int n_prefill = 3;
// -3 dB of headroom
static constexpr int samples_per_frame = 2940;
static constexpr float out_gain = 0.70794578f;
static constexpr int rep_codebooks = 6;
static constexpr int max_frames = 2000;
static constexpr size_t stream_frames = 8;
struct segment {
std::vector<llama_token> tokens; // either tokens or embd rows
std::vector<float> embd;
size_t n_rows(int n) const { return tokens.empty() ? embd.size() / (size_t) n : tokens.size(); }
};
const float * embed_row(const std::vector<int32_t> & codes) {
mtmd_gen_inp inp = mtmd_gen_inp_default(mctx);
inp.type = MTMD_GEN_PROCESS_TYPE_GEN_CODE;
inp.codes = const_cast<int32_t *>(codes.data());
inp.n_codes = codes.size();
mtmd_gen_out out{};
if (mtmd_gen_audio_process(mctx, gen_state.get(), &inp, &out) != 0) {
LOG_ERR("mtmd_helper_gen_audio: codebook embedding failed\n");
return nullptr;
}
return out.embd;
}
// same as the reference sampler (servers/ggml/src/zonos3-sampler.cpp)
int32_t sample(const float * in, int cb) {
std::vector<float> l(in, in + n_vocab);
l[audio_pad] = -INFINITY;
if (rep_pen != 1.0f && cb < rep_codebooks) {
const size_t n_rows = rows.size() / n_cb - n_prefill;
const size_t n_win = std::min(n_rows, (size_t) std::max(rep_win, 0));
std::vector<bool> seen(audio_eoa, false);
for (size_t f = n_prefill + n_rows - n_win; f < n_prefill + n_rows; f++) {
const int32_t c = rows[f * n_cb + (size_t) cb];
if (c >= 0 && c < audio_eoa && !seen[(size_t) c]) {
seen[(size_t) c] = true;
l[c] = l[c] > 0.0f ? l[c] / rep_pen : l[c] * rep_pen;
}
}
}
if (temp <= 0.0f) {
return (int32_t) (std::max_element(l.begin(), l.end()) - l.begin());
}
for (auto & v : l) {
v /= temp;
}
float kth = -INFINITY;
if (top_k > 0 && top_k < n_vocab) {
std::vector<float> tmp(l);
std::nth_element(tmp.begin(), tmp.begin() + (top_k - 1), tmp.end(), std::greater<float>());
kth = tmp[(size_t) top_k - 1];
}
std::vector<int32_t> idx;
for (int32_t i = 0; i < n_vocab; i++) {
if (l[i] >= kth && l[i] > -INFINITY) {
idx.push_back(i);
}
}
float lmax = -INFINITY;
for (int32_t i : idx) {
lmax = std::max(lmax, l[i]);
}
std::vector<double> p(idx.size());
double sum = 0.0;
for (size_t j = 0; j < idx.size(); j++) {
p[j] = std::exp((double) (l[idx[j]] - lmax));
sum += p[j];
}
std::vector<size_t> ord(idx.size());
for (size_t j = 0; j < ord.size(); j++) {
ord[j] = j;
}
std::sort(ord.begin(), ord.end(), [&](size_t a, size_t b) { return p[a] > p[b]; });
if (top_p > 0.0f && top_p < 1.0f) {
double before = 0.0;
for (size_t j : ord) {
const double pj = p[j];
if (before > top_p * sum) {
p[j] = 0.0;
}
before += pj;
}
}
double kept = 0.0;
for (double v : p) {
kept += v;
}
double u = std::uniform_real_distribution<double>(0.0, kept)(rng);
for (size_t j = 0; j < p.size(); j++) {
u -= p[j];
if (u <= 0.0 && p[j] > 0.0) {
return idx[j];
}
}
return idx[ord[0]];
}
// frames with all codebooks known
size_t n_ready() const {
const size_t n_rows = rows.size() / n_cb;
size_t n = n_rows >= (size_t) n_cb ? n_rows - (n_cb - 1) : 0;
if (eos_row >= 0) {
n = std::min(n, (size_t) eos_row);
}
return n;
}
// streaming, as the reference does for the first audio: the silence frames are decoded with silence codes only,
// then the first generated frame from its first code alone; later chunks of complete frames grow 2, 4, 8
bool decode_stream() {
if (n_decoded == 0) {
const size_t n_rows = rows.size() / n_cb;
if (n_rows <= (size_t) n_prefill || rows[(size_t) n_prefill * n_cb] >= audio_eoa) {
return true;
}
std::vector<int32_t> codes;
for (int k = 0; k < n_prefill; k++) {
codes.insert(codes.end(), silence_frame, silence_frame + n_cb);
}
for (int cb = 0; cb < n_cb; cb++) {
const size_t r = (size_t) n_prefill + cb;
const int32_t c = r < n_rows ? rows[r * n_cb + cb] : -1;
codes.push_back(c >= audio_eoa ? -1 : c);
}
if (!run_codec(codes.data(), codes.size())) {
return false;
}
n_decoded = n_prefill + 1;
return true;
}
const size_t n_chunk = std::min(stream_frames, n_decoded - n_prefill + 1);
if (n_ready() >= n_decoded + n_chunk) {
return decode_wav(false);
}
return true;
}
// the silence frames are decoded too, as left context
bool decode_wav(bool final) {
const size_t n_frames = n_ready();
std::vector<int32_t> codes;
while (n_decoded < n_frames) {
const size_t n = std::min(n_frames - n_decoded, (size_t) 64);
codes.resize(n * n_cb);
for (size_t f = 0; f < n; f++) {
for (int cb = 0; cb < n_cb; cb++) {
// EOA and pad have no codec row
const int32_t c = rows[(n_decoded + f + cb) * n_cb + cb];
codes[f * n_cb + cb] = std::clamp(c, 0, audio_eoa - 1);
}
}
if (!run_codec(codes.data(), codes.size())) {
return false;
}
n_decoded += n;
}
if (final && done && n_decoded > 0 && !flushed) {
flushed = true;
return run_codec(nullptr, 0);
}
return true;
}
bool run_codec(const int32_t * codes, size_t n_codes) {
mtmd_gen_inp inp = mtmd_gen_inp_default(mctx);
inp.type = MTMD_GEN_PROCESS_TYPE_GEN_WAV;
inp.codes = const_cast<int32_t *>(codes);
inp.n_codes = n_codes;
mtmd_gen_out out{};
if (mtmd_gen_audio_process(mctx, gen_state.get(), &inp, &out) != 0) {
LOG_ERR("mtmd_helper_gen_audio: codec decode failed\n");
return false;
}
// drop the audio of the silence frames, then the output gain and soft limiter of the reference
const float knee = std::pow(10.0f, -1.0f / 20.0f);
for (size_t i = 0; i < out.n_samples; i++, n_codec_samples++) {
if (n_codec_samples < (size_t) n_prefill * samples_per_frame) {
continue;
}
const float x = out.audio[i] * out_gain;
const float a = std::fabs(x);
audio_pcm.push_back(a <= knee ? x : std::copysign(knee + (1.0f - knee) * std::tanh((a - knee) / (1.0f - knee)), x));
}
return true;
}
std::vector<segment> segments;
size_t seg_idx = 0;
int seg_pos = 0;
std::vector<int32_t> rows; // delayed rows: silence rows, then the generated ones
int step = 0;
int eos_row = -1;
int countdown = 0;
bool done = false;
size_t n_decoded = 0;
size_t n_codec_samples = 0;
bool flushed = false;
float temp = 0.75f;
int32_t top_k = 80;
float top_p = 0.69f;
float rep_pen = 2.4f;
int32_t rep_win = 96;
std::mt19937 rng;
llama_seq_id seq_id = 0;
llama_pos pos = 0;
mtmd_helper_gen_audio_outtype out_type = MTMD_HELPER_GEN_AUDIO_OUTTYPE_WAV;
bool stream = false;
std::vector<float> h_state_buf;
std::vector<float> audio_pcm;
std::vector<char> out_buf;
size_t pcm_sent = 0;
bool wav_header_sent = false;
};
// YuE2: the backbone writes an ABC score (text), then the semantic codes; both are sampled here like the reference (sampling.py)
// the NAR flow matching (in mtmd, on the backbone keys/values) and the VAE then make the stereo PCM
class yue2_gen_audio_pipeline : public mtmd_gen_audio_pipeline {
public:
using mtmd_gen_audio_pipeline::mtmd_gen_audio_pipeline;
bool supports_cfg() const override { return true; }
bool supports_flow_steps() const override { return true; }
// step_gen() stops at the score and code limits of the reference
int32_t max_frames() const override { return 0; }
// the limits and the NAR chunks of the reference need the training context (24576 positions)
bool check_context() const override {
if (llama_vocab_n_tokens(vocab) < tok_codec0 + n_codec) {
LOG_ERR("mtmd_helper_gen_audio: the vocab of the backbone has no yue2 codec tokens\n");
return false;
}
const uint32_t n_ctx_train = (uint32_t) llama_model_n_ctx_train(model);
if (llama_n_ctx_seq(lctx) < n_ctx_train) {
LOG_ERR("mtmd_helper_gen_audio: yue2 needs a context of %u positions per sequence, got %u (use the default context size)\n",
n_ctx_train, llama_n_ctx_seq(lctx));
return false;
}
return true;
}
void reset() override {
seq_id = 0;
pos = 0;
prompt.clear();
prompt_pos = 0;
in_codes = false;
mode_off = false;
cfg_scale = 1.0f;
seq_uncond = 0;
upos = 0;
neg_prefix.clear();
abc_tail.clear();
prefix_tok.clear();
history.clear();
codes.clear();
latents.clear();
n_nar_done = 0;
n_vae_done = 0;
scratch.reset();
scratch_n_ctx = 0;
done = false;
decoded = false;
audio_pcm.clear();
h_state_buf.clear();
out_buf.clear();
stream = false;
pcm_sent = 0;
wav_header_sent = false;
gen_state.reset();
}
int32_t set_input(const mtmd_helper_gen_audio_inp * inp) override {
reset();
seq_id = inp->seq_id;
out_type = inp->out_type;
stream = inp->stream;
code_cfg.temp = inp->temp >= 0.0f ? inp->temp : 1.0f;
flow_steps = inp->flow_steps;
rng.seed(inp->seed == UINT32_MAX ? std::random_device{}() : inp->seed);
if (!init_gen_state(inp->seed)) {
return 1;
}
// the "tts" template renders the whole prompt, special tokens included
const std::string text(inp->prompt, inp->prompt_len);
prompt.resize(text.size() + 16);
const int32_t n = llama_tokenize(vocab, text.c_str(), (int32_t) text.size(), prompt.data(), (int32_t) prompt.size(), false, true);
if (n < 2 || prompt[0] != tok_eod) {
LOG_ERR("mtmd_helper_gen_audio: yue2 expects the prompt of its tts template\n");
return 1;
}
prompt.resize((size_t) n);
// cover mode and cot=off give the score in the prompt, the codes come next
auto abc = std::find(prompt.begin(), prompt.end(), tok_abc_start);
if (prompt.back() == tok_music_start && abc != prompt.end()) {
in_codes = true;
mode_off = prompt.end() - abc == 3; // empty score
} else if (prompt.back() != tok_abc_start) {
LOG_ERR("mtmd_helper_gen_audio: yue2 prompt must end with <abc> or <|music_start|>\n");
return 1;
}
abc_tail.assign(abc, prompt.end());
prefix_tok = prompt;
if (prompt.size() >= llama_n_ctx_seq(lctx)) {
LOG_ERR("mtmd_helper_gen_audio: prompt does not fit in the context\n");
return 1;
}
llama_memory_t mem = llama_get_memory(lctx);
if (!llama_memory_seq_rm(mem, seq_id, 0, -1)) {
LOG_ERR("mtmd_helper_gen_audio: cannot reset the sequence\n");
return 1;
}
// classifier-free guidance of the codes, defaults of the reference (protocol.py)
cfg_scale = inp->cfg_scale > 0.0f ? inp->cfg_scale : mode_off ? 1.01f : 1.0f;
if (cfg_scale != 1.0f) {
// the uncond sequence lives half a seq space away from the cond one
const int32_t n_seq_max = (int32_t) llama_n_seq_max(lctx);
seq_uncond = (llama_seq_id) (n_seq_max / 2 + seq_id);
if (!llama_model_classifier_free_guidance(model) || seq_id >= n_seq_max / 2) {
if (inp->cfg_scale > 0.0f) {
LOG_ERR("mtmd_helper_gen_audio: cfg_scale needs a paired sequence (n_seq_max = %d, seq_id = %d), reconvert the model\n", n_seq_max, seq_id);
return 1;
}
LOG_WRN("mtmd_helper_gen_audio: no paired cfg sequence (n_seq_max = %d, seq_id = %d), cfg is disabled\n", n_seq_max, seq_id);
cfg_scale = 1.0f;
} else if (!llama_memory_seq_rm(mem, seq_uncond, 0, -1)) {
LOG_ERR("mtmd_helper_gen_audio: cannot reset the cfg sequence\n");
return 1;
}
}
if (cfg_scale != 1.0f) {
// the negative prompt is the bare instruction, tokenized alone like the reference
const size_t i0 = text.find("<|endoftext|>");
const size_t i1 = text.find('\n');
if (i0 != 0 || i1 == std::string::npos) {
LOG_ERR("mtmd_helper_gen_audio: yue2 expects the prompt of its tts template\n");
return 1;
}
const std::string instr = text.substr(13, i1 - 13);
neg_prefix.resize(instr.size() + 16);
const int32_t n_instr = llama_tokenize(vocab, instr.c_str(), (int32_t) instr.size(), neg_prefix.data() + 1, (int32_t) neg_prefix.size() - 1, false, false);
if (n_instr < 1) {
LOG_ERR("mtmd_helper_gen_audio: tokenization failed\n");
return 1;
}
neg_prefix[0] = tok_eod;
neg_prefix.resize((size_t) n_instr + 1);
}
return 0;
}
int32_t step_prompt(int32_t n_batch) override {
GGML_ASSERT(n_batch > 0);
const int32_t n_left = (int32_t) prompt.size() - prompt_pos;
if (n_left <= 0) {
return 0;
}
const int32_t n = std::min(n_batch, n_left);
if (!decode(prompt.data() + prompt_pos, n)) {
return -1;
}
prompt_pos += n;
if (n == n_left) {
if (in_codes) {
if (!start_uncond()) {
return -1;
}
} else {
save_logits(llama_get_logits_ith(lctx, -1));
}
}
return n_left - n;
}
int32_t step_gen(llama_token sampled, const float * h_state_in, const float ** h_state_out, bool * out_stop) override {
GGML_UNUSED(sampled);
GGML_UNUSED(h_state_in);
*h_state_out = nullptr;
*out_stop = false;
if (done) {
*out_stop = true;
return 0;
}
const bool use_cfg = in_codes && cfg_scale != 1.0f;
const llama_token tok = sample(logits.data());
int32_t i_out = -1; // output row of the cond sequence
if (!in_codes) {
// the score ends with </abc>, or is cut at max_tokens like the reference
if (tok == tok_abc_end || (int32_t) history.size() + 1 >= abc_cfg.max_tokens) {
const llama_token next[2] = { tok_abc_end, tok_music_start };
if (!decode(next, 2)) {
return 1;
}
abc_tail.insert(abc_tail.end(), next, next + 2);
prefix_tok.insert(prefix_tok.end(), next, next + 2);
in_codes = true;
history.clear();
if (!start_uncond()) {
return 1;
}
} else {
history.push_back(tok);
abc_tail.push_back(tok);
prefix_tok.push_back(tok);
if (!decode(&tok, 1)) {
return 1;
}
save_logits(llama_get_logits_ith(lctx, -1));
}
} else {
// the NAR pass also attends to <|music_end|>, the uncond sequence shares the unified KV cache
const bool full = (int32_t) codes.size() + 1 >= code_cfg.max_tokens || pos + upos + 2 >= (llama_pos) llama_n_ctx_seq(lctx);
if (tok == tok_music_end || full) {
if (!decode(&tok_music_end, 1)) {
return 1;
}
done = true;
*out_stop = true;
return 0;
}
history.push_back(tok);
codes.push_back(tok - tok_codec0);
if (use_cfg) {
// the sampled code feeds both branches, in one decode
if (!decode_pair(tok)) {
return 1;
}
i_out = 0;
} else {
if (!decode(&tok, 1)) {
return 1;
}
save_logits(llama_get_logits_ith(lctx, -1));
}
}
const float * h = llama_get_embeddings_ith(lctx, i_out);
h_state_buf.assign(h, h + n_embd);
*h_state_out = h_state_buf.data();
// streaming: each full chunk of codes goes through the NAR pass and the VAE right away
if (stream && in_codes && codes.size() - n_nar_done >= chunk_size()) {
if (!nar_chunk(n_nar_done, n_nar_done + chunk_size(), false) || !vae_emit(false)) {
return 1;
}
}
return 0;
}
int32_t get_output(int32_t * out_sample_rate, const char ** out_data, size_t * out_data_len, int64_t * out_n_samples) override {
*out_sample_rate = info.sample_rate;
const uint16_t n_ch = (uint16_t) info.n_channels;
if (!stream) {
if (flush() != 0) {
return 1;
}
if (out_n_samples) {
*out_n_samples = (int64_t) (audio_pcm.size() / n_ch);
}
if (out_type == MTMD_HELPER_GEN_AUDIO_OUTTYPE_PCM) {
*out_data = (const char *) audio_pcm.data();
*out_data_len = audio_pcm.size() * sizeof(float);
return 0;
}
out_buf.clear();
if (!write_wav16(out_buf, audio_pcm, info.sample_rate, n_ch)) {
LOG_ERR("mtmd_helper_gen_audio: output too large for WAV\n");
return 1;
}
*out_data = out_buf.data();
*out_data_len = out_buf.size();
return 0;
}
const size_t n_new = audio_pcm.size() - pcm_sent;
if (out_n_samples) {
*out_n_samples = (int64_t) (n_new / n_ch);
}
if (out_type == MTMD_HELPER_GEN_AUDIO_OUTTYPE_PCM) {
*out_data = (const char *) (audio_pcm.data() + pcm_sent);
*out_data_len = n_new * sizeof(float);
pcm_sent = audio_pcm.size();
return 0;
}
out_buf.clear();
if (!wav_header_sent) {
write_wav16_header(out_buf, UINT32_MAX, info.sample_rate, n_ch);
wav_header_sent = true;
}
append_wav16_pcm(out_buf, audio_pcm.data() + pcm_sent, n_new);
pcm_sent = audio_pcm.size();
*out_data = out_buf.data();
*out_data_len = out_buf.size();
return 0;
}
int32_t flush() override {
if (decoded) {
return 0;
}
if (codes.empty()) {
LOG_ERR("mtmd_helper_gen_audio: yue2 stopped before the first code\n");
return 1;
}
if (!done) {
// stopped by the caller: the NAR pass still needs <|music_end|>
if (!decode(&tok_music_end, 1)) {
return 1;
}
done = true;
}
// a song that fits in one chunk uses the backbone sequence itself, as it holds <|music_end|> now
const size_t n_codes = codes.size();
while (n_nar_done < n_codes) {
const size_t a = n_nar_done;
const size_t b = std::min(a + chunk_size(), n_codes);
if (!nar_chunk(a, b, a == 0 && b == n_codes)) {
return 1;
}
}
if (!vae_emit(true)) {
return 1;
}
scratch.reset();
decoded = true;
return 0;
}
private:
// token ids of the yue2_infer protocol (protocol.py)
static constexpr llama_token tok_eod = 151643;
static constexpr llama_token tok_abc_start = 151847;
static constexpr llama_token tok_abc_end = 151848;
static constexpr llama_token tok_music_start = 151851;
static constexpr llama_token tok_music_end = 151852;
static constexpr llama_token tok_codec0 = 151853;
static constexpr int32_t n_codec = 32768;
// NAR chunks and VAE tiles, see synthesize() and decode_tiled() of the reference
static constexpr int64_t n_ctx_max = 24576;
static constexpr size_t stream_chunk = 750; // 30 s
static constexpr size_t n_latent = 64;
static constexpr size_t vae_core = 128; // frames kept per tile
static constexpr size_t vae_halo = 16; // frames of context on each side
static constexpr size_t vae_ratio = 1920;
static constexpr size_t vae_trim = 64;
struct sampling {
float temp;
float top_p;
int32_t top_k;
float penalty;
int32_t penalty_window;
int32_t min_tokens;
int32_t max_tokens;
};
sampling abc_cfg = { 0.7f, 0.9f, 30, 1.005f, 100, 32, 4096 };
sampling code_cfg = { 1.0f, 0.95f, 100, 1.2f, 50, 200, 9000 };
bool decode_ctx(llama_context * ctx, const llama_token * toks, int32_t n, llama_seq_id seq, llama_pos & p) {
llama_batch_ext_ptr batch(llama_batch_ext_init(ctx));
if (!batch_set_tokens(batch.get(), toks, n, p, seq)) {
return false;
}
llama_batch_ext_set_output_logits(batch.get(), n - 1, true);
if (llama_process(ctx, LLAMA_PROCESS_TYPE_DECODE, batch.get()) != 0) {
LOG_ERR("mtmd_helper_gen_audio: decode failed\n");
return false;
}
p += n;
return true;
}
bool decode(const llama_token * toks, int32_t n) {
return decode_ctx(lctx, toks, n, seq_id, pos);
}
// keeps the logits of the next token, as the context logits belong to the last decode of any sequence (server slots)
// only the window that sample() reads: the text and </abc> for the score, <|music_end|> and the codes after it
// with guidance: uncond + scale * (cond - uncond)
void save_logits(const float * cond, const float * uncond = nullptr) {
const llama_token lo = in_codes ? tok_music_end : 0;
const llama_token hi = in_codes ? tok_codec0 + n_codec : tok_abc_end + 1;
logits.resize((size_t) llama_vocab_n_tokens(vocab));
for (llama_token t = lo; t < hi; t++) {
logits[t] = uncond ? uncond[t] + cfg_scale * (cond[t] - uncond[t]) : cond[t];
}
}
// the negative prompt of the reference (negative_prefix): the instruction, then the score of the cond prompt
// the logits of the cond prompt must be the last ones when this is called
bool start_uncond() {
if (cfg_scale == 1.0f) {
save_logits(llama_get_logits_ith(lctx, -1));
return true;
}
const float * c = llama_get_logits_ith(lctx, -1);
std::vector<float> cond(c, c + llama_vocab_n_tokens(vocab));
std::vector<llama_token> neg = neg_prefix;
if (mode_off) {
neg.push_back(tok_music_start);
} else {
neg.insert(neg.end(), abc_tail.begin(), abc_tail.end());
}
if (pos + (llama_pos) neg.size() + 2 >= (llama_pos) llama_n_ctx_seq(lctx)) {
LOG_ERR("mtmd_helper_gen_audio: the cfg prompt does not fit in the context\n");
return false;
}
const int32_t n_batch = (int32_t) llama_n_batch(lctx);
for (size_t i = 0; i < neg.size(); i += (size_t) n_batch) {
const int32_t n = std::min(n_batch, (int32_t) (neg.size() - i));
if (!decode_ctx(lctx, neg.data() + i, n, seq_uncond, upos)) {
return false;
}
}
save_logits(cond.data(), llama_get_logits_ith(lctx, -1));
return true;
}
bool decode_pair(llama_token tok) {
llama_batch_ext_ptr batch(llama_batch_ext_init(lctx));
const llama_seq_id seqs[2] = { seq_id, seq_uncond };
const llama_pos p[2] = { pos, upos };
for (int i = 0; i < 2; i++) {
const int32_t idx = llama_batch_ext_add_token(batch.get(), seqs[i], tok);
if (idx < 0) {
LOG_ERR("mtmd_helper_gen_audio: failed to add token %d to batch (error %d)\n", tok, idx);
return false;
}
llama_batch_ext_set_pos(batch.get(), idx, &p[i]);
llama_batch_ext_set_output_logits(batch.get(), idx, true);
}
if (llama_process(lctx, LLAMA_PROCESS_TYPE_DECODE, batch.get()) != 0) {
LOG_ERR("mtmd_helper_gen_audio: decode failed\n");
return false;
}
pos++;
upos++;
save_logits(llama_get_logits_ith(lctx, 0), llama_get_logits_ith(lctx, 1));
return true;
}
// song_chunks() of the reference: each NAR chunk sees the prefix and its own codes
// the reference uses the largest chunk that fits in n_ctx_max, streaming uses smaller ones
size_t chunk_size() const {
const int64_t ref = std::max<int64_t>(1, (n_ctx_max - (int64_t) prefix_tok.size() - 3) / 2);
return (size_t) (stream ? std::min<int64_t>(ref, stream_chunk) : ref);
}
// backbone keys/values of [prefix, codes[a, b), <|music_end|>] -> latents of the chunk
bool nar_chunk(size_t a, size_t b, bool use_main) {
llama_context * ctx = lctx;
llama_seq_id seq = seq_id;
llama_pos n_past = pos;
if (!use_main) {
// a scratch context on the backbone model, the prefix is decoded once
const llama_pos n_prefix = (llama_pos) prefix_tok.size();
const uint32_t n_need = (uint32_t) (n_prefix + (b - a) + 1);
if (!scratch || scratch_n_ctx < n_need) {
llama_context_params cparams = llama_context_default_params();
cparams.n_ctx = n_need;
cparams.n_seq_max = 1;
cparams.n_threads = llama_n_threads(lctx);
cparams.n_threads_batch = llama_n_threads_batch(lctx);
cparams.no_perf = true;
scratch.reset(llama_init_from_model(const_cast<llama_model *>(model), cparams));
if (!scratch) {
LOG_ERR("mtmd_helper_gen_audio: cannot create the yue2 chunk context\n");
return false;
}
scratch_n_ctx = n_need;
llama_pos p = 0;
const int32_t n_batch = (int32_t) llama_n_batch(scratch.get());
for (size_t i = 0; i < prefix_tok.size(); i += (size_t) n_batch) {
const int32_t n = std::min(n_batch, (int32_t) (prefix_tok.size() - i));
if (!decode_ctx(scratch.get(), prefix_tok.data() + i, n, 0, p)) {
return false;
}
}
}
llama_memory_seq_rm(llama_get_memory(scratch.get()), 0, n_prefix, -1);
std::vector<llama_token> toks;
for (size_t i = a; i < b; i++) {
toks.push_back(codes[i] + tok_codec0);
}
toks.push_back(tok_music_end);
llama_pos p = n_prefix;
const int32_t n_batch = (int32_t) llama_n_batch(scratch.get());
for (size_t i = 0; i < toks.size(); i += (size_t) n_batch) {
const int32_t n = std::min(n_batch, (int32_t) (toks.size() - i));
if (!decode_ctx(scratch.get(), toks.data() + i, n, 0, p)) {
return false;
}
}
ctx = scratch.get();
seq = 0;
n_past = p;
}
std::vector<uint8_t> state(llama_state_seq_get_size(ctx, seq));
if (state.empty() || llama_state_seq_get_data(ctx, state.data(), state.size(), seq) != state.size()) {
LOG_ERR("mtmd_helper_gen_audio: cannot read the backbone state\n");
return false;
}
mtmd_gen_inp inp = mtmd_gen_inp_default(mctx);
inp.type = MTMD_GEN_PROCESS_TYPE_GEN_CODE;
inp.n_codes = b - a;
inp.backbone_state = state.data();
inp.backbone_state_size = state.size();
inp.backbone_n_past = n_past;
inp.n_steps = flow_steps;
mtmd_gen_out out{};
if (mtmd_gen_audio_process(mctx, gen_state.get(), &inp, &out) != 0) {
LOG_ERR("mtmd_helper_gen_audio: flow matching failed\n");
return false;
}
latents.insert(latents.end(), out.feats, out.feats + out.n_feats);
n_nar_done = b;
return true;
}
// decodes the VAE tiles whose context on both sides is known, same cropping as decode_tiled() of the reference
bool vae_emit(bool final) {
const size_t n_lat = latents.size() / n_latent;
while (n_vae_done < n_lat) {
const size_t start = n_vae_done;
const size_t end = std::min(start + vae_core, n_lat);
if (!final && end + vae_halo > n_lat) {
break;
}
const size_t left = start >= vae_halo ? start - vae_halo : 0;
const size_t right = std::min(end + vae_halo, n_lat);
mtmd_gen_inp inp = mtmd_gen_inp_default(mctx);
inp.type = MTMD_GEN_PROCESS_TYPE_GEN_WAV;
inp.feats = latents.data() + left * n_latent;
inp.n_feats = (right - left) * n_latent;
mtmd_gen_out out{};
if (mtmd_gen_audio_process(mctx, gen_state.get(), &inp, &out) != 0) {
LOG_ERR("mtmd_helper_gen_audio: VAE decode failed\n");
return false;
}
// the whole song has vae_ratio * n - vae_trim samples
const size_t n_ch = (size_t) info.n_channels;
const size_t crop0 = (start - left) * vae_ratio;
const size_t n_keep = (end - start) * vae_ratio - (final && end == n_lat ? vae_trim : 0);
if (out.n_samples < (crop0 + n_keep) * n_ch) {
LOG_ERR("mtmd_helper_gen_audio: VAE tile is too short\n");
return false;
}
audio_pcm.insert(audio_pcm.end(), out.audio + crop0 * n_ch, out.audio + (crop0 + n_keep) * n_ch);
n_vae_done = end;
}
return true;
}
// distribution() of the reference: allowed range, no end token before min_tokens, windowed penalty, top-k, top-p
llama_token sample(const float * logits) {
const sampling & c = in_codes ? code_cfg : abc_cfg;
const llama_token end = in_codes ? tok_music_end : tok_abc_end;
const llama_token lo = in_codes ? tok_codec0 : 0;
const llama_token hi = in_codes ? tok_codec0 + n_codec : tok_eod;
std::unordered_map<llama_token, int> freq;
const size_t n_win = std::min(history.size(), (size_t) c.penalty_window);
for (size_t i = history.size() - n_win; i < history.size(); i++) {
freq[history[i]]++;
}
std::vector<std::pair<float, llama_token>> cand;
cand.reserve((size_t) (hi - lo) + 1);
auto push = [&](llama_token t) {
float l = logits[t];
auto it = freq.find(t);
if (it != freq.end()) {
const float alpha = std::pow(c.penalty, (float) it->second);
l = l < 0.0f ? l * alpha : l / alpha;
}
cand.emplace_back(l, t);
};
for (llama_token t = lo; t < hi; t++) {
push(t);
}
if ((int32_t) history.size() >= c.min_tokens) {
push(end);
}
if (c.temp <= 0.0f) {
return std::max_element(cand.begin(), cand.end())->second;
}
const size_t k = std::min(cand.size(), (size_t) std::max(c.top_k, 1));
std::partial_sort(cand.begin(), cand.begin() + k, cand.end(), std::greater<std::pair<float, llama_token>>());
cand.resize(k);
std::vector<double> p(k);
double sum = 0.0;
for (size_t i = 0; i < k; i++) {
p[i] = std::exp((double) ((cand[i].first - cand[0].first) / c.temp));
sum += p[i];
}
// keep the smallest prefix whose mass reaches top_p, the first token always stays (3 for the codes of cot=off)
const size_t n_keep = in_codes && mode_off ? 3 : 1;
double before = 0.0;
for (size_t i = 0; i < k; i++) {
const double pi = p[i];
if (i >= n_keep && before > c.top_p * sum) {
p[i] = 0.0;
}
before += pi;
}
double kept = 0.0;
for (double v : p) {
kept += v;
}
double u = std::uniform_real_distribution<double>(0.0, kept)(rng);
for (size_t i = 0; i < k; i++) {
u -= p[i];
if (u <= 0.0 && p[i] > 0.0) {
return cand[i].second;
}
}
return cand[0].second;
}
llama_seq_id seq_id = 0;
llama_pos pos = 0;
std::vector<llama_token> prompt;
int32_t prompt_pos = 0;
bool in_codes = false;
bool mode_off = false; // cot=off: no score
float cfg_scale = 1.0f;
int32_t flow_steps = 0; // 0 = the model default
llama_seq_id seq_uncond = 0;
llama_pos upos = 0;
std::vector<llama_token> neg_prefix; // EOD + instruction
std::vector<llama_token> abc_tail; // the cond tokens from <abc> on
std::vector<float> logits; // see save_logits()
std::vector<llama_token> prefix_tok; // the cond tokens before the first code
std::vector<float> latents; // of the chunks done so far
size_t n_nar_done = 0; // codes
size_t n_vae_done = 0; // latent frames
llama_context_ptr scratch; // backbone keys/values of a chunk other than the first
uint32_t scratch_n_ctx = 0;
std::vector<llama_token> history; // tokens sampled in the current phase
std::vector<int32_t> codes;
bool done = false;
bool decoded = false;
std::mt19937 rng;
mtmd_helper_gen_audio_outtype out_type = MTMD_HELPER_GEN_AUDIO_OUTTYPE_WAV;
bool stream = false;
std::vector<float> h_state_buf;
std::vector<float> audio_pcm; // interleaved
std::vector<char> out_buf;
size_t pcm_sent = 0;
bool wav_header_sent = false;
};
static std::unique_ptr<mtmd_gen_audio_pipeline> make_pipeline(llama_context * lctx, mtmd_context * mctx) {
switch (mtmd_gen_audio_get_info(mctx).type) {
case MTMD_GEN_AUDIO_TYPE_QWEN3TTS:
return std::unique_ptr<mtmd_gen_audio_pipeline>(new qwen3tts_gen_audio_pipeline(lctx, mctx));
case MTMD_GEN_AUDIO_TYPE_POCKETTTS:
return std::unique_ptr<mtmd_gen_audio_pipeline>(new pockettts_gen_audio_pipeline(lctx, mctx));
case MTMD_GEN_AUDIO_TYPE_SOPRANO:
return std::unique_ptr<mtmd_gen_audio_pipeline>(new soprano_gen_audio_pipeline(lctx, mctx));
case MTMD_GEN_AUDIO_TYPE_LFM3A:
return std::unique_ptr<mtmd_gen_audio_pipeline>(new lfm3a_gen_audio_pipeline(lctx, mctx));
case MTMD_GEN_AUDIO_TYPE_CHATTERBOX:
return std::unique_ptr<mtmd_gen_audio_pipeline>(new chatterbox_gen_audio_pipeline(lctx, mctx));
case MTMD_GEN_AUDIO_TYPE_DRAMA3:
return std::unique_ptr<mtmd_gen_audio_pipeline>(new drama3_gen_audio_pipeline(lctx, mctx));
case MTMD_GEN_AUDIO_TYPE_ZONOS3:
return std::unique_ptr<mtmd_gen_audio_pipeline>(new zonos3_gen_audio_pipeline(lctx, mctx));
case MTMD_GEN_AUDIO_TYPE_YUE2:
return std::unique_ptr<mtmd_gen_audio_pipeline>(new yue2_gen_audio_pipeline(lctx, mctx));
case MTMD_GEN_AUDIO_TYPE_FISH_S2:
return std::unique_ptr<mtmd_gen_audio_pipeline>(new fish_s2_gen_audio_pipeline(lctx, mctx));
default:
return nullptr;
}
}
struct mtmd_helper_gen_audio {
llama_context * lctx = nullptr;
std::unique_ptr<mtmd_gen_audio_pipeline> pipeline;
};
mtmd_helper_gen_audio * mtmd_helper_gen_audio_init(struct llama_context * lctx, struct mtmd_context * mctx) {
auto * ctx = new mtmd_helper_gen_audio();
ctx->lctx = lctx;
ctx->pipeline = make_pipeline(lctx, mctx);
if (ctx->pipeline && !ctx->pipeline->check_context()) {
delete ctx;
return nullptr;
}
return ctx;
}
void mtmd_helper_gen_audio_free(mtmd_helper_gen_audio * ctx) {
delete ctx;
}
int32_t mtmd_helper_gen_audio_max_frames(const mtmd_helper_gen_audio * ctx) {
return ctx && ctx->pipeline ? ctx->pipeline->max_frames() : 0;
}
void mtmd_helper_gen_audio_reset(mtmd_helper_gen_audio * ctx) {
if (ctx && ctx->pipeline) {
ctx->pipeline->reset();
}
}
struct mtmd_helper_gen_audio_inp mtmd_helper_gen_audio_inp_default(void) {
mtmd_helper_gen_audio_inp inp{};
inp.top_k = 50;
inp.top_p = 1.0f;
inp.temp = -1.0f; // model default
inp.seed = UINT32_MAX; // random
inp.speed = 1.0f;
inp.loudness = 0.0f;
inp.cfg_scale = 0.0f; // model default
inp.flow_steps = 0; // model default
inp.out_type = MTMD_HELPER_GEN_AUDIO_OUTTYPE_WAV;
return inp;
}
int32_t mtmd_helper_gen_audio_set_input(mtmd_helper_gen_audio * ctx, const mtmd_helper_gen_audio_inp * inp) {
if (!ctx->pipeline) {
LOG_ERR("mtmd_helper_gen_audio: unsupported or missing gen-audio pipeline\n");
return 1;
}
if (inp->n_past > 0 && !ctx->pipeline->supports_continue()) {
LOG_ERR("mtmd_helper_gen_audio: the pipeline cannot continue a sequence\n");
return 1;
}
if (!std::isfinite(inp->speed) || inp->speed <= 0.0f || !std::isfinite(inp->loudness) || inp->loudness > 0.0f) {
LOG_ERR("mtmd_helper_gen_audio: speed must be > 0 and loudness < 0 LUFS (0 = model default)\n");
return 2;
}
if ((inp->speed != 1.0f || inp->loudness != 0.0f) && !ctx->pipeline->supports_delivery()) {
LOG_ERR("mtmd_helper_gen_audio: the model does not support speed or loudness\n");
return 2;
}
if (!std::isfinite(inp->cfg_scale) || inp->cfg_scale < 0.0f) {
LOG_ERR("mtmd_helper_gen_audio: cfg_scale must be > 0 (0 = model default)\n");
return 2;
}
if (inp->cfg_scale != 0.0f && !ctx->pipeline->supports_cfg()) {
LOG_ERR("mtmd_helper_gen_audio: the model does not support cfg_scale\n");
return 2;
}
if (inp->flow_steps < 0 || inp->flow_steps > 1024) {
LOG_ERR("mtmd_helper_gen_audio: flow_steps must be in [1, 1024] (0 = model default)\n");
return 2;
}
if (inp->flow_steps != 0 && !ctx->pipeline->supports_flow_steps()) {
LOG_ERR("mtmd_helper_gen_audio: the model does not support flow_steps\n");
return 2;
}
ctx->pipeline->logit_bias.clear();
return ctx->pipeline->set_input(inp);
}
int32_t mtmd_helper_gen_audio_step_prompt(mtmd_helper_gen_audio * ctx, int32_t n_batch) {
if (!ctx->pipeline) {
return -1;
}
// the context may use a smaller n_batch than requested (e.g. clamped to n_ctx), llama_process() would fail above it
if (n_batch <= 0 || (uint32_t) n_batch > llama_n_batch(ctx->lctx)) {
LOG_ERR("mtmd_helper_gen_audio: n_batch must be in [1, %u], got %d\n", llama_n_batch(ctx->lctx), n_batch);
return -1;
}
return ctx->pipeline->step_prompt(n_batch);
}
bool mtmd_helper_gen_audio_next_is_text(mtmd_helper_gen_audio * ctx) {
return ctx->pipeline && ctx->pipeline->next_is_text();
}
const llama_logit_bias * mtmd_helper_gen_audio_get_logit_bias(mtmd_helper_gen_audio * ctx, size_t * n_bias) {
*n_bias = ctx->pipeline ? ctx->pipeline->logit_bias.size() : 0;
return *n_bias ? ctx->pipeline->logit_bias.data() : nullptr;
}
int32_t mtmd_helper_audio_stream_get_logit_bias(const mtmd_context * mctx, const llama_model * model,
llama_logit_bias * out, int32_t n_max) {
std::vector<llama_logit_bias> bias;
if (mtmd_gen_audio_get_info(mctx).type == MTMD_GEN_AUDIO_TYPE_LFM3A) {
// LFM3-Audio live ASR: only <|text_end|> ends the text of a chunk, the stream adds the final <|mixed_end|>
const llama_vocab * vocab = llama_model_get_vocab(model);
for (const char * piece : { "<|mixed_end|>", "<|mixed_start|>", "<|audio_start|>", "<|im_start|>", "<|im_end|>", "<|endoftext|>" }) {
const llama_token tok = find_special_token(vocab, piece);
if (tok != LLAMA_TOKEN_NULL) {
bias.push_back({ tok, -INFINITY });
}
}
}
for (int32_t i = 0; i < n_max && i < (int32_t) bias.size(); i++) {
out[i] = bias[i];
}
return (int32_t) bias.size();
}
int32_t mtmd_helper_gen_audio_step_gen(mtmd_helper_gen_audio * ctx, llama_token sampled,
const float * h_state_in, const float ** h_state_out,
bool * out_stop) {
if (!ctx->pipeline) {
return 1;
}
bool stop = false;
const int32_t ret = ctx->pipeline->step_gen(sampled, h_state_in, h_state_out, &stop);
if (out_stop) {
*out_stop = stop;
}
return ret;
}
int32_t mtmd_helper_gen_audio_get_output(mtmd_helper_gen_audio * ctx, int32_t * out_sample_rate,
const char ** out_data, size_t * out_data_len, int64_t * out_n_samples) {
if (!ctx->pipeline) {
return 1;
}
return ctx->pipeline->get_output(out_sample_rate, out_data, out_data_len, out_n_samples);
}
int32_t mtmd_helper_gen_audio_flush(mtmd_helper_gen_audio * ctx) {
if (!ctx->pipeline) {
return 1;
}
return ctx->pipeline->flush();
}