Files
llama.cpp/tools/mtmd/mtmd-helper-gen.cpp
T
Xuan-Son Nguyen a68c6136bb mtmd: support fish audio s2 (#32330)
* add fisk-audio-s2 support

* add voice cloning

* cont merge master

* nits

* nits

* clean up

* nits

* use new batch api
2026-11-16 17:32:27 +01:00

3302 lines
125 KiB
C++

#include "mtmd.h"
#include "mtmd-helper.h"
#include "mtmd-helper-common.h"
#include "llama.h"
#include "../src/llama-ext.h"
#include <algorithm>
#include <cctype>
#include <cmath>
#include <cstring>
#include <functional>
#include <memory>
#include <random>
#include <string>
#include <unordered_map>
#include <vector>
#ifdef MTMD_INTERNAL_HEADER
#error "mtmd-helper is a public library outside of mtmd. it must not include internal headers"
#endif
//
// Audio generation helpers
//
// --tts-lang codes -> language names used by the codec_language special tokens
static const std::unordered_map<std::string, std::string> tts_lang_codes = {
{ "zh", "chinese" },
{ "en", "english" },
{ "de", "german" },
{ "it", "italian" },
{ "pt", "portuguese" },
{ "es", "spanish" },
{ "ja", "japanese" },
{ "ko", "korean" },
{ "fr", "french" },
{ "ru", "russian" },
};
static std::string tts_resolve_lang(const std::string & lang) {
auto it = tts_lang_codes.find(lang);
return it != tts_lang_codes.end() ? it->second : lang;
}
static llama_token find_special_token(const llama_vocab * vocab, const std::string & piece) {
const int32_t n = llama_vocab_n_tokens(vocab);
for (llama_token t = 0; t < n; t++) {
if (piece == llama_vocab_get_text(vocab, t)) {
return t;
}
}
return LLAMA_TOKEN_NULL;
}
static bool batch_set_tokens(llama_batch_ext * batch, const llama_token * tokens, int32_t n, llama_pos pos_0, llama_seq_id seq_id) {
llama_batch_ext_clear(batch);
for (int32_t i = 0; i < n; i++) {
const int32_t idx = llama_batch_ext_add_token(batch, seq_id, tokens[i]);
const llama_pos pos = pos_0 + i;
if (idx < 0) {
LOG_ERR("mtmd_helper_gen_audio: failed to add token %d to batch (error %d)\n", tokens[i], idx);
return false;
}
llama_batch_ext_set_pos(batch, idx, &pos);
}
return true;
}
static void put_bytes(std::vector<char> & buf, const void * p, size_t n) {
const char * c = (const char *) p;
buf.insert(buf.end(), c, c + n);
}
// data_sz == UINT32_MAX writes the "unknown length" sentinel (streaming), same as ffmpeg does on a pipe
static void write_wav16_header(std::vector<char> & buf, uint32_t data_sz, int32_t rate) {
const uint32_t riff_sz = data_sz == UINT32_MAX ? UINT32_MAX : 36 + data_sz;
const uint32_t fmt_sz = 16, byte_rate = (uint32_t) rate * 2;
const uint16_t fmt = 1, ch = 1, align = 2, bits = 16;
const uint32_t rate32 = (uint32_t) rate;
put_bytes(buf, "RIFF", 4); put_bytes(buf, &riff_sz, 4); put_bytes(buf, "WAVE", 4);
put_bytes(buf, "fmt ", 4); put_bytes(buf, &fmt_sz, 4);
put_bytes(buf, &fmt, 2); put_bytes(buf, &ch, 2); put_bytes(buf, &rate32, 4);
put_bytes(buf, &byte_rate, 4); put_bytes(buf, &align, 2); put_bytes(buf, &bits, 2);
put_bytes(buf, "data", 4); put_bytes(buf, &data_sz, 4);
}
static void append_wav16_pcm(std::vector<char> & buf, const float * pcm, size_t n) {
for (size_t i = 0; i < n; i++) {
int16_t s = (int16_t) std::lround(std::max(-1.0f, std::min(1.0f, pcm[i])) * 32767.0f);
put_bytes(buf, &s, 2);
}
}
static bool write_wav16(std::vector<char> & buf, const std::vector<float> & pcm, int32_t rate) {
// RIFF chunk sizes are 32-bit; refuse to emit a file with a truncated header
if (pcm.size() > ((size_t) UINT32_MAX - 36) / 2) {
return false;
}
write_wav16_header(buf, (uint32_t) (pcm.size() * 2), rate);
append_wav16_pcm(buf, pcm.data(), pcm.size());
return true;
}
class mtmd_gen_audio_pipeline {
public:
mtmd_gen_audio_pipeline(llama_context * lctx, mtmd_context * mctx)
: lctx(lctx), mctx(mctx), model(llama_get_model(lctx)), vocab(llama_model_get_vocab(model)),
n_embd(llama_model_n_embd(model)), info(mtmd_gen_audio_get_info(mctx)) {}
virtual ~mtmd_gen_audio_pipeline() = default;
virtual void reset() = 0;
virtual int32_t set_input(const mtmd_helper_gen_audio_inp * inp) = 0;
// decodes at most n_batch prompt tokens; returns remaining count (0 = done), <0 on error
virtual int32_t step_prompt(int32_t n_batch) = 0;
// sampled can be LLAMA_TOKEN_NULL for pipelines with no discrete backbone token,
// those read what they need from h_state_in instead
// set out_stop on end-of-speech, h_state_out must be null if no frame is generated
virtual int32_t step_gen(llama_token sampled, const float * h_state_in, const float ** h_state_out, bool * out_stop) = 0;
virtual int32_t get_output(int32_t * out_sample_rate, const char ** out_data, size_t * out_data_len, int64_t * out_n_samples) = 0;
// forces any buffered codes through code2wav now, regardless of window_frames
virtual int32_t flush() = 0;
// continuing a sequence (inp->n_past > 0) with the model's own text/audio switching
virtual bool supports_continue() const { return false; }
// mtmd_helper_gen_audio_inp::speed and loudness other than the defaults
virtual bool supports_delivery() const { return false; }
virtual bool next_is_text() const { return false; }
std::vector<llama_logit_bias> logit_bias; // see mtmd_helper_gen_audio_get_logit_bias()
protected:
// one state per generation, so a fixed seed always gives the same output
bool init_gen_state(uint32_t seed) {
mtmd_gen_state_params params = mtmd_gen_state_params_default();
params.seed = seed;
gen_state.reset(mtmd_gen_state_init(mctx, params));
if (!gen_state) {
LOG_ERR("mtmd_helper_gen_audio: failed to create the gen state\n");
return false;
}
return true;
}
// encodes a speaker clip or a voice, out receives the encoder embeddings
bool encode_speaker(mtmd_bitmap * bitmap, std::vector<float> & out) {
if (!mtmd_bitmap_is_voice(bitmap) && !mtmd_support_audio(mctx)) {
LOG_ERR("mtmd_helper_gen_audio: mmproj has no speaker/audio encoder\n");
return false;
}
const std::string marker = mtmd_get_marker(mctx);
mtmd_input_text text{ marker.c_str(), marker.size(), false, true };
mtmd_input_chunks * chunks = mtmd_input_chunks_init();
const mtmd_bitmap * bptr = bitmap;
bool ok = mtmd_tokenize(mctx, chunks, &text, &bptr, 1) == 0;
if (ok) {
ok = false;
for (size_t i = 0; i < mtmd_input_chunks_size(chunks); i++) {
const mtmd_input_chunk * chunk = mtmd_input_chunks_get(chunks, i);
if (mtmd_input_chunk_get_type(chunk) != MTMD_INPUT_CHUNK_TYPE_AUDIO) {
continue;
}
if (mtmd_encode_chunk(mctx, chunk) != 0) {
LOG_ERR("mtmd_helper_gen_audio: speaker encode failed\n");
break;
}
const float * embd = mtmd_get_output_embd(mctx);
const size_t n = (size_t) llama_model_n_embd_inp(model) * mtmd_input_chunk_get_n_tokens(chunk);
out.assign(embd, embd + n);
ok = true;
break;
}
}
mtmd_input_chunks_free(chunks);
return ok;
}
llama_context * lctx;
mtmd_context * mctx;
const llama_model * model;
const llama_vocab * vocab;
int n_embd;
mtmd_gen_audio_info info;
mtmd::gen_state_ptr gen_state;
};
// Qwen3-TTS: backbone samples codec_0, code_predictor gives the other 15 codebooks,
// then code2wav decodes them to PCM
class qwen3tts_gen_audio_pipeline : public mtmd_gen_audio_pipeline {
public:
using mtmd_gen_audio_pipeline::mtmd_gen_audio_pipeline;
void reset() override {
seq_id = 0;
pos = 0;
codes_buf.clear();
gen_state.reset();
audio_pcm.clear();
overlay.clear();
h_state_buf.clear();
out_buf.clear();
prompt_embd_buf.clear();
prompt_batch.reset();
n_prompt = 0;
prompt_pos = 0;
stream = false;
pcm_sent = 0;
wav_header_sent = false;
}
int32_t set_input(const mtmd_helper_gen_audio_inp * inp) override {
reset();
seq_id = inp->seq_id;
if (!ensure_cache()) {
return 1;
}
const std::string lang = tts_resolve_lang((inp->lang && inp->lang[0]) ? inp->lang : "english");
const llama_token c_lang = find_special_token(vocab, ("<|codec_language_" + lang + "|>").c_str());
if (c_lang == LLAMA_TOKEN_NULL) {
LOG_ERR("mtmd_helper_gen_audio: unknown language '%s'\n", lang.c_str());
return 1;
}
std::vector<float> speaker_embd;
if (inp->speaker_ref) {
if (!encode_speaker(inp->speaker_ref, speaker_embd)) {
return 1;
}
}
const int n_e = n_embd;
auto row = [&](llama_token t) {
return std::vector<float>(tok_embd.begin() + (size_t) t * n_e,
tok_embd.begin() + (size_t) (t + 1) * n_e);
};
auto sum_row = [&](llama_token a, llama_token b) {
std::vector<float> va = row(a), vb = row(b);
for (int i = 0; i < n_e; i++) va[(size_t) i] += vb[(size_t) i];
return va;
};
auto sum_vec = [&](llama_token a, const std::vector<float> & vb) {
std::vector<float> va = row(a);
for (int i = 0; i < n_e; i++) va[(size_t) i] += vb[(size_t) i];
return va;
};
// upstream chat wrap, then slices: [0:3] role, [3:-5] utterance body
const std::string full = "<|im_start|>assistant\n" + std::string(inp->prompt, inp->prompt_len) +
"<|im_end|>\n<|im_start|>assistant\n";
std::vector<llama_token> ids(full.size() + 16);
int n_ids = llama_tokenize(vocab, full.c_str(), (int32_t) full.size(), ids.data(), (int32_t) ids.size(),
false, true);
if (n_ids < 8) {
LOG_ERR("mtmd_helper_gen_audio: tokenization failed\n");
return 1;
}
ids.resize((size_t) n_ids);
std::vector<std::vector<float>> prompt;
for (int i = 0; i < 3; i++) prompt.push_back(row(ids[(size_t) i]));
prompt.push_back(sum_row(tts_pad, c_think));
prompt.push_back(sum_row(tts_pad, c_think_b));
prompt.push_back(sum_row(tts_pad, c_lang));
prompt.push_back(sum_row(tts_pad, c_think_e));
if (!speaker_embd.empty()) prompt.push_back(sum_vec(tts_pad, speaker_embd));
prompt.push_back(sum_row(tts_bos, codec_pad));
for (int i = 3; i < n_ids - 5; i++) prompt.push_back(sum_row(ids[(size_t) i], codec_pad));
prompt.push_back(sum_row(tts_eos, codec_pad));
prompt.push_back(sum_row(tts_pad, codec_bos));
n_prompt = (int) prompt.size();
// the talker uses the qwen3vl interleaved mrope, all sections are equal for a text/codec stream
mrope = llama_model_rope_type(model) == LLAMA_ROPE_TYPE_MROPE ||
llama_model_rope_type(model) == LLAMA_ROPE_TYPE_IMROPE;
const int n_pos_per_embd = mrope ? 4 : 1;
prompt_embd_buf.resize((size_t) n_prompt * (size_t) n_e);
for (int i = 0; i < n_prompt; i++) {
memcpy(prompt_embd_buf.data() + (size_t) i * n_e, prompt[(size_t) i].data(), (size_t) n_e * sizeof(float));
}
prompt_batch.reset(new decode_embd_batch(prompt_embd_buf.data(), n_prompt, n_pos_per_embd, n_e));
if (mrope) prompt_batch->set_position_mrope_1d(0, seq_id);
else prompt_batch->set_position_normal (0, seq_id);
prompt_pos = 0;
pos = 0;
const mtmd_gen_inp def = mtmd_gen_inp_default(mctx);
top_k = inp->top_k > 0 ? inp->top_k : def.top_k;
top_p = inp->top_p > 0 ? inp->top_p : def.top_p;
out_type = inp->out_type;
stream = inp->stream;
if (!init_gen_state(inp->seed)) {
return 1;
}
// the prompt above holds the whole text stream up to tts_eos, so every generated
// frame adds tts_pad on top of the codes embedding
overlay = row(tts_pad);
return 0;
}
int32_t step_prompt(int32_t n_batch) override {
GGML_ASSERT(n_batch > 0);
if (prompt_pos >= n_prompt) {
return 0;
}
const int32_t n_tokens_batch = std::min(n_batch, n_prompt - prompt_pos);
const bool is_last_batch = (prompt_pos + n_tokens_batch) == n_prompt;
if (is_last_batch) {
prompt_batch->logits[prompt_pos + n_tokens_batch - 1] = 1;
}
if (llama_process(lctx, LLAMA_PROCESS_TYPE_DECODE, prompt_batch->render(lctx, prompt_pos, n_tokens_batch)) != 0) {
LOG_ERR("mtmd_helper_gen_audio: prompt decode failed\n");
return -1;
}
pos += n_tokens_batch;
prompt_pos += n_tokens_batch;
if (prompt_pos >= n_prompt) {
// prompt fully processed, its embedding buffer is no longer needed
prompt_batch.reset();
prompt_embd_buf.clear();
return 0;
}
return n_prompt - prompt_pos;
}
int32_t step_gen(llama_token sampled, const float * h_state_in, const float ** h_state_out, bool * out_stop) override {
if (sampled == LLAMA_TOKEN_NULL) {
LOG_ERR("mtmd_helper_gen_audio: qwen3tts requires a token sampled from the backbone\n");
return 1;
}
// backbone signals end-of-speech with a token, no frame for this step
if (sampled == codec_eos || llama_vocab_is_eog(vocab, sampled)) {
*out_stop = true;
*h_state_out = nullptr;
return 0;
}
mtmd_gen_inp inp = mtmd_gen_inp_default(mctx);
inp.type = MTMD_GEN_PROCESS_TYPE_GEN_CODE;
inp.code0 = sampled - codec_0;
inp.embd = const_cast<float *>(h_state_in);
inp.top_k = top_k;
inp.top_p = top_p;
mtmd_gen_out out{};
if (mtmd_gen_audio_process(mctx, gen_state.get(), &inp, &out) != 0) {
LOG_ERR("mtmd_helper_gen_audio: gen_code process failed\n");
return 1;
}
codes_buf.insert(codes_buf.end(), out.codes, out.codes + out.n_codes);
if (out.n_codes > 0 && codes_buf.size() / out.n_codes >= window_frames) {
if (!flush_gen_wav()) {
return 1;
}
}
std::vector<float> fb(out.embd, out.embd + n_embd);
for (int i = 0; i < n_embd; i++) fb[(size_t) i] += overlay[(size_t) i];
const int n_pos_per_embd = mrope ? 4 : 1;
decode_embd_batch batch_embd(fb.data(), 1, n_pos_per_embd, n_embd);
if (mrope) batch_embd.set_position_mrope_1d(pos, seq_id);
else batch_embd.set_position_normal (pos, seq_id);
batch_embd.logits[0] = 1;
pos++;
if (llama_process(lctx, LLAMA_PROCESS_TYPE_DECODE, batch_embd.render(lctx, 0, 1)) != 0) {
LOG_ERR("mtmd_helper_gen_audio: decode failed\n");
return 1;
}
const float * he = llama_get_embeddings_ith(lctx, -1);
h_state_buf.assign(he, he + n_embd);
*h_state_out = h_state_buf.data();
return 0;
}
int32_t get_output(int32_t * out_sample_rate, const char ** out_data, size_t * out_data_len, int64_t * out_n_samples) override {
*out_sample_rate = info.sample_rate;
if (!stream) {
// one-shot call: force out whatever's left, regardless of window_frames
if (!flush_gen_wav()) {
return 1;
}
if (out_n_samples) {
*out_n_samples = (int64_t) audio_pcm.size();
}
if (out_type == MTMD_HELPER_GEN_AUDIO_OUTTYPE_PCM) {
*out_data = (const char *) audio_pcm.data();
*out_data_len = audio_pcm.size() * sizeof(float);
return 0;
}
out_buf.clear();
if (!write_wav16(out_buf, audio_pcm, info.sample_rate)) {
LOG_ERR("mtmd_helper_gen_audio: output too large for WAV\n");
return 1;
}
*out_data = out_buf.data();
*out_data_len = out_buf.size();
return 0;
}
// streaming: only return audio produced since the previous call
const size_t n_new = audio_pcm.size() - pcm_sent;
if (out_n_samples) {
*out_n_samples = (int64_t) n_new;
}
if (out_type == MTMD_HELPER_GEN_AUDIO_OUTTYPE_PCM) {
*out_data = (const char *) (audio_pcm.data() + pcm_sent);
*out_data_len = n_new * sizeof(float);
pcm_sent = audio_pcm.size();
return 0;
}
out_buf.clear();
if (!wav_header_sent) {
write_wav16_header(out_buf, UINT32_MAX, info.sample_rate);
wav_header_sent = true;
}
append_wav16_pcm(out_buf, audio_pcm.data() + pcm_sent, n_new);
pcm_sent = audio_pcm.size();
*out_data = out_buf.data();
*out_data_len = out_buf.size();
return 0;
}
int32_t flush() override {
return flush_gen_wav() ? 0 : 1;
}
private:
bool ensure_cache() {
if (specials_ok) {
return true;
}
codec_0 = find_special_token(vocab, "<|codec_0|>");
codec_bos = find_special_token(vocab, "<|codec_bos|>");
codec_eos = find_special_token(vocab, "<|codec_eos_token|>");
codec_pad = find_special_token(vocab, "<|codec_pad|>");
c_think = find_special_token(vocab, "<|codec_think|>");
c_think_b = find_special_token(vocab, "<|codec_think_bos|>");
c_think_e = find_special_token(vocab, "<|codec_think_eos|>");
tts_pad = find_special_token(vocab, "<tts_pad>");
tts_bos = find_special_token(vocab, "<tts_text_bos>");
tts_eos = find_special_token(vocab, "<tts_text_eod>");
for (llama_token t : { codec_0, codec_bos, codec_eos, codec_pad,
c_think, c_think_b, c_think_e,
tts_pad, tts_bos, tts_eos }) {
if (t == LLAMA_TOKEN_NULL) {
LOG_ERR("mtmd_helper_gen_audio: missing a required special token in vocab\n");
return false;
}
}
const uint32_t n_tok_embd = llama_model_get_tok_embd(model, nullptr);
if (n_tok_embd == 0) {
LOG_ERR("mtmd_helper_gen_audio: model has no token embeddings\n");
return false;
}
tok_embd.resize(n_tok_embd);
if (llama_model_get_tok_embd(model, tok_embd.data()) != n_tok_embd) {
LOG_ERR("mtmd_helper_gen_audio: token embedding copy failed\n");
return false;
}
specials_ok = true;
return true;
}
// one GEN_WAV process() call over the buffered codes, state is carried across batches
bool flush_gen_wav() {
if (codes_buf.empty()) {
return true;
}
mtmd_gen_inp inp = mtmd_gen_inp_default(mctx);
inp.type = MTMD_GEN_PROCESS_TYPE_GEN_WAV;
inp.codes = codes_buf.data();
inp.n_codes = codes_buf.size();
mtmd_gen_out out{};
if (mtmd_gen_audio_process(mctx, gen_state.get(), &inp, &out) != 0) {
LOG_ERR("mtmd_helper_gen_audio: gen_wav process failed\n");
return false;
}
audio_pcm.insert(audio_pcm.end(), out.audio, out.audio + out.n_samples);
codes_buf.clear();
return true;
}
// vocab specials fixed across the whole session, looked up once
bool specials_ok = false;
llama_token codec_0 = LLAMA_TOKEN_NULL;
llama_token codec_bos = LLAMA_TOKEN_NULL;
llama_token codec_eos = LLAMA_TOKEN_NULL;
llama_token codec_pad = LLAMA_TOKEN_NULL;
llama_token c_think = LLAMA_TOKEN_NULL;
llama_token c_think_b = LLAMA_TOKEN_NULL;
llama_token c_think_e = LLAMA_TOKEN_NULL;
llama_token tts_pad = LLAMA_TOKEN_NULL;
llama_token tts_bos = LLAMA_TOKEN_NULL;
llama_token tts_eos = LLAMA_TOKEN_NULL;
std::vector<float> tok_embd; // whole token embedding matrix, n_vocab * n_embd
// must match hparams.wav_tfm_swa hardcoded in mtmd-model.cpp
size_t window_frames = 72;
// per-generation state, cleared by reset()
llama_seq_id seq_id = 0;
bool mrope = false;
int pos = 0;
// prompt decode state, consumed batch-by-batch by step_prompt()
std::vector<float> prompt_embd_buf;
std::unique_ptr<decode_embd_batch> prompt_batch;
int n_prompt = 0;
int prompt_pos = 0;
int32_t top_k = 50;
float top_p = 1.0f;
std::vector<int32_t> codes_buf;
std::vector<float> audio_pcm;
std::vector<float> overlay;
std::vector<float> h_state_buf;
mtmd_helper_gen_audio_outtype out_type = MTMD_HELPER_GEN_AUDIO_OUTTYPE_WAV;
std::vector<char> out_buf;
bool stream = false;
size_t pcm_sent = 0; // samples already returned by get_output()
bool wav_header_sent = false;
};
// settings that only live in the reference's per-pack yaml, not in the checkpoint
// the english packs share the same shapes and tokenizer, but disagree on these
// both are 0 / false when the pack does not tune them, the model default is then used
struct pockettts_pack_settings {
int frames_after_eos = 0;
bool pad_short_text = false;
};
static pockettts_pack_settings pockettts_pack(const char * variant) {
static const std::unordered_map<std::string, pockettts_pack_settings> packs = {
{ "english", { 0, false } },
{ "english_2026-01", { 0, true } },
{ "english_2026-04", { 0, false } },
{ "french_24l", { 8, false } },
{ "italian", { 0, false } },
};
auto it = packs.find(variant ? variant : "");
if (it == packs.end()) {
LOG_WRN("mtmd_helper_gen_audio: no tuned settings for pocket-tts variant \"%s\"\n",
variant ? variant : "");
return {};
}
return it->second;
}
// pocket-tts: the backbone emits no token, the flow net turns each hidden state into a latent
// the end-of-speech head also lives in the mmproj
class pockettts_gen_audio_pipeline : public mtmd_gen_audio_pipeline {
public:
using mtmd_gen_audio_pipeline::mtmd_gen_audio_pipeline;
void reset() override {
seq_id = 0;
pos = 0;
feats_buf.clear();
gen_state.reset();
audio_pcm.clear();
h_state_buf.clear();
out_buf.clear();
prompt_embd_buf.clear();
prompt_batch.reset();
n_prompt = 0;
prompt_pos = 0;
step_idx = 0;
eos_step = -1;
chunks.clear();
chunk_idx = 0;
n_voice_pos = 0;
chunk_budget = 0;
stream = false;
pcm_sent = 0;
wav_header_sent = false;
}
int32_t set_input(const mtmd_helper_gen_audio_inp * inp) override {
reset();
seq_id = inp->seq_id;
if (!ensure_cache()) {
return 1;
}
std::vector<float> voice;
if (inp->speaker_ref) {
if (!encode_speaker(inp->speaker_ref, voice)) {
return 1;
}
}
pack = pockettts_pack(info.model_variant);
const std::string text = prepare_text(std::string(inp->prompt, inp->prompt_len),
pack.pad_short_text);
if (text.empty()) {
LOG_ERR("mtmd_helper_gen_audio: empty prompt\n");
return 1;
}
std::vector<llama_token> ids(text.size() + 16);
int n_ids = llama_tokenize(vocab, text.c_str(), (int32_t) text.size(), ids.data(),
(int32_t) ids.size(), false, false);
if (n_ids <= 0) {
LOG_ERR("mtmd_helper_gen_audio: tokenization failed\n");
return 1;
}
ids.resize((size_t) n_ids);
// long inputs degrade badly, so each chunk restarts from the voice conditioning
// see split_into_best_sentences() in the reference
chunks = split_chunks(ids);
chunk_idx = 0;
if (chunks.size() > 1) {
LOG_INF("mtmd_helper_gen_audio: %d tokens split into %zu chunks\n", n_ids, chunks.size());
}
const int n_e = n_embd;
// sequence order is voice, then text, then the audio BOS that starts generation
if (!voice.empty()) {
GGML_ASSERT(voice.size() % (size_t) n_e == 0);
if (bos_before_voice != LLAMA_TOKEN_NULL) {
push_embd_row(prompt_embd_buf, bos_before_voice);
}
prompt_embd_buf.insert(prompt_embd_buf.end(), voice.begin(), voice.end());
}
// every later chunk rewinds to here and re-prompts, so the voice stays primed
n_voice_pos = (int) (prompt_embd_buf.size() / (size_t) n_e);
for (llama_token t : chunks[0]) {
push_embd_row(prompt_embd_buf, t);
}
push_embd_row(prompt_embd_buf, audio_bos);
arm_chunk_budget(0);
n_prompt = (int) (prompt_embd_buf.size() / (size_t) n_e);
prompt_batch.reset(new decode_embd_batch(prompt_embd_buf.data(), n_prompt, 1, n_e));
prompt_batch->set_position_normal(0, seq_id);
prompt_pos = 0;
temp = inp->temp;
out_type = inp->out_type;
stream = inp->stream;
if (!init_gen_state(inp->seed)) {
return 1;
}
return 0;
}
int32_t step_prompt(int32_t n_batch) override {
GGML_ASSERT(n_batch > 0);
if (prompt_pos >= n_prompt) {
return 0;
}
const int32_t n_tokens_batch = std::min(n_batch, n_prompt - prompt_pos);
if ((prompt_pos + n_tokens_batch) == n_prompt) {
prompt_batch->logits[prompt_pos + n_tokens_batch - 1] = 1;
}
if (llama_process(lctx, LLAMA_PROCESS_TYPE_DECODE, prompt_batch->render(lctx, prompt_pos, n_tokens_batch)) != 0) {
LOG_ERR("mtmd_helper_gen_audio: prompt decode failed\n");
return -1;
}
pos += n_tokens_batch;
prompt_pos += n_tokens_batch;
if (prompt_pos >= n_prompt) {
prompt_batch.reset();
prompt_embd_buf.clear();
return 0;
}
return n_prompt - prompt_pos;
}
int32_t step_gen(llama_token sampled, const float * h_state_in, const float ** h_state_out, bool * out_stop) override {
(void) sampled; // the backbone output is continuous, there is no token to consume
mtmd_gen_inp inp = mtmd_gen_inp_default(mctx);
inp.type = MTMD_GEN_PROCESS_TYPE_GEN_CODE;
inp.embd = const_cast<float *>(h_state_in);
if (temp >= 0.0f) {
inp.temp = temp;
}
mtmd_gen_out out{};
if (mtmd_gen_audio_process(mctx, gen_state.get(), &inp, &out) != 0) {
LOG_ERR("mtmd_helper_gen_audio: flow decode failed\n");
return 1;
}
if (out.is_eos && eos_step < 0) {
eos_step = step_idx;
}
// the frame of the stopping step is discarded, matching _autoregressive_generation().
// the budget is the reference's fallback for a chunk whose eos head never fires
const bool chunk_done = (eos_step >= 0 && step_idx >= eos_step + frames_after_eos) ||
step_idx >= chunk_budget;
if (chunk_done) {
if (eos_step < 0) {
LOG_WRN("mtmd_helper_gen_audio: chunk %zu hit its budget without end-of-speech\n", chunk_idx);
}
return finish_chunk(h_state_out, out_stop);
}
feats_buf.insert(feats_buf.end(), out.feats, out.feats + out.n_feats);
step_idx++;
if (out.n_feats > 0 && feats_buf.size() / out.n_feats >= window_frames) {
if (!flush_gen_wav()) {
return 1;
}
}
decode_embd_batch batch_embd(out.embd, 1, 1, n_embd);
batch_embd.set_position_normal(pos, seq_id);
batch_embd.logits[0] = 1;
pos++;
if (llama_process(lctx, LLAMA_PROCESS_TYPE_DECODE, batch_embd.render(lctx, 0, 1)) != 0) {
LOG_ERR("mtmd_helper_gen_audio: decode failed\n");
return 1;
}
const float * he = llama_get_embeddings_ith(lctx, -1);
h_state_buf.assign(he, he + n_embd);
*h_state_out = h_state_buf.data();
return 0;
}
int32_t get_output(int32_t * out_sample_rate, const char ** out_data, size_t * out_data_len, int64_t * out_n_samples) override {
*out_sample_rate = info.sample_rate;
if (!stream) {
// one-shot call: force out whatever's left, regardless of window_frames
if (!flush_gen_wav()) {
return 1;
}
if (out_n_samples) {
*out_n_samples = (int64_t) audio_pcm.size();
}
if (out_type == MTMD_HELPER_GEN_AUDIO_OUTTYPE_PCM) {
*out_data = (const char *) audio_pcm.data();
*out_data_len = audio_pcm.size() * sizeof(float);
return 0;
}
out_buf.clear();
if (!write_wav16(out_buf, audio_pcm, info.sample_rate)) {
LOG_ERR("mtmd_helper_gen_audio: output too large for WAV\n");
return 1;
}
*out_data = out_buf.data();
*out_data_len = out_buf.size();
return 0;
}
// streaming: only return audio produced since the previous call
const size_t n_new = audio_pcm.size() - pcm_sent;
if (out_n_samples) {
*out_n_samples = (int64_t) n_new;
}
if (out_type == MTMD_HELPER_GEN_AUDIO_OUTTYPE_PCM) {
*out_data = (const char *) (audio_pcm.data() + pcm_sent);
*out_data_len = n_new * sizeof(float);
pcm_sent = audio_pcm.size();
return 0;
}
out_buf.clear();
if (!wav_header_sent) {
write_wav16_header(out_buf, UINT32_MAX, info.sample_rate);
wav_header_sent = true;
}
append_wav16_pcm(out_buf, audio_pcm.data() + pcm_sent, n_new);
pcm_sent = audio_pcm.size();
*out_data = out_buf.data();
*out_data_len = out_buf.size();
return 0;
}
int32_t flush() override {
return flush_gen_wav() ? 0 : 1;
}
private:
bool ensure_cache() {
if (specials_ok) {
return true;
}
// bos_before_voice is optional, some packs do not insert it
bos_before_voice = find_special_token(vocab, "<|bos_before_voice|>");
audio_bos = find_special_token(vocab, "<|audio_bos|>");
if (audio_bos == LLAMA_TOKEN_NULL) {
LOG_ERR("mtmd_helper_gen_audio: missing <|audio_bos|> in vocab\n");
return false;
}
const uint32_t n_tok_embd = llama_model_get_tok_embd(model, nullptr);
if (n_tok_embd == 0) {
LOG_ERR("mtmd_helper_gen_audio: model has no token embeddings\n");
return false;
}
tok_embd.resize(n_tok_embd);
if (llama_model_get_tok_embd(model, tok_embd.data()) != n_tok_embd) {
LOG_ERR("mtmd_helper_gen_audio: token embedding copy failed\n");
return false;
}
GGML_ASSERT(n_embd > 0 && n_tok_embd % (uint32_t) n_embd == 0);
specials_ok = true;
return true;
}
// the table can be shorter than the vocab, so bound the row lookup
void push_embd_row(std::vector<float> & dst, llama_token t) const {
const size_t n_rows = tok_embd.size() / (size_t) n_embd;
GGML_ASSERT(t >= 0 && (size_t) t < n_rows);
dst.insert(dst.end(),
tok_embd.begin() + (size_t) t * n_embd,
tok_embd.begin() + (size_t) (t + 1) * n_embd);
}
// token ids of the pieces the reference splits on, see split_into_best_sentences().
// the leading token is dropped, it is the tokenizer's dummy prefix
std::vector<llama_token> punct_ids(const char * s) const {
std::vector<llama_token> ids(16);
const int n = llama_tokenize(vocab, s, (int32_t) strlen(s), ids.data(), (int32_t) ids.size(), false, false);
if (n <= 1) {
return {};
}
return std::vector<llama_token>(ids.begin() + 1, ids.begin() + n);
}
// cut after runs of boundary tokens, so punctuation stays with the sentence it ends
static std::vector<std::vector<llama_token>> split_on(const std::vector<llama_token> & ids,
const std::vector<llama_token> & boundary) {
std::vector<std::vector<llama_token>> out;
size_t start = 0;
bool prev_was_boundary = false;
for (size_t i = 0; i < ids.size(); i++) {
const bool is_boundary = std::find(boundary.begin(), boundary.end(), ids[i]) != boundary.end();
if (!is_boundary && prev_was_boundary) {
out.emplace_back(ids.begin() + start, ids.begin() + i);
start = i;
}
prev_was_boundary = is_boundary;
}
out.emplace_back(ids.begin() + start, ids.end());
return out;
}
std::vector<std::vector<llama_token>> split_chunks(const std::vector<llama_token> & ids) const {
if ((int) ids.size() <= max_chunk_tokens) {
return { ids };
}
const std::vector<llama_token> eos_punct = punct_ids(".!...?");
const std::vector<llama_token> mid_punct = punct_ids(",;:");
// oversized sentences are split again on weaker punctuation, else words get skipped
std::vector<std::vector<llama_token>> segments;
for (auto & seg : split_on(ids, eos_punct)) {
if ((int) seg.size() <= max_chunk_tokens) {
segments.push_back(std::move(seg));
continue;
}
auto sub = split_on(seg, mid_punct);
if (sub.size() > 1) {
for (auto & s : sub) {
segments.push_back(std::move(s));
}
} else {
segments.push_back(std::move(seg));
}
}
std::vector<std::vector<llama_token>> out;
for (auto & seg : segments) {
if (seg.empty()) {
continue;
}
if (!out.empty() && (int) (out.back().size() + seg.size()) <= max_chunk_tokens) {
out.back().insert(out.back().end(), seg.begin(), seg.end());
} else {
out.push_back(std::move(seg));
}
}
if (out.empty()) {
out.push_back(ids);
}
for (const auto & c : out) {
if ((int) c.size() > max_chunk_tokens) {
LOG_WRN("mtmd_helper_gen_audio: chunk of %zu tokens exceeds the %d token budget, "
"generation may skip words\n", c.size(), max_chunk_tokens);
}
}
return out;
}
// _estimate_max_gen_len() plus the per-chunk tail guess, both in frames
void arm_chunk_budget(size_t idx) {
const int n_tok = (int) chunks[idx].size();
chunk_budget = (int) std::ceil((n_tok / 3.0 + 2.0) * frame_rate);
// the pack may pin the tail, else the reference guesses it from the word count
frames_after_eos = pack.frames_after_eos > 0 ? pack.frames_after_eos : (n_tok <= 6 ? 5 : 3);
step_idx = 0;
eos_step = -1;
}
// ends the current chunk and, if there is another, re-prompts it on top of the voice
int32_t finish_chunk(const float ** h_state_out, bool * out_stop) {
if (!flush_gen_wav()) {
return 1;
}
// the decoder restarts too, the next chunk's audio is not continuous with this one
mtmd_gen_state_reset(gen_state.get());
if (chunk_idx + 1 >= chunks.size()) {
*out_stop = true;
*h_state_out = nullptr;
return 0;
}
chunk_idx++;
// drop this chunk's text and audio, keep the voice conditioning
llama_memory_seq_rm(llama_get_memory(lctx), seq_id, n_voice_pos, -1);
pos = n_voice_pos;
const int n_e = n_embd;
prompt_embd_buf.clear();
for (llama_token t : chunks[chunk_idx]) {
push_embd_row(prompt_embd_buf, t);
}
push_embd_row(prompt_embd_buf, audio_bos);
arm_chunk_budget(chunk_idx);
const int n_rows = (int) (prompt_embd_buf.size() / (size_t) n_e);
GGML_ASSERT(n_rows > 0);
decode_embd_batch batch(prompt_embd_buf.data(), n_rows, 1, n_e);
batch.set_position_normal(pos, seq_id);
batch.logits[n_rows - 1] = 1;
if (llama_process(lctx, LLAMA_PROCESS_TYPE_DECODE, batch.render(lctx, 0, n_rows)) != 0) {
LOG_ERR("mtmd_helper_gen_audio: chunk prompt decode failed\n");
return 1;
}
pos += n_rows;
prompt_embd_buf.clear();
const float * he = llama_get_embeddings_ith(lctx, -1);
h_state_buf.assign(he, he + n_embd);
*h_state_out = h_state_buf.data();
*out_stop = false;
return 0;
}
// same normalization as prepare_text_prompt() in the reference, it affects quality
static std::string prepare_text(const std::string & in, bool pad_short) {
std::string s;
s.reserve(in.size() + 1);
for (char c : in) {
if (c == '\n' || c == '\r') {
s += ' ';
} else if (c == ';') {
s += ',';
} else {
s += c;
}
}
const size_t b = s.find_first_not_of(' ');
const size_t e = s.find_last_not_of(' ');
if (b == std::string::npos) {
return "";
}
s = s.substr(b, e - b + 1);
if (s[0] >= 'a' && s[0] <= 'z') {
s[0] = (char) (s[0] - 'a' + 'A');
}
const unsigned char last = (unsigned char) s.back();
if (std::isalnum(last)) {
s += '.';
}
if (pad_short && count_words(s) < 5) {
s = std::string(8, ' ') + s;
}
return s;
}
static int count_words(const std::string & s) {
int n = 0;
bool in_word = false;
for (char c : s) {
if (c == ' ') {
in_word = false;
} else if (!in_word) {
in_word = true;
n++;
}
}
return n;
}
// decodes the buffered latents, the mimi decoder state carries over between calls
bool flush_gen_wav() {
if (feats_buf.empty()) {
return true;
}
mtmd_gen_inp inp = mtmd_gen_inp_default(mctx);
inp.type = MTMD_GEN_PROCESS_TYPE_GEN_WAV;
inp.feats = feats_buf.data();
inp.n_feats = feats_buf.size();
mtmd_gen_out out{};
if (mtmd_gen_audio_process(mctx, gen_state.get(), &inp, &out) != 0) {
LOG_ERR("mtmd_helper_gen_audio: mimi decode failed\n");
return false;
}
audio_pcm.insert(audio_pcm.end(), out.audio, out.audio + out.n_samples);
feats_buf.clear();
return true;
}
pockettts_pack_settings pack;
float temp = -1.0f; // the caller's flow noise scale, < 0 for the model default
bool specials_ok = false;
llama_token bos_before_voice = LLAMA_TOKEN_NULL;
llama_token audio_bos = LLAMA_TOKEN_NULL;
std::vector<float> tok_embd;
llama_seq_id seq_id = 0;
int pos = 0;
std::vector<float> prompt_embd_buf;
std::unique_ptr<decode_embd_batch> prompt_batch;
int n_prompt = 0;
int prompt_pos = 0;
// end-of-speech is latched, then a few more frames are generated as tail padding
int step_idx = 0;
int eos_step = -1;
int frames_after_eos = 3;
static constexpr int max_chunk_tokens = 50; // MAX_TOKEN_PER_CHUNK in the reference
static constexpr double frame_rate = 12.5;
std::vector<std::vector<llama_token>> chunks;
size_t chunk_idx = 0;
int n_voice_pos = 0; // KV positions held by the voice conditioning
int chunk_budget = 0;
// latents are decoded a window at a time, the decoder state bridges the windows
size_t window_frames = 8;
std::vector<float> feats_buf;
std::vector<float> audio_pcm;
std::vector<float> h_state_buf;
mtmd_helper_gen_audio_outtype out_type = MTMD_HELPER_GEN_AUDIO_OUTTYPE_WAV;
std::vector<char> out_buf;
bool stream = false;
size_t pcm_sent = 0; // samples already returned by get_output()
bool wav_header_sent = false;
};
// Soprano uses the hidden state that predicts each non-EOS token, including the prompt's last state.
class soprano_gen_audio_pipeline : public mtmd_gen_audio_pipeline {
public:
using mtmd_gen_audio_pipeline::mtmd_gen_audio_pipeline;
void reset() override {
prompt.clear();
features.clear();
h_next.clear();
audio.clear();
wav.clear();
pos = 0;
prompt_pos = 0;
decoded = false;
sent = false;
}
int32_t set_input(const mtmd_helper_gen_audio_inp * inp) override {
reset();
if (inp->speaker_ref || !inp->prompt || inp->prompt_len == 0 || inp->prompt_len > INT32_MAX ||
inp->seq_id < 0 || (uint32_t) inp->seq_id >= llama_n_seq_max(lctx)) {
LOG_ERR("mtmd_helper_gen_audio: soprano requires text, without a speaker reference\n");
return 1;
}
seq_id = inp->seq_id;
out_type = inp->out_type;
stream = inp->stream;
const llama_token text = find_special_token(vocab, "[TEXT]");
const llama_token start = find_special_token(vocab, "[START]");
const llama_token stop = find_special_token(vocab, "[STOP]");
if (text == LLAMA_TOKEN_NULL || start == LLAMA_TOKEN_NULL || stop == LLAMA_TOKEN_NULL) {
LOG_ERR("mtmd_helper_gen_audio: incompatible soprano vocabulary\n");
return 1;
}
const int required = llama_tokenize(vocab, inp->prompt, inp->prompt_len, nullptr, 0, false, false);
if (required >= 0 || (uint32_t) (-required + 3) >= llama_n_ctx_seq(lctx)) {
LOG_ERR("mtmd_helper_gen_audio: soprano prompt does not fit in the context\n");
return 1;
}
const int n = -required;
prompt.resize(n + 3);
prompt[0] = stop;
prompt[1] = text;
if (llama_tokenize(vocab, inp->prompt, inp->prompt_len, prompt.data() + 2, n, false, false) != n) {
return 1;
}
prompt.back() = start;
if (!llama_memory_seq_rm(llama_get_memory(lctx), seq_id, 0, -1)) {
LOG_ERR("mtmd_helper_gen_audio: cannot reset soprano sequence\n");
return 1;
}
if (!init_gen_state(inp->seed)) {
return 1;
}
return 0;
}
int32_t step_prompt(int32_t n_batch) override {
if (n_batch <= 0 || prompt.empty()) {
return -1;
}
const int n = std::min(n_batch, (int) prompt.size() - prompt_pos);
if (n == 0) {
return 0;
}
if (!decode(prompt.data() + prompt_pos, n)) {
return -1;
}
prompt_pos += n;
return (int) prompt.size() - prompt_pos;
}
int32_t step_gen(llama_token sampled, const float * h_state_in, const float ** h_state_out, bool * out_stop) override {
*h_state_out = nullptr;
*out_stop = false;
if (sampled < 0 || sampled >= llama_vocab_n_tokens(vocab) || !h_state_in || prompt_pos != (int) prompt.size() || prompt.empty()) {
return 1;
}
if (llama_vocab_is_eog(vocab, sampled) || (uint32_t) pos >= llama_n_ctx_seq(lctx)) {
*out_stop = true;
return 0;
}
features.insert(features.end(), h_state_in, h_state_in + n_embd);
if (!decode(&sampled, 1)) {
features.resize(features.size() - n_embd);
return 1;
}
const float * h = llama_get_embeddings_ith(lctx, -1);
if (!h) {
return 1;
}
h_next.assign(h, h + n_embd);
*h_state_out = h_next.data();
return 0;
}
int32_t get_output(int32_t * sample_rate, const char ** data, size_t * data_len, int64_t * n_samples) override {
*sample_rate = info.sample_rate;
*data_len = 0;
if (n_samples) { *n_samples = 0; }
if (stream) {
// the decoder needs the whole utterance, so the audio comes out in one chunk after flush()
if (!decoded || sent) {
return 0;
}
sent = true;
} else if (!decode_audio()) {
return 1;
}
if (n_samples) { *n_samples = audio.size(); }
if (out_type == MTMD_HELPER_GEN_AUDIO_OUTTYPE_PCM) {
*data = (const char *) audio.data();
*data_len = audio.size() * sizeof(float);
} else {
wav.clear();
if (!write_wav16(wav, audio, info.sample_rate)) {
return 1;
}
*data = wav.data();
*data_len = wav.size();
}
return 0;
}
// the whole utterance is decoded at once
int32_t flush() override {
return decode_audio() ? 0 : 1;
}
private:
bool decode_audio() {
if (decoded) {
return true;
}
if (features.size() < 2 * (size_t) n_embd) {
LOG_ERR("mtmd_helper_gen_audio: soprano needs at least two generated frames\n");
return false;
}
mtmd_gen_inp inp = mtmd_gen_inp_default(mctx);
inp.type = MTMD_GEN_PROCESS_TYPE_GEN_WAV;
inp.feats = features.data();
inp.n_feats = features.size();
mtmd_gen_out out{};
if (mtmd_gen_audio_process(mctx, gen_state.get(), &inp, &out) != 0) {
return false;
}
audio.assign(out.audio, out.audio + out.n_samples);
decoded = true;
return true;
}
bool decode(llama_token * tokens, int n) {
llama_batch_ext_ptr batch(llama_batch_ext_init(lctx));
if (!batch_set_tokens(batch.get(), tokens, n, pos, seq_id)) {
return false;
}
llama_batch_ext_set_output_logits(batch.get(), n - 1, true);
if (llama_process(lctx, LLAMA_PROCESS_TYPE_DECODE, batch.get()) != 0) {
LOG_ERR("mtmd_helper_gen_audio: soprano backbone decode failed\n");
return false;
}
pos += n;
return true;
}
llama_seq_id seq_id = 0;
llama_pos pos = 0;
int prompt_pos = 0;
std::vector<llama_token> prompt;
std::vector<float> features;
std::vector<float> h_next;
std::vector<float> audio;
std::vector<char> wav;
mtmd_helper_gen_audio_outtype out_type = MTMD_HELPER_GEN_AUDIO_OUTTYPE_WAV;
bool stream = false;
bool decoded = false;
bool sent = false;
};
// LFM3-Audio TTS, see generate() in the reference: each step samples a frame of codes by MaskGIT
// and feeds its embedding back, the end-of-audio code stops
class lfm3a_gen_audio_pipeline : public mtmd_gen_audio_pipeline {
public:
using mtmd_gen_audio_pipeline::mtmd_gen_audio_pipeline;
void reset() override {
seq_id = 0;
pos = 0;
continuing = false;
mode = MODE_AUDIO;
slots_left = 0;
speaking = false;
text_done = false;
prompt_tokens.clear();
prompt_pos = 0;
hist.clear();
gen_state.reset();
pending_tail = false;
pcm_sent = 0;
wav_header_sent = false;
audio_pcm.clear();
h_state_buf.clear();
out_buf.clear();
}
int32_t set_input(const mtmd_helper_gen_audio_inp * inp) override {
reset();
seq_id = inp->seq_id;
// a voice is named in the prompt instead
if (inp->speaker_ref && !mtmd_bitmap_is_voice(inp->speaker_ref)) {
LOG_WRN("mtmd_helper_gen_audio: the model has no voice conditioning, the speaker file is ignored\n");
}
// the prompt is rendered by the caller: the "tts" chat template of the model, or the next turn of a conversation
continuing = inp->n_past > 0;
pos = continuing ? inp->n_past : 0;
const std::string full(inp->prompt, inp->prompt_len);
prompt_tokens.resize(full.size() + 16);
const int n_ids = llama_tokenize(vocab, full.c_str(), (int32_t) full.size(), prompt_tokens.data(),
(int32_t) prompt_tokens.size(), false, true);
if (n_ids <= 0) {
LOG_ERR("mtmd_helper_gen_audio: tokenization failed\n");
return 1;
}
prompt_tokens.resize((size_t) n_ids);
if (!find_tokens()) {
return 1;
}
// a continued response opens with the token of the asked mode, a text-only one cannot switch to audio
if (continuing) {
switch (inp->out_mode) {
case MTMD_HELPER_GEN_AUDIO_OUT_AUDIO: prompt_tokens.push_back(tok_audio_start); break;
case MTMD_HELPER_GEN_AUDIO_OUT_MIXED: prompt_tokens.push_back(tok_mixed_start); break;
case MTMD_HELPER_GEN_AUDIO_OUT_TEXT:
logit_bias = { { tok_audio_start, -INFINITY }, { tok_mixed_start, -INFINITY } };
break;
default: break;
}
}
// the last prompt token picks the mode, see generate() of the reference
const llama_token last = prompt_tokens.back();
mode = last == tok_audio_start ? MODE_AUDIO : last == tok_mixed_start ? MODE_MIXED : MODE_TEXT;
slots_left = n_text_block;
if (!continuing && mode != MODE_AUDIO) {
LOG_ERR("mtmd_helper_gen_audio: the prompt must come from the \"tts\" chat template of the model\n");
return 1;
}
// the sampling defaults of the model, the caller can only override the temperature
const mtmd_gen_inp def = mtmd_gen_inp_default(mctx);
temp = inp->temp >= 0.0f ? inp->temp : def.temp;
top_k = def.top_k;
rep_penalty = def.penalty_repeat;
rep_window = def.penalty_last_n;
out_type = inp->out_type;
stream = inp->stream;
rng.seed(inp->seed == UINT32_MAX ? std::random_device{}() : inp->seed);
return init_gen_state(inp->seed) ? 0 : 1;
}
int32_t step_prompt(int32_t n_batch) override {
GGML_ASSERT(n_batch > 0);
const int n_prompt = (int) prompt_tokens.size();
if (prompt_pos >= n_prompt) {
return 0;
}
const int n = std::min(n_batch, n_prompt - prompt_pos);
llama_batch_ext_ptr batch(llama_batch_ext_init(lctx));
if (!batch_set_tokens(batch.get(), prompt_tokens.data() + prompt_pos, n, pos, seq_id)) {
return -1;
}
if (prompt_pos + n == n_prompt) {
llama_batch_ext_set_output_logits(batch.get(), n - 1, true);
}
if (llama_process(lctx, LLAMA_PROCESS_TYPE_DECODE, batch.get()) != 0) {
LOG_ERR("mtmd_helper_gen_audio: prompt decode failed\n");
return -1;
}
pos += n;
prompt_pos += n;
return n_prompt - prompt_pos;
}
bool supports_continue() const override {
return true;
}
bool next_is_text() const override {
return mode == MODE_TEXT || (mode == MODE_MIXED && !speaking);
}
// one generate() step: a text token from the caller or an audio frame from the hidden state
int32_t step_gen(llama_token sampled, const float * h_state_in, const float ** h_state_out, bool * out_stop) override {
const bool interleaving = mode == MODE_MIXED;
if (next_is_text()) {
// <|im_end|> ends the response in any text slot, <|text_end|> outside an interleaved block too
if (sampled == tok_im_end || (!interleaving && sampled == tok_text_end)) {
*out_stop = true;
*h_state_out = nullptr;
return 0;
}
// the switching tokens are fed back like any text token
if (!interleaving) {
if (sampled == tok_audio_start) {
mode = MODE_AUDIO;
} else if (sampled == tok_mixed_start) {
mode = MODE_MIXED;
slots_left = n_text_block;
speaking = false;
text_done = false;
}
} else if (sampled == tok_text_end) {
text_done = true;
speaking = true;
}
if (!decode(sampled)) {
return 1;
}
} else {
std::vector<int32_t> codes;
const float * fb = nullptr;
bool is_eos = false;
if (!sample_frame(h_state_in, codes, &fb, &is_eos)) {
return 1;
}
if (is_eos) {
if (!continuing) {
// TTS: the end of audio ends the generation
*out_stop = true;
*h_state_out = nullptr;
return 0;
}
// the segment ends, the codec tail goes out and the response continues in text
if (flush() != 0) {
return 1;
}
mode = MODE_TEXT;
} else {
hist.insert(hist.end(), codes.begin(), codes.end());
if (!decode_wav(codes.data(), codes.size())) {
return 1;
}
}
decode_embd_batch batch_embd(const_cast<float *>(fb), 1, 1, n_embd);
batch_embd.set_position_normal(pos, seq_id);
batch_embd.logits[0] = 1;
if (llama_process(lctx, LLAMA_PROCESS_TYPE_DECODE, batch_embd.render(lctx, 0, 1)) != 0) {
LOG_ERR("mtmd_helper_gen_audio: decode failed\n");
return 1;
}
pos++;
}
// a slot emitted while interleaving counts down the block, the switching token itself does not
if (interleaving && !text_done && --slots_left == 0) {
speaking = !speaking;
slots_left = speaking ? n_audio_block : n_text_block;
}
const float * he = llama_get_embeddings_ith(lctx, -1);
h_state_buf.assign(he, he + n_embd);
*h_state_out = h_state_buf.data();
return 0;
}
int32_t get_output(int32_t * out_sample_rate, const char ** out_data, size_t * out_data_len, int64_t * out_n_samples) override {
*out_sample_rate = info.sample_rate;
if (!stream) {
if (flush() != 0) {
return 1;
}
if (out_n_samples) {
*out_n_samples = (int64_t) audio_pcm.size();
}
if (out_type == MTMD_HELPER_GEN_AUDIO_OUTTYPE_PCM) {
*out_data = (const char *) audio_pcm.data();
*out_data_len = audio_pcm.size() * sizeof(float);
return 0;
}
out_buf.clear();
if (!write_wav16(out_buf, audio_pcm, info.sample_rate)) {
LOG_ERR("mtmd_helper_gen_audio: output too large for WAV\n");
return 1;
}
*out_data = out_buf.data();
*out_data_len = out_buf.size();
return 0;
}
// streaming: only the audio produced since the previous call, every frame is decoded as it is sampled
const size_t n_new = audio_pcm.size() - pcm_sent;
if (out_n_samples) {
*out_n_samples = (int64_t) n_new;
}
if (out_type == MTMD_HELPER_GEN_AUDIO_OUTTYPE_PCM) {
*out_data = (const char *) (audio_pcm.data() + pcm_sent);
*out_data_len = n_new * sizeof(float);
pcm_sent = audio_pcm.size();
return 0;
}
out_buf.clear();
if (!wav_header_sent) {
write_wav16_header(out_buf, UINT32_MAX, info.sample_rate);
wav_header_sent = true;
}
append_wav16_pcm(out_buf, audio_pcm.data() + pcm_sent, n_new);
pcm_sent = audio_pcm.size();
*out_data = out_buf.data();
*out_data_len = out_buf.size();
return 0;
}
// the ISTFT holds back the overlap of the last frame
int32_t flush() override {
if (pending_tail) {
if (!decode_wav(nullptr, 0)) {
return 1;
}
pending_tail = false;
}
return 0;
}
private:
// checked by the converter
static constexpr int n_cb = 12;
static constexpr int schedule[] = { 4, 4, 4 }; // codebooks revealed per depthformer pass
// a random order of the codebooks, revealed by the schedule; fb is the feedback embedding of the codes.
// at the end of audio every code is the end code, its feedback embedding is only built when continuing
bool sample_frame(const float * h_state, std::vector<int32_t> & codes, const float ** fb, bool * is_eos) {
std::vector<int> order(n_cb);
for (int i = 0; i < n_cb; i++) {
order[(size_t) i] = i;
}
std::shuffle(order.begin(), order.end(), rng);
codes.assign(n_cb, -1);
mtmd_gen_inp inp = mtmd_gen_inp_default(mctx);
inp.type = MTMD_GEN_PROCESS_TYPE_GEN_CODE;
inp.embd = const_cast<float *>(h_state);
inp.codes = codes.data();
inp.n_codes = codes.size();
mtmd_gen_out out{};
int k = 0;
for (int n_pick : schedule) {
if (mtmd_gen_audio_process(mctx, gen_state.get(), &inp, &out) != 0 || out.n_feats == 0 || out.n_feats % n_cb != 0) {
LOG_ERR("mtmd_helper_gen_audio: depthformer pass failed\n");
return false;
}
const size_t n_vocab = out.n_feats / n_cb;
logits.assign(out.feats, out.feats + out.n_feats);
for (int j = 0; j < n_pick; j++, k++) {
const int i = order[(size_t) k];
float * l = logits.data() + (size_t) i * n_vocab;
apply_rep_penalty(l, n_vocab, i);
codes[(size_t) i] = sample(l, n_vocab);
}
}
// the last code of every codebook ends the audio
const int32_t eos = (int32_t) (logits.size() / n_cb) - 1;
*is_eos = std::find(codes.begin(), codes.end(), eos) != codes.end();
if (*is_eos) {
codes.assign(n_cb, eos);
if (!continuing) {
*fb = nullptr;
return true;
}
}
// all codes known: only the feedback embedding
if (mtmd_gen_audio_process(mctx, gen_state.get(), &inp, &out) != 0) {
LOG_ERR("mtmd_helper_gen_audio: codebook embedding failed\n");
return false;
}
fb_buf.assign(out.embd, out.embd + n_embd);
*fb = fb_buf.data();
return true;
}
// CTRL penalty over the codes codebook cb took in the last rep_window frames
void apply_rep_penalty(float * l, size_t n_vocab, int cb) const {
if (rep_penalty <= 1.0f || rep_window <= 0) {
return;
}
const size_t n_hist = hist.size() / n_cb;
const size_t n_win = std::min(n_hist, (size_t) rep_window);
std::vector<bool> seen(n_vocab, false);
for (size_t f = n_hist - n_win; f < n_hist; f++) {
const int32_t c = hist[f * n_cb + (size_t) cb];
if (c >= 0 && (size_t) c < n_vocab && !seen[(size_t) c]) {
seen[(size_t) c] = true;
l[c] = l[c] > 0.0f ? l[c] / rep_penalty : l[c] * rep_penalty;
}
}
}
// TODO: reuse the samplers of common (top-k, temperature, penalties) instead of this one
int32_t sample(const float * l, size_t n_vocab) {
if (temp <= 0.0f) {
return (int32_t) (std::max_element(l, l + n_vocab) - l);
}
// top-k keeps the codes scoring at least the k-th best, ties included
float lmin = -INFINITY;
if (top_k > 0 && (size_t) top_k < n_vocab) {
std::vector<float> sorted(l, l + n_vocab);
std::nth_element(sorted.begin(), sorted.begin() + (top_k - 1), sorted.end(), std::greater<float>());
lmin = sorted[(size_t) top_k - 1];
}
const float lmax = *std::max_element(l, l + n_vocab);
std::vector<double> prob(n_vocab);
double sum = 0.0;
for (size_t v = 0; v < n_vocab; v++) {
prob[v] = l[v] < lmin ? 0.0 : std::exp((double) (l[v] - lmax) / temp);
sum += prob[v];
}
double r = std::uniform_real_distribution<double>(0.0, 1.0)(rng) * sum;
for (size_t v = 0; v < n_vocab; v++) {
r -= prob[v];
if (r <= 0.0) {
return (int32_t) v;
}
}
return (int32_t) n_vocab - 1;
}
// n_codes == 0 flushes the ISTFT tail
bool decode_wav(const int32_t * codes, size_t n_codes) {
mtmd_gen_inp inp = mtmd_gen_inp_default(mctx);
inp.type = MTMD_GEN_PROCESS_TYPE_GEN_WAV;
inp.codes = const_cast<int32_t *>(codes);
inp.n_codes = n_codes;
mtmd_gen_out out{};
if (mtmd_gen_audio_process(mctx, gen_state.get(), &inp, &out) != 0) {
LOG_ERR("mtmd_helper_gen_audio: codec decode failed\n");
return false;
}
audio_pcm.insert(audio_pcm.end(), out.audio, out.audio + out.n_samples);
pending_tail = n_codes > 0;
return true;
}
bool find_tokens() {
tok_audio_start = find_special_token(vocab, "<|audio_start|>");
tok_mixed_start = find_special_token(vocab, "<|mixed_start|>");
tok_text_end = find_special_token(vocab, "<|text_end|>");
tok_im_end = find_special_token(vocab, "<|im_end|>");
if (tok_audio_start == LLAMA_TOKEN_NULL || tok_mixed_start == LLAMA_TOKEN_NULL ||
tok_text_end == LLAMA_TOKEN_NULL || tok_im_end == LLAMA_TOKEN_NULL) {
LOG_ERR("mtmd_helper_gen_audio: missing LFM3-Audio control tokens in vocab\n");
return false;
}
return true;
}
bool decode(llama_token token) {
llama_batch_ext_ptr batch(llama_batch_ext_init(lctx));
if (!batch_set_tokens(batch.get(), &token, 1, pos, seq_id)) {
return false;
}
llama_batch_ext_set_output_logits(batch.get(), 0, true);
if (llama_process(lctx, LLAMA_PROCESS_TYPE_DECODE, batch.get()) != 0) {
LOG_ERR("mtmd_helper_gen_audio: decode failed\n");
return false;
}
pos += 1;
return true;
}
// generate() modes: text, audio until end of audio, interleaved until <|text_end|>
enum gen_mode { MODE_TEXT, MODE_AUDIO, MODE_MIXED };
// checked by the converter
static constexpr int n_text_block = 6; // interleaved_n_text
static constexpr int n_audio_block = 12; // interleaved_n_audio
llama_token tok_audio_start = LLAMA_TOKEN_NULL;
llama_token tok_mixed_start = LLAMA_TOKEN_NULL;
llama_token tok_text_end = LLAMA_TOKEN_NULL;
llama_token tok_im_end = LLAMA_TOKEN_NULL;
bool continuing = false; // the prompt continues a sequence, the response follows the model's switching
gen_mode mode = MODE_AUDIO;
int slots_left = 0;
bool speaking = false; // interleaving: the current block is audio
bool text_done = false; // interleaving: <|text_end|> was emitted, audio until the end-of-audio frame
llama_seq_id seq_id = 0;
int pos = 0;
std::vector<llama_token> prompt_tokens;
int prompt_pos = 0;
float temp = 0.0f;
int32_t top_k = 0;
float rep_penalty = 1.0f;
int32_t rep_window = 0; // frames
std::mt19937 rng;
std::vector<float> logits;
std::vector<float> fb_buf;
std::vector<int32_t> hist; // frame-major codes of the frames so far
bool pending_tail = false; // the ISTFT holds back the overlap of the last frame
std::vector<float> audio_pcm;
std::vector<float> h_state_buf;
mtmd_helper_gen_audio_outtype out_type = MTMD_HELPER_GEN_AUDIO_OUTTYPE_WAV;
std::vector<char> out_buf;
bool stream = false;
size_t pcm_sent = 0; // samples already returned by get_output()
bool wav_header_sent = false;
};
// Chatterbox: the backbone emits s3 speech tokens, then one flow-matching mel decode and the NSF-iSTFT vocoder make the PCM
// the multilingual variant runs cfg with an uncond sequence (text rows zeroed), paired as seq_id and n_seq_max / 2 + seq_id
class chatterbox_gen_audio_pipeline : public mtmd_gen_audio_pipeline {
public:
using mtmd_gen_audio_pipeline::mtmd_gen_audio_pipeline;
void reset() override {
seq_id = 0;
seq_uncond = 0;
pos = 0;
ar_idx = 0;
codes_buf.clear();
audio_pcm.clear();
h_state_buf.clear();
out_buf.clear();
prompt_embd_buf.clear();
uncond_embd_buf.clear();
cond_logits.clear();
n_prompt = 0;
prompt_pos = 0;
decoded = false;
stream = false;
pcm_sent = 0;
wav_header_sent = false;
gen_state.reset();
}
int32_t set_input(const mtmd_helper_gen_audio_inp * inp) override {
reset();
seq_id = inp->seq_id;
if (!ensure_cache()) {
return 1;
}
out_type = inp->out_type;
stream = inp->stream;
if (!init_gen_state(inp->seed)) {
return 1;
}
if (mtl) {
// the uncond sequence lives half a seq space away from the cond one
const int32_t n_seq_max = (int32_t) llama_n_seq_max(lctx);
seq_uncond = (llama_seq_id) (n_seq_max / 2 + seq_id);
if (!llama_model_classifier_free_guidance(llama_get_model(lctx)) || seq_id >= n_seq_max / 2) {
LOG_ERR("mtmd_helper_gen_audio: multilingual chatterbox needs a backbone with classifier_free_guidance and a paired cfg sequence "
"(n_seq_max = %d, seq_id = %d)\n", n_seq_max, seq_id);
return 1;
}
}
llama_memory_t mem = llama_get_memory(lctx);
if (!llama_memory_seq_rm(mem, seq_id, 0, -1) || (mtl && !llama_memory_seq_rm(mem, seq_uncond, 0, -1))) {
LOG_ERR("mtmd_helper_gen_audio: cannot reset chatterbox sequences\n");
return 1;
}
// the encode gives the talker conditioning rows, set_ref keeps the flow decoder reference
if (!inp->speaker_ref) {
LOG_ERR("mtmd_helper_gen_audio: chatterbox requires a speaker reference or a voice\n");
return 1;
}
if (!encode_speaker(inp->speaker_ref, prompt_embd_buf) ||
mtmd_gen_state_set_ref(mctx, gen_state.get()) != 0) {
LOG_ERR("mtmd_helper_gen_audio: speaker reference encoding failed\n");
return 1;
}
const int n_e = n_embd;
auto push_row = [&](const float * src) {
prompt_embd_buf.insert(prompt_embd_buf.end(), src, src + n_e);
};
auto add_pos = [&](float * r, const std::vector<float> & tab, int idx) {
const float * p = tab.data() + (size_t) idx * n_e;
for (int j = 0; j < n_e; j++) {
r[j] += p[j];
}
};
// prompt rows: [conditioning, text, speech bos]
// no text preprocessing here, the [SPACE] substitution and the case folding are baked in at conversion
std::string txt(inp->prompt, inp->prompt_len);
if (mtl && inp->lang && inp->lang[0]) {
// the reference prefixes the language tag, e.g. "[fr]"
std::string lang = inp->lang;
std::transform(lang.begin(), lang.end(), lang.begin(), [](unsigned char c) { return (char) std::tolower(c); });
txt = "[" + lang + "]" + txt;
}
std::vector<llama_token> ids(txt.size() + 16);
int n_ids = llama_tokenize(vocab, txt.c_str(), (int32_t) txt.size(), ids.data(), (int32_t) ids.size(),
false, true);
if (n_ids < 1) {
LOG_ERR("mtmd_helper_gen_audio: tokenization failed\n");
return 1;
}
ids.resize((size_t) n_ids);
if (mtl) {
// only the multilingual prompt wraps the text in start/stop tokens
ids.insert(ids.begin(), text_start);
ids.push_back(text_stop);
if (ids.size() > text_pos.size() / (size_t) n_e) {
LOG_ERR("mtmd_helper_gen_audio: prompt too long (%zu tokens, max %zu)\n", ids.size(), text_pos.size() / (size_t) n_e);
return 1;
}
}
const size_t text0 = prompt_embd_buf.size() / (size_t) n_e;
for (size_t i = 0; i < ids.size(); i++) {
push_row(tok_embd.data() + (size_t) ids[i] * n_e);
if (mtl) {
add_pos(prompt_embd_buf.data() + (text0 + i) * n_e, text_pos, (int) i);
}
}
// speech bos opens the AR stream
push_row(tok_embd.data() + (size_t) llama_vocab_bos(vocab) * n_e);
if (mtl) {
add_pos(prompt_embd_buf.data() + prompt_embd_buf.size() - n_e, speech_pos, 0);
}
ar_idx = 1;
n_prompt = (int) (prompt_embd_buf.size() / (size_t) n_e);
if (mtl) {
// cfg uncond prompt: text rows are zeroed but keep their learned positions, as in the reference
uncond_embd_buf = prompt_embd_buf;
for (size_t i = 0; i < ids.size(); i++) {
float * r = uncond_embd_buf.data() + (text0 + i) * n_e;
std::fill(r, r + n_e, 0.0f);
add_pos(r, text_pos, (int) i);
}
}
prompt_pos = 0;
return 0;
}
int32_t step_prompt(int32_t n_batch) override {
GGML_ASSERT(n_batch > 0);
// the uncond prompt follows the cond one on mtl
const int n_total = mtl ? 2 * n_prompt : n_prompt;
if (prompt_pos >= n_total) {
return 0;
}
const bool uncond = prompt_pos >= n_prompt;
const int p0 = uncond ? prompt_pos - n_prompt : prompt_pos;
const int32_t n_tokens_batch = std::min(n_batch, n_prompt - p0);
const bool last = p0 + n_tokens_batch == n_prompt;
decode_embd_batch batch((uncond ? uncond_embd_buf : prompt_embd_buf).data(), n_prompt, 1, n_embd);
batch.set_position_normal(0, uncond ? seq_uncond : seq_id);
if (last) {
batch.logits[n_prompt - 1] = 1;
}
if (llama_process(lctx, LLAMA_PROCESS_TYPE_DECODE, batch.render(lctx, p0, n_tokens_batch)) != 0) {
LOG_ERR("mtmd_helper_gen_audio: prompt decode failed\n");
return -1;
}
prompt_pos += n_tokens_batch;
if (last && mtl) {
if (!uncond) {
cfg_read();
} else {
cfg_apply();
}
}
if (prompt_pos >= n_total) {
pos = n_prompt;
prompt_embd_buf.clear();
uncond_embd_buf.clear();
return 0;
}
return n_total - prompt_pos;
}
int32_t step_gen(llama_token sampled, const float * h_state_in, const float ** h_state_out, bool * out_stop) override {
GGML_UNUSED(h_state_in);
if (sampled < 0 || sampled >= llama_vocab_n_tokens(vocab)) {
LOG_ERR("mtmd_helper_gen_audio: chatterbox requires a token sampled from the backbone\n");
return 1;
}
// the speech stop token is the backbone eos, the multilingual speech positions end the stream too
if (llama_vocab_is_eog(vocab, sampled) || (mtl && (size_t) ar_idx >= speech_pos.size() / (size_t) n_embd)) {
*out_stop = true;
*h_state_out = nullptr;
return 0;
}
// keep only the 6561 s3gen codes, dropping start/stop and oov ids
if (sampled >= speech_base && sampled - speech_base < 6561) {
codes_buf.push_back(sampled - speech_base);
}
std::vector<float> e(tok_embd.begin() + (size_t) sampled * n_embd,
tok_embd.begin() + (size_t) (sampled + 1) * n_embd);
if (mtl) {
// the multilingual backbone takes the token row plus the learned speech position
const float * p = speech_pos.data() + (size_t) ar_idx * n_embd;
for (int j = 0; j < n_embd; j++) {
e[(size_t) j] += p[j];
}
ar_idx++;
}
if (!decode_one(e.data(), seq_id)) {
return 1;
}
if (mtl) {
// cfg: the sampled token feeds both branches
cfg_read();
if (!decode_one(e.data(), seq_uncond)) {
return 1;
}
cfg_apply();
}
pos++;
const float * h = llama_get_embeddings_ith(lctx, -1);
h_state_buf.assign(h, h + n_embd);
*h_state_out = h_state_buf.data();
return 0;
}
int32_t get_output(int32_t * out_sample_rate, const char ** out_data, size_t * out_data_len, int64_t * out_n_samples) override {
*out_sample_rate = info.sample_rate;
if (!stream) {
if (!flush_gen_wav()) {
return 1;
}
if (out_n_samples) {
*out_n_samples = (int64_t) audio_pcm.size();
}
if (out_type == MTMD_HELPER_GEN_AUDIO_OUTTYPE_PCM) {
*out_data = (const char *) audio_pcm.data();
*out_data_len = audio_pcm.size() * sizeof(float);
return 0;
}
out_buf.clear();
if (!write_wav16(out_buf, audio_pcm, info.sample_rate)) {
LOG_ERR("mtmd_helper_gen_audio: output too large for WAV\n");
return 1;
}
*out_data = out_buf.data();
*out_data_len = out_buf.size();
return 0;
}
// streaming: the flow decode is one-shot, so all the audio arrives after flush()
const size_t n_new = audio_pcm.size() - pcm_sent;
if (out_n_samples) {
*out_n_samples = (int64_t) n_new;
}
if (out_type == MTMD_HELPER_GEN_AUDIO_OUTTYPE_PCM) {
*out_data = (const char *) (audio_pcm.data() + pcm_sent);
*out_data_len = n_new * sizeof(float);
pcm_sent = audio_pcm.size();
return 0;
}
out_buf.clear();
if (!wav_header_sent) {
write_wav16_header(out_buf, UINT32_MAX, info.sample_rate);
wav_header_sent = true;
}
append_wav16_pcm(out_buf, audio_pcm.data() + pcm_sent, n_new);
pcm_sent = audio_pcm.size();
*out_data = out_buf.data();
*out_data_len = out_buf.size();
return 0;
}
int32_t flush() override {
return flush_gen_wav() ? 0 : 1;
}
private:
bool decode_one(const float * e, llama_seq_id seq) {
decode_embd_batch batch(e, 1, 1, n_embd);
batch.set_position_normal(pos, seq);
batch.logits[0] = 1;
if (llama_process(lctx, LLAMA_PROCESS_TYPE_DECODE, batch.render(lctx, 0, 1)) != 0) {
LOG_ERR("mtmd_helper_gen_audio: step decode failed\n");
return false;
}
return true;
}
// cfg of the reference: logits = cond + 0.5 * (cond - uncond)
// the mix goes into the last output row (the uncond one), the caller samples from it
void cfg_read() {
const float * c = llama_get_logits_ith(lctx, -1);
cond_logits.assign(c, c + llama_vocab_n_tokens(vocab));
}
void cfg_apply() {
float * u = llama_get_logits_ith(lctx, -1);
for (size_t i = 0; i < cond_logits.size(); i++) {
u[i] = cond_logits[i] + 0.5f * (cond_logits[i] - u[i]);
}
}
// one GEN_WAV call over the whole utterance, the flow decode is not streamed
bool flush_gen_wav() {
if (decoded || codes_buf.empty()) {
return true;
}
if (!mtl) {
// turbo appends a short silence tail before vocoding
codes_buf.insert(codes_buf.end(), 3, 4299);
}
mtmd_gen_inp inp = mtmd_gen_inp_default(mctx);
inp.type = MTMD_GEN_PROCESS_TYPE_GEN_WAV;
inp.codes = codes_buf.data();
inp.n_codes = codes_buf.size();
mtmd_gen_out out{};
if (mtmd_gen_audio_process(mctx, gen_state.get(), &inp, &out) != 0) {
LOG_ERR("mtmd_helper_gen_audio: gen_wav decode failed\n");
return false;
}
audio_pcm.assign(out.audio, out.audio + out.n_samples);
decoded = true;
return true;
}
bool ensure_cache() {
if (!tok_embd.empty()) {
return true;
}
// fused vocab layout: [text 0..speech_base) then the speech tokens
speech_base = find_special_token(vocab, "<|speech_0|>");
if (speech_base == LLAMA_TOKEN_NULL) {
LOG_ERR("mtmd_helper_gen_audio: fused speech tokens not found in vocab\n");
return false;
}
const uint32_t n_tok_embd = llama_model_get_tok_embd(model, nullptr);
if (n_tok_embd != (uint32_t) llama_vocab_n_tokens(vocab) * (uint32_t) n_embd) {
LOG_ERR("mtmd_helper_gen_audio: unexpected token embedding size\n");
return false;
}
tok_embd.resize(n_tok_embd);
llama_model_get_tok_embd(model, tok_embd.data());
auto read_table = [&](const char * name, std::vector<float> & dst) {
const size_t n = mtmd_gen_audio_read_tensor(mctx, name, nullptr, 0);
dst.resize(n);
if (n == 0 || mtmd_gen_audio_read_tensor(mctx, name, dst.data(), n) != n) {
LOG_ERR("mtmd_helper_gen_audio: %s read failed\n", name);
return false;
}
return true;
};
// multilingual: the learned position tables
mtl = mtmd_gen_audio_read_tensor(mctx, "a.gen.t3.text_pos_emb", nullptr, 0) > 0;
if (mtl && (!read_table("a.gen.t3.text_pos_emb", text_pos) || !read_table("a.gen.t3.speech_pos_emb", speech_pos))) {
return false;
}
return true;
}
// vocab and mmproj data fixed across the whole session, read once
std::vector<float> tok_embd;
bool mtl = false;
std::vector<float> text_pos;
std::vector<float> speech_pos;
llama_token speech_base = LLAMA_TOKEN_NULL;
// start_text_token and stop_text_token of the reference, only the multilingual prompt uses them
static constexpr llama_token text_start = 255;
static constexpr llama_token text_stop = 0;
// per-generation state, cleared by reset()
llama_seq_id seq_id = 0;
llama_seq_id seq_uncond = 0;
llama_pos pos = 0;
int ar_idx = 0;
std::vector<float> prompt_embd_buf;
std::vector<float> uncond_embd_buf;
int n_prompt = 0;
int prompt_pos = 0;
std::vector<float> cond_logits;
std::vector<int32_t> codes_buf;
bool decoded = false;
std::vector<float> audio_pcm;
std::vector<float> h_state_buf;
mtmd_helper_gen_audio_outtype out_type = MTMD_HELPER_GEN_AUDIO_OUTTYPE_WAV;
std::vector<char> out_buf;
bool stream = false;
size_t pcm_sent = 0; // samples already returned by get_output()
bool wav_header_sent = false;
};
// Drama 3: backbone samples the main token of each frame, codebook tower samples the other codes, Fish DAC makes the PCM
// the tower attends over the whole sequence, so it also runs on every prompt row
class drama3_gen_audio_pipeline : public mtmd_gen_audio_pipeline {
public:
using mtmd_gen_audio_pipeline::mtmd_gen_audio_pipeline;
void reset() override {
seq_id = 0;
pos = 0;
prompt_tokens.clear();
prompt_pos = 0;
frame.clear();
main_logits.clear();
hist.clear();
codes_buf.clear();
n_decoded = 0;
gen_state.reset();
pcm_sent = 0;
wav_header_sent = false;
audio_pcm.clear();
h_state_buf.clear();
out_buf.clear();
}
int32_t set_input(const mtmd_helper_gen_audio_inp * inp) override {
reset();
seq_id = inp->seq_id;
out_type = inp->out_type;
stream = inp->stream;
if (inp->speaker_ref) {
LOG_WRN("mtmd_helper_gen_audio: voice cloning is not supported yet, the speaker file is ignored\n");
}
sem_begin = find_special_token(vocab, "<|semantic:0|>");
eos = find_special_token(vocab, "<|im_end|>");
if (sem_begin == LLAMA_TOKEN_NULL || eos == LLAMA_TOKEN_NULL ||
sem_begin + n_cb_vocab > llama_vocab_n_tokens(vocab)) {
LOG_ERR("mtmd_helper_gen_audio: missing semantic tokens in vocab\n");
return 1;
}
if (!tokenize_segments(prompt_segments(std::string(inp->prompt, inp->prompt_len)), prompt_tokens)) {
return 1;
}
if (prompt_tokens.size() >= llama_n_ctx_seq(lctx) || prompt_tokens.size() > max_rows()) {
LOG_ERR("mtmd_helper_gen_audio: prompt does not fit in the context\n");
return 1;
}
if (!llama_memory_seq_rm(llama_get_memory(lctx), seq_id, 0, -1)) {
LOG_ERR("mtmd_helper_gen_audio: cannot reset the sequence\n");
return 1;
}
rng.seed(inp->seed == UINT32_MAX ? std::random_device{}() : inp->seed);
return init_gen_state(inp->seed) ? 0 : 1;
}
// run the tower on every prompt row to fill its KV
int32_t step_prompt(int32_t n_batch) override {
GGML_ASSERT(n_batch > 0);
const int n_prompt = (int) prompt_tokens.size();
if (prompt_pos >= n_prompt) {
return 0;
}
// small batches: every row also outputs logits
const int n = std::min({ n_batch, n_prompt - prompt_pos, 64 });
llama_batch_ext_ptr batch(llama_batch_ext_init(lctx));
if (!batch_set_tokens(batch.get(), prompt_tokens.data() + prompt_pos, n, pos, seq_id)) {
return -1;
}
for (int i = 0; i < n; i++) {
llama_batch_ext_set_output_logits(batch.get(), i, true);
}
if (llama_process(lctx, LLAMA_PROCESS_TYPE_DECODE, batch.get()) != 0) {
LOG_ERR("mtmd_helper_gen_audio: prompt decode failed\n");
return -1;
}
for (int i = 0; i < n; i++) {
const float * h = llama_get_embeddings_ith(lctx, i);
if (prompt_pos + i == n_prompt - 1) {
h_state_buf.assign(h, h + n_embd); // first frame comes from the last prompt row
save_main_logits(i);
break;
}
if (!run_tower(-1, h)) {
return -1;
}
}
pos += n;
prompt_pos += n;
return n_prompt - prompt_pos;
}
// sample the main token here, ignore the token of the caller
int32_t step_gen(llama_token sampled, const float * h_state_in, const float ** h_state_out, bool * out_stop) override {
GGML_UNUSED(sampled);
GGML_UNUSED(h_state_in);
*h_state_out = nullptr;
*out_stop = false;
const llama_token main = sample_main(main_logits.data());
const size_t n_rows = prompt_tokens.size() - 1 + codes_buf.size() / n_cb;
if (main == eos || (uint32_t) pos >= llama_n_ctx_seq(lctx) || n_rows >= max_rows()) {
*out_stop = true;
return 0;
}
hist.push_back(main);
if (hist.size() > ras_window) {
hist.erase(hist.begin());
}
const float * next_embd = run_tower(main - sem_begin, h_state_buf.data());
if (!next_embd) {
return 1;
}
codes_buf.insert(codes_buf.end(), frame.begin(), frame.end());
// text/speech stream: all mrope sections use the same position
const bool mrope = llama_model_rope_type(model) == LLAMA_ROPE_TYPE_MROPE ||
llama_model_rope_type(model) == LLAMA_ROPE_TYPE_IMROPE;
decode_embd_batch batch(next_embd, 1, mrope ? 4 : 1, n_embd);
if (mrope) {
batch.set_position_mrope_1d(pos, seq_id);
} else {
batch.set_position_normal(pos, seq_id);
}
batch.logits[0] = 1;
if (llama_process(lctx, LLAMA_PROCESS_TYPE_DECODE, batch.render(lctx, 0, 1)) != 0) {
LOG_ERR("mtmd_helper_gen_audio: decode failed\n");
return 1;
}
pos++;
const float * h = llama_get_embeddings_ith(lctx, -1);
h_state_buf.assign(h, h + n_embd);
save_main_logits(-1);
*h_state_out = h_state_buf.data();
if (stream && codes_buf.size() / n_cb - n_decoded >= stream_frames) {
return decode_wav() ? 0 : 1;
}
return 0;
}
int32_t get_output(int32_t * out_sample_rate, const char ** out_data, size_t * out_data_len, int64_t * out_n_samples) override {
*out_sample_rate = info.sample_rate;
if (!stream) {
if (flush() != 0) {
return 1;
}
if (out_n_samples) {
*out_n_samples = (int64_t) audio_pcm.size();
}
if (out_type == MTMD_HELPER_GEN_AUDIO_OUTTYPE_PCM) {
*out_data = (const char *) audio_pcm.data();
*out_data_len = audio_pcm.size() * sizeof(float);
return 0;
}
out_buf.clear();
if (!write_wav16(out_buf, audio_pcm, info.sample_rate)) {
LOG_ERR("mtmd_helper_gen_audio: output too large for WAV\n");
return 1;
}
*out_data = out_buf.data();
*out_data_len = out_buf.size();
return 0;
}
// streaming: only the audio since the previous call
const size_t n_new = audio_pcm.size() - pcm_sent;
if (out_n_samples) {
*out_n_samples = (int64_t) n_new;
}
if (out_type == MTMD_HELPER_GEN_AUDIO_OUTTYPE_PCM) {
*out_data = (const char *) (audio_pcm.data() + pcm_sent);
*out_data_len = n_new * sizeof(float);
pcm_sent = audio_pcm.size();
return 0;
}
out_buf.clear();
if (!wav_header_sent) {
write_wav16_header(out_buf, UINT32_MAX, info.sample_rate);
wav_header_sent = true;
}
append_wav16_pcm(out_buf, audio_pcm.data() + pcm_sent, n_new);
pcm_sent = audio_pcm.size();
*out_data = out_buf.data();
*out_data_len = out_buf.size();
return 0;
}
int32_t flush() override {
return decode_wav() ? 0 : 1;
}
protected:
// tokenize each segment alone as the reference does
bool tokenize_segments(const std::vector<std::string> & segments, std::vector<llama_token> & out) const {
for (const std::string & seg : segments) {
std::vector<llama_token> ids(seg.size() + 16);
const int32_t n = llama_tokenize(vocab, seg.c_str(), (int32_t) seg.size(), ids.data(), (int32_t) ids.size(), false, true);
if (n < 0) {
LOG_ERR("mtmd_helper_gen_audio: tokenization failed\n");
return false;
}
out.insert(out.end(), ids.begin(), ids.begin() + n);
}
return true;
}
// reference-bracket-v1 prompt without references
virtual std::vector<std::string> prompt_segments(std::string text) const {
if (text.rfind("<|speaker:", 0) != 0) {
text = "<|speaker:0|> " + text;
}
return {
"<|im_start|>system\n", "convert the provided text to speech", "<|im_end|>\n",
"<|im_start|>user\n", text, "<|im_end|>\n", "<|im_start|>assistant\n<|voice|>",
};
}
// prompt rows but the last + generated frames
virtual size_t max_rows() const { return n_tower_rows; }
// copy right after our own decode, other sequences may decode on lctx before the next step_gen
// layout: eos, then semantic codes 0..n_cb_vocab-1
void save_main_logits(int32_t i) {
const float * logits = llama_get_logits_ith(lctx, i);
main_logits.resize(n_cb_vocab + 1);
main_logits[0] = logits[eos];
std::copy(logits + sem_begin, logits + sem_begin + n_cb_vocab, main_logits.begin() + 1);
}
// repetition-aware sampling: if a semantic draw repeats one of the last ras_window main tokens, draw again with fixed settings
static constexpr size_t ras_window = 16;
static constexpr size_t n_cb = 10;
// codebook tower KV rows (mtmd_graph_drama3_gen::n_pos_max): one per prompt row but the last, one per frame
static constexpr size_t n_tower_rows = 4096;
// frames per codec call when streaming (0.33 s)
static constexpr size_t stream_frames = 7;
// top-p keeps the token that crosses the threshold, as the reference does
llama_token draw(const std::vector<float> & cand_logits, float temp, int32_t top_k, float top_p) {
std::vector<std::pair<float, int>> cand(cand_logits.size());
for (size_t i = 0; i < cand.size(); i++) {
cand[i] = { cand_logits[i] / temp, (int) i };
}
std::sort(cand.begin(), cand.end(), [](const auto & a, const auto & b) { return a.first > b.first; });
const float lmax = cand[0].first;
double sum = 0.0;
std::vector<double> prob(cand.size());
for (size_t i = 0; i < cand.size(); i++) {
prob[i] = std::exp((double) (cand[i].first - lmax));
sum += prob[i];
}
size_t n_keep = cand.size();
if (top_k > 0) {
n_keep = std::min(n_keep, (size_t) top_k);
}
double cum = 0.0;
for (size_t i = 0; i < n_keep; i++) {
if (cum > top_p * sum) {
n_keep = i;
break;
}
cum += prob[i];
}
double kept = 0.0;
for (size_t i = 0; i < n_keep; i++) {
kept += prob[i];
}
double r = std::uniform_real_distribution<double>(0.0, 1.0)(rng) * kept;
for (size_t i = 0; i < n_keep; i++) {
r -= prob[i];
if (r <= 0.0) {
return cand[i].second;
}
}
return cand[n_keep - 1].second;
}
// candidates: eos, then semantic codes 1.. (the reference never samples code 0)
// logits as saved by save_main_logits()
virtual llama_token sample_main(const float * logits) {
std::vector<float> cand(n_cb_vocab);
cand[0] = logits[0];
for (int i = 1; i < n_cb_vocab; i++) {
cand[i] = logits[1 + i];
}
auto to_token = [&](int i) { return i == 0 ? eos : sem_begin + i; };
const llama_token first = to_token(draw(cand, 1.0f, 0, 0.9f));
if (first != eos && std::find(hist.begin(), hist.end(), first) != hist.end()) {
return to_token(draw(cand, 1.0f, 30, 0.95f));
}
return first;
}
// code0 < 0 for a prompt row, else the main code of the next frame
// returns the next backbone input, the codes of the next frame go to frame
const float * run_tower(int32_t code0, const float * h) {
mtmd_gen_inp inp = mtmd_gen_inp_default(mctx);
inp.type = MTMD_GEN_PROCESS_TYPE_GEN_CODE;
inp.code0 = code0;
inp.embd = const_cast<float *>(h);
inp.codes = frame.empty() ? nullptr : frame.data();
inp.n_codes = frame.size();
mtmd_gen_out out{};
if (mtmd_gen_audio_process(mctx, gen_state.get(), &inp, &out) != 0 || (code0 >= 0 && out.n_codes != n_cb)) {
LOG_ERR("mtmd_helper_gen_audio: codebook tower failed\n");
return nullptr;
}
if (code0 >= 0) {
frame.assign(out.codes, out.codes + out.n_codes);
}
return out.embd;
}
// the codec is stateful, so frames can be decoded in pieces
bool decode_wav() {
const size_t n_frames = codes_buf.size() / n_cb;
while (n_decoded < n_frames) {
const size_t n = std::min(n_frames - n_decoded, (size_t) 64);
mtmd_gen_inp inp = mtmd_gen_inp_default(mctx);
inp.type = MTMD_GEN_PROCESS_TYPE_GEN_WAV;
inp.codes = codes_buf.data() + n_decoded * n_cb;
inp.n_codes = n * n_cb;
mtmd_gen_out out{};
if (mtmd_gen_audio_process(mctx, gen_state.get(), &inp, &out) != 0) {
LOG_ERR("mtmd_helper_gen_audio: codec decode failed\n");
return false;
}
audio_pcm.insert(audio_pcm.end(), out.audio, out.audio + out.n_samples);
n_decoded += n;
}
return true;
}
static constexpr int n_cb_vocab = 4096; // <|semantic:0..4095|>
llama_token sem_begin = LLAMA_TOKEN_NULL;
llama_token eos = LLAMA_TOKEN_NULL;
llama_seq_id seq_id = 0;
llama_pos pos = 0;
std::vector<llama_token> prompt_tokens;
int prompt_pos = 0;
std::mt19937 rng;
std::vector<float> main_logits; // see save_main_logits()
std::vector<int32_t> frame; // codes of the last frame, input of the next tower row
std::vector<llama_token> hist; // last main tokens, for repetition-aware sampling
std::vector<int32_t> codes_buf; // frame-major
size_t n_decoded = 0; // frames already through the codec
std::vector<float> audio_pcm;
std::vector<float> h_state_buf;
mtmd_helper_gen_audio_outtype out_type = MTMD_HELPER_GEN_AUDIO_OUTTYPE_WAV;
std::vector<char> out_buf;
bool stream = false;
size_t pcm_sent = 0; // samples already returned by get_output()
bool wav_header_sent = false;
};
// Fish Audio S2 Pro: same tokens, main-token loop and Fish DAC as drama3
// the fast AR only runs on generated frames, so the prompt is a plain decode
class fish_s2_gen_audio_pipeline : public drama3_gen_audio_pipeline {
public:
using drama3_gen_audio_pipeline::drama3_gen_audio_pipeline;
void reset() override {
drama3_gen_audio_pipeline::reset();
parts.clear();
part_idx = 0;
part_pos = 0;
}
// voice cloning (sglang-omni fishaudio_s2_pro): a system message with the transcript and the codec rows of the clip
int32_t set_input(const mtmd_helper_gen_audio_inp * inp) override {
std::vector<float> ref_rows;
if (inp->speaker_ref && !encode_speaker(inp->speaker_ref, ref_rows)) {
return 1;
}
mtmd_helper_gen_audio_inp text_inp = *inp;
text_inp.speaker_ref = nullptr;
if (drama3_gen_audio_pipeline::set_input(&text_inp) != 0) {
return 1;
}
size_t n_total = prompt_tokens.size();
if (!ref_rows.empty()) {
// optional, the reference leaves the text empty when it has none
std::string ref_text = inp->speaker_ref_text ? inp->speaker_ref_text : "";
if (!ref_text.empty() && ref_text.rfind("<|speaker:", 0) != 0) {
ref_text = "<|speaker:0|>" + ref_text;
}
prompt_part head, rows, tail;
if (!tokenize_segments({ "<|im_start|>system\n", "convert the provided text to speech reference to the following:\n\nText:\n",
ref_text, "\n\nSpeech:\n" }, head.tokens) ||
!tokenize_segments({ "<|im_end|>\n" }, tail.tokens)) {
return 1;
}
rows.rows = std::move(ref_rows);
for (auto * part : { &head, &rows, &tail }) {
n_total += part->size(n_embd);
parts.push_back(std::move(*part));
}
}
if (n_total >= llama_n_ctx_seq(lctx)) {
LOG_ERR("mtmd_helper_gen_audio: prompt does not fit in the context\n");
return 1;
}
prompt_part text;
text.tokens = prompt_tokens;
parts.push_back(std::move(text));
return 0;
}
// the first frame comes from the last prompt row
int32_t step_prompt(int32_t n_batch) override {
GGML_ASSERT(n_batch > 0);
if (part_idx >= parts.size()) {
return 0;
}
const prompt_part & part = parts[part_idx];
const int n_part = part.size(n_embd);
const int n = std::min(n_batch, n_part - part_pos);
const bool last = part_idx + 1 == parts.size() && part_pos + n == n_part;
int ret = 0;
if (part.rows.empty()) {
llama_batch_ext_ptr batch(llama_batch_ext_init(lctx));
if (!batch_set_tokens(batch.get(), part.tokens.data() + part_pos, n, pos, seq_id)) {
return -1;
}
if (last) {
llama_batch_ext_set_output_logits(batch.get(), n - 1, true);
}
ret = llama_process(lctx, LLAMA_PROCESS_TYPE_DECODE, batch.get());
} else {
decode_embd_batch batch(part.rows.data() + (size_t) part_pos * n_embd, n, 1, n_embd);
batch.set_position_normal(pos, seq_id);
ret = llama_process(lctx, LLAMA_PROCESS_TYPE_DECODE, batch.render(lctx, 0, n));
}
if (ret != 0) {
LOG_ERR("mtmd_helper_gen_audio: prompt decode failed\n");
return -1;
}
if (last) {
const float * h = llama_get_embeddings_ith(lctx, -1);
h_state_buf.assign(h, h + n_embd);
save_main_logits(-1);
}
pos += n;
part_pos += n;
if (part_pos == n_part) {
part_idx++;
part_pos = 0;
}
int32_t n_left = -part_pos;
for (size_t i = part_idx; i < parts.size(); i++) {
n_left += parts[i].size(n_embd);
}
return n_left;
}
protected:
// reference (sglang-omni fishaudio_s2_pro) has no system message
std::vector<std::string> prompt_segments(std::string text) const override {
if (text.rfind("<|speaker:", 0) != 0) {
text = "<|speaker:0|>" + text;
}
return {
"<|im_start|>user\n", text, "<|im_end|>\n", "<|im_start|>assistant\n<|voice|>",
};
}
// no codebook tower, only the context limits the length
size_t max_rows() const override { return SIZE_MAX; }
// token ids, or backbone input rows of n_embd floats
struct prompt_part {
std::vector<llama_token> tokens;
std::vector<float> rows;
int size(int n_embd) const { return rows.empty() ? (int) tokens.size() : (int) (rows.size() / n_embd); }
};
std::vector<prompt_part> parts;
size_t part_idx = 0;
int part_pos = 0;
// same as the reference: repetition penalty over the last ras_window main tokens, top-k, top-p on the
// untempered logits (the crossing token is dropped), then temperature
// a repeat within the last 4 main tokens switches to the RAS settings
llama_token sample_main(const float * logits) override {
static constexpr int32_t top_k = 30;
static constexpr float rep_penalty = 1.1f;
std::vector<std::pair<float, llama_token>> cand;
cand.reserve(n_cb_vocab + 1);
cand.push_back({ logits[0], eos });
for (int i = 0; i < n_cb_vocab; i++) {
cand.push_back({ logits[1 + i], sem_begin + i });
}
for (auto & c : cand) {
if (std::find(hist.begin(), hist.end(), c.second) != hist.end()) {
c.first = c.first < 0.0f ? c.first * rep_penalty : c.first / rep_penalty;
}
}
bool ras = false;
if (hist.size() >= 4) {
std::vector<llama_token> last4(hist.end() - 4, hist.end());
std::sort(last4.begin(), last4.end());
ras = std::adjacent_find(last4.begin(), last4.end()) != last4.end();
}
const float temp = ras ? 1.0f : 0.8f;
const float top_p = ras ? 0.9f : 0.8f;
const size_t k = std::min<size_t>(top_k, cand.size());
std::partial_sort(cand.begin(), cand.begin() + k, cand.end(), [](const auto & a, const auto & b) { return a.first > b.first; });
std::vector<double> prob(k);
double sum = 0.0;
for (size_t i = 0; i < k; i++) {
prob[i] = std::exp((double) (cand[i].first - cand[0].first));
sum += prob[i];
}
size_t n_keep = k;
double cum = 0.0;
for (size_t i = 0; i < k; i++) {
cum += prob[i];
if (i > 0 && cum > top_p * sum) {
n_keep = i;
break;
}
}
double kept = 0.0;
for (size_t i = 0; i < n_keep; i++) {
prob[i] = std::exp((double) (cand[i].first - cand[0].first) / temp);
kept += prob[i];
}
double r = std::uniform_real_distribution<double>(0.0, 1.0)(rng) * kept;
for (size_t i = 0; i < n_keep; i++) {
r -= prob[i];
if (r <= 0.0) {
return cand[i].second;
}
}
return cand[n_keep - 1].second;
}
};
// ZONOS3: one backbone row per frame of 10 codebooks, codebook j lags j frames
class zonos3_gen_audio_pipeline : public mtmd_gen_audio_pipeline {
public:
using mtmd_gen_audio_pipeline::mtmd_gen_audio_pipeline;
bool supports_delivery() const override { return true; }
void reset() override {
seq_id = 0;
pos = 0;
segments.clear();
seg_idx = 0;
seg_pos = 0;
rows.clear();
step = 0;
eos_row = -1;
countdown = 0;
done = false;
n_decoded = 0;
n_codec_samples = 0;
flushed = false;
gen_state.reset();
pcm_sent = 0;
wav_header_sent = false;
audio_pcm.clear();
h_state_buf.clear();
out_buf.clear();
}
int32_t set_input(const mtmd_helper_gen_audio_inp * inp) override {
reset();
seq_id = inp->seq_id;
out_type = inp->out_type;
stream = inp->stream;
// settings of the reference, only the temperature can be changed
const mtmd_gen_inp def = mtmd_gen_inp_default(mctx);
temp = inp->temp >= 0.0f ? inp->temp : def.temp;
top_k = def.top_k;
top_p = def.top_p;
rep_pen = def.penalty_repeat;
rep_win = def.penalty_last_n;
rng.seed(inp->seed == UINT32_MAX ? std::random_device{}() : inp->seed);
if (!init_gen_state(inp->seed)) {
return 1;
}
// prompt: [voice rows] [condition rows] BOS text EOS [silence rows]
// the speaker CFG of the reference is omitted: it adds too much code and maintenance cost
if (inp->speaker_ref) {
segment seg;
if (!encode_speaker(inp->speaker_ref, seg.embd)) {
return 1;
}
segments.push_back(std::move(seg));
}
// condition rows: the bucket of each value over the checkpoint edges, or unknown
auto bucket = [](const std::vector<float> & edges, float v) {
return std::to_string(std::upper_bound(edges.begin(), edges.end(), v) - edges.begin());
};
const std::string loudness = inp->loudness != 0.0f ? bucket(loudness_edges, inp->loudness) : "unk";
const std::string rate = inp->speed != 1.0f ? bucket(rate_edges, inp->speed * typical_rate) : "unk";
segment text;
for (const std::string & name : { "<|loudness_" + loudness + "|>", std::string("<|fmax_unk|>"), "<|speaking_rate_" + rate + "|>" }) {
const llama_token t = find_special_token(vocab, name);
if (t == LLAMA_TOKEN_NULL) {
LOG_ERR("mtmd_helper_gen_audio: missing ZONOS3 token %s in vocab\n", name.c_str());
return 1;
}
text.tokens.push_back(t);
}
const std::string prompt(inp->prompt, inp->prompt_len);
std::vector<llama_token> ids(prompt.size() + 16);
const int32_t n_ids = llama_tokenize(vocab, prompt.c_str(), (int32_t) prompt.size(), ids.data(), (int32_t) ids.size(), true, false);
if (n_ids <= 2) {
LOG_ERR("mtmd_helper_gen_audio: empty prompt or tokenization failed\n");
return 1;
}
text.tokens.insert(text.tokens.end(), ids.begin(), ids.begin() + n_ids);
segments.push_back(std::move(text));
// codec silence, delayed
segment silence;
for (int k = 0; k < n_prefill; k++) {
std::vector<int32_t> row(n_cb, audio_pad);
for (int cb = 0; cb <= k && cb < n_cb; cb++) {
row[cb] = silence_frame[cb];
}
const float * embd = embed_row(row);
if (!embd) {
return 1;
}
silence.embd.insert(silence.embd.end(), embd, embd + n_embd);
rows.insert(rows.end(), row.begin(), row.end());
}
segments.push_back(std::move(silence));
size_t n_prompt = 0;
for (const auto & seg : segments) {
n_prompt += seg.n_rows(n_embd);
}
if (n_prompt >= llama_n_ctx_seq(lctx)) {
LOG_ERR("mtmd_helper_gen_audio: prompt does not fit in the context\n");
return 1;
}
if (!llama_memory_seq_rm(llama_get_memory(lctx), seq_id, 0, -1)) {
LOG_ERR("mtmd_helper_gen_audio: cannot reset the sequence\n");
return 1;
}
return 0;
}
int32_t step_prompt(int32_t n_batch) override {
GGML_ASSERT(n_batch > 0);
if (seg_idx >= segments.size()) {
return 0;
}
const segment & seg = segments[seg_idx];
const int n_rows = (int) seg.n_rows(n_embd);
const int n = std::min(n_batch, n_rows - seg_pos);
const bool last = seg_idx + 1 == segments.size() && seg_pos + n == n_rows;
int ret;
if (!seg.tokens.empty()) {
llama_batch_ext_ptr batch(llama_batch_ext_init(lctx));
if (!batch_set_tokens(batch.get(), seg.tokens.data() + seg_pos, n, pos, seq_id)) {
return -1;
}
if (last) {
llama_batch_ext_set_output_logits(batch.get(), n - 1, true);
}
ret = llama_process(lctx, LLAMA_PROCESS_TYPE_DECODE, batch.get());
} else {
decode_embd_batch batch(seg.embd.data() + (size_t) seg_pos * n_embd, n, 1, n_embd);
batch.set_position_normal(pos, seq_id);
batch.logits[n - 1] = last;
ret = llama_process(lctx, LLAMA_PROCESS_TYPE_DECODE, batch.render(lctx, 0, n));
}
if (ret != 0) {
LOG_ERR("mtmd_helper_gen_audio: prompt decode failed\n");
return -1;
}
pos += n;
seg_pos += n;
if (seg_pos == n_rows) {
seg_idx++;
seg_pos = 0;
}
int32_t n_left = 0;
for (size_t i = seg_idx; i < segments.size(); i++) {
n_left += (int32_t) segments[i].n_rows(n_embd) - (i == seg_idx ? seg_pos : 0);
}
if (n_left == 0) {
const float * h = llama_get_embeddings_ith(lctx, -1);
h_state_buf.assign(h, h + n_embd);
}
return n_left;
}
int32_t step_gen(llama_token sampled, const float * h_state_in, const float ** h_state_out, bool * out_stop) override {
GGML_UNUSED(sampled);
GGML_UNUSED(h_state_in);
*h_state_out = nullptr;
*out_stop = false;
if (done) {
*out_stop = true;
return 0;
}
mtmd_gen_inp inp = mtmd_gen_inp_default(mctx);
inp.type = MTMD_GEN_PROCESS_TYPE_GEN_CODE;
inp.embd = h_state_buf.data();
mtmd_gen_out out{};
if (mtmd_gen_audio_process(mctx, gen_state.get(), &inp, &out) != 0 || out.n_feats != (size_t) n_cb * n_vocab) {
LOG_ERR("mtmd_helper_gen_audio: codebook heads failed\n");
return 1;
}
std::vector<int32_t> frame(n_cb);
for (int cb = 0; cb < n_cb; cb++) {
frame[cb] = sample(out.feats + (size_t) cb * n_vocab, cb);
}
// codebooks whose delayed stream has not started yet are pad
const int r = n_prefill + step;
for (int cb = r + 1; cb < n_cb; cb++) {
frame[cb] = audio_pad;
}
rows.insert(rows.end(), frame.begin(), frame.end());
step++;
// after EOA, n_cb + 1 more rows flush the delay
int max_eoa = -1;
for (int cb = 0; cb < n_cb; cb++) {
if (frame[cb] == audio_eoa) {
max_eoa = cb;
}
}
if (eos_row < 0 && max_eoa >= 0) {
eos_row = std::max(0, r - max_eoa);
countdown = n_cb + 1;
}
if ((countdown > 0 && --countdown == 0) || step >= max_frames || (uint32_t) pos >= llama_n_ctx_seq(lctx)) {
done = true;
*out_stop = true;
return 0;
}
const float * next = embed_row(frame);
if (!next) {
return 1;
}
decode_embd_batch batch(next, 1, 1, n_embd);
batch.set_position_normal(pos, seq_id);
batch.logits[0] = 1;
if (llama_process(lctx, LLAMA_PROCESS_TYPE_DECODE, batch.render(lctx, 0, 1)) != 0) {
LOG_ERR("mtmd_helper_gen_audio: decode failed\n");
return 1;
}
pos++;
const float * h = llama_get_embeddings_ith(lctx, -1);
h_state_buf.assign(h, h + n_embd);
*h_state_out = h_state_buf.data();
if (stream) {
return decode_stream() ? 0 : 1;
}
return 0;
}
int32_t get_output(int32_t * out_sample_rate, const char ** out_data, size_t * out_data_len, int64_t * out_n_samples) override {
*out_sample_rate = info.sample_rate;
if (!stream) {
if (flush() != 0) {
return 1;
}
if (out_n_samples) {
*out_n_samples = (int64_t) audio_pcm.size();
}
if (out_type == MTMD_HELPER_GEN_AUDIO_OUTTYPE_PCM) {
*out_data = (const char *) audio_pcm.data();
*out_data_len = audio_pcm.size() * sizeof(float);
return 0;
}
out_buf.clear();
if (!write_wav16(out_buf, audio_pcm, info.sample_rate)) {
LOG_ERR("mtmd_helper_gen_audio: output too large for WAV\n");
return 1;
}
*out_data = out_buf.data();
*out_data_len = out_buf.size();
return 0;
}
// streaming: only the audio since the previous call
const size_t n_new = audio_pcm.size() - pcm_sent;
if (out_n_samples) {
*out_n_samples = (int64_t) n_new;
}
if (out_type == MTMD_HELPER_GEN_AUDIO_OUTTYPE_PCM) {
*out_data = (const char *) (audio_pcm.data() + pcm_sent);
*out_data_len = n_new * sizeof(float);
pcm_sent = audio_pcm.size();
return 0;
}
out_buf.clear();
if (!wav_header_sent) {
write_wav16_header(out_buf, UINT32_MAX, info.sample_rate);
wav_header_sent = true;
}
append_wav16_pcm(out_buf, audio_pcm.data() + pcm_sent, n_new);
pcm_sent = audio_pcm.size();
*out_data = out_buf.data();
*out_data_len = out_buf.size();
return 0;
}
int32_t flush() override {
return decode_wav(true) ? 0 : 1;
}
private:
static constexpr int n_cb = 10;
static constexpr int n_vocab = 4098; // codes, EOA, pad
static constexpr int audio_eoa = 4096;
static constexpr int audio_pad = 4097;
// condition_edges of params.json; speed 1 is this many UTF-8 bytes per second, the middle of the rate buckets
const std::vector<float> loudness_edges = { -29.778f, -26.947f, -25.047f, -23.3f, -21.743f, -20.386f, -19.186f, -17.851f, -15.967f };
const std::vector<float> rate_edges = {
6.546f, 8.182f, 9.273f, 10.31f, 11.054f, 11.667f, 12.193f, 12.656f, 13.083f, 13.476f, 13.846f, 14.196f, 14.53f, 14.887f, 15.193f,
15.517f, 15.865f, 16.208f, 16.579f, 16.972f, 17.411f, 17.898f, 18.434f, 19.069f, 19.818f, 20.816f, 22.286f, 24.722f, 29.25f,
};
static constexpr float typical_rate = 15.0f;
// digital silence of the ZAC codec (registries/silence_frame_*.json)
static constexpr int32_t silence_frame[n_cb] = { 1918, 746, 3544, 234, 1267, 2914, 3944, 2849, 326, 1337 };
static constexpr int n_prefill = 3;
// -3 dB of headroom
static constexpr int samples_per_frame = 2940;
static constexpr float out_gain = 0.70794578f;
static constexpr int rep_codebooks = 6;
static constexpr int max_frames = 2000;
static constexpr size_t stream_frames = 8;
struct segment {
std::vector<llama_token> tokens; // either tokens or embd rows
std::vector<float> embd;
size_t n_rows(int n) const { return tokens.empty() ? embd.size() / (size_t) n : tokens.size(); }
};
const float * embed_row(const std::vector<int32_t> & codes) {
mtmd_gen_inp inp = mtmd_gen_inp_default(mctx);
inp.type = MTMD_GEN_PROCESS_TYPE_GEN_CODE;
inp.codes = const_cast<int32_t *>(codes.data());
inp.n_codes = codes.size();
mtmd_gen_out out{};
if (mtmd_gen_audio_process(mctx, gen_state.get(), &inp, &out) != 0) {
LOG_ERR("mtmd_helper_gen_audio: codebook embedding failed\n");
return nullptr;
}
return out.embd;
}
// same as the reference sampler (servers/ggml/src/zonos3-sampler.cpp)
int32_t sample(const float * in, int cb) {
std::vector<float> l(in, in + n_vocab);
l[audio_pad] = -INFINITY;
if (rep_pen != 1.0f && cb < rep_codebooks) {
const size_t n_rows = rows.size() / n_cb - n_prefill;
const size_t n_win = std::min(n_rows, (size_t) std::max(rep_win, 0));
std::vector<bool> seen(audio_eoa, false);
for (size_t f = n_prefill + n_rows - n_win; f < n_prefill + n_rows; f++) {
const int32_t c = rows[f * n_cb + (size_t) cb];
if (c >= 0 && c < audio_eoa && !seen[(size_t) c]) {
seen[(size_t) c] = true;
l[c] = l[c] > 0.0f ? l[c] / rep_pen : l[c] * rep_pen;
}
}
}
if (temp <= 0.0f) {
return (int32_t) (std::max_element(l.begin(), l.end()) - l.begin());
}
for (auto & v : l) {
v /= temp;
}
float kth = -INFINITY;
if (top_k > 0 && top_k < n_vocab) {
std::vector<float> tmp(l);
std::nth_element(tmp.begin(), tmp.begin() + (top_k - 1), tmp.end(), std::greater<float>());
kth = tmp[(size_t) top_k - 1];
}
std::vector<int32_t> idx;
for (int32_t i = 0; i < n_vocab; i++) {
if (l[i] >= kth && l[i] > -INFINITY) {
idx.push_back(i);
}
}
float lmax = -INFINITY;
for (int32_t i : idx) {
lmax = std::max(lmax, l[i]);
}
std::vector<double> p(idx.size());
double sum = 0.0;
for (size_t j = 0; j < idx.size(); j++) {
p[j] = std::exp((double) (l[idx[j]] - lmax));
sum += p[j];
}
std::vector<size_t> ord(idx.size());
for (size_t j = 0; j < ord.size(); j++) {
ord[j] = j;
}
std::sort(ord.begin(), ord.end(), [&](size_t a, size_t b) { return p[a] > p[b]; });
if (top_p > 0.0f && top_p < 1.0f) {
double before = 0.0;
for (size_t j : ord) {
const double pj = p[j];
if (before > top_p * sum) {
p[j] = 0.0;
}
before += pj;
}
}
double kept = 0.0;
for (double v : p) {
kept += v;
}
double u = std::uniform_real_distribution<double>(0.0, kept)(rng);
for (size_t j = 0; j < p.size(); j++) {
u -= p[j];
if (u <= 0.0 && p[j] > 0.0) {
return idx[j];
}
}
return idx[ord[0]];
}
// frames with all codebooks known
size_t n_ready() const {
const size_t n_rows = rows.size() / n_cb;
size_t n = n_rows >= (size_t) n_cb ? n_rows - (n_cb - 1) : 0;
if (eos_row >= 0) {
n = std::min(n, (size_t) eos_row);
}
return n;
}
// streaming, as the reference does for the first audio: the silence frames are decoded with silence codes only,
// then the first generated frame from its first code alone; later chunks of complete frames grow 2, 4, 8
bool decode_stream() {
if (n_decoded == 0) {
const size_t n_rows = rows.size() / n_cb;
if (n_rows <= (size_t) n_prefill || rows[(size_t) n_prefill * n_cb] >= audio_eoa) {
return true;
}
std::vector<int32_t> codes;
for (int k = 0; k < n_prefill; k++) {
codes.insert(codes.end(), silence_frame, silence_frame + n_cb);
}
for (int cb = 0; cb < n_cb; cb++) {
const size_t r = (size_t) n_prefill + cb;
const int32_t c = r < n_rows ? rows[r * n_cb + cb] : -1;
codes.push_back(c >= audio_eoa ? -1 : c);
}
if (!run_codec(codes.data(), codes.size())) {
return false;
}
n_decoded = n_prefill + 1;
return true;
}
const size_t n_chunk = std::min(stream_frames, n_decoded - n_prefill + 1);
if (n_ready() >= n_decoded + n_chunk) {
return decode_wav(false);
}
return true;
}
// the silence frames are decoded too, as left context
bool decode_wav(bool final) {
const size_t n_frames = n_ready();
std::vector<int32_t> codes;
while (n_decoded < n_frames) {
const size_t n = std::min(n_frames - n_decoded, (size_t) 64);
codes.resize(n * n_cb);
for (size_t f = 0; f < n; f++) {
for (int cb = 0; cb < n_cb; cb++) {
// EOA and pad have no codec row
const int32_t c = rows[(n_decoded + f + cb) * n_cb + cb];
codes[f * n_cb + cb] = std::clamp(c, 0, audio_eoa - 1);
}
}
if (!run_codec(codes.data(), codes.size())) {
return false;
}
n_decoded += n;
}
if (final && done && n_decoded > 0 && !flushed) {
flushed = true;
return run_codec(nullptr, 0);
}
return true;
}
bool run_codec(const int32_t * codes, size_t n_codes) {
mtmd_gen_inp inp = mtmd_gen_inp_default(mctx);
inp.type = MTMD_GEN_PROCESS_TYPE_GEN_WAV;
inp.codes = const_cast<int32_t *>(codes);
inp.n_codes = n_codes;
mtmd_gen_out out{};
if (mtmd_gen_audio_process(mctx, gen_state.get(), &inp, &out) != 0) {
LOG_ERR("mtmd_helper_gen_audio: codec decode failed\n");
return false;
}
// drop the audio of the silence frames, then the output gain and soft limiter of the reference
const float knee = std::pow(10.0f, -1.0f / 20.0f);
for (size_t i = 0; i < out.n_samples; i++, n_codec_samples++) {
if (n_codec_samples < (size_t) n_prefill * samples_per_frame) {
continue;
}
const float x = out.audio[i] * out_gain;
const float a = std::fabs(x);
audio_pcm.push_back(a <= knee ? x : std::copysign(knee + (1.0f - knee) * std::tanh((a - knee) / (1.0f - knee)), x));
}
return true;
}
std::vector<segment> segments;
size_t seg_idx = 0;
int seg_pos = 0;
std::vector<int32_t> rows; // delayed rows: silence rows, then the generated ones
int step = 0;
int eos_row = -1;
int countdown = 0;
bool done = false;
size_t n_decoded = 0;
size_t n_codec_samples = 0;
bool flushed = false;
float temp = 0.75f;
int32_t top_k = 80;
float top_p = 0.69f;
float rep_pen = 2.4f;
int32_t rep_win = 96;
std::mt19937 rng;
llama_seq_id seq_id = 0;
llama_pos pos = 0;
mtmd_helper_gen_audio_outtype out_type = MTMD_HELPER_GEN_AUDIO_OUTTYPE_WAV;
bool stream = false;
std::vector<float> h_state_buf;
std::vector<float> audio_pcm;
std::vector<char> out_buf;
size_t pcm_sent = 0;
bool wav_header_sent = false;
};
static std::unique_ptr<mtmd_gen_audio_pipeline> make_pipeline(llama_context * lctx, mtmd_context * mctx) {
switch (mtmd_gen_audio_get_info(mctx).type) {
case MTMD_GEN_AUDIO_TYPE_QWEN3TTS:
return std::unique_ptr<mtmd_gen_audio_pipeline>(new qwen3tts_gen_audio_pipeline(lctx, mctx));
case MTMD_GEN_AUDIO_TYPE_POCKETTTS:
return std::unique_ptr<mtmd_gen_audio_pipeline>(new pockettts_gen_audio_pipeline(lctx, mctx));
case MTMD_GEN_AUDIO_TYPE_SOPRANO:
return std::unique_ptr<mtmd_gen_audio_pipeline>(new soprano_gen_audio_pipeline(lctx, mctx));
case MTMD_GEN_AUDIO_TYPE_LFM3A:
return std::unique_ptr<mtmd_gen_audio_pipeline>(new lfm3a_gen_audio_pipeline(lctx, mctx));
case MTMD_GEN_AUDIO_TYPE_CHATTERBOX:
return std::unique_ptr<mtmd_gen_audio_pipeline>(new chatterbox_gen_audio_pipeline(lctx, mctx));
case MTMD_GEN_AUDIO_TYPE_DRAMA3:
return std::unique_ptr<mtmd_gen_audio_pipeline>(new drama3_gen_audio_pipeline(lctx, mctx));
case MTMD_GEN_AUDIO_TYPE_ZONOS3:
return std::unique_ptr<mtmd_gen_audio_pipeline>(new zonos3_gen_audio_pipeline(lctx, mctx));
case MTMD_GEN_AUDIO_TYPE_FISH_S2:
return std::unique_ptr<mtmd_gen_audio_pipeline>(new fish_s2_gen_audio_pipeline(lctx, mctx));
default:
return nullptr;
}
}
struct mtmd_helper_gen_audio {
llama_context * lctx = nullptr;
std::unique_ptr<mtmd_gen_audio_pipeline> pipeline;
};
mtmd_helper_gen_audio * mtmd_helper_gen_audio_init(struct llama_context * lctx, struct mtmd_context * mctx) {
auto * ctx = new mtmd_helper_gen_audio();
ctx->lctx = lctx;
ctx->pipeline = make_pipeline(lctx, mctx);
return ctx;
}
void mtmd_helper_gen_audio_free(mtmd_helper_gen_audio * ctx) {
delete ctx;
}
void mtmd_helper_gen_audio_reset(mtmd_helper_gen_audio * ctx) {
if (ctx && ctx->pipeline) {
ctx->pipeline->reset();
}
}
struct mtmd_helper_gen_audio_inp mtmd_helper_gen_audio_inp_default(void) {
mtmd_helper_gen_audio_inp inp{};
inp.top_k = 50;
inp.top_p = 1.0f;
inp.temp = -1.0f; // model default
inp.seed = UINT32_MAX; // random
inp.speed = 1.0f;
inp.loudness = 0.0f;
inp.out_type = MTMD_HELPER_GEN_AUDIO_OUTTYPE_WAV;
return inp;
}
int32_t mtmd_helper_gen_audio_set_input(mtmd_helper_gen_audio * ctx, const mtmd_helper_gen_audio_inp * inp) {
if (!ctx->pipeline) {
LOG_ERR("mtmd_helper_gen_audio: unsupported or missing gen-audio pipeline\n");
return 1;
}
if (inp->n_past > 0 && !ctx->pipeline->supports_continue()) {
LOG_ERR("mtmd_helper_gen_audio: the pipeline cannot continue a sequence\n");
return 1;
}
if (!std::isfinite(inp->speed) || inp->speed <= 0.0f || !std::isfinite(inp->loudness) || inp->loudness > 0.0f) {
LOG_ERR("mtmd_helper_gen_audio: speed must be > 0 and loudness < 0 LUFS (0 = model default)\n");
return 2;
}
if ((inp->speed != 1.0f || inp->loudness != 0.0f) && !ctx->pipeline->supports_delivery()) {
LOG_ERR("mtmd_helper_gen_audio: the model does not support speed or loudness\n");
return 2;
}
ctx->pipeline->logit_bias.clear();
return ctx->pipeline->set_input(inp);
}
int32_t mtmd_helper_gen_audio_step_prompt(mtmd_helper_gen_audio * ctx, int32_t n_batch) {
if (!ctx->pipeline) {
return -1;
}
// the context may use a smaller n_batch than requested (e.g. clamped to n_ctx), llama_process() would fail above it
if (n_batch <= 0 || (uint32_t) n_batch > llama_n_batch(ctx->lctx)) {
LOG_ERR("mtmd_helper_gen_audio: n_batch must be in [1, %u], got %d\n", llama_n_batch(ctx->lctx), n_batch);
return -1;
}
return ctx->pipeline->step_prompt(n_batch);
}
bool mtmd_helper_gen_audio_next_is_text(mtmd_helper_gen_audio * ctx) {
return ctx->pipeline && ctx->pipeline->next_is_text();
}
const llama_logit_bias * mtmd_helper_gen_audio_get_logit_bias(mtmd_helper_gen_audio * ctx, size_t * n_bias) {
*n_bias = ctx->pipeline ? ctx->pipeline->logit_bias.size() : 0;
return *n_bias ? ctx->pipeline->logit_bias.data() : nullptr;
}
int32_t mtmd_helper_audio_stream_get_logit_bias(const mtmd_context * mctx, const llama_model * model,
llama_logit_bias * out, int32_t n_max) {
std::vector<llama_logit_bias> bias;
if (mtmd_gen_audio_get_info(mctx).type == MTMD_GEN_AUDIO_TYPE_LFM3A) {
// LFM3-Audio live ASR: only <|text_end|> ends the text of a chunk, the stream adds the final <|mixed_end|>
const llama_vocab * vocab = llama_model_get_vocab(model);
for (const char * piece : { "<|mixed_end|>", "<|mixed_start|>", "<|audio_start|>", "<|im_start|>", "<|im_end|>", "<|endoftext|>" }) {
const llama_token tok = find_special_token(vocab, piece);
if (tok != LLAMA_TOKEN_NULL) {
bias.push_back({ tok, -INFINITY });
}
}
}
for (int32_t i = 0; i < n_max && i < (int32_t) bias.size(); i++) {
out[i] = bias[i];
}
return (int32_t) bias.size();
}
int32_t mtmd_helper_gen_audio_step_gen(mtmd_helper_gen_audio * ctx, llama_token sampled,
const float * h_state_in, const float ** h_state_out,
bool * out_stop) {
if (!ctx->pipeline) {
return 1;
}
bool stop = false;
const int32_t ret = ctx->pipeline->step_gen(sampled, h_state_in, h_state_out, &stop);
if (out_stop) {
*out_stop = stop;
}
return ret;
}
int32_t mtmd_helper_gen_audio_get_output(mtmd_helper_gen_audio * ctx, int32_t * out_sample_rate,
const char ** out_data, size_t * out_data_len, int64_t * out_n_samples) {
if (!ctx->pipeline) {
return 1;
}
return ctx->pipeline->get_output(out_sample_rate, out_data, out_data_len, out_n_samples);
}
int32_t mtmd_helper_gen_audio_flush(mtmd_helper_gen_audio * ctx) {
if (!ctx->pipeline) {
return 1;
}
return ctx->pipeline->flush();
}