#include "mtmd.h" #include "mtmd-helper.h" #include "mtmd-helper-common.h" #include "llama.h" #include "../src/llama-ext.h" #include #include #include #include #include #include #include #include #include #include #ifdef MTMD_INTERNAL_HEADER #error "mtmd-helper is a public library outside of mtmd. it must not include internal headers" #endif // // Audio generation helpers // // --tts-lang codes -> language names used by the codec_language special tokens static const std::unordered_map tts_lang_codes = { { "zh", "chinese" }, { "en", "english" }, { "de", "german" }, { "it", "italian" }, { "pt", "portuguese" }, { "es", "spanish" }, { "ja", "japanese" }, { "ko", "korean" }, { "fr", "french" }, { "ru", "russian" }, }; static std::string tts_resolve_lang(const std::string & lang) { auto it = tts_lang_codes.find(lang); return it != tts_lang_codes.end() ? it->second : lang; } static llama_token find_special_token(const llama_vocab * vocab, const std::string & piece) { const int32_t n = llama_vocab_n_tokens(vocab); for (llama_token t = 0; t < n; t++) { if (piece == llama_vocab_get_text(vocab, t)) { return t; } } return LLAMA_TOKEN_NULL; } static bool batch_set_tokens(llama_batch_ext * batch, const llama_token * tokens, int32_t n, llama_pos pos_0, llama_seq_id seq_id) { llama_batch_ext_clear(batch); for (int32_t i = 0; i < n; i++) { const int32_t idx = llama_batch_ext_add_token(batch, seq_id, tokens[i]); const llama_pos pos = pos_0 + i; if (idx < 0) { LOG_ERR("mtmd_helper_gen_audio: failed to add token %d to batch (error %d)\n", tokens[i], idx); return false; } llama_batch_ext_set_pos(batch, idx, &pos); } return true; } static void put_bytes(std::vector & buf, const void * p, size_t n) { const char * c = (const char *) p; buf.insert(buf.end(), c, c + n); } // data_sz == UINT32_MAX writes the "unknown length" sentinel (streaming), same as ffmpeg does on a pipe // multi-channel PCM is interleaved static void write_wav16_header(std::vector & buf, uint32_t data_sz, int32_t rate, uint16_t ch = 1) { const uint32_t riff_sz = data_sz == UINT32_MAX ? UINT32_MAX : 36 + data_sz; const uint32_t fmt_sz = 16, byte_rate = (uint32_t) rate * 2 * ch; const uint16_t fmt = 1, align = 2 * ch, bits = 16; const uint32_t rate32 = (uint32_t) rate; put_bytes(buf, "RIFF", 4); put_bytes(buf, &riff_sz, 4); put_bytes(buf, "WAVE", 4); put_bytes(buf, "fmt ", 4); put_bytes(buf, &fmt_sz, 4); put_bytes(buf, &fmt, 2); put_bytes(buf, &ch, 2); put_bytes(buf, &rate32, 4); put_bytes(buf, &byte_rate, 4); put_bytes(buf, &align, 2); put_bytes(buf, &bits, 2); put_bytes(buf, "data", 4); put_bytes(buf, &data_sz, 4); } static void append_wav16_pcm(std::vector & buf, const float * pcm, size_t n) { for (size_t i = 0; i < n; i++) { int16_t s = (int16_t) std::lround(std::max(-1.0f, std::min(1.0f, pcm[i])) * 32767.0f); put_bytes(buf, &s, 2); } } static bool write_wav16(std::vector & buf, const std::vector & pcm, int32_t rate, uint16_t ch = 1) { // RIFF chunk sizes are 32-bit; refuse to emit a file with a truncated header if (pcm.size() > ((size_t) UINT32_MAX - 36) / 2) { return false; } write_wav16_header(buf, (uint32_t) (pcm.size() * 2), rate, ch); append_wav16_pcm(buf, pcm.data(), pcm.size()); return true; } class mtmd_gen_audio_pipeline { public: mtmd_gen_audio_pipeline(llama_context * lctx, mtmd_context * mctx) : lctx(lctx), mctx(mctx), model(llama_get_model(lctx)), vocab(llama_model_get_vocab(model)), n_embd(llama_model_n_embd(model)), info(mtmd_gen_audio_get_info(mctx)) {} virtual ~mtmd_gen_audio_pipeline() = default; // false if the backbone context cannot run the pipeline, checked by mtmd_helper_gen_audio_init() virtual bool check_context() const { return true; } // see mtmd_helper_gen_audio_max_frames() virtual int32_t max_frames() const { return 512; } virtual void reset() = 0; virtual int32_t set_input(const mtmd_helper_gen_audio_inp * inp) = 0; // decodes at most n_batch prompt tokens; returns remaining count (0 = done), <0 on error virtual int32_t step_prompt(int32_t n_batch) = 0; // sampled can be LLAMA_TOKEN_NULL for pipelines with no discrete backbone token, // those read what they need from h_state_in instead // set out_stop on end-of-speech, h_state_out must be null if no frame is generated virtual int32_t step_gen(llama_token sampled, const float * h_state_in, const float ** h_state_out, bool * out_stop) = 0; virtual int32_t get_output(int32_t * out_sample_rate, const char ** out_data, size_t * out_data_len, int64_t * out_n_samples) = 0; // forces any buffered codes through code2wav now, regardless of window_frames virtual int32_t flush() = 0; // continuing a sequence (inp->n_past > 0) with the model's own text/audio switching virtual bool supports_continue() const { return false; } // mtmd_helper_gen_audio_inp::speed and loudness other than the defaults virtual bool supports_delivery() const { return false; } // speed alone, for models without loudness control virtual bool supports_speed() const { return supports_delivery(); } // mtmd_helper_gen_audio_inp::cfg_scale other than the default virtual bool supports_cfg() const { return false; } // mtmd_helper_gen_audio_inp::flow_steps other than the default virtual bool supports_flow_steps() const { return false; } virtual bool next_is_text() const { return false; } std::vector logit_bias; // see mtmd_helper_gen_audio_get_logit_bias() protected: // one state per generation, so a fixed seed always gives the same output bool init_gen_state(uint32_t seed) { mtmd_gen_state_params params = mtmd_gen_state_params_default(); params.seed = seed; gen_state.reset(mtmd_gen_state_init(mctx, params)); if (!gen_state) { LOG_ERR("mtmd_helper_gen_audio: failed to create the gen state\n"); return false; } return true; } // encodes a speaker clip or a voice, out receives the encoder embeddings bool encode_speaker(mtmd_bitmap * bitmap, std::vector & out) { if (!mtmd_bitmap_is_voice(bitmap) && !mtmd_support_audio(mctx)) { LOG_ERR("mtmd_helper_gen_audio: mmproj has no speaker/audio encoder\n"); return false; } const std::string marker = mtmd_get_marker(mctx); mtmd_input_text text{ marker.c_str(), marker.size(), false, true }; mtmd_input_chunks * chunks = mtmd_input_chunks_init(); const mtmd_bitmap * bptr = bitmap; bool ok = mtmd_tokenize(mctx, chunks, &text, &bptr, 1) == 0; if (ok) { ok = false; for (size_t i = 0; i < mtmd_input_chunks_size(chunks); i++) { const mtmd_input_chunk * chunk = mtmd_input_chunks_get(chunks, i); if (mtmd_input_chunk_get_type(chunk) != MTMD_INPUT_CHUNK_TYPE_AUDIO) { continue; } if (mtmd_encode_chunk(mctx, chunk) != 0) { LOG_ERR("mtmd_helper_gen_audio: speaker encode failed\n"); break; } const float * embd = mtmd_get_output_embd(mctx); const size_t n = (size_t) llama_model_n_embd_inp(model) * mtmd_input_chunk_get_n_tokens(chunk); out.assign(embd, embd + n); ok = true; break; } } mtmd_input_chunks_free(chunks); return ok; } llama_context * lctx; mtmd_context * mctx; const llama_model * model; const llama_vocab * vocab; int n_embd; mtmd_gen_audio_info info; mtmd::gen_state_ptr gen_state; }; // Qwen3-TTS: backbone samples codec_0, code_predictor gives the other 15 codebooks, // then code2wav decodes them to PCM class qwen3tts_gen_audio_pipeline : public mtmd_gen_audio_pipeline { public: using mtmd_gen_audio_pipeline::mtmd_gen_audio_pipeline; void reset() override { seq_id = 0; pos = 0; codes_buf.clear(); next_window_frames = window_frames; gen_state.reset(); audio_pcm.clear(); overlay.clear(); h_state_buf.clear(); out_buf.clear(); prompt_embd_buf.clear(); prompt_batch.reset(); n_prompt = 0; prompt_pos = 0; stream = false; pcm_sent = 0; wav_header_sent = false; } int32_t set_input(const mtmd_helper_gen_audio_inp * inp) override { reset(); seq_id = inp->seq_id; if (!ensure_cache()) { return 1; } const std::string lang = tts_resolve_lang((inp->lang && inp->lang[0]) ? inp->lang : "english"); const llama_token c_lang = find_special_token(vocab, ("<|codec_language_" + lang + "|>").c_str()); if (c_lang == LLAMA_TOKEN_NULL) { LOG_ERR("mtmd_helper_gen_audio: unknown language '%s'\n", lang.c_str()); return 1; } std::vector speaker_embd; if (inp->speaker_ref) { mtmd_bitmap_set_transcript(inp->speaker_ref, inp->speaker_ref_text); if (!encode_speaker(inp->speaker_ref, speaker_embd)) { return 1; } } const int n_e = n_embd; // cloning with the reference transcript continues the reference speech, see generate_icl_prompt() upstream const bool has_ref_text = inp->speaker_ref_text && inp->speaker_ref_text[0]; const size_t n_ref_frames = speaker_embd.empty() ? 0 : speaker_embd.size() / (size_t) n_e - 1; const bool icl = has_ref_text && n_ref_frames > 0; if (has_ref_text && !icl) { LOG_WRN("mtmd_helper_gen_audio: no reference codes (mmproj without reference encoder, or clip too long), cloning from the speaker embedding only\n"); } auto row = [&](llama_token t) { return std::vector(tok_embd.begin() + (size_t) t * n_e, tok_embd.begin() + (size_t) (t + 1) * n_e); }; auto sum_row = [&](llama_token a, llama_token b) { std::vector va = row(a), vb = row(b); for (int i = 0; i < n_e; i++) va[(size_t) i] += vb[(size_t) i]; return va; }; auto sum_vec = [&](llama_token a, const std::vector & vb) { std::vector va = row(a); for (int i = 0; i < n_e; i++) va[(size_t) i] += vb[(size_t) i]; return va; }; // upstream chat wrap, then slices: [0:3] role, [3:-5] utterance body const std::string full = "<|im_start|>assistant\n" + std::string(inp->prompt, inp->prompt_len) + "<|im_end|>\n<|im_start|>assistant\n"; auto tokenize = [&](const std::string & text) { std::vector tokens(text.size() + 16); const int n = llama_tokenize(vocab, text.c_str(), (int32_t) text.size(), tokens.data(), (int32_t) tokens.size(), false, true); tokens.resize(n < 0 ? 0 : (size_t) n); return tokens; }; std::vector ids = tokenize(full); const int n_ids = (int) ids.size(); if (n_ids < 8) { LOG_ERR("mtmd_helper_gen_audio: tokenization failed\n"); return 1; } // the reference transcript goes before the text, with the same chat wrap minus the next turn: slices [3:-2] std::vector ref_ids; if (icl) { ref_ids = tokenize("<|im_start|>assistant\n" + std::string(inp->speaker_ref_text) + "<|im_end|>\n"); if (ref_ids.size() < 6) { LOG_ERR("mtmd_helper_gen_audio: tokenization of the reference text failed\n"); return 1; } ref_ids = std::vector(ref_ids.begin() + 3, ref_ids.end() - 2); } std::vector> prompt; for (int i = 0; i < 3; i++) prompt.push_back(row(ids[(size_t) i])); prompt.push_back(sum_row(tts_pad, c_think)); prompt.push_back(sum_row(tts_pad, c_think_b)); prompt.push_back(sum_row(tts_pad, c_lang)); prompt.push_back(sum_row(tts_pad, c_think_e)); if (!speaker_embd.empty()) prompt.push_back(sum_vec(tts_pad, speaker_embd)); prompt.push_back(sum_row(tts_bos, codec_pad)); for (llama_token t : ref_ids) prompt.push_back(sum_row(t, codec_pad)); for (int i = 3; i < n_ids - 5; i++) prompt.push_back(sum_row(ids[(size_t) i], codec_pad)); prompt.push_back(sum_row(tts_eos, codec_pad)); prompt.push_back(sum_row(tts_pad, codec_bos)); // the reference frames, as if the talker had just spoken them if (icl) { for (size_t f = 0; f < n_ref_frames; f++) { const float * frame = speaker_embd.data() + (f + 1) * (size_t) n_e; prompt.push_back(sum_vec(tts_pad, std::vector(frame, frame + n_e))); } } n_prompt = (int) prompt.size(); // the talker uses the qwen3vl interleaved mrope, all sections are equal for a text/codec stream mrope = llama_model_rope_type(model) == LLAMA_ROPE_TYPE_MROPE || llama_model_rope_type(model) == LLAMA_ROPE_TYPE_IMROPE; const int n_pos_per_embd = mrope ? 4 : 1; prompt_embd_buf.resize((size_t) n_prompt * (size_t) n_e); for (int i = 0; i < n_prompt; i++) { memcpy(prompt_embd_buf.data() + (size_t) i * n_e, prompt[(size_t) i].data(), (size_t) n_e * sizeof(float)); } prompt_batch.reset(new decode_embd_batch(prompt_embd_buf.data(), n_prompt, n_pos_per_embd, n_e)); if (mrope) prompt_batch->set_position_mrope_1d(0, seq_id); else prompt_batch->set_position_normal (0, seq_id); prompt_pos = 0; pos = 0; const mtmd_gen_inp def = mtmd_gen_inp_default(mctx); top_k = inp->top_k > 0 ? inp->top_k : def.top_k; top_p = inp->top_p > 0 ? inp->top_p : def.top_p; out_type = inp->out_type; stream = inp->stream; if (!init_gen_state(inp->seed)) { return 1; } // the decoder continues from the reference codes of the encode above if (icl) { if (mtmd_gen_state_set_ref(mctx, gen_state.get()) != 0) { return 1; } next_window_frames = window_frames - std::min(n_ref_frames, window_frames - 1); } // the prompt above holds the whole text stream up to tts_eos, so every generated // frame adds tts_pad on top of the codes embedding overlay = row(tts_pad); return 0; } int32_t step_prompt(int32_t n_batch) override { GGML_ASSERT(n_batch > 0); if (prompt_pos >= n_prompt) { return 0; } const int32_t n_tokens_batch = std::min(n_batch, n_prompt - prompt_pos); const bool is_last_batch = (prompt_pos + n_tokens_batch) == n_prompt; if (is_last_batch) { prompt_batch->logits[prompt_pos + n_tokens_batch - 1] = 1; } if (llama_process(lctx, LLAMA_PROCESS_TYPE_DECODE, prompt_batch->render(lctx, prompt_pos, n_tokens_batch)) != 0) { LOG_ERR("mtmd_helper_gen_audio: prompt decode failed\n"); return -1; } pos += n_tokens_batch; prompt_pos += n_tokens_batch; if (prompt_pos >= n_prompt) { // prompt fully processed, its embedding buffer is no longer needed prompt_batch.reset(); prompt_embd_buf.clear(); return 0; } return n_prompt - prompt_pos; } int32_t step_gen(llama_token sampled, const float * h_state_in, const float ** h_state_out, bool * out_stop) override { if (sampled == LLAMA_TOKEN_NULL) { LOG_ERR("mtmd_helper_gen_audio: qwen3tts requires a token sampled from the backbone\n"); return 1; } // backbone signals end-of-speech with a token, no frame for this step if (sampled == codec_eos || llama_vocab_is_eog(vocab, sampled)) { *out_stop = true; *h_state_out = nullptr; return 0; } mtmd_gen_inp inp = mtmd_gen_inp_default(mctx); inp.type = MTMD_GEN_PROCESS_TYPE_GEN_CODE; inp.code0 = sampled - codec_0; inp.embd = const_cast(h_state_in); inp.top_k = top_k; inp.top_p = top_p; mtmd_gen_out out{}; if (mtmd_gen_audio_process(mctx, gen_state.get(), &inp, &out) != 0) { LOG_ERR("mtmd_helper_gen_audio: gen_code process failed\n"); return 1; } codes_buf.insert(codes_buf.end(), out.codes, out.codes + out.n_codes); if (out.n_codes > 0 && codes_buf.size() / out.n_codes >= next_window_frames) { if (!flush_gen_wav()) { return 1; } } std::vector fb(out.embd, out.embd + n_embd); for (int i = 0; i < n_embd; i++) fb[(size_t) i] += overlay[(size_t) i]; const int n_pos_per_embd = mrope ? 4 : 1; decode_embd_batch batch_embd(fb.data(), 1, n_pos_per_embd, n_embd); if (mrope) batch_embd.set_position_mrope_1d(pos, seq_id); else batch_embd.set_position_normal (pos, seq_id); batch_embd.logits[0] = 1; pos++; if (llama_process(lctx, LLAMA_PROCESS_TYPE_DECODE, batch_embd.render(lctx, 0, 1)) != 0) { LOG_ERR("mtmd_helper_gen_audio: decode failed\n"); return 1; } const float * he = llama_get_embeddings_ith(lctx, -1); h_state_buf.assign(he, he + n_embd); *h_state_out = h_state_buf.data(); return 0; } int32_t get_output(int32_t * out_sample_rate, const char ** out_data, size_t * out_data_len, int64_t * out_n_samples) override { *out_sample_rate = info.sample_rate; if (!stream) { // one-shot call: force out whatever's left, regardless of window_frames if (!flush_gen_wav()) { return 1; } if (out_n_samples) { *out_n_samples = (int64_t) audio_pcm.size(); } if (out_type == MTMD_HELPER_GEN_AUDIO_OUTTYPE_PCM) { *out_data = (const char *) audio_pcm.data(); *out_data_len = audio_pcm.size() * sizeof(float); return 0; } out_buf.clear(); if (!write_wav16(out_buf, audio_pcm, info.sample_rate)) { LOG_ERR("mtmd_helper_gen_audio: output too large for WAV\n"); return 1; } *out_data = out_buf.data(); *out_data_len = out_buf.size(); return 0; } // streaming: only return audio produced since the previous call const size_t n_new = audio_pcm.size() - pcm_sent; if (out_n_samples) { *out_n_samples = (int64_t) n_new; } if (out_type == MTMD_HELPER_GEN_AUDIO_OUTTYPE_PCM) { *out_data = (const char *) (audio_pcm.data() + pcm_sent); *out_data_len = n_new * sizeof(float); pcm_sent = audio_pcm.size(); return 0; } out_buf.clear(); if (!wav_header_sent) { write_wav16_header(out_buf, UINT32_MAX, info.sample_rate); wav_header_sent = true; } append_wav16_pcm(out_buf, audio_pcm.data() + pcm_sent, n_new); pcm_sent = audio_pcm.size(); *out_data = out_buf.data(); *out_data_len = out_buf.size(); return 0; } int32_t flush() override { return flush_gen_wav() ? 0 : 1; } private: bool ensure_cache() { if (specials_ok) { return true; } codec_0 = find_special_token(vocab, "<|codec_0|>"); codec_bos = find_special_token(vocab, "<|codec_bos|>"); codec_eos = find_special_token(vocab, "<|codec_eos_token|>"); codec_pad = find_special_token(vocab, "<|codec_pad|>"); c_think = find_special_token(vocab, "<|codec_think|>"); c_think_b = find_special_token(vocab, "<|codec_think_bos|>"); c_think_e = find_special_token(vocab, "<|codec_think_eos|>"); tts_pad = find_special_token(vocab, ""); tts_bos = find_special_token(vocab, ""); tts_eos = find_special_token(vocab, ""); for (llama_token t : { codec_0, codec_bos, codec_eos, codec_pad, c_think, c_think_b, c_think_e, tts_pad, tts_bos, tts_eos }) { if (t == LLAMA_TOKEN_NULL) { LOG_ERR("mtmd_helper_gen_audio: missing a required special token in vocab\n"); return false; } } const uint32_t n_tok_embd = llama_model_get_tok_embd(model, nullptr); if (n_tok_embd == 0) { LOG_ERR("mtmd_helper_gen_audio: model has no token embeddings\n"); return false; } tok_embd.resize(n_tok_embd); if (llama_model_get_tok_embd(model, tok_embd.data()) != n_tok_embd) { LOG_ERR("mtmd_helper_gen_audio: token embedding copy failed\n"); return false; } specials_ok = true; return true; } // one GEN_WAV process() call over the buffered codes, state is carried across batches bool flush_gen_wav() { if (codes_buf.empty()) { return true; } mtmd_gen_inp inp = mtmd_gen_inp_default(mctx); inp.type = MTMD_GEN_PROCESS_TYPE_GEN_WAV; inp.codes = codes_buf.data(); inp.n_codes = codes_buf.size(); mtmd_gen_out out{}; if (mtmd_gen_audio_process(mctx, gen_state.get(), &inp, &out) != 0) { LOG_ERR("mtmd_helper_gen_audio: gen_wav process failed\n"); return false; } audio_pcm.insert(audio_pcm.end(), out.audio, out.audio + out.n_samples); codes_buf.clear(); next_window_frames = window_frames; return true; } // vocab specials fixed across the whole session, looked up once bool specials_ok = false; llama_token codec_0 = LLAMA_TOKEN_NULL; llama_token codec_bos = LLAMA_TOKEN_NULL; llama_token codec_eos = LLAMA_TOKEN_NULL; llama_token codec_pad = LLAMA_TOKEN_NULL; llama_token c_think = LLAMA_TOKEN_NULL; llama_token c_think_b = LLAMA_TOKEN_NULL; llama_token c_think_e = LLAMA_TOKEN_NULL; llama_token tts_pad = LLAMA_TOKEN_NULL; llama_token tts_bos = LLAMA_TOKEN_NULL; llama_token tts_eos = LLAMA_TOKEN_NULL; std::vector tok_embd; // whole token embedding matrix, n_vocab * n_embd // must match hparams.wav_tfm_swa hardcoded in mtmd-model.cpp size_t window_frames = 72; // per-generation state, cleared by reset() llama_seq_id seq_id = 0; bool mrope = false; int pos = 0; // prompt decode state, consumed batch-by-batch by step_prompt() std::vector prompt_embd_buf; std::unique_ptr prompt_batch; int n_prompt = 0; int prompt_pos = 0; int32_t top_k = 50; float top_p = 1.0f; std::vector codes_buf; size_t next_window_frames = window_frames; // the first window leaves room for the reference codes std::vector audio_pcm; std::vector overlay; std::vector h_state_buf; mtmd_helper_gen_audio_outtype out_type = MTMD_HELPER_GEN_AUDIO_OUTTYPE_WAV; std::vector out_buf; bool stream = false; size_t pcm_sent = 0; // samples already returned by get_output() bool wav_header_sent = false; }; // settings that only live in the reference's per-pack yaml, not in the checkpoint // the english packs share the same shapes and tokenizer, but disagree on these // both are 0 / false when the pack does not tune them, the model default is then used struct pockettts_pack_settings { int frames_after_eos = 0; bool pad_short_text = false; }; static pockettts_pack_settings pockettts_pack(const char * variant) { static const std::unordered_map packs = { { "english", { 0, false } }, { "english_2026-01", { 0, true } }, { "english_2026-04", { 0, false } }, { "french_24l", { 8, false } }, { "italian", { 0, false } }, }; auto it = packs.find(variant ? variant : ""); if (it == packs.end()) { LOG_WRN("mtmd_helper_gen_audio: no tuned settings for pocket-tts variant \"%s\"\n", variant ? variant : ""); return {}; } return it->second; } // pocket-tts: the backbone emits no token, the flow net turns each hidden state into a latent // the end-of-speech head also lives in the mmproj class pockettts_gen_audio_pipeline : public mtmd_gen_audio_pipeline { public: using mtmd_gen_audio_pipeline::mtmd_gen_audio_pipeline; void reset() override { seq_id = 0; pos = 0; feats_buf.clear(); gen_state.reset(); audio_pcm.clear(); h_state_buf.clear(); out_buf.clear(); prompt_embd_buf.clear(); prompt_batch.reset(); n_prompt = 0; prompt_pos = 0; step_idx = 0; eos_step = -1; chunks.clear(); chunk_idx = 0; n_voice_pos = 0; chunk_budget = 0; stream = false; pcm_sent = 0; wav_header_sent = false; } int32_t set_input(const mtmd_helper_gen_audio_inp * inp) override { reset(); seq_id = inp->seq_id; if (!ensure_cache()) { return 1; } std::vector voice; if (inp->speaker_ref) { if (!encode_speaker(inp->speaker_ref, voice)) { return 1; } } pack = pockettts_pack(info.model_variant); const std::string text = prepare_text(std::string(inp->prompt, inp->prompt_len), pack.pad_short_text); if (text.empty()) { LOG_ERR("mtmd_helper_gen_audio: empty prompt\n"); return 1; } std::vector ids(text.size() + 16); int n_ids = llama_tokenize(vocab, text.c_str(), (int32_t) text.size(), ids.data(), (int32_t) ids.size(), false, false); if (n_ids <= 0) { LOG_ERR("mtmd_helper_gen_audio: tokenization failed\n"); return 1; } ids.resize((size_t) n_ids); // long inputs degrade badly, so each chunk restarts from the voice conditioning // see split_into_best_sentences() in the reference chunks = split_chunks(ids); chunk_idx = 0; if (chunks.size() > 1) { LOG_INF("mtmd_helper_gen_audio: %d tokens split into %zu chunks\n", n_ids, chunks.size()); } const int n_e = n_embd; // sequence order is voice, then text, then the audio BOS that starts generation if (!voice.empty()) { GGML_ASSERT(voice.size() % (size_t) n_e == 0); if (bos_before_voice != LLAMA_TOKEN_NULL) { push_embd_row(prompt_embd_buf, bos_before_voice); } prompt_embd_buf.insert(prompt_embd_buf.end(), voice.begin(), voice.end()); } // every later chunk rewinds to here and re-prompts, so the voice stays primed n_voice_pos = (int) (prompt_embd_buf.size() / (size_t) n_e); for (llama_token t : chunks[0]) { push_embd_row(prompt_embd_buf, t); } push_embd_row(prompt_embd_buf, audio_bos); arm_chunk_budget(0); n_prompt = (int) (prompt_embd_buf.size() / (size_t) n_e); prompt_batch.reset(new decode_embd_batch(prompt_embd_buf.data(), n_prompt, 1, n_e)); prompt_batch->set_position_normal(0, seq_id); prompt_pos = 0; temp = inp->temp; out_type = inp->out_type; stream = inp->stream; if (!init_gen_state(inp->seed)) { return 1; } return 0; } int32_t step_prompt(int32_t n_batch) override { GGML_ASSERT(n_batch > 0); if (prompt_pos >= n_prompt) { return 0; } const int32_t n_tokens_batch = std::min(n_batch, n_prompt - prompt_pos); if ((prompt_pos + n_tokens_batch) == n_prompt) { prompt_batch->logits[prompt_pos + n_tokens_batch - 1] = 1; } if (llama_process(lctx, LLAMA_PROCESS_TYPE_DECODE, prompt_batch->render(lctx, prompt_pos, n_tokens_batch)) != 0) { LOG_ERR("mtmd_helper_gen_audio: prompt decode failed\n"); return -1; } pos += n_tokens_batch; prompt_pos += n_tokens_batch; if (prompt_pos >= n_prompt) { prompt_batch.reset(); prompt_embd_buf.clear(); return 0; } return n_prompt - prompt_pos; } int32_t step_gen(llama_token sampled, const float * h_state_in, const float ** h_state_out, bool * out_stop) override { (void) sampled; // the backbone output is continuous, there is no token to consume mtmd_gen_inp inp = mtmd_gen_inp_default(mctx); inp.type = MTMD_GEN_PROCESS_TYPE_GEN_CODE; inp.embd = const_cast(h_state_in); if (temp >= 0.0f) { inp.temp = temp; } mtmd_gen_out out{}; if (mtmd_gen_audio_process(mctx, gen_state.get(), &inp, &out) != 0) { LOG_ERR("mtmd_helper_gen_audio: flow decode failed\n"); return 1; } if (out.is_eos && eos_step < 0) { eos_step = step_idx; } // the frame of the stopping step is discarded, matching _autoregressive_generation(). // the budget is the reference's fallback for a chunk whose eos head never fires const bool chunk_done = (eos_step >= 0 && step_idx >= eos_step + frames_after_eos) || step_idx >= chunk_budget; if (chunk_done) { if (eos_step < 0) { LOG_WRN("mtmd_helper_gen_audio: chunk %zu hit its budget without end-of-speech\n", chunk_idx); } return finish_chunk(h_state_out, out_stop); } feats_buf.insert(feats_buf.end(), out.feats, out.feats + out.n_feats); step_idx++; if (out.n_feats > 0 && feats_buf.size() / out.n_feats >= window_frames) { if (!flush_gen_wav()) { return 1; } } decode_embd_batch batch_embd(out.embd, 1, 1, n_embd); batch_embd.set_position_normal(pos, seq_id); batch_embd.logits[0] = 1; pos++; if (llama_process(lctx, LLAMA_PROCESS_TYPE_DECODE, batch_embd.render(lctx, 0, 1)) != 0) { LOG_ERR("mtmd_helper_gen_audio: decode failed\n"); return 1; } const float * he = llama_get_embeddings_ith(lctx, -1); h_state_buf.assign(he, he + n_embd); *h_state_out = h_state_buf.data(); return 0; } int32_t get_output(int32_t * out_sample_rate, const char ** out_data, size_t * out_data_len, int64_t * out_n_samples) override { *out_sample_rate = info.sample_rate; if (!stream) { // one-shot call: force out whatever's left, regardless of window_frames if (!flush_gen_wav()) { return 1; } if (out_n_samples) { *out_n_samples = (int64_t) audio_pcm.size(); } if (out_type == MTMD_HELPER_GEN_AUDIO_OUTTYPE_PCM) { *out_data = (const char *) audio_pcm.data(); *out_data_len = audio_pcm.size() * sizeof(float); return 0; } out_buf.clear(); if (!write_wav16(out_buf, audio_pcm, info.sample_rate)) { LOG_ERR("mtmd_helper_gen_audio: output too large for WAV\n"); return 1; } *out_data = out_buf.data(); *out_data_len = out_buf.size(); return 0; } // streaming: only return audio produced since the previous call const size_t n_new = audio_pcm.size() - pcm_sent; if (out_n_samples) { *out_n_samples = (int64_t) n_new; } if (out_type == MTMD_HELPER_GEN_AUDIO_OUTTYPE_PCM) { *out_data = (const char *) (audio_pcm.data() + pcm_sent); *out_data_len = n_new * sizeof(float); pcm_sent = audio_pcm.size(); return 0; } out_buf.clear(); if (!wav_header_sent) { write_wav16_header(out_buf, UINT32_MAX, info.sample_rate); wav_header_sent = true; } append_wav16_pcm(out_buf, audio_pcm.data() + pcm_sent, n_new); pcm_sent = audio_pcm.size(); *out_data = out_buf.data(); *out_data_len = out_buf.size(); return 0; } int32_t flush() override { return flush_gen_wav() ? 0 : 1; } private: bool ensure_cache() { if (specials_ok) { return true; } // bos_before_voice is optional, some packs do not insert it bos_before_voice = find_special_token(vocab, "<|bos_before_voice|>"); audio_bos = find_special_token(vocab, "<|audio_bos|>"); if (audio_bos == LLAMA_TOKEN_NULL) { LOG_ERR("mtmd_helper_gen_audio: missing <|audio_bos|> in vocab\n"); return false; } const uint32_t n_tok_embd = llama_model_get_tok_embd(model, nullptr); if (n_tok_embd == 0) { LOG_ERR("mtmd_helper_gen_audio: model has no token embeddings\n"); return false; } tok_embd.resize(n_tok_embd); if (llama_model_get_tok_embd(model, tok_embd.data()) != n_tok_embd) { LOG_ERR("mtmd_helper_gen_audio: token embedding copy failed\n"); return false; } GGML_ASSERT(n_embd > 0 && n_tok_embd % (uint32_t) n_embd == 0); specials_ok = true; return true; } // the table can be shorter than the vocab, so bound the row lookup void push_embd_row(std::vector & dst, llama_token t) const { const size_t n_rows = tok_embd.size() / (size_t) n_embd; GGML_ASSERT(t >= 0 && (size_t) t < n_rows); dst.insert(dst.end(), tok_embd.begin() + (size_t) t * n_embd, tok_embd.begin() + (size_t) (t + 1) * n_embd); } // token ids of the pieces the reference splits on, see split_into_best_sentences(). // the leading token is dropped, it is the tokenizer's dummy prefix std::vector punct_ids(const char * s) const { std::vector ids(16); const int n = llama_tokenize(vocab, s, (int32_t) strlen(s), ids.data(), (int32_t) ids.size(), false, false); if (n <= 1) { return {}; } return std::vector(ids.begin() + 1, ids.begin() + n); } // cut after runs of boundary tokens, so punctuation stays with the sentence it ends static std::vector> split_on(const std::vector & ids, const std::vector & boundary) { std::vector> out; size_t start = 0; bool prev_was_boundary = false; for (size_t i = 0; i < ids.size(); i++) { const bool is_boundary = std::find(boundary.begin(), boundary.end(), ids[i]) != boundary.end(); if (!is_boundary && prev_was_boundary) { out.emplace_back(ids.begin() + start, ids.begin() + i); start = i; } prev_was_boundary = is_boundary; } out.emplace_back(ids.begin() + start, ids.end()); return out; } std::vector> split_chunks(const std::vector & ids) const { if ((int) ids.size() <= max_chunk_tokens) { return { ids }; } const std::vector eos_punct = punct_ids(".!...?"); const std::vector mid_punct = punct_ids(",;:"); // oversized sentences are split again on weaker punctuation, else words get skipped std::vector> segments; for (auto & seg : split_on(ids, eos_punct)) { if ((int) seg.size() <= max_chunk_tokens) { segments.push_back(std::move(seg)); continue; } auto sub = split_on(seg, mid_punct); if (sub.size() > 1) { for (auto & s : sub) { segments.push_back(std::move(s)); } } else { segments.push_back(std::move(seg)); } } std::vector> out; for (auto & seg : segments) { if (seg.empty()) { continue; } if (!out.empty() && (int) (out.back().size() + seg.size()) <= max_chunk_tokens) { out.back().insert(out.back().end(), seg.begin(), seg.end()); } else { out.push_back(std::move(seg)); } } if (out.empty()) { out.push_back(ids); } for (const auto & c : out) { if ((int) c.size() > max_chunk_tokens) { LOG_WRN("mtmd_helper_gen_audio: chunk of %zu tokens exceeds the %d token budget, " "generation may skip words\n", c.size(), max_chunk_tokens); } } return out; } // _estimate_max_gen_len() plus the per-chunk tail guess, both in frames void arm_chunk_budget(size_t idx) { const int n_tok = (int) chunks[idx].size(); chunk_budget = (int) std::ceil((n_tok / 3.0 + 2.0) * frame_rate); // the pack may pin the tail, else the reference guesses it from the word count frames_after_eos = pack.frames_after_eos > 0 ? pack.frames_after_eos : (n_tok <= 6 ? 5 : 3); step_idx = 0; eos_step = -1; } // ends the current chunk and, if there is another, re-prompts it on top of the voice int32_t finish_chunk(const float ** h_state_out, bool * out_stop) { if (!flush_gen_wav()) { return 1; } // the decoder restarts too, the next chunk's audio is not continuous with this one mtmd_gen_state_reset(gen_state.get()); if (chunk_idx + 1 >= chunks.size()) { *out_stop = true; *h_state_out = nullptr; return 0; } chunk_idx++; // drop this chunk's text and audio, keep the voice conditioning llama_memory_seq_rm(llama_get_memory(lctx), seq_id, n_voice_pos, -1); pos = n_voice_pos; const int n_e = n_embd; prompt_embd_buf.clear(); for (llama_token t : chunks[chunk_idx]) { push_embd_row(prompt_embd_buf, t); } push_embd_row(prompt_embd_buf, audio_bos); arm_chunk_budget(chunk_idx); const int n_rows = (int) (prompt_embd_buf.size() / (size_t) n_e); GGML_ASSERT(n_rows > 0); decode_embd_batch batch(prompt_embd_buf.data(), n_rows, 1, n_e); batch.set_position_normal(pos, seq_id); batch.logits[n_rows - 1] = 1; if (llama_process(lctx, LLAMA_PROCESS_TYPE_DECODE, batch.render(lctx, 0, n_rows)) != 0) { LOG_ERR("mtmd_helper_gen_audio: chunk prompt decode failed\n"); return 1; } pos += n_rows; prompt_embd_buf.clear(); const float * he = llama_get_embeddings_ith(lctx, -1); h_state_buf.assign(he, he + n_embd); *h_state_out = h_state_buf.data(); *out_stop = false; return 0; } // same normalization as prepare_text_prompt() in the reference, it affects quality static std::string prepare_text(const std::string & in, bool pad_short) { std::string s; s.reserve(in.size() + 1); for (char c : in) { if (c == '\n' || c == '\r') { s += ' '; } else if (c == ';') { s += ','; } else { s += c; } } const size_t b = s.find_first_not_of(' '); const size_t e = s.find_last_not_of(' '); if (b == std::string::npos) { return ""; } s = s.substr(b, e - b + 1); if (s[0] >= 'a' && s[0] <= 'z') { s[0] = (char) (s[0] - 'a' + 'A'); } const unsigned char last = (unsigned char) s.back(); if (std::isalnum(last)) { s += '.'; } if (pad_short && count_words(s) < 5) { s = std::string(8, ' ') + s; } return s; } static int count_words(const std::string & s) { int n = 0; bool in_word = false; for (char c : s) { if (c == ' ') { in_word = false; } else if (!in_word) { in_word = true; n++; } } return n; } // decodes the buffered latents, the mimi decoder state carries over between calls bool flush_gen_wav() { if (feats_buf.empty()) { return true; } mtmd_gen_inp inp = mtmd_gen_inp_default(mctx); inp.type = MTMD_GEN_PROCESS_TYPE_GEN_WAV; inp.feats = feats_buf.data(); inp.n_feats = feats_buf.size(); mtmd_gen_out out{}; if (mtmd_gen_audio_process(mctx, gen_state.get(), &inp, &out) != 0) { LOG_ERR("mtmd_helper_gen_audio: mimi decode failed\n"); return false; } audio_pcm.insert(audio_pcm.end(), out.audio, out.audio + out.n_samples); feats_buf.clear(); return true; } pockettts_pack_settings pack; float temp = -1.0f; // the caller's flow noise scale, < 0 for the model default bool specials_ok = false; llama_token bos_before_voice = LLAMA_TOKEN_NULL; llama_token audio_bos = LLAMA_TOKEN_NULL; std::vector tok_embd; llama_seq_id seq_id = 0; int pos = 0; std::vector prompt_embd_buf; std::unique_ptr prompt_batch; int n_prompt = 0; int prompt_pos = 0; // end-of-speech is latched, then a few more frames are generated as tail padding int step_idx = 0; int eos_step = -1; int frames_after_eos = 3; static constexpr int max_chunk_tokens = 50; // MAX_TOKEN_PER_CHUNK in the reference static constexpr double frame_rate = 12.5; std::vector> chunks; size_t chunk_idx = 0; int n_voice_pos = 0; // KV positions held by the voice conditioning int chunk_budget = 0; // latents are decoded a window at a time, the decoder state bridges the windows size_t window_frames = 8; std::vector feats_buf; std::vector audio_pcm; std::vector h_state_buf; mtmd_helper_gen_audio_outtype out_type = MTMD_HELPER_GEN_AUDIO_OUTTYPE_WAV; std::vector out_buf; bool stream = false; size_t pcm_sent = 0; // samples already returned by get_output() bool wav_header_sent = false; }; // Kokoro is not autoregressive: the backbone encodes the phonemes once, the mmproj turns its states into the whole utterance // the prompt is a phoneme string (the output of the "tts" chat template), one token per character class kokoro_gen_audio_pipeline : public mtmd_gen_audio_pipeline { public: using mtmd_gen_audio_pipeline::mtmd_gen_audio_pipeline; // the predicted durations are divided by the speed bool supports_speed() const override { return true; } void reset() override { segments.clear(); audio.clear(); wav.clear(); n_encoded = 0; decoded = false; sent = false; } int32_t set_input(const mtmd_helper_gen_audio_inp * inp) override { reset(); if (!inp->prompt || inp->prompt_len == 0 || inp->seq_id < 0 || (uint32_t) inp->seq_id >= llama_n_seq_max(lctx)) { LOG_ERR("mtmd_helper_gen_audio: kokoro requires a prompt\n"); return 1; } if (inp->speaker_ref && !mtmd_bitmap_is_voice(inp->speaker_ref)) { LOG_ERR("mtmd_helper_gen_audio: kokoro has no voice cloning, use one of its voices\n"); return 1; } seq_id = inp->seq_id; out_type = inp->out_type; stream = inp->stream; speed = inp->speed; // one token per phoneme character, unknown characters are dropped like in the reference if (token_of.empty()) { for (llama_token id = 0; id < llama_vocab_n_tokens(vocab); id++) { if (llama_vocab_get_attr(vocab, id) & LLAMA_TOKEN_ATTR_NORMAL) { token_of[llama_vocab_get_text(vocab, id)] = id; } } } std::vector phonemes; const std::string prompt(inp->prompt, inp->prompt_len); for (size_t i = 0; i < prompt.size();) { const unsigned char c = (unsigned char) prompt[i]; const size_t n = c < 0x80 ? 1 : c < 0xE0 ? 2 : c < 0xF0 ? 3 : 4; const auto it = token_of.find(prompt.substr(i, n)); if (it != token_of.end()) { phonemes.push_back(it->second); } i += n; } if (phonemes.empty()) { LOG_ERR("mtmd_helper_gen_audio: the prompt has no phoneme the model knows\n"); return 1; } // a voice is a table of style rows, the row is picked by the number of phonemes const char * voice = inp->speaker_ref ? mtmd_bitmap_get_id(inp->speaker_ref) : mtmd_gen_audio_voice_name(mctx, 0); if (!voice) { LOG_ERR("mtmd_helper_gen_audio: the model has no voice\n"); return 1; } const std::string name = std::string("voice.") + voice; std::vector table(mtmd_gen_audio_read_tensor(mctx, name.c_str(), nullptr, 0)); if (table.size() < n_voice_row || table.size() % n_voice_row != 0 || mtmd_gen_audio_read_tensor(mctx, name.c_str(), table.data(), table.size()) != table.size()) { LOG_ERR("mtmd_helper_gen_audio: unknown voice \"%s\"\n", voice); return 1; } // the backbone takes a whole segment in one batch, padded on both ends const size_t n_max = std::min({(size_t) llama_n_ctx_seq(lctx), (size_t) llama_n_ubatch(lctx), (size_t) llama_model_n_ctx_train(model)}); if (n_max < 3) { LOG_ERR("mtmd_helper_gen_audio: the context is too small\n"); return 1; } const llama_token pad = llama_vocab_bos(vocab); for (size_t pos = 0; pos < phonemes.size();) { const size_t len = split_len(phonemes, pos, n_max - 2); segment seg; seg.tokens.push_back(pad); seg.tokens.insert(seg.tokens.end(), phonemes.begin() + pos, phonemes.begin() + pos + len); seg.tokens.push_back(pad); const size_t row = std::min(len - 1, table.size() / n_voice_row - 1); seg.feats.assign(table.begin() + row * n_voice_row, table.begin() + (row + 1) * n_voice_row); segments.push_back(std::move(seg)); pos += len; // a segment does not start with a space while (pos < phonemes.size() && is_any(phonemes[pos], " ")) { pos++; } } if (!init_gen_state(inp->seed)) { return 1; } return 0; } // the encoder is not causal, it takes one whole segment per call int32_t step_prompt(int32_t) override { if (segments.empty()) { return -1; } if (n_encoded == segments.size()) { return 0; } auto & seg = segments[n_encoded]; const int n = (int) seg.tokens.size(); llama_batch_ext_ptr batch(llama_batch_ext_init(lctx)); if (!batch_set_tokens(batch.get(), seg.tokens.data(), n, 0, seq_id)) { return -1; } for (int i = 0; i < n; ++i) { llama_batch_ext_set_output_embd(batch.get(), i, true); } if (llama_process(lctx, LLAMA_PROCESS_TYPE_DECODE, batch.get()) != 0) { LOG_ERR("mtmd_helper_gen_audio: kokoro backbone failed\n"); return -1; } for (int i = 0; i < n; ++i) { const float * h = llama_get_embeddings_ith(lctx, i); if (!h) { return -1; } seg.feats.insert(seg.feats.end(), h, h + n_embd); } n_encoded++; return (int32_t) (segments.size() - n_encoded); } // nothing is generated step by step int32_t step_gen(llama_token, const float *, const float ** h_state_out, bool * out_stop) override { *h_state_out = nullptr; *out_stop = true; return !segments.empty() && n_encoded == segments.size() ? 0 : 1; } int32_t get_output(int32_t * sample_rate, const char ** data, size_t * data_len, int64_t * n_samples) override { *sample_rate = info.sample_rate; *data_len = 0; if (n_samples) { *n_samples = 0; } if (stream) { // the audio comes out in one chunk after flush() if (!decoded || sent) { return 0; } sent = true; } else if (!decode_audio()) { return 1; } if (n_samples) { *n_samples = audio.size(); } if (out_type == MTMD_HELPER_GEN_AUDIO_OUTTYPE_PCM) { *data = (const char *) audio.data(); *data_len = audio.size() * sizeof(float); } else { wav.clear(); if (!write_wav16(wav, audio, info.sample_rate)) { return 1; } *data = wav.data(); *data_len = wav.size(); } return 0; } int32_t flush() override { return decode_audio() ? 0 : 1; } private: struct segment { std::vector tokens; std::vector feats; // the voice row, then the backbone state of each token }; bool is_any(llama_token id, const char * chars) const { const char * text = llama_vocab_get_text(vocab, id); return text[0] != '\0' && strstr(chars, text) != nullptr; } // length of the segment that starts at pos: a long text is cut at the last sentence end that fits, else at a weaker break size_t split_len(const std::vector & phonemes, size_t pos, size_t n_max) const { if (phonemes.size() - pos <= n_max) { return phonemes.size() - pos; } for (const char * breaks : {"!.?\xe2\x80\xa6", ":;", ",\xe2\x80\x94", " "}) { for (size_t len = n_max; len > 1; len--) { if (is_any(phonemes[pos + len - 1], breaks)) { return len; } } } return n_max; } bool decode_audio() { if (decoded) { return true; } if (segments.empty() || n_encoded != segments.size()) { return false; } for (const auto & seg : segments) { std::vector codes(seg.tokens.begin(), seg.tokens.end()); mtmd_gen_inp inp = mtmd_gen_inp_default(mctx); inp.type = MTMD_GEN_PROCESS_TYPE_GEN_WAV; inp.codes = codes.data(); inp.n_codes = codes.size(); inp.feats = seg.feats.data(); inp.n_feats = seg.feats.size(); inp.speed = speed; mtmd_gen_out out{}; if (mtmd_gen_audio_process(mctx, gen_state.get(), &inp, &out) != 0) { return false; } audio.insert(audio.end(), out.audio, out.audio + out.n_samples); } decoded = true; return true; } static constexpr size_t n_voice_row = 256; // decoder style, then predictor style llama_seq_id seq_id = 0; std::unordered_map token_of; std::vector segments; size_t n_encoded = 0; float speed = 1.0f; std::vector audio; std::vector wav; mtmd_helper_gen_audio_outtype out_type = MTMD_HELPER_GEN_AUDIO_OUTTYPE_WAV; bool stream = false; bool decoded = false; bool sent = false; }; // tiny-voca is not autoregressive: the backbone encodes the phonemes of one sentence, the mmproj turns its states into audio // the prompt is a phoneme string, symbols separated by spaces, a stress mark (ˈ or ˌ) before a stressed vowel // each sentence is one utterance, like the reference pipeline: the "tts" template ends each one with // a prompt without (phonemes given by the user) is cut after each sentence mark, and a missing final mark is added class tiny_voca_gen_audio_pipeline : public mtmd_gen_audio_pipeline { public: using mtmd_gen_audio_pipeline::mtmd_gen_audio_pipeline; // the predicted durations are divided by the speed bool supports_speed() const override { return true; } void reset() override { segments.clear(); audio.clear(); wav.clear(); n_encoded = 0; decoded = false; sent = false; } int32_t set_input(const mtmd_helper_gen_audio_inp * inp) override { reset(); if (!inp->prompt || inp->prompt_len == 0 || inp->seq_id < 0 || (uint32_t) inp->seq_id >= llama_n_seq_max(lctx)) { LOG_ERR("mtmd_helper_gen_audio: tiny-voca requires a prompt\n"); return 1; } if (inp->speaker_ref && !mtmd_bitmap_is_voice(inp->speaker_ref)) { LOG_ERR("mtmd_helper_gen_audio: tiny-voca has no voice cloning, use one of its voices\n"); return 1; } seq_id = inp->seq_id; out_type = inp->out_type; stream = inp->stream; speed = inp->speed; if (!load_tables()) { return 1; } const char * voice = inp->speaker_ref ? mtmd_bitmap_get_id(inp->speaker_ref) : mtmd_gen_audio_voice_name(mctx, 0); std::vector voice_row(2 * (size_t) n_embd); const std::string name = std::string("voice.") + (voice ? voice : ""); if (!voice || mtmd_gen_audio_read_tensor(mctx, name.c_str(), voice_row.data(), voice_row.size()) != voice_row.size()) { LOG_ERR("mtmd_helper_gen_audio: unknown voice \"%s\"\n", voice ? voice : ""); return 1; } // symbols -> (token, stress); the begin and end tokens are added per sentence std::vector> phones; explicit_ends = false; const std::string prompt(inp->prompt, inp->prompt_len); size_t n_unknown = 0; for (size_t i = 0; i < prompt.size();) { while (i < prompt.size() && isspace((unsigned char) prompt[i])) { i++; } size_t j = i; while (j < prompt.size() && !isspace((unsigned char) prompt[j])) { j++; } if (j == i) { break; } std::string sym = prompt.substr(i, j - i); i = j; int32_t stress = 0; if (sym.rfind("\xcb\x88", 0) == 0) { // ˈ primary stress = 1; } else if (sym.rfind("\xcb\x8c", 0) == 0) { // ˌ secondary stress = 2; } if (stress > 0) { sym = sym.substr(2); } if (sym == "") { // the end of a sentence, as the text was split by the template explicit_ends = true; phones.push_back({ tok_eos, 0 }); continue; } if (sym == "" || sym == "") { continue; } auto it = token_of.find(sym); if (it == token_of.end()) { n_unknown++; phones.push_back({ tok_unk, 0 }); continue; } phones.push_back({ it->second, stress }); } if (n_unknown > 0) { LOG_WRN("mtmd_helper_gen_audio: %zu symbols of the prompt are not phonemes of the model, they are read as \n", n_unknown); } // cut after each sentence end, then cut long sentences at weaker breaks const size_t n_max = std::min({ (size_t) llama_n_ctx_seq(lctx), (size_t) llama_n_ubatch(lctx), (size_t) llama_model_n_ctx_train(model) }); if (n_max < 8) { LOG_ERR("mtmd_helper_gen_audio: the context is too small\n"); return 1; } for (size_t pos = 0; pos < phones.size();) { while (pos < phones.size() && (phones[pos].first == tok_sp || phones[pos].first == tok_eos)) { pos++; } if (pos == phones.size()) { break; } const size_t len = split_len(phones, pos, n_max - 3); add_segment(phones, pos, len, voice_row); pos += len; } if (segments.empty()) { LOG_ERR("mtmd_helper_gen_audio: the prompt has no phoneme\n"); return 1; } if (!init_gen_state(inp->seed)) { return 1; } return 0; } // the encoder is not causal, it takes one whole sentence per call int32_t step_prompt(int32_t) override { if (segments.empty()) { return -1; } if (n_encoded == segments.size()) { return 0; } auto & seg = segments[n_encoded]; const int n = (int) seg.tokens.size(); decode_embd_batch batch(seg.embd.data(), n, 1, n_embd); batch.set_position_normal(0, seq_id); std::fill(batch.logits.begin(), batch.logits.end(), 1); if (llama_process(lctx, LLAMA_PROCESS_TYPE_DECODE, batch.render(lctx, 0, n)) != 0) { LOG_ERR("mtmd_helper_gen_audio: tiny-voca encoder failed\n"); return -1; } for (int i = 0; i < n; ++i) { const float * h = llama_get_embeddings_ith(lctx, i); if (!h) { return -1; } seg.feats.insert(seg.feats.end(), h, h + n_embd); } seg.embd.clear(); n_encoded++; return (int32_t) (segments.size() - n_encoded); } // nothing is generated step by step int32_t step_gen(llama_token, const float *, const float ** h_state_out, bool * out_stop) override { *h_state_out = nullptr; *out_stop = true; return !segments.empty() && n_encoded == segments.size() ? 0 : 1; } int32_t get_output(int32_t * sample_rate, const char ** data, size_t * data_len, int64_t * n_samples) override { *sample_rate = info.sample_rate; *data_len = 0; if (n_samples) { *n_samples = 0; } if (stream) { // the audio comes out in one chunk after flush() if (!decoded || sent) { return 0; } sent = true; } else if (!decode_audio()) { return 1; } if (n_samples) { *n_samples = audio.size(); } if (out_type == MTMD_HELPER_GEN_AUDIO_OUTTYPE_PCM) { *data = (const char *) audio.data(); *data_len = audio.size() * sizeof(float); } else { wav.clear(); if (!write_wav16(wav, audio, info.sample_rate)) { return 1; } *data = wav.data(); *data_len = wav.size(); } return 0; } int32_t flush() override { return decode_audio() ? 0 : 1; } private: struct segment { std::vector tokens; std::vector embd; // encoder input rows std::vector feats; // the voice row, then the encoder state of each token }; bool load_tables() { if (!token_of.empty()) { return true; } const llama_token n_vocab = llama_vocab_n_tokens(vocab); for (llama_token id = 0; id < n_vocab; id++) { token_of[llama_vocab_get_text(vocab, id)] = id; } auto find = [&](const char * text) { auto it = token_of.find(text); return it == token_of.end() ? LLAMA_TOKEN_NULL : it->second; }; tok_bos = find(""); tok_eos = find(""); tok_sp = find(""); tok_unk = find(""); tok_dot = find("."); tok_emb.resize((size_t) n_vocab * n_embd); str_emb.resize(3 * (size_t) n_embd); if (tok_bos == LLAMA_TOKEN_NULL || tok_eos == LLAMA_TOKEN_NULL || tok_sp == LLAMA_TOKEN_NULL || tok_unk == LLAMA_TOKEN_NULL || tok_dot == LLAMA_TOKEN_NULL || mtmd_gen_audio_read_tensor(mctx, "a.gen.tok_embd.weight", tok_emb.data(), tok_emb.size()) != tok_emb.size() || mtmd_gen_audio_read_tensor(mctx, "a.gen.stress_embd.weight", str_emb.data(), str_emb.size()) != str_emb.size()) { LOG_ERR("mtmd_helper_gen_audio: the backbone and the mmproj are not a tiny-voca pair\n"); token_of.clear(); return false; } return true; } bool is_any(llama_token id, const char * chars) const { const char * text = llama_vocab_get_text(vocab, id); return text[0] != '\0' && strstr(chars, text) != nullptr; } // phones of the sentence that starts at pos, cut after a weaker break if it is longer than n_max size_t split_len(const std::vector> & phones, size_t pos, size_t n_max) const { size_t n_left = phones.size() - pos; if (explicit_ends) { // up to the next for (size_t len = 0; len < n_left; len++) { if (phones[pos + len].first == tok_eos) { n_left = len; break; } } } else { for (size_t len = 1; len <= std::min(n_left, n_max); len++) { if (is_any(phones[pos + len - 1].first, "!.?\xe2\x80\xa6")) { return len; } } } if (n_left <= n_max) { return n_left; } for (const char * breaks : {":;", ",\xe2\x80\x94"}) { for (size_t len = n_max; len > 1; len--) { if (is_any(phones[pos + len - 1].first, breaks)) { return len; } } } for (size_t len = n_max; len > 1; len--) { if (phones[pos + len - 1].first == tok_sp) { return len; } } return n_max; } // phones ; without a final mark the last word breaks off, so a full stop is added // (not when the template has split the text: it adds the stop where the reference does) void add_segment(const std::vector> & phones, size_t pos, size_t len, const std::vector & voice_row) { segment seg; std::vector stress; seg.tokens.push_back(tok_bos); stress.push_back(0); size_t end = pos + len; while (end > pos && phones[end - 1].first == tok_sp) { end--; } for (size_t i = pos; i < end; i++) { seg.tokens.push_back(phones[i].first); stress.push_back(phones[i].second); } const char * last = llama_vocab_get_text(vocab, seg.tokens.back()); if (!explicit_ends && (!is_any(seg.tokens.back(), "!.?\xe2\x80\xa6,;:") || last[0] == '\0')) { seg.tokens.push_back(tok_dot); stress.push_back(0); } seg.tokens.push_back(tok_eos); stress.push_back(0); seg.embd.resize(seg.tokens.size() * n_embd); for (size_t i = 0; i < seg.tokens.size(); i++) { const float * t = tok_emb.data() + (size_t) seg.tokens[i] * n_embd; const float * s = str_emb.data() + (size_t) stress[i] * n_embd; for (int c = 0; c < n_embd; c++) { seg.embd[i * n_embd + c] = t[c] + s[c] + voice_row[c]; } } seg.feats = voice_row; segments.push_back(std::move(seg)); } bool decode_audio() { if (decoded) { return true; } if (segments.empty() || n_encoded != segments.size()) { return false; } for (const auto & seg : segments) { std::vector codes(seg.tokens.begin(), seg.tokens.end()); mtmd_gen_inp inp = mtmd_gen_inp_default(mctx); inp.type = MTMD_GEN_PROCESS_TYPE_GEN_WAV; inp.codes = codes.data(); inp.n_codes = codes.size(); inp.feats = seg.feats.data(); inp.n_feats = seg.feats.size(); inp.speed = speed; mtmd_gen_out out{}; if (mtmd_gen_audio_process(mctx, gen_state.get(), &inp, &out) != 0) { return false; } audio.insert(audio.end(), out.audio, out.audio + out.n_samples); } // the model can peak above 1.0, the reference scales the whole text down to a peak of 0.99 float peak = 0.0f; for (float v : audio) { peak = std::max(peak, std::fabs(v)); } if (peak > 0.99f) { for (float & v : audio) { v *= 0.99f / peak; } } decoded = true; return true; } llama_seq_id seq_id = 0; std::unordered_map token_of; llama_token tok_bos = LLAMA_TOKEN_NULL; llama_token tok_eos = LLAMA_TOKEN_NULL; llama_token tok_sp = LLAMA_TOKEN_NULL; llama_token tok_unk = LLAMA_TOKEN_NULL; llama_token tok_dot = LLAMA_TOKEN_NULL; bool explicit_ends = false; // the prompt marks the end of each sentence with std::vector tok_emb; // [n_vocab, n_embd] std::vector str_emb; // [3, n_embd] std::vector segments; size_t n_encoded = 0; float speed = 1.0f; std::vector audio; std::vector wav; mtmd_helper_gen_audio_outtype out_type = MTMD_HELPER_GEN_AUDIO_OUTTYPE_WAV; bool stream = false; bool decoded = false; bool sent = false; }; // Soprano uses the hidden state that predicts each non-EOS token, including the prompt's last state. class soprano_gen_audio_pipeline : public mtmd_gen_audio_pipeline { public: using mtmd_gen_audio_pipeline::mtmd_gen_audio_pipeline; void reset() override { prompt.clear(); features.clear(); h_next.clear(); audio.clear(); wav.clear(); pos = 0; prompt_pos = 0; decoded = false; sent = false; } int32_t set_input(const mtmd_helper_gen_audio_inp * inp) override { reset(); if (inp->speaker_ref || !inp->prompt || inp->prompt_len == 0 || inp->prompt_len > INT32_MAX || inp->seq_id < 0 || (uint32_t) inp->seq_id >= llama_n_seq_max(lctx)) { LOG_ERR("mtmd_helper_gen_audio: soprano requires text, without a speaker reference\n"); return 1; } seq_id = inp->seq_id; out_type = inp->out_type; stream = inp->stream; const llama_token text = find_special_token(vocab, "[TEXT]"); const llama_token start = find_special_token(vocab, "[START]"); const llama_token stop = find_special_token(vocab, "[STOP]"); if (text == LLAMA_TOKEN_NULL || start == LLAMA_TOKEN_NULL || stop == LLAMA_TOKEN_NULL) { LOG_ERR("mtmd_helper_gen_audio: incompatible soprano vocabulary\n"); return 1; } const int required = llama_tokenize(vocab, inp->prompt, inp->prompt_len, nullptr, 0, false, false); if (required >= 0 || (uint32_t) (-required + 3) >= llama_n_ctx_seq(lctx)) { LOG_ERR("mtmd_helper_gen_audio: soprano prompt does not fit in the context\n"); return 1; } const int n = -required; prompt.resize(n + 3); prompt[0] = stop; prompt[1] = text; if (llama_tokenize(vocab, inp->prompt, inp->prompt_len, prompt.data() + 2, n, false, false) != n) { return 1; } prompt.back() = start; if (!llama_memory_seq_rm(llama_get_memory(lctx), seq_id, 0, -1)) { LOG_ERR("mtmd_helper_gen_audio: cannot reset soprano sequence\n"); return 1; } if (!init_gen_state(inp->seed)) { return 1; } return 0; } int32_t step_prompt(int32_t n_batch) override { if (n_batch <= 0 || prompt.empty()) { return -1; } const int n = std::min(n_batch, (int) prompt.size() - prompt_pos); if (n == 0) { return 0; } if (!decode(prompt.data() + prompt_pos, n)) { return -1; } prompt_pos += n; return (int) prompt.size() - prompt_pos; } int32_t step_gen(llama_token sampled, const float * h_state_in, const float ** h_state_out, bool * out_stop) override { *h_state_out = nullptr; *out_stop = false; if (sampled < 0 || sampled >= llama_vocab_n_tokens(vocab) || !h_state_in || prompt_pos != (int) prompt.size() || prompt.empty()) { return 1; } if (llama_vocab_is_eog(vocab, sampled) || (uint32_t) pos >= llama_n_ctx_seq(lctx)) { *out_stop = true; return 0; } features.insert(features.end(), h_state_in, h_state_in + n_embd); if (!decode(&sampled, 1)) { features.resize(features.size() - n_embd); return 1; } const float * h = llama_get_embeddings_ith(lctx, -1); if (!h) { return 1; } h_next.assign(h, h + n_embd); *h_state_out = h_next.data(); return 0; } int32_t get_output(int32_t * sample_rate, const char ** data, size_t * data_len, int64_t * n_samples) override { *sample_rate = info.sample_rate; *data_len = 0; if (n_samples) { *n_samples = 0; } if (stream) { // the decoder needs the whole utterance, so the audio comes out in one chunk after flush() if (!decoded || sent) { return 0; } sent = true; } else if (!decode_audio()) { return 1; } if (n_samples) { *n_samples = audio.size(); } if (out_type == MTMD_HELPER_GEN_AUDIO_OUTTYPE_PCM) { *data = (const char *) audio.data(); *data_len = audio.size() * sizeof(float); } else { wav.clear(); if (!write_wav16(wav, audio, info.sample_rate)) { return 1; } *data = wav.data(); *data_len = wav.size(); } return 0; } // the whole utterance is decoded at once int32_t flush() override { return decode_audio() ? 0 : 1; } private: bool decode_audio() { if (decoded) { return true; } if (features.size() < 2 * (size_t) n_embd) { LOG_ERR("mtmd_helper_gen_audio: soprano needs at least two generated frames\n"); return false; } mtmd_gen_inp inp = mtmd_gen_inp_default(mctx); inp.type = MTMD_GEN_PROCESS_TYPE_GEN_WAV; inp.feats = features.data(); inp.n_feats = features.size(); mtmd_gen_out out{}; if (mtmd_gen_audio_process(mctx, gen_state.get(), &inp, &out) != 0) { return false; } audio.assign(out.audio, out.audio + out.n_samples); decoded = true; return true; } bool decode(llama_token * tokens, int n) { llama_batch_ext_ptr batch(llama_batch_ext_init(lctx)); if (!batch_set_tokens(batch.get(), tokens, n, pos, seq_id)) { return false; } llama_batch_ext_set_output_logits(batch.get(), n - 1, true); if (llama_process(lctx, LLAMA_PROCESS_TYPE_DECODE, batch.get()) != 0) { LOG_ERR("mtmd_helper_gen_audio: soprano backbone decode failed\n"); return false; } pos += n; return true; } llama_seq_id seq_id = 0; llama_pos pos = 0; int prompt_pos = 0; std::vector prompt; std::vector features; std::vector h_next; std::vector audio; std::vector wav; mtmd_helper_gen_audio_outtype out_type = MTMD_HELPER_GEN_AUDIO_OUTTYPE_WAV; bool stream = false; bool decoded = false; bool sent = false; }; // LFM3-Audio TTS, see generate() in the reference: each step samples a frame of codes by MaskGIT // and feeds its embedding back, the end-of-audio code stops class lfm3a_gen_audio_pipeline : public mtmd_gen_audio_pipeline { public: using mtmd_gen_audio_pipeline::mtmd_gen_audio_pipeline; void reset() override { seq_id = 0; pos = 0; continuing = false; mode = MODE_AUDIO; slots_left = 0; speaking = false; text_done = false; prompt_tokens.clear(); prompt_pos = 0; hist.clear(); gen_state.reset(); pending_tail = false; pcm_sent = 0; wav_header_sent = false; audio_pcm.clear(); h_state_buf.clear(); out_buf.clear(); } int32_t set_input(const mtmd_helper_gen_audio_inp * inp) override { reset(); seq_id = inp->seq_id; // a voice is named in the prompt instead if (inp->speaker_ref && !mtmd_bitmap_is_voice(inp->speaker_ref)) { LOG_WRN("mtmd_helper_gen_audio: the model has no voice conditioning, the speaker file is ignored\n"); } // the prompt is rendered by the caller: the "tts" chat template of the model, or the next turn of a conversation continuing = inp->n_past > 0; pos = continuing ? inp->n_past : 0; const std::string full(inp->prompt, inp->prompt_len); prompt_tokens.resize(full.size() + 16); const int n_ids = llama_tokenize(vocab, full.c_str(), (int32_t) full.size(), prompt_tokens.data(), (int32_t) prompt_tokens.size(), false, true); if (n_ids <= 0) { LOG_ERR("mtmd_helper_gen_audio: tokenization failed\n"); return 1; } prompt_tokens.resize((size_t) n_ids); if (!find_tokens()) { return 1; } // a continued response opens with the token of the asked mode, a text-only one cannot switch to audio if (continuing) { switch (inp->out_mode) { case MTMD_HELPER_GEN_AUDIO_OUT_AUDIO: prompt_tokens.push_back(tok_audio_start); break; case MTMD_HELPER_GEN_AUDIO_OUT_MIXED: prompt_tokens.push_back(tok_mixed_start); break; case MTMD_HELPER_GEN_AUDIO_OUT_TEXT: logit_bias = { { tok_audio_start, -INFINITY }, { tok_mixed_start, -INFINITY } }; break; default: break; } } // the last prompt token picks the mode, see generate() of the reference const llama_token last = prompt_tokens.back(); mode = last == tok_audio_start ? MODE_AUDIO : last == tok_mixed_start ? MODE_MIXED : MODE_TEXT; slots_left = n_text_block; if (!continuing && mode != MODE_AUDIO) { LOG_ERR("mtmd_helper_gen_audio: the prompt must come from the \"tts\" chat template of the model\n"); return 1; } // the sampling defaults of the model, the caller can only override the temperature const mtmd_gen_inp def = mtmd_gen_inp_default(mctx); temp = inp->temp >= 0.0f ? inp->temp : def.temp; top_k = def.top_k; rep_penalty = def.penalty_repeat; rep_window = def.penalty_last_n; out_type = inp->out_type; stream = inp->stream; rng.seed(inp->seed == UINT32_MAX ? std::random_device{}() : inp->seed); return init_gen_state(inp->seed) ? 0 : 1; } int32_t step_prompt(int32_t n_batch) override { GGML_ASSERT(n_batch > 0); const int n_prompt = (int) prompt_tokens.size(); if (prompt_pos >= n_prompt) { return 0; } const int n = std::min(n_batch, n_prompt - prompt_pos); llama_batch_ext_ptr batch(llama_batch_ext_init(lctx)); if (!batch_set_tokens(batch.get(), prompt_tokens.data() + prompt_pos, n, pos, seq_id)) { return -1; } if (prompt_pos + n == n_prompt) { llama_batch_ext_set_output_logits(batch.get(), n - 1, true); } if (llama_process(lctx, LLAMA_PROCESS_TYPE_DECODE, batch.get()) != 0) { LOG_ERR("mtmd_helper_gen_audio: prompt decode failed\n"); return -1; } pos += n; prompt_pos += n; return n_prompt - prompt_pos; } bool supports_continue() const override { return true; } bool next_is_text() const override { return mode == MODE_TEXT || (mode == MODE_MIXED && !speaking); } // one generate() step: a text token from the caller or an audio frame from the hidden state int32_t step_gen(llama_token sampled, const float * h_state_in, const float ** h_state_out, bool * out_stop) override { const bool interleaving = mode == MODE_MIXED; if (next_is_text()) { // <|im_end|> ends the response in any text slot, <|text_end|> outside an interleaved block too if (sampled == tok_im_end || (!interleaving && sampled == tok_text_end)) { *out_stop = true; *h_state_out = nullptr; return 0; } // the switching tokens are fed back like any text token if (!interleaving) { if (sampled == tok_audio_start) { mode = MODE_AUDIO; } else if (sampled == tok_mixed_start) { mode = MODE_MIXED; slots_left = n_text_block; speaking = false; text_done = false; } } else if (sampled == tok_text_end) { text_done = true; speaking = true; } if (!decode(sampled)) { return 1; } } else { std::vector codes; const float * fb = nullptr; bool is_eos = false; if (!sample_frame(h_state_in, codes, &fb, &is_eos)) { return 1; } if (is_eos) { if (!continuing) { // TTS: the end of audio ends the generation *out_stop = true; *h_state_out = nullptr; return 0; } // the segment ends, the codec tail goes out and the response continues in text if (flush() != 0) { return 1; } mode = MODE_TEXT; } else { hist.insert(hist.end(), codes.begin(), codes.end()); if (!decode_wav(codes.data(), codes.size())) { return 1; } } decode_embd_batch batch_embd(const_cast(fb), 1, 1, n_embd); batch_embd.set_position_normal(pos, seq_id); batch_embd.logits[0] = 1; if (llama_process(lctx, LLAMA_PROCESS_TYPE_DECODE, batch_embd.render(lctx, 0, 1)) != 0) { LOG_ERR("mtmd_helper_gen_audio: decode failed\n"); return 1; } pos++; } // a slot emitted while interleaving counts down the block, the switching token itself does not if (interleaving && !text_done && --slots_left == 0) { speaking = !speaking; slots_left = speaking ? n_audio_block : n_text_block; } const float * he = llama_get_embeddings_ith(lctx, -1); h_state_buf.assign(he, he + n_embd); *h_state_out = h_state_buf.data(); return 0; } int32_t get_output(int32_t * out_sample_rate, const char ** out_data, size_t * out_data_len, int64_t * out_n_samples) override { *out_sample_rate = info.sample_rate; if (!stream) { if (flush() != 0) { return 1; } if (out_n_samples) { *out_n_samples = (int64_t) audio_pcm.size(); } if (out_type == MTMD_HELPER_GEN_AUDIO_OUTTYPE_PCM) { *out_data = (const char *) audio_pcm.data(); *out_data_len = audio_pcm.size() * sizeof(float); return 0; } out_buf.clear(); if (!write_wav16(out_buf, audio_pcm, info.sample_rate)) { LOG_ERR("mtmd_helper_gen_audio: output too large for WAV\n"); return 1; } *out_data = out_buf.data(); *out_data_len = out_buf.size(); return 0; } // streaming: only the audio produced since the previous call, every frame is decoded as it is sampled const size_t n_new = audio_pcm.size() - pcm_sent; if (out_n_samples) { *out_n_samples = (int64_t) n_new; } if (out_type == MTMD_HELPER_GEN_AUDIO_OUTTYPE_PCM) { *out_data = (const char *) (audio_pcm.data() + pcm_sent); *out_data_len = n_new * sizeof(float); pcm_sent = audio_pcm.size(); return 0; } out_buf.clear(); if (!wav_header_sent) { write_wav16_header(out_buf, UINT32_MAX, info.sample_rate); wav_header_sent = true; } append_wav16_pcm(out_buf, audio_pcm.data() + pcm_sent, n_new); pcm_sent = audio_pcm.size(); *out_data = out_buf.data(); *out_data_len = out_buf.size(); return 0; } // the ISTFT holds back the overlap of the last frame int32_t flush() override { if (pending_tail) { if (!decode_wav(nullptr, 0)) { return 1; } pending_tail = false; } return 0; } private: // checked by the converter static constexpr int n_cb = 12; static constexpr int schedule[] = { 4, 4, 4 }; // codebooks revealed per depthformer pass // a random order of the codebooks, revealed by the schedule; fb is the feedback embedding of the codes. // at the end of audio every code is the end code, its feedback embedding is only built when continuing bool sample_frame(const float * h_state, std::vector & codes, const float ** fb, bool * is_eos) { std::vector order(n_cb); for (int i = 0; i < n_cb; i++) { order[(size_t) i] = i; } std::shuffle(order.begin(), order.end(), rng); codes.assign(n_cb, -1); mtmd_gen_inp inp = mtmd_gen_inp_default(mctx); inp.type = MTMD_GEN_PROCESS_TYPE_GEN_CODE; inp.embd = const_cast(h_state); inp.codes = codes.data(); inp.n_codes = codes.size(); mtmd_gen_out out{}; int k = 0; for (int n_pick : schedule) { if (mtmd_gen_audio_process(mctx, gen_state.get(), &inp, &out) != 0 || out.n_feats == 0 || out.n_feats % n_cb != 0) { LOG_ERR("mtmd_helper_gen_audio: depthformer pass failed\n"); return false; } const size_t n_vocab = out.n_feats / n_cb; logits.assign(out.feats, out.feats + out.n_feats); for (int j = 0; j < n_pick; j++, k++) { const int i = order[(size_t) k]; float * l = logits.data() + (size_t) i * n_vocab; apply_rep_penalty(l, n_vocab, i); codes[(size_t) i] = sample(l, n_vocab); } } // the last code of every codebook ends the audio const int32_t eos = (int32_t) (logits.size() / n_cb) - 1; *is_eos = std::find(codes.begin(), codes.end(), eos) != codes.end(); if (*is_eos) { codes.assign(n_cb, eos); if (!continuing) { *fb = nullptr; return true; } } // all codes known: only the feedback embedding if (mtmd_gen_audio_process(mctx, gen_state.get(), &inp, &out) != 0) { LOG_ERR("mtmd_helper_gen_audio: codebook embedding failed\n"); return false; } fb_buf.assign(out.embd, out.embd + n_embd); *fb = fb_buf.data(); return true; } // CTRL penalty over the codes codebook cb took in the last rep_window frames void apply_rep_penalty(float * l, size_t n_vocab, int cb) const { if (rep_penalty <= 1.0f || rep_window <= 0) { return; } const size_t n_hist = hist.size() / n_cb; const size_t n_win = std::min(n_hist, (size_t) rep_window); std::vector seen(n_vocab, false); for (size_t f = n_hist - n_win; f < n_hist; f++) { const int32_t c = hist[f * n_cb + (size_t) cb]; if (c >= 0 && (size_t) c < n_vocab && !seen[(size_t) c]) { seen[(size_t) c] = true; l[c] = l[c] > 0.0f ? l[c] / rep_penalty : l[c] * rep_penalty; } } } // TODO: reuse the samplers of common (top-k, temperature, penalties) instead of this one int32_t sample(const float * l, size_t n_vocab) { if (temp <= 0.0f) { return (int32_t) (std::max_element(l, l + n_vocab) - l); } // top-k keeps the codes scoring at least the k-th best, ties included float lmin = -INFINITY; if (top_k > 0 && (size_t) top_k < n_vocab) { std::vector sorted(l, l + n_vocab); std::nth_element(sorted.begin(), sorted.begin() + (top_k - 1), sorted.end(), std::greater()); lmin = sorted[(size_t) top_k - 1]; } const float lmax = *std::max_element(l, l + n_vocab); std::vector prob(n_vocab); double sum = 0.0; for (size_t v = 0; v < n_vocab; v++) { prob[v] = l[v] < lmin ? 0.0 : std::exp((double) (l[v] - lmax) / temp); sum += prob[v]; } double r = std::uniform_real_distribution(0.0, 1.0)(rng) * sum; for (size_t v = 0; v < n_vocab; v++) { r -= prob[v]; if (r <= 0.0) { return (int32_t) v; } } return (int32_t) n_vocab - 1; } // n_codes == 0 flushes the ISTFT tail bool decode_wav(const int32_t * codes, size_t n_codes) { mtmd_gen_inp inp = mtmd_gen_inp_default(mctx); inp.type = MTMD_GEN_PROCESS_TYPE_GEN_WAV; inp.codes = const_cast(codes); inp.n_codes = n_codes; mtmd_gen_out out{}; if (mtmd_gen_audio_process(mctx, gen_state.get(), &inp, &out) != 0) { LOG_ERR("mtmd_helper_gen_audio: codec decode failed\n"); return false; } audio_pcm.insert(audio_pcm.end(), out.audio, out.audio + out.n_samples); pending_tail = n_codes > 0; return true; } bool find_tokens() { tok_audio_start = find_special_token(vocab, "<|audio_start|>"); tok_mixed_start = find_special_token(vocab, "<|mixed_start|>"); tok_text_end = find_special_token(vocab, "<|text_end|>"); tok_im_end = find_special_token(vocab, "<|im_end|>"); if (tok_audio_start == LLAMA_TOKEN_NULL || tok_mixed_start == LLAMA_TOKEN_NULL || tok_text_end == LLAMA_TOKEN_NULL || tok_im_end == LLAMA_TOKEN_NULL) { LOG_ERR("mtmd_helper_gen_audio: missing LFM3-Audio control tokens in vocab\n"); return false; } return true; } bool decode(llama_token token) { llama_batch_ext_ptr batch(llama_batch_ext_init(lctx)); if (!batch_set_tokens(batch.get(), &token, 1, pos, seq_id)) { return false; } llama_batch_ext_set_output_logits(batch.get(), 0, true); if (llama_process(lctx, LLAMA_PROCESS_TYPE_DECODE, batch.get()) != 0) { LOG_ERR("mtmd_helper_gen_audio: decode failed\n"); return false; } pos += 1; return true; } // generate() modes: text, audio until end of audio, interleaved until <|text_end|> enum gen_mode { MODE_TEXT, MODE_AUDIO, MODE_MIXED }; // checked by the converter static constexpr int n_text_block = 6; // interleaved_n_text static constexpr int n_audio_block = 12; // interleaved_n_audio llama_token tok_audio_start = LLAMA_TOKEN_NULL; llama_token tok_mixed_start = LLAMA_TOKEN_NULL; llama_token tok_text_end = LLAMA_TOKEN_NULL; llama_token tok_im_end = LLAMA_TOKEN_NULL; bool continuing = false; // the prompt continues a sequence, the response follows the model's switching gen_mode mode = MODE_AUDIO; int slots_left = 0; bool speaking = false; // interleaving: the current block is audio bool text_done = false; // interleaving: <|text_end|> was emitted, audio until the end-of-audio frame llama_seq_id seq_id = 0; int pos = 0; std::vector prompt_tokens; int prompt_pos = 0; float temp = 0.0f; int32_t top_k = 0; float rep_penalty = 1.0f; int32_t rep_window = 0; // frames std::mt19937 rng; std::vector logits; std::vector fb_buf; std::vector hist; // frame-major codes of the frames so far bool pending_tail = false; // the ISTFT holds back the overlap of the last frame std::vector audio_pcm; std::vector h_state_buf; mtmd_helper_gen_audio_outtype out_type = MTMD_HELPER_GEN_AUDIO_OUTTYPE_WAV; std::vector out_buf; bool stream = false; size_t pcm_sent = 0; // samples already returned by get_output() bool wav_header_sent = false; }; // Chatterbox: the backbone emits s3 speech tokens, then one flow-matching mel decode and the NSF-iSTFT vocoder make the PCM // the multilingual variant runs cfg with an uncond sequence (text rows zeroed), paired as seq_id and n_seq_max / 2 + seq_id class chatterbox_gen_audio_pipeline : public mtmd_gen_audio_pipeline { public: using mtmd_gen_audio_pipeline::mtmd_gen_audio_pipeline; void reset() override { seq_id = 0; seq_uncond = 0; pos = 0; ar_idx = 0; codes_buf.clear(); audio_pcm.clear(); h_state_buf.clear(); out_buf.clear(); prompt_embd_buf.clear(); uncond_embd_buf.clear(); cond_logits.clear(); n_prompt = 0; prompt_pos = 0; decoded = false; stream = false; pcm_sent = 0; wav_header_sent = false; gen_state.reset(); } int32_t set_input(const mtmd_helper_gen_audio_inp * inp) override { reset(); seq_id = inp->seq_id; if (!ensure_cache()) { return 1; } out_type = inp->out_type; stream = inp->stream; if (!init_gen_state(inp->seed)) { return 1; } if (mtl) { // the uncond sequence lives half a seq space away from the cond one const int32_t n_seq_max = (int32_t) llama_n_seq_max(lctx); seq_uncond = (llama_seq_id) (n_seq_max / 2 + seq_id); if (!llama_model_classifier_free_guidance(llama_get_model(lctx)) || seq_id >= n_seq_max / 2) { LOG_ERR("mtmd_helper_gen_audio: multilingual chatterbox needs a backbone with classifier_free_guidance and a paired cfg sequence " "(n_seq_max = %d, seq_id = %d)\n", n_seq_max, seq_id); return 1; } } llama_memory_t mem = llama_get_memory(lctx); if (!llama_memory_seq_rm(mem, seq_id, 0, -1) || (mtl && !llama_memory_seq_rm(mem, seq_uncond, 0, -1))) { LOG_ERR("mtmd_helper_gen_audio: cannot reset chatterbox sequences\n"); return 1; } // the encode gives the talker conditioning rows, set_ref keeps the flow decoder reference if (!inp->speaker_ref) { LOG_ERR("mtmd_helper_gen_audio: chatterbox requires a speaker reference or a voice\n"); return 1; } if (!encode_speaker(inp->speaker_ref, prompt_embd_buf) || mtmd_gen_state_set_ref(mctx, gen_state.get()) != 0) { LOG_ERR("mtmd_helper_gen_audio: speaker reference encoding failed\n"); return 1; } const int n_e = n_embd; auto push_row = [&](const float * src) { prompt_embd_buf.insert(prompt_embd_buf.end(), src, src + n_e); }; auto add_pos = [&](float * r, const std::vector & tab, int idx) { const float * p = tab.data() + (size_t) idx * n_e; for (int j = 0; j < n_e; j++) { r[j] += p[j]; } }; // prompt rows: [conditioning, text, speech bos] // no text preprocessing here, the [SPACE] substitution and the case folding are baked in at conversion std::string txt(inp->prompt, inp->prompt_len); if (mtl && inp->lang && inp->lang[0]) { // the reference prefixes the language tag, e.g. "[fr]" std::string lang = inp->lang; std::transform(lang.begin(), lang.end(), lang.begin(), [](unsigned char c) { return (char) std::tolower(c); }); txt = "[" + lang + "]" + txt; } std::vector ids(txt.size() + 16); int n_ids = llama_tokenize(vocab, txt.c_str(), (int32_t) txt.size(), ids.data(), (int32_t) ids.size(), false, true); if (n_ids < 1) { LOG_ERR("mtmd_helper_gen_audio: tokenization failed\n"); return 1; } ids.resize((size_t) n_ids); if (mtl) { // only the multilingual prompt wraps the text in start/stop tokens ids.insert(ids.begin(), text_start); ids.push_back(text_stop); if (ids.size() > text_pos.size() / (size_t) n_e) { LOG_ERR("mtmd_helper_gen_audio: prompt too long (%zu tokens, max %zu)\n", ids.size(), text_pos.size() / (size_t) n_e); return 1; } } const size_t text0 = prompt_embd_buf.size() / (size_t) n_e; for (size_t i = 0; i < ids.size(); i++) { push_row(tok_embd.data() + (size_t) ids[i] * n_e); if (mtl) { add_pos(prompt_embd_buf.data() + (text0 + i) * n_e, text_pos, (int) i); } } // speech bos opens the AR stream push_row(tok_embd.data() + (size_t) llama_vocab_bos(vocab) * n_e); if (mtl) { add_pos(prompt_embd_buf.data() + prompt_embd_buf.size() - n_e, speech_pos, 0); } ar_idx = 1; n_prompt = (int) (prompt_embd_buf.size() / (size_t) n_e); if (mtl) { // cfg uncond prompt: text rows are zeroed but keep their learned positions, as in the reference uncond_embd_buf = prompt_embd_buf; for (size_t i = 0; i < ids.size(); i++) { float * r = uncond_embd_buf.data() + (text0 + i) * n_e; std::fill(r, r + n_e, 0.0f); add_pos(r, text_pos, (int) i); } } prompt_pos = 0; return 0; } int32_t step_prompt(int32_t n_batch) override { GGML_ASSERT(n_batch > 0); // the uncond prompt follows the cond one on mtl const int n_total = mtl ? 2 * n_prompt : n_prompt; if (prompt_pos >= n_total) { return 0; } const bool uncond = prompt_pos >= n_prompt; const int p0 = uncond ? prompt_pos - n_prompt : prompt_pos; const int32_t n_tokens_batch = std::min(n_batch, n_prompt - p0); const bool last = p0 + n_tokens_batch == n_prompt; decode_embd_batch batch((uncond ? uncond_embd_buf : prompt_embd_buf).data(), n_prompt, 1, n_embd); batch.set_position_normal(0, uncond ? seq_uncond : seq_id); if (last) { batch.logits[n_prompt - 1] = 1; } if (llama_process(lctx, LLAMA_PROCESS_TYPE_DECODE, batch.render(lctx, p0, n_tokens_batch)) != 0) { LOG_ERR("mtmd_helper_gen_audio: prompt decode failed\n"); return -1; } prompt_pos += n_tokens_batch; if (last && mtl) { if (!uncond) { cfg_read(); } else { cfg_apply(); } } if (prompt_pos >= n_total) { pos = n_prompt; prompt_embd_buf.clear(); uncond_embd_buf.clear(); return 0; } return n_total - prompt_pos; } int32_t step_gen(llama_token sampled, const float * h_state_in, const float ** h_state_out, bool * out_stop) override { GGML_UNUSED(h_state_in); if (sampled < 0 || sampled >= llama_vocab_n_tokens(vocab)) { LOG_ERR("mtmd_helper_gen_audio: chatterbox requires a token sampled from the backbone\n"); return 1; } // the speech stop token is the backbone eos, the multilingual speech positions end the stream too if (llama_vocab_is_eog(vocab, sampled) || (mtl && (size_t) ar_idx >= speech_pos.size() / (size_t) n_embd)) { *out_stop = true; *h_state_out = nullptr; return 0; } // keep only the 6561 s3gen codes, dropping start/stop and oov ids if (sampled >= speech_base && sampled - speech_base < 6561) { codes_buf.push_back(sampled - speech_base); } std::vector e(tok_embd.begin() + (size_t) sampled * n_embd, tok_embd.begin() + (size_t) (sampled + 1) * n_embd); if (mtl) { // the multilingual backbone takes the token row plus the learned speech position const float * p = speech_pos.data() + (size_t) ar_idx * n_embd; for (int j = 0; j < n_embd; j++) { e[(size_t) j] += p[j]; } ar_idx++; } if (!decode_one(e.data(), seq_id)) { return 1; } if (mtl) { // cfg: the sampled token feeds both branches cfg_read(); if (!decode_one(e.data(), seq_uncond)) { return 1; } cfg_apply(); } pos++; const float * h = llama_get_embeddings_ith(lctx, -1); h_state_buf.assign(h, h + n_embd); *h_state_out = h_state_buf.data(); return 0; } int32_t get_output(int32_t * out_sample_rate, const char ** out_data, size_t * out_data_len, int64_t * out_n_samples) override { *out_sample_rate = info.sample_rate; if (!stream) { if (!flush_gen_wav()) { return 1; } if (out_n_samples) { *out_n_samples = (int64_t) audio_pcm.size(); } if (out_type == MTMD_HELPER_GEN_AUDIO_OUTTYPE_PCM) { *out_data = (const char *) audio_pcm.data(); *out_data_len = audio_pcm.size() * sizeof(float); return 0; } out_buf.clear(); if (!write_wav16(out_buf, audio_pcm, info.sample_rate)) { LOG_ERR("mtmd_helper_gen_audio: output too large for WAV\n"); return 1; } *out_data = out_buf.data(); *out_data_len = out_buf.size(); return 0; } // streaming: the flow decode is one-shot, so all the audio arrives after flush() const size_t n_new = audio_pcm.size() - pcm_sent; if (out_n_samples) { *out_n_samples = (int64_t) n_new; } if (out_type == MTMD_HELPER_GEN_AUDIO_OUTTYPE_PCM) { *out_data = (const char *) (audio_pcm.data() + pcm_sent); *out_data_len = n_new * sizeof(float); pcm_sent = audio_pcm.size(); return 0; } out_buf.clear(); if (!wav_header_sent) { write_wav16_header(out_buf, UINT32_MAX, info.sample_rate); wav_header_sent = true; } append_wav16_pcm(out_buf, audio_pcm.data() + pcm_sent, n_new); pcm_sent = audio_pcm.size(); *out_data = out_buf.data(); *out_data_len = out_buf.size(); return 0; } int32_t flush() override { return flush_gen_wav() ? 0 : 1; } private: bool decode_one(const float * e, llama_seq_id seq) { decode_embd_batch batch(e, 1, 1, n_embd); batch.set_position_normal(pos, seq); batch.logits[0] = 1; if (llama_process(lctx, LLAMA_PROCESS_TYPE_DECODE, batch.render(lctx, 0, 1)) != 0) { LOG_ERR("mtmd_helper_gen_audio: step decode failed\n"); return false; } return true; } // cfg of the reference: logits = cond + 0.5 * (cond - uncond) // the mix goes into the last output row (the uncond one), the caller samples from it void cfg_read() { const float * c = llama_get_logits_ith(lctx, -1); cond_logits.assign(c, c + llama_vocab_n_tokens(vocab)); } void cfg_apply() { float * u = llama_get_logits_ith(lctx, -1); for (size_t i = 0; i < cond_logits.size(); i++) { u[i] = cond_logits[i] + 0.5f * (cond_logits[i] - u[i]); } } // one GEN_WAV call over the whole utterance, the flow decode is not streamed bool flush_gen_wav() { if (decoded || codes_buf.empty()) { return true; } if (!mtl) { // turbo appends a short silence tail before vocoding codes_buf.insert(codes_buf.end(), 3, 4299); } mtmd_gen_inp inp = mtmd_gen_inp_default(mctx); inp.type = MTMD_GEN_PROCESS_TYPE_GEN_WAV; inp.codes = codes_buf.data(); inp.n_codes = codes_buf.size(); mtmd_gen_out out{}; if (mtmd_gen_audio_process(mctx, gen_state.get(), &inp, &out) != 0) { LOG_ERR("mtmd_helper_gen_audio: gen_wav decode failed\n"); return false; } audio_pcm.assign(out.audio, out.audio + out.n_samples); decoded = true; return true; } bool ensure_cache() { if (!tok_embd.empty()) { return true; } // fused vocab layout: [text 0..speech_base) then the speech tokens speech_base = find_special_token(vocab, "<|speech_0|>"); if (speech_base == LLAMA_TOKEN_NULL) { LOG_ERR("mtmd_helper_gen_audio: fused speech tokens not found in vocab\n"); return false; } const uint32_t n_tok_embd = llama_model_get_tok_embd(model, nullptr); if (n_tok_embd != (uint32_t) llama_vocab_n_tokens(vocab) * (uint32_t) n_embd) { LOG_ERR("mtmd_helper_gen_audio: unexpected token embedding size\n"); return false; } tok_embd.resize(n_tok_embd); llama_model_get_tok_embd(model, tok_embd.data()); auto read_table = [&](const char * name, std::vector & dst) { const size_t n = mtmd_gen_audio_read_tensor(mctx, name, nullptr, 0); dst.resize(n); if (n == 0 || mtmd_gen_audio_read_tensor(mctx, name, dst.data(), n) != n) { LOG_ERR("mtmd_helper_gen_audio: %s read failed\n", name); return false; } return true; }; // multilingual: the learned position tables mtl = mtmd_gen_audio_read_tensor(mctx, "a.gen.t3.text_pos_emb", nullptr, 0) > 0; if (mtl && (!read_table("a.gen.t3.text_pos_emb", text_pos) || !read_table("a.gen.t3.speech_pos_emb", speech_pos))) { return false; } return true; } // vocab and mmproj data fixed across the whole session, read once std::vector tok_embd; bool mtl = false; std::vector text_pos; std::vector speech_pos; llama_token speech_base = LLAMA_TOKEN_NULL; // start_text_token and stop_text_token of the reference, only the multilingual prompt uses them static constexpr llama_token text_start = 255; static constexpr llama_token text_stop = 0; // per-generation state, cleared by reset() llama_seq_id seq_id = 0; llama_seq_id seq_uncond = 0; llama_pos pos = 0; int ar_idx = 0; std::vector prompt_embd_buf; std::vector uncond_embd_buf; int n_prompt = 0; int prompt_pos = 0; std::vector cond_logits; std::vector codes_buf; bool decoded = false; std::vector audio_pcm; std::vector h_state_buf; mtmd_helper_gen_audio_outtype out_type = MTMD_HELPER_GEN_AUDIO_OUTTYPE_WAV; std::vector out_buf; bool stream = false; size_t pcm_sent = 0; // samples already returned by get_output() bool wav_header_sent = false; }; // Drama 3: backbone samples the main token of each frame, codebook tower samples the other codes, Fish DAC makes the PCM // the tower attends over the whole sequence, so it also runs on every prompt row class drama3_gen_audio_pipeline : public mtmd_gen_audio_pipeline { public: using mtmd_gen_audio_pipeline::mtmd_gen_audio_pipeline; void reset() override { seq_id = 0; pos = 0; prompt_tokens.clear(); ref_codes.clear(); ref_embd.clear(); ref_begin = 0; prompt_pos = 0; frame.clear(); main_logits.clear(); hist.clear(); codes_buf.clear(); n_decoded = 0; gen_state.reset(); pcm_sent = 0; wav_header_sent = false; audio_pcm.clear(); h_state_buf.clear(); out_buf.clear(); } int32_t set_input(const mtmd_helper_gen_audio_inp * inp) override { reset(); seq_id = inp->seq_id; out_type = inp->out_type; stream = inp->stream; // voice cloning: backbone rows from the codec encoder, the codes of the clip go to the gen state std::vector ref_rows; if (inp->speaker_ref && !mtmd_bitmap_is_voice(inp->speaker_ref) && !mtmd_support_audio(mctx)) { LOG_WRN("mtmd_helper_gen_audio: mmproj has no speaker encoder (re-convert it for voice cloning), the speaker file is ignored\n"); } else if (inp->speaker_ref) { if (!encode_speaker(inp->speaker_ref, ref_rows) || !init_gen_state(inp->seed) || mtmd_gen_state_set_ref(mctx, gen_state.get()) != 0) { return 1; } } sem_begin = find_special_token(vocab, "<|semantic:0|>"); eos = find_special_token(vocab, "<|im_end|>"); if (sem_begin == LLAMA_TOKEN_NULL || eos == LLAMA_TOKEN_NULL || sem_begin + n_cb_vocab > llama_vocab_n_tokens(vocab)) { LOG_ERR("mtmd_helper_gen_audio: missing semantic tokens in vocab\n"); return 1; } const std::string text(inp->prompt, inp->prompt_len); if (ref_rows.empty()) { if (!tokenize_segments(prompt_segments(text), prompt_tokens)) { return 1; } } else if (!build_reference_prompt(text, inp->speaker_ref_text ? inp->speaker_ref_text : "", std::move(ref_rows))) { return 1; } if (prompt_tokens.size() >= llama_n_ctx_seq(lctx) || prompt_tokens.size() > max_rows()) { LOG_ERR("mtmd_helper_gen_audio: prompt does not fit in the context\n"); return 1; } if (!llama_memory_seq_rm(llama_get_memory(lctx), seq_id, 0, -1)) { LOG_ERR("mtmd_helper_gen_audio: cannot reset the sequence\n"); return 1; } rng.seed(inp->seed == UINT32_MAX ? std::random_device{}() : inp->seed); return gen_state || init_gen_state(inp->seed) ? 0 : 1; } // run the tower on every prompt row to fill its KV int32_t step_prompt(int32_t n_batch) override { GGML_ASSERT(n_batch > 0); const int n_prompt = (int) prompt_tokens.size(); if (prompt_pos >= n_prompt) { return 0; } // small batches of rows of one kind (text tokens or reference frames): every row also outputs logits const bool speech = is_speech_row(prompt_pos); int n = std::min({ n_batch, n_prompt - prompt_pos, 64 }); for (int i = 1; i < n; i++) { if (is_speech_row(prompt_pos + i) != speech) { n = i; break; } } int ret = 0; if (speech) { ret = decode_embd_rows(ref_embd.data() + (size_t) (prompt_pos - ref_begin) * n_embd, n); } else { llama_batch_ext_ptr batch(llama_batch_ext_init(lctx)); if (!batch_set_tokens(batch.get(), prompt_tokens.data() + prompt_pos, n, pos, seq_id)) { return -1; } for (int i = 0; i < n; i++) { llama_batch_ext_set_output_logits(batch.get(), i, true); } ret = llama_process(lctx, LLAMA_PROCESS_TYPE_DECODE, batch.get()); } if (ret != 0) { LOG_ERR("mtmd_helper_gen_audio: prompt decode failed\n"); return -1; } for (int i = 0; i < n; i++) { const int row = prompt_pos + i; const float * h = llama_get_embeddings_ith(lctx, i); if (row == n_prompt - 1) { h_state_buf.assign(h, h + n_embd); // first frame comes from the last prompt row save_main_logits(i); break; } if (!run_prompt_tower(row, h)) { return -1; } } pos += n; prompt_pos += n; return n_prompt - prompt_pos; } // sample the main token here, ignore the token of the caller int32_t step_gen(llama_token sampled, const float * h_state_in, const float ** h_state_out, bool * out_stop) override { GGML_UNUSED(sampled); GGML_UNUSED(h_state_in); *h_state_out = nullptr; *out_stop = false; const llama_token main = sample_main(main_logits.data()); const size_t n_rows = prompt_tokens.size() - 1 + codes_buf.size() / n_cb; if (main == eos || (uint32_t) pos >= llama_n_ctx_seq(lctx) || n_rows >= max_rows()) { *out_stop = true; return 0; } hist.push_back(main); if (hist.size() > ras_window) { hist.erase(hist.begin()); } const float * next_embd = run_tower(main - sem_begin, h_state_buf.data()); if (!next_embd) { return 1; } codes_buf.insert(codes_buf.end(), frame.begin(), frame.end()); if (decode_embd_rows(next_embd, 1) != 0) { LOG_ERR("mtmd_helper_gen_audio: decode failed\n"); return 1; } pos++; const float * h = llama_get_embeddings_ith(lctx, -1); h_state_buf.assign(h, h + n_embd); save_main_logits(-1); *h_state_out = h_state_buf.data(); if (stream && codes_buf.size() / n_cb - n_decoded >= stream_frames) { return decode_wav() ? 0 : 1; } return 0; } int32_t get_output(int32_t * out_sample_rate, const char ** out_data, size_t * out_data_len, int64_t * out_n_samples) override { *out_sample_rate = info.sample_rate; if (!stream) { if (flush() != 0) { return 1; } if (out_n_samples) { *out_n_samples = (int64_t) audio_pcm.size(); } if (out_type == MTMD_HELPER_GEN_AUDIO_OUTTYPE_PCM) { *out_data = (const char *) audio_pcm.data(); *out_data_len = audio_pcm.size() * sizeof(float); return 0; } out_buf.clear(); if (!write_wav16(out_buf, audio_pcm, info.sample_rate)) { LOG_ERR("mtmd_helper_gen_audio: output too large for WAV\n"); return 1; } *out_data = out_buf.data(); *out_data_len = out_buf.size(); return 0; } // streaming: only the audio since the previous call const size_t n_new = audio_pcm.size() - pcm_sent; if (out_n_samples) { *out_n_samples = (int64_t) n_new; } if (out_type == MTMD_HELPER_GEN_AUDIO_OUTTYPE_PCM) { *out_data = (const char *) (audio_pcm.data() + pcm_sent); *out_data_len = n_new * sizeof(float); pcm_sent = audio_pcm.size(); return 0; } out_buf.clear(); if (!wav_header_sent) { write_wav16_header(out_buf, UINT32_MAX, info.sample_rate); wav_header_sent = true; } append_wav16_pcm(out_buf, audio_pcm.data() + pcm_sent, n_new); pcm_sent = audio_pcm.size(); *out_data = out_buf.data(); *out_data_len = out_buf.size(); return 0; } int32_t flush() override { return decode_wav() ? 0 : 1; } protected: // tokenize each segment alone as the reference does bool tokenize_segments(const std::vector & segments, std::vector & out) const { for (const std::string & seg : segments) { std::vector ids(seg.size() + 16); const int32_t n = llama_tokenize(vocab, seg.c_str(), (int32_t) seg.size(), ids.data(), (int32_t) ids.size(), false, true); if (n < 0) { LOG_ERR("mtmd_helper_gen_audio: tokenization failed\n"); return false; } out.insert(out.end(), ids.begin(), ids.begin() + n); } return true; } // offset of the leading speaker tag <|speaker:N|> after optional whitespace, npos without one static size_t speaker_tag_pos(const std::string & text) { const size_t lead = text.find_first_not_of(" \t\n\r\f\v"); static const std::string open = "<|speaker:"; if (lead == std::string::npos || text.compare(lead, open.size(), open) != 0) { return std::string::npos; } size_t i = lead + open.size(); const size_t digits = i; while (i < text.size() && text[i] >= '0' && text[i] <= '9') { i++; } return i > digits && text.compare(i, 2, "|>") == 0 ? lead : std::string::npos; } // reference-bracket-v1 prompt without references virtual std::vector prompt_segments(std::string text) const { if (speaker_tag_pos(text) == std::string::npos) { text = "<|speaker:0|> " + text; } return { "<|im_start|>system\n", "convert the provided text to speech", "<|im_end|>\n", "<|im_start|>user\n", text, "<|im_end|>\n", "<|im_start|>assistant\n<|voice|>", }; } // prompt rows but the last + generated frames virtual size_t max_rows() const { return n_tower_rows; } // n speech rows from pos, every row outputs logits // text/speech stream: all mrope sections use the same position int32_t decode_embd_rows(const float * embd, int n) { const bool mrope = llama_model_rope_type(model) == LLAMA_ROPE_TYPE_MROPE || llama_model_rope_type(model) == LLAMA_ROPE_TYPE_IMROPE; decode_embd_batch batch(embd, n, mrope ? 4 : 1, n_embd); if (mrope) { batch.set_position_mrope_1d(pos, seq_id); } else { batch.set_position_normal(pos, seq_id); } for (int i = 0; i < n; i++) { batch.logits[i] = 1; } return llama_process(lctx, LLAMA_PROCESS_TYPE_DECODE, batch.render(lctx, 0, n)); } bool is_speech_row(int row) const { return row >= ref_begin && (size_t) (row - ref_begin) < ref_embd.size() / n_embd; } // reference-bracket-v1 with one reference, see vLLM-Omni fish_omni/prompt.py // system: instruction, "\n\n[Reference 1]\nText: \nSpeech: " then one row per frame of the clip bool build_reference_prompt(std::string text, std::string ref_text, std::vector && ref_rows) { size_t n_codes = 0; const int32_t * codes = mtmd_gen_state_get_ref_codes(gen_state.get(), &n_codes); const size_t n_frames = ref_rows.size() / n_embd; if (!codes || n_frames == 0 || n_codes != n_frames * n_cb || ref_rows.size() != n_frames * n_embd) { LOG_ERR("mtmd_helper_gen_audio: the speaker encoder gave no codes for the clip\n"); return false; } // the speaker tag follows "Text: " directly (default speaker 0), an untagged target reuses it std::string tag = "<|speaker:0|>"; const size_t lead = speaker_tag_pos(ref_text); if (lead == std::string::npos) { ref_text = tag + " " + ref_text; } else { ref_text = ref_text.substr(lead); tag = ref_text.substr(0, ref_text.find("|>") + 2); } if (speaker_tag_pos(text) == std::string::npos) { text = tag + " " + text; } std::vector head, tail; if (!tokenize_segments({ "<|im_start|>system\n", "convert the provided text to speech reference to the following:", "\n\n[Reference 1]\nText: " + ref_text + "\nSpeech: " }, head) || !tokenize_segments({ "<|im_end|>\n", "<|im_start|>user\n", text, "<|im_end|>\n", "<|im_start|>assistant\n<|voice|>" }, tail)) { return false; } prompt_tokens.assign(head.begin(), head.end()); prompt_tokens.insert(prompt_tokens.end(), n_frames, LLAMA_TOKEN_NULL); prompt_tokens.insert(prompt_tokens.end(), tail.begin(), tail.end()); ref_codes.assign(codes, codes + n_codes); ref_embd = std::move(ref_rows); ref_begin = (int) head.size(); return true; } // tower row of a prompt row: the input frame is the codes of this row (reference frame) or none, // the codes of the next row are forced when it is a reference frame bool run_prompt_tower(int row, const float * h) { const bool cur = is_speech_row(row); const bool next = is_speech_row(row + 1); const auto ref_frame = [&](int r) { return ref_codes.begin() + (size_t) (r - ref_begin) * n_cb; }; if (next) { // [input frame, -1 for a text row | forced next frame] frame.assign(n_cb, -1); if (cur) { std::copy(ref_frame(row), ref_frame(row + 1), frame.begin()); } frame.insert(frame.end(), ref_frame(row + 1), ref_frame(row + 2)); } else if (cur) { frame.assign(ref_frame(row), ref_frame(row + 1)); } const bool ok = run_tower(next ? frame[n_cb] : -1, h) != nullptr; frame.clear(); return ok; } // copy right after our own decode, other sequences may decode on lctx before the next step_gen // layout: eos, then semantic codes 0..n_cb_vocab-1 void save_main_logits(int32_t i) { const float * logits = llama_get_logits_ith(lctx, i); main_logits.resize(n_cb_vocab + 1); main_logits[0] = logits[eos]; std::copy(logits + sem_begin, logits + sem_begin + n_cb_vocab, main_logits.begin() + 1); } // repetition-aware sampling: if a semantic draw repeats one of the last ras_window main tokens, draw again with fixed settings static constexpr size_t ras_window = 16; static constexpr size_t n_cb = 10; // codebook tower KV rows (mtmd_graph_drama3_gen::n_pos_max): one per prompt row but the last, one per frame static constexpr size_t n_tower_rows = 4096; // frames per codec call when streaming (0.33 s) static constexpr size_t stream_frames = 7; // top-p keeps the token that crosses the threshold, as the reference does llama_token draw(const std::vector & cand_logits, float temp, int32_t top_k, float top_p) { std::vector> cand(cand_logits.size()); for (size_t i = 0; i < cand.size(); i++) { cand[i] = { cand_logits[i] / temp, (int) i }; } std::sort(cand.begin(), cand.end(), [](const auto & a, const auto & b) { return a.first > b.first; }); const float lmax = cand[0].first; double sum = 0.0; std::vector prob(cand.size()); for (size_t i = 0; i < cand.size(); i++) { prob[i] = std::exp((double) (cand[i].first - lmax)); sum += prob[i]; } size_t n_keep = cand.size(); if (top_k > 0) { n_keep = std::min(n_keep, (size_t) top_k); } double cum = 0.0; for (size_t i = 0; i < n_keep; i++) { if (cum > top_p * sum) { n_keep = i; break; } cum += prob[i]; } double kept = 0.0; for (size_t i = 0; i < n_keep; i++) { kept += prob[i]; } double r = std::uniform_real_distribution(0.0, 1.0)(rng) * kept; for (size_t i = 0; i < n_keep; i++) { r -= prob[i]; if (r <= 0.0) { return cand[i].second; } } return cand[n_keep - 1].second; } // candidates: eos, then semantic codes 1.. (the reference never samples code 0) // logits as saved by save_main_logits() virtual llama_token sample_main(const float * logits) { std::vector cand(n_cb_vocab); cand[0] = logits[0]; for (int i = 1; i < n_cb_vocab; i++) { cand[i] = logits[1 + i]; } auto to_token = [&](int i) { return i == 0 ? eos : sem_begin + i; }; const llama_token first = to_token(draw(cand, 1.0f, 0, 0.9f)); if (first != eos && std::find(hist.begin(), hist.end(), first) != hist.end()) { return to_token(draw(cand, 1.0f, 30, 0.95f)); } return first; } // code0 < 0 for a prompt row, else the main code of the next frame // returns the next backbone input, the codes of the next frame go to frame const float * run_tower(int32_t code0, const float * h) { mtmd_gen_inp inp = mtmd_gen_inp_default(mctx); inp.type = MTMD_GEN_PROCESS_TYPE_GEN_CODE; inp.code0 = code0; inp.embd = const_cast(h); inp.codes = frame.empty() ? nullptr : frame.data(); inp.n_codes = frame.size(); mtmd_gen_out out{}; if (mtmd_gen_audio_process(mctx, gen_state.get(), &inp, &out) != 0 || (code0 >= 0 && out.n_codes != n_cb)) { LOG_ERR("mtmd_helper_gen_audio: codebook tower failed\n"); return nullptr; } if (code0 >= 0) { frame.assign(out.codes, out.codes + out.n_codes); } return out.embd; } // the codec is stateful, so frames can be decoded in pieces bool decode_wav() { const size_t n_frames = codes_buf.size() / n_cb; while (n_decoded < n_frames) { const size_t n = std::min(n_frames - n_decoded, (size_t) 64); mtmd_gen_inp inp = mtmd_gen_inp_default(mctx); inp.type = MTMD_GEN_PROCESS_TYPE_GEN_WAV; inp.codes = codes_buf.data() + n_decoded * n_cb; inp.n_codes = n * n_cb; mtmd_gen_out out{}; if (mtmd_gen_audio_process(mctx, gen_state.get(), &inp, &out) != 0) { LOG_ERR("mtmd_helper_gen_audio: codec decode failed\n"); return false; } audio_pcm.insert(audio_pcm.end(), out.audio, out.audio + out.n_samples); n_decoded += n; } return true; } static constexpr int n_cb_vocab = 4096; // <|semantic:0..4095|> llama_token sem_begin = LLAMA_TOKEN_NULL; llama_token eos = LLAMA_TOKEN_NULL; llama_seq_id seq_id = 0; llama_pos pos = 0; std::vector prompt_tokens; // one per prompt row, LLAMA_TOKEN_NULL for a reference frame std::vector ref_codes; // codes of the reference frames, n_cb per frame std::vector ref_embd; // backbone rows of the reference frames int ref_begin = 0; // prompt row of the first reference frame int prompt_pos = 0; std::mt19937 rng; std::vector main_logits; // see save_main_logits() std::vector frame; // codes of the last frame, input of the next tower row std::vector hist; // last main tokens, for repetition-aware sampling std::vector codes_buf; // frame-major size_t n_decoded = 0; // frames already through the codec std::vector audio_pcm; std::vector h_state_buf; mtmd_helper_gen_audio_outtype out_type = MTMD_HELPER_GEN_AUDIO_OUTTYPE_WAV; std::vector out_buf; bool stream = false; size_t pcm_sent = 0; // samples already returned by get_output() bool wav_header_sent = false; }; // Fish Audio S2 Pro: same tokens, main-token loop and Fish DAC as drama3 // the fast AR only runs on generated frames, so the prompt is a plain decode class fish_s2_gen_audio_pipeline : public drama3_gen_audio_pipeline { public: using drama3_gen_audio_pipeline::drama3_gen_audio_pipeline; void reset() override { drama3_gen_audio_pipeline::reset(); parts.clear(); part_idx = 0; part_pos = 0; } // voice cloning (sglang-omni fishaudio_s2_pro): a system message with the transcript and the codec rows of the clip int32_t set_input(const mtmd_helper_gen_audio_inp * inp) override { std::vector ref_rows; if (inp->speaker_ref && !encode_speaker(inp->speaker_ref, ref_rows)) { return 1; } mtmd_helper_gen_audio_inp text_inp = *inp; text_inp.speaker_ref = nullptr; if (drama3_gen_audio_pipeline::set_input(&text_inp) != 0) { return 1; } size_t n_total = prompt_tokens.size(); if (!ref_rows.empty()) { // optional, the reference leaves the text empty when it has none std::string ref_text = inp->speaker_ref_text ? inp->speaker_ref_text : ""; if (!ref_text.empty() && ref_text.rfind("<|speaker:", 0) != 0) { ref_text = "<|speaker:0|>" + ref_text; } prompt_part head, rows, tail; if (!tokenize_segments({ "<|im_start|>system\n", "convert the provided text to speech reference to the following:\n\nText:\n", ref_text, "\n\nSpeech:\n" }, head.tokens) || !tokenize_segments({ "<|im_end|>\n" }, tail.tokens)) { return 1; } rows.rows = std::move(ref_rows); for (auto * part : { &head, &rows, &tail }) { n_total += part->size(n_embd); parts.push_back(std::move(*part)); } } if (n_total >= llama_n_ctx_seq(lctx)) { LOG_ERR("mtmd_helper_gen_audio: prompt does not fit in the context\n"); return 1; } prompt_part text; text.tokens = prompt_tokens; parts.push_back(std::move(text)); return 0; } // the first frame comes from the last prompt row int32_t step_prompt(int32_t n_batch) override { GGML_ASSERT(n_batch > 0); if (part_idx >= parts.size()) { return 0; } const prompt_part & part = parts[part_idx]; const int n_part = part.size(n_embd); const int n = std::min(n_batch, n_part - part_pos); const bool last = part_idx + 1 == parts.size() && part_pos + n == n_part; int ret = 0; if (part.rows.empty()) { llama_batch_ext_ptr batch(llama_batch_ext_init(lctx)); if (!batch_set_tokens(batch.get(), part.tokens.data() + part_pos, n, pos, seq_id)) { return -1; } if (last) { llama_batch_ext_set_output_logits(batch.get(), n - 1, true); } ret = llama_process(lctx, LLAMA_PROCESS_TYPE_DECODE, batch.get()); } else { decode_embd_batch batch(part.rows.data() + (size_t) part_pos * n_embd, n, 1, n_embd); batch.set_position_normal(pos, seq_id); ret = llama_process(lctx, LLAMA_PROCESS_TYPE_DECODE, batch.render(lctx, 0, n)); } if (ret != 0) { LOG_ERR("mtmd_helper_gen_audio: prompt decode failed\n"); return -1; } if (last) { const float * h = llama_get_embeddings_ith(lctx, -1); h_state_buf.assign(h, h + n_embd); save_main_logits(-1); } pos += n; part_pos += n; if (part_pos == n_part) { part_idx++; part_pos = 0; } int32_t n_left = -part_pos; for (size_t i = part_idx; i < parts.size(); i++) { n_left += parts[i].size(n_embd); } return n_left; } protected: // reference (sglang-omni fishaudio_s2_pro) has no system message std::vector prompt_segments(std::string text) const override { if (text.rfind("<|speaker:", 0) != 0) { text = "<|speaker:0|>" + text; } return { "<|im_start|>user\n", text, "<|im_end|>\n", "<|im_start|>assistant\n<|voice|>", }; } // no codebook tower, only the context limits the length size_t max_rows() const override { return SIZE_MAX; } // token ids, or backbone input rows of n_embd floats struct prompt_part { std::vector tokens; std::vector rows; int size(int n_embd) const { return rows.empty() ? (int) tokens.size() : (int) (rows.size() / n_embd); } }; std::vector parts; size_t part_idx = 0; int part_pos = 0; // same as the reference: repetition penalty over the last ras_window main tokens, top-k, top-p on the // untempered logits (the crossing token is dropped), then temperature // a repeat within the last 4 main tokens switches to the RAS settings llama_token sample_main(const float * logits) override { static constexpr int32_t top_k = 30; static constexpr float rep_penalty = 1.1f; std::vector> cand; cand.reserve(n_cb_vocab + 1); cand.push_back({ logits[0], eos }); for (int i = 0; i < n_cb_vocab; i++) { cand.push_back({ logits[1 + i], sem_begin + i }); } for (auto & c : cand) { if (std::find(hist.begin(), hist.end(), c.second) != hist.end()) { c.first = c.first < 0.0f ? c.first * rep_penalty : c.first / rep_penalty; } } bool ras = false; if (hist.size() >= 4) { std::vector last4(hist.end() - 4, hist.end()); std::sort(last4.begin(), last4.end()); ras = std::adjacent_find(last4.begin(), last4.end()) != last4.end(); } const float temp = ras ? 1.0f : 0.8f; const float top_p = ras ? 0.9f : 0.8f; const size_t k = std::min(top_k, cand.size()); std::partial_sort(cand.begin(), cand.begin() + k, cand.end(), [](const auto & a, const auto & b) { return a.first > b.first; }); std::vector prob(k); double sum = 0.0; for (size_t i = 0; i < k; i++) { prob[i] = std::exp((double) (cand[i].first - cand[0].first)); sum += prob[i]; } size_t n_keep = k; double cum = 0.0; for (size_t i = 0; i < k; i++) { cum += prob[i]; if (i > 0 && cum > top_p * sum) { n_keep = i; break; } } double kept = 0.0; for (size_t i = 0; i < n_keep; i++) { prob[i] = std::exp((double) (cand[i].first - cand[0].first) / temp); kept += prob[i]; } double r = std::uniform_real_distribution(0.0, 1.0)(rng) * kept; for (size_t i = 0; i < n_keep; i++) { r -= prob[i]; if (r <= 0.0) { return cand[i].second; } } return cand[n_keep - 1].second; } }; // ZONOS3: one backbone row per frame of 10 codebooks, codebook j lags j frames class zonos3_gen_audio_pipeline : public mtmd_gen_audio_pipeline { public: using mtmd_gen_audio_pipeline::mtmd_gen_audio_pipeline; bool supports_delivery() const override { return true; } void reset() override { seq_id = 0; pos = 0; segments.clear(); seg_idx = 0; seg_pos = 0; rows.clear(); step = 0; eos_row = -1; countdown = 0; done = false; n_decoded = 0; n_codec_samples = 0; flushed = false; gen_state.reset(); pcm_sent = 0; wav_header_sent = false; audio_pcm.clear(); h_state_buf.clear(); out_buf.clear(); } int32_t set_input(const mtmd_helper_gen_audio_inp * inp) override { reset(); seq_id = inp->seq_id; out_type = inp->out_type; stream = inp->stream; // settings of the reference, only the temperature can be changed const mtmd_gen_inp def = mtmd_gen_inp_default(mctx); temp = inp->temp >= 0.0f ? inp->temp : def.temp; top_k = def.top_k; top_p = def.top_p; rep_pen = def.penalty_repeat; rep_win = def.penalty_last_n; rng.seed(inp->seed == UINT32_MAX ? std::random_device{}() : inp->seed); if (!init_gen_state(inp->seed)) { return 1; } // prompt: [voice rows] [condition rows] BOS text EOS [silence rows] // the speaker CFG of the reference is omitted: it adds too much code and maintenance cost if (inp->speaker_ref) { segment seg; if (!encode_speaker(inp->speaker_ref, seg.embd)) { return 1; } segments.push_back(std::move(seg)); } // condition rows: the bucket of each value over the checkpoint edges, or unknown auto bucket = [](const std::vector & edges, float v) { return std::to_string(std::upper_bound(edges.begin(), edges.end(), v) - edges.begin()); }; const std::string loudness = inp->loudness != 0.0f ? bucket(loudness_edges, inp->loudness) : "unk"; const std::string rate = inp->speed != 1.0f ? bucket(rate_edges, inp->speed * typical_rate) : "unk"; segment text; for (const std::string & name : { "<|loudness_" + loudness + "|>", std::string("<|fmax_unk|>"), "<|speaking_rate_" + rate + "|>" }) { const llama_token t = find_special_token(vocab, name); if (t == LLAMA_TOKEN_NULL) { LOG_ERR("mtmd_helper_gen_audio: missing ZONOS3 token %s in vocab\n", name.c_str()); return 1; } text.tokens.push_back(t); } const std::string prompt(inp->prompt, inp->prompt_len); std::vector ids(prompt.size() + 16); const int32_t n_ids = llama_tokenize(vocab, prompt.c_str(), (int32_t) prompt.size(), ids.data(), (int32_t) ids.size(), true, false); if (n_ids <= 2) { LOG_ERR("mtmd_helper_gen_audio: empty prompt or tokenization failed\n"); return 1; } text.tokens.insert(text.tokens.end(), ids.begin(), ids.begin() + n_ids); segments.push_back(std::move(text)); // codec silence, delayed segment silence; for (int k = 0; k < n_prefill; k++) { std::vector row(n_cb, audio_pad); for (int cb = 0; cb <= k && cb < n_cb; cb++) { row[cb] = silence_frame[cb]; } const float * embd = embed_row(row); if (!embd) { return 1; } silence.embd.insert(silence.embd.end(), embd, embd + n_embd); rows.insert(rows.end(), row.begin(), row.end()); } segments.push_back(std::move(silence)); size_t n_prompt = 0; for (const auto & seg : segments) { n_prompt += seg.n_rows(n_embd); } if (n_prompt >= llama_n_ctx_seq(lctx)) { LOG_ERR("mtmd_helper_gen_audio: prompt does not fit in the context\n"); return 1; } if (!llama_memory_seq_rm(llama_get_memory(lctx), seq_id, 0, -1)) { LOG_ERR("mtmd_helper_gen_audio: cannot reset the sequence\n"); return 1; } return 0; } int32_t step_prompt(int32_t n_batch) override { GGML_ASSERT(n_batch > 0); if (seg_idx >= segments.size()) { return 0; } const segment & seg = segments[seg_idx]; const int n_rows = (int) seg.n_rows(n_embd); const int n = std::min(n_batch, n_rows - seg_pos); const bool last = seg_idx + 1 == segments.size() && seg_pos + n == n_rows; int ret; if (!seg.tokens.empty()) { llama_batch_ext_ptr batch(llama_batch_ext_init(lctx)); if (!batch_set_tokens(batch.get(), seg.tokens.data() + seg_pos, n, pos, seq_id)) { return -1; } if (last) { llama_batch_ext_set_output_logits(batch.get(), n - 1, true); } ret = llama_process(lctx, LLAMA_PROCESS_TYPE_DECODE, batch.get()); } else { decode_embd_batch batch(seg.embd.data() + (size_t) seg_pos * n_embd, n, 1, n_embd); batch.set_position_normal(pos, seq_id); batch.logits[n - 1] = last; ret = llama_process(lctx, LLAMA_PROCESS_TYPE_DECODE, batch.render(lctx, 0, n)); } if (ret != 0) { LOG_ERR("mtmd_helper_gen_audio: prompt decode failed\n"); return -1; } pos += n; seg_pos += n; if (seg_pos == n_rows) { seg_idx++; seg_pos = 0; } int32_t n_left = 0; for (size_t i = seg_idx; i < segments.size(); i++) { n_left += (int32_t) segments[i].n_rows(n_embd) - (i == seg_idx ? seg_pos : 0); } if (n_left == 0) { const float * h = llama_get_embeddings_ith(lctx, -1); h_state_buf.assign(h, h + n_embd); } return n_left; } int32_t step_gen(llama_token sampled, const float * h_state_in, const float ** h_state_out, bool * out_stop) override { GGML_UNUSED(sampled); GGML_UNUSED(h_state_in); *h_state_out = nullptr; *out_stop = false; if (done) { *out_stop = true; return 0; } mtmd_gen_inp inp = mtmd_gen_inp_default(mctx); inp.type = MTMD_GEN_PROCESS_TYPE_GEN_CODE; inp.embd = h_state_buf.data(); mtmd_gen_out out{}; if (mtmd_gen_audio_process(mctx, gen_state.get(), &inp, &out) != 0 || out.n_feats != (size_t) n_cb * n_vocab) { LOG_ERR("mtmd_helper_gen_audio: codebook heads failed\n"); return 1; } std::vector frame(n_cb); for (int cb = 0; cb < n_cb; cb++) { frame[cb] = sample(out.feats + (size_t) cb * n_vocab, cb); } // codebooks whose delayed stream has not started yet are pad const int r = n_prefill + step; for (int cb = r + 1; cb < n_cb; cb++) { frame[cb] = audio_pad; } rows.insert(rows.end(), frame.begin(), frame.end()); step++; // after EOA, n_cb + 1 more rows flush the delay int max_eoa = -1; for (int cb = 0; cb < n_cb; cb++) { if (frame[cb] == audio_eoa) { max_eoa = cb; } } if (eos_row < 0 && max_eoa >= 0) { eos_row = std::max(0, r - max_eoa); countdown = n_cb + 1; } if ((countdown > 0 && --countdown == 0) || step >= max_frames || (uint32_t) pos >= llama_n_ctx_seq(lctx)) { done = true; *out_stop = true; return 0; } const float * next = embed_row(frame); if (!next) { return 1; } decode_embd_batch batch(next, 1, 1, n_embd); batch.set_position_normal(pos, seq_id); batch.logits[0] = 1; if (llama_process(lctx, LLAMA_PROCESS_TYPE_DECODE, batch.render(lctx, 0, 1)) != 0) { LOG_ERR("mtmd_helper_gen_audio: decode failed\n"); return 1; } pos++; const float * h = llama_get_embeddings_ith(lctx, -1); h_state_buf.assign(h, h + n_embd); *h_state_out = h_state_buf.data(); if (stream) { return decode_stream() ? 0 : 1; } return 0; } int32_t get_output(int32_t * out_sample_rate, const char ** out_data, size_t * out_data_len, int64_t * out_n_samples) override { *out_sample_rate = info.sample_rate; if (!stream) { if (flush() != 0) { return 1; } if (out_n_samples) { *out_n_samples = (int64_t) audio_pcm.size(); } if (out_type == MTMD_HELPER_GEN_AUDIO_OUTTYPE_PCM) { *out_data = (const char *) audio_pcm.data(); *out_data_len = audio_pcm.size() * sizeof(float); return 0; } out_buf.clear(); if (!write_wav16(out_buf, audio_pcm, info.sample_rate)) { LOG_ERR("mtmd_helper_gen_audio: output too large for WAV\n"); return 1; } *out_data = out_buf.data(); *out_data_len = out_buf.size(); return 0; } // streaming: only the audio since the previous call const size_t n_new = audio_pcm.size() - pcm_sent; if (out_n_samples) { *out_n_samples = (int64_t) n_new; } if (out_type == MTMD_HELPER_GEN_AUDIO_OUTTYPE_PCM) { *out_data = (const char *) (audio_pcm.data() + pcm_sent); *out_data_len = n_new * sizeof(float); pcm_sent = audio_pcm.size(); return 0; } out_buf.clear(); if (!wav_header_sent) { write_wav16_header(out_buf, UINT32_MAX, info.sample_rate); wav_header_sent = true; } append_wav16_pcm(out_buf, audio_pcm.data() + pcm_sent, n_new); pcm_sent = audio_pcm.size(); *out_data = out_buf.data(); *out_data_len = out_buf.size(); return 0; } int32_t flush() override { return decode_wav(true) ? 0 : 1; } private: static constexpr int n_cb = 10; static constexpr int n_vocab = 4098; // codes, EOA, pad static constexpr int audio_eoa = 4096; static constexpr int audio_pad = 4097; // condition_edges of params.json; speed 1 is this many UTF-8 bytes per second, the middle of the rate buckets const std::vector loudness_edges = { -29.778f, -26.947f, -25.047f, -23.3f, -21.743f, -20.386f, -19.186f, -17.851f, -15.967f }; const std::vector rate_edges = { 6.546f, 8.182f, 9.273f, 10.31f, 11.054f, 11.667f, 12.193f, 12.656f, 13.083f, 13.476f, 13.846f, 14.196f, 14.53f, 14.887f, 15.193f, 15.517f, 15.865f, 16.208f, 16.579f, 16.972f, 17.411f, 17.898f, 18.434f, 19.069f, 19.818f, 20.816f, 22.286f, 24.722f, 29.25f, }; static constexpr float typical_rate = 15.0f; // digital silence of the ZAC codec (registries/silence_frame_*.json) static constexpr int32_t silence_frame[n_cb] = { 1918, 746, 3544, 234, 1267, 2914, 3944, 2849, 326, 1337 }; static constexpr int n_prefill = 3; // -3 dB of headroom static constexpr int samples_per_frame = 2940; static constexpr float out_gain = 0.70794578f; static constexpr int rep_codebooks = 6; static constexpr int max_frames = 2000; static constexpr size_t stream_frames = 8; struct segment { std::vector tokens; // either tokens or embd rows std::vector embd; size_t n_rows(int n) const { return tokens.empty() ? embd.size() / (size_t) n : tokens.size(); } }; const float * embed_row(const std::vector & codes) { mtmd_gen_inp inp = mtmd_gen_inp_default(mctx); inp.type = MTMD_GEN_PROCESS_TYPE_GEN_CODE; inp.codes = const_cast(codes.data()); inp.n_codes = codes.size(); mtmd_gen_out out{}; if (mtmd_gen_audio_process(mctx, gen_state.get(), &inp, &out) != 0) { LOG_ERR("mtmd_helper_gen_audio: codebook embedding failed\n"); return nullptr; } return out.embd; } // same as the reference sampler (servers/ggml/src/zonos3-sampler.cpp) int32_t sample(const float * in, int cb) { std::vector l(in, in + n_vocab); l[audio_pad] = -INFINITY; if (rep_pen != 1.0f && cb < rep_codebooks) { const size_t n_rows = rows.size() / n_cb - n_prefill; const size_t n_win = std::min(n_rows, (size_t) std::max(rep_win, 0)); std::vector seen(audio_eoa, false); for (size_t f = n_prefill + n_rows - n_win; f < n_prefill + n_rows; f++) { const int32_t c = rows[f * n_cb + (size_t) cb]; if (c >= 0 && c < audio_eoa && !seen[(size_t) c]) { seen[(size_t) c] = true; l[c] = l[c] > 0.0f ? l[c] / rep_pen : l[c] * rep_pen; } } } if (temp <= 0.0f) { return (int32_t) (std::max_element(l.begin(), l.end()) - l.begin()); } for (auto & v : l) { v /= temp; } float kth = -INFINITY; if (top_k > 0 && top_k < n_vocab) { std::vector tmp(l); std::nth_element(tmp.begin(), tmp.begin() + (top_k - 1), tmp.end(), std::greater()); kth = tmp[(size_t) top_k - 1]; } std::vector idx; for (int32_t i = 0; i < n_vocab; i++) { if (l[i] >= kth && l[i] > -INFINITY) { idx.push_back(i); } } float lmax = -INFINITY; for (int32_t i : idx) { lmax = std::max(lmax, l[i]); } std::vector p(idx.size()); double sum = 0.0; for (size_t j = 0; j < idx.size(); j++) { p[j] = std::exp((double) (l[idx[j]] - lmax)); sum += p[j]; } std::vector ord(idx.size()); for (size_t j = 0; j < ord.size(); j++) { ord[j] = j; } std::sort(ord.begin(), ord.end(), [&](size_t a, size_t b) { return p[a] > p[b]; }); if (top_p > 0.0f && top_p < 1.0f) { double before = 0.0; for (size_t j : ord) { const double pj = p[j]; if (before > top_p * sum) { p[j] = 0.0; } before += pj; } } double kept = 0.0; for (double v : p) { kept += v; } double u = std::uniform_real_distribution(0.0, kept)(rng); for (size_t j = 0; j < p.size(); j++) { u -= p[j]; if (u <= 0.0 && p[j] > 0.0) { return idx[j]; } } return idx[ord[0]]; } // frames with all codebooks known size_t n_ready() const { const size_t n_rows = rows.size() / n_cb; size_t n = n_rows >= (size_t) n_cb ? n_rows - (n_cb - 1) : 0; if (eos_row >= 0) { n = std::min(n, (size_t) eos_row); } return n; } // streaming, as the reference does for the first audio: the silence frames are decoded with silence codes only, // then the first generated frame from its first code alone; later chunks of complete frames grow 2, 4, 8 bool decode_stream() { if (n_decoded == 0) { const size_t n_rows = rows.size() / n_cb; if (n_rows <= (size_t) n_prefill || rows[(size_t) n_prefill * n_cb] >= audio_eoa) { return true; } std::vector codes; for (int k = 0; k < n_prefill; k++) { codes.insert(codes.end(), silence_frame, silence_frame + n_cb); } for (int cb = 0; cb < n_cb; cb++) { const size_t r = (size_t) n_prefill + cb; const int32_t c = r < n_rows ? rows[r * n_cb + cb] : -1; codes.push_back(c >= audio_eoa ? -1 : c); } if (!run_codec(codes.data(), codes.size())) { return false; } n_decoded = n_prefill + 1; return true; } const size_t n_chunk = std::min(stream_frames, n_decoded - n_prefill + 1); if (n_ready() >= n_decoded + n_chunk) { return decode_wav(false); } return true; } // the silence frames are decoded too, as left context bool decode_wav(bool final) { const size_t n_frames = n_ready(); std::vector codes; while (n_decoded < n_frames) { const size_t n = std::min(n_frames - n_decoded, (size_t) 64); codes.resize(n * n_cb); for (size_t f = 0; f < n; f++) { for (int cb = 0; cb < n_cb; cb++) { // EOA and pad have no codec row const int32_t c = rows[(n_decoded + f + cb) * n_cb + cb]; codes[f * n_cb + cb] = std::clamp(c, 0, audio_eoa - 1); } } if (!run_codec(codes.data(), codes.size())) { return false; } n_decoded += n; } if (final && done && n_decoded > 0 && !flushed) { flushed = true; return run_codec(nullptr, 0); } return true; } bool run_codec(const int32_t * codes, size_t n_codes) { mtmd_gen_inp inp = mtmd_gen_inp_default(mctx); inp.type = MTMD_GEN_PROCESS_TYPE_GEN_WAV; inp.codes = const_cast(codes); inp.n_codes = n_codes; mtmd_gen_out out{}; if (mtmd_gen_audio_process(mctx, gen_state.get(), &inp, &out) != 0) { LOG_ERR("mtmd_helper_gen_audio: codec decode failed\n"); return false; } // drop the audio of the silence frames, then the output gain and soft limiter of the reference const float knee = std::pow(10.0f, -1.0f / 20.0f); for (size_t i = 0; i < out.n_samples; i++, n_codec_samples++) { if (n_codec_samples < (size_t) n_prefill * samples_per_frame) { continue; } const float x = out.audio[i] * out_gain; const float a = std::fabs(x); audio_pcm.push_back(a <= knee ? x : std::copysign(knee + (1.0f - knee) * std::tanh((a - knee) / (1.0f - knee)), x)); } return true; } std::vector segments; size_t seg_idx = 0; int seg_pos = 0; std::vector rows; // delayed rows: silence rows, then the generated ones int step = 0; int eos_row = -1; int countdown = 0; bool done = false; size_t n_decoded = 0; size_t n_codec_samples = 0; bool flushed = false; float temp = 0.75f; int32_t top_k = 80; float top_p = 0.69f; float rep_pen = 2.4f; int32_t rep_win = 96; std::mt19937 rng; llama_seq_id seq_id = 0; llama_pos pos = 0; mtmd_helper_gen_audio_outtype out_type = MTMD_HELPER_GEN_AUDIO_OUTTYPE_WAV; bool stream = false; std::vector h_state_buf; std::vector audio_pcm; std::vector out_buf; size_t pcm_sent = 0; bool wav_header_sent = false; }; // YuE2: the backbone writes an ABC score (text), then the semantic codes; both are sampled here like the reference (sampling.py) // the NAR flow matching (in mtmd, on the backbone keys/values) and the VAE then make the stereo PCM class yue2_gen_audio_pipeline : public mtmd_gen_audio_pipeline { public: using mtmd_gen_audio_pipeline::mtmd_gen_audio_pipeline; bool supports_cfg() const override { return true; } bool supports_flow_steps() const override { return true; } // step_gen() stops at the score and code limits of the reference int32_t max_frames() const override { return 0; } // the limits and the NAR chunks of the reference need the training context (24576 positions) bool check_context() const override { if (llama_vocab_n_tokens(vocab) < tok_codec0 + n_codec) { LOG_ERR("mtmd_helper_gen_audio: the vocab of the backbone has no yue2 codec tokens\n"); return false; } const uint32_t n_ctx_train = (uint32_t) llama_model_n_ctx_train(model); if (llama_n_ctx_seq(lctx) < n_ctx_train) { LOG_ERR("mtmd_helper_gen_audio: yue2 needs a context of %u positions per sequence, got %u (use the default context size)\n", n_ctx_train, llama_n_ctx_seq(lctx)); return false; } return true; } void reset() override { seq_id = 0; pos = 0; prompt.clear(); prompt_pos = 0; in_codes = false; mode_off = false; cfg_scale = 1.0f; seq_uncond = 0; upos = 0; neg_prefix.clear(); abc_tail.clear(); prefix_tok.clear(); history.clear(); codes.clear(); latents.clear(); n_nar_done = 0; n_vae_done = 0; scratch.reset(); scratch_n_ctx = 0; done = false; decoded = false; audio_pcm.clear(); h_state_buf.clear(); out_buf.clear(); stream = false; pcm_sent = 0; wav_header_sent = false; gen_state.reset(); } int32_t set_input(const mtmd_helper_gen_audio_inp * inp) override { reset(); seq_id = inp->seq_id; out_type = inp->out_type; stream = inp->stream; code_cfg.temp = inp->temp >= 0.0f ? inp->temp : 1.0f; flow_steps = inp->flow_steps; rng.seed(inp->seed == UINT32_MAX ? std::random_device{}() : inp->seed); if (!init_gen_state(inp->seed)) { return 1; } // the "tts" template renders the whole prompt, special tokens included const std::string text(inp->prompt, inp->prompt_len); prompt.resize(text.size() + 16); const int32_t n = llama_tokenize(vocab, text.c_str(), (int32_t) text.size(), prompt.data(), (int32_t) prompt.size(), false, true); if (n < 2 || prompt[0] != tok_eod) { LOG_ERR("mtmd_helper_gen_audio: yue2 expects the prompt of its tts template\n"); return 1; } prompt.resize((size_t) n); // cover mode and cot=off give the score in the prompt, the codes come next auto abc = std::find(prompt.begin(), prompt.end(), tok_abc_start); if (prompt.back() == tok_music_start && abc != prompt.end()) { in_codes = true; mode_off = prompt.end() - abc == 3; // empty score } else if (prompt.back() != tok_abc_start) { LOG_ERR("mtmd_helper_gen_audio: yue2 prompt must end with or <|music_start|>\n"); return 1; } abc_tail.assign(abc, prompt.end()); prefix_tok = prompt; if (prompt.size() >= llama_n_ctx_seq(lctx)) { LOG_ERR("mtmd_helper_gen_audio: prompt does not fit in the context\n"); return 1; } llama_memory_t mem = llama_get_memory(lctx); if (!llama_memory_seq_rm(mem, seq_id, 0, -1)) { LOG_ERR("mtmd_helper_gen_audio: cannot reset the sequence\n"); return 1; } // classifier-free guidance of the codes, defaults of the reference (protocol.py) cfg_scale = inp->cfg_scale > 0.0f ? inp->cfg_scale : mode_off ? 1.01f : 1.0f; if (cfg_scale != 1.0f) { // the uncond sequence lives half a seq space away from the cond one const int32_t n_seq_max = (int32_t) llama_n_seq_max(lctx); seq_uncond = (llama_seq_id) (n_seq_max / 2 + seq_id); if (!llama_model_classifier_free_guidance(model) || seq_id >= n_seq_max / 2) { if (inp->cfg_scale > 0.0f) { LOG_ERR("mtmd_helper_gen_audio: cfg_scale needs a paired sequence (n_seq_max = %d, seq_id = %d), reconvert the model\n", n_seq_max, seq_id); return 1; } LOG_WRN("mtmd_helper_gen_audio: no paired cfg sequence (n_seq_max = %d, seq_id = %d), cfg is disabled\n", n_seq_max, seq_id); cfg_scale = 1.0f; } else if (!llama_memory_seq_rm(mem, seq_uncond, 0, -1)) { LOG_ERR("mtmd_helper_gen_audio: cannot reset the cfg sequence\n"); return 1; } } if (cfg_scale != 1.0f) { // the negative prompt is the bare instruction, tokenized alone like the reference const size_t i0 = text.find("<|endoftext|>"); const size_t i1 = text.find('\n'); if (i0 != 0 || i1 == std::string::npos) { LOG_ERR("mtmd_helper_gen_audio: yue2 expects the prompt of its tts template\n"); return 1; } const std::string instr = text.substr(13, i1 - 13); neg_prefix.resize(instr.size() + 16); const int32_t n_instr = llama_tokenize(vocab, instr.c_str(), (int32_t) instr.size(), neg_prefix.data() + 1, (int32_t) neg_prefix.size() - 1, false, false); if (n_instr < 1) { LOG_ERR("mtmd_helper_gen_audio: tokenization failed\n"); return 1; } neg_prefix[0] = tok_eod; neg_prefix.resize((size_t) n_instr + 1); } return 0; } int32_t step_prompt(int32_t n_batch) override { GGML_ASSERT(n_batch > 0); const int32_t n_left = (int32_t) prompt.size() - prompt_pos; if (n_left <= 0) { return 0; } const int32_t n = std::min(n_batch, n_left); if (!decode(prompt.data() + prompt_pos, n)) { return -1; } prompt_pos += n; if (n == n_left) { if (in_codes) { if (!start_uncond()) { return -1; } } else { save_logits(llama_get_logits_ith(lctx, -1)); } } return n_left - n; } int32_t step_gen(llama_token sampled, const float * h_state_in, const float ** h_state_out, bool * out_stop) override { GGML_UNUSED(sampled); GGML_UNUSED(h_state_in); *h_state_out = nullptr; *out_stop = false; if (done) { *out_stop = true; return 0; } const bool use_cfg = in_codes && cfg_scale != 1.0f; const llama_token tok = sample(logits.data()); int32_t i_out = -1; // output row of the cond sequence if (!in_codes) { // the score ends with , or is cut at max_tokens like the reference if (tok == tok_abc_end || (int32_t) history.size() + 1 >= abc_cfg.max_tokens) { const llama_token next[2] = { tok_abc_end, tok_music_start }; if (!decode(next, 2)) { return 1; } abc_tail.insert(abc_tail.end(), next, next + 2); prefix_tok.insert(prefix_tok.end(), next, next + 2); in_codes = true; history.clear(); if (!start_uncond()) { return 1; } } else { history.push_back(tok); abc_tail.push_back(tok); prefix_tok.push_back(tok); if (!decode(&tok, 1)) { return 1; } save_logits(llama_get_logits_ith(lctx, -1)); } } else { // the NAR pass also attends to <|music_end|>, the uncond sequence shares the unified KV cache const bool full = (int32_t) codes.size() + 1 >= code_cfg.max_tokens || pos + upos + 2 >= (llama_pos) llama_n_ctx_seq(lctx); if (tok == tok_music_end || full) { if (!decode(&tok_music_end, 1)) { return 1; } done = true; *out_stop = true; return 0; } history.push_back(tok); codes.push_back(tok - tok_codec0); if (use_cfg) { // the sampled code feeds both branches, in one decode if (!decode_pair(tok)) { return 1; } i_out = 0; } else { if (!decode(&tok, 1)) { return 1; } save_logits(llama_get_logits_ith(lctx, -1)); } } const float * h = llama_get_embeddings_ith(lctx, i_out); h_state_buf.assign(h, h + n_embd); *h_state_out = h_state_buf.data(); // streaming: each full chunk of codes goes through the NAR pass and the VAE right away if (stream && in_codes && codes.size() - n_nar_done >= chunk_size()) { if (!nar_chunk(n_nar_done, n_nar_done + chunk_size(), false) || !vae_emit(false)) { return 1; } } return 0; } int32_t get_output(int32_t * out_sample_rate, const char ** out_data, size_t * out_data_len, int64_t * out_n_samples) override { *out_sample_rate = info.sample_rate; const uint16_t n_ch = (uint16_t) info.n_channels; if (!stream) { if (flush() != 0) { return 1; } if (out_n_samples) { *out_n_samples = (int64_t) (audio_pcm.size() / n_ch); } if (out_type == MTMD_HELPER_GEN_AUDIO_OUTTYPE_PCM) { *out_data = (const char *) audio_pcm.data(); *out_data_len = audio_pcm.size() * sizeof(float); return 0; } out_buf.clear(); if (!write_wav16(out_buf, audio_pcm, info.sample_rate, n_ch)) { LOG_ERR("mtmd_helper_gen_audio: output too large for WAV\n"); return 1; } *out_data = out_buf.data(); *out_data_len = out_buf.size(); return 0; } const size_t n_new = audio_pcm.size() - pcm_sent; if (out_n_samples) { *out_n_samples = (int64_t) (n_new / n_ch); } if (out_type == MTMD_HELPER_GEN_AUDIO_OUTTYPE_PCM) { *out_data = (const char *) (audio_pcm.data() + pcm_sent); *out_data_len = n_new * sizeof(float); pcm_sent = audio_pcm.size(); return 0; } out_buf.clear(); if (!wav_header_sent) { write_wav16_header(out_buf, UINT32_MAX, info.sample_rate, n_ch); wav_header_sent = true; } append_wav16_pcm(out_buf, audio_pcm.data() + pcm_sent, n_new); pcm_sent = audio_pcm.size(); *out_data = out_buf.data(); *out_data_len = out_buf.size(); return 0; } int32_t flush() override { if (decoded) { return 0; } if (codes.empty()) { LOG_ERR("mtmd_helper_gen_audio: yue2 stopped before the first code\n"); return 1; } if (!done) { // stopped by the caller: the NAR pass still needs <|music_end|> if (!decode(&tok_music_end, 1)) { return 1; } done = true; } // a song that fits in one chunk uses the backbone sequence itself, as it holds <|music_end|> now const size_t n_codes = codes.size(); while (n_nar_done < n_codes) { const size_t a = n_nar_done; const size_t b = std::min(a + chunk_size(), n_codes); if (!nar_chunk(a, b, a == 0 && b == n_codes)) { return 1; } } if (!vae_emit(true)) { return 1; } scratch.reset(); decoded = true; return 0; } private: // token ids of the yue2_infer protocol (protocol.py) static constexpr llama_token tok_eod = 151643; static constexpr llama_token tok_abc_start = 151847; static constexpr llama_token tok_abc_end = 151848; static constexpr llama_token tok_music_start = 151851; static constexpr llama_token tok_music_end = 151852; static constexpr llama_token tok_codec0 = 151853; static constexpr int32_t n_codec = 32768; // NAR chunks and VAE tiles, see synthesize() and decode_tiled() of the reference static constexpr int64_t n_ctx_max = 24576; static constexpr size_t stream_chunk = 750; // 30 s static constexpr size_t n_latent = 64; static constexpr size_t vae_core = 128; // frames kept per tile static constexpr size_t vae_halo = 16; // frames of context on each side static constexpr size_t vae_ratio = 1920; static constexpr size_t vae_trim = 64; struct sampling { float temp; float top_p; int32_t top_k; float penalty; int32_t penalty_window; int32_t min_tokens; int32_t max_tokens; }; sampling abc_cfg = { 0.7f, 0.9f, 30, 1.005f, 100, 32, 4096 }; sampling code_cfg = { 1.0f, 0.95f, 100, 1.2f, 50, 200, 9000 }; bool decode_ctx(llama_context * ctx, const llama_token * toks, int32_t n, llama_seq_id seq, llama_pos & p) { llama_batch_ext_ptr batch(llama_batch_ext_init(ctx)); if (!batch_set_tokens(batch.get(), toks, n, p, seq)) { return false; } llama_batch_ext_set_output_logits(batch.get(), n - 1, true); if (llama_process(ctx, LLAMA_PROCESS_TYPE_DECODE, batch.get()) != 0) { LOG_ERR("mtmd_helper_gen_audio: decode failed\n"); return false; } p += n; return true; } bool decode(const llama_token * toks, int32_t n) { return decode_ctx(lctx, toks, n, seq_id, pos); } // keeps the logits of the next token, as the context logits belong to the last decode of any sequence (server slots) // only the window that sample() reads: the text and for the score, <|music_end|> and the codes after it // with guidance: uncond + scale * (cond - uncond) void save_logits(const float * cond, const float * uncond = nullptr) { const llama_token lo = in_codes ? tok_music_end : 0; const llama_token hi = in_codes ? tok_codec0 + n_codec : tok_abc_end + 1; logits.resize((size_t) llama_vocab_n_tokens(vocab)); for (llama_token t = lo; t < hi; t++) { logits[t] = uncond ? uncond[t] + cfg_scale * (cond[t] - uncond[t]) : cond[t]; } } // the negative prompt of the reference (negative_prefix): the instruction, then the score of the cond prompt // the logits of the cond prompt must be the last ones when this is called bool start_uncond() { if (cfg_scale == 1.0f) { save_logits(llama_get_logits_ith(lctx, -1)); return true; } const float * c = llama_get_logits_ith(lctx, -1); std::vector cond(c, c + llama_vocab_n_tokens(vocab)); std::vector neg = neg_prefix; if (mode_off) { neg.push_back(tok_music_start); } else { neg.insert(neg.end(), abc_tail.begin(), abc_tail.end()); } if (pos + (llama_pos) neg.size() + 2 >= (llama_pos) llama_n_ctx_seq(lctx)) { LOG_ERR("mtmd_helper_gen_audio: the cfg prompt does not fit in the context\n"); return false; } const int32_t n_batch = (int32_t) llama_n_batch(lctx); for (size_t i = 0; i < neg.size(); i += (size_t) n_batch) { const int32_t n = std::min(n_batch, (int32_t) (neg.size() - i)); if (!decode_ctx(lctx, neg.data() + i, n, seq_uncond, upos)) { return false; } } save_logits(cond.data(), llama_get_logits_ith(lctx, -1)); return true; } bool decode_pair(llama_token tok) { llama_batch_ext_ptr batch(llama_batch_ext_init(lctx)); const llama_seq_id seqs[2] = { seq_id, seq_uncond }; const llama_pos p[2] = { pos, upos }; for (int i = 0; i < 2; i++) { const int32_t idx = llama_batch_ext_add_token(batch.get(), seqs[i], tok); if (idx < 0) { LOG_ERR("mtmd_helper_gen_audio: failed to add token %d to batch (error %d)\n", tok, idx); return false; } llama_batch_ext_set_pos(batch.get(), idx, &p[i]); llama_batch_ext_set_output_logits(batch.get(), idx, true); } if (llama_process(lctx, LLAMA_PROCESS_TYPE_DECODE, batch.get()) != 0) { LOG_ERR("mtmd_helper_gen_audio: decode failed\n"); return false; } pos++; upos++; save_logits(llama_get_logits_ith(lctx, 0), llama_get_logits_ith(lctx, 1)); return true; } // song_chunks() of the reference: each NAR chunk sees the prefix and its own codes // the reference uses the largest chunk that fits in n_ctx_max, streaming uses smaller ones size_t chunk_size() const { const int64_t ref = std::max(1, (n_ctx_max - (int64_t) prefix_tok.size() - 3) / 2); return (size_t) (stream ? std::min(ref, stream_chunk) : ref); } // backbone keys/values of [prefix, codes[a, b), <|music_end|>] -> latents of the chunk bool nar_chunk(size_t a, size_t b, bool use_main) { llama_context * ctx = lctx; llama_seq_id seq = seq_id; llama_pos n_past = pos; if (!use_main) { // a scratch context on the backbone model, the prefix is decoded once const llama_pos n_prefix = (llama_pos) prefix_tok.size(); const uint32_t n_need = (uint32_t) (n_prefix + (b - a) + 1); if (!scratch || scratch_n_ctx < n_need) { llama_context_params cparams = llama_context_default_params(); cparams.n_ctx = n_need; cparams.n_seq_max = 1; cparams.n_threads = llama_n_threads(lctx); cparams.n_threads_batch = llama_n_threads_batch(lctx); cparams.no_perf = true; scratch.reset(llama_init_from_model(const_cast(model), cparams)); if (!scratch) { LOG_ERR("mtmd_helper_gen_audio: cannot create the yue2 chunk context\n"); return false; } scratch_n_ctx = n_need; llama_pos p = 0; const int32_t n_batch = (int32_t) llama_n_batch(scratch.get()); for (size_t i = 0; i < prefix_tok.size(); i += (size_t) n_batch) { const int32_t n = std::min(n_batch, (int32_t) (prefix_tok.size() - i)); if (!decode_ctx(scratch.get(), prefix_tok.data() + i, n, 0, p)) { return false; } } } llama_memory_seq_rm(llama_get_memory(scratch.get()), 0, n_prefix, -1); std::vector toks; for (size_t i = a; i < b; i++) { toks.push_back(codes[i] + tok_codec0); } toks.push_back(tok_music_end); llama_pos p = n_prefix; const int32_t n_batch = (int32_t) llama_n_batch(scratch.get()); for (size_t i = 0; i < toks.size(); i += (size_t) n_batch) { const int32_t n = std::min(n_batch, (int32_t) (toks.size() - i)); if (!decode_ctx(scratch.get(), toks.data() + i, n, 0, p)) { return false; } } ctx = scratch.get(); seq = 0; n_past = p; } std::vector state(llama_state_seq_get_size(ctx, seq)); if (state.empty() || llama_state_seq_get_data(ctx, state.data(), state.size(), seq) != state.size()) { LOG_ERR("mtmd_helper_gen_audio: cannot read the backbone state\n"); return false; } mtmd_gen_inp inp = mtmd_gen_inp_default(mctx); inp.type = MTMD_GEN_PROCESS_TYPE_GEN_CODE; inp.n_codes = b - a; inp.backbone_state = state.data(); inp.backbone_state_size = state.size(); inp.backbone_n_past = n_past; inp.n_steps = flow_steps; mtmd_gen_out out{}; if (mtmd_gen_audio_process(mctx, gen_state.get(), &inp, &out) != 0) { LOG_ERR("mtmd_helper_gen_audio: flow matching failed\n"); return false; } latents.insert(latents.end(), out.feats, out.feats + out.n_feats); n_nar_done = b; return true; } // decodes the VAE tiles whose context on both sides is known, same cropping as decode_tiled() of the reference bool vae_emit(bool final) { const size_t n_lat = latents.size() / n_latent; while (n_vae_done < n_lat) { const size_t start = n_vae_done; const size_t end = std::min(start + vae_core, n_lat); if (!final && end + vae_halo > n_lat) { break; } const size_t left = start >= vae_halo ? start - vae_halo : 0; const size_t right = std::min(end + vae_halo, n_lat); mtmd_gen_inp inp = mtmd_gen_inp_default(mctx); inp.type = MTMD_GEN_PROCESS_TYPE_GEN_WAV; inp.feats = latents.data() + left * n_latent; inp.n_feats = (right - left) * n_latent; mtmd_gen_out out{}; if (mtmd_gen_audio_process(mctx, gen_state.get(), &inp, &out) != 0) { LOG_ERR("mtmd_helper_gen_audio: VAE decode failed\n"); return false; } // the whole song has vae_ratio * n - vae_trim samples const size_t n_ch = (size_t) info.n_channels; const size_t crop0 = (start - left) * vae_ratio; const size_t n_keep = (end - start) * vae_ratio - (final && end == n_lat ? vae_trim : 0); if (out.n_samples < (crop0 + n_keep) * n_ch) { LOG_ERR("mtmd_helper_gen_audio: VAE tile is too short\n"); return false; } audio_pcm.insert(audio_pcm.end(), out.audio + crop0 * n_ch, out.audio + (crop0 + n_keep) * n_ch); n_vae_done = end; } return true; } // distribution() of the reference: allowed range, no end token before min_tokens, windowed penalty, top-k, top-p llama_token sample(const float * logits) { const sampling & c = in_codes ? code_cfg : abc_cfg; const llama_token end = in_codes ? tok_music_end : tok_abc_end; const llama_token lo = in_codes ? tok_codec0 : 0; const llama_token hi = in_codes ? tok_codec0 + n_codec : tok_eod; std::unordered_map freq; const size_t n_win = std::min(history.size(), (size_t) c.penalty_window); for (size_t i = history.size() - n_win; i < history.size(); i++) { freq[history[i]]++; } std::vector> cand; cand.reserve((size_t) (hi - lo) + 1); auto push = [&](llama_token t) { float l = logits[t]; auto it = freq.find(t); if (it != freq.end()) { const float alpha = std::pow(c.penalty, (float) it->second); l = l < 0.0f ? l * alpha : l / alpha; } cand.emplace_back(l, t); }; for (llama_token t = lo; t < hi; t++) { push(t); } if ((int32_t) history.size() >= c.min_tokens) { push(end); } if (c.temp <= 0.0f) { return std::max_element(cand.begin(), cand.end())->second; } const size_t k = std::min(cand.size(), (size_t) std::max(c.top_k, 1)); std::partial_sort(cand.begin(), cand.begin() + k, cand.end(), std::greater>()); cand.resize(k); std::vector p(k); double sum = 0.0; for (size_t i = 0; i < k; i++) { p[i] = std::exp((double) ((cand[i].first - cand[0].first) / c.temp)); sum += p[i]; } // keep the smallest prefix whose mass reaches top_p, the first token always stays (3 for the codes of cot=off) const size_t n_keep = in_codes && mode_off ? 3 : 1; double before = 0.0; for (size_t i = 0; i < k; i++) { const double pi = p[i]; if (i >= n_keep && before > c.top_p * sum) { p[i] = 0.0; } before += pi; } double kept = 0.0; for (double v : p) { kept += v; } double u = std::uniform_real_distribution(0.0, kept)(rng); for (size_t i = 0; i < k; i++) { u -= p[i]; if (u <= 0.0 && p[i] > 0.0) { return cand[i].second; } } return cand[0].second; } llama_seq_id seq_id = 0; llama_pos pos = 0; std::vector prompt; int32_t prompt_pos = 0; bool in_codes = false; bool mode_off = false; // cot=off: no score float cfg_scale = 1.0f; int32_t flow_steps = 0; // 0 = the model default llama_seq_id seq_uncond = 0; llama_pos upos = 0; std::vector neg_prefix; // EOD + instruction std::vector abc_tail; // the cond tokens from on std::vector logits; // see save_logits() std::vector prefix_tok; // the cond tokens before the first code std::vector latents; // of the chunks done so far size_t n_nar_done = 0; // codes size_t n_vae_done = 0; // latent frames llama_context_ptr scratch; // backbone keys/values of a chunk other than the first uint32_t scratch_n_ctx = 0; std::vector history; // tokens sampled in the current phase std::vector codes; bool done = false; bool decoded = false; std::mt19937 rng; mtmd_helper_gen_audio_outtype out_type = MTMD_HELPER_GEN_AUDIO_OUTTYPE_WAV; bool stream = false; std::vector h_state_buf; std::vector audio_pcm; // interleaved std::vector out_buf; size_t pcm_sent = 0; bool wav_header_sent = false; }; // MiniMax Music 3: the backbone samples the semantic code of each frame with guidance, the depth decoder (mtmd) the other codes // the condition rows of the frames go to the flow matching and the vocoder (mtmd) in windows, like the reference (diffusers pipeline) class music3_gen_audio_pipeline : public mtmd_gen_audio_pipeline { public: using mtmd_gen_audio_pipeline::mtmd_gen_audio_pipeline; bool supports_cfg() const override { return true; } bool supports_flow_steps() const override { return true; } // step_gen() stops at the frame limit of the reference int32_t max_frames() const override { return 0; } bool check_context() const override { if (llama_vocab_n_tokens(vocab) < tok_code0 + n_semantic) { LOG_ERR("mtmd_helper_gen_audio: the vocab of the backbone has no minimax_music3 audio codes\n"); return false; } return true; } void reset() override { seq_id = 0; seq_uncond = 0; use_cfg = false; cfg_scale = 1.0f; temp = 1.0f; flow_steps = 0; pos = 0; prompt.clear(); prompt_uncond.clear(); prompt_pos = 0; uprompt_pos = 0; logits_cond.clear(); logits_uncond.clear(); h_rows.clear(); n_gen = 0; cond_rows.clear(); n_rows_dropped = 0; n_win_done = 0; done = false; decoded = false; audio_pcm.clear(); out_buf.clear(); stream = false; pcm_sent = 0; wav_header_sent = false; gen_state.reset(); } int32_t set_input(const mtmd_helper_gen_audio_inp * inp) override { reset(); seq_id = inp->seq_id; out_type = inp->out_type; stream = inp->stream; temp = inp->temp >= 0.0f ? inp->temp : 1.0f; flow_steps = inp->flow_steps; cfg_scale = inp->cfg_scale > 0.0f ? inp->cfg_scale : 1.5f; rng.seed(inp->seed == UINT32_MAX ? std::random_device{}() : inp->seed); if (!init_gen_state(inp->seed)) { return 1; } // the "tts" template renders the whole prompt, special tokens included const std::string text(inp->prompt, inp->prompt_len); prompt.resize(text.size() + 16); const int32_t n = llama_tokenize(vocab, text.c_str(), (int32_t) text.size(), prompt.data(), (int32_t) prompt.size(), false, true); if (n < 3 || prompt[0] != tok_im_start || prompt[n - 1] != tok_audio_start) { LOG_ERR("mtmd_helper_gen_audio: minimax_music3 expects the prompt of its tts template\n"); return 1; } prompt.resize((size_t) n); if (n > max_prompt) { LOG_ERR("mtmd_helper_gen_audio: the prompt has %d tokens, the maximum is %d\n", n, max_prompt); return 1; } llama_memory_t mem = llama_get_memory(lctx); if (!llama_memory_seq_rm(mem, seq_id, 0, -1)) { LOG_ERR("mtmd_helper_gen_audio: cannot reset the sequence\n"); return 1; } use_cfg = cfg_scale != 1.0f; if (use_cfg) { // the uncond sequence lives half a seq space away from the cond one const int32_t n_seq_max = (int32_t) llama_n_seq_max(lctx); seq_uncond = (llama_seq_id) (n_seq_max / 2 + seq_id); if (!llama_model_classifier_free_guidance(model) || seq_id >= n_seq_max / 2) { LOG_ERR("mtmd_helper_gen_audio: cfg needs a paired sequence (n_seq_max = %d, seq_id = %d), reconvert the model or use cfg_scale = 1\n", n_seq_max, seq_id); return 1; } if (!llama_memory_seq_rm(mem, seq_uncond, 0, -1)) { LOG_ERR("mtmd_helper_gen_audio: cannot reset the cfg sequence\n"); return 1; } // the uncond prompt keeps <|im_start|> and the last 2 tokens, the rest is <|audio_cfg|> prompt_uncond = prompt; std::fill(prompt_uncond.begin() + 1, prompt_uncond.end() - 2, tok_audio_cfg); } if (n_pos_needed((llama_pos) n + 1) > (int64_t) llama_n_ctx_seq(lctx)) { LOG_ERR("mtmd_helper_gen_audio: the prompt does not fit in the context\n"); return 1; } return 0; } int32_t step_prompt(int32_t n_batch) override { GGML_ASSERT(n_batch > 0); const int32_t n_prompt = (int32_t) prompt.size(); if (prompt_pos < n_prompt) { const int32_t n = std::min(n_batch, n_prompt - prompt_pos); llama_pos p = prompt_pos; if (!decode_tokens(prompt.data() + prompt_pos, n, seq_id, p)) { return -1; } prompt_pos += n; if (prompt_pos == n_prompt) { save_outputs(0, -1); } } else if (use_cfg && uprompt_pos < n_prompt) { const int32_t n = std::min(n_batch, n_prompt - uprompt_pos); llama_pos p = uprompt_pos; if (!decode_tokens(prompt_uncond.data() + uprompt_pos, n, seq_uncond, p)) { return -1; } uprompt_pos += n; if (uprompt_pos == n_prompt) { save_outputs(1, -1); } } pos = n_prompt; const int32_t n_left = (n_prompt - prompt_pos) + (use_cfg ? n_prompt - uprompt_pos : 0); return n_left; } int32_t step_gen(llama_token sampled, const float * h_state_in, const float ** h_state_out, bool * out_stop) override { GGML_UNUSED(sampled); GGML_UNUSED(h_state_in); *h_state_out = nullptr; *out_stop = false; // the first frame only moves the backbone past <|audio_start|>, its condition row is dropped like the reference const size_t n_rows = n_frames(); while (!done && n_frames() == n_rows) { if (!gen_frame()) { return 1; } } if (n_frames() == n_rows) { *out_stop = true; return 0; } *h_state_out = h_rows.data(); *out_stop = done; // a window is final once a frame past it exists, see MiniMaxMusic3PrepareChunksStep of the reference while (n_frames() > window_start(n_win_done) + win_frames) { if (!run_window(false)) { return 1; } } return 0; } int32_t get_output(int32_t * out_sample_rate, const char ** out_data, size_t * out_data_len, int64_t * out_n_samples) override { *out_sample_rate = info.sample_rate; const uint16_t n_ch = (uint16_t) info.n_channels; if (!stream) { if (flush() != 0) { return 1; } if (out_n_samples) { *out_n_samples = (int64_t) (audio_pcm.size() / n_ch); } if (out_type == MTMD_HELPER_GEN_AUDIO_OUTTYPE_PCM) { *out_data = (const char *) audio_pcm.data(); *out_data_len = audio_pcm.size() * sizeof(float); return 0; } out_buf.clear(); if (!write_wav16(out_buf, audio_pcm, info.sample_rate, n_ch)) { LOG_ERR("mtmd_helper_gen_audio: output too large for WAV\n"); return 1; } *out_data = out_buf.data(); *out_data_len = out_buf.size(); return 0; } const size_t n_new = audio_pcm.size() - pcm_sent; if (out_n_samples) { *out_n_samples = (int64_t) (n_new / n_ch); } if (out_type == MTMD_HELPER_GEN_AUDIO_OUTTYPE_PCM) { *out_data = (const char *) (audio_pcm.data() + pcm_sent); *out_data_len = n_new * sizeof(float); pcm_sent = audio_pcm.size(); return 0; } out_buf.clear(); if (!wav_header_sent) { write_wav16_header(out_buf, UINT32_MAX, info.sample_rate, n_ch); wav_header_sent = true; } append_wav16_pcm(out_buf, audio_pcm.data() + pcm_sent, n_new); pcm_sent = audio_pcm.size(); *out_data = out_buf.data(); *out_data_len = out_buf.size(); return 0; } int32_t flush() override { if (decoded) { return 0; } if (n_frames() == 0) { LOG_ERR("mtmd_helper_gen_audio: minimax_music3 stopped before the first frame\n"); return 1; } done = true; // the windows start every win_hop frames before n_frames - win_hop, the last one ends at n_frames while (n_win_done == 0 || window_start(n_win_done) + win_hop < n_frames()) { const bool last = window_start(n_win_done + 1) + win_hop >= n_frames(); if (!run_window(last)) { return 1; } if (last) { break; } } decoded = true; return 0; } private: // token ids of the checkpoint, see encoders.py of the reference static constexpr llama_token tok_im_start = 151644; static constexpr llama_token tok_audio_cfg = 151654; static constexpr llama_token tok_audio_start = 151669; static constexpr llama_token tok_audio_end = 151670; static constexpr llama_token tok_code0 = 151675; static constexpr int32_t n_semantic = 16384; static constexpr int32_t max_prompt = 5000; static constexpr size_t max_audio_frames = 9000; static constexpr int32_t top_k = 50; // windows of the flow matching and their crop, see before_denoise.py and decoders.py of the reference static constexpr size_t win_frames = 200; static constexpr size_t win_hop = 100; static constexpr size_t crop_left = 86; // latent frames static constexpr size_t crop_right = 344 - 86; // latent frames static constexpr size_t hop_samples = 512; // samples per latent frame // with guidance, both sequences share the unified KV cache int64_t n_pos_needed(llama_pos n) const { return use_cfg ? 2 * (int64_t) n : (int64_t) n; } size_t n_frames() const { return n_rows_dropped + cond_rows.size() / (size_t) n_embd; } static size_t window_start(size_t k) { return k * win_hop; } bool decode_tokens(const llama_token * toks, int32_t n, llama_seq_id seq, llama_pos & p) { llama_batch_ext_ptr batch(llama_batch_ext_init(lctx)); if (!batch_set_tokens(batch.get(), toks, n, p, seq)) { return false; } llama_batch_ext_set_output_logits(batch.get(), n - 1, true); if (llama_process(lctx, LLAMA_PROCESS_TYPE_DECODE, batch.get()) != 0) { LOG_ERR("mtmd_helper_gen_audio: decode failed\n"); return false; } p += n; return true; } // the context outputs belong to the last decode of any sequence (server slots), keep ours right away // only the window that sample() reads: <|audio_end|> and the semantic codes void save_outputs(int row, int32_t idx) { auto & lg = row == 0 ? logits_cond : logits_uncond; const float * l = llama_get_logits_ith(lctx, idx); lg.assign(l + tok_audio_end, l + tok_code0 + n_semantic); const float * h = llama_get_embeddings_ith(lctx, idx); h_rows.resize((size_t) (use_cfg ? 2 : 1) * n_embd); std::copy(h, h + n_embd, h_rows.begin() + (size_t) row * n_embd); } // feeds the next backbone input row to both sequences bool decode_embd(const float * embd) { llama_batch_ext_ptr batch(llama_batch_ext_init(lctx)); const int n_seq = use_cfg ? 2 : 1; const llama_seq_id seqs[2] = { seq_id, seq_uncond }; for (int i = 0; i < n_seq; i++) { const llama_embd row = { embd, 1, (size_t) n_embd }; const int32_t idx = llama_batch_ext_add_embd(batch.get(), seqs[i], row); if (idx < 0) { LOG_ERR("mtmd_helper_gen_audio: failed to add a row to the batch (error %d)\n", idx); return false; } llama_batch_ext_set_pos(batch.get(), idx, &pos); llama_batch_ext_set_output_logits(batch.get(), idx, true); } if (llama_process(lctx, LLAMA_PROCESS_TYPE_DECODE, batch.get()) != 0) { LOG_ERR("mtmd_helper_gen_audio: decode failed\n"); return false; } pos++; for (int i = 0; i < n_seq; i++) { save_outputs(i, i); } return true; } // semantic code or <|audio_end|>: guidance on the top-k of the cond logits, then top-k sampling, see SemanticGenerationStep llama_token sample() { const size_t n = logits_cond.size(); std::vector> cand; cand.reserve(n); for (size_t i = 0; i < n; i++) { const llama_token t = tok_audio_end + (llama_token) i; if (t == tok_audio_end || t >= tok_code0) { cand.emplace_back(logits_cond[i], i); } } const size_t k = std::min(cand.size(), (size_t) top_k); std::nth_element(cand.begin(), cand.begin() + (k - 1), cand.end(), std::greater>()); const float thr = cand[k - 1].first; std::vector> guided; for (const auto & c : cand) { if (c.first >= thr) { const float u = use_cfg ? logits_uncond[c.second] : c.first; guided.emplace_back(u + (c.first - u) * cfg_scale, c.second); } } std::sort(guided.begin(), guided.end(), std::greater>()); if (temp <= 0.0f) { return tok_audio_end + (llama_token) guided[0].second; } // values below the k-th one are masked, ties stay const float gthr = guided[std::min(guided.size(), (size_t) top_k) - 1].first; std::vector p; double sum = 0.0; for (const auto & g : guided) { if (g.first < gthr) { break; } p.push_back(std::exp((double) (g.first - guided[0].first) / temp)); sum += p.back(); } double u = std::uniform_real_distribution(0.0, sum)(rng); for (size_t i = 0; i < p.size(); i++) { u -= p[i]; if (u <= 0.0) { return tok_audio_end + (llama_token) guided[i].second; } } return tok_audio_end + (llama_token) guided[p.size() - 1].second; } // one frame: semantic code, depth decoder, condition row, then the next backbone input bool gen_frame() { const llama_token tok = sample(); if (tok == tok_audio_end) { done = true; return true; } mtmd_gen_inp inp = mtmd_gen_inp_default(mctx); inp.type = MTMD_GEN_PROCESS_TYPE_GEN_CODE; inp.code0 = tok - tok_code0; inp.feats = h_rows.data(); inp.n_feats = h_rows.size(); inp.cfg_scale = use_cfg ? cfg_scale : 1.0f; inp.temp = temp; mtmd_gen_out out{}; if (mtmd_gen_audio_process(mctx, gen_state.get(), &inp, &out) != 0 || out.n_feats != (size_t) n_embd) { LOG_ERR("mtmd_helper_gen_audio: depth decoder failed\n"); return false; } if (n_gen++ > 0) { cond_rows.insert(cond_rows.end(), out.feats, out.feats + out.n_feats); if (n_frames() >= max_audio_frames) { done = true; return true; } } if (n_pos_needed(pos + 1) > (int64_t) llama_n_ctx_seq(lctx)) { LOG_WRN("mtmd_helper_gen_audio: the context is full, the song ends at %zu frames\n", n_frames()); done = true; return true; } return decode_embd(out.embd); } // flow matching and vocoder of one window, then the crop of the reference bool run_window(bool last) { const size_t s = window_start(n_win_done); const size_t e = std::min(s + win_frames, n_frames()); mtmd_gen_inp inp = mtmd_gen_inp_default(mctx); inp.type = MTMD_GEN_PROCESS_TYPE_GEN_WAV; inp.feats = cond_rows.data() + (s - n_rows_dropped) * n_embd; inp.n_feats = (e - s) * n_embd; inp.n_steps = flow_steps; mtmd_gen_out out{}; if (mtmd_gen_audio_process(mctx, gen_state.get(), &inp, &out) != 0) { LOG_ERR("mtmd_helper_gen_audio: flow matching failed\n"); return false; } const size_t n_ch = (size_t) info.n_channels; const size_t n = out.n_samples / n_ch; const size_t left = n_win_done == 0 ? 0 : crop_left * hop_samples; const size_t right = last ? 0 : crop_right * hop_samples; if (n < left + right) { LOG_ERR("mtmd_helper_gen_audio: window is too short\n"); return false; } audio_pcm.insert(audio_pcm.end(), out.audio + left * n_ch, out.audio + (n - right) * n_ch); n_win_done++; // the rows before the next window are not needed anymore const size_t n_drop = std::min(window_start(n_win_done), n_frames()) - n_rows_dropped; cond_rows.erase(cond_rows.begin(), cond_rows.begin() + n_drop * n_embd); n_rows_dropped += n_drop; return true; } llama_seq_id seq_id = 0; llama_seq_id seq_uncond = 0; bool use_cfg = false; float cfg_scale = 1.0f; float temp = 1.0f; int32_t flow_steps = 0; // 0 = the model default llama_pos pos = 0; std::vector prompt; std::vector prompt_uncond; int32_t prompt_pos = 0; int32_t uprompt_pos = 0; std::vector logits_cond; // see save_outputs() std::vector logits_uncond; std::vector h_rows; // backbone hidden of the cond, then uncond sequence size_t n_gen = 0; // frames sampled, the first one included std::vector cond_rows; // one condition row per frame, from frame n_rows_dropped size_t n_rows_dropped = 0; size_t n_win_done = 0; bool done = false; bool decoded = false; std::mt19937 rng; mtmd_helper_gen_audio_outtype out_type = MTMD_HELPER_GEN_AUDIO_OUTTYPE_WAV; bool stream = false; std::vector audio_pcm; // interleaved std::vector out_buf; size_t pcm_sent = 0; bool wav_header_sent = false; }; // VoxCPM2: the backbone emits no token, mtmd turns each hidden state into a patch of AudioVAE latents // every backbone row also goes through the residual LM inside mtmd, see mtmd_gen_code_voxcpm2() class voxcpm2_gen_audio_pipeline : public mtmd_gen_audio_pipeline { public: using mtmd_gen_audio_pipeline::mtmd_gen_audio_pipeline; // step_gen() stops at the limit of the reference, which depends on the text int32_t max_frames() const override { return 0; } bool supports_cfg() const override { return true; } bool supports_flow_steps() const override { return true; } void reset() override { seq_id = 0; pos = 0; rows.clear(); voice.clear(); prompt_pos = 0; n_frames = 0; max_len = 0; h_state_buf.clear(); embd_in.clear(); latents.clear(); n_decoded = 0; gen_state.reset(); audio_pcm.clear(); out_buf.clear(); pcm_sent = 0; wav_header_sent = false; } int32_t set_input(const mtmd_helper_gen_audio_inp * inp) override { reset(); seq_id = inp->seq_id; out_type = inp->out_type; stream = inp->stream; temp = inp->temp; cfg_scale = inp->cfg_scale; flow_steps = inp->flow_steps; if (flow_steps != 0 && (flow_steps < 2 || flow_steps > 32)) { LOG_ERR("mtmd_helper_gen_audio: voxcpm2 needs flow_steps in [2, 32] (0 = model default)\n"); return 2; } if (inp->speaker_ref_text) { LOG_WRN("mtmd_helper_gen_audio: voxcpm2 clones the voice without its transcript, the speaker text is ignored\n"); } if (!ensure_cache()) { return 1; } // re.sub(r"\s+", " ", text) as in VoxCPM._generate() of core.py, a leading or trailing space is kept std::string text; bool space = false; for (char c : std::string(inp->prompt, inp->prompt_len)) { const bool is_space = std::isspace((unsigned char) c); if (!is_space || !space) { text += is_space ? ' ' : c; } space = is_space; } std::vector ids = tokenize(text); if (ids.empty()) { LOG_ERR("mtmd_helper_gen_audio: empty prompt\n"); return 1; } // retry_badcase_ratio_threshold * text length + 10, capped by max_len of the reference max_len = std::min((int) ids.size() * 6 + 10, 4096); // [<|audio_prompt_start|>, voice rows, <|audio_prompt_end|>,] text, <|audio_start|> if (inp->speaker_ref) { if (!encode_speaker(inp->speaker_ref, voice)) { return 1; } rows.push_back({ tok_ref_start, -1 }); for (int i = 0; i < (int) (voice.size() / n_embd); i++) { rows.push_back({ LLAMA_TOKEN_NULL, i }); } rows.push_back({ tok_ref_end, -1 }); } for (llama_token t : ids) { rows.push_back({ t, -1 }); } rows.push_back({ tok_audio_start, -1 }); // the residual LM reads the prompt rows, then one row per frame after the first max_len = std::min(max_len, n_ctx_res + 1 - (int) rows.size()); if (max_len <= 0) { LOG_ERR("mtmd_helper_gen_audio: the prompt is too long for the residual LM (%d rows)\n", n_ctx_res); return 1; } if (rows.size() + (size_t) max_len >= llama_n_ctx_seq(lctx)) { LOG_WRN("mtmd_helper_gen_audio: the context may be too small for the prompt and %d frames\n", max_len); } if (!llama_memory_seq_rm(llama_get_memory(lctx), seq_id, 0, -1)) { LOG_ERR("mtmd_helper_gen_audio: cannot reset the sequence\n"); return 1; } return init_gen_state(inp->seed) ? 0 : 1; } // the residual LM reads every prompt row but the last, which makes the first frame int32_t step_prompt(int32_t n_batch) override { GGML_ASSERT(n_batch > 0); const int n_prompt = (int) rows.size(); if (prompt_pos >= n_prompt) { return 0; } // a batch is either text tokens or voice rows; small batches, every row outputs its hidden state const bool is_voice = rows[prompt_pos].voice >= 0; int n = 0; while (n < std::min(n_batch, 64) && prompt_pos + n < n_prompt && (rows[prompt_pos + n].voice >= 0) == is_voice) { n++; } if (is_voice) { decode_embd_batch batch(voice.data() + (size_t) rows[prompt_pos].voice * n_embd, n, 1, n_embd); batch.set_position_normal(pos, seq_id); std::fill(batch.logits.begin(), batch.logits.end(), 1); if (llama_process(lctx, LLAMA_PROCESS_TYPE_DECODE, batch.render(lctx, 0, n)) != 0) { LOG_ERR("mtmd_helper_gen_audio: prompt decode failed\n"); return -1; } } else { std::vector toks(n); for (int i = 0; i < n; i++) { toks[i] = rows[prompt_pos + i].tok; } llama_batch_ext_ptr batch(llama_batch_ext_init(lctx)); if (!batch_set_tokens(batch.get(), toks.data(), n, pos, seq_id)) { return -1; } for (int i = 0; i < n; i++) { llama_batch_ext_set_output_logits(batch.get(), i, true); } if (llama_process(lctx, LLAMA_PROCESS_TYPE_DECODE, batch.get()) != 0) { LOG_ERR("mtmd_helper_gen_audio: prompt decode failed\n"); return -1; } } // the rows go to the residual LM together, the first frame comes from the last prompt row const bool last = prompt_pos + n == n_prompt; const int n_res = last ? n - 1 : n; std::vector h_rows((size_t) n_res * n_embd); for (int i = 0; i < n_res; i++) { const float * h = llama_get_embeddings_ith(lctx, i); std::copy(h, h + n_embd, h_rows.begin() + (size_t) i * n_embd); } if (last) { const float * h = llama_get_embeddings_ith(lctx, n - 1); h_state_buf.assign(h, h + n_embd); } const float * x = is_voice ? voice.data() + (size_t) rows[prompt_pos].voice * n_embd : nullptr; if (n_res > 0 && !run_row(h_rows.data(), x, -1, nullptr, n_res)) { return -1; } pos += n; prompt_pos += n; return n_prompt - prompt_pos; } // the hidden state of the caller is not read, the server may decode other sequences in between int32_t step_gen(llama_token sampled, const float * h_state_in, const float ** h_state_out, bool * out_stop) override { GGML_UNUSED(sampled); GGML_UNUSED(h_state_in); *h_state_out = nullptr; *out_stop = false; if (n_frames >= max_len || (uint32_t) pos >= llama_n_ctx_seq(lctx)) { *out_stop = true; return 0; } mtmd_gen_out out{}; if (!run_row(h_state_buf.data(), embd_in.empty() ? nullptr : embd_in.data(), 0, &out)) { return 1; } latents.insert(latents.end(), out.feats, out.feats + out.n_feats); n_frames++; *h_state_out = h_state_buf.data(); // the frame of the stopping step is kept, see the loop of VoxCPM2Model._inference() if ((out.is_eos && n_frames > min_len + 1) || n_frames >= max_len) { *out_stop = true; return 0; } embd_in.assign(out.embd, out.embd + n_embd); decode_embd_batch batch(embd_in.data(), 1, 1, n_embd); batch.set_position_normal(pos, seq_id); batch.logits[0] = 1; if (llama_process(lctx, LLAMA_PROCESS_TYPE_DECODE, batch.render(lctx, 0, 1)) != 0) { LOG_ERR("mtmd_helper_gen_audio: decode failed\n"); return 1; } pos++; const float * h = llama_get_embeddings_ith(lctx, -1); h_state_buf.assign(h, h + n_embd); if (stream && latents.size() / n_frame - n_decoded >= stream_frames) { return decode_wav() ? 0 : 1; } return 0; } int32_t get_output(int32_t * out_sample_rate, const char ** out_data, size_t * out_data_len, int64_t * out_n_samples) override { *out_sample_rate = info.sample_rate; if (!stream) { if (flush() != 0) { return 1; } if (out_n_samples) { *out_n_samples = (int64_t) audio_pcm.size(); } if (out_type == MTMD_HELPER_GEN_AUDIO_OUTTYPE_PCM) { *out_data = (const char *) audio_pcm.data(); *out_data_len = audio_pcm.size() * sizeof(float); return 0; } out_buf.clear(); if (!write_wav16(out_buf, audio_pcm, info.sample_rate)) { LOG_ERR("mtmd_helper_gen_audio: output too large for WAV\n"); return 1; } *out_data = out_buf.data(); *out_data_len = out_buf.size(); return 0; } // streaming: only the audio since the previous call const size_t n_new = audio_pcm.size() - pcm_sent; if (out_n_samples) { *out_n_samples = (int64_t) n_new; } if (out_type == MTMD_HELPER_GEN_AUDIO_OUTTYPE_PCM) { *out_data = (const char *) (audio_pcm.data() + pcm_sent); *out_data_len = n_new * sizeof(float); pcm_sent = audio_pcm.size(); return 0; } out_buf.clear(); if (!wav_header_sent) { write_wav16_header(out_buf, UINT32_MAX, info.sample_rate); wav_header_sent = true; } append_wav16_pcm(out_buf, audio_pcm.data() + pcm_sent, n_new); pcm_sent = audio_pcm.size(); *out_data = out_buf.data(); *out_data_len = out_buf.size(); return 0; } int32_t flush() override { return decode_wav() ? 0 : 1; } private: struct prompt_row { llama_token tok; // text row int voice; // row of the voice reference, -1 for a text row }; bool ensure_cache() { if (cache_ready) { return true; } tok_audio_start = find_special_token(vocab, "<|audio_start|>"); tok_ref_start = find_special_token(vocab, "<|audio_prompt_start|>"); tok_ref_end = find_special_token(vocab, "<|audio_prompt_end|>"); if (tok_audio_start == LLAMA_TOKEN_NULL || tok_ref_start == LLAMA_TOKEN_NULL || tok_ref_end == LLAMA_TOKEN_NULL) { LOG_ERR("mtmd_helper_gen_audio: missing voxcpm2 audio tokens in vocab\n"); return false; } // multi-character CJK tokens are split into single characters, see mask_multichar_chinese_tokens() // a token is split when its text without the SPM space (U+2581) is a CJK token of the vocab, each char maps to its plain token const std::string spm_space = "\xe2\x96\x81"; const int32_t n_vocab = llama_vocab_n_tokens(vocab); std::unordered_map pieces; for (llama_token id = 0; id < n_vocab; id++) { pieces.emplace(llama_vocab_get_text(vocab, id), id); } for (llama_token id = 0; id < n_vocab; id++) { std::string clean = llama_vocab_get_text(vocab, id); for (size_t i; (i = clean.find(spm_space)) != std::string::npos;) { clean.erase(i, spm_space.size()); } if (clean.size() < 6 || !is_cjk(clean) || pieces.find(clean) == pieces.end()) { continue; } std::vector ids; for (size_t i = 0; i < clean.size(); i += 3) { auto it = pieces.find(clean.substr(i, 3)); if (it == pieces.end()) { break; } ids.push_back(it->second); } if (ids.size() * 3 == clean.size()) { split_cjk.emplace(id, std::move(ids)); } } cache_ready = true; return true; } // chars of U+4E00..U+9FFF only, each is 3 bytes in UTF-8 static bool is_cjk(const std::string & s) { if (s.empty() || s.size() % 3 != 0) { return false; } for (size_t i = 0; i < s.size(); i += 3) { const auto b0 = (unsigned char) s[i]; const auto b1 = (unsigned char) s[i + 1]; const auto b2 = (unsigned char) s[i + 2]; if ((b0 & 0xF0) != 0xE0 || (b1 & 0xC0) != 0x80 || (b2 & 0xC0) != 0x80) { return false; } const uint32_t cp = ((b0 & 0x0F) << 12) | ((b1 & 0x3F) << 6) | (b2 & 0x3F); if (cp < 0x4E00 || cp > 0x9FFF) { return false; } } return true; } // no BOS, special tokens in the text are plain text, like tokenize() of the reference std::vector tokenize(const std::string & text) const { std::vector ids(text.size() + 16); const int32_t n = llama_tokenize(vocab, text.c_str(), (int32_t) text.size(), ids.data(), (int32_t) ids.size(), false, false); if (n < 0) { LOG_ERR("mtmd_helper_gen_audio: tokenization failed\n"); return {}; } std::vector out; for (int32_t i = 0; i < n; i++) { auto it = split_cjk.find(ids[i]); if (it != split_cjk.end()) { out.insert(out.end(), it->second.begin(), it->second.end()); } else { out.push_back(ids[i]); } } return out; } // backbone rows through mtmd: code0 < 0 for n_rows prompt rows, else one row and the next frame goes to out bool run_row(const float * h, const float * x, int32_t code0, mtmd_gen_out * out = nullptr, int n_rows = 1) { mtmd_gen_inp inp = mtmd_gen_inp_default(mctx); inp.type = MTMD_GEN_PROCESS_TYPE_GEN_CODE; inp.code0 = code0; inp.embd = const_cast(h); inp.feats = x; inp.n_feats = x ? (size_t) n_rows * n_embd : 0; inp.n_codes = (size_t) n_rows; if (temp >= 0.0f) { inp.temp = temp; } if (flow_steps > 0) { inp.n_steps = flow_steps; } if (cfg_scale > 0.0f) { inp.cfg_scale = cfg_scale; } mtmd_gen_out tmp{}; if (mtmd_gen_audio_process(mctx, gen_state.get(), &inp, out ? out : &tmp) != 0) { LOG_ERR("mtmd_helper_gen_audio: voxcpm2 gen_code failed\n"); return false; } return true; } // the AudioVAE decoder is causal and keeps its state, so the latents can be decoded in pieces bool decode_wav() { const size_t n_total = latents.size() / n_frame; while (n_decoded < n_total) { const size_t n = std::min(n_total - n_decoded, (size_t) 16); mtmd_gen_inp inp = mtmd_gen_inp_default(mctx); inp.type = MTMD_GEN_PROCESS_TYPE_GEN_WAV; inp.feats = latents.data() + n_decoded * n_frame; inp.n_feats = n * n_frame; mtmd_gen_out out{}; if (mtmd_gen_audio_process(mctx, gen_state.get(), &inp, &out) != 0) { LOG_ERR("mtmd_helper_gen_audio: AudioVAE decode failed\n"); return false; } audio_pcm.insert(audio_pcm.end(), out.audio, out.audio + out.n_samples); n_decoded += n; } return true; } static constexpr int min_len = 2; // min_len of the reference static constexpr size_t n_frame = 64 * 4; // latents of one frame, n_latent * n_patch static constexpr size_t stream_frames = 2; // frames per decoder call when streaming (0.32 s) static constexpr int n_ctx_res = 8192; // rows of the residual LM, see mtmd_graph_voxcpm2_gen::n_ctx_res llama_token tok_audio_start = LLAMA_TOKEN_NULL; llama_token tok_ref_start = LLAMA_TOKEN_NULL; llama_token tok_ref_end = LLAMA_TOKEN_NULL; std::unordered_map> split_cjk; bool cache_ready = false; float temp = -1.0f; float cfg_scale = 0.0f; int32_t flow_steps = 0; llama_seq_id seq_id = 0; llama_pos pos = 0; std::vector rows; std::vector voice; // backbone rows of the voice reference int prompt_pos = 0; int n_frames = 0; int max_len = 0; std::vector h_state_buf; // hidden state of the last backbone row std::vector embd_in; // input row of the last backbone row, empty for the text row std::vector latents; // frame-major size_t n_decoded = 0; // frames already through the decoder std::vector audio_pcm; mtmd_helper_gen_audio_outtype out_type = MTMD_HELPER_GEN_AUDIO_OUTTYPE_WAV; std::vector out_buf; bool stream = false; size_t pcm_sent = 0; // samples already returned by get_output() bool wav_header_sent = false; }; static std::unique_ptr make_pipeline(llama_context * lctx, mtmd_context * mctx) { switch (mtmd_gen_audio_get_info(mctx).type) { case MTMD_GEN_AUDIO_TYPE_QWEN3TTS: return std::unique_ptr(new qwen3tts_gen_audio_pipeline(lctx, mctx)); case MTMD_GEN_AUDIO_TYPE_POCKETTTS: return std::unique_ptr(new pockettts_gen_audio_pipeline(lctx, mctx)); case MTMD_GEN_AUDIO_TYPE_SOPRANO: return std::unique_ptr(new soprano_gen_audio_pipeline(lctx, mctx)); case MTMD_GEN_AUDIO_TYPE_KOKORO: return std::unique_ptr(new kokoro_gen_audio_pipeline(lctx, mctx)); case MTMD_GEN_AUDIO_TYPE_TINY_VOCA: return std::unique_ptr(new tiny_voca_gen_audio_pipeline(lctx, mctx)); case MTMD_GEN_AUDIO_TYPE_LFM3A: return std::unique_ptr(new lfm3a_gen_audio_pipeline(lctx, mctx)); case MTMD_GEN_AUDIO_TYPE_CHATTERBOX: return std::unique_ptr(new chatterbox_gen_audio_pipeline(lctx, mctx)); case MTMD_GEN_AUDIO_TYPE_DRAMA3: return std::unique_ptr(new drama3_gen_audio_pipeline(lctx, mctx)); case MTMD_GEN_AUDIO_TYPE_ZONOS3: return std::unique_ptr(new zonos3_gen_audio_pipeline(lctx, mctx)); case MTMD_GEN_AUDIO_TYPE_YUE2: return std::unique_ptr(new yue2_gen_audio_pipeline(lctx, mctx)); case MTMD_GEN_AUDIO_TYPE_MINIMAX_MUSIC3: return std::unique_ptr(new music3_gen_audio_pipeline(lctx, mctx)); case MTMD_GEN_AUDIO_TYPE_FISH_S2: return std::unique_ptr(new fish_s2_gen_audio_pipeline(lctx, mctx)); case MTMD_GEN_AUDIO_TYPE_VOXCPM2: return std::unique_ptr(new voxcpm2_gen_audio_pipeline(lctx, mctx)); default: return nullptr; } } struct mtmd_helper_gen_audio { llama_context * lctx = nullptr; std::unique_ptr pipeline; }; mtmd_helper_gen_audio * mtmd_helper_gen_audio_init(struct llama_context * lctx, struct mtmd_context * mctx) { auto * ctx = new mtmd_helper_gen_audio(); ctx->lctx = lctx; ctx->pipeline = make_pipeline(lctx, mctx); if (ctx->pipeline && !ctx->pipeline->check_context()) { delete ctx; return nullptr; } return ctx; } void mtmd_helper_gen_audio_free(mtmd_helper_gen_audio * ctx) { delete ctx; } int32_t mtmd_helper_gen_audio_max_frames(const mtmd_helper_gen_audio * ctx) { return ctx && ctx->pipeline ? ctx->pipeline->max_frames() : 0; } void mtmd_helper_gen_audio_reset(mtmd_helper_gen_audio * ctx) { if (ctx && ctx->pipeline) { ctx->pipeline->reset(); } } struct mtmd_helper_gen_audio_inp mtmd_helper_gen_audio_inp_default(void) { mtmd_helper_gen_audio_inp inp{}; inp.top_k = 50; inp.top_p = 1.0f; inp.temp = -1.0f; // model default inp.seed = UINT32_MAX; // random inp.speed = 1.0f; inp.loudness = 0.0f; inp.cfg_scale = 0.0f; // model default inp.flow_steps = 0; // model default inp.out_type = MTMD_HELPER_GEN_AUDIO_OUTTYPE_WAV; return inp; } int32_t mtmd_helper_gen_audio_set_input(mtmd_helper_gen_audio * ctx, const mtmd_helper_gen_audio_inp * inp) { if (!ctx->pipeline) { LOG_ERR("mtmd_helper_gen_audio: unsupported or missing gen-audio pipeline\n"); return 1; } if (inp->n_past > 0 && !ctx->pipeline->supports_continue()) { LOG_ERR("mtmd_helper_gen_audio: the pipeline cannot continue a sequence\n"); return 1; } if (!std::isfinite(inp->speed) || inp->speed <= 0.0f || !std::isfinite(inp->loudness) || inp->loudness > 0.0f) { LOG_ERR("mtmd_helper_gen_audio: speed must be > 0 and loudness < 0 LUFS (0 = model default)\n"); return 2; } if (inp->speed != 1.0f && !ctx->pipeline->supports_speed()) { LOG_ERR("mtmd_helper_gen_audio: the model does not support speed\n"); return 2; } if (inp->loudness != 0.0f && !ctx->pipeline->supports_delivery()) { LOG_ERR("mtmd_helper_gen_audio: the model does not support loudness\n"); return 2; } if (!std::isfinite(inp->cfg_scale) || inp->cfg_scale < 0.0f) { LOG_ERR("mtmd_helper_gen_audio: cfg_scale must be > 0 (0 = model default)\n"); return 2; } if (inp->cfg_scale != 0.0f && !ctx->pipeline->supports_cfg()) { LOG_ERR("mtmd_helper_gen_audio: the model does not support cfg_scale\n"); return 2; } if (inp->flow_steps < 0 || inp->flow_steps > 1024) { LOG_ERR("mtmd_helper_gen_audio: flow_steps must be in [1, 1024] (0 = model default)\n"); return 2; } if (inp->flow_steps != 0 && !ctx->pipeline->supports_flow_steps()) { LOG_ERR("mtmd_helper_gen_audio: the model does not support flow_steps\n"); return 2; } ctx->pipeline->logit_bias.clear(); return ctx->pipeline->set_input(inp); } int32_t mtmd_helper_gen_audio_step_prompt(mtmd_helper_gen_audio * ctx, int32_t n_batch) { if (!ctx->pipeline) { return -1; } // the context may use a smaller n_batch than requested (e.g. clamped to n_ctx), llama_process() would fail above it if (n_batch <= 0 || (uint32_t) n_batch > llama_n_batch(ctx->lctx)) { LOG_ERR("mtmd_helper_gen_audio: n_batch must be in [1, %u], got %d\n", llama_n_batch(ctx->lctx), n_batch); return -1; } return ctx->pipeline->step_prompt(n_batch); } bool mtmd_helper_gen_audio_next_is_text(mtmd_helper_gen_audio * ctx) { return ctx->pipeline && ctx->pipeline->next_is_text(); } const llama_logit_bias * mtmd_helper_gen_audio_get_logit_bias(mtmd_helper_gen_audio * ctx, size_t * n_bias) { *n_bias = ctx->pipeline ? ctx->pipeline->logit_bias.size() : 0; return *n_bias ? ctx->pipeline->logit_bias.data() : nullptr; } int32_t mtmd_helper_audio_stream_get_logit_bias(const mtmd_context * mctx, const llama_model * model, llama_logit_bias * out, int32_t n_max) { std::vector bias; if (mtmd_gen_audio_get_info(mctx).type == MTMD_GEN_AUDIO_TYPE_LFM3A) { // LFM3-Audio live ASR: only <|text_end|> ends the text of a chunk, the stream adds the final <|mixed_end|> const llama_vocab * vocab = llama_model_get_vocab(model); for (const char * piece : { "<|mixed_end|>", "<|mixed_start|>", "<|audio_start|>", "<|im_start|>", "<|im_end|>", "<|endoftext|>" }) { const llama_token tok = find_special_token(vocab, piece); if (tok != LLAMA_TOKEN_NULL) { bias.push_back({ tok, -INFINITY }); } } } for (int32_t i = 0; i < n_max && i < (int32_t) bias.size(); i++) { out[i] = bias[i]; } return (int32_t) bias.size(); } int32_t mtmd_helper_gen_audio_step_gen(mtmd_helper_gen_audio * ctx, llama_token sampled, const float * h_state_in, const float ** h_state_out, bool * out_stop) { if (!ctx->pipeline) { return 1; } bool stop = false; const int32_t ret = ctx->pipeline->step_gen(sampled, h_state_in, h_state_out, &stop); if (out_stop) { *out_stop = stop; } return ret; } int32_t mtmd_helper_gen_audio_get_output(mtmd_helper_gen_audio * ctx, int32_t * out_sample_rate, const char ** out_data, size_t * out_data_len, int64_t * out_n_samples) { if (!ctx->pipeline) { return 1; } return ctx->pipeline->get_output(out_sample_rate, out_data, out_data_len, out_n_samples); } int32_t mtmd_helper_gen_audio_flush(mtmd_helper_gen_audio * ctx) { if (!ctx->pipeline) { return 1; } return ctx->pipeline->flush(); }