* feat: support PocketTTS voices with no voice cloning The kyutai/pocket-tts-without-voice-cloning weights ship predefined voices as the backbone KV state after the voice prompt. The converter turns each state back into the prompt embeddings and stores them in the mmproj (voice.<name>), the zeroed voice encoder is left out, and the "voice" field of /tts and --tts-voice select one of them. A speaker reference is rejected when the model has no voice encoder. Assisted-by: Claude Fable 5.1 * convert : pocket-tts voices never fail the conversion, log the drift of the state The voice states shipped by Kyutai are not always the ones the pack's weights give for the prompt: italian, german and spanish match within the bf16 rounding, english_2026-04 is 2% off and french 20% (its layer 0 state comes from other weights), while the reference loads them as is and they still sound like the same voice. The converter now re-prompts the backbone with the recovered embeddings, compares the whole state and logs the distance per voice, warning above 5%. A state that does not belong to the weights at all is skipped, so the packs of kyutai/pocket-tts convert too. Assisted-by: Claude Fable 5.1 * mtmd : list the predefined voices, expose them in GET /props mtmd_gen_audio_info gains n_voices and mtmd_gen_audio_voice_name() the names, with the voices worded as a feature of any audio generation model. The server lists them under "voices" in /props when the model ships some. Assisted-by: Claude Fable 5.1 * server : an unknown voice is a bad request A model with predefined voices answers 400 naming the voice instead of 500 "failed to process TTS prompt". A model without them ignores "voice" with a warning, as the other audio generation models do. Assisted-by: Claude Fable 5.1 * refactor api * fix conversion * small renaming * fix lint * tts : a model with predefined voices speaks with the first one by default Without a voice prompt such a model only produces near silence. When neither a voice nor a speaker reference is given, llama-tts and the server now take the first predefined voice and say so in the log, as the LFM3-Audio template does with its default voice. Assisted-by: Claude Fable 5.1 * mtmd : one comment per voice accessor, converter states the rule only The clip.h comment described clip_gen_get_voice but sat above all three functions; the converter comment quoted the drift measured per pack, which goes stale with the next upload of the weights. Assisted-by: Claude Fable 5.1 * clean up redundant code and comments * mtmd: match the LFM3-Audio voice name case-insensitively The tts chat template lowers the voice name, so Male speaks with the male voice; the voice lookup lowers it the same way and names the bitmap after the canonical voice. --------- Co-authored-by: Xuan Son Nguyen <son@huggingface.co> Co-authored-by: Pascal <admin@serveurperso.com>
1841 lines
69 KiB
C++
1841 lines
69 KiB
C++
#include "mtmd.h"
|
|
#include "mtmd-helper.h"
|
|
#include "mtmd-helper-common.h"
|
|
#include "llama.h"
|
|
#include "../src/llama-ext.h"
|
|
|
|
#include <algorithm>
|
|
#include <cctype>
|
|
#include <cmath>
|
|
#include <cstring>
|
|
#include <functional>
|
|
#include <memory>
|
|
#include <random>
|
|
#include <string>
|
|
#include <unordered_map>
|
|
#include <vector>
|
|
|
|
#ifdef MTMD_INTERNAL_HEADER
|
|
#error "mtmd-helper is a public library outside of mtmd. it must not include internal headers"
|
|
#endif
|
|
|
|
//
|
|
// Audio generation helpers
|
|
//
|
|
|
|
// --tts-lang codes -> language names used by the codec_language special tokens
|
|
static const std::unordered_map<std::string, std::string> tts_lang_codes = {
|
|
{ "zh", "chinese" },
|
|
{ "en", "english" },
|
|
{ "de", "german" },
|
|
{ "it", "italian" },
|
|
{ "pt", "portuguese" },
|
|
{ "es", "spanish" },
|
|
{ "ja", "japanese" },
|
|
{ "ko", "korean" },
|
|
{ "fr", "french" },
|
|
{ "ru", "russian" },
|
|
};
|
|
|
|
static std::string tts_resolve_lang(const std::string & lang) {
|
|
auto it = tts_lang_codes.find(lang);
|
|
return it != tts_lang_codes.end() ? it->second : lang;
|
|
}
|
|
|
|
static llama_token find_special_token(const llama_vocab * vocab, const std::string & piece) {
|
|
const int32_t n = llama_vocab_n_tokens(vocab);
|
|
for (llama_token t = 0; t < n; t++) {
|
|
if (piece == llama_vocab_get_text(vocab, t)) {
|
|
return t;
|
|
}
|
|
}
|
|
return LLAMA_TOKEN_NULL;
|
|
}
|
|
|
|
static void put_bytes(std::vector<char> & buf, const void * p, size_t n) {
|
|
const char * c = (const char *) p;
|
|
buf.insert(buf.end(), c, c + n);
|
|
}
|
|
|
|
// data_sz == UINT32_MAX writes the "unknown length" sentinel (streaming), same as ffmpeg does on a pipe
|
|
static void write_wav16_header(std::vector<char> & buf, uint32_t data_sz, int32_t rate) {
|
|
const uint32_t riff_sz = data_sz == UINT32_MAX ? UINT32_MAX : 36 + data_sz;
|
|
const uint32_t fmt_sz = 16, byte_rate = (uint32_t) rate * 2;
|
|
const uint16_t fmt = 1, ch = 1, align = 2, bits = 16;
|
|
const uint32_t rate32 = (uint32_t) rate;
|
|
put_bytes(buf, "RIFF", 4); put_bytes(buf, &riff_sz, 4); put_bytes(buf, "WAVE", 4);
|
|
put_bytes(buf, "fmt ", 4); put_bytes(buf, &fmt_sz, 4);
|
|
put_bytes(buf, &fmt, 2); put_bytes(buf, &ch, 2); put_bytes(buf, &rate32, 4);
|
|
put_bytes(buf, &byte_rate, 4); put_bytes(buf, &align, 2); put_bytes(buf, &bits, 2);
|
|
put_bytes(buf, "data", 4); put_bytes(buf, &data_sz, 4);
|
|
}
|
|
|
|
static void append_wav16_pcm(std::vector<char> & buf, const float * pcm, size_t n) {
|
|
for (size_t i = 0; i < n; i++) {
|
|
int16_t s = (int16_t) (std::max(-1.0f, std::min(1.0f, pcm[i])) * 32767.0f);
|
|
put_bytes(buf, &s, 2);
|
|
}
|
|
}
|
|
|
|
static bool write_wav16(std::vector<char> & buf, const std::vector<float> & pcm, int32_t rate) {
|
|
// RIFF chunk sizes are 32-bit; refuse to emit a file with a truncated header
|
|
if (pcm.size() > ((size_t) UINT32_MAX - 36) / 2) {
|
|
return false;
|
|
}
|
|
write_wav16_header(buf, (uint32_t) (pcm.size() * 2), rate);
|
|
append_wav16_pcm(buf, pcm.data(), pcm.size());
|
|
return true;
|
|
}
|
|
|
|
class mtmd_gen_audio_pipeline {
|
|
public:
|
|
mtmd_gen_audio_pipeline(llama_context * lctx, mtmd_context * mctx)
|
|
: lctx(lctx), mctx(mctx), model(llama_get_model(lctx)), vocab(llama_model_get_vocab(model)),
|
|
n_embd(llama_model_n_embd(model)), info(mtmd_gen_audio_get_info(mctx)) {}
|
|
virtual ~mtmd_gen_audio_pipeline() = default;
|
|
|
|
virtual void reset() = 0;
|
|
virtual int32_t set_input(const mtmd_helper_gen_audio_inp * inp) = 0;
|
|
// decodes at most n_batch prompt tokens; returns remaining count (0 = done), <0 on error
|
|
virtual int32_t step_prompt(int32_t n_batch) = 0;
|
|
// sampled can be LLAMA_TOKEN_NULL for pipelines with no discrete backbone token,
|
|
// those read what they need from h_state_in instead
|
|
// set out_stop on end-of-speech, h_state_out must be null if no frame is generated
|
|
virtual int32_t step_gen(llama_token sampled, const float * h_state_in, const float ** h_state_out, bool * out_stop) = 0;
|
|
virtual int32_t get_output(int32_t * out_sample_rate, const char ** out_data, size_t * out_data_len, int64_t * out_n_samples) = 0;
|
|
// forces any buffered codes through code2wav now, regardless of window_frames
|
|
virtual int32_t flush() = 0;
|
|
// continuing a sequence (inp->n_past > 0) with the model's own text/audio switching
|
|
virtual bool supports_continue() const { return false; }
|
|
virtual bool next_is_text() const { return false; }
|
|
|
|
std::vector<llama_logit_bias> logit_bias; // see mtmd_helper_gen_audio_get_logit_bias()
|
|
|
|
protected:
|
|
// one state per generation, so a fixed seed always gives the same output
|
|
bool init_gen_state(uint32_t seed) {
|
|
mtmd_gen_state_params params = mtmd_gen_state_params_default();
|
|
params.seed = seed;
|
|
gen_state.reset(mtmd_gen_state_init(mctx, params));
|
|
if (!gen_state) {
|
|
LOG_ERR("mtmd_helper_gen_audio: failed to create the gen state\n");
|
|
return false;
|
|
}
|
|
return true;
|
|
}
|
|
|
|
llama_context * lctx;
|
|
mtmd_context * mctx;
|
|
const llama_model * model;
|
|
const llama_vocab * vocab;
|
|
int n_embd;
|
|
mtmd_gen_audio_info info;
|
|
mtmd::gen_state_ptr gen_state;
|
|
};
|
|
|
|
// Qwen3-TTS: backbone samples codec_0, code_predictor gives the other 15 codebooks,
|
|
// then code2wav decodes them to PCM
|
|
class qwen3tts_gen_audio_pipeline : public mtmd_gen_audio_pipeline {
|
|
public:
|
|
using mtmd_gen_audio_pipeline::mtmd_gen_audio_pipeline;
|
|
|
|
void reset() override {
|
|
seq_id = 0;
|
|
pos = 0;
|
|
codes_buf.clear();
|
|
gen_state.reset();
|
|
audio_pcm.clear();
|
|
overlay.clear();
|
|
h_state_buf.clear();
|
|
out_buf.clear();
|
|
prompt_embd_buf.clear();
|
|
prompt_batch.reset();
|
|
n_prompt = 0;
|
|
prompt_pos = 0;
|
|
stream = false;
|
|
pcm_sent = 0;
|
|
wav_header_sent = false;
|
|
}
|
|
|
|
int32_t set_input(const mtmd_helper_gen_audio_inp * inp) override {
|
|
reset();
|
|
seq_id = inp->seq_id;
|
|
|
|
if (!ensure_cache()) {
|
|
return 1;
|
|
}
|
|
|
|
const std::string lang = tts_resolve_lang((inp->lang && inp->lang[0]) ? inp->lang : "english");
|
|
const llama_token c_lang = find_special_token(vocab, ("<|codec_language_" + lang + "|>").c_str());
|
|
if (c_lang == LLAMA_TOKEN_NULL) {
|
|
LOG_ERR("mtmd_helper_gen_audio: unknown language '%s'\n", lang.c_str());
|
|
return 1;
|
|
}
|
|
|
|
std::vector<float> speaker_embd;
|
|
if (inp->speaker_ref) {
|
|
if (!encode_speaker(inp->speaker_ref, speaker_embd)) {
|
|
return 1;
|
|
}
|
|
}
|
|
|
|
const int n_e = n_embd;
|
|
auto row = [&](llama_token t) {
|
|
return std::vector<float>(tok_embd.begin() + (size_t) t * n_e,
|
|
tok_embd.begin() + (size_t) (t + 1) * n_e);
|
|
};
|
|
auto sum_row = [&](llama_token a, llama_token b) {
|
|
std::vector<float> va = row(a), vb = row(b);
|
|
for (int i = 0; i < n_e; i++) va[(size_t) i] += vb[(size_t) i];
|
|
return va;
|
|
};
|
|
auto sum_vec = [&](llama_token a, const std::vector<float> & vb) {
|
|
std::vector<float> va = row(a);
|
|
for (int i = 0; i < n_e; i++) va[(size_t) i] += vb[(size_t) i];
|
|
return va;
|
|
};
|
|
|
|
// upstream chat wrap, then slices: [0:3] role, [3:-5] utterance body
|
|
const std::string full = "<|im_start|>assistant\n" + std::string(inp->prompt, inp->prompt_len) +
|
|
"<|im_end|>\n<|im_start|>assistant\n";
|
|
std::vector<llama_token> ids(full.size() + 16);
|
|
int n_ids = llama_tokenize(vocab, full.c_str(), (int32_t) full.size(), ids.data(), (int32_t) ids.size(),
|
|
false, true);
|
|
if (n_ids < 8) {
|
|
LOG_ERR("mtmd_helper_gen_audio: tokenization failed\n");
|
|
return 1;
|
|
}
|
|
ids.resize((size_t) n_ids);
|
|
|
|
std::vector<std::vector<float>> prompt;
|
|
for (int i = 0; i < 3; i++) prompt.push_back(row(ids[(size_t) i]));
|
|
prompt.push_back(sum_row(tts_pad, c_think));
|
|
prompt.push_back(sum_row(tts_pad, c_think_b));
|
|
prompt.push_back(sum_row(tts_pad, c_lang));
|
|
prompt.push_back(sum_row(tts_pad, c_think_e));
|
|
if (!speaker_embd.empty()) prompt.push_back(sum_vec(tts_pad, speaker_embd));
|
|
prompt.push_back(sum_row(tts_bos, codec_pad));
|
|
for (int i = 3; i < n_ids - 5; i++) prompt.push_back(sum_row(ids[(size_t) i], codec_pad));
|
|
prompt.push_back(sum_row(tts_eos, codec_pad));
|
|
prompt.push_back(sum_row(tts_pad, codec_bos));
|
|
|
|
n_prompt = (int) prompt.size();
|
|
|
|
// the talker uses the qwen3vl interleaved mrope, all sections are equal for a text/codec stream
|
|
mrope = llama_model_rope_type(model) == LLAMA_ROPE_TYPE_MROPE ||
|
|
llama_model_rope_type(model) == LLAMA_ROPE_TYPE_IMROPE;
|
|
const int n_pos_per_embd = mrope ? 4 : 1;
|
|
|
|
prompt_embd_buf.resize((size_t) n_prompt * (size_t) n_e);
|
|
for (int i = 0; i < n_prompt; i++) {
|
|
memcpy(prompt_embd_buf.data() + (size_t) i * n_e, prompt[(size_t) i].data(), (size_t) n_e * sizeof(float));
|
|
}
|
|
|
|
prompt_batch.reset(new decode_embd_batch(prompt_embd_buf.data(), n_prompt, n_pos_per_embd, n_e));
|
|
if (mrope) prompt_batch->set_position_mrope_1d(0, seq_id);
|
|
else prompt_batch->set_position_normal (0, seq_id);
|
|
prompt_pos = 0;
|
|
|
|
pos = 0;
|
|
const mtmd_gen_inp def = mtmd_gen_inp_default(mctx);
|
|
top_k = inp->top_k > 0 ? inp->top_k : def.top_k;
|
|
top_p = inp->top_p > 0 ? inp->top_p : def.top_p;
|
|
out_type = inp->out_type;
|
|
stream = inp->stream;
|
|
if (!init_gen_state(inp->seed)) {
|
|
return 1;
|
|
}
|
|
|
|
// the prompt above holds the whole text stream up to tts_eos, so every generated
|
|
// frame adds tts_pad on top of the codes embedding
|
|
overlay = row(tts_pad);
|
|
|
|
return 0;
|
|
}
|
|
|
|
int32_t step_prompt(int32_t n_batch) override {
|
|
GGML_ASSERT(n_batch > 0);
|
|
if (prompt_pos >= n_prompt) {
|
|
return 0;
|
|
}
|
|
const int32_t n_tokens_batch = std::min(n_batch, n_prompt - prompt_pos);
|
|
|
|
const bool is_last_batch = (prompt_pos + n_tokens_batch) == n_prompt;
|
|
if (is_last_batch) {
|
|
prompt_batch->logits[prompt_pos + n_tokens_batch - 1] = 1;
|
|
}
|
|
|
|
if (llama_process(lctx, LLAMA_PROCESS_TYPE_DECODE, prompt_batch->render(lctx, prompt_pos, n_tokens_batch)) != 0) {
|
|
LOG_ERR("mtmd_helper_gen_audio: prompt decode failed\n");
|
|
return -1;
|
|
}
|
|
|
|
pos += n_tokens_batch;
|
|
prompt_pos += n_tokens_batch;
|
|
|
|
if (prompt_pos >= n_prompt) {
|
|
// prompt fully processed, its embedding buffer is no longer needed
|
|
prompt_batch.reset();
|
|
prompt_embd_buf.clear();
|
|
return 0;
|
|
}
|
|
return n_prompt - prompt_pos;
|
|
}
|
|
|
|
int32_t step_gen(llama_token sampled, const float * h_state_in, const float ** h_state_out, bool * out_stop) override {
|
|
if (sampled == LLAMA_TOKEN_NULL) {
|
|
LOG_ERR("mtmd_helper_gen_audio: qwen3tts requires a token sampled from the backbone\n");
|
|
return 1;
|
|
}
|
|
|
|
// backbone signals end-of-speech with a token, no frame for this step
|
|
if (sampled == codec_eos || llama_vocab_is_eog(vocab, sampled)) {
|
|
*out_stop = true;
|
|
*h_state_out = nullptr;
|
|
return 0;
|
|
}
|
|
|
|
mtmd_gen_inp inp = mtmd_gen_inp_default(mctx);
|
|
inp.type = MTMD_GEN_PROCESS_TYPE_GEN_CODE;
|
|
inp.code0 = sampled - codec_0;
|
|
inp.embd = const_cast<float *>(h_state_in);
|
|
inp.top_k = top_k;
|
|
inp.top_p = top_p;
|
|
mtmd_gen_out out{};
|
|
if (mtmd_gen_audio_process(mctx, gen_state.get(), &inp, &out) != 0) {
|
|
LOG_ERR("mtmd_helper_gen_audio: gen_code process failed\n");
|
|
return 1;
|
|
}
|
|
|
|
codes_buf.insert(codes_buf.end(), out.codes, out.codes + out.n_codes);
|
|
if (out.n_codes > 0 && codes_buf.size() / out.n_codes >= window_frames) {
|
|
if (!flush_gen_wav()) {
|
|
return 1;
|
|
}
|
|
}
|
|
|
|
std::vector<float> fb(out.embd, out.embd + n_embd);
|
|
for (int i = 0; i < n_embd; i++) fb[(size_t) i] += overlay[(size_t) i];
|
|
|
|
const int n_pos_per_embd = mrope ? 4 : 1;
|
|
decode_embd_batch batch_embd(fb.data(), 1, n_pos_per_embd, n_embd);
|
|
if (mrope) batch_embd.set_position_mrope_1d(pos, seq_id);
|
|
else batch_embd.set_position_normal (pos, seq_id);
|
|
batch_embd.logits[0] = 1;
|
|
pos++;
|
|
|
|
if (llama_process(lctx, LLAMA_PROCESS_TYPE_DECODE, batch_embd.render(lctx, 0, 1)) != 0) {
|
|
LOG_ERR("mtmd_helper_gen_audio: decode failed\n");
|
|
return 1;
|
|
}
|
|
|
|
const float * he = llama_get_embeddings_ith(lctx, -1);
|
|
h_state_buf.assign(he, he + n_embd);
|
|
*h_state_out = h_state_buf.data();
|
|
|
|
return 0;
|
|
}
|
|
|
|
int32_t get_output(int32_t * out_sample_rate, const char ** out_data, size_t * out_data_len, int64_t * out_n_samples) override {
|
|
*out_sample_rate = info.sample_rate;
|
|
|
|
if (!stream) {
|
|
// one-shot call: force out whatever's left, regardless of window_frames
|
|
if (!flush_gen_wav()) {
|
|
return 1;
|
|
}
|
|
if (out_n_samples) {
|
|
*out_n_samples = (int64_t) audio_pcm.size();
|
|
}
|
|
if (out_type == MTMD_HELPER_GEN_AUDIO_OUTTYPE_PCM) {
|
|
*out_data = (const char *) audio_pcm.data();
|
|
*out_data_len = audio_pcm.size() * sizeof(float);
|
|
return 0;
|
|
}
|
|
out_buf.clear();
|
|
if (!write_wav16(out_buf, audio_pcm, info.sample_rate)) {
|
|
LOG_ERR("mtmd_helper_gen_audio: output too large for WAV\n");
|
|
return 1;
|
|
}
|
|
*out_data = out_buf.data();
|
|
*out_data_len = out_buf.size();
|
|
return 0;
|
|
}
|
|
|
|
// streaming: only return audio produced since the previous call
|
|
const size_t n_new = audio_pcm.size() - pcm_sent;
|
|
if (out_n_samples) {
|
|
*out_n_samples = (int64_t) n_new;
|
|
}
|
|
|
|
if (out_type == MTMD_HELPER_GEN_AUDIO_OUTTYPE_PCM) {
|
|
*out_data = (const char *) (audio_pcm.data() + pcm_sent);
|
|
*out_data_len = n_new * sizeof(float);
|
|
pcm_sent = audio_pcm.size();
|
|
return 0;
|
|
}
|
|
|
|
out_buf.clear();
|
|
if (!wav_header_sent) {
|
|
write_wav16_header(out_buf, UINT32_MAX, info.sample_rate);
|
|
wav_header_sent = true;
|
|
}
|
|
append_wav16_pcm(out_buf, audio_pcm.data() + pcm_sent, n_new);
|
|
pcm_sent = audio_pcm.size();
|
|
*out_data = out_buf.data();
|
|
*out_data_len = out_buf.size();
|
|
return 0;
|
|
}
|
|
|
|
int32_t flush() override {
|
|
return flush_gen_wav() ? 0 : 1;
|
|
}
|
|
|
|
private:
|
|
bool ensure_cache() {
|
|
if (specials_ok) {
|
|
return true;
|
|
}
|
|
codec_0 = find_special_token(vocab, "<|codec_0|>");
|
|
codec_bos = find_special_token(vocab, "<|codec_bos|>");
|
|
codec_eos = find_special_token(vocab, "<|codec_eos_token|>");
|
|
codec_pad = find_special_token(vocab, "<|codec_pad|>");
|
|
c_think = find_special_token(vocab, "<|codec_think|>");
|
|
c_think_b = find_special_token(vocab, "<|codec_think_bos|>");
|
|
c_think_e = find_special_token(vocab, "<|codec_think_eos|>");
|
|
tts_pad = find_special_token(vocab, "<tts_pad>");
|
|
tts_bos = find_special_token(vocab, "<tts_text_bos>");
|
|
tts_eos = find_special_token(vocab, "<tts_text_eod>");
|
|
for (llama_token t : { codec_0, codec_bos, codec_eos, codec_pad,
|
|
c_think, c_think_b, c_think_e,
|
|
tts_pad, tts_bos, tts_eos }) {
|
|
if (t == LLAMA_TOKEN_NULL) {
|
|
LOG_ERR("mtmd_helper_gen_audio: missing a required special token in vocab\n");
|
|
return false;
|
|
}
|
|
}
|
|
const uint32_t n_tok_embd = llama_model_get_tok_embd(model, nullptr);
|
|
if (n_tok_embd == 0) {
|
|
LOG_ERR("mtmd_helper_gen_audio: model has no token embeddings\n");
|
|
return false;
|
|
}
|
|
tok_embd.resize(n_tok_embd);
|
|
if (llama_model_get_tok_embd(model, tok_embd.data()) != n_tok_embd) {
|
|
LOG_ERR("mtmd_helper_gen_audio: token embedding copy failed\n");
|
|
return false;
|
|
}
|
|
specials_ok = true;
|
|
return true;
|
|
}
|
|
|
|
// runs the reference wav through the speaker encoder, returns one x-vector embedding row
|
|
bool encode_speaker(mtmd_bitmap * bitmap, std::vector<float> & out) {
|
|
if (!mtmd_bitmap_is_voice(bitmap) && !mtmd_support_audio(mctx)) {
|
|
LOG_ERR("mtmd_helper_gen_audio: mmproj has no speaker/audio encoder\n");
|
|
return false;
|
|
}
|
|
const std::string marker = mtmd_get_marker(mctx);
|
|
mtmd_input_text text{ marker.c_str(), marker.size(), false, true };
|
|
mtmd_input_chunks * chunks = mtmd_input_chunks_init();
|
|
const mtmd_bitmap * bptr = bitmap;
|
|
bool ok = mtmd_tokenize(mctx, chunks, &text, &bptr, 1) == 0;
|
|
if (ok) {
|
|
ok = false;
|
|
for (size_t i = 0; i < mtmd_input_chunks_size(chunks); i++) {
|
|
const mtmd_input_chunk * chunk = mtmd_input_chunks_get(chunks, i);
|
|
if (mtmd_input_chunk_get_type(chunk) != MTMD_INPUT_CHUNK_TYPE_AUDIO) {
|
|
continue;
|
|
}
|
|
if (mtmd_encode_chunk(mctx, chunk) != 0) {
|
|
LOG_ERR("mtmd_helper_gen_audio: speaker encode failed\n");
|
|
break;
|
|
}
|
|
const float * embd = mtmd_get_output_embd(mctx);
|
|
const size_t n = (size_t) llama_model_n_embd_inp(model) * mtmd_input_chunk_get_n_tokens(chunk);
|
|
out.assign(embd, embd + n);
|
|
ok = true;
|
|
break;
|
|
}
|
|
}
|
|
mtmd_input_chunks_free(chunks);
|
|
return ok;
|
|
}
|
|
|
|
// one GEN_WAV process() call over the buffered codes, state is carried across batches
|
|
bool flush_gen_wav() {
|
|
if (codes_buf.empty()) {
|
|
return true;
|
|
}
|
|
mtmd_gen_inp inp = mtmd_gen_inp_default(mctx);
|
|
inp.type = MTMD_GEN_PROCESS_TYPE_GEN_WAV;
|
|
inp.codes = codes_buf.data();
|
|
inp.n_codes = codes_buf.size();
|
|
mtmd_gen_out out{};
|
|
if (mtmd_gen_audio_process(mctx, gen_state.get(), &inp, &out) != 0) {
|
|
LOG_ERR("mtmd_helper_gen_audio: gen_wav process failed\n");
|
|
return false;
|
|
}
|
|
audio_pcm.insert(audio_pcm.end(), out.audio, out.audio + out.n_samples);
|
|
codes_buf.clear();
|
|
return true;
|
|
}
|
|
|
|
// vocab specials fixed across the whole session, looked up once
|
|
bool specials_ok = false;
|
|
llama_token codec_0 = LLAMA_TOKEN_NULL;
|
|
llama_token codec_bos = LLAMA_TOKEN_NULL;
|
|
llama_token codec_eos = LLAMA_TOKEN_NULL;
|
|
llama_token codec_pad = LLAMA_TOKEN_NULL;
|
|
llama_token c_think = LLAMA_TOKEN_NULL;
|
|
llama_token c_think_b = LLAMA_TOKEN_NULL;
|
|
llama_token c_think_e = LLAMA_TOKEN_NULL;
|
|
llama_token tts_pad = LLAMA_TOKEN_NULL;
|
|
llama_token tts_bos = LLAMA_TOKEN_NULL;
|
|
llama_token tts_eos = LLAMA_TOKEN_NULL;
|
|
std::vector<float> tok_embd; // whole token embedding matrix, n_vocab * n_embd
|
|
|
|
// must match hparams.wav_tfm_swa hardcoded in clip.cpp
|
|
size_t window_frames = 72;
|
|
|
|
// per-generation state, cleared by reset()
|
|
llama_seq_id seq_id = 0;
|
|
bool mrope = false;
|
|
int pos = 0;
|
|
// prompt decode state, consumed batch-by-batch by step_prompt()
|
|
std::vector<float> prompt_embd_buf;
|
|
std::unique_ptr<decode_embd_batch> prompt_batch;
|
|
int n_prompt = 0;
|
|
int prompt_pos = 0;
|
|
int32_t top_k = 50;
|
|
float top_p = 1.0f;
|
|
std::vector<int32_t> codes_buf;
|
|
std::vector<float> audio_pcm;
|
|
std::vector<float> overlay;
|
|
std::vector<float> h_state_buf;
|
|
mtmd_helper_gen_audio_outtype out_type = MTMD_HELPER_GEN_AUDIO_OUTTYPE_WAV;
|
|
std::vector<char> out_buf;
|
|
bool stream = false;
|
|
size_t pcm_sent = 0; // samples already returned by get_output()
|
|
bool wav_header_sent = false;
|
|
};
|
|
|
|
// settings that only live in the reference's per-pack yaml, not in the checkpoint
|
|
// the english packs share the same shapes and tokenizer, but disagree on these
|
|
// all three are 0 / false when the pack does not tune them, the model default is then used
|
|
struct pockettts_pack_settings {
|
|
float temp = 0.0f;
|
|
int frames_after_eos = 0;
|
|
bool pad_short_text = false;
|
|
};
|
|
|
|
static pockettts_pack_settings pockettts_pack(const char * variant) {
|
|
static const std::unordered_map<std::string, pockettts_pack_settings> packs = {
|
|
{ "english", { 0.3f, 0, false } },
|
|
{ "english_2026-01", { 0.7f, 0, true } },
|
|
{ "english_2026-04", { 0.3f, 0, false } },
|
|
{ "french_24l", { 0.7f, 8, false } },
|
|
};
|
|
auto it = packs.find(variant ? variant : "");
|
|
if (it == packs.end()) {
|
|
LOG_WRN("mtmd_helper_gen_audio: no tuned settings for pocket-tts variant \"%s\"\n",
|
|
variant ? variant : "");
|
|
return {};
|
|
}
|
|
return it->second;
|
|
}
|
|
|
|
// pocket-tts: the backbone emits no token, the flow net turns each hidden state into a latent
|
|
// the end-of-speech head also lives in the mmproj
|
|
class pockettts_gen_audio_pipeline : public mtmd_gen_audio_pipeline {
|
|
public:
|
|
using mtmd_gen_audio_pipeline::mtmd_gen_audio_pipeline;
|
|
|
|
void reset() override {
|
|
seq_id = 0;
|
|
pos = 0;
|
|
feats_buf.clear();
|
|
gen_state.reset();
|
|
audio_pcm.clear();
|
|
h_state_buf.clear();
|
|
out_buf.clear();
|
|
prompt_embd_buf.clear();
|
|
prompt_batch.reset();
|
|
n_prompt = 0;
|
|
prompt_pos = 0;
|
|
step_idx = 0;
|
|
eos_step = -1;
|
|
chunks.clear();
|
|
chunk_idx = 0;
|
|
n_voice_pos = 0;
|
|
chunk_budget = 0;
|
|
stream = false;
|
|
pcm_sent = 0;
|
|
wav_header_sent = false;
|
|
}
|
|
|
|
int32_t set_input(const mtmd_helper_gen_audio_inp * inp) override {
|
|
reset();
|
|
seq_id = inp->seq_id;
|
|
|
|
if (!ensure_cache()) {
|
|
return 1;
|
|
}
|
|
|
|
std::vector<float> voice;
|
|
if (inp->speaker_ref) {
|
|
if (!encode_speaker(inp->speaker_ref, voice)) {
|
|
return 1;
|
|
}
|
|
}
|
|
|
|
pack = pockettts_pack(info.model_variant);
|
|
|
|
const std::string text = prepare_text(std::string(inp->prompt, inp->prompt_len),
|
|
pack.pad_short_text);
|
|
if (text.empty()) {
|
|
LOG_ERR("mtmd_helper_gen_audio: empty prompt\n");
|
|
return 1;
|
|
}
|
|
|
|
std::vector<llama_token> ids(text.size() + 16);
|
|
int n_ids = llama_tokenize(vocab, text.c_str(), (int32_t) text.size(), ids.data(),
|
|
(int32_t) ids.size(), false, false);
|
|
if (n_ids <= 0) {
|
|
LOG_ERR("mtmd_helper_gen_audio: tokenization failed\n");
|
|
return 1;
|
|
}
|
|
ids.resize((size_t) n_ids);
|
|
|
|
// long inputs degrade badly, so each chunk restarts from the voice conditioning
|
|
// see split_into_best_sentences() in the reference
|
|
chunks = split_chunks(ids);
|
|
chunk_idx = 0;
|
|
if (chunks.size() > 1) {
|
|
LOG_INF("mtmd_helper_gen_audio: %d tokens split into %zu chunks\n", n_ids, chunks.size());
|
|
}
|
|
|
|
const int n_e = n_embd;
|
|
|
|
// sequence order is voice, then text, then the audio BOS that starts generation
|
|
if (!voice.empty()) {
|
|
GGML_ASSERT(voice.size() % (size_t) n_e == 0);
|
|
if (bos_before_voice != LLAMA_TOKEN_NULL) {
|
|
push_embd_row(prompt_embd_buf, bos_before_voice);
|
|
}
|
|
prompt_embd_buf.insert(prompt_embd_buf.end(), voice.begin(), voice.end());
|
|
}
|
|
// every later chunk rewinds to here and re-prompts, so the voice stays primed
|
|
n_voice_pos = (int) (prompt_embd_buf.size() / (size_t) n_e);
|
|
|
|
for (llama_token t : chunks[0]) {
|
|
push_embd_row(prompt_embd_buf, t);
|
|
}
|
|
push_embd_row(prompt_embd_buf, audio_bos);
|
|
arm_chunk_budget(0);
|
|
|
|
n_prompt = (int) (prompt_embd_buf.size() / (size_t) n_e);
|
|
prompt_batch.reset(new decode_embd_batch(prompt_embd_buf.data(), n_prompt, 1, n_e));
|
|
prompt_batch->set_position_normal(0, seq_id);
|
|
prompt_pos = 0;
|
|
|
|
out_type = inp->out_type;
|
|
stream = inp->stream;
|
|
if (!init_gen_state(inp->seed)) {
|
|
return 1;
|
|
}
|
|
|
|
return 0;
|
|
}
|
|
|
|
int32_t step_prompt(int32_t n_batch) override {
|
|
GGML_ASSERT(n_batch > 0);
|
|
if (prompt_pos >= n_prompt) {
|
|
return 0;
|
|
}
|
|
const int32_t n_tokens_batch = std::min(n_batch, n_prompt - prompt_pos);
|
|
|
|
if ((prompt_pos + n_tokens_batch) == n_prompt) {
|
|
prompt_batch->logits[prompt_pos + n_tokens_batch - 1] = 1;
|
|
}
|
|
|
|
if (llama_process(lctx, LLAMA_PROCESS_TYPE_DECODE, prompt_batch->render(lctx, prompt_pos, n_tokens_batch)) != 0) {
|
|
LOG_ERR("mtmd_helper_gen_audio: prompt decode failed\n");
|
|
return -1;
|
|
}
|
|
|
|
pos += n_tokens_batch;
|
|
prompt_pos += n_tokens_batch;
|
|
|
|
if (prompt_pos >= n_prompt) {
|
|
prompt_batch.reset();
|
|
prompt_embd_buf.clear();
|
|
return 0;
|
|
}
|
|
return n_prompt - prompt_pos;
|
|
}
|
|
|
|
int32_t step_gen(llama_token sampled, const float * h_state_in, const float ** h_state_out, bool * out_stop) override {
|
|
(void) sampled; // the backbone output is continuous, there is no token to consume
|
|
|
|
mtmd_gen_inp inp = mtmd_gen_inp_default(mctx);
|
|
inp.type = MTMD_GEN_PROCESS_TYPE_GEN_CODE;
|
|
inp.embd = const_cast<float *>(h_state_in);
|
|
if (pack.temp > 0.0f) {
|
|
inp.temp = pack.temp;
|
|
}
|
|
mtmd_gen_out out{};
|
|
if (mtmd_gen_audio_process(mctx, gen_state.get(), &inp, &out) != 0) {
|
|
LOG_ERR("mtmd_helper_gen_audio: flow decode failed\n");
|
|
return 1;
|
|
}
|
|
if (out.is_eos && eos_step < 0) {
|
|
eos_step = step_idx;
|
|
}
|
|
// the frame of the stopping step is discarded, matching _autoregressive_generation().
|
|
// the budget is the reference's fallback for a chunk whose eos head never fires
|
|
const bool chunk_done = (eos_step >= 0 && step_idx >= eos_step + frames_after_eos) ||
|
|
step_idx >= chunk_budget;
|
|
if (chunk_done) {
|
|
if (eos_step < 0) {
|
|
LOG_WRN("mtmd_helper_gen_audio: chunk %zu hit its budget without end-of-speech\n", chunk_idx);
|
|
}
|
|
return finish_chunk(h_state_out, out_stop);
|
|
}
|
|
|
|
feats_buf.insert(feats_buf.end(), out.feats, out.feats + out.n_feats);
|
|
step_idx++;
|
|
if (out.n_feats > 0 && feats_buf.size() / out.n_feats >= window_frames) {
|
|
if (!flush_gen_wav()) {
|
|
return 1;
|
|
}
|
|
}
|
|
|
|
decode_embd_batch batch_embd(out.embd, 1, 1, n_embd);
|
|
batch_embd.set_position_normal(pos, seq_id);
|
|
batch_embd.logits[0] = 1;
|
|
pos++;
|
|
|
|
if (llama_process(lctx, LLAMA_PROCESS_TYPE_DECODE, batch_embd.render(lctx, 0, 1)) != 0) {
|
|
LOG_ERR("mtmd_helper_gen_audio: decode failed\n");
|
|
return 1;
|
|
}
|
|
|
|
const float * he = llama_get_embeddings_ith(lctx, -1);
|
|
h_state_buf.assign(he, he + n_embd);
|
|
*h_state_out = h_state_buf.data();
|
|
|
|
return 0;
|
|
}
|
|
|
|
int32_t get_output(int32_t * out_sample_rate, const char ** out_data, size_t * out_data_len, int64_t * out_n_samples) override {
|
|
*out_sample_rate = info.sample_rate;
|
|
|
|
if (!stream) {
|
|
// one-shot call: force out whatever's left, regardless of window_frames
|
|
if (!flush_gen_wav()) {
|
|
return 1;
|
|
}
|
|
if (out_n_samples) {
|
|
*out_n_samples = (int64_t) audio_pcm.size();
|
|
}
|
|
if (out_type == MTMD_HELPER_GEN_AUDIO_OUTTYPE_PCM) {
|
|
*out_data = (const char *) audio_pcm.data();
|
|
*out_data_len = audio_pcm.size() * sizeof(float);
|
|
return 0;
|
|
}
|
|
out_buf.clear();
|
|
if (!write_wav16(out_buf, audio_pcm, info.sample_rate)) {
|
|
LOG_ERR("mtmd_helper_gen_audio: output too large for WAV\n");
|
|
return 1;
|
|
}
|
|
*out_data = out_buf.data();
|
|
*out_data_len = out_buf.size();
|
|
return 0;
|
|
}
|
|
|
|
// streaming: only return audio produced since the previous call
|
|
const size_t n_new = audio_pcm.size() - pcm_sent;
|
|
if (out_n_samples) {
|
|
*out_n_samples = (int64_t) n_new;
|
|
}
|
|
|
|
if (out_type == MTMD_HELPER_GEN_AUDIO_OUTTYPE_PCM) {
|
|
*out_data = (const char *) (audio_pcm.data() + pcm_sent);
|
|
*out_data_len = n_new * sizeof(float);
|
|
pcm_sent = audio_pcm.size();
|
|
return 0;
|
|
}
|
|
|
|
out_buf.clear();
|
|
if (!wav_header_sent) {
|
|
write_wav16_header(out_buf, UINT32_MAX, info.sample_rate);
|
|
wav_header_sent = true;
|
|
}
|
|
append_wav16_pcm(out_buf, audio_pcm.data() + pcm_sent, n_new);
|
|
pcm_sent = audio_pcm.size();
|
|
*out_data = out_buf.data();
|
|
*out_data_len = out_buf.size();
|
|
return 0;
|
|
}
|
|
|
|
int32_t flush() override {
|
|
return flush_gen_wav() ? 0 : 1;
|
|
}
|
|
|
|
private:
|
|
bool ensure_cache() {
|
|
if (specials_ok) {
|
|
return true;
|
|
}
|
|
// bos_before_voice is optional, some packs do not insert it
|
|
bos_before_voice = find_special_token(vocab, "<|bos_before_voice|>");
|
|
audio_bos = find_special_token(vocab, "<|audio_bos|>");
|
|
if (audio_bos == LLAMA_TOKEN_NULL) {
|
|
LOG_ERR("mtmd_helper_gen_audio: missing <|audio_bos|> in vocab\n");
|
|
return false;
|
|
}
|
|
const uint32_t n_tok_embd = llama_model_get_tok_embd(model, nullptr);
|
|
if (n_tok_embd == 0) {
|
|
LOG_ERR("mtmd_helper_gen_audio: model has no token embeddings\n");
|
|
return false;
|
|
}
|
|
tok_embd.resize(n_tok_embd);
|
|
if (llama_model_get_tok_embd(model, tok_embd.data()) != n_tok_embd) {
|
|
LOG_ERR("mtmd_helper_gen_audio: token embedding copy failed\n");
|
|
return false;
|
|
}
|
|
GGML_ASSERT(n_embd > 0 && n_tok_embd % (uint32_t) n_embd == 0);
|
|
specials_ok = true;
|
|
return true;
|
|
}
|
|
|
|
// the table can be shorter than the vocab, so bound the row lookup
|
|
void push_embd_row(std::vector<float> & dst, llama_token t) const {
|
|
const size_t n_rows = tok_embd.size() / (size_t) n_embd;
|
|
GGML_ASSERT(t >= 0 && (size_t) t < n_rows);
|
|
dst.insert(dst.end(),
|
|
tok_embd.begin() + (size_t) t * n_embd,
|
|
tok_embd.begin() + (size_t) (t + 1) * n_embd);
|
|
}
|
|
|
|
// token ids of the pieces the reference splits on, see split_into_best_sentences().
|
|
// the leading token is dropped, it is the tokenizer's dummy prefix
|
|
std::vector<llama_token> punct_ids(const char * s) const {
|
|
std::vector<llama_token> ids(16);
|
|
const int n = llama_tokenize(vocab, s, (int32_t) strlen(s), ids.data(), (int32_t) ids.size(), false, false);
|
|
if (n <= 1) {
|
|
return {};
|
|
}
|
|
return std::vector<llama_token>(ids.begin() + 1, ids.begin() + n);
|
|
}
|
|
|
|
// cut after runs of boundary tokens, so punctuation stays with the sentence it ends
|
|
static std::vector<std::vector<llama_token>> split_on(const std::vector<llama_token> & ids,
|
|
const std::vector<llama_token> & boundary) {
|
|
std::vector<std::vector<llama_token>> out;
|
|
size_t start = 0;
|
|
bool prev_was_boundary = false;
|
|
for (size_t i = 0; i < ids.size(); i++) {
|
|
const bool is_boundary = std::find(boundary.begin(), boundary.end(), ids[i]) != boundary.end();
|
|
if (!is_boundary && prev_was_boundary) {
|
|
out.emplace_back(ids.begin() + start, ids.begin() + i);
|
|
start = i;
|
|
}
|
|
prev_was_boundary = is_boundary;
|
|
}
|
|
out.emplace_back(ids.begin() + start, ids.end());
|
|
return out;
|
|
}
|
|
|
|
std::vector<std::vector<llama_token>> split_chunks(const std::vector<llama_token> & ids) const {
|
|
if ((int) ids.size() <= max_chunk_tokens) {
|
|
return { ids };
|
|
}
|
|
const std::vector<llama_token> eos_punct = punct_ids(".!...?");
|
|
const std::vector<llama_token> mid_punct = punct_ids(",;:");
|
|
|
|
// oversized sentences are split again on weaker punctuation, else words get skipped
|
|
std::vector<std::vector<llama_token>> segments;
|
|
for (auto & seg : split_on(ids, eos_punct)) {
|
|
if ((int) seg.size() <= max_chunk_tokens) {
|
|
segments.push_back(std::move(seg));
|
|
continue;
|
|
}
|
|
auto sub = split_on(seg, mid_punct);
|
|
if (sub.size() > 1) {
|
|
for (auto & s : sub) {
|
|
segments.push_back(std::move(s));
|
|
}
|
|
} else {
|
|
segments.push_back(std::move(seg));
|
|
}
|
|
}
|
|
|
|
std::vector<std::vector<llama_token>> out;
|
|
for (auto & seg : segments) {
|
|
if (seg.empty()) {
|
|
continue;
|
|
}
|
|
if (!out.empty() && (int) (out.back().size() + seg.size()) <= max_chunk_tokens) {
|
|
out.back().insert(out.back().end(), seg.begin(), seg.end());
|
|
} else {
|
|
out.push_back(std::move(seg));
|
|
}
|
|
}
|
|
if (out.empty()) {
|
|
out.push_back(ids);
|
|
}
|
|
for (const auto & c : out) {
|
|
if ((int) c.size() > max_chunk_tokens) {
|
|
LOG_WRN("mtmd_helper_gen_audio: chunk of %zu tokens exceeds the %d token budget, "
|
|
"generation may skip words\n", c.size(), max_chunk_tokens);
|
|
}
|
|
}
|
|
return out;
|
|
}
|
|
|
|
// _estimate_max_gen_len() plus the per-chunk tail guess, both in frames
|
|
void arm_chunk_budget(size_t idx) {
|
|
const int n_tok = (int) chunks[idx].size();
|
|
chunk_budget = (int) std::ceil((n_tok / 3.0 + 2.0) * frame_rate);
|
|
// the pack may pin the tail, else the reference guesses it from the word count
|
|
frames_after_eos = pack.frames_after_eos > 0 ? pack.frames_after_eos : (n_tok <= 6 ? 5 : 3);
|
|
step_idx = 0;
|
|
eos_step = -1;
|
|
}
|
|
|
|
// ends the current chunk and, if there is another, re-prompts it on top of the voice
|
|
int32_t finish_chunk(const float ** h_state_out, bool * out_stop) {
|
|
if (!flush_gen_wav()) {
|
|
return 1;
|
|
}
|
|
// the decoder restarts too, the next chunk's audio is not continuous with this one
|
|
mtmd_gen_state_reset(gen_state.get());
|
|
|
|
if (chunk_idx + 1 >= chunks.size()) {
|
|
*out_stop = true;
|
|
*h_state_out = nullptr;
|
|
return 0;
|
|
}
|
|
chunk_idx++;
|
|
|
|
// drop this chunk's text and audio, keep the voice conditioning
|
|
llama_memory_seq_rm(llama_get_memory(lctx), seq_id, n_voice_pos, -1);
|
|
pos = n_voice_pos;
|
|
|
|
const int n_e = n_embd;
|
|
prompt_embd_buf.clear();
|
|
for (llama_token t : chunks[chunk_idx]) {
|
|
push_embd_row(prompt_embd_buf, t);
|
|
}
|
|
push_embd_row(prompt_embd_buf, audio_bos);
|
|
arm_chunk_budget(chunk_idx);
|
|
|
|
const int n_rows = (int) (prompt_embd_buf.size() / (size_t) n_e);
|
|
GGML_ASSERT(n_rows > 0);
|
|
decode_embd_batch batch(prompt_embd_buf.data(), n_rows, 1, n_e);
|
|
batch.set_position_normal(pos, seq_id);
|
|
batch.logits[n_rows - 1] = 1;
|
|
if (llama_process(lctx, LLAMA_PROCESS_TYPE_DECODE, batch.render(lctx, 0, n_rows)) != 0) {
|
|
LOG_ERR("mtmd_helper_gen_audio: chunk prompt decode failed\n");
|
|
return 1;
|
|
}
|
|
pos += n_rows;
|
|
prompt_embd_buf.clear();
|
|
|
|
const float * he = llama_get_embeddings_ith(lctx, -1);
|
|
h_state_buf.assign(he, he + n_embd);
|
|
*h_state_out = h_state_buf.data();
|
|
*out_stop = false;
|
|
return 0;
|
|
}
|
|
|
|
// same normalization as prepare_text_prompt() in the reference, it affects quality
|
|
static std::string prepare_text(const std::string & in, bool pad_short) {
|
|
std::string s;
|
|
s.reserve(in.size() + 1);
|
|
for (char c : in) {
|
|
if (c == '\n' || c == '\r') {
|
|
s += ' ';
|
|
} else if (c == ';') {
|
|
s += ',';
|
|
} else {
|
|
s += c;
|
|
}
|
|
}
|
|
const size_t b = s.find_first_not_of(' ');
|
|
const size_t e = s.find_last_not_of(' ');
|
|
if (b == std::string::npos) {
|
|
return "";
|
|
}
|
|
s = s.substr(b, e - b + 1);
|
|
if (s[0] >= 'a' && s[0] <= 'z') {
|
|
s[0] = (char) (s[0] - 'a' + 'A');
|
|
}
|
|
const unsigned char last = (unsigned char) s.back();
|
|
if (std::isalnum(last)) {
|
|
s += '.';
|
|
}
|
|
if (pad_short && count_words(s) < 5) {
|
|
s = std::string(8, ' ') + s;
|
|
}
|
|
return s;
|
|
}
|
|
|
|
static int count_words(const std::string & s) {
|
|
int n = 0;
|
|
bool in_word = false;
|
|
for (char c : s) {
|
|
if (c == ' ') {
|
|
in_word = false;
|
|
} else if (!in_word) {
|
|
in_word = true;
|
|
n++;
|
|
}
|
|
}
|
|
return n;
|
|
}
|
|
|
|
// runs the reference wav through the mimi encoder, returns one row per 12.5Hz frame
|
|
bool encode_speaker(mtmd_bitmap * bitmap, std::vector<float> & out) {
|
|
if (!mtmd_bitmap_is_voice(bitmap) && !mtmd_support_audio(mctx)) {
|
|
LOG_ERR("mtmd_helper_gen_audio: mmproj has no voice encoder\n");
|
|
return false;
|
|
}
|
|
const std::string marker = mtmd_get_marker(mctx);
|
|
mtmd_input_text text{ marker.c_str(), marker.size(), false, true };
|
|
mtmd_input_chunks * chunks = mtmd_input_chunks_init();
|
|
const mtmd_bitmap * bptr = bitmap;
|
|
bool ok = mtmd_tokenize(mctx, chunks, &text, &bptr, 1) == 0;
|
|
if (ok) {
|
|
ok = false;
|
|
for (size_t i = 0; i < mtmd_input_chunks_size(chunks); i++) {
|
|
const mtmd_input_chunk * chunk = mtmd_input_chunks_get(chunks, i);
|
|
if (mtmd_input_chunk_get_type(chunk) != MTMD_INPUT_CHUNK_TYPE_AUDIO) {
|
|
continue;
|
|
}
|
|
if (mtmd_encode_chunk(mctx, chunk) != 0) {
|
|
LOG_ERR("mtmd_helper_gen_audio: voice encode failed\n");
|
|
break;
|
|
}
|
|
const float * embd = mtmd_get_output_embd(mctx);
|
|
const size_t n = (size_t) llama_model_n_embd_inp(model) * mtmd_input_chunk_get_n_tokens(chunk);
|
|
out.assign(embd, embd + n);
|
|
ok = true;
|
|
break;
|
|
}
|
|
}
|
|
mtmd_input_chunks_free(chunks);
|
|
return ok;
|
|
}
|
|
|
|
// decodes the buffered latents, the mimi decoder state carries over between calls
|
|
bool flush_gen_wav() {
|
|
if (feats_buf.empty()) {
|
|
return true;
|
|
}
|
|
mtmd_gen_inp inp = mtmd_gen_inp_default(mctx);
|
|
inp.type = MTMD_GEN_PROCESS_TYPE_GEN_WAV;
|
|
inp.feats = feats_buf.data();
|
|
inp.n_feats = feats_buf.size();
|
|
mtmd_gen_out out{};
|
|
if (mtmd_gen_audio_process(mctx, gen_state.get(), &inp, &out) != 0) {
|
|
LOG_ERR("mtmd_helper_gen_audio: mimi decode failed\n");
|
|
return false;
|
|
}
|
|
audio_pcm.insert(audio_pcm.end(), out.audio, out.audio + out.n_samples);
|
|
feats_buf.clear();
|
|
return true;
|
|
}
|
|
|
|
pockettts_pack_settings pack;
|
|
bool specials_ok = false;
|
|
llama_token bos_before_voice = LLAMA_TOKEN_NULL;
|
|
llama_token audio_bos = LLAMA_TOKEN_NULL;
|
|
std::vector<float> tok_embd;
|
|
|
|
llama_seq_id seq_id = 0;
|
|
int pos = 0;
|
|
std::vector<float> prompt_embd_buf;
|
|
std::unique_ptr<decode_embd_batch> prompt_batch;
|
|
int n_prompt = 0;
|
|
int prompt_pos = 0;
|
|
// end-of-speech is latched, then a few more frames are generated as tail padding
|
|
int step_idx = 0;
|
|
int eos_step = -1;
|
|
int frames_after_eos = 3;
|
|
static constexpr int max_chunk_tokens = 50; // MAX_TOKEN_PER_CHUNK in the reference
|
|
static constexpr double frame_rate = 12.5;
|
|
std::vector<std::vector<llama_token>> chunks;
|
|
size_t chunk_idx = 0;
|
|
int n_voice_pos = 0; // KV positions held by the voice conditioning
|
|
int chunk_budget = 0;
|
|
|
|
// latents are decoded a window at a time, the decoder state bridges the windows
|
|
size_t window_frames = 8;
|
|
std::vector<float> feats_buf;
|
|
std::vector<float> audio_pcm;
|
|
std::vector<float> h_state_buf;
|
|
mtmd_helper_gen_audio_outtype out_type = MTMD_HELPER_GEN_AUDIO_OUTTYPE_WAV;
|
|
std::vector<char> out_buf;
|
|
bool stream = false;
|
|
size_t pcm_sent = 0; // samples already returned by get_output()
|
|
bool wav_header_sent = false;
|
|
};
|
|
|
|
// Soprano uses the hidden state that predicts each non-EOS token, including the prompt's last state.
|
|
class soprano_gen_audio_pipeline : public mtmd_gen_audio_pipeline {
|
|
public:
|
|
using mtmd_gen_audio_pipeline::mtmd_gen_audio_pipeline;
|
|
|
|
void reset() override {
|
|
prompt.clear();
|
|
features.clear();
|
|
h_next.clear();
|
|
audio.clear();
|
|
wav.clear();
|
|
pos = 0;
|
|
prompt_pos = 0;
|
|
decoded = false;
|
|
sent = false;
|
|
}
|
|
|
|
int32_t set_input(const mtmd_helper_gen_audio_inp * inp) override {
|
|
reset();
|
|
if (inp->speaker_ref || !inp->prompt || inp->prompt_len == 0 || inp->prompt_len > INT32_MAX ||
|
|
inp->seq_id < 0 || (uint32_t) inp->seq_id >= llama_n_seq_max(lctx)) {
|
|
LOG_ERR("mtmd_helper_gen_audio: soprano requires text, without a speaker reference\n");
|
|
return 1;
|
|
}
|
|
seq_id = inp->seq_id;
|
|
out_type = inp->out_type;
|
|
stream = inp->stream;
|
|
const llama_token text = find_special_token(vocab, "[TEXT]");
|
|
const llama_token start = find_special_token(vocab, "[START]");
|
|
const llama_token stop = find_special_token(vocab, "[STOP]");
|
|
if (text == LLAMA_TOKEN_NULL || start == LLAMA_TOKEN_NULL || stop == LLAMA_TOKEN_NULL) {
|
|
LOG_ERR("mtmd_helper_gen_audio: incompatible soprano vocabulary\n");
|
|
return 1;
|
|
}
|
|
const int required = llama_tokenize(vocab, inp->prompt, inp->prompt_len, nullptr, 0, false, false);
|
|
if (required >= 0 || (uint32_t) (-required + 3) >= llama_n_ctx_seq(lctx)) {
|
|
LOG_ERR("mtmd_helper_gen_audio: soprano prompt does not fit in the context\n");
|
|
return 1;
|
|
}
|
|
const int n = -required;
|
|
prompt.resize(n + 3);
|
|
prompt[0] = stop;
|
|
prompt[1] = text;
|
|
if (llama_tokenize(vocab, inp->prompt, inp->prompt_len, prompt.data() + 2, n, false, false) != n) {
|
|
return 1;
|
|
}
|
|
prompt.back() = start;
|
|
if (!llama_memory_seq_rm(llama_get_memory(lctx), seq_id, 0, -1)) {
|
|
LOG_ERR("mtmd_helper_gen_audio: cannot reset soprano sequence\n");
|
|
return 1;
|
|
}
|
|
if (!init_gen_state(inp->seed)) {
|
|
return 1;
|
|
}
|
|
return 0;
|
|
}
|
|
|
|
int32_t step_prompt(int32_t n_batch) override {
|
|
if (n_batch <= 0 || prompt.empty()) {
|
|
return -1;
|
|
}
|
|
const int n = std::min(n_batch, (int) prompt.size() - prompt_pos);
|
|
if (n == 0) {
|
|
return 0;
|
|
}
|
|
if (!decode(prompt.data() + prompt_pos, n)) {
|
|
return -1;
|
|
}
|
|
prompt_pos += n;
|
|
return (int) prompt.size() - prompt_pos;
|
|
}
|
|
|
|
int32_t step_gen(llama_token sampled, const float * h_state_in, const float ** h_state_out, bool * out_stop) override {
|
|
*h_state_out = nullptr;
|
|
*out_stop = false;
|
|
if (sampled < 0 || sampled >= llama_vocab_n_tokens(vocab) || !h_state_in || prompt_pos != (int) prompt.size() || prompt.empty()) {
|
|
return 1;
|
|
}
|
|
if (llama_vocab_is_eog(vocab, sampled) || (uint32_t) pos >= llama_n_ctx_seq(lctx)) {
|
|
*out_stop = true;
|
|
return 0;
|
|
}
|
|
features.insert(features.end(), h_state_in, h_state_in + n_embd);
|
|
if (!decode(&sampled, 1)) {
|
|
features.resize(features.size() - n_embd);
|
|
return 1;
|
|
}
|
|
const float * h = llama_get_embeddings_ith(lctx, -1);
|
|
if (!h) {
|
|
return 1;
|
|
}
|
|
h_next.assign(h, h + n_embd);
|
|
*h_state_out = h_next.data();
|
|
return 0;
|
|
}
|
|
|
|
int32_t get_output(int32_t * sample_rate, const char ** data, size_t * data_len, int64_t * n_samples) override {
|
|
*sample_rate = info.sample_rate;
|
|
*data_len = 0;
|
|
if (n_samples) { *n_samples = 0; }
|
|
if (stream) {
|
|
// the decoder needs the whole utterance, so the audio comes out in one chunk after flush()
|
|
if (!decoded || sent) {
|
|
return 0;
|
|
}
|
|
sent = true;
|
|
} else if (!decode_audio()) {
|
|
return 1;
|
|
}
|
|
if (n_samples) { *n_samples = audio.size(); }
|
|
if (out_type == MTMD_HELPER_GEN_AUDIO_OUTTYPE_PCM) {
|
|
*data = (const char *) audio.data();
|
|
*data_len = audio.size() * sizeof(float);
|
|
} else {
|
|
wav.clear();
|
|
if (!write_wav16(wav, audio, info.sample_rate)) {
|
|
return 1;
|
|
}
|
|
*data = wav.data();
|
|
*data_len = wav.size();
|
|
}
|
|
return 0;
|
|
}
|
|
|
|
// the whole utterance is decoded at once
|
|
int32_t flush() override {
|
|
return decode_audio() ? 0 : 1;
|
|
}
|
|
|
|
private:
|
|
bool decode_audio() {
|
|
if (decoded) {
|
|
return true;
|
|
}
|
|
if (features.size() < 2 * (size_t) n_embd) {
|
|
LOG_ERR("mtmd_helper_gen_audio: soprano needs at least two generated frames\n");
|
|
return false;
|
|
}
|
|
mtmd_gen_inp inp = mtmd_gen_inp_default(mctx);
|
|
inp.type = MTMD_GEN_PROCESS_TYPE_GEN_WAV;
|
|
inp.feats = features.data();
|
|
inp.n_feats = features.size();
|
|
mtmd_gen_out out{};
|
|
if (mtmd_gen_audio_process(mctx, gen_state.get(), &inp, &out) != 0) {
|
|
return false;
|
|
}
|
|
audio.assign(out.audio, out.audio + out.n_samples);
|
|
decoded = true;
|
|
return true;
|
|
}
|
|
|
|
bool decode(llama_token * tokens, int n) {
|
|
llama_batch batch = llama_batch_init(n, 0, 1);
|
|
batch.n_tokens = n;
|
|
for (int i = 0; i < n; ++i) {
|
|
batch.token[i] = tokens[i];
|
|
batch.pos[i] = pos + i;
|
|
batch.n_seq_id[i] = 1;
|
|
batch.seq_id[i][0] = seq_id;
|
|
batch.logits[i] = i == n - 1;
|
|
}
|
|
const int ret = llama_decode(lctx, batch);
|
|
llama_batch_free(batch);
|
|
if (ret != 0) {
|
|
LOG_ERR("mtmd_helper_gen_audio: soprano backbone decode failed\n");
|
|
return false;
|
|
}
|
|
pos += n;
|
|
return true;
|
|
}
|
|
|
|
llama_seq_id seq_id = 0;
|
|
llama_pos pos = 0;
|
|
int prompt_pos = 0;
|
|
std::vector<llama_token> prompt;
|
|
std::vector<float> features;
|
|
std::vector<float> h_next;
|
|
std::vector<float> audio;
|
|
std::vector<char> wav;
|
|
mtmd_helper_gen_audio_outtype out_type = MTMD_HELPER_GEN_AUDIO_OUTTYPE_WAV;
|
|
bool stream = false;
|
|
bool decoded = false;
|
|
bool sent = false;
|
|
};
|
|
|
|
// LFM3-Audio TTS, see generate() in the reference: each step samples a frame of codes by MaskGIT
|
|
// and feeds its embedding back, the end-of-audio code stops
|
|
class lfm3a_gen_audio_pipeline : public mtmd_gen_audio_pipeline {
|
|
public:
|
|
using mtmd_gen_audio_pipeline::mtmd_gen_audio_pipeline;
|
|
|
|
void reset() override {
|
|
seq_id = 0;
|
|
pos = 0;
|
|
continuing = false;
|
|
mode = MODE_AUDIO;
|
|
slots_left = 0;
|
|
speaking = false;
|
|
text_done = false;
|
|
prompt_tokens.clear();
|
|
prompt_pos = 0;
|
|
hist.clear();
|
|
gen_state.reset();
|
|
pending_tail = false;
|
|
pcm_sent = 0;
|
|
wav_header_sent = false;
|
|
audio_pcm.clear();
|
|
h_state_buf.clear();
|
|
out_buf.clear();
|
|
}
|
|
|
|
int32_t set_input(const mtmd_helper_gen_audio_inp * inp) override {
|
|
reset();
|
|
seq_id = inp->seq_id;
|
|
|
|
// a voice is named in the prompt instead
|
|
if (inp->speaker_ref && !mtmd_bitmap_is_voice(inp->speaker_ref)) {
|
|
LOG_WRN("mtmd_helper_gen_audio: the model has no voice conditioning, the speaker file is ignored\n");
|
|
}
|
|
|
|
// the prompt is rendered by the caller: the "tts" chat template of the model, or the next turn of a conversation
|
|
continuing = inp->n_past > 0;
|
|
pos = continuing ? inp->n_past : 0;
|
|
const std::string full(inp->prompt, inp->prompt_len);
|
|
prompt_tokens.resize(full.size() + 16);
|
|
const int n_ids = llama_tokenize(vocab, full.c_str(), (int32_t) full.size(), prompt_tokens.data(),
|
|
(int32_t) prompt_tokens.size(), false, true);
|
|
if (n_ids <= 0) {
|
|
LOG_ERR("mtmd_helper_gen_audio: tokenization failed\n");
|
|
return 1;
|
|
}
|
|
prompt_tokens.resize((size_t) n_ids);
|
|
|
|
if (!find_tokens()) {
|
|
return 1;
|
|
}
|
|
// a continued response opens with the token of the asked mode, a text-only one cannot switch to audio
|
|
if (continuing) {
|
|
switch (inp->out_mode) {
|
|
case MTMD_HELPER_GEN_AUDIO_OUT_AUDIO: prompt_tokens.push_back(tok_audio_start); break;
|
|
case MTMD_HELPER_GEN_AUDIO_OUT_MIXED: prompt_tokens.push_back(tok_mixed_start); break;
|
|
case MTMD_HELPER_GEN_AUDIO_OUT_TEXT:
|
|
logit_bias = { { tok_audio_start, -INFINITY }, { tok_mixed_start, -INFINITY } };
|
|
break;
|
|
default: break;
|
|
}
|
|
}
|
|
// the last prompt token picks the mode, see generate() of the reference
|
|
const llama_token last = prompt_tokens.back();
|
|
mode = last == tok_audio_start ? MODE_AUDIO : last == tok_mixed_start ? MODE_MIXED : MODE_TEXT;
|
|
slots_left = n_text_block;
|
|
if (!continuing && mode != MODE_AUDIO) {
|
|
LOG_ERR("mtmd_helper_gen_audio: the prompt must come from the \"tts\" chat template of the model\n");
|
|
return 1;
|
|
}
|
|
|
|
// the sampling defaults of the model, the caller's top-k/top-p are not used
|
|
const mtmd_gen_inp def = mtmd_gen_inp_default(mctx);
|
|
temp = def.temp;
|
|
top_k = def.top_k;
|
|
rep_penalty = def.penalty_repeat;
|
|
rep_window = def.penalty_last_n;
|
|
|
|
out_type = inp->out_type;
|
|
stream = inp->stream;
|
|
rng.seed(inp->seed == UINT32_MAX ? std::random_device{}() : inp->seed);
|
|
return init_gen_state(inp->seed) ? 0 : 1;
|
|
}
|
|
|
|
int32_t step_prompt(int32_t n_batch) override {
|
|
GGML_ASSERT(n_batch > 0);
|
|
const int n_prompt = (int) prompt_tokens.size();
|
|
if (prompt_pos >= n_prompt) {
|
|
return 0;
|
|
}
|
|
const int n = std::min(n_batch, n_prompt - prompt_pos);
|
|
llama_batch batch = llama_batch_init(n, 0, 1);
|
|
for (int i = 0; i < n; i++) {
|
|
batch.token [i] = prompt_tokens[(size_t) (prompt_pos + i)];
|
|
batch.pos [i] = pos + i;
|
|
batch.n_seq_id[i] = 1;
|
|
batch.seq_id [i][0] = seq_id;
|
|
batch.logits [i] = prompt_pos + i == n_prompt - 1;
|
|
}
|
|
batch.n_tokens = n;
|
|
const int ret = llama_decode(lctx, batch);
|
|
llama_batch_free(batch);
|
|
if (ret != 0) {
|
|
LOG_ERR("mtmd_helper_gen_audio: prompt decode failed\n");
|
|
return -1;
|
|
}
|
|
pos += n;
|
|
prompt_pos += n;
|
|
return n_prompt - prompt_pos;
|
|
}
|
|
|
|
bool supports_continue() const override {
|
|
return true;
|
|
}
|
|
|
|
bool next_is_text() const override {
|
|
return mode == MODE_TEXT || (mode == MODE_MIXED && !speaking);
|
|
}
|
|
|
|
// one generate() step: a text token from the caller or an audio frame from the hidden state
|
|
int32_t step_gen(llama_token sampled, const float * h_state_in, const float ** h_state_out, bool * out_stop) override {
|
|
const bool interleaving = mode == MODE_MIXED;
|
|
if (next_is_text()) {
|
|
// <|im_end|> ends the response in any text slot, <|text_end|> outside an interleaved block too
|
|
if (sampled == tok_im_end || (!interleaving && sampled == tok_text_end)) {
|
|
*out_stop = true;
|
|
*h_state_out = nullptr;
|
|
return 0;
|
|
}
|
|
// the switching tokens are fed back like any text token
|
|
if (!interleaving) {
|
|
if (sampled == tok_audio_start) {
|
|
mode = MODE_AUDIO;
|
|
} else if (sampled == tok_mixed_start) {
|
|
mode = MODE_MIXED;
|
|
slots_left = n_text_block;
|
|
speaking = false;
|
|
text_done = false;
|
|
}
|
|
} else if (sampled == tok_text_end) {
|
|
text_done = true;
|
|
speaking = true;
|
|
}
|
|
llama_batch batch = llama_batch_get_one(&sampled, 1);
|
|
if (!decode(batch)) {
|
|
return 1;
|
|
}
|
|
} else {
|
|
std::vector<int32_t> codes;
|
|
const float * fb = nullptr;
|
|
bool is_eos = false;
|
|
if (!sample_frame(h_state_in, codes, &fb, &is_eos)) {
|
|
return 1;
|
|
}
|
|
if (is_eos) {
|
|
if (!continuing) {
|
|
// TTS: the end of audio ends the generation
|
|
*out_stop = true;
|
|
*h_state_out = nullptr;
|
|
return 0;
|
|
}
|
|
// the segment ends, the codec tail goes out and the response continues in text
|
|
if (flush() != 0) {
|
|
return 1;
|
|
}
|
|
mode = MODE_TEXT;
|
|
} else {
|
|
hist.insert(hist.end(), codes.begin(), codes.end());
|
|
if (!decode_wav(codes.data(), codes.size())) {
|
|
return 1;
|
|
}
|
|
}
|
|
decode_embd_batch batch_embd(const_cast<float *>(fb), 1, 1, n_embd);
|
|
batch_embd.set_position_normal(pos, seq_id);
|
|
batch_embd.logits[0] = 1;
|
|
if (llama_process(lctx, LLAMA_PROCESS_TYPE_DECODE, batch_embd.render(lctx, 0, 1)) != 0) {
|
|
LOG_ERR("mtmd_helper_gen_audio: decode failed\n");
|
|
return 1;
|
|
}
|
|
pos++;
|
|
}
|
|
|
|
// a slot emitted while interleaving counts down the block, the switching token itself does not
|
|
if (interleaving && !text_done && --slots_left == 0) {
|
|
speaking = !speaking;
|
|
slots_left = speaking ? n_audio_block : n_text_block;
|
|
}
|
|
|
|
const float * he = llama_get_embeddings_ith(lctx, -1);
|
|
h_state_buf.assign(he, he + n_embd);
|
|
*h_state_out = h_state_buf.data();
|
|
return 0;
|
|
}
|
|
|
|
int32_t get_output(int32_t * out_sample_rate, const char ** out_data, size_t * out_data_len, int64_t * out_n_samples) override {
|
|
*out_sample_rate = info.sample_rate;
|
|
|
|
if (!stream) {
|
|
if (flush() != 0) {
|
|
return 1;
|
|
}
|
|
if (out_n_samples) {
|
|
*out_n_samples = (int64_t) audio_pcm.size();
|
|
}
|
|
if (out_type == MTMD_HELPER_GEN_AUDIO_OUTTYPE_PCM) {
|
|
*out_data = (const char *) audio_pcm.data();
|
|
*out_data_len = audio_pcm.size() * sizeof(float);
|
|
return 0;
|
|
}
|
|
out_buf.clear();
|
|
if (!write_wav16(out_buf, audio_pcm, info.sample_rate)) {
|
|
LOG_ERR("mtmd_helper_gen_audio: output too large for WAV\n");
|
|
return 1;
|
|
}
|
|
*out_data = out_buf.data();
|
|
*out_data_len = out_buf.size();
|
|
return 0;
|
|
}
|
|
|
|
// streaming: only the audio produced since the previous call, every frame is decoded as it is sampled
|
|
const size_t n_new = audio_pcm.size() - pcm_sent;
|
|
if (out_n_samples) {
|
|
*out_n_samples = (int64_t) n_new;
|
|
}
|
|
if (out_type == MTMD_HELPER_GEN_AUDIO_OUTTYPE_PCM) {
|
|
*out_data = (const char *) (audio_pcm.data() + pcm_sent);
|
|
*out_data_len = n_new * sizeof(float);
|
|
pcm_sent = audio_pcm.size();
|
|
return 0;
|
|
}
|
|
out_buf.clear();
|
|
if (!wav_header_sent) {
|
|
write_wav16_header(out_buf, UINT32_MAX, info.sample_rate);
|
|
wav_header_sent = true;
|
|
}
|
|
append_wav16_pcm(out_buf, audio_pcm.data() + pcm_sent, n_new);
|
|
pcm_sent = audio_pcm.size();
|
|
*out_data = out_buf.data();
|
|
*out_data_len = out_buf.size();
|
|
return 0;
|
|
}
|
|
|
|
// the ISTFT holds back the overlap of the last frame
|
|
int32_t flush() override {
|
|
if (pending_tail) {
|
|
if (!decode_wav(nullptr, 0)) {
|
|
return 1;
|
|
}
|
|
pending_tail = false;
|
|
}
|
|
return 0;
|
|
}
|
|
|
|
private:
|
|
// checked by the converter
|
|
static constexpr int n_cb = 12;
|
|
static constexpr int schedule[] = { 4, 4, 4 }; // codebooks revealed per depthformer pass
|
|
|
|
// a random order of the codebooks, revealed by the schedule; fb is the feedback embedding of the codes.
|
|
// at the end of audio every code is the end code, its feedback embedding is only built when continuing
|
|
bool sample_frame(const float * h_state, std::vector<int32_t> & codes, const float ** fb, bool * is_eos) {
|
|
std::vector<int> order(n_cb);
|
|
for (int i = 0; i < n_cb; i++) {
|
|
order[(size_t) i] = i;
|
|
}
|
|
std::shuffle(order.begin(), order.end(), rng);
|
|
|
|
codes.assign(n_cb, -1);
|
|
mtmd_gen_inp inp = mtmd_gen_inp_default(mctx);
|
|
inp.type = MTMD_GEN_PROCESS_TYPE_GEN_CODE;
|
|
inp.embd = const_cast<float *>(h_state);
|
|
inp.codes = codes.data();
|
|
inp.n_codes = codes.size();
|
|
mtmd_gen_out out{};
|
|
|
|
int k = 0;
|
|
for (int n_pick : schedule) {
|
|
if (mtmd_gen_audio_process(mctx, gen_state.get(), &inp, &out) != 0 || out.n_feats == 0 || out.n_feats % n_cb != 0) {
|
|
LOG_ERR("mtmd_helper_gen_audio: depthformer pass failed\n");
|
|
return false;
|
|
}
|
|
const size_t n_vocab = out.n_feats / n_cb;
|
|
logits.assign(out.feats, out.feats + out.n_feats);
|
|
for (int j = 0; j < n_pick; j++, k++) {
|
|
const int i = order[(size_t) k];
|
|
float * l = logits.data() + (size_t) i * n_vocab;
|
|
apply_rep_penalty(l, n_vocab, i);
|
|
codes[(size_t) i] = sample(l, n_vocab);
|
|
}
|
|
}
|
|
|
|
// the last code of every codebook ends the audio
|
|
const int32_t eos = (int32_t) (logits.size() / n_cb) - 1;
|
|
*is_eos = std::find(codes.begin(), codes.end(), eos) != codes.end();
|
|
if (*is_eos) {
|
|
codes.assign(n_cb, eos);
|
|
if (!continuing) {
|
|
*fb = nullptr;
|
|
return true;
|
|
}
|
|
}
|
|
|
|
// all codes known: only the feedback embedding
|
|
if (mtmd_gen_audio_process(mctx, gen_state.get(), &inp, &out) != 0) {
|
|
LOG_ERR("mtmd_helper_gen_audio: codebook embedding failed\n");
|
|
return false;
|
|
}
|
|
fb_buf.assign(out.embd, out.embd + n_embd);
|
|
*fb = fb_buf.data();
|
|
return true;
|
|
}
|
|
|
|
// CTRL penalty over the codes codebook cb took in the last rep_window frames
|
|
void apply_rep_penalty(float * l, size_t n_vocab, int cb) const {
|
|
if (rep_penalty <= 1.0f || rep_window <= 0) {
|
|
return;
|
|
}
|
|
const size_t n_hist = hist.size() / n_cb;
|
|
const size_t n_win = std::min(n_hist, (size_t) rep_window);
|
|
std::vector<bool> seen(n_vocab, false);
|
|
for (size_t f = n_hist - n_win; f < n_hist; f++) {
|
|
const int32_t c = hist[f * n_cb + (size_t) cb];
|
|
if (c >= 0 && (size_t) c < n_vocab && !seen[(size_t) c]) {
|
|
seen[(size_t) c] = true;
|
|
l[c] = l[c] > 0.0f ? l[c] / rep_penalty : l[c] * rep_penalty;
|
|
}
|
|
}
|
|
}
|
|
|
|
// TODO: reuse the samplers of common (top-k, temperature, penalties) instead of this one
|
|
int32_t sample(const float * l, size_t n_vocab) {
|
|
if (temp <= 0.0f) {
|
|
return (int32_t) (std::max_element(l, l + n_vocab) - l);
|
|
}
|
|
// top-k keeps the codes scoring at least the k-th best, ties included
|
|
float lmin = -INFINITY;
|
|
if (top_k > 0 && (size_t) top_k < n_vocab) {
|
|
std::vector<float> sorted(l, l + n_vocab);
|
|
std::nth_element(sorted.begin(), sorted.begin() + (top_k - 1), sorted.end(), std::greater<float>());
|
|
lmin = sorted[(size_t) top_k - 1];
|
|
}
|
|
const float lmax = *std::max_element(l, l + n_vocab);
|
|
std::vector<double> prob(n_vocab);
|
|
double sum = 0.0;
|
|
for (size_t v = 0; v < n_vocab; v++) {
|
|
prob[v] = l[v] < lmin ? 0.0 : std::exp((double) (l[v] - lmax) / temp);
|
|
sum += prob[v];
|
|
}
|
|
double r = std::uniform_real_distribution<double>(0.0, 1.0)(rng) * sum;
|
|
for (size_t v = 0; v < n_vocab; v++) {
|
|
r -= prob[v];
|
|
if (r <= 0.0) {
|
|
return (int32_t) v;
|
|
}
|
|
}
|
|
return (int32_t) n_vocab - 1;
|
|
}
|
|
|
|
// n_codes == 0 flushes the ISTFT tail
|
|
bool decode_wav(const int32_t * codes, size_t n_codes) {
|
|
mtmd_gen_inp inp = mtmd_gen_inp_default(mctx);
|
|
inp.type = MTMD_GEN_PROCESS_TYPE_GEN_WAV;
|
|
inp.codes = const_cast<int32_t *>(codes);
|
|
inp.n_codes = n_codes;
|
|
mtmd_gen_out out{};
|
|
if (mtmd_gen_audio_process(mctx, gen_state.get(), &inp, &out) != 0) {
|
|
LOG_ERR("mtmd_helper_gen_audio: codec decode failed\n");
|
|
return false;
|
|
}
|
|
audio_pcm.insert(audio_pcm.end(), out.audio, out.audio + out.n_samples);
|
|
pending_tail = n_codes > 0;
|
|
return true;
|
|
}
|
|
|
|
bool find_tokens() {
|
|
tok_audio_start = find_special_token(vocab, "<|audio_start|>");
|
|
tok_mixed_start = find_special_token(vocab, "<|mixed_start|>");
|
|
tok_text_end = find_special_token(vocab, "<|text_end|>");
|
|
tok_im_end = find_special_token(vocab, "<|im_end|>");
|
|
if (tok_audio_start == LLAMA_TOKEN_NULL || tok_mixed_start == LLAMA_TOKEN_NULL ||
|
|
tok_text_end == LLAMA_TOKEN_NULL || tok_im_end == LLAMA_TOKEN_NULL) {
|
|
LOG_ERR("mtmd_helper_gen_audio: missing LFM3-Audio control tokens in vocab\n");
|
|
return false;
|
|
}
|
|
return true;
|
|
}
|
|
|
|
bool decode(llama_batch batch) {
|
|
std::vector<llama_pos> positions(batch.n_tokens);
|
|
std::vector<int32_t> n_seq(batch.n_tokens, 1);
|
|
std::vector<llama_seq_id> seq(batch.n_tokens, seq_id);
|
|
std::vector<llama_seq_id *> seq_ptr(batch.n_tokens);
|
|
std::vector<int8_t> logits(batch.n_tokens, 1);
|
|
for (int i = 0; i < batch.n_tokens; i++) {
|
|
positions[i] = pos + i;
|
|
seq_ptr[i] = &seq[i];
|
|
}
|
|
batch.pos = positions.data();
|
|
batch.n_seq_id = n_seq.data();
|
|
batch.seq_id = seq_ptr.data();
|
|
batch.logits = logits.data();
|
|
if (llama_decode(lctx, batch) != 0) {
|
|
LOG_ERR("mtmd_helper_gen_audio: decode failed\n");
|
|
return false;
|
|
}
|
|
pos += batch.n_tokens;
|
|
return true;
|
|
}
|
|
|
|
// generate() modes: text, audio until end of audio, interleaved until <|text_end|>
|
|
enum gen_mode { MODE_TEXT, MODE_AUDIO, MODE_MIXED };
|
|
// checked by the converter
|
|
static constexpr int n_text_block = 6; // interleaved_n_text
|
|
static constexpr int n_audio_block = 12; // interleaved_n_audio
|
|
|
|
llama_token tok_audio_start = LLAMA_TOKEN_NULL;
|
|
llama_token tok_mixed_start = LLAMA_TOKEN_NULL;
|
|
llama_token tok_text_end = LLAMA_TOKEN_NULL;
|
|
llama_token tok_im_end = LLAMA_TOKEN_NULL;
|
|
|
|
bool continuing = false; // the prompt continues a sequence, the response follows the model's switching
|
|
gen_mode mode = MODE_AUDIO;
|
|
int slots_left = 0;
|
|
bool speaking = false; // interleaving: the current block is audio
|
|
bool text_done = false; // interleaving: <|text_end|> was emitted, audio until the end-of-audio frame
|
|
|
|
llama_seq_id seq_id = 0;
|
|
int pos = 0;
|
|
std::vector<llama_token> prompt_tokens;
|
|
int prompt_pos = 0;
|
|
|
|
float temp = 0.0f;
|
|
int32_t top_k = 0;
|
|
float rep_penalty = 1.0f;
|
|
int32_t rep_window = 0; // frames
|
|
std::mt19937 rng;
|
|
std::vector<float> logits;
|
|
std::vector<float> fb_buf;
|
|
std::vector<int32_t> hist; // frame-major codes of the frames so far
|
|
|
|
bool pending_tail = false; // the ISTFT holds back the overlap of the last frame
|
|
std::vector<float> audio_pcm;
|
|
std::vector<float> h_state_buf;
|
|
mtmd_helper_gen_audio_outtype out_type = MTMD_HELPER_GEN_AUDIO_OUTTYPE_WAV;
|
|
std::vector<char> out_buf;
|
|
bool stream = false;
|
|
size_t pcm_sent = 0; // samples already returned by get_output()
|
|
bool wav_header_sent = false;
|
|
};
|
|
|
|
static std::unique_ptr<mtmd_gen_audio_pipeline> make_pipeline(llama_context * lctx, mtmd_context * mctx) {
|
|
switch (mtmd_gen_audio_get_info(mctx).type) {
|
|
case MTMD_GEN_AUDIO_TYPE_QWEN3TTS:
|
|
return std::unique_ptr<mtmd_gen_audio_pipeline>(new qwen3tts_gen_audio_pipeline(lctx, mctx));
|
|
case MTMD_GEN_AUDIO_TYPE_POCKETTTS:
|
|
return std::unique_ptr<mtmd_gen_audio_pipeline>(new pockettts_gen_audio_pipeline(lctx, mctx));
|
|
case MTMD_GEN_AUDIO_TYPE_SOPRANO:
|
|
return std::unique_ptr<mtmd_gen_audio_pipeline>(new soprano_gen_audio_pipeline(lctx, mctx));
|
|
case MTMD_GEN_AUDIO_TYPE_LFM3A:
|
|
return std::unique_ptr<mtmd_gen_audio_pipeline>(new lfm3a_gen_audio_pipeline(lctx, mctx));
|
|
default:
|
|
return nullptr;
|
|
}
|
|
}
|
|
|
|
struct mtmd_helper_gen_audio {
|
|
std::unique_ptr<mtmd_gen_audio_pipeline> pipeline;
|
|
};
|
|
|
|
mtmd_helper_gen_audio * mtmd_helper_gen_audio_init(struct llama_context * lctx, struct mtmd_context * mctx) {
|
|
auto * ctx = new mtmd_helper_gen_audio();
|
|
ctx->pipeline = make_pipeline(lctx, mctx);
|
|
return ctx;
|
|
}
|
|
|
|
void mtmd_helper_gen_audio_free(mtmd_helper_gen_audio * ctx) {
|
|
delete ctx;
|
|
}
|
|
|
|
void mtmd_helper_gen_audio_reset(mtmd_helper_gen_audio * ctx) {
|
|
if (ctx && ctx->pipeline) {
|
|
ctx->pipeline->reset();
|
|
}
|
|
}
|
|
|
|
struct mtmd_helper_gen_audio_inp mtmd_helper_gen_audio_inp_default(void) {
|
|
mtmd_helper_gen_audio_inp inp{};
|
|
inp.top_k = 50;
|
|
inp.top_p = 1.0f;
|
|
inp.seed = UINT32_MAX; // random
|
|
inp.out_type = MTMD_HELPER_GEN_AUDIO_OUTTYPE_WAV;
|
|
return inp;
|
|
}
|
|
|
|
int32_t mtmd_helper_gen_audio_set_input(mtmd_helper_gen_audio * ctx, const mtmd_helper_gen_audio_inp * inp) {
|
|
if (!ctx->pipeline) {
|
|
LOG_ERR("mtmd_helper_gen_audio: unsupported or missing gen-audio pipeline\n");
|
|
return 1;
|
|
}
|
|
if (inp->n_past > 0 && !ctx->pipeline->supports_continue()) {
|
|
LOG_ERR("mtmd_helper_gen_audio: the pipeline cannot continue a sequence\n");
|
|
return 1;
|
|
}
|
|
ctx->pipeline->logit_bias.clear();
|
|
return ctx->pipeline->set_input(inp);
|
|
}
|
|
|
|
int32_t mtmd_helper_gen_audio_step_prompt(mtmd_helper_gen_audio * ctx, int32_t n_batch) {
|
|
if (!ctx->pipeline) {
|
|
return -1;
|
|
}
|
|
return ctx->pipeline->step_prompt(n_batch);
|
|
}
|
|
|
|
bool mtmd_helper_gen_audio_next_is_text(mtmd_helper_gen_audio * ctx) {
|
|
return ctx->pipeline && ctx->pipeline->next_is_text();
|
|
}
|
|
|
|
const llama_logit_bias * mtmd_helper_gen_audio_get_logit_bias(mtmd_helper_gen_audio * ctx, size_t * n_bias) {
|
|
*n_bias = ctx->pipeline ? ctx->pipeline->logit_bias.size() : 0;
|
|
return *n_bias ? ctx->pipeline->logit_bias.data() : nullptr;
|
|
}
|
|
|
|
int32_t mtmd_helper_audio_stream_get_logit_bias(const mtmd_context * mctx, const llama_model * model,
|
|
llama_logit_bias * out, int32_t n_max) {
|
|
std::vector<llama_logit_bias> bias;
|
|
if (mtmd_gen_audio_get_info(mctx).type == MTMD_GEN_AUDIO_TYPE_LFM3A) {
|
|
// LFM3-Audio live ASR: only <|text_end|> ends the text of a chunk, the stream adds the final <|mixed_end|>
|
|
const llama_vocab * vocab = llama_model_get_vocab(model);
|
|
for (const char * piece : { "<|mixed_end|>", "<|mixed_start|>", "<|audio_start|>", "<|im_start|>", "<|im_end|>", "<|endoftext|>" }) {
|
|
const llama_token tok = find_special_token(vocab, piece);
|
|
if (tok != LLAMA_TOKEN_NULL) {
|
|
bias.push_back({ tok, -INFINITY });
|
|
}
|
|
}
|
|
}
|
|
for (int32_t i = 0; i < n_max && i < (int32_t) bias.size(); i++) {
|
|
out[i] = bias[i];
|
|
}
|
|
return (int32_t) bias.size();
|
|
}
|
|
|
|
int32_t mtmd_helper_gen_audio_step_gen(mtmd_helper_gen_audio * ctx, llama_token sampled,
|
|
const float * h_state_in, const float ** h_state_out,
|
|
bool * out_stop) {
|
|
if (!ctx->pipeline) {
|
|
return 1;
|
|
}
|
|
bool stop = false;
|
|
const int32_t ret = ctx->pipeline->step_gen(sampled, h_state_in, h_state_out, &stop);
|
|
if (out_stop) {
|
|
*out_stop = stop;
|
|
}
|
|
return ret;
|
|
}
|
|
|
|
int32_t mtmd_helper_gen_audio_get_output(mtmd_helper_gen_audio * ctx, int32_t * out_sample_rate,
|
|
const char ** out_data, size_t * out_data_len, int64_t * out_n_samples) {
|
|
if (!ctx->pipeline) {
|
|
return 1;
|
|
}
|
|
return ctx->pipeline->get_output(out_sample_rate, out_data, out_data_len, out_n_samples);
|
|
}
|
|
|
|
int32_t mtmd_helper_gen_audio_flush(mtmd_helper_gen_audio * ctx) {
|
|
if (!ctx->pipeline) {
|
|
return 1;
|
|
}
|
|
return ctx->pipeline->flush();
|
|
}
|