diff --git a/tools/mtmd/README-dev.md b/tools/mtmd/README-dev.md index 70d5c5da0..5dad86100 100644 --- a/tools/mtmd/README-dev.md +++ b/tools/mtmd/README-dev.md @@ -55,7 +55,7 @@ For example, Qwen3-TTS: Due to wide variety of audio generation pipelines, the `mtmd_gen_audio` system is designed to be flexible and reusable by new models. `mtmd_gen_audio` is split into 2 main API: -- Core API `mtmd.h`: handles main inference. Important: the API surface must be stateless; caller must handle state management and audio frame accumulation. +- Core API `mtmd.h`: handles main inference. Important: `mtmd_gen_state` holds only the per-sequence data that the mmproj itself writes and reads back (RNG, decoder KV / conv state, output buffers). Data that goes to or comes from the backbone (hidden states, codes, PCM) must stay explicit in `mtmd_gen_inp` / `mtmd_gen_out`; the caller drives the backbone and handles audio frame accumulation. - Helper API `mtmd-helper.h`: provides a model-agnostic stateful API. Usage example can be found in the `tools/tts` directory. ### Checklist for porting new audio generation models to mtmd diff --git a/tools/mtmd/clip-impl.h b/tools/mtmd/clip-impl.h index 8c4ffc9bb..71d349cc5 100644 --- a/tools/mtmd/clip-impl.h +++ b/tools/mtmd/clip-impl.h @@ -16,6 +16,7 @@ #include #include #include +#include #ifdef _WIN32 #ifndef NOMINMAX @@ -463,6 +464,15 @@ struct clip_enc_state { std::vector v; }; +// per-sequence state of audio generation, see clip_gen_state_init() +struct clip_gen_state { + std::mt19937 rng; + + ggml_context_ptr ctx_data; + ggml_backend_buffer_ptr buf; + std::vector slots; // same order as list_gen_state_slots(), F32 [ne0, ne1] +}; + // forward declaration // TODO: improve this later struct clip_ctx; diff --git a/tools/mtmd/clip.cpp b/tools/mtmd/clip.cpp index e4750e52e..359ec5ac3 100644 --- a/tools/mtmd/clip.cpp +++ b/tools/mtmd/clip.cpp @@ -174,10 +174,6 @@ struct clip_ctx { bool support_batch = false; - // for audio gen, reseeded only when the caller asks for another seed - std::mt19937 rng{std::random_device{}()}; - uint32_t rng_seed = UINT32_MAX; - clip_ctx(clip_context_params & ctx_params) { flash_attn_type = ctx_params.flash_attn_type; no_alloc = ctx_params.no_alloc; @@ -4717,6 +4713,59 @@ static std::vector list_gen_state_slots(const clip_hparams & hpa } } +// state_out can have another shape than its slot, only the byte size must match +static void copy_gen_state_out(ggml_tensor * src, ggml_tensor * dst) { + GGML_ASSERT(ggml_nbytes(src) == ggml_nbytes(dst)); + if (src->type == dst->type && ggml_are_same_shape(src, dst) && ggml_is_contiguous(src)) { + ggml_backend_tensor_copy(src, dst); + return; + } + std::vector tmp(ggml_nbytes(src)); + ggml_backend_tensor_get(src, tmp.data(), 0, tmp.size()); + ggml_backend_tensor_set(dst, tmp.data(), 0, tmp.size()); +} + +clip_gen_state * clip_gen_state_init(clip_ctx * ctx, uint32_t seed) { + auto st = std::make_unique(); + st->rng.seed(seed == UINT32_MAX ? std::random_device{}() : seed); + + const auto slots = list_gen_state_slots(ctx->model.hparams, ctx->model); + if (slots.empty()) { + return st.release(); + } + + ggml_init_params params = { + /*.mem_size =*/ ggml_tensor_overhead() * slots.size(), + /*.mem_buffer =*/ nullptr, + /*.no_alloc =*/ true, + }; + st->ctx_data.reset(ggml_init(params)); + for (const auto & slot : slots) { + ggml_tensor * t = ggml_new_tensor_2d(st->ctx_data.get(), GGML_TYPE_F32, slot.ne0, slot.ne1); + ggml_format_name(t, "gen_state_%s", slot.name.c_str()); + st->slots.push_back(t); + } + st->buf.reset(ggml_backend_alloc_ctx_tensors(st->ctx_data.get(), ctx->backend)); + if (!st->buf) { + LOG_ERR("%s: failed to allocate the gen state\n", __func__); + return nullptr; + } + // a zero state is the cold start + ggml_backend_buffer_clear(st->buf.get(), 0); + + return st.release(); +} + +void clip_gen_state_free(clip_gen_state * state) { + delete state; +} + +void clip_gen_state_reset(clip_gen_state * state) { + if (state->buf) { + ggml_backend_buffer_clear(state->buf.get(), 0); + } +} + clip_enc_state * clip_enc_state_init(clip_ctx * ctx, int32_t n_frames_max) { const auto & hparams = ctx->model.hparams; if (ctx->proj_type() != PROJECTOR_TYPE_VOXTRAL_RT || hparams.attn_window_size <= 0) { @@ -4782,9 +4831,9 @@ bool clip_encode(struct clip_ctx * ctx, struct clip_encode_params * params) { clip_model_loader::warmup(*ctx, *params->imgs); } - if (params->seed != ctx->rng_seed) { - ctx->rng_seed = params->seed; - ctx->rng.seed(params->seed == UINT32_MAX ? std::random_device{}() : params->seed); + if (params->gen_process != CLIP_GEN_PROCESS_GEN_UNKNOWN && params->gen_state == nullptr) { + LOG_ERR("%s: audio gen requires a gen state\n", __func__); + return false; } // build the inference graph @@ -4848,19 +4897,13 @@ bool clip_encode(struct clip_ctx * ctx, struct clip_encode_params * params) { ggml_backend_tensor_set(cur, values.data(), 0, ggml_nbytes(cur)); }; - // upload the decoder state from the previous call, or zero-fill on a cold start + // state of the previous call, zero on a cold start auto set_gen_state_in = [&]() { - size_t offset = 0; + const auto & slots = params->gen_state->slots; + size_t i = 0; for (const auto & slot : list_gen_state_slots(hparams, model)) { - ggml_tensor * t = get_inp_tensor(("state_in_" + slot.name).c_str()); - const size_t nb = ggml_nbytes(t); - if (params->state_in && params->state_in->size() >= offset + nb) { - ggml_backend_tensor_set(t, params->state_in->data() + offset, 0, nb); - } else { - std::vector zeros(nb, 0); - ggml_backend_tensor_set(t, zeros.data(), 0, nb); - } - offset += nb; + GGML_ASSERT(i < slots.size()); + ggml_backend_tensor_copy(slots[i++], get_inp_tensor(("state_in_" + slot.name).c_str())); } }; @@ -5584,7 +5627,7 @@ bool clip_encode(struct clip_ctx * ctx, struct clip_encode_params * params) { std::normal_distribution dist(0.0f, std::sqrt(temp)); std::vector noise(ggml_nelements(t)); for (auto & v : noise) { - v = dist(ctx->rng); + v = dist(params->gen_state->rng); } set_input_f32("inp_noise", noise); } @@ -5732,7 +5775,7 @@ bool clip_encode(struct clip_ctx * ctx, struct clip_encode_params * params) { std::uniform_real_distribution dist(0.0f, 1.0f); const int64_t n_acoustic = model.gen_code_head_w->ne[2]; for (int64_t g = 0; g < n_acoustic; g++) { - std::vector r = { dist(ctx->rng) }; + std::vector r = { dist(params->gen_state->rng) }; set_input_f32(("inp_rand_" + std::to_string(g)).c_str(), r); } } @@ -6262,22 +6305,16 @@ bool clip_encode(struct clip_ctx * ctx, struct clip_encode_params * params) { st->n_past += n_new; } - if (params->state_out != nullptr) { - auto & state_out = *params->state_out; - size_t total = 0; - for (const auto & slot : list_gen_state_slots(hparams, model)) { - total += (size_t) (slot.ne0 * slot.ne1) * sizeof(float); - } - state_out.resize(total); - size_t offset = 0; + if (params->gen_process == CLIP_GEN_PROCESS_GEN_WAV) { + const auto & slots = params->gen_state->slots; + size_t i = 0; for (const auto & slot : list_gen_state_slots(hparams, model)) { ggml_tensor * t = ggml_graph_get_tensor(gf, ("state_out_" + slot.name).c_str()); if (t == nullptr) { - GGML_ABORT("state_out requested but graph has no \"state_out_%s\" tensor", slot.name.c_str()); + GGML_ABORT("graph has no \"state_out_%s\" tensor", slot.name.c_str()); } - const size_t nb = ggml_nbytes(t); - ggml_backend_tensor_get(t, state_out.data() + offset, 0, nb); - offset += nb; + GGML_ASSERT(i < slots.size()); + copy_gen_state_out(t, slots[i++]); } } diff --git a/tools/mtmd/clip.h b/tools/mtmd/clip.h index cdbaeae0d..cd3f434da 100644 --- a/tools/mtmd/clip.h +++ b/tools/mtmd/clip.h @@ -14,6 +14,7 @@ struct clip_ctx; struct clip_enc_state; +struct clip_gen_state; struct clip_image_size { int width; @@ -110,7 +111,6 @@ struct clip_encode_params { float top_p = 1.0f; std::vector * out_codes = nullptr; // this frame's 16 sampled codes std::vector * out_feats = nullptr; // continuous counterpart of out_codes - uint32_t seed = UINT32_MAX; // UINT32_MAX for random float temp = 0.0f; // sampling temperature, noise scale for flow-matching decoders bool * out_is_eos = nullptr; @@ -118,8 +118,9 @@ struct clip_encode_params { const std::vector * codes = nullptr; // this frame's 16 RVQ codes const std::vector * feats = nullptr; // continuous counterpart of codes std::vector * out_audio = nullptr; // decoded PCM samples, F32 - const std::vector * state_in = nullptr; // state from previous call, null or wrong size means cold start - std::vector * state_out = nullptr; // state for the next call + + // audio gen: state of one sequence, clip_encode() updates it in place + struct clip_gen_state * gen_state = nullptr; // streaming audio input, updated in place by clip_encode() struct clip_enc_state * enc_state = nullptr; @@ -132,6 +133,12 @@ bool clip_encode(struct clip_ctx * ctx, struct clip_encode_params * params); struct clip_enc_state * clip_enc_state_init(struct clip_ctx * ctx, int32_t n_frames_max); void clip_enc_state_free(struct clip_enc_state * state); +// per-sequence state of audio generation: RNG and the slots kept between GEN_WAV calls +// seed UINT32_MAX means random; reset() clears the slots and keeps the RNG stream +struct clip_gen_state * clip_gen_state_init(struct clip_ctx * ctx, uint32_t seed); +void clip_gen_state_free(struct clip_gen_state * state); +void clip_gen_state_reset(struct clip_gen_state * state); + bool clip_is_llava(const struct clip_ctx * ctx); // note for contributor: this clip_is_(model) pattern is deprecated // do NOT add new functions like this diff --git a/tools/mtmd/mtmd-helper-gen.cpp b/tools/mtmd/mtmd-helper-gen.cpp index 47493d906..d8299ba44 100644 --- a/tools/mtmd/mtmd-helper-gen.cpp +++ b/tools/mtmd/mtmd-helper-gen.cpp @@ -105,12 +105,25 @@ public: virtual int32_t flush() = 0; protected: + // one state per generation, so a fixed seed always gives the same output + bool init_gen_state(uint32_t seed) { + mtmd_gen_state_params params = mtmd_gen_state_params_default(); + params.seed = seed; + gen_state.reset(mtmd_gen_state_init(mctx, params)); + if (!gen_state) { + LOG_ERR("mtmd_helper_gen_audio: failed to create the gen state\n"); + return false; + } + return true; + } + llama_context * lctx; mtmd_context * mctx; const llama_model * model; const llama_vocab * vocab; int n_embd; mtmd_gen_audio_info info; + mtmd::gen_state_ptr gen_state; }; // Qwen3-TTS: backbone samples codec_0, code_predictor gives the other 15 codebooks, @@ -123,7 +136,7 @@ public: seq_id = 0; pos = 0; codes_buf.clear(); - c2w_state.clear(); + gen_state.reset(); audio_pcm.clear(); overlay.clear(); h_state_buf.clear(); @@ -220,9 +233,11 @@ public: const mtmd_gen_inp def = mtmd_gen_inp_default(mctx); top_k = inp->top_k > 0 ? inp->top_k : def.top_k; top_p = inp->top_p > 0 ? inp->top_p : def.top_p; - seed = inp->seed; out_type = inp->out_type; stream = inp->stream; + if (!init_gen_state(inp->seed)) { + return 1; + } // the prompt above holds the whole text stream up to tts_eos, so every generated // frame adds tts_pad on top of the codes embedding @@ -279,9 +294,8 @@ public: inp.embd = const_cast(h_state_in); inp.top_k = top_k; inp.top_p = top_p; - inp.seed = seed; mtmd_gen_out out{}; - if (mtmd_gen_audio_process(mctx, &inp, &out) != 0) { + if (mtmd_gen_audio_process(mctx, gen_state.get(), &inp, &out) != 0) { LOG_ERR("mtmd_helper_gen_audio: gen_code process failed\n"); return 1; } @@ -449,16 +463,12 @@ private: inp.type = MTMD_GEN_PROCESS_TYPE_GEN_WAV; inp.codes = codes_buf.data(); inp.n_codes = codes_buf.size(); - inp.seed = seed; // same seed as gen_code, else clip reseeds mid-generation - inp.state_data = c2w_state.empty() ? nullptr : (const char *) c2w_state.data(); - inp.state_size = c2w_state.size(); mtmd_gen_out out{}; - if (mtmd_gen_audio_process(mctx, &inp, &out) != 0) { + if (mtmd_gen_audio_process(mctx, gen_state.get(), &inp, &out) != 0) { LOG_ERR("mtmd_helper_gen_audio: gen_wav process failed\n"); return false; } audio_pcm.insert(audio_pcm.end(), out.audio, out.audio + out.n_samples); - c2w_state.assign(out.state_data, out.state_data + out.state_size); codes_buf.clear(); return true; } @@ -491,9 +501,7 @@ private: int prompt_pos = 0; int32_t top_k = 50; float top_p = 1.0f; - uint32_t seed = UINT32_MAX; std::vector codes_buf; - std::vector c2w_state; std::vector audio_pcm; std::vector overlay; std::vector h_state_buf; @@ -539,7 +547,7 @@ public: seq_id = 0; pos = 0; feats_buf.clear(); - dec_state.clear(); + gen_state.reset(); audio_pcm.clear(); h_state_buf.clear(); out_buf.clear(); @@ -623,9 +631,11 @@ public: prompt_batch->set_position_normal(0, seq_id); prompt_pos = 0; - seed = inp->seed; out_type = inp->out_type; stream = inp->stream; + if (!init_gen_state(inp->seed)) { + return 1; + } return 0; } @@ -663,13 +673,11 @@ public: mtmd_gen_inp inp = mtmd_gen_inp_default(mctx); inp.type = MTMD_GEN_PROCESS_TYPE_GEN_CODE; inp.embd = const_cast(h_state_in); - // clip only reseeds when the seed changes, so pass the same one on every step - inp.seed = seed; if (pack.temp > 0.0f) { inp.temp = pack.temp; } mtmd_gen_out out{}; - if (mtmd_gen_audio_process(mctx, &inp, &out) != 0) { + if (mtmd_gen_audio_process(mctx, gen_state.get(), &inp, &out) != 0) { LOG_ERR("mtmd_helper_gen_audio: flow decode failed\n"); return 1; } @@ -895,7 +903,7 @@ private: return 1; } // the decoder restarts too, the next chunk's audio is not continuous with this one - dec_state.clear(); + mtmd_gen_state_reset(gen_state.get()); if (chunk_idx + 1 >= chunks.size()) { *out_stop = true; @@ -1023,16 +1031,12 @@ private: inp.type = MTMD_GEN_PROCESS_TYPE_GEN_WAV; inp.feats = feats_buf.data(); inp.n_feats = feats_buf.size(); - inp.seed = seed; - inp.state_data = dec_state.empty() ? nullptr : (const char *) dec_state.data(); - inp.state_size = dec_state.size(); mtmd_gen_out out{}; - if (mtmd_gen_audio_process(mctx, &inp, &out) != 0) { + if (mtmd_gen_audio_process(mctx, gen_state.get(), &inp, &out) != 0) { LOG_ERR("mtmd_helper_gen_audio: mimi decode failed\n"); return false; } audio_pcm.insert(audio_pcm.end(), out.audio, out.audio + out.n_samples); - dec_state.assign(out.state_data, out.state_data + out.state_size); feats_buf.clear(); return true; } @@ -1049,7 +1053,6 @@ private: std::unique_ptr prompt_batch; int n_prompt = 0; int prompt_pos = 0; - uint32_t seed = UINT32_MAX; // end-of-speech is latched, then a few more frames are generated as tail padding int step_idx = 0; int eos_step = -1; @@ -1064,7 +1067,6 @@ private: // latents are decoded a window at a time, the decoder state bridges the windows size_t window_frames = 8; std::vector feats_buf; - std::vector dec_state; std::vector audio_pcm; std::vector h_state_buf; mtmd_helper_gen_audio_outtype out_type = MTMD_HELPER_GEN_AUDIO_OUTTYPE_WAV; diff --git a/tools/mtmd/mtmd.cpp b/tools/mtmd/mtmd.cpp index ffd35b2bb..9e2c5f3c9 100644 --- a/tools/mtmd/mtmd.cpp +++ b/tools/mtmd/mtmd.cpp @@ -523,11 +523,6 @@ struct mtmd_context { // generation context struct clip_ctx * ctx_gen_a; // audio - std::vector gen_out_codes; // this frame's 16 sampled codes (GEN_CODE) - std::vector gen_out_feats; // this frame's continuous features, if any (GEN_CODE) - std::vector gen_out_embd; // next-step hidden state fed back to backbone (GEN_CODE) - std::vector gen_out_audio; // decoded PCM samples for the current frame (GEN_WAV) - std::vector gen_out_state; // state to feed into the next GEN_WAV call bool print_timings; int n_threads; @@ -2080,7 +2075,6 @@ mtmd_gen_audio_info mtmd_gen_audio_get_info(const mtmd_context * ctx) { mtmd_gen_inp mtmd_gen_inp_default(const mtmd_context * ctx) { mtmd_gen_inp inp{}; inp.type = MTMD_GEN_PROCESS_TYPE_GEN_CODE; - inp.seed = UINT32_MAX; if (!ctx->ctx_gen_a) { return inp; } @@ -2104,12 +2098,58 @@ mtmd_gen_inp mtmd_gen_inp_default(const mtmd_context * ctx) { return inp; } -static int32_t mtmd_gen_audio_process_impl(mtmd_context * ctx, const mtmd_gen_inp * inp, mtmd_gen_out * out) { +struct mtmd_gen_state { + clip_ctx * ctx_clip = nullptr; + clip_gen_state * clip_state = nullptr; + + std::vector out_codes; // this frame's 16 sampled codes (GEN_CODE) + std::vector out_feats; // this frame's continuous features, if any (GEN_CODE) + std::vector out_embd; // next-step hidden state fed back to backbone (GEN_CODE) + std::vector out_audio; // decoded PCM samples for the current frame (GEN_WAV) + + ~mtmd_gen_state() { + clip_gen_state_free(clip_state); + } +}; + +mtmd_gen_state_params mtmd_gen_state_params_default(void) { + mtmd_gen_state_params params{}; + params.seed = UINT32_MAX; + return params; +} + +mtmd_gen_state * mtmd_gen_state_init(mtmd_context * ctx, mtmd_gen_state_params params) { + if (!ctx->ctx_gen_a) { + LOG_ERR("%s: model does not support audio generation\n", __func__); + return nullptr; + } + auto st = std::make_unique(); + st->ctx_clip = ctx->ctx_gen_a; + st->clip_state = clip_gen_state_init(ctx->ctx_gen_a, params.seed); + if (!st->clip_state) { + return nullptr; + } + return st.release(); +} + +void mtmd_gen_state_free(mtmd_gen_state * state) { + delete state; +} + +void mtmd_gen_state_reset(mtmd_gen_state * state) { + clip_gen_state_reset(state->clip_state); +} + +static int32_t mtmd_gen_audio_process_impl(mtmd_context * ctx, mtmd_gen_state * state, const mtmd_gen_inp * inp, mtmd_gen_out * out) { clip_ctx * ctx_clip = ctx->ctx_gen_a; if (!ctx_clip) { LOG_ERR("%s: model does not support audio generation\n", __func__); return 1; } + if (!state || state->ctx_clip != ctx_clip) { + LOG_ERR("%s: invalid gen state\n", __func__); + return 1; + } *out = {}; @@ -2139,24 +2179,24 @@ static int32_t mtmd_gen_audio_process_impl(mtmd_context * ctx, const mtmd_gen_in params.code0 = inp->code0; params.top_k = inp->top_k; params.top_p = inp->top_p; - params.seed = inp->seed; params.temp = inp->temp; params.out_is_eos = &is_eos; + params.gen_state = state->clip_state; if (!clip_encode(ctx_clip, ¶ms)) { LOG_ERR("%s: clip_encode failed (gen_code)\n", __func__); return 1; } - ctx->gen_out_embd = std::move(out_embd); - ctx->gen_out_codes = std::move(out_codes); - ctx->gen_out_feats = std::move(out_feats); + state->out_embd = std::move(out_embd); + state->out_codes = std::move(out_codes); + state->out_feats = std::move(out_feats); - out->embd = ctx->gen_out_embd.data(); - out->codes = ctx->gen_out_codes.data(); - out->n_codes = ctx->gen_out_codes.size(); - out->feats = ctx->gen_out_feats.data(); - out->n_feats = ctx->gen_out_feats.size(); + out->embd = state->out_embd.data(); + out->codes = state->out_codes.data(); + out->n_codes = state->out_codes.size(); + out->feats = state->out_feats.data(); + out->n_feats = state->out_feats.size(); out->is_eos = is_eos; return 0; } @@ -2175,10 +2215,6 @@ static int32_t mtmd_gen_audio_process_impl(mtmd_context * ctx, const mtmd_gen_in } else { in_feats.assign(inp->feats, inp->feats + inp->n_feats); } - std::vector in_state; - if (inp->state_data) { - in_state.assign(inp->state_data, inp->state_data + inp->state_size); - } // gen_wav has no hidden-state input, the batch entry is an unused placeholder // TODO @ngxson : some models in the future may require hidden-state input, need to update this code later @@ -2194,30 +2230,25 @@ static int32_t mtmd_gen_audio_process_impl(mtmd_context * ctx, const mtmd_gen_in params.imgs = &batch; params.n_threads = ctx->n_threads; params.gen_process = CLIP_GEN_PROCESS_GEN_WAV; - // gen_wav draws no randomness, but keep the seed so it does not reseed mid-generation - params.seed = inp->seed; params.codes = has_codes ? &in_codes : nullptr; params.feats = has_feats ? &in_feats : nullptr; - params.out_audio = &ctx->gen_out_audio; - params.state_in = inp->state_data ? &in_state : nullptr; - params.state_out = &ctx->gen_out_state; + params.out_audio = &state->out_audio; + params.gen_state = state->clip_state; if (!clip_encode(ctx_clip, ¶ms)) { LOG_ERR("%s: clip_encode failed (code2wav)\n", __func__); return 1; } - out->audio = ctx->gen_out_audio.data(); - out->n_samples = ctx->gen_out_audio.size(); - out->state_data = (const char *) ctx->gen_out_state.data(); - out->state_size = ctx->gen_out_state.size(); + out->audio = state->out_audio.data(); + out->n_samples = state->out_audio.size(); return 0; } -int32_t mtmd_gen_audio_process(mtmd_context * ctx, const struct mtmd_gen_inp * inp, struct mtmd_gen_out * out) { +int32_t mtmd_gen_audio_process(mtmd_context * ctx, mtmd_gen_state * state, const struct mtmd_gen_inp * inp, struct mtmd_gen_out * out) { try { - return mtmd_gen_audio_process_impl(ctx, inp, out); + return mtmd_gen_audio_process_impl(ctx, state, inp, out); } catch (const std::exception & e) { LOG_ERR("%s: error: %s\n", __func__, e.what()); return 1; diff --git a/tools/mtmd/mtmd.h b/tools/mtmd/mtmd.h index c740f1e9f..f248ed739 100644 --- a/tools/mtmd/mtmd.h +++ b/tools/mtmd/mtmd.h @@ -65,6 +65,7 @@ struct mtmd_image_tokens; struct mtmd_input_chunk; struct mtmd_input_chunks; struct mtmd_batch; +struct mtmd_gen_state; struct mtmd_input_text { const char * text; @@ -79,6 +80,7 @@ struct mtmd_input_part { const struct mtmd_bitmap * bitmap; }; + // // C API // @@ -91,6 +93,7 @@ typedef struct mtmd_input_chunks mtmd_input_chunks; typedef struct mtmd_input_text mtmd_input_text; typedef struct mtmd_input_part mtmd_input_part; typedef struct mtmd_batch mtmd_batch; +typedef struct mtmd_gen_state mtmd_gen_state; typedef bool (*mtmd_progress_callback)(float progress, void * user_data); @@ -443,8 +446,7 @@ struct mtmd_gen_inp { float * embd; // the hidden state from backbone, must have n_text_embd elements int32_t top_k; float top_p; - uint32_t seed; // UINT32_MAX for random - float temp; // sampling temperature, or noise scale for flow-matching decoders + float temp; // sampling temperature, or noise scale for flow-matching decoders // for MTMD_GEN_PROCESS_TYPE_GEN_WAV // pass either codes (discrete) or feats (continuous), depending on the pipeline @@ -452,12 +454,10 @@ struct mtmd_gen_inp { size_t n_codes; const float * feats; size_t n_feats; - const char * state_data; - size_t state_size; }; struct mtmd_gen_out { - // note: output memory is allocated by the context, valid until next process() call + // note: output memory is owned by the state, valid until the next process() call with this state // for MTMD_GEN_PROCESS_TYPE_GEN_CODE const int32_t * codes; @@ -471,15 +471,28 @@ struct mtmd_gen_out { // for MTMD_GEN_PROCESS_TYPE_GEN_WAV const float * audio; size_t n_samples; - const char * state_data; - size_t state_size; }; +struct mtmd_gen_state_params { + uint32_t seed; // UINT32_MAX for random +}; + +MTMD_API struct mtmd_gen_state_params mtmd_gen_state_params_default(void); + +// returns nullptr if the model does not support audio generation +MTMD_API mtmd_gen_state * mtmd_gen_state_init(mtmd_context * ctx, struct mtmd_gen_state_params params); +MTMD_API void mtmd_gen_state_free(mtmd_gen_state * state); + +// clear the decoder state (cold start), keep the RNG stream +MTMD_API void mtmd_gen_state_reset(mtmd_gen_state * state); + // defaults tuned for the loaded pipeline, callers override only what they care about MTMD_API struct mtmd_gen_inp mtmd_gen_inp_default(const mtmd_context * ctx); -// note: this API is stateless, caller must handle state management and audio frame accumulation +// run one step and update the state in place +// the caller drives the backbone and collects the audio frames MTMD_API int32_t mtmd_gen_audio_process(mtmd_context * ctx, + mtmd_gen_state * state, const struct mtmd_gen_inp * inp, struct mtmd_gen_out * out); @@ -544,6 +557,11 @@ struct mtmd_audio_stream_deleter { }; using audio_stream_ptr = std::unique_ptr; +struct mtmd_gen_state_deleter { + void operator()(mtmd_gen_state * val) { mtmd_gen_state_free(val); } +}; +using gen_state_ptr = std::unique_ptr; + struct bitmap { bitmap_ptr ptr; bitmap() : ptr(nullptr) {}