#include "models.h" #include // pocket-tts generation stages // // GEN_CODE: backbone hidden state -> next 32-d latent (flow matching) + end-of-speech score // GEN_WAV : a window of latents -> PCM, through the mimi decoder // // there is no codebook anywhere, "codes" in the mtmd API are continuous features here ggml_tensor * mtmd_graph_pockettts_gen::modulate(ggml_tensor * x, ggml_tensor * shift, ggml_tensor * scale) const { ggml_tensor * cur = ggml_mul(ctx0, x, ggml_scale_bias(ctx0, scale, 1.0f, 1.0f)); return ggml_add(ctx0, cur, shift); } // see TimestepEmbedder in the reference ggml_tensor * mtmd_graph_pockettts_gen::time_embed(const mtmd_flow_net::time_embd & te, float t) const { // t is a graph-build constant, so the cos/sin table can be folded into a scaled copy ggml_tensor * args = ggml_scale(ctx0, te.freqs, t); ggml_tensor * emb = ggml_concat(ctx0, ggml_cos(ctx0, args), ggml_sin(ctx0, args), 0); ggml_tensor * cur = build_mm(te.up_w, emb); cur = ggml_add(ctx0, cur, te.up_b); cur = ggml_silu(ctx0, cur); cur = build_mm(te.down_w, cur); cur = ggml_add(ctx0, cur, te.down_b); // this "RMSNorm" divides by the unbiased variance, not the mean square // it also rescales the input, not the centered value, see _rms_norm() in mlp.py { const int64_t n = cur->ne[0]; ggml_tensor * mean = ggml_mean(ctx0, cur); ggml_tensor * dev = ggml_sub(ctx0, cur, mean); ggml_tensor * var = ggml_mean(ctx0, ggml_sqr(ctx0, dev)); var = ggml_scale_bias(ctx0, var, (float) n / (float) (n - 1), 1e-5f); cur = ggml_div(ctx0, cur, ggml_sqrt(ctx0, var)); cur = ggml_mul(ctx0, cur, te.norm); } return cur; } // one velocity evaluation: v(cond, s, t, x) ggml_tensor * mtmd_graph_pockettts_gen::flow_forward(ggml_tensor * cond, ggml_tensor * x, float s, float t) const { const auto & flow = model.flow; ggml_tensor * cur = build_mm(flow.input_proj_w, x); cur = ggml_add(ctx0, cur, flow.input_proj_b); // the two time conditions are averaged, then added to the projected backbone state ggml_tensor * ts = ggml_add(ctx0, time_embed(flow.time[0], s), time_embed(flow.time[1], t)); ts = ggml_scale(ctx0, ts, 1.0f / (float) flow.time.size()); ggml_tensor * c = build_mm(flow.cond_embd_w, cond); c = ggml_add(ctx0, c, flow.cond_embd_b); ggml_tensor * y = ggml_add(ctx0, ts, c); cb(y, "flow_cond", -1); const int64_t n_ch = flow.blocks.empty() ? 0 : flow.blocks[0].norm_w->ne[0]; for (size_t il = 0; il < flow.blocks.size(); il++) { const auto & blk = flow.blocks[il]; ggml_tensor * mod = build_mm(blk.ada_w, ggml_silu(ctx0, y)); mod = ggml_add(ctx0, mod, blk.ada_b); ggml_tensor * shift = ggml_view_1d(ctx0, mod, n_ch, 0); ggml_tensor * scale = ggml_view_1d(ctx0, mod, n_ch, (size_t) n_ch * mod->nb[0]); ggml_tensor * gate = ggml_view_1d(ctx0, mod, n_ch, (size_t) 2 * n_ch * mod->nb[0]); ggml_tensor * h = build_norm(cur, blk.norm_w, blk.norm_b, NORM_TYPE_NORMAL, 1e-6f, (int) il); h = modulate(h, shift, scale); h = build_mm(blk.up_w, h); h = ggml_add(ctx0, h, blk.up_b); h = ggml_silu(ctx0, h); h = build_mm(blk.down_w, h); h = ggml_add(ctx0, h, blk.down_b); cur = ggml_add(ctx0, cur, ggml_mul(ctx0, gate, h)); cb(cur, "flow_blk", (int) il); } // final layer: the norm has no weights, only the AdaLN modulation ggml_tensor * mod = build_mm(flow.final_ada_w, ggml_silu(ctx0, y)); mod = ggml_add(ctx0, mod, flow.final_ada_b); ggml_tensor * shift = ggml_view_1d(ctx0, mod, n_ch, 0); ggml_tensor * scale = ggml_view_1d(ctx0, mod, n_ch, (size_t) n_ch * mod->nb[0]); cur = build_norm(cur, nullptr, nullptr, NORM_TYPE_NORMAL, 1e-6f, -1); cur = modulate(cur, shift, scale); cur = build_mm(flow.final_proj_w, cur); cur = ggml_add(ctx0, cur, flow.final_proj_b); return cur; } // state carried between GEN_WAV calls: rope offset, per-layer KV window, conv left context // and the transposed-conv overlap tails std::vector list_pockettts_state_slots(const mtmd_hparams & hparams, const mtmd_model & model) { std::vector slots; if (model.gen_upsample_w == nullptr) { return slots; // not a pocket-tts decoder } const auto & seanet = model.seanet; // the slots below are sized from these GGML_ASSERT(!model.gen_tfm_layers.empty()); GGML_ASSERT((int) seanet.stages.size() >= hparams.seanet_n_stage); GGML_ASSERT((int) hparams.seanet_ratios.size() >= hparams.seanet_n_stage); GGML_ASSERT(hparams.mimi_tfm_context > 1 && hparams.mimi_downsample > 0); slots.push_back({"pos", 1, 1}); const int64_t n_embd_a = model.gen_tfm_layers[0].q_w->ne[1]; const int64_t prefix = hparams.mimi_tfm_context - 1; for (size_t il = 0; il < model.gen_tfm_layers.size(); il++) { slots.push_back({"tfm_k_" + std::to_string(il), n_embd_a, prefix}); slots.push_back({"tfm_v_" + std::to_string(il), n_embd_a, prefix}); } // upsample is depthwise, its output channel count is the input one slots.push_back({"up", model.gen_upsample_w->ne[0] - hparams.mimi_downsample, model.gen_upsample_w->ne[2]}); slots.push_back({"dec_in", seanet.conv_in_w->ne[0] - 1, seanet.conv_in_w->ne[1]}); for (int i = 0; i < hparams.seanet_n_stage; i++) { const auto & stage = seanet.stages[i]; const int stride = hparams.seanet_ratios[hparams.seanet_n_stage - 1 - i]; slots.push_back({"dec_up_" + std::to_string(i), stage.scale_conv_w->ne[0] - stride, stage.scale_conv_w->ne[1]}); slots.push_back({"dec_res_" + std::to_string(i), stage.res_conv1_w->ne[0] - 1, stage.res_conv1_w->ne[1]}); } slots.push_back({"dec_out", seanet.conv_out_w->ne[0] - 1, seanet.conv_out_w->ne[1]}); return slots; } ggml_cgraph * mtmd_graph_pockettts_gen::build() { if (gen_process == MTMD_SUBCTX_GEN_PROCESS_GEN_CODE) { // the backbone hidden state arrives as the single batch entry ggml_tensor * h_state = build_inp_raw(1); h_state = ggml_reshape_2d(ctx0, h_state, n_mmproj_embd, 1); // end-of-speech probe, thresholded on the host side ggml_tensor * eos = build_mm(model.gen_out_eos_w, h_state); eos = ggml_add(ctx0, eos, model.gen_out_eos_b); ggml_set_name(eos, "out_eos_score"); ggml_set_output(eos); ggml_build_forward_expand(gf, eos); const int64_t n_latent = model.gen_input_lin_w->ne[0]; ggml_tensor * noise = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_latent, 1); ggml_set_name(noise, "inp_noise"); ggml_set_input(noise); // lsd_decode: integrate the velocity field from the noise sample ggml_tensor * cur = noise; for (int i = 0; i < n_step; i++) { const float s = (float) i / (float) n_step; const float t = (float) (i + 1) / (float) n_step; ggml_tensor * v = flow_forward(h_state, cur, s, t); cur = ggml_add(ctx0, cur, ggml_scale(ctx0, v, 1.0f / (float) n_step)); } cb(cur, "flow_latent", -1); ggml_set_name(cur, "out_feats"); ggml_set_output(cur); ggml_build_forward_expand(gf, cur); // the same latent, projected into the backbone's input space for the next step ggml_tensor * embd = build_mm(model.gen_input_lin_w, cur); cb(embd, "gen_embd", -1); ggml_build_forward_expand(gf, embd); return gf; } // GEN_WAV: [32, n_frames] latents -> PCM ggml_tensor * feats = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, model.gen_input_lin_w->ne[0], n_frames); ggml_set_name(feats, "inp_feats"); ggml_set_input(feats); // denormalize, then the DummyQuantizer up-projection ggml_tensor * cur = ggml_add(ctx0, ggml_mul(ctx0, feats, model.gen_emb_std), model.gen_emb_mean); cur = build_mm(model.gen_quant_out_w, cur); cb(cur, "quant_out", -1); mtmd_graph_pockettts_seanet seanet(*this); seanet.state_inputs(list_pockettts_state_slots(hparams, model)); cur = seanet.mimi_decode(cur, n_head, eps); seanet.state_outputs(gf); // [n_samples, 1] -> [n_samples], clamped like the reference output cur = ggml_reshape_1d(ctx0, cur, cur->ne[0]); cur = ggml_clamp(ctx0, cur, -1.0f, 1.0f); ggml_set_name(cur, "out_audio"); ggml_set_output(cur); ggml_build_forward_expand(gf, cur); return gf; }