#include "models.h" // Chatterbox generation graphs: flow encoder, cfm estimator, s3 tokenizer and hift vocoder // x [C, T]: y = W x + b with torch Linear weights stored as [in, out] ggml_tensor * clip_graph_chatterbox_base::cbx_linear(ggml_tensor * w, ggml_tensor * b, ggml_tensor * x) const { ggml_tensor * y = ggml_mul_mat(ctx0, w, x); if (b) { y = ggml_add(ctx0, y, b); } return y; } // conv1d over time with asymmetric zero padding: x [C, T] -> [OC, T_out], kernel [K, IC, OC] ggml_tensor * clip_graph_chatterbox_base::cbx_conv1d(ggml_tensor * k, ggml_tensor * b, ggml_tensor * x, int stride, int pad_l, int pad_r) const { ggml_tensor * xt = ggml_cont(ctx0, ggml_transpose(ctx0, x)); // [T, C] if (pad_l > 0) { ggml_tensor * z = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, pad_l, xt->ne[1]); z = ggml_scale(ctx0, z, 0.0f); xt = ggml_concat(ctx0, z, xt, 0); } if (pad_r > 0) { ggml_tensor * z = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, pad_r, xt->ne[1]); z = ggml_scale(ctx0, z, 0.0f); xt = ggml_concat(ctx0, xt, z, 0); } ggml_tensor * y = ggml_conv_1d(ctx0, k, xt, stride, 0, 1); // [T_out, OC] y = ggml_cont(ctx0, ggml_transpose(ctx0, y)); // [OC, T_out] if (b) { y = ggml_add(ctx0, y, b); } return y; } // x [C, T] -> symmetric-padded dilated conv -> [OC, T] ggml_tensor * clip_graph_chatterbox_base::cbx_conv1d_dil(ggml_tensor * k, ggml_tensor * b, ggml_tensor * x, int pad, int dil) const { ggml_tensor * xt = ggml_cont(ctx0, ggml_transpose(ctx0, x)); ggml_tensor * y = ggml_conv_1d(ctx0, k, xt, 1, pad, dil); y = ggml_cont(ctx0, ggml_transpose(ctx0, y)); if (b) { y = ggml_add(ctx0, y, b); } return y; } // Transformer-XL relative shift: bd [2T-1, T, H] -> [T, T, H] with out[j, i, h] = bd[(T-1) - i + j, i, h] // same buffer walk as the espnet rel_shift: pad one column, drop the first row, keep the first T columns static ggml_tensor * cbx_rel_shift(ggml_context * ctx0, ggml_tensor * bd, int T) { const int H = (int) bd->ne[2]; ggml_tensor * z = ggml_new_tensor_3d(ctx0, GGML_TYPE_F32, 1, T, H); z = ggml_scale(ctx0, z, 0.0f); ggml_tensor * p = ggml_cont(ctx0, ggml_concat(ctx0, z, bd, 0)); // [2T, T, H] p = ggml_reshape_3d(ctx0, p, T, 2 * T, H); // [T, 2T, H] p = ggml_view_3d(ctx0, p, T, 2 * T - 1, H, p->nb[1], p->nb[2], p->nb[1]); // drop first row p = ggml_cont(ctx0, p); p = ggml_reshape_3d(ctx0, p, 2 * T - 1, T, H); // [2T-1, T, H] p = ggml_view_3d(ctx0, p, T, T, H, p->nb[1], p->nb[2], 0); // first T columns return ggml_cont(ctx0, p); } // espnet rel-pos self attention block, pre-norm, x [512, T], pos [512, 2T-1] ggml_tensor * clip_graph_chatterbox::enc_layer(const clip_chatterbox::enc_layer & l, ggml_tensor * x, ggml_tensor * pos, int T) { const int n_head = 8; const int d_head = 64; const float scale = 1.0f / sqrtf((float) d_head); ggml_tensor * res = x; ggml_tensor * cur = build_norm(x, l.norm_mha_w, l.norm_mha_b, NORM_TYPE_NORMAL, 1e-5f, -1); ggml_tensor * q = cbx_linear(l.attn_q_w, l.attn_q_b, cur); ggml_tensor * k = cbx_linear(l.attn_k_w, l.attn_k_b, cur); ggml_tensor * v = cbx_linear(l.attn_v_w, l.attn_v_b, cur); ggml_tensor * pe = cbx_linear(l.attn_pos_w, nullptr, pos); // [512, 2T-1] q = ggml_reshape_3d(ctx0, q, d_head, n_head, T); k = ggml_reshape_3d(ctx0, k, d_head, n_head, T); v = ggml_reshape_3d(ctx0, v, d_head, n_head, T); pe = ggml_reshape_3d(ctx0, pe, d_head, n_head, 2 * T - 1); ggml_tensor * u = l.attn_pos_bias_u; // [64, 8] ggml_tensor * w = l.attn_pos_bias_v; ggml_tensor * qu = ggml_add(ctx0, q, ggml_reshape_3d(ctx0, u, d_head, n_head, 1)); ggml_tensor * qv = ggml_add(ctx0, q, ggml_reshape_3d(ctx0, w, d_head, n_head, 1)); // per head: [64, T] tensors, scores [T(k), T(q)] qu = ggml_cont(ctx0, ggml_permute(ctx0, qu, 0, 2, 1, 3)); // [64, T, 8] qv = ggml_cont(ctx0, ggml_permute(ctx0, qv, 0, 2, 1, 3)); k = ggml_cont(ctx0, ggml_permute(ctx0, k, 0, 2, 1, 3)); v = ggml_cont(ctx0, ggml_permute(ctx0, v, 0, 2, 1, 3)); pe = ggml_cont(ctx0, ggml_permute(ctx0, pe, 0, 2, 1, 3)); // [64, 2T-1, 8] ggml_tensor * ac = ggml_mul_mat(ctx0, k, qu); // [T(k), T(q), 8] ggml_tensor * bd = ggml_mul_mat(ctx0, pe, qv); // [2T-1, T(q), 8] bd = cbx_rel_shift(ctx0, bd, T); // [T(k), T(q), 8] ggml_tensor * scores = ggml_scale(ctx0, ggml_add(ctx0, ac, bd), scale); ggml_tensor * probs = ggml_soft_max(ctx0, scores); ggml_tensor * o = ggml_mul_mat(ctx0, ggml_cont(ctx0, ggml_transpose(ctx0, v)), probs); // [64, T, 8] o = ggml_cont(ctx0, ggml_permute(ctx0, o, 0, 2, 1, 3)); // [64, 8, T] o = ggml_reshape_2d(ctx0, o, n_head * d_head, T); o = cbx_linear(l.attn_out_w, l.attn_out_b, o); x = ggml_add(ctx0, res, o); res = x; cur = build_norm(x, l.norm_ff_w, l.norm_ff_b, NORM_TYPE_NORMAL, 1e-5f, -1); cur = cbx_linear(l.ffn_1_w, l.ffn_1_b, cur); cur = ggml_silu(ctx0, cur); // swish cur = cbx_linear(l.ffn_2_w, l.ffn_2_b, cur); x = ggml_add(ctx0, res, cur); return x; } // mish = x * tanh(softplus(x)) static ggml_tensor * cbx_mish(ggml_context * ctx0, ggml_tensor * x) { return ggml_mul(ctx0, x, ggml_tanh(ctx0, ggml_softplus(ctx0, x))); } // causal block: conv k3 left-padded, layer norm over channels, mish; x [C, T] ggml_tensor * clip_graph_chatterbox::causal_block(const clip_chatterbox::causal_block & b, ggml_tensor * x) { x = cbx_conv1d(b.conv_w, b.conv_b, x, 1, (int) b.conv_w->ne[0] - 1, 0); x = build_norm(x, b.norm_w, b.norm_b, NORM_TYPE_NORMAL, 1e-5f, -1); return cbx_mish(ctx0, x); } // resnet block with time conditioning; x [C, T], temb [1024] ggml_tensor * clip_graph_chatterbox::resnet(const clip_chatterbox::resnet & r, ggml_tensor * x, ggml_tensor * temb) { ggml_tensor * h = causal_block(r.block1, x); ggml_tensor * tproj = cbx_linear(r.mlp_w, r.mlp_b, cbx_mish(ctx0, temb)); h = ggml_add(ctx0, h, tproj); // broadcast [256, 1] over T h = causal_block(r.block2, h); ggml_tensor * res = cbx_conv1d(r.res_conv_w, r.res_conv_b, x, 1, 0, 0); return ggml_add(ctx0, h, res); } // diffusers-style transformer block, full attention; x [256, T] ggml_tensor * clip_graph_chatterbox::tfm_block(const clip_chatterbox::tfm_block & b, ggml_tensor * x) { const int n_head = 8; const int d_head = 64; const int T = (int) x->ne[1]; ggml_tensor * cur = build_norm(x, b.norm1_w, b.norm1_b, NORM_TYPE_NORMAL, 1e-5f, -1); ggml_tensor * q = ggml_reshape_3d(ctx0, ggml_mul_mat(ctx0, b.attn_q_w, cur), d_head, n_head, T); ggml_tensor * k = ggml_reshape_3d(ctx0, ggml_mul_mat(ctx0, b.attn_k_w, cur), d_head, n_head, T); ggml_tensor * v = ggml_reshape_3d(ctx0, ggml_mul_mat(ctx0, b.attn_v_w, cur), d_head, n_head, T); x = ggml_add(ctx0, x, build_attn(b.attn_out_w, b.attn_out_b, q, k, v, nullptr, 1.0f / sqrtf((float) d_head), -1)); cur = build_norm(x, b.norm3_w, b.norm3_b, NORM_TYPE_NORMAL, 1e-5f, -1); cur = cbx_linear(b.ff_in_w, b.ff_in_b, cur); cur = ggml_gelu_erf(ctx0, cur); cur = cbx_linear(b.ff_out_w, b.ff_out_b, cur); return ggml_add(ctx0, x, cur); } // one estimator evaluation; x_noise, mu, cond [80, T], spks [80], temb [1024] ggml_tensor * clip_graph_chatterbox::estimator(ggml_tensor * x_noise, ggml_tensor * mu, ggml_tensor * spks, ggml_tensor * cond, ggml_tensor * temb, int T) { const auto & c = model.cbx; // channels live on ne0, time on ne1: pack along ne0 ggml_tensor * x = ggml_concat(ctx0, x_noise, mu, 0); // [160, T] ggml_tensor * spks_b = ggml_repeat(ctx0, ggml_reshape_2d(ctx0, spks, 80, 1), ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, 80, T)); x = ggml_concat(ctx0, x, spks_b, 0); // [240, T] x = ggml_concat(ctx0, x, cond, 0); // [320, T] x = ggml_cont(ctx0, x); // down ggml_tensor * skip; x = resnet(c.est_down.res, x, temb); for (const auto & b : c.est_down.tfm) { x = tfm_block(b, x); } skip = x; x = cbx_conv1d(c.est_down.conv_w, c.est_down.conv_b, x, 1, (int) c.est_down.conv_w->ne[0] - 1, 0); // mid for (const auto & m : c.est_mid) { x = resnet(m.res, x, temb); for (const auto & b : m.tfm) { x = tfm_block(b, x); } } // up with skip x = ggml_concat(ctx0, x, skip, 0); // [512, T] x = resnet(c.est_up.res, x, temb); for (const auto & b : c.est_up.tfm) { x = tfm_block(b, x); } x = cbx_conv1d(c.est_up.conv_w, c.est_up.conv_b, x, 1, (int) c.est_up.conv_w->ne[0] - 1, 0); x = causal_block(c.est_final_block, x); x = cbx_conv1d(c.est_final_proj_w, c.est_final_proj_b, x, 1, 0, 0); // [80, T] return x; } // s3 tokenizer encoder: log-mel [T, 128] -> fsq code [8, T / 4], rounded to base 3 tokens on the host ggml_cgraph * clip_graph_chatterbox::build_s3tok(int T) { const auto & c = model.cbx; const int n_head = 20; const int d_head = 64; const int T1 = (T - 1) / 2 + 1; const int T2 = (T1 - 1) / 2 + 1; ggml_tensor * inp = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, T, 128); ggml_set_name(inp, "inp_raw"); ggml_set_input(inp); ggml_tensor * pos = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, T2); ggml_set_name(pos, "inp_pos"); ggml_set_input(pos); ggml_tensor * x = ggml_cont(ctx0, ggml_transpose(ctx0, inp)); // [128, T] x = cbx_conv1d(c.s3tok_conv1_w, c.s3tok_conv1_b, x, 2, 1, 1); x = ggml_gelu_erf(ctx0, x); x = cbx_conv1d(c.s3tok_conv2_w, c.s3tok_conv2_b, x, 2, 1, 1); x = ggml_gelu_erf(ctx0, x); // [1280, T2] for (const auto & blk : c.s3tok_blocks) { ggml_tensor * res = x; ggml_tensor * cur = build_norm(x, blk.attn_ln_w, blk.attn_ln_b, NORM_TYPE_NORMAL, 1e-5f, -1); ggml_tensor * q = cbx_linear(blk.attn_q_w, blk.attn_q_b, cur); ggml_tensor * k = ggml_mul_mat(ctx0, blk.attn_k_w, cur); ggml_tensor * v = cbx_linear(blk.attn_v_w, blk.attn_v_b, cur); // fsmn memory: depthwise conv over time on v plus residual, added to the attention output ggml_tensor * fsm = ggml_cont(ctx0, ggml_transpose(ctx0, v)); // [T2, 1280] { ggml_tensor * w = blk.fsmn_w; ggml_tensor * m = ggml_conv_1d_dw(ctx0, w, fsm, 1, ((int) w->ne[0] - 1) / 2, 1); fsm = ggml_add(ctx0, ggml_reshape_2d(ctx0, m, fsm->ne[0], fsm->ne[1]), fsm); } fsm = ggml_cont(ctx0, ggml_transpose(ctx0, fsm)); // [1280, T2] q = ggml_reshape_3d(ctx0, q, d_head, n_head, T2); k = ggml_reshape_3d(ctx0, k, d_head, n_head, T2); v = ggml_reshape_3d(ctx0, v, d_head, n_head, T2); q = ggml_rope_ext(ctx0, q, pos, nullptr, d_head, GGML_ROPE_TYPE_NEOX, 0, 10000.0f, 1.0f, 0.0f, 1.0f, 0.0f, 0.0f); k = ggml_rope_ext(ctx0, k, pos, nullptr, d_head, GGML_ROPE_TYPE_NEOX, 0, 10000.0f, 1.0f, 0.0f, 1.0f, 0.0f, 0.0f); ggml_tensor * o = build_attn(blk.attn_out_w, blk.attn_out_b, q, k, v, nullptr, 1.0f / sqrtf((float) d_head), -1); x = ggml_add(ctx0, res, ggml_add(ctx0, o, fsm)); res = x; cur = build_norm(x, blk.mlp_ln_w, blk.mlp_ln_b, NORM_TYPE_NORMAL, 1e-5f, -1); cur = cbx_linear(blk.mlp_in_w, blk.mlp_in_b, cur); cur = ggml_gelu_erf(ctx0, cur); cur = cbx_linear(blk.mlp_out_w, blk.mlp_out_b, cur); x = ggml_add(ctx0, res, cur); } x = cbx_linear(c.s3tok_down_w, c.s3tok_down_b, x); // [8, T2] x = ggml_tanh(ctx0, x); ggml_set_name(x, "out_fsq"); ggml_set_output(x); ggml_build_forward_expand(gf, x); return gf; } ggml_cgraph * clip_graph_chatterbox::build() { if (gen_process == CLIP_GEN_PROCESS_TTS_VOCODE) { return build_vocoder(vocode_n_mel, vocode_n_stft); } if (gen_process == CLIP_GEN_PROCESS_TOKENIZE) { return build_s3tok(img.nx()); } if (gen_process != CLIP_GEN_PROCESS_TTS) { // load-time buffer sizing path ggml_tensor * inp = ggml_new_tensor_1d(ctx0, GGML_TYPE_F32, 1); ggml_set_name(inp, "inp_stub"); ggml_set_input(inp); ggml_tensor * cur = ggml_dup(ctx0, inp); ggml_set_name(cur, "out_stub"); ggml_set_output(cur); ggml_build_forward_expand(gf, cur); return gf; } const auto & c = model.cbx; const int T1 = n_tokens; // token-rate length (prompt + generated) const int T2 = 2 * n_tokens; // mel-rate length after the x2 upsample ggml_tensor * inp_tokens = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, T1); ggml_set_name(inp_tokens, "inp_tokens"); ggml_set_input(inp_tokens); ggml_tensor * pos1 = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, 512, 2 * T1 - 1); ggml_set_name(pos1, "inp_pos1"); ggml_set_input(pos1); ggml_tensor * pos2 = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, 512, 2 * T2 - 1); ggml_set_name(pos2, "inp_pos2"); ggml_set_input(pos2); // token embedding ggml_tensor * x = ggml_get_rows(ctx0, c.input_embedding_w, inp_tokens); // [512, T1] // embed: linear + layer norm, then the espnet xscale x = cbx_linear(c.embed_linear_w, c.embed_linear_b, x); x = build_norm(x, c.embed_norm_w, c.embed_norm_b, NORM_TYPE_NORMAL, 1e-5f, -1); x = ggml_scale(ctx0, x, sqrtf(512.0f)); cb(x, "fenc_embd", -1); // pre-lookahead: conv k=4 right-padded 3, leaky 0.01, conv k=3 left-padded 2, residual { ggml_tensor * res = x; ggml_tensor * cur = cbx_conv1d(c.pre_conv1_w, c.pre_conv1_b, x, 1, 0, 3); cur = ggml_leaky_relu(ctx0, cur, 0.01f, false); cur = cbx_conv1d(c.pre_conv2_w, c.pre_conv2_b, cur, 1, 2, 0); x = ggml_add(ctx0, res, cur); cb(x, "fenc_pre_lookahead", -1); } for (size_t i = 0; i < c.enc.size(); i++) { x = enc_layer(c.enc[i], x, pos1, T1); cb(x, "fenc_enc", (int) i); } // upsample x2: nearest repeat, left pad 4, conv k=5 { ggml_tensor * xt = ggml_cont(ctx0, ggml_transpose(ctx0, x)); // [T1, 512] xt = ggml_interpolate(ctx0, xt, 2 * T1, 512, 1, 1, GGML_SCALE_MODE_NEAREST); ggml_tensor * z = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, 4, 512); z = ggml_scale(ctx0, z, 0.0f); xt = ggml_concat(ctx0, z, xt, 0); ggml_tensor * y = ggml_conv_1d(ctx0, c.up_conv_w, xt, 1, 0, 1); x = ggml_cont(ctx0, ggml_transpose(ctx0, y)); // [512, T2] x = ggml_add(ctx0, x, c.up_conv_b); cb(x, "fenc_upsample", -1); } // up embed: linear + layer norm + xscale x = cbx_linear(c.up_embed_linear_w, c.up_embed_linear_b, x); x = build_norm(x, c.up_embed_norm_w, c.up_embed_norm_b, NORM_TYPE_NORMAL, 1e-5f, -1); x = ggml_scale(ctx0, x, sqrtf(512.0f)); for (size_t i = 0; i < c.up_enc.size(); i++) { x = enc_layer(c.up_enc[i], x, pos2, T2); cb(x, "fenc_up_enc", (int) i); } x = build_norm(x, c.after_norm_w, c.after_norm_b, NORM_TYPE_NORMAL, 1e-5f, -1); // encoder projection to the mel channel count ggml_tensor * mu = cbx_linear(c.encoder_proj_w, c.encoder_proj_b, x); // [80, T2] cb(mu, "flow_mu", -1); // cfm solver unrolled in the graph: meanflow takes 2 euler steps without cfg, its time embeds mix t and r // the classic solver takes 10 euler steps on the cosine schedule with cfg 0.7 const bool meanflow = c.time_embed_mixer_w != nullptr; const int n_steps = meanflow ? 2 : 10; // span points, same schedule as the host side sinusoid fill in clip.cpp float span[11]; for (int i = 0; i <= n_steps; i++) { const float u = (float) i / n_steps; span[i] = meanflow ? u : 1.0f - cosf(u * (float) M_PI / 2.0f); } ggml_tensor * noise = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, 80, T2); ggml_set_name(noise, "inp_noise"); ggml_set_input(noise); // sinusoidal time embeddings, one row per span point ggml_tensor * temb_sin = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, 320, n_steps + 1); ggml_set_name(temb_sin, "inp_temb"); ggml_set_input(temb_sin); auto time_mlp = [&](ggml_tensor * e) { e = cbx_linear(c.time_mlp_1_w, c.time_mlp_1_b, e); e = ggml_silu(ctx0, e); e = cbx_linear(c.time_mlp_2_w, c.time_mlp_2_b, e); return e; }; auto span_emb = [&](int i) { return ggml_view_2d(ctx0, temb_sin, 320, 1, temb_sin->nb[1], (size_t) i * temb_sin->nb[1]); }; auto step_temb = [&](int i) { if (!meanflow) { return time_mlp(span_emb(i)); } ggml_tensor * e = ggml_concat(ctx0, time_mlp(span_emb(i)), time_mlp(span_emb(i + 1)), 0); // [2048, 1] return ggml_mul_mat(ctx0, c.time_embed_mixer_w, e); // [1024, 1] }; // mel-rate conditions of the voice: prompt features then zeros, and the speaker vector ggml_tensor * pf = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, 80, n_prompt_mel); ggml_set_name(pf, "inp_prompt_feat"); ggml_set_input(pf); ggml_tensor * zc = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, 80, T2 - n_prompt_mel); zc = ggml_scale(ctx0, zc, 0.0f); ggml_tensor * cond = ggml_concat(ctx0, pf, zc, 1); // [80, T2] // raw 192-dim campplus x-vector, normalized then through the flow speaker affine ggml_tensor * xvec = ggml_new_tensor_1d(ctx0, GGML_TYPE_F32, 192); ggml_set_name(xvec, "inp_xvec"); ggml_set_input(xvec); ggml_tensor * n2 = ggml_sqrt(ctx0, ggml_sum(ctx0, ggml_mul(ctx0, xvec, xvec))); ggml_tensor * unit = ggml_div(ctx0, xvec, n2); ggml_tensor * spks = cbx_linear(c.spk_affine_w, c.spk_affine_b, ggml_reshape_2d(ctx0, unit, 192, 1)); spks = ggml_cont(ctx0, ggml_reshape_1d(ctx0, spks, 80)); const float cfg = 0.7f; ggml_tensor * mu_zero = meanflow ? nullptr : ggml_scale(ctx0, mu, 0.0f); ggml_tensor * cond_zero = meanflow ? nullptr : ggml_scale(ctx0, cond, 0.0f); ggml_tensor * spks_zero = meanflow ? nullptr : ggml_scale(ctx0, spks, 0.0f); ggml_tensor * mx = noise; for (int i = 0; i < n_steps; i++) { ggml_tensor * temb = step_temb(i); ggml_tensor * d = estimator(mx, mu, spks, cond, temb, T2); if (!meanflow) { ggml_tensor * du = estimator(mx, mu_zero, spks_zero, cond_zero, temb, T2); d = ggml_add(ctx0, ggml_scale(ctx0, d, 1.0f + cfg), ggml_scale(ctx0, du, -cfg)); } mx = ggml_add(ctx0, mx, ggml_scale(ctx0, d, span[i + 1] - span[i])); cb(mx, "cfm_step", i); } ggml_tensor * mel = mx; // trim the prompt frames at mel rate mel = ggml_view_2d(ctx0, mel, 80, T2 - n_prompt_mel, mel->nb[1], (size_t) n_prompt_mel * mel->nb[1]); mel = ggml_cont(ctx0, mel); ggml_set_name(mel, "out_mel"); ggml_set_output(mel); ggml_build_forward_expand(gf, mel); // f0 predictor on the trimmed mel: 5x (conv k3 same-pad + elu), abs(linear) { ggml_tensor * fx = mel; for (const auto & fc : c.f0_condnet) { fx = cbx_conv1d(fc.w, fc.b, fx, 1, 1, 1); fx = ggml_elu(ctx0, fx); } fx = cbx_linear(c.f0_classifier_w, c.f0_classifier_b, fx); fx = ggml_abs(ctx0, fx); // [1, T] ggml_set_name(fx, "out_f0"); ggml_set_output(fx); ggml_build_forward_expand(gf, fx); } return gf; } // snake activation with per-channel alpha: x + sin^2(alpha x) / alpha static ggml_tensor * cbx_snake(ggml_context * ctx0, ggml_tensor * x, ggml_tensor * alpha) { ggml_tensor * sx = ggml_sin(ctx0, ggml_mul(ctx0, x, alpha)); sx = ggml_mul(ctx0, sx, sx); sx = ggml_div(ctx0, sx, alpha); return ggml_add(ctx0, x, sx); } // hifigan-snake resblock; kernels with dilations 1/3/5 on convs1, 1 on convs2 ggml_tensor * clip_graph_chatterbox::hift_resblock(const clip_chatterbox::hift_res & r, ggml_tensor * x) { static const int dil[3] = {1, 3, 5}; for (size_t j = 0; j < r.units.size(); j++) { const auto & u = r.units[j]; const int d = dil[j % 3]; const int p1 = (int) (u.conv1_w->ne[0] - 1) / 2 * d; const int p2 = (int) (u.conv2_w->ne[0] - 1) / 2; ggml_tensor * xt = cbx_snake(ctx0, x, u.act1_alpha); xt = cbx_conv1d_dil(u.conv1_w, u.conv1_b, xt, p1, d); xt = cbx_snake(ctx0, xt, u.act2_alpha); xt = cbx_conv1d_dil(u.conv2_w, u.conv2_b, xt, p2, 1); x = ggml_add(ctx0, x, xt); } return x; } // mel [80, T] + source stft [18, T_stft] -> conv_post output [18, T_stft2] ggml_cgraph * clip_graph_chatterbox::build_vocoder(int n_mel, int n_stft) { const auto & c = model.cbx; ggml_tensor * mel = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, 80, n_mel); ggml_set_name(mel, "inp_mel"); ggml_set_input(mel); ggml_tensor * sstft = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, 18, n_stft); ggml_set_name(sstft, "inp_sstft"); ggml_set_input(sstft); ggml_tensor * x = cbx_conv1d_dil(c.hift_pre_w, c.hift_pre_b, mel, 3, 1); for (size_t i = 0; i < c.hift_ups.size(); i++) { const auto & up = c.hift_ups[i]; ggml_tensor * uk = up.up_w; const int K = (int) uk->ne[0]; const int S = K / 2; const int P = (K - S) / 2; x = ggml_leaky_relu(ctx0, x, 0.1f, false); // conv transpose then trim the torch padding P on both sides ggml_tensor * xt = ggml_cont(ctx0, ggml_transpose(ctx0, x)); xt = ggml_conv_transpose_1d(ctx0, uk, xt, S, 0, 1); xt = ggml_cont(ctx0, ggml_view_2d(ctx0, xt, xt->ne[0] - 2 * P, xt->ne[1], xt->nb[1], (size_t) P * ggml_element_size(xt))); x = ggml_cont(ctx0, ggml_transpose(ctx0, xt)); x = ggml_add(ctx0, x, up.up_b); const bool is_last = i + 1 == c.hift_ups.size(); if (is_last) { ggml_tensor * xr = ggml_cont(ctx0, ggml_transpose(ctx0, x)); xr = ggml_pad_reflect_1d(ctx0, xr, 1, 0); x = ggml_cont(ctx0, ggml_transpose(ctx0, xr)); } // source injection: strided conv on the source stft, one resblock ggml_tensor * sk = up.source_down_w; const int SK = (int) sk->ne[0]; const int SS = SK > 1 ? SK / 2 : 1; const int SP = SK > 1 ? SS / 2 : 0; ggml_tensor * si; { ggml_tensor * st = ggml_cont(ctx0, ggml_transpose(ctx0, sstft)); st = ggml_conv_1d(ctx0, sk, st, SS, SP, 1); si = ggml_cont(ctx0, ggml_transpose(ctx0, st)); si = ggml_add(ctx0, si, up.source_down_b); } si = hift_resblock(up.source_res, si); // align lengths: the reflection pad on the last stage adds one step if ((int) si->ne[1] != (int) x->ne[1]) { const int n = (int) std::min(si->ne[1], x->ne[1]); si = ggml_cont(ctx0, ggml_view_2d(ctx0, si, si->ne[0], n, si->nb[1], 0)); x = ggml_cont(ctx0, ggml_view_2d(ctx0, x, x->ne[0], n, x->nb[1], 0)); } x = ggml_add(ctx0, x, si); ggml_tensor * acc = hift_resblock(up.res_0, x); acc = ggml_add(ctx0, acc, hift_resblock(up.res_1, x)); acc = ggml_add(ctx0, acc, hift_resblock(up.res_2, x)); x = ggml_scale(ctx0, acc, 1.0f / 3.0f); } x = ggml_leaky_relu(ctx0, x, 0.01f, false); x = cbx_conv1d_dil(c.hift_post_w, c.hift_post_b, x, 3, 1); ggml_set_name(x, "out_spec"); ggml_set_output(x); ggml_build_forward_expand(gf, x); return gf; }