Files
llama.cpp/tools/mtmd/models/kokoro.cpp
T
Xuan-Son Nguyen 307e3f1551 mtmd: add kokoro tts support (#32413)
* jinja: impl scope-chain key lookup

* add test for call_statement lambda

* chat: add tts_dict for kokoro G2P template

* more testing

* value_float_t range check

* no json-in-meta, add llama_model_meta_arr_str/_n

* mtmd: add kokoro tts support

* detailed guide about G2P and text norm port

* refine G2P

* clean up

* migrate batch api

* nits
2026-11-17 22:25:44 +01:00

516 lines
23 KiB
C++

#include "models.h"
#include <algorithm>
#include <cmath>
#include <cstring>
#include <random>
#include <stdexcept>
// Kokoro: StyleTTS2 prosody predictor + iSTFTNet decoder. Tensors keep the names of the reference modules (a.gen.<module>.*)
// One utterance runs the 3 graphs of stage_type. The LSTMs, the alignment, the harmonic source and the ISTFT run on the host in between
// The graphs keep activations as [T, C], the layout of the conv ops
static const int KOKORO_N_HIDDEN = 512;
static const int KOKORO_N_HARM = 9; // fundamental + 8 overtones
static const int KOKORO_UPS = 300; // samples per F0 step: upsample rates 10 * 6, times the ISTFT hop
static const int KOKORO_N_FFT = 20;
static const int KOKORO_HOP = 5;
static const float KOKORO_SR = 24000.0f;
ggml_tensor * mtmd_graph_kokoro::tensor(const std::string & name) const {
const auto it = model.cbx_tensors.find(name);
if (it == model.cbx_tensors.end() || !it->second) {
throw std::runtime_error("kokoro: missing tensor " + name);
}
return it->second;
}
bool mtmd_graph_kokoro::has_tensor(const std::string & name) const {
return model.cbx_tensors.find(name) != model.cbx_tensors.end();
}
// x [T, IC] -> [T_out, OC], with the bias if the module has one
ggml_tensor * mtmd_graph_kokoro::conv(const std::string & name, ggml_tensor * x, int stride, int pad, int dil) const {
ggml_tensor * y = ggml_conv_1d(ctx0, tensor(name + ".weight"), x, stride, pad, dil);
if (has_tensor(name + ".bias")) {
ggml_tensor * b = tensor(name + ".bias");
y = ggml_add(ctx0, y, ggml_reshape_2d(ctx0, b, 1, b->ne[0]));
}
return y;
}
// AdaIN1d: instance norm over time, then scale and shift from the style
ggml_tensor * mtmd_graph_kokoro::adain(const std::string & name, ggml_tensor * x, ggml_tensor * style) const {
const int64_t n_ch = x->ne[1];
ggml_tensor * h = ggml_add(ctx0, ggml_mul_mat(ctx0, tensor(name + ".fc.weight"), style), tensor(name + ".fc.bias"));
ggml_tensor * gamma = ggml_reshape_2d(ctx0, ggml_view_1d(ctx0, h, n_ch, 0), 1, n_ch);
ggml_tensor * beta = ggml_reshape_2d(ctx0, ggml_view_1d(ctx0, h, n_ch, n_ch * ggml_element_size(h)), 1, n_ch);
ggml_tensor * xn = ggml_norm(ctx0, x, 1e-5f);
return ggml_add(ctx0, ggml_add(ctx0, xn, ggml_mul(ctx0, xn, gamma)), beta);
}
// x + sin^2(alpha x) / alpha, alpha per channel
ggml_tensor * mtmd_graph_kokoro::snake(const std::string & name, ggml_tensor * x) const {
ggml_tensor * alpha = tensor(name);
alpha = ggml_reshape_2d(ctx0, alpha, 1, alpha->ne[0]);
ggml_tensor * sx = ggml_sin(ctx0, ggml_mul(ctx0, x, alpha));
sx = ggml_div(ctx0, ggml_mul(ctx0, sx, sx), alpha);
return ggml_add(ctx0, x, sx);
}
// a [T, C] and b [T, C] -> [2T, C] with a on the even steps
ggml_tensor * mtmd_graph_kokoro::interleave(ggml_tensor * a, ggml_tensor * b) const {
const int64_t n_t = a->ne[0], n_ch = a->ne[1];
ggml_tensor * ab = ggml_concat(ctx0, ggml_reshape_3d(ctx0, ggml_cont(ctx0, a), 1, n_t, n_ch),
ggml_reshape_3d(ctx0, ggml_cont(ctx0, b), 1, n_t, n_ch), 0);
return ggml_reshape_2d(ctx0, ggml_cont(ctx0, ab), 2 * n_t, n_ch);
}
// AdainResBlk1d, upsample doubles the length
ggml_tensor * mtmd_graph_kokoro::adain_resblk(const std::string & name, ggml_tensor * x, ggml_tensor * style, bool upsample) const {
const int64_t n_t = x->ne[0], n_ch = x->ne[1];
ggml_tensor * r = adain(name + ".norm1", x, style);
r = ggml_leaky_relu(ctx0, r, 0.2f, false);
if (upsample) {
// depthwise ConvTranspose1d(kernel 3, stride 2, padding 1, output_padding 1), weight stored as [C, 3]:
// y[2i] = w1 x[i], y[2i+1] = w2 x[i] + w0 x[i+1]
ggml_tensor * w = tensor(name + ".pool.weight");
ggml_tensor * b = tensor(name + ".pool.bias");
auto tap = [&](int k) {
return ggml_reshape_2d(ctx0, ggml_view_1d(ctx0, w, n_ch, (size_t) k * n_ch * ggml_element_size(w)), 1, n_ch);
};
ggml_tensor * next = ggml_cont(ctx0, ggml_view_2d(ctx0, r, n_t - 1, n_ch, r->nb[1], ggml_element_size(r)));
next = ggml_pad(ctx0, next, 1, 0, 0, 0);
ggml_tensor * even = ggml_mul(ctx0, r, tap(1));
ggml_tensor * odd = ggml_add(ctx0, ggml_mul(ctx0, r, tap(2)), ggml_mul(ctx0, next, tap(0)));
r = ggml_add(ctx0, interleave(even, odd), ggml_reshape_2d(ctx0, b, 1, n_ch));
}
r = conv(name + ".conv1", r, 1, 1, 1);
r = adain(name + ".norm2", r, style);
r = ggml_leaky_relu(ctx0, r, 0.2f, false);
r = conv(name + ".conv2", r, 1, 1, 1);
ggml_tensor * sc = upsample ? interleave(x, x) : x;
if (has_tensor(name + ".conv1x1.weight")) {
sc = conv(name + ".conv1x1", sc, 1, 0, 1);
}
return ggml_scale(ctx0, ggml_add(ctx0, r, sc), (float) M_SQRT1_2);
}
// AdaINResBlock1 of the generator: three units with dilations 1/3/5 on convs1
ggml_tensor * mtmd_graph_kokoro::gen_resblock(const std::string & name, ggml_tensor * x, ggml_tensor * style) const {
static const int dil[3] = {1, 3, 5};
for (int j = 0; j < 3; j++) {
const std::string js = std::to_string(j);
const int k = (int) tensor(name + ".convs1." + js + ".weight")->ne[0];
ggml_tensor * xt = adain(name + ".adain1." + js, x, style);
xt = snake(name + ".alpha1." + js, xt);
xt = conv(name + ".convs1." + js, xt, 1, (k * dil[j] - dil[j]) / 2, dil[j]);
xt = adain(name + ".adain2." + js, xt, style);
xt = snake(name + ".alpha2." + js, xt);
xt = conv(name + ".convs2." + js, xt, 1, (k - 1) / 2, 1);
x = ggml_add(ctx0, xt, x);
}
return x;
}
ggml_tensor * mtmd_graph_kokoro::input(const char * name, ggml_type type, int64_t ne0, int64_t ne1) const {
ggml_tensor * t = ggml_new_tensor_2d(ctx0, type, ne0, ne1);
ggml_set_name(t, name);
ggml_set_input(t);
return t;
}
ggml_cgraph * mtmd_graph_kokoro::output(const char * name, ggml_tensor * t) const {
ggml_set_name(t, name);
ggml_set_output(t);
ggml_build_forward_expand(gf, t);
return gf;
}
// tokens -> text encoder CNN output [C, n_tok]
ggml_cgraph * mtmd_graph_kokoro::build_text() {
const std::string p = "a.gen.text.";
ggml_tensor * tokens = input("inp_tokens", GGML_TYPE_I32, n_tok, 1);
tokens = ggml_reshape_1d(ctx0, tokens, n_tok);
ggml_tensor * x = ggml_get_rows(ctx0, tensor(p + "embedding.weight"), tokens); // [C, T]
for (int i = 0; has_tensor(p + "cnn." + std::to_string(i) + ".0.weight"); i++) {
const std::string q = p + "cnn." + std::to_string(i);
const int k = (int) tensor(q + ".0.weight")->ne[0];
x = conv(q + ".0", ggml_cont(ctx0, ggml_transpose(ctx0, x)), 1, (k - 1) / 2, 1);
// LayerNorm over the channels
x = ggml_norm(ctx0, ggml_cont(ctx0, ggml_transpose(ctx0, x)), 1e-5f);
x = ggml_add(ctx0, ggml_mul(ctx0, x, tensor(q + ".1.gamma")), tensor(q + ".1.beta"));
x = ggml_leaky_relu(ctx0, x, 0.2f, false);
}
return output("out_text", x);
}
// shared LSTM output [C, n_frames] -> F0 and energy curves, 2 steps per frame
ggml_cgraph * mtmd_graph_kokoro::build_f0n() {
const std::string p = "a.gen.pred.";
ggml_tensor * inp = input("inp_x", GGML_TYPE_F32, KOKORO_N_HIDDEN, n_frames);
ggml_tensor * style = input("inp_style", GGML_TYPE_F32, n_style, 1);
style = ggml_reshape_1d(ctx0, style, n_style);
ggml_tensor * x = ggml_cont(ctx0, ggml_transpose(ctx0, inp));
for (const char * curve : {"F0", "N"}) {
ggml_tensor * cur = x;
for (int i = 0; i < 3; i++) {
const std::string q = p + curve + "." + std::to_string(i);
cur = adain_resblk(q, cur, style, has_tensor(q + ".pool.weight"));
}
cur = conv(p + curve + "_proj", cur, 1, 0, 1);
ggml_set_name(cur, curve[0] == 'F' ? "out_f0" : "out_n");
ggml_set_output(cur);
ggml_build_forward_expand(gf, cur);
}
return gf;
}
// aligned text features, F0, energy and the source STFT -> conv_post output [n_fft + 2, n_har]
ggml_cgraph * mtmd_graph_kokoro::build_dec() {
const std::string p = "a.gen.dec.";
ggml_tensor * asr = input("inp_asr", GGML_TYPE_F32, KOKORO_N_HIDDEN, n_frames);
ggml_tensor * f0 = input("inp_f0", GGML_TYPE_F32, 2 * n_frames, 1);
ggml_tensor * en = input("inp_n", GGML_TYPE_F32, 2 * n_frames, 1);
ggml_tensor * har = input("inp_har", GGML_TYPE_F32, KOKORO_N_FFT + 2, n_har);
ggml_tensor * style = input("inp_style", GGML_TYPE_F32, n_style, 1);
style = ggml_reshape_1d(ctx0, style, n_style);
asr = ggml_cont(ctx0, ggml_transpose(ctx0, asr));
har = ggml_cont(ctx0, ggml_transpose(ctx0, har));
f0 = conv(p + "F0_conv", f0, 2, 1, 1);
en = conv(p + "N_conv", en, 2, 1, 1);
ggml_tensor * x = ggml_concat(ctx0, ggml_concat(ctx0, asr, f0, 1), en, 1);
x = adain_resblk(p + "encode", x, style, false);
ggml_tensor * asr_res = conv(p + "asr_res.0", asr, 1, 0, 1);
for (int i = 0; has_tensor(p + "decode." + std::to_string(i) + ".conv1.weight"); i++) {
const std::string q = p + "decode." + std::to_string(i);
x = ggml_concat(ctx0, ggml_concat(ctx0, ggml_concat(ctx0, x, asr_res, 1), f0, 1), en, 1);
x = adain_resblk(q, x, style, has_tensor(q + ".pool.weight"));
}
const std::string g = p + "generator.";
int n_ups = 0;
while (has_tensor(g + "ups." + std::to_string(n_ups) + ".weight")) {
n_ups++;
}
for (int i = 0; i < n_ups; i++) {
const std::string is = std::to_string(i);
ggml_tensor * up_w = tensor(g + "ups." + is + ".weight");
ggml_tensor * up_b = tensor(g + "ups." + is + ".bias");
// the transposed conv has kernel 2 * stride and torch padding stride / 2
const int stride = (int) up_w->ne[0] / 2;
const int trim = stride / 2;
x = ggml_leaky_relu(ctx0, x, 0.1f, false);
// the source STFT is brought to the rate of this stage
ggml_tensor * src_w = tensor(g + "noise_convs." + is + ".weight");
const int src_k = (int) src_w->ne[0];
const int src_s = src_k > 1 ? src_k / 2 : 1;
ggml_tensor * src = conv(g + "noise_convs." + is, har, src_s, src_k > 1 ? (src_s + 1) / 2 : 0, 1);
src = gen_resblock(g + "noise_res." + is, src, style);
x = ggml_conv_transpose_1d(ctx0, up_w, x, stride, 0, 1);
x = ggml_cont(ctx0, ggml_view_2d(ctx0, x, x->ne[0] - 2 * trim, x->ne[1], x->nb[1], (size_t) trim * ggml_element_size(x)));
x = ggml_add(ctx0, x, ggml_reshape_2d(ctx0, up_b, 1, up_b->ne[0]));
if (i == n_ups - 1) {
x = ggml_pad_reflect_1d(ctx0, x, 1, 0);
}
x = ggml_add(ctx0, x, src);
ggml_tensor * acc = nullptr;
for (int j = 0; j < 3; j++) {
ggml_tensor * r = gen_resblock(g + "resblocks." + std::to_string(i * 3 + j), x, style);
acc = acc ? ggml_add(ctx0, acc, r) : r;
}
x = ggml_scale(ctx0, acc, 1.0f / 3.0f);
}
x = ggml_leaky_relu(ctx0, x, 0.01f, false);
x = conv(g + "conv_post", x, 1, 3, 1);
return output("out_spec", ggml_cont(ctx0, ggml_transpose(ctx0, x)));
}
ggml_cgraph * mtmd_graph_kokoro::build() {
switch (stage) {
case STAGE_TEXT: return build_text();
case STAGE_F0N: return build_f0n();
case STAGE_DEC: return build_dec();
}
GGML_ABORT("invalid kokoro stage");
}
//
// host side
//
static std::vector<float> kokoro_read(const mtmd_model & model, const std::string & name, size_t n_expect) {
std::vector<float> v(n_expect);
if (model.cbx_read_tensor(name.c_str(), v.data(), v.size()) != n_expect) {
throw std::runtime_error("kokoro: unexpected tensor " + name);
}
return v;
}
static inline float kokoro_sigmoid(float x) {
return 1.0f / (1.0f + expf(-x));
}
// y = W x, W row-major [n_out, n_in]
static void kokoro_matvec(const float * w, const float * x, int n_out, int n_in, float * y) {
for (int o = 0; o < n_out; o++) {
const float * row = w + (size_t) o * n_in;
float acc[8] = {0.0f};
int i = 0;
for (; i + 8 <= n_in; i += 8) {
for (int k = 0; k < 8; k++) {
acc[k] += row[i + k] * x[i + k];
}
}
float sum = 0.0f;
for (; i < n_in; i++) {
sum += row[i] * x[i];
}
for (int k = 0; k < 8; k++) {
sum += acc[k];
}
y[o] = sum;
}
}
// bidirectional single-layer LSTM (torch layout, gates i f g o): x [n_t, n_in] -> y [n_t, 2 * n_h]
static void kokoro_lstm(const mtmd_model & model, const std::string & name, const std::vector<float> & x, int n_t, int n_in,
std::vector<float> & y) {
const int n_h = KOKORO_N_HIDDEN / 2;
y.assign((size_t) n_t * 2 * n_h, 0.0f);
std::vector<float> gi(4 * n_h), gh(4 * n_h), h(n_h), c(n_h);
for (int dir = 0; dir < 2; dir++) {
const std::string sfx = dir == 0 ? "" : "_reverse";
const auto w_ih = kokoro_read(model, name + ".weight_ih_l0" + sfx, (size_t) 4 * n_h * n_in);
const auto w_hh = kokoro_read(model, name + ".weight_hh_l0" + sfx, (size_t) 4 * n_h * n_h);
const auto bias = kokoro_read(model, name + ".bias_l0" + sfx, (size_t) 4 * n_h);
std::fill(h.begin(), h.end(), 0.0f);
std::fill(c.begin(), c.end(), 0.0f);
for (int s = 0; s < n_t; s++) {
const int t = dir == 0 ? s : n_t - 1 - s;
kokoro_matvec(w_ih.data(), x.data() + (size_t) t * n_in, 4 * n_h, n_in, gi.data());
kokoro_matvec(w_hh.data(), h.data(), 4 * n_h, n_h, gh.data());
float * out = y.data() + (size_t) t * 2 * n_h + (size_t) dir * n_h;
for (int j = 0; j < n_h; j++) {
const float ig = kokoro_sigmoid(gi[j] + gh[j] + bias[j]);
const float fg = kokoro_sigmoid(gi[n_h + j] + gh[n_h + j] + bias[n_h + j]);
const float gg = tanhf(gi[2 * n_h + j] + gh[2 * n_h + j] + bias[2 * n_h + j]);
const float og = kokoro_sigmoid(gi[3 * n_h + j] + gh[3 * n_h + j] + bias[3 * n_h + j]);
c[j] = fg * c[j] + ig * gg;
h[j] = og * tanhf(c[j]);
out[j] = h[j];
}
}
}
}
// rows of x repeated by their duration: [n_tok, n_ch] -> [n_frames, n_ch]
static std::vector<float> kokoro_align(const std::vector<float> & x, const std::vector<int32_t> & dur, int n_ch) {
std::vector<float> y;
for (size_t t = 0; t < dur.size(); t++) {
for (int32_t r = 0; r < dur[t]; r++) {
y.insert(y.end(), x.begin() + t * n_ch, x.begin() + (t + 1) * n_ch);
}
}
return y;
}
int mtmd_graph_kokoro::prosody(const mtmd_model & model, const float * bert, int n_embd, const float * text_cnn, int n_tok,
const float * style, float speed, std::vector<float> & shared, std::vector<float> & asr) {
const int n_h = KOKORO_N_HIDDEN;
const int n_cat = n_h + n_style;
// bert_encoder, then each row is followed by the style
const auto enc_w = kokoro_read(model, "a.gen.bert_enc.weight", (size_t) n_h * n_embd);
const auto enc_b = kokoro_read(model, "a.gen.bert_enc.bias", n_h);
std::vector<float> x((size_t) n_tok * n_cat);
for (int t = 0; t < n_tok; t++) {
float * row = x.data() + (size_t) t * n_cat;
kokoro_matvec(enc_w.data(), bert + (size_t) t * n_embd, n_h, n_embd, row);
for (int j = 0; j < n_h; j++) {
row[j] += enc_b[j];
}
memcpy(row + n_h, style, n_style * sizeof(float));
}
// DurationEncoder: LSTM, AdaLayerNorm, style again
std::vector<float> y;
std::vector<float> gamma_beta(2 * n_h);
for (int il = 0; model.cbx_tensors.count(mtmd_format("a.gen.pred.text_encoder.lstms.%d.weight_ih_l0", 2 * il)); il++) {
kokoro_lstm(model, mtmd_format("a.gen.pred.text_encoder.lstms.%d", 2 * il), x, n_tok, n_cat, y);
const std::string norm = mtmd_format("a.gen.pred.text_encoder.lstms.%d.fc", 2 * il + 1);
const auto fc_w = kokoro_read(model, norm + ".weight", (size_t) 2 * n_h * n_style);
const auto fc_b = kokoro_read(model, norm + ".bias", (size_t) 2 * n_h);
kokoro_matvec(fc_w.data(), style, 2 * n_h, n_style, gamma_beta.data());
for (int t = 0; t < n_tok; t++) {
const float * src = y.data() + (size_t) t * n_h;
float * dst = x.data() + (size_t) t * n_cat;
double mean = 0.0, var = 0.0;
for (int j = 0; j < n_h; j++) {
mean += src[j];
}
mean /= n_h;
for (int j = 0; j < n_h; j++) {
var += (src[j] - mean) * (src[j] - mean);
}
const double scale = 1.0 / sqrt(var / n_h + 1e-5);
for (int j = 0; j < n_h; j++) {
const float v = (float) ((src[j] - mean) * scale);
dst[j] = (1.0f + gamma_beta[j] + fc_b[j]) * v + gamma_beta[n_h + j] + fc_b[n_h + j];
}
}
}
// durations: frames per token
kokoro_lstm(model, "a.gen.pred.lstm", x, n_tok, n_cat, y);
const ggml_tensor * proj = model.cbx_tensors.at("a.gen.pred.duration_proj.linear_layer.weight");
const int n_bins = (int) proj->ne[1];
const auto proj_w = kokoro_read(model, "a.gen.pred.duration_proj.linear_layer.weight", (size_t) n_bins * n_h);
const auto proj_b = kokoro_read(model, "a.gen.pred.duration_proj.linear_layer.bias", n_bins);
std::vector<int32_t> dur(n_tok);
std::vector<float> logits(n_bins);
int n_frames = 0;
for (int t = 0; t < n_tok; t++) {
kokoro_matvec(proj_w.data(), y.data() + (size_t) t * n_h, n_bins, n_h, logits.data());
float sum = 0.0f;
for (int j = 0; j < n_bins; j++) {
sum += kokoro_sigmoid(logits[j] + proj_b[j]);
}
// round half to even, like torch.round
dur[t] = std::max(1, (int) nearbyintf(sum / speed));
n_frames += dur[t];
}
kokoro_lstm(model, "a.gen.pred.shared", kokoro_align(x, dur, n_cat), n_frames, n_cat, shared);
// text encoder: its LSTM runs on the CNN output
kokoro_lstm(model, "a.gen.text.lstm", std::vector<float>(text_cnn, text_cnn + (size_t) n_tok * n_h), n_tok, n_h, y);
asr = kokoro_align(y, dur, n_h);
return n_frames;
}
// hn-nsf source: f0 [2 * n_frames] -> harmonic source, then its STFT as [n_har, 22] rows of magnitudes and phases
int mtmd_graph_kokoro::har_source(const mtmd_model & model, const std::vector<float> & f0, std::mt19937 & rng, std::vector<float> & har) {
const int n_f0 = (int) f0.size();
const int64_t n_wav = (int64_t) n_f0 * KOKORO_UPS;
const auto lin_w = kokoro_read(model, "a.gen.dec.generator.m_source.l_linear.weight", KOKORO_N_HARM);
const auto lin_b = kokoro_read(model, "a.gen.dec.generator.m_source.l_linear.bias", 1);
// SineGen: the phase adds up at the F0 rate, then it is interpolated to the sample rate
// the reference also adds a random start phase, but its own downsampling drops it
std::vector<float> phase((size_t) n_f0 * KOKORO_N_HARM);
for (int h = 0; h < KOKORO_N_HARM; h++) {
float cum = 0.0f;
for (int i = 0; i < n_f0; i++) {
float rad = f0[i] * (float) (h + 1) / KOKORO_SR;
rad -= floorf(rad);
cum += rad;
phase[(size_t) i * KOKORO_N_HARM + h] = cum * 2.0f * (float) M_PI * KOKORO_UPS;
}
}
std::normal_distribution<float> gauss(0.0f, 1.0f);
std::vector<float> src((size_t) n_wav);
for (int64_t t = 0; t < n_wav; t++) {
const float pos = std::max(0.0f, ((float) t + 0.5f) / KOKORO_UPS - 0.5f);
const int i0 = std::min((int) pos, n_f0 - 1);
const int i1 = std::min(i0 + 1, n_f0 - 1);
const float frac = pos - (float) i0;
const float uv = f0[t / KOKORO_UPS] > 10.0f ? 1.0f : 0.0f; // voiced threshold
const float namp = uv * 0.003f + (1.0f - uv) * 0.1f / 3.0f;
float merged = lin_b[0];
for (int h = 0; h < KOKORO_N_HARM; h++) {
const float ph = (1.0f - frac) * phase[(size_t) i0 * KOKORO_N_HARM + h] + frac * phase[(size_t) i1 * KOKORO_N_HARM + h];
merged += lin_w[h] * (0.1f * sinf(ph) * uv + namp * gauss(rng));
}
src[(size_t) t] = tanhf(merged);
}
// STFT: periodic hann window, centered with reflect padding
const int n_bins = KOKORO_N_FFT / 2 + 1;
const int n_har = (int) (n_wav / KOKORO_HOP) + 1;
float win[KOKORO_N_FFT], tab_cos[KOKORO_N_FFT], tab_sin[KOKORO_N_FFT];
for (int i = 0; i < KOKORO_N_FFT; i++) {
win[i] = 0.5f - 0.5f * cosf(2.0f * (float) M_PI * i / KOKORO_N_FFT);
tab_cos[i] = cosf(2.0f * (float) M_PI * i / KOKORO_N_FFT);
tab_sin[i] = sinf(2.0f * (float) M_PI * i / KOKORO_N_FFT);
}
har.resize((size_t) n_har * 2 * n_bins);
for (int fr = 0; fr < n_har; fr++) {
float frame[KOKORO_N_FFT];
for (int i = 0; i < KOKORO_N_FFT; i++) {
int64_t idx = (int64_t) fr * KOKORO_HOP - KOKORO_N_FFT / 2 + i;
if (idx < 0) {
idx = -idx;
}
if (idx >= n_wav) {
idx = 2 * (n_wav - 1) - idx;
}
frame[i] = src[(size_t) idx] * win[i];
}
float * out = har.data() + (size_t) fr * 2 * n_bins;
for (int k = 0; k < n_bins; k++) {
float re = 0.0f, im = 0.0f;
for (int i = 0; i < KOKORO_N_FFT; i++) {
const int a = (k * i) % KOKORO_N_FFT;
re += frame[i] * tab_cos[a];
im -= frame[i] * tab_sin[a];
}
out[k] = sqrtf(re * re + im * im);
out[n_bins + k] = atan2f(im, re);
}
}
return n_har;
}
// conv_post rows [n_frames, 22] -> waveform: magnitude exp(x), phase sin(x), hann overlap-add
void mtmd_graph_kokoro::istft(const std::vector<float> & spec, std::vector<float> & audio) {
const int n_bins = KOKORO_N_FFT / 2 + 1;
const int n_frames = (int) (spec.size() / (2 * n_bins));
const int64_t n_out = (int64_t) (n_frames - 1) * KOKORO_HOP;
float win[KOKORO_N_FFT], tab_cos[KOKORO_N_FFT], tab_sin[KOKORO_N_FFT];
for (int i = 0; i < KOKORO_N_FFT; i++) {
win[i] = 0.5f - 0.5f * cosf(2.0f * (float) M_PI * i / KOKORO_N_FFT);
tab_cos[i] = cosf(2.0f * (float) M_PI * i / KOKORO_N_FFT);
tab_sin[i] = sinf(2.0f * (float) M_PI * i / KOKORO_N_FFT);
}
std::vector<float> acc((size_t) n_out + KOKORO_N_FFT, 0.0f);
std::vector<float> wsum((size_t) n_out + KOKORO_N_FFT, 0.0f);
for (int fr = 0; fr < n_frames; fr++) {
const float * row = spec.data() + (size_t) fr * 2 * n_bins;
float re[KOKORO_N_FFT / 2 + 1], im[KOKORO_N_FFT / 2 + 1];
for (int k = 0; k < n_bins; k++) {
const float mag = expf(row[k]);
const float ph = sinf(row[n_bins + k]);
re[k] = mag * cosf(ph);
im[k] = mag * sinf(ph);
}
for (int i = 0; i < KOKORO_N_FFT; i++) {
float v = 0.0f;
for (int k = 0; k < n_bins; k++) {
const int a = (k * i) % KOKORO_N_FFT;
const float w = (k == 0 || k == KOKORO_N_FFT / 2) ? 1.0f : 2.0f;
v += w * (re[k] * tab_cos[a] - im[k] * tab_sin[a]);
}
const size_t o = (size_t) fr * KOKORO_HOP + i;
acc[o] += v / KOKORO_N_FFT * win[i];
wsum[o] += win[i] * win[i];
}
}
audio.resize((size_t) std::max<int64_t>(n_out, 0));
for (size_t i = 0; i < audio.size(); i++) {
const size_t j = i + KOKORO_N_FFT / 2; // drop the centering pad
audio[i] = wsum[j] > 1e-11f ? acc[j] / wsum[j] : 0.0f;
}
}