#include "models.h" #include "../mtmd-audio.h" #include #include #include #include #include // same-length conv with 3 taps; x: [T, IC], w: [3, IC, OC], returns [T, OC] ggml_tensor * mtmd_graph_tiny_voca::conv3(ggml_tensor * x, ggml_tensor * w, ggml_tensor * b) const { ggml_tensor * col = ggml_im2col(ctx0, w, x, 1, 0, 1, 0, 1, 0, false, GGML_TYPE_F32); ggml_tensor * y = ggml_mul_mat(ctx0, ggml_reshape_2d(ctx0, col, col->ne[0], col->ne[1] * col->ne[2]), ggml_reshape_2d(ctx0, w, w->ne[0] * w->ne[1], w->ne[2])); return ggml_add(ctx0, y, ggml_reshape_2d(ctx0, b, 1, b->ne[0])); } ggml_cgraph * mtmd_graph_tiny_voca::build() { const auto & tv = model.tiny_voca; ggml_tensor * inp = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, hparams.n_embd, n_rows); ggml_set_name(inp, "inp_feats"); ggml_set_input(inp); ggml_tensor * cur; if (gen_process == MTMD_SUBCTX_GEN_PROCESS_LATENT_IN) { // duration predictor: (conv, gelu, layer norm) x 2, then a linear layer cur = ggml_cont(ctx0, ggml_transpose(ctx0, inp)); cur = conv3(cur, tv.dur_conv1_w, tv.dur_conv1_b); cur = ggml_gelu_erf(ctx0, ggml_cont(ctx0, ggml_transpose(ctx0, cur))); cur = build_norm(cur, tv.dur_norm1_w, tv.dur_norm1_b, NORM_TYPE_NORMAL, eps, -1); cur = conv3(ggml_cont(ctx0, ggml_transpose(ctx0, cur)), tv.dur_conv2_w, tv.dur_conv2_b); cur = ggml_gelu_erf(ctx0, ggml_cont(ctx0, ggml_transpose(ctx0, cur))); cur = build_norm(cur, tv.dur_norm2_w, tv.dur_norm2_b, NORM_TYPE_NORMAL, eps, -1); cur = ggml_add(ctx0, build_mm(tv.dur_out_w, cur), tv.dur_out_b); } else { GGML_ASSERT(gen_process == MTMD_SUBCTX_GEN_PROCESS_LATENT_OUT); ggml_tensor * h = ggml_add(ctx0, build_mm(tv.harm_w, inp), tv.harm_b); ggml_tensor * n = ggml_add(ctx0, build_mm(tv.noise_w, inp), tv.noise_b); cur = ggml_concat(ctx0, h, n, 0); } ggml_set_name(cur, "out_feats"); ggml_set_output(cur); ggml_build_forward_expand(gf, cur); return gf; } // // host: harmonic plus noise synthesis, see SourceFilter in the reference (model.py) // // band-limited pulse train with unit power, F0 given per frame and interpolated in the log domain static void tiny_voca_pulse_train(const std::vector & f0, std::vector & out) { using g = mtmd_graph_tiny_voca; const int64_t n_frames = (int64_t) f0.size(); std::vector knots(n_frames + 1); for (int64_t i = 0; i < n_frames; i++) { knots[i] = std::log(std::max((double) f0[i], 1.0)); } knots[n_frames] = knots[n_frames - 1]; out.resize((size_t) (n_frames * g::hop)); double acc = 0.0; for (int64_t n = 0; n < n_frames * g::hop; n++) { const int64_t low = n / g::hop; const double w = (double) (n - low * g::hop) / g::hop; const double freq = std::exp(knots[low] * (1.0 - w) + knots[low + 1] * w); acc += freq / g::sample_rate; const double theta = std::fmod(2.0 * M_PI * acc, 2.0 * M_PI); const double harmonics = std::max((double) g::harmonic_limit * g::sample_rate / freq, 1.0); const double whole = std::floor(harmonics); const double denom = 2.0 * std::sin(theta / 2.0); const double core = std::fabs(denom) < 1e-7 ? whole : std::sin((whole + 0.5) * theta) / denom - 0.5; const double wave = core + (harmonics - whole) * std::cos((whole + 1.0) * theta); out[n] = (float) (wave * std::sqrt(2.0 / harmonics)); } } // time-varying filter: STFT of the source (center, zero padding), times exp(rfft(cepstrum)) of each frame, ISTFT // cep(j, buf) writes the n_fft cepstrum of STFT frame j, the source gets n_fft / 2 trailing zeros first static void tiny_voca_filter(const std::vector & src, int n_fft, int hop, const std::function & cep, std::vector & out) { const int n_bins = n_fft / 2 + 1; const int64_t n_src = (int64_t) src.size(); const int64_t n_pad = n_src + n_fft / 2; // length given to the STFT const int64_t n_stft = 1 + n_pad / hop; // frames, center = true const int64_t n_full = n_pad + n_fft; // with the center padding mtmd_audio_cache cache; cache.fill_sin_cos_table(n_fft); cache.fill_hann_window(n_fft, true); const float * win = cache.hann_window.data(); std::vector x((size_t) n_full, 0.0f); std::copy(src.begin(), src.end(), x.begin() + n_fft / 2); std::vector ola((size_t) n_full, 0.0f), env((size_t) n_full, 0.0f); std::vector fin(8 * n_fft), fout(8 * n_fft), rin(8 * n_fft), rout(8 * n_fft), cbuf(n_fft); std::vector resp(2 * n_bins); for (int64_t j = 0; j < n_stft; j++) { // response of this frame std::fill(cbuf.begin(), cbuf.end(), 0.0f); cep(j, cbuf.data()); std::copy(cbuf.begin(), cbuf.end(), rin.begin()); mtmd_audio_fft(cache, rin.data(), n_fft, rout.data()); for (int k = 0; k < n_bins; k++) { const float mag = std::exp(std::clamp(rout[2 * k], -14.0f, 6.0f)); resp[2 * k + 0] = mag * std::cos(rout[2 * k + 1]); resp[2 * k + 1] = mag * std::sin(rout[2 * k + 1]); } // spectrum of the source frame const float * seg = x.data() + j * hop; for (int i = 0; i < n_fft; i++) { fin[i] = seg[i] * win[i]; } mtmd_audio_fft(cache, fin.data(), n_fft, fout.data()); // filtered spectrum, mirrored for the complex inverse FFT; irfft ignores the imaginary parts of DC and Nyquist for (int k = 0; k < n_bins; k++) { const float a = fout[2 * k], b = fout[2 * k + 1]; const float c = resp[2 * k], d = resp[2 * k + 1]; rin[2 * k + 0] = a * c - b * d; rin[2 * k + 1] = (k == 0 || k == n_fft / 2) ? 0.0f : a * d + b * c; } for (int k = 1; k < n_fft - n_bins + 1; k++) { rin[2 * (n_fft - k) + 0] = rin[2 * k + 0]; rin[2 * (n_fft - k) + 1] = -rin[2 * k + 1]; } mtmd_audio_ifft(cache, rin.data(), n_fft, rout.data()); float * o = ola.data() + j * hop; float * e = env.data() + j * hop; for (int i = 0; i < n_fft; i++) { o[i] += rout[2 * i] * win[i]; e[i] += win[i] * win[i]; } } out.resize((size_t) n_src); for (int64_t i = 0; i < n_src; i++) { const int64_t k = i + n_fft / 2; // drop the center padding out[i] = env[k] > 1e-11f ? ola[k] / env[k] : 0.0f; } } void mtmd_graph_tiny_voca::synth(const std::vector & voc, const std::vector & f0, std::mt19937 & rng, std::vector & audio) { const int64_t n_frames = (int64_t) f0.size(); GGML_ASSERT(n_frames > 0 && voc.size() == (size_t) (n_frames * n_voc)); // liftering: coefficient i scaled by 1 / (1 + i / 4), the gain (coefficient 0) by 2 auto lifter = [](int i, int first) { return i == 0 && first == 0 ? 2.0f : 1.0f / (1.0f + (float) (i + first) / 4.0f); }; // harmonic branch: envelope interpolated between frames, one gain per sub-frame const int split = n_cep_causal + n_cep_anti; auto harm_cep = [&](int64_t j, float * buf) { const int64_t s = std::min(j, harm_steps * n_frames - 1); const float * here = voc.data() + (s / harm_steps) * n_voc; const float * next = voc.data() + std::min((s + 1) / harm_steps, n_frames - 1) * n_voc; for (int i = 0; i < n_cep_causal; i++) { buf[i] = 0.5f * (here[i] + next[i]) * lifter(i, 0); } buf[0] += here[split + s % harm_steps]; for (int i = 0; i < n_cep_anti; i++) { buf[harm_fft - 1 - i] = 0.5f * (here[n_cep_causal + i] + next[n_cep_causal + i]) * lifter(i, 1); } }; // noise branch: one envelope per frame, one gain per sub-frame auto noise_cep = [&](int64_t j, float * buf) { const int64_t s = std::min(j, noise_steps * n_frames - 1); const float * row = voc.data() + (s / noise_steps) * n_voc + split + harm_steps; for (int i = 0; i < n_noise_cep; i++) { buf[i] = row[i] * lifter(i, 0); } buf[0] += row[n_noise_cep + s % noise_steps]; }; std::vector pulses, noise((size_t) (n_frames * hop)), harm_out, noise_out; tiny_voca_pulse_train(f0, pulses); std::normal_distribution dist(0.0f, 1.0f); for (auto & v : noise) { v = dist(rng); } tiny_voca_filter(pulses, harm_fft, harm_hop, harm_cep, harm_out); tiny_voca_filter(noise, noise_fft, noise_hop, noise_cep, noise_out); audio.resize(pulses.size()); for (size_t i = 0; i < audio.size(); i++) { audio[i] = harm_out[i] + noise_out[i]; } } // // host: unknown-word model of the phonemizer, see g2p.py of dizione // static const char * const tiny_voca_g2p_letters = "abcdefghijklmnopqrstuvwxyz'"; static const char * const tiny_voca_g2p_symbols[mtmd_graph_tiny_voca::n_g2p_symbol] = { "AA0", "AA1", "AA2", "AE0", "AE1", "AE2", "AH0", "AH1", "AH2", "AO0", "AO1", "AO2", "AW0", "AW1", "AW2", "AY0", "AY1", "AY2", "B", "CH", "D", "DH", "EH0", "EH1", "EH2", "ER0", "ER1", "ER2", "EY0", "EY1", "EY2", "F", "G", "HH", "IH0", "IH1", "IH2", "IY0", "IY1", "IY2", "JH", "K", "L", "M", "N", "NG", "OW0", "OW1", "OW2", "OY0", "OY1", "OY2", "P", "R", "S", "SH", "T", "TH", "UH0", "UH1", "UH2", "UW0", "UW1", "UW2", "V", "W", "Y", "Z", "ZH", }; // one direction of a GRU layer over x [n, n_in] -> y [n, n_hidden], written at column offset of rows of width stride // the products are summed in double and rounded once, as close as can be to the reference (numpy, float32): two // symbols are sometimes almost as likely, and the order of a float32 sum would decide between them static void tiny_voca_gru(const mtmd_tiny_voca::g2p_gru & g, const std::vector & x, int64_t n, int64_t n_in, int64_t n_hidden, bool reverse, std::vector & y, int64_t offset, int64_t stride) { const int64_t h3 = 3 * n_hidden; std::vector inp((size_t) (n * h3)); for (int64_t t = 0; t < n; t++) { for (int64_t r = 0; r < h3; r++) { const float * w = g.w_ih.data() + r * n_in; const float * xt = x.data() + t * n_in; double acc = 0.0; for (int64_t c = 0; c < n_in; c++) { acc += (double) w[c] * xt[c]; } inp[t * h3 + r] = (float) acc + g.b_ih[r]; } } std::vector state((size_t) n_hidden, 0.0f), rec((size_t) h3); for (int64_t s = 0; s < n; s++) { const int64_t t = reverse ? n - 1 - s : s; for (int64_t r = 0; r < h3; r++) { const float * w = g.w_hh.data() + r * n_hidden; double acc = 0.0; for (int64_t c = 0; c < n_hidden; c++) { acc += (double) w[c] * state[c]; } rec[r] = (float) acc + g.b_hh[r]; } const float * in = inp.data() + t * h3; for (int64_t k = 0; k < n_hidden; k++) { const float reset = 1.0f / (1.0f + std::exp(-(in[k] + rec[k]))); const float update = 1.0f / (1.0f + std::exp(-(in[n_hidden + k] + rec[n_hidden + k]))); const float cand = std::tanh(in[2 * n_hidden + k] + reset * rec[2 * n_hidden + k]); state[k] = (1.0f - update) * cand + update * state[k]; } std::copy(state.begin(), state.end(), y.begin() + t * stride + offset); } } std::string mtmd_graph_tiny_voca::g2p(const mtmd_tiny_voca & tv, const std::string & word) { const int64_t w = tv.g2p_n_width; const int64_t h = tv.g2p_n_hidden; if (w == 0 || tv.g2p_layers.empty()) { return ""; } std::vector ids; for (char c : word) { const char * p = strchr(tiny_voca_g2p_letters, std::tolower((unsigned char) c)); if (c != '\0' && p) { ids.push_back((int) (p - tiny_voca_g2p_letters) + 1); } } if (ids.empty()) { return ""; } // each letter takes n_g2p_slot positions: its embedding plus the embedding of the slot const int64_t n = (int64_t) ids.size() * n_g2p_slot; std::vector x((size_t) (n * w)); for (size_t l = 0; l < ids.size(); l++) { for (int s = 0; s < n_g2p_slot; s++) { float * row = x.data() + ((int64_t) l * n_g2p_slot + s) * w; for (int64_t c = 0; c < w; c++) { row[c] = tv.g2p_embd[ids[l] * w + c] + tv.g2p_slot[s * w + c]; } } } int64_t n_in = w; for (size_t il = 0; il + 1 < tv.g2p_layers.size(); il += 2) { std::vector y((size_t) (n * 2 * h)); tiny_voca_gru(tv.g2p_layers[il], x, n, n_in, h, false, y, 0, 2 * h); tiny_voca_gru(tv.g2p_layers[il + 1], x, n, n_in, h, true, y, h, 2 * h); x = std::move(y); n_in = 2 * h; } // CTC, read greedily: a label repeated or the blank (0) is no new symbol std::string out; int prev = 0; for (int64_t t = 0; t < n; t++) { int best = 0; float best_v = -INFINITY; for (int k = 0; k <= n_g2p_symbol; k++) { const float * wr = tv.g2p_out_w.data() + k * n_in; double sum = 0.0; for (int64_t c = 0; c < n_in; c++) { sum += (double) wr[c] * x[t * n_in + c]; } const float acc = (float) sum + tv.g2p_out_b[k]; if (acc > best_v) { best_v = acc; best = k; } } if (best != prev && best != 0) { out += (out.empty() ? "" : " ") + std::string(tiny_voca_g2p_symbols[best - 1]); } prev = best; } return out; }