225 lines
10 KiB
C++
225 lines
10 KiB
C++
#include "models.h"
|
|
|
|
#include <cmath>
|
|
|
|
// latent frames of a window of n_frames, see MiniMaxMusic3ConditionEncoder of the reference
|
|
int64_t mtmd_graph_music3_gen::latent_frames(int64_t n_frames) {
|
|
const double n = (double) n_frames * 44100.0 / 24000.0 * 960.0 / 512.0;
|
|
return std::max<int64_t>(1, (int64_t) n);
|
|
}
|
|
|
|
// one frame of the depth decoder (RVQDepthDecoder): row 0 is the backbone hidden, row p > 0 the embedding of code p - 1
|
|
// each step reruns the 4 layers over the rows so far (at most 8, no KV cache), like the reference
|
|
ggml_cgraph * mtmd_graph_music3_gen::build_depth() {
|
|
const int64_t n_cb = model.gen_code_head_w->ne[2]; // residual codebooks
|
|
const int64_t n_voc = model.gen_code_head_w->ne[1];
|
|
const int64_t B = n_rows; // cond, then uncond with guidance
|
|
|
|
ggml_tensor * inp_h = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_embd, B);
|
|
ggml_set_name(inp_h, "inp_feats");
|
|
ggml_set_input(inp_h);
|
|
|
|
ggml_tensor * code0 = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, 1);
|
|
ggml_set_name(code0, "inp_code0");
|
|
ggml_set_input(code0);
|
|
|
|
ggml_tensor * inp_rand = ggml_new_tensor_1d(ctx0, GGML_TYPE_F32, n_cb);
|
|
ggml_set_name(inp_rand, "inp_rand");
|
|
ggml_set_input(inp_rand);
|
|
|
|
mtmd_graph_qwen3tts_gen::code_gen cg(*this, top_k, 1.0f);
|
|
|
|
// [n_embd, 1, B], the codes are the same in all rows
|
|
auto project = [&](ggml_tensor * x) {
|
|
ggml_tensor * y = build_mm(model.gen_code_proj_in_w, x);
|
|
y = ggml_reshape_3d(ctx0, y, n_embd, 1, y->ne[1]);
|
|
return y->ne[2] == B ? y : ggml_repeat_4d(ctx0, y, n_embd, 1, B, 1);
|
|
};
|
|
|
|
ggml_tensor * e0 = ggml_get_rows(ctx0, model.gen_code_out_embd_w, code0);
|
|
ggml_tensor * seq = ggml_concat(ctx0, project(inp_h), project(e0), 1);
|
|
|
|
ggml_tensor * out_codes = code0;
|
|
ggml_tensor * hiddens = ggml_view_2d(ctx0, inp_h, n_embd, 1, inp_h->nb[1], 0); // cond row of each step
|
|
ggml_tensor * embd_sum = e0;
|
|
|
|
for (int64_t g = 0; g < n_cb; g++) {
|
|
const int64_t n_pos = g + 2;
|
|
|
|
ggml_tensor * ones = ggml_fill(ctx0, ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_pos, n_pos), 1.0f);
|
|
build_vit_opts opts;
|
|
opts.attn_mask = ggml_log(ctx0, ggml_tri(ctx0, ones, GGML_TRI_TYPE_LOWER_DIAG)); // 0 = keep, -inf = masked
|
|
if (flash_attn_type == MTMD_FLASH_ATTN_TYPE_ENABLED) {
|
|
opts.attn_mask = ggml_cast(ctx0, opts.attn_mask, GGML_TYPE_F16);
|
|
}
|
|
ggml_tensor * pos = ggml_view_2d(ctx0, model.position_embeddings, n_embd, n_pos, model.position_embeddings->nb[1], 0);
|
|
|
|
ggml_tensor * out = build_vit(seq, n_pos, NORM_TYPE_RMS, FFN_SILU, pos, nullptr, opts); // [n_embd, n_pos, B]
|
|
ggml_tensor * last = ggml_cont(ctx0, ggml_view_2d(ctx0, out, n_embd, B, out->nb[2], (n_pos - 1) * out->nb[1]));
|
|
hiddens = ggml_concat(ctx0, hiddens, ggml_view_2d(ctx0, last, n_embd, 1, last->nb[1], 0), 1);
|
|
|
|
ggml_tensor * head = ggml_view_2d(ctx0, model.gen_code_head_w, n_embd, n_voc, model.gen_code_head_w->nb[1], g * model.gen_code_head_w->nb[2]);
|
|
ggml_tensor * logits = ggml_mul_mat(ctx0, head, last); // [n_voc, B]
|
|
if (B > 1) {
|
|
ggml_tensor * cond = ggml_view_1d(ctx0, logits, n_voc, 0);
|
|
ggml_tensor * uncond = ggml_view_1d(ctx0, logits, n_voc, logits->nb[1]);
|
|
logits = ggml_add(ctx0, uncond, ggml_scale(ctx0, ggml_sub(ctx0, cond, uncond), cfg_scale));
|
|
}
|
|
|
|
ggml_tensor * code;
|
|
if (temp > 0.0f) {
|
|
ggml_tensor * rnd = ggml_view_1d(ctx0, inp_rand, 1, g * inp_rand->nb[0]);
|
|
code = cg.do_sampling(ggml_scale(ctx0, logits, 1.0f / temp), rnd);
|
|
} else {
|
|
code = ggml_argmax(ctx0, logits);
|
|
}
|
|
code = ggml_reshape_1d(ctx0, code, 1);
|
|
out_codes = ggml_concat(ctx0, out_codes, code, 0);
|
|
|
|
ggml_tensor * table = ggml_view_2d(ctx0, model.gen_code_embd_w, n_embd, n_voc, model.gen_code_embd_w->nb[1], g * model.gen_code_embd_w->nb[2]);
|
|
ggml_tensor * e = ggml_get_rows(ctx0, table, code);
|
|
embd_sum = ggml_add(ctx0, embd_sum, e);
|
|
if (g + 1 < n_cb) {
|
|
seq = ggml_concat(ctx0, seq, project(e), 1);
|
|
}
|
|
}
|
|
|
|
ggml_set_name(out_codes, "out_codes");
|
|
ggml_set_output(out_codes);
|
|
ggml_build_forward_expand(gf, out_codes);
|
|
|
|
// condition row of the frame: the 8 hidden states mixed by the condition encoder weights
|
|
ggml_tensor * cond_row = ggml_mul_mat(ctx0, ggml_cont(ctx0, ggml_transpose(ctx0, hiddens)), model.music3.cond_layer_w);
|
|
ggml_set_name(cond_row, "out_feats");
|
|
ggml_set_output(cond_row);
|
|
ggml_build_forward_expand(gf, cond_row);
|
|
|
|
// backbone input of the next frame: (E_lm[semantic] + sum of E_cb[g][code g]) / sqrt(n_cb + 1), must be the last node
|
|
ggml_tensor * out_embd = ggml_scale(ctx0, embd_sum, 1.0f / sqrtf((float) (n_cb + 1)));
|
|
cb(out_embd, "gen_audio_out", -1);
|
|
ggml_build_forward_expand(gf, out_embd);
|
|
return gf;
|
|
}
|
|
|
|
// frame rows [n_embd, n_rows] -> conv k = 3 -> nearest resampling to n_out latent frames
|
|
ggml_cgraph * mtmd_graph_music3_gen::build_cond() {
|
|
const auto & m = model.music3;
|
|
|
|
ggml_tensor * inp = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_embd, n_rows);
|
|
ggml_set_name(inp, "inp_feats");
|
|
ggml_set_input(inp);
|
|
|
|
ggml_tensor * idx = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_out);
|
|
ggml_set_name(idx, "inp_idx");
|
|
ggml_set_input(idx);
|
|
|
|
ggml_tensor * x = ggml_cont(ctx0, ggml_transpose(ctx0, inp)); // [T, n_embd]
|
|
x = conv1d(x, m.cond_w, m.cond_b, 1);
|
|
x = ggml_cont(ctx0, ggml_transpose(ctx0, x)); // [n_cond, T]
|
|
x = ggml_get_rows(ctx0, x, idx);
|
|
ggml_set_name(x, "out_feats");
|
|
ggml_set_output(x);
|
|
ggml_build_forward_expand(gf, x);
|
|
return gf;
|
|
}
|
|
|
|
// one velocity pass of the DiT over a window, the cond and uncond (zero condition) inputs as a batch of 2
|
|
// the input rows are [latent, zeros, condition], the time embedding is prepended as one extra token
|
|
ggml_cgraph * mtmd_graph_music3_gen::build_dit() {
|
|
const auto & m = model.music3;
|
|
const int64_t n_in = m.pre_w->ne[0];
|
|
const int64_t n_dit = m.proj_in_w->ne[1];
|
|
const int64_t n_head = n_dit / dit_d_head;
|
|
const int64_t n_pos = n_rows + 1;
|
|
const int64_t B = 2;
|
|
|
|
ggml_tensor * inp = ggml_new_tensor_3d(ctx0, GGML_TYPE_F32, n_in, n_rows, B);
|
|
ggml_set_name(inp, "inp_feats");
|
|
ggml_set_input(inp);
|
|
|
|
ggml_tensor * inp_t = ggml_new_tensor_1d(ctx0, GGML_TYPE_F32, 1);
|
|
ggml_set_name(inp_t, "inp_t");
|
|
ggml_set_input(inp_t);
|
|
|
|
ggml_tensor * x = ggml_add(ctx0, inp, build_mm(m.pre_w, inp));
|
|
x = build_mm(m.proj_in_w, x); // [n_dit, n_rows, B]
|
|
|
|
// time token: Fourier features -> linear -> silu -> linear
|
|
ggml_tensor * ang = ggml_mul(ctx0, m.time_freq, inp_t);
|
|
ggml_tensor * temb = ggml_concat(ctx0, ggml_cos(ctx0, ang), ggml_sin(ctx0, ang), 0);
|
|
temb = ggml_silu(ctx0, ggml_add(ctx0, build_mm(m.time_in_w, temb), m.time_in_b));
|
|
temb = ggml_add(ctx0, build_mm(m.time_out_w, temb), m.time_out_b);
|
|
temb = ggml_repeat_4d(ctx0, ggml_reshape_3d(ctx0, temb, n_dit, 1, 1), n_dit, 1, B, 1);
|
|
x = ggml_concat(ctx0, temb, x, 1); // [n_dit, n_pos, B]
|
|
x = ggml_reshape_2d(ctx0, x, n_dit, n_pos * B);
|
|
|
|
ggml_tensor * pos = ggml_cast(ctx0, ggml_arange(ctx0, 0.0f, (float) n_pos, 1.0f), GGML_TYPE_I32);
|
|
const float kq = 1.0f / sqrtf((float) dit_d_head);
|
|
|
|
for (size_t il = 0; il < m.dit.size(); il++) {
|
|
const auto & layer = m.dit[il];
|
|
|
|
ggml_tensor * h = build_norm(x, layer.ln_1_w, layer.ln_1_b, NORM_TYPE_NORMAL, dit_eps, (int) il);
|
|
ggml_tensor * q = ggml_reshape_4d(ctx0, build_mm(layer.q_w, h), dit_d_head, n_head, n_pos, B);
|
|
ggml_tensor * k = ggml_reshape_4d(ctx0, build_mm(layer.k_w, h), dit_d_head, n_head, n_pos, B);
|
|
ggml_tensor * v = ggml_reshape_4d(ctx0, build_mm(layer.v_w, h), dit_d_head, n_head, n_pos, B);
|
|
|
|
// only the first dit_n_rot dims of each head rotate
|
|
q = ggml_rope_ext(ctx0, q, pos, nullptr, dit_n_rot, GGML_ROPE_TYPE_NEOX, 0, dit_theta, 1.0f, 0.0f, 1.0f, 0.0f, 0.0f);
|
|
k = ggml_rope_ext(ctx0, k, pos, nullptr, dit_n_rot, GGML_ROPE_TYPE_NEOX, 0, dit_theta, 1.0f, 0.0f, 1.0f, 0.0f, 0.0f);
|
|
|
|
x = ggml_add(ctx0, x, build_attn(layer.o_w, nullptr, q, k, v, nullptr, kq, (int) il));
|
|
|
|
h = build_norm(x, layer.ln_2_w, layer.ln_2_b, NORM_TYPE_NORMAL, dit_eps, (int) il);
|
|
h = build_ffn(h, layer.ff_up_w, layer.ff_up_b, layer.ff_gate_w, layer.ff_gate_b, layer.ff_down_w, layer.ff_down_b, FFN_SILU, (int) il);
|
|
x = ggml_add(ctx0, x, h);
|
|
}
|
|
|
|
// drop the time token
|
|
x = ggml_reshape_3d(ctx0, x, n_dit, n_pos, B);
|
|
x = ggml_view_3d(ctx0, x, n_dit, n_rows, B, x->nb[1], x->nb[2], x->nb[1]);
|
|
x = build_mm(m.proj_out_w, ggml_cont(ctx0, x));
|
|
x = ggml_add(ctx0, x, build_mm(m.post_w, x)); // [n_latent, n_rows, B]
|
|
|
|
// classifier-free guidance
|
|
ggml_tensor * cond = ggml_view_2d(ctx0, x, n_latent, n_rows, x->nb[1], 0);
|
|
ggml_tensor * uncond = ggml_view_2d(ctx0, x, n_latent, n_rows, x->nb[1], x->nb[2]);
|
|
x = ggml_add(ctx0, uncond, ggml_scale(ctx0, ggml_sub(ctx0, cond, uncond), cfg_scale));
|
|
ggml_set_name(x, "out_feats");
|
|
ggml_set_output(x);
|
|
ggml_build_forward_expand(gf, x);
|
|
return gf;
|
|
}
|
|
|
|
// the two channels are decoded as two mono streams of n_latent / 2 channels, then tanh
|
|
ggml_cgraph * mtmd_graph_music3_gen::build_vocoder() {
|
|
ggml_tensor * inp = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_latent, n_rows);
|
|
ggml_set_name(inp, "inp_feats");
|
|
ggml_set_input(inp);
|
|
|
|
ggml_tensor * x = ggml_cont(ctx0, ggml_transpose(ctx0, inp)); // [T, n_latent]
|
|
const int64_t n_half = n_latent / n_channels;
|
|
ggml_tensor * out = nullptr;
|
|
for (int c = 0; c < n_channels; c++) {
|
|
ggml_tensor * xc = ggml_cont(ctx0, ggml_view_2d(ctx0, x, n_rows, n_half, x->nb[1], c * n_half * x->nb[1]));
|
|
ggml_tensor * y = ggml_tanh(ctx0, build_snake_vae(xc, model.snake_vae)); // [T * ratio, 1]
|
|
out = out ? ggml_concat(ctx0, out, y, 1) : y;
|
|
}
|
|
|
|
out = ggml_cont(ctx0, ggml_transpose(ctx0, out)); // interleaved
|
|
ggml_set_name(out, "out_audio");
|
|
ggml_set_output(out);
|
|
ggml_build_forward_expand(gf, out);
|
|
return gf;
|
|
}
|
|
|
|
ggml_cgraph * mtmd_graph_music3_gen::build() {
|
|
switch (gen_process) {
|
|
case MTMD_SUBCTX_GEN_PROCESS_GEN_CODE: return build_depth();
|
|
case MTMD_SUBCTX_GEN_PROCESS_FLOW_COND: return build_cond();
|
|
case MTMD_SUBCTX_GEN_PROCESS_FLOW_STEP: return build_dit();
|
|
case MTMD_SUBCTX_GEN_PROCESS_GEN_WAV: return build_vocoder();
|
|
default: GGML_ABORT("minimax_music3: unsupported gen process");
|
|
}
|
|
}
|