Files
llama.cpp/tools/mtmd/models/snake-vae.cpp
T
Xuan-Son Nguyen d92ea9bb12 mtmd: support Minimax-Music3 (#32435)
* mtmd: support Minimax-Music3

* fix and clean up
2026-11-17 12:53:19 +01:00

65 lines
3.0 KiB
C++

#include "models.h"
// SnakeBeta, alpha and beta folded at conversion: x + beta * sin(alpha * x)^2
// x: [T, C], alpha/beta: [C]
ggml_tensor * mtmd_graph_snake_vae::snake(ggml_tensor * x, const mtmd_snake_vae::snake & s) const {
ggml_tensor * a = ggml_reshape_2d(ctx0, s.alpha, 1, s.alpha->ne[0]);
ggml_tensor * b = ggml_reshape_2d(ctx0, s.beta, 1, s.beta->ne[0]);
// expand reshapes first so mul/sin/sqr/mul/add lands as consecutive nodes, letting backends fuse them
ggml_build_forward_expand(gf, a);
ggml_build_forward_expand(gf, b);
ggml_tensor * y = ggml_sqr(ctx0, ggml_sin(ctx0, ggml_mul(ctx0, x, a)));
return ggml_add(ctx0, x, ggml_mul(ctx0, y, b));
}
// same-length conv, stride 1; x: [T, IC], w: [K, IC, OC], returns [T, OC]
// im2col in F32: the reference decoders run in FP32
ggml_tensor * mtmd_graph_snake_vae::conv1d(ggml_tensor * x, ggml_tensor * w, ggml_tensor * b, int dilation) const {
const int pad = (int) (w->ne[0] - 1) / 2 * dilation;
ggml_tensor * col = ggml_im2col(ctx0, w, x, 1, 0, pad, 0, dilation, 0, false, GGML_TYPE_F32); // [1, T, IC * K]
ggml_tensor * y = ggml_mul_mat(ctx0,
ggml_reshape_2d(ctx0, col, col->ne[0], col->ne[1] * col->ne[2]),
ggml_reshape_2d(ctx0, w, w->ne[0] * w->ne[1], w->ne[2]));
if (b) {
y = ggml_add(ctx0, y, ggml_reshape_2d(ctx0, b, 1, b->ne[0]));
}
return y;
}
ggml_tensor * mtmd_graph_snake_vae::build_snake_vae(ggml_tensor * x, const mtmd_snake_vae & vae) const {
static constexpr int dilations[3] = { 1, 3, 9 };
if (vae.in_proj_w) {
x = conv1d(x, vae.in_proj_w, vae.in_proj_b, 1);
}
x = conv1d(x, vae.in_w, vae.in_b, 1);
for (size_t ib = 0; ib < vae.blocks.size(); ib++) {
const auto & blk = vae.blocks[ib];
const int stride = (int) (blk.up_w->ne[0] / 2);
const int pad = (stride + 1) / 2; // ceil(stride / 2) of the reference
x = snake(x, blk.act);
{
// ConvTranspose1d as GEMM + col2im, same as qwen3tts; up_w: [K, OC, IC]
const int64_t K = blk.up_w->ne[0];
const int64_t OC = blk.up_w->ne[1];
ggml_tensor * w2 = ggml_cont(ctx0, ggml_transpose(ctx0, ggml_reshape_2d(ctx0, blk.up_w, K * OC, blk.up_w->ne[2])));
ggml_tensor * col = ggml_mul_mat(ctx0, w2, ggml_cont(ctx0, ggml_transpose(ctx0, x))); // [K * OC, T]
x = ggml_col2im_1d(ctx0, col, stride, (int) OC, pad);
}
x = ggml_add(ctx0, x, ggml_reshape_2d(ctx0, blk.up_b, 1, blk.up_b->ne[0]));
for (size_t ir = 0; ir < blk.res.size(); ir++) {
const auto & ru = blk.res[ir];
ggml_tensor * h = snake(x, ru.act1);
h = conv1d(h, ru.conv1_w, ru.conv1_b, dilations[ir]);
h = snake(h, ru.act2);
h = conv1d(h, ru.conv2_w, ru.conv2_b, 1);
x = ggml_add(ctx0, x, h);
}
cb(x, "vae_block", (int) ib);
}
x = snake(x, vae.post);
return conv1d(x, vae.out_w, vae.out_b, 1);
}