* mtmd: support MageVL-Cowork model * mtmd: keep magevl rope_freqs at model level and check its shape The rope factors are shared by every layer, so they live in clip_model instead of a pointer copied into each clip_layer. The loader now checks that v.rope_freqs is F32 with d_head elements and that d_head splits into the 4:6:6 pairs the graph expects. --------- Co-authored-by: Pascal <admin@serveurperso.com>
62 lines
2.6 KiB
C++
62 lines
2.6 KiB
C++
#include "models.h"
|
|
|
|
ggml_cgraph * clip_graph_magevl::build() {
|
|
GGML_ASSERT(hparams.n_merge == 2);
|
|
|
|
const int n_pair = d_head / 2;
|
|
const int unit = n_pair / 16;
|
|
|
|
// pairs are grouped as t, h, w at conversion
|
|
int sections[4] = {4 * unit, 6 * unit, 6 * unit, 0};
|
|
|
|
ggml_tensor * positions = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_patches * 4);
|
|
ggml_set_name(positions, "positions");
|
|
ggml_set_input(positions);
|
|
|
|
// a pair uses 2 angles: even outputs come from one rope, odd outputs from the other
|
|
auto add_pos = [&](ggml_tensor * cur, const clip_layer &) {
|
|
ggml_tensor * freqs_a = ggml_view_1d(ctx0, model.rope_freqs, n_pair, 0);
|
|
ggml_tensor * freqs_b = ggml_view_1d(ctx0, model.rope_freqs, n_pair, ggml_row_size(model.rope_freqs->type, n_pair));
|
|
|
|
ggml_tensor * rope_a = ggml_rope_multi(ctx0, cur, positions, freqs_a, d_head, sections,
|
|
GGML_ROPE_TYPE_MROPE, 32768, 1.0f, 1.0f, 0.0f, 1.0f, 32.0f, 1.0f);
|
|
ggml_tensor * rope_b = ggml_rope_multi(ctx0, cur, positions, freqs_b, d_head, sections,
|
|
GGML_ROPE_TYPE_MROPE, 32768, 1.0f, 1.0f, 0.0f, 1.0f, 32.0f, 1.0f);
|
|
|
|
ggml_tensor * even = ggml_view_3d(ctx0, rope_a, n_pair, n_head, n_patches,
|
|
rope_a->nb[1], rope_a->nb[2], 0);
|
|
ggml_tensor * odd = ggml_view_3d(ctx0, rope_b, n_pair, n_head, n_patches,
|
|
rope_b->nb[1], rope_b->nb[2], ggml_row_size(rope_b->type, n_pair));
|
|
|
|
return ggml_concat(ctx0, even, odd, 0);
|
|
};
|
|
|
|
// ggml_conv_2d rounds patches to F16
|
|
ggml_tensor * inp_raw = build_inp_raw();
|
|
ggml_tensor * patches = ggml_im2col(ctx0, model.patch_embeddings_0, inp_raw,
|
|
patch_size, patch_size, 0, 0, 1, 1, true, GGML_TYPE_F32);
|
|
const int64_t n_patch_dim = patches->ne[0];
|
|
ggml_tensor * inp = ggml_mul_mat(ctx0,
|
|
ggml_reshape_2d(ctx0, model.patch_embeddings_0, n_patch_dim, n_embd),
|
|
ggml_reshape_2d(ctx0, patches, n_patch_dim, n_patches));
|
|
|
|
// reorder into 2x2 blocks
|
|
inp = ggml_reshape_4d(ctx0, inp, n_embd * 2, n_patches_x / 2, 2, n_patches_y / 2);
|
|
inp = ggml_permute(ctx0, inp, 0, 2, 1, 3);
|
|
inp = ggml_cont_3d(ctx0, inp, n_embd, n_patches, 1);
|
|
|
|
ggml_tensor * cur = build_vit(inp, n_patches, NORM_TYPE_NORMAL, hparams.ffn_op, nullptr, add_pos);
|
|
|
|
// multimodal projection
|
|
cur = ggml_reshape_3d(ctx0, cur, n_embd * 4, n_patches / 4, 1);
|
|
cur = build_ffn(cur,
|
|
model.mm_0_w, model.mm_0_b,
|
|
nullptr, nullptr,
|
|
model.mm_1_w, model.mm_1_b,
|
|
hparams.ffn_op, -1);
|
|
|
|
ggml_build_forward_expand(gf, cur);
|
|
|
|
return gf;
|
|
}
|