Lumpiasty/llama.cpp
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

mtmd: use ggml_rope_set_offset (#27521)

Browse Source 
* mtmd: use ggml_rope_set_offset

* add comment
This commit is contained in:
Xuan-Son Nguyen
2026-08-22 16:33:47 +02:00
committed by GitHub
parent d9f918d2d0
commit b21e4de745
4 changed files with 62 additions and 104 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tools/mtmd/clip-graph.h
+9 -3
View File
@@ -137,9 +137,15 @@ struct clip_graph {
int il, int il,
ggml_tensor * sinks = nullptr) const; ggml_tensor * sinks = nullptr) const;
// implementation of the 2D RoPE without adding a new op in ggml // implementation of the 2D RoPE using two ggml_rope_ext calls
// this is not efficient (use double the memory), but works on all backends //
// TODO: there was a more efficient which relies on ggml_view and ggml_rope_ext_inplace, but the rope inplace does not work well with non-contiguous tensors ; we should fix that and revert back to the original implementation in https://github.com/ggml-org/llama.cpp/pull/13065 // unlike GGML_ROPE_TYPE_VISION which forces NEOX ordering, this rotates adjacent pairs (normal ordering)
//
// example:
// given a single head with size = 8 --> [00000000]
// dims [0, 4) rotate with pos_a, dims [4, 8) rotate with pos_b --> [aaaabbbb]
// interleave_freq = false --> both halves use the same inv_freq set (like GGML_ROPE_TYPE_VISION)
// interleave_freq = true --> first half uses even inv_freq, second half uses odd inv_freq (used by pixtral)
ggml_tensor * build_rope_2d( ggml_tensor * build_rope_2d(
ggml_context * ctx0, ggml_context * ctx0,
ggml_tensor * cur, ggml_tensor * cur,
tools/mtmd/clip.cpp
+23 -43
View File
@@ -819,8 +819,6 @@ ggml_tensor * clip_graph::build_attn(
} }
// implementation of the 2D RoPE without adding a new op in ggml // implementation of the 2D RoPE without adding a new op in ggml
// this is not efficient (use double the memory), but works on all backends
// TODO: there was a more efficient which relies on ggml_view and ggml_rope_ext_inplace, but the rope inplace does not work well with non-contiguous tensors ; we should fix that and revert back to the original implementation in https://github.com/ggml-org/llama.cpp/pull/13065
ggml_tensor * clip_graph::build_rope_2d( ggml_tensor * clip_graph::build_rope_2d(
ggml_context * ctx0, ggml_context * ctx0,
ggml_tensor * cur, ggml_tensor * cur,
@@ -829,9 +827,7 @@ ggml_tensor * clip_graph::build_rope_2d(
const float freq_base, const float freq_base,
const bool interleave_freq const bool interleave_freq
) { ) {
const int64_t n_dim = cur->ne[0]; const int64_t n_dim = cur->ne[0];
const int64_t n_head = cur->ne[1];
const int64_t n_pos = cur->ne[2];
// for example, if we have cur tensor of shape (n_dim=8, n_head, n_pos) // for example, if we have cur tensor of shape (n_dim=8, n_head, n_pos)
// we will have a list of 4 inv_freq: 1e-0, 1e-1, 1e-2, 1e-3 // we will have a list of 4 inv_freq: 1e-0, 1e-1, 1e-2, 1e-3
@@ -845,46 +841,30 @@ ggml_tensor * clip_graph::build_rope_2d(
? std::pow(freq_base, (float)-2/n_dim) ? std::pow(freq_base, (float)-2/n_dim)
: 1.0; : 1.0;
// first half // first half, dims [0, n_dim/2)
ggml_tensor * first; cur = ggml_rope_ext(
{ ctx0,
first = ggml_view_3d(ctx0, cur, cur,
n_dim/2, n_head, n_pos, pos_a, // positions
cur->nb[1], nullptr, // freq factors
cur->nb[2], n_dim/2, // n_dims
0); 0, 0, freq_base,
first = ggml_rope_ext( 1.0f, 0.0f, 1.0f, 0.0f, 0.0f
ctx0, );
first,
pos_a, // positions
nullptr, // freq factors
n_dim/2, // n_dims
0, 0, freq_base,
1.0f, 0.0f, 1.0f, 0.0f, 0.0f
);
}
// second half // second half, dims [n_dim/2, n_dim)
ggml_tensor * second; cur = ggml_rope_ext(
{ ctx0,
second = ggml_view_3d(ctx0, cur, cur,
n_dim/2, n_head, n_pos, pos_b, // positions
cur->nb[1], nullptr, // freq factors
cur->nb[2], n_dim/2, // n_dims
n_dim/2 * ggml_element_size(cur)); 0, 0, freq_base,
second = ggml_rope_ext( freq_scale_odd,
ctx0, 0.0f, 1.0f, 0.0f, 0.0f
second, );
pos_b, // positions cur = ggml_rope_set_offset(cur, n_dim/2);
nullptr, // freq factors
n_dim/2, // n_dims
0, 0, freq_base,
freq_scale_odd,
0.0f, 1.0f, 0.0f, 0.0f
);
}
cur = ggml_concat(ctx0, first, second, 0);
return cur; return cur;
} }
tools/mtmd/models/gemma4v.cpp
+22 -42
View File
@@ -44,51 +44,31 @@ ggml_cgraph * clip_graph_gemma4v::build() {
// similar to build_rope_2d, but use neox ordering // similar to build_rope_2d, but use neox ordering
auto add_pos = [&](ggml_tensor * cur, const clip_layer &) { auto add_pos = [&](ggml_tensor * cur, const clip_layer &) {
const int64_t n_dim = cur->ne[0]; const int64_t n_dim = cur->ne[0];
const int64_t n_head = cur->ne[1];
const int64_t n_pos = cur->ne[2];
// first half // first half, dims [0, n_dim/2)
ggml_tensor * first; cur = ggml_rope_ext(
{ ctx0,
first = ggml_view_4d(ctx0, cur, cur,
n_dim/2, n_head, n_pos, n_batch, pos_x, // positions
cur->nb[1], nullptr, // freq factors
cur->nb[2], n_dim/2, // n_dims
cur->nb[3], GGML_ROPE_TYPE_NEOX, 0, hparams.rope_theta,
0); 1.0f, 0.0f, 1.0f, 0.0f, 0.0f
first = ggml_rope_ext( );
ctx0,
first,
pos_x, // positions
nullptr, // freq factors
n_dim/2, // n_dims
GGML_ROPE_TYPE_NEOX, 0, hparams.rope_theta,
1.0f, 0.0f, 1.0f, 0.0f, 0.0f
);
}
// second half // second half, dims [n_dim/2, n_dim)
ggml_tensor * second; cur = ggml_rope_ext(
{ ctx0,
second = ggml_view_4d(ctx0, cur, cur,
n_dim/2, n_head, n_pos, n_batch, pos_y, // positions
cur->nb[1], nullptr, // freq factors
cur->nb[2], n_dim/2, // n_dims
cur->nb[3], GGML_ROPE_TYPE_NEOX, 0, hparams.rope_theta,
n_dim/2 * ggml_element_size(cur)); 1.0f, 0.0f, 1.0f, 0.0f, 0.0f
second = ggml_rope_ext( );
ctx0, cur = ggml_rope_set_offset(cur, n_dim/2);
second,
pos_y, // positions
nullptr, // freq factors
n_dim/2, // n_dims
GGML_ROPE_TYPE_NEOX, 0, hparams.rope_theta,
1.0f, 0.0f, 1.0f, 0.0f, 0.0f
);
}
cur = ggml_concat(ctx0, first, second, 0);
return cur; return cur;
}; };
tools/mtmd/models/minimax-m3.cpp
+8 -16
View File
@@ -2,30 +2,22 @@
ggml_tensor * clip_graph_minimax_m3::apply_rope( ggml_tensor * clip_graph_minimax_m3::apply_rope(
ggml_tensor * x, ggml_tensor * pos_h, ggml_tensor * pos_w) { ggml_tensor * x, ggml_tensor * pos_h, ggml_tensor * pos_w) {
const int64_t Hn = x->ne[1]; const int dh = (int) x->ne[0];
const int64_t P = x->ne[2]; const int axd = 2 * ((2 * (dh / 2) / 3) / 2);
const size_t es = ggml_element_size(x);
const int dh = (int) x->ne[0];
const int axd = 2 * ((2 * (dh / 2) / 3) / 2);
GGML_ASSERT(x->nb[0] == es);
GGML_ASSERT(3 * axd <= dh); GGML_ASSERT(3 * axd <= dh);
const float th = hparams.rope_theta; const float th = hparams.rope_theta;
// layout of x is [t, h, w, pad] // layout of x is [t, h, w, pad]
// t is unrotated, h and w are rotated, pad is unrotated // t is unrotated, h and w are rotated, pad is unrotated
// note: everything from n_dims onward untouched, so w and pad are rotated in one call. x = ggml_rope_ext(ctx0, x, pos_h, nullptr, axd, GGML_ROPE_TYPE_NEOX, 0, th, 1.0f, 0.0f, 1.0f, 0.0f, 0.0f);
auto sl = [&](int off, int n) { x = ggml_rope_set_offset(x, axd);
return ggml_cont(ctx0, ggml_view_3d(ctx0, x, n, Hn, P, x->nb[1], x->nb[2], (size_t) off * es));
};
ggml_tensor * t = sl(0, axd);
ggml_tensor * h = sl(axd, axd);
ggml_tensor * w = sl(2 * axd, dh - 2 * axd); // w + pad
h = ggml_rope_ext(ctx0, h, pos_h, nullptr, axd, GGML_ROPE_TYPE_NEOX, 0, th, 1.0f, 0.0f, 1.0f, 0.0f, 0.0f); x = ggml_rope_ext(ctx0, x, pos_w, nullptr, axd, GGML_ROPE_TYPE_NEOX, 0, th, 1.0f, 0.0f, 1.0f, 0.0f, 0.0f);
w = ggml_rope_ext(ctx0, w, pos_w, nullptr, axd, GGML_ROPE_TYPE_NEOX, 0, th, 1.0f, 0.0f, 1.0f, 0.0f, 0.0f); x = ggml_rope_set_offset(x, 2 * axd);
return ggml_concat(ctx0, ggml_concat(ctx0, t, h, 0), w, 0);
return x;
} }
ggml_cgraph * clip_graph_minimax_m3::build() { ggml_cgraph * clip_graph_minimax_m3::build() {