mtmd: use ggml_rope_set_offset (#27521)

* mtmd: use ggml_rope_set_offset

* add comment

tools/mtmd/clip-graph.h

@@ -137,9 +137,15 @@ struct clip_graph {
             int il,
             ggml_tensor * sinks = nullptr) const;
 
-    // implementation of the 2D RoPE without adding a new op in ggml
+    // implementation of the 2D RoPE using two ggml_rope_ext calls
-    // this is not efficient (use double the memory), but works on all backends
+    //
-    // TODO: there was a more efficient which relies on ggml_view and ggml_rope_ext_inplace, but the rope inplace does not work well with non-contiguous tensors ; we should fix that and revert back to the original implementation in https://github.com/ggml-org/llama.cpp/pull/13065
+    // unlike GGML_ROPE_TYPE_VISION which forces NEOX ordering, this rotates adjacent pairs (normal ordering)
+    //
+    // example:
+    //  given a single head with size = 8 --> [00000000]
+    //  dims [0, 4) rotate with pos_a, dims [4, 8) rotate with pos_b --> [aaaabbbb]
+    //  interleave_freq = false --> both halves use the same inv_freq set (like GGML_ROPE_TYPE_VISION)
+    //  interleave_freq = true  --> first half uses even inv_freq, second half uses odd inv_freq (used by pixtral)
     ggml_tensor * build_rope_2d(
         ggml_context * ctx0,
         ggml_tensor * cur,

tools/mtmd/clip.cpp

@@ -819,8 +819,6 @@ ggml_tensor * clip_graph::build_attn(
 }
 
 // implementation of the 2D RoPE without adding a new op in ggml
-// this is not efficient (use double the memory), but works on all backends
-// TODO: there was a more efficient which relies on ggml_view and ggml_rope_ext_inplace, but the rope inplace does not work well with non-contiguous tensors ; we should fix that and revert back to the original implementation in https://github.com/ggml-org/llama.cpp/pull/13065
 ggml_tensor * clip_graph::build_rope_2d(
     ggml_context * ctx0,
     ggml_tensor * cur,
@@ -830,8 +828,6 @@ ggml_tensor * clip_graph::build_rope_2d(
     const bool interleave_freq
 ) {
     const int64_t n_dim = cur->ne[0];
-    const int64_t n_head = cur->ne[1];
-    const int64_t n_pos  = cur->ne[2];
 
     // for example, if we have cur tensor of shape (n_dim=8, n_head, n_pos)
     // we will have a list of 4 inv_freq: 1e-0, 1e-1, 1e-2, 1e-3
@@ -845,36 +841,21 @@ ggml_tensor * clip_graph::build_rope_2d(
                                 ? std::pow(freq_base, (float)-2/n_dim)
                                 : 1.0;
 
-    // first half
+    // first half, dims [0, n_dim/2)
-    ggml_tensor * first;
+    cur = ggml_rope_ext(
-    {
-        first = ggml_view_3d(ctx0, cur,
-            n_dim/2, n_head, n_pos,
-            cur->nb[1],
-            cur->nb[2],
-            0);
-        first = ggml_rope_ext(
         ctx0,
-            first,
+        cur,
         pos_a,      // positions
         nullptr,    // freq factors
         n_dim/2,    // n_dims
         0, 0, freq_base,
         1.0f, 0.0f, 1.0f, 0.0f, 0.0f
     );
-    }
 
-    // second half
+    // second half, dims [n_dim/2, n_dim)
-    ggml_tensor * second;
+    cur = ggml_rope_ext(
-    {
-        second = ggml_view_3d(ctx0, cur,
-            n_dim/2, n_head, n_pos,
-            cur->nb[1],
-            cur->nb[2],
-            n_dim/2 * ggml_element_size(cur));
-        second = ggml_rope_ext(
         ctx0,
-            second,
+        cur,
         pos_b,      // positions
         nullptr,    // freq factors
         n_dim/2,    // n_dims
@@ -882,9 +863,8 @@ ggml_tensor * clip_graph::build_rope_2d(
         freq_scale_odd,
         0.0f, 1.0f, 0.0f, 0.0f
     );
-    }
+    cur = ggml_rope_set_offset(cur, n_dim/2);
 
-    cur = ggml_concat(ctx0, first, second, 0);
     return cur;
 }
 

tools/mtmd/models/gemma4v.cpp

@@ -45,50 +45,30 @@ ggml_cgraph * clip_graph_gemma4v::build() {
     // similar to build_rope_2d, but use neox ordering
     auto add_pos = [&](ggml_tensor * cur, const clip_layer &) {
         const int64_t n_dim = cur->ne[0];
-        const int64_t n_head = cur->ne[1];
-        const int64_t n_pos  = cur->ne[2];
 
-        // first half
+        // first half, dims [0, n_dim/2)
-        ggml_tensor * first;
+        cur = ggml_rope_ext(
-        {
-            first = ggml_view_4d(ctx0, cur,
-                n_dim/2, n_head, n_pos, n_batch,
-                cur->nb[1],
-                cur->nb[2],
-                cur->nb[3],
-                0);
-            first = ggml_rope_ext(
             ctx0,
-                first,
+            cur,
             pos_x,      // positions
             nullptr,    // freq factors
             n_dim/2,    // n_dims
             GGML_ROPE_TYPE_NEOX, 0, hparams.rope_theta,
             1.0f, 0.0f, 1.0f, 0.0f, 0.0f
         );
-        }
 
-        // second half
+        // second half, dims [n_dim/2, n_dim)
-        ggml_tensor * second;
+        cur = ggml_rope_ext(
-        {
-            second = ggml_view_4d(ctx0, cur,
-                n_dim/2, n_head, n_pos, n_batch,
-                cur->nb[1],
-                cur->nb[2],
-                cur->nb[3],
-                n_dim/2 * ggml_element_size(cur));
-            second = ggml_rope_ext(
             ctx0,
-                second,
+            cur,
             pos_y,      // positions
             nullptr,    // freq factors
             n_dim/2,    // n_dims
             GGML_ROPE_TYPE_NEOX, 0, hparams.rope_theta,
             1.0f, 0.0f, 1.0f, 0.0f, 0.0f
         );
-        }
+        cur = ggml_rope_set_offset(cur, n_dim/2);
 
-        cur = ggml_concat(ctx0, first, second, 0);
         return cur;
     };
 

tools/mtmd/models/minimax-m3.cpp

@@ -2,30 +2,22 @@
 
 ggml_tensor * clip_graph_minimax_m3::apply_rope(
         ggml_tensor * x, ggml_tensor * pos_h, ggml_tensor * pos_w) {
-    const int64_t Hn  = x->ne[1];
-    const int64_t P   = x->ne[2];
-    const size_t  es  = ggml_element_size(x);
     const int dh  = (int) x->ne[0];
     const int axd = 2 * ((2 * (dh / 2) / 3) / 2);
 
-    GGML_ASSERT(x->nb[0] == es);
     GGML_ASSERT(3 * axd <= dh);
 
     const float th  = hparams.rope_theta;
 
     // layout of x is [t, h, w, pad]
     // t is unrotated, h and w are rotated, pad is unrotated
-    // note: everything from n_dims onward untouched, so w and pad are rotated in one call.
+    x = ggml_rope_ext(ctx0, x, pos_h, nullptr, axd, GGML_ROPE_TYPE_NEOX, 0, th, 1.0f, 0.0f, 1.0f, 0.0f, 0.0f);
-    auto sl = [&](int off, int n) {
+    x = ggml_rope_set_offset(x, axd);
-        return ggml_cont(ctx0, ggml_view_3d(ctx0, x, n, Hn, P, x->nb[1], x->nb[2], (size_t) off * es));
-    };
-    ggml_tensor * t = sl(0,       axd);
-    ggml_tensor * h = sl(axd,     axd);
-    ggml_tensor * w = sl(2 * axd, dh - 2 * axd); // w + pad
 
-    h = ggml_rope_ext(ctx0, h, pos_h, nullptr, axd, GGML_ROPE_TYPE_NEOX, 0, th, 1.0f, 0.0f, 1.0f, 0.0f, 0.0f);
+    x = ggml_rope_ext(ctx0, x, pos_w, nullptr, axd, GGML_ROPE_TYPE_NEOX, 0, th, 1.0f, 0.0f, 1.0f, 0.0f, 0.0f);
-    w = ggml_rope_ext(ctx0, w, pos_w, nullptr, axd, GGML_ROPE_TYPE_NEOX, 0, th, 1.0f, 0.0f, 1.0f, 0.0f, 0.0f);
+    x = ggml_rope_set_offset(x, 2 * axd);
-    return ggml_concat(ctx0, ggml_concat(ctx0, t, h, 0), w, 0);
+
+    return x;
 }
 
 ggml_cgraph * clip_graph_minimax_m3::build() {