Files
llama.cpp/tests/test-rope.cpp
T
Takamichi Ohara 1fa7f1c488 ggml : support truncate=false for YaRN (#30206)
* ggml : support `truncate=false` for YaRN

YaRN calculates correction intervals to switch frequency interpolation
methods, but the setting to not round the boundaries of these intervals
to integers (`truncate=false`) was not supported.

Models that use `truncate=false` include gpt-oss and some PLaMo-3
models; to support them correctly, the GGML operations and CPU backend
are modified to handle calculations without rounding. The default
behavior is `truncate=true`, and existing calls not explicitly
configured will maintain their previous behavior.

* support CUDA backend

* add tests

* move RoPE helpers to internal header

* format
2026-11-05 12:02:05 +02:00

351 lines
12 KiB
C++

#include "ggml.h"
#include "ggml-cpu.h"
#include <cmath>
#include <cstdio>
#include <cstdlib>
#include <cassert>
#include <vector>
#if defined(_MSC_VER)
#pragma warning(disable: 4244 4267) // possible loss of data
#endif
#if defined(__GNUC__)
#pragma GCC diagnostic ignored "-Wdouble-promotion"
#endif
#define MAX_NARGS 3
#undef MIN
#undef MAX
#define MIN(a, b) ((a) < (b) ? (a) : (b))
#define MAX(a, b) ((a) > (b) ? (a) : (b))
#define GGML_SILU_FP16
//
// logging
//
#if (GGML_DEBUG >= 1)
#define GGML_PRINT_DEBUG(...) printf(__VA_ARGS__)
#else
#define GGML_PRINT_DEBUG(...)
#endif
#if (GGML_DEBUG >= 5)
#define GGML_PRINT_DEBUG_5(...) printf(__VA_ARGS__)
#else
#define GGML_PRINT_DEBUG_5(...)
#endif
#if (GGML_DEBUG >= 10)
#define GGML_PRINT_DEBUG_10(...) printf(__VA_ARGS__)
#else
#define GGML_PRINT_DEBUG_10(...)
#endif
#define GGML_PRINT(...) printf(__VA_ARGS__)
static float frand(void) {
return (float)rand()/(float)RAND_MAX;
}
static int irand(int n) {
if (n == 0) return 0;
return rand()%n;
}
static void get_random_dims(int64_t * dims, int ndims) {
dims[0] = dims[1] = dims[2] = dims[3] = 1;
for (int i = 0; i < ndims; i++) {
dims[i] = 1 + irand(4);
}
}
static struct ggml_tensor * get_random_tensor_f32(
struct ggml_context * ctx0,
int ndims,
const int64_t ne[],
float fmin,
float fmax) {
struct ggml_tensor * result = ggml_new_tensor(ctx0, GGML_TYPE_F32, ndims, ne);
switch (ndims) {
case 1:
for (int i0 = 0; i0 < ne[0]; i0++) {
((float *)result->data)[i0] = frand()*(fmax - fmin) + fmin;
}
break;
case 2:
for (int i1 = 0; i1 < ne[1]; i1++) {
for (int i0 = 0; i0 < ne[0]; i0++) {
((float *)result->data)[i1*ne[0] + i0] = frand()*(fmax - fmin) + fmin;
}
}
break;
case 3:
for (int i2 = 0; i2 < ne[2]; i2++) {
for (int i1 = 0; i1 < ne[1]; i1++) {
for (int i0 = 0; i0 < ne[0]; i0++) {
((float *)result->data)[i2*ne[1]*ne[0] + i1*ne[0] + i0] = frand()*(fmax - fmin) + fmin;
}
}
}
break;
case 4:
for (int i3 = 0; i3 < ne[3]; i3++) {
for (int i2 = 0; i2 < ne[2]; i2++) {
for (int i1 = 0; i1 < ne[1]; i1++) {
for (int i0 = 0; i0 < ne[0]; i0++) {
((float *)result->data)[i3*ne[2]*ne[1]*ne[0] + i2*ne[1]*ne[0] + i1*ne[0] + i0] = frand()*(fmax - fmin) + fmin;
}
}
}
}
break;
default:
assert(false);
};
return result;
}
static void ggml_graph_compute_helper(std::vector<uint8_t> & buf, ggml_cgraph * graph, int n_threads) {
struct ggml_cplan plan = ggml_graph_plan(graph, n_threads, nullptr);
if (plan.work_size > 0) {
buf.resize(plan.work_size);
plan.work_data = buf.data();
}
ggml_graph_compute(graph, &plan);
}
static void test_yarn(ggml_type type, int mode, bool truncate, bool forward, bool inplace) {
const int n_dims = 128;
const int n_offs = 8;
const int ne0 = n_dims + 2*n_offs;
const int32_t positions[] = {0, 1, 4096, 8192, -67};
const int n_pos = sizeof(positions)/sizeof(positions[0]);
const double base = 1000000.0;
const double factor = 256.0;
const double pi = std::acos(-1.0);
double low = n_dims/2 * std::log(4096.0/(32.0*2*pi))/std::log(base);
double high = n_dims/2 * std::log(4096.0/( 1.0*2*pi))/std::log(base);
if (truncate) {
low = std::floor(low);
high = std::ceil(high);
}
const double mscale = 1.0 + 0.1*std::log(factor);
ggml_init_params params = {1024*1024, nullptr, false};
ggml_context * ctx = ggml_init(params);
ggml_tensor * x = ggml_new_tensor_3d(ctx, type, ne0, 2, n_pos);
ggml_tensor * pos = ggml_new_tensor_1d(ctx, GGML_TYPE_I32, n_pos);
std::vector<float> input(ggml_nelements(x));
for (size_t i = 0; i < input.size(); ++i) {
input[i] = (int(i % 31) - 15)/16.0f;
if (type == GGML_TYPE_F32) {
((float *) x->data)[i] = input[i];
} else {
((ggml_fp16_t *) x->data)[i] = ggml_fp32_to_fp16(input[i]);
}
}
for (int i = 0; i < n_pos; ++i) {
((int32_t *) pos->data)[i] = positions[i];
}
ggml_tensor * out;
if (!forward) {
out = ggml_rope_ext_back(ctx, x, pos, nullptr, n_dims, mode, 4096, base, 1.0/factor, 1.0f, 1.0f, 32.0f, 1.0f);
} else if (inplace) {
out = ggml_rope_ext_inplace(ctx, x, pos, nullptr, n_dims, mode, 4096, base, 1.0/factor, 1.0f, 1.0f, 32.0f, 1.0f);
} else {
out = ggml_rope_ext(ctx, x, pos, nullptr, n_dims, mode, 4096, base, 1.0/factor, 1.0f, 1.0f, 32.0f, 1.0f);
}
ggml_rope_set_truncate(out, truncate);
ggml_rope_set_offset(out, n_offs);
ggml_cgraph * graph = ggml_new_graph(ctx);
ggml_build_forward_expand(graph, out);
std::vector<uint8_t> work;
ggml_graph_compute_helper(work, graph, 2);
std::vector<float> expected = input;
for (int p = 0; p < n_pos; ++p) {
for (int h = 0; h < 2; ++h) {
const int row = (p*2 + h)*ne0 + n_offs;
for (int i = 0; i < n_dims/2; ++i) {
const double ramp = 1.0 - MIN(1.0, MAX(0.0, (i - low)/(high - low)));
const double freq = std::pow(base, -2.0*i/n_dims);
const double theta = positions[p]*freq*((1.0 - ramp)/factor + ramp);
const double c = std::cos(theta)*mscale;
const double s = std::sin(theta)*mscale*(forward ? 1.0 : -1.0);
const int i0 = row + (mode == GGML_ROPE_TYPE_NORMAL ? 2*i : i);
const int i1 = i0 + (mode == GGML_ROPE_TYPE_NORMAL ? 1 : n_dims/2);
expected[i0] = input[i0]*c - input[i1]*s;
expected[i1] = input[i0]*s + input[i1]*c;
}
}
}
float max_error = 0.0f;
for (size_t i = 0; i < expected.size(); ++i) {
const float actual = type == GGML_TYPE_F32 ? ((float *) out->data)[i] : ggml_fp16_to_fp32(((ggml_fp16_t *) out->data)[i]);
GGML_ASSERT(std::isfinite(actual));
max_error = MAX(max_error, std::fabs(actual - expected[i]));
}
printf("yarn type=%s mode=%d truncate=%d forward=%d inplace=%d max_error=%g\n",
ggml_type_name(type), mode, truncate, forward, inplace, max_error);
GGML_ASSERT(max_error < (type == GGML_TYPE_F32 ? 0.003f : 0.005f));
ggml_free(ctx);
}
int main(int /*argc*/, const char ** /*argv*/) {
struct ggml_init_params params = {
/* .mem_size = */ 128*1024*1024,
/* .mem_buffer = */ NULL,
/* .no_alloc = */ false,
};
std::vector<uint8_t> work_buffer;
struct ggml_context * ctx0 = ggml_init(params);
struct ggml_tensor * x;
// rope f32
for (int m = 0; m < 5; ++m) {
const int ndims = 4;
const int64_t n_rot = 128;
const int64_t ne[4] = { 2*n_rot, 32, 73, 1 };
const int n_past_0 = 100;
const int n_past_2 = 33;
struct ggml_tensor * r0;
struct ggml_tensor * r1;
struct ggml_tensor * r2;
x = get_random_tensor_f32(ctx0, ndims, ne, -1.0f, 1.0f);
int mode = -1;
if (m < 2) {
struct ggml_tensor * p0 = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, ne[2]);
struct ggml_tensor * p1 = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, ne[2]);
struct ggml_tensor * p2 = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, ne[2]);
for (int i = 0; i < ne[2]; ++i) {
((int32_t *) p0->data)[i] = n_past_0 + i;
((int32_t *) p1->data)[i] = n_past_2 - n_past_0;
((int32_t *) p2->data)[i] = n_past_2 + i;
}
// test mode 0, 2 (standard, GPT-NeoX)
mode = m == 0 ? GGML_ROPE_TYPE_NORMAL : GGML_ROPE_TYPE_NEOX;
// 100, 101, 102, ..., 172
r0 = ggml_rope(ctx0, x, p0, n_rot, mode);
// -67, -67, -67, ..., -67
r1 = ggml_rope(ctx0, r0, p1, n_rot, mode); // "context swap", i.e. forget n_past_0 - n_past_2 tokens
// 33, 34, 35, ..., 105
r2 = ggml_rope(ctx0, x, p2, n_rot, mode);
} else {
// testing multi-dimension rope position embedding mode
struct ggml_tensor * p0 = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, ne[2] * 4);
struct ggml_tensor * p1 = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, ne[2] * 4);
struct ggml_tensor * p2 = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, ne[2] * 4);
int sections[4] = {16, 24, 24, 0};
mode = (m == 2) ? GGML_ROPE_TYPE_MROPE : (m == 3) ? GGML_ROPE_TYPE_VISION : GGML_ROPE_TYPE_IMROPE;
for (int i = 0; i < ne[2]; ++i) {
for (int j = 0; j < 4; ++j) {
((int32_t *) p0->data)[i + ne[2] * j] = n_past_0 + i + j;
((int32_t *) p1->data)[i + ne[2] * j] = n_past_2 - n_past_0;
((int32_t *) p2->data)[i + ne[2] * j] = n_past_2 + i + j;
}
}
// [[100, 101, 102, ..., 172],
// [101, 102, 103, ..., 173],
// [102, 103, 104, ..., 174]]
r0 = ggml_rope_multi(
ctx0, x, p0, nullptr,
n_rot, sections, mode, 32768, 1000000, 1, 0, 1, 32, 1);
// [[-67, -67, -67, ..., -67]
// [-67, -67, -67, ..., -67]
// [-67, -67, -67, ..., -67]]
r1 = ggml_rope_multi(
ctx0, r0, p1, nullptr,
n_rot, sections, mode, 32768, 1000000, 1, 0, 1, 32, 1);
// [[33, 34, 35, ..., 105]
// [34, 35, 36, ..., 106]
// [35, 36, 37, ..., 107]]
r2 = ggml_rope_multi(
ctx0, x, p2, nullptr,
n_rot, sections, mode, 32768, 1000000, 1, 0, 1, 32, 1);
}
ggml_cgraph * gf = ggml_new_graph(ctx0);
ggml_build_forward_expand(gf, r0);
ggml_build_forward_expand(gf, r1);
ggml_build_forward_expand(gf, r2);
ggml_graph_compute_helper(work_buffer, gf, 4);
// check that r1 and r2 are the same
{
double sum0 = 0.0f;
double sum1 = 0.0f;
double diff = 0.0f;
const float * r1_data = (float *) r1->data;
const float * r2_data = (float *) r2->data;
const int n_elements = ggml_nelements(r1);
for (int i = 0; i < n_elements; ++i) {
sum0 += fabs(r1_data[i]);
sum1 += fabs(r2_data[i]);
diff += fabs(r1_data[i] - r2_data[i]);
//if (fabs(r1_data[i] - r2_data[i]) > 0.0001f) {
// printf("%d: %f %f\n", i, r1_data[i], r2_data[i]);
// printf("diff: %f\n", fabs(r1_data[i] - r2_data[i]));
//}
}
//for (int i = 4096; i < 4096 + 128; ++i) {
// printf("%f %f\n", r1_data[i], r2_data[i]);
//}
printf("mode: %d\n", mode);
printf("sum0: %f\n", sum0);
printf("sum1: %f\n", sum1);
printf("diff: %f\n", diff);
printf("rel err: %f\n", diff / sum0);
printf("rel err: %f\n", diff / sum1);
GGML_ASSERT(diff / sum0 < 0.0001f);
GGML_ASSERT(diff / sum1 < 0.0001f);
}
}
ggml_free(ctx0);
for (ggml_type type : {GGML_TYPE_F32, GGML_TYPE_F16}) {
for (int mode : {GGML_ROPE_TYPE_NORMAL, GGML_ROPE_TYPE_NEOX}) {
for (bool truncate : {false, true}) {
test_yarn(type, mode, truncate, true, false);
test_yarn(type, mode, truncate, true, true);
test_yarn(type, mode, truncate, false, false);
}
}
}
return 0;
}