#include "ggml.h" #include "ggml-cpu.h" #include #include #include #include #include #if defined(_MSC_VER) #pragma warning(disable: 4244 4267) // possible loss of data #endif #if defined(__GNUC__) #pragma GCC diagnostic ignored "-Wdouble-promotion" #endif #define MAX_NARGS 3 #undef MIN #undef MAX #define MIN(a, b) ((a) < (b) ? (a) : (b)) #define MAX(a, b) ((a) > (b) ? (a) : (b)) #define GGML_SILU_FP16 // // logging // #if (GGML_DEBUG >= 1) #define GGML_PRINT_DEBUG(...) printf(__VA_ARGS__) #else #define GGML_PRINT_DEBUG(...) #endif #if (GGML_DEBUG >= 5) #define GGML_PRINT_DEBUG_5(...) printf(__VA_ARGS__) #else #define GGML_PRINT_DEBUG_5(...) #endif #if (GGML_DEBUG >= 10) #define GGML_PRINT_DEBUG_10(...) printf(__VA_ARGS__) #else #define GGML_PRINT_DEBUG_10(...) #endif #define GGML_PRINT(...) printf(__VA_ARGS__) static float frand(void) { return (float)rand()/(float)RAND_MAX; } static int irand(int n) { if (n == 0) return 0; return rand()%n; } static void get_random_dims(int64_t * dims, int ndims) { dims[0] = dims[1] = dims[2] = dims[3] = 1; for (int i = 0; i < ndims; i++) { dims[i] = 1 + irand(4); } } static struct ggml_tensor * get_random_tensor_f32( struct ggml_context * ctx0, int ndims, const int64_t ne[], float fmin, float fmax) { struct ggml_tensor * result = ggml_new_tensor(ctx0, GGML_TYPE_F32, ndims, ne); switch (ndims) { case 1: for (int i0 = 0; i0 < ne[0]; i0++) { ((float *)result->data)[i0] = frand()*(fmax - fmin) + fmin; } break; case 2: for (int i1 = 0; i1 < ne[1]; i1++) { for (int i0 = 0; i0 < ne[0]; i0++) { ((float *)result->data)[i1*ne[0] + i0] = frand()*(fmax - fmin) + fmin; } } break; case 3: for (int i2 = 0; i2 < ne[2]; i2++) { for (int i1 = 0; i1 < ne[1]; i1++) { for (int i0 = 0; i0 < ne[0]; i0++) { ((float *)result->data)[i2*ne[1]*ne[0] + i1*ne[0] + i0] = frand()*(fmax - fmin) + fmin; } } } break; case 4: for (int i3 = 0; i3 < ne[3]; i3++) { for (int i2 = 0; i2 < ne[2]; i2++) { for (int i1 = 0; i1 < ne[1]; i1++) { for (int i0 = 0; i0 < ne[0]; i0++) { ((float *)result->data)[i3*ne[2]*ne[1]*ne[0] + i2*ne[1]*ne[0] + i1*ne[0] + i0] = frand()*(fmax - fmin) + fmin; } } } } break; default: assert(false); }; return result; } static void ggml_graph_compute_helper(std::vector & buf, ggml_cgraph * graph, int n_threads) { struct ggml_cplan plan = ggml_graph_plan(graph, n_threads, nullptr); if (plan.work_size > 0) { buf.resize(plan.work_size); plan.work_data = buf.data(); } ggml_graph_compute(graph, &plan); } static void test_yarn(ggml_type type, int mode, bool truncate, bool forward, bool inplace) { const int n_dims = 128; const int n_offs = 8; const int ne0 = n_dims + 2*n_offs; const int32_t positions[] = {0, 1, 4096, 8192, -67}; const int n_pos = sizeof(positions)/sizeof(positions[0]); const double base = 1000000.0; const double factor = 256.0; const double pi = std::acos(-1.0); double low = n_dims/2 * std::log(4096.0/(32.0*2*pi))/std::log(base); double high = n_dims/2 * std::log(4096.0/( 1.0*2*pi))/std::log(base); if (truncate) { low = std::floor(low); high = std::ceil(high); } const double mscale = 1.0 + 0.1*std::log(factor); ggml_init_params params = {1024*1024, nullptr, false}; ggml_context * ctx = ggml_init(params); ggml_tensor * x = ggml_new_tensor_3d(ctx, type, ne0, 2, n_pos); ggml_tensor * pos = ggml_new_tensor_1d(ctx, GGML_TYPE_I32, n_pos); std::vector input(ggml_nelements(x)); for (size_t i = 0; i < input.size(); ++i) { input[i] = (int(i % 31) - 15)/16.0f; if (type == GGML_TYPE_F32) { ((float *) x->data)[i] = input[i]; } else { ((ggml_fp16_t *) x->data)[i] = ggml_fp32_to_fp16(input[i]); } } for (int i = 0; i < n_pos; ++i) { ((int32_t *) pos->data)[i] = positions[i]; } ggml_tensor * out; if (!forward) { out = ggml_rope_ext_back(ctx, x, pos, nullptr, n_dims, mode, 4096, base, 1.0/factor, 1.0f, 1.0f, 32.0f, 1.0f); } else if (inplace) { out = ggml_rope_ext_inplace(ctx, x, pos, nullptr, n_dims, mode, 4096, base, 1.0/factor, 1.0f, 1.0f, 32.0f, 1.0f); } else { out = ggml_rope_ext(ctx, x, pos, nullptr, n_dims, mode, 4096, base, 1.0/factor, 1.0f, 1.0f, 32.0f, 1.0f); } ggml_rope_set_truncate(out, truncate); ggml_rope_set_offset(out, n_offs); ggml_cgraph * graph = ggml_new_graph(ctx); ggml_build_forward_expand(graph, out); std::vector work; ggml_graph_compute_helper(work, graph, 2); std::vector expected = input; for (int p = 0; p < n_pos; ++p) { for (int h = 0; h < 2; ++h) { const int row = (p*2 + h)*ne0 + n_offs; for (int i = 0; i < n_dims/2; ++i) { const double ramp = 1.0 - MIN(1.0, MAX(0.0, (i - low)/(high - low))); const double freq = std::pow(base, -2.0*i/n_dims); const double theta = positions[p]*freq*((1.0 - ramp)/factor + ramp); const double c = std::cos(theta)*mscale; const double s = std::sin(theta)*mscale*(forward ? 1.0 : -1.0); const int i0 = row + (mode == GGML_ROPE_TYPE_NORMAL ? 2*i : i); const int i1 = i0 + (mode == GGML_ROPE_TYPE_NORMAL ? 1 : n_dims/2); expected[i0] = input[i0]*c - input[i1]*s; expected[i1] = input[i0]*s + input[i1]*c; } } } float max_error = 0.0f; for (size_t i = 0; i < expected.size(); ++i) { const float actual = type == GGML_TYPE_F32 ? ((float *) out->data)[i] : ggml_fp16_to_fp32(((ggml_fp16_t *) out->data)[i]); GGML_ASSERT(std::isfinite(actual)); max_error = MAX(max_error, std::fabs(actual - expected[i])); } printf("yarn type=%s mode=%d truncate=%d forward=%d inplace=%d max_error=%g\n", ggml_type_name(type), mode, truncate, forward, inplace, max_error); GGML_ASSERT(max_error < (type == GGML_TYPE_F32 ? 0.003f : 0.005f)); ggml_free(ctx); } int main(int /*argc*/, const char ** /*argv*/) { struct ggml_init_params params = { /* .mem_size = */ 128*1024*1024, /* .mem_buffer = */ NULL, /* .no_alloc = */ false, }; std::vector work_buffer; struct ggml_context * ctx0 = ggml_init(params); struct ggml_tensor * x; // rope f32 for (int m = 0; m < 5; ++m) { const int ndims = 4; const int64_t n_rot = 128; const int64_t ne[4] = { 2*n_rot, 32, 73, 1 }; const int n_past_0 = 100; const int n_past_2 = 33; struct ggml_tensor * r0; struct ggml_tensor * r1; struct ggml_tensor * r2; x = get_random_tensor_f32(ctx0, ndims, ne, -1.0f, 1.0f); int mode = -1; if (m < 2) { struct ggml_tensor * p0 = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, ne[2]); struct ggml_tensor * p1 = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, ne[2]); struct ggml_tensor * p2 = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, ne[2]); for (int i = 0; i < ne[2]; ++i) { ((int32_t *) p0->data)[i] = n_past_0 + i; ((int32_t *) p1->data)[i] = n_past_2 - n_past_0; ((int32_t *) p2->data)[i] = n_past_2 + i; } // test mode 0, 2 (standard, GPT-NeoX) mode = m == 0 ? GGML_ROPE_TYPE_NORMAL : GGML_ROPE_TYPE_NEOX; // 100, 101, 102, ..., 172 r0 = ggml_rope(ctx0, x, p0, n_rot, mode); // -67, -67, -67, ..., -67 r1 = ggml_rope(ctx0, r0, p1, n_rot, mode); // "context swap", i.e. forget n_past_0 - n_past_2 tokens // 33, 34, 35, ..., 105 r2 = ggml_rope(ctx0, x, p2, n_rot, mode); } else { // testing multi-dimension rope position embedding mode struct ggml_tensor * p0 = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, ne[2] * 4); struct ggml_tensor * p1 = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, ne[2] * 4); struct ggml_tensor * p2 = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, ne[2] * 4); int sections[4] = {16, 24, 24, 0}; mode = (m == 2) ? GGML_ROPE_TYPE_MROPE : (m == 3) ? GGML_ROPE_TYPE_VISION : GGML_ROPE_TYPE_IMROPE; for (int i = 0; i < ne[2]; ++i) { for (int j = 0; j < 4; ++j) { ((int32_t *) p0->data)[i + ne[2] * j] = n_past_0 + i + j; ((int32_t *) p1->data)[i + ne[2] * j] = n_past_2 - n_past_0; ((int32_t *) p2->data)[i + ne[2] * j] = n_past_2 + i + j; } } // [[100, 101, 102, ..., 172], // [101, 102, 103, ..., 173], // [102, 103, 104, ..., 174]] r0 = ggml_rope_multi( ctx0, x, p0, nullptr, n_rot, sections, mode, 32768, 1000000, 1, 0, 1, 32, 1); // [[-67, -67, -67, ..., -67] // [-67, -67, -67, ..., -67] // [-67, -67, -67, ..., -67]] r1 = ggml_rope_multi( ctx0, r0, p1, nullptr, n_rot, sections, mode, 32768, 1000000, 1, 0, 1, 32, 1); // [[33, 34, 35, ..., 105] // [34, 35, 36, ..., 106] // [35, 36, 37, ..., 107]] r2 = ggml_rope_multi( ctx0, x, p2, nullptr, n_rot, sections, mode, 32768, 1000000, 1, 0, 1, 32, 1); } ggml_cgraph * gf = ggml_new_graph(ctx0); ggml_build_forward_expand(gf, r0); ggml_build_forward_expand(gf, r1); ggml_build_forward_expand(gf, r2); ggml_graph_compute_helper(work_buffer, gf, 4); // check that r1 and r2 are the same { double sum0 = 0.0f; double sum1 = 0.0f; double diff = 0.0f; const float * r1_data = (float *) r1->data; const float * r2_data = (float *) r2->data; const int n_elements = ggml_nelements(r1); for (int i = 0; i < n_elements; ++i) { sum0 += fabs(r1_data[i]); sum1 += fabs(r2_data[i]); diff += fabs(r1_data[i] - r2_data[i]); //if (fabs(r1_data[i] - r2_data[i]) > 0.0001f) { // printf("%d: %f %f\n", i, r1_data[i], r2_data[i]); // printf("diff: %f\n", fabs(r1_data[i] - r2_data[i])); //} } //for (int i = 4096; i < 4096 + 128; ++i) { // printf("%f %f\n", r1_data[i], r2_data[i]); //} printf("mode: %d\n", mode); printf("sum0: %f\n", sum0); printf("sum1: %f\n", sum1); printf("diff: %f\n", diff); printf("rel err: %f\n", diff / sum0); printf("rel err: %f\n", diff / sum1); GGML_ASSERT(diff / sum0 < 0.0001f); GGML_ASSERT(diff / sum1 < 0.0001f); } } ggml_free(ctx0); for (ggml_type type : {GGML_TYPE_F32, GGML_TYPE_F16}) { for (int mode : {GGML_ROPE_TYPE_NORMAL, GGML_ROPE_TYPE_NEOX}) { for (bool truncate : {false, true}) { test_yarn(type, mode, truncate, true, false); test_yarn(type, mode, truncate, true, true); test_yarn(type, mode, truncate, false, false); } } } return 0; }