From 76f186df71a41abb3ee20fffedee31ca5dbc7d9f Mon Sep 17 00:00:00 2001 From: Bartowski <3266127+bartowski1182@users.noreply.github.com> Date: Thu, 19 Nov 2026 03:04:58 -0500 Subject: [PATCH] ggml : IQ2 creation speedup through caching (#31910) * IQ2 quant creation speedup from caching * Rename variables for clarity * Explicit casts * Initialize all values, add comment to tolerance --- ggml/src/ggml-quants.c | 119 ++++++++++++++++++++++++++++++++++++++--- 1 file changed, 113 insertions(+), 6 deletions(-) diff --git a/ggml/src/ggml-quants.c b/ggml/src/ggml-quants.c index 3586f67ae..40ffe1e6b 100644 --- a/ggml/src/ggml-quants.c +++ b/ggml/src/ggml-quants.c @@ -3307,6 +3307,85 @@ static int iq2_find_best_neighbour(const uint16_t * GGML_RESTRICT neighbours, co return grid_index; } +// same result as iq2_find_best_neighbour: score = s*(s*quad_coef - lin_coef) + const, and the coefs do not depend on the scale s, so cache them per pattern +// take the cached winner only if all other neighbours are worse by more than the float rounding bound of the exact search, else run the exact search +#define IQ2_NEIGHBOUR_CACHE_MAX 512 + +typedef struct { + int pattern; // pattern whose coefs are cached, -1 if none + int last_best; // last winner for this pattern + // sums for the rounding bound: |w|, |w|*|x|, |w|*x^2, and the same with 1 + |w| + double sum_w, sum_wx, sum_wxx, sum_w1, sum_w1x, sum_w1xx; + double lin_term[8][8], quad_term[8][8]; // 2*w*x*q and w*q*q per element, indexed by the grid level q + double lin_coef[IQ2_NEIGHBOUR_CACHE_MAX]; + double quad_coef[IQ2_NEIGHBOUR_CACHE_MAX]; +} iq2_neighbour_cache; + +static void iq2_neighbour_cache_init(iq2_neighbour_cache * cache, const float * GGML_RESTRICT xval, const float * GGML_RESTRICT weight) { + cache->pattern = -1; + cache->sum_w = cache->sum_wx = cache->sum_wxx = cache->sum_w1 = cache->sum_w1x = cache->sum_w1xx = 0; + for (int i = 0; i < 8; ++i) { + const double w = (double)weight[i], x = (double)xval[i], abs_w = fabs(w), abs_x = fabs(x); + cache->sum_w += abs_w; cache->sum_wx += abs_w*abs_x; cache->sum_wxx += abs_w*abs_x*abs_x; + cache->sum_w1 += 1 + abs_w; cache->sum_w1x += (1 + abs_w)*abs_x; cache->sum_w1xx += (1 + abs_w)*abs_x*abs_x; + for (int q = 0; q < 8; ++q) { + cache->lin_term[i][q] = 2*w*x*q; + cache->quad_term[i][q] = w*q*q; + } + } +} + +static int iq2_count_at_most(const double * GGML_RESTRICT values, int n, double limit) { + int count = 0; + for (int j = 0; j < n; ++j) count += values[j] <= limit; + return count; +} + +static int iq2_find_best_neighbour_cached(iq2_neighbour_cache * cache, int pattern, const uint16_t * GGML_RESTRICT neighbours, const uint64_t * GGML_RESTRICT grid, + const float * GGML_RESTRICT xval, const float * GGML_RESTRICT weight, float scale, int8_t * GGML_RESTRICT L) { + const int num_neighbors = neighbours[0]; + const double scale_d = (double)scale; + const double max_level = 7*scale_d; + // size of the numbers in the exact search, its rounding error is a multiple of 2^-24*size (+ size_subnormal for underflow) + const double size = max_level*max_level*cache->sum_w + 2*max_level*cache->sum_wx + cache->sum_wxx; + const double size_subnormal = (1 + max_level)*(1 + max_level)*cache->sum_w1 + 2*(1 + max_level)*cache->sum_w1x + cache->sum_w1xx; + // the cached score is (exact score - const)/s, so the 2^-18*size margin on the exact score is divided by s + const double tolerance = (0x1p-18*size + 0x1p-100*size_subnormal)/scale_d; + if (num_neighbors <= 0 || num_neighbors > IQ2_NEIGHBOUR_CACHE_MAX || !(scale_d > 0 && size < 0x1p100 && size_subnormal < 0x1p100 && tolerance > 0)) { + return iq2_find_best_neighbour(neighbours, grid, xval, weight, scale, L); + } + if (cache->pattern != pattern) { + for (int j = 0; j < num_neighbors; ++j) { + const int8_t * levels = (const int8_t *)(grid + neighbours[j+1]); + double lin = 0, quad = 0; + for (int i = 0; i < 8; ++i) { + lin += cache->lin_term[i][levels[i]]; + quad += cache->quad_term[i][levels[i]]; + } + cache->lin_coef[j] = lin; cache->quad_coef[j] = quad; + } + cache->pattern = pattern; + cache->last_best = 0; + } + double score[IQ2_NEIGHBOUR_CACHE_MAX]; + for (int j = 0; j < num_neighbors; ++j) score[j] = scale_d*cache->quad_coef[j] - cache->lin_coef[j]; + // the winner must be the only neighbour within tolerance, try the previous winner for this pattern first + int best = cache->last_best; + if (iq2_count_at_most(score, num_neighbors, score[best] + tolerance) != 1) { + for (int j = 0; j < num_neighbors; ++j) { + if (score[j] < score[best]) best = j; + } + if (iq2_count_at_most(score, num_neighbors, score[best] + tolerance) != 1) { + return iq2_find_best_neighbour(neighbours, grid, xval, weight, scale, L); + } + cache->last_best = best; + } + const int grid_index = neighbours[best+1]; + const int8_t * levels = (const int8_t *)(grid + grid_index); + for (int i = 0; i < 8; ++i) L[i] = (levels[i] - 1)/2; + return grid_index; +} + static void quantize_row_iq2_xxs_impl(const float * GGML_RESTRICT x, void * GGML_RESTRICT vy, int64_t n, const float * GGML_RESTRICT quant_weights) { const int gindex = iq2_data_index(GGML_TYPE_IQ2_XXS); @@ -3335,6 +3414,8 @@ static void quantize_row_iq2_xxs_impl(const float * GGML_RESTRICT x, void * GGML float waux[32]; uint8_t block_signs[4]; uint32_t q2[2*(QK_K/32)]; + iq2_neighbour_cache neighbour_cache[4]; + int8_t Laux_prev[32]; for (int ibl = 0; ibl < nbl; ++ibl) { @@ -3390,6 +3471,7 @@ static void quantize_row_iq2_xxs_impl(const float * GGML_RESTRICT x, void * GGML continue; } float best = 0; + for (int k = 0; k < 4; ++k) iq2_neighbour_cache_init(&neighbour_cache[k], xval + 8*k, waux + 8*k); for (int is = -6; is <= 6; ++is) { float id = (2*kMaxQ-1+is*0.1f)/eff_max; float this_scale = 1/id; @@ -3403,9 +3485,14 @@ static void quantize_row_iq2_xxs_impl(const float * GGML_RESTRICT x, void * GGML int grid_index = kmap_q2xs[u]; if (grid_index < 0) { const uint16_t * neighbours = kneighbors_q2xs - kmap_q2xs[u] - 1; - grid_index = iq2_find_best_neighbour(neighbours, kgrid_q2xs, xval + 8*k, waux + 8*k, this_scale, Laux + 8*k); + grid_index = iq2_find_best_neighbour_cached(&neighbour_cache[k], u, neighbours, kgrid_q2xs, xval + 8*k, waux + 8*k, this_scale, Laux + 8*k); } } + // same candidate as for the previous scale: it cannot change the result + if (is > -6 && memcmp(Laux, Laux_prev, 32) == 0) { + continue; + } + memcpy(Laux_prev, Laux, 32); float sumqx = 0, sumq2 = 0; for (int i = 0; i < 32; ++i) { float w = weight[i]; @@ -3430,7 +3517,7 @@ static void quantize_row_iq2_xxs_impl(const float * GGML_RESTRICT x, void * GGML int grid_index = kmap_q2xs[u]; if (grid_index < 0) { const uint16_t * neighbours = kneighbors_q2xs - kmap_q2xs[u] - 1; - grid_index = iq2_find_best_neighbour(neighbours, kgrid_q2xs, xval + 8*k, waux + 8*k, scale, L + 8*k); + grid_index = iq2_find_best_neighbour_cached(&neighbour_cache[k], u, neighbours, kgrid_q2xs, xval + 8*k, waux + 8*k, scale, L + 8*k); } const int8_t * pg = (const int8_t *)(kgrid_q2xs + grid_index); for (int i = 0; i < 8; ++i) L[8*k+i] = (pg[i] - 1)/2; @@ -3515,6 +3602,9 @@ static void quantize_row_iq2_xs_impl(const float * GGML_RESTRICT x, void * GGML_ bool is_on_grid_aux[2]; uint8_t block_signs[2]; uint16_t q2[2*(QK_K/16)]; + iq2_neighbour_cache neighbour_cache[2]; + int8_t Laux_prev[16]; + bool is_on_grid_prev[2]; for (int ibl = 0; ibl < nbl; ++ibl) { @@ -3566,6 +3656,7 @@ static void quantize_row_iq2_xs_impl(const float * GGML_RESTRICT x, void * GGML_ float best = 0; float scale = max/(2*kMaxQ-1); is_on_grid[0] = is_on_grid[1] = true; + for (int k = 0; k < 2; ++k) iq2_neighbour_cache_init(&neighbour_cache[k], xval + 8*k, waux + 8*k); for (int is = -9; is <= 9; ++is) { float id = (2*kMaxQ-1+is*0.1f)/max; float this_scale = 1/id; @@ -3581,9 +3672,15 @@ static void quantize_row_iq2_xs_impl(const float * GGML_RESTRICT x, void * GGML_ if (grid_index < 0) { is_on_grid_aux[k] = false; const uint16_t * neighbours = kneighbors_q2xs - kmap_q2xs[u] - 1; - grid_index = iq2_find_best_neighbour(neighbours, kgrid_q2xs, xval + 8*k, waux + 8*k, this_scale, Laux + 8*k); + grid_index = iq2_find_best_neighbour_cached(&neighbour_cache[k], u, neighbours, kgrid_q2xs, xval + 8*k, waux + 8*k, this_scale, Laux + 8*k); } } + // same candidate as for the previous scale: it cannot change the result + if (is > -9 && memcmp(Laux, Laux_prev, 16) == 0 && is_on_grid_aux[0] == is_on_grid_prev[0] && is_on_grid_aux[1] == is_on_grid_prev[1]) { + continue; + } + memcpy(Laux_prev, Laux, 16); + is_on_grid_prev[0] = is_on_grid_aux[0]; is_on_grid_prev[1] = is_on_grid_aux[1]; float sumqx = 0, sumq2 = 0; for (int i = 0; i < 16; ++i) { float w = weight[i]; @@ -3613,7 +3710,7 @@ static void quantize_row_iq2_xs_impl(const float * GGML_RESTRICT x, void * GGML_ int grid_index = kmap_q2xs[u]; if (grid_index < 0) { const uint16_t * neighbours = kneighbors_q2xs - kmap_q2xs[u] - 1; - grid_index = iq2_find_best_neighbour(neighbours, kgrid_q2xs, xval + 8*k, waux + 8*k, scale, L + 8*k); + grid_index = iq2_find_best_neighbour_cached(&neighbour_cache[k], u, neighbours, kgrid_q2xs, xval + 8*k, waux + 8*k, scale, L + 8*k); } } float sumqx = 0, sumq2 = 0; @@ -5155,6 +5252,9 @@ static void quantize_row_iq2_s_impl(const float * GGML_RESTRICT x, void * GGML_R bool is_on_grid[2]; bool is_on_grid_aux[2]; uint8_t block_signs[2]; + iq2_neighbour_cache neighbour_cache[2]; + int8_t Laux_prev[16]; + bool is_on_grid_prev[2]; for (int ibl = 0; ibl < nbl; ++ibl) { @@ -5197,6 +5297,7 @@ static void quantize_row_iq2_s_impl(const float * GGML_RESTRICT x, void * GGML_R float best = 0; float scale = max/(2*kMaxQ-1); is_on_grid[0] = is_on_grid[1] = true; + for (int k = 0; k < 2; ++k) iq2_neighbour_cache_init(&neighbour_cache[k], xval + 8*k, waux + 8*k); for (int is = -9; is <= 9; ++is) { float id = (2*kMaxQ-1+is*0.1f)/max; float this_scale = 1/id; @@ -5212,9 +5313,15 @@ static void quantize_row_iq2_s_impl(const float * GGML_RESTRICT x, void * GGML_R if (grid_index < 0) { is_on_grid_aux[k] = false; const uint16_t * neighbours = kneighbors_q2xs - kmap_q2xs[u] - 1; - grid_index = iq2_find_best_neighbour(neighbours, kgrid_q2xs, xval + 8*k, waux + 8*k, this_scale, Laux + 8*k); + grid_index = iq2_find_best_neighbour_cached(&neighbour_cache[k], u, neighbours, kgrid_q2xs, xval + 8*k, waux + 8*k, this_scale, Laux + 8*k); } } + // same candidate as for the previous scale: it cannot change the result + if (is > -9 && memcmp(Laux, Laux_prev, 16) == 0 && is_on_grid_aux[0] == is_on_grid_prev[0] && is_on_grid_aux[1] == is_on_grid_prev[1]) { + continue; + } + memcpy(Laux_prev, Laux, 16); + is_on_grid_prev[0] = is_on_grid_aux[0]; is_on_grid_prev[1] = is_on_grid_aux[1]; float sumqx = 0, sumq2 = 0; for (int i = 0; i < 16; ++i) { float w = weight[i]; @@ -5244,7 +5351,7 @@ static void quantize_row_iq2_s_impl(const float * GGML_RESTRICT x, void * GGML_R int grid_index = kmap_q2xs[u]; if (grid_index < 0) { const uint16_t * neighbours = kneighbors_q2xs - kmap_q2xs[u] - 1; - grid_index = iq2_find_best_neighbour(neighbours, kgrid_q2xs, xval + 8*k, waux + 8*k, scale, L + 8*k); + grid_index = iq2_find_best_neighbour_cached(&neighbour_cache[k], u, neighbours, kgrid_q2xs, xval + 8*k, waux + 8*k, scale, L + 8*k); } } float sumqx = 0, sumq2 = 0;