ggml : IQ2 creation speedup through caching (#31910)
* IQ2 quant creation speedup from caching * Rename variables for clarity * Explicit casts * Initialize all values, add comment to tolerance
This commit is contained in:
1 parent
95b09b62be
commit
76f186df71
1 file changed
+113
-6
+113
-6
@@ -3307,6 +3307,85 @@ static int iq2_find_best_neighbour(const uint16_t * GGML_RESTRICT neighbours, co
|
||||
return grid_index;
|
||||
}
|
||||
|
||||
// same result as iq2_find_best_neighbour: score = s*(s*quad_coef - lin_coef) + const, and the coefs do not depend on the scale s, so cache them per pattern
|
||||
// take the cached winner only if all other neighbours are worse by more than the float rounding bound of the exact search, else run the exact search
|
||||
#define IQ2_NEIGHBOUR_CACHE_MAX 512
|
||||
|
||||
typedef struct {
|
||||
int pattern; // pattern whose coefs are cached, -1 if none
|
||||
int last_best; // last winner for this pattern
|
||||
// sums for the rounding bound: |w|, |w|*|x|, |w|*x^2, and the same with 1 + |w|
|
||||
double sum_w, sum_wx, sum_wxx, sum_w1, sum_w1x, sum_w1xx;
|
||||
double lin_term[8][8], quad_term[8][8]; // 2*w*x*q and w*q*q per element, indexed by the grid level q
|
||||
double lin_coef[IQ2_NEIGHBOUR_CACHE_MAX];
|
||||
double quad_coef[IQ2_NEIGHBOUR_CACHE_MAX];
|
||||
} iq2_neighbour_cache;
|
||||
|
||||
static void iq2_neighbour_cache_init(iq2_neighbour_cache * cache, const float * GGML_RESTRICT xval, const float * GGML_RESTRICT weight) {
|
||||
cache->pattern = -1;
|
||||
cache->sum_w = cache->sum_wx = cache->sum_wxx = cache->sum_w1 = cache->sum_w1x = cache->sum_w1xx = 0;
|
||||
for (int i = 0; i < 8; ++i) {
|
||||
const double w = (double)weight[i], x = (double)xval[i], abs_w = fabs(w), abs_x = fabs(x);
|
||||
cache->sum_w += abs_w; cache->sum_wx += abs_w*abs_x; cache->sum_wxx += abs_w*abs_x*abs_x;
|
||||
cache->sum_w1 += 1 + abs_w; cache->sum_w1x += (1 + abs_w)*abs_x; cache->sum_w1xx += (1 + abs_w)*abs_x*abs_x;
|
||||
for (int q = 0; q < 8; ++q) {
|
||||
cache->lin_term[i][q] = 2*w*x*q;
|
||||
cache->quad_term[i][q] = w*q*q;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
static int iq2_count_at_most(const double * GGML_RESTRICT values, int n, double limit) {
|
||||
int count = 0;
|
||||
for (int j = 0; j < n; ++j) count += values[j] <= limit;
|
||||
return count;
|
||||
}
|
||||
|
||||
static int iq2_find_best_neighbour_cached(iq2_neighbour_cache * cache, int pattern, const uint16_t * GGML_RESTRICT neighbours, const uint64_t * GGML_RESTRICT grid,
|
||||
const float * GGML_RESTRICT xval, const float * GGML_RESTRICT weight, float scale, int8_t * GGML_RESTRICT L) {
|
||||
const int num_neighbors = neighbours[0];
|
||||
const double scale_d = (double)scale;
|
||||
const double max_level = 7*scale_d;
|
||||
// size of the numbers in the exact search, its rounding error is a multiple of 2^-24*size (+ size_subnormal for underflow)
|
||||
const double size = max_level*max_level*cache->sum_w + 2*max_level*cache->sum_wx + cache->sum_wxx;
|
||||
const double size_subnormal = (1 + max_level)*(1 + max_level)*cache->sum_w1 + 2*(1 + max_level)*cache->sum_w1x + cache->sum_w1xx;
|
||||
// the cached score is (exact score - const)/s, so the 2^-18*size margin on the exact score is divided by s
|
||||
const double tolerance = (0x1p-18*size + 0x1p-100*size_subnormal)/scale_d;
|
||||
if (num_neighbors <= 0 || num_neighbors > IQ2_NEIGHBOUR_CACHE_MAX || !(scale_d > 0 && size < 0x1p100 && size_subnormal < 0x1p100 && tolerance > 0)) {
|
||||
return iq2_find_best_neighbour(neighbours, grid, xval, weight, scale, L);
|
||||
}
|
||||
if (cache->pattern != pattern) {
|
||||
for (int j = 0; j < num_neighbors; ++j) {
|
||||
const int8_t * levels = (const int8_t *)(grid + neighbours[j+1]);
|
||||
double lin = 0, quad = 0;
|
||||
for (int i = 0; i < 8; ++i) {
|
||||
lin += cache->lin_term[i][levels[i]];
|
||||
quad += cache->quad_term[i][levels[i]];
|
||||
}
|
||||
cache->lin_coef[j] = lin; cache->quad_coef[j] = quad;
|
||||
}
|
||||
cache->pattern = pattern;
|
||||
cache->last_best = 0;
|
||||
}
|
||||
double score[IQ2_NEIGHBOUR_CACHE_MAX];
|
||||
for (int j = 0; j < num_neighbors; ++j) score[j] = scale_d*cache->quad_coef[j] - cache->lin_coef[j];
|
||||
// the winner must be the only neighbour within tolerance, try the previous winner for this pattern first
|
||||
int best = cache->last_best;
|
||||
if (iq2_count_at_most(score, num_neighbors, score[best] + tolerance) != 1) {
|
||||
for (int j = 0; j < num_neighbors; ++j) {
|
||||
if (score[j] < score[best]) best = j;
|
||||
}
|
||||
if (iq2_count_at_most(score, num_neighbors, score[best] + tolerance) != 1) {
|
||||
return iq2_find_best_neighbour(neighbours, grid, xval, weight, scale, L);
|
||||
}
|
||||
cache->last_best = best;
|
||||
}
|
||||
const int grid_index = neighbours[best+1];
|
||||
const int8_t * levels = (const int8_t *)(grid + grid_index);
|
||||
for (int i = 0; i < 8; ++i) L[i] = (levels[i] - 1)/2;
|
||||
return grid_index;
|
||||
}
|
||||
|
||||
static void quantize_row_iq2_xxs_impl(const float * GGML_RESTRICT x, void * GGML_RESTRICT vy, int64_t n, const float * GGML_RESTRICT quant_weights) {
|
||||
|
||||
const int gindex = iq2_data_index(GGML_TYPE_IQ2_XXS);
|
||||
@@ -3335,6 +3414,8 @@ static void quantize_row_iq2_xxs_impl(const float * GGML_RESTRICT x, void * GGML
|
||||
float waux[32];
|
||||
uint8_t block_signs[4];
|
||||
uint32_t q2[2*(QK_K/32)];
|
||||
iq2_neighbour_cache neighbour_cache[4];
|
||||
int8_t Laux_prev[32];
|
||||
|
||||
for (int ibl = 0; ibl < nbl; ++ibl) {
|
||||
|
||||
@@ -3390,6 +3471,7 @@ static void quantize_row_iq2_xxs_impl(const float * GGML_RESTRICT x, void * GGML
|
||||
continue;
|
||||
}
|
||||
float best = 0;
|
||||
for (int k = 0; k < 4; ++k) iq2_neighbour_cache_init(&neighbour_cache[k], xval + 8*k, waux + 8*k);
|
||||
for (int is = -6; is <= 6; ++is) {
|
||||
float id = (2*kMaxQ-1+is*0.1f)/eff_max;
|
||||
float this_scale = 1/id;
|
||||
@@ -3403,9 +3485,14 @@ static void quantize_row_iq2_xxs_impl(const float * GGML_RESTRICT x, void * GGML
|
||||
int grid_index = kmap_q2xs[u];
|
||||
if (grid_index < 0) {
|
||||
const uint16_t * neighbours = kneighbors_q2xs - kmap_q2xs[u] - 1;
|
||||
grid_index = iq2_find_best_neighbour(neighbours, kgrid_q2xs, xval + 8*k, waux + 8*k, this_scale, Laux + 8*k);
|
||||
grid_index = iq2_find_best_neighbour_cached(&neighbour_cache[k], u, neighbours, kgrid_q2xs, xval + 8*k, waux + 8*k, this_scale, Laux + 8*k);
|
||||
}
|
||||
}
|
||||
// same candidate as for the previous scale: it cannot change the result
|
||||
if (is > -6 && memcmp(Laux, Laux_prev, 32) == 0) {
|
||||
continue;
|
||||
}
|
||||
memcpy(Laux_prev, Laux, 32);
|
||||
float sumqx = 0, sumq2 = 0;
|
||||
for (int i = 0; i < 32; ++i) {
|
||||
float w = weight[i];
|
||||
@@ -3430,7 +3517,7 @@ static void quantize_row_iq2_xxs_impl(const float * GGML_RESTRICT x, void * GGML
|
||||
int grid_index = kmap_q2xs[u];
|
||||
if (grid_index < 0) {
|
||||
const uint16_t * neighbours = kneighbors_q2xs - kmap_q2xs[u] - 1;
|
||||
grid_index = iq2_find_best_neighbour(neighbours, kgrid_q2xs, xval + 8*k, waux + 8*k, scale, L + 8*k);
|
||||
grid_index = iq2_find_best_neighbour_cached(&neighbour_cache[k], u, neighbours, kgrid_q2xs, xval + 8*k, waux + 8*k, scale, L + 8*k);
|
||||
}
|
||||
const int8_t * pg = (const int8_t *)(kgrid_q2xs + grid_index);
|
||||
for (int i = 0; i < 8; ++i) L[8*k+i] = (pg[i] - 1)/2;
|
||||
@@ -3515,6 +3602,9 @@ static void quantize_row_iq2_xs_impl(const float * GGML_RESTRICT x, void * GGML_
|
||||
bool is_on_grid_aux[2];
|
||||
uint8_t block_signs[2];
|
||||
uint16_t q2[2*(QK_K/16)];
|
||||
iq2_neighbour_cache neighbour_cache[2];
|
||||
int8_t Laux_prev[16];
|
||||
bool is_on_grid_prev[2];
|
||||
|
||||
for (int ibl = 0; ibl < nbl; ++ibl) {
|
||||
|
||||
@@ -3566,6 +3656,7 @@ static void quantize_row_iq2_xs_impl(const float * GGML_RESTRICT x, void * GGML_
|
||||
float best = 0;
|
||||
float scale = max/(2*kMaxQ-1);
|
||||
is_on_grid[0] = is_on_grid[1] = true;
|
||||
for (int k = 0; k < 2; ++k) iq2_neighbour_cache_init(&neighbour_cache[k], xval + 8*k, waux + 8*k);
|
||||
for (int is = -9; is <= 9; ++is) {
|
||||
float id = (2*kMaxQ-1+is*0.1f)/max;
|
||||
float this_scale = 1/id;
|
||||
@@ -3581,9 +3672,15 @@ static void quantize_row_iq2_xs_impl(const float * GGML_RESTRICT x, void * GGML_
|
||||
if (grid_index < 0) {
|
||||
is_on_grid_aux[k] = false;
|
||||
const uint16_t * neighbours = kneighbors_q2xs - kmap_q2xs[u] - 1;
|
||||
grid_index = iq2_find_best_neighbour(neighbours, kgrid_q2xs, xval + 8*k, waux + 8*k, this_scale, Laux + 8*k);
|
||||
grid_index = iq2_find_best_neighbour_cached(&neighbour_cache[k], u, neighbours, kgrid_q2xs, xval + 8*k, waux + 8*k, this_scale, Laux + 8*k);
|
||||
}
|
||||
}
|
||||
// same candidate as for the previous scale: it cannot change the result
|
||||
if (is > -9 && memcmp(Laux, Laux_prev, 16) == 0 && is_on_grid_aux[0] == is_on_grid_prev[0] && is_on_grid_aux[1] == is_on_grid_prev[1]) {
|
||||
continue;
|
||||
}
|
||||
memcpy(Laux_prev, Laux, 16);
|
||||
is_on_grid_prev[0] = is_on_grid_aux[0]; is_on_grid_prev[1] = is_on_grid_aux[1];
|
||||
float sumqx = 0, sumq2 = 0;
|
||||
for (int i = 0; i < 16; ++i) {
|
||||
float w = weight[i];
|
||||
@@ -3613,7 +3710,7 @@ static void quantize_row_iq2_xs_impl(const float * GGML_RESTRICT x, void * GGML_
|
||||
int grid_index = kmap_q2xs[u];
|
||||
if (grid_index < 0) {
|
||||
const uint16_t * neighbours = kneighbors_q2xs - kmap_q2xs[u] - 1;
|
||||
grid_index = iq2_find_best_neighbour(neighbours, kgrid_q2xs, xval + 8*k, waux + 8*k, scale, L + 8*k);
|
||||
grid_index = iq2_find_best_neighbour_cached(&neighbour_cache[k], u, neighbours, kgrid_q2xs, xval + 8*k, waux + 8*k, scale, L + 8*k);
|
||||
}
|
||||
}
|
||||
float sumqx = 0, sumq2 = 0;
|
||||
@@ -5155,6 +5252,9 @@ static void quantize_row_iq2_s_impl(const float * GGML_RESTRICT x, void * GGML_R
|
||||
bool is_on_grid[2];
|
||||
bool is_on_grid_aux[2];
|
||||
uint8_t block_signs[2];
|
||||
iq2_neighbour_cache neighbour_cache[2];
|
||||
int8_t Laux_prev[16];
|
||||
bool is_on_grid_prev[2];
|
||||
|
||||
for (int ibl = 0; ibl < nbl; ++ibl) {
|
||||
|
||||
@@ -5197,6 +5297,7 @@ static void quantize_row_iq2_s_impl(const float * GGML_RESTRICT x, void * GGML_R
|
||||
float best = 0;
|
||||
float scale = max/(2*kMaxQ-1);
|
||||
is_on_grid[0] = is_on_grid[1] = true;
|
||||
for (int k = 0; k < 2; ++k) iq2_neighbour_cache_init(&neighbour_cache[k], xval + 8*k, waux + 8*k);
|
||||
for (int is = -9; is <= 9; ++is) {
|
||||
float id = (2*kMaxQ-1+is*0.1f)/max;
|
||||
float this_scale = 1/id;
|
||||
@@ -5212,9 +5313,15 @@ static void quantize_row_iq2_s_impl(const float * GGML_RESTRICT x, void * GGML_R
|
||||
if (grid_index < 0) {
|
||||
is_on_grid_aux[k] = false;
|
||||
const uint16_t * neighbours = kneighbors_q2xs - kmap_q2xs[u] - 1;
|
||||
grid_index = iq2_find_best_neighbour(neighbours, kgrid_q2xs, xval + 8*k, waux + 8*k, this_scale, Laux + 8*k);
|
||||
grid_index = iq2_find_best_neighbour_cached(&neighbour_cache[k], u, neighbours, kgrid_q2xs, xval + 8*k, waux + 8*k, this_scale, Laux + 8*k);
|
||||
}
|
||||
}
|
||||
// same candidate as for the previous scale: it cannot change the result
|
||||
if (is > -9 && memcmp(Laux, Laux_prev, 16) == 0 && is_on_grid_aux[0] == is_on_grid_prev[0] && is_on_grid_aux[1] == is_on_grid_prev[1]) {
|
||||
continue;
|
||||
}
|
||||
memcpy(Laux_prev, Laux, 16);
|
||||
is_on_grid_prev[0] = is_on_grid_aux[0]; is_on_grid_prev[1] = is_on_grid_aux[1];
|
||||
float sumqx = 0, sumq2 = 0;
|
||||
for (int i = 0; i < 16; ++i) {
|
||||
float w = weight[i];
|
||||
@@ -5244,7 +5351,7 @@ static void quantize_row_iq2_s_impl(const float * GGML_RESTRICT x, void * GGML_R
|
||||
int grid_index = kmap_q2xs[u];
|
||||
if (grid_index < 0) {
|
||||
const uint16_t * neighbours = kneighbors_q2xs - kmap_q2xs[u] - 1;
|
||||
grid_index = iq2_find_best_neighbour(neighbours, kgrid_q2xs, xval + 8*k, waux + 8*k, scale, L + 8*k);
|
||||
grid_index = iq2_find_best_neighbour_cached(&neighbour_cache[k], u, neighbours, kgrid_q2xs, xval + 8*k, waux + 8*k, scale, L + 8*k);
|
||||
}
|
||||
}
|
||||
float sumqx = 0, sumq2 = 0;
|
||||
|
||||
Reference in new issue
Block a user