Files
llama.cpp/ggml/src/ggml-hexagon/htp-repack.cpp
T
Todor BoinovskiandMax Krasnyansky 797de63b2f hexagon: add support for Q1_0 and Q2_0 (#30122)
* hexagon: add support for q1_0 quant

* hexagon: enable Q1_0 weights in the 2D HMX MUL_MAT path

* hex-q1: cleanup gates, generalize tiling and remove in-loop divs

* hex-tiling: document tiled formats in developer.md and update htp-tensor.h

* hex-q2: add support for Q2_0

* hex-q2: remove stale asserts and speacial cases

* hex-mm: fix MUL_MAT_NX assumption about square tiling, Q1/Q2 have non-square tiles

* hex-q1/2: minor optimizations (unrolls/un-unrolls/...)

* hex-q1: optimize Q1_0 layout to reduce unpack/vrmpy overhead

* hex-q1/q2_0: further optimize vec_dots

---------

Co-authored-by: Max Krasnyansky <maxk@qti.qualcomm.com>
2026-10-17 13:36:38 -07:00

1196 lines
53 KiB
C++

#include "htp-repack.h"
#define GGML_COMMON_IMPL_CPP
#include "ggml-common.h"
#include "ggml-impl.h"
#include "ggml-quants.h"
#include <algorithm>
#include <cmath>
#include <cstring>
#include <initializer_list>
// --- 4-bit block-32 nibble helpers (shared by Q4_0, IQ4_NL, Q4_1, MXFP4) ---
static inline void unpack_nibbles_32(uint8_t * dst, const uint8_t * qs) {
for (unsigned int i = 0; i < 16; ++i) {
dst[i + 0] = qs[i] & 0x0F;
dst[i + 16] = qs[i] >> 4;
}
}
static inline void pack_nibbles_32(uint8_t * qs, const uint8_t * src) {
for (unsigned int i = 0; i < 16; ++i) {
qs[i] = src[i + 0] | (src[i + 16] << 4);
}
}
static inline void transpose_nibbles_to_tile(uint8_t * tile_dst, const uint8_t tile_quants[32][32]) {
for (int cp = 0; cp < 16; cp++) {
for (int row = 0; row < 32; row++) {
tile_dst[cp * 32 + row] = (tile_quants[row][2 * cp + 1] << 4) | tile_quants[row][2 * cp];
}
}
}
static inline void transpose_tile_to_nibbles(uint8_t tile_quants[32][32], const uint8_t * tile_src) {
for (int cp = 0; cp < 16; cp++) {
for (int row = 0; row < 32; row++) {
const uint8_t val = tile_src[cp * 32 + row];
tile_quants[row][2 * cp + 0] = val & 0x0F;
tile_quants[row][2 * cp + 1] = val >> 4;
}
}
}
// --- Generic 32x32 tile grid walkers for block-32 and float types ---
template <typename SrcElemT, int ELEMS_PER_KTILE, typename PackTileFn>
static void repack_to_tiled_grid_32(ggml_tensor * t, const void * data, size_t offset, size_t size,
size_t tile_size, PackTileFn && pack_tile) {
const SrcElemT * src_matrix = (const SrcElemT *) data;
const int64_t ne0 = t->ne[0];
const int64_t ne1 = t->ne[1];
const int64_t ne2 = t->ne[2];
const int64_t ne3 = t->ne[3];
const int n_col_tiles = hex_round_up(ne1, 32) / 32;
const int n_k_tiles = hex_round_up(ne0, 32) / 32;
const size_t matrix_size = (size_t) n_col_tiles * n_k_tiles * tile_size;
const size_t slice_size = (size_t) ne1 * ggml_row_size(t->type, ne0);
const int64_t start_slice = offset / slice_size;
int64_t end_slice = (offset + size + slice_size - 1) / slice_size;
if (end_slice > ne2 * ne3) {
end_slice = ne2 * ne3;
}
const int64_t row_stride = (ne0 / (32 / ELEMS_PER_KTILE));
for (int64_t slice_idx = start_slice; slice_idx < end_slice; slice_idx++) {
const SrcElemT * src_slice = src_matrix + (slice_idx - start_slice) * (ne1 * row_stride);
uint8_t * matrix_dst = (uint8_t *) t->data + slice_idx * matrix_size;
for (int ct = 0; ct < n_col_tiles; ct++) {
for (int kt = 0; kt < n_k_tiles; kt++) {
uint8_t * tile_dst = matrix_dst + ((size_t) ct * n_k_tiles + kt) * tile_size;
pack_tile(tile_dst, src_slice, ne0, ne1, row_stride, ct, kt);
}
}
}
}
template <typename DstElemT, int ELEMS_PER_KTILE, typename UnpackTileFn>
static void unpack_from_tiled_grid_32(void * data, const ggml_tensor * t, size_t offset, size_t size,
size_t tile_size, UnpackTileFn && unpack_tile) {
uint8_t * dst_bytes = (uint8_t *) data;
const int64_t ne0 = t->ne[0];
const int64_t ne1 = t->ne[1];
const int64_t ne2 = t->ne[2];
const int64_t ne3 = t->ne[3];
const int n_col_tiles = hex_round_up(ne1, 32) / 32;
const int n_k_tiles = hex_round_up(ne0, 32) / 32;
const size_t matrix_size = (size_t) n_col_tiles * n_k_tiles * tile_size;
const size_t row_size_bytes = ggml_row_size(t->type, ne0);
const size_t slice_size = (size_t) ne1 * row_size_bytes;
const int64_t start_slice = offset / slice_size;
int64_t end_slice = (offset + size + slice_size - 1) / slice_size;
if (end_slice > ne2 * ne3) {
end_slice = ne2 * ne3;
}
const int64_t row_stride = (ne0 / (32 / ELEMS_PER_KTILE));
for (int64_t slice_idx = start_slice; slice_idx < end_slice; slice_idx++) {
const size_t cur_start_byte = (std::max)(offset, (size_t) slice_idx * slice_size);
const size_t cur_end_byte = (std::min)(offset + size, (size_t) (slice_idx + 1) * slice_size);
const size_t slice_offset_start = cur_start_byte - (size_t) slice_idx * slice_size;
const size_t slice_offset_end = cur_end_byte - (size_t) slice_idx * slice_size;
const int64_t start_row = slice_offset_start / row_size_bytes;
const int64_t end_row = (std::min)((int64_t) ((slice_offset_end + row_size_bytes - 1) / row_size_bytes), ne1);
const int start_ct = (int) (start_row / 32);
const int end_ct = (std::min)((int) ((end_row + 31) / 32), n_col_tiles);
DstElemT * dst_slice = (DstElemT *) (dst_bytes + (cur_start_byte - offset));
const uint8_t * matrix_src = (const uint8_t *) t->data + slice_idx * matrix_size;
for (int ct = start_ct; ct < end_ct; ct++) {
for (int kt = 0; kt < n_k_tiles; kt++) {
const uint8_t * tile_src = matrix_src + ((size_t) ct * n_k_tiles + kt) * tile_size;
unpack_tile(dst_slice, tile_src, ne0, start_row, end_row, row_stride, ct, kt);
}
}
}
}
// --- Block-32 Quantized Types (Q4_0, IQ4_NL, Q4_1, Q8_0, MXFP4) ---
static void repack_q4_0_tiled(ggml_tensor * t, const void * data, size_t offset, size_t size) {
repack_to_tiled_grid_32<block_q4_0, 1>(t, data, offset, size, HTP_TENSOR_TILE_SIZE_Q4_0,
[](uint8_t * tile_dst, const block_q4_0 * src_slice, int64_t ne0, int64_t ne1, int64_t row_stride, int ct, int kt) {
uint8_t tile_quants[32][32];
ggml_half * scale_dst = (ggml_half *) (tile_dst + 512);
for (int row = 0; row < 32; row++) {
const int64_t r = ct * 32 + row;
if (r < ne1 && kt < ne0 / 32) {
const block_q4_0 & b = src_slice[r * row_stride + kt];
unpack_nibbles_32(tile_quants[row], b.qs);
scale_dst[row] = b.d;
} else {
memset(tile_quants[row], 8, 32);
scale_dst[row] = 0;
}
}
transpose_nibbles_to_tile(tile_dst, tile_quants);
});
}
static void repack_tiled_q4_0(void * data, const ggml_tensor * t, size_t offset, size_t size) {
unpack_from_tiled_grid_32<block_q4_0, 1>(data, t, offset, size, HTP_TENSOR_TILE_SIZE_Q4_0,
[](block_q4_0 * dst_slice, const uint8_t * tile_src, int64_t ne0, int64_t start_row, int64_t end_row, int64_t row_stride, int ct, int kt) {
uint8_t tile_quants[32][32];
transpose_tile_to_nibbles(tile_quants, tile_src);
const ggml_half * scale_src = (const ggml_half *) (tile_src + 512);
for (int row = 0; row < 32; row++) {
const int64_t r = ct * 32 + row;
if (r >= start_row && r < end_row && kt < ne0 / 32) {
block_q4_0 & b = dst_slice[(r - start_row) * row_stride + kt];
pack_nibbles_32(b.qs, tile_quants[row]);
b.d = scale_src[row];
}
}
});
}
static void repack_q4_1_tiled(ggml_tensor * t, const void * data, size_t offset, size_t size) {
repack_to_tiled_grid_32<block_q4_1, 1>(t, data, offset, size, HTP_TENSOR_TILE_SIZE_Q4_1,
[](uint8_t * tile_dst, const block_q4_1 * src_slice, int64_t ne0, int64_t ne1, int64_t row_stride, int ct, int kt) {
uint8_t tile_quants[32][32];
ggml_half * scale_dst = (ggml_half *) (tile_dst + 512);
for (int row = 0; row < 32; row++) {
const int64_t r = ct * 32 + row;
if (r < ne1 && kt < ne0 / 32) {
const block_q4_1 & b = src_slice[r * row_stride + kt];
unpack_nibbles_32(tile_quants[row], b.qs);
scale_dst[2 * row + 0] = b.d;
scale_dst[2 * row + 1] = b.m;
} else {
memset(tile_quants[row], 0, 32);
scale_dst[2 * row + 0] = 0;
scale_dst[2 * row + 1] = 0;
}
}
transpose_nibbles_to_tile(tile_dst, tile_quants);
});
}
static void repack_tiled_q4_1(void * data, const ggml_tensor * t, size_t offset, size_t size) {
unpack_from_tiled_grid_32<block_q4_1, 1>(data, t, offset, size, HTP_TENSOR_TILE_SIZE_Q4_1,
[](block_q4_1 * dst_slice, const uint8_t * tile_src, int64_t ne0, int64_t start_row, int64_t end_row, int64_t row_stride, int ct, int kt) {
uint8_t tile_quants[32][32];
transpose_tile_to_nibbles(tile_quants, tile_src);
const ggml_half * scale_src = (const ggml_half *) (tile_src + 512);
for (int row = 0; row < 32; row++) {
const int64_t r = ct * 32 + row;
if (r >= start_row && r < end_row && kt < ne0 / 32) {
block_q4_1 & b = dst_slice[(r - start_row) * row_stride + kt];
pack_nibbles_32(b.qs, tile_quants[row]);
b.d = scale_src[2 * row + 0];
b.m = scale_src[2 * row + 1];
}
}
});
}
static void repack_mxfp4_tiled(ggml_tensor * t, const void * data, size_t offset, size_t size) {
repack_to_tiled_grid_32<block_mxfp4, 1>(t, data, offset, size, HTP_TENSOR_TILE_SIZE_MXFP4,
[](uint8_t * tile_dst, const block_mxfp4 * src_slice, int64_t ne0, int64_t ne1, int64_t row_stride, int ct, int kt) {
uint8_t tile_quants[32][32];
uint8_t * scale_dst = tile_dst + 512;
for (int row = 0; row < 32; row++) {
const int64_t r = ct * 32 + row;
if (r < ne1 && kt < ne0 / 32) {
const block_mxfp4 & b = src_slice[r * row_stride + kt];
unpack_nibbles_32(tile_quants[row], b.qs);
scale_dst[row] = b.e;
} else {
memset(tile_quants[row], 0, 32);
scale_dst[row] = 0;
}
}
transpose_nibbles_to_tile(tile_dst, tile_quants);
});
}
static void repack_tiled_mxfp4(void * data, const ggml_tensor * t, size_t offset, size_t size) {
unpack_from_tiled_grid_32<block_mxfp4, 1>(data, t, offset, size, HTP_TENSOR_TILE_SIZE_MXFP4,
[](block_mxfp4 * dst_slice, const uint8_t * tile_src, int64_t ne0, int64_t start_row, int64_t end_row, int64_t row_stride, int ct, int kt) {
uint8_t tile_quants[32][32];
transpose_tile_to_nibbles(tile_quants, tile_src);
const uint8_t * scale_src = tile_src + 512;
for (int row = 0; row < 32; row++) {
const int64_t r = ct * 32 + row;
if (r >= start_row && r < end_row && kt < ne0 / 32) {
block_mxfp4 & b = dst_slice[(r - start_row) * row_stride + kt];
pack_nibbles_32(b.qs, tile_quants[row]);
b.e = scale_src[row];
}
}
});
}
static void repack_q8_0_tiled(ggml_tensor * t, const void * data, size_t offset, size_t size) {
repack_to_tiled_grid_32<block_q8_0, 1>(t, data, offset, size, HTP_TENSOR_TILE_SIZE_Q8_0,
[](uint8_t * tile_dst, const block_q8_0 * src_slice, int64_t ne0, int64_t ne1, int64_t row_stride, int ct, int kt) {
for (int cp = 0; cp < 16; cp++) {
const int col0 = cp * 2;
const int col1 = col0 + 1;
for (int row = 0; row < 32; row++) {
const int64_t r = ct * 32 + row;
const block_q8_0 * b = (r < ne1 && kt < ne0 / 32) ? &src_slice[r * row_stride + kt] : nullptr;
tile_dst[cp * 64 + 2 * row + 0] = b ? b->qs[col0] : 0;
tile_dst[cp * 64 + 2 * row + 1] = b ? b->qs[col1] : 0;
}
}
ggml_half * scale_dst = (ggml_half *) (tile_dst + 1024);
for (int row = 0; row < 32; row++) {
const int64_t r = ct * 32 + row;
scale_dst[row] = (r < ne1 && kt < ne0 / 32) ? src_slice[r * row_stride + kt].d : 0;
}
});
}
static void repack_tiled_q8_0(void * data, const ggml_tensor * t, size_t offset, size_t size) {
unpack_from_tiled_grid_32<block_q8_0, 1>(data, t, offset, size, HTP_TENSOR_TILE_SIZE_Q8_0,
[](block_q8_0 * dst_slice, const uint8_t * tile_src, int64_t ne0, int64_t start_row, int64_t end_row, int64_t row_stride, int ct, int kt) {
for (int cp = 0; cp < 16; cp++) {
const int col0 = cp * 2;
const int col1 = col0 + 1;
for (int row = 0; row < 32; row++) {
const int64_t r = ct * 32 + row;
if (r >= start_row && r < end_row && kt < ne0 / 32) {
block_q8_0 & b = dst_slice[(r - start_row) * row_stride + kt];
b.qs[col0] = tile_src[cp * 64 + 2 * row + 0];
b.qs[col1] = tile_src[cp * 64 + 2 * row + 1];
}
}
}
const ggml_half * scale_src = (const ggml_half *) (tile_src + 1024);
for (int row = 0; row < 32; row++) {
const int64_t r = ct * 32 + row;
if (r >= start_row && r < end_row && kt < ne0 / 32) {
dst_slice[(r - start_row) * row_stride + kt].d = scale_src[row];
}
}
});
}
// --- Float Types (F16, F32, BF16 -> f16_tiled HMX layout) ---
static void repack_q2_0_tiled(ggml_tensor * t, const void * data, size_t offset, size_t size) {
GGML_ASSERT(offset == 0);
GGML_ASSERT(t->ne[0] % 64 == 0);
GGML_ASSERT(size == ggml_nbytes(t));
const block_q2_0 * src = (const block_q2_0 *) data;
const int64_t blocks_per_row = t->ne[0] / 64;
const int64_t n_col_tiles = hex_round_up(t->ne[1], 32) / 32;
const int64_t n_weight_tiles = blocks_per_row;
const size_t matrix_size = (size_t) n_col_tiles * n_weight_tiles * HTP_TENSOR_TILE_SIZE_Q2_0;
for (int64_t slice = 0; slice < t->ne[2] * t->ne[3]; ++slice) {
const block_q2_0 * src_slice = src + slice * t->ne[1] * blocks_per_row;
uint8_t * dst_slice = (uint8_t *) t->data + slice * matrix_size;
memset(dst_slice, 0, matrix_size);
for (int64_t row_index = 0; row_index < t->ne[1]; ++row_index) {
const int64_t ct = row_index / 32;
const int row = (int) (row_index % 32);
for (int64_t kt = 0; kt < n_weight_tiles; ++kt) {
const block_q2_0 * block = src_slice + row_index * blocks_per_row + kt;
uint8_t * tile = dst_slice + (ct * n_weight_tiles + kt) * HTP_TENSOR_TILE_SIZE_Q2_0;
for (int group = 0; group < 16; ++group) {
for (int byte = 0; byte < 4; ++byte) {
const int k = 4 * group + byte;
const uint8_t code = (block->qs[k / 4] >> (2 * (k % 4))) & 3;
tile[(group >> 2) * 128 + 4 * row + byte] |= code << (2 * (group & 3));
}
}
memcpy(tile + 512 + 2 * row, &block->d, sizeof(block->d));
}
}
}
}
static void repack_tiled_q2_0(void * data, const ggml_tensor * t, size_t offset, size_t size) {
GGML_ASSERT(t->ne[0] % 64 == 0);
GGML_ASSERT(offset + size <= ggml_nbytes(t));
const size_t first_block = offset / sizeof(block_q2_0);
const size_t end_block = (offset + size + sizeof(block_q2_0) - 1) / sizeof(block_q2_0);
const size_t blocks_per_row = t->ne[0] / 64;
const size_t blocks_per_slice = t->ne[1] * blocks_per_row;
const size_t n_weight_tiles = blocks_per_row;
const size_t n_col_tiles = hex_round_up(t->ne[1], 32) / 32;
const size_t matrix_size = n_col_tiles * n_weight_tiles * HTP_TENSOR_TILE_SIZE_Q2_0;
for (size_t block_index = first_block; block_index < end_block; ++block_index) {
const size_t slice = block_index / blocks_per_slice;
const size_t in_slice = block_index % blocks_per_slice;
const size_t row_index = in_slice / blocks_per_row;
const size_t kt = in_slice % blocks_per_row;
const size_t ct = row_index / 32;
const size_t row = row_index % 32;
const uint8_t * tile = (const uint8_t *) t->data + slice * matrix_size +
(ct * n_weight_tiles + kt) * HTP_TENSOR_TILE_SIZE_Q2_0;
block_q2_0 block = {};
memcpy(&block.d, tile + 512 + 2 * row, sizeof(block.d));
for (int group = 0; group < 16; ++group) {
for (int byte = 0; byte < 4; ++byte) {
const uint8_t packed = tile[(group >> 2) * 128 + 4 * row + byte];
const uint8_t code = (packed >> (2 * (group & 3))) & 3;
block.qs[group] |= code << (2 * byte);
}
}
const size_t block_begin = block_index * sizeof(block_q2_0);
const size_t copy_begin = (std::max)(block_begin, offset);
const size_t copy_end = (std::min)(block_begin + sizeof(block_q2_0), offset + size);
memcpy((uint8_t *) data + copy_begin - offset, (const uint8_t *) &block + copy_begin - block_begin, copy_end - copy_begin);
}
}
template <typename SrcT, typename ToFp16Fn>
static void repack_float_to_f16_tiled(ggml_tensor * t, const void * data, size_t offset, size_t size, ToFp16Fn && to_fp16) {
repack_to_tiled_grid_32<SrcT, 32>(t, data, offset, size, HTP_TENSOR_TILE_SIZE_F16,
[&](uint8_t * tile_bytes, const SrcT * src_slice, int64_t ne0, int64_t ne1, int64_t row_stride, int ct, int kt) {
uint16_t * tile_dst = (uint16_t *) tile_bytes;
for (int cp = 0; cp < 16; cp++) {
const int64_t c0 = kt * 32 + cp * 2;
const int64_t c1 = c0 + 1;
for (int row = 0; row < 32; row++) {
const int64_t r = ct * 32 + row;
const SrcT * r_ptr = (r < ne1) ? &src_slice[r * row_stride] : nullptr;
tile_dst[cp * 64 + row * 2 + 0] = (r_ptr && c0 < ne0) ? to_fp16(r_ptr[c0]) : 0;
tile_dst[cp * 64 + row * 2 + 1] = (r_ptr && c1 < ne0) ? to_fp16(r_ptr[c1]) : 0;
}
}
});
}
template <typename DstT, typename FromFp16Fn>
static void unpack_f16_tiled_to_float(void * data, const ggml_tensor * t, size_t offset, size_t size, FromFp16Fn && from_fp16) {
unpack_from_tiled_grid_32<DstT, 32>(data, t, offset, size, HTP_TENSOR_TILE_SIZE_F16,
[&](DstT * dst_slice, const uint8_t * tile_bytes, int64_t ne0, int64_t start_row, int64_t end_row, int64_t row_stride, int ct, int kt) {
const uint16_t * tile_src = (const uint16_t *) tile_bytes;
for (int cp = 0; cp < 16; cp++) {
const int64_t c0 = kt * 32 + cp * 2;
const int64_t c1 = c0 + 1;
for (int row = 0; row < 32; row++) {
const int64_t r = ct * 32 + row;
if (r >= start_row && r < end_row) {
DstT * r_ptr = dst_slice + (r - start_row) * row_stride;
if (c0 < ne0) r_ptr[c0] = from_fp16(tile_src[cp * 64 + row * 2 + 0]);
if (c1 < ne0) r_ptr[c1] = from_fp16(tile_src[cp * 64 + row * 2 + 1]);
}
}
}
});
}
// --- Generic K-Quant (super-block 256) grid walkers ---
template <typename BlockT, typename PackKtFn>
static void repack_kquant_to_tiled_grid(ggml_tensor * t, const void * data, size_t offset, size_t size,
size_t tile_size, PackKtFn && pack_kt) {
GGML_ASSERT(offset == 0);
GGML_UNUSED(size);
const BlockT * src_matrix = (const BlockT *) data;
const int64_t ne0 = t->ne[0];
const int64_t ne1 = t->ne[1];
const int64_t ne2 = t->ne[2];
const int64_t ne3 = t->ne[3];
GGML_ASSERT(ne0 % QK_K == 0);
const int n_col_tiles = hex_round_up(ne1, 32) / 32;
const int n_k_tiles = hex_round_up(ne0, 32) / 32;
const size_t matrix_size = (size_t) n_col_tiles * n_k_tiles * tile_size;
const int64_t sb_per_row = ne0 / QK_K;
for (int64_t slice_idx = 0; slice_idx < ne2 * ne3; slice_idx++) {
const BlockT * src_slice = src_matrix + slice_idx * (ne1 * sb_per_row);
uint8_t * matrix_dst = (uint8_t *) t->data + slice_idx * matrix_size;
// Padding rows and OR-ed bit planes require zeroed tiles
memset(matrix_dst, 0, matrix_size);
for (int64_t r = 0; r < ne1; r++) {
const int ct = (int) (r / 32);
const int row = (int) (r % 32);
const BlockT * src_row = src_slice + r * sb_per_row;
for (int kt = 0; kt < n_k_tiles; kt++) {
uint8_t * tile = matrix_dst + ((size_t) ct * n_k_tiles + kt) * tile_size;
pack_kt(tile, &src_row[kt / 8], row, kt % 8);
}
}
}
}
template <typename BlockT, typename UnpackSbFn>
static void unpack_kquant_from_tiled_grid(void * data, const ggml_tensor * t, size_t offset, size_t size,
size_t tile_size, UnpackSbFn && unpack_sb) {
uint8_t * dst_bytes = (uint8_t *) data;
const int64_t ne0 = t->ne[0];
const int64_t ne1 = t->ne[1];
const int64_t ne2 = t->ne[2];
const int64_t ne3 = t->ne[3];
GGML_ASSERT(ne0 % QK_K == 0);
const int n_col_tiles = hex_round_up(ne1, 32) / 32;
const int n_k_tiles = hex_round_up(ne0, 32) / 32;
const size_t matrix_size = (size_t) n_col_tiles * n_k_tiles * tile_size;
const int64_t sb_per_row = ne0 / QK_K;
const size_t row_size_bytes = ggml_row_size(t->type, ne0);
const size_t slice_size = (size_t) ne1 * row_size_bytes;
const int64_t start_slice = offset / slice_size;
int64_t end_slice = (offset + size + slice_size - 1) / slice_size;
if (end_slice > ne2 * ne3) {
end_slice = ne2 * ne3;
}
for (int64_t slice_idx = start_slice; slice_idx < end_slice; slice_idx++) {
const size_t cur_start_byte = (std::max)(offset, (size_t) slice_idx * slice_size);
const size_t cur_end_byte = (std::min)(offset + size, (size_t) (slice_idx + 1) * slice_size);
const size_t slice_offset_start = cur_start_byte - (size_t) slice_idx * slice_size;
const size_t slice_offset_end = cur_end_byte - (size_t) slice_idx * slice_size;
const int64_t start_row = slice_offset_start / row_size_bytes;
const int64_t end_row = (std::min)((int64_t) ((slice_offset_end + row_size_bytes - 1) / row_size_bytes), ne1);
BlockT * dst_slice = (BlockT *) (dst_bytes + (cur_start_byte - offset));
const uint8_t * matrix_src = (const uint8_t *) t->data + slice_idx * matrix_size;
for (int64_t r = start_row; r < end_row; r++) {
const int ct = (int) (r / 32);
const int row = (int) (r % 32);
BlockT * dst_row = dst_slice + (r - start_row) * sb_per_row;
for (int64_t sb = 0; sb < sb_per_row; sb++) {
BlockT * b = &dst_row[sb];
memset(b, 0, sizeof(BlockT));
const uint8_t * sb_tiles = matrix_src + ((size_t) ct * n_k_tiles + sb * 8) * tile_size;
unpack_sb(b, sb_tiles, tile_size, row);
}
}
}
}
// --- K-Quant Helpers & Per-Type Implementations (Q2_K, Q3_K, Q4_K, Q5_K, Q6_K) ---
// unsigned 6-bit value (0..63) of element e of a Q6_K block, same bit layout as dequantize_row_q6_K
static inline uint8_t q6_K_get_quant(const block_q6_K * b, int e) {
const int c = e / 128;
const int w = e % 128;
const int g = w / 32;
const int l = w % 32;
const uint8_t * ql = b->ql + c * 64;
const uint8_t * qh = b->qh + c * 32;
uint8_t lo, hi;
switch (g) {
case 0: lo = ql[l] & 0xF; hi = (qh[l] >> 0) & 3; break;
case 1: lo = ql[l + 32] & 0xF; hi = (qh[l] >> 2) & 3; break;
case 2: lo = ql[l] >> 4; hi = (qh[l] >> 4) & 3; break;
default: lo = ql[l + 32] >> 4; hi = (qh[l] >> 6) & 3; break;
}
return (uint8_t) (lo | (hi << 4));
}
static void repack_q6_K_tiled(ggml_tensor * t, const void * data, size_t offset, size_t size) {
repack_kquant_to_tiled_grid<block_q6_K>(t, data, offset, size, HTP_TENSOR_TILE_SIZE_Q6_K,
[](uint8_t * tile, const block_q6_K * b, int row, int kt_local) {
const float d = GGML_FP16_TO_FP32(b->d);
uint8_t * lo_pl = tile;
uint8_t * hi_pl = tile + 512;
ggml_half * sc_pl = (ggml_half *) (tile + 768);
for (int lk = 0; lk < 32; lk++) {
const uint8_t q6 = q6_K_get_quant(b, kt_local * 32 + lk);
const int g = lk >> 2;
const int pos = row * 4 + (lk & 3);
lo_pl[(g >> 1) * 128 + pos] |= (uint8_t) ((q6 & 0xF) << ((g & 1) * 4));
hi_pl[(g >> 2) * 128 + pos] |= (uint8_t) ((q6 >> 4) << ((g & 3) * 2));
}
for (int sub = 0; sub < 2; sub++) {
sc_pl[sub * 32 + row] = GGML_FP32_TO_FP16(d * (float) b->scales[kt_local * 2 + sub]);
}
});
}
static void repack_tiled_q6_K(void * data, const ggml_tensor * t, size_t offset, size_t size) {
unpack_kquant_from_tiled_grid<block_q6_K>(data, t, offset, size, HTP_TENSOR_TILE_SIZE_Q6_K,
[](block_q6_K * b, const uint8_t * sb_tiles, size_t tile_size, int row) {
float sub_scales[16];
for (int kt_local = 0; kt_local < 8; kt_local++) {
const uint8_t * tile = sb_tiles + kt_local * tile_size;
const uint8_t * lo_pl = tile;
const uint8_t * hi_pl = tile + 512;
const ggml_half * sc_pl = (const ggml_half *) (tile + 768);
const int c = kt_local / 4;
const int g = kt_local % 4;
uint8_t * ql = b->ql + c * 64;
uint8_t * qh = b->qh + c * 32;
for (int lk = 0; lk < 32; lk++) {
const int g_tile = lk >> 2;
const int pos = row * 4 + (lk & 3);
const uint8_t lo = (lo_pl[(g_tile >> 1) * 128 + pos] >> ((g_tile & 1) * 4)) & 0xF;
const uint8_t hi = (hi_pl[(g_tile >> 2) * 128 + pos] >> ((g_tile & 3) * 2)) & 3;
switch (g) {
case 0: ql[lk] |= lo; qh[lk] |= (hi << 0); break;
case 1: ql[lk + 32] |= lo; qh[lk] |= (hi << 2); break;
case 2: ql[lk] |= (lo << 4); qh[lk] |= (hi << 4); break;
default: ql[lk + 32] |= (lo << 4); qh[lk] |= (hi << 6); break;
}
}
for (int sub = 0; sub < 2; sub++) {
sub_scales[kt_local * 2 + sub] = GGML_FP16_TO_FP32(sc_pl[sub * 32 + row]);
}
}
float max_abs_scale = 0.0f;
for (int s = 0; s < 16; s++) {
max_abs_scale = (std::max)(max_abs_scale, fabsf(sub_scales[s]));
}
if (max_abs_scale == 0.0f) {
b->d = GGML_FP32_TO_FP16(0.0f);
} else {
b->d = GGML_FP32_TO_FP16(max_abs_scale / 127.0f);
const float d_actual = GGML_FP16_TO_FP32(b->d);
const float inv_d = (d_actual != 0.0f) ? (1.0f / d_actual) : 0.0f;
for (int s = 0; s < 16; s++) {
const int sc = (int) roundf(sub_scales[s] * inv_d);
b->scales[s] = (int8_t) (std::max)(-128, (std::min)(127, sc));
}
}
});
}
// low 2 bits (0..3) of element e of a Q2_K or Q3_K block, same bit layout as dequantize_row_q2_K / q3_K
static inline uint8_t q2_3_K_get_low2(const uint8_t * qs, int e) {
const int c = e / 128;
const int j = (e % 128) / 32;
const int l = e % 32;
return (qs[c * 32 + l] >> (2 * j)) & 3;
}
// hmask bit of element e of a Q3_K block, same bit layout as dequantize_row_q3_K
static inline bool q3_K_get_hbit(const block_q3_K * b, int e) {
const int c = e / 128;
const int j = (e % 128) / 32;
const int l = e % 32;
return (b->hmask[l] >> (c * 4 + j)) & 1;
}
// signed 6-bit scale j (-32..31) of a Q3_K block, same packing as quantize_row_q3_K_ref
static inline int q3_K_get_scale(const uint8_t * scales, int j) {
const int lo = (j < 8) ? (scales[j] & 0xF) : (scales[j - 8] >> 4);
const int hi = (scales[8 + j % 4] >> (2 * (j / 4))) & 3;
return (lo | (hi << 4)) - 32;
}
// read-back: find fp16 d and l[j] in [lmin, lmax] with fp16(d * l[j]) == prod[j] for all j, false if none
static bool hexagon_recover_k_scales(const ggml_half * prod, int n, int lmin, int lmax, ggml_half * d_out, int * l_out) {
int jmax = 0;
for (int j = 1; j < n; j++) {
if (fabsf(GGML_FP16_TO_FP32(prod[j])) > fabsf(GGML_FP16_TO_FP32(prod[jmax]))) {
jmax = j;
}
}
const float pmax = GGML_FP16_TO_FP32(prod[jmax]);
if (pmax == 0.0f) {
*d_out = GGML_FP32_TO_FP16(0.0f);
for (int j = 0; j < n; j++) {
l_out[j] = 0;
}
return true;
}
// the quantizers put the largest scale at or near the range end, so try large |l| first
const int lext = (std::max)(-lmin, lmax);
for (int a = lext; a >= 1; a--) {
for (int sign : { 1, -1 }) {
const int lj = sign * a;
if (lj < lmin || lj > lmax) {
continue;
}
const ggml_half d0 = GGML_FP32_TO_FP16(pmax / (float) lj);
for (int ulp : { 0, -1, 1 }) {
ggml_half d = d0;
uint16_t bits;
memcpy(&bits, &d, sizeof(bits));
bits = (uint16_t) (bits + ulp);
memcpy(&d, &bits, sizeof(bits));
const float df = GGML_FP16_TO_FP32(d);
if (!std::isfinite(df) || df == 0.0f) {
continue;
}
bool ok = true;
for (int j = 0; j < n && ok; j++) {
const int l = (int) roundf(GGML_FP16_TO_FP32(prod[j]) / df);
const ggml_half p = GGML_FP32_TO_FP16(df * (float) l);
ok = l >= lmin && l <= lmax && memcmp(&p, &prod[j], sizeof(p)) == 0;
l_out[j] = l;
}
if (ok) {
*d_out = d;
return true;
}
}
}
}
return false;
}
static void repack_q3_K_tiled(ggml_tensor * t, const void * data, size_t offset, size_t size) {
repack_kquant_to_tiled_grid<block_q3_K>(t, data, offset, size, HTP_TENSOR_TILE_SIZE_Q3_K,
[](uint8_t * tile, const block_q3_K * b, int row, int kt_local) {
const float d = GGML_FP16_TO_FP32(b->d);
uint8_t * lo_pl = tile;
uint8_t * neg_pl = tile + 256;
ggml_half * sc_pl = (ggml_half *) (tile + 384);
for (int lk = 0; lk < 32; lk++) {
const int e = kt_local * 32 + lk;
const int g = lk >> 2;
const int pos = row * 4 + (lk & 3);
lo_pl[(g >> 2) * 128 + pos] |= (uint8_t) (q2_3_K_get_low2(b->qs, e) << ((g & 3) * 2));
if (!q3_K_get_hbit(b, e)) {
neg_pl[pos] |= (uint8_t) (1 << g);
}
}
for (int sub = 0; sub < 2; sub++) {
sc_pl[sub * 32 + row] = GGML_FP32_TO_FP16(d * (float) q3_K_get_scale(b->scales, kt_local * 2 + sub));
}
});
}
static void repack_tiled_q3_K(void * data, const ggml_tensor * t, size_t offset, size_t size) {
unpack_kquant_from_tiled_grid<block_q3_K>(data, t, offset, size, HTP_TENSOR_TILE_SIZE_Q3_K,
[](block_q3_K * b, const uint8_t * sb_tiles, size_t tile_size, int row) {
ggml_half sub_scales[16];
for (int kt_local = 0; kt_local < 8; kt_local++) {
const uint8_t * tile = sb_tiles + kt_local * tile_size;
const uint8_t * lo_pl = tile;
const uint8_t * neg_pl = tile + 256;
const ggml_half * sc_pl = (const ggml_half *) (tile + 384);
for (int lk = 0; lk < 32; lk++) {
const int e = kt_local * 32 + lk;
const int g = lk >> 2;
const int pos = row * 4 + (lk & 3);
const uint8_t lo = (lo_pl[(g >> 2) * 128 + pos] >> ((g & 3) * 2)) & 3;
const int c = e / 128;
const int j = (e % 128) / 32;
const int l = e % 32;
b->qs[c * 32 + l] |= (uint8_t) (lo << (2 * j));
if (!((neg_pl[pos] >> g) & 1)) {
b->hmask[l] |= (uint8_t) (1 << (c * 4 + j));
}
}
for (int sub = 0; sub < 2; sub++) {
sub_scales[kt_local * 2 + sub] = sc_pl[sub * 32 + row];
}
}
int ls[16];
if (!hexagon_recover_k_scales(sub_scales, 16, -32, 31, &b->d, ls)) {
float max_scale = 0.0f;
for (int s = 0; s < 16; s++) {
if (fabsf(GGML_FP16_TO_FP32(sub_scales[s])) > fabsf(max_scale)) {
max_scale = GGML_FP16_TO_FP32(sub_scales[s]);
}
}
b->d = GGML_FP32_TO_FP16(-max_scale / 32.0f);
const float d_actual = GGML_FP16_TO_FP32(b->d);
const float inv_d = (d_actual != 0.0f) ? (1.0f / d_actual) : 0.0f;
for (int s = 0; s < 16; s++) {
ls[s] = (std::max)(-32, (std::min)(31, (int) roundf(GGML_FP16_TO_FP32(sub_scales[s]) * inv_d)));
}
}
for (int s = 0; s < 16; s++) {
const int l = ls[s] + 32;
if (s < 8) {
b->scales[s] = l & 0xF;
} else {
b->scales[s - 8] |= (uint8_t) ((l & 0xF) << 4);
}
b->scales[s % 4 + 8] |= (uint8_t) ((l >> 4) << (2 * (s / 4)));
}
});
}
static void repack_q2_K_tiled(ggml_tensor * t, const void * data, size_t offset, size_t size) {
repack_kquant_to_tiled_grid<block_q2_K>(t, data, offset, size, HTP_TENSOR_TILE_SIZE_Q2_K,
[](uint8_t * tile, const block_q2_K * b, int row, int kt_local) {
uint8_t * lo_pl = tile;
uint8_t * sc_pl = tile + 256;
ggml_half * dd_pl = (ggml_half *) (tile + 320);
for (int lk = 0; lk < 32; lk++) {
const int g = lk >> 2;
const int pos = row * 4 + (lk & 3);
lo_pl[(g >> 2) * 128 + pos] |= (uint8_t) (q2_3_K_get_low2(b->qs, kt_local * 32 + lk) << ((g & 3) * 2));
}
for (int sub = 0; sub < 2; sub++) {
sc_pl[sub * 32 + row] = b->scales[kt_local * 2 + sub];
}
if (row / 8 == kt_local % 4) {
dd_pl[row % 8] = kt_local < 4 ? b->d : (ggml_half) (b->dmin ^ 0x8000);
}
});
}
static void repack_tiled_q2_K(void * data, const ggml_tensor * t, size_t offset, size_t size) {
unpack_kquant_from_tiled_grid<block_q2_K>(data, t, offset, size, HTP_TENSOR_TILE_SIZE_Q2_K,
[](block_q2_K * b, const uint8_t * sb_tiles, size_t tile_size, int row) {
for (int kt_local = 0; kt_local < 8; kt_local++) {
const uint8_t * tile = sb_tiles + kt_local * tile_size;
const uint8_t * lo_pl = tile;
const uint8_t * sc_pl = tile + 256;
const ggml_half * dd_pl = (const ggml_half *) (tile + 320);
for (int lk = 0; lk < 32; lk++) {
const int e = kt_local * 32 + lk;
const int g = lk >> 2;
const int pos = row * 4 + (lk & 3);
const uint8_t lo = (lo_pl[(g >> 2) * 128 + pos] >> ((g & 3) * 2)) & 3;
b->qs[(e / 128) * 32 + e % 32] |= (uint8_t) (lo << (2 * ((e % 128) / 32)));
}
for (int sub = 0; sub < 2; sub++) {
b->scales[kt_local * 2 + sub] = sc_pl[sub * 32 + row];
}
if (row / 8 == kt_local % 4) {
if (kt_local < 4) {
b->d = dd_pl[row % 8];
} else {
b->dmin = (ggml_half) (dd_pl[row % 8] ^ 0x8000);
}
}
}
});
}
static inline void get_scale_min_k4(int j, const uint8_t * q, uint8_t * d, uint8_t * m) {
if (j < 4) {
*d = q[j] & 63;
*m = q[j + 4] & 63;
} else {
*d = (q[j + 4] & 0xF) | ((q[j - 4] >> 6) << 4);
*m = (q[j + 4] >> 4) | ((q[j - 0] >> 6) << 4);
}
}
static inline void pack_k4_scales_mins(uint8_t * scales, ggml_half * d, ggml_half * dmin,
const float * sub_scales, const float * sub_mins) {
float max_scale = 0.0f;
float max_min = 0.0f;
for (int j = 0; j < 8; j++) {
if (sub_scales[j] > max_scale) max_scale = sub_scales[j];
if (sub_mins[j] > max_min) max_min = sub_mins[j];
}
float inv_scale = 0.0f;
if (max_scale > 0.0f) {
*d = GGML_FP32_TO_FP16(max_scale / 63.0f);
const float d_actual = GGML_FP16_TO_FP32(*d);
inv_scale = (d_actual > 0.0f) ? (1.0f / d_actual) : 0.0f;
} else {
*d = GGML_FP32_TO_FP16(0.0f);
}
float inv_min = 0.0f;
if (max_min > 0.0f) {
*dmin = GGML_FP32_TO_FP16(max_min / 63.0f);
const float dmin_actual = GGML_FP16_TO_FP32(*dmin);
inv_min = (dmin_actual > 0.0f) ? (1.0f / dmin_actual) : 0.0f;
} else {
*dmin = GGML_FP32_TO_FP16(0.0f);
}
for (int j = 0; j < 8; j++) {
uint8_t ls = (std::min)((uint8_t) 63, (uint8_t) roundf(inv_scale * sub_scales[j]));
uint8_t lm = (std::min)((uint8_t) 63, (uint8_t) roundf(inv_min * sub_mins[j]));
if (j < 4) {
scales[j] = ls;
scales[j + 4] = lm;
} else {
scales[j + 4] = (ls & 0xF) | ((lm & 0xF) << 4);
scales[j - 4] |= ((ls >> 4) << 6);
scales[j - 0] |= ((lm >> 4) << 6);
}
}
}
static void repack_q4_K_tiled(ggml_tensor * t, const void * data, size_t offset, size_t size) {
repack_kquant_to_tiled_grid<block_q4_K>(t, data, offset, size, HTP_TENSOR_TILE_SIZE_Q4_K,
[](uint8_t * tile_dst, const block_q4_K * b, int row, int kt_local) {
const float d = GGML_FP16_TO_FP32(b->d);
const float dmin = GGML_FP16_TO_FP32(b->dmin);
uint8_t sc, m;
get_scale_min_k4(kt_local, b->scales, &sc, &m);
const uint8_t * qs_sub = b->qs + (kt_local / 2) * 32;
const int shift = (kt_local & 1) ? 4 : 0;
for (int cp = 0; cp < 16; cp++) {
const uint8_t q0 = (qs_sub[2 * cp + 0] >> shift) & 0x0F;
const uint8_t q1 = (qs_sub[2 * cp + 1] >> shift) & 0x0F;
tile_dst[cp * 32 + row] = (uint8_t) ((q1 << 4) | q0);
}
ggml_half * scale_dst = (ggml_half *) (tile_dst + 512);
scale_dst[2 * row + 0] = GGML_FP32_TO_FP16(d * (float) sc);
scale_dst[2 * row + 1] = GGML_FP32_TO_FP16(-dmin * (float) m);
});
}
static void repack_tiled_q4_K(void * data, const ggml_tensor * t, size_t offset, size_t size) {
unpack_kquant_from_tiled_grid<block_q4_K>(data, t, offset, size, HTP_TENSOR_TILE_SIZE_Q4_K,
[](block_q4_K * b, const uint8_t * sb_tiles, size_t tile_size, int row) {
float sub_scales[8];
float sub_mins[8];
for (int kt_local = 0; kt_local < 8; kt_local++) {
const uint8_t * tile_src = sb_tiles + kt_local * tile_size;
const ggml_half * scale_src = (const ggml_half *) (tile_src + 512);
uint8_t * qs_sub = b->qs + (kt_local / 2) * 32;
const int shift = (kt_local & 1) ? 4 : 0;
for (int cp = 0; cp < 16; cp++) {
const uint8_t val = tile_src[cp * 32 + row];
qs_sub[2 * cp + 0] |= (uint8_t) ((val & 0x0F) << shift);
qs_sub[2 * cp + 1] |= (uint8_t) ((val >> 4) << shift);
}
const float D = GGML_FP16_TO_FP32(scale_src[2 * row + 0]);
const float M = GGML_FP16_TO_FP32(scale_src[2 * row + 1]);
sub_scales[kt_local] = (D > 0.0f) ? D : 0.0f;
sub_mins[kt_local] = (-M > 0.0f) ? -M : 0.0f;
}
pack_k4_scales_mins(b->scales, &b->d, &b->dmin, sub_scales, sub_mins);
});
}
static void repack_q5_K_tiled(ggml_tensor * t, const void * data, size_t offset, size_t size) {
repack_kquant_to_tiled_grid<block_q5_K>(t, data, offset, size, HTP_TENSOR_TILE_SIZE_Q5_K,
[](uint8_t * tile_dst, const block_q5_K * b, int row, int kt_local) {
const float d = GGML_FP16_TO_FP32(b->d);
const float dmin = GGML_FP16_TO_FP32(b->dmin);
uint8_t * plane = tile_dst + 640;
uint8_t sc, m;
get_scale_min_k4(kt_local, b->scales, &sc, &m);
const uint8_t * qs_sub = b->qs + (kt_local / 2) * 32;
const int shift = (kt_local & 1) ? 4 : 0;
const uint8_t hbit = (uint8_t) (1 << kt_local);
for (int cp = 0; cp < 16; cp++) {
const uint8_t q0 = (qs_sub[2 * cp + 0] >> shift) & 0x0F;
const uint8_t q1 = (qs_sub[2 * cp + 1] >> shift) & 0x0F;
tile_dst[cp * 32 + row] = (uint8_t) ((q1 << 4) | q0);
const int i = cp / 4;
const int lane = (cp % 4) * 32 + row;
if (b->qh[2 * cp + 0] & hbit) {
plane[lane] |= (uint8_t) (1 << (2 * i));
}
if (b->qh[2 * cp + 1] & hbit) {
plane[lane] |= (uint8_t) (1 << (2 * i + 1));
}
}
ggml_half * scale_dst = (ggml_half *) (tile_dst + 512);
scale_dst[2 * row + 0] = GGML_FP32_TO_FP16(d * (float) sc);
scale_dst[2 * row + 1] = GGML_FP32_TO_FP16(-dmin * (float) m);
});
}
static void repack_tiled_q5_K(void * data, const ggml_tensor * t, size_t offset, size_t size) {
unpack_kquant_from_tiled_grid<block_q5_K>(data, t, offset, size, HTP_TENSOR_TILE_SIZE_Q5_K,
[](block_q5_K * b, const uint8_t * sb_tiles, size_t tile_size, int row) {
float sub_scales[8];
float sub_mins[8];
for (int kt_local = 0; kt_local < 8; kt_local++) {
const uint8_t * tile_src = sb_tiles + kt_local * tile_size;
const uint8_t * plane = tile_src + 640;
const ggml_half * scale_src = (const ggml_half *) (tile_src + 512);
uint8_t * qs_sub = b->qs + (kt_local / 2) * 32;
const int shift = (kt_local & 1) ? 4 : 0;
const uint8_t hbit = (uint8_t) (1 << kt_local);
for (int cp = 0; cp < 16; cp++) {
const uint8_t val = tile_src[cp * 32 + row];
qs_sub[2 * cp + 0] |= (uint8_t) ((val & 0x0F) << shift);
qs_sub[2 * cp + 1] |= (uint8_t) ((val >> 4) << shift);
const int i = cp / 4;
const int lane = (cp % 4) * 32 + row;
if (plane[lane] & (1 << (2 * i))) {
b->qh[2 * cp + 0] |= hbit;
}
if (plane[lane] & (1 << (2 * i + 1))) {
b->qh[2 * cp + 1] |= hbit;
}
}
const float D = GGML_FP16_TO_FP32(scale_src[2 * row + 0]);
const float M = GGML_FP16_TO_FP32(scale_src[2 * row + 1]);
sub_scales[kt_local] = (D > 0.0f) ? D : 0.0f;
sub_mins[kt_local] = (-M > 0.0f) ? -M : 0.0f;
}
pack_k4_scales_mins(b->scales, &b->d, &b->dmin, sub_scales, sub_mins);
});
}
static inline void q1_0_tiled_bit_location(int k, int row, int & byte_offset, int & bit) {
const int quarter = k / 32;
const int kq = k % 32;
byte_offset = quarter * 128 + 4 * row + (kq % 4);
bit = kq / 4;
}
static void repack_q1_0_tiled(ggml_tensor * t, const void * data, size_t offset, size_t size) {
const block_q1_0 * src_matrix = (const block_q1_0 *) data;
const int64_t ne0 = t->ne[0];
const int64_t ne1 = t->ne[1];
const int64_t ne2 = t->ne[2];
const int64_t ne3 = t->ne[3];
const int n_col_tiles = hex_round_up(ne1, 32) / 32;
const int n_k_tiles = ne0 / QK1_0;
const size_t tile_size = HTP_TENSOR_TILE_SIZE_Q1_0;
const size_t matrix_size = (size_t) n_col_tiles * n_k_tiles * tile_size;
const size_t slice_size = ne1 * ggml_row_size(t->type, ne0);
const int64_t start_slice = offset / slice_size;
const int64_t end_slice = (std::min)((int64_t) ((offset + size + slice_size - 1) / slice_size), ne2 * ne3);
GGML_ASSERT(ne0 % QK1_0 == 0);
for (int64_t slice_idx = start_slice; slice_idx < end_slice; slice_idx++) {
const block_q1_0 * src_slice = src_matrix + (slice_idx - start_slice) * ne1 * n_k_tiles;
uint8_t * matrix_dst = (uint8_t *) t->data + slice_idx * matrix_size;
for (int ct = 0; ct < n_col_tiles; ct++) {
for (int kt = 0; kt < n_k_tiles; kt++) {
uint8_t * tile_dst = matrix_dst + ((size_t) ct * n_k_tiles + kt) * tile_size;
ggml_half * scale_dst = (ggml_half *) (tile_dst + 512);
memset(tile_dst, 0, tile_size);
for (int row = 0; row < 32; row++) {
const int64_t r = ct * 32 + row;
if (r >= ne1) {
continue;
}
const block_q1_0 & block = src_slice[r * n_k_tiles + kt];
scale_dst[row] = block.d;
for (int k = 0; k < QK1_0; k++) {
int byte_offset;
int bit;
q1_0_tiled_bit_location(k, row, byte_offset, bit);
if (block.qs[k / 8] & (1u << (k % 8))) {
tile_dst[byte_offset] |= (uint8_t) (1u << bit);
}
}
}
}
}
}
}
static void repack_tiled_q1_0(void * data, const ggml_tensor * t, size_t offset, size_t size) {
block_q1_0 * dst_matrix = (block_q1_0 *) data;
const int64_t ne0 = t->ne[0];
const int64_t ne1 = t->ne[1];
const int64_t ne2 = t->ne[2];
const int64_t ne3 = t->ne[3];
const int n_col_tiles = hex_round_up(ne1, 32) / 32;
const int n_k_tiles = ne0 / QK1_0;
const size_t tile_size = HTP_TENSOR_TILE_SIZE_Q1_0;
const size_t matrix_size = (size_t) n_col_tiles * n_k_tiles * tile_size;
const size_t row_size_bytes = ggml_row_size(t->type, ne0);
const size_t slice_size = ne1 * row_size_bytes;
const int64_t start_slice = offset / slice_size;
const int64_t end_slice = (std::min)((int64_t) ((offset + size + slice_size - 1) / slice_size), ne2 * ne3);
GGML_ASSERT(ne0 % QK1_0 == 0);
for (int64_t slice_idx = start_slice; slice_idx < end_slice; slice_idx++) {
const size_t cur_start_byte = (std::max)(offset, (size_t) slice_idx * slice_size);
const size_t cur_end_byte = (std::min)(offset + size, (size_t) (slice_idx + 1) * slice_size);
const int64_t start_row = (cur_start_byte - (size_t) slice_idx * slice_size) / row_size_bytes;
const int64_t end_row = (std::min)((int64_t) ((cur_end_byte - (size_t) slice_idx * slice_size + row_size_bytes - 1) / row_size_bytes), ne1);
block_q1_0 * dst_slice = dst_matrix + (cur_start_byte - offset) / sizeof(block_q1_0);
const uint8_t * matrix_src = (const uint8_t *) t->data + slice_idx * matrix_size;
for (int ct = start_row / 32; ct < (end_row + 31) / 32; ct++) {
for (int kt = 0; kt < n_k_tiles; kt++) {
const uint8_t * tile_src = matrix_src + ((size_t) ct * n_k_tiles + kt) * tile_size;
const ggml_half * scale_src = (const ggml_half *) (tile_src + 512);
for (int row = 0; row < 32; row++) {
const int64_t r = ct * 32 + row;
if (r < start_row || r >= end_row) {
continue;
}
block_q1_0 & block = dst_slice[(r - start_row) * n_k_tiles + kt];
memset(block.qs, 0, sizeof(block.qs));
block.d = scale_src[row];
for (int k = 0; k < QK1_0; k++) {
int byte_offset;
int bit;
q1_0_tiled_bit_location(k, row, byte_offset, bit);
if (tile_src[byte_offset] & (1u << bit)) {
block.qs[k / 8] |= (uint8_t) (1u << (k % 8));
}
}
}
}
}
}
}
// --- Top-level Repack / Unpack Dispatchers ---
void htp_repack_tensor_tiled(ggml_tensor * tensor, const void * data, size_t size) {
switch (tensor->type) {
case GGML_TYPE_Q1_0:
repack_q1_0_tiled(tensor, data, 0, size);
break;
case GGML_TYPE_Q4_0:
case GGML_TYPE_IQ4_NL:
repack_q4_0_tiled(tensor, data, 0, size);
break;
case GGML_TYPE_Q4_1:
repack_q4_1_tiled(tensor, data, 0, size);
break;
case GGML_TYPE_Q8_0:
repack_q8_0_tiled(tensor, data, 0, size);
break;
case GGML_TYPE_MXFP4:
repack_mxfp4_tiled(tensor, data, 0, size);
break;
case GGML_TYPE_Q2_0:
repack_q2_0_tiled(tensor, data, 0, size);
break;
case GGML_TYPE_Q2_K:
repack_q2_K_tiled(tensor, data, 0, size);
break;
case GGML_TYPE_Q3_K:
repack_q3_K_tiled(tensor, data, 0, size);
break;
case GGML_TYPE_Q4_K:
repack_q4_K_tiled(tensor, data, 0, size);
break;
case GGML_TYPE_Q5_K:
repack_q5_K_tiled(tensor, data, 0, size);
break;
case GGML_TYPE_Q6_K:
repack_q6_K_tiled(tensor, data, 0, size);
break;
case GGML_TYPE_F16:
repack_float_to_f16_tiled<uint16_t>(tensor, data, 0, size,
[](uint16_t v) { return v; });
break;
case GGML_TYPE_F32:
repack_float_to_f16_tiled<float>(tensor, data, 0, size,
[](float v) { return (uint16_t) ggml_fp32_to_fp16(v); });
break;
case GGML_TYPE_BF16:
repack_float_to_f16_tiled<ggml_bf16_t>(tensor, data, 0, size,
[](ggml_bf16_t v) { return (uint16_t) ggml_fp32_to_fp16(ggml_bf16_to_fp32(v)); });
break;
default:
break;
}
}
void htp_unpack_tensor_tiled(void * data, const ggml_tensor * tensor, size_t offset, size_t size) {
GGML_ASSERT(offset + size <= ggml_nbytes(tensor));
switch (tensor->type) {
case GGML_TYPE_Q1_0:
repack_tiled_q1_0(data, tensor, offset, size);
break;
case GGML_TYPE_Q4_0:
case GGML_TYPE_IQ4_NL:
repack_tiled_q4_0(data, tensor, offset, size);
break;
case GGML_TYPE_Q4_1:
repack_tiled_q4_1(data, tensor, offset, size);
break;
case GGML_TYPE_Q8_0:
repack_tiled_q8_0(data, tensor, offset, size);
break;
case GGML_TYPE_MXFP4:
repack_tiled_mxfp4(data, tensor, offset, size);
break;
case GGML_TYPE_Q2_0:
repack_tiled_q2_0(data, tensor, offset, size);
break;
case GGML_TYPE_Q2_K:
repack_tiled_q2_K(data, tensor, offset, size);
break;
case GGML_TYPE_Q3_K:
repack_tiled_q3_K(data, tensor, offset, size);
break;
case GGML_TYPE_Q4_K:
repack_tiled_q4_K(data, tensor, offset, size);
break;
case GGML_TYPE_Q5_K:
repack_tiled_q5_K(data, tensor, offset, size);
break;
case GGML_TYPE_Q6_K:
repack_tiled_q6_K(data, tensor, offset, size);
break;
case GGML_TYPE_F16:
unpack_f16_tiled_to_float<uint16_t>(data, tensor, offset, size,
[](uint16_t v) { return v; });
break;
case GGML_TYPE_F32:
unpack_f16_tiled_to_float<float>(data, tensor, offset, size,
[](uint16_t v) { return ggml_fp16_to_fp32((ggml_fp16_t) v); });
break;
case GGML_TYPE_BF16:
unpack_f16_tiled_to_float<ggml_bf16_t>(data, tensor, offset, size,
[](uint16_t v) { return ggml_fp32_to_bf16(ggml_fp16_to_fp32((ggml_fp16_t) v)); });
break;
default:
memcpy(data, (const char *) tensor->data + offset, size);
break;
}
}