From 797de63b2fa7396dcc8caca76834c699ac84e1c3 Mon Sep 17 00:00:00 2001 From: Todor Boinovski Date: Sat, 17 Oct 2026 13:36:38 -0700 Subject: [PATCH] hexagon: add support for Q1_0 and Q2_0 (#30122) * hexagon: add support for q1_0 quant * hexagon: enable Q1_0 weights in the 2D HMX MUL_MAT path * hex-q1: cleanup gates, generalize tiling and remove in-loop divs * hex-tiling: document tiled formats in developer.md and update htp-tensor.h * hex-q2: add support for Q2_0 * hex-q2: remove stale asserts and speacial cases * hex-mm: fix MUL_MAT_NX assumption about square tiling, Q1/Q2 have non-square tiles * hex-q1/2: minor optimizations (unrolls/un-unrolls/...) * hex-q1: optimize Q1_0 layout to reduce unpack/vrmpy overhead * hex-q1/q2_0: further optimize vec_dots --------- Co-authored-by: Max Krasnyansky --- docs/backend/snapdragon/developer.md | 151 ++++++++++ ggml/src/ggml-hexagon/ggml-hexagon.cpp | 32 ++- ggml/src/ggml-hexagon/htp-repack.cpp | 195 +++++++++++++ ggml/src/ggml-hexagon/htp-repack.h | 7 +- .../ggml-hexagon/htp/hmx-mm-kernels-tiled.h | 68 +++++ ggml/src/ggml-hexagon/htp/htp-ops.h | 2 + ggml/src/ggml-hexagon/htp/htp-tensor.h | 43 ++- .../ggml-hexagon/htp/hvx-mm-kernels-tiled.h | 271 ++++++++++++++++++ ggml/src/ggml-hexagon/htp/matmul-ops.c | 125 ++++---- ggml/src/ggml-hexagon/htp/matmul-ops.h | 22 +- 10 files changed, 809 insertions(+), 107 deletions(-) diff --git a/docs/backend/snapdragon/developer.md b/docs/backend/snapdragon/developer.md index e088390e8..363be41ba 100644 --- a/docs/backend/snapdragon/developer.md +++ b/docs/backend/snapdragon/developer.md @@ -48,6 +48,157 @@ Alternatively, users can partition and split the model across multiple virtual s tensor-splitting, or row-splitting modes. For user-facing execution modes and examples, see the [Snapdragon user guide](README.md#multi-device-execution-modes). +## Tiled Weight Formats + +Weight tensors in the Hexagon backend are repacked from flat row-major GGML layouts into 2D tiles in memory. +Tiling groups 2D matrix blocks contiguously: +- Eliminates strided row reads when fetching 2D tiles into VTCM via DMA. +- Matches native HVX vector width (128 bytes) and HMX systolic array dimensions (32x32). +- Repacking happens on the host during model load (`set_tensor()`) or via runtime repacking (`rt-repack`). +Implementation: [`htp-repack.cpp`](../../../ggml/src/ggml-hexagon/htp-repack.cpp). + +### Tile Dimensions and Grid Layout + +Each tile represents a submatrix of `N_rows = 32` rows and `K_cols` columns: +- Standard types (`Q4_0`, `Q4_1`, `Q8_0`, `IQ4_NL`, `MXFP4`, and K-quants): `K_cols = 32` (`HTP_TENSOR_TILE_N_COLS = 32`). +- `Q2_0`: `K_cols = 64` (`HTP_TENSOR_TILE_K_Q2_0 = 64`). +- `Q1_0`: `K_cols = 128` (`HTP_TENSOR_TILE_K_Q1_0 = 128`). + +For a weight matrix of logical shape `[ne0, ne1]` (where `ne0 = K`, `ne1 = N`): +- `n_k_tiles = ne0 / K_cols` +- `n_col_tiles = (ne1 + 31) / 32` +- Tile index for column-tile `kt` and row-tile `ct`: `tile_idx = ct * n_k_tiles + kt` +- Byte offset in matrix buffer: `tile_offset = tile_idx * tile_size` + +### Block-32 Quantized Formats + +Block-32 formats use a tile size of 32 rows x 32 columns (`K_cols = 32`): + +- **Q4_0** (576 bytes per tile, `HTP_TENSOR_TILE_SIZE_Q4_0`): + - **Quants** (bytes 0..511, 512 bytes): + Contains 32 rows x 32 columns of 4-bit quants. + Divided into 16 column pairs `cp = 0..15` covering columns `2*cp` and `2*cp + 1`. + For each column pair `cp`, 32 bytes are stored consecutively for rows `row = 0..31`: + `tile[cp * 32 + row]` holds `(quant[row][2*cp + 1] << 4) | quant[row][2*cp]`. + Low nibble holds the even column; high nibble holds the odd column. + - **Scales** (bytes 512..575, 64 bytes): + 32 FP16 values (`ggml_half`), storing scale `d` for each row `row = 0..31`. + +- **Q8_0** (1088 bytes per tile, `HTP_TENSOR_TILE_SIZE_Q8_0`): + - **Quants** (bytes 0..1023, 1024 bytes): + Contains 32 rows x 32 columns of signed 8-bit quants (`int8_t`). + Divided into 16 column pairs `cp = 0..15` covering columns `col0 = 2*cp` and `col1 = 2*cp + 1`. + For each column pair `cp`, 64 bytes store interleaved column elements for each row `row = 0..31`: + `tile[cp * 64 + 2 * row + 0] = quant[row][col0]` + `tile[cp * 64 + 2 * row + 1] = quant[row][col1]` + - **Scales** (bytes 1024..1087, 64 bytes): + 32 FP16 values (`ggml_half`), storing scale `d` for each row `row = 0..31`. + +- **IQ4_NL** (576 bytes per tile, `HTP_TENSOR_TILE_SIZE_IQ4_NL`): + - **Quants** (bytes 0..511, 512 bytes): + Same nibble transposition and storage layout as `Q4_0`. + Contains 32 rows x 32 columns of non-linear 4-bit indices (values 0..15). + Divided into 16 column pairs `cp = 0..15`: + `tile[cp * 32 + row]` holds `(quant[row][2*cp + 1] << 4) | quant[row][2*cp]`. + - **Scales** (bytes 512..575, 64 bytes): + 32 FP16 values (`ggml_half`), storing scale `d` for each row `row = 0..31`. + - **Dequantization**: + In HVX/HMX kernels, each 4-bit index is looked up in the 16-entry non-linear table (`iq4_nl_to_fp16_lut`) to produce an FP16 value, then multiplied by the row scale `d`. + +- **MXFP4** (544 bytes per tile, `HTP_TENSOR_TILE_SIZE_MXFP4`): + - **Quants** (bytes 0..511, 512 bytes): + Same nibble transposition and storage layout as `Q4_0`. + Contains 32 rows x 32 columns of 4-bit OCP Microscaling FP4 (E2M1) values. + Divided into 16 column pairs `cp = 0..15`: + `tile[cp * 32 + row]` holds `(quant[row][2*cp + 1] << 4) | quant[row][2*cp]`. + - **Scales** (bytes 512..543, 32 bytes): + 32 unsigned 8-bit values (`uint8_t`), storing one 8-bit biased exponent (E8M0, bias 127) scale `e` per row `row = 0..31`. + - **Dequantization**: + Kernel converts E8M0 bias to IEEE FP16 bias via `vh = clamp(e - 112, 0, 30) << 10` to form a power-of-two FP16 scale. + Each 4-bit FP4 nibble is unpacked via `mxfp4_to_fp16_lut` and multiplied by this scale. + +- **Q4_1** (640 bytes per tile, `HTP_TENSOR_TILE_SIZE_Q4_1`): + - **Quants** (bytes 0..511, 512 bytes): + Same 4-bit nibble transposition and storage layout as `Q4_0`. + - **Scales** (bytes 512..639, 128 bytes): + 32 pairs of FP16 values (`ggml_half`): `scale[2 * row + 0] = d` (scale) and `scale[2 * row + 1] = m` (min) for each row `row = 0..31`. + +### Q1_0 Format (1-Bit Quantization) + +`Q1_0` packs 128 columns per tile (`K_cols = 128`, `HTP_TENSOR_TILE_K_Q1_0 = 128`): + +- **Tile Layout** (576 bytes per tile, `HTP_TENSOR_TILE_SIZE_Q1_0`): + - **Quants** (bytes 0..511, 512 bytes): + Contains 32 rows x 128 columns of 1-bit quants (-1 or +1). + Divided into 4 quarters (`quarter = 0..3`), where each quarter covers 32 columns in K. + Each quarter occupies 128 bytes (1 HVX vector) in `vrmpy` byte order. + Within a quarter, columns `kq = 0..31` map into 8 groups `g = kq / 4` (0..7) of 4 consecutive columns: + `byte_offset = quarter * 128 + 4 * row + (kq % 4)` + `bit = g` (0..7) + - **Scales** (bytes 512..575, 64 bytes): + 32 FP16 values (`ggml_half`), storing scale `d` for each row `row = 0..31`. + One scale per row applies to all 128 columns (all 4 quarters) of the tile. + +- **HMX Mapping (1-to-4 Tile Expansion)**: + - HMX hardware operates strictly on 32x32 FP16 tiles (`HTP_TENSOR_TILE_N_ROWS = 32`, `HTP_TENSOR_TILE_N_COLS = 32`). + - During kernel execution, HVX dequantization (`dequantize_tiled_weight_to_fp16_task_q1_0`) expands each 32x128 Q1_0 tile (576 bytes) into 4 contiguous 32x32 FP16 HMX tiles (4 x 2048 bytes = 8192 bytes total) in VTCM. + - HMX systolic array processes the 4 resulting 32x32 FP16 tiles over 4 compute steps along K. + +### Q2_0 Format (2-Bit Quantization) + +`Q2_0` packs 64 columns per tile (`K_cols = 64`, `HTP_TENSOR_TILE_K_Q2_0 = 64`): + +- **Tile Layout** (576 bytes per tile, `HTP_TENSOR_TILE_SIZE_Q2_0`): + - **Quants** (bytes 0..511, 512 bytes): + Contains 32 rows x 64 columns of 2-bit quants (codes 0..3 representing signed values -1..2). + Divided into 16 column groups `group = 0..15` of 4 columns each. + Stored in `vrmpy` byte order across 4 HVX vectors (vec 0..3): + Vector `group >> 2` (0..3) holds `group & 3` at bit offset `2 * (group & 3)`. + Byte `4 * row + byte` holds row `row = 0..31` and column byte `0..3`. + Vec 0..1 holds the first 32 columns (half 0), vec 2..3 holds the next 32 columns (half 1). + - **Scales** (bytes 512..575, 64 bytes): + 32 FP16 values (`ggml_half`), storing scale `d` for each row `row = 0..31` (at offset `512 + 2 * row`). + One scale per row applies to all 64 columns of the tile. + +- **HMX Mapping (1-to-2 Tile Expansion)**: + - HMX hardware operates strictly on 32x32 FP16 tiles (`HTP_TENSOR_TILE_N_ROWS = 32`, `HTP_TENSOR_TILE_N_COLS = 32`). + - During kernel execution, HVX dequantization (`dequantize_tiled_weight_to_fp16_task_q2_0`) expands each 32x64 Q2_0 tile (576 bytes) into 2 contiguous 32x32 FP16 HMX tiles (2 x 2048 bytes = 4096 bytes total) in VTCM. + - HMX systolic array processes the 2 resulting 32x32 FP16 tiles over 2 compute steps along K. + +- **HVX Vector Dot-Product**: + - `tiled_vec_dot_q2_0_32x1` and `tiled_vec_dot_q2_0_32x2` unpack 2-bit quants using `unpack_q3_k_low2`, subtract 1, and accumulate against Q8_0 activation tiles using `Q6_Vw_vrmpyacc_VwVbVb`. + +### K-Quants Formats + +In GGML, K-quants pack 256 elements per super-block. +On Hexagon, each 256-element super-block splits along K into 8 tiles of 32 columns (`K_cols = 32`). + +Byte ordering: `Q2_K`, `Q3_K`, and `Q6_K` use `vrmpy` byte order within each 128B HVX vector: +`k = 0..31` is split into 8 groups `g = k/4` of 4 `k` each; byte `4*r + (k%4)` holds row `r`. + +- **Q2_K** (336 bytes per tile, `HTP_TENSOR_TILE_SIZE_Q2_K`, lossless tiling with same size as `block_q2_K`): + - `bytes 0..255` (vec 0..1): unsigned 2-bit quants; vec `m` holds groups `4m..4m+3` at bit offsets 0, 2, 4, 6. + - `bytes 256..319`: raw `scales[]` bytes (`min << 4 | sc`): `[0..31]` for k 0..15, `[32..63]` for k 16..31. + - `bytes 320..335`: 8 FP16 in tile `j` (0..7) of super-block: `d` (`j < 4`) or `-dmin` (`j >= 4`) for rows `8*(j%4)..8*(j%4)+7`. + +- **Q3_K** (512 bytes per tile, `HTP_TENSOR_TILE_SIZE_Q3_K`): + - `bytes 0..255` (vec 0..1): low 2 bits; vec `m` holds groups `4m..4m+3` at bit offsets 0, 2, 4, 6. + - `bytes 256..383` (vec 2): sign mask, bit `g` set when `hmask` is 0 (quant = `low2 - 4`). + - `bytes 384..511` (vec 3): FP16 scales `d * (scales[] - 32)`: `fp16[0..31]` for k 0..15, `fp16[32..63]` for k 16..31. + +- **Q4_K** (640 bytes per tile, `HTP_TENSOR_TILE_SIZE_Q4_K`, same layout as Q4_1 tile; each 256-k super-block splits into 8 tiles of 32 k): + - `bytes 0..511` (vec 0..3): 4-bit quants; byte `32*cp + r` holds `k = 2*cp` (low nibble) and `2*cp + 1` (high nibble). + - `bytes 512..639` (vec 4): 32x2 FP16 per row: `(d * sc, -dmin * m)`. + +- **Q5_K** (768 bytes per tile, `HTP_TENSOR_TILE_SIZE_Q5_K`): + - `bytes 0..639` (vec 0..4): Q4_K tile (low 4 bits + FP16 scale/min pairs). + - `bytes 640..767` (vec 5): 5th-bit plane; byte `l` holds the 8 high bits for lane `l` across vec `i = 0..3` (`bit 2i = lo, 2i+1 = hi`). + +- **Q6_K** (896 bytes per tile, `HTP_TENSOR_TILE_SIZE_Q6_K`): + - `bytes 0..511` (vec 0..3): low 4 bits; vec `i` holds group `2i` (low nibble) and `2i+1` (high nibble). + - `bytes 512..767` (vec 4..5): high 2 bits; vec `4+m` holds groups `4m..4m+3` at bit offsets 0, 2, 4, 6. + - `bytes 768..895` (vec 6): FP16 scales `d * scales[]`: `fp16[0..31]` for k 0..15, `fp16[32..63]` for k 16..31. + ## Op and Kernel Development Guidelines Writing high-performance operators for Hexagon requires following specific guidelines. diff --git a/ggml/src/ggml-hexagon/ggml-hexagon.cpp b/ggml/src/ggml-hexagon/ggml-hexagon.cpp index 54c474122..6e714ff3b 100644 --- a/ggml/src/ggml-hexagon/ggml-hexagon.cpp +++ b/ggml/src/ggml-hexagon/ggml-hexagon.cpp @@ -956,7 +956,6 @@ static void ggml_backend_hexagon_buffer_get_tensor(ggml_backend_buffer_t buffer, return; } - GGML_ASSERT(offset == 0); htp_unpack_tensor_tiled(data, tensor, offset, size); } @@ -1045,7 +1044,6 @@ static void ggml_backend_hexagon_buffer_get_tensor_2d(ggml_backend_buffer_t buff GGML_ASSERT((slice_offset % row_size_bytes) == 0 && "offset must be aligned to row boundary"); GGML_ASSERT((temp_size % row_size_bytes) == 0 && "temp_size must be a multiple of row size"); - GGML_ASSERT((slice_offset / row_size_bytes) % 32 == 0 && "offset must be aligned to tile size (32 rows)"); GGML_ASSERT((offset + temp_size) <= ggml_nbytes(tensor)); std::vector temp_buf(temp_size); @@ -3769,6 +3767,8 @@ static bool ggml_hexagon_matmul_is_hmx_eligible( bool is_matmul_id, bool is_batched ) { + GGML_UNUSED(dst); + if (src1->type != GGML_TYPE_F32 && (src1->type != GGML_TYPE_F16 || is_matmul_id)) { return false; } @@ -3780,7 +3780,7 @@ static bool ggml_hexagon_matmul_is_hmx_eligible( const int wtype = src0->type; // HMX weight tiles accept non-32-aligned N for non-matmul_id. - if (ne01 % 32 != 0 && is_matmul_id) { + if (ne01 % HTP_TENSOR_TILE_N_ROWS != 0 && is_matmul_id) { return false; } @@ -3789,8 +3789,8 @@ static bool ggml_hexagon_matmul_is_hmx_eligible( return false; } - // HMX paths require K aligned to 32. - if (ne00 % 32 != 0) { + // HMX paths require K aligned to HMX tile cols (32). + if (ne00 % HTP_TENSOR_TILE_N_COLS != 0) { return false; } @@ -3952,7 +3952,7 @@ static void ggml_hexagon_precompute_hvx_mm_params( kparams->tile_size = htp_tensor_get_tile_size(wtype); kparams->aligned_tile_size = htp_tensor_get_tile_padded_size(wtype); - const bool k_align = (ne10 % 32 == 0); + const bool k_align = (ne10 % htp_tensor_get_tile_k(wtype) == 0); if (is_matmul_id) { kparams->kernel_type = (src1_nrows < (int) sess->n_threads) @@ -4124,7 +4124,7 @@ static void ggml_hexagon_precompute_matmul_params_impl( const bool is_tiled = ggml_hexagon_tensor_is_tiled(src0); const int wtype = (is_tiled && (src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_BF16)) ? GGML_TYPE_F16 : src0->type; - const int ne00_padded = is_tiled ? hex_round_up(ne00, 32) : ne00; + const int ne00_padded = is_tiled ? hex_round_up(ne00, htp_tensor_get_tile_k(wtype)) : ne00; const int ne11_padded = hex_round_up(ne11, 32); // VTCM has to hold whole 32-row weight tiles, so size for the rounded-up N // even when the tensor itself is ragged. @@ -5372,6 +5372,8 @@ static bool ggml_hexagon_supported_mul_mat(const struct ggml_hexagon_session * s } switch (src0->type) { + case GGML_TYPE_Q1_0: + case GGML_TYPE_Q2_0: case GGML_TYPE_Q4_0: case GGML_TYPE_Q4_1: case GGML_TYPE_Q8_0: @@ -5381,13 +5383,13 @@ static bool ggml_hexagon_supported_mul_mat(const struct ggml_hexagon_session * s case GGML_TYPE_Q5_K: case GGML_TYPE_Q6_K: case GGML_TYPE_Q3_K: - case GGML_TYPE_Q2_K: + case GGML_TYPE_Q2_K: { if (!ggml_is_contiguous(src0) || ggml_is_permuted(src0) || ggml_is_view(src0)) { return false; } - if (src0->ne[0] % ((src0->type == GGML_TYPE_Q6_K || src0->type == GGML_TYPE_Q5_K || src0->type == GGML_TYPE_Q4_K || - src0->type == GGML_TYPE_Q3_K || src0->type == GGML_TYPE_Q2_K) ? QK_K : 32)) { + const int64_t qk = ggml_blck_size(src0->type); + if (src0->ne[0] % qk) { return false; } @@ -5398,6 +5400,7 @@ static bool ggml_hexagon_supported_mul_mat(const struct ggml_hexagon_session * s return false; } break; + } case GGML_TYPE_F16: if (src0->nb[1] < src0->nb[0]) { @@ -5467,6 +5470,8 @@ static bool ggml_hexagon_supported_mul_mat_id(const struct ggml_hexagon_session } switch (src0->type) { + case GGML_TYPE_Q1_0: + case GGML_TYPE_Q2_0: case GGML_TYPE_Q4_0: case GGML_TYPE_Q4_1: case GGML_TYPE_Q8_0: @@ -5481,8 +5486,7 @@ static bool ggml_hexagon_supported_mul_mat_id(const struct ggml_hexagon_session return false; } - if (src0->ne[0] % ((src0->type == GGML_TYPE_Q6_K || src0->type == GGML_TYPE_Q5_K || src0->type == GGML_TYPE_Q4_K || - src0->type == GGML_TYPE_Q3_K || src0->type == GGML_TYPE_Q2_K) ? QK_K : 32)) { + if (src0->ne[0] % ggml_blck_size(src0->type) != 0) { return false; } break; @@ -8317,6 +8321,10 @@ static void ggml_hexagon_init(ggml_backend_reg * reg) { "please update hexagon_type to match ggml_type"); static_assert((unsigned int) HTP_TYPE_Q2_K == (unsigned int) GGML_TYPE_Q2_K, "please update hexagon_type to match ggml_type"); + static_assert((unsigned int) HTP_TYPE_Q1_0 == (unsigned int) GGML_TYPE_Q1_0, + "please update hexagon_type to match ggml_type"); + static_assert((unsigned int) HTP_TYPE_Q2_0 == (unsigned int) GGML_TYPE_Q2_0, + "please update hexagon_type to match ggml_type"); static_assert((unsigned int) HTP_TYPE_BF16 == (unsigned int) GGML_TYPE_BF16, "please update hexagon_type to match ggml_type"); diff --git a/ggml/src/ggml-hexagon/htp-repack.cpp b/ggml/src/ggml-hexagon/htp-repack.cpp index 38beabe82..7ef2f6d0c 100644 --- a/ggml/src/ggml-hexagon/htp-repack.cpp +++ b/ggml/src/ggml-hexagon/htp-repack.cpp @@ -289,6 +289,80 @@ static void repack_tiled_q8_0(void * data, const ggml_tensor * t, size_t offset, // --- Float Types (F16, F32, BF16 -> f16_tiled HMX layout) --- +static void repack_q2_0_tiled(ggml_tensor * t, const void * data, size_t offset, size_t size) { + GGML_ASSERT(offset == 0); + GGML_ASSERT(t->ne[0] % 64 == 0); + GGML_ASSERT(size == ggml_nbytes(t)); + + const block_q2_0 * src = (const block_q2_0 *) data; + const int64_t blocks_per_row = t->ne[0] / 64; + const int64_t n_col_tiles = hex_round_up(t->ne[1], 32) / 32; + const int64_t n_weight_tiles = blocks_per_row; + const size_t matrix_size = (size_t) n_col_tiles * n_weight_tiles * HTP_TENSOR_TILE_SIZE_Q2_0; + + for (int64_t slice = 0; slice < t->ne[2] * t->ne[3]; ++slice) { + const block_q2_0 * src_slice = src + slice * t->ne[1] * blocks_per_row; + uint8_t * dst_slice = (uint8_t *) t->data + slice * matrix_size; + memset(dst_slice, 0, matrix_size); + + for (int64_t row_index = 0; row_index < t->ne[1]; ++row_index) { + const int64_t ct = row_index / 32; + const int row = (int) (row_index % 32); + for (int64_t kt = 0; kt < n_weight_tiles; ++kt) { + const block_q2_0 * block = src_slice + row_index * blocks_per_row + kt; + uint8_t * tile = dst_slice + (ct * n_weight_tiles + kt) * HTP_TENSOR_TILE_SIZE_Q2_0; + for (int group = 0; group < 16; ++group) { + for (int byte = 0; byte < 4; ++byte) { + const int k = 4 * group + byte; + const uint8_t code = (block->qs[k / 4] >> (2 * (k % 4))) & 3; + tile[(group >> 2) * 128 + 4 * row + byte] |= code << (2 * (group & 3)); + } + } + memcpy(tile + 512 + 2 * row, &block->d, sizeof(block->d)); + } + } + } +} + +static void repack_tiled_q2_0(void * data, const ggml_tensor * t, size_t offset, size_t size) { + GGML_ASSERT(t->ne[0] % 64 == 0); + GGML_ASSERT(offset + size <= ggml_nbytes(t)); + + const size_t first_block = offset / sizeof(block_q2_0); + const size_t end_block = (offset + size + sizeof(block_q2_0) - 1) / sizeof(block_q2_0); + const size_t blocks_per_row = t->ne[0] / 64; + const size_t blocks_per_slice = t->ne[1] * blocks_per_row; + const size_t n_weight_tiles = blocks_per_row; + const size_t n_col_tiles = hex_round_up(t->ne[1], 32) / 32; + const size_t matrix_size = n_col_tiles * n_weight_tiles * HTP_TENSOR_TILE_SIZE_Q2_0; + + for (size_t block_index = first_block; block_index < end_block; ++block_index) { + const size_t slice = block_index / blocks_per_slice; + const size_t in_slice = block_index % blocks_per_slice; + const size_t row_index = in_slice / blocks_per_row; + const size_t kt = in_slice % blocks_per_row; + const size_t ct = row_index / 32; + const size_t row = row_index % 32; + const uint8_t * tile = (const uint8_t *) t->data + slice * matrix_size + + (ct * n_weight_tiles + kt) * HTP_TENSOR_TILE_SIZE_Q2_0; + + block_q2_0 block = {}; + memcpy(&block.d, tile + 512 + 2 * row, sizeof(block.d)); + for (int group = 0; group < 16; ++group) { + for (int byte = 0; byte < 4; ++byte) { + const uint8_t packed = tile[(group >> 2) * 128 + 4 * row + byte]; + const uint8_t code = (packed >> (2 * (group & 3))) & 3; + block.qs[group] |= code << (2 * byte); + } + } + + const size_t block_begin = block_index * sizeof(block_q2_0); + const size_t copy_begin = (std::max)(block_begin, offset); + const size_t copy_end = (std::min)(block_begin + sizeof(block_q2_0), offset + size); + memcpy((uint8_t *) data + copy_begin - offset, (const uint8_t *) &block + copy_begin - block_begin, copy_end - copy_begin); + } +} + template static void repack_float_to_f16_tiled(ggml_tensor * t, const void * data, size_t offset, size_t size, ToFp16Fn && to_fp16) { repack_to_tiled_grid_32(t, data, offset, size, HTP_TENSOR_TILE_SIZE_F16, @@ -900,10 +974,122 @@ static void repack_tiled_q5_K(void * data, const ggml_tensor * t, size_t offset, }); } +static inline void q1_0_tiled_bit_location(int k, int row, int & byte_offset, int & bit) { + const int quarter = k / 32; + const int kq = k % 32; + + byte_offset = quarter * 128 + 4 * row + (kq % 4); + bit = kq / 4; +} + +static void repack_q1_0_tiled(ggml_tensor * t, const void * data, size_t offset, size_t size) { + const block_q1_0 * src_matrix = (const block_q1_0 *) data; + const int64_t ne0 = t->ne[0]; + const int64_t ne1 = t->ne[1]; + const int64_t ne2 = t->ne[2]; + const int64_t ne3 = t->ne[3]; + const int n_col_tiles = hex_round_up(ne1, 32) / 32; + const int n_k_tiles = ne0 / QK1_0; + const size_t tile_size = HTP_TENSOR_TILE_SIZE_Q1_0; + const size_t matrix_size = (size_t) n_col_tiles * n_k_tiles * tile_size; + const size_t slice_size = ne1 * ggml_row_size(t->type, ne0); + const int64_t start_slice = offset / slice_size; + const int64_t end_slice = (std::min)((int64_t) ((offset + size + slice_size - 1) / slice_size), ne2 * ne3); + + GGML_ASSERT(ne0 % QK1_0 == 0); + + for (int64_t slice_idx = start_slice; slice_idx < end_slice; slice_idx++) { + const block_q1_0 * src_slice = src_matrix + (slice_idx - start_slice) * ne1 * n_k_tiles; + uint8_t * matrix_dst = (uint8_t *) t->data + slice_idx * matrix_size; + + for (int ct = 0; ct < n_col_tiles; ct++) { + for (int kt = 0; kt < n_k_tiles; kt++) { + uint8_t * tile_dst = matrix_dst + ((size_t) ct * n_k_tiles + kt) * tile_size; + ggml_half * scale_dst = (ggml_half *) (tile_dst + 512); + memset(tile_dst, 0, tile_size); + + for (int row = 0; row < 32; row++) { + const int64_t r = ct * 32 + row; + if (r >= ne1) { + continue; + } + + const block_q1_0 & block = src_slice[r * n_k_tiles + kt]; + scale_dst[row] = block.d; + for (int k = 0; k < QK1_0; k++) { + int byte_offset; + int bit; + q1_0_tiled_bit_location(k, row, byte_offset, bit); + if (block.qs[k / 8] & (1u << (k % 8))) { + tile_dst[byte_offset] |= (uint8_t) (1u << bit); + } + } + } + } + } + } +} + +static void repack_tiled_q1_0(void * data, const ggml_tensor * t, size_t offset, size_t size) { + block_q1_0 * dst_matrix = (block_q1_0 *) data; + const int64_t ne0 = t->ne[0]; + const int64_t ne1 = t->ne[1]; + const int64_t ne2 = t->ne[2]; + const int64_t ne3 = t->ne[3]; + const int n_col_tiles = hex_round_up(ne1, 32) / 32; + const int n_k_tiles = ne0 / QK1_0; + const size_t tile_size = HTP_TENSOR_TILE_SIZE_Q1_0; + const size_t matrix_size = (size_t) n_col_tiles * n_k_tiles * tile_size; + const size_t row_size_bytes = ggml_row_size(t->type, ne0); + const size_t slice_size = ne1 * row_size_bytes; + const int64_t start_slice = offset / slice_size; + const int64_t end_slice = (std::min)((int64_t) ((offset + size + slice_size - 1) / slice_size), ne2 * ne3); + + GGML_ASSERT(ne0 % QK1_0 == 0); + + for (int64_t slice_idx = start_slice; slice_idx < end_slice; slice_idx++) { + const size_t cur_start_byte = (std::max)(offset, (size_t) slice_idx * slice_size); + const size_t cur_end_byte = (std::min)(offset + size, (size_t) (slice_idx + 1) * slice_size); + const int64_t start_row = (cur_start_byte - (size_t) slice_idx * slice_size) / row_size_bytes; + const int64_t end_row = (std::min)((int64_t) ((cur_end_byte - (size_t) slice_idx * slice_size + row_size_bytes - 1) / row_size_bytes), ne1); + block_q1_0 * dst_slice = dst_matrix + (cur_start_byte - offset) / sizeof(block_q1_0); + const uint8_t * matrix_src = (const uint8_t *) t->data + slice_idx * matrix_size; + + for (int ct = start_row / 32; ct < (end_row + 31) / 32; ct++) { + for (int kt = 0; kt < n_k_tiles; kt++) { + const uint8_t * tile_src = matrix_src + ((size_t) ct * n_k_tiles + kt) * tile_size; + const ggml_half * scale_src = (const ggml_half *) (tile_src + 512); + + for (int row = 0; row < 32; row++) { + const int64_t r = ct * 32 + row; + if (r < start_row || r >= end_row) { + continue; + } + + block_q1_0 & block = dst_slice[(r - start_row) * n_k_tiles + kt]; + memset(block.qs, 0, sizeof(block.qs)); + block.d = scale_src[row]; + for (int k = 0; k < QK1_0; k++) { + int byte_offset; + int bit; + q1_0_tiled_bit_location(k, row, byte_offset, bit); + if (tile_src[byte_offset] & (1u << bit)) { + block.qs[k / 8] |= (uint8_t) (1u << (k % 8)); + } + } + } + } + } + } +} + // --- Top-level Repack / Unpack Dispatchers --- void htp_repack_tensor_tiled(ggml_tensor * tensor, const void * data, size_t size) { switch (tensor->type) { + case GGML_TYPE_Q1_0: + repack_q1_0_tiled(tensor, data, 0, size); + break; case GGML_TYPE_Q4_0: case GGML_TYPE_IQ4_NL: repack_q4_0_tiled(tensor, data, 0, size); @@ -917,6 +1103,9 @@ void htp_repack_tensor_tiled(ggml_tensor * tensor, const void * data, size_t siz case GGML_TYPE_MXFP4: repack_mxfp4_tiled(tensor, data, 0, size); break; + case GGML_TYPE_Q2_0: + repack_q2_0_tiled(tensor, data, 0, size); + break; case GGML_TYPE_Q2_K: repack_q2_K_tiled(tensor, data, 0, size); break; @@ -953,6 +1142,9 @@ void htp_unpack_tensor_tiled(void * data, const ggml_tensor * tensor, size_t off GGML_ASSERT(offset + size <= ggml_nbytes(tensor)); switch (tensor->type) { + case GGML_TYPE_Q1_0: + repack_tiled_q1_0(data, tensor, offset, size); + break; case GGML_TYPE_Q4_0: case GGML_TYPE_IQ4_NL: repack_tiled_q4_0(data, tensor, offset, size); @@ -966,6 +1158,9 @@ void htp_unpack_tensor_tiled(void * data, const ggml_tensor * tensor, size_t off case GGML_TYPE_MXFP4: repack_tiled_mxfp4(data, tensor, offset, size); break; + case GGML_TYPE_Q2_0: + repack_tiled_q2_0(data, tensor, offset, size); + break; case GGML_TYPE_Q2_K: repack_tiled_q2_K(data, tensor, offset, size); break; diff --git a/ggml/src/ggml-hexagon/htp-repack.h b/ggml/src/ggml-hexagon/htp-repack.h index ae66faf61..f656e9b8d 100644 --- a/ggml/src/ggml-hexagon/htp-repack.h +++ b/ggml/src/ggml-hexagon/htp-repack.h @@ -10,9 +10,11 @@ static inline bool htp_is_tiled_weight_type(enum ggml_type type) { return type == GGML_TYPE_Q4_0 || type == GGML_TYPE_Q4_1 || type == GGML_TYPE_Q8_0 || type == GGML_TYPE_IQ4_NL || - type == GGML_TYPE_MXFP4 || type == GGML_TYPE_Q6_K || + type == GGML_TYPE_MXFP4 || type == GGML_TYPE_Q1_0 || + type == GGML_TYPE_Q6_K || type == GGML_TYPE_Q4_K || type == GGML_TYPE_Q5_K || type == GGML_TYPE_Q3_K || type == GGML_TYPE_Q2_K || + type == GGML_TYPE_Q2_0 || type == GGML_TYPE_F16 || type == GGML_TYPE_F32 || type == GGML_TYPE_BF16; } @@ -42,6 +44,9 @@ static inline size_t htp_tiled_row_size(enum ggml_type type, int64_t ne0) { if (type == GGML_TYPE_Q6_K) { return (size_t) (ne0 / 32) * (HTP_TENSOR_TILE_SIZE_Q6_K / 32); } + if (type == GGML_TYPE_Q2_0) { + return (size_t) (ne0 / 64) * 18; + } if (type == GGML_TYPE_Q4_K) { return (size_t) (ne0 / 32) * (HTP_TENSOR_TILE_SIZE_Q4_K / 32); } diff --git a/ggml/src/ggml-hexagon/htp/hmx-mm-kernels-tiled.h b/ggml/src/ggml-hexagon/htp/hmx-mm-kernels-tiled.h index 309ec4877..65f90c17a 100644 --- a/ggml/src/ggml-hexagon/htp/hmx-mm-kernels-tiled.h +++ b/ggml/src/ggml-hexagon/htp/hmx-mm-kernels-tiled.h @@ -26,6 +26,7 @@ struct tiled_dequantize_state { struct fastdiv_values n_k_tiles_div; uint32_t n_k_tiles; + uint32_t n_weight_tiles; uint32_t n_tot_tiles; uint32_t n_tiles_per_task; uint32_t tile_size; @@ -37,6 +38,39 @@ struct tiled_dequantize_state { uint32_t weight_type; }; +static void dequantize_tiled_weight_to_fp16_task_q1_0( + const tiled_dequantize_state_t *state, + uint32_t start_tile, uint32_t end_tile) { + + const HVX_Vector v_neg_one = Q6_Vb_vsplat_R(-1); + const HVX_Vector v_pos_one = Q6_Vb_vsplat_R(1); + + for (uint32_t t = start_tile; t < end_tile; t++) { + const uint8_t * tile_src = state->src + t * state->aligned_tile_size; + __fp16 * dst_ptr = state->dst + 4 * t * HTP_MM_HMX_TILE_N_ELMS; + + const HVX_Vector v_sc = hvx_vmem(tile_src + 512); + const HVX_Vector v_scale = Q6_V_lo_W(Q6_W_vshuff_VVR(v_sc, v_sc, -2)); + + for (uint32_t q = 0; q < 4; q++) { + const HVX_Vector v_plane = hvx_vmem(tile_src + q * 128); + __fp16 * quarter_dst = dst_ptr + q * HTP_MM_HMX_TILE_N_ELMS; + + #pragma unroll + for (int g = 0; g < 8; g++) { + const HVX_Vector v_q = unpack_q1_0_group(v_plane, g, v_neg_one, v_pos_one); + const HVX_VectorPair vp16 = Q6_Wh_vunpack_Vb(v_q); + const HVX_VectorPair vp_k = Q6_W_vdeal_VVR(Q6_V_hi_W(vp16), Q6_V_lo_W(vp16), -4); + + hvx_vmem(quarter_dst + (2 * g + 0) * 64) = + Q6_Vhf_equals_Vqf16(Q6_Vqf16_vmpy_VhfVhf(Q6_Vhf_equals_Vh(Q6_V_lo_W(vp_k)), v_scale)); + hvx_vmem(quarter_dst + (2 * g + 1) * 64) = + Q6_Vhf_equals_Vqf16(Q6_Vqf16_vmpy_VhfVhf(Q6_Vhf_equals_Vh(Q6_V_hi_W(vp_k)), v_scale)); + } + } + } +} + // Dequantize a single tile from tiled weight data (already in VTCM) to tile-major FP16. static void dequantize_tiled_weight_to_fp16_task_q4_0( const tiled_dequantize_state_t *state, @@ -751,6 +785,40 @@ static void dequantize_tiled_weight_to_fp16_task_q2_k( } } +// Q2_0 stores codes 0..3 (value code - 1) and one fp16 scale per row, see HTP_MM_WEIGHT_TILE_SIZE_Q2_0. +// One packed 64-k tile expands into two consecutive 32-k fp16 tiles. +static void dequantize_tiled_weight_to_fp16_task_q2_0( + const tiled_dequantize_state_t *state, + uint32_t start_tile, uint32_t end_tile) { + + const HVX_Vector mask_03 = Q6_Vb_vsplat_R(0x03); + const HVX_Vector i1 = Q6_Vb_vsplat_R(1); + + for (uint32_t t = start_tile; t < end_tile; t++) { + const HVX_Vector * vptr = (const HVX_Vector *) (state->src + t * state->aligned_tile_size); + __fp16 * dst_base = state->dst + 2 * t * HTP_MM_HMX_TILE_N_ELMS; + + HVX_Vector v_sc = vptr[4]; + HVX_Vector v_scale = Q6_V_lo_W(Q6_W_vshuff_VVR(v_sc, v_sc, -2)); + + for (int half = 0; half < 2; half++) { + __fp16 * dst_ptr = dst_base + half * HTP_MM_HMX_TILE_N_ELMS; + + #pragma unroll + for (int g = 0; g < 8; g++) { + HVX_Vector v_q = Q6_Vb_vsub_VbVb(unpack_q3_k_low2(vptr + 2 * half, g, mask_03), i1); + HVX_VectorPair vp16 = Q6_Wh_vunpack_Vb(v_q); + HVX_VectorPair vp_k = Q6_W_vdeal_VVR(Q6_V_hi_W(vp16), Q6_V_lo_W(vp16), -4); + + hvx_vmem(dst_ptr + (2 * g + 0) * 64) = + Q6_Vhf_equals_Vqf16(Q6_Vqf16_vmpy_VhfVhf(Q6_Vhf_equals_Vh(Q6_V_lo_W(vp_k)), v_scale)); + hvx_vmem(dst_ptr + (2 * g + 1) * 64) = + Q6_Vhf_equals_Vqf16(Q6_Vqf16_vmpy_VhfVhf(Q6_Vhf_equals_Vh(Q6_V_hi_W(vp_k)), v_scale)); + } + } + } +} + static __attribute__((noinline)) void convert_f16_weight_to_fp16_tiles_task( const tiled_dequantize_state_t *state, diff --git a/ggml/src/ggml-hexagon/htp/htp-ops.h b/ggml/src/ggml-hexagon/htp/htp-ops.h index bb358d09a..3fc558119 100644 --- a/ggml/src/ggml-hexagon/htp/htp-ops.h +++ b/ggml/src/ggml-hexagon/htp/htp-ops.h @@ -32,6 +32,8 @@ enum htp_data_type { HTP_TYPE_I64 = 27, HTP_TYPE_BF16 = 30, HTP_TYPE_MXFP4 = 39, + HTP_TYPE_Q1_0 = 41, + HTP_TYPE_Q2_0 = 42, HTP_TYPE_INVALID }; diff --git a/ggml/src/ggml-hexagon/htp/htp-tensor.h b/ggml/src/ggml-hexagon/htp/htp-tensor.h index 0bce68dd0..a5ea0d533 100644 --- a/ggml/src/ggml-hexagon/htp/htp-tensor.h +++ b/ggml/src/ggml-hexagon/htp/htp-tensor.h @@ -25,32 +25,13 @@ #define HTP_TENSOR_TILE_SIZE_Q6_K 896 #define HTP_TENSOR_TILE_SIZE_Q3_K 512 #define HTP_TENSOR_TILE_SIZE_Q2_K 336 +#define HTP_TENSOR_TILE_K_Q1_0 128 +#define HTP_TENSOR_TILE_SIZE_Q1_0 576 +#define HTP_TENSOR_TILE_K_Q2_0 64 +#define HTP_TENSOR_TILE_SIZE_Q2_0 576 -// Q2_K, Q3_K, Q6_K use vrmpy byte order within each 128B HVX vector: -// k = 0..31 is split into 8 groups g = k/4 of 4 k each; byte 4*r + (k%4) holds row r -// -// Q2_K (336B, lossless tiling with same size as block_q2_K): -// bytes 0..255 (vec 0..1): unsigned 2-bit quants, vec m holds groups 4m..4m+3 at bit offsets 0,2,4,6 -// bytes 256..319: raw scales[] bytes (min << 4 | sc): [0..31] for k 0..15, [32..63] for k 16..31 -// bytes 320..335: 8 fp16 in tile j (0..7) of super-block: d (j < 4) or -dmin (j >= 4) for rows 8*(j%4)..8*(j%4)+7 -// -// Q3_K (512B): -// bytes 0..255 (vec 0..1): low 2 bits, vec m holds groups 4m..4m+3 at bit offsets 0,2,4,6 -// bytes 256..383 (vec 2): sign mask, bit g set when hmask is 0 (quant = low2 - 4) -// bytes 384..511 (vec 3): fp16 scales d * (scales[] - 32): fp16[0..31] for k 0..15, fp16[32..63] for k 16..31 -// -// Q4_K (640B, same layout as Q4_1 tile; each 256-k super-block splits into 8 tiles of 32 k): -// bytes 0..511 (vec 0..3): 4-bit quants, byte 32*cp + r holds k = 2*cp (low nibble) and 2*cp + 1 (high nibble) -// bytes 512..639 (vec 4): 32x2 fp16 per row: (d * sc, -dmin * m) -// -// Q5_K (768B): -// bytes 0..639 (vec 0..4): Q4_K tile (low 4 bits + fp16 scale/min pairs) -// bytes 640..767 (vec 5): 5th-bit plane; byte l holds the 8 high bits for lane l across vec i = 0..3 (bit 2i = lo, 2i+1 = hi) -// -// Q6_K (896B): -// bytes 0..511 (vec 0..3): low 4 bits, vec i holds group 2i (low nibble) and 2i+1 (high nibble) -// bytes 512..767 (vec 4..5): high 2 bits, vec 4+m holds groups 4m..4m+3 at bit offsets 0,2,4,6 -// bytes 768..895 (vec 6): fp16 scales d * scales[]: fp16[0..31] for k 0..15, fp16[32..63] for k 16..31 +// Tiled weight tensor layouts (Q4_0, Q8_0, IQ4_NL, MXFP4, Q1_0, Q2_0, K-quants): +// See docs/backend/snapdragon/developer.md ("Tiled Weight Formats"). struct htp_tensor_mdev_range { uint32_t start; @@ -168,12 +149,22 @@ static inline uint32_t htp_tensor_get_tile_size(int type) { case HTP_TYPE_Q6_K: return HTP_TENSOR_TILE_SIZE_Q6_K; case HTP_TYPE_Q3_K: return HTP_TENSOR_TILE_SIZE_Q3_K; case HTP_TYPE_Q2_K: return HTP_TENSOR_TILE_SIZE_Q2_K; + case HTP_TYPE_Q1_0: return HTP_TENSOR_TILE_SIZE_Q1_0; + case HTP_TYPE_Q2_0: return HTP_TENSOR_TILE_SIZE_Q2_0; case HTP_TYPE_MXFP4: return HTP_TENSOR_TILE_SIZE_MXFP4; case HTP_TYPE_F16: return HTP_TENSOR_TILE_SIZE_F16; default: return 0; } } +static inline uint32_t htp_tensor_get_tile_k(int type) { + switch (type) { + case HTP_TYPE_Q1_0: return HTP_TENSOR_TILE_K_Q1_0; + case HTP_TYPE_Q2_0: return HTP_TENSOR_TILE_K_Q2_0; + default: return 32; + } +} + static inline uint32_t htp_tensor_get_tile_padded_size(int type) { return (htp_tensor_get_tile_size(type) + 127u) & ~127u; } @@ -193,10 +184,12 @@ static inline uint32_t htp_tensor_get_row_size(int type, uint32_t ne00) { case HTP_TYPE_Q4_1: return (ne00 / 32) * 20; case HTP_TYPE_MXFP4: return (ne00 / 32) * 17; case HTP_TYPE_Q2_K: return (ne00 / 256) * 84; + case HTP_TYPE_Q2_0: return (ne00 / 64) * 18; case HTP_TYPE_Q3_K: return (ne00 / 256) * 110; case HTP_TYPE_Q4_K: return (ne00 / 256) * 144; case HTP_TYPE_Q5_K: return (ne00 / 256) * 176; case HTP_TYPE_Q6_K: return (ne00 / 256) * 210; + case HTP_TYPE_Q1_0: return (ne00 / HTP_TENSOR_TILE_K_Q1_0) * 18; case HTP_TYPE_I32: return ne00 * 4; case HTP_TYPE_I64: return ne00 * 8; default: return 0; diff --git a/ggml/src/ggml-hexagon/htp/hvx-mm-kernels-tiled.h b/ggml/src/ggml-hexagon/htp/hvx-mm-kernels-tiled.h index 91e511725..842001c85 100644 --- a/ggml/src/ggml-hexagon/htp/hvx-mm-kernels-tiled.h +++ b/ggml/src/ggml-hexagon/htp/hvx-mm-kernels-tiled.h @@ -395,6 +395,36 @@ static inline HVX_VectorPair unpack_and_interleave_4bit_x2(HVX_Vector v_src, HVX return Q6_W_vcombine_VV(v_W1, v_W0); } +static inline HVX_Vector unpack_q1_0_group( + HVX_Vector v_plane, + int g, + HVX_Vector v_neg_one, + HVX_Vector v_pos_one +) { + HVX_VectorPred q = Q6_Q_vand_VR(v_plane, 0x01010101u << g); + return Q6_V_vmux_QVV(q, v_pos_one, v_neg_one); +} + +static inline HVX_Vector accum_q1_0_32x1( + HVX_Vector v_plane, + const HVX_Vector * restrict v_act, + HVX_Vector v_neg_one, + HVX_Vector v_pos_one +) { + HVX_Vector v_sum_lo = Q6_V_vzero(); + HVX_Vector v_sum_hi = Q6_V_vzero(); + + #pragma unroll + for (int g = 0; g < 4; g++) { + HVX_Vector v_W_lo = unpack_q1_0_group(v_plane, g, v_neg_one, v_pos_one); + HVX_Vector v_W_hi = unpack_q1_0_group(v_plane, g + 4, v_neg_one, v_pos_one); + v_sum_lo = Q6_Vw_vrmpyacc_VwVbVb(v_sum_lo, v_W_lo, v_act[g]); + v_sum_hi = Q6_Vw_vrmpyacc_VwVbVb(v_sum_hi, v_W_hi, v_act[g + 4]); + } + + return Q6_Vw_vadd_VwVw(v_sum_lo, v_sum_hi); +} + static inline HVX_Vector accum_4bit_32x1( const HVX_Vector * restrict vptr, const HVX_Vector * restrict v_act, @@ -437,6 +467,33 @@ static inline HVX_Vector accum_4bit_32x1_lut( return Q6_Vw_vadd_VwVw(v_sum0, v_sum1); } +static inline HVX_VectorPair accum_q1_0_32x2( + HVX_Vector v_plane, + const HVX_Vector * restrict v_act0, + const HVX_Vector * restrict v_act1, + HVX_Vector v_neg_one, + HVX_Vector v_pos_one +) { + HVX_Vector v_sum0_lo = Q6_V_vzero(); + HVX_Vector v_sum0_hi = Q6_V_vzero(); + HVX_Vector v_sum1_lo = Q6_V_vzero(); + HVX_Vector v_sum1_hi = Q6_V_vzero(); + + #pragma unroll + for (int g = 0; g < 4; g++) { + HVX_Vector v_W_lo = unpack_q1_0_group(v_plane, g, v_neg_one, v_pos_one); + HVX_Vector v_W_hi = unpack_q1_0_group(v_plane, g + 4, v_neg_one, v_pos_one); + v_sum0_lo = Q6_Vw_vrmpyacc_VwVbVb(v_sum0_lo, v_W_lo, v_act0[g]); + v_sum0_hi = Q6_Vw_vrmpyacc_VwVbVb(v_sum0_hi, v_W_hi, v_act0[g + 4]); + v_sum1_lo = Q6_Vw_vrmpyacc_VwVbVb(v_sum1_lo, v_W_lo, v_act1[g]); + v_sum1_hi = Q6_Vw_vrmpyacc_VwVbVb(v_sum1_hi, v_W_hi, v_act1[g + 4]); + } + + return Q6_W_vcombine_VV( + Q6_Vw_vadd_VwVw(v_sum1_lo, v_sum1_hi), + Q6_Vw_vadd_VwVw(v_sum0_lo, v_sum0_hi)); +} + static inline HVX_VectorPair accum_4bit_32x2( const HVX_Vector * restrict vptr, const HVX_Vector * restrict v_act0, @@ -771,6 +828,53 @@ static inline void accum_q2_k_32x2( *v_sums1 = Q6_W_vcombine_VV(v_sum1_hi, v_sum1_lo); } +// Q2_0: unsigned codes 0..3; the -1 offset is applied via the Q8_1 block sum (v_act[9]) +static inline HVX_Vector accum_q2_0_32x1( + const HVX_Vector * restrict vptr, + const HVX_Vector * restrict v_act +) { + HVX_Vector v_sum_lo = Q6_V_vzero(); + HVX_Vector v_sum_hi = Q6_V_vzero(); + HVX_Vector mask_03 = Q6_Vb_vsplat_R(0x03); + + #pragma unroll + for (int g = 0; g < 4; g++) { + HVX_Vector v_W_lo = unpack_q3_k_low2(vptr, g, mask_03); + HVX_Vector v_W_hi = unpack_q3_k_low2(vptr, g + 4, mask_03); + v_sum_lo = Q6_Vw_vrmpyacc_VwVubVb(v_sum_lo, v_W_lo, v_act[g]); + v_sum_hi = Q6_Vw_vrmpyacc_VwVubVb(v_sum_hi, v_W_hi, v_act[g + 4]); + } + + return Q6_Vw_vadd_VwVw(v_sum_lo, v_sum_hi); +} + +static inline void accum_q2_0_32x2( + const HVX_Vector * restrict vptr, + const HVX_Vector * restrict v_act0, + const HVX_Vector * restrict v_act1, + HVX_Vector * v_sum0, + HVX_Vector * v_sum1 +) { + HVX_Vector v_sum0_lo = Q6_V_vzero(); + HVX_Vector v_sum0_hi = Q6_V_vzero(); + HVX_Vector v_sum1_lo = Q6_V_vzero(); + HVX_Vector v_sum1_hi = Q6_V_vzero(); + HVX_Vector mask_03 = Q6_Vb_vsplat_R(0x03); + + #pragma unroll + for (int g = 0; g < 4; g++) { + HVX_Vector v_W_lo = unpack_q3_k_low2(vptr, g, mask_03); + HVX_Vector v_W_hi = unpack_q3_k_low2(vptr, g + 4, mask_03); + v_sum0_lo = Q6_Vw_vrmpyacc_VwVubVb(v_sum0_lo, v_W_lo, v_act0[g]); + v_sum0_hi = Q6_Vw_vrmpyacc_VwVubVb(v_sum0_hi, v_W_hi, v_act0[g + 4]); + v_sum1_lo = Q6_Vw_vrmpyacc_VwVubVb(v_sum1_lo, v_W_lo, v_act1[g]); + v_sum1_hi = Q6_Vw_vrmpyacc_VwVubVb(v_sum1_hi, v_W_hi, v_act1[g + 4]); + } + + *v_sum0 = Q6_Vw_vadd_VwVw(v_sum0_lo, v_sum0_hi); + *v_sum1 = Q6_Vw_vadd_VwVw(v_sum1_lo, v_sum1_hi); +} + // Q2_K: fp16 d (*v_d) and -dmin (*v_m) of the 32 rows of a super-block (8 tiles, VTCM stride 384), each in both vector halves static inline void q2_k_load_dmin(const uint8_t * restrict sb, HVX_Vector * restrict v_d, HVX_Vector * restrict v_m) { HVX_Vector v_dd = Q6_V_vzero(); // lanes 0..31: d, lanes 32..63: -dmin @@ -824,6 +928,86 @@ static inline HVX_Vector q2_k_int_scale(HVX_Vector v_isum, HVX_Vector v_msum, HV return hvx_vec_mul_f32_f32(v_t, v_da32); } +static void tiled_vec_dot_q1_0_32x1(const uint32_t n, float * restrict s, const void * restrict vx, const void * restrict vy, uint32_t valid_rows, const float * restrict sz) { + const uint8_t * restrict tile_ptr = vx; + const uint8_t * restrict y_q = vy; + + HVX_Vector v_sum_float = Q6_V_vzero(); + HVX_Vector v_neg_one = Q6_Vb_vsplat_R(-1); + HVX_Vector v_pos_one = Q6_Vb_vsplat_R(1); + + const uint32_t n_k_tiles = n / HTP_TENSOR_TILE_K_Q1_0; + for (uint32_t kt = 0; kt < n_k_tiles; kt++) { + const HVX_Vector * restrict vptr = (const HVX_Vector *) (tile_ptr + kt * htp_tensor_get_tile_padded_size(HTP_TYPE_Q1_0)); + HVX_Vector v_tile_qf = Q6_V_vzero(); + + #pragma unroll + for (int quarter = 0; quarter < 4; quarter++) { + const HVX_Vector * restrict v_act = (const HVX_Vector *) (y_q + (4 * kt + quarter) * HTP_MM_ACT_TILE_SIZE_Q8_0); + HVX_Vector v_sum_sf = Q6_Vsf_equals_Vw(accum_q1_0_32x1(vptr[quarter], v_act, v_neg_one, v_pos_one)); + HVX_Vector v_da_sf = Q6_V_lo_W(hvx_vec_f16_to_f32_shuff(v_act[8])); + HVX_Vector v_q_qf = Q6_Vqf32_vmpy_VsfVsf(v_sum_sf, v_da_sf); + v_tile_qf = (quarter == 0) ? v_q_qf : Q6_Vqf32_vadd_Vqf32Vqf32(v_tile_qf, v_q_qf); + } + + HVX_Vector v_dw_sf = Q6_V_lo_W(hvx_vec_f16_to_f32(vptr[4])); + v_sum_float = hvx_vec_add_f32_f32(v_sum_float, hvx_vec_mul_f32_f32(Q6_Vsf_equals_Vqf32(v_tile_qf), v_dw_sf)); + } + + if (sz) { + hvx_vec_store_u(s, valid_rows * sizeof(float), hvx_vec_add_f32_f32(v_sum_float, hvx_vmemu(sz))); + } else { + hvx_vec_store_u(s, valid_rows * sizeof(float), v_sum_float); + } +} + +static void tiled_vec_dot_q1_0_32x2(const uint32_t n, float * restrict s0, float * restrict s1, const void * restrict vx, const void * restrict vy0, const void * restrict vy1, uint32_t valid_rows, const float * restrict sz0, const float * restrict sz1) { + const uint8_t * restrict tile_ptr = vx; + const uint8_t * restrict y0_q = vy0; + const uint8_t * restrict y1_q = vy1; + + HVX_Vector v_sum_float_c0 = Q6_V_vzero(); + HVX_Vector v_sum_float_c1 = Q6_V_vzero(); + HVX_Vector v_neg_one = Q6_Vb_vsplat_R(-1); + HVX_Vector v_pos_one = Q6_Vb_vsplat_R(1); + + const uint32_t n_k_tiles = n / HTP_TENSOR_TILE_K_Q1_0; + for (uint32_t kt = 0; kt < n_k_tiles; kt++) { + const HVX_Vector * restrict vptr = (const HVX_Vector *) (tile_ptr + kt * htp_tensor_get_tile_padded_size(HTP_TYPE_Q1_0)); + HVX_Vector v_tile_c0 = Q6_V_vzero(); + HVX_Vector v_tile_c1 = Q6_V_vzero(); + + for (int quarter = 0; quarter < 4; quarter++) { + const uint32_t act_tile = 4 * kt + quarter; + const HVX_Vector * restrict v_act0 = (const HVX_Vector *) (y0_q + act_tile * HTP_MM_ACT_TILE_SIZE_Q8_0); + const HVX_Vector * restrict v_act1 = (const HVX_Vector *) (y1_q + act_tile * HTP_MM_ACT_TILE_SIZE_Q8_0); + HVX_VectorPair v_sums = accum_q1_0_32x2(vptr[quarter], v_act0, v_act1, v_neg_one, v_pos_one); + HVX_Vector v_sum_sf_c0 = Q6_Vsf_equals_Vw(Q6_V_lo_W(v_sums)); + HVX_Vector v_sum_sf_c1 = Q6_Vsf_equals_Vw(Q6_V_hi_W(v_sums)); + HVX_Vector v_da_sf_c0 = Q6_V_lo_W(hvx_vec_f16_to_f32_shuff(v_act0[8])); + HVX_Vector v_da_sf_c1 = Q6_V_lo_W(hvx_vec_f16_to_f32_shuff(v_act1[8])); + + v_tile_c0 = hvx_vec_add_f32_f32(v_tile_c0, hvx_vec_mul_f32_f32(v_sum_sf_c0, v_da_sf_c0)); + v_tile_c1 = hvx_vec_add_f32_f32(v_tile_c1, hvx_vec_mul_f32_f32(v_sum_sf_c1, v_da_sf_c1)); + } + + HVX_Vector v_dw_sf = Q6_V_lo_W(hvx_vec_f16_to_f32(vptr[4])); + v_sum_float_c0 = hvx_vec_add_f32_f32(v_sum_float_c0, hvx_vec_mul_f32_f32(v_tile_c0, v_dw_sf)); + v_sum_float_c1 = hvx_vec_add_f32_f32(v_sum_float_c1, hvx_vec_mul_f32_f32(v_tile_c1, v_dw_sf)); + } + + if (sz0) { + hvx_vec_store_u(s0, valid_rows * sizeof(float), hvx_vec_add_f32_f32(v_sum_float_c0, hvx_vmemu(sz0))); + } else { + hvx_vec_store_u(s0, valid_rows * sizeof(float), v_sum_float_c0); + } + if (sz1) { + hvx_vec_store_u(s1, valid_rows * sizeof(float), hvx_vec_add_f32_f32(v_sum_float_c1, hvx_vmemu(sz1))); + } else { + hvx_vec_store_u(s1, valid_rows * sizeof(float), v_sum_float_c1); + } +} + static void tiled_vec_dot_q4_0_32x1(const uint32_t n, float * restrict s, const void * restrict vx, const void * restrict vy, uint32_t valid_rows, const float * restrict sz) { const uint8_t * restrict tile_ptr = vx; const uint8_t * restrict y_q = vy; @@ -1362,6 +1546,93 @@ static void tiled_vec_dot_q2_k_32x2(const uint32_t n, float * restrict s0, float } } +// One Q2_0 weight tile covers 64 k: act tiles 2 * kt and 2 * kt + 1, each 32 k wide (Q8_1) +static void tiled_vec_dot_q2_0_32x1(const uint32_t n, float * restrict s, const void * restrict vx, const void * restrict vy, uint32_t valid_rows, const float * restrict sz) { + const uint8_t * restrict tile_ptr = vx; + const uint8_t * restrict y_q = vy; + + HVX_Vector v_sum_float = Q6_V_vzero(); + HVX_Vector v_one_f16 = hvx_vec_splat_f16(1.0); + + uint32_t n_weight_tiles = n / HTP_MM_WEIGHT_TILE_K_Q2_0; + for (uint32_t kt = 0; kt < n_weight_tiles; kt++) { + const HVX_Vector * restrict vptr = (const HVX_Vector *) (tile_ptr + kt * HTP_MM_WEIGHT_ALIGNED_TILE_SIZE_Q2_0); + HVX_Vector v_tile_qf = Q6_V_vzero(); + + #pragma unroll + for (int half = 0; half < 2; half++) { + const HVX_Vector * restrict v_act = (const HVX_Vector *) (y_q + (2 * kt + half) * HTP_MM_ACT_TILE_SIZE_Q8_1); + + HVX_Vector v_sum_sf = Q6_Vsf_equals_Vw(accum_q2_0_32x1(vptr + 2 * half, v_act)); + HVX_Vector v_da_sf = Q6_V_lo_W(hvx_vec_f16_to_f32_shuff(v_act[8])); + HVX_Vector v_sa_qf = Q6_V_lo_W(Q6_Wqf32_vmpy_VhfVhf(v_act[9], v_one_f16)); + HVX_Vector v_h_qf = Q6_Vqf32_vsub_Vqf32Vqf32(Q6_Vqf32_vmpy_VsfVsf(v_sum_sf, v_da_sf), v_sa_qf); + v_tile_qf = (half == 0) ? v_h_qf : Q6_Vqf32_vadd_Vqf32Vqf32(v_tile_qf, v_h_qf); + } + + HVX_Vector v_dw_sf = Q6_V_lo_W(hvx_vec_f16_to_f32(vptr[4])); + v_sum_float = hvx_vec_add_f32_f32(v_sum_float, hvx_vec_mul_f32_f32(Q6_Vsf_equals_Vqf32(v_tile_qf), v_dw_sf)); + } + + if (sz) { + hvx_vec_store_u(s, valid_rows * sizeof(float), hvx_vec_add_f32_f32(v_sum_float, hvx_vmemu(sz))); + } else { + hvx_vec_store_u(s, valid_rows * sizeof(float), v_sum_float); + } +} + +static void tiled_vec_dot_q2_0_32x2(const uint32_t n, float * restrict s0, float * restrict s1, const void * restrict vx, const void * restrict vy0, const void * restrict vy1, uint32_t valid_rows, const float * restrict sz0, const float * restrict sz1) { + const uint8_t * restrict tile_ptr = vx; + const uint8_t * restrict y0_q = vy0; + const uint8_t * restrict y1_q = vy1; + + HVX_Vector v_sum_float_c0 = Q6_V_vzero(); + HVX_Vector v_sum_float_c1 = Q6_V_vzero(); + HVX_Vector v_one_f16 = hvx_vec_splat_f16(1.0); + + uint32_t n_weight_tiles = n / HTP_MM_WEIGHT_TILE_K_Q2_0; + for (uint32_t kt = 0; kt < n_weight_tiles; kt++) { + const HVX_Vector * restrict vptr = (const HVX_Vector *) (tile_ptr + kt * HTP_MM_WEIGHT_ALIGNED_TILE_SIZE_Q2_0); + HVX_Vector v_tile_qf_c0 = Q6_V_vzero(); + HVX_Vector v_tile_qf_c1 = Q6_V_vzero(); + + #pragma unroll + for (int half = 0; half < 2; half++) { + const HVX_Vector * restrict v_act0 = (const HVX_Vector *) (y0_q + (2 * kt + half) * HTP_MM_ACT_TILE_SIZE_Q8_1); + const HVX_Vector * restrict v_act1 = (const HVX_Vector *) (y1_q + (2 * kt + half) * HTP_MM_ACT_TILE_SIZE_Q8_1); + + HVX_Vector v_sum_c0, v_sum_c1; + accum_q2_0_32x2(vptr + 2 * half, v_act0, v_act1, &v_sum_c0, &v_sum_c1); + + HVX_Vector v_da_sf_c0 = Q6_V_lo_W(hvx_vec_f16_to_f32_shuff(v_act0[8])); + HVX_Vector v_da_sf_c1 = Q6_V_lo_W(hvx_vec_f16_to_f32_shuff(v_act1[8])); + HVX_Vector v_sa_qf_c0 = Q6_V_lo_W(Q6_Wqf32_vmpy_VhfVhf(v_act0[9], v_one_f16)); + HVX_Vector v_sa_qf_c1 = Q6_V_lo_W(Q6_Wqf32_vmpy_VhfVhf(v_act1[9], v_one_f16)); + + HVX_Vector v_h_qf_c0 = Q6_Vqf32_vsub_Vqf32Vqf32(Q6_Vqf32_vmpy_VsfVsf(Q6_Vsf_equals_Vw(v_sum_c0), v_da_sf_c0), v_sa_qf_c0); + HVX_Vector v_h_qf_c1 = Q6_Vqf32_vsub_Vqf32Vqf32(Q6_Vqf32_vmpy_VsfVsf(Q6_Vsf_equals_Vw(v_sum_c1), v_da_sf_c1), v_sa_qf_c1); + + v_tile_qf_c0 = (half == 0) ? v_h_qf_c0 : Q6_Vqf32_vadd_Vqf32Vqf32(v_tile_qf_c0, v_h_qf_c0); + v_tile_qf_c1 = (half == 0) ? v_h_qf_c1 : Q6_Vqf32_vadd_Vqf32Vqf32(v_tile_qf_c1, v_h_qf_c1); + } + + HVX_Vector v_dw_sf = Q6_V_lo_W(hvx_vec_f16_to_f32(vptr[4])); + v_sum_float_c0 = hvx_vec_add_f32_f32(v_sum_float_c0, hvx_vec_mul_f32_f32(Q6_Vsf_equals_Vqf32(v_tile_qf_c0), v_dw_sf)); + v_sum_float_c1 = hvx_vec_add_f32_f32(v_sum_float_c1, hvx_vec_mul_f32_f32(Q6_Vsf_equals_Vqf32(v_tile_qf_c1), v_dw_sf)); + } + + if (sz0) { + hvx_vec_store_u(s0, valid_rows * sizeof(float), hvx_vec_add_f32_f32(v_sum_float_c0, hvx_vmemu(sz0))); + } else { + hvx_vec_store_u(s0, valid_rows * sizeof(float), v_sum_float_c0); + } + if (sz1) { + hvx_vec_store_u(s1, valid_rows * sizeof(float), hvx_vec_add_f32_f32(v_sum_float_c1, hvx_vmemu(sz1))); + } else { + hvx_vec_store_u(s1, valid_rows * sizeof(float), v_sum_float_c1); + } +} + static void tiled_vec_dot_iq4nl_32x1(const uint32_t n, float * restrict s, const void * restrict vx, const void * restrict vy, uint32_t valid_rows, const float * restrict sz) { const uint8_t * restrict tile_ptr = vx; const uint8_t * restrict y_q = vy; diff --git a/ggml/src/ggml-hexagon/htp/matmul-ops.c b/ggml/src/ggml-hexagon/htp/matmul-ops.c index 3837c290d..9af312925 100644 --- a/ggml/src/ggml-hexagon/htp/matmul-ops.c +++ b/ggml/src/ggml-hexagon/htp/matmul-ops.c @@ -277,10 +277,9 @@ static void hvx_mm_2d_tiled(unsigned int nth, unsigned int ith, void * data) { const uint32_t tile_size = htp_tensor_get_tile_size(src0->type); const uint32_t aligned_tile_size = htp_tensor_get_tile_padded_size(src0->type); - uint32_t n_k_tiles_w = ne00 / 32; - uint32_t n_k_tiles_a = ne10 / 32; + uint32_t n_k_tiles_w = ne00 / htp_tensor_get_tile_k(src0->type); uint32_t tile_row_stride = n_k_tiles_w * tile_size; - uint32_t tile_row_transfer_size_aligned = n_k_tiles_a * aligned_tile_size; + uint32_t tile_row_transfer_size_aligned = n_k_tiles_w * aligned_tile_size; uint32_t ct_start = src0_start_row / 32; uint32_t ct_end = (src0_end_row + 31) / 32; @@ -289,7 +288,7 @@ static void hvx_mm_2d_tiled(unsigned int nth, unsigned int ith, void * data) { if (src0_start_row < src0_end_row) { for (uint32_t d = 0; d < n_prefetch && push_ct < ct_end; d++, push_ct++) { dma_queue_push(dma_q, dma_make_data(vtcm_src0_ptr + d * tile_row_transfer_size_aligned, - src0_row + push_ct * tile_row_stride), aligned_tile_size, tile_size, tile_size, n_k_tiles_a); + src0_row + push_ct * tile_row_stride), aligned_tile_size, tile_size, tile_size, n_k_tiles_w); } } @@ -341,7 +340,7 @@ static void hvx_mm_2d_tiled(unsigned int nth, unsigned int ith, void * data) { if (push_ct < ct_end) { dma_queue_push(dma_q, dma_make_data(w_tile, src0_row + push_ct * tile_row_stride), - aligned_tile_size, tile_size, tile_size, n_k_tiles_a); + aligned_tile_size, tile_size, tile_size, n_k_tiles_w); push_ct++; } } @@ -379,10 +378,9 @@ static void hvx_mv_2d_tiled(unsigned int nth, unsigned int ith, void * data) { const uint32_t tile_size = htp_tensor_get_tile_size(src0->type); const uint32_t aligned_tile_size = htp_tensor_get_tile_padded_size(src0->type); - uint32_t n_k_tiles_w = ne00 / 32; - uint32_t n_k_tiles_a = ne10 / 32; + uint32_t n_k_tiles_w = ne00 / htp_tensor_get_tile_k(src0->type); uint32_t tile_row_stride = n_k_tiles_w * tile_size; - uint32_t tile_row_transfer_size_aligned = n_k_tiles_a * aligned_tile_size; + uint32_t tile_row_transfer_size_aligned = n_k_tiles_w * aligned_tile_size; uint32_t ct_start = src0_start_row / 32; uint32_t ct_end = (src0_end_row + 31) / 32; @@ -401,7 +399,7 @@ static void hvx_mv_2d_tiled(unsigned int nth, unsigned int ith, void * data) { } for (uint32_t d = 0; d < n_prefetch && push_ct < ct_end; d++, push_ct++) { dma_queue_push(dma_q, dma_make_data(vtcm_src0_ptr + d * tile_row_transfer_size_aligned, - src0_row + push_ct * tile_row_stride), aligned_tile_size, tile_size, tile_size, n_k_tiles_a); + src0_row + push_ct * tile_row_stride), aligned_tile_size, tile_size, tile_size, n_k_tiles_w); } } @@ -422,7 +420,7 @@ static void hvx_mv_2d_tiled(unsigned int nth, unsigned int ith, void * data) { if (push_ct < ct_end) { dma_queue_push(dma_q, dma_make_data(w_tile, src0_row + push_ct * tile_row_stride), - aligned_tile_size, tile_size, tile_size, n_k_tiles_a); + aligned_tile_size, tile_size, tile_size, n_k_tiles_w); push_ct++; } } @@ -460,8 +458,6 @@ static void hvx_mm_nx_2d_tiled(unsigned int nth, unsigned int ith, void * data) const uint32_t n_prefetch = kparams->n_prefetch; assert(n_prefetch >= 2 && n_prefetch <= HTP_MM_MAX_PREFETCH && (n_prefetch & (n_prefetch - 1)) == 0); - uint32_t n_k_tiles_a = ne10 / 32; - for (uint32_t widx = 0; widx < n_weights; widx++) { const struct htp_tensor * restrict src_w = octx->src[widx]; const struct htp_tensor * restrict dst = octx->dsts[widx]; @@ -470,15 +466,14 @@ static void hvx_mm_nx_2d_tiled(unsigned int nth, unsigned int ith, void * data) const uint32_t tile_size = htp_tensor_get_tile_size(src_w->type); const uint32_t aligned_tile_size = htp_tensor_get_tile_padded_size(src_w->type); - const uint32_t tile_row_transfer_size_aligned = n_k_tiles_a * aligned_tile_size; - const uint32_t ne00 = src_w->ne[0]; const uint32_t ne01 = src_w->ne[1]; const size_t dst_row_size = dst->nb[1]; const dma_addr_t src_w_row = src_w->data; - uint32_t n_k_tiles_w = ne00 / 32; - uint32_t tile_row_stride = n_k_tiles_w * tile_size; + const uint32_t n_k_tiles_w = ne00 / htp_tensor_get_tile_k(src_w->type); + const uint32_t tile_row_stride = n_k_tiles_w * tile_size; + const uint32_t tile_row_transfer_size_aligned = n_k_tiles_w * aligned_tile_size; uint32_t src0_start_row = 0; uint32_t src0_end_row = ne01; @@ -504,7 +499,7 @@ static void hvx_mm_nx_2d_tiled(unsigned int nth, unsigned int ith, void * data) uint32_t push_ct = ct_start; for (uint32_t d = 0; d < n_prefetch && push_ct < ct_end; d++, push_ct++) { dma_queue_push(dma_q, dma_make_data(vtcm_weight_ptr + d * tile_row_transfer_size_aligned, - src_w_row + push_ct * tile_row_stride), aligned_tile_size, tile_size, tile_size, n_k_tiles_a); + src_w_row + push_ct * tile_row_stride), aligned_tile_size, tile_size, tile_size, n_k_tiles_w); } for (uint32_t ct = ct_start; ct < ct_end; ct++) { @@ -536,7 +531,7 @@ static void hvx_mm_nx_2d_tiled(unsigned int nth, unsigned int ith, void * data) if (push_ct < ct_end) { dma_queue_push(dma_q, dma_make_data(w_tile, src_w_row + push_ct * tile_row_stride), - aligned_tile_size, tile_size, tile_size, n_k_tiles_a); + aligned_tile_size, tile_size, tile_size, n_k_tiles_w); push_ct++; } } @@ -840,10 +835,9 @@ static void hvx_mm_4d_tiled(unsigned int nth, unsigned int ith, void * data) { const uint32_t tile_size = htp_tensor_get_tile_size(src0->type); const uint32_t aligned_tile_size = htp_tensor_get_tile_padded_size(src0->type); - const uint32_t n_k_tiles_w = ne00 / 32; - const uint32_t n_k_tiles_a = ne10 / 32; + const uint32_t n_k_tiles_w = ne00 / htp_tensor_get_tile_k(src0->type); const uint32_t tile_row_stride = n_k_tiles_w * tile_size; - const uint32_t tile_row_transfer_size_aligned = n_k_tiles_a * aligned_tile_size; + const uint32_t tile_row_transfer_size_aligned = n_k_tiles_w * aligned_tile_size; const uint32_t src0_slice_stride = ((ne01 + 31) / 32) * tile_row_stride; const uint32_t ct_start = src0_start_row / 32; @@ -884,7 +878,7 @@ static void hvx_mm_4d_tiled(unsigned int nth, unsigned int ith, void * data) { for (uint32_t d = 0; d < n_prefetch && push_ct < ct_end; d++, push_ct++) { dma_queue_push(dma_q, dma_make_data(vtcm_src0_ptr + d * tile_row_transfer_size_aligned, src0_slice + (size_t) push_ct * tile_row_stride), - aligned_tile_size, tile_size, tile_size, n_k_tiles_a); + aligned_tile_size, tile_size, tile_size, n_k_tiles_w); } for (uint32_t ct = ct_start; ct < ct_end; ct++) { @@ -929,7 +923,7 @@ static void hvx_mm_4d_tiled(unsigned int nth, unsigned int ith, void * data) { if (push_ct < ct_end) { dma_queue_push(dma_q, dma_make_data(w_tile, src0_slice + (size_t) push_ct * tile_row_stride), - aligned_tile_size, tile_size, tile_size, n_k_tiles_a); + aligned_tile_size, tile_size, tile_size, n_k_tiles_w); push_ct++; } } @@ -1343,10 +1337,9 @@ static void hvx_mm_id_tiled(unsigned int nth, unsigned int ith, void * data) { const uint32_t tile_size = htp_tensor_get_tile_size(src0->type); const uint32_t aligned_tile_size = htp_tensor_get_tile_padded_size(src0->type); - const uint32_t n_k_tiles_w = ne00 / 32; - const uint32_t n_k_tiles_a = ne10 / 32; + const uint32_t n_k_tiles_w = ne00 / htp_tensor_get_tile_k(src0->type); const uint32_t tile_row_stride = n_k_tiles_w * tile_size; - const uint32_t tile_row_transfer_size_aligned = n_k_tiles_a * aligned_tile_size; + const uint32_t tile_row_transfer_size_aligned = n_k_tiles_w * aligned_tile_size; const uint32_t ct_start = src0_start_row / 32; const uint32_t ct_end = (src0_end_row + 31) / 32; @@ -1354,7 +1347,7 @@ static void hvx_mm_id_tiled(unsigned int nth, unsigned int ith, void * data) { uint32_t push_ct = ct_start; for (uint32_t d = 0; d < n_prefetch && push_ct < ct_end; d++, push_ct++) { dma_queue_push(dma_q, dma_make_data(vtcm_src0_ptr + d * tile_row_transfer_size_aligned, src0_row + push_ct * tile_row_stride), - aligned_tile_size, tile_size, tile_size, n_k_tiles_a); + aligned_tile_size, tile_size, tile_size, n_k_tiles_w); } for (uint32_t ct = ct_start; ct < ct_end; ct++) { @@ -1379,7 +1372,7 @@ static void hvx_mm_id_tiled(unsigned int nth, unsigned int ith, void * data) { if (push_ct < ct_end) { dma_queue_push(dma_q, dma_make_data(w_tile, src0_row + push_ct * tile_row_stride), - aligned_tile_size, tile_size, tile_size, n_k_tiles_a); + aligned_tile_size, tile_size, tile_size, n_k_tiles_w); push_ct++; } } @@ -1431,10 +1424,9 @@ static void hvx_mv_id_tiled(unsigned int nth, unsigned int ith, void * data) { const uint32_t tile_size = htp_tensor_get_tile_size(src0->type); const uint32_t aligned_tile_size = htp_tensor_get_tile_padded_size(src0->type); - const uint32_t n_k_tiles_w = ne00 / 32; - const uint32_t n_k_tiles_a = ne10 / 32; + const uint32_t n_k_tiles_w = ne00 / htp_tensor_get_tile_k(src0->type); const uint32_t tile_row_stride = n_k_tiles_w * tile_size; - const uint32_t tile_row_transfer_size_aligned = n_k_tiles_a * aligned_tile_size; + const uint32_t tile_row_transfer_size_aligned = n_k_tiles_w * aligned_tile_size; const uint32_t ct_start = src0_start_row / 32; const uint32_t ct_end = (src0_end_row + 31) / 32; @@ -1442,7 +1434,7 @@ static void hvx_mv_id_tiled(unsigned int nth, unsigned int ith, void * data) { uint32_t push_ct = ct_start; for (uint32_t d = 0; d < n_prefetch && push_ct < ct_end; d++, push_ct++) { dma_queue_push(dma_q, dma_make_data(vtcm_src0_ptr + d * tile_row_transfer_size_aligned, src0_row + push_ct * tile_row_stride), - aligned_tile_size, tile_size, tile_size, n_k_tiles_a); + aligned_tile_size, tile_size, tile_size, n_k_tiles_w); } for (uint32_t ct = ct_start; ct < ct_end; ct++) { @@ -1457,7 +1449,7 @@ static void hvx_mv_id_tiled(unsigned int nth, unsigned int ith, void * data) { if (push_ct < ct_end) { dma_queue_push(dma_q, dma_make_data(w_tile, src0_row + push_ct * tile_row_stride), - aligned_tile_size, tile_size, tile_size, n_k_tiles_a); + aligned_tile_size, tile_size, tile_size, n_k_tiles_w); push_ct++; } } @@ -1520,10 +1512,9 @@ static void hvx_mv_id_nx_tiled(unsigned int nth, unsigned int ith, void * data) const uint32_t tile_size = htp_tensor_get_tile_size(src_w->type); const uint32_t aligned_tile_size = htp_tensor_get_tile_padded_size(src_w->type); - const uint32_t n_k_tiles_w = src_w->ne[0] / 32; - const uint32_t n_k_tiles_a = act->ne[0] / 32; + const uint32_t n_k_tiles_w = src_w->ne[0] / htp_tensor_get_tile_k(src_w->type); const uint32_t tile_row_stride = n_k_tiles_w * tile_size; - const uint32_t tile_row_transfer_size_aligned = n_k_tiles_a * aligned_tile_size; + const uint32_t tile_row_transfer_size_aligned = n_k_tiles_w * aligned_tile_size; const uint32_t ct_start = src0_start_row / 32; const uint32_t ct_end = (src0_end_row + 31) / 32; @@ -1531,7 +1522,7 @@ static void hvx_mv_id_nx_tiled(unsigned int nth, unsigned int ith, void * data) uint32_t push_ct = ct_start; for (uint32_t d = 0; d < n_prefetch && push_ct < ct_end; d++, push_ct++) { dma_queue_push(dma_q, dma_make_data(vtcm_src0_ptr + d * tile_row_transfer_size_aligned, src0_row + push_ct * tile_row_stride), - aligned_tile_size, tile_size, tile_size, n_k_tiles_a); + aligned_tile_size, tile_size, tile_size, n_k_tiles_w); } for (uint32_t ct = ct_start; ct < ct_end; ct++) { @@ -1546,7 +1537,7 @@ static void hvx_mv_id_nx_tiled(unsigned int nth, unsigned int ith, void * data) if (push_ct < ct_end) { dma_queue_push(dma_q, dma_make_data(w_tile, src0_row + push_ct * tile_row_stride), - aligned_tile_size, tile_size, tile_size, n_k_tiles_a); + aligned_tile_size, tile_size, tile_size, n_k_tiles_w); push_ct++; } } @@ -1611,10 +1602,9 @@ static void hvx_mm_id_nx_tiled(unsigned int nth, unsigned int ith, void * data) const uint32_t tile_size = htp_tensor_get_tile_size(src_w->type); const uint32_t aligned_tile_size = htp_tensor_get_tile_padded_size(src_w->type); - const uint32_t n_k_tiles_w = src_w->ne[0] / 32; - const uint32_t n_k_tiles_a = act->ne[0] / 32; + const uint32_t n_k_tiles_w = src_w->ne[0] / htp_tensor_get_tile_k(src_w->type); const uint32_t tile_row_stride = n_k_tiles_w * tile_size; - const uint32_t tile_row_transfer_size_aligned = n_k_tiles_a * aligned_tile_size; + const uint32_t tile_row_transfer_size_aligned = n_k_tiles_w * aligned_tile_size; const uint32_t ct_start = src0_start_row / 32; const uint32_t ct_end = (src0_end_row + 31) / 32; @@ -1622,7 +1612,7 @@ static void hvx_mm_id_nx_tiled(unsigned int nth, unsigned int ith, void * data) uint32_t push_ct = ct_start; for (uint32_t d = 0; d < n_prefetch && push_ct < ct_end; d++, push_ct++) { dma_queue_push(dma_q, dma_make_data(vtcm_src0_ptr + d * tile_row_transfer_size_aligned, src0_row + push_ct * tile_row_stride), - aligned_tile_size, tile_size, tile_size, n_k_tiles_a); + aligned_tile_size, tile_size, tile_size, n_k_tiles_w); } for (uint32_t ct = ct_start; ct < ct_end; ct++) { @@ -1647,7 +1637,7 @@ static void hvx_mm_id_nx_tiled(unsigned int nth, unsigned int ith, void * data) if (push_ct < ct_end) { dma_queue_push(dma_q, dma_make_data(w_tile, src0_row + push_ct * tile_row_stride), - aligned_tile_size, tile_size, tile_size, n_k_tiles_a); + aligned_tile_size, tile_size, tile_size, n_k_tiles_w); push_ct++; } } @@ -1693,6 +1683,11 @@ static int hvx_mm_init_tiled_vec_dot(struct htp_mm_context * mmctx, enum htp_dat mmctx->vec_dot_32x1 = tiled_vec_dot_q2_k_32x1; mmctx->vec_dot_32x2 = tiled_vec_dot_q2_k_32x2; return 0; + case HTP_TYPE_Q2_0: + mmctx->type = "q2_0_tiled-f32"; + mmctx->vec_dot_32x1 = tiled_vec_dot_q2_0_32x1; + mmctx->vec_dot_32x2 = tiled_vec_dot_q2_0_32x2; + return 0; case HTP_TYPE_IQ4_NL: mmctx->type = "iq4nl_tiled-f32"; mmctx->vec_dot_32x1 = tiled_vec_dot_iq4nl_32x1; @@ -1703,6 +1698,11 @@ static int hvx_mm_init_tiled_vec_dot(struct htp_mm_context * mmctx, enum htp_dat mmctx->vec_dot_32x1 = tiled_vec_dot_mxfp4_32x1; mmctx->vec_dot_32x2 = tiled_vec_dot_mxfp4_32x2; return 0; + case HTP_TYPE_Q1_0: + mmctx->type = "q1_0_tiled-f32"; + mmctx->vec_dot_32x1 = tiled_vec_dot_q1_0_32x1; + mmctx->vec_dot_32x2 = tiled_vec_dot_q1_0_32x2; + return 0; case HTP_TYPE_F16: mmctx->type = "f16_tiled-f16"; mmctx->vec_dot_32x1 = tiled_vec_dot_f16_32x1; @@ -2325,6 +2325,8 @@ static inline tiled_dequant_task_fn_t hmx_get_dequant_worker_fn(int weight_type, case HTP_TYPE_Q6_K: return dequantize_tiled_weight_to_fp16_task_q6_k; case HTP_TYPE_Q3_K: return dequantize_tiled_weight_to_fp16_task_q3_k; case HTP_TYPE_Q2_K: return dequantize_tiled_weight_to_fp16_task_q2_k; + case HTP_TYPE_Q1_0: return dequantize_tiled_weight_to_fp16_task_q1_0; + case HTP_TYPE_Q2_0: return dequantize_tiled_weight_to_fp16_task_q2_0; case HTP_TYPE_F16: return is_tiled ? NULL : convert_f16_weight_to_fp16_tiles_task; case HTP_TYPE_BF16: return convert_bf16_weight_to_fp16_tiles_task; case HTP_TYPE_F32: return quantize_f32_weight_to_fp16_tiles_task; @@ -2346,8 +2348,9 @@ static void dequantize_tiled_weight_chunk_to_fp16_tiles( assert(k_block % HTP_MM_HMX_TILE_N_COLS == 0); - size_t n_col_tiles = hmx_ceil_div(n_cols, HTP_MM_HMX_TILE_N_COLS); - size_t n_tot_tiles = n_col_tiles * n_k_tiles; + const size_t n_col_tiles = hmx_ceil_div(n_cols, HTP_MM_HMX_TILE_N_COLS); + const uint32_t n_weight_tiles = k_block / htp_tensor_get_tile_k(weight_type); + const size_t n_tot_tiles = n_col_tiles * n_weight_tiles; size_t n_tiles_per_task = (n_threads == 1) ? n_tot_tiles : hmx_ceil_div(n_tot_tiles, n_threads); @@ -2363,6 +2366,7 @@ static void dequantize_tiled_weight_chunk_to_fp16_tiles( state.row_stride = row_stride; state.weight_type = weight_type; state.n_k_tiles = n_k_tiles; + state.n_weight_tiles = n_weight_tiles; state.n_k_tiles_div = n_k_tiles_div; state.traces = ctx->trace; state.ctx = ctx; @@ -2666,6 +2670,7 @@ static int hmx_mm_2d_f32(struct htp_context *ctx, tiled_dequant_task_fn_t dequant_worker_fn = hmx_get_dequant_worker_fn(weight_type, is_tiled); const int n_k_tiles = k / HTP_MM_HMX_TILE_N_COLS; + const int n_weight_tiles = k / htp_tensor_get_tile_k(weight_type); const struct fastdiv_values n_k_tiles_div = init_fastdiv_values(n_k_tiles); const size_t vec_dot_size = k * sizeof(__fp16); @@ -2679,8 +2684,6 @@ static int hmx_mm_2d_f32(struct htp_context *ctx, size_t n_chunk_n_cols = n_chunk; size_t vtcm_used = vtcm_size; - const size_t qweight_row_stride = is_tiled ? (size_t)(n_k_tiles * aligned_tile_size) / 32 : 0; - struct htp_mm_hmx_vtcm_layout L; htp_mm_hmx_vtcm_layout_build(&L, HTP_MM_KERNEL_HMX_2D, weight_type, k, m_chunk_n_rows, n_chunk_n_cols, 1, pipeline, act_threads, aligned_tile_size, src2_bytes); @@ -2759,13 +2762,13 @@ static int hmx_mm_2d_f32(struct htp_context *ctx, // Prologue: push A0 and optionally A1 (if n_chunk_cnt > 1) const size_t n_cols_A0 = hex_smin(n - 0 * n_chunk_n_cols, n_chunk_n_cols); - const uint32_t height_A0 = is_tiled ? hmx_ceil_div(n_cols_A0, 32) * n_k_tiles : n_cols_A0; + const uint32_t height_A0 = is_tiled ? hmx_ceil_div(n_cols_A0, 32) * n_weight_tiles : n_cols_A0; dma_queue_push(weight_dma, dma_make_data(vtcm_raw_bufs[0], weight), dma_dst_stride, dma_src_stride, dma_width_bytes, height_A0); if (1 < n_chunk_cnt) { const size_t n_cols_A1 = hex_smin(n - 1 * n_chunk_n_cols, n_chunk_n_cols); - const uint32_t height_A1 = is_tiled ? hmx_ceil_div(n_cols_A1, 32) * n_k_tiles : n_cols_A1; + const uint32_t height_A1 = is_tiled ? hmx_ceil_div(n_cols_A1, 32) * n_weight_tiles : n_cols_A1; dma_queue_push(weight_dma, dma_make_data(vtcm_raw_bufs[1], weight + n_chunk_n_cols * weight_stride), dma_dst_stride, dma_src_stride, dma_width_bytes, height_A1); } @@ -2789,7 +2792,7 @@ static int hmx_mm_2d_f32(struct htp_context *ctx, // 3. push A_{i+2} (if i+2 < n_chunk_cnt) if (i + 2 < n_chunk_cnt) { - const uint32_t height_p2 = is_tiled ? hmx_ceil_div(n_cols_p2, 32) * n_k_tiles : n_cols_p2; + const uint32_t height_p2 = is_tiled ? hmx_ceil_div(n_cols_p2, 32) * n_weight_tiles : n_cols_p2; dma_queue_push(weight_dma, dma_make_data(vtcm_raw_bufs[(i + 2) % 4], weight + nc_p2 * weight_stride), dma_dst_stride, dma_src_stride, dma_width_bytes, height_p2); } @@ -2855,7 +2858,7 @@ static int hmx_mm_2d_f32(struct htp_context *ctx, // A0: Pre-fetch the first weight chunk (nc = 0) if (n > 0) { const size_t n_cols = hex_smin(n, n_chunk_n_cols); - const uint32_t height = is_tiled ? hmx_ceil_div(n_cols, 32) * n_k_tiles : n_cols; + const uint32_t height = is_tiled ? hmx_ceil_div(n_cols, 32) * n_weight_tiles : n_cols; dma_queue_push(weight_dma, dma_make_data(vtcm_raw_bufs[0], weight), dma_dst_stride, dma_src_stride, dma_width_bytes, height); } @@ -2878,7 +2881,7 @@ static int hmx_mm_2d_f32(struct htp_context *ctx, const size_t nc_next = nc + n_chunk_n_cols; if (nc_next < n) { const size_t n_cols_next = hex_smin(n - nc_next, n_chunk_n_cols); - const uint32_t height_next = is_tiled ? hmx_ceil_div(n_cols_next, 32) * n_k_tiles : n_cols_next; + const uint32_t height_next = is_tiled ? hmx_ceil_div(n_cols_next, 32) * n_weight_tiles : n_cols_next; dma_queue_push(weight_dma, dma_make_data(vtcm_raw_bufs[(i + 1) % 2], weight + nc_next * weight_stride), dma_dst_stride, dma_src_stride, dma_width_bytes, height_next); } @@ -2930,6 +2933,7 @@ static int hmx_mm_nx_2d_f32(struct htp_ops_context * octx, const struct htp_mm_k } const int n_k_tiles = k / HTP_MM_HMX_TILE_N_COLS; + const int n_weight_tiles = k / htp_tensor_get_tile_k(weight_type); const struct fastdiv_values n_k_tiles_div = init_fastdiv_values(n_k_tiles); const int tile_size = kparams->tile_size; @@ -3057,13 +3061,13 @@ static int hmx_mm_nx_2d_f32(struct htp_ops_context * octx, const struct htp_mm_k const uint32_t dma_src_stride = is_tiled ? tile_size : weight_stride; const size_t n_cols_A0 = hex_smin(n - 0 * n_chunk_n_cols, n_chunk_n_cols); - const uint32_t height_A0 = is_tiled ? hmx_ceil_div(n_cols_A0, 32) * n_k_tiles : n_cols_A0; + const uint32_t height_A0 = is_tiled ? hmx_ceil_div(n_cols_A0, 32) * n_weight_tiles : n_cols_A0; dma_queue_push(weight_dma, dma_make_data(vtcm_raw_bufs[0], weight), dma_dst_stride, dma_src_stride, dma_width_bytes, height_A0); if (1 < n_chunk_cnt) { const size_t n_cols_A1 = hex_smin(n - 1 * n_chunk_n_cols, n_chunk_n_cols); - const uint32_t height_A1 = is_tiled ? hmx_ceil_div(n_cols_A1, 32) * n_k_tiles : n_cols_A1; + const uint32_t height_A1 = is_tiled ? hmx_ceil_div(n_cols_A1, 32) * n_weight_tiles : n_cols_A1; dma_queue_push(weight_dma, dma_make_data(vtcm_raw_bufs[1], weight + n_chunk_n_cols * weight_stride), dma_dst_stride, dma_src_stride, dma_width_bytes, height_A1); } @@ -3083,7 +3087,7 @@ static int hmx_mm_nx_2d_f32(struct htp_ops_context * octx, const struct htp_mm_k n_k_tiles, n_k_tiles_div, dequant_worker_fn, n_threads); if (i + 2 < n_chunk_cnt) { - const uint32_t height_p2 = is_tiled ? hmx_ceil_div(n_cols_p2, 32) * n_k_tiles : n_cols_p2; + const uint32_t height_p2 = is_tiled ? hmx_ceil_div(n_cols_p2, 32) * n_weight_tiles : n_cols_p2; dma_queue_push(weight_dma, dma_make_data(vtcm_raw_bufs[(i + 2) % 4], weight + nc_p2 * weight_stride), dma_dst_stride, dma_src_stride, dma_width_bytes, height_p2); } @@ -3160,7 +3164,7 @@ static int hmx_mm_nx_2d_f32(struct htp_ops_context * octx, const struct htp_mm_k if (n > 0) { const size_t n_cols = hex_smin(n, n_chunk_n_cols); - const uint32_t height = is_tiled ? hmx_ceil_div(n_cols, 32) * n_k_tiles : n_cols; + const uint32_t height = is_tiled ? hmx_ceil_div(n_cols, 32) * n_weight_tiles : n_cols; dma_queue_push(weight_dma, dma_make_data(vtcm_raw_bufs[0], weight), dma_dst_stride, dma_src_stride, dma_width_bytes, height); } @@ -3180,7 +3184,7 @@ static int hmx_mm_nx_2d_f32(struct htp_ops_context * octx, const struct htp_mm_k const size_t nc_next = nc + n_chunk_n_cols; if (nc_next < n) { const size_t n_cols_next = hex_smin(n - nc_next, n_chunk_n_cols); - const uint32_t height_next = is_tiled ? hmx_ceil_div(n_cols_next, 32) * n_k_tiles : n_cols_next; + const uint32_t height_next = is_tiled ? hmx_ceil_div(n_cols_next, 32) * n_weight_tiles : n_cols_next; dma_queue_push(weight_dma, dma_make_data(vtcm_raw_bufs[(i + 1) % 2], weight + nc_next * weight_stride), dma_dst_stride, dma_src_stride, dma_width_bytes, height_next); } @@ -3544,6 +3548,7 @@ static int hmx_mm_id_2d_f32(struct htp_context *ctx, } const int n_k_tiles = k / HTP_MM_HMX_TILE_N_COLS; + const uint32_t n_weight_tiles = k / htp_tensor_get_tile_k(weight_type); const struct fastdiv_values n_k_tiles_div = init_fastdiv_values(n_k_tiles); const size_t vec_dot_size = k * sizeof(__fp16); @@ -3557,7 +3562,7 @@ static int hmx_mm_id_2d_f32(struct htp_context *ctx, const uint32_t dma_src_stride = is_tiled ? tile_size : weight_stride; const uint32_t dma_width_bytes = is_tiled ? tile_size : row_stride; - const size_t qweight_row_stride = is_tiled ? (size_t)(n_k_tiles * aligned_tile_size) / 32 : 0; + const size_t qweight_row_stride = is_tiled ? (size_t)(n_weight_tiles * aligned_tile_size) / 32 : 0; const size_t weight_row_stride = is_tiled ? qweight_row_stride : row_stride; size_t size_per_n = 0, size_per_m = 0, size_per_mn = 0; @@ -3613,7 +3618,7 @@ static int hmx_mm_id_2d_f32(struct htp_context *ctx, // A0: Pre-fetch the first weight chunk (nc = 0) if (n > 0) { const size_t n_cols = hex_smin((size_t) n, n_chunk_n_cols); - const uint32_t height = is_tiled ? hmx_ceil_div(n_cols, 32) * n_k_tiles : n_cols; + const uint32_t height = is_tiled ? hmx_ceil_div(n_cols, 32) * n_weight_tiles : n_cols; dma_queue_push(weight_dma, dma_make_data(vtcm_raw_bufs[0], weight), dma_dst_stride, dma_src_stride, dma_width_bytes, height); } @@ -3637,7 +3642,7 @@ static int hmx_mm_id_2d_f32(struct htp_context *ctx, const size_t nc_next = nc + n_chunk_n_cols; if (nc_next < (size_t) n) { const size_t n_cols_next = hex_smin((size_t) n - nc_next, n_chunk_n_cols); - const uint32_t height_next = is_tiled ? hmx_ceil_div(n_cols_next, 32) * n_k_tiles : n_cols_next; + const uint32_t height_next = is_tiled ? hmx_ceil_div(n_cols_next, 32) * n_weight_tiles : n_cols_next; dma_queue_push(weight_dma, dma_make_data(vtcm_raw_bufs[(i + 1) % 2], weight + nc_next * weight_stride), dma_dst_stride, dma_src_stride, dma_width_bytes, height_next); } diff --git a/ggml/src/ggml-hexagon/htp/matmul-ops.h b/ggml/src/ggml-hexagon/htp/matmul-ops.h index 339eea4a7..09875a603 100644 --- a/ggml/src/ggml-hexagon/htp/matmul-ops.h +++ b/ggml/src/ggml-hexagon/htp/matmul-ops.h @@ -20,6 +20,10 @@ extern "C" { #define HTP_MM_HMX_TILE_N_ELMS 1024 #define HTP_MM_HMX_MIN_NROWS 4 +#define HTP_MM_WEIGHT_TILE_K_Q2_0 64 +#define HTP_MM_WEIGHT_TILE_SIZE_Q2_0 HTP_TENSOR_TILE_SIZE_Q2_0 +#define HTP_MM_WEIGHT_ALIGNED_TILE_SIZE_Q2_0 640 + // --- Activation Tiled Block Sizes (including padding) --- #define HTP_MM_ACT_TILE_SIZE_Q8_0 1152 #define HTP_MM_ACT_TILE_SIZE_Q8_1 1280 @@ -183,7 +187,7 @@ next_nc: // (Q2_K: per-16 k sums, q8_1_s16) static inline bool htp_mm_weight_has_offset(int weight_type) { return weight_type == HTP_TYPE_Q4_1 || weight_type == HTP_TYPE_Q4_K || weight_type == HTP_TYPE_Q5_K || - weight_type == HTP_TYPE_Q2_K; + weight_type == HTP_TYPE_Q2_K || weight_type == HTP_TYPE_Q2_0; } // --- Activation/Row Size Helpers --- @@ -227,8 +231,8 @@ static inline void htp_mm_hmx_get_2d_chunk_costs( const bool is_quant = (wtype != HTP_TYPE_F16 && wtype != HTP_TYPE_F32 && wtype != HTP_TYPE_BF16); const size_t row_stride = htp_tensor_get_row_size(wtype, k); const size_t vec_dot_size = k * sizeof(uint16_t); - const uint32_t n_k_tiles = k / HTP_MM_HMX_TILE_N_COLS; - const size_t qweight_row_stride = is_quant ? (size_t)(n_k_tiles * aligned_tile_size) / 32 : 0; + const uint32_t n_weight_tiles = k / htp_tensor_get_tile_k(wtype); + const size_t qweight_row_stride = is_quant ? (size_t)(n_weight_tiles * aligned_tile_size) / 32 : 0; *size_per_n_out = (pipeline ? 2 : 1) * (is_quant ? qweight_row_stride : row_stride) + (pipeline ? 2 * vec_dot_size : vec_dot_size); @@ -359,11 +363,11 @@ static inline void htp_mm_hmx_vtcm_layout_build( const bool is_quant = (wtype != HTP_TYPE_F16 && wtype != HTP_TYPE_F32 && wtype != HTP_TYPE_BF16); const size_t row_stride = htp_tensor_get_row_size(wtype, k); const size_t vec_dot_size = k * sizeof(uint16_t); - const uint32_t n_k_tiles = k / HTP_MM_HMX_TILE_N_COLS; + const uint32_t n_weight_tiles = k / htp_tensor_get_tile_k(wtype); const size_t min_f32_size = hex_align_up(act_threads * HTP_MM_DMA_ACT_MULTIPLIER * k * sizeof(float), 128); const size_t weight_area_size = is_quant - ? hex_align_up((nc / 32) * n_k_tiles * aligned_tile_size, HTP_MM_HMX_TILE_SIZE) + ? hex_align_up((nc / 32) * n_weight_tiles * aligned_tile_size, HTP_MM_HMX_TILE_SIZE) : hex_align_up(nc * row_stride, HTP_MM_HMX_TILE_SIZE); const size_t act_area_size = hex_align_up(mc * vec_dot_size, HTP_MM_HMX_TILE_SIZE); const size_t output_area_size = hex_align_up(mc * nc * sizeof(__fp16), HTP_MM_HMX_TILE_SIZE); @@ -435,7 +439,7 @@ static inline void htp_mm_hvx_vtcm_layout_build( if (is_fused_nx) { uint32_t aligned_tile_size = htp_tensor_get_tile_padded_size(wtype); - uint32_t n_k_tiles = hex_round_up(ne10, 32) / 32; + uint32_t n_k_tiles = (ne10 + htp_tensor_get_tile_k(wtype) - 1) / htp_tensor_get_tile_k(wtype); uint32_t tile_row_size = n_k_tiles * aligned_tile_size; size_t weight_sz_per_thread = hex_round_up(n_prefetch * tile_row_size, 128); @@ -451,7 +455,7 @@ static inline void htp_mm_hvx_vtcm_layout_build( } else if (is_matmul_id) { const size_t act_row_size_tiled = htp_mm_tiled_act_row_size(wtype, ne10); const uint32_t aligned_tile_size = htp_tensor_get_tile_padded_size(wtype); - const uint32_t n_k_tiles = ne10 / 32; + const uint32_t n_k_tiles = (ne10 + htp_tensor_get_tile_k(wtype) - 1) / htp_tensor_get_tile_k(wtype); const uint32_t tile_row_size = n_k_tiles * aligned_tile_size; size_t src0_sz_per_thread = htp_mm_round_up(n_prefetch * tile_row_size, 256); @@ -489,8 +493,8 @@ static inline void htp_mm_hvx_vtcm_layout_build( case HTP_MM_KERNEL_HVX_QUANT_ROW: { size_t q_act_row_size = htp_mm_tiled_act_row_size(wtype, ne10); uint32_t aligned_tile_size = htp_tensor_get_tile_padded_size(wtype); - uint32_t n_k_tiles = (ne10 + 31) / 32; - uint32_t tile_row_size = n_k_tiles * aligned_tile_size; + uint32_t n_weight_tiles = (ne10 + htp_tensor_get_tile_k(wtype) - 1) / htp_tensor_get_tile_k(wtype); + uint32_t tile_row_size = n_weight_tiles * aligned_tile_size; size_t tiled_vtcm_size = htp_mm_round_up(n_prefetch * tile_row_size, 256); src0_sz = tiled_vtcm_size * n_threads;