diff --git a/ggml/src/ggml-cpu/CMakeLists.txt b/ggml/src/ggml-cpu/CMakeLists.txt index 50341fd6a..d347757e1 100644 --- a/ggml/src/ggml-cpu/CMakeLists.txt +++ b/ggml/src/ggml-cpu/CMakeLists.txt @@ -650,9 +650,9 @@ function(ggml_add_cpu_backend_variant_impl tag_name) # Fetch KleidiAI sources: include(FetchContent) - set(KLEIDIAI_COMMIT_TAG "v1.24.0") + set(KLEIDIAI_COMMIT_TAG "v1.31.0") set(KLEIDIAI_DOWNLOAD_URL "https://github.com/ARM-software/kleidiai/releases/download/${KLEIDIAI_COMMIT_TAG}/kleidiai-${KLEIDIAI_COMMIT_TAG}-src.tar.gz") - set(KLEIDIAI_RELEASE_ARCHIVE_MD5 "2f02ebe29573d45813e671eb304f2a00") + set(KLEIDIAI_RELEASE_ARCHIVE_MD5 "930495f3aeb16b25dc18a2bde45924f4") set(KLEIDIAI_FETCH_ARGS URL ${KLEIDIAI_DOWNLOAD_URL} diff --git a/ggml/src/ggml-cpu/kleidiai/kernels.cpp b/ggml/src/ggml-cpu/kleidiai/kernels.cpp index d4551298f..89e95968c 100644 --- a/ggml/src/ggml-cpu/kleidiai/kernels.cpp +++ b/ggml/src/ggml-cpu/kleidiai/kernels.cpp @@ -3,6 +3,8 @@ // // KleidiAI micro-kernels +#include + #include "kai/ukernels/matmul/matmul_clamp_f32_qsi8d32p_qsi4c32p/kai_matmul_clamp_f32_qsi8d32p_qsi4c32p_interface.h" #include "kai/ukernels/matmul/matmul_clamp_f32_qai8dxp_qsi8cxp/kai_matmul_clamp_f32_qai8dxp_qsi8cxp_interface.h" #include "kai/ukernels/matmul/matmul_clamp_f32_qsi8d32p_qsi4c32p/kai_matmul_clamp_f32_qsi8d32p1x8_qsi4c32p4x8_1x4x32_neon_dotprod.h" @@ -33,6 +35,10 @@ #include "kai/ukernels/matmul/pack/kai_lhs_quant_pack_qsi8d32p_f32_neon.h" #include "kai/ukernels/matmul/pack/kai_lhs_quant_pack_qai8dxp_f32.h" +#include "kai/ukernels/matmul/kai_matmul.h" +#include "kai/ukernels/matmul/kai_matmul_pack_lhs.h" +#include "kai/ukernels/matmul/kai_matmul_pack_rhs.h" + #include "kai/ukernels/matmul/pack/kai_rhs_pack_kxn_bf16p2vlx2b_f32_x32_sme.h" #include "kai/ukernels/matmul/pack/kai_rhs_pack_nxk_f32p2vlx1biasf32_f32_f32_sme.h" #include "kai/ukernels/matmul/pack/kai_rhs_pack_nxk_qsi4c32pscalef16_qsu4c32s16s0.h" @@ -42,8 +48,6 @@ #include "kai/kai_common.h" -#include "simd-mappings.h" - #define GGML_COMMON_DECL_CPP #include "ggml-common.h" @@ -154,16 +158,6 @@ static inline size_t rhs_ps_fn2(size_t n, size_t k, size_t /*nr*/, size_t /*kr*/ return Fn(n, k); } -template -static inline size_t rhs_stride_fn4(size_t k, size_t nr, size_t kr, size_t bl) { - return Fn(k, nr, kr, bl); -} - -template -static inline size_t rhs_stride_fn1(size_t k, size_t /*nr*/, size_t /*kr*/, size_t /*bl*/) { - return Fn(k); -} - template static inline void rhs_pack_fn12(size_t num_groups, size_t n, size_t k, size_t nr, size_t kr, size_t sr, size_t bl, size_t /*rhs_stride*/, const void* rhs, const void* bias, const void* /*scale*/, @@ -194,137 +188,205 @@ static inline void rhs_pack_fn13(size_t num_groups, size_t n, size_t k, size_t n Fn(num_groups, n, k, nr, kr, sr, rhs_stride, rhs, bias, scale, rhs_packed, extra_bytes, params); } -static const size_t INT4_PER_BYTE = 2; -static const size_t INT4_BITS = 4; -static const int Q4_0_ZERO_POINT = 8; -const size_t INT4_PER_UINT16 = 4; +static constexpr size_t KLEIDIAI_Q4_K_BLOCK_LENGTH = QK_K / 8; -static void dequantize_row_qsi4c32pscalef16( - const void *packed_data, - int32_t row_idx, - int64_t nc, - float *out, - size_t nr_pack, - size_t packed_row_stride, - size_t kr, - size_t bl, - size_t num_bytes_multiplier -) { - size_t group_idx = row_idx / nr_pack; - size_t row_in_group = row_idx % nr_pack; - const uint8_t *packed_group = (const uint8_t *)packed_data + group_idx * packed_row_stride; - size_t num_blocks = nc / bl; - const uint8_t *block_ptr = packed_group; +static_assert(QK_K == 256, "KleidiAI Q4_K kernels require 256-value super-blocks"); +static_assert(sizeof(block_q4_K) == 144, "Unexpected ggml Q4_K block layout"); +static_assert(offsetof(block_q4_K, scales) == 4, "Unexpected ggml Q4_K metadata offset"); +static_assert(offsetof(block_q4_K, qs) == 16, "Unexpected ggml Q4_K data offset"); - for (size_t b = 0; b < num_blocks; ++b) { - uint16_t scale_f16 = *((const uint16_t *)(block_ptr + row_in_group * num_bytes_multiplier)); - float scale = GGML_CPU_FP16_TO_FP32(scale_f16); - - const uint8_t *segment_ptr = block_ptr + nr_pack * num_bytes_multiplier; - size_t num_segments = bl / kr; - size_t num_bytes_per_segment = kr / INT4_PER_BYTE; - - for (size_t s = 0; s < num_segments; ++s) { - const uint8_t *seg_base = segment_ptr + s * nr_pack * num_bytes_per_segment; - const uint8_t *qbytes = seg_base + row_in_group * num_bytes_per_segment; - for (size_t k = 0; k < num_bytes_per_segment; ++k) { - uint8_t byte = qbytes[k] ^ 0x88; - int x0 = (byte & 0x0F) - Q4_0_ZERO_POINT; - int x1 = (byte >> INT4_BITS) - Q4_0_ZERO_POINT; - out[b * bl + s * num_bytes_per_segment + k] = x0 * scale; - out[b * bl + s * num_bytes_per_segment + k + bl/2] = x1 * scale; - } - } - block_ptr += nr_pack * num_bytes_multiplier + num_segments * nr_pack * num_bytes_per_segment; - } +static struct kai_matmul_uker_config q4_k_matmul_config() { + struct kai_matmul_uker_config config = {}; + config.format.bl = KLEIDIAI_Q4_K_BLOCK_LENGTH; + return config; } -static void dequantize_row_qsi4c32ps1s0scalef16( - const void *packed_data, - int32_t row_idx, - int64_t k, - float *out, - size_t nr, - size_t packed_row_stride, - size_t kr, - size_t bl, - size_t num_bytes_multiplier -) { - const size_t num_blocks = k / bl; - const size_t bl4 = bl / INT4_PER_UINT16; - - size_t group_idx = row_idx / nr; - size_t row_in_group = row_idx % nr; - - const uint8_t *packed_group = (const uint8_t *)packed_data + group_idx * packed_row_stride; - const uint16_t *qdata = (const uint16_t *)packed_group; - const uint16_t *scales = (const uint16_t *)(packed_group + packed_row_stride - (nr * num_blocks * num_bytes_multiplier)); - - for (size_t block_idx = 0; block_idx < num_blocks; ++block_idx) { - uint16_t scale_f16 = scales[row_in_group + block_idx * nr]; - float scale = GGML_CPU_FP16_TO_FP32(scale_f16); - - for (size_t bl4_idx = 0; bl4_idx < bl4; ++bl4_idx) { - uint16_t q = qdata[(block_idx * bl4 + bl4_idx) * nr + row_in_group]; - - for (size_t qidx = 0; qidx < INT4_PER_UINT16; ++qidx) { - int v = ((q >> (qidx * 4)) & 0xF) - Q4_0_ZERO_POINT; - out[block_idx * bl + bl4_idx * INT4_BITS + qidx] = v * scale; - } - } - } - GGML_UNUSED(kr); +template +static size_t q4_k_get_m_step() { + const struct kai_matmul_uker_api api = GetApi(); + const struct kai_matmul_uker_config config = q4_k_matmul_config(); + return api.get_step(&config).m; } -static void dequantize_row_qsi8cxp( - const void *packed_data, - int32_t row_idx, - int64_t k, - float *out, - size_t nr, - size_t packed_row_stride, - size_t kr, - size_t bl, - size_t num_bytes_multiplier -) { - GGML_UNUSED(bl); - GGML_UNUSED(num_bytes_multiplier); +template +static size_t q4_k_get_n_step() { + const struct kai_matmul_uker_api api = GetApi(); + const struct kai_matmul_uker_config config = q4_k_matmul_config(); + return api.get_step(&config).n; +} - const size_t k_internal = ((size_t) k + QK8_0 - 1) / QK8_0 * QK8_0; - const size_t group_idx = row_idx / nr; - const size_t row_in_group = row_idx % nr; +template +static size_t q4_k_get_constant() { + return Value; +} - const uint8_t * group_ptr = static_cast(packed_data) + group_idx * packed_row_stride; - const int8_t * data_base = reinterpret_cast(group_ptr); +template +static size_t q4_k_get_dst_offset(size_t m_idx, size_t n_idx, size_t dst_stride) { + const struct kai_matmul_uker_api api = GetApi(); + const struct kai_matmul_uker_config config = q4_k_matmul_config(); + const struct kai_matmul_uker_dst_dim_args index = { m_idx, n_idx }; + const struct kai_matmul_uker_dst_stride_args stride = { dst_stride }; + return api.get_dst_offset(&config, &index, &stride); +} - const size_t num_blocks = k_internal / kr; +template +static size_t q4_k_get_lhs_packed_offset(size_t m_idx, size_t k, size_t bl) { + GGML_ASSERT(bl == KLEIDIAI_Q4_K_BLOCK_LENGTH); - for (size_t block = 0; block < num_blocks; ++block) { - const int8_t * block_ptr = data_base + (block * nr + row_in_group) * kr; - for (size_t i = 0; i < kr; ++i) { - const size_t k_idx = block * kr + i; - if (k_idx < (size_t) k) { - out[k_idx] = static_cast(block_ptr[i]); - } - } - } - - const uint8_t * sums_ptr = group_ptr + nr * k_internal; - GGML_UNUSED(sums_ptr); - - const float * scale_ptr = reinterpret_cast(sums_ptr + nr * sizeof(int32_t)); - const float scale = scale_ptr[row_in_group]; - - if (scale == 0.0f) { - for (size_t i = 0; i < (size_t) k; ++i) { - out[i] = 0.0f; - } - return; - } - - for (size_t i = 0; i < (size_t) k; ++i) { - out[i] *= scale; + const struct kai_matmul_uker_api api = GetApi(); + const struct kai_matmul_uker_config config = q4_k_matmul_config(); + const struct kai_matmul_uker_lhs_dim_args shape = { m_idx, k }; + const struct kai_matmul_uker_lhs_dim_args index = { m_idx, 0 }; + const struct kai_matmul_uker_lhs_stride_args stride = api.get_lhs_stride(&config, &shape); + return api.get_lhs_offset(&config, &index, &stride); +} + +template +static size_t q4_k_get_rhs_packed_offset(size_t n_idx, size_t k, size_t bl) { + GGML_ASSERT(bl == KLEIDIAI_Q4_K_BLOCK_LENGTH); + + const struct kai_matmul_uker_api api = GetApi(); + const struct kai_matmul_uker_config config = q4_k_matmul_config(); + const struct kai_matmul_uker_rhs_dim_args shape = { n_idx, k }; + const struct kai_matmul_uker_rhs_dim_args index = { n_idx, 0 }; + const struct kai_matmul_uker_rhs_stride_args stride = api.get_rhs_stride(&config, &shape); + return api.get_rhs_offset(&config, &index, &stride); +} + +template +static void q4_k_run_matmul( + size_t m, size_t n, size_t k, size_t bl, + const void * lhs_packed, const void * rhs_packed, + void * dst, size_t dst_stride_row, size_t dst_stride_col, + float clamp_min, float clamp_max) { + GGML_ASSERT(bl == KLEIDIAI_Q4_K_BLOCK_LENGTH); + GGML_ASSERT(dst_stride_col == sizeof(float)); + + const struct kai_matmul_uker_api api = GetApi(); + const struct kai_matmul_uker_config config = q4_k_matmul_config(); + const struct kai_matmul_uker_lhs_dim_args lhs_shape = { m, k }; + const struct kai_matmul_uker_rhs_dim_args rhs_shape = { n, k }; + + struct kai_matmul_uker_args args = {}; + args.flags = KAI_MATMUL_UKER_FLAGS_ARGS_CLAMP; + args.shape = { m, n, k }; + args.operand.lhs.ptr = lhs_packed; + args.operand.lhs.stride = api.get_lhs_stride(&config, &lhs_shape); + args.operand.rhs.ptr = rhs_packed; + args.operand.rhs.stride = api.get_rhs_stride(&config, &rhs_shape); + args.operand.dst.ptr = dst; + args.operand.dst.stride.m = dst_stride_row; + args.activation.clamp.min_ptr = &clamp_min; + args.activation.clamp.max_ptr = &clamp_max; + api.run(&config, &args); +} + +static struct kai_matmul_pack_lhs_uker_config q4_k_lhs_pack_config() { + struct kai_matmul_pack_lhs_uker_config config = {}; + return config; +} + +static size_t q4_k_lhs_get_offset(size_t m_idx, size_t lhs_stride) { + const struct kai_matmul_pack_lhs_uker_api api = kai_matmul_pack_lhs_mxk_qsi8d32p1x4sf16_f32_neon(); + const struct kai_matmul_pack_lhs_uker_config config = q4_k_lhs_pack_config(); + const struct kai_matmul_pack_lhs_uker_lhs_dim_args index = { m_idx, 0 }; + const struct kai_matmul_pack_lhs_uker_lhs_stride_args stride = { lhs_stride }; + return api.get_lhs_offset(&config, &index, &stride); +} + +static size_t q4_k_lhs_get_packed_offset( + size_t m_idx, size_t k, size_t bl, size_t mr, size_t kr, size_t sr) { + GGML_ASSERT(bl == KLEIDIAI_Q4_K_BLOCK_LENGTH); + GGML_ASSERT(mr == 1 && kr == 4 && sr == 2); + + const struct kai_matmul_pack_lhs_uker_api api = kai_matmul_pack_lhs_mxk_qsi8d32p1x4sf16_f32_neon(); + const struct kai_matmul_pack_lhs_uker_config config = q4_k_lhs_pack_config(); + const struct kai_matmul_pack_lhs_uker_lhs_packed_dim_args shape = { m_idx, k }; + const struct kai_matmul_pack_lhs_uker_lhs_packed_dim_args index = { m_idx, 0 }; + const struct kai_matmul_pack_lhs_uker_lhs_packed_stride_args stride = api.get_lhs_packed_stride(&config, &shape); + return api.get_lhs_packed_offset(&config, &index, &stride); +} + +static size_t q4_k_lhs_get_packed_size( + size_t m, size_t k, size_t bl, size_t mr, size_t kr, size_t sr) { + GGML_ASSERT(bl == KLEIDIAI_Q4_K_BLOCK_LENGTH); + GGML_ASSERT(mr == 1 && kr == 4 && sr == 2); + + const struct kai_matmul_pack_lhs_uker_api api = kai_matmul_pack_lhs_mxk_qsi8d32p1x4sf16_f32_neon(); + const struct kai_matmul_pack_lhs_uker_config config = q4_k_lhs_pack_config(); + const struct kai_matmul_pack_lhs_uker_lhs_packed_dim_args shape = { m, k }; + const struct kai_matmul_pack_lhs_uker_lhs_packed_stride_args stride = api.get_lhs_packed_stride(&config, &shape); + return api.get_lhs_packed_size(&config, &shape, &stride); +} + +static void q4_k_lhs_pack( + size_t m, size_t k, size_t bl, size_t mr, size_t kr, size_t sr, + size_t m_idx_start, const void * lhs, size_t lhs_stride, void * lhs_packed) { + GGML_ASSERT(bl == KLEIDIAI_Q4_K_BLOCK_LENGTH); + GGML_ASSERT(mr == 1 && kr == 4 && sr == 2); + GGML_ASSERT(m_idx_start == 0); + + const struct kai_matmul_pack_lhs_uker_api api = kai_matmul_pack_lhs_mxk_qsi8d32p1x4sf16_f32_neon(); + const struct kai_matmul_pack_lhs_uker_config config = q4_k_lhs_pack_config(); + const struct kai_matmul_pack_lhs_uker_lhs_packed_dim_args packed_shape = { m, k }; + + struct kai_matmul_pack_lhs_uker_args args = {}; + args.flags = 0; + args.shape = { m, k }; + args.operand.lhs.ptr = lhs; + args.operand.lhs.stride.m = lhs_stride; + args.operand.lhs_packed.ptr = lhs_packed; + args.operand.lhs_packed.stride = api.get_lhs_packed_stride(&config, &packed_shape); + api.run(&config, &args); +} + +static struct kai_matmul_pack_rhs_uker_config q4_k_rhs_pack_config() { + struct kai_matmul_pack_rhs_uker_config config = {}; + return config; +} + +static size_t q4_k_rhs_get_packed_size(size_t n, size_t k, size_t nr, size_t /*kr*/, size_t bl) { + GGML_ASSERT(bl == KLEIDIAI_Q4_K_BLOCK_LENGTH); + + const struct kai_matmul_pack_rhs_uker_api api = + kai_matmul_pack_rhs_nxk_qai4c32p16vsx4s1s0sf16_qai4c32k256sf16s32s0_sme(); + const struct kai_matmul_pack_rhs_uker_config config = q4_k_rhs_pack_config(); + GGML_ASSERT(api.get_step(&config).n == nr); + const struct kai_matmul_pack_rhs_uker_rhs_packed_dim_args shape = { n, k }; + const struct kai_matmul_pack_rhs_uker_rhs_packed_stride_args stride = api.get_rhs_packed_stride(&config, &shape); + return api.get_rhs_packed_size(&config, &shape, &stride); +} + +static void q4_k_rhs_pack( + size_t num_groups, size_t n, size_t k, size_t nr, size_t /*kr*/, size_t /*sr*/, size_t bl, + size_t rhs_stride, const void * rhs, const void * bias, const void * scale, + void * rhs_packed, size_t extra_bytes, const void * params) { + GGML_ASSERT(num_groups == 1); + GGML_ASSERT(bl == KLEIDIAI_Q4_K_BLOCK_LENGTH); + GGML_ASSERT(bias == nullptr); + GGML_ASSERT(scale == nullptr); + GGML_ASSERT(extra_bytes == 0); + GGML_ASSERT(params == nullptr); + + const struct kai_matmul_pack_rhs_uker_api api = + kai_matmul_pack_rhs_nxk_qai4c32p16vsx4s1s0sf16_qai4c32k256sf16s32s0_sme(); + const struct kai_matmul_pack_rhs_uker_config config = q4_k_rhs_pack_config(); + GGML_ASSERT(api.get_step(&config).n == nr); + const struct kai_matmul_pack_rhs_uker_rhs_dim_args rhs_shape = { n, k }; + const struct kai_matmul_pack_rhs_uker_rhs_packed_dim_args packed_shape = { n, k }; + + struct kai_matmul_pack_rhs_uker_args args = {}; + args.flags = 0; + args.shape = { n, k }; + args.operand.rhs.ptr = rhs; + args.operand.rhs.stride = api.get_rhs_stride(&config, &rhs_shape); + if (rhs_stride != 0) { + GGML_ASSERT(rhs_stride >= args.operand.rhs.stride.n); + args.operand.rhs.stride.n = rhs_stride; } + args.operand.rhs_packed.ptr = rhs_packed; + args.operand.rhs_packed.stride = api.get_rhs_packed_stride(&config, &packed_shape); + api.run(&config, &args); } static ggml_kleidiai_kernels gemm_gemv_kernels[] = { @@ -338,7 +400,6 @@ static ggml_kleidiai_kernels gemm_gemv_kernels[] = { /* .get_kr = */ kai_get_kr_matmul_clamp_f32_f16p1vlx2_qsi4c32p4vlx2_1vlx4vl_sme2_mopa, /* .get_sr = */ kai_get_sr_matmul_clamp_f32_f16p1vlx2_qsi4c32p4vlx2_1vlx4vl_sme2_mopa, /* .get_dst_offset = */ kai_get_dst_offset_matmul_clamp_f32_f16p1vlx2_qsi4c32p4vlx2_1vlx4vl_sme2_mopa, - /* .get_dst_size = */ kai_get_dst_size_matmul_clamp_f32_f16p1vlx2_qsi4c32p4vlx2_1vlx4vl_sme2_mopa, /* .get_lhs_offset_ex = */ &kernel_offs_fn3, /* .get_rhs_packed_offset_ex = */ &kernel_offs_fn3, /* .run_kernel_ex = */ &kernel_run_fn11, @@ -359,7 +420,6 @@ static ggml_kleidiai_kernels gemm_gemv_kernels[] = { /* .get_kr = */ kai_get_kr_matmul_clamp_f32_qsi8d32p1x4_qsi4c32p4vlx4_1x4vl_sme2_sdot, /* .get_sr = */ kai_get_sr_matmul_clamp_f32_qsi8d32p1x4_qsi4c32p4vlx4_1x4vl_sme2_sdot, /* .get_dst_offset = */ kai_get_dst_offset_matmul_clamp_f32_qsi8d32p1x4_qsi4c32p4vlx4_1x4vl_sme2_sdot, - /* .get_dst_size = */ kai_get_dst_size_matmul_clamp_f32_qsi8d32p1x4_qsi4c32p4vlx4_1x4vl_sme2_sdot, /* .get_lhs_offset_ex = */ &kernel_offs_fn3, /* .get_rhs_packed_offset_ex = */ &kernel_offs_fn3, /* .run_kernel_ex = */ &kernel_run_fn11, @@ -371,10 +431,7 @@ static ggml_kleidiai_kernels gemm_gemv_kernels[] = { /* .pack_func_ex = */ &lhs_pack_float_fn10, }, /* .rhs_info = */ { - /* .packed_stride = */ kai_get_rhs_packed_stride_rhs_pack_nxk_qsi4c32ps1s0scalef16_qsu4c32s16s0_neon, - /* .to_float = */ dequantize_row_qsi4c32ps1s0scalef16, /* .packed_size_ex = */ &rhs_ps_fn5, - /* .packed_stride_ex = */ &rhs_stride_fn4, /* .pack_func_ex = */ &rhs_pack_fn12, }, /* .required_cpu = */ CPU_FEATURE_SME2 | CPU_FEATURE_FP16, @@ -392,7 +449,6 @@ static ggml_kleidiai_kernels gemm_gemv_kernels[] = { /* .get_kr = */ kai_get_kr_matmul_clamp_f32_bf16p2vlx2_bf16p2vlx2_2vlx2vl_sme2_mopa, /* .get_sr = */ kai_get_sr_matmul_clamp_f32_bf16p2vlx2_bf16p2vlx2_2vlx2vl_sme2_mopa, /* .get_dst_offset = */ kai_get_dst_offset_matmul_clamp_f32_bf16p2vlx2_bf16p2vlx2_2vlx2vl_sme2_mopa, - /* .get_dst_size = */ kai_get_dst_size_matmul_clamp_f32_bf16p2vlx2_bf16p2vlx2_2vlx2vl_sme2_mopa, /* .get_lhs_offset_ex = */ &kernel_offs_fn2, /* .get_rhs_packed_offset_ex = */ &kernel_offs_fn2, /* .run_kernel_ex = */ &kernel_run_fn10, @@ -412,7 +468,6 @@ static ggml_kleidiai_kernels gemm_gemv_kernels[] = { /* .get_kr = */ kai_get_kr_matmul_clamp_f32_bf16p2vlx2_bf16p2vlx2_2vlx2vl_sme2_mopa, /* .get_sr = */ kai_get_sr_matmul_clamp_f32_bf16p2vlx2_bf16p2vlx2_2vlx2vl_sme2_mopa, /* .get_dst_offset = */ kai_get_dst_offset_matmul_clamp_f32_bf16p2vlx2_bf16p2vlx2_2vlx2vl_sme2_mopa, - /* .get_dst_size = */ kai_get_dst_size_matmul_clamp_f32_bf16p2vlx2_bf16p2vlx2_2vlx2vl_sme2_mopa, /* .get_lhs_offset_ex = */ nullptr, /* .get_rhs_packed_offset_ex = */ nullptr, /* .run_kernel_ex = */ nullptr, @@ -424,10 +479,7 @@ static ggml_kleidiai_kernels gemm_gemv_kernels[] = { /* .pack_func_ex = */ &lhs_pack_void_fn9, }, /* .rhs_info = */ { - /* .packed_stride = */ nullptr, - /* .to_float = */ nullptr, /* .packed_size_ex = */ &rhs_ps_fn2, - /* .packed_stride_ex = */ &rhs_stride_fn1, /* .pack_func_ex = */ &rhs_pack_fn13, }, /* .required_cpu = */ CPU_FEATURE_SME2, @@ -446,7 +498,6 @@ static ggml_kleidiai_kernels gemm_gemv_kernels[] = { /* .get_kr = */ kai_get_kr_matmul_clamp_f32_qsi8d32p4x4_qsi4c32p4x4_16x4_neon_dotprod, /* .get_sr = */ kai_get_sr_matmul_clamp_f32_qsi8d32p4x4_qsi4c32p4x4_16x4_neon_dotprod, /* .get_dst_offset = */ kai_get_dst_offset_matmul_clamp_f32_qsi8d32p4x4_qsi4c32p4x4_16x4_neon_dotprod, - /* .get_dst_size = */ kai_get_dst_size_matmul_clamp_f32_qsi8d32p4x4_qsi4c32p4x4_16x4_neon_dotprod, /* .get_lhs_offset_ex = */ &kernel_offs_fn3, /* .get_rhs_packed_offset_ex = */ &kernel_offs_fn3, /* .run_kernel_ex = */ &kernel_run_fn11, @@ -466,7 +517,6 @@ static ggml_kleidiai_kernels gemm_gemv_kernels[] = { /* .get_kr = */ kai_get_kr_matmul_clamp_f32_qsi8d32p1x4_qsi4c32p4x4_1x4_neon_dotprod, /* .get_sr = */ kai_get_sr_matmul_clamp_f32_qsi8d32p1x4_qsi4c32p4x4_1x4_neon_dotprod, /* .get_dst_offset = */ kai_get_dst_offset_matmul_clamp_f32_qsi8d32p1x4_qsi4c32p4x4_1x4_neon_dotprod, - /* .get_dst_size = */ kai_get_dst_size_matmul_clamp_f32_qsi8d32p1x4_qsi4c32p4x4_1x4_neon_dotprod, /* .get_lhs_offset_ex = */ &kernel_offs_fn3, /* .get_rhs_packed_offset_ex = */ &kernel_offs_fn3, /* .run_kernel_ex = */ &kernel_run_fn11, @@ -478,10 +528,7 @@ static ggml_kleidiai_kernels gemm_gemv_kernels[] = { /* .pack_func_ex = */ &lhs_pack_float_fn10, }, /* .rhs_info = */ { - /* .packed_stride = */ kai_get_rhs_packed_stride_rhs_pack_nxk_qsi4c32pscalef16_qsu4c32s16s0, - /* .to_float = */ dequantize_row_qsi4c32pscalef16, /* .packed_size_ex = */ &rhs_ps_fn5, - /* .packed_stride_ex = */ &rhs_stride_fn4, /* .pack_func_ex = */ &rhs_pack_fn12, }, /* .required_cpu = */ CPU_FEATURE_DOTPROD, @@ -499,7 +546,6 @@ static ggml_kleidiai_kernels gemm_gemv_kernels[] = { /* .get_kr = */ kai_get_kr_matmul_clamp_f32_qsi8d32p4x8_qsi4c32p4x8_16x4_neon_i8mm, /* .get_sr = */ kai_get_sr_matmul_clamp_f32_qsi8d32p4x8_qsi4c32p4x8_16x4_neon_i8mm, /* .get_dst_offset = */ kai_get_dst_offset_matmul_clamp_f32_qsi8d32p4x8_qsi4c32p4x8_16x4_neon_i8mm, - /* .get_dst_size = */ kai_get_dst_size_matmul_clamp_f32_qsi8d32p4x8_qsi4c32p4x8_16x4_neon_i8mm, /* .get_lhs_offset_ex = */ &kernel_offs_fn3, /* .get_rhs_packed_offset_ex = */ &kernel_offs_fn3, /* .run_kernel_ex = */ &kernel_run_fn11, @@ -519,7 +565,6 @@ static ggml_kleidiai_kernels gemm_gemv_kernels[] = { /* .get_kr = */ kai_get_kr_matmul_clamp_f32_qsi8d32p1x8_qsi4c32p4x8_1x4x32_neon_dotprod, /* .get_sr = */ kai_get_sr_matmul_clamp_f32_qsi8d32p1x8_qsi4c32p4x8_1x4x32_neon_dotprod, /* .get_dst_offset = */ kai_get_dst_offset_matmul_clamp_f32_qsi8d32p1x8_qsi4c32p4x8_1x4x32_neon_dotprod, - /* .get_dst_size = */ kai_get_dst_size_matmul_clamp_f32_qsi8d32p1x8_qsi4c32p4x8_1x4x32_neon_dotprod, /* .get_lhs_offset_ex = */ &kernel_offs_fn3, /* .get_rhs_packed_offset_ex = */ &kernel_offs_fn3, /* .run_kernel_ex = */ &kernel_run_fn11, @@ -531,10 +576,7 @@ static ggml_kleidiai_kernels gemm_gemv_kernels[] = { /* .pack_func_ex = */ &lhs_pack_float_fn10, }, /* .rhs_info = */ { - /* .packed_stride = */ kai_get_rhs_packed_stride_rhs_pack_nxk_qsi4c32pscalef16_qsu4c32s16s0, - /* .to_float = */ dequantize_row_qsi4c32pscalef16, /* .packed_size_ex = */ &rhs_ps_fn5, - /* .packed_stride_ex = */ &rhs_stride_fn4, /* .pack_func_ex = */ &rhs_pack_fn12, }, /* .required_cpu = */ CPU_FEATURE_I8MM | CPU_FEATURE_DOTPROD, @@ -553,7 +595,6 @@ static ggml_kleidiai_kernels gemm_gemv_kernels[] = { /* .get_kr = */ kai_get_kr_matmul_clamp_f32_qsi8d32p4x8_qsi4c32p8x8_16x8_sve_i8mm, /* .get_sr = */ kai_get_sr_matmul_clamp_f32_qsi8d32p4x8_qsi4c32p8x8_16x8_sve_i8mm, /* .get_dst_offset = */ kai_get_dst_offset_matmul_clamp_f32_qsi8d32p4x8_qsi4c32p8x8_16x8_sve_i8mm, - /* .get_dst_size = */ kai_get_dst_size_matmul_clamp_f32_qsi8d32p4x8_qsi4c32p8x8_16x8_sve_i8mm, /* .get_lhs_offset_ex = */ &kernel_offs_fn3, /* .get_rhs_packed_offset_ex = */ &kernel_offs_fn3, /* .run_kernel_ex = */ &kernel_run_fn11, @@ -573,7 +614,6 @@ static ggml_kleidiai_kernels gemm_gemv_kernels[] = { /* .get_kr = */ kai_get_kr_matmul_clamp_f32_qsi8d32p1x8_qsi4c32p8x8_1x8_sve_dotprod, /* .get_sr = */ kai_get_sr_matmul_clamp_f32_qsi8d32p1x8_qsi4c32p8x8_1x8_sve_dotprod, /* .get_dst_offset = */ kai_get_dst_offset_matmul_clamp_f32_qsi8d32p1x8_qsi4c32p8x8_1x8_sve_dotprod, - /* .get_dst_size = */ kai_get_dst_size_matmul_clamp_f32_qsi8d32p1x8_qsi4c32p8x8_1x8_sve_dotprod, /* .get_lhs_offset_ex = */ &kernel_offs_fn3, /* .get_rhs_packed_offset_ex = */ &kernel_offs_fn3, /* .run_kernel_ex = */ &kernel_run_fn11, @@ -585,10 +625,7 @@ static ggml_kleidiai_kernels gemm_gemv_kernels[] = { /* .pack_func_ex = */ &lhs_pack_float_fn10, }, /* .rhs_info = */ { - /* .packed_stride = */ kai_get_rhs_packed_stride_rhs_pack_nxk_qsi4c32pscalef16_qsu4c32s16s0, - /* .to_float = */ dequantize_row_qsi4c32pscalef16, /* .packed_size_ex = */ &rhs_ps_fn5, - /* .packed_stride_ex = */ &rhs_stride_fn4, /* .pack_func_ex = */ &rhs_pack_fn12, }, /* .required_cpu = */ CPU_FEATURE_SVE | CPU_FEATURE_I8MM | CPU_FEATURE_DOTPROD, @@ -606,7 +643,6 @@ static ggml_kleidiai_kernels gemm_gemv_kernels[] = { /* .get_kr = */ kai_get_kr_matmul_clamp_f32_qsi8d32p4x8_qsi4c32p4x8_16x4_neon_i8mm, /* .get_sr = */ kai_get_sr_matmul_clamp_f32_qsi8d32p4x8_qsi4c32p4x8_16x4_neon_i8mm, /* .get_dst_offset = */ kai_get_dst_offset_matmul_clamp_f32_qsi8d32p4x8_qsi4c32p4x8_16x4_neon_i8mm, - /* .get_dst_size = */ kai_get_dst_size_matmul_clamp_f32_qsi8d32p4x8_qsi4c32p4x8_16x4_neon_i8mm, /* .get_lhs_offset_ex = */ &kernel_offs_fn3, /* .get_rhs_packed_offset_ex = */ &kernel_offs_fn3, /* .run_kernel_ex = */ &kernel_run_fn11, @@ -626,7 +662,6 @@ static ggml_kleidiai_kernels gemm_gemv_kernels[] = { /* .get_kr = */ kai_get_kr_matmul_clamp_f32_qsi8d32p1x8_qsi4c32p4x8_1x4x32_neon_dotprod, /* .get_sr = */ kai_get_sr_matmul_clamp_f32_qsi8d32p1x8_qsi4c32p4x8_1x4x32_neon_dotprod, /* .get_dst_offset = */ kai_get_dst_offset_matmul_clamp_f32_qsi8d32p1x8_qsi4c32p4x8_1x4x32_neon_dotprod, - /* .get_dst_size = */ kai_get_dst_size_matmul_clamp_f32_qsi8d32p1x8_qsi4c32p4x8_1x4x32_neon_dotprod, /* .get_lhs_offset_ex = */ &kernel_offs_fn3, /* .get_rhs_packed_offset_ex = */ &kernel_offs_fn3, /* .run_kernel_ex = */ &kernel_run_fn11, @@ -638,10 +673,7 @@ static ggml_kleidiai_kernels gemm_gemv_kernels[] = { /* .pack_func_ex = */ &lhs_pack_float_fn10, }, /* .rhs_info = */ { - /* .packed_stride = */ kai_get_rhs_packed_stride_rhs_pack_nxk_qsi4c32pscalef16_qsu4c32s16s0, - /* .to_float = */ dequantize_row_qsi4c32pscalef16, /* .packed_size_ex = */ &rhs_ps_fn5, - /* .packed_stride_ex = */ &rhs_stride_fn4, /* .pack_func_ex = */ &rhs_pack_fn12, }, /* .required_cpu = */ CPU_FEATURE_I8MM | CPU_FEATURE_DOTPROD, @@ -659,7 +691,6 @@ static ggml_kleidiai_kernels gemm_gemv_kernels[] = { /* .get_kr = */ kai_get_kr_matmul_clamp_f32_qsi8d32p4x4_qsi4c32p4x4_16x4_neon_dotprod, /* .get_sr = */ kai_get_sr_matmul_clamp_f32_qsi8d32p4x4_qsi4c32p4x4_16x4_neon_dotprod, /* .get_dst_offset = */ kai_get_dst_offset_matmul_clamp_f32_qsi8d32p4x4_qsi4c32p4x4_16x4_neon_dotprod, - /* .get_dst_size = */ kai_get_dst_size_matmul_clamp_f32_qsi8d32p4x4_qsi4c32p4x4_16x4_neon_dotprod, /* .get_lhs_offset_ex = */ &kernel_offs_fn3, /* .get_rhs_packed_offset_ex = */ &kernel_offs_fn3, /* .run_kernel_ex = */ &kernel_run_fn11, @@ -679,7 +710,6 @@ static ggml_kleidiai_kernels gemm_gemv_kernels[] = { /* .get_kr = */ kai_get_kr_matmul_clamp_f32_qsi8d32p1x4_qsi4c32p4x4_1x4_neon_dotprod, /* .get_sr = */ kai_get_sr_matmul_clamp_f32_qsi8d32p1x4_qsi4c32p4x4_1x4_neon_dotprod, /* .get_dst_offset = */ kai_get_dst_offset_matmul_clamp_f32_qsi8d32p1x4_qsi4c32p4x4_1x4_neon_dotprod, - /* .get_dst_size = */ kai_get_dst_size_matmul_clamp_f32_qsi8d32p1x4_qsi4c32p4x4_1x4_neon_dotprod, /* .get_lhs_offset_ex = */ &kernel_offs_fn3, /* .get_rhs_packed_offset_ex = */ &kernel_offs_fn3, /* .run_kernel_ex = */ &kernel_run_fn11, @@ -691,10 +721,7 @@ static ggml_kleidiai_kernels gemm_gemv_kernels[] = { /* .pack_func_ex = */ &lhs_pack_float_fn10, }, /* .rhs_info = */ { - /* .packed_stride = */ kai_get_rhs_packed_stride_rhs_pack_nxk_qsi4c32pscalef16_qsu4c32s16s0, - /* .to_float = */ dequantize_row_qsi4c32pscalef16, /* .packed_size_ex = */ &rhs_ps_fn5, - /* .packed_stride_ex = */ &rhs_stride_fn4, /* .pack_func_ex = */ &rhs_pack_fn12, }, /* .required_cpu = */ CPU_FEATURE_DOTPROD, @@ -706,6 +733,59 @@ static ggml_kleidiai_kernels gemm_gemv_kernels[] = { { /* Sentinel */ } }; +static ggml_kleidiai_kernels gemm_gemv_kernels_q4_k[] = { + { + /* SME2 GEMM */ + { + /* .get_m_step = */ &q4_k_get_m_step, + /* .get_n_step = */ &q4_k_get_n_step, + /* .get_mr = */ &q4_k_get_m_step, + /* .get_nr = */ &q4_k_get_n_step, + /* .get_kr = */ &q4_k_get_constant<4>, + /* .get_sr = */ &q4_k_get_constant<2>, + /* .get_dst_offset = */ &q4_k_get_dst_offset, + /* .get_lhs_offset_ex = */ &q4_k_get_lhs_packed_offset, + /* .get_rhs_packed_offset_ex = */ &q4_k_get_rhs_packed_offset, + /* .run_kernel_ex = */ &q4_k_run_matmul, + }, + { + /* .get_offset = */ kai_get_lhs_offset_lhs_pack_f16pmrx2_f32_neon, + /* .get_packed_offset_ex = */ &lhs_offs_fn6, + /* .packed_size_ex = */ &lhs_ps_fn6, + /* .pack_func_ex = */ &lhs_pack_void_fn10, + }, + /* SME2 GEMV */ + { + /* .get_m_step = */ &q4_k_get_m_step, + /* .get_n_step = */ &q4_k_get_n_step, + /* .get_mr = */ &q4_k_get_m_step, + /* .get_nr = */ &q4_k_get_n_step, + /* .get_kr = */ &q4_k_get_constant<4>, + /* .get_sr = */ &q4_k_get_constant<2>, + /* .get_dst_offset = */ &q4_k_get_dst_offset, + /* .get_lhs_offset_ex = */ &q4_k_get_lhs_packed_offset, + /* .get_rhs_packed_offset_ex = */ &q4_k_get_rhs_packed_offset, + /* .run_kernel_ex = */ &q4_k_run_matmul, + }, + { + /* .get_offset = */ q4_k_lhs_get_offset, + /* .get_packed_offset_ex = */ q4_k_lhs_get_packed_offset, + /* .packed_size_ex = */ q4_k_lhs_get_packed_size, + /* .pack_func_ex = */ q4_k_lhs_pack, + }, + { + /* .packed_size_ex = */ q4_k_rhs_get_packed_size, + /* .pack_func_ex = */ q4_k_rhs_pack, + /* .repack_mode = */ RHS_REPACK_SINGLE_ONLY, + }, + /* .required_cpu = */ CPU_FEATURE_SME2 | CPU_FEATURE_FP16, + /* .lhs_type = */ GGML_TYPE_F32, + /* .rhs_type = */ GGML_TYPE_Q4_K, + /* .op_type = */ GGML_TYPE_F32, + }, + { /* Sentinel */ } +}; + static ggml_kleidiai_kernels gemm_gemv_kernels_q8[] = { { /* SME2 GEMM */ @@ -717,7 +797,6 @@ static ggml_kleidiai_kernels gemm_gemv_kernels_q8[] = { /* .get_kr = */ kai_get_kr_matmul_clamp_f32_qai8dxp1vlx4_qsi8cxp4vlx4_1vlx4vl_sme2_mopa, /* .get_sr = */ kai_get_sr_matmul_clamp_f32_qai8dxp1vlx4_qsi8cxp4vlx4_1vlx4vl_sme2_mopa, /* .get_dst_offset = */ kai_get_dst_offset_matmul_clamp_f32_qai8dxp1vlx4_qsi8cxp4vlx4_1vlx4vl_sme2_mopa, - /* .get_dst_size = */ kai_get_dst_size_matmul_clamp_f32_qai8dxp1vlx4_qsi8cxp4vlx4_1vlx4vl_sme2_mopa, /* .get_lhs_offset_ex = */ &kernel_offs_fn2, /* .get_rhs_packed_offset_ex = */ &kernel_offs_fn2, /* .run_kernel_ex = */ &kernel_run_float_fn10, @@ -737,7 +816,6 @@ static ggml_kleidiai_kernels gemm_gemv_kernels_q8[] = { /* .get_kr = */ kai_get_kr_matmul_clamp_f32_qai8dxp1x4_qsi8cxp4vlx4_1x4vl_sme2_dot, /* .get_sr = */ kai_get_sr_matmul_clamp_f32_qai8dxp1x4_qsi8cxp4vlx4_1x4vl_sme2_dot, /* .get_dst_offset = */ kai_get_dst_offset_matmul_clamp_f32_qai8dxp1x4_qsi8cxp4vlx4_1x4vl_sme2_dot, - /* .get_dst_size = */ kai_get_dst_size_matmul_clamp_f32_qai8dxp1x4_qsi8cxp4vlx4_1x4vl_sme2_dot, /* .get_lhs_offset_ex = */ &kernel_offs_fn2, /* .get_rhs_packed_offset_ex = */ &kernel_offs_fn2, /* .run_kernel_ex = */ &kernel_run_float_fn10, @@ -749,10 +827,7 @@ static ggml_kleidiai_kernels gemm_gemv_kernels_q8[] = { /* .pack_func_ex = */ &lhs_pack_float_fn9_no_bl, }, /* .rhs_info = */ { - /* .packed_stride = */ kai_get_rhs_packed_stride_rhs_pack_nxk_qsi8cxp_qsi8cx_neon, - /* .to_float = */ dequantize_row_qsi8cxp, /* .packed_size_ex = */ &rhs_ps_fn5, - /* .packed_stride_ex = */ &rhs_stride_fn4, /* .pack_func_ex = */ &rhs_pack_scale_fn12, }, /* .required_cpu = */ CPU_FEATURE_SME2, @@ -770,7 +845,6 @@ static ggml_kleidiai_kernels gemm_gemv_kernels_q8[] = { /* .get_kr = */ kai_get_kr_matmul_clamp_f32_qai8dxp1vlx4_qsi8cxp4vlx4_1vlx4vl_sme_mopa, /* .get_sr = */ kai_get_sr_matmul_clamp_f32_qai8dxp1vlx4_qsi8cxp4vlx4_1vlx4vl_sme_mopa, /* .get_dst_offset = */ kai_get_dst_offset_matmul_clamp_f32_qai8dxp1vlx4_qsi8cxp4vlx4_1vlx4vl_sme_mopa, - /* .get_dst_size = */ kai_get_dst_size_matmul_clamp_f32_qai8dxp1vlx4_qsi8cxp4vlx4_1vlx4vl_sme_mopa, /* .get_lhs_offset_ex = */ &kernel_offs_fn2, /* .get_rhs_packed_offset_ex = */ &kernel_offs_fn2, /* .run_kernel_ex = */ &kernel_run_float_fn10, @@ -790,7 +864,6 @@ static ggml_kleidiai_kernels gemm_gemv_kernels_q8[] = { /* .get_kr = */ kai_get_kr_matmul_clamp_f32_qai8dxp1x4_qsi8cxp4vlx4_1x4vl_sme_dot, /* .get_sr = */ kai_get_sr_matmul_clamp_f32_qai8dxp1x4_qsi8cxp4vlx4_1x4vl_sme_dot, /* .get_dst_offset = */ kai_get_dst_offset_matmul_clamp_f32_qai8dxp1x4_qsi8cxp4vlx4_1x4vl_sme_dot, - /* .get_dst_size = */ kai_get_dst_size_matmul_clamp_f32_qai8dxp1x4_qsi8cxp4vlx4_1x4vl_sme_dot, /* .get_lhs_offset_ex = */ &kernel_offs_fn2, /* .get_rhs_packed_offset_ex = */ &kernel_offs_fn2, /* .run_kernel_ex = */ &kernel_run_float_fn10, @@ -802,10 +875,7 @@ static ggml_kleidiai_kernels gemm_gemv_kernels_q8[] = { /* .pack_func_ex = */ &lhs_pack_float_fn9_no_bl, }, /* .rhs_info = */ { - /* .packed_stride = */ kai_get_rhs_packed_stride_rhs_pack_nxk_qsi8cxp_qsi8cx_neon, - /* .to_float = */ dequantize_row_qsi8cxp, /* .packed_size_ex = */ &rhs_ps_fn5, - /* .packed_stride_ex = */ &rhs_stride_fn4, /* .pack_func_ex = */ &rhs_pack_scale_fn12, }, /* .required_cpu = */ CPU_FEATURE_SME, @@ -823,7 +893,6 @@ static ggml_kleidiai_kernels gemm_gemv_kernels_q8[] = { /* .get_kr = */ kai_get_kr_matmul_clamp_f32_qai8dxp4x8_qsi8cxp4x8_16x4_neon_i8mm, /* .get_sr = */ kai_get_sr_matmul_clamp_f32_qai8dxp4x8_qsi8cxp4x8_16x4_neon_i8mm, /* .get_dst_offset = */ kai_get_dst_offset_matmul_clamp_f32_qai8dxp4x8_qsi8cxp4x8_16x4_neon_i8mm, - /* .get_dst_size = */ kai_get_dst_size_matmul_clamp_f32_qai8dxp4x8_qsi8cxp4x8_16x4_neon_i8mm, /* .get_lhs_offset_ex = */ &kernel_offs_fn2, /* .get_rhs_packed_offset_ex = */ &kernel_offs_fn2, /* .run_kernel_ex = */ &kernel_run_float_fn10, @@ -843,7 +912,6 @@ static ggml_kleidiai_kernels gemm_gemv_kernels_q8[] = { /* .get_kr = */ kai_get_kr_matmul_clamp_f32_qai8dxp1x8_qsi8cxp4x8_1x4_neon_dotprod, /* .get_sr = */ kai_get_sr_matmul_clamp_f32_qai8dxp1x8_qsi8cxp4x8_1x4_neon_dotprod, /* .get_dst_offset = */ kai_get_dst_offset_matmul_clamp_f32_qai8dxp1x8_qsi8cxp4x8_1x4_neon_dotprod, - /* .get_dst_size = */ kai_get_dst_size_matmul_clamp_f32_qai8dxp1x8_qsi8cxp4x8_1x4_neon_dotprod, /* .get_lhs_offset_ex = */ &kernel_offs_fn2, /* .get_rhs_packed_offset_ex = */ &kernel_offs_fn2, /* .run_kernel_ex = */ &kernel_run_float_fn10, @@ -855,10 +923,7 @@ static ggml_kleidiai_kernels gemm_gemv_kernels_q8[] = { /* .pack_func_ex = */ &lhs_pack_float_fn9_no_bl, }, /* .rhs_info = */ { - /* .packed_stride = */ kai_get_rhs_packed_stride_rhs_pack_nxk_qsi8cxp_qsi8cx_neon, - /* .to_float = */ dequantize_row_qsi8cxp, /* .packed_size_ex = */ &rhs_ps_fn5, - /* .packed_stride_ex = */ &rhs_stride_fn4, /* .pack_func_ex = */ &rhs_pack_scale_fn12, }, /* .required_cpu = */ CPU_FEATURE_I8MM | CPU_FEATURE_DOTPROD, @@ -876,7 +941,6 @@ static ggml_kleidiai_kernels gemm_gemv_kernels_q8[] = { /* .get_kr = */ kai_get_kr_matmul_clamp_f32_qai8dxp4x4_qsi8cxp4x4_16x4_neon_dotprod, /* .get_sr = */ kai_get_sr_matmul_clamp_f32_qai8dxp4x4_qsi8cxp4x4_16x4_neon_dotprod, /* .get_dst_offset = */ kai_get_dst_offset_matmul_clamp_f32_qai8dxp4x4_qsi8cxp4x4_16x4_neon_dotprod, - /* .get_dst_size = */ kai_get_dst_size_matmul_clamp_f32_qai8dxp4x4_qsi8cxp4x4_16x4_neon_dotprod, /* .get_lhs_offset_ex = */ &kernel_offs_fn2, /* .get_rhs_packed_offset_ex = */ &kernel_offs_fn2, /* .run_kernel_ex = */ &kernel_run_float_fn10, @@ -896,7 +960,6 @@ static ggml_kleidiai_kernels gemm_gemv_kernels_q8[] = { /* .get_kr = */ kai_get_kr_matmul_clamp_f32_qai8dxp1x4_qsi8cxp4x4_1x4_neon_dotprod, /* .get_sr = */ kai_get_sr_matmul_clamp_f32_qai8dxp1x4_qsi8cxp4x4_1x4_neon_dotprod, /* .get_dst_offset = */ kai_get_dst_offset_matmul_clamp_f32_qai8dxp1x4_qsi8cxp4x4_1x4_neon_dotprod, - /* .get_dst_size = */ kai_get_dst_size_matmul_clamp_f32_qai8dxp1x4_qsi8cxp4x4_1x4_neon_dotprod, /* .get_lhs_offset_ex = */ &kernel_offs_fn2, /* .get_rhs_packed_offset_ex = */ &kernel_offs_fn2, /* .run_kernel_ex = */ &kernel_run_float_fn10, @@ -908,10 +971,7 @@ static ggml_kleidiai_kernels gemm_gemv_kernels_q8[] = { /* .pack_func_ex = */ &lhs_pack_float_fn9_no_bl, }, /* .rhs_info = */ { - /* .packed_stride = */ kai_get_rhs_packed_stride_rhs_pack_nxk_qsi8cxp_qsi8cx_neon, - /* .to_float = */ dequantize_row_qsi8cxp, /* .packed_size_ex = */ &rhs_ps_fn5, - /* .packed_stride_ex = */ &rhs_stride_fn4, /* .pack_func_ex = */ &rhs_pack_scale_fn12, }, /* .required_cpu = */ CPU_FEATURE_DOTPROD, @@ -933,7 +993,6 @@ static ggml_kleidiai_kernels ggml_kleidiai_kernels_f32[] = { /* .get_kr = */ kai_get_kr_matmul_clamp_f32_f32p2vlx1_f32p2vlx1biasf32_sme2_mopa, /* .get_sr = */ kai_get_sr_matmul_clamp_f32_f32p2vlx1_f32p2vlx1biasf32_sme2_mopa, /* .get_dst_offset = */ kai_get_dst_offset_matmul_clamp_f32_f32p2vlx1_f32p2vlx1biasf32_sme2_mopa, - /* .get_dst_size = */ kai_get_dst_size_matmul_clamp_f32_f32p2vlx1_f32p2vlx1biasf32_sme2_mopa, /* .get_lhs_offset_ex = */ &kernel_offs_fn2, /* .get_rhs_packed_offset_ex = */ &kernel_offs_fn2, /* .run_kernel_ex = */ &kernel_run_fn10, @@ -953,7 +1012,6 @@ static ggml_kleidiai_kernels ggml_kleidiai_kernels_f32[] = { /* .get_kr = */ kai_get_kr_matmul_clamp_f32_f32_f32p2vlx1b_1x16vl_sme2_mla, /* .get_sr = */ kai_get_sr_matmul_clamp_f32_f32_f32p2vlx1b_1x16vl_sme2_mla, /* .get_dst_offset = */ kai_get_dst_offset_matmul_clamp_f32_f32_f32p2vlx1b_1x16vl_sme2_mla, - /* .get_dst_size = */ kai_get_dst_size_matmul_clamp_f32_f32_f32p2vlx1b_1x16vl_sme2_mla, /* .get_lhs_offset_ex = */ &kernel_offs_fn2, /* .get_rhs_packed_offset_ex = */ &kernel_offs_fn2, /* .run_kernel_ex = */ &kernel_run_lhs_stride_fn10, @@ -965,10 +1023,7 @@ static ggml_kleidiai_kernels ggml_kleidiai_kernels_f32[] = { /* .pack_func_ex = */ nullptr, }, /* .rhs_info = */ { - /* .packed_stride = */ nullptr, - /* .to_float = */ nullptr, /* .packed_size_ex = */ &rhs_ps_fn2, - /* .packed_stride_ex = */ &rhs_stride_fn1, /* .pack_func_ex = */ &rhs_pack_fn13, }, /* .required_cpu = */ CPU_FEATURE_SME2, @@ -986,7 +1041,6 @@ static ggml_kleidiai_kernels ggml_kleidiai_kernels_f32[] = { /* .get_kr = */ kai_get_kr_matmul_clamp_f32_f32p2vlx1_f32p2vlx1b_2vlx2vl_sme_mopa, /* .get_sr = */ kai_get_sr_matmul_clamp_f32_f32p2vlx1_f32p2vlx1b_2vlx2vl_sme_mopa, /* .get_dst_offset = */ kai_get_dst_offset_matmul_clamp_f32_f32p2vlx1_f32p2vlx1b_2vlx2vl_sme_mopa, - /* .get_dst_size = */ kai_get_dst_size_matmul_clamp_f32_f32p2vlx1_f32p2vlx1b_2vlx2vl_sme_mopa, /* .get_lhs_offset_ex = */ &kernel_offs_fn2, /* .get_rhs_packed_offset_ex = */ &kernel_offs_fn2, /* .run_kernel_ex = */ &kernel_run_fn10, @@ -1006,7 +1060,6 @@ static ggml_kleidiai_kernels ggml_kleidiai_kernels_f32[] = { /* .get_kr = */ kai_get_kr_matmul_clamp_f32_f32p2vlx1_f32p2vlx1b_2vlx2vl_sme_mopa, /* .get_sr = */ kai_get_sr_matmul_clamp_f32_f32p2vlx1_f32p2vlx1b_2vlx2vl_sme_mopa, /* .get_dst_offset = */ kai_get_dst_offset_matmul_clamp_f32_f32p2vlx1_f32p2vlx1b_2vlx2vl_sme_mopa, - /* .get_dst_size = */ kai_get_dst_size_matmul_clamp_f32_f32p2vlx1_f32p2vlx1b_2vlx2vl_sme_mopa, /* .get_lhs_offset_ex = */ nullptr, /* .get_rhs_packed_offset_ex = */ nullptr, /* .run_kernel_ex = */ nullptr, @@ -1018,10 +1071,7 @@ static ggml_kleidiai_kernels ggml_kleidiai_kernels_f32[] = { /* .pack_func_ex = */ &lhs_pack_void_fn9, }, /* .rhs_info = */ { - /* .packed_stride = */ nullptr, - /* .to_float = */ nullptr, /* .packed_size_ex = */ &rhs_ps_fn2, - /* .packed_stride_ex = */ &rhs_stride_fn1, /* .pack_func_ex = */ &rhs_pack_fn13, }, /* .required_cpu = */ CPU_FEATURE_SME, @@ -1049,8 +1099,23 @@ ggml_kleidiai_kernels * ggml_kleidiai_select_kernels(cpu_feature cpu_features, c return false; }; - if (tensor->src[0]->type == GGML_TYPE_Q8_0) { - try_table(gemm_gemv_kernels_q8); + if (tensor->src[0]->type == GGML_TYPE_Q4_K) { + try_table(gemm_gemv_kernels_q4_k); + } else if (tensor->src[0]->type == GGML_TYPE_Q8_0 || tensor->src[0]->type == GGML_TYPE_Q6_K) { + const ggml_type original_type = tensor->src[0]->type; + if (original_type == GGML_TYPE_Q6_K) { + for (size_t i = 0; i < NELEMS(gemm_gemv_kernels_q8) - 1; ++i) { + if ((cpu_features & gemm_gemv_kernels_q8[i].required_cpu) == gemm_gemv_kernels_q8[i].required_cpu && + gemm_gemv_kernels_q8[i].lhs_type == tensor->src[1]->type && + gemm_gemv_kernels_q8[i].rhs_type == GGML_TYPE_Q8_0 && + gemm_gemv_kernels_q8[i].op_type == tensor->type) { + kernel = &gemm_gemv_kernels_q8[i]; + break; + } + } + } else { + try_table(gemm_gemv_kernels_q8); + } } else if (tensor->src[0]->type == GGML_TYPE_F32) { try_table(ggml_kleidiai_kernels_f32); } else { @@ -1075,6 +1140,19 @@ ggml_kleidiai_kernels * ggml_kleidiai_select_kernels_q4_0(cpu_feature features) return kernels; } +ggml_kleidiai_kernels * ggml_kleidiai_select_kernels_q4_k(cpu_feature features) { + ggml_kleidiai_kernels * kernels = nullptr; + + for (size_t i = 0; i < NELEMS(gemm_gemv_kernels_q4_k) - 1; ++i) { + if ((features & gemm_gemv_kernels_q4_k[i].required_cpu) == gemm_gemv_kernels_q4_k[i].required_cpu) { + kernels = &gemm_gemv_kernels_q4_k[i]; + break; + } + } + + return kernels; +} + ggml_kleidiai_kernels * ggml_kleidiai_select_kernels_q8_0(cpu_feature features) { ggml_kleidiai_kernels * kernels = nullptr; diff --git a/ggml/src/ggml-cpu/kleidiai/kernels.h b/ggml/src/ggml-cpu/kleidiai/kernels.h index 1da8610ea..308ea9451 100644 --- a/ggml/src/ggml-cpu/kleidiai/kernels.h +++ b/ggml/src/ggml-cpu/kleidiai/kernels.h @@ -33,7 +33,6 @@ struct kernel_info { size_t (*get_sr)(void); size_t (*get_dst_offset)(size_t m_idx, size_t n_idx, size_t stride); - size_t (*get_dst_size)(size_t m, size_t n); size_t (*get_lhs_offset_ex)(size_t m_idx, size_t k, size_t bl); @@ -59,21 +58,12 @@ struct lhs_packing_info { enum rhs_repack_mode { RHS_REPACK_PER_KERNEL, - RHS_REPACK_SHARED, RHS_REPACK_SINGLE_ONLY, }; struct rhs_packing_info { - size_t (*packed_stride)(size_t k, size_t nr, size_t kr, size_t bl); - - void (*to_float)(const void *packed_data, int32_t row_idx, int64_t nc, float *out, - size_t nr_pack, size_t packed_row_stride, size_t kr, size_t bl, - size_t num_bytes_multiplier); - size_t (*packed_size_ex)(size_t n, size_t k, size_t nr, size_t kr, size_t bl); - size_t (*packed_stride_ex)(size_t k, size_t nr, size_t kr, size_t bl); - void (*pack_func_ex)(size_t num_groups, size_t n, size_t k, size_t nr, size_t kr, size_t sr, size_t bl, size_t rhs_stride, const void * rhs, const void * bias, const void * scale, void * rhs_packed, size_t extra_bytes, const void * params); @@ -97,5 +87,6 @@ struct ggml_kleidiai_kernels { ggml_kleidiai_kernels * ggml_kleidiai_select_kernels(cpu_feature cpu_features, const ggml_tensor * tensor); ggml_kleidiai_kernels * ggml_kleidiai_select_kernels_q4_0(cpu_feature features); +ggml_kleidiai_kernels * ggml_kleidiai_select_kernels_q4_k(cpu_feature features); ggml_kleidiai_kernels * ggml_kleidiai_select_kernels_q8_0(cpu_feature features); ggml_kleidiai_kernels * ggml_kleidiai_select_kernels_f32(cpu_feature features); diff --git a/ggml/src/ggml-cpu/kleidiai/kleidiai.cpp b/ggml/src/ggml-cpu/kleidiai/kleidiai.cpp index 1384ca927..dd9bcbf63 100644 --- a/ggml/src/ggml-cpu/kleidiai/kleidiai.cpp +++ b/ggml/src/ggml-cpu/kleidiai/kleidiai.cpp @@ -43,6 +43,7 @@ #include "ggml-impl.h" #include "ggml-feats.h" #include "ggml-backend-impl.h" +#include "ggml-quants.h" #include "ggml-threading.h" #include "traits.h" @@ -57,16 +58,20 @@ static constexpr int GGML_KLEIDIAI_MAX_KERNEL_SLOTS = 2; static constexpr uint32_t GGML_KLEIDIAI_PACK_MAGIC = 0x4b4c4149; // "KLAI" static constexpr uint16_t GGML_KLEIDIAI_PACK_VERSION = 1; static constexpr size_t GGML_KLEIDIAI_PACK_ALIGN = 64; +static constexpr size_t GGML_KLEIDIAI_Q4_K_BL = QK_K / 8; + +static_assert(QK_K == 256, "KleidiAI Q4_K kernels require 256-value super-blocks"); struct ggml_kleidiai_context { cpu_feature features; ggml_kleidiai_kernels * kernels_q4; + ggml_kleidiai_kernels * kernels_q4k; ggml_kleidiai_kernels * kernels_q8; ggml_kleidiai_kernels * kernels_f32; int sme_thread_cap; // <= 0 means "SME disabled/unknown" int thread_hint; // <= 0 means "no hint" int chunk_multiplier; -} static ctx = { CPU_FEATURE_NONE, nullptr, nullptr, nullptr, 0, -1, 4 }; +} static ctx = { CPU_FEATURE_NONE, nullptr, nullptr, nullptr, nullptr, 0, -1, 4 }; static inline bool is_sme_family(cpu_feature f) { return (f & (CPU_FEATURE_SME | CPU_FEATURE_SME2)) != CPU_FEATURE_NONE; @@ -387,6 +392,7 @@ static void init_kleidiai_context(void) { // Kernel selection ctx.kernels_q4 = ggml_kleidiai_select_kernels_q4_0(ctx.features); + ctx.kernels_q4k = ggml_kleidiai_select_kernels_q4_k(ctx.features); ctx.kernels_q8 = ggml_kleidiai_select_kernels_q8_0(ctx.features); ctx.kernels_f32 = ggml_kleidiai_select_kernels_f32(ctx.features); @@ -402,6 +408,12 @@ static void init_kleidiai_context(void) { GGML_LOG_INFO("kleidiai: primary q8 kernel feature %s\n", cpu_feature_to_string(ctx.kernels_q8->required_cpu)); } + if (!ctx.kernels_q4k) { + GGML_LOG_INFO("kleidiai: no compatible q4_k kernels found for CPU features mask %d\n", (int) ctx.features); + } else { + GGML_LOG_INFO("kleidiai: primary q4_k kernel feature %s\n", cpu_feature_to_string(ctx.kernels_q4k->required_cpu)); + } + if (!ctx.kernels_f32) { GGML_LOG_INFO("kleidiai: no compatible f32 kernels found for CPU features mask %d\n", (int)ctx.features); } else { @@ -410,6 +422,7 @@ static void init_kleidiai_context(void) { const bool has_selected_sme_family_kernel = (ctx.kernels_q4 && is_sme_family(ctx.kernels_q4->required_cpu)) || + (ctx.kernels_q4k && is_sme_family(ctx.kernels_q4k->required_cpu)) || (ctx.kernels_q8 && is_sme_family(ctx.kernels_q8->required_cpu)) || (ctx.kernels_f32 && is_sme_family(ctx.kernels_f32->required_cpu)); ctx.sme_thread_cap = has_selected_sme_family_kernel ? sme_cores : 0; @@ -486,6 +499,8 @@ static inline kleidiai_block_args kleidiai_get_block_args(ggml_type rhs_type) { switch (rhs_type) { case GGML_TYPE_Q4_0: return { QK4_0, QK4_0, QK4_0 }; + case GGML_TYPE_Q4_K: + return { GGML_KLEIDIAI_Q4_K_BL, GGML_KLEIDIAI_Q4_K_BL, GGML_KLEIDIAI_Q4_K_BL }; case GGML_TYPE_Q8_0: return { 0, 0, QK8_0 }; default: @@ -556,6 +571,10 @@ static inline ggml_kleidiai_kernels * kleidiai_primary_kernel_q4() { return ctx.kernels_q4; } +static inline ggml_kleidiai_kernels * kleidiai_primary_kernel_q4k() { + return ctx.kernels_q4k; +} + static inline ggml_kleidiai_kernels * kleidiai_primary_kernel_q8() { return ctx.kernels_q8; } @@ -610,6 +629,12 @@ static int kleidiai_collect_q4_chain(std::array & out) { + ggml_kleidiai_kernels * primary = kleidiai_primary_kernel_q4k(); + return kleidiai_collect_kernel_chain_common(primary, ctx.features, out, + [&](cpu_feature mask) { return ggml_kleidiai_select_kernels_q4_k(mask); }); +} + static int kleidiai_collect_q8_chain(std::array & out) { ggml_kleidiai_kernels * primary = kleidiai_primary_kernel_q8(); return kleidiai_collect_kernel_chain_common(primary, ctx.features, out, @@ -662,8 +687,10 @@ class tensor_traits : public ggml::cpu::tensor_traits { const size_t n = op->src[0]->ne[1]; const size_t m = op->src[1]->ne[1]; - if (op->src[0]->type == GGML_TYPE_Q4_0 || op->src[0]->type == GGML_TYPE_Q8_0) { - const size_t qk = (op->src[0]->type == GGML_TYPE_Q4_0) ? QK4_0 : QK8_0; + if (op->src[0]->type == GGML_TYPE_Q4_0 || op->src[0]->type == GGML_TYPE_Q4_K || + op->src[0]->type == GGML_TYPE_Q8_0 || op->src[0]->type == GGML_TYPE_Q6_K) { + const kleidiai_block_args block_args = kleidiai_get_block_args(kernel_chain[0]->rhs_type); + const size_t qk = block_args.pack_bl; size_t cursor = 0; bool any_slot = false; @@ -789,17 +816,14 @@ class tensor_traits : public ggml::cpu::tensor_traits { bool compute_forward(struct ggml_compute_params * params, struct ggml_tensor * dst) override { if (dst->op == GGML_OP_MUL_MAT) { - if (dst->src[0]->type == GGML_TYPE_Q4_0 || dst->src[0]->type == GGML_TYPE_Q8_0) { + if (dst->src[0]->type == GGML_TYPE_Q4_0 || dst->src[0]->type == GGML_TYPE_Q4_K || + dst->src[0]->type == GGML_TYPE_Q8_0 || dst->src[0]->type == GGML_TYPE_Q6_K) { return compute_forward_qx(params, dst); } else if (dst->src[0]->type == GGML_TYPE_F32) { return compute_forward_f32(params, dst); } else if (dst->src[0]->type == GGML_TYPE_F16) { return compute_forward_fp16(params, dst); } - } else if (dst->op == GGML_OP_GET_ROWS) { - if (dst->src[0]->type == GGML_TYPE_Q4_0 || dst->src[0]->type == GGML_TYPE_Q8_0) { - return compute_forward_get_rows(params, dst); - } } return false; } @@ -870,9 +894,12 @@ class tensor_traits : public ggml::cpu::tensor_traits { const bool disable_chunking = ggml_is_numa(); GGML_ASSERT(n <= (size_t) INT_MAX); - for (int64_t batch_idx = 0; batch_idx < ne12; ++batch_idx) { - const uint8_t * lhs_batch_base = static_cast(src1->data) + batch_idx * src1->nb[2]; - uint8_t * dst_batch_base = static_cast(dst->data) + batch_idx * dst->nb[2]; + const int64_t batch_size = ne12 * ne13; + for (int64_t batch_idx = 0; batch_idx < batch_size; ++batch_idx) { + const int64_t i12 = batch_idx % ne12; + const int64_t i13 = batch_idx / ne12; + const uint8_t * lhs_batch_base = static_cast(src1->data) + i12 * src1->nb[2] + i13 * src1->nb[3]; + uint8_t * dst_batch_base = static_cast(dst->data) + i12 * dst->nb[2] + i13 * dst->nb[3]; if (!use_gemv) { const int64_t m_roundup_mr = kai_roundup((int64_t)m, (int64_t)mr); @@ -947,7 +974,7 @@ class tensor_traits : public ggml::cpu::tensor_traits { current_col = ggml_threadpool_chunk_add(params->threadpool, (int) chunk_cols); } - if (batch_idx != ne12 - 1) { + if (batch_idx != batch_size - 1) { ggml_barrier(params->threadpool); } } @@ -1107,7 +1134,8 @@ class tensor_traits : public ggml::cpu::tensor_traits { } bool compute_forward_qx(struct ggml_compute_params * params, struct ggml_tensor * dst) { - GGML_ASSERT(dst->src[0]->type == GGML_TYPE_Q4_0 || dst->src[0]->type == GGML_TYPE_Q8_0); + GGML_ASSERT(dst->src[0]->type == GGML_TYPE_Q4_0 || dst->src[0]->type == GGML_TYPE_Q4_K || + dst->src[0]->type == GGML_TYPE_Q8_0 || dst->src[0]->type == GGML_TYPE_Q6_K); const ggml_tensor * src0 = dst->src[0]; const ggml_tensor * src1 = dst->src[1]; @@ -1265,7 +1293,9 @@ class tensor_traits : public ggml::cpu::tensor_traits { } } - int sme_cap = kleidiai_sme_thread_cap(); + const bool sme_single_only = sme_slot != -1 && + runtime[sme_slot].kernels->rhs_info.repack_mode == RHS_REPACK_SINGLE_ONLY; + int sme_cap = sme_single_only ? nth_total : kleidiai_sme_thread_cap(); if (sme_cap < 0) { sme_cap = nth_total; } @@ -1301,23 +1331,14 @@ class tensor_traits : public ggml::cpu::tensor_traits { threads_remaining -= share; } - if (threads_remaining > 0) { - const int fallback_slot = (sme_slot != -1) ? sme_slot : 0; - runtime[fallback_slot].assigned_threads += threads_remaining; - threads_remaining = 0; - } - int thread_cursor = 0; for (int i = 0; i < runtime_count; ++i) { runtime[i].thread_begin = thread_cursor; thread_cursor += runtime[i].assigned_threads; runtime[i].thread_end = thread_cursor; } - - if (thread_cursor < nth_total && runtime_count > 0) { - runtime[runtime_count - 1].assigned_threads += nth_total - thread_cursor; - runtime[runtime_count - 1].thread_end = nth_total; - } + GGML_ASSERT(thread_cursor > 0); + const int active_threads = thread_cursor; int local_slot = -1; int local_ith = 0; @@ -1328,10 +1349,6 @@ class tensor_traits : public ggml::cpu::tensor_traits { break; } } - if (local_slot == -1) { - return false; - } - const size_t k = ne00; const size_t m = ne11; const size_t n = ne01; @@ -1363,7 +1380,7 @@ class tensor_traits : public ggml::cpu::tensor_traits { const bool disable_chunking = ggml_is_numa(); const size_t chunk_multiplier = std::max(1, ctx.chunk_multiplier); - const size_t chunk_divisor = (nth_total == 1 || disable_chunking) ? (size_t)nth_total : (size_t)nth_total * chunk_multiplier; + const size_t chunk_divisor = (active_threads == 1 || disable_chunking) ? (size_t) active_threads : (size_t) active_threads * chunk_multiplier; size_t chunk_cols = align_up(std::max(1, ceil_div_size(n, chunk_divisor)), common_step); if (chunk_cols == 0) { chunk_cols = common_step; @@ -1397,7 +1414,7 @@ class tensor_traits : public ggml::cpu::tensor_traits { const uint8_t * lhs_batch_base = static_cast(src1->data) + batch_idx * src1->nb[2]; uint8_t * dst_batch_base = static_cast(dst->data) + batch_idx * dst->nb[2]; - if (runtime[local_slot].assigned_threads > 0) { + if (local_slot != -1) { runtime_slot & slot = runtime[local_slot]; const int64_t m_roundup_mr = kai_roundup((int64_t)m, (int64_t)slot.mr); int64_t max_threads = slot.mr ? (m_roundup_mr / (int64_t)slot.mr) : slot.assigned_threads; @@ -1438,28 +1455,30 @@ class tensor_traits : public ggml::cpu::tensor_traits { } if (ith_total == 0) { - ggml_threadpool_chunk_set(params->threadpool, nth_total); + ggml_threadpool_chunk_set(params->threadpool, active_threads); } // Publishes both LHS packing and the initialized dynamic chunk queue. ggml_barrier(params->threadpool); - runtime_slot & slot = runtime[local_slot]; - int current_chunk = ith_total; - while (current_chunk < nchunk) { - const size_t global_start = (size_t)current_chunk * chunk_cols; - if (global_start >= n) { - break; - } + if (local_slot != -1) { + runtime_slot & slot = runtime[local_slot]; + int current_chunk = ith_total; + while (current_chunk < nchunk) { + const size_t global_start = (size_t)current_chunk * chunk_cols; + if (global_start >= n) { + break; + } - const size_t cols = std::min(chunk_cols, n - global_start); - if (cols > 0) { - // KleidiAI GEMM/GEMV kernels accept arbitrary final tail widths; - // only non-tail chunks are guaranteed to be n_step-aligned. - run_chunk(slot, global_start, cols, dst_batch_base); - } + const size_t cols = std::min(chunk_cols, n - global_start); + if (cols > 0) { + // KleidiAI GEMM/GEMV kernels accept arbitrary final tail widths; + // only non-tail chunks are guaranteed to be n_step-aligned. + run_chunk(slot, global_start, cols, dst_batch_base); + } - current_chunk = ggml_threadpool_chunk_add(params->threadpool, 1); + current_chunk = ggml_threadpool_chunk_add(params->threadpool, 1); + } } if (batch_idx != ne12 - 1) { @@ -1470,101 +1489,11 @@ class tensor_traits : public ggml::cpu::tensor_traits { return true; } - bool compute_forward_get_rows(struct ggml_compute_params * params, struct ggml_tensor * dst) { - GGML_ASSERT(dst->src[0]->type == GGML_TYPE_Q4_0 || dst->src[0]->type == GGML_TYPE_Q8_0); - const ggml_tensor * src0 = dst->src[0]; - const ggml_tensor * src1 = dst->src[1]; - - GGML_TENSOR_BINARY_OP_LOCALS - - const kleidiai_weight_header * header = kleidiai_weight_header_from_ptr(src0->data); - const bool has_header = kleidiai_is_weight_header_valid(header); - - std::array kernel_chain; - const bool want_q8 = src0->type == GGML_TYPE_Q8_0; - const int chain_count = want_q8 ? kleidiai_collect_q8_chain(kernel_chain) - : kleidiai_collect_q4_chain(kernel_chain); - - ggml_kleidiai_kernels * kernels = nullptr; - const uint8_t * packed_base = static_cast(src0->data); - - if (has_header && chain_count > 0) { - int select_slot = 0; - if (select_slot >= header->slot_count) { - select_slot = header->slot_count - 1; - } - if (select_slot >= 0 && select_slot < chain_count) { - kernels = kernel_chain[select_slot]; - const uint8_t * slot_ptr = kleidiai_weight_slot_ptr(header, select_slot); - if (slot_ptr) { - packed_base = slot_ptr; - } - } - } - - if (!kernels && chain_count > 0) { - kernels = kernel_chain[0]; - if (has_header) { - const uint8_t * slot_ptr = kleidiai_weight_slot_ptr(header, 0); - if (slot_ptr) { - packed_base = slot_ptr; - } - } - } - - if (!kernels) { - return false; - } - - rhs_packing_info * rhs_info = &kernels->rhs_info; - kernel_info * kernel = &kernels->gemm; - if (!rhs_info->to_float || !kernel->get_nr) { - return false; - } - - const int64_t nc = ne00; - const int64_t nr = ggml_nelements(src1); - - const ggml_type rhs_type = kernels->rhs_type; - size_t block_len = 0; - size_t num_bytes_multiplier = 0; - if (rhs_type == GGML_TYPE_Q4_0) { - block_len = QK4_0; - num_bytes_multiplier = sizeof(uint16_t); - } else if (rhs_type == GGML_TYPE_Q8_0) { - block_len = QK8_0; - num_bytes_multiplier = sizeof(float); - } else { - return false; - } - - const size_t block_rows = kernel->get_nr(); - const size_t kr = kernel->get_kr(); - - const size_t packed_stride = rhs_info->packed_stride(nc, block_rows, kr, block_len); - - const int ith = params->ith; - const int nth = params->nth; - - const int dr = (nr + nth - 1) / nth; - const int ir0 = dr * ith; - const int ir1 = MIN(ir0 + dr, nr); - - for (int64_t i = ir0; i < ir1; ++i) { - GGML_ASSERT(src1->type == GGML_TYPE_I32); - int64_t row_idx = ((const int32_t *)src1->data)[i]; - GGML_ASSERT(row_idx >= 0 && row_idx < src0->ne[1]); - - float *out = (float *)((char *)dst->data + i * nb1); - rhs_info->to_float(packed_base, row_idx, nc, out, block_rows, packed_stride, kr, block_len, num_bytes_multiplier); - } - - return true; - } - public: int repack(struct ggml_tensor * tensor, const void * data, size_t data_size) { - GGML_ASSERT(tensor->type == GGML_TYPE_Q4_0 || tensor->type == GGML_TYPE_Q8_0 || tensor->type == GGML_TYPE_F32); + GGML_ASSERT(tensor->type == GGML_TYPE_Q4_0 || tensor->type == GGML_TYPE_Q4_K || + tensor->type == GGML_TYPE_Q8_0 || tensor->type == GGML_TYPE_Q6_K || + tensor->type == GGML_TYPE_F32); const size_t n = tensor->ne[1]; const size_t k = tensor->ne[0]; @@ -1582,16 +1511,19 @@ public: cursor = align_up(cursor, GGML_KLEIDIAI_PACK_ALIGN); std::array kernel_chain; - const bool want_q8 = tensor->type == GGML_TYPE_Q8_0; + const bool want_q4k = tensor->type == GGML_TYPE_Q4_K; + const bool want_q8 = tensor->type == GGML_TYPE_Q8_0 || tensor->type == GGML_TYPE_Q6_K; const bool want_f32 = tensor->type == GGML_TYPE_F32; const int slot_total = want_f32 ? kleidiai_collect_f32_chain(kernel_chain) : want_q8 ? kleidiai_collect_q8_chain(kernel_chain) - : kleidiai_collect_q4_chain(kernel_chain); + : want_q4k ? kleidiai_collect_q4k_chain(kernel_chain) + : kleidiai_collect_q4_chain(kernel_chain); const bool allow_fallback = kleidiai_pack_fallback_allowed(); std::vector qdata; std::vector scales; std::vector bias; + std::vector row_f32; if (want_q8 && slot_total > 0) { qdata.resize(n * k, 0); @@ -1600,21 +1532,38 @@ public: const size_t row_stride = tensor->nb[1]; const size_t k_blocks = (k + QK8_0 - 1) / QK8_0; + if (tensor->type == GGML_TYPE_Q6_K) { + if (k % QK_K != 0 || k % QK8_0 != 0) { + return -1; + } + row_f32.resize(k); + } + for (size_t row = 0; row < n; ++row) { - const auto * row_blocks = reinterpret_cast( - static_cast(data) + row * row_stride); + const uint8_t * row_data = static_cast(data) + row * row_stride; + + if (tensor->type == GGML_TYPE_Q6_K) { + dequantize_row_q6_K(reinterpret_cast(row_data), row_f32.data(), k); + } float max_abs = 0.0f; - for (size_t block = 0; block < k_blocks; ++block) { - const block_q8_0 & blk = row_blocks[block]; - const float d = GGML_FP16_TO_FP32(blk.d); - for (size_t l = 0; l < QK8_0; ++l) { - const size_t linear_idx = block * QK8_0 + l; - if (linear_idx >= k) { - break; + if (tensor->type == GGML_TYPE_Q6_K) { + for (size_t linear_idx = 0; linear_idx < k; ++linear_idx) { + max_abs = std::max(max_abs, std::fabs(row_f32[linear_idx])); + } + } else { + const block_q8_0 * row_blocks = reinterpret_cast(row_data); + for (size_t block = 0; block < k_blocks; ++block) { + const block_q8_0 & blk = row_blocks[block]; + const float d = GGML_FP16_TO_FP32(blk.d); + for (size_t l = 0; l < QK8_0; ++l) { + const size_t linear_idx = block * QK8_0 + l; + if (linear_idx >= k) { + break; + } + const float value = d * static_cast(blk.qs[l]); + max_abs = std::max(max_abs, std::fabs(value)); } - const float value = d * static_cast(blk.qs[l]); - max_abs = std::max(max_abs, std::fabs(value)); } } @@ -1622,19 +1571,28 @@ public: scales[row] = scale; const float inv_scale = scale > 0.0f ? 1.0f / scale : 0.0f; - for (size_t block = 0; block < k_blocks; ++block) { - const block_q8_0 & blk = row_blocks[block]; - const float d = GGML_FP16_TO_FP32(blk.d); - for (size_t l = 0; l < QK8_0; ++l) { - const size_t linear_idx = block * QK8_0 + l; - if (linear_idx >= k) { - break; - } - const float value = d * static_cast(blk.qs[l]); - int32_t q = scale > 0.0f ? static_cast(std::lround(value * inv_scale)) : 0; + if (tensor->type == GGML_TYPE_Q6_K) { + for (size_t linear_idx = 0; linear_idx < k; ++linear_idx) { + int32_t q = scale > 0.0f ? static_cast(std::lround(row_f32[linear_idx] * inv_scale)) : 0; q = std::clamp(q, -127, 127); qdata[row * k + linear_idx] = static_cast(q); } + } else { + const block_q8_0 * row_blocks = reinterpret_cast(row_data); + for (size_t block = 0; block < k_blocks; ++block) { + const block_q8_0 & blk = row_blocks[block]; + const float d = GGML_FP16_TO_FP32(blk.d); + for (size_t l = 0; l < QK8_0; ++l) { + const size_t linear_idx = block * QK8_0 + l; + if (linear_idx >= k) { + break; + } + const float value = d * static_cast(blk.qs[l]); + int32_t q = scale > 0.0f ? static_cast(std::lround(value * inv_scale)) : 0; + q = std::clamp(q, -127, 127); + qdata[row * k + linear_idx] = static_cast(q); + } + } } } } @@ -1660,6 +1618,7 @@ public: const ggml_type rhs_type = kernels->rhs_type; const size_t block_len = rhs_type == GGML_TYPE_Q8_0 ? QK8_0 : rhs_type == GGML_TYPE_Q4_0 ? QK4_0 : + rhs_type == GGML_TYPE_Q4_K ? GGML_KLEIDIAI_Q4_K_BL : rhs_type == GGML_TYPE_F32 ? 0 : SIZE_MAX; if (block_len == SIZE_MAX) { continue; @@ -1677,6 +1636,9 @@ public: rhs_info->pack_func_ex(1, n, k, nr, kr, sr, QK4_0, 0, static_cast(data), nullptr, nullptr, dst_ptr, 0, ¶ms); + } else if (rhs_type == GGML_TYPE_Q4_K) { + rhs_info->pack_func_ex(1, n, k, nr, kr, sr, GGML_KLEIDIAI_Q4_K_BL, tensor->nb[1], + data, nullptr, nullptr, dst_ptr, 0, nullptr); } else if (rhs_type == GGML_TYPE_Q8_0) { struct kai_rhs_pack_qsi8cx_params params; params.lhs_zero_point = 1; @@ -1719,7 +1681,6 @@ static enum ggml_status ggml_backend_cpu_kleidiai_buffer_init_tensor(ggml_backen tensor->extra = (void *) ggml::cpu::kleidiai::get_tensor_traits(buffer, tensor); return GGML_STATUS_SUCCESS; - GGML_UNUSED(buffer); } static void ggml_backend_cpu_kleidiai_buffer_set_tensor(ggml_backend_buffer_t buffer, struct ggml_tensor * tensor, @@ -1762,7 +1723,9 @@ static size_t ggml_backend_cpu_kleidiai_buffer_type_get_alignment(ggml_backend_b static size_t ggml_backend_cpu_kleidiai_buffer_type_get_alloc_size(ggml_backend_buffer_type_t buft, const struct ggml_tensor * tensor) { GGML_UNUSED(buft); - if (tensor->type != GGML_TYPE_Q4_0 && tensor->type != GGML_TYPE_Q8_0 && tensor->type != GGML_TYPE_F32) { + if (tensor->type != GGML_TYPE_Q4_0 && tensor->type != GGML_TYPE_Q4_K && + tensor->type != GGML_TYPE_Q8_0 && tensor->type != GGML_TYPE_Q6_K && + tensor->type != GGML_TYPE_F32) { return ggml_nbytes(tensor); } @@ -1773,11 +1736,13 @@ static size_t ggml_backend_cpu_kleidiai_buffer_type_get_alloc_size(ggml_backend_ cursor = align_up(cursor, GGML_KLEIDIAI_PACK_ALIGN); std::array kernel_chain; - const bool want_q8 = tensor->type == GGML_TYPE_Q8_0; + const bool want_q4k = tensor->type == GGML_TYPE_Q4_K; + const bool want_q8 = tensor->type == GGML_TYPE_Q8_0 || tensor->type == GGML_TYPE_Q6_K; const bool want_f32 = tensor->type == GGML_TYPE_F32; const int slot_total = want_f32 ? kleidiai_collect_f32_chain(kernel_chain) : want_q8 ? kleidiai_collect_q8_chain(kernel_chain) - : kleidiai_collect_q4_chain(kernel_chain); + : want_q4k ? kleidiai_collect_q4k_chain(kernel_chain) + : kleidiai_collect_q4_chain(kernel_chain); const bool allow_fallback = kleidiai_pack_fallback_allowed(); size_t slot_count = 0; @@ -1797,6 +1762,7 @@ static size_t ggml_backend_cpu_kleidiai_buffer_type_get_alloc_size(ggml_backend_ const ggml_type rhs_type = kernels->rhs_type; const size_t block_len = rhs_type == GGML_TYPE_Q4_0 ? QK4_0 : + rhs_type == GGML_TYPE_Q4_K ? GGML_KLEIDIAI_Q4_K_BL : rhs_type == GGML_TYPE_Q8_0 ? QK8_0 : rhs_type == GGML_TYPE_F32 ? 0 : SIZE_MAX; if (block_len == SIZE_MAX) { @@ -1818,6 +1784,11 @@ static size_t ggml_backend_cpu_kleidiai_buffer_type_get_alloc_size(ggml_backend_ namespace ggml::cpu::kleidiai { class extra_buffer_type : ggml::cpu::extra_buffer_type { bool supports_op(ggml_backend_dev_t, const struct ggml_tensor * op) override { + // GET_ROWS needs the original weights in a native buffer. + if (op->op != GGML_OP_MUL_MAT) { + return false; + } + std::array kernel_chain; const int slot_total = kleidiai_collect_kernel_chain(op, kernel_chain); const bool src0_is_kleidiai = @@ -1826,8 +1797,9 @@ class extra_buffer_type : ggml::cpu::extra_buffer_type { op->src[0]->buffer->buft->context == this && slot_total > 0; - if ((op->op == GGML_OP_MUL_MAT || op->op == GGML_OP_GET_ROWS) && - (op->src[0]->type == GGML_TYPE_Q4_0 || op->src[0]->type == GGML_TYPE_Q8_0 || op->src[0]->type == GGML_TYPE_F32) && + if ((op->src[0]->type == GGML_TYPE_Q4_0 || op->src[0]->type == GGML_TYPE_Q4_K || + op->src[0]->type == GGML_TYPE_Q8_0 || op->src[0]->type == GGML_TYPE_Q6_K || + op->src[0]->type == GGML_TYPE_F32) && src0_is_kleidiai) { if (op->src[0]->type == GGML_TYPE_Q4_0 && ctx.kernels_q4 == nullptr) { return false; @@ -1835,6 +1807,12 @@ class extra_buffer_type : ggml::cpu::extra_buffer_type { if (op->src[0]->type == GGML_TYPE_Q8_0 && ctx.kernels_q8 == nullptr) { return false; } + if (op->src[0]->type == GGML_TYPE_Q6_K && ctx.kernels_q8 == nullptr) { + return false; + } + if (op->src[0]->type == GGML_TYPE_Q4_K && ctx.kernels_q4k == nullptr) { + return false; + } if (op->src[0]->type == GGML_TYPE_F32 && ctx.kernels_f32 == nullptr) { return false; } @@ -1842,8 +1820,18 @@ class extra_buffer_type : ggml::cpu::extra_buffer_type { return false; } - if (op->src[0]->type == GGML_TYPE_Q4_0 || op->src[0]->type == GGML_TYPE_Q8_0) { - if ((op->src[1]->type == GGML_TYPE_F32 || op->src[1]->type == GGML_TYPE_I32) && + if (op->src[0]->type == GGML_TYPE_Q4_K) { + if (op->op != GGML_OP_MUL_MAT || op->src[1]->type != GGML_TYPE_F32 || op->type != GGML_TYPE_F32) { + return false; + } + return ggml_ne(op->src[1], 2) == 1 && ggml_ne(op->src[1], 3) == 1 && + op->src[1]->nb[0] == sizeof(float) && + (op->src[1]->ne[1] == 1 || op->src[1]->nb[1] == op->src[1]->ne[0] * sizeof(float)); + } + + if (op->src[0]->type == GGML_TYPE_Q4_0 || op->src[0]->type == GGML_TYPE_Q8_0 || + op->src[0]->type == GGML_TYPE_Q6_K) { + if (op->src[1]->type == GGML_TYPE_F32 && ggml_ne(op->src[1], 3) == 1) { return true; } @@ -1861,21 +1849,22 @@ class extra_buffer_type : ggml::cpu::extra_buffer_type { } ggml::cpu::tensor_traits * get_tensor_traits(const struct ggml_tensor * op) override { - if (op->op == GGML_OP_MUL_MAT || op->op == GGML_OP_GET_ROWS) { + if (op->op == GGML_OP_MUL_MAT) { if (op->src[0]->buffer && op->src[0]->buffer->buft->context == this) { return (ggml::cpu::tensor_traits *) op->src[0]->extra; } else { - // KleidiAI only has kernels for Q4_0 and Q8_0. For a quantized weight of any + // KleidiAI only has kernels for Q4_0, Q4_K, Q8_0 and Q6_K. For a quantized weight of any // other type (K-quants, IQ) it declines the op and returns nullptr below, so // KleidiAI does not accelerate it. Another CPU backend may still take the op, // and this can run during graph planning, so the message says what KleidiAI // did rather than what ends up executing. Warn once per process. if (ggml_is_quantized(op->src[0]->type) && - op->src[0]->type != GGML_TYPE_Q4_0 && op->src[0]->type != GGML_TYPE_Q8_0) { + op->src[0]->type != GGML_TYPE_Q4_0 && op->src[0]->type != GGML_TYPE_Q4_K && + op->src[0]->type != GGML_TYPE_Q8_0 && op->src[0]->type != GGML_TYPE_Q6_K) { static std::atomic warned(false); if (!warned.exchange(true)) { GGML_LOG_WARN("kleidiai: no kernel for tensor type %s, not accelerated by KleidiAI " - "(kernels available for Q4_0 and Q8_0)\n", + "(kernels available for Q4_0, Q4_K, Q8_0 and Q6_K)\n", ggml_type_name(op->src[0]->type)); } }