kleidiai: add SME2 Q4_K and Q6_K matmul support (#29219)
Change-Id: Ib4aa4c9814c010f9cf084783cc80612e364f9a27 Change-Id: I406a6606c3fea51f8cd32331c1ed71466845d363 Change-Id: Ib4aa4c9814c010f9cf084783cc80612e364f9a27 Signed-off-by: Charles Xu <charles.xu@arm.com>
This commit is contained in:
1 parent
77c4540ce4
commit
45451c366d
4 files changed
+443
-385
No files matched your search
@@ -650,9 +650,9 @@ function(ggml_add_cpu_backend_variant_impl tag_name)
|
||||
|
||||
# Fetch KleidiAI sources:
|
||||
include(FetchContent)
|
||||
set(KLEIDIAI_COMMIT_TAG "v1.24.0")
|
||||
set(KLEIDIAI_COMMIT_TAG "v1.31.0")
|
||||
set(KLEIDIAI_DOWNLOAD_URL "https://github.com/ARM-software/kleidiai/releases/download/${KLEIDIAI_COMMIT_TAG}/kleidiai-${KLEIDIAI_COMMIT_TAG}-src.tar.gz")
|
||||
set(KLEIDIAI_RELEASE_ARCHIVE_MD5 "2f02ebe29573d45813e671eb304f2a00")
|
||||
set(KLEIDIAI_RELEASE_ARCHIVE_MD5 "930495f3aeb16b25dc18a2bde45924f4")
|
||||
|
||||
set(KLEIDIAI_FETCH_ARGS
|
||||
URL ${KLEIDIAI_DOWNLOAD_URL}
|
||||
|
||||
@@ -3,6 +3,8 @@
|
||||
//
|
||||
|
||||
// KleidiAI micro-kernels
|
||||
#include <cstddef>
|
||||
|
||||
#include "kai/ukernels/matmul/matmul_clamp_f32_qsi8d32p_qsi4c32p/kai_matmul_clamp_f32_qsi8d32p_qsi4c32p_interface.h"
|
||||
#include "kai/ukernels/matmul/matmul_clamp_f32_qai8dxp_qsi8cxp/kai_matmul_clamp_f32_qai8dxp_qsi8cxp_interface.h"
|
||||
#include "kai/ukernels/matmul/matmul_clamp_f32_qsi8d32p_qsi4c32p/kai_matmul_clamp_f32_qsi8d32p1x8_qsi4c32p4x8_1x4x32_neon_dotprod.h"
|
||||
@@ -33,6 +35,10 @@
|
||||
#include "kai/ukernels/matmul/pack/kai_lhs_quant_pack_qsi8d32p_f32_neon.h"
|
||||
#include "kai/ukernels/matmul/pack/kai_lhs_quant_pack_qai8dxp_f32.h"
|
||||
|
||||
#include "kai/ukernels/matmul/kai_matmul.h"
|
||||
#include "kai/ukernels/matmul/kai_matmul_pack_lhs.h"
|
||||
#include "kai/ukernels/matmul/kai_matmul_pack_rhs.h"
|
||||
|
||||
#include "kai/ukernels/matmul/pack/kai_rhs_pack_kxn_bf16p2vlx2b_f32_x32_sme.h"
|
||||
#include "kai/ukernels/matmul/pack/kai_rhs_pack_nxk_f32p2vlx1biasf32_f32_f32_sme.h"
|
||||
#include "kai/ukernels/matmul/pack/kai_rhs_pack_nxk_qsi4c32pscalef16_qsu4c32s16s0.h"
|
||||
@@ -42,8 +48,6 @@
|
||||
|
||||
#include "kai/kai_common.h"
|
||||
|
||||
#include "simd-mappings.h"
|
||||
|
||||
#define GGML_COMMON_DECL_CPP
|
||||
#include "ggml-common.h"
|
||||
|
||||
@@ -154,16 +158,6 @@ static inline size_t rhs_ps_fn2(size_t n, size_t k, size_t /*nr*/, size_t /*kr*/
|
||||
return Fn(n, k);
|
||||
}
|
||||
|
||||
template<size_t(*Fn)(size_t,size_t,size_t,size_t)>
|
||||
static inline size_t rhs_stride_fn4(size_t k, size_t nr, size_t kr, size_t bl) {
|
||||
return Fn(k, nr, kr, bl);
|
||||
}
|
||||
|
||||
template<size_t(*Fn)(size_t)>
|
||||
static inline size_t rhs_stride_fn1(size_t k, size_t /*nr*/, size_t /*kr*/, size_t /*bl*/) {
|
||||
return Fn(k);
|
||||
}
|
||||
|
||||
template<void(*Fn)(size_t,size_t,size_t,size_t,size_t,size_t,size_t,const uint8_t*,const float*,void*,size_t,const struct kai_rhs_pack_qs4cxs1s0_param*)>
|
||||
static inline void rhs_pack_fn12(size_t num_groups, size_t n, size_t k, size_t nr, size_t kr, size_t sr, size_t bl,
|
||||
size_t /*rhs_stride*/, const void* rhs, const void* bias, const void* /*scale*/,
|
||||
@@ -194,137 +188,205 @@ static inline void rhs_pack_fn13(size_t num_groups, size_t n, size_t k, size_t n
|
||||
Fn(num_groups, n, k, nr, kr, sr, rhs_stride, rhs, bias, scale, rhs_packed, extra_bytes, params);
|
||||
}
|
||||
|
||||
static const size_t INT4_PER_BYTE = 2;
|
||||
static const size_t INT4_BITS = 4;
|
||||
static const int Q4_0_ZERO_POINT = 8;
|
||||
const size_t INT4_PER_UINT16 = 4;
|
||||
static constexpr size_t KLEIDIAI_Q4_K_BLOCK_LENGTH = QK_K / 8;
|
||||
|
||||
static void dequantize_row_qsi4c32pscalef16(
|
||||
const void *packed_data,
|
||||
int32_t row_idx,
|
||||
int64_t nc,
|
||||
float *out,
|
||||
size_t nr_pack,
|
||||
size_t packed_row_stride,
|
||||
size_t kr,
|
||||
size_t bl,
|
||||
size_t num_bytes_multiplier
|
||||
) {
|
||||
size_t group_idx = row_idx / nr_pack;
|
||||
size_t row_in_group = row_idx % nr_pack;
|
||||
const uint8_t *packed_group = (const uint8_t *)packed_data + group_idx * packed_row_stride;
|
||||
size_t num_blocks = nc / bl;
|
||||
const uint8_t *block_ptr = packed_group;
|
||||
static_assert(QK_K == 256, "KleidiAI Q4_K kernels require 256-value super-blocks");
|
||||
static_assert(sizeof(block_q4_K) == 144, "Unexpected ggml Q4_K block layout");
|
||||
static_assert(offsetof(block_q4_K, scales) == 4, "Unexpected ggml Q4_K metadata offset");
|
||||
static_assert(offsetof(block_q4_K, qs) == 16, "Unexpected ggml Q4_K data offset");
|
||||
|
||||
for (size_t b = 0; b < num_blocks; ++b) {
|
||||
uint16_t scale_f16 = *((const uint16_t *)(block_ptr + row_in_group * num_bytes_multiplier));
|
||||
float scale = GGML_CPU_FP16_TO_FP32(scale_f16);
|
||||
|
||||
const uint8_t *segment_ptr = block_ptr + nr_pack * num_bytes_multiplier;
|
||||
size_t num_segments = bl / kr;
|
||||
size_t num_bytes_per_segment = kr / INT4_PER_BYTE;
|
||||
|
||||
for (size_t s = 0; s < num_segments; ++s) {
|
||||
const uint8_t *seg_base = segment_ptr + s * nr_pack * num_bytes_per_segment;
|
||||
const uint8_t *qbytes = seg_base + row_in_group * num_bytes_per_segment;
|
||||
for (size_t k = 0; k < num_bytes_per_segment; ++k) {
|
||||
uint8_t byte = qbytes[k] ^ 0x88;
|
||||
int x0 = (byte & 0x0F) - Q4_0_ZERO_POINT;
|
||||
int x1 = (byte >> INT4_BITS) - Q4_0_ZERO_POINT;
|
||||
out[b * bl + s * num_bytes_per_segment + k] = x0 * scale;
|
||||
out[b * bl + s * num_bytes_per_segment + k + bl/2] = x1 * scale;
|
||||
}
|
||||
}
|
||||
block_ptr += nr_pack * num_bytes_multiplier + num_segments * nr_pack * num_bytes_per_segment;
|
||||
}
|
||||
static struct kai_matmul_uker_config q4_k_matmul_config() {
|
||||
struct kai_matmul_uker_config config = {};
|
||||
config.format.bl = KLEIDIAI_Q4_K_BLOCK_LENGTH;
|
||||
return config;
|
||||
}
|
||||
|
||||
static void dequantize_row_qsi4c32ps1s0scalef16(
|
||||
const void *packed_data,
|
||||
int32_t row_idx,
|
||||
int64_t k,
|
||||
float *out,
|
||||
size_t nr,
|
||||
size_t packed_row_stride,
|
||||
size_t kr,
|
||||
size_t bl,
|
||||
size_t num_bytes_multiplier
|
||||
) {
|
||||
const size_t num_blocks = k / bl;
|
||||
const size_t bl4 = bl / INT4_PER_UINT16;
|
||||
|
||||
size_t group_idx = row_idx / nr;
|
||||
size_t row_in_group = row_idx % nr;
|
||||
|
||||
const uint8_t *packed_group = (const uint8_t *)packed_data + group_idx * packed_row_stride;
|
||||
const uint16_t *qdata = (const uint16_t *)packed_group;
|
||||
const uint16_t *scales = (const uint16_t *)(packed_group + packed_row_stride - (nr * num_blocks * num_bytes_multiplier));
|
||||
|
||||
for (size_t block_idx = 0; block_idx < num_blocks; ++block_idx) {
|
||||
uint16_t scale_f16 = scales[row_in_group + block_idx * nr];
|
||||
float scale = GGML_CPU_FP16_TO_FP32(scale_f16);
|
||||
|
||||
for (size_t bl4_idx = 0; bl4_idx < bl4; ++bl4_idx) {
|
||||
uint16_t q = qdata[(block_idx * bl4 + bl4_idx) * nr + row_in_group];
|
||||
|
||||
for (size_t qidx = 0; qidx < INT4_PER_UINT16; ++qidx) {
|
||||
int v = ((q >> (qidx * 4)) & 0xF) - Q4_0_ZERO_POINT;
|
||||
out[block_idx * bl + bl4_idx * INT4_BITS + qidx] = v * scale;
|
||||
}
|
||||
}
|
||||
}
|
||||
GGML_UNUSED(kr);
|
||||
template<struct kai_matmul_uker_api (*GetApi)(void)>
|
||||
static size_t q4_k_get_m_step() {
|
||||
const struct kai_matmul_uker_api api = GetApi();
|
||||
const struct kai_matmul_uker_config config = q4_k_matmul_config();
|
||||
return api.get_step(&config).m;
|
||||
}
|
||||
|
||||
static void dequantize_row_qsi8cxp(
|
||||
const void *packed_data,
|
||||
int32_t row_idx,
|
||||
int64_t k,
|
||||
float *out,
|
||||
size_t nr,
|
||||
size_t packed_row_stride,
|
||||
size_t kr,
|
||||
size_t bl,
|
||||
size_t num_bytes_multiplier
|
||||
) {
|
||||
GGML_UNUSED(bl);
|
||||
GGML_UNUSED(num_bytes_multiplier);
|
||||
template<struct kai_matmul_uker_api (*GetApi)(void)>
|
||||
static size_t q4_k_get_n_step() {
|
||||
const struct kai_matmul_uker_api api = GetApi();
|
||||
const struct kai_matmul_uker_config config = q4_k_matmul_config();
|
||||
return api.get_step(&config).n;
|
||||
}
|
||||
|
||||
const size_t k_internal = ((size_t) k + QK8_0 - 1) / QK8_0 * QK8_0;
|
||||
const size_t group_idx = row_idx / nr;
|
||||
const size_t row_in_group = row_idx % nr;
|
||||
template<size_t Value>
|
||||
static size_t q4_k_get_constant() {
|
||||
return Value;
|
||||
}
|
||||
|
||||
const uint8_t * group_ptr = static_cast<const uint8_t *>(packed_data) + group_idx * packed_row_stride;
|
||||
const int8_t * data_base = reinterpret_cast<const int8_t *>(group_ptr);
|
||||
template<struct kai_matmul_uker_api (*GetApi)(void)>
|
||||
static size_t q4_k_get_dst_offset(size_t m_idx, size_t n_idx, size_t dst_stride) {
|
||||
const struct kai_matmul_uker_api api = GetApi();
|
||||
const struct kai_matmul_uker_config config = q4_k_matmul_config();
|
||||
const struct kai_matmul_uker_dst_dim_args index = { m_idx, n_idx };
|
||||
const struct kai_matmul_uker_dst_stride_args stride = { dst_stride };
|
||||
return api.get_dst_offset(&config, &index, &stride);
|
||||
}
|
||||
|
||||
const size_t num_blocks = k_internal / kr;
|
||||
template<struct kai_matmul_uker_api (*GetApi)(void)>
|
||||
static size_t q4_k_get_lhs_packed_offset(size_t m_idx, size_t k, size_t bl) {
|
||||
GGML_ASSERT(bl == KLEIDIAI_Q4_K_BLOCK_LENGTH);
|
||||
|
||||
for (size_t block = 0; block < num_blocks; ++block) {
|
||||
const int8_t * block_ptr = data_base + (block * nr + row_in_group) * kr;
|
||||
for (size_t i = 0; i < kr; ++i) {
|
||||
const size_t k_idx = block * kr + i;
|
||||
if (k_idx < (size_t) k) {
|
||||
out[k_idx] = static_cast<float>(block_ptr[i]);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const uint8_t * sums_ptr = group_ptr + nr * k_internal;
|
||||
GGML_UNUSED(sums_ptr);
|
||||
|
||||
const float * scale_ptr = reinterpret_cast<const float *>(sums_ptr + nr * sizeof(int32_t));
|
||||
const float scale = scale_ptr[row_in_group];
|
||||
|
||||
if (scale == 0.0f) {
|
||||
for (size_t i = 0; i < (size_t) k; ++i) {
|
||||
out[i] = 0.0f;
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
for (size_t i = 0; i < (size_t) k; ++i) {
|
||||
out[i] *= scale;
|
||||
const struct kai_matmul_uker_api api = GetApi();
|
||||
const struct kai_matmul_uker_config config = q4_k_matmul_config();
|
||||
const struct kai_matmul_uker_lhs_dim_args shape = { m_idx, k };
|
||||
const struct kai_matmul_uker_lhs_dim_args index = { m_idx, 0 };
|
||||
const struct kai_matmul_uker_lhs_stride_args stride = api.get_lhs_stride(&config, &shape);
|
||||
return api.get_lhs_offset(&config, &index, &stride);
|
||||
}
|
||||
|
||||
template<struct kai_matmul_uker_api (*GetApi)(void)>
|
||||
static size_t q4_k_get_rhs_packed_offset(size_t n_idx, size_t k, size_t bl) {
|
||||
GGML_ASSERT(bl == KLEIDIAI_Q4_K_BLOCK_LENGTH);
|
||||
|
||||
const struct kai_matmul_uker_api api = GetApi();
|
||||
const struct kai_matmul_uker_config config = q4_k_matmul_config();
|
||||
const struct kai_matmul_uker_rhs_dim_args shape = { n_idx, k };
|
||||
const struct kai_matmul_uker_rhs_dim_args index = { n_idx, 0 };
|
||||
const struct kai_matmul_uker_rhs_stride_args stride = api.get_rhs_stride(&config, &shape);
|
||||
return api.get_rhs_offset(&config, &index, &stride);
|
||||
}
|
||||
|
||||
template<struct kai_matmul_uker_api (*GetApi)(void)>
|
||||
static void q4_k_run_matmul(
|
||||
size_t m, size_t n, size_t k, size_t bl,
|
||||
const void * lhs_packed, const void * rhs_packed,
|
||||
void * dst, size_t dst_stride_row, size_t dst_stride_col,
|
||||
float clamp_min, float clamp_max) {
|
||||
GGML_ASSERT(bl == KLEIDIAI_Q4_K_BLOCK_LENGTH);
|
||||
GGML_ASSERT(dst_stride_col == sizeof(float));
|
||||
|
||||
const struct kai_matmul_uker_api api = GetApi();
|
||||
const struct kai_matmul_uker_config config = q4_k_matmul_config();
|
||||
const struct kai_matmul_uker_lhs_dim_args lhs_shape = { m, k };
|
||||
const struct kai_matmul_uker_rhs_dim_args rhs_shape = { n, k };
|
||||
|
||||
struct kai_matmul_uker_args args = {};
|
||||
args.flags = KAI_MATMUL_UKER_FLAGS_ARGS_CLAMP;
|
||||
args.shape = { m, n, k };
|
||||
args.operand.lhs.ptr = lhs_packed;
|
||||
args.operand.lhs.stride = api.get_lhs_stride(&config, &lhs_shape);
|
||||
args.operand.rhs.ptr = rhs_packed;
|
||||
args.operand.rhs.stride = api.get_rhs_stride(&config, &rhs_shape);
|
||||
args.operand.dst.ptr = dst;
|
||||
args.operand.dst.stride.m = dst_stride_row;
|
||||
args.activation.clamp.min_ptr = &clamp_min;
|
||||
args.activation.clamp.max_ptr = &clamp_max;
|
||||
api.run(&config, &args);
|
||||
}
|
||||
|
||||
static struct kai_matmul_pack_lhs_uker_config q4_k_lhs_pack_config() {
|
||||
struct kai_matmul_pack_lhs_uker_config config = {};
|
||||
return config;
|
||||
}
|
||||
|
||||
static size_t q4_k_lhs_get_offset(size_t m_idx, size_t lhs_stride) {
|
||||
const struct kai_matmul_pack_lhs_uker_api api = kai_matmul_pack_lhs_mxk_qsi8d32p1x4sf16_f32_neon();
|
||||
const struct kai_matmul_pack_lhs_uker_config config = q4_k_lhs_pack_config();
|
||||
const struct kai_matmul_pack_lhs_uker_lhs_dim_args index = { m_idx, 0 };
|
||||
const struct kai_matmul_pack_lhs_uker_lhs_stride_args stride = { lhs_stride };
|
||||
return api.get_lhs_offset(&config, &index, &stride);
|
||||
}
|
||||
|
||||
static size_t q4_k_lhs_get_packed_offset(
|
||||
size_t m_idx, size_t k, size_t bl, size_t mr, size_t kr, size_t sr) {
|
||||
GGML_ASSERT(bl == KLEIDIAI_Q4_K_BLOCK_LENGTH);
|
||||
GGML_ASSERT(mr == 1 && kr == 4 && sr == 2);
|
||||
|
||||
const struct kai_matmul_pack_lhs_uker_api api = kai_matmul_pack_lhs_mxk_qsi8d32p1x4sf16_f32_neon();
|
||||
const struct kai_matmul_pack_lhs_uker_config config = q4_k_lhs_pack_config();
|
||||
const struct kai_matmul_pack_lhs_uker_lhs_packed_dim_args shape = { m_idx, k };
|
||||
const struct kai_matmul_pack_lhs_uker_lhs_packed_dim_args index = { m_idx, 0 };
|
||||
const struct kai_matmul_pack_lhs_uker_lhs_packed_stride_args stride = api.get_lhs_packed_stride(&config, &shape);
|
||||
return api.get_lhs_packed_offset(&config, &index, &stride);
|
||||
}
|
||||
|
||||
static size_t q4_k_lhs_get_packed_size(
|
||||
size_t m, size_t k, size_t bl, size_t mr, size_t kr, size_t sr) {
|
||||
GGML_ASSERT(bl == KLEIDIAI_Q4_K_BLOCK_LENGTH);
|
||||
GGML_ASSERT(mr == 1 && kr == 4 && sr == 2);
|
||||
|
||||
const struct kai_matmul_pack_lhs_uker_api api = kai_matmul_pack_lhs_mxk_qsi8d32p1x4sf16_f32_neon();
|
||||
const struct kai_matmul_pack_lhs_uker_config config = q4_k_lhs_pack_config();
|
||||
const struct kai_matmul_pack_lhs_uker_lhs_packed_dim_args shape = { m, k };
|
||||
const struct kai_matmul_pack_lhs_uker_lhs_packed_stride_args stride = api.get_lhs_packed_stride(&config, &shape);
|
||||
return api.get_lhs_packed_size(&config, &shape, &stride);
|
||||
}
|
||||
|
||||
static void q4_k_lhs_pack(
|
||||
size_t m, size_t k, size_t bl, size_t mr, size_t kr, size_t sr,
|
||||
size_t m_idx_start, const void * lhs, size_t lhs_stride, void * lhs_packed) {
|
||||
GGML_ASSERT(bl == KLEIDIAI_Q4_K_BLOCK_LENGTH);
|
||||
GGML_ASSERT(mr == 1 && kr == 4 && sr == 2);
|
||||
GGML_ASSERT(m_idx_start == 0);
|
||||
|
||||
const struct kai_matmul_pack_lhs_uker_api api = kai_matmul_pack_lhs_mxk_qsi8d32p1x4sf16_f32_neon();
|
||||
const struct kai_matmul_pack_lhs_uker_config config = q4_k_lhs_pack_config();
|
||||
const struct kai_matmul_pack_lhs_uker_lhs_packed_dim_args packed_shape = { m, k };
|
||||
|
||||
struct kai_matmul_pack_lhs_uker_args args = {};
|
||||
args.flags = 0;
|
||||
args.shape = { m, k };
|
||||
args.operand.lhs.ptr = lhs;
|
||||
args.operand.lhs.stride.m = lhs_stride;
|
||||
args.operand.lhs_packed.ptr = lhs_packed;
|
||||
args.operand.lhs_packed.stride = api.get_lhs_packed_stride(&config, &packed_shape);
|
||||
api.run(&config, &args);
|
||||
}
|
||||
|
||||
static struct kai_matmul_pack_rhs_uker_config q4_k_rhs_pack_config() {
|
||||
struct kai_matmul_pack_rhs_uker_config config = {};
|
||||
return config;
|
||||
}
|
||||
|
||||
static size_t q4_k_rhs_get_packed_size(size_t n, size_t k, size_t nr, size_t /*kr*/, size_t bl) {
|
||||
GGML_ASSERT(bl == KLEIDIAI_Q4_K_BLOCK_LENGTH);
|
||||
|
||||
const struct kai_matmul_pack_rhs_uker_api api =
|
||||
kai_matmul_pack_rhs_nxk_qai4c32p16vsx4s1s0sf16_qai4c32k256sf16s32s0_sme();
|
||||
const struct kai_matmul_pack_rhs_uker_config config = q4_k_rhs_pack_config();
|
||||
GGML_ASSERT(api.get_step(&config).n == nr);
|
||||
const struct kai_matmul_pack_rhs_uker_rhs_packed_dim_args shape = { n, k };
|
||||
const struct kai_matmul_pack_rhs_uker_rhs_packed_stride_args stride = api.get_rhs_packed_stride(&config, &shape);
|
||||
return api.get_rhs_packed_size(&config, &shape, &stride);
|
||||
}
|
||||
|
||||
static void q4_k_rhs_pack(
|
||||
size_t num_groups, size_t n, size_t k, size_t nr, size_t /*kr*/, size_t /*sr*/, size_t bl,
|
||||
size_t rhs_stride, const void * rhs, const void * bias, const void * scale,
|
||||
void * rhs_packed, size_t extra_bytes, const void * params) {
|
||||
GGML_ASSERT(num_groups == 1);
|
||||
GGML_ASSERT(bl == KLEIDIAI_Q4_K_BLOCK_LENGTH);
|
||||
GGML_ASSERT(bias == nullptr);
|
||||
GGML_ASSERT(scale == nullptr);
|
||||
GGML_ASSERT(extra_bytes == 0);
|
||||
GGML_ASSERT(params == nullptr);
|
||||
|
||||
const struct kai_matmul_pack_rhs_uker_api api =
|
||||
kai_matmul_pack_rhs_nxk_qai4c32p16vsx4s1s0sf16_qai4c32k256sf16s32s0_sme();
|
||||
const struct kai_matmul_pack_rhs_uker_config config = q4_k_rhs_pack_config();
|
||||
GGML_ASSERT(api.get_step(&config).n == nr);
|
||||
const struct kai_matmul_pack_rhs_uker_rhs_dim_args rhs_shape = { n, k };
|
||||
const struct kai_matmul_pack_rhs_uker_rhs_packed_dim_args packed_shape = { n, k };
|
||||
|
||||
struct kai_matmul_pack_rhs_uker_args args = {};
|
||||
args.flags = 0;
|
||||
args.shape = { n, k };
|
||||
args.operand.rhs.ptr = rhs;
|
||||
args.operand.rhs.stride = api.get_rhs_stride(&config, &rhs_shape);
|
||||
if (rhs_stride != 0) {
|
||||
GGML_ASSERT(rhs_stride >= args.operand.rhs.stride.n);
|
||||
args.operand.rhs.stride.n = rhs_stride;
|
||||
}
|
||||
args.operand.rhs_packed.ptr = rhs_packed;
|
||||
args.operand.rhs_packed.stride = api.get_rhs_packed_stride(&config, &packed_shape);
|
||||
api.run(&config, &args);
|
||||
}
|
||||
|
||||
static ggml_kleidiai_kernels gemm_gemv_kernels[] = {
|
||||
@@ -338,7 +400,6 @@ static ggml_kleidiai_kernels gemm_gemv_kernels[] = {
|
||||
/* .get_kr = */ kai_get_kr_matmul_clamp_f32_f16p1vlx2_qsi4c32p4vlx2_1vlx4vl_sme2_mopa,
|
||||
/* .get_sr = */ kai_get_sr_matmul_clamp_f32_f16p1vlx2_qsi4c32p4vlx2_1vlx4vl_sme2_mopa,
|
||||
/* .get_dst_offset = */ kai_get_dst_offset_matmul_clamp_f32_f16p1vlx2_qsi4c32p4vlx2_1vlx4vl_sme2_mopa,
|
||||
/* .get_dst_size = */ kai_get_dst_size_matmul_clamp_f32_f16p1vlx2_qsi4c32p4vlx2_1vlx4vl_sme2_mopa,
|
||||
/* .get_lhs_offset_ex = */ &kernel_offs_fn3<kai_get_lhs_packed_offset_matmul_clamp_f32_f16p1vlx2_qsi4c32p4vlx2_1vlx4vl_sme2_mopa>,
|
||||
/* .get_rhs_packed_offset_ex = */ &kernel_offs_fn3<kai_get_rhs_packed_offset_matmul_clamp_f32_f16p1vlx2_qsi4c32p4vlx2_1vlx4vl_sme2_mopa>,
|
||||
/* .run_kernel_ex = */ &kernel_run_fn11<kai_run_matmul_clamp_f32_f16p1vlx2_qsi4c32p4vlx2_1vlx4vl_sme2_mopa>,
|
||||
@@ -359,7 +420,6 @@ static ggml_kleidiai_kernels gemm_gemv_kernels[] = {
|
||||
/* .get_kr = */ kai_get_kr_matmul_clamp_f32_qsi8d32p1x4_qsi4c32p4vlx4_1x4vl_sme2_sdot,
|
||||
/* .get_sr = */ kai_get_sr_matmul_clamp_f32_qsi8d32p1x4_qsi4c32p4vlx4_1x4vl_sme2_sdot,
|
||||
/* .get_dst_offset = */ kai_get_dst_offset_matmul_clamp_f32_qsi8d32p1x4_qsi4c32p4vlx4_1x4vl_sme2_sdot,
|
||||
/* .get_dst_size = */ kai_get_dst_size_matmul_clamp_f32_qsi8d32p1x4_qsi4c32p4vlx4_1x4vl_sme2_sdot,
|
||||
/* .get_lhs_offset_ex = */ &kernel_offs_fn3<kai_get_lhs_packed_offset_matmul_clamp_f32_qsi8d32p1x4_qsi4c32p4vlx4_1x4vl_sme2_sdot>,
|
||||
/* .get_rhs_packed_offset_ex = */ &kernel_offs_fn3<kai_get_rhs_packed_offset_matmul_clamp_f32_qsi8d32p1x4_qsi4c32p4vlx4_1x4vl_sme2_sdot>,
|
||||
/* .run_kernel_ex = */ &kernel_run_fn11<kai_run_matmul_clamp_f32_qsi8d32p1x4_qsi4c32p4vlx4_1x4vl_sme2_sdot>,
|
||||
@@ -371,10 +431,7 @@ static ggml_kleidiai_kernels gemm_gemv_kernels[] = {
|
||||
/* .pack_func_ex = */ &lhs_pack_float_fn10<kai_run_lhs_quant_pack_qsi8d32p_f32_neon>,
|
||||
},
|
||||
/* .rhs_info = */ {
|
||||
/* .packed_stride = */ kai_get_rhs_packed_stride_rhs_pack_nxk_qsi4c32ps1s0scalef16_qsu4c32s16s0_neon,
|
||||
/* .to_float = */ dequantize_row_qsi4c32ps1s0scalef16,
|
||||
/* .packed_size_ex = */ &rhs_ps_fn5<kai_get_rhs_packed_size_rhs_pack_nxk_qsi4c32ps1s0scalef16_qsu4c32s16s0_neon>,
|
||||
/* .packed_stride_ex = */ &rhs_stride_fn4<kai_get_rhs_packed_stride_rhs_pack_nxk_qsi4c32ps1s0scalef16_qsu4c32s16s0_neon>,
|
||||
/* .pack_func_ex = */ &rhs_pack_fn12<kai_run_rhs_pack_nxk_qsi4c32ps1s0scalef16_qsu4c32s16s0_neon>,
|
||||
},
|
||||
/* .required_cpu = */ CPU_FEATURE_SME2 | CPU_FEATURE_FP16,
|
||||
@@ -392,7 +449,6 @@ static ggml_kleidiai_kernels gemm_gemv_kernels[] = {
|
||||
/* .get_kr = */ kai_get_kr_matmul_clamp_f32_bf16p2vlx2_bf16p2vlx2_2vlx2vl_sme2_mopa,
|
||||
/* .get_sr = */ kai_get_sr_matmul_clamp_f32_bf16p2vlx2_bf16p2vlx2_2vlx2vl_sme2_mopa,
|
||||
/* .get_dst_offset = */ kai_get_dst_offset_matmul_clamp_f32_bf16p2vlx2_bf16p2vlx2_2vlx2vl_sme2_mopa,
|
||||
/* .get_dst_size = */ kai_get_dst_size_matmul_clamp_f32_bf16p2vlx2_bf16p2vlx2_2vlx2vl_sme2_mopa,
|
||||
/* .get_lhs_offset_ex = */ &kernel_offs_fn2<kai_get_lhs_packed_offset_matmul_clamp_f32_bf16p2vlx2_bf16p2vlx2_2vlx2vl_sme2_mopa>,
|
||||
/* .get_rhs_packed_offset_ex = */ &kernel_offs_fn2<kai_get_rhs_packed_offset_matmul_clamp_f32_bf16p2vlx2_bf16p2vlx2_2vlx2vl_sme2_mopa>,
|
||||
/* .run_kernel_ex = */ &kernel_run_fn10<kai_run_matmul_clamp_f32_bf16p2vlx2_bf16p2vlx2_2vlx2vl_sme2_mopa>,
|
||||
@@ -412,7 +468,6 @@ static ggml_kleidiai_kernels gemm_gemv_kernels[] = {
|
||||
/* .get_kr = */ kai_get_kr_matmul_clamp_f32_bf16p2vlx2_bf16p2vlx2_2vlx2vl_sme2_mopa,
|
||||
/* .get_sr = */ kai_get_sr_matmul_clamp_f32_bf16p2vlx2_bf16p2vlx2_2vlx2vl_sme2_mopa,
|
||||
/* .get_dst_offset = */ kai_get_dst_offset_matmul_clamp_f32_bf16p2vlx2_bf16p2vlx2_2vlx2vl_sme2_mopa,
|
||||
/* .get_dst_size = */ kai_get_dst_size_matmul_clamp_f32_bf16p2vlx2_bf16p2vlx2_2vlx2vl_sme2_mopa,
|
||||
/* .get_lhs_offset_ex = */ nullptr,
|
||||
/* .get_rhs_packed_offset_ex = */ nullptr,
|
||||
/* .run_kernel_ex = */ nullptr,
|
||||
@@ -424,10 +479,7 @@ static ggml_kleidiai_kernels gemm_gemv_kernels[] = {
|
||||
/* .pack_func_ex = */ &lhs_pack_void_fn9<kai_run_lhs_pack_bf16p2vlx2_f32_sme>,
|
||||
},
|
||||
/* .rhs_info = */ {
|
||||
/* .packed_stride = */ nullptr,
|
||||
/* .to_float = */ nullptr,
|
||||
/* .packed_size_ex = */ &rhs_ps_fn2<kai_get_rhs_packed_size_rhs_pack_kxn_bf16p2vlx2b_f32_x32_sme>,
|
||||
/* .packed_stride_ex = */ &rhs_stride_fn1<kai_get_rhs_packed_stride_rhs_pack_kxn_bf16p2vlx2b_f32_x32_sme>,
|
||||
/* .pack_func_ex = */ &rhs_pack_fn13<kai_run_rhs_pack_kxn_bf16p2vlx2b_f32_x32_sme>,
|
||||
},
|
||||
/* .required_cpu = */ CPU_FEATURE_SME2,
|
||||
@@ -446,7 +498,6 @@ static ggml_kleidiai_kernels gemm_gemv_kernels[] = {
|
||||
/* .get_kr = */ kai_get_kr_matmul_clamp_f32_qsi8d32p4x4_qsi4c32p4x4_16x4_neon_dotprod,
|
||||
/* .get_sr = */ kai_get_sr_matmul_clamp_f32_qsi8d32p4x4_qsi4c32p4x4_16x4_neon_dotprod,
|
||||
/* .get_dst_offset = */ kai_get_dst_offset_matmul_clamp_f32_qsi8d32p4x4_qsi4c32p4x4_16x4_neon_dotprod,
|
||||
/* .get_dst_size = */ kai_get_dst_size_matmul_clamp_f32_qsi8d32p4x4_qsi4c32p4x4_16x4_neon_dotprod,
|
||||
/* .get_lhs_offset_ex = */ &kernel_offs_fn3<kai_get_lhs_packed_offset_matmul_clamp_f32_qsi8d32p4x4_qsi4c32p4x4_16x4_neon_dotprod>,
|
||||
/* .get_rhs_packed_offset_ex = */ &kernel_offs_fn3<kai_get_rhs_packed_offset_matmul_clamp_f32_qsi8d32p4x4_qsi4c32p4x4_16x4_neon_dotprod>,
|
||||
/* .run_kernel_ex = */ &kernel_run_fn11<kai_run_matmul_clamp_f32_qsi8d32p4x4_qsi4c32p4x4_16x4_neon_dotprod>,
|
||||
@@ -466,7 +517,6 @@ static ggml_kleidiai_kernels gemm_gemv_kernels[] = {
|
||||
/* .get_kr = */ kai_get_kr_matmul_clamp_f32_qsi8d32p1x4_qsi4c32p4x4_1x4_neon_dotprod,
|
||||
/* .get_sr = */ kai_get_sr_matmul_clamp_f32_qsi8d32p1x4_qsi4c32p4x4_1x4_neon_dotprod,
|
||||
/* .get_dst_offset = */ kai_get_dst_offset_matmul_clamp_f32_qsi8d32p1x4_qsi4c32p4x4_1x4_neon_dotprod,
|
||||
/* .get_dst_size = */ kai_get_dst_size_matmul_clamp_f32_qsi8d32p1x4_qsi4c32p4x4_1x4_neon_dotprod,
|
||||
/* .get_lhs_offset_ex = */ &kernel_offs_fn3<kai_get_lhs_packed_offset_matmul_clamp_f32_qsi8d32p1x4_qsi4c32p4x4_1x4_neon_dotprod>,
|
||||
/* .get_rhs_packed_offset_ex = */ &kernel_offs_fn3<kai_get_rhs_packed_offset_matmul_clamp_f32_qsi8d32p1x4_qsi4c32p4x4_1x4_neon_dotprod>,
|
||||
/* .run_kernel_ex = */ &kernel_run_fn11<kai_run_matmul_clamp_f32_qsi8d32p1x4_qsi4c32p4x4_1x4_neon_dotprod>,
|
||||
@@ -478,10 +528,7 @@ static ggml_kleidiai_kernels gemm_gemv_kernels[] = {
|
||||
/* .pack_func_ex = */ &lhs_pack_float_fn10<kai_run_lhs_quant_pack_qsi8d32p_f32>,
|
||||
},
|
||||
/* .rhs_info = */ {
|
||||
/* .packed_stride = */ kai_get_rhs_packed_stride_rhs_pack_nxk_qsi4c32pscalef16_qsu4c32s16s0,
|
||||
/* .to_float = */ dequantize_row_qsi4c32pscalef16,
|
||||
/* .packed_size_ex = */ &rhs_ps_fn5<kai_get_rhs_packed_size_rhs_pack_nxk_qsi4c32pscalef16_qsu4c32s16s0>,
|
||||
/* .packed_stride_ex = */ &rhs_stride_fn4<kai_get_rhs_packed_stride_rhs_pack_nxk_qsi4c32pscalef16_qsu4c32s16s0>,
|
||||
/* .pack_func_ex = */ &rhs_pack_fn12<kai_run_rhs_pack_nxk_qsi4c32pscalef16_qsu4c32s16s0>,
|
||||
},
|
||||
/* .required_cpu = */ CPU_FEATURE_DOTPROD,
|
||||
@@ -499,7 +546,6 @@ static ggml_kleidiai_kernels gemm_gemv_kernels[] = {
|
||||
/* .get_kr = */ kai_get_kr_matmul_clamp_f32_qsi8d32p4x8_qsi4c32p4x8_16x4_neon_i8mm,
|
||||
/* .get_sr = */ kai_get_sr_matmul_clamp_f32_qsi8d32p4x8_qsi4c32p4x8_16x4_neon_i8mm,
|
||||
/* .get_dst_offset = */ kai_get_dst_offset_matmul_clamp_f32_qsi8d32p4x8_qsi4c32p4x8_16x4_neon_i8mm,
|
||||
/* .get_dst_size = */ kai_get_dst_size_matmul_clamp_f32_qsi8d32p4x8_qsi4c32p4x8_16x4_neon_i8mm,
|
||||
/* .get_lhs_offset_ex = */ &kernel_offs_fn3<kai_get_lhs_packed_offset_matmul_clamp_f32_qsi8d32p4x8_qsi4c32p4x8_16x4_neon_i8mm>,
|
||||
/* .get_rhs_packed_offset_ex = */ &kernel_offs_fn3<kai_get_rhs_packed_offset_matmul_clamp_f32_qsi8d32p4x8_qsi4c32p4x8_16x4_neon_i8mm>,
|
||||
/* .run_kernel_ex = */ &kernel_run_fn11<kai_run_matmul_clamp_f32_qsi8d32p4x8_qsi4c32p4x8_16x4_neon_i8mm>,
|
||||
@@ -519,7 +565,6 @@ static ggml_kleidiai_kernels gemm_gemv_kernels[] = {
|
||||
/* .get_kr = */ kai_get_kr_matmul_clamp_f32_qsi8d32p1x8_qsi4c32p4x8_1x4x32_neon_dotprod,
|
||||
/* .get_sr = */ kai_get_sr_matmul_clamp_f32_qsi8d32p1x8_qsi4c32p4x8_1x4x32_neon_dotprod,
|
||||
/* .get_dst_offset = */ kai_get_dst_offset_matmul_clamp_f32_qsi8d32p1x8_qsi4c32p4x8_1x4x32_neon_dotprod,
|
||||
/* .get_dst_size = */ kai_get_dst_size_matmul_clamp_f32_qsi8d32p1x8_qsi4c32p4x8_1x4x32_neon_dotprod,
|
||||
/* .get_lhs_offset_ex = */ &kernel_offs_fn3<kai_get_lhs_packed_offset_matmul_clamp_f32_qsi8d32p1x8_qsi4c32p4x8_1x4x32_neon_dotprod>,
|
||||
/* .get_rhs_packed_offset_ex = */ &kernel_offs_fn3<kai_get_rhs_packed_offset_matmul_clamp_f32_qsi8d32p1x8_qsi4c32p4x8_1x4x32_neon_dotprod>,
|
||||
/* .run_kernel_ex = */ &kernel_run_fn11<kai_run_matmul_clamp_f32_qsi8d32p1x8_qsi4c32p4x8_1x4x32_neon_dotprod>,
|
||||
@@ -531,10 +576,7 @@ static ggml_kleidiai_kernels gemm_gemv_kernels[] = {
|
||||
/* .pack_func_ex = */ &lhs_pack_float_fn10<kai_run_lhs_quant_pack_qsi8d32p_f32>,
|
||||
},
|
||||
/* .rhs_info = */ {
|
||||
/* .packed_stride = */ kai_get_rhs_packed_stride_rhs_pack_nxk_qsi4c32pscalef16_qsu4c32s16s0,
|
||||
/* .to_float = */ dequantize_row_qsi4c32pscalef16,
|
||||
/* .packed_size_ex = */ &rhs_ps_fn5<kai_get_rhs_packed_size_rhs_pack_nxk_qsi4c32pscalef16_qsu4c32s16s0>,
|
||||
/* .packed_stride_ex = */ &rhs_stride_fn4<kai_get_rhs_packed_stride_rhs_pack_nxk_qsi4c32pscalef16_qsu4c32s16s0>,
|
||||
/* .pack_func_ex = */ &rhs_pack_fn12<kai_run_rhs_pack_nxk_qsi4c32pscalef16_qsu4c32s16s0>,
|
||||
},
|
||||
/* .required_cpu = */ CPU_FEATURE_I8MM | CPU_FEATURE_DOTPROD,
|
||||
@@ -553,7 +595,6 @@ static ggml_kleidiai_kernels gemm_gemv_kernels[] = {
|
||||
/* .get_kr = */ kai_get_kr_matmul_clamp_f32_qsi8d32p4x8_qsi4c32p8x8_16x8_sve_i8mm,
|
||||
/* .get_sr = */ kai_get_sr_matmul_clamp_f32_qsi8d32p4x8_qsi4c32p8x8_16x8_sve_i8mm,
|
||||
/* .get_dst_offset = */ kai_get_dst_offset_matmul_clamp_f32_qsi8d32p4x8_qsi4c32p8x8_16x8_sve_i8mm,
|
||||
/* .get_dst_size = */ kai_get_dst_size_matmul_clamp_f32_qsi8d32p4x8_qsi4c32p8x8_16x8_sve_i8mm,
|
||||
/* .get_lhs_offset_ex = */ &kernel_offs_fn3<kai_get_lhs_packed_offset_matmul_clamp_f32_qsi8d32p4x8_qsi4c32p8x8_16x8_sve_i8mm>,
|
||||
/* .get_rhs_packed_offset_ex = */ &kernel_offs_fn3<kai_get_rhs_packed_offset_matmul_clamp_f32_qsi8d32p4x8_qsi4c32p8x8_16x8_sve_i8mm>,
|
||||
/* .run_kernel_ex = */ &kernel_run_fn11<kai_run_matmul_clamp_f32_qsi8d32p4x8_qsi4c32p8x8_16x8_sve_i8mm>,
|
||||
@@ -573,7 +614,6 @@ static ggml_kleidiai_kernels gemm_gemv_kernels[] = {
|
||||
/* .get_kr = */ kai_get_kr_matmul_clamp_f32_qsi8d32p1x8_qsi4c32p8x8_1x8_sve_dotprod,
|
||||
/* .get_sr = */ kai_get_sr_matmul_clamp_f32_qsi8d32p1x8_qsi4c32p8x8_1x8_sve_dotprod,
|
||||
/* .get_dst_offset = */ kai_get_dst_offset_matmul_clamp_f32_qsi8d32p1x8_qsi4c32p8x8_1x8_sve_dotprod,
|
||||
/* .get_dst_size = */ kai_get_dst_size_matmul_clamp_f32_qsi8d32p1x8_qsi4c32p8x8_1x8_sve_dotprod,
|
||||
/* .get_lhs_offset_ex = */ &kernel_offs_fn3<kai_get_lhs_packed_offset_matmul_clamp_f32_qsi8d32p1x8_qsi4c32p8x8_1x8_sve_dotprod>,
|
||||
/* .get_rhs_packed_offset_ex = */ &kernel_offs_fn3<kai_get_rhs_packed_offset_matmul_clamp_f32_qsi8d32p1x8_qsi4c32p8x8_1x8_sve_dotprod>,
|
||||
/* .run_kernel_ex = */ &kernel_run_fn11<kai_run_matmul_clamp_f32_qsi8d32p1x8_qsi4c32p8x8_1x8_sve_dotprod>,
|
||||
@@ -585,10 +625,7 @@ static ggml_kleidiai_kernels gemm_gemv_kernels[] = {
|
||||
/* .pack_func_ex = */ &lhs_pack_float_fn10<kai_run_lhs_quant_pack_qsi8d32p_f32>,
|
||||
},
|
||||
/* .rhs_info = */ {
|
||||
/* .packed_stride = */ kai_get_rhs_packed_stride_rhs_pack_nxk_qsi4c32pscalef16_qsu4c32s16s0,
|
||||
/* .to_float = */ dequantize_row_qsi4c32pscalef16,
|
||||
/* .packed_size_ex = */ &rhs_ps_fn5<kai_get_rhs_packed_size_rhs_pack_nxk_qsi4c32pscalef16_qsu4c32s16s0>,
|
||||
/* .packed_stride_ex = */ &rhs_stride_fn4<kai_get_rhs_packed_stride_rhs_pack_nxk_qsi4c32pscalef16_qsu4c32s16s0>,
|
||||
/* .pack_func_ex = */ &rhs_pack_fn12<kai_run_rhs_pack_nxk_qsi4c32pscalef16_qsu4c32s16s0>,
|
||||
},
|
||||
/* .required_cpu = */ CPU_FEATURE_SVE | CPU_FEATURE_I8MM | CPU_FEATURE_DOTPROD,
|
||||
@@ -606,7 +643,6 @@ static ggml_kleidiai_kernels gemm_gemv_kernels[] = {
|
||||
/* .get_kr = */ kai_get_kr_matmul_clamp_f32_qsi8d32p4x8_qsi4c32p4x8_16x4_neon_i8mm,
|
||||
/* .get_sr = */ kai_get_sr_matmul_clamp_f32_qsi8d32p4x8_qsi4c32p4x8_16x4_neon_i8mm,
|
||||
/* .get_dst_offset = */ kai_get_dst_offset_matmul_clamp_f32_qsi8d32p4x8_qsi4c32p4x8_16x4_neon_i8mm,
|
||||
/* .get_dst_size = */ kai_get_dst_size_matmul_clamp_f32_qsi8d32p4x8_qsi4c32p4x8_16x4_neon_i8mm,
|
||||
/* .get_lhs_offset_ex = */ &kernel_offs_fn3<kai_get_lhs_packed_offset_matmul_clamp_f32_qsi8d32p4x8_qsi4c32p4x8_16x4_neon_i8mm>,
|
||||
/* .get_rhs_packed_offset_ex = */ &kernel_offs_fn3<kai_get_rhs_packed_offset_matmul_clamp_f32_qsi8d32p4x8_qsi4c32p4x8_16x4_neon_i8mm>,
|
||||
/* .run_kernel_ex = */ &kernel_run_fn11<kai_run_matmul_clamp_f32_qsi8d32p4x8_qsi4c32p4x8_16x4_neon_i8mm>,
|
||||
@@ -626,7 +662,6 @@ static ggml_kleidiai_kernels gemm_gemv_kernels[] = {
|
||||
/* .get_kr = */ kai_get_kr_matmul_clamp_f32_qsi8d32p1x8_qsi4c32p4x8_1x4x32_neon_dotprod,
|
||||
/* .get_sr = */ kai_get_sr_matmul_clamp_f32_qsi8d32p1x8_qsi4c32p4x8_1x4x32_neon_dotprod,
|
||||
/* .get_dst_offset = */ kai_get_dst_offset_matmul_clamp_f32_qsi8d32p1x8_qsi4c32p4x8_1x4x32_neon_dotprod,
|
||||
/* .get_dst_size = */ kai_get_dst_size_matmul_clamp_f32_qsi8d32p1x8_qsi4c32p4x8_1x4x32_neon_dotprod,
|
||||
/* .get_lhs_offset_ex = */ &kernel_offs_fn3<kai_get_lhs_packed_offset_matmul_clamp_f32_qsi8d32p1x8_qsi4c32p4x8_1x4x32_neon_dotprod>,
|
||||
/* .get_rhs_packed_offset_ex = */ &kernel_offs_fn3<kai_get_rhs_packed_offset_matmul_clamp_f32_qsi8d32p1x8_qsi4c32p4x8_1x4x32_neon_dotprod>,
|
||||
/* .run_kernel_ex = */ &kernel_run_fn11<kai_run_matmul_clamp_f32_qsi8d32p1x8_qsi4c32p4x8_1x4x32_neon_dotprod>,
|
||||
@@ -638,10 +673,7 @@ static ggml_kleidiai_kernels gemm_gemv_kernels[] = {
|
||||
/* .pack_func_ex = */ &lhs_pack_float_fn10<kai_run_lhs_quant_pack_qsi8d32p_f32>,
|
||||
},
|
||||
/* .rhs_info = */ {
|
||||
/* .packed_stride = */ kai_get_rhs_packed_stride_rhs_pack_nxk_qsi4c32pscalef16_qsu4c32s16s0,
|
||||
/* .to_float = */ dequantize_row_qsi4c32pscalef16,
|
||||
/* .packed_size_ex = */ &rhs_ps_fn5<kai_get_rhs_packed_size_rhs_pack_nxk_qsi4c32pscalef16_qsu4c32s16s0>,
|
||||
/* .packed_stride_ex = */ &rhs_stride_fn4<kai_get_rhs_packed_stride_rhs_pack_nxk_qsi4c32pscalef16_qsu4c32s16s0>,
|
||||
/* .pack_func_ex = */ &rhs_pack_fn12<kai_run_rhs_pack_nxk_qsi4c32pscalef16_qsu4c32s16s0>,
|
||||
},
|
||||
/* .required_cpu = */ CPU_FEATURE_I8MM | CPU_FEATURE_DOTPROD,
|
||||
@@ -659,7 +691,6 @@ static ggml_kleidiai_kernels gemm_gemv_kernels[] = {
|
||||
/* .get_kr = */ kai_get_kr_matmul_clamp_f32_qsi8d32p4x4_qsi4c32p4x4_16x4_neon_dotprod,
|
||||
/* .get_sr = */ kai_get_sr_matmul_clamp_f32_qsi8d32p4x4_qsi4c32p4x4_16x4_neon_dotprod,
|
||||
/* .get_dst_offset = */ kai_get_dst_offset_matmul_clamp_f32_qsi8d32p4x4_qsi4c32p4x4_16x4_neon_dotprod,
|
||||
/* .get_dst_size = */ kai_get_dst_size_matmul_clamp_f32_qsi8d32p4x4_qsi4c32p4x4_16x4_neon_dotprod,
|
||||
/* .get_lhs_offset_ex = */ &kernel_offs_fn3<kai_get_lhs_packed_offset_matmul_clamp_f32_qsi8d32p4x4_qsi4c32p4x4_16x4_neon_dotprod>,
|
||||
/* .get_rhs_packed_offset_ex = */ &kernel_offs_fn3<kai_get_rhs_packed_offset_matmul_clamp_f32_qsi8d32p4x4_qsi4c32p4x4_16x4_neon_dotprod>,
|
||||
/* .run_kernel_ex = */ &kernel_run_fn11<kai_run_matmul_clamp_f32_qsi8d32p4x4_qsi4c32p4x4_16x4_neon_dotprod>,
|
||||
@@ -679,7 +710,6 @@ static ggml_kleidiai_kernels gemm_gemv_kernels[] = {
|
||||
/* .get_kr = */ kai_get_kr_matmul_clamp_f32_qsi8d32p1x4_qsi4c32p4x4_1x4_neon_dotprod,
|
||||
/* .get_sr = */ kai_get_sr_matmul_clamp_f32_qsi8d32p1x4_qsi4c32p4x4_1x4_neon_dotprod,
|
||||
/* .get_dst_offset = */ kai_get_dst_offset_matmul_clamp_f32_qsi8d32p1x4_qsi4c32p4x4_1x4_neon_dotprod,
|
||||
/* .get_dst_size = */ kai_get_dst_size_matmul_clamp_f32_qsi8d32p1x4_qsi4c32p4x4_1x4_neon_dotprod,
|
||||
/* .get_lhs_offset_ex = */ &kernel_offs_fn3<kai_get_lhs_packed_offset_matmul_clamp_f32_qsi8d32p1x4_qsi4c32p4x4_1x4_neon_dotprod>,
|
||||
/* .get_rhs_packed_offset_ex = */ &kernel_offs_fn3<kai_get_rhs_packed_offset_matmul_clamp_f32_qsi8d32p1x4_qsi4c32p4x4_1x4_neon_dotprod>,
|
||||
/* .run_kernel_ex = */ &kernel_run_fn11<kai_run_matmul_clamp_f32_qsi8d32p1x4_qsi4c32p4x4_1x4_neon_dotprod>,
|
||||
@@ -691,10 +721,7 @@ static ggml_kleidiai_kernels gemm_gemv_kernels[] = {
|
||||
/* .pack_func_ex = */ &lhs_pack_float_fn10<kai_run_lhs_quant_pack_qsi8d32p_f32>,
|
||||
},
|
||||
/* .rhs_info = */ {
|
||||
/* .packed_stride = */ kai_get_rhs_packed_stride_rhs_pack_nxk_qsi4c32pscalef16_qsu4c32s16s0,
|
||||
/* .to_float = */ dequantize_row_qsi4c32pscalef16,
|
||||
/* .packed_size_ex = */ &rhs_ps_fn5<kai_get_rhs_packed_size_rhs_pack_nxk_qsi4c32pscalef16_qsu4c32s16s0>,
|
||||
/* .packed_stride_ex = */ &rhs_stride_fn4<kai_get_rhs_packed_stride_rhs_pack_nxk_qsi4c32pscalef16_qsu4c32s16s0>,
|
||||
/* .pack_func_ex = */ &rhs_pack_fn12<kai_run_rhs_pack_nxk_qsi4c32pscalef16_qsu4c32s16s0>,
|
||||
},
|
||||
/* .required_cpu = */ CPU_FEATURE_DOTPROD,
|
||||
@@ -706,6 +733,59 @@ static ggml_kleidiai_kernels gemm_gemv_kernels[] = {
|
||||
{ /* Sentinel */ }
|
||||
};
|
||||
|
||||
static ggml_kleidiai_kernels gemm_gemv_kernels_q4_k[] = {
|
||||
{
|
||||
/* SME2 GEMM */
|
||||
{
|
||||
/* .get_m_step = */ &q4_k_get_m_step<kai_matmul_clamp_f32_f16p4vsx2_qai4c32p16vsx4s1s0sf16_4vsx16vs_sme2_mopa>,
|
||||
/* .get_n_step = */ &q4_k_get_n_step<kai_matmul_clamp_f32_f16p4vsx2_qai4c32p16vsx4s1s0sf16_4vsx16vs_sme2_mopa>,
|
||||
/* .get_mr = */ &q4_k_get_m_step<kai_matmul_clamp_f32_f16p4vsx2_qai4c32p16vsx4s1s0sf16_4vsx16vs_sme2_mopa>,
|
||||
/* .get_nr = */ &q4_k_get_n_step<kai_matmul_clamp_f32_f16p4vsx2_qai4c32p16vsx4s1s0sf16_4vsx16vs_sme2_mopa>,
|
||||
/* .get_kr = */ &q4_k_get_constant<4>,
|
||||
/* .get_sr = */ &q4_k_get_constant<2>,
|
||||
/* .get_dst_offset = */ &q4_k_get_dst_offset<kai_matmul_clamp_f32_f16p4vsx2_qai4c32p16vsx4s1s0sf16_4vsx16vs_sme2_mopa>,
|
||||
/* .get_lhs_offset_ex = */ &q4_k_get_lhs_packed_offset<kai_matmul_clamp_f32_f16p4vsx2_qai4c32p16vsx4s1s0sf16_4vsx16vs_sme2_mopa>,
|
||||
/* .get_rhs_packed_offset_ex = */ &q4_k_get_rhs_packed_offset<kai_matmul_clamp_f32_f16p4vsx2_qai4c32p16vsx4s1s0sf16_4vsx16vs_sme2_mopa>,
|
||||
/* .run_kernel_ex = */ &q4_k_run_matmul<kai_matmul_clamp_f32_f16p4vsx2_qai4c32p16vsx4s1s0sf16_4vsx16vs_sme2_mopa>,
|
||||
},
|
||||
{
|
||||
/* .get_offset = */ kai_get_lhs_offset_lhs_pack_f16pmrx2_f32_neon,
|
||||
/* .get_packed_offset_ex = */ &lhs_offs_fn6<kai_get_lhs_packed_offset_lhs_pack_f16pmrx2_f32_neon>,
|
||||
/* .packed_size_ex = */ &lhs_ps_fn6<kai_get_lhs_packed_size_lhs_pack_f16pmrx2_f32_neon>,
|
||||
/* .pack_func_ex = */ &lhs_pack_void_fn10<kai_run_lhs_pack_f16pmrx2_f32_neon>,
|
||||
},
|
||||
/* SME2 GEMV */
|
||||
{
|
||||
/* .get_m_step = */ &q4_k_get_m_step<kai_matmul_clamp_f32_qsi8d32p1x4_qai4c32p16vsx4s1s0sf16_1x16vs_sme2_dot>,
|
||||
/* .get_n_step = */ &q4_k_get_n_step<kai_matmul_clamp_f32_qsi8d32p1x4_qai4c32p16vsx4s1s0sf16_1x16vs_sme2_dot>,
|
||||
/* .get_mr = */ &q4_k_get_m_step<kai_matmul_clamp_f32_qsi8d32p1x4_qai4c32p16vsx4s1s0sf16_1x16vs_sme2_dot>,
|
||||
/* .get_nr = */ &q4_k_get_n_step<kai_matmul_clamp_f32_qsi8d32p1x4_qai4c32p16vsx4s1s0sf16_1x16vs_sme2_dot>,
|
||||
/* .get_kr = */ &q4_k_get_constant<4>,
|
||||
/* .get_sr = */ &q4_k_get_constant<2>,
|
||||
/* .get_dst_offset = */ &q4_k_get_dst_offset<kai_matmul_clamp_f32_qsi8d32p1x4_qai4c32p16vsx4s1s0sf16_1x16vs_sme2_dot>,
|
||||
/* .get_lhs_offset_ex = */ &q4_k_get_lhs_packed_offset<kai_matmul_clamp_f32_qsi8d32p1x4_qai4c32p16vsx4s1s0sf16_1x16vs_sme2_dot>,
|
||||
/* .get_rhs_packed_offset_ex = */ &q4_k_get_rhs_packed_offset<kai_matmul_clamp_f32_qsi8d32p1x4_qai4c32p16vsx4s1s0sf16_1x16vs_sme2_dot>,
|
||||
/* .run_kernel_ex = */ &q4_k_run_matmul<kai_matmul_clamp_f32_qsi8d32p1x4_qai4c32p16vsx4s1s0sf16_1x16vs_sme2_dot>,
|
||||
},
|
||||
{
|
||||
/* .get_offset = */ q4_k_lhs_get_offset,
|
||||
/* .get_packed_offset_ex = */ q4_k_lhs_get_packed_offset,
|
||||
/* .packed_size_ex = */ q4_k_lhs_get_packed_size,
|
||||
/* .pack_func_ex = */ q4_k_lhs_pack,
|
||||
},
|
||||
{
|
||||
/* .packed_size_ex = */ q4_k_rhs_get_packed_size,
|
||||
/* .pack_func_ex = */ q4_k_rhs_pack,
|
||||
/* .repack_mode = */ RHS_REPACK_SINGLE_ONLY,
|
||||
},
|
||||
/* .required_cpu = */ CPU_FEATURE_SME2 | CPU_FEATURE_FP16,
|
||||
/* .lhs_type = */ GGML_TYPE_F32,
|
||||
/* .rhs_type = */ GGML_TYPE_Q4_K,
|
||||
/* .op_type = */ GGML_TYPE_F32,
|
||||
},
|
||||
{ /* Sentinel */ }
|
||||
};
|
||||
|
||||
static ggml_kleidiai_kernels gemm_gemv_kernels_q8[] = {
|
||||
{
|
||||
/* SME2 GEMM */
|
||||
@@ -717,7 +797,6 @@ static ggml_kleidiai_kernels gemm_gemv_kernels_q8[] = {
|
||||
/* .get_kr = */ kai_get_kr_matmul_clamp_f32_qai8dxp1vlx4_qsi8cxp4vlx4_1vlx4vl_sme2_mopa,
|
||||
/* .get_sr = */ kai_get_sr_matmul_clamp_f32_qai8dxp1vlx4_qsi8cxp4vlx4_1vlx4vl_sme2_mopa,
|
||||
/* .get_dst_offset = */ kai_get_dst_offset_matmul_clamp_f32_qai8dxp1vlx4_qsi8cxp4vlx4_1vlx4vl_sme2_mopa,
|
||||
/* .get_dst_size = */ kai_get_dst_size_matmul_clamp_f32_qai8dxp1vlx4_qsi8cxp4vlx4_1vlx4vl_sme2_mopa,
|
||||
/* .get_lhs_offset_ex = */ &kernel_offs_fn2<kai_get_lhs_packed_offset_matmul_clamp_f32_qai8dxp1vlx4_qsi8cxp4vlx4_1vlx4vl_sme2_mopa>,
|
||||
/* .get_rhs_packed_offset_ex = */ &kernel_offs_fn2<kai_get_rhs_packed_offset_matmul_clamp_f32_qai8dxp1vlx4_qsi8cxp4vlx4_1vlx4vl_sme2_mopa>,
|
||||
/* .run_kernel_ex = */ &kernel_run_float_fn10<kai_run_matmul_clamp_f32_qai8dxp1vlx4_qsi8cxp4vlx4_1vlx4vl_sme2_mopa>,
|
||||
@@ -737,7 +816,6 @@ static ggml_kleidiai_kernels gemm_gemv_kernels_q8[] = {
|
||||
/* .get_kr = */ kai_get_kr_matmul_clamp_f32_qai8dxp1x4_qsi8cxp4vlx4_1x4vl_sme2_dot,
|
||||
/* .get_sr = */ kai_get_sr_matmul_clamp_f32_qai8dxp1x4_qsi8cxp4vlx4_1x4vl_sme2_dot,
|
||||
/* .get_dst_offset = */ kai_get_dst_offset_matmul_clamp_f32_qai8dxp1x4_qsi8cxp4vlx4_1x4vl_sme2_dot,
|
||||
/* .get_dst_size = */ kai_get_dst_size_matmul_clamp_f32_qai8dxp1x4_qsi8cxp4vlx4_1x4vl_sme2_dot,
|
||||
/* .get_lhs_offset_ex = */ &kernel_offs_fn2<kai_get_lhs_packed_offset_matmul_clamp_f32_qai8dxp1x4_qsi8cxp4vlx4_1x4vl_sme2_dot>,
|
||||
/* .get_rhs_packed_offset_ex = */ &kernel_offs_fn2<kai_get_rhs_packed_offset_matmul_clamp_f32_qai8dxp1x4_qsi8cxp4vlx4_1x4vl_sme2_dot>,
|
||||
/* .run_kernel_ex = */ &kernel_run_float_fn10<kai_run_matmul_clamp_f32_qai8dxp1x4_qsi8cxp4vlx4_1x4vl_sme2_dot>,
|
||||
@@ -749,10 +827,7 @@ static ggml_kleidiai_kernels gemm_gemv_kernels_q8[] = {
|
||||
/* .pack_func_ex = */ &lhs_pack_float_fn9_no_bl<kai_run_lhs_quant_pack_qai8dxp_f32>,
|
||||
},
|
||||
/* .rhs_info = */ {
|
||||
/* .packed_stride = */ kai_get_rhs_packed_stride_rhs_pack_nxk_qsi8cxp_qsi8cx_neon,
|
||||
/* .to_float = */ dequantize_row_qsi8cxp,
|
||||
/* .packed_size_ex = */ &rhs_ps_fn5<kai_get_rhs_packed_size_rhs_pack_nxk_qsi8cxp_qsi8cx_neon>,
|
||||
/* .packed_stride_ex = */ &rhs_stride_fn4<kai_get_rhs_packed_stride_rhs_pack_nxk_qsi8cxp_qsi8cx_neon>,
|
||||
/* .pack_func_ex = */ &rhs_pack_scale_fn12<kai_run_rhs_pack_nxk_qsi8cxp_qsi8cx_neon>,
|
||||
},
|
||||
/* .required_cpu = */ CPU_FEATURE_SME2,
|
||||
@@ -770,7 +845,6 @@ static ggml_kleidiai_kernels gemm_gemv_kernels_q8[] = {
|
||||
/* .get_kr = */ kai_get_kr_matmul_clamp_f32_qai8dxp1vlx4_qsi8cxp4vlx4_1vlx4vl_sme_mopa,
|
||||
/* .get_sr = */ kai_get_sr_matmul_clamp_f32_qai8dxp1vlx4_qsi8cxp4vlx4_1vlx4vl_sme_mopa,
|
||||
/* .get_dst_offset = */ kai_get_dst_offset_matmul_clamp_f32_qai8dxp1vlx4_qsi8cxp4vlx4_1vlx4vl_sme_mopa,
|
||||
/* .get_dst_size = */ kai_get_dst_size_matmul_clamp_f32_qai8dxp1vlx4_qsi8cxp4vlx4_1vlx4vl_sme_mopa,
|
||||
/* .get_lhs_offset_ex = */ &kernel_offs_fn2<kai_get_lhs_packed_offset_matmul_clamp_f32_qai8dxp1vlx4_qsi8cxp4vlx4_1vlx4vl_sme_mopa>,
|
||||
/* .get_rhs_packed_offset_ex = */ &kernel_offs_fn2<kai_get_rhs_packed_offset_matmul_clamp_f32_qai8dxp1vlx4_qsi8cxp4vlx4_1vlx4vl_sme_mopa>,
|
||||
/* .run_kernel_ex = */ &kernel_run_float_fn10<kai_run_matmul_clamp_f32_qai8dxp1vlx4_qsi8cxp4vlx4_1vlx4vl_sme_mopa>,
|
||||
@@ -790,7 +864,6 @@ static ggml_kleidiai_kernels gemm_gemv_kernels_q8[] = {
|
||||
/* .get_kr = */ kai_get_kr_matmul_clamp_f32_qai8dxp1x4_qsi8cxp4vlx4_1x4vl_sme_dot,
|
||||
/* .get_sr = */ kai_get_sr_matmul_clamp_f32_qai8dxp1x4_qsi8cxp4vlx4_1x4vl_sme_dot,
|
||||
/* .get_dst_offset = */ kai_get_dst_offset_matmul_clamp_f32_qai8dxp1x4_qsi8cxp4vlx4_1x4vl_sme_dot,
|
||||
/* .get_dst_size = */ kai_get_dst_size_matmul_clamp_f32_qai8dxp1x4_qsi8cxp4vlx4_1x4vl_sme_dot,
|
||||
/* .get_lhs_offset_ex = */ &kernel_offs_fn2<kai_get_lhs_packed_offset_matmul_clamp_f32_qai8dxp1x4_qsi8cxp4vlx4_1x4vl_sme_dot>,
|
||||
/* .get_rhs_packed_offset_ex = */ &kernel_offs_fn2<kai_get_rhs_packed_offset_matmul_clamp_f32_qai8dxp1x4_qsi8cxp4vlx4_1x4vl_sme_dot>,
|
||||
/* .run_kernel_ex = */ &kernel_run_float_fn10<kai_run_matmul_clamp_f32_qai8dxp1x4_qsi8cxp4vlx4_1x4vl_sme_dot>,
|
||||
@@ -802,10 +875,7 @@ static ggml_kleidiai_kernels gemm_gemv_kernels_q8[] = {
|
||||
/* .pack_func_ex = */ &lhs_pack_float_fn9_no_bl<kai_run_lhs_quant_pack_qai8dxp_f32>,
|
||||
},
|
||||
/* .rhs_info = */ {
|
||||
/* .packed_stride = */ kai_get_rhs_packed_stride_rhs_pack_nxk_qsi8cxp_qsi8cx_neon,
|
||||
/* .to_float = */ dequantize_row_qsi8cxp,
|
||||
/* .packed_size_ex = */ &rhs_ps_fn5<kai_get_rhs_packed_size_rhs_pack_nxk_qsi8cxp_qsi8cx_neon>,
|
||||
/* .packed_stride_ex = */ &rhs_stride_fn4<kai_get_rhs_packed_stride_rhs_pack_nxk_qsi8cxp_qsi8cx_neon>,
|
||||
/* .pack_func_ex = */ &rhs_pack_scale_fn12<kai_run_rhs_pack_nxk_qsi8cxp_qsi8cx_neon>,
|
||||
},
|
||||
/* .required_cpu = */ CPU_FEATURE_SME,
|
||||
@@ -823,7 +893,6 @@ static ggml_kleidiai_kernels gemm_gemv_kernels_q8[] = {
|
||||
/* .get_kr = */ kai_get_kr_matmul_clamp_f32_qai8dxp4x8_qsi8cxp4x8_16x4_neon_i8mm,
|
||||
/* .get_sr = */ kai_get_sr_matmul_clamp_f32_qai8dxp4x8_qsi8cxp4x8_16x4_neon_i8mm,
|
||||
/* .get_dst_offset = */ kai_get_dst_offset_matmul_clamp_f32_qai8dxp4x8_qsi8cxp4x8_16x4_neon_i8mm,
|
||||
/* .get_dst_size = */ kai_get_dst_size_matmul_clamp_f32_qai8dxp4x8_qsi8cxp4x8_16x4_neon_i8mm,
|
||||
/* .get_lhs_offset_ex = */ &kernel_offs_fn2<kai_get_lhs_packed_offset_matmul_clamp_f32_qai8dxp4x8_qsi8cxp4x8_16x4_neon_i8mm>,
|
||||
/* .get_rhs_packed_offset_ex = */ &kernel_offs_fn2<kai_get_rhs_packed_offset_matmul_clamp_f32_qai8dxp4x8_qsi8cxp4x8_16x4_neon_i8mm>,
|
||||
/* .run_kernel_ex = */ &kernel_run_float_fn10<kai_run_matmul_clamp_f32_qai8dxp4x8_qsi8cxp4x8_16x4_neon_i8mm>,
|
||||
@@ -843,7 +912,6 @@ static ggml_kleidiai_kernels gemm_gemv_kernels_q8[] = {
|
||||
/* .get_kr = */ kai_get_kr_matmul_clamp_f32_qai8dxp1x8_qsi8cxp4x8_1x4_neon_dotprod,
|
||||
/* .get_sr = */ kai_get_sr_matmul_clamp_f32_qai8dxp1x8_qsi8cxp4x8_1x4_neon_dotprod,
|
||||
/* .get_dst_offset = */ kai_get_dst_offset_matmul_clamp_f32_qai8dxp1x8_qsi8cxp4x8_1x4_neon_dotprod,
|
||||
/* .get_dst_size = */ kai_get_dst_size_matmul_clamp_f32_qai8dxp1x8_qsi8cxp4x8_1x4_neon_dotprod,
|
||||
/* .get_lhs_offset_ex = */ &kernel_offs_fn2<kai_get_lhs_packed_offset_matmul_clamp_f32_qai8dxp1x8_qsi8cxp4x8_1x4_neon_dotprod>,
|
||||
/* .get_rhs_packed_offset_ex = */ &kernel_offs_fn2<kai_get_rhs_packed_offset_matmul_clamp_f32_qai8dxp1x8_qsi8cxp4x8_1x4_neon_dotprod>,
|
||||
/* .run_kernel_ex = */ &kernel_run_float_fn10<kai_run_matmul_clamp_f32_qai8dxp1x8_qsi8cxp4x8_1x4_neon_dotprod>,
|
||||
@@ -855,10 +923,7 @@ static ggml_kleidiai_kernels gemm_gemv_kernels_q8[] = {
|
||||
/* .pack_func_ex = */ &lhs_pack_float_fn9_no_bl<kai_run_lhs_quant_pack_qai8dxp_f32>,
|
||||
},
|
||||
/* .rhs_info = */ {
|
||||
/* .packed_stride = */ kai_get_rhs_packed_stride_rhs_pack_nxk_qsi8cxp_qsi8cx_neon,
|
||||
/* .to_float = */ dequantize_row_qsi8cxp,
|
||||
/* .packed_size_ex = */ &rhs_ps_fn5<kai_get_rhs_packed_size_rhs_pack_nxk_qsi8cxp_qsi8cx_neon>,
|
||||
/* .packed_stride_ex = */ &rhs_stride_fn4<kai_get_rhs_packed_stride_rhs_pack_nxk_qsi8cxp_qsi8cx_neon>,
|
||||
/* .pack_func_ex = */ &rhs_pack_scale_fn12<kai_run_rhs_pack_nxk_qsi8cxp_qsi8cx_neon>,
|
||||
},
|
||||
/* .required_cpu = */ CPU_FEATURE_I8MM | CPU_FEATURE_DOTPROD,
|
||||
@@ -876,7 +941,6 @@ static ggml_kleidiai_kernels gemm_gemv_kernels_q8[] = {
|
||||
/* .get_kr = */ kai_get_kr_matmul_clamp_f32_qai8dxp4x4_qsi8cxp4x4_16x4_neon_dotprod,
|
||||
/* .get_sr = */ kai_get_sr_matmul_clamp_f32_qai8dxp4x4_qsi8cxp4x4_16x4_neon_dotprod,
|
||||
/* .get_dst_offset = */ kai_get_dst_offset_matmul_clamp_f32_qai8dxp4x4_qsi8cxp4x4_16x4_neon_dotprod,
|
||||
/* .get_dst_size = */ kai_get_dst_size_matmul_clamp_f32_qai8dxp4x4_qsi8cxp4x4_16x4_neon_dotprod,
|
||||
/* .get_lhs_offset_ex = */ &kernel_offs_fn2<kai_get_lhs_packed_offset_matmul_clamp_f32_qai8dxp4x4_qsi8cxp4x4_16x4_neon_dotprod>,
|
||||
/* .get_rhs_packed_offset_ex = */ &kernel_offs_fn2<kai_get_rhs_packed_offset_matmul_clamp_f32_qai8dxp4x4_qsi8cxp4x4_16x4_neon_dotprod>,
|
||||
/* .run_kernel_ex = */ &kernel_run_float_fn10<kai_run_matmul_clamp_f32_qai8dxp4x4_qsi8cxp4x4_16x4_neon_dotprod>,
|
||||
@@ -896,7 +960,6 @@ static ggml_kleidiai_kernels gemm_gemv_kernels_q8[] = {
|
||||
/* .get_kr = */ kai_get_kr_matmul_clamp_f32_qai8dxp1x4_qsi8cxp4x4_1x4_neon_dotprod,
|
||||
/* .get_sr = */ kai_get_sr_matmul_clamp_f32_qai8dxp1x4_qsi8cxp4x4_1x4_neon_dotprod,
|
||||
/* .get_dst_offset = */ kai_get_dst_offset_matmul_clamp_f32_qai8dxp1x4_qsi8cxp4x4_1x4_neon_dotprod,
|
||||
/* .get_dst_size = */ kai_get_dst_size_matmul_clamp_f32_qai8dxp1x4_qsi8cxp4x4_1x4_neon_dotprod,
|
||||
/* .get_lhs_offset_ex = */ &kernel_offs_fn2<kai_get_lhs_packed_offset_matmul_clamp_f32_qai8dxp1x4_qsi8cxp4x4_1x4_neon_dotprod>,
|
||||
/* .get_rhs_packed_offset_ex = */ &kernel_offs_fn2<kai_get_rhs_packed_offset_matmul_clamp_f32_qai8dxp1x4_qsi8cxp4x4_1x4_neon_dotprod>,
|
||||
/* .run_kernel_ex = */ &kernel_run_float_fn10<kai_run_matmul_clamp_f32_qai8dxp1x4_qsi8cxp4x4_1x4_neon_dotprod>,
|
||||
@@ -908,10 +971,7 @@ static ggml_kleidiai_kernels gemm_gemv_kernels_q8[] = {
|
||||
/* .pack_func_ex = */ &lhs_pack_float_fn9_no_bl<kai_run_lhs_quant_pack_qai8dxp_f32>,
|
||||
},
|
||||
/* .rhs_info = */ {
|
||||
/* .packed_stride = */ kai_get_rhs_packed_stride_rhs_pack_nxk_qsi8cxp_qsi8cx_neon,
|
||||
/* .to_float = */ dequantize_row_qsi8cxp,
|
||||
/* .packed_size_ex = */ &rhs_ps_fn5<kai_get_rhs_packed_size_rhs_pack_nxk_qsi8cxp_qsi8cx_neon>,
|
||||
/* .packed_stride_ex = */ &rhs_stride_fn4<kai_get_rhs_packed_stride_rhs_pack_nxk_qsi8cxp_qsi8cx_neon>,
|
||||
/* .pack_func_ex = */ &rhs_pack_scale_fn12<kai_run_rhs_pack_nxk_qsi8cxp_qsi8cx_neon>,
|
||||
},
|
||||
/* .required_cpu = */ CPU_FEATURE_DOTPROD,
|
||||
@@ -933,7 +993,6 @@ static ggml_kleidiai_kernels ggml_kleidiai_kernels_f32[] = {
|
||||
/* .get_kr = */ kai_get_kr_matmul_clamp_f32_f32p2vlx1_f32p2vlx1biasf32_sme2_mopa,
|
||||
/* .get_sr = */ kai_get_sr_matmul_clamp_f32_f32p2vlx1_f32p2vlx1biasf32_sme2_mopa,
|
||||
/* .get_dst_offset = */ kai_get_dst_offset_matmul_clamp_f32_f32p2vlx1_f32p2vlx1biasf32_sme2_mopa,
|
||||
/* .get_dst_size = */ kai_get_dst_size_matmul_clamp_f32_f32p2vlx1_f32p2vlx1biasf32_sme2_mopa,
|
||||
/* .get_lhs_offset_ex = */ &kernel_offs_fn2<kai_get_lhs_packed_offset_matmul_clamp_f32_f32p2vlx1_f32p2vlx1biasf32_sme2_mopa>,
|
||||
/* .get_rhs_packed_offset_ex = */ &kernel_offs_fn2<kai_get_rhs_packed_offset_matmul_clamp_f32_f32p2vlx1_f32p2vlx1biasf32_sme2_mopa>,
|
||||
/* .run_kernel_ex = */ &kernel_run_fn10<kai_run_matmul_clamp_f32_f32p2vlx1_f32p2vlx1biasf32_sme2_mopa>,
|
||||
@@ -953,7 +1012,6 @@ static ggml_kleidiai_kernels ggml_kleidiai_kernels_f32[] = {
|
||||
/* .get_kr = */ kai_get_kr_matmul_clamp_f32_f32_f32p2vlx1b_1x16vl_sme2_mla,
|
||||
/* .get_sr = */ kai_get_sr_matmul_clamp_f32_f32_f32p2vlx1b_1x16vl_sme2_mla,
|
||||
/* .get_dst_offset = */ kai_get_dst_offset_matmul_clamp_f32_f32_f32p2vlx1b_1x16vl_sme2_mla,
|
||||
/* .get_dst_size = */ kai_get_dst_size_matmul_clamp_f32_f32_f32p2vlx1b_1x16vl_sme2_mla,
|
||||
/* .get_lhs_offset_ex = */ &kernel_offs_fn2<kai_get_lhs_offset_matmul_clamp_f32_f32_f32p2vlx1b_1x16vl_sme2_mla>,
|
||||
/* .get_rhs_packed_offset_ex = */ &kernel_offs_fn2<kai_get_rhs_packed_offset_matmul_clamp_f32_f32_f32p2vlx1b_1x16vl_sme2_mla>,
|
||||
/* .run_kernel_ex = */ &kernel_run_lhs_stride_fn10<kai_run_matmul_clamp_f32_f32_f32p2vlx1b_1x16vl_sme2_mla>,
|
||||
@@ -965,10 +1023,7 @@ static ggml_kleidiai_kernels ggml_kleidiai_kernels_f32[] = {
|
||||
/* .pack_func_ex = */ nullptr,
|
||||
},
|
||||
/* .rhs_info = */ {
|
||||
/* .packed_stride = */ nullptr,
|
||||
/* .to_float = */ nullptr,
|
||||
/* .packed_size_ex = */ &rhs_ps_fn2<kai_get_rhs_packed_size_rhs_pack_nxk_f32p2vlx1biasf32_f32_f32_sme>,
|
||||
/* .packed_stride_ex = */ &rhs_stride_fn1<kai_get_rhs_packed_stride_rhs_pack_nxk_f32p2vlx1biasf32_f32_f32_sme>,
|
||||
/* .pack_func_ex = */ &rhs_pack_fn13<kai_run_rhs_pack_nxk_f32p2vlx1biasf32_f32_f32_sme>,
|
||||
},
|
||||
/* .required_cpu = */ CPU_FEATURE_SME2,
|
||||
@@ -986,7 +1041,6 @@ static ggml_kleidiai_kernels ggml_kleidiai_kernels_f32[] = {
|
||||
/* .get_kr = */ kai_get_kr_matmul_clamp_f32_f32p2vlx1_f32p2vlx1b_2vlx2vl_sme_mopa,
|
||||
/* .get_sr = */ kai_get_sr_matmul_clamp_f32_f32p2vlx1_f32p2vlx1b_2vlx2vl_sme_mopa,
|
||||
/* .get_dst_offset = */ kai_get_dst_offset_matmul_clamp_f32_f32p2vlx1_f32p2vlx1b_2vlx2vl_sme_mopa,
|
||||
/* .get_dst_size = */ kai_get_dst_size_matmul_clamp_f32_f32p2vlx1_f32p2vlx1b_2vlx2vl_sme_mopa,
|
||||
/* .get_lhs_offset_ex = */ &kernel_offs_fn2<kai_get_lhs_packed_offset_matmul_clamp_f32_f32p2vlx1_f32p2vlx1b_2vlx2vl_sme_mopa>,
|
||||
/* .get_rhs_packed_offset_ex = */ &kernel_offs_fn2<kai_get_rhs_packed_offset_matmul_clamp_f32_f32p2vlx1_f32p2vlx1b_2vlx2vl_sme_mopa>,
|
||||
/* .run_kernel_ex = */ &kernel_run_fn10<kai_run_matmul_clamp_f32_f32p2vlx1_f32p2vlx1b_2vlx2vl_sme_mopa>,
|
||||
@@ -1006,7 +1060,6 @@ static ggml_kleidiai_kernels ggml_kleidiai_kernels_f32[] = {
|
||||
/* .get_kr = */ kai_get_kr_matmul_clamp_f32_f32p2vlx1_f32p2vlx1b_2vlx2vl_sme_mopa,
|
||||
/* .get_sr = */ kai_get_sr_matmul_clamp_f32_f32p2vlx1_f32p2vlx1b_2vlx2vl_sme_mopa,
|
||||
/* .get_dst_offset = */ kai_get_dst_offset_matmul_clamp_f32_f32p2vlx1_f32p2vlx1b_2vlx2vl_sme_mopa,
|
||||
/* .get_dst_size = */ kai_get_dst_size_matmul_clamp_f32_f32p2vlx1_f32p2vlx1b_2vlx2vl_sme_mopa,
|
||||
/* .get_lhs_offset_ex = */ nullptr,
|
||||
/* .get_rhs_packed_offset_ex = */ nullptr,
|
||||
/* .run_kernel_ex = */ nullptr,
|
||||
@@ -1018,10 +1071,7 @@ static ggml_kleidiai_kernels ggml_kleidiai_kernels_f32[] = {
|
||||
/* .pack_func_ex = */ &lhs_pack_void_fn9<kai_run_lhs_pack_f32p2vlx1_f32_sme>,
|
||||
},
|
||||
/* .rhs_info = */ {
|
||||
/* .packed_stride = */ nullptr,
|
||||
/* .to_float = */ nullptr,
|
||||
/* .packed_size_ex = */ &rhs_ps_fn2<kai_get_rhs_packed_size_rhs_pack_nxk_f32p2vlx1biasf32_f32_f32_sme>,
|
||||
/* .packed_stride_ex = */ &rhs_stride_fn1<kai_get_rhs_packed_stride_rhs_pack_nxk_f32p2vlx1biasf32_f32_f32_sme>,
|
||||
/* .pack_func_ex = */ &rhs_pack_fn13<kai_run_rhs_pack_nxk_f32p2vlx1biasf32_f32_f32_sme>,
|
||||
},
|
||||
/* .required_cpu = */ CPU_FEATURE_SME,
|
||||
@@ -1049,8 +1099,23 @@ ggml_kleidiai_kernels * ggml_kleidiai_select_kernels(cpu_feature cpu_features, c
|
||||
return false;
|
||||
};
|
||||
|
||||
if (tensor->src[0]->type == GGML_TYPE_Q8_0) {
|
||||
try_table(gemm_gemv_kernels_q8);
|
||||
if (tensor->src[0]->type == GGML_TYPE_Q4_K) {
|
||||
try_table(gemm_gemv_kernels_q4_k);
|
||||
} else if (tensor->src[0]->type == GGML_TYPE_Q8_0 || tensor->src[0]->type == GGML_TYPE_Q6_K) {
|
||||
const ggml_type original_type = tensor->src[0]->type;
|
||||
if (original_type == GGML_TYPE_Q6_K) {
|
||||
for (size_t i = 0; i < NELEMS(gemm_gemv_kernels_q8) - 1; ++i) {
|
||||
if ((cpu_features & gemm_gemv_kernels_q8[i].required_cpu) == gemm_gemv_kernels_q8[i].required_cpu &&
|
||||
gemm_gemv_kernels_q8[i].lhs_type == tensor->src[1]->type &&
|
||||
gemm_gemv_kernels_q8[i].rhs_type == GGML_TYPE_Q8_0 &&
|
||||
gemm_gemv_kernels_q8[i].op_type == tensor->type) {
|
||||
kernel = &gemm_gemv_kernels_q8[i];
|
||||
break;
|
||||
}
|
||||
}
|
||||
} else {
|
||||
try_table(gemm_gemv_kernels_q8);
|
||||
}
|
||||
} else if (tensor->src[0]->type == GGML_TYPE_F32) {
|
||||
try_table(ggml_kleidiai_kernels_f32);
|
||||
} else {
|
||||
@@ -1075,6 +1140,19 @@ ggml_kleidiai_kernels * ggml_kleidiai_select_kernels_q4_0(cpu_feature features)
|
||||
return kernels;
|
||||
}
|
||||
|
||||
ggml_kleidiai_kernels * ggml_kleidiai_select_kernels_q4_k(cpu_feature features) {
|
||||
ggml_kleidiai_kernels * kernels = nullptr;
|
||||
|
||||
for (size_t i = 0; i < NELEMS(gemm_gemv_kernels_q4_k) - 1; ++i) {
|
||||
if ((features & gemm_gemv_kernels_q4_k[i].required_cpu) == gemm_gemv_kernels_q4_k[i].required_cpu) {
|
||||
kernels = &gemm_gemv_kernels_q4_k[i];
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
return kernels;
|
||||
}
|
||||
|
||||
ggml_kleidiai_kernels * ggml_kleidiai_select_kernels_q8_0(cpu_feature features) {
|
||||
ggml_kleidiai_kernels * kernels = nullptr;
|
||||
|
||||
|
||||
@@ -33,7 +33,6 @@ struct kernel_info {
|
||||
size_t (*get_sr)(void);
|
||||
|
||||
size_t (*get_dst_offset)(size_t m_idx, size_t n_idx, size_t stride);
|
||||
size_t (*get_dst_size)(size_t m, size_t n);
|
||||
|
||||
size_t (*get_lhs_offset_ex)(size_t m_idx, size_t k, size_t bl);
|
||||
|
||||
@@ -59,21 +58,12 @@ struct lhs_packing_info {
|
||||
|
||||
enum rhs_repack_mode {
|
||||
RHS_REPACK_PER_KERNEL,
|
||||
RHS_REPACK_SHARED,
|
||||
RHS_REPACK_SINGLE_ONLY,
|
||||
};
|
||||
|
||||
struct rhs_packing_info {
|
||||
size_t (*packed_stride)(size_t k, size_t nr, size_t kr, size_t bl);
|
||||
|
||||
void (*to_float)(const void *packed_data, int32_t row_idx, int64_t nc, float *out,
|
||||
size_t nr_pack, size_t packed_row_stride, size_t kr, size_t bl,
|
||||
size_t num_bytes_multiplier);
|
||||
|
||||
size_t (*packed_size_ex)(size_t n, size_t k, size_t nr, size_t kr, size_t bl);
|
||||
|
||||
size_t (*packed_stride_ex)(size_t k, size_t nr, size_t kr, size_t bl);
|
||||
|
||||
void (*pack_func_ex)(size_t num_groups, size_t n, size_t k, size_t nr, size_t kr, size_t sr, size_t bl,
|
||||
size_t rhs_stride, const void * rhs, const void * bias, const void * scale, void * rhs_packed, size_t extra_bytes, const void * params);
|
||||
|
||||
@@ -97,5 +87,6 @@ struct ggml_kleidiai_kernels {
|
||||
|
||||
ggml_kleidiai_kernels * ggml_kleidiai_select_kernels(cpu_feature cpu_features, const ggml_tensor * tensor);
|
||||
ggml_kleidiai_kernels * ggml_kleidiai_select_kernels_q4_0(cpu_feature features);
|
||||
ggml_kleidiai_kernels * ggml_kleidiai_select_kernels_q4_k(cpu_feature features);
|
||||
ggml_kleidiai_kernels * ggml_kleidiai_select_kernels_q8_0(cpu_feature features);
|
||||
ggml_kleidiai_kernels * ggml_kleidiai_select_kernels_f32(cpu_feature features);
|
||||
@@ -43,6 +43,7 @@
|
||||
#include "ggml-impl.h"
|
||||
#include "ggml-feats.h"
|
||||
#include "ggml-backend-impl.h"
|
||||
#include "ggml-quants.h"
|
||||
#include "ggml-threading.h"
|
||||
#include "traits.h"
|
||||
|
||||
@@ -57,16 +58,20 @@ static constexpr int GGML_KLEIDIAI_MAX_KERNEL_SLOTS = 2;
|
||||
static constexpr uint32_t GGML_KLEIDIAI_PACK_MAGIC = 0x4b4c4149; // "KLAI"
|
||||
static constexpr uint16_t GGML_KLEIDIAI_PACK_VERSION = 1;
|
||||
static constexpr size_t GGML_KLEIDIAI_PACK_ALIGN = 64;
|
||||
static constexpr size_t GGML_KLEIDIAI_Q4_K_BL = QK_K / 8;
|
||||
|
||||
static_assert(QK_K == 256, "KleidiAI Q4_K kernels require 256-value super-blocks");
|
||||
|
||||
struct ggml_kleidiai_context {
|
||||
cpu_feature features;
|
||||
ggml_kleidiai_kernels * kernels_q4;
|
||||
ggml_kleidiai_kernels * kernels_q4k;
|
||||
ggml_kleidiai_kernels * kernels_q8;
|
||||
ggml_kleidiai_kernels * kernels_f32;
|
||||
int sme_thread_cap; // <= 0 means "SME disabled/unknown"
|
||||
int thread_hint; // <= 0 means "no hint"
|
||||
int chunk_multiplier;
|
||||
} static ctx = { CPU_FEATURE_NONE, nullptr, nullptr, nullptr, 0, -1, 4 };
|
||||
} static ctx = { CPU_FEATURE_NONE, nullptr, nullptr, nullptr, nullptr, 0, -1, 4 };
|
||||
|
||||
static inline bool is_sme_family(cpu_feature f) {
|
||||
return (f & (CPU_FEATURE_SME | CPU_FEATURE_SME2)) != CPU_FEATURE_NONE;
|
||||
@@ -387,6 +392,7 @@ static void init_kleidiai_context(void) {
|
||||
|
||||
// Kernel selection
|
||||
ctx.kernels_q4 = ggml_kleidiai_select_kernels_q4_0(ctx.features);
|
||||
ctx.kernels_q4k = ggml_kleidiai_select_kernels_q4_k(ctx.features);
|
||||
ctx.kernels_q8 = ggml_kleidiai_select_kernels_q8_0(ctx.features);
|
||||
ctx.kernels_f32 = ggml_kleidiai_select_kernels_f32(ctx.features);
|
||||
|
||||
@@ -402,6 +408,12 @@ static void init_kleidiai_context(void) {
|
||||
GGML_LOG_INFO("kleidiai: primary q8 kernel feature %s\n", cpu_feature_to_string(ctx.kernels_q8->required_cpu));
|
||||
}
|
||||
|
||||
if (!ctx.kernels_q4k) {
|
||||
GGML_LOG_INFO("kleidiai: no compatible q4_k kernels found for CPU features mask %d\n", (int) ctx.features);
|
||||
} else {
|
||||
GGML_LOG_INFO("kleidiai: primary q4_k kernel feature %s\n", cpu_feature_to_string(ctx.kernels_q4k->required_cpu));
|
||||
}
|
||||
|
||||
if (!ctx.kernels_f32) {
|
||||
GGML_LOG_INFO("kleidiai: no compatible f32 kernels found for CPU features mask %d\n", (int)ctx.features);
|
||||
} else {
|
||||
@@ -410,6 +422,7 @@ static void init_kleidiai_context(void) {
|
||||
|
||||
const bool has_selected_sme_family_kernel =
|
||||
(ctx.kernels_q4 && is_sme_family(ctx.kernels_q4->required_cpu)) ||
|
||||
(ctx.kernels_q4k && is_sme_family(ctx.kernels_q4k->required_cpu)) ||
|
||||
(ctx.kernels_q8 && is_sme_family(ctx.kernels_q8->required_cpu)) ||
|
||||
(ctx.kernels_f32 && is_sme_family(ctx.kernels_f32->required_cpu));
|
||||
ctx.sme_thread_cap = has_selected_sme_family_kernel ? sme_cores : 0;
|
||||
@@ -486,6 +499,8 @@ static inline kleidiai_block_args kleidiai_get_block_args(ggml_type rhs_type) {
|
||||
switch (rhs_type) {
|
||||
case GGML_TYPE_Q4_0:
|
||||
return { QK4_0, QK4_0, QK4_0 };
|
||||
case GGML_TYPE_Q4_K:
|
||||
return { GGML_KLEIDIAI_Q4_K_BL, GGML_KLEIDIAI_Q4_K_BL, GGML_KLEIDIAI_Q4_K_BL };
|
||||
case GGML_TYPE_Q8_0:
|
||||
return { 0, 0, QK8_0 };
|
||||
default:
|
||||
@@ -556,6 +571,10 @@ static inline ggml_kleidiai_kernels * kleidiai_primary_kernel_q4() {
|
||||
return ctx.kernels_q4;
|
||||
}
|
||||
|
||||
static inline ggml_kleidiai_kernels * kleidiai_primary_kernel_q4k() {
|
||||
return ctx.kernels_q4k;
|
||||
}
|
||||
|
||||
static inline ggml_kleidiai_kernels * kleidiai_primary_kernel_q8() {
|
||||
return ctx.kernels_q8;
|
||||
}
|
||||
@@ -610,6 +629,12 @@ static int kleidiai_collect_q4_chain(std::array<ggml_kleidiai_kernels *, GGML_KL
|
||||
[&](cpu_feature mask) { return ggml_kleidiai_select_kernels_q4_0(mask); });
|
||||
}
|
||||
|
||||
static int kleidiai_collect_q4k_chain(std::array<ggml_kleidiai_kernels *, GGML_KLEIDIAI_MAX_KERNEL_SLOTS> & out) {
|
||||
ggml_kleidiai_kernels * primary = kleidiai_primary_kernel_q4k();
|
||||
return kleidiai_collect_kernel_chain_common(primary, ctx.features, out,
|
||||
[&](cpu_feature mask) { return ggml_kleidiai_select_kernels_q4_k(mask); });
|
||||
}
|
||||
|
||||
static int kleidiai_collect_q8_chain(std::array<ggml_kleidiai_kernels *, GGML_KLEIDIAI_MAX_KERNEL_SLOTS> & out) {
|
||||
ggml_kleidiai_kernels * primary = kleidiai_primary_kernel_q8();
|
||||
return kleidiai_collect_kernel_chain_common(primary, ctx.features, out,
|
||||
@@ -662,8 +687,10 @@ class tensor_traits : public ggml::cpu::tensor_traits {
|
||||
const size_t n = op->src[0]->ne[1];
|
||||
const size_t m = op->src[1]->ne[1];
|
||||
|
||||
if (op->src[0]->type == GGML_TYPE_Q4_0 || op->src[0]->type == GGML_TYPE_Q8_0) {
|
||||
const size_t qk = (op->src[0]->type == GGML_TYPE_Q4_0) ? QK4_0 : QK8_0;
|
||||
if (op->src[0]->type == GGML_TYPE_Q4_0 || op->src[0]->type == GGML_TYPE_Q4_K ||
|
||||
op->src[0]->type == GGML_TYPE_Q8_0 || op->src[0]->type == GGML_TYPE_Q6_K) {
|
||||
const kleidiai_block_args block_args = kleidiai_get_block_args(kernel_chain[0]->rhs_type);
|
||||
const size_t qk = block_args.pack_bl;
|
||||
|
||||
size_t cursor = 0;
|
||||
bool any_slot = false;
|
||||
@@ -789,17 +816,14 @@ class tensor_traits : public ggml::cpu::tensor_traits {
|
||||
|
||||
bool compute_forward(struct ggml_compute_params * params, struct ggml_tensor * dst) override {
|
||||
if (dst->op == GGML_OP_MUL_MAT) {
|
||||
if (dst->src[0]->type == GGML_TYPE_Q4_0 || dst->src[0]->type == GGML_TYPE_Q8_0) {
|
||||
if (dst->src[0]->type == GGML_TYPE_Q4_0 || dst->src[0]->type == GGML_TYPE_Q4_K ||
|
||||
dst->src[0]->type == GGML_TYPE_Q8_0 || dst->src[0]->type == GGML_TYPE_Q6_K) {
|
||||
return compute_forward_qx(params, dst);
|
||||
} else if (dst->src[0]->type == GGML_TYPE_F32) {
|
||||
return compute_forward_f32(params, dst);
|
||||
} else if (dst->src[0]->type == GGML_TYPE_F16) {
|
||||
return compute_forward_fp16(params, dst);
|
||||
}
|
||||
} else if (dst->op == GGML_OP_GET_ROWS) {
|
||||
if (dst->src[0]->type == GGML_TYPE_Q4_0 || dst->src[0]->type == GGML_TYPE_Q8_0) {
|
||||
return compute_forward_get_rows(params, dst);
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
@@ -870,9 +894,12 @@ class tensor_traits : public ggml::cpu::tensor_traits {
|
||||
const bool disable_chunking = ggml_is_numa();
|
||||
GGML_ASSERT(n <= (size_t) INT_MAX);
|
||||
|
||||
for (int64_t batch_idx = 0; batch_idx < ne12; ++batch_idx) {
|
||||
const uint8_t * lhs_batch_base = static_cast<const uint8_t *>(src1->data) + batch_idx * src1->nb[2];
|
||||
uint8_t * dst_batch_base = static_cast<uint8_t *>(dst->data) + batch_idx * dst->nb[2];
|
||||
const int64_t batch_size = ne12 * ne13;
|
||||
for (int64_t batch_idx = 0; batch_idx < batch_size; ++batch_idx) {
|
||||
const int64_t i12 = batch_idx % ne12;
|
||||
const int64_t i13 = batch_idx / ne12;
|
||||
const uint8_t * lhs_batch_base = static_cast<const uint8_t *>(src1->data) + i12 * src1->nb[2] + i13 * src1->nb[3];
|
||||
uint8_t * dst_batch_base = static_cast<uint8_t *>(dst->data) + i12 * dst->nb[2] + i13 * dst->nb[3];
|
||||
|
||||
if (!use_gemv) {
|
||||
const int64_t m_roundup_mr = kai_roundup((int64_t)m, (int64_t)mr);
|
||||
@@ -947,7 +974,7 @@ class tensor_traits : public ggml::cpu::tensor_traits {
|
||||
current_col = ggml_threadpool_chunk_add(params->threadpool, (int) chunk_cols);
|
||||
}
|
||||
|
||||
if (batch_idx != ne12 - 1) {
|
||||
if (batch_idx != batch_size - 1) {
|
||||
ggml_barrier(params->threadpool);
|
||||
}
|
||||
}
|
||||
@@ -1107,7 +1134,8 @@ class tensor_traits : public ggml::cpu::tensor_traits {
|
||||
}
|
||||
|
||||
bool compute_forward_qx(struct ggml_compute_params * params, struct ggml_tensor * dst) {
|
||||
GGML_ASSERT(dst->src[0]->type == GGML_TYPE_Q4_0 || dst->src[0]->type == GGML_TYPE_Q8_0);
|
||||
GGML_ASSERT(dst->src[0]->type == GGML_TYPE_Q4_0 || dst->src[0]->type == GGML_TYPE_Q4_K ||
|
||||
dst->src[0]->type == GGML_TYPE_Q8_0 || dst->src[0]->type == GGML_TYPE_Q6_K);
|
||||
|
||||
const ggml_tensor * src0 = dst->src[0];
|
||||
const ggml_tensor * src1 = dst->src[1];
|
||||
@@ -1265,7 +1293,9 @@ class tensor_traits : public ggml::cpu::tensor_traits {
|
||||
}
|
||||
}
|
||||
|
||||
int sme_cap = kleidiai_sme_thread_cap();
|
||||
const bool sme_single_only = sme_slot != -1 &&
|
||||
runtime[sme_slot].kernels->rhs_info.repack_mode == RHS_REPACK_SINGLE_ONLY;
|
||||
int sme_cap = sme_single_only ? nth_total : kleidiai_sme_thread_cap();
|
||||
if (sme_cap < 0) {
|
||||
sme_cap = nth_total;
|
||||
}
|
||||
@@ -1301,23 +1331,14 @@ class tensor_traits : public ggml::cpu::tensor_traits {
|
||||
threads_remaining -= share;
|
||||
}
|
||||
|
||||
if (threads_remaining > 0) {
|
||||
const int fallback_slot = (sme_slot != -1) ? sme_slot : 0;
|
||||
runtime[fallback_slot].assigned_threads += threads_remaining;
|
||||
threads_remaining = 0;
|
||||
}
|
||||
|
||||
int thread_cursor = 0;
|
||||
for (int i = 0; i < runtime_count; ++i) {
|
||||
runtime[i].thread_begin = thread_cursor;
|
||||
thread_cursor += runtime[i].assigned_threads;
|
||||
runtime[i].thread_end = thread_cursor;
|
||||
}
|
||||
|
||||
if (thread_cursor < nth_total && runtime_count > 0) {
|
||||
runtime[runtime_count - 1].assigned_threads += nth_total - thread_cursor;
|
||||
runtime[runtime_count - 1].thread_end = nth_total;
|
||||
}
|
||||
GGML_ASSERT(thread_cursor > 0);
|
||||
const int active_threads = thread_cursor;
|
||||
|
||||
int local_slot = -1;
|
||||
int local_ith = 0;
|
||||
@@ -1328,10 +1349,6 @@ class tensor_traits : public ggml::cpu::tensor_traits {
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (local_slot == -1) {
|
||||
return false;
|
||||
}
|
||||
|
||||
const size_t k = ne00;
|
||||
const size_t m = ne11;
|
||||
const size_t n = ne01;
|
||||
@@ -1363,7 +1380,7 @@ class tensor_traits : public ggml::cpu::tensor_traits {
|
||||
|
||||
const bool disable_chunking = ggml_is_numa();
|
||||
const size_t chunk_multiplier = std::max(1, ctx.chunk_multiplier);
|
||||
const size_t chunk_divisor = (nth_total == 1 || disable_chunking) ? (size_t)nth_total : (size_t)nth_total * chunk_multiplier;
|
||||
const size_t chunk_divisor = (active_threads == 1 || disable_chunking) ? (size_t) active_threads : (size_t) active_threads * chunk_multiplier;
|
||||
size_t chunk_cols = align_up(std::max<size_t>(1, ceil_div_size(n, chunk_divisor)), common_step);
|
||||
if (chunk_cols == 0) {
|
||||
chunk_cols = common_step;
|
||||
@@ -1397,7 +1414,7 @@ class tensor_traits : public ggml::cpu::tensor_traits {
|
||||
const uint8_t * lhs_batch_base = static_cast<const uint8_t *>(src1->data) + batch_idx * src1->nb[2];
|
||||
uint8_t * dst_batch_base = static_cast<uint8_t *>(dst->data) + batch_idx * dst->nb[2];
|
||||
|
||||
if (runtime[local_slot].assigned_threads > 0) {
|
||||
if (local_slot != -1) {
|
||||
runtime_slot & slot = runtime[local_slot];
|
||||
const int64_t m_roundup_mr = kai_roundup((int64_t)m, (int64_t)slot.mr);
|
||||
int64_t max_threads = slot.mr ? (m_roundup_mr / (int64_t)slot.mr) : slot.assigned_threads;
|
||||
@@ -1438,28 +1455,30 @@ class tensor_traits : public ggml::cpu::tensor_traits {
|
||||
}
|
||||
|
||||
if (ith_total == 0) {
|
||||
ggml_threadpool_chunk_set(params->threadpool, nth_total);
|
||||
ggml_threadpool_chunk_set(params->threadpool, active_threads);
|
||||
}
|
||||
|
||||
// Publishes both LHS packing and the initialized dynamic chunk queue.
|
||||
ggml_barrier(params->threadpool);
|
||||
|
||||
runtime_slot & slot = runtime[local_slot];
|
||||
int current_chunk = ith_total;
|
||||
while (current_chunk < nchunk) {
|
||||
const size_t global_start = (size_t)current_chunk * chunk_cols;
|
||||
if (global_start >= n) {
|
||||
break;
|
||||
}
|
||||
if (local_slot != -1) {
|
||||
runtime_slot & slot = runtime[local_slot];
|
||||
int current_chunk = ith_total;
|
||||
while (current_chunk < nchunk) {
|
||||
const size_t global_start = (size_t)current_chunk * chunk_cols;
|
||||
if (global_start >= n) {
|
||||
break;
|
||||
}
|
||||
|
||||
const size_t cols = std::min(chunk_cols, n - global_start);
|
||||
if (cols > 0) {
|
||||
// KleidiAI GEMM/GEMV kernels accept arbitrary final tail widths;
|
||||
// only non-tail chunks are guaranteed to be n_step-aligned.
|
||||
run_chunk(slot, global_start, cols, dst_batch_base);
|
||||
}
|
||||
const size_t cols = std::min(chunk_cols, n - global_start);
|
||||
if (cols > 0) {
|
||||
// KleidiAI GEMM/GEMV kernels accept arbitrary final tail widths;
|
||||
// only non-tail chunks are guaranteed to be n_step-aligned.
|
||||
run_chunk(slot, global_start, cols, dst_batch_base);
|
||||
}
|
||||
|
||||
current_chunk = ggml_threadpool_chunk_add(params->threadpool, 1);
|
||||
current_chunk = ggml_threadpool_chunk_add(params->threadpool, 1);
|
||||
}
|
||||
}
|
||||
|
||||
if (batch_idx != ne12 - 1) {
|
||||
@@ -1470,101 +1489,11 @@ class tensor_traits : public ggml::cpu::tensor_traits {
|
||||
return true;
|
||||
}
|
||||
|
||||
bool compute_forward_get_rows(struct ggml_compute_params * params, struct ggml_tensor * dst) {
|
||||
GGML_ASSERT(dst->src[0]->type == GGML_TYPE_Q4_0 || dst->src[0]->type == GGML_TYPE_Q8_0);
|
||||
const ggml_tensor * src0 = dst->src[0];
|
||||
const ggml_tensor * src1 = dst->src[1];
|
||||
|
||||
GGML_TENSOR_BINARY_OP_LOCALS
|
||||
|
||||
const kleidiai_weight_header * header = kleidiai_weight_header_from_ptr(src0->data);
|
||||
const bool has_header = kleidiai_is_weight_header_valid(header);
|
||||
|
||||
std::array<ggml_kleidiai_kernels *, GGML_KLEIDIAI_MAX_KERNEL_SLOTS> kernel_chain;
|
||||
const bool want_q8 = src0->type == GGML_TYPE_Q8_0;
|
||||
const int chain_count = want_q8 ? kleidiai_collect_q8_chain(kernel_chain)
|
||||
: kleidiai_collect_q4_chain(kernel_chain);
|
||||
|
||||
ggml_kleidiai_kernels * kernels = nullptr;
|
||||
const uint8_t * packed_base = static_cast<const uint8_t *>(src0->data);
|
||||
|
||||
if (has_header && chain_count > 0) {
|
||||
int select_slot = 0;
|
||||
if (select_slot >= header->slot_count) {
|
||||
select_slot = header->slot_count - 1;
|
||||
}
|
||||
if (select_slot >= 0 && select_slot < chain_count) {
|
||||
kernels = kernel_chain[select_slot];
|
||||
const uint8_t * slot_ptr = kleidiai_weight_slot_ptr(header, select_slot);
|
||||
if (slot_ptr) {
|
||||
packed_base = slot_ptr;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (!kernels && chain_count > 0) {
|
||||
kernels = kernel_chain[0];
|
||||
if (has_header) {
|
||||
const uint8_t * slot_ptr = kleidiai_weight_slot_ptr(header, 0);
|
||||
if (slot_ptr) {
|
||||
packed_base = slot_ptr;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (!kernels) {
|
||||
return false;
|
||||
}
|
||||
|
||||
rhs_packing_info * rhs_info = &kernels->rhs_info;
|
||||
kernel_info * kernel = &kernels->gemm;
|
||||
if (!rhs_info->to_float || !kernel->get_nr) {
|
||||
return false;
|
||||
}
|
||||
|
||||
const int64_t nc = ne00;
|
||||
const int64_t nr = ggml_nelements(src1);
|
||||
|
||||
const ggml_type rhs_type = kernels->rhs_type;
|
||||
size_t block_len = 0;
|
||||
size_t num_bytes_multiplier = 0;
|
||||
if (rhs_type == GGML_TYPE_Q4_0) {
|
||||
block_len = QK4_0;
|
||||
num_bytes_multiplier = sizeof(uint16_t);
|
||||
} else if (rhs_type == GGML_TYPE_Q8_0) {
|
||||
block_len = QK8_0;
|
||||
num_bytes_multiplier = sizeof(float);
|
||||
} else {
|
||||
return false;
|
||||
}
|
||||
|
||||
const size_t block_rows = kernel->get_nr();
|
||||
const size_t kr = kernel->get_kr();
|
||||
|
||||
const size_t packed_stride = rhs_info->packed_stride(nc, block_rows, kr, block_len);
|
||||
|
||||
const int ith = params->ith;
|
||||
const int nth = params->nth;
|
||||
|
||||
const int dr = (nr + nth - 1) / nth;
|
||||
const int ir0 = dr * ith;
|
||||
const int ir1 = MIN(ir0 + dr, nr);
|
||||
|
||||
for (int64_t i = ir0; i < ir1; ++i) {
|
||||
GGML_ASSERT(src1->type == GGML_TYPE_I32);
|
||||
int64_t row_idx = ((const int32_t *)src1->data)[i];
|
||||
GGML_ASSERT(row_idx >= 0 && row_idx < src0->ne[1]);
|
||||
|
||||
float *out = (float *)((char *)dst->data + i * nb1);
|
||||
rhs_info->to_float(packed_base, row_idx, nc, out, block_rows, packed_stride, kr, block_len, num_bytes_multiplier);
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
public:
|
||||
int repack(struct ggml_tensor * tensor, const void * data, size_t data_size) {
|
||||
GGML_ASSERT(tensor->type == GGML_TYPE_Q4_0 || tensor->type == GGML_TYPE_Q8_0 || tensor->type == GGML_TYPE_F32);
|
||||
GGML_ASSERT(tensor->type == GGML_TYPE_Q4_0 || tensor->type == GGML_TYPE_Q4_K ||
|
||||
tensor->type == GGML_TYPE_Q8_0 || tensor->type == GGML_TYPE_Q6_K ||
|
||||
tensor->type == GGML_TYPE_F32);
|
||||
const size_t n = tensor->ne[1];
|
||||
const size_t k = tensor->ne[0];
|
||||
|
||||
@@ -1582,16 +1511,19 @@ public:
|
||||
cursor = align_up(cursor, GGML_KLEIDIAI_PACK_ALIGN);
|
||||
|
||||
std::array<ggml_kleidiai_kernels *, GGML_KLEIDIAI_MAX_KERNEL_SLOTS> kernel_chain;
|
||||
const bool want_q8 = tensor->type == GGML_TYPE_Q8_0;
|
||||
const bool want_q4k = tensor->type == GGML_TYPE_Q4_K;
|
||||
const bool want_q8 = tensor->type == GGML_TYPE_Q8_0 || tensor->type == GGML_TYPE_Q6_K;
|
||||
const bool want_f32 = tensor->type == GGML_TYPE_F32;
|
||||
const int slot_total = want_f32 ? kleidiai_collect_f32_chain(kernel_chain)
|
||||
: want_q8 ? kleidiai_collect_q8_chain(kernel_chain)
|
||||
: kleidiai_collect_q4_chain(kernel_chain);
|
||||
: want_q4k ? kleidiai_collect_q4k_chain(kernel_chain)
|
||||
: kleidiai_collect_q4_chain(kernel_chain);
|
||||
const bool allow_fallback = kleidiai_pack_fallback_allowed();
|
||||
|
||||
std::vector<int8_t> qdata;
|
||||
std::vector<float> scales;
|
||||
std::vector<float> bias;
|
||||
std::vector<float> row_f32;
|
||||
|
||||
if (want_q8 && slot_total > 0) {
|
||||
qdata.resize(n * k, 0);
|
||||
@@ -1600,21 +1532,38 @@ public:
|
||||
const size_t row_stride = tensor->nb[1];
|
||||
const size_t k_blocks = (k + QK8_0 - 1) / QK8_0;
|
||||
|
||||
if (tensor->type == GGML_TYPE_Q6_K) {
|
||||
if (k % QK_K != 0 || k % QK8_0 != 0) {
|
||||
return -1;
|
||||
}
|
||||
row_f32.resize(k);
|
||||
}
|
||||
|
||||
for (size_t row = 0; row < n; ++row) {
|
||||
const auto * row_blocks = reinterpret_cast<const block_q8_0 *>(
|
||||
static_cast<const uint8_t *>(data) + row * row_stride);
|
||||
const uint8_t * row_data = static_cast<const uint8_t *>(data) + row * row_stride;
|
||||
|
||||
if (tensor->type == GGML_TYPE_Q6_K) {
|
||||
dequantize_row_q6_K(reinterpret_cast<const block_q6_K *>(row_data), row_f32.data(), k);
|
||||
}
|
||||
|
||||
float max_abs = 0.0f;
|
||||
for (size_t block = 0; block < k_blocks; ++block) {
|
||||
const block_q8_0 & blk = row_blocks[block];
|
||||
const float d = GGML_FP16_TO_FP32(blk.d);
|
||||
for (size_t l = 0; l < QK8_0; ++l) {
|
||||
const size_t linear_idx = block * QK8_0 + l;
|
||||
if (linear_idx >= k) {
|
||||
break;
|
||||
if (tensor->type == GGML_TYPE_Q6_K) {
|
||||
for (size_t linear_idx = 0; linear_idx < k; ++linear_idx) {
|
||||
max_abs = std::max(max_abs, std::fabs(row_f32[linear_idx]));
|
||||
}
|
||||
} else {
|
||||
const block_q8_0 * row_blocks = reinterpret_cast<const block_q8_0 *>(row_data);
|
||||
for (size_t block = 0; block < k_blocks; ++block) {
|
||||
const block_q8_0 & blk = row_blocks[block];
|
||||
const float d = GGML_FP16_TO_FP32(blk.d);
|
||||
for (size_t l = 0; l < QK8_0; ++l) {
|
||||
const size_t linear_idx = block * QK8_0 + l;
|
||||
if (linear_idx >= k) {
|
||||
break;
|
||||
}
|
||||
const float value = d * static_cast<float>(blk.qs[l]);
|
||||
max_abs = std::max(max_abs, std::fabs(value));
|
||||
}
|
||||
const float value = d * static_cast<float>(blk.qs[l]);
|
||||
max_abs = std::max(max_abs, std::fabs(value));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1622,19 +1571,28 @@ public:
|
||||
scales[row] = scale;
|
||||
const float inv_scale = scale > 0.0f ? 1.0f / scale : 0.0f;
|
||||
|
||||
for (size_t block = 0; block < k_blocks; ++block) {
|
||||
const block_q8_0 & blk = row_blocks[block];
|
||||
const float d = GGML_FP16_TO_FP32(blk.d);
|
||||
for (size_t l = 0; l < QK8_0; ++l) {
|
||||
const size_t linear_idx = block * QK8_0 + l;
|
||||
if (linear_idx >= k) {
|
||||
break;
|
||||
}
|
||||
const float value = d * static_cast<float>(blk.qs[l]);
|
||||
int32_t q = scale > 0.0f ? static_cast<int32_t>(std::lround(value * inv_scale)) : 0;
|
||||
if (tensor->type == GGML_TYPE_Q6_K) {
|
||||
for (size_t linear_idx = 0; linear_idx < k; ++linear_idx) {
|
||||
int32_t q = scale > 0.0f ? static_cast<int32_t>(std::lround(row_f32[linear_idx] * inv_scale)) : 0;
|
||||
q = std::clamp(q, -127, 127);
|
||||
qdata[row * k + linear_idx] = static_cast<int8_t>(q);
|
||||
}
|
||||
} else {
|
||||
const block_q8_0 * row_blocks = reinterpret_cast<const block_q8_0 *>(row_data);
|
||||
for (size_t block = 0; block < k_blocks; ++block) {
|
||||
const block_q8_0 & blk = row_blocks[block];
|
||||
const float d = GGML_FP16_TO_FP32(blk.d);
|
||||
for (size_t l = 0; l < QK8_0; ++l) {
|
||||
const size_t linear_idx = block * QK8_0 + l;
|
||||
if (linear_idx >= k) {
|
||||
break;
|
||||
}
|
||||
const float value = d * static_cast<float>(blk.qs[l]);
|
||||
int32_t q = scale > 0.0f ? static_cast<int32_t>(std::lround(value * inv_scale)) : 0;
|
||||
q = std::clamp(q, -127, 127);
|
||||
qdata[row * k + linear_idx] = static_cast<int8_t>(q);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1660,6 +1618,7 @@ public:
|
||||
const ggml_type rhs_type = kernels->rhs_type;
|
||||
const size_t block_len = rhs_type == GGML_TYPE_Q8_0 ? QK8_0 :
|
||||
rhs_type == GGML_TYPE_Q4_0 ? QK4_0 :
|
||||
rhs_type == GGML_TYPE_Q4_K ? GGML_KLEIDIAI_Q4_K_BL :
|
||||
rhs_type == GGML_TYPE_F32 ? 0 : SIZE_MAX;
|
||||
if (block_len == SIZE_MAX) {
|
||||
continue;
|
||||
@@ -1677,6 +1636,9 @@ public:
|
||||
rhs_info->pack_func_ex(1, n, k, nr, kr, sr, QK4_0, 0,
|
||||
static_cast<const uint8_t *>(data), nullptr, nullptr,
|
||||
dst_ptr, 0, ¶ms);
|
||||
} else if (rhs_type == GGML_TYPE_Q4_K) {
|
||||
rhs_info->pack_func_ex(1, n, k, nr, kr, sr, GGML_KLEIDIAI_Q4_K_BL, tensor->nb[1],
|
||||
data, nullptr, nullptr, dst_ptr, 0, nullptr);
|
||||
} else if (rhs_type == GGML_TYPE_Q8_0) {
|
||||
struct kai_rhs_pack_qsi8cx_params params;
|
||||
params.lhs_zero_point = 1;
|
||||
@@ -1719,7 +1681,6 @@ static enum ggml_status ggml_backend_cpu_kleidiai_buffer_init_tensor(ggml_backen
|
||||
tensor->extra = (void *) ggml::cpu::kleidiai::get_tensor_traits(buffer, tensor);
|
||||
|
||||
return GGML_STATUS_SUCCESS;
|
||||
GGML_UNUSED(buffer);
|
||||
}
|
||||
|
||||
static void ggml_backend_cpu_kleidiai_buffer_set_tensor(ggml_backend_buffer_t buffer, struct ggml_tensor * tensor,
|
||||
@@ -1762,7 +1723,9 @@ static size_t ggml_backend_cpu_kleidiai_buffer_type_get_alignment(ggml_backend_b
|
||||
static size_t ggml_backend_cpu_kleidiai_buffer_type_get_alloc_size(ggml_backend_buffer_type_t buft, const struct ggml_tensor * tensor) {
|
||||
GGML_UNUSED(buft);
|
||||
|
||||
if (tensor->type != GGML_TYPE_Q4_0 && tensor->type != GGML_TYPE_Q8_0 && tensor->type != GGML_TYPE_F32) {
|
||||
if (tensor->type != GGML_TYPE_Q4_0 && tensor->type != GGML_TYPE_Q4_K &&
|
||||
tensor->type != GGML_TYPE_Q8_0 && tensor->type != GGML_TYPE_Q6_K &&
|
||||
tensor->type != GGML_TYPE_F32) {
|
||||
return ggml_nbytes(tensor);
|
||||
}
|
||||
|
||||
@@ -1773,11 +1736,13 @@ static size_t ggml_backend_cpu_kleidiai_buffer_type_get_alloc_size(ggml_backend_
|
||||
cursor = align_up(cursor, GGML_KLEIDIAI_PACK_ALIGN);
|
||||
|
||||
std::array<ggml_kleidiai_kernels *, GGML_KLEIDIAI_MAX_KERNEL_SLOTS> kernel_chain;
|
||||
const bool want_q8 = tensor->type == GGML_TYPE_Q8_0;
|
||||
const bool want_q4k = tensor->type == GGML_TYPE_Q4_K;
|
||||
const bool want_q8 = tensor->type == GGML_TYPE_Q8_0 || tensor->type == GGML_TYPE_Q6_K;
|
||||
const bool want_f32 = tensor->type == GGML_TYPE_F32;
|
||||
const int slot_total = want_f32 ? kleidiai_collect_f32_chain(kernel_chain)
|
||||
: want_q8 ? kleidiai_collect_q8_chain(kernel_chain)
|
||||
: kleidiai_collect_q4_chain(kernel_chain);
|
||||
: want_q4k ? kleidiai_collect_q4k_chain(kernel_chain)
|
||||
: kleidiai_collect_q4_chain(kernel_chain);
|
||||
const bool allow_fallback = kleidiai_pack_fallback_allowed();
|
||||
|
||||
size_t slot_count = 0;
|
||||
@@ -1797,6 +1762,7 @@ static size_t ggml_backend_cpu_kleidiai_buffer_type_get_alloc_size(ggml_backend_
|
||||
|
||||
const ggml_type rhs_type = kernels->rhs_type;
|
||||
const size_t block_len = rhs_type == GGML_TYPE_Q4_0 ? QK4_0 :
|
||||
rhs_type == GGML_TYPE_Q4_K ? GGML_KLEIDIAI_Q4_K_BL :
|
||||
rhs_type == GGML_TYPE_Q8_0 ? QK8_0 :
|
||||
rhs_type == GGML_TYPE_F32 ? 0 : SIZE_MAX;
|
||||
if (block_len == SIZE_MAX) {
|
||||
@@ -1818,6 +1784,11 @@ static size_t ggml_backend_cpu_kleidiai_buffer_type_get_alloc_size(ggml_backend_
|
||||
namespace ggml::cpu::kleidiai {
|
||||
class extra_buffer_type : ggml::cpu::extra_buffer_type {
|
||||
bool supports_op(ggml_backend_dev_t, const struct ggml_tensor * op) override {
|
||||
// GET_ROWS needs the original weights in a native buffer.
|
||||
if (op->op != GGML_OP_MUL_MAT) {
|
||||
return false;
|
||||
}
|
||||
|
||||
std::array<ggml_kleidiai_kernels *, GGML_KLEIDIAI_MAX_KERNEL_SLOTS> kernel_chain;
|
||||
const int slot_total = kleidiai_collect_kernel_chain(op, kernel_chain);
|
||||
const bool src0_is_kleidiai =
|
||||
@@ -1826,8 +1797,9 @@ class extra_buffer_type : ggml::cpu::extra_buffer_type {
|
||||
op->src[0]->buffer->buft->context == this &&
|
||||
slot_total > 0;
|
||||
|
||||
if ((op->op == GGML_OP_MUL_MAT || op->op == GGML_OP_GET_ROWS) &&
|
||||
(op->src[0]->type == GGML_TYPE_Q4_0 || op->src[0]->type == GGML_TYPE_Q8_0 || op->src[0]->type == GGML_TYPE_F32) &&
|
||||
if ((op->src[0]->type == GGML_TYPE_Q4_0 || op->src[0]->type == GGML_TYPE_Q4_K ||
|
||||
op->src[0]->type == GGML_TYPE_Q8_0 || op->src[0]->type == GGML_TYPE_Q6_K ||
|
||||
op->src[0]->type == GGML_TYPE_F32) &&
|
||||
src0_is_kleidiai) {
|
||||
if (op->src[0]->type == GGML_TYPE_Q4_0 && ctx.kernels_q4 == nullptr) {
|
||||
return false;
|
||||
@@ -1835,6 +1807,12 @@ class extra_buffer_type : ggml::cpu::extra_buffer_type {
|
||||
if (op->src[0]->type == GGML_TYPE_Q8_0 && ctx.kernels_q8 == nullptr) {
|
||||
return false;
|
||||
}
|
||||
if (op->src[0]->type == GGML_TYPE_Q6_K && ctx.kernels_q8 == nullptr) {
|
||||
return false;
|
||||
}
|
||||
if (op->src[0]->type == GGML_TYPE_Q4_K && ctx.kernels_q4k == nullptr) {
|
||||
return false;
|
||||
}
|
||||
if (op->src[0]->type == GGML_TYPE_F32 && ctx.kernels_f32 == nullptr) {
|
||||
return false;
|
||||
}
|
||||
@@ -1842,8 +1820,18 @@ class extra_buffer_type : ggml::cpu::extra_buffer_type {
|
||||
return false;
|
||||
}
|
||||
|
||||
if (op->src[0]->type == GGML_TYPE_Q4_0 || op->src[0]->type == GGML_TYPE_Q8_0) {
|
||||
if ((op->src[1]->type == GGML_TYPE_F32 || op->src[1]->type == GGML_TYPE_I32) &&
|
||||
if (op->src[0]->type == GGML_TYPE_Q4_K) {
|
||||
if (op->op != GGML_OP_MUL_MAT || op->src[1]->type != GGML_TYPE_F32 || op->type != GGML_TYPE_F32) {
|
||||
return false;
|
||||
}
|
||||
return ggml_ne(op->src[1], 2) == 1 && ggml_ne(op->src[1], 3) == 1 &&
|
||||
op->src[1]->nb[0] == sizeof(float) &&
|
||||
(op->src[1]->ne[1] == 1 || op->src[1]->nb[1] == op->src[1]->ne[0] * sizeof(float));
|
||||
}
|
||||
|
||||
if (op->src[0]->type == GGML_TYPE_Q4_0 || op->src[0]->type == GGML_TYPE_Q8_0 ||
|
||||
op->src[0]->type == GGML_TYPE_Q6_K) {
|
||||
if (op->src[1]->type == GGML_TYPE_F32 &&
|
||||
ggml_ne(op->src[1], 3) == 1) {
|
||||
return true;
|
||||
}
|
||||
@@ -1861,21 +1849,22 @@ class extra_buffer_type : ggml::cpu::extra_buffer_type {
|
||||
}
|
||||
|
||||
ggml::cpu::tensor_traits * get_tensor_traits(const struct ggml_tensor * op) override {
|
||||
if (op->op == GGML_OP_MUL_MAT || op->op == GGML_OP_GET_ROWS) {
|
||||
if (op->op == GGML_OP_MUL_MAT) {
|
||||
if (op->src[0]->buffer && op->src[0]->buffer->buft->context == this) {
|
||||
return (ggml::cpu::tensor_traits *) op->src[0]->extra;
|
||||
} else {
|
||||
// KleidiAI only has kernels for Q4_0 and Q8_0. For a quantized weight of any
|
||||
// KleidiAI only has kernels for Q4_0, Q4_K, Q8_0 and Q6_K. For a quantized weight of any
|
||||
// other type (K-quants, IQ) it declines the op and returns nullptr below, so
|
||||
// KleidiAI does not accelerate it. Another CPU backend may still take the op,
|
||||
// and this can run during graph planning, so the message says what KleidiAI
|
||||
// did rather than what ends up executing. Warn once per process.
|
||||
if (ggml_is_quantized(op->src[0]->type) &&
|
||||
op->src[0]->type != GGML_TYPE_Q4_0 && op->src[0]->type != GGML_TYPE_Q8_0) {
|
||||
op->src[0]->type != GGML_TYPE_Q4_0 && op->src[0]->type != GGML_TYPE_Q4_K &&
|
||||
op->src[0]->type != GGML_TYPE_Q8_0 && op->src[0]->type != GGML_TYPE_Q6_K) {
|
||||
static std::atomic<bool> warned(false);
|
||||
if (!warned.exchange(true)) {
|
||||
GGML_LOG_WARN("kleidiai: no kernel for tensor type %s, not accelerated by KleidiAI "
|
||||
"(kernels available for Q4_0 and Q8_0)\n",
|
||||
"(kernels available for Q4_0, Q4_K, Q8_0 and Q6_K)\n",
|
||||
ggml_type_name(op->src[0]->type));
|
||||
}
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user