opencl: add coopmat GEMM for Adreno 850 (Q4_0, Q5_0, Q8_0, Q4_K, Q5_K, Q6_K) (#32686)

* opencl: add Q4_0 coopmat GEMM for Adreno 850

Assisted-by: Claude Opus 5.5

* opencl: add Q5_0 coopmat GEMM for Adreno 850

* opencl: add Q8_0 coopmat GEMM for Adreno 850

* opencl: add Q4_K coopmat GEMM for Adreno 850

* opencl: add Q5_K coopmat GEMM for Adreno 850

* opencl: add Q6_K coopmat GEMM for Adreno 850
This commit is contained in:
shaofeiqi authored and GitHub committed 2026-11-22 00:01:14 -08:00
1 parent 8a97fa98ab
commit 1fb8a26960
9 files changed
+1190 -4

No files matched your search

+7
View File
@@ -227,6 +227,13 @@ set(GGML_OPENCL_KERNELS
gemm_coopk_q4_0_q8_1_dp4a
gemm_coopk_q4_k_q8_1_dp4a
gemm_coopk_q6_k_q8_1_dp4a
pack_b_f16_coopmat
gemm_noshuffle_q4_0_f32_coopmat
gemm_noshuffle_q5_0_f32_coopmat
gemm_noshuffle_q8_0_f32_coopmat
gemm_noshuffle_q4_k_f32_coopmat
gemm_noshuffle_q5_k_f32_coopmat
gemm_noshuffle_q6_k_f32_coopmat
quant_a_q8_1
gemv_moe_q4_k_f32_ns
gemm_moe_q5_k_f32_ns
+219 -4
View File
@@ -1495,6 +1495,13 @@ struct ggml_backend_opencl_context {
cl_kernel kernel_gemm_noshuffle_iq4_nl_q8_1_dp4a = nullptr; // dp4a (int8) dense IQ4_NL prefill GEMM
cl_kernel kernel_gemm_noshuffle_q4_0_q8_1_dp4a = nullptr; // dp4a (int8) dense q4_0 prefill GEMM
cl_kernel kernel_gemm_noshuffle_q4_0_q8_1_dp4a_act_vload4 = nullptr;
cl_kernel kernel_pack_b_f16_coopmat = nullptr;
cl_kernel kernel_gemm_noshuffle_q4_0_f32_coopmat = nullptr;
cl_kernel kernel_gemm_noshuffle_q5_0_f32_coopmat = nullptr;
cl_kernel kernel_gemm_noshuffle_q8_0_f32_coopmat = nullptr;
cl_kernel kernel_gemm_noshuffle_q4_k_f32_coopmat = nullptr;
cl_kernel kernel_gemm_noshuffle_q5_k_f32_coopmat = nullptr;
cl_kernel kernel_gemm_noshuffle_q6_k_f32_coopmat = nullptr;
cl_kernel kernel_gemm_coopk_q4_0_q8_1_dp4a_c4 = nullptr;
cl_kernel kernel_gemm_coopk_q4_0_q8_1_dp4a_c2 = nullptr;
int nsg_coopk_dp4a_q4_0 = 8;
@@ -1979,6 +1986,137 @@ static void load_cl_kernels_gemm_coopk(ggml_backend_opencl_context * backend_ctx
}
}
static void load_cl_kernels_coopmat(ggml_backend_opencl_context * backend_ctx) {
if (backend_ctx->gpu_family != GPU_FAMILY::ADRENO || !strstr(backend_ctx->device_name.c_str(), "850")) {
return;
}
cl_int err;
const std::string opts = "-cl-std=CL" + std::to_string(backend_ctx->opencl_c_version.major) + "." +
std::to_string(backend_ctx->opencl_c_version.minor) +
(backend_ctx->adreno_use_large_buffer ? " -qcom-enable-large-buffer" : "");
// pack_b_f16_coopmat
{
#ifdef GGML_OPENCL_EMBED_KERNELS
const std::string kernel_src {
#include "pack_b_f16_coopmat.cl.h"
};
#else
const std::string kernel_src = read_file("pack_b_f16_coopmat.cl");
#endif
cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), opts, false);
if (!prog) {
return;
}
CL_CHECK((backend_ctx->kernel_pack_b_f16_coopmat = clCreateKernel(prog, "kernel_pack_b_f16_coopmat", &err), err));
CL_CHECK(clReleaseProgram(prog));
GGML_LOG_CONT(".");
}
// gemm_noshuffle_q4_0_f32_coopmat
{
#ifdef GGML_OPENCL_EMBED_KERNELS
const std::string kernel_src {
#include "gemm_noshuffle_q4_0_f32_coopmat.cl.h"
};
#else
const std::string kernel_src = read_file("gemm_noshuffle_q4_0_f32_coopmat.cl");
#endif
cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), opts, false);
if (prog) {
CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q4_0_f32_coopmat = clCreateKernel(prog, "kernel_gemm_noshuffle_q4_0_f32_coopmat", &err), err));
CL_CHECK(clReleaseProgram(prog));
GGML_LOG_CONT(".");
}
}
// gemm_noshuffle_q5_0_f32_coopmat
{
#ifdef GGML_OPENCL_EMBED_KERNELS
const std::string kernel_src {
#include "gemm_noshuffle_q5_0_f32_coopmat.cl.h"
};
#else
const std::string kernel_src = read_file("gemm_noshuffle_q5_0_f32_coopmat.cl");
#endif
cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), opts, false);
if (prog) {
CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q5_0_f32_coopmat = clCreateKernel(prog, "kernel_gemm_noshuffle_q5_0_f32_coopmat", &err), err));
CL_CHECK(clReleaseProgram(prog));
GGML_LOG_CONT(".");
}
}
// gemm_noshuffle_q8_0_f32_coopmat
{
#ifdef GGML_OPENCL_EMBED_KERNELS
const std::string kernel_src {
#include "gemm_noshuffle_q8_0_f32_coopmat.cl.h"
};
#else
const std::string kernel_src = read_file("gemm_noshuffle_q8_0_f32_coopmat.cl");
#endif
cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), opts, false);
if (prog) {
CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q8_0_f32_coopmat = clCreateKernel(prog, "kernel_gemm_noshuffle_q8_0_f32_coopmat", &err), err));
CL_CHECK(clReleaseProgram(prog));
GGML_LOG_CONT(".");
}
}
// gemm_noshuffle_q4_k_f32_coopmat
{
#ifdef GGML_OPENCL_EMBED_KERNELS
const std::string kernel_src {
#include "gemm_noshuffle_q4_k_f32_coopmat.cl.h"
};
#else
const std::string kernel_src = read_file("gemm_noshuffle_q4_k_f32_coopmat.cl");
#endif
cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), opts, false);
if (prog) {
CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q4_k_f32_coopmat = clCreateKernel(prog, "kernel_gemm_noshuffle_q4_k_f32_coopmat", &err), err));
CL_CHECK(clReleaseProgram(prog));
GGML_LOG_CONT(".");
}
}
// gemm_noshuffle_q5_k_f32_coopmat
{
#ifdef GGML_OPENCL_EMBED_KERNELS
const std::string kernel_src {
#include "gemm_noshuffle_q5_k_f32_coopmat.cl.h"
};
#else
const std::string kernel_src = read_file("gemm_noshuffle_q5_k_f32_coopmat.cl");
#endif
cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), opts, false);
if (prog) {
CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q5_k_f32_coopmat = clCreateKernel(prog, "kernel_gemm_noshuffle_q5_k_f32_coopmat", &err), err));
CL_CHECK(clReleaseProgram(prog));
GGML_LOG_CONT(".");
}
}
// gemm_noshuffle_q6_k_f32_coopmat
{
#ifdef GGML_OPENCL_EMBED_KERNELS
const std::string kernel_src {
#include "gemm_noshuffle_q6_k_f32_coopmat.cl.h"
};
#else
const std::string kernel_src = read_file("gemm_noshuffle_q6_k_f32_coopmat.cl");
#endif
cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), opts, false);
if (prog) {
CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q6_k_f32_coopmat = clCreateKernel(prog, "kernel_gemm_noshuffle_q6_k_f32_coopmat", &err), err));
CL_CHECK(clReleaseProgram(prog));
GGML_LOG_CONT(".");
}
}
}
#endif
static std::string ggml_opencl_make_compile_opts(const ggml_backend_opencl_context * backend_ctx) {
@@ -7813,6 +7951,7 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) {
}
}
load_cl_kernels_gemm_coopk(backend_ctx);
load_cl_kernels_coopmat(backend_ctx);
#endif // GGML_OPENCL_USE_ADRENO_KERNELS
GGML_LOG_CONT("\n");
backend_ctx->kernels_loaded = true;
@@ -11739,7 +11878,8 @@ inline bool use_q4_0_bin_kernels(const ggml_backend_opencl_context *backend_ctx,
#ifdef GGML_OPENCL_USE_ADRENO_KERNELS
if (!backend_ctx->kernel_gemv_noshuffle_q4_0_f32_32b_trans ||
!(backend_ctx->kernel_gemm_noshuffle_q4_0_f32_32b_trans_ila_a8_bin ||
backend_ctx->kernel_gemm_noshuffle_q4_0_f32_32b_trans_ila_a7_bin)) {
backend_ctx->kernel_gemm_noshuffle_q4_0_f32_32b_trans_ila_a7_bin ||
backend_ctx->kernel_gemm_noshuffle_q4_0_f32_coopmat)) {
return false;
}
@@ -11893,7 +12033,8 @@ inline bool use_q6_k_bin_kernels(const ggml_backend_opencl_context *backend_ctx,
#ifdef GGML_OPENCL_USE_ADRENO_KERNELS
if (!backend_ctx->kernel_gemv_noshuffle_q6_k_f32_32b_trans ||
!(backend_ctx->kernel_gemm_noshuffle_q6_k_f32_32b_trans_ila_a8_bin ||
backend_ctx->kernel_gemm_noshuffle_q6_k_f32_32b_trans_ila_a7_bin)) {
backend_ctx->kernel_gemm_noshuffle_q6_k_f32_32b_trans_ila_a7_bin ||
backend_ctx->kernel_gemm_noshuffle_q6_k_f32_coopmat)) {
return false;
}
// bin kernels require ne2 == 1 and ne3 == 1 for weights
@@ -11911,7 +12052,8 @@ inline bool use_q4_k_bin_kernels(const ggml_backend_opencl_context *backend_ctx,
#ifdef GGML_OPENCL_USE_ADRENO_KERNELS
if (!backend_ctx->kernel_gemv_noshuffle_q4_k_f32_32b_trans ||
!(backend_ctx->kernel_gemm_noshuffle_q4_k_f32_32b_trans_ila_a8_bin ||
backend_ctx->kernel_gemm_noshuffle_q4_k_f32_32b_trans_ila_a7_bin)) {
backend_ctx->kernel_gemm_noshuffle_q4_k_f32_32b_trans_ila_a7_bin ||
backend_ctx->kernel_gemm_noshuffle_q4_k_f32_coopmat)) {
return false;
}
// bin kernels require ne2 == 1 and ne3 == 1 for weights
@@ -11929,7 +12071,8 @@ inline bool use_q5_k_bin_kernels(const ggml_backend_opencl_context *backend_ctx,
#ifdef GGML_OPENCL_USE_ADRENO_KERNELS
if (!backend_ctx->kernel_gemv_noshuffle_q5_k_f32_32b_trans ||
!(backend_ctx->kernel_gemm_noshuffle_q5_k_f32_32b_trans_ila_a8_bin ||
backend_ctx->kernel_gemm_noshuffle_q5_k_f32_32b_trans_ila_a7_bin)) {
backend_ctx->kernel_gemm_noshuffle_q5_k_f32_32b_trans_ila_a7_bin ||
backend_ctx->kernel_gemm_noshuffle_q5_k_f32_coopmat)) {
return false;
}
return (tensor->ne[0] % 256 == 0) && (tensor->ne[1] % 64 == 0);
@@ -25619,6 +25762,48 @@ static void ggml_cl_mul_mat_q2_0_f32_adreno(ggml_backend_t backend, const ggml_t
CL_CHECK(clReleaseMemObject(b_sub_buf));
}
static bool ggml_cl_mul_mat_coopmat(ggml_backend_opencl_context * backend_ctx, cl_kernel kernel,
std::initializer_list<cl_mem> weights,
const ggml_tensor * src1, ggml_tensor * dst, int K, int M, int N) {
if (kernel == nullptr || M % 64 != 0 || N < 2) {
return false;
}
ggml_tensor_extra_cl * extra1 = (ggml_tensor_extra_cl *)src1->extra;
ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra;
cl_ulong offset1 = extra1->offset + src1->view_offs;
cl_ulong offsetd = extrad->offset + dst->view_offs;
const int n_tiles = CEIL_DIV(N, 64);
backend_ctx->prealloc_act_trans.allocate(backend_ctx->context, (size_t)n_tiles * 64 * K * sizeof(cl_half));
cl_kernel pk = backend_ctx->kernel_pack_b_f16_coopmat;
CL_CHECK(clSetKernelArg(pk, 0, sizeof(cl_mem), &extra1->data_device));
CL_CHECK(clSetKernelArg(pk, 1, sizeof(cl_ulong), &offset1));
CL_CHECK(clSetKernelArg(pk, 2, sizeof(cl_mem), &backend_ctx->prealloc_act_trans.buffer));
CL_CHECK(clSetKernelArg(pk, 3, sizeof(int), &K));
CL_CHECK(clSetKernelArg(pk, 4, sizeof(int), &N));
size_t p_local[2] = { 64, 4 };
size_t p_global[2] = { (size_t)n_tiles * 64, (size_t)CEIL_DIV(K, 64) * 4 };
backend_ctx->enqueue_ndrange_kernel(pk, 2, p_global, p_local, dst);
cl_uint ka = 0;
for (cl_mem w : weights) {
CL_CHECK(clSetKernelArg(kernel, ka++, sizeof(cl_mem), &w));
}
CL_CHECK(clSetKernelArg(kernel, ka++, sizeof(cl_mem), &backend_ctx->prealloc_act_trans.buffer));
CL_CHECK(clSetKernelArg(kernel, ka++, sizeof(cl_mem), &extrad->data_device));
CL_CHECK(clSetKernelArg(kernel, ka++, sizeof(cl_ulong), &offsetd));
CL_CHECK(clSetKernelArg(kernel, ka++, sizeof(int), &K));
CL_CHECK(clSetKernelArg(kernel, ka++, sizeof(int), &M));
CL_CHECK(clSetKernelArg(kernel, ka++, sizeof(int), &N));
size_t c_local[3] = { 64, 1, 4 };
size_t c_global[3] = { 64, (size_t)(M / 64), (size_t)CEIL_DIV(n_tiles, 4) * 4 };
backend_ctx->enqueue_ndrange_kernel(kernel, 3, c_global, c_local, dst);
return true;
}
static void ggml_cl_mul_mat_q4_0_f32_adreno_ila(ggml_backend_t backend, const ggml_tensor * src0,
const ggml_tensor * src1, ggml_tensor * dst) {
GGML_ASSERT(src0);
@@ -25934,6 +26119,11 @@ static void ggml_cl_mul_mat_q4_0_f32_adreno(ggml_backend_t backend, const ggml_t
const bool use_bin = use_q4_0_bin_kernels(backend_ctx, src0);
if (use_bin && ggml_cl_mul_mat_coopmat(backend_ctx, backend_ctx->kernel_gemm_noshuffle_q4_0_f32_coopmat,
{ extra0_q4_0->q, extra0_q4_0->d }, src1, dst, K, M, N)) {
return;
}
if (use_bin) {
if (use_q40_mc3) {
static bool warned = false;
@@ -26498,6 +26688,11 @@ static void ggml_cl_mul_mat_q5_0_f32_adreno(ggml_backend_t backend, const ggml_t
int N = ne1;
int K = ne00;
if (ggml_cl_mul_mat_coopmat(backend_ctx, backend_ctx->kernel_gemm_noshuffle_q5_0_f32_coopmat,
{ extra0_q5_0->qs, extra0_q5_0->qh, extra0_q5_0->d }, src1, dst, K, M, N)) {
return;
}
if (ne1 == 1) {
cl_mem qs_img = nullptr;
cl_mem b_sub_buf = nullptr;
@@ -27340,6 +27535,11 @@ static void ggml_cl_mul_mat_q8_0_f32_adreno(ggml_backend_t backend, const ggml_t
int N = ne1;
int K = ne00;
if (ggml_cl_mul_mat_coopmat(backend_ctx, backend_ctx->kernel_gemm_noshuffle_q8_0_f32_coopmat,
{ extra0_q8_0->q, extra0_q8_0->d }, src1, dst, K, M, N)) {
return;
}
// Multi-column verify GEMV for ne1 2..4, reuses the ne1 == 1 image setup.
// Default on for X2E, GGML_OPENCL_Q8_0_MC=0/1 forces it.
static const char * q8_mc_env = getenv("GGML_OPENCL_Q8_0_MC");
@@ -28321,6 +28521,11 @@ static void ggml_cl_mul_mat_q4_k_f32_adreno(ggml_backend_t backend, const ggml_t
const bool use_bin = use_q4_k_bin_kernels(backend_ctx, src0);
if (use_bin && ggml_cl_mul_mat_coopmat(backend_ctx, backend_ctx->kernel_gemm_noshuffle_q4_k_f32_coopmat,
{ extra0_q4_k->q, extra0_q4_k->s, extra0_q4_k->d, extra0_q4_k->dm }, src1, dst, K, M, N)) {
return;
}
if (use_bin) {
if (use_mc3) {
static bool warned = false;
@@ -29231,6 +29436,11 @@ static void ggml_cl_mul_mat_q6_K_f32_adreno(ggml_backend_t backend, const ggml_t
const bool use_bin = use_q6_k_bin_kernels(backend_ctx, src0);
if (use_bin && ggml_cl_mul_mat_coopmat(backend_ctx, backend_ctx->kernel_gemm_noshuffle_q6_k_f32_coopmat,
{ extra0_q6_K->ql, extra0_q6_K->qh, extra0_q6_K->s, extra0_q6_K->d }, src1, dst, ne00, ne01, ne1)) {
return;
}
if (use_bin) {
if (use_q6k_mc3 || use_q6k_tiled_mc) {
static bool warned = false;
@@ -29971,6 +30181,11 @@ static void ggml_cl_mul_mat_q5_K_f32_adreno(ggml_backend_t backend, const ggml_t
const bool use_bin = use_q5_k_bin_kernels(backend_ctx, src0);
if (use_bin && ggml_cl_mul_mat_coopmat(backend_ctx, backend_ctx->kernel_gemm_noshuffle_q5_k_f32_coopmat,
{ extra0_q5_k->q, extra0_q5_k->qh, extra0_q5_k->s, extra0_q5_k->d, extra0_q5_k->dm }, src1, dst, K, M, N)) {
return;
}
if (use_bin) {
if (use_q5k_mc3) {
static bool warned = false;
@@ -0,0 +1,145 @@
#pragma OPENCL EXTENSION cl_khr_fp16 : enable
#pragma OPENCL EXTENSION cl_qcom_kernel_cooperative_matrix : enable
// Q4_0 x F32 GEMM with cl_qcom_kernel_cooperative_matrix: one 64x64 f16 C tile per wave,
// A dequantized in registers, B packed by kernel_pack_b_f16_coopmat.
typedef half mat_a_t __attribute__((qcom_coop_mat(CLK_COOPERATIVE_MATRIX_SCOPE_SUBGROUP_QCOM, 64, 16, CLK_COOPERATIVE_MATRIX_A_QCOM)));
typedef half mat_b_t __attribute__((qcom_coop_mat(CLK_COOPERATIVE_MATRIX_SCOPE_SUBGROUP_QCOM, 16, 64, CLK_COOPERATIVE_MATRIX_B_QCOM)));
typedef half mat_c_t __attribute__((qcom_coop_mat(CLK_COOPERATIVE_MATRIX_SCOPE_SUBGROUP_QCOM, 64, 64, CLK_COOPERATIVE_MATRIX_ACCUMULATOR_QCOM)));
typedef half half1024 __attribute__((ext_vector_type(1024)));
typedef half half4096 __attribute__((ext_vector_type(4096)));
typedef union {
half1024 v;
half16 h;
} a_row_t;
typedef union {
half4096 v;
struct {
half16 h0;
half16 h1;
half16 h2;
half16 h3;
} s;
} c_row_t;
inline half8 dequant8(uint v, half d) {
uint4 t;
t.s0 = ( v & 0x000F) | ((v << 12) & 0x000F0000);
t.s1 = ((v >> 8) & 0x000F) | ((v << 4) & 0x000F0000);
t.s2 = ((v >> 16) & 0x000F) | ((v >> 4) & 0x000F0000);
t.s3 = ((v >> 24) & 0x000F) | ((v >> 12) & 0x000F0000);
t |= (uint4)(0x64006400);
return (as_half8(t) - (half8)(1032.0h)) * d;
}
inline void store_c16(global float * p, half16 h, int j0, int nv, int M) {
float16 f = convert_float16(h);
if (j0 + 0 < nv) p[(ulong)(j0 + 0) * M] = f.s0;
if (j0 + 1 < nv) p[(ulong)(j0 + 1) * M] = f.s1;
if (j0 + 2 < nv) p[(ulong)(j0 + 2) * M] = f.s2;
if (j0 + 3 < nv) p[(ulong)(j0 + 3) * M] = f.s3;
if (j0 + 4 < nv) p[(ulong)(j0 + 4) * M] = f.s4;
if (j0 + 5 < nv) p[(ulong)(j0 + 5) * M] = f.s5;
if (j0 + 6 < nv) p[(ulong)(j0 + 6) * M] = f.s6;
if (j0 + 7 < nv) p[(ulong)(j0 + 7) * M] = f.s7;
if (j0 + 8 < nv) p[(ulong)(j0 + 8) * M] = f.s8;
if (j0 + 9 < nv) p[(ulong)(j0 + 9) * M] = f.s9;
if (j0 + 10 < nv) p[(ulong)(j0 + 10) * M] = f.sa;
if (j0 + 11 < nv) p[(ulong)(j0 + 11) * M] = f.sb;
if (j0 + 12 < nv) p[(ulong)(j0 + 12) * M] = f.sc;
if (j0 + 13 < nv) p[(ulong)(j0 + 13) * M] = f.sd;
if (j0 + 14 < nv) p[(ulong)(j0 + 14) * M] = f.se;
if (j0 + 15 < nv) p[(ulong)(j0 + 15) * M] = f.sf;
}
#define LOAD_Q(qv, dd, o) { \
qv.s0 = qp[(4 * (o) + 0) * M]; \
qv.s1 = qp[(4 * (o) + 1) * M]; \
qv.s2 = qp[(4 * (o) + 2) * M]; \
qv.s3 = qp[(4 * (o) + 3) * M]; \
dd = dp[(o) * M]; }
#define MMA_K32(qv, dd, b) { \
a_row_t ua0; \
a_row_t ua1; \
ua0.h = (half16)(dequant8(qv.s0, dd), dequant8(qv.s1, dd)); \
ua1.h = (half16)(dequant8(qv.s2, dd), dequant8(qv.s3, dd)); \
mat_a_t a0 = __builtin_astype(ua0.v, mat_a_t); \
mat_a_t a1 = __builtin_astype(ua1.v, mat_a_t); \
mat_b_t b0 = qcom_coop_mat_load((b), CLK_COOPERATIVE_MATRIX_LAYOUT_COLUMN_MAJOR_QCOM, 16); \
mat_b_t b1 = qcom_coop_mat_load((b) + 1024, CLK_COOPERATIVE_MATRIX_LAYOUT_COLUMN_MAJOR_QCOM, 16); \
acc = qcom_coop_mat_mulAdd(a0, b0, acc, CLK_COOPERATIVE_MATRIX_OPERAND_NONE_QCOM); \
acc = qcom_coop_mat_mulAdd(a1, b1, acc, CLK_COOPERATIVE_MATRIX_OPERAND_NONE_QCOM); }
kernel void kernel_gemm_noshuffle_q4_0_f32_coopmat(
global const uint * src0_q, // A: q4_0 quants, [K/8][M] u32
global const half * src0_d, // A: q4_0 scales, [K/32][M] f16
global const half * src1, // B: packed f16 tiles
global char * dst, // C: [N][M] f32
ulong offsetd,
int K,
int M,
int N
) {
const int lane = get_local_id(0);
const int row = get_global_id(1) * 64;
const int tn = get_global_id(2);
const int n_tiles = (N + 63) / 64;
if (tn >= n_tiles) {
return;
}
global const uint * qp = src0_q + row + lane;
global const half * dp = src0_d + row + lane;
global const half * bp = src1 + (ulong)tn * (K / 16) * 1024;
mat_c_t acc = qcom_coop_mat_init(0.0);
{
const int odd = (K / 32) & 1;
const int ob = odd ? 0 : 1;
uint4 qa;
uint4 qb;
half da;
half db;
LOAD_Q(qa, da, 0)
LOAD_Q(qb, db, ob)
da = odd ? 0.0h : da;
MMA_K32(qa, da, bp)
MMA_K32(qb, db, bp + ob * 2048)
qp += (ob + 1) * 4 * M;
dp += (ob + 1) * M;
bp += (ob + 1) * 2048;
}
#pragma unroll 1
for (int k0 = (K / 32) & 1 ? 32 : 64; k0 < K; k0 += 64) {
uint4 qa;
uint4 qb;
half da;
half db;
LOAD_Q(qa, da, 0)
LOAD_Q(qb, db, 1)
MMA_K32(qa, da, bp)
MMA_K32(qb, db, bp + 2048)
qp += 8 * M;
dp += 2 * M;
bp += 4096;
}
c_row_t uc;
uc.v = __builtin_astype(acc, half4096);
global float * p = (global float *)(dst + offsetd) + (ulong)tn * 64 * M + row + lane;
const int nv = N - tn * 64;
store_c16(p, uc.s.h0, 0, nv, M);
store_c16(p, uc.s.h1, 16, nv, M);
store_c16(p, uc.s.h2, 32, nv, M);
store_c16(p, uc.s.h3, 48, nv, M);
}
@@ -0,0 +1,148 @@
#pragma OPENCL EXTENSION cl_khr_fp16 : enable
#pragma OPENCL EXTENSION cl_qcom_kernel_cooperative_matrix : enable
// Q4_K x F32 GEMM with cl_qcom_kernel_cooperative_matrix: one 64x64 f16 C tile per wave,
// A dequantized in registers, B packed by kernel_pack_b_f16_coopmat.
#define K_SCALE_SIZE 12
typedef half mat_a_t __attribute__((qcom_coop_mat(CLK_COOPERATIVE_MATRIX_SCOPE_SUBGROUP_QCOM, 64, 16, CLK_COOPERATIVE_MATRIX_A_QCOM)));
typedef half mat_b_t __attribute__((qcom_coop_mat(CLK_COOPERATIVE_MATRIX_SCOPE_SUBGROUP_QCOM, 16, 64, CLK_COOPERATIVE_MATRIX_B_QCOM)));
typedef half mat_c_t __attribute__((qcom_coop_mat(CLK_COOPERATIVE_MATRIX_SCOPE_SUBGROUP_QCOM, 64, 64, CLK_COOPERATIVE_MATRIX_ACCUMULATOR_QCOM)));
typedef half half1024 __attribute__((ext_vector_type(1024)));
typedef half half4096 __attribute__((ext_vector_type(4096)));
typedef union {
half1024 v;
half16 h;
} a_row_t;
typedef union {
half4096 v;
struct {
half16 h0;
half16 h1;
half16 h2;
half16 h3;
} s;
} c_row_t;
inline half8 dequant8(uint v, half scale, half nminv) {
uint4 t;
t.s0 = ( v & 0x000F) | ((v << 12) & 0x000F0000);
t.s1 = ((v >> 8) & 0x000F) | ((v << 4) & 0x000F0000);
t.s2 = ((v >> 16) & 0x000F) | ((v >> 4) & 0x000F0000);
t.s3 = ((v >> 24) & 0x000F) | ((v >> 12) & 0x000F0000);
t |= (uint4)(0x64006400);
return (as_half8(t) - (half8)(1024.0h)) * scale + nminv;
}
inline void store_c16(global float * p, half16 h, int j0, int nv, int M) {
float16 f = convert_float16(h);
if (j0 + 0 < nv) p[(ulong)(j0 + 0) * M] = f.s0;
if (j0 + 1 < nv) p[(ulong)(j0 + 1) * M] = f.s1;
if (j0 + 2 < nv) p[(ulong)(j0 + 2) * M] = f.s2;
if (j0 + 3 < nv) p[(ulong)(j0 + 3) * M] = f.s3;
if (j0 + 4 < nv) p[(ulong)(j0 + 4) * M] = f.s4;
if (j0 + 5 < nv) p[(ulong)(j0 + 5) * M] = f.s5;
if (j0 + 6 < nv) p[(ulong)(j0 + 6) * M] = f.s6;
if (j0 + 7 < nv) p[(ulong)(j0 + 7) * M] = f.s7;
if (j0 + 8 < nv) p[(ulong)(j0 + 8) * M] = f.s8;
if (j0 + 9 < nv) p[(ulong)(j0 + 9) * M] = f.s9;
if (j0 + 10 < nv) p[(ulong)(j0 + 10) * M] = f.sa;
if (j0 + 11 < nv) p[(ulong)(j0 + 11) * M] = f.sb;
if (j0 + 12 < nv) p[(ulong)(j0 + 12) * M] = f.sc;
if (j0 + 13 < nv) p[(ulong)(j0 + 13) * M] = f.sd;
if (j0 + 14 < nv) p[(ulong)(j0 + 14) * M] = f.se;
if (j0 + 15 < nv) p[(ulong)(j0 + 15) * M] = f.sf;
}
#define MMA_K32(q0, q1, q2, q3, sc, nm, b) { \
a_row_t ua0; \
a_row_t ua1; \
ua0.h = (half16)(dequant8(q0, sc, nm), dequant8(q1, sc, nm)); \
ua1.h = (half16)(dequant8(q2, sc, nm), dequant8(q3, sc, nm)); \
mat_a_t a0 = __builtin_astype(ua0.v, mat_a_t); \
mat_a_t a1 = __builtin_astype(ua1.v, mat_a_t); \
mat_b_t b0 = qcom_coop_mat_load((b), CLK_COOPERATIVE_MATRIX_LAYOUT_COLUMN_MAJOR_QCOM, 16); \
mat_b_t b1 = qcom_coop_mat_load((b) + 1024, CLK_COOPERATIVE_MATRIX_LAYOUT_COLUMN_MAJOR_QCOM, 16); \
acc = qcom_coop_mat_mulAdd(a0, b0, acc, CLK_COOPERATIVE_MATRIX_OPERAND_NONE_QCOM); \
acc = qcom_coop_mat_mulAdd(a1, b1, acc, CLK_COOPERATIVE_MATRIX_OPERAND_NONE_QCOM); }
#define STEP64(sa, na, sb_, nb_) { \
uint8 qv; \
qv.s0 = qp[0]; \
qv.s1 = qp[M]; \
qv.s2 = qp[2 * M]; \
qv.s3 = qp[3 * M]; \
qv.s4 = qp[4 * M]; \
qv.s5 = qp[5 * M]; \
qv.s6 = qp[6 * M]; \
qv.s7 = qp[7 * M]; \
qp += 8 * M; \
MMA_K32(qv.s0, qv.s1, qv.s2, qv.s3, sa, na, bp) \
MMA_K32(qv.s4, qv.s5, qv.s6, qv.s7, sb_, nb_, bp + 2048) \
bp += 4096; }
kernel void kernel_gemm_noshuffle_q4_k_f32_coopmat(
global const uint * src0_q, // A: q4_K quants, [K/8][M] u32
global const uchar * src0_s, // A: 6-bit scale/min codes, [K/256*12][M] u8
global const half * src0_d, // A: per-superblock scale, [K/256][M] f16
global const half * src0_dm, // A: per-superblock min, [K/256][M] f16
global const half * src1, // B: packed f16 tiles
global char * dst, // C: [N][M] f32
ulong offsetd,
int K,
int M,
int N
) {
const int lane = get_local_id(0);
const int row = get_global_id(1) * 64;
const int tn = get_global_id(2);
const int n_tiles = (N + 63) / 64;
if (tn >= n_tiles) {
return;
}
const int lrow = row + lane;
global const uint * qp = src0_q + lrow;
global const half * bp = src1 + (ulong)tn * (K / 16) * 1024;
mat_c_t acc = qcom_coop_mat_init(0.0);
#pragma unroll 1
for (int k0 = 0; k0 < K; k0 += 128) {
const int sb = k0 >> 8;
const half dd = src0_d [(ulong)sb * M + lrow];
const half dmm = src0_dm[(ulong)sb * M + lrow];
global const uchar * sp = src0_s + (ulong)(sb * K_SCALE_SIZE) * M + lrow;
const uchar4 c0 = (uchar4)(sp[0], sp[M], sp[2 * M], sp[3 * M]);
const uchar4 c1 = (uchar4)(sp[4 * M], sp[5 * M], sp[6 * M], sp[7 * M]);
const uchar4 c2 = (uchar4)(sp[8 * M], sp[9 * M], sp[10 * M], sp[11 * M]);
const uchar4 sl = c0 & (uchar)0x3F;
const uchar4 ml = c1 & (uchar)0x3F;
const uchar4 sh = (c2 & (uchar)0x0F) | ((c0 >> (uchar)2) & (uchar)0x30);
const uchar4 mh = (c2 >> (uchar)4) | ((c1 >> (uchar)2) & (uchar)0x30);
const int hi = (k0 >> 7) & 1;
const half4 S = dd * convert_half4(hi ? sh : sl);
const half4 Nm = -(dmm * convert_half4(hi ? mh : ml));
STEP64(S.s0, Nm.s0, S.s1, Nm.s1)
STEP64(S.s2, Nm.s2, S.s3, Nm.s3)
}
c_row_t uc;
uc.v = __builtin_astype(acc, half4096);
global float * p = (global float *)(dst + offsetd) + (ulong)tn * 64 * M + row + lane;
const int nv = N - tn * 64;
store_c16(p, uc.s.h0, 0, nv, M);
store_c16(p, uc.s.h1, 16, nv, M);
store_c16(p, uc.s.h2, 32, nv, M);
store_c16(p, uc.s.h3, 48, nv, M);
}
@@ -0,0 +1,157 @@
#pragma OPENCL EXTENSION cl_khr_fp16 : enable
#pragma OPENCL EXTENSION cl_qcom_kernel_cooperative_matrix : enable
// Q5_0 x F32 GEMM with cl_qcom_kernel_cooperative_matrix: one 64x64 f16 C tile per wave,
// A dequantized in registers, B packed by kernel_pack_b_f16_coopmat.
typedef half mat_a_t __attribute__((qcom_coop_mat(CLK_COOPERATIVE_MATRIX_SCOPE_SUBGROUP_QCOM, 64, 16, CLK_COOPERATIVE_MATRIX_A_QCOM)));
typedef half mat_b_t __attribute__((qcom_coop_mat(CLK_COOPERATIVE_MATRIX_SCOPE_SUBGROUP_QCOM, 16, 64, CLK_COOPERATIVE_MATRIX_B_QCOM)));
typedef half mat_c_t __attribute__((qcom_coop_mat(CLK_COOPERATIVE_MATRIX_SCOPE_SUBGROUP_QCOM, 64, 64, CLK_COOPERATIVE_MATRIX_ACCUMULATOR_QCOM)));
typedef half half1024 __attribute__((ext_vector_type(1024)));
typedef half half4096 __attribute__((ext_vector_type(4096)));
typedef union {
half1024 v;
half16 h;
} a_row_t;
typedef union {
half4096 v;
struct {
half16 h0;
half16 h1;
half16 h2;
half16 h3;
} s;
} c_row_t;
inline half8 dequant8(uint v, uint h, half d) {
uint4 t;
t.s0 = ( v & 0x000F) | ((v << 12) & 0x000F0000) | ((h << 4) & 0x0010) | ((h << 19) & 0x00100000);
t.s1 = ((v >> 8) & 0x000F) | ((v << 4) & 0x000F0000) | ((h << 2) & 0x0010) | ((h << 17) & 0x00100000);
t.s2 = ((v >> 16) & 0x000F) | ((v >> 4) & 0x000F0000) | ( h & 0x0010) | ((h << 15) & 0x00100000);
t.s3 = ((v >> 24) & 0x000F) | ((v >> 12) & 0x000F0000) | ((h >> 2) & 0x0010) | ((h << 13) & 0x00100000);
t |= (uint4)(0x64006400);
return (as_half8(t) - (half8)(1040.0h)) * d;
}
inline void store_c16(global float * p, half16 h, int j0, int nv, int M) {
float16 f = convert_float16(h);
if (j0 + 0 < nv) p[(ulong)(j0 + 0) * M] = f.s0;
if (j0 + 1 < nv) p[(ulong)(j0 + 1) * M] = f.s1;
if (j0 + 2 < nv) p[(ulong)(j0 + 2) * M] = f.s2;
if (j0 + 3 < nv) p[(ulong)(j0 + 3) * M] = f.s3;
if (j0 + 4 < nv) p[(ulong)(j0 + 4) * M] = f.s4;
if (j0 + 5 < nv) p[(ulong)(j0 + 5) * M] = f.s5;
if (j0 + 6 < nv) p[(ulong)(j0 + 6) * M] = f.s6;
if (j0 + 7 < nv) p[(ulong)(j0 + 7) * M] = f.s7;
if (j0 + 8 < nv) p[(ulong)(j0 + 8) * M] = f.s8;
if (j0 + 9 < nv) p[(ulong)(j0 + 9) * M] = f.s9;
if (j0 + 10 < nv) p[(ulong)(j0 + 10) * M] = f.sa;
if (j0 + 11 < nv) p[(ulong)(j0 + 11) * M] = f.sb;
if (j0 + 12 < nv) p[(ulong)(j0 + 12) * M] = f.sc;
if (j0 + 13 < nv) p[(ulong)(j0 + 13) * M] = f.sd;
if (j0 + 14 < nv) p[(ulong)(j0 + 14) * M] = f.se;
if (j0 + 15 < nv) p[(ulong)(j0 + 15) * M] = f.sf;
}
#define LOAD_Q(qv, hv, dd, o) { \
qv.s0 = (uint)qp[(8 * (o) + 0) * M] | ((uint)qp[(8 * (o) + 1) * M] << 16); \
qv.s1 = (uint)qp[(8 * (o) + 2) * M] | ((uint)qp[(8 * (o) + 3) * M] << 16); \
qv.s2 = (uint)qp[(8 * (o) + 4) * M] | ((uint)qp[(8 * (o) + 5) * M] << 16); \
qv.s3 = (uint)qp[(8 * (o) + 6) * M] | ((uint)qp[(8 * (o) + 7) * M] << 16); \
hv.s0 = hp[(4 * (o) + 0) * M]; \
hv.s1 = hp[(4 * (o) + 1) * M]; \
hv.s2 = hp[(4 * (o) + 2) * M]; \
hv.s3 = hp[(4 * (o) + 3) * M]; \
dd = dp[(o) * M]; }
#define MMA_K32(qv, hv, dd, b) { \
a_row_t ua0; \
a_row_t ua1; \
ua0.h = (half16)(dequant8(qv.s0, hv.s0, dd), dequant8(qv.s1, hv.s1, dd)); \
ua1.h = (half16)(dequant8(qv.s2, hv.s2, dd), dequant8(qv.s3, hv.s3, dd)); \
mat_a_t a0 = __builtin_astype(ua0.v, mat_a_t); \
mat_a_t a1 = __builtin_astype(ua1.v, mat_a_t); \
mat_b_t b0 = qcom_coop_mat_load((b), CLK_COOPERATIVE_MATRIX_LAYOUT_COLUMN_MAJOR_QCOM, 16); \
mat_b_t b1 = qcom_coop_mat_load((b) + 1024, CLK_COOPERATIVE_MATRIX_LAYOUT_COLUMN_MAJOR_QCOM, 16); \
acc = qcom_coop_mat_mulAdd(a0, b0, acc, CLK_COOPERATIVE_MATRIX_OPERAND_NONE_QCOM); \
acc = qcom_coop_mat_mulAdd(a1, b1, acc, CLK_COOPERATIVE_MATRIX_OPERAND_NONE_QCOM); }
kernel void kernel_gemm_noshuffle_q5_0_f32_coopmat(
global const ushort * src0_q, // A: q5_0 quants, [K/4][M] u16
global const uchar * src0_qh, // A: q5_0 high bits, [K/8][M] u8
global const half * src0_d, // A: q5_0 scales, [K/32][M] f16
global const half * src1, // B: packed f16 tiles
global char * dst, // C: [N][M] f32
ulong offsetd,
int K,
int M,
int N
) {
const int lane = get_local_id(0);
const int row = get_global_id(1) * 64;
const int tn = get_global_id(2);
const int n_tiles = (N + 63) / 64;
if (tn >= n_tiles) {
return;
}
global const ushort * qp = src0_q + row + lane;
global const uchar * hp = src0_qh + row + lane;
global const half * dp = src0_d + row + lane;
global const half * bp = src1 + (ulong)tn * (K / 16) * 1024;
mat_c_t acc = qcom_coop_mat_init(0.0);
{
const int odd = (K / 32) & 1;
const int ob = odd ? 0 : 1;
uint4 qa;
uint4 qb;
uint4 ha;
uint4 hb;
half da;
half db;
LOAD_Q(qa, ha, da, 0)
LOAD_Q(qb, hb, db, ob)
da = odd ? 0.0h : da;
MMA_K32(qa, ha, da, bp)
MMA_K32(qb, hb, db, bp + ob * 2048)
qp += (ob + 1) * 8 * M;
hp += (ob + 1) * 4 * M;
dp += (ob + 1) * M;
bp += (ob + 1) * 2048;
}
#pragma unroll 1
for (int k0 = (K / 32) & 1 ? 32 : 64; k0 < K; k0 += 64) {
uint4 qa;
uint4 qb;
uint4 ha;
uint4 hb;
half da;
half db;
LOAD_Q(qa, ha, da, 0)
LOAD_Q(qb, hb, db, 1)
MMA_K32(qa, ha, da, bp)
MMA_K32(qb, hb, db, bp + 2048)
qp += 16 * M;
hp += 8 * M;
dp += 2 * M;
bp += 4096;
}
c_row_t uc;
uc.v = __builtin_astype(acc, half4096);
global float * p = (global float *)(dst + offsetd) + (ulong)tn * 64 * M + row + lane;
const int nv = N - tn * 64;
store_c16(p, uc.s.h0, 0, nv, M);
store_c16(p, uc.s.h1, 16, nv, M);
store_c16(p, uc.s.h2, 32, nv, M);
store_c16(p, uc.s.h3, 48, nv, M);
}
@@ -0,0 +1,164 @@
#pragma OPENCL EXTENSION cl_khr_fp16 : enable
#pragma OPENCL EXTENSION cl_qcom_kernel_cooperative_matrix : enable
// Q5_K x F32 GEMM with cl_qcom_kernel_cooperative_matrix: one 64x64 f16 C tile per wave,
// A dequantized in registers, B packed by kernel_pack_b_f16_coopmat.
#define K_SCALE_SIZE 12
typedef half mat_a_t __attribute__((qcom_coop_mat(CLK_COOPERATIVE_MATRIX_SCOPE_SUBGROUP_QCOM, 64, 16, CLK_COOPERATIVE_MATRIX_A_QCOM)));
typedef half mat_b_t __attribute__((qcom_coop_mat(CLK_COOPERATIVE_MATRIX_SCOPE_SUBGROUP_QCOM, 16, 64, CLK_COOPERATIVE_MATRIX_B_QCOM)));
typedef half mat_c_t __attribute__((qcom_coop_mat(CLK_COOPERATIVE_MATRIX_SCOPE_SUBGROUP_QCOM, 64, 64, CLK_COOPERATIVE_MATRIX_ACCUMULATOR_QCOM)));
typedef half half1024 __attribute__((ext_vector_type(1024)));
typedef half half4096 __attribute__((ext_vector_type(4096)));
typedef union {
half1024 v;
half16 h;
} a_row_t;
typedef union {
half4096 v;
struct {
half16 h0;
half16 h1;
half16 h2;
half16 h3;
} s;
} c_row_t;
inline half8 dequant8(uint v, uint h, half scale, half nminv) {
uint4 t;
t.s0 = ( v & 0x000F) | ((v << 12) & 0x000F0000);
t.s1 = ((v >> 8) & 0x000F) | ((v << 4) & 0x000F0000);
t.s2 = ((v >> 16) & 0x000F) | ((v >> 4) & 0x000F0000);
t.s3 = ((v >> 24) & 0x000F) | ((v >> 12) & 0x000F0000);
t.s0 |= ((h & 0x01u) << 4) | ((h & 0x02u) << 19);
t.s1 |= ((h & 0x04u) << 2) | ((h & 0x08u) << 17);
t.s2 |= ( h & 0x10u ) | ((h & 0x20u) << 15);
t.s3 |= ((h & 0x40u) >> 2) | ((h & 0x80u) << 13);
t |= (uint4)(0x64006400);
return (as_half8(t) - (half8)(1024.0h)) * scale + nminv;
}
inline void store_c16(global float * p, half16 h, int j0, int nv, int M) {
float16 f = convert_float16(h);
if (j0 + 0 < nv) p[(ulong)(j0 + 0) * M] = f.s0;
if (j0 + 1 < nv) p[(ulong)(j0 + 1) * M] = f.s1;
if (j0 + 2 < nv) p[(ulong)(j0 + 2) * M] = f.s2;
if (j0 + 3 < nv) p[(ulong)(j0 + 3) * M] = f.s3;
if (j0 + 4 < nv) p[(ulong)(j0 + 4) * M] = f.s4;
if (j0 + 5 < nv) p[(ulong)(j0 + 5) * M] = f.s5;
if (j0 + 6 < nv) p[(ulong)(j0 + 6) * M] = f.s6;
if (j0 + 7 < nv) p[(ulong)(j0 + 7) * M] = f.s7;
if (j0 + 8 < nv) p[(ulong)(j0 + 8) * M] = f.s8;
if (j0 + 9 < nv) p[(ulong)(j0 + 9) * M] = f.s9;
if (j0 + 10 < nv) p[(ulong)(j0 + 10) * M] = f.sa;
if (j0 + 11 < nv) p[(ulong)(j0 + 11) * M] = f.sb;
if (j0 + 12 < nv) p[(ulong)(j0 + 12) * M] = f.sc;
if (j0 + 13 < nv) p[(ulong)(j0 + 13) * M] = f.sd;
if (j0 + 14 < nv) p[(ulong)(j0 + 14) * M] = f.se;
if (j0 + 15 < nv) p[(ulong)(j0 + 15) * M] = f.sf;
}
#define MMA_K32(q0, q1, q2, q3, h0_, h1_, h2_, h3_, sc, nm, b) { \
a_row_t ua0; \
a_row_t ua1; \
ua0.h = (half16)(dequant8(q0, h0_, sc, nm), dequant8(q1, h1_, sc, nm)); \
ua1.h = (half16)(dequant8(q2, h2_, sc, nm), dequant8(q3, h3_, sc, nm)); \
mat_a_t a0 = __builtin_astype(ua0.v, mat_a_t); \
mat_a_t a1 = __builtin_astype(ua1.v, mat_a_t); \
mat_b_t b0 = qcom_coop_mat_load((b), CLK_COOPERATIVE_MATRIX_LAYOUT_COLUMN_MAJOR_QCOM, 16); \
mat_b_t b1 = qcom_coop_mat_load((b) + 1024, CLK_COOPERATIVE_MATRIX_LAYOUT_COLUMN_MAJOR_QCOM, 16); \
acc = qcom_coop_mat_mulAdd(a0, b0, acc, CLK_COOPERATIVE_MATRIX_OPERAND_NONE_QCOM); \
acc = qcom_coop_mat_mulAdd(a1, b1, acc, CLK_COOPERATIVE_MATRIX_OPERAND_NONE_QCOM); }
#define STEP64(sa, na, sb_, nb_) { \
uint8 qv; \
qv.s0 = qp[0]; \
qv.s1 = qp[M]; \
qv.s2 = qp[2 * M]; \
qv.s3 = qp[3 * M]; \
qv.s4 = qp[4 * M]; \
qv.s5 = qp[5 * M]; \
qv.s6 = qp[6 * M]; \
qv.s7 = qp[7 * M]; \
qp += 8 * M; \
uint8 hv; \
hv.s0 = qhp[0]; \
hv.s1 = qhp[M]; \
hv.s2 = qhp[2 * M]; \
hv.s3 = qhp[3 * M]; \
hv.s4 = qhp[4 * M]; \
hv.s5 = qhp[5 * M]; \
hv.s6 = qhp[6 * M]; \
hv.s7 = qhp[7 * M]; \
qhp += 8 * M; \
MMA_K32(qv.s0, qv.s1, qv.s2, qv.s3, hv.s0, hv.s1, hv.s2, hv.s3, sa, na, bp) \
MMA_K32(qv.s4, qv.s5, qv.s6, qv.s7, hv.s4, hv.s5, hv.s6, hv.s7, sb_, nb_, bp + 2048) \
bp += 4096; }
kernel void kernel_gemm_noshuffle_q5_k_f32_coopmat(
global const uint * src0_q, // A: q5_K low nibbles, [K/8][M] u32
global const uchar * src0_qh, // A: q5_K high bits, [K/8][M] u8
global const uchar * src0_s, // A: 6-bit scale/min codes, [K/256*12][M] u8
global const half * src0_d, // A: per-superblock scale, [K/256][M] f16
global const half * src0_dm, // A: per-superblock min, [K/256][M] f16
global const half * src1, // B: packed f16 tiles
global char * dst, // C: [N][M] f32
ulong offsetd,
int K,
int M,
int N
) {
const int lane = get_local_id(0);
const int row = get_global_id(1) * 64;
const int tn = get_global_id(2);
const int n_tiles = (N + 63) / 64;
if (tn >= n_tiles) {
return;
}
const int lrow = row + lane;
global const uint * qp = src0_q + lrow;
global const uchar * qhp = src0_qh + lrow;
global const half * bp = src1 + (ulong)tn * (K / 16) * 1024;
mat_c_t acc = qcom_coop_mat_init(0.0);
#pragma unroll 1
for (int k0 = 0; k0 < K; k0 += 128) {
const int sb = k0 >> 8;
const half dd = src0_d [(ulong)sb * M + lrow];
const half dmm = src0_dm[(ulong)sb * M + lrow];
global const uchar * sp = src0_s + (ulong)(sb * K_SCALE_SIZE) * M + lrow;
const uchar4 c0 = (uchar4)(sp[0], sp[M], sp[2 * M], sp[3 * M]);
const uchar4 c1 = (uchar4)(sp[4 * M], sp[5 * M], sp[6 * M], sp[7 * M]);
const uchar4 c2 = (uchar4)(sp[8 * M], sp[9 * M], sp[10 * M], sp[11 * M]);
const uchar4 sl = c0 & (uchar)0x3F;
const uchar4 ml = c1 & (uchar)0x3F;
const uchar4 sh = (c2 & (uchar)0x0F) | ((c0 >> (uchar)2) & (uchar)0x30);
const uchar4 mh = (c2 >> (uchar)4) | ((c1 >> (uchar)2) & (uchar)0x30);
const int hi = (k0 >> 7) & 1;
const half4 S = dd * convert_half4(hi ? sh : sl);
const half4 Nm = -(dmm * convert_half4(hi ? mh : ml));
STEP64(S.s0, Nm.s0, S.s1, Nm.s1)
STEP64(S.s2, Nm.s2, S.s3, Nm.s3)
}
c_row_t uc;
uc.v = __builtin_astype(acc, half4096);
global float * p = (global float *)(dst + offsetd) + (ulong)tn * 64 * M + row + lane;
const int nv = N - tn * 64;
store_c16(p, uc.s.h0, 0, nv, M);
store_c16(p, uc.s.h1, 16, nv, M);
store_c16(p, uc.s.h2, 32, nv, M);
store_c16(p, uc.s.h3, 48, nv, M);
}
@@ -0,0 +1,145 @@
#pragma OPENCL EXTENSION cl_khr_fp16 : enable
#pragma OPENCL EXTENSION cl_qcom_kernel_cooperative_matrix : enable
// Q6_K x F32 GEMM with cl_qcom_kernel_cooperative_matrix: one 64x64 f16 C tile per wave,
// A dequantized in registers, B packed by kernel_pack_b_f16_coopmat.
typedef half mat_a_t __attribute__((qcom_coop_mat(CLK_COOPERATIVE_MATRIX_SCOPE_SUBGROUP_QCOM, 64, 16, CLK_COOPERATIVE_MATRIX_A_QCOM)));
typedef half mat_b_t __attribute__((qcom_coop_mat(CLK_COOPERATIVE_MATRIX_SCOPE_SUBGROUP_QCOM, 16, 64, CLK_COOPERATIVE_MATRIX_B_QCOM)));
typedef half mat_c_t __attribute__((qcom_coop_mat(CLK_COOPERATIVE_MATRIX_SCOPE_SUBGROUP_QCOM, 64, 64, CLK_COOPERATIVE_MATRIX_ACCUMULATOR_QCOM)));
typedef half half1024 __attribute__((ext_vector_type(1024)));
typedef half half4096 __attribute__((ext_vector_type(4096)));
typedef union {
half1024 v;
half16 h;
} a_row_t;
typedef union {
half4096 v;
struct {
half16 h0;
half16 h1;
half16 h2;
half16 h3;
} s;
} c_row_t;
inline half8 dequant8(uint v, uint hh, half scale) {
uint4 t;
t.s0 = ( v & 0x000F) | ((v << 12) & 0x000F0000);
t.s1 = ((v >> 8) & 0x000F) | ((v << 4) & 0x000F0000);
t.s2 = ((v >> 16) & 0x000F) | ((v >> 4) & 0x000F0000);
t.s3 = ((v >> 24) & 0x000F) | ((v >> 12) & 0x000F0000);
t.s0 |= ((hh & 0x0003u) << 4) | ((hh & 0x000Cu) << 18);
t.s1 |= ( hh & 0x0030u ) | ((hh & 0x00C0u) << 14);
t.s2 |= ((hh & 0x0300u) >> 4) | ((hh & 0x0C00u) << 10);
t.s3 |= ((hh & 0x3000u) >> 8) | ((hh & 0xC000u) << 6);
t |= (uint4)(0x64006400);
return (as_half8(t) - (half8)(1056.0h)) * scale;
}
inline void store_c16(global float * p, half16 h, int j0, int nv, int M) {
float16 f = convert_float16(h);
if (j0 + 0 < nv) p[(ulong)(j0 + 0) * M] = f.s0;
if (j0 + 1 < nv) p[(ulong)(j0 + 1) * M] = f.s1;
if (j0 + 2 < nv) p[(ulong)(j0 + 2) * M] = f.s2;
if (j0 + 3 < nv) p[(ulong)(j0 + 3) * M] = f.s3;
if (j0 + 4 < nv) p[(ulong)(j0 + 4) * M] = f.s4;
if (j0 + 5 < nv) p[(ulong)(j0 + 5) * M] = f.s5;
if (j0 + 6 < nv) p[(ulong)(j0 + 6) * M] = f.s6;
if (j0 + 7 < nv) p[(ulong)(j0 + 7) * M] = f.s7;
if (j0 + 8 < nv) p[(ulong)(j0 + 8) * M] = f.s8;
if (j0 + 9 < nv) p[(ulong)(j0 + 9) * M] = f.s9;
if (j0 + 10 < nv) p[(ulong)(j0 + 10) * M] = f.sa;
if (j0 + 11 < nv) p[(ulong)(j0 + 11) * M] = f.sb;
if (j0 + 12 < nv) p[(ulong)(j0 + 12) * M] = f.sc;
if (j0 + 13 < nv) p[(ulong)(j0 + 13) * M] = f.sd;
if (j0 + 14 < nv) p[(ulong)(j0 + 14) * M] = f.se;
if (j0 + 15 < nv) p[(ulong)(j0 + 15) * M] = f.sf;
}
#define MMA_K32(q0, q1, q2, q3, hA, hB, sc0, sc1, b) { \
a_row_t ua0; \
a_row_t ua1; \
ua0.h = (half16)(dequant8(q0, (hA), sc0), dequant8(q1, (hA) >> 16, sc0)); \
ua1.h = (half16)(dequant8(q2, (hB), sc1), dequant8(q3, (hB) >> 16, sc1)); \
mat_a_t a0 = __builtin_astype(ua0.v, mat_a_t); \
mat_a_t a1 = __builtin_astype(ua1.v, mat_a_t); \
mat_b_t b0 = qcom_coop_mat_load((b), CLK_COOPERATIVE_MATRIX_LAYOUT_COLUMN_MAJOR_QCOM, 16); \
mat_b_t b1 = qcom_coop_mat_load((b) + 1024, CLK_COOPERATIVE_MATRIX_LAYOUT_COLUMN_MAJOR_QCOM, 16); \
acc = qcom_coop_mat_mulAdd(a0, b0, acc, CLK_COOPERATIVE_MATRIX_OPERAND_NONE_QCOM); \
acc = qcom_coop_mat_mulAdd(a1, b1, acc, CLK_COOPERATIVE_MATRIX_OPERAND_NONE_QCOM); }
kernel void kernel_gemm_noshuffle_q6_k_f32_coopmat(
global const uint * src0_ql, // A: q6_K low nibbles, [K/8][M] u32
global const uint * src0_qh, // A: q6_K high 2-bit, [K/16][M] u32
global const uint * src0_s, // A: int8 sub-scales, [M][K/16] (4 per u32)
global const half * src0_d, // A: per-superblock scale, [K/256][M] f16
global const half * src1, // B: packed f16 tiles
global char * dst, // C: [N][M] f32
ulong offsetd,
int K,
int M,
int N
) {
const int lane = get_local_id(0);
const int row = get_global_id(1) * 64;
const int tn = get_global_id(2);
const int n_tiles = (N + 63) / 64;
if (tn >= n_tiles) {
return;
}
const int lrow = row + lane;
global const uint * qp = src0_ql + lrow;
global const uint * qhp = src0_qh + lrow;
global const half * bp = src1 + (ulong)tn * (K / 16) * 1024;
global const uint * sp = src0_s + (ulong)lrow * (K / 64);
mat_c_t acc = qcom_coop_mat_init(0.0);
#pragma unroll 1
for (int k0 = 0; k0 < K; k0 += 64) {
const half dd = src0_d[(ulong)(k0 >> 8) * M + lrow];
const half4 g = convert_half4(as_char4(sp[0])) * dd;
sp += 1;
uint8 qv;
qv.s0 = qp[0];
qv.s1 = qp[M];
qv.s2 = qp[2 * M];
qv.s3 = qp[3 * M];
qv.s4 = qp[4 * M];
qv.s5 = qp[5 * M];
qv.s6 = qp[6 * M];
qv.s7 = qp[7 * M];
qp += 8 * M;
uint4 hv;
hv.s0 = qhp[0];
hv.s1 = qhp[M];
hv.s2 = qhp[2 * M];
hv.s3 = qhp[3 * M];
qhp += 4 * M;
MMA_K32(qv.s0, qv.s1, qv.s2, qv.s3, hv.s0, hv.s1, g.s0, g.s1, bp)
MMA_K32(qv.s4, qv.s5, qv.s6, qv.s7, hv.s2, hv.s3, g.s2, g.s3, bp + 2048)
bp += 4096;
}
c_row_t uc;
uc.v = __builtin_astype(acc, half4096);
global float * p = (global float *)(dst + offsetd) + (ulong)tn * 64 * M + row + lane;
const int nv = N - tn * 64;
store_c16(p, uc.s.h0, 0, nv, M);
store_c16(p, uc.s.h1, 16, nv, M);
store_c16(p, uc.s.h2, 32, nv, M);
store_c16(p, uc.s.h3, 48, nv, M);
}
@@ -0,0 +1,157 @@
#pragma OPENCL EXTENSION cl_khr_fp16 : enable
#pragma OPENCL EXTENSION cl_qcom_kernel_cooperative_matrix : enable
// Q8_0 x F32 GEMM with cl_qcom_kernel_cooperative_matrix: one 64x64 f16 C tile per wave,
// A dequantized in registers, B packed by kernel_pack_b_f16_coopmat.
typedef half mat_a_t __attribute__((qcom_coop_mat(CLK_COOPERATIVE_MATRIX_SCOPE_SUBGROUP_QCOM, 64, 16, CLK_COOPERATIVE_MATRIX_A_QCOM)));
typedef half mat_b_t __attribute__((qcom_coop_mat(CLK_COOPERATIVE_MATRIX_SCOPE_SUBGROUP_QCOM, 16, 64, CLK_COOPERATIVE_MATRIX_B_QCOM)));
typedef half mat_c_t __attribute__((qcom_coop_mat(CLK_COOPERATIVE_MATRIX_SCOPE_SUBGROUP_QCOM, 64, 64, CLK_COOPERATIVE_MATRIX_ACCUMULATOR_QCOM)));
typedef half half1024 __attribute__((ext_vector_type(1024)));
typedef half half4096 __attribute__((ext_vector_type(4096)));
typedef union {
half1024 v;
half16 h;
} a_row_t;
typedef union {
half4096 v;
struct {
half16 h0;
half16 h1;
half16 h2;
half16 h3;
} s;
} c_row_t;
inline half4 dequant4(uint v, half d) {
return convert_half4(as_char4(v)) * d;
}
inline void store_c16(global float * p, half16 h, int j0, int nv, int M) {
float16 f = convert_float16(h);
if (j0 + 0 < nv) p[(ulong)(j0 + 0) * M] = f.s0;
if (j0 + 1 < nv) p[(ulong)(j0 + 1) * M] = f.s1;
if (j0 + 2 < nv) p[(ulong)(j0 + 2) * M] = f.s2;
if (j0 + 3 < nv) p[(ulong)(j0 + 3) * M] = f.s3;
if (j0 + 4 < nv) p[(ulong)(j0 + 4) * M] = f.s4;
if (j0 + 5 < nv) p[(ulong)(j0 + 5) * M] = f.s5;
if (j0 + 6 < nv) p[(ulong)(j0 + 6) * M] = f.s6;
if (j0 + 7 < nv) p[(ulong)(j0 + 7) * M] = f.s7;
if (j0 + 8 < nv) p[(ulong)(j0 + 8) * M] = f.s8;
if (j0 + 9 < nv) p[(ulong)(j0 + 9) * M] = f.s9;
if (j0 + 10 < nv) p[(ulong)(j0 + 10) * M] = f.sa;
if (j0 + 11 < nv) p[(ulong)(j0 + 11) * M] = f.sb;
if (j0 + 12 < nv) p[(ulong)(j0 + 12) * M] = f.sc;
if (j0 + 13 < nv) p[(ulong)(j0 + 13) * M] = f.sd;
if (j0 + 14 < nv) p[(ulong)(j0 + 14) * M] = f.se;
if (j0 + 15 < nv) p[(ulong)(j0 + 15) * M] = f.sf;
}
#define LOAD_Q(qv, dd) { \
qv.s0 = qp[0]; \
qv.s1 = qp[M]; \
qv.s2 = qp[2 * M]; \
qv.s3 = qp[3 * M]; \
qv.s4 = qp[4 * M]; \
qv.s5 = qp[5 * M]; \
qv.s6 = qp[6 * M]; \
qv.s7 = qp[7 * M]; \
dd = dp[0]; \
qp += 8 * M; \
dp += M; }
#define MMA_K32(qv, dd, b) { \
a_row_t ua0; \
a_row_t ua1; \
ua0.h = (half16)(dequant4(qv.s0, dd), dequant4(qv.s1, dd), dequant4(qv.s2, dd), dequant4(qv.s3, dd)); \
ua1.h = (half16)(dequant4(qv.s4, dd), dequant4(qv.s5, dd), dequant4(qv.s6, dd), dequant4(qv.s7, dd)); \
mat_a_t a0 = __builtin_astype(ua0.v, mat_a_t); \
mat_a_t a1 = __builtin_astype(ua1.v, mat_a_t); \
mat_b_t b0 = qcom_coop_mat_load((b), CLK_COOPERATIVE_MATRIX_LAYOUT_COLUMN_MAJOR_QCOM, 16); \
mat_b_t b1 = qcom_coop_mat_load((b) + 1024, CLK_COOPERATIVE_MATRIX_LAYOUT_COLUMN_MAJOR_QCOM, 16); \
acc = qcom_coop_mat_mulAdd(a0, b0, acc, CLK_COOPERATIVE_MATRIX_OPERAND_NONE_QCOM); \
acc = qcom_coop_mat_mulAdd(a1, b1, acc, CLK_COOPERATIVE_MATRIX_OPERAND_NONE_QCOM); }
kernel void kernel_gemm_noshuffle_q8_0_f32_coopmat(
global const uint * src0_q, // A: q8_0 quants, [K/4][M] u32
global const half * src0_d, // A: q8_0 scales, [K/32][M] f16
global const half * src1, // B: packed f16 tiles
global char * dst, // C: [N][M] f32
ulong offsetd,
int K,
int M,
int N
) {
const int lane = get_local_id(0);
const int row = get_global_id(1) * 64;
const int tn = get_global_id(2);
const int n_tiles = (N + 63) / 64;
if (tn >= n_tiles) {
return;
}
global const uint * qp = src0_q + row + lane;
global const half * dp = src0_d + row + lane;
global const half * bp = src1 + (ulong)tn * (K / 16) * 1024;
mat_c_t acc = qcom_coop_mat_init(0.0);
{
const int odd = (K / 32) & 1;
const int kb = odd ? 0 : 32;
uint8 qa;
uint8 qb;
half da;
half db;
qa.s0 = qp[0];
qa.s1 = qp[M];
qa.s2 = qp[2 * M];
qa.s3 = qp[3 * M];
qa.s4 = qp[4 * M];
qa.s5 = qp[5 * M];
qa.s6 = qp[6 * M];
qa.s7 = qp[7 * M];
da = odd ? 0.0h : dp[0];
qb.s0 = qp[(kb / 4 + 0) * M];
qb.s1 = qp[(kb / 4 + 1) * M];
qb.s2 = qp[(kb / 4 + 2) * M];
qb.s3 = qp[(kb / 4 + 3) * M];
qb.s4 = qp[(kb / 4 + 4) * M];
qb.s5 = qp[(kb / 4 + 5) * M];
qb.s6 = qp[(kb / 4 + 6) * M];
qb.s7 = qp[(kb / 4 + 7) * M];
db = dp[(kb / 32) * M];
MMA_K32(qa, da, bp)
MMA_K32(qb, db, bp + (kb / 16) * 1024)
qp += (kb / 4 + 8) * M;
dp += (kb / 32 + 1) * M;
bp += (kb / 16 + 2) * 1024;
}
#pragma unroll 1
for (int k0 = (K / 32) & 1 ? 32 : 64; k0 < K; k0 += 64) {
uint8 qa;
uint8 qb;
half da;
half db;
LOAD_Q(qa, da)
LOAD_Q(qb, db)
MMA_K32(qa, da, bp)
MMA_K32(qb, db, bp + 2048)
bp += 4096;
}
c_row_t uc;
uc.v = __builtin_astype(acc, half4096);
global float * p = (global float *)(dst + offsetd) + (ulong)tn * 64 * M + row + lane;
const int nv = N - tn * 64;
store_c16(p, uc.s.h0, 0, nv, M);
store_c16(p, uc.s.h1, 16, nv, M);
store_c16(p, uc.s.h2, 32, nv, M);
store_c16(p, uc.s.h3, 48, nv, M);
}
@@ -0,0 +1,48 @@
#pragma OPENCL EXTENSION cl_khr_fp16 : enable
// Pack B f32 [N][K] into f16 64x16 tiles (column-major, ld 16) at (tn * (K/16) + ks) * 1024.
// Columns >= N are zero. A workgroup (64, 4) packs 64 columns x 64 k.
#define PACK_LDS (64 + 8)
kernel void kernel_pack_b_f16_coopmat(
global const char * src1,
ulong offset1,
global half * dst,
int K,
int N
) {
local half tile[64 * PACK_LDS];
const int l = get_local_id(1) * 64 + get_local_id(0);
const int tn = get_group_id(0);
const int kt = get_group_id(1);
const int k0 = kt * 64;
global const float * src = (global const float *)(src1 + offset1);
for (int j = 0; j < 4; j++) {
const int f = j * 256 + l;
const int r = f >> 4;
const int c = (f & 15) * 4;
const int n = tn * 64 + r;
float4 v = (float4)(0.0f);
if (n < N && k0 + c < K) {
v = vload4(0, src + (ulong)n * K + k0 + c);
}
vstore_half4(v, 0, tile + r * PACK_LDS + c);
}
barrier(CLK_LOCAL_MEM_FENCE);
global half * out = dst + ((ulong)tn * (K / 16) + kt * 4) * 1024;
for (int j = 0; j < 2; j++) {
const int q = j * 256 + l;
const int ks = q >> 7;
const int n = (q >> 1) & 63;
const int h = q & 1;
if (kt * 4 + ks < K / 16) {
half8 v = vload8(0, tile + n * PACK_LDS + ks * 16 + h * 8);
vstore8(v, 0, out + q * 8);
}
}
}