From 1fb8a269603b497fc1ce7dab2f2fc5c1be7a51cb Mon Sep 17 00:00:00 2001 From: shaofeiqi Date: Sun, 22 Nov 2026 00:01:14 -0800 Subject: [PATCH] opencl: add coopmat GEMM for Adreno 850 (Q4_0, Q5_0, Q8_0, Q4_K, Q5_K, Q6_K) (#32686) * opencl: add Q4_0 coopmat GEMM for Adreno 850 Assisted-by: Claude Opus 5.5 * opencl: add Q5_0 coopmat GEMM for Adreno 850 * opencl: add Q8_0 coopmat GEMM for Adreno 850 * opencl: add Q4_K coopmat GEMM for Adreno 850 * opencl: add Q5_K coopmat GEMM for Adreno 850 * opencl: add Q6_K coopmat GEMM for Adreno 850 --- ggml/src/ggml-opencl/CMakeLists.txt | 7 + ggml/src/ggml-opencl/ggml-opencl.cpp | 223 +++++++++++++++++- .../gemm_noshuffle_q4_0_f32_coopmat.cl | 145 ++++++++++++ .../gemm_noshuffle_q4_k_f32_coopmat.cl | 148 ++++++++++++ .../gemm_noshuffle_q5_0_f32_coopmat.cl | 157 ++++++++++++ .../gemm_noshuffle_q5_k_f32_coopmat.cl | 164 +++++++++++++ .../gemm_noshuffle_q6_k_f32_coopmat.cl | 145 ++++++++++++ .../gemm_noshuffle_q8_0_f32_coopmat.cl | 157 ++++++++++++ .../ggml-opencl/kernels/pack_b_f16_coopmat.cl | 48 ++++ 9 files changed, 1190 insertions(+), 4 deletions(-) create mode 100644 ggml/src/ggml-opencl/kernels/gemm_noshuffle_q4_0_f32_coopmat.cl create mode 100644 ggml/src/ggml-opencl/kernels/gemm_noshuffle_q4_k_f32_coopmat.cl create mode 100644 ggml/src/ggml-opencl/kernels/gemm_noshuffle_q5_0_f32_coopmat.cl create mode 100644 ggml/src/ggml-opencl/kernels/gemm_noshuffle_q5_k_f32_coopmat.cl create mode 100644 ggml/src/ggml-opencl/kernels/gemm_noshuffle_q6_k_f32_coopmat.cl create mode 100644 ggml/src/ggml-opencl/kernels/gemm_noshuffle_q8_0_f32_coopmat.cl create mode 100644 ggml/src/ggml-opencl/kernels/pack_b_f16_coopmat.cl diff --git a/ggml/src/ggml-opencl/CMakeLists.txt b/ggml/src/ggml-opencl/CMakeLists.txt index 325f5c4bf..aadf5ddb3 100644 --- a/ggml/src/ggml-opencl/CMakeLists.txt +++ b/ggml/src/ggml-opencl/CMakeLists.txt @@ -227,6 +227,13 @@ set(GGML_OPENCL_KERNELS gemm_coopk_q4_0_q8_1_dp4a gemm_coopk_q4_k_q8_1_dp4a gemm_coopk_q6_k_q8_1_dp4a + pack_b_f16_coopmat + gemm_noshuffle_q4_0_f32_coopmat + gemm_noshuffle_q5_0_f32_coopmat + gemm_noshuffle_q8_0_f32_coopmat + gemm_noshuffle_q4_k_f32_coopmat + gemm_noshuffle_q5_k_f32_coopmat + gemm_noshuffle_q6_k_f32_coopmat quant_a_q8_1 gemv_moe_q4_k_f32_ns gemm_moe_q5_k_f32_ns diff --git a/ggml/src/ggml-opencl/ggml-opencl.cpp b/ggml/src/ggml-opencl/ggml-opencl.cpp index 894f8235e..d35c6d047 100644 --- a/ggml/src/ggml-opencl/ggml-opencl.cpp +++ b/ggml/src/ggml-opencl/ggml-opencl.cpp @@ -1495,6 +1495,13 @@ struct ggml_backend_opencl_context { cl_kernel kernel_gemm_noshuffle_iq4_nl_q8_1_dp4a = nullptr; // dp4a (int8) dense IQ4_NL prefill GEMM cl_kernel kernel_gemm_noshuffle_q4_0_q8_1_dp4a = nullptr; // dp4a (int8) dense q4_0 prefill GEMM cl_kernel kernel_gemm_noshuffle_q4_0_q8_1_dp4a_act_vload4 = nullptr; + cl_kernel kernel_pack_b_f16_coopmat = nullptr; + cl_kernel kernel_gemm_noshuffle_q4_0_f32_coopmat = nullptr; + cl_kernel kernel_gemm_noshuffle_q5_0_f32_coopmat = nullptr; + cl_kernel kernel_gemm_noshuffle_q8_0_f32_coopmat = nullptr; + cl_kernel kernel_gemm_noshuffle_q4_k_f32_coopmat = nullptr; + cl_kernel kernel_gemm_noshuffle_q5_k_f32_coopmat = nullptr; + cl_kernel kernel_gemm_noshuffle_q6_k_f32_coopmat = nullptr; cl_kernel kernel_gemm_coopk_q4_0_q8_1_dp4a_c4 = nullptr; cl_kernel kernel_gemm_coopk_q4_0_q8_1_dp4a_c2 = nullptr; int nsg_coopk_dp4a_q4_0 = 8; @@ -1979,6 +1986,137 @@ static void load_cl_kernels_gemm_coopk(ggml_backend_opencl_context * backend_ctx } } +static void load_cl_kernels_coopmat(ggml_backend_opencl_context * backend_ctx) { + if (backend_ctx->gpu_family != GPU_FAMILY::ADRENO || !strstr(backend_ctx->device_name.c_str(), "850")) { + return; + } + + cl_int err; + const std::string opts = "-cl-std=CL" + std::to_string(backend_ctx->opencl_c_version.major) + "." + + std::to_string(backend_ctx->opencl_c_version.minor) + + (backend_ctx->adreno_use_large_buffer ? " -qcom-enable-large-buffer" : ""); + + // pack_b_f16_coopmat + { +#ifdef GGML_OPENCL_EMBED_KERNELS + const std::string kernel_src { + #include "pack_b_f16_coopmat.cl.h" + }; +#else + const std::string kernel_src = read_file("pack_b_f16_coopmat.cl"); +#endif + cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), opts, false); + if (!prog) { + return; + } + CL_CHECK((backend_ctx->kernel_pack_b_f16_coopmat = clCreateKernel(prog, "kernel_pack_b_f16_coopmat", &err), err)); + CL_CHECK(clReleaseProgram(prog)); + GGML_LOG_CONT("."); + } + + // gemm_noshuffle_q4_0_f32_coopmat + { +#ifdef GGML_OPENCL_EMBED_KERNELS + const std::string kernel_src { + #include "gemm_noshuffle_q4_0_f32_coopmat.cl.h" + }; +#else + const std::string kernel_src = read_file("gemm_noshuffle_q4_0_f32_coopmat.cl"); +#endif + cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), opts, false); + if (prog) { + CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q4_0_f32_coopmat = clCreateKernel(prog, "kernel_gemm_noshuffle_q4_0_f32_coopmat", &err), err)); + CL_CHECK(clReleaseProgram(prog)); + GGML_LOG_CONT("."); + } + } + + // gemm_noshuffle_q5_0_f32_coopmat + { +#ifdef GGML_OPENCL_EMBED_KERNELS + const std::string kernel_src { + #include "gemm_noshuffle_q5_0_f32_coopmat.cl.h" + }; +#else + const std::string kernel_src = read_file("gemm_noshuffle_q5_0_f32_coopmat.cl"); +#endif + cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), opts, false); + if (prog) { + CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q5_0_f32_coopmat = clCreateKernel(prog, "kernel_gemm_noshuffle_q5_0_f32_coopmat", &err), err)); + CL_CHECK(clReleaseProgram(prog)); + GGML_LOG_CONT("."); + } + } + + // gemm_noshuffle_q8_0_f32_coopmat + { +#ifdef GGML_OPENCL_EMBED_KERNELS + const std::string kernel_src { + #include "gemm_noshuffle_q8_0_f32_coopmat.cl.h" + }; +#else + const std::string kernel_src = read_file("gemm_noshuffle_q8_0_f32_coopmat.cl"); +#endif + cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), opts, false); + if (prog) { + CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q8_0_f32_coopmat = clCreateKernel(prog, "kernel_gemm_noshuffle_q8_0_f32_coopmat", &err), err)); + CL_CHECK(clReleaseProgram(prog)); + GGML_LOG_CONT("."); + } + } + + // gemm_noshuffle_q4_k_f32_coopmat + { +#ifdef GGML_OPENCL_EMBED_KERNELS + const std::string kernel_src { + #include "gemm_noshuffle_q4_k_f32_coopmat.cl.h" + }; +#else + const std::string kernel_src = read_file("gemm_noshuffle_q4_k_f32_coopmat.cl"); +#endif + cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), opts, false); + if (prog) { + CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q4_k_f32_coopmat = clCreateKernel(prog, "kernel_gemm_noshuffle_q4_k_f32_coopmat", &err), err)); + CL_CHECK(clReleaseProgram(prog)); + GGML_LOG_CONT("."); + } + } + + // gemm_noshuffle_q5_k_f32_coopmat + { +#ifdef GGML_OPENCL_EMBED_KERNELS + const std::string kernel_src { + #include "gemm_noshuffle_q5_k_f32_coopmat.cl.h" + }; +#else + const std::string kernel_src = read_file("gemm_noshuffle_q5_k_f32_coopmat.cl"); +#endif + cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), opts, false); + if (prog) { + CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q5_k_f32_coopmat = clCreateKernel(prog, "kernel_gemm_noshuffle_q5_k_f32_coopmat", &err), err)); + CL_CHECK(clReleaseProgram(prog)); + GGML_LOG_CONT("."); + } + } + + // gemm_noshuffle_q6_k_f32_coopmat + { +#ifdef GGML_OPENCL_EMBED_KERNELS + const std::string kernel_src { + #include "gemm_noshuffle_q6_k_f32_coopmat.cl.h" + }; +#else + const std::string kernel_src = read_file("gemm_noshuffle_q6_k_f32_coopmat.cl"); +#endif + cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), opts, false); + if (prog) { + CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q6_k_f32_coopmat = clCreateKernel(prog, "kernel_gemm_noshuffle_q6_k_f32_coopmat", &err), err)); + CL_CHECK(clReleaseProgram(prog)); + GGML_LOG_CONT("."); + } + } +} + #endif static std::string ggml_opencl_make_compile_opts(const ggml_backend_opencl_context * backend_ctx) { @@ -7813,6 +7951,7 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) { } } load_cl_kernels_gemm_coopk(backend_ctx); + load_cl_kernels_coopmat(backend_ctx); #endif // GGML_OPENCL_USE_ADRENO_KERNELS GGML_LOG_CONT("\n"); backend_ctx->kernels_loaded = true; @@ -11739,7 +11878,8 @@ inline bool use_q4_0_bin_kernels(const ggml_backend_opencl_context *backend_ctx, #ifdef GGML_OPENCL_USE_ADRENO_KERNELS if (!backend_ctx->kernel_gemv_noshuffle_q4_0_f32_32b_trans || !(backend_ctx->kernel_gemm_noshuffle_q4_0_f32_32b_trans_ila_a8_bin || - backend_ctx->kernel_gemm_noshuffle_q4_0_f32_32b_trans_ila_a7_bin)) { + backend_ctx->kernel_gemm_noshuffle_q4_0_f32_32b_trans_ila_a7_bin || + backend_ctx->kernel_gemm_noshuffle_q4_0_f32_coopmat)) { return false; } @@ -11893,7 +12033,8 @@ inline bool use_q6_k_bin_kernels(const ggml_backend_opencl_context *backend_ctx, #ifdef GGML_OPENCL_USE_ADRENO_KERNELS if (!backend_ctx->kernel_gemv_noshuffle_q6_k_f32_32b_trans || !(backend_ctx->kernel_gemm_noshuffle_q6_k_f32_32b_trans_ila_a8_bin || - backend_ctx->kernel_gemm_noshuffle_q6_k_f32_32b_trans_ila_a7_bin)) { + backend_ctx->kernel_gemm_noshuffle_q6_k_f32_32b_trans_ila_a7_bin || + backend_ctx->kernel_gemm_noshuffle_q6_k_f32_coopmat)) { return false; } // bin kernels require ne2 == 1 and ne3 == 1 for weights @@ -11911,7 +12052,8 @@ inline bool use_q4_k_bin_kernels(const ggml_backend_opencl_context *backend_ctx, #ifdef GGML_OPENCL_USE_ADRENO_KERNELS if (!backend_ctx->kernel_gemv_noshuffle_q4_k_f32_32b_trans || !(backend_ctx->kernel_gemm_noshuffle_q4_k_f32_32b_trans_ila_a8_bin || - backend_ctx->kernel_gemm_noshuffle_q4_k_f32_32b_trans_ila_a7_bin)) { + backend_ctx->kernel_gemm_noshuffle_q4_k_f32_32b_trans_ila_a7_bin || + backend_ctx->kernel_gemm_noshuffle_q4_k_f32_coopmat)) { return false; } // bin kernels require ne2 == 1 and ne3 == 1 for weights @@ -11929,7 +12071,8 @@ inline bool use_q5_k_bin_kernels(const ggml_backend_opencl_context *backend_ctx, #ifdef GGML_OPENCL_USE_ADRENO_KERNELS if (!backend_ctx->kernel_gemv_noshuffle_q5_k_f32_32b_trans || !(backend_ctx->kernel_gemm_noshuffle_q5_k_f32_32b_trans_ila_a8_bin || - backend_ctx->kernel_gemm_noshuffle_q5_k_f32_32b_trans_ila_a7_bin)) { + backend_ctx->kernel_gemm_noshuffle_q5_k_f32_32b_trans_ila_a7_bin || + backend_ctx->kernel_gemm_noshuffle_q5_k_f32_coopmat)) { return false; } return (tensor->ne[0] % 256 == 0) && (tensor->ne[1] % 64 == 0); @@ -25619,6 +25762,48 @@ static void ggml_cl_mul_mat_q2_0_f32_adreno(ggml_backend_t backend, const ggml_t CL_CHECK(clReleaseMemObject(b_sub_buf)); } +static bool ggml_cl_mul_mat_coopmat(ggml_backend_opencl_context * backend_ctx, cl_kernel kernel, + std::initializer_list weights, + const ggml_tensor * src1, ggml_tensor * dst, int K, int M, int N) { + if (kernel == nullptr || M % 64 != 0 || N < 2) { + return false; + } + ggml_tensor_extra_cl * extra1 = (ggml_tensor_extra_cl *)src1->extra; + ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra; + cl_ulong offset1 = extra1->offset + src1->view_offs; + cl_ulong offsetd = extrad->offset + dst->view_offs; + + const int n_tiles = CEIL_DIV(N, 64); + + backend_ctx->prealloc_act_trans.allocate(backend_ctx->context, (size_t)n_tiles * 64 * K * sizeof(cl_half)); + + cl_kernel pk = backend_ctx->kernel_pack_b_f16_coopmat; + CL_CHECK(clSetKernelArg(pk, 0, sizeof(cl_mem), &extra1->data_device)); + CL_CHECK(clSetKernelArg(pk, 1, sizeof(cl_ulong), &offset1)); + CL_CHECK(clSetKernelArg(pk, 2, sizeof(cl_mem), &backend_ctx->prealloc_act_trans.buffer)); + CL_CHECK(clSetKernelArg(pk, 3, sizeof(int), &K)); + CL_CHECK(clSetKernelArg(pk, 4, sizeof(int), &N)); + size_t p_local[2] = { 64, 4 }; + size_t p_global[2] = { (size_t)n_tiles * 64, (size_t)CEIL_DIV(K, 64) * 4 }; + backend_ctx->enqueue_ndrange_kernel(pk, 2, p_global, p_local, dst); + + cl_uint ka = 0; + for (cl_mem w : weights) { + CL_CHECK(clSetKernelArg(kernel, ka++, sizeof(cl_mem), &w)); + } + CL_CHECK(clSetKernelArg(kernel, ka++, sizeof(cl_mem), &backend_ctx->prealloc_act_trans.buffer)); + CL_CHECK(clSetKernelArg(kernel, ka++, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, ka++, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, ka++, sizeof(int), &K)); + CL_CHECK(clSetKernelArg(kernel, ka++, sizeof(int), &M)); + CL_CHECK(clSetKernelArg(kernel, ka++, sizeof(int), &N)); + + size_t c_local[3] = { 64, 1, 4 }; + size_t c_global[3] = { 64, (size_t)(M / 64), (size_t)CEIL_DIV(n_tiles, 4) * 4 }; + backend_ctx->enqueue_ndrange_kernel(kernel, 3, c_global, c_local, dst); + return true; +} + static void ggml_cl_mul_mat_q4_0_f32_adreno_ila(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { GGML_ASSERT(src0); @@ -25934,6 +26119,11 @@ static void ggml_cl_mul_mat_q4_0_f32_adreno(ggml_backend_t backend, const ggml_t const bool use_bin = use_q4_0_bin_kernels(backend_ctx, src0); + if (use_bin && ggml_cl_mul_mat_coopmat(backend_ctx, backend_ctx->kernel_gemm_noshuffle_q4_0_f32_coopmat, + { extra0_q4_0->q, extra0_q4_0->d }, src1, dst, K, M, N)) { + return; + } + if (use_bin) { if (use_q40_mc3) { static bool warned = false; @@ -26498,6 +26688,11 @@ static void ggml_cl_mul_mat_q5_0_f32_adreno(ggml_backend_t backend, const ggml_t int N = ne1; int K = ne00; + if (ggml_cl_mul_mat_coopmat(backend_ctx, backend_ctx->kernel_gemm_noshuffle_q5_0_f32_coopmat, + { extra0_q5_0->qs, extra0_q5_0->qh, extra0_q5_0->d }, src1, dst, K, M, N)) { + return; + } + if (ne1 == 1) { cl_mem qs_img = nullptr; cl_mem b_sub_buf = nullptr; @@ -27340,6 +27535,11 @@ static void ggml_cl_mul_mat_q8_0_f32_adreno(ggml_backend_t backend, const ggml_t int N = ne1; int K = ne00; + if (ggml_cl_mul_mat_coopmat(backend_ctx, backend_ctx->kernel_gemm_noshuffle_q8_0_f32_coopmat, + { extra0_q8_0->q, extra0_q8_0->d }, src1, dst, K, M, N)) { + return; + } + // Multi-column verify GEMV for ne1 2..4, reuses the ne1 == 1 image setup. // Default on for X2E, GGML_OPENCL_Q8_0_MC=0/1 forces it. static const char * q8_mc_env = getenv("GGML_OPENCL_Q8_0_MC"); @@ -28321,6 +28521,11 @@ static void ggml_cl_mul_mat_q4_k_f32_adreno(ggml_backend_t backend, const ggml_t const bool use_bin = use_q4_k_bin_kernels(backend_ctx, src0); + if (use_bin && ggml_cl_mul_mat_coopmat(backend_ctx, backend_ctx->kernel_gemm_noshuffle_q4_k_f32_coopmat, + { extra0_q4_k->q, extra0_q4_k->s, extra0_q4_k->d, extra0_q4_k->dm }, src1, dst, K, M, N)) { + return; + } + if (use_bin) { if (use_mc3) { static bool warned = false; @@ -29231,6 +29436,11 @@ static void ggml_cl_mul_mat_q6_K_f32_adreno(ggml_backend_t backend, const ggml_t const bool use_bin = use_q6_k_bin_kernels(backend_ctx, src0); + if (use_bin && ggml_cl_mul_mat_coopmat(backend_ctx, backend_ctx->kernel_gemm_noshuffle_q6_k_f32_coopmat, + { extra0_q6_K->ql, extra0_q6_K->qh, extra0_q6_K->s, extra0_q6_K->d }, src1, dst, ne00, ne01, ne1)) { + return; + } + if (use_bin) { if (use_q6k_mc3 || use_q6k_tiled_mc) { static bool warned = false; @@ -29971,6 +30181,11 @@ static void ggml_cl_mul_mat_q5_K_f32_adreno(ggml_backend_t backend, const ggml_t const bool use_bin = use_q5_k_bin_kernels(backend_ctx, src0); + if (use_bin && ggml_cl_mul_mat_coopmat(backend_ctx, backend_ctx->kernel_gemm_noshuffle_q5_k_f32_coopmat, + { extra0_q5_k->q, extra0_q5_k->qh, extra0_q5_k->s, extra0_q5_k->d, extra0_q5_k->dm }, src1, dst, K, M, N)) { + return; + } + if (use_bin) { if (use_q5k_mc3) { static bool warned = false; diff --git a/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q4_0_f32_coopmat.cl b/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q4_0_f32_coopmat.cl new file mode 100644 index 000000000..56210a1ad --- /dev/null +++ b/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q4_0_f32_coopmat.cl @@ -0,0 +1,145 @@ +#pragma OPENCL EXTENSION cl_khr_fp16 : enable +#pragma OPENCL EXTENSION cl_qcom_kernel_cooperative_matrix : enable + +// Q4_0 x F32 GEMM with cl_qcom_kernel_cooperative_matrix: one 64x64 f16 C tile per wave, +// A dequantized in registers, B packed by kernel_pack_b_f16_coopmat. + +typedef half mat_a_t __attribute__((qcom_coop_mat(CLK_COOPERATIVE_MATRIX_SCOPE_SUBGROUP_QCOM, 64, 16, CLK_COOPERATIVE_MATRIX_A_QCOM))); +typedef half mat_b_t __attribute__((qcom_coop_mat(CLK_COOPERATIVE_MATRIX_SCOPE_SUBGROUP_QCOM, 16, 64, CLK_COOPERATIVE_MATRIX_B_QCOM))); +typedef half mat_c_t __attribute__((qcom_coop_mat(CLK_COOPERATIVE_MATRIX_SCOPE_SUBGROUP_QCOM, 64, 64, CLK_COOPERATIVE_MATRIX_ACCUMULATOR_QCOM))); + +typedef half half1024 __attribute__((ext_vector_type(1024))); +typedef half half4096 __attribute__((ext_vector_type(4096))); + +typedef union { + half1024 v; + half16 h; +} a_row_t; + +typedef union { + half4096 v; + struct { + half16 h0; + half16 h1; + half16 h2; + half16 h3; + } s; +} c_row_t; + +inline half8 dequant8(uint v, half d) { + uint4 t; + t.s0 = ( v & 0x000F) | ((v << 12) & 0x000F0000); + t.s1 = ((v >> 8) & 0x000F) | ((v << 4) & 0x000F0000); + t.s2 = ((v >> 16) & 0x000F) | ((v >> 4) & 0x000F0000); + t.s3 = ((v >> 24) & 0x000F) | ((v >> 12) & 0x000F0000); + t |= (uint4)(0x64006400); + + return (as_half8(t) - (half8)(1032.0h)) * d; +} + +inline void store_c16(global float * p, half16 h, int j0, int nv, int M) { + float16 f = convert_float16(h); + if (j0 + 0 < nv) p[(ulong)(j0 + 0) * M] = f.s0; + if (j0 + 1 < nv) p[(ulong)(j0 + 1) * M] = f.s1; + if (j0 + 2 < nv) p[(ulong)(j0 + 2) * M] = f.s2; + if (j0 + 3 < nv) p[(ulong)(j0 + 3) * M] = f.s3; + if (j0 + 4 < nv) p[(ulong)(j0 + 4) * M] = f.s4; + if (j0 + 5 < nv) p[(ulong)(j0 + 5) * M] = f.s5; + if (j0 + 6 < nv) p[(ulong)(j0 + 6) * M] = f.s6; + if (j0 + 7 < nv) p[(ulong)(j0 + 7) * M] = f.s7; + if (j0 + 8 < nv) p[(ulong)(j0 + 8) * M] = f.s8; + if (j0 + 9 < nv) p[(ulong)(j0 + 9) * M] = f.s9; + if (j0 + 10 < nv) p[(ulong)(j0 + 10) * M] = f.sa; + if (j0 + 11 < nv) p[(ulong)(j0 + 11) * M] = f.sb; + if (j0 + 12 < nv) p[(ulong)(j0 + 12) * M] = f.sc; + if (j0 + 13 < nv) p[(ulong)(j0 + 13) * M] = f.sd; + if (j0 + 14 < nv) p[(ulong)(j0 + 14) * M] = f.se; + if (j0 + 15 < nv) p[(ulong)(j0 + 15) * M] = f.sf; +} + +#define LOAD_Q(qv, dd, o) { \ + qv.s0 = qp[(4 * (o) + 0) * M]; \ + qv.s1 = qp[(4 * (o) + 1) * M]; \ + qv.s2 = qp[(4 * (o) + 2) * M]; \ + qv.s3 = qp[(4 * (o) + 3) * M]; \ + dd = dp[(o) * M]; } + +#define MMA_K32(qv, dd, b) { \ + a_row_t ua0; \ + a_row_t ua1; \ + ua0.h = (half16)(dequant8(qv.s0, dd), dequant8(qv.s1, dd)); \ + ua1.h = (half16)(dequant8(qv.s2, dd), dequant8(qv.s3, dd)); \ + mat_a_t a0 = __builtin_astype(ua0.v, mat_a_t); \ + mat_a_t a1 = __builtin_astype(ua1.v, mat_a_t); \ + mat_b_t b0 = qcom_coop_mat_load((b), CLK_COOPERATIVE_MATRIX_LAYOUT_COLUMN_MAJOR_QCOM, 16); \ + mat_b_t b1 = qcom_coop_mat_load((b) + 1024, CLK_COOPERATIVE_MATRIX_LAYOUT_COLUMN_MAJOR_QCOM, 16); \ + acc = qcom_coop_mat_mulAdd(a0, b0, acc, CLK_COOPERATIVE_MATRIX_OPERAND_NONE_QCOM); \ + acc = qcom_coop_mat_mulAdd(a1, b1, acc, CLK_COOPERATIVE_MATRIX_OPERAND_NONE_QCOM); } + +kernel void kernel_gemm_noshuffle_q4_0_f32_coopmat( + global const uint * src0_q, // A: q4_0 quants, [K/8][M] u32 + global const half * src0_d, // A: q4_0 scales, [K/32][M] f16 + global const half * src1, // B: packed f16 tiles + global char * dst, // C: [N][M] f32 + ulong offsetd, + int K, + int M, + int N +) { + const int lane = get_local_id(0); + const int row = get_global_id(1) * 64; + const int tn = get_global_id(2); + const int n_tiles = (N + 63) / 64; + + if (tn >= n_tiles) { + return; + } + + global const uint * qp = src0_q + row + lane; + global const half * dp = src0_d + row + lane; + global const half * bp = src1 + (ulong)tn * (K / 16) * 1024; + + mat_c_t acc = qcom_coop_mat_init(0.0); + + { + const int odd = (K / 32) & 1; + const int ob = odd ? 0 : 1; + uint4 qa; + uint4 qb; + half da; + half db; + LOAD_Q(qa, da, 0) + LOAD_Q(qb, db, ob) + da = odd ? 0.0h : da; + MMA_K32(qa, da, bp) + MMA_K32(qb, db, bp + ob * 2048) + qp += (ob + 1) * 4 * M; + dp += (ob + 1) * M; + bp += (ob + 1) * 2048; + } + +#pragma unroll 1 + for (int k0 = (K / 32) & 1 ? 32 : 64; k0 < K; k0 += 64) { + uint4 qa; + uint4 qb; + half da; + half db; + LOAD_Q(qa, da, 0) + LOAD_Q(qb, db, 1) + MMA_K32(qa, da, bp) + MMA_K32(qb, db, bp + 2048) + qp += 8 * M; + dp += 2 * M; + bp += 4096; + } + + c_row_t uc; + uc.v = __builtin_astype(acc, half4096); + + global float * p = (global float *)(dst + offsetd) + (ulong)tn * 64 * M + row + lane; + const int nv = N - tn * 64; + store_c16(p, uc.s.h0, 0, nv, M); + store_c16(p, uc.s.h1, 16, nv, M); + store_c16(p, uc.s.h2, 32, nv, M); + store_c16(p, uc.s.h3, 48, nv, M); +} diff --git a/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q4_k_f32_coopmat.cl b/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q4_k_f32_coopmat.cl new file mode 100644 index 000000000..65f29309f --- /dev/null +++ b/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q4_k_f32_coopmat.cl @@ -0,0 +1,148 @@ +#pragma OPENCL EXTENSION cl_khr_fp16 : enable +#pragma OPENCL EXTENSION cl_qcom_kernel_cooperative_matrix : enable + +// Q4_K x F32 GEMM with cl_qcom_kernel_cooperative_matrix: one 64x64 f16 C tile per wave, +// A dequantized in registers, B packed by kernel_pack_b_f16_coopmat. + +#define K_SCALE_SIZE 12 + +typedef half mat_a_t __attribute__((qcom_coop_mat(CLK_COOPERATIVE_MATRIX_SCOPE_SUBGROUP_QCOM, 64, 16, CLK_COOPERATIVE_MATRIX_A_QCOM))); +typedef half mat_b_t __attribute__((qcom_coop_mat(CLK_COOPERATIVE_MATRIX_SCOPE_SUBGROUP_QCOM, 16, 64, CLK_COOPERATIVE_MATRIX_B_QCOM))); +typedef half mat_c_t __attribute__((qcom_coop_mat(CLK_COOPERATIVE_MATRIX_SCOPE_SUBGROUP_QCOM, 64, 64, CLK_COOPERATIVE_MATRIX_ACCUMULATOR_QCOM))); + +typedef half half1024 __attribute__((ext_vector_type(1024))); +typedef half half4096 __attribute__((ext_vector_type(4096))); + +typedef union { + half1024 v; + half16 h; +} a_row_t; + +typedef union { + half4096 v; + struct { + half16 h0; + half16 h1; + half16 h2; + half16 h3; + } s; +} c_row_t; + +inline half8 dequant8(uint v, half scale, half nminv) { + uint4 t; + t.s0 = ( v & 0x000F) | ((v << 12) & 0x000F0000); + t.s1 = ((v >> 8) & 0x000F) | ((v << 4) & 0x000F0000); + t.s2 = ((v >> 16) & 0x000F) | ((v >> 4) & 0x000F0000); + t.s3 = ((v >> 24) & 0x000F) | ((v >> 12) & 0x000F0000); + t |= (uint4)(0x64006400); + + return (as_half8(t) - (half8)(1024.0h)) * scale + nminv; +} + +inline void store_c16(global float * p, half16 h, int j0, int nv, int M) { + float16 f = convert_float16(h); + if (j0 + 0 < nv) p[(ulong)(j0 + 0) * M] = f.s0; + if (j0 + 1 < nv) p[(ulong)(j0 + 1) * M] = f.s1; + if (j0 + 2 < nv) p[(ulong)(j0 + 2) * M] = f.s2; + if (j0 + 3 < nv) p[(ulong)(j0 + 3) * M] = f.s3; + if (j0 + 4 < nv) p[(ulong)(j0 + 4) * M] = f.s4; + if (j0 + 5 < nv) p[(ulong)(j0 + 5) * M] = f.s5; + if (j0 + 6 < nv) p[(ulong)(j0 + 6) * M] = f.s6; + if (j0 + 7 < nv) p[(ulong)(j0 + 7) * M] = f.s7; + if (j0 + 8 < nv) p[(ulong)(j0 + 8) * M] = f.s8; + if (j0 + 9 < nv) p[(ulong)(j0 + 9) * M] = f.s9; + if (j0 + 10 < nv) p[(ulong)(j0 + 10) * M] = f.sa; + if (j0 + 11 < nv) p[(ulong)(j0 + 11) * M] = f.sb; + if (j0 + 12 < nv) p[(ulong)(j0 + 12) * M] = f.sc; + if (j0 + 13 < nv) p[(ulong)(j0 + 13) * M] = f.sd; + if (j0 + 14 < nv) p[(ulong)(j0 + 14) * M] = f.se; + if (j0 + 15 < nv) p[(ulong)(j0 + 15) * M] = f.sf; +} + +#define MMA_K32(q0, q1, q2, q3, sc, nm, b) { \ + a_row_t ua0; \ + a_row_t ua1; \ + ua0.h = (half16)(dequant8(q0, sc, nm), dequant8(q1, sc, nm)); \ + ua1.h = (half16)(dequant8(q2, sc, nm), dequant8(q3, sc, nm)); \ + mat_a_t a0 = __builtin_astype(ua0.v, mat_a_t); \ + mat_a_t a1 = __builtin_astype(ua1.v, mat_a_t); \ + mat_b_t b0 = qcom_coop_mat_load((b), CLK_COOPERATIVE_MATRIX_LAYOUT_COLUMN_MAJOR_QCOM, 16); \ + mat_b_t b1 = qcom_coop_mat_load((b) + 1024, CLK_COOPERATIVE_MATRIX_LAYOUT_COLUMN_MAJOR_QCOM, 16); \ + acc = qcom_coop_mat_mulAdd(a0, b0, acc, CLK_COOPERATIVE_MATRIX_OPERAND_NONE_QCOM); \ + acc = qcom_coop_mat_mulAdd(a1, b1, acc, CLK_COOPERATIVE_MATRIX_OPERAND_NONE_QCOM); } + +#define STEP64(sa, na, sb_, nb_) { \ + uint8 qv; \ + qv.s0 = qp[0]; \ + qv.s1 = qp[M]; \ + qv.s2 = qp[2 * M]; \ + qv.s3 = qp[3 * M]; \ + qv.s4 = qp[4 * M]; \ + qv.s5 = qp[5 * M]; \ + qv.s6 = qp[6 * M]; \ + qv.s7 = qp[7 * M]; \ + qp += 8 * M; \ + MMA_K32(qv.s0, qv.s1, qv.s2, qv.s3, sa, na, bp) \ + MMA_K32(qv.s4, qv.s5, qv.s6, qv.s7, sb_, nb_, bp + 2048) \ + bp += 4096; } + +kernel void kernel_gemm_noshuffle_q4_k_f32_coopmat( + global const uint * src0_q, // A: q4_K quants, [K/8][M] u32 + global const uchar * src0_s, // A: 6-bit scale/min codes, [K/256*12][M] u8 + global const half * src0_d, // A: per-superblock scale, [K/256][M] f16 + global const half * src0_dm, // A: per-superblock min, [K/256][M] f16 + global const half * src1, // B: packed f16 tiles + global char * dst, // C: [N][M] f32 + ulong offsetd, + int K, + int M, + int N +) { + const int lane = get_local_id(0); + const int row = get_global_id(1) * 64; + const int tn = get_global_id(2); + const int n_tiles = (N + 63) / 64; + + if (tn >= n_tiles) { + return; + } + + const int lrow = row + lane; + + global const uint * qp = src0_q + lrow; + global const half * bp = src1 + (ulong)tn * (K / 16) * 1024; + + mat_c_t acc = qcom_coop_mat_init(0.0); + +#pragma unroll 1 + for (int k0 = 0; k0 < K; k0 += 128) { + const int sb = k0 >> 8; + const half dd = src0_d [(ulong)sb * M + lrow]; + const half dmm = src0_dm[(ulong)sb * M + lrow]; + + global const uchar * sp = src0_s + (ulong)(sb * K_SCALE_SIZE) * M + lrow; + const uchar4 c0 = (uchar4)(sp[0], sp[M], sp[2 * M], sp[3 * M]); + const uchar4 c1 = (uchar4)(sp[4 * M], sp[5 * M], sp[6 * M], sp[7 * M]); + const uchar4 c2 = (uchar4)(sp[8 * M], sp[9 * M], sp[10 * M], sp[11 * M]); + const uchar4 sl = c0 & (uchar)0x3F; + const uchar4 ml = c1 & (uchar)0x3F; + const uchar4 sh = (c2 & (uchar)0x0F) | ((c0 >> (uchar)2) & (uchar)0x30); + const uchar4 mh = (c2 >> (uchar)4) | ((c1 >> (uchar)2) & (uchar)0x30); + const int hi = (k0 >> 7) & 1; + const half4 S = dd * convert_half4(hi ? sh : sl); + const half4 Nm = -(dmm * convert_half4(hi ? mh : ml)); + + STEP64(S.s0, Nm.s0, S.s1, Nm.s1) + STEP64(S.s2, Nm.s2, S.s3, Nm.s3) + } + + c_row_t uc; + uc.v = __builtin_astype(acc, half4096); + + global float * p = (global float *)(dst + offsetd) + (ulong)tn * 64 * M + row + lane; + const int nv = N - tn * 64; + store_c16(p, uc.s.h0, 0, nv, M); + store_c16(p, uc.s.h1, 16, nv, M); + store_c16(p, uc.s.h2, 32, nv, M); + store_c16(p, uc.s.h3, 48, nv, M); +} diff --git a/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q5_0_f32_coopmat.cl b/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q5_0_f32_coopmat.cl new file mode 100644 index 000000000..1880a86f0 --- /dev/null +++ b/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q5_0_f32_coopmat.cl @@ -0,0 +1,157 @@ +#pragma OPENCL EXTENSION cl_khr_fp16 : enable +#pragma OPENCL EXTENSION cl_qcom_kernel_cooperative_matrix : enable + +// Q5_0 x F32 GEMM with cl_qcom_kernel_cooperative_matrix: one 64x64 f16 C tile per wave, +// A dequantized in registers, B packed by kernel_pack_b_f16_coopmat. + +typedef half mat_a_t __attribute__((qcom_coop_mat(CLK_COOPERATIVE_MATRIX_SCOPE_SUBGROUP_QCOM, 64, 16, CLK_COOPERATIVE_MATRIX_A_QCOM))); +typedef half mat_b_t __attribute__((qcom_coop_mat(CLK_COOPERATIVE_MATRIX_SCOPE_SUBGROUP_QCOM, 16, 64, CLK_COOPERATIVE_MATRIX_B_QCOM))); +typedef half mat_c_t __attribute__((qcom_coop_mat(CLK_COOPERATIVE_MATRIX_SCOPE_SUBGROUP_QCOM, 64, 64, CLK_COOPERATIVE_MATRIX_ACCUMULATOR_QCOM))); + +typedef half half1024 __attribute__((ext_vector_type(1024))); +typedef half half4096 __attribute__((ext_vector_type(4096))); + +typedef union { + half1024 v; + half16 h; +} a_row_t; + +typedef union { + half4096 v; + struct { + half16 h0; + half16 h1; + half16 h2; + half16 h3; + } s; +} c_row_t; + +inline half8 dequant8(uint v, uint h, half d) { + uint4 t; + t.s0 = ( v & 0x000F) | ((v << 12) & 0x000F0000) | ((h << 4) & 0x0010) | ((h << 19) & 0x00100000); + t.s1 = ((v >> 8) & 0x000F) | ((v << 4) & 0x000F0000) | ((h << 2) & 0x0010) | ((h << 17) & 0x00100000); + t.s2 = ((v >> 16) & 0x000F) | ((v >> 4) & 0x000F0000) | ( h & 0x0010) | ((h << 15) & 0x00100000); + t.s3 = ((v >> 24) & 0x000F) | ((v >> 12) & 0x000F0000) | ((h >> 2) & 0x0010) | ((h << 13) & 0x00100000); + t |= (uint4)(0x64006400); + + return (as_half8(t) - (half8)(1040.0h)) * d; +} + +inline void store_c16(global float * p, half16 h, int j0, int nv, int M) { + float16 f = convert_float16(h); + if (j0 + 0 < nv) p[(ulong)(j0 + 0) * M] = f.s0; + if (j0 + 1 < nv) p[(ulong)(j0 + 1) * M] = f.s1; + if (j0 + 2 < nv) p[(ulong)(j0 + 2) * M] = f.s2; + if (j0 + 3 < nv) p[(ulong)(j0 + 3) * M] = f.s3; + if (j0 + 4 < nv) p[(ulong)(j0 + 4) * M] = f.s4; + if (j0 + 5 < nv) p[(ulong)(j0 + 5) * M] = f.s5; + if (j0 + 6 < nv) p[(ulong)(j0 + 6) * M] = f.s6; + if (j0 + 7 < nv) p[(ulong)(j0 + 7) * M] = f.s7; + if (j0 + 8 < nv) p[(ulong)(j0 + 8) * M] = f.s8; + if (j0 + 9 < nv) p[(ulong)(j0 + 9) * M] = f.s9; + if (j0 + 10 < nv) p[(ulong)(j0 + 10) * M] = f.sa; + if (j0 + 11 < nv) p[(ulong)(j0 + 11) * M] = f.sb; + if (j0 + 12 < nv) p[(ulong)(j0 + 12) * M] = f.sc; + if (j0 + 13 < nv) p[(ulong)(j0 + 13) * M] = f.sd; + if (j0 + 14 < nv) p[(ulong)(j0 + 14) * M] = f.se; + if (j0 + 15 < nv) p[(ulong)(j0 + 15) * M] = f.sf; +} + +#define LOAD_Q(qv, hv, dd, o) { \ + qv.s0 = (uint)qp[(8 * (o) + 0) * M] | ((uint)qp[(8 * (o) + 1) * M] << 16); \ + qv.s1 = (uint)qp[(8 * (o) + 2) * M] | ((uint)qp[(8 * (o) + 3) * M] << 16); \ + qv.s2 = (uint)qp[(8 * (o) + 4) * M] | ((uint)qp[(8 * (o) + 5) * M] << 16); \ + qv.s3 = (uint)qp[(8 * (o) + 6) * M] | ((uint)qp[(8 * (o) + 7) * M] << 16); \ + hv.s0 = hp[(4 * (o) + 0) * M]; \ + hv.s1 = hp[(4 * (o) + 1) * M]; \ + hv.s2 = hp[(4 * (o) + 2) * M]; \ + hv.s3 = hp[(4 * (o) + 3) * M]; \ + dd = dp[(o) * M]; } + +#define MMA_K32(qv, hv, dd, b) { \ + a_row_t ua0; \ + a_row_t ua1; \ + ua0.h = (half16)(dequant8(qv.s0, hv.s0, dd), dequant8(qv.s1, hv.s1, dd)); \ + ua1.h = (half16)(dequant8(qv.s2, hv.s2, dd), dequant8(qv.s3, hv.s3, dd)); \ + mat_a_t a0 = __builtin_astype(ua0.v, mat_a_t); \ + mat_a_t a1 = __builtin_astype(ua1.v, mat_a_t); \ + mat_b_t b0 = qcom_coop_mat_load((b), CLK_COOPERATIVE_MATRIX_LAYOUT_COLUMN_MAJOR_QCOM, 16); \ + mat_b_t b1 = qcom_coop_mat_load((b) + 1024, CLK_COOPERATIVE_MATRIX_LAYOUT_COLUMN_MAJOR_QCOM, 16); \ + acc = qcom_coop_mat_mulAdd(a0, b0, acc, CLK_COOPERATIVE_MATRIX_OPERAND_NONE_QCOM); \ + acc = qcom_coop_mat_mulAdd(a1, b1, acc, CLK_COOPERATIVE_MATRIX_OPERAND_NONE_QCOM); } + +kernel void kernel_gemm_noshuffle_q5_0_f32_coopmat( + global const ushort * src0_q, // A: q5_0 quants, [K/4][M] u16 + global const uchar * src0_qh, // A: q5_0 high bits, [K/8][M] u8 + global const half * src0_d, // A: q5_0 scales, [K/32][M] f16 + global const half * src1, // B: packed f16 tiles + global char * dst, // C: [N][M] f32 + ulong offsetd, + int K, + int M, + int N +) { + const int lane = get_local_id(0); + const int row = get_global_id(1) * 64; + const int tn = get_global_id(2); + const int n_tiles = (N + 63) / 64; + + if (tn >= n_tiles) { + return; + } + + global const ushort * qp = src0_q + row + lane; + global const uchar * hp = src0_qh + row + lane; + global const half * dp = src0_d + row + lane; + global const half * bp = src1 + (ulong)tn * (K / 16) * 1024; + + mat_c_t acc = qcom_coop_mat_init(0.0); + + { + const int odd = (K / 32) & 1; + const int ob = odd ? 0 : 1; + uint4 qa; + uint4 qb; + uint4 ha; + uint4 hb; + half da; + half db; + LOAD_Q(qa, ha, da, 0) + LOAD_Q(qb, hb, db, ob) + da = odd ? 0.0h : da; + MMA_K32(qa, ha, da, bp) + MMA_K32(qb, hb, db, bp + ob * 2048) + qp += (ob + 1) * 8 * M; + hp += (ob + 1) * 4 * M; + dp += (ob + 1) * M; + bp += (ob + 1) * 2048; + } + +#pragma unroll 1 + for (int k0 = (K / 32) & 1 ? 32 : 64; k0 < K; k0 += 64) { + uint4 qa; + uint4 qb; + uint4 ha; + uint4 hb; + half da; + half db; + LOAD_Q(qa, ha, da, 0) + LOAD_Q(qb, hb, db, 1) + MMA_K32(qa, ha, da, bp) + MMA_K32(qb, hb, db, bp + 2048) + qp += 16 * M; + hp += 8 * M; + dp += 2 * M; + bp += 4096; + } + + c_row_t uc; + uc.v = __builtin_astype(acc, half4096); + + global float * p = (global float *)(dst + offsetd) + (ulong)tn * 64 * M + row + lane; + const int nv = N - tn * 64; + store_c16(p, uc.s.h0, 0, nv, M); + store_c16(p, uc.s.h1, 16, nv, M); + store_c16(p, uc.s.h2, 32, nv, M); + store_c16(p, uc.s.h3, 48, nv, M); +} diff --git a/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q5_k_f32_coopmat.cl b/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q5_k_f32_coopmat.cl new file mode 100644 index 000000000..2b4f11780 --- /dev/null +++ b/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q5_k_f32_coopmat.cl @@ -0,0 +1,164 @@ +#pragma OPENCL EXTENSION cl_khr_fp16 : enable +#pragma OPENCL EXTENSION cl_qcom_kernel_cooperative_matrix : enable + +// Q5_K x F32 GEMM with cl_qcom_kernel_cooperative_matrix: one 64x64 f16 C tile per wave, +// A dequantized in registers, B packed by kernel_pack_b_f16_coopmat. + +#define K_SCALE_SIZE 12 + +typedef half mat_a_t __attribute__((qcom_coop_mat(CLK_COOPERATIVE_MATRIX_SCOPE_SUBGROUP_QCOM, 64, 16, CLK_COOPERATIVE_MATRIX_A_QCOM))); +typedef half mat_b_t __attribute__((qcom_coop_mat(CLK_COOPERATIVE_MATRIX_SCOPE_SUBGROUP_QCOM, 16, 64, CLK_COOPERATIVE_MATRIX_B_QCOM))); +typedef half mat_c_t __attribute__((qcom_coop_mat(CLK_COOPERATIVE_MATRIX_SCOPE_SUBGROUP_QCOM, 64, 64, CLK_COOPERATIVE_MATRIX_ACCUMULATOR_QCOM))); + +typedef half half1024 __attribute__((ext_vector_type(1024))); +typedef half half4096 __attribute__((ext_vector_type(4096))); + +typedef union { + half1024 v; + half16 h; +} a_row_t; + +typedef union { + half4096 v; + struct { + half16 h0; + half16 h1; + half16 h2; + half16 h3; + } s; +} c_row_t; + +inline half8 dequant8(uint v, uint h, half scale, half nminv) { + uint4 t; + t.s0 = ( v & 0x000F) | ((v << 12) & 0x000F0000); + t.s1 = ((v >> 8) & 0x000F) | ((v << 4) & 0x000F0000); + t.s2 = ((v >> 16) & 0x000F) | ((v >> 4) & 0x000F0000); + t.s3 = ((v >> 24) & 0x000F) | ((v >> 12) & 0x000F0000); + t.s0 |= ((h & 0x01u) << 4) | ((h & 0x02u) << 19); + t.s1 |= ((h & 0x04u) << 2) | ((h & 0x08u) << 17); + t.s2 |= ( h & 0x10u ) | ((h & 0x20u) << 15); + t.s3 |= ((h & 0x40u) >> 2) | ((h & 0x80u) << 13); + t |= (uint4)(0x64006400); + + return (as_half8(t) - (half8)(1024.0h)) * scale + nminv; +} + +inline void store_c16(global float * p, half16 h, int j0, int nv, int M) { + float16 f = convert_float16(h); + if (j0 + 0 < nv) p[(ulong)(j0 + 0) * M] = f.s0; + if (j0 + 1 < nv) p[(ulong)(j0 + 1) * M] = f.s1; + if (j0 + 2 < nv) p[(ulong)(j0 + 2) * M] = f.s2; + if (j0 + 3 < nv) p[(ulong)(j0 + 3) * M] = f.s3; + if (j0 + 4 < nv) p[(ulong)(j0 + 4) * M] = f.s4; + if (j0 + 5 < nv) p[(ulong)(j0 + 5) * M] = f.s5; + if (j0 + 6 < nv) p[(ulong)(j0 + 6) * M] = f.s6; + if (j0 + 7 < nv) p[(ulong)(j0 + 7) * M] = f.s7; + if (j0 + 8 < nv) p[(ulong)(j0 + 8) * M] = f.s8; + if (j0 + 9 < nv) p[(ulong)(j0 + 9) * M] = f.s9; + if (j0 + 10 < nv) p[(ulong)(j0 + 10) * M] = f.sa; + if (j0 + 11 < nv) p[(ulong)(j0 + 11) * M] = f.sb; + if (j0 + 12 < nv) p[(ulong)(j0 + 12) * M] = f.sc; + if (j0 + 13 < nv) p[(ulong)(j0 + 13) * M] = f.sd; + if (j0 + 14 < nv) p[(ulong)(j0 + 14) * M] = f.se; + if (j0 + 15 < nv) p[(ulong)(j0 + 15) * M] = f.sf; +} + +#define MMA_K32(q0, q1, q2, q3, h0_, h1_, h2_, h3_, sc, nm, b) { \ + a_row_t ua0; \ + a_row_t ua1; \ + ua0.h = (half16)(dequant8(q0, h0_, sc, nm), dequant8(q1, h1_, sc, nm)); \ + ua1.h = (half16)(dequant8(q2, h2_, sc, nm), dequant8(q3, h3_, sc, nm)); \ + mat_a_t a0 = __builtin_astype(ua0.v, mat_a_t); \ + mat_a_t a1 = __builtin_astype(ua1.v, mat_a_t); \ + mat_b_t b0 = qcom_coop_mat_load((b), CLK_COOPERATIVE_MATRIX_LAYOUT_COLUMN_MAJOR_QCOM, 16); \ + mat_b_t b1 = qcom_coop_mat_load((b) + 1024, CLK_COOPERATIVE_MATRIX_LAYOUT_COLUMN_MAJOR_QCOM, 16); \ + acc = qcom_coop_mat_mulAdd(a0, b0, acc, CLK_COOPERATIVE_MATRIX_OPERAND_NONE_QCOM); \ + acc = qcom_coop_mat_mulAdd(a1, b1, acc, CLK_COOPERATIVE_MATRIX_OPERAND_NONE_QCOM); } + +#define STEP64(sa, na, sb_, nb_) { \ + uint8 qv; \ + qv.s0 = qp[0]; \ + qv.s1 = qp[M]; \ + qv.s2 = qp[2 * M]; \ + qv.s3 = qp[3 * M]; \ + qv.s4 = qp[4 * M]; \ + qv.s5 = qp[5 * M]; \ + qv.s6 = qp[6 * M]; \ + qv.s7 = qp[7 * M]; \ + qp += 8 * M; \ + uint8 hv; \ + hv.s0 = qhp[0]; \ + hv.s1 = qhp[M]; \ + hv.s2 = qhp[2 * M]; \ + hv.s3 = qhp[3 * M]; \ + hv.s4 = qhp[4 * M]; \ + hv.s5 = qhp[5 * M]; \ + hv.s6 = qhp[6 * M]; \ + hv.s7 = qhp[7 * M]; \ + qhp += 8 * M; \ + MMA_K32(qv.s0, qv.s1, qv.s2, qv.s3, hv.s0, hv.s1, hv.s2, hv.s3, sa, na, bp) \ + MMA_K32(qv.s4, qv.s5, qv.s6, qv.s7, hv.s4, hv.s5, hv.s6, hv.s7, sb_, nb_, bp + 2048) \ + bp += 4096; } + +kernel void kernel_gemm_noshuffle_q5_k_f32_coopmat( + global const uint * src0_q, // A: q5_K low nibbles, [K/8][M] u32 + global const uchar * src0_qh, // A: q5_K high bits, [K/8][M] u8 + global const uchar * src0_s, // A: 6-bit scale/min codes, [K/256*12][M] u8 + global const half * src0_d, // A: per-superblock scale, [K/256][M] f16 + global const half * src0_dm, // A: per-superblock min, [K/256][M] f16 + global const half * src1, // B: packed f16 tiles + global char * dst, // C: [N][M] f32 + ulong offsetd, + int K, + int M, + int N +) { + const int lane = get_local_id(0); + const int row = get_global_id(1) * 64; + const int tn = get_global_id(2); + const int n_tiles = (N + 63) / 64; + + if (tn >= n_tiles) { + return; + } + + const int lrow = row + lane; + + global const uint * qp = src0_q + lrow; + global const uchar * qhp = src0_qh + lrow; + global const half * bp = src1 + (ulong)tn * (K / 16) * 1024; + + mat_c_t acc = qcom_coop_mat_init(0.0); + +#pragma unroll 1 + for (int k0 = 0; k0 < K; k0 += 128) { + const int sb = k0 >> 8; + const half dd = src0_d [(ulong)sb * M + lrow]; + const half dmm = src0_dm[(ulong)sb * M + lrow]; + + global const uchar * sp = src0_s + (ulong)(sb * K_SCALE_SIZE) * M + lrow; + const uchar4 c0 = (uchar4)(sp[0], sp[M], sp[2 * M], sp[3 * M]); + const uchar4 c1 = (uchar4)(sp[4 * M], sp[5 * M], sp[6 * M], sp[7 * M]); + const uchar4 c2 = (uchar4)(sp[8 * M], sp[9 * M], sp[10 * M], sp[11 * M]); + const uchar4 sl = c0 & (uchar)0x3F; + const uchar4 ml = c1 & (uchar)0x3F; + const uchar4 sh = (c2 & (uchar)0x0F) | ((c0 >> (uchar)2) & (uchar)0x30); + const uchar4 mh = (c2 >> (uchar)4) | ((c1 >> (uchar)2) & (uchar)0x30); + const int hi = (k0 >> 7) & 1; + const half4 S = dd * convert_half4(hi ? sh : sl); + const half4 Nm = -(dmm * convert_half4(hi ? mh : ml)); + + STEP64(S.s0, Nm.s0, S.s1, Nm.s1) + STEP64(S.s2, Nm.s2, S.s3, Nm.s3) + } + + c_row_t uc; + uc.v = __builtin_astype(acc, half4096); + + global float * p = (global float *)(dst + offsetd) + (ulong)tn * 64 * M + row + lane; + const int nv = N - tn * 64; + store_c16(p, uc.s.h0, 0, nv, M); + store_c16(p, uc.s.h1, 16, nv, M); + store_c16(p, uc.s.h2, 32, nv, M); + store_c16(p, uc.s.h3, 48, nv, M); +} diff --git a/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q6_k_f32_coopmat.cl b/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q6_k_f32_coopmat.cl new file mode 100644 index 000000000..5c517f94d --- /dev/null +++ b/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q6_k_f32_coopmat.cl @@ -0,0 +1,145 @@ +#pragma OPENCL EXTENSION cl_khr_fp16 : enable +#pragma OPENCL EXTENSION cl_qcom_kernel_cooperative_matrix : enable + +// Q6_K x F32 GEMM with cl_qcom_kernel_cooperative_matrix: one 64x64 f16 C tile per wave, +// A dequantized in registers, B packed by kernel_pack_b_f16_coopmat. + +typedef half mat_a_t __attribute__((qcom_coop_mat(CLK_COOPERATIVE_MATRIX_SCOPE_SUBGROUP_QCOM, 64, 16, CLK_COOPERATIVE_MATRIX_A_QCOM))); +typedef half mat_b_t __attribute__((qcom_coop_mat(CLK_COOPERATIVE_MATRIX_SCOPE_SUBGROUP_QCOM, 16, 64, CLK_COOPERATIVE_MATRIX_B_QCOM))); +typedef half mat_c_t __attribute__((qcom_coop_mat(CLK_COOPERATIVE_MATRIX_SCOPE_SUBGROUP_QCOM, 64, 64, CLK_COOPERATIVE_MATRIX_ACCUMULATOR_QCOM))); + +typedef half half1024 __attribute__((ext_vector_type(1024))); +typedef half half4096 __attribute__((ext_vector_type(4096))); + +typedef union { + half1024 v; + half16 h; +} a_row_t; + +typedef union { + half4096 v; + struct { + half16 h0; + half16 h1; + half16 h2; + half16 h3; + } s; +} c_row_t; + +inline half8 dequant8(uint v, uint hh, half scale) { + uint4 t; + t.s0 = ( v & 0x000F) | ((v << 12) & 0x000F0000); + t.s1 = ((v >> 8) & 0x000F) | ((v << 4) & 0x000F0000); + t.s2 = ((v >> 16) & 0x000F) | ((v >> 4) & 0x000F0000); + t.s3 = ((v >> 24) & 0x000F) | ((v >> 12) & 0x000F0000); + t.s0 |= ((hh & 0x0003u) << 4) | ((hh & 0x000Cu) << 18); + t.s1 |= ( hh & 0x0030u ) | ((hh & 0x00C0u) << 14); + t.s2 |= ((hh & 0x0300u) >> 4) | ((hh & 0x0C00u) << 10); + t.s3 |= ((hh & 0x3000u) >> 8) | ((hh & 0xC000u) << 6); + t |= (uint4)(0x64006400); + + return (as_half8(t) - (half8)(1056.0h)) * scale; +} + +inline void store_c16(global float * p, half16 h, int j0, int nv, int M) { + float16 f = convert_float16(h); + if (j0 + 0 < nv) p[(ulong)(j0 + 0) * M] = f.s0; + if (j0 + 1 < nv) p[(ulong)(j0 + 1) * M] = f.s1; + if (j0 + 2 < nv) p[(ulong)(j0 + 2) * M] = f.s2; + if (j0 + 3 < nv) p[(ulong)(j0 + 3) * M] = f.s3; + if (j0 + 4 < nv) p[(ulong)(j0 + 4) * M] = f.s4; + if (j0 + 5 < nv) p[(ulong)(j0 + 5) * M] = f.s5; + if (j0 + 6 < nv) p[(ulong)(j0 + 6) * M] = f.s6; + if (j0 + 7 < nv) p[(ulong)(j0 + 7) * M] = f.s7; + if (j0 + 8 < nv) p[(ulong)(j0 + 8) * M] = f.s8; + if (j0 + 9 < nv) p[(ulong)(j0 + 9) * M] = f.s9; + if (j0 + 10 < nv) p[(ulong)(j0 + 10) * M] = f.sa; + if (j0 + 11 < nv) p[(ulong)(j0 + 11) * M] = f.sb; + if (j0 + 12 < nv) p[(ulong)(j0 + 12) * M] = f.sc; + if (j0 + 13 < nv) p[(ulong)(j0 + 13) * M] = f.sd; + if (j0 + 14 < nv) p[(ulong)(j0 + 14) * M] = f.se; + if (j0 + 15 < nv) p[(ulong)(j0 + 15) * M] = f.sf; +} + +#define MMA_K32(q0, q1, q2, q3, hA, hB, sc0, sc1, b) { \ + a_row_t ua0; \ + a_row_t ua1; \ + ua0.h = (half16)(dequant8(q0, (hA), sc0), dequant8(q1, (hA) >> 16, sc0)); \ + ua1.h = (half16)(dequant8(q2, (hB), sc1), dequant8(q3, (hB) >> 16, sc1)); \ + mat_a_t a0 = __builtin_astype(ua0.v, mat_a_t); \ + mat_a_t a1 = __builtin_astype(ua1.v, mat_a_t); \ + mat_b_t b0 = qcom_coop_mat_load((b), CLK_COOPERATIVE_MATRIX_LAYOUT_COLUMN_MAJOR_QCOM, 16); \ + mat_b_t b1 = qcom_coop_mat_load((b) + 1024, CLK_COOPERATIVE_MATRIX_LAYOUT_COLUMN_MAJOR_QCOM, 16); \ + acc = qcom_coop_mat_mulAdd(a0, b0, acc, CLK_COOPERATIVE_MATRIX_OPERAND_NONE_QCOM); \ + acc = qcom_coop_mat_mulAdd(a1, b1, acc, CLK_COOPERATIVE_MATRIX_OPERAND_NONE_QCOM); } + +kernel void kernel_gemm_noshuffle_q6_k_f32_coopmat( + global const uint * src0_ql, // A: q6_K low nibbles, [K/8][M] u32 + global const uint * src0_qh, // A: q6_K high 2-bit, [K/16][M] u32 + global const uint * src0_s, // A: int8 sub-scales, [M][K/16] (4 per u32) + global const half * src0_d, // A: per-superblock scale, [K/256][M] f16 + global const half * src1, // B: packed f16 tiles + global char * dst, // C: [N][M] f32 + ulong offsetd, + int K, + int M, + int N +) { + const int lane = get_local_id(0); + const int row = get_global_id(1) * 64; + const int tn = get_global_id(2); + const int n_tiles = (N + 63) / 64; + + if (tn >= n_tiles) { + return; + } + + const int lrow = row + lane; + + global const uint * qp = src0_ql + lrow; + global const uint * qhp = src0_qh + lrow; + global const half * bp = src1 + (ulong)tn * (K / 16) * 1024; + global const uint * sp = src0_s + (ulong)lrow * (K / 64); + + mat_c_t acc = qcom_coop_mat_init(0.0); + +#pragma unroll 1 + for (int k0 = 0; k0 < K; k0 += 64) { + const half dd = src0_d[(ulong)(k0 >> 8) * M + lrow]; + + const half4 g = convert_half4(as_char4(sp[0])) * dd; + sp += 1; + + uint8 qv; + qv.s0 = qp[0]; + qv.s1 = qp[M]; + qv.s2 = qp[2 * M]; + qv.s3 = qp[3 * M]; + qv.s4 = qp[4 * M]; + qv.s5 = qp[5 * M]; + qv.s6 = qp[6 * M]; + qv.s7 = qp[7 * M]; + qp += 8 * M; + + uint4 hv; + hv.s0 = qhp[0]; + hv.s1 = qhp[M]; + hv.s2 = qhp[2 * M]; + hv.s3 = qhp[3 * M]; + qhp += 4 * M; + + MMA_K32(qv.s0, qv.s1, qv.s2, qv.s3, hv.s0, hv.s1, g.s0, g.s1, bp) + MMA_K32(qv.s4, qv.s5, qv.s6, qv.s7, hv.s2, hv.s3, g.s2, g.s3, bp + 2048) + bp += 4096; + } + + c_row_t uc; + uc.v = __builtin_astype(acc, half4096); + + global float * p = (global float *)(dst + offsetd) + (ulong)tn * 64 * M + row + lane; + const int nv = N - tn * 64; + store_c16(p, uc.s.h0, 0, nv, M); + store_c16(p, uc.s.h1, 16, nv, M); + store_c16(p, uc.s.h2, 32, nv, M); + store_c16(p, uc.s.h3, 48, nv, M); +} diff --git a/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q8_0_f32_coopmat.cl b/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q8_0_f32_coopmat.cl new file mode 100644 index 000000000..e0a86a15f --- /dev/null +++ b/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q8_0_f32_coopmat.cl @@ -0,0 +1,157 @@ +#pragma OPENCL EXTENSION cl_khr_fp16 : enable +#pragma OPENCL EXTENSION cl_qcom_kernel_cooperative_matrix : enable + +// Q8_0 x F32 GEMM with cl_qcom_kernel_cooperative_matrix: one 64x64 f16 C tile per wave, +// A dequantized in registers, B packed by kernel_pack_b_f16_coopmat. + +typedef half mat_a_t __attribute__((qcom_coop_mat(CLK_COOPERATIVE_MATRIX_SCOPE_SUBGROUP_QCOM, 64, 16, CLK_COOPERATIVE_MATRIX_A_QCOM))); +typedef half mat_b_t __attribute__((qcom_coop_mat(CLK_COOPERATIVE_MATRIX_SCOPE_SUBGROUP_QCOM, 16, 64, CLK_COOPERATIVE_MATRIX_B_QCOM))); +typedef half mat_c_t __attribute__((qcom_coop_mat(CLK_COOPERATIVE_MATRIX_SCOPE_SUBGROUP_QCOM, 64, 64, CLK_COOPERATIVE_MATRIX_ACCUMULATOR_QCOM))); + +typedef half half1024 __attribute__((ext_vector_type(1024))); +typedef half half4096 __attribute__((ext_vector_type(4096))); + +typedef union { + half1024 v; + half16 h; +} a_row_t; + +typedef union { + half4096 v; + struct { + half16 h0; + half16 h1; + half16 h2; + half16 h3; + } s; +} c_row_t; + +inline half4 dequant4(uint v, half d) { + return convert_half4(as_char4(v)) * d; +} + +inline void store_c16(global float * p, half16 h, int j0, int nv, int M) { + float16 f = convert_float16(h); + if (j0 + 0 < nv) p[(ulong)(j0 + 0) * M] = f.s0; + if (j0 + 1 < nv) p[(ulong)(j0 + 1) * M] = f.s1; + if (j0 + 2 < nv) p[(ulong)(j0 + 2) * M] = f.s2; + if (j0 + 3 < nv) p[(ulong)(j0 + 3) * M] = f.s3; + if (j0 + 4 < nv) p[(ulong)(j0 + 4) * M] = f.s4; + if (j0 + 5 < nv) p[(ulong)(j0 + 5) * M] = f.s5; + if (j0 + 6 < nv) p[(ulong)(j0 + 6) * M] = f.s6; + if (j0 + 7 < nv) p[(ulong)(j0 + 7) * M] = f.s7; + if (j0 + 8 < nv) p[(ulong)(j0 + 8) * M] = f.s8; + if (j0 + 9 < nv) p[(ulong)(j0 + 9) * M] = f.s9; + if (j0 + 10 < nv) p[(ulong)(j0 + 10) * M] = f.sa; + if (j0 + 11 < nv) p[(ulong)(j0 + 11) * M] = f.sb; + if (j0 + 12 < nv) p[(ulong)(j0 + 12) * M] = f.sc; + if (j0 + 13 < nv) p[(ulong)(j0 + 13) * M] = f.sd; + if (j0 + 14 < nv) p[(ulong)(j0 + 14) * M] = f.se; + if (j0 + 15 < nv) p[(ulong)(j0 + 15) * M] = f.sf; +} + +#define LOAD_Q(qv, dd) { \ + qv.s0 = qp[0]; \ + qv.s1 = qp[M]; \ + qv.s2 = qp[2 * M]; \ + qv.s3 = qp[3 * M]; \ + qv.s4 = qp[4 * M]; \ + qv.s5 = qp[5 * M]; \ + qv.s6 = qp[6 * M]; \ + qv.s7 = qp[7 * M]; \ + dd = dp[0]; \ + qp += 8 * M; \ + dp += M; } + +#define MMA_K32(qv, dd, b) { \ + a_row_t ua0; \ + a_row_t ua1; \ + ua0.h = (half16)(dequant4(qv.s0, dd), dequant4(qv.s1, dd), dequant4(qv.s2, dd), dequant4(qv.s3, dd)); \ + ua1.h = (half16)(dequant4(qv.s4, dd), dequant4(qv.s5, dd), dequant4(qv.s6, dd), dequant4(qv.s7, dd)); \ + mat_a_t a0 = __builtin_astype(ua0.v, mat_a_t); \ + mat_a_t a1 = __builtin_astype(ua1.v, mat_a_t); \ + mat_b_t b0 = qcom_coop_mat_load((b), CLK_COOPERATIVE_MATRIX_LAYOUT_COLUMN_MAJOR_QCOM, 16); \ + mat_b_t b1 = qcom_coop_mat_load((b) + 1024, CLK_COOPERATIVE_MATRIX_LAYOUT_COLUMN_MAJOR_QCOM, 16); \ + acc = qcom_coop_mat_mulAdd(a0, b0, acc, CLK_COOPERATIVE_MATRIX_OPERAND_NONE_QCOM); \ + acc = qcom_coop_mat_mulAdd(a1, b1, acc, CLK_COOPERATIVE_MATRIX_OPERAND_NONE_QCOM); } + +kernel void kernel_gemm_noshuffle_q8_0_f32_coopmat( + global const uint * src0_q, // A: q8_0 quants, [K/4][M] u32 + global const half * src0_d, // A: q8_0 scales, [K/32][M] f16 + global const half * src1, // B: packed f16 tiles + global char * dst, // C: [N][M] f32 + ulong offsetd, + int K, + int M, + int N +) { + const int lane = get_local_id(0); + const int row = get_global_id(1) * 64; + const int tn = get_global_id(2); + const int n_tiles = (N + 63) / 64; + + if (tn >= n_tiles) { + return; + } + + global const uint * qp = src0_q + row + lane; + global const half * dp = src0_d + row + lane; + global const half * bp = src1 + (ulong)tn * (K / 16) * 1024; + + mat_c_t acc = qcom_coop_mat_init(0.0); + + { + const int odd = (K / 32) & 1; + const int kb = odd ? 0 : 32; + uint8 qa; + uint8 qb; + half da; + half db; + qa.s0 = qp[0]; + qa.s1 = qp[M]; + qa.s2 = qp[2 * M]; + qa.s3 = qp[3 * M]; + qa.s4 = qp[4 * M]; + qa.s5 = qp[5 * M]; + qa.s6 = qp[6 * M]; + qa.s7 = qp[7 * M]; + da = odd ? 0.0h : dp[0]; + qb.s0 = qp[(kb / 4 + 0) * M]; + qb.s1 = qp[(kb / 4 + 1) * M]; + qb.s2 = qp[(kb / 4 + 2) * M]; + qb.s3 = qp[(kb / 4 + 3) * M]; + qb.s4 = qp[(kb / 4 + 4) * M]; + qb.s5 = qp[(kb / 4 + 5) * M]; + qb.s6 = qp[(kb / 4 + 6) * M]; + qb.s7 = qp[(kb / 4 + 7) * M]; + db = dp[(kb / 32) * M]; + MMA_K32(qa, da, bp) + MMA_K32(qb, db, bp + (kb / 16) * 1024) + qp += (kb / 4 + 8) * M; + dp += (kb / 32 + 1) * M; + bp += (kb / 16 + 2) * 1024; + } + +#pragma unroll 1 + for (int k0 = (K / 32) & 1 ? 32 : 64; k0 < K; k0 += 64) { + uint8 qa; + uint8 qb; + half da; + half db; + LOAD_Q(qa, da) + LOAD_Q(qb, db) + MMA_K32(qa, da, bp) + MMA_K32(qb, db, bp + 2048) + bp += 4096; + } + + c_row_t uc; + uc.v = __builtin_astype(acc, half4096); + + global float * p = (global float *)(dst + offsetd) + (ulong)tn * 64 * M + row + lane; + const int nv = N - tn * 64; + store_c16(p, uc.s.h0, 0, nv, M); + store_c16(p, uc.s.h1, 16, nv, M); + store_c16(p, uc.s.h2, 32, nv, M); + store_c16(p, uc.s.h3, 48, nv, M); +} diff --git a/ggml/src/ggml-opencl/kernels/pack_b_f16_coopmat.cl b/ggml/src/ggml-opencl/kernels/pack_b_f16_coopmat.cl new file mode 100644 index 000000000..3f60223b8 --- /dev/null +++ b/ggml/src/ggml-opencl/kernels/pack_b_f16_coopmat.cl @@ -0,0 +1,48 @@ +#pragma OPENCL EXTENSION cl_khr_fp16 : enable + +// Pack B f32 [N][K] into f16 64x16 tiles (column-major, ld 16) at (tn * (K/16) + ks) * 1024. +// Columns >= N are zero. A workgroup (64, 4) packs 64 columns x 64 k. +#define PACK_LDS (64 + 8) + +kernel void kernel_pack_b_f16_coopmat( + global const char * src1, + ulong offset1, + global half * dst, + int K, + int N +) { + local half tile[64 * PACK_LDS]; + + const int l = get_local_id(1) * 64 + get_local_id(0); + const int tn = get_group_id(0); + const int kt = get_group_id(1); + const int k0 = kt * 64; + + global const float * src = (global const float *)(src1 + offset1); + + for (int j = 0; j < 4; j++) { + const int f = j * 256 + l; + const int r = f >> 4; + const int c = (f & 15) * 4; + const int n = tn * 64 + r; + float4 v = (float4)(0.0f); + if (n < N && k0 + c < K) { + v = vload4(0, src + (ulong)n * K + k0 + c); + } + vstore_half4(v, 0, tile + r * PACK_LDS + c); + } + + barrier(CLK_LOCAL_MEM_FENCE); + + global half * out = dst + ((ulong)tn * (K / 16) + kt * 4) * 1024; + for (int j = 0; j < 2; j++) { + const int q = j * 256 + l; + const int ks = q >> 7; + const int n = (q >> 1) & 63; + const int h = q & 1; + if (kt * 4 + ks < K / 16) { + half8 v = vload8(0, tile + n * PACK_LDS + ks * 16 + h * 8); + vstore8(v, 0, out + q * 8); + } + } +}