From b531e4c7494e5a9349e40d2ea9efc3686f635cdf Mon Sep 17 00:00:00 2001 From: lhez Date: Wed, 21 Oct 2026 00:48:13 -0700 Subject: [PATCH] opencl: add dp4a GEMM variants to vector load activation (#30753) * opencl: add dp4a GEMM variants with vector activation loads * opencl: rename kernels and env vars --- ggml/src/ggml-opencl/ggml-opencl.cpp | 50 +++++-- .../kernels/gemm_noshuffle_q4_0_q8_1_dp4a.cl | 126 ++++++++++++++++ .../kernels/gemm_noshuffle_q4_k_q8_1_dp4a.cl | 138 ++++++++++++++++++ .../kernels/gemm_noshuffle_q5_k_q8_1_dp4a.cl | 138 ++++++++++++++++++ .../kernels/gemm_noshuffle_q6_k_q8_1_dp4a.cl | 114 +++++++++++++++ .../kernels/gemm_noshuffle_q8_0_q8_1_dp4a.cl | 122 ++++++++++++++++ 6 files changed, 679 insertions(+), 9 deletions(-) diff --git a/ggml/src/ggml-opencl/ggml-opencl.cpp b/ggml/src/ggml-opencl/ggml-opencl.cpp index 70f199dba..6071b6e58 100644 --- a/ggml/src/ggml-opencl/ggml-opencl.cpp +++ b/ggml/src/ggml-opencl/ggml-opencl.cpp @@ -1243,6 +1243,7 @@ struct ggml_backend_opencl_context { cl_kernel kernel_gemm_noshuffle_q4_1_f32; cl_kernel kernel_gemm_noshuffle_q8_0_f32, kernel_gemm_noshuffle_q8_0_f32_bin; cl_kernel kernel_gemm_noshuffle_q8_0_q8_1_dp4a = nullptr; // dp4a (int8) dense q8_0 prefill GEMM (opt-in) + cl_kernel kernel_gemm_noshuffle_q8_0_q8_1_dp4a_act_vload4 = nullptr; cl_kernel kernel_gemm_noshuffle_q8_0_q8_1_dp4a_wimg = nullptr; // q8_0 dense dp4a, weights via texture (opt-in) cl_kernel kernel_gemm_noshuffle_q8_0_q8_1_dp4a_ila_a8_bin = nullptr; cl_kernel kernel_gemv_noshuffle_q8_0_f32; @@ -1264,6 +1265,7 @@ struct ggml_backend_opencl_context { cl_kernel kernel_gemm_noshuffle_q4_k_q8_1_dp4a_ila_a8_bin; cl_kernel kernel_gemv_noshuffle_q4_k_f32_32b_trans; cl_kernel kernel_gemm_noshuffle_q4_k_q8_1_dp4a = nullptr; // dp4a (int8) dense prefill GEMM + cl_kernel kernel_gemm_noshuffle_q4_k_q8_1_dp4a_act_vload4 = nullptr; cl_kernel kernel_gemm_noshuffle_q4_k_q8_1_dp4a_wimg = nullptr; // dp4a dense prefill GEMM, weights via texture (X1 opt-in) cl_kernel kernel_gemm_noshuffle_q4_k_q8_1_dp4a_narrow = nullptr; cl_kernel kernel_gemm_noshuffle_q4_k_q8_1_dp4a_narrow_wimg = nullptr; @@ -1271,7 +1273,9 @@ struct ggml_backend_opencl_context { int tile_n_dp4a_narrow_q4_k = 32; int max_n_dp4a_narrow_q4_k = 16; cl_kernel kernel_gemm_noshuffle_q5_k_q8_1_dp4a = nullptr; // dp4a (int8) dense q5_K prefill GEMM + cl_kernel kernel_gemm_noshuffle_q5_k_q8_1_dp4a_act_vload4 = nullptr; cl_kernel kernel_gemm_noshuffle_q6_k_q8_1_dp4a = nullptr; // dp4a (int8) dense q6_K prefill GEMM + cl_kernel kernel_gemm_noshuffle_q6_k_q8_1_dp4a_act_vload4 = nullptr; cl_kernel kernel_quant_a_q8_1; // plain activation q8_1 pre-pass cl_kernel kernel_gemm_noshuffle_q4_k_f32_r1; cl_kernel kernel_gemm_noshuffle_q4_k_f32_kimg; @@ -1305,6 +1309,7 @@ struct ggml_backend_opencl_context { cl_kernel kernel_gemm_noshuffle_iq4_nl_f32; cl_kernel kernel_gemm_noshuffle_iq4_nl_q8_1_dp4a = nullptr; // dp4a (int8) dense IQ4_NL prefill GEMM cl_kernel kernel_gemm_noshuffle_q4_0_q8_1_dp4a = nullptr; // dp4a (int8) dense q4_0 prefill GEMM + cl_kernel kernel_gemm_noshuffle_q4_0_q8_1_dp4a_act_vload4 = nullptr; cl_kernel kernel_gemm_coopk_q4_0_q8_1_dp4a_c4 = nullptr; cl_kernel kernel_gemm_coopk_q4_0_q8_1_dp4a_c2 = nullptr; int nsg_coopk_dp4a_q4_0 = 8; @@ -4347,6 +4352,7 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) { #endif cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q4_0_q8_1_dp4a = clCreateKernel(prog, "kernel_gemm_noshuffle_q4_0_q8_1_dp4a", &err), err)); + CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q4_0_q8_1_dp4a_act_vload4 = clCreateKernel(prog, "kernel_gemm_noshuffle_q4_0_q8_1_dp4a_act_vload4", &err), err)); CL_CHECK(clReleaseProgram(prog)); GGML_LOG_CONT("."); } @@ -4486,6 +4492,7 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) { cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), dp4a_opts); CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q4_k_q8_1_dp4a = clCreateKernel(prog, "kernel_gemm_noshuffle_q4_k_q8_1_dp4a", &err), err)); CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q4_k_q8_1_dp4a_wimg = clCreateKernel(prog, "kernel_gemm_noshuffle_q4_k_q8_1_dp4a_wimg", &err), err)); + CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q4_k_q8_1_dp4a_act_vload4 = clCreateKernel(prog, "kernel_gemm_noshuffle_q4_k_q8_1_dp4a_act_vload4", &err), err)); CL_CHECK(clReleaseProgram(prog)); // Compile a smaller N tile to avoid unused columns in small batches. @@ -4518,6 +4525,7 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) { cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q8_0_q8_1_dp4a = clCreateKernel(prog, "kernel_gemm_noshuffle_q8_0_q8_1_dp4a", &err), err)); CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q8_0_q8_1_dp4a_wimg = clCreateKernel(prog, "kernel_gemm_noshuffle_q8_0_q8_1_dp4a_wimg", &err), err)); + CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q8_0_q8_1_dp4a_act_vload4 = clCreateKernel(prog, "kernel_gemm_noshuffle_q8_0_q8_1_dp4a_act_vload4", &err), err)); CL_CHECK(clReleaseProgram(prog)); GGML_LOG_CONT("."); } @@ -4533,6 +4541,7 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) { #endif cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q5_k_q8_1_dp4a = clCreateKernel(prog, "kernel_gemm_noshuffle_q5_k_q8_1_dp4a", &err), err)); + CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q5_k_q8_1_dp4a_act_vload4 = clCreateKernel(prog, "kernel_gemm_noshuffle_q5_k_q8_1_dp4a_act_vload4", &err), err)); CL_CHECK(clReleaseProgram(prog)); GGML_LOG_CONT("."); } @@ -4548,6 +4557,7 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) { #endif cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q6_k_q8_1_dp4a = clCreateKernel(prog, "kernel_gemm_noshuffle_q6_k_q8_1_dp4a", &err), err)); + CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q6_k_q8_1_dp4a_act_vload4 = clCreateKernel(prog, "kernel_gemm_noshuffle_q6_k_q8_1_dp4a_act_vload4", &err), err)); CL_CHECK(clReleaseProgram(prog)); GGML_LOG_CONT("."); } @@ -20430,11 +20440,13 @@ static void ggml_cl_mul_mat_q4_0_f32_adreno(ggml_backend_t backend, const ggml_t return; } - // dp4a (int8) dense prefill GEMM, default off + static const char * q4_0_act_vload4_env = getenv("GGML_OPENCL_Q4_0_DP4A_ACT_VLOAD4"); + const bool q4_0_act_vload4_on = q4_0_act_vload4_env == nullptr || atoi(q4_0_act_vload4_env) != 0; + // Enable dense prefill by default on X2E when the uint4 activation tile is enabled. static const char * q4_0_dense_dp4a_env = getenv("GGML_OPENCL_Q4_0_DENSE_DP4A"); bool q4_0_dense_dp4a_on = q4_0_dense_dp4a_env ? (atoi(q4_0_dense_dp4a_env) != 0) - : false; + : (backend_ctx->adreno_gen == ADRENO_GPU_GEN::X2E && q4_0_act_vload4_on); // dot prod has to be available q4_0_dense_dp4a_on = backend_ctx->has_integer_dot && q4_0_dense_dp4a_on; @@ -20461,7 +20473,9 @@ static void ggml_cl_mul_mat_q4_0_f32_adreno(ggml_backend_t backend, const ggml_t size_t q_global[1] = { (size_t)(((n_blocks + 63) / 64) * 64) }; backend_ctx->enqueue_ndrange_kernel(qk, 1, q_global, q_local, dst); - cl_kernel dk = backend_ctx->kernel_gemm_noshuffle_q4_0_q8_1_dp4a; + cl_kernel dk = q4_0_act_vload4_on + ? backend_ctx->kernel_gemm_noshuffle_q4_0_q8_1_dp4a_act_vload4 + : backend_ctx->kernel_gemm_noshuffle_q4_0_q8_1_dp4a; int ai = 0; CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &extra0_q4_0->q)); CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &extra0_q4_0->d)); @@ -21754,8 +21768,13 @@ static void ggml_cl_mul_mat_q8_0_f32_adreno(ggml_backend_t backend, const ggml_t } } - cl_kernel dk = use_wimg ? backend_ctx->kernel_gemm_noshuffle_q8_0_q8_1_dp4a_wimg - : backend_ctx->kernel_gemm_noshuffle_q8_0_q8_1_dp4a; + static const char * q8_0_act_vload4_env = getenv("GGML_OPENCL_Q8_0_DP4A_ACT_VLOAD4"); + const bool q8_0_act_vload4_on = q8_0_act_vload4_env == nullptr || atoi(q8_0_act_vload4_env) != 0; + cl_kernel dk = use_wimg + ? backend_ctx->kernel_gemm_noshuffle_q8_0_q8_1_dp4a_wimg + : q8_0_act_vload4_on + ? backend_ctx->kernel_gemm_noshuffle_q8_0_q8_1_dp4a_act_vload4 + : backend_ctx->kernel_gemm_noshuffle_q8_0_q8_1_dp4a; int ai = 0; if (use_wimg) { CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &q8_q_img)); @@ -22703,11 +22722,16 @@ static void ggml_cl_mul_mat_q4_k_f32_adreno(ggml_backend_t backend, const ggml_t const bool use_narrow = backend_ctx->kernel_gemm_noshuffle_q4_k_q8_1_dp4a_narrow != nullptr && N <= backend_ctx->max_n_dp4a_narrow_q4_k; + static const char * q4_k_act_vload4_env = getenv("GGML_OPENCL_Q4_K_DP4A_ACT_VLOAD4"); + const bool q4_k_act_vload4_on = q4_k_act_vload4_env == nullptr || atoi(q4_k_act_vload4_env) != 0; cl_kernel dk = use_narrow ? (use_wimg ? backend_ctx->kernel_gemm_noshuffle_q4_k_q8_1_dp4a_narrow_wimg : backend_ctx->kernel_gemm_noshuffle_q4_k_q8_1_dp4a_narrow) - : (use_wimg ? backend_ctx->kernel_gemm_noshuffle_q4_k_q8_1_dp4a_wimg - : backend_ctx->kernel_gemm_noshuffle_q4_k_q8_1_dp4a); + : use_wimg + ? backend_ctx->kernel_gemm_noshuffle_q4_k_q8_1_dp4a_wimg + : q4_k_act_vload4_on + ? backend_ctx->kernel_gemm_noshuffle_q4_k_q8_1_dp4a_act_vload4 + : backend_ctx->kernel_gemm_noshuffle_q4_k_q8_1_dp4a; int ai = 0; if (use_wimg) { CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &q4k_q_img)); @@ -23369,7 +23393,11 @@ static void ggml_cl_mul_mat_q6_K_f32_adreno(ggml_backend_t backend, const ggml_t size_t q_global[1] = { (size_t)(((n_blocks + 63) / 64) * 64) }; backend_ctx->enqueue_ndrange_kernel(qk, 1, q_global, q_local, dst); - cl_kernel dk = backend_ctx->kernel_gemm_noshuffle_q6_k_q8_1_dp4a; + static const char * q6_k_act_vload4_env = getenv("GGML_OPENCL_Q6_K_DP4A_ACT_VLOAD4"); + const bool q6_k_act_vload4_on = q6_k_act_vload4_env == nullptr || atoi(q6_k_act_vload4_env) != 0; + cl_kernel dk = q6_k_act_vload4_on + ? backend_ctx->kernel_gemm_noshuffle_q6_k_q8_1_dp4a_act_vload4 + : backend_ctx->kernel_gemm_noshuffle_q6_k_q8_1_dp4a; int ai = 0; CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &extra0_q6_K->ql)); CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &extra0_q6_K->qh)); @@ -23925,7 +23953,11 @@ static void ggml_cl_mul_mat_q5_K_f32_adreno(ggml_backend_t backend, const ggml_t size_t q_global[1] = { (size_t)(((n_blocks + 63) / 64) * 64) }; backend_ctx->enqueue_ndrange_kernel(qk, 1, q_global, q_local, dst); - cl_kernel dk = backend_ctx->kernel_gemm_noshuffle_q5_k_q8_1_dp4a; + static const char * q5_k_act_vload4_env = getenv("GGML_OPENCL_Q5_K_DP4A_ACT_VLOAD4"); + const bool q5_k_act_vload4_on = q5_k_act_vload4_env == nullptr || atoi(q5_k_act_vload4_env) != 0; + cl_kernel dk = q5_k_act_vload4_on + ? backend_ctx->kernel_gemm_noshuffle_q5_k_q8_1_dp4a_act_vload4 + : backend_ctx->kernel_gemm_noshuffle_q5_k_q8_1_dp4a; int ai = 0; CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &extra0_q5_k->q)); CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &extra0_q5_k->qh)); diff --git a/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q4_0_q8_1_dp4a.cl b/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q4_0_q8_1_dp4a.cl index 446a8eb18..6418d0f8f 100644 --- a/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q4_0_q8_1_dp4a.cl +++ b/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q4_0_q8_1_dp4a.cl @@ -125,3 +125,129 @@ kernel void kernel_gemm_noshuffle_q4_0_q8_1_dp4a( } #undef NGROUPS } + +// Stage each 32-K activation block as two uint4 values. +inline int dot8_q8a_v(uint8 qw, uint4 a0, uint4 a1) { + int r = 0; + r = dot_acc_sat_4x8packed_ss_int(qw.s0, a0.x, r); + r = dot_acc_sat_4x8packed_ss_int(qw.s1, a0.y, r); + r = dot_acc_sat_4x8packed_ss_int(qw.s2, a0.z, r); + r = dot_acc_sat_4x8packed_ss_int(qw.s3, a0.w, r); + r = dot_acc_sat_4x8packed_ss_int(qw.s4, a1.x, r); + r = dot_acc_sat_4x8packed_ss_int(qw.s5, a1.y, r); + r = dot_acc_sat_4x8packed_ss_int(qw.s6, a1.z, r); + r = dot_acc_sat_4x8packed_ss_int(qw.s7, a1.w, r); + return r; +} + +__attribute__((qcom_wave_pair_mode(1))) +kernel void kernel_gemm_noshuffle_q4_0_q8_1_dp4a_act_vload4( + __global const ushort * src0_q, + __global const half * src0_d, + __global const uint * src1_qa, + __global const half * src1_da, + __global const half * src1_sa, + __global float * dst, + ulong offsetd, + int m, + int n_no_padding, + int k +) { + dst = (global float *)((global char *)dst + offsetd); + + const uint lid = get_local_id(0); + const uint block_id_m = get_global_id(1); + const uint block_id_n = get_global_id(2); + + const uint row = block_id_m * 64 + lid; + const uint col_base = block_id_n * TILESIZE_N; + const bool row_valid = row < (uint)m; + const uint rrow = row_valid ? row : 0; + + const uint k_u = (uint)k >> 2; + const uint k_b = (uint)k >> 5; + + __local uint4 sh_qa4[TILESIZE_N][2]; + __local half sh_d[TILESIZE_N]; + __local half sh_s[TILESIZE_N]; + +#define NGROUPS (TILESIZE_N / 4) + float4 acc[NGROUPS]; + #pragma unroll + for (int g = 0; g < NGROUPS; ++g) { + acc[g] = (float4)(0.0f); + } + + for (uint step = 0; step < (uint)k; step += 32) { + const uint sub = step >> 5; + + const float d_w = (float)src0_d[rrow + sub * (uint)m]; + + const uint qsbase = rrow + (step >> 2) * (uint)m; + uint8 qw; + qw.s0 = EXP4(src0_q[qsbase + 0 * m]); + qw.s1 = EXP4(src0_q[qsbase + 1 * m]); + qw.s2 = EXP4(src0_q[qsbase + 2 * m]); + qw.s3 = EXP4(src0_q[qsbase + 3 * m]); + qw.s4 = EXP4(src0_q[qsbase + 4 * m]); + qw.s5 = EXP4(src0_q[qsbase + 5 * m]); + qw.s6 = EXP4(src0_q[qsbase + 6 * m]); + qw.s7 = EXP4(src0_q[qsbase + 7 * m]); + + for (uint idx = lid; idx < TILESIZE_N * 2; idx += 64) { + const uint t = idx >> 1; + const uint v = idx & 1; + const uint c = col_base + t; + sh_qa4[t][v] = (c < (uint)n_no_padding) + ? vload4(0, src1_qa + c * k_u + (step >> 2) + (v << 2)) + : (uint4)(0u); + } + if (lid < TILESIZE_N) { + const uint c = col_base + lid; + sh_d[lid] = (c < (uint)n_no_padding) ? src1_da[c * k_b + sub] : (half)0; + sh_s[lid] = (c < (uint)n_no_padding) ? src1_sa[c * k_b + sub] : (half)0; + } + barrier(CLK_LOCAL_MEM_FENCE); + +#define LD4(arr, b) ((float4)((float)arr[(b)+0], (float)arr[(b)+1], (float)arr[(b)+2], (float)arr[(b)+3])) + #pragma unroll + for (int g = 0; g < NGROUPS; ++g) { + const int b = g * 4; + float4 rf; +#define DOTV(T) dot8_q8a_v(qw, sh_qa4[T][0], sh_qa4[T][1]) + rf.s0 = (float)DOTV(b+0); + rf.s1 = (float)DOTV(b+1); + rf.s2 = (float)DOTV(b+2); + rf.s3 = (float)DOTV(b+3); +#undef DOTV + + acc[g] += d_w * (LD4(sh_d, b) * rf - 8.0f * LD4(sh_s, b)); + } +#undef LD4 + barrier(CLK_LOCAL_MEM_FENCE); + } + + if (!row_valid) { + return; + } + + #pragma unroll + for (int g = 0; g < NGROUPS; ++g) { + const uint b = (uint)(g * 4); + const float4 a = acc[g]; + const uint c0 = col_base + b; + if (c0 + 0 < (uint)n_no_padding) { + dst[(c0 + 0) * (uint)m + row] = a.s0; + } + if (c0 + 1 < (uint)n_no_padding) { + dst[(c0 + 1) * (uint)m + row] = a.s1; + } + if (c0 + 2 < (uint)n_no_padding) { + dst[(c0 + 2) * (uint)m + row] = a.s2; + } + if (c0 + 3 < (uint)n_no_padding) { + dst[(c0 + 3) * (uint)m + row] = a.s3; + } + } +#undef NGROUPS +} diff --git a/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q4_k_q8_1_dp4a.cl b/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q4_k_q8_1_dp4a.cl index a3b39b6aa..69cabb946 100644 --- a/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q4_k_q8_1_dp4a.cl +++ b/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q4_k_q8_1_dp4a.cl @@ -279,3 +279,141 @@ kernel void kernel_gemm_noshuffle_q4_k_q8_1_dp4a_wimg( } #undef NGROUPS } + +// Stage each 32-K activation block as two uint4 values. +inline int dot8_q8a_v(uint8 qw, uint4 a0, uint4 a1) { + int r = 0; + r = dot_acc_sat_4x8packed_ss_int(qw.s0, a0.x, r); + r = dot_acc_sat_4x8packed_ss_int(qw.s1, a0.y, r); + r = dot_acc_sat_4x8packed_ss_int(qw.s2, a0.z, r); + r = dot_acc_sat_4x8packed_ss_int(qw.s3, a0.w, r); + r = dot_acc_sat_4x8packed_ss_int(qw.s4, a1.x, r); + r = dot_acc_sat_4x8packed_ss_int(qw.s5, a1.y, r); + r = dot_acc_sat_4x8packed_ss_int(qw.s6, a1.z, r); + r = dot_acc_sat_4x8packed_ss_int(qw.s7, a1.w, r); + return r; +} + +__attribute__((qcom_wave_pair_mode(1))) +kernel void kernel_gemm_noshuffle_q4_k_q8_1_dp4a_act_vload4( + __global const ushort * src0_q, + __global const uchar * src0_s, + __global const half * src0_d, + __global const half * src0_dm, + __global const uint * src1_qa, + __global const half * src1_da, + __global const half * src1_sa, + __global float * dst, + ulong offsetd, + int m, + int n_no_padding, + int k, + uchar mask_d6, + uchar mask_d4, + uchar mask_hi2 +) { + dst = (global float *)((global char *)dst + offsetd); + + const uint lid = get_local_id(0); + const uint block_id_m = get_global_id(1); + const uint block_id_n = get_global_id(2); + + const uint row = block_id_m * 64 + lid; + const uint col_base = block_id_n * TILESIZE_N; + const bool row_valid = row < (uint)m; + const uint rrow = row_valid ? row : 0; + + const uint k_u = (uint)k >> 2; + const uint k_b = (uint)k >> 5; + + __local uint4 sh_qa4[TILESIZE_N][2]; + __local half sh_d[TILESIZE_N]; + __local half sh_s[TILESIZE_N]; + +#define NGROUPS (TILESIZE_N / 4) + float4 acc[NGROUPS]; + #pragma unroll + for (int g = 0; g < NGROUPS; ++g) { + acc[g] = (float4)(0.0f); + } + + for (uint step = 0; step < (uint)k; step += 32) { + const uint sub = step >> 5; + const uint sb_idx = step / QK_K; + const uint sub_idx = sub & 7; + + const float dd = (float)src0_d [rrow + sb_idx * m]; + const float dmm = (float)src0_dm[rrow + sb_idx * m]; + global const uchar * sc = src0_s + sb_idx * K_SCALE_SIZE * (uint)m + rrow; + uchar sv, mn; + get_scale_min_k4(sub_idx, sc, (uint)m, &sv, &mn, mask_d6, mask_d4, mask_hi2); + const float scale = dd * (float)sv; + const float minv = dmm * (float)mn; + + const uint wbase = rrow + (step >> 2) * (uint)m; + uint8 qw; + qw.s0 = EXP4(src0_q[wbase + 0 * m]); + qw.s1 = EXP4(src0_q[wbase + 1 * m]); + qw.s2 = EXP4(src0_q[wbase + 2 * m]); + qw.s3 = EXP4(src0_q[wbase + 3 * m]); + qw.s4 = EXP4(src0_q[wbase + 4 * m]); + qw.s5 = EXP4(src0_q[wbase + 5 * m]); + qw.s6 = EXP4(src0_q[wbase + 6 * m]); + qw.s7 = EXP4(src0_q[wbase + 7 * m]); + + for (uint idx = lid; idx < TILESIZE_N * 2; idx += 64) { + const uint t = idx >> 1; + const uint v = idx & 1; + const uint c = col_base + t; + sh_qa4[t][v] = (c < (uint)n_no_padding) + ? vload4(0, src1_qa + c * k_u + (step >> 2) + (v << 2)) + : (uint4)(0u); + } + if (lid < TILESIZE_N) { + const uint c = col_base + lid; + sh_d[lid] = (c < (uint)n_no_padding) ? src1_da[c * k_b + sub] : (half)0; + sh_s[lid] = (c < (uint)n_no_padding) ? src1_sa[c * k_b + sub] : (half)0; + } + barrier(CLK_LOCAL_MEM_FENCE); + +#define LD4(arr, b) ((float4)((float)arr[(b)+0], (float)arr[(b)+1], (float)arr[(b)+2], (float)arr[(b)+3])) + #pragma unroll + for (int g = 0; g < NGROUPS; ++g) { + const int b = g * 4; + float4 rf; +#define DOTV(T) dot8_q8a_v(qw, sh_qa4[T][0], sh_qa4[T][1]) + rf.s0 = (float)DOTV(b+0); + rf.s1 = (float)DOTV(b+1); + rf.s2 = (float)DOTV(b+2); + rf.s3 = (float)DOTV(b+3); +#undef DOTV + acc[g] += scale * LD4(sh_d, b) * rf - minv * LD4(sh_s, b); + } +#undef LD4 + barrier(CLK_LOCAL_MEM_FENCE); + } + + if (!row_valid) { + return; + } + + #pragma unroll + for (int g = 0; g < NGROUPS; ++g) { + const uint b = (uint)(g * 4); + const float4 a = acc[g]; + const uint c0 = col_base + b; + if (c0 + 0 < (uint)n_no_padding) { + dst[(c0 + 0) * (uint)m + row] = a.s0; + } + if (c0 + 1 < (uint)n_no_padding) { + dst[(c0 + 1) * (uint)m + row] = a.s1; + } + if (c0 + 2 < (uint)n_no_padding) { + dst[(c0 + 2) * (uint)m + row] = a.s2; + } + if (c0 + 3 < (uint)n_no_padding) { + dst[(c0 + 3) * (uint)m + row] = a.s3; + } + } +#undef NGROUPS +} diff --git a/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q5_k_q8_1_dp4a.cl b/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q5_k_q8_1_dp4a.cl index aaeed68f6..eef474b47 100644 --- a/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q5_k_q8_1_dp4a.cl +++ b/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q5_k_q8_1_dp4a.cl @@ -162,3 +162,141 @@ kernel void kernel_gemm_noshuffle_q5_k_q8_1_dp4a( } #undef NGROUPS } + +// Stage each 32-K activation block as two uint4 values. +inline int dot8_q8a_v(uint8 qw, uint4 a0, uint4 a1) { + int r = 0; + r = dot_acc_sat_4x8packed_ss_int(qw.s0, a0.x, r); + r = dot_acc_sat_4x8packed_ss_int(qw.s1, a0.y, r); + r = dot_acc_sat_4x8packed_ss_int(qw.s2, a0.z, r); + r = dot_acc_sat_4x8packed_ss_int(qw.s3, a0.w, r); + r = dot_acc_sat_4x8packed_ss_int(qw.s4, a1.x, r); + r = dot_acc_sat_4x8packed_ss_int(qw.s5, a1.y, r); + r = dot_acc_sat_4x8packed_ss_int(qw.s6, a1.z, r); + r = dot_acc_sat_4x8packed_ss_int(qw.s7, a1.w, r); + return r; +} + +__attribute__((qcom_wave_pair_mode(1))) +kernel void kernel_gemm_noshuffle_q5_k_q8_1_dp4a_act_vload4( + __global const ushort * src0_q, + __global const uchar * src0_qh, + __global const uchar * src0_s, + __global const half * src0_d, + __global const half * src0_dm, + __global const uint * src1_qa, + __global const half * src1_da, + __global const half * src1_sa, + __global float * dst, + ulong offsetd, + int m, + int n_no_padding, + int k, + uchar mask_d6, + uchar mask_d4, + uchar mask_hi2 +) { + dst = (global float *)((global char *)dst + offsetd); + + const uint lid = get_local_id(0); + const uint block_id_m = get_global_id(1); + const uint block_id_n = get_global_id(2); + + const uint row = block_id_m * 64 + lid; + const uint col_base = block_id_n * TILESIZE_N; + const bool row_valid = row < (uint)m; + const uint rrow = row_valid ? row : 0; + + const uint num_superblocks = (uint)k / QK_K; + const uint k_u = (uint)k >> 2; + const uint k_b = (uint)k >> 5; + + __local uint4 sh_qa4[TILESIZE_N][2]; + __local half sh_d[TILESIZE_N]; + __local half sh_s[TILESIZE_N]; + +#define NGROUPS (TILESIZE_N / 4) + float4 acc[NGROUPS]; + #pragma unroll + for (int g = 0; g < NGROUPS; ++g) { + acc[g] = (float4)(0.0f); + } + + for (uint step = 0; step < (uint)k; step += 32) { + const uint sub = step >> 5; + const uint sb_idx = step / QK_K; + const uint sub_idx = sub & 7; + + const float dd = (float)src0_d [rrow + sb_idx * m]; + const float dmm = (float)src0_dm[rrow + sb_idx * m]; + global const uchar * sc = src0_s + rrow * num_superblocks * K_SCALE_SIZE + sb_idx * K_SCALE_SIZE; + uchar sv, mn; + get_scale_min_k4(sub_idx, sc, &sv, &mn, mask_d6, mask_d4, mask_hi2); + const float scale = dd * (float)sv; + const float minv = dmm * (float)mn; + + const uint wbase = rrow + (step >> 2) * (uint)m; + const uint qhbase = rrow + (step >> 3) * (uint)m; + uint8 qw; +#define QWU(u) ( EXP4((uint)src0_q[wbase + (uint)(u) * m]) \ + | EXP1( (uint)((src0_qh[qhbase + (uint)((u) >> 1) * m] >> (((u) & 1) * 4)) & 0x0Fu) ) ) + qw.s0 = QWU(0); qw.s1 = QWU(1); qw.s2 = QWU(2); qw.s3 = QWU(3); + qw.s4 = QWU(4); qw.s5 = QWU(5); qw.s6 = QWU(6); qw.s7 = QWU(7); +#undef QWU + + for (uint idx = lid; idx < TILESIZE_N * 2; idx += 64) { + const uint t = idx >> 1; + const uint v = idx & 1; + const uint c = col_base + t; + sh_qa4[t][v] = (c < (uint)n_no_padding) + ? vload4(0, src1_qa + c * k_u + (step >> 2) + (v << 2)) + : (uint4)(0u); + } + if (lid < TILESIZE_N) { + const uint c = col_base + lid; + sh_d[lid] = (c < (uint)n_no_padding) ? src1_da[c * k_b + sub] : (half)0; + sh_s[lid] = (c < (uint)n_no_padding) ? src1_sa[c * k_b + sub] : (half)0; + } + barrier(CLK_LOCAL_MEM_FENCE); + +#define LD4(arr, b) ((float4)((float)arr[(b)+0], (float)arr[(b)+1], (float)arr[(b)+2], (float)arr[(b)+3])) + #pragma unroll + for (int g = 0; g < NGROUPS; ++g) { + const int b = g * 4; + float4 rf; +#define DOTV(T) dot8_q8a_v(qw, sh_qa4[T][0], sh_qa4[T][1]) + rf.s0 = (float)DOTV(b+0); + rf.s1 = (float)DOTV(b+1); + rf.s2 = (float)DOTV(b+2); + rf.s3 = (float)DOTV(b+3); +#undef DOTV + acc[g] += scale * LD4(sh_d, b) * rf - minv * LD4(sh_s, b); + } +#undef LD4 + barrier(CLK_LOCAL_MEM_FENCE); + } + + if (!row_valid) { + return; + } + + #pragma unroll + for (int g = 0; g < NGROUPS; ++g) { + const uint b = (uint)(g * 4); + const float4 a = acc[g]; + const uint c0 = col_base + b; + if (c0 + 0 < (uint)n_no_padding) { + dst[(c0 + 0) * (uint)m + row] = a.s0; + } + if (c0 + 1 < (uint)n_no_padding) { + dst[(c0 + 1) * (uint)m + row] = a.s1; + } + if (c0 + 2 < (uint)n_no_padding) { + dst[(c0 + 2) * (uint)m + row] = a.s2; + } + if (c0 + 3 < (uint)n_no_padding) { + dst[(c0 + 3) * (uint)m + row] = a.s3; + } + } +#undef NGROUPS +} diff --git a/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q6_k_q8_1_dp4a.cl b/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q6_k_q8_1_dp4a.cl index 382d79fdd..f17403a72 100644 --- a/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q6_k_q8_1_dp4a.cl +++ b/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q6_k_q8_1_dp4a.cl @@ -142,3 +142,117 @@ kernel void kernel_gemm_noshuffle_q6_k_q8_1_dp4a( } #undef NGROUPS } + +// Stage each 32-K activation block as two uint4 values. +__attribute__((qcom_wave_pair_mode(1))) +kernel void kernel_gemm_noshuffle_q6_k_q8_1_dp4a_act_vload4( + __global const ushort * src0_ql, + __global const uchar * src0_qh, + __global const ushort * src0_s, + __global const half * src0_d, + __global const uint * src1_qa, + __global const half * src1_da, + __global float * dst, + ulong offsetd, + int m, + int n_no_padding, + int k +) { + dst = (global float *)((global char *)dst + offsetd); + + const uint lid = get_local_id(0); + const uint block_id_m = get_global_id(1); + const uint block_id_n = get_global_id(2); + + const uint row = block_id_m * 64 + lid; + const uint col_base = block_id_n * TILESIZE_N; + const bool row_valid = row < (uint)m; + const uint rrow = row_valid ? row : 0; + + const uint k_u = (uint)k >> 2; + const uint k_b = (uint)k >> 5; + + __local uint4 sh_qa4[TILESIZE_N][2]; + __local half sh_d[TILESIZE_N]; + +#define NGROUPS (TILESIZE_N / 4) + float4 acc[NGROUPS]; + #pragma unroll + for (int g = 0; g < NGROUPS; ++g) { + acc[g] = (float4)(0.0f); + } + + for (uint step = 0; step < (uint)k; step += 32) { + const uint sub = step >> 5; + const uint sb_idx = step / QK_K; + + const float dd = (float)src0_d[rrow + sb_idx * m]; + const char2 sc = as_char2(src0_s[rrow + sub * m]); + const float scale0 = dd * (float)sc.s0; + const float scale1 = dd * (float)sc.s1; + + const uint wbase = rrow + (step >> 2) * (uint)m; + uint qw[8]; + #pragma unroll + for (int u = 0; u < 8; ++u) { + const uint o = wbase + (uint)u * (uint)m; + qw[u] = SIGN6(EXP4((uint)src0_ql[o]) | EXP2((uint)src0_qh[o])); + } + + for (uint idx = lid; idx < TILESIZE_N * 2; idx += 64) { + const uint t = idx >> 1; + const uint v = idx & 1; + const uint c = col_base + t; + sh_qa4[t][v] = (c < (uint)n_no_padding) + ? vload4(0, src1_qa + c * k_u + (step >> 2) + (v << 2)) + : (uint4)(0u); + } + if (lid < TILESIZE_N) { + const uint c = col_base + lid; + sh_d[lid] = (c < (uint)n_no_padding) ? src1_da[c * k_b + sub] : (half)0; + } + barrier(CLK_LOCAL_MEM_FENCE); + + #pragma unroll + for (int g = 0; g < NGROUPS; ++g) { + const int b = g * 4; + float4 rf; + #define DOT_TOK(j) { \ + const uint4 a0 = sh_qa4[b + (j)][0]; \ + const uint4 a1 = sh_qa4[b + (j)][1]; \ + const int raw1 = dot4_q8a(qw[0], qw[1], qw[2], qw[3], a0.x, a0.y, a0.z, a0.w); \ + const int raw2 = dot4_q8a(qw[4], qw[5], qw[6], qw[7], a1.x, a1.y, a1.z, a1.w); \ + rf.s##j = scale0 * (float)raw1 + scale1 * (float)raw2; \ + } + DOT_TOK(0); DOT_TOK(1); DOT_TOK(2); DOT_TOK(3); + #undef DOT_TOK + const float4 ad = (float4)((float)sh_d[b+0], (float)sh_d[b+1], (float)sh_d[b+2], (float)sh_d[b+3]); + acc[g] += ad * rf; + } + barrier(CLK_LOCAL_MEM_FENCE); + } + + if (!row_valid) { + return; + } + + #pragma unroll + for (int g = 0; g < NGROUPS; ++g) { + const uint b = (uint)(g * 4); + const float4 a = acc[g]; + const uint c0 = col_base + b; + if (c0 + 0 < (uint)n_no_padding) { + dst[(c0 + 0) * (uint)m + row] = a.s0; + } + if (c0 + 1 < (uint)n_no_padding) { + dst[(c0 + 1) * (uint)m + row] = a.s1; + } + if (c0 + 2 < (uint)n_no_padding) { + dst[(c0 + 2) * (uint)m + row] = a.s2; + } + if (c0 + 3 < (uint)n_no_padding) { + dst[(c0 + 3) * (uint)m + row] = a.s3; + } + } +#undef NGROUPS +} diff --git a/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q8_0_q8_1_dp4a.cl b/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q8_0_q8_1_dp4a.cl index a481636c2..32ff7b4f2 100644 --- a/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q8_0_q8_1_dp4a.cl +++ b/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q8_0_q8_1_dp4a.cl @@ -210,3 +210,125 @@ kernel void kernel_gemm_noshuffle_q8_0_q8_1_dp4a_wimg( } #undef NGROUPS } + +// Stage each 32-K activation block as two uint4 values. +inline int dot8_q8a_v(uint8 qw, uint4 a0, uint4 a1) { + int r = 0; + r = dot_acc_sat_4x8packed_ss_int(qw.s0, a0.x, r); + r = dot_acc_sat_4x8packed_ss_int(qw.s1, a0.y, r); + r = dot_acc_sat_4x8packed_ss_int(qw.s2, a0.z, r); + r = dot_acc_sat_4x8packed_ss_int(qw.s3, a0.w, r); + r = dot_acc_sat_4x8packed_ss_int(qw.s4, a1.x, r); + r = dot_acc_sat_4x8packed_ss_int(qw.s5, a1.y, r); + r = dot_acc_sat_4x8packed_ss_int(qw.s6, a1.z, r); + r = dot_acc_sat_4x8packed_ss_int(qw.s7, a1.w, r); + return r; +} + +__attribute__((qcom_wave_pair_mode(1))) +kernel void kernel_gemm_noshuffle_q8_0_q8_1_dp4a_act_vload4( + __global const uint * src0_q, + __global const half * src0_d, + __global const uint * src1_qa, + __global const half * src1_da, + __global float * dst, + ulong offsetd, + int m, + int n_no_padding, + int k +) { + dst = (global float *)((global char *)dst + offsetd); + + const uint lid = get_local_id(0); + const uint block_id_m = get_global_id(1); + const uint block_id_n = get_global_id(2); + + const uint row = block_id_m * 64 + lid; + const uint col_base = block_id_n * TILESIZE_N; + const bool row_valid = row < (uint)m; + const uint rrow = row_valid ? row : 0; + + const uint k_u = (uint)k >> 2; + const uint k_b = (uint)k >> 5; + + __local uint4 sh_qa4[TILESIZE_N][2]; + __local half sh_d[TILESIZE_N]; + +#define NGROUPS (TILESIZE_N / 4) + float4 acc[NGROUPS]; + #pragma unroll + for (int g = 0; g < NGROUPS; ++g) { + acc[g] = (float4)(0.0f); + } + + for (uint step = 0; step < (uint)k; step += 32) { + const uint sub = step >> 5; + + const float d_w = (float)src0_d[rrow + sub * (uint)m]; + + const uint wbase = rrow + (step >> 2) * (uint)m; + uint8 qw; + qw.s0 = src0_q[wbase + 0 * m]; + qw.s1 = src0_q[wbase + 1 * m]; + qw.s2 = src0_q[wbase + 2 * m]; + qw.s3 = src0_q[wbase + 3 * m]; + qw.s4 = src0_q[wbase + 4 * m]; + qw.s5 = src0_q[wbase + 5 * m]; + qw.s6 = src0_q[wbase + 6 * m]; + qw.s7 = src0_q[wbase + 7 * m]; + + for (uint idx = lid; idx < TILESIZE_N * 2; idx += 64) { + const uint t = idx >> 1; + const uint v = idx & 1; + const uint c = col_base + t; + sh_qa4[t][v] = (c < (uint)n_no_padding) + ? vload4(0, src1_qa + c * k_u + (step >> 2) + (v << 2)) + : (uint4)(0u); + } + if (lid < TILESIZE_N) { + const uint c = col_base + lid; + sh_d[lid] = (c < (uint)n_no_padding) ? src1_da[c * k_b + sub] : (half)0; + } + barrier(CLK_LOCAL_MEM_FENCE); + +#define LD4(arr, b) ((float4)((float)arr[(b)+0], (float)arr[(b)+1], (float)arr[(b)+2], (float)arr[(b)+3])) + #pragma unroll + for (int g = 0; g < NGROUPS; ++g) { + const int b = g * 4; + float4 rf; +#define DOTV(T) dot8_q8a_v(qw, sh_qa4[T][0], sh_qa4[T][1]) + rf.s0 = (float)DOTV(b+0); + rf.s1 = (float)DOTV(b+1); + rf.s2 = (float)DOTV(b+2); + rf.s3 = (float)DOTV(b+3); +#undef DOTV + acc[g] += d_w * LD4(sh_d, b) * rf; + } +#undef LD4 + barrier(CLK_LOCAL_MEM_FENCE); + } + + if (!row_valid) { + return; + } + + #pragma unroll + for (int g = 0; g < NGROUPS; ++g) { + const uint b = (uint)(g * 4); + const float4 a = acc[g]; + const uint c0 = col_base + b; + if (c0 + 0 < (uint)n_no_padding) { + dst[(c0 + 0) * (uint)m + row] = a.s0; + } + if (c0 + 1 < (uint)n_no_padding) { + dst[(c0 + 1) * (uint)m + row] = a.s1; + } + if (c0 + 2 < (uint)n_no_padding) { + dst[(c0 + 2) * (uint)m + row] = a.s2; + } + if (c0 + 3 < (uint)n_no_padding) { + dst[(c0 + 3) * (uint)m + row] = a.s3; + } + } +#undef NGROUPS +}