From 7e960f40fb0bd2eede76406f807d6bc3f7aa339c Mon Sep 17 00:00:00 2001 From: shaofeiqi Date: Thu, 19 Nov 2026 22:26:28 -0800 Subject: [PATCH] opencl: fold GQA head for f16 FA for better efficiency (#32631) Assisted-by: Codex Co-authored-by: Li He Co-authored-by: Hongqiang Wang --- ggml/src/ggml-opencl/ggml-opencl.cpp | 13 ++++++-- .../kernels/mul_mm_f16_f32_kq_p8.cl | 30 ++++++++++++------- 2 files changed, 31 insertions(+), 12 deletions(-) diff --git a/ggml/src/ggml-opencl/ggml-opencl.cpp b/ggml/src/ggml-opencl/ggml-opencl.cpp index 776a8e757..894f8235e 100644 --- a/ggml/src/ggml-opencl/ggml-opencl.cpp +++ b/ggml/src/ggml-opencl/ggml-opencl.cpp @@ -23128,8 +23128,17 @@ static void ggml_cl_fa_kq_p8_dispatch( CL_CHECK(clSetKernelArg(kernel, a++, sizeof(int), &D_B)); CL_CHECK(clSetKernelArg(kernel, a++, sizeof(int), &p_pitch)); - const size_t n_tiles = (N + 31) / 32; - size_t gws[3] = {64, n_tiles, (size_t)(M / 64) * D_B}; + // Pack query heads of each KV group into a short batch's tile. + static const bool fold_enabled = [] { + const char * e = getenv("GGML_OPENCL_FA_KQ_P8_FOLD"); + return !e || e[0] != '0'; + }(); + const int gqa = D_B / D_A; + const int fold = fold_enabled && gqa > 1 && N < 32 && gqa <= INT_MAX / N ? gqa : 1; + CL_CHECK(clSetKernelArg(kernel, a++, sizeof(int), &fold)); + + const size_t n_tiles = ((size_t) N * fold + 31) / 32; + size_t gws[3] = {64, n_tiles, (size_t)(M / 64) * (fold > 1 ? D_A : D_B)}; const bool two_tiles = n_tiles % 2 == 0 && backend_ctx->get_kernel_workgroup_size(kernel) >= 128; size_t lws[3] = {64, two_tiles ? (size_t) 2 : (size_t) 1, 1}; diff --git a/ggml/src/ggml-opencl/kernels/mul_mm_f16_f32_kq_p8.cl b/ggml/src/ggml-opencl/kernels/mul_mm_f16_f32_kq_p8.cl index 2c0a56c85..9c4f71a6d 100644 --- a/ggml/src/ggml-opencl/kernels/mul_mm_f16_f32_kq_p8.cl +++ b/ggml/src/ggml-opencl/kernels/mul_mm_f16_f32_kq_p8.cl @@ -119,7 +119,8 @@ __kernel void mul_mm_f16_f32_kq_p8( int N, int D_A, int D_B, - int p_pitch // bytes per P row, >= M, multiple of 32 + int p_pitch, // bytes per P row, >= M, multiple of 32 + int gqa_fold ) { // Match the n-tile-first grid in ggml_cl_fa_kq_p8_dispatch. @@ -135,7 +136,8 @@ __kernel void mul_mm_f16_f32_kq_p8( const uint col = block_id_m * TILESIZE_M; const uint row = block_id_n * TILESIZE_N; - const uint depth_A = block_id_d / (D_B/D_A); + const uint fold = (uint)gqa_fold; + const uint depth_A = fold > 1 ? block_id_d : block_id_d / (D_B/D_A); const uint depth_B = block_id_d; int line_stride_matrix_A_in_bytes = K * D_A * 2; @@ -158,11 +160,18 @@ __kernel void mul_mm_f16_f32_kq_p8( __local float matrix_B_local[1024]; - // Clamp Q loads in the last tile; columns past N are not stored. - const uint c0 = min(row + b_globalOffsetInWords_xy.y, (uint)N - 1); - const uint c1 = min(row + b_globalOffsetInWords_xy.y + 16, (uint)N - 1); - const uint b_globalOffsetInWords00 = c0*strideBinElements + depth_B*K + b_globalOffsetInWords_xy.x; - const uint b_globalOffsetInWords16 = c1*strideBinElements + depth_B*K + b_globalOffsetInWords_xy.x; + // Folded columns enumerate (query, head within the KV group). + const uint c0 = min(row + b_globalOffsetInWords_xy.y, (uint)N * fold - 1); + const uint c1 = min(row + b_globalOffsetInWords_xy.y + 16, (uint)N * fold - 1); + uint b_globalOffsetInWords00; + uint b_globalOffsetInWords16; + if (fold > 1) { + b_globalOffsetInWords00 = (c0/fold)*strideBinElements + (depth_B*fold + c0%fold)*K + b_globalOffsetInWords_xy.x; + b_globalOffsetInWords16 = (c1/fold)*strideBinElements + (depth_B*fold + c1%fold)*K + b_globalOffsetInWords_xy.x; + } else { + b_globalOffsetInWords00 = c0*strideBinElements + depth_B*K + b_globalOffsetInWords_xy.x; + b_globalOffsetInWords16 = c1*strideBinElements + depth_B*K + b_globalOffsetInWords_xy.x; + } for (uint step=0; step < K; step+=TILESIZE_K) { regA = mm_load_a(matrix_A, subMatrixAStartInElements, line_stride_matrix_A_in_bytes); @@ -199,9 +208,10 @@ __kernel void mul_mm_f16_f32_kq_p8( if (lane < 32) { const int j = lane >> 1; const int b = lane & 1; - const uint n = row + half_*16 + j; - const uint h = depth_B; - if (n < (uint)N) { + const uint c = row + half_*16 + j; + const uint n = c / fold; + const uint h = depth_B*fold + c % fold; + if (c < (uint)N * fold) { float slope = 1.0f; if (max_bias > 0.0f) { const float base = (int)h < n_head_log2 ? m0 : m1;