opencl: fold GQA head for f16 FA for better efficiency (#32631)

Assisted-by: Codex

Co-authored-by: Li He <lih@qti.qualcomm.com>
Co-authored-by: Hongqiang Wang <wangh@qti.qualcomm.com>
This commit is contained in:
authored and GitHub committed 2026-11-19 22:26:28 -08:00
1 parent 8bf88ba936
commit 7e960f40fb
2 files changed
+31 -12

No files matched your search

+11 -2
View File
@@ -23128,8 +23128,17 @@ static void ggml_cl_fa_kq_p8_dispatch(
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(int), &D_B));
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(int), &p_pitch));
const size_t n_tiles = (N + 31) / 32;
size_t gws[3] = {64, n_tiles, (size_t)(M / 64) * D_B};
// Pack query heads of each KV group into a short batch's tile.
static const bool fold_enabled = [] {
const char * e = getenv("GGML_OPENCL_FA_KQ_P8_FOLD");
return !e || e[0] != '0';
}();
const int gqa = D_B / D_A;
const int fold = fold_enabled && gqa > 1 && N < 32 && gqa <= INT_MAX / N ? gqa : 1;
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(int), &fold));
const size_t n_tiles = ((size_t) N * fold + 31) / 32;
size_t gws[3] = {64, n_tiles, (size_t)(M / 64) * (fold > 1 ? D_A : D_B)};
const bool two_tiles = n_tiles % 2 == 0 && backend_ctx->get_kernel_workgroup_size(kernel) >= 128;
size_t lws[3] = {64, two_tiles ? (size_t) 2 : (size_t) 1, 1};
@@ -119,7 +119,8 @@ __kernel void mul_mm_f16_f32_kq_p8(
int N,
int D_A,
int D_B,
int p_pitch // bytes per P row, >= M, multiple of 32
int p_pitch, // bytes per P row, >= M, multiple of 32
int gqa_fold
) {
// Match the n-tile-first grid in ggml_cl_fa_kq_p8_dispatch.
@@ -135,7 +136,8 @@ __kernel void mul_mm_f16_f32_kq_p8(
const uint col = block_id_m * TILESIZE_M;
const uint row = block_id_n * TILESIZE_N;
const uint depth_A = block_id_d / (D_B/D_A);
const uint fold = (uint)gqa_fold;
const uint depth_A = fold > 1 ? block_id_d : block_id_d / (D_B/D_A);
const uint depth_B = block_id_d;
int line_stride_matrix_A_in_bytes = K * D_A * 2;
@@ -158,11 +160,18 @@ __kernel void mul_mm_f16_f32_kq_p8(
__local float matrix_B_local[1024];
// Clamp Q loads in the last tile; columns past N are not stored.
const uint c0 = min(row + b_globalOffsetInWords_xy.y, (uint)N - 1);
const uint c1 = min(row + b_globalOffsetInWords_xy.y + 16, (uint)N - 1);
const uint b_globalOffsetInWords00 = c0*strideBinElements + depth_B*K + b_globalOffsetInWords_xy.x;
const uint b_globalOffsetInWords16 = c1*strideBinElements + depth_B*K + b_globalOffsetInWords_xy.x;
// Folded columns enumerate (query, head within the KV group).
const uint c0 = min(row + b_globalOffsetInWords_xy.y, (uint)N * fold - 1);
const uint c1 = min(row + b_globalOffsetInWords_xy.y + 16, (uint)N * fold - 1);
uint b_globalOffsetInWords00;
uint b_globalOffsetInWords16;
if (fold > 1) {
b_globalOffsetInWords00 = (c0/fold)*strideBinElements + (depth_B*fold + c0%fold)*K + b_globalOffsetInWords_xy.x;
b_globalOffsetInWords16 = (c1/fold)*strideBinElements + (depth_B*fold + c1%fold)*K + b_globalOffsetInWords_xy.x;
} else {
b_globalOffsetInWords00 = c0*strideBinElements + depth_B*K + b_globalOffsetInWords_xy.x;
b_globalOffsetInWords16 = c1*strideBinElements + depth_B*K + b_globalOffsetInWords_xy.x;
}
for (uint step=0; step < K; step+=TILESIZE_K) {
regA = mm_load_a(matrix_A, subMatrixAStartInElements, line_stride_matrix_A_in_bytes);
@@ -199,9 +208,10 @@ __kernel void mul_mm_f16_f32_kq_p8(
if (lane < 32) {
const int j = lane >> 1;
const int b = lane & 1;
const uint n = row + half_*16 + j;
const uint h = depth_B;
if (n < (uint)N) {
const uint c = row + half_*16 + j;
const uint n = c / fold;
const uint h = depth_B*fold + c % fold;
if (c < (uint)N * fold) {
float slope = 1.0f;
if (max_bias > 0.0f) {
const float base = (int)h < n_head_log2 ? m0 : m1;