From 9e2a04a439edc49f08c5acdafefae19dfa25a658 Mon Sep 17 00:00:00 2001 From: shaofeiqi Date: Wed, 4 Nov 2026 10:45:30 -0800 Subject: [PATCH] opencl: improve decode perf with FA (#31633) * opencl: use more workgroups for q4_K gate up + GLU Co-authored-by: Shaofei Qi * opencl: handle FA for GQA 2 and 6 with head size 256 Co-authored-by: Shaofei Qi * opencl: handle FA for GQA 16 with head size of 128, 512 Co-authored-by: Shaofei Qi * opencl: handle FA for GQA 2 with head size 512 Co-authored-by: Shaofei Qi --------- Co-authored-by: Li He --- ggml/src/ggml-opencl/ggml-opencl.cpp | 23 ++++++++++++++++++----- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/ggml/src/ggml-opencl/ggml-opencl.cpp b/ggml/src/ggml-opencl/ggml-opencl.cpp index 22cf6f67d..a2620135d 100644 --- a/ggml/src/ggml-opencl/ggml-opencl.cpp +++ b/ggml/src/ggml-opencl/ggml-opencl.cpp @@ -7835,7 +7835,8 @@ static void ggml_opencl_ensure_fa_f32_f16_mq_decode(ggml_backend_opencl_context } const bool cluster = dk == 64 || dk == 128; - const int head_sub = cluster ? 2 : (gqa == 8 ? (dk == 512 ? 4 : 2) : 1); + const int head_sub = gqa == 16 ? (cluster ? 4 : 8) : + cluster ? 2 : (gqa == 8 ? (dk == 512 ? 4 : 2) : 1); const int nsg_max = dk == 64 ? 1 : (dk == 128 || (gqa == 8 && dk == 256) ? 2 : 4); const char * kernel_name = cluster ? "flash_attn_f32_f16_q1_vec_mq_split_c8" : "flash_attn_f32_f16_q1_vec_mq_split"; const std::string src = ggml_opencl_fa_kernel_src(FA_VARIANT_F32_F16); @@ -7843,7 +7844,7 @@ static void ggml_opencl_ensure_fa_f32_f16_mq_decode(ggml_backend_opencl_context " -D FA_MQ_ONLY -D MQ_GQA=" + std::to_string(gqa / head_sub) + " -D FA_HEAD_SUB=" + std::to_string(head_sub) + (cluster ? " -D MQ_NSG=" + std::to_string(nsg_max) + " -D FA_CL_C=16" : " -D FA_MQ_SPLIT_ONLY") + - (dk == 64 ? " -D FA_CL_MHRED -D FA_CL_MASK_BCAST" : "") + + (dk == 64 ? " -D FA_CL_MHRED -D FA_CL_MASK_BCAST" : (cluster && gqa == 16 ? " -D FA_CL_MHRED" : "")) + (gqa == 8 && dk == 256 ? " -D FA_Q1_Q_REG" : ""); for (int nsg = nsg_max; nsg >= 1; nsg /= 2) { const size_t wg = 64 * nsg; @@ -10839,7 +10840,8 @@ static void ggml_cl_mul_mat_q4_k_glu_fused(ggml_backend_t backend, ggml_tensor * // byte-identical to the per-op path. Keep the maxwg query as a further floor // for any driver that reports < 512. size_t maxwg = backend_ctx->get_kernel_workgroup_size(kernel); - size_t nsg_y = 8; + // X2E: 4 is faster than 8 on every q4_K FFN shape measured there + size_t nsg_y = backend_ctx->adreno_gen == ADRENO_GPU_GEN::X2E ? 4 : 8; while (nsg_y > 1 && 64 * nsg_y > maxwg) { nsg_y >>= 1; } size_t local_work_size[3] = { 64, nsg_y, 1 }; size_t global_work_size[3] = { (size_t)CEIL_DIV(M / 2, 64) * 64, nsg_y, 1 }; @@ -12148,9 +12150,13 @@ static bool ggml_opencl_supports_op(ggml_backend_dev_t dev, const struct ggml_te } if (q->ne[1] == 1) { const char * decode_env = getenv("GGML_OPENCL_FA_DK512_DECODE"); + const int64_t gqa = k->ne[2] > 0 ? q->ne[2] / k->ne[2] : 0; + // GQA 2 and 16 decode on the X2E MQ kernels (see mq_decode_x2e in ggml_cl_flash_attn) + const bool gqa_ok = gqa == 4 || gqa == 8 || + (backend_ctx->adreno_gen == ADRENO_GPU_GEN::X2E && (gqa == 2 || gqa == 16)); if ((decode_env && decode_env[0] == '0') || backend_ctx->gpu_family != ADRENO || k->ne[2] <= 0 || - (q->ne[2] / k->ne[2] != 4 && q->ne[2] / k->ne[2] != 8) || q->ne[2] % k->ne[2] != 0 || + !gqa_ok || q->ne[2] % k->ne[2] != 0 || !ggml_opencl_ensure_fa_f32_f16_vec_512(backend_ctx)) { return false; } @@ -22596,11 +22602,18 @@ static void ggml_cl_flash_attn(ggml_backend_t backend, const ggml_tensor * q, co const std::pair dk_dv = {d_head_q, d_head_v}; const int mq_gqa = n_head_kv > 0 ? n_head / n_head_kv : 0; const std::tuple mq_decode_key = {d_head_q, d_head_v, mq_gqa}; + // X2E also takes GQA 2 and 6 at head size 256 (gemma-4 SWA layers, Qwen3.5-27B), + // GQA 16 at 512 and 128 (gemma-4-12b global layers, muse-glimmer) and GQA 2 at 512 (gemma-4-E4B MTP drafter) + const bool mq_decode_x2e = backend_ctx->adreno_gen == ADRENO_GPU_GEN::X2E && + ((d_head_q == 256 && (mq_gqa == 2 || mq_gqa == 6)) || + (d_head_q == 512 && (mq_gqa == 2 || mq_gqa == 16)) || + (d_head_q == 128 && mq_gqa == 16 && backend_ctx->has_subgroup_shuffle)); const bool mq_decode_shape = backend_ctx->gpu_family == ADRENO && is_mixed && n_q == 1 && d_head_q == d_head_v && n_head_kv > 0 && n_head % n_head_kv == 0 && ((((d_head_q == 64 && mq_gqa == 8) || (d_head_q == 128 && mq_gqa == 4)) && backend_ctx->has_subgroup_shuffle) || - ((d_head_q == 256 || d_head_q == 512) && (mq_gqa == 4 || mq_gqa == 8))); + ((d_head_q == 256 || d_head_q == 512) && (mq_gqa == 4 || mq_gqa == 8)) || + mq_decode_x2e); if (mq_decode_shape && n_kv >= 32) { ggml_opencl_ensure_fa_f32_f16_mq_decode(backend_ctx, d_head_q, d_head_v, mq_gqa); }