opencl: improve decode perf with FA (#31633)

* opencl: use more workgroups for q4_K gate up + GLU

Co-authored-by: Shaofei Qi <shaoqi@qti.qualcomm.com>

* opencl: handle FA for GQA 2 and 6 with head size 256

Co-authored-by: Shaofei Qi <shaoqi@qti.qualcomm.com>

* opencl: handle FA for GQA 16 with head size of 128, 512

Co-authored-by: Shaofei Qi <shaoqi@qti.qualcomm.com>

* opencl: handle FA for GQA 2 with head size 512

Co-authored-by: Shaofei Qi <shaoqi@qti.qualcomm.com>

---------

Co-authored-by: Li He <lih@qti.qualcomm.com>
This commit is contained in:
shaofeiqiandLi He authored and GitHub committed 2026-11-04 10:45:30 -08:00
1 parent c6a0ff11a0
commit 9e2a04a439
1 file changed
+18 -5
+18 -5
View File
@@ -7835,7 +7835,8 @@ static void ggml_opencl_ensure_fa_f32_f16_mq_decode(ggml_backend_opencl_context
}
const bool cluster = dk == 64 || dk == 128;
const int head_sub = cluster ? 2 : (gqa == 8 ? (dk == 512 ? 4 : 2) : 1);
const int head_sub = gqa == 16 ? (cluster ? 4 : 8) :
cluster ? 2 : (gqa == 8 ? (dk == 512 ? 4 : 2) : 1);
const int nsg_max = dk == 64 ? 1 : (dk == 128 || (gqa == 8 && dk == 256) ? 2 : 4);
const char * kernel_name = cluster ? "flash_attn_f32_f16_q1_vec_mq_split_c8" : "flash_attn_f32_f16_q1_vec_mq_split";
const std::string src = ggml_opencl_fa_kernel_src(FA_VARIANT_F32_F16);
@@ -7843,7 +7844,7 @@ static void ggml_opencl_ensure_fa_f32_f16_mq_decode(ggml_backend_opencl_context
" -D FA_MQ_ONLY -D MQ_GQA=" + std::to_string(gqa / head_sub) +
" -D FA_HEAD_SUB=" + std::to_string(head_sub) +
(cluster ? " -D MQ_NSG=" + std::to_string(nsg_max) + " -D FA_CL_C=16" : " -D FA_MQ_SPLIT_ONLY") +
(dk == 64 ? " -D FA_CL_MHRED -D FA_CL_MASK_BCAST" : "") +
(dk == 64 ? " -D FA_CL_MHRED -D FA_CL_MASK_BCAST" : (cluster && gqa == 16 ? " -D FA_CL_MHRED" : "")) +
(gqa == 8 && dk == 256 ? " -D FA_Q1_Q_REG" : "");
for (int nsg = nsg_max; nsg >= 1; nsg /= 2) {
const size_t wg = 64 * nsg;
@@ -10839,7 +10840,8 @@ static void ggml_cl_mul_mat_q4_k_glu_fused(ggml_backend_t backend, ggml_tensor *
// byte-identical to the per-op path. Keep the maxwg query as a further floor
// for any driver that reports < 512.
size_t maxwg = backend_ctx->get_kernel_workgroup_size(kernel);
size_t nsg_y = 8;
// X2E: 4 is faster than 8 on every q4_K FFN shape measured there
size_t nsg_y = backend_ctx->adreno_gen == ADRENO_GPU_GEN::X2E ? 4 : 8;
while (nsg_y > 1 && 64 * nsg_y > maxwg) { nsg_y >>= 1; }
size_t local_work_size[3] = { 64, nsg_y, 1 };
size_t global_work_size[3] = { (size_t)CEIL_DIV(M / 2, 64) * 64, nsg_y, 1 };
@@ -12148,9 +12150,13 @@ static bool ggml_opencl_supports_op(ggml_backend_dev_t dev, const struct ggml_te
}
if (q->ne[1] == 1) {
const char * decode_env = getenv("GGML_OPENCL_FA_DK512_DECODE");
const int64_t gqa = k->ne[2] > 0 ? q->ne[2] / k->ne[2] : 0;
// GQA 2 and 16 decode on the X2E MQ kernels (see mq_decode_x2e in ggml_cl_flash_attn)
const bool gqa_ok = gqa == 4 || gqa == 8 ||
(backend_ctx->adreno_gen == ADRENO_GPU_GEN::X2E && (gqa == 2 || gqa == 16));
if ((decode_env && decode_env[0] == '0') ||
backend_ctx->gpu_family != ADRENO || k->ne[2] <= 0 ||
(q->ne[2] / k->ne[2] != 4 && q->ne[2] / k->ne[2] != 8) || q->ne[2] % k->ne[2] != 0 ||
!gqa_ok || q->ne[2] % k->ne[2] != 0 ||
!ggml_opencl_ensure_fa_f32_f16_vec_512(backend_ctx)) {
return false;
}
@@ -22596,11 +22602,18 @@ static void ggml_cl_flash_attn(ggml_backend_t backend, const ggml_tensor * q, co
const std::pair<int, int> dk_dv = {d_head_q, d_head_v};
const int mq_gqa = n_head_kv > 0 ? n_head / n_head_kv : 0;
const std::tuple<int, int, int> mq_decode_key = {d_head_q, d_head_v, mq_gqa};
// X2E also takes GQA 2 and 6 at head size 256 (gemma-4 SWA layers, Qwen3.5-27B),
// GQA 16 at 512 and 128 (gemma-4-12b global layers, muse-glimmer) and GQA 2 at 512 (gemma-4-E4B MTP drafter)
const bool mq_decode_x2e = backend_ctx->adreno_gen == ADRENO_GPU_GEN::X2E &&
((d_head_q == 256 && (mq_gqa == 2 || mq_gqa == 6)) ||
(d_head_q == 512 && (mq_gqa == 2 || mq_gqa == 16)) ||
(d_head_q == 128 && mq_gqa == 16 && backend_ctx->has_subgroup_shuffle));
const bool mq_decode_shape = backend_ctx->gpu_family == ADRENO && is_mixed && n_q == 1 &&
d_head_q == d_head_v && n_head_kv > 0 && n_head % n_head_kv == 0 &&
((((d_head_q == 64 && mq_gqa == 8) || (d_head_q == 128 && mq_gqa == 4)) &&
backend_ctx->has_subgroup_shuffle) ||
((d_head_q == 256 || d_head_q == 512) && (mq_gqa == 4 || mq_gqa == 8)));
((d_head_q == 256 || d_head_q == 512) && (mq_gqa == 4 || mq_gqa == 8)) ||
mq_decode_x2e);
if (mq_decode_shape && n_kv >= 32) {
ggml_opencl_ensure_fa_f32_f16_mq_decode(backend_ctx, d_head_q, d_head_v, mq_gqa);
}