From 4442815c02bbcb7be6d98e8a73c227eafcd1d651 Mon Sep 17 00:00:00 2001 From: Lumpiasty Date: Wed, 22 Jul 2026 20:40:34 +0200 Subject: [PATCH] ggml-vulkan: enable flash-attn mask_opt for GCN large head sizes mask_opt was disabled on AMD GCN for head sizes <= 256, but it is beneficial there in high-context prefill: on fully-visible causal blocks it skips the per-block mask load+add, and it skips fully-masked blocks entirely. The attention op on GCN is compute-bound on the softmax path (no matrix cores), so this cuts real work. Verified lossless (perplexity bit-identical with it on vs off) and a measured prefill win on Qwen3.5-35B-A3B (head_dim 256) on an RX 580: pp2048 unchanged at short context, +8% @ 16k, +12% @ 32k, growing with depth. Enable for GCN when HSK/HSV >= 256; the existing large-mask conditions keep it off for decode. Assisted-by: Claude --- ggml/src/ggml-vulkan/ggml-vulkan.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vulkan.cpp index 9aa7d9883..853773f13 100644 --- a/ggml/src/ggml-vulkan/ggml-vulkan.cpp +++ b/ggml/src/ggml-vulkan/ggml-vulkan.cpp @@ -10592,8 +10592,10 @@ static void ggml_vk_flash_attn(ggml_backend_vk_context * ctx, vk_context& subctx } // Only use mask opt when the mask is fairly large. This hasn't been tuned extensively. + // GCN with large head size (>= 256) benefits in high-context prefill (skipping the + // per-block mask add on fully-visible blocks dominates), so enable it there too. bool use_mask_opt = mask && nem1 >= 32 && nem0 * nem1 > 32768 && nem0 >= tuning_params.block_cols * 16 - && (ctx->device->architecture != vk_device_architecture::AMD_GCN || HSK > 256 || HSV > 256); + && (ctx->device->architecture != vk_device_architecture::AMD_GCN || HSK >= 256 || HSV >= 256); vk_fa_pipeline_state fa_pipeline_state = get_fa_pipeline_state(ctx->device, tuning_params, HSK, HSV, aligned, f32acc, mask != nullptr, use_mask_opt, logit_softcap != 0, k->type, v->type);