From 6d483cc3c1f2ce6ecfe783974ce96b1653517fc8 Mon Sep 17 00:00:00 2001 From: Lumpiasty Date: Mon, 27 Jul 2026 23:56:14 +0200 Subject: [PATCH] vulkan: record why flash-attn shmem staging stays off for GCN Staging K/V through shared memory looks like an obvious win on GCN: without it each rowgroup re-reads the whole K/V block from global memory, and with row_split 4 that pulls one 16KB block through a 16KB vector L1 four times. Measured, it loses: -6.7% pp2048 at depth 16k and -7.4% at 32k on Polaris at head size 128. The kvsh stride of D/4+1 dwords is 4 mod 32, which costs an 8-way LDS bank conflict on wave64 - the +1 padding is tuned for warp32 - and the extra shared memory eats occupancy this shader is already short of. Comment only, no behaviour change. Leaving a note so the next person does not spend a GPU on it. Co-Authored-By: Claude Opus 5 --- ggml/src/ggml-vulkan/ggml-vulkan.cpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vulkan.cpp index 38d316661..c954d071d 100644 --- a/ggml/src/ggml-vulkan/ggml-vulkan.cpp +++ b/ggml/src/ggml-vulkan/ggml-vulkan.cpp @@ -3977,6 +3977,10 @@ static vk_fa_tuning_params get_fa_tuning_params_scalar(const vk_device& device, result.d_split = std::min(std::min(result.subgroup_size, 8u), D_lsb / 4); + // Staging K/V through shared memory is a loss on AMD GCN: measured -6.7% at depth 16k and + // -7.4% at 32k on Polaris (head size 128). The kvsh stride of D/4+1 dwords is 4 mod 32, which + // costs an 8-way LDS bank conflict on wave64, and the extra shared memory cuts into occupancy + // that this shader is already short of. Keep it to NVIDIA. result.shmem_staging = (device->vendor_id == VK_VENDOR_ID_NVIDIA && hsk < 256 && hsv < 256) ? 1 : 0; if (!reduce_block_rows && !ggml_vk_flash_attn_scalar_shmem_support(device, result, hsk, hsv, f32acc, k_type, v_type)) {