From ff2a96d2d4a8cb134c690e470ea43b1840163eee Mon Sep 17 00:00:00 2001 From: Lumpiasty Date: Mon, 27 Jul 2026 23:56:14 +0200 Subject: [PATCH] vulkan: record why flash-attn shmem staging stays off for GCN Staging K/V through shared memory looks like an obvious win on GCN: without it each rowgroup re-reads the whole K/V block from global memory, and with row_split 4 that pulls one 16KB block through a 16KB vector L1 four times. Measured, it loses: -6.7% pp2048 at depth 16k and -7.4% at 32k on Polaris at head size 128. The kvsh stride of D/4+1 dwords is 4 mod 32, which costs an 8-way LDS bank conflict on wave64 - the +1 padding is tuned for warp32 - and the extra shared memory eats occupancy this shader is already short of. Comment only, no behaviour change. Leaving a note so the next person does not spend a GPU on it. Co-Authored-By: Claude Opus 5 --- ggml/src/ggml-vulkan/ggml-vulkan.cpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vulkan.cpp index b5644f861..555c6f2a0 100644 --- a/ggml/src/ggml-vulkan/ggml-vulkan.cpp +++ b/ggml/src/ggml-vulkan/ggml-vulkan.cpp @@ -1247,6 +1247,10 @@ static vk_fa_tuning_params get_fa_tuning_params_scalar(const vk_device& device, result.d_split = std::min(std::min(result.subgroup_size, 8u), D_lsb / 4); + // Staging K/V through shared memory is a loss on AMD GCN: measured -6.7% at depth 16k and + // -7.4% at 32k on Polaris (head size 128). The kvsh stride of D/4+1 dwords is 4 mod 32, which + // costs an 8-way LDS bank conflict on wave64, and the extra shared memory cuts into occupancy + // that this shader is already short of. Keep it to NVIDIA. result.shmem_staging = (device->vendor_id == VK_VENDOR_ID_NVIDIA && hsk < 256 && hsv < 256) ? 1 : 0; if (!reduce_block_rows && !ggml_vk_flash_attn_scalar_shmem_support(device, result, hsk, hsv, f32acc, k_type, v_type)) {