From 07ec7c8af09b576792063411d7dc2ec60b2f3a6f Mon Sep 17 00:00:00 2001 From: ravel7524 <58877666+ravel7524@users.noreply.github.com> Date: Mon, 2 Nov 2026 12:54:09 +0100 Subject: [PATCH] vulkan: improve small-n split-K selection on RDNA3 and RDNA4 (#30146) * vulkan: improve small-n split-K selection on RDNA3 and RDNA4 * Clean up test cases by removing commented code Removed tests as requested --- ggml/src/ggml-vulkan/ggml-vulkan.cpp | 42 +++++++++++++++++----------- 1 file changed, 26 insertions(+), 16 deletions(-) diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vulkan.cpp index c0943689f..52d55ac8b 100644 --- a/ggml/src/ggml-vulkan/ggml-vulkan.cpp +++ b/ggml/src/ggml-vulkan/ggml-vulkan.cpp @@ -5920,8 +5920,18 @@ static uint32_t ggml_vk_guess_split_k(ggml_backend_vk_context * ctx, uint32_t m, return 1; } + const bool rdna = ctx->device->architecture == AMD_RDNA3 || ctx->device->architecture == AMD_RDNA4; uint32_t split_k = 1; - if (ctx->device->shader_core_count != 0 && n >= pipeline->wg_denoms[1]) { + if (rdna && ctx->device->shader_core_count != 0 && k >= 2048 && n <= 32) { + // Small n gives too few workgroups to hide latency, aim for 8 per CU. Small tiles are one subgroup each. + const uint32_t tiles = CEIL_DIV(m, pipeline->wg_denoms[0]) * CEIL_DIV(n, pipeline->wg_denoms[1]); + if (tiles < 4 * ctx->device->shader_core_count) { + split_k = std::min(8u, CEIL_DIV(8 * ctx->device->shader_core_count, tiles)); + } + } + + // On RDNA, also split when n <= 64 only partly fills one tile column. + if (split_k == 1 && ctx->device->shader_core_count != 0 && (n >= pipeline->wg_denoms[1] || (rdna && n <= 64))) { // If k is 'large' and the SMs will fill less than halfway, use split_k. uint32_t m_tiles = CEIL_DIV(m, pipeline->wg_denoms[0]); uint32_t n_tiles = CEIL_DIV(n, pipeline->wg_denoms[1]); @@ -5934,24 +5944,24 @@ static uint32_t ggml_vk_guess_split_k(ggml_backend_vk_context * ctx, uint32_t m, } // Cap the split at 8x. Unless k is huge this is a lot of overhead. split_k = std::min(split_k, 8u); - - // ggml_vk_matmul will align the splits to be a multiple of 256. - // If this rounded up size would cause the last split to be empty, - // then reduce the split count. - while (true) { - if (split_k == 1) { - break; - } - uint32_t k_split = CEIL_DIV(k, split_k); - k_split = ROUNDUP_POW2(k_split, 256); - if (k_split * (split_k - 1) < k) { - break; - } - split_k--; - } } } + // ggml_vk_matmul will align the splits to be a multiple of 256. + // If this rounded up size would cause the last split to be empty, + // then reduce the split count. + while (true) { + if (split_k == 1) { + break; + } + uint32_t k_split = CEIL_DIV(k, split_k); + k_split = ROUNDUP_POW2(k_split, 256); + if (k_split * (split_k - 1) < k) { + break; + } + split_k--; + } + return split_k; }