vulkan: improve small-n split-K selection on RDNA3 and RDNA4 (#30146)
* vulkan: improve small-n split-K selection on RDNA3 and RDNA4 * Clean up test cases by removing commented code Removed tests as requested
This commit is contained in:
1 parent
cb02250b5b
commit
07ec7c8af0
1 file changed
+26
-16
@@ -5920,8 +5920,18 @@ static uint32_t ggml_vk_guess_split_k(ggml_backend_vk_context * ctx, uint32_t m,
|
||||
return 1;
|
||||
}
|
||||
|
||||
const bool rdna = ctx->device->architecture == AMD_RDNA3 || ctx->device->architecture == AMD_RDNA4;
|
||||
uint32_t split_k = 1;
|
||||
if (ctx->device->shader_core_count != 0 && n >= pipeline->wg_denoms[1]) {
|
||||
if (rdna && ctx->device->shader_core_count != 0 && k >= 2048 && n <= 32) {
|
||||
// Small n gives too few workgroups to hide latency, aim for 8 per CU. Small tiles are one subgroup each.
|
||||
const uint32_t tiles = CEIL_DIV(m, pipeline->wg_denoms[0]) * CEIL_DIV(n, pipeline->wg_denoms[1]);
|
||||
if (tiles < 4 * ctx->device->shader_core_count) {
|
||||
split_k = std::min(8u, CEIL_DIV(8 * ctx->device->shader_core_count, tiles));
|
||||
}
|
||||
}
|
||||
|
||||
// On RDNA, also split when n <= 64 only partly fills one tile column.
|
||||
if (split_k == 1 && ctx->device->shader_core_count != 0 && (n >= pipeline->wg_denoms[1] || (rdna && n <= 64))) {
|
||||
// If k is 'large' and the SMs will fill less than halfway, use split_k.
|
||||
uint32_t m_tiles = CEIL_DIV(m, pipeline->wg_denoms[0]);
|
||||
uint32_t n_tiles = CEIL_DIV(n, pipeline->wg_denoms[1]);
|
||||
@@ -5934,24 +5944,24 @@ static uint32_t ggml_vk_guess_split_k(ggml_backend_vk_context * ctx, uint32_t m,
|
||||
}
|
||||
// Cap the split at 8x. Unless k is huge this is a lot of overhead.
|
||||
split_k = std::min(split_k, 8u);
|
||||
|
||||
// ggml_vk_matmul will align the splits to be a multiple of 256.
|
||||
// If this rounded up size would cause the last split to be empty,
|
||||
// then reduce the split count.
|
||||
while (true) {
|
||||
if (split_k == 1) {
|
||||
break;
|
||||
}
|
||||
uint32_t k_split = CEIL_DIV(k, split_k);
|
||||
k_split = ROUNDUP_POW2(k_split, 256);
|
||||
if (k_split * (split_k - 1) < k) {
|
||||
break;
|
||||
}
|
||||
split_k--;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ggml_vk_matmul will align the splits to be a multiple of 256.
|
||||
// If this rounded up size would cause the last split to be empty,
|
||||
// then reduce the split count.
|
||||
while (true) {
|
||||
if (split_k == 1) {
|
||||
break;
|
||||
}
|
||||
uint32_t k_split = CEIL_DIV(k, split_k);
|
||||
k_split = ROUNDUP_POW2(k_split, 256);
|
||||
if (k_split * (split_k - 1) < k) {
|
||||
break;
|
||||
}
|
||||
split_k--;
|
||||
}
|
||||
|
||||
return split_k;
|
||||
}
|
||||
|
||||
|
||||
Reference in new issue
Block a user