From ebc9fb933e774bb4ac0ba497a67948e16b7e5656 Mon Sep 17 00:00:00 2001 From: Giorgio Sarno Date: Tue, 17 Nov 2026 15:51:15 +0100 Subject: [PATCH] CUDA: use block reductions for whole-row pool_1d (#31855) * CUDA: use block reductions for whole-row pool_1d Assisted-by: Claude Code * CUDA: template pool1d row kernel by ggml_op_pool --- ggml/src/ggml-cuda/pool1d.cu | 36 +++++++++++++++++++++++++++++++++++ ggml/src/ggml-cuda/pool1d.cuh | 1 + tests/test-backend-ops.cpp | 9 +++++++++ 3 files changed, 46 insertions(+) diff --git a/ggml/src/ggml-cuda/pool1d.cu b/ggml/src/ggml-cuda/pool1d.cu index ac6fb0cbd..495bea0bf 100644 --- a/ggml/src/ggml-cuda/pool1d.cu +++ b/ggml/src/ggml-cuda/pool1d.cu @@ -49,6 +49,36 @@ static __global__ void pool1d_nchw_kernel( o_ptr[cur_ow] = res; } +// a window covering the whole row gives one output per row: reduce it with a block instead of one thread +template +static __global__ void pool1d_row_kernel(const int iw, const float * src, float * dst) { + static_assert(op == GGML_OP_POOL_MAX || op == GGML_OP_POOL_AVG, "unsupported pool op"); + + const float * row = src + (int64_t) blockIdx.x*iw; + + float res = op == GGML_OP_POOL_MAX ? -FLT_MAX : 0.0f; + for (int i = threadIdx.x; i < iw; i += blockDim.x) { + res = op == GGML_OP_POOL_MAX ? max(res, row[i]) : res + row[i]; + } + + __shared__ float buf[WARP_SIZE]; + res = block_reduce(res, buf); + + if (threadIdx.x == 0) { + dst[blockIdx.x] = op == GGML_OP_POOL_MAX ? res : res/iw; + } +} + +static void pool1d_row_f32_cuda(const int iw, const int64_t nr, const float * src, float * dst, const enum ggml_op_pool op, cudaStream_t stream) { + const int nsm = ggml_cuda_info().devices[ggml_cuda_get_device()].nsm; + const dim3 block_dims(nr/nsm < 2 ? 512 : (iw < 1024 ? 32 : 128)); + if (op == GGML_OP_POOL_MAX) { + pool1d_row_kernel<<>>(iw, src, dst); + } else { + pool1d_row_kernel<<>>(iw, src, dst); + } +} + static void pool1d_nchw_kernel_f32_f32_cuda( const int iw, const int ow, const int kw, const int sw, const int pw, @@ -79,6 +109,12 @@ void ggml_cuda_op_pool1d(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { const int64_t OW = dst->ne[0]; const int64_t nr = ggml_nrows(src0); + // short rows: one thread per row is faster + if (OW == 1 && p0 == 0 && k0 == IW && IW >= CUDA_POOL1D_ROW_MIN_IW && (op == GGML_OP_POOL_MAX || op == GGML_OP_POOL_AVG)) { + pool1d_row_f32_cuda(IW, nr, src0_d, dst_d, op, stream); + return; + } + const int parallel_elements = (int)(nr * OW); pool1d_nchw_kernel_f32_f32_cuda(IW, OW, k0, s0, p0, parallel_elements, src0_d, dst_d, op, stream); diff --git a/ggml/src/ggml-cuda/pool1d.cuh b/ggml/src/ggml-cuda/pool1d.cuh index c79461dd8..008b75ef5 100644 --- a/ggml/src/ggml-cuda/pool1d.cuh +++ b/ggml/src/ggml-cuda/pool1d.cuh @@ -1,5 +1,6 @@ #include "common.cuh" #define CUDA_POOL1D_BLOCK_SIZE 256 +#define CUDA_POOL1D_ROW_MIN_IW 256 void ggml_cuda_op_pool1d(ggml_backend_cuda_context & ctx, ggml_tensor * dst); diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cpp index a45d9bad5..da91b51e9 100644 --- a/tests/test-backend-ops.cpp +++ b/tests/test-backend-ops.cpp @@ -10920,6 +10920,11 @@ static std::vector> make_test_cases_eval() { } } } + // one window over the whole row (a row reduction) + for (int64_t iw : {10, 128, 256, 1000, 2048, 4099}) { + test_cases.emplace_back(new test_pool1d(pool_type, type_input, { iw, 3, 2, 1 }, (int) iw, (int) iw, 0)); + test_cases.emplace_back(new test_pool1d(pool_type, type_input, { iw, 1024, 1, 1 }, (int) iw, (int) iw, 0)); + } } } @@ -13652,6 +13657,10 @@ static std::vector> make_test_cases_perf() { } } } + // POOL_1D whole-row reduction with short rows and many rows + for (ggml_op_pool pool_type : {GGML_OP_POOL_AVG, GGML_OP_POOL_MAX}) { + test_cases.emplace_back(new test_pool1d(pool_type, GGML_TYPE_F32, {16, 65536, 1, 1}, 16, 16, 0)); + } // LEAKY_RELU at FFN activation width, for direct comparison with RELU for (int64_t n_tokens : {512, 2048}) {