CUDA: use block reductions for whole-row pool_1d (#31855)

* CUDA: use block reductions for whole-row pool_1d

Assisted-by: Claude Code

* CUDA: template pool1d row kernel by ggml_op_pool
This commit is contained in:
Giorgio Sarno authored and GitHub committed 2026-11-17 15:51:15 +01:00
1 parent 8b8ca4c98e
commit ebc9fb933e
3 files changed
+46

No files matched your search

+36
View File
@@ -49,6 +49,36 @@ static __global__ void pool1d_nchw_kernel(
o_ptr[cur_ow] = res;
}
// a window covering the whole row gives one output per row: reduce it with a block instead of one thread
template <ggml_op_pool op>
static __global__ void pool1d_row_kernel(const int iw, const float * src, float * dst) {
static_assert(op == GGML_OP_POOL_MAX || op == GGML_OP_POOL_AVG, "unsupported pool op");
const float * row = src + (int64_t) blockIdx.x*iw;
float res = op == GGML_OP_POOL_MAX ? -FLT_MAX : 0.0f;
for (int i = threadIdx.x; i < iw; i += blockDim.x) {
res = op == GGML_OP_POOL_MAX ? max(res, row[i]) : res + row[i];
}
__shared__ float buf[WARP_SIZE];
res = block_reduce<op == GGML_OP_POOL_MAX ? block_reduce_method::MAX : block_reduce_method::SUM>(res, buf);
if (threadIdx.x == 0) {
dst[blockIdx.x] = op == GGML_OP_POOL_MAX ? res : res/iw;
}
}
static void pool1d_row_f32_cuda(const int iw, const int64_t nr, const float * src, float * dst, const enum ggml_op_pool op, cudaStream_t stream) {
const int nsm = ggml_cuda_info().devices[ggml_cuda_get_device()].nsm;
const dim3 block_dims(nr/nsm < 2 ? 512 : (iw < 1024 ? 32 : 128));
if (op == GGML_OP_POOL_MAX) {
pool1d_row_kernel<GGML_OP_POOL_MAX><<<nr, block_dims, 0, stream>>>(iw, src, dst);
} else {
pool1d_row_kernel<GGML_OP_POOL_AVG><<<nr, block_dims, 0, stream>>>(iw, src, dst);
}
}
static void pool1d_nchw_kernel_f32_f32_cuda(
const int iw, const int ow,
const int kw, const int sw, const int pw,
@@ -79,6 +109,12 @@ void ggml_cuda_op_pool1d(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {
const int64_t OW = dst->ne[0];
const int64_t nr = ggml_nrows(src0);
// short rows: one thread per row is faster
if (OW == 1 && p0 == 0 && k0 == IW && IW >= CUDA_POOL1D_ROW_MIN_IW && (op == GGML_OP_POOL_MAX || op == GGML_OP_POOL_AVG)) {
pool1d_row_f32_cuda(IW, nr, src0_d, dst_d, op, stream);
return;
}
const int parallel_elements = (int)(nr * OW);
pool1d_nchw_kernel_f32_f32_cuda(IW, OW, k0, s0, p0, parallel_elements, src0_d, dst_d, op, stream);
+1
View File
@@ -1,5 +1,6 @@
#include "common.cuh"
#define CUDA_POOL1D_BLOCK_SIZE 256
#define CUDA_POOL1D_ROW_MIN_IW 256
void ggml_cuda_op_pool1d(ggml_backend_cuda_context & ctx, ggml_tensor * dst);
+9
View File
@@ -10920,6 +10920,11 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() {
}
}
}
// one window over the whole row (a row reduction)
for (int64_t iw : {10, 128, 256, 1000, 2048, 4099}) {
test_cases.emplace_back(new test_pool1d(pool_type, type_input, { iw, 3, 2, 1 }, (int) iw, (int) iw, 0));
test_cases.emplace_back(new test_pool1d(pool_type, type_input, { iw, 1024, 1, 1 }, (int) iw, (int) iw, 0));
}
}
}
@@ -13652,6 +13657,10 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_perf() {
}
}
}
// POOL_1D whole-row reduction with short rows and many rows
for (ggml_op_pool pool_type : {GGML_OP_POOL_AVG, GGML_OP_POOL_MAX}) {
test_cases.emplace_back(new test_pool1d(pool_type, GGML_TYPE_F32, {16, 65536, 1, 1}, 16, 16, 0));
}
// LEAKY_RELU at FFN activation width, for direct comparison with RELU
for (int64_t n_tokens : {512, 2048}) {