cuda: decode the conv2d patch indices with fastdiv (#32492)
The implicit GEMM conv2d kernel recovers the channel and the kernel position of each reduction index with three integer divisions by runtime divisors, once per loaded row of every tile. Outside the 3x3 and 1x1 specializations this integer division bounds the kernel: removing it halves the time on long reductions. Divide with the precomputed magic numbers of fastdiv instead. The 3x3 and 1x1 specializations keep their compile time divisors. test-backend-ops -o CONV_2D: 2006/2006 on CUDA0. CONV_2D, F16 kernel, us/run on an RTX PRO 6000: [2054,1,768] k [7,1,768,768] 221 -> 138 [16390,1,384] k [7,1,384,384] 433 -> 266 [65542,1,192] k [7,1,192,192] 420 -> 271 [8240,1,256] k [11,1,256,256] 165 -> 108 [19,19,256,16] k [4,4,256,4096] 1672 -> 1076
This commit is contained in:
1 parent
401563f10e
commit
aaa807c568
1 file changed
+15
-5
@@ -128,7 +128,9 @@ static __global__ void conv2d_implicit_gemm_f16(const half * __restrict__ input,
|
||||
const half * __restrict__ weight,
|
||||
float * __restrict__ output,
|
||||
const conv_params P,
|
||||
const int split_k) {
|
||||
const int split_k,
|
||||
const uint3 kw_fd,
|
||||
const uint3 kwkh_fd) {
|
||||
using namespace ggml_cuda_mma;
|
||||
constexpr int warp_size = ggml_cuda_get_physical_warp_size();
|
||||
constexpr int nthreads = 4 * warp_size;
|
||||
@@ -202,7 +204,10 @@ static __global__ void conv2d_implicit_gemm_f16(const half * __restrict__ input,
|
||||
}
|
||||
for (int k = load_row; k < BK; k += nthreads / (BN / 2)) {
|
||||
const int ki = k0 + k;
|
||||
const int ci = ki / (kw * kh), ky = ki / kw % kh, kx = ki % kw;
|
||||
const int ci = KW && KH ? ki / (KW * KH) : int(fastdiv(uint32_t(ki), kwkh_fd));
|
||||
const int r = ki - ci * kw * kh;
|
||||
const int ky = KW ? r / KW : int(fastdiv(uint32_t(r), kw_fd));
|
||||
const int kx = r - ky * kw;
|
||||
const int offset = ki < k_total ? (n * ic + ci) * ih * iw + ky * dy * iw + kx * dx : 0;
|
||||
half lo = __float2half(0.0f), hi = lo;
|
||||
if (ki < k_total && spatial < ow * oh) {
|
||||
@@ -314,12 +319,17 @@ static void conv2d_launch_implicit_gemm(const half * input,
|
||||
dim3 grid,
|
||||
dim3 block,
|
||||
cudaStream_t stream) {
|
||||
const uint3 kw_fd = init_fastdiv_values(uint64_t(params.KW));
|
||||
const uint3 kwkh_fd = init_fastdiv_values(uint64_t(params.KW * params.KH));
|
||||
if (params.KW == 3 && params.KH == 3) {
|
||||
conv2d_implicit_gemm_f16<3, 3, use_mma><<<grid, block, 0, stream>>>(input, weight, output, params, split_k);
|
||||
conv2d_implicit_gemm_f16<3, 3, use_mma>
|
||||
<<<grid, block, 0, stream>>>(input, weight, output, params, split_k, kw_fd, kwkh_fd);
|
||||
} else if (params.KW == 1 && params.KH == 1) {
|
||||
conv2d_implicit_gemm_f16<1, 1, use_mma><<<grid, block, 0, stream>>>(input, weight, output, params, split_k);
|
||||
conv2d_implicit_gemm_f16<1, 1, use_mma>
|
||||
<<<grid, block, 0, stream>>>(input, weight, output, params, split_k, kw_fd, kwkh_fd);
|
||||
} else {
|
||||
conv2d_implicit_gemm_f16<0, 0, use_mma><<<grid, block, 0, stream>>>(input, weight, output, params, split_k);
|
||||
conv2d_implicit_gemm_f16<0, 0, use_mma>
|
||||
<<<grid, block, 0, stream>>>(input, weight, output, params, split_k, kw_fd, kwkh_fd);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
Reference in new issue
Block a user