diff --git a/ggml/src/ggml-cuda/fwht.cu b/ggml/src/ggml-cuda/fwht.cu index 67eea594d..d69e2ed81 100644 --- a/ggml/src/ggml-cuda/fwht.cu +++ b/ggml/src/ggml-cuda/fwht.cu @@ -2,6 +2,9 @@ #include "convert.cuh" #include "fwht.cuh" +// wide FWHT blocks use one row per thread block with this many threads +#define GGML_CUDA_FWHT_BLOCK_NT 256 + template __launch_bounds__(4*ggml_cuda_get_physical_warp_size(), 1) __global__ void fwht_cuda(const T * src, float * dst, const int64_t n_rows, const float scale) { @@ -59,6 +62,87 @@ __global__ void fwht_cuda(const T * src, float * dst, const int64_t n_rows, cons } } +// Wide blocks: one row per thread block instead of per warp, so each thread keeps N/NT +// values rather than N/32. Stages below the warp width still shuffle, those up to the +// block width go through shared memory, and the rest stay in registers. +template +__launch_bounds__(NT, 1) +__global__ void fwht_cuda_block(const T * src, float * dst, const int64_t n_rows, const float scale) { + constexpr int warp_size = ggml_cuda_get_physical_warp_size(); + constexpr int NE = N / NT; + static_assert(NE >= 1 && N % NT == 0 && NT % warp_size == 0, "bad FWHT block shape"); + + __shared__ float s[N]; + + const int64_t r = blockIdx.x; + if (r >= n_rows) { + return; + } + + src += r * N; + dst += r * N; + + const int tid = threadIdx.x; + const int lane = tid % warp_size; + + ggml_cuda_pdl_sync(); + + float reg[NE]; +#pragma unroll + for (int i = 0; i < NE; ++i) { + reg[i] = ggml_cuda_cast(src[i * NT + tid]) * scale; + } + + // stages within a warp: partner differs in the lane bits +#pragma unroll + for (int h = 1; h < warp_size; h *= 2) { +#pragma unroll + for (int j = 0; j < NE; j++) { + const float val = reg[j]; + const float val2 = __shfl_xor_sync(0xFFFFFFFF, val, h, warp_size); + reg[j] = (lane & h) == 0 ? val + val2 : val2 - val; + } + } + + // stages across warps: partner differs in the thread-index bits above the lane +#pragma unroll + for (int h = warp_size; h < NT; h *= 2) { +#pragma unroll + for (int j = 0; j < NE; j++) { + s[j * NT + tid] = reg[j]; + } + __syncthreads(); +#pragma unroll + for (int j = 0; j < NE; j++) { + const float val = reg[j]; + const float val2 = s[j * NT + (tid ^ h)]; + reg[j] = (tid & h) == 0 ? val + val2 : val2 - val; + } + __syncthreads(); + } + + // stages above the block width: partner is another register of the same thread +#pragma unroll + for (int h = NT; h < N; h *= 2) { + const int step = h / NT; +#pragma unroll + for (int j = 0; j < NE; j += 2 * step) { +#pragma unroll + for (int k = 0; k < step; k++) { + const float x = reg[j + k]; + const float y = reg[j + k + step]; + reg[j + k] = x + y; + reg[j + k + step] = x - y; + } + } + } + +#pragma unroll + for (int i = 0; i < NE; ++i) { + dst[i * NT + tid] = reg[i]; + } +} + template static bool ggml_cuda_op_fwht_impl(ggml_backend_cuda_context & ctx, const ggml_tensor * src, ggml_tensor * dst) { const int n = src->ne[0]; @@ -94,7 +178,34 @@ static bool ggml_cuda_op_fwht_impl(ggml_backend_cuda_context & ctx, const ggml_t ggml_cuda_kernel_launch(fwht_cuda<512, T>, launch_params, src_d, dst_d, rows, scale); return true; default: - return false; + break; + } + + // wide blocks: one row per thread block + { + constexpr int nt = GGML_CUDA_FWHT_BLOCK_NT; + + dim3 grid_dims_w(rows, 1, 1); + dim3 block_dims_w(nt, 1, 1); + const ggml_cuda_kernel_launch_params launch_params_w = + ggml_cuda_kernel_launch_params(grid_dims_w, block_dims_w, 0, stream); + + switch (n) { + case 1024: + ggml_cuda_kernel_launch(fwht_cuda_block<1024, nt, T>, launch_params_w, src_d, dst_d, rows, scale); + return true; + case 2048: + ggml_cuda_kernel_launch(fwht_cuda_block<2048, nt, T>, launch_params_w, src_d, dst_d, rows, scale); + return true; + case 4096: + ggml_cuda_kernel_launch(fwht_cuda_block<4096, nt, T>, launch_params_w, src_d, dst_d, rows, scale); + return true; + case 8192: + ggml_cuda_kernel_launch(fwht_cuda_block<8192, nt, T>, launch_params_w, src_d, dst_d, rows, scale); + return true; + default: + return false; + } } }