From 0fb8f157702702f33f4e018de8701b2569a1bc28 Mon Sep 17 00:00:00 2001 From: anujj Date: Thu, 12 Nov 2026 18:42:45 +0530 Subject: [PATCH] Title: CUDA: few-row MMQ for BS 2-24 (#31533) * CUDA: few-row MMQ for 2-24 src1 columns New int8 MMA kernel for quantized mat-muls with 2-24 src1 columns, e.g. speculative decoding verify passes. Supports Q8_0, Q3_K, Q4_K, Q5_K, Q6_K, IQ3_S, IQ4_NL and IQ4_XS on NVIDIA GPUs from Ampere on. * nit fixes * CUDA-only guard for few-row MMQ, weights flag in test_mul_mat * few-row quantize: DS4 sum from the quantized values, as in #31247 * few-row MMQ: clearer HIP/MUSA abort, test_mul_mat two-context build * few-row MMQ: dispatch order MMVQ, few-row, MMQ --- ggml/src/ggml-cuda/common.cuh | 8 + ggml/src/ggml-cuda/cp-async.cuh | 20 + ggml/src/ggml-cuda/ggml-cuda.cu | 13 +- ggml/src/ggml-cuda/mmq-few.cu | 876 ++++++++++++++++++++++++++ ggml/src/ggml-cuda/mmq-few.cuh | 7 + ggml/src/ggml-cuda/mmq-load-tiles.cuh | 17 +- ggml/src/ggml-cuda/mmvq.cu | 8 +- ggml/src/ggml-cuda/quantize.cu | 75 +++ ggml/src/ggml-cuda/quantize.cuh | 11 + tests/test-backend-ops.cpp | 67 +- 10 files changed, 1073 insertions(+), 29 deletions(-) create mode 100644 ggml/src/ggml-cuda/mmq-few.cu create mode 100644 ggml/src/ggml-cuda/mmq-few.cuh diff --git a/ggml/src/ggml-cuda/common.cuh b/ggml/src/ggml-cuda/common.cuh index dfb222b4f..414177308 100644 --- a/ggml/src/ggml-cuda/common.cuh +++ b/ggml/src/ggml-cuda/common.cuh @@ -132,6 +132,14 @@ static __device__ __forceinline__ void ggml_cuda_syncwarp() { #endif // GGML_USE_HIP } +static __device__ __forceinline__ void ggml_cuda_prefetch_l2(const void * p) { +#if !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA) + asm volatile("prefetch.global.L2 [%0];" :: "l"(p)); +#else + GGML_UNUSED(p); +#endif // !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA) +} + static __device__ __forceinline__ void ggml_cuda_pdl_sync() { #if defined(GGML_CUDA_USE_PDL) && __CUDA_ARCH__ >= GGML_CUDA_CC_HOPPER cudaGridDependencySynchronize(); diff --git a/ggml/src/ggml-cuda/cp-async.cuh b/ggml/src/ggml-cuda/cp-async.cuh index 63d0c482f..1071352e7 100644 --- a/ggml/src/ggml-cuda/cp-async.cuh +++ b/ggml/src/ggml-cuda/cp-async.cuh @@ -55,3 +55,23 @@ static __device__ __forceinline__ void cp_async_wait_all() { NO_DEVICE_CODE; #endif // CP_ASYNC_AVAILABLE } + +// Commits the copies issued since the last commit as one group. +static __device__ __forceinline__ void cp_async_commit_group() { +#ifdef CP_ASYNC_AVAILABLE + asm volatile("cp.async.commit_group;"); +#else + NO_DEVICE_CODE; +#endif // CP_ASYNC_AVAILABLE +} + +// Makes each thread wait until at most n of its copy groups are pending. +// Like cp_async_wait_all, this does NOT provide any additional synchronization. +template +static __device__ __forceinline__ void cp_async_wait_group() { +#ifdef CP_ASYNC_AVAILABLE + asm volatile("cp.async.wait_group %0;" : : "n"(n)); +#else + NO_DEVICE_CODE; +#endif // CP_ASYNC_AVAILABLE +} diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu index 28073a17f..dd384d91f 100644 --- a/ggml/src/ggml-cuda/ggml-cuda.cu +++ b/ggml/src/ggml-cuda/ggml-cuda.cu @@ -34,6 +34,7 @@ #include "ggml-cuda/im2col.cuh" #include "ggml-cuda/mmf.cuh" #include "ggml-cuda/mmq.cuh" +#include "ggml-cuda/mmq-few.cuh" #include "ggml-cuda/mmvf.cuh" #include "ggml-cuda/mmvq.cuh" #include "ggml-cuda/moe-weighted-reduction.cuh" @@ -2431,6 +2432,10 @@ static bool ggml_cuda_should_fuse_mul_mat_q(const ggml_tensor * up, const ggml_t const int64_t ncols = is_id ? y->ne[2] : y->ne[1]; const int64_t nchannels = is_id ? x_up->ne[2] : y->ne[2]*y->ne[3]; const bool use_mmvq = is_id ? ncols <= get_mmvq_mmid_max_batch(x_up->type, cc) : ggml_cuda_should_use_mmvq(x_up->type, cc, ncols); + // do not fuse the matmuls that few-row MMQ takes in ggml_cuda_mul_mat + if (!is_id && ggml_cuda_should_use_mmq_few(x_up, y, cc)) { + return false; + } if (use_mmvq || !ggml_cuda_should_use_mmq(x_up->type, cc, ncols, /*n_experts =*/ is_id ? x_up->ne[2] : 0)) { return false; } @@ -2483,10 +2488,16 @@ static void ggml_cuda_mul_mat(ggml_backend_cuda_context & ctx, const ggml_tensor ggml_cuda_mul_mat_f(ctx, src0, src1, nullptr, dst); return; } - if (ggml_cuda_should_use_mmvq(src0->type, cc, ne11)) { + // from a per-type number of src1 columns up to 24, few-row MMQ is faster than MMVQ and MMQ + const bool use_mmq_few = ggml_cuda_should_use_mmq_few(src0, src1, cc); + if (!use_mmq_few && ggml_cuda_should_use_mmvq(src0->type, cc, ne11)) { ggml_cuda_mul_mat_vec_q(ctx, src0, src1, nullptr, dst); return; } + if (use_mmq_few) { + ggml_cuda_mul_mat_q_few(ctx, src0, src1, dst); + return; + } if (ggml_cuda_should_use_mmq(src0->type, cc, ne11, /*n_experts =*/ 0)) { ggml_cuda_mul_mat_q(ctx, src0, src1, nullptr, dst); return; diff --git a/ggml/src/ggml-cuda/mmq-few.cu b/ggml/src/ggml-cuda/mmq-few.cu new file mode 100644 index 000000000..2c2a2745e --- /dev/null +++ b/ggml/src/ggml-cuda/mmq-few.cu @@ -0,0 +1,876 @@ +#include "mmq-few.cuh" +#include "cp-async.cuh" +#include "mma.cuh" +#include "quantize.cuh" +#include "vecdotq.cuh" + +#include + +// NVIDIA only: the kernel uses the NVIDIA int8 MMA fragment layout and cp.async. +#if !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA) + +using namespace ggml_cuda_mma; + +// Few-row MMQ: int8 MMA mat-mul for 2-24 src1 columns. A CUDA block has 16 src0 rows (32 for 2 column groups on large matrices). +// Its warps split K in units of 256 values and copy raw src0 rows into their own cp.async ring, src1 is quantized to the MMA fragment layout. + +static constexpr int MMQ_FEW_MAX_NCG = 3; // max. number of column groups +static constexpr int MMQ_FEW_NROWS = 16; // src0 rows of one MMA tile +static constexpr int MMQ_FEW_NWARPS = 4; + +template +static constexpr __host__ __device__ int mmq_few_unit_size_of() { + return QK_K/ggml_cuda_type_traits::qk*ggml_cuda_type_traits::bs; +} + +// src0 bytes per 256 values, 0 for the types without a few-row decoder. +static constexpr __host__ __device__ int mmq_few_unit_size(const ggml_type type) { + switch (type) { + case GGML_TYPE_Q8_0: return mmq_few_unit_size_of(); + case GGML_TYPE_Q3_K: return mmq_few_unit_size_of(); + case GGML_TYPE_Q4_K: return mmq_few_unit_size_of(); + case GGML_TYPE_Q5_K: return mmq_few_unit_size_of(); + case GGML_TYPE_Q6_K: return mmq_few_unit_size_of(); + case GGML_TYPE_IQ3_S: return mmq_few_unit_size_of(); + case GGML_TYPE_IQ4_NL: return mmq_few_unit_size_of(); + case GGML_TYPE_IQ4_XS: return mmq_few_unit_size_of(); + default: return 0; + } +} + +// Q4_K and Q5_K also need the src1 sums for their mins. +static constexpr __host__ __device__ mmq_q8_1_ds_layout mmq_few_ds_layout(const ggml_type type) { + return type == GGML_TYPE_Q4_K || type == GGML_TYPE_Q5_K ? MMQ_Q8_1_DS_LAYOUT_DS4 : MMQ_Q8_1_DS_LAYOUT_D4; +} + +// Units of 256 values per ring stage: 1 for Q6_K, 3 column groups (fewer free registers) and 32-row tiles (2 blocks per SM), else 2. +static constexpr __host__ __device__ int mmq_few_nunits_stage(const ggml_type type, const int ncg, const int nmt) { + return type == GGML_TYPE_Q6_K || ncg > 2 || nmt > 1 ? 1 : 2; +} + +// A third ring stage only makes Q6_K faster, the other types get slower on large matrices. +static constexpr __host__ __device__ int mmq_few_nstages(const ggml_type type) { + return type == GGML_TYPE_Q6_K ? 3 : 2; +} + +// The src0 data of a ring stage does not fill whole 16 byte chunks, it is copied from the preceding 16 byte boundary. +static constexpr __host__ __device__ bool mmq_few_unaligned(const ggml_type type, const int nunits_stage) { + return (nunits_stage*mmq_few_unit_size(type)) % 16 != 0; +} + +// for unaligned data the max. over the 8 possible start offsets +static constexpr __host__ __device__ int mmq_few_nchunks(const ggml_type type, const int nunits_stage) { + const int nbytes = nunits_stage*mmq_few_unit_size(type); + return mmq_few_unaligned(type, nunits_stage) ? (14 + nbytes + 15)/16 : nbytes/16; +} + +// Unaligned rows are padded to an odd number of 16 byte chunks against bank conflicts, for aligned rows the padding costs more than it saves. +static constexpr __host__ __device__ int mmq_few_sram_stride(const ggml_type type, const int nunits_stage) { + const int nchunks = mmq_few_nchunks(type, nunits_stage); + return mmq_few_unaligned(type, nunits_stage) && nchunks % 2 == 0 ? 16*(nchunks + 1) : 16*nchunks; +} + +static constexpr __host__ __device__ int mmq_few_nbytes_shared(const ggml_type type, const int ncg, const int nmt) { + return MMQ_FEW_NWARPS*mmq_few_nstages(type)*nmt*MMQ_FEW_NROWS*mmq_few_sram_stride(type, mmq_few_nunits_stage(type, ncg, nmt)); +} + +// With Q6_K and Q8_0 two CUDA blocks with 32-row tiles would not fit into an SM. +static constexpr __host__ __device__ bool mmq_few_tall_supported(const ggml_type type) { + return type == GGML_TYPE_Q3_K || type == GGML_TYPE_Q4_K || type == GGML_TYPE_Q5_K || + type == GGML_TYPE_IQ3_S || type == GGML_TYPE_IQ4_NL || type == GGML_TYPE_IQ4_XS; +} + +#if defined(CP_ASYNC_AVAILABLE) + +// The src1 fragments of a lane for one unit of 256 values, see quantize.cuh. +struct mmq_few_y_frag { + int4 qs[4]; + int4 ds[4]; +}; + +static __device__ __forceinline__ void mmq_few_load_y(mmq_few_y_frag & yf, const int4 * y) { +#pragma unroll + for (int j = 0; j < 4; ++j) { + yf.qs[j] = y[j*WARP_SIZE + threadIdx.x]; + } +#pragma unroll + for (int j = 0; j < 4; ++j) { + yf.ds[j] = y[4*WARP_SIZE + 4*(threadIdx.x % 4) + j]; + } +} + +static __device__ __forceinline__ int2 mmq_few_get_b(const mmq_few_y_frag & yf, const int s) { + const int4 q = yf.qs[s/2]; + return s % 2 == 0 ? make_int2(q.x, q.y) : make_int2(q.z, q.w); +} + +template +static __device__ __forceinline__ float2 mmq_few_get_d(const mmq_few_y_frag & yf, const int s) { + if constexpr (ds_layout == MMQ_Q8_1_DS_LAYOUT_D4) { + const float * d = (const float *) &yf.ds[s/2]; + return make_float2(d[2*(s % 2) + 0], d[2*(s % 2) + 1]); + } else { + const half2 * ds = (const half2 *) &yf.ds[s/2]; + return make_float2(__low2float(ds[2*(s % 2) + 0]), __low2float(ds[2*(s % 2) + 1])); + } +} + +static __device__ __forceinline__ float2 mmq_few_get_sum(const mmq_few_y_frag & yf, const int s) { + const half2 * ds = (const half2 *) &yf.ds[s/2]; + return make_float2(__high2float(ds[2*(s % 2) + 0]), __high2float(ds[2*(s % 2) + 1])); +} + +// C.x[0], C.x[1]: row g, columns 2t and 2t+1. C.x[2], C.x[3]: row g+8. +static __device__ __forceinline__ tile<16, 8, int> mmq_few_mma(const int (&lo)[2], const int (&hi)[2], const int2 b) { + tile<16, 8, int> A; + A.x[0] = lo[0]; + A.x[1] = lo[1]; + A.x[2] = hi[0]; + A.x[3] = hi[1]; + tile<8, 8, int> B; + B.x[0] = b.x; + B.x[1] = b.y; + tile<16, 8, int> C; + mma(C, A, B); + return C; +} + +static __device__ __forceinline__ tile<16, 8, int> mmq_few_mma_k16(const int (&a)[2], const int b) { + tile<16, 4, int> A; + A.x[0] = a[0]; + A.x[1] = a[1]; + tile<8, 4, int> B; + B.x[0] = b; + tile<16, 8, int> C; + mma(C, A, B); + return C; +} + +// sum_d += d * (C * ls) of sub-block s for all column groups, ls are the src0 row scales +template +static __device__ __forceinline__ void mmq_few_mma_add( + const int (&lo)[2], const int (&hi)[2], const scale_t (&ls)[2], const mmq_few_y_frag (&yf)[ncg], const int s, float (&sum_d)[ncg][4]) { +#pragma unroll + for (int c = 0; c < ncg; ++c) { + const tile<16, 8, int> C = mmq_few_mma(lo, hi, mmq_few_get_b(yf[c], s)); + + const float2 d = mmq_few_get_d(yf[c], s); + sum_d[c][0] += d.x * float(C.x[0]*ls[0]); + sum_d[c][1] += d.y * float(C.x[1]*ls[0]); + sum_d[c][2] += d.x * float(C.x[2]*ls[1]); + sum_d[c][3] += d.y * float(C.x[3]*ls[1]); + } +} + +// Same with one k=16 MMA per half of sub-block s, the int8 scales of the halves are bytes 0 and 1 of sc[r]. +template +static __device__ __forceinline__ void mmq_few_mma_k16_add( + const int (&lo)[2], const int (&hi)[2], const int (&sc)[2], const mmq_few_y_frag (&yf)[ncg], const int s, float (&sum_d)[ncg][4]) { + const int sc_lo[2] = {(int8_t) sc[0], (int8_t) sc[1]}; + const int sc_hi[2] = {(int8_t) (sc[0] >> 8), (int8_t) (sc[1] >> 8)}; +#pragma unroll + for (int c = 0; c < ncg; ++c) { + const int2 b = mmq_few_get_b(yf[c], s); + const tile<16, 8, int> C_lo = mmq_few_mma_k16(lo, b.x); + const tile<16, 8, int> C_hi = mmq_few_mma_k16(hi, b.y); + + const float2 d = mmq_few_get_d(yf[c], s); + sum_d[c][0] += d.x * float(C_lo.x[0]*sc_lo[0] + C_hi.x[0]*sc_hi[0]); + sum_d[c][1] += d.y * float(C_lo.x[1]*sc_lo[0] + C_hi.x[1]*sc_hi[0]); + sum_d[c][2] += d.x * float(C_lo.x[2]*sc_lo[1] + C_hi.x[2]*sc_hi[1]); + sum_d[c][3] += d.y * float(C_lo.x[3]*sc_lo[1] + C_hi.x[3]*sc_hi[1]); + } +} + +template +static __device__ __forceinline__ void mmq_few_add_scaled(float (&sum)[ncg][4], const float (&d)[2], const float (&sum_d)[ncg][4]) { +#pragma unroll + for (int c = 0; c < ncg; ++c) { +#pragma unroll + for (int l = 0; l < 4; ++l) { + sum[c][l] += d[l/2]*sum_d[c][l]; + } + } +} + +// vec_dot: one unit of 256 values of the 16 rows of a warp, x[0] and x[1] are the rows g and g+8 of lane (g, t). +// The lane decodes the values [4t, 4t + 4) and [16 + 4t, 16 + 4t + 4) of each sub-block, its A fragments. + +template +static __device__ __forceinline__ void mmq_few_vec_dot_q45_K(const char * const (&x)[2], const mmq_few_y_frag (&yf)[ncg], float (&sum)[ncg][4]) { + using block_t = std::conditional_t; + const int t = threadIdx.x % 4; + + int qs[2][4][2]; + [[maybe_unused]] int qh[2][2]; // Q5_K: bit s holds the high bit of sub-block s + int sc[2][2]; + int mn[2][2]; + float d[2]; + float dmin[2]; +#pragma unroll + for (int r = 0; r < 2; ++r) { + const block_t * bx = (const block_t *) x[r]; +#pragma unroll + for (int j = 0; j < 4; ++j) { + qs[r][j][0] = get_int_b4(bx->qs + 32*j, t); + qs[r][j][1] = get_int_b4(bx->qs + 32*j + 16, t); + } + if constexpr (type == GGML_TYPE_Q5_K) { + qh[r][0] = get_int_b4(bx->qh, t); + qh[r][1] = get_int_b4(bx->qh + 16, t); + } + const int4 hdr = *(const int4 *) bx; // dm and the 12 scale bytes + const int scales[3] = {hdr.y, hdr.z, hdr.w}; + sc[r][0] = unpack_scales_q45_K(scales, 0); + sc[r][1] = unpack_scales_q45_K(scales, 1); + mn[r][0] = unpack_scales_q45_K(scales, 2); + mn[r][1] = unpack_scales_q45_K(scales, 3); + const float2 dm = __half22float2(bx->dm); + d[r] = dm.x; + dmin[r] = dm.y; + } + + float sum_d[ncg][4] = {{0.0f}}; + float sum_m[ncg][4] = {{0.0f}}; +#pragma unroll + for (int s = 0; s < 8; ++s) { + const int j = s/2; + const int shift = 4*(s % 2); + int lo[2]; + int hi[2]; +#pragma unroll + for (int r = 0; r < 2; ++r) { + lo[r] = (qs[r][j][0] >> shift) & 0x0F0F0F0F; + hi[r] = (qs[r][j][1] >> shift) & 0x0F0F0F0F; + if constexpr (type == GGML_TYPE_Q5_K) { + lo[r] |= ((qh[r][0] >> s) & 0x01010101) << 4; + hi[r] |= ((qh[r][1] >> s) & 0x01010101) << 4; + } + } + const int ls[2] = {(sc[0][s/4] >> (8*(s % 4))) & 0xFF, (sc[1][s/4] >> (8*(s % 4))) & 0xFF}; + const float m0 = (mn[0][s/4] >> (8*(s % 4))) & 0xFF; + const float m1 = (mn[1][s/4] >> (8*(s % 4))) & 0xFF; + mmq_few_mma_add(lo, hi, ls, yf, s, sum_d); +#pragma unroll + for (int c = 0; c < ncg; ++c) { + const float2 sm = mmq_few_get_sum(yf[c], s); + sum_m[c][0] += m0 * sm.x; + sum_m[c][1] += m0 * sm.y; + sum_m[c][2] += m1 * sm.x; + sum_m[c][3] += m1 * sm.y; + } + } +#pragma unroll + for (int c = 0; c < ncg; ++c) { +#pragma unroll + for (int l = 0; l < 4; ++l) { + sum[c][l] += d[l/2]*sum_d[c][l] - dmin[l/2]*sum_m[c][l]; + } + } +} + +// Q6_K and Q3_K have a scale per 16 values: one k=16 MMA per half of a sub-block. +template +static __device__ __forceinline__ void mmq_few_vec_dot_q6_K(const char * const (&x)[2], const mmq_few_y_frag (&yf)[ncg], float (&sum)[ncg][4]) { + const int t = threadIdx.x % 4; + + float d[2]; +#pragma unroll + for (int r = 0; r < 2; ++r) { + d[r] = __half2float(((const block_q6_K *) x[r])->d); + } + + float sum_d[ncg][4] = {{0.0f}}; +#pragma unroll + for (int n = 0; n < 2; ++n) { + int ql[2][2][2]; // [row][ql offset 0 or 32][lo or hi] + int qh[2][2]; + int sc[2][2]; +#pragma unroll + for (int r = 0; r < 2; ++r) { + const block_q6_K * bx = (const block_q6_K *) x[r]; +#pragma unroll + for (int k = 0; k < 2; ++k) { + ql[r][k][0] = get_int_b2(bx->ql + 64*n + 32*k, t); + ql[r][k][1] = get_int_b2(bx->ql + 64*n + 32*k + 16, t); + } + qh[r][0] = get_int_b2(bx->qh + 32*n, t); + qh[r][1] = get_int_b2(bx->qh + 32*n + 16, t); + sc[r][0] = get_int_b2(bx->scales + 8*n, 0); + sc[r][1] = get_int_b2(bx->scales + 8*n, 1); + } +#pragma unroll + for (int j = 0; j < 4; ++j) { + const int s = 4*n + j; + int lo[2]; + int hi[2]; +#pragma unroll + for (int r = 0; r < 2; ++r) { + lo[r] = __vsub4(((ql[r][j % 2][0] >> (4*(j/2))) & 0x0F0F0F0F) | (((qh[r][0] >> (2*j)) & 0x03030303) << 4), 0x20202020); + hi[r] = __vsub4(((ql[r][j % 2][1] >> (4*(j/2))) & 0x0F0F0F0F) | (((qh[r][1] >> (2*j)) & 0x03030303) << 4), 0x20202020); + } + const int sc_s[2] = {sc[0][j/2] >> (16*(j % 2)), sc[1][j/2] >> (16*(j % 2))}; // scales 2s and 2s+1 + mmq_few_mma_k16_add(lo, hi, sc_s, yf, s, sum_d); + } + } + mmq_few_add_scaled(sum, d, sum_d); +} + +template +static __device__ __forceinline__ void mmq_few_vec_dot_q3_K(const char * const (&x)[2], const mmq_few_y_frag (&yf)[ncg], float (&sum)[ncg][4]) { + const int t = threadIdx.x % 4; + + int hm[2][2]; + int sc[2][4]; // the 16 scales of a row as int8, minus 32 + float d[2]; +#pragma unroll + for (int r = 0; r < 2; ++r) { + const block_q3_K * bx = (const block_q3_K *) x[r]; + hm[r][0] = ~get_int_b2(bx->hmask, t); // a set bit means that 4 is subtracted + hm[r][1] = ~get_int_b2(bx->hmask + 16, t); +#pragma unroll + for (int ksc = 0; ksc < 4; ++ksc) { + sc[r][ksc] = unpack_scales_q3_K(bx->scales, ksc); + } + d[r] = __half2float(bx->d); + } + + float sum_d[ncg][4] = {{0.0f}}; +#pragma unroll + for (int n = 0; n < 2; ++n) { + int qs[2][2]; +#pragma unroll + for (int r = 0; r < 2; ++r) { + const block_q3_K * bx = (const block_q3_K *) x[r]; + qs[r][0] = get_int_b2(bx->qs + 32*n, t); + qs[r][1] = get_int_b2(bx->qs + 32*n + 16, t); + } +#pragma unroll + for (int j = 0; j < 4; ++j) { + const int s = 4*n + j; + int lo[2]; + int hi[2]; +#pragma unroll + for (int r = 0; r < 2; ++r) { + lo[r] = __vsub4((qs[r][0] >> (2*j)) & 0x03030303, ((hm[r][0] >> s) & 0x01010101) << 2); + hi[r] = __vsub4((qs[r][1] >> (2*j)) & 0x03030303, ((hm[r][1] >> s) & 0x01010101) << 2); + } + const int sc_s[2] = {sc[0][s/2] >> (16*(s % 2)), sc[1][s/2] >> (16*(s % 2))}; // scales 2s and 2s+1 + mmq_few_mma_k16_add(lo, hi, sc_s, yf, s, sum_d); + } + } + mmq_few_add_scaled(sum, d, sum_d); +} + +template +static __device__ __forceinline__ void mmq_few_vec_dot_iq4_xs(const char * const (&x)[2], const mmq_few_y_frag (&yf)[ncg], float (&sum)[ncg][4]) { + const int t = threadIdx.x % 4; + + float d[2]; + int scales_h[2]; + int scales_l[2]; +#pragma unroll + for (int r = 0; r < 2; ++r) { + const block_iq4_xs * bx = (const block_iq4_xs *) x[r]; + d[r] = __half2float(bx->d); + scales_h[r] = bx->scales_h; + scales_l[r] = get_int_b4(bx->scales_l, 0); + } + + float sum_d[ncg][4] = {{0.0f}}; +#pragma unroll + for (int s = 0; s < 8; ++s) { + int lo[2]; + int hi[2]; + int ls[2]; +#pragma unroll + for (int r = 0; r < 2; ++r) { + const block_iq4_xs * bx = (const block_iq4_xs *) x[r]; + const int2 v = get_int_from_table_16(get_int_b4(bx->qs + 16*s, t), kvalues_iq4nl); + lo[r] = v.x; + hi[r] = v.y; + ls[r] = (((scales_l[r] >> (4*s)) & 0x0F) | (((scales_h[r] >> (2*s)) & 0x03) << 4)) - 32; + } + mmq_few_mma_add(lo, hi, ls, yf, s, sum_d); + } + mmq_few_add_scaled(sum, d, sum_d); +} + +// Q8_0 and IQ4_NL have a block with its own scale per 32 values. +template +static __device__ __forceinline__ void mmq_few_vec_dot_q8_0_iq4_nl(const char * const (&x)[2], const mmq_few_y_frag (&yf)[ncg], float (&sum)[ncg][4]) { + const int t = threadIdx.x % 4; + +#pragma unroll + for (int s = 0; s < 8; ++s) { + int lo[2]; + int hi[2]; + float d[2]; +#pragma unroll + for (int r = 0; r < 2; ++r) { + if constexpr (type == GGML_TYPE_Q8_0) { + const block_q8_0 * bx = (const block_q8_0 *) x[r] + s; + lo[r] = get_int_b2(bx->qs, t); + hi[r] = get_int_b2(bx->qs + 16, t); + d[r] = __half2float(bx->d); + } else { + const block_iq4_nl * bx = (const block_iq4_nl *) x[r] + s; + const int2 v = get_int_from_table_16(get_int_b2(bx->qs, t), kvalues_iq4nl); + lo[r] = v.x; + hi[r] = v.y; + d[r] = __half2float(bx->d); + } + } + mmq_few_mma_add(lo, hi, d, yf, s, sum); + } +} + +template +static __device__ __forceinline__ void mmq_few_vec_dot_iq3_s(const char * const (&x)[2], const mmq_few_y_frag (&yf)[ncg], float (&sum)[ncg][4]) { + const int t = threadIdx.x % 4; + + float d[2]; +#pragma unroll + for (int r = 0; r < 2; ++r) { + d[r] = __half2float(((const block_iq3_s *) x[r])->d); + } + + float sum_d[ncg][4] = {{0.0f}}; +#pragma unroll + for (int s = 0; s < 8; ++s) { + int lo[2]; + int hi[2]; + int ls[2]; +#pragma unroll + for (int r = 0; r < 2; ++r) { + const block_iq3_s * bx = (const block_iq3_s *) x[r]; + const int qh = bx->qh[s]; + // the values [4k, 4k + 4) of a sub-block come from grid entry k, with the signs in bits 4*(k%2) of byte k/2 + const int k_lo = t; + const int k_hi = 4 + t; + const int grid_lo = iq3s_grid[bx->qs[8*s + k_lo] | (((qh >> k_lo) & 1) << 8)]; + const int grid_hi = iq3s_grid[bx->qs[8*s + k_hi] | (((qh >> k_hi) & 1) << 8)]; + const int sign_lo = (bx->signs[4*s + k_lo/2] >> (4*(k_lo % 2))) & 0x0F; + const int sign_hi = (bx->signs[4*s + k_hi/2] >> (4*(k_hi % 2))) & 0x0F; + const int mask_lo = __vcmpne4(((sign_lo & 0x03) << 7) | ((sign_lo & 0x0C) << 21), 0x00000000); + const int mask_hi = __vcmpne4(((sign_hi & 0x03) << 7) | ((sign_hi & 0x0C) << 21), 0x00000000); + lo[r] = __vsub4(grid_lo ^ mask_lo, mask_lo); + hi[r] = __vsub4(grid_hi ^ mask_hi, mask_hi); + ls[r] = 1 + 2*((bx->scales[s/2] >> (4*(s % 2))) & 0x0F); + } + mmq_few_mma_add(lo, hi, ls, yf, s, sum_d); + } + mmq_few_add_scaled(sum, d, sum_d); +} + +template +static __device__ __forceinline__ void mmq_few_vec_dot(const char * const (&x)[2], const mmq_few_y_frag (&yf)[ncg], float (&sum)[ncg][4]) { + if constexpr (type == GGML_TYPE_Q8_0 || type == GGML_TYPE_IQ4_NL) { + mmq_few_vec_dot_q8_0_iq4_nl(x, yf, sum); + } else if constexpr (type == GGML_TYPE_Q3_K) { + mmq_few_vec_dot_q3_K(x, yf, sum); + } else if constexpr (type == GGML_TYPE_Q4_K || type == GGML_TYPE_Q5_K) { + mmq_few_vec_dot_q45_K(x, yf, sum); + } else if constexpr (type == GGML_TYPE_Q6_K) { + mmq_few_vec_dot_q6_K(x, yf, sum); + } else if constexpr (type == GGML_TYPE_IQ3_S) { + mmq_few_vec_dot_iq3_s(x, yf, sum); + } else if constexpr (type == GGML_TYPE_IQ4_XS) { + mmq_few_vec_dot_iq4_xs(x, yf, sum); + } else { + static_assert(type == GGML_TYPE_COUNT, "unsupported type"); + } +} + +#endif // defined(CP_ASYNC_AVAILABLE) + +template +__launch_bounds__(MMQ_FEW_NWARPS*ggml_cuda_get_physical_warp_size(), 1) +static __global__ void mul_mat_q_few( + const char * x, const int4 * y_ptr, float * dst, + const int nunits, const int nrows_x, const int ncols_dst, const int64_t stride_row_x, const int stride_col_dst, + const int channel_ratio, const int64_t stride_channel_x, const int stride_channel_dst, + const int sample_ratio, const int64_t stride_sample_x, const int stride_sample_dst, const bool x_is_constant) { +#if defined(CP_ASYNC_AVAILABLE) + constexpr int nwarps = MMQ_FEW_NWARPS; + constexpr int warp_size = ggml_cuda_get_physical_warp_size(); + static_assert(warp_size == WARP_SIZE, "unsupported warp size"); + + constexpr int unit_size = mmq_few_unit_size(type); + constexpr int nrows = nmt*MMQ_FEW_NROWS; + constexpr int nunits_stage = mmq_few_nunits_stage(type, ncg, nmt); + constexpr int nstages = mmq_few_nstages(type); + constexpr bool unaligned = mmq_few_unaligned(type, nunits_stage); + constexpr int nchunks = mmq_few_nchunks(type, nunits_stage); + constexpr int sram_stride = mmq_few_sram_stride(type, nunits_stage); + constexpr int nbytes_stage = nrows*sram_stride; + constexpr int ncopies = nrows*nchunks; + static_assert(unit_size > 0, "unsupported type"); + + extern __shared__ char data_mmq_few[]; + char * ring = data_mmq_few + threadIdx.y*nstages*nbytes_stage; + + const int row0 = blockIdx.x*nrows; + const int channel_dst = blockIdx.y; + const int sample_dst = blockIdx.z; + const int nchannels = gridDim.y; + + x += int64_t(sample_dst/sample_ratio)*stride_sample_x + int64_t(channel_dst/channel_ratio)*stride_channel_x; + dst += int64_t(sample_dst)*stride_sample_dst + int64_t(channel_dst)*stride_channel_dst; + const int4 * GGML_CUDA_RESTRICT y = y_ptr + (int64_t(sample_dst)*nchannels + channel_dst)*nunits*ncg*MMQ_FEW_Q8_1_INT4; + + // No bounds check for the units, the gate makes nunits a multiple of nunits_stage. + // The extra bytes of unaligned data are inside the allocation, which ggml aligns and pads to 128 bytes. + auto load_stage = [&](const int stage, const int iu0) { + const unsigned int dst_stage = ggml_cuda_cvta_generic_to_shared(ring + stage*nbytes_stage); +#pragma unroll + for (int c0 = 0; c0 < ncopies; c0 += warp_size) { + const int c = c0 + threadIdx.x; + const int i = c / nchunks; + const int ic = c % nchunks; + if (ncopies % warp_size == 0 || c < ncopies) { + const int row = min(row0 + i, nrows_x - 1); + const char * src = x + int64_t(row)*stride_row_x + int64_t(iu0)*unit_size; + if constexpr (unaligned) { + const int off = (uintptr_t) src % 16; + if (16*ic < off + nunits_stage*unit_size) { + cp_async_cg_16<0>(dst_stage + i*sram_stride + 16*ic, src - off + 16*ic); + } + } else { + cp_async_cg_16<0>(dst_stage + i*sram_stride + 16*ic, src + 16*ic); + } + } + } + }; + + // offsets of the rows of this lane from their 16 byte boundary + int off_row[2*nmt] = {0}; + if constexpr (unaligned) { +#pragma unroll + for (int r = 0; r < 2*nmt; ++r) { + const int row = min(row0 + int(threadIdx.x)/4 + 8*r, nrows_x - 1); + off_row[r] = (uintptr_t) (x + int64_t(row)*stride_row_x) % 16; + } + } + + // Only weights, which no preceding kernel writes, are read before the PDL wait. + if (!x_is_constant) { + ggml_cuda_pdl_sync(); + } + +#if __CUDA_ARCH__ == GGML_CUDA_CC_DGX_SPARK + // DGX Spark DRAM serves the short row pieces of the stage copies slowly, so the rows of the block are first prefetched into L2 in address order. + { + const int tid = threadIdx.y*warp_size + threadIdx.x; + const int nrows_blk = min(nrows, nrows_x - row0); + const int64_t row_bytes = int64_t(nunits)*unit_size; + const bool contig = stride_row_x == row_bytes; + for (int ir = 0; ir < (contig ? 1 : nrows_blk); ++ir) { + const uintptr_t p0 = (uintptr_t) (x + int64_t(row0 + ir)*stride_row_x); + const uintptr_t p1 = p0 + (contig ? nrows_blk*row_bytes : row_bytes); + for (uintptr_t p = (p0 & ~uintptr_t(127)) + 128*tid; p < p1; p += 128*nwarps*warp_size) { + ggml_cuda_prefetch_l2((const void *) p); + } + } + } +#endif // __CUDA_ARCH__ == GGML_CUDA_CC_DGX_SPARK + + const int iu_step = nunits_stage*nwarps; + +#pragma unroll + for (int stage = 0; stage < nstages - 1; ++stage) { + const int iu0 = nunits_stage*threadIdx.y + stage*iu_step; + if (iu0 < nunits) { + load_stage(stage, iu0); + } + cp_async_commit_group(); + } + if (x_is_constant) { + ggml_cuda_pdl_sync(); + } + + auto load_y = [&](mmq_few_y_frag (&yf)[ncg], const int iu) { +#pragma unroll + for (int c = 0; c < ncg; ++c) { + mmq_few_load_y(yf[c], y + (int64_t(iu)*ncg + c)*MMQ_FEW_Q8_1_INT4); + } + }; + + // The next src1 fragments are loaded right after the MMAs that use the current ones, an earlier load is slower. + mmq_few_y_frag yf[nunits_stage][ncg]; + if (nunits_stage*threadIdx.y < nunits) { +#pragma unroll + for (int k = 0; k < nunits_stage; ++k) { + load_y(yf[k], nunits_stage*threadIdx.y + k); + } + } + + float sum[nmt][ncg][4] = {{{0.0f}}}; + + for (int iter = 0, iu0 = nunits_stage*threadIdx.y; iu0 < nunits; ++iter, iu0 += iu_step) { + const int iu_load = iu0 + (nstages - 1)*iu_step; + if (iu_load < nunits) { + load_stage((iter + nstages - 1) % nstages, iu_load); + } + cp_async_commit_group(); + + const bool has_next = iu0 + iu_step < nunits; + + cp_async_wait_group(); + ggml_cuda_syncwarp(); + + const char * xs = ring + (iter % nstages)*nbytes_stage; + const int off_stage = unaligned ? (iu0*unit_size) % 16 : 0; +#pragma unroll + for (int k = 0; k < nunits_stage; ++k) { +#pragma unroll + for (int m = 0; m < nmt; ++m) { + const char * const xr[2] = { + xs + (MMQ_FEW_NROWS*m + threadIdx.x/4 + 0)*sram_stride + (off_row[2*m + 0] + off_stage) % 16 + k*unit_size, + xs + (MMQ_FEW_NROWS*m + threadIdx.x/4 + 8)*sram_stride + (off_row[2*m + 1] + off_stage) % 16 + k*unit_size, + }; + mmq_few_vec_dot(xr, yf[k], sum[m]); + } + if (has_next) { + load_y(yf[k], iu0 + iu_step + k); + } + } + + ggml_cuda_syncwarp(); // the stage is overwritten in the next iteration + } + cp_async_wait_all(); + + ggml_cuda_pdl_lc(); + + // reduce the sums over the warps as one flat array + constexpr int nsum = nmt*ncg*4; + float * sum_flat = &sum[0][0][0]; + + float * buf_sum = (float *) data_mmq_few; + __syncthreads(); + if (threadIdx.y > 0) { +#pragma unroll + for (int i = 0; i < nsum; ++i) { + buf_sum[((threadIdx.y - 1)*nsum + i)*warp_size + threadIdx.x] = sum_flat[i]; + } + } + __syncthreads(); + if (threadIdx.y > 0) { + return; + } +#pragma unroll + for (int w = 0; w < nwarps - 1; ++w) { +#pragma unroll + for (int i = 0; i < nsum; ++i) { + sum_flat[i] += buf_sum[(w*nsum + i)*warp_size + threadIdx.x]; + } + } + +#pragma unroll + for (int m = 0; m < nmt; ++m) { +#pragma unroll + for (int c = 0; c < ncg; ++c) { +#pragma unroll + for (int l = 0; l < 4; ++l) { + const int i = row0 + MMQ_FEW_NROWS*m + tile<16, 8, int>::get_i(l); + const int j = c*MMQ_FEW_NCOLS + tile<16, 8, int>::get_j(l); + if (i < nrows_x && j < ncols_dst) { + dst[int64_t(j)*stride_col_dst + i] = sum[m][c][l]; + } + } + } + } +#else + GGML_UNUSED_VARS(x, y_ptr, dst, nunits, nrows_x, ncols_dst, stride_row_x, stride_col_dst, + channel_ratio, stride_channel_x, stride_channel_dst, + sample_ratio, stride_sample_x, stride_sample_dst, x_is_constant); + NO_DEVICE_CODE; +#endif // defined(CP_ASYNC_AVAILABLE) +} + +// ggml_is_constant (static in ggml.c) on the tensor that a view points to +static bool mmq_few_is_constant(const ggml_tensor * src0) { + const ggml_tensor * x_root = src0->view_src ? src0->view_src : src0; + return x_root->buffer && ggml_backend_buffer_get_usage(x_root->buffer) == GGML_BACKEND_BUFFER_USAGE_WEIGHTS && + !(x_root->flags & GGML_TENSOR_FLAG_PARAM); +} + +template +static void launch_mul_mat_q_few(const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst, + const int4 * src1_q, cudaStream_t stream) { + GGML_TENSOR_BINARY_OP_LOCALS; + + const int nunits = ne00 / QK_K; + GGML_ASSERT(nunits % mmq_few_nunits_stage(type, ncg, nmt) == 0); + + const int warp_size = ggml_cuda_info().devices[ggml_cuda_get_device()].warp_size; + const int nbytes_shared = mmq_few_nbytes_shared(type, ncg, nmt); + CUDA_SET_SHARED_MEMORY_LIMIT((mul_mat_q_few), nbytes_shared); + + const bool x_is_constant = mmq_few_is_constant(src0); + + const dim3 block_nums((ne01 + nmt*MMQ_FEW_NROWS - 1) / (nmt*MMQ_FEW_NROWS), ne12, ne13); + const dim3 block_dims(warp_size, MMQ_FEW_NWARPS, 1); + const ggml_cuda_kernel_launch_params params(block_nums, block_dims, nbytes_shared, stream); + ggml_cuda_kernel_launch(mul_mat_q_few, params, + (const char *) src0->data, src1_q, (float *) dst->data, + nunits, (int) ne01, (int) ne1, (int64_t) nb01, (int) (nb1/sizeof(float)), + (int) (ne12/ne02), (int64_t) nb02, (int) (nb2/sizeof(float)), + (int) (ne13/ne03), (int64_t) nb03, (int) (nb3/sizeof(float)), x_is_constant); +} + +static int mmq_few_ncg(const int64_t ncols) { + return (ncols + MMQ_FEW_NCOLS - 1) / MMQ_FEW_NCOLS; +} + +// With 2 column groups, 32-row tiles halve the src1 loads per src0 byte. With 3 column groups they spill registers. +static int mmq_few_nmt(const ggml_type type, const int ncg, const int64_t nrows_x) { + const int nsm = ggml_cuda_info().devices[ggml_cuda_get_device()].nsm; + return mmq_few_tall_supported(type) && ncg == 2 && nrows_x >= int64_t(nsm)*2*MMQ_FEW_NROWS ? 2 : 1; +} + +template +static void launch_mul_mat_q_few_tall(const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst, + const int4 * src1_q, cudaStream_t stream) { + if constexpr (mmq_few_tall_supported(type)) { + if (mmq_few_nmt(type, 2, src0->ne[1]) == 2) { + launch_mul_mat_q_few(src0, src1, dst, src1_q, stream); + return; + } + } + launch_mul_mat_q_few(src0, src1, dst, src1_q, stream); +} + +template +static void launch_mul_mat_q_few_switch_ncg(const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst, + const int4 * src1_q, const int ncg, cudaStream_t stream) { + switch (ncg) { + case 1: + launch_mul_mat_q_few(src0, src1, dst, src1_q, stream); + break; + case 2: + launch_mul_mat_q_few_tall(src0, src1, dst, src1_q, stream); + break; + case 3: + launch_mul_mat_q_few(src0, src1, dst, src1_q, stream); + break; + default: + GGML_ABORT("fatal error"); + } +} + +void ggml_cuda_mul_mat_q_few(ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { + GGML_TENSOR_BINARY_OP_LOCALS; + + GGML_ASSERT(src1->type == GGML_TYPE_F32); + GGML_ASSERT( dst->type == GGML_TYPE_F32); + GGML_ASSERT(nb00 == ggml_type_size(src0->type)); + GGML_ASSERT(nb10 == sizeof(float)); + GGML_ASSERT(nb0 == sizeof(float)); + GGML_ASSERT(ne11 <= MMQ_FEW_MAX_NCG*MMQ_FEW_NCOLS); + GGML_ASSERT(ne00 % QK_K == 0); + GGML_ASSERT(ne12 % ne02 == 0 && ne13 % ne03 == 0); + + cudaStream_t stream = ctx.stream(); + + const int nunits = ne00 / QK_K; + const int ncg = mmq_few_ncg(ne11); + + ggml_cuda_pool_alloc src1_q(ctx.pool(), int64_t(ne12)*ne13*nunits*ncg*MMQ_FEW_Q8_1_INT4); + quantize_mmq_few_q8_1_cuda((const float *) src1->data, src1_q.get(), mmq_few_ds_layout(src0->type), ne10, nb11/sizeof(float), nb12/sizeof(float), nb13/sizeof(float), + ne11, ne12, ne13, stream); + + switch (src0->type) { + case GGML_TYPE_Q8_0: + launch_mul_mat_q_few_switch_ncg(src0, src1, dst, src1_q.get(), ncg, stream); + break; + case GGML_TYPE_Q3_K: + launch_mul_mat_q_few_switch_ncg(src0, src1, dst, src1_q.get(), ncg, stream); + break; + case GGML_TYPE_Q4_K: + launch_mul_mat_q_few_switch_ncg(src0, src1, dst, src1_q.get(), ncg, stream); + break; + case GGML_TYPE_Q5_K: + launch_mul_mat_q_few_switch_ncg(src0, src1, dst, src1_q.get(), ncg, stream); + break; + case GGML_TYPE_Q6_K: + launch_mul_mat_q_few_switch_ncg(src0, src1, dst, src1_q.get(), ncg, stream); + break; + case GGML_TYPE_IQ3_S: + launch_mul_mat_q_few_switch_ncg(src0, src1, dst, src1_q.get(), ncg, stream); + break; + case GGML_TYPE_IQ4_NL: + launch_mul_mat_q_few_switch_ncg(src0, src1, dst, src1_q.get(), ncg, stream); + break; + case GGML_TYPE_IQ4_XS: + launch_mul_mat_q_few_switch_ncg(src0, src1, dst, src1_q.get(), ncg, stream); + break; + default: + GGML_ABORT("unsupported type for few-row MMQ"); + } +} + +// Min. number of src1 columns for the few-row MMQ, 0 if it is not used for this type, GPU and row length. +static int ggml_cuda_mmq_few_min_ncols(const ggml_type type, const int cc, const int64_t ne00) { +#ifdef GGML_CUDA_FORCE_CUBLAS + return 0; +#endif // GGML_CUDA_FORCE_CUBLAS + + // NVIDIA Ampere to Blackwell + if (!ampere_mma_available(cc) || cc >= GGML_CUDA_CC_RUBIN) { + return 0; + } + // on DGX Spark the kernel needs its L2 prefetch, which is only in the DGX Spark device code + if (cc == GGML_CUDA_CC_DGX_SPARK && ggml_cuda_highest_compiled_arch(cc) != GGML_CUDA_CC_DGX_SPARK) { + return 0; + } + // with fewer than 8 units of 256 values per row the warps idle, MMVQ and MMQ are better there + if (mmq_few_unit_size(type) == 0 || ne00 < 8*QK_K) { + return 0; + } + if (cc == GGML_CUDA_CC_DGX_SPARK) { // tuned on DGX Spark GB10 + return type == GGML_TYPE_IQ4_XS || type == GGML_TYPE_IQ4_NL ? 8 : 2; + } + switch (type) { // all GPUs except DGX Spark, tuned on RTX 5090 + case GGML_TYPE_Q3_K: return 3; + case GGML_TYPE_Q5_K: return 3; + case GGML_TYPE_Q4_K: return 4; + case GGML_TYPE_IQ4_NL: return 4; + case GGML_TYPE_Q6_K: return 4; + case GGML_TYPE_Q8_0: return 5; + case GGML_TYPE_IQ3_S: return 5; + case GGML_TYPE_IQ4_XS: return 6; + default: return 0; + } +} + +bool ggml_cuda_should_use_mmq_few(const ggml_tensor * src0, const ggml_tensor * src1, const int cc) { + const int min_ncols = ggml_cuda_mmq_few_min_ncols(src0->type, cc, src0->ne[0]); + if (min_ncols == 0 || src1->ne[1] < min_ncols || src1->ne[1] > MMQ_FEW_MAX_NCG*MMQ_FEW_NCOLS) { + return false; + } + // the row must hold a whole number of ring stages of the launched kernel + const int ncg = mmq_few_ncg(src1->ne[1]); + const int nmt = mmq_few_nmt(src0->type, ncg, src0->ne[1]); + const int nunits_stage = mmq_few_nunits_stage(src0->type, ncg, nmt); + if (src0->ne[0] % (nunits_stage*QK_K) != 0) { + return false; + } + // unaligned data needs the alignment of the decoder loads + const int x_alignment = mmq_few_unaligned(src0->type, nunits_stage) ? (ggml_type_size(src0->type) % 4 == 0 ? 4 : 2) : 16; + return ggml_cuda_is_aligned(src0, x_alignment) && ggml_cuda_is_aligned(src1, 16); +} + +#else // !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA) + +void ggml_cuda_mul_mat_q_few(ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { + GGML_UNUSED_VARS(ctx, src0, src1, dst); + GGML_ABORT("few-row MMQ is not supported on HIP or MUSA"); +} + +bool ggml_cuda_should_use_mmq_few(const ggml_tensor * src0, const ggml_tensor * src1, const int cc) { + GGML_UNUSED_VARS(src0, src1, cc); + return false; +} + +#endif // !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA) diff --git a/ggml/src/ggml-cuda/mmq-few.cuh b/ggml/src/ggml-cuda/mmq-few.cuh new file mode 100644 index 000000000..016834998 --- /dev/null +++ b/ggml/src/ggml-cuda/mmq-few.cuh @@ -0,0 +1,7 @@ +#pragma once + +#include "common.cuh" + +bool ggml_cuda_should_use_mmq_few(const ggml_tensor * src0, const ggml_tensor * src1, int cc); + +void ggml_cuda_mul_mat_q_few(ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst); diff --git a/ggml/src/ggml-cuda/mmq-load-tiles.cuh b/ggml/src/ggml-cuda/mmq-load-tiles.cuh index 8b4c1e8b3..9a5a6a405 100644 --- a/ggml/src/ggml-cuda/mmq-load-tiles.cuh +++ b/ggml/src/ggml-cuda/mmq-load-tiles.cuh @@ -629,6 +629,13 @@ template static __device__ __forceinline_ } } +// The 6-bit scales 4*ksc .. 4*ksc + 3 of a q3_K block, minus 32, as int8. +static __device__ __forceinline__ int unpack_scales_q3_K(const uint8_t * scales, const int ksc) { + const int sc_low = (get_int_b2(scales, ksc % 2) >> (4 * (ksc / 2))) & 0x0F0F0F0F; + const int sc_high = ((get_int_b2(scales, 2) >> (2 * ksc)) << 4) & 0x30303030; + return __vsub4(sc_low | sc_high, 0x20202020); +} + template static __device__ __forceinline__ void ggml_cuda_mmq_load_tiles_q3_K( const char * __restrict__ x, const char * __restrict__ x_gate, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) { GGML_UNUSED(x_gate); @@ -695,15 +702,7 @@ template static __device__ __forceinline_ const int ksc = threadIdx.x % 4; - const int ksc_low = ksc % (QI3_K/8); - const int shift_low = 4 * (ksc / (QI3_K/8)); - const int sc_low = (get_int_b2(bxi->scales, ksc_low) >> shift_low) & 0x0F0F0F0F; - - const int ksc_high = QI3_K/8; - const int shift_high = 2 * ksc; - const int sc_high = ((get_int_b2(bxi->scales, ksc_high) >> shift_high) << 4) & 0x30303030; - - const int sc = __vsub4(sc_low | sc_high, 0x20202020); + const int sc = unpack_scales_q3_K(bxi->scales, ksc); #if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) const int8_t * sc8 = (const int8_t *) ≻ diff --git a/ggml/src/ggml-cuda/mmvq.cu b/ggml/src/ggml-cuda/mmvq.cu index b57b26bee..c6d895db4 100644 --- a/ggml/src/ggml-cuda/mmvq.cu +++ b/ggml/src/ggml-cuda/mmvq.cu @@ -29,10 +29,6 @@ static constexpr __host__ __device__ bool mmvq_should_prefetch(ggml_type type) { return false; } } - -static __device__ __forceinline__ void mmvq_prefetch_l2(const void * p) { - asm volatile("prefetch.global.L2 [%0];" :: "l"(p)); -} #endif typedef float (*vec_dot_q_cuda_t)(const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & kbx, const int & iqs); @@ -757,10 +753,10 @@ static __global__ void mul_mat_vec_q( #pragma unroll for (int i = 0; i < rows_per_cuda_block; ++i) { const size_t off = (size_t)(kbx_offset + i*stride_row_x + kbx_pf) * ggml_cuda_type_traits::bs; - mmvq_prefetch_l2((const char *) vx + off); + ggml_cuda_prefetch_l2((const char *) vx + off); if constexpr (has_fusion) { if (use_gate) { - mmvq_prefetch_l2((const char *) vgate + off); + ggml_cuda_prefetch_l2((const char *) vgate + off); } } } diff --git a/ggml/src/ggml-cuda/quantize.cu b/ggml/src/ggml-cuda/quantize.cu index 4ec8b774b..2acf1f6d0 100644 --- a/ggml/src/ggml-cuda/quantize.cu +++ b/ggml/src/ggml-cuda/quantize.cu @@ -600,6 +600,81 @@ void quantize_mmq_q8_1_cuda( } } +// Each thread quantizes 4 values of one column, see quantize.cuh for the layout. +template +static __global__ void quantize_mmq_few_q8_1( + const float * x_ptr, void * vy_ptr, const int64_t ne00, const int64_t s01, const int64_t s02, const int64_t s03, + const int ne1, const int ne2) { + ggml_cuda_pdl_lc(); + const float * GGML_CUDA_RESTRICT x = x_ptr; + int4 * GGML_CUDA_RESTRICT vy = (int4 *) vy_ptr; + const int64_t i0 = ((int64_t)blockDim.x*blockIdx.y + threadIdx.x)*4; + + if (i0 >= ne00) { + return; + } + ggml_cuda_pdl_sync(); + + const int64_t i2 = blockIdx.z % ne2; + const int64_t i3 = blockIdx.z / ne2; + const float4 xi = (int) blockIdx.x < ne1 ? *(const float4 *) (x + i3*s03 + i2*s02 + blockIdx.x*s01 + i0) : make_float4(0.0f, 0.0f, 0.0f, 0.0f); + + // the 32 values of a sub-block are held by 8 threads + float amax = fmaxf(fmaxf(fabsf(xi.x), fabsf(xi.y)), fmaxf(fabsf(xi.z), fabsf(xi.w))); + amax = warp_reduce_max<8>(amax); + + const float d_inv = 127.0f / amax; + char4 q; + q.x = roundf(xi.x*d_inv); + q.y = roundf(xi.y*d_inv); + q.z = roundf(xi.z*d_inv); + q.w = roundf(xi.w*d_inv); + const float d = 1.0f / d_inv; + + // as in quantize_mmq_q8_1: the sum of the quantized values; the int warp_reduce_sum reduces the whole warp on Ampere+ + [[maybe_unused]] float sum = 0.0f; + if constexpr (ds_layout != MMQ_Q8_1_DS_LAYOUT_D4) { + sum = d*warp_reduce_sum<8>(float(q.x + q.y + q.z + q.w)); + } + + const int s = i0 % QK_K / QK8_1; // sub-block + const int k = i0 % QK8_1; // first value in sub-block + const int g = blockIdx.x % MMQ_FEW_NCOLS; // column in group + int4 * yg = vy + ((blockIdx.z*(ne00/QK_K) + i0/QK_K)*(gridDim.x/MMQ_FEW_NCOLS) + blockIdx.x/MMQ_FEW_NCOLS)*MMQ_FEW_Q8_1_INT4; + ((int *) (yg + (s/2)*WARP_SIZE + 4*g + (k % 16)/4))[2*(s % 2) + k/16] = *(const int *) &q; + if (k == 0) { + int4 * ds = yg + 4*WARP_SIZE + 4*(g/2) + s/2; + if constexpr (ds_layout == MMQ_Q8_1_DS_LAYOUT_D4) { + ((float *) ds)[2*(s % 2) + g % 2] = d; + } else { + ((half2 *) ds)[2*(s % 2) + g % 2] = make_half2(d, sum); + } + } +} + +void quantize_mmq_few_q8_1_cuda( + const float * x, void * vy, const mmq_q8_1_ds_layout ds_layout, const int64_t ne00, const int64_t s01, const int64_t s02, const int64_t s03, + const int64_t ne1, const int64_t ne2, const int64_t ne3, cudaStream_t stream) { + GGML_ASSERT(ne00 % QK_K == 0); + + const int64_t block_num_y = (ne00 + 4*CUDA_QUANTIZE_BLOCK_SIZE_MMQ - 1) / (4*CUDA_QUANTIZE_BLOCK_SIZE_MMQ); + const dim3 num_blocks(GGML_PAD(ne1, MMQ_FEW_NCOLS), block_num_y, ne2*ne3); + const dim3 block_size(CUDA_QUANTIZE_BLOCK_SIZE_MMQ, 1, 1); + const ggml_cuda_kernel_launch_params launch_params = ggml_cuda_kernel_launch_params(num_blocks, block_size, 0, stream); + switch (ds_layout) { + case MMQ_Q8_1_DS_LAYOUT_D4: + ggml_cuda_kernel_launch(quantize_mmq_few_q8_1, launch_params, + x, vy, ne00, s01, s02, s03, (int) ne1, (int) ne2); + break; + case MMQ_Q8_1_DS_LAYOUT_DS4: + ggml_cuda_kernel_launch(quantize_mmq_few_q8_1, launch_params, + x, vy, ne00, s01, s02, s03, (int) ne1, (int) ne2); + break; + default: + GGML_ABORT("fatal error"); + } +} + // scatter=true reuses the quant kernel: grid over tokens, ids = inverse map (token slot -> compact row) void quantize_scatter_mmq_q8_1_cuda( const float * x, const int32_t * ids_src1_inv, void * vy, const ggml_type type_src0, diff --git a/ggml/src/ggml-cuda/quantize.cuh b/ggml/src/ggml-cuda/quantize.cuh index 9ae72ee44..11720ef93 100644 --- a/ggml/src/ggml-cuda/quantize.cuh +++ b/ggml/src/ggml-cuda/quantize.cuh @@ -21,6 +21,17 @@ void quantize_mmq_q8_1_cuda( ggml_type type_src0, int64_t ne00, int64_t s01, int64_t s02, int64_t s03, int64_t ne0, int64_t ne1, int64_t ne2, int64_t ne3, cudaStream_t stream); +// src1 of the few-row MMQ in the layout of the MMA fragments. Per channel, 256 values and group of MMQ_FEW_NCOLS columns: +// qs: int4[4][32], [j][lane] holds the B fragments of sub-blocks 2j and 2j+1 of lane (g, t): the values [4t, 4t + 4) and [16 + 4t, 16 + 4t + 4) of column g. +// ds: int4[4][4], [t][j] holds d (D4, float) or (d, sum) (DS4, half2) of the columns 2t and 2t+1 for sub-block 2j, then 2j+1. +// The column groups of a unit are adjacent, columns beyond ne1 are zero. +static constexpr int MMQ_FEW_NCOLS = 8; // src1 columns per column group +static constexpr int MMQ_FEW_Q8_1_INT4 = 4*WARP_SIZE + 4*4; // int4 per 256 values and column group + +void quantize_mmq_few_q8_1_cuda( + const float * x, void * vy, mmq_q8_1_ds_layout ds_layout, int64_t ne00, int64_t s01, int64_t s02, int64_t s03, + int64_t ne1, int64_t ne2, int64_t ne3, cudaStream_t stream); + void quantize_mmq_fp4_cuda(const float * x, const int32_t * ids, void * vy, diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cpp index b783cccb5..4d7fa1188 100644 --- a/tests/test-backend-ops.cpp +++ b/tests/test-backend-ops.cpp @@ -5474,9 +5474,14 @@ struct test_mul_mat : public test_case { const int64_t m_v; // rows of a in memory, the batches of a are strided for m_v > m, no view for m_v == 0 const int64_t pad; // bytes after the m_v rows of each batch of a, so nb[2] of a is not a multiple of nb[1] const size_t offset_a; // byte offset of the a view when k_v > k or m_v > m + const bool weights; // a in a weights buffer (CUDA with PDL reads weights early) std::string vars() override { - return VARS_TO_STR14(type_a, type_b, m, n, k, bs, nr, per, k_v, o, src_overlap, m_v, pad, offset_a); + std::string s = VARS_TO_STR14(type_a, type_b, m, n, k, bs, nr, per, k_v, o, src_overlap, m_v, pad, offset_a); + if (weights) { + s += ",weights=1"; + } + return s; } double max_nmse_err() override { @@ -5510,10 +5515,21 @@ struct test_mul_mat : public test_case { std::array bs = {10, 10}, std::array nr = {2, 2}, std::array per = {0, 1, 2, 3}, - int64_t k_v = 0, uint32_t o = 1, bool src_overlap = false, int64_t m_v = 0, int64_t pad = 0, size_t offset_a = 0) - : type_a(type_a), type_b(type_b), m(m), n(n), k(k), bs(bs), nr(nr), per(per), k_v(k_v), o(o), src_overlap(src_overlap), m_v(m_v), pad(pad), offset_a(offset_a) {} + int64_t k_v = 0, uint32_t o = 1, bool src_overlap = false, int64_t m_v = 0, int64_t pad = 0, size_t offset_a = 0, bool weights = false) + : type_a(type_a), type_b(type_b), m(m), n(n), k(k), bs(bs), nr(nr), per(per), k_v(k_v), o(o), src_overlap(src_overlap), m_v(m_v), pad(pad), offset_a(offset_a), weights(weights) {} + + bool use_weight_context() override { + return weights; + } ggml_tensor * build_graph(ggml_context * ctx) override { + return build_graph(ctx, ctx); + } + + ggml_tensor * build_graph(ggml_context * ctx, ggml_context * ctx_weights) override { + ggml_context * ctx_a = weights ? ctx_weights : ctx; + GGML_ASSERT(ctx_a); + // C^T = A * B^T: (k, m) * (k, n) => (m, n) ggml_tensor * a; ggml_tensor * b; @@ -5530,7 +5546,7 @@ struct test_mul_mat : public test_case { const int64_t ne_a[4] = {k, m, bs[0], bs[1]}; const int64_t ne_b[4] = {k, n, bs[0]*nr[0], bs[1]*nr[1]}; - a = ggml_new_tensor_4d(ctx, type_a, ne_a[per[0]], ne_a[per[1]], ne_a[per[2]], ne_a[per[3]]); + a = ggml_new_tensor_4d(ctx_a, type_a, ne_a[per[0]], ne_a[per[1]], ne_a[per[2]], ne_a[per[3]]); b = ggml_new_tensor_4d(ctx, type_b, ne_b[per[0]], ne_b[per[1]], ne_b[per[2]], ne_b[per[3]]); if (!ggml_is_quantized(type_a)) { if (bs[1] == 1 && nr[1] == 1) { @@ -5549,6 +5565,7 @@ struct test_mul_mat : public test_case { GGML_ASSERT(type_a == type_b); GGML_ASSERT(k_v == 0); GGML_ASSERT(m_v == 0); + GGML_ASSERT(ctx_a == ctx); // not handled: b is a view of the same tensor // a and b are interleaved views of the same tensor: (e.g. fused QKV in MiniMax-01) ggml_tensor * base = ggml_new_tensor_4d(ctx, type_a, 2*k, std::max(m, n), bs[0]*nr[0], bs[1]*nr[1]); @@ -5561,7 +5578,7 @@ struct test_mul_mat : public test_case { } else { const int64_t k_physical = k_v == 0 ? k : k_v; const int64_t m_physical = m_v == 0 ? m : m_v + (pad != 0); - a = ggml_new_tensor_4d(ctx, type_a, k_physical, m_physical, bs[0], bs[1]); + a = ggml_new_tensor_4d(ctx_a, type_a, k_physical, m_physical, bs[0], bs[1]); b = ggml_new_tensor_4d(ctx, type_b, k_physical, n, bs[0]*nr[0], bs[1]*nr[1]); if (!ggml_is_quantized(type_a)) { @@ -5616,8 +5633,8 @@ struct test_mul_mat_hadamard : public test_mul_mat { GGML_ASSERT(type_a == GGML_TYPE_F32); } - ggml_tensor * build_graph(ggml_context * ctx) override { - ggml_tensor * out = test_mul_mat::build_graph(ctx); + ggml_tensor * build_graph(ggml_context * ctx, ggml_context * ctx_weights) override { + ggml_tensor * out = test_mul_mat::build_graph(ctx, ctx_weights); // Find the mul_mat op in the graph and set the hint for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != NULL; t = ggml_get_next_tensor(ctx, t)) { if (t->op == GGML_OP_MUL_MAT) { @@ -5666,8 +5683,8 @@ struct test_mul_mat_w4a8 : public test_mul_mat { std::array bs = {1, 1}, std::array nr = {1, 1}) : test_mul_mat(type_a, type_b, m, n, k, bs, nr) {} - ggml_tensor * build_graph(ggml_context * ctx) override { - ggml_tensor * out = test_mul_mat::build_graph(ctx); + ggml_tensor * build_graph(ggml_context * ctx, ggml_context * ctx_weights) override { + ggml_tensor * out = test_mul_mat::build_graph(ctx, ctx_weights); for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != NULL; t = ggml_get_next_tensor(ctx, t)) { if (t->op == GGML_OP_MUL_MAT) { ggml_prec_set_src(t, GGML_PREC_Q8, 1); @@ -5688,8 +5705,8 @@ struct test_mul_mat_prec_src1_bf16 : public test_mul_mat { std::array bs = {1, 1}, std::array nr = {1, 1}) : test_mul_mat(type_a, GGML_TYPE_F32, m, n, k, bs, nr) {} - ggml_tensor * build_graph(ggml_context * ctx) override { - ggml_tensor * out = test_mul_mat::build_graph(ctx); + ggml_tensor * build_graph(ggml_context * ctx, ggml_context * ctx_weights) override { + ggml_tensor * out = test_mul_mat::build_graph(ctx, ctx_weights); for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != NULL; t = ggml_get_next_tensor(ctx, t)) { if (t->op == GGML_OP_MUL_MAT) { ggml_prec_set_src(t, GGML_PREC_BF16, 1); @@ -5721,8 +5738,8 @@ struct test_mul_mat_prec_src1_f32 : public test_mul_mat { double max_nmse_err() override { return 1e-10; } - ggml_tensor * build_graph(ggml_context * ctx) override { - ggml_tensor * out = test_mul_mat::build_graph(ctx); + ggml_tensor * build_graph(ggml_context * ctx, ggml_context * ctx_weights) override { + ggml_tensor * out = test_mul_mat::build_graph(ctx, ctx_weights); for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != NULL; t = ggml_get_next_tensor(ctx, t)) { if (t->op == GGML_OP_MUL_MAT) { ggml_prec_set_src(t, GGML_PREC_F32, 1); @@ -11932,6 +11949,28 @@ static std::vector> make_test_cases_eval() { test_cases.emplace_back(new test_mul_mat_pos(type_a, GGML_TYPE_F32, 1000, n, 1024, {1, 1}, {1, 1})); } } + // CUDA few-row MMQ: 1-3 column groups, 32-row tiles (6170 rows), K = 18 units (uneven over the warps) and 9 units (one unit per ring stage) + for (ggml_type type_a : {GGML_TYPE_Q8_0, GGML_TYPE_Q3_K, GGML_TYPE_Q4_K, GGML_TYPE_Q5_K, GGML_TYPE_Q6_K, + GGML_TYPE_IQ3_S, GGML_TYPE_IQ4_NL, GGML_TYPE_IQ4_XS}) { + for (int64_t k : {4608, 2304}) { + test_cases.emplace_back(new test_mul_mat(type_a, GGML_TYPE_F32, 1000, 8, k, {1, 1}, {1, 1})); + test_cases.emplace_back(new test_mul_mat(type_a, GGML_TYPE_F32, 6170, 13, k, {1, 1}, {1, 1})); + test_cases.emplace_back(new test_mul_mat(type_a, GGML_TYPE_F32, 1000, 21, k, {1, 1}, {1, 1})); + } + } + // permuted and broadcast src0/src1 for the 2 byte aligned types + test_cases.emplace_back(new test_mul_mat(GGML_TYPE_Q6_K, GGML_TYPE_F32, 64, 13, 2304, {2, 3}, {1, 1}, {0, 2, 1, 3})); + test_cases.emplace_back(new test_mul_mat(GGML_TYPE_Q6_K, GGML_TYPE_F32, 63, 5, 2304, {3, 2}, {2, 2})); + test_cases.emplace_back(new test_mul_mat(GGML_TYPE_Q3_K, GGML_TYPE_F32, 64, 13, 4608, {2, 3}, {1, 1}, {0, 2, 1, 3})); + test_cases.emplace_back(new test_mul_mat(GGML_TYPE_IQ3_S, GGML_TYPE_F32, 63, 5, 4608, {3, 2}, {2, 2})); + // partial column group, broadcast, padded rows, src1 in [0, 1], src0 in a weights buffer + test_cases.emplace_back(new test_mul_mat(GGML_TYPE_Q4_K, GGML_TYPE_F32, 1000, 5, 4608, {1, 1}, {1, 1})); + test_cases.emplace_back(new test_mul_mat(GGML_TYPE_Q4_K, GGML_TYPE_F32, 64, 13, 4608, {3, 2}, {2, 2})); + test_cases.emplace_back(new test_mul_mat(GGML_TYPE_Q4_K, GGML_TYPE_F32, 1000, 13, 4608, {1, 1}, {1, 1}, {0, 1, 2, 3}, 5120)); + test_cases.emplace_back(new test_mul_mat_pos(GGML_TYPE_Q4_K, GGML_TYPE_F32, 1000, 13, 4608, {1, 1}, {1, 1})); + test_cases.emplace_back(new test_mul_mat_pos(GGML_TYPE_Q5_K, GGML_TYPE_F32, 1000, 13, 4608, {1, 1}, {1, 1})); + test_cases.emplace_back(new test_mul_mat(GGML_TYPE_Q4_K, GGML_TYPE_F32, 6170, 13, 4608, {1, 1}, {1, 1}, {0, 1, 2, 3}, 0, 1, false, 0, 0, 0, true)); + test_cases.emplace_back(new test_mul_mat(GGML_TYPE_Q6_K, GGML_TYPE_F32, 1000, 21, 4608, {1, 1}, {1, 1}, {0, 1, 2, 3}, 0, 1, false, 0, 0, 0, true)); #if 0 { @@ -13261,6 +13300,8 @@ static std::vector> make_test_cases_eval() { test_cases.emplace_back(new test_mul_mat_fusion_w4a8(GGML_TYPE_NVFP4, GGML_GLU_OP_SWIGLU, 150, 2560, 256, true, 16, 8, false)); test_cases.emplace_back(new test_mul_mat_fusion_w4a8(GGML_TYPE_NVFP4, GGML_GLU_OP_SWIGLU, 150, 40960, 256, false, 1, 1, false, false, true, true, { 1, 1 })); test_cases.emplace_back(new test_mul_mat_fusion_w4a8(GGML_TYPE_NVFP4, GGML_GLU_OP_SWIGLU, 150, 2560, 256, true, 16, 8, false, false, true, true)); + // K = 2048: few-row MMQ, where enabled, takes the gate and up matmuls instead of the fused MMQ path + test_cases.emplace_back(new test_mul_mat_fusion(GGML_TYPE_Q4_K, GGML_GLU_OP_SWIGLU, 9, 128, 2048, false, 1, 1, false, false, true, false, { 1, 1 })); for (auto gate : {GATING_FUNC_SOFTMAX, GATING_FUNC_SIGMOID, GATING_FUNC_SOFTMAX_WEIGHT, GATING_FUNC_SQRT_SOFTPLUS}) { for (bool with_norm : {false, true}) {