opencl: fold GQA head for f16 FA for better efficiency (#32631)
Assisted-by: Codex Co-authored-by: Li He <lih@qti.qualcomm.com> Co-authored-by: Hongqiang Wang <wangh@qti.qualcomm.com>
This commit is contained in:
2 files changed
+31
-12
No files matched your search
@@ -23128,8 +23128,17 @@ static void ggml_cl_fa_kq_p8_dispatch(
|
||||
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(int), &D_B));
|
||||
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(int), &p_pitch));
|
||||
|
||||
const size_t n_tiles = (N + 31) / 32;
|
||||
size_t gws[3] = {64, n_tiles, (size_t)(M / 64) * D_B};
|
||||
// Pack query heads of each KV group into a short batch's tile.
|
||||
static const bool fold_enabled = [] {
|
||||
const char * e = getenv("GGML_OPENCL_FA_KQ_P8_FOLD");
|
||||
return !e || e[0] != '0';
|
||||
}();
|
||||
const int gqa = D_B / D_A;
|
||||
const int fold = fold_enabled && gqa > 1 && N < 32 && gqa <= INT_MAX / N ? gqa : 1;
|
||||
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(int), &fold));
|
||||
|
||||
const size_t n_tiles = ((size_t) N * fold + 31) / 32;
|
||||
size_t gws[3] = {64, n_tiles, (size_t)(M / 64) * (fold > 1 ? D_A : D_B)};
|
||||
|
||||
const bool two_tiles = n_tiles % 2 == 0 && backend_ctx->get_kernel_workgroup_size(kernel) >= 128;
|
||||
size_t lws[3] = {64, two_tiles ? (size_t) 2 : (size_t) 1, 1};
|
||||
|
||||
@@ -119,7 +119,8 @@ __kernel void mul_mm_f16_f32_kq_p8(
|
||||
int N,
|
||||
int D_A,
|
||||
int D_B,
|
||||
int p_pitch // bytes per P row, >= M, multiple of 32
|
||||
int p_pitch, // bytes per P row, >= M, multiple of 32
|
||||
int gqa_fold
|
||||
) {
|
||||
|
||||
// Match the n-tile-first grid in ggml_cl_fa_kq_p8_dispatch.
|
||||
@@ -135,7 +136,8 @@ __kernel void mul_mm_f16_f32_kq_p8(
|
||||
|
||||
const uint col = block_id_m * TILESIZE_M;
|
||||
const uint row = block_id_n * TILESIZE_N;
|
||||
const uint depth_A = block_id_d / (D_B/D_A);
|
||||
const uint fold = (uint)gqa_fold;
|
||||
const uint depth_A = fold > 1 ? block_id_d : block_id_d / (D_B/D_A);
|
||||
const uint depth_B = block_id_d;
|
||||
|
||||
int line_stride_matrix_A_in_bytes = K * D_A * 2;
|
||||
@@ -158,11 +160,18 @@ __kernel void mul_mm_f16_f32_kq_p8(
|
||||
|
||||
__local float matrix_B_local[1024];
|
||||
|
||||
// Clamp Q loads in the last tile; columns past N are not stored.
|
||||
const uint c0 = min(row + b_globalOffsetInWords_xy.y, (uint)N - 1);
|
||||
const uint c1 = min(row + b_globalOffsetInWords_xy.y + 16, (uint)N - 1);
|
||||
const uint b_globalOffsetInWords00 = c0*strideBinElements + depth_B*K + b_globalOffsetInWords_xy.x;
|
||||
const uint b_globalOffsetInWords16 = c1*strideBinElements + depth_B*K + b_globalOffsetInWords_xy.x;
|
||||
// Folded columns enumerate (query, head within the KV group).
|
||||
const uint c0 = min(row + b_globalOffsetInWords_xy.y, (uint)N * fold - 1);
|
||||
const uint c1 = min(row + b_globalOffsetInWords_xy.y + 16, (uint)N * fold - 1);
|
||||
uint b_globalOffsetInWords00;
|
||||
uint b_globalOffsetInWords16;
|
||||
if (fold > 1) {
|
||||
b_globalOffsetInWords00 = (c0/fold)*strideBinElements + (depth_B*fold + c0%fold)*K + b_globalOffsetInWords_xy.x;
|
||||
b_globalOffsetInWords16 = (c1/fold)*strideBinElements + (depth_B*fold + c1%fold)*K + b_globalOffsetInWords_xy.x;
|
||||
} else {
|
||||
b_globalOffsetInWords00 = c0*strideBinElements + depth_B*K + b_globalOffsetInWords_xy.x;
|
||||
b_globalOffsetInWords16 = c1*strideBinElements + depth_B*K + b_globalOffsetInWords_xy.x;
|
||||
}
|
||||
|
||||
for (uint step=0; step < K; step+=TILESIZE_K) {
|
||||
regA = mm_load_a(matrix_A, subMatrixAStartInElements, line_stride_matrix_A_in_bytes);
|
||||
@@ -199,9 +208,10 @@ __kernel void mul_mm_f16_f32_kq_p8(
|
||||
if (lane < 32) {
|
||||
const int j = lane >> 1;
|
||||
const int b = lane & 1;
|
||||
const uint n = row + half_*16 + j;
|
||||
const uint h = depth_B;
|
||||
if (n < (uint)N) {
|
||||
const uint c = row + half_*16 + j;
|
||||
const uint n = c / fold;
|
||||
const uint h = depth_B*fold + c % fold;
|
||||
if (c < (uint)N * fold) {
|
||||
float slope = 1.0f;
|
||||
if (max_bias > 0.0f) {
|
||||
const float base = (int)h < n_head_log2 ? m0 : m1;
|
||||
|
||||
Reference in new issue
Block a user