Lumpiasty/llama.cpp
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

vulkan: handle larger batch sizes (>4) efficiently for IQ3_S mat-vec (#27449)

Browse Source 
* vulkan: handle larger batch sizes (>4) efficiently for IQ3_S mat-vec when NUM_COLS > 4. 5x perf at n=8

Assisted-by: Claude Opus 5

* adds 2 cases per quant type at `k=16*256` to the `all_types` mat-vec sweep

---------

Co-authored-by: Marshall <assistant@llama.cpp>
This commit is contained in:
Laurent Zuijdwijk
2026-09-02 09:14:52 +03:00
committed by GitHub
co-authored by Marshall
parent 56dd8150cc
commit ba8818cbf3
2 changed files with 22 additions and 15 deletions
Download Patch File Download Diff File Expand all files Collapse all files
ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq3_s.comp
+18 -15
View File
@@ -7,7 +7,14 @@ layout(local_size_x_id = 0, local_size_y = 1, local_size_z = 1) in;
FLOAT_TYPE temp[NUM_COLS][NUM_ROWS];
void calc_superblock(const uint a_offset, const uint b_offset, const uint ib32, const uint i, const uint num_blocks_per_row, const uint first_row, const uint num_rows) {
// invocations per superblock. with many columns, 8 invocations need too many
// registers and spill, so use 16 to halve the per-invocation B working set
const uint TPB = NUM_COLS <= 4 ? 8 : 16;
const uint NL = 32 / TPB; // l steps per invocation
void calc_superblock(const uint a_offset, const uint b_offset, const uint itid, const uint i, const uint num_blocks_per_row, const uint first_row, const uint num_rows) {
const uint ib32 = itid / (TPB / 8);
const uint l0 = (itid % (TPB / 8)) * NL;
const uint y_idx = i * QUANT_K + 32 * ib32;
uint ibi = a_offset + first_row * num_blocks_per_row + i;
@@ -16,11 +23,8 @@ void calc_superblock(const uint a_offset, const uint b_offset, const uint ib32,
const uint scale = (data_a[ibi].scales[ib32/2] >> (4 * (ib32 & 1))) & 0xF;
const float dscale = d * (1 + 2 * scale);
const uint qh = data_a[ibi].qh[ib32];
FLOAT_TYPE sum[NUM_COLS];
[[unroll]] for (uint j = 0; j < NUM_COLS; ++j) {
sum[j] = 0.0;
}
[[unroll]] for (uint l = 0; l < 4; ++l) {
[[unroll]] for (uint ll = 0; ll < NL; ++ll) {
const uint l = l0 + ll;
const u8vec2 qs = unpack8(uint32_t(data_a_packed16[ibi].qs[4 * ib32 + l])).xy; // vec4 used due to #12147
const uint sign = data_a[ibi].signs[4 * ib32 + l];
const vec4 grid0 = vec4(unpack8(iq3s_grid[qs.x | ((qh << (8 - 2*l)) & 0x100)]));
@@ -30,7 +34,7 @@ void calc_superblock(const uint a_offset, const uint b_offset, const uint ib32,
const vec4 b0 = vec4(data_b_v4[(j*p.batch_stride_b + b_offset + y_idx) / 4 + 2*l + 0]);
const vec4 b4 = vec4(data_b_v4[(j*p.batch_stride_b + b_offset + y_idx) / 4 + 2*l + 1]);
sum[j] =
const FLOAT_TYPE sum =
fma(FLOAT_TYPE(b0.x), FLOAT_TYPE((sign & 1) != 0 ? -grid0.x : grid0.x),
fma(FLOAT_TYPE(b0.y), FLOAT_TYPE((sign & 2) != 0 ? -grid0.y : grid0.y),
fma(FLOAT_TYPE(b0.z), FLOAT_TYPE((sign & 4) != 0 ? -grid0.z : grid0.z),
@@ -39,12 +43,11 @@ void calc_superblock(const uint a_offset, const uint b_offset, const uint ib32,
fma(FLOAT_TYPE(b4.y), FLOAT_TYPE((sign & 32) != 0 ? -grid1.y : grid1.y),
fma(FLOAT_TYPE(b4.z), FLOAT_TYPE((sign & 64) != 0 ? -grid1.z : grid1.z),
fma(FLOAT_TYPE(b4.w), FLOAT_TYPE((sign & 128) != 0 ? -grid1.w : grid1.w),
sum[j]))))))));
FLOAT_TYPE(0.0)))))))));
temp[j][n] = fma(dscale, sum, temp[j][n]);
}
}
[[unroll]] for (uint j = 0; j < NUM_COLS; ++j) {
temp[j][n] = fma(dscale, sum[j], temp[j][n]);
}
ibi += num_blocks_per_row;
}
}
@@ -55,11 +58,11 @@ void compute_outputs(const uint32_t first_row, const uint32_t num_rows) {
const uint num_blocks_per_row = p.ncols / QUANT_K;
// 8 threads are used to process each block
const uint blocks_per_wg = gl_WorkGroupSize.x/8;
// TPB invocations are used to process each block
const uint blocks_per_wg = gl_WorkGroupSize.x/TPB;
const uint tid = gl_LocalInvocationID.x;
const uint itid = tid % 8; // 0...7
const uint ix = tid / 8;
const uint itid = tid % TPB;
const uint ix = tid / TPB;
[[unroll]] for (uint j = 0; j < NUM_COLS; ++j) {
[[unroll]] for (uint i = 0; i < NUM_ROWS; ++i) {
tests/test-backend-ops.cpp
+4
View File
@@ -9395,6 +9395,10 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() {
//test_cases.emplace_back(new test_mul_mat(type_a, GGML_TYPE_F32, 18, i, 32*256, { 1, 1}, {8, 1}));
//test_cases.emplace_back(new test_mul_mat(type_a, GGML_TYPE_F32, 19, i, 33*256, { 1, 1}, {1, 1}));
}
// mat-vec shaders split k across lanes and loop over the blocks in strides. k must be
// long enough that the loop wraps, else the stride is never exercised
test_cases.emplace_back(new test_mul_mat(type_a, GGML_TYPE_F32, 16, 1, 16*256, { 1, 1}, {1, 1}));
test_cases.emplace_back(new test_mul_mat(type_a, GGML_TYPE_F32, 16, 8, 16*256, { 1, 1}, {1, 1}));
}
test_cases.emplace_back(new test_mul_mat(GGML_TYPE_Q4_0, GGML_TYPE_F32, 2880, 32, 2880, {1, 1}, {1, 1}));