Lumpiasty/llama.cpp
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

sycl: reduce redundant work in Q4_K multi-column MMVQ (#27062)

Browse Source 
* sycl: Q4_K Weight unpack optimization and reuse between destination Columns

* sycl: Q4_K small N (N=2..4) + two output rows by subgroup reuse of activation between two rows.

* sycl: gate Q4_K two-row reuse for small N=2

* sycl: Fix on magic number now uses Q4_K_MMVQ_ROW_PAIR_MIN_NROWS=6272 for it, added tests for coverage around Q4_K_MMVQ_ROW_PAIR_MIN_NROWS with perf support to test Q4_K MUL_MAT, applied the same  reuse pattern to the activation as the weights.

Assisted-by: GPT-5.6 Sol

---------

Co-authored-by: RaulAbejonDelgado <raul.abejon.delgado@gmail.com>
This commit is contained in:
Eurekatic
2026-09-03 14:59:06 +08:00
committed by GitHub
co-authored by RaulAbejonDelgado
parent c61b98b875
commit 4aa6ffba25
3 changed files with 293 additions and 80 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/test-backend-ops.cpp
+41
View File
@@ -9435,6 +9435,21 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() {
test_cases.emplace_back(new test_mul_mat(type_a, GGML_TYPE_F32, 16, 8, 16*256, { 1, 1}, {1, 1}));
}
// Multi-column MMVQ coverage for the Q4_K weight-reuse path and a Q5_K control.
for (ggml_type type_a : { GGML_TYPE_Q4_K, GGML_TYPE_Q5_K }) {
for (int n = 1; n <= 8; ++n) {
test_cases.emplace_back(new test_mul_mat(type_a, GGML_TYPE_F32, 4096, n, 1024, { 1, 1 }, { 1, 1 }));
test_cases.emplace_back(new test_mul_mat(type_a, GGML_TYPE_F32, 1023, n, 4096, { 1, 1 }, { 1, 1 }));
}
}
// The SYCL backend picks between one and two output rows per subgroup by row count when there
// are two destination columns (Q4_K_MMVQ_ROW_PAIR_MIN_NROWS in ggml-sycl/mmvq.cpp). Cover both
// sides of that boundary, including an odd row count above it for the row-pair tail.
for (int64_t m : {6271, 6272, 6273}) {
test_cases.emplace_back(new test_mul_mat(GGML_TYPE_Q4_K, GGML_TYPE_F32, m, 2, 1024, { 1, 1 }, { 1, 1 }));
}
test_cases.emplace_back(new test_mul_mat(GGML_TYPE_Q4_0, GGML_TYPE_F32, 2880, 32, 2880, {1, 1}, {1, 1}));
test_cases.emplace_back(new test_mul_mat(GGML_TYPE_Q8_0, GGML_TYPE_F32, 2880, 32, 2880, {1, 1}, {1, 1}));
test_cases.emplace_back(new test_mul_mat(GGML_TYPE_MXFP4, GGML_TYPE_F32, 2880, 32, 2880, {1, 1}, {1, 1}));
@@ -10364,6 +10379,22 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() {
true, 16, 8, b, false, true, false));
}
// Fused row-pair coverage: minimum rows, an even pair, and an odd tail.
for (ggml_glu_op glu_op : { GGML_GLU_OP_SWIGLU, GGML_GLU_OP_GEGLU }) {
for (int64_t m_batch : { 2, 3, 4 }) {
for (int64_t rows : { 1, 2, 3 }) {
test_cases.emplace_back(new test_mul_mat_vec_fusion(GGML_TYPE_Q4_K, glu_op, m_batch, rows, 256,
false, 16, 8, false, false, true, false, { 1, 1 }));
}
}
}
// Both sides of the same row-count boundary as above, on the fused path.
for (int64_t rows : {6271, 6272, 6273}) {
test_cases.emplace_back(new test_mul_mat_vec_fusion(GGML_TYPE_Q4_K, GGML_GLU_OP_SWIGLU, 2, rows, 256,
false, 16, 8, false, false, true, false, { 1, 1 }));
}
for (auto gate : {GATING_FUNC_SOFTMAX, GATING_FUNC_SIGMOID, GATING_FUNC_SOFTMAX_WEIGHT, GATING_FUNC_SQRT_SOFTPLUS}) {
for (bool with_norm : {false, true}) {
for (bool bias_probs : {false, true}) {
@@ -10651,6 +10682,16 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_perf() {
}
}
// Q4_K multi-column mat-vec, at ffn_up/ffn_gate geometry (k = n_embd, m = n_ff): n sweeps the
// per-column specializations used for short prompts and speculative/MTP verify, and m brackets
// the row count at which the SYCL backend switches to two output rows per subgroup
// (Q4_K_MMVQ_ROW_PAIR_MIN_NROWS in ggml-sycl/mmvq.cpp), so both sides of it can be measured.
for (int64_t m : {4096, 6144, 6272, 14336}) {
for (int bs : {1, 2, 3, 4, 8}) {
test_cases.emplace_back(new test_mul_mat(GGML_TYPE_Q4_K, GGML_TYPE_F32, m, bs, 4096, {1, 1}, {1, 1}));
}
}
// qwen3-30b-a3b
for (int bs : {1, 4, 8, 32, 64, 128, 256, 512}) {
for (ggml_type type_a : {GGML_TYPE_F32, GGML_TYPE_F16, GGML_TYPE_Q4_0, GGML_TYPE_Q8_0, GGML_TYPE_Q4_K, GGML_TYPE_Q6_K, GGML_TYPE_IQ2_XS}) {