CUDA: Fuse MMVQ post-scale for NVFP4 (#24481)
* CUDA: Fuse MMVQ for NVFP4 and BS 1 TODO: 1. Add tests to test-backend-ops (did verify correctness manually for one model) 2. Reorder bias/scale once PRs for NVFP4 are merged/landed * Add dense MMVQ fusion as well Perf numbers on B4500. Note qwen35 is FP8->Q8 + ./scripts/compare-llama-bench.py -b master -c osimons/nvfp4_fuse_mmvq --tool llama-bench -i llama-bench.sqlite | Model | Test | t/s master | t/s osimons/nvfp4_fuse_mmvq | Speedup | |:-------------------------|:-------------|-------------:|------------------------------:|----------:| | qwen35moe 35B.A3B NVFP4 | tg128@d32768 | 150.15 | 156.29 | 1.04 | | qwen35moe 35B.A3B Q4_K_M | tg128@d32768 | 157.91 | 157.64 | 1.00 | Perf numbers on DGX Spark + ./scripts/compare-llama-bench.py -b master -c osimons/nvfp4_fuse_mmvq --tool llama-bench -i llama-bench.sqlite | Model | Test | t/s master | t/s osimons/nvfp4_fuse_mmvq | Speedup | |:-------------------------|:-------------|-------------:|------------------------------:|----------:| | qwen35moe 35B.A3B NVFP4 | tg128@d32768 | 58.31 | 59.69 | 1.02 | | qwen35moe 35B.A3B Q4_K_M | tg128@d32768 | 54.94 | 54.79 | 1.00 | * Add tests for the added fusion ops * Cleanup test-backend-ops * Cleanup ggml-cuda/mmvq 1. Unrestrict post-scale fusion 2. Rename names accordingly 3. Remove env variable to disable fusion * Merge old mul_mat patterns into the lane-based approach * Enable fusion for MoE in shared MMVQ * Restrict scale_view_nodes, enroll MM + ADD into lane-matcher * Refactor mmvq loads, still does not help non-nvfp4 kernels * Restrict scale-fusion to NVFP4 This is necessary, as the prolog is quite heavy in GEMV for some quants/model configs, leading to net perf regression. We should really be looking to refactor this such that ratio of prologue/hot-loop/epilogue is better on the hot-loop front: + ./scripts/compare-llama-bench.py -b master -c c1b9381d327e063cc846b46b59708444b66dc4d8 --tool llama-bench -i llama-bench.sqlite | CPU | Model | Test | t/s master | t/s c1b9381d3 | Speedup | |:----------------------------|:-------------------------|:-------------|-------------:|----------------:|----------:| | INTEL(R) XEON(R) GOLD 6542Y | gemma4 26B.A4B NVFP4 | tg128@d32768 | 151.70 | 154.32 | 1.02 | | INTEL(R) XEON(R) GOLD 6542Y | gemma4 26B.A4B Q4_K_M | tg128@d32768 | 187.95 | 185.73 | 0.99 | | INTEL(R) XEON(R) GOLD 6542Y | gpt-oss 20B MXFP4 MoE | tg128@d32768 | 304.62 | 300.69 | 0.99 | | INTEL(R) XEON(R) GOLD 6542Y | qwen35moe 35B.A3B NVFP4 | tg128@d32768 | 193.72 | 211.99 | 1.09 | | INTEL(R) XEON(R) GOLD 6542Y | qwen35moe 35B.A3B Q4_K_M | tg128@d32768 | 217.76 | 218.15 | 1.00 * Reorder scale & bias-add to adhere to #24331 * Restrict lane scale to NVFP4 Don't need to test unfused combinations * Cleanup * Merge single-lane mm-fusion helpers * Refactor and clean-up host-side fusion logic * Move gate_bias and scale into the same active-thread guard Latest perf numbers: B6000 build: 5b7d9f272 (9578) + ./scripts/compare-llama-bench.py -b master -c osimons/nvfp4_fuse_mmvq --tool llama-bench -i llama-bench.sqlite | CPU | Model | Test | t/s master | t/s osimons/nvfp4_fuse_mmvq | Speedup | |:----------------------------|:-------------------------|:-------------|-------------:|------------------------------:|----------:| | INTEL(R) XEON(R) GOLD 6542Y | gemma4 26B.A4B NVFP4 | tg128@d32768 | 151.79 | 154.10 | 1.02 | | INTEL(R) XEON(R) GOLD 6542Y | gemma4 26B.A4B Q4_K_M | tg128@d32768 | 187.90 | 187.27 | 1.00 | | INTEL(R) XEON(R) GOLD 6542Y | gpt-oss 20B MXFP4 MoE | tg128@d32768 | 303.77 | 306.56 | 1.01 | | INTEL(R) XEON(R) GOLD 6542Y | qwen35moe 35B.A3B NVFP4 | tg128@d32768 | 193.41 | 207.99 | 1.08 | | INTEL(R) XEON(R) GOLD 6542Y | qwen35moe 35B.A3B Q4_K_M | tg128@d32768 | 217.60 | 218.58 | 1.00 | DGX Spark build: 5b7d9f272 (9578) + ./scripts/compare-llama-bench.py -b master -c osimons/nvfp4_fuse_mmvq --tool llama-bench -i llama-bench.sqlite | CPU | Model | Test | t/s master | t/s osimons/nvfp4_fuse_mmvq | Speedup | |:------|:-------------------------|:-------------|-------------:|------------------------------:|----------:| | CPU | gemma4 26B.A4B NVFP4 | tg128@d32768 | 34.61 | 34.84 | 1.01 | | CPU | gemma4 26B.A4B Q4_K_M | tg128@d32768 | 46.95 | 46.90 | 1.00 | | CPU | gpt-oss 20B MXFP4 MoE | tg128@d32768 | 64.84 | 64.62 | 1.00 | | CPU | qwen35moe 35B.A3B NVFP4 | tg128@d32768 | 59.63 | 60.72 | 1.02 | | CPU | qwen35moe 35B.A3B Q4_K_M | tg128@d32768 | 56.53 | 56.55 | 1.00 | PPL values for 5 chunks: this PR model mode ppl uncertainty log /mnt/share/gguf/unsloth/Qwen3.6-35B-A3B-GGUF/Qwen3.6-35B-A3B-UD-Q4_K_M.gguf fusion_enabled 5.2892 0.35389 ppl-value-checks/Qwen3.6-35B-A3B-UD-Q4_K_M.fusion_enabled.log /mnt/share/gguf/unsloth/Qwen3.6-35B-A3B-GGUF/Qwen3.6-35B-A3B-UD-Q4_K_M.gguf fusion_disabled 5.2742 0.35215 ppl-value-checks/Qwen3.6-35B-A3B-UD-Q4_K_M.fusion_disabled.log /mnt/share/gguf/nvidia/Qwen3.6-35B-A3B-2.06GB-per-token-CT/Qwen3.6-35B-A3B-2.06GB-per-token-CT_fp8_q8.gguf fusion_enabled 5.4487 0.36866 ppl-value-checks/Qwen3.6-35B-A3B-2.06GB-per-token-CT_fp8_q8.fusion_enabled.log /mnt/share/gguf/nvidia/Qwen3.6-35B-A3B-2.06GB-per-token-CT/Qwen3.6-35B-A3B-2.06GB-per-token-CT_fp8_q8.gguf fusion_disabled 5.4403 0.36782 ppl-value-checks/Qwen3.6-35B-A3B-2.06GB-per-token-CT_fp8_q8.fusion_disabled.log /mnt/share/gguf/nvidia/Gemma-4-26B-A4B-NVFP4/Gemma-4-26B-A4B-NVFP4_fp8_q8.gguf fusion_enabled 17342.4348 3703.13932 ppl-value-checks/Gemma-4-26B-A4B-NVFP4_fp8_q8.fusion_enabled.log /mnt/share/gguf/nvidia/Gemma-4-26B-A4B-NVFP4/Gemma-4-26B-A4B-NVFP4_fp8_q8.gguf fusion_disabled 18627.0624 3998.42475 ppl-value-checks/Gemma-4-26B-A4B-NVFP4_fp8_q8.fusion_disabled.log /mnt/share/gguf/ggml-org/gpt-oss-20b-GGUF/gpt-oss-20b-mxfp4.gguf fusion_enabled 363.8913 33.14007 ppl-value-checks/gpt-oss-20b-mxfp4.fusion_enabled.log /mnt/share/gguf/ggml-org/gpt-oss-20b-GGUF/gpt-oss-20b-mxfp4.gguf fusion_disabled 363.8913 33.14007 ppl-value-checks/gpt-oss-20b-mxfp4.fusion_disabled.log /mnt/share/gguf/unsloth/gemma-4-26B-A4B-it-GGUF/gemma-4-26B-A4B-it-UD-Q4_K_XL.gguf fusion_enabled 17330.3926 3716.70472 ppl-value-checks/gemma-4-26B-A4B-it-UD-Q4_K_XL.fusion_enabled.log /mnt/share/gguf/unsloth/gemma-4-26B-A4B-it-GGUF/gemma-4-26B-A4B-it-UD-Q4_K_XL.gguf fusion_disabled 17933.9524 3883.17066 ppl-value-checks/gemma-4-26B-A4B-it-UD-Q4_K_XL.fusion_disabled.log master: summary: ppl-value-checks/summary.tsv model mode ppl uncertainty log /mnt/share/gguf/unsloth/Qwen3.6-35B-A3B-GGUF/Qwen3.6-35B-A3B-UD-Q4_K_M.gguf fusion_enabled 5.2892 0.35389 ppl-value-checks/Qwen3.6-35B-A3B-UD-Q4_K_M.fusion_enabled.log /mnt/share/gguf/unsloth/Qwen3.6-35B-A3B-GGUF/Qwen3.6-35B-A3B-UD-Q4_K_M.gguf fusion_disabled 5.2742 0.35215 ppl-value-checks/Qwen3.6-35B-A3B-UD-Q4_K_M.fusion_disabled.log /mnt/share/gguf/nvidia/Qwen3.6-35B-A3B-2.06GB-per-token-CT/Qwen3.6-35B-A3B-2.06GB-per-token-CT_fp8_q8.gguf fusion_enabled 5.4487 0.36866 ppl-value-checks/Qwen3.6-35B-A3B-2.06GB-per-token-CT_fp8_q8.fusion_enabled.log /mnt/share/gguf/nvidia/Qwen3.6-35B-A3B-2.06GB-per-token-CT/Qwen3.6-35B-A3B-2.06GB-per-token-CT_fp8_q8.gguf fusion_disabled 5.4403 0.36782 ppl-value-checks/Qwen3.6-35B-A3B-2.06GB-per-token-CT_fp8_q8.fusion_disabled.log /mnt/share/gguf/nvidia/Gemma-4-26B-A4B-NVFP4/Gemma-4-26B-A4B-NVFP4_fp8_q8.gguf fusion_enabled 17342.4348 3703.13932 ppl-value-checks/Gemma-4-26B-A4B-NVFP4_fp8_q8.fusion_enabled.log /mnt/share/gguf/nvidia/Gemma-4-26B-A4B-NVFP4/Gemma-4-26B-A4B-NVFP4_fp8_q8.gguf fusion_disabled 18627.0624 3998.42475 ppl-value-checks/Gemma-4-26B-A4B-NVFP4_fp8_q8.fusion_disabled.log /mnt/share/gguf/ggml-org/gpt-oss-20b-GGUF/gpt-oss-20b-mxfp4.gguf fusion_enabled 363.8913 33.14007 ppl-value-checks/gpt-oss-20b-mxfp4.fusion_enabled.log /mnt/share/gguf/ggml-org/gpt-oss-20b-GGUF/gpt-oss-20b-mxfp4.gguf fusion_disabled 363.8913 33.14007 ppl-value-checks/gpt-oss-20b-mxfp4.fusion_disabled.log /mnt/share/gguf/unsloth/gemma-4-26B-A4B-it-GGUF/gemma-4-26B-A4B-it-UD-Q4_K_XL.gguf fusion_enabled 17330.3926 3716.70472 ppl-value-checks/gemma-4-26B-A4B-it-UD-Q4_K_XL.fusion_enabled.log /mnt/share/gguf/unsloth/gemma-4-26B-A4B-it-GGUF/gemma-4-26B-A4B-it-UD-Q4_K_XL.gguf fusion_disabled 17933.9524 3883.17066 ppl-value-checks/gemma-4-26B-A4B-it-UD-Q4_K_XL.fusion_disabled.log * Allow views to weights in ggml_can_fuse_subgraph * Remove gate_first from test_mul_mat_vec_fusion * Ditch lane-parsing approach in favor of hard-coded patterns * Apply suggestions from code review Co-authored-by: Georgi Gerganov <ggerganov@gmail.com> * Rename ggml_is_constant_view_src to ggml_is_constant * Finish renaming of 0905129e9d12e2bc6f16d6d3cc4e6b40606fc893 * Readd descriptive prints for fusion debugging * Add weight-buffer pre-allocation to `test_case` This is required so we correctly test fusion of NVFP4. * Update ggml/src/ggml.c Co-authored-by: Johannes Gäßler <johannesg@5d6.de> * Add 2nd context for weights as suggested by @JohannesGaessler This reflects more natural use of ggml compared to artifically pre-allocating weights into the same context * Exclude fused tests from gradient mode I'm unsure of the current state, but naively every fusion pattern should require its own backpropagation implementation. I don't see these implemented for the CUDA backend, so we can disable tests to avoid triggering GGML_ASSERT for ggml_tensor * build_graph(ggml_context * ctx) override { GGML_ASSERT(!use_weight_context()); return build_graph(ctx, nullptr); } * Apply suggestions from code review Co-authored-by: Johannes Gäßler <johannesg@5d6.de> --------- Co-authored-by: Georgi Gerganov <ggerganov@gmail.com> Co-authored-by: Johannes Gäßler <johannesg@5d6.de>
This commit is contained in:
Oliver Simons
co-authored by
Georgi Gerganov
Johannes Gäßler
Georgi Gerganov
Johannes Gäßler
parent
f5525f7e7a
commit
3899b39ce2
+148
-32
@@ -1137,6 +1137,10 @@ struct test_case {
|
||||
}
|
||||
|
||||
virtual ggml_tensor * build_graph(ggml_context * ctx) = 0;
|
||||
virtual ggml_tensor * build_graph(ggml_context * ctx, ggml_context * ctx_weights) {
|
||||
GGML_UNUSED(ctx_weights);
|
||||
return build_graph(ctx);
|
||||
}
|
||||
|
||||
virtual double max_nmse_err() {
|
||||
return 1e-7;
|
||||
@@ -1213,6 +1217,7 @@ struct test_case {
|
||||
|
||||
virtual bool run_whole_graph() { return false; }
|
||||
virtual std::vector<ggml_tensor *> fusion_test_nodes() { return {}; }
|
||||
virtual bool use_weight_context() { return false; }
|
||||
|
||||
ggml_cgraph * gf = nullptr;
|
||||
ggml_cgraph * gb = nullptr;
|
||||
@@ -1319,20 +1324,28 @@ struct test_case {
|
||||
/* .mem_base = */ NULL,
|
||||
/* .no_alloc = */ true,
|
||||
};
|
||||
const bool use_weights = use_weight_context();
|
||||
|
||||
ggml_context * ctx = ggml_init(params);
|
||||
GGML_ASSERT(ctx);
|
||||
ggml_context * ctx_weights = use_weights ? ggml_init(params) : nullptr;
|
||||
GGML_ASSERT(!use_weights || ctx_weights);
|
||||
|
||||
gf = ggml_new_graph(ctx);
|
||||
|
||||
// pre-graph sentinel
|
||||
add_sentinel(ctx);
|
||||
if (ctx_weights) {
|
||||
add_sentinel(ctx_weights);
|
||||
}
|
||||
|
||||
ggml_tensor * out = build_graph(ctx);
|
||||
ggml_tensor * out = build_graph(ctx, ctx_weights);
|
||||
current_op_name = op_desc(out);
|
||||
check_for_f16_tensor(ctx);
|
||||
|
||||
if (!matches_filter(out, op_names_filter)) {
|
||||
//printf(" %s: skipping\n", op_desc(out).c_str());
|
||||
ggml_free(ctx_weights);
|
||||
ggml_free(ctx);
|
||||
return test_status_t::SKIPPED;
|
||||
}
|
||||
@@ -1355,18 +1368,36 @@ struct test_case {
|
||||
|
||||
print_test_result_locked(output_printer, result);
|
||||
|
||||
ggml_free(ctx_weights);
|
||||
ggml_free(ctx);
|
||||
return test_status_t::NOT_SUPPORTED;
|
||||
}
|
||||
|
||||
// post-graph sentinel
|
||||
add_sentinel(ctx);
|
||||
if (ctx_weights) {
|
||||
add_sentinel(ctx_weights);
|
||||
}
|
||||
|
||||
ggml_backend_buffer_t buf_weights = nullptr;
|
||||
if (ctx_weights) {
|
||||
buf_weights = ggml_backend_alloc_ctx_tensors(ctx_weights, backend1);
|
||||
if (buf_weights == NULL) {
|
||||
printf("failed to allocate weight tensors [%s] ", ggml_backend_name(backend1));
|
||||
ggml_free(ctx_weights);
|
||||
ggml_free(ctx);
|
||||
return test_status_t::FAIL;
|
||||
}
|
||||
ggml_backend_buffer_set_usage(buf_weights, GGML_BACKEND_BUFFER_USAGE_WEIGHTS);
|
||||
}
|
||||
|
||||
// allocate
|
||||
ggml_backend_buffer_t buf = ggml_backend_alloc_ctx_tensors(ctx, backend1);
|
||||
|
||||
if (buf == NULL) {
|
||||
printf("failed to allocate tensors [%s] ", ggml_backend_name(backend1));
|
||||
ggml_backend_buffer_free(buf_weights);
|
||||
ggml_free(ctx_weights);
|
||||
ggml_free(ctx);
|
||||
return test_status_t::FAIL;
|
||||
}
|
||||
@@ -1381,6 +1412,9 @@ struct test_case {
|
||||
|
||||
// randomize tensors
|
||||
initialize_tensors(ctx);
|
||||
if (ctx_weights) {
|
||||
initialize_tensors(ctx_weights);
|
||||
}
|
||||
|
||||
// compare
|
||||
struct callback_userdata {
|
||||
@@ -1466,7 +1500,8 @@ struct test_case {
|
||||
fused_nodes_to_verify.size());
|
||||
|
||||
ggml_backend_buffer_free(buf);
|
||||
|
||||
ggml_backend_buffer_free(buf_weights);
|
||||
ggml_free(ctx_weights);
|
||||
ggml_free(ctx);
|
||||
|
||||
// Create test result
|
||||
@@ -1490,10 +1525,14 @@ struct test_case {
|
||||
/* .mem_base = */ NULL,
|
||||
/* .no_alloc = */ true,
|
||||
};
|
||||
const bool use_weights = use_weight_context();
|
||||
|
||||
ggml_context_ptr ctx(ggml_init(params)); // smart ptr
|
||||
GGML_ASSERT(ctx);
|
||||
ggml_context_ptr ctx_weights(use_weights ? ggml_init(params) : nullptr);
|
||||
GGML_ASSERT(!use_weights || ctx_weights);
|
||||
|
||||
ggml_tensor * out = build_graph(ctx.get());
|
||||
ggml_tensor * out = build_graph(ctx.get(), ctx_weights.get());
|
||||
current_op_name = op_desc(out);
|
||||
if (!matches_filter(out, op_names_filter)) {
|
||||
//printf(" %s: skipping\n", op_desc(out).c_str());
|
||||
@@ -1510,6 +1549,16 @@ struct test_case {
|
||||
return true;
|
||||
}
|
||||
|
||||
ggml_backend_buffer_ptr buf_weights(nullptr);
|
||||
if (ctx_weights) {
|
||||
buf_weights.reset(ggml_backend_alloc_ctx_tensors(ctx_weights.get(), backend));
|
||||
if (buf_weights == NULL) {
|
||||
printf("failed to allocate weight tensors\n");
|
||||
return false;
|
||||
}
|
||||
ggml_backend_buffer_set_usage(buf_weights.get(), GGML_BACKEND_BUFFER_USAGE_WEIGHTS);
|
||||
}
|
||||
|
||||
// allocate
|
||||
ggml_backend_buffer_ptr buf(ggml_backend_alloc_ctx_tensors(ctx.get(), backend)); // smart ptr
|
||||
|
||||
@@ -1520,6 +1569,9 @@ struct test_case {
|
||||
|
||||
// randomize tensors
|
||||
initialize_tensors(ctx.get());
|
||||
if (ctx_weights) {
|
||||
initialize_tensors(ctx_weights.get());
|
||||
}
|
||||
|
||||
// build graph
|
||||
ggml_cgraph * gf = ggml_new_graph_custom(ctx.get(), graph_nodes, false);
|
||||
@@ -5848,19 +5900,21 @@ struct test_mul_mat_vec_fusion : public test_case {
|
||||
const bool b; // broadcast b matrix (only for use_id)
|
||||
const bool with_bias;
|
||||
const bool with_gate;
|
||||
const bool with_lane_scale;
|
||||
std::array<int64_t, 2> batch_dims;
|
||||
|
||||
test_mul_mat_vec_fusion(ggml_type type, ggml_glu_op op, int64_t m, int64_t n, int64_t k,
|
||||
bool use_id = false, int n_mats = 1, int n_used = 1, bool b = false, bool with_bias = false, bool with_gate = true,
|
||||
std::array<int64_t, 2> batch_dims = {4, 2})
|
||||
: type(type), glu_op(op), m(m), n(n), k(k), use_id(use_id), n_mats(n_mats), n_used(n_used), b(b), with_bias(with_bias), with_gate(with_gate), batch_dims(batch_dims) {
|
||||
bool with_lane_scale = false, std::array<int64_t, 2> batch_dims = {4, 2})
|
||||
: type(type), glu_op(op), m(m), n(n), k(k), use_id(use_id), n_mats(n_mats), n_used(n_used), b(b), with_bias(with_bias),
|
||||
with_gate(with_gate), with_lane_scale(with_lane_scale), batch_dims(batch_dims) {
|
||||
if (use_id) {
|
||||
GGML_ASSERT(n_used <= n_mats);
|
||||
}
|
||||
}
|
||||
|
||||
std::string vars() override {
|
||||
return VARS_TO_STR12(type, glu_op, m, n, k, use_id, n_mats, n_used, b, with_bias, with_gate, batch_dims);
|
||||
return VARS_TO_STR13(type, glu_op, m, n, k, use_id, n_mats, n_used, b, with_bias, with_gate, with_lane_scale, batch_dims);
|
||||
}
|
||||
|
||||
std::string op_desc(ggml_tensor * t) override {
|
||||
@@ -5869,6 +5923,7 @@ struct test_mul_mat_vec_fusion : public test_case {
|
||||
}
|
||||
|
||||
bool run_whole_graph() override { return true; }
|
||||
bool use_weight_context() override { return use_id && with_lane_scale; }
|
||||
|
||||
ggml_tensor * build_gate(ggml_context * ctx, ggml_tensor * ffn_gate, ggml_tensor * ffn_up) {
|
||||
ggml_tensor * out = nullptr;
|
||||
@@ -5884,7 +5939,26 @@ struct test_mul_mat_vec_fusion : public test_case {
|
||||
return out;
|
||||
}
|
||||
|
||||
ggml_tensor * build_lane_scale_dense(ggml_context * ctx, ggml_tensor * out) {
|
||||
ggml_tensor * scale = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, 1);
|
||||
return ggml_mul(ctx, out, scale);
|
||||
}
|
||||
|
||||
ggml_tensor * build_lane_scale_id(ggml_context * ctx, ggml_context * ctx_weights, ggml_tensor * out, ggml_tensor * ids) {
|
||||
GGML_ASSERT(ctx_weights);
|
||||
ggml_tensor * scale = ggml_new_tensor_1d(ctx_weights, GGML_TYPE_F32, n_mats);
|
||||
ggml_tensor * s = ggml_reshape_3d(ctx, scale, 1, n_mats, 1);
|
||||
s = ggml_repeat_4d(ctx, s, 1, n_mats, m, 1);
|
||||
s = ggml_get_rows(ctx, s, ids);
|
||||
return ggml_mul(ctx, out, s);
|
||||
}
|
||||
|
||||
ggml_tensor * build_graph(ggml_context * ctx) override {
|
||||
GGML_ASSERT(!use_weight_context());
|
||||
return build_graph(ctx, nullptr);
|
||||
}
|
||||
|
||||
ggml_tensor * build_graph(ggml_context * ctx, ggml_context * ctx_weights) override {
|
||||
if (!use_id) {
|
||||
const int channels = batch_dims[0];
|
||||
const int samples = batch_dims[1];
|
||||
@@ -5895,19 +5969,34 @@ struct test_mul_mat_vec_fusion : public test_case {
|
||||
ggml_tensor * gate = with_gate ? ggml_new_tensor(ctx, type, 4, ne0.data()) : nullptr;
|
||||
ggml_tensor * up = ggml_new_tensor(ctx, type, 4, ne0.data());
|
||||
|
||||
ggml_tensor * ffn_up = ggml_mul_mat(ctx, up, cur);
|
||||
if (with_bias) {
|
||||
std::array<int64_t, 4> bias_ne = { ffn_up->ne[0], 1, channels, samples };
|
||||
ggml_tensor * up_bias = ggml_new_tensor(ctx, GGML_TYPE_F32, 4, bias_ne.data());
|
||||
ffn_up = ggml_add(ctx, ffn_up, up_bias);
|
||||
}
|
||||
auto build_lane_up = [&]() {
|
||||
ggml_tensor * ffn_up = ggml_mul_mat(ctx, up, cur);
|
||||
if (with_lane_scale) {
|
||||
ffn_up = build_lane_scale_dense(ctx, ffn_up);
|
||||
}
|
||||
if (with_bias) {
|
||||
std::array<int64_t, 4> bias_ne = { ffn_up->ne[0], 1, channels, samples };
|
||||
ggml_tensor * up_bias = ggml_new_tensor(ctx, GGML_TYPE_F32, 4, bias_ne.data());
|
||||
ffn_up = ggml_add(ctx, ffn_up, up_bias);
|
||||
}
|
||||
return ffn_up;
|
||||
};
|
||||
|
||||
ggml_tensor * ffn_gate = with_gate ? ggml_mul_mat(ctx, gate, cur) : nullptr;
|
||||
if (with_bias && with_gate) {
|
||||
std::array<int64_t, 4> bias_ne = { ffn_gate->ne[0], 1, channels, samples };
|
||||
ggml_tensor * gate_bias = ggml_new_tensor(ctx, GGML_TYPE_F32, 4, bias_ne.data());
|
||||
ffn_gate = ggml_add(ctx, ffn_gate, gate_bias);
|
||||
}
|
||||
auto build_lane_gate = [&]() {
|
||||
ggml_tensor * ffn_gate = ggml_mul_mat(ctx, gate, cur);
|
||||
if (with_lane_scale) {
|
||||
ffn_gate = build_lane_scale_dense(ctx, ffn_gate);
|
||||
}
|
||||
if (with_bias) {
|
||||
std::array<int64_t, 4> bias_ne = { ffn_gate->ne[0], 1, channels, samples };
|
||||
ggml_tensor * gate_bias = ggml_new_tensor(ctx, GGML_TYPE_F32, 4, bias_ne.data());
|
||||
ffn_gate = ggml_add(ctx, ffn_gate, gate_bias);
|
||||
}
|
||||
return ffn_gate;
|
||||
};
|
||||
|
||||
ggml_tensor * ffn_up = build_lane_up();
|
||||
ggml_tensor * ffn_gate = with_gate ? build_lane_gate() : nullptr;
|
||||
|
||||
ggml_tensor * out = with_gate ? build_gate(ctx, ffn_gate, ffn_up) : ffn_up;
|
||||
|
||||
@@ -5929,17 +6018,32 @@ struct test_mul_mat_vec_fusion : public test_case {
|
||||
ggml_tensor * cur = ggml_new_tensor_3d(ctx, GGML_TYPE_F32, k, this->b ? 1 : n_used, m);
|
||||
ggml_set_name(cur, "cur");
|
||||
|
||||
ggml_tensor * ffn_up = ggml_mul_mat_id(ctx, ups, cur, ids);
|
||||
if (with_bias) {
|
||||
ggml_tensor * up_bias_param = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, ffn_up->ne[0], n_mats);
|
||||
ffn_up = ggml_add_id(ctx, ffn_up, up_bias_param, ids);
|
||||
}
|
||||
auto build_lane_up = [&]() {
|
||||
ggml_tensor * ffn_up = ggml_mul_mat_id(ctx, ups, cur, ids);
|
||||
if (with_lane_scale) {
|
||||
ffn_up = build_lane_scale_id(ctx, ctx_weights, ffn_up, ids);
|
||||
}
|
||||
if (with_bias) {
|
||||
ggml_tensor * up_bias_param = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, ffn_up->ne[0], n_mats);
|
||||
ffn_up = ggml_add_id(ctx, ffn_up, up_bias_param, ids);
|
||||
}
|
||||
return ffn_up;
|
||||
};
|
||||
|
||||
ggml_tensor * ffn_gate = with_gate? ggml_mul_mat_id(ctx, gates, cur, ids) : nullptr;
|
||||
if (with_bias && with_gate) {
|
||||
ggml_tensor * gate_bias_param = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, ffn_gate->ne[0], n_mats);
|
||||
ffn_gate = ggml_add_id(ctx, ffn_gate, gate_bias_param, ids);
|
||||
}
|
||||
auto build_lane_gate = [&]() {
|
||||
ggml_tensor * ffn_gate = ggml_mul_mat_id(ctx, gates, cur, ids);
|
||||
if (with_lane_scale) {
|
||||
ffn_gate = build_lane_scale_id(ctx, ctx_weights, ffn_gate, ids);
|
||||
}
|
||||
if (with_bias) {
|
||||
ggml_tensor * gate_bias_param = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, ffn_gate->ne[0], n_mats);
|
||||
ffn_gate = ggml_add_id(ctx, ffn_gate, gate_bias_param, ids);
|
||||
}
|
||||
return ffn_gate;
|
||||
};
|
||||
|
||||
ggml_tensor * ffn_up = build_lane_up();
|
||||
ggml_tensor * ffn_gate = with_gate ? build_lane_gate() : nullptr;
|
||||
|
||||
ggml_tensor * out = with_gate ? build_gate(ctx, ffn_gate, ffn_up) : ffn_up;
|
||||
|
||||
@@ -9202,10 +9306,15 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() {
|
||||
if (!with_gate && glu_op != GGML_GLU_OP_SWIGLU) {
|
||||
continue;
|
||||
}
|
||||
test_cases.emplace_back(new test_mul_mat_vec_fusion(type, glu_op, 1, 32, 256,
|
||||
use_id, 16, 8, b, with_bias, with_gate));
|
||||
test_cases.emplace_back(new test_mul_mat_vec_fusion(type, glu_op, 1, 32, 256,
|
||||
use_id, 16, 8, b, with_bias, with_gate, {1, 1}));
|
||||
for (bool with_lane_scale : {false, true}) {
|
||||
if (with_lane_scale && type != GGML_TYPE_NVFP4) {
|
||||
continue;
|
||||
}
|
||||
test_cases.emplace_back(new test_mul_mat_vec_fusion(type, glu_op, 1, 32, 256,
|
||||
use_id, 16, 8, b, with_bias, with_gate, with_lane_scale));
|
||||
test_cases.emplace_back(new test_mul_mat_vec_fusion(type, glu_op, 1, 32, 256,
|
||||
use_id, 16, 8, b, with_bias, with_gate, with_lane_scale, {1, 1}));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -9823,6 +9932,13 @@ static bool test_backend(ggml_backend_t backend, ggml_backend_dev_t dev, test_mo
|
||||
}
|
||||
|
||||
if (mode == MODE_GRAD) {
|
||||
test_cases.erase(
|
||||
std::remove_if(test_cases.begin(), test_cases.end(), [](const std::unique_ptr<test_case> & tc) {
|
||||
return tc->run_whole_graph();
|
||||
}),
|
||||
test_cases.end()
|
||||
);
|
||||
|
||||
size_t n_ok = 0;
|
||||
for (auto & test : test_cases) {
|
||||
if (test->eval_grad(backend, op_names_filter, output_printer)) {
|
||||
|
||||
Reference in New Issue
Block a user