From 7e8324f5feba1d8be3f966e1791e84107e40cdd7 Mon Sep 17 00:00:00 2001 From: Niko Maroulis Date: Wed, 7 Oct 2026 09:41:50 -0400 Subject: [PATCH] metal : fix MUL_MAT+ADD fusion when the residual is itself a MUL_MAT (#30100) * metal : fix MUL_MAT+ADD fusion when the residual is itself a MUL_MAT ggml_metal_op_mul_mat_mma picks the residual of a fused MUL_MAT+ADD as "the ADD operand whose op is not MUL_MAT". When both operands of the ADD are mat-mul outputs (x = W1 @ u + W2 @ v), that test is true for both, so the residual resolves to the fused mat-mul's own, never-written output and the kernel adds whatever that buffer holds. The fusion check (ggml_metal_mul_mat_add_operand) already selects the operand by identity; make the encoder do the same. Clef decision models hit this in their head (proj_option_context @ ctx + proj_option_lexical @ lex, 9 option rows): on Metal, /v1/systemone probabilities collapse toward uniform (billing 0.28 where the CPU backend gives 0.977, Cloudflare_clef-flash Q8_0), deterministic per memory layout, correct with GGML_METAL_FUSION_DISABLE=1. Not a quantization issue: the same file is right on CPU. Add a MUL_MAT_ADD mode to test-backend-ops where the residual is a second mat-mul; on Metal it fails 27 of 28 cases before this change (the one pass is f16 n=2, under the MMA row threshold, so nothing fuses). * Update tests/test-backend-ops.cpp Co-authored-by: Georgi Gerganov --------- Co-authored-by: Georgi Gerganov --- ggml/src/ggml-metal/ggml-metal-ops.cpp | 3 ++- tests/test-backend-ops.cpp | 10 +++++++++- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/ggml/src/ggml-metal/ggml-metal-ops.cpp b/ggml/src/ggml-metal/ggml-metal-ops.cpp index 2a14cd0c2..597d5300c 100644 --- a/ggml/src/ggml-metal/ggml-metal-ops.cpp +++ b/ggml/src/ggml-metal/ggml-metal-ops.cpp @@ -2476,7 +2476,8 @@ static int ggml_metal_op_mul_mat_mma(ggml_metal_op_t ctx, int idx) { if (fuse_add) { dst = ctx->node(idx + n_fuse - 1); - res = dst->src[0]->op == GGML_OP_MUL_MAT ? dst->src[1] : dst->src[0]; + // the residual is the other operand of the ADD, by identity: it can itself be a MUL_MAT output + res = dst->src[0] == op ? dst->src[1] : dst->src[0]; } auto pipeline = ggml_metal_library_get_pipeline_mul_mv_mma_auto(lib, op, fuse_add); diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cpp index b080ab496..efc1777ef 100644 --- a/tests/test-backend-ops.cpp +++ b/tests/test-backend-ops.cpp @@ -7233,6 +7233,7 @@ enum mul_mat_add_mode { MUL_MAT_ADD_ROW, // mm + a one-row res, which a same-shape fusion must leave alone MUL_MAT_ADD_RES_INPLACE, // res += mm MUL_MAT_ADD_B_INPLACE, // b += mm: the sum overwrites the mat-mul input (m == k) + MUL_MAT_ADD_MM_MM, // mm2 + mm: the residual is itself a mat-mul output, so both operands are MUL_MAT }; static std::string var_to_str(mul_mat_add_mode mode) { @@ -7242,6 +7243,7 @@ static std::string var_to_str(mul_mat_add_mode mode) { case MUL_MAT_ADD_ROW: return "mm+row"; case MUL_MAT_ADD_RES_INPLACE: return "res+=mm"; case MUL_MAT_ADD_B_INPLACE: return "b+=mm"; + case MUL_MAT_ADD_MM_MM: return "mm2+mm"; } return "unknown"; } @@ -7287,6 +7289,12 @@ struct test_mul_mat_add : public test_case { case MUL_MAT_ADD_RES_MM: out = ggml_add(ctx, res, mm); break; case MUL_MAT_ADD_RES_INPLACE: out = ggml_add_inplace(ctx, res, mm); break; case MUL_MAT_ADD_B_INPLACE: out = ggml_add_inplace(ctx, b, mm); break; + case MUL_MAT_ADD_MM_MM: + { + ggml_tensor * a2 = ggml_new_tensor_2d(ctx, type_a, k, m); + ggml_tensor * mm2 = ggml_mul_mat(ctx, a2, b); + out = ggml_add(ctx, mm2, mm); + } break; } ggml_set_name(out, "out"); @@ -10409,7 +10417,7 @@ static std::vector> make_test_cases_eval() { } } for (ggml_type type_a : {GGML_TYPE_Q4_0, GGML_TYPE_Q8_0, GGML_TYPE_Q5_K, GGML_TYPE_F16}) { - for (mul_mat_add_mode mode : {MUL_MAT_ADD_MM_RES, MUL_MAT_ADD_RES_MM, MUL_MAT_ADD_ROW, MUL_MAT_ADD_RES_INPLACE}) { + for (mul_mat_add_mode mode : {MUL_MAT_ADD_MM_RES, MUL_MAT_ADD_RES_MM, MUL_MAT_ADD_ROW, MUL_MAT_ADD_RES_INPLACE, MUL_MAT_ADD_MM_MM}) { test_cases.emplace_back(new test_mul_mat_add(type_a, 1000, n, 1024, mode)); } test_cases.emplace_back(new test_mul_mat_add(type_a, 2048, n, 2048, MUL_MAT_ADD_B_INPLACE));