From 0f701c88298606b41a13268a027e97c877ff63ca Mon Sep 17 00:00:00 2001 From: Alex Morgan <36094800+amitmi704@users.noreply.github.com> Date: Mon, 12 Oct 2026 09:19:20 -0500 Subject: [PATCH] vulkan: pack FMA for f16 accumulation, no dot product op (#29877) * vulkan: pack FMA for f16 accumulation, no dot product op * vulkan: dot_product packs two rows per f16 FMA * vulkan: rename dot_product to dot_product_v2 --- .../vulkan-shaders/dot_product_funcs.glsl | 31 +++++++++++++++++++ .../ggml-vulkan/vulkan-shaders/mul_mm.comp | 3 +- .../vulkan-shaders/vulkan-shaders-gen.cpp | 1 + 3 files changed, 33 insertions(+), 2 deletions(-) diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/dot_product_funcs.glsl b/ggml/src/ggml-vulkan/vulkan-shaders/dot_product_funcs.glsl index c474bfe09..d7a9804ba 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/dot_product_funcs.glsl +++ b/ggml/src/ggml-vulkan/vulkan-shaders/dot_product_funcs.glsl @@ -13,6 +13,14 @@ ACC_TYPE dot_product(f16vec2 a, f16vec2 b, ACC_TYPE acc) { return ACC_TYPE(v_dot2_f32_f16(a, b, float(acc))); } +ACC_TYPEV2 dot_product_v2(f16vec4 a0, f16vec4 a1, f16vec4 b, ACC_TYPEV2 acc) { + return ACC_TYPEV2(dot_product(a0, b, acc.x), dot_product(a1, b, acc.y)); +} + +ACC_TYPEV2 dot_product_v2(f16vec2 a0, f16vec2 a1, f16vec2 b, ACC_TYPEV2 acc) { + return ACC_TYPEV2(dot_product(a0, b, acc.x), dot_product(a1, b, acc.y)); +} + #else ACC_TYPE dot_product(FLOAT_TYPEV4 a, FLOAT_TYPEV4 b, ACC_TYPE acc) { @@ -24,4 +32,27 @@ ACC_TYPE dot_product(FLOAT_TYPEV2 a, FLOAT_TYPEV2 b, ACC_TYPE acc) { return fma(ACC_TYPE(a.x), ACC_TYPE(b.x), fma(ACC_TYPE(a.y), ACC_TYPE(b.y), acc)); } +#if defined(ACC_F16) +// two rows at once, so f16 accumulation can use packed fma +ACC_TYPEV2 dot_product_v2(FLOAT_TYPEV4 a0, FLOAT_TYPEV4 a1, FLOAT_TYPEV4 b, ACC_TYPEV2 acc) { + acc = fma(ACC_TYPEV2(a0.w, a1.w), ACC_TYPEV2(b.w), acc); + acc = fma(ACC_TYPEV2(a0.z, a1.z), ACC_TYPEV2(b.z), acc); + acc = fma(ACC_TYPEV2(a0.y, a1.y), ACC_TYPEV2(b.y), acc); + return fma(ACC_TYPEV2(a0.x, a1.x), ACC_TYPEV2(b.x), acc); +} + +ACC_TYPEV2 dot_product_v2(FLOAT_TYPEV2 a0, FLOAT_TYPEV2 a1, FLOAT_TYPEV2 b, ACC_TYPEV2 acc) { + acc = fma(ACC_TYPEV2(a0.y, a1.y), ACC_TYPEV2(b.y), acc); + return fma(ACC_TYPEV2(a0.x, a1.x), ACC_TYPEV2(b.x), acc); +} +#else +ACC_TYPEV2 dot_product_v2(FLOAT_TYPEV4 a0, FLOAT_TYPEV4 a1, FLOAT_TYPEV4 b, ACC_TYPEV2 acc) { + return ACC_TYPEV2(dot_product(a0, b, acc.x), dot_product(a1, b, acc.y)); +} + +ACC_TYPEV2 dot_product_v2(FLOAT_TYPEV2 a0, FLOAT_TYPEV2 a1, FLOAT_TYPEV2 b, ACC_TYPEV2 acc) { + return ACC_TYPEV2(dot_product(a0, b, acc.x), dot_product(a1, b, acc.y)); +} +#endif + #endif diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm.comp b/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm.comp index 9f31ef126..19d37870b 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm.comp +++ b/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm.comp @@ -407,8 +407,7 @@ void main() { [[unroll]] for (uint cr = 0; cr < TM / 2; cr++) { // [WNITER][TN][WMITER][TM / 2] -> [wsic][cc][wsir][cr] const uint sums_idx = (wsic * TN + cc) * WMITER * (TM / 2) + wsir * (TM / 2) + cr; - sums[sums_idx].x = dot_product(cache_a[wsir * TM + 2 * cr ], cache_b, sums[sums_idx].x); - sums[sums_idx].y = dot_product(cache_a[wsir * TM + 2 * cr + 1], cache_b, sums[sums_idx].y); + sums[sums_idx] = dot_product_v2(cache_a[wsir * TM + 2 * cr], cache_a[wsir * TM + 2 * cr + 1], cache_b, sums[sums_idx]); } } } diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/vulkan-shaders-gen.cpp b/ggml/src/ggml-vulkan/vulkan-shaders/vulkan-shaders-gen.cpp index 7597b3d33..94417f62b 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/vulkan-shaders-gen.cpp +++ b/ggml/src/ggml-vulkan/vulkan-shaders/vulkan-shaders-gen.cpp @@ -485,6 +485,7 @@ void matmul_shaders(bool fp16, MatMulIdType matmul_id_type, bool coopmat, bool c base_dict["ACC_TYPE_VEC4"] = f16acc ? "f16vec4" : "vec4"; if (f16acc) { base_dict["ACC_TYPE_MAX"] = "float16_t(65504.0)"; + base_dict["ACC_F16"] = "1"; } if (coopmat) {