From a818a46f3df4b6dee7f59f174283deaeae083c4b Mon Sep 17 00:00:00 2001 From: Clemens Wasser Date: Sat, 21 Nov 2026 13:20:22 +0100 Subject: [PATCH] sycl: add fused ADD, RMS_NORM, MUL and vectorize rms norm (#31753) * sycl: vectorize contiguous rms norm * sycl: add fused ADD, RMS_NORM, MUL * tests: test fused ADD, RMS_NORM, MUL at decode widths --- ggml/src/ggml-sycl/fusion.cpp | 22 ++++ ggml/src/ggml-sycl/ggml-sycl.cpp | 6 + ggml/src/ggml-sycl/norm.cpp | 190 ++++++++++++++++++++++++++++--- ggml/src/ggml-sycl/norm.hpp | 5 + tests/test-backend-ops.cpp | 53 +++++++++ 5 files changed, 262 insertions(+), 14 deletions(-) diff --git a/ggml/src/ggml-sycl/fusion.cpp b/ggml/src/ggml-sycl/fusion.cpp index 9b553829b..9ea6af6b0 100644 --- a/ggml/src/ggml-sycl/fusion.cpp +++ b/ggml/src/ggml-sycl/fusion.cpp @@ -121,6 +121,28 @@ bool ggml_sycl_can_fuse(const ggml_cgraph * cgraph, int node_idx, std::initializ return ggml_sycl_should_fuse_mul_mat_glu(gate, up, glu); } + // the residual sum stays materialised (a later residual add reads it), so ggml_can_fuse cannot + // express this either: the add and the mul are the subgraph's outputs + if (ops.size() == 3 && ops.begin()[0] == GGML_OP_ADD && ops.begin()[1] == GGML_OP_RMS_NORM && + ops.begin()[2] == GGML_OP_MUL) { + if (!ggml_can_fuse_subgraph(cgraph, node_idx, ops, { node_idx, node_idx + 2 })) { + return false; + } + + const ggml_tensor * add = cgraph->nodes[node_idx]; + if (cgraph->nodes[node_idx + 1]->src[0] != add) { + return false; + } + + // the fused kernel does not broadcast the add + if (add->src[0]->type != GGML_TYPE_F32 || add->src[1]->type != GGML_TYPE_F32 || add->type != GGML_TYPE_F32 || + !ggml_are_same_shape(add->src[0], add) || !ggml_are_same_shape(add->src[1], add)) { + return false; + } + + return ggml_sycl_can_fuse(cgraph, node_idx + 1, { GGML_OP_RMS_NORM, GGML_OP_MUL }, {}); + } + if (!ggml_can_fuse(cgraph, node_idx, ops)) { return false; } diff --git a/ggml/src/ggml-sycl/ggml-sycl.cpp b/ggml/src/ggml-sycl/ggml-sycl.cpp index 5b87fa024..3de5f1bd5 100644 --- a/ggml/src/ggml-sycl/ggml-sycl.cpp +++ b/ggml/src/ggml-sycl/ggml-sycl.cpp @@ -6708,6 +6708,12 @@ static void ggml_backend_sycl_graph_compute_impl(ggml_backend_sycl_context * syc continue; } } + if (node->op == GGML_OP_ADD && + ggml_sycl_can_fuse(cgraph, i, { GGML_OP_ADD, GGML_OP_RMS_NORM, GGML_OP_MUL }, {}) && + ggml_sycl_op_add_rms_norm_fused(*sycl_ctx, node, cgraph->nodes[i + 1], cgraph->nodes[i + 2])) { + i += 2; + continue; + } if (node->op == GGML_OP_RMS_NORM && ggml_sycl_can_fuse(cgraph, i, { GGML_OP_RMS_NORM, GGML_OP_MUL, GGML_OP_ADD }, {})) { ggml_sycl_op_rms_norm_fused_add(*sycl_ctx, node, cgraph->nodes[i + 1], cgraph->nodes[i + 2]); diff --git a/ggml/src/ggml-sycl/norm.cpp b/ggml/src/ggml-sycl/norm.cpp index 36576e9c2..0cb30aa35 100644 --- a/ggml/src/ggml-sycl/norm.cpp +++ b/ggml/src/ggml-sycl/norm.cpp @@ -1,4 +1,6 @@ #include "norm.hpp" + +#include "fusion.hpp" #include "ggml-sycl/common.hpp" #include "ggml-sycl/presets.hpp" @@ -188,9 +190,28 @@ static void rms_norm_f32(const float* x, float* dst, const int ncols, float tmp = 0.0f; // partial sum for thread in warp - for (int col = tid; col < ncols; col += block_size) { - const float xi = x[col * src_stride_col]; - tmp += xi * xi; + // float4 path for contiguous, 16-byte aligned rows (decode), views and odd lengths stay strided + bool contig = src_stride_col == 1 && dst_stride_col == 1 && ncols % 4 == 0 && (uintptr_t) x % 16 == 0 && + (uintptr_t) dst % 16 == 0; + if constexpr (do_multiply) { + contig = contig && (uintptr_t) mul % 16 == 0; + } + if constexpr (do_add) { + contig = contig && (uintptr_t) add % 16 == 0; + } + const int n4 = ncols / 4; + + if (contig) { + const sycl::float4 * x4 = reinterpret_cast(x); + for (int i = tid; i < n4; i += block_size) { + const sycl::float4 v = x4[i]; + tmp += sycl::dot(v, v); + } + } else { + for (int col = tid; col < ncols; col += block_size) { + const float xi = x[col * src_stride_col]; + tmp += xi * xi; + } } // sum up partial sums @@ -216,17 +237,37 @@ static void rms_norm_f32(const float* x, float* dst, const int ncols, const float mean = tmp / ncols; const float scale = sycl::rsqrt(mean + eps); - for (int col = tid; col < ncols; col += block_size) { - if constexpr (do_multiply && do_add) { - dst[col * dst_stride_col] = scale * x[col * src_stride_col] * mul[col] + add[col]; - } else if constexpr (do_multiply) { - dst[col * dst_stride_col] = scale * x[col * src_stride_col] * mul[col]; - } else { - // folded epilogue of a fused GGML_OP_SCALE consumer (qwen35 GDN l2 norms); - // the explicit temporary keeps the float evaluation order identical to - // running rms_norm and scale as two separate kernels - const float v = scale * x[col * src_stride_col]; - dst[col * dst_stride_col] = v * scale_mul; + if (contig) { + const sycl::float4 * x4 = reinterpret_cast(x); + sycl::float4 * d4 = reinterpret_cast(dst); + const sycl::float4 * m4 = reinterpret_cast(mul); + const sycl::float4 * a4 = reinterpret_cast(add); + for (int i = tid; i < n4; i += block_size) { + const sycl::float4 v = x4[i]; + if constexpr (do_multiply && do_add) { + const sycl::float4 m = m4[i]; + const sycl::float4 a = a4[i]; + d4[i] = scale * v * m + a; + } else if constexpr (do_multiply) { + const sycl::float4 m = m4[i]; + d4[i] = scale * v * m; + } else { + d4[i] = scale * v * scale_mul; + } + } + } else { + for (int col = tid; col < ncols; col += block_size) { + if constexpr (do_multiply && do_add) { + dst[col * dst_stride_col] = scale * x[col * src_stride_col] * mul[col] + add[col]; + } else if constexpr (do_multiply) { + dst[col * dst_stride_col] = scale * x[col * src_stride_col] * mul[col]; + } else { + // folded epilogue of a fused GGML_OP_SCALE consumer (qwen35 GDN l2 norms); + // the explicit temporary keeps the float evaluation order identical to + // running rms_norm and scale as two separate kernels + const float v = scale * x[col * src_stride_col]; + dst[col * dst_stride_col] = v * scale_mul; + } } } } @@ -918,6 +959,127 @@ void ggml_sycl_op_rms_norm_fused_add(ggml_backend_sycl_context & ctx, ggml_tenso add_s01, add_s02, add_s03, add_nrows, add_nchannels, add_nsamples, eps, main_stream, ctx.device); } +// ADD(a, b) -> RMS_NORM -> MUL(w) in one launch. The sum (read by a later residual add) and +// normed * w are written after a barrier, as mul->data may alias a. +static void add_rms_norm_mul_f32(const float * a, + const float * b, + float * sum, + const float * mul, + float * dst, + const int ncols, + const float eps, + const sycl::nd_item<3> & item_ct1, + float * s_sum, + const int block_size) { + const int row = item_ct1.get_group(2); + const size_t off = (size_t) row * ncols; + const int tid = item_ct1.get_local_id(2); + const int n4 = ncols / 4; + + const sycl::float4 * a4 = reinterpret_cast(a + off); + const sycl::float4 * b4 = reinterpret_cast(b + off); + + sycl::float4 v[2]; + float tmp = 0.0f; +#pragma unroll + for (int k = 0; k < 2; ++k) { + const int i = tid + k * block_size; + if (i < n4) { + const sycl::float4 va = a4[i]; + const sycl::float4 vb = b4[i]; + v[k] = va + vb; + tmp += sycl::dot(v[k], v[k]); + } + } + + const int nwarps = block_size / WARP_SIZE; + tmp = warp_reduce_sum(tmp, item_ct1); + if (block_size > WARP_SIZE) { + const auto sub_group = item_ct1.get_sub_group(); + if (sub_group.get_local_linear_id() == 0) { + s_sum[sub_group.get_group_linear_id()] = tmp; + } + item_ct1.barrier(sycl::access::fence_space::local_space); + const size_t nreduce = ceil_div(nwarps, WARP_SIZE); + tmp = 0.0f; + for (size_t i = 0; i < nreduce; i += 1) { + tmp += s_sum[sub_group.get_local_linear_id() + i * WARP_SIZE]; + } + tmp = warp_reduce_sum(tmp, item_ct1); + } + + const float mean = tmp / ncols; + const float scale = sycl::rsqrt(mean + eps); + + sycl::float4 * s4 = reinterpret_cast(sum + off); + sycl::float4 * d4 = reinterpret_cast(dst + off); + const sycl::float4 * m4 = reinterpret_cast(mul); +#pragma unroll + for (int k = 0; k < 2; ++k) { + const int i = tid + k * block_size; + if (i < n4) { + const sycl::float4 m = m4[i]; + const sycl::float4 u = scale * v[k] * m; + s4[i] = v[k]; + d4[i] = u; + } + } +} + +static void add_rms_norm_mul_f32_sycl(const float * a, + const float * b, + float * sum, + const float * mul, + float * dst, + const int ncols, + const int nrows, + const float eps, + queue_ptr stream, + int device) { + const int work_group_size = ggml_sycl_info().max_work_group_sizes[device]; + GGML_ASSERT(work_group_size % (WARP_SIZE * WARP_SIZE) == 0); + const sycl::range<3> global_dims(1, 1, nrows); + const sycl::range<3> block_dims(1, 1, work_group_size); + stream->submit([&](sycl::handler & cgh) { + sycl::local_accessor s_sum(sycl::range<1>(work_group_size / WARP_SIZE), cgh); + cgh.parallel_for(sycl::nd_range<3>(global_dims * block_dims, block_dims), + [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] { + add_rms_norm_mul_f32(a, b, sum, mul, dst, ncols, eps, item_ct1, get_pointer(s_sum), + work_group_size); + }); + }); +} + +bool ggml_sycl_op_add_rms_norm_fused(ggml_backend_sycl_context & ctx, + ggml_tensor * add, + ggml_tensor * rms, + ggml_tensor * mul) { + ggml_tensor * a = add->src[0]; + ggml_tensor * b = add->src[1]; + ggml_tensor * w = mul->src[0] == rms ? mul->src[1] : mul->src[0]; + const int64_t ncols = add->ne[0]; + const int wg = ggml_sycl_info().max_work_group_sizes[ctx.device]; + // whole rows of float4, at most two per work-item, from 16-byte aligned contiguous tensors + if (add->ne[2] != 1 || add->ne[3] != 1 || ncols < 1024 || ncols % 4 != 0 || ncols > 8 * (int64_t) wg || + w->ne[1] != 1) { + return false; + } + for (const ggml_tensor * t : { a, b, add, mul, w }) { + if (!ggml_is_contiguous(t) || (uintptr_t) t->data % 16 != 0) { + return false; + } + } + float eps = 0.0f; + memcpy(&eps, rms->op_params, sizeof(float)); + if (!(eps >= 0.0f)) { + return false; + } + add_rms_norm_mul_f32_sycl((const float *) a->data, (const float *) b->data, (float *) add->data, + (const float *) w->data, (float *) mul->data, (int) ncols, (int) add->ne[1], eps, + ctx.stream(), ctx.device); + return true; +} + void ggml_sycl_op_rms_norm_back(ggml_backend_sycl_context & ctx, ggml_tensor * dst) { scope_op_debug_print scope_dbg_print(__func__, dst, /*num_src=*/2); diff --git a/ggml/src/ggml-sycl/norm.hpp b/ggml/src/ggml-sycl/norm.hpp index fb667ac6b..59a9ffd6f 100644 --- a/ggml/src/ggml-sycl/norm.hpp +++ b/ggml/src/ggml-sycl/norm.hpp @@ -25,6 +25,11 @@ void ggml_sycl_op_rms_norm_scale_fused(ggml_backend_sycl_context& ctx, ggml_tens void ggml_sycl_op_rms_norm_fused_add(ggml_backend_sycl_context& ctx, ggml_tensor* dst, ggml_tensor* mul_tensor, ggml_tensor* add_tensor); +bool ggml_sycl_op_add_rms_norm_fused(ggml_backend_sycl_context & ctx, + ggml_tensor * add, + ggml_tensor * rms, + ggml_tensor * mul); + void ggml_sycl_op_rms_norm_back(ggml_backend_sycl_context& ctx, ggml_tensor* dst); void ggml_sycl_op_group_norm(ggml_backend_sycl_context& ctx, ggml_tensor* dst); diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cpp index 11bfa42a9..fbf5c12d3 100644 --- a/tests/test-backend-ops.cpp +++ b/tests/test-backend-ops.cpp @@ -4351,6 +4351,52 @@ struct test_add_rms_norm : public test_case { } }; +// GGML_OP_ADD + GGML_OP_RMS_NORM + GGML_OP_MUL (fused operation) +struct test_add_rms_norm_mul : public test_case { + const std::array ne; + const float eps; + + ggml_tensor * sum = nullptr; + ggml_tensor * out = nullptr; + + std::string op_desc(ggml_tensor * t) override { + GGML_UNUSED(t); + return "ADD_RMS_NORM_MUL"; + } + + bool run_whole_graph() override { return true; } + + std::vector fusion_test_nodes() override { return { sum, out }; } + + std::string vars() override { return VARS_TO_STR2(ne, eps); } + + test_add_rms_norm_mul(std::array ne = { 4096, 1, 1, 1 }, float eps = 1e-6f) : ne(ne), eps(eps) {} + + ggml_tensor * build_graph(ggml_context * ctx) override { + ggml_tensor * a = ggml_new_tensor(ctx, GGML_TYPE_F32, 4, ne.data()); + ggml_tensor * b = ggml_new_tensor(ctx, GGML_TYPE_F32, 4, ne.data()); + ggml_tensor * w = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, ne[0]); + ggml_set_name(a, "a"); + ggml_set_name(b, "b"); + ggml_set_name(w, "w"); + + sum = ggml_add(ctx, a, b); + ggml_set_name(sum, "sum"); + ggml_tensor * normed = ggml_mul(ctx, ggml_rms_norm(ctx, sum, eps), w); + ggml_set_name(normed, "normed"); + + out = ggml_add(ctx, normed, sum); + ggml_set_name(out, "out"); + return out; + } + + void initialize_tensors(ggml_context * ctx) override { + for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != NULL; t = ggml_get_next_tensor(ctx, t)) { + init_tensor_uniform(t, -10.f, 10.f); + } + } +}; + // GGML_OP_UNARY(RELU) + GGML_OP_SQR (fused operation) struct test_relu_sqr : public test_case { const ggml_type type; @@ -11801,6 +11847,13 @@ static std::vector> make_test_cases_eval() { } } + // decode-shaped residual rows: below, at and above the fused path's limits, odd row counts, batches + for (int64_t n : { 1020, 1024, 4096, 5120, 8192, 8196, 16384 }) { + for (int64_t rows : { 1, 3, 4 }) { + test_cases.emplace_back(new test_add_rms_norm_mul({ n, rows, 1, 1 }, 1e-6f)); + } + test_cases.emplace_back(new test_add_rms_norm_mul({ n, 2, 3, 1 }, 1e-6f)); + } for (uint32_t n : {64, 1025}) { test_cases.emplace_back(new test_add_add(GGML_TYPE_F32, GGML_TYPE_F32, { n, 5, 4, 3 }, false, false)); test_cases.emplace_back(new test_add_add(GGML_TYPE_F32, GGML_TYPE_F32, { n, 5, 4, 3 }, true, false));