sycl: add fused ADD, RMS_NORM, MUL and vectorize rms norm (#31753)

* sycl: vectorize contiguous rms norm

* sycl: add fused ADD, RMS_NORM, MUL

* tests: test fused ADD, RMS_NORM, MUL at decode widths
This commit is contained in:
Clemens Wasser authored and GitHub committed 2026-11-21 14:20:22 +02:00
1 parent b019be0c86
commit a818a46f3d
5 files changed
+262 -14

No files matched your search

+22
View File
@@ -121,6 +121,28 @@ bool ggml_sycl_can_fuse(const ggml_cgraph * cgraph, int node_idx, std::initializ
return ggml_sycl_should_fuse_mul_mat_glu(gate, up, glu);
}
// the residual sum stays materialised (a later residual add reads it), so ggml_can_fuse cannot
// express this either: the add and the mul are the subgraph's outputs
if (ops.size() == 3 && ops.begin()[0] == GGML_OP_ADD && ops.begin()[1] == GGML_OP_RMS_NORM &&
ops.begin()[2] == GGML_OP_MUL) {
if (!ggml_can_fuse_subgraph(cgraph, node_idx, ops, { node_idx, node_idx + 2 })) {
return false;
}
const ggml_tensor * add = cgraph->nodes[node_idx];
if (cgraph->nodes[node_idx + 1]->src[0] != add) {
return false;
}
// the fused kernel does not broadcast the add
if (add->src[0]->type != GGML_TYPE_F32 || add->src[1]->type != GGML_TYPE_F32 || add->type != GGML_TYPE_F32 ||
!ggml_are_same_shape(add->src[0], add) || !ggml_are_same_shape(add->src[1], add)) {
return false;
}
return ggml_sycl_can_fuse(cgraph, node_idx + 1, { GGML_OP_RMS_NORM, GGML_OP_MUL }, {});
}
if (!ggml_can_fuse(cgraph, node_idx, ops)) {
return false;
}
+6
View File
@@ -6708,6 +6708,12 @@ static void ggml_backend_sycl_graph_compute_impl(ggml_backend_sycl_context * syc
continue;
}
}
if (node->op == GGML_OP_ADD &&
ggml_sycl_can_fuse(cgraph, i, { GGML_OP_ADD, GGML_OP_RMS_NORM, GGML_OP_MUL }, {}) &&
ggml_sycl_op_add_rms_norm_fused(*sycl_ctx, node, cgraph->nodes[i + 1], cgraph->nodes[i + 2])) {
i += 2;
continue;
}
if (node->op == GGML_OP_RMS_NORM &&
ggml_sycl_can_fuse(cgraph, i, { GGML_OP_RMS_NORM, GGML_OP_MUL, GGML_OP_ADD }, {})) {
ggml_sycl_op_rms_norm_fused_add(*sycl_ctx, node, cgraph->nodes[i + 1], cgraph->nodes[i + 2]);
+176 -14
View File
@@ -1,4 +1,6 @@
#include "norm.hpp"
#include "fusion.hpp"
#include "ggml-sycl/common.hpp"
#include "ggml-sycl/presets.hpp"
@@ -188,9 +190,28 @@ static void rms_norm_f32(const float* x, float* dst, const int ncols,
float tmp = 0.0f; // partial sum for thread in warp
for (int col = tid; col < ncols; col += block_size) {
const float xi = x[col * src_stride_col];
tmp += xi * xi;
// float4 path for contiguous, 16-byte aligned rows (decode), views and odd lengths stay strided
bool contig = src_stride_col == 1 && dst_stride_col == 1 && ncols % 4 == 0 && (uintptr_t) x % 16 == 0 &&
(uintptr_t) dst % 16 == 0;
if constexpr (do_multiply) {
contig = contig && (uintptr_t) mul % 16 == 0;
}
if constexpr (do_add) {
contig = contig && (uintptr_t) add % 16 == 0;
}
const int n4 = ncols / 4;
if (contig) {
const sycl::float4 * x4 = reinterpret_cast<const sycl::float4 *>(x);
for (int i = tid; i < n4; i += block_size) {
const sycl::float4 v = x4[i];
tmp += sycl::dot(v, v);
}
} else {
for (int col = tid; col < ncols; col += block_size) {
const float xi = x[col * src_stride_col];
tmp += xi * xi;
}
}
// sum up partial sums
@@ -216,17 +237,37 @@ static void rms_norm_f32(const float* x, float* dst, const int ncols,
const float mean = tmp / ncols;
const float scale = sycl::rsqrt(mean + eps);
for (int col = tid; col < ncols; col += block_size) {
if constexpr (do_multiply && do_add) {
dst[col * dst_stride_col] = scale * x[col * src_stride_col] * mul[col] + add[col];
} else if constexpr (do_multiply) {
dst[col * dst_stride_col] = scale * x[col * src_stride_col] * mul[col];
} else {
// folded epilogue of a fused GGML_OP_SCALE consumer (qwen35 GDN l2 norms);
// the explicit temporary keeps the float evaluation order identical to
// running rms_norm and scale as two separate kernels
const float v = scale * x[col * src_stride_col];
dst[col * dst_stride_col] = v * scale_mul;
if (contig) {
const sycl::float4 * x4 = reinterpret_cast<const sycl::float4 *>(x);
sycl::float4 * d4 = reinterpret_cast<sycl::float4 *>(dst);
const sycl::float4 * m4 = reinterpret_cast<const sycl::float4 *>(mul);
const sycl::float4 * a4 = reinterpret_cast<const sycl::float4 *>(add);
for (int i = tid; i < n4; i += block_size) {
const sycl::float4 v = x4[i];
if constexpr (do_multiply && do_add) {
const sycl::float4 m = m4[i];
const sycl::float4 a = a4[i];
d4[i] = scale * v * m + a;
} else if constexpr (do_multiply) {
const sycl::float4 m = m4[i];
d4[i] = scale * v * m;
} else {
d4[i] = scale * v * scale_mul;
}
}
} else {
for (int col = tid; col < ncols; col += block_size) {
if constexpr (do_multiply && do_add) {
dst[col * dst_stride_col] = scale * x[col * src_stride_col] * mul[col] + add[col];
} else if constexpr (do_multiply) {
dst[col * dst_stride_col] = scale * x[col * src_stride_col] * mul[col];
} else {
// folded epilogue of a fused GGML_OP_SCALE consumer (qwen35 GDN l2 norms);
// the explicit temporary keeps the float evaluation order identical to
// running rms_norm and scale as two separate kernels
const float v = scale * x[col * src_stride_col];
dst[col * dst_stride_col] = v * scale_mul;
}
}
}
}
@@ -918,6 +959,127 @@ void ggml_sycl_op_rms_norm_fused_add(ggml_backend_sycl_context & ctx, ggml_tenso
add_s01, add_s02, add_s03, add_nrows, add_nchannels, add_nsamples, eps, main_stream, ctx.device);
}
// ADD(a, b) -> RMS_NORM -> MUL(w) in one launch. The sum (read by a later residual add) and
// normed * w are written after a barrier, as mul->data may alias a.
static void add_rms_norm_mul_f32(const float * a,
const float * b,
float * sum,
const float * mul,
float * dst,
const int ncols,
const float eps,
const sycl::nd_item<3> & item_ct1,
float * s_sum,
const int block_size) {
const int row = item_ct1.get_group(2);
const size_t off = (size_t) row * ncols;
const int tid = item_ct1.get_local_id(2);
const int n4 = ncols / 4;
const sycl::float4 * a4 = reinterpret_cast<const sycl::float4 *>(a + off);
const sycl::float4 * b4 = reinterpret_cast<const sycl::float4 *>(b + off);
sycl::float4 v[2];
float tmp = 0.0f;
#pragma unroll
for (int k = 0; k < 2; ++k) {
const int i = tid + k * block_size;
if (i < n4) {
const sycl::float4 va = a4[i];
const sycl::float4 vb = b4[i];
v[k] = va + vb;
tmp += sycl::dot(v[k], v[k]);
}
}
const int nwarps = block_size / WARP_SIZE;
tmp = warp_reduce_sum(tmp, item_ct1);
if (block_size > WARP_SIZE) {
const auto sub_group = item_ct1.get_sub_group();
if (sub_group.get_local_linear_id() == 0) {
s_sum[sub_group.get_group_linear_id()] = tmp;
}
item_ct1.barrier(sycl::access::fence_space::local_space);
const size_t nreduce = ceil_div(nwarps, WARP_SIZE);
tmp = 0.0f;
for (size_t i = 0; i < nreduce; i += 1) {
tmp += s_sum[sub_group.get_local_linear_id() + i * WARP_SIZE];
}
tmp = warp_reduce_sum(tmp, item_ct1);
}
const float mean = tmp / ncols;
const float scale = sycl::rsqrt(mean + eps);
sycl::float4 * s4 = reinterpret_cast<sycl::float4 *>(sum + off);
sycl::float4 * d4 = reinterpret_cast<sycl::float4 *>(dst + off);
const sycl::float4 * m4 = reinterpret_cast<const sycl::float4 *>(mul);
#pragma unroll
for (int k = 0; k < 2; ++k) {
const int i = tid + k * block_size;
if (i < n4) {
const sycl::float4 m = m4[i];
const sycl::float4 u = scale * v[k] * m;
s4[i] = v[k];
d4[i] = u;
}
}
}
static void add_rms_norm_mul_f32_sycl(const float * a,
const float * b,
float * sum,
const float * mul,
float * dst,
const int ncols,
const int nrows,
const float eps,
queue_ptr stream,
int device) {
const int work_group_size = ggml_sycl_info().max_work_group_sizes[device];
GGML_ASSERT(work_group_size % (WARP_SIZE * WARP_SIZE) == 0);
const sycl::range<3> global_dims(1, 1, nrows);
const sycl::range<3> block_dims(1, 1, work_group_size);
stream->submit([&](sycl::handler & cgh) {
sycl::local_accessor<float, 1> s_sum(sycl::range<1>(work_group_size / WARP_SIZE), cgh);
cgh.parallel_for(sycl::nd_range<3>(global_dims * block_dims, block_dims),
[=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] {
add_rms_norm_mul_f32(a, b, sum, mul, dst, ncols, eps, item_ct1, get_pointer(s_sum),
work_group_size);
});
});
}
bool ggml_sycl_op_add_rms_norm_fused(ggml_backend_sycl_context & ctx,
ggml_tensor * add,
ggml_tensor * rms,
ggml_tensor * mul) {
ggml_tensor * a = add->src[0];
ggml_tensor * b = add->src[1];
ggml_tensor * w = mul->src[0] == rms ? mul->src[1] : mul->src[0];
const int64_t ncols = add->ne[0];
const int wg = ggml_sycl_info().max_work_group_sizes[ctx.device];
// whole rows of float4, at most two per work-item, from 16-byte aligned contiguous tensors
if (add->ne[2] != 1 || add->ne[3] != 1 || ncols < 1024 || ncols % 4 != 0 || ncols > 8 * (int64_t) wg ||
w->ne[1] != 1) {
return false;
}
for (const ggml_tensor * t : { a, b, add, mul, w }) {
if (!ggml_is_contiguous(t) || (uintptr_t) t->data % 16 != 0) {
return false;
}
}
float eps = 0.0f;
memcpy(&eps, rms->op_params, sizeof(float));
if (!(eps >= 0.0f)) {
return false;
}
add_rms_norm_mul_f32_sycl((const float *) a->data, (const float *) b->data, (float *) add->data,
(const float *) w->data, (float *) mul->data, (int) ncols, (int) add->ne[1], eps,
ctx.stream(), ctx.device);
return true;
}
void ggml_sycl_op_rms_norm_back(ggml_backend_sycl_context & ctx, ggml_tensor * dst) {
scope_op_debug_print scope_dbg_print(__func__, dst, /*num_src=*/2);
+5
View File
@@ -25,6 +25,11 @@ void ggml_sycl_op_rms_norm_scale_fused(ggml_backend_sycl_context& ctx, ggml_tens
void ggml_sycl_op_rms_norm_fused_add(ggml_backend_sycl_context& ctx, ggml_tensor* dst, ggml_tensor* mul_tensor, ggml_tensor* add_tensor);
bool ggml_sycl_op_add_rms_norm_fused(ggml_backend_sycl_context & ctx,
ggml_tensor * add,
ggml_tensor * rms,
ggml_tensor * mul);
void ggml_sycl_op_rms_norm_back(ggml_backend_sycl_context& ctx, ggml_tensor* dst);
void ggml_sycl_op_group_norm(ggml_backend_sycl_context& ctx, ggml_tensor* dst);
+53
View File
@@ -4351,6 +4351,52 @@ struct test_add_rms_norm : public test_case {
}
};
// GGML_OP_ADD + GGML_OP_RMS_NORM + GGML_OP_MUL (fused operation)
struct test_add_rms_norm_mul : public test_case {
const std::array<int64_t, 4> ne;
const float eps;
ggml_tensor * sum = nullptr;
ggml_tensor * out = nullptr;
std::string op_desc(ggml_tensor * t) override {
GGML_UNUSED(t);
return "ADD_RMS_NORM_MUL";
}
bool run_whole_graph() override { return true; }
std::vector<ggml_tensor *> fusion_test_nodes() override { return { sum, out }; }
std::string vars() override { return VARS_TO_STR2(ne, eps); }
test_add_rms_norm_mul(std::array<int64_t, 4> ne = { 4096, 1, 1, 1 }, float eps = 1e-6f) : ne(ne), eps(eps) {}
ggml_tensor * build_graph(ggml_context * ctx) override {
ggml_tensor * a = ggml_new_tensor(ctx, GGML_TYPE_F32, 4, ne.data());
ggml_tensor * b = ggml_new_tensor(ctx, GGML_TYPE_F32, 4, ne.data());
ggml_tensor * w = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, ne[0]);
ggml_set_name(a, "a");
ggml_set_name(b, "b");
ggml_set_name(w, "w");
sum = ggml_add(ctx, a, b);
ggml_set_name(sum, "sum");
ggml_tensor * normed = ggml_mul(ctx, ggml_rms_norm(ctx, sum, eps), w);
ggml_set_name(normed, "normed");
out = ggml_add(ctx, normed, sum);
ggml_set_name(out, "out");
return out;
}
void initialize_tensors(ggml_context * ctx) override {
for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != NULL; t = ggml_get_next_tensor(ctx, t)) {
init_tensor_uniform(t, -10.f, 10.f);
}
}
};
// GGML_OP_UNARY(RELU) + GGML_OP_SQR (fused operation)
struct test_relu_sqr : public test_case {
const ggml_type type;
@@ -11801,6 +11847,13 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() {
}
}
// decode-shaped residual rows: below, at and above the fused path's limits, odd row counts, batches
for (int64_t n : { 1020, 1024, 4096, 5120, 8192, 8196, 16384 }) {
for (int64_t rows : { 1, 3, 4 }) {
test_cases.emplace_back(new test_add_rms_norm_mul({ n, rows, 1, 1 }, 1e-6f));
}
test_cases.emplace_back(new test_add_rms_norm_mul({ n, 2, 3, 1 }, 1e-6f));
}
for (uint32_t n : {64, 1025}) {
test_cases.emplace_back(new test_add_add(GGML_TYPE_F32, GGML_TYPE_F32, { n, 5, 4, 3 }, false, false));
test_cases.emplace_back(new test_add_add(GGML_TYPE_F32, GGML_TYPE_F32, { n, 5, 4, 3 }, true, false));