sycl: add fused ADD, RMS_NORM, MUL and vectorize rms norm (#31753)
* sycl: vectorize contiguous rms norm * sycl: add fused ADD, RMS_NORM, MUL * tests: test fused ADD, RMS_NORM, MUL at decode widths
This commit is contained in:
1 parent
b019be0c86
commit
a818a46f3d
5 files changed
+262
-14
No files matched your search
@@ -121,6 +121,28 @@ bool ggml_sycl_can_fuse(const ggml_cgraph * cgraph, int node_idx, std::initializ
|
||||
return ggml_sycl_should_fuse_mul_mat_glu(gate, up, glu);
|
||||
}
|
||||
|
||||
// the residual sum stays materialised (a later residual add reads it), so ggml_can_fuse cannot
|
||||
// express this either: the add and the mul are the subgraph's outputs
|
||||
if (ops.size() == 3 && ops.begin()[0] == GGML_OP_ADD && ops.begin()[1] == GGML_OP_RMS_NORM &&
|
||||
ops.begin()[2] == GGML_OP_MUL) {
|
||||
if (!ggml_can_fuse_subgraph(cgraph, node_idx, ops, { node_idx, node_idx + 2 })) {
|
||||
return false;
|
||||
}
|
||||
|
||||
const ggml_tensor * add = cgraph->nodes[node_idx];
|
||||
if (cgraph->nodes[node_idx + 1]->src[0] != add) {
|
||||
return false;
|
||||
}
|
||||
|
||||
// the fused kernel does not broadcast the add
|
||||
if (add->src[0]->type != GGML_TYPE_F32 || add->src[1]->type != GGML_TYPE_F32 || add->type != GGML_TYPE_F32 ||
|
||||
!ggml_are_same_shape(add->src[0], add) || !ggml_are_same_shape(add->src[1], add)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
return ggml_sycl_can_fuse(cgraph, node_idx + 1, { GGML_OP_RMS_NORM, GGML_OP_MUL }, {});
|
||||
}
|
||||
|
||||
if (!ggml_can_fuse(cgraph, node_idx, ops)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
@@ -6708,6 +6708,12 @@ static void ggml_backend_sycl_graph_compute_impl(ggml_backend_sycl_context * syc
|
||||
continue;
|
||||
}
|
||||
}
|
||||
if (node->op == GGML_OP_ADD &&
|
||||
ggml_sycl_can_fuse(cgraph, i, { GGML_OP_ADD, GGML_OP_RMS_NORM, GGML_OP_MUL }, {}) &&
|
||||
ggml_sycl_op_add_rms_norm_fused(*sycl_ctx, node, cgraph->nodes[i + 1], cgraph->nodes[i + 2])) {
|
||||
i += 2;
|
||||
continue;
|
||||
}
|
||||
if (node->op == GGML_OP_RMS_NORM &&
|
||||
ggml_sycl_can_fuse(cgraph, i, { GGML_OP_RMS_NORM, GGML_OP_MUL, GGML_OP_ADD }, {})) {
|
||||
ggml_sycl_op_rms_norm_fused_add(*sycl_ctx, node, cgraph->nodes[i + 1], cgraph->nodes[i + 2]);
|
||||
|
||||
+176
-14
@@ -1,4 +1,6 @@
|
||||
#include "norm.hpp"
|
||||
|
||||
#include "fusion.hpp"
|
||||
#include "ggml-sycl/common.hpp"
|
||||
#include "ggml-sycl/presets.hpp"
|
||||
|
||||
@@ -188,9 +190,28 @@ static void rms_norm_f32(const float* x, float* dst, const int ncols,
|
||||
|
||||
float tmp = 0.0f; // partial sum for thread in warp
|
||||
|
||||
for (int col = tid; col < ncols; col += block_size) {
|
||||
const float xi = x[col * src_stride_col];
|
||||
tmp += xi * xi;
|
||||
// float4 path for contiguous, 16-byte aligned rows (decode), views and odd lengths stay strided
|
||||
bool contig = src_stride_col == 1 && dst_stride_col == 1 && ncols % 4 == 0 && (uintptr_t) x % 16 == 0 &&
|
||||
(uintptr_t) dst % 16 == 0;
|
||||
if constexpr (do_multiply) {
|
||||
contig = contig && (uintptr_t) mul % 16 == 0;
|
||||
}
|
||||
if constexpr (do_add) {
|
||||
contig = contig && (uintptr_t) add % 16 == 0;
|
||||
}
|
||||
const int n4 = ncols / 4;
|
||||
|
||||
if (contig) {
|
||||
const sycl::float4 * x4 = reinterpret_cast<const sycl::float4 *>(x);
|
||||
for (int i = tid; i < n4; i += block_size) {
|
||||
const sycl::float4 v = x4[i];
|
||||
tmp += sycl::dot(v, v);
|
||||
}
|
||||
} else {
|
||||
for (int col = tid; col < ncols; col += block_size) {
|
||||
const float xi = x[col * src_stride_col];
|
||||
tmp += xi * xi;
|
||||
}
|
||||
}
|
||||
|
||||
// sum up partial sums
|
||||
@@ -216,17 +237,37 @@ static void rms_norm_f32(const float* x, float* dst, const int ncols,
|
||||
const float mean = tmp / ncols;
|
||||
const float scale = sycl::rsqrt(mean + eps);
|
||||
|
||||
for (int col = tid; col < ncols; col += block_size) {
|
||||
if constexpr (do_multiply && do_add) {
|
||||
dst[col * dst_stride_col] = scale * x[col * src_stride_col] * mul[col] + add[col];
|
||||
} else if constexpr (do_multiply) {
|
||||
dst[col * dst_stride_col] = scale * x[col * src_stride_col] * mul[col];
|
||||
} else {
|
||||
// folded epilogue of a fused GGML_OP_SCALE consumer (qwen35 GDN l2 norms);
|
||||
// the explicit temporary keeps the float evaluation order identical to
|
||||
// running rms_norm and scale as two separate kernels
|
||||
const float v = scale * x[col * src_stride_col];
|
||||
dst[col * dst_stride_col] = v * scale_mul;
|
||||
if (contig) {
|
||||
const sycl::float4 * x4 = reinterpret_cast<const sycl::float4 *>(x);
|
||||
sycl::float4 * d4 = reinterpret_cast<sycl::float4 *>(dst);
|
||||
const sycl::float4 * m4 = reinterpret_cast<const sycl::float4 *>(mul);
|
||||
const sycl::float4 * a4 = reinterpret_cast<const sycl::float4 *>(add);
|
||||
for (int i = tid; i < n4; i += block_size) {
|
||||
const sycl::float4 v = x4[i];
|
||||
if constexpr (do_multiply && do_add) {
|
||||
const sycl::float4 m = m4[i];
|
||||
const sycl::float4 a = a4[i];
|
||||
d4[i] = scale * v * m + a;
|
||||
} else if constexpr (do_multiply) {
|
||||
const sycl::float4 m = m4[i];
|
||||
d4[i] = scale * v * m;
|
||||
} else {
|
||||
d4[i] = scale * v * scale_mul;
|
||||
}
|
||||
}
|
||||
} else {
|
||||
for (int col = tid; col < ncols; col += block_size) {
|
||||
if constexpr (do_multiply && do_add) {
|
||||
dst[col * dst_stride_col] = scale * x[col * src_stride_col] * mul[col] + add[col];
|
||||
} else if constexpr (do_multiply) {
|
||||
dst[col * dst_stride_col] = scale * x[col * src_stride_col] * mul[col];
|
||||
} else {
|
||||
// folded epilogue of a fused GGML_OP_SCALE consumer (qwen35 GDN l2 norms);
|
||||
// the explicit temporary keeps the float evaluation order identical to
|
||||
// running rms_norm and scale as two separate kernels
|
||||
const float v = scale * x[col * src_stride_col];
|
||||
dst[col * dst_stride_col] = v * scale_mul;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -918,6 +959,127 @@ void ggml_sycl_op_rms_norm_fused_add(ggml_backend_sycl_context & ctx, ggml_tenso
|
||||
add_s01, add_s02, add_s03, add_nrows, add_nchannels, add_nsamples, eps, main_stream, ctx.device);
|
||||
}
|
||||
|
||||
// ADD(a, b) -> RMS_NORM -> MUL(w) in one launch. The sum (read by a later residual add) and
|
||||
// normed * w are written after a barrier, as mul->data may alias a.
|
||||
static void add_rms_norm_mul_f32(const float * a,
|
||||
const float * b,
|
||||
float * sum,
|
||||
const float * mul,
|
||||
float * dst,
|
||||
const int ncols,
|
||||
const float eps,
|
||||
const sycl::nd_item<3> & item_ct1,
|
||||
float * s_sum,
|
||||
const int block_size) {
|
||||
const int row = item_ct1.get_group(2);
|
||||
const size_t off = (size_t) row * ncols;
|
||||
const int tid = item_ct1.get_local_id(2);
|
||||
const int n4 = ncols / 4;
|
||||
|
||||
const sycl::float4 * a4 = reinterpret_cast<const sycl::float4 *>(a + off);
|
||||
const sycl::float4 * b4 = reinterpret_cast<const sycl::float4 *>(b + off);
|
||||
|
||||
sycl::float4 v[2];
|
||||
float tmp = 0.0f;
|
||||
#pragma unroll
|
||||
for (int k = 0; k < 2; ++k) {
|
||||
const int i = tid + k * block_size;
|
||||
if (i < n4) {
|
||||
const sycl::float4 va = a4[i];
|
||||
const sycl::float4 vb = b4[i];
|
||||
v[k] = va + vb;
|
||||
tmp += sycl::dot(v[k], v[k]);
|
||||
}
|
||||
}
|
||||
|
||||
const int nwarps = block_size / WARP_SIZE;
|
||||
tmp = warp_reduce_sum(tmp, item_ct1);
|
||||
if (block_size > WARP_SIZE) {
|
||||
const auto sub_group = item_ct1.get_sub_group();
|
||||
if (sub_group.get_local_linear_id() == 0) {
|
||||
s_sum[sub_group.get_group_linear_id()] = tmp;
|
||||
}
|
||||
item_ct1.barrier(sycl::access::fence_space::local_space);
|
||||
const size_t nreduce = ceil_div(nwarps, WARP_SIZE);
|
||||
tmp = 0.0f;
|
||||
for (size_t i = 0; i < nreduce; i += 1) {
|
||||
tmp += s_sum[sub_group.get_local_linear_id() + i * WARP_SIZE];
|
||||
}
|
||||
tmp = warp_reduce_sum(tmp, item_ct1);
|
||||
}
|
||||
|
||||
const float mean = tmp / ncols;
|
||||
const float scale = sycl::rsqrt(mean + eps);
|
||||
|
||||
sycl::float4 * s4 = reinterpret_cast<sycl::float4 *>(sum + off);
|
||||
sycl::float4 * d4 = reinterpret_cast<sycl::float4 *>(dst + off);
|
||||
const sycl::float4 * m4 = reinterpret_cast<const sycl::float4 *>(mul);
|
||||
#pragma unroll
|
||||
for (int k = 0; k < 2; ++k) {
|
||||
const int i = tid + k * block_size;
|
||||
if (i < n4) {
|
||||
const sycl::float4 m = m4[i];
|
||||
const sycl::float4 u = scale * v[k] * m;
|
||||
s4[i] = v[k];
|
||||
d4[i] = u;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
static void add_rms_norm_mul_f32_sycl(const float * a,
|
||||
const float * b,
|
||||
float * sum,
|
||||
const float * mul,
|
||||
float * dst,
|
||||
const int ncols,
|
||||
const int nrows,
|
||||
const float eps,
|
||||
queue_ptr stream,
|
||||
int device) {
|
||||
const int work_group_size = ggml_sycl_info().max_work_group_sizes[device];
|
||||
GGML_ASSERT(work_group_size % (WARP_SIZE * WARP_SIZE) == 0);
|
||||
const sycl::range<3> global_dims(1, 1, nrows);
|
||||
const sycl::range<3> block_dims(1, 1, work_group_size);
|
||||
stream->submit([&](sycl::handler & cgh) {
|
||||
sycl::local_accessor<float, 1> s_sum(sycl::range<1>(work_group_size / WARP_SIZE), cgh);
|
||||
cgh.parallel_for(sycl::nd_range<3>(global_dims * block_dims, block_dims),
|
||||
[=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] {
|
||||
add_rms_norm_mul_f32(a, b, sum, mul, dst, ncols, eps, item_ct1, get_pointer(s_sum),
|
||||
work_group_size);
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
bool ggml_sycl_op_add_rms_norm_fused(ggml_backend_sycl_context & ctx,
|
||||
ggml_tensor * add,
|
||||
ggml_tensor * rms,
|
||||
ggml_tensor * mul) {
|
||||
ggml_tensor * a = add->src[0];
|
||||
ggml_tensor * b = add->src[1];
|
||||
ggml_tensor * w = mul->src[0] == rms ? mul->src[1] : mul->src[0];
|
||||
const int64_t ncols = add->ne[0];
|
||||
const int wg = ggml_sycl_info().max_work_group_sizes[ctx.device];
|
||||
// whole rows of float4, at most two per work-item, from 16-byte aligned contiguous tensors
|
||||
if (add->ne[2] != 1 || add->ne[3] != 1 || ncols < 1024 || ncols % 4 != 0 || ncols > 8 * (int64_t) wg ||
|
||||
w->ne[1] != 1) {
|
||||
return false;
|
||||
}
|
||||
for (const ggml_tensor * t : { a, b, add, mul, w }) {
|
||||
if (!ggml_is_contiguous(t) || (uintptr_t) t->data % 16 != 0) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
float eps = 0.0f;
|
||||
memcpy(&eps, rms->op_params, sizeof(float));
|
||||
if (!(eps >= 0.0f)) {
|
||||
return false;
|
||||
}
|
||||
add_rms_norm_mul_f32_sycl((const float *) a->data, (const float *) b->data, (float *) add->data,
|
||||
(const float *) w->data, (float *) mul->data, (int) ncols, (int) add->ne[1], eps,
|
||||
ctx.stream(), ctx.device);
|
||||
return true;
|
||||
}
|
||||
|
||||
void ggml_sycl_op_rms_norm_back(ggml_backend_sycl_context & ctx, ggml_tensor * dst) {
|
||||
scope_op_debug_print scope_dbg_print(__func__, dst, /*num_src=*/2);
|
||||
|
||||
|
||||
@@ -25,6 +25,11 @@ void ggml_sycl_op_rms_norm_scale_fused(ggml_backend_sycl_context& ctx, ggml_tens
|
||||
|
||||
void ggml_sycl_op_rms_norm_fused_add(ggml_backend_sycl_context& ctx, ggml_tensor* dst, ggml_tensor* mul_tensor, ggml_tensor* add_tensor);
|
||||
|
||||
bool ggml_sycl_op_add_rms_norm_fused(ggml_backend_sycl_context & ctx,
|
||||
ggml_tensor * add,
|
||||
ggml_tensor * rms,
|
||||
ggml_tensor * mul);
|
||||
|
||||
void ggml_sycl_op_rms_norm_back(ggml_backend_sycl_context& ctx, ggml_tensor* dst);
|
||||
|
||||
void ggml_sycl_op_group_norm(ggml_backend_sycl_context& ctx, ggml_tensor* dst);
|
||||
|
||||
@@ -4351,6 +4351,52 @@ struct test_add_rms_norm : public test_case {
|
||||
}
|
||||
};
|
||||
|
||||
// GGML_OP_ADD + GGML_OP_RMS_NORM + GGML_OP_MUL (fused operation)
|
||||
struct test_add_rms_norm_mul : public test_case {
|
||||
const std::array<int64_t, 4> ne;
|
||||
const float eps;
|
||||
|
||||
ggml_tensor * sum = nullptr;
|
||||
ggml_tensor * out = nullptr;
|
||||
|
||||
std::string op_desc(ggml_tensor * t) override {
|
||||
GGML_UNUSED(t);
|
||||
return "ADD_RMS_NORM_MUL";
|
||||
}
|
||||
|
||||
bool run_whole_graph() override { return true; }
|
||||
|
||||
std::vector<ggml_tensor *> fusion_test_nodes() override { return { sum, out }; }
|
||||
|
||||
std::string vars() override { return VARS_TO_STR2(ne, eps); }
|
||||
|
||||
test_add_rms_norm_mul(std::array<int64_t, 4> ne = { 4096, 1, 1, 1 }, float eps = 1e-6f) : ne(ne), eps(eps) {}
|
||||
|
||||
ggml_tensor * build_graph(ggml_context * ctx) override {
|
||||
ggml_tensor * a = ggml_new_tensor(ctx, GGML_TYPE_F32, 4, ne.data());
|
||||
ggml_tensor * b = ggml_new_tensor(ctx, GGML_TYPE_F32, 4, ne.data());
|
||||
ggml_tensor * w = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, ne[0]);
|
||||
ggml_set_name(a, "a");
|
||||
ggml_set_name(b, "b");
|
||||
ggml_set_name(w, "w");
|
||||
|
||||
sum = ggml_add(ctx, a, b);
|
||||
ggml_set_name(sum, "sum");
|
||||
ggml_tensor * normed = ggml_mul(ctx, ggml_rms_norm(ctx, sum, eps), w);
|
||||
ggml_set_name(normed, "normed");
|
||||
|
||||
out = ggml_add(ctx, normed, sum);
|
||||
ggml_set_name(out, "out");
|
||||
return out;
|
||||
}
|
||||
|
||||
void initialize_tensors(ggml_context * ctx) override {
|
||||
for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != NULL; t = ggml_get_next_tensor(ctx, t)) {
|
||||
init_tensor_uniform(t, -10.f, 10.f);
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
// GGML_OP_UNARY(RELU) + GGML_OP_SQR (fused operation)
|
||||
struct test_relu_sqr : public test_case {
|
||||
const ggml_type type;
|
||||
@@ -11801,6 +11847,13 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() {
|
||||
}
|
||||
}
|
||||
|
||||
// decode-shaped residual rows: below, at and above the fused path's limits, odd row counts, batches
|
||||
for (int64_t n : { 1020, 1024, 4096, 5120, 8192, 8196, 16384 }) {
|
||||
for (int64_t rows : { 1, 3, 4 }) {
|
||||
test_cases.emplace_back(new test_add_rms_norm_mul({ n, rows, 1, 1 }, 1e-6f));
|
||||
}
|
||||
test_cases.emplace_back(new test_add_rms_norm_mul({ n, 2, 3, 1 }, 1e-6f));
|
||||
}
|
||||
for (uint32_t n : {64, 1025}) {
|
||||
test_cases.emplace_back(new test_add_add(GGML_TYPE_F32, GGML_TYPE_F32, { n, 5, 4, 3 }, false, false));
|
||||
test_cases.emplace_back(new test_add_add(GGML_TYPE_F32, GGML_TYPE_F32, { n, 5, 4, 3 }, true, false));
|
||||
|
||||
Reference in new issue
Block a user