Lumpiasty/llama.cpp
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

sycl: fuse rms_norm+mul+add and add+add residual chains (#27610)

Browse Source 
Fuse RMS_NORM+MUL+ADD and ADD+ADD under GGML_SYCL_ENABLE_FUSION.

ADD+ADD uses the same binbcast indexing and type matrix as standalone
add() (f32, f16, f16/f32, i32, i16, bf16, including broadcast and
non-contiguous). Unsupported combinations fall back to two add() launches.
This commit is contained in:
Frosty40
2026-09-04 00:05:40 -04:00
committed by GitHub
parent f9f09f02cc
commit 6703d7894c
8 changed files with 593 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/test-backend-ops.cpp
+64
View File
@@ -3700,6 +3700,60 @@ struct test_rms_norm_mul_add : public test_case {
}
};
// GGML_OP_ADD + GGML_OP_ADD (fused residual chain)
struct test_add_add : public test_case {
const ggml_type type;
const ggml_type type_addend;
const std::array<int64_t, 4> ne;
const bool broadcast;
const bool view; // non-contiguous a via view_4d
std::string op_desc(ggml_tensor * t) override {
GGML_UNUSED(t);
return "ADD_ADD";
}
bool run_whole_graph() override { return true; }
std::string vars() override {
return VARS_TO_STR5(type, type_addend, ne, broadcast, view);
}
test_add_add(ggml_type type = GGML_TYPE_F32,
ggml_type type_addend = GGML_TYPE_F32,
std::array<int64_t, 4> ne = {64, 5, 4, 3},
bool broadcast = false,
bool view = false)
: type(type), type_addend(type_addend), ne(ne), broadcast(broadcast), view(view) {}
ggml_tensor * build_graph(ggml_context * ctx) override {
std::array<int64_t, 4> broadcast_dims = {ne[0], 1, 1, 1};
ggml_tensor * a;
if (view) {
std::array<int64_t, 4> parent = { ne[0] * 3, ne[1] * 2, ne[2], ne[3] };
a = ggml_new_tensor(ctx, type, 4, parent.data());
ggml_set_name(a, "a_parent");
a = ggml_view_4d(ctx, a, ne[0], ne[1], ne[2], ne[3], a->nb[1], a->nb[2], a->nb[3], 0);
ggml_set_name(a, "a");
} else {
a = ggml_new_tensor(ctx, type, 4, ne.data());
ggml_set_name(a, "a");
}
ggml_tensor * b = ggml_new_tensor(ctx, type_addend, 4, ne.data());
ggml_tensor * c = ggml_new_tensor(ctx, type_addend, 4, broadcast ? broadcast_dims.data() : ne.data());
ggml_set_name(b, "b");
ggml_set_name(c, "c");
ggml_tensor * out = ggml_add(ctx, ggml_add(ctx, a, b), c);
ggml_set_name(out, "out");
return out;
}
};
// GGML_OP_ADD + GGML_OP_RMS_NORM (fused operation)
struct test_add_rms_norm : public test_case {
const ggml_type type;
@@ -9259,6 +9313,8 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() {
// fusion
test_cases.emplace_back(new test_bin_bcast(ggml_add, GGML_TYPE_F32, {10, 5, 4, 3}, {2, 1, 1, 1}, 2));
test_cases.emplace_back(new test_bin_bcast(ggml_add, GGML_TYPE_F16, {10, 5, 4, 3}, {2, 1, 1, 1}, 2));
test_cases.emplace_back(new test_bin_bcast(ggml_add, GGML_TYPE_F32, {16, 5, 4, 3}, {1, 1, 1, 1}, 2, true));
test_cases.emplace_back(new test_bin_bcast(ggml_add, GGML_TYPE_F32, {16, 5, 4, 3}, {1, 2, 1, 1}, 3));
test_cases.emplace_back(new test_bin_bcast(ggml_add, GGML_TYPE_F32, {10, 5, 4, 3}, {1, 1, 2, 1}, 4));
test_cases.emplace_back(new test_bin_bcast(ggml_add, GGML_TYPE_F32, {16, 5, 4, 3}, {1, 1, 1, 2}, 5));
@@ -9314,6 +9370,14 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() {
}
test_cases.emplace_back(new test_add_rms_norm(GGML_TYPE_F32, {n, 1, 1, 1}, 1e-6f, false));
}
for (uint32_t n : {64, 1025}) {
test_cases.emplace_back(new test_add_add(GGML_TYPE_F32, GGML_TYPE_F32, { n, 5, 4, 3 }, false, false));
test_cases.emplace_back(new test_add_add(GGML_TYPE_F32, GGML_TYPE_F32, { n, 5, 4, 3 }, true, false));
test_cases.emplace_back(new test_add_add(GGML_TYPE_F32, GGML_TYPE_F32, { n, 5, 4, 3 }, false, true));
test_cases.emplace_back(new test_add_add(GGML_TYPE_F16, GGML_TYPE_F16, { n, 5, 4, 3 }, false, false));
test_cases.emplace_back(new test_add_add(GGML_TYPE_F16, GGML_TYPE_F32, { n, 5, 4, 3 }, false, false));
test_cases.emplace_back(new test_add_add(GGML_TYPE_F16, GGML_TYPE_F32, { n, 5, 4, 3 }, true, false));
}
for (auto multi_add : {false, true}) {
for (auto set_rows : {false, true}) {