sycl: add fused ADD, RMS_NORM, MUL and vectorize rms norm (#31753)
* sycl: vectorize contiguous rms norm * sycl: add fused ADD, RMS_NORM, MUL * tests: test fused ADD, RMS_NORM, MUL at decode widths
This commit is contained in:
1 parent
b019be0c86
commit
a818a46f3d
5 files changed
+262
-14
No files matched your search
@@ -4351,6 +4351,52 @@ struct test_add_rms_norm : public test_case {
|
||||
}
|
||||
};
|
||||
|
||||
// GGML_OP_ADD + GGML_OP_RMS_NORM + GGML_OP_MUL (fused operation)
|
||||
struct test_add_rms_norm_mul : public test_case {
|
||||
const std::array<int64_t, 4> ne;
|
||||
const float eps;
|
||||
|
||||
ggml_tensor * sum = nullptr;
|
||||
ggml_tensor * out = nullptr;
|
||||
|
||||
std::string op_desc(ggml_tensor * t) override {
|
||||
GGML_UNUSED(t);
|
||||
return "ADD_RMS_NORM_MUL";
|
||||
}
|
||||
|
||||
bool run_whole_graph() override { return true; }
|
||||
|
||||
std::vector<ggml_tensor *> fusion_test_nodes() override { return { sum, out }; }
|
||||
|
||||
std::string vars() override { return VARS_TO_STR2(ne, eps); }
|
||||
|
||||
test_add_rms_norm_mul(std::array<int64_t, 4> ne = { 4096, 1, 1, 1 }, float eps = 1e-6f) : ne(ne), eps(eps) {}
|
||||
|
||||
ggml_tensor * build_graph(ggml_context * ctx) override {
|
||||
ggml_tensor * a = ggml_new_tensor(ctx, GGML_TYPE_F32, 4, ne.data());
|
||||
ggml_tensor * b = ggml_new_tensor(ctx, GGML_TYPE_F32, 4, ne.data());
|
||||
ggml_tensor * w = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, ne[0]);
|
||||
ggml_set_name(a, "a");
|
||||
ggml_set_name(b, "b");
|
||||
ggml_set_name(w, "w");
|
||||
|
||||
sum = ggml_add(ctx, a, b);
|
||||
ggml_set_name(sum, "sum");
|
||||
ggml_tensor * normed = ggml_mul(ctx, ggml_rms_norm(ctx, sum, eps), w);
|
||||
ggml_set_name(normed, "normed");
|
||||
|
||||
out = ggml_add(ctx, normed, sum);
|
||||
ggml_set_name(out, "out");
|
||||
return out;
|
||||
}
|
||||
|
||||
void initialize_tensors(ggml_context * ctx) override {
|
||||
for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != NULL; t = ggml_get_next_tensor(ctx, t)) {
|
||||
init_tensor_uniform(t, -10.f, 10.f);
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
// GGML_OP_UNARY(RELU) + GGML_OP_SQR (fused operation)
|
||||
struct test_relu_sqr : public test_case {
|
||||
const ggml_type type;
|
||||
@@ -11801,6 +11847,13 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() {
|
||||
}
|
||||
}
|
||||
|
||||
// decode-shaped residual rows: below, at and above the fused path's limits, odd row counts, batches
|
||||
for (int64_t n : { 1020, 1024, 4096, 5120, 8192, 8196, 16384 }) {
|
||||
for (int64_t rows : { 1, 3, 4 }) {
|
||||
test_cases.emplace_back(new test_add_rms_norm_mul({ n, rows, 1, 1 }, 1e-6f));
|
||||
}
|
||||
test_cases.emplace_back(new test_add_rms_norm_mul({ n, 2, 3, 1 }, 1e-6f));
|
||||
}
|
||||
for (uint32_t n : {64, 1025}) {
|
||||
test_cases.emplace_back(new test_add_add(GGML_TYPE_F32, GGML_TYPE_F32, { n, 5, 4, 3 }, false, false));
|
||||
test_cases.emplace_back(new test_add_add(GGML_TYPE_F32, GGML_TYPE_F32, { n, 5, 4, 3 }, true, false));
|
||||
|
||||
Reference in new issue
Block a user