Lumpiasty/llama.cpp
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

sycl: add a batched L2_NORM kernel (#28222)

Browse Source 
* sycl: add a batched L2_NORM kernel

* sycl: batch consecutive L2_NORM siblings in the graph dispatch

Measured on Intel Arc Pro B70 (Battlemage), Qwen3.6-27B Q4_K_M, f16 KV,
npp=128 ntg=128 npl=2, GGML_SYCL profiler:

    L2_NORM dispatches       12480 -> 6240
    L2_NORM device time      68.77 -> 39.14 ms   (-43%)
    total device time        6782 -> 6748 ms     (-0.5%)
    wall decode t/s          flat

* tests: add L2_NORM_BATCH coverage
This commit is contained in:
Titaniumtown
2026-09-07 15:24:14 +02:00
committed by GitHub
parent 7a333e7240
commit dbeb37548e
4 changed files with 226 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/test-backend-ops.cpp
+57
View File
@@ -7206,6 +7206,49 @@ struct test_group_norm_mul_add : public test_case {
}
};
// GGML_OP_L2_NORM x N: independent same-shape norms in one graph (strided qkv views or
// contiguous), consuming adds nested so the norms stay adjacent in the graph.
struct test_l2_norm_batch : public test_case {
const ggml_type type;
const std::array<int64_t, 4> ne;
const int n_norms;
const float eps;
const bool strided;
std::string vars() override { return VARS_TO_STR5(type, ne, n_norms, eps, strided); }
std::string op_desc(ggml_tensor * t) override { GGML_UNUSED(t); return "L2_NORM_BATCH"; }
bool run_whole_graph() override { return true; }
test_l2_norm_batch(ggml_type type = GGML_TYPE_F32, std::array<int64_t, 4> ne = { 128, 16, 16, 1 },
int n_norms = 4, float eps = 1e-12f, bool strided = true)
: type(type), ne(ne), n_norms(n_norms), eps(eps), strided(strided) {}
ggml_tensor * build_graph(ggml_context * ctx) override {
GGML_ASSERT(n_norms >= 2 && n_norms <= 8);
ggml_tensor * parent = nullptr;
if (strided) {
parent = ggml_new_tensor_4d(ctx, type, ne[0], ne[1] * n_norms, ne[2], ne[3]); // qkv buffer
}
ggml_tensor * norms[8];
for (int t = 0; t < n_norms; ++t) {
ggml_tensor * src;
if (strided) {
src = ggml_view_4d(ctx, parent, ne[0], ne[1], ne[2], ne[3], parent->nb[1], parent->nb[2],
parent->nb[3], t * ne[1] * parent->nb[1]);
} else {
src = ggml_new_tensor(ctx, type, 4, ne.data());
}
norms[t] = ggml_l2_norm(ctx, src, eps);
}
ggml_tensor * out = norms[n_norms - 1];
for (int t = n_norms - 2; t >= 0; --t) {
out = ggml_add(ctx, norms[t], out);
}
ggml_set_name(out, "out");
return out;
}
};
// GGML_OP_L2_NORM
struct test_l2_norm : public test_case {
const ggml_type type;
@@ -9495,6 +9538,10 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() {
test_cases.emplace_back(new test_l2_norm(GGML_TYPE_F32, { n, 5, 4, 3 }, eps, false));
test_cases.emplace_back(new test_l2_norm(GGML_TYPE_F32, { n, 5, 4, 3 }, eps, true));
test_cases.emplace_back(new test_l2_norm(GGML_TYPE_F32, { n, 5, 4, 3 }, eps, false, true));
// sibling batching: strided (production shape) and contiguous, 2 and 4 wide
test_cases.emplace_back(new test_l2_norm_batch(GGML_TYPE_F32, { n, 5, 4, 3 }, 2, eps, true));
test_cases.emplace_back(new test_l2_norm_batch(GGML_TYPE_F32, { n, 5, 4, 3 }, 4, eps, true));
test_cases.emplace_back(new test_l2_norm_batch(GGML_TYPE_F32, { n, 5, 4, 3 }, 4, eps, false));
}
// row lengths that are not a multiple of 32, for the scalar (33) and float4 (132, 260) paths
for (uint32_t n : { 33, 132, 260 }) {
@@ -11181,6 +11228,16 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_perf() {
}
}
// launch-overhead isolation: single L2_NORM launch vs batched siblings at the GDN
// production shape (strided qkv views) -- perf-mode only, the eval list has its own
// 2/4-wide coverage
for (int n : { 128, 256 }) {
test_cases.emplace_back(new test_l2_norm(GGML_TYPE_F32, { n, 16, 16, 1 }, 1e-12f, false, false));
test_cases.emplace_back(new test_l2_norm_batch(GGML_TYPE_F32, { n, 16, 16, 1 }, 2, 1e-12f, true));
test_cases.emplace_back(new test_l2_norm_batch(GGML_TYPE_F32, { n, 16, 16, 1 }, 4, 1e-12f, true));
}
return test_cases;
}