From 1548a240e36079f07856c95c53de1ec2770840ab Mon Sep 17 00:00:00 2001 From: Hongqiang Wang Date: Fri, 4 Sep 2026 10:12:26 -0700 Subject: [PATCH] =?UTF-8?q?opencl:=20extend=20the=20elementwise=20and=20da?= =?UTF-8?q?ta=E2=80=90movement=20op=20coverage=20(#27633)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * opencl: add extended elementwise unary ops (sgn, step, elu, hardswish, hardsigmoid, floor, ceil, round, trunc) Adds nine GGML_UNARY_OP_* elementwise ops that were falling back to CPU on the OpenCL backend, following the same variant shape as the existing ABS op: f32, f32_4 (vec4), f16, f16_4 (vec4), and stride-addressed f32_nc / f16_nc for non-contiguous inputs. New kernels/unary_ext.cl (macro-generated), a shared ggml_cl_unary_ext dispatch helper mirroring ggml_cl_abs, the supports_op cases, and the compute-forward cases. Values are computed in float (the f16 variants read/write half and convert), so the conditional ops (step, elu) match the CPU reference; the vec4 forms use select() for the branch. Validated with test-backend-ops on Adreno 840 and 850 (E17): all nine ops pass every case including the vec4 and non-contiguous variants (8/8 or 14/14). * opencl: dispatch a contiguous f32 copy over the whole device kernel_cpy_f32_f32 maps one workgroup to each (i01,i02,i03) row and strides the row across that workgroup's lanes, and the host launches ne01*MIN(64,ne00) work items. A tensor with few long rows therefore runs on a single workgroup. The mamba2 and gated-delta-net recurrent state cache is one row of 524288 floats, copied once per layer per graph, and lands on 64 work items. When both sides are contiguous the copy is a linear move, so dispatch it over the whole device: one work item per float4. Gated on ggml_is_contiguous for both tensors and equal element counts, so copies already spread over many rows keep the existing path. The kernel is created optionally, so a driver that rejects it falls back rather than aborting. vload4/vstore4 rather than a float4 cast: they require only the scalar type's alignment, and these buffers carry an arbitrary 4-byte view offset. CPY, DUP and CONT are 217/217 on Adreno 840 and 740 with the path enabled and disabled. GGML_OPENCL_CPY_FLAT=0 forces the old kernel. * opencl: support all easy-copy types in CONCAT CONCAT was F32-only. Extend it to every "easy-copy" type -- any non-quantized type with a block size of 1 and an element size of 1, 2, 4 or 8 bytes, i.e. f16/bf16/i8/i16/i32/i64 as well as f32. The kernels are keyed by element SIZE rather than by type, which is what CUDA already does for the same op: one kernel per byte width (b1/b2/b4/b8) plus the packed b4 fast path, instead of one per ggml type. supports_op gates on the same property, so a new type of a supported width is picked up with no further work. Validated with test-backend-ops on Adreno 840 / A8X and X2-90 / X2E. --- ggml/src/ggml-opencl/CMakeLists.txt | 1 + ggml/src/ggml-opencl/ggml-opencl.cpp | 257 +++++++++++++++++++++- ggml/src/ggml-opencl/kernels/concat.cl | 118 +++++----- ggml/src/ggml-opencl/kernels/cpy.cl | 25 +++ ggml/src/ggml-opencl/kernels/unary_ext.cl | 85 +++++++ 5 files changed, 422 insertions(+), 64 deletions(-) create mode 100644 ggml/src/ggml-opencl/kernels/unary_ext.cl diff --git a/ggml/src/ggml-opencl/CMakeLists.txt b/ggml/src/ggml-opencl/CMakeLists.txt index 8e6e374f3..37e565ef4 100644 --- a/ggml/src/ggml-opencl/CMakeLists.txt +++ b/ggml/src/ggml-opencl/CMakeLists.txt @@ -222,6 +222,7 @@ set(GGML_OPENCL_KERNELS exp expm1 abs + unary_ext softplus pad repeat diff --git a/ggml/src/ggml-opencl/ggml-opencl.cpp b/ggml/src/ggml-opencl/ggml-opencl.cpp index 864da2536..ad4a995ab 100644 --- a/ggml/src/ggml-opencl/ggml-opencl.cpp +++ b/ggml/src/ggml-opencl/ggml-opencl.cpp @@ -826,6 +826,7 @@ struct ggml_backend_opencl_context { cl_kernel kernel_rope_norm_f32, kernel_rope_norm_f16, kernel_rope_neox_f32, kernel_rope_neox_f16; cl_kernel kernel_rope_multi_f32, kernel_rope_multi_f16, kernel_rope_vision_f32, kernel_rope_vision_f16; cl_kernel kernel_cpy_f16_f16, kernel_cpy_f16_f32, kernel_cpy_f32_f16, kernel_cpy_f32_f32, kernel_cpy_f32_f32_pack, kernel_cpy_i32_i32; + cl_kernel kernel_cpy_f32_f32_flat = nullptr; cl_kernel kernel_mul_mat_f32_f32; cl_kernel kernel_mul_mat_f16_f16; cl_kernel kernel_mul_mat_f16_f32_1row; @@ -932,11 +933,20 @@ struct ggml_backend_opencl_context { cl_kernel kernel_expm1_f16, kernel_expm1_f16_4, kernel_expm1_f16_nc; cl_kernel kernel_abs_f32, kernel_abs_f32_4, kernel_abs_f32_nc; cl_kernel kernel_abs_f16, kernel_abs_f16_4, kernel_abs_f16_nc; + cl_kernel kernel_sgn_f32, kernel_sgn_f32_4, kernel_sgn_f32_nc, kernel_sgn_f16, kernel_sgn_f16_4, kernel_sgn_f16_nc; + cl_kernel kernel_step_f32, kernel_step_f32_4, kernel_step_f32_nc, kernel_step_f16, kernel_step_f16_4, kernel_step_f16_nc; + cl_kernel kernel_elu_f32, kernel_elu_f32_4, kernel_elu_f32_nc, kernel_elu_f16, kernel_elu_f16_4, kernel_elu_f16_nc; + cl_kernel kernel_hardswish_f32, kernel_hardswish_f32_4, kernel_hardswish_f32_nc, kernel_hardswish_f16, kernel_hardswish_f16_4, kernel_hardswish_f16_nc; + cl_kernel kernel_hardsigmoid_f32, kernel_hardsigmoid_f32_4, kernel_hardsigmoid_f32_nc, kernel_hardsigmoid_f16, kernel_hardsigmoid_f16_4, kernel_hardsigmoid_f16_nc; + cl_kernel kernel_floor_f32, kernel_floor_f32_4, kernel_floor_f32_nc, kernel_floor_f16, kernel_floor_f16_4, kernel_floor_f16_nc; + cl_kernel kernel_ceil_f32, kernel_ceil_f32_4, kernel_ceil_f32_nc, kernel_ceil_f16, kernel_ceil_f16_4, kernel_ceil_f16_nc; + cl_kernel kernel_round_f32, kernel_round_f32_4, kernel_round_f32_nc, kernel_round_f16, kernel_round_f16_4, kernel_round_f16_nc; + cl_kernel kernel_trunc_f32, kernel_trunc_f32_4, kernel_trunc_f32_nc, kernel_trunc_f16, kernel_trunc_f16_4, kernel_trunc_f16_nc; cl_kernel kernel_softplus_f32, kernel_softplus_f32_4, kernel_softplus_f32_nc; cl_kernel kernel_softplus_f16, kernel_softplus_f16_4, kernel_softplus_f16_nc; cl_kernel kernel_upscale; cl_kernel kernel_upscale_bilinear; - cl_kernel kernel_concat_f32, kernel_concat_f32_pack; + cl_kernel kernel_concat_b1, kernel_concat_b2, kernel_concat_b4, kernel_concat_b8, kernel_concat_b4_pack; cl_kernel kernel_conv_2d_f16; cl_kernel kernel_conv_2d_f32; cl_kernel kernel_conv_2d_f16_f32; @@ -1537,6 +1547,13 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) { CL_CHECK((backend_ctx->kernel_cpy_f32_f16 = clCreateKernel(prog, "kernel_cpy_f32_f16", &err), err)); CL_CHECK((backend_ctx->kernel_cpy_f32_f32 = clCreateKernel(prog, "kernel_cpy_f32_f32", &err), err)); CL_CHECK((backend_ctx->kernel_cpy_f32_f32_pack = clCreateKernel(prog, "kernel_cpy_f32_f32_pack", &err), err)); + { // optional: without it ggml_cl_cpy keeps the row-mapped kernel + cl_int err_flat = CL_SUCCESS; + cl_kernel k = clCreateKernel(prog, "kernel_cpy_f32_f32_flat", &err_flat); + if (err_flat == CL_SUCCESS) { + backend_ctx->kernel_cpy_f32_f32_flat = k; + } + } CL_CHECK((backend_ctx->kernel_cpy_i32_i32 = clCreateKernel(prog, "kernel_cpy_i32_i32", &err), err)); GGML_LOG_CONT("."); } @@ -3183,6 +3200,38 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) { GGML_LOG_CONT("."); } + // unary_ext (sgn, step, elu, hardswish, hardsigmoid, floor, ceil, round, trunc) + { +#ifdef GGML_OPENCL_EMBED_KERNELS + const std::string kernel_src { + #include "unary_ext.cl.h" + }; +#else + const std::string kernel_src = read_file("unary_ext.cl"); +#endif + cl_program prog = + build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); +#define CL_UNARY_EXT_K(op) \ + CL_CHECK((backend_ctx->kernel_##op##_f32 = clCreateKernel(prog, "kernel_" #op "_f32", &err), err)); \ + CL_CHECK((backend_ctx->kernel_##op##_f32_4 = clCreateKernel(prog, "kernel_" #op "_f32_4", &err), err)); \ + CL_CHECK((backend_ctx->kernel_##op##_f32_nc = clCreateKernel(prog, "kernel_" #op "_f32_nc", &err), err)); \ + CL_CHECK((backend_ctx->kernel_##op##_f16 = clCreateKernel(prog, "kernel_" #op "_f16", &err), err)); \ + CL_CHECK((backend_ctx->kernel_##op##_f16_4 = clCreateKernel(prog, "kernel_" #op "_f16_4", &err), err)); \ + CL_CHECK((backend_ctx->kernel_##op##_f16_nc = clCreateKernel(prog, "kernel_" #op "_f16_nc", &err), err)); + CL_UNARY_EXT_K(sgn) + CL_UNARY_EXT_K(step) + CL_UNARY_EXT_K(elu) + CL_UNARY_EXT_K(hardswish) + CL_UNARY_EXT_K(hardsigmoid) + CL_UNARY_EXT_K(floor) + CL_UNARY_EXT_K(ceil) + CL_UNARY_EXT_K(round) + CL_UNARY_EXT_K(trunc) +#undef CL_UNARY_EXT_K + CL_CHECK(clReleaseProgram(prog)); + GGML_LOG_CONT("."); + } + // softplus { #ifdef GGML_OPENCL_EMBED_KERNELS @@ -3247,8 +3296,11 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) { #endif cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); - CL_CHECK((backend_ctx->kernel_concat_f32 = clCreateKernel(prog, "kernel_concat_f32", &err), err)); - CL_CHECK((backend_ctx->kernel_concat_f32_pack = clCreateKernel(prog, "kernel_concat_f32_pack", &err), err)); + CL_CHECK((backend_ctx->kernel_concat_b1 = clCreateKernel(prog, "kernel_concat_b1", &err), err)); + CL_CHECK((backend_ctx->kernel_concat_b2 = clCreateKernel(prog, "kernel_concat_b2", &err), err)); + CL_CHECK((backend_ctx->kernel_concat_b4 = clCreateKernel(prog, "kernel_concat_b4", &err), err)); + CL_CHECK((backend_ctx->kernel_concat_b8 = clCreateKernel(prog, "kernel_concat_b8", &err), err)); + CL_CHECK((backend_ctx->kernel_concat_b4_pack = clCreateKernel(prog, "kernel_concat_b4_pack", &err), err)); CL_CHECK(clReleaseProgram(prog)); GGML_LOG_CONT("."); } @@ -8452,6 +8504,15 @@ static bool ggml_opencl_supports_op(ggml_backend_dev_t dev, const struct ggml_te case GGML_UNARY_OP_EXPM1: return op->src[0]->type == GGML_TYPE_F32; case GGML_UNARY_OP_ABS: + case GGML_UNARY_OP_SGN: + case GGML_UNARY_OP_STEP: + case GGML_UNARY_OP_ELU: + case GGML_UNARY_OP_HARDSWISH: + case GGML_UNARY_OP_HARDSIGMOID: + case GGML_UNARY_OP_FLOOR: + case GGML_UNARY_OP_CEIL: + case GGML_UNARY_OP_ROUND: + case GGML_UNARY_OP_TRUNC: return op->src[0]->type == GGML_TYPE_F32 || op->src[0]->type == GGML_TYPE_F16; case GGML_UNARY_OP_SOFTPLUS: return op->src[0]->type == GGML_TYPE_F32 || op->src[0]->type == GGML_TYPE_F16; @@ -8531,7 +8592,13 @@ static bool ggml_opencl_supports_op(ggml_backend_dev_t dev, const struct ggml_te return S_v == 16 || S_v == 32 || S_v == 64 || S_v == 128; } case GGML_OP_CONCAT: - return op->src[0]->type == GGML_TYPE_F32 && op->src[1]->type == GGML_TYPE_F32 && op->type == GGML_TYPE_F32; + { + const ggml_type t = op->src[0]->type; + return op->src[1]->type == t && op->type == t && + !ggml_is_quantized(t) && ggml_blck_size(t) == 1 && + (ggml_type_size(t) == 1 || ggml_type_size(t) == 2 || + ggml_type_size(t) == 4 || ggml_type_size(t) == 8); + } case GGML_OP_TIMESTEP_EMBEDDING: return op->src[0]->type == GGML_TYPE_F32 && op->type == GGML_TYPE_F32; case GGML_OP_GROUP_NORM: @@ -15140,6 +15207,97 @@ static void ggml_cl_abs(ggml_backend_t backend, const ggml_tensor * src0, const } } +// Shared driver for the extended unary ops (unary_ext.cl), same selection as +// ggml_cl_abs: contiguous picks the vec4 kernel when the element count is a +// multiple of 4 (else scalar); non-contiguous uses the stride-addressed kernel. +static void ggml_cl_unary_ext(ggml_backend_t backend, const ggml_tensor * src0, ggml_tensor * dst, + cl_kernel k_f32, cl_kernel k_f32_4, cl_kernel k_f32_nc, + cl_kernel k_f16, cl_kernel k_f16_4, cl_kernel k_f16_nc) { + GGML_ASSERT(src0); + GGML_ASSERT(src0->extra); + GGML_ASSERT(dst); + GGML_ASSERT(dst->extra); + + ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context; + + ggml_tensor_extra_cl * extra0 = (ggml_tensor_extra_cl *)src0->extra; + ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra; + + cl_ulong offset0 = extra0->offset + src0->view_offs; + cl_ulong offsetd = extrad->offset + dst->view_offs; + + const int ne00 = src0->ne[0], ne01 = src0->ne[1], ne02 = src0->ne[2], ne03 = src0->ne[3]; + const cl_ulong nb00 = src0->nb[0], nb01 = src0->nb[1], nb02 = src0->nb[2], nb03 = src0->nb[3]; + const cl_ulong nb0 = dst->nb[0], nb1 = dst->nb[1], nb2 = dst->nb[2], nb3 = dst->nb[3]; + + const bool is_f16 = (src0->type == GGML_TYPE_F16); + cl_kernel kernel; + + if (ggml_is_contiguous(src0)) { + int n = ggml_nelements(dst); + if (n % 4 == 0) { + kernel = is_f16 ? k_f16_4 : k_f32_4; + n /= 4; + } else { + kernel = is_f16 ? k_f16 : k_f32; + } + + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offsetd)); + + size_t global_work_size[] = {(size_t)n, 1, 1}; + size_t local_work_size[] = {64, 1, 1}; + size_t * local_work_size_ptr = local_work_size; + if (n % 64 != 0 && !backend_ctx->non_uniform_workgroups) { + local_work_size_ptr = nullptr; + } + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size_ptr, dst); + } else { + kernel = is_f16 ? k_f16_nc : k_f32_nc; + + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &nb00)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_ulong), &nb01)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_ulong), &nb02)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_ulong), &nb03)); + CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_ulong), &nb0)); + CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_ulong), &nb1)); + CL_CHECK(clSetKernelArg(kernel, 11, sizeof(cl_ulong), &nb2)); + CL_CHECK(clSetKernelArg(kernel, 12, sizeof(cl_ulong), &nb3)); + + int nth = 64; + size_t global_work_size[] = {(size_t)ne01*nth, (size_t)ne02, (size_t)ne03}; + size_t local_work_size[] = {(size_t)nth, 1, 1}; + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); + } +} + +#define GGML_CL_UNARY_EXT_WRAP(FN, OP) \ +static void FN(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { \ + UNUSED(src1); \ + ggml_backend_opencl_context *c = (ggml_backend_opencl_context *)backend->context; \ + ggml_cl_unary_ext(backend, src0, dst, c->kernel_##OP##_f32, c->kernel_##OP##_f32_4, c->kernel_##OP##_f32_nc, \ + c->kernel_##OP##_f16, c->kernel_##OP##_f16_4, c->kernel_##OP##_f16_nc); \ +} + +GGML_CL_UNARY_EXT_WRAP(ggml_cl_sgn, sgn) +GGML_CL_UNARY_EXT_WRAP(ggml_cl_step, step) +GGML_CL_UNARY_EXT_WRAP(ggml_cl_elu, elu) +GGML_CL_UNARY_EXT_WRAP(ggml_cl_hardswish, hardswish) +GGML_CL_UNARY_EXT_WRAP(ggml_cl_hardsigmoid, hardsigmoid) +GGML_CL_UNARY_EXT_WRAP(ggml_cl_floor, floor) +GGML_CL_UNARY_EXT_WRAP(ggml_cl_ceil, ceil) +GGML_CL_UNARY_EXT_WRAP(ggml_cl_round, round) +GGML_CL_UNARY_EXT_WRAP(ggml_cl_trunc, trunc) + +#undef GGML_CL_UNARY_EXT_WRAP + static void ggml_cl_softplus(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { GGML_ASSERT(src0); GGML_ASSERT(src0->extra); @@ -15517,9 +15675,8 @@ static void ggml_cl_concat(ggml_backend_t backend, const ggml_tensor * src0, con GGML_ASSERT(src1->extra); GGML_ASSERT(dst); GGML_ASSERT(dst->extra); - GGML_ASSERT(src0->type == GGML_TYPE_F32); - GGML_ASSERT(src1->type == GGML_TYPE_F32); - GGML_ASSERT(dst->type == GGML_TYPE_F32); + GGML_ASSERT(src0->type == src1->type); + GGML_ASSERT(src0->type == dst->type); ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context; @@ -15561,9 +15718,21 @@ static void ggml_cl_concat(ggml_backend_t backend, const ggml_tensor * src0, con int nth = MIN(64, ne0); - const bool concat_pack = (dim == 0 && ne0 < 32); - cl_kernel kernel = concat_pack ? backend_ctx->kernel_concat_f32_pack - : backend_ctx->kernel_concat_f32; + const size_t ts = ggml_type_size(dst->type); + // the pack kernel copies 4-byte elements, so it is only valid for those. + const bool concat_pack = (dim == 0 && ne0 < 32 && ts == 4); + cl_kernel kernel; + if (concat_pack) { + kernel = backend_ctx->kernel_concat_b4_pack; + } else { + switch (ts) { + case 1: kernel = backend_ctx->kernel_concat_b1; break; + case 2: kernel = backend_ctx->kernel_concat_b2; break; + case 4: kernel = backend_ctx->kernel_concat_b4; break; + case 8: kernel = backend_ctx->kernel_concat_b8; break; + default: GGML_ABORT("unsupported concat element size: %zu", ts); + } + } CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); @@ -26021,6 +26190,38 @@ static void ggml_cl_cpy(ggml_backend_t backend, const ggml_tensor * src0, const cl_ulong offset0 = extra0->offset + src0->view_offs; cl_ulong offset1 = extra1->offset + src1->view_offs; + // A contiguous f32 -> f32 copy is a linear move. The kernel below maps one workgroup to + // each row, so a tensor with few long rows runs on a single compute unit; dispatch those + // over the whole device instead. GGML_OPENCL_CPY_FLAT=0 restores the row-mapped path. + static const bool cpy_flat_on = []{ + const char * e = getenv("GGML_OPENCL_CPY_FLAT"); + return !(e && e[0] == '0'); + }(); + if (cpy_flat_on && backend_ctx->kernel_cpy_f32_f32_flat != nullptr && + src0t == GGML_TYPE_F32 && src1t == GGML_TYPE_F32 && + ggml_is_contiguous(src0) && ggml_is_contiguous(src1) && + ggml_nelements(src0) == ggml_nelements(src1)) { + cl_kernel k = backend_ctx->kernel_cpy_f32_f32_flat; + const cl_ulong nelem = (cl_ulong) ggml_nelements(src0); + const cl_ulong n4 = nelem / 4; + + CL_CHECK(clSetKernelArg(k, 0, sizeof(cl_mem), &extra0->data_device)); + CL_CHECK(clSetKernelArg(k, 1, sizeof(cl_ulong), &offset0)); + CL_CHECK(clSetKernelArg(k, 2, sizeof(cl_mem), &extra1->data_device)); + CL_CHECK(clSetKernelArg(k, 3, sizeof(cl_ulong), &offset1)); + CL_CHECK(clSetKernelArg(k, 4, sizeof(cl_ulong), &nelem)); + CL_CHECK(clSetKernelArg(k, 5, sizeof(cl_ulong), &n4)); + + // one work item per float4, plus one for the trailing scalars + const size_t items = (size_t) n4 + ((nelem % 4) ? 1 : 0); + const size_t lsz = MIN((size_t) 64, backend_ctx->max_workgroup_size); + size_t global_work_size[] = { ((items + lsz - 1) / lsz) * lsz, 1, 1 }; + size_t local_work_size[] = { lsz, 1, 1 }; + + backend_ctx->enqueue_ndrange_kernel(k, 1, global_work_size, local_work_size, src1); + return; + } + cl_kernel kernel; switch (src0t) { @@ -27454,6 +27655,42 @@ bool ggml_cl_compute_forward(ggml_backend_t backend, struct ggml_tensor * tensor } func = ggml_cl_abs; break; + case GGML_UNARY_OP_SGN: + if (!any_on_device) { return false; } + func = ggml_cl_sgn; + break; + case GGML_UNARY_OP_STEP: + if (!any_on_device) { return false; } + func = ggml_cl_step; + break; + case GGML_UNARY_OP_ELU: + if (!any_on_device) { return false; } + func = ggml_cl_elu; + break; + case GGML_UNARY_OP_HARDSWISH: + if (!any_on_device) { return false; } + func = ggml_cl_hardswish; + break; + case GGML_UNARY_OP_HARDSIGMOID: + if (!any_on_device) { return false; } + func = ggml_cl_hardsigmoid; + break; + case GGML_UNARY_OP_FLOOR: + if (!any_on_device) { return false; } + func = ggml_cl_floor; + break; + case GGML_UNARY_OP_CEIL: + if (!any_on_device) { return false; } + func = ggml_cl_ceil; + break; + case GGML_UNARY_OP_ROUND: + if (!any_on_device) { return false; } + func = ggml_cl_round; + break; + case GGML_UNARY_OP_TRUNC: + if (!any_on_device) { return false; } + func = ggml_cl_trunc; + break; case GGML_UNARY_OP_SOFTPLUS: if (!any_on_device) { return false; diff --git a/ggml/src/ggml-opencl/kernels/concat.cl b/ggml/src/ggml-opencl/kernels/concat.cl index 2fbd7851d..8ecf7466d 100644 --- a/ggml/src/ggml-opencl/kernels/concat.cl +++ b/ggml/src/ggml-opencl/kernels/concat.cl @@ -1,56 +1,66 @@ -kernel void kernel_concat_f32( - global const char * src0, - ulong offset0, - global const char * src1, - ulong offset1, - global char * dst, - ulong offsetd, - int ne00, - int ne01, - int ne02, - int ne03, - ulong nb00, - ulong nb01, - ulong nb02, - ulong nb03, - ulong nb10, - ulong nb11, - ulong nb12, - ulong nb13, - int ne0, - ulong nb0, - ulong nb1, - ulong nb2, - ulong nb3, - int dim -) { - src0 = src0 + offset0; - src1 = src1 + offset1; - dst = dst + offsetd; +// concat is a pure copy, so the kernels are keyed by element byte size +// (1/2/4/8) rather than logical type, matching the CUDA backend. - const int i3 = get_group_id(2); - const int i2 = get_group_id(1); - const int i1 = get_group_id(0); - - int o[4] = {0, 0, 0, 0}; - o[dim] = dim == 0 ? ne00 : (dim == 1 ? ne01 : (dim == 2 ? ne02 : ne03)); - - global const float * x; - - for (int i0 = get_local_id(0); i0 < ne0; i0 += get_local_size(0)) { - if (i0 < ne00 && i1 < ne01 && i2 < ne02 && i3 < ne03) { - x = (global const float *)(src0 + (i3 )*nb03 + (i2 )*nb02 + (i1 )*nb01 + (i0 )*nb00); - } else { - x = (global const float *)(src1 + (i3 - o[3])*nb13 + (i2 - o[2])*nb12 + (i1 - o[1])*nb11 + (i0 - o[0])*nb10); - } - - global float * y = (global float *)(dst + i3*nb3 + i2*nb2 + i1*nb1 + i0*nb0); - - *y = *x; - } +#define KERNEL_CONCAT(SUFFIX, T) \ +kernel void kernel_concat_##SUFFIX( \ + global const char * src0, \ + ulong offset0, \ + global const char * src1, \ + ulong offset1, \ + global char * dst, \ + ulong offsetd, \ + int ne00, \ + int ne01, \ + int ne02, \ + int ne03, \ + ulong nb00, \ + ulong nb01, \ + ulong nb02, \ + ulong nb03, \ + ulong nb10, \ + ulong nb11, \ + ulong nb12, \ + ulong nb13, \ + int ne0, \ + ulong nb0, \ + ulong nb1, \ + ulong nb2, \ + ulong nb3, \ + int dim \ +) { \ + src0 = src0 + offset0; \ + src1 = src1 + offset1; \ + dst = dst + offsetd; \ + \ + const int i3 = get_group_id(2); \ + const int i2 = get_group_id(1); \ + const int i1 = get_group_id(0); \ + \ + int o[4] = {0, 0, 0, 0}; \ + o[dim] = dim == 0 ? ne00 : (dim == 1 ? ne01 : (dim == 2 ? ne02 : ne03)); \ + \ + global const T * x; \ + \ + for (int i0 = get_local_id(0); i0 < ne0; i0 += get_local_size(0)) { \ + if (i0 < ne00 && i1 < ne01 && i2 < ne02 && i3 < ne03) { \ + x = (global const T *)(src0 + (i3 )*nb03 + (i2 )*nb02 + (i1 )*nb01 + (i0 )*nb00); \ + } else { \ + x = (global const T *)(src1 + (i3 - o[3])*nb13 + (i2 - o[2])*nb12 + (i1 - o[1])*nb11 + (i0 - o[0])*nb10); \ + } \ + \ + global T * y = (global T *)(dst + i3*nb3 + i2*nb2 + i1*nb1 + i0*nb0); \ + \ + *y = *x; \ + } \ } -kernel void kernel_concat_f32_pack( +KERNEL_CONCAT(b1, char) +KERNEL_CONCAT(b2, short) +KERNEL_CONCAT(b4, int) +KERNEL_CONCAT(b8, long) + +// packed variant for the common dim==0, small-ne0 case (4-byte elements only). +kernel void kernel_concat_b4_pack( global const char * src0, ulong offset0, global const char * src1, @@ -104,14 +114,14 @@ kernel void kernel_concat_f32_pack( o[dim] = dim == 0 ? ne00 : (dim == 1 ? ne01 : (dim == 2 ? ne02 : ne03)); for (int i0 = lane; i0 < ne0; i0 += tpr) { - global const float * x; + global const int * x; if (i0 < ne00 && i1 < ne01 && i2 < ne02 && i3 < ne03) { - x = (global const float *)(src0 + (i3 )*nb03 + (i2 )*nb02 + (i1 )*nb01 + (i0 )*nb00); + x = (global const int *)(src0 + (i3 )*nb03 + (i2 )*nb02 + (i1 )*nb01 + (i0 )*nb00); } else { - x = (global const float *)(src1 + (i3 - o[3])*nb13 + (i2 - o[2])*nb12 + (i1 - o[1])*nb11 + (i0 - o[0])*nb10); + x = (global const int *)(src1 + (i3 - o[3])*nb13 + (i2 - o[2])*nb12 + (i1 - o[1])*nb11 + (i0 - o[0])*nb10); } - global float * y = (global float *)(dst + i3*nb3 + i2*nb2 + i1*nb1 + i0*nb0); + global int * y = (global int *)(dst + i3*nb3 + i2*nb2 + i1*nb1 + i0*nb0); *y = *x; } diff --git a/ggml/src/ggml-opencl/kernels/cpy.cl b/ggml/src/ggml-opencl/kernels/cpy.cl index adbd2e766..e875bfaf7 100644 --- a/ggml/src/ggml-opencl/kernels/cpy.cl +++ b/ggml/src/ggml-opencl/kernels/cpy.cl @@ -286,3 +286,28 @@ kernel void kernel_cpy_i32_i32( dst_data[i00] = src[0]; } } + +// Contiguous f32 copy, one work item per float4 over the whole tensor. The kernels above map +// one workgroup to each row, which leaves a tensor with few long rows on a single compute unit. +// vload4/vstore4 rather than a float4 cast: these buffers carry an arbitrary 4-byte view offset. +kernel void kernel_cpy_f32_f32_flat( + global float * src0, + ulong offset0, + global float * dst, + ulong offsetd, + ulong ne, + ulong n4 +) { + src0 = (global float*)((global char*)src0 + offset0); + dst = (global float*)((global char*)dst + offsetd); + + const ulong i = get_global_id(0); + + if (i < n4) { + vstore4(vload4(i, src0), i, dst); + } else if (i == n4) { + for (ulong t = n4 * 4; t < ne; ++t) { + dst[t] = src0[t]; + } + } +} diff --git a/ggml/src/ggml-opencl/kernels/unary_ext.cl b/ggml/src/ggml-opencl/kernels/unary_ext.cl new file mode 100644 index 000000000..e86eadfa5 --- /dev/null +++ b/ggml/src/ggml-opencl/kernels/unary_ext.cl @@ -0,0 +1,85 @@ +#pragma OPENCL EXTENSION cl_khr_fp16 : enable + +//------------------------------------------------------------------------------ +// Extended elementwise unary ops, same variant shape as abs.cl: +// f32, f32_4 (vec4), f16, f16_4 (vec4), f32_nc, f16_nc (stride-addressed). +// +// sgn, step, elu, hardswish, hardsigmoid, floor, ceil, round, trunc. +// +// Semantics match the ggml CPU reference (ggml.c). Values are computed in float +// (the f16 variants read/write half and convert), so the conditional ops match +// the CPU bit-for-bit within tolerance. SEXPR is the scalar form, VEXPR the +// float4 form (vector ternaries need select()). +//------------------------------------------------------------------------------ + +#define UNARY_EXT(NAME, SEXPR, VEXPR) \ +kernel void kernel_##NAME##_f32( \ + global const float * src0, ulong offset0, \ + global float * dst, ulong offsetd) { \ + src0 = (global float*)((global char*)src0 + offset0); \ + dst = (global float*)((global char*)dst + offsetd); \ + float x = src0[get_global_id(0)]; \ + dst[get_global_id(0)] = (SEXPR); \ +} \ +kernel void kernel_##NAME##_f32_4( \ + global const float4 * src0, ulong offset0, \ + global float4 * dst, ulong offsetd) { \ + src0 = (global float4*)((global char*)src0 + offset0); \ + dst = (global float4*)((global char*)dst + offsetd); \ + float4 x = src0[get_global_id(0)]; \ + dst[get_global_id(0)] = (VEXPR); \ +} \ +kernel void kernel_##NAME##_f16( \ + global const half * src0, ulong offset0, \ + global half * dst, ulong offsetd) { \ + src0 = (global half*)((global char*)src0 + offset0); \ + dst = (global half*)((global char*)dst + offsetd); \ + float x = src0[get_global_id(0)]; \ + dst[get_global_id(0)] = (SEXPR); \ +} \ +kernel void kernel_##NAME##_f16_4( \ + global const half4 * src0, ulong offset0, \ + global half4 * dst, ulong offsetd) { \ + src0 = (global half4*)((global char*)src0 + offset0); \ + dst = (global half4*)((global char*)dst + offsetd); \ + float4 x = convert_float4(src0[get_global_id(0)]); \ + dst[get_global_id(0)] = convert_half4(VEXPR); \ +} \ +kernel void kernel_##NAME##_f32_nc( \ + global const char * src0, ulong offset0, \ + global char * dst, ulong offsetd, \ + int ne00, ulong nb00, ulong nb01, ulong nb02, ulong nb03, \ + ulong nb0, ulong nb1, ulong nb2, ulong nb3) { \ + src0 = src0 + offset0; dst = dst + offsetd; \ + const int i3 = get_group_id(2); \ + const int i2 = get_group_id(1); \ + const int i1 = get_group_id(0); \ + for (int i0 = get_local_id(0); i0 < ne00; i0 += get_local_size(0)) { \ + float x = *(global const float *)(src0 + i3*nb03 + i2*nb02 + i1*nb01 + i0*nb00); \ + *(global float *)(dst + i3*nb3 + i2*nb2 + i1*nb1 + i0*nb0) = (SEXPR); \ + } \ +} \ +kernel void kernel_##NAME##_f16_nc( \ + global const char * src0, ulong offset0, \ + global char * dst, ulong offsetd, \ + int ne00, ulong nb00, ulong nb01, ulong nb02, ulong nb03, \ + ulong nb0, ulong nb1, ulong nb2, ulong nb3) { \ + src0 = src0 + offset0; dst = dst + offsetd; \ + const int i3 = get_group_id(2); \ + const int i2 = get_group_id(1); \ + const int i1 = get_group_id(0); \ + for (int i0 = get_local_id(0); i0 < ne00; i0 += get_local_size(0)) {\ + float x = *(global const half *)(src0 + i3*nb03 + i2*nb02 + i1*nb01 + i0*nb00); \ + *(global half *)(dst + i3*nb3 + i2*nb2 + i1*nb1 + i0*nb0) = (SEXPR); \ + } \ +} + +UNARY_EXT(sgn, sign(x), sign(x)) +UNARY_EXT(step, x > 0.0f ? 1.0f : 0.0f, select((float4)0.0f, (float4)1.0f, x > 0.0f)) +UNARY_EXT(elu, x > 0.0f ? x : expm1(x), select(expm1(x), x, x > 0.0f)) +UNARY_EXT(hardswish, x * fmin(1.0f, fmax(0.0f, (x + 3.0f) / 6.0f)), x * fmin((float4)1.0f, fmax((float4)0.0f, (x + 3.0f) / 6.0f))) +UNARY_EXT(hardsigmoid, fmin(1.0f, fmax(0.0f, (x + 3.0f) / 6.0f)), fmin((float4)1.0f, fmax((float4)0.0f, (x + 3.0f) / 6.0f))) +UNARY_EXT(floor, floor(x), floor(x)) +UNARY_EXT(ceil, ceil(x), ceil(x)) +UNARY_EXT(round, round(x), round(x)) +UNARY_EXT(trunc, trunc(x), trunc(x))