Lumpiasty/llama.cpp
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

opencl: extend the elementwise and data‐movement op coverage (#27633)

Browse Source 
* opencl: add extended elementwise unary ops (sgn, step, elu, hardswish, hardsigmoid, floor, ceil, round, trunc)

Adds nine GGML_UNARY_OP_* elementwise ops that were falling back to CPU on the
OpenCL backend, following the same variant shape as the existing ABS op: f32,
f32_4 (vec4), f16, f16_4 (vec4), and stride-addressed f32_nc / f16_nc for
non-contiguous inputs. New kernels/unary_ext.cl (macro-generated), a shared
ggml_cl_unary_ext dispatch helper mirroring ggml_cl_abs, the supports_op cases,
and the compute-forward cases.

Values are computed in float (the f16 variants read/write half and convert), so
the conditional ops (step, elu) match the CPU reference; the vec4 forms use
select() for the branch.

Validated with test-backend-ops on Adreno 840 and 850 (E17): all nine ops pass
every case including the vec4 and non-contiguous variants (8/8 or 14/14).

* opencl: dispatch a contiguous f32 copy over the whole device

kernel_cpy_f32_f32 maps one workgroup to each (i01,i02,i03) row and strides the
row across that workgroup's lanes, and the host launches ne01*MIN(64,ne00) work
items. A tensor with few long rows therefore runs on a single workgroup. The
mamba2 and gated-delta-net recurrent state cache is one row of 524288 floats,
copied once per layer per graph, and lands on 64 work items.

When both sides are contiguous the copy is a linear move, so dispatch it over
the whole device: one work item per float4. Gated on ggml_is_contiguous for both
tensors and equal element counts, so copies already spread over many rows keep
the existing path. The kernel is created optionally, so a driver that rejects it
falls back rather than aborting.

vload4/vstore4 rather than a float4 cast: they require only the scalar type's
alignment, and these buffers carry an arbitrary 4-byte view offset.

CPY, DUP and CONT are 217/217 on Adreno 840 and 740 with the path enabled and
disabled. GGML_OPENCL_CPY_FLAT=0 forces the old kernel.

* opencl: support all easy-copy types in CONCAT

CONCAT was F32-only. Extend it to every "easy-copy" type -- any non-quantized
type with a block size of 1 and an element size of 1, 2, 4 or 8 bytes, i.e.
f16/bf16/i8/i16/i32/i64 as well as f32.

The kernels are keyed by element SIZE rather than by type, which is what CUDA
already does for the same op: one kernel per byte width (b1/b2/b4/b8) plus the
packed b4 fast path, instead of one per ggml type. supports_op gates on the
same property, so a new type of a supported width is picked up with no further
work.

Validated with test-backend-ops on Adreno 840 / A8X and X2-90 / X2E.
This commit is contained in:
Hongqiang Wang
2026-09-04 10:12:26 -07:00
committed by GitHub
parent 4acf4a4cb8
commit 1548a240e3
5 changed files with 422 additions and 64 deletions
Download Patch File Download Diff File Expand all files Collapse all files
ggml/src/ggml-opencl/CMakeLists.txt
+1
View File
@@ -222,6 +222,7 @@ set(GGML_OPENCL_KERNELS
exp
expm1
abs
unary_ext
softplus
pad
repeat
ggml/src/ggml-opencl/ggml-opencl.cpp
+247 -10
View File
@@ -826,6 +826,7 @@ struct ggml_backend_opencl_context {
cl_kernel kernel_rope_norm_f32, kernel_rope_norm_f16, kernel_rope_neox_f32, kernel_rope_neox_f16;
cl_kernel kernel_rope_multi_f32, kernel_rope_multi_f16, kernel_rope_vision_f32, kernel_rope_vision_f16;
cl_kernel kernel_cpy_f16_f16, kernel_cpy_f16_f32, kernel_cpy_f32_f16, kernel_cpy_f32_f32, kernel_cpy_f32_f32_pack, kernel_cpy_i32_i32;
cl_kernel kernel_cpy_f32_f32_flat = nullptr;
cl_kernel kernel_mul_mat_f32_f32;
cl_kernel kernel_mul_mat_f16_f16;
cl_kernel kernel_mul_mat_f16_f32_1row;
@@ -932,11 +933,20 @@ struct ggml_backend_opencl_context {
cl_kernel kernel_expm1_f16, kernel_expm1_f16_4, kernel_expm1_f16_nc;
cl_kernel kernel_abs_f32, kernel_abs_f32_4, kernel_abs_f32_nc;
cl_kernel kernel_abs_f16, kernel_abs_f16_4, kernel_abs_f16_nc;
cl_kernel kernel_sgn_f32, kernel_sgn_f32_4, kernel_sgn_f32_nc, kernel_sgn_f16, kernel_sgn_f16_4, kernel_sgn_f16_nc;
cl_kernel kernel_step_f32, kernel_step_f32_4, kernel_step_f32_nc, kernel_step_f16, kernel_step_f16_4, kernel_step_f16_nc;
cl_kernel kernel_elu_f32, kernel_elu_f32_4, kernel_elu_f32_nc, kernel_elu_f16, kernel_elu_f16_4, kernel_elu_f16_nc;
cl_kernel kernel_hardswish_f32, kernel_hardswish_f32_4, kernel_hardswish_f32_nc, kernel_hardswish_f16, kernel_hardswish_f16_4, kernel_hardswish_f16_nc;
cl_kernel kernel_hardsigmoid_f32, kernel_hardsigmoid_f32_4, kernel_hardsigmoid_f32_nc, kernel_hardsigmoid_f16, kernel_hardsigmoid_f16_4, kernel_hardsigmoid_f16_nc;
cl_kernel kernel_floor_f32, kernel_floor_f32_4, kernel_floor_f32_nc, kernel_floor_f16, kernel_floor_f16_4, kernel_floor_f16_nc;
cl_kernel kernel_ceil_f32, kernel_ceil_f32_4, kernel_ceil_f32_nc, kernel_ceil_f16, kernel_ceil_f16_4, kernel_ceil_f16_nc;
cl_kernel kernel_round_f32, kernel_round_f32_4, kernel_round_f32_nc, kernel_round_f16, kernel_round_f16_4, kernel_round_f16_nc;
cl_kernel kernel_trunc_f32, kernel_trunc_f32_4, kernel_trunc_f32_nc, kernel_trunc_f16, kernel_trunc_f16_4, kernel_trunc_f16_nc;
cl_kernel kernel_softplus_f32, kernel_softplus_f32_4, kernel_softplus_f32_nc;
cl_kernel kernel_softplus_f16, kernel_softplus_f16_4, kernel_softplus_f16_nc;
cl_kernel kernel_upscale;
cl_kernel kernel_upscale_bilinear;
cl_kernel kernel_concat_f32, kernel_concat_f32_pack;
cl_kernel kernel_concat_b1, kernel_concat_b2, kernel_concat_b4, kernel_concat_b8, kernel_concat_b4_pack;
cl_kernel kernel_conv_2d_f16;
cl_kernel kernel_conv_2d_f32;
cl_kernel kernel_conv_2d_f16_f32;
@@ -1537,6 +1547,13 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) {
CL_CHECK((backend_ctx->kernel_cpy_f32_f16 = clCreateKernel(prog, "kernel_cpy_f32_f16", &err), err));
CL_CHECK((backend_ctx->kernel_cpy_f32_f32 = clCreateKernel(prog, "kernel_cpy_f32_f32", &err), err));
CL_CHECK((backend_ctx->kernel_cpy_f32_f32_pack = clCreateKernel(prog, "kernel_cpy_f32_f32_pack", &err), err));
{ // optional: without it ggml_cl_cpy keeps the row-mapped kernel
cl_int err_flat = CL_SUCCESS;
cl_kernel k = clCreateKernel(prog, "kernel_cpy_f32_f32_flat", &err_flat);
if (err_flat == CL_SUCCESS) {
backend_ctx->kernel_cpy_f32_f32_flat = k;
}
}
CL_CHECK((backend_ctx->kernel_cpy_i32_i32 = clCreateKernel(prog, "kernel_cpy_i32_i32", &err), err));
GGML_LOG_CONT(".");
}
@@ -3183,6 +3200,38 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) {
GGML_LOG_CONT(".");
}
// unary_ext (sgn, step, elu, hardswish, hardsigmoid, floor, ceil, round, trunc)
{
#ifdef GGML_OPENCL_EMBED_KERNELS
const std::string kernel_src {
#include "unary_ext.cl.h"
};
#else
const std::string kernel_src = read_file("unary_ext.cl");
#endif
cl_program prog =
build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts);
#define CL_UNARY_EXT_K(op) \
CL_CHECK((backend_ctx->kernel_##op##_f32 = clCreateKernel(prog, "kernel_" #op "_f32", &err), err)); \
CL_CHECK((backend_ctx->kernel_##op##_f32_4 = clCreateKernel(prog, "kernel_" #op "_f32_4", &err), err)); \
CL_CHECK((backend_ctx->kernel_##op##_f32_nc = clCreateKernel(prog, "kernel_" #op "_f32_nc", &err), err)); \
CL_CHECK((backend_ctx->kernel_##op##_f16 = clCreateKernel(prog, "kernel_" #op "_f16", &err), err)); \
CL_CHECK((backend_ctx->kernel_##op##_f16_4 = clCreateKernel(prog, "kernel_" #op "_f16_4", &err), err)); \
CL_CHECK((backend_ctx->kernel_##op##_f16_nc = clCreateKernel(prog, "kernel_" #op "_f16_nc", &err), err));
CL_UNARY_EXT_K(sgn)
CL_UNARY_EXT_K(step)
CL_UNARY_EXT_K(elu)
CL_UNARY_EXT_K(hardswish)
CL_UNARY_EXT_K(hardsigmoid)
CL_UNARY_EXT_K(floor)
CL_UNARY_EXT_K(ceil)
CL_UNARY_EXT_K(round)
CL_UNARY_EXT_K(trunc)
#undef CL_UNARY_EXT_K
CL_CHECK(clReleaseProgram(prog));
GGML_LOG_CONT(".");
}
// softplus
{
#ifdef GGML_OPENCL_EMBED_KERNELS
@@ -3247,8 +3296,11 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) {
#endif
cl_program prog =
build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts);
CL_CHECK((backend_ctx->kernel_concat_f32 = clCreateKernel(prog, "kernel_concat_f32", &err), err));
CL_CHECK((backend_ctx->kernel_concat_f32_pack = clCreateKernel(prog, "kernel_concat_f32_pack", &err), err));
CL_CHECK((backend_ctx->kernel_concat_b1 = clCreateKernel(prog, "kernel_concat_b1", &err), err));
CL_CHECK((backend_ctx->kernel_concat_b2 = clCreateKernel(prog, "kernel_concat_b2", &err), err));
CL_CHECK((backend_ctx->kernel_concat_b4 = clCreateKernel(prog, "kernel_concat_b4", &err), err));
CL_CHECK((backend_ctx->kernel_concat_b8 = clCreateKernel(prog, "kernel_concat_b8", &err), err));
CL_CHECK((backend_ctx->kernel_concat_b4_pack = clCreateKernel(prog, "kernel_concat_b4_pack", &err), err));
CL_CHECK(clReleaseProgram(prog));
GGML_LOG_CONT(".");
}
@@ -8452,6 +8504,15 @@ static bool ggml_opencl_supports_op(ggml_backend_dev_t dev, const struct ggml_te
case GGML_UNARY_OP_EXPM1:
return op->src[0]->type == GGML_TYPE_F32;
case GGML_UNARY_OP_ABS:
case GGML_UNARY_OP_SGN:
case GGML_UNARY_OP_STEP:
case GGML_UNARY_OP_ELU:
case GGML_UNARY_OP_HARDSWISH:
case GGML_UNARY_OP_HARDSIGMOID:
case GGML_UNARY_OP_FLOOR:
case GGML_UNARY_OP_CEIL:
case GGML_UNARY_OP_ROUND:
case GGML_UNARY_OP_TRUNC:
return op->src[0]->type == GGML_TYPE_F32 || op->src[0]->type == GGML_TYPE_F16;
case GGML_UNARY_OP_SOFTPLUS:
return op->src[0]->type == GGML_TYPE_F32 || op->src[0]->type == GGML_TYPE_F16;
@@ -8531,7 +8592,13 @@ static bool ggml_opencl_supports_op(ggml_backend_dev_t dev, const struct ggml_te
return S_v == 16 || S_v == 32 || S_v == 64 || S_v == 128;
}
case GGML_OP_CONCAT:
return op->src[0]->type == GGML_TYPE_F32 && op->src[1]->type == GGML_TYPE_F32 && op->type == GGML_TYPE_F32;
{
const ggml_type t = op->src[0]->type;
return op->src[1]->type == t && op->type == t &&
!ggml_is_quantized(t) && ggml_blck_size(t) == 1 &&
(ggml_type_size(t) == 1 || ggml_type_size(t) == 2 ||
ggml_type_size(t) == 4 || ggml_type_size(t) == 8);
}
case GGML_OP_TIMESTEP_EMBEDDING:
return op->src[0]->type == GGML_TYPE_F32 && op->type == GGML_TYPE_F32;
case GGML_OP_GROUP_NORM:
@@ -15140,6 +15207,97 @@ static void ggml_cl_abs(ggml_backend_t backend, const ggml_tensor * src0, const
}
}
// Shared driver for the extended unary ops (unary_ext.cl), same selection as
// ggml_cl_abs: contiguous picks the vec4 kernel when the element count is a
// multiple of 4 (else scalar); non-contiguous uses the stride-addressed kernel.
static void ggml_cl_unary_ext(ggml_backend_t backend, const ggml_tensor * src0, ggml_tensor * dst,
cl_kernel k_f32, cl_kernel k_f32_4, cl_kernel k_f32_nc,
cl_kernel k_f16, cl_kernel k_f16_4, cl_kernel k_f16_nc) {
GGML_ASSERT(src0);
GGML_ASSERT(src0->extra);
GGML_ASSERT(dst);
GGML_ASSERT(dst->extra);
ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context;
ggml_tensor_extra_cl * extra0 = (ggml_tensor_extra_cl *)src0->extra;
ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra;
cl_ulong offset0 = extra0->offset + src0->view_offs;
cl_ulong offsetd = extrad->offset + dst->view_offs;
const int ne00 = src0->ne[0], ne01 = src0->ne[1], ne02 = src0->ne[2], ne03 = src0->ne[3];
const cl_ulong nb00 = src0->nb[0], nb01 = src0->nb[1], nb02 = src0->nb[2], nb03 = src0->nb[3];
const cl_ulong nb0 = dst->nb[0], nb1 = dst->nb[1], nb2 = dst->nb[2], nb3 = dst->nb[3];
const bool is_f16 = (src0->type == GGML_TYPE_F16);
cl_kernel kernel;
if (ggml_is_contiguous(src0)) {
int n = ggml_nelements(dst);
if (n % 4 == 0) {
kernel = is_f16 ? k_f16_4 : k_f32_4;
n /= 4;
} else {
kernel = is_f16 ? k_f16 : k_f32;
}
CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device));
CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0));
CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extrad->data_device));
CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offsetd));
size_t global_work_size[] = {(size_t)n, 1, 1};
size_t local_work_size[] = {64, 1, 1};
size_t * local_work_size_ptr = local_work_size;
if (n % 64 != 0 && !backend_ctx->non_uniform_workgroups) {
local_work_size_ptr = nullptr;
}
backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size_ptr, dst);
} else {
kernel = is_f16 ? k_f16_nc : k_f32_nc;
CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device));
CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0));
CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extrad->data_device));
CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offsetd));
CL_CHECK(clSetKernelArg(kernel, 4, sizeof(int), &ne00));
CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &nb00));
CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_ulong), &nb01));
CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_ulong), &nb02));
CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_ulong), &nb03));
CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_ulong), &nb0));
CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_ulong), &nb1));
CL_CHECK(clSetKernelArg(kernel, 11, sizeof(cl_ulong), &nb2));
CL_CHECK(clSetKernelArg(kernel, 12, sizeof(cl_ulong), &nb3));
int nth = 64;
size_t global_work_size[] = {(size_t)ne01*nth, (size_t)ne02, (size_t)ne03};
size_t local_work_size[] = {(size_t)nth, 1, 1};
backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst);
}
}
#define GGML_CL_UNARY_EXT_WRAP(FN, OP) \
static void FN(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { \
UNUSED(src1); \
ggml_backend_opencl_context *c = (ggml_backend_opencl_context *)backend->context; \
ggml_cl_unary_ext(backend, src0, dst, c->kernel_##OP##_f32, c->kernel_##OP##_f32_4, c->kernel_##OP##_f32_nc, \
c->kernel_##OP##_f16, c->kernel_##OP##_f16_4, c->kernel_##OP##_f16_nc); \
}
GGML_CL_UNARY_EXT_WRAP(ggml_cl_sgn, sgn)
GGML_CL_UNARY_EXT_WRAP(ggml_cl_step, step)
GGML_CL_UNARY_EXT_WRAP(ggml_cl_elu, elu)
GGML_CL_UNARY_EXT_WRAP(ggml_cl_hardswish, hardswish)
GGML_CL_UNARY_EXT_WRAP(ggml_cl_hardsigmoid, hardsigmoid)
GGML_CL_UNARY_EXT_WRAP(ggml_cl_floor, floor)
GGML_CL_UNARY_EXT_WRAP(ggml_cl_ceil, ceil)
GGML_CL_UNARY_EXT_WRAP(ggml_cl_round, round)
GGML_CL_UNARY_EXT_WRAP(ggml_cl_trunc, trunc)
#undef GGML_CL_UNARY_EXT_WRAP
static void ggml_cl_softplus(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {
GGML_ASSERT(src0);
GGML_ASSERT(src0->extra);
@@ -15517,9 +15675,8 @@ static void ggml_cl_concat(ggml_backend_t backend, const ggml_tensor * src0, con
GGML_ASSERT(src1->extra);
GGML_ASSERT(dst);
GGML_ASSERT(dst->extra);
GGML_ASSERT(src0->type == GGML_TYPE_F32);
GGML_ASSERT(src1->type == GGML_TYPE_F32);
GGML_ASSERT(dst->type == GGML_TYPE_F32);
GGML_ASSERT(src0->type == src1->type);
GGML_ASSERT(src0->type == dst->type);
ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context;
@@ -15561,9 +15718,21 @@ static void ggml_cl_concat(ggml_backend_t backend, const ggml_tensor * src0, con
int nth = MIN(64, ne0);
const bool concat_pack = (dim == 0 && ne0 < 32);
cl_kernel kernel = concat_pack ? backend_ctx->kernel_concat_f32_pack
: backend_ctx->kernel_concat_f32;
const size_t ts = ggml_type_size(dst->type);
// the pack kernel copies 4-byte elements, so it is only valid for those.
const bool concat_pack = (dim == 0 && ne0 < 32 && ts == 4);
cl_kernel kernel;
if (concat_pack) {
kernel = backend_ctx->kernel_concat_b4_pack;
} else {
switch (ts) {
case 1: kernel = backend_ctx->kernel_concat_b1; break;
case 2: kernel = backend_ctx->kernel_concat_b2; break;
case 4: kernel = backend_ctx->kernel_concat_b4; break;
case 8: kernel = backend_ctx->kernel_concat_b8; break;
default: GGML_ABORT("unsupported concat element size: %zu", ts);
}
}
CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device));
CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0));
@@ -26021,6 +26190,38 @@ static void ggml_cl_cpy(ggml_backend_t backend, const ggml_tensor * src0, const
cl_ulong offset0 = extra0->offset + src0->view_offs;
cl_ulong offset1 = extra1->offset + src1->view_offs;
// A contiguous f32 -> f32 copy is a linear move. The kernel below maps one workgroup to
// each row, so a tensor with few long rows runs on a single compute unit; dispatch those
// over the whole device instead. GGML_OPENCL_CPY_FLAT=0 restores the row-mapped path.
static const bool cpy_flat_on = []{
const char * e = getenv("GGML_OPENCL_CPY_FLAT");
return !(e && e[0] == '0');
}();
if (cpy_flat_on && backend_ctx->kernel_cpy_f32_f32_flat != nullptr &&
src0t == GGML_TYPE_F32 && src1t == GGML_TYPE_F32 &&
ggml_is_contiguous(src0) && ggml_is_contiguous(src1) &&
ggml_nelements(src0) == ggml_nelements(src1)) {
cl_kernel k = backend_ctx->kernel_cpy_f32_f32_flat;
const cl_ulong nelem = (cl_ulong) ggml_nelements(src0);
const cl_ulong n4 = nelem / 4;
CL_CHECK(clSetKernelArg(k, 0, sizeof(cl_mem), &extra0->data_device));
CL_CHECK(clSetKernelArg(k, 1, sizeof(cl_ulong), &offset0));
CL_CHECK(clSetKernelArg(k, 2, sizeof(cl_mem), &extra1->data_device));
CL_CHECK(clSetKernelArg(k, 3, sizeof(cl_ulong), &offset1));
CL_CHECK(clSetKernelArg(k, 4, sizeof(cl_ulong), &nelem));
CL_CHECK(clSetKernelArg(k, 5, sizeof(cl_ulong), &n4));
// one work item per float4, plus one for the trailing scalars
const size_t items = (size_t) n4 + ((nelem % 4) ? 1 : 0);
const size_t lsz = MIN((size_t) 64, backend_ctx->max_workgroup_size);
size_t global_work_size[] = { ((items + lsz - 1) / lsz) * lsz, 1, 1 };
size_t local_work_size[] = { lsz, 1, 1 };
backend_ctx->enqueue_ndrange_kernel(k, 1, global_work_size, local_work_size, src1);
return;
}
cl_kernel kernel;
switch (src0t) {
@@ -27454,6 +27655,42 @@ bool ggml_cl_compute_forward(ggml_backend_t backend, struct ggml_tensor * tensor
}
func = ggml_cl_abs;
break;
case GGML_UNARY_OP_SGN:
if (!any_on_device) { return false; }
func = ggml_cl_sgn;
break;
case GGML_UNARY_OP_STEP:
if (!any_on_device) { return false; }
func = ggml_cl_step;
break;
case GGML_UNARY_OP_ELU:
if (!any_on_device) { return false; }
func = ggml_cl_elu;
break;
case GGML_UNARY_OP_HARDSWISH:
if (!any_on_device) { return false; }
func = ggml_cl_hardswish;
break;
case GGML_UNARY_OP_HARDSIGMOID:
if (!any_on_device) { return false; }
func = ggml_cl_hardsigmoid;
break;
case GGML_UNARY_OP_FLOOR:
if (!any_on_device) { return false; }
func = ggml_cl_floor;
break;
case GGML_UNARY_OP_CEIL:
if (!any_on_device) { return false; }
func = ggml_cl_ceil;
break;
case GGML_UNARY_OP_ROUND:
if (!any_on_device) { return false; }
func = ggml_cl_round;
break;
case GGML_UNARY_OP_TRUNC:
if (!any_on_device) { return false; }
func = ggml_cl_trunc;
break;
case GGML_UNARY_OP_SOFTPLUS:
if (!any_on_device) {
return false;
ggml/src/ggml-opencl/kernels/concat.cl
+64 -54
View File
@@ -1,56 +1,66 @@
kernel void kernel_concat_f32(
global const char * src0,
ulong offset0,
global const char * src1,
ulong offset1,
global char * dst,
ulong offsetd,
int ne00,
int ne01,
int ne02,
int ne03,
ulong nb00,
ulong nb01,
ulong nb02,
ulong nb03,
ulong nb10,
ulong nb11,
ulong nb12,
ulong nb13,
int ne0,
ulong nb0,
ulong nb1,
ulong nb2,
ulong nb3,
int dim
) {
src0 = src0 + offset0;
src1 = src1 + offset1;
dst = dst + offsetd;
// concat is a pure copy, so the kernels are keyed by element byte size
// (1/2/4/8) rather than logical type, matching the CUDA backend.
const int i3 = get_group_id(2);
const int i2 = get_group_id(1);
const int i1 = get_group_id(0);
int o[4] = {0, 0, 0, 0};
o[dim] = dim == 0 ? ne00 : (dim == 1 ? ne01 : (dim == 2 ? ne02 : ne03));
global const float * x;
for (int i0 = get_local_id(0); i0 < ne0; i0 += get_local_size(0)) {
if (i0 < ne00 && i1 < ne01 && i2 < ne02 && i3 < ne03) {
x = (global const float *)(src0 + (i3 )*nb03 + (i2 )*nb02 + (i1 )*nb01 + (i0 )*nb00);
} else {
x = (global const float *)(src1 + (i3 - o[3])*nb13 + (i2 - o[2])*nb12 + (i1 - o[1])*nb11 + (i0 - o[0])*nb10);
}
global float * y = (global float *)(dst + i3*nb3 + i2*nb2 + i1*nb1 + i0*nb0);
*y = *x;
}
#define KERNEL_CONCAT(SUFFIX, T) \
kernel void kernel_concat_##SUFFIX( \
global const char * src0, \
ulong offset0, \
global const char * src1, \
ulong offset1, \
global char * dst, \
ulong offsetd, \
int ne00, \
int ne01, \
int ne02, \
int ne03, \
ulong nb00, \
ulong nb01, \
ulong nb02, \
ulong nb03, \
ulong nb10, \
ulong nb11, \
ulong nb12, \
ulong nb13, \
int ne0, \
ulong nb0, \
ulong nb1, \
ulong nb2, \
ulong nb3, \
int dim \
) { \
src0 = src0 + offset0; \
src1 = src1 + offset1; \
dst = dst + offsetd; \
\
const int i3 = get_group_id(2); \
const int i2 = get_group_id(1); \
const int i1 = get_group_id(0); \
\
int o[4] = {0, 0, 0, 0}; \
o[dim] = dim == 0 ? ne00 : (dim == 1 ? ne01 : (dim == 2 ? ne02 : ne03)); \
\
global const T * x; \
\
for (int i0 = get_local_id(0); i0 < ne0; i0 += get_local_size(0)) { \
if (i0 < ne00 && i1 < ne01 && i2 < ne02 && i3 < ne03) { \
x = (global const T *)(src0 + (i3 )*nb03 + (i2 )*nb02 + (i1 )*nb01 + (i0 )*nb00); \
} else { \
x = (global const T *)(src1 + (i3 - o[3])*nb13 + (i2 - o[2])*nb12 + (i1 - o[1])*nb11 + (i0 - o[0])*nb10); \
} \
\
global T * y = (global T *)(dst + i3*nb3 + i2*nb2 + i1*nb1 + i0*nb0); \
\
*y = *x; \
} \
}
kernel void kernel_concat_f32_pack(
KERNEL_CONCAT(b1, char)
KERNEL_CONCAT(b2, short)
KERNEL_CONCAT(b4, int)
KERNEL_CONCAT(b8, long)
// packed variant for the common dim==0, small-ne0 case (4-byte elements only).
kernel void kernel_concat_b4_pack(
global const char * src0,
ulong offset0,
global const char * src1,
@@ -104,14 +114,14 @@ kernel void kernel_concat_f32_pack(
o[dim] = dim == 0 ? ne00 : (dim == 1 ? ne01 : (dim == 2 ? ne02 : ne03));
for (int i0 = lane; i0 < ne0; i0 += tpr) {
global const float * x;
global const int * x;
if (i0 < ne00 && i1 < ne01 && i2 < ne02 && i3 < ne03) {
x = (global const float *)(src0 + (i3 )*nb03 + (i2 )*nb02 + (i1 )*nb01 + (i0 )*nb00);
x = (global const int *)(src0 + (i3 )*nb03 + (i2 )*nb02 + (i1 )*nb01 + (i0 )*nb00);
} else {
x = (global const float *)(src1 + (i3 - o[3])*nb13 + (i2 - o[2])*nb12 + (i1 - o[1])*nb11 + (i0 - o[0])*nb10);
x = (global const int *)(src1 + (i3 - o[3])*nb13 + (i2 - o[2])*nb12 + (i1 - o[1])*nb11 + (i0 - o[0])*nb10);
}
global float * y = (global float *)(dst + i3*nb3 + i2*nb2 + i1*nb1 + i0*nb0);
global int * y = (global int *)(dst + i3*nb3 + i2*nb2 + i1*nb1 + i0*nb0);
*y = *x;
}
ggml/src/ggml-opencl/kernels/cpy.cl
+25
View File
@@ -286,3 +286,28 @@ kernel void kernel_cpy_i32_i32(
dst_data[i00] = src[0];
}
}
// Contiguous f32 copy, one work item per float4 over the whole tensor. The kernels above map
// one workgroup to each row, which leaves a tensor with few long rows on a single compute unit.
// vload4/vstore4 rather than a float4 cast: these buffers carry an arbitrary 4-byte view offset.
kernel void kernel_cpy_f32_f32_flat(
global float * src0,
ulong offset0,
global float * dst,
ulong offsetd,
ulong ne,
ulong n4
) {
src0 = (global float*)((global char*)src0 + offset0);
dst = (global float*)((global char*)dst + offsetd);
const ulong i = get_global_id(0);
if (i < n4) {
vstore4(vload4(i, src0), i, dst);
} else if (i == n4) {
for (ulong t = n4 * 4; t < ne; ++t) {
dst[t] = src0[t];
}
}
}
ggml/src/ggml-opencl/kernels/unary_ext.cl
+85
View File
@@ -0,0 +1,85 @@
#pragma OPENCL EXTENSION cl_khr_fp16 : enable
//------------------------------------------------------------------------------
// Extended elementwise unary ops, same variant shape as abs.cl:
// f32, f32_4 (vec4), f16, f16_4 (vec4), f32_nc, f16_nc (stride-addressed).
//
// sgn, step, elu, hardswish, hardsigmoid, floor, ceil, round, trunc.
//
// Semantics match the ggml CPU reference (ggml.c). Values are computed in float
// (the f16 variants read/write half and convert), so the conditional ops match
// the CPU bit-for-bit within tolerance. SEXPR is the scalar form, VEXPR the
// float4 form (vector ternaries need select()).
//------------------------------------------------------------------------------
#define UNARY_EXT(NAME, SEXPR, VEXPR) \
kernel void kernel_##NAME##_f32( \
global const float * src0, ulong offset0, \
global float * dst, ulong offsetd) { \
src0 = (global float*)((global char*)src0 + offset0); \
dst = (global float*)((global char*)dst + offsetd); \
float x = src0[get_global_id(0)]; \
dst[get_global_id(0)] = (SEXPR); \
} \
kernel void kernel_##NAME##_f32_4( \
global const float4 * src0, ulong offset0, \
global float4 * dst, ulong offsetd) { \
src0 = (global float4*)((global char*)src0 + offset0); \
dst = (global float4*)((global char*)dst + offsetd); \
float4 x = src0[get_global_id(0)]; \
dst[get_global_id(0)] = (VEXPR); \
} \
kernel void kernel_##NAME##_f16( \
global const half * src0, ulong offset0, \
global half * dst, ulong offsetd) { \
src0 = (global half*)((global char*)src0 + offset0); \
dst = (global half*)((global char*)dst + offsetd); \
float x = src0[get_global_id(0)]; \
dst[get_global_id(0)] = (SEXPR); \
} \
kernel void kernel_##NAME##_f16_4( \
global const half4 * src0, ulong offset0, \
global half4 * dst, ulong offsetd) { \
src0 = (global half4*)((global char*)src0 + offset0); \
dst = (global half4*)((global char*)dst + offsetd); \
float4 x = convert_float4(src0[get_global_id(0)]); \
dst[get_global_id(0)] = convert_half4(VEXPR); \
} \
kernel void kernel_##NAME##_f32_nc( \
global const char * src0, ulong offset0, \
global char * dst, ulong offsetd, \
int ne00, ulong nb00, ulong nb01, ulong nb02, ulong nb03, \
ulong nb0, ulong nb1, ulong nb2, ulong nb3) { \
src0 = src0 + offset0; dst = dst + offsetd; \
const int i3 = get_group_id(2); \
const int i2 = get_group_id(1); \
const int i1 = get_group_id(0); \
for (int i0 = get_local_id(0); i0 < ne00; i0 += get_local_size(0)) { \
float x = *(global const float *)(src0 + i3*nb03 + i2*nb02 + i1*nb01 + i0*nb00); \
*(global float *)(dst + i3*nb3 + i2*nb2 + i1*nb1 + i0*nb0) = (SEXPR); \
} \
} \
kernel void kernel_##NAME##_f16_nc( \
global const char * src0, ulong offset0, \
global char * dst, ulong offsetd, \
int ne00, ulong nb00, ulong nb01, ulong nb02, ulong nb03, \
ulong nb0, ulong nb1, ulong nb2, ulong nb3) { \
src0 = src0 + offset0; dst = dst + offsetd; \
const int i3 = get_group_id(2); \
const int i2 = get_group_id(1); \
const int i1 = get_group_id(0); \
for (int i0 = get_local_id(0); i0 < ne00; i0 += get_local_size(0)) {\
float x = *(global const half *)(src0 + i3*nb03 + i2*nb02 + i1*nb01 + i0*nb00); \
*(global half *)(dst + i3*nb3 + i2*nb2 + i1*nb1 + i0*nb0) = (SEXPR); \
} \
}
UNARY_EXT(sgn, sign(x), sign(x))
UNARY_EXT(step, x > 0.0f ? 1.0f : 0.0f, select((float4)0.0f, (float4)1.0f, x > 0.0f))
UNARY_EXT(elu, x > 0.0f ? x : expm1(x), select(expm1(x), x, x > 0.0f))
UNARY_EXT(hardswish, x * fmin(1.0f, fmax(0.0f, (x + 3.0f) / 6.0f)), x * fmin((float4)1.0f, fmax((float4)0.0f, (x + 3.0f) / 6.0f)))
UNARY_EXT(hardsigmoid, fmin(1.0f, fmax(0.0f, (x + 3.0f) / 6.0f)), fmin((float4)1.0f, fmax((float4)0.0f, (x + 3.0f) / 6.0f)))
UNARY_EXT(floor, floor(x), floor(x))
UNARY_EXT(ceil, ceil(x), ceil(x))
UNARY_EXT(round, round(x), round(x))
UNARY_EXT(trunc, trunc(x), trunc(x))