opencl: add DSV4_HC_PRE and DSV4_HC_POST (#31623)

* opencl: add DSV4_HC_PRE and DSV4_HC_POST

Co-authored-by: Shafei Qi <shaoqi@qti.qualcomm.com>

* opencl: improve dsv4_hc_post

Co-authored-by: Shaofei Qi <shaoqi@qti.qualcomm.com>

---------

Co-authored-by: Li He <lih@qti.qualcomm.com>
This commit is contained in:
shaofeiqiandLi He authored and GitHub committed 2026-11-04 19:14:11 -08:00
1 parent 5ce06fc71f
commit 8415bd7bb3
3 files changed
+190 -1

No files matched your search

+1
View File
@@ -78,6 +78,7 @@ set(GGML_OPENCL_KERNELS
diag_mask_inf
diag
div
dsv4_hc
gelu
get_rows
glu
+88 -1
View File
@@ -1104,6 +1104,10 @@ struct ggml_backend_opencl_context {
cl_kernel kernel_argmax_f32_i32;
cl_kernel kernel_argsort_f32_i32;
cl_kernel kernel_sum_rows_f32, kernel_sum_rows_f32_4;
cl_kernel kernel_dsv4_hc_pre = nullptr;
cl_kernel kernel_dsv4_hc_post[2] = {};
size_t local_dsv4_hc_pre = 0;
size_t local_dsv4_hc_post[2] = {};
cl_kernel kernel_cumsum_blk, kernel_cumsum_add;
cl_kernel kernel_repeat_f32;
cl_kernel kernel_pad;
@@ -1463,6 +1467,14 @@ struct ggml_backend_opencl_context {
#endif // GGML_OPENCL_USE_ADRENO_KERNELS
~ggml_backend_opencl_context() {
if (kernel_dsv4_hc_pre) {
CL_CHECK(clReleaseKernel(kernel_dsv4_hc_pre));
}
for (int i = 0; i < 2; ++i) {
if (kernel_dsv4_hc_post[i]) {
CL_CHECK(clReleaseKernel(kernel_dsv4_hc_post[i]));
}
}
if (iq1_s_grid_image) {
CL_CHECK(clReleaseMemObject(iq1_s_grid_image));
}
@@ -4227,6 +4239,28 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) {
GGML_LOG_CONT(".");
}
// dsv4_hc
{
#ifdef GGML_OPENCL_EMBED_KERNELS
const std::string kernel_src {
#include "dsv4_hc.cl.h"
};
#else
const std::string kernel_src = read_file("dsv4_hc.cl");
#endif
for (int i = 0; i < 2; ++i) {
cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts + (i == 0 ? " -DHC_VEC=1" : " -DHC_VEC=4"));
if (i == 0) {
CL_CHECK((backend_ctx->kernel_dsv4_hc_pre = clCreateKernel(prog, "kernel_dsv4_hc_pre", &err), err));
backend_ctx->local_dsv4_hc_pre = MIN((size_t) 64, backend_ctx->get_kernel_workgroup_size(backend_ctx->kernel_dsv4_hc_pre));
}
CL_CHECK((backend_ctx->kernel_dsv4_hc_post[i] = clCreateKernel(prog, "kernel_dsv4_hc_post", &err), err));
backend_ctx->local_dsv4_hc_post[i] = MIN((size_t) 64, backend_ctx->get_kernel_workgroup_size(backend_ctx->kernel_dsv4_hc_post[i]));
CL_CHECK(clReleaseProgram(prog));
}
GGML_LOG_CONT(".");
}
// sum_rows
{
#ifdef GGML_OPENCL_EMBED_KERNELS
@@ -12048,6 +12082,14 @@ static bool ggml_opencl_supports_op(ggml_backend_dev_t dev, const struct ggml_te
case GGML_OP_SUM_ROWS:
case GGML_OP_CUMSUM:
return op->src[0]->type == GGML_TYPE_F32 && ggml_is_contiguous(op->src[0]);
case GGML_OP_DSV4_HC_PRE:
case GGML_OP_DSV4_HC_POST:
for (int i = 0; i < (op->op == GGML_OP_DSV4_HC_PRE ? 2 : 4); ++i) {
if (op->src[i] && op->src[i]->type != GGML_TYPE_F32) {
return false;
}
}
return op->type == GGML_TYPE_F32 && ggml_is_contiguous(op);
case GGML_OP_MEAN:
return op->src[0]->type == GGML_TYPE_F32;
case GGML_OP_FLASH_ATTN_EXT: {
@@ -18274,7 +18316,7 @@ static void ggml_cl_add(ggml_backend_t backend, const ggml_tensor * src0, const
cl_kernel kernel;
const bool bcast_row = ggml_nelements(src1) == ne10 && ggml_is_contiguous(src1) && ne00 % 4 == 0 && ne10 % 4 == 0;
const bool bcast_row = ggml_is_contiguous(src0) && ggml_nelements(src1) == ne10 && ggml_is_contiguous(src1) && ne00 % 4 == 0 && ne10 % 4 == 0;
if (bcast_row) {
GGML_ASSERT(ggml_is_contiguous(src0));
@@ -37839,6 +37881,47 @@ static void ggml_cl_argsort(ggml_backend_t backend, const ggml_tensor * src0, co
#endif // GGML_OPENCL_USE_ADRENO_KERNELS
}
static void ggml_cl_dsv4_hc(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {
auto * backend_ctx = (ggml_backend_opencl_context *) backend->context;
const bool pre = dst->op == GGML_OP_DSV4_HC_PRE;
const bool vec = !pre && src0->ne[0] % 4 == 0 && src0->nb[0] == sizeof(float) && src1->nb[0] == sizeof(float);
cl_kernel kernel = pre ? backend_ctx->kernel_dsv4_hc_pre : backend_ctx->kernel_dsv4_hc_post[vec];
cl_uint arg = 0;
for (int i = 0; i < (pre ? 2 : 4); ++i) {
const ggml_tensor * src = dst->src[i];
auto * extra = src ? (ggml_tensor_extra_cl *) src->extra : nullptr;
const cl_mem buffer = extra ? extra->data_device : nullptr;
const cl_ulong offset = src ? extra->offset + src->view_offs : 0;
CL_CHECK(clSetKernelArg(kernel, arg++, sizeof(cl_mem), &buffer));
CL_CHECK(clSetKernelArg(kernel, arg++, sizeof(cl_ulong), &offset));
for (int j = 0; j < 3; ++j) {
const cl_ulong stride = src ? src->nb[j] : 0;
CL_CHECK(clSetKernelArg(kernel, arg++, sizeof(cl_ulong), &stride));
}
}
auto * extra = (ggml_tensor_extra_cl *) dst->extra;
const cl_ulong offset = extra->offset + dst->view_offs;
const cl_long n_embd = src0->ne[0];
const cl_long n_hc = pre ? src0->ne[1] : src1->ne[1];
CL_CHECK(clSetKernelArg(kernel, arg++, sizeof(cl_mem), &extra->data_device));
CL_CHECK(clSetKernelArg(kernel, arg++, sizeof(cl_ulong), &offset));
CL_CHECK(clSetKernelArg(kernel, arg++, sizeof(cl_long), &n_embd));
CL_CHECK(clSetKernelArg(kernel, arg++, sizeof(cl_long), &n_hc));
if (pre) {
const float scale = ggml_get_op_params_f32(dst, 0);
const cl_int gated = ggml_get_op_params_i32(dst, 1);
CL_CHECK(clSetKernelArg(kernel, arg++, sizeof(float), &scale));
CL_CHECK(clSetKernelArg(kernel, arg++, sizeof(cl_int), &gated));
}
const size_t n_work = (size_t) n_embd / (vec ? 4 : 1);
size_t local_work_size[] = { pre ? backend_ctx->local_dsv4_hc_pre : backend_ctx->local_dsv4_hc_post[vec], 1, 1 };
size_t global_work_size[] = { CEIL_DIV(n_work, local_work_size[0]) * local_work_size[0], (size_t) dst->ne[1], (size_t) dst->ne[2] };
backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst);
}
static void ggml_cl_sum_rows(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {
GGML_ASSERT(src0);
GGML_ASSERT(src0->extra);
@@ -38726,6 +38809,10 @@ bool ggml_cl_compute_forward(ggml_backend_t backend, struct ggml_tensor * tensor
}
func = ggml_cl_sum_rows;
break;
case GGML_OP_DSV4_HC_PRE:
case GGML_OP_DSV4_HC_POST:
func = ggml_cl_dsv4_hc;
break;
case GGML_OP_CUMSUM:
if (!any_on_device) {
return false;
+101
View File
@@ -0,0 +1,101 @@
#if HC_VEC == 4
typedef float4 hc_float;
#define HC_LOAD(p) vload4(0, (global const float *) (p))
#define HC_STORE(v, p) vstore4(v, 0, (global float *) (p))
#else
typedef float hc_float;
#define HC_LOAD(p) (*(global const float *) (p))
#define HC_STORE(v, p) (*(global float *) (p) = (v))
#endif
#if HC_VEC == 1
kernel void kernel_dsv4_hc_pre(
global const char * x,
ulong offset_x,
ulong nbx0,
ulong nbx1,
ulong nbx2,
global const char * weights,
ulong offset_w,
ulong nbw0,
ulong nbw1,
ulong nbw2,
global char * dst,
ulong offset_dst,
long n_embd,
long n_hc,
float scale,
int gated
) {
const size_t i = get_global_id(0);
const size_t t = get_global_id(1);
if (i >= n_embd) {
return;
}
x += offset_x + i*nbx0 + t*nbx2;
weights += offset_w + (gated ? i*nbw0 + t*nbw2 : t*nbw1);
float sum = 0.0f;
for (long h = 0; h < n_hc; ++h) {
const float xv = *(global const float *) (x + h*nbx1);
float wv = *(global const float *) (weights + h*(gated ? nbw1 : nbw0));
if (gated) {
wv = 1.0f / (1.0f + exp(-wv));
}
sum += xv * wv;
}
((global float *) (dst + offset_dst))[i + t*n_embd] = scale * sum;
}
#endif
kernel void kernel_dsv4_hc_post(
global const char * x,
ulong offset_x,
ulong nbx0,
ulong nbx1,
ulong nbx2,
global const char * residual,
ulong offset_r,
ulong nbr0,
ulong nbr1,
ulong nbr2,
global const char * post,
ulong offset_p,
ulong nbp0,
ulong nbp1,
ulong nbp2,
global const char * comb,
ulong offset_c,
ulong nbc0,
ulong nbc1,
ulong nbc2,
global char * dst,
ulong offset_dst,
long n_embd,
long n_hc
) {
const size_t i = get_global_id(0) * HC_VEC;
const size_t h = get_global_id(1);
const size_t t = get_global_id(2);
if (i >= n_embd) {
return;
}
const hc_float xv = HC_LOAD(x + offset_x + i*nbx0 + t*nbx1);
const float pv = *(global const float *) (post + offset_p + h*nbp0 + t*nbp1);
residual += offset_r + i*nbr0 + t*nbr2;
hc_float sum = xv * pv;
if (comb) {
comb += offset_c + h*nbc0 + t*nbc2;
for (long s = 0; s < n_hc; ++s) {
const hc_float rv = HC_LOAD(residual + s*nbr1);
const float cv = *(global const float *) (comb + s*nbc1);
sum += rv * cv;
}
} else {
sum += HC_LOAD(residual + h*nbr1);
}
HC_STORE(sum, dst + offset_dst + (i + (h + t*n_hc)*n_embd)*sizeof(float));
}