opencl: add DSV4_HC_PRE and DSV4_HC_POST (#31623)
* opencl: add DSV4_HC_PRE and DSV4_HC_POST Co-authored-by: Shafei Qi <shaoqi@qti.qualcomm.com> * opencl: improve dsv4_hc_post Co-authored-by: Shaofei Qi <shaoqi@qti.qualcomm.com> --------- Co-authored-by: Li He <lih@qti.qualcomm.com>
This commit is contained in:
1 parent
5ce06fc71f
commit
8415bd7bb3
3 files changed
+190
-1
No files matched your search
@@ -78,6 +78,7 @@ set(GGML_OPENCL_KERNELS
|
||||
diag_mask_inf
|
||||
diag
|
||||
div
|
||||
dsv4_hc
|
||||
gelu
|
||||
get_rows
|
||||
glu
|
||||
|
||||
@@ -1104,6 +1104,10 @@ struct ggml_backend_opencl_context {
|
||||
cl_kernel kernel_argmax_f32_i32;
|
||||
cl_kernel kernel_argsort_f32_i32;
|
||||
cl_kernel kernel_sum_rows_f32, kernel_sum_rows_f32_4;
|
||||
cl_kernel kernel_dsv4_hc_pre = nullptr;
|
||||
cl_kernel kernel_dsv4_hc_post[2] = {};
|
||||
size_t local_dsv4_hc_pre = 0;
|
||||
size_t local_dsv4_hc_post[2] = {};
|
||||
cl_kernel kernel_cumsum_blk, kernel_cumsum_add;
|
||||
cl_kernel kernel_repeat_f32;
|
||||
cl_kernel kernel_pad;
|
||||
@@ -1463,6 +1467,14 @@ struct ggml_backend_opencl_context {
|
||||
#endif // GGML_OPENCL_USE_ADRENO_KERNELS
|
||||
|
||||
~ggml_backend_opencl_context() {
|
||||
if (kernel_dsv4_hc_pre) {
|
||||
CL_CHECK(clReleaseKernel(kernel_dsv4_hc_pre));
|
||||
}
|
||||
for (int i = 0; i < 2; ++i) {
|
||||
if (kernel_dsv4_hc_post[i]) {
|
||||
CL_CHECK(clReleaseKernel(kernel_dsv4_hc_post[i]));
|
||||
}
|
||||
}
|
||||
if (iq1_s_grid_image) {
|
||||
CL_CHECK(clReleaseMemObject(iq1_s_grid_image));
|
||||
}
|
||||
@@ -4227,6 +4239,28 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) {
|
||||
GGML_LOG_CONT(".");
|
||||
}
|
||||
|
||||
// dsv4_hc
|
||||
{
|
||||
#ifdef GGML_OPENCL_EMBED_KERNELS
|
||||
const std::string kernel_src {
|
||||
#include "dsv4_hc.cl.h"
|
||||
};
|
||||
#else
|
||||
const std::string kernel_src = read_file("dsv4_hc.cl");
|
||||
#endif
|
||||
for (int i = 0; i < 2; ++i) {
|
||||
cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts + (i == 0 ? " -DHC_VEC=1" : " -DHC_VEC=4"));
|
||||
if (i == 0) {
|
||||
CL_CHECK((backend_ctx->kernel_dsv4_hc_pre = clCreateKernel(prog, "kernel_dsv4_hc_pre", &err), err));
|
||||
backend_ctx->local_dsv4_hc_pre = MIN((size_t) 64, backend_ctx->get_kernel_workgroup_size(backend_ctx->kernel_dsv4_hc_pre));
|
||||
}
|
||||
CL_CHECK((backend_ctx->kernel_dsv4_hc_post[i] = clCreateKernel(prog, "kernel_dsv4_hc_post", &err), err));
|
||||
backend_ctx->local_dsv4_hc_post[i] = MIN((size_t) 64, backend_ctx->get_kernel_workgroup_size(backend_ctx->kernel_dsv4_hc_post[i]));
|
||||
CL_CHECK(clReleaseProgram(prog));
|
||||
}
|
||||
GGML_LOG_CONT(".");
|
||||
}
|
||||
|
||||
// sum_rows
|
||||
{
|
||||
#ifdef GGML_OPENCL_EMBED_KERNELS
|
||||
@@ -12048,6 +12082,14 @@ static bool ggml_opencl_supports_op(ggml_backend_dev_t dev, const struct ggml_te
|
||||
case GGML_OP_SUM_ROWS:
|
||||
case GGML_OP_CUMSUM:
|
||||
return op->src[0]->type == GGML_TYPE_F32 && ggml_is_contiguous(op->src[0]);
|
||||
case GGML_OP_DSV4_HC_PRE:
|
||||
case GGML_OP_DSV4_HC_POST:
|
||||
for (int i = 0; i < (op->op == GGML_OP_DSV4_HC_PRE ? 2 : 4); ++i) {
|
||||
if (op->src[i] && op->src[i]->type != GGML_TYPE_F32) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
return op->type == GGML_TYPE_F32 && ggml_is_contiguous(op);
|
||||
case GGML_OP_MEAN:
|
||||
return op->src[0]->type == GGML_TYPE_F32;
|
||||
case GGML_OP_FLASH_ATTN_EXT: {
|
||||
@@ -18274,7 +18316,7 @@ static void ggml_cl_add(ggml_backend_t backend, const ggml_tensor * src0, const
|
||||
|
||||
cl_kernel kernel;
|
||||
|
||||
const bool bcast_row = ggml_nelements(src1) == ne10 && ggml_is_contiguous(src1) && ne00 % 4 == 0 && ne10 % 4 == 0;
|
||||
const bool bcast_row = ggml_is_contiguous(src0) && ggml_nelements(src1) == ne10 && ggml_is_contiguous(src1) && ne00 % 4 == 0 && ne10 % 4 == 0;
|
||||
|
||||
if (bcast_row) {
|
||||
GGML_ASSERT(ggml_is_contiguous(src0));
|
||||
@@ -37839,6 +37881,47 @@ static void ggml_cl_argsort(ggml_backend_t backend, const ggml_tensor * src0, co
|
||||
#endif // GGML_OPENCL_USE_ADRENO_KERNELS
|
||||
}
|
||||
|
||||
static void ggml_cl_dsv4_hc(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {
|
||||
auto * backend_ctx = (ggml_backend_opencl_context *) backend->context;
|
||||
const bool pre = dst->op == GGML_OP_DSV4_HC_PRE;
|
||||
const bool vec = !pre && src0->ne[0] % 4 == 0 && src0->nb[0] == sizeof(float) && src1->nb[0] == sizeof(float);
|
||||
cl_kernel kernel = pre ? backend_ctx->kernel_dsv4_hc_pre : backend_ctx->kernel_dsv4_hc_post[vec];
|
||||
|
||||
cl_uint arg = 0;
|
||||
for (int i = 0; i < (pre ? 2 : 4); ++i) {
|
||||
const ggml_tensor * src = dst->src[i];
|
||||
auto * extra = src ? (ggml_tensor_extra_cl *) src->extra : nullptr;
|
||||
const cl_mem buffer = extra ? extra->data_device : nullptr;
|
||||
const cl_ulong offset = src ? extra->offset + src->view_offs : 0;
|
||||
CL_CHECK(clSetKernelArg(kernel, arg++, sizeof(cl_mem), &buffer));
|
||||
CL_CHECK(clSetKernelArg(kernel, arg++, sizeof(cl_ulong), &offset));
|
||||
for (int j = 0; j < 3; ++j) {
|
||||
const cl_ulong stride = src ? src->nb[j] : 0;
|
||||
CL_CHECK(clSetKernelArg(kernel, arg++, sizeof(cl_ulong), &stride));
|
||||
}
|
||||
}
|
||||
|
||||
auto * extra = (ggml_tensor_extra_cl *) dst->extra;
|
||||
const cl_ulong offset = extra->offset + dst->view_offs;
|
||||
const cl_long n_embd = src0->ne[0];
|
||||
const cl_long n_hc = pre ? src0->ne[1] : src1->ne[1];
|
||||
CL_CHECK(clSetKernelArg(kernel, arg++, sizeof(cl_mem), &extra->data_device));
|
||||
CL_CHECK(clSetKernelArg(kernel, arg++, sizeof(cl_ulong), &offset));
|
||||
CL_CHECK(clSetKernelArg(kernel, arg++, sizeof(cl_long), &n_embd));
|
||||
CL_CHECK(clSetKernelArg(kernel, arg++, sizeof(cl_long), &n_hc));
|
||||
if (pre) {
|
||||
const float scale = ggml_get_op_params_f32(dst, 0);
|
||||
const cl_int gated = ggml_get_op_params_i32(dst, 1);
|
||||
CL_CHECK(clSetKernelArg(kernel, arg++, sizeof(float), &scale));
|
||||
CL_CHECK(clSetKernelArg(kernel, arg++, sizeof(cl_int), &gated));
|
||||
}
|
||||
|
||||
const size_t n_work = (size_t) n_embd / (vec ? 4 : 1);
|
||||
size_t local_work_size[] = { pre ? backend_ctx->local_dsv4_hc_pre : backend_ctx->local_dsv4_hc_post[vec], 1, 1 };
|
||||
size_t global_work_size[] = { CEIL_DIV(n_work, local_work_size[0]) * local_work_size[0], (size_t) dst->ne[1], (size_t) dst->ne[2] };
|
||||
backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst);
|
||||
}
|
||||
|
||||
static void ggml_cl_sum_rows(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {
|
||||
GGML_ASSERT(src0);
|
||||
GGML_ASSERT(src0->extra);
|
||||
@@ -38726,6 +38809,10 @@ bool ggml_cl_compute_forward(ggml_backend_t backend, struct ggml_tensor * tensor
|
||||
}
|
||||
func = ggml_cl_sum_rows;
|
||||
break;
|
||||
case GGML_OP_DSV4_HC_PRE:
|
||||
case GGML_OP_DSV4_HC_POST:
|
||||
func = ggml_cl_dsv4_hc;
|
||||
break;
|
||||
case GGML_OP_CUMSUM:
|
||||
if (!any_on_device) {
|
||||
return false;
|
||||
|
||||
@@ -0,0 +1,101 @@
|
||||
#if HC_VEC == 4
|
||||
typedef float4 hc_float;
|
||||
#define HC_LOAD(p) vload4(0, (global const float *) (p))
|
||||
#define HC_STORE(v, p) vstore4(v, 0, (global float *) (p))
|
||||
#else
|
||||
typedef float hc_float;
|
||||
#define HC_LOAD(p) (*(global const float *) (p))
|
||||
#define HC_STORE(v, p) (*(global float *) (p) = (v))
|
||||
#endif
|
||||
|
||||
#if HC_VEC == 1
|
||||
kernel void kernel_dsv4_hc_pre(
|
||||
global const char * x,
|
||||
ulong offset_x,
|
||||
ulong nbx0,
|
||||
ulong nbx1,
|
||||
ulong nbx2,
|
||||
global const char * weights,
|
||||
ulong offset_w,
|
||||
ulong nbw0,
|
||||
ulong nbw1,
|
||||
ulong nbw2,
|
||||
global char * dst,
|
||||
ulong offset_dst,
|
||||
long n_embd,
|
||||
long n_hc,
|
||||
float scale,
|
||||
int gated
|
||||
) {
|
||||
const size_t i = get_global_id(0);
|
||||
const size_t t = get_global_id(1);
|
||||
if (i >= n_embd) {
|
||||
return;
|
||||
}
|
||||
|
||||
x += offset_x + i*nbx0 + t*nbx2;
|
||||
weights += offset_w + (gated ? i*nbw0 + t*nbw2 : t*nbw1);
|
||||
|
||||
float sum = 0.0f;
|
||||
for (long h = 0; h < n_hc; ++h) {
|
||||
const float xv = *(global const float *) (x + h*nbx1);
|
||||
float wv = *(global const float *) (weights + h*(gated ? nbw1 : nbw0));
|
||||
if (gated) {
|
||||
wv = 1.0f / (1.0f + exp(-wv));
|
||||
}
|
||||
sum += xv * wv;
|
||||
}
|
||||
((global float *) (dst + offset_dst))[i + t*n_embd] = scale * sum;
|
||||
}
|
||||
#endif
|
||||
|
||||
kernel void kernel_dsv4_hc_post(
|
||||
global const char * x,
|
||||
ulong offset_x,
|
||||
ulong nbx0,
|
||||
ulong nbx1,
|
||||
ulong nbx2,
|
||||
global const char * residual,
|
||||
ulong offset_r,
|
||||
ulong nbr0,
|
||||
ulong nbr1,
|
||||
ulong nbr2,
|
||||
global const char * post,
|
||||
ulong offset_p,
|
||||
ulong nbp0,
|
||||
ulong nbp1,
|
||||
ulong nbp2,
|
||||
global const char * comb,
|
||||
ulong offset_c,
|
||||
ulong nbc0,
|
||||
ulong nbc1,
|
||||
ulong nbc2,
|
||||
global char * dst,
|
||||
ulong offset_dst,
|
||||
long n_embd,
|
||||
long n_hc
|
||||
) {
|
||||
const size_t i = get_global_id(0) * HC_VEC;
|
||||
const size_t h = get_global_id(1);
|
||||
const size_t t = get_global_id(2);
|
||||
if (i >= n_embd) {
|
||||
return;
|
||||
}
|
||||
|
||||
const hc_float xv = HC_LOAD(x + offset_x + i*nbx0 + t*nbx1);
|
||||
const float pv = *(global const float *) (post + offset_p + h*nbp0 + t*nbp1);
|
||||
residual += offset_r + i*nbr0 + t*nbr2;
|
||||
|
||||
hc_float sum = xv * pv;
|
||||
if (comb) {
|
||||
comb += offset_c + h*nbc0 + t*nbc2;
|
||||
for (long s = 0; s < n_hc; ++s) {
|
||||
const hc_float rv = HC_LOAD(residual + s*nbr1);
|
||||
const float cv = *(global const float *) (comb + s*nbc1);
|
||||
sum += rv * cv;
|
||||
}
|
||||
} else {
|
||||
sum += HC_LOAD(residual + h*nbr1);
|
||||
}
|
||||
HC_STORE(sum, dst + offset_dst + (i + (h + t*n_hc)*n_embd)*sizeof(float));
|
||||
}
|
||||
Reference in new issue
Block a user