diff --git a/ggml/src/ggml-opencl/CMakeLists.txt b/ggml/src/ggml-opencl/CMakeLists.txt index 20dd595c7..58a6d438e 100644 --- a/ggml/src/ggml-opencl/CMakeLists.txt +++ b/ggml/src/ggml-opencl/CMakeLists.txt @@ -78,6 +78,7 @@ set(GGML_OPENCL_KERNELS diag_mask_inf diag div + dsv4_hc gelu get_rows glu diff --git a/ggml/src/ggml-opencl/ggml-opencl.cpp b/ggml/src/ggml-opencl/ggml-opencl.cpp index a2620135d..8aa013edb 100644 --- a/ggml/src/ggml-opencl/ggml-opencl.cpp +++ b/ggml/src/ggml-opencl/ggml-opencl.cpp @@ -1104,6 +1104,10 @@ struct ggml_backend_opencl_context { cl_kernel kernel_argmax_f32_i32; cl_kernel kernel_argsort_f32_i32; cl_kernel kernel_sum_rows_f32, kernel_sum_rows_f32_4; + cl_kernel kernel_dsv4_hc_pre = nullptr; + cl_kernel kernel_dsv4_hc_post[2] = {}; + size_t local_dsv4_hc_pre = 0; + size_t local_dsv4_hc_post[2] = {}; cl_kernel kernel_cumsum_blk, kernel_cumsum_add; cl_kernel kernel_repeat_f32; cl_kernel kernel_pad; @@ -1463,6 +1467,14 @@ struct ggml_backend_opencl_context { #endif // GGML_OPENCL_USE_ADRENO_KERNELS ~ggml_backend_opencl_context() { + if (kernel_dsv4_hc_pre) { + CL_CHECK(clReleaseKernel(kernel_dsv4_hc_pre)); + } + for (int i = 0; i < 2; ++i) { + if (kernel_dsv4_hc_post[i]) { + CL_CHECK(clReleaseKernel(kernel_dsv4_hc_post[i])); + } + } if (iq1_s_grid_image) { CL_CHECK(clReleaseMemObject(iq1_s_grid_image)); } @@ -4227,6 +4239,28 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) { GGML_LOG_CONT("."); } + // dsv4_hc + { +#ifdef GGML_OPENCL_EMBED_KERNELS + const std::string kernel_src { + #include "dsv4_hc.cl.h" + }; +#else + const std::string kernel_src = read_file("dsv4_hc.cl"); +#endif + for (int i = 0; i < 2; ++i) { + cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts + (i == 0 ? " -DHC_VEC=1" : " -DHC_VEC=4")); + if (i == 0) { + CL_CHECK((backend_ctx->kernel_dsv4_hc_pre = clCreateKernel(prog, "kernel_dsv4_hc_pre", &err), err)); + backend_ctx->local_dsv4_hc_pre = MIN((size_t) 64, backend_ctx->get_kernel_workgroup_size(backend_ctx->kernel_dsv4_hc_pre)); + } + CL_CHECK((backend_ctx->kernel_dsv4_hc_post[i] = clCreateKernel(prog, "kernel_dsv4_hc_post", &err), err)); + backend_ctx->local_dsv4_hc_post[i] = MIN((size_t) 64, backend_ctx->get_kernel_workgroup_size(backend_ctx->kernel_dsv4_hc_post[i])); + CL_CHECK(clReleaseProgram(prog)); + } + GGML_LOG_CONT("."); + } + // sum_rows { #ifdef GGML_OPENCL_EMBED_KERNELS @@ -12048,6 +12082,14 @@ static bool ggml_opencl_supports_op(ggml_backend_dev_t dev, const struct ggml_te case GGML_OP_SUM_ROWS: case GGML_OP_CUMSUM: return op->src[0]->type == GGML_TYPE_F32 && ggml_is_contiguous(op->src[0]); + case GGML_OP_DSV4_HC_PRE: + case GGML_OP_DSV4_HC_POST: + for (int i = 0; i < (op->op == GGML_OP_DSV4_HC_PRE ? 2 : 4); ++i) { + if (op->src[i] && op->src[i]->type != GGML_TYPE_F32) { + return false; + } + } + return op->type == GGML_TYPE_F32 && ggml_is_contiguous(op); case GGML_OP_MEAN: return op->src[0]->type == GGML_TYPE_F32; case GGML_OP_FLASH_ATTN_EXT: { @@ -18274,7 +18316,7 @@ static void ggml_cl_add(ggml_backend_t backend, const ggml_tensor * src0, const cl_kernel kernel; - const bool bcast_row = ggml_nelements(src1) == ne10 && ggml_is_contiguous(src1) && ne00 % 4 == 0 && ne10 % 4 == 0; + const bool bcast_row = ggml_is_contiguous(src0) && ggml_nelements(src1) == ne10 && ggml_is_contiguous(src1) && ne00 % 4 == 0 && ne10 % 4 == 0; if (bcast_row) { GGML_ASSERT(ggml_is_contiguous(src0)); @@ -37839,6 +37881,47 @@ static void ggml_cl_argsort(ggml_backend_t backend, const ggml_tensor * src0, co #endif // GGML_OPENCL_USE_ADRENO_KERNELS } +static void ggml_cl_dsv4_hc(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { + auto * backend_ctx = (ggml_backend_opencl_context *) backend->context; + const bool pre = dst->op == GGML_OP_DSV4_HC_PRE; + const bool vec = !pre && src0->ne[0] % 4 == 0 && src0->nb[0] == sizeof(float) && src1->nb[0] == sizeof(float); + cl_kernel kernel = pre ? backend_ctx->kernel_dsv4_hc_pre : backend_ctx->kernel_dsv4_hc_post[vec]; + + cl_uint arg = 0; + for (int i = 0; i < (pre ? 2 : 4); ++i) { + const ggml_tensor * src = dst->src[i]; + auto * extra = src ? (ggml_tensor_extra_cl *) src->extra : nullptr; + const cl_mem buffer = extra ? extra->data_device : nullptr; + const cl_ulong offset = src ? extra->offset + src->view_offs : 0; + CL_CHECK(clSetKernelArg(kernel, arg++, sizeof(cl_mem), &buffer)); + CL_CHECK(clSetKernelArg(kernel, arg++, sizeof(cl_ulong), &offset)); + for (int j = 0; j < 3; ++j) { + const cl_ulong stride = src ? src->nb[j] : 0; + CL_CHECK(clSetKernelArg(kernel, arg++, sizeof(cl_ulong), &stride)); + } + } + + auto * extra = (ggml_tensor_extra_cl *) dst->extra; + const cl_ulong offset = extra->offset + dst->view_offs; + const cl_long n_embd = src0->ne[0]; + const cl_long n_hc = pre ? src0->ne[1] : src1->ne[1]; + CL_CHECK(clSetKernelArg(kernel, arg++, sizeof(cl_mem), &extra->data_device)); + CL_CHECK(clSetKernelArg(kernel, arg++, sizeof(cl_ulong), &offset)); + CL_CHECK(clSetKernelArg(kernel, arg++, sizeof(cl_long), &n_embd)); + CL_CHECK(clSetKernelArg(kernel, arg++, sizeof(cl_long), &n_hc)); + if (pre) { + const float scale = ggml_get_op_params_f32(dst, 0); + const cl_int gated = ggml_get_op_params_i32(dst, 1); + CL_CHECK(clSetKernelArg(kernel, arg++, sizeof(float), &scale)); + CL_CHECK(clSetKernelArg(kernel, arg++, sizeof(cl_int), &gated)); + } + + const size_t n_work = (size_t) n_embd / (vec ? 4 : 1); + size_t local_work_size[] = { pre ? backend_ctx->local_dsv4_hc_pre : backend_ctx->local_dsv4_hc_post[vec], 1, 1 }; + size_t global_work_size[] = { CEIL_DIV(n_work, local_work_size[0]) * local_work_size[0], (size_t) dst->ne[1], (size_t) dst->ne[2] }; + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); +} + static void ggml_cl_sum_rows(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { GGML_ASSERT(src0); GGML_ASSERT(src0->extra); @@ -38726,6 +38809,10 @@ bool ggml_cl_compute_forward(ggml_backend_t backend, struct ggml_tensor * tensor } func = ggml_cl_sum_rows; break; + case GGML_OP_DSV4_HC_PRE: + case GGML_OP_DSV4_HC_POST: + func = ggml_cl_dsv4_hc; + break; case GGML_OP_CUMSUM: if (!any_on_device) { return false; diff --git a/ggml/src/ggml-opencl/kernels/dsv4_hc.cl b/ggml/src/ggml-opencl/kernels/dsv4_hc.cl new file mode 100644 index 000000000..164ab1dae --- /dev/null +++ b/ggml/src/ggml-opencl/kernels/dsv4_hc.cl @@ -0,0 +1,101 @@ +#if HC_VEC == 4 +typedef float4 hc_float; +#define HC_LOAD(p) vload4(0, (global const float *) (p)) +#define HC_STORE(v, p) vstore4(v, 0, (global float *) (p)) +#else +typedef float hc_float; +#define HC_LOAD(p) (*(global const float *) (p)) +#define HC_STORE(v, p) (*(global float *) (p) = (v)) +#endif + +#if HC_VEC == 1 +kernel void kernel_dsv4_hc_pre( + global const char * x, + ulong offset_x, + ulong nbx0, + ulong nbx1, + ulong nbx2, + global const char * weights, + ulong offset_w, + ulong nbw0, + ulong nbw1, + ulong nbw2, + global char * dst, + ulong offset_dst, + long n_embd, + long n_hc, + float scale, + int gated +) { + const size_t i = get_global_id(0); + const size_t t = get_global_id(1); + if (i >= n_embd) { + return; + } + + x += offset_x + i*nbx0 + t*nbx2; + weights += offset_w + (gated ? i*nbw0 + t*nbw2 : t*nbw1); + + float sum = 0.0f; + for (long h = 0; h < n_hc; ++h) { + const float xv = *(global const float *) (x + h*nbx1); + float wv = *(global const float *) (weights + h*(gated ? nbw1 : nbw0)); + if (gated) { + wv = 1.0f / (1.0f + exp(-wv)); + } + sum += xv * wv; + } + ((global float *) (dst + offset_dst))[i + t*n_embd] = scale * sum; +} +#endif + +kernel void kernel_dsv4_hc_post( + global const char * x, + ulong offset_x, + ulong nbx0, + ulong nbx1, + ulong nbx2, + global const char * residual, + ulong offset_r, + ulong nbr0, + ulong nbr1, + ulong nbr2, + global const char * post, + ulong offset_p, + ulong nbp0, + ulong nbp1, + ulong nbp2, + global const char * comb, + ulong offset_c, + ulong nbc0, + ulong nbc1, + ulong nbc2, + global char * dst, + ulong offset_dst, + long n_embd, + long n_hc +) { + const size_t i = get_global_id(0) * HC_VEC; + const size_t h = get_global_id(1); + const size_t t = get_global_id(2); + if (i >= n_embd) { + return; + } + + const hc_float xv = HC_LOAD(x + offset_x + i*nbx0 + t*nbx1); + const float pv = *(global const float *) (post + offset_p + h*nbp0 + t*nbp1); + residual += offset_r + i*nbr0 + t*nbr2; + + hc_float sum = xv * pv; + if (comb) { + comb += offset_c + h*nbc0 + t*nbc2; + for (long s = 0; s < n_hc; ++s) { + const hc_float rv = HC_LOAD(residual + s*nbr1); + const float cv = *(global const float *) (comb + s*nbc1); + sum += rv * cv; + } + } else { + sum += HC_LOAD(residual + h*nbr1); + } + HC_STORE(sum, dst + offset_dst + (i + (h + t*n_hc)*n_embd)*sizeof(float)); +}