opencl: add get_rows for i32, cpy for i32 and f32 (#30756)
* opencl: support i32 GET_ROWS * opencl: support i32 and f32 copy conversions
This commit is contained in:
3 files changed
+151
-3
No files matched your search
@@ -868,7 +868,7 @@ struct ggml_backend_opencl_context {
|
||||
#ifdef GGML_OPENCL_USE_ADRENO_KERNELS
|
||||
ggml_cl_adreno_xmem_attn_state adreno_xmem_attn;
|
||||
#endif
|
||||
cl_kernel kernel_get_rows_f32, kernel_get_rows_f16, kernel_get_rows_q4_0;
|
||||
cl_kernel kernel_get_rows_f32, kernel_get_rows_f16, kernel_get_rows_i32, kernel_get_rows_q4_0;
|
||||
cl_kernel kernel_set_rows_f32_i64, kernel_set_rows_f32_i32, kernel_set_rows_f16_i64, kernel_set_rows_f16_i32;
|
||||
cl_kernel kernel_set_rows_q8_0_i64, kernel_set_rows_q8_0_i32;
|
||||
cl_kernel kernel_set_rows_q8_0_soa_i64, kernel_set_rows_q8_0_soa_i32;
|
||||
@@ -876,7 +876,7 @@ struct ggml_backend_opencl_context {
|
||||
cl_kernel kernel_set_rows_q4_0_soa_i64, kernel_set_rows_q4_0_soa_i32;
|
||||
cl_kernel kernel_rope_norm_f32, kernel_rope_norm_f16, kernel_rope_neox_f32, kernel_rope_neox_f16;
|
||||
cl_kernel kernel_rope_multi_f32, kernel_rope_multi_f16, kernel_rope_vision_f32, kernel_rope_vision_f16;
|
||||
cl_kernel kernel_cpy_f16_f16, kernel_cpy_f16_f32, kernel_cpy_f32_f16, kernel_cpy_f32_f32, kernel_cpy_i32_i32;
|
||||
cl_kernel kernel_cpy_f16_f16, kernel_cpy_f16_f32, kernel_cpy_f32_f16, kernel_cpy_f32_f32, kernel_cpy_i32_i32, kernel_cpy_i32_f32, kernel_cpy_f32_i32;
|
||||
cl_kernel kernel_cpy_f32_f32_pack = nullptr;
|
||||
cl_kernel kernel_cpy_f32_f32_flat = nullptr;
|
||||
cl_kernel kernel_mul_mat_f32_f32;
|
||||
@@ -1652,6 +1652,8 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) {
|
||||
}
|
||||
}
|
||||
CL_CHECK((backend_ctx->kernel_cpy_i32_i32 = clCreateKernel(prog, "kernel_cpy_i32_i32", &err), err));
|
||||
CL_CHECK((backend_ctx->kernel_cpy_i32_f32 = clCreateKernel(prog, "kernel_cpy_i32_f32", &err), err));
|
||||
CL_CHECK((backend_ctx->kernel_cpy_f32_i32 = clCreateKernel(prog, "kernel_cpy_f32_i32", &err), err));
|
||||
GGML_LOG_CONT(".");
|
||||
}
|
||||
|
||||
@@ -1840,6 +1842,7 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) {
|
||||
build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts);
|
||||
|
||||
CL_CHECK((backend_ctx->kernel_get_rows_f32 = clCreateKernel(backend_ctx->program_get_rows, "kernel_get_rows_f32", &err), err));
|
||||
CL_CHECK((backend_ctx->kernel_get_rows_i32 = clCreateKernel(backend_ctx->program_get_rows, "kernel_get_rows_i32", &err), err));
|
||||
CL_CHECK((backend_ctx->kernel_get_rows_f16 = clCreateKernel(backend_ctx->program_get_rows, "kernel_get_rows_f16", &err), err));
|
||||
CL_CHECK((backend_ctx->kernel_get_rows_q4_0 = clCreateKernel(backend_ctx->program_get_rows, "kernel_get_rows_q4_0", &err), err));
|
||||
GGML_LOG_CONT(".");
|
||||
@@ -9102,6 +9105,8 @@ static bool ggml_opencl_supports_op(ggml_backend_dev_t dev, const struct ggml_te
|
||||
return true;
|
||||
case GGML_OP_GET_ROWS:
|
||||
switch (op->src[0]->type) {
|
||||
case GGML_TYPE_I32:
|
||||
return op->type == GGML_TYPE_I32;
|
||||
case GGML_TYPE_F32:
|
||||
case GGML_TYPE_F16:
|
||||
return true;
|
||||
@@ -9139,6 +9144,7 @@ static bool ggml_opencl_supports_op(ggml_backend_dev_t dev, const struct ggml_te
|
||||
switch (op->src[0]->type) {
|
||||
case GGML_TYPE_F32:
|
||||
switch (op->type) {
|
||||
case GGML_TYPE_I32:
|
||||
case GGML_TYPE_F16:
|
||||
case GGML_TYPE_F32:
|
||||
return true;
|
||||
@@ -9155,6 +9161,7 @@ static bool ggml_opencl_supports_op(ggml_backend_dev_t dev, const struct ggml_te
|
||||
}
|
||||
case GGML_TYPE_I32:
|
||||
switch (op->type) {
|
||||
case GGML_TYPE_F32:
|
||||
case GGML_TYPE_I32:
|
||||
return true;
|
||||
default:
|
||||
@@ -13619,6 +13626,9 @@ static void ggml_cl_get_rows(ggml_backend_t backend, const ggml_tensor * src0, c
|
||||
case GGML_TYPE_F32:
|
||||
kernel = backend_ctx->kernel_get_rows_f32;
|
||||
break;
|
||||
case GGML_TYPE_I32:
|
||||
kernel = backend_ctx->kernel_get_rows_i32;
|
||||
break;
|
||||
case GGML_TYPE_F16:
|
||||
kernel = backend_ctx->kernel_get_rows_f16;
|
||||
break;
|
||||
@@ -13654,7 +13664,7 @@ static void ggml_cl_get_rows(ggml_backend_t backend, const ggml_tensor * src0, c
|
||||
}
|
||||
|
||||
int nchunks = 1;
|
||||
if (src0->type == GGML_TYPE_F32) {
|
||||
if (src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_I32) {
|
||||
const int chunk_target = nth * 4;
|
||||
nchunks = (ne00 + chunk_target - 1) / chunk_target;
|
||||
nchunks = MAX(1, MIN(nchunks, 64));
|
||||
@@ -28726,6 +28736,9 @@ static void ggml_cl_cpy(ggml_backend_t backend, const ggml_tensor * src0, const
|
||||
switch (src0t) {
|
||||
case GGML_TYPE_F32:
|
||||
switch (src1t) {
|
||||
case GGML_TYPE_I32:
|
||||
kernel = backend_ctx->kernel_cpy_f32_i32;
|
||||
break;
|
||||
case GGML_TYPE_F16:
|
||||
kernel = backend_ctx->kernel_cpy_f32_f16;
|
||||
break;
|
||||
@@ -28752,6 +28765,9 @@ static void ggml_cl_cpy(ggml_backend_t backend, const ggml_tensor * src0, const
|
||||
break;
|
||||
case GGML_TYPE_I32:
|
||||
switch (src1t) {
|
||||
case GGML_TYPE_F32:
|
||||
kernel = backend_ctx->kernel_cpy_i32_f32;
|
||||
break;
|
||||
case GGML_TYPE_I32:
|
||||
kernel = backend_ctx->kernel_cpy_i32_i32;
|
||||
break;
|
||||
|
||||
@@ -313,3 +313,88 @@ kernel void kernel_cpy_f32_f32_flat(
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
kernel void kernel_cpy_i32_f32(
|
||||
global int * src0,
|
||||
ulong offset0,
|
||||
global float * dst,
|
||||
ulong offsetd,
|
||||
int ne00,
|
||||
int ne01,
|
||||
int ne02,
|
||||
int ne03,
|
||||
ulong nb00,
|
||||
ulong nb01,
|
||||
ulong nb02,
|
||||
ulong nb03,
|
||||
int ne0,
|
||||
int ne1,
|
||||
int ne2,
|
||||
int ne3,
|
||||
ulong nb0,
|
||||
ulong nb1,
|
||||
ulong nb2,
|
||||
ulong nb3
|
||||
) {
|
||||
src0 = (global int *) ((global char *) src0 + offset0);
|
||||
dst = (global float *) ((global char *) dst + offsetd);
|
||||
|
||||
const ulong i01 = get_group_id(0);
|
||||
const ulong i02 = get_group_id(1);
|
||||
const ulong i03 = get_group_id(2);
|
||||
|
||||
for (ulong i00 = get_local_id(0); i00 < ne00; i00 += get_local_size(0)) {
|
||||
const ulong n = ((i03 * ne02 + i02) * ne01 + i01) * ne00 + i00;
|
||||
const ulong i0 = n % ne0;
|
||||
const ulong i1 = n / ne0 % ne1;
|
||||
const ulong i2 = n / ne0 / ne1 % ne2;
|
||||
const ulong i3 = n / ne0 / ne1 / ne2;
|
||||
|
||||
global const int * src = (global const int *) ((global char *) src0 + i00*nb00 + i01*nb01 + i02*nb02 + i03*nb03);
|
||||
global float * out = (global float *) ((global char *) dst + i0*nb0 + i1*nb1 + i2*nb2 + i3*nb3);
|
||||
out[0] = convert_float(src[0]);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
kernel void kernel_cpy_f32_i32(
|
||||
global float * src0,
|
||||
ulong offset0,
|
||||
global int * dst,
|
||||
ulong offsetd,
|
||||
int ne00,
|
||||
int ne01,
|
||||
int ne02,
|
||||
int ne03,
|
||||
ulong nb00,
|
||||
ulong nb01,
|
||||
ulong nb02,
|
||||
ulong nb03,
|
||||
int ne0,
|
||||
int ne1,
|
||||
int ne2,
|
||||
int ne3,
|
||||
ulong nb0,
|
||||
ulong nb1,
|
||||
ulong nb2,
|
||||
ulong nb3
|
||||
) {
|
||||
src0 = (global float *) ((global char *) src0 + offset0);
|
||||
dst = (global int *) ((global char *) dst + offsetd);
|
||||
|
||||
const ulong i01 = get_group_id(0);
|
||||
const ulong i02 = get_group_id(1);
|
||||
const ulong i03 = get_group_id(2);
|
||||
|
||||
for (ulong i00 = get_local_id(0); i00 < ne00; i00 += get_local_size(0)) {
|
||||
const ulong n = ((i03 * ne02 + i02) * ne01 + i01) * ne00 + i00;
|
||||
const ulong i0 = n % ne0;
|
||||
const ulong i1 = n / ne0 % ne1;
|
||||
const ulong i2 = n / ne0 / ne1 % ne2;
|
||||
const ulong i3 = n / ne0 / ne1 / ne2;
|
||||
|
||||
global const float * src = (global const float *) ((global char *) src0 + i00*nb00 + i01*nb01 + i02*nb02 + i03*nb03);
|
||||
global int * out = (global int *) ((global char *) dst + i0*nb0 + i1*nb1 + i2*nb2 + i3*nb3);
|
||||
out[0] = convert_int_rtz(src[0]);
|
||||
}
|
||||
}
|
||||
@@ -106,6 +106,53 @@ kernel void kernel_get_rows_f32(
|
||||
}
|
||||
}
|
||||
|
||||
kernel void kernel_get_rows_i32(
|
||||
global void * src0,
|
||||
ulong offset0,
|
||||
global int * src1,
|
||||
ulong offset1,
|
||||
global int * dst,
|
||||
ulong offsetd,
|
||||
int ne00,
|
||||
ulong nb01,
|
||||
ulong nb02,
|
||||
ulong nb03,
|
||||
int ne10,
|
||||
ulong nb10,
|
||||
ulong nb11,
|
||||
ulong nb12,
|
||||
ulong nb1,
|
||||
ulong nb2,
|
||||
ulong nb3
|
||||
) {
|
||||
src0 = (global void*)((global char*)src0 + offset0);
|
||||
src1 = (global int*)((global char*)src1 + offset1);
|
||||
dst = (global int*)((global char*)dst + offsetd);
|
||||
|
||||
int nchunks = get_num_groups(0) / ne10;
|
||||
int g = get_group_id(0);
|
||||
int i10 = g / nchunks;
|
||||
int chunk = g - i10 * nchunks;
|
||||
int i11 = get_group_id(1);
|
||||
int i12 = get_group_id(2);
|
||||
|
||||
int r = ((global int *) ((global char *) src1 + i12*nb12 + i11*nb11 + i10*nb10))[0];
|
||||
|
||||
int i02 = i11;
|
||||
int i03 = i12;
|
||||
|
||||
global int * dst_row = (global int *) ((global char *) dst + i12*nb3 + i11*nb2 + i10*nb1);
|
||||
global int * src_row = (global int *) ((global char *) src0 + r*nb01 + i02*nb02 + i03*nb03);
|
||||
|
||||
int span = (ne00 + nchunks - 1) / nchunks;
|
||||
int start = chunk * span;
|
||||
int end = min(start + span, ne00);
|
||||
|
||||
for (int ind = start + get_local_id(0); ind < end; ind += get_local_size(0)) {
|
||||
dst_row[ind] = src_row[ind];
|
||||
}
|
||||
}
|
||||
|
||||
kernel void kernel_get_rows_f16(
|
||||
global void * src0,
|
||||
ulong offset0,
|
||||
|
||||
Reference in new issue
Block a user