opencl: add get_rows for i32, cpy for i32 and f32 (#30756)

* opencl: support i32 GET_ROWS

* opencl: support i32 and f32 copy conversions
This commit is contained in:
lhez authored and GitHub committed 2026-10-19 12:22:34 -07:00
1 parent 0ce8cd1141
commit 6fa4f6cd3e
3 files changed
+151 -3

No files matched your search

+19 -3
View File
@@ -868,7 +868,7 @@ struct ggml_backend_opencl_context {
#ifdef GGML_OPENCL_USE_ADRENO_KERNELS
ggml_cl_adreno_xmem_attn_state adreno_xmem_attn;
#endif
cl_kernel kernel_get_rows_f32, kernel_get_rows_f16, kernel_get_rows_q4_0;
cl_kernel kernel_get_rows_f32, kernel_get_rows_f16, kernel_get_rows_i32, kernel_get_rows_q4_0;
cl_kernel kernel_set_rows_f32_i64, kernel_set_rows_f32_i32, kernel_set_rows_f16_i64, kernel_set_rows_f16_i32;
cl_kernel kernel_set_rows_q8_0_i64, kernel_set_rows_q8_0_i32;
cl_kernel kernel_set_rows_q8_0_soa_i64, kernel_set_rows_q8_0_soa_i32;
@@ -876,7 +876,7 @@ struct ggml_backend_opencl_context {
cl_kernel kernel_set_rows_q4_0_soa_i64, kernel_set_rows_q4_0_soa_i32;
cl_kernel kernel_rope_norm_f32, kernel_rope_norm_f16, kernel_rope_neox_f32, kernel_rope_neox_f16;
cl_kernel kernel_rope_multi_f32, kernel_rope_multi_f16, kernel_rope_vision_f32, kernel_rope_vision_f16;
cl_kernel kernel_cpy_f16_f16, kernel_cpy_f16_f32, kernel_cpy_f32_f16, kernel_cpy_f32_f32, kernel_cpy_i32_i32;
cl_kernel kernel_cpy_f16_f16, kernel_cpy_f16_f32, kernel_cpy_f32_f16, kernel_cpy_f32_f32, kernel_cpy_i32_i32, kernel_cpy_i32_f32, kernel_cpy_f32_i32;
cl_kernel kernel_cpy_f32_f32_pack = nullptr;
cl_kernel kernel_cpy_f32_f32_flat = nullptr;
cl_kernel kernel_mul_mat_f32_f32;
@@ -1652,6 +1652,8 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) {
}
}
CL_CHECK((backend_ctx->kernel_cpy_i32_i32 = clCreateKernel(prog, "kernel_cpy_i32_i32", &err), err));
CL_CHECK((backend_ctx->kernel_cpy_i32_f32 = clCreateKernel(prog, "kernel_cpy_i32_f32", &err), err));
CL_CHECK((backend_ctx->kernel_cpy_f32_i32 = clCreateKernel(prog, "kernel_cpy_f32_i32", &err), err));
GGML_LOG_CONT(".");
}
@@ -1840,6 +1842,7 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) {
build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts);
CL_CHECK((backend_ctx->kernel_get_rows_f32 = clCreateKernel(backend_ctx->program_get_rows, "kernel_get_rows_f32", &err), err));
CL_CHECK((backend_ctx->kernel_get_rows_i32 = clCreateKernel(backend_ctx->program_get_rows, "kernel_get_rows_i32", &err), err));
CL_CHECK((backend_ctx->kernel_get_rows_f16 = clCreateKernel(backend_ctx->program_get_rows, "kernel_get_rows_f16", &err), err));
CL_CHECK((backend_ctx->kernel_get_rows_q4_0 = clCreateKernel(backend_ctx->program_get_rows, "kernel_get_rows_q4_0", &err), err));
GGML_LOG_CONT(".");
@@ -9102,6 +9105,8 @@ static bool ggml_opencl_supports_op(ggml_backend_dev_t dev, const struct ggml_te
return true;
case GGML_OP_GET_ROWS:
switch (op->src[0]->type) {
case GGML_TYPE_I32:
return op->type == GGML_TYPE_I32;
case GGML_TYPE_F32:
case GGML_TYPE_F16:
return true;
@@ -9139,6 +9144,7 @@ static bool ggml_opencl_supports_op(ggml_backend_dev_t dev, const struct ggml_te
switch (op->src[0]->type) {
case GGML_TYPE_F32:
switch (op->type) {
case GGML_TYPE_I32:
case GGML_TYPE_F16:
case GGML_TYPE_F32:
return true;
@@ -9155,6 +9161,7 @@ static bool ggml_opencl_supports_op(ggml_backend_dev_t dev, const struct ggml_te
}
case GGML_TYPE_I32:
switch (op->type) {
case GGML_TYPE_F32:
case GGML_TYPE_I32:
return true;
default:
@@ -13619,6 +13626,9 @@ static void ggml_cl_get_rows(ggml_backend_t backend, const ggml_tensor * src0, c
case GGML_TYPE_F32:
kernel = backend_ctx->kernel_get_rows_f32;
break;
case GGML_TYPE_I32:
kernel = backend_ctx->kernel_get_rows_i32;
break;
case GGML_TYPE_F16:
kernel = backend_ctx->kernel_get_rows_f16;
break;
@@ -13654,7 +13664,7 @@ static void ggml_cl_get_rows(ggml_backend_t backend, const ggml_tensor * src0, c
}
int nchunks = 1;
if (src0->type == GGML_TYPE_F32) {
if (src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_I32) {
const int chunk_target = nth * 4;
nchunks = (ne00 + chunk_target - 1) / chunk_target;
nchunks = MAX(1, MIN(nchunks, 64));
@@ -28726,6 +28736,9 @@ static void ggml_cl_cpy(ggml_backend_t backend, const ggml_tensor * src0, const
switch (src0t) {
case GGML_TYPE_F32:
switch (src1t) {
case GGML_TYPE_I32:
kernel = backend_ctx->kernel_cpy_f32_i32;
break;
case GGML_TYPE_F16:
kernel = backend_ctx->kernel_cpy_f32_f16;
break;
@@ -28752,6 +28765,9 @@ static void ggml_cl_cpy(ggml_backend_t backend, const ggml_tensor * src0, const
break;
case GGML_TYPE_I32:
switch (src1t) {
case GGML_TYPE_F32:
kernel = backend_ctx->kernel_cpy_i32_f32;
break;
case GGML_TYPE_I32:
kernel = backend_ctx->kernel_cpy_i32_i32;
break;
+85
View File
@@ -313,3 +313,88 @@ kernel void kernel_cpy_f32_f32_flat(
}
}
}
kernel void kernel_cpy_i32_f32(
global int * src0,
ulong offset0,
global float * dst,
ulong offsetd,
int ne00,
int ne01,
int ne02,
int ne03,
ulong nb00,
ulong nb01,
ulong nb02,
ulong nb03,
int ne0,
int ne1,
int ne2,
int ne3,
ulong nb0,
ulong nb1,
ulong nb2,
ulong nb3
) {
src0 = (global int *) ((global char *) src0 + offset0);
dst = (global float *) ((global char *) dst + offsetd);
const ulong i01 = get_group_id(0);
const ulong i02 = get_group_id(1);
const ulong i03 = get_group_id(2);
for (ulong i00 = get_local_id(0); i00 < ne00; i00 += get_local_size(0)) {
const ulong n = ((i03 * ne02 + i02) * ne01 + i01) * ne00 + i00;
const ulong i0 = n % ne0;
const ulong i1 = n / ne0 % ne1;
const ulong i2 = n / ne0 / ne1 % ne2;
const ulong i3 = n / ne0 / ne1 / ne2;
global const int * src = (global const int *) ((global char *) src0 + i00*nb00 + i01*nb01 + i02*nb02 + i03*nb03);
global float * out = (global float *) ((global char *) dst + i0*nb0 + i1*nb1 + i2*nb2 + i3*nb3);
out[0] = convert_float(src[0]);
}
}
kernel void kernel_cpy_f32_i32(
global float * src0,
ulong offset0,
global int * dst,
ulong offsetd,
int ne00,
int ne01,
int ne02,
int ne03,
ulong nb00,
ulong nb01,
ulong nb02,
ulong nb03,
int ne0,
int ne1,
int ne2,
int ne3,
ulong nb0,
ulong nb1,
ulong nb2,
ulong nb3
) {
src0 = (global float *) ((global char *) src0 + offset0);
dst = (global int *) ((global char *) dst + offsetd);
const ulong i01 = get_group_id(0);
const ulong i02 = get_group_id(1);
const ulong i03 = get_group_id(2);
for (ulong i00 = get_local_id(0); i00 < ne00; i00 += get_local_size(0)) {
const ulong n = ((i03 * ne02 + i02) * ne01 + i01) * ne00 + i00;
const ulong i0 = n % ne0;
const ulong i1 = n / ne0 % ne1;
const ulong i2 = n / ne0 / ne1 % ne2;
const ulong i3 = n / ne0 / ne1 / ne2;
global const float * src = (global const float *) ((global char *) src0 + i00*nb00 + i01*nb01 + i02*nb02 + i03*nb03);
global int * out = (global int *) ((global char *) dst + i0*nb0 + i1*nb1 + i2*nb2 + i3*nb3);
out[0] = convert_int_rtz(src[0]);
}
}
+47
View File
@@ -106,6 +106,53 @@ kernel void kernel_get_rows_f32(
}
}
kernel void kernel_get_rows_i32(
global void * src0,
ulong offset0,
global int * src1,
ulong offset1,
global int * dst,
ulong offsetd,
int ne00,
ulong nb01,
ulong nb02,
ulong nb03,
int ne10,
ulong nb10,
ulong nb11,
ulong nb12,
ulong nb1,
ulong nb2,
ulong nb3
) {
src0 = (global void*)((global char*)src0 + offset0);
src1 = (global int*)((global char*)src1 + offset1);
dst = (global int*)((global char*)dst + offsetd);
int nchunks = get_num_groups(0) / ne10;
int g = get_group_id(0);
int i10 = g / nchunks;
int chunk = g - i10 * nchunks;
int i11 = get_group_id(1);
int i12 = get_group_id(2);
int r = ((global int *) ((global char *) src1 + i12*nb12 + i11*nb11 + i10*nb10))[0];
int i02 = i11;
int i03 = i12;
global int * dst_row = (global int *) ((global char *) dst + i12*nb3 + i11*nb2 + i10*nb1);
global int * src_row = (global int *) ((global char *) src0 + r*nb01 + i02*nb02 + i03*nb03);
int span = (ne00 + nchunks - 1) / nchunks;
int start = chunk * span;
int end = min(start + span, ne00);
for (int ind = start + get_local_id(0); ind < end; ind += get_local_size(0)) {
dst_row[ind] = src_row[ind];
}
}
kernel void kernel_get_rows_f16(
global void * src0,
ulong offset0,