From 6fa4f6cd3ef4b65188acb251d5ef37f8300719a3 Mon Sep 17 00:00:00 2001 From: lhez Date: Mon, 19 Oct 2026 12:22:34 -0700 Subject: [PATCH] opencl: add get_rows for i32, cpy for i32 and f32 (#30756) * opencl: support i32 GET_ROWS * opencl: support i32 and f32 copy conversions --- ggml/src/ggml-opencl/ggml-opencl.cpp | 22 +++++- ggml/src/ggml-opencl/kernels/cpy.cl | 85 ++++++++++++++++++++++++ ggml/src/ggml-opencl/kernels/get_rows.cl | 47 +++++++++++++ 3 files changed, 151 insertions(+), 3 deletions(-) diff --git a/ggml/src/ggml-opencl/ggml-opencl.cpp b/ggml/src/ggml-opencl/ggml-opencl.cpp index de93b0134..96461bcd9 100644 --- a/ggml/src/ggml-opencl/ggml-opencl.cpp +++ b/ggml/src/ggml-opencl/ggml-opencl.cpp @@ -868,7 +868,7 @@ struct ggml_backend_opencl_context { #ifdef GGML_OPENCL_USE_ADRENO_KERNELS ggml_cl_adreno_xmem_attn_state adreno_xmem_attn; #endif - cl_kernel kernel_get_rows_f32, kernel_get_rows_f16, kernel_get_rows_q4_0; + cl_kernel kernel_get_rows_f32, kernel_get_rows_f16, kernel_get_rows_i32, kernel_get_rows_q4_0; cl_kernel kernel_set_rows_f32_i64, kernel_set_rows_f32_i32, kernel_set_rows_f16_i64, kernel_set_rows_f16_i32; cl_kernel kernel_set_rows_q8_0_i64, kernel_set_rows_q8_0_i32; cl_kernel kernel_set_rows_q8_0_soa_i64, kernel_set_rows_q8_0_soa_i32; @@ -876,7 +876,7 @@ struct ggml_backend_opencl_context { cl_kernel kernel_set_rows_q4_0_soa_i64, kernel_set_rows_q4_0_soa_i32; cl_kernel kernel_rope_norm_f32, kernel_rope_norm_f16, kernel_rope_neox_f32, kernel_rope_neox_f16; cl_kernel kernel_rope_multi_f32, kernel_rope_multi_f16, kernel_rope_vision_f32, kernel_rope_vision_f16; - cl_kernel kernel_cpy_f16_f16, kernel_cpy_f16_f32, kernel_cpy_f32_f16, kernel_cpy_f32_f32, kernel_cpy_i32_i32; + cl_kernel kernel_cpy_f16_f16, kernel_cpy_f16_f32, kernel_cpy_f32_f16, kernel_cpy_f32_f32, kernel_cpy_i32_i32, kernel_cpy_i32_f32, kernel_cpy_f32_i32; cl_kernel kernel_cpy_f32_f32_pack = nullptr; cl_kernel kernel_cpy_f32_f32_flat = nullptr; cl_kernel kernel_mul_mat_f32_f32; @@ -1652,6 +1652,8 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) { } } CL_CHECK((backend_ctx->kernel_cpy_i32_i32 = clCreateKernel(prog, "kernel_cpy_i32_i32", &err), err)); + CL_CHECK((backend_ctx->kernel_cpy_i32_f32 = clCreateKernel(prog, "kernel_cpy_i32_f32", &err), err)); + CL_CHECK((backend_ctx->kernel_cpy_f32_i32 = clCreateKernel(prog, "kernel_cpy_f32_i32", &err), err)); GGML_LOG_CONT("."); } @@ -1840,6 +1842,7 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) { build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); CL_CHECK((backend_ctx->kernel_get_rows_f32 = clCreateKernel(backend_ctx->program_get_rows, "kernel_get_rows_f32", &err), err)); + CL_CHECK((backend_ctx->kernel_get_rows_i32 = clCreateKernel(backend_ctx->program_get_rows, "kernel_get_rows_i32", &err), err)); CL_CHECK((backend_ctx->kernel_get_rows_f16 = clCreateKernel(backend_ctx->program_get_rows, "kernel_get_rows_f16", &err), err)); CL_CHECK((backend_ctx->kernel_get_rows_q4_0 = clCreateKernel(backend_ctx->program_get_rows, "kernel_get_rows_q4_0", &err), err)); GGML_LOG_CONT("."); @@ -9102,6 +9105,8 @@ static bool ggml_opencl_supports_op(ggml_backend_dev_t dev, const struct ggml_te return true; case GGML_OP_GET_ROWS: switch (op->src[0]->type) { + case GGML_TYPE_I32: + return op->type == GGML_TYPE_I32; case GGML_TYPE_F32: case GGML_TYPE_F16: return true; @@ -9139,6 +9144,7 @@ static bool ggml_opencl_supports_op(ggml_backend_dev_t dev, const struct ggml_te switch (op->src[0]->type) { case GGML_TYPE_F32: switch (op->type) { + case GGML_TYPE_I32: case GGML_TYPE_F16: case GGML_TYPE_F32: return true; @@ -9155,6 +9161,7 @@ static bool ggml_opencl_supports_op(ggml_backend_dev_t dev, const struct ggml_te } case GGML_TYPE_I32: switch (op->type) { + case GGML_TYPE_F32: case GGML_TYPE_I32: return true; default: @@ -13619,6 +13626,9 @@ static void ggml_cl_get_rows(ggml_backend_t backend, const ggml_tensor * src0, c case GGML_TYPE_F32: kernel = backend_ctx->kernel_get_rows_f32; break; + case GGML_TYPE_I32: + kernel = backend_ctx->kernel_get_rows_i32; + break; case GGML_TYPE_F16: kernel = backend_ctx->kernel_get_rows_f16; break; @@ -13654,7 +13664,7 @@ static void ggml_cl_get_rows(ggml_backend_t backend, const ggml_tensor * src0, c } int nchunks = 1; - if (src0->type == GGML_TYPE_F32) { + if (src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_I32) { const int chunk_target = nth * 4; nchunks = (ne00 + chunk_target - 1) / chunk_target; nchunks = MAX(1, MIN(nchunks, 64)); @@ -28726,6 +28736,9 @@ static void ggml_cl_cpy(ggml_backend_t backend, const ggml_tensor * src0, const switch (src0t) { case GGML_TYPE_F32: switch (src1t) { + case GGML_TYPE_I32: + kernel = backend_ctx->kernel_cpy_f32_i32; + break; case GGML_TYPE_F16: kernel = backend_ctx->kernel_cpy_f32_f16; break; @@ -28752,6 +28765,9 @@ static void ggml_cl_cpy(ggml_backend_t backend, const ggml_tensor * src0, const break; case GGML_TYPE_I32: switch (src1t) { + case GGML_TYPE_F32: + kernel = backend_ctx->kernel_cpy_i32_f32; + break; case GGML_TYPE_I32: kernel = backend_ctx->kernel_cpy_i32_i32; break; diff --git a/ggml/src/ggml-opencl/kernels/cpy.cl b/ggml/src/ggml-opencl/kernels/cpy.cl index 11d7a57c1..18400d37f 100644 --- a/ggml/src/ggml-opencl/kernels/cpy.cl +++ b/ggml/src/ggml-opencl/kernels/cpy.cl @@ -313,3 +313,88 @@ kernel void kernel_cpy_f32_f32_flat( } } } + +kernel void kernel_cpy_i32_f32( + global int * src0, + ulong offset0, + global float * dst, + ulong offsetd, + int ne00, + int ne01, + int ne02, + int ne03, + ulong nb00, + ulong nb01, + ulong nb02, + ulong nb03, + int ne0, + int ne1, + int ne2, + int ne3, + ulong nb0, + ulong nb1, + ulong nb2, + ulong nb3 +) { + src0 = (global int *) ((global char *) src0 + offset0); + dst = (global float *) ((global char *) dst + offsetd); + + const ulong i01 = get_group_id(0); + const ulong i02 = get_group_id(1); + const ulong i03 = get_group_id(2); + + for (ulong i00 = get_local_id(0); i00 < ne00; i00 += get_local_size(0)) { + const ulong n = ((i03 * ne02 + i02) * ne01 + i01) * ne00 + i00; + const ulong i0 = n % ne0; + const ulong i1 = n / ne0 % ne1; + const ulong i2 = n / ne0 / ne1 % ne2; + const ulong i3 = n / ne0 / ne1 / ne2; + + global const int * src = (global const int *) ((global char *) src0 + i00*nb00 + i01*nb01 + i02*nb02 + i03*nb03); + global float * out = (global float *) ((global char *) dst + i0*nb0 + i1*nb1 + i2*nb2 + i3*nb3); + out[0] = convert_float(src[0]); + } +} + + +kernel void kernel_cpy_f32_i32( + global float * src0, + ulong offset0, + global int * dst, + ulong offsetd, + int ne00, + int ne01, + int ne02, + int ne03, + ulong nb00, + ulong nb01, + ulong nb02, + ulong nb03, + int ne0, + int ne1, + int ne2, + int ne3, + ulong nb0, + ulong nb1, + ulong nb2, + ulong nb3 +) { + src0 = (global float *) ((global char *) src0 + offset0); + dst = (global int *) ((global char *) dst + offsetd); + + const ulong i01 = get_group_id(0); + const ulong i02 = get_group_id(1); + const ulong i03 = get_group_id(2); + + for (ulong i00 = get_local_id(0); i00 < ne00; i00 += get_local_size(0)) { + const ulong n = ((i03 * ne02 + i02) * ne01 + i01) * ne00 + i00; + const ulong i0 = n % ne0; + const ulong i1 = n / ne0 % ne1; + const ulong i2 = n / ne0 / ne1 % ne2; + const ulong i3 = n / ne0 / ne1 / ne2; + + global const float * src = (global const float *) ((global char *) src0 + i00*nb00 + i01*nb01 + i02*nb02 + i03*nb03); + global int * out = (global int *) ((global char *) dst + i0*nb0 + i1*nb1 + i2*nb2 + i3*nb3); + out[0] = convert_int_rtz(src[0]); + } +} diff --git a/ggml/src/ggml-opencl/kernels/get_rows.cl b/ggml/src/ggml-opencl/kernels/get_rows.cl index 9ae4fff09..f060e46ae 100644 --- a/ggml/src/ggml-opencl/kernels/get_rows.cl +++ b/ggml/src/ggml-opencl/kernels/get_rows.cl @@ -106,6 +106,53 @@ kernel void kernel_get_rows_f32( } } +kernel void kernel_get_rows_i32( + global void * src0, + ulong offset0, + global int * src1, + ulong offset1, + global int * dst, + ulong offsetd, + int ne00, + ulong nb01, + ulong nb02, + ulong nb03, + int ne10, + ulong nb10, + ulong nb11, + ulong nb12, + ulong nb1, + ulong nb2, + ulong nb3 +) { + src0 = (global void*)((global char*)src0 + offset0); + src1 = (global int*)((global char*)src1 + offset1); + dst = (global int*)((global char*)dst + offsetd); + + int nchunks = get_num_groups(0) / ne10; + int g = get_group_id(0); + int i10 = g / nchunks; + int chunk = g - i10 * nchunks; + int i11 = get_group_id(1); + int i12 = get_group_id(2); + + int r = ((global int *) ((global char *) src1 + i12*nb12 + i11*nb11 + i10*nb10))[0]; + + int i02 = i11; + int i03 = i12; + + global int * dst_row = (global int *) ((global char *) dst + i12*nb3 + i11*nb2 + i10*nb1); + global int * src_row = (global int *) ((global char *) src0 + r*nb01 + i02*nb02 + i03*nb03); + + int span = (ne00 + nchunks - 1) / nchunks; + int start = chunk * span; + int end = min(start + span, ne00); + + for (int ind = start + get_local_id(0); ind < end; ind += get_local_size(0)) { + dst_row[ind] = src_row[ind]; + } +} + kernel void kernel_get_rows_f16( global void * src0, ulong offset0,