diff --git a/ggml/src/ggml-opencl/CMakeLists.txt b/ggml/src/ggml-opencl/CMakeLists.txt index 57620e560..9b977fae7 100644 --- a/ggml/src/ggml-opencl/CMakeLists.txt +++ b/ggml/src/ggml-opencl/CMakeLists.txt @@ -104,7 +104,13 @@ set(GGML_OPENCL_KERNELS gemm_noshuffle_q2_k_q8_1_dp4a gemv_noshuffle_q2_k_f32 gemm_noshuffle_q2_k_f32 + gemm_noshuffle_q3_k_f32 + gemm_noshuffle_q3_k_q8_1_dp4a + gemv_noshuffle_q3_k_f32 mul_mv_q2_k_f32 + mul_mv_q3_k_f32 + mul_mv_q3_k_f32_flat + mul_mm_q3_k_f32_l4_lm mul_mv_q2_k_f32_flat mul_mm_q2_k_f32_l4_lm mul_mv_q4_k_f32 diff --git a/ggml/src/ggml-opencl/ggml-opencl.cpp b/ggml/src/ggml-opencl/ggml-opencl.cpp index 176a6e559..cb4cfdd75 100644 --- a/ggml/src/ggml-opencl/ggml-opencl.cpp +++ b/ggml/src/ggml-opencl/ggml-opencl.cpp @@ -975,6 +975,17 @@ struct ggml_backend_opencl_context { int q2_k_mv_rows = 4; int q2_k_mv_nsg = 4; cl_kernel kernel_mul_mv_q2_K_f32; + cl_kernel kernel_convert_block_q3_K, kernel_restore_block_q3_K; + cl_kernel kernel_mul_mv_q3_K_f32_flat; + cl_kernel kernel_mul_mm_q3_k_f32_l4_lm; + cl_kernel kernel_convert_block_q3_k_noshuffle, kernel_restore_block_q3_k_noshuffle; + cl_kernel kernel_gemv_noshuffle_q3_k_f32 = nullptr; + cl_kernel kernel_gemm_noshuffle_q3_k_f32 = nullptr; + cl_kernel kernel_gemm_noshuffle_q3_k_q8_1_dp4a = nullptr; + cl_kernel kernel_gemm_noshuffle_q3_k_q8_1_dp4a_narrow = nullptr; + int q3_k_mv_rows = 2; + int q3_k_mv_nsg = 8; + cl_kernel kernel_mul_mv_q3_K_f32; cl_kernel kernel_mul_mv_q4_K_f32; cl_kernel kernel_mul_mv_q4_K_f32_flat; cl_kernel kernel_mul_mv_q5_K_f32; @@ -1542,6 +1553,23 @@ static std::string ggml_cl_q2_k_mm_options(const ggml_backend_opencl_context * b return " -DBK=" + std::to_string(bk) + " -DLM_HALF=" + std::to_string(half ? 1 : 0); } +static std::string ggml_cl_q3_k_mm_options(const ggml_backend_opencl_context * backend_ctx) { + const bool x2 = backend_ctx->gpu_family == ADRENO && + (backend_ctx->adreno_gen == ADRENO_GPU_GEN::A8X || backend_ctx->adreno_gen == ADRENO_GPU_GEN::X2E); + int bk = x2 ? 16 : 32; + bool half = x2; + if (const char * env = getenv("GGML_OPENCL_LM_BK")) { + const int value = atoi(env); + if (value == 16 || value == 32) { + bk = value; + } + } + if (const char * env = getenv("GGML_OPENCL_LM_HALF")) { + half = atoi(env) != 0; + } + return " -DBK=" + std::to_string(bk) + " -DLM_HALF=" + std::to_string(half ? 1 : 0); +} + static void load_cl_kernels_argsort(ggml_backend_opencl_context *backend_ctx) { // compiler options for general kernels auto opencl_c_std = @@ -1992,6 +2020,10 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) { CL_CHECK((backend_ctx->kernel_restore_block_q5_K_noshuffle = clCreateKernel(backend_ctx->program_cvt, "kernel_restore_block_q5_K_noshuffle", &err), err)); CL_CHECK((backend_ctx->kernel_convert_block_q2_k_noshuffle = clCreateKernel(backend_ctx->program_cvt, "kernel_convert_block_q2_k_noshuffle", &err), err)); CL_CHECK((backend_ctx->kernel_restore_block_q2_k_noshuffle = clCreateKernel(backend_ctx->program_cvt, "kernel_restore_block_q2_k_noshuffle", &err), err)); + CL_CHECK((backend_ctx->kernel_convert_block_q3_k_noshuffle = clCreateKernel(backend_ctx->program_cvt, "kernel_convert_block_q3_k_noshuffle", &err), err)); + CL_CHECK((backend_ctx->kernel_restore_block_q3_k_noshuffle = clCreateKernel(backend_ctx->program_cvt, "kernel_restore_block_q3_k_noshuffle", &err), err)); + CL_CHECK((backend_ctx->kernel_convert_block_q3_K = clCreateKernel(backend_ctx->program_cvt, "kernel_convert_block_q3_K", &err), err)); + CL_CHECK((backend_ctx->kernel_restore_block_q3_K = clCreateKernel(backend_ctx->program_cvt, "kernel_restore_block_q3_K", &err), err)); CL_CHECK((backend_ctx->kernel_convert_block_q2_K = clCreateKernel(backend_ctx->program_cvt, "kernel_convert_block_q2_K", &err), err)); CL_CHECK((backend_ctx->kernel_restore_block_q2_K = clCreateKernel(backend_ctx->program_cvt, "kernel_restore_block_q2_K", &err), err)); CL_CHECK((backend_ctx->kernel_convert_block_q6_K = clCreateKernel(backend_ctx->program_cvt, "kernel_convert_block_q6_K", &err), err)); @@ -2398,6 +2430,53 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) { GGML_LOG_CONT("."); } + // mul_mv_q3_k_f32_flat + { +#ifdef GGML_OPENCL_EMBED_KERNELS + const std::string kernel_src { + #include "mul_mv_q3_k_f32_flat.cl.h" + }; +#else + const std::string kernel_src = read_file("mul_mv_q3_k_f32_flat.cl"); +#endif + cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); + CL_CHECK((backend_ctx->kernel_mul_mv_q3_K_f32_flat = clCreateKernel(prog, "kernel_mul_mv_q3_K_f32_flat", &err), err)); + CL_CHECK(clReleaseProgram(prog)); + GGML_LOG_CONT("."); + } + + // mul_mm_q3_k_f32_l4_lm + { +#ifdef GGML_OPENCL_EMBED_KERNELS + const std::string kernel_src { + #include "mul_mm_q3_k_f32_l4_lm.cl.h" + }; +#else + const std::string kernel_src = read_file("mul_mm_q3_k_f32_l4_lm.cl"); +#endif + cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts + ggml_cl_q3_k_mm_options(backend_ctx)); + CL_CHECK((backend_ctx->kernel_mul_mm_q3_k_f32_l4_lm = clCreateKernel(prog, "kernel_mul_mm_q3_k_f32_l4_lm", &err), err)); + CL_CHECK(clReleaseProgram(prog)); + GGML_LOG_CONT("."); + } + + // mul_mv_q3_k_f32 + { +#ifdef GGML_OPENCL_EMBED_KERNELS + const std::string kernel_src { + #include "mul_mv_q3_k_f32.cl.h" + }; +#else + const std::string kernel_src = read_file("mul_mv_q3_k_f32.cl"); +#endif + cl_program prog = + build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); + + CL_CHECK((backend_ctx->kernel_mul_mv_q3_K_f32 = clCreateKernel(prog, "kernel_mul_mv_q3_K_f32", &err), err)); + CL_CHECK(clReleaseProgram(prog)); + GGML_LOG_CONT("."); + } + // mul_mv_q4_k_f32 { #ifdef GGML_OPENCL_EMBED_KERNELS @@ -4206,6 +4285,80 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) { GGML_LOG_CONT("."); } + // gemm_noshuffle_q3_k_q8_1_dp4a + { +#ifdef GGML_OPENCL_EMBED_KERNELS + const std::string kernel_src { + #include "gemm_noshuffle_q3_k_q8_1_dp4a.cl.h" + }; +#else + const std::string kernel_src = read_file("gemm_noshuffle_q3_k_q8_1_dp4a.cl"); +#endif + cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); + CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q3_k_q8_1_dp4a = clCreateKernel(prog, "kernel_gemm_noshuffle_q3_k_q8_1_dp4a", &err), err)); + CL_CHECK(clReleaseProgram(prog)); + prog = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts + " -DTILESIZE_N=8"); + CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q3_k_q8_1_dp4a_narrow = clCreateKernel(prog, "kernel_gemm_noshuffle_q3_k_q8_1_dp4a", &err), err)); + CL_CHECK(clReleaseProgram(prog)); + GGML_LOG_CONT("."); + } + + // gemm_noshuffle_q3_k_f32 + { +#ifdef GGML_OPENCL_EMBED_KERNELS + const std::string kernel_src { + #include "gemm_noshuffle_q3_k_f32.cl.h" + }; +#else + const std::string kernel_src = read_file("gemm_noshuffle_q3_k_f32.cl"); +#endif + cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); + CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q3_k_f32 = clCreateKernel(prog, "kernel_gemm_noshuffle_q3_k_f32", &err), err)); + CL_CHECK(clReleaseProgram(prog)); + GGML_LOG_CONT("."); + } + + // gemv_noshuffle_q3_k_f32 + { +#ifdef GGML_OPENCL_EMBED_KERNELS + const std::string kernel_src { + #include "gemv_noshuffle_q3_k_f32.cl.h" + }; +#else + const std::string kernel_src = read_file("gemv_noshuffle_q3_k_f32.cl"); +#endif + if (const char * env = getenv("GGML_OPENCL_Q3_K_MV_R")) { + const int rows = atoi(env); + if (rows == 1 || rows == 2 || rows == 4) { + backend_ctx->q3_k_mv_rows = rows; + } + } + if (const char * env = getenv("GGML_OPENCL_Q3_K_MV_NSG")) { + const int nsg = atoi(env); + if (nsg == 1 || nsg == 2 || nsg == 4 || nsg == 8) { + backend_ctx->q3_k_mv_nsg = nsg; + } + } + for (;;) { + const std::string opts = compile_opts + " -DQ3_K_MV_R=" + std::to_string(backend_ctx->q3_k_mv_rows) + + " -DQ3_K_MV_NSG=" + std::to_string(backend_ctx->q3_k_mv_nsg); + cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), opts); + cl_kernel kernel = clCreateKernel(prog, "kernel_gemv_noshuffle_q3_k_f32", &err); + CL_CHECK(err); + CL_CHECK(clReleaseProgram(prog)); + size_t max_workgroup; + CL_CHECK(clGetKernelWorkGroupInfo(kernel, backend_ctx->device, CL_KERNEL_WORK_GROUP_SIZE, sizeof(max_workgroup), &max_workgroup, nullptr)); + if (64*(size_t)backend_ctx->q3_k_mv_nsg <= max_workgroup) { + backend_ctx->kernel_gemv_noshuffle_q3_k_f32 = kernel; + break; + } + CL_CHECK(clReleaseKernel(kernel)); + GGML_ASSERT(backend_ctx->q3_k_mv_nsg > 1); + backend_ctx->q3_k_mv_nsg /= 2; + } + GGML_LOG_CONT("."); + } + // gemm_noshuffle_q2_k_f32 { #ifdef GGML_OPENCL_EMBED_KERNELS @@ -8245,6 +8398,27 @@ struct ggml_tensor_extra_cl_q2_K { } }; +struct ggml_tensor_extra_cl_q3_K { + bool noshuffle = false; + cl_mem qs = nullptr; // quantized values + cl_mem hm = nullptr; // high bits + cl_mem sc = nullptr; // packed 6-bit scales + cl_mem d = nullptr; // super-block scales + + size_t size_qs = 0, size_hm = 0, size_sc = 0, size_d = 0; + + ~ggml_tensor_extra_cl_q3_K() { reset(); } + + void reset() { + noshuffle = false; + if (qs != nullptr) { CL_CHECK(clReleaseMemObject(qs)); qs = nullptr; } + if (hm != nullptr) { CL_CHECK(clReleaseMemObject(hm)); hm = nullptr; } + if (sc != nullptr) { CL_CHECK(clReleaseMemObject(sc)); sc = nullptr; } + if (d != nullptr) { CL_CHECK(clReleaseMemObject(d)); d = nullptr; } + size_qs = size_hm = size_sc = size_d = 0; + } +}; + struct ggml_tensor_extra_cl_q6_K { // Lower 4 bits of quantized weights. cl_mem ql = nullptr; @@ -9345,6 +9519,24 @@ static bool ggml_cl_q2_k_use_noshuffle(const ggml_backend_opencl_context * backe #endif } +static bool ggml_cl_q3_k_use_noshuffle(const ggml_backend_opencl_context * backend_ctx, const ggml_tensor * tensor) { +#if defined(GGML_OPENCL_SOA_Q) && defined(GGML_OPENCL_USE_ADRENO_KERNELS) + if (const char * env = getenv("GGML_OPENCL_Q3_K_NOSHUFFLE")) { + if (env[0] == '0') { + return false; + } + } + return tensor->type == GGML_TYPE_Q3_K && + backend_ctx->gpu_family == ADRENO && + ggml_is_contiguous(tensor) && !tensor->view_src && + use_adreno_kernels(backend_ctx, tensor); +#else + GGML_UNUSED(backend_ctx); + GGML_UNUSED(tensor); + return false; +#endif +} + inline bool use_adreno_moe_kernels(const ggml_backend_opencl_context *backend_ctx, const ggml_tensor *tensor) { // The moe weight repack kernels *_trans4_ns alias a private ushort8 through a uchar*. // Certain compilers (found with some A7x and A6x) miscompiles this, corrupting the weights. @@ -9868,6 +10060,12 @@ static bool ggml_opencl_supports_op(ggml_backend_dev_t dev, const struct ggml_te } else if (op->src[0]->type == GGML_TYPE_Q4_0) { // Non-contig src0 routes through on-device dequant-to-f16. return op->src[1]->type == GGML_TYPE_F32; + } else if (op->src[0]->type == GGML_TYPE_Q3_K) { + const ggml_tensor * weight = op->src[0]->view_src; + if (weight && ggml_cl_q3_k_use_noshuffle(backend_ctx, weight)) { + return false; + } + return op->src[1]->type == GGML_TYPE_F32 && ggml_is_contiguous_rows(op->src[1]); } else if (op->src[0]->type == GGML_TYPE_Q2_K) { const ggml_tensor * weight = op->src[0]->view_src; if (weight && ggml_cl_q2_k_use_noshuffle(backend_ctx, weight)) { @@ -10254,12 +10452,18 @@ struct ggml_backend_opencl_buffer_context { for (ggml_tensor_extra_cl_q2_K * e : temp_tensor_extras_q2_K) { delete e; } + for (ggml_tensor_extra_cl_q3_K * e : temp_tensor_extras_q3_K) { + delete e; + } for (ggml_tensor_extra_cl_q6_K * e : temp_tensor_extras_q6_K_in_use) { delete e; } for (ggml_tensor_extra_cl_q2_K * e : temp_tensor_extras_q2_K_in_use) { delete e; } + for (ggml_tensor_extra_cl_q3_K * e : temp_tensor_extras_q3_K_in_use) { + delete e; + } for (ggml_tensor_extra_cl_q5_K * e : temp_tensor_extras_q5_K) { delete e; } @@ -10448,6 +10652,21 @@ struct ggml_backend_opencl_buffer_context { return extra; } + ggml_tensor_extra_cl_q3_K * ggml_opencl_alloc_temp_tensor_extra_q3_K() { + ggml_tensor_extra_cl_q3_K * extra; + if (temp_tensor_extras_q3_K.empty()) { + extra = new ggml_tensor_extra_cl_q3_K(); + } else { + extra = temp_tensor_extras_q3_K.back(); + temp_tensor_extras_q3_K.pop_back(); + } + + temp_tensor_extras_q3_K_in_use.push_back(extra); + + extra->reset(); + return extra; + } + ggml_tensor_extra_cl_q6_K * ggml_opencl_alloc_temp_tensor_extra_q6_K() { ggml_tensor_extra_cl_q6_K * extra; if (temp_tensor_extras_q6_K.empty()) { @@ -10529,6 +10748,11 @@ struct ggml_backend_opencl_buffer_context { } temp_tensor_extras_q2_K_in_use.clear(); + for (ggml_tensor_extra_cl_q3_K * e : temp_tensor_extras_q3_K_in_use) { + temp_tensor_extras_q3_K.push_back(e); + } + temp_tensor_extras_q3_K_in_use.clear(); + q8_0_soa_tensors.clear(); q4_0_soa_tensors.clear(); } @@ -10561,7 +10785,9 @@ struct ggml_backend_opencl_buffer_context { std::vector temp_tensor_extras_q5_K; std::vector temp_tensor_extras_q5_K_in_use; std::vector temp_tensor_extras_q2_K; + std::vector temp_tensor_extras_q3_K; std::vector temp_tensor_extras_q2_K_in_use; + std::vector temp_tensor_extras_q3_K_in_use; std::vector temp_tensor_extras_q6_K; std::vector temp_tensor_extras_q6_K_in_use; @@ -10818,6 +11044,91 @@ static void ggml_backend_opencl_buffer_set_tensor(ggml_backend_buffer_t buffer, return; } + if (tensor->type == GGML_TYPE_Q3_K) { + GGML_ASSERT(offset == 0 && size == ggml_nbytes(tensor)); + ggml_tensor_extra_cl * extra_orig = (ggml_tensor_extra_cl *)tensor->extra; + GGML_ASSERT(extra_orig && "Tensors in OpenCL backend should have been allocated and initialized"); + + ggml_backend_opencl_buffer_context * ctx = (ggml_backend_opencl_buffer_context *) buffer->context; + ggml_tensor_extra_cl_q3_K * extra = ctx->ggml_opencl_alloc_temp_tensor_extra_q3_K(); + extra->noshuffle = ggml_cl_q3_k_use_noshuffle(backend_ctx, tensor); + + const size_t blck = (size_t)ggml_blck_size(tensor->type); + const size_t n_blk = ggml_nelements(tensor) / blck; + + const size_t size_qs = n_blk * (blck/4); + const size_t size_hm = n_blk * (blck/8); + const size_t size_sc = n_blk * 12; + const size_t size_d = n_blk * sizeof(ggml_fp16_t); + GGML_ASSERT(size_qs + size_hm + size_sc + size_d == ggml_nbytes(tensor) && "Incorrect tensor size"); + + cl_int err; + cl_mem data_device = clCreateBuffer(context, CL_MEM_READ_WRITE, + ggml_nbytes(tensor), NULL, &err); + CL_CHECK(err); + CL_CHECK(clEnqueueWriteBuffer( + queue, data_device, CL_TRUE, 0, + ggml_nbytes(tensor), data, 0, NULL, NULL)); + + cl_buffer_region region; + region.origin = align_to(extra_orig->offset + tensor->view_offs + offset, backend_ctx->alignment); + region.size = size_qs; + CL_CHECK((extra->qs = clCreateSubBuffer(extra_orig->data_device, CL_MEM_READ_WRITE, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); + auto previous_origin = region.origin; + + region.origin = align_to(previous_origin + size_qs, backend_ctx->alignment); + region.size = size_hm; + CL_CHECK((extra->hm = clCreateSubBuffer(extra_orig->data_device, CL_MEM_READ_WRITE, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); + previous_origin = region.origin; + + region.origin = align_to(previous_origin + size_hm, backend_ctx->alignment); + region.size = size_sc; + CL_CHECK((extra->sc = clCreateSubBuffer(extra_orig->data_device, CL_MEM_READ_WRITE, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); + previous_origin = region.origin; + + region.origin = align_to(previous_origin + size_sc, backend_ctx->alignment); + region.size = size_d; + CL_CHECK((extra->d = clCreateSubBuffer(extra_orig->data_device, CL_MEM_READ_WRITE, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); + + cl_kernel kernel = extra->noshuffle ? backend_ctx->kernel_convert_block_q3_k_noshuffle + : backend_ctx->kernel_convert_block_q3_K; + cl_ulong nb_arg = (cl_ulong)n_blk; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &data_device)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra->qs)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra->hm)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra->sc)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extra->d)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &nb_arg)); + + size_t global_work_size[] = { (size_t)CEIL_DIV(n_blk, 64)*64, 1, 1 }; + size_t local_work_size[] = { 64, 1, 1 }; + + cl_event evt; + CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 3, NULL, global_work_size, local_work_size, 0, NULL, &evt)); + CL_CHECK(clWaitForEvents(1, &evt)); + CL_CHECK(clReleaseEvent(evt)); + CL_CHECK(clReleaseMemObject(data_device)); + +#ifdef GGML_OPENCL_USE_ADRENO_KERNELS + if (extra->noshuffle) { + const int m = tensor->ne[1]; + const int k = tensor->ne[0]; + transpose_2d_as_8b(backend_ctx, extra->qs, extra->qs, size_qs, k/4, m, true, true); + transpose_2d_as_8b(backend_ctx, extra->hm, extra->hm, size_hm, k/8, m, true, true); + transpose_2d(backend_ctx, backend_ctx->kernel_transpose_32_buf, extra->sc, extra->sc, size_sc, 3*k/(int)blck, m, true, true); + transpose_2d(backend_ctx, backend_ctx->kernel_transpose_16_buf, extra->d, extra->d, size_d, k/(int)blck, m, true, true); + } +#endif + + extra->size_qs = size_qs; + extra->size_hm = size_hm; + extra->size_sc = size_sc; + extra->size_d = size_d; + + tensor->extra = extra; + return; + } + if (tensor->type == GGML_TYPE_Q1_0) { ggml_tensor_extra_cl * extra_orig = (ggml_tensor_extra_cl *)tensor->extra; GGML_ASSERT(extra_orig && "Tesnors in OpenCL backend should have been allocated and initialized"); @@ -12553,6 +12864,56 @@ static void ggml_backend_opencl_buffer_get_tensor(ggml_backend_buffer_t buffer, return; } + if (tensor->type == GGML_TYPE_Q3_K) { + ggml_tensor_extra_cl_q3_K * extra = (ggml_tensor_extra_cl_q3_K *)tensor->extra; + + const size_t blck = (size_t)ggml_blck_size(tensor->type); + + ggml_cl_buffer buf_unpacked; + buf_unpacked.allocate(backend_ctx->context, ggml_nbytes(tensor)); + + const size_t n_blk = (size_t)ggml_nelements(tensor) / blck; + cl_ulong nb_arg = (cl_ulong)n_blk; + + cl_mem qs = extra->qs, hm = extra->hm, sc = extra->sc, d = extra->d; +#ifdef GGML_OPENCL_USE_ADRENO_KERNELS + ggml_cl_buffer buf_qs, buf_hm, buf_sc, buf_d; + if (extra->noshuffle) { + buf_qs.allocate(context, extra->size_qs); + buf_hm.allocate(context, extra->size_hm); + buf_sc.allocate(context, extra->size_sc); + buf_d.allocate(context, extra->size_d); + qs = buf_qs.buffer; + hm = buf_hm.buffer; + sc = buf_sc.buffer; + d = buf_d.buffer; + const int m = tensor->ne[1]; + const int k = tensor->ne[0]; + transpose_2d_as_8b(backend_ctx, extra->qs, qs, extra->size_qs, m, k/4, true, true); + transpose_2d_as_8b(backend_ctx, extra->hm, hm, extra->size_hm, m, k/8, true, true); + transpose_2d(backend_ctx, backend_ctx->kernel_transpose_32_buf, extra->sc, sc, extra->size_sc, m, 3*k/(int)blck, true, true); + transpose_2d(backend_ctx, backend_ctx->kernel_transpose_16_buf, extra->d, d, extra->size_d, m, k/(int)blck, true, true); + } +#endif + cl_kernel kernel = extra->noshuffle ? backend_ctx->kernel_restore_block_q3_k_noshuffle + : backend_ctx->kernel_restore_block_q3_K; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &qs)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &hm)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &sc)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &d)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &buf_unpacked.buffer)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &nb_arg)); + + size_t gws[] = { (n_blk + 63) / 64 * 64 }; + size_t lws[] = { 64 }; + cl_event evt; + CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 1, NULL, gws, lws, 0, NULL, &evt)); + CL_CHECK(clWaitForEvents(1, &evt)); + CL_CHECK(clReleaseEvent(evt)); + CL_CHECK(clEnqueueReadBuffer(queue, buf_unpacked.buffer, CL_TRUE, offset, size, data, 0, NULL, NULL)); + return; + } + if (tensor->type == GGML_TYPE_Q1_0) { ggml_tensor_extra_cl_q1_0 * extra = (ggml_tensor_extra_cl_q1_0 *)tensor->extra; @@ -24924,6 +25285,65 @@ static bool ggml_cl_mul_mat_q2_K_dp4a(ggml_backend_t backend, const ggml_tensor return true; } +static bool ggml_cl_mul_mat_q3_K_dp4a(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { + ggml_backend_opencl_context * backend_ctx = (ggml_backend_opencl_context *)backend->context; + ggml_tensor_extra_cl_q3_K * extra0 = (ggml_tensor_extra_cl_q3_K *)src0->extra; + ggml_tensor_extra_cl * extra1 = (ggml_tensor_extra_cl *)src1->extra; + ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra; + + cl_ulong offset1 = extra1->offset + src1->view_offs; + cl_ulong offsetd = extrad->offset + dst->view_offs; + const int m = src0->ne[1]; + const int n = src1->ne[1]; + const int k = src0->ne[0]; + + const int tile_n = n <= 16 ? 8 : 32; + cl_kernel gemm = n <= 16 ? backend_ctx->kernel_gemm_noshuffle_q3_k_q8_1_dp4a_narrow + : backend_ctx->kernel_gemm_noshuffle_q3_k_q8_1_dp4a; + const char * env = getenv("GGML_OPENCL_Q3_K_DENSE_DP4A"); + if (!gemm || !backend_ctx->has_integer_dot || (env && env[0] == '0') || + n <= 1 || !ggml_is_contiguous(src1) || offset1 % backend_ctx->alignment != 0) { + return false; + } + + cl_int err; + cl_buffer_region region = { (size_t)offset1, (size_t)k*n*sizeof(float) }; + cl_mem activation = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err); + CL_CHECK(err); + const size_t n_blocks = (size_t)n*(k/32); + backend_ctx->prealloc_moe_qa.allocate(backend_ctx->context, (size_t)n*k); + backend_ctx->prealloc_moe_da.allocate(backend_ctx->context, n_blocks*sizeof(cl_half)); + backend_ctx->prealloc_moe_sa.allocate(backend_ctx->context, n_blocks*sizeof(cl_half)); + + cl_int nb = n_blocks; + cl_kernel quant = backend_ctx->kernel_quant_a_q8_1; + CL_CHECK(clSetKernelArg(quant, 0, sizeof(cl_mem), &activation)); + CL_CHECK(clSetKernelArg(quant, 1, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer)); + CL_CHECK(clSetKernelArg(quant, 2, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer)); + CL_CHECK(clSetKernelArg(quant, 3, sizeof(cl_mem), &backend_ctx->prealloc_moe_sa.buffer)); + CL_CHECK(clSetKernelArg(quant, 4, sizeof(cl_int), &nb)); + size_t q_global[] = { CEIL_DIV(n_blocks, 64)*64 }; + size_t q_local[] = { 64 }; + backend_ctx->enqueue_ndrange_kernel(quant, 1, q_global, q_local, dst); + + CL_CHECK(clSetKernelArg(gemm, 0, sizeof(cl_mem), &extra0->qs)); + CL_CHECK(clSetKernelArg(gemm, 1, sizeof(cl_mem), &extra0->hm)); + CL_CHECK(clSetKernelArg(gemm, 2, sizeof(cl_mem), &extra0->sc)); + CL_CHECK(clSetKernelArg(gemm, 3, sizeof(cl_mem), &extra0->d)); + CL_CHECK(clSetKernelArg(gemm, 4, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer)); + CL_CHECK(clSetKernelArg(gemm, 5, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer)); + CL_CHECK(clSetKernelArg(gemm, 6, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(gemm, 7, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(gemm, 8, sizeof(int), &m)); + CL_CHECK(clSetKernelArg(gemm, 9, sizeof(int), &n)); + CL_CHECK(clSetKernelArg(gemm, 10, sizeof(int), &k)); + size_t global[] = { 64, (size_t)CEIL_DIV(m, 64), (size_t)CEIL_DIV(n, tile_n) }; + size_t local[] = { 64, 1, 1 }; + backend_ctx->enqueue_ndrange_kernel(gemm, 3, global, local, dst); + CL_CHECK(clReleaseMemObject(activation)); + return true; +} + static void ggml_cl_mul_mat_q2_k_f32_adreno(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { ggml_backend_opencl_context * backend_ctx = (ggml_backend_opencl_context *)backend->context; ggml_tensor_extra_cl_q2_K * extra0 = (ggml_tensor_extra_cl_q2_K *)src0->extra; @@ -25023,6 +25443,108 @@ static void ggml_cl_mul_mat_q2_k_f32_adreno(ggml_backend_t backend, const ggml_t CL_CHECK(clReleaseMemObject(b_img)); CL_CHECK(clReleaseMemObject(b_sub_buf)); } + +static void ggml_cl_mul_mat_q3_k_f32_adreno(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { + ggml_backend_opencl_context * backend_ctx = (ggml_backend_opencl_context *)backend->context; + ggml_tensor_extra_cl_q3_K * extra0 = (ggml_tensor_extra_cl_q3_K *)src0->extra; + ggml_tensor_extra_cl * extra1 = (ggml_tensor_extra_cl *)src1->extra; + ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra; + + cl_ulong offset1 = extra1->offset + src1->view_offs; + cl_ulong offsetd = extrad->offset + dst->view_offs; + const int m = src0->ne[1]; + const int n = src1->ne[1]; + const int k = src0->ne[0]; + + if (n == 1 || !ggml_is_contiguous(src1) || offset1 % backend_ctx->alignment != 0) { + cl_kernel gemv = backend_ctx->kernel_gemv_noshuffle_q3_k_f32; + const int stride_d = dst->ne[0]; + const int stride_b = src1->nb[1]/sizeof(float); + CL_CHECK(clSetKernelArg(gemv, 0, sizeof(cl_mem), &extra0->qs)); + CL_CHECK(clSetKernelArg(gemv, 1, sizeof(cl_mem), &extra0->hm)); + CL_CHECK(clSetKernelArg(gemv, 2, sizeof(cl_mem), &extra0->sc)); + CL_CHECK(clSetKernelArg(gemv, 3, sizeof(cl_mem), &extra0->d)); + CL_CHECK(clSetKernelArg(gemv, 4, sizeof(cl_mem), &extra1->data_device)); + CL_CHECK(clSetKernelArg(gemv, 5, sizeof(cl_ulong), &offset1)); + CL_CHECK(clSetKernelArg(gemv, 6, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(gemv, 7, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(gemv, 8, sizeof(int), &k)); + CL_CHECK(clSetKernelArg(gemv, 9, sizeof(int), &m)); + CL_CHECK(clSetKernelArg(gemv, 10, sizeof(int), &stride_b)); + CL_CHECK(clSetKernelArg(gemv, 11, sizeof(int), &stride_d)); + const size_t rows = 64*(size_t)backend_ctx->q3_k_mv_rows; + const size_t nsg = backend_ctx->q3_k_mv_nsg; + size_t global[] = { CEIL_DIV((size_t)m, rows)*64, (size_t)n*nsg, 1 }; + size_t local[] = { 64, nsg, 1 }; + backend_ctx->enqueue_ndrange_kernel(gemv, 3, global, local, dst); + return; + } + + if (ggml_cl_mul_mat_q3_K_dp4a(backend, src0, src1, dst)) { + return; + } + + cl_context context = backend_ctx->context; + cl_int err; + cl_buffer_region region = { (size_t)offset1, (size_t)k*n*sizeof(float) }; + cl_mem b_sub_buf = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err); + CL_CHECK(err); + + cl_image_format img_fmt = { CL_RGBA, CL_FLOAT }; + cl_image_desc img_desc = {}; + img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; + img_desc.image_width = (size_t)k*n/4; + img_desc.buffer = b_sub_buf; + cl_mem b_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, nullptr, &err); + CL_CHECK(err); + + const int padded_n = CEIL_DIV(n, 8)*8; + region.origin = 0; + region.size = (size_t)k*padded_n*sizeof(cl_half); + backend_ctx->prealloc_act_trans.allocate(context, region.size); + cl_mem b_sub_buf_trans = clCreateSubBuffer(backend_ctx->prealloc_act_trans.buffer, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err); + CL_CHECK(err); + + img_fmt = { CL_RGBA, CL_HALF_FLOAT }; + img_desc.image_width = (size_t)k*padded_n/4; + img_desc.buffer = b_sub_buf_trans; + cl_mem b_img_trans = clCreateImage(context, 0, &img_fmt, &img_desc, nullptr, &err); + CL_CHECK(err); + + const int height_b = std::max(n/4, 1); + const int width_b = k/4; + const int padded_height_b = padded_n/4; + cl_kernel kernel = backend_ctx->kernel_transpose_32_16; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &b_img)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &b_img_trans)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(int), &height_b)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(int), &width_b)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(int), &padded_height_b)); + size_t t_global[] = { (size_t)width_b, (size_t)padded_height_b }; + size_t t_local[] = { 1, 16 }; + backend_ctx->enqueue_ndrange_kernel(kernel, 2, t_global, t_local, dst); + + kernel = backend_ctx->kernel_gemm_noshuffle_q3_k_f32; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->qs)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0->hm)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra0->sc)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra0->d)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &b_img_trans)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &m)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &padded_n)); + CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &k)); + CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &n)); + size_t global[] = { (size_t)CEIL_DIV(n, 8), (size_t)CEIL_DIV(m, 512)*128, 1 }; + size_t local[] = { 1, 128, 1 }; + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global, local, dst); + + CL_CHECK(clReleaseMemObject(b_img_trans)); + CL_CHECK(clReleaseMemObject(b_sub_buf_trans)); + CL_CHECK(clReleaseMemObject(b_img)); + CL_CHECK(clReleaseMemObject(b_sub_buf)); +} #endif static void ggml_cl_mul_mat(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { @@ -25094,6 +25616,7 @@ static void ggml_cl_mul_mat(ggml_backend_t backend, const ggml_tensor * src0, co ggml_tensor_extra_cl_q8_0 * extra0_q8_0 = (ggml_tensor_extra_cl_q8_0 *)soa0_src->extra; ggml_tensor_extra_cl_iq4_nl * extra0_iq4_nl = (ggml_tensor_extra_cl_iq4_nl *)soa0_src->extra; ggml_tensor_extra_cl_q2_K * extra0_q2_K = (ggml_tensor_extra_cl_q2_K *)soa0_src->extra; + ggml_tensor_extra_cl_q3_K * extra0_q3_K = (ggml_tensor_extra_cl_q3_K *)soa0_src->extra; ggml_tensor_extra_cl_q4_K * extra0_q4_K = (ggml_tensor_extra_cl_q4_K *)soa0_src->extra; ggml_tensor_extra_cl_q5_K * extra0_q5_K = (ggml_tensor_extra_cl_q5_K *)soa0_src->extra; ggml_tensor_extra_cl_q6_K * extra0_q6_K = (ggml_tensor_extra_cl_q6_K *)soa0_src->extra; @@ -25495,6 +26018,12 @@ static void ggml_cl_mul_mat(ggml_backend_t backend, const ggml_tensor * src0, co return; } + // q3_k x fp32 + if (src0t == GGML_TYPE_Q3_K && src1t == GGML_TYPE_F32 && extra0_q3_K->noshuffle) { + ggml_cl_mul_mat_q3_k_f32_adreno(backend, src0, src1, dst); + return; + } + // q4_k x fp32 if (src0t == GGML_TYPE_Q4_K && src1t == GGML_TYPE_F32 && !use_flat_gemv_for_large_m_q4_K(backend_ctx, src0)) { ggml_cl_mul_mat_q4_k_f32_adreno(backend, src0, src1, dst); @@ -26045,6 +26574,51 @@ static void ggml_cl_mul_mat(ggml_backend_t backend, const ggml_tensor * src0, co backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); return; } + case GGML_TYPE_Q3_K: { + if (ne11 < 32) { + break; + } + if (!ggml_is_contiguous(src0) || !ggml_is_contiguous(src1)) { + break; + } + + kernel = backend_ctx->kernel_mul_mm_q3_k_f32_l4_lm; + nth0 = (backend_ctx->gpu_family == INTEL) ? 64 : 128; + offset0 = src0->view_offs; + + int batch_stride_a = ne00*ne01; + int batch_stride_b = ne10*ne11; + int batch_stride_d = ne0*ne1; + + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0_q3_K->qs)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_q3_K->hm)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra0_q3_K->sc)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra0_q3_K->d)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_ulong), &offset0)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_mem), &extra1->data_device)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_ulong), &offset1)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &ne01)); + CL_CHECK(clSetKernelArg(kernel, 11, sizeof(int), &ne02)); + CL_CHECK(clSetKernelArg(kernel, 12, sizeof(int), &ne11)); + CL_CHECK(clSetKernelArg(kernel, 13, sizeof(int), &ne12)); + CL_CHECK(clSetKernelArg(kernel, 14, sizeof(int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, 15, sizeof(int), &ne10)); + CL_CHECK(clSetKernelArg(kernel, 16, sizeof(int), &ne0)); + CL_CHECK(clSetKernelArg(kernel, 17, sizeof(int), &batch_stride_a)); + CL_CHECK(clSetKernelArg(kernel, 18, sizeof(int), &batch_stride_b)); + CL_CHECK(clSetKernelArg(kernel, 19, sizeof(int), &batch_stride_d)); + CL_CHECK(clSetKernelArg(kernel, 20, sizeof(int), &r2)); + CL_CHECK(clSetKernelArg(kernel, 21, sizeof(int), &r3)); + + size_t global_work_size[] = {(size_t)(CEIL_DIV(ne01, 64)*nth0), (size_t)CEIL_DIV(ne11, 64), (size_t)ne12*ne13}; + size_t local_work_size[] = {(size_t)nth0, 1, 1}; + + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); + return; + } case GGML_TYPE_Q4_K: { if (ne11 < 32) { break; @@ -27024,7 +27598,70 @@ static void ggml_cl_mul_mat(ggml_backend_t backend, const ggml_tensor * src0, co #endif // GGML_OPENCL_SOA_Q break; } - case GGML_TYPE_Q3_K: + case GGML_TYPE_Q3_K: { + if (backend_ctx->gpu_family == INTEL) { + nth0 = 16; + nth1 = 1; + ndst = 2; + } else if (backend_ctx->gpu_family == ADRENO) { + nth0 = 64; + nth1 = 1; + ndst = 2; + } else { + GGML_ASSERT(false && "TODO: Unknown GPU"); + } + +#ifdef GGML_OPENCL_SOA_Q + kernel = backend_ctx->kernel_mul_mv_q3_K_f32_flat; + offset0 = src0->view_offs; + + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0_q3_K->qs)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_q3_K->hm)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra0_q3_K->sc)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra0_q3_K->d)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_ulong), &offset0)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_mem), &extra1->data_device)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_ulong), &offset1)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &ne01)); + CL_CHECK(clSetKernelArg(kernel, 11, sizeof(cl_ulong), &nb01)); + CL_CHECK(clSetKernelArg(kernel, 12, sizeof(cl_ulong), &nb02)); + CL_CHECK(clSetKernelArg(kernel, 13, sizeof(cl_ulong), &nb03)); + CL_CHECK(clSetKernelArg(kernel, 14, sizeof(int), &ne12)); + CL_CHECK(clSetKernelArg(kernel, 15, sizeof(cl_ulong), &nb11)); + CL_CHECK(clSetKernelArg(kernel, 16, sizeof(cl_ulong), &nb12)); + CL_CHECK(clSetKernelArg(kernel, 17, sizeof(cl_ulong), &nb13)); + CL_CHECK(clSetKernelArg(kernel, 18, sizeof(int), &ne0)); + CL_CHECK(clSetKernelArg(kernel, 19, sizeof(int), &ne1)); + CL_CHECK(clSetKernelArg(kernel, 20, sizeof(int), &r2)); + CL_CHECK(clSetKernelArg(kernel, 21, sizeof(int), &r3)); +#else + kernel = backend_ctx->kernel_mul_mv_q3_K_f32; + + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra1->data_device)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &ne01)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_ulong), &nb01)); + CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_ulong), &nb02)); + CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_ulong), &nb03)); + CL_CHECK(clSetKernelArg(kernel, 11, sizeof(int), &ne12)); + CL_CHECK(clSetKernelArg(kernel, 12, sizeof(cl_ulong), &nb11)); + CL_CHECK(clSetKernelArg(kernel, 13, sizeof(cl_ulong), &nb12)); + CL_CHECK(clSetKernelArg(kernel, 14, sizeof(cl_ulong), &nb13)); + CL_CHECK(clSetKernelArg(kernel, 15, sizeof(int), &ne0)); + CL_CHECK(clSetKernelArg(kernel, 16, sizeof(int), &ne1)); + CL_CHECK(clSetKernelArg(kernel, 17, sizeof(int), &r2)); + CL_CHECK(clSetKernelArg(kernel, 18, sizeof(int), &r3)); +#endif // GGML_OPENCL_SOA_Q + break; + } case GGML_TYPE_Q4_K: { #ifdef GGML_OPENCL_SOA_Q kernel = backend_ctx->kernel_mul_mv_q4_K_f32_flat; @@ -27332,7 +27969,7 @@ static void ggml_cl_mul_mat(ggml_backend_t backend, const ggml_tensor * src0, co src0t == GGML_TYPE_Q8_0 || src0t == GGML_TYPE_Q1_0 || src0t == GGML_TYPE_IQ4_NL || - src0t == GGML_TYPE_Q2_K) { + src0t == GGML_TYPE_Q2_K || src0t == GGML_TYPE_Q3_K) { // Each SIMD group produces N_DST values in the result. Assuming each // workgroup has N_SIMDGROUP SIMD groups, then each workgroup will // produce N_DST*N_SIMDGROUP values in the result. Hence, the grid size @@ -27348,8 +27985,6 @@ static void ggml_cl_mul_mat(ggml_backend_t backend, const ggml_tensor * src0, co size_t local_work_size[] = {(size_t)nth0, (size_t)nth1, 1}; backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); - } else if (src0t == GGML_TYPE_Q3_K) { - GGML_ASSERT(false && "not implemented"); } else if (src0t == GGML_TYPE_Q5_K) { size_t global_work_size[] = {(size_t)(ne01+ndst*nth1-1)/(ndst*nth1)*nth0, (size_t)ne11*nth1, (size_t)ne12*ne13}; size_t local_work_size[] = {(size_t)nth0, (size_t)nth1, 1}; diff --git a/ggml/src/ggml-opencl/kernels/cvt.cl b/ggml/src/ggml-opencl/kernels/cvt.cl index f3309a298..fb21e8eba 100644 --- a/ggml/src/ggml-opencl/kernels/cvt.cl +++ b/ggml/src/ggml-opencl/kernels/cvt.cl @@ -2773,3 +2773,156 @@ kernel void kernel_restore_block_q2_k_noshuffle( } } } + +struct block_q3_K { + uchar hmask[QK_K/8]; + uchar qs[QK_K/4]; + uchar scales[12]; + half d; +}; + +kernel void kernel_convert_block_q3_K( + global struct block_q3_K * src0, + global uchar * dst_qs, + global uchar * dst_hm, + global uchar * dst_sc, + global half * dst_d, + ulong n_blk +) { + const ulong i = get_global_id(0); + if (i >= n_blk) { + return; + } + global struct block_q3_K * b = src0 + i; + for (int j = 0; j < 64; ++j) { + dst_qs[64*i + j] = b->qs[j]; + } + for (int j = 0; j < 32; ++j) { + dst_hm[32*i + j] = b->hmask[j]; + } + for (int j = 0; j < 12; ++j) { + dst_sc[12*i + j] = b->scales[j]; + } + dst_d[i] = b->d; +} + +kernel void kernel_restore_block_q3_K( + global uchar * src_qs, + global uchar * src_hm, + global uchar * src_sc, + global half * src_d, + global struct block_q3_K * dst, + ulong n_blk +) { + const ulong i = get_global_id(0); + if (i >= n_blk) { + return; + } + global struct block_q3_K * b = dst + i; + for (int j = 0; j < 64; ++j) { + b->qs[j] = src_qs[64*i + j]; + } + for (int j = 0; j < 32; ++j) { + b->hmask[j] = src_hm[32*i + j]; + } + for (int j = 0; j < 12; ++j) { + b->scales[j] = src_sc[12*i + j]; + } + b->d = src_d[i]; +} + +kernel void kernel_convert_block_q3_k_noshuffle( + global struct block_q3_K * src0, + global uchar * dst_qs, // QK_K/4 per block + global uchar * dst_hm, // QK_K/8 per block + global uint * dst_sc, // 3 per block + global half * dst_d, // 1 + ulong n_blk +) { + const ulong i = get_global_id(0); + if (i >= n_blk) { + return; + } + global struct block_q3_K * b = (global struct block_q3_K *) src0 + i; + + dst_d[i] = b->d; + + for (int w = 0; w < 3; ++w) { + global const uchar * p = b->scales + 4*w; + dst_sc[3*i + w] = (uint)p[0] | ((uint)p[1] << 8) | ((uint)p[2] << 16) | ((uint)p[3] << 24); + } + + for (int g = 0; g < QK_K/4; ++g) { + uint pk = 0; + for (int t = 0; t < 4; ++t) { + const int e = 4*g + t; + const int sb = e >> 5; + const int ee = e & 31; + pk |= (uint)(((b->qs[32*(sb >> 2) + ee] >> (2*(sb & 3))) & 3) << (2*t)); + } + dst_qs[(QK_K/4) * i + g] = (uchar)pk; + } + + // two groups per hmask byte, so this loop owns the byte and cannot race + for (int t = 0; t < QK_K/8; ++t) { + uint v = 0; + for (int s = 0; s < 2; ++s) { + const int g = 2*t + s; + for (int u = 0; u < 4; ++u) { + const int e = 4*g + u; + const int sb = e >> 5; + const int ee = e & 31; + v |= (uint)(((b->hmask[ee] >> sb) & 1) << (4*s + u)); + } + } + dst_hm[(QK_K/8) * i + t] = (uchar)v; + } +} + +kernel void kernel_restore_block_q3_k_noshuffle( + global uchar * src_qs, + global uchar * src_hm, + global uint * src_sc, + global half * src_d, + global struct block_q3_K * dst, + ulong n_blk +) { + const ulong i = get_global_id(0); + if (i >= n_blk) { + return; + } + global struct block_q3_K * b = (global struct block_q3_K *) dst + i; + + b->d = src_d[i]; + + for (int w = 0; w < 3; ++w) { + const uint x = src_sc[3*i + w]; + global uchar * p = b->scales + 4*w; + p[0] = (uchar)( x & 0xFF); + p[1] = (uchar)((x >> 8) & 0xFF); + p[2] = (uchar)((x >> 16) & 0xFF); + p[3] = (uchar)((x >> 24) & 0xFF); + } + + for (int n2 = 0; n2 < 2; ++n2) { + for (int ee = 0; ee < 32; ++ee) { + uint v = 0; + for (int j = 0; j < 4; ++j) { + const int sb = 4*n2 + j; + const int e = 32*sb + ee; + v |= ((uint)(src_qs[(QK_K/4) * i + (e >> 2)] >> (2*(e & 3))) & 3u) << (2*j); + } + b->qs[32*n2 + ee] = (uchar)v; + } + } + + for (int ee = 0; ee < 32; ++ee) { + uint v = 0; + for (int sb = 0; sb < 8; ++sb) { + const int e = 32*sb + ee; + const int g = e >> 2; + v |= ((uint)(src_hm[(QK_K/8) * i + (g >> 1)] >> (4*(g & 1) + (e & 3))) & 1u) << sb; + } + b->hmask[ee] = (uchar)v; + } +} diff --git a/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q3_k_f32.cl b/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q3_k_f32.cl new file mode 100644 index 000000000..285114678 --- /dev/null +++ b/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q3_k_f32.cl @@ -0,0 +1,92 @@ +#pragma OPENCL EXTENSION cl_khr_fp16 : enable + +#ifdef cl_qcom_reqd_sub_group_size +#pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable +__attribute__((qcom_reqd_sub_group_size("full"))) +#endif +kernel void kernel_gemm_noshuffle_q3_k_f32( + global const uchar * src0_qs, + global const uchar * src0_hm, + global const uint * src0_sc, + global const half * src0_d, + read_only image1d_buffer_t src1, + global float * dst, + ulong offsetd, + int m, + int n, + int k, + int n_no_padding +) { + dst = (global float *)((global char *)dst + offsetd); + const int gy = get_global_id(0); + const int row = get_global_id(1) * 4; + const int n_4 = n / 4; + if (row >= m) { + return; + } + + half8 c0 = 0, c1 = 0, c2 = 0, c3 = 0; + for (int i = 0; i < k; i += 16) { + const int is = (i / 16) & 15; + const int word = is >> 2; + const int shift = 8 * (is & 3); + const int sc_base = row + 3 * (i / 256) * m; + const uint4 low = vload4(0, src0_sc + sc_base + (word & 1) * m) >> (uint4)(4 * (word >> 1)); + const uint4 high = vload4(0, src0_sc + sc_base + 2 * m) >> (uint4)(2 * word); + const int4 sc = convert_int4(((low >> (uint4)(shift)) & (uint4)(15)) | (((high >> (uint4)(shift)) & (uint4)(3)) << (uint4)(4))) - (int4)(32); + const half4 d = vload4(0, src0_d + row + (i / 256) * m); + const half4 scale = convert_half4(convert_float4(d) * convert_float4(sc)); + + for (int l = 0; l < 16; l += 4) { + const int ki = i + l; + const uchar4 bits = vload4(0, src0_qs + row + (ki / 4) * m); + const uchar4 hbits = vload4(0, src0_hm + row + (ki / 8) * m); + #pragma unroll + for (int j = 0; j < 4; ++j) { + half8 b; + b.s0123 = read_imageh(src1, gy * 2 + (ki + j) * n_4); + b.s4567 = read_imageh(src1, gy * 2 + 1 + (ki + j) * n_4); + const int4 low = convert_int4((bits >> (uchar4)(2 * j)) & (uchar4)(3)); + const int4 high = convert_int4((hbits >> (uchar4)((ki & 7) + j)) & (uchar4)(1)); + const half4 w = convert_half4(low - (int4)(4) + (int4)(4) * high) * scale; + c0 += b * w.s0; + c1 += b * w.s1; + c2 += b * w.s2; + c3 += b * w.s3; + } + } + } + + int idx = gy * 8 * m + row; + if (idx + 3 < m * n_no_padding) { + vstore4((float4)(c0.s0, c1.s0, c2.s0, c3.s0), 0, dst + idx); + idx += m; + } + if (idx + 3 < m * n_no_padding) { + vstore4((float4)(c0.s1, c1.s1, c2.s1, c3.s1), 0, dst + idx); + idx += m; + } + if (idx + 3 < m * n_no_padding) { + vstore4((float4)(c0.s2, c1.s2, c2.s2, c3.s2), 0, dst + idx); + idx += m; + } + if (idx + 3 < m * n_no_padding) { + vstore4((float4)(c0.s3, c1.s3, c2.s3, c3.s3), 0, dst + idx); + idx += m; + } + if (idx + 3 < m * n_no_padding) { + vstore4((float4)(c0.s4, c1.s4, c2.s4, c3.s4), 0, dst + idx); + idx += m; + } + if (idx + 3 < m * n_no_padding) { + vstore4((float4)(c0.s5, c1.s5, c2.s5, c3.s5), 0, dst + idx); + idx += m; + } + if (idx + 3 < m * n_no_padding) { + vstore4((float4)(c0.s6, c1.s6, c2.s6, c3.s6), 0, dst + idx); + idx += m; + } + if (idx + 3 < m * n_no_padding) { + vstore4((float4)(c0.s7, c1.s7, c2.s7, c3.s7), 0, dst + idx); + } +} diff --git a/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q3_k_q8_1_dp4a.cl b/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q3_k_q8_1_dp4a.cl new file mode 100644 index 000000000..04fcca0b3 --- /dev/null +++ b/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q3_k_q8_1_dp4a.cl @@ -0,0 +1,160 @@ +#pragma OPENCL EXTENSION cl_khr_fp16 : enable +#pragma OPENCL EXTENSION cl_khr_subgroups : enable +#ifdef cl_khr_integer_dot_product +#pragma OPENCL EXTENSION cl_khr_integer_dot_product : enable +#endif + +#define QK_K 256 + +#ifndef TILESIZE_N +#define TILESIZE_N 32 +#endif + +inline int q3_k_scale(uint sc0, uint sc1, uint sc2, uint is) { + const uint w = is >> 2; + const uint sh = 8u * (is & 3u); + const uint tb = (sc2 >> sh) & 0xFFu; + uint v; + if (w == 0u) { v = ( (sc0 >> sh) & 0xFu) | (((tb >> 0) & 3u) << 4); } + else if (w == 1u) { v = ( (sc1 >> sh) & 0xFu) | (((tb >> 2) & 3u) << 4); } + else if (w == 2u) { v = (((sc0 >> sh) >> 4) & 0xFu) | (((tb >> 4) & 3u) << 4); } + else { v = (((sc1 >> sh) >> 4) & 0xFu) | (((tb >> 6) & 3u) << 4); } + return (int)v - 32; +} + +inline uint q3_k_pack(uint pk, uint hb) { + int v0 = (int)((pk ) & 3u) - 4 + 4*(int)((hb ) & 1u); + int v1 = (int)((pk >> 2) & 3u) - 4 + 4*(int)((hb >> 1) & 1u); + int v2 = (int)((pk >> 4) & 3u) - 4 + 4*(int)((hb >> 2) & 1u); + int v3 = (int)((pk >> 6) & 3u) - 4 + 4*(int)((hb >> 3) & 1u); + return ((uint)v0 & 0xFFu) | (((uint)v1 & 0xFFu) << 8) + | (((uint)v2 & 0xFFu) << 16) | (((uint)v3 & 0xFFu) << 24); +} + +// Pass staged operands by value; local dp4a operands miscompile on X2. +#define KQ_STAGE(p) vload4(0, (p)) + +inline int dot4_q8a_v(uint4 qw, uint4 a) { + int r = 0; + r = dot_acc_sat_4x8packed_ss_int(qw.s0, a.x, r); + r = dot_acc_sat_4x8packed_ss_int(qw.s1, a.y, r); + r = dot_acc_sat_4x8packed_ss_int(qw.s2, a.z, r); + r = dot_acc_sat_4x8packed_ss_int(qw.s3, a.w, r); + return r; +} + +__attribute__((qcom_wave_pair_mode(1))) +kernel void kernel_gemm_noshuffle_q3_k_q8_1_dp4a( + __global const uchar * src0_qs, + __global const uchar * src0_hm, + __global const uint * src0_sc, + __global const half * src0_d, + __global const uint * src1_qa, // q8_1 activations int8 (as uint, 4/elem) [N, K] + __global const half * src1_da, // q8_1 per-block scale [N, K/32] + __global float * dst, + ulong offsetd, + int m, // output features (rows) + int n_no_padding, // tokens (cols) + int k // K (== ne00) +) { + dst = (global float *)((global char *)dst + offsetd); + + const uint lid = get_local_id(0); + const uint block_id_m = get_global_id(1); + const uint block_id_n = get_global_id(2); + + const uint row = block_id_m * 64 + lid; + const uint col_base = block_id_n * TILESIZE_N; + const bool row_valid = row < (uint)m; + const uint rrow = row_valid ? row : 0; + + const uint k_u = (uint)k >> 2; + const uint k_b = (uint)k >> 5; + + __local uint4 sh_qa4[TILESIZE_N][2]; + __local half sh_d[TILESIZE_N]; + +#define NGROUPS (TILESIZE_N / 4) + float4 acc[NGROUPS]; + #pragma unroll + for (int g = 0; g < NGROUPS; ++g) acc[g] = (float4)(0.0f); + + for (uint step = 0; step < (uint)k; step += 32) { + const uint sub = step >> 5; + const uint ib = sub >> 3; // super-block along K + const uint sb = sub & 7u; + + const uint scb = rrow + (3u * ib) * (uint)m; + const uint sc0 = src0_sc[scb + 0u * (uint)m]; + const uint sc1 = src0_sc[scb + 1u * (uint)m]; + const uint sc2 = src0_sc[scb + 2u * (uint)m]; + + const float d_w = (float)src0_d[rrow + ib * (uint)m]; + const float dl0 = d_w * (float)q3_k_scale(sc0, sc1, sc2, 2u*sb + 0u); + const float dl1 = d_w * (float)q3_k_scale(sc0, sc1, sc2, 2u*sb + 1u); + + const uint qsb = rrow + (step >> 2) * (uint)m; + const uint hmb = rrow + (step >> 3) * (uint)m; + + uint4 qlo, qhi; + { + const uint h0 = (uint)src0_hm[hmb + 0u * (uint)m]; + const uint h1 = (uint)src0_hm[hmb + 1u * (uint)m]; + const uint h2 = (uint)src0_hm[hmb + 2u * (uint)m]; + const uint h3 = (uint)src0_hm[hmb + 3u * (uint)m]; + qlo.s0 = q3_k_pack((uint)src0_qs[qsb + 0u * (uint)m], h0 & 0xFu); + qlo.s1 = q3_k_pack((uint)src0_qs[qsb + 1u * (uint)m], (h0 >> 4) & 0xFu); + qlo.s2 = q3_k_pack((uint)src0_qs[qsb + 2u * (uint)m], h1 & 0xFu); + qlo.s3 = q3_k_pack((uint)src0_qs[qsb + 3u * (uint)m], (h1 >> 4) & 0xFu); + qhi.s0 = q3_k_pack((uint)src0_qs[qsb + 4u * (uint)m], h2 & 0xFu); + qhi.s1 = q3_k_pack((uint)src0_qs[qsb + 5u * (uint)m], (h2 >> 4) & 0xFu); + qhi.s2 = q3_k_pack((uint)src0_qs[qsb + 6u * (uint)m], h3 & 0xFu); + qhi.s3 = q3_k_pack((uint)src0_qs[qsb + 7u * (uint)m], (h3 >> 4) & 0xFu); + } + + for (uint idx = lid; idx < TILESIZE_N * 2; idx += 64) { + const uint t = idx >> 1; + const uint v = idx & 1; + const uint c = col_base + t; + sh_qa4[t][v] = (c < (uint)n_no_padding) + ? KQ_STAGE(src1_qa + c * k_u + (step >> 2) + (v << 2)) + : (uint4)(0u); + } + if (lid < TILESIZE_N) { + const uint c = col_base + lid; + sh_d[lid] = (c < (uint)n_no_padding) ? src1_da[c * k_b + sub] : (half)0; + } + barrier(CLK_LOCAL_MEM_FENCE); + +#define LD4(arr, b) ((float4)((float)arr[(b)+0], (float)arr[(b)+1], (float)arr[(b)+2], (float)arr[(b)+3])) + #pragma unroll + for (int g = 0; g < NGROUPS; ++g) { + const int b = g * 4; + float4 rf; +#define Q3_K_COL(T) (dl0 * (float)dot4_q8a_v(qlo, (uint4)(sh_qa4[T][0])) \ + + dl1 * (float)dot4_q8a_v(qhi, (uint4)(sh_qa4[T][1]))) + rf.s0 = Q3_K_COL(b+0); rf.s1 = Q3_K_COL(b+1); + rf.s2 = Q3_K_COL(b+2); rf.s3 = Q3_K_COL(b+3); +#undef Q3_K_COL + acc[g] += LD4(sh_d, b) * rf; + } +#undef LD4 + barrier(CLK_LOCAL_MEM_FENCE); + } + + if (!row_valid) { + return; + } + + #pragma unroll + for (int g = 0; g < NGROUPS; ++g) { + const uint b = (uint)(g * 4); + const float4 a = acc[g]; + const uint c0 = col_base + b; + if (c0 + 0 < (uint)n_no_padding) dst[(c0 + 0) * (uint)m + row] = a.s0; + if (c0 + 1 < (uint)n_no_padding) dst[(c0 + 1) * (uint)m + row] = a.s1; + if (c0 + 2 < (uint)n_no_padding) dst[(c0 + 2) * (uint)m + row] = a.s2; + if (c0 + 3 < (uint)n_no_padding) dst[(c0 + 3) * (uint)m + row] = a.s3; + } +#undef NGROUPS +} diff --git a/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q3_k_f32.cl b/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q3_k_f32.cl new file mode 100644 index 000000000..f44354154 --- /dev/null +++ b/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q3_k_f32.cl @@ -0,0 +1,249 @@ +#pragma OPENCL EXTENSION cl_khr_fp16 : enable + +#define QK_K 256 + +#ifndef Q3_K_MV_NSG +#define Q3_K_MV_NSG 8 +#endif + +#ifndef Q3_K_MV_R +#define Q3_K_MV_R 2 +#endif + +inline int q3_k_scale(uint sc0, uint sc1, uint sc2, uint is) { + const uint w = is >> 2; + const uint sh = 8u * (is & 3u); + const uint tb = (sc2 >> sh) & 0xFFu; + uint v; + if (w == 0u) { v = ( (sc0 >> sh) & 0xFu) | (((tb >> 0) & 3u) << 4); } + else if (w == 1u) { v = ( (sc1 >> sh) & 0xFu) | (((tb >> 2) & 3u) << 4); } + else if (w == 2u) { v = (((sc0 >> sh) >> 4) & 0xFu) | (((tb >> 4) & 3u) << 4); } + else { v = (((sc1 >> sh) >> 4) & 0xFu) | (((tb >> 6) & 3u) << 4); } + return (int)v - 32; +} + +inline float4 q3_k_vals(uint pk, uint hb) { + float4 v; + v.s0 = (float)((int)((pk ) & 3u) - 4 + 4*(int)((hb ) & 1u)); + v.s1 = (float)((int)((pk >> 2) & 3u) - 4 + 4*(int)((hb >> 1) & 1u)); + v.s2 = (float)((int)((pk >> 4) & 3u) - 4 + 4*(int)((hb >> 2) & 1u)); + v.s3 = (float)((int)((pk >> 6) & 3u) - 4 + 4*(int)((hb >> 3) & 1u)); + return v; +} + +kernel void kernel_gemv_noshuffle_q3_k_f32( + global const uchar * src0_qs, + global const uchar * src0_hm, + global const uint * src0_sc, + global const half * src0_d, + global const float * src1, + ulong offset1, + global float * dst, + ulong offsetd, + int ne00, // K + int ne01, // M + int ne10, // activation row stride + int ne0 // dst row stride +) { + src1 = (global const float *)((global const char *)src1 + offset1); + dst = (global float *)((global char *)dst + offsetd); + + const uint m = (uint)ne01; + const uint K = (uint)ne00; + const uint nsb = K / QK_K; + + const uint lid = get_local_id(0); + const uint sgi = get_local_id(1); + const uint col = get_group_id(1); + + global const float * y = src1 + (ulong)col * (uint)ne10; + +#if Q3_K_MV_R == 4 + const uint mq = m >> 2; + const uint j = get_group_id(0) * 64u + lid; + const uint row = j << 2; + + float sumf = 0.f, sumf1 = 0.f, sumf2 = 0.f, sumf3 = 0.f; + + if (j < mq) { + global const uint * qsw = (global const uint *)src0_qs; + global const uint * hmw = (global const uint *)src0_hm; + + for (uint ib = sgi; ib < nsb; ib += Q3_K_MV_NSG) { + const half4 dh = vload4(j + ib * mq, src0_d); + const uint4 s0 = vload4(j + (3u * ib + 0u) * mq, src0_sc); + const uint4 s1 = vload4(j + (3u * ib + 1u) * mq, src0_sc); + const uint4 s2 = vload4(j + (3u * ib + 2u) * mq, src0_sc); + + float acc0 = 0.f, acc1 = 0.f, acc2 = 0.f, acc3 = 0.f; + for (uint sb = 0; sb < 8u; ++sb) { + const uint grp = ib * 64u + sb * 8u; + const uint qsb = j + grp * mq; + const uint hmb = j + (grp >> 1) * mq; + + for (uint h = 0; h < 2u; ++h) { // two 16-weight halves + const int l0 = q3_k_scale(s0.s0, s1.s0, s2.s0, 2u*sb + h); + const int l1 = q3_k_scale(s0.s1, s1.s1, s2.s1, 2u*sb + h); + const int l2 = q3_k_scale(s0.s2, s1.s2, s2.s2, 2u*sb + h); + const int l3 = q3_k_scale(s0.s3, s1.s3, s2.s3, 2u*sb + h); + + float a0 = 0.f, a1 = 0.f, a2 = 0.f, a3 = 0.f; + for (uint u = 0; u < 4u; ++u) { + const uint gg = 4u*h + u; + const uint qsv = qsw[qsb + gg * mq]; // four rows, one load + const uint hmv = hmw[hmb + (gg >> 1) * mq]; + const uint hsh = 4u * (gg & 1u); + const float4 yv = vload4(grp + gg, y); // shared by all four + a0 += dot(yv, q3_k_vals((qsv ) & 0xFFu, ((hmv ) >> hsh) & 0xFu)); + a1 += dot(yv, q3_k_vals((qsv >> 8) & 0xFFu, ((hmv >> 8) >> hsh) & 0xFu)); + a2 += dot(yv, q3_k_vals((qsv >> 16) & 0xFFu, ((hmv >> 16) >> hsh) & 0xFu)); + a3 += dot(yv, q3_k_vals((qsv >> 24) & 0xFFu, ((hmv >> 24) >> hsh) & 0xFu)); + } + acc0 += (float)l0 * a0; + acc1 += (float)l1 * a1; + acc2 += (float)l2 * a2; + acc3 += (float)l3 * a3; + } + } + sumf += (float)dh.s0 * acc0; + sumf1 += (float)dh.s1 * acc1; + sumf2 += (float)dh.s2 * acc2; + sumf3 += (float)dh.s3 * acc3; + } + } +#elif Q3_K_MV_R == 2 + const uint mh = m >> 1; + const uint j = get_group_id(0) * 64u + lid; + const uint row = j << 1; + + float sumf = 0.f; + float sumf1 = 0.f; + + if (j < mh) { + global const ushort * qsu = (global const ushort *)src0_qs; + global const ushort * hmu = (global const ushort *)src0_hm; + + for (uint ib = sgi; ib < nsb; ib += Q3_K_MV_NSG) { + const half2 dh = vload2(j + ib * mh, src0_d); + const uint2 s0 = vload2(j + (3u * ib + 0u) * mh, src0_sc); + const uint2 s1 = vload2(j + (3u * ib + 1u) * mh, src0_sc); + const uint2 s2 = vload2(j + (3u * ib + 2u) * mh, src0_sc); + + float acc0 = 0.f, acc1 = 0.f; + for (uint sb = 0; sb < 8u; ++sb) { + const uint grp = ib * 64u + sb * 8u; + const uint qsb = j + grp * mh; + const uint hmb = j + (grp >> 1) * mh; + + for (uint h = 0; h < 2u; ++h) { // two 16-weight halves + const int l0 = q3_k_scale(s0.s0, s1.s0, s2.s0, 2u*sb + h); + const int l1 = q3_k_scale(s0.s1, s1.s1, s2.s1, 2u*sb + h); + + float a0 = 0.f, a1 = 0.f; + for (uint u = 0; u < 4u; ++u) { + const uint gg = 4u*h + u; + const uint qsv = (uint)qsu[qsb + gg * mh]; + const uint hmv = (uint)hmu[hmb + (gg >> 1) * mh]; + const uint hsh = 4u * (gg & 1u); + const float4 yv = vload4(grp + gg, y); + a0 += dot(yv, q3_k_vals( qsv & 0xFFu, ( hmv >> hsh) & 0xFu)); + a1 += dot(yv, q3_k_vals((qsv >> 8) & 0xFFu, ((hmv >> 8) >> hsh) & 0xFu)); + } + acc0 += (float)l0 * a0; + acc1 += (float)l1 * a1; + } + } + sumf += (float)dh.s0 * acc0; + sumf1 += (float)dh.s1 * acc1; + } + } +#else + const uint row = get_group_id(0) * 64u + lid; + + float sumf = 0.f; + + if (row < m) { + for (uint ib = sgi; ib < nsb; ib += Q3_K_MV_NSG) { + const float d = (float)src0_d[row + ib * m]; + const uint s0 = src0_sc[row + (3u * ib + 0u) * m]; + const uint s1 = src0_sc[row + (3u * ib + 1u) * m]; + const uint s2 = src0_sc[row + (3u * ib + 2u) * m]; + + float acc = 0.f; + for (uint sb = 0; sb < 8u; ++sb) { + const uint grp = ib * 64u + sb * 8u; + const uint qsb = row + grp * m; + const uint hmb = row + (grp >> 1) * m; + + for (uint h = 0; h < 2u; ++h) { + const int l = q3_k_scale(s0, s1, s2, 2u*sb + h); + float a = 0.f; + for (uint u = 0; u < 4u; ++u) { + const uint gg = 4u*h + u; + const uint qsv = (uint)src0_qs[qsb + gg * m]; + const uint hmv = (uint)src0_hm[hmb + (gg >> 1) * m]; + const float4 yv = vload4(grp + gg, y); + a += dot(yv, q3_k_vals(qsv, (hmv >> (4u * (gg & 1u))) & 0xFu)); + } + acc += (float)l * a; + } + } + sumf += d * acc; + } + } +#endif + +#if Q3_K_MV_NSG > 1 +#if Q3_K_MV_R == 4 + __local float4 part[Q3_K_MV_NSG][64]; + part[sgi][lid] = (float4)(sumf, sumf1, sumf2, sumf3); + barrier(CLK_LOCAL_MEM_FENCE); + if (sgi != 0) { + return; + } + for (uint s = 1; s < Q3_K_MV_NSG; ++s) { + const float4 p = part[s][lid]; + sumf += p.s0; + sumf1 += p.s1; + sumf2 += p.s2; + sumf3 += p.s3; + } +#elif Q3_K_MV_R == 2 + __local float2 part[Q3_K_MV_NSG][64]; + part[sgi][lid] = (float2)(sumf, sumf1); + barrier(CLK_LOCAL_MEM_FENCE); + if (sgi != 0) { + return; + } + for (uint s = 1; s < Q3_K_MV_NSG; ++s) { + const float2 p = part[s][lid]; + sumf += p.s0; + sumf1 += p.s1; + } +#else + __local float part[Q3_K_MV_NSG][64]; + part[sgi][lid] = sumf; + barrier(CLK_LOCAL_MEM_FENCE); + if (sgi != 0) { + return; + } + for (uint s = 1; s < Q3_K_MV_NSG; ++s) { + sumf += part[s][lid]; + } +#endif +#endif + +#if Q3_K_MV_R == 4 + if (j < mq) { + vstore4((float4)(sumf, sumf1, sumf2, sumf3), 0, dst + (ulong)col * (uint)ne0 + row); + } +#elif Q3_K_MV_R == 2 + if (j < mh) { + vstore2((float2)(sumf, sumf1), 0, dst + (ulong)col * (uint)ne0 + row); + } +#else + if (row < m) { + dst[(ulong)col * (uint)ne0 + row] = sumf; + } +#endif +} diff --git a/ggml/src/ggml-opencl/kernels/mul_mm_q3_k_f32_l4_lm.cl b/ggml/src/ggml-opencl/kernels/mul_mm_q3_k_f32_l4_lm.cl new file mode 100644 index 000000000..da3997417 --- /dev/null +++ b/ggml/src/ggml-opencl/kernels/mul_mm_q3_k_f32_l4_lm.cl @@ -0,0 +1,219 @@ +#pragma OPENCL EXTENSION cl_khr_fp16 : enable + +#ifdef cl_intel_required_subgroup_size +#define INTEL_GPU 1 +#endif + +#define QK_K 256 + +struct block_q3_K { + uchar hmask[QK_K/8]; + uchar qs[QK_K/4]; + uchar scales[12]; + half d; +}; + +#define LOAD_VEC_A 4 +#define LOAD_VEC_B 4 + +#define BM 64 +#define BN 64 +#ifndef BK +#define BK 32 +#endif +#ifdef INTEL_GPU +#define TM 8 +#else +#define TM 4 +#endif +#define TN 8 + +#ifndef LM_HALF +#define LM_HALF 0 +#endif +#if LM_HALF +typedef half lm_st; +#define LM_LD4(p, i) convert_float4(vload4((i), (p))) +#else +typedef float lm_st; +#define LM_LD4(p, i) vload4((i), (p)) +#endif + +kernel void kernel_mul_mm_q3_k_f32_l4_lm( + global uchar * src0_qs, + global uchar * src0_hm, + global uchar * src0_sc, + global half * src0_d, + ulong offset0, + global float4 * src1, + ulong offset1, + global float * dst, + ulong offsetd, + + int ne00, + int ne01, + int ne02, + int ne11, + int ne12, + + int stride_a, + int stride_b, + int stride_d, + + int batch_stride_a, + int batch_stride_b, + int batch_stride_d, + + int r2, + int r3 +) { + src1 = (global float4*)((global char*)src1 + offset1); + dst = (global float *)((global char*)dst + offsetd); + + local lm_st buf_a[BM * BK]; + local lm_st buf_b[BN * BK]; + + const int batch_idx = get_global_id(2); + + const int i13 = batch_idx / ne12; + const int i12 = batch_idx % ne12; + + const int i03 = i13 / r3; + const int i02 = i12 / r2; + + const int batch_idx_a = i03 * ne02 + i02; + + const int ir = get_group_id(0); + const int ic = get_group_id(1); + + const int tid = get_local_id(0); + const int th_r = tid % (BM / TM); + const int th_c = tid / (BM / TM); + + const int loadr_a = get_local_id(0) % (BK / LOAD_VEC_A); + const int loadc_a = get_local_id(0) / (BK / LOAD_VEC_A); + const int loadr_b = get_local_id(0) % (BK / LOAD_VEC_B); + const int loadc_b = get_local_id(0) / (BK / LOAD_VEC_B); + + const int loadstride_a = get_local_size(0) * LOAD_VEC_A / BK; + const int loadstride_b = get_local_size(0) * LOAD_VEC_B / BK; + + // pos_a counts elements, not blocks - one q3_K super block holds QK_K of them + int pos_a = batch_idx_a * batch_stride_a + ir * BM * stride_a; + int pos_b = (batch_idx * batch_stride_b + ic * BN * stride_b) / LOAD_VEC_B; + + float4 sums4[(TM/4) * TN]; + float4 cache_a4[TM/4]; + + for (int i = 0; i < (TM/4) * TN; i++) { + sums4[i] = (float4)(0.0f); + } + + for (int block = 0; block < ne00; block += BK) { + for (int l = 0; l < BM; l += loadstride_a) { + if (ir*BM + loadc_a + l < ne01) { + int idx = pos_a + (loadc_a + l) * stride_a + loadr_a * LOAD_VEC_A; + int ib = idx / QK_K; + ib += offset0/sizeof(struct block_q3_K); + int e = idx % QK_K; + + // Only qs advances in the second half of a super-block. + int n = e >> 7; + int rem = e & 127; + int j = rem >> 5; + int s = (rem >> 4) & 1; + int l16 = rem & 15; + int is = 8*n + 2*j + s; + int shift = 2*j; + uchar m = (uchar)(1u << (4*n + j)); + + global ushort * a = (global ushort *)(src0_sc + 12*ib); + uint a0 = (uint)a[0] | ((uint)a[1] << 16); + uint a1 = (uint)a[2] | ((uint)a[3] << 16); + uint tmp = (uint)a[4] | ((uint)a[5] << 16); + + uint g = is >> 2; + uint w = (g & 1u) ? a1 : a0; + w = ((w >> ((g >> 1)*4)) & 0x0f0f0f0fu) | (((tmp >> (2*g)) & 0x03030303u) << 4); + + uint sc6 = (w >> (8*(is & 3))) & 0xFFu; + float dl = (float)src0_d[ib] * ((float)sc6 - 32.0f); + + uchar4 q = vload4(0, src0_qs + 64*ib + 32*n + 16*s + l16); + uchar4 hb = vload4(0, src0_hm + 32*ib + 16*s + l16); + + float4 qv = convert_float4((uchar4)((q.s0 >> shift) & 3, + (q.s1 >> shift) & 3, + (q.s2 >> shift) & 3, + (q.s3 >> shift) & 3)); + float4 hv = (float4)((hb.s0 & m) ? 0.f : 4.f, + (hb.s1 & m) ? 0.f : 4.f, + (hb.s2 & m) ? 0.f : 4.f, + (hb.s3 & m) ? 0.f : 4.f); + float4 v1 = (qv - hv) * dl; + + buf_a[(loadr_a * LOAD_VEC_A + 0) * BM + loadc_a + l] = v1.s0; + buf_a[(loadr_a * LOAD_VEC_A + 1) * BM + loadc_a + l] = v1.s1; + buf_a[(loadr_a * LOAD_VEC_A + 2) * BM + loadc_a + l] = v1.s2; + buf_a[(loadr_a * LOAD_VEC_A + 3) * BM + loadc_a + l] = v1.s3; + } else { + buf_a[(loadr_a * LOAD_VEC_A + 0) * BM + loadc_a + l] = 0.0f; + buf_a[(loadr_a * LOAD_VEC_A + 1) * BM + loadc_a + l] = 0.0f; + buf_a[(loadr_a * LOAD_VEC_A + 2) * BM + loadc_a + l] = 0.0f; + buf_a[(loadr_a * LOAD_VEC_A + 3) * BM + loadc_a + l] = 0.0f; + } + } + + for (int l = 0; l < BN; l += loadstride_b) { + if (ic*BN + loadc_b + l < ne11) { + int idx = pos_b + (loadc_b + l) * stride_b / LOAD_VEC_B + loadr_b; + buf_b[(loadr_b * LOAD_VEC_B + 0) * BN + loadc_b + l] = src1[idx].s0; + buf_b[(loadr_b * LOAD_VEC_B + 1) * BN + loadc_b + l] = src1[idx].s1; + buf_b[(loadr_b * LOAD_VEC_B + 2) * BN + loadc_b + l] = src1[idx].s2; + buf_b[(loadr_b * LOAD_VEC_B + 3) * BN + loadc_b + l] = src1[idx].s3; + } else { + buf_b[(loadr_b * LOAD_VEC_B + 0) * BN + loadc_b + l] = 0.0f; + buf_b[(loadr_b * LOAD_VEC_B + 1) * BN + loadc_b + l] = 0.0f; + buf_b[(loadr_b * LOAD_VEC_B + 2) * BN + loadc_b + l] = 0.0f; + buf_b[(loadr_b * LOAD_VEC_B + 3) * BN + loadc_b + l] = 0.0f; + } + } + + barrier(CLK_LOCAL_MEM_FENCE); + + pos_a += BK; + pos_b += BK / LOAD_VEC_B; + + for (int i = 0; i < BK; i++) { + for (int a = 0; a < TM/4; a++) { + cache_a4[a] = LM_LD4(buf_a + (i) * BM + th_r * TM, a); + } + + for (int cc = 0; cc < TN; cc++) { + const float cache_b = buf_b[(i) * BN + th_c * TN + cc]; + for (int a = 0; a < TM/4; a++) { + const int sums_idx = cc*(TM/4) + a; + sums4[sums_idx] = mad(cache_a4[a], (float4)cache_b, sums4[sums_idx]); + } + } + } + barrier(CLK_LOCAL_MEM_FENCE); + } + + const int dr = ir * BM + th_r * TM; + const int dc = ic * BN + th_c * TN; + + const ulong offsets = batch_idx * batch_stride_d; + + for (int cc = 0; cc < TN; cc++) { + for (int a = 0; a < TM/4; a++) { + const float4 v = sums4[cc * (TM/4) + a]; + const float vs[4] = { v.s0, v.s1, v.s2, v.s3 }; + for (int k = 0; k < 4; k++) { + if (dr + 4*a + k < ne01 && dc + cc < ne11) { + dst[offsets + (dc + cc) * stride_d + dr + 4*a + k] = vs[k]; + } + } + } + } +} diff --git a/ggml/src/ggml-opencl/kernels/mul_mv_q3_k_f32.cl b/ggml/src/ggml-opencl/kernels/mul_mv_q3_k_f32.cl new file mode 100644 index 000000000..77c70f4ac --- /dev/null +++ b/ggml/src/ggml-opencl/kernels/mul_mv_q3_k_f32.cl @@ -0,0 +1,197 @@ +#pragma OPENCL EXTENSION cl_khr_fp16 : enable + +#ifdef cl_intel_required_subgroup_size +#pragma OPENCL EXTENSION cl_intel_required_subgroup_size : enable +#define INTEL_GPU 1 +#define REQD_SUBGROUP_SIZE_16 __attribute__((intel_reqd_sub_group_size(16))) +#define REQD_SUBGROUP_SIZE_32 __attribute__((intel_reqd_sub_group_size(32))) +#elif defined(cl_qcom_reqd_sub_group_size) +#pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable +#define ADRENO_GPU 1 +#define REQD_SUBGROUP_SIZE_64 __attribute__((qcom_reqd_sub_group_size("half"))) +#define REQD_SUBGROUP_SIZE_128 __attribute__((qcom_reqd_sub_group_size("full"))) +#endif + +//------------------------------------------------------------------------------ +// block_q3_K +//------------------------------------------------------------------------------ +#define QK_K 256 + +// 16 blocks of 16 elements each +// weight is represented as x = a * q +typedef struct { + uchar hmask[QK_K/8]; // quants - high bit + uchar qs[QK_K/4]; // quants - low 2 bits + uchar scales[12]; // scales, quantized with 6 bits + half d; // super-block scale +} block_q3_K; + +#undef N_DST +#undef N_SIMDGROUP +#undef N_SIMDWIDTH + +#ifdef INTEL_GPU +#define N_DST 2 // number of rows each SIMD group works on +#define N_SIMDGROUP 1 // number of SIMD groups in a thread group +#define N_SIMDWIDTH 16 // SIMD group size +#elif defined (ADRENO_GPU) +#define N_DST 2 +#define N_SIMDGROUP 1 +#define N_SIMDWIDTH 64 +#endif + +#undef BLOCK_STRIDE +// 8 threads cover one super block, so a wave covers N_SIMDWIDTH/8 of them +#define BLOCK_STRIDE (N_SIMDWIDTH/8) + +#ifdef INTEL_GPU +REQD_SUBGROUP_SIZE_16 +#elif defined (ADRENO_GPU) +REQD_SUBGROUP_SIZE_64 +#endif +kernel void kernel_mul_mv_q3_K_f32( + global char * src0, + ulong offset0, + global char * src1, + ulong offset1, + global char * dst, + ulong offsetd, + int ne00, + int ne01, + ulong nb01, + ulong nb02, + ulong nb03, + int ne12, + ulong nb11, + ulong nb12, + ulong nb13, + int ne0, + int ne1, + int r2, + int r3 +) { + src0 = src0 + offset0; + src1 = src1 + offset1; + dst = dst + offsetd; + + int nb = ne00/QK_K; + + int r0 = get_group_id(0); + int r1 = get_group_id(1); + int im = get_group_id(2); + int first_row = (r0 * N_SIMDGROUP + get_sub_group_id()) * N_DST; + + int i12 = im%ne12; + int i13 = im/ne12; + + ulong offset_src0 = first_row*nb01 + (i12/r2)*nb02 + (i13/r3)*nb03; + ulong offset_src1 = r1*nb11 + (i12 )*nb12 + (i13 )*nb13; + + global block_q3_K * x = (global block_q3_K *) (src0 + offset_src0); + global float * yy = (global float *) (src1 + offset_src1); + + float yl[32]; + + // Eight lanes cover one super-block. + int ix = get_sub_group_local_id() % BLOCK_STRIDE; + int tid = get_sub_group_local_id() / BLOCK_STRIDE; + + int ip = tid/4; // 0 or 1 + int il = 2*((tid%4)/2); // 0 or 2 + int ir = tid%2; + int l0 = 8*ir; + + // Shift the masks instead of indexing private arrays to avoid spills. + ushort4 hm = (ushort4)(0x0001, 0x0100, 0x0002, 0x0200) << (ushort4)(2*(2*ip + il/2)); + int4 qm = (int4) (0x0003, 0x0300, 0x000c, 0x0c00) << (int4) (4*(il/2)); + + int shift = 2*il; + + float v1 = il == 0 ? 4.f : 64.f; + float v2 = 4.f * v1; + + int s_shift1 = 4*ip; + int s_shift2 = s_shift1 + il; + + int q_offset = 32*ip + l0; + int y_offset = 128*ip + 32*il + l0; + + global float * y1 = yy + ix*QK_K + y_offset; + + float sumf1[N_DST] = {0.f}; + float sumf2[N_DST] = {0.f}; + float all_sum; + + for (int i = ix; i < nb; i += BLOCK_STRIDE) { + for (int l = 0; l < 8; ++l) { + yl[l+ 0] = y1[l+ 0]; + yl[l+ 8] = y1[l+16]; + yl[l+16] = y1[l+32]; + yl[l+24] = y1[l+48]; + } + + global ushort * q = (global ushort *)(x[i].qs + q_offset); + global ushort * h = (global ushort *)(x[i].hmask + l0); + global ushort * a = (global ushort *)(x[i].scales); + global half * dh = &x[i].d; + + // only ne01 output rows exist; reading past them can pull in an inf scale + for (int row = 0; row < N_DST && first_row + row < ne01; ++row) { + float d_all = dh[0]; + + uint s32 = (uint)a[4] | ((uint)a[5] << 16); + uint aux32 = ((s32 >> s_shift2) << 4) & 0x30303030u; + s32 = (uint)a[il+0] | ((uint)a[il+1] << 16); + s32 = ((s32 >> s_shift1) & 0x0f0f0f0fu) | aux32; + char4 sc = as_char4(s32); + + float s1 = 0, s2 = 0, s3 = 0, s4 = 0, s5 = 0, s6 = 0; + for (int l = 0; l < 8; l += 2) { + int qs = q[l/2]; + s1 += yl[l+0] * (qs & qm.s0); + s2 += yl[l+1] * (qs & qm.s1); + s3 += ((h[l/2] & hm.s0) ? 0.f : yl[l+0]) + ((h[l/2] & hm.s1) ? 0.f : yl[l+1]); + s4 += yl[l+16] * (qs & qm.s2); + s5 += yl[l+17] * (qs & qm.s3); + s6 += ((h[l/2] & hm.s2) ? 0.f : yl[l+16]) + ((h[l/2] & hm.s3) ? 0.f : yl[l+17]); + } + float d1 = d_all * (s1 + 1.f/256.f * s2 - s3*v1); + float d2 = d_all * (s4 + 1.f/256.f * s5 - s6*v2); + sumf1[row] += d1 * (sc.s0 - 32); + sumf2[row] += d2 * (sc.s2 - 32); + + s1 = s2 = s3 = s4 = s5 = s6 = 0; + for (int l = 0; l < 8; l += 2) { + int qs = q[l/2+8]; + s1 += yl[l+8] * (qs & qm.s0); + s2 += yl[l+9] * (qs & qm.s1); + s3 += ((h[l/2+8] & hm.s0) ? 0.f : yl[l+8]) + ((h[l/2+8] & hm.s1) ? 0.f : yl[l+9]); + s4 += yl[l+24] * (qs & qm.s2); + s5 += yl[l+25] * (qs & qm.s3); + s6 += ((h[l/2+8] & hm.s2) ? 0.f : yl[l+24]) + ((h[l/2+8] & hm.s3) ? 0.f : yl[l+25]); + } + d1 = d_all * (s1 + 1.f/256.f * s2 - s3*v1); + d2 = d_all * (s4 + 1.f/256.f * s5 - s6*v2); + sumf1[row] += d1 * (sc.s1 - 32); + sumf2[row] += d2 * (sc.s3 - 32); + + q += nb01/2; + h += nb01/2; + a += nb01/2; + dh += nb01/2; + } + + y1 += BLOCK_STRIDE * QK_K; + } + + global float * dst_f32 = (global float *) dst + im*ne0*ne1 + r1*ne0; + + for (int row = 0; row < N_DST; ++row) { + all_sum = sub_group_reduce_add((sumf1[row] + 0.25f * sumf2[row]) / (1 << shift)); + if (first_row + row < ne01) { + if (get_sub_group_local_id() == 0) { + dst_f32[first_row + row] = all_sum; + } + } + } +} diff --git a/ggml/src/ggml-opencl/kernels/mul_mv_q3_k_f32_flat.cl b/ggml/src/ggml-opencl/kernels/mul_mv_q3_k_f32_flat.cl new file mode 100644 index 000000000..c2c2219d8 --- /dev/null +++ b/ggml/src/ggml-opencl/kernels/mul_mv_q3_k_f32_flat.cl @@ -0,0 +1,199 @@ +#pragma OPENCL EXTENSION cl_khr_fp16 : enable + +#ifdef cl_intel_required_subgroup_size +#pragma OPENCL EXTENSION cl_intel_required_subgroup_size : enable +#define INTEL_GPU 1 +#define REQD_SUBGROUP_SIZE_16 __attribute__((intel_reqd_sub_group_size(16))) +#define REQD_SUBGROUP_SIZE_32 __attribute__((intel_reqd_sub_group_size(32))) +#elif defined(cl_qcom_reqd_sub_group_size) +#pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable +#define ADRENO_GPU 1 +#define REQD_SUBGROUP_SIZE_64 __attribute__((qcom_reqd_sub_group_size("half"))) +#define REQD_SUBGROUP_SIZE_128 __attribute__((qcom_reqd_sub_group_size("full"))) +#endif + +//------------------------------------------------------------------------------ +// block_q3_K +//------------------------------------------------------------------------------ +#define QK_K 256 + +// 16 blocks of 16 elements each +// weight is represented as x = a * q +typedef struct { + uchar hmask[QK_K/8]; // quants - high bit + uchar qs[QK_K/4]; // quants - low 2 bits + uchar scales[12]; // scales, quantized with 6 bits + half d; // super-block scale +} block_q3_K; + +#undef N_DST +#undef N_SIMDGROUP +#undef N_SIMDWIDTH + +#ifdef INTEL_GPU +#define N_DST 2 // number of rows each SIMD group works on +#define N_SIMDGROUP 1 // number of SIMD groups in a thread group +#define N_SIMDWIDTH 16 // SIMD group size +#elif defined (ADRENO_GPU) +#define N_DST 2 +#define N_SIMDGROUP 1 +#define N_SIMDWIDTH 64 +#endif + +#undef BLOCK_STRIDE +// 8 threads cover one super block, so a wave covers N_SIMDWIDTH/8 of them +#define BLOCK_STRIDE (N_SIMDWIDTH/8) + +#ifdef INTEL_GPU +REQD_SUBGROUP_SIZE_16 +#elif defined (ADRENO_GPU) +REQD_SUBGROUP_SIZE_64 +#endif +kernel void kernel_mul_mv_q3_K_f32_flat( + global uchar * src0_qs, + global uchar * src0_hm, + global uchar * src0_sc, + global half * src0_d, + ulong offset0, + global char * src1, + ulong offset1, + global char * dst, + ulong offsetd, + int ne00, + int ne01, + ulong nb01, + ulong nb02, + ulong nb03, + int ne12, + ulong nb11, + ulong nb12, + ulong nb13, + int ne0, + int ne1, + int r2, + int r3 +) { + src1 = src1 + offset1; + dst = dst + offsetd; + + int nb = ne00/QK_K; + + int r0 = get_group_id(0); + int r1 = get_group_id(1); + int im = get_group_id(2); + int first_row = (r0 * N_SIMDGROUP + get_sub_group_id()) * N_DST; + + int i12 = im%ne12; + int i13 = im/ne12; + + ulong offset_src0 = (offset0 + first_row*nb01 + (i12/r2)*nb02 + (i13/r3)*nb03)/sizeof(block_q3_K); + ulong row_stride = nb01/sizeof(block_q3_K); + ulong offset_src1 = r1*nb11 + (i12 )*nb12 + (i13 )*nb13; + + global float * yy = (global float *) (src1 + offset_src1); + + float yl[32]; + + // Eight lanes cover one super-block. + int ix = get_sub_group_local_id() % BLOCK_STRIDE; + int tid = get_sub_group_local_id() / BLOCK_STRIDE; + + int ip = tid/4; // 0 or 1 + int il = 2*((tid%4)/2); // 0 or 2 + int ir = tid%2; + int l0 = 8*ir; + + // Shift the masks instead of indexing private arrays to avoid spills. + ushort4 hm = (ushort4)(0x0001, 0x0100, 0x0002, 0x0200) << (ushort4)(2*(2*ip + il/2)); + int4 qm = (int4) (0x0003, 0x0300, 0x000c, 0x0c00) << (int4) (4*(il/2)); + + int shift = 2*il; + + float v1 = il == 0 ? 4.f : 64.f; + float v2 = 4.f * v1; + + int s_shift1 = 4*ip; + int s_shift2 = s_shift1 + il; + + int q_offset = 32*ip + l0; + int y_offset = 128*ip + 32*il + l0; + + global float * y1 = yy + ix*QK_K + y_offset; + + float sumf1[N_DST] = {0.f}; + float sumf2[N_DST] = {0.f}; + float all_sum; + + for (int i = ix; i < nb; i += BLOCK_STRIDE) { + for (int l = 0; l < 8; ++l) { + yl[l+ 0] = y1[l+ 0]; + yl[l+ 8] = y1[l+16]; + yl[l+16] = y1[l+32]; + yl[l+24] = y1[l+48]; + } + + global ushort * q = (global ushort *)(src0_qs + 64*(offset_src0 + i) + q_offset); + global ushort * h = (global ushort *)(src0_hm + 32*(offset_src0 + i) + l0); + global ushort * a = (global ushort *)(src0_sc + 12*(offset_src0 + i)); + global half * dh = src0_d + offset_src0 + i; + + // only ne01 output rows exist; reading past them can pull in an inf scale + for (int row = 0; row < N_DST && first_row + row < ne01; ++row) { + float d_all = dh[0]; + + uint s32 = (uint)a[4] | ((uint)a[5] << 16); + uint aux32 = ((s32 >> s_shift2) << 4) & 0x30303030u; + s32 = (uint)a[il+0] | ((uint)a[il+1] << 16); + s32 = ((s32 >> s_shift1) & 0x0f0f0f0fu) | aux32; + char4 sc = as_char4(s32); + + float s1 = 0, s2 = 0, s3 = 0, s4 = 0, s5 = 0, s6 = 0; + for (int l = 0; l < 8; l += 2) { + int qs = q[l/2]; + s1 += yl[l+0] * (qs & qm.s0); + s2 += yl[l+1] * (qs & qm.s1); + s3 += ((h[l/2] & hm.s0) ? 0.f : yl[l+0]) + ((h[l/2] & hm.s1) ? 0.f : yl[l+1]); + s4 += yl[l+16] * (qs & qm.s2); + s5 += yl[l+17] * (qs & qm.s3); + s6 += ((h[l/2] & hm.s2) ? 0.f : yl[l+16]) + ((h[l/2] & hm.s3) ? 0.f : yl[l+17]); + } + float d1 = d_all * (s1 + 1.f/256.f * s2 - s3*v1); + float d2 = d_all * (s4 + 1.f/256.f * s5 - s6*v2); + sumf1[row] += d1 * (sc.s0 - 32); + sumf2[row] += d2 * (sc.s2 - 32); + + s1 = s2 = s3 = s4 = s5 = s6 = 0; + for (int l = 0; l < 8; l += 2) { + int qs = q[l/2+8]; + s1 += yl[l+8] * (qs & qm.s0); + s2 += yl[l+9] * (qs & qm.s1); + s3 += ((h[l/2+8] & hm.s0) ? 0.f : yl[l+8]) + ((h[l/2+8] & hm.s1) ? 0.f : yl[l+9]); + s4 += yl[l+24] * (qs & qm.s2); + s5 += yl[l+25] * (qs & qm.s3); + s6 += ((h[l/2+8] & hm.s2) ? 0.f : yl[l+24]) + ((h[l/2+8] & hm.s3) ? 0.f : yl[l+25]); + } + d1 = d_all * (s1 + 1.f/256.f * s2 - s3*v1); + d2 = d_all * (s4 + 1.f/256.f * s5 - s6*v2); + sumf1[row] += d1 * (sc.s1 - 32); + sumf2[row] += d2 * (sc.s3 - 32); + + q += row_stride*32; + h += row_stride*16; + a += row_stride*6; + dh += row_stride*1; + } + + y1 += BLOCK_STRIDE * QK_K; + } + + global float * dst_f32 = (global float *) dst + im*ne0*ne1 + r1*ne0; + + for (int row = 0; row < N_DST; ++row) { + all_sum = sub_group_reduce_add((sumf1[row] + 0.25f * sumf2[row]) / (1 << shift)); + if (first_row + row < ne01) { + if (get_sub_group_local_id() == 0) { + dst_f32[first_row + row] = all_sum; + } + } + } +}