From 6d3568e31d4e4b1e4db4acb8f6ceaad0084932f3 Mon Sep 17 00:00:00 2001 From: shaofeiqi Date: Sat, 14 Nov 2026 21:49:26 -0800 Subject: [PATCH] opencl: add A7 Q6_K non-MoE binary kernel (#30949) --- ggml/src/ggml-opencl/ggml-opencl.cpp | 85 +++++++++++++++++++++++++++- 1 file changed, 84 insertions(+), 1 deletion(-) diff --git a/ggml/src/ggml-opencl/ggml-opencl.cpp b/ggml/src/ggml-opencl/ggml-opencl.cpp index 10165b403..295ec496d 100644 --- a/ggml/src/ggml-opencl/ggml-opencl.cpp +++ b/ggml/src/ggml-opencl/ggml-opencl.cpp @@ -1460,6 +1460,7 @@ struct ggml_backend_opencl_context { cl_kernel kernel_gemm_noshuffle_q6_K_f32; cl_kernel kernel_gemm_noshuffle_q6_K_f32_cok; cl_kernel kernel_gemm_noshuffle_q6_k_f32_32b_trans_ila_a8_bin; + cl_kernel kernel_gemm_noshuffle_q6_k_f32_32b_trans_ila_a7_bin; cl_kernel kernel_gemm_noshuffle_q6_k_q8_1_dp4a_ila_a8_bin; cl_kernel kernel_gemv_noshuffle_q6_k_f32_32b_trans; cl_kernel kernel_gemv_noshuffle_q5_k_f32; @@ -6749,6 +6750,7 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) { backend_ctx->kernel_gemv_noshuffle_q6_k_f32_32b_trans = nullptr; backend_ctx->kernel_gemm_noshuffle_q6_k_f32_32b_trans_ila_a8_bin = nullptr; + backend_ctx->kernel_gemm_noshuffle_q6_k_f32_32b_trans_ila_a7_bin = nullptr; backend_ctx->kernel_gemm_noshuffle_q6_k_q8_1_dp4a_ila_a8_bin = nullptr; { if (backend_ctx->has_vector_subgroup_broadcast) { @@ -6793,6 +6795,17 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) { CL_CHECK(clReleaseProgram(bin_prog)); GGML_LOG_CONT("."); } + + kernel_bin = (const char *)backend_ctx->get_adreno_bin_kernel("gemm_noshuffle_q6_k_f32_32b_trans_ila_a7", &bin_size); + if (kernel_bin && bin_size > 0) { + cl_program bin_prog = + build_program_from_binary(backend_ctx->context, backend_ctx->device, kernel_bin, "", bin_size); + + CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q6_k_f32_32b_trans_ila_a7_bin = + clCreateKernel(bin_prog, "kernel_gemm_noshuffle_q6_k_f32_32b_trans_ila_a7", &err), err)); + CL_CHECK(clReleaseProgram(bin_prog)); + GGML_LOG_CONT("."); + } } } @@ -11744,7 +11757,8 @@ static inline bool use_flat_gemv_for_large_m_q6_K(const ggml_backend_opencl_cont inline bool use_q6_k_bin_kernels(const ggml_backend_opencl_context *backend_ctx, const ggml_tensor *tensor) { #ifdef GGML_OPENCL_USE_ADRENO_KERNELS if (!backend_ctx->kernel_gemv_noshuffle_q6_k_f32_32b_trans || - !backend_ctx->kernel_gemm_noshuffle_q6_k_f32_32b_trans_ila_a8_bin) { + !(backend_ctx->kernel_gemm_noshuffle_q6_k_f32_32b_trans_ila_a8_bin || + backend_ctx->kernel_gemm_noshuffle_q6_k_f32_32b_trans_ila_a7_bin)) { return false; } // bin kernels require ne2 == 1 and ne3 == 1 for weights @@ -28505,6 +28519,8 @@ static void ggml_cl_mul_mat_q6_K_f32_adreno_ila(ggml_backend_t backend, const gg const int gemm_tile_n = 64; int N_pad = CEIL_DIV(N, gemm_tile_n) * gemm_tile_n; + const bool have_a8 = backend_ctx->kernel_gemm_noshuffle_q6_k_f32_32b_trans_ila_a8_bin != nullptr; + static const char * q6_k_bin_dp4a_env = getenv("GGML_OPENCL_Q6_K_BIN_DP4A"); bool q6_k_bin_dp4a_on = q6_k_bin_dp4a_env ? (atoi(q6_k_bin_dp4a_env) != 0) @@ -28573,6 +28589,73 @@ static void ggml_cl_mul_mat_q6_K_f32_adreno_ila(ggml_backend_t backend, const gg return; } + if (!have_a8) { + cl_mem b_src_sub = nullptr; + cl_mem b_pad_sub = nullptr; + cl_mem b_buf = nullptr; + region.origin = offset1; + region.size = (size_t)K * N * sizeof(float); + CL_CHECK((b_src_sub = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); + if (N_pad == N) { + b_buf = b_src_sub; + } else { + size_t padded_bytes = (size_t)K * N_pad * sizeof(float); + backend_ctx->prealloc_act_trans.allocate(context, padded_bytes); + region.origin = 0; + region.size = padded_bytes; + CL_CHECK((b_pad_sub = clCreateSubBuffer(backend_ctx->prealloc_act_trans.buffer, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); + const float b_pad_zero = 0.0f; + CL_CHECK(clEnqueueFillBuffer(backend_ctx->queue, b_pad_sub, &b_pad_zero, sizeof(float), + 0, padded_bytes, 0, NULL, NULL)); + CL_CHECK(clEnqueueCopyBuffer(backend_ctx->queue, b_src_sub, b_pad_sub, 0, 0, (size_t)K * N * sizeof(float), 0, NULL, NULL)); + b_buf = b_pad_sub; + } + + img_fmt = { CL_R, CL_FLOAT }; + memset(&img_desc, 0, sizeof(img_desc)); + img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; + img_desc.image_width = (size_t)K * N_pad; + img_desc.buffer = b_buf; + cl_mem b_img_a7; + CL_CHECK((b_img_a7 = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, NULL, &err), err)); + + const int c_offset_for_kernel = (int)offsetd; + + int line_stride_B = K * (int)sizeof(float); + int line_stride_C = M * (int)sizeof(float); + + kernel = backend_ctx->kernel_gemm_noshuffle_q6_k_f32_32b_trans_ila_a7_bin; + cl_uint k_arg = 0; + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &extra0_q6_K->ql_img)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &extra0_q6_K->d)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &extra0_q6_K->s)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &extra0_q6_K->qh)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &b_img_a7)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &offset1)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &c_offset_for_kernel)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_uint), &ne00)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_uint), &ne01)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &line_stride_B)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &line_stride_C)); + CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &N)); + + size_t local_work_size[3] = { 64, 2, 4 }; + size_t m_tiles = (size_t)CEIL_DIV(M, 64); + size_t global_y = ((m_tiles + 1) / 2) * 2; + size_t global_work_size[3] = { 64, global_y, (size_t)CEIL_DIV(N_pad, 32) }; + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); + + CL_CHECK(clReleaseMemObject(b_img_a7)); + if (b_src_sub) { + CL_CHECK(clReleaseMemObject(b_src_sub)); + } + if (b_pad_sub) { + CL_CHECK(clReleaseMemObject(b_pad_sub)); + } + return; + } + cl_mem b_sub_buf = nullptr; cl_mem b_padded = nullptr; cl_mem b_buf = nullptr;