diff --git a/ggml/src/ggml-opencl/CMakeLists.txt b/ggml/src/ggml-opencl/CMakeLists.txt index 9ec3268b7..ff3ad7e34 100644 --- a/ggml/src/ggml-opencl/CMakeLists.txt +++ b/ggml/src/ggml-opencl/CMakeLists.txt @@ -114,7 +114,9 @@ set(GGML_OPENCL_KERNELS mul_mv_id_mxfp4_f32 mul_mv_id_mxfp4_f32_flat gemm_moe_q4_0_f32_ns + gemm_moe_q4_0_q8_1_dp4a gemv_moe_q4_0_f32_ns + gemm_moe_q8_0_f32_ns gemm_moe_q4_1_f32_ns gemv_moe_q4_1_f32_ns gemm_moe_q5_0_f32_ns @@ -122,6 +124,18 @@ set(GGML_OPENCL_KERNELS gemm_moe_q5_1_f32_ns gemv_moe_q5_1_f32_ns gemm_moe_q4_k_f32_ns + gemm_moe_q4_k_q8_1_dp4a + gemm_moe_q6_k_q8_1_dp4a + gemm_moe_q8_1_dp4a + moe_reorder_quant_a_q8_1 + gemm_noshuffle_q4_k_q8_1_dp4a + gemm_noshuffle_q5_k_q8_1_dp4a + gemm_noshuffle_q6_k_q8_1_dp4a + gemm_noshuffle_q8_0_q8_1_dp4a + gemm_noshuffle_q5_0_q8_1_dp4a + gemm_noshuffle_iq4_nl_q8_1_dp4a + gemm_noshuffle_q4_0_q8_1_dp4a + quant_a_q8_1 gemv_moe_q4_k_f32_ns gemm_moe_q5_k_f32_ns gemv_moe_q5_k_f32_ns @@ -130,8 +144,10 @@ set(GGML_OPENCL_KERNELS gemm_moe_mxfp4_f32 gemv_moe_mxfp4_f32 gemm_moe_mxfp4_f32_ns + gemm_moe_mxfp4_q8_1_dp4a gemv_moe_mxfp4_f32_ns moe_reorder_b + moe_combine moe_sort_by_expert mul_mm_f32_f32_l4_lm mul_mm_f16_f32_l4_lm diff --git a/ggml/src/ggml-opencl/ggml-opencl.cpp b/ggml/src/ggml-opencl/ggml-opencl.cpp index 5c96b9a9f..f283f6569 100644 --- a/ggml/src/ggml-opencl/ggml-opencl.cpp +++ b/ggml/src/ggml-opencl/ggml-opencl.cpp @@ -538,6 +538,10 @@ struct ggml_backend_opencl_context { // ragged moe, use int to directly pass to kernel cl_uint adreno_use_moe_ragged; cl_uint adreno_moe_ragged_skip_gran; + cl_uint adreno_use_moe_ragged_dp4; + + // whether fuse moe combine + cl_uint fuse_moe_combine; bool adreno_has_large_buffer; bool adreno_use_large_buffer; @@ -563,6 +567,12 @@ struct ggml_backend_opencl_context { ggml_cl_buffer prealloc_quant_trans; ggml_cl_buffer prealloc_scales_trans; ggml_cl_buffer prealloc_act_trans; + // q8_1-quantized reordered MoE activations for the dp4a prefill GEMM. + ggml_cl_buffer prealloc_moe_qa; // int8 quants [tok_slots * ne00] + ggml_cl_buffer prealloc_moe_da; // per-block d [tok_slots * ne00/32] (half) + ggml_cl_buffer prealloc_moe_sa; // per-block s [tok_slots * ne00/32] (half) + // scratch copy of the router weights to avoid dst aliasing + ggml_cl_buffer prealloc_moe_combine_w; // pool of persistent image1d_buffer views over kv-cache layers, keyed by // (parent buffer, offset within parent) @@ -816,19 +826,33 @@ struct ggml_backend_opencl_context { // [size_idx][kda][tgpp] where size_idx: 0=S_V=16, 1=32, 2=64, 3=128; kda: 0 or 1. // tgpp 0 = TG variant (COLS_PER_LANE_GROUP=1), tgpp 1 = prefill variant (COLS_PER_LANE_GROUP=4). cl_kernel kernel_gated_delta_net_f32[4][2][2] = {}; - cl_kernel kernel_timestep_embedding; cl_kernel kernel_gemv_moe_q4_0_f32_ns, kernel_gemm_moe_q4_0_f32_ns, kernel_gemm_moe_q4_0_f32_ns_bin; + cl_kernel kernel_gemm_moe_q8_0_f32_ns; cl_kernel kernel_gemv_moe_q4_1_f32_ns, kernel_gemm_moe_q4_1_f32_ns, kernel_gemm_moe_q4_1_f32_ns_bin; cl_kernel kernel_gemv_moe_q5_0_f32_ns, kernel_gemm_moe_q5_0_f32_ns; cl_kernel kernel_gemv_moe_q5_1_f32_ns, kernel_gemm_moe_q5_1_f32_ns; cl_kernel kernel_gemv_moe_q4_k_f32_ns, kernel_gemm_moe_q4_k_f32_ns, kernel_gemm_moe_q4_k_f32_ns_bin; + cl_kernel kernel_gemv_moe_q4_k_f32_ns_wimg = nullptr; // weight-as-texture MoE decode GEMV (opt-in) + cl_kernel kernel_gemm_moe_q4_k_q8_1_dp4a; // dp4a (int8) prefill GEMM variant + cl_kernel kernel_moe_reorder_quant_a_q8_1; // fused reorder + q8_1 quant for the dp4a GEMM + cl_kernel kernel_gemm_moe_q8_1_dp4a_q80 = nullptr; // generic dp4a MoE GEMM (MOE_QT=80), opt-in + cl_kernel kernel_moe_expand_scale_q8_0 = nullptr; // q8_0 per-block d -> uniform scale[16] + cl_kernel kernel_gemm_moe_q8_1_dp4a_q50 = nullptr; // generic dp4a MoE GEMM (MOE_QT=50, q5_0), opt-in + cl_kernel kernel_moe_expand_scale_q5_0 = nullptr; // q5_0 d -> uniform scale[2]/min[1] per 32-block + cl_kernel kernel_gemm_moe_q8_1_dp4a_q5k = nullptr; // generic dp4a MoE GEMM (MOE_QT=5, q5_K), opt-in + cl_kernel kernel_moe_expand_scale_q5_K = nullptr; // q5_K 6-bit s[] -> uniform scale[16]/min[8] cl_kernel kernel_gemv_moe_q5_k_f32_ns, kernel_gemm_moe_q5_k_f32_ns; cl_kernel kernel_gemv_moe_q6_k_f32_ns, kernel_gemm_moe_q6_k_f32_ns; + cl_kernel kernel_gemm_moe_q6_k_q8_1_dp4a; // dp4a (int8) q6_K MoE prefill GEMM cl_kernel kernel_gemv_moe_mxfp4_f32, kernel_gemm_moe_mxfp4_f32; cl_kernel kernel_gemv_moe_mxfp4_f32_ns, kernel_gemm_moe_mxfp4_f32_ns, kernel_gemm_moe_mxfp4_f32_ns_bin; + cl_kernel kernel_gemv_moe_mxfp4_f32_ns_wimg = nullptr; // weight-as-texture MoE decode GEMV + cl_kernel kernel_gemm_moe_mxfp4_q8_1_dp4a; // dp4a (int8) mxfp4 MoE prefill GEMM + cl_kernel kernel_gemm_moe_q4_0_q8_1_dp4a; // dp4a (int8) q4_0 MoE prefill GEMM cl_kernel kernel_moe_reorder_b; cl_kernel kernel_moe_histogram, kernel_moe_scan, kernel_moe_fill, kernel_moe_scatter; + cl_kernel kernel_moe_combine_f32 = nullptr; // fused router-weight mul + cross-expert sum cl_kernel kernel_mul_mv_id_q4_0_f32_8x_flat; cl_kernel kernel_mul_mv_id_q8_0_f32, kernel_mul_mv_id_q8_0_f32_flat; cl_kernel kernel_mul_mv_id_mxfp4_f32; @@ -1006,21 +1030,32 @@ struct ggml_backend_opencl_context { cl_kernel kernel_gemv_noshuffle_q4_1_f32; cl_kernel kernel_gemm_noshuffle_q4_1_f32; cl_kernel kernel_gemm_noshuffle_q8_0_f32, kernel_gemm_noshuffle_q8_0_f32_bin; + cl_kernel kernel_gemm_noshuffle_q8_0_q8_1_dp4a = nullptr; // dp4a (int8) dense q8_0 prefill GEMM (opt-in) + cl_kernel kernel_gemm_noshuffle_q8_0_q8_1_dp4a_wimg = nullptr; // q8_0 dense dp4a, weights via texture (opt-in) cl_kernel kernel_gemv_noshuffle_q8_0_f32; cl_kernel kernel_gemm_noshuffle_q1_0_f32; cl_kernel kernel_gemv_noshuffle_q1_0_f32; cl_kernel kernel_gemv_noshuffle_q4_k_f32; cl_kernel kernel_gemm_noshuffle_q4_k_f32; + cl_kernel kernel_gemm_noshuffle_q4_k_q8_1_dp4a; // dp4a (int8) dense prefill GEMM + cl_kernel kernel_gemm_noshuffle_q4_k_q8_1_dp4a_wimg; // dp4a dense prefill GEMM, weights via texture (X1 opt-in) + cl_kernel kernel_gemm_noshuffle_q5_k_q8_1_dp4a; // dp4a (int8) dense q5_K prefill GEMM + cl_kernel kernel_gemm_noshuffle_q6_k_q8_1_dp4a; // dp4a (int8) dense q6_K prefill GEMM + cl_kernel kernel_quant_a_q8_1; // plain activation q8_1 pre-pass cl_kernel kernel_gemv_noshuffle_q6_K_f32; cl_kernel kernel_gemm_noshuffle_q6_K_f32; cl_kernel kernel_gemv_noshuffle_q5_k_f32; cl_kernel kernel_gemm_noshuffle_q5_k_f32; cl_kernel kernel_gemv_noshuffle_q5_0_f32; cl_kernel kernel_gemm_noshuffle_q5_0_f32; + cl_kernel kernel_gemm_noshuffle_q5_0_q8_1_dp4a = nullptr; // dp4a (int8) dense q5_0 prefill GEMM + cl_kernel kernel_gemm_noshuffle_q5_0_q8_1_dp4a_wimg = nullptr; // q5_0 dense dp4a, qs plane via texture (opt-in) cl_kernel kernel_gemv_noshuffle_q5_1_f32; cl_kernel kernel_gemm_noshuffle_q5_1_f32; cl_kernel kernel_gemv_noshuffle_iq4_nl_f32; cl_kernel kernel_gemm_noshuffle_iq4_nl_f32; + cl_kernel kernel_gemm_noshuffle_iq4_nl_q8_1_dp4a = nullptr; // dp4a (int8) dense IQ4_NL prefill GEMM + cl_kernel kernel_gemm_noshuffle_q4_0_q8_1_dp4a = nullptr; // dp4a (int8) dense q4_0 prefill GEMM #endif // GGML_OPENCL_USE_ADRENO_KERNELS void free() { @@ -1396,6 +1431,11 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) { CL_CHECK((backend_ctx->kernel_restore_block_iq4_nl_noshuffle = clCreateKernel(backend_ctx->program_cvt, "kernel_restore_block_iq4_nl_noshuffle", &err), err)); CL_CHECK((backend_ctx->kernel_convert_bf16_to_f16 = clCreateKernel(backend_ctx->program_cvt, "kernel_convert_bf16_to_f16", &err), err)); CL_CHECK((backend_ctx->kernel_convert_f16_to_bf16 = clCreateKernel(backend_ctx->program_cvt, "kernel_convert_f16_to_bf16", &err), err)); +#ifdef GGML_OPENCL_USE_ADRENO_KERNELS + CL_CHECK((backend_ctx->kernel_moe_expand_scale_q8_0 = clCreateKernel(backend_ctx->program_cvt, "kernel_moe_expand_scale_q8_0", &err), err)); + CL_CHECK((backend_ctx->kernel_moe_expand_scale_q5_0 = clCreateKernel(backend_ctx->program_cvt, "kernel_moe_expand_scale_q5_0", &err), err)); + CL_CHECK((backend_ctx->kernel_moe_expand_scale_q5_K = clCreateKernel(backend_ctx->program_cvt, "kernel_moe_expand_scale_q5_K", &err), err)); +#endif GGML_LOG_CONT("."); } @@ -3115,6 +3155,23 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) { GGML_LOG_CONT("."); } + // moe_combine (fused router-weight mul + cross-expert sum) + { + #ifdef GGML_OPENCL_EMBED_KERNELS + const std::string kernel_src { + #include "moe_combine.cl.h" + }; + #else + const std::string kernel_src = read_file("moe_combine.cl"); + #endif + cl_program prog = build_program_from_source( + backend_ctx->context, backend_ctx->device, kernel_src.c_str(), compile_opts); + CL_CHECK((backend_ctx->kernel_moe_combine_f32 = + clCreateKernel(prog, "kernel_moe_combine_f32", &err), err)); + CL_CHECK(clReleaseProgram(prog)); + GGML_LOG_CONT("."); + } + // mul_mv_id_q4_0_f32_8x_flat { #ifdef GGML_OPENCL_EMBED_KERNELS @@ -3432,6 +3489,22 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) { GGML_LOG_CONT("."); } + // gemm_noshuffle_q5_0_q8_1_dp4a (dp4a dense q5_0 prefill GEMM) + { +#ifdef GGML_OPENCL_EMBED_KERNELS + const std::string kernel_src { + #include "gemm_noshuffle_q5_0_q8_1_dp4a.cl.h" + }; +#else + const std::string kernel_src = read_file("gemm_noshuffle_q5_0_q8_1_dp4a.cl"); +#endif + cl_program prog = build_program_from_source(backend_ctx->context, backend_ctx->device, kernel_src.c_str(), compile_opts); + CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q5_0_q8_1_dp4a = clCreateKernel(prog, "kernel_gemm_noshuffle_q5_0_q8_1_dp4a", &err), err)); + CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q5_0_q8_1_dp4a_wimg = clCreateKernel(prog, "kernel_gemm_noshuffle_q5_0_q8_1_dp4a_wimg", &err), err)); + CL_CHECK(clReleaseProgram(prog)); + GGML_LOG_CONT("."); + } + // gemv_noshuffle_q5_0_f32 { std::string CL_gemv_compile_opts = std::string("-cl-std=") + opencl_c_std + @@ -3506,6 +3579,36 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) { GGML_LOG_CONT("."); } + // gemm_noshuffle_iq4_nl_q8_1_dp4a (dp4a dense IQ4_NL prefill GEMM) + { +#ifdef GGML_OPENCL_EMBED_KERNELS + const std::string kernel_src { + #include "gemm_noshuffle_iq4_nl_q8_1_dp4a.cl.h" + }; +#else + const std::string kernel_src = read_file("gemm_noshuffle_iq4_nl_q8_1_dp4a.cl"); +#endif + cl_program prog = build_program_from_source(backend_ctx->context, backend_ctx->device, kernel_src.c_str(), compile_opts); + CL_CHECK((backend_ctx->kernel_gemm_noshuffle_iq4_nl_q8_1_dp4a = clCreateKernel(prog, "kernel_gemm_noshuffle_iq4_nl_q8_1_dp4a", &err), err)); + CL_CHECK(clReleaseProgram(prog)); + GGML_LOG_CONT("."); + } + + // gemm_noshuffle_q4_0_q8_1_dp4a (dp4a dense q4_0 prefill GEMM) + { +#ifdef GGML_OPENCL_EMBED_KERNELS + const std::string kernel_src { + #include "gemm_noshuffle_q4_0_q8_1_dp4a.cl.h" + }; +#else + const std::string kernel_src = read_file("gemm_noshuffle_q4_0_q8_1_dp4a.cl"); +#endif + cl_program prog = build_program_from_source(backend_ctx->context, backend_ctx->device, kernel_src.c_str(), compile_opts); + CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q4_0_q8_1_dp4a = clCreateKernel(prog, "kernel_gemm_noshuffle_q4_0_q8_1_dp4a", &err), err)); + CL_CHECK(clReleaseProgram(prog)); + GGML_LOG_CONT("."); + } + // gemv_noshuffle_iq4_nl_f32 { std::string CL_gemv_compile_opts = std::string("-cl-std=") + opencl_c_std + @@ -3604,6 +3707,89 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) { GGML_LOG_CONT("."); } + // gemm_noshuffle_q4_k_q8_1_dp4a (dp4a dense prefill GEMM) + { +#ifdef GGML_OPENCL_EMBED_KERNELS + const std::string kernel_src { + #include "gemm_noshuffle_q4_k_q8_1_dp4a.cl.h" + }; +#else + const std::string kernel_src = read_file("gemm_noshuffle_q4_k_q8_1_dp4a.cl"); +#endif + // Per-device dp4a dense tile. The X2-tuned TILESIZE_N=32 over-occupies LDS on + // X1 (1152 B/WG -> few resident WGs); TILESIZE_N=8 (288 B) lifts occupancy on + // X1, byte-identical. X2E keeps 32. Env override wins. + int q4k_dp4a_ts = (backend_ctx->adreno_gen == ADRENO_GPU_GEN::X1E) ? 8 : 32; + if (const char * e = getenv("GGML_OPENCL_Q4K_DP4A_TS")) q4k_dp4a_ts = atoi(e); + std::string dp4a_opts = compile_opts + " -DTILESIZE_N=" + std::to_string(q4k_dp4a_ts); + cl_program prog = build_program_from_source(backend_ctx->context, backend_ctx->device, kernel_src.c_str(), dp4a_opts); + CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q4_k_q8_1_dp4a = clCreateKernel(prog, "kernel_gemm_noshuffle_q4_k_q8_1_dp4a", &err), err)); + CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q4_k_q8_1_dp4a_wimg = clCreateKernel(prog, "kernel_gemm_noshuffle_q4_k_q8_1_dp4a_wimg", &err), err)); + CL_CHECK(clReleaseProgram(prog)); + GGML_LOG_CONT("."); + } + + // gemm_noshuffle_q8_0_q8_1_dp4a (dp4a dense q8_0 prefill GEMM) + { +#ifdef GGML_OPENCL_EMBED_KERNELS + const std::string kernel_src { + #include "gemm_noshuffle_q8_0_q8_1_dp4a.cl.h" + }; +#else + const std::string kernel_src = read_file("gemm_noshuffle_q8_0_q8_1_dp4a.cl"); +#endif + cl_program prog = build_program_from_source(backend_ctx->context, backend_ctx->device, kernel_src.c_str(), compile_opts); + CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q8_0_q8_1_dp4a = clCreateKernel(prog, "kernel_gemm_noshuffle_q8_0_q8_1_dp4a", &err), err)); + CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q8_0_q8_1_dp4a_wimg = clCreateKernel(prog, "kernel_gemm_noshuffle_q8_0_q8_1_dp4a_wimg", &err), err)); + CL_CHECK(clReleaseProgram(prog)); + GGML_LOG_CONT("."); + } + + // gemm_noshuffle_q5_k_q8_1_dp4a (dp4a dense prefill GEMM for q5_K) + { +#ifdef GGML_OPENCL_EMBED_KERNELS + const std::string kernel_src { + #include "gemm_noshuffle_q5_k_q8_1_dp4a.cl.h" + }; +#else + const std::string kernel_src = read_file("gemm_noshuffle_q5_k_q8_1_dp4a.cl"); +#endif + cl_program prog = build_program_from_source(backend_ctx->context, backend_ctx->device, kernel_src.c_str(), compile_opts); + CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q5_k_q8_1_dp4a = clCreateKernel(prog, "kernel_gemm_noshuffle_q5_k_q8_1_dp4a", &err), err)); + CL_CHECK(clReleaseProgram(prog)); + GGML_LOG_CONT("."); + } + + // gemm_noshuffle_q6_k_q8_1_dp4a (dp4a dense prefill GEMM for q6_K ffn_down/output) + { +#ifdef GGML_OPENCL_EMBED_KERNELS + const std::string kernel_src { + #include "gemm_noshuffle_q6_k_q8_1_dp4a.cl.h" + }; +#else + const std::string kernel_src = read_file("gemm_noshuffle_q6_k_q8_1_dp4a.cl"); +#endif + cl_program prog = build_program_from_source(backend_ctx->context, backend_ctx->device, kernel_src.c_str(), compile_opts); + CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q6_k_q8_1_dp4a = clCreateKernel(prog, "kernel_gemm_noshuffle_q6_k_q8_1_dp4a", &err), err)); + CL_CHECK(clReleaseProgram(prog)); + GGML_LOG_CONT("."); + } + + // quant_a_q8_1 (plain activation q8_1 pre-pass for the dense dp4a GEMM) + { +#ifdef GGML_OPENCL_EMBED_KERNELS + const std::string kernel_src { + #include "quant_a_q8_1.cl.h" + }; +#else + const std::string kernel_src = read_file("quant_a_q8_1.cl"); +#endif + cl_program prog = build_program_from_source(backend_ctx->context, backend_ctx->device, kernel_src.c_str(), compile_opts); + CL_CHECK((backend_ctx->kernel_quant_a_q8_1 = clCreateKernel(prog, "kernel_quant_a_q8_1", &err), err)); + CL_CHECK(clReleaseProgram(prog)); + GGML_LOG_CONT("."); + } + // gemv_noshuffle_q4_k_f32 { std::string CL_gemv_compile_opts = std::string("-cl-std=") + opencl_c_std + @@ -3766,6 +3952,23 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) { } } + // gemm_moe_q8_0_f32_ns + { +#ifdef GGML_OPENCL_EMBED_KERNELS + const std::string kernel_src { + #include "gemm_moe_q8_0_f32_ns.cl.h" + }; +#else + const std::string kernel_src = read_file("gemm_moe_q8_0_f32_ns.cl"); +#endif + cl_program prog = + build_program_from_source(backend_ctx->context, backend_ctx->device, kernel_src.c_str(), CL_moe_compile_opts); + + CL_CHECK((backend_ctx->kernel_gemm_moe_q8_0_f32_ns = clCreateKernel(prog, "kernel_gemm_moe_q8_0_f32_ns", &err), err)); + CL_CHECK(clReleaseProgram(prog)); + GGML_LOG_CONT("."); + } + // gemv_moe_q5_0_f32_ns { #ifdef GGML_OPENCL_EMBED_KERNELS @@ -3847,6 +4050,7 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) { build_program_from_source(backend_ctx->context, backend_ctx->device, kernel_src.c_str(), CL_moe_compile_opts); CL_CHECK((backend_ctx->kernel_gemv_moe_q4_k_f32_ns = clCreateKernel(prog, "kernel_gemv_moe_q4_k_f32_ns", &err), err)); + CL_CHECK((backend_ctx->kernel_gemv_moe_q4_k_f32_ns_wimg = clCreateKernel(prog, "kernel_gemv_moe_q4_k_f32_ns_wimg", &err), err)); CL_CHECK(clReleaseProgram(prog)); GGML_LOG_CONT("."); } @@ -3886,6 +4090,103 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) { } } + // gemm_moe_q4_k_q8_1_dp4a (dp4a prefill GEMM) + { +#ifdef GGML_OPENCL_EMBED_KERNELS + const std::string kernel_src { + #include "gemm_moe_q4_k_q8_1_dp4a.cl.h" + }; +#else + const std::string kernel_src = read_file("gemm_moe_q4_k_q8_1_dp4a.cl"); +#endif + cl_program prog = + build_program_from_source(backend_ctx->context, backend_ctx->device, kernel_src.c_str(), CL_moe_compile_opts); + + CL_CHECK((backend_ctx->kernel_gemm_moe_q4_k_q8_1_dp4a = clCreateKernel(prog, "kernel_gemm_moe_q4_k_q8_1_dp4a", &err), err)); + CL_CHECK(clReleaseProgram(prog)); + GGML_LOG_CONT("."); + } + + // gemm_moe_mxfp4_q8_1_dp4a (dp4a prefill GEMM) + { +#ifdef GGML_OPENCL_EMBED_KERNELS + const std::string kernel_src { + #include "gemm_moe_mxfp4_q8_1_dp4a.cl.h" + }; +#else + const std::string kernel_src = read_file("gemm_moe_mxfp4_q8_1_dp4a.cl"); +#endif + cl_program prog = + build_program_from_source(backend_ctx->context, backend_ctx->device, kernel_src.c_str(), CL_moe_compile_opts); + + CL_CHECK((backend_ctx->kernel_gemm_moe_mxfp4_q8_1_dp4a = clCreateKernel(prog, "kernel_gemm_moe_mxfp4_q8_1_dp4a", &err), err)); + CL_CHECK(clReleaseProgram(prog)); + GGML_LOG_CONT("."); + } + + // gemm_moe_q4_0_q8_1_dp4a (dp4a prefill GEMM) + { +#ifdef GGML_OPENCL_EMBED_KERNELS + const std::string kernel_src { + #include "gemm_moe_q4_0_q8_1_dp4a.cl.h" + }; +#else + const std::string kernel_src = read_file("gemm_moe_q4_0_q8_1_dp4a.cl"); +#endif + cl_program prog = + build_program_from_source(backend_ctx->context, backend_ctx->device, kernel_src.c_str(), CL_moe_compile_opts); + + CL_CHECK((backend_ctx->kernel_gemm_moe_q4_0_q8_1_dp4a = clCreateKernel(prog, "kernel_gemm_moe_q4_0_q8_1_dp4a", &err), err)); + CL_CHECK(clReleaseProgram(prog)); + GGML_LOG_CONT("."); + } + + // gemm_moe_q8_1_dp4a (generic dp4a MoE GEMM; MOE_QT=80 -> q8_0 expert variant) + { +#ifdef GGML_OPENCL_EMBED_KERNELS + const std::string kernel_src { + #include "gemm_moe_q8_1_dp4a.cl.h" + }; +#else + const std::string kernel_src = read_file("gemm_moe_q8_1_dp4a.cl"); +#endif + const std::string opts80 = CL_moe_compile_opts + " -DMOE_QT=80"; + cl_program prog = + build_program_from_source(backend_ctx->context, backend_ctx->device, kernel_src.c_str(), opts80.c_str()); + CL_CHECK((backend_ctx->kernel_gemm_moe_q8_1_dp4a_q80 = clCreateKernel(prog, "kernel_gemm_moe_q8_1_dp4a", &err), err)); + CL_CHECK(clReleaseProgram(prog)); + + const std::string opts50 = CL_moe_compile_opts + " -DMOE_QT=50"; + cl_program prog50 = + build_program_from_source(backend_ctx->context, backend_ctx->device, kernel_src.c_str(), opts50.c_str()); + CL_CHECK((backend_ctx->kernel_gemm_moe_q8_1_dp4a_q50 = clCreateKernel(prog50, "kernel_gemm_moe_q8_1_dp4a", &err), err)); + CL_CHECK(clReleaseProgram(prog50)); + + const std::string opts5 = CL_moe_compile_opts + " -DMOE_QT=5"; + cl_program prog5 = + build_program_from_source(backend_ctx->context, backend_ctx->device, kernel_src.c_str(), opts5.c_str()); + CL_CHECK((backend_ctx->kernel_gemm_moe_q8_1_dp4a_q5k = clCreateKernel(prog5, "kernel_gemm_moe_q8_1_dp4a", &err), err)); + CL_CHECK(clReleaseProgram(prog5)); + GGML_LOG_CONT("."); + } + + // moe_reorder_quant_a_q8_1 (fused reorder + q8_1 quant) + { +#ifdef GGML_OPENCL_EMBED_KERNELS + const std::string kernel_src { + #include "moe_reorder_quant_a_q8_1.cl.h" + }; +#else + const std::string kernel_src = read_file("moe_reorder_quant_a_q8_1.cl"); +#endif + cl_program prog = + build_program_from_source(backend_ctx->context, backend_ctx->device, kernel_src.c_str(), CL_moe_compile_opts); + + CL_CHECK((backend_ctx->kernel_moe_reorder_quant_a_q8_1 = clCreateKernel(prog, "kernel_moe_reorder_quant_a_q8_1", &err), err)); + CL_CHECK(clReleaseProgram(prog)); + GGML_LOG_CONT("."); + } + // gemv_moe_q5_k_f32_ns { #ifdef GGML_OPENCL_EMBED_KERNELS @@ -3954,6 +4255,23 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) { GGML_LOG_CONT("."); } + // gemm_moe_q6_k_q8_1_dp4a (dp4a q6_K MoE prefill GEMM) + { +#ifdef GGML_OPENCL_EMBED_KERNELS + const std::string kernel_src { + #include "gemm_moe_q6_k_q8_1_dp4a.cl.h" + }; +#else + const std::string kernel_src = read_file("gemm_moe_q6_k_q8_1_dp4a.cl"); +#endif + cl_program prog = + build_program_from_source(backend_ctx->context, backend_ctx->device, kernel_src.c_str(), CL_moe_compile_opts); + + CL_CHECK((backend_ctx->kernel_gemm_moe_q6_k_q8_1_dp4a = clCreateKernel(prog, "kernel_gemm_moe_q6_k_q8_1_dp4a", &err), err)); + CL_CHECK(clReleaseProgram(prog)); + GGML_LOG_CONT("."); + } + // gemv_moe_mxfp4_f32_ns { #ifdef GGML_OPENCL_EMBED_KERNELS @@ -3967,6 +4285,7 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) { build_program_from_source(backend_ctx->context, backend_ctx->device, kernel_src.c_str(), CL_moe_compile_opts); CL_CHECK((backend_ctx->kernel_gemv_moe_mxfp4_f32_ns = clCreateKernel(prog, "kernel_gemv_moe_mxfp4_f32_ns", &err), err)); + CL_CHECK((backend_ctx->kernel_gemv_moe_mxfp4_f32_ns_wimg = clCreateKernel(prog, "kernel_gemv_moe_mxfp4_f32_ns_wimg", &err), err)); CL_CHECK(clReleaseProgram(prog)); GGML_LOG_CONT("."); } @@ -5484,6 +5803,7 @@ static ggml_backend_opencl_context * ggml_cl_init(ggml_backend_dev_t dev) { // check Adreno large buffer support backend_ctx->adreno_has_large_buffer = strstr(ext_buffer, "cl_qcom_large_buffer") != NULL; + // subgroup shuffle support (N_SPLIT>1 FA kernel) backend_ctx->has_qcom_subgroup_shuffle = strstr(ext_buffer, "cl_qcom_subgroup_shuffle") != NULL; backend_ctx->has_subgroup_shuffle = @@ -5537,6 +5857,14 @@ static ggml_backend_opencl_context * ggml_cl_init(ggml_backend_dev_t dev) { static const char * ragged_gran_env = getenv("GGML_OPENCL_MOE_RAGGED_GRAN"); backend_ctx->adreno_moe_ragged_skip_gran = (ragged_gran_env != NULL) ? atoi(ragged_gran_env) : 8; + // whether fuse moe combine + static const char * fuse_moe_combine_env = getenv("GGML_OPENCL_FUSE_MOE_COMBINE"); + backend_ctx->fuse_moe_combine = fuse_moe_combine_env == NULL ? 1 : (atoi(fuse_moe_combine_env) != 0); + + // ragged moe dp4 variant + static const char * ragged_dp4_env = getenv("GGML_OPENCL_MOE_RAGGED"); + backend_ctx->adreno_use_moe_ragged_dp4 = ragged_dp4_env == NULL ? 1 : (atoi(ragged_dp4_env) != 0); + #ifdef GGML_OPENCL_USE_ADRENO_BIN_KERNELS // try loading adreno binary kernels if enabled // if fails to load, builtin kernels will be used @@ -5863,6 +6191,10 @@ struct ggml_tensor_extra_cl_q5_0 { cl_mem d = nullptr; // Scales in image1d_buffer_t. cl_mem d_img = nullptr; + // Uniform per-32-block scale (2/block) + min (1/block, = d*16 for the -16 centering) + // for the generic dp4a MoE GEMM. Built from d. + cl_mem scale = nullptr; + cl_mem min = nullptr; // Size of quantized values. size_t size_qs = 0; // Size of 5-th bit values. @@ -5891,6 +6223,14 @@ struct ggml_tensor_extra_cl_q5_0 { CL_CHECK(clReleaseMemObject(qs_img)); qs_img = nullptr; } + if (scale != nullptr) { + CL_CHECK(clReleaseMemObject(scale)); + scale = nullptr; + } + if (min != nullptr) { + CL_CHECK(clReleaseMemObject(min)); + min = nullptr; + } qh_img = nullptr; d_img = nullptr; @@ -6015,6 +6355,11 @@ struct ggml_tensor_extra_cl_q8_0 { cl_mem d = nullptr; cl_mem d_img = nullptr; + // Uniform per-16-segment scale (16/superblock) for the generic dp4a MoE GEMM. + // Expanded from d at set_tensor; the int8 codes are reused from q. + // q8_0 is symmetric so no min buffer (has_min=0). + cl_mem scale = nullptr; + size_t size_q = 0; size_t size_d = 0; @@ -6034,6 +6379,10 @@ struct ggml_tensor_extra_cl_q8_0 { CL_CHECK(clReleaseMemObject(d)); d = nullptr; } + if (scale != nullptr) { + CL_CHECK(clReleaseMemObject(scale)); + scale = nullptr; + } // Currently, q_img and d_img are not used. They can be image1d_buffer_t // that wraps around q and d to utilize image access path. q_img = nullptr; @@ -6120,6 +6469,11 @@ struct ggml_tensor_extra_cl_q5_K { cl_mem d = nullptr; // Min for each super block. cl_mem dm = nullptr; + // Uniform per-32-block scale (2/block) + min (1/block, = dm*mn) decoded from the + // 6-bit packed s[] for the generic dp4a MoE GEMM kernel_gemm_moe_q8_1_dp4a. + // Built from s/d/dm at set_tensor; q/qh are reused as-is. + cl_mem scale = nullptr; + cl_mem min = nullptr; size_t size_q = 0; size_t size_qh = 0; @@ -6156,6 +6510,14 @@ struct ggml_tensor_extra_cl_q5_K { CL_CHECK(clReleaseMemObject(q_img)); q_img = nullptr; } + if (scale != nullptr) { + CL_CHECK(clReleaseMemObject(scale)); + scale = nullptr; + } + if (min != nullptr) { + CL_CHECK(clReleaseMemObject(min)); + min = nullptr; + } size_q = 0; size_qh = 0; @@ -6298,6 +6660,122 @@ static void sync_with_other_backends(ggml_backend_t backend) { sync_with_other_backends(backend_ctx); } +// True if two tensors share a device buffer with overlapping byte ranges. The pool +// allocator may place a fused op's output over a sequentially-dead input (safe for the +// original separate kernels, but a read/write race inside one fused kernel). +static bool ggml_cl_tensors_overlap(const ggml_tensor * x, const ggml_tensor * y) { + ggml_tensor_extra_cl * ex = (ggml_tensor_extra_cl *)x->extra; + ggml_tensor_extra_cl * ey = (ggml_tensor_extra_cl *)y->extra; + if (!ex || !ey || ex->data_device != ey->data_device) { return false; } + const cl_ulong xo = ex->offset + x->view_offs, xe = xo + ggml_nbytes(x); + const cl_ulong yo = ey->offset + y->view_offs, ye = yo + ggml_nbytes(y); + return xo < ye && yo < xe; +} + +// Detect the MoE combine epilogue: router-weight MUL ([n_embd,k,nt] * [1,k,nt]) followed +// by k VIEWs of it and a (k-1)-long ADD reduction chain producing [n_embd, nt]. When it +// matches (and the output does not alias the inputs), the whole subgraph collapses to one +// weighted-sum-across-experts kernel. +static bool ggml_opencl_can_fuse_moe_combine(const struct ggml_cgraph * cgraph, int node_idx, + const ggml_tensor ** out_final_add) { + const ggml_tensor * mul = cgraph->nodes[node_idx]; + if (mul->op != GGML_OP_MUL) { return false; } + const ggml_tensor * experts = mul->src[0]; + const ggml_tensor * weights = mul->src[1]; + if (!experts || !weights) { return false; } + if (experts->type != GGML_TYPE_F32 || weights->type != GGML_TYPE_F32 || mul->type != GGML_TYPE_F32) { return false; } + + const int64_t n_embd = experts->ne[0]; + const int64_t k = experts->ne[1]; + const int64_t nt = experts->ne[2]; + if (k < 2 || k > 64 || experts->ne[3] != 1 || n_embd % 4 != 0) { return false; } + if (weights->ne[0] != 1 || weights->ne[1] != k || weights->ne[2] != nt || weights->ne[3] != 1) { return false; } + if (mul->ne[0] != n_embd || mul->ne[1] != k || mul->ne[2] != nt) { return false; } + // the fused kernel needs contiguous experts/weights and a contiguous 2D dst + if (!ggml_is_contiguous(experts) || !ggml_is_contiguous(weights)) { return false; } + + const int n_nodes = 1 + (int)k + (int)(k - 1); // MUL + k*VIEW + (k-1)*ADD + if (n_nodes >= 32) { return false; } + if (node_idx + n_nodes > cgraph->n_nodes) { return false; } + + enum ggml_op ops[1 + 64 + 63]; + int n = 0; + ops[n++] = GGML_OP_MUL; + for (int j = 0; j < (int)k; ++j) { ops[n++] = GGML_OP_VIEW; } + for (int j = 0; j < (int)k - 1; ++j) { ops[n++] = GGML_OP_ADD; } + const int outs[] = { node_idx + n_nodes - 1 }; + if (!ggml_can_fuse_subgraph(cgraph, node_idx, n_nodes, ops, outs, 1)) { return false; } + + for (int j = 0; j < (int)k; ++j) { + const ggml_tensor * vw = cgraph->nodes[node_idx + 1 + j]; + if (vw->op != GGML_OP_VIEW || vw->src[0] != mul || vw->ne[0] != n_embd || vw->ne[1] != nt) { return false; } + } + const ggml_tensor * final_add = cgraph->nodes[node_idx + n_nodes - 1]; + if (final_add->op != GGML_OP_ADD || final_add->type != GGML_TYPE_F32 || + final_add->ne[0] != n_embd || final_add->ne[1] != nt || final_add->ne[2] != 1) { return false; } + if (!ggml_is_contiguous(final_add)) { return false; } + // the fused kernel reads experts + writes final_add in one pass; bail if the + // pool allocator overlapped the output with the (large) experts input -- would race. + // The small weights input is copied to a private scratch in the dispatch, so its own + // aliasing with the output is handled there and does not block the fusion. + if (ggml_cl_tensors_overlap(experts, final_add)) { return false; } + + *out_final_add = final_add; + return true; +} + +static void ggml_cl_moe_combine_fused(ggml_backend_t backend, const ggml_tensor * mul, const ggml_tensor * dst) { + ggml_backend_opencl_context * backend_ctx = (ggml_backend_opencl_context *)backend->context; + const ggml_tensor * experts = mul->src[0]; + const ggml_tensor * weights = mul->src[1]; + + ggml_tensor_extra_cl * ee = (ggml_tensor_extra_cl *)experts->extra; + ggml_tensor_extra_cl * ew = (ggml_tensor_extra_cl *)weights->extra; + ggml_tensor_extra_cl * ed = (ggml_tensor_extra_cl *)dst->extra; + cl_ulong off_e = ee->offset + experts->view_offs; + cl_ulong off_w = ew->offset + weights->view_offs; + cl_ulong off_d = ed->offset + dst->view_offs; + + const int n_embd4 = (int)(experts->ne[0] / 4); + const int k = (int)experts->ne[1]; + const int nt = (int)experts->ne[2]; + const cl_uint e1 = (cl_uint)(experts->nb[1] / sizeof(float)); + const cl_uint e2 = (cl_uint)(experts->nb[2] / sizeof(float)); + const cl_uint w1 = (cl_uint)(weights->nb[1] / sizeof(float)); + const cl_uint w2 = (cl_uint)(weights->nb[2] / sizeof(float)); + const cl_uint d1 = (cl_uint)(dst->nb[1] / sizeof(float)); + + // The router weights are tiny ([1,k,nt]) and may share a pool buffer with the output; + // copy them into a private scratch so the fused kernel never reads aliased memory. + const size_t w_bytes = ggml_nbytes(weights); + backend_ctx->prealloc_moe_combine_w.allocate(backend_ctx->context, w_bytes); + CL_CHECK(clEnqueueCopyBuffer(backend_ctx->queue, ew->data_device, backend_ctx->prealloc_moe_combine_w.buffer, + off_w, 0, w_bytes, 0, NULL, NULL)); + cl_mem w_dev = backend_ctx->prealloc_moe_combine_w.buffer; + cl_ulong w_off = 0; + + cl_kernel kernel = backend_ctx->kernel_moe_combine_f32; + int a = 0; + CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_mem), &ee->data_device)); + CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_ulong), &off_e)); + CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_mem), &w_dev)); + CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_ulong), &w_off)); + CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_mem), &ed->data_device)); + CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_ulong), &off_d)); + CL_CHECK(clSetKernelArg(kernel, a++, sizeof(int), &n_embd4)); + CL_CHECK(clSetKernelArg(kernel, a++, sizeof(int), &k)); + CL_CHECK(clSetKernelArg(kernel, a++, sizeof(int), &nt)); + CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_uint), &e1)); + CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_uint), &e2)); + CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_uint), &w1)); + CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_uint), &w2)); + CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_uint), &d1)); + + size_t lws[2] = { 64, 1 }; + size_t gws[2] = { (size_t)(((n_embd4 + 63) / 64) * 64), (size_t)nt }; + backend_ctx->enqueue_ndrange_kernel(kernel, 2, gws, lws, dst); +} + static bool ggml_opencl_can_fuse(const struct ggml_cgraph * cgraph, int node_idx, std::initializer_list ops) { if (!ggml_can_fuse(cgraph, node_idx, ops)) { return false; @@ -6398,6 +6876,17 @@ static ggml_status ggml_backend_opencl_graph_compute(ggml_backend_t backend, ggm i += 2; continue; } + // Fuse the MoE combine: router-weight mul + cross-expert add chain -> + // one weighted-sum-across-experts kernel. + if (backend_ctx->fuse_moe_combine && !backend_ctx->disable_fusion) { + const ggml_tensor * combine_out = nullptr; + if (ggml_opencl_can_fuse_moe_combine(cgraph, i, &combine_out)) { + ggml_cl_moe_combine_fused(backend, node, combine_out); + i += 2 * (int)node->ne[1] - 1; // skip the k VIEWs + (k-1) ADDs + continue; + } + } + if (!backend_ctx->disable_fusion && ggml_opencl_can_fuse(cgraph, i, { GGML_OP_RMS_NORM, GGML_OP_MUL })) { ggml_opencl_op_rms_norm_fused(backend, node, cgraph->nodes[i+1]); i++; @@ -6446,7 +6935,12 @@ inline bool enable_adreno_trans_weight(const ggml_backend_opencl_context *backen size_t elem_num = tensor->ne[0] * tensor->ne[1] * tensor->ne[2] * tensor->ne[3]; - return ((elem_num < 128 * 1024 * 1024) && adreno_kernel); // max element num: 2**27 + // The 2D weight transpose (transpose_2d_as_*) tiles rows by 4 over a 2D matrix, + // so it requires K(ne0)%32==0, M(ne1)%4==0 and ne2==ne3==1. + const bool shape_ok = (tensor->ne[0] % 32 == 0) && (tensor->ne[1] % 4 == 0) && + (tensor->ne[2] == 1) && (tensor->ne[3] == 1); + + return ((elem_num < 128 * 1024 * 1024) && adreno_kernel && shape_ok); // max element num: 2**27 } static inline bool use_flat_gemv_for_large_m_q4_K(const ggml_tensor *tensor) { @@ -7785,6 +8279,31 @@ static void ggml_backend_opencl_buffer_set_tensor(ggml_backend_buffer_t buffer, extra->qs_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_format_qs, &img_desc_qs, NULL, &err); tensor->extra = extra; + // Generic dp4a MoE path + { + static const char * q5dp4a_env = getenv("GGML_OPENCL_Q5_MOE_DP4A"); + const bool q5dp4a = q5dp4a_env ? (atoi(q5dp4a_env) != 0) + : (backend_ctx->adreno_gen == ADRENO_GPU_GEN::X2E); + if (q5dp4a && ne02 > 1 && (ne00 % 32 == 0)) { + size_t nb32 = (size_t)ne00 / 32; + size_t sc_elems = (size_t)ne02 * ne01 * nb32 * 2; + size_t mn_elems = (size_t)ne02 * ne01 * nb32; + extra->scale = clCreateBuffer(context, CL_MEM_READ_WRITE, sc_elems * sizeof(cl_half), NULL, &err); CL_CHECK(err); + extra->min = clCreateBuffer(context, CL_MEM_READ_WRITE, mn_elems * sizeof(cl_half), NULL, &err); CL_CHECK(err); + cl_kernel ek = backend_ctx->kernel_moe_expand_scale_q5_0; + CL_CHECK(clSetKernelArg(ek, 0, sizeof(cl_mem), &extra->d)); + CL_CHECK(clSetKernelArg(ek, 1, sizeof(cl_mem), &extra->scale)); + CL_CHECK(clSetKernelArg(ek, 2, sizeof(cl_mem), &extra->min)); + CL_CHECK(clSetKernelArg(ek, 3, sizeof(int), &ne00)); + CL_CHECK(clSetKernelArg(ek, 4, sizeof(int), &ne01)); + size_t eg[3] = { (size_t)(((ne01 + 63) / 64) * 64), nb32, (size_t)ne02 }; + size_t el[3] = { 64, 1, 1 }; + cl_event evt; + CL_CHECK(clEnqueueNDRangeKernel(queue, ek, 3, NULL, eg, el, 0, NULL, &evt)); + CL_CHECK(clWaitForEvents(1, &evt)); + } + } + return; } #endif // GGML_OPENCL_USE_ADRENO_KERNELS @@ -8164,6 +8683,34 @@ static void ggml_backend_opencl_buffer_set_tensor(ggml_backend_buffer_t buffer, tensor->extra = extra; ctx->q8_0_soa_tensors.insert(tensor); + // Generic dp4a MoE path (opt-in GGML_OPENCL_Q8_MOE_DP4A) +#ifdef GGML_OPENCL_USE_ADRENO_KERNELS + { + static const char * q8dp4a_env = getenv("GGML_OPENCL_Q8_MOE_DP4A"); + const bool q8dp4a = q8dp4a_env ? (atoi(q8dp4a_env) != 0) + : (backend_ctx->adreno_gen == ADRENO_GPU_GEN::X2E); + if (q8dp4a && tensor->ne[2] > 1 && (tensor->ne[0] % 32 == 0)) { + int ne00 = (int)tensor->ne[0]; + int ne01 = (int)tensor->ne[1]; + int ne02 = (int)tensor->ne[2]; + size_t nb32 = (size_t)ne00 / 32; + size_t scale_elems = (size_t)ne02 * ne01 * nb32 * 2; // 2 per-16-seg scales / 32-block + extra->scale = clCreateBuffer(context, CL_MEM_READ_WRITE, scale_elems * sizeof(cl_half), NULL, &err); + CL_CHECK(err); + cl_kernel ek = backend_ctx->kernel_moe_expand_scale_q8_0; + CL_CHECK(clSetKernelArg(ek, 0, sizeof(cl_mem), &extra->d)); + CL_CHECK(clSetKernelArg(ek, 1, sizeof(cl_mem), &extra->scale)); + CL_CHECK(clSetKernelArg(ek, 2, sizeof(int), &ne00)); + CL_CHECK(clSetKernelArg(ek, 3, sizeof(int), &ne01)); + size_t eg[3] = { (size_t)(((ne01 + 63) / 64) * 64), nb32, (size_t)ne02 }; + size_t el[3] = { 64, 1, 1 }; + cl_event evt; + CL_CHECK(clEnqueueNDRangeKernel(queue, ek, 3, NULL, eg, el, 0, NULL, &evt)); + CL_CHECK(clWaitForEvents(1, &evt)); + } + } +#endif + // Transpose the weights and scales #ifdef GGML_OPENCL_USE_ADRENO_KERNELS if (enable_adreno_trans_weight(backend_ctx, tensor)) { @@ -8516,6 +9063,33 @@ static void ggml_backend_opencl_buffer_set_tensor(ggml_backend_buffer_t buffer, CL_CHECK(err); tensor->extra = extra; + // Generic dp4a MoE path + { + static const char * q5kdp4a_env = getenv("GGML_OPENCL_Q5K_MOE_DP4A"); + const bool q5kdp4a = q5kdp4a_env ? (atoi(q5kdp4a_env) != 0) + : (backend_ctx->adreno_gen == ADRENO_GPU_GEN::X2E); + if (q5kdp4a && ne02 > 1 && (ne00 % 256 == 0)) { + size_t nb32 = (size_t)ne00 / 32; + size_t sc_elems = (size_t)ne02 * ne01 * nb32 * 2; + size_t mn_elems = (size_t)ne02 * ne01 * nb32; + extra->scale = clCreateBuffer(context, CL_MEM_READ_WRITE, sc_elems * sizeof(cl_half), NULL, &err); CL_CHECK(err); + extra->min = clCreateBuffer(context, CL_MEM_READ_WRITE, mn_elems * sizeof(cl_half), NULL, &err); CL_CHECK(err); + cl_kernel ek = backend_ctx->kernel_moe_expand_scale_q5_K; + CL_CHECK(clSetKernelArg(ek, 0, sizeof(cl_mem), &extra->s)); + CL_CHECK(clSetKernelArg(ek, 1, sizeof(cl_mem), &extra->d)); + CL_CHECK(clSetKernelArg(ek, 2, sizeof(cl_mem), &extra->dm)); + CL_CHECK(clSetKernelArg(ek, 3, sizeof(cl_mem), &extra->scale)); + CL_CHECK(clSetKernelArg(ek, 4, sizeof(cl_mem), &extra->min)); + CL_CHECK(clSetKernelArg(ek, 5, sizeof(int), &ne00)); + CL_CHECK(clSetKernelArg(ek, 6, sizeof(int), &ne01)); + size_t eg[3] = { (size_t)(((ne01 + 63) / 64) * 64), (size_t)(ne00 / 256), (size_t)ne02 }; + size_t el[3] = { 64, 1, 1 }; + cl_event evt; + CL_CHECK(clEnqueueNDRangeKernel(queue, ek, 3, NULL, eg, el, 0, NULL, &evt)); + CL_CHECK(clWaitForEvents(1, &evt)); + } + } + return; } #endif // GGML_OPENCL_USE_ADRENO_KERNELS @@ -15245,6 +15819,61 @@ static void ggml_cl_mul_mat_q4_0_f32_adreno(ggml_backend_t backend, const ggml_t CL_CHECK(clReleaseMemObject(b_sub_buf)); CL_CHECK(clReleaseMemObject(b_img)); } else { + // dp4a (int8) dense prefill GEMM: quant activations to q8_1, then the int8 + // dp4a inner-loop GEMM, in place of the transpose + f16 half-dot kernel. + // q4_0 = d*(q-8); mirrors the IQ4_NL/q8_0 dense dp4a paths (+ the sum term). + // OPT-IN / DEFAULT OFF: correct, but neutral on X2E. q4_0's dequant + // ((q-8)*scale) is already trivial so the f16 GEMM is weight-BW-bound and the + // int8 ALU win has nothing to beat -- same as q5_0 dense (unlike IQ4_NL, whose + // codebook dequant is expensive enough for dp4a to help). Kept for A/B; force + // on with GGML_OPENCL_Q4_0_DENSE_DP4A=1. Needs N>8, K%32==0, M%64==0. + static const char * q4_0_dense_dp4a_env = getenv("GGML_OPENCL_Q4_0_DENSE_DP4A"); + const bool q4_0_dense_dp4a_on = q4_0_dense_dp4a_env + ? (atoi(q4_0_dense_dp4a_env) != 0) + : false; + if (q4_0_dense_dp4a_on && backend_ctx->kernel_gemm_noshuffle_q4_0_q8_1_dp4a + && N > 8 && (K % 32 == 0) && (M % 64 == 0)) { + cl_mem a_sub = nullptr; + region.origin = offset1; + region.size = (size_t)K * N * sizeof(float); + CL_CHECK((a_sub = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); + + const size_t n_blocks = (size_t)N * (K / 32); + backend_ctx->prealloc_moe_qa.allocate(context, (size_t)N * K * sizeof(cl_char)); + backend_ctx->prealloc_moe_da.allocate(context, n_blocks * sizeof(cl_half)); + backend_ctx->prealloc_moe_sa.allocate(context, n_blocks * sizeof(cl_half)); + + cl_int tb = (cl_int)n_blocks; + cl_kernel qk = backend_ctx->kernel_quant_a_q8_1; + CL_CHECK(clSetKernelArg(qk, 0, sizeof(cl_mem), &a_sub)); + CL_CHECK(clSetKernelArg(qk, 1, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer)); + CL_CHECK(clSetKernelArg(qk, 2, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer)); + CL_CHECK(clSetKernelArg(qk, 3, sizeof(cl_mem), &backend_ctx->prealloc_moe_sa.buffer)); + CL_CHECK(clSetKernelArg(qk, 4, sizeof(cl_int), &tb)); + size_t q_local[1] = { 64 }; + size_t q_global[1] = { (size_t)(((n_blocks + 63) / 64) * 64) }; + backend_ctx->enqueue_ndrange_kernel(qk, 1, q_global, q_local, dst); + + cl_kernel dk = backend_ctx->kernel_gemm_noshuffle_q4_0_q8_1_dp4a; + int ai = 0; + CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &extra0_q4_0->q)); + CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &extra0_q4_0->d)); + CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer)); + CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer)); + CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &backend_ctx->prealloc_moe_sa.buffer)); + CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_int), &M)); + CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_int), &N)); + CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_int), &K)); + size_t d_local[3] = { 64, 1, 1 }; + size_t d_global[3] = { 64, (size_t)(M / 64), (size_t)CEIL_DIV(N, 32) }; + backend_ctx->enqueue_ndrange_kernel(dk, 3, d_global, d_local, dst); + + CL_CHECK(clReleaseMemObject(a_sub)); + return; + } + cl_mem b_sub_buf = nullptr; cl_mem b_sub_buf_trans = nullptr; cl_mem b_img = nullptr; @@ -15624,6 +16253,98 @@ static void ggml_cl_mul_mat_q5_0_f32_adreno(ggml_backend_t backend, const ggml_t CL_CHECK(clReleaseMemObject(b_sub_buf)); CL_CHECK(clReleaseMemObject(b_img)); } else { + // dp4a (int8) dense q5_0 prefill GEMM. Quantizes the [N,K] activations to + // q8_1 and runs the int8 dot instead of the f16 half-dot. Large-batch + // (ne1>8) only. q5_0 weight = (x-16)*d (x = nibble | hi<<4); x packed as a + // 0..31 byte (dp4a), the -16 centering folded into a single min term + // (d*16) via the q8_1 block sum. Reads the qs/qh/d buffers byte-identically + // to the f16 kernel (greedy byte-identical, MUL_MAT NMSE-OK). + // + // OPT-IN / DEFAULT OFF. Unlike q8_0/q4_K dense, dp4a is not a win for q5_0 on + // X2E: the q5_0 model is bottlenecked elsewhere, so the dense-GEMM int8 win + // has nothing to surface and the q8_1 prepass slightly hurts. Kept correct + + // opt-in for the X1 A/B (different texture-cache dynamic) and the + // weight-texture variant. Env: GGML_OPENCL_Q5_DENSE_DP4A=1. + // Weight-as-texture variant (X1 lever): routes the dominant qs nibble plane + // through an image1d_buffer (qh stays a buffer). Opt-in + // GGML_OPENCL_Q5_DENSE_DP4A_WIMG; when set it also forces the dp4a path on. + static const char * q5_dense_dp4a_env = getenv("GGML_OPENCL_Q5_DENSE_DP4A"); + static const char * q5_dense_wimg_env = getenv("GGML_OPENCL_Q5_DENSE_DP4A_WIMG"); + const bool q5_dense_wimg_on = q5_dense_wimg_env && (atoi(q5_dense_wimg_env) != 0); + const bool q5_dense_dp4a_on = q5_dense_wimg_on + ? true + : (q5_dense_dp4a_env && (atoi(q5_dense_dp4a_env) != 0)); + if (q5_dense_dp4a_on && backend_ctx->kernel_gemm_noshuffle_q5_0_q8_1_dp4a + && N > 8 && (K % 32 == 0) && (M % 64 == 0)) { + cl_mem a_sub = nullptr; + region.origin = offset1; + region.size = (size_t)K * N * sizeof(float); + CL_CHECK((a_sub = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); + + const size_t n_blocks = (size_t)N * (K / 32); + backend_ctx->prealloc_moe_qa.allocate(context, (size_t)N * K * sizeof(cl_char)); + backend_ctx->prealloc_moe_da.allocate(context, n_blocks * sizeof(cl_half)); + backend_ctx->prealloc_moe_sa.allocate(context, n_blocks * sizeof(cl_half)); + + cl_int tb = (cl_int)n_blocks; + cl_kernel qk = backend_ctx->kernel_quant_a_q8_1; + CL_CHECK(clSetKernelArg(qk, 0, sizeof(cl_mem), &a_sub)); + CL_CHECK(clSetKernelArg(qk, 1, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer)); + CL_CHECK(clSetKernelArg(qk, 2, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer)); + CL_CHECK(clSetKernelArg(qk, 3, sizeof(cl_mem), &backend_ctx->prealloc_moe_sa.buffer)); + CL_CHECK(clSetKernelArg(qk, 4, sizeof(cl_int), &tb)); + size_t q_local[1] = { 64 }; + size_t q_global[1] = { (size_t)(((n_blocks + 63) / 64) * 64) }; + backend_ctx->enqueue_ndrange_kernel(qk, 1, q_global, q_local, dst); + + // optional qs texture (image1d_buffer over the nibble plane; the same + // CL_R/UINT32 view, width M*K/8, the GEMV path builds). + cl_mem q5_qs_img = nullptr; + bool use_wimg = q5_dense_wimg_on; + if (use_wimg) { + const size_t tex = (size_t)M * (size_t)K / 8; // uint32 texels (2 ushorts/texel) + if (tex == 0 || tex > backend_ctx->image_max_buffer_size) { + use_wimg = false; + } else { + img_fmt = { CL_R, CL_UNSIGNED_INT32 }; + memset(&img_desc, 0, sizeof(img_desc)); + img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; + img_desc.image_width = tex; + img_desc.buffer = extra0_q5_0->qs; + q5_qs_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, NULL, &err); + if (err != CL_SUCCESS || q5_qs_img == nullptr) { use_wimg = false; q5_qs_img = nullptr; } + } + } + + cl_kernel dk = use_wimg ? backend_ctx->kernel_gemm_noshuffle_q5_0_q8_1_dp4a_wimg + : backend_ctx->kernel_gemm_noshuffle_q5_0_q8_1_dp4a; + int ai = 0; + if (use_wimg) { + CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &q5_qs_img)); + } else { + CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &extra0_q5_0->qs)); + } + CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &extra0_q5_0->qh)); + CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &extra0_q5_0->d)); + CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer)); + CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer)); + CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &backend_ctx->prealloc_moe_sa.buffer)); + CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_int), &M)); + CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_int), &N)); + CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_int), &K)); + size_t d_local[3] = { 64, 1, 1 }; + size_t d_global[3] = { 64, (size_t)(M / 64), (size_t)CEIL_DIV(N, 32) }; + backend_ctx->enqueue_ndrange_kernel(dk, 3, d_global, d_local, dst); + + if (q5_qs_img != nullptr) { + CL_CHECK(clReleaseMemObject(q5_qs_img)); + } + CL_CHECK(clReleaseMemObject(a_sub)); + return; + } + cl_mem b_sub_buf = nullptr; cl_mem b_sub_buf_trans = nullptr; cl_mem b_img = nullptr; @@ -15985,6 +16706,59 @@ static void ggml_cl_mul_mat_iq4_nl_f32_adreno(ggml_backend_t backend, const ggml CL_CHECK(clReleaseMemObject(b_sub_buf)); CL_CHECK(clReleaseMemObject(b_img)); } else { + // dp4a (int8) dense IQ4_NL prefill GEMM. Quantizes the [N,K] activations to + // q8_1 and runs the int8 dot instead of the f16 half-dot. Large-batch + // (ne1>8) only. IQ4_NL weight = kvalues[nibble]*d; the codebook value IS the + // int8 (no min term), so this is the q8_0 dense case plus a nibble->int8 LUT + // unpack. Reads the q/d buffers byte-identically to the f16 kernel. No bin + // kernel for IQ4_NL -> baseline is f16, default ON for X2E (like q4_K/q6_K + // dense dp4a). X1 stays on f16. Env: GGML_OPENCL_IQ4NL_DENSE_DP4A. + static const char * iq4nl_dense_dp4a_env = getenv("GGML_OPENCL_IQ4NL_DENSE_DP4A"); + const bool iq4nl_dense_dp4a_on = iq4nl_dense_dp4a_env + ? (atoi(iq4nl_dense_dp4a_env) != 0) + : (backend_ctx->adreno_gen == ADRENO_GPU_GEN::X2E); + if (iq4nl_dense_dp4a_on && backend_ctx->kernel_gemm_noshuffle_iq4_nl_q8_1_dp4a + && N > 8 && (K % 32 == 0) && (M % 64 == 0)) { + cl_mem a_sub = nullptr; + region.origin = offset1; + region.size = (size_t)K * N * sizeof(float); + CL_CHECK((a_sub = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); + + const size_t n_blocks = (size_t)N * (K / 32); + backend_ctx->prealloc_moe_qa.allocate(context, (size_t)N * K * sizeof(cl_char)); + backend_ctx->prealloc_moe_da.allocate(context, n_blocks * sizeof(cl_half)); + backend_ctx->prealloc_moe_sa.allocate(context, n_blocks * sizeof(cl_half)); + + cl_int tb = (cl_int)n_blocks; + cl_kernel qk = backend_ctx->kernel_quant_a_q8_1; + CL_CHECK(clSetKernelArg(qk, 0, sizeof(cl_mem), &a_sub)); + CL_CHECK(clSetKernelArg(qk, 1, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer)); + CL_CHECK(clSetKernelArg(qk, 2, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer)); + CL_CHECK(clSetKernelArg(qk, 3, sizeof(cl_mem), &backend_ctx->prealloc_moe_sa.buffer)); + CL_CHECK(clSetKernelArg(qk, 4, sizeof(cl_int), &tb)); + size_t q_local[1] = { 64 }; + size_t q_global[1] = { (size_t)(((n_blocks + 63) / 64) * 64) }; + backend_ctx->enqueue_ndrange_kernel(qk, 1, q_global, q_local, dst); + + cl_kernel dk = backend_ctx->kernel_gemm_noshuffle_iq4_nl_q8_1_dp4a; + int ai = 0; + CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &extra0_iq4_nl->q)); + CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &extra0_iq4_nl->d)); + CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer)); + CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer)); + CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_int), &M)); + CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_int), &N)); + CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_int), &K)); + size_t d_local[3] = { 64, 1, 1 }; + size_t d_global[3] = { 64, (size_t)(M / 64), (size_t)CEIL_DIV(N, 32) }; + backend_ctx->enqueue_ndrange_kernel(dk, 3, d_global, d_local, dst); + + CL_CHECK(clReleaseMemObject(a_sub)); + return; + } + cl_mem b_sub_buf = nullptr; cl_mem b_sub_buf_trans = nullptr; cl_mem b_img = nullptr; @@ -16184,6 +16958,90 @@ static void ggml_cl_mul_mat_q8_0_f32_adreno(ggml_backend_t backend, const ggml_t CL_CHECK(clReleaseMemObject(b_img)); CL_CHECK(clReleaseMemObject(b_sub_buf)); } else { + // dp4a dense q8_0 prefill GEMM. Quantizes the [N,K] activations to + // q8_1 and runs the int8 dot instead of the f16 half-dot. Large-batch + // (ne1>8) only; q8_0 weights are already int8 (no requant) and symmetric + // (no min term) + static const char * q8_dense_dp4a_env = getenv("GGML_OPENCL_Q8_DENSE_DP4A"); + static const char * q8_dense_wimg_env = getenv("GGML_OPENCL_Q8_DENSE_DP4A_WIMG"); + const bool q8_dense_wimg_on = q8_dense_wimg_env && (atoi(q8_dense_wimg_env) != 0); + + const bool q8_bin_loaded = (backend_ctx->kernel_gemm_noshuffle_q8_0_f32_bin != nullptr); + // bin kernel takes precedence + const bool q8_dense_dp4a_on = q8_dense_wimg_on + ? true + : q8_dense_dp4a_env + ? (atoi(q8_dense_dp4a_env) != 0) + : (backend_ctx->adreno_gen == ADRENO_GPU_GEN::X2E && !q8_bin_loaded); + + if (q8_dense_dp4a_on && backend_ctx->kernel_gemm_noshuffle_q8_0_q8_1_dp4a + && N > 8 && (K % 32 == 0) && (M % 64 == 0)) { + cl_mem a_sub = nullptr; + region.origin = offset1; + region.size = (size_t)K * N * sizeof(float); + CL_CHECK((a_sub = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); + + const size_t n_blocks = (size_t)N * (K / 32); + backend_ctx->prealloc_moe_qa.allocate(context, (size_t)N * K * sizeof(cl_char)); + backend_ctx->prealloc_moe_da.allocate(context, n_blocks * sizeof(cl_half)); + backend_ctx->prealloc_moe_sa.allocate(context, n_blocks * sizeof(cl_half)); + + cl_int tb = (cl_int)n_blocks; + cl_kernel qk = backend_ctx->kernel_quant_a_q8_1; + CL_CHECK(clSetKernelArg(qk, 0, sizeof(cl_mem), &a_sub)); + CL_CHECK(clSetKernelArg(qk, 1, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer)); + CL_CHECK(clSetKernelArg(qk, 2, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer)); + CL_CHECK(clSetKernelArg(qk, 3, sizeof(cl_mem), &backend_ctx->prealloc_moe_sa.buffer)); + CL_CHECK(clSetKernelArg(qk, 4, sizeof(cl_int), &tb)); + size_t q_local[1] = { 64 }; + size_t q_global[1] = { (size_t)(((n_blocks + 63) / 64) * 64) }; + backend_ctx->enqueue_ndrange_kernel(qk, 1, q_global, q_local, dst); + + // optional weight texture, the same CL_R/UINT32 view, width M*K/4 + cl_mem q8_q_img = nullptr; + bool use_wimg = q8_dense_wimg_on; + if (use_wimg) { + const size_t tex = (size_t)M * (size_t)K / 4; // uint32 texels + if (tex == 0 || tex > backend_ctx->image_max_buffer_size) { + use_wimg = false; + } else { + img_fmt = { CL_R, CL_UNSIGNED_INT32 }; + memset(&img_desc, 0, sizeof(img_desc)); + img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; + img_desc.image_width = tex; + img_desc.buffer = extra0_q8_0->q; + q8_q_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, NULL, &err); + if (err != CL_SUCCESS || q8_q_img == nullptr) { use_wimg = false; q8_q_img = nullptr; } + } + } + + cl_kernel dk = use_wimg ? backend_ctx->kernel_gemm_noshuffle_q8_0_q8_1_dp4a_wimg + : backend_ctx->kernel_gemm_noshuffle_q8_0_q8_1_dp4a; + int ai = 0; + if (use_wimg) { + CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &q8_q_img)); + } else { + CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &extra0_q8_0->q)); + } + CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &extra0_q8_0->d)); + CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer)); + CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer)); + CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_int), &M)); + CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_int), &N)); + CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_int), &K)); + size_t d_local[3] = { 64, 1, 1 }; + size_t d_global[3] = { 64, (size_t)(M / 64), (size_t)CEIL_DIV(N, 32) }; + backend_ctx->enqueue_ndrange_kernel(dk, 3, d_global, d_local, dst); + + if (q8_q_img != nullptr) { + CL_CHECK(clReleaseMemObject(q8_q_img)); + } + CL_CHECK(clReleaseMemObject(a_sub)); + return; + } + // use bin kernel if available if (backend_ctx->kernel_gemm_noshuffle_q8_0_f32_bin) { int K_pad = K; @@ -16517,6 +17375,99 @@ static void ggml_cl_mul_mat_q4_k_f32_adreno(ggml_backend_t backend, const ggml_t size_t global_work_size_t[2] = { (size_t)width_B, (size_t)padded_height_B }; backend_ctx->enqueue_ndrange_kernel(kernel, 2, global_work_size_t, local_work_size_t, dst); + // dp4a (int8) dense prefill GEMM and weight via texture + static const char * q4k_dense_dp4a_env = getenv("GGML_OPENCL_Q4K_DENSE_DP4A"); + static const char * q4k_dense_wimg_env = getenv("GGML_OPENCL_Q4K_DENSE_DP4A_WIMG"); + + const bool q4k_dense_wimg_on = q4k_dense_wimg_env && (atoi(q4k_dense_wimg_env) != 0); + const bool q4k_dense_dp4a_on = q4k_dense_wimg_on + ? true + : q4k_dense_dp4a_env + ? (atoi(q4k_dense_dp4a_env) != 0) + : (backend_ctx->adreno_gen == ADRENO_GPU_GEN::X2E); + + // Min N for the dp4a prefill GEMM, default 9, i.e., ne1 > 8 + static const char * q4k_dp4a_minn_env = getenv("GGML_OPENCL_Q4K_DP4A_MINN"); + const int q4k_dp4a_minn = q4k_dp4a_minn_env ? atoi(q4k_dp4a_minn_env) : 9; + + if (q4k_dense_dp4a_on && N >= q4k_dp4a_minn && (K % 32 == 0) && (M % 64 == 0)) { + const size_t n_blocks = (size_t)N * (K / 32); + backend_ctx->prealloc_moe_qa.allocate(context, (size_t)N * K * sizeof(cl_char)); + backend_ctx->prealloc_moe_da.allocate(context, n_blocks * sizeof(cl_half)); + backend_ctx->prealloc_moe_sa.allocate(context, n_blocks * sizeof(cl_half)); + + cl_int tb = (cl_int)n_blocks; + cl_kernel qk = backend_ctx->kernel_quant_a_q8_1; + CL_CHECK(clSetKernelArg(qk, 0, sizeof(cl_mem), &b_sub_buf)); + CL_CHECK(clSetKernelArg(qk, 1, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer)); + CL_CHECK(clSetKernelArg(qk, 2, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer)); + CL_CHECK(clSetKernelArg(qk, 3, sizeof(cl_mem), &backend_ctx->prealloc_moe_sa.buffer)); + CL_CHECK(clSetKernelArg(qk, 4, sizeof(cl_int), &tb)); + size_t q_local[1] = { 64 }; + size_t q_global[1] = { (size_t)(((n_blocks + 63) / 64) * 64) }; + backend_ctx->enqueue_ndrange_kernel(qk, 1, q_global, q_local, dst); + + // check if weights go through texture + cl_mem q4k_q_img = nullptr; + bool use_wimg = q4k_dense_wimg_on; + if (use_wimg) { + const size_t tex = (size_t)M * (size_t)K / 8; // uint32 texels = bytes/4 + if (tex == 0 || tex > backend_ctx->image_max_buffer_size) { + use_wimg = false; + } else { + img_fmt = { CL_R, CL_UNSIGNED_INT32 }; + memset(&img_desc, 0, sizeof(img_desc)); + img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; + img_desc.image_width = tex; + img_desc.buffer = extra0_q4_k->q; + q4k_q_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, NULL, &err); + if (err != CL_SUCCESS || q4k_q_img == nullptr) { + use_wimg = false; + q4k_q_img = nullptr; + } + } + } + + cl_kernel dk = use_wimg ? backend_ctx->kernel_gemm_noshuffle_q4_k_q8_1_dp4a_wimg + : backend_ctx->kernel_gemm_noshuffle_q4_k_q8_1_dp4a; + int ai = 0; + if (use_wimg) { + CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &q4k_q_img)); + } else { + CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &extra0_q4_k->q)); + } + CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &extra0_q4_k->s)); + CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &extra0_q4_k->d)); + CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &extra0_q4_k->dm)); + CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer)); + CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer)); + CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &backend_ctx->prealloc_moe_sa.buffer)); + CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_int), &M)); + CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_int), &N)); + CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_int), &K)); + CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_uchar), &mask_d6)); + CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_uchar), &mask_d4)); + CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_uchar), &mask_hi2)); + // Must match the compile-time TILESIZE_N chosen at program build (per-device, + // X1E=8 else 32; env override). Same inputs -> same value. + int q4k_dp4a_ts = (backend_ctx->adreno_gen == ADRENO_GPU_GEN::X1E) ? 8 : 32; + if (const char * e = getenv("GGML_OPENCL_Q4K_DP4A_TS")) q4k_dp4a_ts = atoi(e); + size_t d_local[3] = { 64, 1, 1 }; + size_t d_global[3] = { 64, (size_t)(M / 64), (size_t)CEIL_DIV(N, q4k_dp4a_ts) }; + backend_ctx->enqueue_ndrange_kernel(dk, 3, d_global, d_local, dst); + + if (q4k_q_img != nullptr) { + CL_CHECK(clReleaseMemObject(q4k_q_img)); + } + CL_CHECK(clReleaseMemObject(b_sub_buf)); + CL_CHECK(clReleaseMemObject(b_sub_buf_trans)); + CL_CHECK(clReleaseMemObject(b_img)); + CL_CHECK(clReleaseMemObject(b_img_trans)); + return; + } + // gemm kernel = backend_ctx->kernel_gemm_noshuffle_q4_k_f32; int padded_N = N + padding; @@ -16655,6 +17606,54 @@ static void ggml_cl_mul_mat_q6_K_f32_adreno(ggml_backend_t backend, const ggml_t region.size = ne00 * ne1 * sizeof(float); CL_CHECK((b_sub_buf = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); + // dp4a (int8) dense q6_K prefill GEMM + static const char * q6k_dense_dp4a_env = getenv("GGML_OPENCL_Q6K_DENSE_DP4A"); + static const bool q6k_dense_dp4a_on = (q6k_dense_dp4a_env != nullptr) + ? (atoi(q6k_dense_dp4a_env) != 0) + : (backend_ctx->adreno_gen != ADRENO_GPU_GEN::X1E); + + const bool is_output_w_dp4a = strncmp(src0->name, "output", 6) == 0 || + strncmp(src0->name, "token_embd", 10) == 0; + + if (q6k_dense_dp4a_on && !is_output_w_dp4a && ne1 > 8 && (ne00 % 32 == 0) && (ne01 % 64 == 0)) { + const int M = ne01, N = ne1, K = ne00; + const size_t n_blocks = (size_t)N * (K / 32); + backend_ctx->prealloc_moe_qa.allocate(context, (size_t)N * K * sizeof(cl_char)); + backend_ctx->prealloc_moe_da.allocate(context, n_blocks * sizeof(cl_half)); + backend_ctx->prealloc_moe_sa.allocate(context, n_blocks * sizeof(cl_half)); + + cl_int tb = (cl_int)n_blocks; + cl_kernel qk = backend_ctx->kernel_quant_a_q8_1; + CL_CHECK(clSetKernelArg(qk, 0, sizeof(cl_mem), &b_sub_buf)); + CL_CHECK(clSetKernelArg(qk, 1, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer)); + CL_CHECK(clSetKernelArg(qk, 2, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer)); + CL_CHECK(clSetKernelArg(qk, 3, sizeof(cl_mem), &backend_ctx->prealloc_moe_sa.buffer)); + CL_CHECK(clSetKernelArg(qk, 4, sizeof(cl_int), &tb)); + size_t q_local[1] = { 64 }; + size_t q_global[1] = { (size_t)(((n_blocks + 63) / 64) * 64) }; + backend_ctx->enqueue_ndrange_kernel(qk, 1, q_global, q_local, dst); + + cl_kernel dk = backend_ctx->kernel_gemm_noshuffle_q6_k_q8_1_dp4a; + int ai = 0; + CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &extra0_q6_K->ql)); + CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &extra0_q6_K->qh)); + CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &extra0_q6_K->s)); + CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &extra0_q6_K->d)); + CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer)); + CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer)); + CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_int), &M)); + CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_int), &N)); + CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_int), &K)); + size_t d_local[3] = { 64, 1, 1 }; + size_t d_global[3] = { 64, (size_t)(M / 64), (size_t)CEIL_DIV(N, 32) }; + backend_ctx->enqueue_ndrange_kernel(dk, 3, d_global, d_local, dst); + + CL_CHECK(clReleaseMemObject(b_sub_buf)); + return; + } + // image for activation img_fmt.image_channel_order = CL_RGBA; img_fmt.image_channel_data_type = CL_FLOAT; @@ -16900,6 +17899,59 @@ static void ggml_cl_mul_mat_q5_K_f32_adreno(ggml_backend_t backend, const ggml_t size_t global_work_size_t[2] = {(size_t)width_B, (size_t)padded_height_B}; backend_ctx->enqueue_ndrange_kernel(kernel, 2, global_work_size_t, local_work_size_t, dst); + // dp4a (int8) dense q5_K prefill GEMM + static const char * q5k_dense_dp4a_env = getenv("GGML_OPENCL_Q5K_DENSE_DP4A"); + const bool q5k_dense_dp4a_on = q5k_dense_dp4a_env + ? (atoi(q5k_dense_dp4a_env) != 0) + : (backend_ctx->adreno_gen == ADRENO_GPU_GEN::X2E); + + if (q5k_dense_dp4a_on && ne1 > 8 && (ne00 % 32 == 0) && (ne01 % 64 == 0)) { + const int Mm = ne01, Nn = ne1, Kk = ne00; + const size_t n_blocks = (size_t)Nn * (Kk / 32); + backend_ctx->prealloc_moe_qa.allocate(context, (size_t)Nn * Kk * sizeof(cl_char)); + backend_ctx->prealloc_moe_da.allocate(context, n_blocks * sizeof(cl_half)); + backend_ctx->prealloc_moe_sa.allocate(context, n_blocks * sizeof(cl_half)); + + cl_int tb = (cl_int)n_blocks; + cl_kernel qk = backend_ctx->kernel_quant_a_q8_1; + CL_CHECK(clSetKernelArg(qk, 0, sizeof(cl_mem), &b_sub_buf)); + CL_CHECK(clSetKernelArg(qk, 1, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer)); + CL_CHECK(clSetKernelArg(qk, 2, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer)); + CL_CHECK(clSetKernelArg(qk, 3, sizeof(cl_mem), &backend_ctx->prealloc_moe_sa.buffer)); + CL_CHECK(clSetKernelArg(qk, 4, sizeof(cl_int), &tb)); + size_t q_local[1] = { 64 }; + size_t q_global[1] = { (size_t)(((n_blocks + 63) / 64) * 64) }; + backend_ctx->enqueue_ndrange_kernel(qk, 1, q_global, q_local, dst); + + cl_kernel dk = backend_ctx->kernel_gemm_noshuffle_q5_k_q8_1_dp4a; + int ai = 0; + CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &extra0_q5_k->q)); + CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &extra0_q5_k->qh)); + CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &extra0_q5_k->s)); + CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &extra0_q5_k->d)); + CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &extra0_q5_k->dm)); + CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer)); + CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer)); + CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &backend_ctx->prealloc_moe_sa.buffer)); + CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_int), &Mm)); + CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_int), &Nn)); + CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_int), &Kk)); + CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_uchar), &mask_d6)); + CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_uchar), &mask_d4)); + CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_uchar), &mask_hi2)); + size_t d_local[3] = { 64, 1, 1 }; + size_t d_global[3] = { 64, (size_t)(Mm / 64), (size_t)CEIL_DIV(Nn, 32) }; + backend_ctx->enqueue_ndrange_kernel(dk, 3, d_global, d_local, dst); + + CL_CHECK(clReleaseMemObject(b_sub_buf)); + CL_CHECK(clReleaseMemObject(b_sub_buf_trans)); + CL_CHECK(clReleaseMemObject(b_img)); + CL_CHECK(clReleaseMemObject(b_img_trans)); + return; + } + // gemm kernel = backend_ctx->kernel_gemm_noshuffle_q5_k_f32; int padded_N = N + padding; @@ -19380,6 +20432,23 @@ static void moe_router_reoerder(ggml_backend_t backend, const ggml_tensor * src, backend_ctx->enqueue_ndrange_kernel(kernel, 3, histogram_global_size, histogram_local_size, src); + // [MOE_TILES] env-gated padding probe: read back total_tiles (= Sum_e + // ceil(k_e/n_tile_size)) and compare to the ideal tile count for the real + // routing count. Quantifies the per-expert tile-padding waste. Blocking + // readback perturbs timing -> diagnostic only. + if (getenv("GGML_OPENCL_MOE_TILES_DEBUG")) { + int h_total = 0; + clFinish(backend_ctx->queue); + CL_CHECK(clEnqueueReadBuffer(backend_ctx->queue, total_tiles_buf, CL_TRUE, 0, sizeof(int), &h_total, 0, NULL, NULL)); + const int routings = ne20 * ne21; + const int ideal = (routings + n_tile_size - 1) / n_tile_size; + const int slots = h_total * n_tile_size; + fprintf(stderr, "[MOE_TILES] routings=%d (ne20=%d ne21=%d nexp=%d) total_tiles=%d ideal=%d slots=%d pad=%.1f%%\n", + routings, ne20, ne21, ne02, h_total, ideal, slots, + routings > 0 ? 100.0 * (slots - routings) / routings : 0.0); + fflush(stderr); + } + CL_CHECK(clReleaseMemObject(original_router_buf)); CL_CHECK(clReleaseMemObject(hist_buf)); CL_CHECK(clReleaseMemObject(tile_offset_buf)); @@ -19562,9 +20631,18 @@ static void ggml_cl_mul_mat_id(ggml_backend_t backend, const ggml_tensor * src0, backend_ctx->toggle_reorder = false; } - cl_mem sub_buf_src1_pre, buf_src1_reordered, image_src1_reordered, sub_buf_dst, buf_dst_image; + cl_mem sub_buf_src1_pre, sub_buf_dst, buf_dst_image; + cl_mem buf_src1_reordered = nullptr, image_src1_reordered = nullptr; cl_mem buf_src2, buf_src2_emap; + // dp4a (int8) prefill GEMM variant + static const char * q4_0_moe_dp4a_env = getenv("GGML_OPENCL_Q4_0_MOE_DP4A"); + bool use_moe_dp4a = q4_0_moe_dp4a_env + ? (atoi(q4_0_moe_dp4a_env) != 0) + : (backend_ctx->adreno_gen == ADRENO_GPU_GEN::X2E); + // bin kernel takes precedence + use_moe_dp4a = use_moe_dp4a && backend_ctx->kernel_gemm_moe_q4_0_f32_ns_bin == nullptr; + cl_buffer_region region; region.origin = 0; region.size = sizeof(int) * max_post_router_tile * n_tile_size; @@ -19583,45 +20661,48 @@ static void ggml_cl_mul_mat_id(ggml_backend_t backend, const ggml_tensor * src0, sub_buf_src1_pre = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &status); CL_CHECK(status); - // Create image for reordered src1 - // Use pre-allocated placeholder - region.origin = 0; - region.size = ne00 * max_post_router_tile * n_tile_size * sizeof(float); - backend_ctx->prealloc_act_trans.allocate(backend_ctx->context, region.size); - buf_src1_reordered = clCreateSubBuffer( - backend_ctx->prealloc_act_trans.buffer, - 0, - CL_BUFFER_CREATE_TYPE_REGION, - ®ion, - &status); - CL_CHECK(status); - cl_image_format image_format_buf_src1; - cl_image_desc image_desc_buf_src1; - image_format_buf_src1 = {CL_RGBA, CL_FLOAT}; - image_desc_buf_src1 = {CL_MEM_OBJECT_IMAGE1D_BUFFER, static_cast(ne00 * max_post_router_tile * n_tile_size / 4), 0,0,0,0,0,0,0, {buf_src1_reordered}}; - if (backend_ctx->kernel_gemm_moe_q4_0_f32_ns_bin) { - // bin kernel uses slightly different image format - image_format_buf_src1 = {CL_R, CL_FLOAT}; - image_desc_buf_src1.image_width = static_cast(ne00 * max_post_router_tile * n_tile_size); - } - image_src1_reordered = clCreateImage(backend_ctx->context, CL_MEM_READ_ONLY, &image_format_buf_src1, &image_desc_buf_src1, NULL, &status); - CL_CHECK(status); - unsigned short map_ratio = ne20 / ne11; GGML_ASSERT(((map_ratio == 1) || (map_ratio == ne20)) && "Map ratio not supported\n"); - CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 0, sizeof(cl_mem), &sub_buf_src1_pre)); - CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 1, sizeof(cl_mem), &buf_src2)); - CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 2, sizeof(cl_mem), &buf_src1_reordered)); - CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 3, sizeof(cl_mem), &(backend_ctx->prealloc_total_tiles.buffer))); - CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 4, sizeof(unsigned int), &ne00)); - CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 5, sizeof(unsigned short), &map_ratio)); - CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 6, sizeof(unsigned int), &n_tile_size)); - size_t reorder_b_local_size[3] = {256, 1, 1}; - size_t reorder_b_global_size[3] = {static_cast(((ne00 / 4) + 255) / 256 * 256), static_cast(max_post_router_tile * n_tile_size), 1}; + if (!use_moe_dp4a) { + // Create image for reordered src1 + // Use pre-allocated placeholder + region.origin = 0; + region.size = ne00 * max_post_router_tile * n_tile_size * sizeof(float); + backend_ctx->prealloc_act_trans.allocate(backend_ctx->context, region.size); + buf_src1_reordered = clCreateSubBuffer( + backend_ctx->prealloc_act_trans.buffer, + 0, + CL_BUFFER_CREATE_TYPE_REGION, + ®ion, + &status); + CL_CHECK(status); + cl_image_format image_format_buf_src1; + cl_image_desc image_desc_buf_src1; + image_format_buf_src1 = {CL_RGBA, CL_FLOAT}; + image_desc_buf_src1 = {CL_MEM_OBJECT_IMAGE1D_BUFFER, static_cast(ne00 * max_post_router_tile * n_tile_size / 4), 0,0,0,0,0,0,0, {buf_src1_reordered}}; + if (backend_ctx->kernel_gemm_moe_q4_0_f32_ns_bin) { + // bin kernel uses slightly different image format + image_format_buf_src1 = {CL_R, CL_FLOAT}; + image_desc_buf_src1.image_width = static_cast(ne00 * max_post_router_tile * n_tile_size); + } + image_src1_reordered = clCreateImage(backend_ctx->context, CL_MEM_READ_ONLY, &image_format_buf_src1, &image_desc_buf_src1, NULL, &status); + CL_CHECK(status); - // Dispatch reorder kernel - backend_ctx->enqueue_ndrange_kernel(backend_ctx->kernel_moe_reorder_b, 3, reorder_b_global_size, reorder_b_local_size, dst); + CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 0, sizeof(cl_mem), &sub_buf_src1_pre)); + CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 1, sizeof(cl_mem), &buf_src2)); + CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 2, sizeof(cl_mem), &buf_src1_reordered)); + CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 3, sizeof(cl_mem), &(backend_ctx->prealloc_total_tiles.buffer))); + CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 4, sizeof(unsigned int), &ne00)); + CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 5, sizeof(unsigned short), &map_ratio)); + CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 6, sizeof(unsigned int), &n_tile_size)); + + size_t reorder_b_local_size[3] = {256, 1, 1}; + size_t reorder_b_global_size[3] = {static_cast(((ne00 / 4) + 255) / 256 * 256), static_cast(max_post_router_tile * n_tile_size), 1}; + + // Dispatch reorder kernel + backend_ctx->enqueue_ndrange_kernel(backend_ctx->kernel_moe_reorder_b, 3, reorder_b_global_size, reorder_b_local_size, dst); + } // MoE kernel prepare // Create sub buffer for dst @@ -19640,6 +20721,58 @@ static void ggml_cl_mul_mat_id(ggml_backend_t backend, const ggml_tensor * src0, buf_dst_image = clCreateImage(backend_ctx->context, CL_MEM_WRITE_ONLY, &image_format_buf_dst, &image_desc_buf_dst, NULL, &status); CL_CHECK(status); + if (use_moe_dp4a) { + const size_t tok_slots = (size_t)max_post_router_tile * n_tile_size; + const size_t n_blocks = tok_slots * (ne00 / 32); + backend_ctx->prealloc_moe_qa.allocate(backend_ctx->context, tok_slots * ne00 * sizeof(cl_char)); + backend_ctx->prealloc_moe_da.allocate(backend_ctx->context, n_blocks * sizeof(cl_half)); + backend_ctx->prealloc_moe_sa.allocate(backend_ctx->context, n_blocks * sizeof(cl_half)); + + // fused reorder + q8_1 quant straight from the original activations + const cl_uint n_kblocks = (cl_uint)(ne00 / 32); + cl_kernel rq = backend_ctx->kernel_moe_reorder_quant_a_q8_1; + CL_CHECK(clSetKernelArg(rq, 0, sizeof(cl_mem), &sub_buf_src1_pre)); + CL_CHECK(clSetKernelArg(rq, 1, sizeof(cl_mem), &buf_src2)); + CL_CHECK(clSetKernelArg(rq, 2, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer)); + CL_CHECK(clSetKernelArg(rq, 3, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer)); + CL_CHECK(clSetKernelArg(rq, 4, sizeof(cl_mem), &backend_ctx->prealloc_moe_sa.buffer)); + CL_CHECK(clSetKernelArg(rq, 5, sizeof(cl_mem), &(backend_ctx->prealloc_total_tiles.buffer))); + CL_CHECK(clSetKernelArg(rq, 6, sizeof(cl_uint), &ne00)); + CL_CHECK(clSetKernelArg(rq, 7, sizeof(unsigned short), &map_ratio)); + CL_CHECK(clSetKernelArg(rq, 8, sizeof(cl_uint), &n_tile_size)); + CL_CHECK(clSetKernelArg(rq, 9, sizeof(cl_uint), &n_kblocks)); + size_t rq_local[2] = { 32, 1 }; + size_t rq_global[2] = { (size_t)(((n_kblocks + 31) / 32) * 32), tok_slots }; + backend_ctx->enqueue_ndrange_kernel(rq, 2, rq_global, rq_local, dst); + + // dp4a GEMM + cl_kernel dk = backend_ctx->kernel_gemm_moe_q4_0_q8_1_dp4a; + int aidx = 0; + CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &extra0_q4_0->q_img)); + CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &extra0_q4_0->d)); + CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer)); + CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer)); + CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &backend_ctx->prealloc_moe_sa.buffer)); + CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &buf_src2)); + CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &buf_src2_emap)); + CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &buf_dst_image)); + CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &(backend_ctx->prealloc_total_tiles.buffer))); + CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(int), &ne00)); + CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(int), &ne01)); + CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(int), &backend_ctx->adreno_use_moe_ragged_dp4)); + + size_t dp_global[3] = { 64, (size_t)((ne01 + 63) / 64), (size_t)max_post_router_tile }; + size_t dp_local[3] = { 64, 1, 1 }; + backend_ctx->enqueue_ndrange_kernel(dk, 3, dp_global, dp_local, dst); + + clReleaseMemObject(sub_buf_src1_pre); + clReleaseMemObject(buf_src2); + clReleaseMemObject(buf_src2_emap); + clReleaseMemObject(sub_buf_dst); + clReleaseMemObject(buf_dst_image); + return; + } + // Set kernel args int arg_idx = 0; CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &extra0_q4_0->q_img)); @@ -19993,6 +21126,81 @@ static void ggml_cl_mul_mat_id(ggml_backend_t backend, const ggml_tensor * src0, sub_buf_src1_pre = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &status); CL_CHECK(status); + // Generic dp4a MoE GEMM + { + static const char * q5mdp4a_env = getenv("GGML_OPENCL_Q5_MOE_DP4A"); + const bool q5mdp4a_on = q5mdp4a_env ? (atoi(q5mdp4a_env) != 0) + : (backend_ctx->adreno_gen == ADRENO_GPU_GEN::X2E); + const bool use_q5_moe_dp4a = q5mdp4a_on + && backend_ctx->kernel_gemm_moe_q8_1_dp4a_q50 != nullptr + && extra0_q5_0->scale != nullptr; + + if (use_q5_moe_dp4a) { + const size_t tok_slots = (size_t)max_post_router_tile * n_tile_size; + const size_t n_blocks = tok_slots * (ne00 / 32); + backend_ctx->prealloc_moe_qa.allocate(backend_ctx->context, tok_slots * ne00 * sizeof(cl_char)); + backend_ctx->prealloc_moe_da.allocate(backend_ctx->context, n_blocks * sizeof(cl_half)); + backend_ctx->prealloc_moe_sa.allocate(backend_ctx->context, n_blocks * sizeof(cl_half)); + + const cl_uint n_kblocks = (cl_uint)(ne00 / 32); + unsigned short map_ratio_q5 = ne20 / ne11; + cl_kernel rq = backend_ctx->kernel_moe_reorder_quant_a_q8_1; + CL_CHECK(clSetKernelArg(rq, 0, sizeof(cl_mem), &sub_buf_src1_pre)); + CL_CHECK(clSetKernelArg(rq, 1, sizeof(cl_mem), &buf_src2)); + CL_CHECK(clSetKernelArg(rq, 2, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer)); + CL_CHECK(clSetKernelArg(rq, 3, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer)); + CL_CHECK(clSetKernelArg(rq, 4, sizeof(cl_mem), &backend_ctx->prealloc_moe_sa.buffer)); + CL_CHECK(clSetKernelArg(rq, 5, sizeof(cl_mem), &(backend_ctx->prealloc_total_tiles.buffer))); + CL_CHECK(clSetKernelArg(rq, 6, sizeof(cl_uint), &ne00)); + CL_CHECK(clSetKernelArg(rq, 7, sizeof(unsigned short), &map_ratio_q5)); + CL_CHECK(clSetKernelArg(rq, 8, sizeof(cl_uint), &n_tile_size)); + CL_CHECK(clSetKernelArg(rq, 9, sizeof(cl_uint), &n_kblocks)); + size_t rq_local[2] = { 32, 1 }; + size_t rq_global[2] = { (size_t)(((n_kblocks + 31) / 32) * 32), tok_slots }; + backend_ctx->enqueue_ndrange_kernel(rq, 2, rq_global, rq_local, dst); + + region.origin = offsetd; + region.size = ne0 * ne1 * ne2 * sizeof(float); + cl_mem dp_sub_buf_dst = clCreateSubBuffer(extrad->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &status); + CL_CHECK(status); + cl_image_format dp_ifd = {CL_R, CL_FLOAT}; + cl_image_desc dp_idd = {CL_MEM_OBJECT_IMAGE1D_BUFFER, static_cast(ne0 * ne1 * ne2), 0,0,0,0,0,0,0, {dp_sub_buf_dst}}; + cl_mem dp_buf_dst_image = clCreateImage(backend_ctx->context, CL_MEM_WRITE_ONLY, &dp_ifd, &dp_idd, NULL, &status); + CL_CHECK(status); + + int ne00i = (int)ne00, ne01i = (int)ne01; + cl_kernel dk = backend_ctx->kernel_gemm_moe_q8_1_dp4a_q50; + int has_min_q5 = 1; + int aidx = 0; + CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &extra0_q5_0->qs_img)); + CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &extra0_q5_0->qh)); + CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &extra0_q5_0->scale)); + CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &extra0_q5_0->min)); + CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer)); + CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer)); + CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &backend_ctx->prealloc_moe_sa.buffer)); + CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &buf_src2)); + CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &buf_src2_emap)); + CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &dp_buf_dst_image)); + CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &(backend_ctx->prealloc_total_tiles.buffer))); + CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(int), &ne00i)); + CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(int), &ne01i)); + CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(int), &backend_ctx->adreno_use_moe_ragged_dp4)); + CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(int), &has_min_q5)); + + size_t dp_global[3] = { 64, (size_t)((ne01 + 63) / 64), (size_t)max_post_router_tile }; + size_t dp_local[3] = { 64, 1, 1 }; + backend_ctx->enqueue_ndrange_kernel(dk, 3, dp_global, dp_local, dst); + + clReleaseMemObject(sub_buf_src1_pre); + clReleaseMemObject(buf_src2); + clReleaseMemObject(buf_src2_emap); + clReleaseMemObject(dp_sub_buf_dst); + clReleaseMemObject(dp_buf_dst_image); + return; + } + } + // Create image for reordered src1 // Use pre-allocated placeholder region.origin = 0; @@ -20261,6 +21469,183 @@ static void ggml_cl_mul_mat_id(ggml_backend_t backend, const ggml_tensor * src0, #endif //GGML_OPENCL_USE_ADRENO_KERNELS } case GGML_TYPE_Q8_0: { +#ifdef GGML_OPENCL_USE_ADRENO_KERNELS + // MoE GEMM for q8_0 at prefill (ne12>1) + // There is no corresponding gemv_moe, so the code path is different here + static const char * moe_gemm_q8_env = getenv("GGML_OPENCL_MOE_GEMM_Q8"); + const bool moe_gemm_q8 = moe_gemm_q8_env + ? (atoi(moe_gemm_q8_env) != 0) + : (backend_ctx->adreno_gen == ADRENO_GPU_GEN::X2E); + if (moe_gemm_q8 && use_adreno_moe_kernels(backend_ctx, src0) && ne12 > 1) { + cl_int status; + + size_t local_size[3] = {64, 2, 1}; + size_t global_size[3] = {64, 2, 1}; + + kernel = backend_ctx->kernel_gemm_moe_q8_0_f32_ns; + + if ((strstr(src0->name, "as") != NULL) || backend_ctx->toggle_reorder) { + moe_router_reoerder(backend, src2, ne20); + backend_ctx->toggle_reorder = false; + } + + cl_mem sub_buf_src1_pre, buf_src1_reordered, image_src1_reordered, sub_buf_dst, buf_dst_image; + cl_mem buf_src2, buf_src2_emap; + + cl_buffer_region region; + region.origin = 0; + region.size = sizeof(int) * max_post_router_tile * n_tile_size; + buf_src2 = clCreateSubBuffer(backend_ctx->prealloc_post_router.buffer, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &status); + CL_CHECK(status); + + region.origin = 0; + region.size = sizeof(short) * max_post_router_tile; + buf_src2_emap = clCreateSubBuffer(backend_ctx->prealloc_emap.buffer, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &status); + CL_CHECK(status); + + // Reorder activations (group tokens by expert into tiles of 32) + region.origin = offset1; + region.size = ne10 * ne11 * ne12 * sizeof(float); + sub_buf_src1_pre = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &status); + CL_CHECK(status); + + // Generic dp4a MoE GEMM + { + static const char * q8mdp4a_env = getenv("GGML_OPENCL_Q8_MOE_DP4A"); + const bool q8mdp4a_on = q8mdp4a_env ? (atoi(q8mdp4a_env) != 0) + : (backend_ctx->adreno_gen == ADRENO_GPU_GEN::X2E); + const bool use_q8_moe_dp4a = q8mdp4a_on + && backend_ctx->kernel_gemm_moe_q8_1_dp4a_q80 != nullptr + && extra0_q8_0->scale != nullptr; + if (use_q8_moe_dp4a) { + const size_t tok_slots = (size_t)max_post_router_tile * n_tile_size; + const size_t n_blocks = tok_slots * (ne00 / 32); + backend_ctx->prealloc_moe_qa.allocate(backend_ctx->context, tok_slots * ne00 * sizeof(cl_char)); + backend_ctx->prealloc_moe_da.allocate(backend_ctx->context, n_blocks * sizeof(cl_half)); + backend_ctx->prealloc_moe_sa.allocate(backend_ctx->context, n_blocks * sizeof(cl_half)); + + const cl_uint n_kblocks = (cl_uint)(ne00 / 32); + unsigned short map_ratio_q8 = ne20 / ne11; + cl_kernel rq = backend_ctx->kernel_moe_reorder_quant_a_q8_1; + CL_CHECK(clSetKernelArg(rq, 0, sizeof(cl_mem), &sub_buf_src1_pre)); + CL_CHECK(clSetKernelArg(rq, 1, sizeof(cl_mem), &buf_src2)); + CL_CHECK(clSetKernelArg(rq, 2, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer)); + CL_CHECK(clSetKernelArg(rq, 3, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer)); + CL_CHECK(clSetKernelArg(rq, 4, sizeof(cl_mem), &backend_ctx->prealloc_moe_sa.buffer)); + CL_CHECK(clSetKernelArg(rq, 5, sizeof(cl_mem), &(backend_ctx->prealloc_total_tiles.buffer))); + CL_CHECK(clSetKernelArg(rq, 6, sizeof(cl_uint), &ne00)); + CL_CHECK(clSetKernelArg(rq, 7, sizeof(unsigned short), &map_ratio_q8)); + CL_CHECK(clSetKernelArg(rq, 8, sizeof(cl_uint), &n_tile_size)); + CL_CHECK(clSetKernelArg(rq, 9, sizeof(cl_uint), &n_kblocks)); + size_t rq_local[2] = { 32, 1 }; + size_t rq_global[2] = { (size_t)(((n_kblocks + 31) / 32) * 32), tok_slots }; + backend_ctx->enqueue_ndrange_kernel(rq, 2, rq_global, rq_local, dst); + + // dst image + region.origin = offsetd; + region.size = ne0 * ne1 * ne2 * sizeof(float); + cl_mem dp_sub_buf_dst = clCreateSubBuffer(extrad->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &status); + CL_CHECK(status); + cl_image_format dp_ifd = {CL_R, CL_FLOAT}; + cl_image_desc dp_idd = {CL_MEM_OBJECT_IMAGE1D_BUFFER, static_cast(ne0 * ne1 * ne2), 0,0,0,0,0,0,0, {dp_sub_buf_dst}}; + cl_mem dp_buf_dst_image = clCreateImage(backend_ctx->context, CL_MEM_WRITE_ONLY, &dp_ifd, &dp_idd, NULL, &status); + CL_CHECK(status); + + int ne00i = (int)ne00, ne01i = (int)ne01; + cl_kernel dk = backend_ctx->kernel_gemm_moe_q8_1_dp4a_q80; + int has_min_q8 = 0; + int aidx = 0; + CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &extra0_q8_0->q)); // flat int8 codes [expert][row][K] + CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &extra0_q8_0->scale)); // uniform scale[16] + CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &extra0_q8_0->scale)); // dummy min (has_min=0, unread) + CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer)); + CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer)); + CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &backend_ctx->prealloc_moe_sa.buffer)); + CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &buf_src2)); + CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &buf_src2_emap)); + CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &dp_buf_dst_image)); + CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &(backend_ctx->prealloc_total_tiles.buffer))); + CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(int), &ne00i)); + CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(int), &ne01i)); + CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(int), &backend_ctx->adreno_use_moe_ragged_dp4)); + CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(int), &has_min_q8)); + + size_t dp_global[3] = { 64, (size_t)((ne01 + 63) / 64), (size_t)max_post_router_tile }; + size_t dp_local[3] = { 64, 1, 1 }; + backend_ctx->enqueue_ndrange_kernel(dk, 3, dp_global, dp_local, dst); + + clReleaseMemObject(sub_buf_src1_pre); + clReleaseMemObject(buf_src2); + clReleaseMemObject(buf_src2_emap); + clReleaseMemObject(dp_sub_buf_dst); + clReleaseMemObject(dp_buf_dst_image); + return; + } + } + + region.origin = 0; + region.size = ne00 * max_post_router_tile * n_tile_size * sizeof(float); + backend_ctx->prealloc_act_trans.allocate(backend_ctx->context, region.size); + buf_src1_reordered = clCreateSubBuffer( + backend_ctx->prealloc_act_trans.buffer, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &status); + CL_CHECK(status); + cl_image_format image_format_buf_src1 = {CL_RGBA, CL_FLOAT}; + cl_image_desc image_desc_buf_src1 = {CL_MEM_OBJECT_IMAGE1D_BUFFER, static_cast(ne00 * max_post_router_tile * n_tile_size / 4), 0,0,0,0,0,0,0, {buf_src1_reordered}}; + image_src1_reordered = clCreateImage(backend_ctx->context, CL_MEM_READ_ONLY, &image_format_buf_src1, &image_desc_buf_src1, NULL, &status); + CL_CHECK(status); + + unsigned short map_ratio = ne20 / ne11; + GGML_ASSERT(((map_ratio == 1) || (map_ratio == ne20)) && "Map ratio not supported\n"); + CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 0, sizeof(cl_mem), &sub_buf_src1_pre)); + CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 1, sizeof(cl_mem), &buf_src2)); + CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 2, sizeof(cl_mem), &buf_src1_reordered)); + CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 3, sizeof(cl_mem), &(backend_ctx->prealloc_total_tiles.buffer))); + CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 4, sizeof(unsigned int), &ne00)); + CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 5, sizeof(unsigned short), &map_ratio)); + CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 6, sizeof(unsigned int), &n_tile_size)); + + size_t reorder_b_local_size[3] = {256, 1, 1}; + size_t reorder_b_global_size[3] = {static_cast(((ne00 / 4) + 255) / 256 * 256), static_cast(max_post_router_tile * n_tile_size), 1}; + backend_ctx->enqueue_ndrange_kernel(backend_ctx->kernel_moe_reorder_b, 3, reorder_b_global_size, reorder_b_local_size, dst); + + // dst image + region.origin = offsetd; + region.size = ne0 * ne1 * ne2 * sizeof(float); + sub_buf_dst = clCreateSubBuffer(extrad->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &status); + CL_CHECK(status); + cl_image_format image_format_buf_dst = {CL_R, CL_FLOAT}; + cl_image_desc image_desc_buf_dst = {CL_MEM_OBJECT_IMAGE1D_BUFFER, static_cast(ne0 * ne1 * ne2), 0,0,0,0,0,0,0, {sub_buf_dst}}; + buf_dst_image = clCreateImage(backend_ctx->context, CL_MEM_WRITE_ONLY, &image_format_buf_dst, &image_desc_buf_dst, NULL, &status); + CL_CHECK(status); + + int arg_idx = 0; + CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &extra0_q8_0->q)); // flat q8_0 quants + CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &extra0_q8_0->d)); // flat q8_0 scales + CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &image_src1_reordered)); + CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &buf_src2)); + CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &buf_src2_emap)); + CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &buf_dst_image)); + CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &(backend_ctx->prealloc_total_tiles.buffer))); + CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(int), &ne01)); + + global_size[1] = static_cast((ne01 + 63) / 64); + global_size[2] = static_cast(max_post_router_tile); + local_size[1] = 1; + local_size[2] = 1; + + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_size, local_size, dst); + + clReleaseMemObject(sub_buf_src1_pre); + clReleaseMemObject(buf_src1_reordered); + clReleaseMemObject(image_src1_reordered); + clReleaseMemObject(buf_src2); + clReleaseMemObject(buf_src2_emap); + clReleaseMemObject(sub_buf_dst); + clReleaseMemObject(buf_dst_image); + return; + } +#endif // GGML_OPENCL_USE_ADRENO_KERNELS #ifdef GGML_OPENCL_SOA_Q kernel = backend_ctx->kernel_mul_mv_id_q8_0_f32_flat; @@ -20347,6 +21732,18 @@ static void ggml_cl_mul_mat_id(ggml_backend_t backend, const ggml_tensor * src0, if (ne12 == 1) { // for gemv kernel = backend_ctx->kernel_gemv_moe_q4_k_f32_ns; + // Weight-as-texture MoE decode GEMV + static const char * moe_decode_wimg_env = getenv("GGML_OPENCL_MOE_DECODE_WIMG"); + const bool moe_decode_wimg_on = moe_decode_wimg_env + ? (atoi(moe_decode_wimg_env) != 0) + : (backend_ctx->adreno_gen == ADRENO_GPU_GEN::X2E); + const bool use_moe_decode_wimg = moe_decode_wimg_on + && backend_ctx->kernel_gemv_moe_q4_k_f32_ns_wimg != nullptr + && extra0_q4_K->q_img != nullptr; + if (use_moe_decode_wimg) { + kernel = backend_ctx->kernel_gemv_moe_q4_k_f32_ns_wimg; + } + cl_mem src1_sub_buffer, buf_src1_image, buf_src2; // create a sub_buffer for src2 @@ -20376,7 +21773,7 @@ static void ggml_cl_mul_mat_id(ggml_backend_t backend, const ggml_tensor * src0, // Set kernel args int arg_idx = 0; - CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &extra0_q4_K->q)); + CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), use_moe_decode_wimg ? &extra0_q4_K->q_img : &extra0_q4_K->q)); CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &extra0_q4_K->d)); CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &extra0_q4_K->dm)); CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &extra0_q4_K->s)); @@ -20409,9 +21806,18 @@ static void ggml_cl_mul_mat_id(ggml_backend_t backend, const ggml_tensor * src0, backend_ctx->toggle_reorder = false; } - cl_mem sub_buf_src1_pre, buf_src1_reordered, image_src1_reordered, sub_buf_dst, buf_dst_image; + cl_mem sub_buf_src1_pre, sub_buf_dst, buf_dst_image; + cl_mem buf_src1_reordered = nullptr, image_src1_reordered = nullptr; cl_mem buf_src2, buf_src2_emap; + // dp4a (int8) prefill GEMM variant + static const char * q4k_moe_dp4a_env = getenv("GGML_OPENCL_Q4K_MOE_DP4A"); + bool use_moe_dp4a = (q4k_moe_dp4a_env != nullptr) + ? (atoi(q4k_moe_dp4a_env) != 0) + : (backend_ctx->adreno_gen == ADRENO_GPU_GEN::X2E || backend_ctx->adreno_gen == ADRENO_GPU_GEN::X1E); + // bin kernel takes precedence + use_moe_dp4a = use_moe_dp4a && backend_ctx->kernel_gemm_moe_q4_k_f32_ns_bin == nullptr; + cl_buffer_region region; region.origin = 0; region.size = sizeof(int) * max_post_router_tile * n_tile_size; @@ -20429,42 +21835,45 @@ static void ggml_cl_mul_mat_id(ggml_backend_t backend, const ggml_tensor * src0, sub_buf_src1_pre = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &status); CL_CHECK(status); - // Create image for reordered src1 - region.origin = 0; - region.size = ne00 * max_post_router_tile * n_tile_size * sizeof(float); - backend_ctx->prealloc_act_trans.allocate(backend_ctx->context, region.size); - buf_src1_reordered = clCreateSubBuffer( - backend_ctx->prealloc_act_trans.buffer, - 0, - CL_BUFFER_CREATE_TYPE_REGION, - ®ion, - &status); - CL_CHECK(status); - cl_image_format image_format_buf_src1 = {CL_RGBA, CL_FLOAT}; - cl_image_desc image_desc_buf_src1 = {CL_MEM_OBJECT_IMAGE1D_BUFFER, static_cast(ne00 * max_post_router_tile * n_tile_size / 4), 0,0,0,0,0,0,0, {buf_src1_reordered}}; - if (backend_ctx->kernel_gemm_moe_q4_k_f32_ns_bin) { - // bin kernel uses slightly different image format - image_format_buf_src1 = {CL_R, CL_FLOAT}; - image_desc_buf_src1.image_width = static_cast(ne00 * max_post_router_tile * n_tile_size); - } - image_src1_reordered = clCreateImage(backend_ctx->context, CL_MEM_READ_ONLY, &image_format_buf_src1, &image_desc_buf_src1, NULL, &status); - CL_CHECK(status); - unsigned short map_ratio = ne20 / ne11; GGML_ASSERT(((map_ratio == 1) || (map_ratio == ne20)) && "Map ratio not supported\n"); - CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 0, sizeof(cl_mem), &sub_buf_src1_pre)); - CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 1, sizeof(cl_mem), &buf_src2)); - CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 2, sizeof(cl_mem), &buf_src1_reordered)); - CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 3, sizeof(cl_mem), &(backend_ctx->prealloc_total_tiles.buffer))); - CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 4, sizeof(unsigned int), &ne00)); - CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 5, sizeof(unsigned short), &map_ratio)); - CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 6, sizeof(unsigned int), &n_tile_size)); - size_t reorder_b_local_size[3] = {256, 1, 1}; - size_t reorder_b_global_size[3] = {static_cast(((ne00 / 4) + 255) / 256 * 256), static_cast(max_post_router_tile * n_tile_size), 1}; + if (!use_moe_dp4a) { + // Create image for reordered src1 + region.origin = 0; + region.size = ne00 * max_post_router_tile * n_tile_size * sizeof(float); + backend_ctx->prealloc_act_trans.allocate(backend_ctx->context, region.size); + buf_src1_reordered = clCreateSubBuffer( + backend_ctx->prealloc_act_trans.buffer, + 0, + CL_BUFFER_CREATE_TYPE_REGION, + ®ion, + &status); + CL_CHECK(status); + cl_image_format image_format_buf_src1 = {CL_RGBA, CL_FLOAT}; + cl_image_desc image_desc_buf_src1 = {CL_MEM_OBJECT_IMAGE1D_BUFFER, static_cast(ne00 * max_post_router_tile * n_tile_size / 4), 0,0,0,0,0,0,0, {buf_src1_reordered}}; + if (backend_ctx->kernel_gemm_moe_q4_k_f32_ns_bin) { + // bin kernel uses slightly different image format + image_format_buf_src1 = {CL_R, CL_FLOAT}; + image_desc_buf_src1.image_width = static_cast(ne00 * max_post_router_tile * n_tile_size); + } + image_src1_reordered = clCreateImage(backend_ctx->context, CL_MEM_READ_ONLY, &image_format_buf_src1, &image_desc_buf_src1, NULL, &status); + CL_CHECK(status); - // Dispatch reorder kernel - backend_ctx->enqueue_ndrange_kernel(backend_ctx->kernel_moe_reorder_b, 3, reorder_b_global_size, reorder_b_local_size, dst); + CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 0, sizeof(cl_mem), &sub_buf_src1_pre)); + CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 1, sizeof(cl_mem), &buf_src2)); + CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 2, sizeof(cl_mem), &buf_src1_reordered)); + CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 3, sizeof(cl_mem), &(backend_ctx->prealloc_total_tiles.buffer))); + CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 4, sizeof(unsigned int), &ne00)); + CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 5, sizeof(unsigned short), &map_ratio)); + CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 6, sizeof(unsigned int), &n_tile_size)); + + size_t reorder_b_local_size[3] = {256, 1, 1}; + size_t reorder_b_global_size[3] = {static_cast(((ne00 / 4) + 255) / 256 * 256), static_cast(max_post_router_tile * n_tile_size), 1}; + + // Dispatch reorder kernel + backend_ctx->enqueue_ndrange_kernel(backend_ctx->kernel_moe_reorder_b, 3, reorder_b_global_size, reorder_b_local_size, dst); + } // MoE kernel prepare region.origin = offsetd; @@ -20482,6 +21891,61 @@ static void ggml_cl_mul_mat_id(ggml_backend_t backend, const ggml_tensor * src0, buf_dst_image = clCreateImage(backend_ctx->context, CL_MEM_WRITE_ONLY, &image_format_buf_dst, &image_desc_buf_dst, NULL, &status); CL_CHECK(status); + if (use_moe_dp4a) { + const size_t tok_slots = (size_t)max_post_router_tile * n_tile_size; + const size_t n_blocks = tok_slots * (ne00 / 32); + backend_ctx->prealloc_moe_qa.allocate(backend_ctx->context, tok_slots * ne00 * sizeof(cl_char)); + backend_ctx->prealloc_moe_da.allocate(backend_ctx->context, n_blocks * sizeof(cl_half)); + backend_ctx->prealloc_moe_sa.allocate(backend_ctx->context, n_blocks * sizeof(cl_half)); + + // fused reorder + q8_1 quant straight from the original + // activations (no intermediate f32 reorder buffer) + const cl_uint n_kblocks = (cl_uint)(ne00 / 32); + cl_kernel rq = backend_ctx->kernel_moe_reorder_quant_a_q8_1; + CL_CHECK(clSetKernelArg(rq, 0, sizeof(cl_mem), &sub_buf_src1_pre)); + CL_CHECK(clSetKernelArg(rq, 1, sizeof(cl_mem), &buf_src2)); + CL_CHECK(clSetKernelArg(rq, 2, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer)); + CL_CHECK(clSetKernelArg(rq, 3, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer)); + CL_CHECK(clSetKernelArg(rq, 4, sizeof(cl_mem), &backend_ctx->prealloc_moe_sa.buffer)); + CL_CHECK(clSetKernelArg(rq, 5, sizeof(cl_mem), &(backend_ctx->prealloc_total_tiles.buffer))); + CL_CHECK(clSetKernelArg(rq, 6, sizeof(cl_uint), &ne00)); + CL_CHECK(clSetKernelArg(rq, 7, sizeof(unsigned short), &map_ratio)); + CL_CHECK(clSetKernelArg(rq, 8, sizeof(cl_uint), &n_tile_size)); + CL_CHECK(clSetKernelArg(rq, 9, sizeof(cl_uint), &n_kblocks)); + size_t rq_local[2] = { 32, 1 }; + size_t rq_global[2] = { (size_t)(((n_kblocks + 31) / 32) * 32), tok_slots }; + backend_ctx->enqueue_ndrange_kernel(rq, 2, rq_global, rq_local, dst); + + // dp4a GEMM + cl_kernel dk = backend_ctx->kernel_gemm_moe_q4_k_q8_1_dp4a; + int aidx = 0; + CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &extra0_q4_K->q_img)); + CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &extra0_q4_K->d)); + CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &extra0_q4_K->dm)); + CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &extra0_q4_K->s)); + CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer)); + CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer)); + CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &backend_ctx->prealloc_moe_sa.buffer)); + CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &buf_src2)); + CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &buf_src2_emap)); + CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &buf_dst_image)); + CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &(backend_ctx->prealloc_total_tiles.buffer))); + CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(int), &ne00)); + CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(int), &ne01)); + CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(int), &backend_ctx->adreno_use_moe_ragged_dp4)); + + size_t dp_global[3] = { 64, (size_t)((ne01 + 63) / 64), (size_t)max_post_router_tile }; + size_t dp_local[3] = { 64, 1, 1 }; + backend_ctx->enqueue_ndrange_kernel(dk, 3, dp_global, dp_local, dst); + + clReleaseMemObject(sub_buf_src1_pre); + clReleaseMemObject(buf_src2); + clReleaseMemObject(buf_src2_emap); + clReleaseMemObject(sub_buf_dst); + clReleaseMemObject(buf_dst_image); + return; + } + // Set kernel args int arg_idx = 0; CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &extra0_q4_K->q_img)); @@ -20611,6 +22075,83 @@ static void ggml_cl_mul_mat_id(ggml_backend_t backend, const ggml_tensor * src0, sub_buf_src1_pre = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &status); CL_CHECK(status); + // Generic dp4a MoE GEMM + { + static const char * q5kmdp4a_env = getenv("GGML_OPENCL_Q5K_MOE_DP4A"); + const bool q5kmdp4a_on = q5kmdp4a_env ? (atoi(q5kmdp4a_env) != 0) + : (backend_ctx->adreno_gen == ADRENO_GPU_GEN::X2E); + bool use_moe_dp4a = q5kmdp4a_on + && backend_ctx->kernel_gemm_moe_q8_1_dp4a_q5k != nullptr + && extra0_q5_K->scale != nullptr; + // bin kernel takes precedence + use_moe_dp4a = use_moe_dp4a && backend_ctx->kernel_gemm_moe_q4_k_f32_ns_bin == nullptr; + + if (use_moe_dp4a) { + const size_t tok_slots = (size_t)max_post_router_tile * n_tile_size; + const size_t n_blocks = tok_slots * (ne00 / 32); + backend_ctx->prealloc_moe_qa.allocate(backend_ctx->context, tok_slots * ne00 * sizeof(cl_char)); + backend_ctx->prealloc_moe_da.allocate(backend_ctx->context, n_blocks * sizeof(cl_half)); + backend_ctx->prealloc_moe_sa.allocate(backend_ctx->context, n_blocks * sizeof(cl_half)); + + const cl_uint n_kblocks = (cl_uint)(ne00 / 32); + unsigned short map_ratio_q5k = ne20 / ne11; + cl_kernel rq = backend_ctx->kernel_moe_reorder_quant_a_q8_1; + CL_CHECK(clSetKernelArg(rq, 0, sizeof(cl_mem), &sub_buf_src1_pre)); + CL_CHECK(clSetKernelArg(rq, 1, sizeof(cl_mem), &buf_src2)); + CL_CHECK(clSetKernelArg(rq, 2, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer)); + CL_CHECK(clSetKernelArg(rq, 3, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer)); + CL_CHECK(clSetKernelArg(rq, 4, sizeof(cl_mem), &backend_ctx->prealloc_moe_sa.buffer)); + CL_CHECK(clSetKernelArg(rq, 5, sizeof(cl_mem), &(backend_ctx->prealloc_total_tiles.buffer))); + CL_CHECK(clSetKernelArg(rq, 6, sizeof(cl_uint), &ne00)); + CL_CHECK(clSetKernelArg(rq, 7, sizeof(unsigned short), &map_ratio_q5k)); + CL_CHECK(clSetKernelArg(rq, 8, sizeof(cl_uint), &n_tile_size)); + CL_CHECK(clSetKernelArg(rq, 9, sizeof(cl_uint), &n_kblocks)); + size_t rq_local[2] = { 32, 1 }; + size_t rq_global[2] = { (size_t)(((n_kblocks + 31) / 32) * 32), tok_slots }; + backend_ctx->enqueue_ndrange_kernel(rq, 2, rq_global, rq_local, dst); + + region.origin = offsetd; + region.size = ne0 * ne1 * ne2 * sizeof(float); + cl_mem dp_sub_buf_dst = clCreateSubBuffer(extrad->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &status); + CL_CHECK(status); + cl_image_format dp_ifd = {CL_R, CL_FLOAT}; + cl_image_desc dp_idd = {CL_MEM_OBJECT_IMAGE1D_BUFFER, static_cast(ne0 * ne1 * ne2), 0,0,0,0,0,0,0, {dp_sub_buf_dst}}; + cl_mem dp_buf_dst_image = clCreateImage(backend_ctx->context, CL_MEM_WRITE_ONLY, &dp_ifd, &dp_idd, NULL, &status); + CL_CHECK(status); + + int ne00i = (int)ne00, ne01i = (int)ne01; + cl_kernel dk = backend_ctx->kernel_gemm_moe_q8_1_dp4a_q5k; + int has_min_q5k = 1; + int aidx = 0; + CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &extra0_q5_K->q_img)); + CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &extra0_q5_K->qh)); + CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &extra0_q5_K->scale)); + CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &extra0_q5_K->min)); + CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer)); + CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer)); + CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &backend_ctx->prealloc_moe_sa.buffer)); + CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &buf_src2)); + CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &buf_src2_emap)); + CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &dp_buf_dst_image)); + CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &(backend_ctx->prealloc_total_tiles.buffer))); + CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(int), &ne00i)); + CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(int), &ne01i)); + CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(int), &backend_ctx->adreno_use_moe_ragged_dp4)); + CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(int), &has_min_q5k)); + + size_t dp_global[3] = { 64, (size_t)((ne01 + 63) / 64), (size_t)max_post_router_tile }; + size_t dp_local[3] = { 64, 1, 1 }; + backend_ctx->enqueue_ndrange_kernel(dk, 3, dp_global, dp_local, dst); + + clReleaseMemObject(sub_buf_src1_pre); + clReleaseMemObject(buf_src2); + clReleaseMemObject(buf_src2_emap); + clReleaseMemObject(dp_sub_buf_dst); + clReleaseMemObject(dp_buf_dst_image); + return; + } + } + // Create image for reordered src1 // Use pre-allocated placeholder region.origin = 0; @@ -20769,9 +22310,17 @@ static void ggml_cl_mul_mat_id(ggml_backend_t backend, const ggml_tensor * src0, backend_ctx->toggle_reorder = false; } - cl_mem sub_buf_src1_pre, buf_src1_reordered, image_src1_reordered, sub_buf_dst, buf_dst_image; + cl_mem sub_buf_src1_pre, sub_buf_dst, buf_dst_image; + cl_mem buf_src1_reordered = nullptr, image_src1_reordered = nullptr; cl_mem buf_src2, buf_src2_emap; + // dp4a (int8) q6_K MoE prefill GEMM + static const char * q6k_moe_dp4a_env = getenv("GGML_OPENCL_Q6K_MOE_DP4A"); + static const bool use_moe_dp4a = (q6k_moe_dp4a_env != nullptr) + ? (atoi(q6k_moe_dp4a_env) != 0) + : (backend_ctx->adreno_gen == ADRENO_GPU_GEN::X2E + || backend_ctx->adreno_gen == ADRENO_GPU_GEN::X1E); + cl_buffer_region region; region.origin = 0; region.size = sizeof(int) * max_post_router_tile * n_tile_size; @@ -20790,37 +22339,40 @@ static void ggml_cl_mul_mat_id(ggml_backend_t backend, const ggml_tensor * src0, sub_buf_src1_pre = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &status); CL_CHECK(status); - // Create image for reordered src1 - region.origin = 0; - region.size = ne00 * max_post_router_tile * n_tile_size * sizeof(float); - backend_ctx->prealloc_act_trans.allocate(backend_ctx->context, region.size); - buf_src1_reordered = clCreateSubBuffer( - backend_ctx->prealloc_act_trans.buffer, - 0, - CL_BUFFER_CREATE_TYPE_REGION, - ®ion, - &status); - CL_CHECK(status); - cl_image_format image_format_buf_src1 = {CL_RGBA, CL_FLOAT}; - cl_image_desc image_desc_buf_src1 = {CL_MEM_OBJECT_IMAGE1D_BUFFER, static_cast(ne00 * max_post_router_tile * n_tile_size / 4), 0,0,0,0,0,0,0, {buf_src1_reordered}}; - image_src1_reordered = clCreateImage(backend_ctx->context, CL_MEM_READ_ONLY, &image_format_buf_src1, &image_desc_buf_src1, NULL, &status); - CL_CHECK(status); - unsigned short map_ratio = ne20 / ne11; GGML_ASSERT(((map_ratio == 1) || (map_ratio == ne20)) && "Map ratio not supported\n"); - CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 0, sizeof(cl_mem), &sub_buf_src1_pre)); - CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 1, sizeof(cl_mem), &buf_src2)); - CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 2, sizeof(cl_mem), &buf_src1_reordered)); - CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 3, sizeof(cl_mem), &(backend_ctx->prealloc_total_tiles.buffer))); - CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 4, sizeof(unsigned int), &ne00)); - CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 5, sizeof(unsigned short), &map_ratio)); - CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 6, sizeof(unsigned int), &n_tile_size)); - size_t reorder_b_local_size[3] = {256, 1, 1}; - size_t reorder_b_global_size[3] = {static_cast(((ne00 / 4) + 255) / 256 * 256), static_cast(max_post_router_tile * n_tile_size), 1}; + if (!use_moe_dp4a) { + // Create image for reordered src1 + region.origin = 0; + region.size = ne00 * max_post_router_tile * n_tile_size * sizeof(float); + backend_ctx->prealloc_act_trans.allocate(backend_ctx->context, region.size); + buf_src1_reordered = clCreateSubBuffer( + backend_ctx->prealloc_act_trans.buffer, + 0, + CL_BUFFER_CREATE_TYPE_REGION, + ®ion, + &status); + CL_CHECK(status); + cl_image_format image_format_buf_src1 = {CL_RGBA, CL_FLOAT}; + cl_image_desc image_desc_buf_src1 = {CL_MEM_OBJECT_IMAGE1D_BUFFER, static_cast(ne00 * max_post_router_tile * n_tile_size / 4), 0,0,0,0,0,0,0, {buf_src1_reordered}}; + image_src1_reordered = clCreateImage(backend_ctx->context, CL_MEM_READ_ONLY, &image_format_buf_src1, &image_desc_buf_src1, NULL, &status); + CL_CHECK(status); - // Dispatch reorder kernel - backend_ctx->enqueue_ndrange_kernel(backend_ctx->kernel_moe_reorder_b, 3, reorder_b_global_size, reorder_b_local_size, dst); + CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 0, sizeof(cl_mem), &sub_buf_src1_pre)); + CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 1, sizeof(cl_mem), &buf_src2)); + CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 2, sizeof(cl_mem), &buf_src1_reordered)); + CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 3, sizeof(cl_mem), &(backend_ctx->prealloc_total_tiles.buffer))); + CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 4, sizeof(unsigned int), &ne00)); + CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 5, sizeof(unsigned short), &map_ratio)); + CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 6, sizeof(unsigned int), &n_tile_size)); + + size_t reorder_b_local_size[3] = {256, 1, 1}; + size_t reorder_b_global_size[3] = {static_cast(((ne00 / 4) + 255) / 256 * 256), static_cast(max_post_router_tile * n_tile_size), 1}; + + // Dispatch reorder kernel + backend_ctx->enqueue_ndrange_kernel(backend_ctx->kernel_moe_reorder_b, 3, reorder_b_global_size, reorder_b_local_size, dst); + } // MoE kernel prepare // Create sub buffer for dst @@ -20839,6 +22391,58 @@ static void ggml_cl_mul_mat_id(ggml_backend_t backend, const ggml_tensor * src0, buf_dst_image = clCreateImage(backend_ctx->context, CL_MEM_WRITE_ONLY, &image_format_buf_dst, &image_desc_buf_dst, NULL, &status); CL_CHECK(status); + if (use_moe_dp4a) { + const size_t tok_slots = (size_t)max_post_router_tile * n_tile_size; + const size_t n_blocks = tok_slots * (ne00 / 32); + backend_ctx->prealloc_moe_qa.allocate(backend_ctx->context, tok_slots * ne00 * sizeof(cl_char)); + backend_ctx->prealloc_moe_da.allocate(backend_ctx->context, n_blocks * sizeof(cl_half)); + backend_ctx->prealloc_moe_sa.allocate(backend_ctx->context, n_blocks * sizeof(cl_half)); + + // fused reorder + q8_1 quant from the original activations + const cl_uint n_kblocks = (cl_uint)(ne00 / 32); + cl_kernel rq = backend_ctx->kernel_moe_reorder_quant_a_q8_1; + CL_CHECK(clSetKernelArg(rq, 0, sizeof(cl_mem), &sub_buf_src1_pre)); + CL_CHECK(clSetKernelArg(rq, 1, sizeof(cl_mem), &buf_src2)); + CL_CHECK(clSetKernelArg(rq, 2, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer)); + CL_CHECK(clSetKernelArg(rq, 3, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer)); + CL_CHECK(clSetKernelArg(rq, 4, sizeof(cl_mem), &backend_ctx->prealloc_moe_sa.buffer)); + CL_CHECK(clSetKernelArg(rq, 5, sizeof(cl_mem), &(backend_ctx->prealloc_total_tiles.buffer))); + CL_CHECK(clSetKernelArg(rq, 6, sizeof(cl_uint), &ne00)); + CL_CHECK(clSetKernelArg(rq, 7, sizeof(unsigned short), &map_ratio)); + CL_CHECK(clSetKernelArg(rq, 8, sizeof(cl_uint), &n_tile_size)); + CL_CHECK(clSetKernelArg(rq, 9, sizeof(cl_uint), &n_kblocks)); + size_t rq_local[2] = { 32, 1 }; + size_t rq_global[2] = { (size_t)(((n_kblocks + 31) / 32) * 32), tok_slots }; + backend_ctx->enqueue_ndrange_kernel(rq, 2, rq_global, rq_local, dst); + + cl_kernel dk = backend_ctx->kernel_gemm_moe_q6_k_q8_1_dp4a; + int qi = 0; + CL_CHECK(clSetKernelArg(dk, qi++, sizeof(cl_mem), &extra0_q6_K->ql_img)); + CL_CHECK(clSetKernelArg(dk, qi++, sizeof(cl_mem), &extra0_q6_K->qh)); + CL_CHECK(clSetKernelArg(dk, qi++, sizeof(cl_mem), &extra0_q6_K->s)); + CL_CHECK(clSetKernelArg(dk, qi++, sizeof(cl_mem), &extra0_q6_K->d)); + CL_CHECK(clSetKernelArg(dk, qi++, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer)); + CL_CHECK(clSetKernelArg(dk, qi++, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer)); + CL_CHECK(clSetKernelArg(dk, qi++, sizeof(cl_mem), &buf_src2)); + CL_CHECK(clSetKernelArg(dk, qi++, sizeof(cl_mem), &buf_src2_emap)); + CL_CHECK(clSetKernelArg(dk, qi++, sizeof(cl_mem), &buf_dst_image)); + CL_CHECK(clSetKernelArg(dk, qi++, sizeof(cl_mem), &(backend_ctx->prealloc_total_tiles.buffer))); + CL_CHECK(clSetKernelArg(dk, qi++, sizeof(int), &ne00)); + CL_CHECK(clSetKernelArg(dk, qi++, sizeof(int), &ne01)); + CL_CHECK(clSetKernelArg(dk, qi++, sizeof(int), &backend_ctx->adreno_use_moe_ragged_dp4)); + + size_t dp_global[3] = { 64, (size_t)((ne01 + 63) / 64), (size_t)max_post_router_tile }; + size_t dp_local[3] = { 64, 1, 1 }; + backend_ctx->enqueue_ndrange_kernel(dk, 3, dp_global, dp_local, dst); + + clReleaseMemObject(sub_buf_src1_pre); + clReleaseMemObject(buf_src2); + clReleaseMemObject(buf_src2_emap); + clReleaseMemObject(sub_buf_dst); + clReleaseMemObject(buf_dst_image); + return; + } + // Set kernel args int arg_idx = 0; CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &extra0_q6_K->ql_img)); @@ -20887,6 +22491,15 @@ static void ggml_cl_mul_mat_id(ggml_backend_t backend, const ggml_tensor * src0, if (ne12 == 1) { // for gemv kernel = backend_ctx->kernel_gemv_moe_mxfp4_f32_ns; + // Weight-as-texture MoE decode GEMV (see q4_K _wimg) + static const char * moe_decode_wimg_env = getenv("GGML_OPENCL_MOE_DECODE_WIMG"); + const bool use_moe_decode_wimg = (moe_decode_wimg_env && (atoi(moe_decode_wimg_env) != 0)) + && backend_ctx->kernel_gemv_moe_mxfp4_f32_ns_wimg != nullptr + && extra0_mxfp4->q_img != nullptr; + if (use_moe_decode_wimg) { + kernel = backend_ctx->kernel_gemv_moe_mxfp4_f32_ns_wimg; + } + cl_mem src1_sub_buffer, buf_src1_image, buf_src2; // create a sub_buffer for src2 @@ -20916,7 +22529,7 @@ static void ggml_cl_mul_mat_id(ggml_backend_t backend, const ggml_tensor * src0, // Set kernel args int arg_idx = 0; - CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &extra0_mxfp4->q)); + CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), use_moe_decode_wimg ? &extra0_mxfp4->q_img : &extra0_mxfp4->q)); CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &extra0_mxfp4->e)); CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &buf_src1_image)); CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &buf_src2)); @@ -20947,9 +22560,18 @@ static void ggml_cl_mul_mat_id(ggml_backend_t backend, const ggml_tensor * src0, backend_ctx->toggle_reorder = false; } - cl_mem sub_buf_src1_pre, buf_src1_reordered, image_src1_reordered, sub_buf_dst, buf_dst_image; + cl_mem sub_buf_src1_pre, sub_buf_dst, buf_dst_image; + cl_mem buf_src1_reordered = nullptr, image_src1_reordered = nullptr; cl_mem buf_src2, buf_src2_emap; + // dp4a (int8) prefill GEMM variant + static const char * mxfp4_moe_dp4a_env = getenv("GGML_OPENCL_MXFP4_MOE_DP4A"); + bool use_moe_dp4a = mxfp4_moe_dp4a_env + ? (atoi(mxfp4_moe_dp4a_env) != 0) + : (backend_ctx->adreno_gen == ADRENO_GPU_GEN::X2E); + // bin kernel takes precedence + use_moe_dp4a = use_moe_dp4a && backend_ctx->kernel_gemm_moe_mxfp4_f32_ns_bin == nullptr; + cl_buffer_region region; region.origin = 0; region.size = sizeof(int) * max_post_router_tile * n_tile_size; @@ -20969,45 +22591,48 @@ static void ggml_cl_mul_mat_id(ggml_backend_t backend, const ggml_tensor * src0, sub_buf_src1_pre = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &status); CL_CHECK(status); - // Create image for reordered src1 - // Use pre-allocated placeholder - region.origin = 0; - region.size = ne00 * max_post_router_tile * n_tile_size * sizeof(float); - backend_ctx->prealloc_act_trans.allocate(backend_ctx->context, region.size); - buf_src1_reordered = clCreateSubBuffer( - backend_ctx->prealloc_act_trans.buffer, - 0, - CL_BUFFER_CREATE_TYPE_REGION, - ®ion, - &status); - CL_CHECK(status); - cl_image_format image_format_buf_src1; - cl_image_desc image_desc_buf_src1; - image_format_buf_src1 = {CL_RGBA, CL_FLOAT}; - image_desc_buf_src1 = {CL_MEM_OBJECT_IMAGE1D_BUFFER, static_cast(ne00 * max_post_router_tile * n_tile_size / 4), 0,0,0,0,0,0,0, {buf_src1_reordered}}; - if (backend_ctx->kernel_gemm_moe_mxfp4_f32_ns_bin) { - // bin kernel uses slightly different image format - image_format_buf_src1 = {CL_R, CL_FLOAT}; - image_desc_buf_src1.image_width = static_cast(ne00 * max_post_router_tile * n_tile_size); - } - image_src1_reordered = clCreateImage(backend_ctx->context, CL_MEM_READ_ONLY, &image_format_buf_src1, &image_desc_buf_src1, NULL, &status); - CL_CHECK(status); - unsigned short map_ratio = ne20 / ne11; GGML_ASSERT(((map_ratio == 1) || (map_ratio == ne20)) && "Map ratio not supported\n"); - CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 0, sizeof(cl_mem), &sub_buf_src1_pre)); - CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 1, sizeof(cl_mem), &buf_src2)); - CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 2, sizeof(cl_mem), &buf_src1_reordered)); - CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 3, sizeof(cl_mem), &(backend_ctx->prealloc_total_tiles.buffer))); - CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 4, sizeof(unsigned int), &ne00)); - CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 5, sizeof(unsigned short), &map_ratio)); - CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 6, sizeof(unsigned int), &n_tile_size)); - size_t reorder_b_local_size[3] = {256, 1, 1}; - size_t reorder_b_global_size[3] = {static_cast(((ne00 / 4) + 255) / 256 * 256), static_cast(max_post_router_tile * n_tile_size), 1}; + if (!use_moe_dp4a) { + // Create image for reordered src1 + // Use pre-allocated placeholder + region.origin = 0; + region.size = ne00 * max_post_router_tile * n_tile_size * sizeof(float); + backend_ctx->prealloc_act_trans.allocate(backend_ctx->context, region.size); + buf_src1_reordered = clCreateSubBuffer( + backend_ctx->prealloc_act_trans.buffer, + 0, + CL_BUFFER_CREATE_TYPE_REGION, + ®ion, + &status); + CL_CHECK(status); + cl_image_format image_format_buf_src1; + cl_image_desc image_desc_buf_src1; + image_format_buf_src1 = {CL_RGBA, CL_FLOAT}; + image_desc_buf_src1 = {CL_MEM_OBJECT_IMAGE1D_BUFFER, static_cast(ne00 * max_post_router_tile * n_tile_size / 4), 0,0,0,0,0,0,0, {buf_src1_reordered}}; + if (backend_ctx->kernel_gemm_moe_mxfp4_f32_ns_bin) { + // bin kernel uses slightly different image format + image_format_buf_src1 = {CL_R, CL_FLOAT}; + image_desc_buf_src1.image_width = static_cast(ne00 * max_post_router_tile * n_tile_size); + } + image_src1_reordered = clCreateImage(backend_ctx->context, CL_MEM_READ_ONLY, &image_format_buf_src1, &image_desc_buf_src1, NULL, &status); + CL_CHECK(status); - // Dispatch reorder kernel - backend_ctx->enqueue_ndrange_kernel(backend_ctx->kernel_moe_reorder_b, 3, reorder_b_global_size, reorder_b_local_size, dst); + CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 0, sizeof(cl_mem), &sub_buf_src1_pre)); + CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 1, sizeof(cl_mem), &buf_src2)); + CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 2, sizeof(cl_mem), &buf_src1_reordered)); + CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 3, sizeof(cl_mem), &(backend_ctx->prealloc_total_tiles.buffer))); + CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 4, sizeof(unsigned int), &ne00)); + CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 5, sizeof(unsigned short), &map_ratio)); + CL_CHECK(clSetKernelArg(backend_ctx->kernel_moe_reorder_b, 6, sizeof(unsigned int), &n_tile_size)); + + size_t reorder_b_local_size[3] = {256, 1, 1}; + size_t reorder_b_global_size[3] = {static_cast(((ne00 / 4) + 255) / 256 * 256), static_cast(max_post_router_tile * n_tile_size), 1}; + + // Dispatch reorder kernel + backend_ctx->enqueue_ndrange_kernel(backend_ctx->kernel_moe_reorder_b, 3, reorder_b_global_size, reorder_b_local_size, dst); + } // MoE kernel prepare // Create sub buffer for dst @@ -21026,6 +22651,59 @@ static void ggml_cl_mul_mat_id(ggml_backend_t backend, const ggml_tensor * src0, buf_dst_image = clCreateImage(backend_ctx->context, CL_MEM_WRITE_ONLY, &image_format_buf_dst, &image_desc_buf_dst, NULL, &status); CL_CHECK(status); + if (use_moe_dp4a) { + const size_t tok_slots = (size_t)max_post_router_tile * n_tile_size; + const size_t n_blocks = tok_slots * (ne00 / 32); + backend_ctx->prealloc_moe_qa.allocate(backend_ctx->context, tok_slots * ne00 * sizeof(cl_char)); + backend_ctx->prealloc_moe_da.allocate(backend_ctx->context, n_blocks * sizeof(cl_half)); + backend_ctx->prealloc_moe_sa.allocate(backend_ctx->context, n_blocks * sizeof(cl_half)); + + // fused reorder + q8_1 quant straight from the original + // activations (no intermediate f32 reorder buffer). mxfp4 has no + // min term so the GEMM ignores sa, but reorder_quant still writes it. + const cl_uint n_kblocks = (cl_uint)(ne00 / 32); + cl_kernel rq = backend_ctx->kernel_moe_reorder_quant_a_q8_1; + CL_CHECK(clSetKernelArg(rq, 0, sizeof(cl_mem), &sub_buf_src1_pre)); + CL_CHECK(clSetKernelArg(rq, 1, sizeof(cl_mem), &buf_src2)); + CL_CHECK(clSetKernelArg(rq, 2, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer)); + CL_CHECK(clSetKernelArg(rq, 3, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer)); + CL_CHECK(clSetKernelArg(rq, 4, sizeof(cl_mem), &backend_ctx->prealloc_moe_sa.buffer)); + CL_CHECK(clSetKernelArg(rq, 5, sizeof(cl_mem), &(backend_ctx->prealloc_total_tiles.buffer))); + CL_CHECK(clSetKernelArg(rq, 6, sizeof(cl_uint), &ne00)); + CL_CHECK(clSetKernelArg(rq, 7, sizeof(unsigned short), &map_ratio)); + CL_CHECK(clSetKernelArg(rq, 8, sizeof(cl_uint), &n_tile_size)); + CL_CHECK(clSetKernelArg(rq, 9, sizeof(cl_uint), &n_kblocks)); + size_t rq_local[2] = { 32, 1 }; + size_t rq_global[2] = { (size_t)(((n_kblocks + 31) / 32) * 32), tok_slots }; + backend_ctx->enqueue_ndrange_kernel(rq, 2, rq_global, rq_local, dst); + + // dp4a GEMM + cl_kernel dk = backend_ctx->kernel_gemm_moe_mxfp4_q8_1_dp4a; + int aidx = 0; + CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &extra0_mxfp4->q_img)); + CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &extra0_mxfp4->e)); + CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer)); + CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer)); + CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &buf_src2)); + CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &buf_src2_emap)); + CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &buf_dst_image)); + CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &(backend_ctx->prealloc_total_tiles.buffer))); + CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(int), &ne00)); + CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(int), &ne01)); + CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(int), &backend_ctx->adreno_use_moe_ragged_dp4)); + + size_t dp_global[3] = { 64, (size_t)((ne01 + 63) / 64), (size_t)max_post_router_tile }; + size_t dp_local[3] = { 64, 1, 1 }; + backend_ctx->enqueue_ndrange_kernel(dk, 3, dp_global, dp_local, dst); + + clReleaseMemObject(sub_buf_src1_pre); + clReleaseMemObject(buf_src2); + clReleaseMemObject(buf_src2_emap); + clReleaseMemObject(sub_buf_dst); + clReleaseMemObject(buf_dst_image); + return; + } + // Set kernel args int arg_idx = 0; CL_CHECK(clSetKernelArg(kernel, arg_idx++, sizeof(cl_mem), &extra0_mxfp4->q_img)); diff --git a/ggml/src/ggml-opencl/kernels/cvt.cl b/ggml/src/ggml-opencl/kernels/cvt.cl index bf0335a69..3d6cff7cf 100644 --- a/ggml/src/ggml-opencl/kernels/cvt.cl +++ b/ggml/src/ggml-opencl/kernels/cvt.cl @@ -2372,3 +2372,121 @@ kernel void kernel_restore_block_iq4_nl_noshuffle( b->qs[2*i + 1] = convert_uchar(((x0 & mask_F0) >> 4) | (x1 & mask_F0)); } } + +// --------------------------------------------------------------------------- +// kernel_moe_expand_scale_q8_0 +// +// Expand the q8_0 per-32-block scale d (one half/block, [expert][row][block]) into +// the UNIFORM scale[16] format the generic dp4a MoE GEMM (kernel_gemm_moe_q8_1_dp4a, +// MOE_QT=80) consumes: 16 f16 per 256-superblock (per-16-element segment), where the +// two segments of each 32-block share the block's d. q8_0 is symmetric -> no min +// buffer (the GEMM runs with has_min=0). The int8 weight codes are reused verbatim +// from the existing flat q8_0 weight buffer (extra0_q8_0->q), so only the scale is +// rebuilt here. One work-item per (row, superblock, expert). +// --------------------------------------------------------------------------- +kernel void kernel_moe_expand_scale_q8_0( + global const half * src_d, // [expert][row][block], one scale per 32-block + global half * dst_scale, // [expert][row][block][2] (FLAT per-32-block) + int ne00, + int ne01 +) { + int row = get_global_id(0); + int blk = get_global_id(1); // 32-block index along K + int e = get_global_id(2); + if (row >= ne01) { return; } + + long nb = ne00 / 32; // 32-blocks per row (K only needs % 32 == 0) + half d = src_d[((long)e*ne01 + row)*nb + blk]; + long b = (((long)e*ne01 + row)*nb + blk) * 2; + dst_scale[b + 0] = d; + dst_scale[b + 1] = d; +} + +// --------------------------------------------------------------------------- +// kernel_moe_expand_scale_q5_0 +// +// q5_0 = symmetric, value = d*(code-16), code = nibble | (hi<<4) in 0..31. The +// generic dp4a MoE GEMM keeps the unsigned code and centers via the min term: +// scale*dp4a(code,a) - min*sum(a), scale = d, min = d*16. +// Reads the existing q5_0 d ([expert][block][row], one half/32-block, from the +// trans4 convert) and writes the FLAT per-32-block uniform scale[2]/min[1] in +// [expert][row][block] order (a transpose). One work-item per (row, block, expert). +// --------------------------------------------------------------------------- +kernel void kernel_moe_expand_scale_q5_0( + global const half * src_d, // [expert][block][row] + global half * dst_scale, // [expert][row][block][2] + global half * dst_min, // [expert][row][block] + int ne00, + int ne01 +) { + int row = get_global_id(0); + int blk = get_global_id(1); + int e = get_global_id(2); + if (row >= ne01) { return; } + + long nb = ne00 / 32; + half d = src_d[(long)e*nb*ne01 + (long)blk*ne01 + row]; // [expert][block][row] + long sb = (((long)e*ne01 + row)*nb + blk) * 2; + long mb = ((long)e*ne01 + row)*nb + blk; + dst_scale[sb + 0] = d; + dst_scale[sb + 1] = d; + dst_min[mb] = (half)((float)d * 16.0f); +} + +// --------------------------------------------------------------------------- +// kernel_moe_expand_scale_q5_K +// +// q5_K value = d*sv*code + (-dm*mn), with the 6-bit packed per-sub-block scale sv +// and min mn (8 sub-blocks of 32 per 256-superblock, decoded by get_scale_min_k4 +// from the 12-byte s[]). The generic dp4a MoE GEMM (kernel_gemm_moe_q8_1_dp4a, +// MOE_QT=5) keeps the unsigned 5-bit code and applies scale/min via the uniform +// per-32-block buffers: +// acc += sc0*a_d*raw1 + sc1*a_d*raw2 - mn_u*a_s, +// sc0 = sc1 = d*sv (both per-16 segments of a 32-block share the sub-block scale), +// mn_u = dm*mn (positive; the GEMM subtracts it -> the -dm*mn min term). +// q5_K's q_img (low nibbles) + qh (hi-bit plane) are already in the layout the GEMM +// reads (same trans4_ns convert that feeds gemm_moe_q5_k_f32_ns), so only the scale +// is rebuilt here. +// +// One work-item per (row, superblock, expert); each emits 8 sub-blocks. +// --------------------------------------------------------------------------- +kernel void kernel_moe_expand_scale_q5_K( + global const uchar * src_s, // [expert][row][superblock][12] + global const half * src_d, // [expert][superblock][row] + global const half * src_dm, // [expert][superblock][row] + global half * dst_scale, // [expert][row][32block][2] + global half * dst_min, // [expert][row][32block] + int ne00, + int ne01 +) { + int row = get_global_id(0); + int sb = get_global_id(1); // superblock index along K + int e = get_global_id(2); + if (row >= ne01) { return; } + + long nsb = ne00 / 256; // superblocks per row + long nblk32 = ne00 / 32; // 32-blocks per row + + float d = (float)src_d [((long)e*nsb + sb)*ne01 + row]; + float dm = (float)src_dm[((long)e*nsb + sb)*ne01 + row]; + + __global const uchar * sc = src_s + ((long)e*ne01 + row)*nsb*12 + (long)sb*12; + + for (int j = 0; j < 8; ++j) { + uchar sv, mn; + // get_scale_min_k4 (6-bit packed scale/min for sub-block j of 8) + if (j < 4) { + sv = sc[j] & 63; + mn = sc[j+4] & 63; + } else { + sv = (sc[j+4] & 0x0F) | ((sc[j-4] & 0xC0) >> 2); + mn = ((sc[j+4] >> 4) & 0x0F) | ((sc[j] & 0xC0) >> 2); + } + long sub = (long)sb*8 + j; + long sbase = (((long)e*ne01 + row)*nblk32 + sub) * 2; + half s_val = (half)(d * (float)sv); + dst_scale[sbase + 0] = s_val; + dst_scale[sbase + 1] = s_val; + dst_min[((long)e*ne01 + row)*nblk32 + sub] = (half)(dm * (float)mn); + } +} diff --git a/ggml/src/ggml-opencl/kernels/gemm_moe_mxfp4_q8_1_dp4a.cl b/ggml/src/ggml-opencl/kernels/gemm_moe_mxfp4_q8_1_dp4a.cl new file mode 100644 index 000000000..95d063813 --- /dev/null +++ b/ggml/src/ggml-opencl/kernels/gemm_moe_mxfp4_q8_1_dp4a.cl @@ -0,0 +1,186 @@ +#pragma OPENCL EXTENSION cl_khr_fp16 : enable +#pragma OPENCL EXTENSION cl_khr_subgroups : enable +#ifdef cl_khr_integer_dot_product +#pragma OPENCL EXTENSION cl_khr_integer_dot_product : enable +#endif + +#define TILESIZE_M 64 +#define TILESIZE_N 32 + +// 2*mxfp4_value as signed int8, packed 4 codes per uint. Divergent nibble +// lookups read a __constant *uint* array + shift, never a byte array +// (byte-indexed __constant loads serialize on Adreno and are far slower). +// idx 0-3: 0, 1, 2, 3 = 0x03020100 +// idx 4-7: 4, 6, 8, 12 = 0x0C080604 +// idx 8-11: 0, -1, -2, -3 = 0xFDFEFF00 (-1=0xFF,-2=0xFE,-3=0xFD) +// idx 12-15:-4, -6, -8,-12 = 0xF4F8FAFC (-4=0xFC,-6=0xFA,-8=0xF8,-12=0xF4) +__constant uint mxfp4_i8x4[4] = { + 0x03020100u, 0x0C080604u, 0xFDFEFF00u, 0xF4F8FAFCu +}; +inline uint mxfp4_code(uint n) { + return (mxfp4_i8x4[n >> 2] >> ((n & 3u) * 8u)) & 0xFFu; +} +// 4 nibbles in the low 16 bits of u -> 4 codebook int8, packed for dp4a. +inline uint mxfp4_pack(ushort u) { + return mxfp4_code((uint)( u & 0xF)) + | (mxfp4_code((uint)((u >> 4) & 0xF)) << 8) + | (mxfp4_code((uint)((u >> 8) & 0xF)) << 16) + | (mxfp4_code((uint)((u >> 12) & 0xF)) << 24); +} + +static inline float e8m0_to_fp32(uchar x) { + int bits; + bits = (x == 0) ? 0x00400000 : ((uint) x << 23); + return as_float(bits); +} + +// One token's dp4a dot (8 uints = 32 K elems) + mxfp4 block-scale epilogue. +// blk_scale already carries the 0.5 factor (== 0.5 * 2^e). +#define MOE_MXFP4_DP4A_T(t) do { \ + int raw = 0; \ + raw = dot_acc_sat_4x8packed_ss_int(qw[0], sh_qa[t][0], raw); \ + raw = dot_acc_sat_4x8packed_ss_int(qw[1], sh_qa[t][1], raw); \ + raw = dot_acc_sat_4x8packed_ss_int(qw[2], sh_qa[t][2], raw); \ + raw = dot_acc_sat_4x8packed_ss_int(qw[3], sh_qa[t][3], raw); \ + raw = dot_acc_sat_4x8packed_ss_int(qw[4], sh_qa[t][4], raw); \ + raw = dot_acc_sat_4x8packed_ss_int(qw[5], sh_qa[t][5], raw); \ + raw = dot_acc_sat_4x8packed_ss_int(qw[6], sh_qa[t][6], raw); \ + raw = dot_acc_sat_4x8packed_ss_int(qw[7], sh_qa[t][7], raw); \ + acc[t] += blk_scale * (float)sh_d[t] * (float)raw; \ + } while (0) + +__attribute__((qcom_wave_pair_mode(1))) +kernel void kernel_gemm_moe_mxfp4_q8_1_dp4a( + __read_only image1d_buffer_t src0_q, // mxfp4 codes (transposed, packed nibbles) + __global uchar * src0_e, // e8m0 per-32-block scale + __global uint * src1_qa, // q8_1 activations: int8 quants (as uint, 4/elem) + __global half * src1_da, // q8_1 per-block scale [tok_slot * ne00/32] + __global uint * src2, // post-router (orig out positions) + __global ushort * src2_emap, // tile -> expert id + __write_only image1d_buffer_t dst, + __global int * total_tiles, + uint ne00, + uint ne01, + int is_ragged // 1: compute only real tokens per tile +) { + const uint block_id_m = get_global_id(1); // m_tile + const uint block_id_n = get_global_id(2); // n_tile + + if (block_id_n >= total_tiles[0]) { + return; + } + + const uint lid = get_local_id(0); // 0..63, == this WI's output row in the M-tile + + const ushort expert_id = src2_emap[block_id_n]; + const uint row = block_id_m * TILESIZE_M; + const uint col = block_id_n * TILESIZE_N; + + const uint num_blocks = ne00 >> 5; // blocks-of-32 per token + const uint row_idx = row + lid; + + const uint ne00_u = ne00 >> 2; // ne00 in uint (int8x4) units + + __local uint sh_qa[TILESIZE_N][8]; // 32 tokens x 8 uints (32 int8) = 1 KiB + __local half sh_d[TILESIZE_N]; + + // Real token count for this tile. + // Real tokens are packed contiguously at the tile start; padded slots hold + // 0xFFFFFFFF (only the last tile of each expert is partial). is_ragged skips + // the dp4a/staging/scatter for padded slots; is_ragged==0 forces n_real=32. + __local uint sh_src2[TILESIZE_N]; + __local int sh_nreal; + if (lid < TILESIZE_N) { + sh_src2[lid] = src2[col + lid]; + } + barrier(CLK_LOCAL_MEM_FENCE); + if (lid == 0) { + int nr = TILESIZE_N; + if (is_ragged) { + nr = 0; + #pragma unroll + for (int t = 0; t < TILESIZE_N; ++t) { + if (sh_src2[t] != 0xFFFFFFFFu) ++nr; + } + } + sh_nreal = nr; + } + barrier(CLK_LOCAL_MEM_FENCE); + const int n_real = sh_nreal; + + float acc[TILESIZE_N]; + #pragma unroll + for (int t = 0; t < TILESIZE_N; ++t) acc[t] = 0.0f; + + for (uint step = 0; step < ne00; step += 32) { + const uint sub = step >> 5; // 32-block index along K + + // e8m0 block scale for this WI's row, this 32-block (folded x0.5) + const uint e_offset = row_idx + sub * ne01 + expert_id * num_blocks * ne01; + const float blk_scale = 0.5f * e8m0_to_fp32(src0_e[e_offset]); + + // repack this WI's 32 weight nibbles into 8 dp4a uints + const uint qoff0 = row + ((ne01 * step) >> 3) + ((expert_id * ne00 * ne01) >> 3); + const uint qoff1 = row + ((ne01 * (step + 16)) >> 3) + ((expert_id * ne00 * ne01) >> 3); + const uint r0 = read_imageui(src0_q, qoff0 + lid).x; + const uint r1 = read_imageui(src0_q, qoff0 + lid + ne01).x; + const uint r2 = read_imageui(src0_q, qoff1 + lid).x; + const uint r3 = read_imageui(src0_q, qoff1 + lid + ne01).x; + uint qw[8]; + qw[0] = mxfp4_pack((ushort)(r0)); qw[1] = mxfp4_pack((ushort)(r0 >> 16)); + qw[2] = mxfp4_pack((ushort)(r1)); qw[3] = mxfp4_pack((ushort)(r1 >> 16)); + qw[4] = mxfp4_pack((ushort)(r2)); qw[5] = mxfp4_pack((ushort)(r2 >> 16)); + qw[6] = mxfp4_pack((ushort)(r3)); qw[7] = mxfp4_pack((ushort)(r3 >> 16)); + + // cooperatively stage the n_real-token x 32-K int8 activations + const uint stage_lim = (uint)n_real * 8; + for (uint idx = lid; idx < stage_lim; idx += 64) { + const uint t = idx >> 3; + const uint u = idx & 7; + sh_qa[t][u] = src1_qa[(col + t) * ne00_u + (step >> 2) + u]; + } + if (lid < (uint)n_real) { + sh_d[lid] = src1_da[(col + lid) * num_blocks + sub]; + } + barrier(CLK_LOCAL_MEM_FENCE); + + // Full tiles keep the fully-unrolled 32-wide loop; partial tiles run only n_real + if (n_real == TILESIZE_N) { + #pragma unroll + for (int t = 0; t < TILESIZE_N; ++t) { MOE_MXFP4_DP4A_T(t); } + } else { + #pragma unroll 4 + for (int t = 0; t < n_real; ++t) { MOE_MXFP4_DP4A_T(t); } + } + barrier(CLK_LOCAL_MEM_FENCE); + } + + if (row_idx >= ne01) { + return; + } + + // scatter results to original output rows (reuse sh_src2 from the top) + __local uint out_idx[TILESIZE_N]; + if (lid < TILESIZE_N) { + uint idx = sh_src2[lid]; + if (idx == 0xFFFFFFFF) { + idx = sh_src2[0]; + } + out_idx[lid] = idx * ne01; + } + barrier(CLK_LOCAL_MEM_FENCE); + + const uint m_offset = row + lid; + if (n_real == TILESIZE_N) { + #pragma unroll + for (int t = 1; t < TILESIZE_N; ++t) { + write_imagef(dst, out_idx[t] + m_offset, acc[t]); + } + barrier(CLK_GLOBAL_MEM_FENCE); + write_imagef(dst, out_idx[0] + m_offset, acc[0]); + } else { + for (int t = 0; t < n_real; ++t) { + write_imagef(dst, out_idx[t] + m_offset, acc[t]); + } + } +} diff --git a/ggml/src/ggml-opencl/kernels/gemm_moe_q4_0_q8_1_dp4a.cl b/ggml/src/ggml-opencl/kernels/gemm_moe_q4_0_q8_1_dp4a.cl new file mode 100644 index 000000000..86ff943c5 --- /dev/null +++ b/ggml/src/ggml-opencl/kernels/gemm_moe_q4_0_q8_1_dp4a.cl @@ -0,0 +1,165 @@ +#pragma OPENCL EXTENSION cl_khr_fp16 : enable +#pragma OPENCL EXTENSION cl_khr_subgroups : enable +#ifdef cl_khr_integer_dot_product +#pragma OPENCL EXTENSION cl_khr_integer_dot_product : enable +#endif + +#define TILESIZE_M 64 +#define TILESIZE_N 32 + +// Expand the 4 nibbles held in the low 16 bits of `u` into 4 bytes (one nibble +// per byte, value 0..15), packed for the int8 dp4a. The -8 zero-point is applied +// in the epilogue via the activation sum term (cheaper than biasing every byte). +#define EXP4(u) ( ((uint)((u) & 0x000Fu)) | \ + (((uint)((u) & 0x00F0u)) << 4) | \ + (((uint)((u) & 0x0F00u)) << 8) | \ + (((uint)((u) & 0xF000u)) << 12) ) + +// One token's dp4a dot (8 uints = 32 K elems) + q4_0 scale/zero-point epilogue. +#define MOE_Q40_DP4A_T(t) do { \ + int raw = 0; \ + raw = dot_acc_sat_4x8packed_ss_int(qw[0], sh_qa[t][0], raw); \ + raw = dot_acc_sat_4x8packed_ss_int(qw[1], sh_qa[t][1], raw); \ + raw = dot_acc_sat_4x8packed_ss_int(qw[2], sh_qa[t][2], raw); \ + raw = dot_acc_sat_4x8packed_ss_int(qw[3], sh_qa[t][3], raw); \ + raw = dot_acc_sat_4x8packed_ss_int(qw[4], sh_qa[t][4], raw); \ + raw = dot_acc_sat_4x8packed_ss_int(qw[5], sh_qa[t][5], raw); \ + raw = dot_acc_sat_4x8packed_ss_int(qw[6], sh_qa[t][6], raw); \ + raw = dot_acc_sat_4x8packed_ss_int(qw[7], sh_qa[t][7], raw); \ + acc[t] += d_val * ((float)sh_d[t] * (float)raw - 8.0f * (float)sh_s[t]); \ + } while (0) + +__attribute__((qcom_wave_pair_mode(1))) +kernel void kernel_gemm_moe_q4_0_q8_1_dp4a( + __read_only image1d_buffer_t src0_q, // q4_0 weights (transposed, packed nibbles) + __global half * src0_d, // per-32-block scale + __global uint * src1_qa, // q8_1 activations: int8 quants (as uint, 4/elem) + __global half * src1_da, // q8_1 per-block scale [tok_slot * ne00/32] + __global half * src1_sa, // q8_1 per-block sum*d [tok_slot * ne00/32] + __global uint * src2, // post-router (orig out positions) + __global ushort * src2_emap,// tile -> expert id + __write_only image1d_buffer_t dst, + __global int * total_tiles, + uint ne00, + uint ne01, + int is_ragged // 1: compute only real tokens per tile +) { + const uint block_id_m = get_global_id(1); // m_tile + const uint block_id_n = get_global_id(2); // n_tile + + if (block_id_n >= total_tiles[0]) { + return; + } + + const uint lid = get_local_id(0); // 0..63, == this WI's output row in the M-tile + + const ushort expert_id = src2_emap[block_id_n]; + const uint row = block_id_m * TILESIZE_M; + const uint col = block_id_n * TILESIZE_N; + + const uint num_blocks = ne00 >> 5; // blocks-of-32 per token + const uint row_idx = row + lid; + + const uint ne00_u = ne00 >> 2; // ne00 in uint (int8x4) units + + __local uint sh_qa[TILESIZE_N][8]; // 32 tokens x 8 uints (32 int8) = 1 KiB + __local half sh_d[TILESIZE_N]; + __local half sh_s[TILESIZE_N]; + + // Real-token count for this tile + __local uint sh_src2[TILESIZE_N]; + __local int sh_nreal; + if (lid < TILESIZE_N) { + sh_src2[lid] = src2[col + lid]; + } + barrier(CLK_LOCAL_MEM_FENCE); + if (lid == 0) { + int nr = TILESIZE_N; + if (is_ragged) { + nr = 0; + #pragma unroll + for (int t = 0; t < TILESIZE_N; ++t) { + if (sh_src2[t] != 0xFFFFFFFFu) ++nr; + } + } + sh_nreal = nr; + } + barrier(CLK_LOCAL_MEM_FENCE); + const int n_real = sh_nreal; + + float acc[TILESIZE_N]; + #pragma unroll + for (int t = 0; t < TILESIZE_N; ++t) acc[t] = 0.0f; + + for (uint step = 0; step < ne00; step += 32) { + const uint sub = step >> 5; // 32-block index along K + + // per-32-block scale for this WI's row + const uint d_offset = row_idx + sub * ne01 + expert_id * num_blocks * ne01; + const float d_val = (float)src0_d[d_offset]; + + // repack this WI's 32 weight nibbles into 8 dp4a uints + const uint qoff0 = row + ((ne01 * step) >> 3) + ((expert_id * ne00 * ne01) >> 3); + const uint qoff1 = row + ((ne01 * (step + 16)) >> 3) + ((expert_id * ne00 * ne01) >> 3); + const uint r0 = read_imageui(src0_q, qoff0 + lid).x; + const uint r1 = read_imageui(src0_q, qoff0 + lid + ne01).x; + const uint r2 = read_imageui(src0_q, qoff1 + lid).x; + const uint r3 = read_imageui(src0_q, qoff1 + lid + ne01).x; + uint qw[8]; + qw[0] = EXP4(r0); qw[1] = EXP4(r0 >> 16); + qw[2] = EXP4(r1); qw[3] = EXP4(r1 >> 16); + qw[4] = EXP4(r2); qw[5] = EXP4(r2 >> 16); + qw[6] = EXP4(r3); qw[7] = EXP4(r3 >> 16); + + // cooperatively stage the n_real-token x 32-K int8 activations + const uint stage_lim = (uint)n_real * 8; + for (uint idx = lid; idx < stage_lim; idx += 64) { + const uint t = idx >> 3; + const uint u = idx & 7; + sh_qa[t][u] = src1_qa[(col + t) * ne00_u + (step >> 2) + u]; + } + if (lid < (uint)n_real) { + sh_d[lid] = src1_da[(col + lid) * num_blocks + sub]; + sh_s[lid] = src1_sa[(col + lid) * num_blocks + sub]; + } + barrier(CLK_LOCAL_MEM_FENCE); + + if (n_real == TILESIZE_N) { + #pragma unroll + for (int t = 0; t < TILESIZE_N; ++t) { MOE_Q40_DP4A_T(t); } + } else { + #pragma unroll 4 + for (int t = 0; t < n_real; ++t) { MOE_Q40_DP4A_T(t); } + } + barrier(CLK_LOCAL_MEM_FENCE); + } + + if (row_idx >= ne01) { + return; + } + + // scatter results to original output rows (reuse sh_src2 from the top) + __local uint out_idx[TILESIZE_N]; + if (lid < TILESIZE_N) { + uint idx = sh_src2[lid]; + if (idx == 0xFFFFFFFF) { + idx = sh_src2[0]; + } + out_idx[lid] = idx * ne01; + } + barrier(CLK_LOCAL_MEM_FENCE); + + const uint m_offset = row + lid; + if (n_real == TILESIZE_N) { + #pragma unroll + for (int t = 1; t < TILESIZE_N; ++t) { + write_imagef(dst, out_idx[t] + m_offset, acc[t]); + } + barrier(CLK_GLOBAL_MEM_FENCE); + write_imagef(dst, out_idx[0] + m_offset, acc[0]); + } else { + for (int t = 0; t < n_real; ++t) { + write_imagef(dst, out_idx[t] + m_offset, acc[t]); + } + } +} diff --git a/ggml/src/ggml-opencl/kernels/gemm_moe_q4_k_q8_1_dp4a.cl b/ggml/src/ggml-opencl/kernels/gemm_moe_q4_k_q8_1_dp4a.cl new file mode 100644 index 000000000..540897544 --- /dev/null +++ b/ggml/src/ggml-opencl/kernels/gemm_moe_q4_k_q8_1_dp4a.cl @@ -0,0 +1,202 @@ +#pragma OPENCL EXTENSION cl_khr_fp16 : enable +#pragma OPENCL EXTENSION cl_khr_subgroups : enable +#ifdef cl_khr_integer_dot_product +#pragma OPENCL EXTENSION cl_khr_integer_dot_product : enable +#endif + +// q4_K subblock (32 elems): w_i = scale*q_i - minv, q_i in [0,15], scale = +// d_super*sv6, minv = dmin_super*mn6. With activation block (a_d, a_s, qa[32]): +// Sum_i w_i * a_i = scale * a_d * dp4a(q, qa) - minv * a_s +// where a_s = a_d * Sum(qa) (the q8_1 "s" field) + +#define TILESIZE_M 64 +#define TILESIZE_N 32 +#define QK_K 256 +#define K_SCALE_SIZE 12 + +inline void get_scale_min_k4( + int j, + global const uchar * q, + uchar * d, + uchar * m +) { + if (j < 4) { + *d = q[j] & 63; + *m = q[j+4] & 63; + } else { + *d = (q[j+4] & 0x0F) | ((q[j-4] & 0xC0) >> 2); + *m = ((q[j+4] >> 4) & 0x0F) | ((q[j] & 0xC0) >> 2); + } +} + +// Expand the 4 nibbles held in the low 16 bits of `u` into 4 bytes (one nibble +// per byte, value 0..15), packed for the int8 dp4a. +#define EXP4(u) ( ((uint)((u) & 0x000Fu)) | \ + (((uint)((u) & 0x00F0u)) << 4) | \ + (((uint)((u) & 0x0F00u)) << 8) | \ + (((uint)((u) & 0xF000u)) << 12) ) + +// One token's dp4a dot (8 uints = 32 K elems) + q4_K scale/min epilogue into acc[t]. +#define MOE_Q4K_DP4A_T(t) do { \ + int raw = 0; \ + raw = dot_acc_sat_4x8packed_ss_int(qw[0], sh_qa[t][0], raw); \ + raw = dot_acc_sat_4x8packed_ss_int(qw[1], sh_qa[t][1], raw); \ + raw = dot_acc_sat_4x8packed_ss_int(qw[2], sh_qa[t][2], raw); \ + raw = dot_acc_sat_4x8packed_ss_int(qw[3], sh_qa[t][3], raw); \ + raw = dot_acc_sat_4x8packed_ss_int(qw[4], sh_qa[t][4], raw); \ + raw = dot_acc_sat_4x8packed_ss_int(qw[5], sh_qa[t][5], raw); \ + raw = dot_acc_sat_4x8packed_ss_int(qw[6], sh_qa[t][6], raw); \ + raw = dot_acc_sat_4x8packed_ss_int(qw[7], sh_qa[t][7], raw); \ + acc[t] += scale * (float)sh_d[t] * (float)raw - minv * (float)sh_s[t]; \ + } while (0) + +__attribute__((qcom_wave_pair_mode(1))) +kernel void kernel_gemm_moe_q4_k_q8_1_dp4a( + __read_only image1d_buffer_t src0_q, // q4_K weights (transposed, packed nibbles) + __global half * src0_d, // per-superblock scale + __global half * src0_dm, // per-superblock min + __global uchar * src0_s, // 6-bit scale/min codes + __global uint * src1_qa, // q8_1 activations: int8 quants (as uint, 4/elem) + __global half * src1_da, // q8_1 per-block scale [tok_slot * ne00/32] + __global half * src1_sa, // q8_1 per-block sum*d [tok_slot * ne00/32] + __global uint * src2, // post-router (orig out positions) + __global ushort * src2_emap,// tile -> expert id + __write_only image1d_buffer_t dst, + __global int * total_tiles, + uint ne00, + uint ne01, + int is_ragged // 1: compute only real tokens per tile +) { + const uint block_id_m = get_global_id(1); // m_tile + const uint block_id_n = get_global_id(2); // n_tile + + if (block_id_n >= total_tiles[0]) { + return; + } + + const uint lid = get_local_id(0); // 0..63, == this WI's output row in the M-tile + + const ushort expert_id = src2_emap[block_id_n]; + const uint row = block_id_m * TILESIZE_M; + const uint col = block_id_n * TILESIZE_N; + + const uint num_superblocks = ne00 / QK_K; + const uint scales_per_row = num_superblocks * K_SCALE_SIZE; + const uint row_idx = row + lid; + + const uint ne00_u = ne00 >> 2; // ne00 in uint (int8x4) units + const uint ne00_b = ne00 >> 5; // blocks-of-32 per token + + __local uint sh_qa[TILESIZE_N][8]; // 32 tokens x 8 uints (32 int8) = 1 KiB + __local half sh_d[TILESIZE_N]; + __local half sh_s[TILESIZE_N]; + + // Real token count for this tile + __local uint sh_src2[TILESIZE_N]; + __local int sh_nreal; + if (lid < TILESIZE_N) { + sh_src2[lid] = src2[col + lid]; + } + barrier(CLK_LOCAL_MEM_FENCE); + if (lid == 0) { + int nr = TILESIZE_N; + if (is_ragged) { + nr = 0; + #pragma unroll + for (int t = 0; t < TILESIZE_N; ++t) { + if (sh_src2[t] != 0xFFFFFFFFu) ++nr; + } + } + sh_nreal = nr; + } + barrier(CLK_LOCAL_MEM_FENCE); + const int n_real = sh_nreal; + + float acc[TILESIZE_N]; + #pragma unroll + for (int t = 0; t < TILESIZE_N; ++t) acc[t] = 0.0f; + + for (uint step = 0; step < ne00; step += 32) { + const uint sub = step >> 5; // subblock index along K + const uint sb = sub >> 3; // superblock index + const uint j = sub & 7; // subblock within superblock + + // --- weight scale / min for this WI's row, this subblock --- + const uint d_offset = row + sb * ne01 + expert_id * num_superblocks * ne01 + lid; + const float d_val = (float)src0_d[d_offset]; + const float dm_val = (float)src0_dm[d_offset]; + + global const uchar * sc = src0_s + (expert_id * ne01 + row_idx) * scales_per_row + sb * K_SCALE_SIZE; + uchar sv, mn; + get_scale_min_k4(j, sc, &sv, &mn); + const float scale = d_val * (float)sv; + const float minv = dm_val * (float)mn; + + // --- repack this WI's 32 weight nibbles into 8 dp4a uints --- + const uint qoff0 = row + ((ne01 * step) >> 3) + ((expert_id * ne00 * ne01) >> 3); + const uint qoff1 = row + ((ne01 * (step + 16)) >> 3) + ((expert_id * ne00 * ne01) >> 3); + const uint r0 = read_imageui(src0_q, qoff0 + lid).x; + const uint r1 = read_imageui(src0_q, qoff0 + lid + ne01).x; + const uint r2 = read_imageui(src0_q, qoff1 + lid).x; + const uint r3 = read_imageui(src0_q, qoff1 + lid + ne01).x; + uint qw[8]; + qw[0] = EXP4(r0); qw[1] = EXP4(r0 >> 16); + qw[2] = EXP4(r1); qw[3] = EXP4(r1 >> 16); + qw[4] = EXP4(r2); qw[5] = EXP4(r2 >> 16); + qw[6] = EXP4(r3); qw[7] = EXP4(r3 >> 16); + + // --- cooperatively stage the n_real-token x 32-K int8 activations to LDS --- + const uint stage_lim = (uint)n_real * 8; + for (uint idx = lid; idx < stage_lim; idx += 64) { + const uint t = idx >> 3; + const uint u = idx & 7; + sh_qa[t][u] = src1_qa[(col + t) * ne00_u + (step >> 2) + u]; + } + if (lid < (uint)n_real) { + sh_d[lid] = src1_da[(col + lid) * ne00_b + sub]; + sh_s[lid] = src1_sa[(col + lid) * ne00_b + sub]; + } + barrier(CLK_LOCAL_MEM_FENCE); + + // dp4a - each real token sum over 8 uints (32 K), then scale/min + // Full tiles keep the fully-unrolled 32-wide loop; + // partial tiles run only n_real (saves the padded-slot dp4a + staging). + if (n_real == TILESIZE_N) { + #pragma unroll + for (int t = 0; t < TILESIZE_N; ++t) { MOE_Q4K_DP4A_T(t); } + } else { + #pragma unroll 4 + for (int t = 0; t < n_real; ++t) { MOE_Q4K_DP4A_T(t); } + } + barrier(CLK_LOCAL_MEM_FENCE); + } + + if (row_idx >= ne01) { + return; + } + + // scatter results to original output rows + __local uint out_idx[TILESIZE_N]; + if (lid < TILESIZE_N) { + uint idx = sh_src2[lid]; + if (idx == 0xFFFFFFFF) { + idx = sh_src2[0]; + } + out_idx[lid] = idx * ne01; + } + barrier(CLK_LOCAL_MEM_FENCE); + + const uint m_offset = row + lid; + if (n_real == TILESIZE_N) { + #pragma unroll + for (int t = 1; t < TILESIZE_N; ++t) { + write_imagef(dst, out_idx[t] + m_offset, acc[t]); + } + barrier(CLK_GLOBAL_MEM_FENCE); + write_imagef(dst, out_idx[0] + m_offset, acc[0]); + } else { + for (int t = 0; t < n_real; ++t) { + write_imagef(dst, out_idx[t] + m_offset, acc[t]); + } + } +} diff --git a/ggml/src/ggml-opencl/kernels/gemm_moe_q6_k_q8_1_dp4a.cl b/ggml/src/ggml-opencl/kernels/gemm_moe_q6_k_q8_1_dp4a.cl new file mode 100644 index 000000000..35e63dcab --- /dev/null +++ b/ggml/src/ggml-opencl/kernels/gemm_moe_q6_k_q8_1_dp4a.cl @@ -0,0 +1,196 @@ +#pragma OPENCL EXTENSION cl_khr_fp16 : enable +#pragma OPENCL EXTENSION cl_khr_subgroups : enable +#ifdef cl_khr_integer_dot_product +#pragma OPENCL EXTENSION cl_khr_integer_dot_product : enable +#endif + +#define TILESIZE_N 32 +#define QK_K 256 + +// 4 nibbles in the low 16 bits of `u` -> 4 bytes (value 0..15, in bits 0-3). +#define EXP4(u) ( ((uint)((u) & 0x000Fu)) | \ + (((uint)((u) & 0x00F0u)) << 4) | \ + (((uint)((u) & 0x0F00u)) << 8) | \ + (((uint)((u) & 0xF000u)) << 12) ) + +// 4 2-bit highs in byte `b` (8 bits) -> 4 bytes, value 0..3 in bits 4-5 +// (pre-multiplied by 16 so it ORs with the EXP4 nibble to form q6 in 0..63). +#define EXP2(b) ( (((uint)((b) & 0x03u)) << 4) | \ + (((uint)((b) & 0x0Cu)) << 10) | \ + (((uint)((b) & 0x30u)) << 16) | \ + (((uint)((b) & 0xC0u)) << 22) ) + +// q6 (0..63, bits 0-5 of each byte) -> (q6-32) as a signed int8 per byte. +// Flipping bit5 subtracts 32 in 6-bit two's complement; then replicate bit5 +// into bits 6-7 to sign-extend to int8. Per-byte, no inter-byte carry. +inline uint SIGN6(uint q6p) { + uint x = q6p ^ 0x20202020u; + uint s = x & 0x20202020u; + return x | (s << 1) | (s << 2); +} + +inline int dp4a_q6(uint qw0, uint qw1, uint qw2, uint qw3, + uint a0, uint a1, uint a2, uint a3) { + int raw = 0; + raw = dot_acc_sat_4x8packed_ss_int(qw0, a0, raw); + raw = dot_acc_sat_4x8packed_ss_int(qw1, a1, raw); + raw = dot_acc_sat_4x8packed_ss_int(qw2, a2, raw); + raw = dot_acc_sat_4x8packed_ss_int(qw3, a3, raw); + return raw; +} + +// One token's q6_K dp4a dot (two halves, per-16 scales) + epilogue into acc[t]. +#define MOE_Q6K_DP4A_T(t) do { \ + const int raw1 = dp4a_q6(qw[0], qw[1], qw[2], qw[3], sh_qa[t][0], sh_qa[t][1], sh_qa[t][2], sh_qa[t][3]); \ + const int raw2 = dp4a_q6(qw[4], qw[5], qw[6], qw[7], sh_qa[t][4], sh_qa[t][5], sh_qa[t][6], sh_qa[t][7]); \ + const float a_d = (float)sh_d[t]; \ + acc[t] += scale0 * a_d * (float)raw1 + scale1 * a_d * (float)raw2; \ + } while (0) + +__attribute__((qcom_wave_pair_mode(1))) +kernel void kernel_gemm_moe_q6_k_q8_1_dp4a( + __read_only image1d_buffer_t src0_ql, // q6_K low nibbles (image, q4_K-style layout) + __global uint * src0_qh, // q6_K high 2-bit (16 elems/uint) + __global char * src0_s, // int8 scales (one per 16 elems) + __global half * src0_d, // per-superblock scale + __global uint * src1_qa, // q8_1 activations int8 (as uint, 4/elem) + __global half * src1_da, // q8_1 per-block scale [tok_slot * ne00/32] + __global uint * src2, // post-router (orig out positions) + __global ushort * src2_emap, // tile -> expert id + __write_only image1d_buffer_t dst, + __global int * total_tiles, + uint ne00, + uint ne01, + int is_ragged // 1: compute only real tokens per tile +) { + const uint block_id_m = get_global_id(1); + const uint block_id_n = get_global_id(2); + + if (block_id_n >= total_tiles[0]) { + return; + } + + const uint lid = get_local_id(0); // 0..63 -> row within M-tile + + const ushort expert_id = src2_emap[block_id_n]; + const uint row = block_id_m * 64; + const uint col = block_id_n * TILESIZE_N; + + const uint num_superblocks = ne00 / QK_K; + const uint scales_per_row = num_superblocks * 16; + const uint row_idx = row + lid; + + const uint ne00_u = ne00 >> 2; + const uint ne00_b = ne00 >> 5; + + __local uint sh_qa[TILESIZE_N][8]; + __local half sh_d[TILESIZE_N]; + + // Real token count for this tile + __local uint sh_src2[TILESIZE_N]; + __local int sh_nreal; + if (lid < TILESIZE_N) { + sh_src2[lid] = src2[col + lid]; + } + barrier(CLK_LOCAL_MEM_FENCE); + if (lid == 0) { + int nr = TILESIZE_N; + if (is_ragged) { + nr = 0; + #pragma unroll + for (int t = 0; t < TILESIZE_N; ++t) { + if (sh_src2[t] != 0xFFFFFFFFu) ++nr; + } + } + sh_nreal = nr; + } + barrier(CLK_LOCAL_MEM_FENCE); + const int n_real = sh_nreal; + + float acc[TILESIZE_N]; + #pragma unroll + for (int t = 0; t < TILESIZE_N; ++t) acc[t] = 0.0f; + + for (uint step = 0; step < ne00; step += 32) { + const uint sub = step >> 5; + const uint sb = sub >> 3; + const uint j = sub & 7; + + const float d_val = (float)src0_d[row + sb * ne01 + expert_id * num_superblocks * ne01 + lid]; + global const char * sc = src0_s + (expert_id * ne01 + row_idx) * scales_per_row + sb * 16; + const float scale0 = d_val * (float)sc[j * 2]; + const float scale1 = d_val * (float)sc[j * 2 + 1]; + + // high bits: one uint covers 16 elems; first/second 16 of this 32-block + const uint qh_base = row + (sub * 2) * ne01 + expert_id * (num_superblocks * 16) * ne01 + lid; + const uint qh1 = src0_qh[qh_base]; + const uint qh2 = src0_qh[qh_base + ne01]; + + // low nibbles: same image layout as q4_K (8 ushorts over the 32 K) + const uint qoff0 = row + ((ne01 * step) >> 3) + ((expert_id * ne00 * ne01) >> 3); + const uint qoff1 = row + ((ne01 * (step + 16)) >> 3) + ((expert_id * ne00 * ne01) >> 3); + const uint r0 = read_imageui(src0_ql, qoff0 + lid).x; + const uint r1 = read_imageui(src0_ql, qoff0 + lid + ne01).x; + const uint r2 = read_imageui(src0_ql, qoff1 + lid).x; + const uint r3 = read_imageui(src0_ql, qoff1 + lid + ne01).x; + + uint qw[8]; + qw[0] = SIGN6(EXP4(r0) | EXP2((qh1) & 0xFFu)); + qw[1] = SIGN6(EXP4(r0 >> 16) | EXP2((qh1 >> 8) & 0xFFu)); + qw[2] = SIGN6(EXP4(r1) | EXP2((qh1 >> 16) & 0xFFu)); + qw[3] = SIGN6(EXP4(r1 >> 16) | EXP2((qh1 >> 24) & 0xFFu)); + qw[4] = SIGN6(EXP4(r2) | EXP2((qh2) & 0xFFu)); + qw[5] = SIGN6(EXP4(r2 >> 16) | EXP2((qh2 >> 8) & 0xFFu)); + qw[6] = SIGN6(EXP4(r3) | EXP2((qh2 >> 16) & 0xFFu)); + qw[7] = SIGN6(EXP4(r3 >> 16) | EXP2((qh2 >> 24) & 0xFFu)); + + const uint stage_lim = (uint)n_real * 8; + for (uint idx = lid; idx < stage_lim; idx += 64) { + const uint t = idx >> 3; + const uint u = idx & 7; + sh_qa[t][u] = src1_qa[(col + t) * ne00_u + (step >> 2) + u]; + } + if (lid < (uint)n_real) { + sh_d[lid] = src1_da[(col + lid) * ne00_b + sub]; + } + barrier(CLK_LOCAL_MEM_FENCE); + + // Full tiles keep the fully-unrolled 32-wide loop; partial tiles run n_real. + if (n_real == TILESIZE_N) { + #pragma unroll + for (int t = 0; t < TILESIZE_N; ++t) { MOE_Q6K_DP4A_T(t); } + } else { + #pragma unroll 4 + for (int t = 0; t < n_real; ++t) { MOE_Q6K_DP4A_T(t); } + } + barrier(CLK_LOCAL_MEM_FENCE); + } + + if (row_idx >= ne01) { + return; + } + + __local uint out_idx[TILESIZE_N]; + if (lid < TILESIZE_N) { + uint idx = sh_src2[lid]; + if (idx == 0xFFFFFFFF) { + idx = sh_src2[0]; + } + out_idx[lid] = idx * ne01; + } + barrier(CLK_LOCAL_MEM_FENCE); + + const uint m_offset = row + lid; + if (n_real == TILESIZE_N) { + #pragma unroll + for (int t = 1; t < TILESIZE_N; ++t) { + write_imagef(dst, out_idx[t] + m_offset, acc[t]); + } + barrier(CLK_GLOBAL_MEM_FENCE); + write_imagef(dst, out_idx[0] + m_offset, acc[0]); + } else { + for (int t = 0; t < n_real; ++t) { + write_imagef(dst, out_idx[t] + m_offset, acc[t]); + } + } +} diff --git a/ggml/src/ggml-opencl/kernels/gemm_moe_q8_0_f32_ns.cl b/ggml/src/ggml-opencl/kernels/gemm_moe_q8_0_f32_ns.cl new file mode 100644 index 000000000..dc0f0ed86 --- /dev/null +++ b/ggml/src/ggml-opencl/kernels/gemm_moe_q8_0_f32_ns.cl @@ -0,0 +1,221 @@ +#pragma OPENCL EXTENSION cl_khr_fp16 : enable +#pragma OPENCL EXTENSION cl_khr_subgroups : enable +#pragma OPENCL EXTENSION cl_qcom_subgroup_uniform_load: enable +#pragma OPENCL EXTENSION cl_qcom_subgroup_constant_load: enable +#pragma OPENCL EXTENSION cl_qcom_extra_vector_types : enable + +#define TILESIZE_K 16 +#define TILESIZE_M 64 +#define TILESIZE_N 32 + +// q8_0: 16 signed int8 weights (one uint4 = 16 chars) -> half16, scaled. +#define dequantize_q8_0(q4, a_f16, scale) \ + a_f16 = convert_half16(as_char16(q4)) * scale; + +#define dotx16_reduce8(a_reg, b_lm, c_reg, lm_offset) \ + acc.s0 = dot(a_reg.s0123, b_lm[lm_offset + 0]); \ + acc.s1 = dot(a_reg.s0123, b_lm[lm_offset + 1]); \ + acc.s2 = dot(a_reg.s0123, b_lm[lm_offset + 2]); \ + acc.s3 = dot(a_reg.s0123, b_lm[lm_offset + 3]); \ + acc.s4 = dot(a_reg.s0123, b_lm[lm_offset + 4]); \ + acc.s5 = dot(a_reg.s0123, b_lm[lm_offset + 5]); \ + acc.s6 = dot(a_reg.s0123, b_lm[lm_offset + 6]); \ + acc.s7 = dot(a_reg.s0123, b_lm[lm_offset + 7]); \ + acc.s8 = dot(a_reg.s0123, b_lm[lm_offset + 8]); \ + acc.s9 = dot(a_reg.s0123, b_lm[lm_offset + 9]); \ + acc.sa = dot(a_reg.s0123, b_lm[lm_offset + 10]); \ + acc.sb = dot(a_reg.s0123, b_lm[lm_offset + 11]); \ + acc.sc = dot(a_reg.s0123, b_lm[lm_offset + 12]); \ + acc.sd = dot(a_reg.s0123, b_lm[lm_offset + 13]); \ + acc.se = dot(a_reg.s0123, b_lm[lm_offset + 14]); \ + acc.sf = dot(a_reg.s0123, b_lm[lm_offset + 15]); \ + acc.s0 += dot(a_reg.s4567, b_lm[lm_offset + 32]); \ + acc.s1 += dot(a_reg.s4567, b_lm[lm_offset + 33]); \ + acc.s2 += dot(a_reg.s4567, b_lm[lm_offset + 34]); \ + acc.s3 += dot(a_reg.s4567, b_lm[lm_offset + 35]); \ + acc.s4 += dot(a_reg.s4567, b_lm[lm_offset + 36]); \ + acc.s5 += dot(a_reg.s4567, b_lm[lm_offset + 37]); \ + acc.s6 += dot(a_reg.s4567, b_lm[lm_offset + 38]); \ + acc.s7 += dot(a_reg.s4567, b_lm[lm_offset + 39]); \ + acc.s8 += dot(a_reg.s4567, b_lm[lm_offset + 40]); \ + acc.s9 += dot(a_reg.s4567, b_lm[lm_offset + 41]); \ + acc.sa += dot(a_reg.s4567, b_lm[lm_offset + 42]); \ + acc.sb += dot(a_reg.s4567, b_lm[lm_offset + 43]); \ + acc.sc += dot(a_reg.s4567, b_lm[lm_offset + 44]); \ + acc.sd += dot(a_reg.s4567, b_lm[lm_offset + 45]); \ + acc.se += dot(a_reg.s4567, b_lm[lm_offset + 46]); \ + acc.sf += dot(a_reg.s4567, b_lm[lm_offset + 47]); \ + c_reg.lo += convert_float8(acc.lo); \ + c_reg.hi += convert_float8(acc.hi); \ + acc.s0 = dot(a_reg.s89ab, b_lm[lm_offset + 64]); \ + acc.s1 = dot(a_reg.s89ab, b_lm[lm_offset + 65]); \ + acc.s2 = dot(a_reg.s89ab, b_lm[lm_offset + 66]); \ + acc.s3 = dot(a_reg.s89ab, b_lm[lm_offset + 67]); \ + acc.s4 = dot(a_reg.s89ab, b_lm[lm_offset + 68]); \ + acc.s5 = dot(a_reg.s89ab, b_lm[lm_offset + 69]); \ + acc.s6 = dot(a_reg.s89ab, b_lm[lm_offset + 70]); \ + acc.s7 = dot(a_reg.s89ab, b_lm[lm_offset + 71]); \ + acc.s8 = dot(a_reg.s89ab, b_lm[lm_offset + 72]); \ + acc.s9 = dot(a_reg.s89ab, b_lm[lm_offset + 73]); \ + acc.sa = dot(a_reg.s89ab, b_lm[lm_offset + 74]); \ + acc.sb = dot(a_reg.s89ab, b_lm[lm_offset + 75]); \ + acc.sc = dot(a_reg.s89ab, b_lm[lm_offset + 76]); \ + acc.sd = dot(a_reg.s89ab, b_lm[lm_offset + 77]); \ + acc.se = dot(a_reg.s89ab, b_lm[lm_offset + 78]); \ + acc.sf = dot(a_reg.s89ab, b_lm[lm_offset + 79]); \ + acc.s0 += dot(a_reg.scdef, b_lm[lm_offset + 96]); \ + acc.s1 += dot(a_reg.scdef, b_lm[lm_offset + 97]); \ + acc.s2 += dot(a_reg.scdef, b_lm[lm_offset + 98]); \ + acc.s3 += dot(a_reg.scdef, b_lm[lm_offset + 99]); \ + acc.s4 += dot(a_reg.scdef, b_lm[lm_offset + 100]); \ + acc.s5 += dot(a_reg.scdef, b_lm[lm_offset + 101]); \ + acc.s6 += dot(a_reg.scdef, b_lm[lm_offset + 102]); \ + acc.s7 += dot(a_reg.scdef, b_lm[lm_offset + 103]); \ + acc.s8 += dot(a_reg.scdef, b_lm[lm_offset + 104]); \ + acc.s9 += dot(a_reg.scdef, b_lm[lm_offset + 105]); \ + acc.sa += dot(a_reg.scdef, b_lm[lm_offset + 106]); \ + acc.sb += dot(a_reg.scdef, b_lm[lm_offset + 107]); \ + acc.sc += dot(a_reg.scdef, b_lm[lm_offset + 108]); \ + acc.sd += dot(a_reg.scdef, b_lm[lm_offset + 109]); \ + acc.se += dot(a_reg.scdef, b_lm[lm_offset + 110]); \ + acc.sf += dot(a_reg.scdef, b_lm[lm_offset + 111]); \ + c_reg.lo += convert_float8(acc.lo); \ + c_reg.hi += convert_float8(acc.hi); \ + + +__attribute__((qcom_wave_pair_mode(1))) +kernel void kernel_gemm_moe_q8_0_f32_ns( + __global char * src0_q, // flat q8_0 quants [n_expert*ne01*ne00] + __global half * src0_d, // flat q8_0 scales [n_expert*ne01*nb] + __read_only image1d_buffer_t src1, // reordered activations (f32) + __global uint * src2, // post-router out indices + __global ushort * src2_emap,// expert per tile + __write_only image1d_buffer_t dst, + __global int * total_tiles, + uint ne00, + uint ne01 +) { + uint block_id_m = get_global_id(1); // m_tile + uint block_id_n = get_global_id(2); // n_tile + + if (block_id_n >= total_tiles[0]) { + return; + } + + __private half16 reg_a; + __private float32 reg_c = (float32)(0); + __local half4 shared_b[128]; + + const ushort expert_id = src2_emap[block_id_n]; + + const uint row = block_id_m * TILESIZE_M; + const uint col = block_id_n * TILESIZE_N; + + const uint nb = ne00 >> 5; // blocks per row (ne00/32) + const uint w_row = expert_id * ne01 + row + get_local_id(0); // this lane's output row + __global char * w_q = src0_q + (ulong)w_row * ne00; // char base for the row + __global half * w_d = src0_d + (ulong)w_row * nb; // scale base for the row + + uint sub_block_id_m = get_local_id(0); + uint2 b_global_offset; + b_global_offset.x = ((sub_block_id_m & 3) << 2) + (sub_block_id_m >> 2) * ne00; + b_global_offset.y = b_global_offset.x + (16 * ne00); + uint2 b_local_offset; + b_local_offset.x = (sub_block_id_m & 3) * 32 + (sub_block_id_m >> 2); + b_local_offset.y = b_local_offset.x + 16; + + // Loop along K axis, 32 elements per iteration, split into 2 sub-blocks. + for (uint step = 0; step < ne00; step += TILESIZE_K * 2) { + half s = w_d[step >> 5]; // one q8_0 scale per 32-element block + + // First sub-block: 16 weights (16 chars = one uint4) at K=step + uint4 q8x16 = *((__global uint4 *)(w_q + step)); + + uint b_sub_offset = col * ne00 + step; + float8 bx8_f32; + bx8_f32.lo = read_imagef(src1, (b_sub_offset + b_global_offset.x) / 4); + bx8_f32.hi = read_imagef(src1, (b_sub_offset + b_global_offset.y) / 4); + half8 bx8_f16 = convert_half8(bx8_f32); + shared_b[b_local_offset.x] = bx8_f16.lo; + shared_b[b_local_offset.y] = bx8_f16.hi; + + dequantize_q8_0(q8x16, reg_a, s); + + sub_group_barrier(CLK_LOCAL_MEM_FENCE); + + half16 acc; + dotx16_reduce8(reg_a, shared_b, reg_c.lo, 0); + dotx16_reduce8(reg_a, shared_b, reg_c.hi, 16); + + // Second sub-block: next 16 weights at K=step+16 + uint half_step = step + TILESIZE_K; + q8x16 = *((__global uint4 *)(w_q + half_step)); + b_sub_offset = col * ne00 + half_step; + + bx8_f32.lo = read_imagef(src1, (b_sub_offset + b_global_offset.x) / 4); + bx8_f32.hi = read_imagef(src1, (b_sub_offset + b_global_offset.y) / 4); + bx8_f16 = convert_half8(bx8_f32); + shared_b[b_local_offset.x] = bx8_f16.lo; + shared_b[b_local_offset.y] = bx8_f16.hi; + + dequantize_q8_0(q8x16, reg_a, s); + + sub_group_barrier(CLK_LOCAL_MEM_FENCE); + + dotx16_reduce8(reg_a, shared_b, reg_c.lo, 0); + dotx16_reduce8(reg_a, shared_b, reg_c.hi, 16); + } + + if ((get_global_id(0) + block_id_m * TILESIZE_M) >= ne01) { + return; + } + + __local uint out_idx[TILESIZE_N]; + + if (get_local_id(0) < TILESIZE_N) { + uint idx = src2[block_id_n * TILESIZE_N + get_local_id(0)]; + if (idx == 0xFFFFFFFF) { + idx = src2[block_id_n * TILESIZE_N + 0]; + } + out_idx[get_local_id(0)] = idx * ne01; + } + + barrier(CLK_LOCAL_MEM_FENCE); + + uint m_offset = row + get_local_id(0); + + write_imagef(dst, out_idx[1] + m_offset, (reg_c.s1)); + write_imagef(dst, out_idx[2] + m_offset, (reg_c.s2)); + write_imagef(dst, out_idx[3] + m_offset, (reg_c.s3)); + write_imagef(dst, out_idx[4] + m_offset, (reg_c.s4)); + write_imagef(dst, out_idx[5] + m_offset, (reg_c.s5)); + write_imagef(dst, out_idx[6] + m_offset, (reg_c.s6)); + write_imagef(dst, out_idx[7] + m_offset, (reg_c.s7)); + write_imagef(dst, out_idx[8] + m_offset, (reg_c.s8)); + write_imagef(dst, out_idx[9] + m_offset, (reg_c.s9)); + write_imagef(dst, out_idx[10] + m_offset, (reg_c.sa)); + write_imagef(dst, out_idx[11] + m_offset, (reg_c.sb)); + write_imagef(dst, out_idx[12] + m_offset, (reg_c.sc)); + write_imagef(dst, out_idx[13] + m_offset, (reg_c.sd)); + write_imagef(dst, out_idx[14] + m_offset, (reg_c.se)); + write_imagef(dst, out_idx[15] + m_offset, (reg_c.sf)); + write_imagef(dst, out_idx[16] + m_offset, (reg_c.sg)); + write_imagef(dst, out_idx[17] + m_offset, (reg_c.sh)); + write_imagef(dst, out_idx[18] + m_offset, (reg_c.si)); + write_imagef(dst, out_idx[19] + m_offset, (reg_c.sj)); + write_imagef(dst, out_idx[20] + m_offset, (reg_c.sk)); + write_imagef(dst, out_idx[21] + m_offset, (reg_c.sl)); + write_imagef(dst, out_idx[22] + m_offset, (reg_c.sm)); + write_imagef(dst, out_idx[23] + m_offset, (reg_c.sn)); + write_imagef(dst, out_idx[24] + m_offset, (reg_c.so)); + write_imagef(dst, out_idx[25] + m_offset, (reg_c.sp)); + write_imagef(dst, out_idx[26] + m_offset, (reg_c.sq)); + write_imagef(dst, out_idx[27] + m_offset, (reg_c.sr)); + write_imagef(dst, out_idx[28] + m_offset, (reg_c.ss)); + write_imagef(dst, out_idx[29] + m_offset, (reg_c.st)); + write_imagef(dst, out_idx[30] + m_offset, (reg_c.su)); + write_imagef(dst, out_idx[31] + m_offset, (reg_c.sv)); + + barrier(CLK_GLOBAL_MEM_FENCE); + write_imagef(dst, out_idx[0] + m_offset, (reg_c.s0)); +} diff --git a/ggml/src/ggml-opencl/kernels/gemm_moe_q8_1_dp4a.cl b/ggml/src/ggml-opencl/kernels/gemm_moe_q8_1_dp4a.cl new file mode 100644 index 000000000..39bf5d832 --- /dev/null +++ b/ggml/src/ggml-opencl/kernels/gemm_moe_q8_1_dp4a.cl @@ -0,0 +1,221 @@ +#pragma OPENCL EXTENSION cl_khr_fp16 : enable +#pragma OPENCL EXTENSION cl_khr_subgroups : enable +#ifdef cl_khr_integer_dot_product +#pragma OPENCL EXTENSION cl_khr_integer_dot_product : enable +#endif + +// Generic int8 dp4a MoE GEMM, specialized versions also exist +// MOE_QT: +// 4 (q4_K)/41(q4_1)/40(q4_0) NIBBLE image low nibbles -> EXP4 +// 5 (q5_K)/51(q5_1)/50(q5_0) NIBBLE+HI image nibbles + qh high-bit plane +// 6 (q6_K) Q6 image nibbles + qh 2-bit -> SIGN6((nibble|hi2)) +// 80(q8_0)/82(mxfp4) INT8 global int8 codes (mxfp4: convert applies kvalues LUT) + +#define TILESIZE_M 64 +#define TILESIZE_N 32 +#define QK_K 256 + +#ifndef MOE_QT +#define MOE_QT 4 +#endif + +// 4 nibbles in low 16 bits of u -> 4 bytes (value 0..15) +#define EXP4(u) ( ((uint)((u) & 0x000Fu)) | \ + (((uint)((u) & 0x00F0u)) << 4) | \ + (((uint)((u) & 0x0F00u)) << 8) | \ + (((uint)((u) & 0xF000u)) << 12) ) +// 4 2-bit highs in byte b -> 4 bytes, bits 4-5 (q6_K) +#define EXP2(b) ( (((uint)((b) & 0x03u)) << 4) | \ + (((uint)((b) & 0x0Cu)) << 10) | \ + (((uint)((b) & 0x30u)) << 16) | \ + (((uint)((b) & 0xC0u)) << 22) ) + +// q6 (0..63) -> (q6-32) signed int8/byte (no inter-byte carry) +inline uint SIGN6(uint q6p){ uint x=q6p^0x20202020u; uint s=x&0x20202020u; return x|(s<<1)|(s<<2); } + +// 4 high bits (one per element, in bits 0..3 of h) -> bit4 of each of 4 bytes (5-bit hi) +#define EXP1(h) ( (((uint)((h) & 0x1u)) << 4) | \ + (((uint)((h) & 0x2u)) << 11) | \ + (((uint)((h) & 0x4u)) << 18) | \ + (((uint)((h) & 0x8u)) << 25) ) + +// per-type weight params + per-32-step unpack into qw[8] (8 int8 uints) +#if MOE_QT == 4 || MOE_QT == 41 || MOE_QT == 40 + #define WEIGHT_PARAMS __read_only image1d_buffer_t src0_q, + #define LOAD_QW(step, sub) \ + uint qw[8]; { \ + const uint qoff0 = row + ((ne01*(step))>>3) + ((expert_id*ne00*ne01)>>3); \ + const uint qoff1 = row + ((ne01*((step)+16))>>3) + ((expert_id*ne00*ne01)>>3); \ + const uint r0=read_imageui(src0_q,qoff0+lid).x, r1=read_imageui(src0_q,qoff0+lid+ne01).x; \ + const uint r2=read_imageui(src0_q,qoff1+lid).x, r3=read_imageui(src0_q,qoff1+lid+ne01).x; \ + qw[0]=EXP4(r0); qw[1]=EXP4(r0>>16); qw[2]=EXP4(r1); qw[3]=EXP4(r1>>16); \ + qw[4]=EXP4(r2); qw[5]=EXP4(r2>>16); qw[6]=EXP4(r3); qw[7]=EXP4(r3>>16); } + +#elif MOE_QT == 5 || MOE_QT == 51 || MOE_QT == 50 + // low nibbles via image (q4_K layout) + high-bit plane src0_qh: 1 uint per 32-block + // (bit i = high bit of element i). qh laid out [expert][block][row] to match the + // existing q5_0 trans4 convert + #define WEIGHT_PARAMS __read_only image1d_buffer_t src0_q, __global uint * src0_qh, + #define LOAD_QW(step, sub) \ + uint qw[8]; { \ + const uint qoff0 = row + ((ne01*(step))>>3) + ((expert_id*ne00*ne01)>>3); \ + const uint qoff1 = row + ((ne01*((step)+16))>>3) + ((expert_id*ne00*ne01)>>3); \ + const uint r0=read_imageui(src0_q,qoff0+lid).x, r1=read_imageui(src0_q,qoff0+lid+ne01).x; \ + const uint r2=read_imageui(src0_q,qoff1+lid).x, r3=read_imageui(src0_q,qoff1+lid+ne01).x; \ + const uint h = src0_qh[row_idx + (sub)*ne01 + expert_id*(ne00>>5)*ne01]; \ + qw[0]=EXP4(r0)|EXP1(h); qw[1]=EXP4(r0>>16)|EXP1(h>>4); \ + qw[2]=EXP4(r1)|EXP1(h>>8); qw[3]=EXP4(r1>>16)|EXP1(h>>12); \ + qw[4]=EXP4(r2)|EXP1(h>>16); qw[5]=EXP4(r2>>16)|EXP1(h>>20); \ + qw[6]=EXP4(r3)|EXP1(h>>24); qw[7]=EXP4(r3>>16)|EXP1(h>>28); } + +#elif MOE_QT == 6 + #define WEIGHT_PARAMS __read_only image1d_buffer_t src0_ql, __global uint * src0_qh, + #define LOAD_QW(step, sub) \ + uint qw[8]; { \ + const uint qoff0 = row + ((ne01*(step))>>3) + ((expert_id*ne00*ne01)>>3); \ + const uint qoff1 = row + ((ne01*((step)+16))>>3) + ((expert_id*ne00*ne01)>>3); \ + const uint r0=read_imageui(src0_ql,qoff0+lid).x, r1=read_imageui(src0_ql,qoff0+lid+ne01).x; \ + const uint r2=read_imageui(src0_ql,qoff1+lid).x, r3=read_imageui(src0_ql,qoff1+lid+ne01).x; \ + const uint qhb = row + ((sub)*2)*ne01 + expert_id*((ne00>>5)*2)*ne01 + lid; \ + const uint qh1=src0_qh[qhb], qh2=src0_qh[qhb+ne01]; \ + qw[0]=SIGN6(EXP4(r0)|EXP2(qh1&0xFFu)); qw[1]=SIGN6(EXP4(r0>>16)|EXP2((qh1>>8)&0xFFu)); \ + qw[2]=SIGN6(EXP4(r1)|EXP2((qh1>>16)&0xFFu)); qw[3]=SIGN6(EXP4(r1>>16)|EXP2((qh1>>24)&0xFFu)); \ + qw[4]=SIGN6(EXP4(r2)|EXP2(qh2&0xFFu)); qw[5]=SIGN6(EXP4(r2>>16)|EXP2((qh2>>8)&0xFFu)); \ + qw[6]=SIGN6(EXP4(r3)|EXP2((qh2>>16)&0xFFu)); qw[7]=SIGN6(EXP4(r3>>16)|EXP2((qh2>>24)&0xFFu)); } + +#elif MOE_QT == 80 || MOE_QT == 82 + // 8-bit direct: int8 codes 8 uints / 32-block, [expert][row][8*sub]. mxfp4: the + // convert resolves kvalues_mxfp4[nibble] -> int8 and stores the e8m0_half scale. + #define WEIGHT_PARAMS __global uint * src0_q8, + #define LOAD_QW(step, sub) \ + uint qw[8]; { \ + const uint qb = (expert_id*ne01 + row_idx)*(ne00>>2) + (sub)*8; \ + qw[0]=src0_q8[qb+0]; qw[1]=src0_q8[qb+1]; qw[2]=src0_q8[qb+2]; qw[3]=src0_q8[qb+3]; \ + qw[4]=src0_q8[qb+4]; qw[5]=src0_q8[qb+5]; qw[6]=src0_q8[qb+6]; qw[7]=src0_q8[qb+7]; } +#else + #error "unknown MOE_QT" +#endif + +inline int dp4a4(uint w0,uint w1,uint w2,uint w3,uint a0,uint a1,uint a2,uint a3){ + int r=0; r=dot_acc_sat_4x8packed_ss_int(w0,a0,r); r=dot_acc_sat_4x8packed_ss_int(w1,a1,r); + r=dot_acc_sat_4x8packed_ss_int(w2,a2,r); r=dot_acc_sat_4x8packed_ss_int(w3,a3,r); return r; } + +// One token's two-half dp4a + uniform scale/min epilogue into acc[t]. +#define MOE_DP4A_T(t) do { \ + const int raw1 = dp4a4(qw[0],qw[1],qw[2],qw[3], sh_qa[t][0],sh_qa[t][1],sh_qa[t][2],sh_qa[t][3]); \ + const int raw2 = dp4a4(qw[4],qw[5],qw[6],qw[7], sh_qa[t][4],sh_qa[t][5],sh_qa[t][6],sh_qa[t][7]); \ + const float a_d = (float)sh_d[t]; \ + acc[t] += sc0*a_d*(float)raw1 + sc1*a_d*(float)raw2 - mn*(float)sh_s[t]; \ + } while (0) + +__attribute__((qcom_wave_pair_mode(1))) +kernel void kernel_gemm_moe_q8_1_dp4a( + WEIGHT_PARAMS // per-type native weight buffer(s) + __global half * src0_scale,// uniform f16 16/superblock (per-16), [expert,row] + __global half * src0_min, // uniform f16 8/superblock (per-32), [expert,row] + __global uint * src1_qa, // q8_1 activations int8 (as uint, 4/elem) + __global half * src1_da, // q8_1 per-block scale [tok_slot * ne00/32] + __global half * src1_sa, // q8_1 per-block sum*d [tok_slot * ne00/32] + __global uint * src2, // post-router (orig out positions) + __global ushort * src2_emap, // tile -> expert id + __write_only image1d_buffer_t dst, + __global int * total_tiles, + uint ne00, + uint ne01, + int is_ragged, + int has_min // 0 for symmetric types (q8_0/q6_K/q4_0/...): skip min read +) { + const uint block_id_m = get_global_id(1); + const uint block_id_n = get_global_id(2); + if (block_id_n >= total_tiles[0]) return; + + const uint lid = get_local_id(0); // 0..63 -> output row within M-tile + const ushort expert_id = src2_emap[block_id_n]; + const uint row = block_id_m * TILESIZE_M; + const uint col = block_id_n * TILESIZE_N; + const uint row_idx = row + lid; + + // Scale/min are laid out FLAT per-32-block (2 per-16-segment scales + 1 min per + // 32-block), so K only needs to be a multiple of 32 — works for the 32-block + // types (q8_0/q5_0/q4_0/...) as well as the K-quants (K%256==0, same bytes). + const uint nblk32 = ne00 / 32; + const uint sc_per_row = nblk32 * 2; + const uint mn_per_row = nblk32; + const uint ne00_u = ne00 >> 2; + const uint ne00_b = ne00 >> 5; + + __local uint sh_qa[TILESIZE_N][8]; + __local half sh_d[TILESIZE_N]; + __local half sh_s[TILESIZE_N]; + + __local uint sh_src2[TILESIZE_N]; + __local int sh_nreal; + if (lid < TILESIZE_N) sh_src2[lid] = src2[col + lid]; + barrier(CLK_LOCAL_MEM_FENCE); + if (lid == 0) { + int nr = TILESIZE_N; + if (is_ragged) { nr = 0; + #pragma unroll + for (int t = 0; t < TILESIZE_N; ++t) if (sh_src2[t] != 0xFFFFFFFFu) ++nr; } + sh_nreal = nr; + } + barrier(CLK_LOCAL_MEM_FENCE); + const int n_real = sh_nreal; + + float acc[TILESIZE_N]; + #pragma unroll + for (int t = 0; t < TILESIZE_N; ++t) acc[t] = 0.0f; + + for (uint step = 0; step < ne00; step += 32) { + const uint sub = step >> 5; // 32-block index along K + + // uniform pre-decoded scale (2 per-16-seg) + min (1) for this row, this 32-block + __global half * scl = src0_scale + (expert_id*ne01 + row_idx)*sc_per_row + sub*2; + const float sc0 = (float)scl[0]; + const float sc1 = (float)scl[1]; + float mn = 0.0f; + if (has_min) mn = (float)src0_min[(expert_id*ne01 + row_idx)*mn_per_row + sub]; + + LOAD_QW(step, sub) + + const uint stage_lim = (uint)n_real * 8; + for (uint idx = lid; idx < stage_lim; idx += 64) { + const uint t = idx >> 3, u = idx & 7; + sh_qa[t][u] = src1_qa[(col + t) * ne00_u + (step >> 2) + u]; + } + if (lid < (uint)n_real) { + sh_d[lid] = src1_da[(col + lid) * ne00_b + sub]; + sh_s[lid] = src1_sa[(col + lid) * ne00_b + sub]; + } + barrier(CLK_LOCAL_MEM_FENCE); + + if (n_real == TILESIZE_N) { + #pragma unroll + for (int t = 0; t < TILESIZE_N; ++t) { MOE_DP4A_T(t); } + } else { + #pragma unroll 4 + for (int t = 0; t < n_real; ++t) { MOE_DP4A_T(t); } + } + barrier(CLK_LOCAL_MEM_FENCE); + } + + if (row_idx >= ne01) return; + + __local uint out_idx[TILESIZE_N]; + if (lid < TILESIZE_N) { + uint idx = sh_src2[lid]; + if (idx == 0xFFFFFFFF) idx = sh_src2[0]; + out_idx[lid] = idx * ne01; + } + barrier(CLK_LOCAL_MEM_FENCE); + + const uint m_offset = row + lid; + if (n_real == TILESIZE_N) { + #pragma unroll + for (int t = 1; t < TILESIZE_N; ++t) write_imagef(dst, out_idx[t] + m_offset, acc[t]); + barrier(CLK_GLOBAL_MEM_FENCE); + write_imagef(dst, out_idx[0] + m_offset, acc[0]); + } else { + for (int t = 0; t < n_real; ++t) write_imagef(dst, out_idx[t] + m_offset, acc[t]); + } +} diff --git a/ggml/src/ggml-opencl/kernels/gemm_noshuffle_iq4_nl_q8_1_dp4a.cl b/ggml/src/ggml-opencl/kernels/gemm_noshuffle_iq4_nl_q8_1_dp4a.cl new file mode 100644 index 000000000..2941289dd --- /dev/null +++ b/ggml/src/ggml-opencl/kernels/gemm_noshuffle_iq4_nl_q8_1_dp4a.cl @@ -0,0 +1,143 @@ +#pragma OPENCL EXTENSION cl_khr_fp16 : enable +#pragma OPENCL EXTENSION cl_khr_subgroups : enable +#ifdef cl_khr_integer_dot_product +#pragma OPENCL EXTENSION cl_khr_integer_dot_product : enable +#endif + +// Weight layout, feature-major: +// src0_q[row + (k/4)*m] ushort = 4 nibbles (K = 4*grp .. +3) +// src0_d[row + (k/32)*m] half = per-32-block scale + +#define TILESIZE_N 32 + +// IQ4_NL non-linear codebook as signed int8, packed 4 codes per uint. +// divergent nibble lookups read a small __constant uint array + shift, +// never a byte array because byte-indexed __constant loads serialize on Adreno and tank perf +// idx 0-3: -127,-104,-83,-65 = 0x81,0x98,0xAD,0xBF +// idx 4-7: -49,-35,-22,-10 = 0xCF,0xDD,0xEA,0xF6 +// idx 8-11: 1, 13, 25, 38 = 0x01,0x0D,0x19,0x26 +// idx 12-15: 53, 69, 89,113 = 0x35,0x45,0x59,0x71 +__constant uint kvalues_iq4nl_i8x4[4] = { + 0xBFAD9881u, 0xF6EADDCFu, 0x26190D01u, 0x71594535u +}; + +// nibble (0..15) -> its codebook byte in the low 8 bits. +inline uint iq4nl_code(uint n) { + return (kvalues_iq4nl_i8x4[n >> 2] >> ((n & 3u) * 8u)) & 0xFFu; +} + +// 4 nibbles in low 16 bits of u -> 4 codebook int8, packed for dp4a. +inline uint iq4nl_pack(ushort u) { + return iq4nl_code((uint)( u & 0xF)) + | (iq4nl_code((uint)((u >> 4) & 0xF)) << 8) + | (iq4nl_code((uint)((u >> 8) & 0xF)) << 16) + | (iq4nl_code((uint)((u >> 12) & 0xF)) << 24); +} + +inline int dot8_q8a(uint8 qw, __local const uint * a) { + int r = 0; + r = dot_acc_sat_4x8packed_ss_int(qw.s0, a[0], r); + r = dot_acc_sat_4x8packed_ss_int(qw.s1, a[1], r); + r = dot_acc_sat_4x8packed_ss_int(qw.s2, a[2], r); + r = dot_acc_sat_4x8packed_ss_int(qw.s3, a[3], r); + r = dot_acc_sat_4x8packed_ss_int(qw.s4, a[4], r); + r = dot_acc_sat_4x8packed_ss_int(qw.s5, a[5], r); + r = dot_acc_sat_4x8packed_ss_int(qw.s6, a[6], r); + r = dot_acc_sat_4x8packed_ss_int(qw.s7, a[7], r); + return r; +} + +__attribute__((qcom_wave_pair_mode(1))) +kernel void kernel_gemm_noshuffle_iq4_nl_q8_1_dp4a( + __global const ushort * src0_q, // IQ4_NL nibbles (4/ushort, feature-major) + __global const half * src0_d, // per-32-block scale, feature-major + __global const uint * src1_qa, // q8_1 activations int8 (as uint, 4/elem) [N, K] + __global const half * src1_da, // q8_1 per-block scale [N, K/32] + __global float * dst, + ulong offsetd, + int m, // output features (rows) + int n_no_padding, // tokens (cols) + int k // K (== ne00) +) { + dst = (global float *)((global char *)dst + offsetd); + + const uint lid = get_local_id(0); // 0..63 -> row within the M-tile + const uint block_id_m = get_global_id(1); + const uint block_id_n = get_global_id(2); + + const uint row = block_id_m * 64 + lid; + const uint col_base = block_id_n * TILESIZE_N; + const bool row_valid = row < (uint)m; + const uint rrow = row_valid ? row : 0; // clamp OOB rows; their writes are masked + + const uint k_u = (uint)k >> 2; // K in uint (int8x4) units + const uint k_b = (uint)k >> 5; // blocks-of-32 along K + + __local uint sh_qa[TILESIZE_N][8]; + __local half sh_d[TILESIZE_N]; + +#define NGROUPS (TILESIZE_N / 4) + float4 acc[NGROUPS]; + #pragma unroll + for (int g = 0; g < NGROUPS; ++g) acc[g] = (float4)(0.0f); + + for (uint step = 0; step < (uint)k; step += 32) { + const uint sub = step >> 5; + + const float d_w = (float)src0_d[rrow + sub * (uint)m]; + + // 8 weight uints (32 codebook int8) for this row, this 32-block. + const uint qsbase = rrow + (step >> 2) * (uint)m; + uint8 qw; + qw.s0 = iq4nl_pack(src0_q[qsbase + 0 * m]); + qw.s1 = iq4nl_pack(src0_q[qsbase + 1 * m]); + qw.s2 = iq4nl_pack(src0_q[qsbase + 2 * m]); + qw.s3 = iq4nl_pack(src0_q[qsbase + 3 * m]); + qw.s4 = iq4nl_pack(src0_q[qsbase + 4 * m]); + qw.s5 = iq4nl_pack(src0_q[qsbase + 5 * m]); + qw.s6 = iq4nl_pack(src0_q[qsbase + 6 * m]); + qw.s7 = iq4nl_pack(src0_q[qsbase + 7 * m]); + + // cooperatively stage the 32-token x 32-K int8 activations to lm + for (uint idx = lid; idx < TILESIZE_N * 8; idx += 64) { + const uint t = idx >> 3; + const uint u = idx & 7; + const uint c = col_base + t; + sh_qa[t][u] = (c < (uint)n_no_padding) ? src1_qa[c * k_u + (step >> 2) + u] : 0u; + } + if (lid < TILESIZE_N) { + const uint c = col_base + lid; + sh_d[lid] = (c < (uint)n_no_padding) ? src1_da[c * k_b + sub] : (half)0; + } + barrier(CLK_LOCAL_MEM_FENCE); + +#define LD4(arr, b) ((float4)((float)arr[(b)+0], (float)arr[(b)+1], (float)arr[(b)+2], (float)arr[(b)+3])) + #pragma unroll + for (int g = 0; g < NGROUPS; ++g) { + const int b = g * 4; + float4 rf; + rf.s0 = (float)dot8_q8a(qw, sh_qa[b+0]); rf.s1 = (float)dot8_q8a(qw, sh_qa[b+1]); + rf.s2 = (float)dot8_q8a(qw, sh_qa[b+2]); rf.s3 = (float)dot8_q8a(qw, sh_qa[b+3]); + acc[g] += d_w * LD4(sh_d, b) * rf; + } +#undef LD4 + barrier(CLK_LOCAL_MEM_FENCE); + } + + if (!row_valid) { + return; + } + + // dst is [token, feature] row-major (stride m): dst[col*m + row]. + #pragma unroll + for (int g = 0; g < NGROUPS; ++g) { + const uint b = (uint)(g * 4); + const float4 a = acc[g]; + const uint c0 = col_base + b; + if (c0 + 0 < (uint)n_no_padding) dst[(c0 + 0) * (uint)m + row] = a.s0; + if (c0 + 1 < (uint)n_no_padding) dst[(c0 + 1) * (uint)m + row] = a.s1; + if (c0 + 2 < (uint)n_no_padding) dst[(c0 + 2) * (uint)m + row] = a.s2; + if (c0 + 3 < (uint)n_no_padding) dst[(c0 + 3) * (uint)m + row] = a.s3; + } +#undef NGROUPS +} diff --git a/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q4_0_q8_1_dp4a.cl b/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q4_0_q8_1_dp4a.cl new file mode 100644 index 000000000..446a8eb18 --- /dev/null +++ b/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q4_0_q8_1_dp4a.cl @@ -0,0 +1,127 @@ +#pragma OPENCL EXTENSION cl_khr_fp16 : enable +#pragma OPENCL EXTENSION cl_khr_subgroups : enable +#ifdef cl_khr_integer_dot_product +#pragma OPENCL EXTENSION cl_khr_integer_dot_product : enable +#endif + +#define TILESIZE_N 32 + +// Expand the 4 nibbles in the low 16 bits of u into 4 bytes (value 0..15), +// packed for the int8 dp4a. The -8 zero-point is applied via the sum term. +#define EXP4(u) ( ((uint)((u) & 0x000Fu)) | \ + (((uint)((u) & 0x00F0u)) << 4) | \ + (((uint)((u) & 0x0F00u)) << 8) | \ + (((uint)((u) & 0xF000u)) << 12) ) + +inline int dot8_q8a(uint8 qw, __local const uint * a) { + int r = 0; + r = dot_acc_sat_4x8packed_ss_int(qw.s0, a[0], r); + r = dot_acc_sat_4x8packed_ss_int(qw.s1, a[1], r); + r = dot_acc_sat_4x8packed_ss_int(qw.s2, a[2], r); + r = dot_acc_sat_4x8packed_ss_int(qw.s3, a[3], r); + r = dot_acc_sat_4x8packed_ss_int(qw.s4, a[4], r); + r = dot_acc_sat_4x8packed_ss_int(qw.s5, a[5], r); + r = dot_acc_sat_4x8packed_ss_int(qw.s6, a[6], r); + r = dot_acc_sat_4x8packed_ss_int(qw.s7, a[7], r); + return r; +} + +__attribute__((qcom_wave_pair_mode(1))) +kernel void kernel_gemm_noshuffle_q4_0_q8_1_dp4a( + __global const ushort * src0_q, // q4_0 nibbles (4/ushort, feature-major) + __global const half * src0_d, // per-32-block scale, feature-major + __global const uint * src1_qa, // q8_1 activations int8 (as uint, 4/elem) [N, K] + __global const half * src1_da, // q8_1 per-block scale [N, K/32] + __global const half * src1_sa, // q8_1 per-block sum*d [N, K/32] + __global float * dst, + ulong offsetd, + int m, // output features (rows) + int n_no_padding, // tokens (cols) + int k // K (== ne00) +) { + dst = (global float *)((global char *)dst + offsetd); + + const uint lid = get_local_id(0); // 0..63 -> row within the M-tile + const uint block_id_m = get_global_id(1); + const uint block_id_n = get_global_id(2); + + const uint row = block_id_m * 64 + lid; + const uint col_base = block_id_n * TILESIZE_N; + const bool row_valid = row < (uint)m; + const uint rrow = row_valid ? row : 0; // clamp OOB rows; their writes are masked + + const uint k_u = (uint)k >> 2; // K in uint (int8x4) units + const uint k_b = (uint)k >> 5; // blocks-of-32 along K + + __local uint sh_qa[TILESIZE_N][8]; + __local half sh_d[TILESIZE_N]; + __local half sh_s[TILESIZE_N]; + +#define NGROUPS (TILESIZE_N / 4) + float4 acc[NGROUPS]; + #pragma unroll + for (int g = 0; g < NGROUPS; ++g) acc[g] = (float4)(0.0f); + + for (uint step = 0; step < (uint)k; step += 32) { + const uint sub = step >> 5; + + const float d_w = (float)src0_d[rrow + sub * (uint)m]; + + // 8 weight uints (32 nibbles) for this row, this 32-block. Feature-major: + // src0_q[row + (k/4 + u)*m], k/4 = step/4 (= step>>2). EXP4 -> dp4a int8. + const uint qsbase = rrow + (step >> 2) * (uint)m; + uint8 qw; + qw.s0 = EXP4(src0_q[qsbase + 0 * m]); + qw.s1 = EXP4(src0_q[qsbase + 1 * m]); + qw.s2 = EXP4(src0_q[qsbase + 2 * m]); + qw.s3 = EXP4(src0_q[qsbase + 3 * m]); + qw.s4 = EXP4(src0_q[qsbase + 4 * m]); + qw.s5 = EXP4(src0_q[qsbase + 5 * m]); + qw.s6 = EXP4(src0_q[qsbase + 6 * m]); + qw.s7 = EXP4(src0_q[qsbase + 7 * m]); + + // cooperatively stage the 32-token x 32-K int8 activations to LDS + for (uint idx = lid; idx < TILESIZE_N * 8; idx += 64) { + const uint t = idx >> 3; + const uint u = idx & 7; + const uint c = col_base + t; + sh_qa[t][u] = (c < (uint)n_no_padding) ? src1_qa[c * k_u + (step >> 2) + u] : 0u; + } + if (lid < TILESIZE_N) { + const uint c = col_base + lid; + sh_d[lid] = (c < (uint)n_no_padding) ? src1_da[c * k_b + sub] : (half)0; + sh_s[lid] = (c < (uint)n_no_padding) ? src1_sa[c * k_b + sub] : (half)0; + } + barrier(CLK_LOCAL_MEM_FENCE); + +#define LD4(arr, b) ((float4)((float)arr[(b)+0], (float)arr[(b)+1], (float)arr[(b)+2], (float)arr[(b)+3])) + #pragma unroll + for (int g = 0; g < NGROUPS; ++g) { + const int b = g * 4; + float4 rf; + rf.s0 = (float)dot8_q8a(qw, sh_qa[b+0]); rf.s1 = (float)dot8_q8a(qw, sh_qa[b+1]); + rf.s2 = (float)dot8_q8a(qw, sh_qa[b+2]); rf.s3 = (float)dot8_q8a(qw, sh_qa[b+3]); + // q4_0: w = d*(q-8) -> d_w * (a_d * dp4a(q,qa) - 8 * a_s) + acc[g] += d_w * (LD4(sh_d, b) * rf - 8.0f * LD4(sh_s, b)); + } +#undef LD4 + barrier(CLK_LOCAL_MEM_FENCE); + } + + if (!row_valid) { + return; + } + + // dst is [token, feature] row-major (stride m): dst[col*m + row]. + #pragma unroll + for (int g = 0; g < NGROUPS; ++g) { + const uint b = (uint)(g * 4); + const float4 a = acc[g]; + const uint c0 = col_base + b; + if (c0 + 0 < (uint)n_no_padding) dst[(c0 + 0) * (uint)m + row] = a.s0; + if (c0 + 1 < (uint)n_no_padding) dst[(c0 + 1) * (uint)m + row] = a.s1; + if (c0 + 2 < (uint)n_no_padding) dst[(c0 + 2) * (uint)m + row] = a.s2; + if (c0 + 3 < (uint)n_no_padding) dst[(c0 + 3) * (uint)m + row] = a.s3; + } +#undef NGROUPS +} diff --git a/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q4_k_q8_1_dp4a.cl b/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q4_k_q8_1_dp4a.cl new file mode 100644 index 000000000..865618014 --- /dev/null +++ b/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q4_k_q8_1_dp4a.cl @@ -0,0 +1,281 @@ +#pragma OPENCL EXTENSION cl_khr_fp16 : enable +#pragma OPENCL EXTENSION cl_khr_subgroups : enable +#ifdef cl_khr_integer_dot_product +#pragma OPENCL EXTENSION cl_khr_integer_dot_product : enable +#endif + +#ifndef TILESIZE_N +#define TILESIZE_N 32 +#endif +#define QK_K 256 +#define K_SCALE_SIZE 12 + +inline void get_scale_min_k4( + int j, + global const uchar * q, + uchar * d, + uchar * m, + uchar mask_d6, + uchar mask_d4, + uchar mask_hi2 +) { + if (j < 4) { + *d = q[j] & mask_d6; + *m = q[j+4] & mask_d6; + } else { + *d = (q[j+4] & mask_d4) | ((q[j-4] & mask_hi2) >> 2); + *m = ((q[j+4] >> 4) & mask_d4) | ((q[j] & mask_hi2) >> 2); + } +} + +// Expand the 4 nibbles in the low 16 bits of `u` into 4 bytes (one nibble per +// byte, value 0..15), packed for the int8 dp4a. +#define EXP4(u) ( ((uint)((u) & 0x000Fu)) | \ + (((uint)((u) & 0x00F0u)) << 4) | \ + (((uint)((u) & 0x0F00u)) << 8) | \ + (((uint)((u) & 0xF000u)) << 12) ) + +// 32-K dp4a dot of one token's int8 activations (8 packed uints in lm) against the +// row's 8 packed weight uints. qw passed by value as a uint8 (register), not an array. +inline int dot8_q8a(uint8 qw, __local const uint * a) { + int r = 0; + r = dot_acc_sat_4x8packed_ss_int(qw.s0, a[0], r); + r = dot_acc_sat_4x8packed_ss_int(qw.s1, a[1], r); + r = dot_acc_sat_4x8packed_ss_int(qw.s2, a[2], r); + r = dot_acc_sat_4x8packed_ss_int(qw.s3, a[3], r); + r = dot_acc_sat_4x8packed_ss_int(qw.s4, a[4], r); + r = dot_acc_sat_4x8packed_ss_int(qw.s5, a[5], r); + r = dot_acc_sat_4x8packed_ss_int(qw.s6, a[6], r); + r = dot_acc_sat_4x8packed_ss_int(qw.s7, a[7], r); + return r; +} + +__attribute__((qcom_wave_pair_mode(1))) +kernel void kernel_gemm_noshuffle_q4_k_q8_1_dp4a( + __global const ushort * src0_q, // q4_K weights (noshuffle, packed nibbles) + __global const uchar * src0_s, // 6-bit scale/min codes + __global const half * src0_d, // per-superblock scale + __global const half * src0_dm, // per-superblock min + __global const uint * src1_qa, // q8_1 activations int8 (as uint, 4/elem) [N, K] + __global const half * src1_da, // q8_1 per-block scale [N, K/32] + __global const half * src1_sa, // q8_1 per-block sum*d [N, K/32] + __global float * dst, + ulong offsetd, + int m, // output features (rows) + int n_no_padding, // tokens (cols) + int k, // K (== ne00) + uchar mask_d6, + uchar mask_d4, + uchar mask_hi2 +) { + dst = (global float *)((global char *)dst + offsetd); + + const uint lid = get_local_id(0); // 0..63 -> row within the M-tile + const uint block_id_m = get_global_id(1); + const uint block_id_n = get_global_id(2); + + const uint row = block_id_m * 64 + lid; + const uint col_base = block_id_n * TILESIZE_N; + const bool row_valid = row < (uint)m; + const uint rrow = row_valid ? row : 0; // clamp OOB rows; their writes are masked + + const uint num_superblocks = (uint)k / QK_K; + const uint k_u = (uint)k >> 2; // K in uint (int8x4) units + const uint k_b = (uint)k >> 5; // blocks-of-32 along K + + __local uint sh_qa[TILESIZE_N][8]; + __local half sh_d[TILESIZE_N]; + __local half sh_s[TILESIZE_N]; + + // One float4 vector-register accumulator per group of 4 tokens (NGROUPS = TILESIZE_N/4). +#define NGROUPS (TILESIZE_N / 4) + float4 acc[NGROUPS]; + #pragma unroll + for (int g = 0; g < NGROUPS; ++g) { acc[g] = (float4)(0.0f); } + + for (uint step = 0; step < (uint)k; step += 32) { + const uint sub = step >> 5; + const uint sb_idx = step / QK_K; + const uint sub_idx = sub & 7; + + // weight scale/min for this WI's row, this subblock + const float dd = (float)src0_d [rrow + sb_idx * m]; + const float dmm = (float)src0_dm[rrow + sb_idx * m]; + global const uchar * sc = src0_s + rrow * num_superblocks * K_SCALE_SIZE + sb_idx * K_SCALE_SIZE; + uchar sv, mn; + get_scale_min_k4(sub_idx, sc, &sv, &mn, mask_d6, mask_d4, mask_hi2); + const float scale = dd * (float)sv; + const float minv = dmm * (float)mn; + + // repack this row's 32 weight nibbles into 8 dp4a uints. The packed q4_K + // layout stores one ushort = 4 consecutive-K nibbles for a row at + // src0_q[row + (K_group)*m], K_group = step/4 + u. + const uint wbase = rrow + (step >> 2) * (uint)m; + uint8 qw; + qw.s0 = EXP4(src0_q[wbase + 0 * m]); + qw.s1 = EXP4(src0_q[wbase + 1 * m]); + qw.s2 = EXP4(src0_q[wbase + 2 * m]); + qw.s3 = EXP4(src0_q[wbase + 3 * m]); + qw.s4 = EXP4(src0_q[wbase + 4 * m]); + qw.s5 = EXP4(src0_q[wbase + 5 * m]); + qw.s6 = EXP4(src0_q[wbase + 6 * m]); + qw.s7 = EXP4(src0_q[wbase + 7 * m]); + + // cooperatively stage the 32-token x 32-K int8 activations to lm + for (uint idx = lid; idx < TILESIZE_N * 8; idx += 64) { + const uint t = idx >> 3; + const uint u = idx & 7; + const uint c = col_base + t; + sh_qa[t][u] = (c < (uint)n_no_padding) ? src1_qa[c * k_u + (step >> 2) + u] : 0u; + } + if (lid < TILESIZE_N) { + const uint c = col_base + lid; + sh_d[lid] = (c < (uint)n_no_padding) ? src1_da[c * k_b + sub] : (half)0; + sh_s[lid] = (c < (uint)n_no_padding) ? src1_sa[c * k_b + sub] : (half)0; + } + barrier(CLK_LOCAL_MEM_FENCE); + +#define LD4(arr, b) ((float4)((float)arr[(b)+0], (float)arr[(b)+1], (float)arr[(b)+2], (float)arr[(b)+3])) + #pragma unroll + for (int g = 0; g < NGROUPS; ++g) { + const int b = g * 4; + float4 rf; + rf.s0 = (float)dot8_q8a(qw, sh_qa[b+0]); rf.s1 = (float)dot8_q8a(qw, sh_qa[b+1]); + rf.s2 = (float)dot8_q8a(qw, sh_qa[b+2]); rf.s3 = (float)dot8_q8a(qw, sh_qa[b+3]); + acc[g] += scale * LD4(sh_d, b) * rf - minv * LD4(sh_s, b); + } +#undef LD4 + barrier(CLK_LOCAL_MEM_FENCE); + } + + if (!row_valid) { + return; + } + + // dst is [token, feature] row-major (stride m): dst[col*m + row]. Scatter each + // lane with a per-token padding guard (dst is non-contiguous in token). + #pragma unroll + for (int g = 0; g < NGROUPS; ++g) { + const uint b = (uint)(g * 4); + const float4 a = acc[g]; + const uint c0 = col_base + b; + if (c0 + 0 < (uint)n_no_padding) dst[(c0 + 0) * (uint)m + row] = a.s0; + if (c0 + 1 < (uint)n_no_padding) dst[(c0 + 1) * (uint)m + row] = a.s1; + if (c0 + 2 < (uint)n_no_padding) dst[(c0 + 2) * (uint)m + row] = a.s2; + if (c0 + 3 < (uint)n_no_padding) dst[(c0 + 3) * (uint)m + row] = a.s3; + } +#undef NGROUPS +} + +__attribute__((qcom_wave_pair_mode(1))) +kernel void kernel_gemm_noshuffle_q4_k_q8_1_dp4a_wimg( + __read_only image1d_buffer_t src0_q_img, // q4_K weights as uint32 texels (2 ushorts/texel) + __global const uchar * src0_s, // 6-bit scale/min codes + __global const half * src0_d, // per-superblock scale + __global const half * src0_dm, // per-superblock min + __global const uint * src1_qa, // q8_1 activations int8 (as uint, 4/elem) [N, K] + __global const half * src1_da, // q8_1 per-block scale [N, K/32] + __global const half * src1_sa, // q8_1 per-block sum*d [N, K/32] + __global float * dst, + ulong offsetd, + int m, // output features (rows) + int n_no_padding, // tokens (cols) + int k, // K (== ne00) + uchar mask_d6, + uchar mask_d4, + uchar mask_hi2 +) { + dst = (global float *)((global char *)dst + offsetd); + + const uint lid = get_local_id(0); // 0..63 -> row within the M-tile + const uint block_id_m = get_global_id(1); + const uint block_id_n = get_global_id(2); + + const uint row = block_id_m * 64 + lid; + const uint col_base = block_id_n * TILESIZE_N; + const bool row_valid = row < (uint)m; + const uint rrow = row_valid ? row : 0; // clamp OOB rows; their writes are masked + + // Constant per WI: the ushort the row needs always sits in the same half of + // its uint32 texel (m even => index parity == rrow parity). Hoist the shift. + const uint sel = (rrow & 1u) * 16u; + + const uint k_u = (uint)k >> 2; // K in uint (int8x4) units + const uint k_b = (uint)k >> 5; // blocks-of-32 along K + const uint num_superblocks = (uint)k / QK_K; + + __local uint sh_qa[TILESIZE_N][8]; + __local half sh_d[TILESIZE_N]; + __local half sh_s[TILESIZE_N]; + +#define NGROUPS (TILESIZE_N / 4) + float4 acc[NGROUPS]; + #pragma unroll + for (int g = 0; g < NGROUPS; ++g) acc[g] = (float4)(0.0f); + + for (uint step = 0; step < (uint)k; step += 32) { + const uint sub = step >> 5; + const uint sb_idx = step / QK_K; + const uint sub_idx = sub & 7; + + const float dd = (float)src0_d [rrow + sb_idx * m]; + const float dmm = (float)src0_dm[rrow + sb_idx * m]; + global const uchar * sc = src0_s + rrow * num_superblocks * K_SCALE_SIZE + sb_idx * K_SCALE_SIZE; + uchar sv, mn; + get_scale_min_k4(sub_idx, sc, &sv, &mn, mask_d6, mask_d4, mask_hi2); + const float scale = dd * (float)sv; + const float minv = dmm * (float)mn; + + const uint wbase = rrow + (step >> 2) * (uint)m; + uint8 qw; + qw.s0 = EXP4(read_imageui(src0_q_img, (int)((wbase + 0 * m) >> 1)).x >> sel); + qw.s1 = EXP4(read_imageui(src0_q_img, (int)((wbase + 1 * m) >> 1)).x >> sel); + qw.s2 = EXP4(read_imageui(src0_q_img, (int)((wbase + 2 * m) >> 1)).x >> sel); + qw.s3 = EXP4(read_imageui(src0_q_img, (int)((wbase + 3 * m) >> 1)).x >> sel); + qw.s4 = EXP4(read_imageui(src0_q_img, (int)((wbase + 4 * m) >> 1)).x >> sel); + qw.s5 = EXP4(read_imageui(src0_q_img, (int)((wbase + 5 * m) >> 1)).x >> sel); + qw.s6 = EXP4(read_imageui(src0_q_img, (int)((wbase + 6 * m) >> 1)).x >> sel); + qw.s7 = EXP4(read_imageui(src0_q_img, (int)((wbase + 7 * m) >> 1)).x >> sel); + + for (uint idx = lid; idx < TILESIZE_N * 8; idx += 64) { + const uint t = idx >> 3; + const uint u = idx & 7; + const uint c = col_base + t; + sh_qa[t][u] = (c < (uint)n_no_padding) ? src1_qa[c * k_u + (step >> 2) + u] : 0u; + } + if (lid < TILESIZE_N) { + const uint c = col_base + lid; + sh_d[lid] = (c < (uint)n_no_padding) ? src1_da[c * k_b + sub] : (half)0; + sh_s[lid] = (c < (uint)n_no_padding) ? src1_sa[c * k_b + sub] : (half)0; + } + barrier(CLK_LOCAL_MEM_FENCE); + +#define LD4(arr, b) ((float4)((float)arr[(b)+0], (float)arr[(b)+1], (float)arr[(b)+2], (float)arr[(b)+3])) + #pragma unroll + for (int g = 0; g < NGROUPS; ++g) { + const int b = g * 4; + float4 rf; + rf.s0 = (float)dot8_q8a(qw, sh_qa[b+0]); rf.s1 = (float)dot8_q8a(qw, sh_qa[b+1]); + rf.s2 = (float)dot8_q8a(qw, sh_qa[b+2]); rf.s3 = (float)dot8_q8a(qw, sh_qa[b+3]); + acc[g] += scale * LD4(sh_d, b) * rf - minv * LD4(sh_s, b); + } +#undef LD4 + barrier(CLK_LOCAL_MEM_FENCE); + } + + if (!row_valid) { + return; + } + + #pragma unroll + for (int g = 0; g < NGROUPS; ++g) { + const uint b = (uint)(g * 4); + const float4 a = acc[g]; + const uint c0 = col_base + b; + if (c0 + 0 < (uint)n_no_padding) dst[(c0 + 0) * (uint)m + row] = a.s0; + if (c0 + 1 < (uint)n_no_padding) dst[(c0 + 1) * (uint)m + row] = a.s1; + if (c0 + 2 < (uint)n_no_padding) dst[(c0 + 2) * (uint)m + row] = a.s2; + if (c0 + 3 < (uint)n_no_padding) dst[(c0 + 3) * (uint)m + row] = a.s3; + } +#undef NGROUPS +} diff --git a/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q5_0_q8_1_dp4a.cl b/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q5_0_q8_1_dp4a.cl new file mode 100644 index 000000000..4d1c6bdbc --- /dev/null +++ b/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q5_0_q8_1_dp4a.cl @@ -0,0 +1,235 @@ +#pragma OPENCL EXTENSION cl_khr_fp16 : enable +#pragma OPENCL EXTENSION cl_khr_subgroups : enable +#ifdef cl_khr_integer_dot_product +#pragma OPENCL EXTENSION cl_khr_integer_dot_product : enable +#endif + +// Weight layout +// src0_qs[row + (k/4)*m] ushort = 4 low nibbles (K = 4*grp .. +3) +// src0_qh[row + (k/8)*m] uchar = 8 high bits (one per element) +// src0_d [row + (k/32)*m] half = per-32-block scale + +#define TILESIZE_N 32 + +// 4 nibbles in low 16 bits of u -> 4 bytes (value 0..15) +#define EXP4(u) ( ((uint)((u) & 0x000Fu)) | \ + (((uint)((u) & 0x00F0u)) << 4) | \ + (((uint)((u) & 0x0F00u)) << 8) | \ + (((uint)((u) & 0xF000u)) << 12) ) +// 4 high bits (one per element, in bits 0..3 of h) -> bit4 of each of 4 bytes +#define EXP1(h) ( (((uint)((h) & 0x1u)) << 4) | \ + (((uint)((h) & 0x2u)) << 11) | \ + (((uint)((h) & 0x4u)) << 18) | \ + (((uint)((h) & 0x8u)) << 25) ) + +inline int dot8_q8a(uint8 qw, __local const uint * a) { + int r = 0; + r = dot_acc_sat_4x8packed_ss_int(qw.s0, a[0], r); + r = dot_acc_sat_4x8packed_ss_int(qw.s1, a[1], r); + r = dot_acc_sat_4x8packed_ss_int(qw.s2, a[2], r); + r = dot_acc_sat_4x8packed_ss_int(qw.s3, a[3], r); + r = dot_acc_sat_4x8packed_ss_int(qw.s4, a[4], r); + r = dot_acc_sat_4x8packed_ss_int(qw.s5, a[5], r); + r = dot_acc_sat_4x8packed_ss_int(qw.s6, a[6], r); + r = dot_acc_sat_4x8packed_ss_int(qw.s7, a[7], r); + return r; +} + +__attribute__((qcom_wave_pair_mode(1))) +kernel void kernel_gemm_noshuffle_q5_0_q8_1_dp4a( + __global const ushort * src0_qs, // q5_0 low nibbles (4/ushort, feature-major) + __global const uchar * src0_qh, // q5_0 high-bit plane (8/uchar, feature-major) + __global const half * src0_d, // per-32-block scale, feature-major + __global const uint * src1_qa, // q8_1 activations int8 (as uint, 4/elem) [N, K] + __global const half * src1_da, // q8_1 per-block scale [N, K/32] + __global const half * src1_sa, // q8_1 per-block sum*d [N, K/32] + __global float * dst, + ulong offsetd, + int m, // output features (rows) + int n_no_padding, // tokens (cols) + int k // K (== ne00) +) { + dst = (global float *)((global char *)dst + offsetd); + + const uint lid = get_local_id(0); // 0..63 -> row within the M-tile + const uint block_id_m = get_global_id(1); + const uint block_id_n = get_global_id(2); + + const uint row = block_id_m * 64 + lid; + const uint col_base = block_id_n * TILESIZE_N; + const bool row_valid = row < (uint)m; + const uint rrow = row_valid ? row : 0; // clamp OOB rows; their writes are masked + + const uint k_u = (uint)k >> 2; // K in uint (int8x4) units + const uint k_b = (uint)k >> 5; // blocks-of-32 along K + + __local uint sh_qa[TILESIZE_N][8]; + __local half sh_d[TILESIZE_N]; + __local half sh_s[TILESIZE_N]; + +#define NGROUPS (TILESIZE_N / 4) + float4 acc[NGROUPS]; + #pragma unroll + for (int g = 0; g < NGROUPS; ++g) acc[g] = (float4)(0.0f); + + for (uint step = 0; step < (uint)k; step += 32) { + const uint sub = step >> 5; + + const float d_w = (float)src0_d[rrow + sub * (uint)m]; + const float minv = d_w * 16.0f; // -16 centering -> subtract via q8_1 sum + + // 8 weight uints (32 elements) for this row, this 32-block. + // nibbles: src0_qs[row + (step/4 + u)*m]; high bits: src0_qh[row + (step/8 + u/2)*m], + // 4-bit group selected by (u&1)*4. + const uint qsbase = rrow + (step >> 2) * (uint)m; + const uint qhbase = rrow + (step >> 3) * (uint)m; + uint8 qw; + #define QW(u) (EXP4(src0_qs[qsbase + (u) * m]) | \ + EXP1((uint)(src0_qh[qhbase + ((u) >> 1) * m] >> (((u) & 1u) * 4u)) & 0xFu)) + qw.s0 = QW(0); qw.s1 = QW(1); qw.s2 = QW(2); qw.s3 = QW(3); + qw.s4 = QW(4); qw.s5 = QW(5); qw.s6 = QW(6); qw.s7 = QW(7); + #undef QW + + // cooperatively stage the 32-token x 32-K int8 activations to lm + for (uint idx = lid; idx < TILESIZE_N * 8; idx += 64) { + const uint t = idx >> 3; + const uint u = idx & 7; + const uint c = col_base + t; + sh_qa[t][u] = (c < (uint)n_no_padding) ? src1_qa[c * k_u + (step >> 2) + u] : 0u; + } + if (lid < TILESIZE_N) { + const uint c = col_base + lid; + sh_d[lid] = (c < (uint)n_no_padding) ? src1_da[c * k_b + sub] : (half)0; + sh_s[lid] = (c < (uint)n_no_padding) ? src1_sa[c * k_b + sub] : (half)0; + } + barrier(CLK_LOCAL_MEM_FENCE); + +#define LD4(arr, b) ((float4)((float)arr[(b)+0], (float)arr[(b)+1], (float)arr[(b)+2], (float)arr[(b)+3])) + #pragma unroll + for (int g = 0; g < NGROUPS; ++g) { + const int b = g * 4; + float4 rf; + rf.s0 = (float)dot8_q8a(qw, sh_qa[b+0]); rf.s1 = (float)dot8_q8a(qw, sh_qa[b+1]); + rf.s2 = (float)dot8_q8a(qw, sh_qa[b+2]); rf.s3 = (float)dot8_q8a(qw, sh_qa[b+3]); + acc[g] += d_w * LD4(sh_d, b) * rf - minv * LD4(sh_s, b); + } +#undef LD4 + barrier(CLK_LOCAL_MEM_FENCE); + } + + if (!row_valid) { + return; + } + + #pragma unroll + for (int g = 0; g < NGROUPS; ++g) { + const uint b = (uint)(g * 4); + const float4 a = acc[g]; + const uint c0 = col_base + b; + if (c0 + 0 < (uint)n_no_padding) dst[(c0 + 0) * (uint)m + row] = a.s0; + if (c0 + 1 < (uint)n_no_padding) dst[(c0 + 1) * (uint)m + row] = a.s1; + if (c0 + 2 < (uint)n_no_padding) dst[(c0 + 2) * (uint)m + row] = a.s2; + if (c0 + 3 < (uint)n_no_padding) dst[(c0 + 3) * (uint)m + row] = a.s3; + } +#undef NGROUPS +} + +__attribute__((qcom_wave_pair_mode(1))) +kernel void kernel_gemm_noshuffle_q5_0_q8_1_dp4a_wimg( + __read_only image1d_buffer_t src0_qs_img, // q5_0 low nibbles as uint32 texels (2 ushorts/texel) + __global const uchar * src0_qh, + __global const half * src0_d, + __global const uint * src1_qa, + __global const half * src1_da, + __global const half * src1_sa, + __global float * dst, + ulong offsetd, + int m, + int n_no_padding, + int k +) { + dst = (global float *)((global char *)dst + offsetd); + + const uint lid = get_local_id(0); + const uint block_id_m = get_global_id(1); + const uint block_id_n = get_global_id(2); + + const uint row = block_id_m * 64 + lid; + const uint col_base = block_id_n * TILESIZE_N; + const bool row_valid = row < (uint)m; + const uint rrow = row_valid ? row : 0; + + const uint sel = (rrow & 1u) * 16u; // constant per WI: qs ushort half in its uint32 texel + + const uint k_u = (uint)k >> 2; + const uint k_b = (uint)k >> 5; + + __local uint sh_qa[TILESIZE_N][8]; + __local half sh_d[TILESIZE_N]; + __local half sh_s[TILESIZE_N]; + +#define NGROUPS (TILESIZE_N / 4) + float4 acc[NGROUPS]; + #pragma unroll + for (int g = 0; g < NGROUPS; ++g) acc[g] = (float4)(0.0f); + + for (uint step = 0; step < (uint)k; step += 32) { + const uint sub = step >> 5; + + const float d_w = (float)src0_d[rrow + sub * (uint)m]; + const float minv = d_w * 16.0f; + + const uint qsbase = rrow + (step >> 2) * (uint)m; // ushort index + const uint qhbase = rrow + (step >> 3) * (uint)m; + uint8 qw; + // qs ushort via texture: uint32 texel = ushort_index>>1, half = sel. + #define QSU(u) ((read_imageui(src0_qs_img, (int)((qsbase + (u) * m) >> 1)).x >> sel) & 0xFFFFu) + #define QW(u) (EXP4(QSU(u)) | \ + EXP1((uint)(src0_qh[qhbase + ((u) >> 1) * m] >> (((u) & 1u) * 4u)) & 0xFu)) + qw.s0 = QW(0); qw.s1 = QW(1); qw.s2 = QW(2); qw.s3 = QW(3); + qw.s4 = QW(4); qw.s5 = QW(5); qw.s6 = QW(6); qw.s7 = QW(7); + #undef QW + #undef QSU + + for (uint idx = lid; idx < TILESIZE_N * 8; idx += 64) { + const uint t = idx >> 3; + const uint u = idx & 7; + const uint c = col_base + t; + sh_qa[t][u] = (c < (uint)n_no_padding) ? src1_qa[c * k_u + (step >> 2) + u] : 0u; + } + if (lid < TILESIZE_N) { + const uint c = col_base + lid; + sh_d[lid] = (c < (uint)n_no_padding) ? src1_da[c * k_b + sub] : (half)0; + sh_s[lid] = (c < (uint)n_no_padding) ? src1_sa[c * k_b + sub] : (half)0; + } + barrier(CLK_LOCAL_MEM_FENCE); + +#define LD4(arr, b) ((float4)((float)arr[(b)+0], (float)arr[(b)+1], (float)arr[(b)+2], (float)arr[(b)+3])) + #pragma unroll + for (int g = 0; g < NGROUPS; ++g) { + const int b = g * 4; + float4 rf; + rf.s0 = (float)dot8_q8a(qw, sh_qa[b+0]); rf.s1 = (float)dot8_q8a(qw, sh_qa[b+1]); + rf.s2 = (float)dot8_q8a(qw, sh_qa[b+2]); rf.s3 = (float)dot8_q8a(qw, sh_qa[b+3]); + acc[g] += d_w * LD4(sh_d, b) * rf - minv * LD4(sh_s, b); + } +#undef LD4 + barrier(CLK_LOCAL_MEM_FENCE); + } + + if (!row_valid) { + return; + } + + #pragma unroll + for (int g = 0; g < NGROUPS; ++g) { + const uint b = (uint)(g * 4); + const float4 a = acc[g]; + const uint c0 = col_base + b; + if (c0 + 0 < (uint)n_no_padding) dst[(c0 + 0) * (uint)m + row] = a.s0; + if (c0 + 1 < (uint)n_no_padding) dst[(c0 + 1) * (uint)m + row] = a.s1; + if (c0 + 2 < (uint)n_no_padding) dst[(c0 + 2) * (uint)m + row] = a.s2; + if (c0 + 3 < (uint)n_no_padding) dst[(c0 + 3) * (uint)m + row] = a.s3; + } +#undef NGROUPS +} diff --git a/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q5_k_q8_1_dp4a.cl b/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q5_k_q8_1_dp4a.cl new file mode 100644 index 000000000..aaeed68f6 --- /dev/null +++ b/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q5_k_q8_1_dp4a.cl @@ -0,0 +1,164 @@ +#pragma OPENCL EXTENSION cl_khr_fp16 : enable +#pragma OPENCL EXTENSION cl_khr_subgroups : enable +#ifdef cl_khr_integer_dot_product +#pragma OPENCL EXTENSION cl_khr_integer_dot_product : enable +#endif + +#define TILESIZE_N 32 +#define QK_K 256 +#define K_SCALE_SIZE 12 + +inline void get_scale_min_k4( + int j, + global const uchar * q, + uchar * d, + uchar * m, + uchar mask_d6, + uchar mask_d4, + uchar mask_hi2 +) { + if (j < 4) { + *d = q[j] & mask_d6; + *m = q[j+4] & mask_d6; + } else { + *d = (q[j+4] & mask_d4) | ((q[j-4] & mask_hi2) >> 2); + *m = ((q[j+4] >> 4) & mask_d4) | ((q[j] & mask_hi2) >> 2); + } +} + +// 4 nibbles in the low 16 bits of `u` -> 4 bytes (value 0..15, bits 0-3). +#define EXP4(u) ( ((uint)((u) & 0x000Fu)) | \ + (((uint)((u) & 0x00F0u)) << 4) | \ + (((uint)((u) & 0x0F00u)) << 8) | \ + (((uint)((u) & 0xF000u)) << 12) ) + +// 4 high bits (one per element, in bits 0-3 of h) -> bit 4 of each of 4 bytes, +// so OR with EXP4 forms the 5-bit q5_K code 0..31. +#define EXP1(h) ( (((uint)((h) & 0x1u)) << 4) | \ + (((uint)((h) & 0x2u)) << 11) | \ + (((uint)((h) & 0x4u)) << 18) | \ + (((uint)((h) & 0x8u)) << 25) ) + +inline int dot8_q8a(uint8 qw, __local const uint * a) { + int r = 0; + r = dot_acc_sat_4x8packed_ss_int(qw.s0, a[0], r); + r = dot_acc_sat_4x8packed_ss_int(qw.s1, a[1], r); + r = dot_acc_sat_4x8packed_ss_int(qw.s2, a[2], r); + r = dot_acc_sat_4x8packed_ss_int(qw.s3, a[3], r); + r = dot_acc_sat_4x8packed_ss_int(qw.s4, a[4], r); + r = dot_acc_sat_4x8packed_ss_int(qw.s5, a[5], r); + r = dot_acc_sat_4x8packed_ss_int(qw.s6, a[6], r); + r = dot_acc_sat_4x8packed_ss_int(qw.s7, a[7], r); + return r; +} + +__attribute__((qcom_wave_pair_mode(1))) +kernel void kernel_gemm_noshuffle_q5_k_q8_1_dp4a( + __global const ushort * src0_q, // q5_K low nibbles (transposed, ushort = 4 nibbles) + __global const uchar * src0_qh, // q5_K high bits (transposed, uchar = 8 elems/byte) + __global const uchar * src0_s, // 6-bit scale/min codes [row][superblock][12] + __global const half * src0_d, // per-superblock scale (transposed) + __global const half * src0_dm, // per-superblock min (transposed) + __global const uint * src1_qa, // q8_1 activations int8 (as uint, 4/elem) [N, K] + __global const half * src1_da, // q8_1 per-block scale [N, K/32] + __global const half * src1_sa, // q8_1 per-block sum*d [N, K/32] + __global float * dst, + ulong offsetd, + int m, // output features (rows) + int n_no_padding, // tokens (cols) + int k, // K (== ne00) + uchar mask_d6, + uchar mask_d4, + uchar mask_hi2 +) { + dst = (global float *)((global char *)dst + offsetd); + + const uint lid = get_local_id(0); // 0..63 -> row within the M-tile + const uint block_id_m = get_global_id(1); + const uint block_id_n = get_global_id(2); + + const uint row = block_id_m * 64 + lid; + const uint col_base = block_id_n * TILESIZE_N; + const bool row_valid = row < (uint)m; + const uint rrow = row_valid ? row : 0; + + const uint num_superblocks = (uint)k / QK_K; + const uint k_u = (uint)k >> 2; + const uint k_b = (uint)k >> 5; + + __local uint sh_qa[TILESIZE_N][8]; + __local half sh_d[TILESIZE_N]; + __local half sh_s[TILESIZE_N]; + +#define NGROUPS (TILESIZE_N / 4) + float4 acc[NGROUPS]; + #pragma unroll + for (int g = 0; g < NGROUPS; ++g) acc[g] = (float4)(0.0f); + + for (uint step = 0; step < (uint)k; step += 32) { + const uint sub = step >> 5; + const uint sb_idx = step / QK_K; + const uint sub_idx = sub & 7; + + const float dd = (float)src0_d [rrow + sb_idx * m]; + const float dmm = (float)src0_dm[rrow + sb_idx * m]; + global const uchar * sc = src0_s + rrow * num_superblocks * K_SCALE_SIZE + sb_idx * K_SCALE_SIZE; + uchar sv, mn; + get_scale_min_k4(sub_idx, sc, &sv, &mn, mask_d6, mask_d4, mask_hi2); + const float scale = dd * (float)sv; + const float minv = dmm * (float)mn; + + // repack this row's 32 weights (nibble | high-bit) into 8 dp4a uints. + // ushort u -> 4 elements at K = step + u*4; its 4 high bits are nibble + // (u&1) of qh byte (step/8 + u/2). + const uint wbase = rrow + (step >> 2) * (uint)m; + const uint qhbase = rrow + (step >> 3) * (uint)m; + uint8 qw; +#define QWU(u) ( EXP4((uint)src0_q[wbase + (uint)(u) * m]) \ + | EXP1( (uint)((src0_qh[qhbase + (uint)((u) >> 1) * m] >> (((u) & 1) * 4)) & 0x0Fu) ) ) + qw.s0 = QWU(0); qw.s1 = QWU(1); qw.s2 = QWU(2); qw.s3 = QWU(3); + qw.s4 = QWU(4); qw.s5 = QWU(5); qw.s6 = QWU(6); qw.s7 = QWU(7); +#undef QWU + + for (uint idx = lid; idx < TILESIZE_N * 8; idx += 64) { + const uint t = idx >> 3; + const uint u = idx & 7; + const uint c = col_base + t; + sh_qa[t][u] = (c < (uint)n_no_padding) ? src1_qa[c * k_u + (step >> 2) + u] : 0u; + } + if (lid < TILESIZE_N) { + const uint c = col_base + lid; + sh_d[lid] = (c < (uint)n_no_padding) ? src1_da[c * k_b + sub] : (half)0; + sh_s[lid] = (c < (uint)n_no_padding) ? src1_sa[c * k_b + sub] : (half)0; + } + barrier(CLK_LOCAL_MEM_FENCE); + +#define LD4(arr, b) ((float4)((float)arr[(b)+0], (float)arr[(b)+1], (float)arr[(b)+2], (float)arr[(b)+3])) + #pragma unroll + for (int g = 0; g < NGROUPS; ++g) { + const int b = g * 4; + float4 rf; + rf.s0 = (float)dot8_q8a(qw, sh_qa[b+0]); rf.s1 = (float)dot8_q8a(qw, sh_qa[b+1]); + rf.s2 = (float)dot8_q8a(qw, sh_qa[b+2]); rf.s3 = (float)dot8_q8a(qw, sh_qa[b+3]); + acc[g] += scale * LD4(sh_d, b) * rf - minv * LD4(sh_s, b); + } +#undef LD4 + barrier(CLK_LOCAL_MEM_FENCE); + } + + if (!row_valid) { + return; + } + + #pragma unroll + for (int g = 0; g < NGROUPS; ++g) { + const uint b = (uint)(g * 4); + const float4 a = acc[g]; + const uint c0 = col_base + b; + if (c0 + 0 < (uint)n_no_padding) dst[(c0 + 0) * (uint)m + row] = a.s0; + if (c0 + 1 < (uint)n_no_padding) dst[(c0 + 1) * (uint)m + row] = a.s1; + if (c0 + 2 < (uint)n_no_padding) dst[(c0 + 2) * (uint)m + row] = a.s2; + if (c0 + 3 < (uint)n_no_padding) dst[(c0 + 3) * (uint)m + row] = a.s3; + } +#undef NGROUPS +} diff --git a/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q6_k_q8_1_dp4a.cl b/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q6_k_q8_1_dp4a.cl new file mode 100644 index 000000000..382d79fdd --- /dev/null +++ b/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q6_k_q8_1_dp4a.cl @@ -0,0 +1,144 @@ +#pragma OPENCL EXTENSION cl_khr_fp16 : enable +#pragma OPENCL EXTENSION cl_khr_subgroups : enable +#ifdef cl_khr_integer_dot_product +#pragma OPENCL EXTENSION cl_khr_integer_dot_product : enable +#endif + +#define TILESIZE_N 32 +#define QK_K 256 + +// 4 nibbles in the low 16 bits of `u` -> 4 bytes (value 0..15, in bits 0-3). +#define EXP4(u) ( ((uint)((u) & 0x000Fu)) | \ + (((uint)((u) & 0x00F0u)) << 4) | \ + (((uint)((u) & 0x0F00u)) << 8) | \ + (((uint)((u) & 0xF000u)) << 12) ) + +// 4 2-bit highs in byte `b` -> 4 bytes, value 0..3 in bits 4-5 (pre-multiplied +// by 16 so it ORs with the EXP4 nibble to form q6 in 0..63). +#define EXP2(b) ( (((uint)((b) & 0x03u)) << 4) | \ + (((uint)((b) & 0x0Cu)) << 10) | \ + (((uint)((b) & 0x30u)) << 16) | \ + (((uint)((b) & 0xC0u)) << 22) ) + +// q6 (0..63, bits 0-5 of each byte) -> (q6-32) as a signed int8 per byte. +inline uint SIGN6(uint q6p) { + uint x = q6p ^ 0x20202020u; + uint s = x & 0x20202020u; + return x | (s << 1) | (s << 2); +} + +// 16-K dp4a dot: 4 packed weight uints against 4 packed int8 activation uints. +inline int dot4_q8a(uint w0, uint w1, uint w2, uint w3, + uint a0, uint a1, uint a2, uint a3) { + int r = 0; + r = dot_acc_sat_4x8packed_ss_int(w0, a0, r); + r = dot_acc_sat_4x8packed_ss_int(w1, a1, r); + r = dot_acc_sat_4x8packed_ss_int(w2, a2, r); + r = dot_acc_sat_4x8packed_ss_int(w3, a3, r); + return r; +} + +__attribute__((qcom_wave_pair_mode(1))) +kernel void kernel_gemm_noshuffle_q6_k_q8_1_dp4a( + __global const ushort * src0_ql, // q6_K low nibbles (noshuffle) + __global const uchar * src0_qh, // q6_K high 2-bit (uchar, 4 highs/elem) + __global const ushort * src0_s, // int8 scale codes (2 chars/ushort, per 16) + __global const half * src0_d, // per-superblock scale + __global const uint * src1_qa, // q8_1 activations int8 (as uint, 4/elem) [N, K] + __global const half * src1_da, // q8_1 per-block scale [N, K/32] + __global float * dst, + ulong offsetd, + int m, // output features (rows) + int n_no_padding, // tokens (cols) + int k // K (== ne00) +) { + dst = (global float *)((global char *)dst + offsetd); + + const uint lid = get_local_id(0); // 0..63 -> row within the M-tile + const uint block_id_m = get_global_id(1); + const uint block_id_n = get_global_id(2); + + const uint row = block_id_m * 64 + lid; + const uint col_base = block_id_n * TILESIZE_N; + const bool row_valid = row < (uint)m; + const uint rrow = row_valid ? row : 0; // clamp OOB rows; their writes are masked + + const uint k_u = (uint)k >> 2; // K in uint (int8x4) units + const uint k_b = (uint)k >> 5; // blocks-of-32 along K + + __local uint sh_qa[TILESIZE_N][8]; + __local half sh_d[TILESIZE_N]; + +#define NGROUPS (TILESIZE_N / 4) + float4 acc[NGROUPS]; + #pragma unroll + for (int g = 0; g < NGROUPS; ++g) acc[g] = (float4)(0.0f); + + for (uint step = 0; step < (uint)k; step += 32) { + const uint sub = step >> 5; // 32-block index along K + const uint sb_idx = step / QK_K; // superblock index + + // q6_K superblock scale + the two int8 sub-scales spanning this 32-block + const float dd = (float)src0_d[rrow + sb_idx * m]; + const char2 sc = as_char2(src0_s[rrow + sub * m]); + const float scale0 = dd * (float)sc.s0; // K step..step+15 + const float scale1 = dd * (float)sc.s1; // K step+16..step+31 + + // repack this row's 32 weights into 8 dp4a uints (4 K each). ql ushort + + // qh uchar are co-located at src0_*[row + (step/4 + u)*m]. + const uint wbase = rrow + (step >> 2) * (uint)m; + uint qw[8]; + #pragma unroll + for (int u = 0; u < 8; ++u) { + const uint o = wbase + (uint)u * (uint)m; + qw[u] = SIGN6(EXP4((uint)src0_ql[o]) | EXP2((uint)src0_qh[o])); + } + + // cooperatively stage the 32-token x 32-K int8 activations + scale + for (uint idx = lid; idx < TILESIZE_N * 8; idx += 64) { + const uint t = idx >> 3; + const uint u = idx & 7; + const uint c = col_base + t; + sh_qa[t][u] = (c < (uint)n_no_padding) ? src1_qa[c * k_u + (step >> 2) + u] : 0u; + } + if (lid < TILESIZE_N) { + const uint c = col_base + lid; + sh_d[lid] = (c < (uint)n_no_padding) ? src1_da[c * k_b + sub] : (half)0; + } + barrier(CLK_LOCAL_MEM_FENCE); + + #pragma unroll + for (int g = 0; g < NGROUPS; ++g) { + const int b = g * 4; + float4 rf; + #define DOT_TOK(j) { \ + __local const uint * a = sh_qa[b + (j)]; \ + const int raw1 = dot4_q8a(qw[0], qw[1], qw[2], qw[3], a[0], a[1], a[2], a[3]); \ + const int raw2 = dot4_q8a(qw[4], qw[5], qw[6], qw[7], a[4], a[5], a[6], a[7]); \ + rf.s##j = scale0 * (float)raw1 + scale1 * (float)raw2; \ + } + DOT_TOK(0); DOT_TOK(1); DOT_TOK(2); DOT_TOK(3); + #undef DOT_TOK + const float4 ad = (float4)((float)sh_d[b+0], (float)sh_d[b+1], (float)sh_d[b+2], (float)sh_d[b+3]); + acc[g] += ad * rf; + } + barrier(CLK_LOCAL_MEM_FENCE); + } + + if (!row_valid) { + return; + } + + // dst is [token, feature] row-major (stride m): dst[col*m + row]. + #pragma unroll + for (int g = 0; g < NGROUPS; ++g) { + const uint b = (uint)(g * 4); + const float4 a = acc[g]; + const uint c0 = col_base + b; + if (c0 + 0 < (uint)n_no_padding) dst[(c0 + 0) * (uint)m + row] = a.s0; + if (c0 + 1 < (uint)n_no_padding) dst[(c0 + 1) * (uint)m + row] = a.s1; + if (c0 + 2 < (uint)n_no_padding) dst[(c0 + 2) * (uint)m + row] = a.s2; + if (c0 + 3 < (uint)n_no_padding) dst[(c0 + 3) * (uint)m + row] = a.s3; + } +#undef NGROUPS +} diff --git a/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q8_0_q8_1_dp4a.cl b/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q8_0_q8_1_dp4a.cl new file mode 100644 index 000000000..a481636c2 --- /dev/null +++ b/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q8_0_q8_1_dp4a.cl @@ -0,0 +1,212 @@ +#pragma OPENCL EXTENSION cl_khr_fp16 : enable +#pragma OPENCL EXTENSION cl_khr_subgroups : enable +#ifdef cl_khr_integer_dot_product +#pragma OPENCL EXTENSION cl_khr_integer_dot_product : enable +#endif + +// ne1<=8 keeps the f16 / bin small-batch path. + +#define TILESIZE_N 32 + +// 32-K dp4a dot of one token's int8 activations (8 packed uints in lm) against +// 8 packed weight uints. q8_0 weights are already dp4a-format signed int8. +inline int dot8_q8a(uint8 qw, __local const uint * a) { + int r = 0; + r = dot_acc_sat_4x8packed_ss_int(qw.s0, a[0], r); + r = dot_acc_sat_4x8packed_ss_int(qw.s1, a[1], r); + r = dot_acc_sat_4x8packed_ss_int(qw.s2, a[2], r); + r = dot_acc_sat_4x8packed_ss_int(qw.s3, a[3], r); + r = dot_acc_sat_4x8packed_ss_int(qw.s4, a[4], r); + r = dot_acc_sat_4x8packed_ss_int(qw.s5, a[5], r); + r = dot_acc_sat_4x8packed_ss_int(qw.s6, a[6], r); + r = dot_acc_sat_4x8packed_ss_int(qw.s7, a[7], r); + return r; +} + +__attribute__((qcom_wave_pair_mode(1))) +kernel void kernel_gemm_noshuffle_q8_0_q8_1_dp4a( + __global const uint * src0_q, // q8_0 weights: signed int8, 4/uint, feature-major + __global const half * src0_d, // per-32-block scale, feature-major [row + (k/32)*m] + __global const uint * src1_qa, // q8_1 activations int8 (as uint, 4/elem) [N, K] + __global const half * src1_da, // q8_1 per-block scale [N, K/32] + __global float * dst, + ulong offsetd, + int m, // output features (rows) + int n_no_padding, // tokens (cols) + int k // K (== ne00) +) { + dst = (global float *)((global char *)dst + offsetd); + + const uint lid = get_local_id(0); // 0..63 -> row within the M-tile + const uint block_id_m = get_global_id(1); + const uint block_id_n = get_global_id(2); + + const uint row = block_id_m * 64 + lid; + const uint col_base = block_id_n * TILESIZE_N; + const bool row_valid = row < (uint)m; + const uint rrow = row_valid ? row : 0; // clamp OOB rows; their writes are masked + + const uint k_u = (uint)k >> 2; // K in uint (int8x4) units + const uint k_b = (uint)k >> 5; // blocks-of-32 along K + + __local uint sh_qa[TILESIZE_N][8]; + __local half sh_d[TILESIZE_N]; + +#define NGROUPS (TILESIZE_N / 4) + float4 acc[NGROUPS]; + #pragma unroll + for (int g = 0; g < NGROUPS; ++g) acc[g] = (float4)(0.0f); + + for (uint step = 0; step < (uint)k; step += 32) { + const uint sub = step >> 5; + + const float d_w = (float)src0_d[rrow + sub * (uint)m]; + + // 8 weight uints (32 int8) for this row, this 32-block. Feature-major: + // src0_q[row + (k/4 + u)*m], k/4 = step/4 (= step>>2). + const uint wbase = rrow + (step >> 2) * (uint)m; + uint8 qw; + qw.s0 = src0_q[wbase + 0 * m]; + qw.s1 = src0_q[wbase + 1 * m]; + qw.s2 = src0_q[wbase + 2 * m]; + qw.s3 = src0_q[wbase + 3 * m]; + qw.s4 = src0_q[wbase + 4 * m]; + qw.s5 = src0_q[wbase + 5 * m]; + qw.s6 = src0_q[wbase + 6 * m]; + qw.s7 = src0_q[wbase + 7 * m]; + + // cooperatively stage the 32-token x 32-K int8 activations to LDS + for (uint idx = lid; idx < TILESIZE_N * 8; idx += 64) { + const uint t = idx >> 3; + const uint u = idx & 7; + const uint c = col_base + t; + sh_qa[t][u] = (c < (uint)n_no_padding) ? src1_qa[c * k_u + (step >> 2) + u] : 0u; + } + if (lid < TILESIZE_N) { + const uint c = col_base + lid; + sh_d[lid] = (c < (uint)n_no_padding) ? src1_da[c * k_b + sub] : (half)0; + } + barrier(CLK_LOCAL_MEM_FENCE); + +#define LD4(arr, b) ((float4)((float)arr[(b)+0], (float)arr[(b)+1], (float)arr[(b)+2], (float)arr[(b)+3])) + #pragma unroll + for (int g = 0; g < NGROUPS; ++g) { + const int b = g * 4; + float4 rf; + rf.s0 = (float)dot8_q8a(qw, sh_qa[b+0]); rf.s1 = (float)dot8_q8a(qw, sh_qa[b+1]); + rf.s2 = (float)dot8_q8a(qw, sh_qa[b+2]); rf.s3 = (float)dot8_q8a(qw, sh_qa[b+3]); + acc[g] += d_w * LD4(sh_d, b) * rf; + } +#undef LD4 + barrier(CLK_LOCAL_MEM_FENCE); + } + + if (!row_valid) { + return; + } + + // dst is [token, feature] row-major (stride m): dst[col*m + row]. + #pragma unroll + for (int g = 0; g < NGROUPS; ++g) { + const uint b = (uint)(g * 4); + const float4 a = acc[g]; + const uint c0 = col_base + b; + if (c0 + 0 < (uint)n_no_padding) dst[(c0 + 0) * (uint)m + row] = a.s0; + if (c0 + 1 < (uint)n_no_padding) dst[(c0 + 1) * (uint)m + row] = a.s1; + if (c0 + 2 < (uint)n_no_padding) dst[(c0 + 2) * (uint)m + row] = a.s2; + if (c0 + 3 < (uint)n_no_padding) dst[(c0 + 3) * (uint)m + row] = a.s3; + } +#undef NGROUPS +} + +__attribute__((qcom_wave_pair_mode(1))) +kernel void kernel_gemm_noshuffle_q8_0_q8_1_dp4a_wimg( + __read_only image1d_buffer_t src0_q_img, // q8_0 weights as uint32 texels (4 int8/texel) + __global const half * src0_d, + __global const uint * src1_qa, + __global const half * src1_da, + __global float * dst, + ulong offsetd, + int m, + int n_no_padding, + int k +) { + dst = (global float *)((global char *)dst + offsetd); + + const uint lid = get_local_id(0); + const uint block_id_m = get_global_id(1); + const uint block_id_n = get_global_id(2); + + const uint row = block_id_m * 64 + lid; + const uint col_base = block_id_n * TILESIZE_N; + const bool row_valid = row < (uint)m; + const uint rrow = row_valid ? row : 0; + + const uint k_u = (uint)k >> 2; + const uint k_b = (uint)k >> 5; + + __local uint sh_qa[TILESIZE_N][8]; + __local half sh_d[TILESIZE_N]; + +#define NGROUPS (TILESIZE_N / 4) + float4 acc[NGROUPS]; + #pragma unroll + for (int g = 0; g < NGROUPS; ++g) acc[g] = (float4)(0.0f); + + for (uint step = 0; step < (uint)k; step += 32) { + const uint sub = step >> 5; + + const float d_w = (float)src0_d[rrow + sub * (uint)m]; + + const uint wbase = rrow + (step >> 2) * (uint)m; + uint8 qw; + qw.s0 = read_imageui(src0_q_img, (int)(wbase + 0 * m)).x; + qw.s1 = read_imageui(src0_q_img, (int)(wbase + 1 * m)).x; + qw.s2 = read_imageui(src0_q_img, (int)(wbase + 2 * m)).x; + qw.s3 = read_imageui(src0_q_img, (int)(wbase + 3 * m)).x; + qw.s4 = read_imageui(src0_q_img, (int)(wbase + 4 * m)).x; + qw.s5 = read_imageui(src0_q_img, (int)(wbase + 5 * m)).x; + qw.s6 = read_imageui(src0_q_img, (int)(wbase + 6 * m)).x; + qw.s7 = read_imageui(src0_q_img, (int)(wbase + 7 * m)).x; + + for (uint idx = lid; idx < TILESIZE_N * 8; idx += 64) { + const uint t = idx >> 3; + const uint u = idx & 7; + const uint c = col_base + t; + sh_qa[t][u] = (c < (uint)n_no_padding) ? src1_qa[c * k_u + (step >> 2) + u] : 0u; + } + if (lid < TILESIZE_N) { + const uint c = col_base + lid; + sh_d[lid] = (c < (uint)n_no_padding) ? src1_da[c * k_b + sub] : (half)0; + } + barrier(CLK_LOCAL_MEM_FENCE); + +#define LD4(arr, b) ((float4)((float)arr[(b)+0], (float)arr[(b)+1], (float)arr[(b)+2], (float)arr[(b)+3])) + #pragma unroll + for (int g = 0; g < NGROUPS; ++g) { + const int b = g * 4; + float4 rf; + rf.s0 = (float)dot8_q8a(qw, sh_qa[b+0]); rf.s1 = (float)dot8_q8a(qw, sh_qa[b+1]); + rf.s2 = (float)dot8_q8a(qw, sh_qa[b+2]); rf.s3 = (float)dot8_q8a(qw, sh_qa[b+3]); + acc[g] += d_w * LD4(sh_d, b) * rf; + } +#undef LD4 + barrier(CLK_LOCAL_MEM_FENCE); + } + + if (!row_valid) { + return; + } + + #pragma unroll + for (int g = 0; g < NGROUPS; ++g) { + const uint b = (uint)(g * 4); + const float4 a = acc[g]; + const uint c0 = col_base + b; + if (c0 + 0 < (uint)n_no_padding) dst[(c0 + 0) * (uint)m + row] = a.s0; + if (c0 + 1 < (uint)n_no_padding) dst[(c0 + 1) * (uint)m + row] = a.s1; + if (c0 + 2 < (uint)n_no_padding) dst[(c0 + 2) * (uint)m + row] = a.s2; + if (c0 + 3 < (uint)n_no_padding) dst[(c0 + 3) * (uint)m + row] = a.s3; + } +#undef NGROUPS +} diff --git a/ggml/src/ggml-opencl/kernels/gemv_moe_mxfp4_f32_ns.cl b/ggml/src/ggml-opencl/kernels/gemv_moe_mxfp4_f32_ns.cl index 75129e20c..ee8b94f44 100644 --- a/ggml/src/ggml-opencl/kernels/gemv_moe_mxfp4_f32_ns.cl +++ b/ggml/src/ggml-opencl/kernels/gemv_moe_mxfp4_f32_ns.cl @@ -163,3 +163,95 @@ __kernel void kernel_gemv_moe_mxfp4_f32_ns( } } + +__attribute__((qcom_reqd_sub_group_size("half"))) +__kernel void kernel_gemv_moe_mxfp4_f32_ns_wimg( + __read_only image1d_buffer_t src0_q, + __global uchar * src0_e, + __read_only image1d_buffer_t src1, + __global uint * src2, + __global float * dst, + ulong offsetd, + int ne00, + int ne01, + int ne11 +) { + uint i01 = get_global_id(0); + uint i20 = get_global_id(2); + uint sgid = get_local_id(1); + uint slid = get_sub_group_local_id(); + + if (i01 >= ne01) { + return; + } + + uint i11 = i20 % ne11; + + uint expert_id = src2[i20]; + uint expert_offset = expert_id * ne00 * ne01 / 32; + + __private float sum = 0.0f; + + for (uint ib00 = sgid; ib00 < (ne00 / QK_MXFP4); ib00 += N_SIMDGROUP) { + + uint4 regQ; + uint block_offset = expert_offset * 4 + ib00 * ne01 * 4 + i01; + + regQ.s0 = read_imageui(src0_q, (int)(block_offset)).x; + regQ.s1 = read_imageui(src0_q, (int)(block_offset + ne01)).x; + regQ.s2 = read_imageui(src0_q, (int)(block_offset + ne01 * 2)).x; + regQ.s3 = read_imageui(src0_q, (int)(block_offset + ne01 * 3)).x; + + uint offset = i11 * ne00 / 4 + ib00 * 8; + + half8 fp16x8 = mxfp4_to_fp16_packed8(as_ushort2(regQ.s0)); + + float4 shared_y4; + shared_y4 = read_imagef(src1, (offset + 0)); + float4 acc = shared_y4 * convert_float4(fp16x8.lo); + + shared_y4 = read_imagef(src1, (offset + 1)); + acc += shared_y4 * convert_float4(fp16x8.hi); + + fp16x8 = mxfp4_to_fp16_packed8(as_ushort2(regQ.s1)); + + shared_y4 = read_imagef(src1, (offset + 2)); + acc += shared_y4 * convert_float4(fp16x8.lo); + + shared_y4 = read_imagef(src1, (offset + 3)); + acc += shared_y4 * convert_float4(fp16x8.hi); + + fp16x8 = mxfp4_to_fp16_packed8(as_ushort2(regQ.s2)); + + shared_y4 = read_imagef(src1, (offset + 4)); + acc += shared_y4 * convert_float4(fp16x8.lo); + + shared_y4 = read_imagef(src1, (offset + 5)); + acc += shared_y4 * convert_float4(fp16x8.hi); + + fp16x8 = mxfp4_to_fp16_packed8(as_ushort2(regQ.s3)); + + shared_y4 = read_imagef(src1, (offset + 6)); + acc += shared_y4 * convert_float4(fp16x8.lo); + + shared_y4 = read_imagef(src1, (offset + 7)); + acc += shared_y4 * convert_float4(fp16x8.hi); + + uchar regE = src0_e[ib00 * ne01 + i01 + expert_offset]; + sum += e8m0_to_fp32(regE) * ((acc.s0 + acc.s1) + (acc.s2 + acc.s3)); + } + + __local float reduceLM[SIMDGROUP_WIDTH * (N_SIMDGROUP - 1)]; + if (sgid == 1) reduceLM[SIMDGROUP_WIDTH * 0 + slid] = sum; + if (sgid == 2) reduceLM[SIMDGROUP_WIDTH * 1 + slid] = sum; + if (sgid == 3) reduceLM[SIMDGROUP_WIDTH * 2 + slid] = sum; + barrier(CLK_LOCAL_MEM_FENCE); + if (sgid == 0) sum += reduceLM[SIMDGROUP_WIDTH * 0 + slid]; + if (sgid == 0) sum += reduceLM[SIMDGROUP_WIDTH * 1 + slid]; + if (sgid == 0) sum += reduceLM[SIMDGROUP_WIDTH * 2 + slid]; + + if (sgid == 0) { + dst = dst + (offsetd >> 2); + dst[i01 + i20 * ne01] = sum; + } +} diff --git a/ggml/src/ggml-opencl/kernels/gemv_moe_q4_k_f32_ns.cl b/ggml/src/ggml-opencl/kernels/gemv_moe_q4_k_f32_ns.cl index 12464e982..d3a3c7db8 100644 --- a/ggml/src/ggml-opencl/kernels/gemv_moe_q4_k_f32_ns.cl +++ b/ggml/src/ggml-opencl/kernels/gemv_moe_q4_k_f32_ns.cl @@ -153,3 +153,114 @@ __kernel void kernel_gemv_moe_q4_k_f32_ns( dst[i01 + i20 * ne01] = sum; } } + +__attribute__((qcom_reqd_sub_group_size("half"))) +__kernel void kernel_gemv_moe_q4_k_f32_ns_wimg( + __read_only image1d_buffer_t src0_q, + __global half * src0_d, + __global half * src0_dm, + __global uchar * src0_s, + __read_only image1d_buffer_t src1, + __global uint * src2, + __global float * dst, + ulong offsetd, + int ne00, + int ne01, + int ne11 +) { + uint i01 = get_global_id(0); + uint i20 = get_global_id(2); + uint sgid = get_local_id(1); + uint slid = get_sub_group_local_id(); + + if (i01 >= ne01) { + return; + } + + uint i11 = i20 % ne11; + + uint expert_id = src2[i20]; + + int num_superblocks = ne00 / QK_K; + int num_subblocks = ne00 / 32; + int scales_per_row = num_superblocks * K_SCALE_SIZE; + + uint expert_q_offset = expert_id * (ne00 / 8) * ne01; + uint expert_d_offset = expert_id * num_superblocks * ne01; + + __private float sum = 0.0f; + + for (uint ib = sgid; ib < num_subblocks; ib += N_SIMDGROUP) { + uint sb = ib / 8; + uint j = ib % 8; + + half d_val = src0_d[expert_d_offset + sb * ne01 + i01]; + half dm_val = src0_dm[expert_d_offset + sb * ne01 + i01]; + + global const uchar * sc = src0_s + (expert_id * ne01 + i01) * scales_per_row + sb * K_SCALE_SIZE; + uchar sv, mn; + get_scale_min_k4(j, sc, &sv, &mn); + + float scale = (float)d_val * (float)sv; + float minv = (float)dm_val * (float)mn; + + uint q_base = expert_q_offset + ib * ne01 * 4 + i01; + + uint4 regQ; + regQ.s0 = read_imageui(src0_q, (int)(q_base)).x; + regQ.s1 = read_imageui(src0_q, (int)(q_base + ne01)).x; + regQ.s2 = read_imageui(src0_q, (int)(q_base + ne01 * 2)).x; + regQ.s3 = read_imageui(src0_q, (int)(q_base + ne01 * 3)).x; + + uint y_offset = i11 * ne00 / 4 + ib * 8; + + float8 fp32x8 = q4_k_to_fp32_packed8(as_ushort2(regQ.s0), scale, minv); + + float4 shared_y4; + shared_y4 = read_imagef(src1, (y_offset + 0)); + float4 acc = shared_y4 * fp32x8.lo; + + shared_y4 = read_imagef(src1, (y_offset + 1)); + acc += shared_y4 * fp32x8.hi; + + fp32x8 = q4_k_to_fp32_packed8(as_ushort2(regQ.s1), scale, minv); + + shared_y4 = read_imagef(src1, (y_offset + 2)); + acc += shared_y4 * fp32x8.lo; + + shared_y4 = read_imagef(src1, (y_offset + 3)); + acc += shared_y4 * fp32x8.hi; + + fp32x8 = q4_k_to_fp32_packed8(as_ushort2(regQ.s2), scale, minv); + + shared_y4 = read_imagef(src1, (y_offset + 4)); + acc += shared_y4 * fp32x8.lo; + + shared_y4 = read_imagef(src1, (y_offset + 5)); + acc += shared_y4 * fp32x8.hi; + + fp32x8 = q4_k_to_fp32_packed8(as_ushort2(regQ.s3), scale, minv); + + shared_y4 = read_imagef(src1, (y_offset + 6)); + acc += shared_y4 * fp32x8.lo; + + shared_y4 = read_imagef(src1, (y_offset + 7)); + acc += shared_y4 * fp32x8.hi; + + sum += ((acc.s0 + acc.s1) + (acc.s2 + acc.s3)); + } + + __local float reduceLM[SIMDGROUP_WIDTH * (N_SIMDGROUP - 1)]; + if (sgid == 1) reduceLM[SIMDGROUP_WIDTH * 0 + slid] = sum; + if (sgid == 2) reduceLM[SIMDGROUP_WIDTH * 1 + slid] = sum; + if (sgid == 3) reduceLM[SIMDGROUP_WIDTH * 2 + slid] = sum; + barrier(CLK_LOCAL_MEM_FENCE); + if (sgid == 0) sum += reduceLM[SIMDGROUP_WIDTH * 0 + slid]; + if (sgid == 0) sum += reduceLM[SIMDGROUP_WIDTH * 1 + slid]; + if (sgid == 0) sum += reduceLM[SIMDGROUP_WIDTH * 2 + slid]; + + if (sgid == 0) { + dst = dst + (offsetd >> 2); + dst[i01 + i20 * ne01] = sum; + } +} diff --git a/ggml/src/ggml-opencl/kernels/moe_combine.cl b/ggml/src/ggml-opencl/kernels/moe_combine.cl new file mode 100644 index 000000000..c195f1472 --- /dev/null +++ b/ggml/src/ggml-opencl/kernels/moe_combine.cl @@ -0,0 +1,36 @@ +// Fused MoE combine epilogue: replaces the router-weight MUL + the (n_expert_used-1) +// cross-expert ADD chain with ONE weighted-sum-across-experts pass. +// dst[row, tok] = sum_e experts[row, e, tok] * weights[0, e, tok] +// experts: [n_embd, n_expert_used, n_tokens] f32 (contiguous after down-proj GEMM) +// weights: [1, n_expert_used, n_tokens] f32 +// dst: [n_embd, n_tokens] f32 +// One read of experts + one write of dst (eliminates the intermediate weighted +// buffer and the k-1 elementwise add round-trips). Vectorized float4 over rows. +// strides e1/e2/w1/w2/d1 are in ELEMENTS (floats). + +__kernel void kernel_moe_combine_f32( + __global const char * e_buf, ulong off_e, + __global const char * w_buf, ulong off_w, + __global char * d_buf, ulong off_d, + int n_embd4, // n_embd / 4 + int k, // n_expert_used + int n_tokens, + uint e1, uint e2, // experts strides (elements): per-expert, per-token + uint w1, uint w2, // weights strides (elements) + uint d1) // dst per-token stride (elements) +{ + const uint r4 = get_global_id(0); + const uint tok = get_global_id(1); + if (r4 >= (uint)n_embd4 || tok >= (uint)n_tokens) return; + + __global const float * E = (__global const float *)(e_buf + off_e) + tok*e2 + r4*4u; + __global const float * W = (__global const float *)(w_buf + off_w) + tok*w2; + + float4 acc = (float4)(0.0f); + for (int e = 0; e < k; ++e) { + acc = mad(vload4(0, E + (uint)e*e1), (float4)(W[(uint)e*w1]), acc); + } + + __global float * D = (__global float *)(d_buf + off_d) + tok*d1 + r4*4u; + vstore4(acc, 0, D); +} diff --git a/ggml/src/ggml-opencl/kernels/moe_reorder_quant_a_q8_1.cl b/ggml/src/ggml-opencl/kernels/moe_reorder_quant_a_q8_1.cl new file mode 100644 index 000000000..0d16f3abd --- /dev/null +++ b/ggml/src/ggml-opencl/kernels/moe_reorder_quant_a_q8_1.cl @@ -0,0 +1,64 @@ +#pragma OPENCL EXTENSION cl_khr_fp16 : enable + +// Fused MoE activation reorder + q8_1 quantization for the dp4a prefill GEMM. +// Combines kernel_moe_reorder_b (gather src1 rows per the post-router map) with +// the q8_1 quant pre-pass, so the f32 reordered-activation tile buffer is never +// materialised (saves a full write + read of [tok_slots * ne00] floats). +// +// One work-item per (token_slot, 32-block). Padding lanes (router 0xFFFFFFFF) +// emit d=0,s=0,qs=0 so they contribute nothing to the GEMM, exactly as the +// reorder zero-fill did. Output layout matches kernel_moe_quant_a_q8_1: +// qa[token_slot*K + blk*32 + i], da/sa[token_slot*(K/32) + blk]. +__kernel void kernel_moe_reorder_quant_a_q8_1( + __global const float * src, // original activations (offset applied) + __global const uint * router, // post-router indices [tok_slots] + __global char * qa, + __global half * da, + __global half * sa, + __global const int * total_tiles, + uint K, + ushort map_ratio, + uint tile_size, + uint n_kblocks // K / 32 +) { + const uint blk = get_global_id(0); // 32-block along K + const uint tok = get_global_id(1); // token slot (post_router_idx) + + if (blk >= n_kblocks || tok >= (uint)total_tiles[0] * tile_size) { + return; + } + + const uint out_base = tok * K + blk * 32; + const uint bidx = tok * n_kblocks + blk; + + const uint router_idx = router[tok]; + + float v[32]; + float amax = 0.0f; + if (router_idx == 0xFFFFFFFF) { + #pragma unroll + for (int i = 0; i < 32; ++i) v[i] = 0.0f; + } else { + const uint act_idx = router_idx / map_ratio; + const uint in_base = act_idx * K + blk * 32; + #pragma unroll + for (int i = 0; i < 32; ++i) { + v[i] = src[in_base + i]; + amax = fmax(amax, fabs(v[i])); + } + } + + const float d = amax / 127.0f; + const float id = (amax > 0.0f) ? (127.0f / amax) : 0.0f; + + int sum = 0; + #pragma unroll + for (int i = 0; i < 32; ++i) { + const int q = (int)rint(v[i] * id); + qa[out_base + i] = (char)q; + sum += q; + } + + da[bidx] = (half)d; + sa[bidx] = (half)(d * (float)sum); +} diff --git a/ggml/src/ggml-opencl/kernels/quant_a_q8_1.cl b/ggml/src/ggml-opencl/kernels/quant_a_q8_1.cl new file mode 100644 index 000000000..3ce06305a --- /dev/null +++ b/ggml/src/ggml-opencl/kernels/quant_a_q8_1.cl @@ -0,0 +1,42 @@ +#pragma OPENCL EXTENSION cl_khr_fp16 : enable + +// Quantize a contiguous [N, K] f32 activation buffer (token-major, K contiguous +// per token) into q8_1 blocks of 32: int8 quants + per-block scale d + per-block +// sum s (= d * Sum(qs)). Consumed by kernel_gemm_noshuffle_q4_k_q8_1_dp4a for the +// dp4a (int8) dense q4_K prefill GEMM. One work-item per 32-element block. +__kernel void kernel_quant_a_q8_1( + __global const float * src, // [N * K] + __global char * qa, // [N * K] + __global half * da, // [N * (K/32)] + __global half * sa, // [N * (K/32)] + int total_blocks // N * (K/32) +) { + const int blk = get_global_id(0); + if (blk >= total_blocks) { + return; + } + + const int base = blk * 32; + + float v[32]; + float amax = 0.0f; + #pragma unroll + for (int i = 0; i < 32; ++i) { + v[i] = src[base + i]; + amax = fmax(amax, fabs(v[i])); + } + + const float d = amax / 127.0f; + const float id = (amax > 0.0f) ? (127.0f / amax) : 0.0f; + + int sum = 0; + #pragma unroll + for (int i = 0; i < 32; ++i) { + const int q = (int)rint(v[i] * id); + qa[base + i] = (char)q; + sum += q; + } + + da[blk] = (half)d; + sa[blk] = (half)(d * (float)sum); +}