diff --git a/ggml/src/ggml-sycl/ggml-sycl.cpp b/ggml/src/ggml-sycl/ggml-sycl.cpp index 6759af0ed..1cfc84126 100644 --- a/ggml/src/ggml-sycl/ggml-sycl.cpp +++ b/ggml/src/ggml-sycl/ggml-sycl.cpp @@ -4442,6 +4442,47 @@ static bool reorder_qw_q8_0(uint8_t * data_device, const int ncols, const int nr return true; } +// Reorder each Q8_0 expert slice into a self-contained [qs][d] SoA layout. +static bool reorder_qw_q8_0_moe(uint8_t * data_device, size_t expert_bytes, int64_t n_expert, dpct::queue_ptr stream) { + GGML_ASSERT(expert_bytes % sizeof(block_q8_0) == 0); + const int blocks_per_expert = (int) (expert_bytes / sizeof(block_q8_0)); + const size_t total_bytes = expert_bytes * (size_t) n_expert; + + sycl_reorder_temp_buffer tmp(stream, total_bytes); + if (!tmp) { + GGML_LOG_WARN("%s: failed to allocate %zu bytes for reorder temp buffer, skipping reorder\n", __func__, total_bytes); + return false; + } + uint8_t * tmp_buf = static_cast(tmp.ptr); + + sycl::event copy_event; + SYCL_CHECK(CHECK_TRY_ERROR(copy_event = stream->memcpy(tmp_buf, data_device, total_bytes))); + if (!g_ggml_sycl_use_async_mem_op) { + copy_event.wait(); + } + + const int total_blocks = blocks_per_expert * (int) n_expert; + auto reorder_event = stream->parallel_for(total_blocks, [=](auto gb_) { + const int gb = gb_; + const int e = gb / blocks_per_expert; + const int ib = gb % blocks_per_expert; + const block_q8_0 * x = (const block_q8_0 *) (tmp_buf + (size_t) e * expert_bytes); + uint8_t * base = data_device + (size_t) e * expert_bytes; + + auto * qs_ptr = base; + auto * d_ptr = (sycl::half *) (qs_ptr + (size_t) blocks_per_expert * QK8_0); + + for (int j = 0; j < QK8_0; ++j) { + *((int8_t *) qs_ptr + ib * QK8_0 + j) = x[ib].qs[j]; + } + d_ptr[ib] = x[ib].d; + }); + if (!g_ggml_sycl_use_async_mem_op) { + reorder_event.wait_and_throw(); + } + return true; +} + static bool reorder_qw_q4_k(uint8_t * data_device, size_t size, size_t offset, dpct::queue_ptr stream) { GGML_ASSERT(size % sizeof(block_q4_K) == 0); GGML_ASSERT(offset % sizeof(block_q4_K) == 0); @@ -4966,6 +5007,8 @@ static bool reorder_qw(const ggml_tensor * src0, dpct::queue_ptr stream) { if (src0->ne[2] > 1) { GGML_ASSERT((size_t) size == (size_t) src0->ne[2] * src0->nb[2]); switch (src0->type) { + case GGML_TYPE_Q8_0: + return reorder_qw_q8_0_moe(data_device, src0->nb[2], src0->ne[2], stream); case GGML_TYPE_Q4_K: return reorder_qw_q4_k_moe(data_device, src0->nb[2], src0->ne[2], stream); case GGML_TYPE_Q5_K: @@ -5051,8 +5094,8 @@ static void opt_for_reorder_id(ggml_backend_sycl_context * ctx, const ggml_tenso if (!g_ggml_sycl_enable_optimize || !ctx->opt_feature.reorder) { return; } - if (src0->type != GGML_TYPE_Q4_K && src0->type != GGML_TYPE_Q5_K && src0->type != GGML_TYPE_Q6_K && - src0->type != GGML_TYPE_MXFP4) { + if (src0->type != GGML_TYPE_Q8_0 && src0->type != GGML_TYPE_Q4_K && + src0->type != GGML_TYPE_Q5_K && src0->type != GGML_TYPE_Q6_K && src0->type != GGML_TYPE_MXFP4) { return; } // The MXFP4 reorder kernels use 8-byte vector loads, so every expert slice must stay aligned. @@ -5438,7 +5481,7 @@ static bool ggml_sycl_mul_mat_id_mmvq_fused( const int n_experts_used = (int) n_ids_per_group; const int nrows = (int) src0->ne[1]; - // Lazily reorder the (Q4_K) expert weights into a per-expert SoA layout, then run the reorder + // Lazily reorder supported expert weights into a per-expert SoA layout, then run the reorder // GEMV. Placed after the bail checks so a non-dispatchable op does not pay the reorder cost. opt_for_reorder_id(&ctx, src0); const ggml_tensor_extra_gpu * src0_extra = diff --git a/ggml/src/ggml-sycl/mmvq.cpp b/ggml/src/ggml-sycl/mmvq.cpp index eade8949f..e9057777c 100644 --- a/ggml/src/ggml-sycl/mmvq.cpp +++ b/ggml/src/ggml-sycl/mmvq.cpp @@ -3202,9 +3202,11 @@ static void mul_mat_vec_q_moe_reorder( const auto sg = item_ct1.get_sub_group(); + constexpr int vdr_mmvq = reorder_vec_dot_q_sycl::gtype == GGML_TYPE_Q8_0 ? + VDR_Q8_0_Q8_1_MMVQ : block_traits::vdr_mmvq; const int blocks_per_row = ncols / block_traits::qk; - constexpr int blocks_per_subgroup = ceil_div(block_traits::vdr_mmvq * WARP_SIZE, block_traits::qi); - constexpr int block_elements_per_subgroup = block_traits::qi / block_traits::vdr_mmvq; + constexpr int blocks_per_subgroup = ceil_div(vdr_mmvq * WARP_SIZE, block_traits::qi); + constexpr int block_elements_per_subgroup = block_traits::qi / vdr_mmvq; const int nblocks = nrows * (ncols / block_traits::qk); static_assert(blocks_per_subgroup > 0); @@ -3223,12 +3225,24 @@ static void mul_mat_vec_q_moe_reorder( #pragma unroll for (int elem = 0; elem < block_elements_per_subgroup; elem += WARP_SIZE) { - const int iqs = elem + block_traits::vdr_mmvq * (sg.get_local_linear_id() % block_elements_per_subgroup); - partial_sum += reorder_vec_dot_q_sycl()(vx, bx_offset, d_offset, q8_1_quant_ptr, q8_1_ds_ptr, iqs); + const int iqs = elem + vdr_mmvq * (sg.get_local_linear_id() % block_elements_per_subgroup); + if constexpr (reorder_vec_dot_q_sycl::gtype == GGML_TYPE_Q8_0) { + partial_sum += reorder_vec_dot_q_sycl()(vx, bx_offset, d_offset, q8_1_quant_ptr, q8_1_ds_ptr, iqs, vdr_mmvq); + } else { + partial_sum += reorder_vec_dot_q_sycl()(vx, bx_offset, d_offset, q8_1_quant_ptr, q8_1_ds_ptr, iqs); + } } } - auto sum = sycl::reduce_over_group(sg, partial_sum, std::plus<>()); + float sum = partial_sum; + if constexpr (reorder_vec_dot_q_sycl::gtype == GGML_TYPE_Q8_0) { + // Keep the accumulation order of the non-reordered MoE kernel. + for (int mask = WARP_SIZE / 2; mask > 0; mask >>= 1) { + sum += dpct::permute_sub_group_by_xor(sg, sum, mask); + } + } else { + sum = sycl::reduce_over_group(sg, partial_sum, std::plus<>()); + } if (sg.leader()) { dst[row] = sum; } @@ -3269,6 +3283,11 @@ bool ggml_sycl_mul_mat_vec_q_id_reorder( size_t src1_row_stride, dpct::queue_ptr stream) { switch (src0_type) { + case GGML_TYPE_Q8_0: + launch_mul_mat_vec_q_moe_reorder>( + vx_base, vy, ids_dev, dst_base, ncols, nrows, n_experts_used, + expert_weight_stride, dst_row_stride, src1_row_stride, stream); + return true; case GGML_TYPE_Q4_K: launch_mul_mat_vec_q_moe_reorder>( vx_base, vy, ids_dev, dst_base, ncols, nrows, n_experts_used, diff --git a/ggml/src/ggml-sycl/vecdotq.hpp b/ggml/src/ggml-sycl/vecdotq.hpp index d1b6092be..57360a004 100644 --- a/ggml/src/ggml-sycl/vecdotq.hpp +++ b/ggml/src/ggml-sycl/vecdotq.hpp @@ -483,7 +483,8 @@ template <> struct reorder_vec_dot_q_sycl { __dpct_inline__ float operator()(const void * __restrict__ vbq, const std::pair ibx_offset, const std::pair d_offset, const int8_t * q8_1_quant_ptr, - const sycl::half2 * q8_1_ds, const int & iqs) { + const sycl::half2 * q8_1_ds, const int & iqs, + const int vdr = q8_0_traits::vdr_mmvq) { const uint8_t * base = static_cast(vbq); const int8_t * qs = reinterpret_cast(base + ibx_offset.first); const ggml_half d = *reinterpret_cast(base + d_offset.first); @@ -492,14 +493,14 @@ template <> struct reorder_vec_dot_q_sycl { int u[q8_0_traits::vdr_mmvq]; #pragma unroll - for (size_t i = 0; i < q8_0_traits::vdr_mmvq; ++i) { + for (int i = 0; i < vdr; ++i) { v[i] = get_int_from_int8(qs, iqs + i); u[i] = get_int_from_int8_aligned(q8_1_quant_ptr, iqs + i); } int sumi = 0; #pragma unroll - for (size_t i = 0; i < q8_0_traits::vdr_mmvq; ++i) { + for (int i = 0; i < vdr; ++i) { sumi = dpct::dp4a(v[i], u[i], sumi); }