sycl: support reordered Q8_0 MoE weights (#29337)

Preserve the existing MoE accumulation order.

Assisted-by: Codex
Assisted-by: Angel
This commit is contained in:
Frosty40 authored and GitHub committed 2026-10-15 21:59:38 -04:00
1 parent 43d57fb8ca
commit e707790c73
3 files changed
+74 -11

No files matched your search

+46 -3
View File
@@ -4442,6 +4442,47 @@ static bool reorder_qw_q8_0(uint8_t * data_device, const int ncols, const int nr
return true;
}
// Reorder each Q8_0 expert slice into a self-contained [qs][d] SoA layout.
static bool reorder_qw_q8_0_moe(uint8_t * data_device, size_t expert_bytes, int64_t n_expert, dpct::queue_ptr stream) {
GGML_ASSERT(expert_bytes % sizeof(block_q8_0) == 0);
const int blocks_per_expert = (int) (expert_bytes / sizeof(block_q8_0));
const size_t total_bytes = expert_bytes * (size_t) n_expert;
sycl_reorder_temp_buffer tmp(stream, total_bytes);
if (!tmp) {
GGML_LOG_WARN("%s: failed to allocate %zu bytes for reorder temp buffer, skipping reorder\n", __func__, total_bytes);
return false;
}
uint8_t * tmp_buf = static_cast<uint8_t *>(tmp.ptr);
sycl::event copy_event;
SYCL_CHECK(CHECK_TRY_ERROR(copy_event = stream->memcpy(tmp_buf, data_device, total_bytes)));
if (!g_ggml_sycl_use_async_mem_op) {
copy_event.wait();
}
const int total_blocks = blocks_per_expert * (int) n_expert;
auto reorder_event = stream->parallel_for(total_blocks, [=](auto gb_) {
const int gb = gb_;
const int e = gb / blocks_per_expert;
const int ib = gb % blocks_per_expert;
const block_q8_0 * x = (const block_q8_0 *) (tmp_buf + (size_t) e * expert_bytes);
uint8_t * base = data_device + (size_t) e * expert_bytes;
auto * qs_ptr = base;
auto * d_ptr = (sycl::half *) (qs_ptr + (size_t) blocks_per_expert * QK8_0);
for (int j = 0; j < QK8_0; ++j) {
*((int8_t *) qs_ptr + ib * QK8_0 + j) = x[ib].qs[j];
}
d_ptr[ib] = x[ib].d;
});
if (!g_ggml_sycl_use_async_mem_op) {
reorder_event.wait_and_throw();
}
return true;
}
static bool reorder_qw_q4_k(uint8_t * data_device, size_t size, size_t offset, dpct::queue_ptr stream) {
GGML_ASSERT(size % sizeof(block_q4_K) == 0);
GGML_ASSERT(offset % sizeof(block_q4_K) == 0);
@@ -4966,6 +5007,8 @@ static bool reorder_qw(const ggml_tensor * src0, dpct::queue_ptr stream) {
if (src0->ne[2] > 1) {
GGML_ASSERT((size_t) size == (size_t) src0->ne[2] * src0->nb[2]);
switch (src0->type) {
case GGML_TYPE_Q8_0:
return reorder_qw_q8_0_moe(data_device, src0->nb[2], src0->ne[2], stream);
case GGML_TYPE_Q4_K:
return reorder_qw_q4_k_moe(data_device, src0->nb[2], src0->ne[2], stream);
case GGML_TYPE_Q5_K:
@@ -5051,8 +5094,8 @@ static void opt_for_reorder_id(ggml_backend_sycl_context * ctx, const ggml_tenso
if (!g_ggml_sycl_enable_optimize || !ctx->opt_feature.reorder) {
return;
}
if (src0->type != GGML_TYPE_Q4_K && src0->type != GGML_TYPE_Q5_K && src0->type != GGML_TYPE_Q6_K &&
src0->type != GGML_TYPE_MXFP4) {
if (src0->type != GGML_TYPE_Q8_0 && src0->type != GGML_TYPE_Q4_K &&
src0->type != GGML_TYPE_Q5_K && src0->type != GGML_TYPE_Q6_K && src0->type != GGML_TYPE_MXFP4) {
return;
}
// The MXFP4 reorder kernels use 8-byte vector loads, so every expert slice must stay aligned.
@@ -5438,7 +5481,7 @@ static bool ggml_sycl_mul_mat_id_mmvq_fused(
const int n_experts_used = (int) n_ids_per_group;
const int nrows = (int) src0->ne[1];
// Lazily reorder the (Q4_K) expert weights into a per-expert SoA layout, then run the reorder
// Lazily reorder supported expert weights into a per-expert SoA layout, then run the reorder
// GEMV. Placed after the bail checks so a non-dispatchable op does not pay the reorder cost.
opt_for_reorder_id(&ctx, src0);
const ggml_tensor_extra_gpu * src0_extra =
+24 -5
View File
@@ -3202,9 +3202,11 @@ static void mul_mat_vec_q_moe_reorder(
const auto sg = item_ct1.get_sub_group();
constexpr int vdr_mmvq = reorder_vec_dot_q_sycl::gtype == GGML_TYPE_Q8_0 ?
VDR_Q8_0_Q8_1_MMVQ : block_traits::vdr_mmvq;
const int blocks_per_row = ncols / block_traits::qk;
constexpr int blocks_per_subgroup = ceil_div(block_traits::vdr_mmvq * WARP_SIZE, block_traits::qi);
constexpr int block_elements_per_subgroup = block_traits::qi / block_traits::vdr_mmvq;
constexpr int blocks_per_subgroup = ceil_div(vdr_mmvq * WARP_SIZE, block_traits::qi);
constexpr int block_elements_per_subgroup = block_traits::qi / vdr_mmvq;
const int nblocks = nrows * (ncols / block_traits::qk);
static_assert(blocks_per_subgroup > 0);
@@ -3223,12 +3225,24 @@ static void mul_mat_vec_q_moe_reorder(
#pragma unroll
for (int elem = 0; elem < block_elements_per_subgroup; elem += WARP_SIZE) {
const int iqs = elem + block_traits::vdr_mmvq * (sg.get_local_linear_id() % block_elements_per_subgroup);
partial_sum += reorder_vec_dot_q_sycl()(vx, bx_offset, d_offset, q8_1_quant_ptr, q8_1_ds_ptr, iqs);
const int iqs = elem + vdr_mmvq * (sg.get_local_linear_id() % block_elements_per_subgroup);
if constexpr (reorder_vec_dot_q_sycl::gtype == GGML_TYPE_Q8_0) {
partial_sum += reorder_vec_dot_q_sycl()(vx, bx_offset, d_offset, q8_1_quant_ptr, q8_1_ds_ptr, iqs, vdr_mmvq);
} else {
partial_sum += reorder_vec_dot_q_sycl()(vx, bx_offset, d_offset, q8_1_quant_ptr, q8_1_ds_ptr, iqs);
}
}
}
auto sum = sycl::reduce_over_group(sg, partial_sum, std::plus<>());
float sum = partial_sum;
if constexpr (reorder_vec_dot_q_sycl::gtype == GGML_TYPE_Q8_0) {
// Keep the accumulation order of the non-reordered MoE kernel.
for (int mask = WARP_SIZE / 2; mask > 0; mask >>= 1) {
sum += dpct::permute_sub_group_by_xor(sg, sum, mask);
}
} else {
sum = sycl::reduce_over_group(sg, partial_sum, std::plus<>());
}
if (sg.leader()) {
dst[row] = sum;
}
@@ -3269,6 +3283,11 @@ bool ggml_sycl_mul_mat_vec_q_id_reorder(
size_t src1_row_stride,
dpct::queue_ptr stream) {
switch (src0_type) {
case GGML_TYPE_Q8_0:
launch_mul_mat_vec_q_moe_reorder<reorder_vec_dot_q_sycl<GGML_TYPE_Q8_0>>(
vx_base, vy, ids_dev, dst_base, ncols, nrows, n_experts_used,
expert_weight_stride, dst_row_stride, src1_row_stride, stream);
return true;
case GGML_TYPE_Q4_K:
launch_mul_mat_vec_q_moe_reorder<reorder_vec_dot_q_sycl<GGML_TYPE_Q4_K>>(
vx_base, vy, ids_dev, dst_base, ncols, nrows, n_experts_used,
+4 -3
View File
@@ -483,7 +483,8 @@ template <> struct reorder_vec_dot_q_sycl<GGML_TYPE_Q8_0> {
__dpct_inline__ float operator()(const void * __restrict__ vbq, const std::pair<int, int> ibx_offset,
const std::pair<int, int> d_offset, const int8_t * q8_1_quant_ptr,
const sycl::half2 * q8_1_ds, const int & iqs) {
const sycl::half2 * q8_1_ds, const int & iqs,
const int vdr = q8_0_traits::vdr_mmvq) {
const uint8_t * base = static_cast<const uint8_t *>(vbq);
const int8_t * qs = reinterpret_cast<const int8_t *>(base + ibx_offset.first);
const ggml_half d = *reinterpret_cast<const ggml_half *>(base + d_offset.first);
@@ -492,14 +493,14 @@ template <> struct reorder_vec_dot_q_sycl<GGML_TYPE_Q8_0> {
int u[q8_0_traits::vdr_mmvq];
#pragma unroll
for (size_t i = 0; i < q8_0_traits::vdr_mmvq; ++i) {
for (int i = 0; i < vdr; ++i) {
v[i] = get_int_from_int8(qs, iqs + i);
u[i] = get_int_from_int8_aligned(q8_1_quant_ptr, iqs + i);
}
int sumi = 0;
#pragma unroll
for (size_t i = 0; i < q8_0_traits::vdr_mmvq; ++i) {
for (int i = 0; i < vdr; ++i) {
sumi = dpct::dp4a(v[i], u[i], sumi);
}