sycl: support reordered Q8_0 MoE weights (#29337)
Preserve the existing MoE accumulation order. Assisted-by: Codex Assisted-by: Angel
This commit is contained in:
1 parent
43d57fb8ca
commit
e707790c73
3 files changed
+74
-11
No files matched your search
@@ -4442,6 +4442,47 @@ static bool reorder_qw_q8_0(uint8_t * data_device, const int ncols, const int nr
|
||||
return true;
|
||||
}
|
||||
|
||||
// Reorder each Q8_0 expert slice into a self-contained [qs][d] SoA layout.
|
||||
static bool reorder_qw_q8_0_moe(uint8_t * data_device, size_t expert_bytes, int64_t n_expert, dpct::queue_ptr stream) {
|
||||
GGML_ASSERT(expert_bytes % sizeof(block_q8_0) == 0);
|
||||
const int blocks_per_expert = (int) (expert_bytes / sizeof(block_q8_0));
|
||||
const size_t total_bytes = expert_bytes * (size_t) n_expert;
|
||||
|
||||
sycl_reorder_temp_buffer tmp(stream, total_bytes);
|
||||
if (!tmp) {
|
||||
GGML_LOG_WARN("%s: failed to allocate %zu bytes for reorder temp buffer, skipping reorder\n", __func__, total_bytes);
|
||||
return false;
|
||||
}
|
||||
uint8_t * tmp_buf = static_cast<uint8_t *>(tmp.ptr);
|
||||
|
||||
sycl::event copy_event;
|
||||
SYCL_CHECK(CHECK_TRY_ERROR(copy_event = stream->memcpy(tmp_buf, data_device, total_bytes)));
|
||||
if (!g_ggml_sycl_use_async_mem_op) {
|
||||
copy_event.wait();
|
||||
}
|
||||
|
||||
const int total_blocks = blocks_per_expert * (int) n_expert;
|
||||
auto reorder_event = stream->parallel_for(total_blocks, [=](auto gb_) {
|
||||
const int gb = gb_;
|
||||
const int e = gb / blocks_per_expert;
|
||||
const int ib = gb % blocks_per_expert;
|
||||
const block_q8_0 * x = (const block_q8_0 *) (tmp_buf + (size_t) e * expert_bytes);
|
||||
uint8_t * base = data_device + (size_t) e * expert_bytes;
|
||||
|
||||
auto * qs_ptr = base;
|
||||
auto * d_ptr = (sycl::half *) (qs_ptr + (size_t) blocks_per_expert * QK8_0);
|
||||
|
||||
for (int j = 0; j < QK8_0; ++j) {
|
||||
*((int8_t *) qs_ptr + ib * QK8_0 + j) = x[ib].qs[j];
|
||||
}
|
||||
d_ptr[ib] = x[ib].d;
|
||||
});
|
||||
if (!g_ggml_sycl_use_async_mem_op) {
|
||||
reorder_event.wait_and_throw();
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
static bool reorder_qw_q4_k(uint8_t * data_device, size_t size, size_t offset, dpct::queue_ptr stream) {
|
||||
GGML_ASSERT(size % sizeof(block_q4_K) == 0);
|
||||
GGML_ASSERT(offset % sizeof(block_q4_K) == 0);
|
||||
@@ -4966,6 +5007,8 @@ static bool reorder_qw(const ggml_tensor * src0, dpct::queue_ptr stream) {
|
||||
if (src0->ne[2] > 1) {
|
||||
GGML_ASSERT((size_t) size == (size_t) src0->ne[2] * src0->nb[2]);
|
||||
switch (src0->type) {
|
||||
case GGML_TYPE_Q8_0:
|
||||
return reorder_qw_q8_0_moe(data_device, src0->nb[2], src0->ne[2], stream);
|
||||
case GGML_TYPE_Q4_K:
|
||||
return reorder_qw_q4_k_moe(data_device, src0->nb[2], src0->ne[2], stream);
|
||||
case GGML_TYPE_Q5_K:
|
||||
@@ -5051,8 +5094,8 @@ static void opt_for_reorder_id(ggml_backend_sycl_context * ctx, const ggml_tenso
|
||||
if (!g_ggml_sycl_enable_optimize || !ctx->opt_feature.reorder) {
|
||||
return;
|
||||
}
|
||||
if (src0->type != GGML_TYPE_Q4_K && src0->type != GGML_TYPE_Q5_K && src0->type != GGML_TYPE_Q6_K &&
|
||||
src0->type != GGML_TYPE_MXFP4) {
|
||||
if (src0->type != GGML_TYPE_Q8_0 && src0->type != GGML_TYPE_Q4_K &&
|
||||
src0->type != GGML_TYPE_Q5_K && src0->type != GGML_TYPE_Q6_K && src0->type != GGML_TYPE_MXFP4) {
|
||||
return;
|
||||
}
|
||||
// The MXFP4 reorder kernels use 8-byte vector loads, so every expert slice must stay aligned.
|
||||
@@ -5438,7 +5481,7 @@ static bool ggml_sycl_mul_mat_id_mmvq_fused(
|
||||
const int n_experts_used = (int) n_ids_per_group;
|
||||
const int nrows = (int) src0->ne[1];
|
||||
|
||||
// Lazily reorder the (Q4_K) expert weights into a per-expert SoA layout, then run the reorder
|
||||
// Lazily reorder supported expert weights into a per-expert SoA layout, then run the reorder
|
||||
// GEMV. Placed after the bail checks so a non-dispatchable op does not pay the reorder cost.
|
||||
opt_for_reorder_id(&ctx, src0);
|
||||
const ggml_tensor_extra_gpu * src0_extra =
|
||||
|
||||
@@ -3202,9 +3202,11 @@ static void mul_mat_vec_q_moe_reorder(
|
||||
|
||||
const auto sg = item_ct1.get_sub_group();
|
||||
|
||||
constexpr int vdr_mmvq = reorder_vec_dot_q_sycl::gtype == GGML_TYPE_Q8_0 ?
|
||||
VDR_Q8_0_Q8_1_MMVQ : block_traits::vdr_mmvq;
|
||||
const int blocks_per_row = ncols / block_traits::qk;
|
||||
constexpr int blocks_per_subgroup = ceil_div(block_traits::vdr_mmvq * WARP_SIZE, block_traits::qi);
|
||||
constexpr int block_elements_per_subgroup = block_traits::qi / block_traits::vdr_mmvq;
|
||||
constexpr int blocks_per_subgroup = ceil_div(vdr_mmvq * WARP_SIZE, block_traits::qi);
|
||||
constexpr int block_elements_per_subgroup = block_traits::qi / vdr_mmvq;
|
||||
const int nblocks = nrows * (ncols / block_traits::qk);
|
||||
|
||||
static_assert(blocks_per_subgroup > 0);
|
||||
@@ -3223,12 +3225,24 @@ static void mul_mat_vec_q_moe_reorder(
|
||||
|
||||
#pragma unroll
|
||||
for (int elem = 0; elem < block_elements_per_subgroup; elem += WARP_SIZE) {
|
||||
const int iqs = elem + block_traits::vdr_mmvq * (sg.get_local_linear_id() % block_elements_per_subgroup);
|
||||
partial_sum += reorder_vec_dot_q_sycl()(vx, bx_offset, d_offset, q8_1_quant_ptr, q8_1_ds_ptr, iqs);
|
||||
const int iqs = elem + vdr_mmvq * (sg.get_local_linear_id() % block_elements_per_subgroup);
|
||||
if constexpr (reorder_vec_dot_q_sycl::gtype == GGML_TYPE_Q8_0) {
|
||||
partial_sum += reorder_vec_dot_q_sycl()(vx, bx_offset, d_offset, q8_1_quant_ptr, q8_1_ds_ptr, iqs, vdr_mmvq);
|
||||
} else {
|
||||
partial_sum += reorder_vec_dot_q_sycl()(vx, bx_offset, d_offset, q8_1_quant_ptr, q8_1_ds_ptr, iqs);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
auto sum = sycl::reduce_over_group(sg, partial_sum, std::plus<>());
|
||||
float sum = partial_sum;
|
||||
if constexpr (reorder_vec_dot_q_sycl::gtype == GGML_TYPE_Q8_0) {
|
||||
// Keep the accumulation order of the non-reordered MoE kernel.
|
||||
for (int mask = WARP_SIZE / 2; mask > 0; mask >>= 1) {
|
||||
sum += dpct::permute_sub_group_by_xor(sg, sum, mask);
|
||||
}
|
||||
} else {
|
||||
sum = sycl::reduce_over_group(sg, partial_sum, std::plus<>());
|
||||
}
|
||||
if (sg.leader()) {
|
||||
dst[row] = sum;
|
||||
}
|
||||
@@ -3269,6 +3283,11 @@ bool ggml_sycl_mul_mat_vec_q_id_reorder(
|
||||
size_t src1_row_stride,
|
||||
dpct::queue_ptr stream) {
|
||||
switch (src0_type) {
|
||||
case GGML_TYPE_Q8_0:
|
||||
launch_mul_mat_vec_q_moe_reorder<reorder_vec_dot_q_sycl<GGML_TYPE_Q8_0>>(
|
||||
vx_base, vy, ids_dev, dst_base, ncols, nrows, n_experts_used,
|
||||
expert_weight_stride, dst_row_stride, src1_row_stride, stream);
|
||||
return true;
|
||||
case GGML_TYPE_Q4_K:
|
||||
launch_mul_mat_vec_q_moe_reorder<reorder_vec_dot_q_sycl<GGML_TYPE_Q4_K>>(
|
||||
vx_base, vy, ids_dev, dst_base, ncols, nrows, n_experts_used,
|
||||
|
||||
@@ -483,7 +483,8 @@ template <> struct reorder_vec_dot_q_sycl<GGML_TYPE_Q8_0> {
|
||||
|
||||
__dpct_inline__ float operator()(const void * __restrict__ vbq, const std::pair<int, int> ibx_offset,
|
||||
const std::pair<int, int> d_offset, const int8_t * q8_1_quant_ptr,
|
||||
const sycl::half2 * q8_1_ds, const int & iqs) {
|
||||
const sycl::half2 * q8_1_ds, const int & iqs,
|
||||
const int vdr = q8_0_traits::vdr_mmvq) {
|
||||
const uint8_t * base = static_cast<const uint8_t *>(vbq);
|
||||
const int8_t * qs = reinterpret_cast<const int8_t *>(base + ibx_offset.first);
|
||||
const ggml_half d = *reinterpret_cast<const ggml_half *>(base + d_offset.first);
|
||||
@@ -492,14 +493,14 @@ template <> struct reorder_vec_dot_q_sycl<GGML_TYPE_Q8_0> {
|
||||
int u[q8_0_traits::vdr_mmvq];
|
||||
|
||||
#pragma unroll
|
||||
for (size_t i = 0; i < q8_0_traits::vdr_mmvq; ++i) {
|
||||
for (int i = 0; i < vdr; ++i) {
|
||||
v[i] = get_int_from_int8(qs, iqs + i);
|
||||
u[i] = get_int_from_int8_aligned(q8_1_quant_ptr, iqs + i);
|
||||
}
|
||||
|
||||
int sumi = 0;
|
||||
#pragma unroll
|
||||
for (size_t i = 0; i < q8_0_traits::vdr_mmvq; ++i) {
|
||||
for (int i = 0; i < vdr; ++i) {
|
||||
sumi = dpct::dp4a(v[i], u[i], sumi);
|
||||
}
|
||||
|
||||
|
||||
Reference in new issue
Block a user