diff --git a/docs/backend/SYCL.md b/docs/backend/SYCL.md
index 896ce6da1..8649fca0f 100644
--- a/docs/backend/SYCL.md
+++ b/docs/backend/SYCL.md
@@ -829,7 +829,8 @@ User can use the device management in [docs/multi-gpu.md](https://github.com/ggm
| GGML_SYCL_ENABLE_XMX_ESIMD | 0 or 1 (default) | Use the XMX (DPAS) multi-column mat-vec kernels for reordered Q2_K to Q6_K and Q8_0 weights on Xe2 GPUs. Speeds up speculative decoding and short prompt batches. |
| GGML_SYCL_XMX_MAX_COLS | non-negative integer, 80 (default) | Most activation columns (batch size) the `GGML_SYCL_ENABLE_XMX_ESIMD` kernels take; wider batches use the other mat-mul kernels. A larger value keeps XMX for bigger batches (as repeated launches), `0` never uses it. |
| GGML_SYCL_XMX_GLU_MAX_COLS | 0 to 16, 16 (default) | Most columns the fused gate and up XMX kernel takes; wider batches run gate and up as separate mat-vecs. 16 is the most the kernel is built for, larger values are clamped to 16 and `0` turns the fused kernel off. |
-| GGML_SYCL_XMX_GATHER_TYPES | decimal bitmask, all bits set (default) | Weight formats that may use the XMX dequant-GEMM paths, which dequantize weights straight into the XMX tiles. This speeds up prompt processing of MoE models on GPUs with XMX units (Arc A- and B-series, Arc Pro, Data Center GPU Max), for example pp512 of Qwen3-30B-A3B UD-IQ3_XXS by about 50% on an Arc Pro B60. Bits:
* 1: IQ4_NL, 2: IQ3_S, 4: IQ4_XS, 8: IQ3_XXS, 16: IQ2_XXS, 32: IQ2_XS, 64: IQ2_S, 128: IQ1_S, 256: IQ1_M
* 512: Q8_0, 1024: Q4_K, 2048: Q5_K, 4096: Q6_K (MoE `MUL_MAT_ID` only)
Add values to combine them, for example `3` for IQ4_NL and IQ3_S; `0` disables the paths. A set bit does not force the path: batches of more than 64 tokens per expert or row lengths that are not a multiple of 256 (32 for IQ4_NL and Q8_0) use the library GEMM. |
+| GGML_SYCL_XMX_GATHER_TYPES | decimal bitmask, all bits set (default) | Weight formats that may use the XMX dequant-GEMM paths, which dequantize weights straight into the XMX tiles. This speeds up prompt processing of MoE models on GPUs with XMX units (Arc A- and B-series, Arc Pro, Data Center GPU Max), for example pp512 of Qwen3-30B-A3B UD-IQ3_XXS by about 50% on an Arc Pro B60. Bits:
* 1: IQ4_NL, 2: IQ3_S, 4: IQ4_XS, 8: IQ3_XXS, 16: IQ2_XXS, 32: IQ2_XS, 64: IQ2_S, 128: IQ1_S, 256: IQ1_M
* 512: Q8_0, 1024: Q4_K, 2048: Q5_K, 4096: Q6_K, 8192: Q2_K, 16384: Q3_K (MoE `MUL_MAT_ID` only)
Add values to combine them, for example `3` for IQ4_NL and IQ3_S; `0` disables the paths. A set bit does not force the path: batches of more than 64 tokens per expert use the library GEMM unless the device schedule width heuristic is overridden with `GGML_SYCL_MMID_SCHED=3`. Row lengths must be a multiple of 256 (32 for IQ4_NL and Q8_0). |
+| GGML_SYCL_MMID_SCHED | decimal bitmask, 1 (default) | Build the grouped MoE routing and tile schedule on the GPU, avoiding the expert-ID host readback and CPU sort.
* 1: enable the device schedule.
* 2: override the fallback for models with few routed experts by disabling the performance heuristic that rejects batches averaging more than 64 routed rows per expert. This may reduce performance; type, shape, precision and device capability checks still apply.
Use `3` to enable the device schedule and override this heuristic, or `0` to use the host schedule. Bit 2 alone does not enable the device schedule. Higher bits are ignored. |
| GGML_SYCL_XMX_GATHER_SHAPES | decimal bitmask, 255 (default) | XMX `joint_matrix` combinations the paths of `GGML_SYCL_XMX_GATHER_TYPES` may use; the operand type comes from `GGML_SYCL_DYNAMIC_PRECISION` and the best supported combination is picked automatically (logged as `fg_pick_combo`). Bits:
* Xe2, Xe3, Xe-HPC: 1: f16 8x16x16, 2: f16 16x16x16, 4: f16 32x64x16, 8: f16 32x64x32, 32: tf32 8x16x8, 64: bf16 8x16x16
* Xe-HPG (Arc A770, ARL-H): 16: f16 8x8x16, 128: bf16 8x8x16
Clear a bit to exclude a combination, or set a single bit to force one for testing. |
| GGML_SYCL_DYNAMIC_PRECISION | `F16` (default with `GGML_SYCL_F16=ON`), `BF16`, `TF32` or `F32` (default otherwise) | Operand type of the XMX dequant-GEMM paths (`GGML_SYCL_XMX_GATHER_TYPES`); accumulation is always f32. `F16` is the fastest, but activations above 65504 overflow. `BF16` keeps the f32 range at a 7-bit mantissa, `TF32` keeps the range and the f16 mantissa but is about 30% slower and needs Xe2, Xe3 or Xe-HPC, and `F32` turns the XMX paths off. Ops that request a higher src1 precision ([TAG_GGML_PREC]) get it regardless of this setting. |
| GGML_SYCL_DYNAMIC_REQUIRED_PRECISION | `F32` (default), `TF32`, `BF16` or `F16` | Lowest type the XMX paths may use for an op that requests an F32 src1, such as Mistral 4 `ffn_down_exps`. The default runs such ops on the library f32 GEMM; `TF32` or `BF16` trade mantissa for speed while keeping the f32 range. `F16` ignores the request and can overflow; it is meant for testing only. |
diff --git a/ggml/src/ggml-sycl/common.hpp b/ggml/src/ggml-sycl/common.hpp
index 5d1b1c499..a572dd7fe 100644
--- a/ggml/src/ggml-sycl/common.hpp
+++ b/ggml/src/ggml-sycl/common.hpp
@@ -92,9 +92,17 @@ enum ggml_sycl_xmx_gather_type {
GGML_SYCL_XMX_GATHER_Q4_K = 1 << 10,
GGML_SYCL_XMX_GATHER_Q5_K = 1 << 11,
GGML_SYCL_XMX_GATHER_Q6_K = 1 << 12,
+ GGML_SYCL_XMX_GATHER_Q2_K = 1 << 13,
+ GGML_SYCL_XMX_GATHER_Q3_K = 1 << 14,
};
static constexpr int GGML_SYCL_XMX_GATHER_TYPES_DEFAULT = ~0;
extern int g_ggml_sycl_xmx_gather_types;
+
+enum ggml_sycl_mmid_sched_bit {
+ GGML_SYCL_MMID_SCHED_DEVICE = 1 << 0,
+ GGML_SYCL_MMID_SCHED_UNBOUNDED = 1 << 1,
+};
+extern int g_ggml_sycl_mmid_sched;
// Which joint_matrix combinations the XMX dequant-GEMM paths may use, one bit each (see fused-gemm.cpp).
// GGML_SYCL_DYNAMIC_PRECISION picks the operand type, this mask the combinations of that type.
static constexpr int GGML_SYCL_XMX_GATHER_SHAPES_DEFAULT = 0xff;
@@ -443,7 +451,7 @@ struct mmid_row_mapping {
int32_t i2;
};
-struct ggml_sycl_gg_tile {
+struct ggml_sycl_grouped_gemm_tile {
int32_t expert;
int32_t n0;
int32_t n1;
@@ -552,7 +560,7 @@ struct ggml_backend_sycl_context {
std::unique_ptr host_pools[GGML_SYCL_MAX_DEVICES];
std::vector mmid_row_mapping_host;
- std::vector mmid_tile_schedule_host;
+ std::vector mmid_tile_schedule_host;
static std::unique_ptr new_pool_for_device(queue_ptr qptr, int device);
diff --git a/ggml/src/ggml-sycl/fused-gemm.cpp b/ggml/src/ggml-sycl/fused-gemm.cpp
index 3024f139d..803a994c0 100644
--- a/ggml/src/ggml-sycl/fused-gemm.cpp
+++ b/ggml/src/ggml-sycl/fused-gemm.cpp
@@ -411,6 +411,8 @@ template <> struct fg_block_traits { static constexpr int qk = QK
template <> struct fg_block_traits { static constexpr int qk = QK_K; };
template <> struct fg_block_traits { static constexpr int qk = QK_K; };
template <> struct fg_block_traits { static constexpr int qk = QK8_0; };
+template <> struct fg_block_traits { static constexpr int qk = QK_K; };
+template <> struct fg_block_traits { static constexpr int qk = QK_K; };
template <> struct fg_block_traits { static constexpr int qk = QK_K; };
template <> struct fg_block_traits { static constexpr int qk = QK_K; };
template <> struct fg_block_traits { static constexpr int qk = QK_K; };
@@ -595,6 +597,49 @@ static __dpct_inline__ void fg_scale_min_k4(const int j, const uint8_t * __restr
}
}
+template
+static __dpct_inline__ void fg_decode_q2_K(const uint8_t * __restrict__ qs, const uint8_t * __restrict__ scales,
+ const float d, const float dmin, const int ib, typename E::pair * a) {
+ const uint8_t * q = qs + 32 * (ib / 4);
+ const int shift = 2 * (ib % 4);
+#pragma unroll
+ for (int j = 0; j < 16; ++j) {
+ const uint8_t sc = scales[2 * ib + j / 8];
+ const float dl = d * (sc & 0xF);
+ const float ml = dmin * (sc >> 4);
+ a[j] = E::make(dl * ((q[2 * j] >> shift) & 3) - ml,
+ dl * ((q[2 * j + 1] >> shift) & 3) - ml);
+ }
+}
+
+// scales of the two 16-value halves of sub-block ib
+static __dpct_inline__ void fg_q3_K_dl(const uint8_t * __restrict__ scales, const float d, const int ib, float * dl) {
+#pragma unroll
+ for (int i = 0; i < 2; ++i) {
+ const int s = 2 * ib + i;
+ const int sc = ((scales[s % 8] >> (4 * (s / 8))) & 0xF) | (((scales[8 + s % 4] >> (2 * (s / 4))) & 3) << 4);
+ dl[i] = d * (sc - 32);
+ }
+}
+
+template
+static __dpct_inline__ void fg_decode_q3_K(const uint8_t * __restrict__ qs, const uint8_t * __restrict__ hmask,
+ const uint8_t * __restrict__ scales, const float d, const int ib,
+ typename E::pair * a) {
+ // block_q3_K is 110 bytes, so qs and hmask are only 2-byte aligned
+ const uint16_t * q = (const uint16_t *) (qs + 32 * (ib / 4));
+ const uint16_t * h = (const uint16_t *) hmask;
+ const int shift = 2 * (ib % 4);
+ float dl[2];
+ fg_q3_K_dl(scales, d, ib, dl);
+#pragma unroll
+ for (int j = 0; j < 16; ++j) {
+ // per byte: 2 low bits, plus 4 if the hmask bit is set, minus 4
+ const uint32_t v = ((q[j] >> shift) & 0x0303u) | (((h[j] >> ib) & 0x0101u) << 2);
+ a[j] = E::make(dl[j / 8] * ((int) (v & 0xFF) - 4), dl[j / 8] * ((int) (v >> 8) - 4));
+ }
+}
+
// q4_K sub-block ib (0..7) uses scale/min pair ib and the low (even ib) or high (odd ib) nibbles of
// qs[32 * (ib / 2) ...], as in dequantize_row_q4_K
template
@@ -658,6 +703,18 @@ static __dpct_inline__ void fg_stage_a(const block_q8_0 * __restrict__ xrow, con
fg_decode_q8_0(xrow[kb].qs, (float) xrow[kb].d, a);
}
+template
+static __dpct_inline__ void fg_stage_a(const block_q2_K * __restrict__ xrow, const int kb, typename E::pair * a) {
+ FG_SUPERBLOCK(block_q2_K);
+ fg_decode_q2_K(blk->qs, blk->scales, (float) blk->dm[0], (float) blk->dm[1], ib, a);
+}
+
+template
+static __dpct_inline__ void fg_stage_a(const block_q3_K * __restrict__ xrow, const int kb, typename E::pair * a) {
+ FG_SUPERBLOCK(block_q3_K);
+ fg_decode_q3_K(blk->qs, blk->hmask, blk->scales, (float) blk->d, ib, a);
+}
+
template
static __dpct_inline__ void fg_stage_a(const block_q4_K * __restrict__ xrow, const int kb, typename E::pair * a) {
FG_SUPERBLOCK(block_q4_K);
@@ -693,6 +750,50 @@ template <> struct fg_soa {
}
};
+template <> struct fg_soa {
+ static constexpr bool supported = true;
+
+ // [qs][scales][dm]
+ template
+ static __dpct_inline__ void stage(const uint8_t * x, const size_t nblocks, const size_t ib, const int kb,
+ typename E::pair * a) {
+ const uint8_t * scales = x + nblocks * (QK_K / 4);
+ const sycl::half2 dm = ((const sycl::half2 *) (scales + nblocks * (QK_K / 16)))[ib];
+ fg_decode_q2_K(x + ib * (QK_K / 4), scales + ib * (QK_K / 16), (float) dm[0], (float) dm[1],
+ kb % (QK_K / 32), a);
+ }
+};
+
+template <> struct fg_soa {
+ static constexpr bool supported = true;
+
+ // [qs][hmask][scales][d]
+ template
+ static __dpct_inline__ void stage(const uint8_t * x, const size_t nblocks, const size_t ib, const int kb,
+ typename E::pair * a) {
+ const uint8_t * hmask = x + nblocks * (QK_K / 4);
+ const uint8_t * scales = hmask + nblocks * (QK_K / 8);
+ const float d = (float) ((const sycl::half *) (scales + nblocks * 12))[ib];
+ const int sub = kb % (QK_K / 32);
+ // 16-byte aligned, see opt_for_reorder_id
+ using v4 = sycl::vec;
+ const v4 * q = (const v4 *) (x + ib * (QK_K / 4) + 32 * (sub / 4));
+ const v4 * h = (const v4 *) (hmask + ib * (QK_K / 8));
+ const v4 qv[2] = { q[0], q[1] };
+ const v4 hv[2] = { h[0], h[1] };
+ const int shift = 2 * (sub % 4);
+ float dl[2];
+ fg_q3_K_dl(scales + ib * 12, d, sub, dl);
+#pragma unroll
+ for (int w = 0; w < 8; ++w) {
+ const uint32_t v = ((qv[w / 4][w % 4] >> shift) & 0x03030303u) | (((hv[w / 4][w % 4] >> sub) & 0x01010101u) << 2);
+ const float f = dl[w / 4];
+ a[2 * w] = E::make(f * ((int) (v & 0xFF) - 4), f * ((int) ((v >> 8) & 0xFF) - 4));
+ a[2 * w + 1] = E::make(f * ((int) ((v >> 16) & 0xFF) - 4), f * ((int) (v >> 24) - 4));
+ }
+ }
+};
+
template <> struct fg_soa {
static constexpr bool supported = true;
// [qs][scales][dm]
@@ -745,7 +846,7 @@ static void fused_dequant_gemm_tile(
const int b0, const int n0, const int n1,
sycl::local_accessor tile_a,
sycl::local_accessor tile_c,
- const sycl::nd_item<2> & item) {
+ const sycl::nd_item<2> & item, ggml_sycl_grouped_gemm_rows dst_rows = {}) {
if constexpr (fg_built()) {
using EA = typename S::EA;
using TA = typename S::ta;
@@ -854,7 +955,13 @@ static void fused_dequant_gemm_tile(
for (int s = 0; s < FG_KSPLIT; ++s) {
sum += tile_c[s * S::SG_ROWS * S::BN + r * S::BN + c];
}
- dst[(size_t) n * ldd + m] = sum;
+ if (dst_rows.map) {
+ const mmid_row_mapping row = dst_rows.map[n];
+ float * out = (float *) (dst_rows.base + (row.i1 % dst_rows.ne1) * dst_rows.nb1 + row.i2 * dst_rows.nb2);
+ out[m] = sum;
+ } else {
+ dst[(size_t) n * ldd + m] = sum;
+ }
}
}
}
@@ -880,20 +987,24 @@ static void fused_dequant_gemm_launch(const void * src0, const typename S::tsb *
// grouped: work-group (t, mt) is tile t of the schedule; its B columns sit at t * BN
template
static void grouped_dequant_gemm_launch(const char * src0_dd, const size_t expert_stride,
- const ggml_sycl_gg_tile * tiles_ptr, const typename S::tsb * packed, float * dst,
- const int M, const int Npad, const int K, const int64_t n_tiles,
- const int64_t groups_m, dpct::queue_ptr stream) {
+ const ggml_sycl_grouped_gemm_tile * tiles_ptr, const typename S::tsb * packed,
+ float * dst, const int M, const int Npad, const int K, const int64_t n_tiles,
+ const int64_t groups_m, dpct::queue_ptr stream,
+ ggml_sycl_grouped_gemm_rows dst_rows = {}) {
stream->submit([&](sycl::handler & cgh) {
sycl::local_accessor tile_a(FG_KSPLIT * S::SG_ROWS * FG_BK, cgh);
sycl::local_accessor tile_c(FG_KSPLIT * S::SG_ROWS * S::BN, cgh);
cgh.parallel_for(
sycl::nd_range<2>(sycl::range<2>(n_tiles, groups_m * S::WG_SIZE), sycl::range<2>(1, S::WG_SIZE)),
[=](sycl::nd_item<2> item) [[sycl::reqd_sub_group_size(S::SG)]] {
- const int t = item.get_group(0);
- const ggml_sycl_gg_tile tile = tiles_ptr[t];
+ const int t = item.get_group(0);
+ const ggml_sycl_grouped_gemm_tile tile = tiles_ptr[t];
+ if (tile.n0 == tile.n1) {
+ return;
+ }
const block_q_t * x = (const block_q_t *) (src0_dd + (size_t) tile.expert * expert_stride);
fused_dequant_gemm_tile(x, packed, dst, M, Npad, K, M, t * S::BN, tile.n0,
- tile.n1, tile_a, tile_c, item);
+ tile.n1, tile_a, tile_c, item, dst_rows);
});
});
}
@@ -901,17 +1012,17 @@ static void grouped_dequant_gemm_launch(const char * src0_dd, const size_t exper
// src1 f32 rows -> packed [K/V][n_tiles*BN][V], tile t holds its rows [n0, n1) at columns t*BN..,
// zero past n1. The column runs fastest so a sub-group writes one contiguous run.
template
-static void grouped_gemm_pack_b(const float * y, typename S::tsb * packed, const ggml_sycl_gg_tile * tiles, int Npad,
- int K, dpct::queue_ptr stream) {
+static void grouped_gemm_pack_b(const float * y, typename S::tsb * packed, const ggml_sycl_grouped_gemm_tile * tiles,
+ int Npad, int K, dpct::queue_ptr stream) {
using E = typename S::EB;
constexpr int V = S::VNNI;
const int kqs = K / V;
stream->parallel_for(sycl::range<1>((size_t) Npad * kqs), [=](sycl::id<1> id) {
- const size_t idx = id[0];
- const int kq = idx / Npad;
- const int n = idx - (size_t) kq * Npad;
- const ggml_sycl_gg_tile tile = tiles[n / S::BN];
- const int row = tile.n0 + n % S::BN;
+ const size_t idx = id[0];
+ const int kq = idx / Npad;
+ const int n = idx - (size_t) kq * Npad;
+ const ggml_sycl_grouped_gemm_tile tile = tiles[n / S::BN];
+ const int row = tile.n0 + n % S::BN;
// one guarded load run per work-item, as a per-element select costs ~1.5% prefill
typename S::tsb vals[V] = {};
if (row < tile.n1) {
@@ -932,7 +1043,8 @@ static void grouped_gemm_pack_b(const float * y, typename S::tsb * packed, const
// q8_0 and the k-quants take only the grouped path. The plain kernel decodes A again for every BN columns
// of a dense batch, and for these formats that costs more than the one dequantization of the library GEMM.
template static constexpr bool fg_plain_ok() {
- return !std::is_same_v && !std::is_same_v && !std::is_same_v &&
+ return !std::is_same_v && !std::is_same_v && !std::is_same_v &&
+ !std::is_same_v && !std::is_same_v &&
!std::is_same_v;
}
@@ -966,6 +1078,8 @@ template static bool fg_visit_type(ggml_type type, bool reordered,
case GGML_TYPE_IQ1_S: return fg_visit_layout(reordered, f);
case GGML_TYPE_IQ1_M: return fg_visit_layout(reordered, f);
case GGML_TYPE_Q8_0: return fg_visit_layout(reordered, f);
+ case GGML_TYPE_Q2_K: return fg_visit_layout(reordered, f);
+ case GGML_TYPE_Q3_K: return fg_visit_layout(reordered, f);
case GGML_TYPE_Q4_K: return fg_visit_layout(reordered, f);
case GGML_TYPE_Q5_K: return fg_visit_layout(reordered, f);
case GGML_TYPE_Q6_K: return fg_visit_layout(reordered, f);
@@ -1037,7 +1151,7 @@ bool ggml_sycl_fused_dequant_gemm(ggml_type src0_type, bool src0_reordered, cons
template
static bool fg_grouped_run(ggml_type src0_type, bool reordered, const void * src0_base, size_t expert_stride,
const float * src1, float * dst, const int64_t * expert_row_offsets, int64_t n_as, int64_t M,
- int64_t K, std::vector & tiles, ggml_sycl_pool & pool,
+ int64_t K, std::vector & tiles, ggml_sycl_pool & pool,
dpct::queue_ptr stream) {
// the host knows every slice, so it lays out the work-groups: no search on the device
tiles.clear();
@@ -1053,15 +1167,16 @@ static bool fg_grouped_run(ggml_type src0_type, bool reordered, const void * src
const int64_t max_tiles = grouped_gemm_max_tiles(expert_row_offsets[n_as], n_as, S::BN);
GGML_ASSERT(n_tiles <= max_tiles);
- ggml_sycl_pool_alloc tiles_dev(pool, max_tiles);
- SYCL_CHECK(CHECK_TRY_ERROR(stream->memcpy(tiles_dev.get(), tiles.data(), n_tiles * sizeof(ggml_sycl_gg_tile))));
+ ggml_sycl_pool_alloc tiles_dev(pool, max_tiles);
+ SYCL_CHECK(CHECK_TRY_ERROR(
+ stream->memcpy(tiles_dev.get(), tiles.data(), n_tiles * sizeof(ggml_sycl_grouped_gemm_tile))));
ggml_sycl_pool_alloc packed_b(pool, (size_t) K * max_tiles * S::BN);
grouped_gemm_pack_b(src1, packed_b.get(), tiles_dev.get(), Npad, (int) K, stream);
- const typename S::tsb * packed = packed_b.get();
- const ggml_sycl_gg_tile * tiles_ptr = tiles_dev.get();
- const char * src0_dd = (const char *) src0_base;
+ const typename S::tsb * packed = packed_b.get();
+ const ggml_sycl_grouped_gemm_tile * tiles_ptr = tiles_dev.get();
+ const char * src0_dd = (const char *) src0_base;
return fg_visit_type(src0_type, reordered, [&](auto tag) {
using T = decltype(tag);
grouped_dequant_gemm_launch(src0_dd, expert_stride, tiles_ptr, packed, dst,
@@ -1073,7 +1188,7 @@ static bool fg_grouped_run(ggml_type src0_type, bool reordered, const void * src
bool ggml_sycl_grouped_dequant_gemm(ggml_type src0_type, bool reordered, const void * src0_base, size_t expert_stride,
const float * src1, int32_t src1_prec, float * dst,
const int64_t * expert_row_offsets, int64_t n_as, int64_t M, int64_t K,
- int64_t total_rows, std::vector & tiles,
+ int64_t total_rows, std::vector & tiles,
ggml_sycl_pool & pool, dpct::queue_ptr stream) {
int64_t n_active = 0;
for (int64_t e = 0; e < n_as; ++e) {
@@ -1097,3 +1212,227 @@ bool ggml_sycl_grouped_dequant_gemm(ggml_type src0_type, bool reordered, const v
});
return launched;
}
+
+// One work-group sorts routes by expert and builds the tile table.
+static constexpr int MMID_SCHED_WG = 256;
+
+using mmid_sched_atomic = sycl::atomic_ref;
+
+bool ggml_sycl_build_mmid_schedule(const int32_t * ids_dev, size_t ids_token_stride, int64_t n_as, int64_t n_ids,
+ int64_t n_tokens, int64_t n_tiles_max, mmid_row_mapping * row_mapping,
+ ggml_sycl_grouped_gemm_tile * tiles, uint32_t * expert_offsets,
+ dpct::queue_ptr stream) {
+ if (n_as <= 0 || n_as > GGML_SYCL_MMID_SCHED_MAX_EXPERTS || n_ids <= 0 || n_tokens <= 0 || n_ids > INT32_MAX / n_tokens) {
+ return false;
+ }
+ const int64_t n_routes = n_tokens * n_ids;
+ if (n_tiles_max < grouped_gemm_max_tiles(n_routes, n_as, GGML_SYCL_FG_BN)) {
+ return false;
+ }
+
+ stream->submit([&](sycl::handler & cgh) {
+ // l_cnt is reused: route counts, then the write cursors, then the tile count per expert
+ sycl::local_accessor l_cnt(n_as + 1, cgh);
+ sycl::local_accessor l_off(n_as + 1, cgh);
+ sycl::local_accessor l_toff(n_as + 1, cgh);
+ cgh.parallel_for(
+ sycl::nd_range<1>(MMID_SCHED_WG, MMID_SCHED_WG),
+ [=](sycl::nd_item<1> item) {
+ const auto grp = item.get_group();
+ const int64_t tid = item.get_local_id(0);
+ uint32_t * cnt = l_cnt.get_multi_ptr().get();
+ uint32_t * off = l_off.get_multi_ptr().get();
+ uint32_t * toff = l_toff.get_multi_ptr().get();
+
+ for (int64_t e = tid; e < n_as; e += MMID_SCHED_WG) {
+ cnt[e] = 0;
+ }
+ // a dropped route must leave a row that names (0, 0) rather than stale memory
+ for (int64_t r = tid; r < n_routes; r += MMID_SCHED_WG) {
+ row_mapping[r] = { 0, 0 };
+ }
+ sycl::group_barrier(grp);
+
+ for (int64_t r = tid; r < n_routes; r += MMID_SCHED_WG) {
+ const int64_t token = r / n_ids;
+ const int64_t slot = r - token * n_ids;
+ const int32_t * id_row = (const int32_t *) ((const char *) ids_dev + token * ids_token_stride);
+ const int32_t e = id_row[slot];
+ if (e >= 0 && e < n_as) {
+ mmid_sched_atomic(cnt[e]).fetch_add(1u);
+ }
+ }
+ sycl::group_barrier(grp);
+
+ sycl::joint_exclusive_scan(grp, cnt, cnt + n_as, off, 0u, sycl::plus());
+ sycl::group_barrier(grp);
+ if (tid == 0) {
+ off[n_as] = off[n_as - 1] + cnt[n_as - 1];
+ }
+ sycl::group_barrier(grp);
+
+ for (int64_t e = tid; e < n_as; e += MMID_SCHED_WG) {
+ cnt[e] = off[e];
+ }
+ sycl::group_barrier(grp);
+
+ // Atomic arrival order can differ from the host sort; each expert gets the same rows.
+ for (int64_t r = tid; r < n_routes; r += MMID_SCHED_WG) {
+ const int64_t token = r / n_ids;
+ const int64_t slot = r - token * n_ids;
+ const int32_t * id_row = (const int32_t *) ((const char *) ids_dev + token * ids_token_stride);
+ const int32_t e = id_row[slot];
+ if (e >= 0 && e < n_as) {
+ const uint32_t pos = mmid_sched_atomic(cnt[e]).fetch_add(1u);
+ row_mapping[pos] = { (int32_t) slot, (int32_t) token };
+ }
+ }
+ sycl::group_barrier(grp);
+
+ for (int64_t e = tid; e < n_as; e += MMID_SCHED_WG) {
+ cnt[e] = (off[e + 1] - off[e] + GGML_SYCL_FG_BN - 1) / GGML_SYCL_FG_BN;
+ }
+ sycl::group_barrier(grp);
+
+ sycl::joint_exclusive_scan(grp, cnt, cnt + n_as, toff, 0u, sycl::plus());
+ sycl::group_barrier(grp);
+ if (tid == 0) {
+ toff[n_as] = toff[n_as - 1] + cnt[n_as - 1];
+ }
+ sycl::group_barrier(grp);
+
+ const uint32_t n_tiles = toff[n_as];
+ for (int64_t e = tid; e < n_as; e += MMID_SCHED_WG) {
+ const uint32_t base = off[e];
+ const uint32_t end = off[e + 1];
+ for (uint32_t j = 0; j < cnt[e]; ++j) {
+ const uint32_t n0 = base + j * GGML_SYCL_FG_BN;
+ const uint32_t n1 = n0 + GGML_SYCL_FG_BN < end ? n0 + GGML_SYCL_FG_BN : end;
+ tiles[toff[e] + j] = { (int32_t) e, (int32_t) n0, (int32_t) n1 };
+ }
+ }
+ // the launch is bounded, so the unused tail must be tiles the GEMM skips
+ for (int64_t t = n_tiles + tid; t < n_tiles_max; t += MMID_SCHED_WG) {
+ tiles[t] = { 0, 0, 0 };
+ }
+ for (int64_t e = tid; e <= n_as; e += MMID_SCHED_WG) {
+ expert_offsets[e] = off[e];
+ }
+ });
+ });
+ return true;
+}
+
+template
+static bool fg_grouped_device_run(ggml_type src0_type, bool reordered, const void * src0_base, size_t expert_stride,
+ ggml_sycl_grouped_gemm_rows src1, ggml_sycl_grouped_gemm_rows dst,
+ const int32_t * ids_dev, size_t ids_token_stride, int64_t n_as, int64_t n_ids,
+ int64_t n_tokens, int64_t M, int64_t K, ggml_sycl_pool & pool,
+ dpct::queue_ptr stream) {
+ if constexpr (S::BN != GGML_SYCL_FG_BN) {
+ return false;
+ } else {
+ const int64_t total_rows = n_tokens * n_ids;
+ const int64_t max_tiles = grouped_gemm_max_tiles(total_rows, n_as, S::BN);
+ if (max_tiles > INT32_MAX / S::BN) {
+ return false;
+ }
+ const int Npad = (int) (max_tiles * S::BN);
+ ggml_sycl_pool_alloc rows(pool, total_rows);
+ ggml_sycl_pool_alloc tiles(pool, max_tiles);
+ ggml_sycl_pool_alloc offsets(pool, n_as + 1);
+ if (!ggml_sycl_build_mmid_schedule(ids_dev, ids_token_stride, n_as, n_ids, n_tokens, max_tiles,
+ rows.get(), tiles.get(), offsets.get(), stream)) {
+ return false;
+ }
+ src1.map = rows.get();
+ dst.map = rows.get();
+ ggml_sycl_pool_alloc packed_b(pool, (size_t) K * Npad);
+ const ggml_sycl_grouped_gemm_tile * tile_ptr = tiles.get();
+ typename S::tsb * packed = packed_b.get();
+ using E = typename S::EB;
+ constexpr int V = S::VNNI;
+ stream->parallel_for(sycl::range<1>((size_t) Npad * (K / V)), [=](sycl::id<1> id) {
+ const size_t idx = id[0];
+ const int kq = idx / Npad;
+ const int n = idx - (size_t) kq * Npad;
+ const ggml_sycl_grouped_gemm_tile tile = tile_ptr[n / S::BN];
+ const int row = tile.n0 + n % S::BN;
+ typename S::tsb vals[V] = {};
+ if (row < tile.n1) {
+ const mmid_row_mapping mapping = src1.map[row];
+ const float * in = (const float *) (src1.base + (mapping.i1 % src1.ne1) * src1.nb1 + mapping.i2 * src1.nb2) + V * kq;
+#pragma unroll
+ for (int v = 0; v < V; ++v) {
+ vals[v] = E::cvt(in[v]);
+ }
+ }
+ typename S::tsb * out = packed + ((size_t) kq * Npad + n) * V;
+#pragma unroll
+ for (int v = 0; v < V; ++v) {
+ out[v] = vals[v];
+ }
+ });
+ const int64_t groups_m = (M + S::SG_ROWS - 1) / S::SG_ROWS;
+ return fg_visit_type(src0_type, reordered, [&](auto tag) {
+ using T = decltype(tag);
+ grouped_dequant_gemm_launch((const char *) src0_base, expert_stride,
+ tile_ptr, packed, (float *) dst.base, (int) M, Npad, (int) K, max_tiles, groups_m, stream, dst);
+ });
+ }
+}
+
+bool ggml_sycl_grouped_dequant_gemm_device_supported(ggml_type src0_type, bool reordered, int32_t src1_prec,
+ int64_t n_as, int64_t n_ids, int64_t n_tokens,
+ int64_t M, int64_t K, dpct::queue_ptr stream) {
+ if (n_as <= 0 || n_as > GGML_SYCL_MMID_SCHED_MAX_EXPERTS || n_ids <= 0 || n_tokens <= 0 ||
+ n_ids > INT32_MAX / n_tokens) {
+ return false;
+ }
+ const int64_t total_rows = n_ids * n_tokens;
+ // The active expert count is unknown until the GPU builds the schedule.
+ if (g_ggml_sycl_dynamic_precision == GGML_SYCL_DYNAMIC_PRECISION_F32 ||
+ !ggml_sycl_xmx_gather_type_enabled(src0_type) || !fg_visit_type(src0_type, reordered, [](auto) {}) ||
+ !ggml_sycl_grouped_dequant_gemm_shape_ok(src0_type, M, K, total_rows, std::min(n_as, total_rows),
+ (g_ggml_sycl_mmid_sched & GGML_SYCL_MMID_SCHED_UNBOUNDED) != 0)) {
+ return false;
+ }
+ const sycl::device device = stream->get_device();
+ if (device.get_info() < MMID_SCHED_WG ||
+ device.get_info() < 3 * (n_as + 1) * sizeof(uint32_t)) {
+ return false;
+ }
+ if (grouped_gemm_max_tiles(total_rows, n_as, GGML_SYCL_FG_BN) > INT32_MAX / GGML_SYCL_FG_BN) {
+ return false;
+ }
+ const int combo = fg_pick_combo(stream, GGML_TYPE_F32, src1_prec);
+ if (combo < 0) {
+ return false;
+ }
+ // the device schedule is built for GGML_SYCL_FG_BN-wide tiles, see fg_grouped_device_run()
+ bool bn_ok = false;
+ fg_visit_combo(combo, [&](auto s) { bn_ok = decltype(s)::BN == GGML_SYCL_FG_BN; });
+ return bn_ok;
+}
+
+bool ggml_sycl_grouped_dequant_gemm_device(ggml_type src0_type, bool reordered, const void * src0_base,
+ size_t expert_stride, const ggml_sycl_grouped_gemm_rows & src1,
+ int32_t src1_prec, const ggml_sycl_grouped_gemm_rows & dst,
+ const int32_t * ids_dev, size_t ids_token_stride, int64_t n_as,
+ int64_t n_ids, int64_t n_tokens, int64_t M, int64_t K, ggml_sycl_pool & pool,
+ dpct::queue_ptr stream) {
+ if (src1.ne1 <= 0 || dst.ne1 <= 0 ||
+ !ggml_sycl_grouped_dequant_gemm_device_supported(src0_type, reordered, src1_prec, n_as, n_ids,
+ n_tokens, M, K, stream)) {
+ return false;
+ }
+ const int combo = fg_pick_combo(stream, GGML_TYPE_F32, src1_prec);
+ bool launched = false;
+ fg_visit_combo(combo, [&](auto s) {
+ launched = fg_grouped_device_run(src0_type, reordered, src0_base, expert_stride, src1, dst,
+ ids_dev, ids_token_stride, n_as, n_ids, n_tokens, M, K, pool, stream);
+ });
+ return launched;
+}
diff --git a/ggml/src/ggml-sycl/fused-gemm.hpp b/ggml/src/ggml-sycl/fused-gemm.hpp
index 226676d12..0e807a2c0 100644
--- a/ggml/src/ggml-sycl/fused-gemm.hpp
+++ b/ggml/src/ggml-sycl/fused-gemm.hpp
@@ -4,6 +4,18 @@
#include "common.hpp"
+// Routed row n is at base + (map[n].i1 % ne1)*nb1 + map[n].i2*nb2.
+struct ggml_sycl_grouped_gemm_rows {
+ char * base;
+ const mmid_row_mapping * map;
+ int64_t ne1;
+ size_t nb1;
+ size_t nb2;
+};
+
+static constexpr int GGML_SYCL_FG_BN = 32;
+static constexpr int64_t GGML_SYCL_MMID_SCHED_MAX_EXPERTS = 4096;
+
// Shape and type gates for the kernels below. Device capability is separate: it needs a queue to ask.
static constexpr int GGML_SYCL_FG_MAX_N = 64; // widest N taken; each shape covers it in BN-wide tiles
@@ -15,6 +27,8 @@ constexpr bool ggml_sycl_fused_dequant_gemm_type_ok(ggml_type src0_type, int64_t
return K % 32 == 0;
}
const bool superblock =
+ src0_type == GGML_TYPE_Q2_K ||
+ src0_type == GGML_TYPE_Q3_K ||
src0_type == GGML_TYPE_Q4_K ||
src0_type == GGML_TYPE_Q5_K ||
src0_type == GGML_TYPE_Q6_K ||
@@ -39,9 +53,9 @@ constexpr bool ggml_sycl_fused_dequant_gemm_shape_ok(ggml_type src0_type, int64_
// grouped variant: the per-expert fused kernel is only worth it while each expert is narrow,
// so wider average slices are left to the per-expert library GEMM loop
constexpr bool ggml_sycl_grouped_dequant_gemm_shape_ok(ggml_type src0_type, int64_t M, int64_t K,
- int64_t total_rows, int64_t n_active) {
+ int64_t total_rows, int64_t n_active, bool unbounded = false) {
return ggml_sycl_fused_dequant_gemm_shape_ok(src0_type, M, 1, K, M) && total_rows > 0 &&
- total_rows <= INT32_MAX && total_rows <= n_active * GGML_SYCL_FG_MAX_N;
+ total_rows <= INT32_MAX && (unbounded || total_rows <= n_active * GGML_SYCL_FG_MAX_N);
}
// Runtime type gate, kept out of the constexpr predicates above so those stay pure.
@@ -57,6 +71,8 @@ inline bool ggml_sycl_xmx_gather_type_enabled(ggml_type src0_type) {
case GGML_TYPE_IQ1_S: return (g_ggml_sycl_xmx_gather_types & GGML_SYCL_XMX_GATHER_IQ1_S ) != 0;
case GGML_TYPE_IQ1_M: return (g_ggml_sycl_xmx_gather_types & GGML_SYCL_XMX_GATHER_IQ1_M ) != 0;
case GGML_TYPE_Q8_0: return (g_ggml_sycl_xmx_gather_types & GGML_SYCL_XMX_GATHER_Q8_0 ) != 0;
+ case GGML_TYPE_Q2_K: return (g_ggml_sycl_xmx_gather_types & GGML_SYCL_XMX_GATHER_Q2_K ) != 0;
+ case GGML_TYPE_Q3_K: return (g_ggml_sycl_xmx_gather_types & GGML_SYCL_XMX_GATHER_Q3_K ) != 0;
case GGML_TYPE_Q4_K: return (g_ggml_sycl_xmx_gather_types & GGML_SYCL_XMX_GATHER_Q4_K ) != 0;
case GGML_TYPE_Q5_K: return (g_ggml_sycl_xmx_gather_types & GGML_SYCL_XMX_GATHER_Q5_K ) != 0;
case GGML_TYPE_Q6_K: return (g_ggml_sycl_xmx_gather_types & GGML_SYCL_XMX_GATHER_Q6_K ) != 0;
@@ -82,7 +98,25 @@ bool ggml_sycl_fused_dequant_gemm(ggml_type src0_type, bool src0_reordered, cons
bool ggml_sycl_grouped_dequant_gemm(ggml_type src0_type, bool reordered, const void * src0_base, size_t expert_stride,
const float * src1, int32_t src1_prec, float * dst,
const int64_t * expert_row_offsets, int64_t n_as, int64_t M, int64_t K,
- int64_t total_rows, std::vector & tiles,
+ int64_t total_rows, std::vector & tiles,
ggml_sycl_pool & pool, dpct::queue_ptr stream);
+// Build routing and tile tables on the device; unused tile slots are empty.
+bool ggml_sycl_build_mmid_schedule(const int32_t * ids_dev, size_t ids_token_stride, int64_t n_as, int64_t n_ids,
+ int64_t n_tokens, int64_t n_tiles_max, mmid_row_mapping * row_mapping,
+ ggml_sycl_grouped_gemm_tile * tiles, uint32_t * expert_offsets,
+ dpct::queue_ptr stream);
+
+bool ggml_sycl_grouped_dequant_gemm_device_supported(ggml_type src0_type, bool reordered, int32_t src1_prec,
+ int64_t n_as, int64_t n_ids, int64_t n_tokens,
+ int64_t M, int64_t K, dpct::queue_ptr stream);
+
+// Returns false before queuing work when the type, shape, layout or device is unsupported.
+bool ggml_sycl_grouped_dequant_gemm_device(ggml_type src0_type, bool reordered, const void * src0_base,
+ size_t expert_stride, const ggml_sycl_grouped_gemm_rows & src1,
+ int32_t src1_prec, const ggml_sycl_grouped_gemm_rows & dst,
+ const int32_t * ids_dev, size_t ids_token_stride, int64_t n_as,
+ int64_t n_ids, int64_t n_tokens, int64_t M, int64_t K, ggml_sycl_pool & pool,
+ dpct::queue_ptr stream);
+
#endif // GGML_SYCL_FUSED_GEMM_HPP
diff --git a/ggml/src/ggml-sycl/ggml-sycl.cpp b/ggml/src/ggml-sycl/ggml-sycl.cpp
index 7b640a517..7e6df8fdd 100644
--- a/ggml/src/ggml-sycl/ggml-sycl.cpp
+++ b/ggml/src/ggml-sycl/ggml-sycl.cpp
@@ -118,6 +118,7 @@ int g_ggml_sycl_xmx_glu_max_cols = GGML_SYCL_XMX_GLU_MAX_COLS;
int g_ggml_sycl_prioritize_dmmv = 0;
int g_ggml_sycl_xmx_gather_types = GGML_SYCL_XMX_GATHER_TYPES_DEFAULT;
int g_ggml_sycl_xmx_gather_shapes = GGML_SYCL_XMX_GATHER_SHAPES_DEFAULT;
+int g_ggml_sycl_mmid_sched = GGML_SYCL_MMID_SCHED_DEVICE;
int g_ggml_sycl_dynamic_precision = GGML_SYCL_DYNAMIC_PRECISION_DEFAULT;
int g_ggml_sycl_dynamic_required_precision = GGML_SYCL_DYNAMIC_PRECISION_F32;
static const char * ggml_sycl_dynamic_precision_names[] = { "F16", "BF16", "TF32", "F32" };
@@ -451,6 +452,7 @@ static void ggml_check_sycl() try {
g_ggml_sycl_prioritize_dmmv = ggml_sycl_get_env("GGML_SYCL_PRIORITIZE_DMMV", 0);
g_ggml_sycl_graph_eviction_timeout = ggml_sycl_get_env("GGML_SYCL_GRAPH_EVICTION_TIMEOUT", 10);
g_ggml_sycl_xmx_gather_types = ggml_sycl_get_env("GGML_SYCL_XMX_GATHER_TYPES", GGML_SYCL_XMX_GATHER_TYPES_DEFAULT);
+ g_ggml_sycl_mmid_sched = ggml_sycl_get_env("GGML_SYCL_MMID_SCHED", GGML_SYCL_MMID_SCHED_DEVICE) & 3;
g_ggml_sycl_xmx_gather_shapes = ggml_sycl_get_env("GGML_SYCL_XMX_GATHER_SHAPES", GGML_SYCL_XMX_GATHER_SHAPES_DEFAULT);
g_ggml_sycl_dynamic_precision =
ggml_sycl_get_env_precision("GGML_SYCL_DYNAMIC_PRECISION", GGML_SYCL_DYNAMIC_PRECISION_DEFAULT);
@@ -575,6 +577,7 @@ static void ggml_check_sycl() try {
GGML_LOG_INFO(" GGML_SYCL_ENABLE_OPT: %d\n", g_ggml_sycl_enable_optimize);
GGML_LOG_INFO(" GGML_SYCL_XMX_GATHER_TYPES: %d\n", g_ggml_sycl_xmx_gather_types);
+ GGML_LOG_INFO(" GGML_SYCL_MMID_SCHED: %d\n", g_ggml_sycl_mmid_sched);
GGML_LOG_INFO(" GGML_SYCL_XMX_GATHER_SHAPES: %d\n", g_ggml_sycl_xmx_gather_shapes);
GGML_LOG_INFO(" GGML_SYCL_DYNAMIC_PRECISION: %s\n",
ggml_sycl_dynamic_precision_names[g_ggml_sycl_dynamic_precision]);
@@ -812,7 +815,9 @@ ggml_backend_sycl_buffer_init_tensor(ggml_backend_buffer_t buffer,
return GGML_STATUS_SUCCESS;
}
- if (g_ggml_sycl_enable_optimize) {
+ // a compute buffer copy of a host weight (e.g. -ncmoe experts offloaded for prompt processing) is
+ // refilled in the normal layout before every run, so reordering it is wasted work
+ if (g_ggml_sycl_enable_optimize && ggml_backend_buffer_get_usage(buffer) != GGML_BACKEND_BUFFER_USAGE_COMPUTE) {
// set reorder extra buffer based on supported type
switch (tensor->type) {
case GGML_TYPE_Q4_0:
@@ -4474,36 +4479,6 @@ static bool ggml_sycl_supports_dmmv(enum ggml_type type) {
}
}
-static bool ggml_sycl_supports_mmvq_id(enum ggml_type type) {
- switch (type) {
- case GGML_TYPE_Q4_0:
- case GGML_TYPE_Q4_1:
- case GGML_TYPE_Q5_0:
- case GGML_TYPE_Q5_1:
- case GGML_TYPE_Q8_0:
- case GGML_TYPE_Q2_0:
- case GGML_TYPE_Q2_K:
- case GGML_TYPE_Q3_K:
- case GGML_TYPE_Q4_K:
- case GGML_TYPE_Q5_K:
- case GGML_TYPE_Q6_K:
- case GGML_TYPE_MXFP4:
- case GGML_TYPE_NVFP4:
- case GGML_TYPE_IQ2_XXS:
- case GGML_TYPE_IQ2_XS:
- case GGML_TYPE_IQ2_S:
- case GGML_TYPE_IQ3_XXS:
- case GGML_TYPE_IQ3_S:
- case GGML_TYPE_IQ1_S:
- case GGML_TYPE_IQ1_M:
- case GGML_TYPE_IQ4_NL:
- case GGML_TYPE_IQ4_XS:
- return true;
- default:
- return false;
- }
-}
-
// Helper functions to unify device memory allocation for both async and sync paths
static inline void * sycl_ext_malloc_device(dpct::queue_ptr stream, size_t size) {
bool use_async = g_ggml_sycl_use_async_mem_op;
@@ -4979,6 +4954,16 @@ static bool reorder_qw_q3_k(uint8_t * data_device, size_t size, size_t offset, d
}, stream);
}
+// Reorder each Q3_K expert slice on its own.
+static bool reorder_qw_q3_k_moe(uint8_t * data_device, size_t expert_bytes, int64_t n_expert, dpct::queue_ptr stream) {
+ for (int64_t e = 0; e < n_expert; ++e) {
+ if (!reorder_qw_q3_k(data_device + e * expert_bytes, expert_bytes, 0, stream)) {
+ return false;
+ }
+ }
+ return true;
+}
+
static bool reorder_qw_q5_k(uint8_t * data_device, size_t size, size_t offset, dpct::queue_ptr stream) {
GGML_ASSERT(offset == 0);
return reorder_qw_planar(__func__, data_device, size, offsetof(block_q5_K, qs), QK_K / 2,
@@ -5050,6 +5035,8 @@ static bool reorder_qw(const ggml_tensor * src0, dpct::queue_ptr stream) {
switch (src0->type) {
case GGML_TYPE_Q8_0:
return reorder_qw_q8_0_moe(data_device, src0->nb[2], src0->ne[2], stream);
+ case GGML_TYPE_Q3_K:
+ return reorder_qw_q3_k_moe(data_device, src0->nb[2], src0->ne[2], stream);
case GGML_TYPE_Q4_K:
return reorder_qw_q4_k_moe(data_device, src0->nb[2], src0->ne[2], stream);
case GGML_TYPE_Q5_K:
@@ -5135,7 +5122,7 @@ static void opt_for_reorder_id(ggml_backend_sycl_context * ctx, const ggml_tenso
if (!g_ggml_sycl_enable_optimize || !ctx->opt_feature.reorder) {
return;
}
- if (src0->type != GGML_TYPE_Q8_0 && src0->type != GGML_TYPE_Q4_K &&
+ if (src0->type != GGML_TYPE_Q8_0 && src0->type != GGML_TYPE_Q3_K && src0->type != GGML_TYPE_Q4_K &&
src0->type != GGML_TYPE_Q5_K && src0->type != GGML_TYPE_Q6_K && src0->type != GGML_TYPE_MXFP4) {
return;
}
@@ -5143,6 +5130,10 @@ static void opt_for_reorder_id(ggml_backend_sycl_context * ctx, const ggml_tenso
if (src0->type == GGML_TYPE_MXFP4 && (src0->nb[2] % 16 != 0 || (uintptr_t) src0->data % 16 != 0)) {
return;
}
+ // The grouped GEMM loads reordered Q3_K qs and hmask as 16-byte vectors.
+ if (src0->type == GGML_TYPE_Q3_K && ((src0->nb[2] / sizeof(block_q3_K)) % 16 != 0 || (uintptr_t) src0->data % 16 != 0)) {
+ return;
+ }
ggml_tensor_extra_gpu * extra = static_cast(src0->extra);
if (!extra || extra->optimized_feature.reorder) {
return;
@@ -5641,26 +5632,31 @@ __dpct_inline__ static void k_zero_dst_rows(
}
}
-// returns true when ggml_sycl_mul_mat_id takes the fallback path that requires stream synchronization
-static bool ggml_sycl_mul_mat_id_needs_sync(const ggml_tensor * dst) {
- const ggml_tensor * src0 = dst->src[0];
- const ggml_tensor * src1 = dst->src[1];
- const ggml_tensor * ids = dst->src[2];
-
+// layout checks for the fused MMVQ path; the type check is ggml_sycl_mul_mat_vec_q_id_supports_type
+static bool ggml_sycl_mul_mat_id_mmvq_eligible(
+ const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * ids, const ggml_tensor * dst) {
const int64_t ne10 = src1->ne[0];
const int64_t ne11 = src1->ne[1];
const int64_t ne12 = src1->ne[2];
- if (ne12 != 1) return true;
- if (src1->type != GGML_TYPE_F32 || dst->type != GGML_TYPE_F32) return true;
- if (ne10 != src0->ne[0] || ne10 % QK8_1 != 0) return true;
- if (!ggml_is_contiguous(src1)) return true;
+ if (ne12 != 1) return false;
+ if (src1->type != GGML_TYPE_F32 || dst->type != GGML_TYPE_F32) return false;
+ if (ne10 != src0->ne[0] || ne10 % QK8_1 != 0) return false;
+ if (!ggml_is_contiguous(src1)) return false;
const int64_t n_ids_per_group = ids->ne[0];
- if (ids->ne[1] != 1) return true;
- if (ne11 != 1 && ne11 != n_ids_per_group) return true;
+ if (ids->type != GGML_TYPE_I32 || ids->nb[0] != sizeof(int32_t)) return false;
+ if (ids->ne[1] != 1) return false;
+ if (ne11 != 1 && ne11 != n_ids_per_group) return false;
- return !ggml_sycl_supports_mmvq_id(src0->type);
+ return true;
+}
+
+// returns true when ggml_sycl_mul_mat_id takes the fused MMVQ path
+static bool ggml_sycl_mul_mat_id_should_use_mmvq(const ggml_tensor * dst) {
+ const ggml_tensor * src0 = dst->src[0];
+ return ggml_sycl_mul_mat_id_mmvq_eligible(src0, dst->src[1], dst->src[2], dst) &&
+ ggml_sycl_mul_mat_vec_q_id_supports_type(src0->type);
}
// Fused MoE TG fast path. Returns false to fall back to the per-expert loop below.
@@ -5668,13 +5664,11 @@ static bool ggml_sycl_mul_mat_id_mmvq_fused(
ggml_backend_sycl_context & ctx, const ggml_tensor * src0,
const ggml_tensor * src1, const ggml_tensor * ids, ggml_tensor * dst)
{
- if (ggml_sycl_mul_mat_id_needs_sync(dst)) return false;
+ if (!ggml_sycl_mul_mat_id_should_use_mmvq(dst)) return false;
const int64_t ne10 = src1->ne[0];
const int64_t ne11 = src1->ne[1];
-
const int64_t n_ids_per_group = ids->ne[0];
-
const queue_ptr stream = ctx.stream();
const int src1_padded_cols = GGML_PAD((int) ne10, MATRIX_ROW_PADDING);
const int n_experts_used = (int) n_ids_per_group;
@@ -5772,6 +5766,18 @@ static void mmid_counting_sort_rows(
routed_row_src.resize(expert_row_offsets[n_as]);
}
+// true when the device MoE schedule may handle this MUL_MAT_ID: enabled, prefill, and an
+// f32/i32 layout it accepts; ggml_sycl_grouped_dequant_gemm_device decides type and width
+static bool ggml_sycl_mul_mat_id_device_sched_eligible(const ggml_tensor * dst) {
+ const ggml_tensor * src0 = dst->src[0];
+ const ggml_tensor * src1 = dst->src[1];
+ const ggml_tensor * ids = dst->src[2];
+ return (g_ggml_sycl_mmid_sched & GGML_SYCL_MMID_SCHED_DEVICE) && src1->ne[2] != 1 && ggml_is_contiguous(src0) &&
+ ids->type == GGML_TYPE_I32 && ids->nb[0] == sizeof(int32_t) &&
+ src1->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32 &&
+ src1->nb[1] == sizeof(float)*src1->ne[0] && dst->nb[1] == sizeof(float)*dst->ne[0];
+}
+
static void ggml_sycl_mul_mat_id(ggml_backend_sycl_context & ctx,
ggml_tensor *dst) try {
scope_op_debug_print scope_dbg_print(__func__, dst, /*num_src=*/3);
@@ -5792,13 +5798,46 @@ static void ggml_sycl_mul_mat_id(ggml_backend_sycl_context & ctx,
return;
}
}
- GGML_ASSERT(ggml_sycl_mul_mat_id_needs_sync(dst));
+ GGML_ASSERT(!ggml_sycl_mul_mat_id_should_use_mmvq(dst));
// The per-expert loop below reads the experts in whatever layout they have: reorder MXFP4 here as well, so prompt processing does not depend on a single-token decode having run first.
if (src0->type == GGML_TYPE_MXFP4) {
opt_for_reorder_id(&ctx, src0);
}
+ if (ggml_sycl_mul_mat_id_device_sched_eligible(dst)) {
+ if (g_ggml_sycl_dynamic_precision != GGML_SYCL_DYNAMIC_PRECISION_F32 &&
+ ggml_sycl_xmx_gather_type_enabled(src0->type)) {
+ opt_for_reorder_id(&ctx, src0);
+ }
+ const bool reordered = src0->extra && ((const ggml_tensor_extra_gpu *) src0->extra)->optimized_feature.reorder;
+ const ggml_sycl_grouped_gemm_rows src1_rows = { (char *) src1->data, nullptr, ne11, nb11, nb12 };
+ const ggml_sycl_grouped_gemm_rows dst_rows = { (char *) dst->data, nullptr, ne1, nb1, nb2 };
+ if (ggml_sycl_grouped_dequant_gemm_device(src0->type, reordered, src0->data, nb02, src1_rows,
+ ggml_sycl_src1_prec(dst), dst_rows, (const int32_t *) ids->data, ids->nb[1], n_as, n_ids,
+ ids->ne[1], ne01, ne10, ctx.pool(), stream)) {
+ // the device schedule drops skipped slots (id == -1), so their dst rows are zeroed here
+ const char * ids_data = (const char *) ids->data;
+ const size_t ids_nb1 = ids->nb[1];
+ char * dst_data = (char *) dst->data;
+ const int nth = (int) std::min(ne0, 256);
+ stream->parallel_for(
+ sycl::nd_range<2>(sycl::range<2>(ids->ne[1], n_ids * nth), sycl::range<2>(1, nth)),
+ [=](sycl::nd_item<2> item) {
+ const int64_t iid1 = item.get_group(0);
+ const int64_t id = item.get_group(1);
+ if (*(const int32_t *) (ids_data + iid1*ids_nb1 + id*sizeof(int32_t)) != -1) {
+ return;
+ }
+ float * row = (float *) (dst_data + id*nb1 + iid1*nb2);
+ for (int64_t j = item.get_local_id(1); j < ne0; j += nth) {
+ row[j] = 0.0f;
+ }
+ });
+ return;
+ }
+ }
+
std::vector ids_host(ggml_nbytes(ids));
const char * ids_dev = (const char *) ids->data;
@@ -6904,6 +6943,54 @@ static bool mul_mat_uses_onednn(int device, ggml_tensor * dst) {
#endif
}
+// returns true when ggml_sycl_mul_mat_id takes the fallback path that requires stream synchronization:
+// neither the fused MMVQ path nor the device-scheduled grouped GEMM (GGML_SYCL_MMID_SCHED) handles dst
+static bool ggml_sycl_mul_mat_id_needs_sync(ggml_backend_sycl_context * ctx, const ggml_tensor * dst) {
+ const ggml_tensor * src0 = dst->src[0];
+ const ggml_tensor * src1 = dst->src[1];
+ const ggml_tensor * ids = dst->src[2];
+ if (!src0 || !src1 || !ids || ggml_backend_buffer_is_sycl_split(src0->buffer)) {
+ return true;
+ }
+ const bool reordered = src0->extra && ((const ggml_tensor_extra_gpu *) src0->extra)->optimized_feature.reorder;
+ if (ggml_sycl_mul_mat_id_mmvq_eligible(src0, src1, ids, dst)) {
+ return reordered ? !ggml_sycl_mul_mat_vec_q_id_reorder_supports_type(src0->type) :
+ !ggml_sycl_mul_mat_vec_q_id_supports_type(src0->type);
+ }
+ return !ggml_sycl_mul_mat_id_device_sched_eligible(dst) ||
+ !ggml_sycl_grouped_dequant_gemm_device_supported(src0->type, reordered, ggml_sycl_src1_prec(dst),
+ src0->ne[2], ids->ne[0], ids->ne[1], src0->ne[1], src0->ne[0], ctx->stream());
+}
+
+// mirrors the paths in ggml_sycl_mul_mat_id() that call opt_for_reorder_id()
+static bool mul_mat_id_reorders_src0(const ggml_tensor * dst) {
+ const ggml_tensor * src0 = dst->src[0];
+ if (ggml_sycl_mul_mat_id_should_use_mmvq(dst) || src0->type == GGML_TYPE_MXFP4) {
+ return true;
+ }
+ return ggml_sycl_mul_mat_id_device_sched_eligible(dst) &&
+ g_ggml_sycl_dynamic_precision != GGML_SYCL_DYNAMIC_PRECISION_F32 &&
+ ggml_sycl_xmx_gather_type_enabled(src0->type);
+}
+
+// true when the node and its SYCL sources live in this device's buffers (no split or other-device buffers)
+static bool graph_node_is_in_local_device(ggml_backend_buffer_type_t device_buft, const ggml_tensor * node) {
+ if (!node->buffer || node->buffer->buft != device_buft) {
+ return false;
+ }
+ for (int j = 0; j < GGML_MAX_SRC; ++j) {
+ const ggml_tensor * src = node->src[j];
+ if (!src || !src->buffer) {
+ continue;
+ }
+ if (ggml_backend_buffer_is_sycl_split(src->buffer) ||
+ (ggml_backend_buffer_is_sycl(src->buffer) && src->buffer->buft != device_buft)) {
+ return false;
+ }
+ }
+ return true;
+}
+
static bool check_graph_compatibility(ggml_backend_sycl_context * ctx, ggml_cgraph * cgraph) {
if (ggml_sycl_info().active_device_count > 1) {
// A sycl_ex::command_graph object can only be created for a single device
@@ -6911,6 +6998,8 @@ static bool check_graph_compatibility(ggml_backend_sycl_context * ctx, ggml_cgra
return false;
}
+ const ggml_backend_buffer_type_t device_buft = ggml_backend_sycl_buffer_type(ctx->device);
+
for (int i = 0; i < cgraph->n_nodes; i++) {
ggml_tensor * node = cgraph->nodes[i];
@@ -6921,6 +7010,18 @@ static bool check_graph_compatibility(ggml_backend_sycl_context * ctx, ggml_cgra
const ggml_op node_op = node->op;
+ if (!graph_node_is_in_local_device(device_buft, node)) {
+ GGML_LOG_DEBUG("%s: disabling SYCL graphs due to node %s not being device local\n", __func__,
+ ggml_op_name(node_op));
+ return false;
+ }
+
+ // Reorder expert weights before recording, so the reorder state that
+ // ggml_sycl_mul_mat_id_needs_sync() sees is the one the recorded graph runs with.
+ if (node_op == GGML_OP_MUL_MAT_ID && mul_mat_id_reorders_src0(node)) {
+ opt_for_reorder_id(ctx, node->src[0]);
+ }
+
switch (node_op) {
default:
break;
@@ -6928,7 +7029,7 @@ static bool check_graph_compatibility(ggml_backend_sycl_context * ctx, ggml_cgra
// The ggml_sycl_mul_mat_id() fallback path does a blocking host wait on the sycl
// queue after submitting a memcpy operation, but wait() can't be called on a queue
// that is recording to a graph.
- if (ggml_sycl_mul_mat_id_needs_sync(node)) {
+ if (ggml_sycl_mul_mat_id_needs_sync(ctx, node)) {
GGML_LOG_DEBUG("%s: disabling SYCL graphs due to host wait in %s\n", __func__,
ggml_op_name(node_op));
return false;
@@ -6968,6 +7069,13 @@ static bool check_graph_compatibility(ggml_backend_sycl_context * ctx, ggml_cgra
}
#endif
break;
+ case GGML_OP_OUT_PROD:
+ case GGML_OP_CONV_3D:
+ case GGML_OP_SOLVE_TRI:
+ // these call oneMKL and wait on the host
+ GGML_LOG_DEBUG("%s: disabling SYCL graphs due to unsupported node type %s\n", __func__,
+ ggml_op_name(node_op));
+ return false;
}
}
return true;
diff --git a/ggml/src/ggml-sycl/mmvq.cpp b/ggml/src/ggml-sycl/mmvq.cpp
index c3853814a..03464bfae 100644
--- a/ggml/src/ggml-sycl/mmvq.cpp
+++ b/ggml/src/ggml-sycl/mmvq.cpp
@@ -2706,6 +2706,50 @@ static void launch_mul_mat_vec_q_moe(
});
}
+bool ggml_sycl_mul_mat_vec_q_id_supports_type(enum ggml_type src0_type) {
+ switch (src0_type) {
+ case GGML_TYPE_Q4_0:
+ case GGML_TYPE_Q4_1:
+ case GGML_TYPE_Q5_0:
+ case GGML_TYPE_Q5_1:
+ case GGML_TYPE_Q8_0:
+ case GGML_TYPE_Q2_0:
+ case GGML_TYPE_Q2_K:
+ case GGML_TYPE_Q3_K:
+ case GGML_TYPE_Q4_K:
+ case GGML_TYPE_Q5_K:
+ case GGML_TYPE_Q6_K:
+ case GGML_TYPE_MXFP4:
+ case GGML_TYPE_NVFP4:
+ case GGML_TYPE_IQ2_XXS:
+ case GGML_TYPE_IQ2_XS:
+ case GGML_TYPE_IQ2_S:
+ case GGML_TYPE_IQ3_XXS:
+ case GGML_TYPE_IQ3_S:
+ case GGML_TYPE_IQ1_S:
+ case GGML_TYPE_IQ1_M:
+ case GGML_TYPE_IQ4_NL:
+ case GGML_TYPE_IQ4_XS:
+ return true;
+ default:
+ return false;
+ }
+}
+
+bool ggml_sycl_mul_mat_vec_q_id_reorder_supports_type(enum ggml_type src0_type) {
+ switch (src0_type) {
+ case GGML_TYPE_Q8_0:
+ case GGML_TYPE_Q3_K:
+ case GGML_TYPE_Q4_K:
+ case GGML_TYPE_Q5_K:
+ case GGML_TYPE_Q6_K:
+ case GGML_TYPE_MXFP4:
+ return true;
+ default:
+ return false;
+ }
+}
+
bool ggml_sycl_mul_mat_vec_q_id(
enum ggml_type src0_type,
const void * vx_base,
@@ -2719,7 +2763,9 @@ bool ggml_sycl_mul_mat_vec_q_id(
size_t dst_row_stride,
size_t src1_row_stride,
dpct::queue_ptr stream) {
- // if adding a new type, ensure it is also added to ggml_sycl_supports_mmvq_id
+ if (!ggml_sycl_mul_mat_vec_q_id_supports_type(src0_type)) {
+ return false;
+ }
switch (src0_type) {
case GGML_TYPE_Q4_0:
launch_mul_mat_vec_q_moe(
@@ -2952,12 +2998,20 @@ bool ggml_sycl_mul_mat_vec_q_id_reorder(
size_t dst_row_stride,
size_t src1_row_stride,
dpct::queue_ptr stream) {
+ if (!ggml_sycl_mul_mat_vec_q_id_reorder_supports_type(src0_type)) {
+ return false;
+ }
switch (src0_type) {
case GGML_TYPE_Q8_0:
launch_mul_mat_vec_q_moe_reorder>(
vx_base, vy, ids_dev, dst_base, ncols, nrows, n_experts_used,
expert_weight_stride, dst_row_stride, src1_row_stride, stream);
return true;
+ case GGML_TYPE_Q3_K:
+ launch_mul_mat_vec_q_moe_reorder>(
+ vx_base, vy, ids_dev, dst_base, ncols, nrows, n_experts_used,
+ expert_weight_stride, dst_row_stride, src1_row_stride, stream);
+ return true;
case GGML_TYPE_Q4_K:
launch_mul_mat_vec_q_moe_reorder>(
vx_base, vy, ids_dev, dst_base, ncols, nrows, n_experts_used,
diff --git a/ggml/src/ggml-sycl/mmvq.hpp b/ggml/src/ggml-sycl/mmvq.hpp
index 5649cbcd2..e372d50ba 100644
--- a/ggml/src/ggml-sycl/mmvq.hpp
+++ b/ggml/src/ggml-sycl/mmvq.hpp
@@ -117,4 +117,7 @@ bool ggml_sycl_mul_mat_vec_q_glu_plain(
int stride_col_dst, // floats between output columns in dst
dpct::queue_ptr stream);
+bool ggml_sycl_mul_mat_vec_q_id_supports_type(enum ggml_type src0_type);
+bool ggml_sycl_mul_mat_vec_q_id_reorder_supports_type(enum ggml_type src0_type);
+
#endif // GGML_SYCL_MMVQ_HPP
diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cpp
index 840bfecd5..b1a538a31 100644
--- a/tests/test-backend-ops.cpp
+++ b/tests/test-backend-ops.cpp
@@ -11931,6 +11931,23 @@ static std::vector> make_test_cases_eval() {
// the batches of a are padded, nb[2] is not a multiple of nb[1]
test_cases.emplace_back(new test_mul_mat(GGML_TYPE_F32, GGML_TYPE_F32, 8, 1, 64, {8, 1}, {1, 1}, {0, 1, 2, 3}, 0, 1, false, 16, 16));
test_cases.emplace_back(new test_mul_mat(GGML_TYPE_F16, GGML_TYPE_F32, 8, 16, 64, {8, 1}, {1, 1}, {0, 1, 2, 3}, 0, 1, false, 16, 16));
+ // Ragged expert slices, strided routing IDs and broadcast activations.
+ for (ggml_type type : {GGML_TYPE_Q8_0, GGML_TYPE_IQ4_NL, GGML_TYPE_IQ3_S, GGML_TYPE_Q2_K, GGML_TYPE_Q3_K}) {
+ for (int n : {3, 17, 33}) {
+ for (bool broadcast : {false, true}) {
+ test_cases.emplace_back(new test_mul_mat_id(type, GGML_TYPE_F32, 37, 2, broadcast, 65, n, 256));
+ }
+ test_cases.emplace_back(new test_mul_mat_id(type, GGML_TYPE_F32, 37, 2, false, 65, n, 256, 0, false, true));
+ }
+ }
+
+ // Both experts receive every token
+ for (ggml_type type : {GGML_TYPE_Q2_K, GGML_TYPE_Q3_K, GGML_TYPE_Q4_K}) {
+ for (int n : {64, 65}) {
+ test_cases.emplace_back(new test_mul_mat_id(type, GGML_TYPE_F32, 2, 2, false, 33, n, 512));
+ }
+ }
+
// as is a view whose experts are strided by more rows than it uses
test_cases.emplace_back(new test_mul_mat_id(GGML_TYPE_F32, GGML_TYPE_F32, 4, 2, false, 8, 1, 64, 64));
test_cases.emplace_back(new test_mul_mat_id(GGML_TYPE_F16, GGML_TYPE_F32, 4, 2, false, 8, 16, 64, 64));
@@ -13465,7 +13482,7 @@ static std::vector> make_test_cases_perf() {
// qwen3-30b-a3b
for (int bs : {1, 4, 8, 32, 64, 128, 256, 512}) {
- for (ggml_type type_a : {GGML_TYPE_F32, GGML_TYPE_F16, GGML_TYPE_Q4_0, GGML_TYPE_Q8_0, GGML_TYPE_Q4_K, GGML_TYPE_Q6_K, GGML_TYPE_IQ2_XS, GGML_TYPE_IQ4_XS}) {
+ for (ggml_type type_a : {GGML_TYPE_F32, GGML_TYPE_F16, GGML_TYPE_Q4_0, GGML_TYPE_Q8_0, GGML_TYPE_Q2_K, GGML_TYPE_Q3_K, GGML_TYPE_Q4_K, GGML_TYPE_Q6_K, GGML_TYPE_IQ2_XS, GGML_TYPE_IQ4_XS}) {
for (ggml_type type_b : {GGML_TYPE_F32}) {
test_cases.emplace_back(new test_mul_mat_id(type_a, type_b, 128, 8, false, 768, bs, 2048));
test_cases.emplace_back(new test_mul_mat_id_fusion(type_a, type_b, 128, 8, false, 768, bs, 2048, 1));