diff --git a/docs/backend/SYCL.md b/docs/backend/SYCL.md
index ca0b6d7bc..4a1d16aad 100644
--- a/docs/backend/SYCL.md
+++ b/docs/backend/SYCL.md
@@ -827,6 +827,7 @@ User can use the device management in [docs/multi-gpu.md](https://github.com/ggm
| GGML_SYCL_XMX_MAX_COLS | non-negative integer, 80 (default) | Most activation columns (batch size) the `GGML_SYCL_ENABLE_XMX_ESIMD` kernels take; wider batches use the other mat-mul kernels. A larger value keeps XMX for bigger batches (as repeated launches), `0` never uses it. |
| GGML_SYCL_XMX_GLU_MAX_COLS | 0 to 16, 16 (default) | Most columns the fused gate and up XMX kernel takes; wider batches run gate and up as separate mat-vecs. 16 is the most the kernel is built for, larger values are clamped to 16 and `0` turns the fused kernel off. |
| GGML_SYCL_XMX_GATHER_TYPES | decimal bitmask, all bits set (default) | Weight formats that may use the XMX dequant-GEMM paths, which dequantize weights straight into the XMX tiles. This speeds up prompt processing of MoE models on GPUs with XMX units (Arc A- and B-series, Arc Pro, Data Center GPU Max), for example pp512 of Qwen3-30B-A3B UD-IQ3_XXS by about 50% on an Arc Pro B60. Bits:
* 1: IQ4_NL, 2: IQ3_S, 4: IQ4_XS, 8: IQ3_XXS, 16: IQ2_XXS, 32: IQ2_XS, 64: IQ2_S, 128: IQ1_S, 256: IQ1_M
* 512: Q8_0, 1024: Q4_K, 2048: Q5_K, 4096: Q6_K, 8192: Q2_K, 16384: Q3_K (MoE `MUL_MAT_ID` only)
Add values to combine them, for example `3` for IQ4_NL and IQ3_S; `0` disables the paths. A set bit does not force the path: batches of more than 64 tokens per expert use the library GEMM unless the device schedule width heuristic is overridden with `GGML_SYCL_MMID_SCHED=3`. Row lengths must be a multiple of 256 (32 for IQ4_NL and Q8_0). |
+| GGML_SYCL_MMID_ESIMD | 0, 1 (default), 2 | The `MUL_MAT_ID` grouped XMX dequant-GEMM as an ESIMD kernel that dequantizes the weights straight into the `dpas` register operands, with no shared local memory image of them, from either weight layout (the per-expert reorder or the canonical blocks). q8_0, iq4_nl, iq4_xs, iq3_s, iq3_xxs and q2_K to q6_K. It stands in for the 8x16x16 sub-group-16 combination of f16 or bf16 (Xe2, Xe-HPC) and for the 8x8x16 sub-group-8 one (Xe-HPG: Arc A-series, ARL-H; there it works on 16 rows a thread and reads with 1D block loads, since that architecture has half the register file and no 2D block loads); while it is on, a grouped call whose weight type it covers is moved to that combination from any other shape of its type and sub-group size, whatever `GGML_SYCL_XMX_GATHER_SHAPES` allows, because the shape bits choose among `joint_matrix` tiles and the ESIMD kernel tiles on its own. A call it declines runs that combination's `joint_matrix` kernel. 0 uses the `joint_matrix` kernel, 1 the ESIMD one, 2 the ESIMD one summing K in the `joint_matrix` kernel's order (bit-identical output for f16; slower). An AOT build gets its 256-register file through `-ze-opt-enable-auto-large-GRF-mode` (set by the CMake build). Not built with `GGML_SYCL_SUPPORT_XMX=OFF`. |
| GGML_SYCL_MMID_SCHED | decimal bitmask, 1 (default) | Build the grouped MoE routing and tile schedule on the GPU, avoiding the expert-ID host readback and CPU sort.
* 1: enable the device schedule.
* 2: override the fallback for models with few routed experts by disabling the performance heuristic that rejects batches averaging more than 64 routed rows per expert. This may reduce performance; type, shape, precision and device capability checks still apply.
Use `3` to enable the device schedule and override this heuristic, or `0` to use the host schedule. Bit 2 alone does not enable the device schedule. Higher bits are ignored. |
| GGML_SYCL_XMX_GATHER_SHAPES | decimal bitmask, 255 (default) | XMX `joint_matrix` combinations the paths of `GGML_SYCL_XMX_GATHER_TYPES` may use; the operand type comes from `GGML_SYCL_DYNAMIC_PRECISION` and the best supported combination is picked automatically (logged as `fg_pick_combo`). Bits:
* Xe2, Xe3, Xe-HPC: 1: f16 8x16x16, 2: f16 16x16x16, 4: f16 32x64x16, 8: f16 32x64x32, 32: tf32 8x16x8, 64: bf16 8x16x16
* Xe-HPG (Arc A770, ARL-H): 16: f16 8x8x16, 128: bf16 8x8x16
Clear a bit to exclude a combination, or set a single bit to force one for testing. |
| GGML_SYCL_DYNAMIC_PRECISION | `F16` (default with `GGML_SYCL_F16=ON`), `BF16`, `TF32` or `F32` (default otherwise) | Operand type of the XMX dequant-GEMM paths (`GGML_SYCL_XMX_GATHER_TYPES`); accumulation is always f32. `F16` is the fastest, but activations above 65504 overflow. `BF16` keeps the f32 range at a 7-bit mantissa, `TF32` keeps the range and the f16 mantissa but is about 30% slower and needs Xe2, Xe3 or Xe-HPC, and `F32` turns the XMX paths off. Ops that request a higher src1 precision ([TAG_GGML_PREC]) get it regardless of this setting. |
diff --git a/ggml/src/ggml-sycl/CMakeLists.txt b/ggml/src/ggml-sycl/CMakeLists.txt
index 8312a8305..0a5aede0b 100644
--- a/ggml/src/ggml-sycl/CMakeLists.txt
+++ b/ggml/src/ggml-sycl/CMakeLists.txt
@@ -218,6 +218,8 @@ endif()
if (GGML_SYCL_TARGET STREQUAL "INTEL")
add_compile_definitions(GGML_SYCL_WARP_SIZE=16)
+ # With AOT the option goes to ocloc inside -options (see GGML_SYCL_DEVICE_ARCH below): a bare
+ # -Xs hands it to ocloc as a command-line flag, which fails the device link.
if (NOT GGML_SYCL_DEVICE_ARCH)
target_link_options(ggml-sycl PRIVATE -Xs -ze-intel-greater-than-4GB-buffer-required)
else()
@@ -248,10 +250,18 @@ endif()
if (GGML_SYCL_DEVICE_ARCH)
message(STATUS "GGML_SYCL_DEVICE_ARCH=${GGML_SYCL_DEVICE_ARCH} (AOT via spir64_gen)")
+ # AOT (oneAPI 2026.2) ignores the grf_size<256> kernel property on ESIMD kernels and builds them
+ # with 128 registers, which made the ESIMD MoE GEMM 1.7-3.9x slower than under JIT; SPMD kernels
+ # keep honouring the property. A module-wide -doubleGRF also moves the ESIMD kernels that ask for
+ # 128 to 256, and a server built with it hung on its first request, so IGC picks the size per
+ # kernel from its register pressure instead (ESIMD kernels too; -ze-opt-large-grf-kernel does not
+ # reach them).
+ set(GGML_SYCL_AOT_BACKEND_OPTIONS
+ "-device ${GGML_SYCL_DEVICE_ARCH} -options '-ze-intel-greater-than-4GB-buffer-required -ze-opt-enable-auto-large-GRF-mode'")
target_compile_options(
ggml-sycl PRIVATE
-fsycl-targets=spir64_gen
- "SHELL:-Xsycl-target-backend=spir64_gen \"-device ${GGML_SYCL_DEVICE_ARCH} -options -ze-intel-greater-than-4GB-buffer-required\""
+ "SHELL:-Xsycl-target-backend=spir64_gen \"${GGML_SYCL_AOT_BACKEND_OPTIONS}\""
)
# Pass through parallel job (process) count for parallelising the
@@ -266,7 +276,7 @@ if (GGML_SYCL_DEVICE_ARCH)
target_link_options(
ggml-sycl PRIVATE
-fsycl-targets=spir64_gen
- "SHELL:-Xsycl-target-backend=spir64_gen \"-device ${GGML_SYCL_DEVICE_ARCH} -options -ze-intel-greater-than-4GB-buffer-required\""
+ "SHELL:-Xsycl-target-backend=spir64_gen \"${GGML_SYCL_AOT_BACKEND_OPTIONS}\""
-fsycl-max-parallel-link-jobs=${GGML_SYCL_MAX_PARALLEL_LINK_JOBS}
)
diff --git a/ggml/src/ggml-sycl/common.hpp b/ggml/src/ggml-sycl/common.hpp
index 75e783d12..ae156acca 100644
--- a/ggml/src/ggml-sycl/common.hpp
+++ b/ggml/src/ggml-sycl/common.hpp
@@ -72,6 +72,7 @@ extern int g_ggml_sycl_debug;
extern int g_ggml_sycl_enable_optimize;
extern int g_ggml_sycl_enable_fusion;
extern int g_ggml_sycl_enable_esimd;
+extern int g_ggml_sycl_mmid_esimd;
extern int g_ggml_sycl_mmvq_wide;
extern int g_ggml_sycl_enable_xmx_esimd;
extern int g_ggml_sycl_xmx_max_cols;
diff --git a/ggml/src/ggml-sycl/fused-gemm-esimd.cpp b/ggml/src/ggml-sycl/fused-gemm-esimd.cpp
new file mode 100644
index 000000000..37f87e377
--- /dev/null
+++ b/ggml/src/ggml-sycl/fused-gemm-esimd.cpp
@@ -0,0 +1,1160 @@
+#include "fused-gemm-esimd.hpp"
+
+#include
+#include
+#include
+
+// One thread owns MR rows of one expert's weight and one BN-wide tile of the schedule, over the
+// whole K. Per k step (32 values) it loads the rows' stored bits, dequantizes them in registers to
+// the f16 A operand of dpas (RC = 8 rows x 16 k, row major) and multiplies it with the packed B,
+// which is already the dpas VNNI operand: [K/2][Npad][2] halves, NW columns a dpas.
+//
+// The joint_matrix kernel dequantizes into SLM and reads it back in the same sub-group, which is
+// staging, not sharing; dpas takes registers, so that round trip and its barriers are gone here.
+// Its K split over GGML_SYCL_GROUPED_GEMM_KSPLIT sub-groups is reproduced as sequential K ranges summed in
+// the same ascending order as its tile_c reduction, and every format below rounds A exactly as its
+// joint_matrix A stage does, so the output is the same bits.
+//
+// A weight format is a grouped_gemm_esimd_a with: raw, the stored bits of the MR rows for one k
+// step; load(), which fetches them from the expert's slice in either layout (grouped_gemm_esimd::slice); and
+// decode(), the f16 A of one row group.
+// It reads the same fields as the joint_matrix A stage of its format.
+//
+// The kernel is built for one grouped_gemm_arch: the dpas width NW (the columns of B one dpas takes, the
+// sub-group size of the joint_matrix combination it stands in for), the rows of one thread, and
+// whether 2D block loads exist. Xe2 and Xe-HPC: 16 wide, 32 rows, 2D loads. Xe-HPG (Arc A, ARL-H):
+// 8 wide, no 2D block messages, and 256 registers of 32 bytes, half of Xe2's file, so 16 rows: its
+// 32-row accumulators alone would take half the registers.
+static constexpr int GGML_SYCL_GROUPED_GEMM_KSPLIT = FG_KSPLIT;
+static constexpr int GGML_SYCL_GROUPED_GEMM_BN = GGML_SYCL_FG_BN;
+static constexpr int GROUPED_GEMM_MR = 32; // rows of one thread on Xe2: one 2D load covers them
+static constexpr int GROUPED_GEMM_RC = 8; // rows of one dpas
+static constexpr int GROUPED_GEMM_QK = 32; // values of one k step
+
+template struct grouped_gemm_arch {
+ static constexpr int NW = NW_;
+ static constexpr int MR = MR_;
+ static constexpr bool L2D = L2D_;
+};
+
+using grouped_gemm_xe2 = grouped_gemm_arch<16, GROUPED_GEMM_MR, true>;
+using grouped_gemm_xehpg = grouped_gemm_arch<8, 16, false>;
+
+// An AOT build compiles each kernel only for the targets of its dpas width: GGML_SYCL_XMX_AOT_SG for
+// spir64_gen, FG_AOT_SG (fused-gemm.hpp) for intel_gpu_* targets. A build without XMX (GGML_SYCL_SUPPORT_XMX
+// off) compiles none of them
+template static constexpr bool grouped_gemm_built() {
+#if !defined(GGML_SYCL_SUPPORT_XMX)
+ return false;
+#elif defined(GGML_SYCL_XMX_AOT_SG)
+ return A::NW == GGML_SYCL_XMX_AOT_SG;
+#elif defined(FG_AOT_SG)
+ return A::NW == FG_AOT_SG;
+#else
+ return true;
+#endif
+}
+
+namespace grouped_gemm_esimd {
+using namespace sycl::ext::intel::esimd;
+
+// f(std::integral_constant) for I in [0, N): register selects at a runtime offset become
+// indirect accesses, so loops whose index picks registers are spelled out at compile time
+template struct static_for {
+ template static ESIMD_INLINE void run(Fn & f) {
+ if constexpr (I < N) {
+ f(std::integral_constant{});
+ static_for::run(f);
+ }
+ }
+};
+
+// One field of a format's block. In the reorder (SoA) layout it is a stream of its own, `size` bytes
+// a block, after `pre` bytes a block of the streams before it; in the canonical (AoS) layout it is
+// `aos` bytes into the block.
+struct field {
+ int pre;
+ int size;
+ int aos;
+};
+
+// Where one thread's rows sit in an expert's slice, in either layout. nblocks counts the stored blocks
+// of the slice, bpr those of one row, bs is the size of one block.
+template struct slice {
+ const uint8_t * x;
+ int M;
+ int bpr;
+ int nblocks;
+ int m0;
+ bool aos;
+ int bs;
+
+ // per row, the byte offset from x of byte `inner` of field f of block ib
+ ESIMD_INLINE simd at(const field & f, const int ib, const int inner) const {
+ const int stride = aos ? bs : f.size;
+ simd o(0, (uint32_t) (bpr * stride));
+ return o + (uint32_t) ((aos ? f.aos : nblocks * f.pre) + (m0 * bpr + ib) * stride + inner);
+ }
+
+ // a field's bytes at offset `inner` start on a dword in every block (the AoS block and offset
+ // are multiples of 4); the SoA streams always do
+ ESIMD_INLINE bool dword_aligned(const field & f, const int inner) const {
+ return !aos || (bs % 4 == 0 && (f.aos + inner) % 4 == 0);
+ }
+
+ ESIMD_INLINE simd u16(const field & f, const int ib, const int inner) const {
+ return gather((const uint16_t *) x, at(f, ib, inner));
+ }
+
+ ESIMD_INLINE simd u32(const field & f, const int ib, const int inner) const {
+ if (dword_aligned(f, inner)) {
+ return gather((const uint32_t *) x, at(f, ib, inner));
+ }
+ return rows<1>(f, ib, inner);
+ }
+
+ // W dwords of each of the MR rows from byte `inner` (a multiple of 4) of field f of block ib, as
+ // [row][W]. SoA with 2D block loads (L2D): one message over the stream. Otherwise the rows' dwords
+ // are gathered, as [W][row], at offsets rounded down to a dword (every field starts on an even
+ // byte; the SoA streams are dword aligned), shifted by 16 bits where an AoS row's block starts on
+ // 2 mod 4, and transposed with strided reads.
+ template ESIMD_INLINE simd rows(const field & f, const int ib, const int inner) const {
+ if constexpr (L2D) {
+ if (!aos) {
+ const unsigned pitch = (unsigned) (bpr * f.size);
+ return load_2d((const uint32_t *) (x + (size_t) nblocks * f.pre), pitch - 1, M - 1,
+ pitch - 1, (ib * f.size + inner) / 4, m0);
+ }
+ }
+ const simd o = at(f, ib, inner);
+ simd g;
+ if (dword_aligned(f, inner)) {
+ g = gather((const uint32_t *) x, o);
+ } else {
+ // per row: sh is 16 where the run starts on 2 mod 4, else 0; hm keeps the next dword's low
+ // half only where it is needed
+ const simd oa = o & ~3u;
+ const simd sh = (o & 2u) << 3;
+ const simd hm = 0u - (sh >> 4);
+ simd lo = gather((const uint32_t *) x, oa);
+ // the dword after the run holds its last 2 bytes where a row needs it; a row that does not
+ // reads its first dword again rather than past the field
+ const simd tail = gather((const uint32_t *) x, oa + ((4u * W) & hm));
+ auto shift = [&](auto vc) SYCL_ESIMD_FUNCTION {
+ constexpr int v = decltype(vc)::value;
+ simd a = lo.template select(v * MR);
+ simd b;
+ if constexpr (v + 1 < W) {
+ b = lo.template select((v + 1) * MR);
+ } else {
+ b = tail;
+ }
+ g.template select(v * MR) = (a >> sh) | ((b << 16) & hm);
+ };
+ static_for<0, W>::run(shift);
+ }
+ if constexpr (W == 1) {
+ return g;
+ } else {
+ simd t;
+ auto row = [&](auto rc) SYCL_ESIMD_FUNCTION {
+ constexpr int r = decltype(rc)::value;
+ t.template select(r * W) = g.template select(r);
+ };
+ static_for<0, MR>::run(row);
+ return t;
+ }
+ }
+};
+
+// A decode returns its row group's A in the order the two dpas of a k step read it, [k half][row][16
+// k]: both iq formats store k and k + 16 in the same byte or grid dword half, so that order needs
+// only reads with a stride, while [row][32 k] needed strided writes, which split into SIMD4.
+using a_halves = simd;
+
+// f16 of the f32 product code * scale, each row's scale over its 16 values of a k half: the
+// expression the joint_matrix stages evaluate before E::make rounds it
+ESIMD_INLINE simd scale_half(simd v,
+ simd d) {
+ v *= d.template replicate_vs_w_hs(0);
+ return v;
+}
+
+// The iq4 codebook by byte, in SLM: entry b holds the values of its low and its high nibble as the
+// two 16-bit halves of one dword, as T (half, or int16 for an f32 product). A read-only table every
+// lane indexes, which is what SLM is for: register-indirect iselect() over the same lanes cost
+// 7.3 of 12.7 ms, a 16-entry SLM table per nibble about 0.4.
+template ESIMD_INLINE void iq4_pair_table_to_slm() {
+ static_assert(sizeof(T) == 2, "two values a dword");
+ simd lo;
+#pragma unroll
+ for (int i = 0; i < 16; ++i) {
+ lo[i] = sycl::bit_cast((T) kvalues_iq4nl[i]);
+ }
+#pragma unroll
+ for (int hi = 0; hi < 16; ++hi) {
+ slm_block_store(hi * 64, lo | ((uint32_t) sycl::bit_cast((T) kvalues_iq4nl[hi]) << 16));
+ }
+}
+
+// 16 packed bytes a row; byte j holds k = j in its low and k = 16 + j in its high nibble (the order
+// of fg_decode_iq4_xs and of the iq4_nl stage). One gather per byte; the low halves are k half 0,
+// the high halves k half 1, each [row][16].
+template
+ESIMD_INLINE simd iq4_codes(simd q) {
+ simd off = q.template bit_cast_view();
+ simd pr = slm_gather(off * 4);
+ auto pv = pr.template bit_cast_view();
+ simd v;
+ v.template select(0) = pv.template select(0);
+ v.template select(GROUPED_GEMM_RC * 16) = pv.template select(1);
+ return v;
+}
+// A 2D surface: rows `pitch` bytes apart, 16-byte aligned and at least 64 bytes
+constexpr bool surface_ok(const int pitch) {
+ return pitch % 16 == 0 && pitch >= 64;
+}
+
+// byte i of each lane's dword, i a runtime index: a shift by a scalar, not a register pick
+template ESIMD_INLINE simd byte_of(simd w, const int i) {
+ return (w >> (8 * i)) & 0xffu;
+}
+
+// one of four per-lane dwords by a runtime index, as uniform branches rather than a register pick
+template ESIMD_INLINE simd dword_of(simd * w, const int i) {
+ simd r = w[0];
+ if (i == 1) {
+ r = w[1];
+ } else if (i == 2) {
+ r = w[2];
+ } else if (i == 3) {
+ r = w[3];
+ }
+ return r;
+}
+
+// a row group's 32 k-ordered code bytes a row as f32, in [k half][row][16 k]
+ESIMD_INLINE simd byte_codes(simd q) {
+ auto bm = q.template bit_cast_view();
+ simd v;
+ v.template select(0) = bm.template select(0, 0);
+ v.template select(GROUPED_GEMM_RC * 16) =
+ bm.template select(0, 16);
+ return v;
+}
+
+// f16 of d * code - m in f32, each row's d and m over its 16 values of a k half: the expression of
+// the K-quant stages with a min
+ESIMD_INLINE simd scale_min_half(simd v,
+ simd d,
+ simd m) {
+ v = v * d.template replicate_vs_w_hs(0) -
+ m.template replicate_vs_w_hs(0);
+ return v;
+}
+
+// the two k halves of a row group, the code bytes times one (d, m) per row and half
+ESIMD_INLINE a_halves halves_dm(simd q,
+ simd d0,
+ simd m0,
+ simd d1,
+ simd m1,
+ const float bias = 0.0f) {
+ simd c = byte_codes(q);
+ if (bias != 0.0f) {
+ c -= bias;
+ }
+ a_halves v;
+ v.template select(0) =
+ scale_min_half(c.template select(0), d0, m0);
+ v.template select(GROUPED_GEMM_RC * 16) =
+ scale_min_half(c.template select(GROUPED_GEMM_RC * 16), d1, m1);
+ return v;
+}
+
+// a row group's 32 values from grid dwords (byte b of entry e is k = 4e + b) and a sign dword per
+// row (bit k negates k), times d: the expression of the iq3 stages
+ESIMD_INLINE a_halves grid_signed(simd ge,
+ simd signs,
+ simd d) {
+ auto gm = ge.template bit_cast_view();
+ simd sg = signs.template replicate_vs_w_hs(0);
+ simd k(0, 1);
+ k &= 15;
+ a_halves v;
+#pragma unroll
+ for (int h = 0; h < 2; ++h) {
+ simd eh = gm.template select(0, 4 * h);
+ simd f = eh.template bit_cast_view();
+ f.merge(-f, ((sg >> (k + 16 * h)) & 1) != 0);
+ v.template select(h * GROUPED_GEMM_RC * 16) = scale_half(f, d);
+ }
+ return v;
+}
+} // namespace grouped_gemm_esimd
+
+template struct grouped_gemm_esimd_a {
+ static constexpr bool supported = false;
+};
+
+// [qs 32][d], one block a k step
+template struct grouped_gemm_esimd_a {
+ static constexpr bool supported = true;
+ static constexpr int qk = QK8_0;
+ static constexpr bool whole = true; // decode_all() for the whole tile
+ static constexpr int slm_bytes = 0;
+ static ESIMD_INLINE void init_slm() {}
+ struct raw {
+ grouped_gemm_esimd::simd q;
+ grouped_gemm_esimd::simd d;
+ };
+
+ static constexpr grouped_gemm_esimd::field QS = { 0, QK8_0, 2 }, D = { QK8_0, 2, 0 };
+ static bool ok(const int K, const bool aos) { return K % qk == 0 && (aos || K >= 64); }
+ struct sb_t {};
+ template static ESIMD_INLINE void load_sb(sb_t &, const Sl &, const int) {}
+ template static ESIMD_INLINE void load(raw & r, sb_t &, const Sl & s, const int kb, const int) {
+ r.q = s.template rows<8>(QS, kb, 0);
+ r.d = s.u16(D, kb, 0);
+ }
+ // all MR x 32 quants at once: int8 -> half, times the row's scale. q * d needs at most 19
+ // significant bits, so the f32 product the joint_matrix stage rounds to half is exact and one
+ // correctly rounded half multiply gives the same bits.
+ static ESIMD_INLINE grouped_gemm_esimd::simd decode_all(raw & r) {
+ using namespace grouped_gemm_esimd;
+ simd av = r.q.template bit_cast_view();
+ simd dh = r.d.template bit_cast_view();
+ av *= dh.template replicate_vs_w_hs(0);
+ return av;
+ }
+};
+
+// [qs 16][d], one block a k step
+template struct grouped_gemm_esimd_a {
+ static constexpr bool supported = true;
+ static constexpr int qk = QK4_NL;
+ static constexpr bool whole = false;
+ static constexpr int slm_bytes = 256 * sizeof(uint32_t);
+
+ static ESIMD_INLINE void init_slm() { grouped_gemm_esimd::iq4_pair_table_to_slm(); }
+ struct raw {
+ grouped_gemm_esimd::simd q;
+ grouped_gemm_esimd::simd d;
+ };
+
+ static constexpr grouped_gemm_esimd::field QS = { 0, QK4_NL / 2, 2 }, D = { QK4_NL / 2, 2, 0 };
+ static bool ok(const int K, const bool aos) { return K % qk == 0 && (aos || K / 2 >= 64); }
+ struct sb_t {};
+ template static ESIMD_INLINE void load_sb(sb_t &, const Sl &, const int) {}
+ template static ESIMD_INLINE void load(raw & r, sb_t &, const Sl & s, const int kb, const int) {
+ using namespace grouped_gemm_esimd;
+ r.q = s.template rows<4>(QS, kb, 0);
+ simd d = s.u16(D, kb, 0);
+ r.d = d.template bit_cast_view();
+ }
+ // the codebook values are integers below 128, exact in half, and d * value has at most 18
+ // significant bits: one correctly rounded half multiply is the stage's f32 product rounded
+ template static ESIMD_INLINE grouped_gemm_esimd::a_halves decode(raw & r) {
+ using namespace grouped_gemm_esimd;
+ a_halves v = iq4_codes(r.q.template select(g * GROUPED_GEMM_RC * 4));
+ simd dg = r.d.template select(g * GROUPED_GEMM_RC);
+ simd dr = dg.template replicate_vs_w_hs(0);
+ v.template select(0) *= dr;
+ v.template select(GROUPED_GEMM_RC * 16) *= dr;
+ return v;
+ }
+};
+
+// [qs 128][d 2][scales_h 2][scales_l 4] per 256 values; k step ib8 of a superblock is sub-block ib8
+template struct grouped_gemm_esimd_a {
+ static constexpr bool supported = true;
+ static constexpr int qk = QK_K;
+ static constexpr bool whole = false;
+ // int16 table and f32 products: d * (scale - 32) * value can need 24 significant bits, more
+ // than a half multiply keeps
+ static constexpr int slm_bytes = 256 * sizeof(uint32_t);
+
+ static ESIMD_INLINE void init_slm() { grouped_gemm_esimd::iq4_pair_table_to_slm(); }
+ struct raw {
+ grouped_gemm_esimd::simd q;
+ grouped_gemm_esimd::simd d;
+ };
+ struct sb_t {
+ grouped_gemm_esimd::simd d;
+ grouped_gemm_esimd::simd sh; // scales_h
+ grouped_gemm_esimd::simd sl; // scales_l, 4 bytes
+ };
+
+ static constexpr grouped_gemm_esimd::field QS = { 0, QK_K / 2, 8 }, D = { QK_K / 2, 2, 0 },
+ SH = { QK_K / 2 + 2, 2, 2 }, SL = { QK_K / 2 + 4, 4, 4 };
+ static bool ok(const int K, const bool) { return K % qk == 0; }
+ template static ESIMD_INLINE void load_sb(sb_t & sb, const Sl & s, const int ib) {
+ using namespace grouped_gemm_esimd;
+ simd dh = s.u16(D, ib, 0);
+ sb.d = simd(dh.template bit_cast_view());
+ sb.sh = s.u16(SH, ib, 0);
+ sb.sl = s.u32(SL, ib, 0);
+ }
+ template static ESIMD_INLINE void load(raw & r, sb_t & sb, const Sl & s, const int kb, const int j) {
+ using namespace grouped_gemm_esimd;
+ r.q = s.template rows<4>(QS, kb / 8, 16 * j);
+ // d * (6-bit scale - 32), in f32 as the stage computes it
+ simd sc = simd(((sb.sl >> (4 * j)) & 0xf) | (((sb.sh >> (2 * j)) & 3) << 4)) - 32;
+ r.d = sb.d * simd(sc);
+ }
+
+ template static ESIMD_INLINE grouped_gemm_esimd::a_halves decode(raw & r) {
+ using namespace grouped_gemm_esimd;
+ simd c =
+ iq4_codes(r.q.template select(g * GROUPED_GEMM_RC * 4));
+ simd dg = r.d.template select(g * GROUPED_GEMM_RC);
+ a_halves v;
+ v.template select(0) = scale_half(c.template select(0), dg);
+ v.template select(GROUPED_GEMM_RC * 16) =
+ scale_half(c.template select(GROUPED_GEMM_RC * 16), dg);
+ return v;
+ }
+};
+
+// [qs 64][qh 8][signs 32][d 2, scales 4] per 256 values. Value k of sub-block ib8 is byte k % 4 of
+// grid entry k / 4, the entry index being qs[k / 4] with bit 8 from qh bit k / 4, negated where bit
+// k of the sub-block's sign dword is set; times d * (1 + 2 * 4-bit scale)
+template struct grouped_gemm_esimd_a {
+ static constexpr bool supported = true;
+ static constexpr int qk = QK_K;
+ static constexpr bool whole = false;
+ // the 512-entry grid in SLM, read by every lane: the global gather of it cost 2.4x the kernel
+ static constexpr int slm_bytes = 512 * sizeof(uint32_t);
+ static ESIMD_INLINE void init_slm() {
+ using namespace grouped_gemm_esimd;
+#pragma unroll
+ for (int i = 0; i < 512; i += 64) {
+ slm_block_store(i * 4, block_load(iq3s_grid + i));
+ }
+ }
+ struct raw {
+ grouped_gemm_esimd::simd q; // [row][8 qs bytes]
+ grouped_gemm_esimd::simd qh; // the sub-block's qh byte
+ grouped_gemm_esimd::simd signs; // the sub-block's 32 sign bits
+ grouped_gemm_esimd::simd d;
+ };
+ // d and the scale bytes are a block's, kept over its k steps; the sign dword and the qh byte of
+ // a k step are a 2D load each at that step's column, so no register is picked by a runtime index
+ struct sb_t {
+ grouped_gemm_esimd::simd d;
+ grouped_gemm_esimd::simd sc; // the 4 scale bytes
+ };
+ // the SoA stream {d, scales} is d and scales of the block in AoS, which are not adjacent there: MS
+ // reads the scales at their SoA offset 2 in the stream, which is AoS offset 2 past 104
+ static constexpr int META = QK_K / 4 + QK_K / 32 + QK_K / 8;
+ static constexpr grouped_gemm_esimd::field QS = { 0, QK_K / 4, 2 }, QH = { QK_K / 4, QK_K / 32, 2 + QK_K / 4 },
+ SG = { QK_K / 4 + QK_K / 32, QK_K / 8, 2 + QK_K / 4 + QK_K / 32 },
+ MD = { META, 2 + IQ3S_N_SCALE, 0 },
+ MS = { META, 2 + IQ3S_N_SCALE, META };
+ // the SoA qh and signs streams are 2D surfaces: their row pitches must be 16-byte aligned
+ static bool ok(const int K, const bool aos) {
+ return K % qk == 0 && (aos || ((K / QK_K * 8) % 16 == 0 && K / QK_K * 8 >= 64));
+ }
+ template static ESIMD_INLINE void load_sb(sb_t & sb, const Sl & s, const int ib) {
+ using namespace grouped_gemm_esimd;
+ simd dh = s.u16(MD, ib, 0);
+ simd s0 = s.u16(MS, ib, 2);
+ simd s1 = s.u16(MS, ib, 4);
+ sb.d = simd(dh.template bit_cast_view());
+ sb.sc = s0 | (s1 << 16);
+ }
+ template static ESIMD_INLINE void load(raw & r, sb_t & sb, const Sl & s, const int kb, const int j) {
+ using namespace grouped_gemm_esimd;
+ const int ib = kb / 8;
+ r.q = s.template rows<2>(QS, ib, 8 * j);
+ r.qh = (s.template rows<1>(QH, ib, 4 * (j / 4)) >> (8 * (j % 4))) & 0xffu;
+ r.signs = s.template rows<1>(SG, ib, 4 * j);
+ r.d = sb.d * simd(simd(1 + 2 * ((sb.sc >> (4 * j)) & 0xf)));
+ }
+
+ template static ESIMD_INLINE grouped_gemm_esimd::a_halves decode(raw & r) {
+ using namespace grouped_gemm_esimd;
+ // the 8 grid entries of each of the row group's rows
+ simd qb =
+ r.q.template select(g * GROUPED_GEMM_RC * 2).template bit_cast_view();
+ simd qr = r.qh.template select(g * GROUPED_GEMM_RC);
+ simd qh = qr.template replicate_vs_w_hs(0);
+ simd j(0, 1);
+ j &= 7;
+ simd idx = simd(qb) | (((qh >> j) & 1) << 8);
+ simd ge = slm_gather(idx * 4);
+ return grid_signed(ge, r.signs.template select(g * GROUPED_GEMM_RC),
+ r.d.template select(g * GROUPED_GEMM_RC));
+ }
+};
+
+// The K-quants and iq3_xxs: a 256-value superblock, k step j its sub-block j. Their value bytes for
+// a k step are one rectangle of a stream, in k order after a shift and a mask per dword; scales,
+// mins and d are the block's and are loaded once per block.
+
+// [qs 128][scales 12][dm]: value l of sub-block j is nibble j & 1 of byte 32 (j / 2) + l, times
+// dall * scale_j, minus dmin * min_j (6-bit scale and min, get_scale_min_k4)
+template struct grouped_gemm_esimd_a {
+ static constexpr bool supported = true;
+ static constexpr int qk = QK_K;
+ static constexpr bool whole = false;
+ static constexpr int slm_bytes = 0;
+ static ESIMD_INLINE void init_slm() {}
+ struct raw {
+ grouped_gemm_esimd::simd q;
+ grouped_gemm_esimd::simd d, m;
+ };
+ struct sb_t {
+ grouped_gemm_esimd::simd sc[3]; // the 12 scale bytes
+ grouped_gemm_esimd::simd dall, dmin;
+ };
+
+ static constexpr grouped_gemm_esimd::field QS = { 0, QK_K / 2, 16 }, SC = { QK_K / 2, K_SCALE_SIZE, 4 },
+ DM = { QK_K / 2 + K_SCALE_SIZE, 4, 0 };
+
+ static bool ok(const int K, const bool aos) {
+ return K % qk == 0 && (aos || grouped_gemm_esimd::surface_ok(K / 2));
+ }
+
+ template
+ static ESIMD_INLINE void load_sb(sb_t & sb,
+ const Sl & s,
+ const int ib,
+ const grouped_gemm_esimd::field & sc = SC,
+ const grouped_gemm_esimd::field & dmf = DM) {
+ using namespace grouped_gemm_esimd;
+#pragma unroll
+ for (int w = 0; w < 3; ++w) {
+ sb.sc[w] = s.u32(sc, ib, 4 * w);
+ }
+ simd dm = s.u32(dmf, ib, 0);
+ auto dh = dm.template bit_cast_view();
+ sb.dall = simd(dh.template select(0));
+ sb.dmin = simd(dh.template select(1));
+ }
+ // get_scale_min_k4 for sub-block j over the lanes
+ static ESIMD_INLINE void scale_min(sb_t & sb,
+ const int j,
+ grouped_gemm_esimd::simd & d,
+ grouped_gemm_esimd::simd & m) {
+ using namespace grouped_gemm_esimd;
+ simd sc, mb;
+ if (j < 4) {
+ sc = byte_of(sb.sc[0], j) & 63;
+ mb = byte_of(sb.sc[1], j) & 63;
+ } else {
+ sc = (byte_of(sb.sc[2], j - 4) & 0xf) | ((byte_of(sb.sc[0], j - 4) >> 6) << 4);
+ mb = (byte_of(sb.sc[2], j - 4) >> 4) | ((byte_of(sb.sc[1], j - 4) >> 6) << 4);
+ }
+ d = sb.dall * simd(sc);
+ m = sb.dmin * simd(mb);
+ }
+ template static ESIMD_INLINE void load(raw & r, sb_t & sb, const Sl & s, const int kb, const int j) {
+ using namespace grouped_gemm_esimd;
+ r.q = (s.template rows<8>(QS, kb / 8, 32 * (j / 2)) >> (4 * (j & 1))) & 0x0F0F0F0Fu;
+ scale_min(sb, j, r.d, r.m);
+ }
+
+ template static ESIMD_INLINE grouped_gemm_esimd::a_halves decode(raw & r) {
+ using namespace grouped_gemm_esimd;
+ simd d = r.d.template select(g * GROUPED_GEMM_RC),
+ m = r.m.template select(g * GROUPED_GEMM_RC);
+ return halves_dm(r.q.template select(g * GROUPED_GEMM_RC * 8), d, m, d, m);
+ }
+};
+
+// [qs 128][qh 32][scales 12][dm]: q4_K plus bit j of qh byte l as 16
+template struct grouped_gemm_esimd_a {
+ using q4 = grouped_gemm_esimd_a;
+ static constexpr bool supported = true;
+ static constexpr int qk = QK_K;
+ static constexpr bool whole = false;
+ static constexpr int slm_bytes = 0;
+ static ESIMD_INLINE void init_slm() {}
+ using raw = typename q4::raw;
+ using sb_t = typename q4::sb_t;
+ static constexpr grouped_gemm_esimd::field QS = { 0, QK_K / 2, 48 }, QH = { QK_K / 2, QK_K / 8, 16 },
+ SC = { QK_K / 2 + QK_K / 8, K_SCALE_SIZE, 4 },
+ DM = { QK_K / 2 + QK_K / 8 + K_SCALE_SIZE, 4, 0 };
+ static bool ok(const int K, const bool aos) {
+ return K % qk == 0 && (aos || (grouped_gemm_esimd::surface_ok(K / 2) && grouped_gemm_esimd::surface_ok(K / 8)));
+ }
+ template static ESIMD_INLINE void load_sb(sb_t & sb, const Sl & s, const int ib) { q4::load_sb(sb, s, ib, SC, DM); }
+ template static ESIMD_INLINE void load(raw & r, sb_t & sb, const Sl & s, const int kb, const int j) {
+ using namespace grouped_gemm_esimd;
+ const int ib = kb / 8;
+ simd qh = s.template rows<8>(QH, ib, 0);
+ r.q = ((s.template rows<8>(QS, ib, 32 * (j / 2)) >> (4 * (j & 1))) & 0x0F0F0F0Fu) |
+ (((qh >> j) & 0x01010101u) << 4);
+ q4::scale_min(sb, j, r.d, r.m);
+ }
+
+ template static ESIMD_INLINE grouped_gemm_esimd::a_halves decode(raw & r) {
+ return q4::template decode(r);
+ }
+};
+
+// [ql 128][qh 64][scales 16][d]: sub-block j is quarter r = j % 4 of half h = j / 4; value l is
+// nibble r >> 1 of ql byte 64h + 32 (r & 1) + l with bits 2r, 2r+1 of qh byte 32h + l above it,
+// minus 32, times d * the int8 scale 2j + l / 16
+template struct grouped_gemm_esimd_a {
+ static constexpr bool supported = true;
+ static constexpr int qk = QK_K;
+ static constexpr bool whole = false;
+ static constexpr int slm_bytes = 0;
+ static ESIMD_INLINE void init_slm() {}
+ struct raw {
+ grouped_gemm_esimd::simd q;
+ grouped_gemm_esimd::simd d0, d1; // per k half
+ };
+ struct sb_t {
+ grouped_gemm_esimd::simd sc[4]; // the 16 int8 scales
+ grouped_gemm_esimd::simd d;
+ };
+
+ static constexpr grouped_gemm_esimd::field QL = { 0, QK_K / 2, 0 }, QH = { QK_K / 2, QK_K / 4, QK_K / 2 },
+ SC = { QK_K / 2 + QK_K / 4, QK_K / 16, QK_K / 2 + QK_K / 4 },
+ D = { QK_K / 2 + QK_K / 4 + QK_K / 16, 2,
+ QK_K / 2 + QK_K / 4 + QK_K / 16 };
+ static bool ok(const int K, const bool aos) {
+ return K % qk == 0 && (aos || (grouped_gemm_esimd::surface_ok(K / 2) && grouped_gemm_esimd::surface_ok(K / 4)));
+ }
+ template static ESIMD_INLINE void load_sb(sb_t & sb, const Sl & s, const int ib) {
+ using namespace grouped_gemm_esimd;
+#pragma unroll
+ for (int w = 0; w < 4; ++w) {
+ sb.sc[w] = s.u32(SC, ib, 4 * w);
+ }
+ simd dh = s.u16(D, ib, 0);
+ sb.d = simd(dh.template bit_cast_view());
+ }
+ template static ESIMD_INLINE void load(raw & r, sb_t & sb, const Sl & s, const int kb, const int j) {
+ using namespace grouped_gemm_esimd;
+ const int ib = kb / 8, h = j / 4, q = j % 4;
+ simd ql = s.template rows<8>(QL, ib, 64 * h + 32 * (q & 1));
+ simd qh = s.template rows<8>(QH, ib, 32 * h);
+ r.q = ((ql >> (4 * (q >> 1))) & 0x0F0F0F0Fu) | (((qh >> (2 * q)) & 0x03030303u) << 4);
+ // scales 2j and 2j + 1: bytes 2 (j % 2) and 2 (j % 2) + 1 of dword j / 2, sign extended
+ simd w = dword_of(sb.sc, j / 2);
+ simd s0 = simd(w << (24 - 16 * (j % 2))) >> 24;
+ simd s1 = simd(w << (16 - 16 * (j % 2))) >> 24;
+ r.d0 = sb.d * simd(s0);
+ r.d1 = sb.d * simd(s1);
+ }
+
+ template static ESIMD_INLINE grouped_gemm_esimd::a_halves decode(raw & r) {
+ using namespace grouped_gemm_esimd;
+ simd z = 0.0f;
+ return halves_dm(r.q.template select(g * GROUPED_GEMM_RC * 8),
+ r.d0.template select(g * GROUPED_GEMM_RC), z,
+ r.d1.template select(g * GROUPED_GEMM_RC), z, 32.0f);
+ }
+};
+
+// [qs 64][scales 16][dm]: value l of sub-block j is bits 2 (j % 4) of byte 32 (j / 4) + l, times
+// d * (scale & 15), minus dmin * (scale >> 4), scale byte 2j + l / 16
+template struct grouped_gemm_esimd_a