sycl: Improve Grouped XMX GEMM by avoiding SLM staging (#31428)
This commit is contained in:
1 parent
7476e47261
commit
418fe97b5f
8 files changed
+1299
-29
No files matched your search
@@ -827,6 +827,7 @@ User can use the device management in [docs/multi-gpu.md](https://github.com/ggm
|
||||
| GGML_SYCL_XMX_MAX_COLS | non-negative integer, 80 (default) | Most activation columns (batch size) the `GGML_SYCL_ENABLE_XMX_ESIMD` kernels take; wider batches use the other mat-mul kernels. A larger value keeps XMX for bigger batches (as repeated launches), `0` never uses it. |
|
||||
| GGML_SYCL_XMX_GLU_MAX_COLS | 0 to 16, 16 (default) | Most columns the fused gate and up XMX kernel takes; wider batches run gate and up as separate mat-vecs. 16 is the most the kernel is built for, larger values are clamped to 16 and `0` turns the fused kernel off. |
|
||||
| GGML_SYCL_XMX_GATHER_TYPES | decimal bitmask, all bits set (default) | Weight formats that may use the XMX dequant-GEMM paths, which dequantize weights straight into the XMX tiles. This speeds up prompt processing of MoE models on GPUs with XMX units (Arc A- and B-series, Arc Pro, Data Center GPU Max), for example pp512 of Qwen3-30B-A3B UD-IQ3_XXS by about 50% on an Arc Pro B60. Bits:<br>* 1: IQ4_NL, 2: IQ3_S, 4: IQ4_XS, 8: IQ3_XXS, 16: IQ2_XXS, 32: IQ2_XS, 64: IQ2_S, 128: IQ1_S, 256: IQ1_M<br>* 512: Q8_0, 1024: Q4_K, 2048: Q5_K, 4096: Q6_K, 8192: Q2_K, 16384: Q3_K (MoE `MUL_MAT_ID` only)<br>Add values to combine them, for example `3` for IQ4_NL and IQ3_S; `0` disables the paths. A set bit does not force the path: batches of more than 64 tokens per expert use the library GEMM unless the device schedule width heuristic is overridden with `GGML_SYCL_MMID_SCHED=3`. Row lengths must be a multiple of 256 (32 for IQ4_NL and Q8_0). |
|
||||
| GGML_SYCL_MMID_ESIMD | 0, 1 (default), 2 | The `MUL_MAT_ID` grouped XMX dequant-GEMM as an ESIMD kernel that dequantizes the weights straight into the `dpas` register operands, with no shared local memory image of them, from either weight layout (the per-expert reorder or the canonical blocks). q8_0, iq4_nl, iq4_xs, iq3_s, iq3_xxs and q2_K to q6_K. It stands in for the 8x16x16 sub-group-16 combination of f16 or bf16 (Xe2, Xe-HPC) and for the 8x8x16 sub-group-8 one (Xe-HPG: Arc A-series, ARL-H; there it works on 16 rows a thread and reads with 1D block loads, since that architecture has half the register file and no 2D block loads); while it is on, a grouped call whose weight type it covers is moved to that combination from any other shape of its type and sub-group size, whatever `GGML_SYCL_XMX_GATHER_SHAPES` allows, because the shape bits choose among `joint_matrix` tiles and the ESIMD kernel tiles on its own. A call it declines runs that combination's `joint_matrix` kernel. 0 uses the `joint_matrix` kernel, 1 the ESIMD one, 2 the ESIMD one summing K in the `joint_matrix` kernel's order (bit-identical output for f16; slower). An AOT build gets its 256-register file through `-ze-opt-enable-auto-large-GRF-mode` (set by the CMake build). Not built with `GGML_SYCL_SUPPORT_XMX=OFF`. |
|
||||
| GGML_SYCL_MMID_SCHED | decimal bitmask, 1 (default) | Build the grouped MoE routing and tile schedule on the GPU, avoiding the expert-ID host readback and CPU sort.<br>* 1: enable the device schedule.<br>* 2: override the fallback for models with few routed experts by disabling the performance heuristic that rejects batches averaging more than 64 routed rows per expert. This may reduce performance; type, shape, precision and device capability checks still apply.<br>Use `3` to enable the device schedule and override this heuristic, or `0` to use the host schedule. Bit 2 alone does not enable the device schedule. Higher bits are ignored. |
|
||||
| GGML_SYCL_XMX_GATHER_SHAPES | decimal bitmask, 255 (default) | XMX `joint_matrix` combinations the paths of `GGML_SYCL_XMX_GATHER_TYPES` may use; the operand type comes from `GGML_SYCL_DYNAMIC_PRECISION` and the best supported combination is picked automatically (logged as `fg_pick_combo`). Bits:<br>* Xe2, Xe3, Xe-HPC: 1: f16 8x16x16, 2: f16 16x16x16, 4: f16 32x64x16, 8: f16 32x64x32, 32: tf32 8x16x8, 64: bf16 8x16x16<br>* Xe-HPG (Arc A770, ARL-H): 16: f16 8x8x16, 128: bf16 8x8x16<br>Clear a bit to exclude a combination, or set a single bit to force one for testing. |
|
||||
| GGML_SYCL_DYNAMIC_PRECISION | `F16` (default with `GGML_SYCL_F16=ON`), `BF16`, `TF32` or `F32` (default otherwise) | Operand type of the XMX dequant-GEMM paths (`GGML_SYCL_XMX_GATHER_TYPES`); accumulation is always f32. `F16` is the fastest, but activations above 65504 overflow. `BF16` keeps the f32 range at a 7-bit mantissa, `TF32` keeps the range and the f16 mantissa but is about 30% slower and needs Xe2, Xe3 or Xe-HPC, and `F32` turns the XMX paths off. Ops that request a higher src1 precision ([TAG_GGML_PREC]) get it regardless of this setting. |
|
||||
|
||||
@@ -218,6 +218,8 @@ endif()
|
||||
|
||||
if (GGML_SYCL_TARGET STREQUAL "INTEL")
|
||||
add_compile_definitions(GGML_SYCL_WARP_SIZE=16)
|
||||
# With AOT the option goes to ocloc inside -options (see GGML_SYCL_DEVICE_ARCH below): a bare
|
||||
# -Xs hands it to ocloc as a command-line flag, which fails the device link.
|
||||
if (NOT GGML_SYCL_DEVICE_ARCH)
|
||||
target_link_options(ggml-sycl PRIVATE -Xs -ze-intel-greater-than-4GB-buffer-required)
|
||||
else()
|
||||
@@ -248,10 +250,18 @@ endif()
|
||||
|
||||
if (GGML_SYCL_DEVICE_ARCH)
|
||||
message(STATUS "GGML_SYCL_DEVICE_ARCH=${GGML_SYCL_DEVICE_ARCH} (AOT via spir64_gen)")
|
||||
# AOT (oneAPI 2026.2) ignores the grf_size<256> kernel property on ESIMD kernels and builds them
|
||||
# with 128 registers, which made the ESIMD MoE GEMM 1.7-3.9x slower than under JIT; SPMD kernels
|
||||
# keep honouring the property. A module-wide -doubleGRF also moves the ESIMD kernels that ask for
|
||||
# 128 to 256, and a server built with it hung on its first request, so IGC picks the size per
|
||||
# kernel from its register pressure instead (ESIMD kernels too; -ze-opt-large-grf-kernel does not
|
||||
# reach them).
|
||||
set(GGML_SYCL_AOT_BACKEND_OPTIONS
|
||||
"-device ${GGML_SYCL_DEVICE_ARCH} -options '-ze-intel-greater-than-4GB-buffer-required -ze-opt-enable-auto-large-GRF-mode'")
|
||||
target_compile_options(
|
||||
ggml-sycl PRIVATE
|
||||
-fsycl-targets=spir64_gen
|
||||
"SHELL:-Xsycl-target-backend=spir64_gen \"-device ${GGML_SYCL_DEVICE_ARCH} -options -ze-intel-greater-than-4GB-buffer-required\""
|
||||
"SHELL:-Xsycl-target-backend=spir64_gen \"${GGML_SYCL_AOT_BACKEND_OPTIONS}\""
|
||||
)
|
||||
|
||||
# Pass through parallel job (process) count for parallelising the
|
||||
@@ -266,7 +276,7 @@ if (GGML_SYCL_DEVICE_ARCH)
|
||||
target_link_options(
|
||||
ggml-sycl PRIVATE
|
||||
-fsycl-targets=spir64_gen
|
||||
"SHELL:-Xsycl-target-backend=spir64_gen \"-device ${GGML_SYCL_DEVICE_ARCH} -options -ze-intel-greater-than-4GB-buffer-required\""
|
||||
"SHELL:-Xsycl-target-backend=spir64_gen \"${GGML_SYCL_AOT_BACKEND_OPTIONS}\""
|
||||
-fsycl-max-parallel-link-jobs=${GGML_SYCL_MAX_PARALLEL_LINK_JOBS}
|
||||
)
|
||||
|
||||
|
||||
@@ -72,6 +72,7 @@ extern int g_ggml_sycl_debug;
|
||||
extern int g_ggml_sycl_enable_optimize;
|
||||
extern int g_ggml_sycl_enable_fusion;
|
||||
extern int g_ggml_sycl_enable_esimd;
|
||||
extern int g_ggml_sycl_mmid_esimd;
|
||||
extern int g_ggml_sycl_mmvq_wide;
|
||||
extern int g_ggml_sycl_enable_xmx_esimd;
|
||||
extern int g_ggml_sycl_xmx_max_cols;
|
||||
|
||||
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,33 @@
|
||||
#ifndef GGML_SYCL_FUSED_GEMM_ESIMD_HPP
|
||||
#define GGML_SYCL_FUSED_GEMM_ESIMD_HPP
|
||||
|
||||
#include "fused-gemm.hpp"
|
||||
|
||||
// The grouped (MUL_MAT_ID) dequant GEMM, written with ESIMD so the dequantized weights go from
|
||||
// registers straight into the dpas A operand: no SLM, no barrier. Same inputs and outputs as the
|
||||
// joint_matrix grouped kernel of the 8 x NW x 16 sub-group-NW combination of T (sycl::half or
|
||||
// bfloat16): NW = 16 on Xe2 and Xe-HPC, 8 on Xe-HPG (Arc A, ARL-H). The schedule's tiles, B packed as
|
||||
// [K/2][Npad][2] T, and dst through dst_rows. The weight is in the reorder (SoA) layout if
|
||||
// `reordered`, else in the canonical block layout. Returns false when the shape is not covered.
|
||||
template <typename block_q_t>
|
||||
constexpr bool ggml_sycl_grouped_gemm_esimd_has = std::is_same_v<block_q_t, block_q8_0> ||
|
||||
std::is_same_v<block_q_t, block_iq4_nl> ||
|
||||
std::is_same_v<block_q_t, block_iq4_xs> ||
|
||||
std::is_same_v<block_q_t, block_iq3_s> ||
|
||||
std::is_same_v<block_q_t, block_iq3_xxs> ||
|
||||
std::is_same_v<block_q_t, block_q2_K> ||
|
||||
std::is_same_v<block_q_t, block_q3_K> ||
|
||||
std::is_same_v<block_q_t, block_q4_K> ||
|
||||
std::is_same_v<block_q_t, block_q5_K> ||
|
||||
std::is_same_v<block_q_t, block_q6_K>;
|
||||
|
||||
template <typename block_q_t, typename T, int NW>
|
||||
bool ggml_sycl_grouped_gemm_esimd(const char * src0_dd, size_t expert_stride, const ggml_sycl_grouped_gemm_tile * tiles,
|
||||
const T * packed, float * dst, int M, int Npad, int K,
|
||||
const ggml_sycl_grouped_gemm_rows & dst_rows, bool reordered, int64_t n_tiles,
|
||||
dpct::queue_ptr stream);
|
||||
|
||||
// GGML_SYCL_MMID_ESIMD: 1 (default) takes the ESIMD kernel above where it applies, 0 never does
|
||||
bool ggml_sycl_grouped_gemm_esimd_enabled();
|
||||
|
||||
#endif // GGML_SYCL_FUSED_GEMM_ESIMD_HPP
|
||||
@@ -1,4 +1,5 @@
|
||||
#include "fused-gemm.hpp"
|
||||
#include "fused-gemm-esimd.hpp"
|
||||
|
||||
#include <sycl/ext/oneapi/matrix/matrix.hpp>
|
||||
|
||||
@@ -17,7 +18,6 @@ namespace mx = sycl::ext::oneapi::experimental::matrix;
|
||||
// superblock into steps of this width. The sub-groups of a work-group each walk their own K range
|
||||
// and are summed at the end.
|
||||
static constexpr int FG_BK = QK4_NL;
|
||||
static constexpr int FG_KSPLIT = 4;
|
||||
|
||||
// Element traits of one joint_matrix operand type. The A stage and the B pack compute in f32 and
|
||||
// convert once, in registers, when they write the element, so any type costs the same one pass.
|
||||
@@ -113,6 +113,15 @@ using fg_half = sycl::half;
|
||||
using fg_bf16 = sycl::ext::oneapi::bfloat16;
|
||||
using fg_tf32 = mx::precision::tf32;
|
||||
|
||||
// The combinations the ESIMD grouped GEMM (fused-gemm-esimd.cpp) stands in for: 8 x SG x 16 with f16
|
||||
// or bf16 operands, the shape of one dpas, on sub-group 16 (Xe2, Xe-HPC) or 8 (Xe-HPG). Its B is
|
||||
// packed and tiled as theirs is.
|
||||
template <typename S> static constexpr bool fg_esimd_combo() {
|
||||
return S::TM == 8 && S::TN == S::SG && S::TK == 16 && (S::SG == 16 || S::SG == 8) &&
|
||||
std::is_same_v<typename S::ta, typename S::tb> &&
|
||||
(std::is_same_v<typename S::ta, fg_half> || std::is_same_v<typename S::ta, fg_bf16>);
|
||||
}
|
||||
|
||||
// One bit of GGML_SYCL_XMX_GATHER_SHAPES per combination. Only combinations some device lists in
|
||||
// matrix_combinations are built (appendix of sycl_ext_oneapi_matrix and the runtime's own list).
|
||||
template <typename F> static void fg_visit_combo(int idx, F && f);
|
||||
@@ -152,29 +161,7 @@ template <typename F> static void fg_visit_combo(int idx, F && f) {
|
||||
// size, since IGC fails on the other ones. A JIT build keeps them all, but each combination lands in
|
||||
// its own device image (joint_matrix is an optional kernel feature) and only a combination the
|
||||
// device reports is launched, so the runtime never asks IGC for the others.
|
||||
#if defined(__SYCL_DEVICE_ONLY__)
|
||||
# if __SYCL_TARGET_INTEL_GPU_ACM_G10__ || __SYCL_TARGET_INTEL_GPU_ACM_G11__ || __SYCL_TARGET_INTEL_GPU_ACM_G12__ || \
|
||||
__SYCL_TARGET_INTEL_GPU_ARL_H__
|
||||
# define FG_AOT_SG 8
|
||||
# elif __SYCL_TARGET_INTEL_GPU_PVC__ || __SYCL_TARGET_INTEL_GPU_PVC_VG__ || __SYCL_TARGET_INTEL_GPU_BMG_G21__ || \
|
||||
__SYCL_TARGET_INTEL_GPU_BMG_G31__ || __SYCL_TARGET_INTEL_GPU_LNL_M__ || __SYCL_TARGET_INTEL_GPU_PTL_H__ || \
|
||||
__SYCL_TARGET_INTEL_GPU_PTL_U__ || __SYCL_TARGET_INTEL_GPU_WCL__ || __SYCL_TARGET_INTEL_GPU_NVL_S__ || \
|
||||
__SYCL_TARGET_INTEL_GPU_NVL_U__ || __SYCL_TARGET_INTEL_GPU_NVL_P__
|
||||
# define FG_AOT_SG 16
|
||||
# elif __SYCL_TARGET_INTEL_GPU_TGLLP__ || __SYCL_TARGET_INTEL_GPU_RKL__ || __SYCL_TARGET_INTEL_GPU_ADL_S__ || \
|
||||
__SYCL_TARGET_INTEL_GPU_ADL_P__ || __SYCL_TARGET_INTEL_GPU_ADL_N__ || __SYCL_TARGET_INTEL_GPU_DG1__ || \
|
||||
__SYCL_TARGET_INTEL_GPU_MTL_U__ || __SYCL_TARGET_INTEL_GPU_MTL_H__
|
||||
# define FG_AOT_SG 0 // no XMX
|
||||
# endif
|
||||
#endif
|
||||
|
||||
template <int SG> static constexpr bool fg_built() {
|
||||
#if defined(FG_AOT_SG)
|
||||
return SG == FG_AOT_SG;
|
||||
#else
|
||||
return true;
|
||||
#endif
|
||||
}
|
||||
// (FG_AOT_SG is defined in fused-gemm.hpp, which the ESIMD kernels share)
|
||||
|
||||
// Upper bound on the tile count when total_rows rows are routed to n_as experts: the worst case gives
|
||||
// each expert one row and fills whole tiles with the rest. The bound depends only on the shape, not
|
||||
@@ -991,6 +978,15 @@ static void grouped_dequant_gemm_launch(const char * src0_dd, const size_t exper
|
||||
float * dst, const int M, const int Npad, const int K, const int64_t n_tiles,
|
||||
const int64_t groups_m, dpct::queue_ptr stream,
|
||||
ggml_sycl_grouped_gemm_rows dst_rows = {}) {
|
||||
if constexpr (fg_esimd_combo<S>() && ggml_sycl_grouped_gemm_esimd_has<block_q_t>) {
|
||||
// the register-operand ESIMD kernel, either weight layout: no SLM A image (GGML_SYCL_MMID_ESIMD)
|
||||
if (ggml_sycl_grouped_gemm_esimd_enabled() &&
|
||||
ggml_sycl_grouped_gemm_esimd<block_q_t, typename S::tsb, S::SG>(src0_dd, expert_stride, tiles_ptr, packed,
|
||||
dst, M, Npad, K, dst_rows, reordered,
|
||||
n_tiles, stream)) {
|
||||
return;
|
||||
}
|
||||
}
|
||||
stream->submit([&](sycl::handler & cgh) {
|
||||
sycl::local_accessor<typename S::tsa, 1> tile_a(FG_KSPLIT * S::SG_ROWS * FG_BK, cgh);
|
||||
sycl::local_accessor<float, 1> tile_c(FG_KSPLIT * S::SG_ROWS * S::BN, cgh);
|
||||
@@ -1087,6 +1083,45 @@ template <typename F> static bool fg_visit_type(ggml_type type, bool reordered,
|
||||
}
|
||||
}
|
||||
|
||||
// The ESIMD grouped GEMM loads A straight into its dpas operands, so the joint_matrix tile shape
|
||||
// means nothing to it; it runs on the fg_esimd_combo of the picked operand type and sub-group size. A
|
||||
// grouped call whose weight it covers is moved there from any other shape of that type and sub-group
|
||||
// size, if the device lists it, whatever GGML_SYCL_XMX_GATHER_SHAPES allows: the shape bits choose
|
||||
// among joint_matrix tiles.
|
||||
// A call the ESIMD kernel then declines (a shape it does not take) runs that combination's joint_matrix
|
||||
// kernel.
|
||||
static int fg_esimd_remap(dpct::queue_ptr stream, int combo, ggml_type src0_type, bool reordered) {
|
||||
if (combo < 0 || !ggml_sycl_grouped_gemm_esimd_enabled()) {
|
||||
return combo;
|
||||
}
|
||||
bool covered = false;
|
||||
fg_visit_type(src0_type, reordered, [&](auto tag) {
|
||||
using T = decltype(tag);
|
||||
covered = ggml_sycl_grouped_gemm_esimd_has<typename T::type>;
|
||||
});
|
||||
if (!covered) {
|
||||
return combo;
|
||||
}
|
||||
int target = -1;
|
||||
fg_visit_combo(combo, [&](auto s) {
|
||||
using S = decltype(s);
|
||||
if constexpr (std::is_same_v<typename S::ta, typename S::tb>) {
|
||||
for (int idx = 0; idx < FG_N_COMBOS; ++idx) {
|
||||
fg_visit_combo(idx, [&](auto t) {
|
||||
using U = decltype(t);
|
||||
if constexpr (fg_esimd_combo<U>() && U::SG == S::SG && std::is_same_v<typename U::ta, typename S::ta>) {
|
||||
target = idx;
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
});
|
||||
if (target < 0 || target == combo || !(fg_device_combos(stream->get_device()) & (1 << target))) {
|
||||
return combo;
|
||||
}
|
||||
return target;
|
||||
}
|
||||
|
||||
template <typename S>
|
||||
static bool fg_fused_run(ggml_type src0_type, bool src0_reordered, const void * src0, const void * src1, ggml_type src1_type,
|
||||
float * dst, int64_t M, int64_t N, int64_t K, int64_t ldd, ggml_sycl_pool & pool,
|
||||
@@ -1201,7 +1236,7 @@ bool ggml_sycl_grouped_dequant_gemm(ggml_type src0_type, bool reordered, const v
|
||||
if (!ggml_sycl_grouped_dequant_gemm_shape_ok(src0_type, M, K, total_rows, n_active)) {
|
||||
return false;
|
||||
}
|
||||
const int combo = fg_pick_combo(stream, GGML_TYPE_F32, src1_prec);
|
||||
const int combo = fg_esimd_remap(stream, fg_pick_combo(stream, GGML_TYPE_F32, src1_prec), src0_type, reordered);
|
||||
if (combo < 0) {
|
||||
return false;
|
||||
}
|
||||
@@ -1407,7 +1442,7 @@ bool ggml_sycl_grouped_dequant_gemm_device_supported(ggml_type src0_type, bool r
|
||||
if (grouped_gemm_max_tiles(total_rows, n_as, GGML_SYCL_FG_BN) > INT32_MAX / GGML_SYCL_FG_BN) {
|
||||
return false;
|
||||
}
|
||||
const int combo = fg_pick_combo(stream, GGML_TYPE_F32, src1_prec);
|
||||
const int combo = fg_esimd_remap(stream, fg_pick_combo(stream, GGML_TYPE_F32, src1_prec), src0_type, reordered);
|
||||
if (combo < 0) {
|
||||
return false;
|
||||
}
|
||||
@@ -1428,7 +1463,7 @@ bool ggml_sycl_grouped_dequant_gemm_device(ggml_type src0_type, bool reordered,
|
||||
n_tokens, M, K, stream)) {
|
||||
return false;
|
||||
}
|
||||
const int combo = fg_pick_combo(stream, GGML_TYPE_F32, src1_prec);
|
||||
const int combo = fg_esimd_remap(stream, fg_pick_combo(stream, GGML_TYPE_F32, src1_prec), src0_type, reordered);
|
||||
bool launched = false;
|
||||
fg_visit_combo(combo, [&](auto s) {
|
||||
launched = fg_grouped_device_run<decltype(s)>(src0_type, reordered, src0_base, expert_stride, src1, dst,
|
||||
|
||||
@@ -13,7 +13,34 @@ struct ggml_sycl_grouped_gemm_rows {
|
||||
size_t nb2;
|
||||
};
|
||||
|
||||
// AOT: the sub-group size of the XMX combinations of the target, 0 for a target without XMX.
|
||||
// Undefined in a JIT build, which keeps every combination.
|
||||
#if defined(__SYCL_DEVICE_ONLY__)
|
||||
# if __SYCL_TARGET_INTEL_GPU_ACM_G10__ || __SYCL_TARGET_INTEL_GPU_ACM_G11__ || __SYCL_TARGET_INTEL_GPU_ACM_G12__ || \
|
||||
__SYCL_TARGET_INTEL_GPU_ARL_H__
|
||||
# define FG_AOT_SG 8
|
||||
# elif __SYCL_TARGET_INTEL_GPU_PVC__ || __SYCL_TARGET_INTEL_GPU_PVC_VG__ || __SYCL_TARGET_INTEL_GPU_BMG_G21__ || \
|
||||
__SYCL_TARGET_INTEL_GPU_BMG_G31__ || __SYCL_TARGET_INTEL_GPU_LNL_M__ || __SYCL_TARGET_INTEL_GPU_PTL_H__ || \
|
||||
__SYCL_TARGET_INTEL_GPU_PTL_U__ || __SYCL_TARGET_INTEL_GPU_WCL__ || __SYCL_TARGET_INTEL_GPU_NVL_S__ || \
|
||||
__SYCL_TARGET_INTEL_GPU_NVL_U__ || __SYCL_TARGET_INTEL_GPU_NVL_P__
|
||||
# define FG_AOT_SG 16
|
||||
# elif __SYCL_TARGET_INTEL_GPU_TGLLP__ || __SYCL_TARGET_INTEL_GPU_RKL__ || __SYCL_TARGET_INTEL_GPU_ADL_S__ || \
|
||||
__SYCL_TARGET_INTEL_GPU_ADL_P__ || __SYCL_TARGET_INTEL_GPU_ADL_N__ || __SYCL_TARGET_INTEL_GPU_DG1__ || \
|
||||
__SYCL_TARGET_INTEL_GPU_MTL_U__ || __SYCL_TARGET_INTEL_GPU_MTL_H__
|
||||
# define FG_AOT_SG 0 // no XMX
|
||||
# endif
|
||||
#endif
|
||||
|
||||
template <int SG> static constexpr bool fg_built() {
|
||||
#if defined(FG_AOT_SG)
|
||||
return SG == FG_AOT_SG;
|
||||
#else
|
||||
return true;
|
||||
#endif
|
||||
}
|
||||
|
||||
static constexpr int GGML_SYCL_FG_BN = 32;
|
||||
static constexpr int FG_KSPLIT = 4;
|
||||
static constexpr int64_t GGML_SYCL_MMID_SCHED_MAX_EXPERTS = 4096;
|
||||
|
||||
// Shape and type gates for the kernels below. Device capability is separate: it needs a queue to ask.
|
||||
|
||||
@@ -111,6 +111,7 @@ int g_ggml_sycl_memtrace_step = 64;
|
||||
int g_ggml_sycl_enable_vmm = 1;
|
||||
int g_ggml_sycl_enable_fusion = 1;
|
||||
int g_ggml_sycl_enable_esimd = 1;
|
||||
int g_ggml_sycl_mmid_esimd = 1;
|
||||
int g_ggml_sycl_mmvq_wide = 1;
|
||||
int g_ggml_sycl_enable_xmx_esimd = 1;
|
||||
int g_ggml_sycl_xmx_max_cols = GGML_SYCL_XMX_MAX_COLS_DEFAULT;
|
||||
@@ -444,6 +445,7 @@ static void ggml_check_sycl() try {
|
||||
g_ggml_sycl_enable_vmm = ggml_sycl_get_env("GGML_SYCL_ENABLE_VMM", 1);
|
||||
g_ggml_sycl_enable_fusion = ggml_sycl_get_env("GGML_SYCL_ENABLE_FUSION", 1);
|
||||
g_ggml_sycl_enable_esimd = ggml_sycl_get_env("GGML_SYCL_ENABLE_ESIMD", 1);
|
||||
g_ggml_sycl_mmid_esimd = ggml_sycl_get_env("GGML_SYCL_MMID_ESIMD", 1);
|
||||
g_ggml_sycl_mmvq_wide = ggml_sycl_get_env("GGML_SYCL_MMVQ_WIDE", 1);
|
||||
g_ggml_sycl_enable_xmx_esimd = ggml_sycl_get_env("GGML_SYCL_ENABLE_XMX_ESIMD", 1);
|
||||
g_ggml_sycl_xmx_max_cols = std::max(0, ggml_sycl_get_env("GGML_SYCL_XMX_MAX_COLS", GGML_SYCL_XMX_MAX_COLS_DEFAULT));
|
||||
@@ -578,6 +580,7 @@ static void ggml_check_sycl() try {
|
||||
GGML_LOG_INFO(" GGML_SYCL_ENABLE_OPT: %d\n", g_ggml_sycl_enable_optimize);
|
||||
GGML_LOG_INFO(" GGML_SYCL_XMX_GATHER_TYPES: %d\n", g_ggml_sycl_xmx_gather_types);
|
||||
GGML_LOG_INFO(" GGML_SYCL_MMID_SCHED: %d\n", g_ggml_sycl_mmid_sched);
|
||||
GGML_LOG_INFO(" GGML_SYCL_MMID_ESIMD: %d\n", g_ggml_sycl_mmid_esimd);
|
||||
GGML_LOG_INFO(" GGML_SYCL_XMX_GATHER_SHAPES: %d\n", g_ggml_sycl_xmx_gather_shapes);
|
||||
GGML_LOG_INFO(" GGML_SYCL_DYNAMIC_PRECISION: %s\n",
|
||||
ggml_sycl_dynamic_precision_names[g_ggml_sycl_dynamic_precision]);
|
||||
|
||||
Reference in new issue
Block a user