sycl: add grouped MoE XMX GEMM (#29245)
Co-authored-by: cwriter <cwriter@localhost>
This commit is contained in:
1 parent
75118a3a59
commit
847f447c31
6 files changed
+1358
-16
No files matched your search
@@ -816,6 +816,10 @@ User can use the device management in [docs/multi-gpu.md](https://github.com/ggm
|
||||
| GGML_SYCL_MKL_FA_DIAG | 0 (default) or 1 | Enable output fingerprinting for MKL flash attention. Dumps the first 64 float output values for the first 6 FA calls with n_kv ≥ 1024, labeled with kernel type (MKL/TILE/VEC) for cross-kernel comparison. |
|
||||
| GGML_SYCL_ENABLE_FUSION | 0 or 1 (default) | Enable fused-kernel dispatch in graph compute. Unsupported types and layouts fall back to the standalone op kernels. See `ggml_sycl_can_fuse()`. |
|
||||
| GGML_SYCL_ENABLE_ESIMD | 0 or 1 (default)| Enable ESIMD kernels when available. |
|
||||
| GGML_SYCL_XMX_GATHER_TYPES | decimal bitmask, all bits set (default) | Weight formats that may use the XMX dequant-GEMM paths, which dequantize weights straight into the XMX tiles. This speeds up prompt processing of MoE models on GPUs with XMX units (Arc A- and B-series, Arc Pro, Data Center GPU Max), for example pp512 of Qwen3-30B-A3B UD-IQ3_XXS by about 50% on an Arc Pro B60. Bits:<br>* 1: IQ4_NL, 2: IQ3_S, 4: IQ4_XS, 8: IQ3_XXS, 16: IQ2_XXS, 32: IQ2_XS, 64: IQ2_S, 128: IQ1_S, 256: IQ1_M<br>* 512: Q8_0, 1024: Q4_K, 2048: Q5_K, 4096: Q6_K (MoE `MUL_MAT_ID` only)<br>Add values to combine them, for example `3` for IQ4_NL and IQ3_S; `0` disables the paths. A set bit does not force the path: batches of more than 64 tokens per expert or row lengths that are not a multiple of 256 (32 for IQ4_NL and Q8_0) use the library GEMM. |
|
||||
| GGML_SYCL_XMX_GATHER_SHAPES | decimal bitmask, 255 (default) | XMX `joint_matrix` combinations the paths of `GGML_SYCL_XMX_GATHER_TYPES` may use; the operand type comes from `GGML_SYCL_DYNAMIC_PRECISION` and the best supported combination is picked automatically (logged as `fg_pick_combo`). Bits:<br>* Xe2, Xe3, Xe-HPC: 1: f16 8x16x16, 2: f16 16x16x16, 4: f16 32x64x16, 8: f16 32x64x32, 32: tf32 8x16x8, 64: bf16 8x16x16<br>* Xe-HPG (Arc A770, ARL-H): 16: f16 8x8x16, 128: bf16 8x8x16<br>Clear a bit to exclude a combination, or set a single bit to force one for testing. |
|
||||
| GGML_SYCL_DYNAMIC_PRECISION | `F16` (default with `GGML_SYCL_F16=ON`), `BF16`, `TF32` or `F32` (default otherwise) | Operand type of the XMX dequant-GEMM paths (`GGML_SYCL_XMX_GATHER_TYPES`); accumulation is always f32. `F16` is the fastest, but activations above 65504 overflow. `BF16` keeps the f32 range at a 7-bit mantissa, `TF32` keeps the range and the f16 mantissa but is about 30% slower and needs Xe2, Xe3 or Xe-HPC, and `F32` turns the XMX paths off. Ops that request a higher src1 precision ([TAG_GGML_PREC]) get it regardless of this setting. |
|
||||
| GGML_SYCL_DYNAMIC_REQUIRED_PRECISION | `F32` (default), `TF32`, `BF16` or `F16` | Lowest type the XMX paths may use for an op that requests an F32 src1, such as Mistral 4 `ffn_down_exps`. The default runs such ops on the library f32 GEMM; `TF32` or `BF16` trade mantissa for speed while keeping the f32 range. `F16` ignores the request and can overflow; it is meant for testing only. |
|
||||
| GGML_SYCL_MMVQ_WIDE | 0 or 1 (default) | Use the wide-load variant of the reordered Q8_0 mat-vec kernel, which reads four contiguous dwords per operand instead of one value at a time. Set to 0 to fall back to the per-value loads. Only affects Q8_0 weights in the reordered layout. |
|
||||
| GGML_SYCL_SPARSE_FA | 0 (default) or 1 | Enable Sparse Flash-attention.|
|
||||
| GGML_SYCL_SPARSE_FA_DEBUG | 0 (default) or 1 | Enable to debug for Sparse Flash-attention.|
|
||||
|
||||
@@ -221,4 +221,27 @@ if (GGML_SYCL_DEVICE_ARCH)
|
||||
"SHELL:-Xsycl-target-backend=spir64_gen \"-device ${GGML_SYCL_DEVICE_ARCH}\""
|
||||
-fsycl-max-parallel-link-jobs=${GGML_SYCL_MAX_PARALLEL_LINK_JOBS}
|
||||
)
|
||||
|
||||
# The XMX dequant-GEMM tiles need the sub-group size of the target: 8 on Xe-HPG (DG2, ARL-H),
|
||||
# 16 on Xe-HPC and Xe2 or newer. ocloc fails on the other size, so build only the one that fits.
|
||||
# 0 (unknown name, mixed list, or no XMX) builds no XMX tile and the path stays off.
|
||||
set(_ggml_sycl_xmx_sg "")
|
||||
string(TOLOWER "${GGML_SYCL_DEVICE_ARCH}" _ggml_sycl_archs)
|
||||
string(REPLACE "," ";" _ggml_sycl_archs "${_ggml_sycl_archs}")
|
||||
foreach(_arch IN LISTS _ggml_sycl_archs)
|
||||
if (_arch MATCHES "^(dg2|acm|ats-m|arl-h|xe-hpg|12\\.5[567]\\.|12\\.74\\.)")
|
||||
set(_sg 8)
|
||||
elseif (_arch MATCHES "^(pvc|bmg|lnl|ptl|wcl|nvl|cri|xe2|xe3|xe-hpc|12\\.60\\.|20\\.|30\\.)")
|
||||
set(_sg 16)
|
||||
else()
|
||||
set(_sg 0)
|
||||
endif()
|
||||
if (_ggml_sycl_xmx_sg STREQUAL "" OR _ggml_sycl_xmx_sg EQUAL _sg)
|
||||
set(_ggml_sycl_xmx_sg ${_sg})
|
||||
else()
|
||||
set(_ggml_sycl_xmx_sg 0)
|
||||
endif()
|
||||
endforeach()
|
||||
message(STATUS "GGML_SYCL_DEVICE_ARCH: XMX dequant-GEMM sub-group size ${_ggml_sycl_xmx_sg} (0 = off)")
|
||||
target_compile_definitions(ggml-sycl PRIVATE GGML_SYCL_XMX_AOT_SG=${_ggml_sycl_xmx_sg})
|
||||
endif()
|
||||
@@ -65,6 +65,61 @@ extern int g_ggml_sycl_enable_fusion;
|
||||
extern int g_ggml_sycl_enable_esimd;
|
||||
extern int g_ggml_sycl_mmvq_wide;
|
||||
extern int g_ggml_sycl_prioritize_dmmv;
|
||||
|
||||
// Which quantized weight formats may take the XMX dequant-GEMM paths. A bitmask rather than one
|
||||
// flag per path, so a format can be enabled or measured on its own and adding a format is one bit.
|
||||
enum ggml_sycl_xmx_gather_type {
|
||||
GGML_SYCL_XMX_GATHER_IQ4_NL = 1 << 0,
|
||||
GGML_SYCL_XMX_GATHER_IQ3_S = 1 << 1,
|
||||
GGML_SYCL_XMX_GATHER_IQ4_XS = 1 << 2,
|
||||
GGML_SYCL_XMX_GATHER_IQ3_XXS = 1 << 3,
|
||||
GGML_SYCL_XMX_GATHER_IQ2_XXS = 1 << 4,
|
||||
GGML_SYCL_XMX_GATHER_IQ2_XS = 1 << 5,
|
||||
GGML_SYCL_XMX_GATHER_IQ2_S = 1 << 6,
|
||||
GGML_SYCL_XMX_GATHER_IQ1_S = 1 << 7,
|
||||
GGML_SYCL_XMX_GATHER_IQ1_M = 1 << 8,
|
||||
GGML_SYCL_XMX_GATHER_Q8_0 = 1 << 9,
|
||||
GGML_SYCL_XMX_GATHER_Q4_K = 1 << 10,
|
||||
GGML_SYCL_XMX_GATHER_Q5_K = 1 << 11,
|
||||
GGML_SYCL_XMX_GATHER_Q6_K = 1 << 12,
|
||||
};
|
||||
static constexpr int GGML_SYCL_XMX_GATHER_TYPES_DEFAULT = ~0;
|
||||
extern int g_ggml_sycl_xmx_gather_types;
|
||||
// Which joint_matrix combinations the XMX dequant-GEMM paths may use, one bit each (see fused-gemm.cpp).
|
||||
// GGML_SYCL_DYNAMIC_PRECISION picks the operand type, this mask the combinations of that type.
|
||||
static constexpr int GGML_SYCL_XMX_GATHER_SHAPES_DEFAULT = 0xff;
|
||||
extern int g_ggml_sycl_xmx_gather_shapes;
|
||||
|
||||
// GGML_SYCL_DYNAMIC_PRECISION: operand type of the XMX dequant-GEMM paths. F32 turns them off and
|
||||
// keeps the library GEMM in f32. A src1 precision request of an op [TAG_GGML_PREC] is always met.
|
||||
enum ggml_sycl_dynamic_precision {
|
||||
GGML_SYCL_DYNAMIC_PRECISION_F16,
|
||||
GGML_SYCL_DYNAMIC_PRECISION_BF16,
|
||||
GGML_SYCL_DYNAMIC_PRECISION_TF32,
|
||||
GGML_SYCL_DYNAMIC_PRECISION_F32,
|
||||
};
|
||||
#ifdef GGML_SYCL_F16
|
||||
static constexpr int GGML_SYCL_DYNAMIC_PRECISION_DEFAULT = GGML_SYCL_DYNAMIC_PRECISION_F16;
|
||||
#else
|
||||
static constexpr int GGML_SYCL_DYNAMIC_PRECISION_DEFAULT = GGML_SYCL_DYNAMIC_PRECISION_F32;
|
||||
#endif
|
||||
extern int g_ggml_sycl_dynamic_precision;
|
||||
// GGML_SYCL_DYNAMIC_REQUIRED_PRECISION: the XMX type an F32 src1 request may run on instead of f32
|
||||
// (TF32, or BF16 which also allows tf32). F32 (default): none. F16: src1 requests are ignored.
|
||||
extern int g_ggml_sycl_dynamic_required_precision;
|
||||
|
||||
// [TAG_GGML_PREC] src1 precision request of the MUL_MAT/MUL_MAT_ID op dst
|
||||
static inline int32_t ggml_sycl_src1_prec(const ggml_tensor * dst) {
|
||||
return g_ggml_sycl_dynamic_required_precision == GGML_SYCL_DYNAMIC_PRECISION_F16 ? GGML_PREC_UNDEFINED :
|
||||
dst->op_params[3];
|
||||
}
|
||||
|
||||
// [TAG_GGML_PREC] the library GEMM and dmmv may convert src1 of the MUL_MAT/MUL_MAT_ID op dst to f16
|
||||
static inline bool ggml_sycl_src1_f16_ok(const ggml_tensor * dst) {
|
||||
const int32_t src1_prec = ggml_sycl_src1_prec(dst);
|
||||
return g_ggml_sycl_dynamic_precision != GGML_SYCL_DYNAMIC_PRECISION_F32 &&
|
||||
(src1_prec == GGML_PREC_UNDEFINED || src1_prec >= GGML_PREC_F16);
|
||||
}
|
||||
extern int g_ggml_sycl_enable_flash_attention;
|
||||
extern int g_ggml_sycl_dev2dev_memcpy;
|
||||
extern int g_ggml_sycl_fa_onednn;
|
||||
@@ -333,6 +388,12 @@ struct mmid_row_mapping {
|
||||
int32_t i2;
|
||||
};
|
||||
|
||||
struct ggml_sycl_gg_tile {
|
||||
int32_t expert;
|
||||
int32_t n0;
|
||||
int32_t n1;
|
||||
};
|
||||
|
||||
namespace sycl_ex = sycl::ext::oneapi::experimental;
|
||||
struct ggml_backend_sycl_context {
|
||||
int device;
|
||||
@@ -410,6 +471,7 @@ struct ggml_backend_sycl_context {
|
||||
std::unique_ptr<ggml_sycl_pool> host_pools[GGML_SYCL_MAX_DEVICES];
|
||||
|
||||
std::vector<mmid_row_mapping> mmid_row_mapping_host;
|
||||
std::vector<ggml_sycl_gg_tile> mmid_tile_schedule_host;
|
||||
|
||||
static std::unique_ptr<ggml_sycl_pool> new_pool_for_device(queue_ptr qptr, int device);
|
||||
|
||||
|
||||
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,88 @@
|
||||
#ifndef GGML_SYCL_FUSED_GEMM_HPP
|
||||
#define GGML_SYCL_FUSED_GEMM_HPP
|
||||
|
||||
#include "common.hpp"
|
||||
|
||||
|
||||
// Shape and type gates for the kernels below. Device capability is separate: it needs a queue to ask.
|
||||
static constexpr int GGML_SYCL_FG_MAX_N = 64; // widest N taken; each shape covers it in BN-wide tiles
|
||||
|
||||
// weight formats the fused A stage decodes; K must cover whole stored blocks
|
||||
constexpr bool ggml_sycl_fused_dequant_gemm_type_ok(ggml_type src0_type, int64_t K) {
|
||||
// iq4_nl and q8_0 store 32 values per block; every other format here is a 256-value superblock
|
||||
// that the A stage walks in steps of 32, so K must cover whole superblocks.
|
||||
if (src0_type == GGML_TYPE_IQ4_NL || src0_type == GGML_TYPE_Q8_0) {
|
||||
return K % 32 == 0;
|
||||
}
|
||||
const bool superblock =
|
||||
src0_type == GGML_TYPE_Q4_K ||
|
||||
src0_type == GGML_TYPE_Q5_K ||
|
||||
src0_type == GGML_TYPE_Q6_K ||
|
||||
src0_type == GGML_TYPE_IQ3_S ||
|
||||
src0_type == GGML_TYPE_IQ4_XS ||
|
||||
src0_type == GGML_TYPE_IQ3_XXS ||
|
||||
src0_type == GGML_TYPE_IQ2_XXS ||
|
||||
src0_type == GGML_TYPE_IQ2_XS ||
|
||||
src0_type == GGML_TYPE_IQ2_S ||
|
||||
src0_type == GGML_TYPE_IQ1_S ||
|
||||
src0_type == GGML_TYPE_IQ1_M;
|
||||
return superblock && QK_K == 256 && K % QK_K == 0;
|
||||
}
|
||||
|
||||
constexpr bool ggml_sycl_fused_dequant_gemm_shape_ok(ggml_type src0_type, int64_t M, int64_t N, int64_t K,
|
||||
int64_t ldd) {
|
||||
return ggml_sycl_fused_dequant_gemm_type_ok(src0_type, K) && M > 0 && N > 0 && K > 0 &&
|
||||
N <= GGML_SYCL_FG_MAX_N &&
|
||||
M <= INT32_MAX && N <= INT32_MAX && K <= INT32_MAX && ldd <= INT32_MAX;
|
||||
}
|
||||
|
||||
// grouped variant: the per-expert fused kernel is only worth it while each expert is narrow,
|
||||
// so wider average slices are left to the per-expert library GEMM loop
|
||||
constexpr bool ggml_sycl_grouped_dequant_gemm_shape_ok(ggml_type src0_type, int64_t M, int64_t K,
|
||||
int64_t total_rows, int64_t n_active) {
|
||||
return ggml_sycl_fused_dequant_gemm_shape_ok(src0_type, M, 1, K, M) && total_rows > 0 &&
|
||||
total_rows <= INT32_MAX && total_rows <= n_active * GGML_SYCL_FG_MAX_N;
|
||||
}
|
||||
|
||||
// Runtime type gate, kept out of the constexpr predicates above so those stay pure.
|
||||
inline bool ggml_sycl_xmx_gather_type_enabled(ggml_type src0_type) {
|
||||
switch (src0_type) {
|
||||
case GGML_TYPE_IQ4_NL: return (g_ggml_sycl_xmx_gather_types & GGML_SYCL_XMX_GATHER_IQ4_NL ) != 0;
|
||||
case GGML_TYPE_IQ3_S: return (g_ggml_sycl_xmx_gather_types & GGML_SYCL_XMX_GATHER_IQ3_S ) != 0;
|
||||
case GGML_TYPE_IQ4_XS: return (g_ggml_sycl_xmx_gather_types & GGML_SYCL_XMX_GATHER_IQ4_XS ) != 0;
|
||||
case GGML_TYPE_IQ3_XXS: return (g_ggml_sycl_xmx_gather_types & GGML_SYCL_XMX_GATHER_IQ3_XXS ) != 0;
|
||||
case GGML_TYPE_IQ2_XXS: return (g_ggml_sycl_xmx_gather_types & GGML_SYCL_XMX_GATHER_IQ2_XXS ) != 0;
|
||||
case GGML_TYPE_IQ2_XS: return (g_ggml_sycl_xmx_gather_types & GGML_SYCL_XMX_GATHER_IQ2_XS ) != 0;
|
||||
case GGML_TYPE_IQ2_S: return (g_ggml_sycl_xmx_gather_types & GGML_SYCL_XMX_GATHER_IQ2_S ) != 0;
|
||||
case GGML_TYPE_IQ1_S: return (g_ggml_sycl_xmx_gather_types & GGML_SYCL_XMX_GATHER_IQ1_S ) != 0;
|
||||
case GGML_TYPE_IQ1_M: return (g_ggml_sycl_xmx_gather_types & GGML_SYCL_XMX_GATHER_IQ1_M ) != 0;
|
||||
case GGML_TYPE_Q8_0: return (g_ggml_sycl_xmx_gather_types & GGML_SYCL_XMX_GATHER_Q8_0 ) != 0;
|
||||
case GGML_TYPE_Q4_K: return (g_ggml_sycl_xmx_gather_types & GGML_SYCL_XMX_GATHER_Q4_K ) != 0;
|
||||
case GGML_TYPE_Q5_K: return (g_ggml_sycl_xmx_gather_types & GGML_SYCL_XMX_GATHER_Q5_K ) != 0;
|
||||
case GGML_TYPE_Q6_K: return (g_ggml_sycl_xmx_gather_types & GGML_SYCL_XMX_GATHER_Q6_K ) != 0;
|
||||
default: return false;
|
||||
}
|
||||
}
|
||||
|
||||
// dst[n*ldd + m] = sum_k dequant(src0)[m*K + k] * src1[n*K + k], src1 is F32, F16 or BF16.
|
||||
// The XMX combination is picked per call from the src1 type and its precision request src1_prec
|
||||
// (op_params[3], [TAG_GGML_PREC]); the accumulator is f32, which meets any request.
|
||||
// q8_0 and the k-quants are not handled here, only in the grouped path below.
|
||||
// Returns false when the case is not handled (type, device, precision, or shape).
|
||||
bool ggml_sycl_fused_dequant_gemm(ggml_type src0_type, const void * src0, const void * src1, ggml_type src1_type,
|
||||
int32_t src1_prec, float * dst, int64_t M, int64_t N, int64_t K, int64_t ldd,
|
||||
ggml_sycl_pool & pool, dpct::queue_ptr stream);
|
||||
|
||||
// One launch for every expert of a MUL_MAT_ID: rows of src1/dst are grouped by expert, expert e
|
||||
// owns rows [expert_row_offsets[e], expert_row_offsets[e+1]) and reads its weights at
|
||||
// src0_base + e*expert_stride. tiles is host scratch that must stay alive until the queue drains.
|
||||
// reordered: each expert slice is in the reorder (SoA) layout of reorder_qw().
|
||||
// dst[n*M + m] = sum_k dequant(src0_e)[m*K + k] * src1[n*K + k]
|
||||
// Returns false when the case is not handled (type, layout, device, precision, or shape).
|
||||
bool ggml_sycl_grouped_dequant_gemm(ggml_type src0_type, bool reordered, const void * src0_base, size_t expert_stride,
|
||||
const float * src1, int32_t src1_prec, float * dst,
|
||||
const int64_t * expert_row_offsets, int64_t n_as, int64_t M, int64_t K,
|
||||
int64_t total_rows, std::vector<ggml_sycl_gg_tile> & tiles,
|
||||
ggml_sycl_pool & pool, dpct::queue_ptr stream);
|
||||
|
||||
#endif // GGML_SYCL_FUSED_GEMM_HPP
|
||||
@@ -14,6 +14,7 @@
|
||||
#include <array>
|
||||
#include <assert.h>
|
||||
#include <atomic>
|
||||
#include <cctype>
|
||||
#include <cinttypes>
|
||||
#include <cstddef>
|
||||
#include <cstdint>
|
||||
@@ -60,6 +61,7 @@
|
||||
#include "ggml-sycl/common.hpp"
|
||||
#include "ggml-sycl/element_wise.hpp"
|
||||
#include "ggml-sycl/fwht.hpp"
|
||||
#include "ggml-sycl/fused-gemm.hpp"
|
||||
#include "ggml-sycl/gemm.hpp"
|
||||
#include "ggml-sycl/getrows.hpp"
|
||||
#include "ggml-sycl/mem.hpp"
|
||||
@@ -105,6 +107,30 @@ int g_ggml_sycl_enable_fusion = 1;
|
||||
int g_ggml_sycl_enable_esimd = 1;
|
||||
int g_ggml_sycl_mmvq_wide = 1;
|
||||
int g_ggml_sycl_prioritize_dmmv = 0;
|
||||
int g_ggml_sycl_xmx_gather_types = GGML_SYCL_XMX_GATHER_TYPES_DEFAULT;
|
||||
int g_ggml_sycl_xmx_gather_shapes = GGML_SYCL_XMX_GATHER_SHAPES_DEFAULT;
|
||||
int g_ggml_sycl_dynamic_precision = GGML_SYCL_DYNAMIC_PRECISION_DEFAULT;
|
||||
int g_ggml_sycl_dynamic_required_precision = GGML_SYCL_DYNAMIC_PRECISION_F32;
|
||||
static const char * ggml_sycl_dynamic_precision_names[] = { "F16", "BF16", "TF32", "F32" };
|
||||
|
||||
// value of a GGML_SYCL_DYNAMIC_PRECISION-style variable; def if unset or invalid
|
||||
static int ggml_sycl_get_env_precision(const char * name, int def) {
|
||||
const char * env = getenv(name);
|
||||
if (!env) {
|
||||
return def;
|
||||
}
|
||||
std::string mode(env);
|
||||
for (char & c : mode) {
|
||||
c = (char) std::toupper((unsigned char) c);
|
||||
}
|
||||
for (int i = GGML_SYCL_DYNAMIC_PRECISION_F16; i <= GGML_SYCL_DYNAMIC_PRECISION_F32; i++) {
|
||||
if (mode == ggml_sycl_dynamic_precision_names[i]) {
|
||||
return i;
|
||||
}
|
||||
}
|
||||
GGML_LOG_WARN("%s: unknown %s=%s, using %s\n", __func__, name, env, ggml_sycl_dynamic_precision_names[def]);
|
||||
return def;
|
||||
}
|
||||
int g_ggml_sycl_use_async_mem_op = 0;
|
||||
int g_ggml_sycl_use_async_mem_op_requested = 1;
|
||||
int g_ggml_sycl_use_level_zero_api = 0;
|
||||
@@ -401,6 +427,12 @@ static void ggml_check_sycl() try {
|
||||
g_ggml_sycl_enable_esimd = ggml_sycl_get_env("GGML_SYCL_ENABLE_ESIMD", 1);
|
||||
g_ggml_sycl_mmvq_wide = ggml_sycl_get_env("GGML_SYCL_MMVQ_WIDE", 1);
|
||||
g_ggml_sycl_prioritize_dmmv = ggml_sycl_get_env("GGML_SYCL_PRIORITIZE_DMMV", 0);
|
||||
g_ggml_sycl_xmx_gather_types = ggml_sycl_get_env("GGML_SYCL_XMX_GATHER_TYPES", GGML_SYCL_XMX_GATHER_TYPES_DEFAULT);
|
||||
g_ggml_sycl_xmx_gather_shapes = ggml_sycl_get_env("GGML_SYCL_XMX_GATHER_SHAPES", GGML_SYCL_XMX_GATHER_SHAPES_DEFAULT);
|
||||
g_ggml_sycl_dynamic_precision =
|
||||
ggml_sycl_get_env_precision("GGML_SYCL_DYNAMIC_PRECISION", GGML_SYCL_DYNAMIC_PRECISION_DEFAULT);
|
||||
g_ggml_sycl_dynamic_required_precision =
|
||||
ggml_sycl_get_env_precision("GGML_SYCL_DYNAMIC_REQUIRED_PRECISION", GGML_SYCL_DYNAMIC_PRECISION_F32);
|
||||
|
||||
#ifdef GGML_SYCL_SUPPORT_LEVEL_ZERO_API
|
||||
g_ggml_sycl_use_level_zero_api = ggml_sycl_get_env("GGML_SYCL_USE_LEVEL_ZERO_API", 1);
|
||||
@@ -509,6 +541,12 @@ static void ggml_check_sycl() try {
|
||||
#endif
|
||||
|
||||
GGML_LOG_INFO(" GGML_SYCL_ENABLE_OPT: %d\n", g_ggml_sycl_enable_optimize);
|
||||
GGML_LOG_INFO(" GGML_SYCL_XMX_GATHER_TYPES: %d\n", g_ggml_sycl_xmx_gather_types);
|
||||
GGML_LOG_INFO(" GGML_SYCL_XMX_GATHER_SHAPES: %d\n", g_ggml_sycl_xmx_gather_shapes);
|
||||
GGML_LOG_INFO(" GGML_SYCL_DYNAMIC_PRECISION: %s\n",
|
||||
ggml_sycl_dynamic_precision_names[g_ggml_sycl_dynamic_precision]);
|
||||
GGML_LOG_INFO(" GGML_SYCL_DYNAMIC_REQUIRED_PRECISION: %s\n",
|
||||
ggml_sycl_dynamic_precision_names[g_ggml_sycl_dynamic_required_precision]);
|
||||
|
||||
#if defined(GGML_SYCL_SUPPORT_VMM)
|
||||
GGML_LOG_INFO(" GGML_SYCL_ENABLE_VMM: %d\n", g_ggml_sycl_enable_vmm);
|
||||
@@ -3027,22 +3065,18 @@ inline void ggml_sycl_op_mul_mat_sycl(
|
||||
}
|
||||
#endif
|
||||
|
||||
// dequantize inside the GEMM instead of writing the f16 weights out and reading them back; src1
|
||||
// goes in its own type, so there is no separate conversion pass
|
||||
if (ggml_is_quantized(src0->type) && ggml_is_contiguous(src0) && row_diff == src0->ne[1] &&
|
||||
ggml_sycl_fused_dequant_gemm(src0->type, src0_dd_i, src1_ddf_i, src1->type, ggml_sycl_src1_prec(dst), dst_dd_i,
|
||||
row_diff, src1_ncols, ne10, ldc, ctx.pool(), stream)) {
|
||||
return;
|
||||
}
|
||||
|
||||
// the f16 route converts src1 to f16 [TAG_GGML_PREC]
|
||||
use_fp16 = use_fp16 && ggml_sycl_src1_f16_ok(dst);
|
||||
if ((src0->type == GGML_TYPE_F16 || ggml_is_quantized(src0->type)) && use_fp16 && ggml_is_contiguous(src0) &&
|
||||
row_diff == src0->ne[1] && dst->op_params[0] == GGML_PREC_DEFAULT) {
|
||||
ggml_sycl_pool_alloc<sycl::half> src0_as_f16(ctx.pool());
|
||||
if (src0->type != GGML_TYPE_F16) {
|
||||
scope_op_debug_print scope_dbg_print(__func__, "/to_fp16_sycl", dst, /*num_src=*/2,
|
||||
" : converting src0 to fp16");
|
||||
const to_fp16_sycl_t to_fp16_sycl = ggml_get_to_fp16_sycl(src0->type, dst);
|
||||
GGML_ASSERT(to_fp16_sycl != nullptr);
|
||||
size_t ne = row_diff*ne00;
|
||||
src0_as_f16.alloc(ne);
|
||||
to_fp16_sycl(src0_dd_i, src0_as_f16.get(), ne, stream);
|
||||
}
|
||||
const sycl::half *src0_ptr = src0->type == GGML_TYPE_F16
|
||||
? (const sycl::half *)src0_dd_i
|
||||
: src0_as_f16.get();
|
||||
|
||||
ggml_sycl_pool_alloc<sycl::half> src1_as_f16(ctx.pool());
|
||||
if (src1->type != GGML_TYPE_F16) {
|
||||
scope_op_debug_print scope_dbg_print(__func__, "/to_fp16_sycl", dst, /*num_src=*/2,
|
||||
@@ -3057,6 +3091,20 @@ inline void ggml_sycl_op_mul_mat_sycl(
|
||||
? (const sycl::half *)src1->data + src1_padded_row_size
|
||||
: src1_as_f16.get();
|
||||
|
||||
ggml_sycl_pool_alloc<sycl::half> src0_as_f16(ctx.pool());
|
||||
if (src0->type != GGML_TYPE_F16) {
|
||||
scope_op_debug_print scope_dbg_print(__func__, "/to_fp16_sycl", dst, /*num_src=*/2,
|
||||
" : converting src0 to fp16");
|
||||
const to_fp16_sycl_t to_fp16_sycl = ggml_get_to_fp16_sycl(src0->type, dst);
|
||||
GGML_ASSERT(to_fp16_sycl != nullptr);
|
||||
size_t ne = row_diff*ne00;
|
||||
src0_as_f16.alloc(ne);
|
||||
to_fp16_sycl(src0_dd_i, src0_as_f16.get(), ne, stream);
|
||||
}
|
||||
const sycl::half *src0_ptr = src0->type == GGML_TYPE_F16
|
||||
? (const sycl::half *)src0_dd_i
|
||||
: src0_as_f16.get();
|
||||
|
||||
#if GGML_SYCL_DNNL
|
||||
if (g_ggml_sycl_enable_dnn && ggml_sycl_dnnl_has_optimized_gemm(ggml_sycl_get_device())) {
|
||||
DnnlGemmWrapper::row_gemm(ctx,row_diff, src1_ncols , ne10, src0_ptr,
|
||||
@@ -4859,6 +4907,10 @@ static void ggml_sycl_mul_mat(ggml_backend_sycl_context & ctx, const ggml_tensor
|
||||
|
||||
// check data types and tensor shapes for custom matrix multiplication kernels:
|
||||
bool use_dequantize_mul_mat_vec = can_use_dequantize_mul_mat_vec(src0, src1, dst);
|
||||
#ifdef GGML_SYCL_F16
|
||||
// dmmv may convert src1 to f16 in this build [TAG_GGML_PREC]
|
||||
use_dequantize_mul_mat_vec = use_dequantize_mul_mat_vec && ggml_sycl_src1_f16_ok(dst);
|
||||
#endif
|
||||
|
||||
bool use_mul_mat_vec_q = can_use_mul_mat_vec_q(src0, src1, dst);
|
||||
|
||||
@@ -5272,7 +5324,9 @@ static void ggml_sycl_mul_mat_id(ggml_backend_sycl_context & ctx,
|
||||
SYCL_CHECK(CHECK_TRY_ERROR(
|
||||
stream->memcpy(ids_host.data(), ids_dev, ggml_nbytes(ids))));
|
||||
|
||||
// also ensures ctx.mmid_row_mapping_host is drained before we use it again
|
||||
// also ensures ctx.mmid_row_mapping_host and ctx.mmid_tile_schedule_host are drained before we
|
||||
// refill them: the grouped GEMM enqueues an async copy out of the tile schedule, so removing
|
||||
// this wait would let the next node overwrite a buffer the device is still reading
|
||||
SYCL_CHECK(CHECK_TRY_ERROR(stream->wait()));
|
||||
|
||||
ggml_tensor src0_row = *src0;
|
||||
@@ -5363,7 +5417,25 @@ static void ggml_sycl_mul_mat_id(ggml_backend_sycl_context & ctx,
|
||||
});
|
||||
}
|
||||
|
||||
for (int64_t i02 = 0; i02 < n_as; i02++) {
|
||||
bool grouped = false;
|
||||
if (ggml_is_contiguous(src0) && src1->type == GGML_TYPE_F32 &&
|
||||
dst->type == GGML_TYPE_F32 && nb11 == sizeof(float)*ne10 && nb1 == sizeof(float)*ne0) {
|
||||
// the grouped GEMM reads the reorder (SoA) layout faster, and the first decode step installs it
|
||||
// anyway: install it here already, so prefill does not depend on whether a decode ran before
|
||||
if (g_ggml_sycl_dynamic_precision != GGML_SYCL_DYNAMIC_PRECISION_F32 &&
|
||||
ggml_sycl_xmx_gather_type_enabled(src0->type)) {
|
||||
opt_for_reorder_id(&ctx, src0);
|
||||
}
|
||||
const bool src0_reordered =
|
||||
src0->extra && ((const ggml_tensor_extra_gpu *) src0->extra)->optimized_feature.reorder;
|
||||
grouped = ggml_sycl_grouped_dequant_gemm(src0->type, src0_reordered, src0_original, nb02,
|
||||
(const float *) src1_contiguous.get(), ggml_sycl_src1_prec(dst),
|
||||
(float *) dst_contiguous.get(),
|
||||
expert_row_offsets.data(), n_as, ne01, ne10, n_routed_rows,
|
||||
ctx.mmid_tile_schedule_host, ctx.pool(), stream);
|
||||
}
|
||||
|
||||
for (int64_t i02 = 0; i02 < n_as && !grouped; i02++) {
|
||||
const int64_t num_src1_rows = expert_row_counts[i02];
|
||||
|
||||
if (num_src1_rows == 0) {
|
||||
|
||||
Reference in new issue
Block a user