Restore ESIMD kernels on FFN path (#30239)
* Reenable ESIMD on the FFN path Signed-off-by: Todd Malsbary <todd.malsbary@intel.com> * Extend ESIMD fusion on FFN path to include Q8_0 Signed-off-by: Todd Malsbary <todd.malsbary@intel.com> * Remove comment explicitly referencing ESIMD in test-backend-ops Signed-off-by: Todd Malsbary <todd.malsbary@intel.com> --------- Signed-off-by: Todd Malsbary <todd.malsbary@intel.com>
This commit is contained in:
1 parent
848bbda904
commit
547208dca2
6 files changed
+265
-81
No files matched your search
@@ -37,6 +37,10 @@ namespace syclexp = sycl::ext::oneapi::experimental;
|
||||
#endif
|
||||
#endif
|
||||
|
||||
#if defined(__INTEL_LLVM_COMPILER)
|
||||
#define GGML_SYCL_DMMV_HAS_ESIMD
|
||||
#endif
|
||||
|
||||
#if GGML_SYCL_DNNL
|
||||
#include "dnnl.hpp"
|
||||
#include "dnnl_sycl.hpp"
|
||||
|
||||
+181
-73
@@ -10,7 +10,6 @@
|
||||
#endif
|
||||
#include <sycl/ext/intel/esimd.hpp>
|
||||
#include "esimd.hpp"
|
||||
#define GGML_SYCL_DMMV_HAS_ESIMD
|
||||
#endif
|
||||
|
||||
static void convert_f16(const void * vx, const int64_t ib, const int iqs, dfloat2 & v){
|
||||
@@ -1873,11 +1872,67 @@ static void dequantize_mul_mat_vec_q6_K_sycl(const void *vx, const float *y,
|
||||
#ifdef GGML_SYCL_DMMV_HAS_ESIMD
|
||||
using ggml_sycl_esimd::GGML_SYCL_DMMV_ESIMD_WG_SIZE;
|
||||
|
||||
// generic reordered dequantize-matvec: each work-group owns a pair of
|
||||
// consecutive output rows and updates one 32-wide accumulator per row
|
||||
template <ggml_type T>
|
||||
// policies for dequantize_mul_mat_vec_reorder_esimd: row mapping and what to do with the two sums
|
||||
// plain mat-vec: a work-group owns two consecutive rows of one matrix
|
||||
struct esimd_dmmv_row_pair_policy {
|
||||
const void * vx;
|
||||
float * dst;
|
||||
|
||||
ESIMD_INLINE const void * w_a() const { return vx; }
|
||||
ESIMD_INLINE const void * w_b() const { return vx; }
|
||||
|
||||
ESIMD_INLINE int row_a(int g) const { return g * 2; }
|
||||
ESIMD_INLINE int row_b(int g) const { return g * 2 + 1; }
|
||||
ESIMD_INLINE bool has_b(int g, int nrows) const { return row_b(g) < nrows; }
|
||||
|
||||
ESIMD_INLINE void epilogue(int g, int nrows, float sum_a, float sum_b) const {
|
||||
dst[row_a(g)] = sum_a;
|
||||
if (has_b(g, nrows)) {
|
||||
dst[row_b(g)] = sum_b;
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
// fused gate+up+GLU: a work-group owns one row of both matrices; a is the gate, b the up
|
||||
struct esimd_dmmv_glu_policy {
|
||||
const void * vgate;
|
||||
const void * vup;
|
||||
float * dst;
|
||||
ggml_glu_op glu_op;
|
||||
|
||||
ESIMD_INLINE const void * w_a() const { return vgate; }
|
||||
ESIMD_INLINE const void * w_b() const { return vup; }
|
||||
|
||||
ESIMD_INLINE int row_a(int g) const { return g; }
|
||||
ESIMD_INLINE int row_b(int g) const { return g; }
|
||||
ESIMD_INLINE bool has_b(int, int) const { return true; } // up always contributes
|
||||
|
||||
ESIMD_INLINE void epilogue(int g, int, float gate, float up) const {
|
||||
using namespace sycl::ext::intel::esimd;
|
||||
|
||||
// SwiGLU: silu(gate)*up = gate*sigmoid(gate)*up. GEGLU's tanh form reduces
|
||||
// to the same shape via tanh(y) = 2*sigmoid(2y) - 1, so only the sigmoid
|
||||
// argument differs. sigmoid via exp2 (width-1 simd is a degenerate case for
|
||||
// the EM pipe, so broadcast to 16 lanes); lane 0 holds the result.
|
||||
const float GELU_COEF_A = 0.044715f;
|
||||
const float SQRT_2_OVER_PI = 0.79788456080286535587989211986876f;
|
||||
const float NEG_LOG2E = -1.44269504088896341f;
|
||||
|
||||
const float arg = (glu_op == GGML_GLU_OP_GEGLU) ?
|
||||
2.0f * SQRT_2_OVER_PI * gate * (1.0f + GELU_COEF_A * gate * gate) :
|
||||
gate;
|
||||
|
||||
simd<float, 16> t = arg * NEG_LOG2E;
|
||||
simd<float, 16> e = exp2(t);
|
||||
dst[g] = gate / (1.0f + e[0]) * up;
|
||||
}
|
||||
};
|
||||
|
||||
// generic reordered dequantize-matvec: each work-group owns a pair of rows chosen by the policy
|
||||
// and updates one 32-wide accumulator per row
|
||||
template <ggml_type T, typename Policy>
|
||||
ESIMD_INLINE void dequantize_mul_mat_vec_reorder_esimd(
|
||||
const void * vx, const float * y, float * dst,
|
||||
const Policy & policy, const float * y,
|
||||
const int ncols, const int nrows,
|
||||
sycl::local_accessor<float, 1> lmem,
|
||||
const sycl::nd_item<1> & it) {
|
||||
@@ -1886,41 +1941,41 @@ ESIMD_INLINE void dequantize_mul_mat_vec_reorder_esimd(
|
||||
|
||||
const int num_blocks_per_row = ncols / QK_K;
|
||||
const size_t nb = (size_t) nrows * num_blocks_per_row;
|
||||
const auto ps = traits::make_ptrs(vx, nb);
|
||||
const auto pa = traits::make_ptrs(policy.w_a(), nb);
|
||||
const auto pb = traits::make_ptrs(policy.w_b(), nb);
|
||||
|
||||
const int tid = it.get_local_id(0);
|
||||
const int row_pair = it.get_group(0);
|
||||
const int row0 = row_pair * 2; // two consecutive output rows
|
||||
const bool has_row1 = row0 + 1 < nrows;
|
||||
const int tid = it.get_local_id(0);
|
||||
const int g = it.get_group(0);
|
||||
const int row_a = policy.row_a(g);
|
||||
const int row_b = policy.row_b(g);
|
||||
const bool has_b = policy.has_b(g, nrows);
|
||||
|
||||
// one 32-wide accumulator per output row (small footprint, no spill)
|
||||
simd<float, 32> acc0 = 0.0f;
|
||||
simd<float, 32> acc1 = 0.0f;
|
||||
// one 32-wide accumulator per row (small footprint, no spill)
|
||||
simd<float, 32> acc_a = 0.0f;
|
||||
simd<float, 32> acc_b = 0.0f;
|
||||
|
||||
for (int ib = tid; ib < num_blocks_per_row; ib += GGML_SYCL_DMMV_ESIMD_WG_SIZE) {
|
||||
simd<float, 256> y_vec = block_load<float, 256>(y + (size_t) ib * QK_K);
|
||||
|
||||
const size_t bi0 = (size_t) (row0 + 0) * num_blocks_per_row + ib;
|
||||
const size_t bi1 = (size_t) (row0 + 1) * num_blocks_per_row + ib;
|
||||
const size_t bi_a = (size_t) row_a * num_blocks_per_row + ib;
|
||||
const size_t bi_b = (size_t) row_b * num_blocks_per_row + ib;
|
||||
|
||||
traits::mac_pair(ps, bi0, ps, bi1, has_row1, y_vec, acc0, acc1);
|
||||
traits::mac_pair(pa, bi_a, pb, bi_b, has_b, y_vec, acc_a, acc_b);
|
||||
}
|
||||
|
||||
lmem[tid * 2 + 0] = reduce<float>(acc0, std::plus<>{});
|
||||
lmem[tid * 2 + 1] = reduce<float>(acc1, std::plus<>{});
|
||||
lmem[tid * 2 + 0] = reduce<float>(acc_a, std::plus<>{});
|
||||
lmem[tid * 2 + 1] = reduce<float>(acc_b, std::plus<>{});
|
||||
it.barrier(sycl::access::fence_space::local_space);
|
||||
|
||||
if (tid == 0) {
|
||||
float sum0 = 0.0f;
|
||||
float sum1 = 0.0f;
|
||||
float sum_a = 0.0f;
|
||||
float sum_b = 0.0f;
|
||||
for (int p = 0; p < GGML_SYCL_DMMV_ESIMD_WG_SIZE; ++p) {
|
||||
sum0 += lmem[p * 2 + 0];
|
||||
sum1 += lmem[p * 2 + 1];
|
||||
}
|
||||
dst[row0 + 0] = sum0;
|
||||
if (has_row1) {
|
||||
dst[row0 + 1] = sum1;
|
||||
sum_a += lmem[p * 2 + 0];
|
||||
sum_b += lmem[p * 2 + 1];
|
||||
}
|
||||
|
||||
policy.epilogue(g, nrows, sum_a, sum_b);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1936,7 +1991,7 @@ static void dequantize_mul_mat_vec_q2_K_sycl_reorder_esimd(const void *vx, const
|
||||
sycl::nd_range<1>(sycl::range<1>((size_t)workgroups * GGML_SYCL_DMMV_ESIMD_WG_SIZE), sycl::range<1>(GGML_SYCL_DMMV_ESIMD_WG_SIZE)),
|
||||
[=](sycl::nd_item<1> it) [[intel::sycl_explicit_simd]] {
|
||||
dequantize_mul_mat_vec_reorder_esimd<GGML_TYPE_Q2_K>(
|
||||
vx, y, dst, ncols, nrows, lmem, it);
|
||||
esimd_dmmv_row_pair_policy{ vx, dst }, y, ncols, nrows, lmem, it);
|
||||
});
|
||||
});
|
||||
}
|
||||
@@ -1953,7 +2008,7 @@ static void dequantize_mul_mat_vec_q3_K_sycl_reorder_esimd(const void *vx, const
|
||||
sycl::nd_range<1>(sycl::range<1>((size_t)workgroups * GGML_SYCL_DMMV_ESIMD_WG_SIZE), sycl::range<1>(GGML_SYCL_DMMV_ESIMD_WG_SIZE)),
|
||||
[=](sycl::nd_item<1> it) [[intel::sycl_explicit_simd]] {
|
||||
dequantize_mul_mat_vec_reorder_esimd<GGML_TYPE_Q3_K>(
|
||||
vx, y, dst, ncols, nrows, lmem, it);
|
||||
esimd_dmmv_row_pair_policy{ vx, dst }, y, ncols, nrows, lmem, it);
|
||||
});
|
||||
});
|
||||
}
|
||||
@@ -1970,7 +2025,7 @@ static void dequantize_mul_mat_vec_q4_K_sycl_reorder_esimd(const void *vx, const
|
||||
sycl::nd_range<1>(sycl::range<1>((size_t)workgroups * GGML_SYCL_DMMV_ESIMD_WG_SIZE), sycl::range<1>(GGML_SYCL_DMMV_ESIMD_WG_SIZE)),
|
||||
[=](sycl::nd_item<1> it) [[intel::sycl_explicit_simd]] {
|
||||
dequantize_mul_mat_vec_reorder_esimd<GGML_TYPE_Q4_K>(
|
||||
vx, y, dst, ncols, nrows, lmem, it);
|
||||
esimd_dmmv_row_pair_policy{ vx, dst }, y, ncols, nrows, lmem, it);
|
||||
});
|
||||
});
|
||||
}
|
||||
@@ -1987,7 +2042,7 @@ static void dequantize_mul_mat_vec_q5_K_sycl_reorder_esimd(const void *vx, const
|
||||
sycl::nd_range<1>(sycl::range<1>((size_t)workgroups * GGML_SYCL_DMMV_ESIMD_WG_SIZE), sycl::range<1>(GGML_SYCL_DMMV_ESIMD_WG_SIZE)),
|
||||
[=](sycl::nd_item<1> it) [[intel::sycl_explicit_simd]] {
|
||||
dequantize_mul_mat_vec_reorder_esimd<GGML_TYPE_Q5_K>(
|
||||
vx, y, dst, ncols, nrows, lmem, it);
|
||||
esimd_dmmv_row_pair_policy{ vx, dst }, y, ncols, nrows, lmem, it);
|
||||
});
|
||||
});
|
||||
}
|
||||
@@ -2004,7 +2059,7 @@ static void dequantize_mul_mat_vec_q6_K_sycl_reorder_esimd(const void *vx, const
|
||||
sycl::nd_range<1>(sycl::range<1>((size_t)workgroups * GGML_SYCL_DMMV_ESIMD_WG_SIZE), sycl::range<1>(GGML_SYCL_DMMV_ESIMD_WG_SIZE)),
|
||||
[=](sycl::nd_item<1> it) [[intel::sycl_explicit_simd]] {
|
||||
dequantize_mul_mat_vec_reorder_esimd<GGML_TYPE_Q6_K>(
|
||||
vx, y, dst, ncols, nrows, lmem, it);
|
||||
esimd_dmmv_row_pair_policy{ vx, dst }, y, ncols, nrows, lmem, it);
|
||||
});
|
||||
});
|
||||
}
|
||||
@@ -2045,9 +2100,9 @@ ESIMD_INLINE void q8_0_mac_stripe(
|
||||
}
|
||||
}
|
||||
|
||||
template <int WG>
|
||||
template <int WG, typename Policy>
|
||||
ESIMD_INLINE void dequantize_mul_mat_vec_q8_0_reorder_esimd(
|
||||
const void * vx, const float * y, float * dst,
|
||||
const Policy & policy, const float * y,
|
||||
const int ncols, const int nrows,
|
||||
sycl::local_accessor<float, 1> lmem,
|
||||
const sycl::nd_item<1> & it) {
|
||||
@@ -2057,84 +2112,137 @@ ESIMD_INLINE void dequantize_mul_mat_vec_q8_0_reorder_esimd(
|
||||
|
||||
const int nblk_row = ncols / QK8_0;
|
||||
const size_t nb = (size_t) nrows * nblk_row;
|
||||
const int8_t * qs = (const int8_t *) vx;
|
||||
const sycl::half * d = (const sycl::half *) (qs + nb * QK8_0);
|
||||
const int8_t * qs_a = (const int8_t *) policy.w_a();
|
||||
const int8_t * qs_b = (const int8_t *) policy.w_b();
|
||||
const sycl::half * d_a = (const sycl::half *) (qs_a + nb * QK8_0);
|
||||
const sycl::half * d_b = (const sycl::half *) (qs_b + nb * QK8_0);
|
||||
|
||||
const int tid = it.get_local_id(0);
|
||||
const int row_pair = it.get_group(0);
|
||||
const int row0 = row_pair * 2;
|
||||
const bool has_row1 = row0 + 1 < nrows;
|
||||
const int tid = it.get_local_id(0);
|
||||
const int g = it.get_group(0);
|
||||
const bool has_b = policy.has_b(g, nrows);
|
||||
|
||||
const size_t base0 = (size_t) row0 * nblk_row;
|
||||
const size_t base1 = has_row1 ? (size_t) (row0 + 1) * nblk_row : base0;
|
||||
const size_t base_a = (size_t) policy.row_a(g) * nblk_row;
|
||||
const size_t base_b = (size_t) policy.row_b(g) * nblk_row;
|
||||
|
||||
simd<float, 32> acc0 = 0.0f;
|
||||
simd<float, 32> acc1 = 0.0f;
|
||||
simd<float, 32> acc_a = 0.0f;
|
||||
simd<float, 32> acc_b = 0.0f;
|
||||
|
||||
// Each thread processes one contiguous stripe.
|
||||
int ib = 0;
|
||||
for (; ib + WG * STRIPE <= nblk_row; ib += WG * STRIPE) {
|
||||
const int b = ib + tid * STRIPE;
|
||||
simd<float, 256> y_vec = block_load<float, 256>(y + (size_t) b * QK8_0);
|
||||
q8_0_mac_stripe<STRIPE>(qs + (base0 + b) * QK8_0, qs + (base1 + b) * QK8_0,
|
||||
d + base0 + b, d + base1 + b, has_row1, y_vec, acc0, acc1);
|
||||
q8_0_mac_stripe<STRIPE>(qs_a + (base_a + b) * QK8_0, qs_b + (base_b + b) * QK8_0,
|
||||
d_a + base_a + b, d_b + base_b + b, has_b, y_vec, acc_a, acc_b);
|
||||
}
|
||||
|
||||
// Distribute remaining blocks across the work-group.
|
||||
for (int b = ib + tid; b < nblk_row; b += WG) {
|
||||
simd<float, 32> y_vec = block_load<float, 32>(y + (size_t) b * QK8_0);
|
||||
q8_0_mac_stripe<1>(qs + (base0 + b) * QK8_0, qs + (base1 + b) * QK8_0,
|
||||
d + base0 + b, d + base1 + b, has_row1, y_vec, acc0, acc1);
|
||||
q8_0_mac_stripe<1>(qs_a + (base_a + b) * QK8_0, qs_b + (base_b + b) * QK8_0,
|
||||
d_a + base_a + b, d_b + base_b + b, has_b, y_vec, acc_a, acc_b);
|
||||
}
|
||||
|
||||
lmem[tid * 2 + 0] = reduce<float>(acc0, std::plus<>{});
|
||||
lmem[tid * 2 + 1] = reduce<float>(acc1, std::plus<>{});
|
||||
lmem[tid * 2 + 0] = reduce<float>(acc_a, std::plus<>{});
|
||||
lmem[tid * 2 + 1] = reduce<float>(acc_b, std::plus<>{});
|
||||
it.barrier(sycl::access::fence_space::local_space);
|
||||
|
||||
if (tid == 0) {
|
||||
float sum0 = 0.0f;
|
||||
float sum1 = 0.0f;
|
||||
float sum_a = 0.0f;
|
||||
float sum_b = 0.0f;
|
||||
for (int p = 0; p < WG; ++p) {
|
||||
sum0 += lmem[p * 2 + 0];
|
||||
sum1 += lmem[p * 2 + 1];
|
||||
}
|
||||
dst[row0 + 0] = sum0;
|
||||
if (has_row1) {
|
||||
dst[row0 + 1] = sum1;
|
||||
sum_a += lmem[p * 2 + 0];
|
||||
sum_b += lmem[p * 2 + 1];
|
||||
}
|
||||
|
||||
policy.epilogue(g, nrows, sum_a, sum_b);
|
||||
}
|
||||
}
|
||||
|
||||
template <int WG>
|
||||
static void q8_0_esimd_launch(const void * vx, const float * y, float * dst, const int ncols,
|
||||
const int nrows, dpct::queue_ptr stream) {
|
||||
const int workgroups = (nrows + 1) / 2;
|
||||
template <int WG, typename Policy>
|
||||
static void q8_0_esimd_launch(const Policy & policy, const float * y, const int ncols, const int nrows,
|
||||
const int workgroups, dpct::queue_ptr stream) {
|
||||
stream->submit([&](sycl::handler & h) {
|
||||
sycl::local_accessor<float, 1> lmem(sycl::range<1>(WG * 2), h);
|
||||
h.parallel_for(
|
||||
sycl::nd_range<1>(sycl::range<1>((size_t) workgroups * WG), sycl::range<1>(WG)),
|
||||
[=](sycl::nd_item<1> it) [[intel::sycl_explicit_simd]] {
|
||||
dequantize_mul_mat_vec_q8_0_reorder_esimd<WG>(vx, y, dst, ncols, nrows, lmem, it);
|
||||
dequantize_mul_mat_vec_q8_0_reorder_esimd<WG>(policy, y, ncols, nrows, lmem, it);
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
// Scale the work-group with the number of blocks per row.
|
||||
template <typename Policy>
|
||||
static void q8_0_esimd_launch_scaled(const Policy & policy, const float * y, const int ncols, const int nrows,
|
||||
const int workgroups, dpct::queue_ptr stream) {
|
||||
GGML_ASSERT(ncols % QK8_0 == 0);
|
||||
|
||||
const int nblk_row = ncols / QK8_0;
|
||||
if (nblk_row >= 64) {
|
||||
q8_0_esimd_launch<8>(policy, y, ncols, nrows, workgroups, stream);
|
||||
} else if (nblk_row >= 32) {
|
||||
q8_0_esimd_launch<4>(policy, y, ncols, nrows, workgroups, stream);
|
||||
} else if (nblk_row >= 16) {
|
||||
q8_0_esimd_launch<2>(policy, y, ncols, nrows, workgroups, stream);
|
||||
} else {
|
||||
q8_0_esimd_launch<1>(policy, y, ncols, nrows, workgroups, stream);
|
||||
}
|
||||
}
|
||||
|
||||
static void dequantize_mul_mat_vec_q8_0_sycl_reorder_esimd(const void *vx, const float *y,
|
||||
float *dst, const int ncols,
|
||||
const int nrows,
|
||||
dpct::queue_ptr stream) {
|
||||
GGML_ASSERT(ncols % QK8_0 == 0);
|
||||
q8_0_esimd_launch_scaled(esimd_dmmv_row_pair_policy{ vx, dst }, y, ncols, nrows, (nrows + 1) / 2, stream);
|
||||
}
|
||||
|
||||
// Scale the work-group with the number of blocks per row.
|
||||
const int nblk_row = ncols / QK8_0;
|
||||
if (nblk_row >= 64) {
|
||||
q8_0_esimd_launch<8>(vx, y, dst, ncols, nrows, stream);
|
||||
} else if (nblk_row >= 32) {
|
||||
q8_0_esimd_launch<4>(vx, y, dst, ncols, nrows, stream);
|
||||
} else if (nblk_row >= 16) {
|
||||
q8_0_esimd_launch<2>(vx, y, dst, ncols, nrows, stream);
|
||||
} else {
|
||||
q8_0_esimd_launch<1>(vx, y, dst, ncols, nrows, stream);
|
||||
// ESIMD fused gate+up+GLU on reordered K-quant SOA weights: one output row (gate
|
||||
// and up) per work-group, two independent 32-wide accumulators per super-block.
|
||||
// Reuses the DMMV kernels' dequant+MAC primitive (esimd_reorder_q_traits<T>::mac_pair).
|
||||
template <ggml_type T>
|
||||
static void dequantize_mul_mat_vec_glu_reorder_esimd(const void * vx, const void * vgate,
|
||||
const float * y, float * dst,
|
||||
const int ncols, const int nrows,
|
||||
const ggml_glu_op glu_op,
|
||||
dpct::queue_ptr stream) {
|
||||
GGML_ASSERT(ncols % QK_K == 0);
|
||||
const int workgroups = nrows; // one output row (gate+up) per work-group
|
||||
stream->submit([&](sycl::handler & h) {
|
||||
sycl::local_accessor<float, 1> lmem(sycl::range<1>(GGML_SYCL_DMMV_ESIMD_WG_SIZE * 2), h);
|
||||
h.parallel_for(
|
||||
sycl::nd_range<1>(sycl::range<1>((size_t) workgroups * GGML_SYCL_DMMV_ESIMD_WG_SIZE), sycl::range<1>(GGML_SYCL_DMMV_ESIMD_WG_SIZE)),
|
||||
[=](sycl::nd_item<1> it) [[intel::sycl_explicit_simd]] {
|
||||
dequantize_mul_mat_vec_reorder_esimd<T>(
|
||||
esimd_dmmv_glu_policy{ vgate, vx, dst, glu_op }, y, ncols, nrows, lmem, it);
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
bool ggml_sycl_dequantize_mul_mat_vec_glu_reorder_esimd(enum ggml_type src0_type, enum ggml_glu_op glu_op,
|
||||
const void * vx, const void * vgate, const float * y,
|
||||
float * dst, int ncols, int nrows, dpct::queue_ptr stream) {
|
||||
switch (src0_type) {
|
||||
case GGML_TYPE_Q2_K:
|
||||
dequantize_mul_mat_vec_glu_reorder_esimd<GGML_TYPE_Q2_K>(vx, vgate, y, dst, ncols, nrows, glu_op, stream);
|
||||
return true;
|
||||
case GGML_TYPE_Q3_K:
|
||||
dequantize_mul_mat_vec_glu_reorder_esimd<GGML_TYPE_Q3_K>(vx, vgate, y, dst, ncols, nrows, glu_op, stream);
|
||||
return true;
|
||||
case GGML_TYPE_Q4_K:
|
||||
dequantize_mul_mat_vec_glu_reorder_esimd<GGML_TYPE_Q4_K>(vx, vgate, y, dst, ncols, nrows, glu_op, stream);
|
||||
return true;
|
||||
case GGML_TYPE_Q5_K:
|
||||
dequantize_mul_mat_vec_glu_reorder_esimd<GGML_TYPE_Q5_K>(vx, vgate, y, dst, ncols, nrows, glu_op, stream);
|
||||
return true;
|
||||
case GGML_TYPE_Q6_K:
|
||||
dequantize_mul_mat_vec_glu_reorder_esimd<GGML_TYPE_Q6_K>(vx, vgate, y, dst, ncols, nrows, glu_op, stream);
|
||||
return true;
|
||||
case GGML_TYPE_Q8_0:
|
||||
q8_0_esimd_launch_scaled(esimd_dmmv_glu_policy{ vgate, vx, dst, glu_op }, y, ncols, nrows, nrows, stream);
|
||||
return true;
|
||||
default:
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -24,4 +24,13 @@ void ggml_sycl_op_dequantize_mul_mat_vec(
|
||||
const int64_t src1_ncols, const int64_t src1_padded_row_size,
|
||||
const dpct::queue_ptr &stream);
|
||||
|
||||
// Fused dense-FFN mat-vec: writes glu(gate . y, up . y) for same-type reordered
|
||||
// weights (Q2_K..Q6_K, Q8_0) and a single F32 activation column, using the ESIMD DMMV kernels.
|
||||
// vx is the up weight, vgate the gate weight. Only defined when GGML_SYCL_DMMV_HAS_ESIMD;
|
||||
// returns false if the type is unhandled.
|
||||
bool ggml_sycl_dequantize_mul_mat_vec_glu_reorder_esimd(
|
||||
enum ggml_type src0_type, enum ggml_glu_op glu_op,
|
||||
const void * vx, const void * vgate, const float * y,
|
||||
float * dst, int ncols, int nrows, dpct::queue_ptr stream);
|
||||
|
||||
#endif // GGML_SYCL_DMMV_HPP
|
||||
@@ -4,7 +4,7 @@
|
||||
#include <algorithm>
|
||||
|
||||
// mul_mat(gate) + mul_mat(up) + GLU: graph shape and tensor properties only. Backend state
|
||||
// (weight layout, split buffers, DMMV) is checked by ggml_sycl_mul_mat_glu_mmvq_fused().
|
||||
// (weight layout, split buffers, DMMV) is checked by ggml_sycl_op_mul_mat_glu_fused().
|
||||
static bool ggml_sycl_should_fuse_mul_mat_glu(const ggml_tensor * gate, const ggml_tensor * up,
|
||||
const ggml_tensor * glu) {
|
||||
// the fused epilogue implements these two; the rest fall back to the standalone GLU kernels
|
||||
@@ -32,12 +32,22 @@ static bool ggml_sycl_should_fuse_mul_mat_glu(const ggml_tensor * gate, const gg
|
||||
return false;
|
||||
}
|
||||
|
||||
// fused GEMVs walk whole QK_K super-blocks: the reorder kernel covers same-type
|
||||
// fused K-quant GEMVs walk whole QK_K super-blocks: the reorder kernel covers same-type
|
||||
// q4_K, the plain-layout kernel covers q5_K / iq4_xs pairs incl. mixed gate/up types
|
||||
const bool reorder_pair = wu->type == GGML_TYPE_Q4_K && wg->type == GGML_TYPE_Q4_K;
|
||||
const bool plain_pair = (wu->type == GGML_TYPE_Q5_K || wu->type == GGML_TYPE_IQ4_XS) &&
|
||||
(wg->type == GGML_TYPE_Q5_K || wg->type == GGML_TYPE_IQ4_XS);
|
||||
if ((!reorder_pair && !plain_pair) || wu->ne[0] % QK_K != 0) {
|
||||
// the ESIMD kernel covers same-type q2_K..q6_K and q8_0, single column only (as unfused DMMV)
|
||||
#ifdef GGML_SYCL_DMMV_HAS_ESIMD
|
||||
const bool esimd_pair = g_ggml_sycl_enable_esimd && wu->type == wg->type && act->ne[1] == 1 &&
|
||||
(wu->type == GGML_TYPE_Q2_K || wu->type == GGML_TYPE_Q3_K || wu->type == GGML_TYPE_Q4_K ||
|
||||
wu->type == GGML_TYPE_Q5_K || wu->type == GGML_TYPE_Q6_K || wu->type == GGML_TYPE_Q8_0);
|
||||
#else
|
||||
const bool esimd_pair = false;
|
||||
#endif
|
||||
// q8_0 rows are walked in 32-wide blocks, so they only need to be a multiple of QK8_0
|
||||
const int64_t row_align = esimd_pair && wu->type == GGML_TYPE_Q8_0 ? QK8_0 : QK_K;
|
||||
if ((!reorder_pair && !plain_pair && !esimd_pair) || wu->ne[0] % row_align != 0) {
|
||||
return false;
|
||||
}
|
||||
|
||||
|
||||
@@ -46,9 +46,6 @@
|
||||
# include <sycl/ext/oneapi/virtual_mem/virtual_mem.hpp>
|
||||
# define GGML_SYCL_SUPPORT_VMM
|
||||
#endif
|
||||
#if defined(__INTEL_LLVM_COMPILER)
|
||||
#define GGML_SYCL_DMMV_HAS_ESIMD
|
||||
#endif
|
||||
#include <sycl/half_type.hpp>
|
||||
|
||||
#include "ggml.h"
|
||||
@@ -5287,6 +5284,12 @@ static bool ggml_sycl_mul_mat_glu_mmvq_plain(ggml_backend_sycl_context & ctx, gg
|
||||
return false;
|
||||
}
|
||||
|
||||
// the plain kernel only covers q5_K / iq4_xs pairs: bail before quantizing the activation
|
||||
const auto plain_type = [](ggml_type t) { return t == GGML_TYPE_Q5_K || t == GGML_TYPE_IQ4_XS; };
|
||||
if (!plain_type(wu->type) || !plain_type(wg->type)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
// log the up mat-mul: glu's own srcs are the two intermediates the fusion never materialises
|
||||
scope_op_debug_print scope_dbg_print(__func__, up, /*num_src=*/2, " : fused with gate + GLU (plain layout)");
|
||||
|
||||
@@ -5311,9 +5314,42 @@ static bool ggml_sycl_mul_mat_glu_mmvq_plain(ggml_backend_sycl_context & ctx, gg
|
||||
GGML_UNUSED(gate);
|
||||
}
|
||||
|
||||
#ifdef GGML_SYCL_DMMV_HAS_ESIMD
|
||||
// {mul_mat(gate), mul_mat(up), GLU} on the reorder layout via the ESIMD DMMV kernel, for
|
||||
// same-type K-quant or Q8_0 pairs and a single activation column. It reads the F32 activation
|
||||
// directly, so one launch replaces three and no q8_1 quantization is needed.
|
||||
static bool ggml_sycl_mul_mat_glu_dmmv_esimd(ggml_backend_sycl_context & ctx, ggml_tensor * glu,
|
||||
ggml_tensor * gate, ggml_tensor * up, const ggml_tensor * wu,
|
||||
const ggml_tensor * wg, const ggml_tensor * act) {
|
||||
// the kernel reads the activation while writing the output; glu may reuse the dead activation's memory
|
||||
const char * act_begin = (const char *) act->data;
|
||||
const char * glu_begin = (const char *) glu->data;
|
||||
if (act_begin < glu_begin + ggml_nbytes(glu) && glu_begin < act_begin + ggml_nbytes(act)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
// install the reorder (SoA) layout the kernel needs, as the unfused DMMV path would
|
||||
opt_for_reorder(&ctx, wu, act, up, mul_mat_algo::DMMV);
|
||||
opt_for_reorder(&ctx, wg, act, gate, mul_mat_algo::DMMV);
|
||||
|
||||
const auto * extra_u = static_cast<const ggml_tensor_extra_gpu *>(wu->extra);
|
||||
const auto * extra_g = static_cast<const ggml_tensor_extra_gpu *>(wg->extra);
|
||||
if (!extra_u || !extra_g || !extra_u->optimized_feature.reorder || !extra_g->optimized_feature.reorder) {
|
||||
return false;
|
||||
}
|
||||
|
||||
// log the up mat-mul: glu's own srcs are the two intermediates the fusion never materialises
|
||||
scope_op_debug_print scope_dbg_print(__func__, up, /*num_src=*/2, " : fused with gate + GLU (ESIMD)");
|
||||
|
||||
return ggml_sycl_dequantize_mul_mat_vec_glu_reorder_esimd(wu->type, ggml_get_glu_op(glu), wu->data, wg->data,
|
||||
(const float *) act->data, (float *) glu->data,
|
||||
(int) wu->ne[0], (int) wu->ne[1], ctx.stream());
|
||||
}
|
||||
#endif
|
||||
|
||||
// Fused dense-FFN mat-vec for the {mul_mat(gate), mul_mat(up), GLU} subgraph at node_idx.
|
||||
// Returns false if it declined, in which case the caller runs the three nodes normally.
|
||||
static bool ggml_sycl_mul_mat_glu_mmvq_fused(ggml_backend_sycl_context & ctx, ggml_cgraph * cgraph, int node_idx) {
|
||||
static bool ggml_sycl_mul_mat_glu_fused(ggml_backend_sycl_context & ctx, ggml_cgraph * cgraph, int node_idx) {
|
||||
if (!ggml_sycl_can_fuse(cgraph, node_idx, { GGML_OP_MUL_MAT, GGML_OP_MUL_MAT, GGML_OP_GLU }, {})) {
|
||||
return false;
|
||||
}
|
||||
@@ -5331,6 +5367,16 @@ static bool ggml_sycl_mul_mat_glu_mmvq_fused(ggml_backend_sycl_context & ctx, gg
|
||||
return false;
|
||||
}
|
||||
|
||||
#ifdef GGML_SYCL_DMMV_HAS_ESIMD
|
||||
// single-column same-type K-quant or Q8_0 pairs take the ESIMD kernel, as the unfused mul-mats
|
||||
// would via DMMV; if the reorder layout could not be installed, fall through to mmvq
|
||||
if (g_ggml_sycl_enable_esimd && wu->type == wg->type && act->ne[1] == 1 &&
|
||||
ggml_sycl_supports_reorder_esimd(wu->type) &&
|
||||
ggml_sycl_mul_mat_glu_dmmv_esimd(ctx, glu, gate, up, wu, wg, act)) {
|
||||
return true;
|
||||
}
|
||||
#endif
|
||||
|
||||
// with DMMV prioritised the unfused path would not have gone through mmvq at all
|
||||
if (g_ggml_sycl_prioritize_dmmv) {
|
||||
return false;
|
||||
@@ -6594,7 +6640,7 @@ static void ggml_backend_sycl_graph_compute_impl(ggml_backend_sycl_context * syc
|
||||
continue;
|
||||
}
|
||||
|
||||
if (node->op == GGML_OP_MUL_MAT && ggml_sycl_mul_mat_glu_mmvq_fused(*sycl_ctx, cgraph, i)) {
|
||||
if (node->op == GGML_OP_MUL_MAT && ggml_sycl_mul_mat_glu_fused(*sycl_ctx, cgraph, i)) {
|
||||
i += 2;
|
||||
continue;
|
||||
}
|
||||
|
||||
@@ -11927,6 +11927,13 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() {
|
||||
true, 16, 8, b, false, true, false));
|
||||
}
|
||||
|
||||
for (int64_t k : {96, 896, 1024, 2880}) {
|
||||
for (ggml_glu_op glu_op : {GGML_GLU_OP_SWIGLU, GGML_GLU_OP_GEGLU}) {
|
||||
test_cases.emplace_back(new test_mul_mat_vec_fusion(GGML_TYPE_Q8_0, glu_op, 1, 32, k,
|
||||
false, 16, 8, false, false, true, false, {1, 1}));
|
||||
}
|
||||
}
|
||||
|
||||
// Fused row-pair coverage: minimum rows, an even pair, and an odd tail.
|
||||
// TODO: the max_nmse_err() for these cases is not estimated correctly causing sporadic false failures.
|
||||
//for (ggml_glu_op glu_op : { GGML_GLU_OP_SWIGLU, GGML_GLU_OP_GEGLU }) {
|
||||
|
||||
Reference in new issue
Block a user