Lumpiasty/llama.cpp
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

sycl : fix get_rows Q2_K, Q4_K, Q5_K (#25656)

Browse Source
This commit is contained in:
Neo Zhang
2026-07-15 10:32:28 +03:00
committed by GitHub
parent ae9291e16b
commit d3fba0c79d
2 changed files with 195 additions and 19 deletions
Download Patch File Download Diff File Expand all files Collapse all files
ggml/src/ggml-sycl/dequantize.hpp
+115 -16
View File
@@ -19,6 +19,7 @@
typedef void (*dequantize_kernel_t)(const void * vx, const int64_t ib, const int iqs, dfloat2 & v);
typedef void (*dequantize_kernel_t_reorder)(const void *d, const int64_t ib, const void *qs,
const int iqs, dfloat2 &v);
typedef void (*dequantize_kernel_f32_t)(const void * vx, const int64_t ib, const int iqs, float & v0, float & v1);
#if QK_K == 256
static inline void get_scale_min_k4(int j, const uint8_t * q, uint8_t & d, uint8_t & m);
@@ -85,6 +86,21 @@ static __dpct_inline__ void dequantize_q1_0_reorder(const void *d_ptr, const int
v.y() = (2 * bit_1 - 1) * d;
}
static __dpct_inline__ void dequantize_q1_0(const void *vx, const int64_t ib,
const int iqs, dfloat2 &v) {
const block_q1_0 * x = (const block_q1_0 *) vx;
const dfloat d = x[ib].d;
const int bit_index_0 = iqs + 0;
const int bit_index_1 = iqs + 1;
const int bit_0 = (x[ib].qs[bit_index_0 / 8] >> (bit_index_0 % 8)) & 1;
const int bit_1 = (x[ib].qs[bit_index_1 / 8] >> (bit_index_1 % 8)) & 1;
v.x() = (2 * bit_0 - 1) * d;
v.y() = (2 * bit_1 - 1) * d;
}
static __dpct_inline__ void dequantize_q4_1(const void *vx, const int64_t ib,
const int iqs, dfloat2 &v) {
const block_q4_1 * x = (const block_q4_1 *) vx;
@@ -140,6 +156,39 @@ static __dpct_inline__ void dequantize_q4_K(const void *vx, const int64_t ib,
#endif
}
static __dpct_inline__ void dequantize_q4_K_f32(const void *vx, const int64_t ib,
const int iqs, float &v0, float &v1) {
#if QK_K == 256
const block_q4_K * x = (const block_q4_K *) vx;
const sycl::half2 dm = x[ib].dm;
const float dall = dm[0];
const float dmin = dm[1];
auto dequantize_one = [&](const int idx) -> float {
const int il = idx / 64;
const int in = idx % 64;
const int is = 2 * il + (in >= 32 ? 1 : 0);
const int qsi = 32 * il + (in & 31);
uint8_t sc;
uint8_t m;
get_scale_min_k4(is, x[ib].scales, sc, m);
const float d = dall * sc;
const float mn = dmin * m;
const uint8_t q = x[ib].qs[qsi];
const uint8_t qv = (in >= 32) ? (q >> 4) : (q & 0xF);
return d * qv - mn;
};
v0 = dequantize_one(iqs + 0);
v1 = dequantize_one(iqs + 1);
#else
GGML_ABORT("Q4_K dequantize not supported for QK_K != 256");
#endif
}
static __dpct_inline__ void dequantize_q2_K(const void *vx, const int64_t ib,
const int iqs, dfloat2 &v) {
#if QK_K == 256
@@ -159,7 +208,7 @@ static __dpct_inline__ void dequantize_q2_K(const void *vx, const int64_t ib,
const float d = dall * (sc & 0xF);
const float m = dmin * (sc >> 4);
return sycl::fma((dfloat) ((q >> (2 * g)) & 3), (dfloat) d, (dfloat) (-m));
return (dfloat) d * (dfloat) ((q >> (2 * g)) & 3) - (dfloat) m;
};
v.x() = dequantize_one(iqs + 0);
@@ -169,6 +218,35 @@ static __dpct_inline__ void dequantize_q2_K(const void *vx, const int64_t ib,
#endif
}
static __dpct_inline__ void dequantize_q2_K_f32(const void *vx, const int64_t ib,
const int iqs, float &v0, float &v1) {
#if QK_K == 256
const block_q2_K * x = (const block_q2_K *) vx;
const float dall = x[ib].dm[0];
const float dmin = x[ib].dm[1];
auto dequantize_one = [&](const int idx) -> float {
const int n = idx / 128;
const int r = idx % 128;
const int g = r / 32;
const int l = r % 32;
const int is = 8 * n + l / 16;
const uint8_t q = x[ib].qs[32 * n + l];
const uint8_t sc = x[ib].scales[is + 2 * g];
const float d = dall * (sc & 0xF);
const float m = dmin * (sc >> 4);
return d * ((q >> (2 * g)) & 3) - m;
};
v0 = dequantize_one(iqs + 0);
v1 = dequantize_one(iqs + 1);
#else
GGML_ABORT("Q2_K dequantize not supported for QK_K != 256");
#endif
}
static __dpct_inline__ void dequantize_q3_K(const void *vx, const int64_t ib,
const int iqs, dfloat2 &v) {
#if QK_K == 256
@@ -242,6 +320,42 @@ static __dpct_inline__ void dequantize_q5_K(const void *vx, const int64_t ib,
#endif
}
static __dpct_inline__ void dequantize_q5_K_f32(const void *vx, const int64_t ib,
const int iqs, float &v0, float &v1) {
#if QK_K == 256
const block_q5_K * x = (const block_q5_K *) vx;
const float dall = x[ib].dm[0];
const float dmin = x[ib].dm[1];
auto dequantize_one = [&](const int idx) -> float {
const int il = idx / 64;
const int in = idx % 64;
const int is = 2 * il + (in >= 32 ? 1 : 0);
const int ir = (in & 31) / 2;
const int iq = in & 1;
const uint8_t q = x[ib].qs[32 * il + 2 * ir + iq];
const uint8_t h = x[ib].qh[2 * ir + iq];
const uint8_t qv = (in >= 32) ? (q >> 4) : (q & 0xF);
uint8_t sc;
uint8_t m;
get_scale_min_k4(is, x[ib].scales, sc, m);
const float d = dall * sc;
const float mn = dmin * m;
const uint8_t hm = 1 << (2 * il + (in >= 32 ? 1 : 0));
return (qv + ((h & hm) ? 16 : 0)) * d - mn;
};
v0 = dequantize_one(iqs + 0);
v1 = dequantize_one(iqs + 1);
#else
GGML_ABORT("Q5_K dequantize not supported for QK_K != 256");
#endif
}
static __dpct_inline__ void dequantize_q6_K(const void *vx, const int64_t ib,
const int iqs, dfloat2 &v) {
#if QK_K == 256
@@ -296,21 +410,6 @@ static __dpct_inline__ void dequantize_mxfp4(const void *vx, const int64_t ib,
v.y() = d * kvalues_mxfp4[q >> 4] * 0.5f;
}
static __dpct_inline__ void dequantize_q1_0(const void *vx, const int64_t ib,
const int iqs, dfloat2 &v) {
const block_q1_0 * x = (const block_q1_0 *) vx;
const dfloat d = x[ib].d;
const int bit_index_0 = iqs + 0;
const int bit_index_1 = iqs + 1;
const int bit_0 = (x[ib].qs[bit_index_0 / 8] >> (bit_index_0 % 8)) & 1;
const int bit_1 = (x[ib].qs[bit_index_1 / 8] >> (bit_index_1 % 8)) & 1;
v.x() = (2 * bit_0 - 1) * d;
v.y() = (2 * bit_1 - 1) * d;
}
static __dpct_inline__ void dequantize_nvfp4(const void *vx, const int64_t ib,
const int iqs, dfloat2 &v) {
const block_nvfp4 & xb = ((const block_nvfp4 *) vx)[ib];