opencl: fix image limit for q4_k dense bin kernels, q4_0 bcast and rms_norm (#30310)

* opencl: fallback when Q4_K weight images exceed device limits

Co-authored-by: Hongqiang Wang <wangh@qti.qualcomm.com>

* opencl: avoid incomplete subgroup in rms_norm

* opencl: refine conditions for dense bin kernels and fix bcast for q4_0

Co-authored-by: Hongqiang Wang <wangh@qti.qualcomm.com>

---------

Co-authored-by: Hongqiang Wang <wangh@qti.qualcomm.com>
This commit is contained in:
lhezandHongqiang Wang authored and GitHub committed 2026-10-11 09:41:29 -07:00
1 parent 5c8854fd88
commit c83f3058b6
1 file changed
+41 -11
+41 -11
View File
@@ -8583,6 +8583,11 @@ inline bool use_adreno_kernels(const ggml_backend_opencl_context *backend_ctx, c
bool threashold_ok = tensor->ne[0] >= threshold_ne0 && tensor->ne[1] >= threshold_ne1 &&
tensor->ne[2] == 1 && tensor->ne[3] == 1;
// the transposed layout needs K % 32 == 0 and M % 4 == 0
if (tensor->ne[0] % 32 != 0 || tensor->ne[1] % 4 != 0) {
return false;
}
// The noshuffle layout packs 2 rows per 32-bit texel and the GEMV reads it at an
// ne1/2 texel stride with an exact-cover dispatch, so it is only addressable when
// ne1 is a multiple of 64; an unaligned ne1 truncates the stride and the weight is
@@ -8731,7 +8736,10 @@ inline bool use_q4_0_bin_kernels(const ggml_backend_opencl_context *backend_ctx,
!backend_ctx->kernel_gemm_noshuffle_q4_0_f32_32b_trans_ila_a8_bin) {
return false;
}
return (tensor->ne[0] % 32 == 0) && (tensor->ne[1] % 64 == 0);
// bin kernels require ne2 == 1 and ne3 == 1 for weights
return tensor->ne[2] == 1 && tensor->ne[3] == 1 &&
(tensor->ne[0] % 32 == 0) && (tensor->ne[1] % 64 == 0);
#else
GGML_UNUSED(backend_ctx);
GGML_UNUSED(tensor);
@@ -8779,22 +8787,30 @@ static inline bool flat_large_m_enabled() {
return en;
}
// The noshuffle Q4_K weight image stores eight weights per uint texel.
static inline bool q4_K_weight_image_fits(const ggml_backend_opencl_context *backend_ctx, const ggml_tensor *tensor) {
const size_t texels = (size_t) ggml_nelements(tensor) / 8;
return texels != 0 && texels <= backend_ctx->image_max_buffer_size;
}
static inline bool use_flat_gemv_for_large_m_q4_K(const ggml_backend_opencl_context *backend_ctx, const ggml_tensor *tensor) {
if (tensor->ne[1] % 4 != 0 && tensor->ne[2] == 1 && tensor->ne[3] == 1) {
return true;
}
if (!flat_large_m_enabled()) {
if (tensor->ne[2] != 1 || tensor->ne[3] != 1 || use_q4k_tiled(backend_ctx, tensor)) {
return false;
}
// The image limit is a correctness guard, independent of the large-M opt-in.
if (!q4_K_weight_image_fits(backend_ctx, tensor)) {
return true;
}
// gemv_noshuffle variant perf drops for large M, use flat variant for large M.
// threshold is well above typical hidden/FFN dims, but below typical vocab sizes.
// note that this forces large M weights to use LM GEMM.
// EXCEPT when this branch's tiled-canonical lm_head/embed layout is active: the
// weight is converted to the 64-row tiled layout, which the flat gemv would
// misread as garbage. use_q4k_tiled owns these large-M weights, so defer to it.
return tensor->ne[1] >= 32768 && tensor->ne[2] == 1 && tensor->ne[3] == 1
&& !use_q4k_tiled(backend_ctx, tensor);
return flat_large_m_enabled() && tensor->ne[1] >= 32768;
}
static inline bool use_flat_gemv_for_large_m_q6_K(const ggml_backend_opencl_context *backend_ctx, const ggml_tensor *tensor) {
@@ -8846,7 +8862,9 @@ inline bool use_q6_k_bin_kernels(const ggml_backend_opencl_context *backend_ctx,
!backend_ctx->kernel_gemm_noshuffle_q6_k_f32_32b_trans_ila_a8_bin) {
return false;
}
return (tensor->ne[0] % 256 == 0) && (tensor->ne[1] % 64 == 0) &&
// bin kernels require ne2 == 1 and ne3 == 1 for weights
return tensor->ne[2] == 1 && tensor->ne[3] == 1 &&
(tensor->ne[0] % 256 == 0) && (tensor->ne[1] % 64 == 0) &&
!use_q6k_tiled(backend_ctx, tensor) && !use_flat_gemv_for_large_m_q6_K(backend_ctx, tensor);
#else
GGML_UNUSED(backend_ctx);
@@ -8861,7 +8879,9 @@ inline bool use_q4_k_bin_kernels(const ggml_backend_opencl_context *backend_ctx,
!backend_ctx->kernel_gemm_noshuffle_q4_k_f32_32b_trans_ila_a8_bin) {
return false;
}
return (tensor->ne[0] % 256 == 0) && (tensor->ne[1] % 64 == 0) &&
// bin kernels require ne2 == 1 and ne3 == 1 for weights
return tensor->ne[2] == 1 && tensor->ne[3] == 1 &&
(tensor->ne[0] % 256 == 0) && (tensor->ne[1] % 64 == 0) &&
!use_q4k_tiled(backend_ctx, tensor) && !use_flat_gemv_for_large_m_q4_K(backend_ctx, tensor);
#else
GGML_UNUSED(backend_ctx);
@@ -15071,7 +15091,7 @@ static void ggml_cl_rms_norm(ggml_backend_t backend, const ggml_tensor * src0, c
GGML_ASSERT(ne00 % 4 == 0);
const int nth = MIN(64, ne00);
const int nth = 64;
size_t global_work_size[] = {(size_t)ne01*nth, (size_t)ne02, (size_t)ne03};
size_t local_work_size[] = {(size_t)nth, 1, 1};
@@ -23453,7 +23473,17 @@ static void ggml_cl_mul_mat(ggml_backend_t backend, const ggml_tensor * src0, co
// quant kv without FA
// used for non-contiguous src0 (the usual head-major permuted K view when n_head_kv>1)
// AND for the contiguous case that occurs when n_head_kv==1 (e.g. Gemma-4 E2B)
if ((src0t == GGML_TYPE_Q4_0 || src0t == GGML_TYPE_Q8_0) &&
// Q4_0 bin kernels use a special weight layout, which the restore below does not support.
bool q4_0_bin_layout = false;
#ifdef GGML_OPENCL_USE_ADRENO_KERNELS
q4_0_bin_layout = src0t == GGML_TYPE_Q4_0 && src0->view_src == nullptr &&
ggml_is_contiguous(src0) &&
src0->ne[2] == 1 && src0->ne[3] == 1 &&
use_adreno_kernels(backend_ctx, src0) &&
!use_adreno_moe_kernels(backend_ctx, src0) &&
use_q4_0_bin_kernels(backend_ctx, src0);
#endif
if ((src0t == GGML_TYPE_Q4_0 || src0t == GGML_TYPE_Q8_0) && !q4_0_bin_layout &&
(!ggml_is_contiguous(src0) || src1->ne[2] > src0->ne[2])) {
cl_mem f16_buf = ggml_cl_mul_mat_dequant_quant_to_f16(backend_ctx, src0, nullptr);