opencl: add initial iq4_xs support (#31339)

* opencl: add iq4_xs AOS mv

Assisted-by: Codex

* opencl: add iq4_xs SOA mv and tiled mm

Assisted-by: Codex

* opencl: add iq4_xs noshuffle gemm/gemv for adreno

Co-authored-by: Shaofei Qi <shaoqi@qti.qualcomm.com>
Assisted-by: Codex

* opencl: add iq4_xs dp4a for adreno

Co-authored-by: Shaofei Qi <shaoqi@qti.qualcomm.com>
Assisted-by: Codex

* opencl: optimize iq4_xs adreno decode and narrow batches

Co-authored-by: Shaofei Qi <shaoqi@qti.qualcomm.com>

Assisted-by: Codex

---------

Co-authored-by: Li He <lih@qti.qualcomm.com>
This commit is contained in:
shaofeiqiandLi He authored and GitHub committed 2026-11-02 21:58:19 -08:00
1 parent 4402e65401
commit e6192de0a6
10 files changed
+1496 -1

No files matched your search

+7
View File
@@ -131,6 +131,9 @@ set(GGML_OPENCL_KERNELS
gemm_noshuffle_iq3_xxs_f32
gemm_noshuffle_iq3_xxs_q8_1_dp4a
gemv_noshuffle_iq3_xxs_f32
gemm_noshuffle_iq4_xs_f32
gemm_noshuffle_iq4_xs_q8_1_dp4a
gemv_noshuffle_iq4_xs_f32
gemm_noshuffle_q3_k_q8_1_dp4a
gemv_noshuffle_q3_k_f32
mul_mv_q2_k_f32
@@ -165,6 +168,10 @@ set(GGML_OPENCL_KERNELS
mul_mv_iq3_s_f32
mul_mv_iq3_s_f32_flat
mul_mm_iq3_s_f32_l4_lm
iq4_xs_grid
mul_mv_iq4_xs_f32
mul_mv_iq4_xs_f32_flat
mul_mm_iq4_xs_f32_l4_lm
mul_mv_q3_k_f32_flat
mul_mm_q3_k_f32_l4_lm
mul_mv_q2_k_f32_flat
+622 -1
View File
@@ -1074,6 +1074,16 @@ struct ggml_backend_opencl_context {
ggml_cl_buffer iq3_s_grid_buffer;
cl_mem iq3_s_grid_image = nullptr;
cl_kernel kernel_mul_mv_iq3_s_f32;
cl_kernel kernel_convert_block_iq4_xs, kernel_restore_block_iq4_xs;
cl_kernel kernel_convert_block_iq4_xs_noshuffle = nullptr, kernel_restore_block_iq4_xs_noshuffle = nullptr;
cl_kernel kernel_mul_mv_iq4_xs_f32_flat;
cl_kernel kernel_mul_mm_iq4_xs_f32_l4_lm;
cl_kernel kernel_gemv_noshuffle_iq4_xs_f32 = nullptr;
cl_kernel kernel_gemm_noshuffle_iq4_xs_f32 = nullptr;
cl_kernel kernel_gemm_noshuffle_iq4_xs_q8_1_dp4a = nullptr;
cl_kernel kernel_gemm_noshuffle_iq4_xs_q8_1_dp4a_narrow = nullptr;
int iq4_xs_mv_nsg = 8;
cl_kernel kernel_mul_mv_iq4_xs_f32;
cl_kernel kernel_mul_mv_q4_K_f32;
cl_kernel kernel_mul_mv_q4_K_f32_flat;
cl_kernel kernel_mul_mv_q5_K_f32;
@@ -2163,6 +2173,10 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) {
CL_CHECK((backend_ctx->kernel_restore_block_iq3_xxs = clCreateKernel(backend_ctx->program_cvt, "kernel_restore_block_iq3_xxs", &err), err));
CL_CHECK((backend_ctx->kernel_convert_block_iq3_s = clCreateKernel(backend_ctx->program_cvt, "kernel_convert_block_iq3_s", &err), err));
CL_CHECK((backend_ctx->kernel_restore_block_iq3_s = clCreateKernel(backend_ctx->program_cvt, "kernel_restore_block_iq3_s", &err), err));
CL_CHECK((backend_ctx->kernel_convert_block_iq4_xs_noshuffle = clCreateKernel(backend_ctx->program_cvt, "kernel_convert_block_iq4_xs_noshuffle", &err), err));
CL_CHECK((backend_ctx->kernel_restore_block_iq4_xs_noshuffle = clCreateKernel(backend_ctx->program_cvt, "kernel_restore_block_iq4_xs_noshuffle", &err), err));
CL_CHECK((backend_ctx->kernel_convert_block_iq4_xs = clCreateKernel(backend_ctx->program_cvt, "kernel_convert_block_iq4_xs", &err), err));
CL_CHECK((backend_ctx->kernel_restore_block_iq4_xs = clCreateKernel(backend_ctx->program_cvt, "kernel_restore_block_iq4_xs", &err), err));
CL_CHECK((backend_ctx->kernel_convert_block_q3_K = clCreateKernel(backend_ctx->program_cvt, "kernel_convert_block_q3_K", &err), err));
CL_CHECK((backend_ctx->kernel_restore_block_q3_K = clCreateKernel(backend_ctx->program_cvt, "kernel_restore_block_q3_K", &err), err));
CL_CHECK((backend_ctx->kernel_convert_block_q2_K = clCreateKernel(backend_ctx->program_cvt, "kernel_convert_block_q2_K", &err), err));
@@ -3054,6 +3068,59 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) {
GGML_LOG_CONT(".");
}
#ifdef GGML_OPENCL_EMBED_KERNELS
const std::string iq4_xs_grid_src {
#include "iq4_xs_grid.cl.h"
};
#else
const std::string iq4_xs_grid_src = read_file("iq4_xs_grid.cl");
#endif
// mul_mv_iq4_xs_f32_flat
{
#ifdef GGML_OPENCL_EMBED_KERNELS
const std::string kernel_src {
#include "mul_mv_iq4_xs_f32_flat.cl.h"
};
#else
const std::string kernel_src = read_file("mul_mv_iq4_xs_f32_flat.cl");
#endif
cl_program prog = build_program_from_source(backend_ctx, (iq4_xs_grid_src + kernel_src).c_str(), compile_opts);
CL_CHECK((backend_ctx->kernel_mul_mv_iq4_xs_f32_flat = clCreateKernel(prog, "kernel_mul_mv_iq4_xs_f32_flat", &err), err));
CL_CHECK(clReleaseProgram(prog));
GGML_LOG_CONT(".");
}
// mul_mm_iq4_xs_f32_l4_lm
{
#ifdef GGML_OPENCL_EMBED_KERNELS
const std::string kernel_src {
#include "mul_mm_iq4_xs_f32_l4_lm.cl.h"
};
#else
const std::string kernel_src = read_file("mul_mm_iq4_xs_f32_l4_lm.cl");
#endif
cl_program prog = build_program_from_source(backend_ctx, (iq4_xs_grid_src + kernel_src).c_str(), compile_opts);
CL_CHECK((backend_ctx->kernel_mul_mm_iq4_xs_f32_l4_lm = clCreateKernel(prog, "kernel_mul_mm_iq4_xs_f32_l4_lm", &err), err));
CL_CHECK(clReleaseProgram(prog));
GGML_LOG_CONT(".");
}
// mul_mv_iq4_xs_f32
{
#ifdef GGML_OPENCL_EMBED_KERNELS
const std::string kernel_src {
#include "mul_mv_iq4_xs_f32.cl.h"
};
#else
const std::string kernel_src = read_file("mul_mv_iq4_xs_f32.cl");
#endif
cl_program prog = build_program_from_source(backend_ctx, (iq4_xs_grid_src + kernel_src).c_str(), compile_opts);
CL_CHECK((backend_ctx->kernel_mul_mv_iq4_xs_f32 = clCreateKernel(prog, "kernel_mul_mv_iq4_xs_f32", &err), err));
CL_CHECK(clReleaseProgram(prog));
GGML_LOG_CONT(".");
}
// mul_mv_q3_k_f32
{
#ifdef GGML_OPENCL_EMBED_KERNELS
@@ -4915,6 +4982,39 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) {
GGML_LOG_CONT(".");
}
// gemm_noshuffle_iq4_xs_q8_1_dp4a
{
#ifdef GGML_OPENCL_EMBED_KERNELS
const std::string kernel_src {
#include "gemm_noshuffle_iq4_xs_q8_1_dp4a.cl.h"
};
#else
const std::string kernel_src = read_file("gemm_noshuffle_iq4_xs_q8_1_dp4a.cl");
#endif
cl_program prog = build_program_from_source(backend_ctx, (iq4_xs_grid_src + kernel_src).c_str(), compile_opts);
CL_CHECK((backend_ctx->kernel_gemm_noshuffle_iq4_xs_q8_1_dp4a = clCreateKernel(prog, "kernel_gemm_noshuffle_iq4_xs_q8_1_dp4a", &err), err));
CL_CHECK(clReleaseProgram(prog));
prog = build_program_from_source(backend_ctx, (iq4_xs_grid_src + kernel_src).c_str(), compile_opts + " -DTILESIZE_N=8");
CL_CHECK((backend_ctx->kernel_gemm_noshuffle_iq4_xs_q8_1_dp4a_narrow = clCreateKernel(prog, "kernel_gemm_noshuffle_iq4_xs_q8_1_dp4a", &err), err));
CL_CHECK(clReleaseProgram(prog));
GGML_LOG_CONT(".");
}
// gemm_noshuffle_iq4_xs_f32
{
#ifdef GGML_OPENCL_EMBED_KERNELS
const std::string kernel_src {
#include "gemm_noshuffle_iq4_xs_f32.cl.h"
};
#else
const std::string kernel_src = read_file("gemm_noshuffle_iq4_xs_f32.cl");
#endif
cl_program prog = build_program_from_source(backend_ctx, (iq4_xs_grid_src + kernel_src).c_str(), compile_opts);
CL_CHECK((backend_ctx->kernel_gemm_noshuffle_iq4_xs_f32 = clCreateKernel(prog, "kernel_gemm_noshuffle_iq4_xs_f32", &err), err));
CL_CHECK(clReleaseProgram(prog));
GGML_LOG_CONT(".");
}
// gemm_noshuffle_q2_0_f32
{
#ifdef GGML_OPENCL_EMBED_KERNELS
@@ -4930,6 +5030,41 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) {
GGML_LOG_CONT(".");
}
// gemv_noshuffle_iq4_xs_f32
{
#ifdef GGML_OPENCL_EMBED_KERNELS
const std::string kernel_src {
#include "gemv_noshuffle_iq4_xs_f32.cl.h"
};
#else
const std::string kernel_src = read_file("gemv_noshuffle_iq4_xs_f32.cl");
#endif
if (const char * env = getenv("GGML_OPENCL_IQ4_XS_MV_NSG")) {
const int nsg = atoi(env);
if (nsg == 1 || nsg == 2 || nsg == 4 || nsg == 8) {
backend_ctx->iq4_xs_mv_nsg = nsg;
}
}
for (;;) {
const std::string opts = compile_opts + " -DIQ4_XS_MV_NSG=" + std::to_string(backend_ctx->iq4_xs_mv_nsg);
cl_program prog = build_program_from_source(backend_ctx, (iq4_xs_grid_src + kernel_src).c_str(), opts);
cl_kernel kernel = clCreateKernel(prog, "kernel_gemv_noshuffle_iq4_xs_f32", &err);
CL_CHECK(err);
size_t max_workgroup;
CL_CHECK(clGetKernelWorkGroupInfo(kernel, backend_ctx->device, CL_KERNEL_WORK_GROUP_SIZE, sizeof(max_workgroup), &max_workgroup, nullptr));
if (64*(size_t)backend_ctx->iq4_xs_mv_nsg <= max_workgroup) {
backend_ctx->kernel_gemv_noshuffle_iq4_xs_f32 = kernel;
CL_CHECK(clReleaseProgram(prog));
break;
}
CL_CHECK(clReleaseKernel(kernel));
CL_CHECK(clReleaseProgram(prog));
GGML_ASSERT(backend_ctx->iq4_xs_mv_nsg > 1);
backend_ctx->iq4_xs_mv_nsg /= 2;
}
GGML_LOG_CONT(".");
}
// gemv_noshuffle_q2_0_f32
{
#ifdef GGML_OPENCL_EMBED_KERNELS
@@ -9848,6 +9983,27 @@ struct ggml_tensor_extra_cl_iq1_m {
}
};
struct ggml_tensor_extra_cl_iq4_xs {
bool noshuffle = false;
cl_mem qs = nullptr;
cl_mem sl = nullptr;
cl_mem sh = nullptr;
cl_mem d = nullptr;
size_t size_qs = 0, size_sl = 0, size_sh = 0, size_d = 0;
~ggml_tensor_extra_cl_iq4_xs() { reset(); }
void reset() {
noshuffle = false;
if (qs != nullptr) { CL_CHECK(clReleaseMemObject(qs)); qs = nullptr; }
if (sl != nullptr) { CL_CHECK(clReleaseMemObject(sl)); sl = nullptr; }
if (sh != nullptr) { CL_CHECK(clReleaseMemObject(sh)); sh = nullptr; }
if (d != nullptr) { CL_CHECK(clReleaseMemObject(d)); d = nullptr; }
size_qs = size_sl = size_sh = size_d = 0;
}
};
struct ggml_tensor_extra_cl_q6_K {
// Lower 4 bits of quantized weights.
cl_mem ql = nullptr;
@@ -10966,6 +11122,24 @@ static bool ggml_cl_q3_k_use_noshuffle(const ggml_backend_opencl_context * backe
#endif
}
static bool ggml_cl_iq4_xs_use_noshuffle(const ggml_backend_opencl_context * backend_ctx, const ggml_tensor * tensor) {
#if defined(GGML_OPENCL_SOA_Q) && defined(GGML_OPENCL_USE_ADRENO_KERNELS)
if (const char * env = getenv("GGML_OPENCL_IQ4_XS_NOSHUFFLE")) {
if (env[0] == '0') {
return false;
}
}
return tensor->type == GGML_TYPE_IQ4_XS &&
backend_ctx->gpu_family == ADRENO &&
ggml_is_contiguous(tensor) && !tensor->view_src &&
use_adreno_kernels(backend_ctx, tensor);
#else
GGML_UNUSED(backend_ctx);
GGML_UNUSED(tensor);
return false;
#endif
}
static bool ggml_cl_iq3_s_use_noshuffle(const ggml_backend_opencl_context * backend_ctx, const ggml_tensor * tensor) {
#if defined(GGML_OPENCL_SOA_Q) && defined(GGML_OPENCL_USE_ADRENO_KERNELS)
if (const char * env = getenv("GGML_OPENCL_IQ3_S_NOSHUFFLE")) {
@@ -11699,6 +11873,12 @@ static bool ggml_opencl_supports_op(ggml_backend_dev_t dev, const struct ggml_te
return false;
}
return op->src[1]->type == GGML_TYPE_F32 && ggml_is_contiguous_rows(op->src[1]);
} else if (op->src[0]->type == GGML_TYPE_IQ4_XS) {
const ggml_tensor * weight = op->src[0]->view_src;
if (weight && ggml_cl_iq4_xs_use_noshuffle(backend_ctx, weight)) {
return false;
}
return op->src[1]->type == GGML_TYPE_F32 && ggml_is_contiguous_rows(op->src[1]);
} else if (op->src[0]->type == GGML_TYPE_Q3_K) {
const ggml_tensor * weight = op->src[0]->view_src;
if (weight && ggml_cl_q3_k_use_noshuffle(backend_ctx, weight)) {
@@ -12130,6 +12310,9 @@ struct ggml_backend_opencl_buffer_context {
for (ggml_tensor_extra_cl_iq3_xxs * e : temp_tensor_extras_iq3_xxs) {
delete e;
}
for (ggml_tensor_extra_cl_iq4_xs * e : temp_tensor_extras_iq4_xs) {
delete e;
}
for (ggml_tensor_extra_cl_q6_K * e : temp_tensor_extras_q6_K_in_use) {
delete e;
}
@@ -12163,6 +12346,9 @@ struct ggml_backend_opencl_buffer_context {
for (ggml_tensor_extra_cl_iq3_xxs * e : temp_tensor_extras_iq3_xxs_in_use) {
delete e;
}
for (ggml_tensor_extra_cl_iq4_xs * e : temp_tensor_extras_iq4_xs_in_use) {
delete e;
}
for (ggml_tensor_extra_cl_q5_K * e : temp_tensor_extras_q5_K) {
delete e;
}
@@ -12481,6 +12667,20 @@ struct ggml_backend_opencl_buffer_context {
}
temp_tensor_extras_iq3_xxs_in_use.push_back(extra);
extra->reset();
return extra;
}
ggml_tensor_extra_cl_iq4_xs * ggml_opencl_alloc_temp_tensor_extra_iq4_xs() {
ggml_tensor_extra_cl_iq4_xs * extra;
if (temp_tensor_extras_iq4_xs.empty()) {
extra = new ggml_tensor_extra_cl_iq4_xs();
} else {
extra = temp_tensor_extras_iq4_xs.back();
temp_tensor_extras_iq4_xs.pop_back();
}
temp_tensor_extras_iq4_xs_in_use.push_back(extra);
extra->reset();
return extra;
@@ -12611,6 +12811,10 @@ struct ggml_backend_opencl_buffer_context {
temp_tensor_extras_iq3_xxs.push_back(e);
}
temp_tensor_extras_iq3_xxs_in_use.clear();
for (ggml_tensor_extra_cl_iq4_xs * e : temp_tensor_extras_iq4_xs_in_use) {
temp_tensor_extras_iq4_xs.push_back(e);
}
temp_tensor_extras_iq4_xs_in_use.clear();
q8_0_soa_tensors.clear();
q4_0_soa_tensors.clear();
@@ -12663,6 +12867,8 @@ struct ggml_backend_opencl_buffer_context {
std::vector<ggml_tensor_extra_cl_iq3_s *> temp_tensor_extras_iq3_s;
std::vector<ggml_tensor_extra_cl_iq3_s *> temp_tensor_extras_iq3_s_in_use;
std::vector<ggml_tensor_extra_cl_iq3_xxs *> temp_tensor_extras_iq3_xxs_in_use;
std::vector<ggml_tensor_extra_cl_iq4_xs *> temp_tensor_extras_iq4_xs;
std::vector<ggml_tensor_extra_cl_iq4_xs *> temp_tensor_extras_iq4_xs_in_use;
std::vector<ggml_tensor_extra_cl_q6_K *> temp_tensor_extras_q6_K;
std::vector<ggml_tensor_extra_cl_q6_K *> temp_tensor_extras_q6_K_in_use;
@@ -14599,6 +14805,87 @@ static void ggml_backend_opencl_buffer_set_tensor(ggml_backend_buffer_t buffer,
#endif
return;
}
if (tensor->type == GGML_TYPE_IQ4_XS) {
GGML_ASSERT(offset == 0 && size == ggml_nbytes(tensor));
ggml_tensor_extra_cl * extra_orig = (ggml_tensor_extra_cl *)tensor->extra;
GGML_ASSERT(extra_orig && "Tensors in OpenCL backend should have been allocated and initialized");
ggml_backend_opencl_buffer_context * ctx = (ggml_backend_opencl_buffer_context *) buffer->context;
ggml_tensor_extra_cl_iq4_xs * extra = ctx->ggml_opencl_alloc_temp_tensor_extra_iq4_xs();
extra->noshuffle = ggml_cl_iq4_xs_use_noshuffle(backend_ctx, tensor);
const size_t blck = (size_t)ggml_blck_size(tensor->type);
const size_t n_blk = ggml_nelements(tensor) / blck;
const size_t size_qs = n_blk * 128;
const size_t size_sl = n_blk * 4;
const size_t size_sh = n_blk * 2;
const size_t size_d = n_blk * 2;
GGML_ASSERT(size_qs + size_sl + size_sh + size_d == ggml_nbytes(tensor) && "Incorrect tensor size");
cl_int err;
cl_mem data_device = clCreateBuffer(context, CL_MEM_READ_WRITE,
ggml_nbytes(tensor), NULL, &err);
CL_CHECK(err);
CL_CHECK(clEnqueueWriteBuffer(
queue, data_device, CL_TRUE, 0,
ggml_nbytes(tensor), data, 0, NULL, NULL));
cl_buffer_region region;
region.origin = align_to(extra_orig->offset + tensor->view_offs + offset, backend_ctx->alignment);
region.size = size_qs;
CL_CHECK((extra->qs = clCreateSubBuffer(extra_orig->data_device, CL_MEM_READ_WRITE, CL_BUFFER_CREATE_TYPE_REGION, &region, &err), err));
region.origin = align_to(region.origin + size_qs, backend_ctx->alignment);
region.size = size_sl;
CL_CHECK((extra->sl = clCreateSubBuffer(extra_orig->data_device, CL_MEM_READ_WRITE, CL_BUFFER_CREATE_TYPE_REGION, &region, &err), err));
region.origin = align_to(region.origin + size_sl, backend_ctx->alignment);
region.size = size_sh;
CL_CHECK((extra->sh = clCreateSubBuffer(extra_orig->data_device, CL_MEM_READ_WRITE, CL_BUFFER_CREATE_TYPE_REGION, &region, &err), err));
region.origin = align_to(region.origin + size_sh, backend_ctx->alignment);
region.size = size_d;
CL_CHECK((extra->d = clCreateSubBuffer(extra_orig->data_device, CL_MEM_READ_WRITE, CL_BUFFER_CREATE_TYPE_REGION, &region, &err), err));
cl_kernel kernel = extra->noshuffle ? backend_ctx->kernel_convert_block_iq4_xs_noshuffle : backend_ctx->kernel_convert_block_iq4_xs;
cl_ulong nb_arg = (cl_ulong)n_blk;
CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &data_device));
CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra->qs));
CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra->sl));
CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra->sh));
CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extra->d));
CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &nb_arg));
size_t global_work_size[] = { (size_t)CEIL_DIV(n_blk, 64)*64, 1, 1 };
size_t local_work_size[] = { 64, 1, 1 };
cl_event evt;
CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 3, NULL, global_work_size, local_work_size, 0, NULL, &evt));
CL_CHECK(clWaitForEvents(1, &evt));
CL_CHECK(clReleaseEvent(evt));
CL_CHECK(clReleaseMemObject(data_device));
#ifdef GGML_OPENCL_USE_ADRENO_KERNELS
if (extra->noshuffle) {
const int m = tensor->ne[1];
const int k = tensor->ne[0];
transpose_2d_as_16b(backend_ctx, extra->qs, extra->qs, size_qs, k/4, m, true, true);
transpose_2d_as_32b(backend_ctx, extra->sl, extra->sl, size_sl, k/256, m, true, true);
transpose_2d_as_16b(backend_ctx, extra->sh, extra->sh, size_sh, k/256, m, true, true);
transpose_2d_as_16b(backend_ctx, extra->d, extra->d, size_d, k/256, m, true, true);
}
#endif
extra->size_qs = size_qs;
extra->size_sl = size_sl;
extra->size_sh = size_sh;
extra->size_d = size_d;
tensor->extra = extra;
return;
}
if (tensor->type == GGML_TYPE_Q4_K) {
ggml_tensor_extra_cl * extra_orig = (ggml_tensor_extra_cl *)tensor->extra;
GGML_ASSERT(extra_orig && "Tesnors in OpenCL backend should have been allocated and initialized");
@@ -16480,6 +16767,61 @@ static void ggml_backend_opencl_buffer_get_tensor(ggml_backend_buffer_t buffer,
CL_CHECK(clReleaseMemObject(data_device));
return;
}
if (tensor->type == GGML_TYPE_IQ4_XS) {
if (tensor->view_src) {
offset += tensor->view_offs;
tensor = tensor->view_src;
}
ggml_tensor_extra_cl_iq4_xs * extra = (ggml_tensor_extra_cl_iq4_xs *)tensor->extra;
const size_t blck = (size_t)ggml_blck_size(tensor->type);
ggml_cl_buffer buf_unpacked;
buf_unpacked.allocate(backend_ctx->context, ggml_nbytes(tensor));
const size_t n_blk = (size_t)ggml_nelements(tensor) / blck;
cl_ulong nb_arg = (cl_ulong)n_blk;
cl_mem qs = extra->qs, sl = extra->sl, sh = extra->sh, d = extra->d;
#ifdef GGML_OPENCL_USE_ADRENO_KERNELS
ggml_cl_buffer buf_qs, buf_sl, buf_sh, buf_d;
if (extra->noshuffle) {
buf_qs.allocate(context, extra->size_qs);
buf_sl.allocate(context, extra->size_sl);
buf_sh.allocate(context, extra->size_sh);
buf_d.allocate(context, extra->size_d);
qs = buf_qs.buffer;
sl = buf_sl.buffer;
sh = buf_sh.buffer;
d = buf_d.buffer;
const int m = tensor->ne[1];
const int k = tensor->ne[0];
transpose_2d_as_16b(backend_ctx, extra->qs, qs, extra->size_qs, m, k/4, true, true);
transpose_2d_as_32b(backend_ctx, extra->sl, sl, extra->size_sl, m, k/256, true, true);
transpose_2d_as_16b(backend_ctx, extra->sh, sh, extra->size_sh, m, k/256, true, true);
transpose_2d_as_16b(backend_ctx, extra->d, d, extra->size_d, m, k/256, true, true);
}
#endif
cl_kernel kernel = extra->noshuffle ? backend_ctx->kernel_restore_block_iq4_xs_noshuffle : backend_ctx->kernel_restore_block_iq4_xs;
CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &qs));
CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &sl));
CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &sh));
CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &d));
CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &buf_unpacked.buffer));
CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &nb_arg));
size_t gws[] = { (n_blk + 63) / 64 * 64 };
size_t lws[] = { 64 };
cl_event evt;
CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 1, NULL, gws, lws, 0, NULL, &evt));
CL_CHECK(clWaitForEvents(1, &evt));
CL_CHECK(clReleaseEvent(evt));
CL_CHECK(clEnqueueReadBuffer(queue, buf_unpacked.buffer, CL_TRUE, offset, size, data, 0, NULL, NULL));
return;
}
if (tensor->type == GGML_TYPE_Q4_K) {
ggml_tensor_extra_cl_q4_K * extra = (ggml_tensor_extra_cl_q4_K *)tensor->extra;
@@ -25466,6 +25808,169 @@ static void ggml_cl_mul_mat_iq4_nl_f32_adreno(ggml_backend_t backend, const ggml
#endif
}
#ifdef GGML_OPENCL_USE_ADRENO_KERNELS
static bool ggml_cl_mul_mat_iq4_xs_dp4a(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {
ggml_backend_opencl_context * backend_ctx = (ggml_backend_opencl_context *)backend->context;
ggml_tensor_extra_cl_iq4_xs * extra0 = (ggml_tensor_extra_cl_iq4_xs *)src0->extra;
ggml_tensor_extra_cl * extra1 = (ggml_tensor_extra_cl *)src1->extra;
ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra;
cl_ulong offset1 = extra1->offset + src1->view_offs;
cl_ulong offsetd = extrad->offset + dst->view_offs;
const int m = src0->ne[1];
const int n = src1->ne[1];
const int k = src0->ne[0];
const int tile_n = n <= 8 ? 8 : 32;
cl_kernel gemm = n <= 8 ? backend_ctx->kernel_gemm_noshuffle_iq4_xs_q8_1_dp4a_narrow
: backend_ctx->kernel_gemm_noshuffle_iq4_xs_q8_1_dp4a;
const char * env = getenv("GGML_OPENCL_IQ4_XS_DENSE_DP4A");
if (!gemm || !backend_ctx->has_integer_dot || (env && env[0] == '0') ||
n <= 1 || !ggml_is_contiguous(src1) || offset1 % backend_ctx->alignment != 0) {
return false;
}
cl_int err;
cl_buffer_region region = { (size_t)offset1, (size_t)k*n*sizeof(float) };
cl_mem activation = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, &region, &err);
CL_CHECK(err);
const size_t n_blocks = (size_t)n*(k/32);
backend_ctx->prealloc_moe_qa.allocate(backend_ctx->context, (size_t)n*k);
backend_ctx->prealloc_moe_da.allocate(backend_ctx->context, n_blocks*sizeof(cl_half));
backend_ctx->prealloc_moe_sa.allocate(backend_ctx->context, n_blocks*sizeof(cl_half));
cl_int nb = n_blocks;
cl_kernel quant = backend_ctx->kernel_quant_a_q8_1;
CL_CHECK(clSetKernelArg(quant, 0, sizeof(cl_mem), &activation));
CL_CHECK(clSetKernelArg(quant, 1, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer));
CL_CHECK(clSetKernelArg(quant, 2, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer));
CL_CHECK(clSetKernelArg(quant, 3, sizeof(cl_mem), &backend_ctx->prealloc_moe_sa.buffer));
CL_CHECK(clSetKernelArg(quant, 4, sizeof(cl_int), &nb));
size_t q_global[] = { CEIL_DIV(n_blocks, 64)*64 };
size_t q_local[] = { 64 };
backend_ctx->enqueue_ndrange_kernel(quant, 1, q_global, q_local, dst);
CL_CHECK(clSetKernelArg(gemm, 0, sizeof(cl_mem), &extra0->qs));
CL_CHECK(clSetKernelArg(gemm, 1, sizeof(cl_mem), &extra0->sl));
CL_CHECK(clSetKernelArg(gemm, 2, sizeof(cl_mem), &extra0->sh));
CL_CHECK(clSetKernelArg(gemm, 3, sizeof(cl_mem), &extra0->d));
CL_CHECK(clSetKernelArg(gemm, 4, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer));
CL_CHECK(clSetKernelArg(gemm, 5, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer));
CL_CHECK(clSetKernelArg(gemm, 6, sizeof(cl_mem), &extrad->data_device));
CL_CHECK(clSetKernelArg(gemm, 7, sizeof(cl_ulong), &offsetd));
CL_CHECK(clSetKernelArg(gemm, 8, sizeof(int), &m));
CL_CHECK(clSetKernelArg(gemm, 9, sizeof(int), &n));
CL_CHECK(clSetKernelArg(gemm, 10, sizeof(int), &k));
size_t global[] = { 64, (size_t)CEIL_DIV(m, 64), (size_t)CEIL_DIV(n, tile_n) };
size_t local[] = { 64, 1, 1 };
backend_ctx->enqueue_ndrange_kernel(gemm, 3, global, local, dst);
CL_CHECK(clReleaseMemObject(activation));
return true;
}
static void ggml_cl_mul_mat_iq4_xs_f32_adreno(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {
ggml_backend_opencl_context * backend_ctx = (ggml_backend_opencl_context *)backend->context;
ggml_tensor_extra_cl_iq4_xs * extra0 = (ggml_tensor_extra_cl_iq4_xs *)src0->extra;
ggml_tensor_extra_cl * extra1 = (ggml_tensor_extra_cl *)src1->extra;
ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra;
cl_ulong offset1 = extra1->offset + src1->view_offs;
cl_ulong offsetd = extrad->offset + dst->view_offs;
const int m = src0->ne[1];
const int n = src1->ne[1];
const int k = src0->ne[0];
if (n == 1 || !ggml_is_contiguous(src1) || offset1 % backend_ctx->alignment != 0) {
cl_kernel gemv = backend_ctx->kernel_gemv_noshuffle_iq4_xs_f32;
const int stride_d = dst->ne[0];
const int stride_b = src1->nb[1]/sizeof(float);
CL_CHECK(clSetKernelArg(gemv, 0, sizeof(cl_mem), &extra0->qs));
CL_CHECK(clSetKernelArg(gemv, 1, sizeof(cl_mem), &extra0->sl));
CL_CHECK(clSetKernelArg(gemv, 2, sizeof(cl_mem), &extra0->sh));
CL_CHECK(clSetKernelArg(gemv, 3, sizeof(cl_mem), &extra0->d));
CL_CHECK(clSetKernelArg(gemv, 4, sizeof(cl_mem), &extra1->data_device));
CL_CHECK(clSetKernelArg(gemv, 5, sizeof(cl_ulong), &offset1));
CL_CHECK(clSetKernelArg(gemv, 6, sizeof(cl_mem), &extrad->data_device));
CL_CHECK(clSetKernelArg(gemv, 7, sizeof(cl_ulong), &offsetd));
CL_CHECK(clSetKernelArg(gemv, 8, sizeof(int), &k));
CL_CHECK(clSetKernelArg(gemv, 9, sizeof(int), &m));
CL_CHECK(clSetKernelArg(gemv, 10, sizeof(int), &stride_b));
CL_CHECK(clSetKernelArg(gemv, 11, sizeof(int), &stride_d));
const size_t nsg = backend_ctx->iq4_xs_mv_nsg;
size_t global[] = { (size_t)CEIL_DIV(m, 128)*64, (size_t)n*nsg, 1 };
size_t local[] = { 64, nsg, 1 };
backend_ctx->enqueue_ndrange_kernel(gemv, 3, global, local, dst);
return;
}
if (ggml_cl_mul_mat_iq4_xs_dp4a(backend, src0, src1, dst)) {
return;
}
cl_context context = backend_ctx->context;
cl_int err;
cl_buffer_region region = { (size_t)offset1, (size_t)k*n*sizeof(float) };
cl_mem b_sub_buf = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, &region, &err);
CL_CHECK(err);
cl_image_format img_fmt = { CL_RGBA, CL_FLOAT };
cl_image_desc img_desc = {};
img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER;
img_desc.image_width = (size_t)k*n/4;
img_desc.buffer = b_sub_buf;
cl_mem b_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, nullptr, &err);
CL_CHECK(err);
const int padded_n = CEIL_DIV(n, 8)*8;
region.origin = 0;
region.size = (size_t)k*padded_n*sizeof(cl_half);
backend_ctx->prealloc_act_trans.allocate(context, region.size);
cl_mem b_sub_buf_trans = clCreateSubBuffer(backend_ctx->prealloc_act_trans.buffer, 0, CL_BUFFER_CREATE_TYPE_REGION, &region, &err);
CL_CHECK(err);
img_fmt = { CL_RGBA, CL_HALF_FLOAT };
img_desc.image_width = (size_t)k*padded_n/4;
img_desc.buffer = b_sub_buf_trans;
cl_mem b_img_trans = clCreateImage(context, 0, &img_fmt, &img_desc, nullptr, &err);
CL_CHECK(err);
const int height_b = std::max(n/4, 1);
const int width_b = k/4;
const int padded_height_b = padded_n/4;
cl_kernel kernel = backend_ctx->kernel_transpose_32_16;
CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &b_img));
CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &b_img_trans));
CL_CHECK(clSetKernelArg(kernel, 2, sizeof(int), &height_b));
CL_CHECK(clSetKernelArg(kernel, 3, sizeof(int), &width_b));
CL_CHECK(clSetKernelArg(kernel, 4, sizeof(int), &padded_height_b));
size_t t_global[] = { (size_t)width_b, (size_t)padded_height_b };
size_t t_local[] = { 1, 16 };
backend_ctx->enqueue_ndrange_kernel(kernel, 2, t_global, t_local, dst);
kernel = backend_ctx->kernel_gemm_noshuffle_iq4_xs_f32;
CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->qs));
CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0->sl));
CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra0->sh));
CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra0->d));
CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &b_img_trans));
CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_mem), &extrad->data_device));
CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_ulong), &offsetd));
CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &m));
CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &padded_n));
CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &k));
CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &n));
size_t global[] = { (size_t)CEIL_DIV(n, 8), (size_t)CEIL_DIV(m, 512)*128, 1 };
size_t local[] = { 1, 128, 1 };
backend_ctx->enqueue_ndrange_kernel(kernel, 3, global, local, dst);
CL_CHECK(clReleaseMemObject(b_img_trans));
CL_CHECK(clReleaseMemObject(b_sub_buf_trans));
CL_CHECK(clReleaseMemObject(b_img));
CL_CHECK(clReleaseMemObject(b_sub_buf));
}
#endif
static void ggml_cl_mul_mat_q8_0_f32_adreno(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {
#ifdef GGML_OPENCL_USE_ADRENO_KERNELS
GGML_ASSERT(src0);
@@ -29875,6 +30380,7 @@ static void ggml_cl_mul_mat(ggml_backend_t backend, const ggml_tensor * src0, co
ggml_tensor_extra_cl_mxfp4 * extra0_mxfp4 = (ggml_tensor_extra_cl_mxfp4 *)soa0_src->extra;
ggml_tensor_extra_cl_q8_0 * extra0_q8_0 = (ggml_tensor_extra_cl_q8_0 *)soa0_src->extra;
ggml_tensor_extra_cl_iq4_nl * extra0_iq4_nl = (ggml_tensor_extra_cl_iq4_nl *)soa0_src->extra;
ggml_tensor_extra_cl_iq4_xs * extra0_iq4_xs = (ggml_tensor_extra_cl_iq4_xs *)soa0_src->extra;
ggml_tensor_extra_cl_q2_K * extra0_q2_K = (ggml_tensor_extra_cl_q2_K *)soa0_src->extra;
ggml_tensor_extra_cl_q3_K * extra0_q3_K = (ggml_tensor_extra_cl_q3_K *)soa0_src->extra;
ggml_tensor_extra_cl_iq2_s * extra0_iq2_s = (ggml_tensor_extra_cl_iq2_s *)soa0_src->extra;
@@ -30278,6 +30784,12 @@ static void ggml_cl_mul_mat(ggml_backend_t backend, const ggml_tensor * src0, co
return;
}
// iq4_xs x fp32
if (src0t == GGML_TYPE_IQ4_XS && src1t == GGML_TYPE_F32 && extra0_iq4_xs->noshuffle) {
ggml_cl_mul_mat_iq4_xs_f32_adreno(backend, src0, src1, dst);
return;
}
// q8_0 x fp32
if (src0t == GGML_TYPE_Q8_0 && src1t == GGML_TYPE_F32 &&
enable_adreno_trans_weight(backend_ctx, src0)) {
@@ -30888,6 +31400,51 @@ static void ggml_cl_mul_mat(ggml_backend_t backend, const ggml_tensor * src0, co
backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst);
return;
}
case GGML_TYPE_IQ4_XS: {
if (ne11 < 32) {
break;
}
if (!ggml_is_contiguous(src0) || !ggml_is_contiguous(src1)) {
break;
}
kernel = backend_ctx->kernel_mul_mm_iq4_xs_f32_l4_lm;
nth0 = (backend_ctx->gpu_family == INTEL) ? 64 : 128;
offset0 = src0->view_offs;
int batch_stride_a = ne00*ne01;
int batch_stride_b = ne10*ne11;
int batch_stride_d = ne0*ne1;
CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0_iq4_xs->qs));
CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_iq4_xs->sl));
CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra0_iq4_xs->sh));
CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra0_iq4_xs->d));
CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_ulong), &offset0));
CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_mem), &extra1->data_device));
CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_ulong), &offset1));
CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_mem), &extrad->data_device));
CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_ulong), &offsetd));
CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &ne00));
CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &ne01));
CL_CHECK(clSetKernelArg(kernel, 11, sizeof(int), &ne02));
CL_CHECK(clSetKernelArg(kernel, 12, sizeof(int), &ne11));
CL_CHECK(clSetKernelArg(kernel, 13, sizeof(int), &ne12));
CL_CHECK(clSetKernelArg(kernel, 14, sizeof(int), &ne00));
CL_CHECK(clSetKernelArg(kernel, 15, sizeof(int), &ne10));
CL_CHECK(clSetKernelArg(kernel, 16, sizeof(int), &ne0));
CL_CHECK(clSetKernelArg(kernel, 17, sizeof(int), &batch_stride_a));
CL_CHECK(clSetKernelArg(kernel, 18, sizeof(int), &batch_stride_b));
CL_CHECK(clSetKernelArg(kernel, 19, sizeof(int), &batch_stride_d));
CL_CHECK(clSetKernelArg(kernel, 20, sizeof(int), &r2));
CL_CHECK(clSetKernelArg(kernel, 21, sizeof(int), &r3));
size_t global_work_size[] = {(size_t)(CEIL_DIV(ne01, 64)*nth0), (size_t)CEIL_DIV(ne11, 64), (size_t)ne12*ne13};
size_t local_work_size[] = {(size_t)nth0, 1, 1};
backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst);
return;
}
case GGML_TYPE_Q2_K: {
if (ne11 < 32) {
break;
@@ -32264,6 +32821,70 @@ static void ggml_cl_mul_mat(ggml_backend_t backend, const ggml_tensor * src0, co
CL_CHECK(clSetKernelArg(kernel, 12, sizeof(int), &ne1));
CL_CHECK(clSetKernelArg(kernel, 13, sizeof(int), &r2));
CL_CHECK(clSetKernelArg(kernel, 14, sizeof(int), &r3));
#endif // GGML_OPENCL_SOA_Q
break;
}
case GGML_TYPE_IQ4_XS: {
if (backend_ctx->gpu_family == INTEL) {
nth0 = 16;
nth1 = 1;
ndst = 4;
} else if (backend_ctx->gpu_family == ADRENO) {
nth0 = 64;
nth1 = 1;
ndst = 4;
} else {
GGML_ASSERT(false && "TODO: Unknown GPU");
}
#ifdef GGML_OPENCL_SOA_Q
kernel = backend_ctx->kernel_mul_mv_iq4_xs_f32_flat;
offset0 = src0->view_offs;
CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0_iq4_xs->qs));
CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_iq4_xs->sl));
CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra0_iq4_xs->sh));
CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra0_iq4_xs->d));
CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_ulong), &offset0));
CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_mem), &extra1->data_device));
CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_ulong), &offset1));
CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_mem), &extrad->data_device));
CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_ulong), &offsetd));
CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &ne00));
CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &ne01));
CL_CHECK(clSetKernelArg(kernel, 11, sizeof(cl_ulong), &nb01));
CL_CHECK(clSetKernelArg(kernel, 12, sizeof(cl_ulong), &nb02));
CL_CHECK(clSetKernelArg(kernel, 13, sizeof(cl_ulong), &nb03));
CL_CHECK(clSetKernelArg(kernel, 14, sizeof(int), &ne12));
CL_CHECK(clSetKernelArg(kernel, 15, sizeof(cl_ulong), &nb11));
CL_CHECK(clSetKernelArg(kernel, 16, sizeof(cl_ulong), &nb12));
CL_CHECK(clSetKernelArg(kernel, 17, sizeof(cl_ulong), &nb13));
CL_CHECK(clSetKernelArg(kernel, 18, sizeof(int), &ne0));
CL_CHECK(clSetKernelArg(kernel, 19, sizeof(int), &ne1));
CL_CHECK(clSetKernelArg(kernel, 20, sizeof(int), &r2));
CL_CHECK(clSetKernelArg(kernel, 21, sizeof(int), &r3));
#else
kernel = backend_ctx->kernel_mul_mv_iq4_xs_f32;
CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device));
CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0));
CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra1->data_device));
CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1));
CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extrad->data_device));
CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offsetd));
CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne00));
CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &ne01));
CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_ulong), &nb01));
CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_ulong), &nb02));
CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_ulong), &nb03));
CL_CHECK(clSetKernelArg(kernel, 11, sizeof(int), &ne12));
CL_CHECK(clSetKernelArg(kernel, 12, sizeof(cl_ulong), &nb11));
CL_CHECK(clSetKernelArg(kernel, 13, sizeof(cl_ulong), &nb12));
CL_CHECK(clSetKernelArg(kernel, 14, sizeof(cl_ulong), &nb13));
CL_CHECK(clSetKernelArg(kernel, 15, sizeof(int), &ne0));
CL_CHECK(clSetKernelArg(kernel, 16, sizeof(int), &ne1));
CL_CHECK(clSetKernelArg(kernel, 17, sizeof(int), &r2));
CL_CHECK(clSetKernelArg(kernel, 18, sizeof(int), &r3));
#endif // GGML_OPENCL_SOA_Q
break;
}
@@ -33147,7 +33768,7 @@ static void ggml_cl_mul_mat(ggml_backend_t backend, const ggml_tensor * src0, co
src0t == GGML_TYPE_Q1_0 ||
src0t == GGML_TYPE_Q2_0 ||
src0t == GGML_TYPE_IQ4_NL ||
src0t == GGML_TYPE_Q2_K || src0t == GGML_TYPE_Q3_K || src0t == GGML_TYPE_IQ2_S || src0t == GGML_TYPE_IQ2_XS || src0t == GGML_TYPE_IQ2_XXS || src0t == GGML_TYPE_IQ1_S || src0t == GGML_TYPE_IQ1_M || src0t == GGML_TYPE_IQ3_S || src0t == GGML_TYPE_IQ3_XXS) {
src0t == GGML_TYPE_Q2_K || src0t == GGML_TYPE_Q3_K || src0t == GGML_TYPE_IQ2_S || src0t == GGML_TYPE_IQ2_XS || src0t == GGML_TYPE_IQ2_XXS || src0t == GGML_TYPE_IQ1_S || src0t == GGML_TYPE_IQ1_M || src0t == GGML_TYPE_IQ3_S || src0t == GGML_TYPE_IQ3_XXS || src0t == GGML_TYPE_IQ4_XS) {
// Each SIMD group produces N_DST values in the result. Assuming each
// workgroup has N_SIMDGROUP SIMD groups, then each workgroup will
// produce N_DST*N_SIMDGROUP values in the result. Hence, the grid size
+95
View File
@@ -3283,3 +3283,98 @@ kernel void kernel_restore_block_iq3_xxs(
q[1] = (ushort)(sas >> 16);
}
}
struct block_iq4_xs {
half d;
ushort scales_h;
uchar scales_l[QK_K/64];
uchar qs[QK_K/2];
};
kernel void kernel_convert_block_iq4_xs(
global struct block_iq4_xs * src0,
global uchar * dst_qs,
global uchar * dst_sl,
global ushort * dst_sh,
global half * dst_d,
ulong n_blocks
) {
const ulong i = get_global_id(0);
if (i >= n_blocks) { return; }
dst_d[i] = src0[i].d;
dst_sh[i] = src0[i].scales_h;
for (int j = 0; j < 128; ++j) {
dst_qs[128*i + j] = src0[i].qs[j];
}
for (int j = 0; j < 4; ++j) {
dst_sl[4*i + j] = src0[i].scales_l[j];
}
}
kernel void kernel_restore_block_iq4_xs(
global uchar * src_qs,
global uchar * src_sl,
global ushort * src_sh,
global half * src_d,
global struct block_iq4_xs * dst,
ulong n_blocks
) {
const ulong i = get_global_id(0);
if (i >= n_blocks) { return; }
dst[i].d = src_d[i];
dst[i].scales_h = src_sh[i];
for (int j = 0; j < 128; ++j) {
dst[i].qs[j] = src_qs[128*i + j];
}
for (int j = 0; j < 4; ++j) {
dst[i].scales_l[j] = src_sl[4*i + j];
}
}
kernel void kernel_convert_block_iq4_xs_noshuffle(
global struct block_iq4_xs * src0,
global ushort * dst_qs,
global uint * dst_sl,
global ushort * dst_sh,
global half * dst_d,
ulong n_blocks
) {
const ulong i = get_global_id(0);
if (i >= n_blocks) { return; }
dst_d[i] = src0[i].d;
dst_sh[i] = src0[i].scales_h;
dst_sl[i] = as_uint(vload4(0, src0[i].scales_l));
for (int sb = 0; sb < 8; ++sb) {
for (int u = 0; u < 8; ++u) {
ushort q = 0;
for (int j = 0; j < 4; ++j) {
uchar v = src0[i].qs[16*sb + 4*(u & 3) + j];
q |= (ushort)(((v >> (4*(u >> 2))) & 15) << (4*j));
}
dst_qs[64*i + 8*sb + u] = q;
}
}
}
kernel void kernel_restore_block_iq4_xs_noshuffle(
global ushort * src_qs,
global uint * src_sl,
global ushort * src_sh,
global half * src_d,
global struct block_iq4_xs * dst,
ulong n_blocks
) {
const ulong i = get_global_id(0);
if (i >= n_blocks) { return; }
dst[i].d = src_d[i];
dst[i].scales_h = src_sh[i];
vstore4(as_uchar4(src_sl[i]), 0, dst[i].scales_l);
for (int sb = 0; sb < 8; ++sb) {
for (int j = 0; j < 16; ++j) {
const int shift = 4*(j & 3);
const ushort lo = src_qs[64*i + 8*sb + j/4];
const ushort hi = src_qs[64*i + 8*sb + j/4 + 4];
dst[i].qs[16*sb + j] = (uchar)(((lo >> shift) & 15) | (((hi >> shift) & 15) << 4));
}
}
}
@@ -0,0 +1,88 @@
#pragma OPENCL EXTENSION cl_khr_fp16 : enable
#ifdef cl_qcom_reqd_sub_group_size
#pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable
__attribute__((qcom_reqd_sub_group_size("full")))
#endif
kernel void kernel_gemm_noshuffle_iq4_xs_f32(
global const ushort * src0_qs,
global const uint * src0_sl,
global const ushort * src0_sh,
global const half * src0_d,
read_only image1d_buffer_t src1,
global float * dst,
ulong offsetd,
int m,
int n,
int k,
int n_no_padding
) {
dst = (global float *)((global char *)dst + offsetd);
const int gy = get_global_id(0);
const int row = get_global_id(1) * 4;
const int n_4 = n / 4;
if (row >= m) {
return;
}
half8 c0 = 0, c1 = 0, c2 = 0, c3 = 0;
for (int i = 0; i < k; i += 16) {
const uint4 sl = vload4(0, src0_sl + row + (i / 256) * m);
const uint4 sh = convert_uint4(vload4(0, src0_sh + row + (i / 256) * m));
const half4 d = vload4(0, src0_d + row + (i / 256) * m);
const int sb = (i / 32) & 7;
const int4 ls = convert_int4(((sl >> (uint4)(4*sb)) & (uint4)(15)) | (((sh >> (uint4)(2*sb)) & (uint4)(3)) << (uint4)(4))) - (int4)(32);
const half4 scale = convert_half4(convert_float4(d) * convert_float4(ls));
for (int l = 0; l < 16; l += 4) {
const int ki = i + l;
const ushort4 q = vload4(0, src0_qs + row + (ki / 4) * m);
#pragma unroll
for (int j = 0; j < 4; ++j) {
half8 b;
b.s0123 = read_imageh(src1, gy * 2 + (ki + j) * n_4);
b.s4567 = read_imageh(src1, gy * 2 + 1 + (ki + j) * n_4);
const ushort4 indices = (q >> (ushort4)(4*j)) & (ushort4)(15);
const half4 w = (half4)(iq4_xs_values[indices.s0], iq4_xs_values[indices.s1],
iq4_xs_values[indices.s2], iq4_xs_values[indices.s3]) * scale;
c0 += b * w.s0;
c1 += b * w.s1;
c2 += b * w.s2;
c3 += b * w.s3;
}
}
}
int idx = gy * 8 * m + row;
if (idx + 3 < m * n_no_padding) {
vstore4((float4)(c0.s0, c1.s0, c2.s0, c3.s0), 0, dst + idx);
idx += m;
}
if (idx + 3 < m * n_no_padding) {
vstore4((float4)(c0.s1, c1.s1, c2.s1, c3.s1), 0, dst + idx);
idx += m;
}
if (idx + 3 < m * n_no_padding) {
vstore4((float4)(c0.s2, c1.s2, c2.s2, c3.s2), 0, dst + idx);
idx += m;
}
if (idx + 3 < m * n_no_padding) {
vstore4((float4)(c0.s3, c1.s3, c2.s3, c3.s3), 0, dst + idx);
idx += m;
}
if (idx + 3 < m * n_no_padding) {
vstore4((float4)(c0.s4, c1.s4, c2.s4, c3.s4), 0, dst + idx);
idx += m;
}
if (idx + 3 < m * n_no_padding) {
vstore4((float4)(c0.s5, c1.s5, c2.s5, c3.s5), 0, dst + idx);
idx += m;
}
if (idx + 3 < m * n_no_padding) {
vstore4((float4)(c0.s6, c1.s6, c2.s6, c3.s6), 0, dst + idx);
idx += m;
}
if (idx + 3 < m * n_no_padding) {
vstore4((float4)(c0.s7, c1.s7, c2.s7, c3.s7), 0, dst + idx);
}
}
@@ -0,0 +1,135 @@
#pragma OPENCL EXTENSION cl_khr_fp16 : enable
#pragma OPENCL EXTENSION cl_khr_subgroups : enable
#ifdef cl_khr_integer_dot_product
#pragma OPENCL EXTENSION cl_khr_integer_dot_product : enable
#endif
#define QK_K 256
#ifndef TILESIZE_N
#define TILESIZE_N 32
#endif
inline uint iq4_xs_pack(ushort q) {
return iq4_xs_code(q & 15)
| (iq4_xs_code((q >> 4) & 15) << 8)
| (iq4_xs_code((q >> 8) & 15) << 16)
| (iq4_xs_code(q >> 12) << 24);
}
// Pass staged operands by value; local dp4a operands miscompile on X2.
inline int dot8_q8a_v(uint8 qw, uint4 a0, uint4 a1) {
int r = 0;
r = dot_acc_sat_4x8packed_ss_int(qw.s0, a0.x, r);
r = dot_acc_sat_4x8packed_ss_int(qw.s1, a0.y, r);
r = dot_acc_sat_4x8packed_ss_int(qw.s2, a0.z, r);
r = dot_acc_sat_4x8packed_ss_int(qw.s3, a0.w, r);
r = dot_acc_sat_4x8packed_ss_int(qw.s4, a1.x, r);
r = dot_acc_sat_4x8packed_ss_int(qw.s5, a1.y, r);
r = dot_acc_sat_4x8packed_ss_int(qw.s6, a1.z, r);
r = dot_acc_sat_4x8packed_ss_int(qw.s7, a1.w, r);
return r;
}
__attribute__((qcom_wave_pair_mode(1)))
kernel void kernel_gemm_noshuffle_iq4_xs_q8_1_dp4a(
__global const ushort * src0_qs,
__global const uint * src0_sl,
__global const ushort * src0_sh,
__global const half * src0_d,
__global const uint * src1_qa,
__global const half * src1_da,
__global float * dst,
ulong offsetd,
int m,
int n_no_padding,
int k
) {
dst = (global float *)((global char *)dst + offsetd);
const uint lid = get_local_id(0);
const uint block_id_m = get_global_id(1);
const uint block_id_n = get_global_id(2);
const uint row = block_id_m * 64 + lid;
const uint col_base = block_id_n * TILESIZE_N;
const bool row_valid = row < (uint)m;
const uint rrow = row_valid ? row : 0;
const uint k_u = (uint)k >> 2;
const uint k_b = (uint)k >> 5;
__local uint4 sh_qa4[TILESIZE_N][2];
__local half sh_d[TILESIZE_N];
#define NGROUPS (TILESIZE_N / 4)
float4 acc[NGROUPS];
#pragma unroll
for (int g = 0; g < NGROUPS; ++g) acc[g] = (float4)(0.0f);
for (uint step = 0; step < (uint)k; step += 32) {
const uint sub = step >> 5;
const uint ib = sub >> 3;
const uint sb = sub & 7u;
const uint sl = src0_sl[rrow + ib*(uint)m];
const uint sh = src0_sh[rrow + ib*(uint)m];
const int ls = (int)(((sl >> (4*sb)) & 15u) | (((sh >> (2*sb)) & 3u) << 4)) - 32;
const float d_w = (float)src0_d[rrow + ib*(uint)m] * (float)ls;
const uint qsb = rrow + sub*8u*(uint)m;
uint8 qw;
qw.s0 = iq4_xs_pack(src0_qs[qsb + 0u*(uint)m]);
qw.s1 = iq4_xs_pack(src0_qs[qsb + 1u*(uint)m]);
qw.s2 = iq4_xs_pack(src0_qs[qsb + 2u*(uint)m]);
qw.s3 = iq4_xs_pack(src0_qs[qsb + 3u*(uint)m]);
qw.s4 = iq4_xs_pack(src0_qs[qsb + 4u*(uint)m]);
qw.s5 = iq4_xs_pack(src0_qs[qsb + 5u*(uint)m]);
qw.s6 = iq4_xs_pack(src0_qs[qsb + 6u*(uint)m]);
qw.s7 = iq4_xs_pack(src0_qs[qsb + 7u*(uint)m]);
for (uint idx = lid; idx < TILESIZE_N * 2; idx += 64) {
const uint t = idx >> 1;
const uint v = idx & 1;
const uint c = col_base + t;
sh_qa4[t][v] = (c < (uint)n_no_padding)
? vload4(0, src1_qa + c * k_u + (step >> 2) + (v << 2))
: (uint4)(0u);
}
if (lid < TILESIZE_N) {
const uint c = col_base + lid;
sh_d[lid] = (c < (uint)n_no_padding) ? src1_da[c * k_b + sub] : (half)0;
}
barrier(CLK_LOCAL_MEM_FENCE);
#define LD4(arr, b) ((float4)((float)arr[(b)+0], (float)arr[(b)+1], (float)arr[(b)+2], (float)arr[(b)+3]))
#pragma unroll
for (int g = 0; g < NGROUPS; ++g) {
const int b = g * 4;
float4 rf;
#define TDOT4(T) dot8_q8a_v(qw, (uint4)(sh_qa4[T][0]), (uint4)(sh_qa4[T][1]))
rf.s0 = (float)TDOT4(b+0); rf.s1 = (float)TDOT4(b+1);
rf.s2 = (float)TDOT4(b+2); rf.s3 = (float)TDOT4(b+3);
acc[g] += d_w * LD4(sh_d, b) * rf;
}
#undef TDOT4
#undef LD4
barrier(CLK_LOCAL_MEM_FENCE);
}
if (!row_valid) {
return;
}
#pragma unroll
for (int g = 0; g < NGROUPS; ++g) {
const uint b = (uint)(g * 4);
const float4 a = acc[g];
const uint c0 = col_base + b;
if (c0 + 0 < (uint)n_no_padding) dst[(c0 + 0) * (uint)m + row] = a.s0;
if (c0 + 1 < (uint)n_no_padding) dst[(c0 + 1) * (uint)m + row] = a.s1;
if (c0 + 2 < (uint)n_no_padding) dst[(c0 + 2) * (uint)m + row] = a.s2;
if (c0 + 3 < (uint)n_no_padding) dst[(c0 + 3) * (uint)m + row] = a.s3;
}
#undef NGROUPS
}
@@ -0,0 +1,85 @@
#pragma OPENCL EXTENSION cl_khr_fp16 : enable
#define QK_K 256
#ifndef IQ4_XS_MV_NSG
#define IQ4_XS_MV_NSG 8
#endif
inline float4 iq4_xs_vals(ushort q) {
return (float4)((char)iq4_xs_code(q & 15), (char)iq4_xs_code((q >> 4) & 15),
(char)iq4_xs_code((q >> 8) & 15), (char)iq4_xs_code(q >> 12));
}
kernel void kernel_gemv_noshuffle_iq4_xs_f32(
global const ushort * src0_qs,
global const uint * src0_sl,
global const ushort * src0_sh,
global const half * src0_d,
global const float * src1,
ulong offset1,
global float * dst,
ulong offsetd,
int ne00,
int ne01,
int ne10,
int ne0
) {
src1 = (global const float *)((global const char *)src1 + offset1);
dst = (global float *)((global char *)dst + offsetd);
const uint m = (uint)ne01;
const uint mh = m >> 1;
const uint lid = get_local_id(0);
const uint sgi = get_local_id(1);
const uint col = get_group_id(1);
const uint j = get_group_id(0)*64u + lid;
const uint row = j << 1;
global const float * y = src1 + (ulong)col * (uint)ne10;
float sum0 = 0.0f, sum1 = 0.0f;
if (j < mh) {
global const uint * qs = (global const uint *)src0_qs;
global const uint * sh = (global const uint *)src0_sh;
for (uint ib = sgi; ib < (uint)ne00/QK_K; ib += IQ4_XS_MV_NSG) {
const uint base = j + ib*mh;
const uint2 sl = vload2(base, src0_sl);
const uint shp = sh[base];
const half2 d = vload2(base, src0_d);
float acc0 = 0.0f, acc1 = 0.0f;
for (uint sb = 0; sb < 8; ++sb) {
const int ls0 = (int)(((sl.s0 >> (4*sb)) & 15u) | (((shp >> (2*sb)) & 3u) << 4)) - 32;
const int ls1 = (int)(((sl.s1 >> (4*sb)) & 15u) | (((shp >> (16 + 2*sb)) & 3u) << 4)) - 32;
float a0 = 0.0f, a1 = 0.0f;
for (uint u = 0; u < 8; ++u) {
const uint group = ib*64 + sb*8 + u;
const uint q = qs[j + group*mh];
const float4 yv = vload4(group, y);
a0 += dot(yv, iq4_xs_vals((ushort)q));
a1 += dot(yv, iq4_xs_vals((ushort)(q >> 16)));
}
acc0 += (float)ls0*a0;
acc1 += (float)ls1*a1;
}
sum0 += (float)d.s0*acc0;
sum1 += (float)d.s1*acc1;
}
}
#if IQ4_XS_MV_NSG > 1
local float2 part[IQ4_XS_MV_NSG][64];
part[sgi][lid] = (float2)(sum0, sum1);
barrier(CLK_LOCAL_MEM_FENCE);
if (sgi != 0) {
return;
}
for (uint s = 1; s < IQ4_XS_MV_NSG; ++s) {
const float2 v = part[s][lid];
sum0 += v.s0;
sum1 += v.s1;
}
#endif
if (j < mh) {
vstore2((float2)(sum0, sum1), 0, dst + (ulong)col*(uint)ne0 + row);
}
}
@@ -0,0 +1,10 @@
constant char iq4_xs_values[16] = {
-127, -104, -83, -65, -49, -35, -22, -10, 1, 13, 25, 38, 53, 69, 89, 113,
};
inline uint iq4_xs_code(uint n) {
const uint lo = (n & 4u) ? 0xF6EADDCFu : 0xBFAD9881u;
const uint hi = (n & 4u) ? 0x71594535u : 0x26190D01u;
const uint t = (n & 8u) ? hi : lo;
return (t >> (8*(n & 3u))) & 255u;
}
@@ -0,0 +1,186 @@
#pragma OPENCL EXTENSION cl_khr_fp16 : enable
#ifdef cl_intel_required_subgroup_size
#define INTEL_GPU 1
#endif
#define QK_K 256
struct block_iq4_xs {
half d;
ushort scales_h;
uchar scales_l[QK_K/64];
uchar qs[QK_K/2];
};
#define LOAD_VEC_A 4
#define LOAD_VEC_B 4
#define BM 64
#define BN 64
#define BK 32
#ifdef INTEL_GPU
#define TM 8
#define TN 8
#else
#define TM 4
#define TN 8
#endif
kernel void kernel_mul_mm_iq4_xs_f32_l4_lm(
global uchar * src0_qs,
global uchar * src0_sl,
global ushort * src0_sh,
global half * src0_d,
ulong offset0,
global float4 * src1,
ulong offset1,
global float * dst,
ulong offsetd,
int ne00,
int ne01,
int ne02,
int ne11,
int ne12,
int stride_a,
int stride_b,
int stride_d,
int batch_stride_a,
int batch_stride_b,
int batch_stride_d,
int r2,
int r3
) {
src1 = (global float4*)((global char*)src1 + offset1);
dst = (global float *)((global char*)dst + offsetd);
local float buf_a[BM * BK];
local float buf_b[BN * BK];
const int batch_idx = get_global_id(2);
const int i13 = batch_idx / ne12;
const int i12 = batch_idx % ne12;
const int i03 = i13 / r3;
const int i02 = i12 / r2;
const int batch_idx_a = i03 * ne02 + i02;
const int ir = get_group_id(0);
const int ic = get_group_id(1);
const int tid = get_local_id(0);
const int th_r = tid % (BM / TM);
const int th_c = tid / (BM / TM);
const int loadr_a = get_local_id(0) % (BK / LOAD_VEC_A);
const int loadc_a = get_local_id(0) / (BK / LOAD_VEC_A);
const int loadr_b = get_local_id(0) % (BK / LOAD_VEC_B);
const int loadc_b = get_local_id(0) / (BK / LOAD_VEC_B);
const int loadstride_a = get_local_size(0) * LOAD_VEC_A / BK;
const int loadstride_b = get_local_size(0) * LOAD_VEC_B / BK;
// pos_a counts elements, not blocks - one iq4_xs super block holds QK_K of them
int pos_a = batch_idx_a * batch_stride_a + ir * BM * stride_a;
int pos_b = (batch_idx * batch_stride_b + ic * BN * stride_b) / LOAD_VEC_B;
float sums[TM * TN];
float cache_a[TM];
float cache_b[TN];
for (int i = 0; i < TM * TN; i++) {
sums[i] = 0.0f;
}
for (int block = 0; block < ne00; block += BK) {
for (int l = 0; l < BM; l += loadstride_a) {
if (ir*BM + loadc_a + l < ne01) {
int idx = pos_a + (loadc_a + l) * stride_a + loadr_a * LOAD_VEC_A;
int ib = idx / QK_K;
ib += offset0/sizeof(struct block_iq4_xs);
int e = idx % QK_K;
int ib32 = e >> 5;
int j = e & 15;
int shift = (e & 16) >> 2;
int ls = ((src0_sl[4*ib + ib32/2] >> (4*(ib32 & 1))) & 15) | (((src0_sh[ib] >> (2*ib32)) & 3) << 4);
float db = (float)src0_d[ib] * (float)(ls - 32);
uchar4 q = vload4(0, src0_qs + 128*ib + 16*ib32 + j);
q = (q >> (uchar4)(shift)) & (uchar4)(15);
float4 v1;
v1.s0 = db * (float)iq4_xs_values[q.s0];
v1.s1 = db * (float)iq4_xs_values[q.s1];
v1.s2 = db * (float)iq4_xs_values[q.s2];
v1.s3 = db * (float)iq4_xs_values[q.s3];
buf_a[(loadr_a * LOAD_VEC_A + 0) * BM + loadc_a + l] = v1.s0;
buf_a[(loadr_a * LOAD_VEC_A + 1) * BM + loadc_a + l] = v1.s1;
buf_a[(loadr_a * LOAD_VEC_A + 2) * BM + loadc_a + l] = v1.s2;
buf_a[(loadr_a * LOAD_VEC_A + 3) * BM + loadc_a + l] = v1.s3;
} else {
buf_a[(loadr_a * LOAD_VEC_A + 0) * BM + loadc_a + l] = 0.0f;
buf_a[(loadr_a * LOAD_VEC_A + 1) * BM + loadc_a + l] = 0.0f;
buf_a[(loadr_a * LOAD_VEC_A + 2) * BM + loadc_a + l] = 0.0f;
buf_a[(loadr_a * LOAD_VEC_A + 3) * BM + loadc_a + l] = 0.0f;
}
}
for (int l = 0; l < BN; l += loadstride_b) {
if (ic*BN + loadc_b + l < ne11) {
int idx = pos_b + (loadc_b + l) * stride_b / LOAD_VEC_B + loadr_b;
buf_b[(loadr_b * LOAD_VEC_B + 0) * BN + loadc_b + l] = src1[idx].s0;
buf_b[(loadr_b * LOAD_VEC_B + 1) * BN + loadc_b + l] = src1[idx].s1;
buf_b[(loadr_b * LOAD_VEC_B + 2) * BN + loadc_b + l] = src1[idx].s2;
buf_b[(loadr_b * LOAD_VEC_B + 3) * BN + loadc_b + l] = src1[idx].s3;
} else {
buf_b[(loadr_b * LOAD_VEC_B + 0) * BN + loadc_b + l] = 0.0f;
buf_b[(loadr_b * LOAD_VEC_B + 1) * BN + loadc_b + l] = 0.0f;
buf_b[(loadr_b * LOAD_VEC_B + 2) * BN + loadc_b + l] = 0.0f;
buf_b[(loadr_b * LOAD_VEC_B + 3) * BN + loadc_b + l] = 0.0f;
}
}
barrier(CLK_LOCAL_MEM_FENCE);
pos_a += BK;
pos_b += BK / LOAD_VEC_B;
for (int i = 0; i < BK; i++) {
for (int j = 0; j < TM; j++) {
cache_a[j] = buf_a[(i) * BM + th_r * TM + j];
}
for (int j = 0; j < TN; j++) {
cache_b[j] = buf_b[(i) * BN + th_c * TN + j];
}
for (int cc = 0; cc < TN; cc++) {
for (int cr = 0; cr < TM; cr++) {
const int sums_idx = cc*TM + cr;
sums[sums_idx] = mad(cache_a[cr], cache_b[cc], sums[sums_idx]);
}
}
}
barrier(CLK_LOCAL_MEM_FENCE);
}
const int dr = ir * BM + th_r * TM;
const int dc = ic * BN + th_c * TN;
const ulong offsets = batch_idx * batch_stride_d;
for (int cc = 0; cc < TN; cc++) {
for (int cr = 0; cr < TM; cr++) {
if (dr + cr < ne01 && dc + cc < ne11) {
dst[offsets + (dc + cc) * stride_d + dr + cr] = sums[cc * TM + cr];
}
}
}
}
@@ -0,0 +1,134 @@
#pragma OPENCL EXTENSION cl_khr_fp16 : enable
#ifdef cl_intel_required_subgroup_size
#pragma OPENCL EXTENSION cl_intel_required_subgroup_size : enable
#define INTEL_GPU 1
#define REQD_SUBGROUP_SIZE_16 __attribute__((intel_reqd_sub_group_size(16)))
#define REQD_SUBGROUP_SIZE_32 __attribute__((intel_reqd_sub_group_size(32)))
#elif defined(cl_qcom_reqd_sub_group_size)
#pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable
#define ADRENO_GPU 1
#define REQD_SUBGROUP_SIZE_64 __attribute__((qcom_reqd_sub_group_size("half")))
#define REQD_SUBGROUP_SIZE_128 __attribute__((qcom_reqd_sub_group_size("full")))
#endif
#define QK_K 256
typedef struct {
half d;
ushort scales_h;
uchar scales_l[QK_K/64];
uchar qs[QK_K/2];
} block_iq4_xs;
#undef N_DST
#undef N_SIMDGROUP
#undef N_SIMDWIDTH
#ifdef INTEL_GPU
#define N_DST 4
#define N_SIMDGROUP 1
#define N_SIMDWIDTH 16
#elif defined (ADRENO_GPU)
#define N_DST 4
#define N_SIMDGROUP 1
#define N_SIMDWIDTH 64
#endif
#undef BLOCK_STRIDE
// 8 threads cover one super block, one 32 element sub block each
#define BLOCK_STRIDE (N_SIMDWIDTH/8)
#ifdef INTEL_GPU
REQD_SUBGROUP_SIZE_16
#elif defined (ADRENO_GPU)
REQD_SUBGROUP_SIZE_64
#endif
kernel void kernel_mul_mv_iq4_xs_f32(
global char * src0,
ulong offset0,
global char * src1,
ulong offset1,
global char * dst,
ulong offsetd,
int ne00,
int ne01,
ulong nb01,
ulong nb02,
ulong nb03,
int ne12,
ulong nb11,
ulong nb12,
ulong nb13,
int ne0,
int ne1,
int r2,
int r3
) {
src0 = src0 + offset0;
src1 = src1 + offset1;
dst = dst + offsetd;
int ix = get_sub_group_local_id()/8; // super block index
int it = get_sub_group_local_id()%8; // sub block inside the super block
int nb = ne00/QK_K;
int r0 = get_group_id(0);
int r1 = get_group_id(1);
int im = get_group_id(2);
int first_row = (r0 * N_SIMDGROUP + get_sub_group_id()) * N_DST;
int i12 = im%ne12;
int i13 = im/ne12;
ulong offset_src0 = first_row*nb01 + (i12/r2)*nb02 + (i13/r3)*nb03;
ulong offset_src1 = r1*nb11 + (i12 )*nb12 + (i13 )*nb13;
global block_iq4_xs * x = (global block_iq4_xs *) (src0 + offset_src0);
global float * y = (global float *) (src1 + offset_src1);
float yl[32];
float sumf[N_DST] = {0.f};
float all_sum;
global float * y4 = y + ix * QK_K + 32 * it;
for (int ib = ix; ib < nb; ib += BLOCK_STRIDE) {
for (int i = 0; i < 32; ++i) {
yl[i] = y4[i];
}
global char * xrow = (global char *)(x + ib);
// only ne01 output rows exist; reading past them can pull in an inf scale
for (int row = 0; row < N_DST && first_row + row < ne01; row++) {
global block_iq4_xs * xb = (global block_iq4_xs *)(xrow + row*nb01);
int ls = ((xb->scales_l[it/2] >> (4*(it%2))) & 15) | (((xb->scales_h >> (2*it)) & 3) << 4);
float db = (float)xb->d * (float)(ls - 32);
global uchar * qs = xb->qs + 16*it;
float acc = 0.f;
for (int j = 0; j < 16; ++j) {
acc += yl[j] * (float)iq4_xs_values[qs[j] & 15];
acc += yl[j+16] * (float)iq4_xs_values[qs[j] >> 4];
}
sumf[row] += db * acc;
}
y4 += BLOCK_STRIDE * QK_K;
}
global float * dst_f32 = (global float *) dst + im*ne0*ne1 + r1*ne0;
for (int row = 0; row < N_DST; ++row) {
all_sum = sub_group_reduce_add(sumf[row]);
if (first_row + row < ne01) {
if (get_sub_group_local_id() == 0) {
dst_f32[first_row + row] = all_sum;
}
}
}
}
@@ -0,0 +1,134 @@
#pragma OPENCL EXTENSION cl_khr_fp16 : enable
#ifdef cl_intel_required_subgroup_size
#pragma OPENCL EXTENSION cl_intel_required_subgroup_size : enable
#define INTEL_GPU 1
#define REQD_SUBGROUP_SIZE_16 __attribute__((intel_reqd_sub_group_size(16)))
#define REQD_SUBGROUP_SIZE_32 __attribute__((intel_reqd_sub_group_size(32)))
#elif defined(cl_qcom_reqd_sub_group_size)
#pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable
#define ADRENO_GPU 1
#define REQD_SUBGROUP_SIZE_64 __attribute__((qcom_reqd_sub_group_size("half")))
#define REQD_SUBGROUP_SIZE_128 __attribute__((qcom_reqd_sub_group_size("full")))
#endif
#define QK_K 256
typedef struct {
half d;
ushort scales_h;
uchar scales_l[QK_K/64];
uchar qs[QK_K/2];
} block_iq4_xs;
#undef N_DST
#undef N_SIMDGROUP
#undef N_SIMDWIDTH
#ifdef INTEL_GPU
#define N_DST 4
#define N_SIMDGROUP 1
#define N_SIMDWIDTH 16
#elif defined (ADRENO_GPU)
#define N_DST 4
#define N_SIMDGROUP 1
#define N_SIMDWIDTH 64
#endif
#undef BLOCK_STRIDE
// 8 threads cover one super block, one 32 element sub block each
#define BLOCK_STRIDE (N_SIMDWIDTH/8)
#ifdef INTEL_GPU
REQD_SUBGROUP_SIZE_16
#elif defined (ADRENO_GPU)
REQD_SUBGROUP_SIZE_64
#endif
kernel void kernel_mul_mv_iq4_xs_f32_flat(
global uchar * src0_qs,
global uchar * src0_sl,
global ushort * src0_sh,
global half * src0_d,
ulong offset0,
global char * src1,
ulong offset1,
global char * dst,
ulong offsetd,
int ne00,
int ne01,
ulong nb01,
ulong nb02,
ulong nb03,
int ne12,
ulong nb11,
ulong nb12,
ulong nb13,
int ne0,
int ne1,
int r2,
int r3
) {
src1 = src1 + offset1;
dst = dst + offsetd;
int ix = get_sub_group_local_id()/8; // super block index
int it = get_sub_group_local_id()%8; // sub block inside the super block
int nb = ne00/QK_K;
int r0 = get_group_id(0);
int r1 = get_group_id(1);
int im = get_group_id(2);
int first_row = (r0 * N_SIMDGROUP + get_sub_group_id()) * N_DST;
int i12 = im%ne12;
int i13 = im/ne12;
ulong offset_src0 = (offset0 + first_row*nb01 + (i12/r2)*nb02 + (i13/r3)*nb03)/sizeof(block_iq4_xs);
ulong row_stride = nb01/sizeof(block_iq4_xs);
ulong offset_src1 = r1*nb11 + (i12 )*nb12 + (i13 )*nb13;
global float * y = (global float *) (src1 + offset_src1);
float yl[32];
float sumf[N_DST] = {0.f};
float all_sum;
global float * y4 = y + ix * QK_K + 32 * it;
for (int ib = ix; ib < nb; ib += BLOCK_STRIDE) {
for (int i = 0; i < 32; ++i) {
yl[i] = y4[i];
}
// only ne01 output rows exist; reading past them can pull in an inf scale
for (int row = 0; row < N_DST && first_row + row < ne01; row++) {
const ulong b = offset_src0 + ib + row*row_stride;
int ls = ((src0_sl[4*b + it/2] >> (4*(it%2))) & 15) | (((src0_sh[b] >> (2*it)) & 3) << 4);
float db = (float)src0_d[b] * (float)(ls - 32);
global uchar * qs = src0_qs + 128*b + 16*it;
float acc = 0.f;
for (int j = 0; j < 16; ++j) {
acc += yl[j] * (float)iq4_xs_values[qs[j] & 15];
acc += yl[j+16] * (float)iq4_xs_values[qs[j] >> 4];
}
sumf[row] += db * acc;
}
y4 += BLOCK_STRIDE * QK_K;
}
global float * dst_f32 = (global float *) dst + im*ne0*ne1 + r1*ne0;
for (int row = 0; row < N_DST; ++row) {
all_sum = sub_group_reduce_add(sumf[row]);
if (first_row + row < ne01) {
if (get_sub_group_local_id() == 0) {
dst_f32[first_row + row] = all_sum;
}
}
}
}