ggml : allow MUL_MAT_ID -1 index to skip the computation (#26631)
* ggml : allow MUL_MAT_ID -1 index to skip the computation * nits * update vulkan code * ggml-webgpu: ensure all threads reach workgroup barriers in mul_mat_id_vec (cherry picked from commit d8a4b9efb6e2b9abf3a866e847756b7e6bbc6b69) * fix cuda * openvino * exclude CUDA from this PR * add TODO tag --------- Co-authored-by: yomaytk <yoshimura.masashi.frbs@gmail.com>
This commit is contained in:
1 parent
3642238cc1
commit
701fd25b3a
45 files changed
+586
-67
No files matched your search
@@ -926,6 +926,7 @@ extern "C" {
|
||||
enum ggml_type type);
|
||||
|
||||
// dst[i0, i1, i2] = a[i0, i1, i2] + b[i0, ids[i1, i2]]
|
||||
// an id of -1 adds nothing: dst[i0, i1, i2] = a[i0, i1, i2]
|
||||
GGML_API struct ggml_tensor * ggml_add_id(
|
||||
struct ggml_context * ctx,
|
||||
struct ggml_tensor * a,
|
||||
@@ -1508,6 +1509,7 @@ extern "C" {
|
||||
enum ggml_op_hint hint);
|
||||
|
||||
// indirect matrix multiplication
|
||||
// an id of -1 skips the slot and zeroes the matching row of the result
|
||||
//
|
||||
// [TAG_MUL_MAT_ID_DUPLICATE_ID]
|
||||
// note: this op has undefined behaviour if a given `a` is referenced more than once for a given `b`
|
||||
|
||||
@@ -1652,7 +1652,12 @@ static void ggml_compute_forward_mul_mat_id(
|
||||
for (int id = 0; id < n_ids; ++id) {
|
||||
const int32_t i02 = *(const int32_t *) ((const char *) ids->data + iid1*ids->nb[1] + id*ids->nb[0]);
|
||||
|
||||
assert(i02 >= 0 && i02 < n_as);
|
||||
assert(i02 == -1 || (i02 >= 0 && i02 < n_as));
|
||||
|
||||
if (i02 == -1) {
|
||||
memset((char *) dst->data + id*nb1 + iid1*nb2, 0, ne0*sizeof(float));
|
||||
continue;
|
||||
}
|
||||
|
||||
MMID_MATRIX_ROW(i02, matrix_row_counts[i02]) = (struct mmid_row_mapping) {id, iid1};
|
||||
matrix_row_counts[i02] += 1;
|
||||
|
||||
@@ -743,12 +743,17 @@ static void ggml_compute_forward_add_id_f32(
|
||||
// src1 indices
|
||||
const int i11 = *(int32_t *) ((char *) src2->data + i1*nb20 + i2*nb21);
|
||||
|
||||
GGML_ASSERT(i11 >= 0 && i11 < ne11);
|
||||
GGML_ASSERT(i11 == -1 || (i11 >= 0 && i11 < ne11));
|
||||
|
||||
ggml_vec_add_f32(ne0,
|
||||
(float *) ((char *) dst->data + i3*nb3 + i2*nb2 + i1*nb1 ),
|
||||
(float *) ((char *) src0->data + i3*nb03 + i2*nb02 + i1*nb01),
|
||||
(float *) ((char *) src1->data + i11*nb11));
|
||||
float * dst_row = (float *) ((char *) dst->data + i3*nb3 + i2*nb2 + i1*nb1 );
|
||||
const float * src0_row = (float *) ((char *) src0->data + i3*nb03 + i2*nb02 + i1*nb01);
|
||||
|
||||
if (i11 == -1) {
|
||||
memcpy(dst_row, src0_row, ne0*sizeof(float));
|
||||
continue;
|
||||
}
|
||||
|
||||
ggml_vec_add_f32(ne0, dst_row, src0_row, (float *) ((char *) src1->data + i11*nb11));
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -4855,7 +4855,12 @@ template <typename BLOC_TYPE, int64_t INTER_SIZE, int64_t NB_COLS, ggml_type PAR
|
||||
const int32_t i02 =
|
||||
*(const int32_t *) ((const char *) ids->data + iid1 * ids->nb[1] + id * ids->nb[0]);
|
||||
|
||||
GGML_ASSERT(i02 >= 0 && i02 < n_as);
|
||||
GGML_ASSERT(i02 == -1 || (i02 >= 0 && i02 < n_as));
|
||||
|
||||
if (i02 == -1) {
|
||||
memset((char *) dst->data + id*nb1 + iid1*nb2, 0, ne0*sizeof(float));
|
||||
continue;
|
||||
}
|
||||
|
||||
MMID_MATRIX_ROW(i02, matrix_row_counts[i02]) = { id, iid1 };
|
||||
matrix_row_counts[i02] += 1;
|
||||
|
||||
@@ -682,7 +682,12 @@ template <typename BLOC_TYPE, int64_t INTER_SIZE, int64_t NB_COLS> class tensor_
|
||||
const int32_t i02 =
|
||||
*(const int32_t *) ((const char *) ids->data + iid1 * ids->nb[1] + id * ids->nb[0]);
|
||||
|
||||
GGML_ASSERT(i02 >= 0 && i02 < n_as);
|
||||
GGML_ASSERT(i02 == -1 || (i02 >= 0 && i02 < n_as));
|
||||
|
||||
if (i02 == -1) {
|
||||
memset((char *) dst->data + id * nb1 + iid1 * nb2, 0, ne0 * sizeof(float));
|
||||
continue;
|
||||
}
|
||||
|
||||
MMID_MATRIX_ROW(i02, matrix_row_counts[i02]) = { id, iid1 };
|
||||
matrix_row_counts[i02] += 1;
|
||||
|
||||
@@ -1413,10 +1413,10 @@ static void hvx_mv_id_tiled(unsigned int nth, unsigned int ith, void * data) {
|
||||
|
||||
for (uint32_t ie1 = 0; ie1 < n_aids; ++ie1) { // for each expert
|
||||
const int32_t eid = *(const int32_t *) ((const uint8_t *) src2->data + ie1 * src2->nb[0]);
|
||||
if (eid < 0) {
|
||||
if (eid == -1) {
|
||||
continue;
|
||||
}
|
||||
assert(eid < (int32_t) n_ids);
|
||||
assert(eid >= 0 && eid < (int32_t) n_ids);
|
||||
|
||||
const dma_addr_t src0_row = src0->data + eid * nb02;
|
||||
const uint8_t * restrict act_col = (const uint8_t *) act_data;
|
||||
@@ -4166,10 +4166,10 @@ static inline void scan_expert_ids_n(
|
||||
const int32_t * row_ptr = (const int32_t *) (ids_data + iid1 * ids_nb1);
|
||||
for (uint32_t id = 0; id < n_ids; ++id) {
|
||||
const int32_t i02 = row_ptr[id];
|
||||
if (i02 < 0) {
|
||||
if (i02 == -1) {
|
||||
continue;
|
||||
}
|
||||
assert(i02 < n_as);
|
||||
assert(i02 >= 0 && i02 < n_as);
|
||||
|
||||
if (matrix_rows) {
|
||||
matrix_rows[i02 * mapping_stride + counts[i02]] = (struct mmid_row_mapping) { id, iid1 };
|
||||
@@ -4179,6 +4179,28 @@ static inline void scan_expert_ids_n(
|
||||
}
|
||||
}
|
||||
|
||||
// the matmuls only write the rows that an expert owns, so the rows of the skipped slots are zeroed here
|
||||
static inline void zero_skipped_dst_rows(
|
||||
const struct htp_tensor * ids,
|
||||
uint32_t n_ids,
|
||||
const struct htp_tensor * dst,
|
||||
uint32_t ne0,
|
||||
size_t nb1,
|
||||
size_t nb2
|
||||
) {
|
||||
const uint8_t * ids_data = (const uint8_t *) ids->data;
|
||||
|
||||
for (uint32_t iid1 = 0; iid1 < ids->ne[1]; ++iid1) {
|
||||
for (uint32_t id = 0; id < n_ids; ++id) {
|
||||
const int32_t i02 = *(const int32_t *) (ids_data + iid1 * ids->nb[1] + id * ids->nb[0]);
|
||||
if (i02 != -1) {
|
||||
continue;
|
||||
}
|
||||
memset((uint8_t *) (dst->data + id * nb1 + iid1 * nb2), 0, ne0 * sizeof(float));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
static inline void scan_expert_ids(
|
||||
const struct htp_tensor * ids,
|
||||
uint32_t n_ids,
|
||||
@@ -4204,10 +4226,10 @@ static inline void scan_expert_ids(
|
||||
const int32_t * row_ptr = (const int32_t *) (ids_data + iid1 * ids_nb1);
|
||||
for (uint32_t id = 0; id < n_ids; ++id) {
|
||||
const int32_t i02 = *(const int32_t *) ((const uint8_t *) row_ptr + id * ids_nb0);
|
||||
if (i02 < 0) {
|
||||
if (i02 == -1) {
|
||||
continue;
|
||||
}
|
||||
assert(i02 < n_as);
|
||||
assert(i02 >= 0 && i02 < n_as);
|
||||
|
||||
if (matrix_rows) {
|
||||
matrix_rows[i02 * mapping_stride + counts[i02]] = (struct mmid_row_mapping) { id, iid1 };
|
||||
@@ -4287,6 +4309,8 @@ int op_matmul_id(struct htp_ops_context * octx) {
|
||||
scan_expert_ids(ids, n_ids, n_as, matrix_row_counts, matrix_rows, mapping_stride);
|
||||
}
|
||||
|
||||
zero_skipped_dst_rows(ids, n_ids, dst, ne0, nb1, nb2);
|
||||
|
||||
mmctx->matrix_row_counts = matrix_row_counts;
|
||||
mmctx->matrix_rows = matrix_rows;
|
||||
mmctx->mapping_stride = mapping_stride;
|
||||
|
||||
@@ -1283,6 +1283,15 @@ ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_mul_mm_id_map0(g
|
||||
return res;
|
||||
}
|
||||
|
||||
ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_mul_mm_id_zero(ggml_metal_library_t lib) {
|
||||
ggml_metal_pipeline_with_params res = ggml_metal_library_get_pipeline(lib, "kernel_mul_mm_id_zero");
|
||||
if (!res.pipeline) {
|
||||
res = ggml_metal_library_compile_pipeline(lib, "kernel_mul_mm_id_zero", "kernel_mul_mm_id_zero", nullptr);
|
||||
}
|
||||
|
||||
return res;
|
||||
}
|
||||
|
||||
ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_mul_mm_id(ggml_metal_library_t lib, const ggml_tensor * op) {
|
||||
char base[256];
|
||||
char name[256];
|
||||
|
||||
@@ -140,6 +140,7 @@ struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_mul_mv_mm
|
||||
struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_mul_mm (ggml_metal_library_t lib, const struct ggml_tensor * op);
|
||||
struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_mul_mv (ggml_metal_library_t lib, const struct ggml_tensor * op);
|
||||
struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_mul_mm_id_map0 (ggml_metal_library_t lib, int ne02, int ne20);
|
||||
struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_mul_mm_id_zero (ggml_metal_library_t lib);
|
||||
struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_mul_mm_id_amax(ggml_metal_library_t lib);
|
||||
struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_mul_mm_id_amax_part(ggml_metal_library_t lib);
|
||||
struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_mul_mm_id (ggml_metal_library_t lib, const struct ggml_tensor * op);
|
||||
|
||||
@@ -572,6 +572,13 @@ typedef struct {
|
||||
uint64_t nb21;
|
||||
} ggml_metal_kargs_mul_mm_id_map0;
|
||||
|
||||
typedef struct {
|
||||
int32_t ne0;
|
||||
uint64_t nb1;
|
||||
uint64_t nb2;
|
||||
uint64_t nb21;
|
||||
} ggml_metal_kargs_mul_mm_id_zero;
|
||||
|
||||
typedef struct {
|
||||
int32_t ne00;
|
||||
int32_t ne01;
|
||||
|
||||
@@ -2935,6 +2935,25 @@ int ggml_metal_op_mul_mat_id(ggml_metal_op_t ctx, int idx) {
|
||||
ggml_metal_encoder_dispatch_threadgroups(enc, 1, 1, 1, ne02, 1, 1);
|
||||
}
|
||||
|
||||
// zero the rows that no expert owns - disjoint from what kernel_mul_mm_id writes
|
||||
{
|
||||
ggml_metal_kargs_mul_mm_id_zero args = {
|
||||
/*.ne0 =*/ ne0,
|
||||
/*.nb1 =*/ nb1,
|
||||
/*.nb2 =*/ nb2,
|
||||
/*.nb21 =*/ nb21,
|
||||
};
|
||||
|
||||
auto pipeline = ggml_metal_library_get_pipeline_mul_mm_id_zero(lib);
|
||||
|
||||
ggml_metal_encoder_set_pipeline(enc, pipeline);
|
||||
ggml_metal_encoder_set_bytes (enc, &args, sizeof(args), 0);
|
||||
ggml_metal_encoder_set_buffer (enc, bid_src2, 1);
|
||||
ggml_metal_encoder_set_buffer (enc, bid_dst, 2);
|
||||
|
||||
ggml_metal_encoder_dispatch_threadgroups(enc, ne20, ne21, 1, 32, 1, 1);
|
||||
}
|
||||
|
||||
ggml_metal_op_concurrency_reset(ctx);
|
||||
|
||||
if (use_amax) {
|
||||
|
||||
@@ -185,6 +185,14 @@ kernel void kernel_add_id(
|
||||
|
||||
device float * dst_row = (device float *)((device char *)dst + i1*nb1 + i2*nb2);
|
||||
device const float * src0_row = (device const float *)((device char *)src0 + i1*args.nb01 + i2*args.nb02);
|
||||
|
||||
if (i11 == -1) {
|
||||
for (int i0 = tpitg.x; i0 < args.ne0; i0 += ntg.x) {
|
||||
dst_row[i0] = src0_row[i0];
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
device const float * src1_row = (device const float *)((device char *)src1 + i11*args.nb11);
|
||||
|
||||
for (int i0 = tpitg.x; i0 < args.ne0; i0 += ntg.x) {
|
||||
|
||||
@@ -391,8 +391,10 @@ kernel void kernel_mul_mm_id_map0(
|
||||
if (i21 + tpitg < args.ne21) {
|
||||
device const int32_t * src2_i32 = (device const int32_t *) (src2 + (i21 + tpitg)*args.nb21);
|
||||
|
||||
// 0xFFFF marks a skipped slot, it never matches an expert since ne02 fits in a threadgroup
|
||||
FOR_UNROLL (short i20 = 0; i20 < ne20; i20++) {
|
||||
sids[tpitg*ne20 + i20] = src2_i32[i20];
|
||||
const int32_t id = src2_i32[i20];
|
||||
sids[tpitg*ne20 + i20] = id == -1 ? 0xFFFF : (uint16_t) id;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -506,6 +508,30 @@ kernel void kernel_mul_mm_id_map0(
|
||||
}
|
||||
}
|
||||
|
||||
// kernel_mul_mm_id only writes the rows that an expert owns, so the skipped rows are zeroed here
|
||||
kernel void kernel_mul_mm_id_zero(
|
||||
constant ggml_metal_kargs_mul_mm_id_zero & args,
|
||||
device const char * src2,
|
||||
device char * dst,
|
||||
uint3 tgpig[[threadgroup_position_in_grid]],
|
||||
ushort3 tpitg[[thread_position_in_threadgroup]],
|
||||
ushort3 ntg[[threads_per_threadgroup]]) {
|
||||
const int i20 = tgpig.x; // slot
|
||||
const int i21 = tgpig.y; // token
|
||||
|
||||
const int32_t id = ((device const int32_t *) (src2 + i21*args.nb21))[i20];
|
||||
|
||||
if (id != -1) {
|
||||
return;
|
||||
}
|
||||
|
||||
device float * dst_row = (device float *) (dst + i20*args.nb1 + i21*args.nb2);
|
||||
|
||||
for (int i0 = tpitg.x; i0 < args.ne0; i0 += ntg.x) {
|
||||
dst_row[i0] = 0.0f;
|
||||
}
|
||||
}
|
||||
|
||||
kernel void kernel_mul_mm_id_amax_part_f32(
|
||||
constant ggml_metal_kargs_mul_mm_id_amax & args,
|
||||
device const char * src1,
|
||||
|
||||
@@ -3302,7 +3302,8 @@ kernel void kernel_mul_mv_id(
|
||||
uint3 tgpig[[threadgroup_position_in_grid]],
|
||||
ushort tiitg[[thread_index_in_threadgroup]],
|
||||
ushort tiisg[[thread_index_in_simdgroup]],
|
||||
ushort sgitg[[simdgroup_index_in_threadgroup]]) {
|
||||
ushort sgitg[[simdgroup_index_in_threadgroup]],
|
||||
ushort3 ntg [[threads_per_threadgroup]]) {
|
||||
const int iid1 = tgpig.z/args.nei0;
|
||||
const int idx = tgpig.z%args.nei0;
|
||||
|
||||
@@ -3316,11 +3317,20 @@ kernel void kernel_mul_mv_id(
|
||||
const int64_t i1 = idx;
|
||||
const int64_t i2 = i12;
|
||||
|
||||
device char * dst_cur = dst + (i1*args.ne0 + i2*args.ne1*args.ne0)*sizeof(float);
|
||||
|
||||
if (i02 == -1) {
|
||||
device float * dst_f32 = (device float *) dst_cur;
|
||||
const ushort nth = ntg.x*ntg.y*ntg.z;
|
||||
for (int i0 = tiitg; i0 < args.ne0; i0 += nth) {
|
||||
dst_f32[i0] = 0.0f;
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
device const char * src0_cur = src0s + i02*args.nb02;
|
||||
device const char * src1_cur = src1 + i11*args.nb11 + i12*args.nb12;
|
||||
|
||||
device char * dst_cur = dst + (i1*args.ne0 + i2*args.ne1*args.ne0)*sizeof(float);
|
||||
|
||||
ggml_metal_kargs_mul_mv args0 = {
|
||||
/*.ne00 =*/ args.ne00,
|
||||
/*.ne01 =*/ args.ne01,
|
||||
|
||||
@@ -1047,6 +1047,7 @@ struct ggml_backend_opencl_context {
|
||||
cl_kernel kernel_moe_reorder_b;
|
||||
cl_kernel kernel_moe_histogram, kernel_moe_scan, kernel_moe_fill, kernel_moe_scatter;
|
||||
cl_kernel kernel_moe_scatter_stable = nullptr; // deterministic slot assignment
|
||||
cl_kernel kernel_moe_zero_dst;
|
||||
cl_kernel kernel_moe_combine_f32 = nullptr; // fused router-weight mul + cross-expert sum
|
||||
cl_kernel kernel_moe_combine_bias_f32 = nullptr; // same, with the down-projection bias add folded in
|
||||
cl_kernel kernel_mul_mv_id_q4_0_f32_8x_flat;
|
||||
@@ -5547,6 +5548,7 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) {
|
||||
build_program_from_source(backend_ctx, kernel_src.c_str(), CL_moe_compile_opts);
|
||||
|
||||
CL_CHECK((backend_ctx->kernel_moe_histogram = clCreateKernel(prog, "kernel_moe_histogram", &err), err));
|
||||
CL_CHECK((backend_ctx->kernel_moe_zero_dst = clCreateKernel(prog, "kernel_moe_zero_dst", &err), err));
|
||||
CL_CHECK((backend_ctx->kernel_moe_scan = clCreateKernel(prog, "kernel_moe_scan", &err), err));
|
||||
CL_CHECK((backend_ctx->kernel_moe_fill = clCreateKernel(prog, "kernel_moe_fill", &err), err));
|
||||
CL_CHECK((backend_ctx->kernel_moe_scatter = clCreateKernel(prog, "kernel_moe_scatter", &err), err));
|
||||
@@ -26961,6 +26963,39 @@ static void ggml_cl_mul_mat_id(ggml_backend_t backend, const ggml_tensor * src0,
|
||||
|
||||
GGML_ASSERT(ne00 == ne10);
|
||||
|
||||
// the matmul kernels only write the rows an expert owns, so zero the skipped rows here
|
||||
{
|
||||
cl_int err;
|
||||
|
||||
cl_buffer_region region;
|
||||
region.origin = offset2;
|
||||
region.size = nb21 * ne21;
|
||||
cl_mem buf_ids = clCreateSubBuffer(extra2->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err);
|
||||
CL_CHECK(err);
|
||||
|
||||
region.origin = offsetd;
|
||||
region.size = (size_t) ne0 * ne1 * ne2 * sizeof(float);
|
||||
cl_mem buf_dst = clCreateSubBuffer(extrad->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err);
|
||||
CL_CHECK(err);
|
||||
|
||||
const cl_uint ids_stride = nb21 / sizeof(int);
|
||||
|
||||
cl_kernel kernel_zero = backend_ctx->kernel_moe_zero_dst;
|
||||
CL_CHECK(clSetKernelArg(kernel_zero, 0, sizeof(cl_mem), &buf_ids));
|
||||
CL_CHECK(clSetKernelArg(kernel_zero, 1, sizeof(cl_mem), &buf_dst));
|
||||
CL_CHECK(clSetKernelArg(kernel_zero, 2, sizeof(cl_uint), &ne21));
|
||||
CL_CHECK(clSetKernelArg(kernel_zero, 3, sizeof(cl_uint), &ne20));
|
||||
CL_CHECK(clSetKernelArg(kernel_zero, 4, sizeof(cl_uint), &ids_stride));
|
||||
CL_CHECK(clSetKernelArg(kernel_zero, 5, sizeof(cl_uint), &ne0));
|
||||
|
||||
size_t zero_local[3] = {1, 1, 64};
|
||||
size_t zero_global[3] = {(size_t) ne21, (size_t) ne20, 64};
|
||||
backend_ctx->enqueue_ndrange_kernel(kernel_zero, 3, zero_global, zero_local, dst);
|
||||
|
||||
CL_CHECK(clReleaseMemObject(buf_ids));
|
||||
CL_CHECK(clReleaseMemObject(buf_dst));
|
||||
}
|
||||
|
||||
int sgs = 32; // subgroup size
|
||||
int nsg = 1; // number of subgroups
|
||||
int nrows = 1; // number of row in src1
|
||||
|
||||
@@ -34,6 +34,14 @@ kernel void kernel_add_id(
|
||||
|
||||
global float * dst_row = (global float *)((global char *)dst + i1*nb1 + i2*nb2);
|
||||
global float * src0_row = (global float *)((global char *)src0 + i1*nb01 + i2*nb02);
|
||||
|
||||
if (i11 == -1) { // skipped slot, add nothing
|
||||
for (int i0 = get_local_id(0); i0 < ne0; i0 += get_local_size(0)) {
|
||||
dst_row[i0] = src0_row[i0];
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
global float * src1_row = (global float *)((global char *)src1 + i11*nb11);
|
||||
|
||||
for (int i0 = get_local_id(0); i0 < ne0; i0 += get_local_size(0)) {
|
||||
|
||||
@@ -89,6 +89,9 @@ __kernel void kernel_gemv_moe_mxfp4_f32_ns(
|
||||
uint i11 = i20 % ne11;
|
||||
|
||||
uint expert_id = src2[i20];
|
||||
if ((int)expert_id == -1) { // skipped slot, the dst row is zeroed by kernel_moe_zero_dst
|
||||
return;
|
||||
}
|
||||
uint expert_offset = expert_id * ne00 * ne01 / 32;
|
||||
|
||||
__private float sum = 0.0f; // each thread calculate partial sum of one output
|
||||
@@ -188,6 +191,9 @@ __kernel void kernel_gemv_moe_mxfp4_f32_ns_wimg(
|
||||
uint i11 = i20 % ne11;
|
||||
|
||||
uint expert_id = src2[i20];
|
||||
if ((int)expert_id == -1) { // skipped slot, the dst row is zeroed by kernel_moe_zero_dst
|
||||
return;
|
||||
}
|
||||
uint expert_offset = expert_id * ne00 * ne01 / 32;
|
||||
|
||||
__private float sum = 0.0f;
|
||||
|
||||
@@ -44,6 +44,9 @@ __kernel void kernel_gemv_moe_q4_0_f32_ns(
|
||||
uint i11 = i20 % ne11;
|
||||
|
||||
uint expert_id = src2[i20];
|
||||
if ((int)expert_id == -1) { // skipped slot, the dst row is zeroed by kernel_moe_zero_dst
|
||||
return;
|
||||
}
|
||||
uint expert_offset = expert_id * ne00 * ne01 / 32;
|
||||
|
||||
__private float sum = 0.0f; // each thread calculate partial sum of one output
|
||||
|
||||
@@ -45,6 +45,9 @@ __kernel void kernel_gemv_moe_q4_1_f32_ns(
|
||||
uint i11 = i20 % ne11;
|
||||
|
||||
uint expert_id = src2[i20];
|
||||
if ((int)expert_id == -1) { // skipped slot, the dst row is zeroed by kernel_moe_zero_dst
|
||||
return;
|
||||
}
|
||||
uint expert_offset = expert_id * ne00 * ne01 / 32;
|
||||
|
||||
__private float sum = 0.0f; // each thread calculate partial sum of one output
|
||||
|
||||
@@ -61,6 +61,9 @@ __kernel void kernel_gemv_moe_q4_k_f32_ns(
|
||||
uint i11 = i20 % ne11;
|
||||
|
||||
uint expert_id = src2[i20];
|
||||
if ((int)expert_id == -1) { // skipped slot, the dst row is zeroed by kernel_moe_zero_dst
|
||||
return;
|
||||
}
|
||||
|
||||
int num_superblocks = ne00 / QK_K;
|
||||
int num_subblocks = ne00 / 32;
|
||||
@@ -180,6 +183,9 @@ __kernel void kernel_gemv_moe_q4_k_f32_ns_wimg(
|
||||
uint i11 = i20 % ne11;
|
||||
|
||||
uint expert_id = src2[i20];
|
||||
if ((int)expert_id == -1) { // skipped slot, the dst row is zeroed by kernel_moe_zero_dst
|
||||
return;
|
||||
}
|
||||
|
||||
int num_superblocks = ne00 / QK_K;
|
||||
int num_subblocks = ne00 / 32;
|
||||
|
||||
@@ -45,6 +45,9 @@ __kernel void kernel_gemv_moe_q5_0_f32_ns(
|
||||
uint i11 = i20 % ne11;
|
||||
|
||||
uint expert_id = src2[i20];
|
||||
if ((int)expert_id == -1) { // skipped slot, the dst row is zeroed by kernel_moe_zero_dst
|
||||
return;
|
||||
}
|
||||
uint expert_offset = expert_id * ne00 * ne01 / 32;
|
||||
|
||||
__private float sum = 0.0f; // each thread calculate partial sum of one output
|
||||
|
||||
@@ -46,6 +46,9 @@ __kernel void kernel_gemv_moe_q5_1_f32_ns(
|
||||
uint i11 = i20 % ne11;
|
||||
|
||||
uint expert_id = src2[i20];
|
||||
if ((int)expert_id == -1) { // skipped slot, the dst row is zeroed by kernel_moe_zero_dst
|
||||
return;
|
||||
}
|
||||
uint expert_offset = expert_id * ne00 * ne01 / 32;
|
||||
|
||||
__private float sum = 0.0f; // each thread calculate partial sum of one output
|
||||
|
||||
@@ -62,6 +62,9 @@ __kernel void kernel_gemv_moe_q5_k_f32_ns(
|
||||
uint i11 = i20 % ne11;
|
||||
|
||||
uint expert_id = src2[i20];
|
||||
if ((int)expert_id == -1) { // skipped slot, the dst row is zeroed by kernel_moe_zero_dst
|
||||
return;
|
||||
}
|
||||
|
||||
int num_superblocks = ne00 / QK_K;
|
||||
int num_subblocks = ne00 / 32;
|
||||
|
||||
@@ -45,6 +45,9 @@ __kernel void kernel_gemv_moe_q6_k_f32_ns(
|
||||
uint i11 = i20 % ne11;
|
||||
|
||||
uint expert_id = src2[i20];
|
||||
if ((int)expert_id == -1) { // skipped slot, the dst row is zeroed by kernel_moe_zero_dst
|
||||
return;
|
||||
}
|
||||
|
||||
int num_superblocks = ne00 / QK_K;
|
||||
int num_subblocks = ne00 / 32; // 8 sub-blocks of 32 per super-block
|
||||
|
||||
@@ -18,6 +18,9 @@ __kernel void kernel_moe_histogram(
|
||||
}
|
||||
|
||||
int expert_id = input[n * n_experts + k];
|
||||
if (expert_id == -1) { // skipped slot, owned by no expert
|
||||
return;
|
||||
}
|
||||
atomic_inc(&hist[expert_id]);
|
||||
}
|
||||
|
||||
@@ -63,6 +66,9 @@ __kernel void kernel_moe_scatter(
|
||||
}
|
||||
|
||||
int val = input[n * n_experts + k];
|
||||
if (val == -1) { // skipped slot, owned by no expert
|
||||
return;
|
||||
}
|
||||
|
||||
int local_slot = atomic_inc(&slot_counter[val]);
|
||||
|
||||
@@ -74,6 +80,32 @@ __kernel void kernel_moe_scatter(
|
||||
emap[tile_idx] = val;
|
||||
}
|
||||
|
||||
// the gemm only writes the rows an expert owns, so the skipped rows (negative ids) are zeroed here
|
||||
__kernel void kernel_moe_zero_dst(
|
||||
__global const int * input,
|
||||
__global float * dst,
|
||||
uint N,
|
||||
uint topK,
|
||||
uint n_experts,
|
||||
uint ne0
|
||||
) {
|
||||
uint n = get_global_id(0);
|
||||
uint k = get_global_id(1);
|
||||
|
||||
if (n >= N || k >= topK) {
|
||||
return;
|
||||
}
|
||||
|
||||
if (input[n * n_experts + k] != -1) {
|
||||
return;
|
||||
}
|
||||
|
||||
__global float * dst_row = dst + ((ulong)n * topK + k) * ne0;
|
||||
for (uint i = get_local_id(2); i < ne0; i += get_local_size(2)) {
|
||||
dst_row[i] = 0.0f;
|
||||
}
|
||||
}
|
||||
|
||||
// Deterministic replacement for kernel_moe_scatter.
|
||||
//
|
||||
// kernel_moe_scatter takes each token's slot from atomic_inc(slot_counter[expert]),
|
||||
|
||||
@@ -172,6 +172,9 @@ kernel void kernel_mul_mv_id_mxfp4_f32(
|
||||
const int idx = get_group_id(2)%ne20;
|
||||
|
||||
int i02 = ((global int *) (src2 + iid1*nb21))[idx];
|
||||
if (i02 == -1) { // skipped slot, the dst row is zeroed by kernel_moe_zero_dst
|
||||
return;
|
||||
}
|
||||
|
||||
int i11 = idx % ne11;
|
||||
int i12 = iid1;
|
||||
|
||||
@@ -101,6 +101,9 @@ kernel void kernel_mul_mv_id_mxfp4_f32_flat(
|
||||
const int idx = get_group_id(2) % ne20;
|
||||
|
||||
uint i02 = ((global uint *) (src2 + offset2 + iid1 * nb21))[idx];
|
||||
if ((int)i02 == -1) { // skipped slot, the dst row is zeroed by kernel_moe_zero_dst
|
||||
return;
|
||||
}
|
||||
|
||||
int i11 = idx % ne11;
|
||||
|
||||
|
||||
@@ -267,6 +267,9 @@ kernel void kernel_mul_mv_id_q4_0_f32_8x_flat(
|
||||
const int idx = get_group_id(2)%ne20;
|
||||
|
||||
const int i02 = ((global int *)(src2 + iid1*nb21))[idx];
|
||||
if (i02 == -1) { // skipped slot, the dst row is zeroed by kernel_moe_zero_dst
|
||||
return;
|
||||
}
|
||||
|
||||
const int i11 = idx%ne11;
|
||||
const int i12 = iid1;
|
||||
|
||||
@@ -73,6 +73,9 @@ kernel void kernel_mul_mv_id_q8_0_f32(
|
||||
int idx = get_group_id(2)%ne20;
|
||||
|
||||
int i02 = ((global int *) (src2 + iid1*nb21))[idx];
|
||||
if (i02 == -1) { // skipped slot, the dst row is zeroed by kernel_moe_zero_dst
|
||||
return;
|
||||
}
|
||||
|
||||
int i11_ = idx % ne11;
|
||||
int i12_ = iid1;
|
||||
|
||||
@@ -72,6 +72,9 @@ kernel void kernel_mul_mv_id_q8_0_f32_flat(
|
||||
int idx = (int)get_group_id(2)%ne20;
|
||||
|
||||
int i02 = ((global int *) (src2 + iid1*nb21))[idx];
|
||||
if (i02 == -1) { // skipped slot, the dst row is zeroed by kernel_moe_zero_dst
|
||||
return;
|
||||
}
|
||||
|
||||
int i11_ = idx % ne11;
|
||||
int i12_ = iid1;
|
||||
|
||||
@@ -9,8 +9,12 @@
|
||||
#include <openvino/op/constant.hpp>
|
||||
#include <openvino/op/convert.hpp>
|
||||
#include <openvino/op/gather.hpp>
|
||||
#include <openvino/op/greater_eq.hpp>
|
||||
#include <openvino/op/maximum.hpp>
|
||||
#include <openvino/op/multiply.hpp>
|
||||
#include <openvino/op/reshape.hpp>
|
||||
#include <openvino/op/shape_of.hpp>
|
||||
#include <openvino/op/unsqueeze.hpp>
|
||||
|
||||
namespace ov {
|
||||
namespace frontend {
|
||||
@@ -52,8 +56,16 @@ OutputVector translate_add_id(const NodeContext & context) {
|
||||
ids = std::make_shared<ov::op::v0::Convert>(ids, ov::element::i32);
|
||||
}
|
||||
|
||||
// an id of -1 marks a skipped slot: gather row 0 for it and zero the bias with a mask
|
||||
auto zero = ov::op::v0::Constant::create(ids.get_element_type(), ov::Shape{}, {0});
|
||||
ov::Output<ov::Node> keep = std::make_shared<ov::op::v1::GreaterEqual>(ids, zero);
|
||||
keep = std::make_shared<ov::op::v0::Convert>(keep, bias.get_element_type());
|
||||
keep = std::make_shared<ov::op::v0::Unsqueeze>(keep, ov::op::v0::Constant::create(ov::element::i64, ov::Shape{1}, {-1}));
|
||||
ids = std::make_shared<ov::op::v1::Maximum>(ids, zero);
|
||||
|
||||
auto gather_axis = ov::op::v0::Constant::create(ov::element::i32, ov::Shape{}, {0});
|
||||
ov::Output<ov::Node> selected_bias = std::make_shared<ov::op::v8::Gather>(bias, ids, gather_axis);
|
||||
selected_bias = std::make_shared<ov::op::v1::Multiply>(selected_bias, keep);
|
||||
selected_bias = std::make_shared<ov::op::v1::Reshape>(
|
||||
selected_bias, std::make_shared<ov::op::v3::ShapeOf>(input, ov::element::i64), false);
|
||||
|
||||
|
||||
@@ -26,6 +26,15 @@ static void add_id_kernel(
|
||||
float* dst_row = (float*)((char*)dst + i1 * nb1 + i2 * nb2);
|
||||
const float* src0_row =
|
||||
(const float*)((const char*)src0 + i1 * nb01 + i2 * nb02);
|
||||
|
||||
if (i11 == -1) { // skipped slot, add nothing
|
||||
for (int64_t i0 = item_ct1.get_local_id(2); i0 < ne0;
|
||||
i0 += item_ct1.get_local_range(2)) {
|
||||
dst_row[i0] = src0_row[i0];
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
const float* src1_row = (const float*)((const char*)src1 + i11 * nb11);
|
||||
|
||||
for (int64_t i0 = item_ct1.get_local_id(2); i0 < ne0;
|
||||
|
||||
@@ -5700,6 +5700,23 @@ __dpct_inline__ static void k_copy_dst_from_contiguous(
|
||||
}
|
||||
}
|
||||
|
||||
__dpct_inline__ static void k_zero_dst_rows(
|
||||
char *__restrict__ dst_original,
|
||||
const mmid_row_mapping *__restrict__ row_mapping, int64_t ne0, size_t nb1,
|
||||
size_t nb2, const sycl::nd_item<3> &item_ct1) {
|
||||
int32_t i = item_ct1.get_group(2);
|
||||
|
||||
const int32_t i1 = row_mapping[i].i1;
|
||||
const int32_t i2 = row_mapping[i].i2;
|
||||
|
||||
float * dst_row_original = (float *)(dst_original + i1*nb1 + i2*nb2);
|
||||
|
||||
for (int j = item_ct1.get_local_id(2); j < ne0;
|
||||
j += item_ct1.get_local_range(2)) {
|
||||
dst_row_original[j] = 0.0f;
|
||||
}
|
||||
}
|
||||
|
||||
// returns true when ggml_sycl_mul_mat_id takes the fallback path that requires stream synchronization
|
||||
static bool ggml_sycl_mul_mat_id_needs_sync(const ggml_tensor * dst) {
|
||||
const ggml_tensor * src0 = dst->src[0];
|
||||
@@ -5785,14 +5802,18 @@ static void mmid_counting_sort_rows(
|
||||
int64_t n_ids, int64_t n_as, int64_t n_routed_rows,
|
||||
std::vector<int64_t> & expert_counts,
|
||||
std::vector<int64_t> & expert_row_offsets,
|
||||
std::vector<mmid_row_mapping> & routed_row_src) {
|
||||
std::vector<mmid_row_mapping> & routed_row_src,
|
||||
std::vector<mmid_row_mapping> & skipped_row_dst) {
|
||||
|
||||
// frequencies: how many routed rows each expert "owns"
|
||||
expert_counts.assign(n_as, 0);
|
||||
for (int64_t iid1 = 0; iid1 < ids->ne[1]; iid1++) {
|
||||
for (int64_t id = 0; id < n_ids; id++) {
|
||||
const int32_t row_id_i = *(const int32_t *) (ids_host + iid1*ids->nb[1] + id*ids->nb[0]);
|
||||
GGML_ASSERT(row_id_i >= 0 && row_id_i < n_as);
|
||||
GGML_ASSERT(row_id_i == -1 || (row_id_i >= 0 && row_id_i < n_as));
|
||||
if (row_id_i == -1) {
|
||||
continue;
|
||||
}
|
||||
expert_counts[row_id_i]++;
|
||||
}
|
||||
}
|
||||
@@ -5805,10 +5826,17 @@ static void mmid_counting_sort_rows(
|
||||
|
||||
std::vector<int64_t> expert_row_next = expert_row_offsets;
|
||||
routed_row_src.resize(n_routed_rows);
|
||||
skipped_row_dst.clear();
|
||||
for (int64_t iid1 = 0; iid1 < ids->ne[1]; iid1++) {
|
||||
for (int64_t id = 0; id < n_ids; id++) {
|
||||
const int32_t row_id_i = *(const int32_t *) (ids_host + iid1*ids->nb[1] + id*ids->nb[0]);
|
||||
GGML_ASSERT(row_id_i >= 0 && row_id_i < n_as);
|
||||
GGML_ASSERT(row_id_i == -1 || (row_id_i >= 0 && row_id_i < n_as));
|
||||
|
||||
// a skipped slot has no expert, its dst row is zeroed instead
|
||||
if (row_id_i == -1) {
|
||||
skipped_row_dst.push_back({(int32_t) id, (int32_t) iid1});
|
||||
continue;
|
||||
}
|
||||
|
||||
// find and validate the next free row for a given expert (row_id_i)
|
||||
const int64_t routed_row = expert_row_next[row_id_i]++;
|
||||
@@ -5817,6 +5845,7 @@ static void mmid_counting_sort_rows(
|
||||
routed_row_src[routed_row] = {(int32_t) id, (int32_t) iid1};
|
||||
}
|
||||
}
|
||||
routed_row_src.resize(expert_row_offsets[n_as]);
|
||||
}
|
||||
|
||||
static void ggml_sycl_mul_mat_id(ggml_backend_sycl_context & ctx,
|
||||
@@ -5884,7 +5913,7 @@ static void ggml_sycl_mul_mat_id(ggml_backend_sycl_context & ctx,
|
||||
for (int64_t iid1 = 0; iid1 < ids->ne[1]; iid1++) {
|
||||
for (int64_t id = 0; id < n_ids; id++) {
|
||||
const int32_t i02 = *(const int32_t *) (ids_host.data() + iid1*ids->nb[1] + id*ids->nb[0]);
|
||||
GGML_ASSERT(i02 >= 0 && i02 < n_as);
|
||||
GGML_ASSERT(i02 == -1 || (i02 >= 0 && i02 < n_as));
|
||||
|
||||
const int64_t i11 = id % ne11;
|
||||
const int64_t i12 = iid1;
|
||||
@@ -5892,6 +5921,12 @@ static void ggml_sycl_mul_mat_id(ggml_backend_sycl_context & ctx,
|
||||
const int64_t i1 = id;
|
||||
const int64_t i2 = i12;
|
||||
|
||||
if (i02 == -1) { // skipped slot
|
||||
SYCL_CHECK(CHECK_TRY_ERROR(
|
||||
stream->memset(dst_original + i1*nb1 + i2*nb2, 0, ne0*sizeof(float))));
|
||||
continue;
|
||||
}
|
||||
|
||||
src0_row.data = src0_original + i02*nb02;
|
||||
src1_row.data = src1_original + i11*nb11 + i12*nb12;
|
||||
dst_row.data = dst_original + i1*nb1 + i2*nb2;
|
||||
@@ -5913,20 +5948,43 @@ static void ggml_sycl_mul_mat_id(ggml_backend_sycl_context & ctx,
|
||||
std::vector<int64_t> expert_row_offsets;
|
||||
// the sources (slot/token pairs) of contiguous rows to guide k_copy_src1_to_contiguous
|
||||
std::vector<mmid_row_mapping> & routed_row_src = ctx.mmid_row_mapping_host;
|
||||
// the slot/token pairs of the skipped slots, whose dst rows are zeroed
|
||||
std::vector<mmid_row_mapping> skipped_row_dst;
|
||||
|
||||
mmid_counting_sort_rows(ids, ids_host.data(), n_ids, n_as, n_routed_rows,
|
||||
expert_row_counts, expert_row_offsets, routed_row_src);
|
||||
expert_row_counts, expert_row_offsets, routed_row_src, skipped_row_dst);
|
||||
|
||||
const int64_t n_valid_rows = (int64_t) routed_row_src.size();
|
||||
|
||||
ggml_sycl_pool_alloc<mmid_row_mapping> dev_row_mapping(ctx.pool(), n_routed_rows);
|
||||
SYCL_CHECK(CHECK_TRY_ERROR(
|
||||
stream->memcpy(dev_row_mapping.get(), routed_row_src.data(), n_routed_rows*sizeof(mmid_row_mapping))));
|
||||
stream->memcpy(dev_row_mapping.get(), routed_row_src.data(), n_valid_rows*sizeof(mmid_row_mapping))));
|
||||
|
||||
const unsigned int max_work_group_size = ggml_sycl_info().max_work_group_sizes[ctx.device];
|
||||
assert(max_work_group_size % (WARP_SIZE * WARP_SIZE) == 0);
|
||||
|
||||
{
|
||||
if (!skipped_row_dst.empty()) {
|
||||
ggml_sycl_pool_alloc<mmid_row_mapping> dev_skipped(ctx.pool(), skipped_row_dst.size());
|
||||
SYCL_CHECK(CHECK_TRY_ERROR(
|
||||
stream->memcpy(dev_skipped.get(), skipped_row_dst.data(), skipped_row_dst.size()*sizeof(mmid_row_mapping))));
|
||||
|
||||
sycl::range<3> block_dims(1, 1, std::min((unsigned int)ne0, max_work_group_size));
|
||||
sycl::range<3> grid_dims(1, 1, skipped_row_dst.size());
|
||||
stream->submit([&](sycl::handler &cgh) {
|
||||
mmid_row_mapping *__restrict dev_skipped_get = dev_skipped.get();
|
||||
|
||||
cgh.parallel_for(
|
||||
sycl::nd_range<3>(grid_dims * block_dims, block_dims),
|
||||
[=](sycl::nd_item<3> item_ct1) {
|
||||
k_zero_dst_rows(dst_original, dev_skipped_get, ne0, nb1, nb2, item_ct1);
|
||||
});
|
||||
});
|
||||
SYCL_CHECK(CHECK_TRY_ERROR(stream->wait()));
|
||||
}
|
||||
|
||||
if (n_valid_rows > 0) {
|
||||
sycl::range<3> block_dims(1, 1, std::min((unsigned int)ne10, max_work_group_size));
|
||||
sycl::range<3> grid_dims(1, 1, n_routed_rows);
|
||||
sycl::range<3> grid_dims(1, 1, n_valid_rows);
|
||||
stream->submit([&](sycl::handler &cgh) {
|
||||
char *__restrict src1_contiguous_get =
|
||||
src1_contiguous.get();
|
||||
@@ -5992,9 +6050,9 @@ static void ggml_sycl_mul_mat_id(ggml_backend_sycl_context & ctx,
|
||||
ggml_sycl_mul_mat(ctx, &src0_row, &src1_row, &dst_row);
|
||||
}
|
||||
|
||||
{
|
||||
if (n_valid_rows > 0) {
|
||||
sycl::range<3> block_dims(1, 1, std::min((unsigned int)ne0, max_work_group_size));
|
||||
sycl::range<3> grid_dims(1, 1, n_routed_rows);
|
||||
sycl::range<3> grid_dims(1, 1, n_valid_rows);
|
||||
stream->submit([&](sycl::handler &cgh) {
|
||||
const char *__restrict dst_contiguous_get =
|
||||
dst_contiguous.get();
|
||||
|
||||
@@ -3149,7 +3149,6 @@ static void mul_mat_vec_q_moe(
|
||||
const int expert_idx = item_ct1.get_group(1);
|
||||
const int i02 = ids_dev[expert_idx];
|
||||
|
||||
const char * vx = (const char *) vx_base + (size_t) i02 * expert_weight_stride;
|
||||
const char * vy = (const char *) vy_base + (size_t) expert_idx * src1_row_stride;
|
||||
float * dst = (float *) ((char *) dst_base + (size_t) expert_idx * dst_row_stride);
|
||||
|
||||
@@ -3159,6 +3158,15 @@ static void mul_mat_vec_q_moe(
|
||||
return;
|
||||
}
|
||||
|
||||
if (i02 == -1) { // skipped slot
|
||||
if (item_ct1.get_local_id(2) == 0) {
|
||||
dst[row] = 0.0f;
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
const char * vx = (const char *) vx_base + (size_t) i02 * expert_weight_stride;
|
||||
|
||||
const int blocks_per_row = ncols / qk;
|
||||
constexpr int blocks_per_warp = (vdr * WARP_SIZE + qi - 1) / qi;
|
||||
|
||||
@@ -3355,7 +3363,6 @@ static void mul_mat_vec_q_moe_reorder(
|
||||
const int expert_idx = item_ct1.get_group(1);
|
||||
const int i02 = ids_dev[expert_idx];
|
||||
|
||||
const char * vx = (const char *) vx_base + (size_t) i02 * expert_weight_stride;
|
||||
const char * vy = (const char *) vy_base + (size_t) expert_idx * src1_row_stride;
|
||||
float * dst = (float *) ((char *) dst_base + (size_t) expert_idx * dst_row_stride);
|
||||
|
||||
@@ -3366,6 +3373,15 @@ static void mul_mat_vec_q_moe_reorder(
|
||||
|
||||
const auto sg = item_ct1.get_sub_group();
|
||||
|
||||
if (i02 == -1) { // skipped slot
|
||||
if (sg.leader()) {
|
||||
dst[row] = 0.0f;
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
const char * vx = (const char *) vx_base + (size_t) i02 * expert_weight_stride;
|
||||
|
||||
constexpr int vdr_mmvq = reorder_vec_dot_q_sycl::gtype == GGML_TYPE_Q8_0 ?
|
||||
VDR_Q8_0_Q8_1_MMVQ : block_traits::vdr_mmvq;
|
||||
const int blocks_per_row = ncols / block_traits::qk;
|
||||
|
||||
@@ -222,6 +222,9 @@ struct vk_op_count_experts_push_constants {
|
||||
uint32_t ne00L;
|
||||
uint32_t row_tile_size;
|
||||
uint32_t row_ids_offset;
|
||||
uint32_t ne0_dst;
|
||||
uint32_t stride_dst;
|
||||
uint32_t batch_stride_dst;
|
||||
};
|
||||
|
||||
struct vk_op_glu_push_constants {
|
||||
|
||||
@@ -3585,9 +3585,9 @@ void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) {
|
||||
ggml_vk_create_pipeline(device, device->pipeline_count_equal_i32, "count_equal_i32", count_equal_i32_len, count_equal_i32_data, "main", 3, sizeof(vk_op_push_constants), {512, 1, 1}, { device->subgroup_size }, 1);
|
||||
|
||||
if (device->subgroup_arithmetic && device->subgroup_vote && device->subgroup_require_full_support) {
|
||||
ggml_vk_create_pipeline(device, device->pipeline_count_experts, "count_experts", count_experts_subgroup_len, count_experts_subgroup_data, "main", 2, sizeof(vk_op_count_experts_push_constants), {1, 1, 1}, {}, 1, true, true);
|
||||
ggml_vk_create_pipeline(device, device->pipeline_count_experts, "count_experts", count_experts_subgroup_len, count_experts_subgroup_data, "main", 3, sizeof(vk_op_count_experts_push_constants), {1, 1, 1}, {}, 1, true, true);
|
||||
} else {
|
||||
ggml_vk_create_pipeline(device, device->pipeline_count_experts, "count_experts", count_experts_len, count_experts_data, "main", 2, sizeof(vk_op_count_experts_push_constants), {1, 1, 1}, {}, 1, true);
|
||||
ggml_vk_create_pipeline(device, device->pipeline_count_experts, "count_experts", count_experts_len, count_experts_data, "main", 3, sizeof(vk_op_count_experts_push_constants), {1, 1, 1}, {}, 1, true);
|
||||
}
|
||||
|
||||
// comb holds a token's 4x4 matrix in one 16-lane slice of a subgroup, so it
|
||||
@@ -7643,16 +7643,20 @@ static void ggml_vk_mul_mat_id_q_f16(ggml_backend_vk_context * ctx, vk_context&
|
||||
ggml_vk_sync_buffers(ctx, subctx);
|
||||
}
|
||||
{
|
||||
// this also zeroes the dst rows of the skipped slots (negative ids)
|
||||
vk_op_count_experts_push_constants pc = { (uint32_t)nei0,
|
||||
(uint32_t)nei1,
|
||||
(uint32_t)(nbi0 / ggml_type_size(ids->type)),
|
||||
(uint32_t)(nbi1 / ggml_type_size(ids->type)),
|
||||
(uint32_t)(get_misalign_bytes(ctx, ids) / ggml_type_size(ids->type)),
|
||||
(uint32_t)n_as,
|
||||
0, 0, bn, (uint32_t)row_ids_offset };
|
||||
0, 0, bn, (uint32_t)row_ids_offset,
|
||||
(uint32_t)ne01,
|
||||
(uint32_t)(dst->nb[1] / sizeof(float)),
|
||||
(uint32_t)(dst->nb[2] / sizeof(float)) };
|
||||
init_pushconst_fastdiv(pc);
|
||||
ggml_vk_dispatch_pipeline(ctx, subctx, count_experts,
|
||||
{ d_ids, expert_count_buf }, pc,
|
||||
{ d_ids, expert_count_buf, vk_subbuffer{ d_D, d_buf_offset, d_sz } }, pc,
|
||||
{ 1, 1, 1 });
|
||||
}
|
||||
|
||||
|
||||
@@ -27,14 +27,22 @@ void main() {
|
||||
const uint i1 = gl_WorkGroupID.x;
|
||||
const uint i2 = gl_WorkGroupID.y;
|
||||
|
||||
const uint i11 = data_c[i1 + i2 * p.s21];
|
||||
const int i11 = data_c[i1 + i2 * p.s21];
|
||||
|
||||
const uint s1 = p.ne0;
|
||||
const uint s2 = p.ne0 * p.ne1;
|
||||
|
||||
const uint d0 = i1 * s1 + i2 * s2;
|
||||
const uint a0 = i1 * p.s01 + i2 * p.s02;
|
||||
const uint b0 = i11 * p.s11;
|
||||
|
||||
if (i11 == -1) { // skipped slot, add nothing
|
||||
for (uint i0 = gl_LocalInvocationID.x; i0 < p.ne0; i0 += BLOCK_SIZE) {
|
||||
data_d[d0 + i0] = data_a[a0 + i0];
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
const uint b0 = uint(i11) * p.s11;
|
||||
|
||||
for (uint i0 = gl_LocalInvocationID.x; i0 < p.ne0; i0 += BLOCK_SIZE) {
|
||||
data_d[d0 + i0] = data_a[a0 + i0] + data_b[b0 + i0];
|
||||
|
||||
@@ -23,6 +23,9 @@ layout (push_constant) uniform parameter
|
||||
uint32_t ne00L;
|
||||
uint32_t row_tile_size;
|
||||
uint32_t row_ids_offset;
|
||||
uint32_t ne0_dst;
|
||||
uint32_t stride_dst;
|
||||
uint32_t batch_stride_dst;
|
||||
} p;
|
||||
|
||||
#define BLOCK_SIZE 256
|
||||
@@ -32,6 +35,7 @@ layout(local_size_x = BLOCK_SIZE, local_size_y = 1, local_size_z = 1) in;
|
||||
|
||||
layout (binding = 0) readonly buffer A {uint data_a[];};
|
||||
layout (binding = 1) writeonly buffer D {uint data_d[];};
|
||||
layout (binding = 2) writeonly buffer DST {float data_dst[];};
|
||||
|
||||
shared uint vals[EXPERTS_PER_CHUNK];
|
||||
shared uint offsets[EXPERTS_PER_CHUNK];
|
||||
@@ -47,6 +51,26 @@ void write_tiles(uint expert, uint count, uint tile_begin, uint n_tiles) {
|
||||
}
|
||||
}
|
||||
|
||||
// the matmul writes only the rows that an expert owns, so zero the rows of the skipped slots
|
||||
void zero_skipped_rows() {
|
||||
const uint num_elements = p.ne00 * p.ne01;
|
||||
const uint tid = gl_LocalInvocationID.x;
|
||||
|
||||
for (uint idx = 0; idx < num_elements; idx++) {
|
||||
const uint i01 = fastdiv(idx, p.ne00mp, p.ne00L);
|
||||
const uint i00 = idx - i01 * p.ne00;
|
||||
|
||||
if (int(data_a[p.a_offset + i01 * p.nb01 + i00 * p.nb00]) != -1) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const uint d_off = i00 * p.stride_dst + i01 * p.batch_stride_dst;
|
||||
for (uint i = tid; i < p.ne0_dst; i += BLOCK_SIZE) {
|
||||
data_dst[d_off + i] = 0.0f;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void main() {
|
||||
const uint num_elements = p.ne00 * p.ne01;
|
||||
const uint tid = gl_LocalInvocationID.x;
|
||||
@@ -123,4 +147,6 @@ void main() {
|
||||
if (tid == 0) {
|
||||
data_d[0] = total_tiles;
|
||||
}
|
||||
|
||||
zero_skipped_rows();
|
||||
}
|
||||
@@ -42,6 +42,7 @@ layout (push_constant) uniform parameter
|
||||
|
||||
#ifdef MUL_MAT_ID
|
||||
uint expert_id;
|
||||
bool expert_skip; // an id of -1 skips the slot, the result row is zeroed
|
||||
#endif
|
||||
|
||||
void get_offsets(out uint a_offset, out uint b_offset, out uint d_offset) {
|
||||
@@ -63,7 +64,9 @@ void get_offsets(out uint a_offset, out uint b_offset, out uint d_offset) {
|
||||
batch_idx_a = i03 * p.ne02 + i02;
|
||||
}
|
||||
#else
|
||||
expert_id = data_ids[expert_i0 + p.expert_i1 * p.nbi1];
|
||||
const int expert = data_ids[expert_i0 + p.expert_i1 * p.nbi1];
|
||||
expert_skip = expert == -1;
|
||||
expert_id = expert_skip ? 0 : uint(expert); // clamp, so that the offsets below stay in range
|
||||
#endif
|
||||
|
||||
a_offset =
|
||||
@@ -113,6 +116,9 @@ void reduce_result(inout FLOAT_TYPE temp[NUM_COLS][NUM_ROWS], const in uint32_t
|
||||
const uint expert_i0 = gl_GlobalInvocationID.y;
|
||||
temp[j][n] *= FLOAT_TYPE(data_fuse1[expert_i0]);
|
||||
}
|
||||
if (expert_skip) {
|
||||
temp[j][n] = FLOAT_TYPE(0);
|
||||
}
|
||||
#else
|
||||
if ((p.fusion_flags & MAT_VEC_FUSION_FLAGS_BIAS0) != 0) {
|
||||
temp[j][n] += FLOAT_TYPE(data_fuse0[j*p.batch_stride_d + d_offset + first_row + n]);
|
||||
@@ -168,6 +174,9 @@ void reduce_result(FLOAT_TYPE temp[NUM_COLS][NUM_ROWS], const in uint32_t d_offs
|
||||
const uint expert_i0 = gl_GlobalInvocationID.y;
|
||||
temp[j][n] *= FLOAT_TYPE(data_fuse1[expert_i0]);
|
||||
}
|
||||
if (expert_skip) {
|
||||
temp[j][n] = FLOAT_TYPE(0);
|
||||
}
|
||||
#else
|
||||
if ((p.fusion_flags & MAT_VEC_FUSION_FLAGS_BIAS0) != 0) {
|
||||
temp[j][n] += FLOAT_TYPE(data_fuse0[j*p.batch_stride_d + d_offset + first_row + n]);
|
||||
@@ -213,6 +222,9 @@ void reduce_result(FLOAT_TYPE temp[NUM_COLS][NUM_ROWS], const in uint32_t d_offs
|
||||
const uint expert_i0 = gl_GlobalInvocationID.y;
|
||||
tmpsh[j][n][0] *= FLOAT_TYPE(data_fuse1[expert_i0]);
|
||||
}
|
||||
if (expert_skip) {
|
||||
tmpsh[j][n][0] = FLOAT_TYPE(0);
|
||||
}
|
||||
#else
|
||||
if ((p.fusion_flags & MAT_VEC_FUSION_FLAGS_BIAS0) != 0) {
|
||||
tmpsh[j][n][0] += FLOAT_TYPE(data_fuse0[j*p.batch_stride_d + d_offset + first_row + n]);
|
||||
|
||||
@@ -1863,10 +1863,12 @@ static webgpu_encoded_op ggml_webgpu_mul_mat_id(webgpu_context & ctx,
|
||||
// params for mul_mat_id_gather.wgsl
|
||||
std::vector<uint32_t> gather_params = {
|
||||
(uint32_t) (ggml_webgpu_tensor_misalignment(ctx, src2) / ggml_type_size(src2->type)),
|
||||
(uint32_t) (ggml_webgpu_tensor_misalignment(ctx, dst) / ggml_type_size(dst->type)),
|
||||
param_n_expert,
|
||||
param_n_expert_used,
|
||||
param_n_tokens,
|
||||
(uint32_t) (src2->nb[1] / ggml_type_size(src2->type)),
|
||||
(uint32_t) dst->ne[0],
|
||||
};
|
||||
|
||||
const size_t dst_offset = ggml_webgpu_tensor_offset(dst);
|
||||
@@ -1888,6 +1890,8 @@ static webgpu_encoded_op ggml_webgpu_mul_mat_id(webgpu_context & ctx,
|
||||
ggml_webgpu_tensor_binding_size(ctx, src2)),
|
||||
ggml_webgpu_make_bind_group_entry(1, ggml_webgpu_tensor_buf(dst), ids_cache_align_offset, ids_cache_bind_size),
|
||||
ggml_webgpu_make_bind_group_entry(2, ggml_webgpu_tensor_buf(dst), ids_tiles_align_offset, ids_tiles_bind_size),
|
||||
ggml_webgpu_make_bind_group_entry(3, ggml_webgpu_tensor_buf(dst), ggml_webgpu_tensor_align_offset(ctx, dst),
|
||||
ggml_webgpu_tensor_binding_size(ctx, dst)),
|
||||
};
|
||||
|
||||
dispatches.push_back({
|
||||
|
||||
@@ -46,12 +46,24 @@ fn main(@builtin(workgroup_id) wg_id: vec3<u32>,
|
||||
let i2 = wg_linear / params.ne1;
|
||||
let i1 = wg_linear % params.ne1;
|
||||
|
||||
let i11 = u32(ids[params.offset_ids + i1 * params.nb20 + i2 * params.nb21]);
|
||||
let i11 = i32(ids[params.offset_ids + i1 * params.nb20 + i2 * params.nb21]);
|
||||
|
||||
let src0_row = params.offset_src0 + i1 * params.nb01 + i2 * params.nb02;
|
||||
let src1_row = params.offset_src1 + i11 * params.nb11;
|
||||
let dst_row = params.offset_dst + i1 * params.ne0 + i2 * (params.ne0 * params.ne1);
|
||||
|
||||
if (i11 == -1) { // skipped slot, add nothing
|
||||
#ifdef INPLACE
|
||||
// src0 already holds the result
|
||||
#else
|
||||
for (var i = thread_id;i < params.ne0; i += WG_SIZE) {
|
||||
dst[dst_row + i] = src0[src0_row + i];
|
||||
}
|
||||
#endif
|
||||
return;
|
||||
}
|
||||
|
||||
let src1_row = params.offset_src1 + u32(i11) * params.nb11;
|
||||
|
||||
for (var i = thread_id;i < params.ne0; i += WG_SIZE) {
|
||||
#ifdef INPLACE
|
||||
src0[src0_row + i] = src0[src0_row + i] + src1[src1_row + i];
|
||||
|
||||
@@ -2,19 +2,24 @@ enable f16;
|
||||
|
||||
struct MulMatIdGatherParams {
|
||||
offset_ids: u32,
|
||||
offset_dst: u32,
|
||||
|
||||
n_expert: u32,
|
||||
n_expert_used: u32,
|
||||
n_tokens: u32,
|
||||
|
||||
stride_ids_1: u32,
|
||||
|
||||
m: u32,
|
||||
};
|
||||
|
||||
@group(0) @binding(0) var<storage, read_write> ids: array<i32>; // [n_expert_used, n_tokens]
|
||||
@group(0) @binding(1) var<storage, read_write> ids_slot_pos: array<u32>; // [n_expert_used * n_tokens]
|
||||
@group(0) @binding(2) var<storage, read_write> ids_tiles: array<u32>; // [n_tiles, (expert, first slot, n slots) * n_tiles]
|
||||
|
||||
@group(0) @binding(3) var<uniform> params: MulMatIdGatherParams;
|
||||
@group(0) @binding(3) var<storage, read_write> dst: array<f32>; // [m, n_expert_used, n_tokens]
|
||||
|
||||
@group(0) @binding(4) var<uniform> params: MulMatIdGatherParams;
|
||||
|
||||
// per expert: the slot count, replaced by the first slot of the expert in ids_slot_pos by the prefix sum, then advanced as its write cursor
|
||||
var<workgroup> ids_count: array<atomic<u32>, N_EXPERT>;
|
||||
@@ -36,8 +41,10 @@ fn main(@builtin(local_invocation_id) local_id: vec3<u32>) {
|
||||
for (var i = thread_id; i < ids_slots_num; i += WG_SIZE) {
|
||||
let row = i / params.n_expert_used;
|
||||
let col = i % params.n_expert_used;
|
||||
let expert = u32(ids[params.offset_ids + row * params.stride_ids_1 + col]);
|
||||
atomicAdd(&ids_count[expert], 1u);
|
||||
let expert = ids[params.offset_ids + row * params.stride_ids_1 + col];
|
||||
if (expert != -1) { // skipped slot, owned by no expert
|
||||
atomicAdd(&ids_count[u32(expert)], 1u);
|
||||
}
|
||||
}
|
||||
|
||||
workgroupBarrier();
|
||||
@@ -71,8 +78,23 @@ fn main(@builtin(local_invocation_id) local_id: vec3<u32>) {
|
||||
for (var i = thread_id; i < ids_slots_num; i += WG_SIZE) {
|
||||
let row = i / params.n_expert_used;
|
||||
let col = i % params.n_expert_used;
|
||||
let expert = u32(ids[params.offset_ids + row * params.stride_ids_1 + col]);
|
||||
let pos = atomicAdd(&ids_count[expert], 1u);
|
||||
ids_slot_pos[pos] = i;
|
||||
let expert = ids[params.offset_ids + row * params.stride_ids_1 + col];
|
||||
if (expert != -1) {
|
||||
let pos = atomicAdd(&ids_count[u32(expert)], 1u);
|
||||
ids_slot_pos[pos] = i;
|
||||
}
|
||||
}
|
||||
|
||||
// the matmul writes only the rows that an expert owns, so zero the rows of the skipped slots
|
||||
for (var i: u32 = 0; i < ids_slots_num; i += 1) {
|
||||
let row = i / params.n_expert_used;
|
||||
let col = i % params.n_expert_used;
|
||||
if (ids[params.offset_ids + row * params.stride_ids_1 + col] != -1) {
|
||||
continue;
|
||||
}
|
||||
let dst_row = params.offset_dst + i * params.m;
|
||||
for (var r = thread_id; r < params.m; r += WG_SIZE) {
|
||||
dst[dst_row + r] = 0.0f;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -70,11 +70,18 @@ fn main(
|
||||
let own_expert = ids[params.offset_ids + slot];
|
||||
let row_base = (wg_linear % output_groups) * OUTPUTS_PER_WG;
|
||||
|
||||
let src0_batch_offset = params.offset_src0 + own_expert * params.stride_02;
|
||||
// a skipped slot (id -1) keeps the zero accumulator, so the reduction below writes zeros
|
||||
// no early return: the barriers below must stay in uniform control flow
|
||||
let skipped = i32(own_expert) == -1;
|
||||
|
||||
let src0_batch_offset = params.offset_src0 + select(own_expert, 0u, skipped) * params.stride_02;
|
||||
let src1_idx_base = params.offset_src1 + (slot % params.b_ne1) * params.stride_11;
|
||||
let dst_idx_base = params.offset_dst + slot * params.m + row_base;
|
||||
|
||||
let acc = accumulate_vec_dot(thread_id, row_base, src0_batch_offset, src1_idx_base);
|
||||
var acc: array<array<f32, OUTPUTS_PER_WG>, NUM_COLS>;
|
||||
if (!skipped) {
|
||||
acc = accumulate_vec_dot(thread_id, row_base, src0_batch_offset, src1_idx_base);
|
||||
}
|
||||
|
||||
#ifdef USE_SUBGROUP_REDUCTION
|
||||
for (var row = 0u; row < OUTPUTS_PER_WG; row++) {
|
||||
|
||||
@@ -375,7 +375,12 @@ static void ggml_zendnn_compute_forward_mul_mat_id(
|
||||
for (int id = 0; id < n_ids; ++id) {
|
||||
const int32_t i02 = *(const int32_t *)((const char *)ids->data + iid1*ids->nb[1] + id*ids->nb[0]);
|
||||
|
||||
GGML_ASSERT(i02 >= 0 && i02 < n_as);
|
||||
GGML_ASSERT(i02 == -1 || (i02 >= 0 && i02 < n_as));
|
||||
|
||||
if (i02 == -1) {
|
||||
memset((char *) dst->data + id*nb1 + iid1*nb2, 0, ne0*sizeof(float));
|
||||
continue;
|
||||
}
|
||||
|
||||
if (matrix_row_counts[i02] == 0) {
|
||||
n_active_experts++;
|
||||
|
||||
@@ -3438,6 +3438,8 @@ void ggml_mul_mat_set_hint(
|
||||
in b, n_expert_used can be broadcasted to match the n_expert_used of ids
|
||||
|
||||
c ~= as[:,:,i] @ b[:,i%r,t], i = ids[e,t] for all e,t in ids
|
||||
|
||||
an id of -1 skips the slot: no matrix is selected, c[:,e,t] is set to zero
|
||||
*/
|
||||
struct ggml_tensor * ggml_mul_mat_id(
|
||||
struct ggml_context * ctx,
|
||||
|
||||
+92
-19
@@ -1194,6 +1194,29 @@ static bool op_filter_entry_matches(std::string_view entry, std::string_view op_
|
||||
return std::regex_search(op_name.data(), op_name.data() + op_name.size(), re);
|
||||
}
|
||||
|
||||
// TODO: [TAG_MUL_MAT_ID_SKIP] see https://github.com/ggml-org/llama.cpp/pull/26631
|
||||
static bool backend_lacks_skipped_ids(ggml_backend_t backend, ggml_cgraph * gf) {
|
||||
const char * name = ggml_backend_reg_name(ggml_backend_dev_backend_reg(ggml_backend_get_device(backend)));
|
||||
if (strcmp(name, "CUDA") != 0 && strcmp(name, "ROCm") != 0 && strcmp(name, "MUSA") != 0) {
|
||||
return false;
|
||||
}
|
||||
for (int i = 0; i < ggml_graph_n_nodes(gf); i++) {
|
||||
const ggml_tensor * node = ggml_graph_node(gf, i);
|
||||
if (node->op != GGML_OP_MUL_MAT_ID && node->op != GGML_OP_ADD_ID) {
|
||||
continue;
|
||||
}
|
||||
const ggml_tensor * ids = node->src[2];
|
||||
std::vector<int32_t> row(ids->ne[0]);
|
||||
for (int64_t i1 = 0; i1 < ids->ne[1]; i1++) {
|
||||
ggml_backend_tensor_get(ids, row.data(), i1*ids->nb[1], ids->ne[0]*sizeof(int32_t));
|
||||
if (std::find(row.begin(), row.end(), -1) != row.end()) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
struct test_case {
|
||||
virtual ~test_case() {}
|
||||
|
||||
@@ -1475,6 +1498,13 @@ struct test_case {
|
||||
initialize_tensors(ctx_weights.get());
|
||||
}
|
||||
|
||||
// TODO: [TAG_MUL_MAT_ID_SKIP] @ngxson : remove this once CUDA implements mul_mat_id with -1 = skipped
|
||||
if (backend_lacks_skipped_ids(backend1, gf)) {
|
||||
test_result result(ggml_backend_name(backend1), current_op_name, vars(), "test", false, false, "not supported");
|
||||
print_test_result_locked(output_printer, result);
|
||||
return test_status_t::NOT_SUPPORTED;
|
||||
}
|
||||
|
||||
// compare
|
||||
struct callback_userdata {
|
||||
bool ok;
|
||||
@@ -3564,6 +3594,16 @@ struct test_bin_bcast : public test_case {
|
||||
}
|
||||
};
|
||||
|
||||
// set some ids to -1, so that MUL_MAT_ID and ADD_ID skip those slots
|
||||
// every 4th row skips all of its slots, the other rows skip only some of them
|
||||
static void mask_skipped_ids(std::vector<int32_t> & ids, int64_t row) {
|
||||
for (size_t i = 0; i < ids.size(); i++) {
|
||||
if (row % 4 == 1 || (row + (int64_t) i) % 3 == 0) {
|
||||
ids[i] = -1;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// GGML_OP_ADD_ID
|
||||
struct test_add_id : public test_case {
|
||||
const ggml_type type_a;
|
||||
@@ -3572,9 +3612,10 @@ struct test_add_id : public test_case {
|
||||
const int64_t n_experts;
|
||||
const int64_t n_experts_used;
|
||||
const int64_t n_token;
|
||||
const bool skip; // set some ids to a negative value
|
||||
|
||||
std::string vars() override {
|
||||
return VARS_TO_STR6(type_a, type_b, n_embd, n_experts, n_experts_used, n_token);
|
||||
return VARS_TO_STR7(type_a, type_b, n_embd, n_experts, n_experts_used, n_token, skip);
|
||||
}
|
||||
|
||||
size_t op_size(ggml_tensor * t) override {
|
||||
@@ -3586,9 +3627,10 @@ struct test_add_id : public test_case {
|
||||
int64_t n_embd = 128,
|
||||
int64_t n_experts = 16,
|
||||
int64_t n_experts_used = 8,
|
||||
int64_t n_token = 10)
|
||||
int64_t n_token = 10,
|
||||
bool skip = false)
|
||||
: type_a(type_a), type_b(type_b), n_embd(n_embd),
|
||||
n_experts(n_experts), n_experts_used(n_experts_used), n_token(n_token) {}
|
||||
n_experts(n_experts), n_experts_used(n_experts_used), n_token(n_token), skip(skip) {}
|
||||
|
||||
ggml_tensor * build_graph(ggml_context * ctx) override {
|
||||
ggml_tensor * a = ggml_new_tensor_3d(ctx, type_a, n_embd, n_experts_used, n_token);
|
||||
@@ -3617,6 +3659,9 @@ struct test_add_id : public test_case {
|
||||
data[i] = i % n_experts;
|
||||
}
|
||||
std::shuffle(data.begin(), data.end(), rng);
|
||||
if (skip) {
|
||||
mask_skipped_ids(data, r);
|
||||
}
|
||||
ggml_backend_tensor_set(t, data.data(), r * t->nb[1], t->ne[0] * sizeof(int32_t));
|
||||
}
|
||||
} else {
|
||||
@@ -5550,7 +5595,8 @@ struct test_mul_mat_w4a4 : public test_mul_mat {
|
||||
}
|
||||
};
|
||||
|
||||
static void init_mul_mat_id_ids(ggml_context * ctx, int n_mats) {
|
||||
static void init_mul_mat_id_ids(ggml_context * ctx, int n_mats, bool skip = false) {
|
||||
|
||||
std::random_device rd;
|
||||
std::default_random_engine rng(rd());
|
||||
for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != NULL; t = ggml_get_next_tensor(ctx, t)) {
|
||||
@@ -5563,12 +5609,15 @@ static void init_mul_mat_id_ids(ggml_context * ctx, int n_mats) {
|
||||
data[i] = i % n_mats;
|
||||
}
|
||||
std::shuffle(data.begin(), data.end(), rng);
|
||||
if (skip) {
|
||||
mask_skipped_ids(data, r);
|
||||
}
|
||||
ggml_backend_tensor_set(t, data.data(), r * t->nb[1], t->ne[0] * sizeof(int32_t));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
static void init_mul_mat_id_tensors(ggml_context * ctx, int n_mats, float amax = 1.0f) {
|
||||
static void init_mul_mat_id_tensors(ggml_context * ctx, int n_mats, bool skip = false, float amax = 1.0f) {
|
||||
for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != NULL; t = ggml_get_next_tensor(ctx, t)) {
|
||||
if (t->type == GGML_TYPE_I32) {
|
||||
continue;
|
||||
@@ -5579,7 +5628,7 @@ static void init_mul_mat_id_tensors(ggml_context * ctx, int n_mats, float amax =
|
||||
init_tensor_uniform(t);
|
||||
}
|
||||
}
|
||||
init_mul_mat_id_ids(ctx, n_mats);
|
||||
init_mul_mat_id_ids(ctx, n_mats, skip);
|
||||
}
|
||||
|
||||
// GGML_OP_MUL_MAT_ID
|
||||
@@ -5595,9 +5644,10 @@ struct test_mul_mat_id : public test_case {
|
||||
const float amax; // magnitude of src1
|
||||
const int64_t m_v; // rows of as in memory, the experts of as are strided for m_v > m, no view for m_v == 0
|
||||
const bool ids_offset;
|
||||
const bool skip; // set some ids to a negative value
|
||||
|
||||
std::string vars() override {
|
||||
return VARS_TO_STR11(type_a, type_b, n_mats, n_used, b, m, n, k, amax, m_v, ids_offset);
|
||||
return VARS_TO_STR12(type_a, type_b, n_mats, n_used, b, m, n, k, amax, m_v, ids_offset, skip);
|
||||
}
|
||||
|
||||
double max_nmse_err() override {
|
||||
@@ -5622,9 +5672,9 @@ struct test_mul_mat_id : public test_case {
|
||||
test_mul_mat_id(ggml_type type_a = GGML_TYPE_F32, ggml_type type_b = GGML_TYPE_F32,
|
||||
int n_mats = 8, int n_used = 2, bool b = false,
|
||||
int64_t m = 32, int64_t n = 32, int64_t k = 32,
|
||||
float amax = 1.0f, int64_t m_v = 0, bool ids_offset = false)
|
||||
float amax = 1.0f, int64_t m_v = 0, bool ids_offset = false, bool skip = false)
|
||||
: type_a(type_a), type_b(type_b), n_mats(n_mats), n_used(n_used), b(b),
|
||||
m(m), n(n), k(k), amax(amax), m_v(m_v), ids_offset(ids_offset) {
|
||||
m(m), n(n), k(k), amax(amax), m_v(m_v), ids_offset(ids_offset), skip(skip) {
|
||||
GGML_ASSERT(n_used <= n_mats);
|
||||
GGML_ASSERT(m_v == 0 || m_v > m);
|
||||
}
|
||||
@@ -5659,11 +5709,11 @@ struct test_mul_mat_id : public test_case {
|
||||
}
|
||||
|
||||
void initialize_tensors(ggml_context * ctx) override {
|
||||
init_mul_mat_id_tensors(ctx, n_mats, amax);
|
||||
init_mul_mat_id_tensors(ctx, n_mats, skip, amax);
|
||||
}
|
||||
|
||||
void reinit_perf_iter(ggml_context * ctx) override {
|
||||
init_mul_mat_id_ids(ctx, n_mats);
|
||||
init_mul_mat_id_ids(ctx, n_mats, skip);
|
||||
}
|
||||
};
|
||||
|
||||
@@ -5738,9 +5788,10 @@ struct test_mul_mat_id_fusion : public test_case {
|
||||
const int64_t k;
|
||||
const uint32_t o; // number of outputs
|
||||
const bool mul;
|
||||
const bool skip; // set some ids to a negative value
|
||||
|
||||
std::string vars() override {
|
||||
return VARS_TO_STR10(type_a, type_b, n_mats, n_used, b, m, n, k, o, mul);
|
||||
return VARS_TO_STR11(type_a, type_b, n_mats, n_used, b, m, n, k, o, mul, skip);
|
||||
}
|
||||
|
||||
double max_nmse_err() override {
|
||||
@@ -5754,9 +5805,9 @@ struct test_mul_mat_id_fusion : public test_case {
|
||||
|
||||
test_mul_mat_id_fusion(ggml_type type_a = GGML_TYPE_F32, ggml_type type_b = GGML_TYPE_F32,
|
||||
int n_mats = 8, int n_used = 2, bool b = false,
|
||||
int64_t m = 32, int64_t n = 32, int64_t k = 32, uint32_t o = 1, bool mul = false)
|
||||
int64_t m = 32, int64_t n = 32, int64_t k = 32, uint32_t o = 1, bool mul = false, bool skip = false)
|
||||
: type_a(type_a), type_b(type_b), n_mats(n_mats), n_used(n_used), b(b),
|
||||
m(m), n(n), k(k), o(o), mul(mul) {
|
||||
m(m), n(n), k(k), o(o), mul(mul), skip(skip) {
|
||||
GGML_ASSERT(n_used <= n_mats);
|
||||
}
|
||||
|
||||
@@ -5796,7 +5847,7 @@ struct test_mul_mat_id_fusion : public test_case {
|
||||
}
|
||||
|
||||
void initialize_tensors(ggml_context * ctx) override {
|
||||
init_mul_mat_id_tensors(ctx, n_mats);
|
||||
init_mul_mat_id_tensors(ctx, n_mats, skip);
|
||||
}
|
||||
|
||||
bool run_whole_graph() override { return true; }
|
||||
@@ -7754,19 +7805,20 @@ struct test_mul_mat_vec_fusion : public test_case {
|
||||
const bool with_gate;
|
||||
const bool with_lane_scale;
|
||||
std::array<int64_t, 2> batch_dims;
|
||||
const bool skip; // set some ids to a negative value (only for use_id)
|
||||
|
||||
test_mul_mat_vec_fusion(ggml_type type, ggml_glu_op op, int64_t m, int64_t n, int64_t k,
|
||||
bool use_id = false, int n_mats = 1, int n_used = 1, bool b = false, bool with_bias = false, bool with_gate = true,
|
||||
bool with_lane_scale = false, std::array<int64_t, 2> batch_dims = {4, 2})
|
||||
bool with_lane_scale = false, std::array<int64_t, 2> batch_dims = {4, 2}, bool skip = false)
|
||||
: type(type), glu_op(op), m(m), n(n), k(k), use_id(use_id), n_mats(n_mats), n_used(n_used), b(b), with_bias(with_bias),
|
||||
with_gate(with_gate), with_lane_scale(with_lane_scale), batch_dims(batch_dims) {
|
||||
with_gate(with_gate), with_lane_scale(with_lane_scale), batch_dims(batch_dims), skip(skip) {
|
||||
if (use_id) {
|
||||
GGML_ASSERT(n_used <= n_mats);
|
||||
}
|
||||
}
|
||||
|
||||
std::string vars() override {
|
||||
return VARS_TO_STR13(type, glu_op, m, n, k, use_id, n_mats, n_used, b, with_bias, with_gate, with_lane_scale, batch_dims);
|
||||
return VARS_TO_STR14(type, glu_op, m, n, k, use_id, n_mats, n_used, b, with_bias, with_gate, with_lane_scale, batch_dims, skip);
|
||||
}
|
||||
|
||||
std::string op_desc(ggml_tensor * t) override {
|
||||
@@ -7917,7 +7969,7 @@ struct test_mul_mat_vec_fusion : public test_case {
|
||||
init_tensor_uniform(t);
|
||||
}
|
||||
} else {
|
||||
init_mul_mat_id_tensors(ctx, n_mats);
|
||||
init_mul_mat_id_tensors(ctx, n_mats, skip);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -11453,6 +11505,21 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() {
|
||||
}
|
||||
}
|
||||
|
||||
// skipped slots (negative ids)
|
||||
for (ggml_type type_a : {GGML_TYPE_F32, GGML_TYPE_F16, GGML_TYPE_BF16, GGML_TYPE_Q4_0, GGML_TYPE_Q4_K, GGML_TYPE_Q8_0, GGML_TYPE_MXFP4}) {
|
||||
for (int n_used : {1, 2, 4}) {
|
||||
for (bool b : {false, true}) {
|
||||
for (int n : {1, 4, 33, 129}) {
|
||||
test_cases.emplace_back(new test_mul_mat_id(type_a, GGML_TYPE_F32, 8, n_used, b, 512, n, 256, 1.0f, 0, false, true));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
// n_used == n_mats, so every expert can be skipped
|
||||
test_cases.emplace_back(new test_mul_mat_id(GGML_TYPE_F16, GGML_TYPE_F32, 4, 4, false, 512, 64, 256, 1.0f, 0, false, true));
|
||||
test_cases.emplace_back(new test_mul_mat_id(GGML_TYPE_F16, GGML_TYPE_F32, 1, 1, false, 8, 16, 1, 1.0f, 0, false, true));
|
||||
test_cases.emplace_back(new test_mul_mat_id_fusion(GGML_TYPE_F16, GGML_TYPE_F32, 16, 16, false, 32, 32, 32, 3, false, true));
|
||||
|
||||
for (int bs : {1, 4, 512}) {
|
||||
for (ggml_type type_a : {GGML_TYPE_F32, GGML_TYPE_F16, GGML_TYPE_Q4_0, GGML_TYPE_Q4_K}) {
|
||||
for (ggml_type type_b : {GGML_TYPE_F32}) {
|
||||
@@ -11500,6 +11567,7 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() {
|
||||
for (int n_embd : {32, 129}) {
|
||||
for (int n_token : {1, 32, 129}) {
|
||||
test_cases.emplace_back(new test_add_id(type_a, type_b, n_embd, n_mats, n_used, n_token));
|
||||
test_cases.emplace_back(new test_add_id(type_a, type_b, n_embd, n_mats, n_used, n_token, true));
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -12238,6 +12306,11 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() {
|
||||
test_cases.emplace_back(new test_mul_mat_vec_fusion(type, glu_op, m_batch, 32, 256,
|
||||
use_id, 16, 8, b, with_bias, with_gate, with_lane_scale, {1, 1}));
|
||||
}
|
||||
// the lane scale does a GET_ROWS on the ids, which does not accept -1
|
||||
if (use_id && !with_lane_scale) {
|
||||
test_cases.emplace_back(new test_mul_mat_vec_fusion(type, glu_op, 8, 32, 256,
|
||||
use_id, 16, 8, b, with_bias, with_gate, with_lane_scale, {1, 1}, true));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user