metal : fix mul_mat_id with duplicate ids (#30373)

* metal : fix mul_mat_id limits and kernel name

* metal : fix mul_mat_id with duplicate ids

kernel_mul_mm_id_map0 assumed at most one match per token and a fixed
capacity of ne21 entries per expert, which produced wrong or dropped
output rows and could overflow into the slice of the next expert when
the ids contain duplicates.

build a compact per-expert list of the (token, slot) entries instead:

- pass 1 counts the entries of every expert, thread 0 computes the
  offsets of the expert slices (tpe now holds [base, count])
- pass 2 writes the entries with one unconditional store per token
  (conditional stores of divergent lanes are very slow), the dummy
  value of the tokens without a match lands on a padding slot of the
  slice that is never read - duplicates are handled by a rare branch
- kernel_mul_mm_id reads the base/count of its expert and loops over
  the entry tiles that do not fit into the grid

The ids buffer is sized as ne20*ne21 + ne02 (entries + padding).

883/883 test-backend-ops MUL_MAT_ID tests pass, llama-bench is within
run-to-run variance of the previous behavior.

Assisted-by: pi:llama.cpp/MiMo-V2.6-Flash-MOPD

* metal : make mul_mat_id map0 ne02 a function constant

use FOR_UNROLL for the id loops of kernel_mul_mm_id_map0 and turn ne02
into the function constant FC_mul_mm_id_map0_ne02 (FC_MUL_MM + 7), so
that the threadgroup offsets of cnt/bas/sids and the prefix-sum bound
fold at compile time.

the pipeline name now encodes ne02 (same convention as the ssm_conv
nc/silu constants) and the map0 pipeline is compiled with the constant
value set. ne02 is no longer part of ggml_metal_kargs_mul_mm_id_map0 -
the dispatch grid still comes from the tensor ne[2] on the host.

no measurable perf change: 883/883 test-backend-ops MUL_MAT_ID tests
pass, op-level and llama-bench results are within run variance.

Assisted-by: pi:llama.cpp/MiMo-V2.6-Flash-MOPD

* cont : clean-up + TODO

* metal : dispatch mul_mat_id entry tiles from the prepass

follow the approach of the vulkan implementation (#29998): instead of
launching a grid over experts x ne21 entry tiles and letting most of
the threadgroups early exit, kernel_mul_mm_id_map0 now also emits a
row map - the total tile count plus one (expert, first entry, valid
rows) descriptor per 32-entry tile that has work.

kernel_mul_mm_id is launched with a static upper bound on the number
of tiles (full tiles over all entries plus at most one partial tile
per expert), reads its descriptor and processes a single tile - the
tail early exits. This removes the round loop that was needed to cover
duplicate ids past the grid (register pressure: th_max 768 -> back to
the plain mul_mm level) and duplicates are now handled implicitly,
since they only produce more descriptors within the bound.

The grid shrinks by ~8-16x on typical shapes (e.g. 128 experts,
8 used, 512 tokens: 128 tiles instead of 2048 entry tiles, of which
94% only launched to exit) and ne21 is no longer needed by the mm
kernel kargs.

883/883 test-backend-ops MUL_MAT_ID tests pass. llama-bench
(Qwen3.6-35B-A3B Q8_0) is now faster than master on all pp sizes
(pp2048/4096/8192 +2%, pp512 +0.5%), tg32 unchanged.

Assisted-by: pi:llama.cpp/MiMo-V2.6-Flash-MOPD

* cont : add smem assert
This commit is contained in:
Georgi Gerganov authored and GitHub committed 2026-10-12 13:31:41 +03:00
1 parent 77757a3467
commit 073b74034b
5 files changed
+184 -61

No files matched your search

+2 -1
View File
@@ -43,9 +43,10 @@ bool ggml_metal_op_mul_mat_use_mm(const struct ggml_tensor * op, bool has_simdgr
bool ggml_metal_op_mul_mat_id_use_mm(const struct ggml_tensor * op, bool has_simdgroup_mm) {
const int64_t ne00 = op->src[0]->ne[0];
const int64_t ne02 = op->src[0]->ne[2];
const int64_t ne21 = op->src[2]->ne[1];
return has_simdgroup_mm && ne00 >= 64 && ne21 >= 32;
return has_simdgroup_mm && ne00 >= 64 && ne02 <= 1024 && ne21 >= 32;
}
// the most src1 rows of the few-row MMA kernels
+8 -2
View File
@@ -1268,10 +1268,16 @@ ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_mul_mm_id_map0(g
ggml_metal_pipeline_with_params res = ggml_metal_library_get_pipeline(lib, name);
if (!res.pipeline) {
res = ggml_metal_library_compile_pipeline(lib, base, name, nullptr);
ggml_metal_cv_t cv = ggml_metal_cv_init();
ggml_metal_cv_set_int32(cv, ne02, FC_MUL_MM + 7);
res = ggml_metal_library_compile_pipeline(lib, base, name, cv);
ggml_metal_cv_free(cv);
}
res.smem = (size_t) ne02*ne20*sizeof(uint16_t);
// per expert: entry count + base offset, followed by the staging of the ids of ntg tokens
res.smem = 2*(size_t) ne02*sizeof(uint32_t) + (size_t) ne02*ne20*sizeof(uint16_t);
res.smem = GGML_PAD(res.smem, 16);
return res;
-2
View File
@@ -560,7 +560,6 @@ typedef struct {
} ggml_metal_kargs_mul_mv_ext;
typedef struct {
int32_t ne02;
int32_t ne10;
int32_t ne11; // n_expert_used (bcast)
uint64_t nb11;
@@ -590,7 +589,6 @@ typedef struct {
uint64_t nb12;
uint64_t nb13;
int32_t ne20;
int32_t ne21;
int32_t ne0;
int32_t ne1;
int16_t r2;
+26 -6
View File
@@ -2772,21 +2772,39 @@ int ggml_metal_op_mul_mat(ggml_metal_op_t ctx, int idx) {
return 1;
}
// upper bound on the number of entry tiles emitted by kernel_mul_mm_id_map0: the full tiles
// over all entries plus at most one partial tile per expert
static int64_t ggml_metal_op_mul_mat_id_max_tiles(const ggml_tensor * op) {
const int64_t ne02 = op->src[0]->ne[2]; // n_expert
const int64_t ne20 = op->src[2]->ne[0]; // n_expert_used
const int64_t ne21 = op->src[2]->ne[1]; // n_token
const int64_t n_rows = ne20*ne21;
const int64_t tiles = (n_rows + 31)/32 + (ne02 < n_rows ? ne02 : n_rows) - 1;
return n_rows < tiles ? n_rows : tiles;
}
size_t ggml_metal_op_mul_mat_id_extra_tpe(const ggml_tensor * op) {
assert(op->op == GGML_OP_MUL_MAT_ID);
const int64_t ne02 = op->src[0]->ne[2]; // n_expert
// row map of kernel_mul_mm_id: total tile count + one
// (expert, first entry, valid rows) descriptor per tile
const int64_t max_tiles = ggml_metal_op_mul_mat_id_max_tiles(op);
return ggml_type_size(GGML_TYPE_I32)*ne02;
return ggml_type_size(GGML_TYPE_I32)*(1 + 3*max_tiles);
}
size_t ggml_metal_op_mul_mat_id_extra_ids(const ggml_tensor * op) {
assert(op->op == GGML_OP_MUL_MAT_ID);
const int64_t ne02 = op->src[0]->ne[2]; // n_expert
const int64_t ne20 = op->src[2]->ne[0]; // n_expert_used
const int64_t ne21 = op->src[2]->ne[1]; // n_token
return ggml_type_size(GGML_TYPE_I32)*ne02*ne21;
// one entry per (expert used, token) pair of ids plus one padding slot per expert
// (see kernel_mul_mm_id_map0)
return ggml_type_size(GGML_TYPE_I32)*(ne20*ne21 + ne02);
}
size_t ggml_metal_op_mul_mat_id_extra_amax(const ggml_tensor * op) {
@@ -2884,7 +2902,6 @@ int ggml_metal_op_mul_mat_id(ggml_metal_op_t ctx, int idx) {
{
ggml_metal_kargs_mul_mm_id_map0 args = {
ne02,
ne10,
ne11, // n_expert_used (bcast)
nb11,
@@ -2942,7 +2959,6 @@ int ggml_metal_op_mul_mat_id(ggml_metal_op_t ctx, int idx) {
/*.nb12 =*/ nb12,
/*.nb13 =*/ nb13,
/*.ne20 =*/ ne20, // n_expert_used
/*.ne21 =*/ ne21, // n_tokens
/*.ne0 =*/ ne0,
/*.ne1 =*/ ne1,
/*.r2 =*/ r2,
@@ -2960,9 +2976,13 @@ int ggml_metal_op_mul_mat_id(ggml_metal_op_t ctx, int idx) {
const size_t smem = pipeline.smem;
GGML_ASSERT(smem <= props_dev->max_theadgroup_memory_size);
ggml_metal_encoder_set_threadgroup_memory_size(enc, smem, 0);
ggml_metal_encoder_dispatch_threadgroups(enc, (ne21 + 31)/32, (ne01 + 63)/64, ne02, 128, 1, 1);
const int max_tiles = (int) ggml_metal_op_mul_mat_id_max_tiles(op);
ggml_metal_encoder_dispatch_threadgroups(enc, max_tiles, (ne01 + 63)/64, 1, 128, 1, 1);
}
} else {
auto pipeline = ggml_metal_library_get_pipeline_mul_mv_id(lib, op);
+148 -50
View File
@@ -8,6 +8,7 @@ constant short FC_mul_mm_ne13 [[function_constant(FC_MUL_MM + 3)]];
constant short FC_mul_mm_r2 [[function_constant(FC_MUL_MM + 4)]];
constant short FC_mul_mm_r3 [[function_constant(FC_MUL_MM + 5)]];
constant bool FC_mul_mm_id_amax [[function_constant(FC_MUL_MM + 6)]];
constant int32_t FC_mul_mm_id_map0_ne02 [[function_constant(FC_MUL_MM + 7)]];
// each block_q contains 16*nl weights
#ifdef GGML_METAL_HAS_TENSOR
@@ -371,19 +372,23 @@ kernel void kernel_mul_mm_id_map0(
ushort ntg[[threads_per_threadgroup]]) {
const short ide = tpitg; // expert id
uint32_t n_all = 0;
// per expert: entry count (pass 1), replaced by the base offset of the slice (pass 2)
threadgroup uint32_t * cnt = (threadgroup uint32_t *) (shmem);
threadgroup uint32_t * bas = cnt + FC_mul_mm_id_map0_ne02;
device int32_t * ids_i32 = (device int32_t *) hids + ide*args.ne21;
// staging of the ids of ntg tokens
threadgroup uint16_t * sids = (threadgroup uint16_t *) (bas + FC_mul_mm_id_map0_ne02);
// pass 1: count the entries of this expert - every occurrence of this expert within the
// ids is an entry (duplicate ids produce multiple entries for the same token)
uint32_t n_all = 0;
for (int i21 = 0; i21 < args.ne21; i21 += ntg) { // n_tokens
if (i21 + tpitg < args.ne21) {
device const int32_t * src2_i32 = (device const int32_t *) (src2 + (i21 + tpitg)*args.nb21);
threadgroup uint16_t * sids = (threadgroup uint16_t *) shmem + tpitg*ne20;
#pragma unroll(ne20)
for (short i20 = 0; i20 < ne20; i20++) {
sids[i20] = src2_i32[i20];
FOR_UNROLL (short i20 = 0; i20 < ne20; i20++) {
sids[tpitg*ne20 + i20] = src2_i32[i20];
}
}
@@ -394,24 +399,107 @@ kernel void kernel_mul_mm_id_map0(
break;
}
threadgroup const uint16_t * sids = (threadgroup const uint16_t *) shmem + t*ne20;
threadgroup const uint16_t * sids_t = sids + t*ne20;
short sel = 0;
#pragma unroll(ne20)
for (short i20 = 0; i20 < ne20; i20++) {
sel += (sids[i20] == ide)*(i20 + 1);
FOR_UNROLL (short i20 = 0; i20 < ne20; i20++) {
n_all += sids_t[i20] == ide;
}
ids_i32[n_all] = (i21 + t)*ne20 + sel - 1;
n_all += sel > 0;
}
threadgroup_barrier(mem_flags::mem_threadgroup);
}
device uint32_t * tpe_u32 = (device uint32_t *) (htpe);
tpe_u32[ide] = n_all;
cnt[ide] = n_all;
threadgroup_barrier(mem_flags::mem_threadgroup);
// exclusive prefix sum over the counts: the entries of this expert start at bas[ide] -
// every slice is padded by one entry for the uniform dummy store of pass 2 - and emit the
// entry tile descriptors of kernel_mul_mm_id: (expert, first entry, valid rows) per tile
if (tpitg == 0) {
constexpr uint32_t NTILE = 32; // must match NR1 of kernel_mul_mm_id
uint32_t base = 0;
uint32_t tile = 0;
device uint32_t * tpe_u32 = (device uint32_t *) (htpe);
for (int e = 0; e < FC_mul_mm_id_map0_ne02; ++e) {
const uint32_t n = cnt[e];
bas[e] = base;
for (uint32_t k = 0; k < n; k += NTILE) {
tpe_u32[1 + 3*tile ] = e;
tpe_u32[1 + 3*tile + 1] = base + k;
tpe_u32[1 + 3*tile + 2] = n - k < NTILE ? n - k : NTILE;
tile++;
}
base += n + 1;
}
tpe_u32[0] = tile;
}
threadgroup_barrier(mem_flags::mem_threadgroup);
// pass 2: write the entries of this expert into its slice of the compact list
device int32_t * ids_i32 = (device int32_t *) (hids);
uint32_t ient = bas[ide];
for (int i21 = 0; i21 < args.ne21; i21 += ntg) { // n_tokens
if (i21 + tpitg < args.ne21) {
device const int32_t * src2_i32 = (device const int32_t *) (src2 + (i21 + tpitg)*args.nb21);
FOR_UNROLL (short i20 = 0; i20 < ne20; i20++) {
sids[tpitg*ne20 + i20] = src2_i32[i20];
}
}
threadgroup_barrier(mem_flags::mem_threadgroup);
for (short t = 0; t < ntg; t++) {
if (i21 + t >= args.ne21) {
break;
}
threadgroup const uint16_t * sids_t = sids + t*ne20;
short first = -1;
short n_ents = 0;
FOR_UNROLL (short i20 = 0; i20 < ne20; i20++) {
const short hit = sids_t[i20] == ide;
first = hit && first < 0 ? i20 : first;
n_ents += hit;
}
// one store per token for every thread, regardless of the match - conditional
// stores of the divergent lanes are very slow - the dummy value of the tokens
// without a match lands on the padding slot of this slice and is never read
ids_i32[ient] = first >= 0 ? (i21 + t)*ne20 + first : 0;
if (n_ents > 1) {
// duplicate ids: the remaining matches of this token
short j = 1;
FOR_UNROLL (short i20 = 0; i20 < ne20; i20++) {
if (sids_t[i20] == ide && i20 != first) {
ids_i32[ient + j] = (i21 + t)*ne20 + i20;
j += 1;
}
}
}
ient += n_ents;
}
threadgroup_barrier(mem_flags::mem_threadgroup);
}
}
kernel void kernel_mul_mm_id_amax_part_f32(
@@ -505,6 +593,7 @@ template [[host_name("kernel_mul_mm_id_map0_ne20_10")]] kernel kernel_mul_mm_id_
template [[host_name("kernel_mul_mm_id_map0_ne20_16")]] kernel kernel_mul_mm_id_map0_t kernel_mul_mm_id_map0<16>;
template [[host_name("kernel_mul_mm_id_map0_ne20_22")]] kernel kernel_mul_mm_id_map0_t kernel_mul_mm_id_map0<22>;
// TODO: refactor into two seperate functions to avoid ifdef branching: with and without tensor support
template<typename S0, typename S0_4x4, typename S0_8x8, typename S1, typename S1_2x4, typename S1_8x8, typename block_q, short nl, void (*dequantize_func)(device const block_q *, short, thread S0_4x4 &), typename T0, typename T0_4x4, typename T1, typename T1_2x4>
kernel void kernel_mul_mm_id(
constant ggml_metal_kargs_mul_mm_id & args,
@@ -533,55 +622,42 @@ kernel void kernel_mul_mm_id(
constexpr int NL0 = NK/16;
constexpr int NL1 = NK/8;
const int im = tgpig.z; // expert
const int r0 = tgpig.y*NR0;
const int r1 = tgpig.x*NR1;
const int tile = tgpig.x; // entry tile, described by the prepass
const int r0 = tgpig.y*NR0;
device const uint32_t * tpe_u32 = (device const uint32_t *) (htpe);
device const int32_t * ids_i32 = (device const int32_t *) (hids);
const int32_t neh1 = tpe_u32[im];
if (r1 >= neh1) {
// the prepass emits one (expert, first entry, valid rows) descriptor per tile that has
// work - the grid is sized by a static upper bound, so the tail early exits
if (tile >= (int) tpe_u32[0]) {
return;
}
const int im = tpe_u32[1 + 3*tile]; // expert
const int idb = tpe_u32[1 + 3*tile + 1]; // first entry of this tile in hids
// number of entries in this tile, always <= NR1
const short nr1 = (short) ((tpe_u32[1 + 3*tile + 2] < NR1) ? tpe_u32[1 + 3*tile + 2] : NR1);
// if this block is of 64x32 shape or smaller
const short nr0 = (args.ne0 - r0 < NR0) ? (args.ne0 - r0) : NR0;
const short nr1 = ( neh1 - r1 < NR1) ? ( neh1 - r1) : NR1;
// a thread shouldn't load data outside of the matrix
const short lr0 = ((short)tiitg/NL0) < nr0 ? ((short)tiitg/NL0) : nr0 - 1; // 0 .. 63
const short lr1 = ((short)tiitg/NL1) < nr1 ? ((short)tiitg/NL1) : nr1 - 1; // 0 .. 31
const short il0 = (tiitg % NL0);
short il = il0;
const int id = ids_i32[im*args.ne21 + r1 + lr1];
const short i11 = (id % args.ne20) % args.ne11;
const short i12 = (id / args.ne20);
const short i13 = 0;
const uint64_t offset0 = im*args.nb02 + i13*args.nb03;
const short offset1 = il0/nl;
device const block_q * x = (device const block_q *)(src0 + args.nb01*(r0 + lr0) + offset0) + offset1;
const short iy = 8*(tiitg % NL1);
device const T1 * y = (device const T1 *)(src1
+ args.nb13*i13
+ args.nb12*i12
+ args.nb11*i11
+ args.nb10*iy);
// skip the upper half of the token tile when the expert did not fill it
constexpr short NR1H = NR1/2;
const bool has_hi = nr1 > NR1H;
// power-of-two rescaling
const float s1_inv = FC_mul_mm_id_amax ? ((device const float *) amax)[0] : 1.0f;
const float s1_scale = FC_mul_mm_id_amax ? ((device const float *) amax)[1] : 1.0f;
@@ -591,13 +667,6 @@ kernel void kernel_mul_mm_id(
S1_8x8 mb[2];
simdgroup_float8x8 mc[8];
for (short i = 0; i < 8; i++){
mc[i] = make_filled_simdgroup_matrix<float, 8>(0.f);
}
// simdgroups 2,3 own rows NR1H..NR1-1
const bool sg_active = has_hi || sgitg < 2;
#else
auto tA = tensor<threadgroup S0, dextents<int32_t, 2>, tensor_inline>(sa, dextents<int32_t, 2>(NK, NR0));
@@ -609,6 +678,35 @@ kernel void kernel_mul_mm_id(
mpp::tensor_ops::matmul2d_descriptor(NR1H, NR0, NK, false, true, false, mpp::tensor_ops::matmul2d_descriptor::mode::multiply_accumulate),
execution_simdgroups<4>> mm;
#endif
const short lr1 = ((short)tiitg/NL1) < nr1 ? ((short)tiitg/NL1) : nr1 - 1; // 0 .. 31
short il = il0;
const int id = ids_i32[idb + lr1];
const short i11 = (id % args.ne20) % args.ne11;
const short i12 = (id / args.ne20);
device const block_q * x = (device const block_q *)(src0 + args.nb01*(r0 + lr0) + offset0) + offset1;
device const T1 * y = (device const T1 *)(src1
+ args.nb13*i13
+ args.nb12*i12
+ args.nb11*i11
+ args.nb10*iy);
const bool has_hi = nr1 > NR1H;
#ifndef GGML_METAL_HAS_TENSOR
for (short i = 0; i < 8; i++){
mc[i] = make_filled_simdgroup_matrix<float, 8>(0.f);
}
// simdgroups 2,3 own rows NR1H..NR1-1
const bool sg_active = has_hi || sgitg < 2;
#else
auto cT0 = mm.get_destination_cooperative_tensor<decltype(tA), decltype(tB0), float>();
auto cT1 = mm.get_destination_cooperative_tensor<decltype(tA), decltype(tB1), float>();
#endif
@@ -819,7 +917,7 @@ kernel void kernel_mul_mm_id(
threadgroup_barrier(mem_flags::mem_threadgroup);
for (short j = sgitg; j < nr1; j += 4) {
const int id = ids_i32[im*args.ne21 + r1 + j];
const int id = ids_i32[idb + j];
const short ide = id % args.ne20;
const short idt = id / args.ne20;