vulkan: avoid redundant scans when zeroing skipped rows (#31266)

* vulkan: avoid redundant scans when zeroing skipped rows

* vulkan: remove unreachable zero-expert fallback
This commit is contained in:
Amadeus dyw authored and GitHub committed 2026-11-02 13:53:27 +02:00
1 parent 2c93f0b6e1
commit cb02250b5b
1 file changed
+47 -12
@@ -40,6 +40,8 @@ layout (binding = 2) writeonly buffer DST {float data_dst[];};
shared uint vals[EXPERTS_PER_CHUNK];
shared uint offsets[EXPERTS_PER_CHUNK];
shared uint cursors[EXPERTS_PER_CHUNK];
shared uint skipped_rows;
shared uint skip_count;
// data_d[0] is the tile count. Each tile has (expert, valid rows).
// Packed (token << 16) | slot row ids start at p.row_ids_offset, with each expert on a tile boundary.
@@ -51,23 +53,38 @@ void write_tiles(uint expert, uint count, uint tile_begin, uint n_tiles) {
}
}
// the matmul writes only the rows that an expert owns, so zero the rows of the skipped slots
// The matmul does not write skipped rows, so collect their offsets and zero them cooperatively.
void zero_skipped_rows() {
const uint num_elements = p.ne00 * p.ne01;
const uint tid = gl_LocalInvocationID.x;
for (uint idx = 0; idx < num_elements; idx++) {
const uint i01 = fastdiv(idx, p.ne00mp, p.ne00L);
const uint i00 = idx - i01 * p.ne00;
if (int(data_a[p.a_offset + i01 * p.nb01 + i00 * p.nb00]) != -1) {
continue;
// Reuse offsets[] as a queue, so each batch must fit in EXPERTS_PER_CHUNK.
for (uint base = 0; base < num_elements; base += EXPERTS_PER_CHUNK) {
const uint end = min(base + EXPERTS_PER_CHUNK, num_elements);
if (tid == 0) {
skip_count = 0;
}
barrier();
const uint d_off = i00 * p.stride_dst + i01 * p.batch_stride_dst;
for (uint i = tid; i < p.ne0_dst; i += BLOCK_SIZE) {
data_dst[d_off + i] = 0.0f;
for (uint idx = base + tid; idx < end; idx += BLOCK_SIZE) {
const uint i01 = fastdiv(idx, p.ne00mp, p.ne00L);
const uint i00 = idx - i01 * p.ne00;
if (int(data_a[p.a_offset + i01 * p.nb01 + i00 * p.nb00]) == -1) {
const uint slot = atomicAdd(skip_count, 1);
offsets[slot] = i00 * p.stride_dst + i01 * p.batch_stride_dst;
}
}
barrier();
const uint n_skipped = skip_count;
for (uint r = 0; r < n_skipped; ++r) {
const uint d_off = offsets[r];
for (uint i = tid; i < p.ne0_dst; i += BLOCK_SIZE) {
data_dst[d_off + i] = 0.0f;
}
}
barrier();
}
}
@@ -76,6 +93,11 @@ void main() {
const uint tid = gl_LocalInvocationID.x;
const uint tile_shift = findLSB(p.row_tile_size);
uint total_tiles = 0;
uint local_skipped = 0;
if (tid == 0) {
skipped_rows = 0;
}
barrier();
#ifdef USE_SUBGROUPS
// Use the subgroup that contains invocation 0.
const bool prefix_subgroup = subgroupAny(tid == 0);
@@ -135,7 +157,12 @@ void main() {
for (uint idx = tid; idx < num_elements; idx += BLOCK_SIZE) {
const uint i01 = fastdiv(idx, p.ne00mp, p.ne00L);
const uint i00 = idx - i01 * p.ne00;
const uint expert = data_a[p.a_offset + i01 * p.nb01 + i00 * p.nb00] - expert_base;
const uint id = data_a[p.a_offset + i01 * p.nb01 + i00 * p.nb00];
// Count skipped slots only in the first expert chunk.
if (expert_base == 0 && int(id) == -1) {
local_skipped++;
}
const uint expert = id - expert_base;
if (expert < n_experts) {
const uint row = atomicAdd(cursors[expert], 1);
data_d[p.row_ids_offset + offsets[expert] + row] = (i01 << 16) | i00;
@@ -144,9 +171,17 @@ void main() {
// The next chunk clears vals. Scatter uses only offsets and cursors.
expert_base += n_experts;
}
if (local_skipped != 0) {
atomicAdd(skipped_rows, local_skipped);
}
// All scatter reads must finish before reusing offsets[] as a queue.
barrier();
if (tid == 0) {
data_d[0] = total_tiles;
}
zero_skipped_rows();
if (skipped_rows != 0) {
zero_skipped_rows();
}
}