webgpu: (Fix CI) fix mul_mat_id with duplicate expert ids (#30451)
* Fix mm-id of webgpu in the case of ids duplication * formatting * fix comments to avoid breaking up sentences in comments
This commit is contained in:
1 parent
04f7226a2c
commit
fcac19079f
5 files changed
+143
-136
No files matched your search
@@ -1053,6 +1053,22 @@ struct ggml_webgpu_mul_mat_shader_decisions {
|
||||
|
||||
/** MUL_MAT_ID **/
|
||||
|
||||
struct ggml_webgpu_mul_mat_id_gather_pipeline_key {
|
||||
uint32_t n_experts;
|
||||
|
||||
bool operator==(const ggml_webgpu_mul_mat_id_gather_pipeline_key & other) const {
|
||||
return n_experts == other.n_experts;
|
||||
}
|
||||
};
|
||||
|
||||
struct ggml_webgpu_mul_mat_id_gather_pipeline_key_hash {
|
||||
size_t operator()(const ggml_webgpu_mul_mat_id_gather_pipeline_key & key) const {
|
||||
size_t seed = 0;
|
||||
ggml_webgpu_hash_combine(seed, key.n_experts);
|
||||
return seed;
|
||||
}
|
||||
};
|
||||
|
||||
struct ggml_webgpu_mul_mat_id_pipeline_key {
|
||||
ggml_type src0_type;
|
||||
ggml_type src1_type;
|
||||
@@ -1263,12 +1279,15 @@ class ggml_webgpu_shader_lib {
|
||||
std::unordered_map<ggml_webgpu_mul_mat_pipeline_key, webgpu_pipeline, ggml_webgpu_mul_mat_pipeline_key_hash>
|
||||
mul_mat_fast_pipelines; // fast mat-mat (reg-tile or subgroup)
|
||||
std::unordered_map<ggml_webgpu_quantize_q8_pipeline_key, webgpu_pipeline, ggml_webgpu_quantize_q8_pipeline_key_hash>
|
||||
quantize_q8_pipelines;
|
||||
std::unordered_map<int, webgpu_pipeline> mul_mat_id_gather_pipelines; // key is fixed
|
||||
quantize_q8_pipelines;
|
||||
std::unordered_map<ggml_webgpu_mul_mat_id_gather_pipeline_key,
|
||||
webgpu_pipeline,
|
||||
ggml_webgpu_mul_mat_id_gather_pipeline_key_hash>
|
||||
mul_mat_id_gather_pipelines; // n_experts
|
||||
std::unordered_map<ggml_webgpu_mul_mat_id_pipeline_key, webgpu_pipeline, ggml_webgpu_mul_mat_id_pipeline_key_hash>
|
||||
mul_mat_id_pipelines; // src0_type/src1_type
|
||||
mul_mat_id_pipelines; // src0_type/src1_type
|
||||
std::unordered_map<ggml_webgpu_mul_mat_id_pipeline_key, webgpu_pipeline, ggml_webgpu_mul_mat_id_pipeline_key_hash>
|
||||
mul_mat_id_vec_pipelines; // src0_type/src1_type
|
||||
mul_mat_id_vec_pipelines; // src0_type/src1_type
|
||||
|
||||
std::unordered_map<ggml_webgpu_set_rows_pipeline_key, webgpu_pipeline, ggml_webgpu_set_rows_pipeline_key_hash>
|
||||
set_rows_pipelines;
|
||||
@@ -2325,20 +2344,26 @@ class ggml_webgpu_shader_lib {
|
||||
}
|
||||
|
||||
webgpu_pipeline get_mul_mat_id_gather_pipeline(const ggml_webgpu_shader_lib_context & context) {
|
||||
auto it = mul_mat_id_gather_pipelines.find(1);
|
||||
ggml_webgpu_mul_mat_id_gather_pipeline_key key = {};
|
||||
key.n_experts = (uint32_t) context.src0->ne[2];
|
||||
|
||||
auto it = mul_mat_id_gather_pipelines.find(key);
|
||||
if (it != mul_mat_id_gather_pipelines.end()) {
|
||||
return it->second;
|
||||
}
|
||||
std::vector<std::string> defines;
|
||||
defines.push_back(std::string("WG_SIZE=") + std::to_string(context.max_wg_size));
|
||||
defines.push_back(std::string("WG_SIZE=") + std::to_string(context.max_wg_size) + "u");
|
||||
defines.push_back(std::string("N_EXPERT=") + std::to_string(key.n_experts) + "u");
|
||||
defines.push_back(std::string("TILE_N_SLOTS=") +
|
||||
std::to_string(WEBGPU_MUL_MAT_TILE_N * WEBGPU_MUL_MAT_WG_SIZE_N) + "u");
|
||||
|
||||
auto processed = preprocessor.preprocess(wgsl_mul_mat_id_gather, defines);
|
||||
auto decisions = std::make_shared<ggml_webgpu_generic_shader_decisions>();
|
||||
decisions->wg_size = context.max_wg_size;
|
||||
|
||||
webgpu_pipeline pipeline = ggml_webgpu_create_pipeline(device, processed, "mul_mat_id_gather");
|
||||
pipeline.context = decisions;
|
||||
mul_mat_id_gather_pipelines[1] = pipeline;
|
||||
webgpu_pipeline pipeline = ggml_webgpu_create_pipeline(device, processed, "mul_mat_id_gather");
|
||||
pipeline.context = decisions;
|
||||
mul_mat_id_gather_pipelines[key] = pipeline;
|
||||
return pipeline;
|
||||
}
|
||||
|
||||
|
||||
@@ -1817,6 +1817,19 @@ static webgpu_encoded_op ggml_webgpu_mul_mat_id_vec(webgpu_context & ctx,
|
||||
return ggml_backend_webgpu_build(ctx, pipeline, params, entries, wg_x, wg_y);
|
||||
}
|
||||
|
||||
// upper bound on the number of slot tiles emitted by mul_mat_id_gather.wgsl
|
||||
static uint32_t ggml_webgpu_mul_mat_id_max_tiles(const ggml_tensor * dst) {
|
||||
const uint32_t n_expert = (uint32_t) dst->src[0]->ne[2];
|
||||
const uint32_t n_slots = (uint32_t) (dst->ne[1] * dst->ne[2]); // n_expert_used * n_tokens
|
||||
|
||||
return CEIL_DIV(n_slots, WEBGPU_MUL_MAT_TILE_N * WEBGPU_MUL_MAT_WG_SIZE_N) + std::min(n_expert, n_slots);
|
||||
}
|
||||
|
||||
// tile count followed by (expert, first slot, n slots) per tile
|
||||
static size_t ggml_webgpu_mul_mat_id_tiles_nbytes(const ggml_tensor * dst) {
|
||||
return (1 + 3 * (size_t) ggml_webgpu_mul_mat_id_max_tiles(dst)) * sizeof(uint32_t);
|
||||
}
|
||||
|
||||
static webgpu_encoded_op ggml_webgpu_mul_mat_id(webgpu_context & ctx,
|
||||
ggml_tensor * src0,
|
||||
ggml_tensor * src1,
|
||||
@@ -1856,39 +1869,29 @@ static webgpu_encoded_op ggml_webgpu_mul_mat_id(webgpu_context & ctx,
|
||||
(uint32_t) (src2->nb[1] / ggml_type_size(src2->type)),
|
||||
};
|
||||
|
||||
const size_t dst_offset = ggml_webgpu_tensor_offset(dst);
|
||||
const size_t gathered_buf_nbytes = src0->ne[2] * src1->ne[2] * sizeof(uint32_t);
|
||||
const size_t dst_offset = ggml_webgpu_tensor_offset(dst);
|
||||
const size_t ids_cache_nbytes = src2->ne[0] * src1->ne[2] * sizeof(uint32_t);
|
||||
const size_t ids_tiles_nbytes = ggml_webgpu_mul_mat_id_tiles_nbytes(dst);
|
||||
|
||||
const size_t gathered_expert_used_align_offset = ROUNDUP_POW2(
|
||||
const size_t ids_cache_align_offset = ROUNDUP_POW2(
|
||||
dst_offset + ggml_nbytes(dst), ctx->global_ctx->capabilities.limits.minStorageBufferOffsetAlignment);
|
||||
const size_t gathered_tokens_align_offset =
|
||||
ROUNDUP_POW2(gathered_expert_used_align_offset + gathered_buf_nbytes,
|
||||
ctx->global_ctx->capabilities.limits.minStorageBufferOffsetAlignment);
|
||||
const size_t gathered_count_ids_align_offset =
|
||||
ROUNDUP_POW2(gathered_tokens_align_offset + gathered_buf_nbytes,
|
||||
const size_t ids_tiles_align_offset =
|
||||
ROUNDUP_POW2(ids_cache_align_offset + ids_cache_nbytes,
|
||||
ctx->global_ctx->capabilities.limits.minStorageBufferOffsetAlignment);
|
||||
|
||||
const size_t gathered_binding_size = ROUNDUP_POW2(gathered_buf_nbytes, WEBGPU_STORAGE_BUF_BINDING_MULT);
|
||||
const size_t gathered_count_ids_binding_size =
|
||||
ROUNDUP_POW2(src0->ne[2] * sizeof(uint32_t), WEBGPU_STORAGE_BUF_BINDING_MULT);
|
||||
const size_t ids_cache_bind_size = ROUNDUP_POW2(ids_cache_nbytes, WEBGPU_STORAGE_BUF_BINDING_MULT);
|
||||
const size_t ids_tiles_bind_size = ROUNDUP_POW2(ids_tiles_nbytes, WEBGPU_STORAGE_BUF_BINDING_MULT);
|
||||
|
||||
// bind group entries for mul_mat_id_gather.wgsl
|
||||
std::vector<wgpu::BindGroupEntry> gather_entries = {
|
||||
ggml_webgpu_make_bind_group_entry(0, ggml_webgpu_tensor_buf(src2), ggml_webgpu_tensor_align_offset(ctx, src2),
|
||||
ggml_webgpu_tensor_binding_size(ctx, src2)),
|
||||
ggml_webgpu_make_bind_group_entry(1, ggml_webgpu_tensor_buf(dst), gathered_expert_used_align_offset,
|
||||
gathered_binding_size),
|
||||
ggml_webgpu_make_bind_group_entry(2, ggml_webgpu_tensor_buf(dst), gathered_tokens_align_offset,
|
||||
gathered_binding_size),
|
||||
ggml_webgpu_make_bind_group_entry(3, ggml_webgpu_tensor_buf(dst), gathered_count_ids_align_offset,
|
||||
gathered_count_ids_binding_size),
|
||||
ggml_webgpu_make_bind_group_entry(1, ggml_webgpu_tensor_buf(dst), ids_cache_align_offset, ids_cache_bind_size),
|
||||
ggml_webgpu_make_bind_group_entry(2, ggml_webgpu_tensor_buf(dst), ids_tiles_align_offset, ids_tiles_bind_size),
|
||||
};
|
||||
|
||||
// n_expert is much less than maxComputeWorkgroupsPerDimension (e.g., n_exeprt=256 at Qwen3.5-35B-A3B)
|
||||
const uint32_t gather_wg_x = param_n_expert;
|
||||
|
||||
dispatches.push_back({
|
||||
gather_pipeline, std::move(gather_params), std::move(gather_entries), { gather_wg_x, 1 }
|
||||
gather_pipeline, std::move(gather_params), std::move(gather_entries), { 1, 1 }
|
||||
});
|
||||
|
||||
// params for mul_mat_id.wgsl
|
||||
@@ -1916,12 +1919,8 @@ static webgpu_encoded_op ggml_webgpu_mul_mat_id(webgpu_context & ctx,
|
||||
ggml_webgpu_tensor_binding_size(ctx, src1)),
|
||||
ggml_webgpu_make_bind_group_entry(2, ggml_webgpu_tensor_buf(dst), ggml_webgpu_tensor_align_offset(ctx, dst),
|
||||
ggml_webgpu_tensor_binding_size(ctx, dst)),
|
||||
ggml_webgpu_make_bind_group_entry(3, ggml_webgpu_tensor_buf(dst), gathered_expert_used_align_offset,
|
||||
gathered_binding_size),
|
||||
ggml_webgpu_make_bind_group_entry(4, ggml_webgpu_tensor_buf(dst), gathered_tokens_align_offset,
|
||||
gathered_binding_size),
|
||||
ggml_webgpu_make_bind_group_entry(5, ggml_webgpu_tensor_buf(dst), gathered_count_ids_align_offset,
|
||||
gathered_count_ids_binding_size),
|
||||
ggml_webgpu_make_bind_group_entry(3, ggml_webgpu_tensor_buf(dst), ids_cache_align_offset, ids_cache_bind_size),
|
||||
ggml_webgpu_make_bind_group_entry(4, ggml_webgpu_tensor_buf(dst), ids_tiles_align_offset, ids_tiles_bind_size),
|
||||
};
|
||||
|
||||
// Calculate workgroup dimensions
|
||||
@@ -1930,15 +1929,12 @@ static webgpu_encoded_op ggml_webgpu_mul_mat_id(webgpu_context & ctx,
|
||||
|
||||
auto * main_decisions = static_cast<ggml_webgpu_mul_mat_shader_decisions *>(main_pipeline.context.get());
|
||||
|
||||
uint32_t wg_m;
|
||||
// must match `TILE_N_SLOTS` of mul_mat_id_gather.wgsl
|
||||
GGML_ASSERT(main_decisions->tile_n * main_decisions->wg_size_n == WEBGPU_MUL_MAT_TILE_N * WEBGPU_MUL_MAT_WG_SIZE_N);
|
||||
|
||||
uint32_t tile_m_s = main_decisions->tile_m * main_decisions->wg_size_m;
|
||||
uint32_t tile_n_s = main_decisions->tile_n * main_decisions->wg_size_n;
|
||||
wg_m = CEIL_DIV(dst->ne[0], tile_m_s);
|
||||
uint32_t total_gathered = dst->ne[1] * dst->ne[2];
|
||||
uint32_t max_active_experts = std::min((uint32_t) src0->ne[2], total_gathered);
|
||||
uint32_t max_wg_n = CEIL_DIV(total_gathered, tile_n_s) + max_active_experts;
|
||||
uint32_t total_wg = wg_m * max_wg_n;
|
||||
const uint32_t tile_m_s = main_decisions->tile_m * main_decisions->wg_size_m;
|
||||
const uint32_t wg_m = CEIL_DIV(dst->ne[0], tile_m_s);
|
||||
const uint32_t total_wg = wg_m * ggml_webgpu_mul_mat_id_max_tiles(dst);
|
||||
|
||||
compute_2d_workgroups(total_wg, ctx->global_ctx->capabilities.limits.maxComputeWorkgroupsPerDimension, wg_x, wg_y);
|
||||
|
||||
@@ -4065,11 +4061,11 @@ static size_t ggml_backend_webgpu_buffer_type_get_alloc_size(ggml_backend_buffer
|
||||
const ggml_tensor * src0 = tensor->src[0];
|
||||
const ggml_tensor * src1 = tensor->src[1];
|
||||
if (src0 && src1) {
|
||||
const size_t gathered_size = sizeof(uint32_t) * tensor->src[0]->ne[2] * tensor->src[1]->ne[2];
|
||||
const size_t gathered_count_ids_size = sizeof(uint32_t) * tensor->src[0]->ne[2];
|
||||
res = ROUNDUP_POW2(
|
||||
res + gathered_size * 2 + gathered_count_ids_size +
|
||||
ctx->webgpu_global_ctx->capabilities.limits.minStorageBufferOffsetAlignment * 3,
|
||||
const size_t ids_cache_size = sizeof(uint32_t) * tensor->ne[1] * tensor->ne[2];
|
||||
const size_t ids_tiles_size = ggml_webgpu_mul_mat_id_tiles_nbytes(tensor);
|
||||
res = ROUNDUP_POW2(
|
||||
res + ids_cache_size + ids_tiles_size +
|
||||
ctx->webgpu_global_ctx->capabilities.limits.minStorageBufferOffsetAlignment * 2,
|
||||
WEBGPU_STORAGE_BUF_BINDING_MULT);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -38,11 +38,11 @@ struct MulMatIdParams {
|
||||
@group(0) @binding(0) var<storage, read_write> src0: array<SRC0_TYPE>; // [cols, rows, n_expert]
|
||||
@group(0) @binding(1) var<storage, read_write> src1: array<SRC1_TYPE>; // [cols, b_ne1, n_tokens]
|
||||
@group(0) @binding(2) var<storage, read_write> dst: array<DST_TYPE>; // [rows, n_expert_used, n_tokens]
|
||||
@group(0) @binding(3) var<storage, read_write> global_gathered_expert_used: array<u32>; // [n_expert][n_tokens]
|
||||
@group(0) @binding(4) var<storage, read_write> global_gathered_tokens: array<u32>; // [n_expert][n_tokens]
|
||||
@group(0) @binding(5) var<storage, read_write> gathered_count_ids: array<u32>; // [n_expert]
|
||||
|
||||
@group(0) @binding(6) var<uniform> params: MulMatIdParams;
|
||||
@group(0) @binding(3) var<storage, read_write> ids_slot_pos: array<u32>; // [n_expert_used * n_tokens]
|
||||
@group(0) @binding(4) var<storage, read_write> ids_tiles: array<u32>; // [n_tiles, (expert, first slot, n slots) * n_tiles]
|
||||
|
||||
@group(0) @binding(5) var<uniform> params: MulMatIdParams;
|
||||
|
||||
fn get_local_n(thread_id: u32) -> u32 {
|
||||
return thread_id / WORKGROUP_SIZE_M;
|
||||
@@ -90,44 +90,29 @@ fn main(@builtin(workgroup_id) wg_id: vec3<u32>,
|
||||
let local_m = get_local_m(thread_id);
|
||||
let local_n = get_local_n(thread_id);
|
||||
|
||||
var expert_idx:u32 = 0xFFFFFFFFu;
|
||||
var wg_in_batch:u32 = 0;
|
||||
var wg_sum:u32 = 0;
|
||||
let wg_m_count = (params.m + WORKGROUP_SIZE_M * TILE_M - 1u) / (WORKGROUP_SIZE_M * TILE_M);
|
||||
let wg_linear = wg_id.y * num_wg.x + wg_id.x;
|
||||
|
||||
for (var i = 0u;i < params.n_expert;i += 1) {
|
||||
let wg_n_count = (gathered_count_ids[i] + WORKGROUP_SIZE_N * TILE_N - 1u) / (WORKGROUP_SIZE_N * TILE_N);
|
||||
let wg_per_matrix = wg_m_count * wg_n_count;
|
||||
if (wg_sum <= wg_linear && wg_linear < wg_sum + wg_per_matrix) {
|
||||
expert_idx = i;
|
||||
wg_in_batch = wg_linear - wg_sum;
|
||||
break;
|
||||
}
|
||||
wg_sum += wg_per_matrix;
|
||||
}
|
||||
// the slot tile of this workgroup, described by mul_mat_id_gather.wgsl
|
||||
let tile = wg_linear / wg_m_count;
|
||||
let wg_m = wg_linear % wg_m_count;
|
||||
|
||||
let is_valid = expert_idx != 0xFFFFFFFFu;
|
||||
let is_valid = tile < ids_tiles[0];
|
||||
|
||||
var wg_m: u32 = 0;
|
||||
var wg_n: u32 = 0;
|
||||
var offset_wg_m: u32 = 0;
|
||||
var offset_wg_n: u32 = 0;
|
||||
var expert_idx: u32 = 0;
|
||||
var rest_token_n: u32 = 0;
|
||||
var src0_batch_offset: u32 = 0;
|
||||
|
||||
wg_m = wg_in_batch % wg_m_count;
|
||||
wg_n = wg_in_batch / wg_m_count;
|
||||
|
||||
offset_wg_m = wg_m * WORKGROUP_SIZE_M * TILE_M;
|
||||
offset_wg_n = wg_n * WORKGROUP_SIZE_N * TILE_N;
|
||||
let offset_wg_m = wg_m * WORKGROUP_SIZE_M * TILE_M;
|
||||
|
||||
if (is_valid) {
|
||||
rest_token_n = gathered_count_ids[expert_idx] - offset_wg_n;
|
||||
let global_gathered_base = expert_idx * params.n_tokens + offset_wg_n;
|
||||
for (var i = thread_id; i < TILE_N * WORKGROUP_SIZE_N && offset_wg_n + i < gathered_count_ids[expert_idx]; i += TOTAL_WORKGROUP_SIZE) {
|
||||
gathered_expert_used[i] = global_gathered_expert_used[global_gathered_base + i];
|
||||
gathered_tokens[i] = global_gathered_tokens[global_gathered_base + i];
|
||||
expert_idx = ids_tiles[1 + 3 * tile];
|
||||
let slot_base = ids_tiles[1 + 3 * tile + 1];
|
||||
rest_token_n = ids_tiles[1 + 3 * tile + 2];
|
||||
for (var i = thread_id; i < rest_token_n; i += TOTAL_WORKGROUP_SIZE) {
|
||||
let pos_in_ids = ids_slot_pos[slot_base + i];
|
||||
gathered_expert_used[i] = pos_in_ids % params.n_expert_used;
|
||||
gathered_tokens[i] = pos_in_ids / params.n_expert_used;
|
||||
}
|
||||
src0_batch_offset = params.offset_src0 + expert_idx * params.stride_02;
|
||||
}
|
||||
@@ -135,7 +120,7 @@ fn main(@builtin(workgroup_id) wg_id: vec3<u32>,
|
||||
workgroupBarrier();
|
||||
|
||||
let output_row_base = offset_wg_m + local_m * TILE_M;
|
||||
let output_col_base = offset_wg_n + local_n * TILE_N;
|
||||
let output_col_base = local_n * TILE_N;
|
||||
|
||||
let dst2_stride = params.m * params.n_expert_used;
|
||||
let dst1_stride = params.m;
|
||||
@@ -178,9 +163,9 @@ fn main(@builtin(workgroup_id) wg_id: vec3<u32>,
|
||||
if (is_valid) {
|
||||
for (var tn = 0u; tn < TILE_N; tn++) {
|
||||
let n_idx = output_col_base + tn;
|
||||
if (n_idx < gathered_count_ids[expert_idx]) {
|
||||
let dst1_idx = gathered_expert_used[n_idx - offset_wg_n];
|
||||
let dst2_idx = gathered_tokens[n_idx - offset_wg_n];
|
||||
if (n_idx < rest_token_n) {
|
||||
let dst1_idx = gathered_expert_used[n_idx];
|
||||
let dst2_idx = gathered_tokens[n_idx];
|
||||
let dst12_offset = params.offset_dst + dst2_idx * dst2_stride + dst1_idx * dst1_stride;
|
||||
for (var tm = 0u; tm < TILE_M; tm += VEC_SIZE) {
|
||||
let global_row = output_row_base + tm;
|
||||
|
||||
@@ -11,42 +11,68 @@ struct MulMatIdGatherParams {
|
||||
};
|
||||
|
||||
@group(0) @binding(0) var<storage, read_write> ids: array<i32>; // [n_expert_used, n_tokens]
|
||||
@group(0) @binding(1) var<storage, read_write> global_gathered_expert_used: array<u32>; // [n_expert][n_tokens]
|
||||
@group(0) @binding(2) var<storage, read_write> global_gathered_tokens: array<u32>; // [n_expert][n_tokens]
|
||||
@group(0) @binding(3) var<storage, read_write> gathered_count_ids: array<u32>; // [n_expert]
|
||||
@group(0) @binding(1) var<storage, read_write> ids_slot_pos: array<u32>; // [n_expert_used * n_tokens]
|
||||
@group(0) @binding(2) var<storage, read_write> ids_tiles: array<u32>; // [n_tiles, (expert, first slot, n slots) * n_tiles]
|
||||
|
||||
@group(0) @binding(4) var<uniform> params: MulMatIdGatherParams;
|
||||
@group(0) @binding(3) var<uniform> params: MulMatIdGatherParams;
|
||||
|
||||
var<workgroup> count:atomic<u32>;
|
||||
// per expert: the slot count, replaced by the first slot of the expert in ids_slot_pos by the prefix sum, then advanced as its write cursor
|
||||
var<workgroup> ids_count: array<atomic<u32>, N_EXPERT>;
|
||||
|
||||
// launched as a single workgroup, where the threads process the elements of ids in parallel with atomics
|
||||
@compute @workgroup_size(WG_SIZE)
|
||||
fn main(@builtin(workgroup_id) wg_id: vec3<u32>,
|
||||
@builtin(local_invocation_id) local_id: vec3<u32>) {
|
||||
fn main(@builtin(local_invocation_id) local_id: vec3<u32>) {
|
||||
|
||||
let thread_id = local_id.x;
|
||||
let own_expert = wg_id.x; // the expert assigned to this workgroup
|
||||
let ids_slots_num = params.n_expert_used * params.n_tokens;
|
||||
|
||||
if (thread_id == 0u) {
|
||||
atomicStore(&count, 0);
|
||||
for (var e = thread_id; e < N_EXPERT; e += WG_SIZE) {
|
||||
atomicStore(&ids_count[e], 0u);
|
||||
}
|
||||
|
||||
workgroupBarrier();
|
||||
|
||||
for (var i = thread_id;i < params.n_expert_used * params.n_tokens;i += WG_SIZE) {
|
||||
// count the slots of each expert
|
||||
for (var i = thread_id; i < ids_slots_num; i += WG_SIZE) {
|
||||
let row = i / params.n_expert_used;
|
||||
let col = i % params.n_expert_used;
|
||||
let expert = u32(ids[params.offset_ids + row * params.stride_ids_1 + col]);
|
||||
if (own_expert == expert) {
|
||||
let pos = atomicAdd(&count, 1u);
|
||||
let gathered_id = own_expert * params.n_tokens + pos;
|
||||
global_gathered_expert_used[gathered_id] = col;
|
||||
global_gathered_tokens[gathered_id] = row;
|
||||
}
|
||||
atomicAdd(&ids_count[expert], 1u);
|
||||
}
|
||||
|
||||
workgroupBarrier();
|
||||
|
||||
// exclusive prefix sum over the counts, and emit one tile descriptor per `TILE_N_SLOTS` slots of each expert
|
||||
if (thread_id == 0u) {
|
||||
gathered_count_ids[own_expert] = atomicLoad(&count);
|
||||
var base: u32 = 0;
|
||||
var tile: u32 = 0;
|
||||
|
||||
for (var e: u32 = 0; e < N_EXPERT; e += 1) {
|
||||
let n = atomicLoad(&ids_count[e]);
|
||||
|
||||
atomicStore(&ids_count[e], base);
|
||||
|
||||
for (var k: u32 = 0; k < n; k += TILE_N_SLOTS) {
|
||||
ids_tiles[1 + 3 * tile] = e;
|
||||
ids_tiles[1 + 3 * tile + 1] = base + k;
|
||||
ids_tiles[1 + 3 * tile + 2] = min(n - k, TILE_N_SLOTS);
|
||||
tile += 1;
|
||||
}
|
||||
|
||||
base += n;
|
||||
}
|
||||
|
||||
ids_tiles[0] = tile;
|
||||
}
|
||||
|
||||
workgroupBarrier();
|
||||
|
||||
// store the position in ids tensor to ids_slot_pos
|
||||
for (var i = thread_id; i < ids_slots_num; i += WG_SIZE) {
|
||||
let row = i / params.n_expert_used;
|
||||
let col = i % params.n_expert_used;
|
||||
let expert = u32(ids[params.offset_ids + row * params.stride_ids_1 + col]);
|
||||
let pos = atomicAdd(&ids_count[expert], 1u);
|
||||
ids_slot_pos[pos] = i;
|
||||
}
|
||||
}
|
||||
@@ -59,45 +59,20 @@ fn main(
|
||||
|
||||
let thread_id = local_id.x;
|
||||
|
||||
for (var i = thread_id;i < params.n_expert;i += WG_SIZE) {
|
||||
gathered_count_ids[i] = 0;
|
||||
}
|
||||
|
||||
workgroupBarrier();
|
||||
|
||||
// gather the selected experts for the target token.
|
||||
for (var col = thread_id;col < params.n_expert_used;col += WG_SIZE) {
|
||||
let expert = ids[params.offset_ids + col];
|
||||
gathered_count_ids[expert] = 1;
|
||||
gathered_expert_used[expert] = col;
|
||||
}
|
||||
|
||||
workgroupBarrier();
|
||||
|
||||
let output_groups:u32 = (params.m + OUTPUTS_PER_WG - 1u) / OUTPUTS_PER_WG;
|
||||
let output_groups: u32 = (params.m + OUTPUTS_PER_WG - 1u) / OUTPUTS_PER_WG;
|
||||
let wg_linear = wg_id.y * num_wg.x + wg_id.x;
|
||||
|
||||
var own_expert:u32 = 0;
|
||||
var wg_in_batch:u32 = 0;
|
||||
var wg_sum:u32 = 0;
|
||||
|
||||
for (var i = 0u;i < params.n_expert;i += 1) {
|
||||
let wg_vec_count = gathered_count_ids[i]; // 1 or 0
|
||||
let wg_per_matrix = output_groups * wg_vec_count;
|
||||
if (wg_sum <= wg_linear && wg_linear < wg_sum + wg_per_matrix) {
|
||||
own_expert = i;
|
||||
wg_in_batch = wg_linear - wg_sum;
|
||||
break;
|
||||
}
|
||||
wg_sum += wg_per_matrix;
|
||||
if (wg_linear >= output_groups * params.n_expert_used) {
|
||||
return;
|
||||
}
|
||||
|
||||
let row_base = (wg_linear % output_groups) * OUTPUTS_PER_WG;
|
||||
let dst1_stride = params.m;
|
||||
let slot = wg_linear / output_groups;
|
||||
let own_expert = ids[params.offset_ids + slot];
|
||||
let row_base = (wg_linear % output_groups) * OUTPUTS_PER_WG;
|
||||
|
||||
let src0_batch_offset = params.offset_src0 + own_expert * params.stride_02;
|
||||
let src1_idx_base = params.offset_src1 + (gathered_expert_used[own_expert] % params.b_ne1) * params.stride_11;
|
||||
let dst_idx_base = params.offset_dst + gathered_expert_used[own_expert] * dst1_stride + row_base;
|
||||
let src1_idx_base = params.offset_src1 + (slot % params.b_ne1) * params.stride_11;
|
||||
let dst_idx_base = params.offset_dst + slot * params.m + row_base;
|
||||
|
||||
let acc = accumulate_vec_dot(thread_id, row_base, src0_batch_offset, src1_idx_base);
|
||||
|
||||
|
||||
Reference in new issue
Block a user