llama: support MoE cache over multiple GPUs (#30112)

This commit is contained in:
Aman Gupta authored and GitHub committed 2026-10-08 15:50:49 +05:30
1 parent 033df86b69
commit c811cb8f0a
11 files changed
+164 -73

No files matched your search

+2 -2
View File
@@ -2448,10 +2448,10 @@ ggml_tensor * llm_graph_context::build_moe_cache_slots(
// the slot map is a host weight, so the scheduler starts a new split here and copies it with the copy callback
// the callback reads the selected experts, uploads the missing ones and updates the slot map
ggml_tensor * slots = ggml_get_rows(ctx0, slot_map, ids); // [1, n_expert_used*n_tokens]
if (!ggml_backend_supports_op(moe_cache->backend(), slots)) {
if (!ggml_backend_supports_op(moe_cache->backend(il), slots)) {
return nullptr;
}
ggml_backend_sched_set_tensor_backend(sched, slots, moe_cache->backend());
ggml_backend_sched_set_tensor_backend(sched, slots, moe_cache->backend(il));
cb(slots, "ffn_moe_slots", il);
return ggml_reshape_2d(ctx0, slots, selected_experts->ne[0], selected_experts->ne[1]); // [n_expert_used, n_tokens]