ggml-backend : prefetch only inputs computed by earlier splits (#32100)

* ggml-backend : prefetch only inputs computed by earlier splits

The copy callback of a split runs after the prefetch and can write tensors that a later split reads, e.g. the MoE cache uploads missed experts into its bank. A prefetch of such a tensor reads it before the write.

* llama : skip the MoE cache when its device cannot run the cached mul_mat_id

Otherwise the scheduler runs the matmul on another backend and copies the whole bank every ubatch.
This commit is contained in:
Aman Gupta authored and GitHub committed 2026-11-12 17:24:40 +08:00
1 parent 81f21f4b27
commit 3bce8bd50c
2 files changed
+16 -2

No files matched your search

+10 -1
View File
@@ -2687,7 +2687,16 @@ llm_graph_moe_cache_slots llm_graph_context::build_moe_cache_slots(
return { nullptr, nullptr };
}
for (ggml_tensor * w : { up_exps, gate_exps, down_exps, gate_up_exps }) {
if (w != nullptr && moe_cache->get_experts(w) == nullptr) {
if (w == nullptr) {
continue;
}
ggml_tensor * experts = moe_cache->get_experts(w);
if (experts == nullptr) {
return { nullptr, nullptr };
}
// the device of the cache must compute the matmul, else the scheduler copies the cache to another backend
ggml_tensor * b = ggml_new_tensor_3d(ctx0, GGML_TYPE_F32, experts->ne[0], w == down_exps ? selected_experts->ne[0] : 1, selected_experts->ne[1]);
if (!ggml_backend_supports_op(moe_cache->backend(il), ggml_mul_mat_id(ctx0, experts, b, selected_experts))) {
return { nullptr, nullptr };
}
}