diff --git a/ggml/src/ggml-backend.cpp b/ggml/src/ggml-backend.cpp index 961990f85..b70d0d874 100644 --- a/ggml/src/ggml-backend.cpp +++ b/ggml/src/ggml-backend.cpp @@ -2172,8 +2172,13 @@ static void ggml_backend_sched_prefetch_inputs(ggml_backend_sched_t sched, int s continue; } - // not if this split computes or writes the input + // only results of earlier splits, the copy callback of this split runs later and can write other tensors (e.g. the MoE cache) const struct ggml_tensor * base = input->view_src ? input->view_src : input; + if (base->buffer == NULL || ggml_backend_buffer_get_usage(base->buffer) != GGML_BACKEND_BUFFER_USAGE_COMPUTE) { + continue; + } + + // not if this split computes or writes the input bool written = false; for (int j = 0; j < split->graph.n_nodes && !written; j++) { const struct ggml_tensor * node = split->graph.nodes[j]; diff --git a/src/llama-graph.cpp b/src/llama-graph.cpp index cc7779164..50a76f691 100644 --- a/src/llama-graph.cpp +++ b/src/llama-graph.cpp @@ -2687,7 +2687,16 @@ llm_graph_moe_cache_slots llm_graph_context::build_moe_cache_slots( return { nullptr, nullptr }; } for (ggml_tensor * w : { up_exps, gate_exps, down_exps, gate_up_exps }) { - if (w != nullptr && moe_cache->get_experts(w) == nullptr) { + if (w == nullptr) { + continue; + } + ggml_tensor * experts = moe_cache->get_experts(w); + if (experts == nullptr) { + return { nullptr, nullptr }; + } + // the device of the cache must compute the matmul, else the scheduler copies the cache to another backend + ggml_tensor * b = ggml_new_tensor_3d(ctx0, GGML_TYPE_F32, experts->ne[0], w == down_exps ? selected_experts->ne[0] : 1, selected_experts->ne[1]); + if (!ggml_backend_supports_op(moe_cache->backend(il), ggml_mul_mat_id(ctx0, experts, b, selected_experts))) { return { nullptr, nullptr }; } }