diff --git a/src/llama-context.cpp b/src/llama-context.cpp index 9dcaf05a6..136b1e061 100644 --- a/src/llama-context.cpp +++ b/src/llama-context.cpp @@ -2680,16 +2680,30 @@ bool llama_context::sched_copy_experts(ggml_backend_t backend, const ggml_tensor const size_t expert_size = src->nb[2]; if (ids != st.ids || (int64_t) st.used.size() != n_expert) { - st.ids_data.resize(ggml_nbytes(ids)/sizeof(int32_t)); - ggml_backend_tensor_get_async(backend, ids, st.ids_data.data(), 0, ggml_nbytes(ids)); - ggml_backend_synchronize(backend); + // Reading the ids back forces a full pipeline flush, since the ids are + // produced on the same device we are about to upload to. Once the batch + // draws enough experts, essentially all of them come back used and the + // readback has bought nothing, so skip it and copy the whole tensor as one + // range. Copying an expert that no id selects cannot change the result: + // mul_mat_id only reads the rows the ids point at, so this stays exact. + // With the MoE cache on, cached experts are still copied device-to-device. + const bool all_experts_used = ggml_nelements(ids) >= 4*n_expert; - st.used.assign(n_expert, false); - for (int64_t i1 = 0; i1 < ids->ne[1]; i1++) { - for (int64_t i0 = 0; i0 < ids->ne[0]; i0++) { - const int32_t id = st.ids_data[i1*ids->nb[1]/sizeof(int32_t) + i0*ids->nb[0]/sizeof(int32_t)]; - GGML_ASSERT(id >= 0 && id < n_expert); - st.used[id] = true; + if (all_experts_used) { + st.ids_data.clear(); + st.used.assign(n_expert, true); + } else { + st.ids_data.resize(ggml_nbytes(ids)/sizeof(int32_t)); + ggml_backend_tensor_get_async(backend, ids, st.ids_data.data(), 0, ggml_nbytes(ids)); + ggml_backend_synchronize(backend); + + st.used.assign(n_expert, false); + for (int64_t i1 = 0; i1 < ids->ne[1]; i1++) { + for (int64_t i0 = 0; i0 < ids->ne[0]; i0++) { + const int32_t id = st.ids_data[i1*ids->nb[1]/sizeof(int32_t) + i0*ids->nb[0]/sizeof(int32_t)]; + GGML_ASSERT(id >= 0 && id < n_expert); + st.used[id] = true; + } } }