llama : copy all experts of a host weight shared by several nodes (#32807)
* llama : copy all experts of a host weight shared by several nodes The scheduler copies a host weight to the device once per graph, so a partial expert copy made for the first MUL_MAT_ID is stale for the others (e.g. the shared layers of inkling2_loop with --n-cpu-moe). Assisted-by: Claude Opus 5.5 * llama : copy all experts of a host weight shared by several nodes Assisted-by: Codex
This commit is contained in:
1 parent
d2bceda3de
commit
5bacb7c868
1 file changed
+6
@@ -3143,6 +3143,12 @@ bool llama_context::sched_copy_experts(ggml_backend_t backend, const ggml_tensor
|
||||
return false;
|
||||
}
|
||||
|
||||
// Later loop passes can select different experts from the same device copy.
|
||||
// TODO: copy only the experts needed by each pass (see #32807 and #32809).
|
||||
if (lctx->model.arch == LLM_ARCH_INKLING2_LOOP) {
|
||||
return false;
|
||||
}
|
||||
|
||||
const ggml_tensor * ids = node->src[2];
|
||||
if (ggml_nelements(ids) == 0) {
|
||||
return true;
|
||||
|
||||
Reference in new issue
Block a user