llama: support MoE cache over multiple GPUs (#30112)

This commit is contained in:
Aman Gupta authored and GitHub committed 2026-10-08 15:50:49 +05:30
1 parent 033df86b69
commit c811cb8f0a
11 files changed
+164 -73

No files matched your search

+3 -14
View File
@@ -436,7 +436,8 @@ llama_context::llama_context(
model.n_gpu_layers() > model.hparams.n_layer_all &&
model.split_mode() == LLAMA_SPLIT_MODE_LAYER &&
cparams.offload_kqv &&
!model.has_tensor_overrides();
!model.has_tensor_overrides() &&
cparams.moe_cache_size == 0; // not supported by the MoE cache
// pipeline parallelism requires support for async compute and events in all devices
if (pipeline_parallel) {
@@ -465,19 +466,7 @@ llama_context::llama_context(
}
if (cparams.moe_cache_size > 0) {
if (cparams.pipeline_parallel || model.n_devices() > 1) {
throw std::runtime_error("MoE cache does not support multiple devices");
}
for (size_t i = 0; i < backend_ptrs.size(); ++i) {
const auto type = ggml_backend_dev_type(ggml_backend_get_device(backend_ptrs[i]));
if (type == GGML_BACKEND_DEVICE_TYPE_GPU || type == GGML_BACKEND_DEVICE_TYPE_IGPU) {
moe_cache = std::make_unique<llama_moe_cache>(model, backend_ptrs[i], backend_buft[i], cparams.moe_cache_size);
break;
}
}
if (!moe_cache) {
throw std::runtime_error("MoE cache requires a GPU backend");
}
moe_cache = std::make_unique<llama_moe_cache>(model, backend_ptrs, backend_buft, cparams.moe_cache_size);
}
sched_reserve();