llama: support MoE cache over multiple GPUs (#30112)
This commit is contained in:
1 parent
033df86b69
commit
c811cb8f0a
11 files changed
+164
-73
No files matched your search
+3
-14
@@ -436,7 +436,8 @@ llama_context::llama_context(
|
||||
model.n_gpu_layers() > model.hparams.n_layer_all &&
|
||||
model.split_mode() == LLAMA_SPLIT_MODE_LAYER &&
|
||||
cparams.offload_kqv &&
|
||||
!model.has_tensor_overrides();
|
||||
!model.has_tensor_overrides() &&
|
||||
cparams.moe_cache_size == 0; // not supported by the MoE cache
|
||||
|
||||
// pipeline parallelism requires support for async compute and events in all devices
|
||||
if (pipeline_parallel) {
|
||||
@@ -465,19 +466,7 @@ llama_context::llama_context(
|
||||
}
|
||||
|
||||
if (cparams.moe_cache_size > 0) {
|
||||
if (cparams.pipeline_parallel || model.n_devices() > 1) {
|
||||
throw std::runtime_error("MoE cache does not support multiple devices");
|
||||
}
|
||||
for (size_t i = 0; i < backend_ptrs.size(); ++i) {
|
||||
const auto type = ggml_backend_dev_type(ggml_backend_get_device(backend_ptrs[i]));
|
||||
if (type == GGML_BACKEND_DEVICE_TYPE_GPU || type == GGML_BACKEND_DEVICE_TYPE_IGPU) {
|
||||
moe_cache = std::make_unique<llama_moe_cache>(model, backend_ptrs[i], backend_buft[i], cparams.moe_cache_size);
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (!moe_cache) {
|
||||
throw std::runtime_error("MoE cache requires a GPU backend");
|
||||
}
|
||||
moe_cache = std::make_unique<llama_moe_cache>(model, backend_ptrs, backend_buft, cparams.moe_cache_size);
|
||||
}
|
||||
|
||||
sched_reserve();
|
||||
|
||||
Reference in new issue
Block a user