model: split layers evenly when no device reports free memory (#29677)
When every device reports zero free memory the split points are normalized by a zero sum, the NaN split sends layers to a device index past the end and loading aborts with vector::_M_range_check.
This commit is contained in:
1 parent
3eb101360f
commit
df18bee11e
1 file changed
+1
-1
+1
-1
@@ -1645,7 +1645,7 @@ bool llama_model_base::load_tensors(llama_model_loader & ml) {
|
||||
splits[i] = split_sum;
|
||||
}
|
||||
for (size_t i = 0; i < n_devices(); ++i) {
|
||||
splits[i] /= split_sum;
|
||||
splits[i] = split_sum != 0.0f ? splits[i] / split_sum : float(i + 1) / n_devices();
|
||||
}
|
||||
|
||||
const int i_gpu_start = std::max(n_layer_all + 1 - n_gpu_layers, 0);
|
||||
|
||||
Reference in new issue
Block a user