llama: add default load-mode auto, which avoids mmap on iGPUs (#26081)
* llama: add new default load-mode auto which picks mmap unless a non-Metal iGPU is used * Update ggml/src/ggml-hexagon/ggml-hexagon.cpp Co-authored-by: Max Krasnyansky <maxk@qti.qualcomm.com> * set mmap_support to false on OpenCL backend * fix order of load modes * use -1 for auto * resolve load mode auto earlier to correctly pick gpu host or cpu memory * add load mode auto to llama-bench * bump virtgpu api version, regenerate docs --------- Co-authored-by: Piotr Wilkin (ilintar) <piotr.wilkin@syndatis.com> Co-authored-by: Max Krasnyansky <maxk@qti.qualcomm.com> Co-authored-by: Georgi Gerganov <ggerganov@gmail.com>
This commit is contained in:
co-authored by
Max Krasnyansky
Piotr Wilkin
Georgi Gerganov
parent
b3df57286c
commit
153d324bcf
+1
-1
@@ -473,7 +473,7 @@ struct common_params {
|
||||
std::vector<size_t> fit_params_target = std::vector<size_t>(llama_max_devices(), 1024 * 1024*1024);
|
||||
|
||||
enum llama_split_mode split_mode = LLAMA_SPLIT_MODE_LAYER; // how to split the model across GPUs
|
||||
enum llama_load_mode load_mode = LLAMA_LOAD_MODE_MMAP; // how to load the model
|
||||
enum llama_load_mode load_mode = LLAMA_LOAD_MODE_AUTO; // how to load the model
|
||||
|
||||
common_cpu_params cpuparams;
|
||||
common_cpu_params cpuparams_batch;
|
||||
|
||||
Reference in New Issue
Block a user