llama: add default load-mode auto, which avoids mmap on iGPUs (#26081)

* llama: add new default load-mode auto which picks mmap unless a non-Metal iGPU is used

* Update ggml/src/ggml-hexagon/ggml-hexagon.cpp

Co-authored-by: Max Krasnyansky <maxk@qti.qualcomm.com>

* set mmap_support to false on OpenCL backend

* fix order of load modes

* use -1 for auto

* resolve load mode auto earlier to correctly pick gpu host or cpu memory

* add load mode auto to llama-bench

* bump virtgpu api version, regenerate docs

---------

Co-authored-by: Piotr Wilkin (ilintar) <piotr.wilkin@syndatis.com>
Co-authored-by: Max Krasnyansky <maxk@qti.qualcomm.com>
Co-authored-by: Georgi Gerganov <ggerganov@gmail.com>

ggml/src/ggml-virtgpu/backend/backend-dispatched-device.cpp

@@ -111,6 +111,7 @@ uint32_t backend_device_get_props(apir_encoder * enc, apir_decoder * dec, virgl_
     apir_encode_bool_t(enc, &props.caps.host_buffer);
     apir_encode_bool_t(enc, &props.caps.buffer_from_host_ptr);
     apir_encode_bool_t(enc, &props.caps.events);
+    apir_encode_bool_t(enc, &props.caps.mmap_support);
 
     return 0;
 }

ggml/src/ggml-virtgpu/backend/shared/api_remoting.h

@@ -7,7 +7,7 @@
 #include <cstdint>
 
 #define APIR_PROTOCOL_MAJOR 0
-#define APIR_PROTOCOL_MINOR 1
+#define APIR_PROTOCOL_MINOR 2
 
 #define APIR_HANDSHAKE_MAGIC 0xab1e
 

ggml/src/ggml-virtgpu/ggml-backend-buffer-type.cpp

@@ -11,9 +11,9 @@ static ggml_backend_buffer_t ggml_backend_remoting_buffer_type_alloc_buffer(ggml
 
     context->gpu = gpu;
 
-    bool async__unused, host_buffer__unused, events__unused;
+    bool async__unused, host_buffer__unused, events__unused, mmap_support__unused;
     bool buffer_from_host_ptr;
-    apir_device_get_props(gpu, &async__unused, &host_buffer__unused, &buffer_from_host_ptr, &events__unused);
+    apir_device_get_props(gpu, &async__unused, &host_buffer__unused, &buffer_from_host_ptr, &events__unused, &mmap_support__unused);
 
     if (buffer_from_host_ptr) {
         context->apir_context = apir_device_buffer_from_ptr(gpu, size, size);

ggml/src/ggml-virtgpu/ggml-backend-device.cpp

@@ -65,7 +65,7 @@ static void ggml_backend_remoting_device_get_props(ggml_backend_dev_t dev, ggml_
 
     virtgpu * gpu = DEV_TO_GPU(dev);
     apir_device_get_props(gpu, &props->caps.async, &props->caps.host_buffer, &props->caps.buffer_from_host_ptr,
-                          &props->caps.events);
+                          &props->caps.events, &props->caps.mmap_support);
 
     props->caps.buffer_from_host_ptr = false;
     props->caps.async                = false;

ggml/src/ggml-virtgpu/virtgpu-forward-device.cpp

@@ -144,7 +144,8 @@ void apir_device_get_props(virtgpu * gpu,
                            bool *    async,
                            bool *    host_buffer,
                            bool *    buffer_from_host_ptr,
-                           bool *    events) {
+                           bool *    events,
+                           bool *    mmap_support) {
     apir_encoder *        encoder;
     apir_decoder *        decoder;
     ApirForwardReturnCode ret;
@@ -157,6 +158,7 @@ void apir_device_get_props(virtgpu * gpu,
     apir_decode_bool_t(decoder, host_buffer);
     apir_decode_bool_t(decoder, buffer_from_host_ptr);
     apir_decode_bool_t(decoder, events);
+    apir_decode_bool_t(decoder, mmap_support);
 
     remote_call_finish(gpu, encoder, decoder);
 

ggml/src/ggml-virtgpu/virtgpu-forward.gen.h

@@ -13,7 +13,8 @@ void                           apir_device_get_props(struct virtgpu * gpu,
                                                      bool *           async,
                                                      bool *           host_buffer,
                                                      bool *           buffer_from_host_ptr,
-                                                     bool *           events);
+                                                     bool *           events,
+                                                     bool *           mmap_support);
 apir_buffer_context_t          apir_device_buffer_from_ptr(struct virtgpu * gpu, size_t size, size_t max_tensor_size);
 
 /* buffer-type */