diff --git a/ggml/src/ggml-vulkan/ggml-vulkan-buffers.cpp b/ggml/src/ggml-vulkan/ggml-vulkan-buffers.cpp index eabd8974a..b687b1390 100644 --- a/ggml/src/ggml-vulkan/ggml-vulkan-buffers.cpp +++ b/ggml/src/ggml-vulkan/ggml-vulkan-buffers.cpp @@ -1,6 +1,5 @@ #include "ggml-vulkan-common.h" -#include #include #include #include @@ -363,21 +362,16 @@ void ggml_vk_host_get(const vk_device& device, const void * ptr, vk_buffer& buf, vk_host_allocation & entry = vk_instance.host_allocations[index]; if (entry.views[device->idx] == nullptr) { - if (!vk_instance.multi_device) { - return; - } - if (!device->external_memory_host) { - static std::atomic warned_mask{0}; - const uint32_t bit = 1u << device->idx; - if (!(warned_mask.fetch_or(bit) & bit)) { - GGML_LOG_WARN("ggml_vulkan: %s does not support VK_EXT_external_memory_host; host memory " - "will be staged, not pinned, on this device\n", device->name.c_str()); - } + const uint32_t bit = 1u << device->idx; + if (!vk_instance.multi_device || !vk_instance.device_host_buffer_import[device->idx] || (vk_instance.host_import_failed & bit)) { return; } + GGML_ASSERT(device->external_memory_host); // const only propagates from the const context; the pointee is mutable entry.views[device->idx] = ggml_vk_buffer_from_host_ptr(const_cast(device), entry.base, entry.size); if (entry.views[device->idx] == nullptr) { + vk_instance.host_import_failed |= bit; + GGML_LOG_WARN("ggml_vulkan: %s: host memory import failed, using staging\n", device->name.c_str()); return; } } diff --git a/ggml/src/ggml-vulkan/ggml-vulkan-types.h b/ggml/src/ggml-vulkan/ggml-vulkan-types.h index a41ce5966..00d20e843 100644 --- a/ggml/src/ggml-vulkan/ggml-vulkan-types.h +++ b/ggml/src/ggml-vulkan/ggml-vulkan-types.h @@ -1329,12 +1329,14 @@ struct vk_instance_t { std::vector device_indices; std::vector device_supports_membudget; + std::vector device_host_buffer_import; vk_device devices[GGML_VK_MAX_DEVICES]; std::vector host_allocations; std::shared_mutex host_allocations_mutex; size_t host_import_alignment = 0; + uint32_t host_import_failed = 0; // bitmask by device->idx, guarded by host_allocations_mutex bool multi_device = false; }; diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vulkan.cpp index 436bfe3bb..fa038bb5b 100644 --- a/ggml/src/ggml-vulkan/ggml-vulkan.cpp +++ b/ggml/src/ggml-vulkan/ggml-vulkan.cpp @@ -3987,6 +3987,17 @@ void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) { } } +static bool ggml_vk_external_memory_host_usable(vk::DriverId driver_id) { + // Disable external_memory_host until https://github.com/KhronosGroup/MoltenVK/pull/2622 + // is available in the Vulkan SDK. + return driver_id != vk::DriverId::eMoltenvk; +} + +static bool ggml_vk_host_buffer_import_usable(uint32_t vendor_id) { + // amdgpu re-faults all pages of every imported host allocation on each submission + return vendor_id != VK_VENDOR_ID_AMD; +} + vk_device ggml_vk_get_device(size_t idx) { VK_LOG_DEBUG("ggml_vk_get_device(" << idx << ")"); @@ -4174,11 +4185,7 @@ vk_device ggml_vk_get_device(size_t idx) { device->vendor_id = device->properties.vendorID; device->driver_id = driver_props.driverID; - if (device->driver_id == vk::DriverId::eMoltenvk) { - // Disable external_memory_host until https://github.com/KhronosGroup/MoltenVK/pull/2622 - // is available in the Vulkan SDK. - device->external_memory_host = false; - } + device->external_memory_host = device->external_memory_host && ggml_vk_external_memory_host_usable(device->driver_id); // Implementing the async backend interfaces seems broken on older Intel HW, // see https://github.com/ggml-org/llama.cpp/issues/17302. @@ -5421,7 +5428,7 @@ void ggml_vk_instance_init() { } GGML_LOG_DEBUG("ggml_vulkan: Found %zu Vulkan devices:\n", vk_instance.device_indices.size()); - bool all_external_memory_host = vk_instance.device_indices.size() > 1; + size_t host_buffer_import_count = 0; for (size_t i = 0; i < vk_instance.device_indices.size(); i++) { vk::PhysicalDevice vkdev = devices[vk_instance.device_indices[i]]; std::vector extensionprops = vkdev.enumerateDeviceExtensionProperties(); @@ -5438,20 +5445,27 @@ void ggml_vk_instance_init() { vk_instance.device_supports_membudget.push_back(membudget_supported); + bool host_buffer_import = false; if (external_memory_host_supported) { vk::PhysicalDeviceProperties2 props2; + vk::PhysicalDeviceDriverProperties driver_props; vk::PhysicalDeviceExternalMemoryHostPropertiesEXT ext_props; - props2.pNext = &ext_props; + props2.pNext = &driver_props; + driver_props.pNext = &ext_props; vkdev.getProperties2(&props2); - vk_instance.host_import_alignment = std::max(vk_instance.host_import_alignment, ext_props.minImportedHostPointerAlignment); - } else { - all_external_memory_host = false; + host_buffer_import = ggml_vk_external_memory_host_usable(driver_props.driverID) && + ggml_vk_host_buffer_import_usable(props2.properties.vendorID); + if (host_buffer_import) { + vk_instance.host_import_alignment = std::max(vk_instance.host_import_alignment, ext_props.minImportedHostPointerAlignment); + host_buffer_import_count++; + } } + vk_instance.device_host_buffer_import.push_back(host_buffer_import); ggml_vk_print_gpu_info(i); } - vk_instance.multi_device = all_external_memory_host; + vk_instance.multi_device = host_buffer_import_count > 1 && vk_instance.device_host_buffer_import[0]; } void ggml_vk_init(ggml_backend_vk_context * ctx, size_t idx) {