diff --git a/ggml/src/ggml-vulkan/CMakeLists.txt b/ggml/src/ggml-vulkan/CMakeLists.txt index d7dec92a6..682b90e16 100644 --- a/ggml/src/ggml-vulkan/CMakeLists.txt +++ b/ggml/src/ggml-vulkan/CMakeLists.txt @@ -66,6 +66,7 @@ if (Vulkan_FOUND) ggml-vulkan-push-constants.h ggml-vulkan-common.h ggml-vulkan-buffers.cpp + ggml-vulkan-comm.cpp ggml-vulkan-debug.cpp ) diff --git a/ggml/src/ggml-vulkan/ggml-vulkan-comm.cpp b/ggml/src/ggml-vulkan/ggml-vulkan-comm.cpp new file mode 100644 index 000000000..a2370b8cd --- /dev/null +++ b/ggml/src/ggml-vulkan/ggml-vulkan-comm.cpp @@ -0,0 +1,591 @@ +#include "ggml-vulkan-common.h" + +#if !defined(_WIN32) +#include +#include +#if defined(__linux__) +#include +#endif + +static constexpr size_t ggml_vk_comm_f16_min_size = 1024 * 1024; + +// follows ops ggml-backend-meta may delay a reduction across +// TODO: the required precision should come from the meta backend instead. +static bool ggml_vk_comm_needs_f32(const ggml_tensor * t, int depth = 0) { + if (t == nullptr || depth > 64) { + return false; + } + switch (t->op) { + case GGML_OP_MUL_MAT: + case GGML_OP_MUL_MAT_ID: + return ggml_get_op_params_i32(t, 0) == GGML_PREC_F32; + case GGML_OP_ADD: + return ggml_vk_comm_needs_f32(t->src[0], depth + 1) || ggml_vk_comm_needs_f32(t->src[1], depth + 1); + case GGML_OP_MUL: + case GGML_OP_ADD_ID: + return ggml_vk_comm_needs_f32(t->src[0], depth + 1); + case GGML_OP_VIEW: + return ggml_vk_comm_needs_f32(t->src[0], depth + 1); + default: + return false; + } +} + +// host_bridge: device i signals done[i], a host thread signals go[i] once all peers of i are done +// sync_fd: a host thread exchanges SYNC_FD semaphores between peers, the wait resolves in the kernel +enum class vk_comm_sync { + host_bridge, + sync_fd, +}; + +struct vk_comm_device { + ggml_backend_t backend = nullptr; + ggml_backend_vk_context * ctx = nullptr; + void * host_ptr = nullptr; + std::vector host_buf; // host_ptr imported into each peer + + struct { + PFN_vkGetSemaphoreFdKHR get_fd = nullptr; + PFN_vkImportSemaphoreFdKHR import_fd = nullptr; + vk::Semaphore sem; + std::vector imp; // [peer], pooled, reused only after ggml_vk_graph_cleanup + } sfd; + + struct { + vk::Semaphore done; + vk::Semaphore go; + } bridge; +}; + +struct ggml_backend_vk_comm_context { + std::vector devs; + vk_comm_sync mode = vk_comm_sync::host_bridge; + size_t align = 4096; + size_t cap = 0; + uint64_t round = 0; + ggml_context * tctx = nullptr; + ggml_tensor * slot = nullptr; + bool disabled = false; + + std::thread thread; + std::chrono::nanoseconds spin{ 200'000 }; + std::atomic submitted{ 0 }; + std::atomic imported{ 0 }; + std::atomic sleeping{ false }; + std::atomic stop{ false }; + std::mutex mutex; + std::condition_variable cv; +}; + +static void ggml_vk_comm_free_buffers(ggml_backend_vk_comm_context * comm) { + for (vk_comm_device & d : comm->devs) { + for (ggml_backend_buffer_t & buf : d.host_buf) { + ggml_backend_buffer_free(buf); + buf = nullptr; + } + std::free(d.host_ptr); + d.host_ptr = nullptr; + } + comm->cap = 0; +} + +static bool ggml_backend_vk_comm_ensure(ggml_backend_vk_comm_context * comm, size_t nbytes) { + if (comm->disabled) { + return false; + } + if (nbytes <= comm->cap) { + return true; + } + auto fail = [comm]() { + GGML_LOG_WARN("ggml_vulkan: shared host buffer allocation failed, using generic all-reduce\n"); + comm->disabled = true; + return false; + }; + for (vk_comm_device & d : comm->devs) { + ggml_backend_synchronize(d.backend); + } + ggml_vk_comm_free_buffers(comm); + + const size_t n = comm->devs.size(); + const size_t cap = GGML_PAD(nbytes, comm->align); + for (size_t i = 0; i < n; i++) { + vk_comm_device & d = comm->devs[i]; + if (posix_memalign(&d.host_ptr, comm->align, 2 * cap) != 0) { + d.host_ptr = nullptr; + return fail(); + } + for (size_t j = 0; j < n; j++) { + d.host_buf[j] = ggml_backend_vk_device_buffer_from_host_ptr( + ggml_backend_get_device(comm->devs[j].backend), d.host_ptr, 2 * cap, 2 * cap); + if (!d.host_buf[j]) { + return fail(); + } + } + } + comm->cap = cap; + return true; +} + +static void ggml_vk_comm_set_slot(ggml_tensor * slot, const ggml_tensor * src, ggml_type type, ggml_backend_buffer_t buf, size_t offs) { + slot->type = type; + slot->nb[0] = ggml_type_size(type); + for (int d = 0; d < GGML_MAX_DIMS; d++) { + slot->ne[d] = src->ne[d]; + if (d > 0) { + slot->nb[d] = slot->nb[d - 1] * slot->ne[d - 1]; + } + } + slot->buffer = buf; + slot->data = (char *) ggml_backend_buffer_get_base(buf) + offs; +} + +static void ggml_vk_comm_submit(ggml_backend_vk_context * ctx, vk_context & c, vk::Semaphore signal, uint64_t value) { + ggml_vk_ctx_end(c); + for (auto & cpy : c->in_memcpys) { + memcpy(cpy.dst, cpy.src, cpy.n); + } + for (auto & mset : c->memsets) { + memset(mset.dst, mset.val, mset.n); + } + c->in_memcpys.clear(); + c->memsets.clear(); + if (signal) { + c->seqs.back().back().signal_semaphores.push_back({ signal, value }); + } + ggml_vk_submit(c, {}); + ctx->submit_pending = true; + ctx->compute_ctx.reset(); +} + +static void ggml_vk_comm_notify_submitted(ggml_backend_vk_comm_context * comm, uint64_t round) { + comm->submitted.store(round); + if (comm->sleeping.load()) { + std::lock_guard lock(comm->mutex); + comm->cv.notify_one(); + } +} + +// false on stop +static bool ggml_vk_comm_wait_submitted(ggml_backend_vk_comm_context * comm, uint64_t r) { + using clock = std::chrono::steady_clock; + for (const auto t0 = clock::now(); comm->submitted.load() < r;) { + if (comm->stop.load()) { + return false; + } + if (clock::now() - t0 < comm->spin) { + YIELD(); + continue; + } + comm->sleeping.store(true); + if (comm->submitted.load() < r) { + std::unique_lock lock(comm->mutex); + comm->cv.wait(lock, [&] { return comm->submitted.load() >= r || comm->stop.load(); }); + } + comm->sleeping.store(false); + } + return true; +} + +// signals go[i] once all peers of i are done; false on stop +static bool ggml_vk_comm_bridge_release(ggml_backend_vk_comm_context * comm, uint64_t r, std::vector & seen, std::vector & sent) { + using clock = std::chrono::steady_clock; + const size_t n = comm->devs.size(); + std::fill(sent.begin(), sent.end(), false); + size_t left = n; + for (const auto t0 = clock::now();;) { + size_t pending = n; + for (size_t i = 0; i < n; i++) { + if (sent[i]) { + continue; + } + bool ready = true; + for (size_t j = 0; j < n && ready; j++) { + if (j != i && seen[j] < r) { + const vk_comm_device & p = comm->devs[j]; + seen[j] = p.ctx->device->device.getSemaphoreCounterValue(p.bridge.done); + ready = seen[j] >= r; + pending = ready ? pending : j; + } + } + if (ready) { + const vk_comm_device & d = comm->devs[i]; + d.ctx->device->device.signalSemaphore({ d.bridge.go, r }); + sent[i] = true; + left--; + } + } + if (left == 0) { + return true; + } + if (clock::now() - t0 < comm->spin) { + YIELD(); + continue; + } + const vk_comm_device & p = comm->devs[pending]; + const vk::SemaphoreWaitInfo wi{ {}, 1, &p.bridge.done, &r }; + while (p.ctx->device->device.waitSemaphores(wi, 100'000'000) != vk::Result::eSuccess) { + if (comm->stop.load()) { + return false; + } + } + } +} + +static void ggml_vk_comm_bridge_thread(ggml_backend_vk_comm_context * comm) { +#if defined(__linux__) + pthread_setname_np(pthread_self(), "ggml-vk-comm"); +#endif + std::vector seen(comm->devs.size(), 0); + std::vector sent(comm->devs.size()); + try { + for (uint64_t r = 1;; r++) { + if (!ggml_vk_comm_wait_submitted(comm, r) || !ggml_vk_comm_bridge_release(comm, r, seen, sent)) { + return; + } + } + } catch (const vk::SystemError & e) { + GGML_ABORT("ggml_vulkan: all-reduce bridge failed: %s", e.what()); + } +} + +static void ggml_vk_comm_bridge_init(ggml_backend_vk_comm_context * comm) { + vk::SemaphoreTypeCreateInfo stci{ vk::SemaphoreType::eTimeline, 0 }; + vk::SemaphoreCreateInfo sci{}; + sci.pNext = &stci; + for (vk_comm_device & d : comm->devs) { + d.bridge.done = d.ctx->device->device.createSemaphore(sci); + d.bridge.go = d.ctx->device->device.createSemaphore(sci); + } + comm->thread = std::thread(ggml_vk_comm_bridge_thread, comm); +} + +static void ggml_vk_comm_bridge_free(vk_comm_device & d) { + if (d.bridge.done) { + d.ctx->device->device.destroySemaphore(d.bridge.done); + } + if (d.bridge.go) { + d.ctx->device->device.destroySemaphore(d.bridge.go); + } +} + +static void ggml_vk_comm_barrier_to_host(vk_context & c) { + const vk::MemoryBarrier to_host{ vk::AccessFlagBits::eTransferWrite | vk::AccessFlagBits::eShaderWrite, vk::AccessFlagBits::eHostRead }; + c->s->buffer->buf.pipelineBarrier(c->p->q->stage_flags, vk::PipelineStageFlagBits::eHost, {}, { to_host }, {}, {}); +} + +// the peers' slots were made available to the host before the wait resolved, the submit predates that +static void ggml_vk_comm_barrier_from_host(vk_context & c) { + const vk::MemoryBarrier from_host{ vk::AccessFlagBits::eHostWrite, vk::AccessFlagBits::eShaderRead | vk::AccessFlagBits::eTransferRead }; + c->s->buffer->buf.pipelineBarrier(vk::PipelineStageFlagBits::eHost, c->p->q->stage_flags, {}, { from_host }, {}, {}); +} + +static void ggml_vk_comm_bridge_signal(ggml_backend_vk_comm_context * comm, size_t i, vk_context & c, uint64_t round) { + const vk_comm_device & d = comm->devs[i]; + ggml_vk_comm_barrier_to_host(c); + ggml_vk_comm_submit(d.ctx, c, d.bridge.done, round); +} + +static void ggml_vk_comm_bridge_wait(ggml_backend_vk_comm_context * comm, size_t i, vk_context & c, uint64_t round) { + c->s->wait_semaphores.push_back({ comm->devs[i].bridge.go, round }); + ggml_vk_comm_barrier_from_host(c); +} + +static bool ggml_vk_comm_sfd_supported(vk_comm_device & d) { + const vk_device & dev = d.ctx->device; + const bool mesa = dev->driver_id == vk::DriverId::eMesaRadv || + dev->driver_id == vk::DriverId::eIntelOpenSourceMESA || + dev->driver_id == vk::DriverId::eMesaNvk; + if (!mesa || !dev->external_semaphore) { + return false; + } + d.sfd.get_fd = (PFN_vkGetSemaphoreFdKHR) dev->device.getProcAddr("vkGetSemaphoreFdKHR"); + d.sfd.import_fd = (PFN_vkImportSemaphoreFdKHR) dev->device.getProcAddr("vkImportSemaphoreFdKHR"); + if (!d.sfd.get_fd || !d.sfd.import_fd) { + return false; + } + vk::PhysicalDeviceExternalSemaphoreInfo esi{ vk::ExternalSemaphoreHandleTypeFlagBits::eSyncFd }; + vk::ExternalSemaphoreProperties esp = dev->physical_device.getExternalSemaphoreProperties(esi); + const vk::ExternalSemaphoreFeatureFlags need = + vk::ExternalSemaphoreFeatureFlagBits::eExportable | vk::ExternalSemaphoreFeatureFlagBits::eImportable; + return (esp.externalSemaphoreFeatures & need) == need && + (esp.compatibleHandleTypes & vk::ExternalSemaphoreHandleTypeFlagBits::eSyncFd); +} + +// Mesa moves a queue to its submit thread on the first wait-before-signal, taking the CS ioctls off this thread +static void ggml_vk_comm_sfd_enable_submit_thread(const vk_device & dev) { + vk::SemaphoreTypeCreateInfo stci{ vk::SemaphoreType::eTimeline, 0 }; + vk::SemaphoreCreateInfo sci{}; + sci.pNext = &stci; + const vk::Semaphore sem = dev->device.createSemaphore(sci); + const vk::Fence fence = dev->device.createFence({}); + const uint64_t one = 1; + vk::TimelineSemaphoreSubmitInfo tsi{ 1, &one, 0, nullptr }; + const vk::PipelineStageFlags stage = vk::PipelineStageFlagBits::eAllCommands; + vk::SubmitInfo si{ 1, &sem, &stage, 0, nullptr, 0, nullptr }; + si.setPNext(&tsi); + dev->compute_queue->handle->submit(si, fence); + dev->device.signalSemaphore({ sem, 1 }); + VK_CHECK(dev->device.waitForFences({ fence }, true, UINT64_MAX), "ggml_vk_comm_sfd_enable_submit_thread", dev); + dev->device.destroyFence(fence); + dev->device.destroySemaphore(sem); +} + +// Requires a submitted signal on sem; the export resets it. +static int ggml_vk_comm_sfd_export(vk_comm_device & d) { + int fd = -1; + VkSemaphoreGetFdInfoKHR gi{}; + gi.sType = VK_STRUCTURE_TYPE_SEMAPHORE_GET_FD_INFO_KHR; + gi.semaphore = d.sfd.sem; + gi.handleType = VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_SYNC_FD_BIT; + const VkResult result = d.sfd.get_fd((VkDevice) d.ctx->device->device, &gi, &fd); + if (result != VK_SUCCESS) { + GGML_ABORT("ggml_vulkan: vkGetSemaphoreFdKHR failed: %d", (int) result); + } + return fd; +} + +// fd -1 imports an already signalled payload +static void ggml_vk_comm_sfd_import(vk_comm_device & d, vk::Semaphore sem, int fd) { + VkImportSemaphoreFdInfoKHR isi{}; + isi.sType = VK_STRUCTURE_TYPE_IMPORT_SEMAPHORE_FD_INFO_KHR; + isi.semaphore = sem; + isi.flags = VK_SEMAPHORE_IMPORT_TEMPORARY_BIT; + isi.handleType = VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_SYNC_FD_BIT; + isi.fd = fd; + const VkResult result = d.sfd.import_fd((VkDevice) d.ctx->device->device, &isi); + if (result != VK_SUCCESS) { + GGML_ABORT("ggml_vulkan: vkImportSemaphoreFdKHR failed: %d", (int) result); + } +} + +static void ggml_vk_comm_sfd_exchange(ggml_backend_vk_comm_context * comm, uint64_t r) { + const size_t n = comm->devs.size(); + for (size_t j = 0; j < n; j++) { + const int fd = ggml_vk_comm_sfd_export(comm->devs[j]); + const size_t last = j == n - 1 ? n - 2 : n - 1; + for (size_t i = 0; i < n; i++) { + if (i == j) { + continue; + } + const int imp_fd = fd < 0 || i == last ? fd : fcntl(fd, F_DUPFD_CLOEXEC, 0); + GGML_ASSERT(imp_fd >= 0 || fd < 0); + ggml_vk_comm_sfd_import(comm->devs[i], comm->devs[i].sfd.imp[j], imp_fd); + } + } + comm->imported.store(r, std::memory_order_release); +} + +static void ggml_vk_comm_sfd_thread(ggml_backend_vk_comm_context * comm) { +#if defined(__linux__) + pthread_setname_np(pthread_self(), "ggml-vk-comm"); +#endif + for (uint64_t r = 1; ggml_vk_comm_wait_submitted(comm, r); r++) { + ggml_vk_comm_sfd_exchange(comm, r); + } +} + +static void ggml_vk_comm_sfd_init(ggml_backend_vk_comm_context * comm) { + vk::ExportSemaphoreCreateInfo esci{ vk::ExternalSemaphoreHandleTypeFlagBits::eSyncFd }; + vk::SemaphoreCreateInfo sci{}; + sci.pNext = &esci; + for (vk_comm_device & d : comm->devs) { + d.sfd.sem = d.ctx->device->device.createSemaphore(sci); + d.sfd.imp.assign(comm->devs.size(), {}); + ggml_vk_comm_sfd_enable_submit_thread(d.ctx->device); + } + comm->thread = std::thread(ggml_vk_comm_sfd_thread, comm); +} + +static void ggml_vk_comm_sfd_free(vk_comm_device & d) { + if (d.sfd.sem) { + d.ctx->device->device.destroySemaphore(d.sfd.sem); + } +} + +static void ggml_vk_comm_sfd_signal(ggml_backend_vk_comm_context * comm, size_t i, vk_context & c) { + vk_comm_device & d = comm->devs[i]; + ggml_vk_comm_barrier_to_host(c); + ggml_vk_comm_submit(d.ctx, c, d.sfd.sem, 0); + for (size_t j = 0; j < comm->devs.size(); j++) { + d.sfd.imp[j] = j == i ? vk::Semaphore{} : ggml_vk_create_binary_semaphore(d.ctx); + } +} + +static void ggml_vk_comm_sfd_wait(ggml_backend_vk_comm_context * comm, size_t i, vk_context & c, uint64_t round) { + for (size_t j = 0; j < comm->devs.size(); j++) { + if (j != i) { + c->s->wait_semaphores.push_back({ comm->devs[i].sfd.imp[j], 0 }); + } + } + c->s->ready = &comm->imported; + c->s->ready_value = round; + ggml_vk_comm_barrier_from_host(c); +} + +void ggml_backend_vk_comm_free(void * comm_ctx) { + ggml_backend_vk_comm_context * comm = static_cast(comm_ctx); + // pending reductions need the thread + for (vk_comm_device & d : comm->devs) { + ggml_backend_synchronize(d.backend); + } + if (comm->thread.joinable()) { + { + std::lock_guard lock(comm->mutex); + comm->stop.store(true); + } + comm->cv.notify_all(); + comm->thread.join(); + } + for (vk_comm_device & d : comm->devs) { + ggml_vk_comm_sfd_free(d); + ggml_vk_comm_bridge_free(d); + } + ggml_vk_comm_free_buffers(comm); + if (comm->tctx) { + ggml_free(comm->tctx); + } + delete comm; +} + +void * ggml_backend_vk_comm_init(ggml_backend_t * backends, size_t n_backends) { + if (vk_instance.allreduce_disabled || n_backends < 2) { + return nullptr; + } + for (size_t i = 0; i < n_backends; i++) { + if (!ggml_backend_is_vk(backends[i])) { + return nullptr; + } + } + ggml_backend_vk_comm_context * comm = new ggml_backend_vk_comm_context; + comm->devs.resize(n_backends); + for (size_t i = 0; i < n_backends; i++) { + vk_comm_device & d = comm->devs[i]; + d.backend = backends[i]; + d.ctx = (ggml_backend_vk_context *) backends[i]->context; + d.host_buf.assign(n_backends, nullptr); + } + + for (size_t i = 0; i < n_backends; i++) { + const vk_device & dev = comm->devs[i].ctx->device; + comm->align = std::max(comm->align, (size_t) dev->min_imported_host_pointer_alignment); + if (!dev->external_memory_host) { + GGML_LOG_WARN("ggml_vulkan: device %zu: VK_EXT_external_memory_host missing, using generic all-reduce\n", i); + ggml_backend_vk_comm_free(comm); + return nullptr; + } + if (dev->serialize_submissions) { + GGML_LOG_WARN("ggml_vulkan: device %zu: GGML_VK_SERIALIZE_SUBMISSIONS set, using generic all-reduce\n", i); + ggml_backend_vk_comm_free(comm); + return nullptr; + } + } + + if (std::all_of(comm->devs.begin(), comm->devs.end(), ggml_vk_comm_sfd_supported)) { + comm->mode = vk_comm_sync::sync_fd; + } + + if (const char * spin = getenv("GGML_VK_ALLREDUCE_SPIN_US")) { + comm->spin = std::chrono::microseconds(std::max(0, atoi(spin))); + } + const ggml_init_params ip = { ggml_tensor_overhead(), nullptr, true }; + comm->tctx = ggml_init(ip); + comm->slot = ggml_new_tensor_1d(comm->tctx, GGML_TYPE_F32, 1); + + try { + if (comm->mode == vk_comm_sync::sync_fd) { + ggml_vk_comm_sfd_init(comm); + } else { + ggml_vk_comm_bridge_init(comm); + } + } catch (const vk::SystemError & e) { + GGML_LOG_WARN("ggml_vulkan: semaphore initialization failed: %s, using generic all-reduce\n", e.what()); + ggml_backend_vk_comm_free(comm); + return nullptr; + } + return comm; +} + +// A: partial to own host slot (round & 1), signal; B: wait on peers' A, add their slots; all A precede any B +// slot reuse: A(k) follows own B(k-1), which waited on peers' A(k-1), submitted after their B(k-2), the last reader +bool ggml_backend_vk_comm_allreduce_tensor(void * comm_ctx, ggml_tensor ** tensors) { + ggml_backend_vk_comm_context * comm = static_cast(comm_ctx); + const size_t n = comm->devs.size(); + + const int64_t ne = ggml_nelements(tensors[0]); + for (size_t i = 0; i < n; i++) { + const ggml_tensor * t = tensors[i]; + if (t->type != GGML_TYPE_F32 || ggml_nelements(t) != ne || !ggml_is_contiguous(t) || !ggml_backend_buffer_is_vk(t->buffer)) { + return false; + } + const ggml_backend_vk_context * ctx = comm->devs[i].ctx; + if (!ctx->compute_ctx.expired() && !ctx->compute_ctx.lock()->out_memcpys.empty()) { + return false; + } + } + if (ne == 0) { + return true; + } + const size_t nbytes = ggml_nbytes(tensors[0]); + if (!ggml_backend_vk_comm_ensure(comm, nbytes)) { + return false; + } + + const bool f32_wire = nbytes < ggml_vk_comm_f16_min_size || ggml_vk_comm_needs_f32(tensors[0]); + const ggml_type wire = f32_wire ? GGML_TYPE_F32 : GGML_TYPE_F16; + const uint64_t round = ++comm->round; + const size_t slot_offs = (round & 1) * comm->cap; + const size_t slot_size = ne * ggml_type_size(wire); + + for (size_t i = 0; i < n; i++) { + const vk_comm_device & d = comm->devs[i]; + ggml_backend_vk_context * ctx = d.ctx; + ggml_tensor * t = tensors[i]; + vk_buffer & tbuf = ((ggml_backend_vk_buffer_context *) t->buffer->context)->dev_buffer; + vk_buffer & hbuf = ((ggml_backend_vk_buffer_context *) d.host_buf[i]->context)->dev_buffer; + const size_t toffs = vk_tensor_offset(t) + t->view_offs; + + vk_context c = ggml_vk_get_compute_ctx(ctx); + ggml_vk_sync_buffers(ctx, c); + if (!(t->flags & GGML_TENSOR_FLAG_COMPUTE)) { + c->s->buffer->buf.fillBuffer(hbuf->buffer, slot_offs, GGML_PAD(slot_size, 4), 0); + c->s->buffer->buf.fillBuffer(tbuf->buffer, toffs, nbytes, 0); + } else if (wire == GGML_TYPE_F32) { + ggml_vk_buffer_copy_async(c, hbuf, slot_offs, tbuf, toffs, nbytes); + } else { + ggml_vk_comm_set_slot(comm->slot, t, wire, d.host_buf[i], slot_offs); + ggml_vk_cpy(ctx, c, t, comm->slot); + } + if (comm->mode == vk_comm_sync::sync_fd) { + ggml_vk_comm_sfd_signal(comm, i, c); + } else { + ggml_vk_comm_bridge_signal(comm, i, c, round); + } + } + ggml_vk_comm_notify_submitted(comm, round); + + for (size_t i = 0; i < n; i++) { + ggml_backend_vk_context * ctx = comm->devs[i].ctx; + vk_context c = ggml_vk_get_compute_ctx(ctx); + ctx->do_add_rms_partials = false; + if (comm->mode == vk_comm_sync::sync_fd) { + ggml_vk_comm_sfd_wait(comm, i, c, round); + } else { + ggml_vk_comm_bridge_wait(comm, i, c, round); + } + ggml_vk_sync_buffers(ctx, c); + for (size_t j = 0; j < n; j++) { + if (j == i) { + continue; + } + ggml_vk_comm_set_slot(comm->slot, tensors[i], wire, comm->devs[j].host_buf[i], slot_offs); + ggml_vk_add(ctx, c, tensors[i], comm->slot, tensors[i]); + ggml_vk_sync_buffers(ctx, c); + } + // small reductions go out with the next graph's first submit + if (wire != GGML_TYPE_F32 || vk_perf_logger_enabled) { + ggml_vk_comm_submit(ctx, c, {}, 0); + } + } + return true; +} +#endif diff --git a/ggml/src/ggml-vulkan/ggml-vulkan-common.h b/ggml/src/ggml-vulkan/ggml-vulkan-common.h index 66b7557be..0ac0f8b83 100644 --- a/ggml/src/ggml-vulkan/ggml-vulkan-common.h +++ b/ggml/src/ggml-vulkan/ggml-vulkan-common.h @@ -86,6 +86,7 @@ void ggml_vk_cmd_label_begin(vk::CommandBuffer buf, const char * name); void ggml_vk_ctx_end(vk_context& ctx); void ggml_vk_ctx_begin(vk_device& device, vk_context& subctx); vk_context ggml_vk_get_compute_ctx(ggml_backend_vk_context * ctx); +vk::Semaphore ggml_vk_create_binary_semaphore(ggml_backend_vk_context * ctx); vk_context ggml_vk_get_transfer_ctx(ggml_backend_vk_context * ctx); bool ggml_vk_submit_transfer_ctx(ggml_backend_vk_context * ctx); size_t ggml_vk_align_size(size_t width, size_t align); @@ -233,8 +234,16 @@ size_t ggml_backend_vk_buffer_type_get_alignment(ggml_backend_buffer_type_t buft size_t ggml_backend_vk_buffer_type_get_max_size(ggml_backend_buffer_type_t buft); size_t ggml_backend_vk_buffer_type_get_alloc_size(ggml_backend_buffer_type_t buft, const ggml_tensor * tensor); void ggml_backend_vk_free(ggml_backend_t backend); +ggml_backend_buffer_t ggml_backend_vk_device_buffer_from_host_ptr(ggml_backend_dev_t dev, void * ptr, size_t size, size_t max_tensor_size); ggml_backend_reg_t ggml_backend_vk_reg(); +// comm +#if !defined(_WIN32) +void * ggml_backend_vk_comm_init(ggml_backend_t * backends, size_t n_backends); +void ggml_backend_vk_comm_free(void * comm_ctx); +bool ggml_backend_vk_comm_allreduce_tensor(void * comm_ctx, ggml_tensor ** tensors); +#endif + // debug int64_t ggml_vk_get_op_batch_size(const ggml_tensor * op); #ifdef GGML_VULKAN_RUN_TESTS diff --git a/ggml/src/ggml-vulkan/ggml-vulkan-types.h b/ggml/src/ggml-vulkan/ggml-vulkan-types.h index a1505555a..d3b1a8dd2 100644 --- a/ggml/src/ggml-vulkan/ggml-vulkan-types.h +++ b/ggml/src/ggml-vulkan/ggml-vulkan-types.h @@ -679,6 +679,7 @@ struct vk_device_struct { uint64_t suballocation_block_size; uint64_t min_imported_host_pointer_alignment; bool external_memory_host {}; + bool external_semaphore {}; bool fp16; bool bf16; bool pipeline_robustness; @@ -1087,6 +1088,9 @@ struct vk_submission { vk_command_buffer* buffer = nullptr; std::vector wait_semaphores; std::vector signal_semaphores; + // submit waits until *ready >= ready_value + const std::atomic * ready = nullptr; + uint64_t ready_value = 0; }; typedef std::vector vk_sequence; @@ -1225,7 +1229,7 @@ struct ggml_backend_vk_context { vk_device device; - size_t semaphore_idx, event_idx; + size_t semaphore_idx, binary_semaphore_idx, event_idx; ggml_vk_garbage_collector gc; size_t prealloc_size_x, prealloc_size_y, prealloc_size_split_k, prealloc_size_add_rms_partials, prealloc_size_add_rms_partials_offset; vk_buffer prealloc_x, prealloc_y, prealloc_split_k, prealloc_add_rms_partials, sync_staging; @@ -1341,6 +1345,7 @@ struct vk_instance_t { uint32_t host_import_failed = 0; // bitmask by device->idx, guarded by host_allocations_mutex bool multi_device = false; + bool allreduce_disabled = false; }; typedef void (*ggml_vk_func_t)(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst); diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vulkan.cpp index 52d55ac8b..2b6561d43 100644 --- a/ggml/src/ggml-vulkan/ggml-vulkan.cpp +++ b/ggml/src/ggml-vulkan/ggml-vulkan.cpp @@ -934,6 +934,11 @@ void ggml_vk_submit(vk_context& ctx, vk::Fence fence) { for (const auto& sequence : ctx->seqs) { for (const auto& submission : sequence) { + if (submission.ready) { + while (submission.ready->load(std::memory_order_acquire) < submission.ready_value) { + YIELD(); + } + } stage_flags.push_back({}); idx++; tl_wait_vals.push_back({}); @@ -1082,14 +1087,16 @@ vk_context ggml_vk_create_temporary_context(vk_command_pool& p) { return result; } -static vk_semaphore * ggml_vk_create_binary_semaphore(ggml_backend_vk_context * ctx) { - VK_LOG_DEBUG("ggml_vk_create_timeline_semaphore()"); - vk::SemaphoreTypeCreateInfo tci{ vk::SemaphoreType::eBinary, 0 }; - vk::SemaphoreCreateInfo ci{}; - ci.setPNext(&tci); - vk::Semaphore semaphore = ctx->device->device.createSemaphore(ci); - ctx->gc.semaphores.push_back({ semaphore, 0 }); - return &ctx->gc.semaphores[ctx->gc.semaphores.size() - 1]; +vk::Semaphore ggml_vk_create_binary_semaphore(ggml_backend_vk_context * ctx) { + VK_LOG_DEBUG("ggml_vk_create_binary_semaphore()"); + if (ctx->binary_semaphore_idx >= ctx->gc.semaphores.size()) { + vk::SemaphoreTypeCreateInfo tci{ vk::SemaphoreType::eBinary, 0 }; + vk::SemaphoreCreateInfo ci{}; + ci.setPNext(&tci); + vk::Semaphore semaphore = ctx->device->device.createSemaphore(ci); + ctx->gc.semaphores.push_back({ semaphore, 0 }); + } + return ctx->gc.semaphores[ctx->binary_semaphore_idx++].s; } static vk_semaphore * ggml_vk_create_timeline_semaphore(ggml_backend_vk_context * ctx) { @@ -4128,6 +4135,8 @@ vk_device ggml_vk_get_device(size_t idx) { device->memory_priority = true; } else if (strcmp("VK_EXT_external_memory_host", properties.extensionName) == 0) { device->external_memory_host = true; + } else if (strcmp("VK_KHR_external_semaphore_fd", properties.extensionName) == 0) { + device->external_semaphore = true; #if defined(VK_EXT_shader_64bit_indexing) } else if (strcmp("VK_EXT_shader_64bit_indexing", properties.extensionName) == 0) { device->shader_64b_indexing = true; @@ -4475,6 +4484,9 @@ vk_device ggml_vk_get_device(size_t idx) { if (device->external_memory_host) { device_extensions.push_back("VK_EXT_external_memory_host"); } + if (device->external_semaphore) { + device_extensions.push_back("VK_KHR_external_semaphore_fd"); + } #if defined(VK_EXT_shader_64bit_indexing) VkPhysicalDeviceShader64BitIndexingFeaturesEXT shader_64bit_indexing_features {}; @@ -5287,6 +5299,7 @@ void ggml_vk_instance_init() { vk_perf_logger_concurrent = getenv("GGML_VK_PERF_LOGGER_CONCURRENT") != nullptr; vk_enable_sync_logger = getenv("GGML_VK_SYNC_LOGGER") != nullptr; vk_memory_logger_enabled = getenv("GGML_VK_MEMORY_LOGGER") != nullptr; + vk_instance.allreduce_disabled = getenv("GGML_VK_DISABLE_ALLREDUCE") != nullptr; const char* GGML_VK_PIPELINE_STATS = getenv("GGML_VK_PIPELINE_STATS"); if (GGML_VK_PIPELINE_STATS != nullptr) { vk_pipeline_stats_filter = GGML_VK_PIPELINE_STATS; @@ -5493,6 +5506,7 @@ void ggml_vk_init(ggml_backend_vk_context * ctx, size_t idx) { ctx->device = ggml_vk_get_device(idx); ctx->semaphore_idx = 0; + ctx->binary_semaphore_idx = 0; ctx->event_idx = 0; ctx->prealloc_size_x = 0; @@ -13013,10 +13027,7 @@ void ggml_vk_graph_cleanup(ggml_backend_vk_context * ctx) { ggml_vk_command_pool_cleanup(ctx->device, ctx->transfer_cmd_pool); } - for (size_t i = 0; i < ctx->gc.semaphores.size(); i++) { - ctx->device->device.destroySemaphore({ ctx->gc.semaphores[i].s }); - } - ctx->gc.semaphores.clear(); + ctx->binary_semaphore_idx = 0; for (size_t i = 0; i < ctx->gc.tl_semaphores.size(); i++) { ctx->device->device.destroySemaphore({ ctx->gc.tl_semaphores[i].s }); @@ -13064,6 +13075,11 @@ void ggml_vk_cleanup(ggml_backend_vk_context * ctx) { } ctx->gc.events.clear(); + for (auto & sem : ctx->gc.semaphores) { + ctx->device->device.destroySemaphore(sem.s); + } + ctx->gc.semaphores.clear(); + ctx->device->device.destroyFence(ctx->fence); ctx->device->device.destroyFence(ctx->almost_ready_fence); @@ -14464,7 +14480,9 @@ static ggml_status ggml_backend_vk_graph_compute(ggml_backend_t backend, ggml_cg flops_cap = 2'000'000'000ULL * ctx->device->shader_core_count; } } - uint64_t flops_per_submit = std::min(flops_cap, ctx->last_total_flops / 40u); + // small graphs gain nothing from splitting, each submit costs CPU time + uint64_t flops_per_submit = ctx->last_total_flops < 1'000'000'000ULL ? 0 : + std::clamp(ctx->last_total_flops / 40u, std::min(100'000'000ULL, flops_cap), flops_cap); auto const submit_after = [&](int start, int end) { if (ctx->device->serialize_submissions) { @@ -16266,7 +16284,7 @@ static void ggml_backend_vk_device_event_synchronize(ggml_backend_dev_t dev, ggm } } -static ggml_backend_buffer_t ggml_backend_vk_device_buffer_from_host_ptr(ggml_backend_dev_t dev, void * ptr, size_t size, size_t max_tensor_size) { +ggml_backend_buffer_t ggml_backend_vk_device_buffer_from_host_ptr(ggml_backend_dev_t dev, void * ptr, size_t size, size_t max_tensor_size) { VK_LOG_DEBUG("ggml_backend_vk_device_buffer_from_host_ptr(backend=" << dev << ", ptr=" << ptr << ", size=" << size << ")"); GGML_UNUSED(max_tensor_size); @@ -16348,11 +16366,29 @@ static ggml_backend_dev_t ggml_backend_vk_reg_get_device(ggml_backend_reg_t reg, return devices[device]; } +static void * ggml_backend_vk_reg_get_proc_address(ggml_backend_reg_t reg, const char * name) { + UNUSED(reg); +#if !defined(_WIN32) + if (strcmp(name, "ggml_backend_comm_init") == 0) { + return (void *) ggml_backend_vk_comm_init; + } + if (strcmp(name, "ggml_backend_comm_free") == 0) { + return (void *) ggml_backend_vk_comm_free; + } + if (strcmp(name, "ggml_backend_comm_allreduce_tensor") == 0) { + return (void *) ggml_backend_vk_comm_allreduce_tensor; + } +#else + UNUSED(name); +#endif + return nullptr; +} + static const struct ggml_backend_reg_i ggml_backend_vk_reg_i = { /* .get_name = */ ggml_backend_vk_reg_get_name, /* .get_device_count = */ ggml_backend_vk_reg_get_device_count, /* .get_device = */ ggml_backend_vk_reg_get_device, - /* .get_proc_address = */ NULL, + /* .get_proc_address = */ ggml_backend_vk_reg_get_proc_address, }; ggml_backend_reg_t ggml_backend_vk_reg() {