From 61d447234482cc0807cab652d763e0fdb5f5d816 Mon Sep 17 00:00:00 2001 From: Jeff Bolz Date: Thu, 5 Nov 2026 06:31:44 -0600 Subject: [PATCH] vulkan: Implement test-fusion interface (#30769) --- ggml/src/ggml-vulkan/ggml-vulkan-common.h | 6 +++ ggml/src/ggml-vulkan/ggml-vulkan-debug.cpp | 52 ++++++++++++++++++++++ ggml/src/ggml-vulkan/ggml-vulkan-types.h | 3 ++ ggml/src/ggml-vulkan/ggml-vulkan.cpp | 33 +++++++++++--- 4 files changed, 89 insertions(+), 5 deletions(-) diff --git a/ggml/src/ggml-vulkan/ggml-vulkan-common.h b/ggml/src/ggml-vulkan/ggml-vulkan-common.h index 0ac0f8b83..b5491fc4b 100644 --- a/ggml/src/ggml-vulkan/ggml-vulkan-common.h +++ b/ggml/src/ggml-vulkan/ggml-vulkan-common.h @@ -245,6 +245,12 @@ bool ggml_backend_vk_comm_allreduce_tensor(void * comm_ctx, ggml_tensor ** tenso #endif // debug +void ggml_vk_count_fusion(const vk_device & device, const char * label); +void * ggml_backend_vk_fusion_get(ggml_backend_dev_t dev); +void ggml_backend_vk_fusion_stats_init(void * finfo); +void ggml_backend_vk_fusion_stats_reset(void * finfo); +int ggml_backend_vk_fusion_stats_get(void * finfo, const char ** labels, uint64_t * counts, int n); +void ggml_backend_vk_fusion_set_enabled(void * finfo, bool enabled); int64_t ggml_vk_get_op_batch_size(const ggml_tensor * op); #ifdef GGML_VULKAN_RUN_TESTS template diff --git a/ggml/src/ggml-vulkan/ggml-vulkan-debug.cpp b/ggml/src/ggml-vulkan/ggml-vulkan-debug.cpp index 97747e866..5fa919511 100644 --- a/ggml/src/ggml-vulkan/ggml-vulkan-debug.cpp +++ b/ggml/src/ggml-vulkan/ggml-vulkan-debug.cpp @@ -12,6 +12,58 @@ uint32_t vk_perf_logger_frequency = 1; std::string vk_pipeline_stats_filter; +void ggml_vk_count_fusion(const vk_device & device, const char * label) { + if (!device->fusion_stats_enabled.load(std::memory_order_relaxed)) { + return; + } + std::lock_guard lock(device->fusion_mutex); + device->fusion_counts[label]++; +} + +void * ggml_backend_vk_fusion_get(ggml_backend_dev_t dev) { + auto * ctx = (ggml_backend_vk_device_context *) dev->context; + return ggml_vk_get_device(ctx->device).get(); +} + +void ggml_backend_vk_fusion_stats_init(void * finfo) { + auto * device = (vk_device_struct *) finfo; + device->fusion_stats_enabled.store(true, std::memory_order_relaxed); +} + +void ggml_backend_vk_fusion_stats_reset(void * finfo) { + auto * device = (vk_device_struct *) finfo; + std::lock_guard lock(device->fusion_mutex); + // Keep labels and their order stable between fused and unfused runs. + for (auto & entry : device->fusion_counts) { + entry.second = 0; + } +} + +int ggml_backend_vk_fusion_stats_get(void * finfo, const char ** labels, uint64_t * counts, int n) { + auto * device = (vk_device_struct *) finfo; + std::lock_guard lock(device->fusion_mutex); + if (labels == nullptr) { + return (int) device->fusion_counts.size(); + } + int i = 0; + for (const auto & entry : device->fusion_counts) { + if (i >= n) { + break; + } + labels[i] = entry.first.c_str(); + if (counts) { + counts[i] = entry.second; + } + i++; + } + return i; +} + +void ggml_backend_vk_fusion_set_enabled(void * finfo, bool enabled) { + auto * device = (vk_device_struct *) finfo; + device->disable_fusion = !enabled; +} + void vk_memory_logger::log_allocation(vk_buffer_ref buf_ref, size_t size) { if (!vk_memory_logger_enabled) { return; diff --git a/ggml/src/ggml-vulkan/ggml-vulkan-types.h b/ggml/src/ggml-vulkan/ggml-vulkan-types.h index f529a13ae..abc01ebe2 100644 --- a/ggml/src/ggml-vulkan/ggml-vulkan-types.h +++ b/ggml/src/ggml-vulkan/ggml-vulkan-types.h @@ -1018,6 +1018,9 @@ struct vk_device_struct { ggml_backend_buffer_type buffer_type; bool disable_fusion; + std::atomic fusion_stats_enabled {false}; + std::mutex fusion_mutex; + std::map fusion_counts; bool disable_descriptor_reuse; std::atomic buffer_destroy_count {}; bool disable_host_visible_vidmem; diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vulkan.cpp index fc0ff83dd..1e203695b 100644 --- a/ggml/src/ggml-vulkan/ggml-vulkan.cpp +++ b/ggml/src/ggml-vulkan/ggml-vulkan.cpp @@ -4964,8 +4964,7 @@ vk_device ggml_vk_get_device(size_t idx) { device->disable_descriptor_reuse = getenv("GGML_VK_DISABLE_DESCRIPTOR_REUSE") != nullptr; - device->add_rms_fusion = !device->disable_fusion && - device->subgroup_arithmetic && + device->add_rms_fusion = device->subgroup_arithmetic && device->vendor_id != VK_VENDOR_ID_INTEL; device->partials_binding_alignment = std::max(4u, (uint32_t)device->properties.limits.minStorageBufferOffsetAlignment); @@ -12448,11 +12447,12 @@ bool ggml_vk_build_graph(ggml_backend_vk_context * ctx, ggml_cgraph * cgraph, in cgraph->nodes[next_node_idx]->op == GGML_OP_RMS_NORM && cgraph->nodes[next_node_idx]->src[0] == cgraph->nodes[next_node_idx - 1] && ggml_nrows(cgraph->nodes[next_node_idx]) == 1 && - ctx->device->add_rms_fusion) { + ctx->device->add_rms_fusion && !ctx->device->disable_fusion) { uint32_t size = ggml_vk_rms_partials_size(ctx, cgraph->nodes[node_idx]); ctx->do_add_rms_partials_offset_calculation = true; if (ctx->prealloc_size_add_rms_partials_offset + size <= ctx->prealloc_size_add_rms_partials) { ctx->do_add_rms_partials = true; + ggml_vk_count_fusion(ctx->device, "ADD_RMS_NORM_PARTIALS"); } } } @@ -14860,6 +14860,16 @@ static ggml_status ggml_backend_vk_graph_compute(ggml_backend_t backend, ggml_cg } if (enqueued) { + if (fusion_string && ctx->device->fusion_stats_enabled.load(std::memory_order_relaxed)) { + std::string label = fusion_string; + if (cgraph->nodes[i]->op == GGML_OP_ADD) { + label += "_" + std::to_string(ctx->num_additional_fused_ops + 1); + } + if (ctx->fused_topk_moe_scale) { + label += "_SCALE"; + } + ggml_vk_count_fusion(ctx->device, label.c_str()); + } ++submitted_nodes; #ifndef GGML_VULKAN_CHECK_RESULTS @@ -16428,9 +16438,22 @@ static void * ggml_backend_vk_reg_get_proc_address(ggml_backend_reg_t reg, const if (strcmp(name, "ggml_backend_comm_allreduce_tensor") == 0) { return (void *) ggml_backend_vk_comm_allreduce_tensor; } -#else - UNUSED(name); #endif + if (strcmp(name, "ggml_backend_fusion_get") == 0) { + return (void *) ggml_backend_vk_fusion_get; + } + if (strcmp(name, "ggml_backend_fusion_stats_init") == 0) { + return (void *) ggml_backend_vk_fusion_stats_init; + } + if (strcmp(name, "ggml_backend_fusion_stats_reset") == 0) { + return (void *) ggml_backend_vk_fusion_stats_reset; + } + if (strcmp(name, "ggml_backend_fusion_stats_get") == 0) { + return (void *) ggml_backend_vk_fusion_stats_get; + } + if (strcmp(name, "ggml_backend_fusion_set_enabled") == 0) { + return (void *) ggml_backend_vk_fusion_set_enabled; + } return nullptr; }