vulkan: Implement test-fusion interface (#30769)
This commit is contained in:
1 parent
016c1d914c
commit
61d4472344
4 files changed
+89
-5
No files matched your search
@@ -245,6 +245,12 @@ bool ggml_backend_vk_comm_allreduce_tensor(void * comm_ctx, ggml_tensor ** tenso
|
||||
#endif
|
||||
|
||||
// debug
|
||||
void ggml_vk_count_fusion(const vk_device & device, const char * label);
|
||||
void * ggml_backend_vk_fusion_get(ggml_backend_dev_t dev);
|
||||
void ggml_backend_vk_fusion_stats_init(void * finfo);
|
||||
void ggml_backend_vk_fusion_stats_reset(void * finfo);
|
||||
int ggml_backend_vk_fusion_stats_get(void * finfo, const char ** labels, uint64_t * counts, int n);
|
||||
void ggml_backend_vk_fusion_set_enabled(void * finfo, bool enabled);
|
||||
int64_t ggml_vk_get_op_batch_size(const ggml_tensor * op);
|
||||
#ifdef GGML_VULKAN_RUN_TESTS
|
||||
template <typename X_TYPE, typename Y_TYPE>
|
||||
|
||||
@@ -12,6 +12,58 @@ uint32_t vk_perf_logger_frequency = 1;
|
||||
|
||||
std::string vk_pipeline_stats_filter;
|
||||
|
||||
void ggml_vk_count_fusion(const vk_device & device, const char * label) {
|
||||
if (!device->fusion_stats_enabled.load(std::memory_order_relaxed)) {
|
||||
return;
|
||||
}
|
||||
std::lock_guard<std::mutex> lock(device->fusion_mutex);
|
||||
device->fusion_counts[label]++;
|
||||
}
|
||||
|
||||
void * ggml_backend_vk_fusion_get(ggml_backend_dev_t dev) {
|
||||
auto * ctx = (ggml_backend_vk_device_context *) dev->context;
|
||||
return ggml_vk_get_device(ctx->device).get();
|
||||
}
|
||||
|
||||
void ggml_backend_vk_fusion_stats_init(void * finfo) {
|
||||
auto * device = (vk_device_struct *) finfo;
|
||||
device->fusion_stats_enabled.store(true, std::memory_order_relaxed);
|
||||
}
|
||||
|
||||
void ggml_backend_vk_fusion_stats_reset(void * finfo) {
|
||||
auto * device = (vk_device_struct *) finfo;
|
||||
std::lock_guard<std::mutex> lock(device->fusion_mutex);
|
||||
// Keep labels and their order stable between fused and unfused runs.
|
||||
for (auto & entry : device->fusion_counts) {
|
||||
entry.second = 0;
|
||||
}
|
||||
}
|
||||
|
||||
int ggml_backend_vk_fusion_stats_get(void * finfo, const char ** labels, uint64_t * counts, int n) {
|
||||
auto * device = (vk_device_struct *) finfo;
|
||||
std::lock_guard<std::mutex> lock(device->fusion_mutex);
|
||||
if (labels == nullptr) {
|
||||
return (int) device->fusion_counts.size();
|
||||
}
|
||||
int i = 0;
|
||||
for (const auto & entry : device->fusion_counts) {
|
||||
if (i >= n) {
|
||||
break;
|
||||
}
|
||||
labels[i] = entry.first.c_str();
|
||||
if (counts) {
|
||||
counts[i] = entry.second;
|
||||
}
|
||||
i++;
|
||||
}
|
||||
return i;
|
||||
}
|
||||
|
||||
void ggml_backend_vk_fusion_set_enabled(void * finfo, bool enabled) {
|
||||
auto * device = (vk_device_struct *) finfo;
|
||||
device->disable_fusion = !enabled;
|
||||
}
|
||||
|
||||
void vk_memory_logger::log_allocation(vk_buffer_ref buf_ref, size_t size) {
|
||||
if (!vk_memory_logger_enabled) {
|
||||
return;
|
||||
|
||||
@@ -1018,6 +1018,9 @@ struct vk_device_struct {
|
||||
ggml_backend_buffer_type buffer_type;
|
||||
|
||||
bool disable_fusion;
|
||||
std::atomic<bool> fusion_stats_enabled {false};
|
||||
std::mutex fusion_mutex;
|
||||
std::map<std::string, uint64_t> fusion_counts;
|
||||
bool disable_descriptor_reuse;
|
||||
std::atomic<uint64_t> buffer_destroy_count {};
|
||||
bool disable_host_visible_vidmem;
|
||||
|
||||
@@ -4964,8 +4964,7 @@ vk_device ggml_vk_get_device(size_t idx) {
|
||||
|
||||
device->disable_descriptor_reuse = getenv("GGML_VK_DISABLE_DESCRIPTOR_REUSE") != nullptr;
|
||||
|
||||
device->add_rms_fusion = !device->disable_fusion &&
|
||||
device->subgroup_arithmetic &&
|
||||
device->add_rms_fusion = device->subgroup_arithmetic &&
|
||||
device->vendor_id != VK_VENDOR_ID_INTEL;
|
||||
device->partials_binding_alignment =
|
||||
std::max(4u, (uint32_t)device->properties.limits.minStorageBufferOffsetAlignment);
|
||||
@@ -12448,11 +12447,12 @@ bool ggml_vk_build_graph(ggml_backend_vk_context * ctx, ggml_cgraph * cgraph, in
|
||||
cgraph->nodes[next_node_idx]->op == GGML_OP_RMS_NORM &&
|
||||
cgraph->nodes[next_node_idx]->src[0] == cgraph->nodes[next_node_idx - 1] &&
|
||||
ggml_nrows(cgraph->nodes[next_node_idx]) == 1 &&
|
||||
ctx->device->add_rms_fusion) {
|
||||
ctx->device->add_rms_fusion && !ctx->device->disable_fusion) {
|
||||
uint32_t size = ggml_vk_rms_partials_size(ctx, cgraph->nodes[node_idx]);
|
||||
ctx->do_add_rms_partials_offset_calculation = true;
|
||||
if (ctx->prealloc_size_add_rms_partials_offset + size <= ctx->prealloc_size_add_rms_partials) {
|
||||
ctx->do_add_rms_partials = true;
|
||||
ggml_vk_count_fusion(ctx->device, "ADD_RMS_NORM_PARTIALS");
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -14860,6 +14860,16 @@ static ggml_status ggml_backend_vk_graph_compute(ggml_backend_t backend, ggml_cg
|
||||
}
|
||||
|
||||
if (enqueued) {
|
||||
if (fusion_string && ctx->device->fusion_stats_enabled.load(std::memory_order_relaxed)) {
|
||||
std::string label = fusion_string;
|
||||
if (cgraph->nodes[i]->op == GGML_OP_ADD) {
|
||||
label += "_" + std::to_string(ctx->num_additional_fused_ops + 1);
|
||||
}
|
||||
if (ctx->fused_topk_moe_scale) {
|
||||
label += "_SCALE";
|
||||
}
|
||||
ggml_vk_count_fusion(ctx->device, label.c_str());
|
||||
}
|
||||
++submitted_nodes;
|
||||
|
||||
#ifndef GGML_VULKAN_CHECK_RESULTS
|
||||
@@ -16428,9 +16438,22 @@ static void * ggml_backend_vk_reg_get_proc_address(ggml_backend_reg_t reg, const
|
||||
if (strcmp(name, "ggml_backend_comm_allreduce_tensor") == 0) {
|
||||
return (void *) ggml_backend_vk_comm_allreduce_tensor;
|
||||
}
|
||||
#else
|
||||
UNUSED(name);
|
||||
#endif
|
||||
if (strcmp(name, "ggml_backend_fusion_get") == 0) {
|
||||
return (void *) ggml_backend_vk_fusion_get;
|
||||
}
|
||||
if (strcmp(name, "ggml_backend_fusion_stats_init") == 0) {
|
||||
return (void *) ggml_backend_vk_fusion_stats_init;
|
||||
}
|
||||
if (strcmp(name, "ggml_backend_fusion_stats_reset") == 0) {
|
||||
return (void *) ggml_backend_vk_fusion_stats_reset;
|
||||
}
|
||||
if (strcmp(name, "ggml_backend_fusion_stats_get") == 0) {
|
||||
return (void *) ggml_backend_vk_fusion_stats_get;
|
||||
}
|
||||
if (strcmp(name, "ggml_backend_fusion_set_enabled") == 0) {
|
||||
return (void *) ggml_backend_vk_fusion_set_enabled;
|
||||
}
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
|
||||
Reference in new issue
Block a user