From 16f80bd0c0fa6b27e7d0ec7fcb5a0e0dc5f3fd47 Mon Sep 17 00:00:00 2001 From: Ruben Ortlam Date: Fri, 20 Nov 2026 08:38:14 +0100 Subject: [PATCH] vulkan: chunked GATED_DELTA_NET implementation (#30207) * vulkan: add cm2 chunked GATED_DELTA_NET shader * fix shmem bank conflict bottleneck * optimize further * add coopmat maintenance1 variant * optimize for RDNA4 * add gmem version RDNA4 * optimize for Ampere * fix nvidia shmem overuse * fix cm2 validation errors * improve cm2 perf * improve cm1 perf * clean up * enable by default * show cm maintenance1 marker in device info * properly check for cm_m1 features * add tests * add v32 mode, improve chunked condition * allow other subgroup sizes * vulkan: specialize the GDN CM2 workgroup size Assisted-by: Codex * vulkan: coopmat2 GDN implementation * work around Nvidia shader compiler crash * remove shared shader cm2 code --------- Co-authored-by: Jeff Bolz --- ggml/src/ggml-vulkan/CMakeLists.txt | 6 + .../ggml-vulkan/ggml-vulkan-push-constants.h | 1 + ggml/src/ggml-vulkan/ggml-vulkan-types.h | 31 ++ ggml/src/ggml-vulkan/ggml-vulkan.cpp | 212 ++++++++- .../ggml-vulkan/vulkan-shaders/CMakeLists.txt | 4 + .../feature-tests/coopmat_maintenance1.comp | 8 + .../vulkan-shaders/gated_delta_net.comp | 22 +- .../vulkan-shaders/gated_delta_net_cm.comp | 437 ++++++++++++++++++ .../vulkan-shaders/gated_delta_net_cm2.comp | 320 +++++++++++++ .../vulkan-shaders/vulkan-shaders-gen.cpp | 19 +- tests/test-backend-ops.cpp | 96 +++- 11 files changed, 1134 insertions(+), 22 deletions(-) create mode 100644 ggml/src/ggml-vulkan/vulkan-shaders/feature-tests/coopmat_maintenance1.comp create mode 100644 ggml/src/ggml-vulkan/vulkan-shaders/gated_delta_net_cm.comp create mode 100644 ggml/src/ggml-vulkan/vulkan-shaders/gated_delta_net_cm2.comp diff --git a/ggml/src/ggml-vulkan/CMakeLists.txt b/ggml/src/ggml-vulkan/CMakeLists.txt index 791753a8c..d8fa6ef40 100644 --- a/ggml/src/ggml-vulkan/CMakeLists.txt +++ b/ggml/src/ggml-vulkan/CMakeLists.txt @@ -101,6 +101,12 @@ if (Vulkan_FOUND) "GGML_VULKAN_COOPMAT2_DECODE_VECTOR_GLSLC_SUPPORT" ) + test_shader_extension_support( + "GL_EXT_cooperative_matrix_maintenance1" + "${CMAKE_CURRENT_SOURCE_DIR}/vulkan-shaders/feature-tests/coopmat_maintenance1.comp" + "GGML_VULKAN_COOPMAT_MAINTENANCE1_GLSLC_SUPPORT" + ) + test_shader_extension_support( "GL_EXT_integer_dot_product" "${CMAKE_CURRENT_SOURCE_DIR}/vulkan-shaders/feature-tests/integer_dot.comp" diff --git a/ggml/src/ggml-vulkan/ggml-vulkan-push-constants.h b/ggml/src/ggml-vulkan/ggml-vulkan-push-constants.h index 859cca181..0a04c1d56 100644 --- a/ggml/src/ggml-vulkan/ggml-vulkan-push-constants.h +++ b/ggml/src/ggml-vulkan/ggml-vulkan-push-constants.h @@ -787,6 +787,7 @@ struct vk_op_gated_delta_net_push_constants { uint32_t n_seqs; uint32_t s_off; uint32_t sq1, sq2, sq3; + uint32_t sk1, sk2, sk3; uint32_t sv1, sv2, sv3; uint32_t sb1, sb2, sb3; uint32_t neq1, rq3; diff --git a/ggml/src/ggml-vulkan/ggml-vulkan-types.h b/ggml/src/ggml-vulkan/ggml-vulkan-types.h index b4fec2da4..211948ee7 100644 --- a/ggml/src/ggml-vulkan/ggml-vulkan-types.h +++ b/ggml/src/ggml-vulkan/ggml-vulkan-types.h @@ -36,6 +36,21 @@ typedef struct VkPhysicalDeviceCooperativeMatrixDecodeVectorFeaturesNV { } VkPhysicalDeviceCooperativeMatrixDecodeVectorFeaturesNV; #endif +#ifndef VK_EXT_cooperative_matrix_maintenance1 +#define VK_EXT_cooperative_matrix_maintenance1 1 +#define VK_EXT_COOPERATIVE_MATRIX_MAINTENANCE_1_EXTENSION_NAME "VK_EXT_cooperative_matrix_maintenance1" +#define VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_COOPERATIVE_MATRIX_MAINTENANCE_1_FEATURES_EXT ((VkStructureType)1000659000) +typedef struct VkPhysicalDeviceCooperativeMatrixMaintenance1FeaturesEXT { + VkStructureType sType; + void* pNext; + VkBool32 cooperativeMatrixProperties2; + VkBool32 cooperativeMatrixReductions; + VkBool32 cooperativeMatrixConversions; + VkBool32 cooperativeMatrixPerElementOperations; + VkBool32 cooperativeMatrixGetCoordinate; +} VkPhysicalDeviceCooperativeMatrixMaintenance1FeaturesEXT; +#endif + #if __has_include() # include #elif __has_include() @@ -221,6 +236,12 @@ static bool is_pow2(uint32_t x) { return x > 1 && (x & (x-1)) == 0; } #define MAX_FUSED_ADDS (MAX_PARAMETER_COUNT - 3) +#define GGML_VK_GDN_CM_V 64u +#define GGML_VK_GDN_CM2_V 32u +#define GGML_VK_GDN_CM_D 128u +#define GGML_VK_GDN_CM_PAD 8u +#define GGML_VK_GDN_CM_LDP (GGML_VK_GDN_CM_D + GGML_VK_GDN_CM_PAD) + struct vk_pipeline_struct; typedef std::shared_ptr vk_pipeline; @@ -767,6 +788,12 @@ struct vk_device_struct { bool coopmat_acc_f32_support {}; bool coopmat_acc_f16_support {}; bool coopmat_bf16_support {}; + bool coopmat_bf16_acc_support {}; + bool coopmat_maintenance1 {}; + bool coopmat_m1_reductions {}; + bool coopmat_m1_conversions {}; + bool coopmat_m1_per_element_ops {}; + bool coopmat_m1_get_coordinate {}; bool coopmat_support_16x16x16_f16acc {}; bool coopmat_support_16x16x16_f32acc {}; bool coopmat1_fa_support {}; @@ -781,6 +808,7 @@ struct vk_device_struct { bool coopmat2; bool coopmat2_bf16_support {}; + bool coopmat2_gdn_support {}; bool coopmat2_decode_vector; bool dot2_f16 {}; @@ -987,6 +1015,9 @@ struct vk_device_struct { vk_pipeline pipeline_lightning_indexer_f32[GGML_TYPE_COUNT]; // [size_idx][kda] where size_idx: 0=d16, 1=d32, 2=d64, 3=d128 vk_pipeline pipeline_gated_delta_net[4][2]; + vk_pipeline pipeline_gated_delta_net_cm; + vk_pipeline pipeline_gated_delta_net_cm_v32; + vk_pipeline pipeline_gated_delta_net_cm_v128; vk_pipeline pipeline_ssm_scan_f32_d128; vk_pipeline pipeline_ssm_scan_f32_d256; vk_pipeline pipeline_ssm_conv_f32; diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vulkan.cpp index cd8ef9ba3..a9d60d6af 100644 --- a/ggml/src/ggml-vulkan/ggml-vulkan.cpp +++ b/ggml/src/ggml-vulkan/ggml-vulkan.cpp @@ -1667,6 +1667,22 @@ static uint32_t ggml_vk_mul_mat_vec_max_cols(const vk_device & device) { return device->coopmat2 && !device->disable_narrow_mul_mat ? 4u : mul_mat_vec_max_cols; } +static uint32_t ggml_vk_gated_delta_net_shmem_size(uint32_t v, bool coopmat2) { + const uint32_t C = 16; + const uint32_t LDP = GGML_VK_GDN_CM_LDP; + const uint32_t bf16_size = sizeof(ggml_bf16_t); + const bool gmem_mirror = !coopmat2 && v == 128; + + const uint32_t qk = 4 * C * LDP * bf16_size; + const uint32_t state = gmem_mirror ? 0 : 2 * v * LDP * bf16_size; + const uint32_t p_inv = 4 * C * C * bf16_size; + const uint32_t delta_solved = coopmat2 ? 2 * C * (v + GGML_VK_GDN_CM_PAD) * bf16_size : 4 * C * v * bf16_size; + const uint32_t lower = (coopmat2 ? C * (C - 1) / 2 : C * C) * sizeof(float); + const uint32_t scan = 4 * C * sizeof(float); + + return qk + state + p_inv + delta_solved + lower + scan; +} + static bool ggml_vk_matmul_shmem_support(const vk_device& device, const std::vector& warptile, bool mul_mat_id, ggml_type src0_type) { uint32_t lut_size = 0; @@ -4124,13 +4140,14 @@ void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) { device->subgroup_min_size <= 16u && device->subgroup_max_size >= 16u) { gdn_subgroup_size = 16u; } + const bool gdn_packed_rows = S_V == 128u && device->vendor_id == VK_VENDOR_ID_NVIDIA && + gdn_subgroup_size == 32u && device->subgroup_arithmetic && device->subgroup_clustered; uint32_t lanes_per_column; if (device->vendor_id == VK_VENDOR_ID_INTEL) { // Intel Xe: full-width reduction (min rows/lane) ~10x over the COLS_PER_WG=8 rule. lanes_per_column = std::min(gdn_subgroup_size, S_V); } else if (S_V >= 128u && device->subgroup_clustered) { - // COLS_PER_WG=8: measured optimum (Ampere sg32->4, Vega20 sg64->8). - lanes_per_column = std::max(1u, gdn_subgroup_size / 8u); + lanes_per_column = gdn_packed_rows ? 8u : std::max(1u, gdn_subgroup_size / 8u); } else { // Use largest power-of-two that divides both S_V and subgroup_size so that // (1) S_V % lanes_per_column == 0 and (2) S_V % (subgroup_size / lanes_per_column) == 0. @@ -4177,9 +4194,60 @@ void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) { for (uint32_t kda = 0; kda < 2; kda++) { ggml_vk_create_pipeline(device, device->pipeline_gated_delta_net[si][kda], gdn_names[si][kda], gdn_len, gdn_data, "main", 7, sizeof(vk_op_gated_delta_net_push_constants), - wg_denoms, {S_V, kda, gdn_subgroup_size, lanes_per_column}, 1, true, use_subgroup_ops, gdn_subgroup_size); + wg_denoms, {S_V, kda, gdn_subgroup_size, lanes_per_column, uint32_t(gdn_packed_rows)}, 1, true, use_subgroup_ops, gdn_subgroup_size); } } + + // Chunked GDN prefill uses workgroup matrices on CM2 and subgroup matrices on CM1. + const bool gdn_chunked = getenv("GGML_VK_DISABLE_GDN_CHUNKED") == nullptr; + const uint32_t gdn_shmem = device->properties.limits.maxComputeSharedMemorySize; +#if defined(VK_NV_cooperative_matrix2) && defined(GGML_VULKAN_COOPMAT2_GLSLC_SUPPORT) && defined(GGML_VULKAN_BFLOAT16_GLSLC_SUPPORT) + if (gdn_chunked && device->coopmat2 && device->coopmat2_gdn_support) { + ggml_vk_create_pipeline(device, device->pipeline_gated_delta_net_cm, + "gated_delta_net_f32_cm2", gated_delta_net_f32_cm2_len, gated_delta_net_f32_cm2_data, + "main", 7, sizeof(vk_op_gated_delta_net_push_constants), {1, 1, 1}, {}, 1, true, true, 32); + } +#endif +#if defined(GGML_VULKAN_COOPMAT_GLSLC_SUPPORT) && defined(GGML_VULKAN_COOPMAT_MAINTENANCE1_GLSLC_SUPPORT) && defined(GGML_VULKAN_BFLOAT16_GLSLC_SUPPORT) + // shader needs >= 16 lanes and 16 subgroups per WG; prefer wave64 + auto gdn_sg_ok = [&](uint32_t sg) { + const bool reachable = device->subgroup_size_control ? device->subgroup_min_size <= sg && sg <= device->subgroup_max_size + : device->subgroup_size == sg; + return reachable && 16u * sg <= device->properties.limits.maxComputeWorkGroupInvocations && + 16u * sg <= device->properties.limits.maxComputeWorkGroupSize[0]; + }; + const uint32_t gdn_sg = gdn_sg_ok(64) ? 64u : device->subgroup_size; + if (gdn_chunked && !device->coopmat2 && device->coopmat_support && device->coopmat_bf16_support && device->coopmat_m1_per_element_ops && + gdn_sg >= 16 && gdn_sg_ok(gdn_sg)) { + const bool bf16acc = device->coopmat_bf16_acc_support; + const std::vector gdn_spec = { 16u * gdn_sg }; + // V=64: state mirror in shared, 7 bindings + if (gdn_shmem >= ggml_vk_gated_delta_net_shmem_size(GGML_VK_GDN_CM_V, false)) { + ggml_vk_create_pipeline(device, device->pipeline_gated_delta_net_cm, + bf16acc ? "gated_delta_net_f32_cm1_bf16acc" : "gated_delta_net_f32_cm1", + bf16acc ? gated_delta_net_f32_cm1_bf16acc_len : gated_delta_net_f32_cm1_len, + bf16acc ? (const void *)gated_delta_net_f32_cm1_bf16acc_data : (const void *)gated_delta_net_f32_cm1_data, + "main", 7, sizeof(vk_op_gated_delta_net_push_constants), {1, 1, 1}, gdn_spec, 1, true, true, gdn_sg); + } + if (gdn_shmem >= ggml_vk_gated_delta_net_shmem_size(32u, false)) { + ggml_vk_create_pipeline(device, device->pipeline_gated_delta_net_cm_v32, + bf16acc ? "gated_delta_net_f32_cm1_bf16acc_v32" : "gated_delta_net_f32_cm1_v32", + bf16acc ? gated_delta_net_f32_cm1_bf16acc_v32_len : gated_delta_net_f32_cm1_v32_len, + bf16acc ? (const void *)gated_delta_net_f32_cm1_bf16acc_v32_data : (const void *)gated_delta_net_f32_cm1_v32_data, + "main", 7, sizeof(vk_op_gated_delta_net_push_constants), {1, 1, 1}, gdn_spec, 1, true, true, gdn_sg); + } + // V=128: state mirror in a gmem scratch buffer, 8 bindings + if (gdn_shmem >= ggml_vk_gated_delta_net_shmem_size(128u, false)) { + ggml_vk_create_pipeline(device, device->pipeline_gated_delta_net_cm_v128, + bf16acc ? "gated_delta_net_f32_cm1_bf16acc_v128" : "gated_delta_net_f32_cm1_v128", + bf16acc ? gated_delta_net_f32_cm1_bf16acc_v128_len : gated_delta_net_f32_cm1_v128_len, + bf16acc ? (const void *)gated_delta_net_f32_cm1_bf16acc_v128_data : (const void *)gated_delta_net_f32_cm1_v128_data, + "main", 8, sizeof(vk_op_gated_delta_net_push_constants), {1, 1, 1}, gdn_spec, 1, true, true, gdn_sg); + } + } +#endif + GGML_UNUSED(gdn_chunked); + GGML_UNUSED(gdn_shmem); } if (device->subgroup_arithmetic && device->subgroup_require_full_support) { @@ -4554,6 +4622,11 @@ vk_device ggml_vk_get_device(size_t idx) { } else if (strcmp("VK_NV_cooperative_matrix2", properties.extensionName) == 0 && !getenv("GGML_VK_DISABLE_COOPMAT2")) { coopmat2_support = true; +#endif +#if defined(GGML_VULKAN_COOPMAT_MAINTENANCE1_GLSLC_SUPPORT) + } else if (strcmp("VK_EXT_cooperative_matrix_maintenance1", properties.extensionName) == 0 && + !getenv("GGML_VK_DISABLE_COOPMAT_MAINTENANCE1")) { + device->coopmat_maintenance1 = true; #endif } else if (strcmp(VK_NV_COOPERATIVE_MATRIX_DECODE_VECTOR_EXTENSION_NAME, properties.extensionName) == 0 && !getenv("GGML_VK_DISABLE_COOPMAT2_DECODE_VECTOR")) { @@ -4880,6 +4953,15 @@ vk_device ggml_vk_get_device(size_t idx) { } #endif +#if defined(GGML_VULKAN_COOPMAT_MAINTENANCE1_GLSLC_SUPPORT) + VkPhysicalDeviceCooperativeMatrixMaintenance1FeaturesEXT coopmat_m1_features {}; + coopmat_m1_features.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_COOPERATIVE_MATRIX_MAINTENANCE_1_FEATURES_EXT; + if (device->coopmat_support && device->coopmat_maintenance1) { + last_struct->pNext = (VkBaseOutStructure *)&coopmat_m1_features; + last_struct = (VkBaseOutStructure *)&coopmat_m1_features; + } +#endif + VkPhysicalDeviceCooperativeMatrixDecodeVectorFeaturesNV coopmat2_decode_vector_features {}; coopmat2_decode_vector_features.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_COOPERATIVE_MATRIX_DECODE_VECTOR_FEATURES_NV; if (coopmat2_decode_vector_support) { @@ -5064,6 +5146,17 @@ vk_device ggml_vk_get_device(size_t idx) { device->coopmat1_fa_support = device->coopmat_support && device->subgroup_require_full_support; #endif +#if defined(GGML_VULKAN_COOPMAT_MAINTENANCE1_GLSLC_SUPPORT) + device->coopmat_maintenance1 = device->coopmat_support && device->coopmat_maintenance1; + if (device->coopmat_maintenance1) { + device->coopmat_m1_reductions = coopmat_m1_features.cooperativeMatrixReductions; + device->coopmat_m1_conversions = coopmat_m1_features.cooperativeMatrixConversions; + device->coopmat_m1_per_element_ops = coopmat_m1_features.cooperativeMatrixPerElementOperations; + device->coopmat_m1_get_coordinate = coopmat_m1_features.cooperativeMatrixGetCoordinate; + device_extensions.push_back(VK_EXT_COOPERATIVE_MATRIX_MAINTENANCE_1_EXTENSION_NAME); + } +#endif + if (coopmat2_support) { #if defined(VK_NV_cooperative_matrix2) && defined(GGML_VULKAN_COOPMAT2_GLSLC_SUPPORT) if (coopmat2_features.cooperativeMatrixWorkgroupScope && @@ -5147,6 +5240,13 @@ vk_device ggml_vk_get_device(size_t idx) { prop.KGranularity <= 16) { found_bf16_128 = true; } + // GDN uses M=32, with N/K dimensions that are multiples of 16. + if (prop.workgroupInvocations == 128 && + prop.MGranularity != 0 && 32 % prop.MGranularity == 0 && + prop.NGranularity != 0 && 16 % prop.NGranularity == 0 && + prop.KGranularity != 0 && 16 % prop.KGranularity == 0) { + device->coopmat2_gdn_support = true; + } if (prop.workgroupInvocations == 256 && prop.MGranularity <= 32 && prop.NGranularity <= 32 && @@ -5157,6 +5257,12 @@ vk_device ggml_vk_get_device(size_t idx) { #endif } } + device->coopmat2_gdn_support = device->coopmat2_gdn_support && + coopmat2_props.cooperativeMatrixWorkgroupScopeMaxWorkgroupSize >= 128 && + device->properties.limits.maxComputeWorkGroupInvocations >= 128 && + device->properties.limits.maxComputeWorkGroupSize[0] >= 128 && + uint64_t(ggml_vk_gated_delta_net_shmem_size(GGML_VK_GDN_CM2_V, true)) + coopmat2_props.cooperativeMatrixWorkgroupScopeReservedSharedMemory <= + device->properties.limits.maxComputeSharedMemorySize; if (found_fp16_128 && found_fp16_256 && found_fp32_128 && found_fp32_256 && coopmat2_props.cooperativeMatrixFlexibleDimensionsMaxDimension >= 512) { @@ -5274,6 +5380,18 @@ vk_device ggml_vk_get_device(size_t idx) { device->coopmat_bf16_support = true; } } + // bf16 accumulator (bf16/bf16/bf16/bf16) at the shader's fixed 16x16x16 + // shape lets the GDN coopmat1 path keep the hi/lo split as accumulators. + if (bfloat16_support && + prop.AType == VK_COMPONENT_TYPE_BFLOAT16_KHR && + prop.BType == VK_COMPONENT_TYPE_BFLOAT16_KHR && + prop.CType == VK_COMPONENT_TYPE_BFLOAT16_KHR && + prop.ResultType == VK_COMPONENT_TYPE_BFLOAT16_KHR && + (vk::ScopeKHR)prop.scope == vk::ScopeKHR::eSubgroup && + prop.MSize == 16 && prop.NSize == 16 && prop.KSize == 16 + ) { + device->coopmat_bf16_acc_support = true; + } #endif } @@ -5489,6 +5607,7 @@ static void ggml_vk_print_gpu_info(size_t idx) { bool coopmat_support = false; bool coopmat2_support = false; bool coopmat2_decode_vector_support = false; + bool coopmat_maintenance1_support = false; bool integer_dot_product = false; bool bfloat16_support = false; bool dot2_f16_support = false; @@ -5509,6 +5628,11 @@ static void ggml_vk_print_gpu_info(size_t idx) { } else if (strcmp("VK_NV_cooperative_matrix2", properties.extensionName) == 0 && !getenv("GGML_VK_DISABLE_COOPMAT2")) { coopmat2_support = true; +#endif +#if defined(GGML_VULKAN_COOPMAT_MAINTENANCE1_GLSLC_SUPPORT) + } else if (strcmp("VK_EXT_cooperative_matrix_maintenance1", properties.extensionName) == 0 && + !getenv("GGML_VK_DISABLE_COOPMAT_MAINTENANCE1")) { + coopmat_maintenance1_support = true; #endif } else if (strcmp(VK_NV_COOPERATIVE_MATRIX_DECODE_VECTOR_EXTENSION_NAME, properties.extensionName) == 0 && !getenv("GGML_VK_DISABLE_COOPMAT2_DECODE_VECTOR")) { @@ -5592,6 +5716,15 @@ static void ggml_vk_print_gpu_info(size_t idx) { } #endif +#if defined(GGML_VULKAN_COOPMAT_MAINTENANCE1_GLSLC_SUPPORT) + VkPhysicalDeviceCooperativeMatrixMaintenance1FeaturesEXT coopmat_m1_features {}; + coopmat_m1_features.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_COOPERATIVE_MATRIX_MAINTENANCE_1_FEATURES_EXT; + if (coopmat_support && coopmat_maintenance1_support) { + last_struct->pNext = (VkBaseOutStructure *)&coopmat_m1_features; + last_struct = (VkBaseOutStructure *)&coopmat_m1_features; + } +#endif + VkPhysicalDeviceShaderIntegerDotProductFeaturesKHR shader_integer_dot_product_features {}; shader_integer_dot_product_features.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_SHADER_INTEGER_DOT_PRODUCT_FEATURES_KHR; if (integer_dot_product) { @@ -5685,13 +5818,17 @@ static void ggml_vk_print_gpu_info(size_t idx) { coopmat2_support = false; #endif +#if defined(GGML_VULKAN_COOPMAT_MAINTENANCE1_GLSLC_SUPPORT) + coopmat_maintenance1_support = coopmat_maintenance1_support && coopmat_m1_features.cooperativeMatrixPerElementOperations; +#endif + coopmat2_decode_vector_support = coopmat2_decode_vector_support && coopmat2_decode_vector_features.cooperativeMatrixDecodeVector; #if !defined(GGML_VULKAN_COOPMAT2_DECODE_VECTOR_GLSLC_SUPPORT) coopmat2_decode_vector_support = false; #endif std::string matrix_cores = coopmat2_support ? (coopmat2_decode_vector_support ? "NV_coopmat2v" : "NV_coopmat2") - : coopmat_support ? "KHR_coopmat" + : coopmat_support ? (coopmat_maintenance1_support ? "KHR_coopmat_m1" : "KHR_coopmat") : "none"; bool dot2_f16 = dot2_f16_support && dot2_features.shaderMixedFloatDotProductFloat16AccFloat32; @@ -11172,6 +11309,7 @@ void ggml_vk_lightning_indexer(ggml_backend_vk_context * ctx, vk_context& subctx void ggml_vk_gated_delta_net(ggml_backend_vk_context * ctx, vk_context& subctx, ggml_tensor * dst) { const ggml_tensor * src_q = dst->src[0]; + const ggml_tensor * src_k = dst->src[1]; const ggml_tensor * src_v = dst->src[2]; const ggml_tensor * src_beta = dst->src[4]; @@ -11187,7 +11325,39 @@ void ggml_vk_gated_delta_net(ggml_backend_vk_context * ctx, vk_context& subctx, const uint32_t s_off = S_v * H * n_tokens * n_seqs; - vk_pipeline pipeline = ggml_vk_op_get_pipeline(ctx, dst->src[0], dst->src[1], dst->src[2], dst, dst->op); + const ggml_tensor * src_g = dst->src[3]; + // RDNA3: the scan is faster below ~4k tokens + const uint32_t cm_min_tokens = ctx->device->architecture == AMD_RDNA3 ? 4096u : 64u; + const bool use_cm = (ctx->device->pipeline_gated_delta_net_cm != nullptr || + ctx->device->pipeline_gated_delta_net_cm_v32 != nullptr || + ctx->device->pipeline_gated_delta_net_cm_v128 != nullptr) && + src_g->ne[0] == 1 && S_v == 128 && dst->src[0]->ne[0] == 128 && K == 1 && n_tokens >= cm_min_tokens; + + const bool use_cm2 = use_cm && ctx->device->coopmat2; + const bool have_v64 = ctx->device->pipeline_gated_delta_net_cm != nullptr; + bool use_v32 = use_cm && !use_cm2 && !have_v64 && ctx->device->pipeline_gated_delta_net_cm_v32 != nullptr; + bool use_v128 = use_cm && !use_cm2 && !have_v64 && !use_v32; + const uint32_t nsm = ctx->device->shader_core_count; + if (use_cm && !use_cm2 && have_v64 && nsm != 0) { + const uint32_t wg64 = 2u * H * n_seqs; + // V=32 only pays off while V=64 leaves at least half the cores idle + use_v32 = ctx->device->pipeline_gated_delta_net_cm_v32 != nullptr && 2u * wg64 <= nsm; + if (!use_v32 && ctx->device->pipeline_gated_delta_net_cm_v128 != nullptr) { + const uint32_t wg128 = H * n_seqs; + const uint32_t waves64 = (wg64 + nsm - 1) / nsm; + const uint32_t waves128 = (wg128 + nsm - 1) / nsm; + // prefer v128 when occupancy (wgs / (waves * nsm)) is at least as good + use_v128 = (uint64_t)wg128 * waves64 >= (uint64_t)wg64 * waves128; + } + } + + const uint32_t cm_V = use_cm2 ? GGML_VK_GDN_CM2_V : (use_v128 ? 128u : (use_v32 ? 32u : GGML_VK_GDN_CM_V)); + + vk_pipeline pipeline = use_v128 + ? ctx->device->pipeline_gated_delta_net_cm_v128 + : use_v32 ? ctx->device->pipeline_gated_delta_net_cm_v32 + : (use_cm ? ctx->device->pipeline_gated_delta_net_cm + : ggml_vk_op_get_pipeline(ctx, dst->src[0], dst->src[1], dst->src[2], dst, dst->op)); GGML_ASSERT(pipeline != nullptr); ggml_pipeline_request_descriptor_sets(ctx, pipeline, 1); @@ -11198,9 +11368,27 @@ void ggml_vk_gated_delta_net(ggml_backend_vk_context * ctx, vk_context& subctx, src_buf[i] = ggml_vk_tensor_subbuffer(ctx, dst->src[i]); } + vk_subbuffer mirror_buf{}; + if (use_v128) { + const uint32_t LDP = GGML_VK_GDN_CM_LDP; + const uint32_t blocks = (S_v / cm_V) * n_seqs * H; + const size_t scratch_size = (size_t)blocks * LDP * cm_V * 2u * sizeof(uint16_t); + if (ctx->prealloc_size_x < scratch_size) { + ctx->prealloc_size_x = scratch_size; + ggml_vk_preallocate_buffers(ctx, subctx); + } + if (ctx->prealloc_x_need_sync) { + ggml_vk_sync_buffers(ctx, subctx); + } + mirror_buf = vk_subbuffer{ ctx->prealloc_x, 0, ctx->prealloc_x->size }; + } + const uint32_t sq1 = (uint32_t)(src_q->nb[1] / sizeof(float)); const uint32_t sq2 = (uint32_t)(src_q->nb[2] / sizeof(float)); const uint32_t sq3 = (uint32_t)(src_q->nb[3] / sizeof(float)); + const uint32_t sk1 = (uint32_t)(src_k->nb[1] / sizeof(float)); + const uint32_t sk2 = (uint32_t)(src_k->nb[2] / sizeof(float)); + const uint32_t sk3 = (uint32_t)(src_k->nb[3] / sizeof(float)); const uint32_t sv1 = (uint32_t)(src_v->nb[1] / sizeof(float)); const uint32_t sv2 = (uint32_t)(src_v->nb[2] / sizeof(float)); const uint32_t sv3 = (uint32_t)(src_v->nb[3] / sizeof(float)); @@ -11215,6 +11403,7 @@ void ggml_vk_gated_delta_net(ggml_backend_vk_context * ctx, vk_context& subctx, const vk_op_gated_delta_net_push_constants pc = { H, n_tokens, n_seqs, s_off, sq1, sq2, sq3, + sk1, sk2, sk3, sv1, sv2, sv3, sb1, sb2, sb3, neq1, rq3, @@ -11222,9 +11411,16 @@ void ggml_vk_gated_delta_net(ggml_backend_vk_context * ctx, vk_context& subctx, K }; - ggml_vk_dispatch_pipeline(ctx, subctx, pipeline, - {src_buf[0], src_buf[1], src_buf[2], src_buf[3], src_buf[4], src_buf[5], dst_buf}, - pc, { H, n_seqs, S_v }); + if (use_v128) { + ggml_vk_dispatch_pipeline(ctx, subctx, pipeline, + {src_buf[0], src_buf[1], src_buf[2], src_buf[3], src_buf[4], src_buf[5], dst_buf, mirror_buf}, + pc, std::array{ S_v / cm_V, n_seqs, H }); + ctx->prealloc_x_need_sync = true; + } else { + ggml_vk_dispatch_pipeline(ctx, subctx, pipeline, + {src_buf[0], src_buf[1], src_buf[2], src_buf[3], src_buf[4], src_buf[5], dst_buf}, + pc, use_cm ? std::array{ S_v / cm_V, n_seqs, H } : std::array{ H, n_seqs, S_v }); + } } void ggml_vk_ssm_scan(ggml_backend_vk_context * ctx, vk_context& subctx, ggml_tensor * dst) { diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/CMakeLists.txt b/ggml/src/ggml-vulkan/vulkan-shaders/CMakeLists.txt index cbe7a68bf..273a80233 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/CMakeLists.txt +++ b/ggml/src/ggml-vulkan/vulkan-shaders/CMakeLists.txt @@ -15,6 +15,10 @@ if (GGML_VULKAN_COOPMAT2_DECODE_VECTOR_GLSLC_SUPPORT) add_compile_definitions(GGML_VULKAN_COOPMAT2_DECODE_VECTOR_GLSLC_SUPPORT) message(STATUS "Enabling coopmat2 decode_vector glslc support") endif() +if (GGML_VULKAN_COOPMAT_MAINTENANCE1_GLSLC_SUPPORT) + add_compile_definitions(GGML_VULKAN_COOPMAT_MAINTENANCE1_GLSLC_SUPPORT) + message(STATUS "Enabling coopmat maintenance1 glslc support") +endif() if (GGML_VULKAN_INTEGER_DOT_GLSLC_SUPPORT) add_compile_definitions(GGML_VULKAN_INTEGER_DOT_GLSLC_SUPPORT) message(STATUS "Enabling dot glslc support") diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/feature-tests/coopmat_maintenance1.comp b/ggml/src/ggml-vulkan/vulkan-shaders/feature-tests/coopmat_maintenance1.comp new file mode 100644 index 000000000..049ca2867 --- /dev/null +++ b/ggml/src/ggml-vulkan/vulkan-shaders/feature-tests/coopmat_maintenance1.comp @@ -0,0 +1,8 @@ +#version 460 + +#extension GL_KHR_cooperative_matrix : require +#extension GL_EXT_cooperative_matrix_maintenance1 : require + +void main() +{ +} diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/gated_delta_net.comp b/ggml/src/ggml-vulkan/vulkan-shaders/gated_delta_net.comp index 0e384330b..ad2ced78a 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/gated_delta_net.comp +++ b/ggml/src/ggml-vulkan/vulkan-shaders/gated_delta_net.comp @@ -15,10 +15,19 @@ layout(constant_id = 0) const uint S_V = 128; layout(constant_id = 1) const uint KDA = 0; layout(constant_id = 2) const uint SUBGROUP_SIZE = 32; layout(constant_id = 3) const uint LANES_PER_COLUMN = 32; +layout(constant_id = 4) const bool PACKED_ROWS = false; const uint COLS_PER_WG = SUBGROUP_SIZE / LANES_PER_COLUMN; const uint ROWS_PER_LANE = S_V / LANES_PER_COLUMN; +uint row_index(uint r, uint lane) { + // Keep four adjacent rows together in each lane. + if (PACKED_ROWS) { + return (r / 4u * LANES_PER_COLUMN + lane) * 4u + r % 4u; + } + return r * LANES_PER_COLUMN + lane; +} + layout(local_size_x_id = 2, local_size_y = 1, local_size_z = 1) in; layout(push_constant) uniform Parameters { @@ -27,6 +36,7 @@ layout(push_constant) uniform Parameters { uint n_seqs; uint s_off; uint sq1, sq2, sq3; + uint sk1, sk2, sk3; uint sv1, sv2, sv3; uint sb1, sb2, sb3; uint neq1, rq3; @@ -110,7 +120,7 @@ void main() { FLOAT_TYPE s_shard[ROWS_PER_LANE]; [[unroll]] for (uint r = 0; r < ROWS_PER_LANE; r++) { - s_shard[r] = FLOAT_TYPE(data_state[state_in_base + col * S_V + r * LANES_PER_COLUMN + lane]); + s_shard[r] = FLOAT_TYPE(data_state[state_in_base + col * S_V + row_index(r, lane)]); } // snapshot slot mapping: slot 0 = most recent state, slot s = s tokens back. @@ -120,7 +130,7 @@ void main() { for (uint t = 0; t < n_tokens; t++) { const uint q_off = iq3 * sq3 + t * sq2 + iq1 * sq1; - const uint k_off = q_off; + const uint k_off = iq3 * sk3 + t * sk2 + iq1 * sk1; const uint v_off = seq_id * sv3 + t * sv2 + head_id * sv1; const uint gb_off = seq_id * sb3 + t * sb2 + head_id * sb1; const FLOAT_TYPE beta_val = FLOAT_TYPE(data_beta[gb_off]); @@ -128,7 +138,7 @@ void main() { FLOAT_TYPE k_reg[ROWS_PER_LANE]; FLOAT_TYPE q_reg[ROWS_PER_LANE]; [[unroll]] for (uint r = 0; r < ROWS_PER_LANE; r++) { - const uint i = r * LANES_PER_COLUMN + lane; + const uint i = row_index(r, lane); k_reg[r] = FLOAT_TYPE(data_k[k_off + i]); q_reg[r] = FLOAT_TYPE(data_q[q_off + i]); } @@ -142,7 +152,7 @@ void main() { } else { const uint g_base = gb_off * S_V; [[unroll]] for (uint r = 0; r < ROWS_PER_LANE; r++) { - const uint i = r * LANES_PER_COLUMN + lane; + const uint i = row_index(r, lane); g_exp[r] = exp(FLOAT_TYPE(data_g[g_base + i])); } } @@ -175,7 +185,7 @@ void main() { if (target_slot >= 0 && target_slot < int(K)) { const uint slot_base = s_off + uint(target_slot) * state_size_per_snap + state_out_base; [[unroll]] for (uint r = 0; r < ROWS_PER_LANE; r++) { - data_dst[slot_base + col * S_V + r * LANES_PER_COLUMN + lane] = s_shard[r]; + data_dst[slot_base + col * S_V + row_index(r, lane)] = s_shard[r]; } } } @@ -183,7 +193,7 @@ void main() { if (K == 1u) { [[unroll]] for (uint r = 0; r < ROWS_PER_LANE; r++) { - data_dst[s_off + state_out_base + col * S_V + r * LANES_PER_COLUMN + lane] = s_shard[r]; + data_dst[s_off + state_out_base + col * S_V + row_index(r, lane)] = s_shard[r]; } } } diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/gated_delta_net_cm.comp b/ggml/src/ggml-vulkan/vulkan-shaders/gated_delta_net_cm.comp new file mode 100644 index 000000000..c5b4e4084 --- /dev/null +++ b/ggml/src/ggml-vulkan/vulkan-shaders/gated_delta_net_cm.comp @@ -0,0 +1,437 @@ +#version 450 + +#extension GL_EXT_control_flow_attributes : enable +#extension GL_EXT_shader_explicit_arithmetic_types_float16 : require +#extension GL_EXT_shader_explicit_arithmetic_types_int32 : require +#extension GL_EXT_bfloat16 : require +#extension GL_KHR_memory_scope_semantics : enable +#extension GL_KHR_shader_subgroup_basic : enable +#extension GL_KHR_shader_subgroup_arithmetic : enable +#extension GL_KHR_shader_subgroup_shuffle : enable +#extension GL_KHR_cooperative_matrix : enable +#extension GL_EXT_cooperative_matrix_maintenance1 : enable + +// Chunked gated-delta-net prefill (non-KDA, S_v == D == 128, K == 1), port of CUDA PR 29353. +// State in register tiles, mirrored ([D x V]) for the projections. +// bf16 hi/lo 3x products for ~fp32 accuracy. +#define OPD bfloat16_t + +#define C 16 // chunk size (tokens per step) +#define D 128 // head dim / value dim +#define TM 16 // tile rows +#define TN 16 // tile cols (bf16 cm has only m16n16k16; n8 would pad) +#ifndef V +#define V 64 // value columns per workgroup block (V=128 = whole head) +#endif +#define NSUB 16 // subgroups per workgroup +layout(constant_id = 0) const uint WGS = NSUB * 32; // NSUB * subgroup size +#define KT 16 // K-tile width for deep (K=D) matmuls (bf16 cm: m16n16k16 only) +#define PAD 8 // shared padding to break COL-load bank conflicts (keep LDP*2 a multiple of 16B) +#define LDP (D + PAD) + +// GMEM_MIRROR: state mirror in a gmem scratch buffer (too big for shared) + +#define STATE_TILES ((D / TM) * (V / TN)) +#define PER_WARP (STATE_TILES / NSUB) +#define VT (V / TN) // value tiles +#define CT (C / TN) // C-column tiles (gram/pmat halves) +const uint NQK = C * D / WGS; + +#define TILE_R(ti) (((ti) / VT) * TM) +#define TILE_C(ti) (((ti) % VT) * TN) + +layout(local_size_x_id = 0, local_size_y = 1, local_size_z = 1) in; + +layout(push_constant) uniform Parameters { + uint H; + uint n_tokens; + uint n_seqs; + uint s_off; + uint sq1, sq2, sq3; + uint sk1, sk2, sk3; + uint sv1, sv2, sv3; + uint sb1, sb2, sb3; + uint neq1, rq3; + float scale; + uint K; +} p; + +layout(binding = 0) readonly buffer QBuf { float data_q[]; }; +layout(binding = 1) readonly buffer KBuf { float data_k[]; }; +layout(binding = 2) readonly buffer VBuf { float data_v[]; }; +layout(binding = 3) readonly buffer GBuf { float data_g[]; }; +layout(binding = 4) readonly buffer BetaBuf { float data_beta[]; }; +layout(binding = 5) readonly buffer StateBuf { float data_state[]; }; +layout(binding = 6) buffer DstBuf { float data_dst[]; }; +#ifdef GMEM_MIRROR +layout(binding = 7) coherent buffer MirrorBuf { OPD data_mirror[]; }; +#endif + +shared OPD sh_qh[C * LDP], sh_ql[C * LDP]; // q*scale hi/lo (D padded) +shared OPD sh_kh[C * LDP], sh_kl[C * LDP]; // k hi/lo (D padded) +#ifndef GMEM_MIRROR +shared OPD sh_sh[LDP * V], sh_sl[LDP * V]; // state mirror [D x V] column-major (D padded) +#endif +shared OPD sh_dh[C * V], sh_dl[C * V]; // delta (pre-inv) / delta2 [C x V] row-major +shared OPD sh_soh[C * V], sh_sol[C * V]; // solved [C x V] row-major +shared OPD sh_ph[C * C], sh_pl[C * C]; // decayed P +shared OPD sh_ih[C * C], sh_il[C * C]; // (I + L)^-1 +shared float sh_lower[C * C]; // strictly-lower decayed L + +shared float sh_prefix[C], sh_beta[C], sh_ep[C], sh_carry[C]; + +#define SG gl_ScopeSubgroup +#define MUA gl_MatrixUseA +#define MUB gl_MatrixUseB +#define MACC gl_MatrixUseAccumulator +#define ROW gl_CooperativeMatrixLayoutRowMajor +#define COL gl_CooperativeMatrixLayoutColumnMajor + + +// 3x bf16 product accumulated into acc (hi/lo split for ~fp32 accuracy). +#define PRODUCT(acc, ah, al, bh, bl) \ + acc = coopMatMulAdd(al, bh, acc); \ + acc = coopMatMulAdd(ah, bl, acc); \ + acc = coopMatMulAdd(ah, bh, acc); + +void split(const float x, out OPD h, out OPD l) { + h = OPD(x); + l = OPD(x - float(h)); +} + +#ifdef BF16ACC +// split through a bf16 accumulator +#define SPLIT_STORE(m, arr_h, off_h, arr_l, off_l, ld, layout) { \ + const coopmat h_ = coopmat(m); \ + const coopmat l_ = coopmat(m - coopmat(h_)); \ + coopMatStore(h_, arr_h, off_h, ld, layout); \ + coopMatStore(l_, arr_l, off_l, ld, layout); \ +} + +#ifdef GMEM_MIRROR +#define STORE_MIRROR(m, off) SPLIT_STORE(m, data_mirror, mir_h + off, data_mirror, mir_l + off, LDP, COL) +#else +#define STORE_MIRROR(m, off) SPLIT_STORE(m, sh_sh, off, sh_sl, off, LDP, COL) +#endif +#define STORE_DELTA(m, off) SPLIT_STORE(m, sh_dh, off, sh_dl, off, V, ROW) +#define STORE_SOLVED(m, off) SPLIT_STORE(m, sh_soh, off, sh_sol, off, V, ROW) +#else +// split per element +#ifdef GMEM_MIRROR +float SplitColS(const in uint32_t r, const in uint32_t c, const in float elem, const in uint32_t off) { + const uint idx = off + c * LDP + r; + split(elem, data_mirror[idx], data_mirror[idx + LDP * V]); + return elem; +} +#define STORE_MIRROR(m, off) coopMatPerElementEXT(m, m, SplitColS, mir_h + off) +#else +float SplitColS(const in uint32_t r, const in uint32_t c, const in float elem, const in uint32_t off) { + const uint idx = off + c * LDP + r; + split(elem, sh_sh[idx], sh_sl[idx]); + return elem; +} +#define STORE_MIRROR(m, off) coopMatPerElementEXT(m, m, SplitColS, off) +#endif + +float SplitRowD(const in uint32_t r, const in uint32_t c, const in float elem, const in uint32_t off) { + const uint idx = off + r * V + c; + split(elem, sh_dh[idx], sh_dl[idx]); + return elem; +} + +float SplitRowSO(const in uint32_t r, const in uint32_t c, const in float elem, const in uint32_t off) { + const uint idx = off + r * V + c; + split(elem, sh_soh[idx], sh_sol[idx]); + return elem; +} + +#define STORE_DELTA(m, off) coopMatPerElementEXT(m, m, SplitRowD, off) +#define STORE_SOLVED(m, off) coopMatPerElementEXT(m, m, SplitRowSO, off) +#endif + +#ifdef GMEM_MIRROR +#define LOAD_S(h, l, off) \ + coopMatLoad(h, data_mirror, mir_h + off, LDP, COL); \ + coopMatLoad(l, data_mirror, mir_l + off, LDP, COL); +// mirror is workgroup-private +#define MIRROR_BARRIER() controlBarrier(gl_ScopeWorkgroup, gl_ScopeWorkgroup, \ + gl_StorageSemanticsBuffer | gl_StorageSemanticsShared, \ + gl_SemanticsAcquireRelease) +#else +#define LOAD_S(h, l, off) \ + coopMatLoad(h, sh_sh, off, LDP, COL); \ + coopMatLoad(l, sh_sl, off, LDP, COL); +#define MIRROR_BARRIER() barrier() +#endif + +float StoreOut(const in uint32_t r, const in uint32_t c, const in float elem, + const in uint32_t out_base, const in uint32_t valid) { + if (r < valid) { + data_dst[out_base + r * (D * p.H) + c] = elem; + } + return elem; +} + +float StoreState(const in uint32_t r, const in uint32_t c, const in float elem, + const in uint32_t base) { + data_dst[base + c * D + r] = elem; + return elem; +} + +float LoadV(const in uint32_t r, const in uint32_t c, const in float elem, + const in uint32_t v_base, const in uint32_t sv2, const in uint32_t valid) { + return r < valid ? data_v[v_base + r * sv2 + c] : 0.0; +} + +// sh_beta[r] == 0 for r >= valid +float ComputeDeltaV(const in uint32_t r, const in uint32_t c, const in float ks, + const in float vv) { + return sh_beta[r] * (vv - sh_ep[r] * ks); +} + +float ApplyExpPrefix(const in uint32_t r, const in uint32_t c, const in float qs) { + return sh_ep[r] * qs; +} + +float StoreLower(const in uint32_t r, const in uint32_t c, const in float elem, + const in uint32_t c_o) { + const uint cc = c_o + c; + if (cc < r) { + sh_lower[r * C + cc] = elem * exp(sh_prefix[r] - sh_prefix[cc]) * sh_beta[r]; + } + return elem; +} + +float StorePmat(const in uint32_t r, const in uint32_t c, const in float elem, + const in uint32_t c_o) { + const uint cc = c_o + c; + float v = 0.0; + if (cc <= r) { + v = elem * exp(sh_prefix[r] - sh_prefix[cc]); + } + split(v, sh_ph[r * C + cc], sh_pl[r * C + cc]); + return elem; +} + +void prefetch(const uint t0, const uint iq1, const uint iq3, const uint seq_id, const uint head_id, + out float q[NQK], out float k[NQK], out float g, out float b) { + const uint n = t0 < p.n_tokens ? min(C, p.n_tokens - t0) : 0u; + [[unroll]] for (uint j = 0; j < NQK; j++) { + const uint i = gl_LocalInvocationID.x + j * WGS; + const uint t = i / D; + q[j] = 0.0; + k[j] = 0.0; + if (t < n) { + const uint q_off = iq3 * p.sq3 + (t0 + t) * p.sq2 + iq1 * p.sq1 + i % D; + const uint k_off = iq3 * p.sk3 + (t0 + t) * p.sk2 + iq1 * p.sk1 + i % D; + q[j] = data_q[q_off] * p.scale; + k[j] = data_k[k_off]; + } + } + g = 0.0; + b = 0.0; + const uint t = gl_SubgroupInvocationID; + if (gl_SubgroupID == 0 && t < n) { + const uint gb = seq_id * p.sb3 + (t0 + t) * p.sb2 + head_id * p.sb1; + g = data_g[gb]; + b = data_beta[gb]; + } +} + +// sg0: gram K@K^T -> L, sg1: pmat Q@K^T -> P +void gram_pmat(const uint sg) { + const bool gram = sg < CT; + const uint c_o = (sg % CT) * TN; + coopmat x = coopmat(0); + [[unroll]] for (uint kk = 0; kk < D; kk += KT) { + coopmat ah, al; + if (gram) { + coopMatLoad(ah, sh_kh, kk, LDP, ROW); + coopMatLoad(al, sh_kl, kk, LDP, ROW); + } else { + coopMatLoad(ah, sh_qh, kk, LDP, ROW); + coopMatLoad(al, sh_ql, kk, LDP, ROW); + } + coopmat bh, bl; + coopMatLoad(bh, sh_kh, c_o * LDP + kk, LDP, COL); + coopMatLoad(bl, sh_kl, c_o * LDP + kk, LDP, COL); + PRODUCT(x, ah, al, bh, bl); + } + if (gram) { + coopMatPerElementEXT(x, x, StoreLower, c_o); + } else { + coopMatPerElementEXT(x, x, StorePmat, c_o); + } +} + +// (I + L)^-1, forward substitution: subgroup per row, lane per column +void invert(const uint sg) { + const uint col = gl_SubgroupInvocationID; + [[unroll]] for (uint row = sg; row < C; row += NSUB) { + float x = col < C ? (col == row ? 1.0 : (col < row ? -sh_lower[row * C + col] : 0.0)) : 0.0; + for (int pivot = int(row) - 1; pivot > 0; pivot--) { + const float xp = subgroupShuffle(x, uint(pivot)); + if (col < uint(pivot)) { + x -= xp * sh_lower[uint(pivot) * C + col]; + } + } + if (col < C) { + split(x, sh_ih[row * C + col], sh_il[row * C + col]); + } + } +} + +float carry_decay(const uint valid) { + const float plast = sh_prefix[valid - 1]; + if (gl_LocalInvocationID.x < C) { + sh_carry[gl_LocalInvocationID.x] = exp(plast - sh_prefix[gl_LocalInvocationID.x]); + } + return exp(plast); +} + +void main() { + const uint head_id = gl_WorkGroupID.z; + const uint seq_id = gl_WorkGroupID.y; + const uint sg = gl_SubgroupID; + const uint tid = gl_LocalInvocationID.x; + const uint v0 = gl_WorkGroupID.x * V; + + const uint iq1 = head_id % p.neq1; + const uint iq3 = seq_id / p.rq3; + + const uint state_base = (seq_id * p.H + head_id) * (D * D); + +#ifdef GMEM_MIRROR + const uint mir_stride = uint(LDP) * V; + const uint mir_block = (head_id * p.n_seqs + seq_id) * gl_NumWorkGroups.x + gl_WorkGroupID.x; + const uint mir_h = mir_block * mir_stride * 2u; + const uint mir_l = mir_h + mir_stride; +#endif + + coopmat state[PER_WARP]; + [[unroll]] for (uint j = 0; j < PER_WARP; j++) { + const uint ti = sg + j * NSUB; + coopMatLoad(state[j], data_state, state_base + (v0 + TILE_C(ti)) * D + TILE_R(ti), D, COL); + } + + float qreg[NQK], kreg[NQK], greg, breg; + prefetch(0u, iq1, iq3, seq_id, head_id, qreg, kreg, greg, breg); + + for (uint t0 = 0; t0 < p.n_tokens; t0 += C) { + const uint valid = min(C, p.n_tokens - t0); + + barrier(); + if (sg == 0) { + const uint t = gl_SubgroupInvocationID; + const float acc = subgroupInclusiveAdd(greg); + if (t < C) { + sh_prefix[t] = acc; + sh_ep[t] = exp(acc); + sh_beta[t] = breg; + } + } + [[unroll]] for (uint j = 0; j < NQK; j++) { + const uint i = tid + j * WGS; + const uint pi = (i / D) * LDP + (i % D); + split(qreg[j], sh_qh[pi], sh_ql[pi]); + split(kreg[j], sh_kh[pi], sh_kl[pi]); + } + + [[unroll]] for (uint j = 0; j < PER_WARP; j++) { + const uint ti = sg + j * NSUB; + STORE_MIRROR(state[j], TILE_C(ti) * LDP + TILE_R(ti)); + } + MIRROR_BARRIER(); + + prefetch(t0 + C, iq1, iq3, seq_id, head_id, qreg, kreg, greg, breg); + const float cdecay = carry_decay(valid); + + // qs -> base_r and ks -> delta on sg2..2+VT-1 + coopmat base_r = coopmat(0); + if (sg < 2 * CT) { + gram_pmat(sg); + } else if (sg - 2 * CT < VT) { + const uint c_o = (sg - 2 * CT) * TN; + const uint v_base = seq_id * p.sv3 + head_id * p.sv1 + v0 + c_o + t0 * p.sv2; + coopmat vmat = coopmat(0); + coopMatPerElementEXT(vmat, vmat, LoadV, v_base, p.sv2, valid); + coopmat qs = coopmat(0); + coopmat ks = coopmat(0); + [[unroll]] for (uint kk = 0; kk < D; kk += KT) { + coopmat qh, ql, kh, kl; + coopMatLoad(qh, sh_qh, kk, LDP, ROW); + coopMatLoad(ql, sh_ql, kk, LDP, ROW); + coopMatLoad(kh, sh_kh, kk, LDP, ROW); + coopMatLoad(kl, sh_kl, kk, LDP, ROW); + coopmat sbh, sbl; + LOAD_S(sbh, sbl, c_o * LDP + kk); + PRODUCT(qs, qh, ql, sbh, sbl); + PRODUCT(ks, kh, kl, sbh, sbl); + } + coopMatPerElementEXT(base_r, qs, ApplyExpPrefix); + coopmat delta; + coopMatPerElementEXT(delta, ks, ComputeDeltaV, vmat); + STORE_DELTA(delta, c_o); + } + barrier(); + + invert(sg); + barrier(); + + // solved = (I+L)^-1 @ delta (sg0..VT-1) + if (sg < VT) { + const uint c_o = sg * TN; + coopmat solved = coopmat(0); + coopmat ih, il; + coopMatLoad(ih, sh_ih, 0, C, ROW); + coopMatLoad(il, sh_il, 0, C, ROW); + coopmat dh, dl; + coopMatLoad(dh, sh_dh, c_o, V, ROW); + coopMatLoad(dl, sh_dl, c_o, V, ROW); + PRODUCT(solved, ih, il, dh, dl); + STORE_SOLVED(solved, c_o); + } + barrier(); + + // out = base + P@solved (sg2..2+VT-1) + if (sg >= 2 * CT && sg - 2 * CT < VT) { + const uint c_o = (sg - 2 * CT) * TN; + coopmat O = base_r; + coopmat ph, pl; + coopMatLoad(ph, sh_ph, 0, C, ROW); + coopMatLoad(pl, sh_pl, 0, C, ROW); + coopmat svh, svl; + coopMatLoad(svh, sh_soh, c_o, V, ROW); + coopMatLoad(svl, sh_sol, c_o, V, ROW); + PRODUCT(O, ph, pl, svh, svl); + const uint out_base = (seq_id * p.n_tokens * p.H + head_id) * D + v0 + c_o + t0 * (D * p.H); + coopMatPerElementEXT(O, O, StoreOut, out_base, valid); + } + // delta2 = solved * carry -> sh_dh/dl (all threads) + for (uint i = tid; i < C * V; i += WGS) { + const float sv = float(sh_soh[i]) + float(sh_sol[i]); + split(sv * sh_carry[i / V], sh_dh[i], sh_dl[i]); + } + barrier(); + + // state = cdecay*state + K^T @ delta2 + [[unroll]] for (uint j = 0; j < PER_WARP; j++) { + const uint ti = sg + j * NSUB; + coopmat kth, ktl; + coopMatLoad(kth, sh_kh, TILE_R(ti), LDP, COL); + coopMatLoad(ktl, sh_kl, TILE_R(ti), LDP, COL); + coopmat dh, dl; + coopMatLoad(dh, sh_dh, TILE_C(ti), V, ROW); + coopMatLoad(dl, sh_dl, TILE_C(ti), V, ROW); + state[j] = state[j] * cdecay; + PRODUCT(state[j], kth, ktl, dh, dl); + } + // no end-of-chunk barrier: covered by the chunk-top barrier + } + + [[unroll]] for (uint j = 0; j < PER_WARP; j++) { + const uint ti = sg + j * NSUB; + coopMatPerElementEXT(state[j], state[j], StoreState, + p.s_off + state_base + (v0 + TILE_C(ti)) * D + TILE_R(ti)); + } +} diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/gated_delta_net_cm2.comp b/ggml/src/ggml-vulkan/vulkan-shaders/gated_delta_net_cm2.comp new file mode 100644 index 000000000..270ef35db --- /dev/null +++ b/ggml/src/ggml-vulkan/vulkan-shaders/gated_delta_net_cm2.comp @@ -0,0 +1,320 @@ +#version 450 + +#extension GL_EXT_control_flow_attributes : require +#extension GL_EXT_shader_explicit_arithmetic_types_int32 : require +#extension GL_EXT_bfloat16 : require +#extension GL_KHR_memory_scope_semantics : require +#extension GL_KHR_shader_subgroup_basic : require +#extension GL_KHR_shader_subgroup_arithmetic : require +#extension GL_KHR_shader_subgroup_shuffle : require +#extension GL_KHR_shader_subgroup_shuffle_relative : require +#extension GL_KHR_cooperative_matrix : require +#extension GL_NV_cooperative_matrix2 : require + +// Chunked gated-delta-net prefill (non-KDA, S_v == D == 128, K == 1). +// BF16 hi/lo products use FP32 accumulators. +#define OPD bfloat16_t + +#define C 16 // chunk size (tokens per step) +#define D 128 // head dim / value dim +#define V 32 // value columns per workgroup block +#define WGS 128 +#define NSUB (WGS / 32) +#define KT 32 // K-tile width for deep (K=D) matmuls +#define PAD 8 +#define LDP (D + PAD) +#define LDV (V + PAD) +#define KQ_ROWS (2 * C) +#define NQK (C * D / WGS) + +layout(local_size_x = WGS, local_size_y = 1, local_size_z = 1) in; + +layout(push_constant) uniform Parameters { + uint H; + uint n_tokens; + uint n_seqs; + uint s_off; + uint sq1, sq2, sq3; + uint sk1, sk2, sk3; + uint sv1, sv2, sv3; + uint sb1, sb2, sb3; + uint neq1, rq3; + float scale; + uint K; +} p; + +layout(binding = 0) readonly buffer QBuf { float data_q[]; }; +layout(binding = 1) readonly buffer KBuf { float data_k[]; }; +layout(binding = 2) readonly buffer VBuf { float data_v[]; }; +layout(binding = 3) readonly buffer GBuf { float data_g[]; }; +layout(binding = 4) readonly buffer BetaBuf { float data_beta[]; }; +layout(binding = 5) readonly buffer StateBuf { float data_state[]; }; +layout(binding = 6) buffer DstBuf { float data_dst[]; }; + +// K and scaled Q occupy the two row blocks of one 32-row operand. +shared OPD sh_kqh[2 * C * LDP], sh_kql[2 * C * LDP]; +shared OPD sh_sh[V * LDP], sh_sl[V * LDP]; // state mirror [V x D] row-major (D padded) +shared OPD sh_ph[C * C], sh_pl[C * C]; +shared OPD sh_ih[C * C], sh_il[C * C]; +shared OPD sh_dh[C * LDV], sh_dl[C * LDV]; // delta / solved [C x V] row-major (V padded) +shared float sh_lower[C * (C - 1) / 2]; // packed strictly-lower Gram entries +shared float sh_prefix[C], sh_ep[C], sh_beta[C], sh_carry[C]; + +#define WG gl_ScopeWorkgroup +#define MUA gl_MatrixUseA +#define MUB gl_MatrixUseB +#define MACC gl_MatrixUseAccumulator +#define ROW gl_CooperativeMatrixLayoutRowMajor +#define COL gl_CooperativeMatrixLayoutColumnMajor + +#define PRODUCT(acc, ah, al, bh, bl) \ + acc = coopMatMulAdd(ah, bh, acc); \ + acc = coopMatMulAdd(al, bh, acc); \ + acc = coopMatMulAdd(ah, bl, acc); + +void split(const float x, out OPD h, out OPD l) { + h = OPD(x); + l = OPD(x - float(h)); +} + +float StoreGramPmat(const in uint32_t r, const in uint32_t c, const in float elem) { + if (r < C) { + if (c < r) { + sh_lower[r * (r - 1) / 2 + c] = elem * exp(sh_prefix[r] - sh_prefix[c]) * sh_beta[r]; + } + } else { + const uint row = r - C; + const float y = c <= row ? elem * exp(sh_prefix[row] - sh_prefix[c]) : 0.0; + split(y, sh_ph[row * C + c], sh_pl[row * C + c]); + } + return elem; +} + +float SplitRowS(const in uint32_t r, const in uint32_t c, const in float elem) { + const uint i = r * LDP + c; + split(elem, sh_sh[i], sh_sl[i]); + return elem; +} + +float ApplyExpPrefix(const in uint32_t r, const in uint32_t c, const in float qs) { + return sh_ep[c] * qs; +} + +float LoadV(const in uint32_t r, const in uint32_t c, const in float elem, + const in uint32_t v_base, const in uint32_t sv2, const in uint32_t valid) { + return c < valid ? data_v[v_base + c * sv2 + r] : 0.0; +} + +float StoreDeltaV(const in uint32_t r, const in uint32_t c, const in float ks, const in float vv) { + const float x = sh_beta[c] * (vv - sh_ep[c] * ks); + const uint i = c * LDV + r; + split(x, sh_dh[i], sh_dl[i]); + return ks; +} + +float StoreSolved(const in uint32_t r, const in uint32_t c, const in float elem) { + const uint i = c * LDV + r; + split(elem, sh_dh[i], sh_dl[i]); + // The state mirror is no longer needed after KS/QS. + const uint j = r * LDP + c; + split(sh_carry[c] * elem, sh_sh[j], sh_sl[j]); + return elem; +} + +float StoreOut(const in uint32_t r, const in uint32_t c, const in float elem, + const in uint32_t out_base, const in uint32_t valid) { + if (c < valid) { + data_dst[out_base + c * (p.H * D) + r] = elem; + } + return elem; +} + +void prefetch(const uint t0, const uint q_base, const uint k_base, const uint gb_base, + out float q[NQK], out float k[NQK], out float g, out float b) { + const uint valid = t0 < p.n_tokens ? min(C, p.n_tokens - t0) : 0u; + const uint r = gl_SubgroupInvocationID; + g = 0.0; + b = 0.0; + if (gl_SubgroupID == 0 && r < valid) { + const uint off = gb_base + (t0 + r) * p.sb2; + g = data_g[off]; + b = data_beta[off]; + } + [[unroll]] for (uint j = 0; j < NQK; ++j) { + const uint i = gl_LocalInvocationID.x + j * WGS; + const uint token = i / D; + q[j] = 0.0; + k[j] = 0.0; + if (token < valid) { + const uint q_off = q_base + (t0 + token) * p.sq2 + i % D; + const uint k_off = k_base + (t0 + token) * p.sk2 + i % D; + q[j] = data_q[q_off]; + k[j] = data_k[k_off]; + } + } +} + +void stage_inputs(const in float q[NQK], const in float k[NQK]) { + [[unroll]] for (uint j = 0; j < NQK; ++j) { + const uint i = gl_LocalInvocationID.x + j * WGS; + const uint token = i / D; + const uint key = i % D; + const uint ki = token * LDP + key; + const uint qi = (C + token) * LDP + key; + split(k[j], sh_kqh[ki], sh_kql[ki]); + split(p.scale * q[j], sh_kqh[qi], sh_kql[qi]); + } +} + +// [K; Q] * K^T computes Gram and P in one 32-row matrix. +void gram_pmat() { + coopmat x = coopmat(0.0); + [[unroll]] for (uint kk = 0; kk < D; kk += KT) { + coopmat ah, al; + coopmat bh, bl; + coopMatLoad(ah, sh_kqh, kk, LDP, ROW); + coopMatLoad(al, sh_kql, kk, LDP, ROW); + coopMatLoad(bh, sh_kqh, kk, LDP, COL); + coopMatLoad(bl, sh_kql, kk, LDP, COL); + PRODUCT(x, ah, al, bh, bl); + } + coopMatPerElementNV(x, x, StoreGramPmat); +} + +void invert() { + const uint col = gl_SubgroupInvocationID % C; + // Each half-subgroup covers all columns of its assigned rows. + const uint section = gl_SubgroupInvocationID / C; + const uint row_base = (gl_SubgroupID * 2u + section) * (C / (2 * NSUB)); + // Keep the diagonal zero so pivots at or above the row do nothing. + float x[C / (2 * NSUB)]; + [[unroll]] for (uint j = 0; j < C / (2 * NSUB); ++j) { + const uint row = row_base + j; + x[j] = col < row ? -sh_lower[row * (row - 1) / 2 + col] : 0.0; + } + [[dont_unroll]] for (int pivot = C - 2; pivot > 0; --pivot) { + const float lower = col < uint(pivot) ? sh_lower[uint(pivot) * uint(pivot - 1) / 2 + col] : 0.0; + [[unroll]] for (uint j = 0; j < C / (2 * NSUB); ++j) { + const float xp = subgroupShuffle(x[j], uint(pivot) + section * C); + x[j] -= xp * lower; + } + } + [[unroll]] for (uint j = 0; j < C / (2 * NSUB); ++j) { + const uint row = row_base + j; + split(col == row ? 1.0 : x[j], sh_ih[row * C + col], sh_il[row * C + col]); + } +} + +void main() { + const uint head_id = gl_WorkGroupID.z; + const uint seq_id = gl_WorkGroupID.y; + const uint v0 = gl_WorkGroupID.x * V; + const uint iq1 = head_id % p.neq1; + const uint iq3 = seq_id / p.rq3; + const uint q_base = iq3 * p.sq3 + iq1 * p.sq1; + const uint k_base = iq3 * p.sk3 + iq1 * p.sk1; + const uint v_base = seq_id * p.sv3 + head_id * p.sv1 + v0; + const uint gb_base = seq_id * p.sb3 + head_id * p.sb1; + const uint state_base = (seq_id * p.H + head_id) * (D * D) + v0 * D; + const uint out_base = (seq_id * p.n_tokens * p.H + head_id) * D + v0; + + // Transposing the state-side products gives them V=32 rows with C=16 columns. + coopmat state; + coopMatLoad(state, data_state, state_base, D, ROW); + + float qreg[NQK], kreg[NQK], greg, breg; + prefetch(0u, q_base, k_base, gb_base, qreg, kreg, greg, breg); + + [[dont_unroll]] for (uint t0 = 0; t0 < p.n_tokens; t0 += C) { + const uint valid = min(C, p.n_tokens - t0); + barrier(); + if (gl_SubgroupID == 0) { + const uint r = gl_SubgroupInvocationID; + // shuffle scan instead of subgroupInclusiveAdd: works around an NVIDIA r620 shader compiler crash + float sum = greg; + [[unroll]] for (uint d = 1; d < C; d <<= 1) { + const float t = subgroupShuffleUp(sum, d); + if (r >= d) { + sum += t; + } + } + const float last = subgroupShuffle(sum, valid - 1); + if (r < C) { + sh_prefix[r] = sum; + sh_ep[r] = exp(sum); + sh_beta[r] = breg; + sh_carry[r] = exp(last - sum); + } + } + stage_inputs(qreg, kreg); + coopmat vmat; + if (valid == C && (v_base % 4u) == 0 && (p.sv2 % 4u) == 0) { + coopMatLoad(vmat, data_v, v_base + t0 * p.sv2, p.sv2, COL); + } else { + vmat = coopmat(0.0); + coopMatPerElementNV(vmat, vmat, LoadV, v_base + t0 * p.sv2, p.sv2, valid); + } + barrier(); + // Current inputs are in shared memory; keep the next chunk in registers. + prefetch(t0 + C, q_base, k_base, gb_base, qreg, kreg, greg, breg); + + gram_pmat(); + barrier(); + invert(); + coopMatPerElementNV(state, state, SplitRowS); + barrier(); + + coopmat ks = coopmat(0.0), base_r = coopmat(0.0); + [[unroll]] for (uint kk = 0; kk < D; kk += KT) { + coopmat ah, al; + coopmat kh, kl, qh, ql; + coopMatLoad(ah, sh_sh, kk, LDP, ROW); + coopMatLoad(al, sh_sl, kk, LDP, ROW); + coopMatLoad(kh, sh_kqh, kk, LDP, COL); + coopMatLoad(kl, sh_kql, kk, LDP, COL); + coopMatLoad(qh, sh_kqh, C * LDP + kk, LDP, COL); + coopMatLoad(ql, sh_kql, C * LDP + kk, LDP, COL); + PRODUCT(ks, ah, al, kh, kl); + PRODUCT(base_r, ah, al, qh, ql); + } + coopMatPerElementNV(base_r, base_r, ApplyExpPrefix); + coopMatPerElementNV(ks, ks, StoreDeltaV, vmat); + barrier(); + + coopmat solved = coopmat(0.0); + { + coopmat rh, rl; + coopmat ih, il; + coopMatLoad(rh, sh_dh, 0, LDV, COL); + coopMatLoad(rl, sh_dl, 0, LDV, COL); + coopMatLoad(ih, sh_ih, 0, C, COL); + coopMatLoad(il, sh_il, 0, C, COL); + PRODUCT(solved, rh, rl, ih, il); + } + barrier(); + coopMatPerElementNV(solved, solved, StoreSolved); + barrier(); + + { + coopmat xh, xl; + coopmat ph, pl; + coopMatLoad(xh, sh_dh, 0, LDV, COL); + coopMatLoad(xl, sh_dl, 0, LDV, COL); + coopMatLoad(ph, sh_ph, 0, C, COL); + coopMatLoad(pl, sh_pl, 0, C, COL); + PRODUCT(base_r, xh, xl, ph, pl); + } + coopMatPerElementNV(base_r, base_r, StoreOut, out_base + t0 * (p.H * D), valid); + + coopmat xh, xl; + coopmat kh, kl; + coopMatLoad(xh, sh_sh, 0, LDP, ROW); + coopMatLoad(xl, sh_sl, 0, LDP, ROW); + coopMatLoad(kh, sh_kqh, 0, LDP, ROW); + coopMatLoad(kl, sh_kql, 0, LDP, ROW); + state *= sh_ep[valid - 1]; + PRODUCT(state, xh, xl, kh, kl); + } + + coopMatStore(state, data_dst, p.s_off + state_base, D, ROW); +} diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/vulkan-shaders-gen.cpp b/ggml/src/ggml-vulkan/vulkan-shaders/vulkan-shaders-gen.cpp index e988600e3..e20c94758 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/vulkan-shaders-gen.cpp +++ b/ggml/src/ggml-vulkan/vulkan-shaders/vulkan-shaders-gen.cpp @@ -348,7 +348,10 @@ compile_count_guard acquire_compile_slot() { } void string_to_spv_func(std::string name, std::string in_path, std::string out_path, std::map defines, bool coopmat, bool dep_file, compile_count_guard slot) { - std::string target_env = (name.find("_cm2") != std::string::npos) ? "--target-env=vulkan1.3" : "--target-env=vulkan1.2"; + // coopmat2 and the maintenance1 GDN coopmat1 variants need SPIR-V 1.6 (vulkan1.3) + const bool needs_spv16 = name.find("_cm2") != std::string::npos || + name.find("gated_delta_net_f32_cm1") != std::string::npos; + std::string target_env = needs_spv16 ? "--target-env=vulkan1.3" : "--target-env=vulkan1.2"; #ifdef _WIN32 std::vector cmd = {GLSLC, "-fshader-stage=compute", target_env, "\"" + in_path + "\"", "-o", "\"" + out_path + "\""}; @@ -1128,6 +1131,20 @@ void process_shaders() { string_to_spv("gated_delta_net_f32_nocluster", "gated_delta_net.comp", merge_maps(base_dict, {{"FLOAT_TYPE", "float"}, {"USE_SUBGROUP_ADD", "1"}, {"USE_SUBGROUP_CLUSTERED", "0"}})); string_to_spv("gated_delta_net_f32_shmem", "gated_delta_net.comp", merge_maps(base_dict, {{"FLOAT_TYPE", "float"}, {"USE_SUBGROUP_ADD", "0"}, {"USE_SUBGROUP_CLUSTERED", "0"}})); +#if defined(GGML_VULKAN_BFLOAT16_GLSLC_SUPPORT) && defined(GGML_VULKAN_COOPMAT2_GLSLC_SUPPORT) + string_to_spv("gated_delta_net_f32", "gated_delta_net_cm2.comp", base_dict, true, false, true); +#endif +#if defined(GGML_VULKAN_BFLOAT16_GLSLC_SUPPORT) && defined(GGML_VULKAN_COOPMAT_GLSLC_SUPPORT) && defined(GGML_VULKAN_COOPMAT_MAINTENANCE1_GLSLC_SUPPORT) + string_to_spv("gated_delta_net_f32", "gated_delta_net_cm.comp", merge_maps(base_dict, {{"BF16ACC", "1"}}), true, true, false, false, "_bf16acc"); + string_to_spv("gated_delta_net_f32", "gated_delta_net_cm.comp", base_dict, true, true); + string_to_spv("gated_delta_net_f32", "gated_delta_net_cm.comp", merge_maps(base_dict, {{"BF16ACC", "1"}, {"V", "32"}}), true, true, false, false, "_bf16acc_v32"); + string_to_spv("gated_delta_net_f32", "gated_delta_net_cm.comp", merge_maps(base_dict, {{"V", "32"}}), true, true, false, false, "_v32"); + // whole-head V=128 variant: state mirror in a gmem scratch buffer (won't fit LDS). + // For head counts that tail the CU grid at V=64 (e.g. 48-head 27B on 64 CUs). + string_to_spv("gated_delta_net_f32", "gated_delta_net_cm.comp", merge_maps(base_dict, {{"BF16ACC", "1"}, {"GMEM_MIRROR", "1"}, {"V", "128"}}), true, true, false, false, "_bf16acc_v128"); + string_to_spv("gated_delta_net_f32", "gated_delta_net_cm.comp", merge_maps(base_dict, {{"GMEM_MIRROR", "1"}, {"V", "128"}}), true, true, false, false, "_v128"); +#endif + string_to_spv("opt_step_adamw_f32", "opt_step_adamw.comp", merge_maps(base_dict, {{"A_TYPE", "float"}})); string_to_spv("opt_step_sgd_f32", "opt_step_sgd.comp", merge_maps(base_dict, {{"A_TYPE", "float"}})); diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cpp index a870016b0..f75d93bc3 100644 --- a/tests/test-backend-ops.cpp +++ b/tests/test-backend-ops.cpp @@ -5261,16 +5261,20 @@ struct test_gated_delta_net : public test_case { const bool permuted; const bool kda; const int64_t K; // snapshot slot count: 1 = final-only, >1 = last K states + const bool correlated; + const bool padded_v; + const bool padded_q; + const bool permuted_k; std::string vars() override { - return VARS_TO_STR9(type, head_count, head_size, n_seq_tokens, n_seqs, v_repeat, permuted, kda, K); + return VARS_TO_STR13(type, head_count, head_size, n_seq_tokens, n_seqs, v_repeat, permuted, kda, K, correlated, padded_v, padded_q, permuted_k); } test_gated_delta_net(ggml_type type = GGML_TYPE_F32, int64_t head_count = 4, int64_t head_size = 16, int64_t n_seq_tokens = 1, int64_t n_seqs = 1, - int v_repeat = 1, bool permuted = false, bool kda = false, int64_t K = 1) + int v_repeat = 1, bool permuted = false, bool kda = false, int64_t K = 1, bool correlated = false, bool padded_v = false, bool padded_q = false, bool permuted_k = false) : type(type), head_count(head_count), head_size(head_size), n_seq_tokens(n_seq_tokens), n_seqs(n_seqs), - v_repeat(v_repeat), permuted(permuted), kda(kda), K(K) {} + v_repeat(v_repeat), permuted(permuted), kda(kda), K(K), correlated(correlated), padded_v(padded_v), padded_q(padded_q), permuted_k(permuted_k) {} ggml_tensor * build_graph(ggml_context * ctx) override { ggml_tensor * q; @@ -5280,11 +5284,15 @@ struct test_gated_delta_net : public test_case { // create with dims 1 and 2 swapped, then permute back to get non-contiguous layout q = ggml_permute(ctx, ggml_new_tensor_4d(ctx, type, head_size, n_seq_tokens, head_count, n_seqs), 0, 2, 1, 3); k = ggml_permute(ctx, ggml_new_tensor_4d(ctx, type, head_size, n_seq_tokens, head_count, n_seqs), 0, 2, 1, 3); - v = ggml_permute(ctx, ggml_new_tensor_4d(ctx, type, head_size, n_seq_tokens, head_count * v_repeat, n_seqs), 0, 2, 1, 3); + v = ggml_permute(ctx, ggml_new_tensor_4d(ctx, type, head_size + padded_v, n_seq_tokens, head_count * v_repeat, n_seqs), 0, 2, 1, 3); } else { q = ggml_new_tensor_4d(ctx, type, head_size, head_count, n_seq_tokens, n_seqs); k = ggml_new_tensor_4d(ctx, type, head_size, head_count, n_seq_tokens, n_seqs); - v = ggml_new_tensor_4d(ctx, type, head_size, head_count * v_repeat, n_seq_tokens, n_seqs); + v = ggml_new_tensor_4d(ctx, type, head_size + padded_v, head_count * v_repeat, n_seq_tokens, n_seqs); + } + if (padded_v) { + ggml_set_name(v, "v"); + v = ggml_view_4d(ctx, v, head_size, head_count * v_repeat, n_seq_tokens, n_seqs, v->nb[1], v->nb[2], v->nb[3], 0); } ggml_set_name(q, "q"); ggml_set_name(k, "k"); @@ -5299,6 +5307,17 @@ struct test_gated_delta_net : public test_case { // q/k are L2-normalised in qwen35/kimi-linear before delta_net q = ggml_l2_norm(ctx, q, 1e-6f); k = ggml_l2_norm(ctx, k, 1e-6f); + // Set layouts after normalization, which produces contiguous tensors. + if (padded_q) { + ggml_tensor * q_padded = ggml_new_tensor_4d(ctx, type, head_size + 1, head_count, n_seq_tokens, n_seqs); + q_padded = ggml_view_4d(ctx, q_padded, head_size, head_count, n_seq_tokens, n_seqs, q_padded->nb[1], q_padded->nb[2], q_padded->nb[3], 0); + q = ggml_cpy(ctx, q, q_padded); + } + if (permuted_k) { + ggml_tensor * k_permuted = ggml_new_tensor_4d(ctx, type, head_size, n_seq_tokens, head_count, n_seqs); + k_permuted = ggml_permute(ctx, k_permuted, 0, 2, 1, 3); + k = ggml_cpy(ctx, k, k_permuted); + } ggml_tensor * out = ggml_gated_delta_net(ctx, q, k, v, g, beta, state, K); return out; } @@ -5307,9 +5326,17 @@ struct test_gated_delta_net : public test_case { for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != nullptr; t = ggml_get_next_tensor(ctx, t)) { if (ggml_is_view_op(t->op)) { continue; } if (strcmp(t->name, "g") == 0) { - init_tensor_uniform(t, -20.0f, -1e-4f); + init_tensor_uniform(t, correlated ? -1e-4f : -20.0f, -1e-4f); } else if (strcmp(t->name, "beta") == 0) { - init_tensor_uniform(t, 0.0f, 1.0f); + init_tensor_uniform(t, correlated ? 0.9f : 0.0f, correlated ? 0.9f : 1.0f); + } else if (correlated && strcmp(t->name, "k") == 0) { + // Identical unit keys and weak decay stress cancellation in the triangular solve. + GGML_ASSERT(t->type == GGML_TYPE_F32); + std::vector data(ggml_nelements(t), 0.0f); + for (size_t i = 0; i < data.size(); i += size_t(t->ne[0])) { + data[i] = 1.0f; + } + ggml_backend_tensor_set(t, data.data(), 0, data.size() * sizeof(float)); } else if (strcmp(t->name, "v") == 0) { init_tensor_uniform(t, -0.3f, 5.0f); } else { @@ -13634,6 +13661,15 @@ static std::vector> make_test_cases_eval() { test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 4, 64, 33, 1, 1, false, true)); test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 4, 64, 100, 1, 1, false, true)); + // KDA at D=128: decode, prefill, tails, strides, and snapshots. + test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 32, 128, 1, 1, 1, false, true)); + test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 4, 128, 64, 1, 1, false, true)); + test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 4, 128, 65, 2, 2, true, true)); + test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 4, 128, 257, 1, 1, false, true)); + test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 4, 128, 65, 1, 1, false, true, 1, true)); + test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 4, 128, 257, 1, 1, true, true, 1, true, true)); + test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 4, 128, 4, 2, 2, false, true, 4)); + // MMA-eligible GDN shapes: whole heads, value slices, tails, and the HIP prefill threshold. test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 16, 128, 65, 1, 1)); test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 16, 128, 65, 3, 4)); @@ -13645,6 +13681,30 @@ static std::vector> make_test_cases_eval() { test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 16, 128, 2048, 1, 2)); // Keep the WebGPU dispatch below 65535 workgroups (32 per token for this shape). test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 16, 128, 2047, 2, 3)); + test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 4, 128, 4097, 1, 2)); + test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 16, 128, 127, 2)); + test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 16, 128, 256, 1, 2)); + test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 16, 128, 256, 1, 3)); + test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 16, 128, 513, 1, 2)); + test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 16, 128, 512, 2, 2)); + + test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 4, 128, 257, 2, 2, true)); + test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 3, 128, 129, 1, 1, false, false, 1, false, true)); + test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 4, 128, 4095, 1, 2)); + + // Correlated keys, high beta, and weak decay stress the chunked triangular solve. + for (int64_t n_tokens : {64, 65, 257}) { + test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 4, 128, n_tokens, 1, 1, false, false, 1, true)); + } + + // Odd V strides require element loads instead of aligned matrix loads. + test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 3, 128, 65, 1, 1, false, false, 1, false, true)); + + // Q and K can have different strides in both scalar and chunked paths. + for (int64_t n_tokens : {4, 65}) { + test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 2, 128, n_tokens, 2, 2, false, false, 1, false, false, true)); + test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 2, 128, n_tokens, 2, 2, false, false, 1, false, false, false, true)); + } // K > 1: output keeps the last min(n_tokens, K) per-token snapshots, ordered most-recent-first // (slot 0 = final state, slot s = state s tokens back). @@ -14296,12 +14356,34 @@ static std::vector> make_test_cases_perf() { test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 32, 128, 256, 1)); // PP-256 test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 32, 128, 512, 1)); // PP-512 test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 32, 128, 1024, 1)); // PP-1024 + // MMA-eligible prefill sweep (16 heads, d=128, K=1): Vulkan cm2 vs CUDA MMA + test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 16, 128, 64, 1)); + test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 16, 128, 128, 1)); + test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 16, 128, 256, 1)); + test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 16, 128, 512, 1)); + test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 16, 128, 1024, 1)); + test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 16, 128, 2048, 1)); + test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 16, 128, 4096, 1)); // Small model configs (fewer heads = less GPU occupancy for autoregressive) test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 4, 128, 64, 1)); // 4h PP-64 test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 4, 128, 256, 1)); // 4h PP-256 test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 4, 128, 512, 1)); // 4h PP-512 test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 4, 128, 1024, 1)); // 4h PP-1024 test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 32, 128, 64, 1, 1, false, true)); // KDA PP-64 + test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 4, 128, 64, 1, 1, false, true)); + test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 16, 128, 256, 1, 1, false, true)); + test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 4, 128, 1024, 1, 1, false, true)); + // GDN model shapes: 35B (v_repeat=2), 27B (v_repeat=3), batch sweep + for (int T : { 2048, 4096, 8192, 16384, 32768 }) { + test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 16, 128, T, 1, 2)); + test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 16, 128, T, 1, 3)); + } + + test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 4, 128, 2048, 1)); + test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 4, 128, 4096, 1)); + test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 4, 128, 8192, 1)); + test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 20, 128, 1536, 1)); + test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 20, 128, 2048, 1)); // lightning_indexer for (int kv : { 256, 4096, 65536 }) {