vulkan: chunked GATED_DELTA_NET implementation (#30207)

* vulkan: add cm2 chunked GATED_DELTA_NET shader

* fix shmem bank conflict bottleneck

* optimize further

* add coopmat maintenance1 variant

* optimize for RDNA4

* add gmem version RDNA4

* optimize for Ampere

* fix nvidia shmem overuse

* fix cm2 validation errors

* improve cm2 perf

* improve cm1 perf

* clean up

* enable by default

* show cm maintenance1 marker in device info

* properly check for cm_m1 features

* add tests

* add v32 mode, improve chunked condition

* allow other subgroup sizes

* vulkan: specialize the GDN CM2 workgroup size

Assisted-by: Codex

* vulkan: coopmat2 GDN implementation

* work around Nvidia shader compiler crash

* remove shared shader cm2 code

---------

Co-authored-by: Jeff Bolz <jbolz@nvidia.com>
This commit is contained in:
Ruben OrtlamandJeff Bolz authored and GitHub committed 2026-11-20 09:38:14 +02:00
1 parent 2d8d003937
commit 16f80bd0c0
11 files changed
+1134 -22

No files matched your search

+6
View File
@@ -101,6 +101,12 @@ if (Vulkan_FOUND)
"GGML_VULKAN_COOPMAT2_DECODE_VECTOR_GLSLC_SUPPORT"
)
test_shader_extension_support(
"GL_EXT_cooperative_matrix_maintenance1"
"${CMAKE_CURRENT_SOURCE_DIR}/vulkan-shaders/feature-tests/coopmat_maintenance1.comp"
"GGML_VULKAN_COOPMAT_MAINTENANCE1_GLSLC_SUPPORT"
)
test_shader_extension_support(
"GL_EXT_integer_dot_product"
"${CMAKE_CURRENT_SOURCE_DIR}/vulkan-shaders/feature-tests/integer_dot.comp"
@@ -787,6 +787,7 @@ struct vk_op_gated_delta_net_push_constants {
uint32_t n_seqs;
uint32_t s_off;
uint32_t sq1, sq2, sq3;
uint32_t sk1, sk2, sk3;
uint32_t sv1, sv2, sv3;
uint32_t sb1, sb2, sb3;
uint32_t neq1, rq3;
+31
View File
@@ -36,6 +36,21 @@ typedef struct VkPhysicalDeviceCooperativeMatrixDecodeVectorFeaturesNV {
} VkPhysicalDeviceCooperativeMatrixDecodeVectorFeaturesNV;
#endif
#ifndef VK_EXT_cooperative_matrix_maintenance1
#define VK_EXT_cooperative_matrix_maintenance1 1
#define VK_EXT_COOPERATIVE_MATRIX_MAINTENANCE_1_EXTENSION_NAME "VK_EXT_cooperative_matrix_maintenance1"
#define VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_COOPERATIVE_MATRIX_MAINTENANCE_1_FEATURES_EXT ((VkStructureType)1000659000)
typedef struct VkPhysicalDeviceCooperativeMatrixMaintenance1FeaturesEXT {
VkStructureType sType;
void* pNext;
VkBool32 cooperativeMatrixProperties2;
VkBool32 cooperativeMatrixReductions;
VkBool32 cooperativeMatrixConversions;
VkBool32 cooperativeMatrixPerElementOperations;
VkBool32 cooperativeMatrixGetCoordinate;
} VkPhysicalDeviceCooperativeMatrixMaintenance1FeaturesEXT;
#endif
#if __has_include(<spirv/unified1/spirv.hpp>)
# include <spirv/unified1/spirv.hpp>
#elif __has_include(<spirv-headers/spirv.hpp>)
@@ -221,6 +236,12 @@ static bool is_pow2(uint32_t x) { return x > 1 && (x & (x-1)) == 0; }
#define MAX_FUSED_ADDS (MAX_PARAMETER_COUNT - 3)
#define GGML_VK_GDN_CM_V 64u
#define GGML_VK_GDN_CM2_V 32u
#define GGML_VK_GDN_CM_D 128u
#define GGML_VK_GDN_CM_PAD 8u
#define GGML_VK_GDN_CM_LDP (GGML_VK_GDN_CM_D + GGML_VK_GDN_CM_PAD)
struct vk_pipeline_struct;
typedef std::shared_ptr<struct vk_pipeline_struct> vk_pipeline;
@@ -767,6 +788,12 @@ struct vk_device_struct {
bool coopmat_acc_f32_support {};
bool coopmat_acc_f16_support {};
bool coopmat_bf16_support {};
bool coopmat_bf16_acc_support {};
bool coopmat_maintenance1 {};
bool coopmat_m1_reductions {};
bool coopmat_m1_conversions {};
bool coopmat_m1_per_element_ops {};
bool coopmat_m1_get_coordinate {};
bool coopmat_support_16x16x16_f16acc {};
bool coopmat_support_16x16x16_f32acc {};
bool coopmat1_fa_support {};
@@ -781,6 +808,7 @@ struct vk_device_struct {
bool coopmat2;
bool coopmat2_bf16_support {};
bool coopmat2_gdn_support {};
bool coopmat2_decode_vector;
bool dot2_f16 {};
@@ -987,6 +1015,9 @@ struct vk_device_struct {
vk_pipeline pipeline_lightning_indexer_f32[GGML_TYPE_COUNT];
// [size_idx][kda] where size_idx: 0=d16, 1=d32, 2=d64, 3=d128
vk_pipeline pipeline_gated_delta_net[4][2];
vk_pipeline pipeline_gated_delta_net_cm;
vk_pipeline pipeline_gated_delta_net_cm_v32;
vk_pipeline pipeline_gated_delta_net_cm_v128;
vk_pipeline pipeline_ssm_scan_f32_d128;
vk_pipeline pipeline_ssm_scan_f32_d256;
vk_pipeline pipeline_ssm_conv_f32;
+204 -8
View File
@@ -1667,6 +1667,22 @@ static uint32_t ggml_vk_mul_mat_vec_max_cols(const vk_device & device) {
return device->coopmat2 && !device->disable_narrow_mul_mat ? 4u : mul_mat_vec_max_cols;
}
static uint32_t ggml_vk_gated_delta_net_shmem_size(uint32_t v, bool coopmat2) {
const uint32_t C = 16;
const uint32_t LDP = GGML_VK_GDN_CM_LDP;
const uint32_t bf16_size = sizeof(ggml_bf16_t);
const bool gmem_mirror = !coopmat2 && v == 128;
const uint32_t qk = 4 * C * LDP * bf16_size;
const uint32_t state = gmem_mirror ? 0 : 2 * v * LDP * bf16_size;
const uint32_t p_inv = 4 * C * C * bf16_size;
const uint32_t delta_solved = coopmat2 ? 2 * C * (v + GGML_VK_GDN_CM_PAD) * bf16_size : 4 * C * v * bf16_size;
const uint32_t lower = (coopmat2 ? C * (C - 1) / 2 : C * C) * sizeof(float);
const uint32_t scan = 4 * C * sizeof(float);
return qk + state + p_inv + delta_solved + lower + scan;
}
static bool ggml_vk_matmul_shmem_support(const vk_device& device, const std::vector<uint32_t>& warptile, bool mul_mat_id, ggml_type src0_type) {
uint32_t lut_size = 0;
@@ -4124,13 +4140,14 @@ void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) {
device->subgroup_min_size <= 16u && device->subgroup_max_size >= 16u) {
gdn_subgroup_size = 16u;
}
const bool gdn_packed_rows = S_V == 128u && device->vendor_id == VK_VENDOR_ID_NVIDIA &&
gdn_subgroup_size == 32u && device->subgroup_arithmetic && device->subgroup_clustered;
uint32_t lanes_per_column;
if (device->vendor_id == VK_VENDOR_ID_INTEL) {
// Intel Xe: full-width reduction (min rows/lane) ~10x over the COLS_PER_WG=8 rule.
lanes_per_column = std::min(gdn_subgroup_size, S_V);
} else if (S_V >= 128u && device->subgroup_clustered) {
// COLS_PER_WG=8: measured optimum (Ampere sg32->4, Vega20 sg64->8).
lanes_per_column = std::max(1u, gdn_subgroup_size / 8u);
lanes_per_column = gdn_packed_rows ? 8u : std::max(1u, gdn_subgroup_size / 8u);
} else {
// Use largest power-of-two that divides both S_V and subgroup_size so that
// (1) S_V % lanes_per_column == 0 and (2) S_V % (subgroup_size / lanes_per_column) == 0.
@@ -4177,9 +4194,60 @@ void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) {
for (uint32_t kda = 0; kda < 2; kda++) {
ggml_vk_create_pipeline(device, device->pipeline_gated_delta_net[si][kda],
gdn_names[si][kda], gdn_len, gdn_data, "main", 7, sizeof(vk_op_gated_delta_net_push_constants),
wg_denoms, {S_V, kda, gdn_subgroup_size, lanes_per_column}, 1, true, use_subgroup_ops, gdn_subgroup_size);
wg_denoms, {S_V, kda, gdn_subgroup_size, lanes_per_column, uint32_t(gdn_packed_rows)}, 1, true, use_subgroup_ops, gdn_subgroup_size);
}
}
// Chunked GDN prefill uses workgroup matrices on CM2 and subgroup matrices on CM1.
const bool gdn_chunked = getenv("GGML_VK_DISABLE_GDN_CHUNKED") == nullptr;
const uint32_t gdn_shmem = device->properties.limits.maxComputeSharedMemorySize;
#if defined(VK_NV_cooperative_matrix2) && defined(GGML_VULKAN_COOPMAT2_GLSLC_SUPPORT) && defined(GGML_VULKAN_BFLOAT16_GLSLC_SUPPORT)
if (gdn_chunked && device->coopmat2 && device->coopmat2_gdn_support) {
ggml_vk_create_pipeline(device, device->pipeline_gated_delta_net_cm,
"gated_delta_net_f32_cm2", gated_delta_net_f32_cm2_len, gated_delta_net_f32_cm2_data,
"main", 7, sizeof(vk_op_gated_delta_net_push_constants), {1, 1, 1}, {}, 1, true, true, 32);
}
#endif
#if defined(GGML_VULKAN_COOPMAT_GLSLC_SUPPORT) && defined(GGML_VULKAN_COOPMAT_MAINTENANCE1_GLSLC_SUPPORT) && defined(GGML_VULKAN_BFLOAT16_GLSLC_SUPPORT)
// shader needs >= 16 lanes and 16 subgroups per WG; prefer wave64
auto gdn_sg_ok = [&](uint32_t sg) {
const bool reachable = device->subgroup_size_control ? device->subgroup_min_size <= sg && sg <= device->subgroup_max_size
: device->subgroup_size == sg;
return reachable && 16u * sg <= device->properties.limits.maxComputeWorkGroupInvocations &&
16u * sg <= device->properties.limits.maxComputeWorkGroupSize[0];
};
const uint32_t gdn_sg = gdn_sg_ok(64) ? 64u : device->subgroup_size;
if (gdn_chunked && !device->coopmat2 && device->coopmat_support && device->coopmat_bf16_support && device->coopmat_m1_per_element_ops &&
gdn_sg >= 16 && gdn_sg_ok(gdn_sg)) {
const bool bf16acc = device->coopmat_bf16_acc_support;
const std::vector<uint32_t> gdn_spec = { 16u * gdn_sg };
// V=64: state mirror in shared, 7 bindings
if (gdn_shmem >= ggml_vk_gated_delta_net_shmem_size(GGML_VK_GDN_CM_V, false)) {
ggml_vk_create_pipeline(device, device->pipeline_gated_delta_net_cm,
bf16acc ? "gated_delta_net_f32_cm1_bf16acc" : "gated_delta_net_f32_cm1",
bf16acc ? gated_delta_net_f32_cm1_bf16acc_len : gated_delta_net_f32_cm1_len,
bf16acc ? (const void *)gated_delta_net_f32_cm1_bf16acc_data : (const void *)gated_delta_net_f32_cm1_data,
"main", 7, sizeof(vk_op_gated_delta_net_push_constants), {1, 1, 1}, gdn_spec, 1, true, true, gdn_sg);
}
if (gdn_shmem >= ggml_vk_gated_delta_net_shmem_size(32u, false)) {
ggml_vk_create_pipeline(device, device->pipeline_gated_delta_net_cm_v32,
bf16acc ? "gated_delta_net_f32_cm1_bf16acc_v32" : "gated_delta_net_f32_cm1_v32",
bf16acc ? gated_delta_net_f32_cm1_bf16acc_v32_len : gated_delta_net_f32_cm1_v32_len,
bf16acc ? (const void *)gated_delta_net_f32_cm1_bf16acc_v32_data : (const void *)gated_delta_net_f32_cm1_v32_data,
"main", 7, sizeof(vk_op_gated_delta_net_push_constants), {1, 1, 1}, gdn_spec, 1, true, true, gdn_sg);
}
// V=128: state mirror in a gmem scratch buffer, 8 bindings
if (gdn_shmem >= ggml_vk_gated_delta_net_shmem_size(128u, false)) {
ggml_vk_create_pipeline(device, device->pipeline_gated_delta_net_cm_v128,
bf16acc ? "gated_delta_net_f32_cm1_bf16acc_v128" : "gated_delta_net_f32_cm1_v128",
bf16acc ? gated_delta_net_f32_cm1_bf16acc_v128_len : gated_delta_net_f32_cm1_v128_len,
bf16acc ? (const void *)gated_delta_net_f32_cm1_bf16acc_v128_data : (const void *)gated_delta_net_f32_cm1_v128_data,
"main", 8, sizeof(vk_op_gated_delta_net_push_constants), {1, 1, 1}, gdn_spec, 1, true, true, gdn_sg);
}
}
#endif
GGML_UNUSED(gdn_chunked);
GGML_UNUSED(gdn_shmem);
}
if (device->subgroup_arithmetic && device->subgroup_require_full_support) {
@@ -4554,6 +4622,11 @@ vk_device ggml_vk_get_device(size_t idx) {
} else if (strcmp("VK_NV_cooperative_matrix2", properties.extensionName) == 0 &&
!getenv("GGML_VK_DISABLE_COOPMAT2")) {
coopmat2_support = true;
#endif
#if defined(GGML_VULKAN_COOPMAT_MAINTENANCE1_GLSLC_SUPPORT)
} else if (strcmp("VK_EXT_cooperative_matrix_maintenance1", properties.extensionName) == 0 &&
!getenv("GGML_VK_DISABLE_COOPMAT_MAINTENANCE1")) {
device->coopmat_maintenance1 = true;
#endif
} else if (strcmp(VK_NV_COOPERATIVE_MATRIX_DECODE_VECTOR_EXTENSION_NAME, properties.extensionName) == 0 &&
!getenv("GGML_VK_DISABLE_COOPMAT2_DECODE_VECTOR")) {
@@ -4880,6 +4953,15 @@ vk_device ggml_vk_get_device(size_t idx) {
}
#endif
#if defined(GGML_VULKAN_COOPMAT_MAINTENANCE1_GLSLC_SUPPORT)
VkPhysicalDeviceCooperativeMatrixMaintenance1FeaturesEXT coopmat_m1_features {};
coopmat_m1_features.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_COOPERATIVE_MATRIX_MAINTENANCE_1_FEATURES_EXT;
if (device->coopmat_support && device->coopmat_maintenance1) {
last_struct->pNext = (VkBaseOutStructure *)&coopmat_m1_features;
last_struct = (VkBaseOutStructure *)&coopmat_m1_features;
}
#endif
VkPhysicalDeviceCooperativeMatrixDecodeVectorFeaturesNV coopmat2_decode_vector_features {};
coopmat2_decode_vector_features.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_COOPERATIVE_MATRIX_DECODE_VECTOR_FEATURES_NV;
if (coopmat2_decode_vector_support) {
@@ -5064,6 +5146,17 @@ vk_device ggml_vk_get_device(size_t idx) {
device->coopmat1_fa_support = device->coopmat_support && device->subgroup_require_full_support;
#endif
#if defined(GGML_VULKAN_COOPMAT_MAINTENANCE1_GLSLC_SUPPORT)
device->coopmat_maintenance1 = device->coopmat_support && device->coopmat_maintenance1;
if (device->coopmat_maintenance1) {
device->coopmat_m1_reductions = coopmat_m1_features.cooperativeMatrixReductions;
device->coopmat_m1_conversions = coopmat_m1_features.cooperativeMatrixConversions;
device->coopmat_m1_per_element_ops = coopmat_m1_features.cooperativeMatrixPerElementOperations;
device->coopmat_m1_get_coordinate = coopmat_m1_features.cooperativeMatrixGetCoordinate;
device_extensions.push_back(VK_EXT_COOPERATIVE_MATRIX_MAINTENANCE_1_EXTENSION_NAME);
}
#endif
if (coopmat2_support) {
#if defined(VK_NV_cooperative_matrix2) && defined(GGML_VULKAN_COOPMAT2_GLSLC_SUPPORT)
if (coopmat2_features.cooperativeMatrixWorkgroupScope &&
@@ -5147,6 +5240,13 @@ vk_device ggml_vk_get_device(size_t idx) {
prop.KGranularity <= 16) {
found_bf16_128 = true;
}
// GDN uses M=32, with N/K dimensions that are multiples of 16.
if (prop.workgroupInvocations == 128 &&
prop.MGranularity != 0 && 32 % prop.MGranularity == 0 &&
prop.NGranularity != 0 && 16 % prop.NGranularity == 0 &&
prop.KGranularity != 0 && 16 % prop.KGranularity == 0) {
device->coopmat2_gdn_support = true;
}
if (prop.workgroupInvocations == 256 &&
prop.MGranularity <= 32 &&
prop.NGranularity <= 32 &&
@@ -5157,6 +5257,12 @@ vk_device ggml_vk_get_device(size_t idx) {
#endif
}
}
device->coopmat2_gdn_support = device->coopmat2_gdn_support &&
coopmat2_props.cooperativeMatrixWorkgroupScopeMaxWorkgroupSize >= 128 &&
device->properties.limits.maxComputeWorkGroupInvocations >= 128 &&
device->properties.limits.maxComputeWorkGroupSize[0] >= 128 &&
uint64_t(ggml_vk_gated_delta_net_shmem_size(GGML_VK_GDN_CM2_V, true)) + coopmat2_props.cooperativeMatrixWorkgroupScopeReservedSharedMemory <=
device->properties.limits.maxComputeSharedMemorySize;
if (found_fp16_128 && found_fp16_256 &&
found_fp32_128 && found_fp32_256 &&
coopmat2_props.cooperativeMatrixFlexibleDimensionsMaxDimension >= 512) {
@@ -5274,6 +5380,18 @@ vk_device ggml_vk_get_device(size_t idx) {
device->coopmat_bf16_support = true;
}
}
// bf16 accumulator (bf16/bf16/bf16/bf16) at the shader's fixed 16x16x16
// shape lets the GDN coopmat1 path keep the hi/lo split as accumulators.
if (bfloat16_support &&
prop.AType == VK_COMPONENT_TYPE_BFLOAT16_KHR &&
prop.BType == VK_COMPONENT_TYPE_BFLOAT16_KHR &&
prop.CType == VK_COMPONENT_TYPE_BFLOAT16_KHR &&
prop.ResultType == VK_COMPONENT_TYPE_BFLOAT16_KHR &&
(vk::ScopeKHR)prop.scope == vk::ScopeKHR::eSubgroup &&
prop.MSize == 16 && prop.NSize == 16 && prop.KSize == 16
) {
device->coopmat_bf16_acc_support = true;
}
#endif
}
@@ -5489,6 +5607,7 @@ static void ggml_vk_print_gpu_info(size_t idx) {
bool coopmat_support = false;
bool coopmat2_support = false;
bool coopmat2_decode_vector_support = false;
bool coopmat_maintenance1_support = false;
bool integer_dot_product = false;
bool bfloat16_support = false;
bool dot2_f16_support = false;
@@ -5509,6 +5628,11 @@ static void ggml_vk_print_gpu_info(size_t idx) {
} else if (strcmp("VK_NV_cooperative_matrix2", properties.extensionName) == 0 &&
!getenv("GGML_VK_DISABLE_COOPMAT2")) {
coopmat2_support = true;
#endif
#if defined(GGML_VULKAN_COOPMAT_MAINTENANCE1_GLSLC_SUPPORT)
} else if (strcmp("VK_EXT_cooperative_matrix_maintenance1", properties.extensionName) == 0 &&
!getenv("GGML_VK_DISABLE_COOPMAT_MAINTENANCE1")) {
coopmat_maintenance1_support = true;
#endif
} else if (strcmp(VK_NV_COOPERATIVE_MATRIX_DECODE_VECTOR_EXTENSION_NAME, properties.extensionName) == 0 &&
!getenv("GGML_VK_DISABLE_COOPMAT2_DECODE_VECTOR")) {
@@ -5592,6 +5716,15 @@ static void ggml_vk_print_gpu_info(size_t idx) {
}
#endif
#if defined(GGML_VULKAN_COOPMAT_MAINTENANCE1_GLSLC_SUPPORT)
VkPhysicalDeviceCooperativeMatrixMaintenance1FeaturesEXT coopmat_m1_features {};
coopmat_m1_features.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_COOPERATIVE_MATRIX_MAINTENANCE_1_FEATURES_EXT;
if (coopmat_support && coopmat_maintenance1_support) {
last_struct->pNext = (VkBaseOutStructure *)&coopmat_m1_features;
last_struct = (VkBaseOutStructure *)&coopmat_m1_features;
}
#endif
VkPhysicalDeviceShaderIntegerDotProductFeaturesKHR shader_integer_dot_product_features {};
shader_integer_dot_product_features.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_SHADER_INTEGER_DOT_PRODUCT_FEATURES_KHR;
if (integer_dot_product) {
@@ -5685,13 +5818,17 @@ static void ggml_vk_print_gpu_info(size_t idx) {
coopmat2_support = false;
#endif
#if defined(GGML_VULKAN_COOPMAT_MAINTENANCE1_GLSLC_SUPPORT)
coopmat_maintenance1_support = coopmat_maintenance1_support && coopmat_m1_features.cooperativeMatrixPerElementOperations;
#endif
coopmat2_decode_vector_support = coopmat2_decode_vector_support && coopmat2_decode_vector_features.cooperativeMatrixDecodeVector;
#if !defined(GGML_VULKAN_COOPMAT2_DECODE_VECTOR_GLSLC_SUPPORT)
coopmat2_decode_vector_support = false;
#endif
std::string matrix_cores = coopmat2_support ? (coopmat2_decode_vector_support ? "NV_coopmat2v" : "NV_coopmat2")
: coopmat_support ? "KHR_coopmat"
: coopmat_support ? (coopmat_maintenance1_support ? "KHR_coopmat_m1" : "KHR_coopmat")
: "none";
bool dot2_f16 = dot2_f16_support && dot2_features.shaderMixedFloatDotProductFloat16AccFloat32;
@@ -11172,6 +11309,7 @@ void ggml_vk_lightning_indexer(ggml_backend_vk_context * ctx, vk_context& subctx
void ggml_vk_gated_delta_net(ggml_backend_vk_context * ctx, vk_context& subctx, ggml_tensor * dst) {
const ggml_tensor * src_q = dst->src[0];
const ggml_tensor * src_k = dst->src[1];
const ggml_tensor * src_v = dst->src[2];
const ggml_tensor * src_beta = dst->src[4];
@@ -11187,7 +11325,39 @@ void ggml_vk_gated_delta_net(ggml_backend_vk_context * ctx, vk_context& subctx,
const uint32_t s_off = S_v * H * n_tokens * n_seqs;
vk_pipeline pipeline = ggml_vk_op_get_pipeline(ctx, dst->src[0], dst->src[1], dst->src[2], dst, dst->op);
const ggml_tensor * src_g = dst->src[3];
// RDNA3: the scan is faster below ~4k tokens
const uint32_t cm_min_tokens = ctx->device->architecture == AMD_RDNA3 ? 4096u : 64u;
const bool use_cm = (ctx->device->pipeline_gated_delta_net_cm != nullptr ||
ctx->device->pipeline_gated_delta_net_cm_v32 != nullptr ||
ctx->device->pipeline_gated_delta_net_cm_v128 != nullptr) &&
src_g->ne[0] == 1 && S_v == 128 && dst->src[0]->ne[0] == 128 && K == 1 && n_tokens >= cm_min_tokens;
const bool use_cm2 = use_cm && ctx->device->coopmat2;
const bool have_v64 = ctx->device->pipeline_gated_delta_net_cm != nullptr;
bool use_v32 = use_cm && !use_cm2 && !have_v64 && ctx->device->pipeline_gated_delta_net_cm_v32 != nullptr;
bool use_v128 = use_cm && !use_cm2 && !have_v64 && !use_v32;
const uint32_t nsm = ctx->device->shader_core_count;
if (use_cm && !use_cm2 && have_v64 && nsm != 0) {
const uint32_t wg64 = 2u * H * n_seqs;
// V=32 only pays off while V=64 leaves at least half the cores idle
use_v32 = ctx->device->pipeline_gated_delta_net_cm_v32 != nullptr && 2u * wg64 <= nsm;
if (!use_v32 && ctx->device->pipeline_gated_delta_net_cm_v128 != nullptr) {
const uint32_t wg128 = H * n_seqs;
const uint32_t waves64 = (wg64 + nsm - 1) / nsm;
const uint32_t waves128 = (wg128 + nsm - 1) / nsm;
// prefer v128 when occupancy (wgs / (waves * nsm)) is at least as good
use_v128 = (uint64_t)wg128 * waves64 >= (uint64_t)wg64 * waves128;
}
}
const uint32_t cm_V = use_cm2 ? GGML_VK_GDN_CM2_V : (use_v128 ? 128u : (use_v32 ? 32u : GGML_VK_GDN_CM_V));
vk_pipeline pipeline = use_v128
? ctx->device->pipeline_gated_delta_net_cm_v128
: use_v32 ? ctx->device->pipeline_gated_delta_net_cm_v32
: (use_cm ? ctx->device->pipeline_gated_delta_net_cm
: ggml_vk_op_get_pipeline(ctx, dst->src[0], dst->src[1], dst->src[2], dst, dst->op));
GGML_ASSERT(pipeline != nullptr);
ggml_pipeline_request_descriptor_sets(ctx, pipeline, 1);
@@ -11198,9 +11368,27 @@ void ggml_vk_gated_delta_net(ggml_backend_vk_context * ctx, vk_context& subctx,
src_buf[i] = ggml_vk_tensor_subbuffer(ctx, dst->src[i]);
}
vk_subbuffer mirror_buf{};
if (use_v128) {
const uint32_t LDP = GGML_VK_GDN_CM_LDP;
const uint32_t blocks = (S_v / cm_V) * n_seqs * H;
const size_t scratch_size = (size_t)blocks * LDP * cm_V * 2u * sizeof(uint16_t);
if (ctx->prealloc_size_x < scratch_size) {
ctx->prealloc_size_x = scratch_size;
ggml_vk_preallocate_buffers(ctx, subctx);
}
if (ctx->prealloc_x_need_sync) {
ggml_vk_sync_buffers(ctx, subctx);
}
mirror_buf = vk_subbuffer{ ctx->prealloc_x, 0, ctx->prealloc_x->size };
}
const uint32_t sq1 = (uint32_t)(src_q->nb[1] / sizeof(float));
const uint32_t sq2 = (uint32_t)(src_q->nb[2] / sizeof(float));
const uint32_t sq3 = (uint32_t)(src_q->nb[3] / sizeof(float));
const uint32_t sk1 = (uint32_t)(src_k->nb[1] / sizeof(float));
const uint32_t sk2 = (uint32_t)(src_k->nb[2] / sizeof(float));
const uint32_t sk3 = (uint32_t)(src_k->nb[3] / sizeof(float));
const uint32_t sv1 = (uint32_t)(src_v->nb[1] / sizeof(float));
const uint32_t sv2 = (uint32_t)(src_v->nb[2] / sizeof(float));
const uint32_t sv3 = (uint32_t)(src_v->nb[3] / sizeof(float));
@@ -11215,6 +11403,7 @@ void ggml_vk_gated_delta_net(ggml_backend_vk_context * ctx, vk_context& subctx,
const vk_op_gated_delta_net_push_constants pc = {
H, n_tokens, n_seqs, s_off,
sq1, sq2, sq3,
sk1, sk2, sk3,
sv1, sv2, sv3,
sb1, sb2, sb3,
neq1, rq3,
@@ -11222,9 +11411,16 @@ void ggml_vk_gated_delta_net(ggml_backend_vk_context * ctx, vk_context& subctx,
K
};
ggml_vk_dispatch_pipeline(ctx, subctx, pipeline,
{src_buf[0], src_buf[1], src_buf[2], src_buf[3], src_buf[4], src_buf[5], dst_buf},
pc, { H, n_seqs, S_v });
if (use_v128) {
ggml_vk_dispatch_pipeline(ctx, subctx, pipeline,
{src_buf[0], src_buf[1], src_buf[2], src_buf[3], src_buf[4], src_buf[5], dst_buf, mirror_buf},
pc, std::array<uint32_t, 3>{ S_v / cm_V, n_seqs, H });
ctx->prealloc_x_need_sync = true;
} else {
ggml_vk_dispatch_pipeline(ctx, subctx, pipeline,
{src_buf[0], src_buf[1], src_buf[2], src_buf[3], src_buf[4], src_buf[5], dst_buf},
pc, use_cm ? std::array<uint32_t, 3>{ S_v / cm_V, n_seqs, H } : std::array<uint32_t, 3>{ H, n_seqs, S_v });
}
}
void ggml_vk_ssm_scan(ggml_backend_vk_context * ctx, vk_context& subctx, ggml_tensor * dst) {
@@ -15,6 +15,10 @@ if (GGML_VULKAN_COOPMAT2_DECODE_VECTOR_GLSLC_SUPPORT)
add_compile_definitions(GGML_VULKAN_COOPMAT2_DECODE_VECTOR_GLSLC_SUPPORT)
message(STATUS "Enabling coopmat2 decode_vector glslc support")
endif()
if (GGML_VULKAN_COOPMAT_MAINTENANCE1_GLSLC_SUPPORT)
add_compile_definitions(GGML_VULKAN_COOPMAT_MAINTENANCE1_GLSLC_SUPPORT)
message(STATUS "Enabling coopmat maintenance1 glslc support")
endif()
if (GGML_VULKAN_INTEGER_DOT_GLSLC_SUPPORT)
add_compile_definitions(GGML_VULKAN_INTEGER_DOT_GLSLC_SUPPORT)
message(STATUS "Enabling dot glslc support")
@@ -0,0 +1,8 @@
#version 460
#extension GL_KHR_cooperative_matrix : require
#extension GL_EXT_cooperative_matrix_maintenance1 : require
void main()
{
}
@@ -15,10 +15,19 @@ layout(constant_id = 0) const uint S_V = 128;
layout(constant_id = 1) const uint KDA = 0;
layout(constant_id = 2) const uint SUBGROUP_SIZE = 32;
layout(constant_id = 3) const uint LANES_PER_COLUMN = 32;
layout(constant_id = 4) const bool PACKED_ROWS = false;
const uint COLS_PER_WG = SUBGROUP_SIZE / LANES_PER_COLUMN;
const uint ROWS_PER_LANE = S_V / LANES_PER_COLUMN;
uint row_index(uint r, uint lane) {
// Keep four adjacent rows together in each lane.
if (PACKED_ROWS) {
return (r / 4u * LANES_PER_COLUMN + lane) * 4u + r % 4u;
}
return r * LANES_PER_COLUMN + lane;
}
layout(local_size_x_id = 2, local_size_y = 1, local_size_z = 1) in;
layout(push_constant) uniform Parameters {
@@ -27,6 +36,7 @@ layout(push_constant) uniform Parameters {
uint n_seqs;
uint s_off;
uint sq1, sq2, sq3;
uint sk1, sk2, sk3;
uint sv1, sv2, sv3;
uint sb1, sb2, sb3;
uint neq1, rq3;
@@ -110,7 +120,7 @@ void main() {
FLOAT_TYPE s_shard[ROWS_PER_LANE];
[[unroll]] for (uint r = 0; r < ROWS_PER_LANE; r++) {
s_shard[r] = FLOAT_TYPE(data_state[state_in_base + col * S_V + r * LANES_PER_COLUMN + lane]);
s_shard[r] = FLOAT_TYPE(data_state[state_in_base + col * S_V + row_index(r, lane)]);
}
// snapshot slot mapping: slot 0 = most recent state, slot s = s tokens back.
@@ -120,7 +130,7 @@ void main() {
for (uint t = 0; t < n_tokens; t++) {
const uint q_off = iq3 * sq3 + t * sq2 + iq1 * sq1;
const uint k_off = q_off;
const uint k_off = iq3 * sk3 + t * sk2 + iq1 * sk1;
const uint v_off = seq_id * sv3 + t * sv2 + head_id * sv1;
const uint gb_off = seq_id * sb3 + t * sb2 + head_id * sb1;
const FLOAT_TYPE beta_val = FLOAT_TYPE(data_beta[gb_off]);
@@ -128,7 +138,7 @@ void main() {
FLOAT_TYPE k_reg[ROWS_PER_LANE];
FLOAT_TYPE q_reg[ROWS_PER_LANE];
[[unroll]] for (uint r = 0; r < ROWS_PER_LANE; r++) {
const uint i = r * LANES_PER_COLUMN + lane;
const uint i = row_index(r, lane);
k_reg[r] = FLOAT_TYPE(data_k[k_off + i]);
q_reg[r] = FLOAT_TYPE(data_q[q_off + i]);
}
@@ -142,7 +152,7 @@ void main() {
} else {
const uint g_base = gb_off * S_V;
[[unroll]] for (uint r = 0; r < ROWS_PER_LANE; r++) {
const uint i = r * LANES_PER_COLUMN + lane;
const uint i = row_index(r, lane);
g_exp[r] = exp(FLOAT_TYPE(data_g[g_base + i]));
}
}
@@ -175,7 +185,7 @@ void main() {
if (target_slot >= 0 && target_slot < int(K)) {
const uint slot_base = s_off + uint(target_slot) * state_size_per_snap + state_out_base;
[[unroll]] for (uint r = 0; r < ROWS_PER_LANE; r++) {
data_dst[slot_base + col * S_V + r * LANES_PER_COLUMN + lane] = s_shard[r];
data_dst[slot_base + col * S_V + row_index(r, lane)] = s_shard[r];
}
}
}
@@ -183,7 +193,7 @@ void main() {
if (K == 1u) {
[[unroll]] for (uint r = 0; r < ROWS_PER_LANE; r++) {
data_dst[s_off + state_out_base + col * S_V + r * LANES_PER_COLUMN + lane] = s_shard[r];
data_dst[s_off + state_out_base + col * S_V + row_index(r, lane)] = s_shard[r];
}
}
}
@@ -0,0 +1,437 @@
#version 450
#extension GL_EXT_control_flow_attributes : enable
#extension GL_EXT_shader_explicit_arithmetic_types_float16 : require
#extension GL_EXT_shader_explicit_arithmetic_types_int32 : require
#extension GL_EXT_bfloat16 : require
#extension GL_KHR_memory_scope_semantics : enable
#extension GL_KHR_shader_subgroup_basic : enable
#extension GL_KHR_shader_subgroup_arithmetic : enable
#extension GL_KHR_shader_subgroup_shuffle : enable
#extension GL_KHR_cooperative_matrix : enable
#extension GL_EXT_cooperative_matrix_maintenance1 : enable
// Chunked gated-delta-net prefill (non-KDA, S_v == D == 128, K == 1), port of CUDA PR 29353.
// State in register tiles, mirrored ([D x V]) for the projections.
// bf16 hi/lo 3x products for ~fp32 accuracy.
#define OPD bfloat16_t
#define C 16 // chunk size (tokens per step)
#define D 128 // head dim / value dim
#define TM 16 // tile rows
#define TN 16 // tile cols (bf16 cm has only m16n16k16; n8 would pad)
#ifndef V
#define V 64 // value columns per workgroup block (V=128 = whole head)
#endif
#define NSUB 16 // subgroups per workgroup
layout(constant_id = 0) const uint WGS = NSUB * 32; // NSUB * subgroup size
#define KT 16 // K-tile width for deep (K=D) matmuls (bf16 cm: m16n16k16 only)
#define PAD 8 // shared padding to break COL-load bank conflicts (keep LDP*2 a multiple of 16B)
#define LDP (D + PAD)
// GMEM_MIRROR: state mirror in a gmem scratch buffer (too big for shared)
#define STATE_TILES ((D / TM) * (V / TN))
#define PER_WARP (STATE_TILES / NSUB)
#define VT (V / TN) // value tiles
#define CT (C / TN) // C-column tiles (gram/pmat halves)
const uint NQK = C * D / WGS;
#define TILE_R(ti) (((ti) / VT) * TM)
#define TILE_C(ti) (((ti) % VT) * TN)
layout(local_size_x_id = 0, local_size_y = 1, local_size_z = 1) in;
layout(push_constant) uniform Parameters {
uint H;
uint n_tokens;
uint n_seqs;
uint s_off;
uint sq1, sq2, sq3;
uint sk1, sk2, sk3;
uint sv1, sv2, sv3;
uint sb1, sb2, sb3;
uint neq1, rq3;
float scale;
uint K;
} p;
layout(binding = 0) readonly buffer QBuf { float data_q[]; };
layout(binding = 1) readonly buffer KBuf { float data_k[]; };
layout(binding = 2) readonly buffer VBuf { float data_v[]; };
layout(binding = 3) readonly buffer GBuf { float data_g[]; };
layout(binding = 4) readonly buffer BetaBuf { float data_beta[]; };
layout(binding = 5) readonly buffer StateBuf { float data_state[]; };
layout(binding = 6) buffer DstBuf { float data_dst[]; };
#ifdef GMEM_MIRROR
layout(binding = 7) coherent buffer MirrorBuf { OPD data_mirror[]; };
#endif
shared OPD sh_qh[C * LDP], sh_ql[C * LDP]; // q*scale hi/lo (D padded)
shared OPD sh_kh[C * LDP], sh_kl[C * LDP]; // k hi/lo (D padded)
#ifndef GMEM_MIRROR
shared OPD sh_sh[LDP * V], sh_sl[LDP * V]; // state mirror [D x V] column-major (D padded)
#endif
shared OPD sh_dh[C * V], sh_dl[C * V]; // delta (pre-inv) / delta2 [C x V] row-major
shared OPD sh_soh[C * V], sh_sol[C * V]; // solved [C x V] row-major
shared OPD sh_ph[C * C], sh_pl[C * C]; // decayed P
shared OPD sh_ih[C * C], sh_il[C * C]; // (I + L)^-1
shared float sh_lower[C * C]; // strictly-lower decayed L
shared float sh_prefix[C], sh_beta[C], sh_ep[C], sh_carry[C];
#define SG gl_ScopeSubgroup
#define MUA gl_MatrixUseA
#define MUB gl_MatrixUseB
#define MACC gl_MatrixUseAccumulator
#define ROW gl_CooperativeMatrixLayoutRowMajor
#define COL gl_CooperativeMatrixLayoutColumnMajor
// 3x bf16 product accumulated into acc (hi/lo split for ~fp32 accuracy).
#define PRODUCT(acc, ah, al, bh, bl) \
acc = coopMatMulAdd(al, bh, acc); \
acc = coopMatMulAdd(ah, bl, acc); \
acc = coopMatMulAdd(ah, bh, acc);
void split(const float x, out OPD h, out OPD l) {
h = OPD(x);
l = OPD(x - float(h));
}
#ifdef BF16ACC
// split through a bf16 accumulator
#define SPLIT_STORE(m, arr_h, off_h, arr_l, off_l, ld, layout) { \
const coopmat<OPD, SG, TM, TN, MACC> h_ = coopmat<OPD, SG, TM, TN, MACC>(m); \
const coopmat<OPD, SG, TM, TN, MACC> l_ = coopmat<OPD, SG, TM, TN, MACC>(m - coopmat<float, SG, TM, TN, MACC>(h_)); \
coopMatStore(h_, arr_h, off_h, ld, layout); \
coopMatStore(l_, arr_l, off_l, ld, layout); \
}
#ifdef GMEM_MIRROR
#define STORE_MIRROR(m, off) SPLIT_STORE(m, data_mirror, mir_h + off, data_mirror, mir_l + off, LDP, COL)
#else
#define STORE_MIRROR(m, off) SPLIT_STORE(m, sh_sh, off, sh_sl, off, LDP, COL)
#endif
#define STORE_DELTA(m, off) SPLIT_STORE(m, sh_dh, off, sh_dl, off, V, ROW)
#define STORE_SOLVED(m, off) SPLIT_STORE(m, sh_soh, off, sh_sol, off, V, ROW)
#else
// split per element
#ifdef GMEM_MIRROR
float SplitColS(const in uint32_t r, const in uint32_t c, const in float elem, const in uint32_t off) {
const uint idx = off + c * LDP + r;
split(elem, data_mirror[idx], data_mirror[idx + LDP * V]);
return elem;
}
#define STORE_MIRROR(m, off) coopMatPerElementEXT(m, m, SplitColS, mir_h + off)
#else
float SplitColS(const in uint32_t r, const in uint32_t c, const in float elem, const in uint32_t off) {
const uint idx = off + c * LDP + r;
split(elem, sh_sh[idx], sh_sl[idx]);
return elem;
}
#define STORE_MIRROR(m, off) coopMatPerElementEXT(m, m, SplitColS, off)
#endif
float SplitRowD(const in uint32_t r, const in uint32_t c, const in float elem, const in uint32_t off) {
const uint idx = off + r * V + c;
split(elem, sh_dh[idx], sh_dl[idx]);
return elem;
}
float SplitRowSO(const in uint32_t r, const in uint32_t c, const in float elem, const in uint32_t off) {
const uint idx = off + r * V + c;
split(elem, sh_soh[idx], sh_sol[idx]);
return elem;
}
#define STORE_DELTA(m, off) coopMatPerElementEXT(m, m, SplitRowD, off)
#define STORE_SOLVED(m, off) coopMatPerElementEXT(m, m, SplitRowSO, off)
#endif
#ifdef GMEM_MIRROR
#define LOAD_S(h, l, off) \
coopMatLoad(h, data_mirror, mir_h + off, LDP, COL); \
coopMatLoad(l, data_mirror, mir_l + off, LDP, COL);
// mirror is workgroup-private
#define MIRROR_BARRIER() controlBarrier(gl_ScopeWorkgroup, gl_ScopeWorkgroup, \
gl_StorageSemanticsBuffer | gl_StorageSemanticsShared, \
gl_SemanticsAcquireRelease)
#else
#define LOAD_S(h, l, off) \
coopMatLoad(h, sh_sh, off, LDP, COL); \
coopMatLoad(l, sh_sl, off, LDP, COL);
#define MIRROR_BARRIER() barrier()
#endif
float StoreOut(const in uint32_t r, const in uint32_t c, const in float elem,
const in uint32_t out_base, const in uint32_t valid) {
if (r < valid) {
data_dst[out_base + r * (D * p.H) + c] = elem;
}
return elem;
}
float StoreState(const in uint32_t r, const in uint32_t c, const in float elem,
const in uint32_t base) {
data_dst[base + c * D + r] = elem;
return elem;
}
float LoadV(const in uint32_t r, const in uint32_t c, const in float elem,
const in uint32_t v_base, const in uint32_t sv2, const in uint32_t valid) {
return r < valid ? data_v[v_base + r * sv2 + c] : 0.0;
}
// sh_beta[r] == 0 for r >= valid
float ComputeDeltaV(const in uint32_t r, const in uint32_t c, const in float ks,
const in float vv) {
return sh_beta[r] * (vv - sh_ep[r] * ks);
}
float ApplyExpPrefix(const in uint32_t r, const in uint32_t c, const in float qs) {
return sh_ep[r] * qs;
}
float StoreLower(const in uint32_t r, const in uint32_t c, const in float elem,
const in uint32_t c_o) {
const uint cc = c_o + c;
if (cc < r) {
sh_lower[r * C + cc] = elem * exp(sh_prefix[r] - sh_prefix[cc]) * sh_beta[r];
}
return elem;
}
float StorePmat(const in uint32_t r, const in uint32_t c, const in float elem,
const in uint32_t c_o) {
const uint cc = c_o + c;
float v = 0.0;
if (cc <= r) {
v = elem * exp(sh_prefix[r] - sh_prefix[cc]);
}
split(v, sh_ph[r * C + cc], sh_pl[r * C + cc]);
return elem;
}
void prefetch(const uint t0, const uint iq1, const uint iq3, const uint seq_id, const uint head_id,
out float q[NQK], out float k[NQK], out float g, out float b) {
const uint n = t0 < p.n_tokens ? min(C, p.n_tokens - t0) : 0u;
[[unroll]] for (uint j = 0; j < NQK; j++) {
const uint i = gl_LocalInvocationID.x + j * WGS;
const uint t = i / D;
q[j] = 0.0;
k[j] = 0.0;
if (t < n) {
const uint q_off = iq3 * p.sq3 + (t0 + t) * p.sq2 + iq1 * p.sq1 + i % D;
const uint k_off = iq3 * p.sk3 + (t0 + t) * p.sk2 + iq1 * p.sk1 + i % D;
q[j] = data_q[q_off] * p.scale;
k[j] = data_k[k_off];
}
}
g = 0.0;
b = 0.0;
const uint t = gl_SubgroupInvocationID;
if (gl_SubgroupID == 0 && t < n) {
const uint gb = seq_id * p.sb3 + (t0 + t) * p.sb2 + head_id * p.sb1;
g = data_g[gb];
b = data_beta[gb];
}
}
// sg0: gram K@K^T -> L, sg1: pmat Q@K^T -> P
void gram_pmat(const uint sg) {
const bool gram = sg < CT;
const uint c_o = (sg % CT) * TN;
coopmat<float, SG, C, TN, MACC> x = coopmat<float, SG, C, TN, MACC>(0);
[[unroll]] for (uint kk = 0; kk < D; kk += KT) {
coopmat<OPD, SG, C, KT, MUA> ah, al;
if (gram) {
coopMatLoad(ah, sh_kh, kk, LDP, ROW);
coopMatLoad(al, sh_kl, kk, LDP, ROW);
} else {
coopMatLoad(ah, sh_qh, kk, LDP, ROW);
coopMatLoad(al, sh_ql, kk, LDP, ROW);
}
coopmat<OPD, SG, KT, TN, MUB> bh, bl;
coopMatLoad(bh, sh_kh, c_o * LDP + kk, LDP, COL);
coopMatLoad(bl, sh_kl, c_o * LDP + kk, LDP, COL);
PRODUCT(x, ah, al, bh, bl);
}
if (gram) {
coopMatPerElementEXT(x, x, StoreLower, c_o);
} else {
coopMatPerElementEXT(x, x, StorePmat, c_o);
}
}
// (I + L)^-1, forward substitution: subgroup per row, lane per column
void invert(const uint sg) {
const uint col = gl_SubgroupInvocationID;
[[unroll]] for (uint row = sg; row < C; row += NSUB) {
float x = col < C ? (col == row ? 1.0 : (col < row ? -sh_lower[row * C + col] : 0.0)) : 0.0;
for (int pivot = int(row) - 1; pivot > 0; pivot--) {
const float xp = subgroupShuffle(x, uint(pivot));
if (col < uint(pivot)) {
x -= xp * sh_lower[uint(pivot) * C + col];
}
}
if (col < C) {
split(x, sh_ih[row * C + col], sh_il[row * C + col]);
}
}
}
float carry_decay(const uint valid) {
const float plast = sh_prefix[valid - 1];
if (gl_LocalInvocationID.x < C) {
sh_carry[gl_LocalInvocationID.x] = exp(plast - sh_prefix[gl_LocalInvocationID.x]);
}
return exp(plast);
}
void main() {
const uint head_id = gl_WorkGroupID.z;
const uint seq_id = gl_WorkGroupID.y;
const uint sg = gl_SubgroupID;
const uint tid = gl_LocalInvocationID.x;
const uint v0 = gl_WorkGroupID.x * V;
const uint iq1 = head_id % p.neq1;
const uint iq3 = seq_id / p.rq3;
const uint state_base = (seq_id * p.H + head_id) * (D * D);
#ifdef GMEM_MIRROR
const uint mir_stride = uint(LDP) * V;
const uint mir_block = (head_id * p.n_seqs + seq_id) * gl_NumWorkGroups.x + gl_WorkGroupID.x;
const uint mir_h = mir_block * mir_stride * 2u;
const uint mir_l = mir_h + mir_stride;
#endif
coopmat<float, SG, TM, TN, MACC> state[PER_WARP];
[[unroll]] for (uint j = 0; j < PER_WARP; j++) {
const uint ti = sg + j * NSUB;
coopMatLoad(state[j], data_state, state_base + (v0 + TILE_C(ti)) * D + TILE_R(ti), D, COL);
}
float qreg[NQK], kreg[NQK], greg, breg;
prefetch(0u, iq1, iq3, seq_id, head_id, qreg, kreg, greg, breg);
for (uint t0 = 0; t0 < p.n_tokens; t0 += C) {
const uint valid = min(C, p.n_tokens - t0);
barrier();
if (sg == 0) {
const uint t = gl_SubgroupInvocationID;
const float acc = subgroupInclusiveAdd(greg);
if (t < C) {
sh_prefix[t] = acc;
sh_ep[t] = exp(acc);
sh_beta[t] = breg;
}
}
[[unroll]] for (uint j = 0; j < NQK; j++) {
const uint i = tid + j * WGS;
const uint pi = (i / D) * LDP + (i % D);
split(qreg[j], sh_qh[pi], sh_ql[pi]);
split(kreg[j], sh_kh[pi], sh_kl[pi]);
}
[[unroll]] for (uint j = 0; j < PER_WARP; j++) {
const uint ti = sg + j * NSUB;
STORE_MIRROR(state[j], TILE_C(ti) * LDP + TILE_R(ti));
}
MIRROR_BARRIER();
prefetch(t0 + C, iq1, iq3, seq_id, head_id, qreg, kreg, greg, breg);
const float cdecay = carry_decay(valid);
// qs -> base_r and ks -> delta on sg2..2+VT-1
coopmat<float, SG, C, TN, MACC> base_r = coopmat<float, SG, C, TN, MACC>(0);
if (sg < 2 * CT) {
gram_pmat(sg);
} else if (sg - 2 * CT < VT) {
const uint c_o = (sg - 2 * CT) * TN;
const uint v_base = seq_id * p.sv3 + head_id * p.sv1 + v0 + c_o + t0 * p.sv2;
coopmat<float, SG, C, TN, MACC> vmat = coopmat<float, SG, C, TN, MACC>(0);
coopMatPerElementEXT(vmat, vmat, LoadV, v_base, p.sv2, valid);
coopmat<float, SG, C, TN, MACC> qs = coopmat<float, SG, C, TN, MACC>(0);
coopmat<float, SG, C, TN, MACC> ks = coopmat<float, SG, C, TN, MACC>(0);
[[unroll]] for (uint kk = 0; kk < D; kk += KT) {
coopmat<OPD, SG, C, KT, MUA> qh, ql, kh, kl;
coopMatLoad(qh, sh_qh, kk, LDP, ROW);
coopMatLoad(ql, sh_ql, kk, LDP, ROW);
coopMatLoad(kh, sh_kh, kk, LDP, ROW);
coopMatLoad(kl, sh_kl, kk, LDP, ROW);
coopmat<OPD, SG, KT, TN, MUB> sbh, sbl;
LOAD_S(sbh, sbl, c_o * LDP + kk);
PRODUCT(qs, qh, ql, sbh, sbl);
PRODUCT(ks, kh, kl, sbh, sbl);
}
coopMatPerElementEXT(base_r, qs, ApplyExpPrefix);
coopmat<float, SG, C, TN, MACC> delta;
coopMatPerElementEXT(delta, ks, ComputeDeltaV, vmat);
STORE_DELTA(delta, c_o);
}
barrier();
invert(sg);
barrier();
// solved = (I+L)^-1 @ delta (sg0..VT-1)
if (sg < VT) {
const uint c_o = sg * TN;
coopmat<float, SG, C, TN, MACC> solved = coopmat<float, SG, C, TN, MACC>(0);
coopmat<OPD, SG, C, C, MUA> ih, il;
coopMatLoad(ih, sh_ih, 0, C, ROW);
coopMatLoad(il, sh_il, 0, C, ROW);
coopmat<OPD, SG, C, TN, MUB> dh, dl;
coopMatLoad(dh, sh_dh, c_o, V, ROW);
coopMatLoad(dl, sh_dl, c_o, V, ROW);
PRODUCT(solved, ih, il, dh, dl);
STORE_SOLVED(solved, c_o);
}
barrier();
// out = base + P@solved (sg2..2+VT-1)
if (sg >= 2 * CT && sg - 2 * CT < VT) {
const uint c_o = (sg - 2 * CT) * TN;
coopmat<float, SG, C, TN, MACC> O = base_r;
coopmat<OPD, SG, C, C, MUA> ph, pl;
coopMatLoad(ph, sh_ph, 0, C, ROW);
coopMatLoad(pl, sh_pl, 0, C, ROW);
coopmat<OPD, SG, C, TN, MUB> svh, svl;
coopMatLoad(svh, sh_soh, c_o, V, ROW);
coopMatLoad(svl, sh_sol, c_o, V, ROW);
PRODUCT(O, ph, pl, svh, svl);
const uint out_base = (seq_id * p.n_tokens * p.H + head_id) * D + v0 + c_o + t0 * (D * p.H);
coopMatPerElementEXT(O, O, StoreOut, out_base, valid);
}
// delta2 = solved * carry -> sh_dh/dl (all threads)
for (uint i = tid; i < C * V; i += WGS) {
const float sv = float(sh_soh[i]) + float(sh_sol[i]);
split(sv * sh_carry[i / V], sh_dh[i], sh_dl[i]);
}
barrier();
// state = cdecay*state + K^T @ delta2
[[unroll]] for (uint j = 0; j < PER_WARP; j++) {
const uint ti = sg + j * NSUB;
coopmat<OPD, SG, TM, C, MUA> kth, ktl;
coopMatLoad(kth, sh_kh, TILE_R(ti), LDP, COL);
coopMatLoad(ktl, sh_kl, TILE_R(ti), LDP, COL);
coopmat<OPD, SG, C, TN, MUB> dh, dl;
coopMatLoad(dh, sh_dh, TILE_C(ti), V, ROW);
coopMatLoad(dl, sh_dl, TILE_C(ti), V, ROW);
state[j] = state[j] * cdecay;
PRODUCT(state[j], kth, ktl, dh, dl);
}
// no end-of-chunk barrier: covered by the chunk-top barrier
}
[[unroll]] for (uint j = 0; j < PER_WARP; j++) {
const uint ti = sg + j * NSUB;
coopMatPerElementEXT(state[j], state[j], StoreState,
p.s_off + state_base + (v0 + TILE_C(ti)) * D + TILE_R(ti));
}
}
@@ -0,0 +1,320 @@
#version 450
#extension GL_EXT_control_flow_attributes : require
#extension GL_EXT_shader_explicit_arithmetic_types_int32 : require
#extension GL_EXT_bfloat16 : require
#extension GL_KHR_memory_scope_semantics : require
#extension GL_KHR_shader_subgroup_basic : require
#extension GL_KHR_shader_subgroup_arithmetic : require
#extension GL_KHR_shader_subgroup_shuffle : require
#extension GL_KHR_shader_subgroup_shuffle_relative : require
#extension GL_KHR_cooperative_matrix : require
#extension GL_NV_cooperative_matrix2 : require
// Chunked gated-delta-net prefill (non-KDA, S_v == D == 128, K == 1).
// BF16 hi/lo products use FP32 accumulators.
#define OPD bfloat16_t
#define C 16 // chunk size (tokens per step)
#define D 128 // head dim / value dim
#define V 32 // value columns per workgroup block
#define WGS 128
#define NSUB (WGS / 32)
#define KT 32 // K-tile width for deep (K=D) matmuls
#define PAD 8
#define LDP (D + PAD)
#define LDV (V + PAD)
#define KQ_ROWS (2 * C)
#define NQK (C * D / WGS)
layout(local_size_x = WGS, local_size_y = 1, local_size_z = 1) in;
layout(push_constant) uniform Parameters {
uint H;
uint n_tokens;
uint n_seqs;
uint s_off;
uint sq1, sq2, sq3;
uint sk1, sk2, sk3;
uint sv1, sv2, sv3;
uint sb1, sb2, sb3;
uint neq1, rq3;
float scale;
uint K;
} p;
layout(binding = 0) readonly buffer QBuf { float data_q[]; };
layout(binding = 1) readonly buffer KBuf { float data_k[]; };
layout(binding = 2) readonly buffer VBuf { float data_v[]; };
layout(binding = 3) readonly buffer GBuf { float data_g[]; };
layout(binding = 4) readonly buffer BetaBuf { float data_beta[]; };
layout(binding = 5) readonly buffer StateBuf { float data_state[]; };
layout(binding = 6) buffer DstBuf { float data_dst[]; };
// K and scaled Q occupy the two row blocks of one 32-row operand.
shared OPD sh_kqh[2 * C * LDP], sh_kql[2 * C * LDP];
shared OPD sh_sh[V * LDP], sh_sl[V * LDP]; // state mirror [V x D] row-major (D padded)
shared OPD sh_ph[C * C], sh_pl[C * C];
shared OPD sh_ih[C * C], sh_il[C * C];
shared OPD sh_dh[C * LDV], sh_dl[C * LDV]; // delta / solved [C x V] row-major (V padded)
shared float sh_lower[C * (C - 1) / 2]; // packed strictly-lower Gram entries
shared float sh_prefix[C], sh_ep[C], sh_beta[C], sh_carry[C];
#define WG gl_ScopeWorkgroup
#define MUA gl_MatrixUseA
#define MUB gl_MatrixUseB
#define MACC gl_MatrixUseAccumulator
#define ROW gl_CooperativeMatrixLayoutRowMajor
#define COL gl_CooperativeMatrixLayoutColumnMajor
#define PRODUCT(acc, ah, al, bh, bl) \
acc = coopMatMulAdd(ah, bh, acc); \
acc = coopMatMulAdd(al, bh, acc); \
acc = coopMatMulAdd(ah, bl, acc);
void split(const float x, out OPD h, out OPD l) {
h = OPD(x);
l = OPD(x - float(h));
}
float StoreGramPmat(const in uint32_t r, const in uint32_t c, const in float elem) {
if (r < C) {
if (c < r) {
sh_lower[r * (r - 1) / 2 + c] = elem * exp(sh_prefix[r] - sh_prefix[c]) * sh_beta[r];
}
} else {
const uint row = r - C;
const float y = c <= row ? elem * exp(sh_prefix[row] - sh_prefix[c]) : 0.0;
split(y, sh_ph[row * C + c], sh_pl[row * C + c]);
}
return elem;
}
float SplitRowS(const in uint32_t r, const in uint32_t c, const in float elem) {
const uint i = r * LDP + c;
split(elem, sh_sh[i], sh_sl[i]);
return elem;
}
float ApplyExpPrefix(const in uint32_t r, const in uint32_t c, const in float qs) {
return sh_ep[c] * qs;
}
float LoadV(const in uint32_t r, const in uint32_t c, const in float elem,
const in uint32_t v_base, const in uint32_t sv2, const in uint32_t valid) {
return c < valid ? data_v[v_base + c * sv2 + r] : 0.0;
}
float StoreDeltaV(const in uint32_t r, const in uint32_t c, const in float ks, const in float vv) {
const float x = sh_beta[c] * (vv - sh_ep[c] * ks);
const uint i = c * LDV + r;
split(x, sh_dh[i], sh_dl[i]);
return ks;
}
float StoreSolved(const in uint32_t r, const in uint32_t c, const in float elem) {
const uint i = c * LDV + r;
split(elem, sh_dh[i], sh_dl[i]);
// The state mirror is no longer needed after KS/QS.
const uint j = r * LDP + c;
split(sh_carry[c] * elem, sh_sh[j], sh_sl[j]);
return elem;
}
float StoreOut(const in uint32_t r, const in uint32_t c, const in float elem,
const in uint32_t out_base, const in uint32_t valid) {
if (c < valid) {
data_dst[out_base + c * (p.H * D) + r] = elem;
}
return elem;
}
void prefetch(const uint t0, const uint q_base, const uint k_base, const uint gb_base,
out float q[NQK], out float k[NQK], out float g, out float b) {
const uint valid = t0 < p.n_tokens ? min(C, p.n_tokens - t0) : 0u;
const uint r = gl_SubgroupInvocationID;
g = 0.0;
b = 0.0;
if (gl_SubgroupID == 0 && r < valid) {
const uint off = gb_base + (t0 + r) * p.sb2;
g = data_g[off];
b = data_beta[off];
}
[[unroll]] for (uint j = 0; j < NQK; ++j) {
const uint i = gl_LocalInvocationID.x + j * WGS;
const uint token = i / D;
q[j] = 0.0;
k[j] = 0.0;
if (token < valid) {
const uint q_off = q_base + (t0 + token) * p.sq2 + i % D;
const uint k_off = k_base + (t0 + token) * p.sk2 + i % D;
q[j] = data_q[q_off];
k[j] = data_k[k_off];
}
}
}
void stage_inputs(const in float q[NQK], const in float k[NQK]) {
[[unroll]] for (uint j = 0; j < NQK; ++j) {
const uint i = gl_LocalInvocationID.x + j * WGS;
const uint token = i / D;
const uint key = i % D;
const uint ki = token * LDP + key;
const uint qi = (C + token) * LDP + key;
split(k[j], sh_kqh[ki], sh_kql[ki]);
split(p.scale * q[j], sh_kqh[qi], sh_kql[qi]);
}
}
// [K; Q] * K^T computes Gram and P in one 32-row matrix.
void gram_pmat() {
coopmat<float, WG, KQ_ROWS, C, MACC> x = coopmat<float, WG, KQ_ROWS, C, MACC>(0.0);
[[unroll]] for (uint kk = 0; kk < D; kk += KT) {
coopmat<OPD, WG, KQ_ROWS, KT, MUA> ah, al;
coopmat<OPD, WG, KT, C, MUB> bh, bl;
coopMatLoad(ah, sh_kqh, kk, LDP, ROW);
coopMatLoad(al, sh_kql, kk, LDP, ROW);
coopMatLoad(bh, sh_kqh, kk, LDP, COL);
coopMatLoad(bl, sh_kql, kk, LDP, COL);
PRODUCT(x, ah, al, bh, bl);
}
coopMatPerElementNV(x, x, StoreGramPmat);
}
void invert() {
const uint col = gl_SubgroupInvocationID % C;
// Each half-subgroup covers all columns of its assigned rows.
const uint section = gl_SubgroupInvocationID / C;
const uint row_base = (gl_SubgroupID * 2u + section) * (C / (2 * NSUB));
// Keep the diagonal zero so pivots at or above the row do nothing.
float x[C / (2 * NSUB)];
[[unroll]] for (uint j = 0; j < C / (2 * NSUB); ++j) {
const uint row = row_base + j;
x[j] = col < row ? -sh_lower[row * (row - 1) / 2 + col] : 0.0;
}
[[dont_unroll]] for (int pivot = C - 2; pivot > 0; --pivot) {
const float lower = col < uint(pivot) ? sh_lower[uint(pivot) * uint(pivot - 1) / 2 + col] : 0.0;
[[unroll]] for (uint j = 0; j < C / (2 * NSUB); ++j) {
const float xp = subgroupShuffle(x[j], uint(pivot) + section * C);
x[j] -= xp * lower;
}
}
[[unroll]] for (uint j = 0; j < C / (2 * NSUB); ++j) {
const uint row = row_base + j;
split(col == row ? 1.0 : x[j], sh_ih[row * C + col], sh_il[row * C + col]);
}
}
void main() {
const uint head_id = gl_WorkGroupID.z;
const uint seq_id = gl_WorkGroupID.y;
const uint v0 = gl_WorkGroupID.x * V;
const uint iq1 = head_id % p.neq1;
const uint iq3 = seq_id / p.rq3;
const uint q_base = iq3 * p.sq3 + iq1 * p.sq1;
const uint k_base = iq3 * p.sk3 + iq1 * p.sk1;
const uint v_base = seq_id * p.sv3 + head_id * p.sv1 + v0;
const uint gb_base = seq_id * p.sb3 + head_id * p.sb1;
const uint state_base = (seq_id * p.H + head_id) * (D * D) + v0 * D;
const uint out_base = (seq_id * p.n_tokens * p.H + head_id) * D + v0;
// Transposing the state-side products gives them V=32 rows with C=16 columns.
coopmat<float, WG, V, D, MACC> state;
coopMatLoad(state, data_state, state_base, D, ROW);
float qreg[NQK], kreg[NQK], greg, breg;
prefetch(0u, q_base, k_base, gb_base, qreg, kreg, greg, breg);
[[dont_unroll]] for (uint t0 = 0; t0 < p.n_tokens; t0 += C) {
const uint valid = min(C, p.n_tokens - t0);
barrier();
if (gl_SubgroupID == 0) {
const uint r = gl_SubgroupInvocationID;
// shuffle scan instead of subgroupInclusiveAdd: works around an NVIDIA r620 shader compiler crash
float sum = greg;
[[unroll]] for (uint d = 1; d < C; d <<= 1) {
const float t = subgroupShuffleUp(sum, d);
if (r >= d) {
sum += t;
}
}
const float last = subgroupShuffle(sum, valid - 1);
if (r < C) {
sh_prefix[r] = sum;
sh_ep[r] = exp(sum);
sh_beta[r] = breg;
sh_carry[r] = exp(last - sum);
}
}
stage_inputs(qreg, kreg);
coopmat<float, WG, V, C, MACC> vmat;
if (valid == C && (v_base % 4u) == 0 && (p.sv2 % 4u) == 0) {
coopMatLoad(vmat, data_v, v_base + t0 * p.sv2, p.sv2, COL);
} else {
vmat = coopmat<float, WG, V, C, MACC>(0.0);
coopMatPerElementNV(vmat, vmat, LoadV, v_base + t0 * p.sv2, p.sv2, valid);
}
barrier();
// Current inputs are in shared memory; keep the next chunk in registers.
prefetch(t0 + C, q_base, k_base, gb_base, qreg, kreg, greg, breg);
gram_pmat();
barrier();
invert();
coopMatPerElementNV(state, state, SplitRowS);
barrier();
coopmat<float, WG, V, C, MACC> ks = coopmat<float, WG, V, C, MACC>(0.0), base_r = coopmat<float, WG, V, C, MACC>(0.0);
[[unroll]] for (uint kk = 0; kk < D; kk += KT) {
coopmat<OPD, WG, V, KT, MUA> ah, al;
coopmat<OPD, WG, KT, C, MUB> kh, kl, qh, ql;
coopMatLoad(ah, sh_sh, kk, LDP, ROW);
coopMatLoad(al, sh_sl, kk, LDP, ROW);
coopMatLoad(kh, sh_kqh, kk, LDP, COL);
coopMatLoad(kl, sh_kql, kk, LDP, COL);
coopMatLoad(qh, sh_kqh, C * LDP + kk, LDP, COL);
coopMatLoad(ql, sh_kql, C * LDP + kk, LDP, COL);
PRODUCT(ks, ah, al, kh, kl);
PRODUCT(base_r, ah, al, qh, ql);
}
coopMatPerElementNV(base_r, base_r, ApplyExpPrefix);
coopMatPerElementNV(ks, ks, StoreDeltaV, vmat);
barrier();
coopmat<float, WG, V, C, MACC> solved = coopmat<float, WG, V, C, MACC>(0.0);
{
coopmat<OPD, WG, V, C, MUA> rh, rl;
coopmat<OPD, WG, C, C, MUB> ih, il;
coopMatLoad(rh, sh_dh, 0, LDV, COL);
coopMatLoad(rl, sh_dl, 0, LDV, COL);
coopMatLoad(ih, sh_ih, 0, C, COL);
coopMatLoad(il, sh_il, 0, C, COL);
PRODUCT(solved, rh, rl, ih, il);
}
barrier();
coopMatPerElementNV(solved, solved, StoreSolved);
barrier();
{
coopmat<OPD, WG, V, C, MUA> xh, xl;
coopmat<OPD, WG, C, C, MUB> ph, pl;
coopMatLoad(xh, sh_dh, 0, LDV, COL);
coopMatLoad(xl, sh_dl, 0, LDV, COL);
coopMatLoad(ph, sh_ph, 0, C, COL);
coopMatLoad(pl, sh_pl, 0, C, COL);
PRODUCT(base_r, xh, xl, ph, pl);
}
coopMatPerElementNV(base_r, base_r, StoreOut, out_base + t0 * (p.H * D), valid);
coopmat<OPD, WG, V, C, MUA> xh, xl;
coopmat<OPD, WG, C, D, MUB> kh, kl;
coopMatLoad(xh, sh_sh, 0, LDP, ROW);
coopMatLoad(xl, sh_sl, 0, LDP, ROW);
coopMatLoad(kh, sh_kqh, 0, LDP, ROW);
coopMatLoad(kl, sh_kql, 0, LDP, ROW);
state *= sh_ep[valid - 1];
PRODUCT(state, xh, xl, kh, kl);
}
coopMatStore(state, data_dst, p.s_off + state_base, D, ROW);
}
@@ -348,7 +348,10 @@ compile_count_guard acquire_compile_slot() {
}
void string_to_spv_func(std::string name, std::string in_path, std::string out_path, std::map<std::string, std::string> defines, bool coopmat, bool dep_file, compile_count_guard slot) {
std::string target_env = (name.find("_cm2") != std::string::npos) ? "--target-env=vulkan1.3" : "--target-env=vulkan1.2";
// coopmat2 and the maintenance1 GDN coopmat1 variants need SPIR-V 1.6 (vulkan1.3)
const bool needs_spv16 = name.find("_cm2") != std::string::npos ||
name.find("gated_delta_net_f32_cm1") != std::string::npos;
std::string target_env = needs_spv16 ? "--target-env=vulkan1.3" : "--target-env=vulkan1.2";
#ifdef _WIN32
std::vector<std::string> cmd = {GLSLC, "-fshader-stage=compute", target_env, "\"" + in_path + "\"", "-o", "\"" + out_path + "\""};
@@ -1128,6 +1131,20 @@ void process_shaders() {
string_to_spv("gated_delta_net_f32_nocluster", "gated_delta_net.comp", merge_maps(base_dict, {{"FLOAT_TYPE", "float"}, {"USE_SUBGROUP_ADD", "1"}, {"USE_SUBGROUP_CLUSTERED", "0"}}));
string_to_spv("gated_delta_net_f32_shmem", "gated_delta_net.comp", merge_maps(base_dict, {{"FLOAT_TYPE", "float"}, {"USE_SUBGROUP_ADD", "0"}, {"USE_SUBGROUP_CLUSTERED", "0"}}));
#if defined(GGML_VULKAN_BFLOAT16_GLSLC_SUPPORT) && defined(GGML_VULKAN_COOPMAT2_GLSLC_SUPPORT)
string_to_spv("gated_delta_net_f32", "gated_delta_net_cm2.comp", base_dict, true, false, true);
#endif
#if defined(GGML_VULKAN_BFLOAT16_GLSLC_SUPPORT) && defined(GGML_VULKAN_COOPMAT_GLSLC_SUPPORT) && defined(GGML_VULKAN_COOPMAT_MAINTENANCE1_GLSLC_SUPPORT)
string_to_spv("gated_delta_net_f32", "gated_delta_net_cm.comp", merge_maps(base_dict, {{"BF16ACC", "1"}}), true, true, false, false, "_bf16acc");
string_to_spv("gated_delta_net_f32", "gated_delta_net_cm.comp", base_dict, true, true);
string_to_spv("gated_delta_net_f32", "gated_delta_net_cm.comp", merge_maps(base_dict, {{"BF16ACC", "1"}, {"V", "32"}}), true, true, false, false, "_bf16acc_v32");
string_to_spv("gated_delta_net_f32", "gated_delta_net_cm.comp", merge_maps(base_dict, {{"V", "32"}}), true, true, false, false, "_v32");
// whole-head V=128 variant: state mirror in a gmem scratch buffer (won't fit LDS).
// For head counts that tail the CU grid at V=64 (e.g. 48-head 27B on 64 CUs).
string_to_spv("gated_delta_net_f32", "gated_delta_net_cm.comp", merge_maps(base_dict, {{"BF16ACC", "1"}, {"GMEM_MIRROR", "1"}, {"V", "128"}}), true, true, false, false, "_bf16acc_v128");
string_to_spv("gated_delta_net_f32", "gated_delta_net_cm.comp", merge_maps(base_dict, {{"GMEM_MIRROR", "1"}, {"V", "128"}}), true, true, false, false, "_v128");
#endif
string_to_spv("opt_step_adamw_f32", "opt_step_adamw.comp", merge_maps(base_dict, {{"A_TYPE", "float"}}));
string_to_spv("opt_step_sgd_f32", "opt_step_sgd.comp", merge_maps(base_dict, {{"A_TYPE", "float"}}));
+89 -7
View File
@@ -5261,16 +5261,20 @@ struct test_gated_delta_net : public test_case {
const bool permuted;
const bool kda;
const int64_t K; // snapshot slot count: 1 = final-only, >1 = last K states
const bool correlated;
const bool padded_v;
const bool padded_q;
const bool permuted_k;
std::string vars() override {
return VARS_TO_STR9(type, head_count, head_size, n_seq_tokens, n_seqs, v_repeat, permuted, kda, K);
return VARS_TO_STR13(type, head_count, head_size, n_seq_tokens, n_seqs, v_repeat, permuted, kda, K, correlated, padded_v, padded_q, permuted_k);
}
test_gated_delta_net(ggml_type type = GGML_TYPE_F32,
int64_t head_count = 4, int64_t head_size = 16, int64_t n_seq_tokens = 1, int64_t n_seqs = 1,
int v_repeat = 1, bool permuted = false, bool kda = false, int64_t K = 1)
int v_repeat = 1, bool permuted = false, bool kda = false, int64_t K = 1, bool correlated = false, bool padded_v = false, bool padded_q = false, bool permuted_k = false)
: type(type), head_count(head_count), head_size(head_size), n_seq_tokens(n_seq_tokens), n_seqs(n_seqs),
v_repeat(v_repeat), permuted(permuted), kda(kda), K(K) {}
v_repeat(v_repeat), permuted(permuted), kda(kda), K(K), correlated(correlated), padded_v(padded_v), padded_q(padded_q), permuted_k(permuted_k) {}
ggml_tensor * build_graph(ggml_context * ctx) override {
ggml_tensor * q;
@@ -5280,11 +5284,15 @@ struct test_gated_delta_net : public test_case {
// create with dims 1 and 2 swapped, then permute back to get non-contiguous layout
q = ggml_permute(ctx, ggml_new_tensor_4d(ctx, type, head_size, n_seq_tokens, head_count, n_seqs), 0, 2, 1, 3);
k = ggml_permute(ctx, ggml_new_tensor_4d(ctx, type, head_size, n_seq_tokens, head_count, n_seqs), 0, 2, 1, 3);
v = ggml_permute(ctx, ggml_new_tensor_4d(ctx, type, head_size, n_seq_tokens, head_count * v_repeat, n_seqs), 0, 2, 1, 3);
v = ggml_permute(ctx, ggml_new_tensor_4d(ctx, type, head_size + padded_v, n_seq_tokens, head_count * v_repeat, n_seqs), 0, 2, 1, 3);
} else {
q = ggml_new_tensor_4d(ctx, type, head_size, head_count, n_seq_tokens, n_seqs);
k = ggml_new_tensor_4d(ctx, type, head_size, head_count, n_seq_tokens, n_seqs);
v = ggml_new_tensor_4d(ctx, type, head_size, head_count * v_repeat, n_seq_tokens, n_seqs);
v = ggml_new_tensor_4d(ctx, type, head_size + padded_v, head_count * v_repeat, n_seq_tokens, n_seqs);
}
if (padded_v) {
ggml_set_name(v, "v");
v = ggml_view_4d(ctx, v, head_size, head_count * v_repeat, n_seq_tokens, n_seqs, v->nb[1], v->nb[2], v->nb[3], 0);
}
ggml_set_name(q, "q");
ggml_set_name(k, "k");
@@ -5299,6 +5307,17 @@ struct test_gated_delta_net : public test_case {
// q/k are L2-normalised in qwen35/kimi-linear before delta_net
q = ggml_l2_norm(ctx, q, 1e-6f);
k = ggml_l2_norm(ctx, k, 1e-6f);
// Set layouts after normalization, which produces contiguous tensors.
if (padded_q) {
ggml_tensor * q_padded = ggml_new_tensor_4d(ctx, type, head_size + 1, head_count, n_seq_tokens, n_seqs);
q_padded = ggml_view_4d(ctx, q_padded, head_size, head_count, n_seq_tokens, n_seqs, q_padded->nb[1], q_padded->nb[2], q_padded->nb[3], 0);
q = ggml_cpy(ctx, q, q_padded);
}
if (permuted_k) {
ggml_tensor * k_permuted = ggml_new_tensor_4d(ctx, type, head_size, n_seq_tokens, head_count, n_seqs);
k_permuted = ggml_permute(ctx, k_permuted, 0, 2, 1, 3);
k = ggml_cpy(ctx, k, k_permuted);
}
ggml_tensor * out = ggml_gated_delta_net(ctx, q, k, v, g, beta, state, K);
return out;
}
@@ -5307,9 +5326,17 @@ struct test_gated_delta_net : public test_case {
for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != nullptr; t = ggml_get_next_tensor(ctx, t)) {
if (ggml_is_view_op(t->op)) { continue; }
if (strcmp(t->name, "g") == 0) {
init_tensor_uniform(t, -20.0f, -1e-4f);
init_tensor_uniform(t, correlated ? -1e-4f : -20.0f, -1e-4f);
} else if (strcmp(t->name, "beta") == 0) {
init_tensor_uniform(t, 0.0f, 1.0f);
init_tensor_uniform(t, correlated ? 0.9f : 0.0f, correlated ? 0.9f : 1.0f);
} else if (correlated && strcmp(t->name, "k") == 0) {
// Identical unit keys and weak decay stress cancellation in the triangular solve.
GGML_ASSERT(t->type == GGML_TYPE_F32);
std::vector<float> data(ggml_nelements(t), 0.0f);
for (size_t i = 0; i < data.size(); i += size_t(t->ne[0])) {
data[i] = 1.0f;
}
ggml_backend_tensor_set(t, data.data(), 0, data.size() * sizeof(float));
} else if (strcmp(t->name, "v") == 0) {
init_tensor_uniform(t, -0.3f, 5.0f);
} else {
@@ -13634,6 +13661,15 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() {
test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 4, 64, 33, 1, 1, false, true));
test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 4, 64, 100, 1, 1, false, true));
// KDA at D=128: decode, prefill, tails, strides, and snapshots.
test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 32, 128, 1, 1, 1, false, true));
test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 4, 128, 64, 1, 1, false, true));
test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 4, 128, 65, 2, 2, true, true));
test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 4, 128, 257, 1, 1, false, true));
test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 4, 128, 65, 1, 1, false, true, 1, true));
test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 4, 128, 257, 1, 1, true, true, 1, true, true));
test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 4, 128, 4, 2, 2, false, true, 4));
// MMA-eligible GDN shapes: whole heads, value slices, tails, and the HIP prefill threshold.
test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 16, 128, 65, 1, 1));
test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 16, 128, 65, 3, 4));
@@ -13645,6 +13681,30 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() {
test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 16, 128, 2048, 1, 2));
// Keep the WebGPU dispatch below 65535 workgroups (32 per token for this shape).
test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 16, 128, 2047, 2, 3));
test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 4, 128, 4097, 1, 2));
test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 16, 128, 127, 2));
test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 16, 128, 256, 1, 2));
test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 16, 128, 256, 1, 3));
test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 16, 128, 513, 1, 2));
test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 16, 128, 512, 2, 2));
test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 4, 128, 257, 2, 2, true));
test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 3, 128, 129, 1, 1, false, false, 1, false, true));
test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 4, 128, 4095, 1, 2));
// Correlated keys, high beta, and weak decay stress the chunked triangular solve.
for (int64_t n_tokens : {64, 65, 257}) {
test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 4, 128, n_tokens, 1, 1, false, false, 1, true));
}
// Odd V strides require element loads instead of aligned matrix loads.
test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 3, 128, 65, 1, 1, false, false, 1, false, true));
// Q and K can have different strides in both scalar and chunked paths.
for (int64_t n_tokens : {4, 65}) {
test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 2, 128, n_tokens, 2, 2, false, false, 1, false, false, true));
test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 2, 128, n_tokens, 2, 2, false, false, 1, false, false, false, true));
}
// K > 1: output keeps the last min(n_tokens, K) per-token snapshots, ordered most-recent-first
// (slot 0 = final state, slot s = state s tokens back).
@@ -14296,12 +14356,34 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_perf() {
test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 32, 128, 256, 1)); // PP-256
test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 32, 128, 512, 1)); // PP-512
test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 32, 128, 1024, 1)); // PP-1024
// MMA-eligible prefill sweep (16 heads, d=128, K=1): Vulkan cm2 vs CUDA MMA
test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 16, 128, 64, 1));
test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 16, 128, 128, 1));
test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 16, 128, 256, 1));
test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 16, 128, 512, 1));
test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 16, 128, 1024, 1));
test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 16, 128, 2048, 1));
test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 16, 128, 4096, 1));
// Small model configs (fewer heads = less GPU occupancy for autoregressive)
test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 4, 128, 64, 1)); // 4h PP-64
test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 4, 128, 256, 1)); // 4h PP-256
test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 4, 128, 512, 1)); // 4h PP-512
test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 4, 128, 1024, 1)); // 4h PP-1024
test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 32, 128, 64, 1, 1, false, true)); // KDA PP-64
test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 4, 128, 64, 1, 1, false, true));
test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 16, 128, 256, 1, 1, false, true));
test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 4, 128, 1024, 1, 1, false, true));
// GDN model shapes: 35B (v_repeat=2), 27B (v_repeat=3), batch sweep
for (int T : { 2048, 4096, 8192, 16384, 32768 }) {
test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 16, 128, T, 1, 2));
test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 16, 128, T, 1, 3));
}
test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 4, 128, 2048, 1));
test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 4, 128, 4096, 1));
test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 4, 128, 8192, 1));
test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 20, 128, 1536, 1));
test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 20, 128, 2048, 1));
// lightning_indexer
for (int kv : { 256, 4096, 65536 }) {