vulkan: chunked GATED_DELTA_NET implementation (#30207)
* vulkan: add cm2 chunked GATED_DELTA_NET shader * fix shmem bank conflict bottleneck * optimize further * add coopmat maintenance1 variant * optimize for RDNA4 * add gmem version RDNA4 * optimize for Ampere * fix nvidia shmem overuse * fix cm2 validation errors * improve cm2 perf * improve cm1 perf * clean up * enable by default * show cm maintenance1 marker in device info * properly check for cm_m1 features * add tests * add v32 mode, improve chunked condition * allow other subgroup sizes * vulkan: specialize the GDN CM2 workgroup size Assisted-by: Codex * vulkan: coopmat2 GDN implementation * work around Nvidia shader compiler crash * remove shared shader cm2 code --------- Co-authored-by: Jeff Bolz <jbolz@nvidia.com>
This commit is contained in:
1 parent
2d8d003937
commit
16f80bd0c0
11 files changed
+1134
-22
No files matched your search
@@ -101,6 +101,12 @@ if (Vulkan_FOUND)
|
||||
"GGML_VULKAN_COOPMAT2_DECODE_VECTOR_GLSLC_SUPPORT"
|
||||
)
|
||||
|
||||
test_shader_extension_support(
|
||||
"GL_EXT_cooperative_matrix_maintenance1"
|
||||
"${CMAKE_CURRENT_SOURCE_DIR}/vulkan-shaders/feature-tests/coopmat_maintenance1.comp"
|
||||
"GGML_VULKAN_COOPMAT_MAINTENANCE1_GLSLC_SUPPORT"
|
||||
)
|
||||
|
||||
test_shader_extension_support(
|
||||
"GL_EXT_integer_dot_product"
|
||||
"${CMAKE_CURRENT_SOURCE_DIR}/vulkan-shaders/feature-tests/integer_dot.comp"
|
||||
|
||||
@@ -787,6 +787,7 @@ struct vk_op_gated_delta_net_push_constants {
|
||||
uint32_t n_seqs;
|
||||
uint32_t s_off;
|
||||
uint32_t sq1, sq2, sq3;
|
||||
uint32_t sk1, sk2, sk3;
|
||||
uint32_t sv1, sv2, sv3;
|
||||
uint32_t sb1, sb2, sb3;
|
||||
uint32_t neq1, rq3;
|
||||
|
||||
@@ -36,6 +36,21 @@ typedef struct VkPhysicalDeviceCooperativeMatrixDecodeVectorFeaturesNV {
|
||||
} VkPhysicalDeviceCooperativeMatrixDecodeVectorFeaturesNV;
|
||||
#endif
|
||||
|
||||
#ifndef VK_EXT_cooperative_matrix_maintenance1
|
||||
#define VK_EXT_cooperative_matrix_maintenance1 1
|
||||
#define VK_EXT_COOPERATIVE_MATRIX_MAINTENANCE_1_EXTENSION_NAME "VK_EXT_cooperative_matrix_maintenance1"
|
||||
#define VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_COOPERATIVE_MATRIX_MAINTENANCE_1_FEATURES_EXT ((VkStructureType)1000659000)
|
||||
typedef struct VkPhysicalDeviceCooperativeMatrixMaintenance1FeaturesEXT {
|
||||
VkStructureType sType;
|
||||
void* pNext;
|
||||
VkBool32 cooperativeMatrixProperties2;
|
||||
VkBool32 cooperativeMatrixReductions;
|
||||
VkBool32 cooperativeMatrixConversions;
|
||||
VkBool32 cooperativeMatrixPerElementOperations;
|
||||
VkBool32 cooperativeMatrixGetCoordinate;
|
||||
} VkPhysicalDeviceCooperativeMatrixMaintenance1FeaturesEXT;
|
||||
#endif
|
||||
|
||||
#if __has_include(<spirv/unified1/spirv.hpp>)
|
||||
# include <spirv/unified1/spirv.hpp>
|
||||
#elif __has_include(<spirv-headers/spirv.hpp>)
|
||||
@@ -221,6 +236,12 @@ static bool is_pow2(uint32_t x) { return x > 1 && (x & (x-1)) == 0; }
|
||||
|
||||
#define MAX_FUSED_ADDS (MAX_PARAMETER_COUNT - 3)
|
||||
|
||||
#define GGML_VK_GDN_CM_V 64u
|
||||
#define GGML_VK_GDN_CM2_V 32u
|
||||
#define GGML_VK_GDN_CM_D 128u
|
||||
#define GGML_VK_GDN_CM_PAD 8u
|
||||
#define GGML_VK_GDN_CM_LDP (GGML_VK_GDN_CM_D + GGML_VK_GDN_CM_PAD)
|
||||
|
||||
struct vk_pipeline_struct;
|
||||
|
||||
typedef std::shared_ptr<struct vk_pipeline_struct> vk_pipeline;
|
||||
@@ -767,6 +788,12 @@ struct vk_device_struct {
|
||||
bool coopmat_acc_f32_support {};
|
||||
bool coopmat_acc_f16_support {};
|
||||
bool coopmat_bf16_support {};
|
||||
bool coopmat_bf16_acc_support {};
|
||||
bool coopmat_maintenance1 {};
|
||||
bool coopmat_m1_reductions {};
|
||||
bool coopmat_m1_conversions {};
|
||||
bool coopmat_m1_per_element_ops {};
|
||||
bool coopmat_m1_get_coordinate {};
|
||||
bool coopmat_support_16x16x16_f16acc {};
|
||||
bool coopmat_support_16x16x16_f32acc {};
|
||||
bool coopmat1_fa_support {};
|
||||
@@ -781,6 +808,7 @@ struct vk_device_struct {
|
||||
|
||||
bool coopmat2;
|
||||
bool coopmat2_bf16_support {};
|
||||
bool coopmat2_gdn_support {};
|
||||
bool coopmat2_decode_vector;
|
||||
|
||||
bool dot2_f16 {};
|
||||
@@ -987,6 +1015,9 @@ struct vk_device_struct {
|
||||
vk_pipeline pipeline_lightning_indexer_f32[GGML_TYPE_COUNT];
|
||||
// [size_idx][kda] where size_idx: 0=d16, 1=d32, 2=d64, 3=d128
|
||||
vk_pipeline pipeline_gated_delta_net[4][2];
|
||||
vk_pipeline pipeline_gated_delta_net_cm;
|
||||
vk_pipeline pipeline_gated_delta_net_cm_v32;
|
||||
vk_pipeline pipeline_gated_delta_net_cm_v128;
|
||||
vk_pipeline pipeline_ssm_scan_f32_d128;
|
||||
vk_pipeline pipeline_ssm_scan_f32_d256;
|
||||
vk_pipeline pipeline_ssm_conv_f32;
|
||||
|
||||
@@ -1667,6 +1667,22 @@ static uint32_t ggml_vk_mul_mat_vec_max_cols(const vk_device & device) {
|
||||
return device->coopmat2 && !device->disable_narrow_mul_mat ? 4u : mul_mat_vec_max_cols;
|
||||
}
|
||||
|
||||
static uint32_t ggml_vk_gated_delta_net_shmem_size(uint32_t v, bool coopmat2) {
|
||||
const uint32_t C = 16;
|
||||
const uint32_t LDP = GGML_VK_GDN_CM_LDP;
|
||||
const uint32_t bf16_size = sizeof(ggml_bf16_t);
|
||||
const bool gmem_mirror = !coopmat2 && v == 128;
|
||||
|
||||
const uint32_t qk = 4 * C * LDP * bf16_size;
|
||||
const uint32_t state = gmem_mirror ? 0 : 2 * v * LDP * bf16_size;
|
||||
const uint32_t p_inv = 4 * C * C * bf16_size;
|
||||
const uint32_t delta_solved = coopmat2 ? 2 * C * (v + GGML_VK_GDN_CM_PAD) * bf16_size : 4 * C * v * bf16_size;
|
||||
const uint32_t lower = (coopmat2 ? C * (C - 1) / 2 : C * C) * sizeof(float);
|
||||
const uint32_t scan = 4 * C * sizeof(float);
|
||||
|
||||
return qk + state + p_inv + delta_solved + lower + scan;
|
||||
}
|
||||
|
||||
static bool ggml_vk_matmul_shmem_support(const vk_device& device, const std::vector<uint32_t>& warptile, bool mul_mat_id, ggml_type src0_type) {
|
||||
|
||||
uint32_t lut_size = 0;
|
||||
@@ -4124,13 +4140,14 @@ void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) {
|
||||
device->subgroup_min_size <= 16u && device->subgroup_max_size >= 16u) {
|
||||
gdn_subgroup_size = 16u;
|
||||
}
|
||||
const bool gdn_packed_rows = S_V == 128u && device->vendor_id == VK_VENDOR_ID_NVIDIA &&
|
||||
gdn_subgroup_size == 32u && device->subgroup_arithmetic && device->subgroup_clustered;
|
||||
uint32_t lanes_per_column;
|
||||
if (device->vendor_id == VK_VENDOR_ID_INTEL) {
|
||||
// Intel Xe: full-width reduction (min rows/lane) ~10x over the COLS_PER_WG=8 rule.
|
||||
lanes_per_column = std::min(gdn_subgroup_size, S_V);
|
||||
} else if (S_V >= 128u && device->subgroup_clustered) {
|
||||
// COLS_PER_WG=8: measured optimum (Ampere sg32->4, Vega20 sg64->8).
|
||||
lanes_per_column = std::max(1u, gdn_subgroup_size / 8u);
|
||||
lanes_per_column = gdn_packed_rows ? 8u : std::max(1u, gdn_subgroup_size / 8u);
|
||||
} else {
|
||||
// Use largest power-of-two that divides both S_V and subgroup_size so that
|
||||
// (1) S_V % lanes_per_column == 0 and (2) S_V % (subgroup_size / lanes_per_column) == 0.
|
||||
@@ -4177,9 +4194,60 @@ void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) {
|
||||
for (uint32_t kda = 0; kda < 2; kda++) {
|
||||
ggml_vk_create_pipeline(device, device->pipeline_gated_delta_net[si][kda],
|
||||
gdn_names[si][kda], gdn_len, gdn_data, "main", 7, sizeof(vk_op_gated_delta_net_push_constants),
|
||||
wg_denoms, {S_V, kda, gdn_subgroup_size, lanes_per_column}, 1, true, use_subgroup_ops, gdn_subgroup_size);
|
||||
wg_denoms, {S_V, kda, gdn_subgroup_size, lanes_per_column, uint32_t(gdn_packed_rows)}, 1, true, use_subgroup_ops, gdn_subgroup_size);
|
||||
}
|
||||
}
|
||||
|
||||
// Chunked GDN prefill uses workgroup matrices on CM2 and subgroup matrices on CM1.
|
||||
const bool gdn_chunked = getenv("GGML_VK_DISABLE_GDN_CHUNKED") == nullptr;
|
||||
const uint32_t gdn_shmem = device->properties.limits.maxComputeSharedMemorySize;
|
||||
#if defined(VK_NV_cooperative_matrix2) && defined(GGML_VULKAN_COOPMAT2_GLSLC_SUPPORT) && defined(GGML_VULKAN_BFLOAT16_GLSLC_SUPPORT)
|
||||
if (gdn_chunked && device->coopmat2 && device->coopmat2_gdn_support) {
|
||||
ggml_vk_create_pipeline(device, device->pipeline_gated_delta_net_cm,
|
||||
"gated_delta_net_f32_cm2", gated_delta_net_f32_cm2_len, gated_delta_net_f32_cm2_data,
|
||||
"main", 7, sizeof(vk_op_gated_delta_net_push_constants), {1, 1, 1}, {}, 1, true, true, 32);
|
||||
}
|
||||
#endif
|
||||
#if defined(GGML_VULKAN_COOPMAT_GLSLC_SUPPORT) && defined(GGML_VULKAN_COOPMAT_MAINTENANCE1_GLSLC_SUPPORT) && defined(GGML_VULKAN_BFLOAT16_GLSLC_SUPPORT)
|
||||
// shader needs >= 16 lanes and 16 subgroups per WG; prefer wave64
|
||||
auto gdn_sg_ok = [&](uint32_t sg) {
|
||||
const bool reachable = device->subgroup_size_control ? device->subgroup_min_size <= sg && sg <= device->subgroup_max_size
|
||||
: device->subgroup_size == sg;
|
||||
return reachable && 16u * sg <= device->properties.limits.maxComputeWorkGroupInvocations &&
|
||||
16u * sg <= device->properties.limits.maxComputeWorkGroupSize[0];
|
||||
};
|
||||
const uint32_t gdn_sg = gdn_sg_ok(64) ? 64u : device->subgroup_size;
|
||||
if (gdn_chunked && !device->coopmat2 && device->coopmat_support && device->coopmat_bf16_support && device->coopmat_m1_per_element_ops &&
|
||||
gdn_sg >= 16 && gdn_sg_ok(gdn_sg)) {
|
||||
const bool bf16acc = device->coopmat_bf16_acc_support;
|
||||
const std::vector<uint32_t> gdn_spec = { 16u * gdn_sg };
|
||||
// V=64: state mirror in shared, 7 bindings
|
||||
if (gdn_shmem >= ggml_vk_gated_delta_net_shmem_size(GGML_VK_GDN_CM_V, false)) {
|
||||
ggml_vk_create_pipeline(device, device->pipeline_gated_delta_net_cm,
|
||||
bf16acc ? "gated_delta_net_f32_cm1_bf16acc" : "gated_delta_net_f32_cm1",
|
||||
bf16acc ? gated_delta_net_f32_cm1_bf16acc_len : gated_delta_net_f32_cm1_len,
|
||||
bf16acc ? (const void *)gated_delta_net_f32_cm1_bf16acc_data : (const void *)gated_delta_net_f32_cm1_data,
|
||||
"main", 7, sizeof(vk_op_gated_delta_net_push_constants), {1, 1, 1}, gdn_spec, 1, true, true, gdn_sg);
|
||||
}
|
||||
if (gdn_shmem >= ggml_vk_gated_delta_net_shmem_size(32u, false)) {
|
||||
ggml_vk_create_pipeline(device, device->pipeline_gated_delta_net_cm_v32,
|
||||
bf16acc ? "gated_delta_net_f32_cm1_bf16acc_v32" : "gated_delta_net_f32_cm1_v32",
|
||||
bf16acc ? gated_delta_net_f32_cm1_bf16acc_v32_len : gated_delta_net_f32_cm1_v32_len,
|
||||
bf16acc ? (const void *)gated_delta_net_f32_cm1_bf16acc_v32_data : (const void *)gated_delta_net_f32_cm1_v32_data,
|
||||
"main", 7, sizeof(vk_op_gated_delta_net_push_constants), {1, 1, 1}, gdn_spec, 1, true, true, gdn_sg);
|
||||
}
|
||||
// V=128: state mirror in a gmem scratch buffer, 8 bindings
|
||||
if (gdn_shmem >= ggml_vk_gated_delta_net_shmem_size(128u, false)) {
|
||||
ggml_vk_create_pipeline(device, device->pipeline_gated_delta_net_cm_v128,
|
||||
bf16acc ? "gated_delta_net_f32_cm1_bf16acc_v128" : "gated_delta_net_f32_cm1_v128",
|
||||
bf16acc ? gated_delta_net_f32_cm1_bf16acc_v128_len : gated_delta_net_f32_cm1_v128_len,
|
||||
bf16acc ? (const void *)gated_delta_net_f32_cm1_bf16acc_v128_data : (const void *)gated_delta_net_f32_cm1_v128_data,
|
||||
"main", 8, sizeof(vk_op_gated_delta_net_push_constants), {1, 1, 1}, gdn_spec, 1, true, true, gdn_sg);
|
||||
}
|
||||
}
|
||||
#endif
|
||||
GGML_UNUSED(gdn_chunked);
|
||||
GGML_UNUSED(gdn_shmem);
|
||||
}
|
||||
|
||||
if (device->subgroup_arithmetic && device->subgroup_require_full_support) {
|
||||
@@ -4554,6 +4622,11 @@ vk_device ggml_vk_get_device(size_t idx) {
|
||||
} else if (strcmp("VK_NV_cooperative_matrix2", properties.extensionName) == 0 &&
|
||||
!getenv("GGML_VK_DISABLE_COOPMAT2")) {
|
||||
coopmat2_support = true;
|
||||
#endif
|
||||
#if defined(GGML_VULKAN_COOPMAT_MAINTENANCE1_GLSLC_SUPPORT)
|
||||
} else if (strcmp("VK_EXT_cooperative_matrix_maintenance1", properties.extensionName) == 0 &&
|
||||
!getenv("GGML_VK_DISABLE_COOPMAT_MAINTENANCE1")) {
|
||||
device->coopmat_maintenance1 = true;
|
||||
#endif
|
||||
} else if (strcmp(VK_NV_COOPERATIVE_MATRIX_DECODE_VECTOR_EXTENSION_NAME, properties.extensionName) == 0 &&
|
||||
!getenv("GGML_VK_DISABLE_COOPMAT2_DECODE_VECTOR")) {
|
||||
@@ -4880,6 +4953,15 @@ vk_device ggml_vk_get_device(size_t idx) {
|
||||
}
|
||||
#endif
|
||||
|
||||
#if defined(GGML_VULKAN_COOPMAT_MAINTENANCE1_GLSLC_SUPPORT)
|
||||
VkPhysicalDeviceCooperativeMatrixMaintenance1FeaturesEXT coopmat_m1_features {};
|
||||
coopmat_m1_features.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_COOPERATIVE_MATRIX_MAINTENANCE_1_FEATURES_EXT;
|
||||
if (device->coopmat_support && device->coopmat_maintenance1) {
|
||||
last_struct->pNext = (VkBaseOutStructure *)&coopmat_m1_features;
|
||||
last_struct = (VkBaseOutStructure *)&coopmat_m1_features;
|
||||
}
|
||||
#endif
|
||||
|
||||
VkPhysicalDeviceCooperativeMatrixDecodeVectorFeaturesNV coopmat2_decode_vector_features {};
|
||||
coopmat2_decode_vector_features.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_COOPERATIVE_MATRIX_DECODE_VECTOR_FEATURES_NV;
|
||||
if (coopmat2_decode_vector_support) {
|
||||
@@ -5064,6 +5146,17 @@ vk_device ggml_vk_get_device(size_t idx) {
|
||||
device->coopmat1_fa_support = device->coopmat_support && device->subgroup_require_full_support;
|
||||
#endif
|
||||
|
||||
#if defined(GGML_VULKAN_COOPMAT_MAINTENANCE1_GLSLC_SUPPORT)
|
||||
device->coopmat_maintenance1 = device->coopmat_support && device->coopmat_maintenance1;
|
||||
if (device->coopmat_maintenance1) {
|
||||
device->coopmat_m1_reductions = coopmat_m1_features.cooperativeMatrixReductions;
|
||||
device->coopmat_m1_conversions = coopmat_m1_features.cooperativeMatrixConversions;
|
||||
device->coopmat_m1_per_element_ops = coopmat_m1_features.cooperativeMatrixPerElementOperations;
|
||||
device->coopmat_m1_get_coordinate = coopmat_m1_features.cooperativeMatrixGetCoordinate;
|
||||
device_extensions.push_back(VK_EXT_COOPERATIVE_MATRIX_MAINTENANCE_1_EXTENSION_NAME);
|
||||
}
|
||||
#endif
|
||||
|
||||
if (coopmat2_support) {
|
||||
#if defined(VK_NV_cooperative_matrix2) && defined(GGML_VULKAN_COOPMAT2_GLSLC_SUPPORT)
|
||||
if (coopmat2_features.cooperativeMatrixWorkgroupScope &&
|
||||
@@ -5147,6 +5240,13 @@ vk_device ggml_vk_get_device(size_t idx) {
|
||||
prop.KGranularity <= 16) {
|
||||
found_bf16_128 = true;
|
||||
}
|
||||
// GDN uses M=32, with N/K dimensions that are multiples of 16.
|
||||
if (prop.workgroupInvocations == 128 &&
|
||||
prop.MGranularity != 0 && 32 % prop.MGranularity == 0 &&
|
||||
prop.NGranularity != 0 && 16 % prop.NGranularity == 0 &&
|
||||
prop.KGranularity != 0 && 16 % prop.KGranularity == 0) {
|
||||
device->coopmat2_gdn_support = true;
|
||||
}
|
||||
if (prop.workgroupInvocations == 256 &&
|
||||
prop.MGranularity <= 32 &&
|
||||
prop.NGranularity <= 32 &&
|
||||
@@ -5157,6 +5257,12 @@ vk_device ggml_vk_get_device(size_t idx) {
|
||||
#endif
|
||||
}
|
||||
}
|
||||
device->coopmat2_gdn_support = device->coopmat2_gdn_support &&
|
||||
coopmat2_props.cooperativeMatrixWorkgroupScopeMaxWorkgroupSize >= 128 &&
|
||||
device->properties.limits.maxComputeWorkGroupInvocations >= 128 &&
|
||||
device->properties.limits.maxComputeWorkGroupSize[0] >= 128 &&
|
||||
uint64_t(ggml_vk_gated_delta_net_shmem_size(GGML_VK_GDN_CM2_V, true)) + coopmat2_props.cooperativeMatrixWorkgroupScopeReservedSharedMemory <=
|
||||
device->properties.limits.maxComputeSharedMemorySize;
|
||||
if (found_fp16_128 && found_fp16_256 &&
|
||||
found_fp32_128 && found_fp32_256 &&
|
||||
coopmat2_props.cooperativeMatrixFlexibleDimensionsMaxDimension >= 512) {
|
||||
@@ -5274,6 +5380,18 @@ vk_device ggml_vk_get_device(size_t idx) {
|
||||
device->coopmat_bf16_support = true;
|
||||
}
|
||||
}
|
||||
// bf16 accumulator (bf16/bf16/bf16/bf16) at the shader's fixed 16x16x16
|
||||
// shape lets the GDN coopmat1 path keep the hi/lo split as accumulators.
|
||||
if (bfloat16_support &&
|
||||
prop.AType == VK_COMPONENT_TYPE_BFLOAT16_KHR &&
|
||||
prop.BType == VK_COMPONENT_TYPE_BFLOAT16_KHR &&
|
||||
prop.CType == VK_COMPONENT_TYPE_BFLOAT16_KHR &&
|
||||
prop.ResultType == VK_COMPONENT_TYPE_BFLOAT16_KHR &&
|
||||
(vk::ScopeKHR)prop.scope == vk::ScopeKHR::eSubgroup &&
|
||||
prop.MSize == 16 && prop.NSize == 16 && prop.KSize == 16
|
||||
) {
|
||||
device->coopmat_bf16_acc_support = true;
|
||||
}
|
||||
#endif
|
||||
}
|
||||
|
||||
@@ -5489,6 +5607,7 @@ static void ggml_vk_print_gpu_info(size_t idx) {
|
||||
bool coopmat_support = false;
|
||||
bool coopmat2_support = false;
|
||||
bool coopmat2_decode_vector_support = false;
|
||||
bool coopmat_maintenance1_support = false;
|
||||
bool integer_dot_product = false;
|
||||
bool bfloat16_support = false;
|
||||
bool dot2_f16_support = false;
|
||||
@@ -5509,6 +5628,11 @@ static void ggml_vk_print_gpu_info(size_t idx) {
|
||||
} else if (strcmp("VK_NV_cooperative_matrix2", properties.extensionName) == 0 &&
|
||||
!getenv("GGML_VK_DISABLE_COOPMAT2")) {
|
||||
coopmat2_support = true;
|
||||
#endif
|
||||
#if defined(GGML_VULKAN_COOPMAT_MAINTENANCE1_GLSLC_SUPPORT)
|
||||
} else if (strcmp("VK_EXT_cooperative_matrix_maintenance1", properties.extensionName) == 0 &&
|
||||
!getenv("GGML_VK_DISABLE_COOPMAT_MAINTENANCE1")) {
|
||||
coopmat_maintenance1_support = true;
|
||||
#endif
|
||||
} else if (strcmp(VK_NV_COOPERATIVE_MATRIX_DECODE_VECTOR_EXTENSION_NAME, properties.extensionName) == 0 &&
|
||||
!getenv("GGML_VK_DISABLE_COOPMAT2_DECODE_VECTOR")) {
|
||||
@@ -5592,6 +5716,15 @@ static void ggml_vk_print_gpu_info(size_t idx) {
|
||||
}
|
||||
#endif
|
||||
|
||||
#if defined(GGML_VULKAN_COOPMAT_MAINTENANCE1_GLSLC_SUPPORT)
|
||||
VkPhysicalDeviceCooperativeMatrixMaintenance1FeaturesEXT coopmat_m1_features {};
|
||||
coopmat_m1_features.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_COOPERATIVE_MATRIX_MAINTENANCE_1_FEATURES_EXT;
|
||||
if (coopmat_support && coopmat_maintenance1_support) {
|
||||
last_struct->pNext = (VkBaseOutStructure *)&coopmat_m1_features;
|
||||
last_struct = (VkBaseOutStructure *)&coopmat_m1_features;
|
||||
}
|
||||
#endif
|
||||
|
||||
VkPhysicalDeviceShaderIntegerDotProductFeaturesKHR shader_integer_dot_product_features {};
|
||||
shader_integer_dot_product_features.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_SHADER_INTEGER_DOT_PRODUCT_FEATURES_KHR;
|
||||
if (integer_dot_product) {
|
||||
@@ -5685,13 +5818,17 @@ static void ggml_vk_print_gpu_info(size_t idx) {
|
||||
coopmat2_support = false;
|
||||
#endif
|
||||
|
||||
#if defined(GGML_VULKAN_COOPMAT_MAINTENANCE1_GLSLC_SUPPORT)
|
||||
coopmat_maintenance1_support = coopmat_maintenance1_support && coopmat_m1_features.cooperativeMatrixPerElementOperations;
|
||||
#endif
|
||||
|
||||
coopmat2_decode_vector_support = coopmat2_decode_vector_support && coopmat2_decode_vector_features.cooperativeMatrixDecodeVector;
|
||||
#if !defined(GGML_VULKAN_COOPMAT2_DECODE_VECTOR_GLSLC_SUPPORT)
|
||||
coopmat2_decode_vector_support = false;
|
||||
#endif
|
||||
|
||||
std::string matrix_cores = coopmat2_support ? (coopmat2_decode_vector_support ? "NV_coopmat2v" : "NV_coopmat2")
|
||||
: coopmat_support ? "KHR_coopmat"
|
||||
: coopmat_support ? (coopmat_maintenance1_support ? "KHR_coopmat_m1" : "KHR_coopmat")
|
||||
: "none";
|
||||
|
||||
bool dot2_f16 = dot2_f16_support && dot2_features.shaderMixedFloatDotProductFloat16AccFloat32;
|
||||
@@ -11172,6 +11309,7 @@ void ggml_vk_lightning_indexer(ggml_backend_vk_context * ctx, vk_context& subctx
|
||||
|
||||
void ggml_vk_gated_delta_net(ggml_backend_vk_context * ctx, vk_context& subctx, ggml_tensor * dst) {
|
||||
const ggml_tensor * src_q = dst->src[0];
|
||||
const ggml_tensor * src_k = dst->src[1];
|
||||
const ggml_tensor * src_v = dst->src[2];
|
||||
const ggml_tensor * src_beta = dst->src[4];
|
||||
|
||||
@@ -11187,7 +11325,39 @@ void ggml_vk_gated_delta_net(ggml_backend_vk_context * ctx, vk_context& subctx,
|
||||
|
||||
const uint32_t s_off = S_v * H * n_tokens * n_seqs;
|
||||
|
||||
vk_pipeline pipeline = ggml_vk_op_get_pipeline(ctx, dst->src[0], dst->src[1], dst->src[2], dst, dst->op);
|
||||
const ggml_tensor * src_g = dst->src[3];
|
||||
// RDNA3: the scan is faster below ~4k tokens
|
||||
const uint32_t cm_min_tokens = ctx->device->architecture == AMD_RDNA3 ? 4096u : 64u;
|
||||
const bool use_cm = (ctx->device->pipeline_gated_delta_net_cm != nullptr ||
|
||||
ctx->device->pipeline_gated_delta_net_cm_v32 != nullptr ||
|
||||
ctx->device->pipeline_gated_delta_net_cm_v128 != nullptr) &&
|
||||
src_g->ne[0] == 1 && S_v == 128 && dst->src[0]->ne[0] == 128 && K == 1 && n_tokens >= cm_min_tokens;
|
||||
|
||||
const bool use_cm2 = use_cm && ctx->device->coopmat2;
|
||||
const bool have_v64 = ctx->device->pipeline_gated_delta_net_cm != nullptr;
|
||||
bool use_v32 = use_cm && !use_cm2 && !have_v64 && ctx->device->pipeline_gated_delta_net_cm_v32 != nullptr;
|
||||
bool use_v128 = use_cm && !use_cm2 && !have_v64 && !use_v32;
|
||||
const uint32_t nsm = ctx->device->shader_core_count;
|
||||
if (use_cm && !use_cm2 && have_v64 && nsm != 0) {
|
||||
const uint32_t wg64 = 2u * H * n_seqs;
|
||||
// V=32 only pays off while V=64 leaves at least half the cores idle
|
||||
use_v32 = ctx->device->pipeline_gated_delta_net_cm_v32 != nullptr && 2u * wg64 <= nsm;
|
||||
if (!use_v32 && ctx->device->pipeline_gated_delta_net_cm_v128 != nullptr) {
|
||||
const uint32_t wg128 = H * n_seqs;
|
||||
const uint32_t waves64 = (wg64 + nsm - 1) / nsm;
|
||||
const uint32_t waves128 = (wg128 + nsm - 1) / nsm;
|
||||
// prefer v128 when occupancy (wgs / (waves * nsm)) is at least as good
|
||||
use_v128 = (uint64_t)wg128 * waves64 >= (uint64_t)wg64 * waves128;
|
||||
}
|
||||
}
|
||||
|
||||
const uint32_t cm_V = use_cm2 ? GGML_VK_GDN_CM2_V : (use_v128 ? 128u : (use_v32 ? 32u : GGML_VK_GDN_CM_V));
|
||||
|
||||
vk_pipeline pipeline = use_v128
|
||||
? ctx->device->pipeline_gated_delta_net_cm_v128
|
||||
: use_v32 ? ctx->device->pipeline_gated_delta_net_cm_v32
|
||||
: (use_cm ? ctx->device->pipeline_gated_delta_net_cm
|
||||
: ggml_vk_op_get_pipeline(ctx, dst->src[0], dst->src[1], dst->src[2], dst, dst->op));
|
||||
GGML_ASSERT(pipeline != nullptr);
|
||||
|
||||
ggml_pipeline_request_descriptor_sets(ctx, pipeline, 1);
|
||||
@@ -11198,9 +11368,27 @@ void ggml_vk_gated_delta_net(ggml_backend_vk_context * ctx, vk_context& subctx,
|
||||
src_buf[i] = ggml_vk_tensor_subbuffer(ctx, dst->src[i]);
|
||||
}
|
||||
|
||||
vk_subbuffer mirror_buf{};
|
||||
if (use_v128) {
|
||||
const uint32_t LDP = GGML_VK_GDN_CM_LDP;
|
||||
const uint32_t blocks = (S_v / cm_V) * n_seqs * H;
|
||||
const size_t scratch_size = (size_t)blocks * LDP * cm_V * 2u * sizeof(uint16_t);
|
||||
if (ctx->prealloc_size_x < scratch_size) {
|
||||
ctx->prealloc_size_x = scratch_size;
|
||||
ggml_vk_preallocate_buffers(ctx, subctx);
|
||||
}
|
||||
if (ctx->prealloc_x_need_sync) {
|
||||
ggml_vk_sync_buffers(ctx, subctx);
|
||||
}
|
||||
mirror_buf = vk_subbuffer{ ctx->prealloc_x, 0, ctx->prealloc_x->size };
|
||||
}
|
||||
|
||||
const uint32_t sq1 = (uint32_t)(src_q->nb[1] / sizeof(float));
|
||||
const uint32_t sq2 = (uint32_t)(src_q->nb[2] / sizeof(float));
|
||||
const uint32_t sq3 = (uint32_t)(src_q->nb[3] / sizeof(float));
|
||||
const uint32_t sk1 = (uint32_t)(src_k->nb[1] / sizeof(float));
|
||||
const uint32_t sk2 = (uint32_t)(src_k->nb[2] / sizeof(float));
|
||||
const uint32_t sk3 = (uint32_t)(src_k->nb[3] / sizeof(float));
|
||||
const uint32_t sv1 = (uint32_t)(src_v->nb[1] / sizeof(float));
|
||||
const uint32_t sv2 = (uint32_t)(src_v->nb[2] / sizeof(float));
|
||||
const uint32_t sv3 = (uint32_t)(src_v->nb[3] / sizeof(float));
|
||||
@@ -11215,6 +11403,7 @@ void ggml_vk_gated_delta_net(ggml_backend_vk_context * ctx, vk_context& subctx,
|
||||
const vk_op_gated_delta_net_push_constants pc = {
|
||||
H, n_tokens, n_seqs, s_off,
|
||||
sq1, sq2, sq3,
|
||||
sk1, sk2, sk3,
|
||||
sv1, sv2, sv3,
|
||||
sb1, sb2, sb3,
|
||||
neq1, rq3,
|
||||
@@ -11222,9 +11411,16 @@ void ggml_vk_gated_delta_net(ggml_backend_vk_context * ctx, vk_context& subctx,
|
||||
K
|
||||
};
|
||||
|
||||
ggml_vk_dispatch_pipeline(ctx, subctx, pipeline,
|
||||
{src_buf[0], src_buf[1], src_buf[2], src_buf[3], src_buf[4], src_buf[5], dst_buf},
|
||||
pc, { H, n_seqs, S_v });
|
||||
if (use_v128) {
|
||||
ggml_vk_dispatch_pipeline(ctx, subctx, pipeline,
|
||||
{src_buf[0], src_buf[1], src_buf[2], src_buf[3], src_buf[4], src_buf[5], dst_buf, mirror_buf},
|
||||
pc, std::array<uint32_t, 3>{ S_v / cm_V, n_seqs, H });
|
||||
ctx->prealloc_x_need_sync = true;
|
||||
} else {
|
||||
ggml_vk_dispatch_pipeline(ctx, subctx, pipeline,
|
||||
{src_buf[0], src_buf[1], src_buf[2], src_buf[3], src_buf[4], src_buf[5], dst_buf},
|
||||
pc, use_cm ? std::array<uint32_t, 3>{ S_v / cm_V, n_seqs, H } : std::array<uint32_t, 3>{ H, n_seqs, S_v });
|
||||
}
|
||||
}
|
||||
|
||||
void ggml_vk_ssm_scan(ggml_backend_vk_context * ctx, vk_context& subctx, ggml_tensor * dst) {
|
||||
|
||||
@@ -15,6 +15,10 @@ if (GGML_VULKAN_COOPMAT2_DECODE_VECTOR_GLSLC_SUPPORT)
|
||||
add_compile_definitions(GGML_VULKAN_COOPMAT2_DECODE_VECTOR_GLSLC_SUPPORT)
|
||||
message(STATUS "Enabling coopmat2 decode_vector glslc support")
|
||||
endif()
|
||||
if (GGML_VULKAN_COOPMAT_MAINTENANCE1_GLSLC_SUPPORT)
|
||||
add_compile_definitions(GGML_VULKAN_COOPMAT_MAINTENANCE1_GLSLC_SUPPORT)
|
||||
message(STATUS "Enabling coopmat maintenance1 glslc support")
|
||||
endif()
|
||||
if (GGML_VULKAN_INTEGER_DOT_GLSLC_SUPPORT)
|
||||
add_compile_definitions(GGML_VULKAN_INTEGER_DOT_GLSLC_SUPPORT)
|
||||
message(STATUS "Enabling dot glslc support")
|
||||
|
||||
@@ -0,0 +1,8 @@
|
||||
#version 460
|
||||
|
||||
#extension GL_KHR_cooperative_matrix : require
|
||||
#extension GL_EXT_cooperative_matrix_maintenance1 : require
|
||||
|
||||
void main()
|
||||
{
|
||||
}
|
||||
@@ -15,10 +15,19 @@ layout(constant_id = 0) const uint S_V = 128;
|
||||
layout(constant_id = 1) const uint KDA = 0;
|
||||
layout(constant_id = 2) const uint SUBGROUP_SIZE = 32;
|
||||
layout(constant_id = 3) const uint LANES_PER_COLUMN = 32;
|
||||
layout(constant_id = 4) const bool PACKED_ROWS = false;
|
||||
|
||||
const uint COLS_PER_WG = SUBGROUP_SIZE / LANES_PER_COLUMN;
|
||||
const uint ROWS_PER_LANE = S_V / LANES_PER_COLUMN;
|
||||
|
||||
uint row_index(uint r, uint lane) {
|
||||
// Keep four adjacent rows together in each lane.
|
||||
if (PACKED_ROWS) {
|
||||
return (r / 4u * LANES_PER_COLUMN + lane) * 4u + r % 4u;
|
||||
}
|
||||
return r * LANES_PER_COLUMN + lane;
|
||||
}
|
||||
|
||||
layout(local_size_x_id = 2, local_size_y = 1, local_size_z = 1) in;
|
||||
|
||||
layout(push_constant) uniform Parameters {
|
||||
@@ -27,6 +36,7 @@ layout(push_constant) uniform Parameters {
|
||||
uint n_seqs;
|
||||
uint s_off;
|
||||
uint sq1, sq2, sq3;
|
||||
uint sk1, sk2, sk3;
|
||||
uint sv1, sv2, sv3;
|
||||
uint sb1, sb2, sb3;
|
||||
uint neq1, rq3;
|
||||
@@ -110,7 +120,7 @@ void main() {
|
||||
|
||||
FLOAT_TYPE s_shard[ROWS_PER_LANE];
|
||||
[[unroll]] for (uint r = 0; r < ROWS_PER_LANE; r++) {
|
||||
s_shard[r] = FLOAT_TYPE(data_state[state_in_base + col * S_V + r * LANES_PER_COLUMN + lane]);
|
||||
s_shard[r] = FLOAT_TYPE(data_state[state_in_base + col * S_V + row_index(r, lane)]);
|
||||
}
|
||||
|
||||
// snapshot slot mapping: slot 0 = most recent state, slot s = s tokens back.
|
||||
@@ -120,7 +130,7 @@ void main() {
|
||||
|
||||
for (uint t = 0; t < n_tokens; t++) {
|
||||
const uint q_off = iq3 * sq3 + t * sq2 + iq1 * sq1;
|
||||
const uint k_off = q_off;
|
||||
const uint k_off = iq3 * sk3 + t * sk2 + iq1 * sk1;
|
||||
const uint v_off = seq_id * sv3 + t * sv2 + head_id * sv1;
|
||||
const uint gb_off = seq_id * sb3 + t * sb2 + head_id * sb1;
|
||||
const FLOAT_TYPE beta_val = FLOAT_TYPE(data_beta[gb_off]);
|
||||
@@ -128,7 +138,7 @@ void main() {
|
||||
FLOAT_TYPE k_reg[ROWS_PER_LANE];
|
||||
FLOAT_TYPE q_reg[ROWS_PER_LANE];
|
||||
[[unroll]] for (uint r = 0; r < ROWS_PER_LANE; r++) {
|
||||
const uint i = r * LANES_PER_COLUMN + lane;
|
||||
const uint i = row_index(r, lane);
|
||||
k_reg[r] = FLOAT_TYPE(data_k[k_off + i]);
|
||||
q_reg[r] = FLOAT_TYPE(data_q[q_off + i]);
|
||||
}
|
||||
@@ -142,7 +152,7 @@ void main() {
|
||||
} else {
|
||||
const uint g_base = gb_off * S_V;
|
||||
[[unroll]] for (uint r = 0; r < ROWS_PER_LANE; r++) {
|
||||
const uint i = r * LANES_PER_COLUMN + lane;
|
||||
const uint i = row_index(r, lane);
|
||||
g_exp[r] = exp(FLOAT_TYPE(data_g[g_base + i]));
|
||||
}
|
||||
}
|
||||
@@ -175,7 +185,7 @@ void main() {
|
||||
if (target_slot >= 0 && target_slot < int(K)) {
|
||||
const uint slot_base = s_off + uint(target_slot) * state_size_per_snap + state_out_base;
|
||||
[[unroll]] for (uint r = 0; r < ROWS_PER_LANE; r++) {
|
||||
data_dst[slot_base + col * S_V + r * LANES_PER_COLUMN + lane] = s_shard[r];
|
||||
data_dst[slot_base + col * S_V + row_index(r, lane)] = s_shard[r];
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -183,7 +193,7 @@ void main() {
|
||||
|
||||
if (K == 1u) {
|
||||
[[unroll]] for (uint r = 0; r < ROWS_PER_LANE; r++) {
|
||||
data_dst[s_off + state_out_base + col * S_V + r * LANES_PER_COLUMN + lane] = s_shard[r];
|
||||
data_dst[s_off + state_out_base + col * S_V + row_index(r, lane)] = s_shard[r];
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,437 @@
|
||||
#version 450
|
||||
|
||||
#extension GL_EXT_control_flow_attributes : enable
|
||||
#extension GL_EXT_shader_explicit_arithmetic_types_float16 : require
|
||||
#extension GL_EXT_shader_explicit_arithmetic_types_int32 : require
|
||||
#extension GL_EXT_bfloat16 : require
|
||||
#extension GL_KHR_memory_scope_semantics : enable
|
||||
#extension GL_KHR_shader_subgroup_basic : enable
|
||||
#extension GL_KHR_shader_subgroup_arithmetic : enable
|
||||
#extension GL_KHR_shader_subgroup_shuffle : enable
|
||||
#extension GL_KHR_cooperative_matrix : enable
|
||||
#extension GL_EXT_cooperative_matrix_maintenance1 : enable
|
||||
|
||||
// Chunked gated-delta-net prefill (non-KDA, S_v == D == 128, K == 1), port of CUDA PR 29353.
|
||||
// State in register tiles, mirrored ([D x V]) for the projections.
|
||||
// bf16 hi/lo 3x products for ~fp32 accuracy.
|
||||
#define OPD bfloat16_t
|
||||
|
||||
#define C 16 // chunk size (tokens per step)
|
||||
#define D 128 // head dim / value dim
|
||||
#define TM 16 // tile rows
|
||||
#define TN 16 // tile cols (bf16 cm has only m16n16k16; n8 would pad)
|
||||
#ifndef V
|
||||
#define V 64 // value columns per workgroup block (V=128 = whole head)
|
||||
#endif
|
||||
#define NSUB 16 // subgroups per workgroup
|
||||
layout(constant_id = 0) const uint WGS = NSUB * 32; // NSUB * subgroup size
|
||||
#define KT 16 // K-tile width for deep (K=D) matmuls (bf16 cm: m16n16k16 only)
|
||||
#define PAD 8 // shared padding to break COL-load bank conflicts (keep LDP*2 a multiple of 16B)
|
||||
#define LDP (D + PAD)
|
||||
|
||||
// GMEM_MIRROR: state mirror in a gmem scratch buffer (too big for shared)
|
||||
|
||||
#define STATE_TILES ((D / TM) * (V / TN))
|
||||
#define PER_WARP (STATE_TILES / NSUB)
|
||||
#define VT (V / TN) // value tiles
|
||||
#define CT (C / TN) // C-column tiles (gram/pmat halves)
|
||||
const uint NQK = C * D / WGS;
|
||||
|
||||
#define TILE_R(ti) (((ti) / VT) * TM)
|
||||
#define TILE_C(ti) (((ti) % VT) * TN)
|
||||
|
||||
layout(local_size_x_id = 0, local_size_y = 1, local_size_z = 1) in;
|
||||
|
||||
layout(push_constant) uniform Parameters {
|
||||
uint H;
|
||||
uint n_tokens;
|
||||
uint n_seqs;
|
||||
uint s_off;
|
||||
uint sq1, sq2, sq3;
|
||||
uint sk1, sk2, sk3;
|
||||
uint sv1, sv2, sv3;
|
||||
uint sb1, sb2, sb3;
|
||||
uint neq1, rq3;
|
||||
float scale;
|
||||
uint K;
|
||||
} p;
|
||||
|
||||
layout(binding = 0) readonly buffer QBuf { float data_q[]; };
|
||||
layout(binding = 1) readonly buffer KBuf { float data_k[]; };
|
||||
layout(binding = 2) readonly buffer VBuf { float data_v[]; };
|
||||
layout(binding = 3) readonly buffer GBuf { float data_g[]; };
|
||||
layout(binding = 4) readonly buffer BetaBuf { float data_beta[]; };
|
||||
layout(binding = 5) readonly buffer StateBuf { float data_state[]; };
|
||||
layout(binding = 6) buffer DstBuf { float data_dst[]; };
|
||||
#ifdef GMEM_MIRROR
|
||||
layout(binding = 7) coherent buffer MirrorBuf { OPD data_mirror[]; };
|
||||
#endif
|
||||
|
||||
shared OPD sh_qh[C * LDP], sh_ql[C * LDP]; // q*scale hi/lo (D padded)
|
||||
shared OPD sh_kh[C * LDP], sh_kl[C * LDP]; // k hi/lo (D padded)
|
||||
#ifndef GMEM_MIRROR
|
||||
shared OPD sh_sh[LDP * V], sh_sl[LDP * V]; // state mirror [D x V] column-major (D padded)
|
||||
#endif
|
||||
shared OPD sh_dh[C * V], sh_dl[C * V]; // delta (pre-inv) / delta2 [C x V] row-major
|
||||
shared OPD sh_soh[C * V], sh_sol[C * V]; // solved [C x V] row-major
|
||||
shared OPD sh_ph[C * C], sh_pl[C * C]; // decayed P
|
||||
shared OPD sh_ih[C * C], sh_il[C * C]; // (I + L)^-1
|
||||
shared float sh_lower[C * C]; // strictly-lower decayed L
|
||||
|
||||
shared float sh_prefix[C], sh_beta[C], sh_ep[C], sh_carry[C];
|
||||
|
||||
#define SG gl_ScopeSubgroup
|
||||
#define MUA gl_MatrixUseA
|
||||
#define MUB gl_MatrixUseB
|
||||
#define MACC gl_MatrixUseAccumulator
|
||||
#define ROW gl_CooperativeMatrixLayoutRowMajor
|
||||
#define COL gl_CooperativeMatrixLayoutColumnMajor
|
||||
|
||||
|
||||
// 3x bf16 product accumulated into acc (hi/lo split for ~fp32 accuracy).
|
||||
#define PRODUCT(acc, ah, al, bh, bl) \
|
||||
acc = coopMatMulAdd(al, bh, acc); \
|
||||
acc = coopMatMulAdd(ah, bl, acc); \
|
||||
acc = coopMatMulAdd(ah, bh, acc);
|
||||
|
||||
void split(const float x, out OPD h, out OPD l) {
|
||||
h = OPD(x);
|
||||
l = OPD(x - float(h));
|
||||
}
|
||||
|
||||
#ifdef BF16ACC
|
||||
// split through a bf16 accumulator
|
||||
#define SPLIT_STORE(m, arr_h, off_h, arr_l, off_l, ld, layout) { \
|
||||
const coopmat<OPD, SG, TM, TN, MACC> h_ = coopmat<OPD, SG, TM, TN, MACC>(m); \
|
||||
const coopmat<OPD, SG, TM, TN, MACC> l_ = coopmat<OPD, SG, TM, TN, MACC>(m - coopmat<float, SG, TM, TN, MACC>(h_)); \
|
||||
coopMatStore(h_, arr_h, off_h, ld, layout); \
|
||||
coopMatStore(l_, arr_l, off_l, ld, layout); \
|
||||
}
|
||||
|
||||
#ifdef GMEM_MIRROR
|
||||
#define STORE_MIRROR(m, off) SPLIT_STORE(m, data_mirror, mir_h + off, data_mirror, mir_l + off, LDP, COL)
|
||||
#else
|
||||
#define STORE_MIRROR(m, off) SPLIT_STORE(m, sh_sh, off, sh_sl, off, LDP, COL)
|
||||
#endif
|
||||
#define STORE_DELTA(m, off) SPLIT_STORE(m, sh_dh, off, sh_dl, off, V, ROW)
|
||||
#define STORE_SOLVED(m, off) SPLIT_STORE(m, sh_soh, off, sh_sol, off, V, ROW)
|
||||
#else
|
||||
// split per element
|
||||
#ifdef GMEM_MIRROR
|
||||
float SplitColS(const in uint32_t r, const in uint32_t c, const in float elem, const in uint32_t off) {
|
||||
const uint idx = off + c * LDP + r;
|
||||
split(elem, data_mirror[idx], data_mirror[idx + LDP * V]);
|
||||
return elem;
|
||||
}
|
||||
#define STORE_MIRROR(m, off) coopMatPerElementEXT(m, m, SplitColS, mir_h + off)
|
||||
#else
|
||||
float SplitColS(const in uint32_t r, const in uint32_t c, const in float elem, const in uint32_t off) {
|
||||
const uint idx = off + c * LDP + r;
|
||||
split(elem, sh_sh[idx], sh_sl[idx]);
|
||||
return elem;
|
||||
}
|
||||
#define STORE_MIRROR(m, off) coopMatPerElementEXT(m, m, SplitColS, off)
|
||||
#endif
|
||||
|
||||
float SplitRowD(const in uint32_t r, const in uint32_t c, const in float elem, const in uint32_t off) {
|
||||
const uint idx = off + r * V + c;
|
||||
split(elem, sh_dh[idx], sh_dl[idx]);
|
||||
return elem;
|
||||
}
|
||||
|
||||
float SplitRowSO(const in uint32_t r, const in uint32_t c, const in float elem, const in uint32_t off) {
|
||||
const uint idx = off + r * V + c;
|
||||
split(elem, sh_soh[idx], sh_sol[idx]);
|
||||
return elem;
|
||||
}
|
||||
|
||||
#define STORE_DELTA(m, off) coopMatPerElementEXT(m, m, SplitRowD, off)
|
||||
#define STORE_SOLVED(m, off) coopMatPerElementEXT(m, m, SplitRowSO, off)
|
||||
#endif
|
||||
|
||||
#ifdef GMEM_MIRROR
|
||||
#define LOAD_S(h, l, off) \
|
||||
coopMatLoad(h, data_mirror, mir_h + off, LDP, COL); \
|
||||
coopMatLoad(l, data_mirror, mir_l + off, LDP, COL);
|
||||
// mirror is workgroup-private
|
||||
#define MIRROR_BARRIER() controlBarrier(gl_ScopeWorkgroup, gl_ScopeWorkgroup, \
|
||||
gl_StorageSemanticsBuffer | gl_StorageSemanticsShared, \
|
||||
gl_SemanticsAcquireRelease)
|
||||
#else
|
||||
#define LOAD_S(h, l, off) \
|
||||
coopMatLoad(h, sh_sh, off, LDP, COL); \
|
||||
coopMatLoad(l, sh_sl, off, LDP, COL);
|
||||
#define MIRROR_BARRIER() barrier()
|
||||
#endif
|
||||
|
||||
float StoreOut(const in uint32_t r, const in uint32_t c, const in float elem,
|
||||
const in uint32_t out_base, const in uint32_t valid) {
|
||||
if (r < valid) {
|
||||
data_dst[out_base + r * (D * p.H) + c] = elem;
|
||||
}
|
||||
return elem;
|
||||
}
|
||||
|
||||
float StoreState(const in uint32_t r, const in uint32_t c, const in float elem,
|
||||
const in uint32_t base) {
|
||||
data_dst[base + c * D + r] = elem;
|
||||
return elem;
|
||||
}
|
||||
|
||||
float LoadV(const in uint32_t r, const in uint32_t c, const in float elem,
|
||||
const in uint32_t v_base, const in uint32_t sv2, const in uint32_t valid) {
|
||||
return r < valid ? data_v[v_base + r * sv2 + c] : 0.0;
|
||||
}
|
||||
|
||||
// sh_beta[r] == 0 for r >= valid
|
||||
float ComputeDeltaV(const in uint32_t r, const in uint32_t c, const in float ks,
|
||||
const in float vv) {
|
||||
return sh_beta[r] * (vv - sh_ep[r] * ks);
|
||||
}
|
||||
|
||||
float ApplyExpPrefix(const in uint32_t r, const in uint32_t c, const in float qs) {
|
||||
return sh_ep[r] * qs;
|
||||
}
|
||||
|
||||
float StoreLower(const in uint32_t r, const in uint32_t c, const in float elem,
|
||||
const in uint32_t c_o) {
|
||||
const uint cc = c_o + c;
|
||||
if (cc < r) {
|
||||
sh_lower[r * C + cc] = elem * exp(sh_prefix[r] - sh_prefix[cc]) * sh_beta[r];
|
||||
}
|
||||
return elem;
|
||||
}
|
||||
|
||||
float StorePmat(const in uint32_t r, const in uint32_t c, const in float elem,
|
||||
const in uint32_t c_o) {
|
||||
const uint cc = c_o + c;
|
||||
float v = 0.0;
|
||||
if (cc <= r) {
|
||||
v = elem * exp(sh_prefix[r] - sh_prefix[cc]);
|
||||
}
|
||||
split(v, sh_ph[r * C + cc], sh_pl[r * C + cc]);
|
||||
return elem;
|
||||
}
|
||||
|
||||
void prefetch(const uint t0, const uint iq1, const uint iq3, const uint seq_id, const uint head_id,
|
||||
out float q[NQK], out float k[NQK], out float g, out float b) {
|
||||
const uint n = t0 < p.n_tokens ? min(C, p.n_tokens - t0) : 0u;
|
||||
[[unroll]] for (uint j = 0; j < NQK; j++) {
|
||||
const uint i = gl_LocalInvocationID.x + j * WGS;
|
||||
const uint t = i / D;
|
||||
q[j] = 0.0;
|
||||
k[j] = 0.0;
|
||||
if (t < n) {
|
||||
const uint q_off = iq3 * p.sq3 + (t0 + t) * p.sq2 + iq1 * p.sq1 + i % D;
|
||||
const uint k_off = iq3 * p.sk3 + (t0 + t) * p.sk2 + iq1 * p.sk1 + i % D;
|
||||
q[j] = data_q[q_off] * p.scale;
|
||||
k[j] = data_k[k_off];
|
||||
}
|
||||
}
|
||||
g = 0.0;
|
||||
b = 0.0;
|
||||
const uint t = gl_SubgroupInvocationID;
|
||||
if (gl_SubgroupID == 0 && t < n) {
|
||||
const uint gb = seq_id * p.sb3 + (t0 + t) * p.sb2 + head_id * p.sb1;
|
||||
g = data_g[gb];
|
||||
b = data_beta[gb];
|
||||
}
|
||||
}
|
||||
|
||||
// sg0: gram K@K^T -> L, sg1: pmat Q@K^T -> P
|
||||
void gram_pmat(const uint sg) {
|
||||
const bool gram = sg < CT;
|
||||
const uint c_o = (sg % CT) * TN;
|
||||
coopmat<float, SG, C, TN, MACC> x = coopmat<float, SG, C, TN, MACC>(0);
|
||||
[[unroll]] for (uint kk = 0; kk < D; kk += KT) {
|
||||
coopmat<OPD, SG, C, KT, MUA> ah, al;
|
||||
if (gram) {
|
||||
coopMatLoad(ah, sh_kh, kk, LDP, ROW);
|
||||
coopMatLoad(al, sh_kl, kk, LDP, ROW);
|
||||
} else {
|
||||
coopMatLoad(ah, sh_qh, kk, LDP, ROW);
|
||||
coopMatLoad(al, sh_ql, kk, LDP, ROW);
|
||||
}
|
||||
coopmat<OPD, SG, KT, TN, MUB> bh, bl;
|
||||
coopMatLoad(bh, sh_kh, c_o * LDP + kk, LDP, COL);
|
||||
coopMatLoad(bl, sh_kl, c_o * LDP + kk, LDP, COL);
|
||||
PRODUCT(x, ah, al, bh, bl);
|
||||
}
|
||||
if (gram) {
|
||||
coopMatPerElementEXT(x, x, StoreLower, c_o);
|
||||
} else {
|
||||
coopMatPerElementEXT(x, x, StorePmat, c_o);
|
||||
}
|
||||
}
|
||||
|
||||
// (I + L)^-1, forward substitution: subgroup per row, lane per column
|
||||
void invert(const uint sg) {
|
||||
const uint col = gl_SubgroupInvocationID;
|
||||
[[unroll]] for (uint row = sg; row < C; row += NSUB) {
|
||||
float x = col < C ? (col == row ? 1.0 : (col < row ? -sh_lower[row * C + col] : 0.0)) : 0.0;
|
||||
for (int pivot = int(row) - 1; pivot > 0; pivot--) {
|
||||
const float xp = subgroupShuffle(x, uint(pivot));
|
||||
if (col < uint(pivot)) {
|
||||
x -= xp * sh_lower[uint(pivot) * C + col];
|
||||
}
|
||||
}
|
||||
if (col < C) {
|
||||
split(x, sh_ih[row * C + col], sh_il[row * C + col]);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
float carry_decay(const uint valid) {
|
||||
const float plast = sh_prefix[valid - 1];
|
||||
if (gl_LocalInvocationID.x < C) {
|
||||
sh_carry[gl_LocalInvocationID.x] = exp(plast - sh_prefix[gl_LocalInvocationID.x]);
|
||||
}
|
||||
return exp(plast);
|
||||
}
|
||||
|
||||
void main() {
|
||||
const uint head_id = gl_WorkGroupID.z;
|
||||
const uint seq_id = gl_WorkGroupID.y;
|
||||
const uint sg = gl_SubgroupID;
|
||||
const uint tid = gl_LocalInvocationID.x;
|
||||
const uint v0 = gl_WorkGroupID.x * V;
|
||||
|
||||
const uint iq1 = head_id % p.neq1;
|
||||
const uint iq3 = seq_id / p.rq3;
|
||||
|
||||
const uint state_base = (seq_id * p.H + head_id) * (D * D);
|
||||
|
||||
#ifdef GMEM_MIRROR
|
||||
const uint mir_stride = uint(LDP) * V;
|
||||
const uint mir_block = (head_id * p.n_seqs + seq_id) * gl_NumWorkGroups.x + gl_WorkGroupID.x;
|
||||
const uint mir_h = mir_block * mir_stride * 2u;
|
||||
const uint mir_l = mir_h + mir_stride;
|
||||
#endif
|
||||
|
||||
coopmat<float, SG, TM, TN, MACC> state[PER_WARP];
|
||||
[[unroll]] for (uint j = 0; j < PER_WARP; j++) {
|
||||
const uint ti = sg + j * NSUB;
|
||||
coopMatLoad(state[j], data_state, state_base + (v0 + TILE_C(ti)) * D + TILE_R(ti), D, COL);
|
||||
}
|
||||
|
||||
float qreg[NQK], kreg[NQK], greg, breg;
|
||||
prefetch(0u, iq1, iq3, seq_id, head_id, qreg, kreg, greg, breg);
|
||||
|
||||
for (uint t0 = 0; t0 < p.n_tokens; t0 += C) {
|
||||
const uint valid = min(C, p.n_tokens - t0);
|
||||
|
||||
barrier();
|
||||
if (sg == 0) {
|
||||
const uint t = gl_SubgroupInvocationID;
|
||||
const float acc = subgroupInclusiveAdd(greg);
|
||||
if (t < C) {
|
||||
sh_prefix[t] = acc;
|
||||
sh_ep[t] = exp(acc);
|
||||
sh_beta[t] = breg;
|
||||
}
|
||||
}
|
||||
[[unroll]] for (uint j = 0; j < NQK; j++) {
|
||||
const uint i = tid + j * WGS;
|
||||
const uint pi = (i / D) * LDP + (i % D);
|
||||
split(qreg[j], sh_qh[pi], sh_ql[pi]);
|
||||
split(kreg[j], sh_kh[pi], sh_kl[pi]);
|
||||
}
|
||||
|
||||
[[unroll]] for (uint j = 0; j < PER_WARP; j++) {
|
||||
const uint ti = sg + j * NSUB;
|
||||
STORE_MIRROR(state[j], TILE_C(ti) * LDP + TILE_R(ti));
|
||||
}
|
||||
MIRROR_BARRIER();
|
||||
|
||||
prefetch(t0 + C, iq1, iq3, seq_id, head_id, qreg, kreg, greg, breg);
|
||||
const float cdecay = carry_decay(valid);
|
||||
|
||||
// qs -> base_r and ks -> delta on sg2..2+VT-1
|
||||
coopmat<float, SG, C, TN, MACC> base_r = coopmat<float, SG, C, TN, MACC>(0);
|
||||
if (sg < 2 * CT) {
|
||||
gram_pmat(sg);
|
||||
} else if (sg - 2 * CT < VT) {
|
||||
const uint c_o = (sg - 2 * CT) * TN;
|
||||
const uint v_base = seq_id * p.sv3 + head_id * p.sv1 + v0 + c_o + t0 * p.sv2;
|
||||
coopmat<float, SG, C, TN, MACC> vmat = coopmat<float, SG, C, TN, MACC>(0);
|
||||
coopMatPerElementEXT(vmat, vmat, LoadV, v_base, p.sv2, valid);
|
||||
coopmat<float, SG, C, TN, MACC> qs = coopmat<float, SG, C, TN, MACC>(0);
|
||||
coopmat<float, SG, C, TN, MACC> ks = coopmat<float, SG, C, TN, MACC>(0);
|
||||
[[unroll]] for (uint kk = 0; kk < D; kk += KT) {
|
||||
coopmat<OPD, SG, C, KT, MUA> qh, ql, kh, kl;
|
||||
coopMatLoad(qh, sh_qh, kk, LDP, ROW);
|
||||
coopMatLoad(ql, sh_ql, kk, LDP, ROW);
|
||||
coopMatLoad(kh, sh_kh, kk, LDP, ROW);
|
||||
coopMatLoad(kl, sh_kl, kk, LDP, ROW);
|
||||
coopmat<OPD, SG, KT, TN, MUB> sbh, sbl;
|
||||
LOAD_S(sbh, sbl, c_o * LDP + kk);
|
||||
PRODUCT(qs, qh, ql, sbh, sbl);
|
||||
PRODUCT(ks, kh, kl, sbh, sbl);
|
||||
}
|
||||
coopMatPerElementEXT(base_r, qs, ApplyExpPrefix);
|
||||
coopmat<float, SG, C, TN, MACC> delta;
|
||||
coopMatPerElementEXT(delta, ks, ComputeDeltaV, vmat);
|
||||
STORE_DELTA(delta, c_o);
|
||||
}
|
||||
barrier();
|
||||
|
||||
invert(sg);
|
||||
barrier();
|
||||
|
||||
// solved = (I+L)^-1 @ delta (sg0..VT-1)
|
||||
if (sg < VT) {
|
||||
const uint c_o = sg * TN;
|
||||
coopmat<float, SG, C, TN, MACC> solved = coopmat<float, SG, C, TN, MACC>(0);
|
||||
coopmat<OPD, SG, C, C, MUA> ih, il;
|
||||
coopMatLoad(ih, sh_ih, 0, C, ROW);
|
||||
coopMatLoad(il, sh_il, 0, C, ROW);
|
||||
coopmat<OPD, SG, C, TN, MUB> dh, dl;
|
||||
coopMatLoad(dh, sh_dh, c_o, V, ROW);
|
||||
coopMatLoad(dl, sh_dl, c_o, V, ROW);
|
||||
PRODUCT(solved, ih, il, dh, dl);
|
||||
STORE_SOLVED(solved, c_o);
|
||||
}
|
||||
barrier();
|
||||
|
||||
// out = base + P@solved (sg2..2+VT-1)
|
||||
if (sg >= 2 * CT && sg - 2 * CT < VT) {
|
||||
const uint c_o = (sg - 2 * CT) * TN;
|
||||
coopmat<float, SG, C, TN, MACC> O = base_r;
|
||||
coopmat<OPD, SG, C, C, MUA> ph, pl;
|
||||
coopMatLoad(ph, sh_ph, 0, C, ROW);
|
||||
coopMatLoad(pl, sh_pl, 0, C, ROW);
|
||||
coopmat<OPD, SG, C, TN, MUB> svh, svl;
|
||||
coopMatLoad(svh, sh_soh, c_o, V, ROW);
|
||||
coopMatLoad(svl, sh_sol, c_o, V, ROW);
|
||||
PRODUCT(O, ph, pl, svh, svl);
|
||||
const uint out_base = (seq_id * p.n_tokens * p.H + head_id) * D + v0 + c_o + t0 * (D * p.H);
|
||||
coopMatPerElementEXT(O, O, StoreOut, out_base, valid);
|
||||
}
|
||||
// delta2 = solved * carry -> sh_dh/dl (all threads)
|
||||
for (uint i = tid; i < C * V; i += WGS) {
|
||||
const float sv = float(sh_soh[i]) + float(sh_sol[i]);
|
||||
split(sv * sh_carry[i / V], sh_dh[i], sh_dl[i]);
|
||||
}
|
||||
barrier();
|
||||
|
||||
// state = cdecay*state + K^T @ delta2
|
||||
[[unroll]] for (uint j = 0; j < PER_WARP; j++) {
|
||||
const uint ti = sg + j * NSUB;
|
||||
coopmat<OPD, SG, TM, C, MUA> kth, ktl;
|
||||
coopMatLoad(kth, sh_kh, TILE_R(ti), LDP, COL);
|
||||
coopMatLoad(ktl, sh_kl, TILE_R(ti), LDP, COL);
|
||||
coopmat<OPD, SG, C, TN, MUB> dh, dl;
|
||||
coopMatLoad(dh, sh_dh, TILE_C(ti), V, ROW);
|
||||
coopMatLoad(dl, sh_dl, TILE_C(ti), V, ROW);
|
||||
state[j] = state[j] * cdecay;
|
||||
PRODUCT(state[j], kth, ktl, dh, dl);
|
||||
}
|
||||
// no end-of-chunk barrier: covered by the chunk-top barrier
|
||||
}
|
||||
|
||||
[[unroll]] for (uint j = 0; j < PER_WARP; j++) {
|
||||
const uint ti = sg + j * NSUB;
|
||||
coopMatPerElementEXT(state[j], state[j], StoreState,
|
||||
p.s_off + state_base + (v0 + TILE_C(ti)) * D + TILE_R(ti));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,320 @@
|
||||
#version 450
|
||||
|
||||
#extension GL_EXT_control_flow_attributes : require
|
||||
#extension GL_EXT_shader_explicit_arithmetic_types_int32 : require
|
||||
#extension GL_EXT_bfloat16 : require
|
||||
#extension GL_KHR_memory_scope_semantics : require
|
||||
#extension GL_KHR_shader_subgroup_basic : require
|
||||
#extension GL_KHR_shader_subgroup_arithmetic : require
|
||||
#extension GL_KHR_shader_subgroup_shuffle : require
|
||||
#extension GL_KHR_shader_subgroup_shuffle_relative : require
|
||||
#extension GL_KHR_cooperative_matrix : require
|
||||
#extension GL_NV_cooperative_matrix2 : require
|
||||
|
||||
// Chunked gated-delta-net prefill (non-KDA, S_v == D == 128, K == 1).
|
||||
// BF16 hi/lo products use FP32 accumulators.
|
||||
#define OPD bfloat16_t
|
||||
|
||||
#define C 16 // chunk size (tokens per step)
|
||||
#define D 128 // head dim / value dim
|
||||
#define V 32 // value columns per workgroup block
|
||||
#define WGS 128
|
||||
#define NSUB (WGS / 32)
|
||||
#define KT 32 // K-tile width for deep (K=D) matmuls
|
||||
#define PAD 8
|
||||
#define LDP (D + PAD)
|
||||
#define LDV (V + PAD)
|
||||
#define KQ_ROWS (2 * C)
|
||||
#define NQK (C * D / WGS)
|
||||
|
||||
layout(local_size_x = WGS, local_size_y = 1, local_size_z = 1) in;
|
||||
|
||||
layout(push_constant) uniform Parameters {
|
||||
uint H;
|
||||
uint n_tokens;
|
||||
uint n_seqs;
|
||||
uint s_off;
|
||||
uint sq1, sq2, sq3;
|
||||
uint sk1, sk2, sk3;
|
||||
uint sv1, sv2, sv3;
|
||||
uint sb1, sb2, sb3;
|
||||
uint neq1, rq3;
|
||||
float scale;
|
||||
uint K;
|
||||
} p;
|
||||
|
||||
layout(binding = 0) readonly buffer QBuf { float data_q[]; };
|
||||
layout(binding = 1) readonly buffer KBuf { float data_k[]; };
|
||||
layout(binding = 2) readonly buffer VBuf { float data_v[]; };
|
||||
layout(binding = 3) readonly buffer GBuf { float data_g[]; };
|
||||
layout(binding = 4) readonly buffer BetaBuf { float data_beta[]; };
|
||||
layout(binding = 5) readonly buffer StateBuf { float data_state[]; };
|
||||
layout(binding = 6) buffer DstBuf { float data_dst[]; };
|
||||
|
||||
// K and scaled Q occupy the two row blocks of one 32-row operand.
|
||||
shared OPD sh_kqh[2 * C * LDP], sh_kql[2 * C * LDP];
|
||||
shared OPD sh_sh[V * LDP], sh_sl[V * LDP]; // state mirror [V x D] row-major (D padded)
|
||||
shared OPD sh_ph[C * C], sh_pl[C * C];
|
||||
shared OPD sh_ih[C * C], sh_il[C * C];
|
||||
shared OPD sh_dh[C * LDV], sh_dl[C * LDV]; // delta / solved [C x V] row-major (V padded)
|
||||
shared float sh_lower[C * (C - 1) / 2]; // packed strictly-lower Gram entries
|
||||
shared float sh_prefix[C], sh_ep[C], sh_beta[C], sh_carry[C];
|
||||
|
||||
#define WG gl_ScopeWorkgroup
|
||||
#define MUA gl_MatrixUseA
|
||||
#define MUB gl_MatrixUseB
|
||||
#define MACC gl_MatrixUseAccumulator
|
||||
#define ROW gl_CooperativeMatrixLayoutRowMajor
|
||||
#define COL gl_CooperativeMatrixLayoutColumnMajor
|
||||
|
||||
#define PRODUCT(acc, ah, al, bh, bl) \
|
||||
acc = coopMatMulAdd(ah, bh, acc); \
|
||||
acc = coopMatMulAdd(al, bh, acc); \
|
||||
acc = coopMatMulAdd(ah, bl, acc);
|
||||
|
||||
void split(const float x, out OPD h, out OPD l) {
|
||||
h = OPD(x);
|
||||
l = OPD(x - float(h));
|
||||
}
|
||||
|
||||
float StoreGramPmat(const in uint32_t r, const in uint32_t c, const in float elem) {
|
||||
if (r < C) {
|
||||
if (c < r) {
|
||||
sh_lower[r * (r - 1) / 2 + c] = elem * exp(sh_prefix[r] - sh_prefix[c]) * sh_beta[r];
|
||||
}
|
||||
} else {
|
||||
const uint row = r - C;
|
||||
const float y = c <= row ? elem * exp(sh_prefix[row] - sh_prefix[c]) : 0.0;
|
||||
split(y, sh_ph[row * C + c], sh_pl[row * C + c]);
|
||||
}
|
||||
return elem;
|
||||
}
|
||||
|
||||
float SplitRowS(const in uint32_t r, const in uint32_t c, const in float elem) {
|
||||
const uint i = r * LDP + c;
|
||||
split(elem, sh_sh[i], sh_sl[i]);
|
||||
return elem;
|
||||
}
|
||||
|
||||
float ApplyExpPrefix(const in uint32_t r, const in uint32_t c, const in float qs) {
|
||||
return sh_ep[c] * qs;
|
||||
}
|
||||
|
||||
float LoadV(const in uint32_t r, const in uint32_t c, const in float elem,
|
||||
const in uint32_t v_base, const in uint32_t sv2, const in uint32_t valid) {
|
||||
return c < valid ? data_v[v_base + c * sv2 + r] : 0.0;
|
||||
}
|
||||
|
||||
float StoreDeltaV(const in uint32_t r, const in uint32_t c, const in float ks, const in float vv) {
|
||||
const float x = sh_beta[c] * (vv - sh_ep[c] * ks);
|
||||
const uint i = c * LDV + r;
|
||||
split(x, sh_dh[i], sh_dl[i]);
|
||||
return ks;
|
||||
}
|
||||
|
||||
float StoreSolved(const in uint32_t r, const in uint32_t c, const in float elem) {
|
||||
const uint i = c * LDV + r;
|
||||
split(elem, sh_dh[i], sh_dl[i]);
|
||||
// The state mirror is no longer needed after KS/QS.
|
||||
const uint j = r * LDP + c;
|
||||
split(sh_carry[c] * elem, sh_sh[j], sh_sl[j]);
|
||||
return elem;
|
||||
}
|
||||
|
||||
float StoreOut(const in uint32_t r, const in uint32_t c, const in float elem,
|
||||
const in uint32_t out_base, const in uint32_t valid) {
|
||||
if (c < valid) {
|
||||
data_dst[out_base + c * (p.H * D) + r] = elem;
|
||||
}
|
||||
return elem;
|
||||
}
|
||||
|
||||
void prefetch(const uint t0, const uint q_base, const uint k_base, const uint gb_base,
|
||||
out float q[NQK], out float k[NQK], out float g, out float b) {
|
||||
const uint valid = t0 < p.n_tokens ? min(C, p.n_tokens - t0) : 0u;
|
||||
const uint r = gl_SubgroupInvocationID;
|
||||
g = 0.0;
|
||||
b = 0.0;
|
||||
if (gl_SubgroupID == 0 && r < valid) {
|
||||
const uint off = gb_base + (t0 + r) * p.sb2;
|
||||
g = data_g[off];
|
||||
b = data_beta[off];
|
||||
}
|
||||
[[unroll]] for (uint j = 0; j < NQK; ++j) {
|
||||
const uint i = gl_LocalInvocationID.x + j * WGS;
|
||||
const uint token = i / D;
|
||||
q[j] = 0.0;
|
||||
k[j] = 0.0;
|
||||
if (token < valid) {
|
||||
const uint q_off = q_base + (t0 + token) * p.sq2 + i % D;
|
||||
const uint k_off = k_base + (t0 + token) * p.sk2 + i % D;
|
||||
q[j] = data_q[q_off];
|
||||
k[j] = data_k[k_off];
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void stage_inputs(const in float q[NQK], const in float k[NQK]) {
|
||||
[[unroll]] for (uint j = 0; j < NQK; ++j) {
|
||||
const uint i = gl_LocalInvocationID.x + j * WGS;
|
||||
const uint token = i / D;
|
||||
const uint key = i % D;
|
||||
const uint ki = token * LDP + key;
|
||||
const uint qi = (C + token) * LDP + key;
|
||||
split(k[j], sh_kqh[ki], sh_kql[ki]);
|
||||
split(p.scale * q[j], sh_kqh[qi], sh_kql[qi]);
|
||||
}
|
||||
}
|
||||
|
||||
// [K; Q] * K^T computes Gram and P in one 32-row matrix.
|
||||
void gram_pmat() {
|
||||
coopmat<float, WG, KQ_ROWS, C, MACC> x = coopmat<float, WG, KQ_ROWS, C, MACC>(0.0);
|
||||
[[unroll]] for (uint kk = 0; kk < D; kk += KT) {
|
||||
coopmat<OPD, WG, KQ_ROWS, KT, MUA> ah, al;
|
||||
coopmat<OPD, WG, KT, C, MUB> bh, bl;
|
||||
coopMatLoad(ah, sh_kqh, kk, LDP, ROW);
|
||||
coopMatLoad(al, sh_kql, kk, LDP, ROW);
|
||||
coopMatLoad(bh, sh_kqh, kk, LDP, COL);
|
||||
coopMatLoad(bl, sh_kql, kk, LDP, COL);
|
||||
PRODUCT(x, ah, al, bh, bl);
|
||||
}
|
||||
coopMatPerElementNV(x, x, StoreGramPmat);
|
||||
}
|
||||
|
||||
void invert() {
|
||||
const uint col = gl_SubgroupInvocationID % C;
|
||||
// Each half-subgroup covers all columns of its assigned rows.
|
||||
const uint section = gl_SubgroupInvocationID / C;
|
||||
const uint row_base = (gl_SubgroupID * 2u + section) * (C / (2 * NSUB));
|
||||
// Keep the diagonal zero so pivots at or above the row do nothing.
|
||||
float x[C / (2 * NSUB)];
|
||||
[[unroll]] for (uint j = 0; j < C / (2 * NSUB); ++j) {
|
||||
const uint row = row_base + j;
|
||||
x[j] = col < row ? -sh_lower[row * (row - 1) / 2 + col] : 0.0;
|
||||
}
|
||||
[[dont_unroll]] for (int pivot = C - 2; pivot > 0; --pivot) {
|
||||
const float lower = col < uint(pivot) ? sh_lower[uint(pivot) * uint(pivot - 1) / 2 + col] : 0.0;
|
||||
[[unroll]] for (uint j = 0; j < C / (2 * NSUB); ++j) {
|
||||
const float xp = subgroupShuffle(x[j], uint(pivot) + section * C);
|
||||
x[j] -= xp * lower;
|
||||
}
|
||||
}
|
||||
[[unroll]] for (uint j = 0; j < C / (2 * NSUB); ++j) {
|
||||
const uint row = row_base + j;
|
||||
split(col == row ? 1.0 : x[j], sh_ih[row * C + col], sh_il[row * C + col]);
|
||||
}
|
||||
}
|
||||
|
||||
void main() {
|
||||
const uint head_id = gl_WorkGroupID.z;
|
||||
const uint seq_id = gl_WorkGroupID.y;
|
||||
const uint v0 = gl_WorkGroupID.x * V;
|
||||
const uint iq1 = head_id % p.neq1;
|
||||
const uint iq3 = seq_id / p.rq3;
|
||||
const uint q_base = iq3 * p.sq3 + iq1 * p.sq1;
|
||||
const uint k_base = iq3 * p.sk3 + iq1 * p.sk1;
|
||||
const uint v_base = seq_id * p.sv3 + head_id * p.sv1 + v0;
|
||||
const uint gb_base = seq_id * p.sb3 + head_id * p.sb1;
|
||||
const uint state_base = (seq_id * p.H + head_id) * (D * D) + v0 * D;
|
||||
const uint out_base = (seq_id * p.n_tokens * p.H + head_id) * D + v0;
|
||||
|
||||
// Transposing the state-side products gives them V=32 rows with C=16 columns.
|
||||
coopmat<float, WG, V, D, MACC> state;
|
||||
coopMatLoad(state, data_state, state_base, D, ROW);
|
||||
|
||||
float qreg[NQK], kreg[NQK], greg, breg;
|
||||
prefetch(0u, q_base, k_base, gb_base, qreg, kreg, greg, breg);
|
||||
|
||||
[[dont_unroll]] for (uint t0 = 0; t0 < p.n_tokens; t0 += C) {
|
||||
const uint valid = min(C, p.n_tokens - t0);
|
||||
barrier();
|
||||
if (gl_SubgroupID == 0) {
|
||||
const uint r = gl_SubgroupInvocationID;
|
||||
// shuffle scan instead of subgroupInclusiveAdd: works around an NVIDIA r620 shader compiler crash
|
||||
float sum = greg;
|
||||
[[unroll]] for (uint d = 1; d < C; d <<= 1) {
|
||||
const float t = subgroupShuffleUp(sum, d);
|
||||
if (r >= d) {
|
||||
sum += t;
|
||||
}
|
||||
}
|
||||
const float last = subgroupShuffle(sum, valid - 1);
|
||||
if (r < C) {
|
||||
sh_prefix[r] = sum;
|
||||
sh_ep[r] = exp(sum);
|
||||
sh_beta[r] = breg;
|
||||
sh_carry[r] = exp(last - sum);
|
||||
}
|
||||
}
|
||||
stage_inputs(qreg, kreg);
|
||||
coopmat<float, WG, V, C, MACC> vmat;
|
||||
if (valid == C && (v_base % 4u) == 0 && (p.sv2 % 4u) == 0) {
|
||||
coopMatLoad(vmat, data_v, v_base + t0 * p.sv2, p.sv2, COL);
|
||||
} else {
|
||||
vmat = coopmat<float, WG, V, C, MACC>(0.0);
|
||||
coopMatPerElementNV(vmat, vmat, LoadV, v_base + t0 * p.sv2, p.sv2, valid);
|
||||
}
|
||||
barrier();
|
||||
// Current inputs are in shared memory; keep the next chunk in registers.
|
||||
prefetch(t0 + C, q_base, k_base, gb_base, qreg, kreg, greg, breg);
|
||||
|
||||
gram_pmat();
|
||||
barrier();
|
||||
invert();
|
||||
coopMatPerElementNV(state, state, SplitRowS);
|
||||
barrier();
|
||||
|
||||
coopmat<float, WG, V, C, MACC> ks = coopmat<float, WG, V, C, MACC>(0.0), base_r = coopmat<float, WG, V, C, MACC>(0.0);
|
||||
[[unroll]] for (uint kk = 0; kk < D; kk += KT) {
|
||||
coopmat<OPD, WG, V, KT, MUA> ah, al;
|
||||
coopmat<OPD, WG, KT, C, MUB> kh, kl, qh, ql;
|
||||
coopMatLoad(ah, sh_sh, kk, LDP, ROW);
|
||||
coopMatLoad(al, sh_sl, kk, LDP, ROW);
|
||||
coopMatLoad(kh, sh_kqh, kk, LDP, COL);
|
||||
coopMatLoad(kl, sh_kql, kk, LDP, COL);
|
||||
coopMatLoad(qh, sh_kqh, C * LDP + kk, LDP, COL);
|
||||
coopMatLoad(ql, sh_kql, C * LDP + kk, LDP, COL);
|
||||
PRODUCT(ks, ah, al, kh, kl);
|
||||
PRODUCT(base_r, ah, al, qh, ql);
|
||||
}
|
||||
coopMatPerElementNV(base_r, base_r, ApplyExpPrefix);
|
||||
coopMatPerElementNV(ks, ks, StoreDeltaV, vmat);
|
||||
barrier();
|
||||
|
||||
coopmat<float, WG, V, C, MACC> solved = coopmat<float, WG, V, C, MACC>(0.0);
|
||||
{
|
||||
coopmat<OPD, WG, V, C, MUA> rh, rl;
|
||||
coopmat<OPD, WG, C, C, MUB> ih, il;
|
||||
coopMatLoad(rh, sh_dh, 0, LDV, COL);
|
||||
coopMatLoad(rl, sh_dl, 0, LDV, COL);
|
||||
coopMatLoad(ih, sh_ih, 0, C, COL);
|
||||
coopMatLoad(il, sh_il, 0, C, COL);
|
||||
PRODUCT(solved, rh, rl, ih, il);
|
||||
}
|
||||
barrier();
|
||||
coopMatPerElementNV(solved, solved, StoreSolved);
|
||||
barrier();
|
||||
|
||||
{
|
||||
coopmat<OPD, WG, V, C, MUA> xh, xl;
|
||||
coopmat<OPD, WG, C, C, MUB> ph, pl;
|
||||
coopMatLoad(xh, sh_dh, 0, LDV, COL);
|
||||
coopMatLoad(xl, sh_dl, 0, LDV, COL);
|
||||
coopMatLoad(ph, sh_ph, 0, C, COL);
|
||||
coopMatLoad(pl, sh_pl, 0, C, COL);
|
||||
PRODUCT(base_r, xh, xl, ph, pl);
|
||||
}
|
||||
coopMatPerElementNV(base_r, base_r, StoreOut, out_base + t0 * (p.H * D), valid);
|
||||
|
||||
coopmat<OPD, WG, V, C, MUA> xh, xl;
|
||||
coopmat<OPD, WG, C, D, MUB> kh, kl;
|
||||
coopMatLoad(xh, sh_sh, 0, LDP, ROW);
|
||||
coopMatLoad(xl, sh_sl, 0, LDP, ROW);
|
||||
coopMatLoad(kh, sh_kqh, 0, LDP, ROW);
|
||||
coopMatLoad(kl, sh_kql, 0, LDP, ROW);
|
||||
state *= sh_ep[valid - 1];
|
||||
PRODUCT(state, xh, xl, kh, kl);
|
||||
}
|
||||
|
||||
coopMatStore(state, data_dst, p.s_off + state_base, D, ROW);
|
||||
}
|
||||
@@ -348,7 +348,10 @@ compile_count_guard acquire_compile_slot() {
|
||||
}
|
||||
|
||||
void string_to_spv_func(std::string name, std::string in_path, std::string out_path, std::map<std::string, std::string> defines, bool coopmat, bool dep_file, compile_count_guard slot) {
|
||||
std::string target_env = (name.find("_cm2") != std::string::npos) ? "--target-env=vulkan1.3" : "--target-env=vulkan1.2";
|
||||
// coopmat2 and the maintenance1 GDN coopmat1 variants need SPIR-V 1.6 (vulkan1.3)
|
||||
const bool needs_spv16 = name.find("_cm2") != std::string::npos ||
|
||||
name.find("gated_delta_net_f32_cm1") != std::string::npos;
|
||||
std::string target_env = needs_spv16 ? "--target-env=vulkan1.3" : "--target-env=vulkan1.2";
|
||||
|
||||
#ifdef _WIN32
|
||||
std::vector<std::string> cmd = {GLSLC, "-fshader-stage=compute", target_env, "\"" + in_path + "\"", "-o", "\"" + out_path + "\""};
|
||||
@@ -1128,6 +1131,20 @@ void process_shaders() {
|
||||
string_to_spv("gated_delta_net_f32_nocluster", "gated_delta_net.comp", merge_maps(base_dict, {{"FLOAT_TYPE", "float"}, {"USE_SUBGROUP_ADD", "1"}, {"USE_SUBGROUP_CLUSTERED", "0"}}));
|
||||
string_to_spv("gated_delta_net_f32_shmem", "gated_delta_net.comp", merge_maps(base_dict, {{"FLOAT_TYPE", "float"}, {"USE_SUBGROUP_ADD", "0"}, {"USE_SUBGROUP_CLUSTERED", "0"}}));
|
||||
|
||||
#if defined(GGML_VULKAN_BFLOAT16_GLSLC_SUPPORT) && defined(GGML_VULKAN_COOPMAT2_GLSLC_SUPPORT)
|
||||
string_to_spv("gated_delta_net_f32", "gated_delta_net_cm2.comp", base_dict, true, false, true);
|
||||
#endif
|
||||
#if defined(GGML_VULKAN_BFLOAT16_GLSLC_SUPPORT) && defined(GGML_VULKAN_COOPMAT_GLSLC_SUPPORT) && defined(GGML_VULKAN_COOPMAT_MAINTENANCE1_GLSLC_SUPPORT)
|
||||
string_to_spv("gated_delta_net_f32", "gated_delta_net_cm.comp", merge_maps(base_dict, {{"BF16ACC", "1"}}), true, true, false, false, "_bf16acc");
|
||||
string_to_spv("gated_delta_net_f32", "gated_delta_net_cm.comp", base_dict, true, true);
|
||||
string_to_spv("gated_delta_net_f32", "gated_delta_net_cm.comp", merge_maps(base_dict, {{"BF16ACC", "1"}, {"V", "32"}}), true, true, false, false, "_bf16acc_v32");
|
||||
string_to_spv("gated_delta_net_f32", "gated_delta_net_cm.comp", merge_maps(base_dict, {{"V", "32"}}), true, true, false, false, "_v32");
|
||||
// whole-head V=128 variant: state mirror in a gmem scratch buffer (won't fit LDS).
|
||||
// For head counts that tail the CU grid at V=64 (e.g. 48-head 27B on 64 CUs).
|
||||
string_to_spv("gated_delta_net_f32", "gated_delta_net_cm.comp", merge_maps(base_dict, {{"BF16ACC", "1"}, {"GMEM_MIRROR", "1"}, {"V", "128"}}), true, true, false, false, "_bf16acc_v128");
|
||||
string_to_spv("gated_delta_net_f32", "gated_delta_net_cm.comp", merge_maps(base_dict, {{"GMEM_MIRROR", "1"}, {"V", "128"}}), true, true, false, false, "_v128");
|
||||
#endif
|
||||
|
||||
string_to_spv("opt_step_adamw_f32", "opt_step_adamw.comp", merge_maps(base_dict, {{"A_TYPE", "float"}}));
|
||||
string_to_spv("opt_step_sgd_f32", "opt_step_sgd.comp", merge_maps(base_dict, {{"A_TYPE", "float"}}));
|
||||
|
||||
|
||||
@@ -5261,16 +5261,20 @@ struct test_gated_delta_net : public test_case {
|
||||
const bool permuted;
|
||||
const bool kda;
|
||||
const int64_t K; // snapshot slot count: 1 = final-only, >1 = last K states
|
||||
const bool correlated;
|
||||
const bool padded_v;
|
||||
const bool padded_q;
|
||||
const bool permuted_k;
|
||||
|
||||
std::string vars() override {
|
||||
return VARS_TO_STR9(type, head_count, head_size, n_seq_tokens, n_seqs, v_repeat, permuted, kda, K);
|
||||
return VARS_TO_STR13(type, head_count, head_size, n_seq_tokens, n_seqs, v_repeat, permuted, kda, K, correlated, padded_v, padded_q, permuted_k);
|
||||
}
|
||||
|
||||
test_gated_delta_net(ggml_type type = GGML_TYPE_F32,
|
||||
int64_t head_count = 4, int64_t head_size = 16, int64_t n_seq_tokens = 1, int64_t n_seqs = 1,
|
||||
int v_repeat = 1, bool permuted = false, bool kda = false, int64_t K = 1)
|
||||
int v_repeat = 1, bool permuted = false, bool kda = false, int64_t K = 1, bool correlated = false, bool padded_v = false, bool padded_q = false, bool permuted_k = false)
|
||||
: type(type), head_count(head_count), head_size(head_size), n_seq_tokens(n_seq_tokens), n_seqs(n_seqs),
|
||||
v_repeat(v_repeat), permuted(permuted), kda(kda), K(K) {}
|
||||
v_repeat(v_repeat), permuted(permuted), kda(kda), K(K), correlated(correlated), padded_v(padded_v), padded_q(padded_q), permuted_k(permuted_k) {}
|
||||
|
||||
ggml_tensor * build_graph(ggml_context * ctx) override {
|
||||
ggml_tensor * q;
|
||||
@@ -5280,11 +5284,15 @@ struct test_gated_delta_net : public test_case {
|
||||
// create with dims 1 and 2 swapped, then permute back to get non-contiguous layout
|
||||
q = ggml_permute(ctx, ggml_new_tensor_4d(ctx, type, head_size, n_seq_tokens, head_count, n_seqs), 0, 2, 1, 3);
|
||||
k = ggml_permute(ctx, ggml_new_tensor_4d(ctx, type, head_size, n_seq_tokens, head_count, n_seqs), 0, 2, 1, 3);
|
||||
v = ggml_permute(ctx, ggml_new_tensor_4d(ctx, type, head_size, n_seq_tokens, head_count * v_repeat, n_seqs), 0, 2, 1, 3);
|
||||
v = ggml_permute(ctx, ggml_new_tensor_4d(ctx, type, head_size + padded_v, n_seq_tokens, head_count * v_repeat, n_seqs), 0, 2, 1, 3);
|
||||
} else {
|
||||
q = ggml_new_tensor_4d(ctx, type, head_size, head_count, n_seq_tokens, n_seqs);
|
||||
k = ggml_new_tensor_4d(ctx, type, head_size, head_count, n_seq_tokens, n_seqs);
|
||||
v = ggml_new_tensor_4d(ctx, type, head_size, head_count * v_repeat, n_seq_tokens, n_seqs);
|
||||
v = ggml_new_tensor_4d(ctx, type, head_size + padded_v, head_count * v_repeat, n_seq_tokens, n_seqs);
|
||||
}
|
||||
if (padded_v) {
|
||||
ggml_set_name(v, "v");
|
||||
v = ggml_view_4d(ctx, v, head_size, head_count * v_repeat, n_seq_tokens, n_seqs, v->nb[1], v->nb[2], v->nb[3], 0);
|
||||
}
|
||||
ggml_set_name(q, "q");
|
||||
ggml_set_name(k, "k");
|
||||
@@ -5299,6 +5307,17 @@ struct test_gated_delta_net : public test_case {
|
||||
// q/k are L2-normalised in qwen35/kimi-linear before delta_net
|
||||
q = ggml_l2_norm(ctx, q, 1e-6f);
|
||||
k = ggml_l2_norm(ctx, k, 1e-6f);
|
||||
// Set layouts after normalization, which produces contiguous tensors.
|
||||
if (padded_q) {
|
||||
ggml_tensor * q_padded = ggml_new_tensor_4d(ctx, type, head_size + 1, head_count, n_seq_tokens, n_seqs);
|
||||
q_padded = ggml_view_4d(ctx, q_padded, head_size, head_count, n_seq_tokens, n_seqs, q_padded->nb[1], q_padded->nb[2], q_padded->nb[3], 0);
|
||||
q = ggml_cpy(ctx, q, q_padded);
|
||||
}
|
||||
if (permuted_k) {
|
||||
ggml_tensor * k_permuted = ggml_new_tensor_4d(ctx, type, head_size, n_seq_tokens, head_count, n_seqs);
|
||||
k_permuted = ggml_permute(ctx, k_permuted, 0, 2, 1, 3);
|
||||
k = ggml_cpy(ctx, k, k_permuted);
|
||||
}
|
||||
ggml_tensor * out = ggml_gated_delta_net(ctx, q, k, v, g, beta, state, K);
|
||||
return out;
|
||||
}
|
||||
@@ -5307,9 +5326,17 @@ struct test_gated_delta_net : public test_case {
|
||||
for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != nullptr; t = ggml_get_next_tensor(ctx, t)) {
|
||||
if (ggml_is_view_op(t->op)) { continue; }
|
||||
if (strcmp(t->name, "g") == 0) {
|
||||
init_tensor_uniform(t, -20.0f, -1e-4f);
|
||||
init_tensor_uniform(t, correlated ? -1e-4f : -20.0f, -1e-4f);
|
||||
} else if (strcmp(t->name, "beta") == 0) {
|
||||
init_tensor_uniform(t, 0.0f, 1.0f);
|
||||
init_tensor_uniform(t, correlated ? 0.9f : 0.0f, correlated ? 0.9f : 1.0f);
|
||||
} else if (correlated && strcmp(t->name, "k") == 0) {
|
||||
// Identical unit keys and weak decay stress cancellation in the triangular solve.
|
||||
GGML_ASSERT(t->type == GGML_TYPE_F32);
|
||||
std::vector<float> data(ggml_nelements(t), 0.0f);
|
||||
for (size_t i = 0; i < data.size(); i += size_t(t->ne[0])) {
|
||||
data[i] = 1.0f;
|
||||
}
|
||||
ggml_backend_tensor_set(t, data.data(), 0, data.size() * sizeof(float));
|
||||
} else if (strcmp(t->name, "v") == 0) {
|
||||
init_tensor_uniform(t, -0.3f, 5.0f);
|
||||
} else {
|
||||
@@ -13634,6 +13661,15 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() {
|
||||
test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 4, 64, 33, 1, 1, false, true));
|
||||
test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 4, 64, 100, 1, 1, false, true));
|
||||
|
||||
// KDA at D=128: decode, prefill, tails, strides, and snapshots.
|
||||
test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 32, 128, 1, 1, 1, false, true));
|
||||
test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 4, 128, 64, 1, 1, false, true));
|
||||
test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 4, 128, 65, 2, 2, true, true));
|
||||
test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 4, 128, 257, 1, 1, false, true));
|
||||
test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 4, 128, 65, 1, 1, false, true, 1, true));
|
||||
test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 4, 128, 257, 1, 1, true, true, 1, true, true));
|
||||
test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 4, 128, 4, 2, 2, false, true, 4));
|
||||
|
||||
// MMA-eligible GDN shapes: whole heads, value slices, tails, and the HIP prefill threshold.
|
||||
test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 16, 128, 65, 1, 1));
|
||||
test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 16, 128, 65, 3, 4));
|
||||
@@ -13645,6 +13681,30 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() {
|
||||
test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 16, 128, 2048, 1, 2));
|
||||
// Keep the WebGPU dispatch below 65535 workgroups (32 per token for this shape).
|
||||
test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 16, 128, 2047, 2, 3));
|
||||
test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 4, 128, 4097, 1, 2));
|
||||
test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 16, 128, 127, 2));
|
||||
test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 16, 128, 256, 1, 2));
|
||||
test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 16, 128, 256, 1, 3));
|
||||
test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 16, 128, 513, 1, 2));
|
||||
test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 16, 128, 512, 2, 2));
|
||||
|
||||
test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 4, 128, 257, 2, 2, true));
|
||||
test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 3, 128, 129, 1, 1, false, false, 1, false, true));
|
||||
test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 4, 128, 4095, 1, 2));
|
||||
|
||||
// Correlated keys, high beta, and weak decay stress the chunked triangular solve.
|
||||
for (int64_t n_tokens : {64, 65, 257}) {
|
||||
test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 4, 128, n_tokens, 1, 1, false, false, 1, true));
|
||||
}
|
||||
|
||||
// Odd V strides require element loads instead of aligned matrix loads.
|
||||
test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 3, 128, 65, 1, 1, false, false, 1, false, true));
|
||||
|
||||
// Q and K can have different strides in both scalar and chunked paths.
|
||||
for (int64_t n_tokens : {4, 65}) {
|
||||
test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 2, 128, n_tokens, 2, 2, false, false, 1, false, false, true));
|
||||
test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 2, 128, n_tokens, 2, 2, false, false, 1, false, false, false, true));
|
||||
}
|
||||
|
||||
// K > 1: output keeps the last min(n_tokens, K) per-token snapshots, ordered most-recent-first
|
||||
// (slot 0 = final state, slot s = state s tokens back).
|
||||
@@ -14296,12 +14356,34 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_perf() {
|
||||
test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 32, 128, 256, 1)); // PP-256
|
||||
test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 32, 128, 512, 1)); // PP-512
|
||||
test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 32, 128, 1024, 1)); // PP-1024
|
||||
// MMA-eligible prefill sweep (16 heads, d=128, K=1): Vulkan cm2 vs CUDA MMA
|
||||
test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 16, 128, 64, 1));
|
||||
test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 16, 128, 128, 1));
|
||||
test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 16, 128, 256, 1));
|
||||
test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 16, 128, 512, 1));
|
||||
test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 16, 128, 1024, 1));
|
||||
test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 16, 128, 2048, 1));
|
||||
test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 16, 128, 4096, 1));
|
||||
// Small model configs (fewer heads = less GPU occupancy for autoregressive)
|
||||
test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 4, 128, 64, 1)); // 4h PP-64
|
||||
test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 4, 128, 256, 1)); // 4h PP-256
|
||||
test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 4, 128, 512, 1)); // 4h PP-512
|
||||
test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 4, 128, 1024, 1)); // 4h PP-1024
|
||||
test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 32, 128, 64, 1, 1, false, true)); // KDA PP-64
|
||||
test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 4, 128, 64, 1, 1, false, true));
|
||||
test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 16, 128, 256, 1, 1, false, true));
|
||||
test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 4, 128, 1024, 1, 1, false, true));
|
||||
// GDN model shapes: 35B (v_repeat=2), 27B (v_repeat=3), batch sweep
|
||||
for (int T : { 2048, 4096, 8192, 16384, 32768 }) {
|
||||
test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 16, 128, T, 1, 2));
|
||||
test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 16, 128, T, 1, 3));
|
||||
}
|
||||
|
||||
test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 4, 128, 2048, 1));
|
||||
test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 4, 128, 4096, 1));
|
||||
test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 4, 128, 8192, 1));
|
||||
test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 20, 128, 1536, 1));
|
||||
test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 20, 128, 2048, 1));
|
||||
|
||||
// lightning_indexer
|
||||
for (int kv : { 256, 4096, 65536 }) {
|
||||
|
||||
Reference in new issue
Block a user