vulkan: pack FMA for f16 accumulation, no dot product op (#29877)
* vulkan: pack FMA for f16 accumulation, no dot product op * vulkan: dot_product packs two rows per f16 FMA * vulkan: rename dot_product to dot_product_v2
This commit is contained in:
1 parent
efe3626b11
commit
0f701c8829
3 files changed
+33
-2
No files matched your search
@@ -13,6 +13,14 @@ ACC_TYPE dot_product(f16vec2 a, f16vec2 b, ACC_TYPE acc) {
|
||||
return ACC_TYPE(v_dot2_f32_f16(a, b, float(acc)));
|
||||
}
|
||||
|
||||
ACC_TYPEV2 dot_product_v2(f16vec4 a0, f16vec4 a1, f16vec4 b, ACC_TYPEV2 acc) {
|
||||
return ACC_TYPEV2(dot_product(a0, b, acc.x), dot_product(a1, b, acc.y));
|
||||
}
|
||||
|
||||
ACC_TYPEV2 dot_product_v2(f16vec2 a0, f16vec2 a1, f16vec2 b, ACC_TYPEV2 acc) {
|
||||
return ACC_TYPEV2(dot_product(a0, b, acc.x), dot_product(a1, b, acc.y));
|
||||
}
|
||||
|
||||
#else
|
||||
|
||||
ACC_TYPE dot_product(FLOAT_TYPEV4 a, FLOAT_TYPEV4 b, ACC_TYPE acc) {
|
||||
@@ -24,4 +32,27 @@ ACC_TYPE dot_product(FLOAT_TYPEV2 a, FLOAT_TYPEV2 b, ACC_TYPE acc) {
|
||||
return fma(ACC_TYPE(a.x), ACC_TYPE(b.x), fma(ACC_TYPE(a.y), ACC_TYPE(b.y), acc));
|
||||
}
|
||||
|
||||
#if defined(ACC_F16)
|
||||
// two rows at once, so f16 accumulation can use packed fma
|
||||
ACC_TYPEV2 dot_product_v2(FLOAT_TYPEV4 a0, FLOAT_TYPEV4 a1, FLOAT_TYPEV4 b, ACC_TYPEV2 acc) {
|
||||
acc = fma(ACC_TYPEV2(a0.w, a1.w), ACC_TYPEV2(b.w), acc);
|
||||
acc = fma(ACC_TYPEV2(a0.z, a1.z), ACC_TYPEV2(b.z), acc);
|
||||
acc = fma(ACC_TYPEV2(a0.y, a1.y), ACC_TYPEV2(b.y), acc);
|
||||
return fma(ACC_TYPEV2(a0.x, a1.x), ACC_TYPEV2(b.x), acc);
|
||||
}
|
||||
|
||||
ACC_TYPEV2 dot_product_v2(FLOAT_TYPEV2 a0, FLOAT_TYPEV2 a1, FLOAT_TYPEV2 b, ACC_TYPEV2 acc) {
|
||||
acc = fma(ACC_TYPEV2(a0.y, a1.y), ACC_TYPEV2(b.y), acc);
|
||||
return fma(ACC_TYPEV2(a0.x, a1.x), ACC_TYPEV2(b.x), acc);
|
||||
}
|
||||
#else
|
||||
ACC_TYPEV2 dot_product_v2(FLOAT_TYPEV4 a0, FLOAT_TYPEV4 a1, FLOAT_TYPEV4 b, ACC_TYPEV2 acc) {
|
||||
return ACC_TYPEV2(dot_product(a0, b, acc.x), dot_product(a1, b, acc.y));
|
||||
}
|
||||
|
||||
ACC_TYPEV2 dot_product_v2(FLOAT_TYPEV2 a0, FLOAT_TYPEV2 a1, FLOAT_TYPEV2 b, ACC_TYPEV2 acc) {
|
||||
return ACC_TYPEV2(dot_product(a0, b, acc.x), dot_product(a1, b, acc.y));
|
||||
}
|
||||
#endif
|
||||
|
||||
#endif
|
||||
@@ -407,8 +407,7 @@ void main() {
|
||||
[[unroll]] for (uint cr = 0; cr < TM / 2; cr++) {
|
||||
// [WNITER][TN][WMITER][TM / 2] -> [wsic][cc][wsir][cr]
|
||||
const uint sums_idx = (wsic * TN + cc) * WMITER * (TM / 2) + wsir * (TM / 2) + cr;
|
||||
sums[sums_idx].x = dot_product(cache_a[wsir * TM + 2 * cr ], cache_b, sums[sums_idx].x);
|
||||
sums[sums_idx].y = dot_product(cache_a[wsir * TM + 2 * cr + 1], cache_b, sums[sums_idx].y);
|
||||
sums[sums_idx] = dot_product_v2(cache_a[wsir * TM + 2 * cr], cache_a[wsir * TM + 2 * cr + 1], cache_b, sums[sums_idx]);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -485,6 +485,7 @@ void matmul_shaders(bool fp16, MatMulIdType matmul_id_type, bool coopmat, bool c
|
||||
base_dict["ACC_TYPE_VEC4"] = f16acc ? "f16vec4" : "vec4";
|
||||
if (f16acc) {
|
||||
base_dict["ACC_TYPE_MAX"] = "float16_t(65504.0)";
|
||||
base_dict["ACC_F16"] = "1";
|
||||
}
|
||||
|
||||
if (coopmat) {
|
||||
|
||||
Reference in new issue
Block a user