metal : add BF16 variant of the tensor API flash attention kernel (#30491)

* metal : add BF16 variant of the tensor API flash attention kernel

* cont : move tensor FA kernels to kernels/fa_tensor.metal
This commit is contained in:
Ethan Guo authored and GitHub committed 2026-10-15 09:39:47 +03:00
1 parent e467e5b5e5
commit 6e66cb5d4e
7 files changed
+316 -295

No files matched your search

+5 -4
View File
@@ -45,6 +45,7 @@ set(METALLIB_KERNEL_SOURCES
kernels/fa_q5_0.metal
kernels/fa_q5_1.metal
kernels/fa_q8_0.metal
kernels/fa_tensor.metal
kernels/fa_vec_f16.metal
kernels/fa_vec_f32.metal
kernels/fa_vec_q4_0.metal
@@ -233,12 +234,12 @@ else()
VERBATIM
)
set(AIR_FA_TENSOR "${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/fa_f16_tensor.air")
set(AIR_FA_TENSOR "${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/fa_tensor_tensor.air")
add_custom_command(
OUTPUT ${AIR_FA_TENSOR}
COMMAND xcrun -sdk ${METAL_SDK} metal ${XC_FLAGS_TENSOR} -DGGML_METAL_HAS_TENSOR -I ${CMAKE_RUNTIME_OUTPUT_DIRECTORY} -c ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/kernels/fa_f16.metal -o ${AIR_FA_TENSOR}
DEPENDS kernels/fa_f16.metal ${METALLIB_KERNELS_FA_SHARED} kernels/common.h kernels/dequantize.h ${METALLIB_COMMON} ggml-metal-impl.h
COMMENT "Compiling kernels/fa_f16.metal (tensor API)"
COMMAND xcrun -sdk ${METAL_SDK} metal ${XC_FLAGS_TENSOR} -DGGML_METAL_HAS_TENSOR -I ${CMAKE_RUNTIME_OUTPUT_DIRECTORY} -c ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/kernels/fa_tensor.metal -o ${AIR_FA_TENSOR}
DEPENDS kernels/fa_tensor.metal kernels/common.h ${METALLIB_COMMON} ggml-metal-impl.h
COMMENT "Compiling kernels/fa_tensor.metal (tensor API)"
VERBATIM
)
+3 -1
View File
@@ -1866,7 +1866,9 @@ ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_flash_attn_ext_t
const int32_t dk = (int32_t) op->src[1]->ne[0];
const int32_t dv = (int32_t) op->src[2]->ne[0];
snprintf(base, 256, "kernel_flash_attn_ext_tensor_f16_dk%d_dv%d", dk, dv);
const char * type = op->src[1]->type == GGML_TYPE_BF16 ? "bf16" : "f16";
snprintf(base, 256, "kernel_flash_attn_ext_tensor_%s_dk%d_dv%d", type, dk, dv);
snprintf(name, 256, "%s_mask=%d_sinks=%d_bias=%d_scap=%d", base, has_mask, has_sinks, has_bias, has_scap);
ggml_metal_pipeline_with_params res = ggml_metal_library_get_pipeline(lib, name);
+1
View File
@@ -118,6 +118,7 @@ int ggml_metal_pipeline_max_theads_per_threadgroup(struct ggml_metal_pipeline_wi
X(FA_Q5_0, fa_q5_0) \
X(FA_Q5_1, fa_q5_1) \
X(FA_Q8_0, fa_q8_0) \
X(FA_TENSOR, fa_tensor) \
X(FA_VEC_F16, fa_vec_f16) \
X(FA_VEC_F32, fa_vec_f32) \
X(FA_VEC_Q4_0, fa_vec_q4_0) \
+2 -2
View File
@@ -3161,7 +3161,7 @@ static bool ggml_metal_op_flash_attn_ext_use_tensor(const ggml_tensor * op, bool
return false;
}
if (op->src[1]->type != GGML_TYPE_F16 && !ggml_metal_op_flash_attn_ext_use_kv_f16(op)) {
if (op->src[1]->type != GGML_TYPE_F16 && op->src[1]->type != GGML_TYPE_BF16 && !ggml_metal_op_flash_attn_ext_use_kv_f16(op)) {
return false;
}
@@ -3678,7 +3678,7 @@ int ggml_metal_op_flash_attn_ext(ggml_metal_op_t ctx, int idx) {
/*.logit_softcap =*/ logit_softcap,
};
// shared memory layout: queries (half), scores (float), probabilities (half), row scale (float), rescale flag (int)
// shared memory layout: queries (16-bit), scores (float), probabilities (16-bit), row scale (float), rescale flag (int)
const size_t smem = GGML_PAD(nqptg*ne00*sizeof(ggml_fp16_t) + nqptg*ncpsg*(sizeof(float) + sizeof(ggml_fp16_t)) + nqptg*sizeof(float) + sizeof(int32_t), 16);
auto pipeline = ggml_metal_library_get_pipeline_flash_attn_ext_tensor(lib, op, has_mask, has_sinks, has_bias, has_scap);
-288
View File
@@ -75,291 +75,3 @@ template [[host_name("kernel_flash_attn_ext_bf16_dk576_dv512")]] kernel flash_at
#undef FA_TYPES
#undef FA_TYPES_BF
#undef FA_TYPES_F32
#ifdef GGML_METAL_HAS_TENSOR
constant bool FC_flash_attn_ext_tensor_has_mask [[function_constant(FC_FLASH_ATTN_EXT_TENSOR + 0)]];
constant bool FC_flash_attn_ext_tensor_has_sinks [[function_constant(FC_FLASH_ATTN_EXT_TENSOR + 1)]];
constant bool FC_flash_attn_ext_tensor_has_bias [[function_constant(FC_FLASH_ATTN_EXT_TENSOR + 2)]];
constant bool FC_flash_attn_ext_tensor_has_scap [[function_constant(FC_FLASH_ATTN_EXT_TENSOR + 3)]];
// ref: https://arxiv.org/pdf/2307.08691.pdf
template<
short DK, // K head size
short DV, // V head size
short Q = OP_FLASH_ATTN_EXT_TENSOR_NQPSG, // queries per threadgroup
short C = OP_FLASH_ATTN_EXT_TENSOR_NCPSG, // cache items per threadgroup
short NSG = OP_FLASH_ATTN_EXT_TENSOR_NSG> // number of simd groups
kernel void kernel_flash_attn_ext_tensor(
constant ggml_metal_kargs_flash_attn_ext & args,
device const char * q,
device const char * k,
device const char * v,
device const char * mask,
device const char * sinks,
device const char * blk,
device char * dst,
threadgroup char * shmem [[threadgroup(0)]],
uint3 tgpig [[threadgroup_position_in_grid]],
ushort tiisg [[thread_index_in_simdgroup]],
ushort sgitg [[simdgroup_index_in_threadgroup]]) {
constexpr short NW = N_SIMDWIDTH;
constexpr short NT = NW*NSG;
constexpr short NQ = Q/NSG;
constexpr short NC = C/NW; // columns per thread
static_assert(DK % 4 == 0, "DK must be divisible by 4");
static_assert(Q % NSG == 0, "Q must be divisible by NSG");
static_assert(C % NW == 0, "C must be divisible by NW");
const int iq3 = tgpig[2];
const int iq2 = tgpig[1];
const int iq1 = tgpig[0]*Q;
const short tiitg = sgitg*NW + tiisg;
threadgroup half * sq = (threadgroup half *) shmem; // [Q, DK] queries
threadgroup float * ss = (threadgroup float *) (sq + Q*DK); // [Q, C] scores
threadgroup half * sp = (threadgroup half *) (ss + Q*C); // [Q, C] probabilities
threadgroup float * sr = (threadgroup float *) (sp + Q*C); // [Q] per-row scale of O
threadgroup int * sf = (threadgroup int *) (sr + Q); // [1] last iteration (ic0 + 1) that rescaled O
q += iq1*args.nb01 + iq2*args.nb02 + iq3*args.nb03;
{
const int ikv2 = iq2/(args.ne02/args.ne_12_2);
const int ikv3 = iq3/(args.ne03/args.ne_12_3);
k += ikv2*args.nb12 + ikv3*args.nb13;
v += ikv2*args.nb22 + ikv3*args.nb23;
}
// with softcap the scale is small (scale/softcap), so it is applied to the scores to keep the precision of Q
const float qscale = FC_flash_attn_ext_tensor_has_scap ? 1.0f : args.scale;
// load the queries, with the scale folded in
for (int i = tiitg; i < Q*DK/4; i += NT) {
const int j = i/(DK/4);
float4 q4 = 0.0f;
if (iq1 + j < args.ne01) {
q4 = ((device const float4 *) (q + j*args.nb01))[i%(DK/4)];
}
((threadgroup half4 *) sq)[i] = (half4) (q4*qscale);
}
device const half * pm[NQ];
FOR_UNROLL (short jj = 0; jj < NQ; ++jj) {
const short j = jj*NSG + sgitg;
pm[jj] = (device const half *) (mask + (iq1 + j)*args.nb31 + (iq2%args.ne32)*args.nb32 + (iq3%args.ne33)*args.nb33);
}
{
const int nblk1 = (args.ne01 + Q - 1)/Q;
const int nblk0 = (args.ne11 + C - 1)/C;
blk += (((iq3%args.ne33)*args.ne32 + (iq2%args.ne32))*nblk1 + iq1/Q)*nblk0;
}
float M[NQ];
float S[NQ];
FOR_UNROLL (short jj = 0; jj < NQ; ++jj) {
M[jj] = -FLT_MAX/2;
S[jj] = 0.0f;
}
float slope = 1.0f;
// ALiBi
if (FC_flash_attn_ext_tensor_has_bias) {
const short h = iq2;
const float base = h < args.n_head_log2 ? args.m0 : args.m1;
const short exph = h < args.n_head_log2 ? h + 1 : 2*(h - args.n_head_log2) + 1;
slope = pow(base, exph);
}
const int sk = args.ns10;
const int sv = args.ns20;
auto tq = tensor(sq, dextents<int32_t, 2>(DK, Q));
auto ts = tensor(ss, dextents<int32_t, 2>(C, Q));
auto tp = tensor(sp, dextents<int32_t, 2>(C, Q));
mpp::tensor_ops::matmul2d<
mpp::tensor_ops::matmul2d_descriptor(Q, C, DK, false, true, false, mpp::tensor_ops::matmul2d_descriptor::mode::multiply),
execution_simdgroups<NSG>> mm_qk;
mpp::tensor_ops::matmul2d<
mpp::tensor_ops::matmul2d_descriptor(Q, DV, C, false, false, false, mpp::tensor_ops::matmul2d_descriptor::mode::multiply_accumulate),
execution_simdgroups<NSG>> mm_pv;
auto tv0 = tensor((device half *) v, dextents<int32_t, 2>(DV, C), array<int, 2>({1, sv}));
// the O matrix from the paper
auto co = mm_pv.template get_destination_cooperative_tensor<decltype(tp), decltype(tv0), float>();
FOR_UNROLL (short i = 0; i < co.get_capacity(); ++i) {
if (co.is_valid_element(i)) {
co[i] = 0.0f;
}
}
if (tiitg == 0) {
sf[0] = 0;
}
threadgroup_barrier(mem_flags::mem_threadgroup);
// the host guarantees ne11 % C == 0
for (int ic0 = 0, ic = 0; ic < args.ne11; ++ic0, ic += C) {
char blk_cur = 1;
if (FC_flash_attn_ext_tensor_has_mask) {
blk_cur = blk[ic0];
if (blk_cur == 0) {
continue;
}
}
// Q*K^T
{
auto tk = tensor((device half *) (k + (uint64_t) ic*args.nb11), dextents<int32_t, 2>(DK, C), array<int, 2>({1, sk}));
mm_qk.run(tq, tk, ts);
}
threadgroup_barrier(mem_flags::mem_threadgroup);
// online softmax
FOR_UNROLL (short jj = 0; jj < NQ; ++jj) {
const short j = jj*NSG + sgitg;
float s[NC];
FOR_UNROLL (short ii = 0; ii < NC; ++ii) {
s[ii] = ss[j*C + ii*NW + tiisg];
}
if (FC_flash_attn_ext_tensor_has_scap) {
FOR_UNROLL (short ii = 0; ii < NC; ++ii) {
s[ii] = args.logit_softcap*precise::tanh(s[ii]*args.scale);
}
}
if (FC_flash_attn_ext_tensor_has_mask && blk_cur != 2 && iq1 + j < args.ne31) {
FOR_UNROLL (short ii = 0; ii < NC; ++ii) {
s[ii] += slope*(float) pm[jj][ic + ii*NW + tiisg];
}
}
float m = M[jj];
FOR_UNROLL (short ii = 0; ii < NC; ++ii) {
m = max(m, s[ii]);
}
m = simd_max(m);
// lazy rescaling: move the running max only when it grows by more than 8 (e^8 fits in half)
float ms = 1.0f;
if (m > M[jj] + 8.0f) {
ms = exp(M[jj] - m);
M[jj] = m;
if (tiisg == 0) {
sf[0] = ic0 + 1;
}
}
float sum = 0.0f;
FOR_UNROLL (short ii = 0; ii < NC; ++ii) {
// the sum uses the same rounded values as P*V
const half p = (half) exp(s[ii] - M[jj]);
sp[j*C + ii*NW + tiisg] = p;
sum += (float) p;
}
S[jj] = S[jj]*ms + simd_sum(sum);
if (tiisg == 0) {
sr[j] = ms;
}
}
threadgroup_barrier(mem_flags::mem_threadgroup);
// O = diag(ms)*O + P*V
if (sf[0] == ic0 + 1) {
FOR_UNROLL (short i = 0; i < co.get_capacity(); ++i) {
if (co.is_valid_element(i)) {
co[i] *= sr[co.get_multidimensional_index(i)[1]];
}
}
}
{
auto tv = tensor((device half *) (v + (uint64_t) ic*args.nb21), dextents<int32_t, 2>(DV, C), array<int, 2>({1, sv}));
mm_pv.run(tp, tv, co);
}
threadgroup_barrier(mem_flags::mem_threadgroup);
}
FOR_UNROLL (short jj = 0; jj < NQ; ++jj) {
const short j = jj*NSG + sgitg;
// the sink only adds to the denominator - its rescale of O is folded into the final scale
float ms = 1.0f;
if (FC_flash_attn_ext_tensor_has_sinks) {
const float s = ((device const float *) sinks)[iq2];
const float m = max(M[jj], s);
ms = exp(M[jj] - m);
S[jj] = S[jj]*ms + exp(s - m);
}
if (tiisg == 0) {
sr[j] = S[jj] == 0.0f ? 0.0f : ms/S[jj];
}
}
threadgroup_barrier(mem_flags::mem_threadgroup);
FOR_UNROLL (short i = 0; i < co.get_capacity(); ++i) {
if (co.is_valid_element(i)) {
co[i] *= sr[co.get_multidimensional_index(i)[1]];
}
}
// store to global memory - rows past ne01 are clipped by the tensor extents
device float * pdst = (device float *) dst + ((uint64_t) iq3*args.ne2*args.ne1 + iq2 + (uint64_t) iq1*args.ne1)*DV;
auto td = tensor(pdst, dextents<int32_t, 2>(DV, args.ne01 - iq1), array<int, 2>({1, args.ne1*DV}));
co.store(td);
}
typedef decltype(kernel_flash_attn_ext_tensor<64, 64>) flash_attn_ext_tensor_t;
template [[host_name("kernel_flash_attn_ext_tensor_f16_dk64_dv64" )]] kernel flash_attn_ext_tensor_t kernel_flash_attn_ext_tensor<64, 64>;
template [[host_name("kernel_flash_attn_ext_tensor_f16_dk128_dv128")]] kernel flash_attn_ext_tensor_t kernel_flash_attn_ext_tensor<128, 128>;
template [[host_name("kernel_flash_attn_ext_tensor_f16_dk192_dv128")]] kernel flash_attn_ext_tensor_t kernel_flash_attn_ext_tensor<192, 128>;
template [[host_name("kernel_flash_attn_ext_tensor_f16_dk256_dv256")]] kernel flash_attn_ext_tensor_t kernel_flash_attn_ext_tensor<256, 256>;
template [[host_name("kernel_flash_attn_ext_tensor_f16_dk512_dv512")]] kernel flash_attn_ext_tensor_t kernel_flash_attn_ext_tensor<512, 512, OP_FLASH_ATTN_EXT_TENSOR_NQPSG_LARGE>;
template [[host_name("kernel_flash_attn_ext_tensor_f16_dk576_dv512")]] kernel flash_attn_ext_tensor_t kernel_flash_attn_ext_tensor<576, 512, OP_FLASH_ATTN_EXT_TENSOR_NQPSG_LARGE>;
#endif // GGML_METAL_HAS_TENSOR
+298
View File
@@ -0,0 +1,298 @@
#include "common.h"
#ifdef GGML_METAL_HAS_TENSOR
constant bool FC_flash_attn_ext_tensor_has_mask [[function_constant(FC_FLASH_ATTN_EXT_TENSOR + 0)]];
constant bool FC_flash_attn_ext_tensor_has_sinks [[function_constant(FC_FLASH_ATTN_EXT_TENSOR + 1)]];
constant bool FC_flash_attn_ext_tensor_has_bias [[function_constant(FC_FLASH_ATTN_EXT_TENSOR + 2)]];
constant bool FC_flash_attn_ext_tensor_has_scap [[function_constant(FC_FLASH_ATTN_EXT_TENSOR + 3)]];
// ref: https://arxiv.org/pdf/2307.08691.pdf
template<
typename T, // Q, K, V and P types
typename T4,
short DK, // K head size
short DV, // V head size
short Q = OP_FLASH_ATTN_EXT_TENSOR_NQPSG, // queries per threadgroup
short C = OP_FLASH_ATTN_EXT_TENSOR_NCPSG, // cache items per threadgroup
short NSG = OP_FLASH_ATTN_EXT_TENSOR_NSG> // number of simd groups
kernel void kernel_flash_attn_ext_tensor(
constant ggml_metal_kargs_flash_attn_ext & args,
device const char * q,
device const char * k,
device const char * v,
device const char * mask,
device const char * sinks,
device const char * blk,
device char * dst,
threadgroup char * shmem [[threadgroup(0)]],
uint3 tgpig [[threadgroup_position_in_grid]],
ushort tiisg [[thread_index_in_simdgroup]],
ushort sgitg [[simdgroup_index_in_threadgroup]]) {
constexpr short NW = N_SIMDWIDTH;
constexpr short NT = NW*NSG;
constexpr short NQ = Q/NSG;
constexpr short NC = C/NW; // columns per thread
static_assert(DK % 4 == 0, "DK must be divisible by 4");
static_assert(Q % NSG == 0, "Q must be divisible by NSG");
static_assert(C % NW == 0, "C must be divisible by NW");
const int iq3 = tgpig[2];
const int iq2 = tgpig[1];
const int iq1 = tgpig[0]*Q;
const short tiitg = sgitg*NW + tiisg;
threadgroup T * sq = (threadgroup T *) shmem; // [Q, DK] queries
threadgroup float * ss = (threadgroup float *) (sq + Q*DK); // [Q, C] scores
threadgroup T * sp = (threadgroup T *) (ss + Q*C); // [Q, C] probabilities
threadgroup float * sr = (threadgroup float *) (sp + Q*C); // [Q] per-row scale of O
threadgroup int * sf = (threadgroup int *) (sr + Q); // [1] last iteration (ic0 + 1) that rescaled O
q += iq1*args.nb01 + iq2*args.nb02 + iq3*args.nb03;
{
const int ikv2 = iq2/(args.ne02/args.ne_12_2);
const int ikv3 = iq3/(args.ne03/args.ne_12_3);
k += ikv2*args.nb12 + ikv3*args.nb13;
v += ikv2*args.nb22 + ikv3*args.nb23;
}
// with softcap the scale is small (scale/softcap), so it is applied to the scores to keep the precision of Q
const float qscale = FC_flash_attn_ext_tensor_has_scap ? 1.0f : args.scale;
// load the queries, with the scale folded in
for (int i = tiitg; i < Q*DK/4; i += NT) {
const int j = i/(DK/4);
float4 q4 = 0.0f;
if (iq1 + j < args.ne01) {
q4 = ((device const float4 *) (q + j*args.nb01))[i%(DK/4)];
}
((threadgroup T4 *) sq)[i] = (T4) (q4*qscale);
}
device const half * pm[NQ];
FOR_UNROLL (short jj = 0; jj < NQ; ++jj) {
const short j = jj*NSG + sgitg;
pm[jj] = (device const half *) (mask + (iq1 + j)*args.nb31 + (iq2%args.ne32)*args.nb32 + (iq3%args.ne33)*args.nb33);
}
{
const int nblk1 = (args.ne01 + Q - 1)/Q;
const int nblk0 = (args.ne11 + C - 1)/C;
blk += (((iq3%args.ne33)*args.ne32 + (iq2%args.ne32))*nblk1 + iq1/Q)*nblk0;
}
float M[NQ];
float S[NQ];
FOR_UNROLL (short jj = 0; jj < NQ; ++jj) {
M[jj] = -FLT_MAX/2;
S[jj] = 0.0f;
}
float slope = 1.0f;
// ALiBi
if (FC_flash_attn_ext_tensor_has_bias) {
const short h = iq2;
const float base = h < args.n_head_log2 ? args.m0 : args.m1;
const short exph = h < args.n_head_log2 ? h + 1 : 2*(h - args.n_head_log2) + 1;
slope = pow(base, exph);
}
const int sk = args.ns10;
const int sv = args.ns20;
auto tq = tensor(sq, dextents<int32_t, 2>(DK, Q));
auto ts = tensor(ss, dextents<int32_t, 2>(C, Q));
auto tp = tensor(sp, dextents<int32_t, 2>(C, Q));
mpp::tensor_ops::matmul2d<
mpp::tensor_ops::matmul2d_descriptor(Q, C, DK, false, true, false, mpp::tensor_ops::matmul2d_descriptor::mode::multiply),
execution_simdgroups<NSG>> mm_qk;
mpp::tensor_ops::matmul2d<
mpp::tensor_ops::matmul2d_descriptor(Q, DV, C, false, false, false, mpp::tensor_ops::matmul2d_descriptor::mode::multiply_accumulate),
execution_simdgroups<NSG>> mm_pv;
auto tv0 = tensor((device T *) v, dextents<int32_t, 2>(DV, C), array<int, 2>({1, sv}));
// the O matrix from the paper
auto co = mm_pv.template get_destination_cooperative_tensor<decltype(tp), decltype(tv0), float>();
FOR_UNROLL (short i = 0; i < co.get_capacity(); ++i) {
if (co.is_valid_element(i)) {
co[i] = 0.0f;
}
}
if (tiitg == 0) {
sf[0] = 0;
}
threadgroup_barrier(mem_flags::mem_threadgroup);
// the host guarantees ne11 % C == 0
for (int ic0 = 0, ic = 0; ic < args.ne11; ++ic0, ic += C) {
char blk_cur = 1;
if (FC_flash_attn_ext_tensor_has_mask) {
blk_cur = blk[ic0];
if (blk_cur == 0) {
continue;
}
}
// Q*K^T
{
auto tk = tensor((device T *) (k + (uint64_t) ic*args.nb11), dextents<int32_t, 2>(DK, C), array<int, 2>({1, sk}));
mm_qk.run(tq, tk, ts);
}
threadgroup_barrier(mem_flags::mem_threadgroup);
// online softmax
FOR_UNROLL (short jj = 0; jj < NQ; ++jj) {
const short j = jj*NSG + sgitg;
float s[NC];
FOR_UNROLL (short ii = 0; ii < NC; ++ii) {
s[ii] = ss[j*C + ii*NW + tiisg];
}
if (FC_flash_attn_ext_tensor_has_scap) {
FOR_UNROLL (short ii = 0; ii < NC; ++ii) {
s[ii] = args.logit_softcap*precise::tanh(s[ii]*args.scale);
}
}
if (FC_flash_attn_ext_tensor_has_mask && blk_cur != 2 && iq1 + j < args.ne31) {
FOR_UNROLL (short ii = 0; ii < NC; ++ii) {
s[ii] += slope*(float) pm[jj][ic + ii*NW + tiisg];
}
}
float m = M[jj];
FOR_UNROLL (short ii = 0; ii < NC; ++ii) {
m = max(m, s[ii]);
}
m = simd_max(m);
// lazy rescaling: move the running max only when it grows by more than 8 (e^8 fits in half)
float ms = 1.0f;
if (m > M[jj] + 8.0f) {
ms = exp(M[jj] - m);
M[jj] = m;
if (tiisg == 0) {
sf[0] = ic0 + 1;
}
}
float sum = 0.0f;
FOR_UNROLL (short ii = 0; ii < NC; ++ii) {
// the sum uses the same rounded values as P*V
const T p = (T) exp(s[ii] - M[jj]);
sp[j*C + ii*NW + tiisg] = p;
sum += (float) p;
}
S[jj] = S[jj]*ms + simd_sum(sum);
if (tiisg == 0) {
sr[j] = ms;
}
}
threadgroup_barrier(mem_flags::mem_threadgroup);
// O = diag(ms)*O + P*V
if (sf[0] == ic0 + 1) {
FOR_UNROLL (short i = 0; i < co.get_capacity(); ++i) {
if (co.is_valid_element(i)) {
co[i] *= sr[co.get_multidimensional_index(i)[1]];
}
}
}
{
auto tv = tensor((device T *) (v + (uint64_t) ic*args.nb21), dextents<int32_t, 2>(DV, C), array<int, 2>({1, sv}));
mm_pv.run(tp, tv, co);
}
threadgroup_barrier(mem_flags::mem_threadgroup);
}
FOR_UNROLL (short jj = 0; jj < NQ; ++jj) {
const short j = jj*NSG + sgitg;
// the sink only adds to the denominator - its rescale of O is folded into the final scale
float ms = 1.0f;
if (FC_flash_attn_ext_tensor_has_sinks) {
const float s = ((device const float *) sinks)[iq2];
const float m = max(M[jj], s);
ms = exp(M[jj] - m);
S[jj] = S[jj]*ms + exp(s - m);
}
if (tiisg == 0) {
sr[j] = S[jj] == 0.0f ? 0.0f : ms/S[jj];
}
}
threadgroup_barrier(mem_flags::mem_threadgroup);
FOR_UNROLL (short i = 0; i < co.get_capacity(); ++i) {
if (co.is_valid_element(i)) {
co[i] *= sr[co.get_multidimensional_index(i)[1]];
}
}
// store to global memory - rows past ne01 are clipped by the tensor extents
device float * pdst = (device float *) dst + ((uint64_t) iq3*args.ne2*args.ne1 + iq2 + (uint64_t) iq1*args.ne1)*DV;
auto td = tensor(pdst, dextents<int32_t, 2>(DV, args.ne01 - iq1), array<int, 2>({1, args.ne1*DV}));
co.store(td);
}
typedef decltype(kernel_flash_attn_ext_tensor<half, half4, 64, 64>) flash_attn_ext_tensor_t;
template [[host_name("kernel_flash_attn_ext_tensor_f16_dk64_dv64" )]] kernel flash_attn_ext_tensor_t kernel_flash_attn_ext_tensor<half, half4, 64, 64>;
template [[host_name("kernel_flash_attn_ext_tensor_f16_dk128_dv128")]] kernel flash_attn_ext_tensor_t kernel_flash_attn_ext_tensor<half, half4, 128, 128>;
template [[host_name("kernel_flash_attn_ext_tensor_f16_dk192_dv128")]] kernel flash_attn_ext_tensor_t kernel_flash_attn_ext_tensor<half, half4, 192, 128>;
template [[host_name("kernel_flash_attn_ext_tensor_f16_dk256_dv256")]] kernel flash_attn_ext_tensor_t kernel_flash_attn_ext_tensor<half, half4, 256, 256>;
template [[host_name("kernel_flash_attn_ext_tensor_f16_dk512_dv512")]] kernel flash_attn_ext_tensor_t kernel_flash_attn_ext_tensor<half, half4, 512, 512, OP_FLASH_ATTN_EXT_TENSOR_NQPSG_LARGE>;
template [[host_name("kernel_flash_attn_ext_tensor_f16_dk576_dv512")]] kernel flash_attn_ext_tensor_t kernel_flash_attn_ext_tensor<half, half4, 576, 512, OP_FLASH_ATTN_EXT_TENSOR_NQPSG_LARGE>;
template [[host_name("kernel_flash_attn_ext_tensor_bf16_dk64_dv64" )]] kernel flash_attn_ext_tensor_t kernel_flash_attn_ext_tensor<bfloat, bfloat4, 64, 64>;
template [[host_name("kernel_flash_attn_ext_tensor_bf16_dk128_dv128")]] kernel flash_attn_ext_tensor_t kernel_flash_attn_ext_tensor<bfloat, bfloat4, 128, 128>;
template [[host_name("kernel_flash_attn_ext_tensor_bf16_dk192_dv128")]] kernel flash_attn_ext_tensor_t kernel_flash_attn_ext_tensor<bfloat, bfloat4, 192, 128>;
template [[host_name("kernel_flash_attn_ext_tensor_bf16_dk256_dv256")]] kernel flash_attn_ext_tensor_t kernel_flash_attn_ext_tensor<bfloat, bfloat4, 256, 256>;
template [[host_name("kernel_flash_attn_ext_tensor_bf16_dk512_dv512")]] kernel flash_attn_ext_tensor_t kernel_flash_attn_ext_tensor<bfloat, bfloat4, 512, 512, OP_FLASH_ATTN_EXT_TENSOR_NQPSG_LARGE>;
template [[host_name("kernel_flash_attn_ext_tensor_bf16_dk576_dv512")]] kernel flash_attn_ext_tensor_t kernel_flash_attn_ext_tensor<bfloat, bfloat4, 576, 512, OP_FLASH_ATTN_EXT_TENSOR_NQPSG_LARGE>;
#endif // GGML_METAL_HAS_TENSOR
+7
View File
@@ -11813,11 +11813,18 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() {
test_cases.emplace_back(new test_flash_attn_ext(256, 256, 4, {2, 1}, 1024, 32, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_F16, GGML_TYPE_F16));
test_cases.emplace_back(new test_flash_attn_ext(512, 512, 4, {2, 1}, 1024, 4, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_F16, GGML_TYPE_F16));
// BF16 K/V for the other head sizes of the Metal tensor API kernel
test_cases.emplace_back(new test_flash_attn_ext(128, 128, 4, {12, 1}, 512, 75, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_BF16, GGML_TYPE_BF16));
test_cases.emplace_back(new test_flash_attn_ext(192, 128, 4, { 8, 1}, 512, 75, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_BF16, GGML_TYPE_BF16));
test_cases.emplace_back(new test_flash_attn_ext(512, 512, 4, { 4, 1}, 512, 75, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_BF16, GGML_TYPE_BF16));
test_cases.emplace_back(new test_flash_attn_ext(576, 512, 4, { 4, 1}, 512, 75, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_BF16, GGML_TYPE_BF16, {0, 1, 2, 3}, true, true));
// FLASH_ATTN_EXT: large logits
test_cases.emplace_back(new test_flash_attn_ext_large_logits( 64, 64, 16, {4, 1}, 1024, 75, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_F16, GGML_TYPE_F16));
test_cases.emplace_back(new test_flash_attn_ext_large_logits(128, 128, 8, {4, 1}, 1024, 75, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_F16, GGML_TYPE_F16));
test_cases.emplace_back(new test_flash_attn_ext_large_logits(256, 256, 4, {4, 1}, 1024, 75, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_F16, GGML_TYPE_F16));
test_cases.emplace_back(new test_flash_attn_ext_large_logits(256, 256, 4, {4, 1}, 1024, 75, true, false, 0, 10.0f, GGML_PREC_F32, GGML_TYPE_F16, GGML_TYPE_F16));
test_cases.emplace_back(new test_flash_attn_ext_large_logits(256, 256, 4, {4, 1}, 1024, 75, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_BF16, GGML_TYPE_BF16));
test_cases.emplace_back(new test_cross_entropy_loss (GGML_TYPE_F32, { 10, 5, 4, 3}));
test_cases.emplace_back(new test_cross_entropy_loss (GGML_TYPE_F32, {30000, 1, 1, 1}));