#version 450

#extension GL_EXT_control_flow_attributes : enable
#extension GL_EXT_shader_16bit_storage : require

#extension GL_EXT_shader_explicit_arithmetic_types_float16 : require
#extension GL_EXT_shader_explicit_arithmetic_types_int32 : require

#if defined(BFLOAT16)
#extension GL_EXT_bfloat16 : enable
#endif

#extension GL_KHR_shader_subgroup_basic : enable
#extension GL_KHR_shader_subgroup_arithmetic : enable
#extension GL_KHR_shader_subgroup_vote : enable
#extension GL_KHR_memory_scope_semantics : enable
#extension GL_KHR_cooperative_matrix : enable

#include "types.glsl"
#include "flash_attn_base.glsl"
#if !defined(BFLOAT16)
#include "flash_attn_dequant.glsl"
#endif

// These need to be supported N,M values for a MatBc x MatBr x 16 coopmatmuladd
const uint32_t MatBr = 16;
const uint32_t MatBc = 16;

const uint32_t rows_per_thread = Br / row_split;
const uint32_t cols_per_iter = gl_WorkGroupSize.x / row_split;
const uint32_t cols_per_thread = Bc / cols_per_iter;


layout (binding = 0) readonly buffer Q {float data_q[];};
layout (binding = 0) readonly buffer QV4 {vec4 data_qv4[];};
layout (binding = 1) readonly buffer K {FLOAT_TYPE data_k[];};
layout (binding = 1) readonly buffer KV4 {FLOAT_TYPEV4 data_kv4[];};
layout (binding = 2) readonly buffer V {FLOAT_TYPE data_v[];};
layout (binding = 2) readonly buffer VV4 {FLOAT_TYPEV4 data_vv4[];};
layout (binding = 3) readonly buffer M {float16_t data_m[];};

shared float tmpsh[row_split];

const uint32_t qstride = HSK_pad / 4 + 2;
shared FLOAT_TYPEV4 Qf[Br * qstride];

const uint psh_stride = Br / 4 + 2;
shared FLOAT_TYPEV4 Psh[Bc * psh_stride];

// Avoid padding for hsk==256 to make it fit in 48KB shmem.
const uint32_t sfshstride = (HSK <= 128) ? (Br / 4 + 2) : Br / 4;
shared ACC_TYPEV4 sfsh[Bc * sfshstride];

const uint32_t D_pad = HSK_pad > HSV_pad ? HSK_pad : HSV_pad;
const uint32_t kvsh_stride = (SHMEM_STAGING != 0 ? D_pad : MatBr) / 4 + 2;
const uint v_cols = MatBc / 4 * row_split; // total cols, 4 vec4s per MatBc * number of subgroups
const uint vsh_stride = v_cols;
shared FLOAT_TYPEV4 kvsh[(kvsh_stride >= vsh_stride) ? (Bc * kvsh_stride) : (Bc * vsh_stride)];

const uint32_t osh_stride = row_split * MatBr / 4;
shared O_TYPEV4 pvsh[MatBc * osh_stride];

shared ACC_TYPE slope[Br];

void main() {
#ifdef NEEDS_INIT_IQ_SHMEM
    if (fa_type_needs_shmem(FaTypeK) || fa_type_needs_shmem(FaTypeV)) {
        init_iq_shmem(gl_WorkGroupSize);
    }
#endif

    init_indices();

    const uint32_t tid = gl_LocalInvocationIndex;

    const uint32_t threads_per_rowgroup = gl_WorkGroupSize.x / row_split;
    const uint32_t d_per_thread = (HSV/4 + threads_per_rowgroup - 1) / threads_per_rowgroup;
    const uint32_t row_tid = gl_LocalInvocationIndex / threads_per_rowgroup;
    const uint32_t col_tid = gl_LocalInvocationIndex % threads_per_rowgroup;

#define tile_row(r) (row_tid * rows_per_thread + (r))

    // Zero-initialize shared memory for Q/K when HSK is not a multiple of 16 (HSK_pad > HSK).
    if ((HSK % 16) != 0) {
        [[unroll]] for (uint i = 0; i < Br * qstride; i += gl_WorkGroupSize.x) {
            if (i + tid < Br * qstride) {
                Qf[i + tid] = FLOAT_TYPEV4(0);
            }
        }
        barrier();
    }

    uint32_t q_offset = gqa_iq1*p.nb01 + (iq2*p.nb02+iq3*p.nb03) / 4;

    [[unroll]] for (uint32_t idx = 0; idx < Br * HSK / 4; idx += gl_WorkGroupSize.x) {
        uint32_t d = (idx + tid) % (HSK / 4);
        uint32_t r = (idx + tid) / (HSK / 4);
        if (r < Br && d < HSK / 4 &&
            i * Br + r < N) {
            Qf[r * qstride + d] = FLOAT_TYPEV4(data_qv4[q_offset / 4 + (i * Br + r) * q_stride / 4 + d] * p.scale);
        }
    }
    barrier();

    O_TYPEV4 Of[rows_per_thread][d_per_thread];
    [[unroll]] for (uint32_t r = 0; r < rows_per_thread; ++r) {
        [[unroll]] for (uint32_t d = 0; d < d_per_thread; ++d) {
            Of[r][d] = O_TYPEV4(0.0);
        }
    }

    float Lf[rows_per_thread], Mf[rows_per_thread];

    // Use -FLT_MAX/2 rather than -inf to reduce the possibility of NaNs, e.g. when computing Mold-M.
    const float NEG_FLT_MAX_OVER_2 = uintBitsToFloat(0xFEFFFFFF);

    [[unroll]] for (uint32_t r = 0; r < rows_per_thread; ++r) {
        Lf[r] = 0;
        Mf[r] = NEG_FLT_MAX_OVER_2;
    }

    // ALiBi
    if (p.max_bias > 0.0f) {
        if (tid < Br) {
            uint r = tid;
            slope[r] = perElemOpComputeSlope(r, col_tid, ACC_TYPE(0), iq2);
        }
    } else {
        if (tid < Br) {
            uint r = tid;
            slope[r] = ACC_TYPE(1.0);
        }
    }

    const uint32_t mo_stride = CEIL_DIV(KV, 16 * Bc);
    // mo_offset will point to the tile starting at row i*Br and col 0
    uint32_t mo_offset = mo_stride * i;

    // FaBlockBytesK/V == 2 for f16 (sizeof f16) and == 16 for f32 (vec4) and == ggml block size for quants.
    uint32_t k_offset = (ik2*p.nb12 + ik3*p.nb13) / FaBlockBytesK;
    uint32_t v_offset = (iv2*p.nb22 + iv3*p.nb23) / FaBlockBytesV;
    uint32_t m_offset = gqa_iq1*KV;
    if (p.nem2 != 1 || p.nem3 != 1) {
        m_offset += ((iq3 % p.nem3) * p.nem2 + (iq2 % p.nem2)) * p.nem1 * KV;
        mo_offset += ((iq3 % p.nem3) * p.nem2 + (iq2 % p.nem2)) * CEIL_DIV(p.nem1, Br) * mo_stride;
    }

    uint32_t mask_opt = 0;
    uint32_t mask_opt_idx = ~0;
    uint32_t mask_opt_bits = 0;
    f16vec4 mask_cache[Bc * Br / 4 / WorkGroupSize];

    [[dont_unroll]]
    for (uint32_t j = start_j; j < end_j; ++j) {

        [[unroll]] for (uint32_t idx = 0; idx < mask_cache.length(); ++idx) {
            mask_cache[idx] = f16vec4(0);
        }

        if (MASK_ENABLE) {
            if (USE_MASK_OPT && mask_opt_idx != j / 16) {
                mask_opt_idx = j / 16;
                mask_opt = data_mask_opt[mo_offset + mask_opt_idx];
            }
            mask_opt_bits = (mask_opt >> ((j % 16) * 2)) & 0x3;
            if (mask_opt_bits == MASK_OPT_ALL_NEG_INF) {
                // skip this block
                continue;
            }
            // Only load if the block is not all zeros
            if (mask_opt_bits != MASK_OPT_ALL_ZERO) {
                bool nem1_bounds_check = !(p.gqa_ratio > 1) && (p.nem1 % Br) != 0;

                float max_mask = NEG_FLT_MAX_OVER_2;
                [[unroll]] for (uint32_t idx = 0; idx < Bc * Br / 4; idx += gl_WorkGroupSize.x) {
                    uint32_t c = (idx + tid) / (Br / 4);
                    uint32_t r = (idx + tid) % (Br / 4);
                    if (idx + tid < Bc * Br / 4 || idx + gl_WorkGroupSize.x <= Bc * Br / 4) {
                        if ((!KV_bounds_check || j * Bc + c < KV)) {
                            f16vec4 m;
                            if (!nem1_bounds_check || i * Br + r * 4 + 3 < p.nem1) {
                                m = f16vec4(data_m[m_offset + (i * Br + r * 4    ) * m_stride + (j * Bc + c)],
                                            data_m[m_offset + (i * Br + r * 4 + 1) * m_stride + (j * Bc + c)],
                                            data_m[m_offset + (i * Br + r * 4 + 2) * m_stride + (j * Bc + c)],
                                            data_m[m_offset + (i * Br + r * 4 + 3) * m_stride + (j * Bc + c)]);
                                max_mask = max(max(max(max(max_mask, float(m[0])), float(m[1])), float(m[2])), float(m[3]));
                            } else if (i * Br + r * 4 + 2 < p.nem1) {
                                m = f16vec4(data_m[m_offset + (i * Br + r * 4    ) * m_stride + (j * Bc + c)],
                                            data_m[m_offset + (i * Br + r * 4 + 1) * m_stride + (j * Bc + c)],
                                            data_m[m_offset + (i * Br + r * 4 + 2) * m_stride + (j * Bc + c)],
                                            0.0);
                                max_mask = max(max(max(max_mask, float(m[0])), float(m[1])), float(m[2]));
                            } else if (i * Br + r * 4 + 1 < p.nem1) {
                                m = f16vec4(data_m[m_offset + (i * Br + r * 4    ) * m_stride + (j * Bc + c)],
                                            data_m[m_offset + (i * Br + r * 4 + 1) * m_stride + (j * Bc + c)],
                                            0.0,
                                            0.0);
                                max_mask = max(max(max_mask, float(m[0])), float(m[1]));
                            } else if (i * Br + r * 4 < p.nem1) {
                                m = f16vec4(data_m[m_offset + (i * Br + r * 4    ) * m_stride + (j * Bc + c)],
                                            0.0,
                                            0.0,
                                            0.0);
                                max_mask = max(max_mask, float(m[0]));
                            } else {
                                m = f16vec4(0.0);
                            }
                            mask_cache[idx / WorkGroupSize] = m;
                        }
                    }
                }
                // skip the block if the mask is entirely -inf
                bool all_less = subgroupAll(max_mask <= NEG_FLT_MAX_OVER_2);
                barrier();
                if (gl_SubgroupInvocationID == 0) {
                    tmpsh[gl_SubgroupID] = all_less ? NEG_FLT_MAX_OVER_2 : 0.0f;
                }
                barrier();
                [[unroll]] for (uint s = 0; s < gl_NumSubgroups; ++s) {
                    max_mask = max(max_mask, tmpsh[s]);
                }
                if (max_mask <= NEG_FLT_MAX_OVER_2) {
                    continue;
                }
            }
        }

        if (SHMEM_STAGING != 0) {
            [[unroll]] for (uint32_t idx = 0; idx < Bc * HSK_pad / 4; idx += gl_WorkGroupSize.x) {
                uint32_t d = (idx + tid) % (HSK_pad / 4);
                uint32_t c = (idx + tid) / (HSK_pad / 4);
                if (idx + gl_WorkGroupSize.x <= Bc * HSK_pad / 4 || c < Bc) {
                    FLOAT_TYPEV4 K_Tf = FLOAT_TYPEV4(0);
                    if ((!KV_bounds_check || j * Bc + c < KV) && (HSK == HSK_pad || d < HSK / 4)) {
#if !defined(BFLOAT16)
                        if (USE_DECODE_K) {
                            uint coord = (j * Bc + c) * k_stride * BLOCK_SIZE_K + 4 * d;
                            uint ib = coord / BLOCK_SIZE_K;
                            uint iqs = (coord % BLOCK_SIZE_K);
                            K_Tf = dequantize4(ib, iqs, k_offset, BINDING_IDX_K);
                        } else
#endif
                        {
                            K_Tf = FLOAT_TYPEV4(data_kv4[k_offset / 4 + (j * Bc + c) * k_stride / 4 + d]);
                        }
                    }

                    kvsh[c * kvsh_stride + d] = K_Tf;
                }
            }
            barrier();
        }

        // K * Q^T -> S^T: Bc x HSK_pad * HSK_pad x Br -> Bc x Br
        // Bc split across workgroup (four subgroups), loop over HSK in chunks of 16: 16 x 16 * 16 x 16 -> 16 x 16
        // This is written transposed in order to allow for N being 8 if implementations need it
        coopmat<ACC_TYPE, gl_ScopeSubgroup, MatBc, MatBr, gl_MatrixUseAccumulator> SfMat = coopmat<ACC_TYPE, gl_ScopeSubgroup, MatBc, MatBr, gl_MatrixUseAccumulator>(0);
        coopmat<FLOAT_TYPE, gl_ScopeSubgroup, MatBc, 16, gl_MatrixUseA> KMat;
        coopmat<FLOAT_TYPE, gl_ScopeSubgroup, 16, MatBr, gl_MatrixUseB> QMat;

        [[unroll]] for (uint32_t d = 0; d < HSK_pad / 16; ++d) {
            // If SHMEM_STAGING is set, a Bc * HSK_pad size tile of K is loaded to shmem
            // If not, K is loaded directly from global memory if aligned, otherwise
            // staged through a Bc * MatBr size staging buffer.
            // If K is a quant type, then it is always staged for dequantization.
            if (SHMEM_STAGING == 0) {
            // For quants we always need to dequant into kvsh; for f16/bf16 we can load
            // directly from global memory when alignment / bounds allow it.
            const bool stage_k = USE_DECODE_K || KV_bounds_check || d * 16 + 16 > HSK;
            if (stage_k) {
                barrier();
                [[unroll]] for (uint32_t idx = 0; idx < Bc * MatBr / 4; idx += gl_WorkGroupSize.x) {
                    uint32_t col_vec = (idx + tid) % (MatBr / 4);
                    uint32_t row = (idx + tid) / (MatBr / 4);
                    if (idx + tid < Bc * MatBr / 4) {
                        FLOAT_TYPEV4 K_Tf = FLOAT_TYPEV4(0);
                        if ((!KV_bounds_check || j * Bc + row < KV) && (HSK == HSK_pad || d * 16 + col_vec * 4 < HSK)) {
#if !defined(BFLOAT16)
                            if (USE_DECODE_K) {
                                uint coord = (j * Bc + row) * k_stride * BLOCK_SIZE_K + d * 16 + col_vec * 4;
                                uint ib = coord / BLOCK_SIZE_K;
                                uint iqs = (coord % BLOCK_SIZE_K);
                                K_Tf = dequantize4(ib, iqs, k_offset, BINDING_IDX_K);
                            } else
#endif
                            {
                                K_Tf = FLOAT_TYPEV4(data_kv4[k_offset / 4 + (j * Bc + row) * k_stride / 4 + d * 16 / 4 + col_vec]);
                            }
                        }

                        kvsh[row * kvsh_stride + col_vec] = K_Tf;
                    }
                }
                barrier();
            }

            if (stage_k) {
                uint coord = (gl_SubgroupID * MatBc) * kvsh_stride;
                coopMatLoad(KMat, kvsh, coord, kvsh_stride, gl_CooperativeMatrixLayoutRowMajor);
            } else {
                const uint coord = k_offset / 4 + (j * Bc + gl_SubgroupID * MatBc) * k_stride / 4 + d * 16 / 4;
                coopMatLoad(KMat, data_kv4, coord, k_stride / 4, gl_CooperativeMatrixLayoutRowMajor);
            }
            } else {
                uint coord = (gl_SubgroupID * MatBc) * kvsh_stride + d * 16 / 4;
                coopMatLoad(KMat, kvsh, coord, kvsh_stride, gl_CooperativeMatrixLayoutRowMajor);
            }

            coopMatLoad(QMat, Qf, d * 16 / 4, qstride, gl_CooperativeMatrixLayoutColumnMajor);

            SfMat = coopMatMulAdd(KMat, QMat, SfMat);
        }

        uint coord = gl_SubgroupID * MatBc * sfshstride;
        coopMatStore(SfMat, sfsh, coord, sfshstride, gl_CooperativeMatrixLayoutRowMajor);
        barrier();

        if (LOGIT_SOFTCAP) {
            [[unroll]] for (uint32_t idx = 0; idx < Bc * Br / 4; idx += gl_WorkGroupSize.x) {
                uint32_t c = (idx + tid) / (Br / 4);
                uint32_t r = (idx + tid) % (Br / 4);
                if (idx + tid < Bc * Br / 4 || idx + gl_WorkGroupSize.x <= Bc * Br / 4) {
                    sfsh[c * sfshstride + r] = ACC_TYPEV4(p.logit_softcap * tanh(sfsh[c * sfshstride + r]));
                }
            }
            barrier();
        }

        if (MASK_ENABLE && mask_opt_bits != MASK_OPT_ALL_ZERO) {
            [[unroll]] for (uint32_t idx = 0; idx < Bc * Br / 4; idx += gl_WorkGroupSize.x) {
                uint32_t c = (idx + tid) / (Br / 4);
                uint32_t r = (idx + tid) % (Br / 4);
                if (idx + tid < Bc * Br / 4 || idx + gl_WorkGroupSize.x <= Bc * Br / 4) {
                    if (!KV_bounds_check || j * Bc + c < KV) {
                        // Mask nem1 bounds check is handled when loading masks
                        ACC_TYPEV4 masks = ACC_TYPEV4(mask_cache[idx / WorkGroupSize]);
                        ACC_TYPEV4 slopes = ACC_TYPEV4(slope[r * 4], slope[r * 4 + 1], slope[r * 4 + 2], slope[r * 4 + 3]);
                        sfsh[c * sfshstride + r] += slopes * masks;
                    }
                }
            }
            barrier();
        }

        float eMf[rows_per_thread];
        [[unroll]] for (uint32_t r = 0; r < rows_per_thread; ++r) {
            const uint r_vec  = tile_row(r) / 4;
            const uint r_comp = tile_row(r) % 4;

            float rowmaxf = NEG_FLT_MAX_OVER_2;
            [[unroll]] for (uint32_t c = 0; c < cols_per_thread; ++c) {
                if (KV_bounds_check && j * Bc + c * cols_per_iter + col_tid >= KV) {
                    continue;
                }
                rowmaxf = max(rowmaxf, float(sfsh[r_vec + (c * cols_per_iter + col_tid) * sfshstride][r_comp]));
            }
            rowmaxf += FATTN_KQ_MAX_OFFSET;
            float Moldf = Mf[r];

            // Compute max across the row
            rowmaxf = subgroupMax(rowmaxf);

            // M = max(rowmax, Mold)
            // P = e^(S - M)
            // eM = e^(Mold - M)
            Mf[r] = max(rowmaxf, Moldf);
            eMf[r] = exp(Moldf - Mf[r]);

            Lf[r] = eMf[r]*Lf[r];
        }

        [[unroll]] for (uint32_t d0 = 0; d0 < HSV / 4; d0 += threads_per_rowgroup) {
            const uint d_local = d0 / threads_per_rowgroup;
            [[unroll]] for (uint32_t r = 0; r < rows_per_thread; ++r) {
                Of[r][d_local] = O_TYPE(eMf[r]) * Of[r][d_local];
            }
        }

        // Calculate and store Pf in Psh
        [[unroll]] for (uint32_t c = 0; c < cols_per_thread; ++c) {
            const uint col = c * cols_per_iter + col_tid;

            [[unroll]] for (uint32_t r = 0; r < rows_per_thread; r += 4) {
                const uint row = tile_row(r);
                if (KV_bounds_check && j * Bc + col >= KV) {
                    Psh[col * psh_stride + row / 4] = FLOAT_TYPEV4(0.0f);
                } else {
                    const vec4 mfvec = vec4(Mf[r], Mf[r + 1], Mf[r + 2], Mf[r + 3]);
                    const FLOAT_TYPEV4 Pf = FLOAT_TYPEV4(exp(vec4(sfsh[row / 4 + col * sfshstride]) - mfvec));
                    [[unroll]] for (uint32_t vec_idx = 0; vec_idx < 4; ++vec_idx) {
                        Lf[r + vec_idx] += Pf[vec_idx];
                    }
                    Psh[col * psh_stride + row / 4] = Pf;
                }
            }
        }

        if (SHMEM_STAGING != 0) {
            [[unroll]] for (uint32_t idx = 0; idx < Bc * HSV_pad / 4; idx += gl_WorkGroupSize.x) {
                uint32_t d = (idx + tid) % (HSV_pad / 4);
                uint32_t c = (idx + tid) / (HSV_pad / 4);
                if (idx + gl_WorkGroupSize.x <= Bc * HSV_pad / 4 || c < Bc) {
                    FLOAT_TYPEV4 V_Tf = FLOAT_TYPEV4(0);
                    if ((!KV_bounds_check || j * Bc + c < KV) && (HSV == HSV_pad || d < HSV / 4)) {
#if !defined(BFLOAT16)
                        if (USE_DECODE_V) {
                            uint coord = (j * Bc + c) * v_stride * BLOCK_SIZE_V + 4 * d;
                            uint ib = coord / BLOCK_SIZE_V;
                            uint iqs = (coord % BLOCK_SIZE_V);
                            V_Tf = dequantize4(ib, iqs, v_offset, BINDING_IDX_V);
                        } else
#endif
                        {
                            V_Tf = FLOAT_TYPEV4(data_vv4[v_offset / 4 + (j * Bc + c) * v_stride / 4 + d]);
                        }
                    }

                    kvsh[c * kvsh_stride + d] = V_Tf;
                }
            }
        }
        barrier();

        const uint num_hsv_tiles = (HSV + MatBc * row_split - 1) / (MatBc * row_split); // round up

        // Each subgroup handles HSV/4 columns
        [[unroll]] for (uint32_t hsv_tile = 0; hsv_tile < num_hsv_tiles; ++hsv_tile) {
            const uint hsv_offset = (hsv_tile * row_split + gl_SubgroupID) * 16;

            coopmat<O_TYPE, gl_ScopeSubgroup, MatBc, MatBr, gl_MatrixUseAccumulator> PVMat = coopmat<O_TYPE, gl_ScopeSubgroup, MatBc, MatBr, gl_MatrixUseAccumulator>(0);

            // Preload V tiles for [Bc, 16 * num subgroups]
            const uint v_rows = Bc;
            const uint v_total = v_rows * v_cols;
            const uint v_loads_per_thread = v_total / gl_WorkGroupSize.x;

            // If SHMEM_STAGING is set, a Bc * HSV_pad size tile of V is loaded to shmem.
            // If not, V is loaded directly from global memory if aligned, otherwise
            // staged through a Bc * MatBr size staging buffer.
            // If V is a quant type, then it is always staged for dequantization.
            if (SHMEM_STAGING == 0) {
            // For quants we always preload via kvsh. For f16/bf16 we only preload when
            // alignment / bounds force it (otherwise we coopMatLoad direct from data_vv4).
            const bool stage_v = USE_DECODE_V || KV_bounds_check;
            if (stage_v) {
                [[unroll]] for (uint32_t i = 0; i < v_loads_per_thread; ++i) {
                    const uint idx = i * gl_WorkGroupSize.x + tid;
                    const uint row = idx / v_cols;
                    const uint col = idx % v_cols;

                    const uint v_row = j * Bc + row;
                    const uint v_col = hsv_tile * MatBc * row_split + col * 4;

                    const uint coord = v_row * v_stride * BLOCK_SIZE_V + v_col;
                    const uint ib = coord / BLOCK_SIZE_V;
                    const uint iqs = coord % BLOCK_SIZE_V;

                    if (!KV_bounds_check || (v_row < KV && v_col < HSV)) {
#if !defined(BFLOAT16)
                        if (USE_DECODE_V) {
                            kvsh[row * vsh_stride + col] = dequantize4(ib, iqs, v_offset, BINDING_IDX_V);
                        } else
#endif
                        {
                            kvsh[row * vsh_stride + col] = data_vv4[(v_offset + v_row * v_stride + v_col) / 4];
                        }
                    } else {
                        kvsh[row * vsh_stride + col] = FLOAT_TYPEV4(0.0f);
                    }
                }
            }
            }
            barrier();

            const uint o_offset = gl_SubgroupID * MatBr / 4;

            if (hsv_offset < HSV_pad) {
                [[unroll]] for (uint32_t bc_chunk = 0; bc_chunk < Bc / MatBc; ++bc_chunk) {
                    coopMatLoad(KMat, Psh, bc_chunk * MatBc * psh_stride, psh_stride, gl_CooperativeMatrixLayoutColumnMajor);

                    if (SHMEM_STAGING == 0) {
                    if (!USE_DECODE_V && !KV_bounds_check) {
                        // F16/BF16 values can be loaded directly from global memory
                        const uint v_tile_row = j * Bc + bc_chunk * MatBc;
                        const uint v_tile_offset = v_offset / 4 + v_tile_row * v_stride / 4 + hsv_offset / 4;
                        coopMatLoad(QMat, data_vv4, v_tile_offset, v_stride / 4, gl_CooperativeMatrixLayoutRowMajor);
                    } else {
                        const uint v_tile_offset = bc_chunk * MatBr * v_cols + gl_SubgroupID * (MatBc / 4);
                        coopMatLoad(QMat, kvsh, v_tile_offset, vsh_stride, gl_CooperativeMatrixLayoutRowMajor);
                    }
                    } else {
                        const uint v_tile_offset = bc_chunk * MatBc * kvsh_stride + (hsv_tile * row_split + gl_SubgroupID) * (MatBc / 4);
                        coopMatLoad(QMat, kvsh, v_tile_offset, kvsh_stride, gl_CooperativeMatrixLayoutRowMajor);
                    }

                    PVMat = coopMatMulAdd(KMat, QMat, PVMat);
                }

                // Store PVMat to pvsh and load into Of
                coopMatStore(PVMat, pvsh, o_offset, osh_stride, gl_CooperativeMatrixLayoutRowMajor);
            }

            barrier();

            const uint hsv_per_tile = row_split * MatBc;
            const uint hsv_base = hsv_tile * hsv_per_tile;
            const uint d_values_per_tile = hsv_per_tile / 4;

            const uint d_start = hsv_tile * d_values_per_tile;
            const uint d_end = min(d_start + d_values_per_tile, HSV / 4);

            [[unroll]] for (uint32_t r = 0; r < rows_per_thread; ++r) {
                const uint row = tile_row(r);

                [[unroll]] for (uint32_t d_local = 0; d_local < d_per_thread; ++d_local) {
                    const uint d = d_local * threads_per_rowgroup + col_tid;
                    const uint hsv_col = 4 * d;

                    if (hsv_col >= hsv_base && hsv_col < hsv_base + hsv_per_tile && hsv_col < HSV) {
                        const uint local_hsv = (hsv_col - hsv_base) / 4;
                        Of[r][d_local] += pvsh[row * osh_stride + local_hsv];
                    }
                }
            }
        }

        barrier();
    }

    [[unroll]] for (uint32_t r = 0; r < rows_per_thread; ++r) {
        Lf[r] = subgroupAdd(Lf[r]);
    }

    // If there is split_k, then the split_k resolve shader does the final
    // division by L. Store the intermediate O value and per-row m and L values.
    if (p.k_num > 1) {
        if (p.gqa_ratio > 1) {
            // note: O and Q have swapped coord 1,2.
            uint32_t o_offset = HSV * p.ne1 * (split_k_index + p.k_num * (gqa_iq1 + p.ne2 * iq3)) / 4;

            [[unroll]] for (uint32_t r = 0; r < rows_per_thread; ++r) {
                if (tile_row(r) < N) {
                    [[unroll]] for (uint32_t d0 = 0; d0 < HSV / 4; d0 += threads_per_rowgroup) {
                        const uint d = d0 + col_tid;
                        if (d >= HSV/4) break;
                        const uint d_local = d0 / threads_per_rowgroup;
                        gqaStore(tile_row(r), d, Of[r][d_local], o_offset, iq2, N);
                    }
                }
            }

            o_offset = HSV * p.ne1 * p.k_num * p.ne2 * p.ne3 + p.ne1 * 2 * (split_k_index + p.k_num * (gqa_iq1 + p.ne2 * iq3));
            [[unroll]] for (uint32_t r = 0; r < rows_per_thread; ++r) {
                if (tile_row(r) < N) {
                    perElemOpStoreCol0(tile_row(r), 0u, ACC_TYPE(Lf[r]), o_offset, iq2, N);
                    perElemOpStoreCol0(tile_row(r), 0u, ACC_TYPE(Mf[r]), o_offset + p.ne1, iq2, N);
                }
            }
        } else {
            [[unroll]] for (uint32_t r = 0; r < rows_per_thread; ++r) {
                const uint row = tile_row(r);
                const uint global_row = i * Br + row;

                if (global_row < N) {
                    uint32_t o_offset = HSV * p.ne1 * (split_k_index + p.k_num * (global_row + p.ne2 * iq3)) / 4;

                    [[unroll]] for (uint32_t d0 = 0; d0 < HSV / 4; d0 += threads_per_rowgroup) {
                        const uint d = d0 + col_tid;
                        if (d >= HSV/4) break;
                        data_ov4[o_offset + iq2 * HSV/4 + d] = D_TYPEV4(Of[r][d/threads_per_rowgroup]);
                    }
                }

                if (global_row < N && col_tid == 0) {
                    uint32_t lm_offset = HSV * p.ne1 * p.k_num * p.ne2 * p.ne3 + p.ne1 * 2 * (split_k_index + p.k_num * (global_row + p.ne2 * iq3));
                    data_o[lm_offset + iq2] = D_TYPE(Lf[r]);
                    data_o[lm_offset + p.ne1 + iq2] = D_TYPE(Mf[r]);
                }
            }
        }

        return;
    }

    if ((p.mask_n_head_log2 & SINK_ENABLE_BIT) != 0) {
        [[unroll]] for (uint32_t r = 0; r < rows_per_thread; ++r) {
            float sink = perElemOpGetSink(tile_row(r), 0u, ACC_TYPE(0), iq2);

            float ms = 1.0f;
            float vs = 1.0f;

            if (sink > Mf[r]) {
                ms = exp(Mf[r] - sink);

                [[unroll]] for (uint32_t d0 = 0; d0 < HSV / 4; d0 += threads_per_rowgroup) {
                    const uint d_local = d0 / threads_per_rowgroup;
                    Of[r][d_local] *= O_TYPE(ms);
                }
            } else {
                vs = exp(sink - Mf[r]);
            }

            Lf[r] = Lf[r]*ms + vs;
        }
    }

    float Lfrcp[rows_per_thread];
    [[unroll]] for (uint32_t r = 0; r < rows_per_thread; ++r) {
        Lfrcp[r] = (Lf[r] == 0.0) ? 0.0 : (1.0 / Lf[r]);
    }

    [[unroll]] for (uint32_t d0 = 0; d0 < HSV / 4; d0 += threads_per_rowgroup) {
        const uint d_local = d0 / threads_per_rowgroup;
        [[unroll]] for (uint32_t r = 0; r < rows_per_thread; ++r) {
            Of[r][d_local] *= O_TYPE(Lfrcp[r]);
#if defined(FLOAT_TYPE_MAX)
            Of[r][d_local] = clamp(Of[r][d_local], -FLOAT_TYPE_MAX, FLOAT_TYPE_MAX);
#endif
        }
    }

    uint32_t o_offset = (gqa_iq1*p.ne1*HSV + iq3*p.ne2*p.ne1*HSV) / 4;

    if (p.gqa_ratio > 1) {
        [[unroll]] for (uint32_t r = 0; r < rows_per_thread; ++r) {
            if (tile_row(r) < N) {
                [[unroll]] for (uint32_t d0 = 0; d0 < HSV / 4; d0 += threads_per_rowgroup) {
                    const uint d = d0 + col_tid;
                    if (d >= HSV / 4) break;
                    const uint d_local = d0 / threads_per_rowgroup;
                    gqaStore(tile_row(r), d, Of[r][d_local], o_offset, iq2, N);
                }
            }
        }
    } else {
        [[unroll]] for (uint32_t r = 0; r < rows_per_thread; ++r) {
            if (i * Br + tile_row(r) < N) {
                [[unroll]] for (uint32_t d0 = 0; d0 < HSV / 4; d0 += threads_per_rowgroup) {
                    const uint d = d0 + col_tid;
                    if (d >= HSV / 4) break;
                    const uint d_local = d0 / threads_per_rowgroup;
                    data_ov4[o_offset + (iq2 * HSV + (i * Br + tile_row(r)) * p.ne1 * HSV) / 4 + d] = D_TYPEV4(Of[r][d_local]);
                }
            }
        }
    }
}