// // MIT license // Copyright (C) 2025 Intel Corporation // SPDX-License-Identifier: MIT // // // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. // See https://llvm.org/LICENSE.txt for license information. // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // #include #include "dpct/helper.hpp" #include "common.hpp" #include "fattn-common.hpp" #include "fattn-tile.hpp" #include "fattn-vec.hpp" #include "fattn.hpp" #include "fattn-onednn.hpp" #define FATTN_VEC_CASE(D, type_K, type_V) \ { \ const bool type_K_okay = K->type == (type_K) || (K->type == GGML_TYPE_F32 && (type_K) == GGML_TYPE_F16); \ const bool type_V_okay = V->type == (type_V) || (V->type == GGML_TYPE_F32 && (type_V) == GGML_TYPE_F16); \ if (Q->ne[0] == (D) && type_K_okay && type_V_okay) { \ ggml_sycl_flash_attn_ext_vec_case(ctx, dst); \ return; \ } \ } \ #define FATTN_VEC_CASES_ALL_D(type_K, type_V) \ FATTN_VEC_CASE( 64, type_K, type_V) \ FATTN_VEC_CASE(128, type_K, type_V) \ FATTN_VEC_CASE(256, type_K, type_V) \ FATTN_VEC_CASE(512, type_K, type_V) \ static void ggml_sycl_flash_attn_ext_vec(ggml_backend_sycl_context & ctx, ggml_tensor * dst) { ggml_tensor * Q = dst->src[0]; ggml_tensor * K = dst->src[1]; ggml_tensor * V = dst->src[2]; #ifdef GGML_SYCL_FA_ALL_QUANTS FATTN_VEC_CASES_ALL_D(GGML_TYPE_F16, GGML_TYPE_F16) FATTN_VEC_CASES_ALL_D(GGML_TYPE_Q4_0, GGML_TYPE_F16) FATTN_VEC_CASES_ALL_D(GGML_TYPE_Q4_1, GGML_TYPE_F16) FATTN_VEC_CASES_ALL_D(GGML_TYPE_Q5_0, GGML_TYPE_F16) FATTN_VEC_CASES_ALL_D(GGML_TYPE_Q5_1, GGML_TYPE_F16) FATTN_VEC_CASES_ALL_D(GGML_TYPE_Q8_0, GGML_TYPE_F16) FATTN_VEC_CASES_ALL_D(GGML_TYPE_F16, GGML_TYPE_Q4_0) FATTN_VEC_CASES_ALL_D(GGML_TYPE_Q4_0, GGML_TYPE_Q4_0) FATTN_VEC_CASES_ALL_D(GGML_TYPE_Q4_1, GGML_TYPE_Q4_0) FATTN_VEC_CASES_ALL_D(GGML_TYPE_Q5_0, GGML_TYPE_Q4_0) FATTN_VEC_CASES_ALL_D(GGML_TYPE_Q5_1, GGML_TYPE_Q4_0) FATTN_VEC_CASES_ALL_D(GGML_TYPE_Q8_0, GGML_TYPE_Q4_0) FATTN_VEC_CASES_ALL_D(GGML_TYPE_F16, GGML_TYPE_Q4_1) FATTN_VEC_CASES_ALL_D(GGML_TYPE_Q4_0, GGML_TYPE_Q4_1) FATTN_VEC_CASES_ALL_D(GGML_TYPE_Q4_1, GGML_TYPE_Q4_1) FATTN_VEC_CASES_ALL_D(GGML_TYPE_Q5_0, GGML_TYPE_Q4_1) FATTN_VEC_CASES_ALL_D(GGML_TYPE_Q5_1, GGML_TYPE_Q4_1) FATTN_VEC_CASES_ALL_D(GGML_TYPE_Q8_0, GGML_TYPE_Q4_1) FATTN_VEC_CASES_ALL_D(GGML_TYPE_F16, GGML_TYPE_Q5_0) FATTN_VEC_CASES_ALL_D(GGML_TYPE_Q4_0, GGML_TYPE_Q5_0) FATTN_VEC_CASES_ALL_D(GGML_TYPE_Q4_1, GGML_TYPE_Q5_0) FATTN_VEC_CASES_ALL_D(GGML_TYPE_Q5_0, GGML_TYPE_Q5_0) FATTN_VEC_CASES_ALL_D(GGML_TYPE_Q5_1, GGML_TYPE_Q5_0) FATTN_VEC_CASES_ALL_D(GGML_TYPE_Q8_0, GGML_TYPE_Q5_0) FATTN_VEC_CASES_ALL_D(GGML_TYPE_F16, GGML_TYPE_Q5_1) FATTN_VEC_CASES_ALL_D(GGML_TYPE_Q4_0, GGML_TYPE_Q5_1) FATTN_VEC_CASES_ALL_D(GGML_TYPE_Q4_1, GGML_TYPE_Q5_1) FATTN_VEC_CASES_ALL_D(GGML_TYPE_Q5_0, GGML_TYPE_Q5_1) FATTN_VEC_CASES_ALL_D(GGML_TYPE_Q5_1, GGML_TYPE_Q5_1) FATTN_VEC_CASES_ALL_D(GGML_TYPE_Q8_0, GGML_TYPE_Q5_1) FATTN_VEC_CASES_ALL_D(GGML_TYPE_F16, GGML_TYPE_Q8_0) FATTN_VEC_CASES_ALL_D(GGML_TYPE_Q4_0, GGML_TYPE_Q8_0) FATTN_VEC_CASES_ALL_D(GGML_TYPE_Q4_1, GGML_TYPE_Q8_0) FATTN_VEC_CASES_ALL_D(GGML_TYPE_Q5_0, GGML_TYPE_Q8_0) FATTN_VEC_CASES_ALL_D(GGML_TYPE_Q5_1, GGML_TYPE_Q8_0) FATTN_VEC_CASES_ALL_D(GGML_TYPE_Q8_0, GGML_TYPE_Q8_0) #else FATTN_VEC_CASES_ALL_D(GGML_TYPE_F16, GGML_TYPE_F16) FATTN_VEC_CASES_ALL_D(GGML_TYPE_Q4_0, GGML_TYPE_Q4_0) FATTN_VEC_CASES_ALL_D(GGML_TYPE_Q8_0, GGML_TYPE_Q8_0) #endif // GGML_SYCL_FA_ALL_QUANTS GGML_ABORT("Not match KV type in vec"); } // Best FlashAttention kernel for a specific GPU: enum best_fattn_kernel { BEST_FATTN_KERNEL_NONE = 0, BEST_FATTN_KERNEL_VEC = 100, BEST_FATTN_KERNEL_ONEDNN = 150, // oneDNN SDPA: native F16 (PR #25222) BEST_FATTN_KERNEL_TILE = 200, BEST_FATTN_KERNEL_MKL = 300, }; static best_fattn_kernel ggml_sycl_get_best_fattn_kernel(const int device, const ggml_tensor * dst) { GGML_UNUSED(device); #ifndef SYCL_FLASH_ATTN GGML_UNUSED(dst); return BEST_FATTN_KERNEL_NONE; #endif// SYCL_FLASH_ATTN if(!g_ggml_sycl_enable_flash_attention) return BEST_FATTN_KERNEL_NONE; const ggml_tensor * KQV = dst; const ggml_tensor * Q = dst->src[0]; const ggml_tensor * K = dst->src[1]; const ggml_tensor * V = dst->src[2]; const ggml_tensor * mask = dst->src[3]; const ggml_tensor * sinks = dst->src[4]; const int gqa_ratio = Q->ne[2] / K->ne[2]; GGML_ASSERT(Q->ne[2] % K->ne[2] == 0); float max_bias = 0.0f; memcpy(&max_bias, (const float *) KQV->op_params + 1, sizeof(float)); float logit_softcap = 0.0f; memcpy(&logit_softcap, (const float *) KQV->op_params + 2, sizeof(float)); bool gqa_opt_applies = gqa_ratio >= 2 && mask && max_bias == 0.0f && K->ne[1] % FATTN_KQ_STRIDE == 0; // XMX-accelerated path: oneDNN SDPA (native F16 and dequant+non-F16). // ONEDNN requires min 32 query tokens — short-circuit decode to avoid // calling _supported() on every decode FA call. if (Q->ne[1] >= 32 && ggml_sycl_flash_attn_ext_onednn_supported(dst)) { return BEST_FATTN_KERNEL_ONEDNN; } // MKL path: XMX-accelerated GEMM for prompt processing (all KV cache types). // The MKL kernel converts non-F16 K/V to F16 via to_fp16_sycl before GEMM, // so quantized, F16, BF16, and F32 caches all benefit from XMX acceleration. // Activates automatically when flash-attn is enabled (--flash-attn on or -fa) // and n_kv >= 1024. Falls through to TILE/VEC for ALiBi, logit softcap, // and mismatched batch dimensions (unsupported by the MKL kernel). // Set GGML_SYCL_ENABLE_MKL_FA=0 to force TILE/VEC path for A/B testing. // Example: GGML_SYCL_ENABLE_MKL_FA=0 llama-cli -m model.gguf -fa -ngl 99 ... // Note: MKL GEMM calls are incompatible with SYCL graph capture replay. static int mkl_enable = ggml_sycl_get_env("GGML_SYCL_ENABLE_MKL_FA", 1); // MKL is validated for the mainstream GQA envelope: grouped-query // (gqa_ratio >= 2), head_dim a multiple of 64 in [64,512] with matching // K/V head size, mask, no sinks/ALiBi/softcap. Gemma's global layers use // head_dim 512, so the cap must include it. Head sizes not a multiple of // 64 (72/80/96), MHA (gqa_ratio == 1), and MLA (DKQ != DV, e.g. 576/512) // fall through to TILE/VEC; see follow-up work. if (mkl_enable == 1 && mask && !sinks && gqa_ratio >= 2 && Q->ne[0] >= 64 && Q->ne[0] <= 512 && Q->ne[0] % 64 == 0 && Q->ne[0] == V->ne[0] && Q->ne[1] >= 32 && K->ne[1] >= 1024 && max_bias == 0.0f && logit_softcap == 0.0f && (Q->ne[3] == K->ne[3] || K->ne[3] == 1)) { // F16 K/V strides must be a multiple of ne[0]*2 (the natural row size // in bytes). This passes both dense (nb1 == ne0*2) and interleaved // (nb1 == H * ne0*2). Only pathological test strides like nb1=32 or // nb1=75 for ne0=40 fall through to TILE. bool kv_strides_ok = true; for (const ggml_tensor * t : {K, V}) { if (t->type == GGML_TYPE_F16 && t->nb[1] % (t->ne[0] * 2) != 0) { kv_strides_ok = false; break; } } if (kv_strides_ok) { return BEST_FATTN_KERNEL_MKL; } } for (const ggml_tensor * t : {Q, K, V, mask}) { if (t == nullptr || ggml_is_quantized(t->type)) { continue; } for (size_t i = 1; i < GGML_MAX_DIMS; ++i) { if (t->nb[i] % 16 != 0) { gqa_opt_applies = false; break; } } } switch (K->ne[0]) { case 40: case 64: case 72: case 80: case 96: case 128: case 112: case 256: case 512: if (V->ne[0] != K->ne[0]) { return BEST_FATTN_KERNEL_NONE; } break; case 576: if (V->ne[0] != 512) { return BEST_FATTN_KERNEL_NONE; } if (!gqa_opt_applies) { return BEST_FATTN_KERNEL_NONE; } break; default: return BEST_FATTN_KERNEL_NONE; } #ifndef GGML_SYCL_FA_ALL_QUANTS if (K->type != V->type) { return BEST_FATTN_KERNEL_NONE; } #endif // GGML_SYCL_FA_ALL_QUANTS switch (K->type) { case GGML_TYPE_F32: case GGML_TYPE_F16: case GGML_TYPE_BF16: break; case GGML_TYPE_Q4_1: case GGML_TYPE_Q5_0: case GGML_TYPE_Q5_1: #ifndef GGML_SYCL_FA_ALL_QUANTS return BEST_FATTN_KERNEL_NONE; #endif // GGML_SYCL_FA_ALL_QUANTS case GGML_TYPE_Q4_0: case GGML_TYPE_Q8_0: break; default: return BEST_FATTN_KERNEL_NONE; } if (mask && mask->ne[2] != 1) { return BEST_FATTN_KERNEL_NONE; } // For small batch sizes the vector kernel may be preferable over the kernels optimized for large batch sizes. // BF16 is excluded: the VEC kernel has no BF16 template (it needs GGML_SYCL_FA_ALL_QUANTS for non-F16/Q4_0/Q8_0). const bool has_bf16 = (K->type == GGML_TYPE_BF16 || V->type == GGML_TYPE_BF16); const bool can_use_vector_kernel = Q->ne[0] <= 512 && Q->ne[0] % 64 == 0 && K->ne[1] % FATTN_KQ_STRIDE == 0 && !has_bf16; // Fused-XMX path: oneDNN Graph SDPA (flash attention). Strictly // additive -- taken only when statically supported, otherwise falls through to VEC/TILE below. if (ggml_sycl_flash_attn_ext_onednn_supported(dst)) { return BEST_FATTN_KERNEL_ONEDNN; } // If there are no tensor cores available, use the generic tile kernel: if (can_use_vector_kernel) { if (!ggml_is_quantized(K->type) && !ggml_is_quantized(V->type)) { if (Q->ne[1] == 1) { if (!gqa_opt_applies) { return BEST_FATTN_KERNEL_VEC; } } } else { if (Q->ne[1] <= 2) { return BEST_FATTN_KERNEL_VEC; } } } return BEST_FATTN_KERNEL_TILE; } void ggml_sycl_flash_attn_ext(ggml_backend_sycl_context & ctx, ggml_tensor * dst) { ggml_sycl_set_device(ctx.device); // n_kv watchdog: log when n_kv differs from the last FA call with // the same D — helps detect cache-truncation issues. static int nkv_debug = ggml_sycl_get_env("GGML_SYCL_MKL_FA_DEBUG", 0); if (nkv_debug == 1) { const ggml_tensor * K_dbg = dst->src[1]; const ggml_tensor * V_dbg = dst->src[2]; static int64_t last_nkv_d256 = 0, last_nkv_d512 = 0; static int fa_call_seq = 0; fa_call_seq++; int64_t cur_nkv = K_dbg->ne[1]; int Dk = (int)K_dbg->ne[0]; const char * kname = "TILE"; best_fattn_kernel k = ggml_sycl_get_best_fattn_kernel(ctx.device, dst); if (k == BEST_FATTN_KERNEL_MKL) kname = "MKL"; if (k == BEST_FATTN_KERNEL_ONEDNN) kname = "ONEDNN"; if (k == BEST_FATTN_KERNEL_VEC) kname = "VEC"; int64_t delta = 0; if (Dk == 256) { delta = cur_nkv - last_nkv_d256; last_nkv_d256 = cur_nkv; } else if (Dk == 512) { delta = cur_nkv - last_nkv_d512; last_nkv_d512 = cur_nkv; } GGML_LOG_INFO("[FA-DISP] #%d %s D=%d n_kv=%lld delta=%lld " "V_ne1=%lld\n", fa_call_seq, kname, Dk, (long long)cur_nkv, (long long)delta, (long long)V_dbg->ne[1]); } const best_fattn_kernel fk = ggml_sycl_get_best_fattn_kernel(ggml_sycl_get_device(), dst); switch (fk) { case BEST_FATTN_KERNEL_NONE: GGML_ABORT("Not support Flash-Attention"); case BEST_FATTN_KERNEL_ONEDNN: // guarded: ggml_sycl_flash_attn_ext_onednn() is only defined under GGML_SYCL_DNNL; // the reference must be compiled out here or the GGML_SYCL_DNNL=0 build fails to link. #if GGML_SYCL_DNNL ggml_sycl_flash_attn_ext_onednn(ctx, dst); #endif break; case BEST_FATTN_KERNEL_TILE: ggml_sycl_flash_attn_ext_tile(ctx, dst); break; case BEST_FATTN_KERNEL_VEC: ggml_sycl_flash_attn_ext_vec(ctx, dst); break; case BEST_FATTN_KERNEL_MKL: ggml_sycl_flash_attn_ext_mkl(ctx, dst); break; } // --- Output fingerprint (GGML_SYCL_MKL_FA_DIAG=1) --- // Copy first 64 float output values to host for fingerprinting. // Compare MKL vs TILE (GGML_SYCL_ENABLE_MKL_FA=0) to detect divergence. // Only fingerprints the first 6 FA calls with n_kv >= 1024. static int fa_diag = ggml_sycl_get_env("GGML_SYCL_MKL_FA_DIAG", 0); static int fa_diag_count = 0; if (fa_diag == 1 && fa_diag_count < 6) { const ggml_tensor * K_diag = dst->src[1]; const ggml_tensor * V_diag = dst->src[2]; const ggml_tensor * Q_diag = dst->src[0]; if (K_diag->ne[1] >= 1024) { fa_diag_count++; float diag_buf[64]; dpct::queue_ptr q = ctx.stream(); q->memcpy(diag_buf, dst->data, 64 * sizeof(float)); q->wait(); const char * kname = "???"; best_fattn_kernel kb = ggml_sycl_get_best_fattn_kernel(ctx.device, dst); if (kb == BEST_FATTN_KERNEL_ONEDNN) kname = "ONEDNN"; if (kb == BEST_FATTN_KERNEL_MKL) kname = "MKL"; if (kb == BEST_FATTN_KERNEL_TILE) kname = "TILE"; if (kb == BEST_FATTN_KERNEL_VEC) kname = "VEC"; GGML_LOG_INFO("[FA-DIAG] #%d %s D=%d n_kv=%lld n_q=%lld " "n_qh=%lld n_kvh=%lld K=%s V=%s " "nb1=%zu nb2=%zu first 64 floats:\n", fa_diag_count, kname, (int)K_diag->ne[0], (long long)K_diag->ne[1], (long long)Q_diag->ne[1], (long long)Q_diag->ne[2], (long long)K_diag->ne[2], ggml_type_name(K_diag->type), ggml_type_name(V_diag->type), K_diag->nb[1], K_diag->nb[2]); for (int i = 0; i < 64; i += 8) { GGML_LOG_INFO(" [%2d] %08x %08x %08x %08x %08x %08x %08x %08x\n", i, *(unsigned *)&diag_buf[i+0], *(unsigned *)&diag_buf[i+1], *(unsigned *)&diag_buf[i+2], *(unsigned *)&diag_buf[i+3], *(unsigned *)&diag_buf[i+4], *(unsigned *)&diag_buf[i+5], *(unsigned *)&diag_buf[i+6], *(unsigned *)&diag_buf[i+7]); } } } } bool ggml_sycl_flash_attn_ext_supported(int device, const ggml_tensor * dst) { return ggml_sycl_get_best_fattn_kernel(device, dst) != BEST_FATTN_KERNEL_NONE; }