From 09338b85e8b230742fa56f93a1d9f0bf006ca2d3 Mon Sep 17 00:00:00 2001 From: Titaniumtown Date: Wed, 11 Nov 2026 22:58:26 -0800 Subject: [PATCH] sycl: make oneDNN FA cache thread-safe and memoize runtime failures (#31138) --- ggml/src/ggml-sycl/common.hpp | 13 +++++++++ ggml/src/ggml-sycl/fattn-onednn.cpp | 42 +++++++++++++++++------------ ggml/src/ggml-sycl/fattn-onednn.hpp | 4 +++ ggml/src/ggml-sycl/fattn.cpp | 4 +-- 4 files changed, 44 insertions(+), 19 deletions(-) diff --git a/ggml/src/ggml-sycl/common.hpp b/ggml/src/ggml-sycl/common.hpp index f46e79784..f19b0631d 100644 --- a/ggml/src/ggml-sycl/common.hpp +++ b/ggml/src/ggml-sycl/common.hpp @@ -47,6 +47,7 @@ namespace syclexp = sycl::ext::oneapi::experimental; #if GGML_SYCL_DNNL #include "dnnl.hpp" #include "dnnl_sycl.hpp" +#include "oneapi/dnnl/dnnl_graph.hpp" #endif #define GGML_COMMON_DECL_SYCL @@ -484,6 +485,16 @@ struct ggml_sycl_graph { static_assert(std::is_trivial::value, "node_properties must be trivial"); #endif +#if GGML_SYCL_DNNL +struct sdpa_partition { + dnnl::graph::compiled_partition cp; + std::vector ins; + dnnl::graph::logical_tensor out; + size_t id_q = 0, id_k = 0, id_v = 0, id_scale = 0, id_mask = 0; + bool ok = false; +}; +#endif + struct ggml_backend_sycl_context { int device; std::string name; @@ -552,6 +563,8 @@ struct ggml_backend_sycl_context { dnnl::stream stream_dnnl() { return stream_dnnl(device, 0); } + + std::unordered_map sdpa_partitions; #endif // pool diff --git a/ggml/src/ggml-sycl/fattn-onednn.cpp b/ggml/src/ggml-sycl/fattn-onednn.cpp index bd7ce8936..66ff32f15 100644 --- a/ggml/src/ggml-sycl/fattn-onednn.cpp +++ b/ggml/src/ggml-sycl/fattn-onednn.cpp @@ -1,4 +1,5 @@ #include +#include #include #include #include @@ -25,6 +26,20 @@ bool ggml_sycl_fattn_onednn_binds_kv(const ggml_tensor * K, const ggml_tensor * return bindable(K) && bindable(V); } +#if GGML_SYCL_DNNL +// per device: set once a oneDNN SDPA execute throws, so dispatch stops picking oneDNN there +static std::atomic g_ggml_sycl_fa_onednn_failed[GGML_SYCL_MAX_DEVICES] = {}; +#endif + +bool ggml_sycl_flash_attn_ext_onednn_failed(int device) { +#if GGML_SYCL_DNNL + return g_ggml_sycl_fa_onednn_failed[device].load(std::memory_order_relaxed); +#else + GGML_UNUSED(device); + return false; +#endif +} + bool ggml_sycl_flash_attn_ext_onednn_supported(const ggml_tensor * dst, bool use_shape_limit) { #if !GGML_SYCL_DNNL GGML_UNUSED(dst); @@ -146,14 +161,6 @@ static void permute_sdpa_out_sycl(const sycl::half * out, float * dst, }); } -struct sdpa_partition { - compiled_partition cp; - std::vector ins; - logical_tensor out; - size_t id_q = 0, id_k = 0, id_v = 0, id_scale = 0, id_mask = 0; - bool ok = false; -}; - // Build + compile the contiguous-input GQA SDPA graph (MatMul->Divide->Add->SoftMax->MatMul), f32 out. // Mirrors the hardware-verified scratch/onednn_sdpa_probe.cpp build_gqa (partitions=1, sdp_primitive_kernel_t). static sdpa_partition build_sdpa(const engine & eng, int H, int Hkv, int q, int seq, int d, @@ -361,17 +368,16 @@ void ggml_sycl_flash_attn_ext_onednn(ggml_backend_sycl_context & ctx, ggml_tenso outf_ptr = outf_pool->get(); } - // compile once per (device, shape, KV strides), reuse across layers/calls. Stride 2 always - // repeats stride 1 and stride 4 is always 1, so the key covers every entry that can differ. - static std::unordered_map cache; + // compile once per (shape, KV strides) in this device's context, reuse across layers/calls. Stride 2 + // always repeats stride 1 and stride 4 is always 1, so the key covers every entry that can differ. char keyb[256]; - snprintf(keyb, sizeof(keyb), "%d:%lld:%lld:%lld:%lld:%lld:%lld:%lld:%lld:%lld:%lld:%lld", ggml_sycl_get_device(), + snprintf(keyb, sizeof(keyb), "%lld:%lld:%lld:%lld:%lld:%lld:%lld:%lld:%lld:%lld:%lld", (long long) H, (long long) Hkv, (long long) q, (long long) seq, (long long) d, (long long) k_str[0], (long long) k_str[1], (long long) k_str[3], (long long) v_str[0], (long long) v_str[1], (long long) v_str[3]); - auto it = cache.find(keyb); - if (it == cache.end()) { - it = cache.emplace(keyb, build_sdpa(eng, (int) H, (int) Hkv, (int) q, (int) seq, (int) d, k_str, v_str)).first; + auto it = ctx.sdpa_partitions.find(keyb); + if (it == ctx.sdpa_partitions.end()) { + it = ctx.sdpa_partitions.emplace(keyb, build_sdpa(eng, (int) H, (int) Hkv, (int) q, (int) seq, (int) d, k_str, v_str)).first; } sdpa_partition & E = it->second; if (!E.ok) { @@ -407,8 +413,10 @@ void ggml_sycl_flash_attn_ext_onednn(ggml_backend_sycl_context & ctx, ggml_tenso } } catch (const std::exception & e) { - // any oneDNN/SYCL failure is non-fatal: fall back to the existing kernel (strictly additive). - GGML_LOG_WARN("%s: oneDNN SDPA failed (%s); falling back to TILE kernel\n", __func__, e.what()); + // any oneDNN/SYCL failure is non-fatal: warn once, stop picking oneDNN on this device, run TILE instead + if (!g_ggml_sycl_fa_onednn_failed[ctx.device].exchange(true)) { + GGML_LOG_WARN("%s: oneDNN SDPA failed (%s); falling back to TILE kernel\n", __func__, e.what()); + } ggml_sycl_flash_attn_ext_tile(ctx, dst); } diff --git a/ggml/src/ggml-sycl/fattn-onednn.hpp b/ggml/src/ggml-sycl/fattn-onednn.hpp index 9669d1bd2..4756b240a 100644 --- a/ggml/src/ggml-sycl/fattn-onednn.hpp +++ b/ggml/src/ggml-sycl/fattn-onednn.hpp @@ -7,6 +7,10 @@ // (f16 KV, no softcap/ALiBi, single stream, tuned head_dim, prefill-sized q.) bool ggml_sycl_flash_attn_ext_onednn_supported(const ggml_tensor * dst, bool use_shape_limit = true); +// True once a oneDNN SDPA execute has failed on this device. Not part of _supported(): that also sizes the +// FA scratch buffer, and a size that depends on runtime state can come out smaller than a later call needs. +bool ggml_sycl_flash_attn_ext_onednn_failed(int device); + // True when the oneDNN path binds an F16 KV cache in place instead of staging a dense copy of // it. Depends only on the types and strides of K and V, so the answer holds for every call. bool ggml_sycl_fattn_onednn_binds_kv(const ggml_tensor * K, const ggml_tensor * V); diff --git a/ggml/src/ggml-sycl/fattn.cpp b/ggml/src/ggml-sycl/fattn.cpp index 6f5f8a832..525e6fd8c 100644 --- a/ggml/src/ggml-sycl/fattn.cpp +++ b/ggml/src/ggml-sycl/fattn.cpp @@ -134,7 +134,7 @@ static best_fattn_kernel ggml_sycl_get_best_fattn_kernel(const int device, const // XMX-accelerated path: oneDNN SDPA (native F16 and dequant+non-F16). // ONEDNN requires min 32 query tokens — short-circuit decode to avoid // calling _supported() on every decode FA call. - if (Q->ne[1] >= 32 + if (Q->ne[1] >= 32 && !ggml_sycl_flash_attn_ext_onednn_failed(device) && ggml_sycl_flash_attn_ext_onednn_supported(dst)) { return BEST_FATTN_KERNEL_ONEDNN; } @@ -259,7 +259,7 @@ static best_fattn_kernel ggml_sycl_get_best_fattn_kernel(const int device, const // Fused-XMX path: oneDNN Graph SDPA (flash attention). Strictly // additive -- taken only when statically supported, otherwise falls through to VEC/TILE below. - if (ggml_sycl_flash_attn_ext_onednn_supported(dst)) { + if (!ggml_sycl_flash_attn_ext_onednn_failed(device) && ggml_sycl_flash_attn_ext_onednn_supported(dst)) { return BEST_FATTN_KERNEL_ONEDNN; }