sycl: make oneDNN FA cache thread-safe and memoize runtime failures (#31138)

This commit is contained in:
Titaniumtown authored and GitHub committed 2026-11-12 08:58:26 +02:00
1 parent dcbff21661
commit 09338b85e8
4 files changed
+44 -19

No files matched your search

+13
View File
@@ -47,6 +47,7 @@ namespace syclexp = sycl::ext::oneapi::experimental;
#if GGML_SYCL_DNNL
#include "dnnl.hpp"
#include "dnnl_sycl.hpp"
#include "oneapi/dnnl/dnnl_graph.hpp"
#endif
#define GGML_COMMON_DECL_SYCL
@@ -484,6 +485,16 @@ struct ggml_sycl_graph {
static_assert(std::is_trivial<ggml_sycl_graph::node_properties>::value, "node_properties must be trivial");
#endif
#if GGML_SYCL_DNNL
struct sdpa_partition {
dnnl::graph::compiled_partition cp;
std::vector<dnnl::graph::logical_tensor> ins;
dnnl::graph::logical_tensor out;
size_t id_q = 0, id_k = 0, id_v = 0, id_scale = 0, id_mask = 0;
bool ok = false;
};
#endif
struct ggml_backend_sycl_context {
int device;
std::string name;
@@ -552,6 +563,8 @@ struct ggml_backend_sycl_context {
dnnl::stream stream_dnnl() {
return stream_dnnl(device, 0);
}
std::unordered_map<std::string, sdpa_partition> sdpa_partitions;
#endif
// pool
+25 -17
View File
@@ -1,4 +1,5 @@
#include <array>
#include <atomic>
#include <cstdint>
#include <cstdio>
#include <cstring>
@@ -25,6 +26,20 @@ bool ggml_sycl_fattn_onednn_binds_kv(const ggml_tensor * K, const ggml_tensor *
return bindable(K) && bindable(V);
}
#if GGML_SYCL_DNNL
// per device: set once a oneDNN SDPA execute throws, so dispatch stops picking oneDNN there
static std::atomic<bool> g_ggml_sycl_fa_onednn_failed[GGML_SYCL_MAX_DEVICES] = {};
#endif
bool ggml_sycl_flash_attn_ext_onednn_failed(int device) {
#if GGML_SYCL_DNNL
return g_ggml_sycl_fa_onednn_failed[device].load(std::memory_order_relaxed);
#else
GGML_UNUSED(device);
return false;
#endif
}
bool ggml_sycl_flash_attn_ext_onednn_supported(const ggml_tensor * dst, bool use_shape_limit) {
#if !GGML_SYCL_DNNL
GGML_UNUSED(dst);
@@ -146,14 +161,6 @@ static void permute_sdpa_out_sycl(const sycl::half * out, float * dst,
});
}
struct sdpa_partition {
compiled_partition cp;
std::vector<logical_tensor> ins;
logical_tensor out;
size_t id_q = 0, id_k = 0, id_v = 0, id_scale = 0, id_mask = 0;
bool ok = false;
};
// Build + compile the contiguous-input GQA SDPA graph (MatMul->Divide->Add->SoftMax->MatMul), f32 out.
// Mirrors the hardware-verified scratch/onednn_sdpa_probe.cpp build_gqa (partitions=1, sdp_primitive_kernel_t).
static sdpa_partition build_sdpa(const engine & eng, int H, int Hkv, int q, int seq, int d,
@@ -361,17 +368,16 @@ void ggml_sycl_flash_attn_ext_onednn(ggml_backend_sycl_context & ctx, ggml_tenso
outf_ptr = outf_pool->get();
}
// compile once per (device, shape, KV strides), reuse across layers/calls. Stride 2 always
// repeats stride 1 and stride 4 is always 1, so the key covers every entry that can differ.
static std::unordered_map<std::string, sdpa_partition> cache;
// compile once per (shape, KV strides) in this device's context, reuse across layers/calls. Stride 2
// always repeats stride 1 and stride 4 is always 1, so the key covers every entry that can differ.
char keyb[256];
snprintf(keyb, sizeof(keyb), "%d:%lld:%lld:%lld:%lld:%lld:%lld:%lld:%lld:%lld:%lld:%lld", ggml_sycl_get_device(),
snprintf(keyb, sizeof(keyb), "%lld:%lld:%lld:%lld:%lld:%lld:%lld:%lld:%lld:%lld:%lld",
(long long) H, (long long) Hkv, (long long) q, (long long) seq, (long long) d,
(long long) k_str[0], (long long) k_str[1], (long long) k_str[3],
(long long) v_str[0], (long long) v_str[1], (long long) v_str[3]);
auto it = cache.find(keyb);
if (it == cache.end()) {
it = cache.emplace(keyb, build_sdpa(eng, (int) H, (int) Hkv, (int) q, (int) seq, (int) d, k_str, v_str)).first;
auto it = ctx.sdpa_partitions.find(keyb);
if (it == ctx.sdpa_partitions.end()) {
it = ctx.sdpa_partitions.emplace(keyb, build_sdpa(eng, (int) H, (int) Hkv, (int) q, (int) seq, (int) d, k_str, v_str)).first;
}
sdpa_partition & E = it->second;
if (!E.ok) {
@@ -407,8 +413,10 @@ void ggml_sycl_flash_attn_ext_onednn(ggml_backend_sycl_context & ctx, ggml_tenso
}
}
catch (const std::exception & e) {
// any oneDNN/SYCL failure is non-fatal: fall back to the existing kernel (strictly additive).
GGML_LOG_WARN("%s: oneDNN SDPA failed (%s); falling back to TILE kernel\n", __func__, e.what());
// any oneDNN/SYCL failure is non-fatal: warn once, stop picking oneDNN on this device, run TILE instead
if (!g_ggml_sycl_fa_onednn_failed[ctx.device].exchange(true)) {
GGML_LOG_WARN("%s: oneDNN SDPA failed (%s); falling back to TILE kernel\n", __func__, e.what());
}
ggml_sycl_flash_attn_ext_tile(ctx, dst);
}
+4
View File
@@ -7,6 +7,10 @@
// (f16 KV, no softcap/ALiBi, single stream, tuned head_dim, prefill-sized q.)
bool ggml_sycl_flash_attn_ext_onednn_supported(const ggml_tensor * dst, bool use_shape_limit = true);
// True once a oneDNN SDPA execute has failed on this device. Not part of _supported(): that also sizes the
// FA scratch buffer, and a size that depends on runtime state can come out smaller than a later call needs.
bool ggml_sycl_flash_attn_ext_onednn_failed(int device);
// True when the oneDNN path binds an F16 KV cache in place instead of staging a dense copy of
// it. Depends only on the types and strides of K and V, so the answer holds for every call.
bool ggml_sycl_fattn_onednn_binds_kv(const ggml_tensor * K, const ggml_tensor * V);
+2 -2
View File
@@ -134,7 +134,7 @@ static best_fattn_kernel ggml_sycl_get_best_fattn_kernel(const int device, const
// XMX-accelerated path: oneDNN SDPA (native F16 and dequant+non-F16).
// ONEDNN requires min 32 query tokens — short-circuit decode to avoid
// calling _supported() on every decode FA call.
if (Q->ne[1] >= 32
if (Q->ne[1] >= 32 && !ggml_sycl_flash_attn_ext_onednn_failed(device)
&& ggml_sycl_flash_attn_ext_onednn_supported(dst)) {
return BEST_FATTN_KERNEL_ONEDNN;
}
@@ -259,7 +259,7 @@ static best_fattn_kernel ggml_sycl_get_best_fattn_kernel(const int device, const
// Fused-XMX path: oneDNN Graph SDPA (flash attention). Strictly
// additive -- taken only when statically supported, otherwise falls through to VEC/TILE below.
if (ggml_sycl_flash_attn_ext_onednn_supported(dst)) {
if (!ggml_sycl_flash_attn_ext_onednn_failed(device) && ggml_sycl_flash_attn_ext_onednn_supported(dst)) {
return BEST_FATTN_KERNEL_ONEDNN;
}