sycl: make oneDNN FA cache thread-safe and memoize runtime failures (#31138)
This commit is contained in:
1 parent
dcbff21661
commit
09338b85e8
4 files changed
+44
-19
No files matched your search
@@ -47,6 +47,7 @@ namespace syclexp = sycl::ext::oneapi::experimental;
|
||||
#if GGML_SYCL_DNNL
|
||||
#include "dnnl.hpp"
|
||||
#include "dnnl_sycl.hpp"
|
||||
#include "oneapi/dnnl/dnnl_graph.hpp"
|
||||
#endif
|
||||
|
||||
#define GGML_COMMON_DECL_SYCL
|
||||
@@ -484,6 +485,16 @@ struct ggml_sycl_graph {
|
||||
static_assert(std::is_trivial<ggml_sycl_graph::node_properties>::value, "node_properties must be trivial");
|
||||
#endif
|
||||
|
||||
#if GGML_SYCL_DNNL
|
||||
struct sdpa_partition {
|
||||
dnnl::graph::compiled_partition cp;
|
||||
std::vector<dnnl::graph::logical_tensor> ins;
|
||||
dnnl::graph::logical_tensor out;
|
||||
size_t id_q = 0, id_k = 0, id_v = 0, id_scale = 0, id_mask = 0;
|
||||
bool ok = false;
|
||||
};
|
||||
#endif
|
||||
|
||||
struct ggml_backend_sycl_context {
|
||||
int device;
|
||||
std::string name;
|
||||
@@ -552,6 +563,8 @@ struct ggml_backend_sycl_context {
|
||||
dnnl::stream stream_dnnl() {
|
||||
return stream_dnnl(device, 0);
|
||||
}
|
||||
|
||||
std::unordered_map<std::string, sdpa_partition> sdpa_partitions;
|
||||
#endif
|
||||
|
||||
// pool
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
#include <array>
|
||||
#include <atomic>
|
||||
#include <cstdint>
|
||||
#include <cstdio>
|
||||
#include <cstring>
|
||||
@@ -25,6 +26,20 @@ bool ggml_sycl_fattn_onednn_binds_kv(const ggml_tensor * K, const ggml_tensor *
|
||||
return bindable(K) && bindable(V);
|
||||
}
|
||||
|
||||
#if GGML_SYCL_DNNL
|
||||
// per device: set once a oneDNN SDPA execute throws, so dispatch stops picking oneDNN there
|
||||
static std::atomic<bool> g_ggml_sycl_fa_onednn_failed[GGML_SYCL_MAX_DEVICES] = {};
|
||||
#endif
|
||||
|
||||
bool ggml_sycl_flash_attn_ext_onednn_failed(int device) {
|
||||
#if GGML_SYCL_DNNL
|
||||
return g_ggml_sycl_fa_onednn_failed[device].load(std::memory_order_relaxed);
|
||||
#else
|
||||
GGML_UNUSED(device);
|
||||
return false;
|
||||
#endif
|
||||
}
|
||||
|
||||
bool ggml_sycl_flash_attn_ext_onednn_supported(const ggml_tensor * dst, bool use_shape_limit) {
|
||||
#if !GGML_SYCL_DNNL
|
||||
GGML_UNUSED(dst);
|
||||
@@ -146,14 +161,6 @@ static void permute_sdpa_out_sycl(const sycl::half * out, float * dst,
|
||||
});
|
||||
}
|
||||
|
||||
struct sdpa_partition {
|
||||
compiled_partition cp;
|
||||
std::vector<logical_tensor> ins;
|
||||
logical_tensor out;
|
||||
size_t id_q = 0, id_k = 0, id_v = 0, id_scale = 0, id_mask = 0;
|
||||
bool ok = false;
|
||||
};
|
||||
|
||||
// Build + compile the contiguous-input GQA SDPA graph (MatMul->Divide->Add->SoftMax->MatMul), f32 out.
|
||||
// Mirrors the hardware-verified scratch/onednn_sdpa_probe.cpp build_gqa (partitions=1, sdp_primitive_kernel_t).
|
||||
static sdpa_partition build_sdpa(const engine & eng, int H, int Hkv, int q, int seq, int d,
|
||||
@@ -361,17 +368,16 @@ void ggml_sycl_flash_attn_ext_onednn(ggml_backend_sycl_context & ctx, ggml_tenso
|
||||
outf_ptr = outf_pool->get();
|
||||
}
|
||||
|
||||
// compile once per (device, shape, KV strides), reuse across layers/calls. Stride 2 always
|
||||
// repeats stride 1 and stride 4 is always 1, so the key covers every entry that can differ.
|
||||
static std::unordered_map<std::string, sdpa_partition> cache;
|
||||
// compile once per (shape, KV strides) in this device's context, reuse across layers/calls. Stride 2
|
||||
// always repeats stride 1 and stride 4 is always 1, so the key covers every entry that can differ.
|
||||
char keyb[256];
|
||||
snprintf(keyb, sizeof(keyb), "%d:%lld:%lld:%lld:%lld:%lld:%lld:%lld:%lld:%lld:%lld:%lld", ggml_sycl_get_device(),
|
||||
snprintf(keyb, sizeof(keyb), "%lld:%lld:%lld:%lld:%lld:%lld:%lld:%lld:%lld:%lld:%lld",
|
||||
(long long) H, (long long) Hkv, (long long) q, (long long) seq, (long long) d,
|
||||
(long long) k_str[0], (long long) k_str[1], (long long) k_str[3],
|
||||
(long long) v_str[0], (long long) v_str[1], (long long) v_str[3]);
|
||||
auto it = cache.find(keyb);
|
||||
if (it == cache.end()) {
|
||||
it = cache.emplace(keyb, build_sdpa(eng, (int) H, (int) Hkv, (int) q, (int) seq, (int) d, k_str, v_str)).first;
|
||||
auto it = ctx.sdpa_partitions.find(keyb);
|
||||
if (it == ctx.sdpa_partitions.end()) {
|
||||
it = ctx.sdpa_partitions.emplace(keyb, build_sdpa(eng, (int) H, (int) Hkv, (int) q, (int) seq, (int) d, k_str, v_str)).first;
|
||||
}
|
||||
sdpa_partition & E = it->second;
|
||||
if (!E.ok) {
|
||||
@@ -407,8 +413,10 @@ void ggml_sycl_flash_attn_ext_onednn(ggml_backend_sycl_context & ctx, ggml_tenso
|
||||
}
|
||||
}
|
||||
catch (const std::exception & e) {
|
||||
// any oneDNN/SYCL failure is non-fatal: fall back to the existing kernel (strictly additive).
|
||||
GGML_LOG_WARN("%s: oneDNN SDPA failed (%s); falling back to TILE kernel\n", __func__, e.what());
|
||||
// any oneDNN/SYCL failure is non-fatal: warn once, stop picking oneDNN on this device, run TILE instead
|
||||
if (!g_ggml_sycl_fa_onednn_failed[ctx.device].exchange(true)) {
|
||||
GGML_LOG_WARN("%s: oneDNN SDPA failed (%s); falling back to TILE kernel\n", __func__, e.what());
|
||||
}
|
||||
ggml_sycl_flash_attn_ext_tile(ctx, dst);
|
||||
}
|
||||
|
||||
|
||||
@@ -7,6 +7,10 @@
|
||||
// (f16 KV, no softcap/ALiBi, single stream, tuned head_dim, prefill-sized q.)
|
||||
bool ggml_sycl_flash_attn_ext_onednn_supported(const ggml_tensor * dst, bool use_shape_limit = true);
|
||||
|
||||
// True once a oneDNN SDPA execute has failed on this device. Not part of _supported(): that also sizes the
|
||||
// FA scratch buffer, and a size that depends on runtime state can come out smaller than a later call needs.
|
||||
bool ggml_sycl_flash_attn_ext_onednn_failed(int device);
|
||||
|
||||
// True when the oneDNN path binds an F16 KV cache in place instead of staging a dense copy of
|
||||
// it. Depends only on the types and strides of K and V, so the answer holds for every call.
|
||||
bool ggml_sycl_fattn_onednn_binds_kv(const ggml_tensor * K, const ggml_tensor * V);
|
||||
|
||||
@@ -134,7 +134,7 @@ static best_fattn_kernel ggml_sycl_get_best_fattn_kernel(const int device, const
|
||||
// XMX-accelerated path: oneDNN SDPA (native F16 and dequant+non-F16).
|
||||
// ONEDNN requires min 32 query tokens — short-circuit decode to avoid
|
||||
// calling _supported() on every decode FA call.
|
||||
if (Q->ne[1] >= 32
|
||||
if (Q->ne[1] >= 32 && !ggml_sycl_flash_attn_ext_onednn_failed(device)
|
||||
&& ggml_sycl_flash_attn_ext_onednn_supported(dst)) {
|
||||
return BEST_FATTN_KERNEL_ONEDNN;
|
||||
}
|
||||
@@ -259,7 +259,7 @@ static best_fattn_kernel ggml_sycl_get_best_fattn_kernel(const int device, const
|
||||
|
||||
// Fused-XMX path: oneDNN Graph SDPA (flash attention). Strictly
|
||||
// additive -- taken only when statically supported, otherwise falls through to VEC/TILE below.
|
||||
if (ggml_sycl_flash_attn_ext_onednn_supported(dst)) {
|
||||
if (!ggml_sycl_flash_attn_ext_onednn_failed(device) && ggml_sycl_flash_attn_ext_onednn_supported(dst)) {
|
||||
return BEST_FATTN_KERNEL_ONEDNN;
|
||||
}
|
||||
|
||||
|
||||
Reference in new issue
Block a user