sycl: column-blocked gated_delta_net kernel (#32456)
This commit is contained in:
1 parent
80325823d4
commit
44c20b1c0a
2 files changed
+178
-118
No files matched your search
@@ -6,7 +6,13 @@
|
||||
#include <cmath>
|
||||
|
||||
|
||||
template <int S_v, bool KDA, bool keep_rs_t>
|
||||
// Warps per work-group. 2 is 13% faster than 4 at 4 columns per warp on Arc Pro B70.
|
||||
static constexpr int gdn_warps = 2;
|
||||
// State columns per warp when there are enough heads, see launch_gated_delta_net.
|
||||
static constexpr int gdn_cols_wide = 4;
|
||||
|
||||
// C is the number of state columns per warp: a token's q/k shard is loaded once and shared by these columns.
|
||||
template <int S_v, int C, bool KDA, bool keep_rs_t>
|
||||
void gated_delta_net_sycl(const float * q,
|
||||
const float * k,
|
||||
const float * v,
|
||||
@@ -34,9 +40,10 @@ void gated_delta_net_sycl(const float * q,
|
||||
auto item_ct1 = sycl::ext::oneapi::this_work_item::get_nd_item<3>();
|
||||
const uint32_t h_idx = item_ct1.get_group(2);
|
||||
const uint32_t sequence = item_ct1.get_group(1);
|
||||
// each warp owns one column, using warp-level primitives to reduce across rows
|
||||
// each warp owns C columns, using warp-level primitives to reduce across rows
|
||||
const int lane = item_ct1.get_local_id(2);
|
||||
const int col = item_ct1.get_group(0) * item_ct1.get_local_range(1) + item_ct1.get_local_id(1);
|
||||
const int warp = item_ct1.get_group(0) * item_ct1.get_local_range(1) + item_ct1.get_local_id(1);
|
||||
const int col0 = warp * C;
|
||||
|
||||
const uint32_t iq1 = fastmodulo(h_idx, neqk1_magic);
|
||||
const uint32_t iq3 = fastdiv(sequence, rq3_magic);
|
||||
@@ -48,94 +55,145 @@ void gated_delta_net_sycl(const float * q,
|
||||
const int64_t state_in_offset = sequence * H * S_v * S_v + h_idx * S_v * S_v;
|
||||
const int64_t state_out_offset = (sequence * H + h_idx) * S_v * S_v;
|
||||
state += state_out_offset;
|
||||
curr_state += state_in_offset + col * S_v;
|
||||
curr_state += state_in_offset;
|
||||
attn_data += (sequence * n_tokens * H + h_idx) * S_v;
|
||||
|
||||
constexpr int warp_size = ggml_sycl_get_physical_warp_size() < S_v ? ggml_sycl_get_physical_warp_size() : S_v;
|
||||
static_assert(S_v % warp_size == 0, "S_v must be a multiple of warp_size");
|
||||
static_assert(S_v % (C * gdn_warps) == 0, "S_v must be a multiple of the columns per work-group");
|
||||
static_assert(C <= warp_size, "the attn store puts column c in lane c");
|
||||
constexpr int rows_per_lane = (S_v + warp_size - 1) / warp_size;
|
||||
float s_shard[rows_per_lane];
|
||||
float s_shard[C][rows_per_lane];
|
||||
#pragma unroll
|
||||
for (int r = 0; r < rows_per_lane; r++) {
|
||||
const int i = r * warp_size + lane;
|
||||
s_shard[r] = curr_state[i];
|
||||
for (int c = 0; c < C; c++) {
|
||||
#pragma unroll
|
||||
for (int r = 0; r < rows_per_lane; r++) {
|
||||
const int i = r * warp_size + lane;
|
||||
s_shard[c][r] = curr_state[(col0 + c) * S_v + i];
|
||||
}
|
||||
}
|
||||
|
||||
// snapshot slot mapping: slot 0 = most recent state, slot s = s tokens back.
|
||||
// When n_tokens < K only slots 0..n_tokens-1 are written; older slots are caller-owned.
|
||||
|
||||
const float * qp = q + iq3 * sq3 + iq1 * sq1 + lane;
|
||||
const float * kp = k + iq3 * sq3 + iq1 * sq1 + lane;
|
||||
const float * vp = v + sequence * sv3 + h_idx * sv1 + col0;
|
||||
const float * bp = beta + sequence * sb3 + h_idx * sb1;
|
||||
const float * gp = g + (sequence * sb3 + h_idx * sb1) * (KDA ? S_v : 1);
|
||||
|
||||
// q/k of the next token are loaded one iteration ahead to hide the load latency.
|
||||
float q_next[rows_per_lane];
|
||||
float k_next[rows_per_lane];
|
||||
#pragma unroll
|
||||
for (int r = 0; r < rows_per_lane; r++) {
|
||||
q_next[r] = qp[r * warp_size];
|
||||
k_next[r] = kp[r * warp_size];
|
||||
}
|
||||
|
||||
for (int t = 0; t < n_tokens; t++) {
|
||||
const float * q_t = q + iq3 * sq3 + t * sq2 + iq1 * sq1;
|
||||
const float * k_t = k + iq3 * sq3 + t * sq2 + iq1 * sq1;
|
||||
const float * v_t = v + sequence * sv3 + t * sv2 + h_idx * sv1;
|
||||
const float beta_val = *bp;
|
||||
const float * g_t = gp;
|
||||
bp += sb2;
|
||||
gp += sb2 * (KDA ? S_v : 1);
|
||||
|
||||
const int64_t gb_offset = sequence * sb3 + t * sb2 + h_idx * sb1;
|
||||
const float * beta_t = beta + gb_offset;
|
||||
const float * g_t = g + gb_offset * (KDA ? S_v : 1);
|
||||
float q_reg[rows_per_lane];
|
||||
float k_reg[rows_per_lane];
|
||||
#pragma unroll
|
||||
for (int r = 0; r < rows_per_lane; r++) {
|
||||
q_reg[r] = q_next[r];
|
||||
k_reg[r] = k_next[r];
|
||||
}
|
||||
if (t + 1 < n_tokens) {
|
||||
qp += sq2;
|
||||
kp += sq2;
|
||||
}
|
||||
// Load outside the branch: the last token reads its own row again.
|
||||
// A guarded load is 20% slower at 32 heads on Arc Pro B70.
|
||||
#pragma unroll
|
||||
for (int r = 0; r < rows_per_lane; r++) {
|
||||
q_next[r] = qp[r * warp_size];
|
||||
k_next[r] = kp[r * warp_size];
|
||||
}
|
||||
|
||||
const float beta_val = *beta_t;
|
||||
float v_reg[C];
|
||||
#pragma unroll
|
||||
for (int c = 0; c < C; c++) {
|
||||
v_reg[c] = vp[c];
|
||||
}
|
||||
vp += sv2;
|
||||
|
||||
// attn[col] = g * (S^T q)[col] + delta[col] * (k . q), so no reduction waits for delta
|
||||
float kq_shard = 0.0f;
|
||||
#pragma unroll
|
||||
for (int r = 0; r < rows_per_lane; r++) {
|
||||
kq_shard += k_reg[r] * q_reg[r];
|
||||
}
|
||||
const float kq = warp_reduce_sum<warp_size>(kq_shard);
|
||||
|
||||
float out_lane = 0.0f;
|
||||
if constexpr (!KDA) {
|
||||
const float g_val = sycl::native::exp(*g_t);
|
||||
|
||||
// kv[col] = (S^T @ k)[col] = sum_i S[i][col] * k[i]
|
||||
float kv_shard = 0.0f;
|
||||
#pragma unroll
|
||||
for (int r = 0; r < rows_per_lane; r++) {
|
||||
const int i = r * warp_size + lane;
|
||||
kv_shard += s_shard[r] * k_t[i];
|
||||
}
|
||||
float kv_col = warp_reduce_sum<warp_size>(kv_shard);
|
||||
|
||||
// delta[col] = (v[col] - g * kv[col]) * beta
|
||||
float delta_col = (v_t[col] - g_val * kv_col) * beta_val;
|
||||
|
||||
// fused: S[i][col] = g * S[i][col] + k[i] * delta[col]
|
||||
// attn[col] = (S^T @ q)[col] = sum_i S[i][col] * q[i]
|
||||
float attn_partial = 0.0f;
|
||||
for (int c = 0; c < C; c++) {
|
||||
// x: kv[col] = (S^T k)[col], y: (S^T q)[col]
|
||||
sycl::float2 acc(0.0f, 0.0f);
|
||||
#pragma unroll
|
||||
for (int r = 0; r < rows_per_lane; r++) {
|
||||
const int i = r * warp_size + lane;
|
||||
s_shard[r] = g_val * s_shard[r] + k_t[i] * delta_col;
|
||||
attn_partial += s_shard[r] * q_t[i];
|
||||
}
|
||||
for (int r = 0; r < rows_per_lane; r++) {
|
||||
acc.x() += s_shard[c][r] * k_reg[r];
|
||||
acc.y() += s_shard[c][r] * q_reg[r];
|
||||
}
|
||||
acc = warp_reduce_sum<warp_size>(acc);
|
||||
|
||||
float attn_col = warp_reduce_sum<warp_size>(attn_partial);
|
||||
// delta[col] = (v[col] - g * kv[col]) * beta
|
||||
const float delta_col = (v_reg[c] - g_val * acc.x()) * beta_val;
|
||||
|
||||
if (lane == 0) {
|
||||
attn_data[col] = attn_col * scale;
|
||||
// S[i][col] = g * S[i][col] + k[i] * delta[col]
|
||||
#pragma unroll
|
||||
for (int r = 0; r < rows_per_lane; r++) {
|
||||
s_shard[c][r] = g_val * s_shard[c][r] + k_reg[r] * delta_col;
|
||||
}
|
||||
|
||||
// the reductions broadcast, so lane c already holds column c's result
|
||||
out_lane = (lane == c) ? (g_val * acc.y() + delta_col * kq) * scale : out_lane;
|
||||
}
|
||||
} else {
|
||||
// kv[col] = sum_i g[i] * S[i][col] * k[i]
|
||||
float kv_shard = 0.0f;
|
||||
float g_reg[rows_per_lane];
|
||||
#pragma unroll
|
||||
for (int r = 0; r < rows_per_lane; r++) {
|
||||
const int i = r * warp_size + lane;
|
||||
kv_shard += sycl::native::exp(g_t[i]) * s_shard[r] * k_t[i];
|
||||
g_reg[r] = sycl::native::exp(g_t[r * warp_size + lane]);
|
||||
}
|
||||
|
||||
float kv_col = warp_reduce_sum<warp_size>(kv_shard);
|
||||
|
||||
// delta[col] = (v[col] - kv[col]) * beta
|
||||
float delta_col = (v_t[col] - kv_col) * beta_val;
|
||||
|
||||
// fused: S[i][col] = g[i] * S[i][col] + k[i] * delta[col]
|
||||
// attn[col] = (S^T @ q)[col] = sum_i S[i][col] * q[i]
|
||||
float attn_partial = 0.0f;
|
||||
#pragma unroll
|
||||
for (int r = 0; r < rows_per_lane; r++) {
|
||||
const int i = r * warp_size + lane;
|
||||
s_shard[r] = sycl::native::exp(g_t[i]) * s_shard[r] + k_t[i] * delta_col;
|
||||
attn_partial += s_shard[r] * q_t[i];
|
||||
}
|
||||
for (int c = 0; c < C; c++) {
|
||||
// x: kv[col] = sum_i g[i] * S[i][col] * k[i], y: sum_i g[i] * S[i][col] * q[i]
|
||||
sycl::float2 acc(0.0f, 0.0f);
|
||||
#pragma unroll
|
||||
for (int r = 0; r < rows_per_lane; r++) {
|
||||
const float gs = g_reg[r] * s_shard[c][r];
|
||||
acc.x() += gs * k_reg[r];
|
||||
acc.y() += gs * q_reg[r];
|
||||
}
|
||||
acc = warp_reduce_sum<warp_size>(acc);
|
||||
|
||||
float attn_col = warp_reduce_sum<warp_size>(attn_partial);
|
||||
// delta[col] = (v[col] - kv[col]) * beta
|
||||
const float delta_col = (v_reg[c] - acc.x()) * beta_val;
|
||||
|
||||
if (lane == 0) {
|
||||
attn_data[col] = attn_col * scale;
|
||||
// S[i][col] = g[i] * S[i][col] + k[i] * delta[col]
|
||||
#pragma unroll
|
||||
for (int r = 0; r < rows_per_lane; r++) {
|
||||
s_shard[c][r] = g_reg[r] * s_shard[c][r] + k_reg[r] * delta_col;
|
||||
}
|
||||
|
||||
out_lane = (lane == c) ? (acc.y() + delta_col * kq) * scale : out_lane;
|
||||
}
|
||||
}
|
||||
|
||||
if (lane < C) {
|
||||
attn_data[col0 + lane] = out_lane;
|
||||
}
|
||||
|
||||
attn_data += S_v * H;
|
||||
|
||||
|
||||
@@ -143,21 +201,26 @@ void gated_delta_net_sycl(const float * q,
|
||||
if constexpr (keep_rs_t) {
|
||||
const int target_slot = (int) n_tokens - 1 - t;
|
||||
if (target_slot >= 0 && target_slot < K) {
|
||||
float * curr_state = state + target_slot * state_slot_stride;
|
||||
float * slot_col = state + target_slot * state_slot_stride + col0 * S_v + lane;
|
||||
#pragma unroll
|
||||
for (int r = 0; r < rows_per_lane; r++) {
|
||||
const int i = r * warp_size + lane;
|
||||
curr_state[col * S_v + i] = s_shard[r];
|
||||
for (int c = 0; c < C; c++) {
|
||||
#pragma unroll
|
||||
for (int r = 0; r < rows_per_lane; r++) {
|
||||
slot_col[c * S_v + r * warp_size] = s_shard[c][r];
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if constexpr (!keep_rs_t) {
|
||||
float * out_col = state + col0 * S_v + lane;
|
||||
#pragma unroll
|
||||
for (int r = 0; r < rows_per_lane; r++) {
|
||||
const int i = r * warp_size + lane;
|
||||
state[col * S_v + i] = s_shard[r];
|
||||
for (int c = 0; c < C; c++) {
|
||||
#pragma unroll
|
||||
for (int r = 0; r < rows_per_lane; r++) {
|
||||
out_col[c * S_v + r * warp_size] = s_shard[c][r];
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -193,60 +256,36 @@ static void launch_gated_delta_net(const float * q_d,
|
||||
//TODO: Add chunked kernel for even faster pre-fill
|
||||
const int warp_size = ggml_sycl_info().devices[ggml_sycl_get_device()].warp_size;
|
||||
|
||||
const int num_warps = 4;
|
||||
dpct::dim3 grid_dims(H, n_seqs, (S_v + num_warps - 1) / num_warps);
|
||||
dpct::dim3 block_dims(warp_size <= S_v ? warp_size : S_v, num_warps, 1);
|
||||
|
||||
const sycl::uint3 neqk1_magic = init_fastdiv_values(neqk1);
|
||||
const sycl::uint3 rq3_magic = init_fastdiv_values(rq3);
|
||||
|
||||
auto launch = [&](auto sv_c, auto c_c) {
|
||||
constexpr int sv = decltype(sv_c)::value;
|
||||
constexpr int C = decltype(c_c)::value;
|
||||
dpct::dim3 grid_dims(H, n_seqs, sv / (C * gdn_warps));
|
||||
dpct::dim3 block_dims(warp_size <= sv ? warp_size : sv, gdn_warps, 1);
|
||||
stream->parallel_for(sycl::nd_range<3>(grid_dims * block_dims, block_dims),
|
||||
[=](sycl::nd_item<3> /*item_ct1*/) [[sycl::reqd_sub_group_size(WARP_SIZE)]] {
|
||||
gated_delta_net_sycl<sv, C, KDA, keep_rs_t>(
|
||||
q_d, k_d, v_d, g_d, b_d, s_d, dst_d, state_d, H, n_tokens, sq1, sq2, sq3, sv1, sv2,
|
||||
sv3, sb1, sb2, sb3, neqk1_magic, rq3_magic, scale, state_slot_stride, K);
|
||||
});
|
||||
};
|
||||
// gdn_cols_wide columns per warp cut the q/k loads but launch fewer warps, so with few heads 1 column is faster.
|
||||
// The 1024-warp switch point is measured on Arc Pro B70.
|
||||
auto launch_sv = [&](auto sv_c) {
|
||||
if (H * n_seqs * S_v / gdn_cols_wide >= 1024) {
|
||||
launch(sv_c, std::integral_constant<int, gdn_cols_wide>{});
|
||||
} else {
|
||||
launch(sv_c, std::integral_constant<int, 1>{});
|
||||
}
|
||||
};
|
||||
|
||||
switch (S_v) {
|
||||
case 16:
|
||||
{
|
||||
constexpr int sv = 16;
|
||||
stream->parallel_for(sycl::nd_range<3>(grid_dims * block_dims, block_dims),
|
||||
[=](sycl::nd_item<3> /*item_ct1*/) [[sycl::reqd_sub_group_size(WARP_SIZE)]] {
|
||||
gated_delta_net_sycl<sv, KDA, keep_rs_t>(q_d, k_d, v_d, g_d, b_d, s_d, dst_d, state_d, H, n_tokens,
|
||||
sq1, sq2, sq3, sv1, sv2, sv3, sb1, sb2,
|
||||
sb3, neqk1_magic, rq3_magic, scale, state_slot_stride, K);
|
||||
});
|
||||
}
|
||||
break;
|
||||
case 32:
|
||||
{
|
||||
constexpr int sv = 32;
|
||||
stream->parallel_for(sycl::nd_range<3>(grid_dims * block_dims, block_dims),
|
||||
[=](sycl::nd_item<3> /*item_ct1*/) [[sycl::reqd_sub_group_size(WARP_SIZE)]] {
|
||||
gated_delta_net_sycl<sv, KDA, keep_rs_t>(q_d, k_d, v_d, g_d, b_d, s_d, dst_d, state_d, H, n_tokens,
|
||||
sq1, sq2, sq3, sv1, sv2, sv3, sb1, sb2,
|
||||
sb3, neqk1_magic, rq3_magic, scale, state_slot_stride, K);
|
||||
});
|
||||
}
|
||||
break;
|
||||
case 64: {
|
||||
{
|
||||
constexpr int sv = 64;
|
||||
stream->parallel_for(sycl::nd_range<3>(grid_dims * block_dims, block_dims),
|
||||
[=](sycl::nd_item<3> /*item_ct1*/) [[sycl::reqd_sub_group_size(WARP_SIZE)]] {
|
||||
gated_delta_net_sycl<sv, KDA, keep_rs_t>(
|
||||
q_d, k_d, v_d, g_d, b_d, s_d, dst_d, state_d, H, n_tokens, sq1, sq2,
|
||||
sq3, sv1, sv2, sv3, sb1, sb2, sb3, neqk1_magic, rq3_magic, scale, state_slot_stride, K);
|
||||
});
|
||||
}
|
||||
break;
|
||||
}
|
||||
case 128: {
|
||||
{
|
||||
constexpr int sv = 128;
|
||||
stream->parallel_for(sycl::nd_range<3>(grid_dims * block_dims, block_dims),
|
||||
[=](sycl::nd_item<3> /*item_ct1*/) [[sycl::reqd_sub_group_size(WARP_SIZE)]] {
|
||||
gated_delta_net_sycl<sv, KDA, keep_rs_t>(
|
||||
q_d, k_d, v_d, g_d, b_d, s_d, dst_d, state_d, H, n_tokens, sq1, sq2,
|
||||
sq3, sv1, sv2, sv3, sb1, sb2, sb3, neqk1_magic, rq3_magic, scale, state_slot_stride, K);
|
||||
});
|
||||
}
|
||||
break;
|
||||
}
|
||||
case 16: launch_sv(std::integral_constant<int, 16>{}); break;
|
||||
case 32: launch_sv(std::integral_constant<int, 32>{}); break;
|
||||
case 64: launch_sv(std::integral_constant<int, 64>{}); break;
|
||||
case 128: launch_sv(std::integral_constant<int, 128>{}); break;
|
||||
default:
|
||||
GGML_ABORT("fatal error");
|
||||
break;
|
||||
|
||||
@@ -5317,16 +5317,22 @@ struct test_gated_delta_net : public test_case {
|
||||
const bool padded_v;
|
||||
const bool padded_q;
|
||||
const bool permuted_k;
|
||||
// lower bound of the log-gate; a mild gate keeps the state alive over the whole sequence
|
||||
const float g_min;
|
||||
|
||||
std::string vars() override {
|
||||
return VARS_TO_STR13(type, head_count, head_size, n_seq_tokens, n_seqs, v_repeat, permuted, kda, K, correlated, padded_v, padded_q, permuted_k);
|
||||
std::string s = VARS_TO_STR13(type, head_count, head_size, n_seq_tokens, n_seqs, v_repeat, permuted, kda, K, correlated, padded_v, padded_q, permuted_k);
|
||||
if (g_min != -20.0f) {
|
||||
s += "," + VAR_TO_STR(g_min);
|
||||
}
|
||||
return s;
|
||||
}
|
||||
|
||||
test_gated_delta_net(ggml_type type = GGML_TYPE_F32,
|
||||
int64_t head_count = 4, int64_t head_size = 16, int64_t n_seq_tokens = 1, int64_t n_seqs = 1,
|
||||
int v_repeat = 1, bool permuted = false, bool kda = false, int64_t K = 1, bool correlated = false, bool padded_v = false, bool padded_q = false, bool permuted_k = false)
|
||||
int v_repeat = 1, bool permuted = false, bool kda = false, int64_t K = 1, bool correlated = false, bool padded_v = false, bool padded_q = false, bool permuted_k = false, float g_min = -20.0f)
|
||||
: type(type), head_count(head_count), head_size(head_size), n_seq_tokens(n_seq_tokens), n_seqs(n_seqs),
|
||||
v_repeat(v_repeat), permuted(permuted), kda(kda), K(K), correlated(correlated), padded_v(padded_v), padded_q(padded_q), permuted_k(permuted_k) {}
|
||||
v_repeat(v_repeat), permuted(permuted), kda(kda), K(K), correlated(correlated), padded_v(padded_v), padded_q(padded_q), permuted_k(permuted_k), g_min(g_min) {}
|
||||
|
||||
ggml_tensor * build_graph(ggml_context * ctx) override {
|
||||
ggml_tensor * q;
|
||||
@@ -5378,7 +5384,7 @@ struct test_gated_delta_net : public test_case {
|
||||
for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != nullptr; t = ggml_get_next_tensor(ctx, t)) {
|
||||
if (ggml_is_view_op(t->op)) { continue; }
|
||||
if (strcmp(t->name, "g") == 0) {
|
||||
init_tensor_uniform(t, correlated ? -1e-4f : -20.0f, -1e-4f);
|
||||
init_tensor_uniform(t, correlated ? -1e-4f : g_min, -1e-4f);
|
||||
} else if (strcmp(t->name, "beta") == 0) {
|
||||
init_tensor_uniform(t, correlated ? 0.9f : 0.0f, correlated ? 0.9f : 1.0f);
|
||||
} else if (correlated && strcmp(t->name, "k") == 0) {
|
||||
@@ -13765,6 +13771,19 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() {
|
||||
test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 2, 128, n_tokens, 2, 2, false, false, 1, false, false, true));
|
||||
test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 2, 128, n_tokens, 2, 2, false, false, 1, false, false, false, true));
|
||||
}
|
||||
// mild decay keeps the state alive across the sequence; qwen35 prefill layout with K=5 snapshots
|
||||
for (int64_t n : { 5, 300, 2048 }) {
|
||||
test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 16, 128, n, 1, 3, false, false, /*K=*/5, false, false, false, false, /*g_min=*/-0.05f));
|
||||
}
|
||||
test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 4, 128, 300, 2, 2, true, false, /*K=*/1, false, false, false, false, /*g_min=*/-0.05f));
|
||||
test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 4, 64, 200, 1, 1, false, false, /*K=*/5, false, false, false, false, /*g_min=*/-0.05f));
|
||||
// a backend may pick its kernel by head count: few and many heads, per head size, plain and KDA
|
||||
for (int64_t s : { 16, 32, 64, 128 }) {
|
||||
for (int64_t h : { 4 * 128 / s, 64 * 128 / s }) {
|
||||
test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, h, s, 37, 1, 1, false, false, /*K=*/4, false, false, false, false, /*g_min=*/-0.05f));
|
||||
test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, h, s, 37, 1, 1, false, true, /*K=*/1, false, false, false, false, /*g_min=*/-0.05f));
|
||||
}
|
||||
}
|
||||
|
||||
// K > 1: output keeps the last min(n_tokens, K) per-token snapshots, ordered most-recent-first
|
||||
// (slot 0 = final state, slot s = state s tokens back).
|
||||
@@ -13781,6 +13800,7 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() {
|
||||
test_cases.emplace_back(new test_gated_delta_net_cache_fusion(GGML_TYPE_F32, 4, 64, 4, 1, 2));
|
||||
test_cases.emplace_back(new test_gated_delta_net_cache_fusion(GGML_TYPE_F32, 4, 32, 4, 1, 4));
|
||||
test_cases.emplace_back(new test_gated_delta_net_cache_fusion(GGML_TYPE_F32, 8, 32, 4, 2, 4));
|
||||
test_cases.emplace_back(new test_gated_delta_net_cache_fusion(GGML_TYPE_F32, 128, 64, 4, 1, 4));
|
||||
test_cases.emplace_back(new test_gated_delta_net_cache_fusion(GGML_TYPE_F32, 4, 32, 8, 1, 4));
|
||||
|
||||
#if 0
|
||||
@@ -14444,6 +14464,7 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_perf() {
|
||||
test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 4, 128, 8192, 1));
|
||||
test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 20, 128, 1536, 1));
|
||||
test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 20, 128, 2048, 1));
|
||||
test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 16, 128, 2048, 1, 3, false, false, 5)); // qwen35 PP-2048, K=5
|
||||
|
||||
// lightning_indexer
|
||||
for (int kv : { 256, 4096, 65536 }) {
|
||||
|
||||
Reference in new issue
Block a user