llama: properly handle KV on training (#28520)
* llama: properly handle KV on training * improve
This commit is contained in:
1 parent
60e9cf7a7b
commit
81ff93ea1d
3 files changed
+61
-21
No files matched your search
+38
-16
@@ -468,8 +468,13 @@ void llm_graph_input_attn_no_cache::set_input(const llama_ubatch * ubatch) {
|
||||
}
|
||||
|
||||
void llm_graph_input_attn_kv::set_input(const llama_ubatch * ubatch) {
|
||||
mctx->set_input_k_idxs(self_k_idxs, ubatch);
|
||||
mctx->set_input_v_idxs(self_v_idxs, ubatch);
|
||||
// the idxs are left unallocated when the KV cache is bypassed during training
|
||||
if (self_k_idxs && self_k_idxs->buffer) {
|
||||
mctx->set_input_k_idxs(self_k_idxs, ubatch);
|
||||
}
|
||||
if (self_v_idxs && self_v_idxs->buffer) {
|
||||
mctx->set_input_v_idxs(self_v_idxs, ubatch);
|
||||
}
|
||||
|
||||
// the mask is left unallocated when the graph only stores K/V without attending
|
||||
// (e.g. DFlash's KV-injection pass)
|
||||
@@ -2893,21 +2898,30 @@ ggml_tensor * llm_graph_context::build_attn(
|
||||
|
||||
const auto * mctx_cur = inp->mctx;
|
||||
|
||||
// store to KV cache
|
||||
{
|
||||
const auto & k_idxs = inp->get_k_idxs();
|
||||
const auto & v_idxs = inp->get_v_idxs();
|
||||
ggml_tensor * q = q_cur;
|
||||
ggml_tensor * k;
|
||||
ggml_tensor * v;
|
||||
|
||||
ggml_build_forward_expand(gf, mctx_cur->cpy_k(ctx0, k_cur, k_idxs, il));
|
||||
ggml_build_forward_expand(gf, mctx_cur->cpy_v(ctx0, v_cur, v_idxs, il));
|
||||
if (cparams.training) {
|
||||
GGML_ASSERT(mctx_cur->get_n_kv() == n_tokens);
|
||||
|
||||
k = k_cur;
|
||||
v = v_cur;
|
||||
} else {
|
||||
{
|
||||
const auto & k_idxs = inp->get_k_idxs();
|
||||
const auto & v_idxs = inp->get_v_idxs();
|
||||
|
||||
ggml_build_forward_expand(gf, mctx_cur->cpy_k(ctx0, k_cur, k_idxs, il));
|
||||
ggml_build_forward_expand(gf, mctx_cur->cpy_v(ctx0, v_cur, v_idxs, il));
|
||||
}
|
||||
|
||||
k = mctx_cur->get_k(ctx0, il);
|
||||
v = mctx_cur->get_v(ctx0, il);
|
||||
}
|
||||
|
||||
ggml_tensor * kq_mask = inp->get_kq_mask();
|
||||
|
||||
ggml_tensor * q = q_cur;
|
||||
ggml_tensor * k = mctx_cur->get_k(ctx0, il);
|
||||
ggml_tensor * v = mctx_cur->get_v(ctx0, il);
|
||||
|
||||
ggml_tensor * cur = build_attn_mha(q, k, v, kq_b, kq_mask, sinks, v_mla, 0, kq_scale, il);
|
||||
cb(cur, "kqv_out", il);
|
||||
|
||||
@@ -3144,14 +3158,22 @@ ggml_tensor * llm_graph_context::build_attn(
|
||||
|
||||
const auto * mctx_cur = is_swa ? mctx_iswa->get_swa() : mctx_iswa->get_base();
|
||||
|
||||
// whole seq fits into batch in training mode
|
||||
const bool use_kv_cur = cparams.training && k_cur && v_cur;
|
||||
if (use_kv_cur) {
|
||||
GGML_ASSERT(mctx_cur->get_n_kv() == n_tokens);
|
||||
}
|
||||
|
||||
const bool store_kv = !use_kv_cur || hparams.n_layer_kv_from_start >= 0;
|
||||
|
||||
// optionally store to KV cache
|
||||
if (k_cur) {
|
||||
if (store_kv && k_cur) {
|
||||
const auto & k_idxs = is_swa ? inp->get_k_idxs_swa() : inp->get_k_idxs();
|
||||
|
||||
ggml_build_forward_expand(gf, mctx_cur->cpy_k(ctx0, k_cur, k_idxs, il));
|
||||
}
|
||||
|
||||
if (v_cur) {
|
||||
if (store_kv && v_cur) {
|
||||
const auto & v_idxs = is_swa ? inp->get_v_idxs_swa() : inp->get_v_idxs();
|
||||
|
||||
ggml_build_forward_expand(gf, mctx_cur->cpy_v(ctx0, v_cur, v_idxs, il));
|
||||
@@ -3160,8 +3182,8 @@ ggml_tensor * llm_graph_context::build_attn(
|
||||
const auto & kq_mask = is_swa ? inp->get_kq_mask_swa() : inp->get_kq_mask();
|
||||
|
||||
ggml_tensor * q = q_cur;
|
||||
ggml_tensor * k = mctx_cur->get_k(ctx0, il);
|
||||
ggml_tensor * v = mctx_cur->get_v(ctx0, il);
|
||||
ggml_tensor * k = use_kv_cur ? k_cur : mctx_cur->get_k(ctx0, il);
|
||||
ggml_tensor * v = use_kv_cur ? v_cur : mctx_cur->get_v(ctx0, il);
|
||||
|
||||
ggml_tensor * cur = build_attn_mha(q, k, v, kq_b, kq_mask, sinks, v_mla, 0, kq_scale, il);
|
||||
cb(cur, "kqv_out", il);
|
||||
|
||||
Reference in new issue
Block a user