llama: properly handle KV on training (#28520)

* llama: properly handle KV on training

* improve
This commit is contained in:
Xuan-Son Nguyen authored and GitHub committed 2026-09-30 18:09:08 +02:00
1 parent 60e9cf7a7b
commit 81ff93ea1d
3 files changed
+61 -21

No files matched your search

+22 -5
View File
@@ -275,6 +275,7 @@ llama_context::llama_context(
// initialized later
cparams.pipeline_parallel = false;
cparams.training = false;
{
const char * LLAMA_GRAPH_REUSE_DISABLE = getenv("LLAMA_GRAPH_REUSE_DISABLE");
@@ -691,7 +692,11 @@ void llama_context::sched_reserve() {
}
// reserve with tg (token generation) graph to get the number of splits and nodes
{
if (cparams.training) {
// no tg graph for training
n_splits_tg = n_splits_pp;
n_nodes_tg = n_nodes_pp;
} else {
auto * gf = graph_reserve(n_seqs, n_seqs, n_seqs, mctx.get(), model.hparams.no_alloc);
if (!gf) {
throw std::runtime_error("failed to allocate compute tg buffers");
@@ -2410,6 +2415,11 @@ uint32_t llama_context::graph_max_nodes(uint32_t n_tokens) const {
if (n_sampling_outputs_max > 1) {
res += (n_sampling_outputs_max - 1) * n_sampling_nodes_max;
}
if (cparams.training) {
res *= 4;
}
return res;
}
@@ -3500,12 +3510,19 @@ void llama_context::opt_init(struct llama_model * model, struct llama_opt_params
if (cparams.flash_attn) {
LLAMA_LOG_INFO("%s: disabling flash attention, FLASH_ATTN_EXT has no backward pass\n", __func__);
cparams.flash_attn = false;
// the graph changes without flash attention, need to reserve again
sched_need_reserve = true;
sched_reserve();
}
// gradients cannot flow through the KV cache, so the attention reads the K and V of the current ubatch directly
if (n_ubatch == cparams.n_ctx) {
cparams.training = true;
} else {
LLAMA_LOG_WARN("%s: n_ubatch (%u) != n_ctx (%u), the K and V projections will not receive gradients\n", __func__, n_ubatch, cparams.n_ctx);
}
// the training graph is different, need to reserve again
sched_need_reserve = true;
sched_reserve();
ggml_opt_params opt_params = ggml_opt_default_params(sched.get(), GGML_OPT_LOSS_TYPE_CROSS_ENTROPY);
opt_params.opt_period = n_batch / n_ubatch;
opt_params.get_opt_pars = lopt_params.get_opt_pars;