llama: properly handle KV on training (#28520)
* llama: properly handle KV on training * improve
This commit is contained in:
1 parent
60e9cf7a7b
commit
81ff93ea1d
3 files changed
+61
-21
No files matched your search
+22
-5
@@ -275,6 +275,7 @@ llama_context::llama_context(
|
||||
|
||||
// initialized later
|
||||
cparams.pipeline_parallel = false;
|
||||
cparams.training = false;
|
||||
|
||||
{
|
||||
const char * LLAMA_GRAPH_REUSE_DISABLE = getenv("LLAMA_GRAPH_REUSE_DISABLE");
|
||||
@@ -691,7 +692,11 @@ void llama_context::sched_reserve() {
|
||||
}
|
||||
|
||||
// reserve with tg (token generation) graph to get the number of splits and nodes
|
||||
{
|
||||
if (cparams.training) {
|
||||
// no tg graph for training
|
||||
n_splits_tg = n_splits_pp;
|
||||
n_nodes_tg = n_nodes_pp;
|
||||
} else {
|
||||
auto * gf = graph_reserve(n_seqs, n_seqs, n_seqs, mctx.get(), model.hparams.no_alloc);
|
||||
if (!gf) {
|
||||
throw std::runtime_error("failed to allocate compute tg buffers");
|
||||
@@ -2410,6 +2415,11 @@ uint32_t llama_context::graph_max_nodes(uint32_t n_tokens) const {
|
||||
if (n_sampling_outputs_max > 1) {
|
||||
res += (n_sampling_outputs_max - 1) * n_sampling_nodes_max;
|
||||
}
|
||||
|
||||
if (cparams.training) {
|
||||
res *= 4;
|
||||
}
|
||||
|
||||
return res;
|
||||
}
|
||||
|
||||
@@ -3500,12 +3510,19 @@ void llama_context::opt_init(struct llama_model * model, struct llama_opt_params
|
||||
if (cparams.flash_attn) {
|
||||
LLAMA_LOG_INFO("%s: disabling flash attention, FLASH_ATTN_EXT has no backward pass\n", __func__);
|
||||
cparams.flash_attn = false;
|
||||
|
||||
// the graph changes without flash attention, need to reserve again
|
||||
sched_need_reserve = true;
|
||||
sched_reserve();
|
||||
}
|
||||
|
||||
// gradients cannot flow through the KV cache, so the attention reads the K and V of the current ubatch directly
|
||||
if (n_ubatch == cparams.n_ctx) {
|
||||
cparams.training = true;
|
||||
} else {
|
||||
LLAMA_LOG_WARN("%s: n_ubatch (%u) != n_ctx (%u), the K and V projections will not receive gradients\n", __func__, n_ubatch, cparams.n_ctx);
|
||||
}
|
||||
|
||||
// the training graph is different, need to reserve again
|
||||
sched_need_reserve = true;
|
||||
sched_reserve();
|
||||
|
||||
ggml_opt_params opt_params = ggml_opt_default_params(sched.get(), GGML_OPT_LOSS_TYPE_CROSS_ENTROPY);
|
||||
opt_params.opt_period = n_batch / n_ubatch;
|
||||
opt_params.get_opt_pars = lopt_params.get_opt_pars;
|
||||
|
||||
Reference in new issue
Block a user