llama: limit max outputs of llama_context (#23861)

* llama: save more VRAM by reserving n_outputs == n_seqs when possible * add n_outputs_per_seq * move n_outputs_max to server-context * change ubatch to batch everywhere
2026-06-01 18:01:38 +03:00
parent 95b8b8ec1a
commit de6f727aae
6 changed files with 71 additions and 11 deletions
@@ -13,6 +13,7 @@ struct llama_cparams {
    uint32_t n_ubatch;
    uint32_t n_seq_max;
    uint32_t n_rs_seq;        // number of recurrent-state snapshots per seq for rollback
+    uint32_t n_outputs_max;   // max outputs supported by the context
    int32_t  n_threads;       // number of threads to use for generation
    int32_t  n_threads_batch; // number of threads to use for batch processing