Lumpiasty/llama.cpp
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

context : autoscale n_ctx_train when yarn scaling specified (#28030)

Browse Source
This commit is contained in:
Sigbjørn Skjæret
2026-09-01 19:59:54 +03:00
committed by GitHub
parent f28493c783
commit dfc29b64eb
2 changed files with 16 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/llama-context.cpp
+13 -5
View File
@@ -126,6 +126,7 @@ llama_context::llama_context(
embd_layer_inp.resize(hparams.n_layer() + 1); embd_layer_inp.resize(hparams.n_layer() + 1);
cparams.ctx_type = params.ctx_type; cparams.ctx_type = params.ctx_type;
cparams.rope_scaling_type = params.rope_scaling_type;
cparams.pooling_type = params.pooling_type; cparams.pooling_type = params.pooling_type;
cparams.n_ctx = params.n_ctx == 0 ? hparams.n_ctx_train : params.n_ctx; cparams.n_ctx = params.n_ctx == 0 ? hparams.n_ctx_train : params.n_ctx;
@@ -160,17 +161,16 @@ llama_context::llama_context(
} }
} }
auto rope_scaling_type = params.rope_scaling_type; if (cparams.rope_scaling_type == LLAMA_ROPE_SCALING_TYPE_UNSPECIFIED) {
if (rope_scaling_type == LLAMA_ROPE_SCALING_TYPE_UNSPECIFIED) { cparams.rope_scaling_type = hparams.rope_scaling_type_train;
rope_scaling_type = hparams.rope_scaling_type_train;
} }
if (rope_scaling_type == LLAMA_ROPE_SCALING_TYPE_NONE) { if (cparams.rope_scaling_type == LLAMA_ROPE_SCALING_TYPE_NONE) {
cparams.rope_freq_scale = 1.0f; // never scale if scaling type is none cparams.rope_freq_scale = 1.0f; // never scale if scaling type is none
} }
if (cparams.yarn_ext_factor < 0.0f) { // negative indicates 'not set' if (cparams.yarn_ext_factor < 0.0f) { // negative indicates 'not set'
cparams.yarn_ext_factor = rope_scaling_type == LLAMA_ROPE_SCALING_TYPE_YARN ? 1.0f : 0.0f; cparams.yarn_ext_factor = cparams.rope_scaling_type == LLAMA_ROPE_SCALING_TYPE_YARN ? 1.0f : 0.0f;
} }
if (cparams.yarn_ext_factor != 0) { if (cparams.yarn_ext_factor != 0) {
@@ -3734,6 +3734,14 @@ llama_context * llama_init_from_model(
try { try {
auto * ctx = new llama_context(*model, params); auto * ctx = new llama_context(*model, params);
const auto & cparams = ctx->get_cparams();
if (cparams.rope_scaling_type == LLAMA_ROPE_SCALING_TYPE_YARN && cparams.rope_freq_scale != model->hparams.rope_freq_scale_train) {
LLAMA_LOG_INFO("%s: custom YaRN scaling detected, re-adjusting n_ctx_train(%u)...\n", __func__, model->hparams.n_ctx_train);
model->hparams.n_ctx_train = cparams.n_ctx_orig_yarn / cparams.rope_freq_scale;
LLAMA_LOG_INFO("%s: n_ctx_train adjusted to %u\n", __func__, model->hparams.n_ctx_train);
}
return ctx; return ctx;
} catch (const std::exception & err) { } catch (const std::exception & err) {
LLAMA_LOG_ERROR("%s: failed to initialize the context: %s\n", __func__, err.what()); LLAMA_LOG_ERROR("%s: failed to initialize the context: %s\n", __func__, err.what());
src/llama-cparams.h
+1
View File
@@ -57,6 +57,7 @@ struct llama_cparams {
std::vector<bool> embeddings_layer_inp; // [n_layer()] extract input embeddings for layer std::vector<bool> embeddings_layer_inp; // [n_layer()] extract input embeddings for layer
enum llama_context_type ctx_type; enum llama_context_type ctx_type;
enum llama_rope_scaling_type rope_scaling_type;
enum llama_pooling_type pooling_type; enum llama_pooling_type pooling_type;
ggml_backend_sched_eval_callback cb_eval; ggml_backend_sched_eval_callback cb_eval;