Lumpiasty/llama.cpp
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

sampler : remove "full-context windows" from history-based samplers (#26524)

Browse Source 
* Resolve -1 to 1024 instead of ctx-len for samplers

Because of backend-sampling we initialize samplers before the complete
llama_context is there. Therefore, we cannot infer the resolved context
length yet at the time we construct the samplers.

* Shared default of 64 for history-based samplers, remove context_size
This commit is contained in:
Oliver Simons
2026-08-04 21:28:55 +03:00
committed by GitHub
parent 76c956c137
commit a6aa6f5450
18 changed files with 52 additions and 80 deletions
Download Patch File Download Diff File Expand all files Collapse all files
common/arg.cpp
+4 -4
View File
@@ -2008,9 +2008,9 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
).set_sampling());
add_opt(common_arg(
{"--repeat-last-n"}, "N",
string_format("last n tokens to consider for penalize (default: %d, 0 = disabled, -1 = ctx_size)", params.sampling.penalty_last_n),
string_format("last n tokens to consider for penalize (default: %d, 0 = disabled)", params.sampling.penalty_last_n),
[](common_params & params, int value) {
if (value < -1) {
if (value < 0) {
throw std::runtime_error(string_format("error: invalid repeat-last-n = %d\n", value));
}
params.sampling.penalty_last_n = value;
@@ -2081,9 +2081,9 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
).set_sampling());
add_opt(common_arg(
{"--dry-penalty-last-n"}, "N",
string_format("set DRY penalty for the last n tokens (default: %d, 0 = disable, -1 = context size)", params.sampling.dry_penalty_last_n),
string_format("set DRY penalty for the last n tokens (default: %d, 0 = disable)", params.sampling.dry_penalty_last_n),
[](common_params & params, int value) {
if (value < -1) {
if (value < 0) {
throw std::runtime_error(string_format("error: invalid dry-penalty-last-n = %d\n", value));
}
params.sampling.dry_penalty_last_n = value;
common/common.cpp
+1 -12
View File
@@ -1302,23 +1302,12 @@ common_init_result::common_init_result(common_params & params, bool model_only)
params.sampling.logit_bias_eog.begin(), params.sampling.logit_bias_eog.end());
}
//if (params.sampling.penalty_last_n == -1) {
// LOG_TRC("%s: setting penalty_last_n to ctx_size = %d\n", __func__, llama_n_ctx(lctx));
// params.sampling.penalty_last_n = llama_n_ctx(lctx);
//}
//if (params.sampling.dry_penalty_last_n == -1) {
// LOG_TRC("%s: setting dry_penalty_last_n to ctx_size = %d\n", __func__, llama_n_ctx(lctx));
// params.sampling.dry_penalty_last_n = llama_n_ctx(lctx);
//}
// init the backend samplers as part of the context creation
pimpl->samplers.resize(cparams.n_seq_max);
pimpl->samplers_seq_config.resize(cparams.n_seq_max);
const int32_t n_ctx = cparams.n_ctx > 0 ? (int32_t) cparams.n_ctx : llama_model_n_ctx_train(model);
for (int i = 0; i < (int) cparams.n_seq_max; ++i) {
pimpl->samplers[i].reset(common_sampler_init(model, params.sampling, n_ctx));
pimpl->samplers[i].reset(common_sampler_init(model, params.sampling));
pimpl->samplers_seq_config[i] = { i, common_sampler_get(pimpl->samplers[i].get()) };
}
common/common.h
+2 -2
View File
@@ -235,14 +235,14 @@ struct common_params_sampling {
float temp = 0.80f; // <= 0.0 to sample greedily, 0.0 to not output probabilities
float dynatemp_range = 0.00f; // 0.0 = disabled
float dynatemp_exponent = 1.00f; // controls how entropy maps to temperature in dynamic temperature sampler
int32_t penalty_last_n = 64; // last n tokens to penalize (0 = disable penalty, -1 = context size)
int32_t penalty_last_n = 64; // last n tokens to penalize (0 = disable penalty)
float penalty_repeat = 1.00f; // 1.0 = disabled
float penalty_freq = 0.00f; // 0.0 = disabled
float penalty_present = 0.00f; // 0.0 = disabled
float dry_multiplier = 0.0f; // 0.0 = disabled; DRY repetition penalty for tokens extending repetition:
float dry_base = 1.75f; // 0.0 = disabled; multiplier * base ^ (length of sequence before token - allowed length)
int32_t dry_allowed_length = 2; // tokens extending repetitions beyond this receive penalty
int32_t dry_penalty_last_n = -1; // how many tokens to scan for repetitions (0 = disable penalty, -1 = context size)
int32_t dry_penalty_last_n = 64; // how many tokens to scan for repetitions (0 = disable penalty)
float adaptive_target = -1.0f; // select tokens near this probability (valid range 0.0 to 1.0; negative = disabled)
float adaptive_decay = 0.90f; // EMA decay for adaptation; history ≈ 1/(1-decay) tokens (0.0 - 0.99)
int32_t mirostat = 0; // 0 = disabled, 1 = mirostat, 2 = mirostat 2.0
common/sampling.cpp
+2 -7
View File
@@ -186,8 +186,7 @@ std::string common_params_sampling::print() const {
struct common_sampler * common_sampler_init(
const struct llama_model * model,
struct common_params_sampling & params,
int32_t n_ctx) {
struct common_params_sampling & params) {
if (!std::isfinite(params.penalty_repeat) ||
params.penalty_repeat <= 0.0f ||
!std::isfinite(1.0f/params.penalty_repeat)) {
@@ -199,10 +198,6 @@ struct common_sampler * common_sampler_init(
if (!std::isfinite(params.penalty_present)) {
throw std::invalid_argument("penalty_present must be finite");
}
if (params.penalty_last_n == -1) {
params.penalty_last_n = n_ctx > 0 ? n_ctx : llama_model_n_ctx_train(model);
}
const llama_vocab * vocab = llama_model_get_vocab(model);
llama_sampler_chain_params lparams = llama_sampler_chain_default_params();
@@ -355,7 +350,7 @@ struct common_sampler * common_sampler_init(
for (const auto & str : params.dry_sequence_breakers) {
c_breakers.push_back(str.c_str());
}
samplers.push_back(llama_sampler_init_dry(vocab, llama_model_n_ctx_train(model), params.dry_multiplier, params.dry_base, params.dry_allowed_length, params.dry_penalty_last_n, c_breakers.data(), c_breakers.size()));
samplers.push_back(llama_sampler_init_dry(vocab, params.dry_multiplier, params.dry_base, params.dry_allowed_length, params.dry_penalty_last_n, c_breakers.data(), c_breakers.size()));
}
break;
case COMMON_SAMPLER_TYPE_TOP_K:
common/sampling.h
+1 -2
View File
@@ -39,8 +39,7 @@ struct common_sampler;
// note: can mutate params in some cases
struct common_sampler * common_sampler_init(
const struct llama_model * model,
struct common_params_sampling & params,
int32_t n_ctx = 0);
struct common_params_sampling & params);
void common_sampler_free(struct common_sampler * gsmpl);