sampler : remove "full-context windows" from history-based samplers (#26524)
* Resolve -1 to 1024 instead of ctx-len for samplers Because of backend-sampling we initialize samplers before the complete llama_context is there. Therefore, we cannot infer the resolved context length yet at the time we construct the samplers. * Shared default of 64 for history-based samplers, remove context_size
This commit is contained in:
Oliver Simons
+2
-2
@@ -116,14 +116,14 @@
|
||||
| `--xtc-probability N` | xtc probability (default: 0.00, 0.0 = disabled) |
|
||||
| `--xtc-threshold N` | xtc threshold (default: 0.10, 1.0 = disabled) |
|
||||
| `--typical, --typical-p N` | locally typical sampling, parameter p (default: 1.00, 1.0 = disabled) |
|
||||
| `--repeat-last-n N` | last n tokens to consider for penalize (default: 64, 0 = disabled, -1 = ctx_size) |
|
||||
| `--repeat-last-n N` | last n tokens to consider for penalize (default: 64, 0 = disabled) |
|
||||
| `--repeat-penalty N` | penalize repeat sequence of tokens (default: 1.00, 1.0 = disabled) |
|
||||
| `--presence-penalty N` | repeat alpha presence penalty (default: 0.00, 0.0 = disabled) |
|
||||
| `--frequency-penalty N` | repeat alpha frequency penalty (default: 0.00, 0.0 = disabled) |
|
||||
| `--dry-multiplier N` | set DRY sampling multiplier (default: 0.00, 0.0 = disabled) |
|
||||
| `--dry-base N` | set DRY sampling base value (default: 1.75) |
|
||||
| `--dry-allowed-length N` | set allowed length for DRY sampling (default: 2) |
|
||||
| `--dry-penalty-last-n N` | set DRY penalty for the last n tokens (default: -1, 0 = disable, -1 = context size) |
|
||||
| `--dry-penalty-last-n N` | set DRY penalty for the last n tokens (default: 64, 0 = disable) |
|
||||
| `--dry-sequence-breaker STRING` | add sequence breaker for DRY sampling, clearing out default breakers ('\n', ':', '"', '*') in the process; use "none" to not use any sequence breakers |
|
||||
| `--adaptive-target N` | adaptive-p: select tokens near this probability (valid range 0.0 to 1.0; negative = disabled) (default: -1.00)<br/>[(more info)](https://github.com/ggml-org/llama.cpp/pull/17927) |
|
||||
| `--adaptive-decay N` | adaptive-p: decay rate for target adaptation over time. lower values are more reactive, higher values are more stable.<br/>(valid range 0.0 to 0.99) (default: 0.90) |
|
||||
|
||||
@@ -199,14 +199,14 @@ llama-completion.exe -m models\gemma-1.1-7b-it.Q4_K_M.gguf --ignore-eos -n -1
|
||||
| `--xtc-probability N` | xtc probability (default: 0.00, 0.0 = disabled) |
|
||||
| `--xtc-threshold N` | xtc threshold (default: 0.10, 1.0 = disabled) |
|
||||
| `--typical, --typical-p N` | locally typical sampling, parameter p (default: 1.00, 1.0 = disabled) |
|
||||
| `--repeat-last-n N` | last n tokens to consider for penalize (default: 64, 0 = disabled, -1 = ctx_size) |
|
||||
| `--repeat-last-n N` | last n tokens to consider for penalize (default: 64, 0 = disabled) |
|
||||
| `--repeat-penalty N` | penalize repeat sequence of tokens (default: 1.00, 1.0 = disabled) |
|
||||
| `--presence-penalty N` | repeat alpha presence penalty (default: 0.00, 0.0 = disabled) |
|
||||
| `--frequency-penalty N` | repeat alpha frequency penalty (default: 0.00, 0.0 = disabled) |
|
||||
| `--dry-multiplier N` | set DRY sampling multiplier (default: 0.00, 0.0 = disabled) |
|
||||
| `--dry-base N` | set DRY sampling base value (default: 1.75) |
|
||||
| `--dry-allowed-length N` | set allowed length for DRY sampling (default: 2) |
|
||||
| `--dry-penalty-last-n N` | set DRY penalty for the last n tokens (default: -1, 0 = disable, -1 = context size) |
|
||||
| `--dry-penalty-last-n N` | set DRY penalty for the last n tokens (default: 64, 0 = disable) |
|
||||
| `--dry-sequence-breaker STRING` | add sequence breaker for DRY sampling, clearing out default breakers ('\n', ':', '"', '*') in the process; use "none" to not use any sequence breakers |
|
||||
| `--adaptive-target N` | adaptive-p: select tokens near this probability (valid range 0.0 to 1.0; negative = disabled) (default: -1.00)<br/>[(more info)](https://github.com/ggml-org/llama.cpp/pull/17927) |
|
||||
| `--adaptive-decay N` | adaptive-p: decay rate for target adaptation over time. lower values are more reactive, higher values are more stable.<br/>(valid range 0.0 to 0.99) (default: 0.90) |
|
||||
@@ -388,11 +388,11 @@ Example usage: `--temp 0`
|
||||
### Repeat Penalty
|
||||
|
||||
- `--repeat-penalty N`: Control the repetition of token sequences in the generated text default: 1.0, 1.0 = disabled).
|
||||
- `--repeat-last-n N`: Last n tokens to consider for penalizing repetition (default: 64, 0 = disabled, -1 = ctx-size).
|
||||
- `--repeat-last-n N`: Last n tokens to consider for penalizing repetition (default: 64, 0 = disabled).
|
||||
|
||||
The `repeat-penalty` option helps prevent the model from generating repetitive or monotonous text. A higher value (e.g., 1.5) will penalize repetitions more strongly, while a lower value (e.g., 0.9) will be more lenient. The default value is 1.
|
||||
|
||||
The `repeat-last-n` option controls the number of tokens in the history to consider for penalizing repetition. A larger value will look further back in the generated text to prevent repetitions, while a smaller value will only consider recent tokens. A value of 0 disables the penalty, and a value of -1 sets the number of tokens considered equal to the context size (`ctx-size`).
|
||||
The `repeat-last-n` option controls the number of tokens in the history to consider for penalizing repetition. A larger value will look further back in the generated text to prevent repetitions, while a smaller value will only consider recent tokens. A value of 0 disables the penalty.
|
||||
|
||||
### DRY Repetition Penalty
|
||||
|
||||
@@ -401,7 +401,7 @@ DRY (Don't Repeat Yourself) sampling is an effective technique for reducing repe
|
||||
- `--dry-multiplier N`: Set the DRY sampling multiplier (default: 0.0, 0.0 = disabled).
|
||||
- `--dry-base N`: Set the DRY sampling base value (default: 1.75).
|
||||
- `--dry-allowed-length N`: Set the allowed length for DRY sampling (default: 2).
|
||||
- `--dry-penalty-last-n N`: Set DRY penalty for the last n tokens (default: -1, 0 = disable, -1 = context size).
|
||||
- `--dry-penalty-last-n N`: Set DRY penalty for the last n tokens (default: 64, 0 = disable).
|
||||
- `--dry-sequence-breaker STRING`: Add a sequence breaker for DRY sampling. Can be used more than once to add multiple sequence breakers. Using this clears out the default breakers, which consist of: `['\n', ':', '"', '*']`. If the string `"none"` is supplied, no sequence breakers are used.
|
||||
|
||||
The `dry-multiplier` option controls the strength of the DRY sampling effect. A value of 0.0 disables DRY sampling, while higher values increase its influence. A typical recommended value is 0.8.
|
||||
@@ -410,13 +410,13 @@ The `dry-base` option sets the base value for the exponential penalty calculatio
|
||||
|
||||
The `dry-allowed-length` option sets the maximum length of repeated sequences that will not be penalized. Repetitions shorter than or equal to this length are not penalized, allowing for natural repetitions of short phrases or common words.
|
||||
|
||||
The `dry-penalty-last-n` option controls how many recent tokens to consider when applying the DRY penalty. A value of -1 considers the entire context. Use a positive value to limit the consideration to a specific number of recent tokens.
|
||||
The `dry-penalty-last-n` option controls how many recent tokens to consider when applying the DRY penalty. A value of 0 disables the penalty. Use a positive value to limit the consideration to a specific number of recent tokens.
|
||||
|
||||
The `dry-sequence-breaker` option adds a single sequence breaker and can be used more than once to specify multiple sequence breakers. Sequence breakers interrupt sequence matching and break the input into parts where matching can be applied.
|
||||
|
||||
DRY sampling provides more nuanced control over text generation, particularly for reducing long-range repetitions and maintaining global coherence.
|
||||
|
||||
Example usage: `--dry-multiplier 0.8 --dry-base 1.75 --dry-allowed-length 2 --dry-penalty-last-n -1 --dry-sequence-breaker "—" --dry-sequence-breaker "##"`
|
||||
Example usage: `--dry-multiplier 0.8 --dry-base 1.75 --dry-allowed-length 2 --dry-penalty-last-n 64 --dry-sequence-breaker "—" --dry-sequence-breaker "##"`
|
||||
|
||||
### Top-K Sampling
|
||||
|
||||
|
||||
@@ -215,7 +215,7 @@ def run_mtmd_cli(spec: "ModelSpec", model_path, mmproj_path, image_path, bin_pat
|
||||
"--dry-multiplier", "0.8",
|
||||
"--dry-base", "1.75",
|
||||
"--dry-allowed-length", "2",
|
||||
"--dry-penalty-last-n", "-1",
|
||||
"--dry-penalty-last-n", "64",
|
||||
"--dry-sequence-breaker", "none",
|
||||
]
|
||||
if spec.n_ctx is not None:
|
||||
|
||||
@@ -133,14 +133,14 @@ For the full list of features, please refer to [server's changelog](https://gith
|
||||
| `--xtc-probability N` | xtc probability (default: 0.00, 0.0 = disabled) |
|
||||
| `--xtc-threshold N` | xtc threshold (default: 0.10, 1.0 = disabled) |
|
||||
| `--typical, --typical-p N` | locally typical sampling, parameter p (default: 1.00, 1.0 = disabled) |
|
||||
| `--repeat-last-n N` | last n tokens to consider for penalize (default: 64, 0 = disabled, -1 = ctx_size) |
|
||||
| `--repeat-last-n N` | last n tokens to consider for penalize (default: 64, 0 = disabled) |
|
||||
| `--repeat-penalty N` | penalize repeat sequence of tokens (default: 1.00, 1.0 = disabled) |
|
||||
| `--presence-penalty N` | repeat alpha presence penalty (default: 0.00, 0.0 = disabled) |
|
||||
| `--frequency-penalty N` | repeat alpha frequency penalty (default: 0.00, 0.0 = disabled) |
|
||||
| `--dry-multiplier N` | set DRY sampling multiplier (default: 0.00, 0.0 = disabled) |
|
||||
| `--dry-base N` | set DRY sampling base value (default: 1.75) |
|
||||
| `--dry-allowed-length N` | set allowed length for DRY sampling (default: 2) |
|
||||
| `--dry-penalty-last-n N` | set DRY penalty for the last n tokens (default: -1, 0 = disable, -1 = context size) |
|
||||
| `--dry-penalty-last-n N` | set DRY penalty for the last n tokens (default: 64, 0 = disable) |
|
||||
| `--dry-sequence-breaker STRING` | add sequence breaker for DRY sampling, clearing out default breakers ('\n', ':', '"', '*') in the process; use "none" to not use any sequence breakers |
|
||||
| `--adaptive-target N` | adaptive-p: select tokens near this probability (valid range 0.0 to 1.0; negative = disabled) (default: -1.00)<br/>[(more info)](https://github.com/ggml-org/llama.cpp/pull/17927) |
|
||||
| `--adaptive-decay N` | adaptive-p: decay rate for target adaptation over time. lower values are more reactive, higher values are more stable.<br/>(valid range 0.0 to 0.99) (default: 0.90) |
|
||||
@@ -476,7 +476,7 @@ These words will not be included in the completion, so make sure to add them to
|
||||
|
||||
`repeat_penalty`: Control the repetition of token sequences in the generated text. Default: `1.1`
|
||||
|
||||
`repeat_last_n`: Last n tokens to consider for penalizing repetition. Default: `64`, where `0` is disabled and `-1` is ctx-size.
|
||||
`repeat_last_n`: Last n tokens to consider for penalizing repetition. Default: `64`, where `0` is disabled.
|
||||
|
||||
`presence_penalty`: Repeat alpha presence penalty. Default: `0.0`, which is disabled.
|
||||
|
||||
@@ -488,7 +488,7 @@ These words will not be included in the completion, so make sure to add them to
|
||||
|
||||
`dry_allowed_length`: Tokens that extend repetition beyond this receive exponentially increasing penalty: multiplier * base ^ (length of repeating sequence before token - allowed length). Default: `2`
|
||||
|
||||
`dry_penalty_last_n`: How many tokens to scan for repetitions. Default: `-1`, where `0` is disabled and `-1` is context size.
|
||||
`dry_penalty_last_n`: How many tokens to scan for repetitions. Default: `64`, where `0` is disabled.
|
||||
|
||||
`dry_sequence_breakers`: Specify an array of sequence breakers for DRY sampling. Only a JSON array of strings is accepted. Default: `['\n', ':', '"', '*']`
|
||||
|
||||
@@ -796,7 +796,7 @@ By default, it is read-only. To make POST request to change global properties, y
|
||||
"dry_multiplier": 0.0,
|
||||
"dry_base": 1.75,
|
||||
"dry_allowed_length": 2,
|
||||
"dry_penalty_last_n": -1,
|
||||
"dry_penalty_last_n": 64,
|
||||
"dry_sequence_breakers": [
|
||||
"\n",
|
||||
":",
|
||||
|
||||
@@ -1807,8 +1807,7 @@ private:
|
||||
// initialize samplers
|
||||
if (task.need_sampling()) {
|
||||
try {
|
||||
slot.smpl.reset(common_sampler_init(
|
||||
model_tgt, task.params.sampling, (int32_t) llama_n_ctx(ctx_tgt)));
|
||||
slot.smpl.reset(common_sampler_init(model_tgt, task.params.sampling));
|
||||
} catch (std::exception & e) {
|
||||
std::string err_msg = std::string("Failed to initialize samplers: ") + e.what();
|
||||
send_error(task, err_msg, ERROR_TYPE_INVALID_REQUEST);
|
||||
@@ -4148,7 +4147,6 @@ std::unique_ptr<server_res_generator> server_routes::handle_completions_impl(
|
||||
task.params = server_schema::eval_llama_cmpl_schema(
|
||||
ctx_server.vocab,
|
||||
params,
|
||||
meta->slot_n_ctx,
|
||||
meta->logit_bias_eog,
|
||||
data);
|
||||
|
||||
|
||||
@@ -124,8 +124,8 @@ std::vector<std::unique_ptr<field>> make_llama_cmpl_schema(const common_params &
|
||||
->set_desc("Dynamic temperature exponent, controls how entropy maps to temperature"));
|
||||
|
||||
add((new field_num("repeat_last_n", params.sampling.penalty_last_n))
|
||||
->set_hard_limits(-1, INT32_MAX)
|
||||
->set_desc("Last n tokens to consider for penalizing repetition (0 = disabled, -1 = ctx-size)"));
|
||||
->set_hard_limits(0, INT32_MAX)
|
||||
->set_desc("Last n tokens to consider for penalizing repetition (0 = disabled)"));
|
||||
|
||||
add((new field_num("repeat_penalty", params.sampling.penalty_repeat))
|
||||
->set_desc("Control the repetition of token sequences in the generated text (1.0 = disabled)"));
|
||||
@@ -151,8 +151,8 @@ std::vector<std::unique_ptr<field>> make_llama_cmpl_schema(const common_params &
|
||||
->set_desc("Tokens that extend repetition beyond this length receive exponentially increasing penalty: multiplier * base ^ (sequence_length - allowed_length)"));
|
||||
|
||||
add((new field_num("dry_penalty_last_n", params.sampling.dry_penalty_last_n))
|
||||
->set_hard_limits(-1, INT32_MAX)
|
||||
->set_desc("How many tokens to scan for repetitions (0 = disabled, -1 = context size)"));
|
||||
->set_hard_limits(0, INT32_MAX)
|
||||
->set_desc("How many tokens to scan for repetitions (0 = disabled)"));
|
||||
|
||||
add((new field_num("mirostat", params.sampling.mirostat))
|
||||
->set_limits(0, 2)
|
||||
@@ -515,7 +515,6 @@ std::vector<std::unique_ptr<field>> make_llama_cmpl_schema(const common_params &
|
||||
task_params eval_llama_cmpl_schema(
|
||||
const llama_vocab * vocab,
|
||||
const common_params & params_base,
|
||||
const int n_ctx_slot,
|
||||
const std::vector<llama_logit_bias> & logit_bias_eog,
|
||||
const json & data) {
|
||||
task_params params;
|
||||
@@ -549,15 +548,6 @@ task_params eval_llama_cmpl_schema(
|
||||
|
||||
// post-processing
|
||||
{
|
||||
if (params.sampling.penalty_last_n == -1) {
|
||||
// note: should be the slot's context and not the full context, but it's ok
|
||||
params.sampling.penalty_last_n = n_ctx_slot;
|
||||
}
|
||||
|
||||
if (params.sampling.dry_penalty_last_n == -1) {
|
||||
params.sampling.dry_penalty_last_n = n_ctx_slot;
|
||||
}
|
||||
|
||||
// if "reasoning_format" is not provided, its handler will not be called, we will need to handle it here
|
||||
auto reasoning_format = params.chat_parser_params.reasoning_format;
|
||||
params.chat_parser_params.reasoning_in_content = params.stream && (reasoning_format == COMMON_REASONING_FORMAT_DEEPSEEK_LEGACY);
|
||||
|
||||
@@ -98,7 +98,6 @@ std::vector<std::unique_ptr<field>> make_llama_cmpl_schema(
|
||||
task_params eval_llama_cmpl_schema(
|
||||
const llama_vocab * vocab,
|
||||
const common_params & params_base,
|
||||
const int n_ctx_slot,
|
||||
const std::vector<llama_logit_bias> & logit_bias_eog,
|
||||
const json & data);
|
||||
|
||||
|
||||
@@ -30,7 +30,7 @@ describe('ParameterSyncService', () => {
|
||||
dry_multiplier: 0.0,
|
||||
dry_base: 1.75,
|
||||
dry_allowed_length: 2,
|
||||
dry_penalty_last_n: -1,
|
||||
dry_penalty_last_n: 64,
|
||||
mirostat: 0,
|
||||
mirostat_tau: 5.0,
|
||||
mirostat_eta: 0.1,
|
||||
@@ -96,7 +96,7 @@ describe('ParameterSyncService', () => {
|
||||
dry_multiplier: 0.0,
|
||||
dry_base: 1.75,
|
||||
dry_allowed_length: 2,
|
||||
dry_penalty_last_n: -1,
|
||||
dry_penalty_last_n: 64,
|
||||
mirostat: 0,
|
||||
mirostat_tau: 5.0,
|
||||
mirostat_eta: 0.1,
|
||||
|
||||
Reference in New Issue
Block a user