sampler : remove "full-context windows" from history-based samplers (#26524)
* Resolve -1 to 1024 instead of ctx-len for samplers Because of backend-sampling we initialize samplers before the complete llama_context is there. Therefore, we cannot infer the resolved context length yet at the time we construct the samplers. * Shared default of 64 for history-based samplers, remove context_size
This commit is contained in:
@@ -1807,8 +1807,7 @@ private:
|
||||
// initialize samplers
|
||||
if (task.need_sampling()) {
|
||||
try {
|
||||
slot.smpl.reset(common_sampler_init(
|
||||
model_tgt, task.params.sampling, (int32_t) llama_n_ctx(ctx_tgt)));
|
||||
slot.smpl.reset(common_sampler_init(model_tgt, task.params.sampling));
|
||||
} catch (std::exception & e) {
|
||||
std::string err_msg = std::string("Failed to initialize samplers: ") + e.what();
|
||||
send_error(task, err_msg, ERROR_TYPE_INVALID_REQUEST);
|
||||
@@ -4148,7 +4147,6 @@ std::unique_ptr<server_res_generator> server_routes::handle_completions_impl(
|
||||
task.params = server_schema::eval_llama_cmpl_schema(
|
||||
ctx_server.vocab,
|
||||
params,
|
||||
meta->slot_n_ctx,
|
||||
meta->logit_bias_eog,
|
||||
data);
|
||||
|
||||
|
||||
Reference in New Issue
Block a user