spec: Add benchmark-only synthetic speculative acceptance options (#27711)
* Add benchmark-only synthetic speculative acceptance to llama-server and llama-cli * Address review comments * Address review comments * Add some comments in the code
This commit is contained in:
@@ -99,6 +99,8 @@ class ServerProcess:
|
||||
spec_type: str | None = None
|
||||
spec_draft_n_min: int | None = None
|
||||
spec_draft_n_max: int | None = None
|
||||
spec_synth_len: float | None = None
|
||||
spec_synth_rates: List[float] | None = None
|
||||
no_ui: bool | None = None
|
||||
jinja: bool | None = None
|
||||
reasoning_format: Literal['deepseek', 'none', 'nothink'] | None = None
|
||||
@@ -245,6 +247,11 @@ class ServerProcess:
|
||||
server_args.extend(["--spec-draft-n-max", self.spec_draft_n_max])
|
||||
if self.spec_draft_n_min:
|
||||
server_args.extend(["--spec-draft-n-min", self.spec_draft_n_min])
|
||||
if self.spec_synth_len is not None:
|
||||
server_args.extend(["--spec-synth-len", self.spec_synth_len])
|
||||
if self.spec_synth_rates is not None:
|
||||
rates = ",".join(str(rate) for rate in self.spec_synth_rates)
|
||||
server_args.extend(["--spec-synth-rates", rates])
|
||||
if self.no_ui:
|
||||
server_args.append("--no-ui")
|
||||
if self.no_models_autoload:
|
||||
|
||||
Reference in New Issue
Block a user