common : add --spec-draft-n-ubatch (#30459)
* common : add --spec-draft-n-ubatch Allow configuring the physical batch size of the draft context. The drafters feed it at most the drafted tokens per sequence and chunk larger batches by its own n_ubatch, so it does not need the ubatch of the target. A smaller value shrinks the draft compute buffer and leaves more VRAM for the target KV cache (the draft is fitted together with the target). The default 0 keeps the current behavior of following --n-ubatch. Assisted-by: pi:llama.cpp/Qwen3.8-Flash-Next * cont : remove comment [no ci]
This commit is contained in:
1 parent
318a471a62
commit
3fb47677ea
9 files changed
+63
-3
No files matched your search
@@ -95,10 +95,20 @@ static void test(void) {
|
||||
common_params base;
|
||||
base.n_parallel = 4;
|
||||
base.n_outputs_max_per_seq = 8;
|
||||
base.n_ubatch = 512;
|
||||
|
||||
const auto draft = common_base_params_to_speculative(base);
|
||||
assert(draft.n_outputs_max == 4);
|
||||
assert(draft.n_outputs_max_per_seq == 1);
|
||||
{
|
||||
const auto draft = common_base_params_to_speculative(base);
|
||||
assert(draft.n_outputs_max == 4);
|
||||
assert(draft.n_outputs_max_per_seq == 1);
|
||||
assert(draft.n_ubatch == 512); // inherits the target by default
|
||||
}
|
||||
|
||||
base.speculative.draft.n_ubatch = 8;
|
||||
{
|
||||
const auto draft = common_base_params_to_speculative(base);
|
||||
assert(draft.n_ubatch == 8);
|
||||
}
|
||||
}
|
||||
|
||||
printf("test-arg-parser: make sure there is no duplicated arguments in any examples\n\n");
|
||||
|
||||
Reference in new issue
Block a user