server : enable packing for d1-2B and d1-Omni-2B models (#32553)

* llama-bench : size the batch to the prompt for models without memory

Assisted-by: Claude Opus 5.5

* server : read up to 16 lfm3-d1 questions in one pass

Assisted-by: Claude Opus 5.5
This commit is contained in:
Tarek Dakhran authored and GitHub committed 2026-11-18 20:15:43 +01:00
1 parent 2cbfc3cf34
commit bafa0420c1
14 files changed
+220 -31

No files matched your search

+24
View File
@@ -259,6 +259,10 @@ bool llama_batch_allocr::init(
decision_order.resize(n_tok, 0);
decision_order[i] = batch_inp.tokens[i].decision_order;
}
if (batch_inp.tokens[i].decision_group != 0) {
decision_group.resize(n_tok, 0);
decision_group[i] = batch_inp.tokens[i].decision_group;
}
}
//
@@ -353,6 +357,7 @@ bool llama_batch_allocr::init(
/*.output =*/ batch.logits,
/*.type =*/ is_embd_vec.empty() ? nullptr : is_embd_vec.data(),
/*.decision_order =*/ decision_order.empty() ? nullptr : decision_order.data(),
/*.decision_group =*/ decision_group.empty() ? nullptr : decision_group.data(),
/*.data =*/ {},
};
@@ -579,6 +584,7 @@ llama_ubatch llama_batch_allocr::ubatch_reserve(uint32_t n_seq_tokens, uint32_t
/*.output =*/ udata->output.data(),
/*.type =*/ nullptr,
/*.decision_order =*/ nullptr,
/*.decision_group =*/ nullptr,
/*.data =*/ std::move(udata),
};
@@ -887,6 +893,7 @@ void llama_batch_allocr::clear() {
seq_id_unq .clear();
output .clear();
decision_order.clear();
decision_group.clear();
for (auto & cur : seq_pos) {
cur.clear();
@@ -943,6 +950,7 @@ llama_ubatch llama_batch_allocr::ubatch_add(const std::vector<int32_t> & idxs, u
udata->output .resize(n_tokens);
udata->type .resize(mixed ? n_tokens : 0);
udata->decision_order.resize(decision_order.empty() ? 0 : n_tokens);
udata->decision_group.resize(decision_group.empty() ? 0 : n_tokens);
udata->batch_idxs = idxs;
udata->seq_id_data.reserve(n_tokens);
@@ -984,6 +992,9 @@ llama_ubatch llama_batch_allocr::ubatch_add(const std::vector<int32_t> & idxs, u
if (!decision_order.empty()) {
udata->decision_order[i] = decision_order[idxs[i]];
}
if (!decision_group.empty()) {
udata->decision_group[i] = decision_group[idxs[i]];
}
for (int s = 0; s < udata->n_seq_id[i]; ++s) {
const llama_seq_id seq_id = batch.seq_id[idxs[i]][s];
@@ -1030,6 +1041,7 @@ llama_ubatch llama_batch_allocr::ubatch_add(const std::vector<int32_t> & idxs, u
/*.output =*/ udata->output.data(),
/*.type =*/ mixed ? udata->type.data() : nullptr,
/*.decision_order =*/ udata->decision_order.empty() ? nullptr : udata->decision_order.data(),
/*.decision_group =*/ udata->decision_group.empty() ? nullptr : udata->decision_group.data(),
/*.data =*/ std::move(udata),
};
@@ -1430,6 +1442,14 @@ bool llama_batch_ext::set_decision_order(int32_t idx, int32_t order) {
return true;
}
bool llama_batch_ext::set_decision_group(int32_t idx, int32_t group) {
if (idx < 0 || idx >= (int32_t) tokens.size() || group < 0) {
return false;
}
tokens[idx].decision_group = group;
return true;
}
// llama_batch_ext C API
llama_batch_ext * llama_batch_ext_init(llama_context * ctx) {
@@ -1513,6 +1533,10 @@ bool llama_batch_ext_set_decision_order(llama_batch_ext * batch, int32_t idx, ll
return batch->set_decision_order(idx, order);
}
bool llama_batch_ext_set_decision_group(llama_batch_ext * batch, int32_t idx, int32_t group) {
return batch->set_decision_group(idx, group);
}
// llama_batch_compat
void llama_batch_compat::init(llama_batch_ext & dst, const llama_batch & batch_inp, size_t n_embd_row) {