server : enable packing for d1-2B and d1-Omni-2B models (#32553)
* llama-bench : size the batch to the prompt for models without memory Assisted-by: Claude Opus 5.5 * server : read up to 16 lfm3-d1 questions in one pass Assisted-by: Claude Opus 5.5
This commit is contained in:
1 parent
2cbfc3cf34
commit
bafa0420c1
14 files changed
+220
-31
No files matched your search
@@ -259,6 +259,10 @@ bool llama_batch_allocr::init(
|
||||
decision_order.resize(n_tok, 0);
|
||||
decision_order[i] = batch_inp.tokens[i].decision_order;
|
||||
}
|
||||
if (batch_inp.tokens[i].decision_group != 0) {
|
||||
decision_group.resize(n_tok, 0);
|
||||
decision_group[i] = batch_inp.tokens[i].decision_group;
|
||||
}
|
||||
}
|
||||
|
||||
//
|
||||
@@ -353,6 +357,7 @@ bool llama_batch_allocr::init(
|
||||
/*.output =*/ batch.logits,
|
||||
/*.type =*/ is_embd_vec.empty() ? nullptr : is_embd_vec.data(),
|
||||
/*.decision_order =*/ decision_order.empty() ? nullptr : decision_order.data(),
|
||||
/*.decision_group =*/ decision_group.empty() ? nullptr : decision_group.data(),
|
||||
/*.data =*/ {},
|
||||
};
|
||||
|
||||
@@ -579,6 +584,7 @@ llama_ubatch llama_batch_allocr::ubatch_reserve(uint32_t n_seq_tokens, uint32_t
|
||||
/*.output =*/ udata->output.data(),
|
||||
/*.type =*/ nullptr,
|
||||
/*.decision_order =*/ nullptr,
|
||||
/*.decision_group =*/ nullptr,
|
||||
/*.data =*/ std::move(udata),
|
||||
};
|
||||
|
||||
@@ -887,6 +893,7 @@ void llama_batch_allocr::clear() {
|
||||
seq_id_unq .clear();
|
||||
output .clear();
|
||||
decision_order.clear();
|
||||
decision_group.clear();
|
||||
|
||||
for (auto & cur : seq_pos) {
|
||||
cur.clear();
|
||||
@@ -943,6 +950,7 @@ llama_ubatch llama_batch_allocr::ubatch_add(const std::vector<int32_t> & idxs, u
|
||||
udata->output .resize(n_tokens);
|
||||
udata->type .resize(mixed ? n_tokens : 0);
|
||||
udata->decision_order.resize(decision_order.empty() ? 0 : n_tokens);
|
||||
udata->decision_group.resize(decision_group.empty() ? 0 : n_tokens);
|
||||
|
||||
udata->batch_idxs = idxs;
|
||||
udata->seq_id_data.reserve(n_tokens);
|
||||
@@ -984,6 +992,9 @@ llama_ubatch llama_batch_allocr::ubatch_add(const std::vector<int32_t> & idxs, u
|
||||
if (!decision_order.empty()) {
|
||||
udata->decision_order[i] = decision_order[idxs[i]];
|
||||
}
|
||||
if (!decision_group.empty()) {
|
||||
udata->decision_group[i] = decision_group[idxs[i]];
|
||||
}
|
||||
|
||||
for (int s = 0; s < udata->n_seq_id[i]; ++s) {
|
||||
const llama_seq_id seq_id = batch.seq_id[idxs[i]][s];
|
||||
@@ -1030,6 +1041,7 @@ llama_ubatch llama_batch_allocr::ubatch_add(const std::vector<int32_t> & idxs, u
|
||||
/*.output =*/ udata->output.data(),
|
||||
/*.type =*/ mixed ? udata->type.data() : nullptr,
|
||||
/*.decision_order =*/ udata->decision_order.empty() ? nullptr : udata->decision_order.data(),
|
||||
/*.decision_group =*/ udata->decision_group.empty() ? nullptr : udata->decision_group.data(),
|
||||
/*.data =*/ std::move(udata),
|
||||
};
|
||||
|
||||
@@ -1430,6 +1442,14 @@ bool llama_batch_ext::set_decision_order(int32_t idx, int32_t order) {
|
||||
return true;
|
||||
}
|
||||
|
||||
bool llama_batch_ext::set_decision_group(int32_t idx, int32_t group) {
|
||||
if (idx < 0 || idx >= (int32_t) tokens.size() || group < 0) {
|
||||
return false;
|
||||
}
|
||||
tokens[idx].decision_group = group;
|
||||
return true;
|
||||
}
|
||||
|
||||
// llama_batch_ext C API
|
||||
|
||||
llama_batch_ext * llama_batch_ext_init(llama_context * ctx) {
|
||||
@@ -1513,6 +1533,10 @@ bool llama_batch_ext_set_decision_order(llama_batch_ext * batch, int32_t idx, ll
|
||||
return batch->set_decision_order(idx, order);
|
||||
}
|
||||
|
||||
bool llama_batch_ext_set_decision_group(llama_batch_ext * batch, int32_t idx, int32_t group) {
|
||||
return batch->set_decision_group(idx, group);
|
||||
}
|
||||
|
||||
// llama_batch_compat
|
||||
|
||||
void llama_batch_compat::init(llama_batch_ext & dst, const llama_batch & batch_inp, size_t n_embd_row) {
|
||||
|
||||
Reference in new issue
Block a user