server : enable packing for d1-2B and d1-Omni-2B models (#32553)

* llama-bench : size the batch to the prompt for models without memory

Assisted-by: Claude Opus 5.5

* server : read up to 16 lfm3-d1 questions in one pass

Assisted-by: Claude Opus 5.5
This commit is contained in:
Tarek Dakhran authored and GitHub committed 2026-11-18 20:15:43 +01:00
1 parent 2cbfc3cf34
commit bafa0420c1
14 files changed
+220 -31

No files matched your search

+66
View File
@@ -1451,6 +1451,72 @@ void server_decision_context::fill_task_d1omni(
//
// given to the template: text between the pieces of the prompt, and at the start of the span of a question or of an option
// lfm3-d1 packs the questions of a state as d1 attention.py pack_set does
static const std::string D1_HEAD = "<|startoftext|><|im_start|>user\n";
static const std::string D1_HEADER = "QUESTION:\n";
size_t server_decision_context::set_size(const std::vector<server_decision_question> & questions) const {
if (!is_lfm3_d1()) {
return 0;
}
// TODO: pack spoken questions
for (const auto & question : questions) {
if (decision_is_spoken(question.instructions)) {
return 0;
}
}
return COMMON_DECISION_LFM3_D1_SET_SIZE;
}
void server_decision_context::fill_task_set(
const json & state,
const std::vector<server_decision_question> & questions,
const std::vector<raw_buffer> & files,
mtmd_context * mctx,
const mtmd_helper_init_opt & init_opt,
server_task & task) const {
GGML_ASSERT(!questions.empty());
std::string head = D1_HEAD;
for (size_t i = 0; i < files.size(); i++) {
head += get_media_marker();
}
auto suffix = [&](const server_decision_question & question) {
const std::string lone = render(json(), questions, question, 0, files.size());
if (!string_starts_with(lone, head)) {
throw std::runtime_error("unexpected layout of the decision prompt");
}
return lone.substr(head.size());
};
// the first question is its lone prompt
fill_task(state, questions, questions[0], 0, files, mctx, init_opt, task);
const size_t n_first = common_tokenize(vocab, suffix(questions[0]), false, true).size();
if (n_first >= task.tokens.size()) {
throw std::runtime_error("unexpected layout of the decision prompt");
}
task.decision.group.assign(task.tokens.size(), 0);
std::fill(task.decision.group.end() - n_first, task.decision.group.end(), 1);
task.decision.label_pos.assign(task.decision.label_groups.size(), task.tokens.size() - 1);
for (size_t k = 1; k < questions.size(); k++) {
const std::string text = (state.is_null() ? "" : D1_HEADER) + suffix(questions[k]);
for (const llama_token token : common_tokenize(vocab, text, false, true)) {
task.tokens.push_back(token);
task.decision.group.push_back(k + 1);
}
std::vector<std::string> texts;
std::vector<llama_tokens> groups;
d1_labels(questions[k], texts, groups);
for (const auto & group : groups) {
task.decision.labels.insert(task.decision.labels.end(), group.begin(), group.end());
task.decision.label_groups.push_back(group.size());
task.decision.label_pos.push_back(task.tokens.size() - 1);
}
}
}
static const std::string CLEF_MARKER = "<<clef:";
static const std::string CLEF_SEP = "<<clef:sep>>";
static const std::string CLEF_MARK_QUESTION = "<<clef:question>>";