server : enable packing for d1-2B and d1-Omni-2B models (#32553)

* llama-bench : size the batch to the prompt for models without memory

Assisted-by: Claude Opus 5.5

* server : read up to 16 lfm3-d1 questions in one pass

Assisted-by: Claude Opus 5.5
This commit is contained in:
Tarek Dakhran authored and GitHub committed 2026-11-18 20:15:43 +01:00
1 parent 2cbfc3cf34
commit bafa0420c1
14 files changed
+220 -31

No files matched your search

+8
View File
@@ -1313,6 +1313,11 @@ common_init_result::common_init_result(common_params & params, bool model_only)
LOG_INF("%s", "decision model reads the embeddings output, enabling embedding mode\n");
}
// 0 is n_batch
if ((decision_type == COMMON_DECISION_TYPE_LFM3_D1 || decision_type == COMMON_DECISION_TYPE_LFM3_D1_OMNI) && cparams.n_outputs_max != 0) {
cparams.n_outputs_max = std::max(cparams.n_outputs_max, (uint32_t) COMMON_DECISION_LFM3_D1_SET_SIZE);
}
if (llama_model_classifier_free_guidance(model)) {
params.kv_unified = true;
cparams.kv_unified = true;
@@ -2334,6 +2339,9 @@ llama_batch_ext * common_batch::get_sub_batch(int32_t off, int32_t n) {
if (t.decision_order != 0) {
llama_batch_ext_set_decision_order(res, idx, (llama_decision_order) t.decision_order);
}
if (t.decision_group != 0 && !llama_batch_ext_set_decision_group(res, idx, t.decision_group)) {
GGML_ABORT("%s: failed to set the decision group of the entry at index %d\n", __func__, i);
}
}
return res;
+4
View File
@@ -1002,6 +1002,9 @@ enum common_decision_type {
COMMON_DECISION_TYPE_UNKNOWN, // a decision model of a type that is not supported
};
// lfm3-d1: max questions packed in one prompt
#define COMMON_DECISION_LFM3_D1_SET_SIZE 16
common_decision_type common_get_decision_type(const struct llama_model * model);
// metadata of a GGUF file, read without loading the model
@@ -1117,6 +1120,7 @@ struct common_batch {
llama_embd state; // non-owning view of the data passed to set_embd_state(), data == NULL if none
std::vector<llama_seq_id> seq_ids_extra; // see add_seq()
int32_t decision_order = 0; // see llama_batch_ext_set_decision_order()
int32_t decision_group = 0; // see llama_batch_ext_set_decision_group()
int32_t embd_row = -1; // see add_embd_row(), -1 if none
int32_t state_row = -1; // see set_embd_state_row(), -1 if none
};
+24
View File
@@ -259,6 +259,10 @@ bool llama_batch_allocr::init(
decision_order.resize(n_tok, 0);
decision_order[i] = batch_inp.tokens[i].decision_order;
}
if (batch_inp.tokens[i].decision_group != 0) {
decision_group.resize(n_tok, 0);
decision_group[i] = batch_inp.tokens[i].decision_group;
}
}
//
@@ -353,6 +357,7 @@ bool llama_batch_allocr::init(
/*.output =*/ batch.logits,
/*.type =*/ is_embd_vec.empty() ? nullptr : is_embd_vec.data(),
/*.decision_order =*/ decision_order.empty() ? nullptr : decision_order.data(),
/*.decision_group =*/ decision_group.empty() ? nullptr : decision_group.data(),
/*.data =*/ {},
};
@@ -579,6 +584,7 @@ llama_ubatch llama_batch_allocr::ubatch_reserve(uint32_t n_seq_tokens, uint32_t
/*.output =*/ udata->output.data(),
/*.type =*/ nullptr,
/*.decision_order =*/ nullptr,
/*.decision_group =*/ nullptr,
/*.data =*/ std::move(udata),
};
@@ -887,6 +893,7 @@ void llama_batch_allocr::clear() {
seq_id_unq .clear();
output .clear();
decision_order.clear();
decision_group.clear();
for (auto & cur : seq_pos) {
cur.clear();
@@ -943,6 +950,7 @@ llama_ubatch llama_batch_allocr::ubatch_add(const std::vector<int32_t> & idxs, u
udata->output .resize(n_tokens);
udata->type .resize(mixed ? n_tokens : 0);
udata->decision_order.resize(decision_order.empty() ? 0 : n_tokens);
udata->decision_group.resize(decision_group.empty() ? 0 : n_tokens);
udata->batch_idxs = idxs;
udata->seq_id_data.reserve(n_tokens);
@@ -984,6 +992,9 @@ llama_ubatch llama_batch_allocr::ubatch_add(const std::vector<int32_t> & idxs, u
if (!decision_order.empty()) {
udata->decision_order[i] = decision_order[idxs[i]];
}
if (!decision_group.empty()) {
udata->decision_group[i] = decision_group[idxs[i]];
}
for (int s = 0; s < udata->n_seq_id[i]; ++s) {
const llama_seq_id seq_id = batch.seq_id[idxs[i]][s];
@@ -1030,6 +1041,7 @@ llama_ubatch llama_batch_allocr::ubatch_add(const std::vector<int32_t> & idxs, u
/*.output =*/ udata->output.data(),
/*.type =*/ mixed ? udata->type.data() : nullptr,
/*.decision_order =*/ udata->decision_order.empty() ? nullptr : udata->decision_order.data(),
/*.decision_group =*/ udata->decision_group.empty() ? nullptr : udata->decision_group.data(),
/*.data =*/ std::move(udata),
};
@@ -1430,6 +1442,14 @@ bool llama_batch_ext::set_decision_order(int32_t idx, int32_t order) {
return true;
}
bool llama_batch_ext::set_decision_group(int32_t idx, int32_t group) {
if (idx < 0 || idx >= (int32_t) tokens.size() || group < 0) {
return false;
}
tokens[idx].decision_group = group;
return true;
}
// llama_batch_ext C API
llama_batch_ext * llama_batch_ext_init(llama_context * ctx) {
@@ -1513,6 +1533,10 @@ bool llama_batch_ext_set_decision_order(llama_batch_ext * batch, int32_t idx, ll
return batch->set_decision_order(idx, order);
}
bool llama_batch_ext_set_decision_group(llama_batch_ext * batch, int32_t idx, int32_t group) {
return batch->set_decision_group(idx, group);
}
// llama_batch_compat
void llama_batch_compat::init(llama_batch_ext & dst, const llama_batch & batch_inp, size_t n_embd_row) {
+5
View File
@@ -61,6 +61,7 @@ struct llama_ubatch {
int8_t * output; // [n_tokens] | i | -
int8_t * type; // [n_tokens] | i | - (mixed ubatch only, 0 - token, 1 - embd)
int32_t * decision_order; // [n_tokens], NULL if no entry has one, see llama_batch_ext_set_decision_order()
int32_t * decision_group; // [n_tokens], NULL if no entry has one, see llama_batch_ext_set_decision_group()
struct data_t {
std::vector<llama_token> token;
@@ -77,6 +78,7 @@ struct llama_ubatch {
std::vector<int8_t> type;
std::vector<int32_t> batch_idxs; // original batch index for each token
std::vector<int32_t> decision_order;
std::vector<int32_t> decision_group;
std::vector<llama_seq_id> seq_id_data;
};
@@ -121,6 +123,7 @@ struct llama_batch_ext {
int32_t state_row = -1; // row of ctx_state_src's h_nextn device buffer used as state, -1 if none
bool output = false; // TODO: have dedicated output flags
int32_t decision_order = 0; // see llama_batch_ext_set_decision_order()
int32_t decision_group = 0; // see llama_batch_ext_set_decision_group()
std::unordered_set<llama_seq_id> seq_ids;
std::array<llama_pos, GGML_MROPE_SECTIONS> pos = {0, 0, 0, 0};
};
@@ -155,6 +158,7 @@ struct llama_batch_ext {
bool set_token_pos(int32_t idx, const llama_pos * pos_in);
bool set_output(int32_t idx, bool output_last);
bool set_decision_order(int32_t idx, int32_t order);
bool set_decision_group(int32_t idx, int32_t group);
};
// a helper for sanitizing, fulfilling and splitting a batch
@@ -241,6 +245,7 @@ private:
std::vector<int32_t> seq_idx;
std::vector<int8_t> output;
std::vector<int32_t> decision_order; // empty if no entry has one
std::vector<int32_t> decision_group; // empty if no entry has one
using pos_set_t = std::set<llama_pos>;
using seq_cpl_t = std::vector<bool>;
+5 -1
View File
@@ -1934,7 +1934,11 @@ int llama_context::encode(const llama_batch_ext & batch_inp) {
const llama_ubatch ubatch = balloc->split_simple(n_tokens);
// micro-batching is not possible for non-causal encoding, so we process the batch in a single shot
GGML_ASSERT(cparams.n_ubatch >= n_tokens && "encoder requires n_ubatch >= n_tokens");
if (n_tokens > cparams.n_ubatch) {
LLAMA_LOG_ERROR("%s: the encoder reads the batch in one pass, n_tokens = %u > n_ubatch = %u, increase n_ubatch (-ub)\n",
__func__, n_tokens, cparams.n_ubatch);
return -1;
}
// TODO: this clear of the buffer can easily be forgotten - need something better
// sync first so any in-flight async copies into embd_seq complete before it is freed
+4
View File
@@ -137,6 +137,10 @@ enum llama_decision_order {
// The embeddings output has one value per entry: row i is the score of option i
LLAMA_API bool llama_batch_ext_set_decision_order(struct llama_batch_ext * batch, int32_t idx, enum llama_decision_order order);
// Questions packed after a shared state in one sequence, contexts without memory only
// 0 (default) is the state, k > 0 the k-th question; full attention layers hide other questions from a question
LLAMA_API bool llama_batch_ext_set_decision_group(struct llama_batch_ext * batch, int32_t idx, int32_t group);
// Label of option i of a decision model ("<arch>.decision.labels"), nullptr past the last one
LLAMA_API const char * llama_model_decision_label(const struct llama_model * model, uint32_t i);
+12 -5
View File
@@ -608,25 +608,32 @@ void llm_graph_input_attn_no_cache::set_input(const llama_ubatch * ubatch) {
const int64_t n_kv = self_attn_mask.raw->ne[0];
const int64_t n_tokens = ubatch->n_tokens;
const auto fill_mask = [&](auto * data, int64_t ne, int n_swa, llama_swa_type swa_type, bool causal) {
const auto fill_mask = [&](auto * data, int64_t ne, int n_swa, llama_swa_type swa_type, bool causal, const int32_t * group) {
using T = std::remove_reference_t<decltype(*data)>;
std::fill(data, data + ne, llama_cast<T>(-INFINITY));
for (int i1 = 0; i1 < n_tokens; ++i1) {
const llama_seq_id s1 = ubatch->seq_id[i1][0];
const llama_pos p1 = ubatch->pos[i1];
const int32_t g1 = group ? group[i1] : 0;
const uint64_t idst = i1*n_kv;
for (int i0 = 0; i0 < n_tokens; ++i0) {
const llama_seq_id s0 = ubatch->seq_id[i0][0];
const llama_pos p0 = ubatch->pos[i0];
const int32_t g0 = group ? group[i0] : 0;
// mask different sequences
if (s0 != s1) {
continue;
}
// mask other questions, see llama_batch_ext_set_decision_group()
if (g0 != 0 && g1 != 0 && g0 != g1) {
continue;
}
// mask future tokens
if (causal && p0 > p1) {
continue;
@@ -653,18 +660,18 @@ void llm_graph_input_attn_no_cache::set_input(const llama_ubatch * ubatch) {
GGML_ASSERT(self_attn_mask.raw);
GGML_ASSERT(ggml_backend_buffer_is_host(self_attn_mask.raw->buffer));
if (self_attn_mask.raw->type == GGML_TYPE_F16) {
fill_mask((ggml_fp16_t *) self_attn_mask.raw->data, ggml_nelements(self_attn_mask.raw), 0, LLAMA_SWA_TYPE_NONE, causal);
fill_mask((ggml_fp16_t *) self_attn_mask.raw->data, ggml_nelements(self_attn_mask.raw), 0, LLAMA_SWA_TYPE_NONE, causal, ubatch->decision_group);
} else {
fill_mask((float *) self_attn_mask.raw->data, ggml_nelements(self_attn_mask.raw), 0, LLAMA_SWA_TYPE_NONE, causal);
fill_mask((float *) self_attn_mask.raw->data, ggml_nelements(self_attn_mask.raw), 0, LLAMA_SWA_TYPE_NONE, causal, ubatch->decision_group);
}
if (hparams.swa_type != LLAMA_SWA_TYPE_NONE) {
GGML_ASSERT(self_attn_mask_swa.raw);
GGML_ASSERT(ggml_backend_buffer_is_host(self_attn_mask_swa.raw->buffer));
if (self_attn_mask_swa.raw->type == GGML_TYPE_F16) {
fill_mask((ggml_fp16_t *) self_attn_mask_swa.raw->data, ggml_nelements(self_attn_mask_swa.raw), hparams.n_swa, hparams.swa_type, causal_swa);
fill_mask((ggml_fp16_t *) self_attn_mask_swa.raw->data, ggml_nelements(self_attn_mask_swa.raw), hparams.n_swa, hparams.swa_type, causal_swa, nullptr);
} else {
fill_mask((float *) self_attn_mask_swa.raw->data, ggml_nelements(self_attn_mask_swa.raw), hparams.n_swa, hparams.swa_type, causal_swa);
fill_mask((float *) self_attn_mask_swa.raw->data, ggml_nelements(self_attn_mask_swa.raw), hparams.n_swa, hparams.swa_type, causal_swa, nullptr);
}
}
}
+1
View File
@@ -169,6 +169,7 @@ static llama_ubatch dsv4_build_raw_write_ubatch(const llama_ubatch & ubatch) {
/*.output =*/ data->output.data(),
/*.type =*/ nullptr,
/*.decision_order =*/ nullptr,
/*.decision_group =*/ nullptr,
/*.data =*/ data,
};
+16
View File
@@ -2379,6 +2379,18 @@ int llama_bench(int argc, char ** argv) {
}
llama_context * ctx = llama_init_from_model(lmodel, cparams);
// a model without memory reads a prompt in one ubatch
const int n_tokens_max = std::max(inst.n_prompt, inst.n_depth);
const bool resize = ctx != NULL && !llama_get_memory(ctx) && (int) llama_n_ubatch(ctx) < n_tokens_max;
if (resize) {
fprintf(stderr, "%s: the model has no memory, setting n_batch = n_ubatch = %d\n", __func__, n_tokens_max);
llama_free(ctx);
cparams.n_batch = n_tokens_max;
cparams.n_ubatch = n_tokens_max;
ctx = llama_init_from_model(lmodel, cparams);
}
if (ctx == NULL) {
fprintf(stderr, "%s: error: failed to create context with model '%s'\n", __func__, inst.model.c_str());
llama_model_free(lmodel);
@@ -2386,6 +2398,10 @@ int llama_bench(int argc, char ** argv) {
}
test t(inst, lmodel, ctx);
if (resize) {
t.n_batch = n_tokens_max;
t.n_ubatch = n_tokens_max;
}
llama_memory_clear(llama_get_memory(ctx), false);
+2 -2
View File
@@ -1931,11 +1931,11 @@ Follows the [TypeSafe API](https://docs.typesafe.ai/api), streaming is not suppo
- `score`: An array of 2 to 10 level descriptions, lowest level first.
- `noul`: Optional. An object with the descriptions of `true` and `false`.
The questions of a request are answered independently, an answer does not depend on the other questions. The exception is clef: it reads all the questions in one prompt and decides them jointly.
The questions of a request are answered independently, an answer does not depend on the other questions. The exceptions are clef, which reads all the questions in one prompt and decides them jointly, and lfm3-d1 and lfm3-d1-omni, which read up to 16 questions after the state in one pass, as the models were trained (requests with a spoken question read one question per pass).
The number of options of a `choice` question is limited by the model, for example: 52 for openjev, 255 for granite-switch, laya, clef, pplx-decider, vllm-decision2, vllm-d3, lfm2-d1, lfm2-d1-omni and h2o-lightning, 2 for openbmb-minicpm-ocr-jev. For laya, long questions and options are truncated to the token budget the model was trained with.
For laya, clef, vllm-d3 and lfm2-d1-omni, the whole prompt is evaluated in one batch: it must fit in `--ubatch-size`. An lfm2-d1-omni prompt is cut to 16384 tokens. For vllm-decision2, the options of a question and the end of the prompt must fit in one batch. An h2o-lightning state longer than 32000 tokens keeps its first and last 16000 tokens. A server that runs clef only serves this endpoint, text generation is not available. A decision model that is also a text model, such as granite-switch, serves both and reports `["text", "decisions"]`.
For laya, clef, vllm-d3, lfm2-d1-omni, lfm3-d1 and lfm3-d1-omni, the whole prompt is evaluated in one batch: it must fit in `--ubatch-size`. An lfm2-d1-omni prompt is cut to 16384 tokens. For vllm-decision2, the options of a question and the end of the prompt must fit in one batch. An h2o-lightning state longer than 32000 tokens keeps its first and last 16000 tokens. A server that runs clef only serves this endpoint, text generation is not available. A decision model that is also a text model, such as granite-switch, serves both and reports `["text", "decisions"]`.
openbmb-minicpm-ocr-jev answers one fixed question: is an OCR text, rendered as an image, consistent with the original image? `state` is the OCR text, as a string, special tokens written in it are escaped. `files` (or `images`) has exactly 2 images: the original, then the rendered one, both are required. The request has one question and no `instructions`: the task is part of the prompt of the model, so a request that sends one is rejected. It is either a `choice` with the `criteria` `{"0": null, "1": null}`, where `1` means consistent, or a `noul` without `criteria`, which returns the probability of `1`. The model has no LM head, so text generation is not available.
+57 -23
View File
@@ -159,6 +159,7 @@ struct server_batch {
bool output;
bool is_prompt; // for stats tracking
int32_t decision_order = 0;
int32_t decision_group = 0;
int32_t i_embd = -1; // row in embd, -1 if this is a token
};
std::vector<token> tokens;
@@ -239,6 +240,11 @@ struct server_batch {
tokens[idx].decision_order = order;
}
void set_decision_group(int32_t idx, int32_t group) {
GGML_ASSERT(idx >= 0 && idx < (int32_t)tokens.size());
tokens[idx].decision_group = group;
}
// render the sub-batch [off, off + n_tokens) into view, index i in view is index off + i here
void render(int32_t off, int32_t n_tokens) {
GGML_ASSERT(off >= 0 && off < size());
@@ -253,6 +259,7 @@ struct server_batch {
view.add(t.token, t.pos[0], t.id_slot, t.output);
}
view.tokens.back().decision_order = t.decision_order;
view.tokens.back().decision_group = t.decision_group;
}
}
};
@@ -2469,25 +2476,26 @@ private:
}
res->scores.assign(embd, embd + llama_model_n_embd_out(model_tgt));
} else if (!decision.labels.empty()) {
const float * logits = llama_get_logits_ith(slot.ctx_tgt, i_batch);
if (logits == nullptr) {
send_error(slot, "failed to get logits", ERROR_TYPE_SERVER);
return;
}
const int32_t n_vocab = llama_vocab_n_tokens(vocab);
for (const llama_token label : decision.labels) {
GGML_ASSERT(label >= 0 && label < n_vocab);
res->scores.push_back(logits[label]);
}
if (!decision.label_groups.empty()) {
std::vector<float> scores;
size_t i = 0;
for (const int32_t n : decision.label_groups) {
GGML_ASSERT(n > 0 && i + n <= res->scores.size());
scores.push_back(*std::max_element(res->scores.begin() + i, res->scores.begin() + i + n));
i += n;
const size_t n_out = decision.label_groups.empty() ? decision.labels.size() : decision.label_groups.size();
GGML_ASSERT(decision.label_pos.empty() || decision.label_pos.size() == n_out);
size_t i = 0;
for (size_t o = 0; o < n_out; o++) {
const int32_t i_out = decision.label_pos.empty() ? i_batch : i_batch - (slot.task->n_tokens() - 1 - decision.label_pos[o]);
const float * logits = llama_get_logits_ith(slot.ctx_tgt, i_out);
if (logits == nullptr) {
send_error(slot, "failed to get logits", ERROR_TYPE_SERVER);
return;
}
res->scores = std::move(scores);
const size_t n = decision.label_groups.empty() ? 1 : decision.label_groups[o];
GGML_ASSERT(n > 0 && i + n <= decision.labels.size());
float score = -INFINITY;
for (size_t j = i; j < i + n; j++) {
GGML_ASSERT(decision.labels[j] >= 0 && decision.labels[j] < n_vocab);
score = std::max(score, logits[decision.labels[j]]);
}
res->scores.push_back(score);
i += n;
}
} else {
// the outputs of this slot in this batch are the last tokens of the prompt
@@ -4430,6 +4438,9 @@ private:
if (!slot.task->decision.order.empty()) {
batch.set_decision_order(batch.size() - 1, slot.task->decision.order[slot.prompt.n_tokens()]);
}
if (!slot.task->decision.group.empty()) {
batch.set_decision_group(batch.size() - 1, slot.task->decision.group[slot.prompt.n_tokens()]);
}
slot.prompt.tokens.push_back(cur_tok);
// break at the last user message, or at user messages at least min step past the last checkpoint
@@ -4483,6 +4494,11 @@ private:
// extract the logits only for the last token
batch.set_output(batch.size() - 1, true);
// and at the answer of each packed question
for (const int32_t pos : slot.task->decision.label_pos) {
batch.set_output(batch.size() - slot.task->n_tokens() + pos, true);
}
slot.stats.n_gen = 0;
slot.i_batch = batch.size() - 1;
@@ -4555,6 +4571,9 @@ private:
if (!slot.task->decision.order.empty()) {
batch.set_decision_order(batch.size() - 1, slot.task->decision.order[cur_token_idx]);
}
if (!slot.task->decision.group.empty()) {
batch.set_decision_group(batch.size() - 1, slot.task->decision.group[cur_token_idx]);
}
slot.prompt.tokens.push_back(cur_tok);
continue;
}
@@ -6464,6 +6483,7 @@ void server_routes::init_routes() {
const json body = json::parse(req.body);
const auto questions = decision.parse_questions(body);
const size_t set_size = decision.set_size(questions);
std::vector<raw_buffer> files;
const json state = decision.prepare_state(decision.parse_state(body, files));
@@ -6472,7 +6492,8 @@ void server_routes::init_routes() {
return res;
}
// one task per variant of each question, or one task for all the questions
// one task per variant of each question, or one task for all the questions, or per packed set
const bool is_packed = decision.is_joint() || set_size > 0;
auto & rd = res->rd;
{
std::vector<server_task> tasks;
@@ -6481,6 +6502,14 @@ void server_routes::init_routes() {
task.id = rd.get_new_id();
decision.fill_task_joint(state, questions, files, ctx_server.mctx, ctx_server.init_opt, task);
tasks.push_back(std::move(task));
} else if (set_size > 0) {
for (size_t i = 0; i < questions.size(); i += set_size) {
const std::vector<server_decision_question> set(questions.begin() + i, questions.begin() + std::min(questions.size(), i + set_size));
server_task task = server_task(SERVER_TASK_TYPE_DECISION);
task.id = rd.get_new_id();
decision.fill_task_set(state, set, files, ctx_server.mctx, ctx_server.init_opt, task);
tasks.push_back(std::move(task));
}
} else {
for (const auto & question : questions) {
for (size_t variant = 0; variant < decision.n_variants(question); variant++) {
@@ -6512,15 +6541,20 @@ void server_routes::init_routes() {
size_t i_score = 0;
for (const auto & question : questions) {
std::vector<std::vector<float>> scores;
if (decision.is_joint()) {
// one result with the scores of all the questions, in order
auto * result = dynamic_cast<server_task_result_decision *>(all_results.results[0].get());
if (is_packed) {
auto * result = dynamic_cast<server_task_result_decision *>(all_results.results[i_result].get());
GGML_ASSERT(result != nullptr && i_score + question.options.size() <= result->scores.size());
scores.emplace_back(result->scores.begin() + i_score, result->scores.begin() + i_score + question.options.size());
if (i_score == 0) {
n_tokens += result->n_tokens;
}
i_score += question.options.size();
n_tokens = result->n_tokens;
if (i_score == result->scores.size()) {
i_result++;
i_score = 0;
}
}
for (size_t variant = 0; !decision.is_joint() && variant < decision.n_variants(question); variant++) {
for (size_t variant = 0; !is_packed && variant < decision.n_variants(question); variant++) {
auto * result = dynamic_cast<server_task_result_decision *>(all_results.results[i_result++].get());
GGML_ASSERT(result != nullptr);
scores.push_back(result->scores);
+66
View File
@@ -1451,6 +1451,72 @@ void server_decision_context::fill_task_d1omni(
//
// given to the template: text between the pieces of the prompt, and at the start of the span of a question or of an option
// lfm3-d1 packs the questions of a state as d1 attention.py pack_set does
static const std::string D1_HEAD = "<|startoftext|><|im_start|>user\n";
static const std::string D1_HEADER = "QUESTION:\n";
size_t server_decision_context::set_size(const std::vector<server_decision_question> & questions) const {
if (!is_lfm3_d1()) {
return 0;
}
// TODO: pack spoken questions
for (const auto & question : questions) {
if (decision_is_spoken(question.instructions)) {
return 0;
}
}
return COMMON_DECISION_LFM3_D1_SET_SIZE;
}
void server_decision_context::fill_task_set(
const json & state,
const std::vector<server_decision_question> & questions,
const std::vector<raw_buffer> & files,
mtmd_context * mctx,
const mtmd_helper_init_opt & init_opt,
server_task & task) const {
GGML_ASSERT(!questions.empty());
std::string head = D1_HEAD;
for (size_t i = 0; i < files.size(); i++) {
head += get_media_marker();
}
auto suffix = [&](const server_decision_question & question) {
const std::string lone = render(json(), questions, question, 0, files.size());
if (!string_starts_with(lone, head)) {
throw std::runtime_error("unexpected layout of the decision prompt");
}
return lone.substr(head.size());
};
// the first question is its lone prompt
fill_task(state, questions, questions[0], 0, files, mctx, init_opt, task);
const size_t n_first = common_tokenize(vocab, suffix(questions[0]), false, true).size();
if (n_first >= task.tokens.size()) {
throw std::runtime_error("unexpected layout of the decision prompt");
}
task.decision.group.assign(task.tokens.size(), 0);
std::fill(task.decision.group.end() - n_first, task.decision.group.end(), 1);
task.decision.label_pos.assign(task.decision.label_groups.size(), task.tokens.size() - 1);
for (size_t k = 1; k < questions.size(); k++) {
const std::string text = (state.is_null() ? "" : D1_HEADER) + suffix(questions[k]);
for (const llama_token token : common_tokenize(vocab, text, false, true)) {
task.tokens.push_back(token);
task.decision.group.push_back(k + 1);
}
std::vector<std::string> texts;
std::vector<llama_tokens> groups;
d1_labels(questions[k], texts, groups);
for (const auto & group : groups) {
task.decision.labels.insert(task.decision.labels.end(), group.begin(), group.end());
task.decision.label_groups.push_back(group.size());
task.decision.label_pos.push_back(task.tokens.size() - 1);
}
}
}
static const std::string CLEF_MARKER = "<<clef:";
static const std::string CLEF_SEP = "<<clef:sep>>";
static const std::string CLEF_MARK_QUESTION = "<<clef:question>>";
+12
View File
@@ -124,6 +124,18 @@ struct server_decision_context {
const mtmd_helper_init_opt & init_opt,
server_task & task) const;
// lfm3-d1: max questions per prompt, 0 if each question has its own prompt
size_t set_size(const std::vector<server_decision_question> & questions) const;
// set the prompt of these questions packed after their state, the result has the scores of all their options
void fill_task_set(
const json & state,
const std::vector<server_decision_question> & questions,
const std::vector<raw_buffer> & files,
mtmd_context * mctx,
const mtmd_helper_init_opt & init_opt,
server_task & task) const;
// scores: the raw model outputs of each variant
json format_answer(const server_decision_question & question, const std::vector<std::vector<float>> & scores, bool has_media = false) const;
+4
View File
@@ -209,6 +209,7 @@ struct server_task {
struct decision {
std::vector<llama_token> labels; // logits of these tokens, at the last prompt token
std::vector<int32_t> label_groups; // if set, number of labels per output, the output is their max
std::vector<int32_t> label_pos; // if set, prompt position of each output, else the last token
std::vector<int32_t> markers; // embeddings[column] at these prompt positions
int32_t column = 0;
// if set, embeddings is [q | k], and the output is instead the scaled dot product of q[pointer] and k[marker]
@@ -237,6 +238,9 @@ struct server_task {
// the scores are the first n_scores rows of the embeddings
std::vector<int32_t> order;
int32_t n_scores = 0;
// one value per prompt token, see llama_batch_ext_set_decision_group()
std::vector<int32_t> group;
};
decision decision;