server : enable packing for d1-2B and d1-Omni-2B models (#32553)
* llama-bench : size the batch to the prompt for models without memory Assisted-by: Claude Opus 5.5 * server : read up to 16 lfm3-d1 questions in one pass Assisted-by: Claude Opus 5.5
This commit is contained in:
1 parent
2cbfc3cf34
commit
bafa0420c1
14 files changed
+220
-31
No files matched your search
@@ -159,6 +159,7 @@ struct server_batch {
|
||||
bool output;
|
||||
bool is_prompt; // for stats tracking
|
||||
int32_t decision_order = 0;
|
||||
int32_t decision_group = 0;
|
||||
int32_t i_embd = -1; // row in embd, -1 if this is a token
|
||||
};
|
||||
std::vector<token> tokens;
|
||||
@@ -239,6 +240,11 @@ struct server_batch {
|
||||
tokens[idx].decision_order = order;
|
||||
}
|
||||
|
||||
void set_decision_group(int32_t idx, int32_t group) {
|
||||
GGML_ASSERT(idx >= 0 && idx < (int32_t)tokens.size());
|
||||
tokens[idx].decision_group = group;
|
||||
}
|
||||
|
||||
// render the sub-batch [off, off + n_tokens) into view, index i in view is index off + i here
|
||||
void render(int32_t off, int32_t n_tokens) {
|
||||
GGML_ASSERT(off >= 0 && off < size());
|
||||
@@ -253,6 +259,7 @@ struct server_batch {
|
||||
view.add(t.token, t.pos[0], t.id_slot, t.output);
|
||||
}
|
||||
view.tokens.back().decision_order = t.decision_order;
|
||||
view.tokens.back().decision_group = t.decision_group;
|
||||
}
|
||||
}
|
||||
};
|
||||
@@ -2469,25 +2476,26 @@ private:
|
||||
}
|
||||
res->scores.assign(embd, embd + llama_model_n_embd_out(model_tgt));
|
||||
} else if (!decision.labels.empty()) {
|
||||
const float * logits = llama_get_logits_ith(slot.ctx_tgt, i_batch);
|
||||
if (logits == nullptr) {
|
||||
send_error(slot, "failed to get logits", ERROR_TYPE_SERVER);
|
||||
return;
|
||||
}
|
||||
const int32_t n_vocab = llama_vocab_n_tokens(vocab);
|
||||
for (const llama_token label : decision.labels) {
|
||||
GGML_ASSERT(label >= 0 && label < n_vocab);
|
||||
res->scores.push_back(logits[label]);
|
||||
}
|
||||
if (!decision.label_groups.empty()) {
|
||||
std::vector<float> scores;
|
||||
size_t i = 0;
|
||||
for (const int32_t n : decision.label_groups) {
|
||||
GGML_ASSERT(n > 0 && i + n <= res->scores.size());
|
||||
scores.push_back(*std::max_element(res->scores.begin() + i, res->scores.begin() + i + n));
|
||||
i += n;
|
||||
const size_t n_out = decision.label_groups.empty() ? decision.labels.size() : decision.label_groups.size();
|
||||
GGML_ASSERT(decision.label_pos.empty() || decision.label_pos.size() == n_out);
|
||||
size_t i = 0;
|
||||
for (size_t o = 0; o < n_out; o++) {
|
||||
const int32_t i_out = decision.label_pos.empty() ? i_batch : i_batch - (slot.task->n_tokens() - 1 - decision.label_pos[o]);
|
||||
const float * logits = llama_get_logits_ith(slot.ctx_tgt, i_out);
|
||||
if (logits == nullptr) {
|
||||
send_error(slot, "failed to get logits", ERROR_TYPE_SERVER);
|
||||
return;
|
||||
}
|
||||
res->scores = std::move(scores);
|
||||
const size_t n = decision.label_groups.empty() ? 1 : decision.label_groups[o];
|
||||
GGML_ASSERT(n > 0 && i + n <= decision.labels.size());
|
||||
float score = -INFINITY;
|
||||
for (size_t j = i; j < i + n; j++) {
|
||||
GGML_ASSERT(decision.labels[j] >= 0 && decision.labels[j] < n_vocab);
|
||||
score = std::max(score, logits[decision.labels[j]]);
|
||||
}
|
||||
res->scores.push_back(score);
|
||||
i += n;
|
||||
}
|
||||
} else {
|
||||
// the outputs of this slot in this batch are the last tokens of the prompt
|
||||
@@ -4430,6 +4438,9 @@ private:
|
||||
if (!slot.task->decision.order.empty()) {
|
||||
batch.set_decision_order(batch.size() - 1, slot.task->decision.order[slot.prompt.n_tokens()]);
|
||||
}
|
||||
if (!slot.task->decision.group.empty()) {
|
||||
batch.set_decision_group(batch.size() - 1, slot.task->decision.group[slot.prompt.n_tokens()]);
|
||||
}
|
||||
slot.prompt.tokens.push_back(cur_tok);
|
||||
|
||||
// break at the last user message, or at user messages at least min step past the last checkpoint
|
||||
@@ -4483,6 +4494,11 @@ private:
|
||||
// extract the logits only for the last token
|
||||
batch.set_output(batch.size() - 1, true);
|
||||
|
||||
// and at the answer of each packed question
|
||||
for (const int32_t pos : slot.task->decision.label_pos) {
|
||||
batch.set_output(batch.size() - slot.task->n_tokens() + pos, true);
|
||||
}
|
||||
|
||||
slot.stats.n_gen = 0;
|
||||
slot.i_batch = batch.size() - 1;
|
||||
|
||||
@@ -4555,6 +4571,9 @@ private:
|
||||
if (!slot.task->decision.order.empty()) {
|
||||
batch.set_decision_order(batch.size() - 1, slot.task->decision.order[cur_token_idx]);
|
||||
}
|
||||
if (!slot.task->decision.group.empty()) {
|
||||
batch.set_decision_group(batch.size() - 1, slot.task->decision.group[cur_token_idx]);
|
||||
}
|
||||
slot.prompt.tokens.push_back(cur_tok);
|
||||
continue;
|
||||
}
|
||||
@@ -6464,6 +6483,7 @@ void server_routes::init_routes() {
|
||||
|
||||
const json body = json::parse(req.body);
|
||||
const auto questions = decision.parse_questions(body);
|
||||
const size_t set_size = decision.set_size(questions);
|
||||
|
||||
std::vector<raw_buffer> files;
|
||||
const json state = decision.prepare_state(decision.parse_state(body, files));
|
||||
@@ -6472,7 +6492,8 @@ void server_routes::init_routes() {
|
||||
return res;
|
||||
}
|
||||
|
||||
// one task per variant of each question, or one task for all the questions
|
||||
// one task per variant of each question, or one task for all the questions, or per packed set
|
||||
const bool is_packed = decision.is_joint() || set_size > 0;
|
||||
auto & rd = res->rd;
|
||||
{
|
||||
std::vector<server_task> tasks;
|
||||
@@ -6481,6 +6502,14 @@ void server_routes::init_routes() {
|
||||
task.id = rd.get_new_id();
|
||||
decision.fill_task_joint(state, questions, files, ctx_server.mctx, ctx_server.init_opt, task);
|
||||
tasks.push_back(std::move(task));
|
||||
} else if (set_size > 0) {
|
||||
for (size_t i = 0; i < questions.size(); i += set_size) {
|
||||
const std::vector<server_decision_question> set(questions.begin() + i, questions.begin() + std::min(questions.size(), i + set_size));
|
||||
server_task task = server_task(SERVER_TASK_TYPE_DECISION);
|
||||
task.id = rd.get_new_id();
|
||||
decision.fill_task_set(state, set, files, ctx_server.mctx, ctx_server.init_opt, task);
|
||||
tasks.push_back(std::move(task));
|
||||
}
|
||||
} else {
|
||||
for (const auto & question : questions) {
|
||||
for (size_t variant = 0; variant < decision.n_variants(question); variant++) {
|
||||
@@ -6512,15 +6541,20 @@ void server_routes::init_routes() {
|
||||
size_t i_score = 0;
|
||||
for (const auto & question : questions) {
|
||||
std::vector<std::vector<float>> scores;
|
||||
if (decision.is_joint()) {
|
||||
// one result with the scores of all the questions, in order
|
||||
auto * result = dynamic_cast<server_task_result_decision *>(all_results.results[0].get());
|
||||
if (is_packed) {
|
||||
auto * result = dynamic_cast<server_task_result_decision *>(all_results.results[i_result].get());
|
||||
GGML_ASSERT(result != nullptr && i_score + question.options.size() <= result->scores.size());
|
||||
scores.emplace_back(result->scores.begin() + i_score, result->scores.begin() + i_score + question.options.size());
|
||||
if (i_score == 0) {
|
||||
n_tokens += result->n_tokens;
|
||||
}
|
||||
i_score += question.options.size();
|
||||
n_tokens = result->n_tokens;
|
||||
if (i_score == result->scores.size()) {
|
||||
i_result++;
|
||||
i_score = 0;
|
||||
}
|
||||
}
|
||||
for (size_t variant = 0; !decision.is_joint() && variant < decision.n_variants(question); variant++) {
|
||||
for (size_t variant = 0; !is_packed && variant < decision.n_variants(question); variant++) {
|
||||
auto * result = dynamic_cast<server_task_result_decision *>(all_results.results[i_result++].get());
|
||||
GGML_ASSERT(result != nullptr);
|
||||
scores.push_back(result->scores);
|
||||
|
||||
Reference in new issue
Block a user