llama, server: add /v1/systemone API (models: laya, julia-1, lev, openjev, kev) (#29818)

* init conversion

* convert: ok

* model loaded

* add server code

* improve conversion script

* support shared prompt prefix

* add docs, imorove UX a bit

* add vision support

* add openjev tiny model for testing

* add dev docs

* support lev & kev

* clean up

* fix lev noul

* fix py lint

* nits docs

* clarify about not supporting date_facts
This commit is contained in:
Xuan-Son Nguyen authored and GitHub committed 2026-10-02 11:56:04 +02:00
1 parent 70849ee82c
commit a4cb4c61fd
35 files changed
+2139 -15

No files matched your search

+210
View File
@@ -1,6 +1,7 @@
#include "server-context.h"
#include "server-chat.h"
#include "server-common.h"
#include "server-decision.h"
#include "server-http.h"
#include "server-task.h"
#include "server-queue.h"
@@ -413,6 +414,10 @@ struct server_slot {
if (pooling == LLAMA_POOLING_TYPE_RANK && llama_get_causal_attn(ctx_tgt)) {
return true;
}
// a decision task reads its outputs from the last batch
if (task->type == SERVER_TASK_TYPE_DECISION) {
return true;
}
return false;
}
@@ -689,6 +694,14 @@ struct server_slot {
return res;
}
// the other slot continues from the tokens processed so far
void copy_prompt_to(server_slot & other) const {
mem.seq_rm(other.id, -1, -1);
mem.seq_cp(id, other.id, -1, -1);
other.prompt = prompt.clone();
}
void copy_state_to(server_slot & other) const {
GGML_ASSERT(state == SLOT_STATE_DONE_PROMPT);
@@ -825,6 +838,8 @@ public:
mtmd_helper_init_opt init_opt = mtmd_helper_init_opt_default();
const llama_vocab * vocab = nullptr;
server_decision_context decision;
server_queue queue_tasks;
server_response queue_results;
@@ -1102,6 +1117,13 @@ private:
vocab = llama_model_get_vocab(model_tgt);
try {
decision.init(model_tgt);
} catch (const std::exception & e) {
SRV_ERR("failed to init decision model: %s\n", e.what());
return false;
}
n_ctx = llama_n_ctx(ctx_tgt);
add_bos_token = llama_vocab_get_add_bos(vocab);
@@ -2194,6 +2216,67 @@ private:
queue_results.send(std::move(res));
}
void send_decision(const server_slot & slot, const common_batch & batch, int32_t i_batch) {
auto res = std::make_unique<server_task_result_decision>();
res->id = slot.task->id;
res->index = slot.task->index;
res->n_tokens = slot.task->n_tokens();
const auto & decision = slot.task->decision;
if (!decision.labels.empty()) {
const float * logits = llama_get_logits_ith(slot.ctx_tgt, i_batch);
if (logits == nullptr) {
send_error(slot, "failed to get logits", ERROR_TYPE_SERVER);
return;
}
const int32_t n_vocab = llama_vocab_n_tokens(vocab);
for (const llama_token label : decision.labels) {
GGML_ASSERT(label >= 0 && label < n_vocab);
res->scores.push_back(logits[label]);
}
} else {
// the outputs of this slot in this batch are the last tokens of the prompt
std::vector<int32_t> idx;
for (int i = 0; i < batch.size(); ++i) {
if (batch.tokens[i].output && batch.tokens[i].seq_id == slot.id) {
idx.push_back(i);
}
}
const int32_t pos_first = slot.prompt.n_tokens() - (int32_t) idx.size();
auto get_embd = [&](int32_t pos) -> const float * {
const int32_t i = pos - pos_first;
return i >= 0 && i < (int32_t) idx.size() ? llama_get_embeddings_ith(slot.ctx_tgt, idx[i]) : nullptr;
};
const int32_t n_embd_out = llama_model_n_embd_out(model_tgt);
const int32_t n_pointer = n_embd_out / 2;
const float * embd_q = decision.pointer >= 0 ? get_embd(decision.pointer) : nullptr;
GGML_ASSERT(decision.column >= 0 && decision.column < n_embd_out);
for (const int32_t marker : decision.markers) {
const float * embd = get_embd(marker);
if (embd == nullptr || (decision.pointer >= 0 && embd_q == nullptr)) {
send_error(slot, "failed to get embeddings, the question and its options must fit in one batch", ERROR_TYPE_SERVER);
return;
}
if (decision.pointer < 0) {
res->scores.push_back(embd[decision.column]);
continue;
}
float dot = 0.0f;
for (int32_t i = 0; i < n_pointer; i++) {
dot += embd_q[i] * embd[n_pointer + i];
}
res->scores.push_back(dot / sqrtf((float) n_pointer));
}
}
SLT_DBG(slot, "%s", "sending decision result\n");
queue_results.send(std::move(res));
}
void send_rerank(const server_slot & slot, const common_batch & batch) {
auto res = std::make_unique<server_task_result_rerank>();
res->id = slot.task->id;
@@ -2384,6 +2467,7 @@ private:
case SERVER_TASK_TYPE_INFILL:
case SERVER_TASK_TYPE_EMBEDDING:
case SERVER_TASK_TYPE_RERANK:
case SERVER_TASK_TYPE_DECISION:
{
// special case: if input is provided via CLI, tokenize it first
// otherwise, no need to tokenize as it's already done inside the HTTP thread
@@ -3212,12 +3296,29 @@ private:
return;
}
// the outputs of a decision are read from one batch
const int32_t n_decision_first = slot.task->type == SERVER_TASK_TYPE_DECISION ? slot.task->decision.pos_first() : -1;
if (n_decision_first >= 0 && slot.task->n_tokens() - n_decision_first > n_batch) {
send_error(slot,
string_format("the question and its options (%d tokens) are too large to process. "
"increase the batch size (current batch size: %d)",
slot.task->n_tokens() - n_decision_first, n_batch),
ERROR_TYPE_INVALID_REQUEST);
slot.release();
return;
}
const bool is_stateless_task = slot.task->type == SERVER_TASK_TYPE_EMBEDDING || slot.task->type == SERVER_TASK_TYPE_RERANK;
if (slot.task->params.cache_prompt && !is_stateless_task) {
// reuse any previously computed tokens that are common with the new prompt
n_past = slot.prompt.tokens.get_common_prefix(input_tokens);
// the children start from the shared prefix, do not go past it
if (slot.task->n_tokens_shared > 0) {
n_past = std::min(n_past, slot.task->n_tokens_shared);
}
// if there is an alora invoked, don't cache after the invocation start
if (slot.alora_invocation_start > 0) {
SLT_DBG(slot, "only caching to alora invocation start (n_past = %d, alora_invocation_start = %d)\n", n_past, slot.alora_invocation_start);
@@ -3443,6 +3544,24 @@ private:
slot.mem.seq_rm(slot.id, p0, -1);
// shared prompt prefix: once it is processed, the children continue from it with their own prompt
bool wait_shared = false;
if (slot.task->n_tokens_shared > 0) {
const bool is_shared_done = slot.prompt.n_tokens() == slot.task->n_tokens_shared;
for (auto & other : slots) {
if (other.state != SLOT_STATE_WAIT_OTHER || other.task->id_parent != slot.task->id) {
continue;
}
if (is_shared_done) {
SLT_TRC(slot, " - copying shared prompt (%d tokens) to child %d\n", slot.prompt.n_tokens(), other.id);
slot.copy_prompt_to(other);
other.state = SLOT_STATE_STARTED;
} else {
wait_shared = true;
}
}
}
// If using an alora, there may be uncached tokens that come
// before the invocation sequence. When this happens, the
// tokens before the invocation sequence need to be
@@ -3520,6 +3639,8 @@ private:
const auto & spans = slot.task->params.message_spans;
const auto last_user_pos = spans.last_user_message_pos();
const int32_t n_decision_first = slot.task->type == SERVER_TASK_TYPE_DECISION ? slot.task->decision.pos_first() : -1;
// add prompt tokens for processing in the current batch
while (slot.prompt.n_tokens() < slot.task->n_tokens() && batch.size() < n_batch) {
// get next token to process
@@ -3528,6 +3649,16 @@ private:
break; // end of text chunk
}
// stop at the end of the shared prefix, the children are started from this state
if (wait_shared && slot.prompt.n_tokens() == slot.task->n_tokens_shared) {
break;
}
// the outputs of a decision are read from one batch, do not split them
if (slot.prompt.n_tokens() == n_decision_first && batch.size() + slot.task->n_tokens() - n_decision_first > n_batch) {
break;
}
// if this is an alora request with pre-invocation
// tokens that are not cached, we need to stop filling
// this batch at those pre-invocation tokens.
@@ -3771,6 +3902,8 @@ private:
SLT_TRC(slot, " - copying state to child %d\n", child->id);
GGML_ASSERT(child->state == SLOT_STATE_WAIT_OTHER);
// children with their own prompt are started at the end of the shared prefix
GGML_ASSERT(slot.task->n_tokens_shared == 0);
slot.copy_state_to(*child);
child->state = SLOT_STATE_DONE_PROMPT;
@@ -3831,6 +3964,13 @@ private:
return;
}
if (slot.task->type == SERVER_TASK_TYPE_DECISION) {
send_decision(slot, batch.view, slot.i_batch - off);
slot.release();
slot.i_batch = -1;
return;
}
GGML_ASSERT(slot.task->need_sampling());
// prompt evaluated for next-token prediction
@@ -5224,6 +5364,76 @@ void server_routes::init_routes() {
return res;
};
this->post_systemone = [this](const server_http_req & req) {
auto res = create_response();
const auto & decision = ctx_server.decision;
if (decision.type == COMMON_DECISION_TYPE_NONE) {
res->error(format_error_response("This model is not a decision model", ERROR_TYPE_NOT_SUPPORTED));
return res;
}
const json body = json::parse(req.body);
const auto questions = decision.parse_questions(body);
std::vector<raw_buffer> files;
const json state = decision.parse_state(body, files);
if (!files.empty() && (!decision.can_use_images() || !meta->has_inp_image)) {
res->error(format_error_response("This server does not support image input for decisions. For a model that supports it, start it with `--mmproj`", ERROR_TYPE_NOT_SUPPORTED));
return res;
}
// one task per variant of each question
auto & rd = res->rd;
{
std::vector<server_task> tasks;
for (const auto & question : questions) {
for (size_t variant = 0; variant < decision.n_variants(question); variant++) {
server_task task = server_task(SERVER_TASK_TYPE_DECISION);
task.id = rd.get_new_id();
decision.fill_task(state, question, variant, files, ctx_server.mctx, ctx_server.init_opt, task);
tasks.push_back(std::move(task));
}
}
if (decision.can_share_prompt()) {
tasks = server_decision_group_tasks(std::move(tasks), params.n_parallel);
}
rd.post_tasks(std::move(tasks));
}
auto all_results = rd.wait_for_all(req.should_stop);
if (all_results.is_terminated) {
return res; // connection is closed
} else if (all_results.error) {
res->error(all_results.error->to_json());
return res;
}
json answers = json::object();
int32_t n_tokens = 0;
size_t i_result = 0;
for (const auto & question : questions) {
std::vector<std::vector<float>> scores;
for (size_t variant = 0; variant < decision.n_variants(question); variant++) {
auto * result = dynamic_cast<server_task_result_decision *>(all_results.results[i_result++].get());
GGML_ASSERT(result != nullptr);
scores.push_back(result->scores);
n_tokens += result->n_tokens;
}
answers[question.id] = decision.format_answer(question, scores);
}
res->ok(json{
{"model", meta->model_name},
{"answers", answers},
{"usage", {
{"input_tokens", n_tokens},
{"output_tokens", 0},
}},
});
return res;
};
this->get_lora_adapters = [this](const server_http_req & req) {
auto res = create_response();