llama, server: add /v1/systemone API (models: laya, julia-1, lev, openjev, kev) (#29818)
* init conversion * convert: ok * model loaded * add server code * improve conversion script * support shared prompt prefix * add docs, imorove UX a bit * add vision support * add openjev tiny model for testing * add dev docs * support lev & kev * clean up * fix lev noul * fix py lint * nits docs * clarify about not supporting date_facts
This commit is contained in:
1 parent
70849ee82c
commit
a4cb4c61fd
35 files changed
+2139
-15
No files matched your search
@@ -1,6 +1,7 @@
|
||||
#include "server-context.h"
|
||||
#include "server-chat.h"
|
||||
#include "server-common.h"
|
||||
#include "server-decision.h"
|
||||
#include "server-http.h"
|
||||
#include "server-task.h"
|
||||
#include "server-queue.h"
|
||||
@@ -413,6 +414,10 @@ struct server_slot {
|
||||
if (pooling == LLAMA_POOLING_TYPE_RANK && llama_get_causal_attn(ctx_tgt)) {
|
||||
return true;
|
||||
}
|
||||
// a decision task reads its outputs from the last batch
|
||||
if (task->type == SERVER_TASK_TYPE_DECISION) {
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
@@ -689,6 +694,14 @@ struct server_slot {
|
||||
return res;
|
||||
}
|
||||
|
||||
// the other slot continues from the tokens processed so far
|
||||
void copy_prompt_to(server_slot & other) const {
|
||||
mem.seq_rm(other.id, -1, -1);
|
||||
mem.seq_cp(id, other.id, -1, -1);
|
||||
|
||||
other.prompt = prompt.clone();
|
||||
}
|
||||
|
||||
void copy_state_to(server_slot & other) const {
|
||||
GGML_ASSERT(state == SLOT_STATE_DONE_PROMPT);
|
||||
|
||||
@@ -825,6 +838,8 @@ public:
|
||||
mtmd_helper_init_opt init_opt = mtmd_helper_init_opt_default();
|
||||
const llama_vocab * vocab = nullptr;
|
||||
|
||||
server_decision_context decision;
|
||||
|
||||
server_queue queue_tasks;
|
||||
server_response queue_results;
|
||||
|
||||
@@ -1102,6 +1117,13 @@ private:
|
||||
|
||||
vocab = llama_model_get_vocab(model_tgt);
|
||||
|
||||
try {
|
||||
decision.init(model_tgt);
|
||||
} catch (const std::exception & e) {
|
||||
SRV_ERR("failed to init decision model: %s\n", e.what());
|
||||
return false;
|
||||
}
|
||||
|
||||
n_ctx = llama_n_ctx(ctx_tgt);
|
||||
|
||||
add_bos_token = llama_vocab_get_add_bos(vocab);
|
||||
@@ -2194,6 +2216,67 @@ private:
|
||||
queue_results.send(std::move(res));
|
||||
}
|
||||
|
||||
void send_decision(const server_slot & slot, const common_batch & batch, int32_t i_batch) {
|
||||
auto res = std::make_unique<server_task_result_decision>();
|
||||
res->id = slot.task->id;
|
||||
res->index = slot.task->index;
|
||||
res->n_tokens = slot.task->n_tokens();
|
||||
|
||||
const auto & decision = slot.task->decision;
|
||||
|
||||
if (!decision.labels.empty()) {
|
||||
const float * logits = llama_get_logits_ith(slot.ctx_tgt, i_batch);
|
||||
if (logits == nullptr) {
|
||||
send_error(slot, "failed to get logits", ERROR_TYPE_SERVER);
|
||||
return;
|
||||
}
|
||||
const int32_t n_vocab = llama_vocab_n_tokens(vocab);
|
||||
for (const llama_token label : decision.labels) {
|
||||
GGML_ASSERT(label >= 0 && label < n_vocab);
|
||||
res->scores.push_back(logits[label]);
|
||||
}
|
||||
} else {
|
||||
// the outputs of this slot in this batch are the last tokens of the prompt
|
||||
std::vector<int32_t> idx;
|
||||
for (int i = 0; i < batch.size(); ++i) {
|
||||
if (batch.tokens[i].output && batch.tokens[i].seq_id == slot.id) {
|
||||
idx.push_back(i);
|
||||
}
|
||||
}
|
||||
const int32_t pos_first = slot.prompt.n_tokens() - (int32_t) idx.size();
|
||||
auto get_embd = [&](int32_t pos) -> const float * {
|
||||
const int32_t i = pos - pos_first;
|
||||
return i >= 0 && i < (int32_t) idx.size() ? llama_get_embeddings_ith(slot.ctx_tgt, idx[i]) : nullptr;
|
||||
};
|
||||
|
||||
const int32_t n_embd_out = llama_model_n_embd_out(model_tgt);
|
||||
const int32_t n_pointer = n_embd_out / 2;
|
||||
const float * embd_q = decision.pointer >= 0 ? get_embd(decision.pointer) : nullptr;
|
||||
GGML_ASSERT(decision.column >= 0 && decision.column < n_embd_out);
|
||||
|
||||
for (const int32_t marker : decision.markers) {
|
||||
const float * embd = get_embd(marker);
|
||||
if (embd == nullptr || (decision.pointer >= 0 && embd_q == nullptr)) {
|
||||
send_error(slot, "failed to get embeddings, the question and its options must fit in one batch", ERROR_TYPE_SERVER);
|
||||
return;
|
||||
}
|
||||
if (decision.pointer < 0) {
|
||||
res->scores.push_back(embd[decision.column]);
|
||||
continue;
|
||||
}
|
||||
float dot = 0.0f;
|
||||
for (int32_t i = 0; i < n_pointer; i++) {
|
||||
dot += embd_q[i] * embd[n_pointer + i];
|
||||
}
|
||||
res->scores.push_back(dot / sqrtf((float) n_pointer));
|
||||
}
|
||||
}
|
||||
|
||||
SLT_DBG(slot, "%s", "sending decision result\n");
|
||||
|
||||
queue_results.send(std::move(res));
|
||||
}
|
||||
|
||||
void send_rerank(const server_slot & slot, const common_batch & batch) {
|
||||
auto res = std::make_unique<server_task_result_rerank>();
|
||||
res->id = slot.task->id;
|
||||
@@ -2384,6 +2467,7 @@ private:
|
||||
case SERVER_TASK_TYPE_INFILL:
|
||||
case SERVER_TASK_TYPE_EMBEDDING:
|
||||
case SERVER_TASK_TYPE_RERANK:
|
||||
case SERVER_TASK_TYPE_DECISION:
|
||||
{
|
||||
// special case: if input is provided via CLI, tokenize it first
|
||||
// otherwise, no need to tokenize as it's already done inside the HTTP thread
|
||||
@@ -3212,12 +3296,29 @@ private:
|
||||
return;
|
||||
}
|
||||
|
||||
// the outputs of a decision are read from one batch
|
||||
const int32_t n_decision_first = slot.task->type == SERVER_TASK_TYPE_DECISION ? slot.task->decision.pos_first() : -1;
|
||||
if (n_decision_first >= 0 && slot.task->n_tokens() - n_decision_first > n_batch) {
|
||||
send_error(slot,
|
||||
string_format("the question and its options (%d tokens) are too large to process. "
|
||||
"increase the batch size (current batch size: %d)",
|
||||
slot.task->n_tokens() - n_decision_first, n_batch),
|
||||
ERROR_TYPE_INVALID_REQUEST);
|
||||
slot.release();
|
||||
return;
|
||||
}
|
||||
|
||||
const bool is_stateless_task = slot.task->type == SERVER_TASK_TYPE_EMBEDDING || slot.task->type == SERVER_TASK_TYPE_RERANK;
|
||||
|
||||
if (slot.task->params.cache_prompt && !is_stateless_task) {
|
||||
// reuse any previously computed tokens that are common with the new prompt
|
||||
n_past = slot.prompt.tokens.get_common_prefix(input_tokens);
|
||||
|
||||
// the children start from the shared prefix, do not go past it
|
||||
if (slot.task->n_tokens_shared > 0) {
|
||||
n_past = std::min(n_past, slot.task->n_tokens_shared);
|
||||
}
|
||||
|
||||
// if there is an alora invoked, don't cache after the invocation start
|
||||
if (slot.alora_invocation_start > 0) {
|
||||
SLT_DBG(slot, "only caching to alora invocation start (n_past = %d, alora_invocation_start = %d)\n", n_past, slot.alora_invocation_start);
|
||||
@@ -3443,6 +3544,24 @@ private:
|
||||
|
||||
slot.mem.seq_rm(slot.id, p0, -1);
|
||||
|
||||
// shared prompt prefix: once it is processed, the children continue from it with their own prompt
|
||||
bool wait_shared = false;
|
||||
if (slot.task->n_tokens_shared > 0) {
|
||||
const bool is_shared_done = slot.prompt.n_tokens() == slot.task->n_tokens_shared;
|
||||
for (auto & other : slots) {
|
||||
if (other.state != SLOT_STATE_WAIT_OTHER || other.task->id_parent != slot.task->id) {
|
||||
continue;
|
||||
}
|
||||
if (is_shared_done) {
|
||||
SLT_TRC(slot, " - copying shared prompt (%d tokens) to child %d\n", slot.prompt.n_tokens(), other.id);
|
||||
slot.copy_prompt_to(other);
|
||||
other.state = SLOT_STATE_STARTED;
|
||||
} else {
|
||||
wait_shared = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// If using an alora, there may be uncached tokens that come
|
||||
// before the invocation sequence. When this happens, the
|
||||
// tokens before the invocation sequence need to be
|
||||
@@ -3520,6 +3639,8 @@ private:
|
||||
const auto & spans = slot.task->params.message_spans;
|
||||
const auto last_user_pos = spans.last_user_message_pos();
|
||||
|
||||
const int32_t n_decision_first = slot.task->type == SERVER_TASK_TYPE_DECISION ? slot.task->decision.pos_first() : -1;
|
||||
|
||||
// add prompt tokens for processing in the current batch
|
||||
while (slot.prompt.n_tokens() < slot.task->n_tokens() && batch.size() < n_batch) {
|
||||
// get next token to process
|
||||
@@ -3528,6 +3649,16 @@ private:
|
||||
break; // end of text chunk
|
||||
}
|
||||
|
||||
// stop at the end of the shared prefix, the children are started from this state
|
||||
if (wait_shared && slot.prompt.n_tokens() == slot.task->n_tokens_shared) {
|
||||
break;
|
||||
}
|
||||
|
||||
// the outputs of a decision are read from one batch, do not split them
|
||||
if (slot.prompt.n_tokens() == n_decision_first && batch.size() + slot.task->n_tokens() - n_decision_first > n_batch) {
|
||||
break;
|
||||
}
|
||||
|
||||
// if this is an alora request with pre-invocation
|
||||
// tokens that are not cached, we need to stop filling
|
||||
// this batch at those pre-invocation tokens.
|
||||
@@ -3771,6 +3902,8 @@ private:
|
||||
SLT_TRC(slot, " - copying state to child %d\n", child->id);
|
||||
|
||||
GGML_ASSERT(child->state == SLOT_STATE_WAIT_OTHER);
|
||||
// children with their own prompt are started at the end of the shared prefix
|
||||
GGML_ASSERT(slot.task->n_tokens_shared == 0);
|
||||
|
||||
slot.copy_state_to(*child);
|
||||
child->state = SLOT_STATE_DONE_PROMPT;
|
||||
@@ -3831,6 +3964,13 @@ private:
|
||||
return;
|
||||
}
|
||||
|
||||
if (slot.task->type == SERVER_TASK_TYPE_DECISION) {
|
||||
send_decision(slot, batch.view, slot.i_batch - off);
|
||||
slot.release();
|
||||
slot.i_batch = -1;
|
||||
return;
|
||||
}
|
||||
|
||||
GGML_ASSERT(slot.task->need_sampling());
|
||||
|
||||
// prompt evaluated for next-token prediction
|
||||
@@ -5224,6 +5364,76 @@ void server_routes::init_routes() {
|
||||
return res;
|
||||
};
|
||||
|
||||
this->post_systemone = [this](const server_http_req & req) {
|
||||
auto res = create_response();
|
||||
const auto & decision = ctx_server.decision;
|
||||
if (decision.type == COMMON_DECISION_TYPE_NONE) {
|
||||
res->error(format_error_response("This model is not a decision model", ERROR_TYPE_NOT_SUPPORTED));
|
||||
return res;
|
||||
}
|
||||
|
||||
const json body = json::parse(req.body);
|
||||
const auto questions = decision.parse_questions(body);
|
||||
|
||||
std::vector<raw_buffer> files;
|
||||
const json state = decision.parse_state(body, files);
|
||||
if (!files.empty() && (!decision.can_use_images() || !meta->has_inp_image)) {
|
||||
res->error(format_error_response("This server does not support image input for decisions. For a model that supports it, start it with `--mmproj`", ERROR_TYPE_NOT_SUPPORTED));
|
||||
return res;
|
||||
}
|
||||
|
||||
// one task per variant of each question
|
||||
auto & rd = res->rd;
|
||||
{
|
||||
std::vector<server_task> tasks;
|
||||
for (const auto & question : questions) {
|
||||
for (size_t variant = 0; variant < decision.n_variants(question); variant++) {
|
||||
server_task task = server_task(SERVER_TASK_TYPE_DECISION);
|
||||
task.id = rd.get_new_id();
|
||||
decision.fill_task(state, question, variant, files, ctx_server.mctx, ctx_server.init_opt, task);
|
||||
tasks.push_back(std::move(task));
|
||||
}
|
||||
}
|
||||
if (decision.can_share_prompt()) {
|
||||
tasks = server_decision_group_tasks(std::move(tasks), params.n_parallel);
|
||||
}
|
||||
rd.post_tasks(std::move(tasks));
|
||||
}
|
||||
|
||||
auto all_results = rd.wait_for_all(req.should_stop);
|
||||
|
||||
if (all_results.is_terminated) {
|
||||
return res; // connection is closed
|
||||
} else if (all_results.error) {
|
||||
res->error(all_results.error->to_json());
|
||||
return res;
|
||||
}
|
||||
|
||||
json answers = json::object();
|
||||
int32_t n_tokens = 0;
|
||||
size_t i_result = 0;
|
||||
for (const auto & question : questions) {
|
||||
std::vector<std::vector<float>> scores;
|
||||
for (size_t variant = 0; variant < decision.n_variants(question); variant++) {
|
||||
auto * result = dynamic_cast<server_task_result_decision *>(all_results.results[i_result++].get());
|
||||
GGML_ASSERT(result != nullptr);
|
||||
scores.push_back(result->scores);
|
||||
n_tokens += result->n_tokens;
|
||||
}
|
||||
answers[question.id] = decision.format_answer(question, scores);
|
||||
}
|
||||
|
||||
res->ok(json{
|
||||
{"model", meta->model_name},
|
||||
{"answers", answers},
|
||||
{"usage", {
|
||||
{"input_tokens", n_tokens},
|
||||
{"output_tokens", 0},
|
||||
}},
|
||||
});
|
||||
return res;
|
||||
};
|
||||
|
||||
this->get_lora_adapters = [this](const server_http_req & req) {
|
||||
auto res = create_response();
|
||||
|
||||
|
||||
Reference in new issue
Block a user