model : add LiquidAI/d1-omni-600M decision model (#30114)
* model : add LiquidAI/d1-omni-600M decision model Assisted-by: Claude Opus 5.5 * mtmd : keep conformer GLU sigmoid on CUDA Assisted-by: Claude Opus 5.5 * server : take d1omni audio through images and input_audio, scope memory-less lfm2 to non-causal Assisted-by: Claude Opus 5.5 * common : rename decision type d1omni to lfm2-d1-omni, server : make images an alias of files Assisted-by: Claude Opus 5.5
This commit is contained in:
1 parent
aa5e0092fd
commit
a657f7e981
21 files changed
+833
-35
No files matched your search
@@ -126,6 +126,12 @@ void server_decision_context::init(const llama_model * model) {
|
||||
} else if (model_type == COMMON_DECISION_TYPE_LFM2_D1) {
|
||||
n_options_max = 255;
|
||||
noul_true_first = true;
|
||||
} else if (model_type == COMMON_DECISION_TYPE_LFM2_D1_OMNI) {
|
||||
token_marker = llama_vocab_mask(vocab);
|
||||
if (token_marker == LLAMA_TOKEN_NULL) {
|
||||
throw std::runtime_error("decision model has no mask token");
|
||||
}
|
||||
n_options_max = 255;
|
||||
} else {
|
||||
throw std::runtime_error("unsupported decision model type: " + type_name);
|
||||
}
|
||||
@@ -139,8 +145,8 @@ void server_decision_context::init(const llama_model * model) {
|
||||
//
|
||||
|
||||
std::vector<server_decision_question> server_decision_context::parse_questions(const json & body) const {
|
||||
// d1 accepts a null state (images only)
|
||||
if (!body.contains("state") || (body.at("state").is_null() && type != COMMON_DECISION_TYPE_LFM2_D1)) {
|
||||
// lfm2-d1 and lfm2-d1-omni accept a null state, for example to ask about images only
|
||||
if (!body.contains("state") || (body.at("state").is_null() && type != COMMON_DECISION_TYPE_LFM2_D1 && type != COMMON_DECISION_TYPE_LFM2_D1_OMNI)) {
|
||||
throw std::invalid_argument("\"state\" must be provided");
|
||||
}
|
||||
if (!body.contains("questions") || !body.at("questions").is_object() || body.at("questions").empty()) {
|
||||
@@ -193,7 +199,15 @@ std::vector<server_decision_question> server_decision_context::parse_questions(c
|
||||
throw err("\"criteria\" must be an object");
|
||||
}
|
||||
for (const char * key : {"false", "true"}) {
|
||||
question.options.push_back({key, criteria.is_object() && criteria.contains(key) ? criteria.at(key) : json()});
|
||||
json description;
|
||||
if (criteria.is_object() && criteria.contains(key)) {
|
||||
description = criteria.at(key);
|
||||
} else if (criteria.is_object() && type == COMMON_DECISION_TYPE_LFM2_D1_OMNI) {
|
||||
// lfm2-d1-omni also reads the descriptions under "no" and "yes"
|
||||
const char * alias = std::string(key) == "true" ? "yes" : "no";
|
||||
description = criteria.contains(alias) ? criteria.at(alias) : json();
|
||||
}
|
||||
question.options.push_back({key, description});
|
||||
}
|
||||
if (noul_true_first) {
|
||||
std::swap(question.options[0], question.options[1]);
|
||||
@@ -217,9 +231,10 @@ std::vector<server_decision_question> server_decision_context::parse_questions(c
|
||||
|
||||
static const size_t DECISION_MAX_IMAGES = 8;
|
||||
|
||||
// any media, mtmd tells an audio clip from an image by its content
|
||||
static void decision_load_image(const json & url, std::vector<raw_buffer> & files) {
|
||||
if (!url.is_string() || !string_starts_with(url.get<std::string>(), "data:image/")) {
|
||||
throw std::invalid_argument("images must be data URLs (data:image/...;base64,...)");
|
||||
if (!url.is_string() || !string_starts_with(url.get<std::string>(), "data:")) {
|
||||
throw std::invalid_argument("images must be data URLs (data:image/...;base64,... or data:audio/...;base64,...)");
|
||||
}
|
||||
if (files.size() >= DECISION_MAX_IMAGES) {
|
||||
throw std::invalid_argument(string_format("too many images, the maximum is %zu", DECISION_MAX_IMAGES));
|
||||
@@ -227,15 +242,30 @@ static void decision_load_image(const json & url, std::vector<raw_buffer> & file
|
||||
handle_media(files, url.get<std::string>(), "");
|
||||
}
|
||||
|
||||
// audio is base64 data, as a data URL or not (OpenAI input_audio)
|
||||
static void decision_load_audio(const json & data, std::vector<raw_buffer> & files) {
|
||||
if (!data.is_string() || string_starts_with(data.get<std::string>(), "http") || string_starts_with(data.get<std::string>(), "file://")) {
|
||||
throw std::invalid_argument("audio must be base64 data");
|
||||
}
|
||||
if (files.size() >= DECISION_MAX_IMAGES) {
|
||||
throw std::invalid_argument(string_format("too many media files, the maximum is %zu", DECISION_MAX_IMAGES));
|
||||
}
|
||||
handle_media(files, data.get<std::string>(), "");
|
||||
}
|
||||
|
||||
json server_decision_context::parse_state(const json & body, std::vector<raw_buffer> & files) const {
|
||||
if (body.contains("videos") && !body.at("videos").is_null() && !body.at("videos").empty()) {
|
||||
throw std::invalid_argument("\"videos\" is not supported");
|
||||
}
|
||||
if (body.contains("images") && !body.at("images").is_null()) {
|
||||
if (!body.at("images").is_array()) {
|
||||
throw std::invalid_argument("\"images\" must be an array");
|
||||
// "images" is an alias of "files"
|
||||
for (const char * key : {"files", "images"}) {
|
||||
if (!body.contains(key) || body.at(key).is_null()) {
|
||||
continue;
|
||||
}
|
||||
for (const auto & url : body.at("images")) {
|
||||
if (!body.at(key).is_array()) {
|
||||
throw std::invalid_argument(string_format("\"%s\" must be an array", key));
|
||||
}
|
||||
for (const auto & url : body.at(key)) {
|
||||
decision_load_image(url, files);
|
||||
}
|
||||
}
|
||||
@@ -247,7 +277,7 @@ json server_decision_context::parse_state(const json & body, std::vector<raw_buf
|
||||
return state;
|
||||
}
|
||||
|
||||
// chat messages: take the image parts out of the content
|
||||
// chat messages: take the image and audio parts out of the content
|
||||
json messages_out = json::array();
|
||||
for (const auto & msg : messages) {
|
||||
if (!msg.is_object() || !msg.contains("content") || !msg.at("content").is_array()) {
|
||||
@@ -259,6 +289,9 @@ json server_decision_context::parse_state(const json & body, std::vector<raw_buf
|
||||
if (part.is_object() && json_value(part, "type", std::string()) == "image_url" && part.contains("image_url")) {
|
||||
const json & image_url = part.at("image_url");
|
||||
decision_load_image(image_url.is_object() && image_url.contains("url") ? image_url.at("url") : image_url, files);
|
||||
} else if (part.is_object() && json_value(part, "type", std::string()) == "input_audio" && part.contains("input_audio")) {
|
||||
const json input_audio = json_value(part, "input_audio", json::object());
|
||||
decision_load_audio(input_audio.contains("data") ? input_audio.at("data") : json_value(input_audio, "url", json()), files);
|
||||
} else {
|
||||
content.push_back(part);
|
||||
}
|
||||
@@ -364,6 +397,41 @@ static std::string decision_kev_text(const json & val) {
|
||||
return std::regex_replace(decision_kev_render(val), re_special, "<\xC2\xA6$1\xC2\xA6>");
|
||||
}
|
||||
|
||||
// lfm2-d1-omni: special tokens written in the input must not be parsed as such, in keys too (d1-omni prompt.py: escape)
|
||||
static json decision_d1omni_escape(const json & val) {
|
||||
static const std::regex re_special("<\\|([A-Za-z0-9_]+)\\|>");
|
||||
if (val.is_string()) {
|
||||
return std::regex_replace(val.get<std::string>(), re_special, "<\xC2\xA6$1\xC2\xA6>");
|
||||
}
|
||||
if (val.is_array()) {
|
||||
json out = json::array();
|
||||
for (const auto & item : val) {
|
||||
out.push_back(decision_d1omni_escape(item));
|
||||
}
|
||||
return out;
|
||||
}
|
||||
if (val.is_object()) {
|
||||
json out = json::object();
|
||||
for (const auto & [key, item] : val.items()) {
|
||||
out[std::regex_replace(key, re_special, "<\xC2\xA6$1\xC2\xA6>")] = decision_d1omni_escape(item);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
return val;
|
||||
}
|
||||
|
||||
// given to the lfm2-d1-omni template: text between the pieces of the prompt, and at the start of the pieces that are cut to a token budget
|
||||
static const std::string D1OMNI_MARKER = "<<d1omni:";
|
||||
static const std::string D1OMNI_SEP = "<<d1omni:sep>>";
|
||||
static const std::string D1OMNI_MARK_STATE = "<<d1omni:state>>";
|
||||
static const std::string D1OMNI_MARK_QUESTION = "<<d1omni:question>>";
|
||||
static const std::string D1OMNI_MARK_OPTION = "<<d1omni:option>>";
|
||||
|
||||
// max_length, image_text_length and audio_text_length of the model config, the converter checks them
|
||||
static const size_t D1OMNI_MAX_TOKENS = 16384;
|
||||
static const size_t D1OMNI_MAX_TOKENS_IMAGE = 896;
|
||||
static const size_t D1OMNI_MAX_TOKENS_AUDIO = 15360;
|
||||
|
||||
size_t server_decision_context::n_variants(const server_decision_question & question) const {
|
||||
// lev shows the options of a choice in 2 orders, to cancel the preference for the first label
|
||||
if (type == COMMON_DECISION_TYPE_LEV && question.type == SERVER_DECISION_QUESTION_CHOICE && question.options.size() > 1) {
|
||||
@@ -515,7 +583,8 @@ std::string server_decision_context::render(
|
||||
const std::vector<server_decision_question> & questions,
|
||||
const server_decision_question & question,
|
||||
size_t variant,
|
||||
size_t n_images) const {
|
||||
size_t n_images,
|
||||
bool is_audio) const {
|
||||
// the template is given raw JSON values, it serializes the ones that are not strings
|
||||
json inp = json{
|
||||
{"id", question.id},
|
||||
@@ -554,6 +623,15 @@ std::string server_decision_context::render(
|
||||
inp = decision_replace_text(inp, text_marker, " ");
|
||||
}
|
||||
|
||||
if (type == COMMON_DECISION_TYPE_LFM2_D1_OMNI) {
|
||||
inp = decision_replace_text(decision_d1omni_escape(inp), D1OMNI_MARKER, "<<d1omni ");
|
||||
inp["audio"] = is_audio;
|
||||
inp["sep"] = D1OMNI_SEP;
|
||||
inp["mark_state"] = D1OMNI_MARK_STATE;
|
||||
inp["mark_question"] = D1OMNI_MARK_QUESTION;
|
||||
inp["mark_option"] = D1OMNI_MARK_OPTION;
|
||||
}
|
||||
|
||||
// the template puts one media marker per image
|
||||
json images = json::array();
|
||||
if (n_images > 0) {
|
||||
@@ -580,6 +658,11 @@ void server_decision_context::fill_task(
|
||||
mtmd_context * mctx,
|
||||
const mtmd_helper_init_opt & init_opt,
|
||||
server_task & task) const {
|
||||
if (type == COMMON_DECISION_TYPE_LFM2_D1_OMNI) {
|
||||
fill_task_d1omni(state, questions, question, variant, files, mctx, init_opt, task);
|
||||
return;
|
||||
}
|
||||
|
||||
const std::string prompt = render(state, questions, question, variant, files.size());
|
||||
|
||||
if (type == COMMON_DECISION_TYPE_OPENJEV || type == COMMON_DECISION_TYPE_LEV || type == COMMON_DECISION_TYPE_NIMBLE || type == COMMON_DECISION_TYPE_PPLX_DECIDER) {
|
||||
@@ -678,6 +761,129 @@ void server_decision_context::fill_task_laya(llama_tokens & tokens, const server
|
||||
task.decision.column = question.type;
|
||||
}
|
||||
|
||||
// each piece is cut to its budget as the model was trained (d1-omni prompt.py: encode), the state gets the room that is left
|
||||
// the media come first, the text after them has a budget of its own
|
||||
void server_decision_context::fill_task_d1omni(
|
||||
const json & state,
|
||||
const std::vector<server_decision_question> & questions,
|
||||
const server_decision_question & question,
|
||||
size_t variant,
|
||||
const std::vector<raw_buffer> & files,
|
||||
mtmd_context * mctx,
|
||||
const mtmd_helper_init_opt & init_opt,
|
||||
server_task & task) const {
|
||||
const auto invalid = std::runtime_error("unexpected layout of the decision prompt");
|
||||
|
||||
// the prompt depends on the kind of media, mtmd tells an audio clip from an image by its content
|
||||
std::string markers;
|
||||
server_tokens media(llama_tokens(), false);
|
||||
bool is_audio = false;
|
||||
if (!files.empty()) {
|
||||
for (size_t i = 0; i < files.size(); i++) {
|
||||
markers += get_media_marker();
|
||||
}
|
||||
media = process_mtmd_prompt(mctx, markers, files, init_opt);
|
||||
for (size_t i = 0; i < media.size(); i++) {
|
||||
if (media[i] == LLAMA_TOKEN_NULL) {
|
||||
const auto & chunk = media.find_chunk(i);
|
||||
is_audio = is_audio || mtmd_input_chunk_get_type(chunk.get()) == MTMD_INPUT_CHUNK_TYPE_AUDIO;
|
||||
i += mtmd_input_chunk_get_n_tokens(chunk.get()) - 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
if (is_audio && files.size() > 1) {
|
||||
throw std::invalid_argument("a request has images or one audio clip, not both");
|
||||
}
|
||||
const size_t n_media = media.size();
|
||||
|
||||
const std::string prompt = render(state, questions, question, variant, files.size(), is_audio);
|
||||
std::vector<std::string> pieces = string_split(prompt, D1OMNI_SEP);
|
||||
if (pieces.empty() || pieces[0] != markers) {
|
||||
throw invalid;
|
||||
}
|
||||
|
||||
size_t n_max = D1OMNI_MAX_TOKENS;
|
||||
if (!files.empty()) {
|
||||
n_max = std::min(is_audio ? D1OMNI_MAX_TOKENS_AUDIO : D1OMNI_MAX_TOKENS_IMAGE, D1OMNI_MAX_TOKENS - std::min(D1OMNI_MAX_TOKENS, n_media));
|
||||
if (n_max < 64) {
|
||||
throw std::invalid_argument(string_format("the media take %zu of the %zu positions, send fewer images", n_media, D1OMNI_MAX_TOKENS));
|
||||
}
|
||||
}
|
||||
|
||||
// the options get max(96, min(24 n + 32, max / 2)) tokens, shared evenly
|
||||
const int64_t n_options = question.options.size();
|
||||
const int64_t n_budget = std::max<int64_t>(96, std::min<int64_t>(24 * n_options + 32, n_max / 2));
|
||||
const int64_t n_option_max = std::max<int64_t>(2, (n_budget - 3 * n_options) / n_options);
|
||||
const int64_t n_question_max = std::max<int64_t>(16, n_budget);
|
||||
|
||||
llama_tokens head; // before the state
|
||||
llama_tokens body; // the state
|
||||
llama_tokens tail; // after the state
|
||||
bool has_state = false;
|
||||
for (size_t i_piece = 1; i_piece < pieces.size(); i_piece++) {
|
||||
std::string piece = pieces[i_piece];
|
||||
int64_t n_piece_max = -1;
|
||||
bool is_state = false;
|
||||
if (string_starts_with(piece, D1OMNI_MARK_STATE)) {
|
||||
piece = piece.substr(D1OMNI_MARK_STATE.size());
|
||||
is_state = true;
|
||||
} else if (string_starts_with(piece, D1OMNI_MARK_QUESTION)) {
|
||||
piece = piece.substr(D1OMNI_MARK_QUESTION.size());
|
||||
n_piece_max = n_question_max;
|
||||
} else if (string_starts_with(piece, D1OMNI_MARK_OPTION)) {
|
||||
piece = piece.substr(D1OMNI_MARK_OPTION.size());
|
||||
n_piece_max = n_option_max;
|
||||
}
|
||||
|
||||
llama_tokens tokens = common_tokenize(vocab, piece, false, true);
|
||||
if (n_piece_max >= 0 && (int64_t) tokens.size() > n_piece_max) {
|
||||
tokens.resize(n_piece_max);
|
||||
}
|
||||
|
||||
if (is_state) {
|
||||
if (has_state) {
|
||||
throw invalid;
|
||||
}
|
||||
body = std::move(tokens);
|
||||
has_state = true;
|
||||
} else {
|
||||
llama_tokens & dst = has_state ? tail : head;
|
||||
dst.insert(dst.end(), tokens.begin(), tokens.end());
|
||||
}
|
||||
}
|
||||
if (!has_state) {
|
||||
throw invalid;
|
||||
}
|
||||
|
||||
const size_t n_room = n_max - std::min(n_max, head.size() + tail.size());
|
||||
body.resize(std::min(body.size(), n_room));
|
||||
|
||||
llama_tokens tokens = std::move(head);
|
||||
tokens.insert(tokens.end(), body.begin(), body.end());
|
||||
tokens.insert(tokens.end(), tail.begin(), tail.end());
|
||||
tokens.resize(std::min(tokens.size(), n_max));
|
||||
|
||||
for (size_t i = 0; i < tokens.size(); i++) {
|
||||
if (tokens[i] == token_marker) {
|
||||
task.decision.markers.push_back(n_media + i);
|
||||
}
|
||||
}
|
||||
if ((int64_t) task.decision.markers.size() != n_options) {
|
||||
throw std::invalid_argument("the options do not fit in the context");
|
||||
}
|
||||
|
||||
// the output has one score per question type
|
||||
task.decision.column = question.type;
|
||||
if (files.empty()) {
|
||||
task.tokens = server_tokens(tokens, false);
|
||||
} else {
|
||||
task.tokens = std::move(media);
|
||||
for (const llama_token token : tokens) {
|
||||
task.tokens.push_back(token);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
//
|
||||
// joint prompt (clef)
|
||||
//
|
||||
@@ -850,14 +1056,15 @@ static double decision_confidence_score(const std::vector<double> & probs) {
|
||||
return std::max(0.0, 1.0 - dist / dist_uniform);
|
||||
}
|
||||
|
||||
json server_decision_context::format_answer(const server_decision_question & question, const std::vector<std::vector<float>> & scores) const {
|
||||
json server_decision_context::format_answer(const server_decision_question & question, const std::vector<std::vector<float>> & scores, bool has_media) const {
|
||||
const size_t n = n_outputs(question);
|
||||
if (scores.size() != n_variants(question)) {
|
||||
throw std::runtime_error("decision result does not match the number of variants");
|
||||
}
|
||||
|
||||
// softmax over the outputs of each variant, then the average of the variants
|
||||
const float temperature = get_temperature(question);
|
||||
// lfm2-d1-omni: image and audio answers are not calibrated
|
||||
const float temperature = has_media && type == COMMON_DECISION_TYPE_LFM2_D1_OMNI ? 1.0f : get_temperature(question);
|
||||
std::vector<double> probs(n, 0.0);
|
||||
for (size_t v = 0; v < scores.size(); v++) {
|
||||
const auto & s = scores[v];
|
||||
|
||||
Reference in new issue
Block a user