model: Add MiniCPM-OCR-Jev (#30980)
* model : add MiniCPM-OCR-Jev support The model is a Qwen3.5-VL trunk with a value head read at the last token: it scores an OCR text against an image rendered from it, it does not generate. Add the openbmb-minicpm-ocr-jev decision type, served by POST /v1/systemone. The class scores are the pooled output of the sequence, read with last pooling, so the conversion turns score.weight into cls.output and sets embedding_length_out. The checkpoint has no LM head, so text generation is rejected instead of using the output weights duplicated from the token embeddings. The task is fixed, so its prompt is entirely in the systemone template, the same way h2o-lightning keeps its default there. The processor of the checkpoint allows up to 16384 merged tokens per image, more than the Qwen-VL default of mtmd, so the mmproj keeps the image budget of the preprocessor config. The checkpoint declares MiniCPMOCRForSequenceClassification in its config.json, as other models do. Signed-off-by: tc-mb <tianchi_cai@icloud.com> * fix code Signed-off-by: tc-mb <tianchi_cai@icloud.com> * use gguf meta classifier_output_labels * remove redudant code changes --------- Signed-off-by: tc-mb <tianchi_cai@icloud.com> Co-authored-by: Xuan Son Nguyen <son@huggingface.co>
This commit is contained in:
1 parent
276378515c
commit
7eed6b7685
13 files changed
+270
-10
No files matched your search
@@ -204,6 +204,15 @@ void server_decision_context::init(const llama_model * model) {
|
||||
}
|
||||
n_options_max = labels.size();
|
||||
noul_true_first = true;
|
||||
} else if (model_type == COMMON_DECISION_TYPE_MINICPM_OCR_JEV) {
|
||||
// the classes come from the GGUF classifier labels, the request API is binary: [inconsistent, consistent]
|
||||
const uint32_t n_classes = llama_model_n_cls_out(model);
|
||||
if (n_classes != 2 || llama_model_n_embd_out(model) != (int32_t) n_classes) {
|
||||
throw std::runtime_error(string_format("decision model must have 2 classifier labels and output one score per class, got %u labels and %d scores",
|
||||
n_classes, llama_model_n_embd_out(model)));
|
||||
}
|
||||
n_options_max = n_classes;
|
||||
choice_sorted = true;
|
||||
} else {
|
||||
throw std::runtime_error("unsupported decision model type: " + type_name);
|
||||
}
|
||||
@@ -231,6 +240,9 @@ std::vector<server_decision_question> server_decision_context::parse_questions(c
|
||||
if (!body.contains("questions") || !body.at("questions").is_object() || body.at("questions").empty()) {
|
||||
throw std::invalid_argument("\"questions\" must be a non-empty object");
|
||||
}
|
||||
if (type == COMMON_DECISION_TYPE_MINICPM_OCR_JEV && body.at("questions").size() != 1) {
|
||||
throw std::invalid_argument("this model answers exactly one question");
|
||||
}
|
||||
|
||||
std::vector<server_decision_question> questions;
|
||||
for (const auto & [id, q] : body.at("questions").items()) {
|
||||
@@ -244,7 +256,9 @@ std::vector<server_decision_question> server_decision_context::parse_questions(c
|
||||
// h2o-lightning: the instructions can be left out, the template has a default, and the type defaults to choice
|
||||
const bool is_h2o = type == COMMON_DECISION_TYPE_H2O_LIGHTNING;
|
||||
const json instructions = q.contains("instructions") ? q.at("instructions") : (is_h2o ? json(std::string()) : json());
|
||||
if (instructions.is_null() && type != COMMON_DECISION_TYPE_VLLM_D3 && !is_h2o) {
|
||||
// minicpm-ocr-jev answers one fixed question, the prompt is entirely in the template
|
||||
const bool is_ocr_jev = type == COMMON_DECISION_TYPE_MINICPM_OCR_JEV;
|
||||
if (instructions.is_null() && type != COMMON_DECISION_TYPE_VLLM_D3 && !is_h2o && !is_ocr_jev) {
|
||||
throw err("\"instructions\" must be provided");
|
||||
}
|
||||
if (strict_payload && (!decision_is_payload(instructions) || instructions == "")) {
|
||||
@@ -322,6 +336,26 @@ std::vector<server_decision_question> server_decision_context::parse_questions(c
|
||||
throw err("\"type\" must be one of: choice, score, noul");
|
||||
}
|
||||
|
||||
// the classes are fixed: "0" is inconsistent, "1" is consistent
|
||||
if (type == COMMON_DECISION_TYPE_MINICPM_OCR_JEV) {
|
||||
// the task text is in the template, an instructions from the request would be dropped
|
||||
if (!instructions.is_null()) {
|
||||
throw err("\"instructions\" is not used by this model, the task is part of its prompt");
|
||||
}
|
||||
if (question.type == SERVER_DECISION_QUESTION_CHOICE) {
|
||||
const auto & opts = question.options;
|
||||
if (opts.size() != 2 || opts[0].key != "0" || opts[1].key != "1" || !opts[0].description.is_null() || !opts[1].description.is_null()) {
|
||||
throw err("\"criteria\" must be {\"0\": null, \"1\": null}");
|
||||
}
|
||||
} else if (question.type == SERVER_DECISION_QUESTION_NOUL) {
|
||||
if (!criteria.is_null()) {
|
||||
throw err("\"criteria\" is not supported for a noul question");
|
||||
}
|
||||
} else {
|
||||
throw err("only choice and noul questions are supported");
|
||||
}
|
||||
}
|
||||
|
||||
if (question.options.size() > n_options_max) {
|
||||
throw err(string_format("too many options (%zu), this model supports at most %zu", question.options.size(), n_options_max));
|
||||
}
|
||||
@@ -360,6 +394,9 @@ static void decision_load_audio(const json & data, std::vector<raw_buffer> & fil
|
||||
}
|
||||
|
||||
json server_decision_context::parse_state(const json & body, std::vector<raw_buffer> & files) const {
|
||||
if (type == COMMON_DECISION_TYPE_MINICPM_OCR_JEV && !body.at("state").is_string()) {
|
||||
throw std::invalid_argument("\"state\" must be the OCR text");
|
||||
}
|
||||
if (body.contains("videos") && !body.at("videos").is_null() && !body.at("videos").empty()) {
|
||||
throw std::invalid_argument("\"videos\" is not supported");
|
||||
}
|
||||
@@ -376,6 +413,11 @@ json server_decision_context::parse_state(const json & body, std::vector<raw_buf
|
||||
}
|
||||
}
|
||||
|
||||
// the prompt has the original image, then the image rendered from the OCR text
|
||||
if (type == COMMON_DECISION_TYPE_MINICPM_OCR_JEV && files.size() != 2) {
|
||||
throw std::invalid_argument(string_format("this model needs exactly 2 images (the original, then the rendered one), got %zu", files.size()));
|
||||
}
|
||||
|
||||
const json & state = body.at("state");
|
||||
const bool is_wrapped = state.is_object() && state.contains("messages");
|
||||
const json & messages = is_wrapped ? state.at("messages") : state;
|
||||
@@ -1040,6 +1082,11 @@ std::string server_decision_context::render(
|
||||
inp["instructions"] = decision_kev_text(question.instructions);
|
||||
}
|
||||
|
||||
// the OCR text must not be parsed as special tokens
|
||||
if (type == COMMON_DECISION_TYPE_MINICPM_OCR_JEV) {
|
||||
inp["state"] = server_decision_escape_special_tokens(state.get<std::string>());
|
||||
}
|
||||
|
||||
// the input must not contain the marker of the options
|
||||
if (!text_marker.empty()) {
|
||||
inp = decision_replace_text(inp, text_marker, " ");
|
||||
@@ -1092,6 +1139,12 @@ void server_decision_context::fill_task(
|
||||
|
||||
const std::string prompt = render(state, questions, question, variant, files.size());
|
||||
|
||||
if (type == COMMON_DECISION_TYPE_MINICPM_OCR_JEV) {
|
||||
task.tokens = process_mtmd_prompt(mctx, prompt, files, init_opt);
|
||||
task.decision.pooled = true;
|
||||
return;
|
||||
}
|
||||
|
||||
if (type == COMMON_DECISION_TYPE_OPENJEV || type == COMMON_DECISION_TYPE_GRANITE_SWITCH || type == COMMON_DECISION_TYPE_LEV || type == COMMON_DECISION_TYPE_NIMBLE || type == COMMON_DECISION_TYPE_PPLX_DECIDER || type == COMMON_DECISION_TYPE_VLLM_D3 ||
|
||||
type == COMMON_DECISION_TYPE_H2O_LIGHTNING) {
|
||||
// lev reads the ratings of a noul question at its first labels, not at the digits
|
||||
@@ -1531,8 +1584,8 @@ json server_decision_context::format_answer(const server_decision_question & que
|
||||
if (s.size() != n) {
|
||||
throw std::runtime_error("decision result does not match the number of options");
|
||||
}
|
||||
// a joint head returns NaN if it could not use the decision order
|
||||
if (std::any_of(s.begin(), s.end(), [](float v) { return std::isnan(v); })) {
|
||||
// a joint head returns NaN if it could not use the decision order, +inf would make the softmax NaN
|
||||
if (std::any_of(s.begin(), s.end(), [](float v) { return std::isnan(v) || v == INFINITY; })) {
|
||||
throw std::runtime_error("the model could not evaluate the decision");
|
||||
}
|
||||
std::vector<double> logits(s.begin(), s.end());
|
||||
|
||||
Reference in new issue
Block a user