Lumpiasty/llama.cpp
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

args: add --video-* CLI arguments (#24318)

Browse Source 
* args: add --video-* CLI arguments

* gen docs

* nits

* add mtmd_helper_init_opt
This commit is contained in:
Xuan-Son Nguyen
2026-08-27 12:11:12 +02:00
committed by GitHub
parent 915dc6d38c
commit f29551215b
12 changed files with 138 additions and 48 deletions
Download Patch File Download Diff File Expand all files Collapse all files
common/arg.cpp
+21
View File
@@ -2644,6 +2644,27 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
params.mtmd_batch_max_tokens = value; params.mtmd_batch_max_tokens = value;
} }
).set_examples({LLAMA_EXAMPLE_SERVER}).set_env("LLAMA_ARG_MTMD_BATCH_MAX_TOKENS")); ).set_examples({LLAMA_EXAMPLE_SERVER}).set_env("LLAMA_ARG_MTMD_BATCH_MAX_TOKENS"));
add_opt(common_arg(
{"--video-fps"}, "N",
string_format("target video frame rate (default: %.1f)", params.video_fps),
[](common_params & params, const std::string & value) {
params.video_fps = std::stof(value);
}
).set_examples(mmproj_examples).set_env("LLAMA_ARG_VIDEO_FPS"));
add_opt(common_arg(
{"--video-timestamp-interval"}, "N",
string_format("interval in milliseconds between text timestamps (default: %" PRId64 ")", params.video_timestamp_interval_ms),
[](common_params & params, int value) {
params.video_timestamp_interval_ms = value;
}
).set_examples(mmproj_examples).set_env("LLAMA_ARG_VIDEO_TIMESTAMP_INTERVAL"));
add_opt(common_arg(
{"--video-ffmpeg-dir"}, "DIR",
"path to the directory containing ffmpeg and ffprobe (default: search in PATH)",
[](common_params & params, const std::string & value) {
params.video_ffmpeg_bin_dir = value;
}
).set_examples(mmproj_examples).set_env("LLAMA_ARG_VIDEO_FFMPEG_DIR"));
if (params.is_gen_docs || llama_supports_rpc()) { if (params.is_gen_docs || llama_supports_rpc()) {
add_opt(common_arg( add_opt(common_arg(
{"--rpc"}, "SERVERS", {"--rpc"}, "SERVERS",
common/common.h
+5
View File
@@ -590,6 +590,11 @@ struct common_params {
int image_max_tokens = -1; int image_max_tokens = -1;
int mtmd_batch_max_tokens = 1024; int mtmd_batch_max_tokens = 1024;
// for video input
float video_fps = 4.0f;
int64_t video_timestamp_interval_ms = 5000;
std::string video_ffmpeg_bin_dir = "";
// finetune // finetune
struct lr_opt lr; struct lr_opt lr;
enum ggml_opt_optimizer_type optimizer = GGML_OPT_OPTIMIZER_TYPE_ADAMW; enum ggml_opt_optimizer_type optimizer = GGML_OPT_OPTIMIZER_TYPE_ADAMW;
tools/cli/README.md
+3
View File
@@ -166,6 +166,9 @@
| `--image, --audio, --video FILE` | path to an image, audio, or video file. use with multimodal models, use comma-separated values for multiple files | | `--image, --audio, --video FILE` | path to an image, audio, or video file. use with multimodal models, use comma-separated values for multiple files |
| `--image-min-tokens N` | minimum number of tokens each image can take, only used by vision models with dynamic resolution (default: read from model)<br/>(env: LLAMA_ARG_IMAGE_MIN_TOKENS) | | `--image-min-tokens N` | minimum number of tokens each image can take, only used by vision models with dynamic resolution (default: read from model)<br/>(env: LLAMA_ARG_IMAGE_MIN_TOKENS) |
| `--image-max-tokens N` | maximum number of tokens each image can take, only used by vision models with dynamic resolution (default: read from model)<br/>(env: LLAMA_ARG_IMAGE_MAX_TOKENS) | | `--image-max-tokens N` | maximum number of tokens each image can take, only used by vision models with dynamic resolution (default: read from model)<br/>(env: LLAMA_ARG_IMAGE_MAX_TOKENS) |
| `--video-fps N` | target video frame rate (default: 4.0)<br/>(env: LLAMA_ARG_VIDEO_FPS) |
| `--video-timestamp-interval N` | interval in milliseconds between text timestamps (default: 5000)<br/>(env: LLAMA_ARG_VIDEO_TIMESTAMP_INTERVAL) |
| `--video-ffmpeg-dir DIR` | path to the directory containing ffmpeg and ffprobe (default: search in PATH)<br/>(env: LLAMA_ARG_VIDEO_FFMPEG_DIR) |
| `-o, --output, --output-file FNAME` | output file (default: '') | | `-o, --output, --output-file FNAME` | output file (default: '') |
| `--chat-template-kwargs STRING` | sets additional params for the json template parser, must be a valid json object string, e.g. '{"key1":"value1","key2":"value2"}'<br/>(env: LLAMA_ARG_CHAT_TEMPLATE_KWARGS) | | `--chat-template-kwargs STRING` | sets additional params for the json template parser, must be a valid json object string, e.g. '{"key1":"value1","key2":"value2"}'<br/>(env: LLAMA_ARG_CHAT_TEMPLATE_KWARGS) |
| `--jinja, --no-jinja` | whether to use jinja template engine for chat (default: enabled)<br/>(env: LLAMA_ARG_JINJA) | | `--jinja, --no-jinja` | whether to use jinja template engine for chat (default: enabled)<br/>(env: LLAMA_ARG_JINJA) |
tools/mtmd/mtmd-cli.cpp
+10 -1
View File
@@ -87,6 +87,9 @@ struct mtmd_cli_context {
mtmd::bitmaps bitmaps; mtmd::bitmaps bitmaps;
std::vector<mtmd_helper::video_ptr> videos; std::vector<mtmd_helper::video_ptr> videos;
mtmd_helper_init_opt init_opt = mtmd_helper_init_opt_default();
std::string video_ffmpeg_bin_dir;
mtmd::batch_ptr mbatch; mtmd::batch_ptr mbatch;
// chat template // chat template
@@ -170,6 +173,12 @@ struct mtmd_cli_context {
LOG_ERR("Failed to load vision model from %s\n", clip_path); LOG_ERR("Failed to load vision model from %s\n", clip_path);
exit(1); exit(1);
} }
video_ffmpeg_bin_dir = params.video_ffmpeg_bin_dir;
init_opt.video_params.fps_target = params.video_fps;
init_opt.video_params.timestamp_interval_ms = params.video_timestamp_interval_ms;
init_opt.video_params.ffmpeg_bin_dir = video_ffmpeg_bin_dir.empty()
? nullptr : video_ffmpeg_bin_dir.c_str();
} }
bool check_antiprompt(const llama_tokens & generated_tokens) { bool check_antiprompt(const llama_tokens & generated_tokens) {
@@ -184,7 +193,7 @@ struct mtmd_cli_context {
} }
bool load_media(const std::string & fname) { bool load_media(const std::string & fname) {
auto res = mtmd_helper_bitmap_init_from_file(ctx_vision.get(), fname.c_str(), false); auto res = mtmd_helper_bitmap_init_from_file(ctx_vision.get(), fname.c_str(), false, init_opt);
if (!res.bitmap) { if (!res.bitmap) {
return false; return false;
} }
tools/mtmd/mtmd-helper.cpp
+19 -9
View File
@@ -369,14 +369,18 @@ static bool is_webp_file(const unsigned char * buf, size_t len) {
} }
#ifdef MTMD_VIDEO #ifdef MTMD_VIDEO
static mtmd_bitmap * decode_webp_with_ffmpeg(mtmd_context * mctx, const unsigned char * buf, size_t len, bool placeholder); static mtmd_bitmap * decode_webp_with_ffmpeg(mtmd_context * mctx, const unsigned char * buf, size_t len, bool placeholder,
const mtmd_helper_video_init_params & params);
#endif #endif
mtmd_helper_bitmap_wrapper mtmd_helper_bitmap_init_from_buf(mtmd_context * ctx, const unsigned char * buf, size_t len, bool placeholder) { mtmd_helper_bitmap_wrapper mtmd_helper_bitmap_init_from_buf(mtmd_context * ctx, const unsigned char * buf, size_t len, bool placeholder,
mtmd_helper_init_opt opt) {
// calculate the hash if needed // calculate the hash if needed
std::string id; std::string id;
mtmd_bitmap * result = nullptr; mtmd_bitmap * result = nullptr;
GGML_UNUSED(opt); // only used by video code paths
if (!placeholder) { if (!placeholder) {
// use sha256 to prevent cache poisoning // use sha256 to prevent cache poisoning
id = hash_sha256_hex(buf, len); id = hash_sha256_hex(buf, len);
@@ -414,7 +418,7 @@ mtmd_helper_bitmap_wrapper mtmd_helper_bitmap_init_from_buf(mtmd_context * ctx,
#ifdef MTMD_VIDEO #ifdef MTMD_VIDEO
// stb_image does not support webp; decode it with ffmpeg as a single frame // stb_image does not support webp; decode it with ffmpeg as a single frame
if (!result && is_webp_file(buf, len)) { if (!result && is_webp_file(buf, len)) {
result = decode_webp_with_ffmpeg(ctx, buf, len, placeholder); result = decode_webp_with_ffmpeg(ctx, buf, len, placeholder, opt.video_params);
if (!result) { if (!result) {
LOG_ERR("%s: failed to decode webp buffer\n", __func__); LOG_ERR("%s: failed to decode webp buffer\n", __func__);
return {nullptr, nullptr}; return {nullptr, nullptr};
@@ -427,8 +431,7 @@ mtmd_helper_bitmap_wrapper mtmd_helper_bitmap_init_from_buf(mtmd_context * ctx,
// last try: load as video // last try: load as video
#ifdef MTMD_VIDEO #ifdef MTMD_VIDEO
if (!result) { if (!result) {
auto params = mtmd_helper_video_init_params_default(); auto video_ctx = mtmd_helper_video_init_from_buf(ctx, buf, len, opt.video_params);
auto video_ctx = mtmd_helper_video_init_from_buf(ctx, buf, len, params);
if (!video_ctx) { if (!video_ctx) {
LOG_ERR("%s: failed to decode buffer as either image/audio/video\n", __func__); LOG_ERR("%s: failed to decode buffer as either image/audio/video\n", __func__);
return {nullptr, nullptr}; return {nullptr, nullptr};
@@ -456,7 +459,8 @@ mtmd_helper_bitmap_wrapper mtmd_helper_bitmap_init_from_buf(mtmd_context * ctx,
return {nullptr, nullptr}; return {nullptr, nullptr};
} }
mtmd_helper_bitmap_wrapper mtmd_helper_bitmap_init_from_file(mtmd_context * ctx, const char * fname, bool placeholder) { mtmd_helper_bitmap_wrapper mtmd_helper_bitmap_init_from_file(mtmd_context * ctx, const char * fname, bool placeholder,
mtmd_helper_init_opt opt) {
#ifdef _WIN32 #ifdef _WIN32
int wlen = MultiByteToWideChar(CP_UTF8, 0, fname, -1, NULL, 0); int wlen = MultiByteToWideChar(CP_UTF8, 0, fname, -1, NULL, 0);
if (!wlen) { if (!wlen) {
@@ -497,7 +501,7 @@ mtmd_helper_bitmap_wrapper mtmd_helper_bitmap_init_from_file(mtmd_context * ctx,
return {nullptr, nullptr}; return {nullptr, nullptr};
} }
return mtmd_helper_bitmap_init_from_buf(ctx, buf.data(), buf.size(), placeholder); return mtmd_helper_bitmap_init_from_buf(ctx, buf.data(), buf.size(), placeholder, opt);
} }
bool mtmd_helper_support_video(mtmd_context * ctx) { bool mtmd_helper_support_video(mtmd_context * ctx) {
@@ -855,6 +859,12 @@ mtmd_helper_video_init_params mtmd_helper_video_init_params_default() {
}; };
} }
mtmd_helper_init_opt mtmd_helper_init_opt_default() {
return {
/* video_params */ mtmd_helper_video_init_params_default(),
};
}
static std::string video_resolve_bin(const char * bin_dir, const char * name) { static std::string video_resolve_bin(const char * bin_dir, const char * name) {
if (!bin_dir || bin_dir[0] == '\0') { if (!bin_dir || bin_dir[0] == '\0') {
return name; // rely on PATH return name; // rely on PATH
@@ -876,8 +886,8 @@ static std::string video_resolve_bin(const char * bin_dir, const char * name) {
} }
#ifdef MTMD_VIDEO #ifdef MTMD_VIDEO
static mtmd_bitmap * decode_webp_with_ffmpeg(mtmd_context * mctx, const unsigned char * buf, size_t len, bool placeholder) { static mtmd_bitmap * decode_webp_with_ffmpeg(mtmd_context * mctx, const unsigned char * buf, size_t len, bool placeholder,
auto params = mtmd_helper_video_init_params_default(); const mtmd_helper_video_init_params & params) {
mtmd_helper_video vctx; mtmd_helper_video vctx;
vctx.mctx = mctx; vctx.mctx = mctx;
vctx.input_buf.assign(buf, buf + len); vctx.input_buf.assign(buf, buf + len);
tools/mtmd/mtmd-helper.h
+28 -10
View File
@@ -23,6 +23,23 @@ extern "C" {
struct mtmd_helper_video; struct mtmd_helper_video;
typedef struct mtmd_helper_video mtmd_helper_video; typedef struct mtmd_helper_video mtmd_helper_video;
struct mtmd_helper_video_init_params {
float fps_target; // desired output fps; <= 0 means use the video's native fps, defaulted to 4.0f
const char * ffmpeg_bin_dir; // directory containing ffmpeg/ffprobe binaries; NULL means search PATH
int64_t timestamp_interval_ms; // interval for adding timestamp as text chunk (example: "[10m50.5s]"); <= 0 means no timestamp, defaulted to 5000ms
// TODO @ngxson : allow "placeholder" bitmap output for counting tokens
};
MTMD_API struct mtmd_helper_video_init_params mtmd_helper_video_init_params_default(void);
// opt for mtmd_helper_bitmap_init_from_*()
struct mtmd_helper_init_opt {
struct mtmd_helper_video_init_params video_params;
};
typedef struct mtmd_helper_init_opt mtmd_helper_init_opt;
MTMD_API struct mtmd_helper_init_opt mtmd_helper_init_opt_default(void);
// Set callback for all future logging events. // Set callback for all future logging events.
// If this is not called, or NULL is supplied, everything is output on stderr. // If this is not called, or NULL is supplied, everything is output on stderr.
// Note: this also call mtmd_log_set() internally // Note: this also call mtmd_log_set() internally
@@ -40,7 +57,11 @@ struct mtmd_helper_bitmap_wrapper {
// it calls mtmd_helper_bitmap_init_from_buf() internally // it calls mtmd_helper_bitmap_init_from_buf() internally
// returns nullptr on failure // returns nullptr on failure
// this function is thread-safe // this function is thread-safe
MTMD_API struct mtmd_helper_bitmap_wrapper mtmd_helper_bitmap_init_from_file(mtmd_context * ctx, const char * fname, bool placeholder); MTMD_API struct mtmd_helper_bitmap_wrapper mtmd_helper_bitmap_init_from_file(
mtmd_context * ctx,
const char * fname,
bool placeholder,
struct mtmd_helper_init_opt opt);
// helper function to construct a mtmd_bitmap from a buffer containing a file // helper function to construct a mtmd_bitmap from a buffer containing a file
// supported formats: // supported formats:
@@ -53,7 +74,11 @@ MTMD_API struct mtmd_helper_bitmap_wrapper mtmd_helper_bitmap_init_from_file(mtm
// - output bitmap will have SHA-256 hash (hex string) as the ID // - output bitmap will have SHA-256 hash (hex string) as the ID
// returns nullptr on failure // returns nullptr on failure
// this function is thread-safe // this function is thread-safe
MTMD_API struct mtmd_helper_bitmap_wrapper mtmd_helper_bitmap_init_from_buf(mtmd_context * ctx, const unsigned char * buf, size_t len, bool placeholder); MTMD_API struct mtmd_helper_bitmap_wrapper mtmd_helper_bitmap_init_from_buf(
mtmd_context * ctx,
const unsigned char * buf, size_t len,
bool placeholder,
struct mtmd_helper_init_opt opt);
// helper to count the total number of tokens from a list of chunks, useful to keep track of KV cache // helper to count the total number of tokens from a list of chunks, useful to keep track of KV cache
MTMD_API size_t mtmd_helper_get_n_tokens(const mtmd_input_chunks * chunks); MTMD_API size_t mtmd_helper_get_n_tokens(const mtmd_input_chunks * chunks);
@@ -124,14 +149,7 @@ struct mtmd_helper_video_info {
int32_t n_frames; // estimated total frames at effective fps (-1 if unknown) int32_t n_frames; // estimated total frames at effective fps (-1 if unknown)
}; };
struct mtmd_helper_video_init_params { // note: mtmd_helper_video_init_params is defined at the top, as it is part of mtmd_helper_init_opt
float fps_target; // desired output fps; <= 0 means use the video's native fps, defaulted to 4.0f
const char * ffmpeg_bin_dir; // directory containing ffmpeg/ffprobe binaries; NULL means search PATH
int64_t timestamp_interval_ms; // interval for adding timestamp as text chunk (example: "[10m50.5s]"); <= 0 means no timestamp, defaulted to 5000ms
// TODO @ngxson : allow "placeholder" bitmap output for counting tokens
};
MTMD_API struct mtmd_helper_video_init_params mtmd_helper_video_init_params_default(void);
// returns NULL on failure (ffprobe not found, file unreadable, etc.) // returns NULL on failure (ffprobe not found, file unreadable, etc.)
MTMD_API mtmd_helper_video * mtmd_helper_video_init( MTMD_API mtmd_helper_video * mtmd_helper_video_init(
tools/server/README.md
+3
View File
@@ -182,6 +182,9 @@ For the full list of features, please refer to [server's changelog](https://gith
| `--image-min-tokens N` | minimum number of tokens each image can take, only used by vision models with dynamic resolution (default: read from model)<br/>(env: LLAMA_ARG_IMAGE_MIN_TOKENS) | | `--image-min-tokens N` | minimum number of tokens each image can take, only used by vision models with dynamic resolution (default: read from model)<br/>(env: LLAMA_ARG_IMAGE_MIN_TOKENS) |
| `--image-max-tokens N` | maximum number of tokens each image can take, only used by vision models with dynamic resolution (default: read from model)<br/>(env: LLAMA_ARG_IMAGE_MAX_TOKENS) | | `--image-max-tokens N` | maximum number of tokens each image can take, only used by vision models with dynamic resolution (default: read from model)<br/>(env: LLAMA_ARG_IMAGE_MAX_TOKENS) |
| `--mtmd-batch-max-tokens N` | maximum number of image tokens per batch when encoding images (default: 1024)<br/>(env: LLAMA_ARG_MTMD_BATCH_MAX_TOKENS) | | `--mtmd-batch-max-tokens N` | maximum number of image tokens per batch when encoding images (default: 1024)<br/>(env: LLAMA_ARG_MTMD_BATCH_MAX_TOKENS) |
| `--video-fps N` | target video frame rate (default: 4.0)<br/>(env: LLAMA_ARG_VIDEO_FPS) |
| `--video-timestamp-interval N` | interval in milliseconds between text timestamps (default: 5000)<br/>(env: LLAMA_ARG_VIDEO_TIMESTAMP_INTERVAL) |
| `--video-ffmpeg-dir DIR` | path to the directory containing ffmpeg and ffprobe (default: search in PATH)<br/>(env: LLAMA_ARG_VIDEO_FFMPEG_DIR) |
| `-a, --alias STRING` | set model name aliases, comma-separated (to be used by API)<br/>(env: LLAMA_ARG_ALIAS) | | `-a, --alias STRING` | set model name aliases, comma-separated (to be used by API)<br/>(env: LLAMA_ARG_ALIAS) |
| `--tags STRING` | set model tags, comma-separated (informational, not used for routing)<br/>(env: LLAMA_ARG_TAGS) | | `--tags STRING` | set model tags, comma-separated (informational, not used for routing)<br/>(env: LLAMA_ARG_TAGS) |
| `--embd-normalize N` | normalisation for embeddings (default: 2) (-1=none, 0=max absolute int16, 1=taxicab, 2=euclidean, >2=p-norm) | | `--embd-normalize N` | normalisation for embeddings (default: 2) (-1=none, 0=max absolute int16, 1=taxicab, 2=euclidean, >2=p-norm) |
tools/server/server-common.cpp
+17 -11
View File
@@ -910,12 +910,17 @@ size_t validate_utf8(const std::string& text) {
return len; return len;
} }
server_tokens process_mtmd_prompt(mtmd_context * mctx, const std::string & prompt, const std::vector<raw_buffer> & files, bool is_placeholder) { server_tokens process_mtmd_prompt(
mtmd_context * mctx,
const std::string & prompt,
const std::vector<raw_buffer> & files,
const mtmd_helper_init_opt & init_opt,
bool is_placeholder) {
// these will be freed upon going out of scope // these will be freed upon going out of scope
mtmd::bitmaps bitmaps; mtmd::bitmaps bitmaps;
std::vector<mtmd_helper::video_ptr> videos; std::vector<mtmd_helper::video_ptr> videos;
for (auto & file : files) { for (auto & file : files) {
auto out = mtmd_helper_bitmap_init_from_buf(mctx, file.data(), file.size(), is_placeholder); auto out = mtmd_helper_bitmap_init_from_buf(mctx, file.data(), file.size(), is_placeholder, init_opt);
if (!out.bitmap) { if (!out.bitmap) {
throw std::runtime_error("Failed to load image or audio file"); throw std::runtime_error("Failed to load image or audio file");
} }
@@ -956,7 +961,7 @@ server_tokens process_mtmd_prompt(mtmd_context * mctx, const std::string & promp
* - "prompt": [12, 34, "string", 56, 78] * - "prompt": [12, 34, "string", 56, 78]
* - "prompt": { "prompt_string": "string", "multimodal_data": [ "base64" ] } * - "prompt": { "prompt_string": "string", "multimodal_data": [ "base64" ] }
*/ */
static server_tokens tokenize_input_subprompt(const llama_vocab * vocab, mtmd_context * mctx, const json & json_prompt, bool add_special, bool parse_special) { static server_tokens tokenize_input_subprompt(const llama_vocab * vocab, mtmd_context * mctx, const json & json_prompt, bool add_special, bool parse_special, const mtmd_helper_init_opt & init_opt) {
constexpr char JSON_STRING_PROMPT_KEY[] = "prompt_string"; constexpr char JSON_STRING_PROMPT_KEY[] = "prompt_string";
constexpr char JSON_MTMD_DATA_KEY[] = "multimodal_data"; constexpr char JSON_MTMD_DATA_KEY[] = "multimodal_data";
const bool has_mtmd = mctx != nullptr; const bool has_mtmd = mctx != nullptr;
@@ -979,7 +984,7 @@ static server_tokens tokenize_input_subprompt(const llama_vocab * vocab, mtmd_co
for (const auto & entry : json_prompt.at(JSON_MTMD_DATA_KEY)) { for (const auto & entry : json_prompt.at(JSON_MTMD_DATA_KEY)) {
files.push_back(base64_decode(entry)); files.push_back(base64_decode(entry));
} }
return process_mtmd_prompt(mctx, json_prompt.at(JSON_STRING_PROMPT_KEY), files); return process_mtmd_prompt(mctx, json_prompt.at(JSON_STRING_PROMPT_KEY), files, init_opt);
} else { } else {
// Not multimodal, but contains a subobject. // Not multimodal, but contains a subobject.
llama_tokens tmp = tokenize_mixed(vocab, json_prompt.at(JSON_STRING_PROMPT_KEY), add_special, parse_special); llama_tokens tmp = tokenize_mixed(vocab, json_prompt.at(JSON_STRING_PROMPT_KEY), add_special, parse_special);
@@ -990,15 +995,15 @@ static server_tokens tokenize_input_subprompt(const llama_vocab * vocab, mtmd_co
} }
} }
std::vector<server_tokens> tokenize_input_prompts(const llama_vocab * vocab, mtmd_context * mctx, const json & json_prompt, bool add_special, bool parse_special) { std::vector<server_tokens> tokenize_input_prompts(const llama_vocab * vocab, mtmd_context * mctx, const json & json_prompt, bool add_special, bool parse_special, const mtmd_helper_init_opt & init_opt) {
std::vector<server_tokens> result; std::vector<server_tokens> result;
if (json_prompt.is_array() && !json_is_array_and_contains_numbers(json_prompt)) { if (json_prompt.is_array() && !json_is_array_and_contains_numbers(json_prompt)) {
result.reserve(json_prompt.size()); result.reserve(json_prompt.size());
for (const auto & p : json_prompt) { for (const auto & p : json_prompt) {
result.push_back(tokenize_input_subprompt(vocab, mctx, p,add_special, parse_special)); result.push_back(tokenize_input_subprompt(vocab, mctx, p, add_special, parse_special, init_opt));
} }
} else { } else {
result.push_back(tokenize_input_subprompt(vocab, mctx, json_prompt, add_special, parse_special)); result.push_back(tokenize_input_subprompt(vocab, mctx, json_prompt, add_special, parse_special, init_opt));
} }
if (result.empty()) { if (result.empty()) {
throw std::runtime_error("\"prompt\" must not be empty"); throw std::runtime_error("\"prompt\" must not be empty");
@@ -1787,7 +1792,8 @@ server_tokens format_prompt_rerank(
const struct llama_vocab * vocab, const struct llama_vocab * vocab,
mtmd_context * mctx, mtmd_context * mctx,
const std::string & query, const std::string & query,
const std::string & doc) { const std::string & doc,
const mtmd_helper_init_opt & init_opt) {
server_tokens result = {}; server_tokens result = {};
const char * rerank_prompt = llama_model_chat_template(model, "rerank"); const char * rerank_prompt = llama_model_chat_template(model, "rerank");
@@ -1796,12 +1802,12 @@ server_tokens format_prompt_rerank(
std::string prompt = rerank_prompt; std::string prompt = rerank_prompt;
string_replace_all(prompt, "{query}" , query); string_replace_all(prompt, "{query}" , query);
string_replace_all(prompt, "{document}", doc ); string_replace_all(prompt, "{document}", doc );
server_tokens tokens = tokenize_input_subprompt(vocab, mctx, prompt, false, true); server_tokens tokens = tokenize_input_subprompt(vocab, mctx, prompt, false, true, init_opt);
result.push_back(tokens); result.push_back(tokens);
} else { } else {
// Get EOS token - use SEP token as fallback if EOS is not available // Get EOS token - use SEP token as fallback if EOS is not available
server_tokens query_tokens = tokenize_input_subprompt(vocab, mctx, query, false, false); server_tokens query_tokens = tokenize_input_subprompt(vocab, mctx, query, false, false, init_opt);
server_tokens doc_tokens = tokenize_input_subprompt(vocab, mctx, doc, false, false); server_tokens doc_tokens = tokenize_input_subprompt(vocab, mctx, doc, false, false, init_opt);
llama_token eos_token = llama_vocab_eos(vocab); llama_token eos_token = llama_vocab_eos(vocab);
if (eos_token == LLAMA_TOKEN_NULL) { if (eos_token == LLAMA_TOKEN_NULL) {
eos_token = llama_vocab_sep(vocab); eos_token = llama_vocab_sep(vocab);
tools/server/server-common.h
+11 -3
View File
@@ -5,6 +5,7 @@
#include "llama.h" #include "llama.h"
#include "chat.h" #include "chat.h"
#include "mtmd.h" #include "mtmd.h"
#include "mtmd-helper.h"
#include "json.h" #include "json.h"
@@ -269,7 +270,12 @@ size_t validate_utf8(const std::string& text);
// process mtmd prompt, return the server_tokens containing both text tokens and media chunks // process mtmd prompt, return the server_tokens containing both text tokens and media chunks
// if is_placeholder is true, the media chunk will be treated as placeholder for counting tokens; the output tokens are not usable for actual inference (e.g. for submitting a task to server_queue) // if is_placeholder is true, the media chunk will be treated as placeholder for counting tokens; the output tokens are not usable for actual inference (e.g. for submitting a task to server_queue)
server_tokens process_mtmd_prompt(mtmd_context * mctx, const std::string & prompt, const std::vector<raw_buffer> & files, bool is_placeholder = false); server_tokens process_mtmd_prompt(
mtmd_context * mctx,
const std::string & prompt,
const std::vector<raw_buffer> & files,
const mtmd_helper_init_opt & init_opt,
bool is_placeholder = false);
/** /**
* break the input "prompt" object into multiple prompt if needed, then tokenize them * break the input "prompt" object into multiple prompt if needed, then tokenize them
@@ -289,7 +295,8 @@ std::vector<server_tokens> tokenize_input_prompts(
mtmd_context * mctx, mtmd_context * mctx,
const json & json_prompt, const json & json_prompt,
bool add_special, bool add_special,
bool parse_special); bool parse_special,
const mtmd_helper_init_opt & init_opt);
// //
// OAI utils // OAI utils
@@ -538,7 +545,8 @@ server_tokens format_prompt_rerank(
const struct llama_vocab * vocab, const struct llama_vocab * vocab,
mtmd_context * mctx, mtmd_context * mctx,
const std::string & query, const std::string & query,
const std::string & doc); const std::string & doc,
const mtmd_helper_init_opt & init_opt);
// simple implementation of a pipe // simple implementation of a pipe
// used for streaming data between threads // used for streaming data between threads
tools/server/server-context.cpp
+19 -12
View File
@@ -794,6 +794,8 @@ public:
llama_model * model_tgt = nullptr; llama_model * model_tgt = nullptr;
mtmd_context * mctx = nullptr; mtmd_context * mctx = nullptr;
// note: video_params.ffmpeg_bin_dir points into params_base, which outlives this struct
mtmd_helper_init_opt init_opt = mtmd_helper_init_opt_default();
const llama_vocab * vocab = nullptr; const llama_vocab * vocab = nullptr;
server_queue queue_tasks; server_queue queue_tasks;
@@ -1118,6 +1120,11 @@ private:
} }
SRV_INF("loaded multimodal model, '%s'\n", mmproj_path.c_str()); SRV_INF("loaded multimodal model, '%s'\n", mmproj_path.c_str());
init_opt.video_params.fps_target = params_base.video_fps;
init_opt.video_params.timestamp_interval_ms = params_base.video_timestamp_interval_ms;
init_opt.video_params.ffmpeg_bin_dir = params_base.video_ffmpeg_bin_dir.empty()
? nullptr : params_base.video_ffmpeg_bin_dir.c_str();
if (params_base.ctx_shift) { if (params_base.ctx_shift) {
params_base.ctx_shift = false; params_base.ctx_shift = false;
SRV_WRN("%s\n", "ctx_shift is not supported by multimodal, it will be disabled"); SRV_WRN("%s\n", "ctx_shift is not supported by multimodal, it will be disabled");
@@ -2134,9 +2141,9 @@ private:
try { try {
auto & prompt = task.cli_prompt; auto & prompt = task.cli_prompt;
if (mctx != nullptr) { if (mctx != nullptr) {
task.tokens = process_mtmd_prompt(mctx, prompt, task.cli_files); task.tokens = process_mtmd_prompt(mctx, prompt, task.cli_files, init_opt);
} else { } else {
task.tokens = std::move(tokenize_input_prompts(vocab, mctx, prompt, true, true)[0]); task.tokens = std::move(tokenize_input_prompts(vocab, mctx, prompt, true, true, init_opt)[0]);
} }
task.cli_prompt.clear(); task.cli_prompt.clear();
task.cli_files.clear(); task.cli_files.clear();
@@ -4165,10 +4172,10 @@ std::unique_ptr<server_res_generator> server_routes::handle_completions_impl(
if (res_type != TASK_RESPONSE_TYPE_NONE && ctx_server.mctx != nullptr) { if (res_type != TASK_RESPONSE_TYPE_NONE && ctx_server.mctx != nullptr) {
// This is the case used by OAI compatible chat path with MTMD. TODO It can be moved to the path below. // This is the case used by OAI compatible chat path with MTMD. TODO It can be moved to the path below.
inputs.push_back(process_mtmd_prompt(ctx_server.mctx, prompt.get<std::string>(), files)); inputs.push_back(process_mtmd_prompt(ctx_server.mctx, prompt.get<std::string>(), files, ctx_server.init_opt));
} else { } else {
// Everything else, including multimodal completions. // Everything else, including multimodal completions.
inputs = tokenize_input_prompts(ctx_server.vocab, ctx_server.mctx, prompt, true, true); inputs = tokenize_input_prompts(ctx_server.vocab, ctx_server.mctx, prompt, true, true, ctx_server.init_opt);
} }
// tasks.reserve(inputs.size()); // TODO: this is inaccurate due to child tasks // tasks.reserve(inputs.size()); // TODO: this is inaccurate due to child tasks
@@ -4752,7 +4759,7 @@ void server_routes::init_routes() {
data["input_extra"] = input_extra; // default to empty array if it's not exist data["input_extra"] = input_extra; // default to empty array if it's not exist
std::string prompt = json_value(data, "prompt", std::string()); std::string prompt = json_value(data, "prompt", std::string());
std::vector<server_tokens> tokenized_prompts = tokenize_input_prompts(ctx_server.vocab, ctx_server.mctx, prompt, false, true); std::vector<server_tokens> tokenized_prompts = tokenize_input_prompts(ctx_server.vocab, ctx_server.mctx, prompt, false, true, ctx_server.init_opt);
SRV_DBG("creating infill tasks, n_prompts = %d\n", (int) tokenized_prompts.size()); SRV_DBG("creating infill tasks, n_prompts = %d\n", (int) tokenized_prompts.size());
data["prompt"] = format_prompt_infill( data["prompt"] = format_prompt_infill(
ctx_server.vocab, ctx_server.vocab,
@@ -4816,7 +4823,7 @@ void server_routes::init_routes() {
}; };
this->post_chat_completions_tok = [this](const server_http_req & req) { this->post_chat_completions_tok = [this](const server_http_req & req) {
return handle_count_tokens(ctx_server.vocab, ctx_server.mctx, req, TASK_RESPONSE_TYPE_OAI_CHAT); return handle_count_tokens(ctx_server.vocab, ctx_server.mctx, ctx_server.init_opt, req, TASK_RESPONSE_TYPE_OAI_CHAT);
}; };
this->post_control = [this](const server_http_req & req) { this->post_control = [this](const server_http_req & req) {
@@ -4875,7 +4882,7 @@ void server_routes::init_routes() {
}; };
this->post_responses_tok_oai = [this](const server_http_req & req) { this->post_responses_tok_oai = [this](const server_http_req & req) {
return handle_count_tokens(ctx_server.vocab, ctx_server.mctx, req, TASK_RESPONSE_TYPE_OAI_RESP); return handle_count_tokens(ctx_server.vocab, ctx_server.mctx, ctx_server.init_opt, req, TASK_RESPONSE_TYPE_OAI_RESP);
}; };
this->post_transcriptions_oai = [this](const server_http_req & req) { this->post_transcriptions_oai = [this](const server_http_req & req) {
@@ -4925,7 +4932,7 @@ void server_routes::init_routes() {
}; };
this->post_anthropic_count_tokens = [this](const server_http_req & req) { this->post_anthropic_count_tokens = [this](const server_http_req & req) {
return handle_count_tokens(ctx_server.vocab, ctx_server.mctx, req, TASK_RESPONSE_TYPE_ANTHROPIC); return handle_count_tokens(ctx_server.vocab, ctx_server.mctx, ctx_server.init_opt, req, TASK_RESPONSE_TYPE_ANTHROPIC);
}; };
// same with handle_chat_completions, but without inference part // same with handle_chat_completions, but without inference part
@@ -5058,7 +5065,7 @@ void server_routes::init_routes() {
std::vector<server_task> tasks; std::vector<server_task> tasks;
tasks.reserve(documents.size()); tasks.reserve(documents.size());
for (size_t i = 0; i < documents.size(); i++) { for (size_t i = 0; i < documents.size(); i++) {
auto tmp = format_prompt_rerank(ctx_server.model_tgt, ctx_server.vocab, ctx_server.mctx, query, documents[i]); auto tmp = format_prompt_rerank(ctx_server.model_tgt, ctx_server.vocab, ctx_server.mctx, query, documents[i], ctx_server.init_opt);
server_task task = server_task(SERVER_TASK_TYPE_RERANK); server_task task = server_task(SERVER_TASK_TYPE_RERANK);
task.id = rd.get_new_id(); task.id = rd.get_new_id();
task.tokens = std::move(tmp); task.tokens = std::move(tmp);
@@ -5296,7 +5303,7 @@ std::unique_ptr<server_res_generator> server_routes::handle_embeddings_impl(cons
} }
} }
auto tokenized_prompts = tokenize_input_prompts(ctx_server.vocab, ctx_server.mctx, prompt, true, true); auto tokenized_prompts = tokenize_input_prompts(ctx_server.vocab, ctx_server.mctx, prompt, true, true, ctx_server.init_opt);
for (const auto & tokens : tokenized_prompts) { for (const auto & tokens : tokenized_prompts) {
// this check is necessary for models that do not add BOS token to the input // this check is necessary for models that do not add BOS token to the input
if (tokens.empty()) { if (tokens.empty()) {
@@ -5357,7 +5364,7 @@ std::unique_ptr<server_res_generator> server_routes::handle_embeddings_impl(cons
return res; return res;
} }
std::unique_ptr<server_res_generator> server_routes::handle_count_tokens(const llama_vocab * vocab, mtmd_context * mctx, const server_http_req & req, task_response_type res_type) { std::unique_ptr<server_res_generator> server_routes::handle_count_tokens(const llama_vocab * vocab, mtmd_context * mctx, const mtmd_helper_init_opt & init_opt, const server_http_req & req, task_response_type res_type) {
auto res = create_response(); auto res = create_response();
std::vector<raw_buffer> files; std::vector<raw_buffer> files;
json body = json::parse(req.body); json body = json::parse(req.body);
@@ -5395,7 +5402,7 @@ std::unique_ptr<server_res_generator> server_routes::handle_count_tokens(const l
if (!prompt.is_string()) { if (!prompt.is_string()) {
throw std::runtime_error("for mtmd, input prompt must be a string."); throw std::runtime_error("for mtmd, input prompt must be a string.");
} }
n_tokens = process_mtmd_prompt(mctx, prompt.get<std::string>(), files, true).size(); n_tokens = process_mtmd_prompt(mctx, prompt.get<std::string>(), files, init_opt, true).size();
} else { } else {
n_tokens = tokenize_mixed(vocab, prompt, true, true).size(); n_tokens = tokenize_mixed(vocab, prompt, true, true).size();
} }
tools/server/server-context.h
+1 -1
View File
@@ -169,7 +169,7 @@ private:
std::unique_ptr<server_res_generator> handle_slots_restore(const server_http_req & req, int id_slot); std::unique_ptr<server_res_generator> handle_slots_restore(const server_http_req & req, int id_slot);
std::unique_ptr<server_res_generator> handle_slots_erase(const server_http_req &, int id_slot); std::unique_ptr<server_res_generator> handle_slots_erase(const server_http_req &, int id_slot);
std::unique_ptr<server_res_generator> handle_embeddings_impl(const server_http_req & req, task_response_type res_type); std::unique_ptr<server_res_generator> handle_embeddings_impl(const server_http_req & req, task_response_type res_type);
std::unique_ptr<server_res_generator> handle_count_tokens(const llama_vocab * vocab, mtmd_context * mctx, const server_http_req & req, task_response_type res_type); std::unique_ptr<server_res_generator> handle_count_tokens(const llama_vocab * vocab, mtmd_context * mctx, const mtmd_helper_init_opt & init_opt, const server_http_req & req, task_response_type res_type);
// using unique_ptr to allow late initialization of const // using unique_ptr to allow late initialization of const
std::unique_ptr<const server_context_meta> meta; std::unique_ptr<const server_context_meta> meta;
tools/tts/tts.cpp
+1 -1
View File
@@ -103,7 +103,7 @@ int main(int argc, char ** argv) {
mtmd::bitmap_ptr speaker_bitmap; mtmd::bitmap_ptr speaker_bitmap;
if (!params.tts_speaker_file.empty()) { if (!params.tts_speaker_file.empty()) {
auto wrapper = mtmd_helper_bitmap_init_from_file(mctx.get(), params.tts_speaker_file.c_str(), false); auto wrapper = mtmd_helper_bitmap_init_from_file(mctx.get(), params.tts_speaker_file.c_str(), false, mtmd_helper_init_opt_default());
if (!wrapper.bitmap) { if (!wrapper.bitmap) {
LOG_ERR("failed to load speaker file %s\n", params.tts_speaker_file.c_str()); LOG_ERR("failed to load speaker file %s\n", params.tts_speaker_file.c_str());
return 1; return 1;