Lumpiasty/llama.cpp
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

args: add --video-* CLI arguments (#24318)

Browse Source 
* args: add --video-* CLI arguments

* gen docs

* nits

* add mtmd_helper_init_opt
This commit is contained in:
Xuan-Son Nguyen
2026-08-27 12:11:12 +02:00
committed by GitHub
parent 915dc6d38c
commit f29551215b
12 changed files with 138 additions and 48 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tools/mtmd/mtmd-cli.cpp
+10 -1
View File
@@ -87,6 +87,9 @@ struct mtmd_cli_context {
mtmd::bitmaps bitmaps;
std::vector<mtmd_helper::video_ptr> videos;
mtmd_helper_init_opt init_opt = mtmd_helper_init_opt_default();
std::string video_ffmpeg_bin_dir;
mtmd::batch_ptr mbatch;
// chat template
@@ -170,6 +173,12 @@ struct mtmd_cli_context {
LOG_ERR("Failed to load vision model from %s\n", clip_path);
exit(1);
}
video_ffmpeg_bin_dir = params.video_ffmpeg_bin_dir;
init_opt.video_params.fps_target = params.video_fps;
init_opt.video_params.timestamp_interval_ms = params.video_timestamp_interval_ms;
init_opt.video_params.ffmpeg_bin_dir = video_ffmpeg_bin_dir.empty()
? nullptr : video_ffmpeg_bin_dir.c_str();
}
bool check_antiprompt(const llama_tokens & generated_tokens) {
@@ -184,7 +193,7 @@ struct mtmd_cli_context {
}
bool load_media(const std::string & fname) {
auto res = mtmd_helper_bitmap_init_from_file(ctx_vision.get(), fname.c_str(), false);
auto res = mtmd_helper_bitmap_init_from_file(ctx_vision.get(), fname.c_str(), false, init_opt);
if (!res.bitmap) {
return false;
}
tools/mtmd/mtmd-helper.cpp
+19 -9
View File
@@ -369,14 +369,18 @@ static bool is_webp_file(const unsigned char * buf, size_t len) {
}
#ifdef MTMD_VIDEO
static mtmd_bitmap * decode_webp_with_ffmpeg(mtmd_context * mctx, const unsigned char * buf, size_t len, bool placeholder);
static mtmd_bitmap * decode_webp_with_ffmpeg(mtmd_context * mctx, const unsigned char * buf, size_t len, bool placeholder,
const mtmd_helper_video_init_params & params);
#endif
mtmd_helper_bitmap_wrapper mtmd_helper_bitmap_init_from_buf(mtmd_context * ctx, const unsigned char * buf, size_t len, bool placeholder) {
mtmd_helper_bitmap_wrapper mtmd_helper_bitmap_init_from_buf(mtmd_context * ctx, const unsigned char * buf, size_t len, bool placeholder,
mtmd_helper_init_opt opt) {
// calculate the hash if needed
std::string id;
mtmd_bitmap * result = nullptr;
GGML_UNUSED(opt); // only used by video code paths
if (!placeholder) {
// use sha256 to prevent cache poisoning
id = hash_sha256_hex(buf, len);
@@ -414,7 +418,7 @@ mtmd_helper_bitmap_wrapper mtmd_helper_bitmap_init_from_buf(mtmd_context * ctx,
#ifdef MTMD_VIDEO
// stb_image does not support webp; decode it with ffmpeg as a single frame
if (!result && is_webp_file(buf, len)) {
result = decode_webp_with_ffmpeg(ctx, buf, len, placeholder);
result = decode_webp_with_ffmpeg(ctx, buf, len, placeholder, opt.video_params);
if (!result) {
LOG_ERR("%s: failed to decode webp buffer\n", __func__);
return {nullptr, nullptr};
@@ -427,8 +431,7 @@ mtmd_helper_bitmap_wrapper mtmd_helper_bitmap_init_from_buf(mtmd_context * ctx,
// last try: load as video
#ifdef MTMD_VIDEO
if (!result) {
auto params = mtmd_helper_video_init_params_default();
auto video_ctx = mtmd_helper_video_init_from_buf(ctx, buf, len, params);
auto video_ctx = mtmd_helper_video_init_from_buf(ctx, buf, len, opt.video_params);
if (!video_ctx) {
LOG_ERR("%s: failed to decode buffer as either image/audio/video\n", __func__);
return {nullptr, nullptr};
@@ -456,7 +459,8 @@ mtmd_helper_bitmap_wrapper mtmd_helper_bitmap_init_from_buf(mtmd_context * ctx,
return {nullptr, nullptr};
}
mtmd_helper_bitmap_wrapper mtmd_helper_bitmap_init_from_file(mtmd_context * ctx, const char * fname, bool placeholder) {
mtmd_helper_bitmap_wrapper mtmd_helper_bitmap_init_from_file(mtmd_context * ctx, const char * fname, bool placeholder,
mtmd_helper_init_opt opt) {
#ifdef _WIN32
int wlen = MultiByteToWideChar(CP_UTF8, 0, fname, -1, NULL, 0);
if (!wlen) {
@@ -497,7 +501,7 @@ mtmd_helper_bitmap_wrapper mtmd_helper_bitmap_init_from_file(mtmd_context * ctx,
return {nullptr, nullptr};
}
return mtmd_helper_bitmap_init_from_buf(ctx, buf.data(), buf.size(), placeholder);
return mtmd_helper_bitmap_init_from_buf(ctx, buf.data(), buf.size(), placeholder, opt);
}
bool mtmd_helper_support_video(mtmd_context * ctx) {
@@ -855,6 +859,12 @@ mtmd_helper_video_init_params mtmd_helper_video_init_params_default() {
};
}
mtmd_helper_init_opt mtmd_helper_init_opt_default() {
return {
/* video_params */ mtmd_helper_video_init_params_default(),
};
}
static std::string video_resolve_bin(const char * bin_dir, const char * name) {
if (!bin_dir || bin_dir[0] == '\0') {
return name; // rely on PATH
@@ -876,8 +886,8 @@ static std::string video_resolve_bin(const char * bin_dir, const char * name) {
}
#ifdef MTMD_VIDEO
static mtmd_bitmap * decode_webp_with_ffmpeg(mtmd_context * mctx, const unsigned char * buf, size_t len, bool placeholder) {
auto params = mtmd_helper_video_init_params_default();
static mtmd_bitmap * decode_webp_with_ffmpeg(mtmd_context * mctx, const unsigned char * buf, size_t len, bool placeholder,
const mtmd_helper_video_init_params & params) {
mtmd_helper_video vctx;
vctx.mctx = mctx;
vctx.input_buf.assign(buf, buf + len);
tools/mtmd/mtmd-helper.h
+28 -10
View File
@@ -23,6 +23,23 @@ extern "C" {
struct mtmd_helper_video;
typedef struct mtmd_helper_video mtmd_helper_video;
struct mtmd_helper_video_init_params {
float fps_target; // desired output fps; <= 0 means use the video's native fps, defaulted to 4.0f
const char * ffmpeg_bin_dir; // directory containing ffmpeg/ffprobe binaries; NULL means search PATH
int64_t timestamp_interval_ms; // interval for adding timestamp as text chunk (example: "[10m50.5s]"); <= 0 means no timestamp, defaulted to 5000ms
// TODO @ngxson : allow "placeholder" bitmap output for counting tokens
};
MTMD_API struct mtmd_helper_video_init_params mtmd_helper_video_init_params_default(void);
// opt for mtmd_helper_bitmap_init_from_*()
struct mtmd_helper_init_opt {
struct mtmd_helper_video_init_params video_params;
};
typedef struct mtmd_helper_init_opt mtmd_helper_init_opt;
MTMD_API struct mtmd_helper_init_opt mtmd_helper_init_opt_default(void);
// Set callback for all future logging events.
// If this is not called, or NULL is supplied, everything is output on stderr.
// Note: this also call mtmd_log_set() internally
@@ -40,7 +57,11 @@ struct mtmd_helper_bitmap_wrapper {
// it calls mtmd_helper_bitmap_init_from_buf() internally
// returns nullptr on failure
// this function is thread-safe
MTMD_API struct mtmd_helper_bitmap_wrapper mtmd_helper_bitmap_init_from_file(mtmd_context * ctx, const char * fname, bool placeholder);
MTMD_API struct mtmd_helper_bitmap_wrapper mtmd_helper_bitmap_init_from_file(
mtmd_context * ctx,
const char * fname,
bool placeholder,
struct mtmd_helper_init_opt opt);
// helper function to construct a mtmd_bitmap from a buffer containing a file
// supported formats:
@@ -53,7 +74,11 @@ MTMD_API struct mtmd_helper_bitmap_wrapper mtmd_helper_bitmap_init_from_file(mtm
// - output bitmap will have SHA-256 hash (hex string) as the ID
// returns nullptr on failure
// this function is thread-safe
MTMD_API struct mtmd_helper_bitmap_wrapper mtmd_helper_bitmap_init_from_buf(mtmd_context * ctx, const unsigned char * buf, size_t len, bool placeholder);
MTMD_API struct mtmd_helper_bitmap_wrapper mtmd_helper_bitmap_init_from_buf(
mtmd_context * ctx,
const unsigned char * buf, size_t len,
bool placeholder,
struct mtmd_helper_init_opt opt);
// helper to count the total number of tokens from a list of chunks, useful to keep track of KV cache
MTMD_API size_t mtmd_helper_get_n_tokens(const mtmd_input_chunks * chunks);
@@ -124,14 +149,7 @@ struct mtmd_helper_video_info {
int32_t n_frames; // estimated total frames at effective fps (-1 if unknown)
};
struct mtmd_helper_video_init_params {
float fps_target; // desired output fps; <= 0 means use the video's native fps, defaulted to 4.0f
const char * ffmpeg_bin_dir; // directory containing ffmpeg/ffprobe binaries; NULL means search PATH
int64_t timestamp_interval_ms; // interval for adding timestamp as text chunk (example: "[10m50.5s]"); <= 0 means no timestamp, defaulted to 5000ms
// TODO @ngxson : allow "placeholder" bitmap output for counting tokens
};
MTMD_API struct mtmd_helper_video_init_params mtmd_helper_video_init_params_default(void);
// note: mtmd_helper_video_init_params is defined at the top, as it is part of mtmd_helper_init_opt
// returns NULL on failure (ffprobe not found, file unreadable, etc.)
MTMD_API mtmd_helper_video * mtmd_helper_video_init(