From e3483e40b02326383a54a23ae71f2bfa9f81181a Mon Sep 17 00:00:00 2001 From: Jhen-Jie Hong Date: Sat, 14 Nov 2026 21:12:07 +0800 Subject: [PATCH] mtmd : allow custom video decoder (#32188) * mtmd : allow custom video decoder * also adapt to wllama requirements * clean up * add test --------- Co-authored-by: Xuan Son Nguyen --- tests/test-mtmd-impl.cpp | 54 +- tools/mtmd/CMakeLists.txt | 4 + tools/mtmd/mtmd-helper-common.h | 11 +- tools/mtmd/mtmd-helper-video-ffmpeg.cpp | 410 ++++++++++++++ tools/mtmd/mtmd-helper.cpp | 675 ++++++------------------ tools/mtmd/mtmd-helper.h | 57 +- tools/server/server-context.cpp | 6 + tools/server/server-context.h | 5 + 8 files changed, 691 insertions(+), 531 deletions(-) create mode 100644 tools/mtmd/mtmd-helper-video-ffmpeg.cpp diff --git a/tests/test-mtmd-impl.cpp b/tests/test-mtmd-impl.cpp index 45c15f8c3..8730fb91e 100644 --- a/tests/test-mtmd-impl.cpp +++ b/tests/test-mtmd-impl.cpp @@ -367,10 +367,20 @@ MAKE_TEST(test_video_config) { // n_frames source frames, each one starts with its index struct test_video_source : mtmd_helper_video_seq { int32_t n_frames; - int32_t n_read = 0; + int32_t n_read = 0; + int32_t n_skipped = 0; test_video_source(int32_t n_frames) : n_frames(n_frames) {} + bool skip_raw_frame() override { + if (n_read >= n_frames) { + return false; + } + n_read++; + n_skipped++; + return true; + } + bool read_raw_frame() override { if (n_read >= n_frames) { return false; @@ -412,17 +422,17 @@ MAKE_TEST(test_video_sequence_qwen3vl) { const mtmd_image_preprocessor_qwen3vl preproc(hparams); // { source video, expected frame size and sequence from Qwen3VLProcessor (transformers 4.57.1) } - const std::vector> cases = { - { { 720, 358, 24.0f, 240 }, { 704, 352 }, + const std::vector> cases = { + { { 720, 358, 24.0f, 240, -1.0f }, { 704, 352 }, "<0.3 seconds>[0][13]<1.3 seconds>[25][38]<2.4 seconds>[50][63]<3.4 seconds>[75][88]<4.5 seconds>[101][113]" "<5.5 seconds>[126][138]<6.6 seconds>[151][164]<7.6 seconds>[176][189]<8.6 seconds>[201][214]<9.7 seconds>[226][239]" }, // odd frame count: the last group repeats its last frame - { { 640, 480, 10.0f, 35 }, { 640, 480 }, + { { 640, 480, 10.0f, 35, -1.0f }, { 640, 480 }, "<0.3 seconds>[0][6]<1.4 seconds>[11][17]<2.5 seconds>[23][28]<3.4 seconds>[34]" }, // fewer frames than the minimum of 4 - { { 320, 240, 30.0f, 3 }, { 320, 256 }, + { { 320, 240, 30.0f, 3, -1.0f }, { 320, 256 }, "<0.0 seconds>[0][1]<0.1 seconds>[2]" }, - { { 1280, 720, 30000.0f / 1001.0f, 300 }, { 1280, 704 }, + { { 1280, 720, 30000.0f / 1001.0f, 300, -1.0f }, { 1280, 704 }, "<0.3 seconds>[0][16]<1.3 seconds>[31][47]<2.4 seconds>[63][79]<3.4 seconds>[94][110]<4.5 seconds>[126][142]" "<5.5 seconds>[157][173]<6.6 seconds>[189][205]<7.6 seconds>[220][236]<8.7 seconds>[252][268]<9.7 seconds>[283][299]" }, }; @@ -445,19 +455,43 @@ MAKE_TEST(test_video_sequence_lfm3_omni) { hparams.n_merge = 2; const mtmd_image_preprocessor_lfm3vl preproc(hparams); - const mtmd_helper_video_src src = { 64, 64, 2.0f, 9 }; + const mtmd_helper_video_stream_info src = { 64, 64, 2.0f, 9, -1.0f }; const mtmd_video_config cfg = mtmd_video_config_init(MTMD_VIDEO_TEMPLATE_LFM3_OMNI, &preproc, src.width, src.height, src.n_frames, src.fps); test_video_source video(src.n_frames); video.configure(cfg, src, 0.0f); - video.src_alive = true; - video.audio_rate = 100; - video.soundtrack.assign(450, 0.0f); + video.src_alive = true; + // the video is 4.5 seconds long, the rest of the soundtrack is dropped + const std::vector pcm(500, 0.0f); + video.set_soundtrack(pcm.data(), pcm.size(), 100); t.assert_equal("sequence", std::string("<0.2 seconds>[0][1]{100}<1.2 seconds>[2][3]{100}<2.2 seconds>[4][5]{100}<3.2 seconds>[6][7]{100}<4.0 seconds>[8]{50}"), video_sequence_to_str(t, video)); } +// a source that gives every frame: the frames of fps_target are picked by index +MAKE_TEST(test_video_sequence_sample_by_index) { + const mtmd_video_config cfg = { MTMD_VIDEO_TEMPLATE_DEFAULT, 0.0f, 0, 0, 0 }; + + // { source video, frames in the source, expected sequence, expected number of skipped frames } + const std::vector> cases = { + { { 64, 64, 8.0f, 16, -1.0f }, 16, "Video:[0][0m0.00s][4][8][0m1.00s][12]", 9 }, + // unknown length: every frame is kept + { { 64, 64, 8.0f, -1, -1.0f }, 3, "Video:[0][0m0.00s][1][2]", 0 }, + }; + + for (const auto & [src, n_frames, expected, expected_skipped] : cases) { + const std::string name = std::to_string(src.n_frames) + " frames"; + test_video_source video(n_frames); + video.timestamp_interval_ms = 1000; + video.configure(cfg, src, 2.0f); + video.sample_by_index(src); + video.src_alive = true; + t.assert_equal(name + " sequence", expected, video_sequence_to_str(t, video)); + t.assert_equal(name + " skipped", expected_skipped, video.n_skipped); + } +} + // // mtmd_audio // diff --git a/tools/mtmd/CMakeLists.txt b/tools/mtmd/CMakeLists.txt index c19815d67..44ad17733 100644 --- a/tools/mtmd/CMakeLists.txt +++ b/tools/mtmd/CMakeLists.txt @@ -31,6 +31,10 @@ set(MTMD_CORE_SOURCES mtmd-graph.h ) +if (MTMD_VIDEO) + list(APPEND MTMD_CORE_SOURCES mtmd-helper-video-ffmpeg.cpp) +endif() + add_library(mtmd ${MTMD_CORE_SOURCES} models/models.h diff --git a/tools/mtmd/mtmd-helper-common.h b/tools/mtmd/mtmd-helper-common.h index e347868ef..91b1df835 100644 --- a/tools/mtmd/mtmd-helper-common.h +++ b/tools/mtmd/mtmd-helper-common.h @@ -1,7 +1,7 @@ #pragma once // shared internal utilities for the mtmd-helper-*.cpp translation units -// (mtmd-helper.cpp, mtmd-helper-gen.cpp) +// (mtmd-helper.cpp, mtmd-helper-gen.cpp, mtmd-helper-video-ffmpeg.cpp) // NOT part of the public mtmd-helper.h API #include "ggml.h" @@ -65,6 +65,15 @@ inline mtmd_helper_logger g_logger; #define LOG_WRN(...) g_logger.log(GGML_LOG_LEVEL_WARN, __VA_ARGS__) #define LOG_ERR(...) g_logger.log(GGML_LOG_LEVEL_ERROR, __VA_ARGS__) +// +// video +// + +struct mtmd_helper_video_decoder; + +// only with MTMD_VIDEO; the user_data of open() is ffmpeg_bin_dir +const mtmd_helper_video_decoder * mtmd_helper_video_decoder_ffmpeg(); + // // embd batch // diff --git a/tools/mtmd/mtmd-helper-video-ffmpeg.cpp b/tools/mtmd/mtmd-helper-video-ffmpeg.cpp new file mode 100644 index 000000000..0e9fd2603 --- /dev/null +++ b/tools/mtmd/mtmd-helper-video-ffmpeg.cpp @@ -0,0 +1,410 @@ +// mtmd_helper_video_decoder backed by ffmpeg/ffprobe subprocesses + +#ifdef MTMD_VIDEO + +// fix problem with std::min and std::max +#if defined(_WIN32) +#define WIN32_LEAN_AND_MEAN +#ifndef NOMINMAX +# define NOMINMAX +#endif +#include +#endif + +#include "mtmd-helper.h" +#include "mtmd-helper-common.h" + +#include "sheredom/subprocess.h" + +#include +#include +#include +#include +#include +#include +#include + +#ifndef _WIN32 +#include +#include +#include +#endif + +#ifdef MTMD_INTERNAL_HEADER +#error "mtmd-helper is a public library outside of mtmd. it must not include internal headers" +#endif + +namespace { + +// RAII wrapper for managing subprocess +struct subprocess_handle { + struct subprocess_s proc = {}; + bool created = false; // process exists and must be cleaned up + bool alive = false; // process can still give us data + std::thread feeder; + + subprocess_handle() = default; + subprocess_handle(const subprocess_handle &) = delete; + subprocess_handle & operator=(const subprocess_handle &) = delete; + ~subprocess_handle() { stop(); } + + void stop() { + // note: alive becomes false on stdout EOF, but the process still needs cleanup + if (!created) { + return; + } + subprocess_terminate(&proc); +#ifdef _WIN32 + // no SIGPIPE on windows: a blocked feeder only gets a broken pipe once we close our read end of the child stdin + if (proc.hStdInput) { + CloseHandle(proc.hStdInput); + proc.hStdInput = nullptr; + } +#endif + // join before destroy: feeder holds a FILE* from subprocess_stdin; + // subprocess_destroy closes it, so the thread must finish first + if (feeder.joinable()) { + feeder.join(); + } + subprocess_join(&proc, nullptr); // reap the child, or else it stays a zombie + subprocess_destroy(&proc); + created = false; + alive = false; + } + + FILE * stdout_pipe() { + return subprocess_stdout(&proc); + } + + // buf must outlive the feeder thread, i.e. this handle + void start_feeder(const std::vector & buf) { + feeder = std::thread([this, &buf]() { +#ifndef _WIN32 + // ffmpeg can exit before it reads all the input, for example when ffprobe already got the metadata. + // the write below must then fail with EPIPE, instead of killing the process with SIGPIPE + sigset_t sigpipe_set; + sigemptyset(&sigpipe_set); + sigaddset(&sigpipe_set, SIGPIPE); + pthread_sigmask(SIG_BLOCK, &sigpipe_set, nullptr); // linux sends the signal to the writing thread +#endif + FILE * f = subprocess_stdin(&proc); + if (!f) { + return; + } +#ifdef F_SETNOSIGPIPE + fcntl(fileno(f), F_SETNOSIGPIPE, 1); // macos/bsd send it to the process, so turn it off per fd +#endif + fwrite(buf.data(), 1, buf.size(), f); + fclose(f); + proc.stdin_file = nullptr; // prevent double-close in subprocess_destroy + }); + } +}; + +std::string resolve_bin(const char * bin_dir, const char * name) { + if (!bin_dir || bin_dir[0] == '\0') { + return name; // rely on PATH + } + std::string result = bin_dir; + char last = result.back(); + if (last != '/' && last != '\\') { +#ifdef _WIN32 + result += '\\'; +#else + result += '/'; +#endif + } + result += name; +#ifdef _WIN32 + result += ".exe"; +#endif + return result; +} + +float parse_rational(const char * s) { + int num = 0, den = 1; + if (sscanf(s, "%d/%d", &num, &den) == 2 && den > 0) { + return (float)num / (float)den; + } + float val; + if (sscanf(s, "%f", &val) == 1) { + return val; + } + return 0.0f; +} + +struct ffmpeg_video { + std::string path; + std::vector input_buf; // non-empty when opened from a buffer + std::string ffmpeg_bin; + std::string ffprobe_bin; + + uint32_t src_width = 0; + uint32_t src_height = 0; + double video_start_s = 0.0; + + std::vector pcm; + size_t pcm_start = 0; + std::vector skip_buf; + + // must be destroyed before input_buf, which its feeder thread reads + subprocess_handle sp; + + bool is_buf_input() const { + return !input_buf.empty(); + } + + bool probe(mtmd_helper_video_stream_info & info) { + // cache: makes stdin seekable; without it ffprobe reports width/height 0 + // for mp4 files whose moov atom sits at the end + const char * input_arg = is_buf_input() ? "cache:pipe:0" : path.c_str(); + std::vector cmd = { + ffprobe_bin.c_str(), + "-v", "quiet", + "-show_entries", "stream=width,height,r_frame_rate,nb_frames,duration,start_time", + "-select_streams", "v:0", + "-of", "default=noprint_wrappers=1", + }; + if (is_buf_input()) { + // remove the 64KB read-ahead limit of cache:, or else ffprobe cannot reach a moov atom at end of file + cmd.insert(cmd.end(), {"-read_ahead_limit", "-1"}); + } + cmd.insert(cmd.end(), {input_arg, nullptr}); + + LOG_DBG("%s: launching:", __func__); + for (size_t i = 0; cmd[i]; i++) { LOG_DBG(" %s", cmd[i]); } + LOG_DBG("\n"); + + subprocess_handle probe_sp; + if (subprocess_create(cmd.data(), + subprocess_option_search_user_path | subprocess_option_inherit_environment, + &probe_sp.proc) != 0) { + LOG_ERR("%s: failed to launch ffprobe\n", __func__); + return false; + } + probe_sp.created = true; + probe_sp.alive = true; + + if (is_buf_input()) { + probe_sp.start_feeder(input_buf); + } + + char line[256]; + FILE * fp = probe_sp.stdout_pipe(); + + while (fgets(line, sizeof(line), fp)) { + char * eq = strchr(line, '='); + if (!eq) continue; + *eq = '\0'; + const char * key = line; + const char * val = eq + 1; + char * nl = (char *)strchr(val, '\n'); + if (nl) *nl = '\0'; + + if (strcmp(key, "width") == 0) { + info.width = (uint32_t)atoi(val); + } else if (strcmp(key, "height") == 0) { + info.height = (uint32_t)atoi(val); + } else if (strcmp(key, "r_frame_rate") == 0) { + info.fps = parse_rational(val); + } else if (strcmp(key, "nb_frames") == 0 && strcmp(val, "N/A") != 0) { + info.n_frames = atoi(val); + } else if (strcmp(key, "duration") == 0 && strcmp(val, "N/A") != 0) { + info.duration = (float)atof(val); + } else if (strcmp(key, "start_time") == 0 && strcmp(val, "N/A") != 0) { + video_start_s = atof(val); + } + } + + probe_sp.stop(); + + // TODO: this is the encoded size, it is wrong for a rotated video (ffmpeg rotates the frames) + src_width = info.width; + src_height = info.height; + return info.width > 0 && info.height > 0 && info.fps > 0.0f; + } + + void load_soundtrack(int rate) { + const std::string rate_str = std::to_string(rate); + std::vector cmd = {ffmpeg_bin.c_str(), "-nostdin"}; + cmd.reserve(20); // GCC 11 -Wstringop-overflow false positive on the inserts below otherwise + if (is_buf_input()) { + cmd.insert(cmd.end(), {"-read_ahead_limit", "-1"}); + } + cmd.insert(cmd.end(), {"-i", is_buf_input() ? "cache:pipe:0" : path.c_str(), "-map", "0:a:0", "-vn", "-ac", "1", + "-ar", rate_str.c_str(), "-f", "f32le", "pipe:1", "-loglevel", "error", nullptr}); + subprocess_handle audio_sp; + if (subprocess_create(cmd.data(), subprocess_option_search_user_path | subprocess_option_inherit_environment, + &audio_sp.proc) != 0) { + LOG_WRN("%s: failed to launch ffmpeg for the soundtrack\n", __func__); + return; + } + audio_sp.created = true; + audio_sp.alive = true; + if (is_buf_input()) { + audio_sp.start_feeder(input_buf); + } + float block[4096]; + size_t n; + while ((n = fread(block, sizeof(float), 4096, audio_sp.stdout_pipe())) > 0) { + pcm.insert(pcm.end(), block, block + n); + } + audio_sp.stop(); + + // align to the first video frame + pcm_start = std::min(pcm.size(), (size_t) std::max(0.0, std::nearbyint(video_start_s * rate))); + } + + bool start(uint32_t width, uint32_t height, float fps) { + std::vector cmd; + cmd.push_back(ffmpeg_bin.c_str()); + + cmd.push_back("-nostdin"); + if (is_buf_input()) { + // remove the 64KB read-ahead limit of cache:, or else ffmpeg cannot reach a moov atom at end of file + cmd.push_back("-read_ahead_limit"); + cmd.push_back("-1"); + } + cmd.push_back("-i"); + // cache:pipe:0 wraps stdin with a seekable in-memory cache, letting ffmpeg seek + // backwards for container headers (e.g. MP4 moov atom at end of file) + cmd.push_back(is_buf_input() ? "cache:pipe:0" : path.c_str()); + + const bool scale = width != src_width || height != src_height; + char vf[160] = ""; + if (fps > 0.0f) { + if (scale) { + snprintf(vf, sizeof(vf), "fps=%.6f,scale=%u:%u", fps, width, height); + } else { + snprintf(vf, sizeof(vf), "fps=%.6f", fps); + } + } else if (scale) { + // B=0 C=0.5 bicubic on RGB is closest to Pillow + snprintf(vf, sizeof(vf), + "format=rgb24,scale=%u:%u:flags=bicubic+accurate_rnd+full_chroma_int:param0=0:param1=0.5", + width, height); + } + if (vf[0]) { + cmd.push_back("-vf"); + cmd.push_back(vf); + } + + cmd.push_back("-f"); + cmd.push_back("rawvideo"); + cmd.push_back("-pix_fmt"); + cmd.push_back("rgb24"); + cmd.push_back("pipe:1"); + cmd.push_back("-loglevel"); + cmd.push_back("error"); + cmd.push_back(nullptr); + + LOG_DBG("%s: launching:", __func__); + for (size_t i = 0; cmd[i]; i++) { + LOG_DBG(" %s", cmd[i]); + } + LOG_DBG("\n"); + + int ret = subprocess_create( + cmd.data(), + subprocess_option_search_user_path | subprocess_option_inherit_environment, + &sp.proc); + + sp.created = (ret == 0); + sp.alive = (ret == 0); + LOG_DBG("%s: subprocess_create ret=%d proc_alive=%d\n", __func__, ret, (int)sp.alive); + if (!sp.alive) { + LOG_ERR("%s: failed to start ffmpeg (is ffmpeg in PATH?)\n", __func__); + return false; + } + + if (is_buf_input()) { + LOG_DBG("%s: starting feeder thread for %zu-byte buffer\n", __func__, input_buf.size()); + sp.start_feeder(input_buf); + } + return true; + } + + bool read(unsigned char * dst, size_t n) { + FILE * fp = sp.stdout_pipe(); + if (!dst) { + skip_buf.resize(n); + dst = skip_buf.data(); + } + size_t total_read = 0; + while (total_read < n) { + const size_t got = fread(dst + total_read, 1, n - total_read, fp); + if (got == 0) { + // clean EOF only if no bytes read yet; partial frame is an error + LOG_DBG("%s: fread returned 0 after %zu/%zu bytes (ferror=%d)\n", + __func__, total_read, n, ferror(fp)); + return false; + } + total_read += got; + } + return true; + } +}; + +void * ffmpeg_open(void * user_data, const char * path, const unsigned char * buf, size_t len, + mtmd_helper_video_stream_info * info) { + auto v = std::make_unique(); + const char * bin_dir = static_cast(user_data); + v->ffmpeg_bin = resolve_bin(bin_dir, "ffmpeg"); + v->ffprobe_bin = resolve_bin(bin_dir, "ffprobe"); + if (buf && len > 0) { + v->input_buf.assign(buf, buf + len); + } else if (path) { + v->path = path; + } + if (!v->probe(*info)) { + LOG_ERR("%s: ffprobe failed for '%s' (is ffprobe in PATH?)\n", __func__, v->is_buf_input() ? "" : v->path.c_str()); + return nullptr; + } + return v.release(); +} + +bool ffmpeg_start(void * state, uint32_t width, uint32_t height, float fps, bool * fps_picked) { + *fps_picked = fps > 0.0f; + return static_cast(state)->start(width, height, fps); +} + +bool ffmpeg_read_frame(void * state, unsigned char * rgb, uint32_t width, uint32_t height) { + auto * v = static_cast(state); + if (!v->sp.alive) { + return false; + } + if (!v->pcm.empty()) { + std::vector().swap(v->pcm); // no longer needed + } + LOG_DBG("%s: reading a frame, expecting %zu bytes (%ux%u)\n", __func__, (size_t) width * height * 3, width, height); + return v->read(rgb, (size_t) width * height * 3); +} + +const float * ffmpeg_read_audio(void * state, int sample_rate, size_t * n_samples) { + auto * v = static_cast(state); + v->pcm.clear(); + v->pcm_start = 0; + v->load_soundtrack(sample_rate); + *n_samples = v->pcm.size() - v->pcm_start; + return *n_samples > 0 ? v->pcm.data() + v->pcm_start : nullptr; +} + +void ffmpeg_close(void * state) { + delete static_cast(state); +} + +} // namespace + +const mtmd_helper_video_decoder * mtmd_helper_video_decoder_ffmpeg() { + static const mtmd_helper_video_decoder decoder = { + /* open */ ffmpeg_open, + /* start */ ffmpeg_start, + /* read_frame */ ffmpeg_read_frame, + /* read_audio */ ffmpeg_read_audio, + /* close */ ffmpeg_close, + /* user_data */ nullptr, + }; + return &decoder; +} + +#endif // MTMD_VIDEO diff --git a/tools/mtmd/mtmd-helper.cpp b/tools/mtmd/mtmd-helper.cpp index aefe504b2..e14476788 100644 --- a/tools/mtmd/mtmd-helper.cpp +++ b/tools/mtmd/mtmd-helper.cpp @@ -41,16 +41,6 @@ #error "mtmd-helper is a public library outside of mtmd. it must not include internal headers" #endif -#ifdef MTMD_VIDEO -#include "sheredom/subprocess.h" -#include -#ifndef _WIN32 -#include -#include -#include -#endif -#endif - // // internal logging functions // @@ -433,20 +423,19 @@ static bool is_webp_file(const unsigned char * buf, size_t len) { return len >= 12 && memcmp(buf, "RIFF", 4) == 0 && memcmp(buf + 8, "WEBP", 4) == 0; } -#ifdef MTMD_VIDEO -static mtmd_bitmap * decode_webp_with_ffmpeg(const mtmd_context * mctx, const unsigned char * buf, size_t len, bool placeholder, - const mtmd_helper_video_init_params & params); +static mtmd_bitmap * decode_still_image(const char * path, const unsigned char * buf, size_t len, bool placeholder, + const mtmd_helper_video_init_params & params); +static mtmd_helper_video * video_init(const mtmd_context * mctx, const char * path, const unsigned char * buf, size_t len, + const mtmd_helper_video_init_params & params); static void mtmd_helper_video_set_id(mtmd_helper_video * vctx, const std::string & id); -#endif -mtmd_helper_bitmap_wrapper mtmd_helper_bitmap_init_from_buf(const mtmd_context * ctx, const unsigned char * buf, size_t len, bool placeholder, - mtmd_helper_init_opt opt) { +static mtmd_helper_bitmap_wrapper mtmd_helper_bitmap_init_impl(const mtmd_context * ctx, const char * fname, + const unsigned char * buf, size_t len, bool placeholder, + mtmd_helper_init_opt opt) { // calculate the hash if needed std::string id; mtmd_bitmap * result = nullptr; - GGML_UNUSED(opt); // only used by video code paths - if (!placeholder) { // use sha256 to prevent cache poisoning id = hash_sha256_hex(buf, len); @@ -481,10 +470,9 @@ mtmd_helper_bitmap_wrapper mtmd_helper_bitmap_init_from_buf(const mtmd_context * // otherwise, fallthrough to video decoding (if supported) } -#ifdef MTMD_VIDEO - // stb_image does not support webp; decode it with ffmpeg as a single frame + // stb_image does not support webp; decode it with the video decoder as a single frame if (!result && is_webp_file(buf, len)) { - result = decode_webp_with_ffmpeg(ctx, buf, len, placeholder, opt.video_params); + result = decode_still_image(fname, buf, len, placeholder, opt.video_params); if (!result) { LOG_ERR("%s: failed to decode webp buffer\n", __func__); return {nullptr, nullptr}; @@ -492,12 +480,10 @@ mtmd_helper_bitmap_wrapper mtmd_helper_bitmap_init_from_buf(const mtmd_context * mtmd_bitmap_set_id(result, id.empty() ? nullptr : id.c_str()); return {result, nullptr}; } -#endif // last try: load as video -#ifdef MTMD_VIDEO if (!result) { - auto video_ctx = mtmd_helper_video_init_from_buf(ctx, buf, len, opt.video_params); + auto video_ctx = video_init(ctx, fname, buf, len, opt.video_params); if (!video_ctx) { LOG_ERR("%s: failed to decode buffer as either image/audio/video\n", __func__); return {nullptr, nullptr}; @@ -516,17 +502,16 @@ mtmd_helper_bitmap_wrapper mtmd_helper_bitmap_init_from_buf(const mtmd_context * mtmd_bitmap_set_is_video(result, true); return {result, video_ctx}; } -#else - if (!result) { - LOG_ERR("%s: failed to decode buffer as either image or audio (video support not compiled in)\n", __func__); - return {nullptr, nullptr}; - } -#endif // should not reach here return {nullptr, nullptr}; } +mtmd_helper_bitmap_wrapper mtmd_helper_bitmap_init_from_buf(const mtmd_context * ctx, const unsigned char * buf, size_t len, bool placeholder, + mtmd_helper_init_opt opt) { + return mtmd_helper_bitmap_init_impl(ctx, nullptr, buf, len, placeholder, opt); +} + mtmd_helper_bitmap_wrapper mtmd_helper_bitmap_init_from_file(const mtmd_context * ctx, const char * fname, bool placeholder, mtmd_helper_init_opt opt) { #ifdef _WIN32 @@ -569,7 +554,7 @@ mtmd_helper_bitmap_wrapper mtmd_helper_bitmap_init_from_file(const mtmd_context return {nullptr, nullptr}; } - return mtmd_helper_bitmap_init_from_buf(ctx, buf.data(), buf.size(), placeholder, opt); + return mtmd_helper_bitmap_init_impl(ctx, fname, buf.data(), buf.size(), placeholder, opt); } // @@ -595,7 +580,7 @@ int32_t mtmd_helper_video_seq::ts_per_n_frames() const { return tmpl == MTMD_VIDEO_TEMPLATE_LFM3 || tmpl == MTMD_VIDEO_TEMPLATE_LFM3_OMNI || tmpl == MTMD_VIDEO_TEMPLATE_QWEN3VL ? 2 : 0; } -void mtmd_helper_video_seq::configure(const mtmd_video_config & cfg, const mtmd_helper_video_src & src, float fps_target_arg) { +void mtmd_helper_video_seq::configure(const mtmd_video_config & cfg, const mtmd_helper_video_stream_info & src, float fps_target_arg) { tmpl = cfg.tmpl; uint32_t width = src.width; uint32_t height = src.height; @@ -644,6 +629,37 @@ void mtmd_helper_video_seq::configure(const mtmd_video_config & cfg, const mtmd_ frame_buf.resize((size_t)width * height * 3); } +void mtmd_helper_video_seq::sample_by_index(const mtmd_helper_video_stream_info & src) { + if (!frame_indices.empty() || src.fps <= 0.0f) { + return; + } + const int32_t n_src = src.n_frames > 0 ? src.n_frames + : src.duration > 0.0f ? (int32_t) std::nearbyint(src.duration * src.fps) : 0; + if (info.n_frames <= 0 || n_src <= 0) { + // unknown length: keep every frame + fps_target = src.fps; + info.fps = src.fps; + return; + } + frame_indices.resize(info.n_frames); + for (int32_t i = 0; i < info.n_frames; i++) { + frame_indices[i] = std::min(n_src - 1, (int32_t) ((double) i * src.fps / fps_target)); + } + src_fps = src.fps; + n_src_frames = n_src; +} + +void mtmd_helper_video_seq::set_soundtrack(const float * pcm, size_t n, int rate) { + if (!pcm || n == 0) { + LOG_WRN("%s: the video has no soundtrack\n", __func__); + return; + } + // keep only the span of the video stream: n_frames / fps seconds from its first frame + const size_t len = (size_t) std::nearbyint((double) n_src_frames / src_fps * rate); + audio_rate = rate; + soundtrack.assign(pcm, pcm + std::min(n, len)); +} + // the soundtrack of the group of frames that ends with frame idx, nullptr if the soundtrack has ended mtmd_bitmap * mtmd_helper_video_seq::soundtrack_slice(int32_t idx) const { const int32_t ts_per_n_frames = this->ts_per_n_frames(); @@ -675,7 +691,7 @@ mtmd_bitmap * mtmd_helper_video_seq::read_next_frame() { // skip to the wanted source frame; a repeated index reuses the frame in frame_buf const int32_t want = frame_indices[next_index]; while (src_frame <= want) { - if (!read_raw_frame()) { + if (!(src_frame < want ? skip_raw_frame() : read_raw_frame())) { src_alive = false; return nullptr; } @@ -790,345 +806,138 @@ int32_t mtmd_helper_video_seq::read_next(mtmd_bitmap ** out_bitmap, char ** out_ return 0; } -#ifdef MTMD_VIDEO - struct mtmd_helper_video : mtmd_helper_video_seq { - const mtmd_context * mctx; - std::string path; - std::vector input_buf; // non-empty when initialized from buffer - std::string ffmpeg_bin; - std::string ffprobe_bin; + mtmd_helper_video_decoder dec = {}; + void * state = nullptr; - // RAII wrapper for managing subprocess - struct subprocess_handle { - struct subprocess_s proc = {}; - bool created = false; // process exists and must be cleaned up - bool alive = false; // process can still give us data - std::thread feeder; - - subprocess_handle() = default; - subprocess_handle(const subprocess_handle &) = delete; - subprocess_handle & operator=(const subprocess_handle &) = delete; - ~subprocess_handle() { stop(); } - - void stop() { - // note: alive becomes false on stdout EOF, but the process still needs cleanup - if (!created) { - return; - } - subprocess_terminate(&proc); -#ifdef _WIN32 - // no SIGPIPE on windows: a blocked feeder only gets a broken pipe once we close our read end of the child stdin - if (proc.hStdInput) { - CloseHandle(proc.hStdInput); - proc.hStdInput = nullptr; - } -#endif - // join before destroy: feeder holds a FILE* from subprocess_stdin; - // subprocess_destroy closes it, so the thread must finish first - if (feeder.joinable()) { - feeder.join(); - } - subprocess_join(&proc, nullptr); // reap the child, or else it stays a zombie - subprocess_destroy(&proc); - created = false; - alive = false; + ~mtmd_helper_video() override { + if (state) { + dec.close(state); } - - FILE * stdout_pipe() { - return subprocess_stdout(&proc); - } - - // buf is tied to lifetime of mtmd_helper_video, so it's guaranteed to outlive the feeder thread - void start_feeder(const std::vector & buf) { - feeder = std::thread([this, &buf]() { -#ifndef _WIN32 - // ffmpeg can exit before it reads all the input, for example when ffprobe already got the metadata. - // the write below must then fail with EPIPE, instead of killing the process with SIGPIPE - sigset_t sigpipe_set; - sigemptyset(&sigpipe_set); - sigaddset(&sigpipe_set, SIGPIPE); - pthread_sigmask(SIG_BLOCK, &sigpipe_set, nullptr); // linux sends the signal to the writing thread -#endif - FILE * f = subprocess_stdin(&proc); - if (!f) { - return; - } -#ifdef F_SETNOSIGPIPE - fcntl(fileno(f), F_SETNOSIGPIPE, 1); // macos/bsd send it to the process, so turn it off per fd -#endif - fwrite(buf.data(), 1, buf.size(), f); - fclose(f); - proc.stdin_file = nullptr; // prevent double-close in subprocess_destroy - }); - } - }; - - subprocess_handle sp; - - // apply mtmd_get_video_config(); false for still images decoded via ffmpeg - bool use_video_config = false; - double video_start_s = 0.0; // start time of the video stream, to align the soundtrack - - bool is_buf_input() const { - return !input_buf.empty(); } - bool probe(float fps_target_arg) { - // cache: makes stdin seekable; without it ffprobe reports width/height 0 - // for mp4 files whose moov atom sits at the end - const char * input_arg = is_buf_input() ? "cache:pipe:0" : path.c_str(); - std::vector cmd = { - ffprobe_bin.c_str(), - "-v", "quiet", - "-show_entries", "stream=width,height,r_frame_rate,nb_frames,duration,start_time", - "-select_streams", "v:0", - "-of", "default=noprint_wrappers=1", - }; - if (is_buf_input()) { - // remove the 64KB read-ahead limit of cache:, or else ffprobe cannot reach a moov atom at end of file - cmd.insert(cmd.end(), {"-read_ahead_limit", "-1"}); - } - cmd.insert(cmd.end(), {input_arg, nullptr}); - - LOG_DBG("%s: launching:", __func__); - for (size_t i = 0; cmd[i]; i++) { LOG_DBG(" %s", cmd[i]); } - LOG_DBG("\n"); - - subprocess_handle probe_sp; - if (subprocess_create(cmd.data(), - subprocess_option_search_user_path | subprocess_option_inherit_environment, - &probe_sp.proc) != 0) { - LOG_ERR("%s: failed to launch ffprobe\n", __func__); - return false; - } - probe_sp.created = true; - probe_sp.alive = true; - - if (is_buf_input()) { - probe_sp.start_feeder(input_buf); - } - - mtmd_helper_video_src src; - char line[256]; - FILE * fp = probe_sp.stdout_pipe(); - - while (fgets(line, sizeof(line), fp)) { - char * eq = strchr(line, '='); - if (!eq) continue; - *eq = '\0'; - const char * key = line; - const char * val = eq + 1; - char * nl = (char *)strchr(val, '\n'); - if (nl) *nl = '\0'; - - if (strcmp(key, "width") == 0) { - src.width = (uint32_t)atoi(val); - } else if (strcmp(key, "height") == 0) { - src.height = (uint32_t)atoi(val); - } else if (strcmp(key, "r_frame_rate") == 0) { - src.fps = parse_rational(val); - } else if (strcmp(key, "nb_frames") == 0 && strcmp(val, "N/A") != 0) { - src.n_frames = atoi(val); - } else if (strcmp(key, "duration") == 0 && strcmp(val, "N/A") != 0) { - src.duration = (float)atof(val); - } else if (strcmp(key, "start_time") == 0 && strcmp(val, "N/A") != 0) { - video_start_s = atof(val); - } - } - - probe_sp.stop(); - - if (src.width == 0 || src.height == 0 || src.fps <= 0.0f) { - return false; - } - - mtmd_video_config cfg{MTMD_VIDEO_TEMPLATE_DEFAULT, 0.0f, 0, 0, 0}; - if (use_video_config) { - cfg = mtmd_get_video_config(mctx, src.width, src.height, src.n_frames, src.fps); - } - configure(cfg, src, fps_target_arg); - return true; - } - - // the soundtrack over the span of the video stream: from its first frame for n_frames / fps seconds, mono at the - // rate of the audio encoder; empty if the file has no audio - void load_soundtrack() { - const int rate = mtmd_get_audio_sample_rate(mctx); - if (tmpl != MTMD_VIDEO_TEMPLATE_LFM3_OMNI || frame_indices.empty() || rate <= 0) { - return; - } - const std::string rate_str = std::to_string(rate); - std::vector cmd = {ffmpeg_bin.c_str(), "-nostdin"}; - cmd.reserve(20); // GCC 11 -Wstringop-overflow false positive on the inserts below otherwise - if (is_buf_input()) { - cmd.insert(cmd.end(), {"-read_ahead_limit", "-1"}); - } - cmd.insert(cmd.end(), {"-i", is_buf_input() ? "cache:pipe:0" : path.c_str(), "-map", "0:a:0", "-vn", "-ac", "1", - "-ar", rate_str.c_str(), "-f", "f32le", "pipe:1", "-loglevel", "error", nullptr}); - subprocess_handle audio_sp; - if (subprocess_create(cmd.data(), subprocess_option_search_user_path | subprocess_option_inherit_environment, - &audio_sp.proc) != 0) { - LOG_WRN("%s: failed to launch ffmpeg for the soundtrack\n", __func__); - return; - } - audio_sp.created = true; - audio_sp.alive = true; - if (is_buf_input()) { - audio_sp.start_feeder(input_buf); - } - std::vector pcm; - float block[4096]; - size_t n; - while ((n = fread(block, sizeof(float), 4096, audio_sp.stdout_pipe())) > 0) { - pcm.insert(pcm.end(), block, block + n); - } - audio_sp.stop(); - - const size_t start = (size_t) std::max(0.0, std::nearbyint(video_start_s * rate)); - const size_t len = (size_t) std::nearbyint((double) n_src_frames / src_fps * rate); - if (start >= pcm.size()) { - LOG_WRN("%s: the video has no soundtrack\n", __func__); - return; - } - audio_rate = rate; - soundtrack.assign(pcm.begin() + start, pcm.begin() + std::min(pcm.size(), start + len)); - } - - bool start_ffmpeg(float seek_seconds) { - char seek_buf[64]; - char fps_buf[64]; - - std::vector cmd; - cmd.push_back(ffmpeg_bin.c_str()); - - if (!is_buf_input() && seek_seconds > 0.0f) { - // input-side seek: fast, keyframe-accurate; only valid for seekable file inputs - snprintf(seek_buf, sizeof(seek_buf), "%.6f", seek_seconds); - cmd.push_back("-ss"); - cmd.push_back(seek_buf); - } - - cmd.push_back("-nostdin"); - if (is_buf_input()) { - // remove the 64KB read-ahead limit of cache:, or else ffmpeg cannot reach a moov atom at end of file - cmd.push_back("-read_ahead_limit"); - cmd.push_back("-1"); - } - cmd.push_back("-i"); - // cache:pipe:0 wraps stdin with a seekable in-memory cache, letting ffmpeg seek - // backwards for container headers (e.g. MP4 moov atom at end of file) - cmd.push_back(is_buf_input() ? "cache:pipe:0" : path.c_str()); - - if (seek_seconds > 0.0f && is_buf_input()) { - // output-side seek: frame-accurate but decodes and discards frames up to seek point - snprintf(seek_buf, sizeof(seek_buf), "%.6f", seek_seconds); - cmd.push_back("-ss"); - cmd.push_back(seek_buf); - } - - std::string vf; - if (!frame_indices.empty()) { - // frames are picked by index in read_next_frame(); B=0 C=0.5 bicubic on RGB is closest to Pillow - if (out_width > 0 && out_height > 0) { - char scale_buf[160]; - snprintf(scale_buf, sizeof(scale_buf), - "format=rgb24,scale=%d:%d:flags=bicubic+accurate_rnd+full_chroma_int:param0=0:param1=0.5", - out_width, out_height); - vf = scale_buf; - } - } else if (fps_target > 0.0f) { - if (out_width > 0 && out_height > 0) { - snprintf(fps_buf, sizeof(fps_buf), "fps=%.6f,scale=%d:%d", fps_target, out_width, out_height); - } else { - snprintf(fps_buf, sizeof(fps_buf), "fps=%.6f", fps_target); - } - vf = fps_buf; - } - if (!vf.empty()) { - cmd.push_back("-vf"); - cmd.push_back(vf.c_str()); - } - - char frames_buf[16]; - const int32_t n_out = !frame_indices.empty() ? frame_indices.back() + 1 : target_frames; - if (n_out > 0) { - snprintf(frames_buf, sizeof(frames_buf), "%d", n_out); - cmd.push_back("-frames:v"); - cmd.push_back(frames_buf); - } - - cmd.push_back("-f"); - cmd.push_back("rawvideo"); - cmd.push_back("-pix_fmt"); - cmd.push_back("rgb24"); - cmd.push_back("pipe:1"); - cmd.push_back("-loglevel"); - cmd.push_back("error"); - cmd.push_back(nullptr); - - LOG_DBG("%s: launching:", __func__); - for (size_t i = 0; cmd[i]; i++) { - LOG_DBG(" %s", cmd[i]); - } - LOG_DBG("\n"); - - int ret = subprocess_create( - cmd.data(), - subprocess_option_search_user_path | subprocess_option_inherit_environment, - &sp.proc); - - sp.created = (ret == 0); - sp.alive = (ret == 0); - LOG_DBG("%s: subprocess_create ret=%d proc_alive=%d\n", __func__, ret, (int)sp.alive); - - if (sp.alive && is_buf_input()) { - LOG_DBG("%s: starting feeder thread for %zu-byte buffer\n", __func__, input_buf.size()); - sp.start_feeder(input_buf); - } - - src_alive = sp.alive; - return sp.alive; - } - - void stop_ffmpeg() { - sp.stop(); - } - - // read one raw frame from ffmpeg into frame_buf bool read_raw_frame() override { - FILE * fp = sp.stdout_pipe(); - const size_t frame_size = (size_t)info.width * info.height * 3; - LOG_DBG("%s: reading frame %d, expecting %zu bytes (%ux%u)\n", - __func__, current_frame, frame_size, info.width, info.height); + return dec.read_frame(state, frame_buf.data(), info.width, info.height); + } - size_t total_read = 0; - while (total_read < frame_size) { - size_t n = fread(frame_buf.data() + total_read, 1, frame_size - total_read, fp); - if (n == 0) { - // clean EOF only if no bytes read yet; partial frame is an error - LOG_DBG("%s: fread returned 0 after %zu/%zu bytes (ferror=%d)\n", - __func__, total_read, frame_size, ferror(fp)); - return false; - } - total_read += n; + bool skip_raw_frame() override { + return dec.read_frame(state, nullptr, info.width, info.height); + } + + bool start(const mtmd_helper_video_stream_info & src, float fps) { + bool fps_picked = false; + if (dec.start && !dec.start(state, info.width, info.height, fps, &fps_picked)) { + LOG_ERR("%s: the video decoder failed to start\n", __func__); + return false; + } + if (fps > 0.0f && !fps_picked) { + sample_by_index(src); } return true; } - static float parse_rational(const char * s) { - int num = 0, den = 1; - if (sscanf(s, "%d/%d", &num, &den) == 2 && den > 0) { - return (float)num / (float)den; + void load_soundtrack(const mtmd_context * mctx) { + const int rate = mtmd_get_audio_sample_rate(mctx); + if (tmpl != MTMD_VIDEO_TEMPLATE_LFM3_OMNI || frame_indices.empty() || rate <= 0 || !dec.read_audio) { + return; } - float val; - if (sscanf(s, "%f", &val) == 1) { - return val; - } - return 0.0f; + size_t n = 0; + const float * pcm = dec.read_audio(state, rate, &n); + set_soundtrack(pcm, n, rate); } }; + +static const mtmd_helper_video_decoder * video_builtin_decoder() { +#ifdef MTMD_VIDEO + return mtmd_helper_video_decoder_ffmpeg(); +#else + return nullptr; #endif +} + +static std::unique_ptr video_open(const char * path, const unsigned char * buf, size_t len, + const mtmd_helper_video_init_params & params, + mtmd_helper_video_stream_info & src) { + const mtmd_helper_video_decoder * dec = params.decoder; + void * user_data = dec ? dec->user_data : nullptr; + if (!dec) { + dec = video_builtin_decoder(); + if (!dec) { + LOG_ERR("%s: video is not supported in this build (MTMD_VIDEO is set to OFF) and no decoder is given\n", __func__); + return nullptr; + } + user_data = const_cast(params.ffmpeg_bin_dir); + } + + if (!dec->open || !dec->read_frame || !dec->close) { + LOG_ERR("%s: the video decoder must have open, read_frame and close\n", __func__); + return nullptr; + } + + auto ctx = std::make_unique(); + ctx->dec = *dec; + + src = {0, 0, 0.0f, -1, -1.0f}; + ctx->state = ctx->dec.open(user_data, path, buf, len, &src); + if (!ctx->state || src.width == 0 || src.height == 0 || src.fps <= 0.0f) { + LOG_ERR("%s: the video decoder failed to open the video\n", __func__); + return nullptr; + } + return ctx; +} + +static mtmd_helper_video * video_init(const mtmd_context * mctx, const char * path, const unsigned char * buf, size_t len, + const mtmd_helper_video_init_params & params) { + mtmd_helper_video_stream_info src; + auto ctx = video_open(path, buf, len, params, src); + if (!ctx) { + return nullptr; + } + + ctx->timestamp_interval_ms = params.timestamp_interval_ms; + ctx->configure(mtmd_get_video_config(mctx, src.width, src.height, src.n_frames, src.fps), src, params.fps_target); + if (!ctx->start(src, ctx->frame_indices.empty() ? ctx->fps_target : 0.0f)) { + return nullptr; + } + if (!params.no_soundtrack) { + ctx->load_soundtrack(mctx); + } + ctx->src_alive = true; + return ctx.release(); +} + +static mtmd_bitmap * decode_still_image(const char * path, const unsigned char * buf, size_t len, bool placeholder, + const mtmd_helper_video_init_params & params) { + // an application's decoder may not read images, prefer the built-in one + mtmd_helper_video_init_params still_params = params; + if (video_builtin_decoder()) { + still_params.decoder = nullptr; + } + mtmd_helper_video_stream_info src; + auto vctx = video_open(path, buf, len, still_params, src); + if (!vctx) { + return nullptr; + } + if (placeholder) { + return mtmd_bitmap_init(src.width, src.height, nullptr); + } + // still image: the fps filter would output no frame, so disable it + vctx->configure({MTMD_VIDEO_TEMPLATE_DEFAULT, 0.0f, 0, 0, 0}, src, 0.0f); + if (!vctx->start(src, 0.0f)) { + return nullptr; + } + vctx->src_alive = true; + mtmd_bitmap * frame = vctx->read_next_frame(); + if (frame) { + mtmd_bitmap_set_mergeable(frame, false); + mtmd_bitmap_set_is_video(frame, false); + } + return frame; +} + +static void mtmd_helper_video_set_id(mtmd_helper_video * vctx, const std::string & id) { + vctx->id = id; +} mtmd_helper_video_init_params mtmd_helper_video_init_params_default() { return { @@ -1136,6 +945,7 @@ mtmd_helper_video_init_params mtmd_helper_video_init_params_default() { /* ffmpeg_bin_dir */ nullptr, /* timestamp_interval_ms */ 5000, /* no_soundtrack */ false, + /* decoder */ nullptr, }; } @@ -1145,168 +955,32 @@ mtmd_helper_init_opt mtmd_helper_init_opt_default() { }; } -static std::string video_resolve_bin(const char * bin_dir, const char * name) { - if (!bin_dir || bin_dir[0] == '\0') { - return name; // rely on PATH - } - std::string result = bin_dir; - char last = result.back(); - if (last != '/' && last != '\\') { -#ifdef _WIN32 - result += '\\'; -#else - result += '/'; -#endif - } - result += name; -#ifdef _WIN32 - result += ".exe"; -#endif - return result; -} - -#ifdef MTMD_VIDEO -static void mtmd_helper_video_set_id(mtmd_helper_video * vctx, const std::string & id) { - vctx->id = id; -} - -static mtmd_bitmap * decode_webp_with_ffmpeg(const mtmd_context * mctx, const unsigned char * buf, size_t len, bool placeholder, - const mtmd_helper_video_init_params & params) { - mtmd_helper_video vctx; - vctx.mctx = mctx; - vctx.input_buf.assign(buf, buf + len); - vctx.ffmpeg_bin = video_resolve_bin(params.ffmpeg_bin_dir, "ffmpeg"); - vctx.ffprobe_bin = video_resolve_bin(params.ffmpeg_bin_dir, "ffprobe"); - if (!vctx.probe(0.0f)) { - return nullptr; - } - if (placeholder) { - return mtmd_bitmap_init(vctx.info.width, vctx.info.height, nullptr); - } - // still image: the fps filter would output no frame, so disable it - vctx.fps_target = 0.0f; - if (!vctx.start_ffmpeg(0.0f)) { - return nullptr; - } - mtmd_bitmap * frame = vctx.read_next_frame(); - if (frame) { - mtmd_bitmap_set_mergeable(frame, false); - mtmd_bitmap_set_is_video(frame, false); - } - return frame; -} -#endif - mtmd_helper_video * mtmd_helper_video_init( const mtmd_context * mctx, const char * path, mtmd_helper_video_init_params params) { -#ifdef MTMD_VIDEO - auto * ctx = new mtmd_helper_video(); - - ctx->mctx = mctx; - ctx->path = path; - ctx->ffmpeg_bin = video_resolve_bin(params.ffmpeg_bin_dir, "ffmpeg"); - ctx->ffprobe_bin = video_resolve_bin(params.ffmpeg_bin_dir, "ffprobe"); - ctx->timestamp_interval_ms = params.timestamp_interval_ms; - ctx->use_video_config = true; - - if (!ctx->probe(params.fps_target)) { - LOG_ERR("%s: ffprobe failed for '%s' (is ffprobe in PATH?)\n", __func__, path); - delete ctx; - return nullptr; - } - if (!params.no_soundtrack) { - ctx->load_soundtrack(); - } - - if (!ctx->start_ffmpeg(0.0f)) { - LOG_ERR("%s: failed to start ffmpeg for '%s' (is ffmpeg in PATH?)\n", __func__, path); - delete ctx; - return nullptr; - } - - return ctx; -#else - GGML_UNUSED(mctx); - GGML_UNUSED(path); - GGML_UNUSED(params); - LOG_ERR("%s: video is not supported in this build (MTMD_VIDEO is set to OFF)\n", __func__); - return nullptr; -#endif + return video_init(mctx, path, nullptr, 0, params); } mtmd_helper_video * mtmd_helper_video_init_from_buf( const mtmd_context * mctx, const unsigned char * buf, size_t len, mtmd_helper_video_init_params params) { -#ifdef MTMD_VIDEO - auto * ctx = new mtmd_helper_video(); - - ctx->mctx = mctx; - ctx->input_buf.assign(buf, buf + len); - ctx->ffmpeg_bin = video_resolve_bin(params.ffmpeg_bin_dir, "ffmpeg"); - ctx->ffprobe_bin = video_resolve_bin(params.ffmpeg_bin_dir, "ffprobe"); - ctx->timestamp_interval_ms = params.timestamp_interval_ms; - ctx->use_video_config = true; - - if (!ctx->probe(params.fps_target)) { - LOG_ERR("%s: ffprobe failed on buffer (is ffprobe in PATH?)\n", __func__); - delete ctx; - return nullptr; - } - if (!params.no_soundtrack) { - ctx->load_soundtrack(); - } - - if (!ctx->start_ffmpeg(0.0f)) { - LOG_ERR("%s: failed to start ffmpeg on buffer (is ffmpeg in PATH?)\n", __func__); - delete ctx; - return nullptr; - } - - return ctx; -#else - GGML_UNUSED(mctx); - GGML_UNUSED(buf); - GGML_UNUSED(len); - GGML_UNUSED(params); - LOG_ERR("%s: video is not supported in this build (MTMD_VIDEO is set to OFF)\n", __func__); - return nullptr; -#endif + return video_init(mctx, nullptr, buf, len, params); } void mtmd_helper_video_free(mtmd_helper_video * ctx) { -#ifdef MTMD_VIDEO - if (!ctx) return; - ctx->stop_ffmpeg(); delete ctx; -#else - GGML_UNUSED(ctx); - LOG_ERR("%s: video is not supported in this build (MTMD_VIDEO is set to OFF)\n", __func__); -#endif } mtmd_helper_video_info mtmd_helper_video_get_info(const mtmd_helper_video * ctx) { -#ifdef MTMD_VIDEO return ctx->info; -#else - GGML_UNUSED(ctx); - GGML_ASSERT(false && "video is not supported in this build (MTMD_VIDEO is set to OFF)"); -#endif } int32_t mtmd_helper_video_read_next(mtmd_helper_video * ctx, mtmd_bitmap ** out_bitmap, char ** out_text) { -#ifdef MTMD_VIDEO if (!ctx) return -2; return ctx->read_next(out_bitmap, out_text); -#else - GGML_UNUSED(ctx); - GGML_UNUSED(out_bitmap); - GGML_UNUSED(out_text); - GGML_ASSERT(false && "video is not supported in this build (MTMD_VIDEO is set to OFF)"); -#endif } mtmd_helper_cap mtmd_helper_get_capabilities(const llama_model * model, const mtmd_context * mctx) { @@ -1320,9 +994,7 @@ mtmd_helper_cap mtmd_helper_get_capabilities(const llama_model * model, const mt cap.vision = mtmd_support_vision(mctx); cap.audio = mtmd_support_audio(mctx); cap.audio_stream = mtmd_support_audio_stream(mctx); -#ifdef MTMD_VIDEO - cap.video = cap.vision; -#endif + cap.video = cap.vision && video_builtin_decoder() != nullptr; const auto gen_audio_type = mtmd_gen_audio_get_info(mctx).type; cap.audio_gen = gen_audio_type != MTMD_GEN_AUDIO_TYPE_NONE; cap.audio_gen_spk_ref = gen_audio_type == MTMD_GEN_AUDIO_TYPE_QWEN3TTS @@ -1341,10 +1013,5 @@ bool mtmd_helper_model_can_chat(const llama_context * lctx, const mtmd_context * } bool mtmd_helper_support_video(const mtmd_context * ctx) { -#ifdef MTMD_VIDEO - return mtmd_support_vision(ctx); -#else - GGML_UNUSED(ctx); - return false; -#endif + return mtmd_support_vision(ctx) && video_builtin_decoder() != nullptr; } diff --git a/tools/mtmd/mtmd-helper.h b/tools/mtmd/mtmd-helper.h index 7d72fdb99..9bb4f99c3 100644 --- a/tools/mtmd/mtmd-helper.h +++ b/tools/mtmd/mtmd-helper.h @@ -23,11 +23,40 @@ extern "C" { struct mtmd_helper_video; typedef struct mtmd_helper_video mtmd_helper_video; +struct mtmd_helper_video_stream_info { + uint32_t width; // as displayed (rotation applied) + uint32_t height; + float fps; + int32_t n_frames; // -1 if unknown + float duration; // in seconds, -1 if unknown +}; + +// Video decoder: the built-in one runs ffmpeg/ffprobe (MTMD_VIDEO), an application can provide its own. +// Call order: open, start, read_audio, read_frame..., close +struct mtmd_helper_video_decoder { + // path and/or buf is set; buf is only valid during the call + // returns the decoder state, NULL on failure + void * (*open)(void * user_data, const char * path, const unsigned char * buf, size_t len, + struct mtmd_helper_video_stream_info * info); + // optional: called once with the size read_frame() will ask for + // if fps > 0, a decoder that resamples to fps itself sets *fps_picked; otherwise the helper picks frames by index + bool (*start)(void * state, uint32_t width, uint32_t height, float fps, bool * fps_picked); + // next frame in presentation order, as RGB24 scaled to width x height (same on every call) + // rgb == NULL skips the frame; returns false at the end of the stream or on error + bool (*read_frame)(void * state, unsigned char * rgb, uint32_t width, uint32_t height); + // optional: the soundtrack as mono F32 PCM at sample_rate, starting at the first video frame + // the samples are valid until the next call to the decoder; NULL if there is no soundtrack + const float * (*read_audio)(void * state, int sample_rate, size_t * n_samples); + void (*close)(void * state); + void * user_data; +}; + struct mtmd_helper_video_init_params { float fps_target; // desired output fps; <= 0 means use the video's native fps, defaulted to 4.0f const char * ffmpeg_bin_dir; // directory containing ffmpeg/ffprobe binaries; NULL means search PATH int64_t timestamp_interval_ms; // interval for adding timestamp as text chunk (example: "[10m50.5s]"); <= 0 means no timestamp, defaulted to 5000ms bool no_soundtrack; // frames only, also for models that hear the soundtrack of a video (LFM3-Omni) + const struct mtmd_helper_video_decoder * decoder; // NULL means ffmpeg (requires MTMD_VIDEO) // TODO @ngxson : allow "placeholder" bitmap output for counting tokens }; @@ -64,7 +93,7 @@ MTMD_API struct mtmd_helper_bitmap_wrapper mtmd_helper_bitmap_init_from_file( // helper function to construct a mtmd_bitmap from a buffer containing a file // supported formats: // image: formats supported by stb_image: jpg, png, bmp, gif, etc. -// webp is decoded via ffmpeg, requires MTMD_VIDEO build with ffmpeg in PATH +// webp is decoded via ffmpeg (MTMD_VIDEO build with ffmpeg in PATH), or via opt.video_params.decoder if ffmpeg is not built in // audio: formats supported by miniaudio: wav, mp3, flac // note: // - for now, video input is only supported via C++ helper functions @@ -155,7 +184,7 @@ MTMD_API int32_t mtmd_helper_decode_image_chunk(mtmd_context * ctx, void * user_data); // -// video input helpers (requires ffmpeg/ffprobe installed on the system) +// video input helpers (requires ffmpeg/ffprobe installed on the system, or a mtmd_helper_video_decoder) // the notion of video only exists at the helper level, it is not visible to the core mtmd library // // NOTE: this implementation is model-agnostic, it can be used with any vision-capable model @@ -172,16 +201,14 @@ struct mtmd_helper_video_info { // note: mtmd_helper_video_init_params is defined at the top, as it is part of mtmd_helper_init_opt -// returns NULL on failure (ffprobe not found, file unreadable, etc.) +// returns NULL on failure (no decoder, ffprobe not found, file unreadable, etc.) MTMD_API mtmd_helper_video * mtmd_helper_video_init( const struct mtmd_context * mctx, const char * path, struct mtmd_helper_video_init_params params); // Same as mtmd_helper_video_init(), but reads from an in-memory buffer. -// The buffer is copied internally; the caller does not need to keep it alive. -// Note: pipe input is not seekable, so seeking will use output-side seeking -// (ffmpeg decodes and discards frames up to the target position). +// The caller does not need to keep the buffer alive. MTMD_API mtmd_helper_video * mtmd_helper_video_init_from_buf( const struct mtmd_context * mctx, const unsigned char * buf, size_t len, @@ -203,7 +230,7 @@ struct mtmd_helper_cap { // input capabilities bool vision; bool audio; - bool video; // also depending on MTMD_VIDEO build option + bool video; // built-in decoder only (MTMD_VIDEO), a custom decoder is not counted // realtime capabilities bool audio_stream; // realtime transcription, see mtmd_audio_stream_init() @@ -435,14 +462,6 @@ inline std::vector audio_stream_logit_bias(const mtmd_context // internals of mtmd_helper_video, do not use in applications // -struct mtmd_helper_video_src { - uint32_t width = 0; - uint32_t height = 0; - float fps = 0.0f; - int32_t n_frames = -1; - float duration = -1.0f; -}; - struct mtmd_helper_video_seq { float fps_target = 0.0f; mtmd_helper_video_info info = {}; @@ -482,8 +501,14 @@ struct mtmd_helper_video_seq { virtual ~mtmd_helper_video_seq(); virtual bool read_raw_frame() = 0; + virtual bool skip_raw_frame() { return read_raw_frame(); } - void configure(const mtmd_video_config & cfg, const mtmd_helper_video_src & src, float fps_target_arg); + void configure(const mtmd_video_config & cfg, const mtmd_helper_video_stream_info & src, float fps_target_arg); + + // pick the frames of fps_target by index, for a source that yields every frame + void sample_by_index(const mtmd_helper_video_stream_info & src); + // pcm: mono, starting at the first video frame + void set_soundtrack(const float * pcm, size_t n, int rate); int32_t ts_per_n_frames() const; diff --git a/tools/server/server-context.cpp b/tools/server/server-context.cpp index a6059ab38..0d613667d 100644 --- a/tools/server/server-context.cpp +++ b/tools/server/server-context.cpp @@ -1416,6 +1416,8 @@ private: } mtmd_cap = mtmd_helper_get_capabilities(model_tgt, mctx); + // the helper only knows about the built-in video decoder + mtmd_cap.video = mtmd_cap.video || (mtmd_cap.vision && init_opt.video_params.decoder != nullptr); if (!llama_memory_can_shift(llama_get_memory(ctx_tgt))) { if (params_base.ctx_shift) { @@ -5332,6 +5334,10 @@ void server_context::set_state_callback(server_state_callback_t callback) { impl->callback_state = std::move(callback); } +void server_context::set_video_decoder(const mtmd_helper_video_decoder * decoder) { + impl->init_opt.video_params.decoder = decoder; +} + // // server_routes // diff --git a/tools/server/server-context.h b/tools/server/server-context.h index 4b145cb8a..2c91d53a4 100644 --- a/tools/server/server-context.h +++ b/tools/server/server-context.h @@ -12,6 +12,7 @@ #include struct server_context_impl; // private implementation +struct mtmd_helper_video_decoder; // see mtmd-helper.h struct server_context_meta { std::string build_info; @@ -90,6 +91,10 @@ struct server_context { // note: must be set before load_model() is called void set_state_callback(server_state_callback_t callback); + + // decode video with a custom decoder instead of ffmpeg, the decoder must outlive the server + // note: must be set before load_model() is called + void set_video_decoder(const mtmd_helper_video_decoder * decoder); };