// mtmd_helper_video_decoder backed by ffmpeg/ffprobe subprocesses #ifdef MTMD_VIDEO // fix problem with std::min and std::max #if defined(_WIN32) #define WIN32_LEAN_AND_MEAN #ifndef NOMINMAX # define NOMINMAX #endif #include #endif #include "mtmd-helper.h" #include "mtmd-helper-common.h" #include "sheredom/subprocess.h" #include #include #include #include #include #include #include #ifndef _WIN32 #include #include #include #endif #ifdef MTMD_INTERNAL_HEADER #error "mtmd-helper is a public library outside of mtmd. it must not include internal headers" #endif namespace { // RAII wrapper for managing subprocess struct subprocess_handle { struct subprocess_s proc = {}; bool created = false; // process exists and must be cleaned up bool alive = false; // process can still give us data std::thread feeder; subprocess_handle() = default; subprocess_handle(const subprocess_handle &) = delete; subprocess_handle & operator=(const subprocess_handle &) = delete; ~subprocess_handle() { stop(); } void stop() { // note: alive becomes false on stdout EOF, but the process still needs cleanup if (!created) { return; } subprocess_terminate(&proc); #ifdef _WIN32 // no SIGPIPE on windows: a blocked feeder only gets a broken pipe once we close our read end of the child stdin if (proc.hStdInput) { CloseHandle(proc.hStdInput); proc.hStdInput = nullptr; } #endif // join before destroy: feeder holds a FILE* from subprocess_stdin; // subprocess_destroy closes it, so the thread must finish first if (feeder.joinable()) { feeder.join(); } subprocess_join(&proc, nullptr); // reap the child, or else it stays a zombie subprocess_destroy(&proc); created = false; alive = false; } FILE * stdout_pipe() { return subprocess_stdout(&proc); } // buf must outlive the feeder thread, i.e. this handle void start_feeder(const std::vector & buf) { feeder = std::thread([this, &buf]() { #ifndef _WIN32 // ffmpeg can exit before it reads all the input, for example when ffprobe already got the metadata. // the write below must then fail with EPIPE, instead of killing the process with SIGPIPE sigset_t sigpipe_set; sigemptyset(&sigpipe_set); sigaddset(&sigpipe_set, SIGPIPE); pthread_sigmask(SIG_BLOCK, &sigpipe_set, nullptr); // linux sends the signal to the writing thread #endif FILE * f = subprocess_stdin(&proc); if (!f) { return; } #ifdef F_SETNOSIGPIPE fcntl(fileno(f), F_SETNOSIGPIPE, 1); // macos/bsd send it to the process, so turn it off per fd #endif fwrite(buf.data(), 1, buf.size(), f); fclose(f); proc.stdin_file = nullptr; // prevent double-close in subprocess_destroy }); } }; std::string resolve_bin(const char * bin_dir, const char * name) { if (!bin_dir || bin_dir[0] == '\0') { return name; // rely on PATH } std::string result = bin_dir; char last = result.back(); if (last != '/' && last != '\\') { #ifdef _WIN32 result += '\\'; #else result += '/'; #endif } result += name; #ifdef _WIN32 result += ".exe"; #endif return result; } float parse_rational(const char * s) { int num = 0, den = 1; if (sscanf(s, "%d/%d", &num, &den) == 2 && den > 0) { return (float)num / (float)den; } float val; if (sscanf(s, "%f", &val) == 1) { return val; } return 0.0f; } struct ffmpeg_video { std::string path; std::vector input_buf; // non-empty when opened from a buffer std::string ffmpeg_bin; std::string ffprobe_bin; uint32_t src_width = 0; uint32_t src_height = 0; double video_start_s = 0.0; std::vector pcm; size_t pcm_start = 0; std::vector skip_buf; // must be destroyed before input_buf, which its feeder thread reads subprocess_handle sp; bool is_buf_input() const { return !input_buf.empty(); } bool probe(mtmd_helper_video_stream_info & info) { // cache: makes stdin seekable; without it ffprobe reports width/height 0 // for mp4 files whose moov atom sits at the end const char * input_arg = is_buf_input() ? "cache:pipe:0" : path.c_str(); std::vector cmd = { ffprobe_bin.c_str(), "-v", "quiet", "-show_entries", "stream=width,height,sample_aspect_ratio,r_frame_rate,nb_frames,duration,start_time:stream_side_data=rotation", "-select_streams", "v:0", "-of", "default=noprint_wrappers=1", }; if (is_buf_input()) { // remove the 64KB read-ahead limit of cache:, or else ffprobe cannot reach a moov atom at end of file cmd.insert(cmd.end(), {"-read_ahead_limit", "-1"}); } cmd.insert(cmd.end(), {input_arg, nullptr}); LOG_DBG("%s: launching:", __func__); for (size_t i = 0; cmd[i]; i++) { LOG_DBG(" %s", cmd[i]); } LOG_DBG("\n"); subprocess_handle probe_sp; if (subprocess_create(cmd.data(), subprocess_option_search_user_path | subprocess_option_inherit_environment, &probe_sp.proc) != 0) { LOG_ERR("%s: failed to launch ffprobe\n", __func__); return false; } probe_sp.created = true; probe_sp.alive = true; if (is_buf_input()) { probe_sp.start_feeder(input_buf); } int sar_num = 1; int sar_den = 1; int rotation = 0; char line[256]; FILE * fp = probe_sp.stdout_pipe(); while (fgets(line, sizeof(line), fp)) { char * eq = strchr(line, '='); if (!eq) continue; *eq = '\0'; const char * key = line; const char * val = eq + 1; char * nl = (char *)strchr(val, '\n'); if (nl) *nl = '\0'; if (strcmp(key, "width") == 0) { info.width = (uint32_t)atoi(val); } else if (strcmp(key, "height") == 0) { info.height = (uint32_t)atoi(val); } else if (strcmp(key, "r_frame_rate") == 0) { info.fps = parse_rational(val); } else if (strcmp(key, "nb_frames") == 0 && strcmp(val, "N/A") != 0) { info.n_frames = atoi(val); } else if (strcmp(key, "duration") == 0 && strcmp(val, "N/A") != 0) { info.duration = (float)atof(val); } else if (strcmp(key, "start_time") == 0 && strcmp(val, "N/A") != 0) { video_start_s = atof(val); } else if (strcmp(key, "sample_aspect_ratio") == 0) { if (sscanf(val, "%d:%d", &sar_num, &sar_den) != 2) { sar_num = sar_den = 1; // N/A } } else if (strcmp(key, "rotation") == 0) { rotation = atoi(val); } } probe_sp.stop(); // ffmpeg gives frames of the encoded size; the displayed size also has the pixel aspect ratio applied src_width = info.width; src_height = info.height; const double sar = sar_den > 0 ? (double) sar_num / sar_den : 1.0; if (sar >= 0.1 && sar <= 10.0) { // the value is from the file, ignore it if it is not plausible info.width = (uint32_t) std::nearbyint(info.width * sar); } // ffmpeg rotates the frames by default if (rotation % 180 == 90 || rotation % 180 == -90) { std::swap(src_width, src_height); std::swap(info.width, info.height); } return info.width > 0 && info.height > 0 && info.fps > 0.0f; } void load_soundtrack(int rate) { const std::string rate_str = std::to_string(rate); std::vector cmd = {ffmpeg_bin.c_str(), "-nostdin"}; cmd.reserve(20); // GCC 11 -Wstringop-overflow false positive on the inserts below otherwise if (is_buf_input()) { cmd.insert(cmd.end(), {"-read_ahead_limit", "-1"}); } cmd.insert(cmd.end(), {"-i", is_buf_input() ? "cache:pipe:0" : path.c_str(), "-map", "0:a:0", "-vn", "-ac", "1", "-ar", rate_str.c_str(), "-f", "f32le", "pipe:1", "-loglevel", "error", nullptr}); subprocess_handle audio_sp; if (subprocess_create(cmd.data(), subprocess_option_search_user_path | subprocess_option_inherit_environment, &audio_sp.proc) != 0) { LOG_WRN("%s: failed to launch ffmpeg for the soundtrack\n", __func__); return; } audio_sp.created = true; audio_sp.alive = true; if (is_buf_input()) { audio_sp.start_feeder(input_buf); } float block[4096]; size_t n; while ((n = fread(block, sizeof(float), 4096, audio_sp.stdout_pipe())) > 0) { pcm.insert(pcm.end(), block, block + n); } audio_sp.stop(); // align to the first video frame pcm_start = std::min(pcm.size(), (size_t) std::max(0.0, std::nearbyint(video_start_s * rate))); } bool start(uint32_t width, uint32_t height, float fps) { std::vector cmd; cmd.push_back(ffmpeg_bin.c_str()); cmd.push_back("-nostdin"); if (is_buf_input()) { // remove the 64KB read-ahead limit of cache:, or else ffmpeg cannot reach a moov atom at end of file cmd.push_back("-read_ahead_limit"); cmd.push_back("-1"); } cmd.push_back("-i"); // cache:pipe:0 wraps stdin with a seekable in-memory cache, letting ffmpeg seek // backwards for container headers (e.g. MP4 moov atom at end of file) cmd.push_back(is_buf_input() ? "cache:pipe:0" : path.c_str()); const bool scale = width != src_width || height != src_height; char vf[160] = ""; if (fps > 0.0f) { if (scale) { snprintf(vf, sizeof(vf), "fps=%.6f,scale=%u:%u", fps, width, height); } else { snprintf(vf, sizeof(vf), "fps=%.6f", fps); } } else if (scale) { // B=0 C=0.5 bicubic on RGB is closest to Pillow snprintf(vf, sizeof(vf), "format=rgb24,scale=%u:%u:flags=bicubic+accurate_rnd+full_chroma_int:param0=0:param1=0.5", width, height); } if (vf[0]) { cmd.push_back("-vf"); cmd.push_back(vf); } cmd.push_back("-f"); cmd.push_back("rawvideo"); cmd.push_back("-pix_fmt"); cmd.push_back("rgb24"); cmd.push_back("pipe:1"); cmd.push_back("-loglevel"); cmd.push_back("error"); cmd.push_back(nullptr); LOG_DBG("%s: launching:", __func__); for (size_t i = 0; cmd[i]; i++) { LOG_DBG(" %s", cmd[i]); } LOG_DBG("\n"); int ret = subprocess_create( cmd.data(), subprocess_option_search_user_path | subprocess_option_inherit_environment, &sp.proc); sp.created = (ret == 0); sp.alive = (ret == 0); LOG_DBG("%s: subprocess_create ret=%d proc_alive=%d\n", __func__, ret, (int)sp.alive); if (!sp.alive) { LOG_ERR("%s: failed to start ffmpeg (is ffmpeg in PATH?)\n", __func__); return false; } if (is_buf_input()) { LOG_DBG("%s: starting feeder thread for %zu-byte buffer\n", __func__, input_buf.size()); sp.start_feeder(input_buf); } return true; } bool read(unsigned char * dst, size_t n) { FILE * fp = sp.stdout_pipe(); if (!dst) { skip_buf.resize(n); dst = skip_buf.data(); } size_t total_read = 0; while (total_read < n) { const size_t got = fread(dst + total_read, 1, n - total_read, fp); if (got == 0) { // clean EOF only if no bytes read yet; partial frame is an error LOG_DBG("%s: fread returned 0 after %zu/%zu bytes (ferror=%d)\n", __func__, total_read, n, ferror(fp)); return false; } total_read += got; } return true; } }; void * ffmpeg_open(void * user_data, const char * path, const unsigned char * buf, size_t len, mtmd_helper_video_stream_info * info) { auto v = std::make_unique(); const char * bin_dir = static_cast(user_data); v->ffmpeg_bin = resolve_bin(bin_dir, "ffmpeg"); v->ffprobe_bin = resolve_bin(bin_dir, "ffprobe"); if (buf && len > 0) { v->input_buf.assign(buf, buf + len); } else if (path) { v->path = path; } if (!v->probe(*info)) { LOG_ERR("%s: ffprobe failed for '%s' (is ffprobe in PATH?)\n", __func__, v->is_buf_input() ? "" : v->path.c_str()); return nullptr; } return v.release(); } bool ffmpeg_start(void * state, uint32_t width, uint32_t height, float fps, bool * fps_picked) { *fps_picked = fps > 0.0f; return static_cast(state)->start(width, height, fps); } bool ffmpeg_read_frame(void * state, unsigned char * rgb, uint32_t width, uint32_t height) { auto * v = static_cast(state); if (!v->sp.alive) { return false; } if (!v->pcm.empty()) { std::vector().swap(v->pcm); // no longer needed } LOG_DBG("%s: reading a frame, expecting %zu bytes (%ux%u)\n", __func__, (size_t) width * height * 3, width, height); return v->read(rgb, (size_t) width * height * 3); } const float * ffmpeg_read_audio(void * state, int sample_rate, size_t * n_samples) { auto * v = static_cast(state); v->pcm.clear(); v->pcm_start = 0; v->load_soundtrack(sample_rate); *n_samples = v->pcm.size() - v->pcm_start; return *n_samples > 0 ? v->pcm.data() + v->pcm_start : nullptr; } void ffmpeg_close(void * state) { delete static_cast(state); } } // namespace const mtmd_helper_video_decoder * mtmd_helper_video_decoder_ffmpeg() { static const mtmd_helper_video_decoder decoder = { /* open */ ffmpeg_open, /* start */ ffmpeg_start, /* read_frame */ ffmpeg_read_frame, /* read_audio */ ffmpeg_read_audio, /* close */ ffmpeg_close, /* user_data */ nullptr, }; return &decoder; } #endif // MTMD_VIDEO