From 9cffdcc801582616250520966699cb5b25d28243 Mon Sep 17 00:00:00 2001 From: Abhiram <78226909+geckguy@users.noreply.github.com> Date: Thu, 3 Sep 2026 01:54:31 +0530 Subject: [PATCH] server : accept data: URLs for input_video and input_audio (#27735) * server : accept data: URLs for input_video and input_audio input_video and input_audio passed accept_base64_uri=false to handle_media(), so data: URLs got treated as raw base64 strings and failed later with a confusing media probe error (#27724). pass true for these two content types the same way image_url already does, and allow video/audio mime types in the data: url check instead of image only. data URL validation now throws std::invalid_argument so malformed input comes back as 400 instead of 500, matching the other input validation in this file. * server : simplify handle_media and drop unused accept_base64_uri flag * server : update comment and add unit test for invalid data URI MIME --- tools/server/server-common.cpp | 23 +++++++++++----------- tools/server/tests/unit/test_vision_api.py | 1 + 2 files changed, 13 insertions(+), 11 deletions(-) diff --git a/tools/server/server-common.cpp b/tools/server/server-common.cpp index c30955e89..2ac98b6fd 100644 --- a/tools/server/server-common.cpp +++ b/tools/server/server-common.cpp @@ -1062,8 +1062,7 @@ json oaicompat_completion_params_parse(const json & body) { static void handle_media( std::vector & out_files, const std::string & url, - const std::string & media_path, - bool accept_base64_uri) { + const std::string & media_path) { if (!media_path.empty()) { // should already be enforced by arg.cpp, but checking just in case GGML_ASSERT(media_path.back() == DIRECTORY_SEPARATOR); @@ -1104,15 +1103,17 @@ static void handle_media( data.assign((std::istreambuf_iterator(file)), std::istreambuf_iterator()); out_files.push_back(data); - } else if (accept_base64_uri && string_starts_with(url, "data:")) { - // try to decode base64 image + } else if (string_starts_with(url, "data:")) { + // try to decode base64 image, video, or audio std::vector parts = string_split(url, /*separator*/ ','); if (parts.size() != 2) { - throw std::runtime_error("Invalid uri-encoded base64 value"); - } else if (!string_starts_with(parts[0], "data:image/")) { - throw std::runtime_error("Invalid uri format: " + parts[0]); + throw std::invalid_argument("Invalid uri-encoded base64 value"); + } else if (!string_starts_with(parts[0], "data:image/") + && !string_starts_with(parts[0], "data:video/") + && !string_starts_with(parts[0], "data:audio/")) { + throw std::invalid_argument("Invalid uri format: " + parts[0]); } else if (!string_ends_with(parts[0], "base64")) { - throw std::runtime_error("uri must be base64 encoded"); + throw std::invalid_argument("uri must be base64 encoded"); } else { auto base64_data = parts[1]; auto decoded_data = base64_decode(base64_data); @@ -1219,7 +1220,7 @@ json oaicompat_chat_params_parse( json image_url = json_value(p, "image_url", json::object()); std::string url = json_value(image_url, "url", std::string()); - handle_media(out_files, url, opt.media_path, true); + handle_media(out_files, url, opt.media_path); p["type"] = "media_marker"; p["text"] = get_media_marker(); @@ -1234,7 +1235,7 @@ json oaicompat_chat_params_parse( json input_audio = json_value(p, "input_audio", json::object()); std::string url = json_value(input_audio, "data", json_value(input_audio, "url", std::string())); - handle_media(out_files, url, opt.media_path, false); + handle_media(out_files, url, opt.media_path); p["type"] = "media_marker"; p["text"] = get_media_marker(); @@ -1248,7 +1249,7 @@ json oaicompat_chat_params_parse( json input_video = json_value(p, "input_video", json::object()); std::string url = json_value(input_video, "data", json_value(input_video, "url", std::string())); - handle_media(out_files, url, opt.media_path, false); + handle_media(out_files, url, opt.media_path); p["type"] = "media_marker"; p["text"] = get_media_marker(); diff --git a/tools/server/tests/unit/test_vision_api.py b/tools/server/tests/unit/test_vision_api.py index 8b01c5372..3bf868e66 100644 --- a/tools/server/tests/unit/test_vision_api.py +++ b/tools/server/tests/unit/test_vision_api.py @@ -71,6 +71,7 @@ def test_v1_models_supports_multimodal_capability(): ("What is this:\n", "malformed", False, None), ("What is this:\n", "https://google.com/404", False, None), # non-existent image ("What is this:\n", "https://ggml.ai", False, None), # non-image data + ("What is this:\n", "data:text/html;base64,aGVsbG8=", False, None), # unsupported data uri mime # TODO @ngxson : test with multiple images, no images and with audio ] )