Lumpiasty/llama.cpp
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

server : accept data: URLs for input_video and input_audio (#27735)

Browse Source 
* server : accept data: URLs for input_video and input_audio

input_video and input_audio passed accept_base64_uri=false to
handle_media(), so data: URLs got treated as raw base64 strings and
failed later with a confusing media probe error (#27724).

pass true for these two content types the same way image_url already
does, and allow video/audio mime types in the data: url check instead
of image only. data URL validation now throws std::invalid_argument so
malformed input comes back as 400 instead of 500, matching the other
input validation in this file.

* server : simplify handle_media and drop unused accept_base64_uri flag

* server : update comment and add unit test for invalid data URI MIME
This commit is contained in:
Abhiram
2026-09-02 22:24:31 +02:00
committed by GitHub
parent f027c4f1b0
commit 9cffdcc801
2 changed files with 13 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tools/server/server-common.cpp
+12 -11
View File
@@ -1062,8 +1062,7 @@ json oaicompat_completion_params_parse(const json & body) {
static void handle_media(
std::vector<raw_buffer> & out_files,
const std::string & url,
const std::string & media_path,
bool accept_base64_uri) {
const std::string & media_path) {
if (!media_path.empty()) {
// should already be enforced by arg.cpp, but checking just in case
GGML_ASSERT(media_path.back() == DIRECTORY_SEPARATOR);
@@ -1104,15 +1103,17 @@ static void handle_media(
data.assign((std::istreambuf_iterator<char>(file)), std::istreambuf_iterator<char>());
out_files.push_back(data);
} else if (accept_base64_uri && string_starts_with(url, "data:")) {
// try to decode base64 image
} else if (string_starts_with(url, "data:")) {
// try to decode base64 image, video, or audio
std::vector<std::string> parts = string_split<std::string>(url, /*separator*/ ',');
if (parts.size() != 2) {
throw std::runtime_error("Invalid uri-encoded base64 value");
} else if (!string_starts_with(parts[0], "data:image/")) {
throw std::runtime_error("Invalid uri format: " + parts[0]);
throw std::invalid_argument("Invalid uri-encoded base64 value");
} else if (!string_starts_with(parts[0], "data:image/")
&& !string_starts_with(parts[0], "data:video/")
&& !string_starts_with(parts[0], "data:audio/")) {
throw std::invalid_argument("Invalid uri format: " + parts[0]);
} else if (!string_ends_with(parts[0], "base64")) {
throw std::runtime_error("uri must be base64 encoded");
throw std::invalid_argument("uri must be base64 encoded");
} else {
auto base64_data = parts[1];
auto decoded_data = base64_decode(base64_data);
@@ -1219,7 +1220,7 @@ json oaicompat_chat_params_parse(
json image_url = json_value(p, "image_url", json::object());
std::string url = json_value(image_url, "url", std::string());
handle_media(out_files, url, opt.media_path, true);
handle_media(out_files, url, opt.media_path);
p["type"] = "media_marker";
p["text"] = get_media_marker();
@@ -1234,7 +1235,7 @@ json oaicompat_chat_params_parse(
json input_audio = json_value(p, "input_audio", json::object());
std::string url = json_value(input_audio, "data",
json_value(input_audio, "url", std::string()));
handle_media(out_files, url, opt.media_path, false);
handle_media(out_files, url, opt.media_path);
p["type"] = "media_marker";
p["text"] = get_media_marker();
@@ -1248,7 +1249,7 @@ json oaicompat_chat_params_parse(
json input_video = json_value(p, "input_video", json::object());
std::string url = json_value(input_video, "data",
json_value(input_video, "url", std::string()));
handle_media(out_files, url, opt.media_path, false);
handle_media(out_files, url, opt.media_path);
p["type"] = "media_marker";
p["text"] = get_media_marker();
tools/server/tests/unit/test_vision_api.py
+1
View File
@@ -71,6 +71,7 @@ def test_v1_models_supports_multimodal_capability():
("What is this:\n", "malformed", False, None),
("What is this:\n", "https://google.com/404", False, None), # non-existent image
("What is this:\n", "https://ggml.ai", False, None), # non-image data
("What is this:\n", "data:text/html;base64,aGVsbG8=", False, None), # unsupported data uri mime
# TODO @ngxson : test with multiple images, no images and with audio
]
)