ui: add read_media tool (#25877)
* server: add read_image tool (#25875) Adds a server-tool that allows vision models to analyze server-side images. This tool is reading a single file for now: The image data is base64 encoded and passed to the UI, which decodes it, fills the <img> tag and removes the data URI before passing the tool result back to the model. * cleanup read_image tool: move magic strings to constants * Add dedicated constants file: tools/ui/src/lib/constants/read-image.ts with PREFIX_IMAGE, PREFIX_SIZE, PREFIX_MIME constants * Use ATTACHMENT_SAVED_REGEX from agentic.ts in ChatMessageToolCallBlockReadImage.svelte * Use NEWLINE constant from code.ts instead of hardcoded '\n' * Use PREFIX_SIZE in regex pattern for size parsing * Add SERVER_TOOL_READ_IMAGE_PREFIX_* constants in C++ server-tools.cpp to match the TypeScript PREFIX_* constants for consistency * server: rename read_image tool to read_media for images and audio * Rename server_tool_read_image to server_tool_read_media in C++ * Rename enum BuiltInTool.READ_IMAGE to READ_MEDIA * Rename UI constants, parser, and Svelte component files * Update display label from 'Read image' to 'Read media' * ui: consolidate audio data URI handling into shared utility * Extract getAudioInputFormat to a shared utility (was duplicated inline) * Store raw base64 in base64Data on the message object * Use base64Data to construct data URIs for audio rendering * Update agentic store to build INPUT_AUDIO parts from base64Data * server: read_media: restrict audio to wav/mp3 and minor fixes * Server get_mime_from_extension now only advertises audio/wav and audio/mpeg (the only formats the model's input_audio API accepts) * Case-insensitive extension matching (fixes .MP3, .Wav, etc.) * Unknown extensions return an error instead of a multi-MB data URI that inflates model context with garbage * Updated tool description to document supported formats * Frontend AUDIO_MIME_TO_EXTENSION trimmed to match server * fix a missing import in tools/ui/src/lib/stores/agentic.svelte.ts * server: read_media: add to --tools help text and README tool list * ui: fix indentation in ChatMessageToolCallBlockDefault.svelte * server: read_media tool: fix a cast to use the correct type * server: read_media: multiple fixes * server-tools.cpp import cctype, remove UTF-8 char, check mime before reading file * ui: add MimeTypePrefix.AUDIO and use it in agentic.svelte.ts * server: make read_media inherit from read_file and add uses_cwd * ui: fix formating issues * rm from server * move it to frontend-only tool * correct partial commit * rm unused * ui: address review from allozaur Replace the magic strings, regexes and number in the read_media parser and service with named constants. Path splitting reuses FILE_PATH_SEPARATOR_REGEX, the size header regex moves to READ_MEDIA_SIZE_REGEX derived from PREFIX_SIZE, and FILE_EXTENSION_SEPARATOR lands next to it in constants/code.ts. --------- Co-authored-by: ckrafft <ckrafft@epyc> Co-authored-by: Xuan Son Nguyen <son@huggingface.co> Co-authored-by: Pascal <admin@serveurperso.com>
This commit is contained in:
co-authored by
ckrafft
Xuan Son Nguyen
Pascal
parent
89e0aa6fd3
commit
4dd127584b
@@ -1,6 +1,7 @@
|
||||
#include "server-tools.h"
|
||||
|
||||
#include "subproc.h"
|
||||
#include "base64.hpp"
|
||||
|
||||
#include <filesystem>
|
||||
#include <fstream>
|
||||
@@ -864,6 +865,7 @@ static bool path_glob_match(const std::string & pattern, const std::string & rel
|
||||
//
|
||||
|
||||
static constexpr size_t SERVER_TOOL_READ_FILE_MAX_SIZE = 16 * 1024; // 16 KB
|
||||
static constexpr size_t SERVER_TOOL_READ_FILE_MAX_SIZE_BASE64 = 32 * 1024 * 1024; // 32 MB
|
||||
|
||||
struct server_tool_read_file : server_tool {
|
||||
server_tool_read_file() {
|
||||
@@ -899,6 +901,8 @@ struct server_tool_read_file : server_tool {
|
||||
int start_line = json_value(params, "start_line", 1);
|
||||
int end_line = json_value(params, "end_line", -1); // -1 = no limit
|
||||
bool append_loc = json_value(params, "append_loc", false);
|
||||
// comes from the x-resp-type header, the model cannot ask for it
|
||||
bool as_base64 = json_value(params, "resp_type", std::string()) == "base64";
|
||||
|
||||
auto io = make_tools_io(params);
|
||||
|
||||
@@ -906,6 +910,23 @@ struct server_tool_read_file : server_tool {
|
||||
if (!io->file_size(path, file_size)) {
|
||||
return {{"error", "cannot stat file: " + path}};
|
||||
}
|
||||
|
||||
if (as_base64) {
|
||||
if (file_size > SERVER_TOOL_READ_FILE_MAX_SIZE_BASE64) {
|
||||
return {{"error", string_format(
|
||||
"file too large (%zu bytes, max %zu)",
|
||||
(size_t)file_size, SERVER_TOOL_READ_FILE_MAX_SIZE_BASE64)}};
|
||||
}
|
||||
std::string content;
|
||||
if (!io->read_file(path, content)) {
|
||||
return {{"error", "failed to open file: " + path}};
|
||||
}
|
||||
return {
|
||||
{"base64", base64::encode(content.data(), content.size())},
|
||||
{"size_bytes", (size_t) content.size()},
|
||||
};
|
||||
}
|
||||
|
||||
if (file_size > SERVER_TOOL_READ_FILE_MAX_SIZE && end_line == -1) {
|
||||
return {{"error", string_format(
|
||||
"file too large (%zu bytes, max %zu). Use start_line/end_line to read a portion.",
|
||||
@@ -2135,6 +2156,15 @@ void server_tools::setup(const std::vector<std::string> & enabled_tools,
|
||||
params["runtime"] = runtime->spec();
|
||||
}
|
||||
|
||||
// x-resp-type header is only used by read_file for now
|
||||
if (params.contains("resp_type")) {
|
||||
params.erase("resp_type");
|
||||
}
|
||||
auto resp_type = get_header(req.headers, "x-resp-type");
|
||||
if (!resp_type.empty()) {
|
||||
params["resp_type"] = resp_type;
|
||||
}
|
||||
|
||||
server_tool & tool = find_tool(tools, tool_name, stream);
|
||||
|
||||
if (stream) {
|
||||
|
||||
Reference in New Issue
Block a user