ui: add read_media tool (#25877)
* server: add read_image tool (#25875) Adds a server-tool that allows vision models to analyze server-side images. This tool is reading a single file for now: The image data is base64 encoded and passed to the UI, which decodes it, fills the <img> tag and removes the data URI before passing the tool result back to the model. * cleanup read_image tool: move magic strings to constants * Add dedicated constants file: tools/ui/src/lib/constants/read-image.ts with PREFIX_IMAGE, PREFIX_SIZE, PREFIX_MIME constants * Use ATTACHMENT_SAVED_REGEX from agentic.ts in ChatMessageToolCallBlockReadImage.svelte * Use NEWLINE constant from code.ts instead of hardcoded '\n' * Use PREFIX_SIZE in regex pattern for size parsing * Add SERVER_TOOL_READ_IMAGE_PREFIX_* constants in C++ server-tools.cpp to match the TypeScript PREFIX_* constants for consistency * server: rename read_image tool to read_media for images and audio * Rename server_tool_read_image to server_tool_read_media in C++ * Rename enum BuiltInTool.READ_IMAGE to READ_MEDIA * Rename UI constants, parser, and Svelte component files * Update display label from 'Read image' to 'Read media' * ui: consolidate audio data URI handling into shared utility * Extract getAudioInputFormat to a shared utility (was duplicated inline) * Store raw base64 in base64Data on the message object * Use base64Data to construct data URIs for audio rendering * Update agentic store to build INPUT_AUDIO parts from base64Data * server: read_media: restrict audio to wav/mp3 and minor fixes * Server get_mime_from_extension now only advertises audio/wav and audio/mpeg (the only formats the model's input_audio API accepts) * Case-insensitive extension matching (fixes .MP3, .Wav, etc.) * Unknown extensions return an error instead of a multi-MB data URI that inflates model context with garbage * Updated tool description to document supported formats * Frontend AUDIO_MIME_TO_EXTENSION trimmed to match server * fix a missing import in tools/ui/src/lib/stores/agentic.svelte.ts * server: read_media: add to --tools help text and README tool list * ui: fix indentation in ChatMessageToolCallBlockDefault.svelte * server: read_media tool: fix a cast to use the correct type * server: read_media: multiple fixes * server-tools.cpp import cctype, remove UTF-8 char, check mime before reading file * ui: add MimeTypePrefix.AUDIO and use it in agentic.svelte.ts * server: make read_media inherit from read_file and add uses_cwd * ui: fix formating issues * rm from server * move it to frontend-only tool * correct partial commit * rm unused * ui: address review from allozaur Replace the magic strings, regexes and number in the read_media parser and service with named constants. Path splitting reuses FILE_PATH_SEPARATOR_REGEX, the size header regex moves to READ_MEDIA_SIZE_REGEX derived from PREFIX_SIZE, and FILE_EXTENSION_SEPARATOR lands next to it in constants/code.ts. --------- Co-authored-by: ckrafft <ckrafft@epyc> Co-authored-by: Xuan Son Nguyen <son@huggingface.co> Co-authored-by: Pascal <admin@serveurperso.com>
This commit is contained in:
parabelboi
co-authored by
ckrafft
Xuan Son Nguyen
Pascal
ckrafft
Xuan Son Nguyen
Pascal
parent
89e0aa6fd3
commit
4dd127584b
@@ -10,6 +10,7 @@
|
||||
import {
|
||||
Braces,
|
||||
Clock,
|
||||
Eye,
|
||||
FilePen,
|
||||
FilePlus,
|
||||
FileSearch,
|
||||
@@ -47,6 +48,7 @@ export const BUILTIN_TOOL_UI: Readonly<Record<BuiltInTool, BuiltinToolUiEntry>>
|
||||
source: ToolSource.BUILTIN
|
||||
},
|
||||
[BuiltInTool.READ_FILE]: { icon: FileText, label: 'Read file', source: ToolSource.BUILTIN },
|
||||
[BuiltInTool.READ_MEDIA]: { icon: Eye, label: 'Read media', source: ToolSource.FRONTEND },
|
||||
[BuiltInTool.RUN_JAVASCRIPT]: {
|
||||
icon: Braces,
|
||||
label: 'Run JavaScript',
|
||||
|
||||
@@ -18,6 +18,9 @@ export const TRIM_TRAILING_PADDING_REGEX = /(?:\n[ \t]*)+$/;
|
||||
// `C:\foo\bar.txt`. Used wherever a parameter accepts a user-supplied path.
|
||||
export const FILE_PATH_SEPARATOR_REGEX = /[\\/]/;
|
||||
|
||||
// Separates a file name from its extension, e.g. the '.' in `cover.png`.
|
||||
export const FILE_EXTENSION_SEPARATOR = '.';
|
||||
|
||||
// Matches the `text:` prefix that file-type identifiers use to denote a
|
||||
// plain-text language (e.g. `text:typescript`). Used by tool-call renderers
|
||||
// to recover the underlying highlight.js language.
|
||||
|
||||
@@ -49,6 +49,7 @@ export * from './sse';
|
||||
export * from './precision';
|
||||
export * from './processing-info';
|
||||
export * from './pwa';
|
||||
export * from './read-media';
|
||||
export * from './routes';
|
||||
export * from './sandbox';
|
||||
export * from './settings-keys';
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import { MimeTypeImage } from '$lib/enums';
|
||||
import { MimeTypeAudio, MimeTypeImage } from '$lib/enums';
|
||||
|
||||
// File extension patterns for resource type detection
|
||||
export const IMAGE_FILE_EXTENSION_REGEX = /\.(png|jpg|jpeg|gif|svg|webp)$/i;
|
||||
@@ -27,6 +27,9 @@ export const MCP_RESOURCE_ATTACHMENT_ID_PREFIX = 'res';
|
||||
// Default file extension for unknown image types
|
||||
export const DEFAULT_IMAGE_EXTENSION = 'img';
|
||||
|
||||
// Default file extension for unknown audio types
|
||||
export const DEFAULT_AUDIO_EXTENSION = 'mp3';
|
||||
|
||||
// Default filename for resource content downloads
|
||||
export const DEFAULT_RESOURCE_FILENAME = 'resource.txt';
|
||||
|
||||
@@ -53,3 +56,18 @@ export const IMAGE_MIME_TO_EXTENSION: Record<string, string> = {
|
||||
[MimeTypeImage.PNG]: 'png',
|
||||
[MimeTypeImage.WEBP]: 'webp'
|
||||
} as const;
|
||||
|
||||
/**
|
||||
* Mapping from audio MIME types to file extensions.
|
||||
* Used for generating attachment filenames from MIME types.
|
||||
*/
|
||||
export const AUDIO_MIME_TO_EXTENSION: Record<string, string> = {
|
||||
[MimeTypeAudio.MP3]: 'mp3',
|
||||
[MimeTypeAudio.MP3_MPEG]: 'mp3',
|
||||
[MimeTypeAudio.VND_WAVE]: 'wav',
|
||||
[MimeTypeAudio.WAV]: 'wav',
|
||||
[MimeTypeAudio.WAVE]: 'wav',
|
||||
[MimeTypeAudio.X_PN_WAV]: 'wav',
|
||||
[MimeTypeAudio.X_WAV]: 'wav',
|
||||
[MimeTypeAudio.X_WAVE]: 'wav'
|
||||
} as const;
|
||||
|
||||
@@ -0,0 +1,66 @@
|
||||
import {
|
||||
BuiltInTool,
|
||||
JsonSchemaType,
|
||||
MimeTypeAudio,
|
||||
MimeTypeImage,
|
||||
ToolCallType
|
||||
} from '$lib/enums';
|
||||
import type { OpenAIToolDefinition } from '$lib/types';
|
||||
|
||||
export const READ_MEDIA_TOOL_NAME = BuiltInTool.READ_MEDIA;
|
||||
|
||||
// header lines of the tool result, parsed back by the read_media renderer
|
||||
export const PREFIX_FILE = 'File: ';
|
||||
export const PREFIX_SIZE = 'Size: ';
|
||||
export const PREFIX_MIME = 'MIME: ';
|
||||
|
||||
/** Byte count of the `Size: ` header line, e.g. `Size: 12345 bytes` -> capture group 1 is `12345`. */
|
||||
export const READ_MEDIA_SIZE_REGEX = new RegExp(`^${PREFIX_SIZE}\\s*(\\d+)\\s*bytes`);
|
||||
|
||||
/** Image extensions the tool accepts. The server decodes images with stb_image, which has no webp or tiff. */
|
||||
export const READ_MEDIA_IMAGE_MIME: Record<string, string> = {
|
||||
gif: MimeTypeImage.GIF,
|
||||
jpeg: MimeTypeImage.JPEG,
|
||||
jpg: MimeTypeImage.JPEG,
|
||||
png: MimeTypeImage.PNG
|
||||
} as const;
|
||||
|
||||
/** Audio extensions the tool accepts. The `input_audio` API only takes wav and mp3. */
|
||||
export const READ_MEDIA_AUDIO_MIME: Record<string, string> = {
|
||||
mp3: MimeTypeAudio.MP3_MPEG,
|
||||
wav: MimeTypeAudio.WAV
|
||||
} as const;
|
||||
|
||||
/**
|
||||
* Build the read_media tool definition for the modalities the active model has.
|
||||
* At least one of the two flags must be true, otherwise the tool is not offered
|
||||
* at all - a model that cannot see or hear has nothing to do with the bytes.
|
||||
*/
|
||||
export function buildReadMediaToolDefinition(
|
||||
supportsVision: boolean,
|
||||
supportsAudio: boolean
|
||||
): OpenAIToolDefinition {
|
||||
const kinds: string[] = [];
|
||||
|
||||
if (supportsVision) kinds.push(`images (${Object.keys(READ_MEDIA_IMAGE_MIME).join(', ')})`);
|
||||
|
||||
if (supportsAudio) kinds.push(`audio (${Object.keys(READ_MEDIA_AUDIO_MIME).join(', ')})`);
|
||||
|
||||
return {
|
||||
function: {
|
||||
description: `Read a media file and attach it to the conversation so it can be perceived directly. Supports ${kinds.join(' and ')}.`,
|
||||
name: READ_MEDIA_TOOL_NAME,
|
||||
parameters: {
|
||||
properties: {
|
||||
path: {
|
||||
description: 'Path to the media file',
|
||||
type: JsonSchemaType.STRING
|
||||
}
|
||||
},
|
||||
required: ['path'],
|
||||
type: JsonSchemaType.OBJECT
|
||||
}
|
||||
},
|
||||
type: ToolCallType.FUNCTION
|
||||
};
|
||||
}
|
||||
@@ -3,6 +3,12 @@ import { ToolSource } from '$lib/enums/tools.enums';
|
||||
/** HTTP header carrying the working directory a tool call runs in. The server resolves relative paths against it; the model cannot override it. */
|
||||
export const X_TOOL_CWD_HEADER = 'x-tool-cwd';
|
||||
|
||||
/** HTTP header asking the server to encode a tool's output differently, e.g. read_file returning base64. Not a tool parameter, so it stays out of the definition the model sees. */
|
||||
export const X_RESP_TYPE_HEADER = 'x-resp-type';
|
||||
|
||||
/** `X_RESP_TYPE_HEADER` value that makes read_file return the raw bytes as base64 instead of text. */
|
||||
export const RESP_TYPE_BASE64 = 'base64';
|
||||
|
||||
export const TOOL_GROUP_LABELS = {
|
||||
[ToolSource.BUILTIN]: 'Built-in',
|
||||
[ToolSource.CUSTOM]: 'JSON Schema',
|
||||
|
||||
Reference in New Issue
Block a user