ui: add read_media tool (#25877)

* server: add read_image tool (#25875)

Adds a server-tool that allows vision models to analyze server-side images.
This tool is reading a single file for now:
The image data is base64 encoded and passed to the UI, which
decodes it, fills the <img> tag and removes the data URI before
passing the tool result back to the model.

* cleanup read_image tool: move magic strings to constants

* Add dedicated constants file: tools/ui/src/lib/constants/read-image.ts
  with PREFIX_IMAGE, PREFIX_SIZE, PREFIX_MIME constants
* Use ATTACHMENT_SAVED_REGEX from agentic.ts in ChatMessageToolCallBlockReadImage.svelte
* Use NEWLINE constant from code.ts instead of hardcoded '\n'
* Use PREFIX_SIZE in regex pattern for size parsing
* Add SERVER_TOOL_READ_IMAGE_PREFIX_* constants in C++ server-tools.cpp
  to match the TypeScript PREFIX_* constants for consistency

* server: rename read_image tool to read_media for images and audio

* Rename server_tool_read_image to server_tool_read_media in C++
* Rename enum BuiltInTool.READ_IMAGE to READ_MEDIA
* Rename UI constants, parser, and Svelte component files
* Update display label from 'Read image' to 'Read media'

* ui: consolidate audio data URI handling into shared utility

* Extract getAudioInputFormat to a shared utility (was duplicated inline)
* Store raw base64 in base64Data on the message object
* Use base64Data to construct data URIs for audio rendering
* Update agentic store to build INPUT_AUDIO parts from base64Data

* server: read_media: restrict audio to wav/mp3 and minor fixes

* Server get_mime_from_extension now only advertises audio/wav and
  audio/mpeg (the only formats the model's input_audio API accepts)
* Case-insensitive extension matching (fixes .MP3, .Wav, etc.)
* Unknown extensions return an error instead of a multi-MB data URI
  that inflates model context with garbage
* Updated tool description to document supported formats
* Frontend AUDIO_MIME_TO_EXTENSION trimmed to match server
* fix a missing import in tools/ui/src/lib/stores/agentic.svelte.ts

* server: read_media: add to --tools help text and README tool list

* ui: fix indentation in ChatMessageToolCallBlockDefault.svelte

* server: read_media tool: fix a cast to use the correct type

* server: read_media: multiple fixes

* server-tools.cpp import cctype, remove UTF-8 char, check mime before reading file
* ui: add MimeTypePrefix.AUDIO and use it in agentic.svelte.ts

* server: make read_media inherit from read_file and add uses_cwd

* ui: fix formating issues

* rm from server

* move it to frontend-only tool

* correct partial commit

* rm unused

* ui: address review from allozaur

Replace the magic strings, regexes and number in the read_media parser
and service with named constants. Path splitting reuses
FILE_PATH_SEPARATOR_REGEX, the size header regex moves to
READ_MEDIA_SIZE_REGEX derived from PREFIX_SIZE, and
FILE_EXTENSION_SEPARATOR lands next to it in constants/code.ts.

---------

Co-authored-by: ckrafft <ckrafft@epyc>
Co-authored-by: Xuan Son Nguyen <son@huggingface.co>
Co-authored-by: Pascal <admin@serveurperso.com>

tools/ui/src/lib/services/chat.service.ts

@@ -1,4 +1,5 @@
 import { settingsStore } from '../stores/settings.svelte';
+import { getAudioInputFormat } from '../utils/audio-format';
 import { capImageDataURLSize } from '../utils/cap-img-size';
 import {
 	API_CHAT,
@@ -20,18 +21,12 @@ import {
 import {
 	AttachmentType,
 	ContentPartType,
-	FileTypeAudio,
 	MessageRole,
-	MimeTypeAudio,
 	ReasoningFormat,
 	StreamConnectionState
 } from '$lib/enums';
 import { modelsStore } from '$lib/stores/models.svelte';
-import type {
-	AudioInputFormat,
-	DatabaseMessageExtraMcpPrompt,
-	DatabaseMessageExtraMcpResource
-} from '$lib/types';
+import type { DatabaseMessageExtraMcpPrompt, DatabaseMessageExtraMcpResource } from '$lib/types';
 import type {
 	ApiChatCompletionToolCall,
 	ApiChatMessageContentPart,
@@ -43,23 +38,6 @@ import { getAuthHeaders, getJsonHeaders } from '$lib/utils/api-headers';
 import { formatAttachmentText } from '$lib/utils/formatters';
 import { streamIdentity } from '$lib/utils/stream-identity';
 
-function getAudioInputFormat(mimeType: string): AudioInputFormat {
-	const normalizedMimeType = mimeType.trim().toLowerCase();
-
-	if (
-		normalizedMimeType === MimeTypeAudio.WAV ||
-		normalizedMimeType === MimeTypeAudio.WAVE ||
-		normalizedMimeType === MimeTypeAudio.X_WAV ||
-		normalizedMimeType === MimeTypeAudio.X_WAVE ||
-		normalizedMimeType === MimeTypeAudio.VND_WAVE ||
-		normalizedMimeType === MimeTypeAudio.X_PN_WAV
-	) {
-		return FileTypeAudio.WAV;
-	}
-
-	return FileTypeAudio.MP3;
-}
-
 interface ResumableStreamState {
 	bytesReceived: number;
 	updatedAt: number;

tools/ui/src/lib/services/read-media.service.ts

@@ -0,0 +1,112 @@
+import { ToolsService } from './tools.service';
+import {
+	FILE_EXTENSION_SEPARATOR,
+	FILE_PATH_SEPARATOR_REGEX,
+	NEWLINE,
+	PREFIX_FILE,
+	PREFIX_MIME,
+	PREFIX_SIZE,
+	READ_MEDIA_AUDIO_MIME,
+	READ_MEDIA_IMAGE_MIME,
+	RESP_TYPE_BASE64
+} from '$lib/constants';
+import { BuiltInTool, ToolResponseField } from '$lib/enums';
+import type { ToolExecutionResult } from '$lib/types';
+
+/** Modalities of the model the tool call runs for. */
+export interface ReadMediaCapabilities {
+	audio: boolean;
+	vision: boolean;
+}
+
+/** Lowercase extension of a path, without the dot. Empty when the file name has none. */
+function fileExtension(path: string): string {
+	const name = path.split(FILE_PATH_SEPARATOR_REGEX).pop() ?? '';
+	const dot = name.lastIndexOf(FILE_EXTENSION_SEPARATOR);
+
+	return dot > 0 ? name.slice(dot + 1).toLowerCase() : '';
+}
+
+/**
+ * **ReadMediaService** - frontend executor for the `read_media` tool
+ *
+ * The tool is synthetic: no such tool exists on the server. It reads the file
+ * through the built-in `read_file` tool with the `base64` response type, then
+ * turns the bytes into a data URI line. The agentic store lifts that line into
+ * an image or audio attachment on the tool result message, which is what makes
+ * the model perceive the file instead of reading a wall of base64.
+ *
+ * Living in the frontend is what lets it exist only for models that can
+ * actually use the result - the server has no idea which model is selected.
+ *
+ * @see buildReadMediaToolDefinition in constants/read-media.ts - tool schema sent to the LLM
+ * @see agenticStore in stores/agentic.svelte.ts - tool dispatch and attachment extraction
+ */
+export class ReadMediaService {
+	static async executeTool(
+		params: Record<string, unknown>,
+		capabilities: ReadMediaCapabilities,
+		signal?: AbortSignal,
+		cwd?: string
+	): Promise<ToolExecutionResult> {
+		const path = typeof params.path === 'string' ? params.path : '';
+
+		if (!path) {
+			return { content: 'Error: missing "path" argument.', isError: true };
+		}
+
+		const extension = fileExtension(path);
+		const imageMime = READ_MEDIA_IMAGE_MIME[extension];
+		const audioMime = READ_MEDIA_AUDIO_MIME[extension];
+
+		let resolvedMime: string | undefined;
+
+		if (imageMime && capabilities.vision) resolvedMime = imageMime;
+		else if (audioMime && capabilities.audio) resolvedMime = audioMime;
+
+		if (!resolvedMime) {
+			const supported = [
+				...(capabilities.vision ? Object.keys(READ_MEDIA_IMAGE_MIME) : []),
+				...(capabilities.audio ? Object.keys(READ_MEDIA_AUDIO_MIME) : [])
+			];
+			// an unreadable-by-this-model file is a dead end, so say why instead of failing silently
+			const reason =
+				imageMime || audioMime
+					? `the current model cannot perceive ".${extension}" files`
+					: `".${extension}" is not a supported media type`;
+
+			return {
+				content: `Error: ${reason}. Supported: ${supported.join(', ')}.`,
+				isError: true
+			};
+		}
+
+		const raw = await ToolsService.executeToolRaw(
+			BuiltInTool.READ_FILE,
+			{ path },
+			signal,
+			cwd,
+			RESP_TYPE_BASE64
+		);
+
+		if (ToolResponseField.ERROR in raw) {
+			return { content: String(raw[ToolResponseField.ERROR]), isError: true };
+		}
+
+		const base64 = typeof raw.base64 === 'string' ? raw.base64 : '';
+
+		if (!base64) {
+			return { content: `Error: no data returned for ${path}.`, isError: true };
+		}
+
+		const sizeBytes = typeof raw.size_bytes === 'number' ? raw.size_bytes : 0;
+		const content = [
+			`${PREFIX_FILE}${path}`,
+			`${PREFIX_SIZE}${sizeBytes} bytes`,
+			`${PREFIX_MIME}${resolvedMime}`,
+			`data:${resolvedMime};base64,${base64}`
+		].join(NEWLINE);
+
+		return { content, isError: false };
+	}
+}

tools/ui/src/lib/services/tools.service.ts

@@ -1,5 +1,5 @@
 import { base } from '$app/paths';
-import { API_TOOLS, X_TOOL_CWD_HEADER } from '$lib/constants';
+import { API_TOOLS, X_RESP_TYPE_HEADER, X_TOOL_CWD_HEADER } from '$lib/constants';
 import { ToolResponseField } from '$lib/enums';
 import type { ServerBuiltinToolInfo, ToolExecutionResult } from '$lib/types';
 import { apiFetch } from '$lib/utils';
@@ -51,16 +51,26 @@ export class ToolsService {
 	 * Execute a built-in tool and return the raw JSON response. Unlike
 	 * executeTool, this preserves structured fields (e.g. file_glob_search's
 	 * `entries` and `base`) that the flattened ToolExecutionResult drops.
+	 *
+	 * @param respType - sent as the x-resp-type request header. Only read_file
+	 * honors it, with `base64` to get the raw bytes instead of decoded text.
 	 */
 	static async executeToolRaw(
 		toolName: string,
 		params: Record<string, unknown>,
 		signal?: AbortSignal,
-		cwd?: string
+		cwd?: string,
+		respType?: string
 	): Promise<Record<string, unknown>> {
+		const headers: Record<string, string> = {};
+
+		if (cwd) headers[X_TOOL_CWD_HEADER] = cwd;
+
+		if (respType) headers[X_RESP_TYPE_HEADER] = respType;
+
 		return apiFetch<Record<string, unknown>>(API_TOOLS.EXECUTE, {
 			body: JSON.stringify({ params, tool: toolName }),
-			headers: cwd ? { [X_TOOL_CWD_HEADER]: cwd } : undefined,
+			headers: Object.keys(headers).length > 0 ? headers : undefined,
 			method: 'POST',
 			signal
 		});