ui: add read_media tool (#25877)

* server: add read_image tool (#25875) Adds a server-tool that allows vision models to analyze server-side images. This tool is reading a single file for now: The image data is base64 encoded and passed to the UI, which decodes it, fills the <img> tag and removes the data URI before passing the tool result back to the model. * cleanup read_image tool: move magic strings to constants * Add dedicated constants file: tools/ui/src/lib/constants/read-image.ts with PREFIX_IMAGE, PREFIX_SIZE, PREFIX_MIME constants * Use ATTACHMENT_SAVED_REGEX from agentic.ts in ChatMessageToolCallBlockReadImage.svelte * Use NEWLINE constant from code.ts instead of hardcoded '\n' * Use PREFIX_SIZE in regex pattern for size parsing * Add SERVER_TOOL_READ_IMAGE_PREFIX_* constants in C++ server-tools.cpp to match the TypeScript PREFIX_* constants for consistency * server: rename read_image tool to read_media for images and audio * Rename server_tool_read_image to server_tool_read_media in C++ * Rename enum BuiltInTool.READ_IMAGE to READ_MEDIA * Rename UI constants, parser, and Svelte component files * Update display label from 'Read image' to 'Read media' * ui: consolidate audio data URI handling into shared utility * Extract getAudioInputFormat to a shared utility (was duplicated inline) * Store raw base64 in base64Data on the message object * Use base64Data to construct data URIs for audio rendering * Update agentic store to build INPUT_AUDIO parts from base64Data * server: read_media: restrict audio to wav/mp3 and minor fixes * Server get_mime_from_extension now only advertises audio/wav and audio/mpeg (the only formats the model's input_audio API accepts) * Case-insensitive extension matching (fixes .MP3, .Wav, etc.) * Unknown extensions return an error instead of a multi-MB data URI that inflates model context with garbage * Updated tool description to document supported formats * Frontend AUDIO_MIME_TO_EXTENSION trimmed to match server * fix a missing import in tools/ui/src/lib/stores/agentic.svelte.ts * server: read_media: add to --tools help text and README tool list * ui: fix indentation in ChatMessageToolCallBlockDefault.svelte * server: read_media tool: fix a cast to use the correct type * server: read_media: multiple fixes * server-tools.cpp import cctype, remove UTF-8 char, check mime before reading file * ui: add MimeTypePrefix.AUDIO and use it in agentic.svelte.ts * server: make read_media inherit from read_file and add uses_cwd * ui: fix formating issues * rm from server * move it to frontend-only tool * correct partial commit * rm unused * ui: address review from allozaur Replace the magic strings, regexes and number in the read_media parser and service with named constants. Path splitting reuses FILE_PATH_SEPARATOR_REGEX, the size header regex moves to READ_MEDIA_SIZE_REGEX derived from PREFIX_SIZE, and FILE_EXTENSION_SEPARATOR lands next to it in constants/code.ts. --------- Co-authored-by: ckrafft <ckrafft@epyc> Co-authored-by: Xuan Son Nguyen <son@huggingface.co> Co-authored-by: Pascal <admin@serveurperso.com>
2026-08-12 12:03:32 +02:00
parent 89e0aa6fd3
commit 4dd127584b
25 changed files with 579 additions and 73 deletions
@@ -7,6 +7,7 @@
 	import ChatMessageToolCallBlockGetInfo from './ChatMessageToolCallBlockGetInfo.svelte';
 	import ChatMessageToolCallBlockGrepSearch from './ChatMessageToolCallBlockGrepSearch.svelte';
 	import ChatMessageToolCallBlockReadFile from './ChatMessageToolCallBlockReadFile.svelte';
+	import ChatMessageToolCallBlockReadMedia from './ChatMessageToolCallBlockReadMedia.svelte';
 	import ChatMessageToolCallBlockRunJavascript from './ChatMessageToolCallBlockRunJavascript.svelte';
 	import ChatMessageToolCallBlockSearchResults from './ChatMessageToolCallBlockSearchResults.svelte';
 	import ChatMessageToolCallBlockWriteFile from './ChatMessageToolCallBlockWriteFile.svelte';
@@ -45,6 +46,8 @@
 	<ChatMessageToolCallBlockGetInfo {section} {isStreaming} />
 {:else if section.toolName === BuiltInTool.READ_FILE}
 	<ChatMessageToolCallBlockReadFile {section} {open} {isStreaming} {onToggle} />
+{:else if section.toolName === BuiltInTool.READ_MEDIA}
+	<ChatMessageToolCallBlockReadMedia {section} {open} {isStreaming} {onToggle} />
 {:else if section.toolName === BuiltInTool.EDIT_FILE}
 	<ChatMessageToolCallBlockEditFile {section} {open} {isStreaming} {onToggle} />
 {:else if section.toolName === BuiltInTool.WRITE_FILE}
@@ -8,14 +8,16 @@
 	import { MarkdownContent, SyntaxHighlightedCode } from '$lib/components/app';
 	import { MAX_HEIGHT_CODE_BLOCK } from '$lib/constants';
 	import { getBuiltinToolUi } from '$lib/constants/built-in-tools';
-	import { FileTypeText, ToolResultKind } from '$lib/enums';
+	import { AttachmentType, FileTypeText, MimeTypeAudio, ToolResultKind } from '$lib/enums';
 	import type { DatabaseMessageExtra } from '$lib/types';
 	import {
 		type AgenticSection,
 		classifyToolResult,
 		formatJsonPretty,
-		parseToolResultWithImages
+		parseToolResultWithMedia,
+		type ToolResultLine
 	} from '$lib/utils';
+	import { createBase64DataUrl } from '$lib/utils/data-url';

 	interface Props {
 		section: AgenticSection;
@@ -29,8 +31,8 @@

 	const title = $derived(getBuiltinToolUi(section.toolName)?.label ?? section.toolName ?? '');
 	const outputKind = $derived(classifyToolResult(section.toolResult));
-	const parsedLines = $derived(
-		section.toolResult ? parseToolResultWithImages(section.toolResult, attachments) : []
+	const parsedLines: ToolResultLine[] = $derived(
+		section.toolResult ? parseToolResultWithMedia(section.toolResult, attachments) : []
 	);
 </script>

@@ -103,13 +105,26 @@
 							<div class="font-mono text-[11px] leading-relaxed whitespace-pre-wrap">
 								{line.text}
 							</div>
-							{#if line.image}
-								<img
-									src={line.image.base64Url}
-									alt={line.image.name}
-									class="mt-2 mb-2 h-auto max-w-full rounded-lg"
-									loading="lazy"
-								/>
+							{#if line.media}
+								{#if line.media.type === AttachmentType.AUDIO}
+									{@const audioMimeType = line.media.mimeType ?? MimeTypeAudio.MP3_MPEG}
+									<div class="mt-2 mb-2">
+										<audio controls class="w-full rounded-lg">
+											<source
+												src={createBase64DataUrl(audioMimeType, line.media.base64Data)}
+												type={audioMimeType}
+											/>
+											Your browser does not support the audio element.
+										</audio>
+									</div>
+								{:else}
+									<img
+										src={line.media.base64Url}
+										alt={line.media.name}
+										class="mt-2 mb-2 h-auto max-w-full rounded-lg"
+										loading="lazy"
+									/>
+								{/if}
 							{/if}
 						{/each}
 					</div>
@@ -23,7 +23,7 @@
 		isExitCodeSummaryLine,
 		parseExecShellCommandError,
 		parseExecShellCommandExitStatus,
-		parseToolResultWithImages,
+		parseToolResultWithMedia,
 		type ToolResultLine
 	} from '$lib/utils';

@@ -53,7 +53,7 @@
 	);

 	const parsedLines: ToolResultLine[] = $derived(
-		section.toolResult ? parseToolResultWithImages(section.toolResult, attachments) : []
+		section.toolResult ? parseToolResultWithMedia(section.toolResult, attachments) : []
 	);

 	// Drop the trailing "[exit code: N]" line - rendered as a colored
@@ -223,10 +223,10 @@
 			>
 				{#each outputLines as line, i (i)}
 					<div class="font-mono text-[11px] leading-relaxed whitespace-pre-wrap">{line.text}</div>
-					{#if line.image}
+					{#if line.media}
 						<img
-							src={line.image.base64Url}
-							alt={line.image.name}
+							src={line.media.base64Url}
+							alt={line.media.name}
 							class="mt-2 mb-2 h-auto max-w-full rounded-lg"
 							loading="lazy"
 						/>
@@ -0,0 +1,99 @@
+<script lang="ts">
+	import { parseReadMediaMeta } from './parsers/read-media';
+	import ToolCallBlock from './ToolCallBlock.svelte';
+	import { ATTACHMENT_SAVED_REGEX } from '$lib/constants/agentic';
+	import { AttachmentType, MimeTypeAudio } from '$lib/enums';
+	import type { DatabaseMessageExtraAudioFile, DatabaseMessageExtraImageFile } from '$lib/types';
+	import { type AgenticSection } from '$lib/utils';
+	import { createBase64DataUrl } from '$lib/utils/data-url';
+
+	interface Props {
+		section: AgenticSection;
+		open: boolean;
+		isStreaming: boolean;
+		onToggle?: () => void;
+	}
+
+	let { isStreaming, onToggle, open, section }: Props = $props();
+
+	const readMediaMeta = $derived(parseReadMediaMeta(section));
+
+	// extractBase64Attachments swapped the data URI line for [Attachment saved: name]
+	// and moved the bytes to the message extras, so the name is the only link back
+	const mediaAttachment = $derived.by(() => {
+		const extras = section.toolResultExtras;
+
+		if (!extras || extras.length === 0) return null;
+
+		const match = section.toolResult?.match(ATTACHMENT_SAVED_REGEX);
+
+		if (!match) return null;
+
+		const attachmentName = match[1];
+
+		return (
+			extras.find(
+				(e): e is DatabaseMessageExtraImageFile | DatabaseMessageExtraAudioFile =>
+					(e.type === AttachmentType.IMAGE || e.type === AttachmentType.AUDIO) &&
+					e.name === attachmentName
+			) ?? null
+		);
+	});
+
+	const audioMimeType = $derived(readMediaMeta?.mimeType ?? MimeTypeAudio.MP3_MPEG);
+</script>
+
+<ToolCallBlock {section} {open} {isStreaming} meta={readMediaMeta} {onToggle}>
+	{#snippet titleSnippet()}
+		<span class="text-muted-foreground">Read media </span>
+		<span class="font-mono">{readMediaMeta?.fileName}</span>
+	{/snippet}
+
+	{#snippet children(_meta, _ctx)}
+		{#if section.toolResult}
+			{#if !mediaAttachment}
+				<div class="rounded bg-muted/20 p-2 text-xs text-muted-foreground/70 italic">
+					Media attachment not found in message extras
+				</div>
+			{:else if mediaAttachment.type === AttachmentType.AUDIO}
+				<div class="mt-2">
+					<audio controls class="w-full rounded-lg">
+						<source
+							src={createBase64DataUrl(audioMimeType, mediaAttachment.base64Data)}
+							type={audioMimeType}
+						/>
+						Your browser does not support the audio element.
+					</audio>
+				</div>
+			{:else}
+				<div class="mt-2">
+					<img
+						src={mediaAttachment.base64Url}
+						alt={readMediaMeta?.fileName ?? 'media'}
+						class="max-h-[60vh] max-w-full rounded-lg object-contain shadow-lg"
+						loading="lazy"
+					/>
+				</div>
+			{/if}
+
+			{#if readMediaMeta?.sizeBytes || readMediaMeta?.mimeType}
+				<div class="mt-2 flex gap-4 text-xs text-muted-foreground">
+					{#if readMediaMeta?.sizeBytes}
+						<span>Size: {readMediaMeta.sizeBytes} bytes</span>
+					{/if}
+					{#if readMediaMeta?.mimeType}
+						<span>MIME: {readMediaMeta.mimeType}</span>
+					{/if}
+				</div>
+			{/if}
+
+			{#if readMediaMeta?.path}
+				<div class="mt-1 font-mono text-xs text-muted-foreground/60">{readMediaMeta.path}</div>
+			{/if}
+		{:else}
+			<div class="rounded bg-muted/20 p-2 text-xs text-muted-foreground/70 italic">
+				Waiting for media data...
+			</div>
+		{/if}
+	{/snippet}
+</ToolCallBlock>
@@ -0,0 +1,56 @@
+import { FILE_PATH_SEPARATOR_REGEX, NEWLINE } from '$lib/constants/code';
+import {
+	PREFIX_FILE,
+	PREFIX_MIME,
+	PREFIX_SIZE,
+	READ_MEDIA_SIZE_REGEX
+} from '$lib/constants/read-media';
+import type { AgenticSection } from '$lib/utils';
+
+export interface ReadMediaMeta {
+	fileName: string;
+	path: string;
+	sizeBytes?: number;
+	mimeType?: string;
+}
+
+/**
+ * Parse read_media tool result to extract metadata.
+ * Expected format (after extractBase64Attachments processing):
+ *   File: /path/to/file.png
+ *   Size: 12345 bytes
+ *   MIME: image/png
+ *   [Attachment saved: mcp-attachment-xxx.png]
+ *
+ * The data URI line is replaced by the attachment marker by
+ * agenticStore.extractBase64Attachments before storage.
+ */
+export function parseReadMediaMeta(section: AgenticSection): ReadMediaMeta | null {
+	if (!section.toolResult) return null;
+
+	const lines = section.toolResult.split(NEWLINE);
+
+	let fileName = '';
+	let path = '';
+	let sizeBytes: number | undefined;
+	let mimeType: string | undefined;
+
+	for (const line of lines) {
+		const trimmed = line.trim();
+
+		if (trimmed.startsWith(PREFIX_FILE)) {
+			path = trimmed.slice(PREFIX_FILE.length).trim();
+			fileName = path.split(FILE_PATH_SEPARATOR_REGEX).pop() ?? path;
+		} else if (trimmed.startsWith(PREFIX_SIZE)) {
+			const match = trimmed.match(READ_MEDIA_SIZE_REGEX);
+
+			if (match) sizeBytes = Number(match[1]);
+		} else if (trimmed.startsWith(PREFIX_MIME)) {
+			mimeType = trimmed.slice(PREFIX_MIME.length).trim();
+		}
+	}
+
+	if (!path) return null;
+
+	return { fileName, mimeType, path, sizeBytes };
+}