ui: add read_media tool (#25877)
* server: add read_image tool (#25875) Adds a server-tool that allows vision models to analyze server-side images. This tool is reading a single file for now: The image data is base64 encoded and passed to the UI, which decodes it, fills the <img> tag and removes the data URI before passing the tool result back to the model. * cleanup read_image tool: move magic strings to constants * Add dedicated constants file: tools/ui/src/lib/constants/read-image.ts with PREFIX_IMAGE, PREFIX_SIZE, PREFIX_MIME constants * Use ATTACHMENT_SAVED_REGEX from agentic.ts in ChatMessageToolCallBlockReadImage.svelte * Use NEWLINE constant from code.ts instead of hardcoded '\n' * Use PREFIX_SIZE in regex pattern for size parsing * Add SERVER_TOOL_READ_IMAGE_PREFIX_* constants in C++ server-tools.cpp to match the TypeScript PREFIX_* constants for consistency * server: rename read_image tool to read_media for images and audio * Rename server_tool_read_image to server_tool_read_media in C++ * Rename enum BuiltInTool.READ_IMAGE to READ_MEDIA * Rename UI constants, parser, and Svelte component files * Update display label from 'Read image' to 'Read media' * ui: consolidate audio data URI handling into shared utility * Extract getAudioInputFormat to a shared utility (was duplicated inline) * Store raw base64 in base64Data on the message object * Use base64Data to construct data URIs for audio rendering * Update agentic store to build INPUT_AUDIO parts from base64Data * server: read_media: restrict audio to wav/mp3 and minor fixes * Server get_mime_from_extension now only advertises audio/wav and audio/mpeg (the only formats the model's input_audio API accepts) * Case-insensitive extension matching (fixes .MP3, .Wav, etc.) * Unknown extensions return an error instead of a multi-MB data URI that inflates model context with garbage * Updated tool description to document supported formats * Frontend AUDIO_MIME_TO_EXTENSION trimmed to match server * fix a missing import in tools/ui/src/lib/stores/agentic.svelte.ts * server: read_media: add to --tools help text and README tool list * ui: fix indentation in ChatMessageToolCallBlockDefault.svelte * server: read_media tool: fix a cast to use the correct type * server: read_media: multiple fixes * server-tools.cpp import cctype, remove UTF-8 char, check mime before reading file * ui: add MimeTypePrefix.AUDIO and use it in agentic.svelte.ts * server: make read_media inherit from read_file and add uses_cwd * ui: fix formating issues * rm from server * move it to frontend-only tool * correct partial commit * rm unused * ui: address review from allozaur Replace the magic strings, regexes and number in the read_media parser and service with named constants. Path splitting reuses FILE_PATH_SEPARATOR_REGEX, the size header regex moves to READ_MEDIA_SIZE_REGEX derived from PREFIX_SIZE, and FILE_EXTENSION_SEPARATOR lands next to it in constants/code.ts. --------- Co-authored-by: ckrafft <ckrafft@epyc> Co-authored-by: Xuan Son Nguyen <son@huggingface.co> Co-authored-by: Pascal <admin@serveurperso.com>
This commit is contained in:
parabelboi
co-authored by
ckrafft
Xuan Son Nguyen
Pascal
ckrafft
Xuan Son Nguyen
Pascal
parent
89e0aa6fd3
commit
4dd127584b
@@ -22,7 +22,9 @@
|
||||
|
||||
import { DEFAULT_AGENTIC_CONFIG, NEWLINE } from '$lib/constants';
|
||||
import {
|
||||
AUDIO_MIME_TO_EXTENSION,
|
||||
DATA_URI_BASE64_REGEX,
|
||||
DEFAULT_AUDIO_EXTENSION,
|
||||
DEFAULT_IMAGE_EXTENSION,
|
||||
IMAGE_MIME_TO_EXTENSION,
|
||||
MCP_ATTACHMENT_NAME_PREFIX
|
||||
@@ -36,6 +38,7 @@ import {
|
||||
ToolCallType
|
||||
} from '$lib/enums';
|
||||
import { ChatService } from '$lib/services';
|
||||
import { ReadMediaService } from '$lib/services/read-media.service';
|
||||
import { SandboxService } from '$lib/services/sandbox.service';
|
||||
import { ToolsService } from '$lib/services/tools.service';
|
||||
import { conversationsStore } from '$lib/stores/conversations.svelte';
|
||||
@@ -75,9 +78,10 @@ import type {
|
||||
import type {
|
||||
DatabaseMessage,
|
||||
DatabaseMessageExtra,
|
||||
DatabaseMessageExtraAudioFile,
|
||||
DatabaseMessageExtraImageFile
|
||||
} from '$lib/types/database';
|
||||
import { isAbortError } from '$lib/utils';
|
||||
import { getAudioInputFormat, isAbortError } from '$lib/utils';
|
||||
import { SvelteMap } from 'svelte/reactivity';
|
||||
|
||||
function createDefaultSession(): AgenticSession {
|
||||
@@ -900,7 +904,18 @@ class AgenticStore {
|
||||
if (executionResult.isError) toolSuccess = false;
|
||||
} else if (toolSource === ToolSource.FRONTEND) {
|
||||
const args = this.parseToolArguments(toolCall.function.arguments);
|
||||
const executionResult = await SandboxService.executeTool(toolName, args, signal);
|
||||
const executionResult =
|
||||
toolName === BuiltInTool.READ_MEDIA
|
||||
? await ReadMediaService.executeTool(
|
||||
args,
|
||||
{
|
||||
audio: modelsStore.modelSupportsAudio(effectiveModel),
|
||||
vision: modelsStore.modelSupportsVision(effectiveModel)
|
||||
},
|
||||
signal,
|
||||
conversationsStore.activeConversation?.cwd
|
||||
)
|
||||
: await SandboxService.executeTool(toolName, args, signal);
|
||||
|
||||
result = executionResult.content;
|
||||
|
||||
@@ -990,7 +1005,19 @@ class AgenticStore {
|
||||
];
|
||||
|
||||
for (const attachment of attachments) {
|
||||
if (attachment.type === AttachmentType.IMAGE) {
|
||||
if (attachment.type === AttachmentType.AUDIO) {
|
||||
if (modelsStore.modelSupportsAudio(effectiveModel)) {
|
||||
contentParts.push({
|
||||
input_audio: {
|
||||
data: (attachment as DatabaseMessageExtraAudioFile).base64Data,
|
||||
format: getAudioInputFormat(
|
||||
(attachment as DatabaseMessageExtraAudioFile).mimeType
|
||||
)
|
||||
},
|
||||
type: ContentPartType.INPUT_AUDIO
|
||||
});
|
||||
}
|
||||
} else if (attachment.type === AttachmentType.IMAGE) {
|
||||
if (modelsStore.modelSupportsVision(effectiveModel)) {
|
||||
contentParts.push({
|
||||
image_url: {
|
||||
@@ -1101,6 +1128,18 @@ class AgenticStore {
|
||||
return `[Attachment saved: ${name}]`;
|
||||
}
|
||||
|
||||
if (mimeType.startsWith(MimeTypePrefix.AUDIO)) {
|
||||
// audio extras hold the bare base64, the input_audio part has no room for a data URI
|
||||
attachments.push({
|
||||
base64Data,
|
||||
mimeType,
|
||||
name,
|
||||
type: AttachmentType.AUDIO
|
||||
});
|
||||
|
||||
return `[Attachment saved: ${name}]`;
|
||||
}
|
||||
|
||||
return line;
|
||||
});
|
||||
|
||||
@@ -1108,7 +1147,9 @@ class AgenticStore {
|
||||
}
|
||||
|
||||
private buildAttachmentName(mimeType: string, index: number): string {
|
||||
const extension = IMAGE_MIME_TO_EXTENSION[mimeType] ?? DEFAULT_IMAGE_EXTENSION;
|
||||
const extension = mimeType.startsWith(MimeTypePrefix.AUDIO)
|
||||
? (AUDIO_MIME_TO_EXTENSION[mimeType] ?? DEFAULT_AUDIO_EXTENSION)
|
||||
: (IMAGE_MIME_TO_EXTENSION[mimeType] ?? DEFAULT_IMAGE_EXTENSION);
|
||||
|
||||
return `${MCP_ATTACHMENT_NAME_PREFIX}-${Date.now()}-${index}.${extension}`;
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user