ui: add browser get_info tool (#27251)

* ui: add browser get_info tool

* format

* nits

* Update tools/ui/src/lib/stores/tools.svelte.ts

Co-authored-by: Aleksander Grygier <aleksander.grygier@gmail.com>

---------

Co-authored-by: Aleksander Grygier <aleksander.grygier@gmail.com>

tools/ui/src/lib/constants/browser-info.ts

@@ -0,0 +1,36 @@
+import { CLI_FLAGS } from './cli-flags.constants';
+import { BuiltInTool, JsonSchemaType, ToolCallType } from '$lib/enums';
+import type { OpenAIToolDefinition } from '$lib/types';
+
+export const BROWSER_INFO_TOOL_NAME = BuiltInTool.GET_INFO;
+
+/** UA token to OS name, first match wins - Android and iOS UAs also carry the Linux / Mac OS X tokens */
+export const BROWSER_INFO_OS_UA_PATTERNS: readonly [RegExp, string][] = [
+	[/Windows NT/, 'Windows'],
+	[/Android/, 'Android'],
+	[/iPhone|iPad|iPod/, 'iOS'],
+	[/CrOS/, 'ChromeOS'],
+	[/Mac OS X/, 'macOS'],
+	[/Linux/, 'Linux']
+];
+
+export const BROWSER_INFO_OS_UNKNOWN = 'unknown';
+
+/** Sent to the model as the `note` field of the tool result, next to the OS name */
+export const BROWSER_INFO_NOTE = `This environment is browser-only, it cannot read or modify local files, and it cannot run shell commands. To get local file access, tell user to launch llama-server with the ${CLI_FLAGS.AGENT} argument.`;
+
+export function buildBrowserInfoToolDefinition(): OpenAIToolDefinition {
+	return {
+		function: {
+			description:
+				'Get runtime info (OS name), may call when user asks about local files or shell commands',
+			name: BROWSER_INFO_TOOL_NAME,
+			parameters: {
+				properties: {},
+				required: [],
+				type: JsonSchemaType.OBJECT
+			}
+		},
+		type: ToolCallType.FUNCTION
+	};
+}

tools/ui/src/lib/constants/cli-flags.constants.ts

@@ -1,4 +1,5 @@
 export const CLI_FLAGS = {
+	AGENT: '--agent',
 	API_KEY: '--api-key',
 	MCP_PROXY: '--ui-mcp-proxy',
 	SLOTS: '--slots',

tools/ui/src/lib/constants/index.ts

@@ -59,3 +59,4 @@ export * from './uri-template.constants';
 export * from './url.constants';
 export * from './working-directory.constants';
 export * from './read-media';
+export * from './browser-info';

tools/ui/src/lib/stores/agentic.svelte.ts

@@ -56,7 +56,8 @@ import type {
 	AgenticSession,
 	McpServerOverride,
 	MCPToolCall,
-	SettingsConfigType
+	SettingsConfigType,
+	ToolExecutionResult
 } from '$lib/types';
 import type {
 	AgenticFlowCallbacks,
@@ -83,7 +84,7 @@ import type {
 	DatabaseMessageExtraAudioFile,
 	DatabaseMessageExtraImageFile
 } from '$lib/types/database';
-import { getAudioInputFormat, isAbortError } from '$lib/utils';
+import { executeBrowserInfoTool, getAudioInputFormat, isAbortError } from '$lib/utils';
 import { SvelteMap } from 'svelte/reactivity';
 
 function createDefaultSession(): AgenticSession {
@@ -942,18 +943,24 @@ class AgenticStore {
 							if (executionResult.isError) toolSuccess = false;
 						} else if (toolSource === ToolSource.FRONTEND) {
 							const args = this.parseToolArguments(toolCall.function.arguments);
-							const executionResult =
-								toolName === BuiltInTool.READ_MEDIA
-									? await ReadMediaService.executeTool(
-											args,
-											{
-												audio: modelsStore.modelSupportsAudio(effectiveModel),
-												vision: modelsStore.modelSupportsVision(effectiveModel)
-											},
-											signal,
-											conversationsStore.activeConversation?.cwd
-										)
-									: await SandboxService.executeTool(toolName, args, signal);
+
+							let executionResult: ToolExecutionResult;
+
+							if (toolName === BuiltInTool.GET_INFO) {
+								executionResult = executeBrowserInfoTool();
+							} else if (toolName === BuiltInTool.READ_MEDIA) {
+								executionResult = await ReadMediaService.executeTool(
+									args,
+									{
+										audio: modelsStore.modelSupportsAudio(effectiveModel),
+										vision: modelsStore.modelSupportsVision(effectiveModel)
+									},
+									signal,
+									conversationsStore.activeConversation?.cwd
+								);
+							} else {
+								executionResult = await SandboxService.executeTool(toolName, args, signal);
+							}
 
 							result = executionResult.content;
 

tools/ui/src/lib/stores/tools.svelte.ts

@@ -1,5 +1,6 @@
 import { browser } from '$app/environment';
 import {
+	buildBrowserInfoToolDefinition,
 	buildReadMediaToolDefinition,
 	DISABLED_TOOL_KEYS_LOCALSTORAGE_KEY,
 	HOME_TILDE,
@@ -185,9 +186,18 @@ class ToolsStore {
 
 		if (readMedia) tools.push(readMedia);
 
+		// provide browser's get_info tool if server doesn't provide one
+		if (!this.hasBuiltinTool(BuiltInTool.GET_INFO)) {
+			tools.push(buildBrowserInfoToolDefinition());
+		}
+
 		return tools;
 	}
 
+	private hasBuiltinTool(name: BuiltInTool): boolean {
+		return this._builtinTools.some((def) => def.function.name === name);
+	}
+
 	/**
 	 * `read_media` runs in the frontend on top of the server's `read_file`, so it
 	 * exists only when that tool is served and the active model can perceive the
@@ -195,11 +205,7 @@ class ToolsStore {
 	 * conversation uses.
 	 */
 	private readMediaTool(): OpenAIToolDefinition | null {
-		const hasReadFile = this._builtinTools.some(
-			(def) => def.function.name === BuiltInTool.READ_FILE
-		);
-
-		if (!hasReadFile) return null;
+		if (!this.hasBuiltinTool(BuiltInTool.READ_FILE)) return null;
 
 		const model = modelsStore.selectedModelName ?? modelsStore.models[0]?.model ?? '';
 

tools/ui/src/lib/utils/browser-info.ts

@@ -0,0 +1,39 @@
+/**
+ * Browser fallback for the server's `get_info` tool, offered only when the
+ * server does not serve one (llama-server without --agent). It tells the model
+ * which OS the browser runs on and that there is no local file or shell access,
+ * so it does not plan around tools that are not there.
+ *
+ * @see server_tool_get_info in tools/server/server-tools.cpp - the served variant
+ * @see buildBrowserInfoToolDefinition in constants/browser-info.ts - tool schema sent to the LLM
+ */
+
+import { browser } from '$app/environment';
+import {
+	BROWSER_INFO_NOTE,
+	BROWSER_INFO_OS_UA_PATTERNS,
+	BROWSER_INFO_OS_UNKNOWN
+} from '$lib/constants';
+import type { ToolExecutionResult } from '$lib/types';
+
+function detectOs(userAgent: string): string {
+	for (const [pattern, os] of BROWSER_INFO_OS_UA_PATTERNS) {
+		if (pattern.test(userAgent)) return os;
+	}
+
+	return BROWSER_INFO_OS_UNKNOWN;
+}
+
+/**
+ * Result shape mirrors the server tool's JSON so the `get_info` renderer reads
+ * `os` the same way, minus `cwd` - there is no working directory to report.
+ */
+export function executeBrowserInfoTool(): ToolExecutionResult {
+	return {
+		content: JSON.stringify({
+			note: BROWSER_INFO_NOTE,
+			os: browser ? detectOs(navigator.userAgent) : BROWSER_INFO_OS_UNKNOWN
+		}),
+		isError: false
+	};
+}

tools/ui/src/lib/utils/index.ts

@@ -331,6 +331,9 @@ export { getChatCommands } from './chat-commands';
 // SANDBOX_TOOL_DEFINITION is deprecated; kept for backward compatibility.
 export { buildSandboxToolDefinition, SANDBOX_TOOL_DEFINITION } from './sandbox-tool';
 
+// Browser fallback for the server's get_info tool
+export { executeBrowserInfoTool } from './browser-info';
+
 // Cryptography utilities
 
 export { uuid } from './uuid';