From 555881ebc8b0fc0402b30e09258a32a7bfd13c52 Mon Sep 17 00:00:00 2001 From: Zach Winter Date: Fri, 24 Jul 2026 16:09:46 -0400 Subject: [PATCH] ui: reduce per-token render cost when streaming (#26053) * performance harness - the empirical root Assisted-by: Claude Opus 4.8 * 210.36ms -> 2.67ms per streamed token Assisted-by: Claude Opus 4.8 * 11.58ms -> 0.62ms per streamed token Assisted-by: Claude Opus 4.8 * 22.02ms -> 3.33ms per streamed token Assisted-by: Claude Opus 4.8 * 3.07ms -> 1.36ms per streamed token at 40 messages Assisted-by: Claude Opus 4.8 --------- Co-authored-by: Zach Winter --- .../content/CollapsibleContentBlock.svelte | 13 +- .../content/CollapsibleTerminalBlock.svelte | 10 +- .../MarkdownContent/MarkdownContent.svelte | 14 +- .../ui/src/lib/constants/latex-protection.ts | 8 + .../ui/src/lib/stores/conversations.svelte.ts | 17 +- tools/ui/src/lib/utils/code.ts | 19 +- tools/ui/src/lib/utils/latex-protection.ts | 49 ++- tools/ui/tests/client/README-perf.md | 58 +++ .../client/agentic-stream.perf.svelte.test.ts | 410 ++++++++++++++++++ .../collapsible-lazy-body.svelte.test.ts | 37 ++ .../components/AgenticPerfWrapper.svelte | 16 + .../components/ChatMessagesPerfWrapper.svelte | 12 + .../CollapsibleLazyBodyHarness.svelte | 21 + .../components/agentic-perf-state.svelte.ts | 17 + .../update-message-in-place.svelte.test.ts | 64 +++ tools/ui/tests/unit/agentic-hotpath.bench.ts | 248 +++++++++++ tools/ui/tests/unit/code.test.ts | 21 + tools/ui/tests/unit/latex-protection.test.ts | 38 ++ 18 files changed, 1037 insertions(+), 35 deletions(-) create mode 100644 tools/ui/tests/client/README-perf.md create mode 100644 tools/ui/tests/client/agentic-stream.perf.svelte.test.ts create mode 100644 tools/ui/tests/client/collapsible-lazy-body.svelte.test.ts create mode 100644 tools/ui/tests/client/components/AgenticPerfWrapper.svelte create mode 100644 tools/ui/tests/client/components/ChatMessagesPerfWrapper.svelte create mode 100644 tools/ui/tests/client/components/CollapsibleLazyBodyHarness.svelte create mode 100644 tools/ui/tests/client/components/agentic-perf-state.svelte.ts create mode 100644 tools/ui/tests/client/update-message-in-place.svelte.test.ts create mode 100644 tools/ui/tests/unit/agentic-hotpath.bench.ts diff --git a/tools/ui/src/lib/components/app/content/CollapsibleContentBlock.svelte b/tools/ui/src/lib/components/app/content/CollapsibleContentBlock.svelte index ea7fe7b95..ad7032261 100644 --- a/tools/ui/src/lib/components/app/content/CollapsibleContentBlock.svelte +++ b/tools/ui/src/lib/components/app/content/CollapsibleContentBlock.svelte @@ -89,10 +89,15 @@ -
-
- {@render children()} + + {#if open} +
+
+ {@render children()} +
-
+ {/if} diff --git a/tools/ui/src/lib/components/app/content/CollapsibleTerminalBlock.svelte b/tools/ui/src/lib/components/app/content/CollapsibleTerminalBlock.svelte index 4370aea42..5cbe003bd 100644 --- a/tools/ui/src/lib/components/app/content/CollapsibleTerminalBlock.svelte +++ b/tools/ui/src/lib/components/app/content/CollapsibleTerminalBlock.svelte @@ -90,8 +90,12 @@ -
- {@render children()} -
+ + {#if open} +
+ {@render children()} +
+ {/if} diff --git a/tools/ui/src/lib/components/app/content/MarkdownContent/MarkdownContent.svelte b/tools/ui/src/lib/components/app/content/MarkdownContent/MarkdownContent.svelte index 2c17f4234..fc7e31412 100644 --- a/tools/ui/src/lib/components/app/content/MarkdownContent/MarkdownContent.svelte +++ b/tools/ui/src/lib/components/app/content/MarkdownContent/MarkdownContent.svelte @@ -107,6 +107,15 @@ return null; }); const liveSvgHtml = $derived(streamingSvgCode !== null ? sanitizeSvg(streamingSvgCode) : ''); + + // Derived rather than called inline in the template so it only recomputes when + // the block actually changes. Auto-detection is disabled while streaming: it + // costs ~38ms a call and re-guesses the language on every chunk. + const streamingCodeHtml = $derived( + incompleteCodeBlock + ? highlightCode(incompleteCodeBlock.code, incompleteCodeBlock.language || 'text', false) + : '' + ); let previewDialogOpen = $state(false); let previewCode = $state(''); let previewLanguage = $state('text'); @@ -903,10 +912,7 @@ > 
{@html highlightCode(
-								incompleteCodeBlock.code,
-								incompleteCodeBlock.language || 'text'
-							)}{@html streamingCodeHtml}
diff --git a/tools/ui/src/lib/constants/latex-protection.ts b/tools/ui/src/lib/constants/latex-protection.ts index 27c88e725..da27c0083 100644 --- a/tools/ui/src/lib/constants/latex-protection.ts +++ b/tools/ui/src/lib/constants/latex-protection.ts @@ -28,6 +28,14 @@ export const LATEX_MATH_AND_CODE_PATTERN = /** Regex to capture the content of a $$...\\\\...$$ block (display-formula with line-break) */ export const LATEX_LINEBREAK_REGEXP = /\$\$([\s\S]*?\\\\[\s\S]*?)\$\$/; +/** + * Cheap gate for `preprocessLaTeX`. Every transformation it performs is triggered + * by a `$` (inline/display math, currency escaping) or a backslash escape + * (`\(`, `\[`, `\ce{`, `\pu{`). Text containing neither is returned untouched, so + * this lets the caller skip the whole protect/restore pipeline. + */ +export const LATEX_TRIGGER_REGEXP = /[$\\]/; + /** map from mchem-regexp to replacement */ export const MHCHEM_PATTERN_MAP: readonly [RegExp, string][] = [ [/(\s)\$\\ce{/g, '$1$\\\\ce{'], diff --git a/tools/ui/src/lib/stores/conversations.svelte.ts b/tools/ui/src/lib/stores/conversations.svelte.ts index 9649cfe03..bc2feefd3 100644 --- a/tools/ui/src/lib/stores/conversations.svelte.ts +++ b/tools/ui/src/lib/stores/conversations.svelte.ts @@ -169,8 +169,21 @@ class ConversationsStore { * Updates a message at a specific index in active messages */ updateMessageAtIndex(index: number, updates: Partial): void { - if (index !== -1 && this.activeMessages[index]) { - this.activeMessages[index] = { ...this.activeMessages[index], ...updates }; + const message = index === -1 ? undefined : this.activeMessages[index]; + + if (!message) return; + + // Assign field by field rather than replacing the object. Replacing it + // changes the array slot, which invalidates every consumer that merely + // walks the list - notably ChatMessages.displayMessages, which rebuilds + // entries for every message in the conversation. Deep $state proxies make + // per-field writes fine-grained, so only readers of the changed field wake. + const target = message as unknown as Record; + + for (const [key, value] of Object.entries(updates)) { + if (target[key] !== value) { + target[key] = value; + } } } diff --git a/tools/ui/src/lib/utils/code.ts b/tools/ui/src/lib/utils/code.ts index a29028c74..44b9b4841 100644 --- a/tools/ui/src/lib/utils/code.ts +++ b/tools/ui/src/lib/utils/code.ts @@ -30,13 +30,21 @@ function trimCodePadding(code: string): string { return code.replace(TRIM_LEADING_PADDING_REGEX, '').replace(TRIM_TRAILING_PADDING_REGEX, ''); } +function escapeCode(code: string): string { + return code.replace(AMPERSAND_REGEX, '&').replace(LT_REGEX, '<').replace(GT_REGEX, '>'); +} + /** * Highlights code using highlight.js * @param code - The code to highlight * @param language - The programming language + * @param autoDetect - Fall back to `highlightAuto` when `language` is unknown. + * Callers rendering a still-streaming block should pass false: auto-detection + * costs ~38ms per call and re-guesses on every chunk, so the language (and + * therefore the whole highlight) flickers as the block grows. * @returns HTML string with syntax highlighting */ -export function highlightCode(code: string, language: string): string { +export function highlightCode(code: string, language: string, autoDetect = true): string { if (!code) return ''; const trimmed = trimCodePadding(code); @@ -47,15 +55,14 @@ export function highlightCode(code: string, language: string): string { if (isSupported) { return hljs.highlight(trimmed, { language: lang }).value; - } else { + } else if (autoDetect) { return hljs.highlightAuto(trimmed).value; + } else { + return escapeCode(trimmed); } } catch { // Fallback to escaped plain text - return trimmed - .replace(AMPERSAND_REGEX, '&') - .replace(LT_REGEX, '<') - .replace(GT_REGEX, '>'); + return escapeCode(trimmed); } } diff --git a/tools/ui/src/lib/utils/latex-protection.ts b/tools/ui/src/lib/utils/latex-protection.ts index 839306978..573eb9297 100644 --- a/tools/ui/src/lib/utils/latex-protection.ts +++ b/tools/ui/src/lib/utils/latex-protection.ts @@ -2,6 +2,7 @@ import { CODE_BLOCK_REGEXP, LATEX_MATH_AND_CODE_PATTERN, LATEX_LINEBREAK_REGEXP, + LATEX_TRIGGER_REGEXP, MHCHEM_PATTERN_MAP } from '$lib/constants'; @@ -148,6 +149,15 @@ export function preprocessLaTeX(content: string): string { // See also: // https://github.com/danny-avila/LibreChat/blob/main/client/src/utils/latex.ts + // Every step below keys off a `$` or a backslash escape (\[ \] \( \) \ce{ \pu{). + // With neither present the protect/restore passes round-trip the input + // unchanged, so skip them: the step 2 scan is O(n^2) in line length and costs + // ~90ms on a 26KB single-line message that contains no math at all. This + // matters during streaming, where the whole message is reprocessed per frame. + if (!LATEX_TRIGGER_REGEXP.test(content)) { + return content; + } + // Step 0: Temporarily remove blockquote markers (>) to process LaTeX correctly // Store the structure so we can restore it later const blockquoteMarkers: Map = new Map(); @@ -175,24 +185,31 @@ export function preprocessLaTeX(content: string): string { const latexExpressions: string[] = []; // Match \S...\[...\] and protect them and insert a line-break. - content = content.replace(/([\S].*?)\\\[([\s\S]*?)\\\](.*)/g, (match, group1, group2, group3) => { - // Check if there are characters following the formula (display-formula in a table-cell?) - if (group1.endsWith('\\')) { - return match; // Backslash before \[, do nothing. - } - const hasSuffix = /\S/.test(group3); - let optBreak; + // Guarded: with no `\[` present this pattern still probes every start offset, + // expanding `.*?` to the end of each line before failing - O(n^2) for nothing. + if (content.includes('\\[')) { + content = content.replace( + /([\S].*?)\\\[([\s\S]*?)\\\](.*)/g, + (match, group1, group2, group3) => { + // Check if there are characters following the formula (display-formula in a table-cell?) + if (group1.endsWith('\\')) { + return match; // Backslash before \[, do nothing. + } + const hasSuffix = /\S/.test(group3); + let optBreak; - if (hasSuffix) { - latexExpressions.push(`\\(${group2.trim()}\\)`); // Convert into inline. - optBreak = ''; - } else { - latexExpressions.push(`\\[${group2}\\]`); - optBreak = '\n'; - } + if (hasSuffix) { + latexExpressions.push(`\\(${group2.trim()}\\)`); // Convert into inline. + optBreak = ''; + } else { + latexExpressions.push(`\\[${group2}\\]`); + optBreak = '\n'; + } - return `${group1}${optBreak}<>${optBreak}${group3}`; - }); + return `${group1}${optBreak}<>${optBreak}${group3}`; + } + ); + } // Match \(...\), \[...\], $$...$$ and protect them content = content.replace( diff --git a/tools/ui/tests/client/README-perf.md b/tools/ui/tests/client/README-perf.md new file mode 100644 index 000000000..a4e3e4c64 --- /dev/null +++ b/tools/ui/tests/client/README-perf.md @@ -0,0 +1,58 @@ +# Agentic thread perf harness + +Two tiers, both reusing the existing vitest projects (see `vite.config.ts`). + +## Tier 1 - `agentic-stream.perf.svelte.test.ts` (project: `client`, real Chromium) + +Mounts `ChatMessageAgenticContent` and replays a stream, replacing the message +object on each chunk exactly as the real pipeline does: + +- `chat.svelte.ts` `updateStreamingUI()` runs per SSE chunk +- `conversations.svelte.ts` `updateMessageAtIndex` does `{ ...old, ...updates }` + +That new object identity is the thing under test: it cascades through +`deriveAgenticSections` (which returns fresh `AgenticSection` objects) into every +tool-call block in the message, including completed ones. + +``` +npx vitest --project=client --run tests/client/agentic-stream.perf.svelte.test.ts +``` + +### Reading the output + +- `mean` / `p95` / `max` - the synchronous window per token: prop write, + `await tick()`, then a forced `offsetHeight` read so style and layout are + included rather than deferred. +- `sync` - sum of those windows. This is the number to optimize. +- `wall` - the whole run including work `MarkdownContent` defers into its own + `requestAnimationFrame`. It carries a ~16.7ms/token idle floor because the + harness yields a frame each iteration, so compare `wall` **across fixtures**, + never against `sync`. + +### The knobs, and what each one discriminates + +The point of the harness is the _scaling curve_, not any single number. + +| Knob | Reads on | +| --------------------------- | ---------------------------------------------------------------------------------------------------- | +| `priorToolCalls` (0/1/5/20) | the reactive fan-out. Flat => no fan-out. Linear => confirmed. | +| `toolResultBytes` | whole-blob string scans (`extractSearchResults`, `parseToolResultWithImages`, `classifyToolResult`). | +| `editFileEdits` | `computeLineDiff`, the O(m\*n) LCS. | +| `openCodeFence` | `hljs.highlightAuto` on partial code. | + +Deliberately no hard assertions: CI timing is noisy and the value here is the +before/after delta, not a gate. + +### Caveat + +This measures one message's subtree. In the real app `ChatMessages.svelte` +rebuilds its whole `displayMessages` list per token, so multiply by the number +of rendered messages to get the conversation-level cost. + +## Tier 2 - `../unit/agentic-hotpath.bench.ts` (project: `unit`, node) + +Per-call costs for the pure functions the curve implicates. + +``` +npx vitest bench --project=unit --run tests/unit/agentic-hotpath.bench.ts +``` diff --git a/tools/ui/tests/client/agentic-stream.perf.svelte.test.ts b/tools/ui/tests/client/agentic-stream.perf.svelte.test.ts new file mode 100644 index 000000000..c13f98cfc --- /dev/null +++ b/tools/ui/tests/client/agentic-stream.perf.svelte.test.ts @@ -0,0 +1,410 @@ +// Tier 1 perf harness for the agentic thread. +// +// Drives ChatMessageAgenticContent the way the real streaming pipeline does: +// chat.svelte.ts -> conversations.svelte.ts:184 replaces the message object per +// SSE chunk (`{ ...old, ...updates }`), which changes prop identity and cascades +// through deriveAgenticSections into every tool-call block in the message. +// +// The fixture is parameterized so the *scaling curve* identifies the culprit - +// a single number would not. See tests/client/README-perf.md. +// +// Run: npx vitest --project=client --run tests/client/agentic-stream.perf.svelte.test.ts + +import { describe, it } from 'vitest'; +import { render } from 'vitest-browser-svelte'; +import { tick } from 'svelte'; +import AgenticPerfWrapper from './components/AgenticPerfWrapper.svelte'; +import ChatMessagesPerfWrapper from './components/ChatMessagesPerfWrapper.svelte'; +import { perfState } from './components/agentic-perf-state.svelte'; +import { conversationsStore } from '$lib/stores/conversations.svelte'; +import type { DatabaseMessage } from '$lib/types'; +import { MessageRole } from '$lib/enums'; + +// --- fixture construction ------------------------------------------------- + +interface FixtureOpts { + /** Completed tool-call sections preceding the streaming text. */ + priorToolCalls: number; + /** Size of each tool result blob. */ + toolResultBytes: number; + /** Number of edit_file calls (each a 400x400-line diff). */ + editFileEdits: number; + /** Leave an unclosed ``` fence at the end of the streamed content. */ + openCodeFence: boolean; + /** + * Emit a blank line every N chunks so the content forms real markdown + * blocks. 0 = one unbroken paragraph, which defeats MarkdownContent's + * stable-block cache entirely (worst case). Typical prose has breaks. + */ + paragraphEvery: number; +} + +const DEFAULTS: FixtureOpts = { + priorToolCalls: 0, + toolResultBytes: 1024, + editFileEdits: 0, + openCodeFence: false, + paragraphEvery: 0 +}; + +function blob(bytes: number, seed: string): string { + const line = `${seed} output line with some representative width to it`; + const n = Math.max(1, Math.ceil(bytes / (line.length + 1))); + const out: string[] = []; + for (let i = 0; i < n; i++) out.push(`${line} ${i}`); + return out.join('\n'); +} + +function diffLines(n: number, seed: string): string { + const out: string[] = []; + for (let i = 0; i < n; i++) out.push(`${seed} line ${i} const value_${i} = compute(${i});`); + return out.join('\n'); +} + +let msgSeq = 0; + +function baseMessage(overrides: Partial): DatabaseMessage { + return { + id: `m${msgSeq++}`, + convId: 'perf-conv', + type: 'text', + timestamp: 0, + role: MessageRole.ASSISTANT, + content: '', + parent: null, + children: [], + ...overrides + } as DatabaseMessage; +} + +function buildFixture(opts: FixtureOpts): { + message: DatabaseMessage; + toolMessages: DatabaseMessage[]; +} { + const toolCalls: unknown[] = []; + const toolMessages: DatabaseMessage[] = []; + + for (let i = 0; i < opts.priorToolCalls; i++) { + const id = `call_${i}`; + toolCalls.push({ + id, + type: 'function', + function: { + name: 'exec_shell_command', + arguments: JSON.stringify({ command: `grep -rn "thing_${i}" src/` }) + } + }); + toolMessages.push( + baseMessage({ + role: MessageRole.TOOL, + toolCallId: id, + content: `${blob(opts.toolResultBytes, `t${i}`)}\n[exit code: 0]` + }) + ); + } + + for (let i = 0; i < opts.editFileEdits; i++) { + const id = `edit_${i}`; + toolCalls.push({ + id, + type: 'function', + function: { + name: 'edit_file', + arguments: JSON.stringify({ + path: `/src/file_${i}.ts`, + edits: [{ old_text: diffLines(400, 'old'), new_text: diffLines(400, 'new') }] + }) + } + }); + toolMessages.push( + baseMessage({ + role: MessageRole.TOOL, + toolCallId: id, + content: JSON.stringify({ result: 'ok', edits_applied: 1 }) + }) + ); + } + + const message = baseMessage({ + toolCalls: toolCalls.length > 0 ? JSON.stringify(toolCalls) : undefined, + content: '' + }); + + return { message, toolMessages }; +} + +// --- the driver ----------------------------------------------------------- + +interface Sample { + label: string; + tokens: number; + mean: number; + p95: number; + max: number; + /** Sum of the synchronous per-token windows. */ + total: number; + /** Wall-clock for the whole run incl. deferred rAF work, then settle. */ + wall: number; +} + +function nextFrame(): Promise { + return new Promise((resolve) => requestAnimationFrame(() => resolve())); +} + +const results: Sample[] = []; + +/** + * Replays `tokens` streamed chunks, replacing the message object each time + * exactly as conversations.svelte.ts:184 does, flushing Svelte and forcing + * layout so the measurement includes render + style/layout, not just script. + */ +async function measure(label: string, partial: Partial, tokens = 60) { + const opts = { ...DEFAULTS, ...partial }; + const { message, toolMessages } = buildFixture(opts); + + perfState.message = message; + perfState.toolMessages = toolMessages; + perfState.isStreaming = true; + + // Every wrapper reads the same module state, so a leftover mount from a + // previous fixture would re-render on each mutation and fold its cost into + // this measurement. Tear down explicitly between fixtures. + const { unmount } = render(AgenticPerfWrapper); + + await tick(); + + const CHUNK = 'The quick brown fox jumps over the lazy dog. '; + let accumulated = opts.openCodeFence ? '```notalanguage\n' : ''; + const durations: number[] = []; + + const wallStart = performance.now(); + + for (let i = 0; i < tokens; i++) { + accumulated += CHUNK; + if (opts.paragraphEvery > 0 && (i + 1) % opts.paragraphEvery === 0) { + accumulated += '\n\n'; + } + + const t0 = performance.now(); + // Mirrors updateMessageAtIndex: a brand-new object identity per chunk. + perfState.message = { ...perfState.message!, content: accumulated }; + await tick(); + void document.body.offsetHeight; // force style + layout + durations.push(performance.now() - t0); + + // MarkdownContent coalesces its parse into a rAF, so that work lands + // outside the window above. Yield a frame each iteration so it is + // captured in `wall` - the gap between `wall` and `total` is the + // deferred cost. + await nextFrame(); + } + + // Let any trailing coalesced work drain before stopping the clock. + await nextFrame(); + await nextFrame(); + const wall = performance.now() - wallStart; + + await unmount(); + + durations.sort((a, b) => a - b); + const total = durations.reduce((a, b) => a + b, 0); + + results.push({ + label, + tokens, + mean: total / durations.length, + p95: durations[Math.floor(durations.length * 0.95)], + max: durations[durations.length - 1], + total, + wall + }); +} + +function report() { + const pad = (s: string, n: number) => s.padEnd(n); + const num = (n: number) => n.toFixed(2).padStart(8); + + const header = `${pad('fixture', 40)}${pad('tok', 5)}${'mean'.padStart(8)}${'p95'.padStart(8)}${'max'.padStart(8)}${'sync'.padStart(9)}${'wall'.padStart(9)}`; + const lines = [ + '', + '=== Tier 1: ms per streamed token (agentic content subtree) ===', + 'mean/p95/max = synchronous window per token (script + style + layout).', + 'sync = sum of those windows.', + 'wall = whole run, incl. work MarkdownContent defers into its own rAF.', + ' NOTE: wall carries a ~16.7ms/token idle floor from the harness', + ' yielding a frame each iteration (60 tokens => ~1000ms floor).', + ' Compare wall ACROSS fixtures / against the baseline row,', + ' never against sync.', + '', + header, + '-'.repeat(header.length) + ]; + + for (const r of results) { + lines.push( + `${pad(r.label, 40)}${pad(String(r.tokens), 5)}${num(r.mean)}${num(r.p95)}${num(r.max)}${num(r.total)}${num(r.wall)}` + ); + } + + lines.push(''); + console.log(lines.join('\n')); +} + +// --- conversation-level driver -------------------------------------------- +// The per-message driver above cannot see the fan-out in ChatMessages: a single +// token mutation invalidates `displayMessages`, which rebuilds a fresh +// toolMessages array for EVERY message in the conversation. Drive the real +// store through the real list component to measure that. + +async function measureConversation( + label: string, + priorMessages: number, + tokens = 60, + /** + * Give each prior assistant turn a resolved tool call, so the fixture pays + * `hasAgenticContent`'s JSON.parse and the tool-message grouping walk that a + * real agent thread would - plain prose messages skip both. + */ + agentic = false +) { + const history: DatabaseMessage[] = []; + for (let i = 0; i < priorMessages; i++) { + const isAssistant = i % 2 !== 0; + + if (isAssistant && agentic) { + const id = `prior_call_${i}`; + history.push( + baseMessage({ + role: MessageRole.ASSISTANT, + content: `Message ${i}`, + toolCalls: JSON.stringify([ + { + id, + type: 'function', + function: { + name: 'exec_shell_command', + arguments: JSON.stringify({ command: `grep -rn "thing_${i}" src/` }) + } + } + ]) + }) + ); + history.push( + baseMessage({ + role: MessageRole.TOOL, + toolCallId: id, + content: `${blob(1024, `r${i}`)}\n[exit code: 0]` + }) + ); + continue; + } + + history.push( + baseMessage({ + role: isAssistant ? MessageRole.ASSISTANT : MessageRole.USER, + content: `Message ${i}: ${blob(512, `m${i}`)}` + }) + ); + } + + const streaming = baseMessage({ role: MessageRole.ASSISTANT, content: '' }); + history.push(streaming); + + conversationsStore.activeMessages = history; + + const { unmount } = render(ChatMessagesPerfWrapper); + await tick(); + + const idx = conversationsStore.findMessageIndex(streaming.id); + const CHUNK = 'The quick brown fox jumps over the lazy dog. '; + let accumulated = ''; + const durations: number[] = []; + const wallStart = performance.now(); + + for (let i = 0; i < tokens; i++) { + accumulated += CHUNK; + + const t0 = performance.now(); + // The real path: chat.svelte.ts -> conversations.svelte.ts. + conversationsStore.updateMessageAtIndex(idx, { content: accumulated }); + await tick(); + void document.body.offsetHeight; + durations.push(performance.now() - t0); + + await nextFrame(); + } + + await nextFrame(); + await nextFrame(); + const wall = performance.now() - wallStart; + + await unmount(); + conversationsStore.activeMessages = []; + + durations.sort((a, b) => a - b); + const total = durations.reduce((a, b) => a + b, 0); + + results.push({ + label, + tokens, + mean: total / durations.length, + p95: durations[Math.floor(durations.length * 0.95)], + max: durations[durations.length - 1], + total, + wall + }); +} + +// --- the matrix ----------------------------------------------------------- +// Sequential, in one test, so the table prints together and the samples do not +// interleave with other suites competing for the main thread. + +describe('agentic streaming perf', () => { + it('scales', { timeout: 600_000 }, async () => { + // Baseline: plain text streaming, nothing agentic. + await measure('baseline: no tool calls', {}); + + // Knob: priorToolCalls. Flat => no fan-out. Linear => fan-out confirmed. + await measure('priorToolCalls=1 (1KB results)', { priorToolCalls: 1 }); + await measure('priorToolCalls=5 (1KB results)', { priorToolCalls: 5 }); + await measure('priorToolCalls=20 (1KB results)', { priorToolCalls: 20 }); + + // Knob: toolResultBytes, at fixed section count. + await measure('5 calls x 200KB results', { + priorToolCalls: 5, + toolResultBytes: 200 * 1024 + }); + + // Knob: editFileEdits (computeLineDiff, 400x400 LCS). + await measure('3 edit_file calls (400x400 diff)', { editFileEdits: 3 }); + + // Knob: openCodeFence (hljs highlightAuto on partial code). + await measure('open code fence, unknown language', { openCodeFence: true }); + + // Conversation level: does streaming one message cost more as the + // conversation grows? Flat => no fan-out. Linear => confirmed. + await measureConversation('convo: 1 prior message', 1); + await measureConversation('convo: 10 prior messages', 10); + await measureConversation('convo: 40 prior messages', 40); + + // Same, but each prior assistant turn carries a resolved tool call. + await measureConversation('convo: 10 prior, agentic', 10, 60, true); + await measureConversation('convo: 40 prior, agentic', 40, 60, true); + + // Message length: MarkdownContent re-parses the whole accumulated string + // each frame, so per-token cost should climb as the response grows. + // A rising mean across these three rows means O(n^2) over the stream. + // One unbroken paragraph: worst case, stable-block cache never applies. + await measure('len 60tok 1-para (worst case)', {}, 60); + await measure('len 250tok 1-para (worst case)', {}, 250); + await measure('len 600tok 1-para (worst case)', {}, 600); + + // Same lengths, broken into paragraphs every 8 chunks: the typical shape, + // where only the trailing paragraph should be unstable. + await measure('len 60tok paras (typical)', { paragraphEvery: 8 }, 60); + await measure('len 250tok paras (typical)', { paragraphEvery: 8 }, 250); + await measure('len 600tok paras (typical)', { paragraphEvery: 8 }, 600); + + report(); + }); +}); diff --git a/tools/ui/tests/client/collapsible-lazy-body.svelte.test.ts b/tools/ui/tests/client/collapsible-lazy-body.svelte.test.ts new file mode 100644 index 000000000..9b29bea73 --- /dev/null +++ b/tools/ui/tests/client/collapsible-lazy-body.svelte.test.ts @@ -0,0 +1,37 @@ +// Guards the lazy-body behaviour of the shared collapsible wrappers. +// +// bits-ui's Collapsible.Content renders its children unconditionally and only +// sets `hidden`, so before this was gated a collapsed tool result kept its whole +// body in the DOM and re-rendered it on every streamed token (measured at +// ~41ms/section/token for a 200KB result). These tests pin the fix: closed means +// not rendered, and opening still mounts the body. + +import { describe, it, expect } from 'vitest'; +import { render } from 'vitest-browser-svelte'; +import { tick } from 'svelte'; +import CollapsibleLazyBodyHarness from './components/CollapsibleLazyBodyHarness.svelte'; + +const MARKER = 'collapsible-body-marker'; + +describe('collapsible wrappers render their body lazily', () => { + for (const variant of ['content', 'terminal'] as const) { + it(`${variant}: body is absent while closed and present once open`, async () => { + const screen = render(CollapsibleLazyBodyHarness, { variant, open: false }); + await tick(); + + expect(document.body.textContent).not.toContain(MARKER); + + await screen.rerender({ variant, open: true }); + await tick(); + + expect(document.body.textContent).toContain(MARKER); + + // And it unmounts again on close, so a collapsed block stops costing + // anything during streaming. + await screen.rerender({ variant, open: false }); + await tick(); + + expect(document.body.textContent).not.toContain(MARKER); + }); + } +}); diff --git a/tools/ui/tests/client/components/AgenticPerfWrapper.svelte b/tools/ui/tests/client/components/AgenticPerfWrapper.svelte new file mode 100644 index 000000000..8f95860b7 --- /dev/null +++ b/tools/ui/tests/client/components/AgenticPerfWrapper.svelte @@ -0,0 +1,16 @@ + + + + {#if perfState.message} + + {/if} + diff --git a/tools/ui/tests/client/components/ChatMessagesPerfWrapper.svelte b/tools/ui/tests/client/components/ChatMessagesPerfWrapper.svelte new file mode 100644 index 000000000..14d938978 --- /dev/null +++ b/tools/ui/tests/client/components/ChatMessagesPerfWrapper.svelte @@ -0,0 +1,12 @@ + + + + + diff --git a/tools/ui/tests/client/components/CollapsibleLazyBodyHarness.svelte b/tools/ui/tests/client/components/CollapsibleLazyBodyHarness.svelte new file mode 100644 index 000000000..53d3b9cfd --- /dev/null +++ b/tools/ui/tests/client/components/CollapsibleLazyBodyHarness.svelte @@ -0,0 +1,21 @@ + + +{#if variant === 'content'} + + collapsible-body-marker + +{:else} + + collapsible-body-marker + +{/if} diff --git a/tools/ui/tests/client/components/agentic-perf-state.svelte.ts b/tools/ui/tests/client/components/agentic-perf-state.svelte.ts new file mode 100644 index 000000000..f0cc8c3b8 --- /dev/null +++ b/tools/ui/tests/client/components/agentic-perf-state.svelte.ts @@ -0,0 +1,17 @@ +// Shared reactive fixture state for the Tier 1 perf harness. +// +// The wrapper component reads this module directly rather than taking props, +// so the driver can mutate it without depending on how the test renderer +// forwards props. + +import type { DatabaseMessage } from '$lib/types'; + +export const perfState = $state<{ + message: DatabaseMessage | null; + toolMessages: DatabaseMessage[]; + isStreaming: boolean; +}>({ + message: null, + toolMessages: [], + isStreaming: true +}); diff --git a/tools/ui/tests/client/update-message-in-place.svelte.test.ts b/tools/ui/tests/client/update-message-in-place.svelte.test.ts new file mode 100644 index 000000000..54c215fef --- /dev/null +++ b/tools/ui/tests/client/update-message-in-place.svelte.test.ts @@ -0,0 +1,64 @@ +// Pins the contract that makes streaming cheap: updateMessageAtIndex mutates the +// existing message object instead of replacing it. +// +// Replacing it (`{ ...old, ...updates }`) changes the array slot, which +// invalidates every consumer that merely walks the list - ChatMessages' +// `displayMessages` rebuilds entries for EVERY message in the conversation. That +// made per-token cost scale with conversation length (1.26ms at 1 prior message +// -> 3.07ms at 40). Mutating in place keeps it flat. + +import { describe, it, expect } from 'vitest'; +import { conversationsStore } from '$lib/stores/conversations.svelte'; +import type { DatabaseMessage } from '$lib/types'; +import { MessageRole } from '$lib/enums'; + +function makeMessage(id: string): DatabaseMessage { + return { + id, + convId: 'c1', + type: 'text', + timestamp: 0, + role: MessageRole.ASSISTANT, + content: '', + parent: null, + children: [] + } as DatabaseMessage; +} + +describe('conversationsStore.updateMessageAtIndex', () => { + it('mutates in place, preserving object identity', () => { + conversationsStore.activeMessages = [makeMessage('a'), makeMessage('b')]; + const before = conversationsStore.activeMessages[1]; + + conversationsStore.updateMessageAtIndex(1, { content: 'hello' }); + + expect(conversationsStore.activeMessages[1].content).toBe('hello'); + expect(conversationsStore.activeMessages[1]).toBe(before); + + conversationsStore.activeMessages = []; + }); + + it('leaves other messages and unrelated fields untouched', () => { + conversationsStore.activeMessages = [makeMessage('a'), makeMessage('b')]; + const untouched = conversationsStore.activeMessages[0]; + + conversationsStore.updateMessageAtIndex(1, { content: 'x', model: 'm1' }); + + expect(conversationsStore.activeMessages[0]).toBe(untouched); + expect(conversationsStore.activeMessages[0].content).toBe(''); + expect(conversationsStore.activeMessages[1].model).toBe('m1'); + expect(conversationsStore.activeMessages[1].id).toBe('b'); + + conversationsStore.activeMessages = []; + }); + + it('is a no-op for an index of -1 or out of range', () => { + conversationsStore.activeMessages = [makeMessage('a')]; + + expect(() => conversationsStore.updateMessageAtIndex(-1, { content: 'x' })).not.toThrow(); + expect(() => conversationsStore.updateMessageAtIndex(9, { content: 'x' })).not.toThrow(); + expect(conversationsStore.activeMessages[0].content).toBe(''); + + conversationsStore.activeMessages = []; + }); +}); diff --git a/tools/ui/tests/unit/agentic-hotpath.bench.ts b/tools/ui/tests/unit/agentic-hotpath.bench.ts new file mode 100644 index 000000000..51b99c31f --- /dev/null +++ b/tools/ui/tests/unit/agentic-hotpath.bench.ts @@ -0,0 +1,248 @@ +// Microbenchmarks for the functions on the agentic-thread streaming hot path. +// +// Every function here is invoked from a `$derived` that is invalidated on each +// streamed token, for every tool-call section in the message. These numbers give +// the per-call cost; multiply by (tokens x sections) for the real damage. +// +// Run: npx vitest bench --project=unit tests/unit/agentic-hotpath.bench.ts + +import { bench, describe } from 'vitest'; + +import { remark } from 'remark'; +import remarkBreaks from 'remark-breaks'; +import remarkGfm from 'remark-gfm'; +import remarkMath from 'remark-math'; +import remarkRehype from 'remark-rehype'; +import rehypeKatex from 'rehype-katex'; +import rehypeHighlight from 'rehype-highlight'; +import rehypeStringify from 'rehype-stringify'; +import { all as lowlightAll } from 'lowlight'; + +import { computeLineDiff } from '$lib/utils/compute-line-diff'; +import { extractSearchResults, extractSearchQuery } from '$lib/utils/search-results'; +import { parsePartialJsonArgs } from '$lib/utils/parse-partial-json-args'; +import { highlightCode, detectIncompleteCodeBlock } from '$lib/utils/code'; +import { classifyToolResult, parseToolResultWithImages } from '$lib/utils/agentic'; +import { preprocessLaTeX } from '$lib/utils/latex-protection'; + +// --- fixtures ------------------------------------------------------------- + +function lines(n: number, seed: string): string { + const out: string[] = []; + for (let i = 0; i < n; i++) out.push(`${seed} line ${i} const value_${i} = compute(${i});`); + return out.join('\n'); +} + +const SHELL_OUTPUT_1KB = lines(12, 'out'); +const SHELL_OUTPUT_200KB = lines(2600, 'out'); +const SHELL_OUTPUT_2MB = lines(26000, 'out'); + +// Realistic exec_shell_command result: the exit-code marker is the final line, +// which is what the un-anchored EXIT_CODE regex has to scan the whole blob for. +const SHELL_2MB_WITH_EXIT = `${SHELL_OUTPUT_2MB}\n[exit code: 0]`; + +const EDIT_OLD_400 = lines(400, 'old'); +const EDIT_NEW_400 = lines(400, 'new'); +const EDIT_OLD_50 = lines(50, 'old'); +const EDIT_NEW_50 = lines(50, 'new'); + +const WRITE_FILE_ARGS = JSON.stringify({ + path: '/src/lib/thing.ts', + content: lines(1500, 'src') +}); + +const MARKDOWN_50KB = Array.from( + { length: 400 }, + (_, i) => + `## Section ${i}\n\nSome **bold** prose with a [link](https://example.com) and \`inline\` code.\n\n- bullet one\n- bullet two\n\n\`\`\`ts\nconst x${i} = ${i};\n\`\`\`\n` +).join('\n'); + +const CODE_BLOCK_5KB = lines(60, 'code'); + +// --- A: computeLineDiff (O(m*n) LCS, allocates a full matrix) -------------- + +describe('computeLineDiff', () => { + bench('50 x 50 lines', () => { + computeLineDiff(EDIT_OLD_50, EDIT_NEW_50); + }); + + bench('400 x 400 lines', () => { + computeLineDiff(EDIT_OLD_400, EDIT_NEW_400); + }); +}); + +// --- B: the unified processor, rebuilt per call --------------------------- +// Mirrors MarkdownContent.svelte:144-176. The local rehype/remark plugins are +// omitted (they pull in browser-only modules); rehypeHighlight + lowlightAll is +// the dominant term, so this is a lower bound on the real cost. + +function buildProcessor() { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + let proc: any = remark().use(remarkGfm); + proc = proc.use(remarkMath).use(remarkBreaks).use(remarkRehype).use(rehypeKatex); + return proc.use(rehypeHighlight, { languages: lowlightAll }).use(rehypeStringify, { + allowDangerousHtml: true + }); +} + +describe('markdown processor', () => { + bench('build processor (per processMarkdown call, x2)', () => { + buildProcessor(); + }); + + const prebuilt = buildProcessor(); + + bench('parse 50KB markdown with a prebuilt processor', () => { + prebuilt.parse(MARKDOWN_50KB); + }); +}); + +// Isolates the O(n^2) term measured in Tier 1: processMarkdown re-parses the +// WHOLE accumulated string every frame, while only the last block can have +// changed. If parse() dominates at these sizes, incremental parsing (re-parsing +// only the tail after the last stable block) is the fix; if not, the cost is +// downstream in transform/stringify/DOM. +describe('markdown parse scaling (whole-string reparse per frame)', () => { + const proc = buildProcessor(); + const prose = (kb: number) => + Array.from( + { length: Math.ceil((kb * 1024) / 64) }, + (_, i) => `The quick brown fox jumps over the lazy dog. Sentence ${i}.` + ).join(' '); + + const MD_3KB = prose(3); + const MD_11KB = prose(11); + const MD_26KB = prose(26); + + bench('parse 3KB', () => { + proc.parse(MD_3KB); + }); + + bench('parse 11KB', () => { + proc.parse(MD_11KB); + }); + + bench('parse 26KB', () => { + proc.parse(MD_26KB); + }); + + bench('parse only a 200-char tail (proposed incremental)', () => { + proc.parse(MD_26KB.slice(-200)); + }); +}); + +// processMarkdown runs these over the WHOLE accumulated string every frame too, +// before parse() even starts. Measure them before assuming parse is the target. +describe('other whole-string passes per frame', () => { + const prose = (kb: number) => + Array.from( + { length: Math.ceil((kb * 1024) / 64) }, + (_, i) => `The quick brown fox jumps over the lazy dog. Sentence ${i}.` + ).join(' '); + + const MD_3KB = prose(3); + const MD_11KB = prose(11); + const MD_26KB = prose(26); + const MD_26KB_LATEX = `${MD_26KB} and some math $x^2 + y^2 = z^2$ inline.`; + + bench('preprocessLaTeX 3KB (no latex present)', () => { + preprocessLaTeX(MD_3KB); + }); + + bench('preprocessLaTeX 11KB (no latex present)', () => { + preprocessLaTeX(MD_11KB); + }); + + bench('preprocessLaTeX 26KB (no latex present)', () => { + preprocessLaTeX(MD_26KB); + }); + + bench('preprocessLaTeX 26KB (latex present)', () => { + preprocessLaTeX(MD_26KB_LATEX); + }); + + bench('detectIncompleteCodeBlock 26KB', () => { + detectIncompleteCodeBlock(MD_26KB); + }); +}); + +// --- C: extractSearchResults, run for EVERY tool of every type ------------ + +describe('extractSearchResults (only .length > 0 is consumed)', () => { + bench('1KB non-search tool result', () => { + extractSearchResults(SHELL_OUTPUT_1KB); + }); + + bench('200KB non-search tool result', () => { + extractSearchResults(SHELL_OUTPUT_200KB); + }); + + bench('2MB non-search tool result', () => { + extractSearchResults(SHELL_OUTPUT_2MB); + }); +}); + +describe('extractSearchQuery', () => { + bench('write_file args (throws on the JSON.parse path)', () => { + extractSearchQuery(WRITE_FILE_ARGS); + }); +}); + +// --- E: the un-anchored exit-code regex ----------------------------------- +// Reproduced inline so the bench is independent of the current call site. + +const EXIT_CODE_TAIL = /\[exit code: (-?\d+)\]\s*$/; + +describe('exit-code regex', () => { + bench('whole 2MB blob (current behaviour)', () => { + SHELL_2MB_WITH_EXIT.match(EXIT_CODE_TAIL); + }); + + bench('last 64 chars only (proposed)', () => { + SHELL_2MB_WITH_EXIT.slice(-64).match(EXIT_CODE_TAIL); + }); +}); + +// --- per-line result parsers ---------------------------------------------- + +describe('parseToolResultWithImages', () => { + bench('1KB', () => { + parseToolResultWithImages(SHELL_OUTPUT_1KB, []); + }); + + bench('200KB', () => { + parseToolResultWithImages(SHELL_OUTPUT_200KB, []); + }); + + bench('2MB', () => { + parseToolResultWithImages(SHELL_OUTPUT_2MB, []); + }); +}); + +describe('classifyToolResult', () => { + bench('200KB plain text (falls through to looksLikeMarkdown)', () => { + classifyToolResult(SHELL_OUTPUT_200KB); + }); +}); + +describe('parsePartialJsonArgs', () => { + bench('write_file args, ~60KB (char-by-char scan)', () => { + parsePartialJsonArgs(WRITE_FILE_ARGS); + }); +}); + +// --- F / I: highlight.js --------------------------------------------------- + +describe('highlightCode', () => { + bench('short bash command (title row, per token)', () => { + highlightCode('grep -rn "foo" src/ | head -50', 'bash'); + }); + + bench('5KB known language', () => { + highlightCode(CODE_BLOCK_5KB, 'typescript'); + }); + + bench('5KB unknown language -> highlightAuto', () => { + highlightCode(CODE_BLOCK_5KB, 'not-a-language'); + }); +}); diff --git a/tools/ui/tests/unit/code.test.ts b/tools/ui/tests/unit/code.test.ts index 944cf8b73..7820c9807 100644 --- a/tools/ui/tests/unit/code.test.ts +++ b/tools/ui/tests/unit/code.test.ts @@ -79,4 +79,25 @@ describe('highlightCode', () => { const framed = highlightCode('\nfunction foo() {}\n', 'javascript'); expect(framed).toBe(trimmed); }); + + it('auto-detects an unknown language by default', () => { + const html = highlightCode('const answer = 42;', 'not-a-language'); + expect(html).toContain('hljs-'); + }); + + it('escapes instead of auto-detecting when autoDetect is false', () => { + const html = highlightCode('const answer = 42;', 'not-a-language', false); + expect(html).not.toContain('hljs-'); + expect(html).toBe('const answer = 42;'); + }); + + it('still highlights a known language when autoDetect is false', () => { + const html = highlightCode('const answer = 42;', 'javascript', false); + expect(html).toContain('hljs-'); + }); + + it('escapes html metacharacters when falling back to plain text', () => { + const html = highlightCode('', 'not-a-language', false); + expect(html).toBe('<script>a && b</script>'); + }); }); diff --git a/tools/ui/tests/unit/latex-protection.test.ts b/tools/ui/tests/unit/latex-protection.test.ts index 84328dbc1..011b9c0af 100644 --- a/tools/ui/tests/unit/latex-protection.test.ts +++ b/tools/ui/tests/unit/latex-protection.test.ts @@ -374,3 +374,41 @@ $$\n\\pi_n(\\mathbb{S}^3) = \\begin{cases} expect(output).toBe('Regular text with $x^2$.\n\n> Quote with $y^2$.\n\nMore text with $z^2$.'); }); }); + +// The fast path skips the whole protect/restore pipeline when the content has +// neither `$` nor a backslash. These pin the assumption it relies on: such +// content is returned byte-for-byte unchanged, and anything carrying a trigger +// character still goes through the full pipeline. +describe('preprocessLaTeX fast path', () => { + const untouched = [ + ['plain prose', 'The quick brown fox jumps over the lazy dog.'], + ['headings and lists', '# Title\n\n- one\n- two\n\n## Sub\n\n1. first\n2. second'], + ['fenced code without escapes', '```ts\nconst x = 1;\n```'], + ['inline code and emphasis', 'Use `npm run build` with **bold** and _italic_.'], + ['blockquotes', '> quoted line\n> another line'], + ['tables', '| a | b |\n| --- | --- |\n| 1 | 2 |'], + ['brackets and parens without escapes', 'array[0] and call(arg) and [link](/url)'], + ['multi-paragraph prose', 'First para.\n\nSecond para.\n\nThird para.'] + ]; + + for (const [label, input] of untouched) { + test(`returns ${label} unchanged`, () => { + expect(preprocessLaTeX(input)).toBe(input); + }); + } + + test('still processes content once a trigger character is present', () => { + expect(preprocessLaTeX('inline \\(x^2\\) here')).toBe('inline $x^2$ here'); + expect(preprocessLaTeX('Price: $5')).toBe('Price: \\$5'); + }); + + test('fast path result matches the full pipeline for trigger-free content', () => { + // Appending a trigger char forces the slow path; stripping it back off + // must agree with what the fast path returned for the same prefix. + const body = 'Plain paragraph with no math.\n\nAnother paragraph here.'; + const viaFastPath = preprocessLaTeX(body); + const viaFullPipeline = preprocessLaTeX(`${body}\n\n\\(z\\)`); + + expect(viaFullPipeline.startsWith(viaFastPath)).toBe(true); + }); +});