diff --git a/.github/workflows/ui-self-hosted.yml b/.github/workflows/ui-self-hosted.yml index 346f1cfb4..4967de07c 100644 --- a/.github/workflows/ui-self-hosted.yml +++ b/.github/workflows/ui-self-hosted.yml @@ -19,6 +19,7 @@ on: '.github/workflows/ui-build-self-hosted.yml', 'ui/apps/chat/**.*', 'ui/lib/ui/**.*', + 'ui/lib/voice/**.*', 'tools/server/tests/**.*' ] pull_request: @@ -28,6 +29,7 @@ on: '.github/workflows/ui-build-self-hosted.yml', 'ui/apps/chat/**.*', 'ui/lib/ui/**.*', + 'ui/lib/voice/**.*', 'tools/server/tests/**.*' ] diff --git a/.github/workflows/ui.yml b/.github/workflows/ui.yml index 10d7ed0ed..65e5c36ff 100644 --- a/.github/workflows/ui.yml +++ b/.github/workflows/ui.yml @@ -150,6 +150,37 @@ jobs: run: npm run prepack working-directory: ui/lib/ui + voice-checks: + name: Voice checks + runs-on: ubuntu-24.04 + steps: + - name: Checkout code + uses: actions/checkout@v6 + with: + ref: ${{ github.event.inputs.sha || github.event.pull_request.head.sha || github.sha || github.head_ref || github.ref_name }} + + - name: Setup Node.js + uses: actions/setup-node@v6 + with: + node-version: "24" + package-manager-cache: false + + - name: Install dependencies + run: npm ci + working-directory: ui/lib/voice + + - name: Run type checking + run: npm run check + working-directory: ui/lib/voice + + - name: Run tests + run: npm test + working-directory: ui/lib/voice + + - name: Build the package + run: npm run prepack + working-directory: ui/lib/voice + e2e-tests: name: E2E Tests needs: ui-build diff --git a/ui/README.md b/ui/README.md index b70228d0d..0f210a40c 100644 --- a/ui/README.md +++ b/ui/README.md @@ -5,7 +5,8 @@ The UI of llama.cpp: the packages in `lib` and the apps built on them. ```text ui/ ├── lib/ -│ └── ui/ @ggml-org/llama-ui: design tokens, fonts, surfaces and components +│ ├── ui/ @ggml-org/llama-ui: design tokens, fonts, surfaces and components +│ └── voice/ @ggml-org/llama-voice: microphone capture, transcription and dictation └── apps/ └── chat/ Llama Chat, the chat app that llama-server serves ``` @@ -13,12 +14,15 @@ ui/ - [`lib/ui`](lib/ui/README.md) draws the parts and handles interaction, with no business logic. It is built with `@sveltejs/package` and published to npm as `@ggml-org/llama-ui`. Apps in this repo skip the package: they compile its sources as part of their own build, so they hold no copy of a component, a token or a font. +- [`lib/voice`](lib/voice/README.md) records the microphone and turns speech into text through llama-server, with no + UI: Llama UI draws the voice parts. It is plain TypeScript built with `tsc`, plus a Svelte entry + (`@ggml-org/llama-voice/svelte`). Apps in this repo compile its sources too. It is not published to npm yet. - [`apps/chat`](apps/chat/README.md) talks to llama-server, stores conversations and runs the agentic loop. CMake builds it into llama-server (see `scripts/ui-assets.cmake`), and it is published to npm as `@ggml-org/llama-chat` (with `llama-chat` as its short name), so `npx llama-chat` runs it against any llama-server. Each folder is its own npm package with its own `package-lock.json`: run `npm ci` in the one you work on. Building, -checking and testing `apps/chat` needs no install in `lib/ui`. +checking and testing `apps/chat` needs no install in `lib/ui` or `lib/voice`. ## Releases diff --git a/ui/apps/chat/.storybook/main.ts b/ui/apps/chat/.storybook/main.ts index 3e8c1a39d..e2b8dad14 100644 --- a/ui/apps/chat/.storybook/main.ts +++ b/ui/apps/chat/.storybook/main.ts @@ -16,7 +16,7 @@ const config: StorybookConfig = { '../tests/stories/**/*.stories.@(js|ts|svelte)', { directory: '../../../lib/ui/tests/stories', - files: '{foundations,components,app,models,mockups}/**/*.stories.svelte' + files: '{foundations,components,app,models,mockups,voice}/**/*.stories.svelte' } ], addons: [ diff --git a/ui/apps/chat/DESIGN.md b/ui/apps/chat/DESIGN.md index f3a1eaf98..efbc65d93 100644 --- a/ui/apps/chat/DESIGN.md +++ b/ui/apps/chat/DESIGN.md @@ -1154,6 +1154,12 @@ screens. Status lamps, avatars and the round icon buttons in the chat form are f (`--sidebar-reveal-x`), taking no taps while the page is aside. The page keeps room under its last row (`max-md:pb-safe-24`) so the key never covers it. One per page, only for its main action; from `md` up the action stays a button in the header. +- **Voice clips:** a recorded clip is a round primary play key, its waveform and the time in + `caption`: bars in the muted foreground at 40%, the played part in the primary colour, so + the clip stays neutral in both modes. The live waveform while recording is as tall as the + input's line (24px), the newest level on the right, and holds still with reduced motion. A + voice message's transcript opens under the clip on hover, keyboard focus or a tap, and + stays in the DOM for screen readers while closed. - **Copy feedback:** a copy action turns its icon into a `fg-success` check for 1.5s (`COPY_FEEDBACK_RESET_MS`), only when the copy succeeded, then back. The icon pops in with a 150ms fade and scale, skipped on first render and with reduced motion. diff --git a/ui/apps/chat/README.md b/ui/apps/chat/README.md index 5fa37286a..abee5d43b 100644 --- a/ui/apps/chat/README.md +++ b/ui/apps/chat/README.md @@ -89,6 +89,7 @@ The app imports the library as `@ggml-org/llama-ui/ui/` and compiles its s | Routes | `src/routes` | `/`, `/chat/[id]`, `/models`, `/mcp-servers`, `/settings`, `/search` | | App components | `src/lib/components/app` | Chat, navigation, models, MCP, settings and dialogs, grouped by area | | UI components | `ui/lib/ui` | shadcn-svelte on bits-ui, from Llama UI | +| Voice | `ui/lib/voice` | Microphone capture, transcription and dictation, from Llama Voice (`@ggml-org/llama-voice`) | | Hooks | `src/lib/hooks` | View logic between components and stores: scroll, drag and drop, pickers, keyboard | | Stores | `src/lib/stores` | Application state as Svelte 5 runes: chat, conversations, the agentic loop, tools, models, settings | | Services | `src/lib/services` | Stateless I/O: the llama-server API, IndexedDB, MCP, the Hugging Face Hub, the tool sandbox | diff --git a/ui/apps/chat/demo/README.md b/ui/apps/chat/demo/README.md index 8769da6ef..e92962687 100644 --- a/ui/apps/chat/demo/README.md +++ b/ui/apps/chat/demo/README.md @@ -115,7 +115,7 @@ Records every `scripts/docs/*.demo.ts` from the storage target and uploads `vide ## Writing a demo -Scripts get `test` and `expect` from `helpers.ts` without importing them. `demo.click`, `demo.hover` and `demo.type` glide a visible cursor to the target before acting, with timing varied like a person's (seeded per demo, so re-takes match); `demo.scroll` turns the mouse wheel over a target; `demo.zoom` pinches in on a control or a point to show a detail too small to read in the frame, holds, and pinches back out (with the fingertips drawn on a touch device); `demo.caption` shows a caption at the top of the frame. Use `test.use({ settings, startPath })` to seed other settings or start on another route. +Scripts get `test` and `expect` from `helpers.ts` without importing them. `demo.click`, `demo.hover` and `demo.type` glide a visible cursor to the target before acting, with timing varied like a person's (seeded per demo, so re-takes match); `demo.scroll` turns the mouse wheel over a target; `demo.zoom` pinches in on a control or a point to show a detail too small to read in the frame, holds, and pinches back out (with the fingertips drawn on a touch device); `demo.caption` shows a caption at the top of the frame. For voice input, `demo.microphone()` gives the page a microphone that hears nothing but `demo.speak(text)`, which reads the text in a synthetic voice (macOS `say`) and shows it as a caption, since the video has no sound. Use `test.use({ settings, startPath })` to seed other settings or start on another route. ## Conversations diff --git a/ui/apps/chat/demo/helpers.ts b/ui/apps/chat/demo/helpers.ts index 1c0ad7734..97e7bdb11 100644 --- a/ui/apps/chat/demo/helpers.ts +++ b/ui/apps/chat/demo/helpers.ts @@ -3,7 +3,15 @@ import { generateTrace } from './fixtures/traces.ts'; import { ScreencastRecorder } from './recorder.ts'; import { type CDPSession, type Locator, type Page, test as base } from '@playwright/test'; import { execFileSync } from 'node:child_process'; -import { appendFileSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs'; +import { + appendFileSync, + mkdirSync, + mkdtempSync, + readFileSync, + rmSync, + writeFileSync +} from 'node:fs'; +import { tmpdir } from 'node:os'; import { basename, join, resolve } from 'node:path'; const VIDEOS_DIR = resolve(import.meta.dirname, 'videos'); @@ -112,6 +120,68 @@ function seedSettings(config: Record): void { } } +/** + * Swaps the page's microphone for one that hears only what `window.demoSpeak` plays into it, + * silence otherwise. Chromium's fake capture device crashes the page, so voice input is + * recorded with this instead. Runs in the page. + */ +function installFakeMicrophone(): void { + const context = new AudioContext(); + const output = context.createMediaStreamDestination(); + + navigator.mediaDevices.getUserMedia = async (constraints) => { + if (!constraints?.audio || constraints.video) { + throw new DOMException('demo: only the microphone is faked', 'NotSupportedError'); + } + + await context.resume(); + + // each opening gets its own tracks, so a recording that stops them leaves the source on + return new MediaStream(output.stream.getAudioTracks().map((track) => track.clone())); + }; + + Object.assign(window, { + demoSpeak: async (wavBase64: string) => { + const bytes = Uint8Array.from(atob(wavBase64), (c) => c.charCodeAt(0)); + const source = context.createBufferSource(); + + source.buffer = await context.decodeAudioData(bytes.buffer); + source.connect(output); + await context.resume(); + await new Promise((done) => { + source.onended = done; + source.start(); + }); + } + }); +} + +/** `text` read by a synthetic voice, as a base64 WAV. Uses macOS `say`. */ +function synthesizeSpeech(text: string, voice: string): string { + const dir = mkdtempSync(join(tmpdir(), 'demo-speech-')); + const file = join(dir, 'speech.wav'); + + try { + execFileSync('say', [ + '-v', + voice, + '-o', + file, + '--file-format=WAVE', + '--data-format=LEI16@24000', + text + ]); + + return readFileSync(file).toString('base64'); + } catch (error) { + throw new Error(`demo: speak() needs macOS \`say\` with the voice ${voice}`, { + cause: error + }); + } finally { + rmSync(dir, { force: true, recursive: true }); + } +} + /** Seeded randomness, so a demo moves the same way on every recording. */ function createRandom(seed: string): () => number { let state = [...seed].reduce((h, c) => Math.imul(h ^ c.charCodeAt(0), 2654435761), 1); @@ -143,19 +213,9 @@ export class Demo { /** Shows a caption at the top of the frame for `ms`, clear of the chat form. */ async caption(text: string, ms = 2200): Promise { - await this.page.evaluate((t) => { - const el = document.getElementById('demo-caption'); - - if (!el) return; - - el.textContent = t; - el.classList.add('visible'); - }, text); + await this.showCaption(text); await this.page.waitForTimeout(ms); - await this.page.evaluate(() => - document.getElementById('demo-caption')?.classList.remove('visible') - ); - await this.page.waitForTimeout(250); + await this.hideCaption(); } async click(target: Locator, ms = 400): Promise { @@ -182,6 +242,15 @@ export class Demo { await this.pause(ms); } + /** + * Gives the page a microphone that hears only what `speak` says, and allows the page to + * use it. Call it before the microphone is first used. + */ + async microphone(): Promise { + await this.page.context().grantPermissions(['microphone']); + await this.page.evaluate(installFakeMicrophone); + } + /** Glides the cursor to a point near the center of `target`. */ async moveTo(target: Locator): Promise { await target.scrollIntoViewIfNeeded(); @@ -240,6 +309,23 @@ export class Demo { await this.pause(ms); } + /** + * Says `text` into the microphone from `microphone` in a synthetic voice (macOS `say`), and + * returns once it is said. The words show as a caption, since the video has no sound. + */ + async speak(text: string, voice = 'Samantha', ms = 400): Promise { + const wav = synthesizeSpeech(text, voice); + + await this.showCaption(`“${text}”`); + await this.page.evaluate( + (base64) => + (window as unknown as { demoSpeak(wav: string): Promise }).demoSpeak(base64), + wav + ); + await this.hideCaption(); + await this.pause(ms); + } + /** Types quickly with uneven key timing, a little slower after spaces and punctuation. */ async type(target: Locator, text: string, ms = 500): Promise { await this.click(target, 200); @@ -305,6 +391,13 @@ export class Demo { return { x: box.x + box.width / 2, y: box.y + box.height / 2 }; } + private async hideCaption(): Promise { + await this.page.evaluate(() => + document.getElementById('demo-caption')?.classList.remove('visible') + ); + await this.page.waitForTimeout(250); + } + /** A pinch that reaches the screen's edge can end without its last touchend reaching the page. */ private async liftFingers(): Promise { await this.page.evaluate(() => @@ -340,6 +433,17 @@ export class Demo { return this.cdp; } + + private async showCaption(text: string): Promise { + await this.page.evaluate((t) => { + const el = document.getElementById('demo-caption'); + + if (!el) return; + + el.textContent = t; + el.classList.add('visible'); + }, text); + } } interface DemoOptions { diff --git a/ui/apps/chat/scripts/llama-ui.js b/ui/apps/chat/scripts/llama-ui.js index c4eb26af7..f8963593a 100644 --- a/ui/apps/chat/scripts/llama-ui.js +++ b/ui/apps/chat/scripts/llama-ui.js @@ -26,6 +26,7 @@ const LLAMA_UI_EXPORTS = [ ['@ggml-org/llama-ui/enums', `${source}/enums/index.ts`], ['@ggml-org/llama-ui/code', `${source}/utils/index.ts`], ['@ggml-org/llama-ui/utils', `${source}/components/ui/utils.ts`], + ['@ggml-org/llama-ui/voice', `${source}/components/voice/index.ts`], ['@ggml-org/llama-ui/styles/*', `${source}/styles/*`], ['@ggml-org/llama-ui/theme.css', `${source}/styles/theme.css`], ['@ggml-org/llama-ui/tokens.css', `${source}/styles/tokens.css`], diff --git a/ui/apps/chat/src/lib/components/app/chat/ChatAttachments/ChatAttachmentsList/ChatAttachmentsListItem/ChatAttachmentsListItem.svelte b/ui/apps/chat/src/lib/components/app/chat/ChatAttachments/ChatAttachmentsList/ChatAttachmentsListItem/ChatAttachmentsListItem.svelte index d7284aa61..5a0987154 100644 --- a/ui/apps/chat/src/lib/components/app/chat/ChatAttachments/ChatAttachmentsList/ChatAttachmentsListItem/ChatAttachmentsListItem.svelte +++ b/ui/apps/chat/src/lib/components/app/chat/ChatAttachments/ChatAttachmentsList/ChatAttachmentsListItem/ChatAttachmentsListItem.svelte @@ -1,9 +1,11 @@ + +
+ {#key collapsed} + + + + {/key} + + {#if !source.voice} +

{source.name}

+ {/if} + + {#if !readonly && onRemove} +
+ onRemove?.(item.id)} + stopPropagationOnClick + tooltip="Remove" + /> +
+ {/if} +
diff --git a/ui/apps/chat/src/lib/components/app/chat/ChatAttachments/ChatAttachmentsPreview/ChatAttachmentsPreviewCurrentItem/ChatAttachmentsPreviewCurrentItemAudio.svelte b/ui/apps/chat/src/lib/components/app/chat/ChatAttachments/ChatAttachmentsPreview/ChatAttachmentsPreviewCurrentItem/ChatAttachmentsPreviewCurrentItemAudio.svelte index f1d6a46cc..2d2ee228b 100644 --- a/ui/apps/chat/src/lib/components/app/chat/ChatAttachments/ChatAttachmentsPreview/ChatAttachmentsPreviewCurrentItem/ChatAttachmentsPreviewCurrentItemAudio.svelte +++ b/ui/apps/chat/src/lib/components/app/chat/ChatAttachments/ChatAttachmentsPreview/ChatAttachmentsPreviewCurrentItem/ChatAttachmentsPreviewCurrentItemAudio.svelte @@ -1,12 +1,23 @@
@@ -21,6 +32,19 @@

Audio preview not available

{/if} -

{currentItem?.name || 'Audio'}

+

+ {source?.voice ? VOICE_MESSAGE_LABEL : currentItem?.name || 'Audio'} +

+ + {#if transcript} +

+ Transcript: {transcript} +

+ {:else if transcribing} +

Transcribing...

+ {/if}
diff --git a/ui/apps/chat/src/lib/components/app/chat/ChatForm/ChatForm.svelte b/ui/apps/chat/src/lib/components/app/chat/ChatForm/ChatForm.svelte index e99c1eac3..727ceccd7 100644 --- a/ui/apps/chat/src/lib/components/app/chat/ChatForm/ChatForm.svelte +++ b/ui/apps/chat/src/lib/components/app/chat/ChatForm/ChatForm.svelte @@ -1,10 +1,13 @@ + +
{ event.preventDefault(); - - if (!canSubmit || disabled || hasLoadingAttachments) return; - - onSubmit?.(); + void submit(); }} > @@ -753,13 +807,14 @@ {disabled} expanded={isExpanded} onInput={() => { + voice.handleUserInput(); expansion.handleInput(); pickers.handleInput(); onValueChange?.(value); }} onKeydown={handleKeydown} onPaste={handlePaste} - {placeholder} + placeholder={voicePlaceholder ?? placeholder} /> {#if mcpStore.resources.hasAttachments} @@ -774,29 +829,55 @@
- - onSystemPromptClick?.({ files: uploadedFiles, message: value })} - onWorkingDirectoryClick={isWorkingDirectoryInAddMenu - ? pickers.handleWorkingDirectoryOpen - : undefined} - {showAddButton} - {showModelSelector} - {uploadedFiles} - workingDirectory={cwd} - /> + {#if voice.isActive} + void handleStopAndSend()} + onStop={() => void voice.stop()} + quiet={voice.quiet} + state={voice.state} + /> + {/if} + + +
+ + onSystemPromptClick?.({ files: uploadedFiles, message: value })} + onWorkingDirectoryClick={isWorkingDirectoryInAddMenu + ? pickers.handleWorkingDirectoryOpen + : undefined} + {showAddButton} + {showModelSelector} + voice={{ + action: voice.label, + hint: voice.hint, + prepare: () => void voice.prepare(), + show: voice.showButton, + unavailableReason: voice.unavailableReason, + unprepare: voice.unprepare + }} + workingDirectory={cwd} + /> +
diff --git a/ui/apps/chat/src/lib/components/app/chat/ChatForm/ChatFormActions/ChatFormActionRecord.svelte b/ui/apps/chat/src/lib/components/app/chat/ChatForm/ChatFormActions/ChatFormActionRecord.svelte index 8664af6b2..33d52171d 100644 --- a/ui/apps/chat/src/lib/components/app/chat/ChatForm/ChatFormActions/ChatFormActionRecord.svelte +++ b/ui/apps/chat/src/lib/components/app/chat/ChatForm/ChatFormActions/ChatFormActionRecord.svelte @@ -2,34 +2,95 @@ import { Button } from '@ggml-org/llama-ui/ui/button'; import * as Tooltip from '@ggml-org/llama-ui/ui/tooltip'; import { Mic, MicOff, Square } from '@lucide/svelte'; - import { ICON_CLASS_DEFAULT } from '#lib/constants/index.ts'; + import { + ICON_CLASS_DEFAULT, + VOICE_MODE_MESSAGE, + VOICE_PRESS_CLICK_MS + } from '#lib/constants/index.ts'; + import { MouseButton, PointerType } from '#lib/enums/index.ts'; interface Props { + /** What a press does: send a voice message or dictate. */ + action?: string; class?: string; disabled?: boolean; - hasAudioModality?: boolean; + /** Said after the action in the tooltip, e.g. that another model may be loaded. */ + hint?: string | null; isLoading?: boolean; isRecording?: boolean; /** Why the microphone can't record (no HTTPS, access denied...), shown on the button. */ micWarning?: string | null; + /** Why no model can take the recording; disables the button. */ + unavailableReason?: string | null; + /** A quieter (tertiary) button, beside the main one: send or stop. */ + secondary?: boolean; + /** With `secondary`: filled like the stop button it stands beside, not bare. */ + filled?: boolean; onMicClick?: () => void; + /** A mouse or the focus came to the button: a press may follow, the microphone can open. */ + onPrepare?: () => void; + /** It left again without a press. */ + onPrepareEnd?: () => void; } let { + action = VOICE_MODE_MESSAGE.voiceMessage, class: className = '', disabled = false, - hasAudioModality = false, + filled = false, + hint = null, isLoading = false, isRecording = false, micWarning = null, - onMicClick + onMicClick, + onPrepare, + onPrepareEnd, + secondary = false, + unavailableReason = null }: Props = $props(); - // a model without audio input disables the button; a microphone that can't record keeps it + // no model to take the recording disables the button; a microphone that can't record keeps it // tappable, so a tap can say why - const showWarning = $derived(hasAudioModality && !isRecording && !!micWarning); + const showWarning = $derived(!unavailableReason && !isRecording && !!micWarning); + // a mouse starts recording on press, not on release, so the first words are not lost; + // a touch or a key goes through the click, which is when the browser allows it + let pressedAt = -Infinity; + + function handlePointerDown(event: PointerEvent) { + if (isRecording || disabled || isLoading || unavailableReason) return; + + if (event.pointerType !== PointerType.MOUSE || event.button !== MouseButton.PRIMARY) return; + + pressedAt = performance.now(); + onMicClick?.(); + } + + // a touch presses at once, so only a mouse, a pen or the keyboard come ahead of the press + function handlePointerEnter(event: PointerEvent) { + if (event.pointerType !== PointerType.TOUCH && !isRecording) onPrepare?.(); + } + + function handlePointerLeave(event: PointerEvent) { + if (event.pointerType !== PointerType.TOUCH) onPrepareEnd?.(); + } + + function handleClick() { + // the click that ends the press that already started it; the recording bar may have + // taken the button's place meanwhile, so this one can also never come + if (performance.now() - pressedAt < VOICE_PRESS_CLICK_MS) { + pressedAt = -Infinity; + + return; + } + + onMicClick?.(); + } + const tooltip = $derived( - !hasAudioModality ? 'Current model does not support audio' : showWarning ? micWarning : null + isRecording + ? null + : (unavailableReason ?? + (showWarning ? micWarning : [action, hint].filter(Boolean).join('. '))) ); @@ -37,28 +98,41 @@ {/snippet} diff --git a/ui/apps/chat/src/lib/components/app/chat/ChatForm/ChatFormActions/ChatFormActions.svelte b/ui/apps/chat/src/lib/components/app/chat/ChatForm/ChatFormActions/ChatFormActions.svelte index d71a8da5d..c895ef0c3 100644 --- a/ui/apps/chat/src/lib/components/app/chat/ChatForm/ChatFormActions/ChatFormActions.svelte +++ b/ui/apps/chat/src/lib/components/app/chat/ChatForm/ChatFormActions/ChatFormActions.svelte @@ -1,6 +1,6 @@ + + + {#snippet stop({ disabled })} + + {/snippet} + diff --git a/ui/apps/chat/src/lib/components/app/chat/index.ts b/ui/apps/chat/src/lib/components/app/chat/index.ts index fce3962c4..f7626c73a 100644 --- a/ui/apps/chat/src/lib/components/app/chat/index.ts +++ b/ui/apps/chat/src/lib/components/app/chat/index.ts @@ -79,6 +79,14 @@ export { default as ChatAttachmentsListItemMcpResource } from './ChatAttachments */ export { default as ChatAttachmentsListItemThumbnailFile } from './ChatAttachments/ChatAttachmentsList/ChatAttachmentsListItem/ChatAttachmentsListItemThumbnailFile.svelte'; +/** + * An audio attachment drawn as a clip: a playable waveform (peaks computed from + * the audio when it was stored without them) and, for a voice message, its + * transcript under it on hover, focus or a tap, with the transcribing and failed + * states. `sm` in the chat form, removable; `md` in a sent message. + */ +export { default as ChatAttachmentsListItemVoice } from './ChatAttachments/ChatAttachmentsList/ChatAttachmentsListItem/ChatAttachmentsListItemVoice.svelte'; + /** * Thumbnail for image attachments with lazy loading and error fallback. * Displays image preview with configurable dimensions. Falls back to placeholder @@ -170,12 +178,20 @@ export { default as ChatForm } from './ChatForm/ChatForm.svelte'; export { default as ChatFormActionsAdd } from './ChatForm/ChatFormActions/ChatFormActionAdd/ChatFormActionsAdd.svelte'; /** - * Audio recording button with real-time recording indicator. Records audio - * and converts to WAV format for upload. Only visible when the active model - * supports audio modality and setting for automatic audio input is enabled. Shows recording duration while active. + * Microphone button: starts a voice message or a dictation (its label says which), + * or, disabled, says why no model can take a recording; while recording it is the + * stop button. Shown on an empty message when voice input is available and the + * setting for the microphone on empty input is on. */ export { default as ChatFormActionRecord } from './ChatForm/ChatFormActions/ChatFormActionRecord.svelte'; +/** + * The actions row while the chat form records: cancel, the live waveform and the + * elapsed time, and stop, with a voice message's streaming text under the waveform + * and a waiting state while dictated text is transcribed after recording. + */ +export { default as ChatFormActionsRecording } from './ChatForm/ChatFormActions/ChatFormActionsRecording.svelte'; + /** * Container for chat form action buttons. Arranges file attachment, audio record, * and submit/stop buttons in a horizontal layout. Handles conditional visibility diff --git a/ui/apps/chat/src/lib/components/app/models/ModelRowActions.svelte b/ui/apps/chat/src/lib/components/app/models/ModelRowActions.svelte index 33be4c624..071b71000 100644 --- a/ui/apps/chat/src/lib/components/app/models/ModelRowActions.svelte +++ b/ui/apps/chat/src/lib/components/app/models/ModelRowActions.svelte @@ -1,7 +1,8 @@ @@ -35,6 +48,26 @@ /> + {#if canDictate} + + + voiceStore.toggleDictation(option.model)} + tooltip={dictationLabel} + tooltipAsTitle + /> + + {/if} + {#if isFav} diff --git a/ui/apps/chat/src/lib/components/app/models/ModelVoiceRole.svelte b/ui/apps/chat/src/lib/components/app/models/ModelVoiceRole.svelte new file mode 100644 index 000000000..cabc88139 --- /dev/null +++ b/ui/apps/chat/src/lib/components/app/models/ModelVoiceRole.svelte @@ -0,0 +1,60 @@ + + + +{#if role} + {@const Icon = ICONS[role]} + + + + {#snippet child({ props })} + + + {/snippet} + + + +

{description}

+
+
+{/if} diff --git a/ui/apps/chat/src/lib/components/app/models/ModelsManager/ModelsManagerModelRow.svelte b/ui/apps/chat/src/lib/components/app/models/ModelsManager/ModelsManagerModelRow.svelte index 2c25dddd1..9f8913451 100644 --- a/ui/apps/chat/src/lib/components/app/models/ModelsManager/ModelsManagerModelRow.svelte +++ b/ui/apps/chat/src/lib/components/app/models/ModelsManager/ModelsManagerModelRow.svelte @@ -4,15 +4,20 @@ import ModelContext from '../ModelContext.svelte'; import ModelDraftSubline from '../ModelDraftSubline.svelte'; import ModelId from '../ModelId.svelte'; + import ModelVoiceRole from '../ModelVoiceRole.svelte'; import ModelsManagerStatusCell from './ModelsManagerStatusCell.svelte'; import { modelRowActions } from './row-actions'; import { configuredContext, downloadProgressFor, isRowControlClick } from './utils'; import { ModelDownloadProgressBar } from '@ggml-org/llama-ui/models'; - import { Heart, MoreHorizontal } from '@lucide/svelte'; + import { Heart, Mic, MoreHorizontal } from '@lucide/svelte'; import { ContextMenuActions, DropdownMenuActions } from '#lib/components/app/index.ts'; - import { MODEL_ROW_GRID_CLASS, MODEL_ROW_TRAILING_CELL_CLASS } from '#lib/constants/index.ts'; + import { + MODEL_ROW_GRID_CLASS, + MODEL_ROW_TRAILING_CELL_CLASS, + VOICE_DICTATION_MODEL_LABEL + } from '#lib/constants/index.ts'; import { ModelRowDownloadState } from '#lib/enums/index.ts'; - import { deviceStore, modelsStore } from '#lib/stores/index.ts'; + import { deviceStore, modelsStore, voiceStore } from '#lib/stores/index.ts'; import type { ModelOption, ModelSidecarBadge } from '#lib/types/models.js'; interface Props { @@ -38,6 +43,7 @@ }: Props = $props(); let favorite = $derived(isFavorite(option)); + let dictates = $derived(voiceStore.isDictationModel(option.model)); let isHidden = $derived(modelsStore.isHidden(option.id)); // live while the download runs, frozen while it is paused let downloadProgress = $derived(downloadProgressFor(option.model)); @@ -101,9 +107,18 @@ title={option.model} /> + + + {#if dictates} + +