diff --git a/tools/ui/src/lib/components/app/settings/SettingsChat/SettingsChatImportExportTab.svelte b/tools/ui/src/lib/components/app/settings/SettingsChat/SettingsChatImportExportTab.svelte index a86d68584..57dbba30b 100644 --- a/tools/ui/src/lib/components/app/settings/SettingsChat/SettingsChatImportExportTab.svelte +++ b/tools/ui/src/lib/components/app/settings/SettingsChat/SettingsChatImportExportTab.svelte @@ -159,8 +159,10 @@ try { const input = document.createElement('input'); + // No `accept` filter: iOS resolves each entry to a UTI and has none for + // `.jsonl`, which greys out exported conversations in the file picker. + // `parseImportFile` detects the format from the file contents instead. input.type = HtmlInputType.FILE; - input.accept = `${FileExtensionText.JSON},${FileExtensionText.JSONL},${FileExtensionText.ZIP}`; input.onchange = async (e) => { const file = (e.target as HTMLInputElement)?.files?.[0]; @@ -199,9 +201,17 @@ .snapshot(fullImportData) .filter((item) => selectedIds.has(item.conv.id)); - await conversationsStore.importConversationsData(selectedData); + const { imported, skipped } = await conversationsStore.importConversationsData(selectedData); - importedConversations = selectedConversations; + // A conversation already in the database is left untouched, so the summary + // lists what was written and the toast accounts for the rest. + if (skipped.length > 0) { + toast.info( + `Skipped ${skipped.length} conversation${skipped.length === 1 ? '' : 's'} already in your library` + ); + } + + importedConversations = imported; showImportSummary = true; showExportSummary = false; showImportDialog = false; diff --git a/tools/ui/src/lib/constants/conversation-import.ts b/tools/ui/src/lib/constants/conversation-import.ts new file mode 100644 index 000000000..ed500440a --- /dev/null +++ b/tools/ui/src/lib/constants/conversation-import.ts @@ -0,0 +1,3 @@ +// First bytes of every ZIP local file header ("PK"). Import detects an archive +// from these bytes rather than from the filename, which the OS may not preserve. +export const ZIP_MAGIC = [0x50, 0x4b]; diff --git a/tools/ui/src/lib/constants/index.ts b/tools/ui/src/lib/constants/index.ts index a80e0cb63..100432c18 100644 --- a/tools/ui/src/lib/constants/index.ts +++ b/tools/ui/src/lib/constants/index.ts @@ -13,6 +13,7 @@ export * from './storage'; export * from './attachment-menu'; export * from './auto-scroll'; export * from './context-gauge-popup'; +export * from './conversation-import'; export * from './binary-detection'; export * from './built-in-tools'; export * from './cache'; diff --git a/tools/ui/src/lib/constants/message-export.ts b/tools/ui/src/lib/constants/message-export.ts index 79fa36f91..fc4dbe259 100644 --- a/tools/ui/src/lib/constants/message-export.ts +++ b/tools/ui/src/lib/constants/message-export.ts @@ -7,6 +7,9 @@ export const EXPORT_CONV_NAME_SUFFIX_MAX_LENGTH = 20; // Characters to keep in the ISO timestamp. 19 keeps 2026-01-01T00:00:00 export const ISO_TIMESTAMP_SLICE_LENGTH = 19; +// Producer marker carried by the session record of a JSONL export +export const SESSION_HARNESS = 'llama.app'; + // Replacements for making the conversation title filename-friendly export const NON_ALPHANUMERIC_REGEX = /[^a-z0-9]/gi; export const EXPORT_CONV_NONALNUM_REPLACEMENT = '_'; diff --git a/tools/ui/src/lib/enums/conversation-import.enums.ts b/tools/ui/src/lib/enums/conversation-import.enums.ts new file mode 100644 index 000000000..eef47c5cc --- /dev/null +++ b/tools/ui/src/lib/enums/conversation-import.enums.ts @@ -0,0 +1,9 @@ +/** + * Discriminator of a record line in the JSONL conversation format. A session + * record opens a conversation and carries its properties; every following + * message record belongs to it. + */ +export enum SessionRecordType { + SESSION = 'session', + MESSAGE = 'message' +} diff --git a/tools/ui/src/lib/enums/index.ts b/tools/ui/src/lib/enums/index.ts index 2f70e063d..ee14293fc 100644 --- a/tools/ui/src/lib/enums/index.ts +++ b/tools/ui/src/lib/enums/index.ts @@ -27,6 +27,8 @@ export { ReasoningFormat } from './chat.enums'; +export { SessionRecordType } from './conversation-import.enums'; + export { ReasoningEffort } from './reasoning-effort.enums'; export { diff --git a/tools/ui/src/lib/services/database.service.ts b/tools/ui/src/lib/services/database.service.ts index 4fb70e29a..bc65caaca 100644 --- a/tools/ui/src/lib/services/database.service.ts +++ b/tools/ui/src/lib/services/database.service.ts @@ -554,12 +554,13 @@ export class DatabaseService { * Skips conversations that already exist. * * @param data - Array of { conv, messages } objects + * @returns The conversations written to the database and the ones skipped */ static async importConversations( data: { conv: DatabaseConversation; messages: DatabaseMessage[] }[] - ): Promise<{ imported: number; skipped: number }> { - let importedCount = 0; - let skippedCount = 0; + ): Promise<{ imported: DatabaseConversation[]; skipped: DatabaseConversation[] }> { + const imported: DatabaseConversation[] = []; + const skipped: DatabaseConversation[] = []; return await db.transaction( 'rw', @@ -570,8 +571,7 @@ export class DatabaseService { const existing = await db[IDXDB_TABLES.conversations].get(conv.id); if (existing) { - console.warn(`Conversation "${conv.name}" already exists, skipping...`); - skippedCount++; + skipped.push(conv); continue; } @@ -580,10 +580,10 @@ export class DatabaseService { await db[IDXDB_TABLES.messages].put(msg); } - importedCount++; + imported.push(conv); } - return { imported: importedCount, skipped: skippedCount }; + return { imported, skipped }; } ); } diff --git a/tools/ui/src/lib/stores/conversations.svelte.ts b/tools/ui/src/lib/stores/conversations.svelte.ts index bc2feefd3..e467c8fad 100644 --- a/tools/ui/src/lib/stores/conversations.svelte.ts +++ b/tools/ui/src/lib/stores/conversations.svelte.ts @@ -30,11 +30,11 @@ import type { McpServerOverride } from '$lib/types/database'; import { zipSync, unzipSync, strToU8, strFromU8 } from 'fflate'; import { MessageRole, - HtmlInputType, FileExtensionText, MimeTypeText, MimeTypeApplication, - ReasoningEffort + ReasoningEffort, + SessionRecordType } from '$lib/enums'; import { ISO_DATE_TIME_SEPARATOR, @@ -47,7 +47,10 @@ import { ISO_TIME_SEPARATOR_REPLACEMENT, NON_ALPHANUMERIC_REGEX, MULTIPLE_UNDERSCORE_REGEX, - REASONING_EFFORT_DEFAULT_LOCALSTORAGE_KEY + REASONING_EFFORT_DEFAULT_LOCALSTORAGE_KEY, + NEWLINE, + SESSION_HARNESS, + ZIP_MAGIC } from '$lib/constants'; import { ROUTES } from '$lib/constants/routes'; @@ -914,30 +917,35 @@ class ConversationsStore { /** * Serializes a session (a conversation with its messages) as JSONL. - * The first line is the session header (a `type: 'session'` record carrying the - * conversation properties); each subsequent line is a single message. + * The first line is the session header (a `SessionRecordType.SESSION` record + * carrying the conversation properties); each subsequent line is a single message. * @param data - The exported conversation payload * @returns The JSONL string (one record per line) */ serializeSessionToJsonl(data: ExportedConversation): string { const { conv, messages } = data; - const sessionLine = JSON.stringify({ type: 'session', harness: 'llama.app', ...conv }); + const sessionLine = JSON.stringify({ + type: SessionRecordType.SESSION, + harness: SESSION_HARNESS, + ...conv + }); const messageLines = messages.map((message: DatabaseMessage) => { // `toolCalls` is stored as a JSON string; drop it when empty, otherwise parse it. const { toolCalls, ...rest } = message; const normalized = toolCalls ? { ...rest, toolCalls: JSON.parse(toolCalls) } : rest; - return JSON.stringify({ type: 'message', message: normalized }); + return JSON.stringify({ type: SessionRecordType.MESSAGE, message: normalized }); }); - return [sessionLine, ...messageLines].join('\n'); + return [sessionLine, ...messageLines].join(NEWLINE); } /** * Parses the JSONL session format produced by {@link serializeSessionToJsonl}. - * A `type: 'session'` line starts a new session; following `type: 'message'` - * lines are appended to it. Supports multiple sessions in a single file. + * A `SessionRecordType.SESSION` line starts a new session; following + * `SessionRecordType.MESSAGE` lines are appended to it. Supports multiple + * sessions in a single file. * @param text - The JSONL file contents * @returns The parsed conversations with their messages */ @@ -945,20 +953,20 @@ class ConversationsStore { const sessions: ExportedConversation[] = []; let current: ExportedConversation | null = null; - for (const line of text.split('\n')) { + for (const line of text.split(NEWLINE)) { const trimmed = line.trim(); if (!trimmed) continue; const record = JSON.parse(trimmed); - if (record.type === 'session') { + if (record.type === SessionRecordType.SESSION) { // Drop the discriminator and harness marker; the rest is the conversation. const conv = { ...record }; delete conv.type; delete conv.harness; current = { conv: conv as DatabaseConversation, messages: [] }; sessions.push(current); - } else if (record.type === 'message') { + } else if (record.type === SessionRecordType.MESSAGE) { if (!current) { throw new Error('Invalid JSONL: message record before any session record'); } @@ -977,27 +985,47 @@ class ConversationsStore { } /** - * Parses an import file into conversations, accepting the current `.jsonl` and - * `.zip` formats as well as the legacy `.json` format. + * Reports whether the text is the JSONL session format, whose first non-empty + * line is a `SessionRecordType.SESSION` record. A legacy JSON export starts + * with an array or an object that has no such discriminator. + * @param text - The file contents + */ + private isSessionsJsonl(text: string): boolean { + const trimmed = text.trimStart(); + const lineEnd = trimmed.indexOf(NEWLINE); + const firstLine = lineEnd === -1 ? trimmed : trimmed.slice(0, lineEnd); + + try { + return JSON.parse(firstLine).type === SessionRecordType.SESSION; + } catch { + // Not a standalone JSON record, so not the JSONL format. + return false; + } + } + + /** + * Parses an import file into conversations, accepting the current JSONL and + * ZIP formats as well as the legacy JSON format. The format comes from the + * contents, so an import works whatever the file is named. * @param file - The user-selected file * @returns The parsed conversations with their messages */ async parseImportFile(file: File): Promise { - const name = file.name.toLowerCase(); + const bytes = new Uint8Array(await file.arrayBuffer()); - if (name.endsWith(FileExtensionText.ZIP)) { - const entries = unzipSync(new Uint8Array(await file.arrayBuffer())); + if (ZIP_MAGIC.every((byte, index) => bytes[index] === byte)) { + const entries = unzipSync(bytes); const sessions: ExportedConversation[] = []; - for (const [entryName, bytes] of Object.entries(entries)) { + for (const [entryName, entryBytes] of Object.entries(entries)) { if (!entryName.toLowerCase().endsWith(FileExtensionText.JSONL)) continue; - sessions.push(...this.parseSessionsJsonl(strFromU8(bytes))); + sessions.push(...this.parseSessionsJsonl(strFromU8(entryBytes))); } return sessions; } - const text = await file.text(); + const text = strFromU8(bytes); - if (name.endsWith(FileExtensionText.JSONL)) { + if (this.isSessionsJsonl(text)) { return this.parseSessionsJsonl(text); } @@ -1103,73 +1131,14 @@ class ConversationsStore { this.downloadConversationFile({ conv: conversation, messages }); } - /** - * Imports conversations from a JSON file - * Opens file picker and processes the selected file - * @returns The list of imported conversations - */ - async importConversations(): Promise { - return new Promise((resolve, reject) => { - const input = document.createElement('input'); - input.type = HtmlInputType.FILE; - input.accept = FileExtensionText.JSON; - - input.onchange = async (e) => { - const file = (e.target as HTMLInputElement)?.files?.[0]; - - if (!file) { - reject(new Error('No file selected')); - return; - } - - try { - const text = await file.text(); - const parsedData = JSON.parse(text); - let importedData: ExportedConversations; - - if (Array.isArray(parsedData)) { - importedData = parsedData; - } else if ( - parsedData && - typeof parsedData === 'object' && - 'conv' in parsedData && - 'messages' in parsedData - ) { - importedData = [parsedData]; - } else { - throw new Error('Invalid file format'); - } - - const result = await DatabaseService.importConversations(importedData); - toast.success(`Imported ${result.imported} conversation(s), skipped ${result.skipped}`); - - await this.loadConversations(); - - const importedConversations = ( - Array.isArray(importedData) ? importedData : [importedData] - ).map((item) => item.conv); - - resolve(importedConversations); - } catch (err: unknown) { - const message = err instanceof Error ? err.message : 'Unknown error'; - console.error('Failed to import conversations:', err); - toast.error('Import failed', { description: message }); - reject(new Error(`Import failed: ${message}`)); - } - }; - - input.click(); - }); - } - /** * Imports conversations from provided data (without file picker) * @param data - Array of conversation data with messages - * @returns Import result with counts + * @returns The conversations written to the database and the ones skipped */ async importConversationsData( data: ExportedConversations - ): Promise<{ imported: number; skipped: number }> { + ): Promise<{ imported: DatabaseConversation[]; skipped: DatabaseConversation[] }> { const result = await DatabaseService.importConversations(data); await this.loadConversations(); return result; diff --git a/tools/ui/src/lib/utils/modality-file-validation.ts b/tools/ui/src/lib/utils/modality-file-validation.ts index bf78a7008..bfdee75ce 100644 --- a/tools/ui/src/lib/utils/modality-file-validation.ts +++ b/tools/ui/src/lib/utils/modality-file-validation.ts @@ -161,9 +161,3 @@ export function generateModalityErrorMessage( return message; } - -/** - * Generate file input accept string based on model modalities - * @param capabilities - The modality capabilities to check against - * @returns Accept string for HTML file input element - */ diff --git a/tools/ui/tests/client/conversation-import-db.svelte.test.ts b/tools/ui/tests/client/conversation-import-db.svelte.test.ts new file mode 100644 index 000000000..2a27be9b0 --- /dev/null +++ b/tools/ui/tests/client/conversation-import-db.svelte.test.ts @@ -0,0 +1,68 @@ +import { afterEach, describe, expect, it } from 'vitest'; +import { DatabaseService } from '$lib/services/database.service'; +import { MessageRole, MessageType } from '$lib/enums'; +import type { ExportedConversation } from '$lib/types/database'; + +function makeSession(id: string): ExportedConversation { + return { + conv: { id, currNode: `${id}-msg`, lastModified: 0, name: `Chat ${id}` }, + messages: [ + { + id: `${id}-msg`, + convId: id, + type: MessageType.TEXT, + timestamp: 0, + role: MessageRole.USER, + content: `hello from ${id}`, + parent: null, + children: [] + } + ] + } as unknown as ExportedConversation; +} + +afterEach(async () => { + const conversations = await DatabaseService.getAllConversations(); + await DatabaseService.bulkDeleteConversations(conversations.map((conv) => conv.id)); +}); + +/** + * An import leaves a conversation already in the database untouched, so the + * caller needs to know what was written to report it instead of echoing the + * selection back at the user. + */ +describe('DatabaseService.importConversations', () => { + it('reports the conversations it wrote', async () => { + const { imported, skipped } = await DatabaseService.importConversations([ + makeSession('a'), + makeSession('b') + ]); + + expect(imported.map((conv) => conv.id)).toEqual(['a', 'b']); + expect(skipped).toEqual([]); + expect(await DatabaseService.getConversationMessages('a')).toHaveLength(1); + }); + + it('reports an existing conversation as skipped and leaves it untouched', async () => { + await DatabaseService.importConversations([makeSession('a')]); + await DatabaseService.updateConversation('a', { name: 'Renamed locally' }); + + const { imported, skipped } = await DatabaseService.importConversations([makeSession('a')]); + + expect(imported).toEqual([]); + expect(skipped.map((conv) => conv.id)).toEqual(['a']); + expect((await DatabaseService.getConversation('a'))?.name).toBe('Renamed locally'); + }); + + it('imports the new conversations of a partially known selection', async () => { + await DatabaseService.importConversations([makeSession('a')]); + + const { imported, skipped } = await DatabaseService.importConversations([ + makeSession('a'), + makeSession('b') + ]); + + expect(imported.map((conv) => conv.id)).toEqual(['b']); + expect(skipped.map((conv) => conv.id)).toEqual(['a']); + }); +}); diff --git a/tools/ui/tests/unit/conversation-import.test.ts b/tools/ui/tests/unit/conversation-import.test.ts new file mode 100644 index 000000000..4565ed5b5 --- /dev/null +++ b/tools/ui/tests/unit/conversation-import.test.ts @@ -0,0 +1,112 @@ +import { beforeAll, describe, expect, it } from 'vitest'; +import { zipSync, strToU8 } from 'fflate'; +import { MessageRole, MessageType } from '$lib/enums'; +import { NEWLINE } from '$lib/constants'; +import type { ExportedConversation } from '$lib/types/database'; + +let conversationsStore: typeof import('$lib/stores/conversations.svelte').conversationsStore; + +// node env unit project has no DOM, install a minimal localStorage backed by a +// Map before the store module reads it. Transforming the store takes seconds, +// so import it once for the whole file. +beforeAll(async () => { + const store = new Map(); + const polyfill: Storage = { + get length() { + return store.size; + }, + clear: () => store.clear(), + getItem: (k) => (store.has(k) ? store.get(k)! : null), + key: (i) => Array.from(store.keys())[i] ?? null, + removeItem: (k) => { + store.delete(k); + }, + setItem: (k, v) => { + store.set(k, String(v)); + } + }; + (globalThis as unknown as { localStorage: Storage }).localStorage = polyfill; + + ({ conversationsStore } = await import('$lib/stores/conversations.svelte')); +}, 30000); + +function makeSession(id: string): ExportedConversation { + return { + conv: { id, currNode: `${id}-msg`, lastModified: 0, name: `Chat ${id}` }, + messages: [ + { + id: `${id}-msg`, + convId: id, + type: MessageType.TEXT, + timestamp: 0, + role: MessageRole.USER, + content: `hello from ${id}`, + parent: null, + children: [] + } + ] + } as unknown as ExportedConversation; +} + +/** + * `parseImportFile` detects the format from the file contents. iOS has no UTI + * for `.jsonl`, so the picker cannot filter on it and the filename carries no + * guarantee: a JSONL export must import under any name. + */ +describe('conversationsStore.parseImportFile', () => { + it('imports a JSONL export whose name has no meaningful extension', async () => { + const jsonl = conversationsStore.serializeSessionToJsonl(makeSession('a')); + + const sessions = await conversationsStore.parseImportFile(new File([jsonl], 'export')); + + expect(sessions).toHaveLength(1); + expect(sessions[0].conv.id).toBe('a'); + expect(sessions[0].messages[0].content).toBe('hello from a'); + }); + + it('imports several sessions from one JSONL file', async () => { + const jsonl = [makeSession('a'), makeSession('b')] + .map((session) => conversationsStore.serializeSessionToJsonl(session)) + .join(NEWLINE); + + const sessions = await conversationsStore.parseImportFile(new File([jsonl], 'export.txt')); + + expect(sessions.map((session) => session.conv.id)).toEqual(['a', 'b']); + }); + + it('imports a ZIP archive whose name has no meaningful extension', async () => { + const zipped = zipSync({ + 'a.jsonl': strToU8(conversationsStore.serializeSessionToJsonl(makeSession('a'))), + 'b.jsonl': strToU8(conversationsStore.serializeSessionToJsonl(makeSession('b'))), + 'notes.txt': strToU8('ignored') + }); + + const sessions = await conversationsStore.parseImportFile(new File([zipped], 'archive')); + + expect(sessions.map((session) => session.conv.id).sort()).toEqual(['a', 'b']); + }); + + it('imports the legacy JSON array format', async () => { + const json = JSON.stringify([makeSession('a')], null, 2); + + const sessions = await conversationsStore.parseImportFile(new File([json], 'export.jsonl')); + + expect(sessions).toHaveLength(1); + expect(sessions[0].conv.id).toBe('a'); + }); + + it('imports the legacy JSON single object format', async () => { + const json = JSON.stringify(makeSession('a')); + + const sessions = await conversationsStore.parseImportFile(new File([json], 'export')); + + expect(sessions).toHaveLength(1); + expect(sessions[0].conv.id).toBe('a'); + }); + + it('rejects a file that holds neither format', async () => { + await expect( + conversationsStore.parseImportFile(new File(['not an export'], 'export.jsonl')) + ).rejects.toThrow(); + }); +});