ui: detect the conversation import format from file contents (#26121)

* ui: detect the conversation import format from file contents

iOS resolves every accept entry to a UTI and has none for ".jsonl", so the
picker greyed out exported conversations. Drop the accept filter and pick the
parser from the file contents: ZIP magic bytes, then a first "session" record
for JSONL, otherwise the legacy JSON format.

Also remove the unused importConversations() picker and an orphan doc comment,
and cover each format with unit tests.

* ui: report what a conversation import actually wrote

The import summary echoed the selection back, so re-importing conversations
already in the database claimed success while nothing was written and only a
console warning said otherwise.

Return the imported and skipped conversations from the database layer, list the
written ones in the summary, and count the rest in a toast.

* ui: name the literals of the JSONL conversation format

Introduce SessionRecordType and SESSION_HARNESS, and reuse the existing
NEWLINE constant, so the record format lives in one place.

This also covers the writer side, which predates the import path under
review and carried the same literals: an enum stated by the reader alone
lets the two sides drift.

Values are unchanged, so an export stays byte identical.
This commit is contained in:
Pascal
2026-07-26 23:32:58 +02:00
committed by GitHub
parent d2a818231e
commit 55b7d6c4c7
11 changed files with 270 additions and 99 deletions
@@ -159,8 +159,10 @@
try { try {
const input = document.createElement('input'); const input = document.createElement('input');
// No `accept` filter: iOS resolves each entry to a UTI and has none for
// `.jsonl`, which greys out exported conversations in the file picker.
// `parseImportFile` detects the format from the file contents instead.
input.type = HtmlInputType.FILE; input.type = HtmlInputType.FILE;
input.accept = `${FileExtensionText.JSON},${FileExtensionText.JSONL},${FileExtensionText.ZIP}`;
input.onchange = async (e) => { input.onchange = async (e) => {
const file = (e.target as HTMLInputElement)?.files?.[0]; const file = (e.target as HTMLInputElement)?.files?.[0];
@@ -199,9 +201,17 @@
.snapshot(fullImportData) .snapshot(fullImportData)
.filter((item) => selectedIds.has(item.conv.id)); .filter((item) => selectedIds.has(item.conv.id));
await conversationsStore.importConversationsData(selectedData); const { imported, skipped } = await conversationsStore.importConversationsData(selectedData);
importedConversations = selectedConversations; // A conversation already in the database is left untouched, so the summary
// lists what was written and the toast accounts for the rest.
if (skipped.length > 0) {
toast.info(
`Skipped ${skipped.length} conversation${skipped.length === 1 ? '' : 's'} already in your library`
);
}
importedConversations = imported;
showImportSummary = true; showImportSummary = true;
showExportSummary = false; showExportSummary = false;
showImportDialog = false; showImportDialog = false;
@@ -0,0 +1,3 @@
// First bytes of every ZIP local file header ("PK"). Import detects an archive
// from these bytes rather than from the filename, which the OS may not preserve.
export const ZIP_MAGIC = [0x50, 0x4b];
+1
View File
@@ -13,6 +13,7 @@ export * from './storage';
export * from './attachment-menu'; export * from './attachment-menu';
export * from './auto-scroll'; export * from './auto-scroll';
export * from './context-gauge-popup'; export * from './context-gauge-popup';
export * from './conversation-import';
export * from './binary-detection'; export * from './binary-detection';
export * from './built-in-tools'; export * from './built-in-tools';
export * from './cache'; export * from './cache';
@@ -7,6 +7,9 @@ export const EXPORT_CONV_NAME_SUFFIX_MAX_LENGTH = 20;
// Characters to keep in the ISO timestamp. 19 keeps 2026-01-01T00:00:00 // Characters to keep in the ISO timestamp. 19 keeps 2026-01-01T00:00:00
export const ISO_TIMESTAMP_SLICE_LENGTH = 19; export const ISO_TIMESTAMP_SLICE_LENGTH = 19;
// Producer marker carried by the session record of a JSONL export
export const SESSION_HARNESS = 'llama.app';
// Replacements for making the conversation title filename-friendly // Replacements for making the conversation title filename-friendly
export const NON_ALPHANUMERIC_REGEX = /[^a-z0-9]/gi; export const NON_ALPHANUMERIC_REGEX = /[^a-z0-9]/gi;
export const EXPORT_CONV_NONALNUM_REPLACEMENT = '_'; export const EXPORT_CONV_NONALNUM_REPLACEMENT = '_';
@@ -0,0 +1,9 @@
/**
* Discriminator of a record line in the JSONL conversation format. A session
* record opens a conversation and carries its properties; every following
* message record belongs to it.
*/
export enum SessionRecordType {
SESSION = 'session',
MESSAGE = 'message'
}
+2
View File
@@ -27,6 +27,8 @@ export {
ReasoningFormat ReasoningFormat
} from './chat.enums'; } from './chat.enums';
export { SessionRecordType } from './conversation-import.enums';
export { ReasoningEffort } from './reasoning-effort.enums'; export { ReasoningEffort } from './reasoning-effort.enums';
export { export {
@@ -554,12 +554,13 @@ export class DatabaseService {
* Skips conversations that already exist. * Skips conversations that already exist.
* *
* @param data - Array of { conv, messages } objects * @param data - Array of { conv, messages } objects
* @returns The conversations written to the database and the ones skipped
*/ */
static async importConversations( static async importConversations(
data: { conv: DatabaseConversation; messages: DatabaseMessage[] }[] data: { conv: DatabaseConversation; messages: DatabaseMessage[] }[]
): Promise<{ imported: number; skipped: number }> { ): Promise<{ imported: DatabaseConversation[]; skipped: DatabaseConversation[] }> {
let importedCount = 0; const imported: DatabaseConversation[] = [];
let skippedCount = 0; const skipped: DatabaseConversation[] = [];
return await db.transaction( return await db.transaction(
'rw', 'rw',
@@ -570,8 +571,7 @@ export class DatabaseService {
const existing = await db[IDXDB_TABLES.conversations].get(conv.id); const existing = await db[IDXDB_TABLES.conversations].get(conv.id);
if (existing) { if (existing) {
console.warn(`Conversation "${conv.name}" already exists, skipping...`); skipped.push(conv);
skippedCount++;
continue; continue;
} }
@@ -580,10 +580,10 @@ export class DatabaseService {
await db[IDXDB_TABLES.messages].put(msg); await db[IDXDB_TABLES.messages].put(msg);
} }
importedCount++; imported.push(conv);
} }
return { imported: importedCount, skipped: skippedCount }; return { imported, skipped };
} }
); );
} }
+52 -83
View File
@@ -30,11 +30,11 @@ import type { McpServerOverride } from '$lib/types/database';
import { zipSync, unzipSync, strToU8, strFromU8 } from 'fflate'; import { zipSync, unzipSync, strToU8, strFromU8 } from 'fflate';
import { import {
MessageRole, MessageRole,
HtmlInputType,
FileExtensionText, FileExtensionText,
MimeTypeText, MimeTypeText,
MimeTypeApplication, MimeTypeApplication,
ReasoningEffort ReasoningEffort,
SessionRecordType
} from '$lib/enums'; } from '$lib/enums';
import { import {
ISO_DATE_TIME_SEPARATOR, ISO_DATE_TIME_SEPARATOR,
@@ -47,7 +47,10 @@ import {
ISO_TIME_SEPARATOR_REPLACEMENT, ISO_TIME_SEPARATOR_REPLACEMENT,
NON_ALPHANUMERIC_REGEX, NON_ALPHANUMERIC_REGEX,
MULTIPLE_UNDERSCORE_REGEX, MULTIPLE_UNDERSCORE_REGEX,
REASONING_EFFORT_DEFAULT_LOCALSTORAGE_KEY REASONING_EFFORT_DEFAULT_LOCALSTORAGE_KEY,
NEWLINE,
SESSION_HARNESS,
ZIP_MAGIC
} from '$lib/constants'; } from '$lib/constants';
import { ROUTES } from '$lib/constants/routes'; import { ROUTES } from '$lib/constants/routes';
@@ -914,30 +917,35 @@ class ConversationsStore {
/** /**
* Serializes a session (a conversation with its messages) as JSONL. * Serializes a session (a conversation with its messages) as JSONL.
* The first line is the session header (a `type: 'session'` record carrying the * The first line is the session header (a `SessionRecordType.SESSION` record
* conversation properties); each subsequent line is a single message. * carrying the conversation properties); each subsequent line is a single message.
* @param data - The exported conversation payload * @param data - The exported conversation payload
* @returns The JSONL string (one record per line) * @returns The JSONL string (one record per line)
*/ */
serializeSessionToJsonl(data: ExportedConversation): string { serializeSessionToJsonl(data: ExportedConversation): string {
const { conv, messages } = data; const { conv, messages } = data;
const sessionLine = JSON.stringify({ type: 'session', harness: 'llama.app', ...conv }); const sessionLine = JSON.stringify({
type: SessionRecordType.SESSION,
harness: SESSION_HARNESS,
...conv
});
const messageLines = messages.map((message: DatabaseMessage) => { const messageLines = messages.map((message: DatabaseMessage) => {
// `toolCalls` is stored as a JSON string; drop it when empty, otherwise parse it. // `toolCalls` is stored as a JSON string; drop it when empty, otherwise parse it.
const { toolCalls, ...rest } = message; const { toolCalls, ...rest } = message;
const normalized = toolCalls ? { ...rest, toolCalls: JSON.parse(toolCalls) } : rest; const normalized = toolCalls ? { ...rest, toolCalls: JSON.parse(toolCalls) } : rest;
return JSON.stringify({ type: 'message', message: normalized }); return JSON.stringify({ type: SessionRecordType.MESSAGE, message: normalized });
}); });
return [sessionLine, ...messageLines].join('\n'); return [sessionLine, ...messageLines].join(NEWLINE);
} }
/** /**
* Parses the JSONL session format produced by {@link serializeSessionToJsonl}. * Parses the JSONL session format produced by {@link serializeSessionToJsonl}.
* A `type: 'session'` line starts a new session; following `type: 'message'` * A `SessionRecordType.SESSION` line starts a new session; following
* lines are appended to it. Supports multiple sessions in a single file. * `SessionRecordType.MESSAGE` lines are appended to it. Supports multiple
* sessions in a single file.
* @param text - The JSONL file contents * @param text - The JSONL file contents
* @returns The parsed conversations with their messages * @returns The parsed conversations with their messages
*/ */
@@ -945,20 +953,20 @@ class ConversationsStore {
const sessions: ExportedConversation[] = []; const sessions: ExportedConversation[] = [];
let current: ExportedConversation | null = null; let current: ExportedConversation | null = null;
for (const line of text.split('\n')) { for (const line of text.split(NEWLINE)) {
const trimmed = line.trim(); const trimmed = line.trim();
if (!trimmed) continue; if (!trimmed) continue;
const record = JSON.parse(trimmed); const record = JSON.parse(trimmed);
if (record.type === 'session') { if (record.type === SessionRecordType.SESSION) {
// Drop the discriminator and harness marker; the rest is the conversation. // Drop the discriminator and harness marker; the rest is the conversation.
const conv = { ...record }; const conv = { ...record };
delete conv.type; delete conv.type;
delete conv.harness; delete conv.harness;
current = { conv: conv as DatabaseConversation, messages: [] }; current = { conv: conv as DatabaseConversation, messages: [] };
sessions.push(current); sessions.push(current);
} else if (record.type === 'message') { } else if (record.type === SessionRecordType.MESSAGE) {
if (!current) { if (!current) {
throw new Error('Invalid JSONL: message record before any session record'); throw new Error('Invalid JSONL: message record before any session record');
} }
@@ -977,27 +985,47 @@ class ConversationsStore {
} }
/** /**
* Parses an import file into conversations, accepting the current `.jsonl` and * Reports whether the text is the JSONL session format, whose first non-empty
* `.zip` formats as well as the legacy `.json` format. * line is a `SessionRecordType.SESSION` record. A legacy JSON export starts
* with an array or an object that has no such discriminator.
* @param text - The file contents
*/
private isSessionsJsonl(text: string): boolean {
const trimmed = text.trimStart();
const lineEnd = trimmed.indexOf(NEWLINE);
const firstLine = lineEnd === -1 ? trimmed : trimmed.slice(0, lineEnd);
try {
return JSON.parse(firstLine).type === SessionRecordType.SESSION;
} catch {
// Not a standalone JSON record, so not the JSONL format.
return false;
}
}
/**
* Parses an import file into conversations, accepting the current JSONL and
* ZIP formats as well as the legacy JSON format. The format comes from the
* contents, so an import works whatever the file is named.
* @param file - The user-selected file * @param file - The user-selected file
* @returns The parsed conversations with their messages * @returns The parsed conversations with their messages
*/ */
async parseImportFile(file: File): Promise<ExportedConversation[]> { async parseImportFile(file: File): Promise<ExportedConversation[]> {
const name = file.name.toLowerCase(); const bytes = new Uint8Array(await file.arrayBuffer());
if (name.endsWith(FileExtensionText.ZIP)) { if (ZIP_MAGIC.every((byte, index) => bytes[index] === byte)) {
const entries = unzipSync(new Uint8Array(await file.arrayBuffer())); const entries = unzipSync(bytes);
const sessions: ExportedConversation[] = []; const sessions: ExportedConversation[] = [];
for (const [entryName, bytes] of Object.entries(entries)) { for (const [entryName, entryBytes] of Object.entries(entries)) {
if (!entryName.toLowerCase().endsWith(FileExtensionText.JSONL)) continue; if (!entryName.toLowerCase().endsWith(FileExtensionText.JSONL)) continue;
sessions.push(...this.parseSessionsJsonl(strFromU8(bytes))); sessions.push(...this.parseSessionsJsonl(strFromU8(entryBytes)));
} }
return sessions; return sessions;
} }
const text = await file.text(); const text = strFromU8(bytes);
if (name.endsWith(FileExtensionText.JSONL)) { if (this.isSessionsJsonl(text)) {
return this.parseSessionsJsonl(text); return this.parseSessionsJsonl(text);
} }
@@ -1103,73 +1131,14 @@ class ConversationsStore {
this.downloadConversationFile({ conv: conversation, messages }); this.downloadConversationFile({ conv: conversation, messages });
} }
/**
* Imports conversations from a JSON file
* Opens file picker and processes the selected file
* @returns The list of imported conversations
*/
async importConversations(): Promise<DatabaseConversation[]> {
return new Promise((resolve, reject) => {
const input = document.createElement('input');
input.type = HtmlInputType.FILE;
input.accept = FileExtensionText.JSON;
input.onchange = async (e) => {
const file = (e.target as HTMLInputElement)?.files?.[0];
if (!file) {
reject(new Error('No file selected'));
return;
}
try {
const text = await file.text();
const parsedData = JSON.parse(text);
let importedData: ExportedConversations;
if (Array.isArray(parsedData)) {
importedData = parsedData;
} else if (
parsedData &&
typeof parsedData === 'object' &&
'conv' in parsedData &&
'messages' in parsedData
) {
importedData = [parsedData];
} else {
throw new Error('Invalid file format');
}
const result = await DatabaseService.importConversations(importedData);
toast.success(`Imported ${result.imported} conversation(s), skipped ${result.skipped}`);
await this.loadConversations();
const importedConversations = (
Array.isArray(importedData) ? importedData : [importedData]
).map((item) => item.conv);
resolve(importedConversations);
} catch (err: unknown) {
const message = err instanceof Error ? err.message : 'Unknown error';
console.error('Failed to import conversations:', err);
toast.error('Import failed', { description: message });
reject(new Error(`Import failed: ${message}`));
}
};
input.click();
});
}
/** /**
* Imports conversations from provided data (without file picker) * Imports conversations from provided data (without file picker)
* @param data - Array of conversation data with messages * @param data - Array of conversation data with messages
* @returns Import result with counts * @returns The conversations written to the database and the ones skipped
*/ */
async importConversationsData( async importConversationsData(
data: ExportedConversations data: ExportedConversations
): Promise<{ imported: number; skipped: number }> { ): Promise<{ imported: DatabaseConversation[]; skipped: DatabaseConversation[] }> {
const result = await DatabaseService.importConversations(data); const result = await DatabaseService.importConversations(data);
await this.loadConversations(); await this.loadConversations();
return result; return result;
@@ -161,9 +161,3 @@ export function generateModalityErrorMessage(
return message; return message;
} }
/**
* Generate file input accept string based on model modalities
* @param capabilities - The modality capabilities to check against
* @returns Accept string for HTML file input element
*/
@@ -0,0 +1,68 @@
import { afterEach, describe, expect, it } from 'vitest';
import { DatabaseService } from '$lib/services/database.service';
import { MessageRole, MessageType } from '$lib/enums';
import type { ExportedConversation } from '$lib/types/database';
function makeSession(id: string): ExportedConversation {
return {
conv: { id, currNode: `${id}-msg`, lastModified: 0, name: `Chat ${id}` },
messages: [
{
id: `${id}-msg`,
convId: id,
type: MessageType.TEXT,
timestamp: 0,
role: MessageRole.USER,
content: `hello from ${id}`,
parent: null,
children: []
}
]
} as unknown as ExportedConversation;
}
afterEach(async () => {
const conversations = await DatabaseService.getAllConversations();
await DatabaseService.bulkDeleteConversations(conversations.map((conv) => conv.id));
});
/**
* An import leaves a conversation already in the database untouched, so the
* caller needs to know what was written to report it instead of echoing the
* selection back at the user.
*/
describe('DatabaseService.importConversations', () => {
it('reports the conversations it wrote', async () => {
const { imported, skipped } = await DatabaseService.importConversations([
makeSession('a'),
makeSession('b')
]);
expect(imported.map((conv) => conv.id)).toEqual(['a', 'b']);
expect(skipped).toEqual([]);
expect(await DatabaseService.getConversationMessages('a')).toHaveLength(1);
});
it('reports an existing conversation as skipped and leaves it untouched', async () => {
await DatabaseService.importConversations([makeSession('a')]);
await DatabaseService.updateConversation('a', { name: 'Renamed locally' });
const { imported, skipped } = await DatabaseService.importConversations([makeSession('a')]);
expect(imported).toEqual([]);
expect(skipped.map((conv) => conv.id)).toEqual(['a']);
expect((await DatabaseService.getConversation('a'))?.name).toBe('Renamed locally');
});
it('imports the new conversations of a partially known selection', async () => {
await DatabaseService.importConversations([makeSession('a')]);
const { imported, skipped } = await DatabaseService.importConversations([
makeSession('a'),
makeSession('b')
]);
expect(imported.map((conv) => conv.id)).toEqual(['b']);
expect(skipped.map((conv) => conv.id)).toEqual(['a']);
});
});
@@ -0,0 +1,112 @@
import { beforeAll, describe, expect, it } from 'vitest';
import { zipSync, strToU8 } from 'fflate';
import { MessageRole, MessageType } from '$lib/enums';
import { NEWLINE } from '$lib/constants';
import type { ExportedConversation } from '$lib/types/database';
let conversationsStore: typeof import('$lib/stores/conversations.svelte').conversationsStore;
// node env unit project has no DOM, install a minimal localStorage backed by a
// Map before the store module reads it. Transforming the store takes seconds,
// so import it once for the whole file.
beforeAll(async () => {
const store = new Map<string, string>();
const polyfill: Storage = {
get length() {
return store.size;
},
clear: () => store.clear(),
getItem: (k) => (store.has(k) ? store.get(k)! : null),
key: (i) => Array.from(store.keys())[i] ?? null,
removeItem: (k) => {
store.delete(k);
},
setItem: (k, v) => {
store.set(k, String(v));
}
};
(globalThis as unknown as { localStorage: Storage }).localStorage = polyfill;
({ conversationsStore } = await import('$lib/stores/conversations.svelte'));
}, 30000);
function makeSession(id: string): ExportedConversation {
return {
conv: { id, currNode: `${id}-msg`, lastModified: 0, name: `Chat ${id}` },
messages: [
{
id: `${id}-msg`,
convId: id,
type: MessageType.TEXT,
timestamp: 0,
role: MessageRole.USER,
content: `hello from ${id}`,
parent: null,
children: []
}
]
} as unknown as ExportedConversation;
}
/**
* `parseImportFile` detects the format from the file contents. iOS has no UTI
* for `.jsonl`, so the picker cannot filter on it and the filename carries no
* guarantee: a JSONL export must import under any name.
*/
describe('conversationsStore.parseImportFile', () => {
it('imports a JSONL export whose name has no meaningful extension', async () => {
const jsonl = conversationsStore.serializeSessionToJsonl(makeSession('a'));
const sessions = await conversationsStore.parseImportFile(new File([jsonl], 'export'));
expect(sessions).toHaveLength(1);
expect(sessions[0].conv.id).toBe('a');
expect(sessions[0].messages[0].content).toBe('hello from a');
});
it('imports several sessions from one JSONL file', async () => {
const jsonl = [makeSession('a'), makeSession('b')]
.map((session) => conversationsStore.serializeSessionToJsonl(session))
.join(NEWLINE);
const sessions = await conversationsStore.parseImportFile(new File([jsonl], 'export.txt'));
expect(sessions.map((session) => session.conv.id)).toEqual(['a', 'b']);
});
it('imports a ZIP archive whose name has no meaningful extension', async () => {
const zipped = zipSync({
'a.jsonl': strToU8(conversationsStore.serializeSessionToJsonl(makeSession('a'))),
'b.jsonl': strToU8(conversationsStore.serializeSessionToJsonl(makeSession('b'))),
'notes.txt': strToU8('ignored')
});
const sessions = await conversationsStore.parseImportFile(new File([zipped], 'archive'));
expect(sessions.map((session) => session.conv.id).sort()).toEqual(['a', 'b']);
});
it('imports the legacy JSON array format', async () => {
const json = JSON.stringify([makeSession('a')], null, 2);
const sessions = await conversationsStore.parseImportFile(new File([json], 'export.jsonl'));
expect(sessions).toHaveLength(1);
expect(sessions[0].conv.id).toBe('a');
});
it('imports the legacy JSON single object format', async () => {
const json = JSON.stringify(makeSession('a'));
const sessions = await conversationsStore.parseImportFile(new File([json], 'export'));
expect(sessions).toHaveLength(1);
expect(sessions[0].conv.id).toBe('a');
});
it('rejects a file that holds neither format', async () => {
await expect(
conversationsStore.parseImportFile(new File(['not an export'], 'export.jsonl'))
).rejects.toThrow();
});
});