ui: detect the conversation import format from file contents (#26121)
* ui: detect the conversation import format from file contents iOS resolves every accept entry to a UTI and has none for ".jsonl", so the picker greyed out exported conversations. Drop the accept filter and pick the parser from the file contents: ZIP magic bytes, then a first "session" record for JSONL, otherwise the legacy JSON format. Also remove the unused importConversations() picker and an orphan doc comment, and cover each format with unit tests. * ui: report what a conversation import actually wrote The import summary echoed the selection back, so re-importing conversations already in the database claimed success while nothing was written and only a console warning said otherwise. Return the imported and skipped conversations from the database layer, list the written ones in the summary, and count the rest in a toast. * ui: name the literals of the JSONL conversation format Introduce SessionRecordType and SESSION_HARNESS, and reuse the existing NEWLINE constant, so the record format lives in one place. This also covers the writer side, which predates the import path under review and carried the same literals: an enum stated by the reader alone lets the two sides drift. Values are unchanged, so an export stays byte identical.
This commit is contained in:
Pascal
+13
-3
@@ -159,8 +159,10 @@
|
||||
try {
|
||||
const input = document.createElement('input');
|
||||
|
||||
// No `accept` filter: iOS resolves each entry to a UTI and has none for
|
||||
// `.jsonl`, which greys out exported conversations in the file picker.
|
||||
// `parseImportFile` detects the format from the file contents instead.
|
||||
input.type = HtmlInputType.FILE;
|
||||
input.accept = `${FileExtensionText.JSON},${FileExtensionText.JSONL},${FileExtensionText.ZIP}`;
|
||||
|
||||
input.onchange = async (e) => {
|
||||
const file = (e.target as HTMLInputElement)?.files?.[0];
|
||||
@@ -199,9 +201,17 @@
|
||||
.snapshot(fullImportData)
|
||||
.filter((item) => selectedIds.has(item.conv.id));
|
||||
|
||||
await conversationsStore.importConversationsData(selectedData);
|
||||
const { imported, skipped } = await conversationsStore.importConversationsData(selectedData);
|
||||
|
||||
importedConversations = selectedConversations;
|
||||
// A conversation already in the database is left untouched, so the summary
|
||||
// lists what was written and the toast accounts for the rest.
|
||||
if (skipped.length > 0) {
|
||||
toast.info(
|
||||
`Skipped ${skipped.length} conversation${skipped.length === 1 ? '' : 's'} already in your library`
|
||||
);
|
||||
}
|
||||
|
||||
importedConversations = imported;
|
||||
showImportSummary = true;
|
||||
showExportSummary = false;
|
||||
showImportDialog = false;
|
||||
|
||||
@@ -0,0 +1,3 @@
|
||||
// First bytes of every ZIP local file header ("PK"). Import detects an archive
|
||||
// from these bytes rather than from the filename, which the OS may not preserve.
|
||||
export const ZIP_MAGIC = [0x50, 0x4b];
|
||||
@@ -13,6 +13,7 @@ export * from './storage';
|
||||
export * from './attachment-menu';
|
||||
export * from './auto-scroll';
|
||||
export * from './context-gauge-popup';
|
||||
export * from './conversation-import';
|
||||
export * from './binary-detection';
|
||||
export * from './built-in-tools';
|
||||
export * from './cache';
|
||||
|
||||
@@ -7,6 +7,9 @@ export const EXPORT_CONV_NAME_SUFFIX_MAX_LENGTH = 20;
|
||||
// Characters to keep in the ISO timestamp. 19 keeps 2026-01-01T00:00:00
|
||||
export const ISO_TIMESTAMP_SLICE_LENGTH = 19;
|
||||
|
||||
// Producer marker carried by the session record of a JSONL export
|
||||
export const SESSION_HARNESS = 'llama.app';
|
||||
|
||||
// Replacements for making the conversation title filename-friendly
|
||||
export const NON_ALPHANUMERIC_REGEX = /[^a-z0-9]/gi;
|
||||
export const EXPORT_CONV_NONALNUM_REPLACEMENT = '_';
|
||||
|
||||
@@ -0,0 +1,9 @@
|
||||
/**
|
||||
* Discriminator of a record line in the JSONL conversation format. A session
|
||||
* record opens a conversation and carries its properties; every following
|
||||
* message record belongs to it.
|
||||
*/
|
||||
export enum SessionRecordType {
|
||||
SESSION = 'session',
|
||||
MESSAGE = 'message'
|
||||
}
|
||||
@@ -27,6 +27,8 @@ export {
|
||||
ReasoningFormat
|
||||
} from './chat.enums';
|
||||
|
||||
export { SessionRecordType } from './conversation-import.enums';
|
||||
|
||||
export { ReasoningEffort } from './reasoning-effort.enums';
|
||||
|
||||
export {
|
||||
|
||||
@@ -554,12 +554,13 @@ export class DatabaseService {
|
||||
* Skips conversations that already exist.
|
||||
*
|
||||
* @param data - Array of { conv, messages } objects
|
||||
* @returns The conversations written to the database and the ones skipped
|
||||
*/
|
||||
static async importConversations(
|
||||
data: { conv: DatabaseConversation; messages: DatabaseMessage[] }[]
|
||||
): Promise<{ imported: number; skipped: number }> {
|
||||
let importedCount = 0;
|
||||
let skippedCount = 0;
|
||||
): Promise<{ imported: DatabaseConversation[]; skipped: DatabaseConversation[] }> {
|
||||
const imported: DatabaseConversation[] = [];
|
||||
const skipped: DatabaseConversation[] = [];
|
||||
|
||||
return await db.transaction(
|
||||
'rw',
|
||||
@@ -570,8 +571,7 @@ export class DatabaseService {
|
||||
|
||||
const existing = await db[IDXDB_TABLES.conversations].get(conv.id);
|
||||
if (existing) {
|
||||
console.warn(`Conversation "${conv.name}" already exists, skipping...`);
|
||||
skippedCount++;
|
||||
skipped.push(conv);
|
||||
continue;
|
||||
}
|
||||
|
||||
@@ -580,10 +580,10 @@ export class DatabaseService {
|
||||
await db[IDXDB_TABLES.messages].put(msg);
|
||||
}
|
||||
|
||||
importedCount++;
|
||||
imported.push(conv);
|
||||
}
|
||||
|
||||
return { imported: importedCount, skipped: skippedCount };
|
||||
return { imported, skipped };
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
@@ -30,11 +30,11 @@ import type { McpServerOverride } from '$lib/types/database';
|
||||
import { zipSync, unzipSync, strToU8, strFromU8 } from 'fflate';
|
||||
import {
|
||||
MessageRole,
|
||||
HtmlInputType,
|
||||
FileExtensionText,
|
||||
MimeTypeText,
|
||||
MimeTypeApplication,
|
||||
ReasoningEffort
|
||||
ReasoningEffort,
|
||||
SessionRecordType
|
||||
} from '$lib/enums';
|
||||
import {
|
||||
ISO_DATE_TIME_SEPARATOR,
|
||||
@@ -47,7 +47,10 @@ import {
|
||||
ISO_TIME_SEPARATOR_REPLACEMENT,
|
||||
NON_ALPHANUMERIC_REGEX,
|
||||
MULTIPLE_UNDERSCORE_REGEX,
|
||||
REASONING_EFFORT_DEFAULT_LOCALSTORAGE_KEY
|
||||
REASONING_EFFORT_DEFAULT_LOCALSTORAGE_KEY,
|
||||
NEWLINE,
|
||||
SESSION_HARNESS,
|
||||
ZIP_MAGIC
|
||||
} from '$lib/constants';
|
||||
|
||||
import { ROUTES } from '$lib/constants/routes';
|
||||
@@ -914,30 +917,35 @@ class ConversationsStore {
|
||||
|
||||
/**
|
||||
* Serializes a session (a conversation with its messages) as JSONL.
|
||||
* The first line is the session header (a `type: 'session'` record carrying the
|
||||
* conversation properties); each subsequent line is a single message.
|
||||
* The first line is the session header (a `SessionRecordType.SESSION` record
|
||||
* carrying the conversation properties); each subsequent line is a single message.
|
||||
* @param data - The exported conversation payload
|
||||
* @returns The JSONL string (one record per line)
|
||||
*/
|
||||
serializeSessionToJsonl(data: ExportedConversation): string {
|
||||
const { conv, messages } = data;
|
||||
|
||||
const sessionLine = JSON.stringify({ type: 'session', harness: 'llama.app', ...conv });
|
||||
const sessionLine = JSON.stringify({
|
||||
type: SessionRecordType.SESSION,
|
||||
harness: SESSION_HARNESS,
|
||||
...conv
|
||||
});
|
||||
const messageLines = messages.map((message: DatabaseMessage) => {
|
||||
// `toolCalls` is stored as a JSON string; drop it when empty, otherwise parse it.
|
||||
const { toolCalls, ...rest } = message;
|
||||
const normalized = toolCalls ? { ...rest, toolCalls: JSON.parse(toolCalls) } : rest;
|
||||
|
||||
return JSON.stringify({ type: 'message', message: normalized });
|
||||
return JSON.stringify({ type: SessionRecordType.MESSAGE, message: normalized });
|
||||
});
|
||||
|
||||
return [sessionLine, ...messageLines].join('\n');
|
||||
return [sessionLine, ...messageLines].join(NEWLINE);
|
||||
}
|
||||
|
||||
/**
|
||||
* Parses the JSONL session format produced by {@link serializeSessionToJsonl}.
|
||||
* A `type: 'session'` line starts a new session; following `type: 'message'`
|
||||
* lines are appended to it. Supports multiple sessions in a single file.
|
||||
* A `SessionRecordType.SESSION` line starts a new session; following
|
||||
* `SessionRecordType.MESSAGE` lines are appended to it. Supports multiple
|
||||
* sessions in a single file.
|
||||
* @param text - The JSONL file contents
|
||||
* @returns The parsed conversations with their messages
|
||||
*/
|
||||
@@ -945,20 +953,20 @@ class ConversationsStore {
|
||||
const sessions: ExportedConversation[] = [];
|
||||
let current: ExportedConversation | null = null;
|
||||
|
||||
for (const line of text.split('\n')) {
|
||||
for (const line of text.split(NEWLINE)) {
|
||||
const trimmed = line.trim();
|
||||
if (!trimmed) continue;
|
||||
|
||||
const record = JSON.parse(trimmed);
|
||||
|
||||
if (record.type === 'session') {
|
||||
if (record.type === SessionRecordType.SESSION) {
|
||||
// Drop the discriminator and harness marker; the rest is the conversation.
|
||||
const conv = { ...record };
|
||||
delete conv.type;
|
||||
delete conv.harness;
|
||||
current = { conv: conv as DatabaseConversation, messages: [] };
|
||||
sessions.push(current);
|
||||
} else if (record.type === 'message') {
|
||||
} else if (record.type === SessionRecordType.MESSAGE) {
|
||||
if (!current) {
|
||||
throw new Error('Invalid JSONL: message record before any session record');
|
||||
}
|
||||
@@ -977,27 +985,47 @@ class ConversationsStore {
|
||||
}
|
||||
|
||||
/**
|
||||
* Parses an import file into conversations, accepting the current `.jsonl` and
|
||||
* `.zip` formats as well as the legacy `.json` format.
|
||||
* Reports whether the text is the JSONL session format, whose first non-empty
|
||||
* line is a `SessionRecordType.SESSION` record. A legacy JSON export starts
|
||||
* with an array or an object that has no such discriminator.
|
||||
* @param text - The file contents
|
||||
*/
|
||||
private isSessionsJsonl(text: string): boolean {
|
||||
const trimmed = text.trimStart();
|
||||
const lineEnd = trimmed.indexOf(NEWLINE);
|
||||
const firstLine = lineEnd === -1 ? trimmed : trimmed.slice(0, lineEnd);
|
||||
|
||||
try {
|
||||
return JSON.parse(firstLine).type === SessionRecordType.SESSION;
|
||||
} catch {
|
||||
// Not a standalone JSON record, so not the JSONL format.
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Parses an import file into conversations, accepting the current JSONL and
|
||||
* ZIP formats as well as the legacy JSON format. The format comes from the
|
||||
* contents, so an import works whatever the file is named.
|
||||
* @param file - The user-selected file
|
||||
* @returns The parsed conversations with their messages
|
||||
*/
|
||||
async parseImportFile(file: File): Promise<ExportedConversation[]> {
|
||||
const name = file.name.toLowerCase();
|
||||
const bytes = new Uint8Array(await file.arrayBuffer());
|
||||
|
||||
if (name.endsWith(FileExtensionText.ZIP)) {
|
||||
const entries = unzipSync(new Uint8Array(await file.arrayBuffer()));
|
||||
if (ZIP_MAGIC.every((byte, index) => bytes[index] === byte)) {
|
||||
const entries = unzipSync(bytes);
|
||||
const sessions: ExportedConversation[] = [];
|
||||
for (const [entryName, bytes] of Object.entries(entries)) {
|
||||
for (const [entryName, entryBytes] of Object.entries(entries)) {
|
||||
if (!entryName.toLowerCase().endsWith(FileExtensionText.JSONL)) continue;
|
||||
sessions.push(...this.parseSessionsJsonl(strFromU8(bytes)));
|
||||
sessions.push(...this.parseSessionsJsonl(strFromU8(entryBytes)));
|
||||
}
|
||||
return sessions;
|
||||
}
|
||||
|
||||
const text = await file.text();
|
||||
const text = strFromU8(bytes);
|
||||
|
||||
if (name.endsWith(FileExtensionText.JSONL)) {
|
||||
if (this.isSessionsJsonl(text)) {
|
||||
return this.parseSessionsJsonl(text);
|
||||
}
|
||||
|
||||
@@ -1103,73 +1131,14 @@ class ConversationsStore {
|
||||
this.downloadConversationFile({ conv: conversation, messages });
|
||||
}
|
||||
|
||||
/**
|
||||
* Imports conversations from a JSON file
|
||||
* Opens file picker and processes the selected file
|
||||
* @returns The list of imported conversations
|
||||
*/
|
||||
async importConversations(): Promise<DatabaseConversation[]> {
|
||||
return new Promise((resolve, reject) => {
|
||||
const input = document.createElement('input');
|
||||
input.type = HtmlInputType.FILE;
|
||||
input.accept = FileExtensionText.JSON;
|
||||
|
||||
input.onchange = async (e) => {
|
||||
const file = (e.target as HTMLInputElement)?.files?.[0];
|
||||
|
||||
if (!file) {
|
||||
reject(new Error('No file selected'));
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
const text = await file.text();
|
||||
const parsedData = JSON.parse(text);
|
||||
let importedData: ExportedConversations;
|
||||
|
||||
if (Array.isArray(parsedData)) {
|
||||
importedData = parsedData;
|
||||
} else if (
|
||||
parsedData &&
|
||||
typeof parsedData === 'object' &&
|
||||
'conv' in parsedData &&
|
||||
'messages' in parsedData
|
||||
) {
|
||||
importedData = [parsedData];
|
||||
} else {
|
||||
throw new Error('Invalid file format');
|
||||
}
|
||||
|
||||
const result = await DatabaseService.importConversations(importedData);
|
||||
toast.success(`Imported ${result.imported} conversation(s), skipped ${result.skipped}`);
|
||||
|
||||
await this.loadConversations();
|
||||
|
||||
const importedConversations = (
|
||||
Array.isArray(importedData) ? importedData : [importedData]
|
||||
).map((item) => item.conv);
|
||||
|
||||
resolve(importedConversations);
|
||||
} catch (err: unknown) {
|
||||
const message = err instanceof Error ? err.message : 'Unknown error';
|
||||
console.error('Failed to import conversations:', err);
|
||||
toast.error('Import failed', { description: message });
|
||||
reject(new Error(`Import failed: ${message}`));
|
||||
}
|
||||
};
|
||||
|
||||
input.click();
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Imports conversations from provided data (without file picker)
|
||||
* @param data - Array of conversation data with messages
|
||||
* @returns Import result with counts
|
||||
* @returns The conversations written to the database and the ones skipped
|
||||
*/
|
||||
async importConversationsData(
|
||||
data: ExportedConversations
|
||||
): Promise<{ imported: number; skipped: number }> {
|
||||
): Promise<{ imported: DatabaseConversation[]; skipped: DatabaseConversation[] }> {
|
||||
const result = await DatabaseService.importConversations(data);
|
||||
await this.loadConversations();
|
||||
return result;
|
||||
|
||||
@@ -161,9 +161,3 @@ export function generateModalityErrorMessage(
|
||||
|
||||
return message;
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate file input accept string based on model modalities
|
||||
* @param capabilities - The modality capabilities to check against
|
||||
* @returns Accept string for HTML file input element
|
||||
*/
|
||||
|
||||
Reference in New Issue
Block a user