Lumpiasty/llama.cpp
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

refactor: Naming (#27001)

Browse Source
This commit is contained in:
Aleksander Grygier
2026-08-13 19:45:32 +02:00
committed by GitHub
parent 9c5531e2bf
commit fa4ec4590c
28 changed files with 275 additions and 212 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tools/ui/src/lib/utils/contenteditable-tokenizer.ts → tools/ui/src/lib/utils/chat-form-input-rich-tokenizer.ts
+43 -39
View File
@@ -1,5 +1,5 @@
/**
* Maps between the chat-form contenteditable's markdown source and the
* Maps between the chat-form-input-rich's markdown source and the
* badge/code/text token stream the DOM is built from. A badge is one
* opaque source contribution (`[name](file://path)`); its own subtree
* is never walked, and the caret cannot land inside it, so offsets
@@ -35,10 +35,10 @@ import {
MENTION_BADGE_SVG_ATTRIBUTES,
SETTINGS_KEYS
} from '$lib/constants';
import { ContentEditableTokenKind } from '$lib/enums';
import { ChatFormInputRichTokenKind } from '$lib/enums';
import { settingsStore } from '$lib/stores/settings.svelte';
import { toolsStore } from '$lib/stores/tools.svelte';
import type { ContentEditableToken } from '$lib/types/contenteditable';
import type { ChatFormInputRichToken } from '$lib/types/chat-form-input-rich';
// Block wrappers browsers insert for newlines; each folds back into a
// single `\n` during serialization.
@@ -66,7 +66,7 @@ const CODE_SPAN_RE = /(```[\s\S]*?```)|(`[^`\n]+`)/g;
/**
* Cheap gate check for `ChatForm`: does the buffer contain a
* complete code span (inline or fenced)? Used to promote the plain
* textarea to the contenteditable renderer.
* textarea to the chat-form-input-rich renderer.
*/
export function containsCodeSpan(value: string): boolean {
CODE_SPAN_RE.lastIndex = 0;
@@ -102,14 +102,14 @@ export function isOffsetInCodeBlock(source: string, offset: number): boolean {
/**
* Tokenize a markdown source value into the segments the
* contenteditable will render. Code spans are carved out first
* chat-form-input-rich will render. Code spans are carved out first
* (their content is literal - a `file://` link inside backticks
* must NOT render as a badge), then plain text and badges
* interleave in the remaining gaps. Any whitespace after a badge
* stays in a plain text token so the round trip is byte-exact.
*/
export function tokenizeContent(input: string): ContentEditableToken[] {
const tokens: ContentEditableToken[] = [];
export function tokenizeContent(input: string): ChatFormInputRichToken[] {
const tokens: ChatFormInputRichToken[] = [];
let cursor = 0;
@@ -126,8 +126,8 @@ export function tokenizeContent(input: string): ContentEditableToken[] {
tokens.push(
match[1] !== undefined
? { kind: ContentEditableTokenKind.CODE_BLOCK, text: match[1] }
: { kind: ContentEditableTokenKind.INLINE_CODE, text: match[2] }
? { kind: ChatFormInputRichTokenKind.CODE_BLOCK, text: match[1] }
: { kind: ChatFormInputRichTokenKind.CODE_INLINE, text: match[2] }
);
cursor = start + match[0].length;
}
@@ -142,7 +142,7 @@ export function tokenizeContent(input: string): ContentEditableToken[] {
/**
* Tokenize a code-free segment into text and badge tokens.
*/
function pushTextAndBadgeTokens(input: string, tokens: ContentEditableToken[]) {
function pushTextAndBadgeTokens(input: string, tokens: ChatFormInputRichToken[]) {
let cursor = 0;
MENTION_BADGE_RE.lastIndex = 0;
@@ -154,24 +154,26 @@ function pushTextAndBadgeTokens(input: string, tokens: ContentEditableToken[]) {
const start = match.index;
if (start > cursor) {
tokens.push({ kind: ContentEditableTokenKind.TEXT, text: input.slice(cursor, start) });
tokens.push({ kind: ChatFormInputRichTokenKind.TEXT, text: input.slice(cursor, start) });
}
tokens.push({ kind: ContentEditableTokenKind.BADGE, name, path });
tokens.push({ kind: ChatFormInputRichTokenKind.BADGE, name, path });
cursor = start + whole.length;
}
if (cursor < input.length) {
tokens.push({ kind: ContentEditableTokenKind.TEXT, text: input.slice(cursor) });
tokens.push({ kind: ChatFormInputRichTokenKind.TEXT, text: input.slice(cursor) });
}
}
function isCodeBlockElement(node: Node | null): node is HTMLElement {
return node instanceof HTMLElement && node.dataset.codeToken === 'block';
return (
node instanceof HTMLElement && node.dataset.codeToken === ChatFormInputRichTokenKind.CODE_BLOCK
);
}
/**
* Serialize a contenteditable subtree back to source. `<br>` and block
* Serialize a chat-form-input-rich subtree back to source. `<br>` and block
* wrappers the browser inserted for newlines fold back into `\n` (a
* trailing `<br>` is the browser's caret placeholder, not a newline);
* any other element is transparent. Code spans serialize their
@@ -226,7 +228,7 @@ export function serializeContent(root: HTMLElement): string {
}
if (el.dataset.codeToken !== undefined) {
const isBlock = el.dataset.codeToken === 'block';
const isBlock = el.dataset.codeToken === ChatFormInputRichTokenKind.CODE_BLOCK;
if (isBlock && (pendingBlockBoundary || !first)) out += '\n';
@@ -285,8 +287,8 @@ export function serializeContent(root: HTMLElement): string {
* A mismatch means token boundaries shifted (a code span was just
* completed or broken) and the DOM needs a rebuild to restyle.
*/
export function domMatchesTokens(root: HTMLElement, tokens: ContentEditableToken[]): boolean {
const expected = tokens.filter((token) => token.kind !== ContentEditableTokenKind.TEXT);
export function domMatchesTokens(root: HTMLElement, tokens: ChatFormInputRichToken[]): boolean {
const expected = tokens.filter((token) => token.kind !== ChatFormInputRichTokenKind.TEXT);
let index = 0;
@@ -309,7 +311,7 @@ export function domMatchesTokens(root: HTMLElement, tokens: ContentEditableToken
if (!token) return false;
if (isBadge) {
if (token.kind !== ContentEditableTokenKind.BADGE) return false;
if (token.kind !== ChatFormInputRichTokenKind.BADGE) return false;
if (token.name !== (el.dataset.mentionName ?? '')) return false;
@@ -318,16 +320,16 @@ export function domMatchesTokens(root: HTMLElement, tokens: ContentEditableToken
continue;
}
const codeKind: ContentEditableTokenKind =
el.dataset.codeToken === 'block'
? ContentEditableTokenKind.CODE_BLOCK
: ContentEditableTokenKind.INLINE_CODE;
const codeKind: ChatFormInputRichTokenKind =
el.dataset.codeToken === ChatFormInputRichTokenKind.CODE_BLOCK
? ChatFormInputRichTokenKind.CODE_BLOCK
: ChatFormInputRichTokenKind.CODE_INLINE;
if (token.kind !== codeKind) return false;
if (
token.kind === ContentEditableTokenKind.INLINE_CODE ||
token.kind === ContentEditableTokenKind.CODE_BLOCK
token.kind === ChatFormInputRichTokenKind.CODE_INLINE ||
token.kind === ChatFormInputRichTokenKind.CODE_BLOCK
) {
if (token.text !== (el.textContent ?? '')) return false;
}
@@ -446,7 +448,7 @@ export function rangeToTextOffset(root: HTMLElement, range: Range | null): numbe
}
if (el.dataset.codeToken !== undefined) {
const isBlock = el.dataset.codeToken === 'block';
const isBlock = el.dataset.codeToken === ChatFormInputRichTokenKind.CODE_BLOCK;
if (isBlock && !first) {
if (!atOrBeforeCaret(el, 0)) {
@@ -521,26 +523,29 @@ export function rangeToTextOffset(root: HTMLElement, range: Range | null): numbe
* string + inline SVG are shared with the rehype plugin via
* `$lib/constants`.
*/
export function buildFragment(tokens: ContentEditableToken[]): DocumentFragment {
export function buildFragment(tokens: ChatFormInputRichToken[]): DocumentFragment {
const fragment = document.createDocumentFragment();
for (let index = 0; index < tokens.length; index++) {
const token = tokens[index];
if (token.kind === ContentEditableTokenKind.TEXT) {
if (token.kind === ChatFormInputRichTokenKind.TEXT) {
let text = token.text;
// The separator \n at a fenced-block boundary is synthesized
// at serialization time; keeping it in the DOM would render a
// phantom empty line next to the block.
if (
tokens[index - 1]?.kind === ContentEditableTokenKind.CODE_BLOCK &&
tokens[index - 1]?.kind === ChatFormInputRichTokenKind.CODE_BLOCK &&
text.startsWith('\n')
) {
text = text.slice(1);
}
if (tokens[index + 1]?.kind === ContentEditableTokenKind.CODE_BLOCK && text.endsWith('\n')) {
if (
tokens[index + 1]?.kind === ChatFormInputRichTokenKind.CODE_BLOCK &&
text.endsWith('\n')
) {
text = text.slice(0, -1);
}
@@ -552,13 +557,12 @@ export function buildFragment(tokens: ContentEditableToken[]): DocumentFragment
}
if (
token.kind === ContentEditableTokenKind.INLINE_CODE ||
token.kind === ContentEditableTokenKind.CODE_BLOCK
token.kind === ChatFormInputRichTokenKind.CODE_INLINE ||
token.kind === ChatFormInputRichTokenKind.CODE_BLOCK
) {
const code = document.createElement('code');
code.dataset.codeToken =
token.kind === ContentEditableTokenKind.CODE_BLOCK ? 'block' : 'inline';
code.dataset.codeToken = token.kind;
code.textContent = token.text;
fragment.appendChild(code);
@@ -734,14 +738,14 @@ export function badgeAwareWordJump(
for (const token of tokenizeContent(source)) {
const len =
token.kind === ContentEditableTokenKind.BADGE
token.kind === ChatFormInputRichTokenKind.BADGE
? badgeSourceLength(token.name, token.path)
: token.text.length;
if (token.kind === ContentEditableTokenKind.BADGE)
if (token.kind === ChatFormInputRichTokenKind.BADGE)
badgeSpans.push([masked.length, masked.length + len]);
masked += token.kind === ContentEditableTokenKind.BADGE ? 'a'.repeat(len) : token.text;
masked += token.kind === ChatFormInputRichTokenKind.BADGE ? 'a'.repeat(len) : token.text;
}
if (badgeSpans.length === 0) return null;
@@ -804,7 +808,7 @@ export function badgeAwareWordJump(
export function leadingBadgeEdgeOffset(source: string, caret: number): number | null {
const [first] = tokenizeContent(source);
if (!first || first.kind !== ContentEditableTokenKind.BADGE) return null;
if (!first || first.kind !== ChatFormInputRichTokenKind.BADGE) return null;
return caret === badgeSourceLength(first.name, first.path) ? 0 : null;
}
@@ -912,7 +916,7 @@ export function textOffsetToRange(root: HTMLElement, offset: number): Range {
}
if (el.dataset.codeToken !== undefined) {
const isBlock = el.dataset.codeToken === 'block';
const isBlock = el.dataset.codeToken === ChatFormInputRichTokenKind.CODE_BLOCK;
if (isBlock && (pendingBlockBoundary || !first)) {
pendingBlockBoundary = false;
tools/ui/src/lib/utils/index.ts
+1 -1
View File
@@ -221,7 +221,7 @@ export {
textOffsetToRange,
badgeAwareWordJump,
leadingBadgeEdgeOffset
} from './contenteditable-tokenizer';
} from './chat-form-input-rich-tokenizer';
// Source-space undo/redo history for the chat-form contenteditable
export { SourceHistory, type SourceHistoryEntry } from './source-history';