ui: Context usage gauge and panel (#25340)
* feat: WIP * feat: Retire ChatScreenProcessingInfo component, context, and keepStatsVisible settings * feat: Always-on gauge with active-model /props, conversation stats and live-reactive reading/output/avg * feat: Add /tokenize endpoint, TokenizeService, FNV-1a and JSON Schema utilities * feat: Surface enabled-tools token count in context hover card * refactor(tools): make toolsStore the sole owner of the OpenAI wire format Previously mcpStore.getToolDefinitionsForLLM() owned the MCP->OpenAI shape conversion (plus normalizeSchemaProperties). That created two sources of truth for what gets sent to the LLM, with the duplication-prone risk of the deduplicated enabled list (which feeds the token-count cache) drifting from the bytes actually shipped on chat. Now: - mcpStore: pure protocol state + routing. Drop getToolDefinitionsForLLM and the inline OpenAIToolDefinition conversion + normalizeSchemaProperties. Doc comment adjusted to declare wire-format ownership as belonging to toolsStore. Connection lifecycle, health checks, executeTool, and the connections/toolsIndex remain. - toolsStore: owns the wire shape (added earlier this series). mcpEntries() inlines the MCP tool conversion; uses normalizeJsonSchema (the JSON Schema util extracted in the prior commit) so missing 'type' fields are inferred from defaults. mcpTools getter iterates mcpEntries() so the Settings UI and the deduplicated enabled list see the same definitions. getEnabledToolsForLLM iterates mcpEntries() instead of calling mcpStore, so the JSON sent to the LLM is identical to what toolsStore.refreshEnabledToolsTokenCount tokenizes. - agentic: the chat-completion tools field's type was annotated as ReturnType<typeof mcpStore.getToolDefinitionsForLLM>, claiming the shape was owned by mcpStore. Switch to ReturnType<typeof toolsStore.getEnabledToolsForLLM>, the actual source. Assisted-by: Claude * feat: UI WIP * feat: UI WIP * feat: UI WIP * feat: Adjust reasoning submenu layout and spacing * feat: Adjust context usage gauge thresholds and styling * feat: Split context usage gauge stats into current and cumulative breakdowns * chore: Format * refactor: Cleanup * refactor: Cleanup * feat: improve token gauge accuracy and display * refactor: remove MCP recommendation gating and simplify server visibility * feat: add token audit logging to ChatStore for debugging * refactor: Simplify context token reading to use server promptTokens directly * feat: Replace last-known token tracking with live server-derived stats for accurate streaming gauges * feat: UI Improvements * feat: Move prompt processing stats to the preceding user message * feat: Fix context token double-counting and refine gauge layout * refactor: remove always-show-agentic-turns setting and simplify agentic turn display * feat: track and display cache tokens in context gauge * feat: add diagnostic logging for chat completion requests * refactor: improve token audit console output with fresh/cached breakdown * fix: invalidate enabled tools token count cache on tool changes * test: add unit tests for tools store token count invalidation * refactor: Remove tools token counting infrastructure * refactor: Update ChatFormContextGauge to use simplified token tracking * refactor: Update ChatStore to remove tools token counting * chore: Formatting * feat: Improve UI text * feat: simplify context usage derivation and refine gauge labels * refactor: cleanup logs * cleaning * fix: UI * refactor: Enums * refactor: Extract context gauge logic into hook and split UI into sub-components * refactor: Cleanup comments --------- Co-authored-by: Pascal <admin@serveurperso.com>
This commit is contained in:
Aleksander Grygier
co-authored by
Pascal
Pascal
parent
da46e59cbf
commit
f1161b15f2
@@ -0,0 +1,295 @@
|
||||
/**
|
||||
* Reactive state for the context usage gauge: resolves the active model,
|
||||
* fetches its cached props, parses live server stats, and exposes per-turn
|
||||
* read / fresh / cache / output and cumulative token counts.
|
||||
*/
|
||||
|
||||
import {
|
||||
modelsStore,
|
||||
modelOptions,
|
||||
selectedModelId,
|
||||
singleModelName
|
||||
} from '$lib/stores/models.svelte';
|
||||
import { chatStore } from '$lib/stores/chat.svelte';
|
||||
import { activeMessages } from '$lib/stores/conversations.svelte';
|
||||
import { isRouterMode } from '$lib/stores/server.svelte';
|
||||
import { MessageRole } from '$lib/enums';
|
||||
import { STATS_UNITS } from '$lib/constants';
|
||||
import type { ChatMessageTimings, DatabaseMessage } from '$lib/types';
|
||||
import { useProcessingState } from './use-processing-state.svelte';
|
||||
import {
|
||||
colorLevelFromPercent,
|
||||
type ColorLevel
|
||||
} from '$lib/components/app/chat/ChatForm/ChatFormContextGauge/context-gauge';
|
||||
|
||||
interface LiveStats {
|
||||
freshTokens: number;
|
||||
promptTokens: number;
|
||||
cacheTokens: number;
|
||||
outputTokens: number;
|
||||
}
|
||||
|
||||
export interface UseContextGaugeReturn {
|
||||
readonly activeModelId: string | null;
|
||||
readonly isActiveModelLoaded: boolean;
|
||||
readonly isActiveModelLoading: boolean;
|
||||
readonly contextTotal: number | null;
|
||||
readonly contextUsed: number;
|
||||
readonly currentRead: number;
|
||||
readonly currentFresh: number;
|
||||
readonly currentCache: number;
|
||||
readonly currentOutput: number;
|
||||
readonly kvTotal: number;
|
||||
readonly cumulativeRead: number;
|
||||
readonly cumulativeOutput: number;
|
||||
readonly cumulativeCacheTotal: number;
|
||||
readonly averageTokensPerSecond: number | null;
|
||||
readonly contextPercent: number | null;
|
||||
readonly colorLevel: ColorLevel;
|
||||
readonly transientDetails: string[];
|
||||
readonly hasAnyUsage: boolean;
|
||||
loadModel(): Promise<void>;
|
||||
startMonitoring(): void;
|
||||
}
|
||||
|
||||
function lastAssistantTimings(messages: DatabaseMessage[]): ChatMessageTimings | undefined {
|
||||
for (let i = messages.length - 1; i >= 0; i--) {
|
||||
const m = messages[i];
|
||||
if (m.role === MessageRole.ASSISTANT && m.timings) return m.timings;
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
function deriveLiveStats(
|
||||
state: ReturnType<typeof useProcessingState>['processingState']
|
||||
): LiveStats | null {
|
||||
if (!state || (state.status !== 'preparing' && state.status !== 'generating')) {
|
||||
return null;
|
||||
}
|
||||
const promptTokens = state.promptTokens ?? 0;
|
||||
const cacheTokens = state.cacheTokens ?? 0;
|
||||
return {
|
||||
freshTokens: promptTokens,
|
||||
promptTokens: promptTokens + cacheTokens,
|
||||
cacheTokens,
|
||||
outputTokens: state.outputTokensUsed ?? 0
|
||||
};
|
||||
}
|
||||
|
||||
const TRANSIENT_DETAILS_EXCLUDED_PREFIXES = ['Context:', 'Output:'];
|
||||
|
||||
function filterTransientDetails(raw: string[]): string[] {
|
||||
return raw.filter((detail) => {
|
||||
if (TRANSIENT_DETAILS_EXCLUDED_PREFIXES.some((prefix) => detail.startsWith(prefix))) {
|
||||
return false;
|
||||
}
|
||||
return !detail.includes(STATS_UNITS.TOKENS_PER_SECOND);
|
||||
});
|
||||
}
|
||||
|
||||
export function useContextGauge(): UseContextGaugeReturn {
|
||||
const processingState = useProcessingState();
|
||||
|
||||
// Resolve the model the gauge reports context for: explicit selection >
|
||||
// last assistant model > single-model mode (mirrors useChatScreenActiveModel).
|
||||
const activeModelId = $derived.by(() => {
|
||||
if (!isRouterMode()) {
|
||||
return singleModelName();
|
||||
}
|
||||
|
||||
const selectedId = selectedModelId();
|
||||
if (selectedId) {
|
||||
const model = modelOptions().find((m) => m.id === selectedId);
|
||||
if (model) return model.model;
|
||||
}
|
||||
|
||||
return chatStore.getConversationModel(activeMessages() as DatabaseMessage[]);
|
||||
});
|
||||
|
||||
const isActiveModelLoaded = $derived(
|
||||
activeModelId !== null && modelsStore.isModelLoaded(activeModelId)
|
||||
);
|
||||
|
||||
const isActiveModelLoading = $derived(
|
||||
activeModelId !== null && modelsStore.isModelOperationInProgress(activeModelId)
|
||||
);
|
||||
|
||||
// Pull /props on demand so n_ctx surfaces before the first chat request.
|
||||
$effect(() => {
|
||||
if (activeModelId && isActiveModelLoaded) {
|
||||
const cached = modelsStore.getModelProps(activeModelId);
|
||||
if (!cached) {
|
||||
void modelsStore.fetchModelProps(activeModelId);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
const contextTotal = $derived.by(() => {
|
||||
void modelsStore.propsCacheVersion;
|
||||
return activeModelId ? modelsStore.getModelContextSize(activeModelId) : null;
|
||||
});
|
||||
|
||||
const liveStats = $derived(deriveLiveStats(processingState.processingState));
|
||||
|
||||
const currentRead = $derived.by(() => {
|
||||
const timings = lastAssistantTimings(activeMessages() as DatabaseMessage[]);
|
||||
let read = 0;
|
||||
if (timings) {
|
||||
read = (timings.prompt_n ?? 0) + (timings.cache_n ?? 0);
|
||||
}
|
||||
// live.promptTokens is already the combined reading (prompt + cache),
|
||||
// so do not also add live.cacheTokens.
|
||||
if (liveStats && liveStats.promptTokens > 0) {
|
||||
read = Math.max(read, liveStats.promptTokens);
|
||||
}
|
||||
return read;
|
||||
});
|
||||
|
||||
const currentFresh = $derived.by(() => {
|
||||
const timings = lastAssistantTimings(activeMessages() as DatabaseMessage[]);
|
||||
const fresh = timings?.prompt_n ?? 0;
|
||||
return Math.max(fresh, liveStats?.freshTokens ?? 0);
|
||||
});
|
||||
|
||||
const currentCache = $derived.by(() => {
|
||||
const timings = lastAssistantTimings(activeMessages() as DatabaseMessage[]);
|
||||
const cached = timings?.cache_n ?? 0;
|
||||
if (liveStats && liveStats.promptTokens > 0) {
|
||||
return Math.max(cached, liveStats.cacheTokens);
|
||||
}
|
||||
return cached;
|
||||
});
|
||||
|
||||
const currentOutput = $derived.by(() => {
|
||||
if (liveStats && liveStats.outputTokens > 0) return liveStats.outputTokens;
|
||||
const timings = lastAssistantTimings(activeMessages() as DatabaseMessage[]);
|
||||
return timings?.predicted_n ?? 0;
|
||||
});
|
||||
|
||||
const kvTotal = $derived(currentRead + currentOutput);
|
||||
const contextUsed = $derived(currentRead + currentOutput);
|
||||
|
||||
const cumulative = $derived.by(() => {
|
||||
const messages = activeMessages() as DatabaseMessage[];
|
||||
|
||||
// Agentic sessions stamp the same agentic.llm totals onto every
|
||||
// assistant message; cache_n is never per-turn so cache_total stays 0.
|
||||
const agenticMessages = messages.filter(
|
||||
(m) => m.role === MessageRole.ASSISTANT && m.timings?.agentic?.llm?.predicted_n != null
|
||||
);
|
||||
|
||||
if (agenticMessages.length > 0) {
|
||||
const llm = agenticMessages[agenticMessages.length - 1].timings!.agentic!.llm;
|
||||
const output = llm.predicted_n ?? 0;
|
||||
const outputMs = llm.predicted_ms ?? 0;
|
||||
const averageTokensPerSecond = outputMs > 0 && output > 0 ? (output / outputMs) * 1000 : null;
|
||||
return {
|
||||
read: llm.prompt_n ?? 0,
|
||||
output,
|
||||
cacheTotal: 0,
|
||||
averageTokensPerSecond
|
||||
};
|
||||
}
|
||||
|
||||
let read = 0;
|
||||
let output = 0;
|
||||
let outputMs = 0;
|
||||
let cacheTotal = 0;
|
||||
for (const m of messages) {
|
||||
if (m.role !== MessageRole.ASSISTANT || !m.timings) continue;
|
||||
read += m.timings.prompt_n ?? 0;
|
||||
cacheTotal += m.timings.cache_n ?? 0;
|
||||
output += m.timings.predicted_n ?? 0;
|
||||
outputMs += m.timings.predicted_ms ?? 0;
|
||||
}
|
||||
const averageTokensPerSecond = outputMs > 0 && output > 0 ? (output / outputMs) * 1000 : null;
|
||||
return { read, output, cacheTotal, averageTokensPerSecond };
|
||||
});
|
||||
|
||||
const contextPercent = $derived.by(() => {
|
||||
if (contextTotal === null || contextTotal <= 0) return null;
|
||||
return Math.round((contextUsed / contextTotal) * 100);
|
||||
});
|
||||
|
||||
const colorLevel = $derived(colorLevelFromPercent(contextPercent));
|
||||
|
||||
// Drop lines the surrounding Context / Output / speed rows already render.
|
||||
const transientDetails = $derived(filterTransientDetails(processingState.getTechnicalDetails()));
|
||||
|
||||
const hasAnyUsage = $derived(
|
||||
cumulative.read > 0 ||
|
||||
cumulative.output > 0 ||
|
||||
currentRead > 0 ||
|
||||
currentOutput > 0 ||
|
||||
cumulative.averageTokensPerSecond !== null ||
|
||||
transientDetails.length > 0
|
||||
);
|
||||
|
||||
async function loadModel() {
|
||||
if (!activeModelId || isActiveModelLoading) return;
|
||||
try {
|
||||
await modelsStore.loadModel(activeModelId);
|
||||
} catch {
|
||||
// toast already surfaced by modelsStore.loadModel
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
get activeModelId() {
|
||||
return activeModelId;
|
||||
},
|
||||
get isActiveModelLoaded() {
|
||||
return isActiveModelLoaded;
|
||||
},
|
||||
get isActiveModelLoading() {
|
||||
return isActiveModelLoading;
|
||||
},
|
||||
get contextTotal() {
|
||||
return contextTotal;
|
||||
},
|
||||
get contextUsed() {
|
||||
return contextUsed;
|
||||
},
|
||||
get currentRead() {
|
||||
return currentRead;
|
||||
},
|
||||
get currentFresh() {
|
||||
return currentFresh;
|
||||
},
|
||||
get currentCache() {
|
||||
return currentCache;
|
||||
},
|
||||
get currentOutput() {
|
||||
return currentOutput;
|
||||
},
|
||||
get kvTotal() {
|
||||
return kvTotal;
|
||||
},
|
||||
get cumulativeRead() {
|
||||
return cumulative.read;
|
||||
},
|
||||
get cumulativeOutput() {
|
||||
return cumulative.output;
|
||||
},
|
||||
get cumulativeCacheTotal() {
|
||||
return cumulative.cacheTotal;
|
||||
},
|
||||
get averageTokensPerSecond() {
|
||||
return cumulative.averageTokensPerSecond;
|
||||
},
|
||||
get contextPercent() {
|
||||
return contextPercent;
|
||||
},
|
||||
get colorLevel() {
|
||||
return colorLevel;
|
||||
},
|
||||
get transientDetails() {
|
||||
return transientDetails;
|
||||
},
|
||||
get hasAnyUsage() {
|
||||
return hasAnyUsage;
|
||||
},
|
||||
loadModel,
|
||||
startMonitoring: () => processingState.startMonitoring()
|
||||
};
|
||||
}
|
||||
@@ -54,11 +54,6 @@ export function useMcpRecommendations() {
|
||||
// effect, and we must not wipe the timeout that was just scheduled.
|
||||
if (checked) return;
|
||||
|
||||
if (mcpStore.optedInRecommendationIds.size > 0) {
|
||||
checked = true;
|
||||
return;
|
||||
}
|
||||
|
||||
const hasRecommendations = mcpStore
|
||||
.getServers()
|
||||
.some((server) => RECOMMENDED_MCP_SERVER_IDS.has(server.id));
|
||||
|
||||
@@ -1,5 +1,4 @@
|
||||
import { activeProcessingState } from '$lib/stores/chat.svelte';
|
||||
import { config } from '$lib/stores/settings.svelte';
|
||||
import { STATS_UNITS } from '$lib/constants';
|
||||
import type { ApiProcessingState, LiveProcessingStats, LiveGenerationStats } from '$lib/types';
|
||||
|
||||
@@ -46,7 +45,6 @@ export function useProcessingState(): UseProcessingStateReturn {
|
||||
return activeProcessingState();
|
||||
});
|
||||
|
||||
// Track last known state for keepStatsVisible functionality
|
||||
$effect(() => {
|
||||
if (processingState && isMonitoring) {
|
||||
lastKnownState = processingState;
|
||||
@@ -88,14 +86,8 @@ export function useProcessingState(): UseProcessingStateReturn {
|
||||
|
||||
function stopMonitoring(): void {
|
||||
if (!isMonitoring) return;
|
||||
isMonitoring = false;
|
||||
|
||||
// Only clear last known state if keepStatsVisible is disabled
|
||||
const currentConfig = config();
|
||||
if (!currentConfig.keepStatsVisible) {
|
||||
lastKnownState = null;
|
||||
lastKnownProcessingStats = null;
|
||||
}
|
||||
isMonitoring = false;
|
||||
}
|
||||
|
||||
function getProcessingMessage(): string {
|
||||
|
||||
Reference in New Issue
Block a user