From 3f9aa5d10b35223abf6146f960ff5cb5c68053ee Mon Sep 17 00:00:00 2001 From: Armin Ronacher Date: Thu, 9 Jul 2026 14:12:57 +0200 Subject: [PATCH] feat(coding-agent): add prompt cache miss tracking (#6427) Detect prompt cache misses per turn by comparing each assistant message's cache reads against the previous request's prompt tokens (core/cache-stats.ts). Significant misses emit a warning-colored transcript notice at the turn they occur, noting idle gaps past the cache TTL and model switches when relevant. /session gained cache statistics: a compact token/cache breakdown with hit rate, a $-prefixed cost section with per-model cost breakdown, and the cumulative cost re-billed due to cache misses. --- packages/coding-agent/CHANGELOG.md | 1 + packages/coding-agent/docs/settings.md | 1 + .../coding-agent/src/core/agent-session.ts | 42 +++-- packages/coding-agent/src/core/cache-stats.ts | 164 ++++++++++++++++++ .../coding-agent/src/core/settings-manager.ts | 11 ++ .../modes/interactive/components/footer.ts | 3 +- .../components/settings-selector.ts | 12 ++ .../src/modes/interactive/interactive-mode.ts | 116 +++++++++++-- .../test/agent-session-stats.test.ts | 6 +- .../coding-agent/test/cache-stats.test.ts | 143 +++++++++++++++ ...hinking-toggle-pending-tool-render.test.ts | 8 +- 11 files changed, 471 insertions(+), 36 deletions(-) create mode 100644 packages/coding-agent/src/core/cache-stats.ts create mode 100644 packages/coding-agent/test/cache-stats.test.ts diff --git a/packages/coding-agent/CHANGELOG.md b/packages/coding-agent/CHANGELOG.md index 3df85629..bd6c5289 100644 --- a/packages/coding-agent/CHANGELOG.md +++ b/packages/coding-agent/CHANGELOG.md @@ -8,6 +8,7 @@ - Added public SDK exports for CLI-equivalent model and scoped-model resolution ([#6201](https://github.com/earendil-works/pi/issues/6201)). - Added extension and RPC `agent_settled` events plus session-level idle waiting for fully settled agent runs ([#6363](https://github.com/earendil-works/pi/issues/6363)). - Added extension entry renderers for persisted display-only session entries that are rendered in interactive mode without being sent to the model context. +- Added a `showCacheMissNotices` setting and `/settings` toggle for significant prompt-cache miss transcript notices. ### Fixed diff --git a/packages/coding-agent/docs/settings.md b/packages/coding-agent/docs/settings.md index d1a49fc7..02f21c3c 100644 --- a/packages/coding-agent/docs/settings.md +++ b/packages/coding-agent/docs/settings.md @@ -31,6 +31,7 @@ Use `/trust` in interactive mode to save a project trust decision for future ses | `defaultModel` | string | - | Default model ID | | `defaultThinkingLevel` | string | - | `"off"`, `"minimal"`, `"low"`, `"medium"`, `"high"`, `"xhigh"` | | `hideThinkingBlock` | boolean | `false` | Hide thinking blocks in output | +| `showCacheMissNotices` | boolean | `false` | Show transcript notices for significant prompt-cache misses | | `thinkingBudgets` | object | - | Custom token budgets per thinking level | #### thinkingBudgets diff --git a/packages/coding-agent/src/core/agent-session.ts b/packages/coding-agent/src/core/agent-session.ts index 9a3a5c5f..c4c2d366 100644 --- a/packages/coding-agent/src/core/agent-session.ts +++ b/packages/coding-agent/src/core/agent-session.ts @@ -3016,14 +3016,15 @@ export class AgentSession { } /** - * Get session statistics. + * Get session statistics. Aggregates over ALL session entries (including + * history that was compacted away), so token/cost totals reflect what was + * actually billed across the session. */ getSessionStats(): SessionStats { - const state = this.state; - const userMessages = state.messages.filter((m) => m.role === "user").length; - const assistantMessages = state.messages.filter((m) => m.role === "assistant").length; - const toolResults = state.messages.filter((m) => m.role === "toolResult").length; - + let userMessages = 0; + let assistantMessages = 0; + let toolResults = 0; + let totalMessages = 0; let toolCalls = 0; let totalInput = 0; let totalOutput = 0; @@ -3031,15 +3032,26 @@ export class AgentSession { let totalCacheWrite = 0; let totalCost = 0; - for (const message of state.messages) { - if (message.role === "assistant") { + for (const entry of this.sessionManager.getEntries()) { + if (entry.type !== "message") continue; + totalMessages++; + const message = entry.message; + if (message.role === "user") { + userMessages++; + } else if (message.role === "toolResult") { + toolResults++; + } else if (message.role === "assistant") { + assistantMessages++; const assistantMsg = message as AssistantMessage; - toolCalls += assistantMsg.content.filter((c) => c.type === "toolCall").length; - totalInput += assistantMsg.usage.input; - totalOutput += assistantMsg.usage.output; - totalCacheRead += assistantMsg.usage.cacheRead; - totalCacheWrite += assistantMsg.usage.cacheWrite; - totalCost += assistantMsg.usage.cost.total; + if (Array.isArray(assistantMsg.content)) { + toolCalls += assistantMsg.content.filter((c) => c.type === "toolCall").length; + } + const usage = assistantMsg.usage; + totalInput += usage.input; + totalOutput += usage.output; + totalCacheRead += usage.cacheRead; + totalCacheWrite += usage.cacheWrite; + totalCost += usage.cost.total; } } @@ -3050,7 +3062,7 @@ export class AgentSession { assistantMessages, toolCalls, toolResults, - totalMessages: state.messages.length, + totalMessages, tokens: { input: totalInput, output: totalOutput, diff --git a/packages/coding-agent/src/core/cache-stats.ts b/packages/coding-agent/src/core/cache-stats.ts new file mode 100644 index 00000000..8c9a4943 --- /dev/null +++ b/packages/coding-agent/src/core/cache-stats.ts @@ -0,0 +1,164 @@ +import type { AssistantMessage } from "@earendil-works/pi-ai"; +import type { SessionEntry } from "./session-manager.ts"; + +/** + * Prompt-cache TTL: idle gaps longer than this are worth mentioning as the + * likely cause of a miss. Anthropic's default cache TTL is 5 minutes. + */ +export const CACHE_TTL_MS = 5 * 60 * 1000; + +/** Per-turn misses at or below this are cache breakpoint granularity noise. */ +const NOISE_FLOOR_TOKENS = 1024; + +/** A counted cache miss on a single assistant message. */ +export interface CacheMiss { + /** Prompt tokens that were in the previous turn's prompt but not read from cache. */ + missedTokens: number; + /** Extra dollars paid vs. a full cache hit; 0 when pricing is unknown. */ + missedCost: number; + /** Milliseconds since the previous request (which last refreshed the cache). */ + idleMs: number; + /** True when the model changed relative to the previous request. */ + modelChanged: boolean; +} + +export interface CacheWasteTotals { + missedTokens: number; + missedCost: number; + /** Number of counted misses (turns above the noise floor). */ + missCount: number; +} + +/** Minimal pricing lookup, satisfied by ModelRegistry. Cost is $/million tokens. */ +export interface ModelPriceSource { + find(provider: string, modelId: string): { cost: { cacheRead: number } } | undefined; +} + +/** The last request seen by the scan; everything in its prompt should be cached. */ +interface PreviousRequest { + promptTokens: number; + modelKey: string; + timestamp: number; + /** + * Sticky: some earlier request in this scan segment reported cache activity. + * Distinguishes a total miss on a cache-read-only provider (OpenAI-style, + * writes unreported) from a provider that never reports caching at all. + */ + reportedCache: boolean; +} + +/** + * Compute the cache miss for one assistant message relative to the previous + * request. Returns undefined when nothing is counted: first turn, after a + * reset, no cache activity ever reported (provider without cache support), or + * miss below the noise floor. + */ +function detectMiss( + prev: PreviousRequest | undefined, + message: AssistantMessage, + models: ModelPriceSource, +): CacheMiss | undefined { + const usage = message.usage; + const promptTokens = usage.input + usage.cacheRead + usage.cacheWrite; + // A zero-cache turn only counts when cache activity was reported before: + // on cache-read-only providers that is a total miss, while on providers + // that never report caching it means nothing. + if (!prev || promptTokens <= 0 || (usage.cacheRead + usage.cacheWrite === 0 && !prev.reportedCache)) { + return undefined; + } + + const missedTokens = Math.min(prev.promptTokens, promptTokens) - usage.cacheRead; + if (missedTokens <= NOISE_FLOOR_TOKENS) return undefined; + + // Extra cost = missed tokens billed at the actual paid rate (input/cacheWrite, + // incl. write premium) instead of the cache-read rate. Missed tokens can only + // land in the input or cacheWrite buckets, so the paid rate comes straight + // from this message's own cost breakdown. + const paidTokens = usage.input + usage.cacheWrite; + const paidPerToken = paidTokens > 0 ? (usage.cost.input + usage.cost.cacheWrite) / paidTokens : 0; + const readPerToken = + usage.cacheRead > 0 + ? usage.cost.cacheRead / usage.cacheRead + : (models.find(message.provider, message.model)?.cost.cacheRead ?? 0) / 1_000_000; + + return { + missedTokens, + missedCost: missedTokens * Math.max(0, paidPerToken - readPerToken), + idleMs: Math.max(0, message.timestamp - prev.timestamp), + modelChanged: `${message.provider}/${message.model}` !== prev.modelKey, + }; +} + +function asPreviousRequest(message: AssistantMessage, reportedCache: boolean): PreviousRequest | undefined { + const usage = message.usage; + const promptTokens = usage.input + usage.cacheRead + usage.cacheWrite; + if (promptTokens <= 0) return undefined; + return { + promptTokens, + modelKey: `${message.provider}/${message.model}`, + timestamp: message.timestamp, + reportedCache: reportedCache || usage.cacheRead + usage.cacheWrite > 0, + }; +} + +function scan( + entries: SessionEntry[], + models: ModelPriceSource, +): { prev: PreviousRequest | undefined; totals: CacheWasteTotals; misses: Map } { + let prev: PreviousRequest | undefined; + const totals: CacheWasteTotals = { missedTokens: 0, missedCost: 0, missCount: 0 }; + const misses = new Map(); + + for (const entry of entries) { + if (entry.type === "compaction" || entry.type === "branch_summary") { + // The context legitimately changed; the next turn's prompt is new content, + // not re-billed content. Model switches are NOT exempt: they re-bill the + // full prompt and should be counted. + prev = undefined; + continue; + } + if (entry.type === "message" && entry.message.role === "assistant") { + const miss = detectMiss(prev, entry.message, models); + if (miss) { + totals.missedTokens += miss.missedTokens; + totals.missedCost += miss.missedCost; + totals.missCount += 1; + misses.set(entry.message, miss); + } + prev = asPreviousRequest(entry.message, prev?.reportedCache ?? false) ?? prev; + } + } + return { prev, totals, misses }; +} + +/** + * Cumulative cache waste across a session: prompt tokens that should have been + * cache reads (they were in the previous turn's prompt) but were re-billed. + */ +export function computeCacheWaste(entries: SessionEntry[], models: ModelPriceSource): CacheWasteTotals { + return scan(entries, models).totals; +} + +/** + * All counted cache misses across a session, keyed by the assistant message + * (by reference) that paid for them. Used to re-derive transcript notices when + * rebuilding the chat from entries (resume, post-compaction rebuild). + */ +export function collectCacheMisses( + entries: SessionEntry[], + models: ModelPriceSource, +): Map { + return scan(entries, models).misses; +} + +/** + * Detect a cache miss on a just-completed assistant message. + * `entries` must not yet contain `message` (message_end fires before persistence). + */ +export function detectCacheMiss( + entries: SessionEntry[], + message: AssistantMessage, + models: ModelPriceSource, +): CacheMiss | undefined { + return detectMiss(scan(entries, models).prev, message, models); +} diff --git a/packages/coding-agent/src/core/settings-manager.ts b/packages/coding-agent/src/core/settings-manager.ts index 74ee902a..141b984b 100644 --- a/packages/coding-agent/src/core/settings-manager.ts +++ b/packages/coding-agent/src/core/settings-manager.ts @@ -92,6 +92,7 @@ export interface Settings { branchSummary?: BranchSummarySettings; retry?: RetrySettings; hideThinkingBlock?: boolean; + showCacheMissNotices?: boolean; // default: false - show transcript notices for significant prompt-cache misses externalEditor?: string; // Command for Ctrl+G external editor; takes precedence over VISUAL/EDITOR shellPath?: string; // Custom shell path (e.g., for Cygwin users on Windows) quietStartup?: boolean; @@ -845,6 +846,10 @@ export class SettingsManager { return this.settings.hideThinkingBlock ?? false; } + getShowCacheMissNotices(): boolean { + return this.settings.showCacheMissNotices ?? false; + } + getExternalEditorCommand(): string | undefined { const configuredEditor = this.settings.externalEditor; if (typeof configuredEditor === "string" && configuredEditor.trim() !== "") { @@ -863,6 +868,12 @@ export class SettingsManager { this.save(); } + setShowCacheMissNotices(show: boolean): void { + this.globalSettings.showCacheMissNotices = show; + this.markModified("showCacheMissNotices"); + this.save(); + } + getShellPath(): string | undefined { return this.settings.shellPath; } diff --git a/packages/coding-agent/src/modes/interactive/components/footer.ts b/packages/coding-agent/src/modes/interactive/components/footer.ts index aa537127..4b2914fd 100644 --- a/packages/coding-agent/src/modes/interactive/components/footer.ts +++ b/packages/coding-agent/src/modes/interactive/components/footer.ts @@ -20,7 +20,7 @@ function sanitizeStatusText(text: string): string { /** * Format token counts for compact footer display. */ -function formatTokens(count: number): string { +export function formatTokens(count: number): string { if (count < 1000) return count.toString(); if (count < 10000) return `${(count / 1000).toFixed(1)}k`; if (count < 1000000) return `${Math.round(count / 1000)}k`; @@ -137,7 +137,6 @@ export class FooterComponent implements Component { if ((totalCacheRead > 0 || totalCacheWrite > 0) && latestCacheHitRate !== undefined) { statsParts.push(`CH${latestCacheHitRate.toFixed(1)}%`); } - // Show cost with "(sub)" indicator if using OAuth subscription const usingSubscription = state.model ? this.session.modelRegistry.isUsingOAuth(state.model) : false; if (totalCost || usingSubscription) { diff --git a/packages/coding-agent/src/modes/interactive/components/settings-selector.ts b/packages/coding-agent/src/modes/interactive/components/settings-selector.ts index 81f7ccea..3436b240 100644 --- a/packages/coding-agent/src/modes/interactive/components/settings-selector.ts +++ b/packages/coding-agent/src/modes/interactive/components/settings-selector.ts @@ -65,6 +65,7 @@ export interface SettingsConfig { terminalTheme: TerminalTheme; availableThemes: string[]; hideThinkingBlock: boolean; + showCacheMissNotices: boolean; collapseChangelog: boolean; enableInstallTelemetry: boolean; doubleEscapeAction: "fork" | "tree" | "none"; @@ -95,6 +96,7 @@ export interface SettingsCallbacks { onThemeChange: (theme: string) => void; onThemePreview?: (theme: string) => void; onHideThinkingBlockChange: (hidden: boolean) => void; + onShowCacheMissNoticesChange: (shown: boolean) => void; onCollapseChangelogChange: (collapsed: boolean) => void; onEnableInstallTelemetryChange: (enabled: boolean) => void; onDoubleEscapeActionChange: (action: "fork" | "tree" | "none") => void; @@ -521,6 +523,13 @@ export class SettingsSelectorComponent extends Container { currentValue: config.hideThinkingBlock ? "true" : "false", values: ["true", "false"], }, + { + id: "cache-miss-notices", + label: "Cache miss notices", + description: "Show transcript notices for significant prompt-cache misses", + currentValue: config.showCacheMissNotices ? "true" : "false", + values: ["true", "false"], + }, { id: "collapse-changelog", label: "Collapse changelog", @@ -764,6 +773,9 @@ export class SettingsSelectorComponent extends Container { case "hide-thinking": callbacks.onHideThinkingBlockChange(newValue === "true"); break; + case "cache-miss-notices": + callbacks.onShowCacheMissNoticesChange(newValue === "true"); + break; case "collapse-changelog": callbacks.onCollapseChangelogChange(newValue === "true"); break; diff --git a/packages/coding-agent/src/modes/interactive/interactive-mode.ts b/packages/coding-agent/src/modes/interactive/interactive-mode.ts index 8773cc0c..8f492a18 100644 --- a/packages/coding-agent/src/modes/interactive/interactive-mode.ts +++ b/packages/coding-agent/src/modes/interactive/interactive-mode.ts @@ -60,6 +60,13 @@ import { } from "../../config.ts"; import { type AgentSession, type AgentSessionEvent, parseSkillBlock } from "../../core/agent-session.ts"; import { type AgentSessionRuntime, SessionImportFileNotFoundError } from "../../core/agent-session-runtime.ts"; +import { + CACHE_TTL_MS, + type CacheMiss, + collectCacheMisses, + computeCacheWaste, + detectCacheMiss, +} from "../../core/cache-stats.ts"; import type { AutocompleteProviderFactory, EditorFactory, @@ -111,7 +118,7 @@ import { EarendilAnnouncementComponent } from "./components/earendil-announcemen import { ExtensionEditorComponent } from "./components/extension-editor.ts"; import { ExtensionInputComponent } from "./components/extension-input.ts"; import { ExtensionSelectorComponent } from "./components/extension-selector.ts"; -import { FooterComponent } from "./components/footer.ts"; +import { FooterComponent, formatTokens } from "./components/footer.ts"; import { formatKeyText, keyDisplayText, keyHint, keyText, rawKeyHint } from "./components/keybinding-hints.ts"; import { LoginDialogComponent } from "./components/login-dialog.ts"; import { ModelSelectorComponent } from "./components/model-selector.ts"; @@ -2961,6 +2968,7 @@ export class InteractiveMode { for (const [, component] of this.pendingTools.entries()) { component.setArgsComplete(); } + this.maybeShowCacheMissNotice(this.streamingMessage); } this.streamingComponent = undefined; this.streamingMessage = undefined; @@ -3277,6 +3285,11 @@ export class InteractiveMode { ): void { this.pendingTools.clear(); const renderedPendingTools = new Map(); + // Cache-miss notices are not persisted; re-derive them from the full entry + // list and re-inject them after the assistant messages that paid for them. + const cacheMisses = this.settingsManager.getShowCacheMissNotices() + ? collectCacheMisses(this.sessionManager.getEntries(), this.session.modelRegistry) + : new Map(); if (options.updateFooter) { this.footer.invalidate(); @@ -3328,6 +3341,10 @@ export class InteractiveMode { } } } + if (message.stopReason !== "aborted" && message.stopReason !== "error") { + const miss = cacheMisses.get(message); + if (miss) this.addCacheMissNotice(miss); + } } else if (message.role === "toolResult") { // Match tool results to pending tool components const component = renderedPendingTools.get(message.toolCallId); @@ -3366,6 +3383,35 @@ export class InteractiveMode { this.renderSessionItems(items, options); } + /** + * Show a transcript notice when a completed assistant message paid for a + * significant cache miss. Only states observable facts: the miss itself, + * a model switch, or an idle gap past the cache TTL. + */ + private maybeShowCacheMissNotice(message: AssistantMessage): void { + if (!this.settingsManager.getShowCacheMissNotices()) return; + + // Entries don't contain `message` yet: message_end fires before persistence. + const miss = detectCacheMiss(this.sessionManager.getEntries(), message, this.session.modelRegistry); + if (miss) this.addCacheMissNotice(miss); + } + + private addCacheMissNotice(miss: CacheMiss): void { + if (miss.missedTokens < 20_000 && miss.missedCost < 0.1) return; + + const cost = miss.missedCost >= 0.01 ? ` (~$${miss.missedCost.toFixed(2)})` : ""; + const reBilled = `${formatTokens(miss.missedTokens)} tokens re-billed${cost}`; + let label = "Cache miss"; + if (miss.modelChanged) { + label = "Cache miss after model switch"; + } else if (miss.idleMs >= CACHE_TTL_MS) { + label = `Cache miss after ${Math.round(miss.idleMs / 60_000)}m idle`; + } + const text = theme.fg("warning", `${label}: ${reBilled}`); + this.chatContainer.addChild(new Spacer(1)); + this.chatContainer.addChild(new Text(text, 1, 0)); + } + renderInitialMessages(): void { const entries = this.sessionManager.buildContextEntries(); this.renderSessionEntries(entries, { @@ -4088,6 +4134,7 @@ export class InteractiveMode { doubleEscapeAction: this.settingsManager.getDoubleEscapeAction(), treeFilterMode: this.settingsManager.getTreeFilterMode(), showHardwareCursor: this.settingsManager.getShowHardwareCursor(), + showCacheMissNotices: this.settingsManager.getShowCacheMissNotices(), defaultProjectTrust: this.settingsManager.getDefaultProjectTrust(), editorPaddingX: this.settingsManager.getEditorPaddingX(), outputPad: this.settingsManager.getOutputPad(), @@ -4164,6 +4211,10 @@ export class InteractiveMode { this.chatContainer.clear(); this.rebuildChatFromMessages(); }, + onShowCacheMissNoticesChange: (shown) => { + this.settingsManager.setShowCacheMissNotices(shown); + this.rebuildChatFromMessages(); + }, onCollapseChangelogChange: (collapsed) => { this.settingsManager.setCollapseChangelog(collapsed); }, @@ -5561,6 +5612,26 @@ export class InteractiveMode { private handleSessionCommand(): void { const stats = this.session.getSessionStats(); const sessionName = this.sessionManager.getSessionName(); + const entries = this.sessionManager.getEntries(); + const cacheWaste = computeCacheWaste(entries, this.session.modelRegistry); + + // Cost/token totals per provider/model actually used (e.g. OpenRouter `auto` + // resolves to a concrete responseModel), sorted by cost descending. + const perModelMap = new Map(); + for (const entry of entries) { + if (entry.type !== "message" || entry.message.role !== "assistant") continue; + const message = entry.message; + const usage = message.usage; + const key = `${message.provider}/${message.responseModel ?? message.model}`; + let bucket = perModelMap.get(key); + if (!bucket) { + bucket = { key, cost: 0, tokens: 0 }; + perModelMap.set(key, bucket); + } + bucket.cost += usage.cost.total; + bucket.tokens += usage.input + usage.output + usage.cacheRead + usage.cacheWrite; + } + const perModel = Array.from(perModelMap.values()).sort((a, b) => b.cost - a.cost); let info = `${theme.bold("Session Info")}\n\n`; if (sessionName) { @@ -5569,25 +5640,44 @@ export class InteractiveMode { info += `${theme.fg("dim", "File:")} ${stats.sessionFile ?? "In-memory"}\n`; info += `${theme.fg("dim", "ID:")} ${stats.sessionId}\n\n`; info += `${theme.bold("Messages")}\n`; + info += `${theme.fg("dim", "Total:")} ${stats.totalMessages}\n`; info += `${theme.fg("dim", "User:")} ${stats.userMessages}\n`; info += `${theme.fg("dim", "Assistant:")} ${stats.assistantMessages}\n`; - info += `${theme.fg("dim", "Tool Calls:")} ${stats.toolCalls}\n`; - info += `${theme.fg("dim", "Tool Results:")} ${stats.toolResults}\n`; - info += `${theme.fg("dim", "Total:")} ${stats.totalMessages}\n\n`; + info += `${theme.fg("dim", "Tools:")} ${stats.toolCalls} calls, ${stats.toolResults} results\n\n`; info += `${theme.bold("Tokens")}\n`; - info += `${theme.fg("dim", "Input:")} ${stats.tokens.input.toLocaleString()}\n`; + // "Input" is the full prompt volume. With cache activity, split it into + // cached (served from cache) vs uncached (everything else) - the only + // provider-independent split. Cache writes, where reported, are a detail + // of the uncached portion. + const { input, cacheRead, cacheWrite } = stats.tokens; + const promptTokens = input + cacheRead + cacheWrite; + info += `${theme.fg("dim", "Input:")} ${promptTokens.toLocaleString()}\n`; + if (promptTokens > 0 && (cacheRead > 0 || cacheWrite > 0)) { + const hitRate = theme.fg("dim", `(${((cacheRead / promptTokens) * 100).toFixed(1)}%)`); + info += ` ${theme.fg("dim", "Cached:")} ${cacheRead.toLocaleString()} ${hitRate}\n`; + const written = + cacheWrite > 0 ? ` ${theme.fg("dim", `(${cacheWrite.toLocaleString()} written to cache)`)}` : ""; + info += ` ${theme.fg("dim", "Uncached:")} ${(input + cacheWrite).toLocaleString()}${written}\n`; + } info += `${theme.fg("dim", "Output:")} ${stats.tokens.output.toLocaleString()}\n`; - if (stats.tokens.cacheRead > 0) { - info += `${theme.fg("dim", "Cache Read:")} ${stats.tokens.cacheRead.toLocaleString()}\n`; - } - if (stats.tokens.cacheWrite > 0) { - info += `${theme.fg("dim", "Cache Write:")} ${stats.tokens.cacheWrite.toLocaleString()}\n`; - } info += `${theme.fg("dim", "Total:")} ${stats.tokens.total.toLocaleString()}\n`; - if (stats.cost > 0) { + if (stats.cost > 0 || cacheWaste.missedTokens > 0) { info += `\n${theme.bold("Cost")}\n`; - info += `${theme.fg("dim", "Total:")} ${stats.cost.toFixed(4)}`; + info += `${theme.fg("dim", "Total:")} $${stats.cost.toFixed(3)}`; + if (perModel.length > 1) { + for (const entry of perModel) { + info += `\n ${theme.fg("dim", `${entry.key}:`)} $${entry.cost.toFixed(3)} ${theme.fg("dim", `(${formatTokens(entry.tokens)} tokens)`)}`; + } + } + if (cacheWaste.missedTokens > 0) { + const missLabel = cacheWaste.missCount === 1 ? "1 miss" : `${cacheWaste.missCount} misses`; + const detail = `${cacheWaste.missedTokens.toLocaleString()} tokens, ${missLabel}`; + info += + cacheWaste.missedCost >= 0.0001 + ? `\n${theme.fg("dim", "Cache Re-billed:")} $${cacheWaste.missedCost.toFixed(3)} ${theme.fg("dim", `(${detail})`)}` + : `\n${theme.fg("dim", "Cache Re-billed:")} ${detail}`; + } } this.chatContainer.addChild(new Spacer(1)); diff --git a/packages/coding-agent/test/agent-session-stats.test.ts b/packages/coding-agent/test/agent-session-stats.test.ts index 7e2a5f3f..0b6c4853 100644 --- a/packages/coding-agent/test/agent-session-stats.test.ts +++ b/packages/coding-agent/test/agent-session-stats.test.ts @@ -109,7 +109,8 @@ describe("AgentSession.getSessionStats", () => { syncAgentMessages(session, sessionManager); const stats = session.getSessionStats(); - expect(stats.tokens.input).toBe(195_000); + // Totals cover ALL entries, including history compacted away (180k + 195k). + expect(stats.tokens.input).toBe(375_000); expect(stats.contextUsage).toBeDefined(); expect(stats.contextUsage?.tokens).toBeNull(); expect(stats.contextUsage?.percent).toBeNull(); @@ -132,7 +133,8 @@ describe("AgentSession.getSessionStats", () => { syncAgentMessages(session, sessionManager); const stats = session.getSessionStats(); - expect(stats.tokens.input).toBe(220_000); + // Totals cover ALL entries, including history compacted away (180k + 195k + 25k). + expect(stats.tokens.input).toBe(400_000); expect(stats.contextUsage).toBeDefined(); expect(stats.contextUsage?.tokens).toBe(25_000); expect(stats.contextUsage?.percent).toBe((25_000 / model.contextWindow) * 100); diff --git a/packages/coding-agent/test/cache-stats.test.ts b/packages/coding-agent/test/cache-stats.test.ts new file mode 100644 index 00000000..1d43bfed --- /dev/null +++ b/packages/coding-agent/test/cache-stats.test.ts @@ -0,0 +1,143 @@ +import type { AssistantMessage } from "@earendil-works/pi-ai"; +import { describe, expect, it } from "vitest"; +import { + collectCacheMisses, + computeCacheWaste, + detectCacheMiss, + type ModelPriceSource, +} from "../src/core/cache-stats.ts"; +import type { SessionEntry } from "../src/core/session-manager.ts"; + +const zeroCost = { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 }; + +const models: ModelPriceSource = { + // $/million tokens; used as cache-read price fallback on full-miss turns + find: () => ({ cost: { cacheRead: 0.3 } }), +}; + +function assistant(options: { + input?: number; + cacheRead?: number; + cacheWrite?: number; + cost?: Partial; + model?: string; + timestamp?: number; +}): AssistantMessage { + return { + role: "assistant", + content: [], + api: "anthropic-messages", + provider: "test", + model: options.model ?? "test-model", + usage: { + input: options.input ?? 0, + output: 10, + cacheRead: options.cacheRead ?? 0, + cacheWrite: options.cacheWrite ?? 0, + totalTokens: 0, + cost: { ...zeroCost, ...options.cost }, + }, + stopReason: "stop", + timestamp: options.timestamp ?? 0, + } as AssistantMessage; +} + +function entry(message: AssistantMessage): SessionEntry { + return { type: "message", id: "x", parentId: null, timestamp: "", message } as SessionEntry; +} + +// Turn 1: fresh 100k cache write at $3.75/M +const turn1 = assistant({ cacheWrite: 100_000, cost: { cacheWrite: 0.375 }, timestamp: 0 }); +// Turn 2: healthy, everything read back at $0.30/M +const turn2 = assistant({ + cacheRead: 100_000, + cacheWrite: 5_000, + cost: { cacheRead: 0.03, cacheWrite: 0.019 }, + timestamp: 60_000, +}); + +describe("computeCacheWaste", () => { + it("accumulates missed tokens and cost across turns", () => { + // Turn 3: full miss, previous 105k prompt re-billed at $3.75/M write + const turn3 = assistant({ cacheWrite: 110_000, cost: { cacheWrite: 0.4125 }, timestamp: 120_000 }); + const totals = computeCacheWaste([entry(turn1), entry(turn2), entry(turn3)], models); + expect(totals.missedTokens).toBe(105_000); + // 105k at ($3.75 - $0.30)/M + expect(totals.missedCost).toBeCloseTo(0.36225, 5); + }); + + it("counts nothing for healthy sessions", () => { + const totals = computeCacheWaste([entry(turn1), entry(turn2)], models); + expect(totals.missedTokens).toBe(0); + expect(totals.missedCost).toBe(0); + }); + + it("skips the turn after a compaction reset", () => { + const reset = { type: "compaction", id: "c", parentId: null, timestamp: "" } as SessionEntry; + const afterReset = assistant({ cacheWrite: 20_000, cost: { cacheWrite: 0.075 } }); + const totals = computeCacheWaste([entry(turn1), reset, entry(afterReset)], models); + expect(totals.missedTokens).toBe(0); + }); + + it("counts misses caused by model switches", () => { + const otherModel = assistant({ cacheWrite: 100_000, cost: { cacheWrite: 0.375 }, model: "other-model" }); + const totals = computeCacheWaste([entry(turn1), entry(otherModel)], models); + expect(totals.missedTokens).toBe(100_000); + expect(totals.missCount).toBe(1); + }); + + it("skips providers that report no cache activity", () => { + const a = assistant({ input: 100_000 }); + const b = assistant({ input: 110_000 }); + const totals = computeCacheWaste([entry(a), entry(b)], models); + expect(totals.missedTokens).toBe(0); + }); +}); + +describe("collectCacheMisses", () => { + it("maps counted misses to their assistant messages by reference", () => { + const missTurn = assistant({ cacheWrite: 110_000, cost: { cacheWrite: 0.4125 }, timestamp: 120_000 }); + const misses = collectCacheMisses([entry(turn1), entry(turn2), entry(missTurn)], models); + expect(misses.size).toBe(1); + expect(misses.get(missTurn)?.missedTokens).toBe(105_000); + }); +}); + +describe("detectCacheMiss", () => { + it("detects a miss on a just-completed message with idle time", () => { + const missMessage = assistant({ cacheWrite: 110_000, cost: { cacheWrite: 0.4125 }, timestamp: 600_000 }); + const miss = detectCacheMiss([entry(turn1), entry(turn2)], missMessage, models); + expect(miss).toBeDefined(); + expect(miss?.missedTokens).toBe(105_000); + expect(miss?.missedCost).toBeCloseTo(0.36225, 5); + // 600s - 60s since the previous request + expect(miss?.idleMs).toBe(540_000); + expect(miss?.modelChanged).toBe(false); + }); + + it("flags model switches on detected misses", () => { + const otherModel = assistant({ + cacheWrite: 110_000, + cost: { cacheWrite: 0.4125 }, + model: "other-model", + timestamp: 120_000, + }); + const miss = detectCacheMiss([entry(turn1), entry(turn2)], otherModel, models); + expect(miss?.missedTokens).toBe(105_000); + expect(miss?.modelChanged).toBe(true); + }); + + it("returns undefined for healthy turns", () => { + const healthy = assistant({ + cacheRead: 105_000, + cacheWrite: 2_000, + cost: { cacheRead: 0.0315, cacheWrite: 0.0075 }, + timestamp: 120_000, + }); + expect(detectCacheMiss([entry(turn1), entry(turn2)], healthy, models)).toBeUndefined(); + }); + + it("returns undefined for the first turn of a session", () => { + expect(detectCacheMiss([], turn1, models)).toBeUndefined(); + }); +}); diff --git a/packages/coding-agent/test/suite/regressions/4167-thinking-toggle-pending-tool-render.test.ts b/packages/coding-agent/test/suite/regressions/4167-thinking-toggle-pending-tool-render.test.ts index 2c8039b5..0b2fd842 100644 --- a/packages/coding-agent/test/suite/regressions/4167-thinking-toggle-pending-tool-render.test.ts +++ b/packages/coding-agent/test/suite/regressions/4167-thinking-toggle-pending-tool-render.test.ts @@ -42,8 +42,8 @@ type RenderSessionContextThis = { getShowImages(): boolean; getImageWidthCells(): number; }; - sessionManager: { getCwd(): string }; - session: { retryAttempt: number }; + sessionManager: { getCwd(): string; getEntries(): SessionEntry[] }; + session: { retryAttempt: number; modelRegistry: { find(provider: string, modelId: string): undefined } }; toolOutputExpanded: boolean; isInitialized: boolean; updateEditorBorderColor(): void; @@ -71,8 +71,8 @@ function createFakeInteractiveModeThis(): RenderSessionContextThis { getShowImages: () => false, getImageWidthCells: () => 60, }, - sessionManager: { getCwd: () => process.cwd() }, - session: { retryAttempt: 0 }, + sessionManager: { getCwd: () => process.cwd(), getEntries: () => [] }, + session: { retryAttempt: 0, modelRegistry: { find: () => undefined } }, toolOutputExpanded: false, isInitialized: true, updateEditorBorderColor: vi.fn(),