feat(coding-agent): add prompt cache miss tracking (#6427)

Detect prompt cache misses per turn by comparing each assistant message's
cache reads against the previous request's prompt tokens (core/cache-stats.ts).
Significant misses emit a warning-colored transcript notice at the turn they
occur, noting idle gaps past the cache TTL and model switches when relevant.

/session gained cache statistics: a compact token/cache breakdown with hit
rate, a $-prefixed cost section with per-model cost breakdown, and the
cumulative cost re-billed due to cache misses.
This commit is contained in:
Armin Ronacher
2026-07-09 14:12:57 +02:00
committed by GitHub
parent 57d96d72ed
commit 3f9aa5d10b
11 changed files with 471 additions and 36 deletions
+27 -15
View File
@@ -3016,14 +3016,15 @@ export class AgentSession {
}
/**
* Get session statistics.
* Get session statistics. Aggregates over ALL session entries (including
* history that was compacted away), so token/cost totals reflect what was
* actually billed across the session.
*/
getSessionStats(): SessionStats {
const state = this.state;
const userMessages = state.messages.filter((m) => m.role === "user").length;
const assistantMessages = state.messages.filter((m) => m.role === "assistant").length;
const toolResults = state.messages.filter((m) => m.role === "toolResult").length;
let userMessages = 0;
let assistantMessages = 0;
let toolResults = 0;
let totalMessages = 0;
let toolCalls = 0;
let totalInput = 0;
let totalOutput = 0;
@@ -3031,15 +3032,26 @@ export class AgentSession {
let totalCacheWrite = 0;
let totalCost = 0;
for (const message of state.messages) {
if (message.role === "assistant") {
for (const entry of this.sessionManager.getEntries()) {
if (entry.type !== "message") continue;
totalMessages++;
const message = entry.message;
if (message.role === "user") {
userMessages++;
} else if (message.role === "toolResult") {
toolResults++;
} else if (message.role === "assistant") {
assistantMessages++;
const assistantMsg = message as AssistantMessage;
toolCalls += assistantMsg.content.filter((c) => c.type === "toolCall").length;
totalInput += assistantMsg.usage.input;
totalOutput += assistantMsg.usage.output;
totalCacheRead += assistantMsg.usage.cacheRead;
totalCacheWrite += assistantMsg.usage.cacheWrite;
totalCost += assistantMsg.usage.cost.total;
if (Array.isArray(assistantMsg.content)) {
toolCalls += assistantMsg.content.filter((c) => c.type === "toolCall").length;
}
const usage = assistantMsg.usage;
totalInput += usage.input;
totalOutput += usage.output;
totalCacheRead += usage.cacheRead;
totalCacheWrite += usage.cacheWrite;
totalCost += usage.cost.total;
}
}
@@ -3050,7 +3062,7 @@ export class AgentSession {
assistantMessages,
toolCalls,
toolResults,
totalMessages: state.messages.length,
totalMessages,
tokens: {
input: totalInput,
output: totalOutput,
@@ -0,0 +1,164 @@
import type { AssistantMessage } from "@earendil-works/pi-ai";
import type { SessionEntry } from "./session-manager.ts";
/**
* Prompt-cache TTL: idle gaps longer than this are worth mentioning as the
* likely cause of a miss. Anthropic's default cache TTL is 5 minutes.
*/
export const CACHE_TTL_MS = 5 * 60 * 1000;
/** Per-turn misses at or below this are cache breakpoint granularity noise. */
const NOISE_FLOOR_TOKENS = 1024;
/** A counted cache miss on a single assistant message. */
export interface CacheMiss {
/** Prompt tokens that were in the previous turn's prompt but not read from cache. */
missedTokens: number;
/** Extra dollars paid vs. a full cache hit; 0 when pricing is unknown. */
missedCost: number;
/** Milliseconds since the previous request (which last refreshed the cache). */
idleMs: number;
/** True when the model changed relative to the previous request. */
modelChanged: boolean;
}
export interface CacheWasteTotals {
missedTokens: number;
missedCost: number;
/** Number of counted misses (turns above the noise floor). */
missCount: number;
}
/** Minimal pricing lookup, satisfied by ModelRegistry. Cost is $/million tokens. */
export interface ModelPriceSource {
find(provider: string, modelId: string): { cost: { cacheRead: number } } | undefined;
}
/** The last request seen by the scan; everything in its prompt should be cached. */
interface PreviousRequest {
promptTokens: number;
modelKey: string;
timestamp: number;
/**
* Sticky: some earlier request in this scan segment reported cache activity.
* Distinguishes a total miss on a cache-read-only provider (OpenAI-style,
* writes unreported) from a provider that never reports caching at all.
*/
reportedCache: boolean;
}
/**
* Compute the cache miss for one assistant message relative to the previous
* request. Returns undefined when nothing is counted: first turn, after a
* reset, no cache activity ever reported (provider without cache support), or
* miss below the noise floor.
*/
function detectMiss(
prev: PreviousRequest | undefined,
message: AssistantMessage,
models: ModelPriceSource,
): CacheMiss | undefined {
const usage = message.usage;
const promptTokens = usage.input + usage.cacheRead + usage.cacheWrite;
// A zero-cache turn only counts when cache activity was reported before:
// on cache-read-only providers that is a total miss, while on providers
// that never report caching it means nothing.
if (!prev || promptTokens <= 0 || (usage.cacheRead + usage.cacheWrite === 0 && !prev.reportedCache)) {
return undefined;
}
const missedTokens = Math.min(prev.promptTokens, promptTokens) - usage.cacheRead;
if (missedTokens <= NOISE_FLOOR_TOKENS) return undefined;
// Extra cost = missed tokens billed at the actual paid rate (input/cacheWrite,
// incl. write premium) instead of the cache-read rate. Missed tokens can only
// land in the input or cacheWrite buckets, so the paid rate comes straight
// from this message's own cost breakdown.
const paidTokens = usage.input + usage.cacheWrite;
const paidPerToken = paidTokens > 0 ? (usage.cost.input + usage.cost.cacheWrite) / paidTokens : 0;
const readPerToken =
usage.cacheRead > 0
? usage.cost.cacheRead / usage.cacheRead
: (models.find(message.provider, message.model)?.cost.cacheRead ?? 0) / 1_000_000;
return {
missedTokens,
missedCost: missedTokens * Math.max(0, paidPerToken - readPerToken),
idleMs: Math.max(0, message.timestamp - prev.timestamp),
modelChanged: `${message.provider}/${message.model}` !== prev.modelKey,
};
}
function asPreviousRequest(message: AssistantMessage, reportedCache: boolean): PreviousRequest | undefined {
const usage = message.usage;
const promptTokens = usage.input + usage.cacheRead + usage.cacheWrite;
if (promptTokens <= 0) return undefined;
return {
promptTokens,
modelKey: `${message.provider}/${message.model}`,
timestamp: message.timestamp,
reportedCache: reportedCache || usage.cacheRead + usage.cacheWrite > 0,
};
}
function scan(
entries: SessionEntry[],
models: ModelPriceSource,
): { prev: PreviousRequest | undefined; totals: CacheWasteTotals; misses: Map<AssistantMessage, CacheMiss> } {
let prev: PreviousRequest | undefined;
const totals: CacheWasteTotals = { missedTokens: 0, missedCost: 0, missCount: 0 };
const misses = new Map<AssistantMessage, CacheMiss>();
for (const entry of entries) {
if (entry.type === "compaction" || entry.type === "branch_summary") {
// The context legitimately changed; the next turn's prompt is new content,
// not re-billed content. Model switches are NOT exempt: they re-bill the
// full prompt and should be counted.
prev = undefined;
continue;
}
if (entry.type === "message" && entry.message.role === "assistant") {
const miss = detectMiss(prev, entry.message, models);
if (miss) {
totals.missedTokens += miss.missedTokens;
totals.missedCost += miss.missedCost;
totals.missCount += 1;
misses.set(entry.message, miss);
}
prev = asPreviousRequest(entry.message, prev?.reportedCache ?? false) ?? prev;
}
}
return { prev, totals, misses };
}
/**
* Cumulative cache waste across a session: prompt tokens that should have been
* cache reads (they were in the previous turn's prompt) but were re-billed.
*/
export function computeCacheWaste(entries: SessionEntry[], models: ModelPriceSource): CacheWasteTotals {
return scan(entries, models).totals;
}
/**
* All counted cache misses across a session, keyed by the assistant message
* (by reference) that paid for them. Used to re-derive transcript notices when
* rebuilding the chat from entries (resume, post-compaction rebuild).
*/
export function collectCacheMisses(
entries: SessionEntry[],
models: ModelPriceSource,
): Map<AssistantMessage, CacheMiss> {
return scan(entries, models).misses;
}
/**
* Detect a cache miss on a just-completed assistant message.
* `entries` must not yet contain `message` (message_end fires before persistence).
*/
export function detectCacheMiss(
entries: SessionEntry[],
message: AssistantMessage,
models: ModelPriceSource,
): CacheMiss | undefined {
return detectMiss(scan(entries, models).prev, message, models);
}
@@ -92,6 +92,7 @@ export interface Settings {
branchSummary?: BranchSummarySettings;
retry?: RetrySettings;
hideThinkingBlock?: boolean;
showCacheMissNotices?: boolean; // default: false - show transcript notices for significant prompt-cache misses
externalEditor?: string; // Command for Ctrl+G external editor; takes precedence over VISUAL/EDITOR
shellPath?: string; // Custom shell path (e.g., for Cygwin users on Windows)
quietStartup?: boolean;
@@ -845,6 +846,10 @@ export class SettingsManager {
return this.settings.hideThinkingBlock ?? false;
}
getShowCacheMissNotices(): boolean {
return this.settings.showCacheMissNotices ?? false;
}
getExternalEditorCommand(): string | undefined {
const configuredEditor = this.settings.externalEditor;
if (typeof configuredEditor === "string" && configuredEditor.trim() !== "") {
@@ -863,6 +868,12 @@ export class SettingsManager {
this.save();
}
setShowCacheMissNotices(show: boolean): void {
this.globalSettings.showCacheMissNotices = show;
this.markModified("showCacheMissNotices");
this.save();
}
getShellPath(): string | undefined {
return this.settings.shellPath;
}
@@ -20,7 +20,7 @@ function sanitizeStatusText(text: string): string {
/**
* Format token counts for compact footer display.
*/
function formatTokens(count: number): string {
export function formatTokens(count: number): string {
if (count < 1000) return count.toString();
if (count < 10000) return `${(count / 1000).toFixed(1)}k`;
if (count < 1000000) return `${Math.round(count / 1000)}k`;
@@ -137,7 +137,6 @@ export class FooterComponent implements Component {
if ((totalCacheRead > 0 || totalCacheWrite > 0) && latestCacheHitRate !== undefined) {
statsParts.push(`CH${latestCacheHitRate.toFixed(1)}%`);
}
// Show cost with "(sub)" indicator if using OAuth subscription
const usingSubscription = state.model ? this.session.modelRegistry.isUsingOAuth(state.model) : false;
if (totalCost || usingSubscription) {
@@ -65,6 +65,7 @@ export interface SettingsConfig {
terminalTheme: TerminalTheme;
availableThemes: string[];
hideThinkingBlock: boolean;
showCacheMissNotices: boolean;
collapseChangelog: boolean;
enableInstallTelemetry: boolean;
doubleEscapeAction: "fork" | "tree" | "none";
@@ -95,6 +96,7 @@ export interface SettingsCallbacks {
onThemeChange: (theme: string) => void;
onThemePreview?: (theme: string) => void;
onHideThinkingBlockChange: (hidden: boolean) => void;
onShowCacheMissNoticesChange: (shown: boolean) => void;
onCollapseChangelogChange: (collapsed: boolean) => void;
onEnableInstallTelemetryChange: (enabled: boolean) => void;
onDoubleEscapeActionChange: (action: "fork" | "tree" | "none") => void;
@@ -521,6 +523,13 @@ export class SettingsSelectorComponent extends Container {
currentValue: config.hideThinkingBlock ? "true" : "false",
values: ["true", "false"],
},
{
id: "cache-miss-notices",
label: "Cache miss notices",
description: "Show transcript notices for significant prompt-cache misses",
currentValue: config.showCacheMissNotices ? "true" : "false",
values: ["true", "false"],
},
{
id: "collapse-changelog",
label: "Collapse changelog",
@@ -764,6 +773,9 @@ export class SettingsSelectorComponent extends Container {
case "hide-thinking":
callbacks.onHideThinkingBlockChange(newValue === "true");
break;
case "cache-miss-notices":
callbacks.onShowCacheMissNoticesChange(newValue === "true");
break;
case "collapse-changelog":
callbacks.onCollapseChangelogChange(newValue === "true");
break;
@@ -60,6 +60,13 @@ import {
} from "../../config.ts";
import { type AgentSession, type AgentSessionEvent, parseSkillBlock } from "../../core/agent-session.ts";
import { type AgentSessionRuntime, SessionImportFileNotFoundError } from "../../core/agent-session-runtime.ts";
import {
CACHE_TTL_MS,
type CacheMiss,
collectCacheMisses,
computeCacheWaste,
detectCacheMiss,
} from "../../core/cache-stats.ts";
import type {
AutocompleteProviderFactory,
EditorFactory,
@@ -111,7 +118,7 @@ import { EarendilAnnouncementComponent } from "./components/earendil-announcemen
import { ExtensionEditorComponent } from "./components/extension-editor.ts";
import { ExtensionInputComponent } from "./components/extension-input.ts";
import { ExtensionSelectorComponent } from "./components/extension-selector.ts";
import { FooterComponent } from "./components/footer.ts";
import { FooterComponent, formatTokens } from "./components/footer.ts";
import { formatKeyText, keyDisplayText, keyHint, keyText, rawKeyHint } from "./components/keybinding-hints.ts";
import { LoginDialogComponent } from "./components/login-dialog.ts";
import { ModelSelectorComponent } from "./components/model-selector.ts";
@@ -2961,6 +2968,7 @@ export class InteractiveMode {
for (const [, component] of this.pendingTools.entries()) {
component.setArgsComplete();
}
this.maybeShowCacheMissNotice(this.streamingMessage);
}
this.streamingComponent = undefined;
this.streamingMessage = undefined;
@@ -3277,6 +3285,11 @@ export class InteractiveMode {
): void {
this.pendingTools.clear();
const renderedPendingTools = new Map<string, ToolExecutionComponent>();
// Cache-miss notices are not persisted; re-derive them from the full entry
// list and re-inject them after the assistant messages that paid for them.
const cacheMisses = this.settingsManager.getShowCacheMissNotices()
? collectCacheMisses(this.sessionManager.getEntries(), this.session.modelRegistry)
: new Map<AssistantMessage, CacheMiss>();
if (options.updateFooter) {
this.footer.invalidate();
@@ -3328,6 +3341,10 @@ export class InteractiveMode {
}
}
}
if (message.stopReason !== "aborted" && message.stopReason !== "error") {
const miss = cacheMisses.get(message);
if (miss) this.addCacheMissNotice(miss);
}
} else if (message.role === "toolResult") {
// Match tool results to pending tool components
const component = renderedPendingTools.get(message.toolCallId);
@@ -3366,6 +3383,35 @@ export class InteractiveMode {
this.renderSessionItems(items, options);
}
/**
* Show a transcript notice when a completed assistant message paid for a
* significant cache miss. Only states observable facts: the miss itself,
* a model switch, or an idle gap past the cache TTL.
*/
private maybeShowCacheMissNotice(message: AssistantMessage): void {
if (!this.settingsManager.getShowCacheMissNotices()) return;
// Entries don't contain `message` yet: message_end fires before persistence.
const miss = detectCacheMiss(this.sessionManager.getEntries(), message, this.session.modelRegistry);
if (miss) this.addCacheMissNotice(miss);
}
private addCacheMissNotice(miss: CacheMiss): void {
if (miss.missedTokens < 20_000 && miss.missedCost < 0.1) return;
const cost = miss.missedCost >= 0.01 ? ` (~$${miss.missedCost.toFixed(2)})` : "";
const reBilled = `${formatTokens(miss.missedTokens)} tokens re-billed${cost}`;
let label = "Cache miss";
if (miss.modelChanged) {
label = "Cache miss after model switch";
} else if (miss.idleMs >= CACHE_TTL_MS) {
label = `Cache miss after ${Math.round(miss.idleMs / 60_000)}m idle`;
}
const text = theme.fg("warning", `${label}: ${reBilled}`);
this.chatContainer.addChild(new Spacer(1));
this.chatContainer.addChild(new Text(text, 1, 0));
}
renderInitialMessages(): void {
const entries = this.sessionManager.buildContextEntries();
this.renderSessionEntries(entries, {
@@ -4088,6 +4134,7 @@ export class InteractiveMode {
doubleEscapeAction: this.settingsManager.getDoubleEscapeAction(),
treeFilterMode: this.settingsManager.getTreeFilterMode(),
showHardwareCursor: this.settingsManager.getShowHardwareCursor(),
showCacheMissNotices: this.settingsManager.getShowCacheMissNotices(),
defaultProjectTrust: this.settingsManager.getDefaultProjectTrust(),
editorPaddingX: this.settingsManager.getEditorPaddingX(),
outputPad: this.settingsManager.getOutputPad(),
@@ -4164,6 +4211,10 @@ export class InteractiveMode {
this.chatContainer.clear();
this.rebuildChatFromMessages();
},
onShowCacheMissNoticesChange: (shown) => {
this.settingsManager.setShowCacheMissNotices(shown);
this.rebuildChatFromMessages();
},
onCollapseChangelogChange: (collapsed) => {
this.settingsManager.setCollapseChangelog(collapsed);
},
@@ -5561,6 +5612,26 @@ export class InteractiveMode {
private handleSessionCommand(): void {
const stats = this.session.getSessionStats();
const sessionName = this.sessionManager.getSessionName();
const entries = this.sessionManager.getEntries();
const cacheWaste = computeCacheWaste(entries, this.session.modelRegistry);
// Cost/token totals per provider/model actually used (e.g. OpenRouter `auto`
// resolves to a concrete responseModel), sorted by cost descending.
const perModelMap = new Map<string, { key: string; cost: number; tokens: number }>();
for (const entry of entries) {
if (entry.type !== "message" || entry.message.role !== "assistant") continue;
const message = entry.message;
const usage = message.usage;
const key = `${message.provider}/${message.responseModel ?? message.model}`;
let bucket = perModelMap.get(key);
if (!bucket) {
bucket = { key, cost: 0, tokens: 0 };
perModelMap.set(key, bucket);
}
bucket.cost += usage.cost.total;
bucket.tokens += usage.input + usage.output + usage.cacheRead + usage.cacheWrite;
}
const perModel = Array.from(perModelMap.values()).sort((a, b) => b.cost - a.cost);
let info = `${theme.bold("Session Info")}\n\n`;
if (sessionName) {
@@ -5569,25 +5640,44 @@ export class InteractiveMode {
info += `${theme.fg("dim", "File:")} ${stats.sessionFile ?? "In-memory"}\n`;
info += `${theme.fg("dim", "ID:")} ${stats.sessionId}\n\n`;
info += `${theme.bold("Messages")}\n`;
info += `${theme.fg("dim", "Total:")} ${stats.totalMessages}\n`;
info += `${theme.fg("dim", "User:")} ${stats.userMessages}\n`;
info += `${theme.fg("dim", "Assistant:")} ${stats.assistantMessages}\n`;
info += `${theme.fg("dim", "Tool Calls:")} ${stats.toolCalls}\n`;
info += `${theme.fg("dim", "Tool Results:")} ${stats.toolResults}\n`;
info += `${theme.fg("dim", "Total:")} ${stats.totalMessages}\n\n`;
info += `${theme.fg("dim", "Tools:")} ${stats.toolCalls} calls, ${stats.toolResults} results\n\n`;
info += `${theme.bold("Tokens")}\n`;
info += `${theme.fg("dim", "Input:")} ${stats.tokens.input.toLocaleString()}\n`;
// "Input" is the full prompt volume. With cache activity, split it into
// cached (served from cache) vs uncached (everything else) - the only
// provider-independent split. Cache writes, where reported, are a detail
// of the uncached portion.
const { input, cacheRead, cacheWrite } = stats.tokens;
const promptTokens = input + cacheRead + cacheWrite;
info += `${theme.fg("dim", "Input:")} ${promptTokens.toLocaleString()}\n`;
if (promptTokens > 0 && (cacheRead > 0 || cacheWrite > 0)) {
const hitRate = theme.fg("dim", `(${((cacheRead / promptTokens) * 100).toFixed(1)}%)`);
info += ` ${theme.fg("dim", "Cached:")} ${cacheRead.toLocaleString()} ${hitRate}\n`;
const written =
cacheWrite > 0 ? ` ${theme.fg("dim", `(${cacheWrite.toLocaleString()} written to cache)`)}` : "";
info += ` ${theme.fg("dim", "Uncached:")} ${(input + cacheWrite).toLocaleString()}${written}\n`;
}
info += `${theme.fg("dim", "Output:")} ${stats.tokens.output.toLocaleString()}\n`;
if (stats.tokens.cacheRead > 0) {
info += `${theme.fg("dim", "Cache Read:")} ${stats.tokens.cacheRead.toLocaleString()}\n`;
}
if (stats.tokens.cacheWrite > 0) {
info += `${theme.fg("dim", "Cache Write:")} ${stats.tokens.cacheWrite.toLocaleString()}\n`;
}
info += `${theme.fg("dim", "Total:")} ${stats.tokens.total.toLocaleString()}\n`;
if (stats.cost > 0) {
if (stats.cost > 0 || cacheWaste.missedTokens > 0) {
info += `\n${theme.bold("Cost")}\n`;
info += `${theme.fg("dim", "Total:")} ${stats.cost.toFixed(4)}`;
info += `${theme.fg("dim", "Total:")} $${stats.cost.toFixed(3)}`;
if (perModel.length > 1) {
for (const entry of perModel) {
info += `\n ${theme.fg("dim", `${entry.key}:`)} $${entry.cost.toFixed(3)} ${theme.fg("dim", `(${formatTokens(entry.tokens)} tokens)`)}`;
}
}
if (cacheWaste.missedTokens > 0) {
const missLabel = cacheWaste.missCount === 1 ? "1 miss" : `${cacheWaste.missCount} misses`;
const detail = `${cacheWaste.missedTokens.toLocaleString()} tokens, ${missLabel}`;
info +=
cacheWaste.missedCost >= 0.0001
? `\n${theme.fg("dim", "Cache Re-billed:")} $${cacheWaste.missedCost.toFixed(3)} ${theme.fg("dim", `(${detail})`)}`
: `\n${theme.fg("dim", "Cache Re-billed:")} ${detail}`;
}
}
this.chatContainer.addChild(new Spacer(1));