feat(coding-agent): add prompt cache miss tracking (#6427)

Detect prompt cache misses per turn by comparing each assistant message's cache reads against the previous request's prompt tokens (core/cache-stats.ts). Significant misses emit a warning-colored transcript notice at the turn they occur, noting idle gaps past the cache TTL and model switches when relevant. /session gained cache statistics: a compact token/cache breakdown with hit rate, a $-prefixed cost section with per-model cost breakdown, and the cumulative cost re-billed due to cache misses.
2026-07-09 14:12:57 +02:00
parent 57d96d72ed
commit 3f9aa5d10b
11 changed files with 471 additions and 36 deletions
@@ -109,7 +109,8 @@ describe("AgentSession.getSessionStats", () => {
 			syncAgentMessages(session, sessionManager);

 			const stats = session.getSessionStats();
-			expect(stats.tokens.input).toBe(195_000);
+			// Totals cover ALL entries, including history compacted away (180k + 195k).
+			expect(stats.tokens.input).toBe(375_000);
 			expect(stats.contextUsage).toBeDefined();
 			expect(stats.contextUsage?.tokens).toBeNull();
 			expect(stats.contextUsage?.percent).toBeNull();
@@ -132,7 +133,8 @@ describe("AgentSession.getSessionStats", () => {
 			syncAgentMessages(session, sessionManager);

 			const stats = session.getSessionStats();
-			expect(stats.tokens.input).toBe(220_000);
+			// Totals cover ALL entries, including history compacted away (180k + 195k + 25k).
+			expect(stats.tokens.input).toBe(400_000);
 			expect(stats.contextUsage).toBeDefined();
 			expect(stats.contextUsage?.tokens).toBe(25_000);
 			expect(stats.contextUsage?.percent).toBe((25_000 / model.contextWindow) * 100);
@@ -0,0 +1,143 @@
+import type { AssistantMessage } from "@earendil-works/pi-ai";
+import { describe, expect, it } from "vitest";
+import {
+	collectCacheMisses,
+	computeCacheWaste,
+	detectCacheMiss,
+	type ModelPriceSource,
+} from "../src/core/cache-stats.ts";
+import type { SessionEntry } from "../src/core/session-manager.ts";
+
+const zeroCost = { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 };
+
+const models: ModelPriceSource = {
+	// $/million tokens; used as cache-read price fallback on full-miss turns
+	find: () => ({ cost: { cacheRead: 0.3 } }),
+};
+
+function assistant(options: {
+	input?: number;
+	cacheRead?: number;
+	cacheWrite?: number;
+	cost?: Partial<typeof zeroCost>;
+	model?: string;
+	timestamp?: number;
+}): AssistantMessage {
+	return {
+		role: "assistant",
+		content: [],
+		api: "anthropic-messages",
+		provider: "test",
+		model: options.model ?? "test-model",
+		usage: {
+			input: options.input ?? 0,
+			output: 10,
+			cacheRead: options.cacheRead ?? 0,
+			cacheWrite: options.cacheWrite ?? 0,
+			totalTokens: 0,
+			cost: { ...zeroCost, ...options.cost },
+		},
+		stopReason: "stop",
+		timestamp: options.timestamp ?? 0,
+	} as AssistantMessage;
+}
+
+function entry(message: AssistantMessage): SessionEntry {
+	return { type: "message", id: "x", parentId: null, timestamp: "", message } as SessionEntry;
+}
+
+// Turn 1: fresh 100k cache write at $3.75/M
+const turn1 = assistant({ cacheWrite: 100_000, cost: { cacheWrite: 0.375 }, timestamp: 0 });
+// Turn 2: healthy, everything read back at $0.30/M
+const turn2 = assistant({
+	cacheRead: 100_000,
+	cacheWrite: 5_000,
+	cost: { cacheRead: 0.03, cacheWrite: 0.019 },
+	timestamp: 60_000,
+});
+
+describe("computeCacheWaste", () => {
+	it("accumulates missed tokens and cost across turns", () => {
+		// Turn 3: full miss, previous 105k prompt re-billed at $3.75/M write
+		const turn3 = assistant({ cacheWrite: 110_000, cost: { cacheWrite: 0.4125 }, timestamp: 120_000 });
+		const totals = computeCacheWaste([entry(turn1), entry(turn2), entry(turn3)], models);
+		expect(totals.missedTokens).toBe(105_000);
+		// 105k at ($3.75 - $0.30)/M
+		expect(totals.missedCost).toBeCloseTo(0.36225, 5);
+	});
+
+	it("counts nothing for healthy sessions", () => {
+		const totals = computeCacheWaste([entry(turn1), entry(turn2)], models);
+		expect(totals.missedTokens).toBe(0);
+		expect(totals.missedCost).toBe(0);
+	});
+
+	it("skips the turn after a compaction reset", () => {
+		const reset = { type: "compaction", id: "c", parentId: null, timestamp: "" } as SessionEntry;
+		const afterReset = assistant({ cacheWrite: 20_000, cost: { cacheWrite: 0.075 } });
+		const totals = computeCacheWaste([entry(turn1), reset, entry(afterReset)], models);
+		expect(totals.missedTokens).toBe(0);
+	});
+
+	it("counts misses caused by model switches", () => {
+		const otherModel = assistant({ cacheWrite: 100_000, cost: { cacheWrite: 0.375 }, model: "other-model" });
+		const totals = computeCacheWaste([entry(turn1), entry(otherModel)], models);
+		expect(totals.missedTokens).toBe(100_000);
+		expect(totals.missCount).toBe(1);
+	});
+
+	it("skips providers that report no cache activity", () => {
+		const a = assistant({ input: 100_000 });
+		const b = assistant({ input: 110_000 });
+		const totals = computeCacheWaste([entry(a), entry(b)], models);
+		expect(totals.missedTokens).toBe(0);
+	});
+});
+
+describe("collectCacheMisses", () => {
+	it("maps counted misses to their assistant messages by reference", () => {
+		const missTurn = assistant({ cacheWrite: 110_000, cost: { cacheWrite: 0.4125 }, timestamp: 120_000 });
+		const misses = collectCacheMisses([entry(turn1), entry(turn2), entry(missTurn)], models);
+		expect(misses.size).toBe(1);
+		expect(misses.get(missTurn)?.missedTokens).toBe(105_000);
+	});
+});
+
+describe("detectCacheMiss", () => {
+	it("detects a miss on a just-completed message with idle time", () => {
+		const missMessage = assistant({ cacheWrite: 110_000, cost: { cacheWrite: 0.4125 }, timestamp: 600_000 });
+		const miss = detectCacheMiss([entry(turn1), entry(turn2)], missMessage, models);
+		expect(miss).toBeDefined();
+		expect(miss?.missedTokens).toBe(105_000);
+		expect(miss?.missedCost).toBeCloseTo(0.36225, 5);
+		// 600s - 60s since the previous request
+		expect(miss?.idleMs).toBe(540_000);
+		expect(miss?.modelChanged).toBe(false);
+	});
+
+	it("flags model switches on detected misses", () => {
+		const otherModel = assistant({
+			cacheWrite: 110_000,
+			cost: { cacheWrite: 0.4125 },
+			model: "other-model",
+			timestamp: 120_000,
+		});
+		const miss = detectCacheMiss([entry(turn1), entry(turn2)], otherModel, models);
+		expect(miss?.missedTokens).toBe(105_000);
+		expect(miss?.modelChanged).toBe(true);
+	});
+
+	it("returns undefined for healthy turns", () => {
+		const healthy = assistant({
+			cacheRead: 105_000,
+			cacheWrite: 2_000,
+			cost: { cacheRead: 0.0315, cacheWrite: 0.0075 },
+			timestamp: 120_000,
+		});
+		expect(detectCacheMiss([entry(turn1), entry(turn2)], healthy, models)).toBeUndefined();
+	});
+
+	it("returns undefined for the first turn of a session", () => {
+		expect(detectCacheMiss([], turn1, models)).toBeUndefined();
+	});
+});
@@ -42,8 +42,8 @@ type RenderSessionContextThis = {
 		getShowImages(): boolean;
 		getImageWidthCells(): number;
 	};
-	sessionManager: { getCwd(): string };
-	session: { retryAttempt: number };
+	sessionManager: { getCwd(): string; getEntries(): SessionEntry[] };
+	session: { retryAttempt: number; modelRegistry: { find(provider: string, modelId: string): undefined } };
 	toolOutputExpanded: boolean;
 	isInitialized: boolean;
 	updateEditorBorderColor(): void;
@@ -71,8 +71,8 @@ function createFakeInteractiveModeThis(): RenderSessionContextThis {
 			getShowImages: () => false,
 			getImageWidthCells: () => 60,
 		},
-		sessionManager: { getCwd: () => process.cwd() },
-		session: { retryAttempt: 0 },
+		sessionManager: { getCwd: () => process.cwd(), getEntries: () => [] },
+		session: { retryAttempt: 0, modelRegistry: { find: () => undefined } },
 		toolOutputExpanded: false,
 		isInitialized: true,
 		updateEditorBorderColor: vi.fn(),