ton/pi_harness
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(coding-agent): add prompt cache miss tracking (#6427)

Browse Source 
Detect prompt cache misses per turn by comparing each assistant message's
cache reads against the previous request's prompt tokens (core/cache-stats.ts).
Significant misses emit a warning-colored transcript notice at the turn they
occur, noting idle gaps past the cache TTL and model switches when relevant.

/session gained cache statistics: a compact token/cache breakdown with hit
rate, a $-prefixed cost section with per-model cost breakdown, and the
cumulative cost re-billed due to cache misses.
This commit is contained in:
Armin Ronacher
2026-07-09 14:12:57 +02:00
committed by GitHub
parent 57d96d72ed
commit 3f9aa5d10b
11 changed files with 471 additions and 36 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/coding-agent/src/modes/interactive/interactive-mode.ts
+103 -13
View File
@@ -60,6 +60,13 @@ import {
} from "../../config.ts";
import { type AgentSession, type AgentSessionEvent, parseSkillBlock } from "../../core/agent-session.ts";
import { type AgentSessionRuntime, SessionImportFileNotFoundError } from "../../core/agent-session-runtime.ts";
import {
CACHE_TTL_MS,
type CacheMiss,
collectCacheMisses,
computeCacheWaste,
detectCacheMiss,
} from "../../core/cache-stats.ts";
import type {
AutocompleteProviderFactory,
EditorFactory,
@@ -111,7 +118,7 @@ import { EarendilAnnouncementComponent } from "./components/earendil-announcemen
import { ExtensionEditorComponent } from "./components/extension-editor.ts";
import { ExtensionInputComponent } from "./components/extension-input.ts";
import { ExtensionSelectorComponent } from "./components/extension-selector.ts";
import { FooterComponent } from "./components/footer.ts";
import { FooterComponent, formatTokens } from "./components/footer.ts";
import { formatKeyText, keyDisplayText, keyHint, keyText, rawKeyHint } from "./components/keybinding-hints.ts";
import { LoginDialogComponent } from "./components/login-dialog.ts";
import { ModelSelectorComponent } from "./components/model-selector.ts";
@@ -2961,6 +2968,7 @@ export class InteractiveMode {
for (const [, component] of this.pendingTools.entries()) {
component.setArgsComplete();
}
this.maybeShowCacheMissNotice(this.streamingMessage);
}
this.streamingComponent = undefined;
this.streamingMessage = undefined;
@@ -3277,6 +3285,11 @@ export class InteractiveMode {
): void {
this.pendingTools.clear();
const renderedPendingTools = new Map<string, ToolExecutionComponent>();
// Cache-miss notices are not persisted; re-derive them from the full entry
// list and re-inject them after the assistant messages that paid for them.
const cacheMisses = this.settingsManager.getShowCacheMissNotices()
? collectCacheMisses(this.sessionManager.getEntries(), this.session.modelRegistry)
: new Map<AssistantMessage, CacheMiss>();
if (options.updateFooter) {
this.footer.invalidate();
@@ -3328,6 +3341,10 @@ export class InteractiveMode {
}
}
}
if (message.stopReason !== "aborted" && message.stopReason !== "error") {
const miss = cacheMisses.get(message);
if (miss) this.addCacheMissNotice(miss);
}
} else if (message.role === "toolResult") {
// Match tool results to pending tool components
const component = renderedPendingTools.get(message.toolCallId);
@@ -3366,6 +3383,35 @@ export class InteractiveMode {
this.renderSessionItems(items, options);
}
/**
* Show a transcript notice when a completed assistant message paid for a
* significant cache miss. Only states observable facts: the miss itself,
* a model switch, or an idle gap past the cache TTL.
*/
private maybeShowCacheMissNotice(message: AssistantMessage): void {
if (!this.settingsManager.getShowCacheMissNotices()) return;
// Entries don't contain `message` yet: message_end fires before persistence.
const miss = detectCacheMiss(this.sessionManager.getEntries(), message, this.session.modelRegistry);
if (miss) this.addCacheMissNotice(miss);
}
private addCacheMissNotice(miss: CacheMiss): void {
if (miss.missedTokens < 20_000 && miss.missedCost < 0.1) return;
const cost = miss.missedCost >= 0.01 ? ` (~$${miss.missedCost.toFixed(2)})` : "";
const reBilled = `${formatTokens(miss.missedTokens)} tokens re-billed${cost}`;
let label = "Cache miss";
if (miss.modelChanged) {
label = "Cache miss after model switch";
} else if (miss.idleMs >= CACHE_TTL_MS) {
label = `Cache miss after ${Math.round(miss.idleMs / 60_000)}m idle`;
}
const text = theme.fg("warning", `${label}: ${reBilled}`);
this.chatContainer.addChild(new Spacer(1));
this.chatContainer.addChild(new Text(text, 1, 0));
}
renderInitialMessages(): void {
const entries = this.sessionManager.buildContextEntries();
this.renderSessionEntries(entries, {
@@ -4088,6 +4134,7 @@ export class InteractiveMode {
doubleEscapeAction: this.settingsManager.getDoubleEscapeAction(),
treeFilterMode: this.settingsManager.getTreeFilterMode(),
showHardwareCursor: this.settingsManager.getShowHardwareCursor(),
showCacheMissNotices: this.settingsManager.getShowCacheMissNotices(),
defaultProjectTrust: this.settingsManager.getDefaultProjectTrust(),
editorPaddingX: this.settingsManager.getEditorPaddingX(),
outputPad: this.settingsManager.getOutputPad(),
@@ -4164,6 +4211,10 @@ export class InteractiveMode {
this.chatContainer.clear();
this.rebuildChatFromMessages();
},
onShowCacheMissNoticesChange: (shown) => {
this.settingsManager.setShowCacheMissNotices(shown);
this.rebuildChatFromMessages();
},
onCollapseChangelogChange: (collapsed) => {
this.settingsManager.setCollapseChangelog(collapsed);
},
@@ -5561,6 +5612,26 @@ export class InteractiveMode {
private handleSessionCommand(): void {
const stats = this.session.getSessionStats();
const sessionName = this.sessionManager.getSessionName();
const entries = this.sessionManager.getEntries();
const cacheWaste = computeCacheWaste(entries, this.session.modelRegistry);
// Cost/token totals per provider/model actually used (e.g. OpenRouter `auto`
// resolves to a concrete responseModel), sorted by cost descending.
const perModelMap = new Map<string, { key: string; cost: number; tokens: number }>();
for (const entry of entries) {
if (entry.type !== "message" || entry.message.role !== "assistant") continue;
const message = entry.message;
const usage = message.usage;
const key = `${message.provider}/${message.responseModel ?? message.model}`;
let bucket = perModelMap.get(key);
if (!bucket) {
bucket = { key, cost: 0, tokens: 0 };
perModelMap.set(key, bucket);
}
bucket.cost += usage.cost.total;
bucket.tokens += usage.input + usage.output + usage.cacheRead + usage.cacheWrite;
}
const perModel = Array.from(perModelMap.values()).sort((a, b) => b.cost - a.cost);
let info = `${theme.bold("Session Info")}\n\n`;
if (sessionName) {
@@ -5569,25 +5640,44 @@ export class InteractiveMode {
info += `${theme.fg("dim", "File:")} ${stats.sessionFile ?? "In-memory"}\n`;
info += `${theme.fg("dim", "ID:")} ${stats.sessionId}\n\n`;
info += `${theme.bold("Messages")}\n`;
info += `${theme.fg("dim", "Total:")} ${stats.totalMessages}\n`;
info += `${theme.fg("dim", "User:")} ${stats.userMessages}\n`;
info += `${theme.fg("dim", "Assistant:")} ${stats.assistantMessages}\n`;
info += `${theme.fg("dim", "Tool Calls:")} ${stats.toolCalls}\n`;
info += `${theme.fg("dim", "Tool Results:")} ${stats.toolResults}\n`;
info += `${theme.fg("dim", "Total:")} ${stats.totalMessages}\n\n`;
info += `${theme.fg("dim", "Tools:")} ${stats.toolCalls} calls, ${stats.toolResults} results\n\n`;
info += `${theme.bold("Tokens")}\n`;
info += `${theme.fg("dim", "Input:")} ${stats.tokens.input.toLocaleString()}\n`;
// "Input" is the full prompt volume. With cache activity, split it into
// cached (served from cache) vs uncached (everything else) - the only
// provider-independent split. Cache writes, where reported, are a detail
// of the uncached portion.
const { input, cacheRead, cacheWrite } = stats.tokens;
const promptTokens = input + cacheRead + cacheWrite;
info += `${theme.fg("dim", "Input:")} ${promptTokens.toLocaleString()}\n`;
if (promptTokens > 0 && (cacheRead > 0 || cacheWrite > 0)) {
const hitRate = theme.fg("dim", `(${((cacheRead / promptTokens) * 100).toFixed(1)}%)`);
info += ` ${theme.fg("dim", "Cached:")} ${cacheRead.toLocaleString()} ${hitRate}\n`;
const written =
cacheWrite > 0 ? ` ${theme.fg("dim", `(${cacheWrite.toLocaleString()} written to cache)`)}` : "";
info += ` ${theme.fg("dim", "Uncached:")} ${(input + cacheWrite).toLocaleString()}${written}\n`;
}
info += `${theme.fg("dim", "Output:")} ${stats.tokens.output.toLocaleString()}\n`;
if (stats.tokens.cacheRead > 0) {
info += `${theme.fg("dim", "Cache Read:")} ${stats.tokens.cacheRead.toLocaleString()}\n`;
}
if (stats.tokens.cacheWrite > 0) {
info += `${theme.fg("dim", "Cache Write:")} ${stats.tokens.cacheWrite.toLocaleString()}\n`;
}
info += `${theme.fg("dim", "Total:")} ${stats.tokens.total.toLocaleString()}\n`;
if (stats.cost > 0) {
if (stats.cost > 0 || cacheWaste.missedTokens > 0) {
info += `\n${theme.bold("Cost")}\n`;
info += `${theme.fg("dim", "Total:")} ${stats.cost.toFixed(4)}`;
info += `${theme.fg("dim", "Total:")} $${stats.cost.toFixed(3)}`;
if (perModel.length > 1) {
for (const entry of perModel) {
info += `\n ${theme.fg("dim", `${entry.key}:`)} $${entry.cost.toFixed(3)} ${theme.fg("dim", `(${formatTokens(entry.tokens)} tokens)`)}`;
}
}
if (cacheWaste.missedTokens > 0) {
const missLabel = cacheWaste.missCount === 1 ? "1 miss" : `${cacheWaste.missCount} misses`;
const detail = `${cacheWaste.missedTokens.toLocaleString()} tokens, ${missLabel}`;
info +=
cacheWaste.missedCost >= 0.0001
? `\n${theme.fg("dim", "Cache Re-billed:")} $${cacheWaste.missedCost.toFixed(3)} ${theme.fg("dim", `(${detail})`)}`
: `\n${theme.fg("dim", "Cache Re-billed:")} ${detail}`;
}
}
this.chatContainer.addChild(new Spacer(1));