add usage info to branch summary, compaction and tool result entries (#6671)

* add usage info to branch summary entries

* add usage to compaction entries

* allow custom tools to report llm usage in tool results

* allow observing and patching usage in tool_result hooks

* agent-harness: save usage in entries

for compaction, branch summaries and tool results
This commit is contained in:
David Brailovsky
2026-07-20 16:41:43 +02:00
committed by GitHub
parent c179395218
commit 2fd3868401
29 changed files with 844 additions and 122 deletions
+33 -20
View File
@@ -32,6 +32,7 @@ import type {
Model,
ProviderHeaders,
TextContent,
Usage,
} from "@earendil-works/pi-ai/compat";
import {
clampThinkingLevel,
@@ -104,6 +105,7 @@ import { type BuildSystemPromptOptions, buildSystemPrompt } from "./system-promp
import { type BashOperations, createLocalBashOperations } from "./tools/bash.ts";
import { createAllToolDefinitions } from "./tools/index.ts";
import { createToolDefinitionFromAgentTool } from "./tools/tool-definition-wrapper.ts";
import { addUsageToTotals, createUsageTotals } from "./usage-totals.ts";
// ============================================================================
// Skill Block Parsing
@@ -482,6 +484,7 @@ export class AgentSession {
content: result.content,
details: result.details,
isError,
usage: result.usage,
});
if (!hookResult) {
@@ -492,6 +495,7 @@ export class AgentSession {
content: hookResult.content,
details: hookResult.details,
isError: hookResult.isError ?? isError,
usage: hookResult.usage,
};
};
}
@@ -1812,6 +1816,7 @@ export class AgentSession {
let summary: string;
let firstKeptEntryId: string;
let tokensBefore: number;
let usage: Usage | undefined;
let details: unknown;
if (extensionCompaction) {
@@ -1819,6 +1824,7 @@ export class AgentSession {
summary = extensionCompaction.summary;
firstKeptEntryId = extensionCompaction.firstKeptEntryId;
tokensBefore = extensionCompaction.tokensBefore;
usage = extensionCompaction.usage;
details = extensionCompaction.details;
} else {
// Generate compaction result
@@ -1836,6 +1842,7 @@ export class AgentSession {
summary = result.summary;
firstKeptEntryId = result.firstKeptEntryId;
tokensBefore = result.tokensBefore;
usage = result.usage;
details = result.details;
}
@@ -1843,7 +1850,7 @@ export class AgentSession {
throw new Error("Compaction cancelled");
}
this.sessionManager.appendCompaction(summary, firstKeptEntryId, tokensBefore, details, fromExtension);
this.sessionManager.appendCompaction(summary, firstKeptEntryId, tokensBefore, details, fromExtension, usage);
const newEntries = this.sessionManager.getEntries();
const sessionContext = this.sessionManager.buildSessionContext();
this.agent.state.messages = sessionContext.messages;
@@ -1869,6 +1876,7 @@ export class AgentSession {
firstKeptEntryId,
tokensBefore,
estimatedTokensAfter,
usage,
details,
};
this._emit({
@@ -2084,6 +2092,7 @@ export class AgentSession {
let summary: string;
let firstKeptEntryId: string;
let tokensBefore: number;
let usage: Usage | undefined;
let details: unknown;
if (extensionCompaction) {
@@ -2091,6 +2100,7 @@ export class AgentSession {
summary = extensionCompaction.summary;
firstKeptEntryId = extensionCompaction.firstKeptEntryId;
tokensBefore = extensionCompaction.tokensBefore;
usage = extensionCompaction.usage;
details = extensionCompaction.details;
} else {
// Generate compaction result
@@ -2108,6 +2118,7 @@ export class AgentSession {
summary = compactResult.summary;
firstKeptEntryId = compactResult.firstKeptEntryId;
tokensBefore = compactResult.tokensBefore;
usage = compactResult.usage;
details = compactResult.details;
}
@@ -2122,7 +2133,7 @@ export class AgentSession {
return false;
}
this.sessionManager.appendCompaction(summary, firstKeptEntryId, tokensBefore, details, fromExtension);
this.sessionManager.appendCompaction(summary, firstKeptEntryId, tokensBefore, details, fromExtension, usage);
const newEntries = this.sessionManager.getEntries();
const sessionContext = this.sessionManager.buildSessionContext();
this.agent.state.messages = sessionContext.messages;
@@ -2148,6 +2159,7 @@ export class AgentSession {
firstKeptEntryId,
tokensBefore,
estimatedTokensAfter,
usage,
details,
};
this._emit({ type: "compaction_end", reason, result, aborted: false, willRetry });
@@ -2872,7 +2884,7 @@ export class AgentSession {
this._branchSummaryAbortController = new AbortController();
try {
let extensionSummary: { summary: string; details?: unknown } | undefined;
let extensionSummary: { summary: string; details?: unknown; usage?: Usage } | undefined;
let fromExtension = false;
// Emit session_before_tree event
@@ -2907,6 +2919,7 @@ export class AgentSession {
// Run default summarizer if needed
let summaryText: string | undefined;
let summaryDetails: unknown;
let summaryUsage: Usage | undefined;
if (options.summarize && entriesToSummarize.length > 0 && !extensionSummary) {
const model = this.model!;
const { apiKey, headers, env } = await this._getSummarizationRequestAuth(model);
@@ -2929,6 +2942,7 @@ export class AgentSession {
throw new Error(result.error);
}
summaryText = result.summary;
summaryUsage = result.usage;
summaryDetails = {
readFiles: result.readFiles || [],
modifiedFiles: result.modifiedFiles || [],
@@ -2936,6 +2950,7 @@ export class AgentSession {
} else if (extensionSummary) {
summaryText = extensionSummary.summary;
summaryDetails = extensionSummary.details;
summaryUsage = extensionSummary.usage;
}
// Determine the new leaf position based on target type
@@ -2965,6 +2980,7 @@ export class AgentSession {
summaryText,
summaryDetails,
fromExtension,
summaryUsage,
);
summaryEntry = this.sessionManager.getEntry(summaryId) as BranchSummaryEntry;
@@ -3037,13 +3053,12 @@ export class AgentSession {
let toolResults = 0;
let totalMessages = 0;
let toolCalls = 0;
let totalInput = 0;
let totalOutput = 0;
let totalCacheRead = 0;
let totalCacheWrite = 0;
let totalCost = 0;
const usageTotals = createUsageTotals();
for (const entry of this.sessionManager.getEntries()) {
if ((entry.type === "branch_summary" || entry.type === "compaction") && entry.usage) {
addUsageToTotals(usageTotals, entry.usage);
}
if (entry.type !== "message") continue;
totalMessages++;
const message = entry.message;
@@ -3051,18 +3066,16 @@ export class AgentSession {
userMessages++;
} else if (message.role === "toolResult") {
toolResults++;
if (message.usage) {
addUsageToTotals(usageTotals, message.usage);
}
} else if (message.role === "assistant") {
assistantMessages++;
const assistantMsg = message as AssistantMessage;
if (Array.isArray(assistantMsg.content)) {
toolCalls += assistantMsg.content.filter((c) => c.type === "toolCall").length;
}
const usage = assistantMsg.usage;
totalInput += usage.input;
totalOutput += usage.output;
totalCacheRead += usage.cacheRead;
totalCacheWrite += usage.cacheWrite;
totalCost += usage.cost.total;
addUsageToTotals(usageTotals, assistantMsg.usage);
}
}
@@ -3075,13 +3088,13 @@ export class AgentSession {
toolResults,
totalMessages,
tokens: {
input: totalInput,
output: totalOutput,
cacheRead: totalCacheRead,
cacheWrite: totalCacheWrite,
total: totalInput + totalOutput + totalCacheRead + totalCacheWrite,
input: usageTotals.input,
output: usageTotals.output,
cacheRead: usageTotals.cacheRead,
cacheWrite: usageTotals.cacheWrite,
total: usageTotals.input + usageTotals.output + usageTotals.cacheRead + usageTotals.cacheWrite,
},
cost: totalCost,
cost: usageTotals.cost,
contextUsage: this.getContextUsage(),
};
}
@@ -7,7 +7,7 @@
import type { AgentMessage, StreamFn } from "@earendil-works/pi-agent-core";
import { contentText } from "@earendil-works/pi-ai";
import type { Model, SimpleStreamOptions } from "@earendil-works/pi-ai/compat";
import type { Model, SimpleStreamOptions, Usage } from "@earendil-works/pi-ai/compat";
import { completeSimple } from "@earendil-works/pi-ai/compat";
import {
convertToLlm,
@@ -33,6 +33,7 @@ import {
export interface BranchSummaryResult {
summary?: string;
usage?: Usage;
readFiles?: string[];
modifiedFiles?: string[];
aborted?: boolean;
@@ -363,6 +364,7 @@ export async function generateBranchSummary(
return {
summary: summary || "No summary generated",
usage: response.usage,
readFiles,
modifiedFiles,
};
@@ -90,10 +90,35 @@ export interface CompactionResult<T = unknown> {
firstKeptEntryId: string;
tokensBefore: number;
estimatedTokensAfter?: number;
/** Usage from the LLM call(s) that generated this summary, if available */
usage?: Usage;
/** Extension-specific data (e.g., ArtifactIndex, version markers for structured compaction) */
details?: T;
}
function combineUsage(first: Usage, second: Usage): Usage {
return {
input: first.input + second.input,
output: first.output + second.output,
cacheRead: first.cacheRead + second.cacheRead,
cacheWrite: first.cacheWrite + second.cacheWrite,
...(first.cacheWrite1h !== undefined || second.cacheWrite1h !== undefined
? { cacheWrite1h: (first.cacheWrite1h ?? 0) + (second.cacheWrite1h ?? 0) }
: {}),
...(first.reasoning !== undefined || second.reasoning !== undefined
? { reasoning: (first.reasoning ?? 0) + (second.reasoning ?? 0) }
: {}),
totalTokens: first.totalTokens + second.totalTokens,
cost: {
input: first.cost.input + second.cost.input,
output: first.cost.output + second.cost.output,
cacheRead: first.cost.cacheRead + second.cost.cacheRead,
cacheWrite: first.cost.cacheWrite + second.cost.cacheWrite,
total: first.cost.total + second.cost.total,
},
};
}
// ============================================================================
// Types
// ============================================================================
@@ -556,7 +581,7 @@ export async function generateSummary(
thinkingLevel?: ThinkingLevel,
streamFn?: StreamFn,
env?: Record<string, string>,
): Promise<string> {
): Promise<{ text: string; usage: Usage }> {
const maxTokens = Math.min(
Math.floor(0.8 * reserveTokens),
model.maxTokens > 0 ? model.maxTokens : Number.POSITIVE_INFINITY,
@@ -603,7 +628,7 @@ export async function generateSummary(
const textContent = contentText(response.content);
return textContent;
return { text: textContent, usage: response.usage };
}
// ============================================================================
@@ -759,24 +784,28 @@ export async function compact(
// Generate summaries and merge into one
let summary: string;
let summaryUsage: Usage;
if (isSplitTurn && turnPrefixMessages.length > 0) {
const historyResult =
messagesToSummarize.length > 0
? await generateSummary(
messagesToSummarize,
model,
settings.reserveTokens,
apiKey,
headers,
signal,
customInstructions,
previousSummary,
thinkingLevel,
streamFn,
env,
)
: "No prior history.";
let historyText = "No prior history.";
let historyUsage: Usage | undefined;
if (messagesToSummarize.length > 0) {
const historyResult = await generateSummary(
messagesToSummarize,
model,
settings.reserveTokens,
apiKey,
headers,
signal,
customInstructions,
previousSummary,
thinkingLevel,
streamFn,
env,
);
historyText = historyResult.text;
historyUsage = historyResult.usage;
}
const turnPrefixResult = await generateTurnPrefixSummary(
turnPrefixMessages,
model,
@@ -789,10 +818,11 @@ export async function compact(
streamFn,
);
// Merge into single summary
summary = `${historyResult}\n\n---\n\n**Turn Context (split turn):**\n\n${turnPrefixResult}`;
summary = `${historyText}\n\n---\n\n**Turn Context (split turn):**\n\n${turnPrefixResult.text}`;
summaryUsage = historyUsage ? combineUsage(historyUsage, turnPrefixResult.usage) : turnPrefixResult.usage;
} else {
// Just generate history summary
summary = await generateSummary(
const result = await generateSummary(
messagesToSummarize,
model,
settings.reserveTokens,
@@ -805,6 +835,8 @@ export async function compact(
streamFn,
env,
);
summary = result.text;
summaryUsage = result.usage;
}
// Compute file lists and append to summary
@@ -819,6 +851,7 @@ export async function compact(
summary,
firstKeptEntryId,
tokensBefore,
usage: summaryUsage,
details: { readFiles, modifiedFiles } as CompactionDetails,
};
}
@@ -836,7 +869,7 @@ async function generateTurnPrefixSummary(
signal?: AbortSignal,
thinkingLevel?: ThinkingLevel,
streamFn?: StreamFn,
): Promise<string> {
): Promise<{ text: string; usage: Usage }> {
const maxTokens = Math.min(
Math.floor(0.5 * reserveTokens),
model.maxTokens > 0 ? model.maxTokens : Number.POSITIVE_INFINITY,
@@ -863,5 +896,8 @@ async function generateTurnPrefixSummary(
throw new Error(`Turn prefix summarization failed: ${response.errorMessage || "Unknown error"}`);
}
return contentText(response.content);
return {
text: contentText(response.content),
usage: response.usage,
};
}
@@ -883,6 +883,10 @@ export class ExtensionRunner {
currentEvent.isError = handlerResult.isError;
modified = true;
}
if (handlerResult.usage !== undefined) {
currentEvent.usage = handlerResult.usage;
modified = true;
}
} catch (err) {
const message = err instanceof Error ? err.message : String(err);
const stack = err instanceof Error ? err.stack : undefined;
@@ -904,6 +908,7 @@ export class ExtensionRunner {
content: currentEvent.content,
details: currentEvent.details,
isError: currentEvent.isError,
usage: currentEvent.usage,
};
}
@@ -30,6 +30,7 @@ import type {
SimpleStreamOptions,
TextContent,
ToolResultMessage,
Usage,
} from "@earendil-works/pi-ai";
import type {
AutocompleteItem,
@@ -905,6 +906,8 @@ interface ToolResultEventBase {
input: Record<string, unknown>;
content: (TextContent | ImageContent)[];
isError: boolean;
/** Usage from the tool execution itself, if available. */
usage?: Usage;
}
export interface BashToolResultEvent extends ToolResultEventBase {
@@ -1072,6 +1075,7 @@ export interface ToolResultEventResult {
content?: (TextContent | ImageContent)[];
details?: unknown;
isError?: boolean;
usage?: Usage;
}
export interface MessageEndEventResult {
@@ -1104,6 +1108,7 @@ export interface SessionBeforeTreeResult {
summary?: {
summary: string;
details?: unknown;
usage?: Usage;
};
/** Override custom instructions for summarization */
customInstructions?: string;
@@ -1,5 +1,5 @@
import type { AgentMessage } from "@earendil-works/pi-agent-core";
import { type ImageContent, type Message, type TextContent, uuidv7 } from "@earendil-works/pi-ai";
import { type ImageContent, type Message, type TextContent, type Usage, uuidv7 } from "@earendil-works/pi-ai";
import { randomUUID } from "crypto";
import {
appendFileSync,
@@ -73,6 +73,8 @@ export interface CompactionEntry<T = unknown> extends SessionEntryBase {
tokensBefore: number;
/** Extension-specific data (e.g., ArtifactIndex, version markers for structured compaction) */
details?: T;
/** Usage from the LLM call(s) that generated this summary, if available */
usage?: Usage;
/** True if generated by an extension, undefined/false if pi-generated (backward compatible) */
fromHook?: boolean;
}
@@ -83,6 +85,8 @@ export interface BranchSummaryEntry<T = unknown> extends SessionEntryBase {
summary: string;
/** Extension-specific data (not sent to LLM) */
details?: T;
/** Usage from the LLM call that generated this summary, if available */
usage?: Usage;
/** True if generated by an extension, false if pi-generated */
fromHook?: boolean;
}
@@ -1096,6 +1100,7 @@ export class SessionManager {
tokensBefore: number,
details?: T,
fromHook?: boolean,
usage?: Usage,
): string {
const entry: CompactionEntry<T> = {
type: "compaction",
@@ -1106,6 +1111,7 @@ export class SessionManager {
firstKeptEntryId,
tokensBefore,
details,
usage,
fromHook,
};
this._appendEntry(entry);
@@ -1372,7 +1378,13 @@ export class SessionManager {
* Same as branch(), but also appends a branch_summary entry that captures
* context from the abandoned conversation path.
*/
branchWithSummary(branchFromId: string | null, summary: string, details?: unknown, fromHook?: boolean): string {
branchWithSummary(
branchFromId: string | null,
summary: string,
details?: unknown,
fromHook?: boolean,
usage?: Usage,
): string {
if (branchFromId !== null && !this.byId.has(branchFromId)) {
throw new Error(`Entry ${branchFromId} not found`);
}
@@ -1385,6 +1397,7 @@ export class SessionManager {
fromId: branchFromId ?? "root",
summary,
details,
usage,
fromHook,
};
this._appendEntry(entry);
@@ -0,0 +1,27 @@
import type { Usage } from "@earendil-works/pi-ai/compat";
export interface UsageTotals {
input: number;
output: number;
cacheRead: number;
cacheWrite: number;
cost: number;
}
export function createUsageTotals(): UsageTotals {
return {
input: 0,
output: 0,
cacheRead: 0,
cacheWrite: 0,
cost: 0,
};
}
export function addUsageToTotals(totals: UsageTotals, usage: Usage): void {
totals.input += usage.input;
totals.output += usage.output;
totals.cacheRead += usage.cacheRead;
totals.cacheWrite += usage.cacheWrite;
totals.cost += usage.cost.total;
}
@@ -3,6 +3,7 @@ import { type Component, truncateToWidth, visibleWidth } from "@earendil-works/p
import type { AgentSession } from "../../../core/agent-session.ts";
import { areExperimentalFeaturesEnabled } from "../../../core/experimental.ts";
import type { ReadonlyFooterDataProvider } from "../../../core/footer-data-provider.ts";
import { addUsageToTotals, createUsageTotals } from "../../../core/usage-totals.ts";
import { theme } from "../theme/theme.ts";
/**
@@ -84,25 +85,21 @@ export class FooterComponent implements Component {
const state = this.session.state;
// Calculate cumulative usage from ALL session entries (not just post-compaction messages)
let totalInput = 0;
let totalOutput = 0;
let totalCacheRead = 0;
let totalCacheWrite = 0;
let totalCost = 0;
const usageTotals = createUsageTotals();
let latestCacheHitRate: number | undefined;
for (const entry of this.session.sessionManager.getEntries()) {
if (entry.type === "message" && entry.message.role === "assistant") {
totalInput += entry.message.usage.input;
totalOutput += entry.message.usage.output;
totalCacheRead += entry.message.usage.cacheRead;
totalCacheWrite += entry.message.usage.cacheWrite;
totalCost += entry.message.usage.cost.total;
addUsageToTotals(usageTotals, entry.message.usage);
const latestPromptTokens =
entry.message.usage.input + entry.message.usage.cacheRead + entry.message.usage.cacheWrite;
latestCacheHitRate =
latestPromptTokens > 0 ? (entry.message.usage.cacheRead / latestPromptTokens) * 100 : undefined;
} else if (entry.type === "message" && entry.message.role === "toolResult" && entry.message.usage) {
addUsageToTotals(usageTotals, entry.message.usage);
} else if ((entry.type === "branch_summary" || entry.type === "compaction") && entry.usage) {
addUsageToTotals(usageTotals, entry.usage);
}
}
@@ -130,19 +127,20 @@ export class FooterComponent implements Component {
// Build stats line
const statsParts = [];
if (totalInput) statsParts.push(`${formatTokens(totalInput)}`);
if (totalOutput) statsParts.push(`${formatTokens(totalOutput)}`);
if (totalCacheRead) statsParts.push(`R${formatTokens(totalCacheRead)}`);
if (totalCacheWrite) statsParts.push(`W${formatTokens(totalCacheWrite)}`);
if ((totalCacheRead > 0 || totalCacheWrite > 0) && latestCacheHitRate !== undefined) {
if (usageTotals.input) statsParts.push(`${formatTokens(usageTotals.input)}`);
if (usageTotals.output) statsParts.push(`${formatTokens(usageTotals.output)}`);
if (usageTotals.cacheRead) statsParts.push(`R${formatTokens(usageTotals.cacheRead)}`);
if (usageTotals.cacheWrite) statsParts.push(`W${formatTokens(usageTotals.cacheWrite)}`);
if ((usageTotals.cacheRead > 0 || usageTotals.cacheWrite > 0) && latestCacheHitRate !== undefined) {
statsParts.push(`CH${latestCacheHitRate.toFixed(1)}%`);
}
// Kimi Coding is subscription-backed despite using API-key authentication.
const usingSubscription = state.model
? state.model.provider === "kimi-coding" || this.session.modelRuntime.isUsingOAuth(state.model.provider)
: false;
if (totalCost || usingSubscription) {
const costStr = `$${totalCost.toFixed(3)}${usingSubscription ? " (sub)" : ""}`;
if (usageTotals.cost || usingSubscription) {
const costStr = `$${usageTotals.cost.toFixed(3)}${usingSubscription ? " (sub)" : ""}`;
statsParts.push(costStr);
}
@@ -1,5 +1,5 @@
import { Agent } from "@earendil-works/pi-agent-core";
import { type AssistantMessage, getModel, type Usage } from "@earendil-works/pi-ai/compat";
import { type AssistantMessage, getModel, type ToolResultMessage, type Usage } from "@earendil-works/pi-ai/compat";
import { describe, expect, it } from "vitest";
import { AgentSession } from "../src/core/agent-session.ts";
import { AuthStorage } from "../src/core/auth-storage.ts";
@@ -48,6 +48,18 @@ function createUserMessage(text: string, timestamp: number) {
};
}
function createToolResultMessage(usage: Usage): ToolResultMessage {
return {
role: "toolResult",
toolCallId: "tool-call-1",
toolName: "test_tool",
content: [{ type: "text", text: "tool result" }],
usage,
isError: false,
timestamp: 1,
};
}
async function createSession() {
const settingsManager = SettingsManager.inMemory();
const sessionManager = SessionManager.inMemory();
@@ -143,6 +155,75 @@ describe("AgentSession.getSessionStats", () => {
}
});
it("includes branch summary usage in session totals", async () => {
const { session, sessionManager } = await createSession();
try {
sessionManager.branchWithSummary(null, "summary", undefined, false, {
input: 10,
output: 20,
cacheRead: 30,
cacheWrite: 40,
totalTokens: 100,
cost: { input: 0.1, output: 0.2, cacheRead: 0.3, cacheWrite: 0.4, total: 1 },
});
syncAgentMessages(session, sessionManager);
const stats = session.getSessionStats();
expect(stats.tokens).toEqual({ input: 10, output: 20, cacheRead: 30, cacheWrite: 40, total: 100 });
expect(stats.cost).toBe(1);
} finally {
session.dispose();
}
});
it("includes compaction usage in session totals", async () => {
const { session, sessionManager } = await createSession();
try {
const firstKeptEntryId = sessionManager.appendMessage(createUserMessage("hello", 1));
sessionManager.appendCompaction("summary", firstKeptEntryId, 100, undefined, false, {
input: 10,
output: 20,
cacheRead: 30,
cacheWrite: 40,
totalTokens: 100,
cost: { input: 0.1, output: 0.2, cacheRead: 0.3, cacheWrite: 0.4, total: 1 },
});
syncAgentMessages(session, sessionManager);
const stats = session.getSessionStats();
expect(stats.tokens).toEqual({ input: 10, output: 20, cacheRead: 30, cacheWrite: 40, total: 100 });
expect(stats.cost).toBe(1);
} finally {
session.dispose();
}
});
it("includes tool result usage in session totals", async () => {
const { session, sessionManager } = await createSession();
try {
sessionManager.appendMessage(
createToolResultMessage({
input: 10,
output: 20,
cacheRead: 30,
cacheWrite: 40,
totalTokens: 100,
cost: { input: 0.1, output: 0.2, cacheRead: 0.3, cacheWrite: 0.4, total: 1 },
}),
);
syncAgentMessages(session, sessionManager);
const stats = session.getSessionStats();
expect(stats.tokens).toEqual({ input: 10, output: 20, cacheRead: 30, cacheWrite: 40, total: 100 });
expect(stats.cost).toBe(1);
} finally {
session.dispose();
}
});
it("ignores zero-usage messages when checking for post-compaction context usage", async () => {
const { session, sessionManager } = await createSession();
@@ -0,0 +1,55 @@
import type { Usage } from "@earendil-works/pi-ai/compat";
import { afterEach, describe, expect, it } from "vitest";
import { createHarness, type Harness } from "./suite/harness.ts";
import { assistantMsg, userMsg } from "./utilities.ts";
describe("Branch summary extensions", () => {
const harnesses: Harness[] = [];
afterEach(() => {
while (harnesses.length > 0) {
harnesses.pop()?.cleanup();
}
});
it("persists extension-provided summary usage in session totals", async () => {
const usage: Usage = {
input: 10,
output: 20,
cacheRead: 30,
cacheWrite: 40,
totalTokens: 100,
cost: { input: 0.1, output: 0.2, cacheRead: 0.3, cacheWrite: 0.4, total: 1 },
};
const harness = await createHarness({
extensionFactories: [
(pi) => {
pi.on("session_before_tree", () => ({
summary: {
summary: "Summary provided by extension",
usage,
},
}));
},
],
});
harnesses.push(harness);
const targetId = harness.sessionManager.appendMessage(userMsg("first branch"));
harness.sessionManager.appendMessage(assistantMsg("first reply"));
harness.sessionManager.appendMessage(userMsg("abandoned branch work"));
harness.sessionManager.appendMessage(assistantMsg("abandoned reply"));
const result = await harness.session.navigateTree(targetId, { summarize: true });
const summaryEntry = result.summaryEntry;
expect(summaryEntry?.type).toBe("branch_summary");
expect(summaryEntry?.fromHook).toBe(true);
expect(summaryEntry?.summary).toBe("Summary provided by extension");
expect(summaryEntry?.usage).toEqual(usage);
const stats = harness.session.getSessionStats();
expect(stats.tokens).toEqual({ input: 12, output: 22, cacheRead: 30, cacheWrite: 40, total: 104 });
expect(stats.cost).toBe(1);
});
});
@@ -57,7 +57,7 @@ describe("generateSummary reasoning options", () => {
});
it("uses the provided thinking level for reasoning-capable models", async () => {
await generateSummary(
const result = await generateSummary(
messages,
createModel(true),
2000,
@@ -69,6 +69,9 @@ describe("generateSummary reasoning options", () => {
"medium",
);
expect(result.text).toBe("## Goal\nTest summary");
expect(result.usage).toEqual(mockSummaryResponse.usage);
expect(completeSimpleMock).toHaveBeenCalledTimes(1);
expect(completeSimpleMock.mock.calls[0][2]).toMatchObject({
reasoning: "medium",
@@ -127,8 +130,15 @@ describe("generateSummary reasoning options", () => {
settings: { enabled: true, reserveTokens: 500000, keepRecentTokens: 20000 },
};
await compact(preparation, createModel(false, 128000), "test-key");
const result = await compact(preparation, createModel(false, 128000), "test-key");
expect(result.usage).toEqual({
...mockSummaryResponse.usage,
input: 20,
output: 20,
totalTokens: 40,
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 },
});
expect(completeSimpleMock.mock.calls.map((call) => call[2]?.maxTokens)).toEqual([128000, 128000]);
});
});
+76 -12
View File
@@ -21,20 +21,46 @@ function createSession(options: {
reasoning?: boolean;
thinkingLevel?: string;
usage?: AssistantUsage;
branchUsage?: AssistantUsage;
compactionUsage?: AssistantUsage;
toolUsage?: AssistantUsage;
}): AgentSession {
const usage = options.usage;
const entries =
usage === undefined
? []
: [
{
type: "message",
message: {
role: "assistant",
usage,
},
},
];
const entries: Array<Record<string, unknown>> = [];
if (usage !== undefined) {
entries.push({
type: "message",
message: {
role: "assistant",
usage,
},
});
}
if (options.branchUsage !== undefined) {
entries.push({
type: "branch_summary",
usage: options.branchUsage,
});
}
if (options.compactionUsage !== undefined) {
entries.push({
type: "compaction",
usage: options.compactionUsage,
});
}
if (options.toolUsage !== undefined) {
entries.push({
type: "message",
message: {
role: "toolResult",
usage: options.toolUsage,
},
});
}
const session = {
state: {
@@ -125,6 +151,44 @@ describe("FooterComponent width handling", () => {
}
});
it("includes summary and tool result usage in the total cost", () => {
const session = createSession({
sessionName: "",
usage: {
input: 100,
output: 10,
cacheRead: 0,
cacheWrite: 0,
cost: { total: 0.5 },
},
branchUsage: {
input: 20,
output: 5,
cacheRead: 0,
cacheWrite: 0,
cost: { total: 0.25 },
},
compactionUsage: {
input: 5,
output: 2,
cacheRead: 0,
cacheWrite: 0,
cost: { total: 0.125 },
},
toolUsage: {
input: 15,
output: 3,
cacheRead: 0,
cacheWrite: 0,
cost: { total: 0.375 },
},
});
const footer = new FooterComponent(session, createFooterData(1));
const statsLine = stripAnsi(footer.render(120)[1]);
expect(statsLine).toContain("$1.250");
});
it("shows the latest cache hit rate when cache usage is present", () => {
const session = createSession({
sessionName: "",
@@ -71,7 +71,15 @@ describe("SessionManager append and tree traversal", () => {
const id1 = session.appendMessage(userMsg("1"));
const id2 = session.appendMessage(assistantMsg("2"));
const compactionId = session.appendCompaction("summary", id1, 1000);
const usage = {
input: 10,
output: 20,
cacheRead: 30,
cacheWrite: 40,
totalTokens: 100,
cost: { input: 0.1, output: 0.2, cacheRead: 0.3, cacheWrite: 0.4, total: 1 },
};
const compactionId = session.appendCompaction("summary", id1, 1000, undefined, false, usage);
const _id3 = session.appendMessage(userMsg("3"));
const entries = session.getEntries();
@@ -83,6 +91,7 @@ describe("SessionManager append and tree traversal", () => {
expect(compactionEntry.summary).toBe("summary");
expect(compactionEntry.firstKeptEntryId).toBe(id1);
expect(compactionEntry.tokensBefore).toBe(1000);
expect(compactionEntry.usage).toEqual(usage);
}
expect(entries[3].parentId).toBe(compactionId);
@@ -319,7 +328,15 @@ describe("SessionManager append and tree traversal", () => {
const _id2 = session.appendMessage(assistantMsg("2"));
const _id3 = session.appendMessage(userMsg("3"));
const summaryId = session.branchWithSummary(id1, "Summary of abandoned work");
const usage = {
input: 10,
output: 20,
cacheRead: 30,
cacheWrite: 40,
totalTokens: 100,
cost: { input: 0.1, output: 0.2, cacheRead: 0.3, cacheWrite: 0.4, total: 1 },
};
const summaryId = session.branchWithSummary(id1, "Summary of abandoned work", undefined, false, usage);
expect(session.getLeafId()).toBe(summaryId);
@@ -329,6 +346,7 @@ describe("SessionManager append and tree traversal", () => {
expect(summaryEntry?.parentId).toBe(id1);
if (summaryEntry?.type === "branch_summary") {
expect(summaryEntry.summary).toBe("Summary of abandoned work");
expect(summaryEntry.usage).toEqual(usage);
}
});
@@ -97,6 +97,14 @@ describe("AgentSession compaction characterization", () => {
});
it("manually compacts using an extension-provided summary", async () => {
const summaryUsage = {
input: 10,
output: 20,
cacheRead: 30,
cacheWrite: 40,
totalTokens: 100,
cost: { input: 0.1, output: 0.2, cacheRead: 0.3, cacheWrite: 0.4, total: 1 },
};
const harness = await createHarness({
settings: { compaction: { keepRecentTokens: 1 } },
extensionFactories: [
@@ -106,6 +114,7 @@ describe("AgentSession compaction characterization", () => {
summary: "summary from extension",
firstKeptEntryId: event.preparation.firstKeptEntryId,
tokensBefore: event.preparation.tokensBefore,
usage: summaryUsage,
details: { source: "extension" },
},
}));
@@ -116,14 +125,26 @@ describe("AgentSession compaction characterization", () => {
await harness.session.prompt("one");
await harness.session.prompt("two");
const statsBefore = harness.session.getSessionStats();
const result = await harness.session.compact();
const compactionEntries = harness.sessionManager.getEntries().filter((entry) => entry.type === "compaction");
const estimatedTokensAfter = harness.session.messages.reduce((sum, message) => sum + estimateTokens(message), 0);
expect(result.summary).toBe("summary from extension");
expect(result.usage).toEqual(summaryUsage);
expect(result.estimatedTokensAfter).toBe(estimatedTokensAfter);
expect(compactionEntries).toHaveLength(1);
const compactionEntry = compactionEntries[0];
if (compactionEntry?.type === "compaction") {
expect(compactionEntry.usage).toEqual(summaryUsage);
}
const statsAfter = harness.session.getSessionStats();
expect(statsAfter.tokens.input).toBe(statsBefore.tokens.input + summaryUsage.input);
expect(statsAfter.tokens.output).toBe(statsBefore.tokens.output + summaryUsage.output);
expect(statsAfter.tokens.cacheRead).toBe(statsBefore.tokens.cacheRead + summaryUsage.cacheRead);
expect(statsAfter.tokens.cacheWrite).toBe(statsBefore.tokens.cacheWrite + summaryUsage.cacheWrite);
expect(statsAfter.cost).toBe(statsBefore.cost + summaryUsage.cost.total);
expect(harness.session.messages[0]?.role).toBe("compactionSummary");
});
@@ -154,6 +175,22 @@ describe("AgentSession compaction characterization", () => {
expect(getStreamCallCount()).toBe(1);
});
it("persists usage from pi-generated manual compaction", async () => {
const harness = await createHarness({ withConfiguredAuth: false });
harnesses.push(harness);
seedCompactableSession(harness);
useSummaryStreamFn(harness, "summary from custom stream");
const result = await harness.session.compact();
const compactionEntries = harness.sessionManager.getEntries().filter((entry) => entry.type === "compaction");
expect(result.usage).toEqual(createUsage(10));
expect(compactionEntries).toHaveLength(1);
expect(compactionEntries[0]?.type === "compaction" ? compactionEntries[0].usage : undefined).toEqual(
createUsage(10),
);
});
it("auto-compacts with a custom streamFn when registry auth is absent", async () => {
const harness = await createHarness({ withConfiguredAuth: false });
harnesses.push(harness);
@@ -1,5 +1,5 @@
import type { AgentTool, ThinkingLevel } from "@earendil-works/pi-agent-core";
import { fauxAssistantMessage, fauxToolCall, type Model } from "@earendil-works/pi-ai";
import { fauxAssistantMessage, fauxToolCall, type Model, type Usage } from "@earendil-works/pi-ai";
import { Type } from "typebox";
import { afterEach, describe, expect, it } from "vitest";
import type { BuildSystemPromptOptions, ExtensionAPI } from "../../src/index.ts";
@@ -156,6 +156,23 @@ describe("AgentSession model and extension characterization", () => {
});
it("allows extension tool_result handlers to modify tool results", async () => {
const toolUsage: Usage = {
input: 1,
output: 2,
cacheRead: 3,
cacheWrite: 4,
totalTokens: 10,
cost: { input: 0.1, output: 0.2, cacheRead: 0.3, cacheWrite: 0.4, total: 1 },
};
const patchedToolUsage: Usage = {
input: 5,
output: 6,
cacheRead: 7,
cacheWrite: 8,
totalTokens: 26,
cost: { input: 0.5, output: 0.6, cacheRead: 0.7, cacheWrite: 0.8, total: 2.6 },
};
let observedToolUsage: Usage | undefined;
const echoTool: AgentTool = {
name: "echo",
label: "Echo",
@@ -163,17 +180,21 @@ describe("AgentSession model and extension characterization", () => {
parameters: Type.Object({ text: Type.String() }),
execute: async (_toolCallId, params) => {
const text = typeof params === "object" && params !== null && "text" in params ? String(params.text) : "";
return { content: [{ type: "text", text }], details: { text } };
return { content: [{ type: "text", text }], details: { text }, usage: toolUsage };
},
};
const harness = await createHarness({
tools: [echoTool],
extensionFactories: [
(pi) => {
pi.on("tool_result", async () => ({
content: [{ type: "text", text: "patched result" }],
details: { patched: true },
}));
pi.on("tool_result", async (event) => {
observedToolUsage = event.usage;
return {
content: [{ type: "text", text: "patched result" }],
details: { patched: true },
usage: patchedToolUsage,
};
});
},
],
});
@@ -196,9 +217,12 @@ describe("AgentSession model and extension characterization", () => {
await harness.session.prompt("hi");
expect(getAssistantTexts(harness)).toContain("patched result");
expect(
harness.session.messages.find((message) => message.role === "toolResult" && message.details?.patched === true),
).toBeDefined();
const toolResult = harness.session.messages.find(
(message) => message.role === "toolResult" && message.details?.patched === true,
);
expect(observedToolUsage).toEqual(toolUsage);
expect(toolResult).toBeDefined();
expect(toolResult?.role === "toolResult" ? toolResult.usage : undefined).toEqual(patchedToolUsage);
});
it("allows extension context handlers to modify messages before the LLM call", async () => {
@@ -37,7 +37,7 @@ describe("issue #6324 branch summary ambient auth", () => {
cacheRead: 0,
cacheWrite: 0,
totalTokens: 2,
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 },
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0.25 },
},
stopReason: "stop",
timestamp: Date.now(),
@@ -57,5 +57,6 @@ describe("issue #6324 branch summary ambient auth", () => {
expect(streamCallCount).toBe(1);
expect(result.summaryEntry?.type).toBe("branch_summary");
expect(result.summaryEntry?.summary).toContain("branch summary text");
expect(result.summaryEntry?.usage?.cost.total).toBe(0.25);
});
});