add usage info to branch summary, compaction and tool result entries (#6671)

* add usage info to branch summary entries

* add usage to compaction entries

* allow custom tools to report llm usage in tool results

* allow observing and patching usage in tool_result hooks

* agent-harness: save usage in entries

for compaction, branch summaries and tool results
This commit is contained in:
David Brailovsky
2026-07-20 16:41:43 +02:00
committed by GitHub
parent c179395218
commit 2fd3868401
29 changed files with 844 additions and 122 deletions
+2
View File
@@ -737,6 +737,7 @@ async function finalizeExecutedToolCall(
...result,
content: afterResult.content ?? result.content,
details: afterResult.details ?? result.details,
usage: afterResult.usage ?? result.usage,
terminate: afterResult.terminate ?? result.terminate,
};
isError = afterResult.isError ?? isError;
@@ -780,6 +781,7 @@ function createToolResultMessage(finalized: FinalizedToolCallOutcome): ToolResul
// so the null never enters session history or provider payloads.
content: finalized.result.content ?? [],
details: finalized.result.details,
usage: finalized.result.usage,
...(finalized.result.addedToolNames?.length ? { addedToolNames: finalized.result.addedToolNames } : {}),
isError: finalized.isError,
timestamp: Date.now(),
+19 -5
View File
@@ -32,6 +32,7 @@ import type {
AgentHarnessResources,
AgentHarnessStreamOptions,
AgentHarnessStreamOptionsPatch,
CompactResult,
ExecutionEnv,
NavigateTreeResult,
PendingSessionWrite,
@@ -434,9 +435,16 @@ export class AgentHarness<
content: result.content,
details: result.details,
isError,
usage: result.usage,
});
return patch
? { content: patch.content, details: patch.details, isError: patch.isError, terminate: patch.terminate }
? {
content: patch.content,
details: patch.details,
isError: patch.isError,
usage: patch.usage,
terminate: patch.terminate,
}
: undefined;
},
prepareNextTurn: async () => {
@@ -690,9 +698,7 @@ export class AgentHarness<
}
}
async compact(
customInstructions?: string,
): Promise<{ summary: string; firstKeptEntryId: string; tokensBefore: number; details?: unknown }> {
async compact(customInstructions?: string): Promise<CompactResult> {
if (this.phase !== "idle") throw new AgentHarnessError("busy", "compact() requires idle harness");
this.phase = "compaction";
try {
@@ -723,6 +729,7 @@ export class AgentHarness<
result.tokensBefore,
result.details,
provided !== undefined,
result.usage,
);
const entry = await this.session.getEntry(entryId);
if (entry?.type === "compaction") {
@@ -764,6 +771,7 @@ export class AgentHarness<
let summaryEntry: NavigateTreeResult["summaryEntry"];
let summaryText: string | undefined = hookResult?.summary?.summary;
let summaryDetails: unknown = hookResult?.summary?.details;
let summaryUsage = hookResult?.summary?.usage;
if (!summaryText && options?.summarize && entries.length > 0) {
const model = this.model;
if (!model) throw new AgentHarnessError("invalid_state", "No model set for branch summary");
@@ -779,6 +787,7 @@ export class AgentHarness<
throw new AgentHarnessError("branch_summary", branchSummary.error.message, branchSummary.error);
}
summaryText = branchSummary.value.summary;
summaryUsage = branchSummary.value.usage;
summaryDetails = {
readFiles: branchSummary.value.readFiles,
modifiedFiles: branchSummary.value.modifiedFiles,
@@ -798,7 +807,12 @@ export class AgentHarness<
const summaryId = await this.session.moveTo(
newLeafId,
summaryText
? { summary: summaryText, details: summaryDetails, fromHook: hookResult?.summary !== undefined }
? {
summary: summaryText,
details: summaryDetails,
usage: summaryUsage,
fromHook: hookResult?.summary !== undefined,
}
: undefined,
);
if (summaryId) {
@@ -252,6 +252,7 @@ export async function generateBranchSummary(
return ok({
summary: summary || "No summary generated",
usage: response.usage,
readFiles,
modifiedFiles,
});
@@ -101,10 +101,35 @@ export interface CompactionResult<T = unknown> {
firstKeptEntryId: string;
/** Estimated context tokens before compaction. */
tokensBefore: number;
/** Usage from the LLM call(s) that generated this summary, if available. */
usage?: Usage;
/** Optional implementation-specific details stored with the compaction entry. */
details?: T;
}
function combineUsage(first: Usage, second: Usage): Usage {
return {
input: first.input + second.input,
output: first.output + second.output,
cacheRead: first.cacheRead + second.cacheRead,
cacheWrite: first.cacheWrite + second.cacheWrite,
...(first.cacheWrite1h !== undefined || second.cacheWrite1h !== undefined
? { cacheWrite1h: (first.cacheWrite1h ?? 0) + (second.cacheWrite1h ?? 0) }
: {}),
...(first.reasoning !== undefined || second.reasoning !== undefined
? { reasoning: (first.reasoning ?? 0) + (second.reasoning ?? 0) }
: {}),
totalTokens: first.totalTokens + second.totalTokens,
cost: {
input: first.cost.input + second.cost.input,
output: first.cost.output + second.cost.output,
cacheRead: first.cost.cacheRead + second.cost.cacheRead,
cacheWrite: first.cost.cacheWrite + second.cost.cacheWrite,
total: first.cost.total + second.cost.total,
},
};
}
/** Compaction thresholds and retention settings. */
export interface CompactionSettings {
/** Enable automatic compaction decisions. */
@@ -474,7 +499,7 @@ export async function generateSummary(
customInstructions?: string,
previousSummary?: string,
thinkingLevel?: ThinkingLevel,
): Promise<Result<string, CompactionError>> {
): Promise<Result<{ text: string; usage: Usage }, CompactionError>> {
const maxTokens = Math.min(
Math.floor(0.8 * reserveTokens),
model.maxTokens > 0 ? model.maxTokens : Number.POSITIVE_INFINITY,
@@ -523,7 +548,7 @@ export async function generateSummary(
const textContent = contentText(response.content);
return ok(textContent);
return ok({ text: textContent, usage: response.usage });
}
/** Prepared inputs for a compaction run. */
@@ -656,11 +681,13 @@ export async function compact(
}
let summary: string;
let summaryUsage: Usage;
if (isSplitTurn && turnPrefixMessages.length > 0) {
const historyResult =
messagesToSummarize.length > 0
? await generateSummary(
let historyText = "No prior history.";
let historyUsage: Usage | undefined;
if (messagesToSummarize.length > 0) {
const historyResult = await generateSummary(
messagesToSummarize,
models,
model,
@@ -669,9 +696,11 @@ export async function compact(
customInstructions,
previousSummary,
thinkingLevel,
)
: ok<string, CompactionError>("No prior history.");
);
if (!historyResult.ok) return err(historyResult.error);
historyText = historyResult.value.text;
historyUsage = historyResult.value.usage;
}
const turnPrefixResult = await generateTurnPrefixSummary(
turnPrefixMessages,
models,
@@ -681,7 +710,10 @@ export async function compact(
thinkingLevel,
);
if (!turnPrefixResult.ok) return err(turnPrefixResult.error);
summary = `${historyResult.value}\n\n---\n\n**Turn Context (split turn):**\n\n${turnPrefixResult.value}`;
summary = `${historyText}\n\n---\n\n**Turn Context (split turn):**\n\n${turnPrefixResult.value.text}`;
summaryUsage = historyUsage
? combineUsage(historyUsage, turnPrefixResult.value.usage)
: turnPrefixResult.value.usage;
} else {
const summaryResult = await generateSummary(
messagesToSummarize,
@@ -694,7 +726,8 @@ export async function compact(
thinkingLevel,
);
if (!summaryResult.ok) return err(summaryResult.error);
summary = summaryResult.value;
summary = summaryResult.value.text;
summaryUsage = summaryResult.value.usage;
}
const { readFiles, modifiedFiles } = computeFileLists(fileOps);
@@ -704,6 +737,7 @@ export async function compact(
summary,
firstKeptEntryId,
tokensBefore,
usage: summaryUsage,
details: { readFiles, modifiedFiles } as CompactionDetails,
});
}
@@ -714,7 +748,7 @@ async function generateTurnPrefixSummary(
reserveTokens: number,
signal?: AbortSignal,
thinkingLevel?: ThinkingLevel,
): Promise<Result<string, CompactionError>> {
): Promise<Result<{ text: string; usage: Usage }, CompactionError>> {
const maxTokens = Math.min(
Math.floor(0.5 * reserveTokens),
model.maxTokens > 0 ? model.maxTokens : Number.POSITIVE_INFINITY,
@@ -749,5 +783,8 @@ async function generateTurnPrefixSummary(
);
}
return ok(contentText(response.content));
return ok({
text: contentText(response.content),
usage: response.usage,
});
}
@@ -1,4 +1,4 @@
import type { ImageContent, TextContent } from "@earendil-works/pi-ai";
import type { ImageContent, TextContent, Usage } from "@earendil-works/pi-ai";
import type { AgentMessage } from "../../types.ts";
import { createBranchSummaryMessage, createCompactionSummaryMessage, createCustomMessage } from "../messages.ts";
import type {
@@ -247,6 +247,7 @@ export class Session<TMetadata extends SessionMetadata = SessionMetadata> {
tokensBefore: number,
details?: T,
fromHook?: boolean,
usage?: Usage,
): Promise<string> {
return this.appendTypedEntry({
type: "compaction",
@@ -257,6 +258,7 @@ export class Session<TMetadata extends SessionMetadata = SessionMetadata> {
firstKeptEntryId,
tokensBefore,
details,
usage,
fromHook,
} satisfies CompactionEntry<T>);
}
@@ -317,7 +319,7 @@ export class Session<TMetadata extends SessionMetadata = SessionMetadata> {
async moveTo(
entryId: string | null,
summary?: { summary: string; details?: unknown; fromHook?: boolean },
summary?: { summary: string; details?: unknown; usage?: Usage; fromHook?: boolean },
): Promise<string | undefined> {
if (entryId !== null && !(await this.storage.getEntry(entryId))) {
throw new SessionError("not_found", `Entry ${entryId} not found`);
@@ -332,6 +334,7 @@ export class Session<TMetadata extends SessionMetadata = SessionMetadata> {
fromId: entryId ?? "root",
summary: summary.summary,
details: summary.details,
usage: summary.usage,
fromHook: summary.fromHook,
} satisfies BranchSummaryEntry);
}
+22 -2
View File
@@ -1,4 +1,12 @@
import type { ImageContent, Model, Models, SimpleStreamOptions, TextContent, Transport } from "@earendil-works/pi-ai";
import type {
ImageContent,
Model,
Models,
SimpleStreamOptions,
TextContent,
Transport,
Usage,
} from "@earendil-works/pi-ai";
import type { AgentEvent, AgentMessage, AgentTool, QueueMode, ThinkingLevel } from "../index.ts";
import type { Session } from "./session/session.ts";
@@ -365,6 +373,7 @@ export interface CompactionEntry<T = unknown> extends SessionTreeEntryBase {
firstKeptEntryId: string;
tokensBefore: number;
details?: T;
usage?: Usage;
fromHook?: boolean;
}
@@ -373,6 +382,7 @@ export interface BranchSummaryEntry<T = unknown> extends SessionTreeEntryBase {
fromId: string;
summary: string;
details?: T;
usage?: Usage;
fromHook?: boolean;
}
@@ -572,6 +582,7 @@ export interface ToolResultEvent {
content: Array<TextContent | ImageContent>;
details: unknown;
isError: boolean;
usage?: Usage;
}
export interface SessionBeforeCompactEvent {
@@ -687,6 +698,7 @@ export interface ToolResultPatch {
content?: Array<TextContent | ImageContent>;
details?: unknown;
isError?: boolean;
usage?: Usage;
terminate?: boolean;
}
@@ -697,7 +709,12 @@ export interface SessionBeforeCompactResult {
export interface SessionBeforeTreeResult {
cancel?: boolean;
summary?: { summary: string; details?: unknown };
summary?: {
summary: string;
details?: unknown;
/** Usage from the LLM call that generated this summary, if available. */
usage?: Usage;
};
customInstructions?: string;
replaceInstructions?: boolean;
label?: string;
@@ -738,6 +755,8 @@ export interface CompactResult {
summary: string;
firstKeptEntryId: string;
tokensBefore: number;
/** Usage from the LLM call(s) that generated this summary, if available. */
usage?: Usage;
details?: unknown;
}
@@ -793,6 +812,7 @@ export interface GenerateBranchSummaryOptions {
export interface BranchSummaryResult {
summary: string;
usage?: Usage;
readFiles: string[];
modifiedFiles: string[];
}
+8 -1
View File
@@ -11,6 +11,7 @@ import type {
TextContent,
Tool,
ToolResultMessage,
Usage,
} from "@earendil-works/pi-ai";
import type { Static, TSchema } from "typebox";
@@ -69,15 +70,18 @@ export interface BeforeToolCallResult {
* - `content`: if provided, replaces the tool result content array in full
* - `details`: if provided, replaces the tool result details value in full
* - `isError`: if provided, replaces the tool result error flag
* - `usage`: if provided, replaces the tool result usage
* - `terminate`: if provided, replaces the early-termination hint
*
* Omitted fields keep the original executed tool result values.
* There is no deep merge for `content` or `details`.
* There is no deep merge for `content`, `details`, or `usage`.
*/
export interface AfterToolCallResult {
content?: (TextContent | ImageContent)[];
details?: unknown;
isError?: boolean;
/** Usage from the final tool execution itself, if available. Not used for main LLM context accounting. */
usage?: Usage;
/**
* Hint that the agent should stop after the current tool batch.
* Early termination only happens when every finalized tool result in the batch sets this to true.
@@ -273,6 +277,7 @@ export interface AgentLoopConfig extends SimpleStreamOptions {
* - `content` replaces the full content array
* - `details` replaces the full details payload
* - `isError` replaces the error flag
* - `usage` replaces the tool result usage
* - `terminate` replaces the early-termination hint
*
* Any omitted fields keep their original values. No deep merge is performed.
@@ -352,6 +357,8 @@ export interface AgentToolResult<T> {
content: (TextContent | ImageContent)[];
/** Arbitrary structured details for logs or UI rendering. */
details: T;
/** Usage from the final tool execution itself, if available. Not used for main LLM context accounting. */
usage?: Usage;
/** Names of tools introduced by this result and available from this transcript point onward. */
addedToolNames?: string[];
/**
+26
View File
@@ -239,6 +239,23 @@ describe("agentLoop with AgentMessage", () => {
it("should handle tool calls and results", async () => {
const toolSchema = Type.Object({ value: Type.String() });
const executed: string[] = [];
const toolUsage = {
input: 1,
output: 2,
cacheRead: 3,
cacheWrite: 4,
totalTokens: 10,
cost: { input: 0.1, output: 0.2, cacheRead: 0.3, cacheWrite: 0.4, total: 1 },
};
const patchedToolUsage = {
input: 5,
output: 6,
cacheRead: 7,
cacheWrite: 8,
totalTokens: 26,
cost: { input: 0.5, output: 0.6, cacheRead: 0.7, cacheWrite: 0.8, total: 2.6 },
};
let observedToolUsage: typeof toolUsage | undefined;
const tool: AgentTool<typeof toolSchema, { value: string }> = {
name: "echo",
label: "Echo",
@@ -249,6 +266,7 @@ describe("agentLoop with AgentMessage", () => {
return {
content: [{ type: "text", text: `echoed: ${params.value}` }],
details: { value: params.value },
usage: toolUsage,
};
},
};
@@ -264,6 +282,10 @@ describe("agentLoop with AgentMessage", () => {
const config: AgentLoopConfig = {
model: createModel(),
convertToLlm: identityConverter,
afterToolCall: async ({ result }) => {
observedToolUsage = result.usage;
return { usage: patchedToolUsage };
},
};
let callIndex = 0;
@@ -305,6 +327,10 @@ describe("agentLoop with AgentMessage", () => {
if (toolEnd?.type === "tool_execution_end") {
expect(toolEnd.isError).toBe(false);
}
expect(observedToolUsage).toEqual(toolUsage);
const messages = await stream.result();
const toolResult = messages.find((message) => message.role === "toolResult");
expect(toolResult?.role === "toolResult" ? toolResult.usage : undefined).toEqual(patchedToolUsage);
});
it("should not execute tool calls from a length-truncated assistant message", async () => {
@@ -5,6 +5,7 @@ import {
fauxProvider,
fauxToolCall,
type RegisterFauxProviderOptions,
type Usage,
} from "@earendil-works/pi-ai";
import { getModel } from "@earendil-works/pi-ai/compat";
import { describe, expect, it } from "vitest";
@@ -14,7 +15,7 @@ import { InMemorySessionStorage } from "../../src/harness/session/memory-storage
import { Session } from "../../src/harness/session/session.ts";
import type { PromptTemplate, Skill } from "../../src/harness/types.ts";
import type { AgentMessage, AgentTool } from "../../src/types.ts";
import { calculateTool } from "../utils/calculate.ts";
import { calculateTool, createCalculateToolWithUsage } from "../utils/calculate.ts";
import { getCurrentTimeTool } from "../utils/get-current-time.ts";
interface AppSkill extends Skill {
@@ -60,6 +61,34 @@ function getReasoning(options: unknown): unknown {
return options.reasoning;
}
function createUsage(input: number, output: number, cacheRead = 0, cacheWrite = 0): Usage {
return {
input,
output,
cacheRead,
cacheWrite,
totalTokens: input + output + cacheRead + cacheWrite,
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 },
};
}
function createUserMessage(text: string): AgentMessage {
return { role: "user", content: [{ type: "text", text }], timestamp: Date.now() };
}
function createAssistantMessage(text: string): AgentMessage {
return {
role: "assistant",
content: [{ type: "text", text }],
api: "faux",
provider: "faux",
model: "faux-1",
usage: createUsage(100, 50),
stopReason: "stop",
timestamp: Date.now(),
};
}
describe("AgentHarness", () => {
it("constructs directly and exposes queue modes", () => {
const session = new Session(new InMemorySessionStorage());
@@ -427,14 +456,18 @@ describe("AgentHarness", () => {
}),
]);
const session = new Session(new InMemorySessionStorage());
const toolUsage = createUsage(1, 2, 3, 4);
const patchedToolUsage = createUsage(5, 6, 7, 8);
const calculateToolWithUsage = createCalculateToolWithUsage(toolUsage);
const harness = new AgentHarness({
models,
env: new NodeExecutionEnv({ cwd: process.cwd() }),
session,
model: registration.getModel(),
tools: [calculateTool],
tools: [calculateToolWithUsage],
});
const seenToolCalls: Array<{ id: string; name: string; expression: unknown }> = [];
let seenToolUsage: Usage | undefined;
harness.on("tool_call", (event) => {
seenToolCalls.push({ id: event.toolCallId, name: event.toolName, expression: event.input.expression });
return undefined;
@@ -442,9 +475,11 @@ describe("AgentHarness", () => {
harness.on("tool_result", (event) => {
expect(event.toolCallId).toBe("call-1");
expect(event.toolName).toBe("calculate");
seenToolUsage = event.usage;
return {
content: [{ type: "text", text: "patched result" }],
details: { patched: true },
usage: patchedToolUsage,
terminate: true,
};
});
@@ -455,16 +490,109 @@ describe("AgentHarness", () => {
(entry) => entry.type === "message" && entry.message.role === "toolResult",
);
expect(seenToolCalls).toEqual([{ id: "call-1", name: "calculate", expression: "2 + 2" }]);
expect(seenToolUsage).toEqual(toolUsage);
expect(toolResult).toMatchObject({
type: "message",
message: {
role: "toolResult",
content: [{ type: "text", text: "patched result" }],
details: { patched: true },
usage: patchedToolUsage,
},
});
});
it("persists generated compaction usage", async () => {
const registration = newFaux();
registration.setResponses([fauxAssistantMessage("## Goal\nTest summary")]);
const session = new Session(new InMemorySessionStorage());
await session.appendMessage(createUserMessage("one"));
await session.appendMessage(createAssistantMessage("two"));
const harness = new AgentHarness({
models,
env: new NodeExecutionEnv({ cwd: process.cwd() }),
session,
model: registration.getModel(),
});
const result = await harness.compact();
const compaction = (await session.getEntries()).find((entry) => entry.type === "compaction");
expect(result.usage?.totalTokens).toBeGreaterThan(0);
expect(compaction?.type === "compaction" ? compaction.usage : undefined).toEqual(result.usage);
});
it("persists hook-provided compaction usage", async () => {
const registration = newFaux();
const usage = createUsage(5, 6, 7, 8);
const session = new Session(new InMemorySessionStorage());
await session.appendMessage(createUserMessage("one"));
await session.appendMessage(createAssistantMessage("two"));
const harness = new AgentHarness({
models,
env: new NodeExecutionEnv({ cwd: process.cwd() }),
session,
model: registration.getModel(),
});
harness.on("session_before_compact", (event) => ({
compaction: {
summary: "hook summary",
firstKeptEntryId: event.preparation.firstKeptEntryId,
tokensBefore: event.preparation.tokensBefore,
usage,
},
}));
const result = await harness.compact();
const compaction = (await session.getEntries()).find((entry) => entry.type === "compaction");
expect(result.usage).toEqual(usage);
expect(compaction?.type === "compaction" ? compaction.usage : undefined).toEqual(usage);
});
it("persists generated branch summary usage", async () => {
const registration = newFaux();
registration.setResponses([fauxAssistantMessage("## Goal\nBranch summary")]);
const session = new Session(new InMemorySessionStorage());
const targetId = await session.appendMessage(createUserMessage("first branch"));
await session.appendMessage(createAssistantMessage("first reply"));
await session.appendMessage(createUserMessage("abandoned work"));
await session.appendMessage(createAssistantMessage("abandoned reply"));
const harness = new AgentHarness({
models,
env: new NodeExecutionEnv({ cwd: process.cwd() }),
session,
model: registration.getModel(),
});
const result = await harness.navigateTree(targetId, { summarize: true });
expect(result.summaryEntry?.usage?.totalTokens).toBeGreaterThan(0);
});
it("persists hook-provided branch summary usage", async () => {
const registration = newFaux();
const usage = createUsage(13, 14, 15, 16);
const session = new Session(new InMemorySessionStorage());
const targetId = await session.appendMessage(createUserMessage("first branch"));
await session.appendMessage(createAssistantMessage("first reply"));
await session.appendMessage(createUserMessage("abandoned work"));
await session.appendMessage(createAssistantMessage("abandoned reply"));
const harness = new AgentHarness({
models,
env: new NodeExecutionEnv({ cwd: process.cwd() }),
session,
model: registration.getModel(),
});
harness.on("session_before_tree", () => ({
summary: { summary: "hook branch summary", usage },
}));
const result = await harness.navigateTree(targetId, { summarize: true });
expect(result.summaryEntry?.usage).toEqual(usage);
});
it("preserves app tool types for getters and update events", async () => {
const session = new Session(new InMemorySessionStorage());
const env = new NodeExecutionEnv({ cwd: process.cwd() });
+43 -1
View File
@@ -6,6 +6,7 @@ import {
fauxProvider,
type Message,
type Model,
type Models,
type Usage,
} from "@earendil-works/pi-ai";
import { beforeEach, describe, expect, it } from "vitest";
@@ -142,6 +143,17 @@ function createFauxModel(reasoning: boolean, maxTokens = 8192): { faux: FauxProv
return { faux, model: faux.getModel() };
}
function createModelsWithSimpleResponses(responses: AssistantMessage[]): Models {
const remaining = [...responses];
const stub = Object.create(models) as Models;
stub.completeSimple = async () => {
const response = remaining.shift();
if (!response) throw new Error("No faux completeSimple response queued");
return response;
};
return stub;
}
describe("harness compaction", () => {
beforeEach(() => {
nextId = 0;
@@ -501,7 +513,12 @@ describe("harness compaction", () => {
await generateSummary(messages, models, model, 2000, undefined, "focus", "old summary"),
);
expect(summary).toContain("Test summary");
expect(summary.text).toContain("Test summary");
expect(summary.usage.input).toBeGreaterThan(0);
expect(summary.usage.output).toBeGreaterThan(0);
expect(summary.usage.totalTokens).toBe(
summary.usage.input + summary.usage.output + summary.usage.cacheRead + summary.usage.cacheWrite,
);
expect(promptText).toContain("<previous-summary>\nold summary\n</previous-summary>");
expect(promptText).toContain("Additional focus: focus");
});
@@ -578,6 +595,30 @@ describe("harness compaction", () => {
expect(invalidResult).toMatchObject({ ok: false, error: { code: "invalid_session" } });
});
it("combines usage for split-turn compaction summaries", async () => {
const messages: AgentMessage[] = [createUserMessage("Summarize this.")];
const { model } = createFauxModel(false);
const historyUsage = createMockUsage(1, 2, 3, 4);
const turnPrefixUsage = createMockUsage(5, 6, 7, 8);
const usageModels = createModelsWithSimpleResponses([
{ ...fauxAssistantMessage("history summary"), usage: historyUsage },
{ ...fauxAssistantMessage("turn prefix summary"), usage: turnPrefixUsage },
]);
const preparation: CompactionPreparation = {
firstKeptEntryId: "entry-keep",
messagesToSummarize: messages,
turnPrefixMessages: messages,
isSplitTurn: true,
tokensBefore: 100,
fileOps: { read: new Set(), written: new Set(), edited: new Set() },
settings: { enabled: true, reserveTokens: 2000, keepRecentTokens: 20 },
};
const result = getOrThrow(await compact(preparation, usageModels, model));
expect(result.usage).toEqual(createMockUsage(6, 8, 10, 12));
});
it("passes reasoning through turn-prefix summaries when enabled", async () => {
const messages: AgentMessage[] = [createUserMessage("Summarize this.")];
const seenOptions: Array<Record<string, unknown> | undefined> = [];
@@ -646,6 +687,7 @@ describe("harness compaction", () => {
const result = getOrThrow(await compact(preparation!, models, model));
expect(result.summary.length).toBeGreaterThan(0);
expect(result.firstKeptEntryId).toBeTruthy();
expect(result.usage?.totalTokens).toBeGreaterThan(0);
expect(result.details).toBeDefined();
});
});
@@ -86,6 +86,49 @@ async function runSessionSuite(
expect(context.messages[1]?.role).toBe("branchSummary");
});
it("persists compaction usage", async () => {
const session = new Session(await createStorage());
const firstKeptEntryId = await session.appendMessage(createUserMessage("one"));
const usage = {
input: 1,
output: 2,
cacheRead: 3,
cacheWrite: 4,
totalTokens: 10,
cost: { input: 0.1, output: 0.2, cacheRead: 0.3, cacheWrite: 0.4, total: 1 },
};
const compactionId = await session.appendCompaction(
"summary",
firstKeptEntryId,
1234,
undefined,
false,
usage,
);
const compactionEntry = await session.getEntry(compactionId);
expect(compactionEntry?.type === "compaction" ? compactionEntry.usage : undefined).toEqual(usage);
});
it("persists branch summary usage", async () => {
const session = new Session(await createStorage());
const user1 = await session.appendMessage(createUserMessage("one"));
const usage = {
input: 1,
output: 2,
cacheRead: 3,
cacheWrite: 4,
totalTokens: 10,
cost: { input: 0.1, output: 0.2, cacheRead: 0.3, cacheWrite: 0.4, total: 1 },
};
const summaryId = await session.moveTo(user1, { summary: "summary text", usage });
const summaryEntry = await session.getEntry(summaryId!);
expect(summaryEntry?.type === "branch_summary" ? summaryEntry.usage : undefined).toEqual(usage);
});
it("supports custom message entries in context", async () => {
const session = new Session(await createStorage());
await session.appendMessage(createUserMessage("one"));
+8
View File
@@ -1,3 +1,4 @@
import type { Usage } from "@earendil-works/pi-ai";
import { type Static, Type } from "typebox";
import type { AgentTool, AgentToolResult } from "../../src/types.ts";
@@ -30,3 +31,10 @@ export const calculateTool: AgentTool<typeof calculateSchema, undefined> = {
return calculate(args.expression);
},
};
export function createCalculateToolWithUsage(usage: Usage): AgentTool<typeof calculateSchema, undefined> {
return {
...calculateTool,
execute: async (_toolCallId: string, args: CalculateParams) => ({ ...calculate(args.expression), usage }),
};
}
+2
View File
@@ -408,6 +408,8 @@ export interface ToolResultMessage<TDetails = any> {
toolName: string;
content: (TextContent | ImageContent)[]; // Supports text and images
details?: TDetails;
/** Usage from the tool execution itself, if available. Not part of main LLM context accounting. */
usage?: Usage;
/**
* Names from `Context.tools` that became available after this result.
* Providers with native deferred tool loading use this as the load point;
+33 -20
View File
@@ -32,6 +32,7 @@ import type {
Model,
ProviderHeaders,
TextContent,
Usage,
} from "@earendil-works/pi-ai/compat";
import {
clampThinkingLevel,
@@ -104,6 +105,7 @@ import { type BuildSystemPromptOptions, buildSystemPrompt } from "./system-promp
import { type BashOperations, createLocalBashOperations } from "./tools/bash.ts";
import { createAllToolDefinitions } from "./tools/index.ts";
import { createToolDefinitionFromAgentTool } from "./tools/tool-definition-wrapper.ts";
import { addUsageToTotals, createUsageTotals } from "./usage-totals.ts";
// ============================================================================
// Skill Block Parsing
@@ -482,6 +484,7 @@ export class AgentSession {
content: result.content,
details: result.details,
isError,
usage: result.usage,
});
if (!hookResult) {
@@ -492,6 +495,7 @@ export class AgentSession {
content: hookResult.content,
details: hookResult.details,
isError: hookResult.isError ?? isError,
usage: hookResult.usage,
};
};
}
@@ -1812,6 +1816,7 @@ export class AgentSession {
let summary: string;
let firstKeptEntryId: string;
let tokensBefore: number;
let usage: Usage | undefined;
let details: unknown;
if (extensionCompaction) {
@@ -1819,6 +1824,7 @@ export class AgentSession {
summary = extensionCompaction.summary;
firstKeptEntryId = extensionCompaction.firstKeptEntryId;
tokensBefore = extensionCompaction.tokensBefore;
usage = extensionCompaction.usage;
details = extensionCompaction.details;
} else {
// Generate compaction result
@@ -1836,6 +1842,7 @@ export class AgentSession {
summary = result.summary;
firstKeptEntryId = result.firstKeptEntryId;
tokensBefore = result.tokensBefore;
usage = result.usage;
details = result.details;
}
@@ -1843,7 +1850,7 @@ export class AgentSession {
throw new Error("Compaction cancelled");
}
this.sessionManager.appendCompaction(summary, firstKeptEntryId, tokensBefore, details, fromExtension);
this.sessionManager.appendCompaction(summary, firstKeptEntryId, tokensBefore, details, fromExtension, usage);
const newEntries = this.sessionManager.getEntries();
const sessionContext = this.sessionManager.buildSessionContext();
this.agent.state.messages = sessionContext.messages;
@@ -1869,6 +1876,7 @@ export class AgentSession {
firstKeptEntryId,
tokensBefore,
estimatedTokensAfter,
usage,
details,
};
this._emit({
@@ -2084,6 +2092,7 @@ export class AgentSession {
let summary: string;
let firstKeptEntryId: string;
let tokensBefore: number;
let usage: Usage | undefined;
let details: unknown;
if (extensionCompaction) {
@@ -2091,6 +2100,7 @@ export class AgentSession {
summary = extensionCompaction.summary;
firstKeptEntryId = extensionCompaction.firstKeptEntryId;
tokensBefore = extensionCompaction.tokensBefore;
usage = extensionCompaction.usage;
details = extensionCompaction.details;
} else {
// Generate compaction result
@@ -2108,6 +2118,7 @@ export class AgentSession {
summary = compactResult.summary;
firstKeptEntryId = compactResult.firstKeptEntryId;
tokensBefore = compactResult.tokensBefore;
usage = compactResult.usage;
details = compactResult.details;
}
@@ -2122,7 +2133,7 @@ export class AgentSession {
return false;
}
this.sessionManager.appendCompaction(summary, firstKeptEntryId, tokensBefore, details, fromExtension);
this.sessionManager.appendCompaction(summary, firstKeptEntryId, tokensBefore, details, fromExtension, usage);
const newEntries = this.sessionManager.getEntries();
const sessionContext = this.sessionManager.buildSessionContext();
this.agent.state.messages = sessionContext.messages;
@@ -2148,6 +2159,7 @@ export class AgentSession {
firstKeptEntryId,
tokensBefore,
estimatedTokensAfter,
usage,
details,
};
this._emit({ type: "compaction_end", reason, result, aborted: false, willRetry });
@@ -2872,7 +2884,7 @@ export class AgentSession {
this._branchSummaryAbortController = new AbortController();
try {
let extensionSummary: { summary: string; details?: unknown } | undefined;
let extensionSummary: { summary: string; details?: unknown; usage?: Usage } | undefined;
let fromExtension = false;
// Emit session_before_tree event
@@ -2907,6 +2919,7 @@ export class AgentSession {
// Run default summarizer if needed
let summaryText: string | undefined;
let summaryDetails: unknown;
let summaryUsage: Usage | undefined;
if (options.summarize && entriesToSummarize.length > 0 && !extensionSummary) {
const model = this.model!;
const { apiKey, headers, env } = await this._getSummarizationRequestAuth(model);
@@ -2929,6 +2942,7 @@ export class AgentSession {
throw new Error(result.error);
}
summaryText = result.summary;
summaryUsage = result.usage;
summaryDetails = {
readFiles: result.readFiles || [],
modifiedFiles: result.modifiedFiles || [],
@@ -2936,6 +2950,7 @@ export class AgentSession {
} else if (extensionSummary) {
summaryText = extensionSummary.summary;
summaryDetails = extensionSummary.details;
summaryUsage = extensionSummary.usage;
}
// Determine the new leaf position based on target type
@@ -2965,6 +2980,7 @@ export class AgentSession {
summaryText,
summaryDetails,
fromExtension,
summaryUsage,
);
summaryEntry = this.sessionManager.getEntry(summaryId) as BranchSummaryEntry;
@@ -3037,13 +3053,12 @@ export class AgentSession {
let toolResults = 0;
let totalMessages = 0;
let toolCalls = 0;
let totalInput = 0;
let totalOutput = 0;
let totalCacheRead = 0;
let totalCacheWrite = 0;
let totalCost = 0;
const usageTotals = createUsageTotals();
for (const entry of this.sessionManager.getEntries()) {
if ((entry.type === "branch_summary" || entry.type === "compaction") && entry.usage) {
addUsageToTotals(usageTotals, entry.usage);
}
if (entry.type !== "message") continue;
totalMessages++;
const message = entry.message;
@@ -3051,18 +3066,16 @@ export class AgentSession {
userMessages++;
} else if (message.role === "toolResult") {
toolResults++;
if (message.usage) {
addUsageToTotals(usageTotals, message.usage);
}
} else if (message.role === "assistant") {
assistantMessages++;
const assistantMsg = message as AssistantMessage;
if (Array.isArray(assistantMsg.content)) {
toolCalls += assistantMsg.content.filter((c) => c.type === "toolCall").length;
}
const usage = assistantMsg.usage;
totalInput += usage.input;
totalOutput += usage.output;
totalCacheRead += usage.cacheRead;
totalCacheWrite += usage.cacheWrite;
totalCost += usage.cost.total;
addUsageToTotals(usageTotals, assistantMsg.usage);
}
}
@@ -3075,13 +3088,13 @@ export class AgentSession {
toolResults,
totalMessages,
tokens: {
input: totalInput,
output: totalOutput,
cacheRead: totalCacheRead,
cacheWrite: totalCacheWrite,
total: totalInput + totalOutput + totalCacheRead + totalCacheWrite,
input: usageTotals.input,
output: usageTotals.output,
cacheRead: usageTotals.cacheRead,
cacheWrite: usageTotals.cacheWrite,
total: usageTotals.input + usageTotals.output + usageTotals.cacheRead + usageTotals.cacheWrite,
},
cost: totalCost,
cost: usageTotals.cost,
contextUsage: this.getContextUsage(),
};
}
@@ -7,7 +7,7 @@
import type { AgentMessage, StreamFn } from "@earendil-works/pi-agent-core";
import { contentText } from "@earendil-works/pi-ai";
import type { Model, SimpleStreamOptions } from "@earendil-works/pi-ai/compat";
import type { Model, SimpleStreamOptions, Usage } from "@earendil-works/pi-ai/compat";
import { completeSimple } from "@earendil-works/pi-ai/compat";
import {
convertToLlm,
@@ -33,6 +33,7 @@ import {
export interface BranchSummaryResult {
summary?: string;
usage?: Usage;
readFiles?: string[];
modifiedFiles?: string[];
aborted?: boolean;
@@ -363,6 +364,7 @@ export async function generateBranchSummary(
return {
summary: summary || "No summary generated",
usage: response.usage,
readFiles,
modifiedFiles,
};
@@ -90,10 +90,35 @@ export interface CompactionResult<T = unknown> {
firstKeptEntryId: string;
tokensBefore: number;
estimatedTokensAfter?: number;
/** Usage from the LLM call(s) that generated this summary, if available */
usage?: Usage;
/** Extension-specific data (e.g., ArtifactIndex, version markers for structured compaction) */
details?: T;
}
function combineUsage(first: Usage, second: Usage): Usage {
return {
input: first.input + second.input,
output: first.output + second.output,
cacheRead: first.cacheRead + second.cacheRead,
cacheWrite: first.cacheWrite + second.cacheWrite,
...(first.cacheWrite1h !== undefined || second.cacheWrite1h !== undefined
? { cacheWrite1h: (first.cacheWrite1h ?? 0) + (second.cacheWrite1h ?? 0) }
: {}),
...(first.reasoning !== undefined || second.reasoning !== undefined
? { reasoning: (first.reasoning ?? 0) + (second.reasoning ?? 0) }
: {}),
totalTokens: first.totalTokens + second.totalTokens,
cost: {
input: first.cost.input + second.cost.input,
output: first.cost.output + second.cost.output,
cacheRead: first.cost.cacheRead + second.cost.cacheRead,
cacheWrite: first.cost.cacheWrite + second.cost.cacheWrite,
total: first.cost.total + second.cost.total,
},
};
}
// ============================================================================
// Types
// ============================================================================
@@ -556,7 +581,7 @@ export async function generateSummary(
thinkingLevel?: ThinkingLevel,
streamFn?: StreamFn,
env?: Record<string, string>,
): Promise<string> {
): Promise<{ text: string; usage: Usage }> {
const maxTokens = Math.min(
Math.floor(0.8 * reserveTokens),
model.maxTokens > 0 ? model.maxTokens : Number.POSITIVE_INFINITY,
@@ -603,7 +628,7 @@ export async function generateSummary(
const textContent = contentText(response.content);
return textContent;
return { text: textContent, usage: response.usage };
}
// ============================================================================
@@ -759,11 +784,13 @@ export async function compact(
// Generate summaries and merge into one
let summary: string;
let summaryUsage: Usage;
if (isSplitTurn && turnPrefixMessages.length > 0) {
const historyResult =
messagesToSummarize.length > 0
? await generateSummary(
let historyText = "No prior history.";
let historyUsage: Usage | undefined;
if (messagesToSummarize.length > 0) {
const historyResult = await generateSummary(
messagesToSummarize,
model,
settings.reserveTokens,
@@ -775,8 +802,10 @@ export async function compact(
thinkingLevel,
streamFn,
env,
)
: "No prior history.";
);
historyText = historyResult.text;
historyUsage = historyResult.usage;
}
const turnPrefixResult = await generateTurnPrefixSummary(
turnPrefixMessages,
model,
@@ -789,10 +818,11 @@ export async function compact(
streamFn,
);
// Merge into single summary
summary = `${historyResult}\n\n---\n\n**Turn Context (split turn):**\n\n${turnPrefixResult}`;
summary = `${historyText}\n\n---\n\n**Turn Context (split turn):**\n\n${turnPrefixResult.text}`;
summaryUsage = historyUsage ? combineUsage(historyUsage, turnPrefixResult.usage) : turnPrefixResult.usage;
} else {
// Just generate history summary
summary = await generateSummary(
const result = await generateSummary(
messagesToSummarize,
model,
settings.reserveTokens,
@@ -805,6 +835,8 @@ export async function compact(
streamFn,
env,
);
summary = result.text;
summaryUsage = result.usage;
}
// Compute file lists and append to summary
@@ -819,6 +851,7 @@ export async function compact(
summary,
firstKeptEntryId,
tokensBefore,
usage: summaryUsage,
details: { readFiles, modifiedFiles } as CompactionDetails,
};
}
@@ -836,7 +869,7 @@ async function generateTurnPrefixSummary(
signal?: AbortSignal,
thinkingLevel?: ThinkingLevel,
streamFn?: StreamFn,
): Promise<string> {
): Promise<{ text: string; usage: Usage }> {
const maxTokens = Math.min(
Math.floor(0.5 * reserveTokens),
model.maxTokens > 0 ? model.maxTokens : Number.POSITIVE_INFINITY,
@@ -863,5 +896,8 @@ async function generateTurnPrefixSummary(
throw new Error(`Turn prefix summarization failed: ${response.errorMessage || "Unknown error"}`);
}
return contentText(response.content);
return {
text: contentText(response.content),
usage: response.usage,
};
}
@@ -883,6 +883,10 @@ export class ExtensionRunner {
currentEvent.isError = handlerResult.isError;
modified = true;
}
if (handlerResult.usage !== undefined) {
currentEvent.usage = handlerResult.usage;
modified = true;
}
} catch (err) {
const message = err instanceof Error ? err.message : String(err);
const stack = err instanceof Error ? err.stack : undefined;
@@ -904,6 +908,7 @@ export class ExtensionRunner {
content: currentEvent.content,
details: currentEvent.details,
isError: currentEvent.isError,
usage: currentEvent.usage,
};
}
@@ -30,6 +30,7 @@ import type {
SimpleStreamOptions,
TextContent,
ToolResultMessage,
Usage,
} from "@earendil-works/pi-ai";
import type {
AutocompleteItem,
@@ -905,6 +906,8 @@ interface ToolResultEventBase {
input: Record<string, unknown>;
content: (TextContent | ImageContent)[];
isError: boolean;
/** Usage from the tool execution itself, if available. */
usage?: Usage;
}
export interface BashToolResultEvent extends ToolResultEventBase {
@@ -1072,6 +1075,7 @@ export interface ToolResultEventResult {
content?: (TextContent | ImageContent)[];
details?: unknown;
isError?: boolean;
usage?: Usage;
}
export interface MessageEndEventResult {
@@ -1104,6 +1108,7 @@ export interface SessionBeforeTreeResult {
summary?: {
summary: string;
details?: unknown;
usage?: Usage;
};
/** Override custom instructions for summarization */
customInstructions?: string;
@@ -1,5 +1,5 @@
import type { AgentMessage } from "@earendil-works/pi-agent-core";
import { type ImageContent, type Message, type TextContent, uuidv7 } from "@earendil-works/pi-ai";
import { type ImageContent, type Message, type TextContent, type Usage, uuidv7 } from "@earendil-works/pi-ai";
import { randomUUID } from "crypto";
import {
appendFileSync,
@@ -73,6 +73,8 @@ export interface CompactionEntry<T = unknown> extends SessionEntryBase {
tokensBefore: number;
/** Extension-specific data (e.g., ArtifactIndex, version markers for structured compaction) */
details?: T;
/** Usage from the LLM call(s) that generated this summary, if available */
usage?: Usage;
/** True if generated by an extension, undefined/false if pi-generated (backward compatible) */
fromHook?: boolean;
}
@@ -83,6 +85,8 @@ export interface BranchSummaryEntry<T = unknown> extends SessionEntryBase {
summary: string;
/** Extension-specific data (not sent to LLM) */
details?: T;
/** Usage from the LLM call that generated this summary, if available */
usage?: Usage;
/** True if generated by an extension, false if pi-generated */
fromHook?: boolean;
}
@@ -1096,6 +1100,7 @@ export class SessionManager {
tokensBefore: number,
details?: T,
fromHook?: boolean,
usage?: Usage,
): string {
const entry: CompactionEntry<T> = {
type: "compaction",
@@ -1106,6 +1111,7 @@ export class SessionManager {
firstKeptEntryId,
tokensBefore,
details,
usage,
fromHook,
};
this._appendEntry(entry);
@@ -1372,7 +1378,13 @@ export class SessionManager {
* Same as branch(), but also appends a branch_summary entry that captures
* context from the abandoned conversation path.
*/
branchWithSummary(branchFromId: string | null, summary: string, details?: unknown, fromHook?: boolean): string {
branchWithSummary(
branchFromId: string | null,
summary: string,
details?: unknown,
fromHook?: boolean,
usage?: Usage,
): string {
if (branchFromId !== null && !this.byId.has(branchFromId)) {
throw new Error(`Entry ${branchFromId} not found`);
}
@@ -1385,6 +1397,7 @@ export class SessionManager {
fromId: branchFromId ?? "root",
summary,
details,
usage,
fromHook,
};
this._appendEntry(entry);
@@ -0,0 +1,27 @@
import type { Usage } from "@earendil-works/pi-ai/compat";
export interface UsageTotals {
input: number;
output: number;
cacheRead: number;
cacheWrite: number;
cost: number;
}
export function createUsageTotals(): UsageTotals {
return {
input: 0,
output: 0,
cacheRead: 0,
cacheWrite: 0,
cost: 0,
};
}
export function addUsageToTotals(totals: UsageTotals, usage: Usage): void {
totals.input += usage.input;
totals.output += usage.output;
totals.cacheRead += usage.cacheRead;
totals.cacheWrite += usage.cacheWrite;
totals.cost += usage.cost.total;
}
@@ -3,6 +3,7 @@ import { type Component, truncateToWidth, visibleWidth } from "@earendil-works/p
import type { AgentSession } from "../../../core/agent-session.ts";
import { areExperimentalFeaturesEnabled } from "../../../core/experimental.ts";
import type { ReadonlyFooterDataProvider } from "../../../core/footer-data-provider.ts";
import { addUsageToTotals, createUsageTotals } from "../../../core/usage-totals.ts";
import { theme } from "../theme/theme.ts";
/**
@@ -84,25 +85,21 @@ export class FooterComponent implements Component {
const state = this.session.state;
// Calculate cumulative usage from ALL session entries (not just post-compaction messages)
let totalInput = 0;
let totalOutput = 0;
let totalCacheRead = 0;
let totalCacheWrite = 0;
let totalCost = 0;
const usageTotals = createUsageTotals();
let latestCacheHitRate: number | undefined;
for (const entry of this.session.sessionManager.getEntries()) {
if (entry.type === "message" && entry.message.role === "assistant") {
totalInput += entry.message.usage.input;
totalOutput += entry.message.usage.output;
totalCacheRead += entry.message.usage.cacheRead;
totalCacheWrite += entry.message.usage.cacheWrite;
totalCost += entry.message.usage.cost.total;
addUsageToTotals(usageTotals, entry.message.usage);
const latestPromptTokens =
entry.message.usage.input + entry.message.usage.cacheRead + entry.message.usage.cacheWrite;
latestCacheHitRate =
latestPromptTokens > 0 ? (entry.message.usage.cacheRead / latestPromptTokens) * 100 : undefined;
} else if (entry.type === "message" && entry.message.role === "toolResult" && entry.message.usage) {
addUsageToTotals(usageTotals, entry.message.usage);
} else if ((entry.type === "branch_summary" || entry.type === "compaction") && entry.usage) {
addUsageToTotals(usageTotals, entry.usage);
}
}
@@ -130,19 +127,20 @@ export class FooterComponent implements Component {
// Build stats line
const statsParts = [];
if (totalInput) statsParts.push(`${formatTokens(totalInput)}`);
if (totalOutput) statsParts.push(`${formatTokens(totalOutput)}`);
if (totalCacheRead) statsParts.push(`R${formatTokens(totalCacheRead)}`);
if (totalCacheWrite) statsParts.push(`W${formatTokens(totalCacheWrite)}`);
if ((totalCacheRead > 0 || totalCacheWrite > 0) && latestCacheHitRate !== undefined) {
if (usageTotals.input) statsParts.push(`${formatTokens(usageTotals.input)}`);
if (usageTotals.output) statsParts.push(`${formatTokens(usageTotals.output)}`);
if (usageTotals.cacheRead) statsParts.push(`R${formatTokens(usageTotals.cacheRead)}`);
if (usageTotals.cacheWrite) statsParts.push(`W${formatTokens(usageTotals.cacheWrite)}`);
if ((usageTotals.cacheRead > 0 || usageTotals.cacheWrite > 0) && latestCacheHitRate !== undefined) {
statsParts.push(`CH${latestCacheHitRate.toFixed(1)}%`);
}
// Kimi Coding is subscription-backed despite using API-key authentication.
const usingSubscription = state.model
? state.model.provider === "kimi-coding" || this.session.modelRuntime.isUsingOAuth(state.model.provider)
: false;
if (totalCost || usingSubscription) {
const costStr = `$${totalCost.toFixed(3)}${usingSubscription ? " (sub)" : ""}`;
if (usageTotals.cost || usingSubscription) {
const costStr = `$${usageTotals.cost.toFixed(3)}${usingSubscription ? " (sub)" : ""}`;
statsParts.push(costStr);
}
@@ -1,5 +1,5 @@
import { Agent } from "@earendil-works/pi-agent-core";
import { type AssistantMessage, getModel, type Usage } from "@earendil-works/pi-ai/compat";
import { type AssistantMessage, getModel, type ToolResultMessage, type Usage } from "@earendil-works/pi-ai/compat";
import { describe, expect, it } from "vitest";
import { AgentSession } from "../src/core/agent-session.ts";
import { AuthStorage } from "../src/core/auth-storage.ts";
@@ -48,6 +48,18 @@ function createUserMessage(text: string, timestamp: number) {
};
}
function createToolResultMessage(usage: Usage): ToolResultMessage {
return {
role: "toolResult",
toolCallId: "tool-call-1",
toolName: "test_tool",
content: [{ type: "text", text: "tool result" }],
usage,
isError: false,
timestamp: 1,
};
}
async function createSession() {
const settingsManager = SettingsManager.inMemory();
const sessionManager = SessionManager.inMemory();
@@ -143,6 +155,75 @@ describe("AgentSession.getSessionStats", () => {
}
});
it("includes branch summary usage in session totals", async () => {
const { session, sessionManager } = await createSession();
try {
sessionManager.branchWithSummary(null, "summary", undefined, false, {
input: 10,
output: 20,
cacheRead: 30,
cacheWrite: 40,
totalTokens: 100,
cost: { input: 0.1, output: 0.2, cacheRead: 0.3, cacheWrite: 0.4, total: 1 },
});
syncAgentMessages(session, sessionManager);
const stats = session.getSessionStats();
expect(stats.tokens).toEqual({ input: 10, output: 20, cacheRead: 30, cacheWrite: 40, total: 100 });
expect(stats.cost).toBe(1);
} finally {
session.dispose();
}
});
it("includes compaction usage in session totals", async () => {
const { session, sessionManager } = await createSession();
try {
const firstKeptEntryId = sessionManager.appendMessage(createUserMessage("hello", 1));
sessionManager.appendCompaction("summary", firstKeptEntryId, 100, undefined, false, {
input: 10,
output: 20,
cacheRead: 30,
cacheWrite: 40,
totalTokens: 100,
cost: { input: 0.1, output: 0.2, cacheRead: 0.3, cacheWrite: 0.4, total: 1 },
});
syncAgentMessages(session, sessionManager);
const stats = session.getSessionStats();
expect(stats.tokens).toEqual({ input: 10, output: 20, cacheRead: 30, cacheWrite: 40, total: 100 });
expect(stats.cost).toBe(1);
} finally {
session.dispose();
}
});
it("includes tool result usage in session totals", async () => {
const { session, sessionManager } = await createSession();
try {
sessionManager.appendMessage(
createToolResultMessage({
input: 10,
output: 20,
cacheRead: 30,
cacheWrite: 40,
totalTokens: 100,
cost: { input: 0.1, output: 0.2, cacheRead: 0.3, cacheWrite: 0.4, total: 1 },
}),
);
syncAgentMessages(session, sessionManager);
const stats = session.getSessionStats();
expect(stats.tokens).toEqual({ input: 10, output: 20, cacheRead: 30, cacheWrite: 40, total: 100 });
expect(stats.cost).toBe(1);
} finally {
session.dispose();
}
});
it("ignores zero-usage messages when checking for post-compaction context usage", async () => {
const { session, sessionManager } = await createSession();
@@ -0,0 +1,55 @@
import type { Usage } from "@earendil-works/pi-ai/compat";
import { afterEach, describe, expect, it } from "vitest";
import { createHarness, type Harness } from "./suite/harness.ts";
import { assistantMsg, userMsg } from "./utilities.ts";
describe("Branch summary extensions", () => {
const harnesses: Harness[] = [];
afterEach(() => {
while (harnesses.length > 0) {
harnesses.pop()?.cleanup();
}
});
it("persists extension-provided summary usage in session totals", async () => {
const usage: Usage = {
input: 10,
output: 20,
cacheRead: 30,
cacheWrite: 40,
totalTokens: 100,
cost: { input: 0.1, output: 0.2, cacheRead: 0.3, cacheWrite: 0.4, total: 1 },
};
const harness = await createHarness({
extensionFactories: [
(pi) => {
pi.on("session_before_tree", () => ({
summary: {
summary: "Summary provided by extension",
usage,
},
}));
},
],
});
harnesses.push(harness);
const targetId = harness.sessionManager.appendMessage(userMsg("first branch"));
harness.sessionManager.appendMessage(assistantMsg("first reply"));
harness.sessionManager.appendMessage(userMsg("abandoned branch work"));
harness.sessionManager.appendMessage(assistantMsg("abandoned reply"));
const result = await harness.session.navigateTree(targetId, { summarize: true });
const summaryEntry = result.summaryEntry;
expect(summaryEntry?.type).toBe("branch_summary");
expect(summaryEntry?.fromHook).toBe(true);
expect(summaryEntry?.summary).toBe("Summary provided by extension");
expect(summaryEntry?.usage).toEqual(usage);
const stats = harness.session.getSessionStats();
expect(stats.tokens).toEqual({ input: 12, output: 22, cacheRead: 30, cacheWrite: 40, total: 104 });
expect(stats.cost).toBe(1);
});
});
@@ -57,7 +57,7 @@ describe("generateSummary reasoning options", () => {
});
it("uses the provided thinking level for reasoning-capable models", async () => {
await generateSummary(
const result = await generateSummary(
messages,
createModel(true),
2000,
@@ -69,6 +69,9 @@ describe("generateSummary reasoning options", () => {
"medium",
);
expect(result.text).toBe("## Goal\nTest summary");
expect(result.usage).toEqual(mockSummaryResponse.usage);
expect(completeSimpleMock).toHaveBeenCalledTimes(1);
expect(completeSimpleMock.mock.calls[0][2]).toMatchObject({
reasoning: "medium",
@@ -127,8 +130,15 @@ describe("generateSummary reasoning options", () => {
settings: { enabled: true, reserveTokens: 500000, keepRecentTokens: 20000 },
};
await compact(preparation, createModel(false, 128000), "test-key");
const result = await compact(preparation, createModel(false, 128000), "test-key");
expect(result.usage).toEqual({
...mockSummaryResponse.usage,
input: 20,
output: 20,
totalTokens: 40,
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 },
});
expect(completeSimpleMock.mock.calls.map((call) => call[2]?.maxTokens)).toEqual([128000, 128000]);
});
});
@@ -21,20 +21,46 @@ function createSession(options: {
reasoning?: boolean;
thinkingLevel?: string;
usage?: AssistantUsage;
branchUsage?: AssistantUsage;
compactionUsage?: AssistantUsage;
toolUsage?: AssistantUsage;
}): AgentSession {
const usage = options.usage;
const entries =
usage === undefined
? []
: [
{
const entries: Array<Record<string, unknown>> = [];
if (usage !== undefined) {
entries.push({
type: "message",
message: {
role: "assistant",
usage,
},
});
}
if (options.branchUsage !== undefined) {
entries.push({
type: "branch_summary",
usage: options.branchUsage,
});
}
if (options.compactionUsage !== undefined) {
entries.push({
type: "compaction",
usage: options.compactionUsage,
});
}
if (options.toolUsage !== undefined) {
entries.push({
type: "message",
message: {
role: "toolResult",
usage: options.toolUsage,
},
];
});
}
const session = {
state: {
@@ -125,6 +151,44 @@ describe("FooterComponent width handling", () => {
}
});
it("includes summary and tool result usage in the total cost", () => {
const session = createSession({
sessionName: "",
usage: {
input: 100,
output: 10,
cacheRead: 0,
cacheWrite: 0,
cost: { total: 0.5 },
},
branchUsage: {
input: 20,
output: 5,
cacheRead: 0,
cacheWrite: 0,
cost: { total: 0.25 },
},
compactionUsage: {
input: 5,
output: 2,
cacheRead: 0,
cacheWrite: 0,
cost: { total: 0.125 },
},
toolUsage: {
input: 15,
output: 3,
cacheRead: 0,
cacheWrite: 0,
cost: { total: 0.375 },
},
});
const footer = new FooterComponent(session, createFooterData(1));
const statsLine = stripAnsi(footer.render(120)[1]);
expect(statsLine).toContain("$1.250");
});
it("shows the latest cache hit rate when cache usage is present", () => {
const session = createSession({
sessionName: "",
@@ -71,7 +71,15 @@ describe("SessionManager append and tree traversal", () => {
const id1 = session.appendMessage(userMsg("1"));
const id2 = session.appendMessage(assistantMsg("2"));
const compactionId = session.appendCompaction("summary", id1, 1000);
const usage = {
input: 10,
output: 20,
cacheRead: 30,
cacheWrite: 40,
totalTokens: 100,
cost: { input: 0.1, output: 0.2, cacheRead: 0.3, cacheWrite: 0.4, total: 1 },
};
const compactionId = session.appendCompaction("summary", id1, 1000, undefined, false, usage);
const _id3 = session.appendMessage(userMsg("3"));
const entries = session.getEntries();
@@ -83,6 +91,7 @@ describe("SessionManager append and tree traversal", () => {
expect(compactionEntry.summary).toBe("summary");
expect(compactionEntry.firstKeptEntryId).toBe(id1);
expect(compactionEntry.tokensBefore).toBe(1000);
expect(compactionEntry.usage).toEqual(usage);
}
expect(entries[3].parentId).toBe(compactionId);
@@ -319,7 +328,15 @@ describe("SessionManager append and tree traversal", () => {
const _id2 = session.appendMessage(assistantMsg("2"));
const _id3 = session.appendMessage(userMsg("3"));
const summaryId = session.branchWithSummary(id1, "Summary of abandoned work");
const usage = {
input: 10,
output: 20,
cacheRead: 30,
cacheWrite: 40,
totalTokens: 100,
cost: { input: 0.1, output: 0.2, cacheRead: 0.3, cacheWrite: 0.4, total: 1 },
};
const summaryId = session.branchWithSummary(id1, "Summary of abandoned work", undefined, false, usage);
expect(session.getLeafId()).toBe(summaryId);
@@ -329,6 +346,7 @@ describe("SessionManager append and tree traversal", () => {
expect(summaryEntry?.parentId).toBe(id1);
if (summaryEntry?.type === "branch_summary") {
expect(summaryEntry.summary).toBe("Summary of abandoned work");
expect(summaryEntry.usage).toEqual(usage);
}
});
@@ -97,6 +97,14 @@ describe("AgentSession compaction characterization", () => {
});
it("manually compacts using an extension-provided summary", async () => {
const summaryUsage = {
input: 10,
output: 20,
cacheRead: 30,
cacheWrite: 40,
totalTokens: 100,
cost: { input: 0.1, output: 0.2, cacheRead: 0.3, cacheWrite: 0.4, total: 1 },
};
const harness = await createHarness({
settings: { compaction: { keepRecentTokens: 1 } },
extensionFactories: [
@@ -106,6 +114,7 @@ describe("AgentSession compaction characterization", () => {
summary: "summary from extension",
firstKeptEntryId: event.preparation.firstKeptEntryId,
tokensBefore: event.preparation.tokensBefore,
usage: summaryUsage,
details: { source: "extension" },
},
}));
@@ -116,14 +125,26 @@ describe("AgentSession compaction characterization", () => {
await harness.session.prompt("one");
await harness.session.prompt("two");
const statsBefore = harness.session.getSessionStats();
const result = await harness.session.compact();
const compactionEntries = harness.sessionManager.getEntries().filter((entry) => entry.type === "compaction");
const estimatedTokensAfter = harness.session.messages.reduce((sum, message) => sum + estimateTokens(message), 0);
expect(result.summary).toBe("summary from extension");
expect(result.usage).toEqual(summaryUsage);
expect(result.estimatedTokensAfter).toBe(estimatedTokensAfter);
expect(compactionEntries).toHaveLength(1);
const compactionEntry = compactionEntries[0];
if (compactionEntry?.type === "compaction") {
expect(compactionEntry.usage).toEqual(summaryUsage);
}
const statsAfter = harness.session.getSessionStats();
expect(statsAfter.tokens.input).toBe(statsBefore.tokens.input + summaryUsage.input);
expect(statsAfter.tokens.output).toBe(statsBefore.tokens.output + summaryUsage.output);
expect(statsAfter.tokens.cacheRead).toBe(statsBefore.tokens.cacheRead + summaryUsage.cacheRead);
expect(statsAfter.tokens.cacheWrite).toBe(statsBefore.tokens.cacheWrite + summaryUsage.cacheWrite);
expect(statsAfter.cost).toBe(statsBefore.cost + summaryUsage.cost.total);
expect(harness.session.messages[0]?.role).toBe("compactionSummary");
});
@@ -154,6 +175,22 @@ describe("AgentSession compaction characterization", () => {
expect(getStreamCallCount()).toBe(1);
});
it("persists usage from pi-generated manual compaction", async () => {
const harness = await createHarness({ withConfiguredAuth: false });
harnesses.push(harness);
seedCompactableSession(harness);
useSummaryStreamFn(harness, "summary from custom stream");
const result = await harness.session.compact();
const compactionEntries = harness.sessionManager.getEntries().filter((entry) => entry.type === "compaction");
expect(result.usage).toEqual(createUsage(10));
expect(compactionEntries).toHaveLength(1);
expect(compactionEntries[0]?.type === "compaction" ? compactionEntries[0].usage : undefined).toEqual(
createUsage(10),
);
});
it("auto-compacts with a custom streamFn when registry auth is absent", async () => {
const harness = await createHarness({ withConfiguredAuth: false });
harnesses.push(harness);
@@ -1,5 +1,5 @@
import type { AgentTool, ThinkingLevel } from "@earendil-works/pi-agent-core";
import { fauxAssistantMessage, fauxToolCall, type Model } from "@earendil-works/pi-ai";
import { fauxAssistantMessage, fauxToolCall, type Model, type Usage } from "@earendil-works/pi-ai";
import { Type } from "typebox";
import { afterEach, describe, expect, it } from "vitest";
import type { BuildSystemPromptOptions, ExtensionAPI } from "../../src/index.ts";
@@ -156,6 +156,23 @@ describe("AgentSession model and extension characterization", () => {
});
it("allows extension tool_result handlers to modify tool results", async () => {
const toolUsage: Usage = {
input: 1,
output: 2,
cacheRead: 3,
cacheWrite: 4,
totalTokens: 10,
cost: { input: 0.1, output: 0.2, cacheRead: 0.3, cacheWrite: 0.4, total: 1 },
};
const patchedToolUsage: Usage = {
input: 5,
output: 6,
cacheRead: 7,
cacheWrite: 8,
totalTokens: 26,
cost: { input: 0.5, output: 0.6, cacheRead: 0.7, cacheWrite: 0.8, total: 2.6 },
};
let observedToolUsage: Usage | undefined;
const echoTool: AgentTool = {
name: "echo",
label: "Echo",
@@ -163,17 +180,21 @@ describe("AgentSession model and extension characterization", () => {
parameters: Type.Object({ text: Type.String() }),
execute: async (_toolCallId, params) => {
const text = typeof params === "object" && params !== null && "text" in params ? String(params.text) : "";
return { content: [{ type: "text", text }], details: { text } };
return { content: [{ type: "text", text }], details: { text }, usage: toolUsage };
},
};
const harness = await createHarness({
tools: [echoTool],
extensionFactories: [
(pi) => {
pi.on("tool_result", async () => ({
pi.on("tool_result", async (event) => {
observedToolUsage = event.usage;
return {
content: [{ type: "text", text: "patched result" }],
details: { patched: true },
}));
usage: patchedToolUsage,
};
});
},
],
});
@@ -196,9 +217,12 @@ describe("AgentSession model and extension characterization", () => {
await harness.session.prompt("hi");
expect(getAssistantTexts(harness)).toContain("patched result");
expect(
harness.session.messages.find((message) => message.role === "toolResult" && message.details?.patched === true),
).toBeDefined();
const toolResult = harness.session.messages.find(
(message) => message.role === "toolResult" && message.details?.patched === true,
);
expect(observedToolUsage).toEqual(toolUsage);
expect(toolResult).toBeDefined();
expect(toolResult?.role === "toolResult" ? toolResult.usage : undefined).toEqual(patchedToolUsage);
});
it("allows extension context handlers to modify messages before the LLM call", async () => {
@@ -37,7 +37,7 @@ describe("issue #6324 branch summary ambient auth", () => {
cacheRead: 0,
cacheWrite: 0,
totalTokens: 2,
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 },
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0.25 },
},
stopReason: "stop",
timestamp: Date.now(),
@@ -57,5 +57,6 @@ describe("issue #6324 branch summary ambient auth", () => {
expect(streamCallCount).toBe(1);
expect(result.summaryEntry?.type).toBe("branch_summary");
expect(result.summaryEntry?.summary).toContain("branch summary text");
expect(result.summaryEntry?.usage?.cost.total).toBe(0.25);
});
});