add usage info to branch summary, compaction and tool result entries (#6671)

* add usage info to branch summary entries

* add usage to compaction entries

* allow custom tools to report llm usage in tool results

* allow observing and patching usage in tool_result hooks

* agent-harness: save usage in entries

for compaction, branch summaries and tool results

packages/agent/test/agent-loop.test.ts

@@ -239,6 +239,23 @@ describe("agentLoop with AgentMessage", () => {
 	it("should handle tool calls and results", async () => {
 		const toolSchema = Type.Object({ value: Type.String() });
 		const executed: string[] = [];
+		const toolUsage = {
+			input: 1,
+			output: 2,
+			cacheRead: 3,
+			cacheWrite: 4,
+			totalTokens: 10,
+			cost: { input: 0.1, output: 0.2, cacheRead: 0.3, cacheWrite: 0.4, total: 1 },
+		};
+		const patchedToolUsage = {
+			input: 5,
+			output: 6,
+			cacheRead: 7,
+			cacheWrite: 8,
+			totalTokens: 26,
+			cost: { input: 0.5, output: 0.6, cacheRead: 0.7, cacheWrite: 0.8, total: 2.6 },
+		};
+		let observedToolUsage: typeof toolUsage | undefined;
 		const tool: AgentTool<typeof toolSchema, { value: string }> = {
 			name: "echo",
 			label: "Echo",
@@ -249,6 +266,7 @@ describe("agentLoop with AgentMessage", () => {
 				return {
 					content: [{ type: "text", text: `echoed: ${params.value}` }],
 					details: { value: params.value },
+					usage: toolUsage,
 				};
 			},
 		};
@@ -264,6 +282,10 @@ describe("agentLoop with AgentMessage", () => {
 		const config: AgentLoopConfig = {
 			model: createModel(),
 			convertToLlm: identityConverter,
+			afterToolCall: async ({ result }) => {
+				observedToolUsage = result.usage;
+				return { usage: patchedToolUsage };
+			},
 		};
 
 		let callIndex = 0;
@@ -305,6 +327,10 @@ describe("agentLoop with AgentMessage", () => {
 		if (toolEnd?.type === "tool_execution_end") {
 			expect(toolEnd.isError).toBe(false);
 		}
+		expect(observedToolUsage).toEqual(toolUsage);
+		const messages = await stream.result();
+		const toolResult = messages.find((message) => message.role === "toolResult");
+		expect(toolResult?.role === "toolResult" ? toolResult.usage : undefined).toEqual(patchedToolUsage);
 	});
 
 	it("should not execute tool calls from a length-truncated assistant message", async () => {

packages/agent/test/harness/agent-harness.test.ts

@@ -5,6 +5,7 @@ import {
 	fauxProvider,
 	fauxToolCall,
 	type RegisterFauxProviderOptions,
+	type Usage,
 } from "@earendil-works/pi-ai";
 import { getModel } from "@earendil-works/pi-ai/compat";
 import { describe, expect, it } from "vitest";
@@ -14,7 +15,7 @@ import { InMemorySessionStorage } from "../../src/harness/session/memory-storage
 import { Session } from "../../src/harness/session/session.ts";
 import type { PromptTemplate, Skill } from "../../src/harness/types.ts";
 import type { AgentMessage, AgentTool } from "../../src/types.ts";
-import { calculateTool } from "../utils/calculate.ts";
+import { calculateTool, createCalculateToolWithUsage } from "../utils/calculate.ts";
 import { getCurrentTimeTool } from "../utils/get-current-time.ts";
 
 interface AppSkill extends Skill {
@@ -60,6 +61,34 @@ function getReasoning(options: unknown): unknown {
 	return options.reasoning;
 }
 
+function createUsage(input: number, output: number, cacheRead = 0, cacheWrite = 0): Usage {
+	return {
+		input,
+		output,
+		cacheRead,
+		cacheWrite,
+		totalTokens: input + output + cacheRead + cacheWrite,
+		cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 },
+	};
+}
+
+function createUserMessage(text: string): AgentMessage {
+	return { role: "user", content: [{ type: "text", text }], timestamp: Date.now() };
+}
+
+function createAssistantMessage(text: string): AgentMessage {
+	return {
+		role: "assistant",
+		content: [{ type: "text", text }],
+		api: "faux",
+		provider: "faux",
+		model: "faux-1",
+		usage: createUsage(100, 50),
+		stopReason: "stop",
+		timestamp: Date.now(),
+	};
+}
+
 describe("AgentHarness", () => {
 	it("constructs directly and exposes queue modes", () => {
 		const session = new Session(new InMemorySessionStorage());
@@ -427,14 +456,18 @@ describe("AgentHarness", () => {
 				}),
 		]);
 		const session = new Session(new InMemorySessionStorage());
+		const toolUsage = createUsage(1, 2, 3, 4);
+		const patchedToolUsage = createUsage(5, 6, 7, 8);
+		const calculateToolWithUsage = createCalculateToolWithUsage(toolUsage);
 		const harness = new AgentHarness({
 			models,
 			env: new NodeExecutionEnv({ cwd: process.cwd() }),
 			session,
 			model: registration.getModel(),
-			tools: [calculateTool],
+			tools: [calculateToolWithUsage],
 		});
 		const seenToolCalls: Array<{ id: string; name: string; expression: unknown }> = [];
+		let seenToolUsage: Usage | undefined;
 		harness.on("tool_call", (event) => {
 			seenToolCalls.push({ id: event.toolCallId, name: event.toolName, expression: event.input.expression });
 			return undefined;
@@ -442,9 +475,11 @@ describe("AgentHarness", () => {
 		harness.on("tool_result", (event) => {
 			expect(event.toolCallId).toBe("call-1");
 			expect(event.toolName).toBe("calculate");
+			seenToolUsage = event.usage;
 			return {
 				content: [{ type: "text", text: "patched result" }],
 				details: { patched: true },
+				usage: patchedToolUsage,
 				terminate: true,
 			};
 		});
@@ -455,16 +490,109 @@ describe("AgentHarness", () => {
 			(entry) => entry.type === "message" && entry.message.role === "toolResult",
 		);
 		expect(seenToolCalls).toEqual([{ id: "call-1", name: "calculate", expression: "2 + 2" }]);
+		expect(seenToolUsage).toEqual(toolUsage);
 		expect(toolResult).toMatchObject({
 			type: "message",
 			message: {
 				role: "toolResult",
 				content: [{ type: "text", text: "patched result" }],
 				details: { patched: true },
+				usage: patchedToolUsage,
 			},
 		});
 	});
 
+	it("persists generated compaction usage", async () => {
+		const registration = newFaux();
+		registration.setResponses([fauxAssistantMessage("## Goal\nTest summary")]);
+		const session = new Session(new InMemorySessionStorage());
+		await session.appendMessage(createUserMessage("one"));
+		await session.appendMessage(createAssistantMessage("two"));
+		const harness = new AgentHarness({
+			models,
+			env: new NodeExecutionEnv({ cwd: process.cwd() }),
+			session,
+			model: registration.getModel(),
+		});
+
+		const result = await harness.compact();
+		const compaction = (await session.getEntries()).find((entry) => entry.type === "compaction");
+
+		expect(result.usage?.totalTokens).toBeGreaterThan(0);
+		expect(compaction?.type === "compaction" ? compaction.usage : undefined).toEqual(result.usage);
+	});
+
+	it("persists hook-provided compaction usage", async () => {
+		const registration = newFaux();
+		const usage = createUsage(5, 6, 7, 8);
+		const session = new Session(new InMemorySessionStorage());
+		await session.appendMessage(createUserMessage("one"));
+		await session.appendMessage(createAssistantMessage("two"));
+		const harness = new AgentHarness({
+			models,
+			env: new NodeExecutionEnv({ cwd: process.cwd() }),
+			session,
+			model: registration.getModel(),
+		});
+		harness.on("session_before_compact", (event) => ({
+			compaction: {
+				summary: "hook summary",
+				firstKeptEntryId: event.preparation.firstKeptEntryId,
+				tokensBefore: event.preparation.tokensBefore,
+				usage,
+			},
+		}));
+
+		const result = await harness.compact();
+		const compaction = (await session.getEntries()).find((entry) => entry.type === "compaction");
+
+		expect(result.usage).toEqual(usage);
+		expect(compaction?.type === "compaction" ? compaction.usage : undefined).toEqual(usage);
+	});
+
+	it("persists generated branch summary usage", async () => {
+		const registration = newFaux();
+		registration.setResponses([fauxAssistantMessage("## Goal\nBranch summary")]);
+		const session = new Session(new InMemorySessionStorage());
+		const targetId = await session.appendMessage(createUserMessage("first branch"));
+		await session.appendMessage(createAssistantMessage("first reply"));
+		await session.appendMessage(createUserMessage("abandoned work"));
+		await session.appendMessage(createAssistantMessage("abandoned reply"));
+		const harness = new AgentHarness({
+			models,
+			env: new NodeExecutionEnv({ cwd: process.cwd() }),
+			session,
+			model: registration.getModel(),
+		});
+
+		const result = await harness.navigateTree(targetId, { summarize: true });
+
+		expect(result.summaryEntry?.usage?.totalTokens).toBeGreaterThan(0);
+	});
+
+	it("persists hook-provided branch summary usage", async () => {
+		const registration = newFaux();
+		const usage = createUsage(13, 14, 15, 16);
+		const session = new Session(new InMemorySessionStorage());
+		const targetId = await session.appendMessage(createUserMessage("first branch"));
+		await session.appendMessage(createAssistantMessage("first reply"));
+		await session.appendMessage(createUserMessage("abandoned work"));
+		await session.appendMessage(createAssistantMessage("abandoned reply"));
+		const harness = new AgentHarness({
+			models,
+			env: new NodeExecutionEnv({ cwd: process.cwd() }),
+			session,
+			model: registration.getModel(),
+		});
+		harness.on("session_before_tree", () => ({
+			summary: { summary: "hook branch summary", usage },
+		}));
+
+		const result = await harness.navigateTree(targetId, { summarize: true });
+
+		expect(result.summaryEntry?.usage).toEqual(usage);
+	});
+
 	it("preserves app tool types for getters and update events", async () => {
 		const session = new Session(new InMemorySessionStorage());
 		const env = new NodeExecutionEnv({ cwd: process.cwd() });

packages/agent/test/harness/compaction.test.ts

@@ -6,6 +6,7 @@ import {
 	fauxProvider,
 	type Message,
 	type Model,
+	type Models,
 	type Usage,
 } from "@earendil-works/pi-ai";
 import { beforeEach, describe, expect, it } from "vitest";
@@ -142,6 +143,17 @@ function createFauxModel(reasoning: boolean, maxTokens = 8192): { faux: FauxProv
 	return { faux, model: faux.getModel() };
 }
 
+function createModelsWithSimpleResponses(responses: AssistantMessage[]): Models {
+	const remaining = [...responses];
+	const stub = Object.create(models) as Models;
+	stub.completeSimple = async () => {
+		const response = remaining.shift();
+		if (!response) throw new Error("No faux completeSimple response queued");
+		return response;
+	};
+	return stub;
+}
+
 describe("harness compaction", () => {
 	beforeEach(() => {
 		nextId = 0;
@@ -501,7 +513,12 @@ describe("harness compaction", () => {
 			await generateSummary(messages, models, model, 2000, undefined, "focus", "old summary"),
 		);
 
-		expect(summary).toContain("Test summary");
+		expect(summary.text).toContain("Test summary");
+		expect(summary.usage.input).toBeGreaterThan(0);
+		expect(summary.usage.output).toBeGreaterThan(0);
+		expect(summary.usage.totalTokens).toBe(
+			summary.usage.input + summary.usage.output + summary.usage.cacheRead + summary.usage.cacheWrite,
+		);
 		expect(promptText).toContain("<previous-summary>\nold summary\n</previous-summary>");
 		expect(promptText).toContain("Additional focus: focus");
 	});
@@ -578,6 +595,30 @@ describe("harness compaction", () => {
 		expect(invalidResult).toMatchObject({ ok: false, error: { code: "invalid_session" } });
 	});
 
+	it("combines usage for split-turn compaction summaries", async () => {
+		const messages: AgentMessage[] = [createUserMessage("Summarize this.")];
+		const { model } = createFauxModel(false);
+		const historyUsage = createMockUsage(1, 2, 3, 4);
+		const turnPrefixUsage = createMockUsage(5, 6, 7, 8);
+		const usageModels = createModelsWithSimpleResponses([
+			{ ...fauxAssistantMessage("history summary"), usage: historyUsage },
+			{ ...fauxAssistantMessage("turn prefix summary"), usage: turnPrefixUsage },
+		]);
+		const preparation: CompactionPreparation = {
+			firstKeptEntryId: "entry-keep",
+			messagesToSummarize: messages,
+			turnPrefixMessages: messages,
+			isSplitTurn: true,
+			tokensBefore: 100,
+			fileOps: { read: new Set(), written: new Set(), edited: new Set() },
+			settings: { enabled: true, reserveTokens: 2000, keepRecentTokens: 20 },
+		};
+
+		const result = getOrThrow(await compact(preparation, usageModels, model));
+
+		expect(result.usage).toEqual(createMockUsage(6, 8, 10, 12));
+	});
+
 	it("passes reasoning through turn-prefix summaries when enabled", async () => {
 		const messages: AgentMessage[] = [createUserMessage("Summarize this.")];
 		const seenOptions: Array<Record<string, unknown> | undefined> = [];
@@ -646,6 +687,7 @@ describe("harness compaction", () => {
 		const result = getOrThrow(await compact(preparation!, models, model));
 		expect(result.summary.length).toBeGreaterThan(0);
 		expect(result.firstKeptEntryId).toBeTruthy();
+		expect(result.usage?.totalTokens).toBeGreaterThan(0);
 		expect(result.details).toBeDefined();
 	});
 });

packages/agent/test/harness/session.test.ts

@@ -86,6 +86,49 @@ async function runSessionSuite(
 			expect(context.messages[1]?.role).toBe("branchSummary");
 		});
 
+		it("persists compaction usage", async () => {
+			const session = new Session(await createStorage());
+			const firstKeptEntryId = await session.appendMessage(createUserMessage("one"));
+			const usage = {
+				input: 1,
+				output: 2,
+				cacheRead: 3,
+				cacheWrite: 4,
+				totalTokens: 10,
+				cost: { input: 0.1, output: 0.2, cacheRead: 0.3, cacheWrite: 0.4, total: 1 },
+			};
+
+			const compactionId = await session.appendCompaction(
+				"summary",
+				firstKeptEntryId,
+				1234,
+				undefined,
+				false,
+				usage,
+			);
+
+			const compactionEntry = await session.getEntry(compactionId);
+			expect(compactionEntry?.type === "compaction" ? compactionEntry.usage : undefined).toEqual(usage);
+		});
+
+		it("persists branch summary usage", async () => {
+			const session = new Session(await createStorage());
+			const user1 = await session.appendMessage(createUserMessage("one"));
+			const usage = {
+				input: 1,
+				output: 2,
+				cacheRead: 3,
+				cacheWrite: 4,
+				totalTokens: 10,
+				cost: { input: 0.1, output: 0.2, cacheRead: 0.3, cacheWrite: 0.4, total: 1 },
+			};
+
+			const summaryId = await session.moveTo(user1, { summary: "summary text", usage });
+
+			const summaryEntry = await session.getEntry(summaryId!);
+			expect(summaryEntry?.type === "branch_summary" ? summaryEntry.usage : undefined).toEqual(usage);
+		});
+
 		it("supports custom message entries in context", async () => {
 			const session = new Session(await createStorage());
 			await session.appendMessage(createUserMessage("one"));

packages/agent/test/utils/calculate.ts

@@ -1,3 +1,4 @@
+import type { Usage } from "@earendil-works/pi-ai";
 import { type Static, Type } from "typebox";
 import type { AgentTool, AgentToolResult } from "../../src/types.ts";
 
@@ -30,3 +31,10 @@ export const calculateTool: AgentTool<typeof calculateSchema, undefined> = {
 		return calculate(args.expression);
 	},
 };
+
+export function createCalculateToolWithUsage(usage: Usage): AgentTool<typeof calculateSchema, undefined> {
+	return {
+		...calculateTool,
+		execute: async (_toolCallId: string, args: CalculateParams) => ({ ...calculate(args.expression), usage }),
+	};
+}