add usage info to branch summary, compaction and tool result entries (#6671)
* add usage info to branch summary entries * add usage to compaction entries * allow custom tools to report llm usage in tool results * allow observing and patching usage in tool_result hooks * agent-harness: save usage in entries for compaction, branch summaries and tool results
This commit is contained in:
@@ -1,5 +1,5 @@
|
||||
import { Agent } from "@earendil-works/pi-agent-core";
|
||||
import { type AssistantMessage, getModel, type Usage } from "@earendil-works/pi-ai/compat";
|
||||
import { type AssistantMessage, getModel, type ToolResultMessage, type Usage } from "@earendil-works/pi-ai/compat";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { AgentSession } from "../src/core/agent-session.ts";
|
||||
import { AuthStorage } from "../src/core/auth-storage.ts";
|
||||
@@ -48,6 +48,18 @@ function createUserMessage(text: string, timestamp: number) {
|
||||
};
|
||||
}
|
||||
|
||||
function createToolResultMessage(usage: Usage): ToolResultMessage {
|
||||
return {
|
||||
role: "toolResult",
|
||||
toolCallId: "tool-call-1",
|
||||
toolName: "test_tool",
|
||||
content: [{ type: "text", text: "tool result" }],
|
||||
usage,
|
||||
isError: false,
|
||||
timestamp: 1,
|
||||
};
|
||||
}
|
||||
|
||||
async function createSession() {
|
||||
const settingsManager = SettingsManager.inMemory();
|
||||
const sessionManager = SessionManager.inMemory();
|
||||
@@ -143,6 +155,75 @@ describe("AgentSession.getSessionStats", () => {
|
||||
}
|
||||
});
|
||||
|
||||
it("includes branch summary usage in session totals", async () => {
|
||||
const { session, sessionManager } = await createSession();
|
||||
|
||||
try {
|
||||
sessionManager.branchWithSummary(null, "summary", undefined, false, {
|
||||
input: 10,
|
||||
output: 20,
|
||||
cacheRead: 30,
|
||||
cacheWrite: 40,
|
||||
totalTokens: 100,
|
||||
cost: { input: 0.1, output: 0.2, cacheRead: 0.3, cacheWrite: 0.4, total: 1 },
|
||||
});
|
||||
syncAgentMessages(session, sessionManager);
|
||||
|
||||
const stats = session.getSessionStats();
|
||||
expect(stats.tokens).toEqual({ input: 10, output: 20, cacheRead: 30, cacheWrite: 40, total: 100 });
|
||||
expect(stats.cost).toBe(1);
|
||||
} finally {
|
||||
session.dispose();
|
||||
}
|
||||
});
|
||||
|
||||
it("includes compaction usage in session totals", async () => {
|
||||
const { session, sessionManager } = await createSession();
|
||||
|
||||
try {
|
||||
const firstKeptEntryId = sessionManager.appendMessage(createUserMessage("hello", 1));
|
||||
sessionManager.appendCompaction("summary", firstKeptEntryId, 100, undefined, false, {
|
||||
input: 10,
|
||||
output: 20,
|
||||
cacheRead: 30,
|
||||
cacheWrite: 40,
|
||||
totalTokens: 100,
|
||||
cost: { input: 0.1, output: 0.2, cacheRead: 0.3, cacheWrite: 0.4, total: 1 },
|
||||
});
|
||||
syncAgentMessages(session, sessionManager);
|
||||
|
||||
const stats = session.getSessionStats();
|
||||
expect(stats.tokens).toEqual({ input: 10, output: 20, cacheRead: 30, cacheWrite: 40, total: 100 });
|
||||
expect(stats.cost).toBe(1);
|
||||
} finally {
|
||||
session.dispose();
|
||||
}
|
||||
});
|
||||
|
||||
it("includes tool result usage in session totals", async () => {
|
||||
const { session, sessionManager } = await createSession();
|
||||
|
||||
try {
|
||||
sessionManager.appendMessage(
|
||||
createToolResultMessage({
|
||||
input: 10,
|
||||
output: 20,
|
||||
cacheRead: 30,
|
||||
cacheWrite: 40,
|
||||
totalTokens: 100,
|
||||
cost: { input: 0.1, output: 0.2, cacheRead: 0.3, cacheWrite: 0.4, total: 1 },
|
||||
}),
|
||||
);
|
||||
syncAgentMessages(session, sessionManager);
|
||||
|
||||
const stats = session.getSessionStats();
|
||||
expect(stats.tokens).toEqual({ input: 10, output: 20, cacheRead: 30, cacheWrite: 40, total: 100 });
|
||||
expect(stats.cost).toBe(1);
|
||||
} finally {
|
||||
session.dispose();
|
||||
}
|
||||
});
|
||||
|
||||
it("ignores zero-usage messages when checking for post-compaction context usage", async () => {
|
||||
const { session, sessionManager } = await createSession();
|
||||
|
||||
|
||||
Reference in New Issue
Block a user