add usage info to branch summary, compaction and tool result entries (#6671)
* add usage info to branch summary entries * add usage to compaction entries * allow custom tools to report llm usage in tool results * allow observing and patching usage in tool_result hooks * agent-harness: save usage in entries for compaction, branch summaries and tool results
This commit is contained in:
@@ -239,6 +239,23 @@ describe("agentLoop with AgentMessage", () => {
|
||||
it("should handle tool calls and results", async () => {
|
||||
const toolSchema = Type.Object({ value: Type.String() });
|
||||
const executed: string[] = [];
|
||||
const toolUsage = {
|
||||
input: 1,
|
||||
output: 2,
|
||||
cacheRead: 3,
|
||||
cacheWrite: 4,
|
||||
totalTokens: 10,
|
||||
cost: { input: 0.1, output: 0.2, cacheRead: 0.3, cacheWrite: 0.4, total: 1 },
|
||||
};
|
||||
const patchedToolUsage = {
|
||||
input: 5,
|
||||
output: 6,
|
||||
cacheRead: 7,
|
||||
cacheWrite: 8,
|
||||
totalTokens: 26,
|
||||
cost: { input: 0.5, output: 0.6, cacheRead: 0.7, cacheWrite: 0.8, total: 2.6 },
|
||||
};
|
||||
let observedToolUsage: typeof toolUsage | undefined;
|
||||
const tool: AgentTool<typeof toolSchema, { value: string }> = {
|
||||
name: "echo",
|
||||
label: "Echo",
|
||||
@@ -249,6 +266,7 @@ describe("agentLoop with AgentMessage", () => {
|
||||
return {
|
||||
content: [{ type: "text", text: `echoed: ${params.value}` }],
|
||||
details: { value: params.value },
|
||||
usage: toolUsage,
|
||||
};
|
||||
},
|
||||
};
|
||||
@@ -264,6 +282,10 @@ describe("agentLoop with AgentMessage", () => {
|
||||
const config: AgentLoopConfig = {
|
||||
model: createModel(),
|
||||
convertToLlm: identityConverter,
|
||||
afterToolCall: async ({ result }) => {
|
||||
observedToolUsage = result.usage;
|
||||
return { usage: patchedToolUsage };
|
||||
},
|
||||
};
|
||||
|
||||
let callIndex = 0;
|
||||
@@ -305,6 +327,10 @@ describe("agentLoop with AgentMessage", () => {
|
||||
if (toolEnd?.type === "tool_execution_end") {
|
||||
expect(toolEnd.isError).toBe(false);
|
||||
}
|
||||
expect(observedToolUsage).toEqual(toolUsage);
|
||||
const messages = await stream.result();
|
||||
const toolResult = messages.find((message) => message.role === "toolResult");
|
||||
expect(toolResult?.role === "toolResult" ? toolResult.usage : undefined).toEqual(patchedToolUsage);
|
||||
});
|
||||
|
||||
it("should not execute tool calls from a length-truncated assistant message", async () => {
|
||||
|
||||
Reference in New Issue
Block a user