add usage info to branch summary, compaction and tool result entries (#6671)
* add usage info to branch summary entries * add usage to compaction entries * allow custom tools to report llm usage in tool results * allow observing and patching usage in tool_result hooks * agent-harness: save usage in entries for compaction, branch summaries and tool results
This commit is contained in:
@@ -32,6 +32,7 @@ import type {
|
||||
Model,
|
||||
ProviderHeaders,
|
||||
TextContent,
|
||||
Usage,
|
||||
} from "@earendil-works/pi-ai/compat";
|
||||
import {
|
||||
clampThinkingLevel,
|
||||
@@ -104,6 +105,7 @@ import { type BuildSystemPromptOptions, buildSystemPrompt } from "./system-promp
|
||||
import { type BashOperations, createLocalBashOperations } from "./tools/bash.ts";
|
||||
import { createAllToolDefinitions } from "./tools/index.ts";
|
||||
import { createToolDefinitionFromAgentTool } from "./tools/tool-definition-wrapper.ts";
|
||||
import { addUsageToTotals, createUsageTotals } from "./usage-totals.ts";
|
||||
|
||||
// ============================================================================
|
||||
// Skill Block Parsing
|
||||
@@ -482,6 +484,7 @@ export class AgentSession {
|
||||
content: result.content,
|
||||
details: result.details,
|
||||
isError,
|
||||
usage: result.usage,
|
||||
});
|
||||
|
||||
if (!hookResult) {
|
||||
@@ -492,6 +495,7 @@ export class AgentSession {
|
||||
content: hookResult.content,
|
||||
details: hookResult.details,
|
||||
isError: hookResult.isError ?? isError,
|
||||
usage: hookResult.usage,
|
||||
};
|
||||
};
|
||||
}
|
||||
@@ -1812,6 +1816,7 @@ export class AgentSession {
|
||||
let summary: string;
|
||||
let firstKeptEntryId: string;
|
||||
let tokensBefore: number;
|
||||
let usage: Usage | undefined;
|
||||
let details: unknown;
|
||||
|
||||
if (extensionCompaction) {
|
||||
@@ -1819,6 +1824,7 @@ export class AgentSession {
|
||||
summary = extensionCompaction.summary;
|
||||
firstKeptEntryId = extensionCompaction.firstKeptEntryId;
|
||||
tokensBefore = extensionCompaction.tokensBefore;
|
||||
usage = extensionCompaction.usage;
|
||||
details = extensionCompaction.details;
|
||||
} else {
|
||||
// Generate compaction result
|
||||
@@ -1836,6 +1842,7 @@ export class AgentSession {
|
||||
summary = result.summary;
|
||||
firstKeptEntryId = result.firstKeptEntryId;
|
||||
tokensBefore = result.tokensBefore;
|
||||
usage = result.usage;
|
||||
details = result.details;
|
||||
}
|
||||
|
||||
@@ -1843,7 +1850,7 @@ export class AgentSession {
|
||||
throw new Error("Compaction cancelled");
|
||||
}
|
||||
|
||||
this.sessionManager.appendCompaction(summary, firstKeptEntryId, tokensBefore, details, fromExtension);
|
||||
this.sessionManager.appendCompaction(summary, firstKeptEntryId, tokensBefore, details, fromExtension, usage);
|
||||
const newEntries = this.sessionManager.getEntries();
|
||||
const sessionContext = this.sessionManager.buildSessionContext();
|
||||
this.agent.state.messages = sessionContext.messages;
|
||||
@@ -1869,6 +1876,7 @@ export class AgentSession {
|
||||
firstKeptEntryId,
|
||||
tokensBefore,
|
||||
estimatedTokensAfter,
|
||||
usage,
|
||||
details,
|
||||
};
|
||||
this._emit({
|
||||
@@ -2084,6 +2092,7 @@ export class AgentSession {
|
||||
let summary: string;
|
||||
let firstKeptEntryId: string;
|
||||
let tokensBefore: number;
|
||||
let usage: Usage | undefined;
|
||||
let details: unknown;
|
||||
|
||||
if (extensionCompaction) {
|
||||
@@ -2091,6 +2100,7 @@ export class AgentSession {
|
||||
summary = extensionCompaction.summary;
|
||||
firstKeptEntryId = extensionCompaction.firstKeptEntryId;
|
||||
tokensBefore = extensionCompaction.tokensBefore;
|
||||
usage = extensionCompaction.usage;
|
||||
details = extensionCompaction.details;
|
||||
} else {
|
||||
// Generate compaction result
|
||||
@@ -2108,6 +2118,7 @@ export class AgentSession {
|
||||
summary = compactResult.summary;
|
||||
firstKeptEntryId = compactResult.firstKeptEntryId;
|
||||
tokensBefore = compactResult.tokensBefore;
|
||||
usage = compactResult.usage;
|
||||
details = compactResult.details;
|
||||
}
|
||||
|
||||
@@ -2122,7 +2133,7 @@ export class AgentSession {
|
||||
return false;
|
||||
}
|
||||
|
||||
this.sessionManager.appendCompaction(summary, firstKeptEntryId, tokensBefore, details, fromExtension);
|
||||
this.sessionManager.appendCompaction(summary, firstKeptEntryId, tokensBefore, details, fromExtension, usage);
|
||||
const newEntries = this.sessionManager.getEntries();
|
||||
const sessionContext = this.sessionManager.buildSessionContext();
|
||||
this.agent.state.messages = sessionContext.messages;
|
||||
@@ -2148,6 +2159,7 @@ export class AgentSession {
|
||||
firstKeptEntryId,
|
||||
tokensBefore,
|
||||
estimatedTokensAfter,
|
||||
usage,
|
||||
details,
|
||||
};
|
||||
this._emit({ type: "compaction_end", reason, result, aborted: false, willRetry });
|
||||
@@ -2872,7 +2884,7 @@ export class AgentSession {
|
||||
this._branchSummaryAbortController = new AbortController();
|
||||
|
||||
try {
|
||||
let extensionSummary: { summary: string; details?: unknown } | undefined;
|
||||
let extensionSummary: { summary: string; details?: unknown; usage?: Usage } | undefined;
|
||||
let fromExtension = false;
|
||||
|
||||
// Emit session_before_tree event
|
||||
@@ -2907,6 +2919,7 @@ export class AgentSession {
|
||||
// Run default summarizer if needed
|
||||
let summaryText: string | undefined;
|
||||
let summaryDetails: unknown;
|
||||
let summaryUsage: Usage | undefined;
|
||||
if (options.summarize && entriesToSummarize.length > 0 && !extensionSummary) {
|
||||
const model = this.model!;
|
||||
const { apiKey, headers, env } = await this._getSummarizationRequestAuth(model);
|
||||
@@ -2929,6 +2942,7 @@ export class AgentSession {
|
||||
throw new Error(result.error);
|
||||
}
|
||||
summaryText = result.summary;
|
||||
summaryUsage = result.usage;
|
||||
summaryDetails = {
|
||||
readFiles: result.readFiles || [],
|
||||
modifiedFiles: result.modifiedFiles || [],
|
||||
@@ -2936,6 +2950,7 @@ export class AgentSession {
|
||||
} else if (extensionSummary) {
|
||||
summaryText = extensionSummary.summary;
|
||||
summaryDetails = extensionSummary.details;
|
||||
summaryUsage = extensionSummary.usage;
|
||||
}
|
||||
|
||||
// Determine the new leaf position based on target type
|
||||
@@ -2965,6 +2980,7 @@ export class AgentSession {
|
||||
summaryText,
|
||||
summaryDetails,
|
||||
fromExtension,
|
||||
summaryUsage,
|
||||
);
|
||||
summaryEntry = this.sessionManager.getEntry(summaryId) as BranchSummaryEntry;
|
||||
|
||||
@@ -3037,13 +3053,12 @@ export class AgentSession {
|
||||
let toolResults = 0;
|
||||
let totalMessages = 0;
|
||||
let toolCalls = 0;
|
||||
let totalInput = 0;
|
||||
let totalOutput = 0;
|
||||
let totalCacheRead = 0;
|
||||
let totalCacheWrite = 0;
|
||||
let totalCost = 0;
|
||||
const usageTotals = createUsageTotals();
|
||||
|
||||
for (const entry of this.sessionManager.getEntries()) {
|
||||
if ((entry.type === "branch_summary" || entry.type === "compaction") && entry.usage) {
|
||||
addUsageToTotals(usageTotals, entry.usage);
|
||||
}
|
||||
if (entry.type !== "message") continue;
|
||||
totalMessages++;
|
||||
const message = entry.message;
|
||||
@@ -3051,18 +3066,16 @@ export class AgentSession {
|
||||
userMessages++;
|
||||
} else if (message.role === "toolResult") {
|
||||
toolResults++;
|
||||
if (message.usage) {
|
||||
addUsageToTotals(usageTotals, message.usage);
|
||||
}
|
||||
} else if (message.role === "assistant") {
|
||||
assistantMessages++;
|
||||
const assistantMsg = message as AssistantMessage;
|
||||
if (Array.isArray(assistantMsg.content)) {
|
||||
toolCalls += assistantMsg.content.filter((c) => c.type === "toolCall").length;
|
||||
}
|
||||
const usage = assistantMsg.usage;
|
||||
totalInput += usage.input;
|
||||
totalOutput += usage.output;
|
||||
totalCacheRead += usage.cacheRead;
|
||||
totalCacheWrite += usage.cacheWrite;
|
||||
totalCost += usage.cost.total;
|
||||
addUsageToTotals(usageTotals, assistantMsg.usage);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -3075,13 +3088,13 @@ export class AgentSession {
|
||||
toolResults,
|
||||
totalMessages,
|
||||
tokens: {
|
||||
input: totalInput,
|
||||
output: totalOutput,
|
||||
cacheRead: totalCacheRead,
|
||||
cacheWrite: totalCacheWrite,
|
||||
total: totalInput + totalOutput + totalCacheRead + totalCacheWrite,
|
||||
input: usageTotals.input,
|
||||
output: usageTotals.output,
|
||||
cacheRead: usageTotals.cacheRead,
|
||||
cacheWrite: usageTotals.cacheWrite,
|
||||
total: usageTotals.input + usageTotals.output + usageTotals.cacheRead + usageTotals.cacheWrite,
|
||||
},
|
||||
cost: totalCost,
|
||||
cost: usageTotals.cost,
|
||||
contextUsage: this.getContextUsage(),
|
||||
};
|
||||
}
|
||||
|
||||
@@ -7,7 +7,7 @@
|
||||
|
||||
import type { AgentMessage, StreamFn } from "@earendil-works/pi-agent-core";
|
||||
import { contentText } from "@earendil-works/pi-ai";
|
||||
import type { Model, SimpleStreamOptions } from "@earendil-works/pi-ai/compat";
|
||||
import type { Model, SimpleStreamOptions, Usage } from "@earendil-works/pi-ai/compat";
|
||||
import { completeSimple } from "@earendil-works/pi-ai/compat";
|
||||
import {
|
||||
convertToLlm,
|
||||
@@ -33,6 +33,7 @@ import {
|
||||
|
||||
export interface BranchSummaryResult {
|
||||
summary?: string;
|
||||
usage?: Usage;
|
||||
readFiles?: string[];
|
||||
modifiedFiles?: string[];
|
||||
aborted?: boolean;
|
||||
@@ -363,6 +364,7 @@ export async function generateBranchSummary(
|
||||
|
||||
return {
|
||||
summary: summary || "No summary generated",
|
||||
usage: response.usage,
|
||||
readFiles,
|
||||
modifiedFiles,
|
||||
};
|
||||
|
||||
@@ -90,10 +90,35 @@ export interface CompactionResult<T = unknown> {
|
||||
firstKeptEntryId: string;
|
||||
tokensBefore: number;
|
||||
estimatedTokensAfter?: number;
|
||||
/** Usage from the LLM call(s) that generated this summary, if available */
|
||||
usage?: Usage;
|
||||
/** Extension-specific data (e.g., ArtifactIndex, version markers for structured compaction) */
|
||||
details?: T;
|
||||
}
|
||||
|
||||
function combineUsage(first: Usage, second: Usage): Usage {
|
||||
return {
|
||||
input: first.input + second.input,
|
||||
output: first.output + second.output,
|
||||
cacheRead: first.cacheRead + second.cacheRead,
|
||||
cacheWrite: first.cacheWrite + second.cacheWrite,
|
||||
...(first.cacheWrite1h !== undefined || second.cacheWrite1h !== undefined
|
||||
? { cacheWrite1h: (first.cacheWrite1h ?? 0) + (second.cacheWrite1h ?? 0) }
|
||||
: {}),
|
||||
...(first.reasoning !== undefined || second.reasoning !== undefined
|
||||
? { reasoning: (first.reasoning ?? 0) + (second.reasoning ?? 0) }
|
||||
: {}),
|
||||
totalTokens: first.totalTokens + second.totalTokens,
|
||||
cost: {
|
||||
input: first.cost.input + second.cost.input,
|
||||
output: first.cost.output + second.cost.output,
|
||||
cacheRead: first.cost.cacheRead + second.cost.cacheRead,
|
||||
cacheWrite: first.cost.cacheWrite + second.cost.cacheWrite,
|
||||
total: first.cost.total + second.cost.total,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Types
|
||||
// ============================================================================
|
||||
@@ -556,7 +581,7 @@ export async function generateSummary(
|
||||
thinkingLevel?: ThinkingLevel,
|
||||
streamFn?: StreamFn,
|
||||
env?: Record<string, string>,
|
||||
): Promise<string> {
|
||||
): Promise<{ text: string; usage: Usage }> {
|
||||
const maxTokens = Math.min(
|
||||
Math.floor(0.8 * reserveTokens),
|
||||
model.maxTokens > 0 ? model.maxTokens : Number.POSITIVE_INFINITY,
|
||||
@@ -603,7 +628,7 @@ export async function generateSummary(
|
||||
|
||||
const textContent = contentText(response.content);
|
||||
|
||||
return textContent;
|
||||
return { text: textContent, usage: response.usage };
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
@@ -759,24 +784,28 @@ export async function compact(
|
||||
|
||||
// Generate summaries and merge into one
|
||||
let summary: string;
|
||||
let summaryUsage: Usage;
|
||||
|
||||
if (isSplitTurn && turnPrefixMessages.length > 0) {
|
||||
const historyResult =
|
||||
messagesToSummarize.length > 0
|
||||
? await generateSummary(
|
||||
messagesToSummarize,
|
||||
model,
|
||||
settings.reserveTokens,
|
||||
apiKey,
|
||||
headers,
|
||||
signal,
|
||||
customInstructions,
|
||||
previousSummary,
|
||||
thinkingLevel,
|
||||
streamFn,
|
||||
env,
|
||||
)
|
||||
: "No prior history.";
|
||||
let historyText = "No prior history.";
|
||||
let historyUsage: Usage | undefined;
|
||||
if (messagesToSummarize.length > 0) {
|
||||
const historyResult = await generateSummary(
|
||||
messagesToSummarize,
|
||||
model,
|
||||
settings.reserveTokens,
|
||||
apiKey,
|
||||
headers,
|
||||
signal,
|
||||
customInstructions,
|
||||
previousSummary,
|
||||
thinkingLevel,
|
||||
streamFn,
|
||||
env,
|
||||
);
|
||||
historyText = historyResult.text;
|
||||
historyUsage = historyResult.usage;
|
||||
}
|
||||
const turnPrefixResult = await generateTurnPrefixSummary(
|
||||
turnPrefixMessages,
|
||||
model,
|
||||
@@ -789,10 +818,11 @@ export async function compact(
|
||||
streamFn,
|
||||
);
|
||||
// Merge into single summary
|
||||
summary = `${historyResult}\n\n---\n\n**Turn Context (split turn):**\n\n${turnPrefixResult}`;
|
||||
summary = `${historyText}\n\n---\n\n**Turn Context (split turn):**\n\n${turnPrefixResult.text}`;
|
||||
summaryUsage = historyUsage ? combineUsage(historyUsage, turnPrefixResult.usage) : turnPrefixResult.usage;
|
||||
} else {
|
||||
// Just generate history summary
|
||||
summary = await generateSummary(
|
||||
const result = await generateSummary(
|
||||
messagesToSummarize,
|
||||
model,
|
||||
settings.reserveTokens,
|
||||
@@ -805,6 +835,8 @@ export async function compact(
|
||||
streamFn,
|
||||
env,
|
||||
);
|
||||
summary = result.text;
|
||||
summaryUsage = result.usage;
|
||||
}
|
||||
|
||||
// Compute file lists and append to summary
|
||||
@@ -819,6 +851,7 @@ export async function compact(
|
||||
summary,
|
||||
firstKeptEntryId,
|
||||
tokensBefore,
|
||||
usage: summaryUsage,
|
||||
details: { readFiles, modifiedFiles } as CompactionDetails,
|
||||
};
|
||||
}
|
||||
@@ -836,7 +869,7 @@ async function generateTurnPrefixSummary(
|
||||
signal?: AbortSignal,
|
||||
thinkingLevel?: ThinkingLevel,
|
||||
streamFn?: StreamFn,
|
||||
): Promise<string> {
|
||||
): Promise<{ text: string; usage: Usage }> {
|
||||
const maxTokens = Math.min(
|
||||
Math.floor(0.5 * reserveTokens),
|
||||
model.maxTokens > 0 ? model.maxTokens : Number.POSITIVE_INFINITY,
|
||||
@@ -863,5 +896,8 @@ async function generateTurnPrefixSummary(
|
||||
throw new Error(`Turn prefix summarization failed: ${response.errorMessage || "Unknown error"}`);
|
||||
}
|
||||
|
||||
return contentText(response.content);
|
||||
return {
|
||||
text: contentText(response.content),
|
||||
usage: response.usage,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -883,6 +883,10 @@ export class ExtensionRunner {
|
||||
currentEvent.isError = handlerResult.isError;
|
||||
modified = true;
|
||||
}
|
||||
if (handlerResult.usage !== undefined) {
|
||||
currentEvent.usage = handlerResult.usage;
|
||||
modified = true;
|
||||
}
|
||||
} catch (err) {
|
||||
const message = err instanceof Error ? err.message : String(err);
|
||||
const stack = err instanceof Error ? err.stack : undefined;
|
||||
@@ -904,6 +908,7 @@ export class ExtensionRunner {
|
||||
content: currentEvent.content,
|
||||
details: currentEvent.details,
|
||||
isError: currentEvent.isError,
|
||||
usage: currentEvent.usage,
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
@@ -30,6 +30,7 @@ import type {
|
||||
SimpleStreamOptions,
|
||||
TextContent,
|
||||
ToolResultMessage,
|
||||
Usage,
|
||||
} from "@earendil-works/pi-ai";
|
||||
import type {
|
||||
AutocompleteItem,
|
||||
@@ -905,6 +906,8 @@ interface ToolResultEventBase {
|
||||
input: Record<string, unknown>;
|
||||
content: (TextContent | ImageContent)[];
|
||||
isError: boolean;
|
||||
/** Usage from the tool execution itself, if available. */
|
||||
usage?: Usage;
|
||||
}
|
||||
|
||||
export interface BashToolResultEvent extends ToolResultEventBase {
|
||||
@@ -1072,6 +1075,7 @@ export interface ToolResultEventResult {
|
||||
content?: (TextContent | ImageContent)[];
|
||||
details?: unknown;
|
||||
isError?: boolean;
|
||||
usage?: Usage;
|
||||
}
|
||||
|
||||
export interface MessageEndEventResult {
|
||||
@@ -1104,6 +1108,7 @@ export interface SessionBeforeTreeResult {
|
||||
summary?: {
|
||||
summary: string;
|
||||
details?: unknown;
|
||||
usage?: Usage;
|
||||
};
|
||||
/** Override custom instructions for summarization */
|
||||
customInstructions?: string;
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import type { AgentMessage } from "@earendil-works/pi-agent-core";
|
||||
import { type ImageContent, type Message, type TextContent, uuidv7 } from "@earendil-works/pi-ai";
|
||||
import { type ImageContent, type Message, type TextContent, type Usage, uuidv7 } from "@earendil-works/pi-ai";
|
||||
import { randomUUID } from "crypto";
|
||||
import {
|
||||
appendFileSync,
|
||||
@@ -73,6 +73,8 @@ export interface CompactionEntry<T = unknown> extends SessionEntryBase {
|
||||
tokensBefore: number;
|
||||
/** Extension-specific data (e.g., ArtifactIndex, version markers for structured compaction) */
|
||||
details?: T;
|
||||
/** Usage from the LLM call(s) that generated this summary, if available */
|
||||
usage?: Usage;
|
||||
/** True if generated by an extension, undefined/false if pi-generated (backward compatible) */
|
||||
fromHook?: boolean;
|
||||
}
|
||||
@@ -83,6 +85,8 @@ export interface BranchSummaryEntry<T = unknown> extends SessionEntryBase {
|
||||
summary: string;
|
||||
/** Extension-specific data (not sent to LLM) */
|
||||
details?: T;
|
||||
/** Usage from the LLM call that generated this summary, if available */
|
||||
usage?: Usage;
|
||||
/** True if generated by an extension, false if pi-generated */
|
||||
fromHook?: boolean;
|
||||
}
|
||||
@@ -1096,6 +1100,7 @@ export class SessionManager {
|
||||
tokensBefore: number,
|
||||
details?: T,
|
||||
fromHook?: boolean,
|
||||
usage?: Usage,
|
||||
): string {
|
||||
const entry: CompactionEntry<T> = {
|
||||
type: "compaction",
|
||||
@@ -1106,6 +1111,7 @@ export class SessionManager {
|
||||
firstKeptEntryId,
|
||||
tokensBefore,
|
||||
details,
|
||||
usage,
|
||||
fromHook,
|
||||
};
|
||||
this._appendEntry(entry);
|
||||
@@ -1372,7 +1378,13 @@ export class SessionManager {
|
||||
* Same as branch(), but also appends a branch_summary entry that captures
|
||||
* context from the abandoned conversation path.
|
||||
*/
|
||||
branchWithSummary(branchFromId: string | null, summary: string, details?: unknown, fromHook?: boolean): string {
|
||||
branchWithSummary(
|
||||
branchFromId: string | null,
|
||||
summary: string,
|
||||
details?: unknown,
|
||||
fromHook?: boolean,
|
||||
usage?: Usage,
|
||||
): string {
|
||||
if (branchFromId !== null && !this.byId.has(branchFromId)) {
|
||||
throw new Error(`Entry ${branchFromId} not found`);
|
||||
}
|
||||
@@ -1385,6 +1397,7 @@ export class SessionManager {
|
||||
fromId: branchFromId ?? "root",
|
||||
summary,
|
||||
details,
|
||||
usage,
|
||||
fromHook,
|
||||
};
|
||||
this._appendEntry(entry);
|
||||
|
||||
@@ -0,0 +1,27 @@
|
||||
import type { Usage } from "@earendil-works/pi-ai/compat";
|
||||
|
||||
export interface UsageTotals {
|
||||
input: number;
|
||||
output: number;
|
||||
cacheRead: number;
|
||||
cacheWrite: number;
|
||||
cost: number;
|
||||
}
|
||||
|
||||
export function createUsageTotals(): UsageTotals {
|
||||
return {
|
||||
input: 0,
|
||||
output: 0,
|
||||
cacheRead: 0,
|
||||
cacheWrite: 0,
|
||||
cost: 0,
|
||||
};
|
||||
}
|
||||
|
||||
export function addUsageToTotals(totals: UsageTotals, usage: Usage): void {
|
||||
totals.input += usage.input;
|
||||
totals.output += usage.output;
|
||||
totals.cacheRead += usage.cacheRead;
|
||||
totals.cacheWrite += usage.cacheWrite;
|
||||
totals.cost += usage.cost.total;
|
||||
}
|
||||
@@ -3,6 +3,7 @@ import { type Component, truncateToWidth, visibleWidth } from "@earendil-works/p
|
||||
import type { AgentSession } from "../../../core/agent-session.ts";
|
||||
import { areExperimentalFeaturesEnabled } from "../../../core/experimental.ts";
|
||||
import type { ReadonlyFooterDataProvider } from "../../../core/footer-data-provider.ts";
|
||||
import { addUsageToTotals, createUsageTotals } from "../../../core/usage-totals.ts";
|
||||
import { theme } from "../theme/theme.ts";
|
||||
|
||||
/**
|
||||
@@ -84,25 +85,21 @@ export class FooterComponent implements Component {
|
||||
const state = this.session.state;
|
||||
|
||||
// Calculate cumulative usage from ALL session entries (not just post-compaction messages)
|
||||
let totalInput = 0;
|
||||
let totalOutput = 0;
|
||||
let totalCacheRead = 0;
|
||||
let totalCacheWrite = 0;
|
||||
let totalCost = 0;
|
||||
const usageTotals = createUsageTotals();
|
||||
let latestCacheHitRate: number | undefined;
|
||||
|
||||
for (const entry of this.session.sessionManager.getEntries()) {
|
||||
if (entry.type === "message" && entry.message.role === "assistant") {
|
||||
totalInput += entry.message.usage.input;
|
||||
totalOutput += entry.message.usage.output;
|
||||
totalCacheRead += entry.message.usage.cacheRead;
|
||||
totalCacheWrite += entry.message.usage.cacheWrite;
|
||||
totalCost += entry.message.usage.cost.total;
|
||||
addUsageToTotals(usageTotals, entry.message.usage);
|
||||
|
||||
const latestPromptTokens =
|
||||
entry.message.usage.input + entry.message.usage.cacheRead + entry.message.usage.cacheWrite;
|
||||
latestCacheHitRate =
|
||||
latestPromptTokens > 0 ? (entry.message.usage.cacheRead / latestPromptTokens) * 100 : undefined;
|
||||
} else if (entry.type === "message" && entry.message.role === "toolResult" && entry.message.usage) {
|
||||
addUsageToTotals(usageTotals, entry.message.usage);
|
||||
} else if ((entry.type === "branch_summary" || entry.type === "compaction") && entry.usage) {
|
||||
addUsageToTotals(usageTotals, entry.usage);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -130,19 +127,20 @@ export class FooterComponent implements Component {
|
||||
|
||||
// Build stats line
|
||||
const statsParts = [];
|
||||
if (totalInput) statsParts.push(`↑${formatTokens(totalInput)}`);
|
||||
if (totalOutput) statsParts.push(`↓${formatTokens(totalOutput)}`);
|
||||
if (totalCacheRead) statsParts.push(`R${formatTokens(totalCacheRead)}`);
|
||||
if (totalCacheWrite) statsParts.push(`W${formatTokens(totalCacheWrite)}`);
|
||||
if ((totalCacheRead > 0 || totalCacheWrite > 0) && latestCacheHitRate !== undefined) {
|
||||
if (usageTotals.input) statsParts.push(`↑${formatTokens(usageTotals.input)}`);
|
||||
if (usageTotals.output) statsParts.push(`↓${formatTokens(usageTotals.output)}`);
|
||||
if (usageTotals.cacheRead) statsParts.push(`R${formatTokens(usageTotals.cacheRead)}`);
|
||||
if (usageTotals.cacheWrite) statsParts.push(`W${formatTokens(usageTotals.cacheWrite)}`);
|
||||
if ((usageTotals.cacheRead > 0 || usageTotals.cacheWrite > 0) && latestCacheHitRate !== undefined) {
|
||||
statsParts.push(`CH${latestCacheHitRate.toFixed(1)}%`);
|
||||
}
|
||||
|
||||
// Kimi Coding is subscription-backed despite using API-key authentication.
|
||||
const usingSubscription = state.model
|
||||
? state.model.provider === "kimi-coding" || this.session.modelRuntime.isUsingOAuth(state.model.provider)
|
||||
: false;
|
||||
if (totalCost || usingSubscription) {
|
||||
const costStr = `$${totalCost.toFixed(3)}${usingSubscription ? " (sub)" : ""}`;
|
||||
if (usageTotals.cost || usingSubscription) {
|
||||
const costStr = `$${usageTotals.cost.toFixed(3)}${usingSubscription ? " (sub)" : ""}`;
|
||||
statsParts.push(costStr);
|
||||
}
|
||||
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import { Agent } from "@earendil-works/pi-agent-core";
|
||||
import { type AssistantMessage, getModel, type Usage } from "@earendil-works/pi-ai/compat";
|
||||
import { type AssistantMessage, getModel, type ToolResultMessage, type Usage } from "@earendil-works/pi-ai/compat";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { AgentSession } from "../src/core/agent-session.ts";
|
||||
import { AuthStorage } from "../src/core/auth-storage.ts";
|
||||
@@ -48,6 +48,18 @@ function createUserMessage(text: string, timestamp: number) {
|
||||
};
|
||||
}
|
||||
|
||||
function createToolResultMessage(usage: Usage): ToolResultMessage {
|
||||
return {
|
||||
role: "toolResult",
|
||||
toolCallId: "tool-call-1",
|
||||
toolName: "test_tool",
|
||||
content: [{ type: "text", text: "tool result" }],
|
||||
usage,
|
||||
isError: false,
|
||||
timestamp: 1,
|
||||
};
|
||||
}
|
||||
|
||||
async function createSession() {
|
||||
const settingsManager = SettingsManager.inMemory();
|
||||
const sessionManager = SessionManager.inMemory();
|
||||
@@ -143,6 +155,75 @@ describe("AgentSession.getSessionStats", () => {
|
||||
}
|
||||
});
|
||||
|
||||
it("includes branch summary usage in session totals", async () => {
|
||||
const { session, sessionManager } = await createSession();
|
||||
|
||||
try {
|
||||
sessionManager.branchWithSummary(null, "summary", undefined, false, {
|
||||
input: 10,
|
||||
output: 20,
|
||||
cacheRead: 30,
|
||||
cacheWrite: 40,
|
||||
totalTokens: 100,
|
||||
cost: { input: 0.1, output: 0.2, cacheRead: 0.3, cacheWrite: 0.4, total: 1 },
|
||||
});
|
||||
syncAgentMessages(session, sessionManager);
|
||||
|
||||
const stats = session.getSessionStats();
|
||||
expect(stats.tokens).toEqual({ input: 10, output: 20, cacheRead: 30, cacheWrite: 40, total: 100 });
|
||||
expect(stats.cost).toBe(1);
|
||||
} finally {
|
||||
session.dispose();
|
||||
}
|
||||
});
|
||||
|
||||
it("includes compaction usage in session totals", async () => {
|
||||
const { session, sessionManager } = await createSession();
|
||||
|
||||
try {
|
||||
const firstKeptEntryId = sessionManager.appendMessage(createUserMessage("hello", 1));
|
||||
sessionManager.appendCompaction("summary", firstKeptEntryId, 100, undefined, false, {
|
||||
input: 10,
|
||||
output: 20,
|
||||
cacheRead: 30,
|
||||
cacheWrite: 40,
|
||||
totalTokens: 100,
|
||||
cost: { input: 0.1, output: 0.2, cacheRead: 0.3, cacheWrite: 0.4, total: 1 },
|
||||
});
|
||||
syncAgentMessages(session, sessionManager);
|
||||
|
||||
const stats = session.getSessionStats();
|
||||
expect(stats.tokens).toEqual({ input: 10, output: 20, cacheRead: 30, cacheWrite: 40, total: 100 });
|
||||
expect(stats.cost).toBe(1);
|
||||
} finally {
|
||||
session.dispose();
|
||||
}
|
||||
});
|
||||
|
||||
it("includes tool result usage in session totals", async () => {
|
||||
const { session, sessionManager } = await createSession();
|
||||
|
||||
try {
|
||||
sessionManager.appendMessage(
|
||||
createToolResultMessage({
|
||||
input: 10,
|
||||
output: 20,
|
||||
cacheRead: 30,
|
||||
cacheWrite: 40,
|
||||
totalTokens: 100,
|
||||
cost: { input: 0.1, output: 0.2, cacheRead: 0.3, cacheWrite: 0.4, total: 1 },
|
||||
}),
|
||||
);
|
||||
syncAgentMessages(session, sessionManager);
|
||||
|
||||
const stats = session.getSessionStats();
|
||||
expect(stats.tokens).toEqual({ input: 10, output: 20, cacheRead: 30, cacheWrite: 40, total: 100 });
|
||||
expect(stats.cost).toBe(1);
|
||||
} finally {
|
||||
session.dispose();
|
||||
}
|
||||
});
|
||||
|
||||
it("ignores zero-usage messages when checking for post-compaction context usage", async () => {
|
||||
const { session, sessionManager } = await createSession();
|
||||
|
||||
|
||||
@@ -0,0 +1,55 @@
|
||||
import type { Usage } from "@earendil-works/pi-ai/compat";
|
||||
import { afterEach, describe, expect, it } from "vitest";
|
||||
import { createHarness, type Harness } from "./suite/harness.ts";
|
||||
import { assistantMsg, userMsg } from "./utilities.ts";
|
||||
|
||||
describe("Branch summary extensions", () => {
|
||||
const harnesses: Harness[] = [];
|
||||
|
||||
afterEach(() => {
|
||||
while (harnesses.length > 0) {
|
||||
harnesses.pop()?.cleanup();
|
||||
}
|
||||
});
|
||||
|
||||
it("persists extension-provided summary usage in session totals", async () => {
|
||||
const usage: Usage = {
|
||||
input: 10,
|
||||
output: 20,
|
||||
cacheRead: 30,
|
||||
cacheWrite: 40,
|
||||
totalTokens: 100,
|
||||
cost: { input: 0.1, output: 0.2, cacheRead: 0.3, cacheWrite: 0.4, total: 1 },
|
||||
};
|
||||
const harness = await createHarness({
|
||||
extensionFactories: [
|
||||
(pi) => {
|
||||
pi.on("session_before_tree", () => ({
|
||||
summary: {
|
||||
summary: "Summary provided by extension",
|
||||
usage,
|
||||
},
|
||||
}));
|
||||
},
|
||||
],
|
||||
});
|
||||
harnesses.push(harness);
|
||||
|
||||
const targetId = harness.sessionManager.appendMessage(userMsg("first branch"));
|
||||
harness.sessionManager.appendMessage(assistantMsg("first reply"));
|
||||
harness.sessionManager.appendMessage(userMsg("abandoned branch work"));
|
||||
harness.sessionManager.appendMessage(assistantMsg("abandoned reply"));
|
||||
|
||||
const result = await harness.session.navigateTree(targetId, { summarize: true });
|
||||
const summaryEntry = result.summaryEntry;
|
||||
|
||||
expect(summaryEntry?.type).toBe("branch_summary");
|
||||
expect(summaryEntry?.fromHook).toBe(true);
|
||||
expect(summaryEntry?.summary).toBe("Summary provided by extension");
|
||||
expect(summaryEntry?.usage).toEqual(usage);
|
||||
|
||||
const stats = harness.session.getSessionStats();
|
||||
expect(stats.tokens).toEqual({ input: 12, output: 22, cacheRead: 30, cacheWrite: 40, total: 104 });
|
||||
expect(stats.cost).toBe(1);
|
||||
});
|
||||
});
|
||||
@@ -57,7 +57,7 @@ describe("generateSummary reasoning options", () => {
|
||||
});
|
||||
|
||||
it("uses the provided thinking level for reasoning-capable models", async () => {
|
||||
await generateSummary(
|
||||
const result = await generateSummary(
|
||||
messages,
|
||||
createModel(true),
|
||||
2000,
|
||||
@@ -69,6 +69,9 @@ describe("generateSummary reasoning options", () => {
|
||||
"medium",
|
||||
);
|
||||
|
||||
expect(result.text).toBe("## Goal\nTest summary");
|
||||
expect(result.usage).toEqual(mockSummaryResponse.usage);
|
||||
|
||||
expect(completeSimpleMock).toHaveBeenCalledTimes(1);
|
||||
expect(completeSimpleMock.mock.calls[0][2]).toMatchObject({
|
||||
reasoning: "medium",
|
||||
@@ -127,8 +130,15 @@ describe("generateSummary reasoning options", () => {
|
||||
settings: { enabled: true, reserveTokens: 500000, keepRecentTokens: 20000 },
|
||||
};
|
||||
|
||||
await compact(preparation, createModel(false, 128000), "test-key");
|
||||
const result = await compact(preparation, createModel(false, 128000), "test-key");
|
||||
|
||||
expect(result.usage).toEqual({
|
||||
...mockSummaryResponse.usage,
|
||||
input: 20,
|
||||
output: 20,
|
||||
totalTokens: 40,
|
||||
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 },
|
||||
});
|
||||
expect(completeSimpleMock.mock.calls.map((call) => call[2]?.maxTokens)).toEqual([128000, 128000]);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -21,20 +21,46 @@ function createSession(options: {
|
||||
reasoning?: boolean;
|
||||
thinkingLevel?: string;
|
||||
usage?: AssistantUsage;
|
||||
branchUsage?: AssistantUsage;
|
||||
compactionUsage?: AssistantUsage;
|
||||
toolUsage?: AssistantUsage;
|
||||
}): AgentSession {
|
||||
const usage = options.usage;
|
||||
const entries =
|
||||
usage === undefined
|
||||
? []
|
||||
: [
|
||||
{
|
||||
type: "message",
|
||||
message: {
|
||||
role: "assistant",
|
||||
usage,
|
||||
},
|
||||
},
|
||||
];
|
||||
const entries: Array<Record<string, unknown>> = [];
|
||||
|
||||
if (usage !== undefined) {
|
||||
entries.push({
|
||||
type: "message",
|
||||
message: {
|
||||
role: "assistant",
|
||||
usage,
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
if (options.branchUsage !== undefined) {
|
||||
entries.push({
|
||||
type: "branch_summary",
|
||||
usage: options.branchUsage,
|
||||
});
|
||||
}
|
||||
|
||||
if (options.compactionUsage !== undefined) {
|
||||
entries.push({
|
||||
type: "compaction",
|
||||
usage: options.compactionUsage,
|
||||
});
|
||||
}
|
||||
|
||||
if (options.toolUsage !== undefined) {
|
||||
entries.push({
|
||||
type: "message",
|
||||
message: {
|
||||
role: "toolResult",
|
||||
usage: options.toolUsage,
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
const session = {
|
||||
state: {
|
||||
@@ -125,6 +151,44 @@ describe("FooterComponent width handling", () => {
|
||||
}
|
||||
});
|
||||
|
||||
it("includes summary and tool result usage in the total cost", () => {
|
||||
const session = createSession({
|
||||
sessionName: "",
|
||||
usage: {
|
||||
input: 100,
|
||||
output: 10,
|
||||
cacheRead: 0,
|
||||
cacheWrite: 0,
|
||||
cost: { total: 0.5 },
|
||||
},
|
||||
branchUsage: {
|
||||
input: 20,
|
||||
output: 5,
|
||||
cacheRead: 0,
|
||||
cacheWrite: 0,
|
||||
cost: { total: 0.25 },
|
||||
},
|
||||
compactionUsage: {
|
||||
input: 5,
|
||||
output: 2,
|
||||
cacheRead: 0,
|
||||
cacheWrite: 0,
|
||||
cost: { total: 0.125 },
|
||||
},
|
||||
toolUsage: {
|
||||
input: 15,
|
||||
output: 3,
|
||||
cacheRead: 0,
|
||||
cacheWrite: 0,
|
||||
cost: { total: 0.375 },
|
||||
},
|
||||
});
|
||||
const footer = new FooterComponent(session, createFooterData(1));
|
||||
|
||||
const statsLine = stripAnsi(footer.render(120)[1]);
|
||||
expect(statsLine).toContain("$1.250");
|
||||
});
|
||||
|
||||
it("shows the latest cache hit rate when cache usage is present", () => {
|
||||
const session = createSession({
|
||||
sessionName: "",
|
||||
|
||||
@@ -71,7 +71,15 @@ describe("SessionManager append and tree traversal", () => {
|
||||
|
||||
const id1 = session.appendMessage(userMsg("1"));
|
||||
const id2 = session.appendMessage(assistantMsg("2"));
|
||||
const compactionId = session.appendCompaction("summary", id1, 1000);
|
||||
const usage = {
|
||||
input: 10,
|
||||
output: 20,
|
||||
cacheRead: 30,
|
||||
cacheWrite: 40,
|
||||
totalTokens: 100,
|
||||
cost: { input: 0.1, output: 0.2, cacheRead: 0.3, cacheWrite: 0.4, total: 1 },
|
||||
};
|
||||
const compactionId = session.appendCompaction("summary", id1, 1000, undefined, false, usage);
|
||||
const _id3 = session.appendMessage(userMsg("3"));
|
||||
|
||||
const entries = session.getEntries();
|
||||
@@ -83,6 +91,7 @@ describe("SessionManager append and tree traversal", () => {
|
||||
expect(compactionEntry.summary).toBe("summary");
|
||||
expect(compactionEntry.firstKeptEntryId).toBe(id1);
|
||||
expect(compactionEntry.tokensBefore).toBe(1000);
|
||||
expect(compactionEntry.usage).toEqual(usage);
|
||||
}
|
||||
|
||||
expect(entries[3].parentId).toBe(compactionId);
|
||||
@@ -319,7 +328,15 @@ describe("SessionManager append and tree traversal", () => {
|
||||
const _id2 = session.appendMessage(assistantMsg("2"));
|
||||
const _id3 = session.appendMessage(userMsg("3"));
|
||||
|
||||
const summaryId = session.branchWithSummary(id1, "Summary of abandoned work");
|
||||
const usage = {
|
||||
input: 10,
|
||||
output: 20,
|
||||
cacheRead: 30,
|
||||
cacheWrite: 40,
|
||||
totalTokens: 100,
|
||||
cost: { input: 0.1, output: 0.2, cacheRead: 0.3, cacheWrite: 0.4, total: 1 },
|
||||
};
|
||||
const summaryId = session.branchWithSummary(id1, "Summary of abandoned work", undefined, false, usage);
|
||||
|
||||
expect(session.getLeafId()).toBe(summaryId);
|
||||
|
||||
@@ -329,6 +346,7 @@ describe("SessionManager append and tree traversal", () => {
|
||||
expect(summaryEntry?.parentId).toBe(id1);
|
||||
if (summaryEntry?.type === "branch_summary") {
|
||||
expect(summaryEntry.summary).toBe("Summary of abandoned work");
|
||||
expect(summaryEntry.usage).toEqual(usage);
|
||||
}
|
||||
});
|
||||
|
||||
|
||||
@@ -97,6 +97,14 @@ describe("AgentSession compaction characterization", () => {
|
||||
});
|
||||
|
||||
it("manually compacts using an extension-provided summary", async () => {
|
||||
const summaryUsage = {
|
||||
input: 10,
|
||||
output: 20,
|
||||
cacheRead: 30,
|
||||
cacheWrite: 40,
|
||||
totalTokens: 100,
|
||||
cost: { input: 0.1, output: 0.2, cacheRead: 0.3, cacheWrite: 0.4, total: 1 },
|
||||
};
|
||||
const harness = await createHarness({
|
||||
settings: { compaction: { keepRecentTokens: 1 } },
|
||||
extensionFactories: [
|
||||
@@ -106,6 +114,7 @@ describe("AgentSession compaction characterization", () => {
|
||||
summary: "summary from extension",
|
||||
firstKeptEntryId: event.preparation.firstKeptEntryId,
|
||||
tokensBefore: event.preparation.tokensBefore,
|
||||
usage: summaryUsage,
|
||||
details: { source: "extension" },
|
||||
},
|
||||
}));
|
||||
@@ -116,14 +125,26 @@ describe("AgentSession compaction characterization", () => {
|
||||
|
||||
await harness.session.prompt("one");
|
||||
await harness.session.prompt("two");
|
||||
const statsBefore = harness.session.getSessionStats();
|
||||
|
||||
const result = await harness.session.compact();
|
||||
const compactionEntries = harness.sessionManager.getEntries().filter((entry) => entry.type === "compaction");
|
||||
const estimatedTokensAfter = harness.session.messages.reduce((sum, message) => sum + estimateTokens(message), 0);
|
||||
|
||||
expect(result.summary).toBe("summary from extension");
|
||||
expect(result.usage).toEqual(summaryUsage);
|
||||
expect(result.estimatedTokensAfter).toBe(estimatedTokensAfter);
|
||||
expect(compactionEntries).toHaveLength(1);
|
||||
const compactionEntry = compactionEntries[0];
|
||||
if (compactionEntry?.type === "compaction") {
|
||||
expect(compactionEntry.usage).toEqual(summaryUsage);
|
||||
}
|
||||
const statsAfter = harness.session.getSessionStats();
|
||||
expect(statsAfter.tokens.input).toBe(statsBefore.tokens.input + summaryUsage.input);
|
||||
expect(statsAfter.tokens.output).toBe(statsBefore.tokens.output + summaryUsage.output);
|
||||
expect(statsAfter.tokens.cacheRead).toBe(statsBefore.tokens.cacheRead + summaryUsage.cacheRead);
|
||||
expect(statsAfter.tokens.cacheWrite).toBe(statsBefore.tokens.cacheWrite + summaryUsage.cacheWrite);
|
||||
expect(statsAfter.cost).toBe(statsBefore.cost + summaryUsage.cost.total);
|
||||
expect(harness.session.messages[0]?.role).toBe("compactionSummary");
|
||||
});
|
||||
|
||||
@@ -154,6 +175,22 @@ describe("AgentSession compaction characterization", () => {
|
||||
expect(getStreamCallCount()).toBe(1);
|
||||
});
|
||||
|
||||
it("persists usage from pi-generated manual compaction", async () => {
|
||||
const harness = await createHarness({ withConfiguredAuth: false });
|
||||
harnesses.push(harness);
|
||||
seedCompactableSession(harness);
|
||||
useSummaryStreamFn(harness, "summary from custom stream");
|
||||
|
||||
const result = await harness.session.compact();
|
||||
|
||||
const compactionEntries = harness.sessionManager.getEntries().filter((entry) => entry.type === "compaction");
|
||||
expect(result.usage).toEqual(createUsage(10));
|
||||
expect(compactionEntries).toHaveLength(1);
|
||||
expect(compactionEntries[0]?.type === "compaction" ? compactionEntries[0].usage : undefined).toEqual(
|
||||
createUsage(10),
|
||||
);
|
||||
});
|
||||
|
||||
it("auto-compacts with a custom streamFn when registry auth is absent", async () => {
|
||||
const harness = await createHarness({ withConfiguredAuth: false });
|
||||
harnesses.push(harness);
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import type { AgentTool, ThinkingLevel } from "@earendil-works/pi-agent-core";
|
||||
import { fauxAssistantMessage, fauxToolCall, type Model } from "@earendil-works/pi-ai";
|
||||
import { fauxAssistantMessage, fauxToolCall, type Model, type Usage } from "@earendil-works/pi-ai";
|
||||
import { Type } from "typebox";
|
||||
import { afterEach, describe, expect, it } from "vitest";
|
||||
import type { BuildSystemPromptOptions, ExtensionAPI } from "../../src/index.ts";
|
||||
@@ -156,6 +156,23 @@ describe("AgentSession model and extension characterization", () => {
|
||||
});
|
||||
|
||||
it("allows extension tool_result handlers to modify tool results", async () => {
|
||||
const toolUsage: Usage = {
|
||||
input: 1,
|
||||
output: 2,
|
||||
cacheRead: 3,
|
||||
cacheWrite: 4,
|
||||
totalTokens: 10,
|
||||
cost: { input: 0.1, output: 0.2, cacheRead: 0.3, cacheWrite: 0.4, total: 1 },
|
||||
};
|
||||
const patchedToolUsage: Usage = {
|
||||
input: 5,
|
||||
output: 6,
|
||||
cacheRead: 7,
|
||||
cacheWrite: 8,
|
||||
totalTokens: 26,
|
||||
cost: { input: 0.5, output: 0.6, cacheRead: 0.7, cacheWrite: 0.8, total: 2.6 },
|
||||
};
|
||||
let observedToolUsage: Usage | undefined;
|
||||
const echoTool: AgentTool = {
|
||||
name: "echo",
|
||||
label: "Echo",
|
||||
@@ -163,17 +180,21 @@ describe("AgentSession model and extension characterization", () => {
|
||||
parameters: Type.Object({ text: Type.String() }),
|
||||
execute: async (_toolCallId, params) => {
|
||||
const text = typeof params === "object" && params !== null && "text" in params ? String(params.text) : "";
|
||||
return { content: [{ type: "text", text }], details: { text } };
|
||||
return { content: [{ type: "text", text }], details: { text }, usage: toolUsage };
|
||||
},
|
||||
};
|
||||
const harness = await createHarness({
|
||||
tools: [echoTool],
|
||||
extensionFactories: [
|
||||
(pi) => {
|
||||
pi.on("tool_result", async () => ({
|
||||
content: [{ type: "text", text: "patched result" }],
|
||||
details: { patched: true },
|
||||
}));
|
||||
pi.on("tool_result", async (event) => {
|
||||
observedToolUsage = event.usage;
|
||||
return {
|
||||
content: [{ type: "text", text: "patched result" }],
|
||||
details: { patched: true },
|
||||
usage: patchedToolUsage,
|
||||
};
|
||||
});
|
||||
},
|
||||
],
|
||||
});
|
||||
@@ -196,9 +217,12 @@ describe("AgentSession model and extension characterization", () => {
|
||||
await harness.session.prompt("hi");
|
||||
|
||||
expect(getAssistantTexts(harness)).toContain("patched result");
|
||||
expect(
|
||||
harness.session.messages.find((message) => message.role === "toolResult" && message.details?.patched === true),
|
||||
).toBeDefined();
|
||||
const toolResult = harness.session.messages.find(
|
||||
(message) => message.role === "toolResult" && message.details?.patched === true,
|
||||
);
|
||||
expect(observedToolUsage).toEqual(toolUsage);
|
||||
expect(toolResult).toBeDefined();
|
||||
expect(toolResult?.role === "toolResult" ? toolResult.usage : undefined).toEqual(patchedToolUsage);
|
||||
});
|
||||
|
||||
it("allows extension context handlers to modify messages before the LLM call", async () => {
|
||||
|
||||
+2
-1
@@ -37,7 +37,7 @@ describe("issue #6324 branch summary ambient auth", () => {
|
||||
cacheRead: 0,
|
||||
cacheWrite: 0,
|
||||
totalTokens: 2,
|
||||
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 },
|
||||
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0.25 },
|
||||
},
|
||||
stopReason: "stop",
|
||||
timestamp: Date.now(),
|
||||
@@ -57,5 +57,6 @@ describe("issue #6324 branch summary ambient auth", () => {
|
||||
expect(streamCallCount).toBe(1);
|
||||
expect(result.summaryEntry?.type).toBe("branch_summary");
|
||||
expect(result.summaryEntry?.summary).toContain("branch summary text");
|
||||
expect(result.summaryEntry?.usage?.cost.total).toBe(0.25);
|
||||
});
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user