ton/pi_harness
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

add branch/compact retries to agent-harness

Browse Source
This commit is contained in:
David Brailovsky
2026-07-21 12:11:39 +00:00
parent 8e53e0e49c
commit 162179af5a
10 changed files with 354 additions and 67 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/agent/src/harness/agent-harness.ts
+25 -1
View File
@@ -4,6 +4,8 @@ import {
type ImageContent,
type Model,
type Models,
type RetryCallbacks,
type RetryPolicy,
type UserMessage,
} from "@earendil-works/pi-ai";
import { runAgentLoop } from "../agent-loop.ts";
@@ -178,6 +180,7 @@ export class AgentHarness<
private thinkingLevel: ThinkingLevel;
private systemPrompt: AgentHarnessOptions<TSkill, TPromptTemplate, TTool>["systemPrompt"];
private streamOptions: AgentHarnessStreamOptions;
private retry: RetryPolicy | undefined;
private resources: AgentHarnessResources<TSkill, TPromptTemplate>;
private tools = new Map<string, TTool>();
private activeToolNames: string[];
@@ -194,6 +197,7 @@ export class AgentHarness<
this.models = options.models;
this.resources = options.resources ?? {};
this.streamOptions = cloneStreamOptions(options.streamOptions);
this.retry = options.retry;
this.systemPrompt = options.systemPrompt;
this.validateUniqueNames(
(options.tools ?? []).map((tool) => tool.name),
@@ -256,6 +260,15 @@ export class AgentHarness<
return lastResult;
}
private retryCallbacks(operation: "compaction" | "branch_summary"): RetryCallbacks {
return {
onRetryScheduled: (attempt, maxAttempts, delayMs, errorMessage) =>
this.emitOwn({ type: "retry_scheduled", operation, attempt, maxAttempts, delayMs, errorMessage }),
onRetryAttemptStart: () => this.emitOwn({ type: "retry_attempt_start", operation }),
onRetryFinished: () => this.emitOwn({ type: "retry_finished", operation }),
};
}
private async emitBeforeProviderRequest(
model: Model<any>,
sessionId: string,
@@ -720,7 +733,16 @@ export class AgentHarness<
const provided = hookResult?.compaction;
const compactResult = provided
? { ok: true as const, value: provided }
: await compact(preparation, this.models, model, customInstructions, undefined, this.thinkingLevel);
: await compact(
preparation,
this.models,
model,
customInstructions,
undefined,
this.thinkingLevel,
this.retry,
this.retryCallbacks("compaction"),
);
if (!compactResult.ok) throw compactResult.error;
const result = compactResult.value;
const entryId = await this.session.appendCompaction(
@@ -781,6 +803,8 @@ export class AgentHarness<
signal: new AbortController().signal,
customInstructions: hookResult?.customInstructions ?? options?.customInstructions,
replaceInstructions: hookResult?.replaceInstructions ?? options?.replaceInstructions,
retry: this.retry,
callbacks: this.retryCallbacks("branch_summary"),
});
if (!branchSummary.ok) {
if (branchSummary.error.code === "aborted") return { cancelled: true };
packages/agent/src/harness/compaction/branch-summarization.ts
+20 -4
View File
@@ -1,4 +1,4 @@
import { contentText, type Model, type Models } from "@earendil-works/pi-ai";
import { contentText, type Model, type Models, type RetryCallbacks, type RetryPolicy } from "@earendil-works/pi-ai";
import type { AgentMessage } from "../../types.ts";
import {
@@ -9,7 +9,7 @@ import {
} from "../messages.ts";
import type { BranchSummaryResult, Session, SessionTreeEntry } from "../types.ts";
import { BranchSummaryError, err, ok, type Result, SessionError } from "../types.ts";
import { estimateTokens, SUMMARIZATION_SYSTEM_PROMPT } from "./compaction.ts";
import { completeSimpleWithRetries, estimateTokens, SUMMARIZATION_SYSTEM_PROMPT } from "./compaction.ts";
import {
computeFileLists,
createFileOps,
@@ -61,6 +61,10 @@ export interface GenerateBranchSummaryOptions {
replaceInstructions?: boolean;
/** Tokens reserved for prompt and model output. Defaults to 16384. */
reserveTokens?: number;
/** Optional retry policy for transient summarization errors. */
retry?: RetryPolicy;
/** Optional callbacks for retry reporting. */
callbacks?: RetryCallbacks;
}
/** Collect entries that should be summarized before navigating to a different session tree entry. */
@@ -200,7 +204,16 @@ export async function generateBranchSummary(
entries: SessionTreeEntry[],
options: GenerateBranchSummaryOptions,
): Promise<Result<BranchSummaryResult, BranchSummaryError>> {
const { models, model, signal, customInstructions, replaceInstructions, reserveTokens = 16384 } = options;
const {
models,
model,
signal,
customInstructions,
replaceInstructions,
reserveTokens = 16384,
retry,
callbacks,
} = options;
const contextWindow = model.contextWindow || 128000;
const tokenBudget = contextWindow - reserveTokens;
@@ -228,10 +241,13 @@ export async function generateBranchSummary(
timestamp: Date.now(),
},
];
const response = await models.completeSimple(
const response = await completeSimpleWithRetries(
models,
model,
{ systemPrompt: SUMMARIZATION_SYSTEM_PROMPT, messages: summarizationMessages },
{ signal, maxTokens: 2048 },
retry,
callbacks,
);
if (response.stopReason === "aborted") {
return err(new BranchSummaryError("aborted", response.errorMessage || "Branch summary aborted"));
packages/agent/src/harness/compaction/compaction.ts
+45 -5
View File
@@ -1,9 +1,14 @@
import {
type AssistantMessage,
type Context,
contentText,
type ImageContent,
type Model,
type Models,
type RetryCallbacks,
type RetryPolicy,
retryAssistantCall,
type SimpleStreamOptions,
type TextContent,
type Usage,
} from "@earendil-works/pi-ai";
@@ -107,6 +112,17 @@ export interface CompactionResult<T = unknown> {
details?: T;
}
export async function completeSimpleWithRetries(
models: Models,
model: Model<any>,
context: Context,
options: SimpleStreamOptions,
retry?: RetryPolicy,
callbacks?: RetryCallbacks,
): Promise<AssistantMessage> {
return retryAssistantCall(() => models.completeSimple(model, context, options), retry, options.signal, callbacks);
}
function combineUsage(first: Usage, second: Usage): Usage {
return {
input: first.input + second.input,
@@ -499,6 +515,8 @@ export async function generateSummary(
customInstructions?: string,
previousSummary?: string,
thinkingLevel?: ThinkingLevel,
retry?: RetryPolicy,
callbacks?: RetryCallbacks,
): Promise<Result<string, CompactionError>> {
const result = await generateSummaryWithUsage(
currentMessages,
@@ -509,6 +527,8 @@ export async function generateSummary(
customInstructions,
previousSummary,
thinkingLevel,
retry,
callbacks,
);
return result.ok ? ok(result.value.text) : err(result.error);
}
@@ -523,6 +543,8 @@ export async function generateSummaryWithUsage(
customInstructions?: string,
previousSummary?: string,
thinkingLevel?: ThinkingLevel,
retry?: RetryPolicy,
callbacks?: RetryCallbacks,
): Promise<Result<{ text: string; usage: Usage }, CompactionError>> {
const maxTokens = Math.min(
Math.floor(0.8 * reserveTokens),
@@ -553,10 +575,13 @@ export async function generateSummaryWithUsage(
? { maxTokens, signal, reasoning: thinkingLevel }
: { maxTokens, signal };
const response = await models.completeSimple(
const response = await completeSimpleWithRetries(
models,
model,
{ systemPrompt: SUMMARIZATION_SYSTEM_PROMPT, messages: summarizationMessages },
completionOptions,
retry,
callbacks,
);
if (response.stopReason === "aborted") {
return err(new CompactionError("aborted", response.errorMessage || "Summarization aborted"));
@@ -688,6 +713,8 @@ export async function compact(
customInstructions?: string,
signal?: AbortSignal,
thinkingLevel?: ThinkingLevel,
retry?: RetryPolicy,
callbacks?: RetryCallbacks,
): Promise<Result<CompactionResult, CompactionError>> {
const {
firstKeptEntryId,
@@ -720,6 +747,8 @@ export async function compact(
customInstructions,
previousSummary,
thinkingLevel,
retry,
callbacks,
);
if (!historyResult.ok) return err(historyResult.error);
historyText = historyResult.value.text;
@@ -732,6 +761,8 @@ export async function compact(
settings.reserveTokens,
signal,
thinkingLevel,
retry,
callbacks,
);
if (!turnPrefixResult.ok) return err(turnPrefixResult.error);
summary = `${historyText}\n\n---\n\n**Turn Context (split turn):**\n\n${turnPrefixResult.value.text}`;
@@ -748,6 +779,8 @@ export async function compact(
customInstructions,
previousSummary,
thinkingLevel,
retry,
callbacks,
);
if (!summaryResult.ok) return err(summaryResult.error);
summary = summaryResult.value.text;
@@ -772,6 +805,8 @@ async function generateTurnPrefixSummary(
reserveTokens: number,
signal?: AbortSignal,
thinkingLevel?: ThinkingLevel,
retry?: RetryPolicy,
callbacks?: RetryCallbacks,
): Promise<Result<{ text: string; usage: Usage }, CompactionError>> {
const maxTokens = Math.min(
Math.floor(0.5 * reserveTokens),
@@ -788,12 +823,17 @@ async function generateTurnPrefixSummary(
},
];
const response = await models.completeSimple(
model,
{ systemPrompt: SUMMARIZATION_SYSTEM_PROMPT, messages: summarizationMessages },
const completionOptions =
model.reasoning && thinkingLevel && thinkingLevel !== "off"
? { maxTokens, signal, reasoning: thinkingLevel }
: { maxTokens, signal },
: { maxTokens, signal };
const response = await completeSimpleWithRetries(
models,
model,
{ systemPrompt: SUMMARIZATION_SYSTEM_PROMPT, messages: summarizationMessages },
completionOptions,
retry,
callbacks,
);
if (response.stopReason === "aborted") {
return err(new CompactionError("aborted", response.errorMessage || "Turn prefix summarization aborted"));
packages/agent/src/harness/types.ts
+28
View File
@@ -2,6 +2,7 @@ import type {
ImageContent,
Model,
Models,
RetryPolicy,
SimpleStreamOptions,
TextContent,
Transport,
@@ -613,6 +614,25 @@ export interface SessionTreeEvent {
fromHook?: boolean;
}
export interface RetryScheduledEvent {
type: "retry_scheduled";
operation: "compaction" | "branch_summary";
attempt: number;
maxAttempts: number;
delayMs: number;
errorMessage: string;
}
export interface RetryAttemptStartEvent {
type: "retry_attempt_start";
operation: "compaction" | "branch_summary";
}
export interface RetryFinishedEvent {
type: "retry_finished";
operation: "compaction" | "branch_summary";
}
export interface ModelUpdateEvent {
type: "model_update";
model: Model<any>;
@@ -663,6 +683,9 @@ export type AgentHarnessOwnEvent<
| SessionCompactEvent
| SessionBeforeTreeEvent
| SessionTreeEvent
| RetryScheduledEvent
| RetryAttemptStartEvent
| RetryFinishedEvent
| ModelUpdateEvent
| ThinkingLevelUpdateEvent
| ResourcesUpdateEvent<TSkill, TPromptTemplate>
@@ -732,6 +755,9 @@ export type AgentHarnessEventResultMap = {
session_compact: undefined;
session_before_tree: SessionBeforeTreeResult | undefined;
session_tree: undefined;
retry_scheduled: undefined;
retry_attempt_start: undefined;
retry_finished: undefined;
model_update: undefined;
thinking_level_update: undefined;
resources_update: undefined;
@@ -848,6 +874,8 @@ export interface AgentHarnessOptions<
}) => string | Promise<string>);
/** Curated stream/provider request options. Snapshotted at turn start. */
streamOptions?: AgentHarnessStreamOptions;
/** Optional retry policy for generated compaction and branch-summary requests. */
retry?: RetryPolicy;
model: Model<any>;
thinkingLevel?: ThinkingLevel;
activeToolNames?: string[];
packages/agent/test/harness/agent-harness.test.ts
+174
View File
@@ -550,6 +550,180 @@ describe("AgentHarness", () => {
expect(compaction?.type === "compaction" ? compaction.usage : undefined).toEqual(usage);
});
describe("summarization retries", () => {
it("retries transient compaction errors and emits retry events", async () => {
const registration = newFaux();
let calls = 0;
registration.setResponses([
() => {
calls++;
return fauxAssistantMessage("", { stopReason: "error", errorMessage: "terminated" });
},
() => {
calls++;
return fauxAssistantMessage("## Goal\nRecovered summary");
},
]);
const session = new Session(new InMemorySessionStorage());
await session.appendMessage(createUserMessage("one"));
await session.appendMessage(createAssistantMessage("two"));
const harness = new AgentHarness({
models,
env: new NodeExecutionEnv({ cwd: process.cwd() }),
session,
model: registration.getModel(),
retry: { enabled: true, maxRetries: 1, baseDelayMs: 0 },
});
const retryEvents: string[] = [];
harness.subscribe((event) => {
if (
event.type === "retry_scheduled" ||
event.type === "retry_attempt_start" ||
event.type === "retry_finished"
) {
retryEvents.push(`${event.type}:${event.operation}`);
}
});
const result = await harness.compact();
expect(result.summary).toContain("Recovered summary");
expect(calls).toBe(2);
expect(retryEvents).toEqual([
"retry_scheduled:compaction",
"retry_attempt_start:compaction",
"retry_finished:compaction",
]);
});
it("does not retry non-retryable compaction errors", async () => {
const registration = newFaux();
let calls = 0;
registration.setResponses([
() => {
calls++;
return fauxAssistantMessage("", { stopReason: "error", errorMessage: "insufficient_quota" });
},
]);
const session = new Session(new InMemorySessionStorage());
await session.appendMessage(createUserMessage("one"));
await session.appendMessage(createAssistantMessage("two"));
const harness = new AgentHarness({
models,
env: new NodeExecutionEnv({ cwd: process.cwd() }),
session,
model: registration.getModel(),
retry: { enabled: true, maxRetries: 1, baseDelayMs: 0 },
});
const retryEvents: string[] = [];
harness.subscribe((event) => {
if (
event.type === "retry_scheduled" ||
event.type === "retry_attempt_start" ||
event.type === "retry_finished"
) {
retryEvents.push(event.type);
}
});
await expect(harness.compact()).rejects.toThrow("insufficient_quota");
expect(calls).toBe(1);
expect(retryEvents).toEqual([]);
});
it("exhausts transient compaction retries after maxRetries failures", async () => {
const registration = newFaux();
let calls = 0;
registration.setResponses(
Array.from({ length: 4 }, () => () => {
calls++;
return fauxAssistantMessage("", { stopReason: "error", errorMessage: "terminated" });
}),
);
const session = new Session(new InMemorySessionStorage());
await session.appendMessage(createUserMessage("one"));
await session.appendMessage(createAssistantMessage("two"));
const harness = new AgentHarness({
models,
env: new NodeExecutionEnv({ cwd: process.cwd() }),
session,
model: registration.getModel(),
retry: { enabled: true, maxRetries: 3, baseDelayMs: 0 },
});
const retryEvents: string[] = [];
harness.subscribe((event) => {
if (
event.type === "retry_scheduled" ||
event.type === "retry_attempt_start" ||
event.type === "retry_finished"
) {
retryEvents.push(`${event.type}:${event.operation}`);
}
});
await expect(harness.compact()).rejects.toThrow("terminated");
expect(calls).toBe(4);
expect(retryEvents).toEqual([
"retry_scheduled:compaction",
"retry_attempt_start:compaction",
"retry_scheduled:compaction",
"retry_attempt_start:compaction",
"retry_scheduled:compaction",
"retry_attempt_start:compaction",
"retry_finished:compaction",
]);
});
it("retries transient branch summary errors and emits retry events", async () => {
const registration = newFaux();
let calls = 0;
registration.setResponses([
() => {
calls++;
return fauxAssistantMessage("", { stopReason: "error", errorMessage: "terminated" });
},
() => {
calls++;
return fauxAssistantMessage("## Goal\nRecovered branch summary");
},
]);
const session = new Session(new InMemorySessionStorage());
const targetId = await session.appendMessage(createUserMessage("first branch"));
await session.appendMessage(createAssistantMessage("first reply"));
await session.appendMessage(createUserMessage("abandoned work"));
await session.appendMessage(createAssistantMessage("abandoned reply"));
const harness = new AgentHarness({
models,
env: new NodeExecutionEnv({ cwd: process.cwd() }),
session,
model: registration.getModel(),
retry: { enabled: true, maxRetries: 1, baseDelayMs: 0 },
});
const retryEvents: string[] = [];
harness.subscribe((event) => {
if (
event.type === "retry_scheduled" ||
event.type === "retry_attempt_start" ||
event.type === "retry_finished"
) {
retryEvents.push(`${event.type}:${event.operation}`);
}
});
const result = await harness.navigateTree(targetId, { summarize: true });
expect(result.summaryEntry?.summary).toContain("Recovered branch summary");
expect(calls).toBe(2);
expect(retryEvents).toEqual([
"retry_scheduled:branch_summary",
"retry_attempt_start:branch_summary",
"retry_finished:branch_summary",
]);
});
});
it("persists generated branch summary usage", async () => {
const registration = newFaux();
registration.setResponses([fauxAssistantMessage("## Goal\nBranch summary")]);