feat(agent): add AgentHarness execution tools

This commit is contained in:
Mario Zechner
2026-07-21 17:03:28 +02:00
parent 3dcb411e89
commit 37eb243d26
23 changed files with 1766 additions and 75 deletions
@@ -8,9 +8,9 @@ import {
} from "@earendil-works/pi-ai";
import { describe, expect, it } from "vitest";
import { AgentHarness } from "../../src/harness/agent-harness.ts";
import { NodeExecutionEnv } from "../../src/harness/env/nodejs.ts";
import { InMemorySessionStorage } from "../../src/harness/session/memory-storage.ts";
import { Session } from "../../src/harness/session/session.ts";
import type { AgentHarnessOptions } from "../../src/harness/types.ts";
import { calculateTool } from "../utils/calculate.ts";
/** Shared collection; each faux provider gets a unique id so coexisting fakes route correctly. */
@@ -23,7 +23,7 @@ function newFaux(): FauxProviderHandle {
return faux;
}
function createHarness(options: ConstructorParameters<typeof AgentHarness>[0]): AgentHarness {
function createHarness(options: AgentHarnessOptions): AgentHarness {
return new AgentHarness(options);
}
@@ -49,7 +49,6 @@ describe("AgentHarness stream configuration", () => {
const session = new Session(new InMemorySessionStorage({ metadata: { id: "session-1", createdAt: "now" } }));
const harness = createHarness({
models,
env: new NodeExecutionEnv({ cwd: process.cwd() }),
session,
model: registration.getModel(),
streamOptions: {
@@ -98,7 +97,6 @@ describe("AgentHarness stream configuration", () => {
const harness = createHarness({
models,
env: new NodeExecutionEnv({ cwd: process.cwd() }),
session: new Session(new InMemorySessionStorage()),
model: registration.getModel(),
streamOptions: {
@@ -156,7 +154,6 @@ describe("AgentHarness stream configuration", () => {
const harness = createHarness({
models,
env: new NodeExecutionEnv({ cwd: process.cwd() }),
session: new Session(new InMemorySessionStorage()),
model: registration.getModel(),
tools: [calculateTool],
@@ -191,7 +188,6 @@ describe("AgentHarness stream configuration", () => {
const harness = createHarness({
models,
env: new NodeExecutionEnv({ cwd: process.cwd() }),
session: new Session(new InMemorySessionStorage()),
model: registration.getModel(),
});
@@ -13,7 +13,7 @@ import { AgentHarness } from "../../src/harness/agent-harness.ts";
import { NodeExecutionEnv } from "../../src/harness/env/nodejs.ts";
import { InMemorySessionStorage } from "../../src/harness/session/memory-storage.ts";
import { Session } from "../../src/harness/session/session.ts";
import type { PromptTemplate, Skill } from "../../src/harness/types.ts";
import type { AgentHarnessTool, PromptTemplate, Skill } from "../../src/harness/types.ts";
import type { AgentMessage, AgentTool } from "../../src/types.ts";
import { calculateTool, createCalculateToolWithUsage } from "../utils/calculate.ts";
import { getCurrentTimeTool } from "../utils/get-current-time.ts";
@@ -92,11 +92,9 @@ function createAssistantMessage(text: string): AgentMessage {
describe("AgentHarness", () => {
it("constructs directly and exposes queue modes", () => {
const session = new Session(new InMemorySessionStorage());
const env = new NodeExecutionEnv({ cwd: process.cwd() });
const initialModel = getModel("anthropic", "claude-sonnet-4-5");
const harness = new AgentHarness({
models,
env,
session,
model: initialModel,
thinkingLevel: "high",
@@ -104,7 +102,6 @@ describe("AgentHarness", () => {
steeringMode: "all",
followUpMode: "all",
});
expect(harness.env).toBe(env);
expect(harness.getModel()).toBe(initialModel);
expect(harness.getThinkingLevel()).toBe("high");
expect(harness.getSteeringMode()).toBe("all");
@@ -134,7 +131,6 @@ describe("AgentHarness", () => {
]);
const harness = new AgentHarness({
models,
env: new NodeExecutionEnv({ cwd: process.cwd() }),
session: new Session(new InMemorySessionStorage()),
model: registration.getModel(),
steeringMode: "one-at-a-time",
@@ -170,7 +166,6 @@ describe("AgentHarness", () => {
const session = new Session(new InMemorySessionStorage());
const harness = new AgentHarness({
models,
env: new NodeExecutionEnv({ cwd: process.cwd() }),
session,
model: registration.getModel(),
});
@@ -211,7 +206,6 @@ describe("AgentHarness", () => {
]);
const harness = new AgentHarness({
models,
env: new NodeExecutionEnv({ cwd: process.cwd() }),
session: new Session(new InMemorySessionStorage()),
model: registration.getModel(),
});
@@ -264,7 +258,6 @@ describe("AgentHarness", () => {
]);
const harness = new AgentHarness({
models,
env: new NodeExecutionEnv({ cwd: process.cwd() }),
session: new Session(new InMemorySessionStorage()),
model: registration.getModel(),
followUpMode: "one-at-a-time",
@@ -294,7 +287,6 @@ describe("AgentHarness", () => {
const session = new Session(new InMemorySessionStorage());
const harness = new AgentHarness({
models,
env: new NodeExecutionEnv({ cwd: process.cwd() }),
session,
model: registration.getModel(),
});
@@ -351,9 +343,8 @@ describe("AgentHarness", () => {
return fauxAssistantMessage("done");
},
]);
const harness = new AgentHarness<Skill, PromptTemplate, AgentTool>({
const harness = new AgentHarness<undefined, Skill, PromptTemplate, AgentTool>({
models,
env: new NodeExecutionEnv({ cwd: process.cwd() }),
session: new Session(new InMemorySessionStorage()),
model: registration.getModel(),
thinkingLevel: "off",
@@ -390,7 +381,6 @@ describe("AgentHarness", () => {
const session = new Session(new InMemorySessionStorage());
const harness = new AgentHarness({
models,
env: new NodeExecutionEnv({ cwd: process.cwd() }),
session,
model: registration.getModel(),
});
@@ -421,7 +411,6 @@ describe("AgentHarness", () => {
const barrier = deferred();
const harness = new AgentHarness({
models,
env: new NodeExecutionEnv({ cwd: process.cwd() }),
session: new Session(new InMemorySessionStorage()),
model: registration.getModel(),
});
@@ -461,7 +450,6 @@ describe("AgentHarness", () => {
const calculateToolWithUsage = createCalculateToolWithUsage(toolUsage);
const harness = new AgentHarness({
models,
env: new NodeExecutionEnv({ cwd: process.cwd() }),
session,
model: registration.getModel(),
tools: [calculateToolWithUsage],
@@ -502,6 +490,75 @@ describe("AgentHarness", () => {
});
});
it("passes a static application context to harness tools", async () => {
const registration = newFaux();
registration.setResponses([
() =>
fauxAssistantMessage(fauxToolCall("context", { expression: "2 + 2" }, { id: "call-1" }), {
stopReason: "toolUse",
}),
]);
const env = new NodeExecutionEnv({ cwd: process.cwd() });
const toolContext = { env, sessionId: "session-1" };
let receivedContext: typeof toolContext | undefined;
const contextTool: AgentHarnessTool<typeof toolContext, typeof calculateTool.parameters, undefined> = {
...calculateTool,
name: "context",
execute: async (toolCallId, params, signal, onUpdate, context) => {
receivedContext = context;
return { ...(await calculateTool.execute(toolCallId, params, signal, onUpdate)), terminate: true };
},
};
const harness = new AgentHarness({
models,
session: new Session(new InMemorySessionStorage()),
model: registration.getModel(),
tools: [contextTool],
toolContext,
});
await harness.prompt("hello");
expect(receivedContext).toBe(toolContext);
});
it("resolves async tool context providers for each turn snapshot", async () => {
const registration = newFaux();
registration.setResponses([
() =>
fauxAssistantMessage(fauxToolCall("context", { expression: "1 + 1" }, { id: "call-1" }), {
stopReason: "toolUse",
}),
() =>
fauxAssistantMessage(fauxToolCall("context", { expression: "2 + 2" }, { id: "call-2" }), {
stopReason: "toolUse",
}),
() => fauxAssistantMessage("done"),
]);
type ToolContext = { generation: number };
const generations: number[] = [];
const contextTool: AgentHarnessTool<ToolContext, typeof calculateTool.parameters, undefined> = {
...calculateTool,
name: "context",
execute: async (toolCallId, params, signal, onUpdate, context) => {
generations.push(context.generation);
return await calculateTool.execute(toolCallId, params, signal, onUpdate);
},
};
let generation = 0;
const harness = new AgentHarness({
models,
session: new Session(new InMemorySessionStorage()),
model: registration.getModel(),
tools: [contextTool],
toolContext: async (): Promise<ToolContext> => ({ generation: ++generation }),
});
await harness.prompt("hello");
expect(generations).toEqual([1, 2]);
});
it("persists generated compaction usage", async () => {
const registration = newFaux();
registration.setResponses([fauxAssistantMessage("## Goal\nTest summary")]);
@@ -510,7 +567,6 @@ describe("AgentHarness", () => {
await session.appendMessage(createAssistantMessage("two"));
const harness = new AgentHarness({
models,
env: new NodeExecutionEnv({ cwd: process.cwd() }),
session,
model: registration.getModel(),
});
@@ -530,7 +586,6 @@ describe("AgentHarness", () => {
await session.appendMessage(createAssistantMessage("two"));
const harness = new AgentHarness({
models,
env: new NodeExecutionEnv({ cwd: process.cwd() }),
session,
model: registration.getModel(),
});
@@ -560,7 +615,6 @@ describe("AgentHarness", () => {
await session.appendMessage(createAssistantMessage("abandoned reply"));
const harness = new AgentHarness({
models,
env: new NodeExecutionEnv({ cwd: process.cwd() }),
session,
model: registration.getModel(),
});
@@ -580,7 +634,6 @@ describe("AgentHarness", () => {
await session.appendMessage(createAssistantMessage("abandoned reply"));
const harness = new AgentHarness({
models,
env: new NodeExecutionEnv({ cwd: process.cwd() }),
session,
model: registration.getModel(),
});
@@ -595,14 +648,12 @@ describe("AgentHarness", () => {
it("preserves app tool types for getters and update events", async () => {
const session = new Session(new InMemorySessionStorage());
const env = new NodeExecutionEnv({ cwd: process.cwd() });
const model = getModel("anthropic", "claude-sonnet-4-5");
type AppTool = AgentTool<typeof calculateTool.parameters, undefined> & { source: "builtin" | "extension" };
const inspectTool: AppTool = { ...calculateTool, name: "inspect", source: "builtin" };
const searchTool: AppTool = { ...calculateTool, name: "search", source: "extension" };
const harness = new AgentHarness<AppSkill, AppPromptTemplate, AppTool>({
const harness = new AgentHarness<undefined, AppSkill, AppPromptTemplate, AppTool>({
models,
env,
session,
model,
tools: [inspectTool, searchTool],
@@ -667,16 +718,14 @@ describe("AgentHarness", () => {
it("validates constructor tool names", () => {
const session = new Session(new InMemorySessionStorage());
const env = new NodeExecutionEnv({ cwd: process.cwd() });
const model = getModel("anthropic", "claude-sonnet-4-5");
expect(
() => new AgentHarness({ env, session, models, model, tools: [calculateTool], activeToolNames: ["missing"] }),
() => new AgentHarness({ session, models, model, tools: [calculateTool], activeToolNames: ["missing"] }),
).toThrow(/Unknown tool/);
expect(
() =>
new AgentHarness({
models,
env,
session,
model,
tools: [calculateTool, calculateTool],
@@ -687,7 +736,6 @@ describe("AgentHarness", () => {
() =>
new AgentHarness({
models,
env,
session,
model,
tools: [calculateTool],
@@ -698,9 +746,12 @@ describe("AgentHarness", () => {
it("preserves app resource types for getters and update events", async () => {
const session = new Session(new InMemorySessionStorage());
const env = new NodeExecutionEnv({ cwd: process.cwd() });
const model = getModel("anthropic", "claude-sonnet-4-5");
const harness = new AgentHarness<AppSkill, AppPromptTemplate, AgentTool>({ env, session, models, model });
const harness = new AgentHarness<undefined, AppSkill, AppPromptTemplate, AgentTool>({
session,
models,
model,
});
const skill: AppSkill = {
name: "inspect",
description: "Inspect things",
+338
View File
@@ -0,0 +1,338 @@
import { applyPatch } from "diff";
import { describe, expect, it } from "vitest";
import { NodeExecutionEnv } from "../../src/harness/env/nodejs.ts";
import { createBashTool } from "../../src/harness/tools/bash.ts";
import { createEditTool } from "../../src/harness/tools/edit.ts";
import { createReadTool } from "../../src/harness/tools/read.ts";
import { createWriteTool } from "../../src/harness/tools/write.ts";
import { getOrThrow } from "../../src/harness/types.ts";
import { createTempDir } from "./session-test-utils.ts";
function textOutput(result: { content: Array<{ type: string; text?: string }> }): string {
return result.content.flatMap((part) => (part.type === "text" ? [part.text ?? ""] : [])).join("\n");
}
function createContext() {
const env = new NodeExecutionEnv({ cwd: createTempDir() });
return { env, sessionId: "session-1" };
}
function createTinyBmp(): Uint8Array {
const bytes = new Uint8Array(58);
const view = new DataView(bytes.buffer);
bytes[0] = 0x42;
bytes[1] = 0x4d;
view.setUint32(2, bytes.length, true);
view.setUint32(10, 54, true);
view.setUint32(14, 40, true);
view.setInt32(18, 1, true);
view.setInt32(22, 1, true);
view.setUint16(26, 1, true);
view.setUint16(28, 24, true);
view.setUint32(34, 4, true);
return bytes;
}
describe("AgentHarness tools", () => {
describe("read", () => {
it("reads text with offsets, limits, and continuation notices", async () => {
const context = createContext();
getOrThrow(
await context.env.writeFile(
"test.txt",
Array.from({ length: 100 }, (_, index) => `Line ${index + 1}`).join("\n"),
),
);
const result = await createReadTool().execute(
"read-1",
{ path: "test.txt", offset: 41, limit: 20 },
undefined,
undefined,
context,
);
const output = textOutput(result);
expect(output).not.toContain("Line 40");
expect(output).toContain("Line 41");
expect(output).toContain("Line 60");
expect(output).not.toContain("Line 61");
expect(output).toContain("[40 more lines in file. Use offset=61 to continue.]");
});
it("truncates large text by line count", async () => {
const context = createContext();
getOrThrow(
await context.env.writeFile(
"large.txt",
Array.from({ length: 2500 }, (_, index) => `Line ${index + 1}`).join("\n"),
),
);
const result = await createReadTool().execute("read-2", { path: "large.txt" }, undefined, undefined, context);
expect(textOutput(result)).toContain("[Showing lines 1-2000 of 2500. Use offset=2001 to continue.]");
expect(result.details?.truncation).toMatchObject({
truncated: true,
truncatedBy: "lines",
totalLines: 2500,
outputLines: 2000,
});
});
it("rejects offsets beyond the file", async () => {
const context = createContext();
getOrThrow(await context.env.writeFile("short.txt", "one\ntwo\nthree"));
await expect(
createReadTool().execute("read-3", { path: "short.txt", offset: 100 }, undefined, undefined, context),
).rejects.toThrow("Offset 100 is beyond end of file (3 lines total)");
});
it("detects supported images by content", async () => {
const context = createContext();
const png = Uint8Array.from(
Buffer.from(
"iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR4nGNgYGD4DwABBAEAX+XDSwAAAABJRU5ErkJggg==",
"base64",
),
);
getOrThrow(await context.env.writeFile("image.txt", png));
const result = await createReadTool().execute("read-4", { path: "image.txt" }, undefined, undefined, context);
expect(textOutput(result)).toContain("Read image file [image/png]");
expect(result.content).toContainEqual({
type: "image",
data: Buffer.from(png).toString("base64"),
mimeType: "image/png",
});
});
it("delegates image conversion and resizing to an injected processor", async () => {
const context = createContext();
const bmp = createTinyBmp();
getOrThrow(await context.env.writeFile("image.bmp", bmp));
let received: { bytes: Uint8Array; mimeType: string; autoResizeImages: boolean } | undefined;
const tool = createReadTool({
autoResizeImages: false,
imageProcessor: async (bytes, mimeType, options) => {
received = { bytes, mimeType, autoResizeImages: options.autoResizeImages };
return {
ok: true,
data: "converted",
mimeType: "image/png",
hints: ["[Image converted from image/bmp to image/png.]"],
};
},
});
const result = await tool.execute("read-bmp", { path: "image.bmp" }, undefined, undefined, context);
expect(received).toMatchObject({ mimeType: "image/bmp", autoResizeImages: false });
expect(Array.from(received?.bytes ?? [])).toEqual(Array.from(bmp));
expect(textOutput(result)).toContain("[Image converted from image/bmp to image/png.]");
expect(result.content).toContainEqual({ type: "image", data: "converted", mimeType: "image/png" });
});
});
describe("write", () => {
it("writes files and creates parent directories", async () => {
const context = createContext();
const result = await createWriteTool().execute(
"write-1",
{ path: "nested/dir/file.txt", content: "hello" },
undefined,
undefined,
context,
);
expect(textOutput(result)).toBe("Successfully wrote 5 bytes to nested/dir/file.txt");
expect(getOrThrow(await context.env.readTextFile("nested/dir/file.txt"))).toBe("hello");
});
});
describe("edit", () => {
it("applies disjoint edits and returns both diff formats", async () => {
const context = createContext();
const original = "alpha\nbeta\ngamma\ndelta\n";
getOrThrow(await context.env.writeFile("edit.txt", original));
const result = await createEditTool().execute(
"edit-1",
{
path: "edit.txt",
edits: [
{ oldText: "alpha\n", newText: "ALPHA\n" },
{ oldText: "gamma\n", newText: "GAMMA\n" },
],
},
undefined,
undefined,
context,
);
expect(textOutput(result)).toBe("Successfully replaced 2 block(s) in edit.txt.");
expect(result.details?.diff).toContain("ALPHA");
expect(result.details?.diff).toContain("GAMMA");
expect(applyPatch(original, result.details?.patch ?? "")).toBe("ALPHA\nbeta\nGAMMA\ndelta\n");
expect(getOrThrow(await context.env.readTextFile("edit.txt"))).toBe("ALPHA\nbeta\nGAMMA\ndelta\n");
});
it("matches all edits against the original and rejects overlaps", async () => {
const context = createContext();
getOrThrow(await context.env.writeFile("edit.txt", "one\ntwo\nthree\n"));
await expect(
createEditTool().execute(
"edit-2",
{
path: "edit.txt",
edits: [
{ oldText: "one\ntwo\n", newText: "ONE\nTWO\n" },
{ oldText: "two\nthree\n", newText: "TWO\nTHREE\n" },
],
},
undefined,
undefined,
context,
),
).rejects.toThrow(/overlap/);
expect(getOrThrow(await context.env.readTextFile("edit.txt"))).toBe("one\ntwo\nthree\n");
});
it("rejects missing and duplicate target text", async () => {
const context = createContext();
getOrThrow(await context.env.writeFile("edit.txt", "foo foo foo"));
const tool = createEditTool();
await expect(
tool.execute(
"edit-3",
{ path: "edit.txt", edits: [{ oldText: "bar", newText: "baz" }] },
undefined,
undefined,
context,
),
).rejects.toThrow(/Could not find the exact text/);
await expect(
tool.execute(
"edit-4",
{ path: "edit.txt", edits: [{ oldText: "foo", newText: "bar" }] },
undefined,
undefined,
context,
),
).rejects.toThrow(/Found 3 occurrences/);
});
it("preserves BOM and CRLF line endings", async () => {
const context = createContext();
getOrThrow(await context.env.writeFile("edit.txt", "\uFEFFone\r\ntwo\r\n"));
await createEditTool().execute(
"edit-5",
{ path: "edit.txt", edits: [{ oldText: "two", newText: "TWO" }] },
undefined,
undefined,
context,
);
expect(getOrThrow(await context.env.readTextFile("edit.txt"))).toBe("\uFEFFone\r\nTWO\r\n");
});
});
describe("bash", () => {
it("executes commands and combines stdout and stderr", async () => {
const context = createContext();
const result = await createBashTool().execute(
"bash-1",
{ command: "printf out; printf err >&2" },
undefined,
undefined,
context,
);
expect(textOutput(result)).toContain("out");
expect(textOutput(result)).toContain("err");
});
it("reports nonzero exits and timeouts", async () => {
const context = createContext();
const tool = createBashTool();
await expect(
tool.execute("bash-2", { command: "printf failed; exit 7" }, undefined, undefined, context),
).rejects.toThrow(/failed[\s\S]*Command exited with code 7/);
await expect(
tool.execute("bash-3", { command: "sleep 2", timeout: 0.01 }, undefined, undefined, context),
).rejects.toThrow(/Command timed out after 0.01 seconds/);
});
it("preserves truncated output when a command times out", async () => {
const context = createContext();
let error: unknown;
try {
await createBashTool().execute(
"bash-timeout-output",
{
command: "i=1; while [ $i -le 3000 ]; do echo line-$i; i=$((i + 1)); done; sleep 2",
timeout: 0.05,
},
undefined,
undefined,
context,
);
} catch (cause) {
error = cause;
}
expect(error).toBeInstanceOf(Error);
const message = (error as Error).message;
expect(message).toContain("Command timed out after 0.05 seconds");
const fullOutputPath = message.match(/Full output: ([^\]\n]+)/)?.[1];
expect(fullOutputPath).toBeDefined();
const fullOutput = getOrThrow(await context.env.readTextFile(fullOutputPath!));
expect(fullOutput).toContain("line-1\nline-2");
expect(fullOutput).toContain("line-2999\nline-3000");
});
it("supports command prefixes", async () => {
const context = createContext();
const result = await createBashTool({ commandPrefix: "value=hello" }).execute(
"bash-4",
{ command: "printf $value" },
undefined,
undefined,
context,
);
expect(textOutput(result)).toBe("hello");
});
it("coalesces updates and persists truncated full output", async () => {
const context = createContext();
const updates: string[] = [];
const result = await createBashTool().execute(
"bash-5",
{ command: "i=1; while [ $i -le 3000 ]; do echo line-$i; i=$((i + 1)); done" },
undefined,
(update) => updates.push(textOutput(update)),
context,
);
expect(updates.length).toBeLessThan(25);
expect(result.details?.truncation).toMatchObject({
truncated: true,
truncatedBy: "lines",
totalLines: 3000,
outputLines: 2000,
});
expect(textOutput(result)).toContain("line-3000");
expect(result.details?.fullOutputPath).toBeDefined();
const fullOutput = getOrThrow(await context.env.readTextFile(result.details!.fullOutputPath!));
expect(fullOutput).toContain("line-1\nline-2");
expect(fullOutput).toContain("line-2999\nline-3000");
});
});
});
+7 -2
View File
@@ -7,6 +7,10 @@ import { NodeExecutionEnv } from "../../src/harness/env/nodejs.ts";
import { InMemorySessionStorage } from "../../src/harness/session/memory-storage.ts";
import {
AgentHarness,
createBashTool,
createEditTool,
createReadTool,
createWriteTool,
formatSkillsForSystemPrompt,
loadSourcedPromptTemplates,
loadSourcedSkills,
@@ -49,12 +53,13 @@ if (!model) {
const session = new Session(new InMemorySessionStorage());
const agent = new AgentHarness({
env,
session,
models,
model,
thinkingLevel: "low",
systemPrompt: ({ env, resources }) =>
tools: [createReadTool(), createWriteTool(), createEditTool(), createBashTool()],
toolContext: async () => ({ env, sessionId: (await session.getMetadata()).id }),
systemPrompt: ({ resources }) =>
[
"You are a helpful assistant.",
formatSkillsForSystemPrompt(resources.skills ?? []),