new_pull

2026-07-26 14:02:37 +07:00
parent bc56546b49
commit 367ebc1c7f
171 changed files with 4617 additions and 10402 deletions
@@ -0,0 +1,28 @@
+# Pi evals
+
+Behavioral evaluations for Pi using `vitest-evals`.
+
+The Pi harness in `src/pi-harness.ts` is built on `createHarness` from `vitest-evals/harness`. It drives a real
+`createAgentSession` in a temporary workspace and reports the final assistant message, transcript, and token usage.
+
+## Running
+
+From the repository root, run with an explicit provider and model:
+
+```bash
+npm run eval -- --provider openai-codex --model gpt-5.4
+```
+
+When invoked from a Pi Bash tool, the current session supplies `PI_PROVIDER` and `PI_MODEL`, so this is sufficient:
+
+```bash
+npm run eval
+```
+
+The runner requires both values and never falls back to another model. Additional arguments are forwarded to Vitest, for example:
+
+```bash
+npm run eval -- -t "capital of France"
+```
+
+Authentication is resolved by Pi's normal `ModelRuntime`. Subscription-backed providers such as `openai-codex` use credentials from the user's Pi configuration. API-backed providers use their standard environment variables, such as `OPENAI_API_KEY` for `openai`.
@@ -0,0 +1,18 @@
+{
+	"name": "@earendil-works/pi-evals",
+	"version": "0.82.1",
+	"private": true,
+	"type": "module",
+	"scripts": {
+		"clean": "shx rm -rf .eval",
+		"eval": "node scripts/run-evals.mjs"
+	},
+	"devDependencies": {
+		"@earendil-works/pi-coding-agent": "^0.82.1",
+		"@types/node": "24.12.4",
+		"shx": "0.4.0",
+		"typescript": "5.9.3",
+		"vitest-evals": "0.15.0",
+		"vitest": "4.1.9"
+	}
+}
@@ -0,0 +1,78 @@
+import { spawnSync } from "node:child_process";
+import { createRequire } from "node:module";
+import { dirname, resolve } from "node:path";
+import { fileURLToPath } from "node:url";
+
+const packageRoot = resolve(dirname(fileURLToPath(import.meta.url)), "..");
+const args = process.argv.slice(2);
+let provider;
+let model;
+let hasCliModelSelection = false;
+const vitestArgs = [];
+
+for (let index = 0; index < args.length; index += 1) {
+	const arg = args[index];
+	if (arg === "--provider" || arg === "--model") {
+		const value = args[index + 1];
+		if (!value || value.startsWith("-")) {
+			console.error(`Missing value for ${arg}`);
+			process.exit(1);
+		}
+		if (arg === "--provider") provider = value;
+		else model = value;
+		hasCliModelSelection = true;
+		index += 1;
+		continue;
+	}
+	if (arg.startsWith("--provider=")) {
+		provider = arg.slice("--provider=".length);
+		hasCliModelSelection = true;
+		continue;
+	}
+	if (arg.startsWith("--model=")) {
+		model = arg.slice("--model=".length);
+		hasCliModelSelection = true;
+		continue;
+	}
+	vitestArgs.push(arg);
+}
+
+if (hasCliModelSelection && (!provider || !model)) {
+	console.error("CLI model selection requires both --provider and --model.");
+	process.exit(1);
+}
+
+provider ??= process.env.PI_PROVIDER;
+model ??= process.env.PI_MODEL;
+
+if (!provider || !model) {
+	console.error(
+		"No eval model selected. Pass --provider and --model, or set PI_PROVIDER and PI_MODEL.",
+	);
+	process.exit(1);
+}
+
+const require = createRequire(import.meta.url);
+const vitestPackagePath = require.resolve("vitest/package.json");
+const vitestCliPath = resolve(dirname(vitestPackagePath), "vitest.mjs");
+
+console.error(`[eval] provider=${provider} model=${model}`);
+const result = spawnSync(
+	process.execPath,
+	[vitestCliPath, "run", "--config", "vitest.config.ts", ...vitestArgs],
+	{
+		cwd: packageRoot,
+		stdio: "inherit",
+		env: {
+			...process.env,
+			PI_PROVIDER: provider,
+			PI_MODEL: model,
+		},
+	},
+);
+
+if (result.error) {
+	throw result.error;
+}
+
+process.exit(result.status ?? 1);
@@ -0,0 +1,15 @@
+import { expect } from "vitest";
+import { describeEval } from "vitest-evals";
+import { piCodingAgentHarness } from "./pi-harness.ts";
+
+describeEval("general knowledge", { harness: piCodingAgentHarness }, (it) => {
+	it("knows the capital of France", async ({ run }) => {
+		const result = await run("What's the capital of France? Respond with only the city name.");
+
+		expect(result.output.trim()).toBe("Paris");
+		expect(result.errors).toEqual([]);
+		expect(result.usage.provider).toBe(process.env.PI_PROVIDER);
+		expect(result.usage.model).toBe(process.env.PI_MODEL);
+		expect(result.usage.totalTokens).toBeGreaterThan(0);
+	});
+});
@@ -0,0 +1,166 @@
+import { mkdir, mkdtemp, rm } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import {
+	createAgentSession,
+	DefaultResourceLoader,
+	ModelRuntime,
+	SessionManager,
+	SettingsManager,
+} from "@earendil-works/pi-coding-agent";
+import { createHarness, type SimpleHarnessResult } from "vitest-evals/harness";
+
+type EvalSession = Awaited<ReturnType<typeof createAgentSession>>["session"];
+
+async function cleanupEvalResources(session: EvalSession | undefined, removeRoot: () => Promise<void>): Promise<void> {
+	const failures: unknown[] = [];
+	if (session) {
+		try {
+			session.dispose();
+		} catch (error) {
+			failures.push(error);
+		}
+	}
+	try {
+		await removeRoot();
+	} catch (error) {
+		failures.push(error);
+	}
+	if (failures.length === 1) {
+		throw failures[0];
+	}
+	if (failures.length > 1) {
+		throw new AggregateError(failures, "Pi eval session disposal and temporary directory cleanup failed.");
+	}
+}
+
+function getRequiredModelSelection(): { provider: string; model: string } {
+	const provider = process.env.PI_PROVIDER?.trim();
+	const model = process.env.PI_MODEL?.trim();
+	if (!provider || !model) {
+		throw new Error("PI_PROVIDER and PI_MODEL must both be set for eval runs.");
+	}
+	return { provider, model };
+}
+
+async function runPiEval(input: string, signal: AbortSignal | undefined): Promise<SimpleHarnessResult<string>> {
+	signal?.throwIfAborted();
+	const selection = getRequiredModelSelection();
+	const modelRuntime = await ModelRuntime.create();
+	signal?.throwIfAborted();
+	const model = modelRuntime.getModel(selection.provider, selection.model);
+	if (!model) {
+		throw new Error(`Eval model not found: ${selection.provider}/${selection.model}`);
+	}
+	const auth = await modelRuntime.checkAuth(selection.provider);
+	signal?.throwIfAborted();
+	if (!auth) {
+		throw new Error(`No authentication configured for eval model ${selection.provider}/${selection.model}.`);
+	}
+
+	const root = await mkdtemp(join(tmpdir(), "pi-eval-"));
+	const cwd = join(root, "workspace");
+	const agentDir = join(root, "agent");
+	const cleanup = () => rm(root, { recursive: true, force: true });
+	let createdSession: EvalSession | undefined;
+	try {
+		signal?.throwIfAborted();
+		await Promise.all([mkdir(cwd), mkdir(agentDir)]);
+		signal?.throwIfAborted();
+
+		const settingsManager = SettingsManager.inMemory();
+		const resourceLoader = new DefaultResourceLoader({
+			cwd,
+			agentDir,
+			settingsManager,
+			noExtensions: true,
+			noSkills: true,
+			noPromptTemplates: true,
+			noThemes: true,
+			noContextFiles: true,
+		});
+		await resourceLoader.reload();
+		signal?.throwIfAborted();
+
+		createdSession = (
+			await createAgentSession({
+				cwd,
+				agentDir,
+				model,
+				modelRuntime,
+				thinkingLevel: "off",
+				noTools: "all",
+				resourceLoader,
+				settingsManager,
+				sessionManager: SessionManager.inMemory(cwd),
+			})
+		).session;
+		signal?.throwIfAborted();
+	} catch (error) {
+		try {
+			await cleanupEvalResources(createdSession, cleanup);
+		} catch (cleanupError) {
+			throw new AggregateError([error, cleanupError], "Pi eval setup failed and cleanup also failed.");
+		}
+		throw error;
+	}
+	if (!createdSession) {
+		throw new Error("Pi eval session setup completed without a session.");
+	}
+
+	const session = createdSession;
+	const abort = () => void session.abort();
+	signal?.addEventListener("abort", abort, { once: true });
+	let result: SimpleHarnessResult<string>;
+	try {
+		signal?.throwIfAborted();
+		await session.prompt(input);
+		const assistant = session.messages
+			.slice()
+			.reverse()
+			.find((message) => message.role === "assistant");
+		if (!assistant) {
+			throw new Error("Pi eval did not produce an assistant message.");
+		}
+		if (assistant.stopReason !== "stop") {
+			throw new Error(assistant.errorMessage ?? `Pi eval stopped unexpectedly: ${assistant.stopReason}`);
+		}
+
+		const output = session.getLastAssistantText();
+		if (!output) {
+			throw new Error("Pi eval produced an empty assistant response.");
+		}
+
+		const stats = session.getSessionStats();
+		result = {
+			output,
+			events: [
+				{ type: "message", role: "user", content: input },
+				{ type: "message", role: "assistant", content: output },
+			],
+			usage: {
+				provider: model.provider,
+				model: model.id,
+				inputTokens: stats.tokens.input,
+				outputTokens: stats.tokens.output,
+				totalTokens: stats.tokens.total,
+			},
+		};
+	} catch (error) {
+		signal?.removeEventListener("abort", abort);
+		try {
+			await cleanupEvalResources(session, cleanup);
+		} catch (cleanupError) {
+			throw new AggregateError([error, cleanupError], "Pi eval run failed and cleanup also failed.");
+		}
+		throw error;
+	}
+	signal?.removeEventListener("abort", abort);
+	await cleanupEvalResources(session, cleanup);
+	return result;
+}
+
+export const piCodingAgentHarness = createHarness<string, string>({
+	name: "pi-coding-agent",
+	run: ({ input, signal }) => runPiEval(input, signal),
+});
@@ -0,0 +1,10 @@
+{
+	"extends": "../../tsconfig.json",
+	"compilerOptions": {
+		"noEmit": true,
+		"module": "NodeNext",
+		"moduleResolution": "NodeNext",
+		"types": ["node", "vitest"]
+	},
+	"include": ["src/**/*.ts"]
+}
@@ -0,0 +1,19 @@
+import { defineConfig, mergeConfig } from "vitest/config";
+import baseConfig, { workspaceSourcePaths } from "../../vitest.base.ts";
+
+export default mergeConfig(
+	baseConfig,
+	defineConfig({
+		test: {
+			environment: "node",
+			fileParallelism: false,
+			include: ["src/**/*.eval.ts"],
+			testTimeout: 120000,
+			hookTimeout: 30000,
+			reporters: ["vitest-evals/reporter"],
+		},
+		resolve: {
+			alias: [{ find: /^@earendil-works\/pi-coding-agent$/, replacement: workspaceSourcePaths.codingAgentIndex }],
+		},
+	}),
+);