fix(ai): validate generated model data before builds

packages/ai/CHANGELOG.md

@@ -8,6 +8,10 @@
 - Added a shared `uuidv7` utility for time-ordered identifiers.
 - Added optional usage metadata to tool result messages ([#6671](https://github.com/earendil-works/pi/pull/6671) by [@davidbrai](https://github.com/davidbrai)).
 
+### Changed
+
+- Changed model generation to validate ignored provider data before compilation; `npm run build` refreshes model data as before, while `npm run build:offline` reuses existing data without network access.
+
 ### Fixed
 
 - Fixed sessionless OpenAI Codex WebSocket requests to use UUIDv7 request IDs, enabling models that reject UUIDv4 IDs.

packages/ai/package.json

@@ -49,10 +49,13 @@
 	],
 	"scripts": {
 		"clean": "shx rm -rf dist",
-		"generate-models": "node scripts/generate-models.ts",
+		"generate-models": "node scripts/generate-models.ts --strict",
+		"hydrate-model-data": "node scripts/generate-models.ts --strict --data-only",
 		"generate-model-catalog": "node scripts/generate-models.ts --strict --json-only --json-output ../../.artifacts/model-catalog",
-		"generate-image-models": "node scripts/generate-image-models.ts",
-		"build": "npm run generate-models && tsgo -p tsconfig.build.json && shx rm -rf dist/providers/data && shx cp -r src/providers/data dist/providers/data",
+		"generate-image-models": "node scripts/generate-image-models.ts --strict",
+		"check:model-data": "node scripts/check-model-data.ts",
+		"build": "npm run generate-models && npm run build:offline",
+		"build:offline": "npm run check:model-data && tsgo -p tsconfig.build.json && shx rm -rf dist/providers/data && shx cp -r src/providers/data dist/providers/data",
 		"test": "vitest --run",
 		"prepublishOnly": "npm run clean && npm run build"
 	},

packages/ai/scripts/check-model-data.ts

@@ -0,0 +1,16 @@
+#!/usr/bin/env node
+
+import { dirname, join } from "node:path";
+import { fileURLToPath } from "node:url";
+import { validateGeneratedModelData } from "./model-data.ts";
+
+const packageRoot = join(dirname(fileURLToPath(import.meta.url)), "..");
+
+try {
+	validateGeneratedModelData(packageRoot);
+	console.log("Generated model data is valid.");
+} catch (error) {
+	console.error(error instanceof Error ? error.message : String(error));
+	console.error("\nModel data is missing or stale. Run `npm run hydrate:model-data` from the repository root.");
+	process.exitCode = 1;
+}

packages/ai/scripts/generate-image-models.ts

@@ -1,7 +1,7 @@
 #!/usr/bin/env node
 
 import { writeFileSync } from "fs";
-import { dirname, join } from "path";
+import { dirname, join, resolve } from "path";
 import { fileURLToPath } from "url";
 import type { ImagesModel } from "../src/types.ts";
 
@@ -10,6 +10,13 @@ const __dirname = dirname(__filename);
 const packageRoot = join(__dirname, "..");
 const OPENROUTER_BASE_URL = "https://openrouter.ai/api/v1";
 
+function readStrictOption(args: string[]): boolean {
+	for (const arg of args) {
+		if (arg !== "--strict") throw new Error(`Unknown argument: ${arg}`);
+	}
+	return args.includes("--strict");
+}
+
 interface OpenRouterModelRecord {
 	id: string;
 	name: string;
@@ -26,52 +33,73 @@ interface OpenRouterModelRecord {
 	};
 }
 
-async function fetchOpenRouterImageModels(): Promise<ImagesModel<"openrouter-images">[]> {
+export function parseOpenRouterImageModels(
+	payload: unknown,
+	strict: boolean,
+): ImagesModel<"openrouter-images">[] {
+	const data =
+		typeof payload === "object" && payload !== null
+			? (payload as { data?: OpenRouterModelRecord[] }).data
+			: undefined;
+	if (!Array.isArray(data) || data.length === 0) {
+		if (strict) throw new Error("OpenRouter API returned a missing or empty image model list");
+		return [];
+	}
+
+	const models: ImagesModel<"openrouter-images">[] = [];
+	for (const model of data) {
+		const input = Array.from(
+			new Set(
+				(model.architecture?.input_modalities ?? []).filter(
+					(modality): modality is "text" | "image" => modality === "text" || modality === "image",
+				),
+			),
+		);
+		const output = Array.from(
+			new Set(
+				(model.architecture?.output_modalities ?? []).filter(
+					(modality): modality is "text" | "image" => modality === "text" || modality === "image",
+				),
+			),
+		);
+
+		if (!output.includes("image")) continue;
+		if (input.length === 0) input.push("text");
+
+		models.push({
+			id: model.id,
+			name: model.name,
+			api: "openrouter-images",
+			provider: "openrouter",
+			baseUrl: OPENROUTER_BASE_URL,
+			input,
+			output,
+			cost: {
+				input: parseFloat(model.pricing?.prompt || "0") * 1_000_000,
+				output: parseFloat(model.pricing?.completion || "0") * 1_000_000,
+				cacheRead: parseFloat(model.pricing?.input_cache_read || "0") * 1_000_000,
+				cacheWrite: parseFloat(model.pricing?.input_cache_write || "0") * 1_000_000,
+			},
+		});
+	}
+
+	if (strict && models.length === 0) {
+		throw new Error("OpenRouter API returned no usable image models");
+	}
+	return models;
+}
+
+async function fetchOpenRouterImageModels(strict: boolean): Promise<ImagesModel<"openrouter-images">[]> {
 	try {
 		console.log("Fetching image models from OpenRouter API...");
 		const response = await fetch(`${OPENROUTER_BASE_URL}/models?output_modalities=image`);
-		const data = (await response.json()) as { data?: OpenRouterModelRecord[] };
-		const models: ImagesModel<"openrouter-images">[] = [];
-
-		for (const model of data.data ?? []) {
-			const input = Array.from(
-				new Set(
-					(model.architecture?.input_modalities ?? [])
-						.filter((modality): modality is "text" | "image" => modality === "text" || modality === "image"),
-				),
-			);
-			const output = Array.from(
-				new Set(
-					(model.architecture?.output_modalities ?? []).filter(
-						(modality): modality is "text" | "image" => modality === "text" || modality === "image",
-					),
-				),
-			);
-
-			if (!output.includes("image")) continue;
-			if (input.length === 0) input.push("text");
-
-			models.push({
-				id: model.id,
-				name: model.name,
-				api: "openrouter-images",
-				provider: "openrouter",
-				baseUrl: OPENROUTER_BASE_URL,
-				input,
-				output,
-				cost: {
-					input: parseFloat(model.pricing?.prompt || "0") * 1_000_000,
-					output: parseFloat(model.pricing?.completion || "0") * 1_000_000,
-					cacheRead: parseFloat(model.pricing?.input_cache_read || "0") * 1_000_000,
-					cacheWrite: parseFloat(model.pricing?.input_cache_write || "0") * 1_000_000,
-				},
-			});
-		}
-
+		if (!response.ok) throw new Error(`OpenRouter API returned ${response.status}`);
+		const models = parseOpenRouterImageModels(await response.json(), strict);
 		console.log(`Fetched ${models.length} image models from OpenRouter`);
 		return models;
 	} catch (error) {
 		console.error("Failed to fetch OpenRouter image models:", error);
+		if (strict) throw error;
 		return [];
 	}
 }
@@ -118,14 +146,17 @@ ${providerEntries}
 }
 
 async function main(): Promise<void> {
-	const models = await fetchOpenRouterImageModels();
+	const strict = readStrictOption(process.argv.slice(2));
+	const models = await fetchOpenRouterImageModels(strict);
 	const output = generateImageModelsFile(models);
 	const outputPath = join(packageRoot, "src", "image-models.generated.ts");
 	writeFileSync(outputPath, output, "utf-8");
 	console.log(`Generated ${outputPath}`);
 }
 
-main().catch((error) => {
-	console.error(error);
-	process.exit(1);
-});
+if (process.argv[1] && resolve(process.argv[1]) === __filename) {
+	main().catch((error) => {
+		console.error(error);
+		process.exit(1);
+	});
+}

packages/ai/scripts/generate-models.ts

@@ -1,6 +1,6 @@
 #!/usr/bin/env node
 
-import { mkdirSync, readdirSync, rmSync, writeFileSync } from "fs";
+import { existsSync, mkdirSync, mkdtempSync, readFileSync, readdirSync, renameSync, rmSync, writeFileSync } from "fs";
 import { dirname, join, resolve } from "path";
 import { fileURLToPath } from "url";
 import {
@@ -18,6 +18,14 @@ import type {
 	OpenAICompletionsCompat,
 	OpenAIResponsesCompat,
 } from "../src/types.ts";
+import {
+	createModelDataManifest,
+	type ModelDataStructure,
+	MODEL_DATA_MANIFEST_FILE,
+	readModelDataStructure,
+	validateGeneratedModelData,
+	validateModelDataDirectory,
+} from "./model-data.ts";
 
 const __filename = fileURLToPath(import.meta.url);
 const __dirname = dirname(__filename);
@@ -25,11 +33,13 @@ const packageRoot = join(__dirname, "..");
 
 function readGeneratorOptions(args: string[]): {
 	strict: boolean;
+	dataOnly: boolean;
 	jsonOnly: boolean;
 	jsonOutputDir: string | undefined;
 	pretty: boolean;
 } {
 	let strict = false;
+	let dataOnly = false;
 	let jsonOnly = false;
 	let jsonOutputDir: string | undefined;
 	let pretty = false;
@@ -40,6 +50,10 @@ function readGeneratorOptions(args: string[]): {
 			strict = true;
 			continue;
 		}
+		if (arg === "--data-only") {
+			dataOnly = true;
+			continue;
+		}
 		if (arg === "--json-only") {
 			jsonOnly = true;
 			continue;
@@ -58,7 +72,8 @@ function readGeneratorOptions(args: string[]): {
 	}
 
 	if (jsonOnly && !jsonOutputDir) throw new Error("--json-only requires --json-output");
-	return { strict, jsonOnly, jsonOutputDir, pretty };
+	if (dataOnly && (jsonOnly || jsonOutputDir)) throw new Error("--data-only cannot be combined with JSON catalog output");
+	return { strict, dataOnly, jsonOnly, jsonOutputDir, pretty };
 }
 
 const generatorOptions = readGeneratorOptions(process.argv.slice(2));
@@ -2403,63 +2418,154 @@ async function generateModels() {
 			jsonProviders[providerId][modelId] = providers[providerId][modelId];
 		}
 	}
-	const writeJson = (path: string, value: unknown) =>
-		writeFileSync(path, `${JSON.stringify(value, null, generatorOptions.pretty ? 2 : undefined)}\n`);
+
+	const serializeJson = (value: unknown) => `${JSON.stringify(value, null, generatorOptions.pretty ? 2 : undefined)}\n`;
+	const writeJson = (path: string, value: unknown) => writeFileSync(path, serializeJson(value));
+	let generatedDataProviderIds = sortedProviderIds;
+	let generatedDataProviders = jsonProviders;
+	let modelDataStructure: ModelDataStructure = Object.fromEntries(
+		sortedProviderIds.map((providerId) => [
+			providerId,
+			Object.fromEntries(
+				Object.entries(jsonProviders[providerId]).map(([modelId, model]) => [modelId, model.api]),
+			),
+		]),
+	);
+
+	if (generatorOptions.dataOnly) {
+		modelDataStructure = readModelDataStructure(packageRoot);
+		generatedDataProviderIds = Object.keys(modelDataStructure);
+		const hydratedProviders: typeof jsonProviders = {};
+		const hydrationErrors: string[] = [];
+		for (const [providerId, expectedModels] of Object.entries(modelDataStructure)) {
+			hydratedProviders[providerId] = {};
+			for (const [modelId, expectedApi] of Object.entries(expectedModels)) {
+				const model = jsonProviders[providerId]?.[modelId];
+				if (!model) {
+					hydrationErrors.push(`missing ${providerId}/${modelId}`);
+					continue;
+				}
+				if (model.api !== expectedApi) {
+					hydrationErrors.push(`${providerId}/${modelId} uses ${model.api}, expected ${expectedApi}`);
+					continue;
+				}
+				hydratedProviders[providerId][modelId] = model;
+			}
+		}
+		if (hydrationErrors.length > 0) {
+			throw new Error(`Cannot hydrate the committed model catalog:\n${hydrationErrors.map((error) => `  - ${error}`).join("\n")}`);
+		}
+		generatedDataProviders = hydratedProviders;
+	}
 
 	if (!generatorOptions.jsonOnly) {
-		// Generate TypeScript structural catalogs and adjacent JSON values.
-		const generatedHeader = `// This file is auto-generated by scripts/generate-models.ts
+		// Stage and validate all provider values before replacing the current generated data.
+		const providersDir = join(packageRoot, "src/providers");
+		const dataDir = join(providersDir, "data");
+		const stagingRoot = mkdtempSync(join(providersDir, ".model-generation-"));
+		const stagedDataDir = join(stagingRoot, "data");
+		const previousDataDir = join(stagingRoot, "previous-data");
+		let restoreStructuralCatalog: (() => void) | undefined;
+		try {
+			mkdirSync(stagedDataDir, { recursive: true });
+			const fileContents: Record<string, string> = {};
+			for (const providerId of generatedDataProviderIds) {
+				const filename = `${providerId}.json`;
+				const content = serializeJson(generatedDataProviders[providerId]);
+				fileContents[filename] = content;
+				writeFileSync(join(stagedDataDir, filename), content);
+			}
+			writeJson(
+				join(stagedDataDir, MODEL_DATA_MANIFEST_FILE),
+				createModelDataManifest(modelDataStructure, fileContents),
+			);
+			validateModelDataDirectory(modelDataStructure, stagedDataDir);
+
+			if (!generatorOptions.dataOnly) {
+				// Generate TypeScript structural catalogs only after the model data is complete and valid.
+				const previousShardContents = new Map(
+					readdirSync(providersDir)
+						.filter((entry) => entry.endsWith(".models.ts"))
+						.map((entry) => [entry, readFileSync(join(providersDir, entry), "utf8")] as const),
+				);
+				const aggregatorPath = join(packageRoot, "src/models.generated.ts");
+				const previousAggregator = readFileSync(aggregatorPath, "utf8");
+				restoreStructuralCatalog = () => {
+					for (const entry of readdirSync(providersDir)) {
+						if (entry.endsWith(".models.ts")) rmSync(join(providersDir, entry));
+					}
+					for (const [entry, content] of previousShardContents) {
+						writeFileSync(join(providersDir, entry), content);
+					}
+					writeFileSync(aggregatorPath, previousAggregator);
+				};
+
+				const generatedHeader = `// This file is auto-generated by scripts/generate-models.ts
 // Do not edit manually - run 'npm run generate-models' to update
 
 `;
-		const catalogConstName = (providerId: string) =>
-			`${providerId.toUpperCase().replace(/[^A-Z0-9]+/g, "_")}_MODELS`;
-		const providersDir = join(packageRoot, "src/providers");
-		const dataDir = join(providersDir, "data");
+				const catalogConstName = (providerId: string) =>
+					`${providerId.toUpperCase().replace(/[^A-Z0-9]+/g, "_")}_MODELS`;
+				const generatedShardFiles = new Set<string>();
 
-		function emitModelShape(model: Model<any>, indent: string): string {
-			return `${indent}${JSON.stringify(model.id)}: Model<${JSON.stringify(model.api)}> & {\n${indent}\tid: ${JSON.stringify(model.id)};\n${indent}\tprovider: ${JSON.stringify(model.provider)};\n${indent}};\n`;
-		}
+				function emitModelShape(model: Model<any>, indent: string): string {
+					return `${indent}${JSON.stringify(model.id)}: Model<${JSON.stringify(model.api)}> & {\n${indent}\tid: ${JSON.stringify(model.id)};\n${indent}\tprovider: ${JSON.stringify(model.provider)};\n${indent}};\n`;
+				}
 
-		// Remove stale per-provider catalogs and their generated values.
-		for (const entry of readdirSync(providersDir)) {
-			if (entry.endsWith(".models.ts")) {
-				rmSync(join(providersDir, entry));
+				for (const providerId of sortedProviderIds) {
+					const models = providers[providerId];
+					let output = generatedHeader;
+					output += `import values from "./data/${providerId}.json" with { type: "json" };\n`;
+					output += `import type { Model } from "../types.ts";\n\n`;
+					output += `export const ${catalogConstName(providerId)} = values as {\n`;
+					for (const modelId of Object.keys(models).sort()) {
+						output += emitModelShape(models[modelId], "\t");
+					}
+					output += `};\n`;
+					const filename = `${providerId}.models.ts`;
+					generatedShardFiles.add(filename);
+					writeFileSync(join(providersDir, filename), output);
+				}
+				for (const entry of readdirSync(providersDir)) {
+					if (entry.endsWith(".models.ts") && !generatedShardFiles.has(entry)) rmSync(join(providersDir, entry));
+				}
+				console.log(`Generated ${sortedProviderIds.length} catalog structures under src/providers/`);
+
+				let output = generatedHeader;
+				for (const providerId of sortedProviderIds) {
+					output += `import { ${catalogConstName(providerId)} } from "./providers/${providerId}.models.ts";\n`;
+				}
+				output += `\nexport const MODELS = {\n`;
+				for (const providerId of sortedProviderIds) {
+					output += `\t${JSON.stringify(providerId)}: ${catalogConstName(providerId)},\n`;
+				}
+				output += `} as const;\n`;
+				writeFileSync(aggregatorPath, output);
+				console.log("Generated src/models.generated.ts");
 			}
-		}
-		rmSync(dataDir, { recursive: true, force: true });
-		mkdirSync(dataDir, { recursive: true });
 
-		// Per-provider catalog structure and values (sorted for deterministic output).
-		for (const providerId of sortedProviderIds) {
-			const models = providers[providerId];
-			const sortedModelIds = Object.keys(models).sort();
-			let output = generatedHeader;
-			output += `import values from "./data/${providerId}.json" with { type: "json" };\n`;
-			output += `import type { Model } from "../types.ts";\n\n`;
-			output += `export const ${catalogConstName(providerId)} = values as {\n`;
-			for (const modelId of sortedModelIds) {
-				output += emitModelShape(models[modelId], "\t");
+			const hadPreviousData = existsSync(dataDir);
+			if (hadPreviousData) renameSync(dataDir, previousDataDir);
+			try {
+				renameSync(stagedDataDir, dataDir);
+				validateGeneratedModelData(packageRoot);
+			} catch (error) {
+				rmSync(dataDir, { recursive: true, force: true });
+				if (hadPreviousData && existsSync(previousDataDir)) renameSync(previousDataDir, dataDir);
+				throw error;
 			}
-			output += `};\n`;
-			writeFileSync(join(providersDir, `${providerId}.models.ts`), output);
-			writeJson(join(dataDir, `${providerId}.json`), jsonProviders[providerId]);
+			restoreStructuralCatalog = undefined;
+			console.log(
+				generatorOptions.dataOnly
+					? "Hydrated JSON model values under src/providers/data/"
+					: "Generated JSON model values under src/providers/data/",
+			);
+		} catch (error) {
+			restoreStructuralCatalog?.();
+			throw error;
+		} finally {
+			rmSync(stagingRoot, { recursive: true, force: true });
 		}
-		console.log(`Generated ${sortedProviderIds.length} catalog structures under src/providers/`);
-		console.log("Generated JSON model values under src/providers/data/");
-
-		// Aggregator
-		let output = generatedHeader;
-		for (const providerId of sortedProviderIds) {
-			output += `import { ${catalogConstName(providerId)} } from "./providers/${providerId}.models.ts";\n`;
-		}
-		output += `\nexport const MODELS = {\n`;
-		for (const providerId of sortedProviderIds) {
-			output += `\t${JSON.stringify(providerId)}: ${catalogConstName(providerId)},\n`;
-		}
-		output += `} as const;\n`;
-		writeFileSync(join(packageRoot, "src/models.generated.ts"), output);
-		console.log("Generated src/models.generated.ts");
 	}
 
 	if (generatorOptions.jsonOutputDir) {

packages/ai/scripts/model-data.ts

@@ -0,0 +1,261 @@
+import { createHash } from "node:crypto";
+import { existsSync, readFileSync, readdirSync, statSync } from "node:fs";
+import { join } from "node:path";
+
+export const MODEL_DATA_SCHEMA_VERSION = 1;
+export const MODEL_DATA_MANIFEST_FILE = ".manifest.json";
+
+export type ModelDataStructure = Record<string, Record<string, string>>;
+
+export interface ModelDataManifest {
+	schemaVersion: number;
+	structureHash: string;
+	files: Record<string, string>;
+}
+
+const JSON_STRING_PATTERN = '"(?:\\\\.|[^"\\\\])*"';
+const MODEL_SHAPE_PATTERN = new RegExp(`^\\t(${JSON_STRING_PATTERN}): Model<(${JSON_STRING_PATTERN})> & \\{$`);
+const MODEL_ID_PATTERN = new RegExp(`^\\t\\tid: (${JSON_STRING_PATTERN});$`);
+const MODEL_PROVIDER_PATTERN = new RegExp(`^\\t\\tprovider: (${JSON_STRING_PATTERN});$`);
+
+function sha256(value: string): string {
+	return createHash("sha256").update(value).digest("hex");
+}
+
+function parseJsonString(value: string, description: string): string {
+	const parsed: unknown = JSON.parse(value);
+	if (typeof parsed !== "string") throw new Error(`${description} is not a string`);
+	return parsed;
+}
+
+function sortedRecord<T>(entries: Iterable<readonly [string, T]>): Record<string, T> {
+	return Object.fromEntries(Array.from(entries).sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0)));
+}
+
+function sameStrings(a: readonly string[], b: readonly string[]): boolean {
+	return a.length === b.length && a.every((value, index) => value === b[index]);
+}
+
+function describeSetDifference(expected: readonly string[], actual: readonly string[]): string {
+	const expectedSet = new Set(expected);
+	const actualSet = new Set(actual);
+	const missing = expected.filter((value) => !actualSet.has(value));
+	const extra = actual.filter((value) => !expectedSet.has(value));
+	return [missing.length > 0 ? `missing: ${missing.join(", ")}` : "", extra.length > 0 ? `extra: ${extra.join(", ")}` : ""]
+		.filter(Boolean)
+		.join("; ");
+}
+
+function parseProviderStructure(path: string, providerId: string): Record<string, string> {
+	const source = readFileSync(path, "utf8");
+	const expectedImport = `import values from "./data/${providerId}.json" with { type: "json" };`;
+	if (!source.includes(expectedImport)) {
+		throw new Error(`${path} does not import ${providerId}.json`);
+	}
+
+	const models = new Map<string, string>();
+	const lines = source.split("\n");
+	for (let index = 0; index < lines.length; index++) {
+		const shapeMatch = MODEL_SHAPE_PATTERN.exec(lines[index]);
+		if (!shapeMatch) continue;
+
+		const idMatch = MODEL_ID_PATTERN.exec(lines[index + 1] ?? "");
+		const providerMatch = MODEL_PROVIDER_PATTERN.exec(lines[index + 2] ?? "");
+		if (!idMatch || !providerMatch || lines[index + 3] !== "\t};") {
+			throw new Error(`${path}:${index + 1} has a malformed generated model declaration`);
+		}
+
+		const key = parseJsonString(shapeMatch[1], `${path}:${index + 1} model key`);
+		const api = parseJsonString(shapeMatch[2], `${path}:${index + 1} model API`);
+		const id = parseJsonString(idMatch[1], `${path}:${index + 2} model ID`);
+		const provider = parseJsonString(providerMatch[1], `${path}:${index + 3} provider ID`);
+		if (id !== key) throw new Error(`${path}:${index + 1} declares key ${key} with ID ${id}`);
+		if (provider !== providerId) {
+			throw new Error(`${path}:${index + 1} declares provider ${provider} instead of ${providerId}`);
+		}
+		if (models.has(key)) throw new Error(`${path} declares model ${key} more than once`);
+		models.set(key, api);
+		index += 3;
+	}
+
+	if (models.size === 0) throw new Error(`${path} contains no generated model declarations`);
+	return sortedRecord(models);
+}
+
+export function readModelDataStructure(packageRoot: string): ModelDataStructure {
+	const providersDir = join(packageRoot, "src", "providers");
+	const shardProviderIds = readdirSync(providersDir)
+		.filter((entry) => entry.endsWith(".models.ts"))
+		.map((entry) => entry.slice(0, -".models.ts".length))
+		.sort();
+	if (shardProviderIds.length === 0) throw new Error(`No generated provider shards found under ${providersDir}`);
+
+	const aggregator = readFileSync(join(packageRoot, "src", "models.generated.ts"), "utf8");
+	const importedProviderIds = Array.from(
+		aggregator.matchAll(/^import \{ [A-Z0-9_]+_MODELS \} from "\.\/providers\/([^"/]+)\.models\.ts";$/gm),
+		(match) => match[1],
+	).sort();
+	if (!sameStrings(shardProviderIds, importedProviderIds)) {
+		throw new Error(
+			`Generated model aggregator and provider shards do not match (${describeSetDifference(shardProviderIds, importedProviderIds)})`,
+		);
+	}
+
+	return sortedRecord(
+		shardProviderIds.map((providerId) => [
+			providerId,
+			parseProviderStructure(join(providersDir, `${providerId}.models.ts`), providerId),
+		] as const),
+	);
+}
+
+export function modelDataStructureHash(structure: ModelDataStructure): string {
+	return sha256(JSON.stringify(structure));
+}
+
+export function createModelDataManifest(
+	structure: ModelDataStructure,
+	fileContents: Readonly<Record<string, string>>,
+): ModelDataManifest {
+	return {
+		schemaVersion: MODEL_DATA_SCHEMA_VERSION,
+		structureHash: modelDataStructureHash(structure),
+		files: sortedRecord(Object.entries(fileContents).map(([file, content]) => [file, sha256(content)] as const)),
+	};
+}
+
+function isRecord(value: unknown): value is Record<string, unknown> {
+	return typeof value === "object" && value !== null && !Array.isArray(value);
+}
+
+function readJsonObject(path: string, description: string, errors: string[]): Record<string, unknown> | undefined {
+	let parsed: unknown;
+	try {
+		parsed = JSON.parse(readFileSync(path, "utf8"));
+	} catch (error) {
+		errors.push(`${description} is not valid JSON: ${error instanceof Error ? error.message : String(error)}`);
+		return undefined;
+	}
+	if (!isRecord(parsed)) {
+		errors.push(`${description} must contain a JSON object`);
+		return undefined;
+	}
+	return parsed;
+}
+
+function validateModelValue(
+	value: unknown,
+	providerId: string,
+	modelId: string,
+	expectedApi: string,
+	errors: string[],
+): void {
+	const label = `${providerId}/${modelId}`;
+	if (!isRecord(value)) {
+		errors.push(`${label} must be an object`);
+		return;
+	}
+	if (value.id !== modelId) errors.push(`${label} has id ${JSON.stringify(value.id)}, expected ${JSON.stringify(modelId)}`);
+	if (value.provider !== providerId) {
+		errors.push(`${label} has provider ${JSON.stringify(value.provider)}, expected ${JSON.stringify(providerId)}`);
+	}
+	if (value.api !== expectedApi) {
+		errors.push(`${label} has api ${JSON.stringify(value.api)}, expected ${JSON.stringify(expectedApi)}`);
+	}
+	if (typeof value.name !== "string" || value.name.length === 0) errors.push(`${label} has no model name`);
+	if (typeof value.baseUrl !== "string") errors.push(`${label} has no baseUrl string`);
+	if (typeof value.reasoning !== "boolean") errors.push(`${label} has no reasoning boolean`);
+	if (
+		!Array.isArray(value.input) ||
+		value.input.length === 0 ||
+		value.input.some((entry) => entry !== "text" && entry !== "image")
+	) {
+		errors.push(`${label} has invalid input modalities`);
+	}
+	if (typeof value.contextWindow !== "number" || !Number.isFinite(value.contextWindow) || value.contextWindow <= 0) {
+		errors.push(`${label} has invalid contextWindow`);
+	}
+	if (typeof value.maxTokens !== "number" || !Number.isFinite(value.maxTokens) || value.maxTokens <= 0) {
+		errors.push(`${label} has invalid maxTokens`);
+	}
+	if (!isRecord(value.cost)) {
+		errors.push(`${label} has invalid cost metadata`);
+	} else {
+		for (const field of ["input", "output", "cacheRead", "cacheWrite"] as const) {
+			const cost = value.cost[field];
+			if (typeof cost !== "number" || !Number.isFinite(cost)) {
+				errors.push(`${label} has invalid cost.${field}`);
+			}
+		}
+	}
+}
+
+function throwValidationErrors(errors: string[]): never {
+	const visible = errors.slice(0, 30);
+	const suffix = errors.length > visible.length ? `\n  ... and ${errors.length - visible.length} more` : "";
+	throw new Error(`Invalid generated model data:\n${visible.map((error) => `  - ${error}`).join("\n")}${suffix}`);
+}
+
+export function validateModelDataDirectory(structure: ModelDataStructure, dataDir: string): void {
+	if (!existsSync(dataDir) || !statSync(dataDir).isDirectory()) {
+		throw new Error(`Generated model data directory does not exist: ${dataDir}`);
+	}
+
+	const errors: string[] = [];
+	const expectedFiles = Object.keys(structure)
+		.map((providerId) => `${providerId}.json`)
+		.sort();
+	const actualFiles = readdirSync(dataDir)
+		.filter((entry) => entry.endsWith(".json") && entry !== MODEL_DATA_MANIFEST_FILE)
+		.sort();
+	if (!sameStrings(expectedFiles, actualFiles)) {
+		errors.push(`provider data files do not match the structural catalog (${describeSetDifference(expectedFiles, actualFiles)})`);
+	}
+
+	const manifestPath = join(dataDir, MODEL_DATA_MANIFEST_FILE);
+	const manifest = readJsonObject(manifestPath, "model data manifest", errors);
+	if (manifest?.schemaVersion !== MODEL_DATA_SCHEMA_VERSION) {
+		errors.push(
+			`model data schema is ${JSON.stringify(manifest?.schemaVersion)}, expected ${MODEL_DATA_SCHEMA_VERSION}`,
+		);
+	}
+	const expectedStructureHash = modelDataStructureHash(structure);
+	if (manifest?.structureHash !== expectedStructureHash) {
+		errors.push("model data generation stamp does not match the structural catalog");
+	}
+	const manifestFiles = isRecord(manifest?.files) ? manifest.files : undefined;
+	if (!manifestFiles) errors.push("model data manifest has no file hashes");
+	else {
+		const manifestFileNames = Object.keys(manifestFiles).sort();
+		if (!sameStrings(expectedFiles, manifestFileNames)) {
+			errors.push(`manifest file hashes do not match provider data files (${describeSetDifference(expectedFiles, manifestFileNames)})`);
+		}
+	}
+
+	for (const [providerId, expectedModels] of Object.entries(structure)) {
+		const filename = `${providerId}.json`;
+		const path = join(dataDir, filename);
+		if (!existsSync(path)) continue;
+		const content = readFileSync(path, "utf8");
+		if (manifestFiles && manifestFiles[filename] !== sha256(content)) {
+			errors.push(`${filename} does not match its manifest hash`);
+		}
+		const values = readJsonObject(path, filename, errors);
+		if (!values) continue;
+		const expectedModelIds = Object.keys(expectedModels).sort();
+		const actualModelIds = Object.keys(values).sort();
+		if (!sameStrings(expectedModelIds, actualModelIds)) {
+			errors.push(`${filename} model IDs do not match the structural catalog (${describeSetDifference(expectedModelIds, actualModelIds)})`);
+		}
+		for (const [modelId, api] of Object.entries(expectedModels)) {
+			if (modelId in values) validateModelValue(values[modelId], providerId, modelId, api, errors);
+		}
+	}
+
+	if (errors.length > 0) throwValidationErrors(errors);
+}
+
+export function validateGeneratedModelData(packageRoot: string): void {
+	const structure = readModelDataStructure(packageRoot);
+	validateModelDataDirectory(structure, join(packageRoot, "src", "providers", "data"));
+}

packages/ai/test/image-model-data.test.ts

@@ -0,0 +1,47 @@
+import { describe, expect, it } from "vitest";
+import { parseOpenRouterImageModels } from "../scripts/generate-image-models.ts";
+
+const validImageModel = {
+	id: "example/image-model",
+	name: "Example Image Model",
+	architecture: {
+		input_modalities: ["text", "image"],
+		output_modalities: ["image"],
+	},
+	pricing: {
+		prompt: "0.000001",
+		completion: "0.000002",
+	},
+};
+
+describe("OpenRouter image model parsing", () => {
+	it.each([{}, { data: [] }, { data: "invalid" }])("rejects a missing or empty strict catalog", (payload) => {
+		expect(() => parseOpenRouterImageModels(payload, true)).toThrow("missing or empty image model list");
+	});
+
+	it("rejects a strict catalog with no usable image models", () => {
+		expect(() =>
+			parseOpenRouterImageModels(
+				{
+					data: [
+						{
+							...validImageModel,
+							architecture: { input_modalities: ["text"], output_modalities: ["text"] },
+						},
+					],
+				},
+				true,
+			),
+		).toThrow("no usable image models");
+	});
+
+	it("parses a non-empty image model catalog", () => {
+		expect(parseOpenRouterImageModels({ data: [validImageModel] }, true)).toEqual([
+			expect.objectContaining({
+				id: "example/image-model",
+				input: ["text", "image"],
+				output: ["image"],
+			}),
+		]);
+	});
+});

packages/ai/test/model-data-validation.test.ts

@@ -0,0 +1,123 @@
+import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { afterEach, describe, expect, it } from "vitest";
+import {
+	createModelDataManifest,
+	MODEL_DATA_MANIFEST_FILE,
+	MODEL_DATA_SCHEMA_VERSION,
+	type ModelDataStructure,
+	readModelDataStructure,
+	validateModelDataDirectory,
+} from "../scripts/model-data.ts";
+
+const temporaryRoots: string[] = [];
+
+afterEach(() => {
+	for (const root of temporaryRoots.splice(0)) rmSync(root, { force: true, recursive: true });
+});
+
+function createFixture(): {
+	dataDir: string;
+	packageRoot: string;
+	structure: ModelDataStructure;
+	values: Record<string, unknown>;
+} {
+	const packageRoot = mkdtempSync(join(tmpdir(), "pi-model-data-"));
+	temporaryRoots.push(packageRoot);
+	const providersDir = join(packageRoot, "src", "providers");
+	const dataDir = join(providersDir, "data");
+	mkdirSync(dataDir, { recursive: true });
+	writeFileSync(
+		join(packageRoot, "src", "models.generated.ts"),
+		'import { TEST_PROVIDER_MODELS } from "./providers/test-provider.models.ts";\n\nexport const MODELS = {\n\t"test-provider": TEST_PROVIDER_MODELS,\n} as const;\n',
+	);
+	writeFileSync(
+		join(providersDir, "test-provider.models.ts"),
+		'// generated\n\nimport values from "./data/test-provider.json" with { type: "json" };\nimport type { Model } from "../types.ts";\n\nexport const TEST_PROVIDER_MODELS = values as {\n\t"model-a": Model<"openai-completions"> & {\n\t\tid: "model-a";\n\t\tprovider: "test-provider";\n\t};\n};\n',
+	);
+
+	const structure = readModelDataStructure(packageRoot);
+	const values: Record<string, unknown> = {
+		"model-a": {
+			id: "model-a",
+			name: "Model A",
+			api: "openai-completions",
+			provider: "test-provider",
+			baseUrl: "https://example.test/v1",
+			reasoning: false,
+			input: ["text"],
+			cost: { input: 1, output: 2, cacheRead: 0, cacheWrite: 0 },
+			contextWindow: 1000,
+			maxTokens: 100,
+		},
+	};
+	writeFixtureData(dataDir, structure, values);
+	return { dataDir, packageRoot, structure, values };
+}
+
+function writeFixtureData(
+	dataDir: string,
+	structure: ModelDataStructure,
+	values: Record<string, unknown>,
+	manifestSchemaVersion = MODEL_DATA_SCHEMA_VERSION,
+): void {
+	const filename = "test-provider.json";
+	const content = `${JSON.stringify(values)}\n`;
+	writeFileSync(join(dataDir, filename), content);
+	const manifest = createModelDataManifest(structure, { [filename]: content });
+	manifest.schemaVersion = manifestSchemaVersion;
+	writeFileSync(join(dataDir, MODEL_DATA_MANIFEST_FILE), `${JSON.stringify(manifest)}\n`);
+}
+
+describe("generated model data validation", () => {
+	it("validates complete data against generated structural catalogs", () => {
+		const { dataDir, structure } = createFixture();
+		expect(() => validateModelDataDirectory(structure, dataDir)).not.toThrow();
+	});
+
+	it("rejects a missing model data directory", () => {
+		const { dataDir, structure } = createFixture();
+		rmSync(dataDir, { recursive: true });
+		expect(() => validateModelDataDirectory(structure, dataDir)).toThrow("does not exist");
+	});
+
+	it.each([
+		["id", "wrong-id", "has id"],
+		["provider", "wrong-provider", "has provider"],
+		["api", "anthropic-messages", "has api"],
+	] as const)("rejects a wrong model %s", (field, value, expectedMessage) => {
+		const fixture = createFixture();
+		const model = fixture.values["model-a"] as Record<string, unknown>;
+		model[field] = value;
+		writeFixtureData(fixture.dataDir, fixture.structure, fixture.values);
+		expect(() => validateModelDataDirectory(fixture.structure, fixture.dataDir)).toThrow(expectedMessage);
+	});
+
+	it("rejects missing model IDs and stale file hashes", () => {
+		const fixture = createFixture();
+		writeFileSync(join(fixture.dataDir, "test-provider.json"), "{}\n");
+		expect(() => validateModelDataDirectory(fixture.structure, fixture.dataDir)).toThrow(/manifest hash|model IDs/);
+	});
+
+	it("rejects incompatible schema and generation stamps", () => {
+		const fixture = createFixture();
+		writeFixtureData(fixture.dataDir, fixture.structure, fixture.values, MODEL_DATA_SCHEMA_VERSION + 1);
+		expect(() => validateModelDataDirectory(fixture.structure, fixture.dataDir)).toThrow("model data schema");
+
+		const manifestPath = join(fixture.dataDir, MODEL_DATA_MANIFEST_FILE);
+		const manifest = JSON.parse(readFileSync(manifestPath, "utf8")) as Record<string, unknown>;
+		manifest.structureHash = "stale";
+		writeFileSync(manifestPath, `${JSON.stringify(manifest)}\n`);
+		expect(() => validateModelDataDirectory(fixture.structure, fixture.dataDir)).toThrow("generation stamp");
+	});
+
+	it("rejects missing provider shards referenced by the aggregator", () => {
+		const { packageRoot } = createFixture();
+		writeFileSync(
+			join(packageRoot, "src", "models.generated.ts"),
+			'import { TEST_PROVIDER_MODELS } from "./providers/test-provider.models.ts";\nimport { MISSING_MODELS } from "./providers/missing.models.ts";\n',
+		);
+		expect(() => readModelDataStructure(packageRoot)).toThrow("aggregator and provider shards do not match");
+	});
+});