From c8c3cd499f4d35c0f9cfebfec5f4e3822411a49f Mon Sep 17 00:00:00 2001 From: Armin Ronacher Date: Mon, 20 Jul 2026 22:28:48 +0200 Subject: [PATCH] fix(ai): validate generated model data before builds --- .gitignore | 1 + README.md | 5 +- package.json | 3 + packages/ai/CHANGELOG.md | 4 + packages/ai/package.json | 9 +- packages/ai/scripts/check-model-data.ts | 16 ++ packages/ai/scripts/generate-image-models.ts | 121 +++++--- packages/ai/scripts/generate-models.ts | 202 ++++++++++---- packages/ai/scripts/model-data.ts | 261 ++++++++++++++++++ packages/ai/test/image-model-data.test.ts | 47 ++++ .../ai/test/model-data-validation.test.ts | 123 +++++++++ scripts/check-browser-smoke.mjs | 2 +- scripts/local-release.mjs | 8 +- scripts/release.mjs | 1 + 14 files changed, 700 insertions(+), 103 deletions(-) create mode 100644 packages/ai/scripts/check-model-data.ts create mode 100644 packages/ai/scripts/model-data.ts create mode 100644 packages/ai/test/image-model-data.test.ts create mode 100644 packages/ai/test/model-data-validation.test.ts diff --git a/.gitignore b/.gitignore index 8cfb05c7..0e241a9d 100644 --- a/.gitignore +++ b/.gitignore @@ -9,6 +9,7 @@ packages/*/dist/ packages/*/dist-chrome/ packages/*/dist-firefox/ packages/ai/src/providers/data/ +packages/ai/src/providers/.model-generation-*/ *.cpuprofile # Environment diff --git a/README.md b/README.md index 130a412f..900779b0 100644 --- a/README.md +++ b/README.md @@ -52,8 +52,9 @@ See [CONTRIBUTING.md](CONTRIBUTING.md) for contribution guidelines and [AGENTS.m ```bash npm install --ignore-scripts # Install all dependencies without running lifecycle scripts -npm run build # Build all packages -npm run check # Lint, format, and type check +npm run build # Refresh model data, then build all packages +npm run build:offline # Rebuild using existing model data without network access +npm run check # Lint, format, and type check ./test.sh # Run tests (skips LLM-dependent tests without API keys) ./pi-test.sh # Run pi from sources (can be run from any directory) ``` diff --git a/package.json b/package.json index c919d33e..fc3dd5db 100644 --- a/package.json +++ b/package.json @@ -13,6 +13,7 @@ "scripts": { "clean": "npm run clean --workspaces", "build": "cd packages/tui && npm run build && cd ../ai && npm run build && cd ../agent && npm run build && cd ../coding-agent && npm run build && cd ../orchestrator && npm run build", + "build:offline": "cd packages/tui && npm run build && cd ../ai && npm run build:offline && cd ../agent && npm run build && cd ../coding-agent && npm run build && cd ../orchestrator && npm run build", "check": "biome check --write --error-on-warnings . && npm run check:pinned-deps && npm run check:ts-imports && npm run check:shrinkwrap && npm run check:install-lock:coding-agent && tsgo --noEmit && npm run check:browser-smoke", "check:browser-smoke": "node scripts/check-browser-smoke.mjs", "check:pinned-deps": "node scripts/check-pinned-deps.mjs", @@ -20,6 +21,8 @@ "check:install-lock:coding-agent": "node scripts/generate-coding-agent-install-lock.mjs --check", "check:ts-imports": "node scripts/check-ts-relative-imports.mjs", "generate:models": "npm --prefix packages/ai run generate-models && npm --prefix packages/ai run generate-image-models", + "hydrate:model-data": "npm --prefix packages/ai run hydrate-model-data", + "check:model-data": "npm --prefix packages/ai run check:model-data", "generate:model-catalog": "npm --prefix packages/ai run generate-model-catalog", "diff:model-catalog": "node scripts/diff-model-catalog.mjs", "check:model-catalog": "node scripts/publish-model-catalog.mjs --input .artifacts/model-catalog --dry-run", diff --git a/packages/ai/CHANGELOG.md b/packages/ai/CHANGELOG.md index 56693a28..b3a89736 100644 --- a/packages/ai/CHANGELOG.md +++ b/packages/ai/CHANGELOG.md @@ -8,6 +8,10 @@ - Added a shared `uuidv7` utility for time-ordered identifiers. - Added optional usage metadata to tool result messages ([#6671](https://github.com/earendil-works/pi/pull/6671) by [@davidbrai](https://github.com/davidbrai)). +### Changed + +- Changed model generation to validate ignored provider data before compilation; `npm run build` refreshes model data as before, while `npm run build:offline` reuses existing data without network access. + ### Fixed - Fixed sessionless OpenAI Codex WebSocket requests to use UUIDv7 request IDs, enabling models that reject UUIDv4 IDs. diff --git a/packages/ai/package.json b/packages/ai/package.json index 58b9113f..ce936c9e 100644 --- a/packages/ai/package.json +++ b/packages/ai/package.json @@ -49,10 +49,13 @@ ], "scripts": { "clean": "shx rm -rf dist", - "generate-models": "node scripts/generate-models.ts", + "generate-models": "node scripts/generate-models.ts --strict", + "hydrate-model-data": "node scripts/generate-models.ts --strict --data-only", "generate-model-catalog": "node scripts/generate-models.ts --strict --json-only --json-output ../../.artifacts/model-catalog", - "generate-image-models": "node scripts/generate-image-models.ts", - "build": "npm run generate-models && tsgo -p tsconfig.build.json && shx rm -rf dist/providers/data && shx cp -r src/providers/data dist/providers/data", + "generate-image-models": "node scripts/generate-image-models.ts --strict", + "check:model-data": "node scripts/check-model-data.ts", + "build": "npm run generate-models && npm run build:offline", + "build:offline": "npm run check:model-data && tsgo -p tsconfig.build.json && shx rm -rf dist/providers/data && shx cp -r src/providers/data dist/providers/data", "test": "vitest --run", "prepublishOnly": "npm run clean && npm run build" }, diff --git a/packages/ai/scripts/check-model-data.ts b/packages/ai/scripts/check-model-data.ts new file mode 100644 index 00000000..5c61b52c --- /dev/null +++ b/packages/ai/scripts/check-model-data.ts @@ -0,0 +1,16 @@ +#!/usr/bin/env node + +import { dirname, join } from "node:path"; +import { fileURLToPath } from "node:url"; +import { validateGeneratedModelData } from "./model-data.ts"; + +const packageRoot = join(dirname(fileURLToPath(import.meta.url)), ".."); + +try { + validateGeneratedModelData(packageRoot); + console.log("Generated model data is valid."); +} catch (error) { + console.error(error instanceof Error ? error.message : String(error)); + console.error("\nModel data is missing or stale. Run `npm run hydrate:model-data` from the repository root."); + process.exitCode = 1; +} diff --git a/packages/ai/scripts/generate-image-models.ts b/packages/ai/scripts/generate-image-models.ts index 2744da48..26e46e45 100644 --- a/packages/ai/scripts/generate-image-models.ts +++ b/packages/ai/scripts/generate-image-models.ts @@ -1,7 +1,7 @@ #!/usr/bin/env node import { writeFileSync } from "fs"; -import { dirname, join } from "path"; +import { dirname, join, resolve } from "path"; import { fileURLToPath } from "url"; import type { ImagesModel } from "../src/types.ts"; @@ -10,6 +10,13 @@ const __dirname = dirname(__filename); const packageRoot = join(__dirname, ".."); const OPENROUTER_BASE_URL = "https://openrouter.ai/api/v1"; +function readStrictOption(args: string[]): boolean { + for (const arg of args) { + if (arg !== "--strict") throw new Error(`Unknown argument: ${arg}`); + } + return args.includes("--strict"); +} + interface OpenRouterModelRecord { id: string; name: string; @@ -26,52 +33,73 @@ interface OpenRouterModelRecord { }; } -async function fetchOpenRouterImageModels(): Promise[]> { +export function parseOpenRouterImageModels( + payload: unknown, + strict: boolean, +): ImagesModel<"openrouter-images">[] { + const data = + typeof payload === "object" && payload !== null + ? (payload as { data?: OpenRouterModelRecord[] }).data + : undefined; + if (!Array.isArray(data) || data.length === 0) { + if (strict) throw new Error("OpenRouter API returned a missing or empty image model list"); + return []; + } + + const models: ImagesModel<"openrouter-images">[] = []; + for (const model of data) { + const input = Array.from( + new Set( + (model.architecture?.input_modalities ?? []).filter( + (modality): modality is "text" | "image" => modality === "text" || modality === "image", + ), + ), + ); + const output = Array.from( + new Set( + (model.architecture?.output_modalities ?? []).filter( + (modality): modality is "text" | "image" => modality === "text" || modality === "image", + ), + ), + ); + + if (!output.includes("image")) continue; + if (input.length === 0) input.push("text"); + + models.push({ + id: model.id, + name: model.name, + api: "openrouter-images", + provider: "openrouter", + baseUrl: OPENROUTER_BASE_URL, + input, + output, + cost: { + input: parseFloat(model.pricing?.prompt || "0") * 1_000_000, + output: parseFloat(model.pricing?.completion || "0") * 1_000_000, + cacheRead: parseFloat(model.pricing?.input_cache_read || "0") * 1_000_000, + cacheWrite: parseFloat(model.pricing?.input_cache_write || "0") * 1_000_000, + }, + }); + } + + if (strict && models.length === 0) { + throw new Error("OpenRouter API returned no usable image models"); + } + return models; +} + +async function fetchOpenRouterImageModels(strict: boolean): Promise[]> { try { console.log("Fetching image models from OpenRouter API..."); const response = await fetch(`${OPENROUTER_BASE_URL}/models?output_modalities=image`); - const data = (await response.json()) as { data?: OpenRouterModelRecord[] }; - const models: ImagesModel<"openrouter-images">[] = []; - - for (const model of data.data ?? []) { - const input = Array.from( - new Set( - (model.architecture?.input_modalities ?? []) - .filter((modality): modality is "text" | "image" => modality === "text" || modality === "image"), - ), - ); - const output = Array.from( - new Set( - (model.architecture?.output_modalities ?? []).filter( - (modality): modality is "text" | "image" => modality === "text" || modality === "image", - ), - ), - ); - - if (!output.includes("image")) continue; - if (input.length === 0) input.push("text"); - - models.push({ - id: model.id, - name: model.name, - api: "openrouter-images", - provider: "openrouter", - baseUrl: OPENROUTER_BASE_URL, - input, - output, - cost: { - input: parseFloat(model.pricing?.prompt || "0") * 1_000_000, - output: parseFloat(model.pricing?.completion || "0") * 1_000_000, - cacheRead: parseFloat(model.pricing?.input_cache_read || "0") * 1_000_000, - cacheWrite: parseFloat(model.pricing?.input_cache_write || "0") * 1_000_000, - }, - }); - } - + if (!response.ok) throw new Error(`OpenRouter API returned ${response.status}`); + const models = parseOpenRouterImageModels(await response.json(), strict); console.log(`Fetched ${models.length} image models from OpenRouter`); return models; } catch (error) { console.error("Failed to fetch OpenRouter image models:", error); + if (strict) throw error; return []; } } @@ -118,14 +146,17 @@ ${providerEntries} } async function main(): Promise { - const models = await fetchOpenRouterImageModels(); + const strict = readStrictOption(process.argv.slice(2)); + const models = await fetchOpenRouterImageModels(strict); const output = generateImageModelsFile(models); const outputPath = join(packageRoot, "src", "image-models.generated.ts"); writeFileSync(outputPath, output, "utf-8"); console.log(`Generated ${outputPath}`); } -main().catch((error) => { - console.error(error); - process.exit(1); -}); +if (process.argv[1] && resolve(process.argv[1]) === __filename) { + main().catch((error) => { + console.error(error); + process.exit(1); + }); +} diff --git a/packages/ai/scripts/generate-models.ts b/packages/ai/scripts/generate-models.ts index 82700ac9..66e64e5a 100644 --- a/packages/ai/scripts/generate-models.ts +++ b/packages/ai/scripts/generate-models.ts @@ -1,6 +1,6 @@ #!/usr/bin/env node -import { mkdirSync, readdirSync, rmSync, writeFileSync } from "fs"; +import { existsSync, mkdirSync, mkdtempSync, readFileSync, readdirSync, renameSync, rmSync, writeFileSync } from "fs"; import { dirname, join, resolve } from "path"; import { fileURLToPath } from "url"; import { @@ -18,6 +18,14 @@ import type { OpenAICompletionsCompat, OpenAIResponsesCompat, } from "../src/types.ts"; +import { + createModelDataManifest, + type ModelDataStructure, + MODEL_DATA_MANIFEST_FILE, + readModelDataStructure, + validateGeneratedModelData, + validateModelDataDirectory, +} from "./model-data.ts"; const __filename = fileURLToPath(import.meta.url); const __dirname = dirname(__filename); @@ -25,11 +33,13 @@ const packageRoot = join(__dirname, ".."); function readGeneratorOptions(args: string[]): { strict: boolean; + dataOnly: boolean; jsonOnly: boolean; jsonOutputDir: string | undefined; pretty: boolean; } { let strict = false; + let dataOnly = false; let jsonOnly = false; let jsonOutputDir: string | undefined; let pretty = false; @@ -40,6 +50,10 @@ function readGeneratorOptions(args: string[]): { strict = true; continue; } + if (arg === "--data-only") { + dataOnly = true; + continue; + } if (arg === "--json-only") { jsonOnly = true; continue; @@ -58,7 +72,8 @@ function readGeneratorOptions(args: string[]): { } if (jsonOnly && !jsonOutputDir) throw new Error("--json-only requires --json-output"); - return { strict, jsonOnly, jsonOutputDir, pretty }; + if (dataOnly && (jsonOnly || jsonOutputDir)) throw new Error("--data-only cannot be combined with JSON catalog output"); + return { strict, dataOnly, jsonOnly, jsonOutputDir, pretty }; } const generatorOptions = readGeneratorOptions(process.argv.slice(2)); @@ -2403,63 +2418,154 @@ async function generateModels() { jsonProviders[providerId][modelId] = providers[providerId][modelId]; } } - const writeJson = (path: string, value: unknown) => - writeFileSync(path, `${JSON.stringify(value, null, generatorOptions.pretty ? 2 : undefined)}\n`); + + const serializeJson = (value: unknown) => `${JSON.stringify(value, null, generatorOptions.pretty ? 2 : undefined)}\n`; + const writeJson = (path: string, value: unknown) => writeFileSync(path, serializeJson(value)); + let generatedDataProviderIds = sortedProviderIds; + let generatedDataProviders = jsonProviders; + let modelDataStructure: ModelDataStructure = Object.fromEntries( + sortedProviderIds.map((providerId) => [ + providerId, + Object.fromEntries( + Object.entries(jsonProviders[providerId]).map(([modelId, model]) => [modelId, model.api]), + ), + ]), + ); + + if (generatorOptions.dataOnly) { + modelDataStructure = readModelDataStructure(packageRoot); + generatedDataProviderIds = Object.keys(modelDataStructure); + const hydratedProviders: typeof jsonProviders = {}; + const hydrationErrors: string[] = []; + for (const [providerId, expectedModels] of Object.entries(modelDataStructure)) { + hydratedProviders[providerId] = {}; + for (const [modelId, expectedApi] of Object.entries(expectedModels)) { + const model = jsonProviders[providerId]?.[modelId]; + if (!model) { + hydrationErrors.push(`missing ${providerId}/${modelId}`); + continue; + } + if (model.api !== expectedApi) { + hydrationErrors.push(`${providerId}/${modelId} uses ${model.api}, expected ${expectedApi}`); + continue; + } + hydratedProviders[providerId][modelId] = model; + } + } + if (hydrationErrors.length > 0) { + throw new Error(`Cannot hydrate the committed model catalog:\n${hydrationErrors.map((error) => ` - ${error}`).join("\n")}`); + } + generatedDataProviders = hydratedProviders; + } if (!generatorOptions.jsonOnly) { - // Generate TypeScript structural catalogs and adjacent JSON values. - const generatedHeader = `// This file is auto-generated by scripts/generate-models.ts + // Stage and validate all provider values before replacing the current generated data. + const providersDir = join(packageRoot, "src/providers"); + const dataDir = join(providersDir, "data"); + const stagingRoot = mkdtempSync(join(providersDir, ".model-generation-")); + const stagedDataDir = join(stagingRoot, "data"); + const previousDataDir = join(stagingRoot, "previous-data"); + let restoreStructuralCatalog: (() => void) | undefined; + try { + mkdirSync(stagedDataDir, { recursive: true }); + const fileContents: Record = {}; + for (const providerId of generatedDataProviderIds) { + const filename = `${providerId}.json`; + const content = serializeJson(generatedDataProviders[providerId]); + fileContents[filename] = content; + writeFileSync(join(stagedDataDir, filename), content); + } + writeJson( + join(stagedDataDir, MODEL_DATA_MANIFEST_FILE), + createModelDataManifest(modelDataStructure, fileContents), + ); + validateModelDataDirectory(modelDataStructure, stagedDataDir); + + if (!generatorOptions.dataOnly) { + // Generate TypeScript structural catalogs only after the model data is complete and valid. + const previousShardContents = new Map( + readdirSync(providersDir) + .filter((entry) => entry.endsWith(".models.ts")) + .map((entry) => [entry, readFileSync(join(providersDir, entry), "utf8")] as const), + ); + const aggregatorPath = join(packageRoot, "src/models.generated.ts"); + const previousAggregator = readFileSync(aggregatorPath, "utf8"); + restoreStructuralCatalog = () => { + for (const entry of readdirSync(providersDir)) { + if (entry.endsWith(".models.ts")) rmSync(join(providersDir, entry)); + } + for (const [entry, content] of previousShardContents) { + writeFileSync(join(providersDir, entry), content); + } + writeFileSync(aggregatorPath, previousAggregator); + }; + + const generatedHeader = `// This file is auto-generated by scripts/generate-models.ts // Do not edit manually - run 'npm run generate-models' to update `; - const catalogConstName = (providerId: string) => - `${providerId.toUpperCase().replace(/[^A-Z0-9]+/g, "_")}_MODELS`; - const providersDir = join(packageRoot, "src/providers"); - const dataDir = join(providersDir, "data"); + const catalogConstName = (providerId: string) => + `${providerId.toUpperCase().replace(/[^A-Z0-9]+/g, "_")}_MODELS`; + const generatedShardFiles = new Set(); - function emitModelShape(model: Model, indent: string): string { - return `${indent}${JSON.stringify(model.id)}: Model<${JSON.stringify(model.api)}> & {\n${indent}\tid: ${JSON.stringify(model.id)};\n${indent}\tprovider: ${JSON.stringify(model.provider)};\n${indent}};\n`; - } + function emitModelShape(model: Model, indent: string): string { + return `${indent}${JSON.stringify(model.id)}: Model<${JSON.stringify(model.api)}> & {\n${indent}\tid: ${JSON.stringify(model.id)};\n${indent}\tprovider: ${JSON.stringify(model.provider)};\n${indent}};\n`; + } - // Remove stale per-provider catalogs and their generated values. - for (const entry of readdirSync(providersDir)) { - if (entry.endsWith(".models.ts")) { - rmSync(join(providersDir, entry)); + for (const providerId of sortedProviderIds) { + const models = providers[providerId]; + let output = generatedHeader; + output += `import values from "./data/${providerId}.json" with { type: "json" };\n`; + output += `import type { Model } from "../types.ts";\n\n`; + output += `export const ${catalogConstName(providerId)} = values as {\n`; + for (const modelId of Object.keys(models).sort()) { + output += emitModelShape(models[modelId], "\t"); + } + output += `};\n`; + const filename = `${providerId}.models.ts`; + generatedShardFiles.add(filename); + writeFileSync(join(providersDir, filename), output); + } + for (const entry of readdirSync(providersDir)) { + if (entry.endsWith(".models.ts") && !generatedShardFiles.has(entry)) rmSync(join(providersDir, entry)); + } + console.log(`Generated ${sortedProviderIds.length} catalog structures under src/providers/`); + + let output = generatedHeader; + for (const providerId of sortedProviderIds) { + output += `import { ${catalogConstName(providerId)} } from "./providers/${providerId}.models.ts";\n`; + } + output += `\nexport const MODELS = {\n`; + for (const providerId of sortedProviderIds) { + output += `\t${JSON.stringify(providerId)}: ${catalogConstName(providerId)},\n`; + } + output += `} as const;\n`; + writeFileSync(aggregatorPath, output); + console.log("Generated src/models.generated.ts"); } - } - rmSync(dataDir, { recursive: true, force: true }); - mkdirSync(dataDir, { recursive: true }); - // Per-provider catalog structure and values (sorted for deterministic output). - for (const providerId of sortedProviderIds) { - const models = providers[providerId]; - const sortedModelIds = Object.keys(models).sort(); - let output = generatedHeader; - output += `import values from "./data/${providerId}.json" with { type: "json" };\n`; - output += `import type { Model } from "../types.ts";\n\n`; - output += `export const ${catalogConstName(providerId)} = values as {\n`; - for (const modelId of sortedModelIds) { - output += emitModelShape(models[modelId], "\t"); + const hadPreviousData = existsSync(dataDir); + if (hadPreviousData) renameSync(dataDir, previousDataDir); + try { + renameSync(stagedDataDir, dataDir); + validateGeneratedModelData(packageRoot); + } catch (error) { + rmSync(dataDir, { recursive: true, force: true }); + if (hadPreviousData && existsSync(previousDataDir)) renameSync(previousDataDir, dataDir); + throw error; } - output += `};\n`; - writeFileSync(join(providersDir, `${providerId}.models.ts`), output); - writeJson(join(dataDir, `${providerId}.json`), jsonProviders[providerId]); + restoreStructuralCatalog = undefined; + console.log( + generatorOptions.dataOnly + ? "Hydrated JSON model values under src/providers/data/" + : "Generated JSON model values under src/providers/data/", + ); + } catch (error) { + restoreStructuralCatalog?.(); + throw error; + } finally { + rmSync(stagingRoot, { recursive: true, force: true }); } - console.log(`Generated ${sortedProviderIds.length} catalog structures under src/providers/`); - console.log("Generated JSON model values under src/providers/data/"); - - // Aggregator - let output = generatedHeader; - for (const providerId of sortedProviderIds) { - output += `import { ${catalogConstName(providerId)} } from "./providers/${providerId}.models.ts";\n`; - } - output += `\nexport const MODELS = {\n`; - for (const providerId of sortedProviderIds) { - output += `\t${JSON.stringify(providerId)}: ${catalogConstName(providerId)},\n`; - } - output += `} as const;\n`; - writeFileSync(join(packageRoot, "src/models.generated.ts"), output); - console.log("Generated src/models.generated.ts"); } if (generatorOptions.jsonOutputDir) { diff --git a/packages/ai/scripts/model-data.ts b/packages/ai/scripts/model-data.ts new file mode 100644 index 00000000..588bc695 --- /dev/null +++ b/packages/ai/scripts/model-data.ts @@ -0,0 +1,261 @@ +import { createHash } from "node:crypto"; +import { existsSync, readFileSync, readdirSync, statSync } from "node:fs"; +import { join } from "node:path"; + +export const MODEL_DATA_SCHEMA_VERSION = 1; +export const MODEL_DATA_MANIFEST_FILE = ".manifest.json"; + +export type ModelDataStructure = Record>; + +export interface ModelDataManifest { + schemaVersion: number; + structureHash: string; + files: Record; +} + +const JSON_STRING_PATTERN = '"(?:\\\\.|[^"\\\\])*"'; +const MODEL_SHAPE_PATTERN = new RegExp(`^\\t(${JSON_STRING_PATTERN}): Model<(${JSON_STRING_PATTERN})> & \\{$`); +const MODEL_ID_PATTERN = new RegExp(`^\\t\\tid: (${JSON_STRING_PATTERN});$`); +const MODEL_PROVIDER_PATTERN = new RegExp(`^\\t\\tprovider: (${JSON_STRING_PATTERN});$`); + +function sha256(value: string): string { + return createHash("sha256").update(value).digest("hex"); +} + +function parseJsonString(value: string, description: string): string { + const parsed: unknown = JSON.parse(value); + if (typeof parsed !== "string") throw new Error(`${description} is not a string`); + return parsed; +} + +function sortedRecord(entries: Iterable): Record { + return Object.fromEntries(Array.from(entries).sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0))); +} + +function sameStrings(a: readonly string[], b: readonly string[]): boolean { + return a.length === b.length && a.every((value, index) => value === b[index]); +} + +function describeSetDifference(expected: readonly string[], actual: readonly string[]): string { + const expectedSet = new Set(expected); + const actualSet = new Set(actual); + const missing = expected.filter((value) => !actualSet.has(value)); + const extra = actual.filter((value) => !expectedSet.has(value)); + return [missing.length > 0 ? `missing: ${missing.join(", ")}` : "", extra.length > 0 ? `extra: ${extra.join(", ")}` : ""] + .filter(Boolean) + .join("; "); +} + +function parseProviderStructure(path: string, providerId: string): Record { + const source = readFileSync(path, "utf8"); + const expectedImport = `import values from "./data/${providerId}.json" with { type: "json" };`; + if (!source.includes(expectedImport)) { + throw new Error(`${path} does not import ${providerId}.json`); + } + + const models = new Map(); + const lines = source.split("\n"); + for (let index = 0; index < lines.length; index++) { + const shapeMatch = MODEL_SHAPE_PATTERN.exec(lines[index]); + if (!shapeMatch) continue; + + const idMatch = MODEL_ID_PATTERN.exec(lines[index + 1] ?? ""); + const providerMatch = MODEL_PROVIDER_PATTERN.exec(lines[index + 2] ?? ""); + if (!idMatch || !providerMatch || lines[index + 3] !== "\t};") { + throw new Error(`${path}:${index + 1} has a malformed generated model declaration`); + } + + const key = parseJsonString(shapeMatch[1], `${path}:${index + 1} model key`); + const api = parseJsonString(shapeMatch[2], `${path}:${index + 1} model API`); + const id = parseJsonString(idMatch[1], `${path}:${index + 2} model ID`); + const provider = parseJsonString(providerMatch[1], `${path}:${index + 3} provider ID`); + if (id !== key) throw new Error(`${path}:${index + 1} declares key ${key} with ID ${id}`); + if (provider !== providerId) { + throw new Error(`${path}:${index + 1} declares provider ${provider} instead of ${providerId}`); + } + if (models.has(key)) throw new Error(`${path} declares model ${key} more than once`); + models.set(key, api); + index += 3; + } + + if (models.size === 0) throw new Error(`${path} contains no generated model declarations`); + return sortedRecord(models); +} + +export function readModelDataStructure(packageRoot: string): ModelDataStructure { + const providersDir = join(packageRoot, "src", "providers"); + const shardProviderIds = readdirSync(providersDir) + .filter((entry) => entry.endsWith(".models.ts")) + .map((entry) => entry.slice(0, -".models.ts".length)) + .sort(); + if (shardProviderIds.length === 0) throw new Error(`No generated provider shards found under ${providersDir}`); + + const aggregator = readFileSync(join(packageRoot, "src", "models.generated.ts"), "utf8"); + const importedProviderIds = Array.from( + aggregator.matchAll(/^import \{ [A-Z0-9_]+_MODELS \} from "\.\/providers\/([^"/]+)\.models\.ts";$/gm), + (match) => match[1], + ).sort(); + if (!sameStrings(shardProviderIds, importedProviderIds)) { + throw new Error( + `Generated model aggregator and provider shards do not match (${describeSetDifference(shardProviderIds, importedProviderIds)})`, + ); + } + + return sortedRecord( + shardProviderIds.map((providerId) => [ + providerId, + parseProviderStructure(join(providersDir, `${providerId}.models.ts`), providerId), + ] as const), + ); +} + +export function modelDataStructureHash(structure: ModelDataStructure): string { + return sha256(JSON.stringify(structure)); +} + +export function createModelDataManifest( + structure: ModelDataStructure, + fileContents: Readonly>, +): ModelDataManifest { + return { + schemaVersion: MODEL_DATA_SCHEMA_VERSION, + structureHash: modelDataStructureHash(structure), + files: sortedRecord(Object.entries(fileContents).map(([file, content]) => [file, sha256(content)] as const)), + }; +} + +function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value); +} + +function readJsonObject(path: string, description: string, errors: string[]): Record | undefined { + let parsed: unknown; + try { + parsed = JSON.parse(readFileSync(path, "utf8")); + } catch (error) { + errors.push(`${description} is not valid JSON: ${error instanceof Error ? error.message : String(error)}`); + return undefined; + } + if (!isRecord(parsed)) { + errors.push(`${description} must contain a JSON object`); + return undefined; + } + return parsed; +} + +function validateModelValue( + value: unknown, + providerId: string, + modelId: string, + expectedApi: string, + errors: string[], +): void { + const label = `${providerId}/${modelId}`; + if (!isRecord(value)) { + errors.push(`${label} must be an object`); + return; + } + if (value.id !== modelId) errors.push(`${label} has id ${JSON.stringify(value.id)}, expected ${JSON.stringify(modelId)}`); + if (value.provider !== providerId) { + errors.push(`${label} has provider ${JSON.stringify(value.provider)}, expected ${JSON.stringify(providerId)}`); + } + if (value.api !== expectedApi) { + errors.push(`${label} has api ${JSON.stringify(value.api)}, expected ${JSON.stringify(expectedApi)}`); + } + if (typeof value.name !== "string" || value.name.length === 0) errors.push(`${label} has no model name`); + if (typeof value.baseUrl !== "string") errors.push(`${label} has no baseUrl string`); + if (typeof value.reasoning !== "boolean") errors.push(`${label} has no reasoning boolean`); + if ( + !Array.isArray(value.input) || + value.input.length === 0 || + value.input.some((entry) => entry !== "text" && entry !== "image") + ) { + errors.push(`${label} has invalid input modalities`); + } + if (typeof value.contextWindow !== "number" || !Number.isFinite(value.contextWindow) || value.contextWindow <= 0) { + errors.push(`${label} has invalid contextWindow`); + } + if (typeof value.maxTokens !== "number" || !Number.isFinite(value.maxTokens) || value.maxTokens <= 0) { + errors.push(`${label} has invalid maxTokens`); + } + if (!isRecord(value.cost)) { + errors.push(`${label} has invalid cost metadata`); + } else { + for (const field of ["input", "output", "cacheRead", "cacheWrite"] as const) { + const cost = value.cost[field]; + if (typeof cost !== "number" || !Number.isFinite(cost)) { + errors.push(`${label} has invalid cost.${field}`); + } + } + } +} + +function throwValidationErrors(errors: string[]): never { + const visible = errors.slice(0, 30); + const suffix = errors.length > visible.length ? `\n ... and ${errors.length - visible.length} more` : ""; + throw new Error(`Invalid generated model data:\n${visible.map((error) => ` - ${error}`).join("\n")}${suffix}`); +} + +export function validateModelDataDirectory(structure: ModelDataStructure, dataDir: string): void { + if (!existsSync(dataDir) || !statSync(dataDir).isDirectory()) { + throw new Error(`Generated model data directory does not exist: ${dataDir}`); + } + + const errors: string[] = []; + const expectedFiles = Object.keys(structure) + .map((providerId) => `${providerId}.json`) + .sort(); + const actualFiles = readdirSync(dataDir) + .filter((entry) => entry.endsWith(".json") && entry !== MODEL_DATA_MANIFEST_FILE) + .sort(); + if (!sameStrings(expectedFiles, actualFiles)) { + errors.push(`provider data files do not match the structural catalog (${describeSetDifference(expectedFiles, actualFiles)})`); + } + + const manifestPath = join(dataDir, MODEL_DATA_MANIFEST_FILE); + const manifest = readJsonObject(manifestPath, "model data manifest", errors); + if (manifest?.schemaVersion !== MODEL_DATA_SCHEMA_VERSION) { + errors.push( + `model data schema is ${JSON.stringify(manifest?.schemaVersion)}, expected ${MODEL_DATA_SCHEMA_VERSION}`, + ); + } + const expectedStructureHash = modelDataStructureHash(structure); + if (manifest?.structureHash !== expectedStructureHash) { + errors.push("model data generation stamp does not match the structural catalog"); + } + const manifestFiles = isRecord(manifest?.files) ? manifest.files : undefined; + if (!manifestFiles) errors.push("model data manifest has no file hashes"); + else { + const manifestFileNames = Object.keys(manifestFiles).sort(); + if (!sameStrings(expectedFiles, manifestFileNames)) { + errors.push(`manifest file hashes do not match provider data files (${describeSetDifference(expectedFiles, manifestFileNames)})`); + } + } + + for (const [providerId, expectedModels] of Object.entries(structure)) { + const filename = `${providerId}.json`; + const path = join(dataDir, filename); + if (!existsSync(path)) continue; + const content = readFileSync(path, "utf8"); + if (manifestFiles && manifestFiles[filename] !== sha256(content)) { + errors.push(`${filename} does not match its manifest hash`); + } + const values = readJsonObject(path, filename, errors); + if (!values) continue; + const expectedModelIds = Object.keys(expectedModels).sort(); + const actualModelIds = Object.keys(values).sort(); + if (!sameStrings(expectedModelIds, actualModelIds)) { + errors.push(`${filename} model IDs do not match the structural catalog (${describeSetDifference(expectedModelIds, actualModelIds)})`); + } + for (const [modelId, api] of Object.entries(expectedModels)) { + if (modelId in values) validateModelValue(values[modelId], providerId, modelId, api, errors); + } + } + + if (errors.length > 0) throwValidationErrors(errors); +} + +export function validateGeneratedModelData(packageRoot: string): void { + const structure = readModelDataStructure(packageRoot); + validateModelDataDirectory(structure, join(packageRoot, "src", "providers", "data")); +} diff --git a/packages/ai/test/image-model-data.test.ts b/packages/ai/test/image-model-data.test.ts new file mode 100644 index 00000000..21d46805 --- /dev/null +++ b/packages/ai/test/image-model-data.test.ts @@ -0,0 +1,47 @@ +import { describe, expect, it } from "vitest"; +import { parseOpenRouterImageModels } from "../scripts/generate-image-models.ts"; + +const validImageModel = { + id: "example/image-model", + name: "Example Image Model", + architecture: { + input_modalities: ["text", "image"], + output_modalities: ["image"], + }, + pricing: { + prompt: "0.000001", + completion: "0.000002", + }, +}; + +describe("OpenRouter image model parsing", () => { + it.each([{}, { data: [] }, { data: "invalid" }])("rejects a missing or empty strict catalog", (payload) => { + expect(() => parseOpenRouterImageModels(payload, true)).toThrow("missing or empty image model list"); + }); + + it("rejects a strict catalog with no usable image models", () => { + expect(() => + parseOpenRouterImageModels( + { + data: [ + { + ...validImageModel, + architecture: { input_modalities: ["text"], output_modalities: ["text"] }, + }, + ], + }, + true, + ), + ).toThrow("no usable image models"); + }); + + it("parses a non-empty image model catalog", () => { + expect(parseOpenRouterImageModels({ data: [validImageModel] }, true)).toEqual([ + expect.objectContaining({ + id: "example/image-model", + input: ["text", "image"], + output: ["image"], + }), + ]); + }); +}); diff --git a/packages/ai/test/model-data-validation.test.ts b/packages/ai/test/model-data-validation.test.ts new file mode 100644 index 00000000..1e7798d3 --- /dev/null +++ b/packages/ai/test/model-data-validation.test.ts @@ -0,0 +1,123 @@ +import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, describe, expect, it } from "vitest"; +import { + createModelDataManifest, + MODEL_DATA_MANIFEST_FILE, + MODEL_DATA_SCHEMA_VERSION, + type ModelDataStructure, + readModelDataStructure, + validateModelDataDirectory, +} from "../scripts/model-data.ts"; + +const temporaryRoots: string[] = []; + +afterEach(() => { + for (const root of temporaryRoots.splice(0)) rmSync(root, { force: true, recursive: true }); +}); + +function createFixture(): { + dataDir: string; + packageRoot: string; + structure: ModelDataStructure; + values: Record; +} { + const packageRoot = mkdtempSync(join(tmpdir(), "pi-model-data-")); + temporaryRoots.push(packageRoot); + const providersDir = join(packageRoot, "src", "providers"); + const dataDir = join(providersDir, "data"); + mkdirSync(dataDir, { recursive: true }); + writeFileSync( + join(packageRoot, "src", "models.generated.ts"), + 'import { TEST_PROVIDER_MODELS } from "./providers/test-provider.models.ts";\n\nexport const MODELS = {\n\t"test-provider": TEST_PROVIDER_MODELS,\n} as const;\n', + ); + writeFileSync( + join(providersDir, "test-provider.models.ts"), + '// generated\n\nimport values from "./data/test-provider.json" with { type: "json" };\nimport type { Model } from "../types.ts";\n\nexport const TEST_PROVIDER_MODELS = values as {\n\t"model-a": Model<"openai-completions"> & {\n\t\tid: "model-a";\n\t\tprovider: "test-provider";\n\t};\n};\n', + ); + + const structure = readModelDataStructure(packageRoot); + const values: Record = { + "model-a": { + id: "model-a", + name: "Model A", + api: "openai-completions", + provider: "test-provider", + baseUrl: "https://example.test/v1", + reasoning: false, + input: ["text"], + cost: { input: 1, output: 2, cacheRead: 0, cacheWrite: 0 }, + contextWindow: 1000, + maxTokens: 100, + }, + }; + writeFixtureData(dataDir, structure, values); + return { dataDir, packageRoot, structure, values }; +} + +function writeFixtureData( + dataDir: string, + structure: ModelDataStructure, + values: Record, + manifestSchemaVersion = MODEL_DATA_SCHEMA_VERSION, +): void { + const filename = "test-provider.json"; + const content = `${JSON.stringify(values)}\n`; + writeFileSync(join(dataDir, filename), content); + const manifest = createModelDataManifest(structure, { [filename]: content }); + manifest.schemaVersion = manifestSchemaVersion; + writeFileSync(join(dataDir, MODEL_DATA_MANIFEST_FILE), `${JSON.stringify(manifest)}\n`); +} + +describe("generated model data validation", () => { + it("validates complete data against generated structural catalogs", () => { + const { dataDir, structure } = createFixture(); + expect(() => validateModelDataDirectory(structure, dataDir)).not.toThrow(); + }); + + it("rejects a missing model data directory", () => { + const { dataDir, structure } = createFixture(); + rmSync(dataDir, { recursive: true }); + expect(() => validateModelDataDirectory(structure, dataDir)).toThrow("does not exist"); + }); + + it.each([ + ["id", "wrong-id", "has id"], + ["provider", "wrong-provider", "has provider"], + ["api", "anthropic-messages", "has api"], + ] as const)("rejects a wrong model %s", (field, value, expectedMessage) => { + const fixture = createFixture(); + const model = fixture.values["model-a"] as Record; + model[field] = value; + writeFixtureData(fixture.dataDir, fixture.structure, fixture.values); + expect(() => validateModelDataDirectory(fixture.structure, fixture.dataDir)).toThrow(expectedMessage); + }); + + it("rejects missing model IDs and stale file hashes", () => { + const fixture = createFixture(); + writeFileSync(join(fixture.dataDir, "test-provider.json"), "{}\n"); + expect(() => validateModelDataDirectory(fixture.structure, fixture.dataDir)).toThrow(/manifest hash|model IDs/); + }); + + it("rejects incompatible schema and generation stamps", () => { + const fixture = createFixture(); + writeFixtureData(fixture.dataDir, fixture.structure, fixture.values, MODEL_DATA_SCHEMA_VERSION + 1); + expect(() => validateModelDataDirectory(fixture.structure, fixture.dataDir)).toThrow("model data schema"); + + const manifestPath = join(fixture.dataDir, MODEL_DATA_MANIFEST_FILE); + const manifest = JSON.parse(readFileSync(manifestPath, "utf8")) as Record; + manifest.structureHash = "stale"; + writeFileSync(manifestPath, `${JSON.stringify(manifest)}\n`); + expect(() => validateModelDataDirectory(fixture.structure, fixture.dataDir)).toThrow("generation stamp"); + }); + + it("rejects missing provider shards referenced by the aggregator", () => { + const { packageRoot } = createFixture(); + writeFileSync( + join(packageRoot, "src", "models.generated.ts"), + 'import { TEST_PROVIDER_MODELS } from "./providers/test-provider.models.ts";\nimport { MISSING_MODELS } from "./providers/missing.models.ts";\n', + ); + expect(() => readModelDataStructure(packageRoot)).toThrow("aggregator and provider shards do not match"); + }); +}); diff --git a/scripts/check-browser-smoke.mjs b/scripts/check-browser-smoke.mjs index eeab27b1..d11b22d2 100644 --- a/scripts/check-browser-smoke.mjs +++ b/scripts/check-browser-smoke.mjs @@ -8,7 +8,7 @@ const agentTreeshakeOutputPath = join(tmpdir(), "pi-agent-treeshake-smoke.js"); const errorLogPath = join(tmpdir(), "pi-browser-smoke-errors.log"); const generatedCatalogDataDir = join(process.cwd(), "packages/ai/src/providers/data"); -// Fresh checkouts do not materialize provider JSON until npm run build. +// Fresh checkouts do not materialize provider JSON until model data is hydrated. const generatedCatalogDataPlugin = { name: "generated-model-catalog", setup(build) { diff --git a/scripts/local-release.mjs b/scripts/local-release.mjs index e06996d0..181379d1 100644 --- a/scripts/local-release.mjs +++ b/scripts/local-release.mjs @@ -209,9 +209,9 @@ const bunInstallDirectory = join(outDir, "bun-install"); const binaryDirectory = join(outDir, "bun"); mkdirSync(tarballDirectory, { recursive: true }); -if (!options.skipCheck || !options.skipTest) { - run("npm", ["--prefix", "packages/ai", "run", "generate-models"], { cwd: repoRoot }); -} +// Release artifacts always use a freshly generated, strictly validated catalog, +// including when checks or tests are explicitly skipped. +run("npm", ["run", "generate:models"], { cwd: repoRoot }); if (!options.skipCheck) { run("npm", ["run", "check"], { cwd: repoRoot }); @@ -223,7 +223,7 @@ if (!options.skipTest) { for (const pkg of packages) { run("npm", ["run", "clean"], { cwd: pkg.directory }); - run("npm", ["run", "build"], { cwd: pkg.directory }); + run("npm", ["run", pkg.directory === "packages/ai" ? "build:offline" : "build"], { cwd: pkg.directory }); } const tarballs = new Map(); diff --git a/scripts/release.mjs b/scripts/release.mjs index e8a3d725..95cac6e6 100755 --- a/scripts/release.mjs +++ b/scripts/release.mjs @@ -167,6 +167,7 @@ console.log(); // 4. Regenerate release artifacts console.log("Regenerating release artifacts..."); run("npm run generate:models"); +run("npm run check:model-data"); run("npm run shrinkwrap:coding-agent"); run("npm run install-lock:coding-agent"); console.log();