fix(ai): validate generated model data before builds
This commit is contained in:
Armin Ronacher
@@ -0,0 +1,16 @@
|
||||
#!/usr/bin/env node
|
||||
|
||||
import { dirname, join } from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { validateGeneratedModelData } from "./model-data.ts";
|
||||
|
||||
const packageRoot = join(dirname(fileURLToPath(import.meta.url)), "..");
|
||||
|
||||
try {
|
||||
validateGeneratedModelData(packageRoot);
|
||||
console.log("Generated model data is valid.");
|
||||
} catch (error) {
|
||||
console.error(error instanceof Error ? error.message : String(error));
|
||||
console.error("\nModel data is missing or stale. Run `npm run hydrate:model-data` from the repository root.");
|
||||
process.exitCode = 1;
|
||||
}
|
||||
@@ -1,7 +1,7 @@
|
||||
#!/usr/bin/env node
|
||||
|
||||
import { writeFileSync } from "fs";
|
||||
import { dirname, join } from "path";
|
||||
import { dirname, join, resolve } from "path";
|
||||
import { fileURLToPath } from "url";
|
||||
import type { ImagesModel } from "../src/types.ts";
|
||||
|
||||
@@ -10,6 +10,13 @@ const __dirname = dirname(__filename);
|
||||
const packageRoot = join(__dirname, "..");
|
||||
const OPENROUTER_BASE_URL = "https://openrouter.ai/api/v1";
|
||||
|
||||
function readStrictOption(args: string[]): boolean {
|
||||
for (const arg of args) {
|
||||
if (arg !== "--strict") throw new Error(`Unknown argument: ${arg}`);
|
||||
}
|
||||
return args.includes("--strict");
|
||||
}
|
||||
|
||||
interface OpenRouterModelRecord {
|
||||
id: string;
|
||||
name: string;
|
||||
@@ -26,52 +33,73 @@ interface OpenRouterModelRecord {
|
||||
};
|
||||
}
|
||||
|
||||
async function fetchOpenRouterImageModels(): Promise<ImagesModel<"openrouter-images">[]> {
|
||||
export function parseOpenRouterImageModels(
|
||||
payload: unknown,
|
||||
strict: boolean,
|
||||
): ImagesModel<"openrouter-images">[] {
|
||||
const data =
|
||||
typeof payload === "object" && payload !== null
|
||||
? (payload as { data?: OpenRouterModelRecord[] }).data
|
||||
: undefined;
|
||||
if (!Array.isArray(data) || data.length === 0) {
|
||||
if (strict) throw new Error("OpenRouter API returned a missing or empty image model list");
|
||||
return [];
|
||||
}
|
||||
|
||||
const models: ImagesModel<"openrouter-images">[] = [];
|
||||
for (const model of data) {
|
||||
const input = Array.from(
|
||||
new Set(
|
||||
(model.architecture?.input_modalities ?? []).filter(
|
||||
(modality): modality is "text" | "image" => modality === "text" || modality === "image",
|
||||
),
|
||||
),
|
||||
);
|
||||
const output = Array.from(
|
||||
new Set(
|
||||
(model.architecture?.output_modalities ?? []).filter(
|
||||
(modality): modality is "text" | "image" => modality === "text" || modality === "image",
|
||||
),
|
||||
),
|
||||
);
|
||||
|
||||
if (!output.includes("image")) continue;
|
||||
if (input.length === 0) input.push("text");
|
||||
|
||||
models.push({
|
||||
id: model.id,
|
||||
name: model.name,
|
||||
api: "openrouter-images",
|
||||
provider: "openrouter",
|
||||
baseUrl: OPENROUTER_BASE_URL,
|
||||
input,
|
||||
output,
|
||||
cost: {
|
||||
input: parseFloat(model.pricing?.prompt || "0") * 1_000_000,
|
||||
output: parseFloat(model.pricing?.completion || "0") * 1_000_000,
|
||||
cacheRead: parseFloat(model.pricing?.input_cache_read || "0") * 1_000_000,
|
||||
cacheWrite: parseFloat(model.pricing?.input_cache_write || "0") * 1_000_000,
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
if (strict && models.length === 0) {
|
||||
throw new Error("OpenRouter API returned no usable image models");
|
||||
}
|
||||
return models;
|
||||
}
|
||||
|
||||
async function fetchOpenRouterImageModels(strict: boolean): Promise<ImagesModel<"openrouter-images">[]> {
|
||||
try {
|
||||
console.log("Fetching image models from OpenRouter API...");
|
||||
const response = await fetch(`${OPENROUTER_BASE_URL}/models?output_modalities=image`);
|
||||
const data = (await response.json()) as { data?: OpenRouterModelRecord[] };
|
||||
const models: ImagesModel<"openrouter-images">[] = [];
|
||||
|
||||
for (const model of data.data ?? []) {
|
||||
const input = Array.from(
|
||||
new Set(
|
||||
(model.architecture?.input_modalities ?? [])
|
||||
.filter((modality): modality is "text" | "image" => modality === "text" || modality === "image"),
|
||||
),
|
||||
);
|
||||
const output = Array.from(
|
||||
new Set(
|
||||
(model.architecture?.output_modalities ?? []).filter(
|
||||
(modality): modality is "text" | "image" => modality === "text" || modality === "image",
|
||||
),
|
||||
),
|
||||
);
|
||||
|
||||
if (!output.includes("image")) continue;
|
||||
if (input.length === 0) input.push("text");
|
||||
|
||||
models.push({
|
||||
id: model.id,
|
||||
name: model.name,
|
||||
api: "openrouter-images",
|
||||
provider: "openrouter",
|
||||
baseUrl: OPENROUTER_BASE_URL,
|
||||
input,
|
||||
output,
|
||||
cost: {
|
||||
input: parseFloat(model.pricing?.prompt || "0") * 1_000_000,
|
||||
output: parseFloat(model.pricing?.completion || "0") * 1_000_000,
|
||||
cacheRead: parseFloat(model.pricing?.input_cache_read || "0") * 1_000_000,
|
||||
cacheWrite: parseFloat(model.pricing?.input_cache_write || "0") * 1_000_000,
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
if (!response.ok) throw new Error(`OpenRouter API returned ${response.status}`);
|
||||
const models = parseOpenRouterImageModels(await response.json(), strict);
|
||||
console.log(`Fetched ${models.length} image models from OpenRouter`);
|
||||
return models;
|
||||
} catch (error) {
|
||||
console.error("Failed to fetch OpenRouter image models:", error);
|
||||
if (strict) throw error;
|
||||
return [];
|
||||
}
|
||||
}
|
||||
@@ -118,14 +146,17 @@ ${providerEntries}
|
||||
}
|
||||
|
||||
async function main(): Promise<void> {
|
||||
const models = await fetchOpenRouterImageModels();
|
||||
const strict = readStrictOption(process.argv.slice(2));
|
||||
const models = await fetchOpenRouterImageModels(strict);
|
||||
const output = generateImageModelsFile(models);
|
||||
const outputPath = join(packageRoot, "src", "image-models.generated.ts");
|
||||
writeFileSync(outputPath, output, "utf-8");
|
||||
console.log(`Generated ${outputPath}`);
|
||||
}
|
||||
|
||||
main().catch((error) => {
|
||||
console.error(error);
|
||||
process.exit(1);
|
||||
});
|
||||
if (process.argv[1] && resolve(process.argv[1]) === __filename) {
|
||||
main().catch((error) => {
|
||||
console.error(error);
|
||||
process.exit(1);
|
||||
});
|
||||
}
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
#!/usr/bin/env node
|
||||
|
||||
import { mkdirSync, readdirSync, rmSync, writeFileSync } from "fs";
|
||||
import { existsSync, mkdirSync, mkdtempSync, readFileSync, readdirSync, renameSync, rmSync, writeFileSync } from "fs";
|
||||
import { dirname, join, resolve } from "path";
|
||||
import { fileURLToPath } from "url";
|
||||
import {
|
||||
@@ -18,6 +18,14 @@ import type {
|
||||
OpenAICompletionsCompat,
|
||||
OpenAIResponsesCompat,
|
||||
} from "../src/types.ts";
|
||||
import {
|
||||
createModelDataManifest,
|
||||
type ModelDataStructure,
|
||||
MODEL_DATA_MANIFEST_FILE,
|
||||
readModelDataStructure,
|
||||
validateGeneratedModelData,
|
||||
validateModelDataDirectory,
|
||||
} from "./model-data.ts";
|
||||
|
||||
const __filename = fileURLToPath(import.meta.url);
|
||||
const __dirname = dirname(__filename);
|
||||
@@ -25,11 +33,13 @@ const packageRoot = join(__dirname, "..");
|
||||
|
||||
function readGeneratorOptions(args: string[]): {
|
||||
strict: boolean;
|
||||
dataOnly: boolean;
|
||||
jsonOnly: boolean;
|
||||
jsonOutputDir: string | undefined;
|
||||
pretty: boolean;
|
||||
} {
|
||||
let strict = false;
|
||||
let dataOnly = false;
|
||||
let jsonOnly = false;
|
||||
let jsonOutputDir: string | undefined;
|
||||
let pretty = false;
|
||||
@@ -40,6 +50,10 @@ function readGeneratorOptions(args: string[]): {
|
||||
strict = true;
|
||||
continue;
|
||||
}
|
||||
if (arg === "--data-only") {
|
||||
dataOnly = true;
|
||||
continue;
|
||||
}
|
||||
if (arg === "--json-only") {
|
||||
jsonOnly = true;
|
||||
continue;
|
||||
@@ -58,7 +72,8 @@ function readGeneratorOptions(args: string[]): {
|
||||
}
|
||||
|
||||
if (jsonOnly && !jsonOutputDir) throw new Error("--json-only requires --json-output");
|
||||
return { strict, jsonOnly, jsonOutputDir, pretty };
|
||||
if (dataOnly && (jsonOnly || jsonOutputDir)) throw new Error("--data-only cannot be combined with JSON catalog output");
|
||||
return { strict, dataOnly, jsonOnly, jsonOutputDir, pretty };
|
||||
}
|
||||
|
||||
const generatorOptions = readGeneratorOptions(process.argv.slice(2));
|
||||
@@ -2403,63 +2418,154 @@ async function generateModels() {
|
||||
jsonProviders[providerId][modelId] = providers[providerId][modelId];
|
||||
}
|
||||
}
|
||||
const writeJson = (path: string, value: unknown) =>
|
||||
writeFileSync(path, `${JSON.stringify(value, null, generatorOptions.pretty ? 2 : undefined)}\n`);
|
||||
|
||||
const serializeJson = (value: unknown) => `${JSON.stringify(value, null, generatorOptions.pretty ? 2 : undefined)}\n`;
|
||||
const writeJson = (path: string, value: unknown) => writeFileSync(path, serializeJson(value));
|
||||
let generatedDataProviderIds = sortedProviderIds;
|
||||
let generatedDataProviders = jsonProviders;
|
||||
let modelDataStructure: ModelDataStructure = Object.fromEntries(
|
||||
sortedProviderIds.map((providerId) => [
|
||||
providerId,
|
||||
Object.fromEntries(
|
||||
Object.entries(jsonProviders[providerId]).map(([modelId, model]) => [modelId, model.api]),
|
||||
),
|
||||
]),
|
||||
);
|
||||
|
||||
if (generatorOptions.dataOnly) {
|
||||
modelDataStructure = readModelDataStructure(packageRoot);
|
||||
generatedDataProviderIds = Object.keys(modelDataStructure);
|
||||
const hydratedProviders: typeof jsonProviders = {};
|
||||
const hydrationErrors: string[] = [];
|
||||
for (const [providerId, expectedModels] of Object.entries(modelDataStructure)) {
|
||||
hydratedProviders[providerId] = {};
|
||||
for (const [modelId, expectedApi] of Object.entries(expectedModels)) {
|
||||
const model = jsonProviders[providerId]?.[modelId];
|
||||
if (!model) {
|
||||
hydrationErrors.push(`missing ${providerId}/${modelId}`);
|
||||
continue;
|
||||
}
|
||||
if (model.api !== expectedApi) {
|
||||
hydrationErrors.push(`${providerId}/${modelId} uses ${model.api}, expected ${expectedApi}`);
|
||||
continue;
|
||||
}
|
||||
hydratedProviders[providerId][modelId] = model;
|
||||
}
|
||||
}
|
||||
if (hydrationErrors.length > 0) {
|
||||
throw new Error(`Cannot hydrate the committed model catalog:\n${hydrationErrors.map((error) => ` - ${error}`).join("\n")}`);
|
||||
}
|
||||
generatedDataProviders = hydratedProviders;
|
||||
}
|
||||
|
||||
if (!generatorOptions.jsonOnly) {
|
||||
// Generate TypeScript structural catalogs and adjacent JSON values.
|
||||
const generatedHeader = `// This file is auto-generated by scripts/generate-models.ts
|
||||
// Stage and validate all provider values before replacing the current generated data.
|
||||
const providersDir = join(packageRoot, "src/providers");
|
||||
const dataDir = join(providersDir, "data");
|
||||
const stagingRoot = mkdtempSync(join(providersDir, ".model-generation-"));
|
||||
const stagedDataDir = join(stagingRoot, "data");
|
||||
const previousDataDir = join(stagingRoot, "previous-data");
|
||||
let restoreStructuralCatalog: (() => void) | undefined;
|
||||
try {
|
||||
mkdirSync(stagedDataDir, { recursive: true });
|
||||
const fileContents: Record<string, string> = {};
|
||||
for (const providerId of generatedDataProviderIds) {
|
||||
const filename = `${providerId}.json`;
|
||||
const content = serializeJson(generatedDataProviders[providerId]);
|
||||
fileContents[filename] = content;
|
||||
writeFileSync(join(stagedDataDir, filename), content);
|
||||
}
|
||||
writeJson(
|
||||
join(stagedDataDir, MODEL_DATA_MANIFEST_FILE),
|
||||
createModelDataManifest(modelDataStructure, fileContents),
|
||||
);
|
||||
validateModelDataDirectory(modelDataStructure, stagedDataDir);
|
||||
|
||||
if (!generatorOptions.dataOnly) {
|
||||
// Generate TypeScript structural catalogs only after the model data is complete and valid.
|
||||
const previousShardContents = new Map(
|
||||
readdirSync(providersDir)
|
||||
.filter((entry) => entry.endsWith(".models.ts"))
|
||||
.map((entry) => [entry, readFileSync(join(providersDir, entry), "utf8")] as const),
|
||||
);
|
||||
const aggregatorPath = join(packageRoot, "src/models.generated.ts");
|
||||
const previousAggregator = readFileSync(aggregatorPath, "utf8");
|
||||
restoreStructuralCatalog = () => {
|
||||
for (const entry of readdirSync(providersDir)) {
|
||||
if (entry.endsWith(".models.ts")) rmSync(join(providersDir, entry));
|
||||
}
|
||||
for (const [entry, content] of previousShardContents) {
|
||||
writeFileSync(join(providersDir, entry), content);
|
||||
}
|
||||
writeFileSync(aggregatorPath, previousAggregator);
|
||||
};
|
||||
|
||||
const generatedHeader = `// This file is auto-generated by scripts/generate-models.ts
|
||||
// Do not edit manually - run 'npm run generate-models' to update
|
||||
|
||||
`;
|
||||
const catalogConstName = (providerId: string) =>
|
||||
`${providerId.toUpperCase().replace(/[^A-Z0-9]+/g, "_")}_MODELS`;
|
||||
const providersDir = join(packageRoot, "src/providers");
|
||||
const dataDir = join(providersDir, "data");
|
||||
const catalogConstName = (providerId: string) =>
|
||||
`${providerId.toUpperCase().replace(/[^A-Z0-9]+/g, "_")}_MODELS`;
|
||||
const generatedShardFiles = new Set<string>();
|
||||
|
||||
function emitModelShape(model: Model<any>, indent: string): string {
|
||||
return `${indent}${JSON.stringify(model.id)}: Model<${JSON.stringify(model.api)}> & {\n${indent}\tid: ${JSON.stringify(model.id)};\n${indent}\tprovider: ${JSON.stringify(model.provider)};\n${indent}};\n`;
|
||||
}
|
||||
function emitModelShape(model: Model<any>, indent: string): string {
|
||||
return `${indent}${JSON.stringify(model.id)}: Model<${JSON.stringify(model.api)}> & {\n${indent}\tid: ${JSON.stringify(model.id)};\n${indent}\tprovider: ${JSON.stringify(model.provider)};\n${indent}};\n`;
|
||||
}
|
||||
|
||||
// Remove stale per-provider catalogs and their generated values.
|
||||
for (const entry of readdirSync(providersDir)) {
|
||||
if (entry.endsWith(".models.ts")) {
|
||||
rmSync(join(providersDir, entry));
|
||||
for (const providerId of sortedProviderIds) {
|
||||
const models = providers[providerId];
|
||||
let output = generatedHeader;
|
||||
output += `import values from "./data/${providerId}.json" with { type: "json" };\n`;
|
||||
output += `import type { Model } from "../types.ts";\n\n`;
|
||||
output += `export const ${catalogConstName(providerId)} = values as {\n`;
|
||||
for (const modelId of Object.keys(models).sort()) {
|
||||
output += emitModelShape(models[modelId], "\t");
|
||||
}
|
||||
output += `};\n`;
|
||||
const filename = `${providerId}.models.ts`;
|
||||
generatedShardFiles.add(filename);
|
||||
writeFileSync(join(providersDir, filename), output);
|
||||
}
|
||||
for (const entry of readdirSync(providersDir)) {
|
||||
if (entry.endsWith(".models.ts") && !generatedShardFiles.has(entry)) rmSync(join(providersDir, entry));
|
||||
}
|
||||
console.log(`Generated ${sortedProviderIds.length} catalog structures under src/providers/`);
|
||||
|
||||
let output = generatedHeader;
|
||||
for (const providerId of sortedProviderIds) {
|
||||
output += `import { ${catalogConstName(providerId)} } from "./providers/${providerId}.models.ts";\n`;
|
||||
}
|
||||
output += `\nexport const MODELS = {\n`;
|
||||
for (const providerId of sortedProviderIds) {
|
||||
output += `\t${JSON.stringify(providerId)}: ${catalogConstName(providerId)},\n`;
|
||||
}
|
||||
output += `} as const;\n`;
|
||||
writeFileSync(aggregatorPath, output);
|
||||
console.log("Generated src/models.generated.ts");
|
||||
}
|
||||
}
|
||||
rmSync(dataDir, { recursive: true, force: true });
|
||||
mkdirSync(dataDir, { recursive: true });
|
||||
|
||||
// Per-provider catalog structure and values (sorted for deterministic output).
|
||||
for (const providerId of sortedProviderIds) {
|
||||
const models = providers[providerId];
|
||||
const sortedModelIds = Object.keys(models).sort();
|
||||
let output = generatedHeader;
|
||||
output += `import values from "./data/${providerId}.json" with { type: "json" };\n`;
|
||||
output += `import type { Model } from "../types.ts";\n\n`;
|
||||
output += `export const ${catalogConstName(providerId)} = values as {\n`;
|
||||
for (const modelId of sortedModelIds) {
|
||||
output += emitModelShape(models[modelId], "\t");
|
||||
const hadPreviousData = existsSync(dataDir);
|
||||
if (hadPreviousData) renameSync(dataDir, previousDataDir);
|
||||
try {
|
||||
renameSync(stagedDataDir, dataDir);
|
||||
validateGeneratedModelData(packageRoot);
|
||||
} catch (error) {
|
||||
rmSync(dataDir, { recursive: true, force: true });
|
||||
if (hadPreviousData && existsSync(previousDataDir)) renameSync(previousDataDir, dataDir);
|
||||
throw error;
|
||||
}
|
||||
output += `};\n`;
|
||||
writeFileSync(join(providersDir, `${providerId}.models.ts`), output);
|
||||
writeJson(join(dataDir, `${providerId}.json`), jsonProviders[providerId]);
|
||||
restoreStructuralCatalog = undefined;
|
||||
console.log(
|
||||
generatorOptions.dataOnly
|
||||
? "Hydrated JSON model values under src/providers/data/"
|
||||
: "Generated JSON model values under src/providers/data/",
|
||||
);
|
||||
} catch (error) {
|
||||
restoreStructuralCatalog?.();
|
||||
throw error;
|
||||
} finally {
|
||||
rmSync(stagingRoot, { recursive: true, force: true });
|
||||
}
|
||||
console.log(`Generated ${sortedProviderIds.length} catalog structures under src/providers/`);
|
||||
console.log("Generated JSON model values under src/providers/data/");
|
||||
|
||||
// Aggregator
|
||||
let output = generatedHeader;
|
||||
for (const providerId of sortedProviderIds) {
|
||||
output += `import { ${catalogConstName(providerId)} } from "./providers/${providerId}.models.ts";\n`;
|
||||
}
|
||||
output += `\nexport const MODELS = {\n`;
|
||||
for (const providerId of sortedProviderIds) {
|
||||
output += `\t${JSON.stringify(providerId)}: ${catalogConstName(providerId)},\n`;
|
||||
}
|
||||
output += `} as const;\n`;
|
||||
writeFileSync(join(packageRoot, "src/models.generated.ts"), output);
|
||||
console.log("Generated src/models.generated.ts");
|
||||
}
|
||||
|
||||
if (generatorOptions.jsonOutputDir) {
|
||||
|
||||
@@ -0,0 +1,261 @@
|
||||
import { createHash } from "node:crypto";
|
||||
import { existsSync, readFileSync, readdirSync, statSync } from "node:fs";
|
||||
import { join } from "node:path";
|
||||
|
||||
export const MODEL_DATA_SCHEMA_VERSION = 1;
|
||||
export const MODEL_DATA_MANIFEST_FILE = ".manifest.json";
|
||||
|
||||
export type ModelDataStructure = Record<string, Record<string, string>>;
|
||||
|
||||
export interface ModelDataManifest {
|
||||
schemaVersion: number;
|
||||
structureHash: string;
|
||||
files: Record<string, string>;
|
||||
}
|
||||
|
||||
const JSON_STRING_PATTERN = '"(?:\\\\.|[^"\\\\])*"';
|
||||
const MODEL_SHAPE_PATTERN = new RegExp(`^\\t(${JSON_STRING_PATTERN}): Model<(${JSON_STRING_PATTERN})> & \\{$`);
|
||||
const MODEL_ID_PATTERN = new RegExp(`^\\t\\tid: (${JSON_STRING_PATTERN});$`);
|
||||
const MODEL_PROVIDER_PATTERN = new RegExp(`^\\t\\tprovider: (${JSON_STRING_PATTERN});$`);
|
||||
|
||||
function sha256(value: string): string {
|
||||
return createHash("sha256").update(value).digest("hex");
|
||||
}
|
||||
|
||||
function parseJsonString(value: string, description: string): string {
|
||||
const parsed: unknown = JSON.parse(value);
|
||||
if (typeof parsed !== "string") throw new Error(`${description} is not a string`);
|
||||
return parsed;
|
||||
}
|
||||
|
||||
function sortedRecord<T>(entries: Iterable<readonly [string, T]>): Record<string, T> {
|
||||
return Object.fromEntries(Array.from(entries).sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0)));
|
||||
}
|
||||
|
||||
function sameStrings(a: readonly string[], b: readonly string[]): boolean {
|
||||
return a.length === b.length && a.every((value, index) => value === b[index]);
|
||||
}
|
||||
|
||||
function describeSetDifference(expected: readonly string[], actual: readonly string[]): string {
|
||||
const expectedSet = new Set(expected);
|
||||
const actualSet = new Set(actual);
|
||||
const missing = expected.filter((value) => !actualSet.has(value));
|
||||
const extra = actual.filter((value) => !expectedSet.has(value));
|
||||
return [missing.length > 0 ? `missing: ${missing.join(", ")}` : "", extra.length > 0 ? `extra: ${extra.join(", ")}` : ""]
|
||||
.filter(Boolean)
|
||||
.join("; ");
|
||||
}
|
||||
|
||||
function parseProviderStructure(path: string, providerId: string): Record<string, string> {
|
||||
const source = readFileSync(path, "utf8");
|
||||
const expectedImport = `import values from "./data/${providerId}.json" with { type: "json" };`;
|
||||
if (!source.includes(expectedImport)) {
|
||||
throw new Error(`${path} does not import ${providerId}.json`);
|
||||
}
|
||||
|
||||
const models = new Map<string, string>();
|
||||
const lines = source.split("\n");
|
||||
for (let index = 0; index < lines.length; index++) {
|
||||
const shapeMatch = MODEL_SHAPE_PATTERN.exec(lines[index]);
|
||||
if (!shapeMatch) continue;
|
||||
|
||||
const idMatch = MODEL_ID_PATTERN.exec(lines[index + 1] ?? "");
|
||||
const providerMatch = MODEL_PROVIDER_PATTERN.exec(lines[index + 2] ?? "");
|
||||
if (!idMatch || !providerMatch || lines[index + 3] !== "\t};") {
|
||||
throw new Error(`${path}:${index + 1} has a malformed generated model declaration`);
|
||||
}
|
||||
|
||||
const key = parseJsonString(shapeMatch[1], `${path}:${index + 1} model key`);
|
||||
const api = parseJsonString(shapeMatch[2], `${path}:${index + 1} model API`);
|
||||
const id = parseJsonString(idMatch[1], `${path}:${index + 2} model ID`);
|
||||
const provider = parseJsonString(providerMatch[1], `${path}:${index + 3} provider ID`);
|
||||
if (id !== key) throw new Error(`${path}:${index + 1} declares key ${key} with ID ${id}`);
|
||||
if (provider !== providerId) {
|
||||
throw new Error(`${path}:${index + 1} declares provider ${provider} instead of ${providerId}`);
|
||||
}
|
||||
if (models.has(key)) throw new Error(`${path} declares model ${key} more than once`);
|
||||
models.set(key, api);
|
||||
index += 3;
|
||||
}
|
||||
|
||||
if (models.size === 0) throw new Error(`${path} contains no generated model declarations`);
|
||||
return sortedRecord(models);
|
||||
}
|
||||
|
||||
export function readModelDataStructure(packageRoot: string): ModelDataStructure {
|
||||
const providersDir = join(packageRoot, "src", "providers");
|
||||
const shardProviderIds = readdirSync(providersDir)
|
||||
.filter((entry) => entry.endsWith(".models.ts"))
|
||||
.map((entry) => entry.slice(0, -".models.ts".length))
|
||||
.sort();
|
||||
if (shardProviderIds.length === 0) throw new Error(`No generated provider shards found under ${providersDir}`);
|
||||
|
||||
const aggregator = readFileSync(join(packageRoot, "src", "models.generated.ts"), "utf8");
|
||||
const importedProviderIds = Array.from(
|
||||
aggregator.matchAll(/^import \{ [A-Z0-9_]+_MODELS \} from "\.\/providers\/([^"/]+)\.models\.ts";$/gm),
|
||||
(match) => match[1],
|
||||
).sort();
|
||||
if (!sameStrings(shardProviderIds, importedProviderIds)) {
|
||||
throw new Error(
|
||||
`Generated model aggregator and provider shards do not match (${describeSetDifference(shardProviderIds, importedProviderIds)})`,
|
||||
);
|
||||
}
|
||||
|
||||
return sortedRecord(
|
||||
shardProviderIds.map((providerId) => [
|
||||
providerId,
|
||||
parseProviderStructure(join(providersDir, `${providerId}.models.ts`), providerId),
|
||||
] as const),
|
||||
);
|
||||
}
|
||||
|
||||
export function modelDataStructureHash(structure: ModelDataStructure): string {
|
||||
return sha256(JSON.stringify(structure));
|
||||
}
|
||||
|
||||
export function createModelDataManifest(
|
||||
structure: ModelDataStructure,
|
||||
fileContents: Readonly<Record<string, string>>,
|
||||
): ModelDataManifest {
|
||||
return {
|
||||
schemaVersion: MODEL_DATA_SCHEMA_VERSION,
|
||||
structureHash: modelDataStructureHash(structure),
|
||||
files: sortedRecord(Object.entries(fileContents).map(([file, content]) => [file, sha256(content)] as const)),
|
||||
};
|
||||
}
|
||||
|
||||
function isRecord(value: unknown): value is Record<string, unknown> {
|
||||
return typeof value === "object" && value !== null && !Array.isArray(value);
|
||||
}
|
||||
|
||||
function readJsonObject(path: string, description: string, errors: string[]): Record<string, unknown> | undefined {
|
||||
let parsed: unknown;
|
||||
try {
|
||||
parsed = JSON.parse(readFileSync(path, "utf8"));
|
||||
} catch (error) {
|
||||
errors.push(`${description} is not valid JSON: ${error instanceof Error ? error.message : String(error)}`);
|
||||
return undefined;
|
||||
}
|
||||
if (!isRecord(parsed)) {
|
||||
errors.push(`${description} must contain a JSON object`);
|
||||
return undefined;
|
||||
}
|
||||
return parsed;
|
||||
}
|
||||
|
||||
function validateModelValue(
|
||||
value: unknown,
|
||||
providerId: string,
|
||||
modelId: string,
|
||||
expectedApi: string,
|
||||
errors: string[],
|
||||
): void {
|
||||
const label = `${providerId}/${modelId}`;
|
||||
if (!isRecord(value)) {
|
||||
errors.push(`${label} must be an object`);
|
||||
return;
|
||||
}
|
||||
if (value.id !== modelId) errors.push(`${label} has id ${JSON.stringify(value.id)}, expected ${JSON.stringify(modelId)}`);
|
||||
if (value.provider !== providerId) {
|
||||
errors.push(`${label} has provider ${JSON.stringify(value.provider)}, expected ${JSON.stringify(providerId)}`);
|
||||
}
|
||||
if (value.api !== expectedApi) {
|
||||
errors.push(`${label} has api ${JSON.stringify(value.api)}, expected ${JSON.stringify(expectedApi)}`);
|
||||
}
|
||||
if (typeof value.name !== "string" || value.name.length === 0) errors.push(`${label} has no model name`);
|
||||
if (typeof value.baseUrl !== "string") errors.push(`${label} has no baseUrl string`);
|
||||
if (typeof value.reasoning !== "boolean") errors.push(`${label} has no reasoning boolean`);
|
||||
if (
|
||||
!Array.isArray(value.input) ||
|
||||
value.input.length === 0 ||
|
||||
value.input.some((entry) => entry !== "text" && entry !== "image")
|
||||
) {
|
||||
errors.push(`${label} has invalid input modalities`);
|
||||
}
|
||||
if (typeof value.contextWindow !== "number" || !Number.isFinite(value.contextWindow) || value.contextWindow <= 0) {
|
||||
errors.push(`${label} has invalid contextWindow`);
|
||||
}
|
||||
if (typeof value.maxTokens !== "number" || !Number.isFinite(value.maxTokens) || value.maxTokens <= 0) {
|
||||
errors.push(`${label} has invalid maxTokens`);
|
||||
}
|
||||
if (!isRecord(value.cost)) {
|
||||
errors.push(`${label} has invalid cost metadata`);
|
||||
} else {
|
||||
for (const field of ["input", "output", "cacheRead", "cacheWrite"] as const) {
|
||||
const cost = value.cost[field];
|
||||
if (typeof cost !== "number" || !Number.isFinite(cost)) {
|
||||
errors.push(`${label} has invalid cost.${field}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function throwValidationErrors(errors: string[]): never {
|
||||
const visible = errors.slice(0, 30);
|
||||
const suffix = errors.length > visible.length ? `\n ... and ${errors.length - visible.length} more` : "";
|
||||
throw new Error(`Invalid generated model data:\n${visible.map((error) => ` - ${error}`).join("\n")}${suffix}`);
|
||||
}
|
||||
|
||||
export function validateModelDataDirectory(structure: ModelDataStructure, dataDir: string): void {
|
||||
if (!existsSync(dataDir) || !statSync(dataDir).isDirectory()) {
|
||||
throw new Error(`Generated model data directory does not exist: ${dataDir}`);
|
||||
}
|
||||
|
||||
const errors: string[] = [];
|
||||
const expectedFiles = Object.keys(structure)
|
||||
.map((providerId) => `${providerId}.json`)
|
||||
.sort();
|
||||
const actualFiles = readdirSync(dataDir)
|
||||
.filter((entry) => entry.endsWith(".json") && entry !== MODEL_DATA_MANIFEST_FILE)
|
||||
.sort();
|
||||
if (!sameStrings(expectedFiles, actualFiles)) {
|
||||
errors.push(`provider data files do not match the structural catalog (${describeSetDifference(expectedFiles, actualFiles)})`);
|
||||
}
|
||||
|
||||
const manifestPath = join(dataDir, MODEL_DATA_MANIFEST_FILE);
|
||||
const manifest = readJsonObject(manifestPath, "model data manifest", errors);
|
||||
if (manifest?.schemaVersion !== MODEL_DATA_SCHEMA_VERSION) {
|
||||
errors.push(
|
||||
`model data schema is ${JSON.stringify(manifest?.schemaVersion)}, expected ${MODEL_DATA_SCHEMA_VERSION}`,
|
||||
);
|
||||
}
|
||||
const expectedStructureHash = modelDataStructureHash(structure);
|
||||
if (manifest?.structureHash !== expectedStructureHash) {
|
||||
errors.push("model data generation stamp does not match the structural catalog");
|
||||
}
|
||||
const manifestFiles = isRecord(manifest?.files) ? manifest.files : undefined;
|
||||
if (!manifestFiles) errors.push("model data manifest has no file hashes");
|
||||
else {
|
||||
const manifestFileNames = Object.keys(manifestFiles).sort();
|
||||
if (!sameStrings(expectedFiles, manifestFileNames)) {
|
||||
errors.push(`manifest file hashes do not match provider data files (${describeSetDifference(expectedFiles, manifestFileNames)})`);
|
||||
}
|
||||
}
|
||||
|
||||
for (const [providerId, expectedModels] of Object.entries(structure)) {
|
||||
const filename = `${providerId}.json`;
|
||||
const path = join(dataDir, filename);
|
||||
if (!existsSync(path)) continue;
|
||||
const content = readFileSync(path, "utf8");
|
||||
if (manifestFiles && manifestFiles[filename] !== sha256(content)) {
|
||||
errors.push(`${filename} does not match its manifest hash`);
|
||||
}
|
||||
const values = readJsonObject(path, filename, errors);
|
||||
if (!values) continue;
|
||||
const expectedModelIds = Object.keys(expectedModels).sort();
|
||||
const actualModelIds = Object.keys(values).sort();
|
||||
if (!sameStrings(expectedModelIds, actualModelIds)) {
|
||||
errors.push(`${filename} model IDs do not match the structural catalog (${describeSetDifference(expectedModelIds, actualModelIds)})`);
|
||||
}
|
||||
for (const [modelId, api] of Object.entries(expectedModels)) {
|
||||
if (modelId in values) validateModelValue(values[modelId], providerId, modelId, api, errors);
|
||||
}
|
||||
}
|
||||
|
||||
if (errors.length > 0) throwValidationErrors(errors);
|
||||
}
|
||||
|
||||
export function validateGeneratedModelData(packageRoot: string): void {
|
||||
const structure = readModelDataStructure(packageRoot);
|
||||
validateModelDataDirectory(structure, join(packageRoot, "src", "providers", "data"));
|
||||
}
|
||||
Reference in New Issue
Block a user