fix(ai): validate generated model data before builds
This commit is contained in:
Armin Ronacher
@@ -1,6 +1,6 @@
|
||||
#!/usr/bin/env node
|
||||
|
||||
import { mkdirSync, readdirSync, rmSync, writeFileSync } from "fs";
|
||||
import { existsSync, mkdirSync, mkdtempSync, readFileSync, readdirSync, renameSync, rmSync, writeFileSync } from "fs";
|
||||
import { dirname, join, resolve } from "path";
|
||||
import { fileURLToPath } from "url";
|
||||
import {
|
||||
@@ -18,6 +18,14 @@ import type {
|
||||
OpenAICompletionsCompat,
|
||||
OpenAIResponsesCompat,
|
||||
} from "../src/types.ts";
|
||||
import {
|
||||
createModelDataManifest,
|
||||
type ModelDataStructure,
|
||||
MODEL_DATA_MANIFEST_FILE,
|
||||
readModelDataStructure,
|
||||
validateGeneratedModelData,
|
||||
validateModelDataDirectory,
|
||||
} from "./model-data.ts";
|
||||
|
||||
const __filename = fileURLToPath(import.meta.url);
|
||||
const __dirname = dirname(__filename);
|
||||
@@ -25,11 +33,13 @@ const packageRoot = join(__dirname, "..");
|
||||
|
||||
function readGeneratorOptions(args: string[]): {
|
||||
strict: boolean;
|
||||
dataOnly: boolean;
|
||||
jsonOnly: boolean;
|
||||
jsonOutputDir: string | undefined;
|
||||
pretty: boolean;
|
||||
} {
|
||||
let strict = false;
|
||||
let dataOnly = false;
|
||||
let jsonOnly = false;
|
||||
let jsonOutputDir: string | undefined;
|
||||
let pretty = false;
|
||||
@@ -40,6 +50,10 @@ function readGeneratorOptions(args: string[]): {
|
||||
strict = true;
|
||||
continue;
|
||||
}
|
||||
if (arg === "--data-only") {
|
||||
dataOnly = true;
|
||||
continue;
|
||||
}
|
||||
if (arg === "--json-only") {
|
||||
jsonOnly = true;
|
||||
continue;
|
||||
@@ -58,7 +72,8 @@ function readGeneratorOptions(args: string[]): {
|
||||
}
|
||||
|
||||
if (jsonOnly && !jsonOutputDir) throw new Error("--json-only requires --json-output");
|
||||
return { strict, jsonOnly, jsonOutputDir, pretty };
|
||||
if (dataOnly && (jsonOnly || jsonOutputDir)) throw new Error("--data-only cannot be combined with JSON catalog output");
|
||||
return { strict, dataOnly, jsonOnly, jsonOutputDir, pretty };
|
||||
}
|
||||
|
||||
const generatorOptions = readGeneratorOptions(process.argv.slice(2));
|
||||
@@ -2403,63 +2418,154 @@ async function generateModels() {
|
||||
jsonProviders[providerId][modelId] = providers[providerId][modelId];
|
||||
}
|
||||
}
|
||||
const writeJson = (path: string, value: unknown) =>
|
||||
writeFileSync(path, `${JSON.stringify(value, null, generatorOptions.pretty ? 2 : undefined)}\n`);
|
||||
|
||||
const serializeJson = (value: unknown) => `${JSON.stringify(value, null, generatorOptions.pretty ? 2 : undefined)}\n`;
|
||||
const writeJson = (path: string, value: unknown) => writeFileSync(path, serializeJson(value));
|
||||
let generatedDataProviderIds = sortedProviderIds;
|
||||
let generatedDataProviders = jsonProviders;
|
||||
let modelDataStructure: ModelDataStructure = Object.fromEntries(
|
||||
sortedProviderIds.map((providerId) => [
|
||||
providerId,
|
||||
Object.fromEntries(
|
||||
Object.entries(jsonProviders[providerId]).map(([modelId, model]) => [modelId, model.api]),
|
||||
),
|
||||
]),
|
||||
);
|
||||
|
||||
if (generatorOptions.dataOnly) {
|
||||
modelDataStructure = readModelDataStructure(packageRoot);
|
||||
generatedDataProviderIds = Object.keys(modelDataStructure);
|
||||
const hydratedProviders: typeof jsonProviders = {};
|
||||
const hydrationErrors: string[] = [];
|
||||
for (const [providerId, expectedModels] of Object.entries(modelDataStructure)) {
|
||||
hydratedProviders[providerId] = {};
|
||||
for (const [modelId, expectedApi] of Object.entries(expectedModels)) {
|
||||
const model = jsonProviders[providerId]?.[modelId];
|
||||
if (!model) {
|
||||
hydrationErrors.push(`missing ${providerId}/${modelId}`);
|
||||
continue;
|
||||
}
|
||||
if (model.api !== expectedApi) {
|
||||
hydrationErrors.push(`${providerId}/${modelId} uses ${model.api}, expected ${expectedApi}`);
|
||||
continue;
|
||||
}
|
||||
hydratedProviders[providerId][modelId] = model;
|
||||
}
|
||||
}
|
||||
if (hydrationErrors.length > 0) {
|
||||
throw new Error(`Cannot hydrate the committed model catalog:\n${hydrationErrors.map((error) => ` - ${error}`).join("\n")}`);
|
||||
}
|
||||
generatedDataProviders = hydratedProviders;
|
||||
}
|
||||
|
||||
if (!generatorOptions.jsonOnly) {
|
||||
// Generate TypeScript structural catalogs and adjacent JSON values.
|
||||
const generatedHeader = `// This file is auto-generated by scripts/generate-models.ts
|
||||
// Stage and validate all provider values before replacing the current generated data.
|
||||
const providersDir = join(packageRoot, "src/providers");
|
||||
const dataDir = join(providersDir, "data");
|
||||
const stagingRoot = mkdtempSync(join(providersDir, ".model-generation-"));
|
||||
const stagedDataDir = join(stagingRoot, "data");
|
||||
const previousDataDir = join(stagingRoot, "previous-data");
|
||||
let restoreStructuralCatalog: (() => void) | undefined;
|
||||
try {
|
||||
mkdirSync(stagedDataDir, { recursive: true });
|
||||
const fileContents: Record<string, string> = {};
|
||||
for (const providerId of generatedDataProviderIds) {
|
||||
const filename = `${providerId}.json`;
|
||||
const content = serializeJson(generatedDataProviders[providerId]);
|
||||
fileContents[filename] = content;
|
||||
writeFileSync(join(stagedDataDir, filename), content);
|
||||
}
|
||||
writeJson(
|
||||
join(stagedDataDir, MODEL_DATA_MANIFEST_FILE),
|
||||
createModelDataManifest(modelDataStructure, fileContents),
|
||||
);
|
||||
validateModelDataDirectory(modelDataStructure, stagedDataDir);
|
||||
|
||||
if (!generatorOptions.dataOnly) {
|
||||
// Generate TypeScript structural catalogs only after the model data is complete and valid.
|
||||
const previousShardContents = new Map(
|
||||
readdirSync(providersDir)
|
||||
.filter((entry) => entry.endsWith(".models.ts"))
|
||||
.map((entry) => [entry, readFileSync(join(providersDir, entry), "utf8")] as const),
|
||||
);
|
||||
const aggregatorPath = join(packageRoot, "src/models.generated.ts");
|
||||
const previousAggregator = readFileSync(aggregatorPath, "utf8");
|
||||
restoreStructuralCatalog = () => {
|
||||
for (const entry of readdirSync(providersDir)) {
|
||||
if (entry.endsWith(".models.ts")) rmSync(join(providersDir, entry));
|
||||
}
|
||||
for (const [entry, content] of previousShardContents) {
|
||||
writeFileSync(join(providersDir, entry), content);
|
||||
}
|
||||
writeFileSync(aggregatorPath, previousAggregator);
|
||||
};
|
||||
|
||||
const generatedHeader = `// This file is auto-generated by scripts/generate-models.ts
|
||||
// Do not edit manually - run 'npm run generate-models' to update
|
||||
|
||||
`;
|
||||
const catalogConstName = (providerId: string) =>
|
||||
`${providerId.toUpperCase().replace(/[^A-Z0-9]+/g, "_")}_MODELS`;
|
||||
const providersDir = join(packageRoot, "src/providers");
|
||||
const dataDir = join(providersDir, "data");
|
||||
const catalogConstName = (providerId: string) =>
|
||||
`${providerId.toUpperCase().replace(/[^A-Z0-9]+/g, "_")}_MODELS`;
|
||||
const generatedShardFiles = new Set<string>();
|
||||
|
||||
function emitModelShape(model: Model<any>, indent: string): string {
|
||||
return `${indent}${JSON.stringify(model.id)}: Model<${JSON.stringify(model.api)}> & {\n${indent}\tid: ${JSON.stringify(model.id)};\n${indent}\tprovider: ${JSON.stringify(model.provider)};\n${indent}};\n`;
|
||||
}
|
||||
function emitModelShape(model: Model<any>, indent: string): string {
|
||||
return `${indent}${JSON.stringify(model.id)}: Model<${JSON.stringify(model.api)}> & {\n${indent}\tid: ${JSON.stringify(model.id)};\n${indent}\tprovider: ${JSON.stringify(model.provider)};\n${indent}};\n`;
|
||||
}
|
||||
|
||||
// Remove stale per-provider catalogs and their generated values.
|
||||
for (const entry of readdirSync(providersDir)) {
|
||||
if (entry.endsWith(".models.ts")) {
|
||||
rmSync(join(providersDir, entry));
|
||||
for (const providerId of sortedProviderIds) {
|
||||
const models = providers[providerId];
|
||||
let output = generatedHeader;
|
||||
output += `import values from "./data/${providerId}.json" with { type: "json" };\n`;
|
||||
output += `import type { Model } from "../types.ts";\n\n`;
|
||||
output += `export const ${catalogConstName(providerId)} = values as {\n`;
|
||||
for (const modelId of Object.keys(models).sort()) {
|
||||
output += emitModelShape(models[modelId], "\t");
|
||||
}
|
||||
output += `};\n`;
|
||||
const filename = `${providerId}.models.ts`;
|
||||
generatedShardFiles.add(filename);
|
||||
writeFileSync(join(providersDir, filename), output);
|
||||
}
|
||||
for (const entry of readdirSync(providersDir)) {
|
||||
if (entry.endsWith(".models.ts") && !generatedShardFiles.has(entry)) rmSync(join(providersDir, entry));
|
||||
}
|
||||
console.log(`Generated ${sortedProviderIds.length} catalog structures under src/providers/`);
|
||||
|
||||
let output = generatedHeader;
|
||||
for (const providerId of sortedProviderIds) {
|
||||
output += `import { ${catalogConstName(providerId)} } from "./providers/${providerId}.models.ts";\n`;
|
||||
}
|
||||
output += `\nexport const MODELS = {\n`;
|
||||
for (const providerId of sortedProviderIds) {
|
||||
output += `\t${JSON.stringify(providerId)}: ${catalogConstName(providerId)},\n`;
|
||||
}
|
||||
output += `} as const;\n`;
|
||||
writeFileSync(aggregatorPath, output);
|
||||
console.log("Generated src/models.generated.ts");
|
||||
}
|
||||
}
|
||||
rmSync(dataDir, { recursive: true, force: true });
|
||||
mkdirSync(dataDir, { recursive: true });
|
||||
|
||||
// Per-provider catalog structure and values (sorted for deterministic output).
|
||||
for (const providerId of sortedProviderIds) {
|
||||
const models = providers[providerId];
|
||||
const sortedModelIds = Object.keys(models).sort();
|
||||
let output = generatedHeader;
|
||||
output += `import values from "./data/${providerId}.json" with { type: "json" };\n`;
|
||||
output += `import type { Model } from "../types.ts";\n\n`;
|
||||
output += `export const ${catalogConstName(providerId)} = values as {\n`;
|
||||
for (const modelId of sortedModelIds) {
|
||||
output += emitModelShape(models[modelId], "\t");
|
||||
const hadPreviousData = existsSync(dataDir);
|
||||
if (hadPreviousData) renameSync(dataDir, previousDataDir);
|
||||
try {
|
||||
renameSync(stagedDataDir, dataDir);
|
||||
validateGeneratedModelData(packageRoot);
|
||||
} catch (error) {
|
||||
rmSync(dataDir, { recursive: true, force: true });
|
||||
if (hadPreviousData && existsSync(previousDataDir)) renameSync(previousDataDir, dataDir);
|
||||
throw error;
|
||||
}
|
||||
output += `};\n`;
|
||||
writeFileSync(join(providersDir, `${providerId}.models.ts`), output);
|
||||
writeJson(join(dataDir, `${providerId}.json`), jsonProviders[providerId]);
|
||||
restoreStructuralCatalog = undefined;
|
||||
console.log(
|
||||
generatorOptions.dataOnly
|
||||
? "Hydrated JSON model values under src/providers/data/"
|
||||
: "Generated JSON model values under src/providers/data/",
|
||||
);
|
||||
} catch (error) {
|
||||
restoreStructuralCatalog?.();
|
||||
throw error;
|
||||
} finally {
|
||||
rmSync(stagingRoot, { recursive: true, force: true });
|
||||
}
|
||||
console.log(`Generated ${sortedProviderIds.length} catalog structures under src/providers/`);
|
||||
console.log("Generated JSON model values under src/providers/data/");
|
||||
|
||||
// Aggregator
|
||||
let output = generatedHeader;
|
||||
for (const providerId of sortedProviderIds) {
|
||||
output += `import { ${catalogConstName(providerId)} } from "./providers/${providerId}.models.ts";\n`;
|
||||
}
|
||||
output += `\nexport const MODELS = {\n`;
|
||||
for (const providerId of sortedProviderIds) {
|
||||
output += `\t${JSON.stringify(providerId)}: ${catalogConstName(providerId)},\n`;
|
||||
}
|
||||
output += `} as const;\n`;
|
||||
writeFileSync(join(packageRoot, "src/models.generated.ts"), output);
|
||||
console.log("Generated src/models.generated.ts");
|
||||
}
|
||||
|
||||
if (generatorOptions.jsonOutputDir) {
|
||||
|
||||
Reference in New Issue
Block a user