ton/pi_harness
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(ai): separate generated model data (#6765)

Browse Source
This commit is contained in:
Armin Ronacher
2026-07-17 11:08:48 +02:00
committed by GitHub
parent b0c2a90e51
commit a9f6a3159a
44 changed files with 4455 additions and 19265 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/ai/scripts/generate-models.ts
+26 -53
View File
@@ -2287,74 +2287,56 @@ async function generateModels() {
}
const sortedProviderIds = Object.keys(providers).sort();
const jsonProviders: Record<string, Record<string, Model<any>>> = {};
for (const providerId of sortedProviderIds) {
jsonProviders[providerId] = {};
for (const modelId of Object.keys(providers[providerId]).sort()) {
jsonProviders[providerId][modelId] = providers[providerId][modelId];
}
}
const writeJson = (path: string, value: unknown) => writeFileSync(path, `${JSON.stringify(value)}\n`);
if (!generatorOptions.jsonOnly) {
// Generate TypeScript files: one catalog per provider plus an aggregator
// Generate TypeScript structural catalogs and adjacent JSON values.
const generatedHeader = `// This file is auto-generated by scripts/generate-models.ts
// Do not edit manually - run 'npm run generate-models' to update
`;
const catalogConstName = (providerId: string) =>
`${providerId.toUpperCase().replace(/[^A-Z0-9]+/g, "_")}_MODELS`;
const providersDir = join(packageRoot, "src/providers");
const dataDir = join(providersDir, "data");
function emitModel(model: Model<any>, indent: string): string {
let output = `${indent}"${model.id}": {\n`;
output += `${indent}\tid: "${model.id}",\n`;
output += `${indent}\tname: "${model.name}",\n`;
output += `${indent}\tapi: "${model.api}",\n`;
output += `${indent}\tprovider: "${model.provider}",\n`;
if (model.baseUrl !== undefined) {
output += `${indent}\tbaseUrl: "${model.baseUrl}",\n`;
}
if (model.headers) {
output += `${indent}\theaders: ${JSON.stringify(model.headers)},\n`;
}
if (model.compat) {
output += `${indent}\tcompat: ${JSON.stringify(model.compat)},\n`;
}
output += `${indent}\treasoning: ${model.reasoning},\n`;
if (model.thinkingLevelMap) {
output += `${indent}\tthinkingLevelMap: ${JSON.stringify(model.thinkingLevelMap)},\n`;
}
output += `${indent}\tinput: [${model.input.map(i => `"${i}"`).join(", ")}],\n`;
output += `${indent}\tcost: {\n`;
output += `${indent}\t\tinput: ${model.cost.input},\n`;
output += `${indent}\t\toutput: ${model.cost.output},\n`;
output += `${indent}\t\tcacheRead: ${model.cost.cacheRead},\n`;
output += `${indent}\t\tcacheWrite: ${model.cost.cacheWrite},\n`;
if (model.cost.tiers) {
output += `${indent}\t\ttiers: ${JSON.stringify(model.cost.tiers)},\n`;
}
output += `${indent}\t},\n`;
output += `${indent}\tcontextWindow: ${model.contextWindow},\n`;
output += `${indent}\tmaxTokens: ${model.maxTokens},\n`;
output += `${indent}} satisfies Model<"${model.api}">,\n`;
return output;
function emitModelShape(model: Model<any>, indent: string): string {
return `${indent}${JSON.stringify(model.id)}: Model<${JSON.stringify(model.api)}> & {\n${indent}\tid: ${JSON.stringify(model.id)};\n${indent}\tprovider: ${JSON.stringify(model.provider)};\n${indent}};\n`;
}
const providersDir = join(packageRoot, "src/providers");
// Remove stale per-provider catalogs
// Remove stale per-provider catalogs and their generated values.
for (const entry of readdirSync(providersDir)) {
if (entry.endsWith(".models.ts")) {
rmSync(join(providersDir, entry));
}
}
rmSync(dataDir, { recursive: true, force: true });
mkdirSync(dataDir, { recursive: true });
// Per-provider catalogs (sorted for deterministic output)
// Per-provider catalog structure and values (sorted for deterministic output).
for (const providerId of sortedProviderIds) {
const models = providers[providerId];
let output = generatedHeader;
output += `import type { Model } from "../types.ts";\n\n`;
output += `export const ${catalogConstName(providerId)} = {\n`;
const sortedModelIds = Object.keys(models).sort();
let output = generatedHeader;
output += `import values from "./data/${providerId}.json" with { type: "json" };\n`;
output += `import type { Model } from "../types.ts";\n\n`;
output += `export const ${catalogConstName(providerId)} = values as {\n`;
for (const modelId of sortedModelIds) {
output += emitModel(models[modelId], "\t");
output += emitModelShape(models[modelId], "\t");
}
output += `} as const;\n`;
output += `};\n`;
writeFileSync(join(providersDir, `${providerId}.models.ts`), output);
writeJson(join(dataDir, `${providerId}.json`), jsonProviders[providerId]);
}
console.log(`Generated ${sortedProviderIds.length} catalogs under src/providers/`);
console.log(`Generated ${sortedProviderIds.length} catalog structures under src/providers/`);
console.log("Generated JSON model values under src/providers/data/");
// Aggregator
let output = generatedHeader;
@@ -2371,18 +2353,9 @@ async function generateModels() {
}
if (generatorOptions.jsonOutputDir) {
const jsonProviders: Record<string, Record<string, Model<any>>> = {};
for (const providerId of sortedProviderIds) {
jsonProviders[providerId] = {};
for (const modelId of Object.keys(providers[providerId]).sort()) {
jsonProviders[providerId][modelId] = providers[providerId][modelId];
}
}
const providerOutputDir = join(generatorOptions.jsonOutputDir, "providers");
rmSync(generatorOptions.jsonOutputDir, { recursive: true, force: true });
mkdirSync(providerOutputDir, { recursive: true });
const writeJson = (path: string, value: unknown) => writeFileSync(path, `${JSON.stringify(value)}\n`);
writeJson(join(generatorOptions.jsonOutputDir, "models.json"), jsonProviders);
writeJson(join(generatorOptions.jsonOutputDir, "providers.json"), sortedProviderIds);
for (const providerId of sortedProviderIds) {