diff --git a/.github/workflows/publish-model-catalog.yml b/.github/workflows/publish-model-catalog.yml new file mode 100644 index 00000000..03641352 --- /dev/null +++ b/.github/workflows/publish-model-catalog.yml @@ -0,0 +1,112 @@ +name: Publish Model Catalog + +on: + workflow_run: + workflows: + - CI + types: + - completed + pull_request: + paths: + - '.github/workflows/publish-model-catalog.yml' + - '.gitignore' + - 'package.json' + - 'packages/ai/**' + - 'scripts/publish-model-catalog.mjs' + schedule: + - cron: '17 */4 * * *' + workflow_dispatch: + inputs: + source_ref: + description: 'Commit, branch, or tag to generate from' + required: false + default: 'main' + type: string + publish: + description: 'Upload the generated catalog to production R2' + required: true + default: false + type: boolean + +permissions: + contents: read + +jobs: + generate: + if: ${{ github.event_name != 'workflow_run' || (github.event.workflow_run.conclusion == 'success' && github.event.workflow_run.head_branch == 'main') }} + runs-on: ubuntu-latest + env: + SOURCE_REF: ${{ github.event.workflow_run.head_sha || github.event.pull_request.head.sha || inputs.source_ref || github.sha }} + steps: + - name: Checkout + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + ref: ${{ env.SOURCE_REF }} + persist-credentials: false + + - name: Setup Node.js + uses: actions/setup-node@39370e3970a6d050c480ffad4ff0ed4d3fdee5af # v4.1.0 + with: + node-version: '22' + cache: npm + + - name: Install dependencies + run: npm ci --ignore-scripts + + - name: Generate model catalog JSON + run: npm run generate:model-catalog + + - name: Validate model catalog JSON + run: npm run check:model-catalog + + - name: Upload model catalog JSON + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: model-catalog-json + path: .artifacts/model-catalog + if-no-files-found: error + retention-days: 14 + + publish: + if: ${{ github.event_name == 'schedule' || github.event_name == 'workflow_run' || (github.event_name == 'workflow_dispatch' && inputs.publish) }} + needs: generate + runs-on: ubuntu-latest + environment: pi-model-upload + concurrency: + group: publish-model-catalog-r2 + cancel-in-progress: true + env: + SOURCE_REF: ${{ github.event.workflow_run.head_sha || inputs.source_ref || github.sha }} + AWS_ACCESS_KEY_ID: ${{ secrets.PI_ARTIFACTS_R2_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.PI_ARTIFACTS_R2_SECRET_ACCESS_KEY }} + AWS_DEFAULT_REGION: auto + AWS_EC2_METADATA_DISABLED: 'true' + R2_ENDPOINT: https://67c0d357268b0fca6e0b465bb9d01b84.r2.cloudflarestorage.com + steps: + - name: Checkout + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + ref: ${{ env.SOURCE_REF }} + persist-credentials: false + + - name: Setup Node.js + uses: actions/setup-node@39370e3970a6d050c480ffad4ff0ed4d3fdee5af # v4.1.0 + with: + node-version: '22' + + - name: Download model catalog JSON + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 + with: + name: model-catalog-json + path: .artifacts/model-catalog + + - name: Verify AWS CLI + run: aws --version + + - name: Publish model catalog to R2 + run: | + node scripts/publish-model-catalog.mjs \ + --input .artifacts/model-catalog \ + --bucket pi-artifacts \ + --endpoint "$R2_ENDPOINT" \ + --source-commit "$(git rev-parse HEAD)" diff --git a/.gitignore b/.gitignore index 12a09a8f..49dc29c7 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ node_modules/ dist/ +.artifacts/ *.log .DS_Store *.tsbuildinfo diff --git a/package.json b/package.json index 4ec90289..83462d28 100644 --- a/package.json +++ b/package.json @@ -19,6 +19,8 @@ "check:shrinkwrap": "node scripts/generate-coding-agent-shrinkwrap.mjs --check", "check:install-lock:coding-agent": "node scripts/generate-coding-agent-install-lock.mjs --check", "check:ts-imports": "node scripts/check-ts-relative-imports.mjs", + "generate:model-catalog": "npm --prefix packages/ai run generate-model-catalog", + "check:model-catalog": "node scripts/publish-model-catalog.mjs --input .artifacts/model-catalog --dry-run", "profile:tui": "node scripts/profile-coding-agent-node.mjs --mode tui", "profile:rpc": "node scripts/profile-coding-agent-node.mjs --mode rpc", "test": "npm run test --workspaces --if-present", diff --git a/packages/ai/package.json b/packages/ai/package.json index d0edb56b..e2ecd28c 100644 --- a/packages/ai/package.json +++ b/packages/ai/package.json @@ -46,6 +46,7 @@ "scripts": { "clean": "shx rm -rf dist", "generate-models": "node scripts/generate-models.ts", + "generate-model-catalog": "node scripts/generate-models.ts --strict --json-only --json-output ../../.artifacts/model-catalog", "generate-image-models": "node scripts/generate-image-models.ts", "build": "npm run generate-models && npm run generate-image-models && tsgo -p tsconfig.build.json", "test": "vitest --run", diff --git a/packages/ai/scripts/generate-models.ts b/packages/ai/scripts/generate-models.ts index 6f194818..d740182b 100644 --- a/packages/ai/scripts/generate-models.ts +++ b/packages/ai/scripts/generate-models.ts @@ -1,7 +1,7 @@ #!/usr/bin/env node -import { readdirSync, rmSync, writeFileSync } from "fs"; -import { join, dirname } from "path"; +import { mkdirSync, readdirSync, rmSync, writeFileSync } from "fs"; +import { dirname, join, resolve } from "path"; import { fileURLToPath } from "url"; import { CLOUDFLARE_AI_GATEWAY_ANTHROPIC_BASE_URL, @@ -22,6 +22,40 @@ const __filename = fileURLToPath(import.meta.url); const __dirname = dirname(__filename); const packageRoot = join(__dirname, ".."); +function readGeneratorOptions(args: string[]): { + strict: boolean; + jsonOnly: boolean; + jsonOutputDir: string | undefined; +} { + let strict = false; + let jsonOnly = false; + let jsonOutputDir: string | undefined; + + for (let index = 0; index < args.length; index++) { + const arg = args[index]; + if (arg === "--strict") { + strict = true; + continue; + } + if (arg === "--json-only") { + jsonOnly = true; + continue; + } + if (arg === "--json-output") { + const value = args[++index]; + if (!value) throw new Error("--json-output requires a directory"); + jsonOutputDir = resolve(value); + continue; + } + throw new Error(`Unknown argument: ${arg}`); + } + + if (jsonOnly && !jsonOutputDir) throw new Error("--json-only requires --json-output"); + return { strict, jsonOnly, jsonOutputDir }; +} + +const generatorOptions = readGeneratorOptions(process.argv.slice(2)); + interface ModelsDevModel { id: string; name: string; @@ -688,6 +722,7 @@ async function fetchNvidiaNimModelIds(): Promise> { try { console.log("Fetching models from NVIDIA NIM API..."); const response = await fetch(`${NVIDIA_BASE_URL}/models`); + if (!response.ok) throw new Error(`NVIDIA NIM API returned ${response.status}`); const data = (await response.json()) as { data?: NvidiaNimModelListItem[] }; const modelIds = new Map(); @@ -700,6 +735,7 @@ async function fetchNvidiaNimModelIds(): Promise> { return modelIds; } catch (error) { console.error("Failed to fetch NVIDIA NIM models:", error); + if (generatorOptions.strict) throw error; return new Map(); } } @@ -708,6 +744,7 @@ async function fetchOpenRouterModels(): Promise[]> { try { console.log("Fetching models from OpenRouter API..."); const response = await fetch("https://openrouter.ai/api/v1/models"); + if (!response.ok) throw new Error(`OpenRouter API returned ${response.status}`); const data = await response.json(); const models: Model[] = []; @@ -760,6 +797,7 @@ async function fetchOpenRouterModels(): Promise[]> { return models; } catch (error) { console.error("Failed to fetch OpenRouter models:", error); + if (generatorOptions.strict) throw error; return []; } } @@ -768,6 +806,7 @@ async function fetchAiGatewayModels(): Promise[]> { try { console.log("Fetching models from Vercel AI Gateway API..."); const response = await fetch(`${AI_GATEWAY_MODELS_URL}/models`); + if (!response.ok) throw new Error(`Vercel AI Gateway API returned ${response.status}`); const data = await response.json(); const models: Model[] = []; @@ -818,6 +857,7 @@ async function fetchAiGatewayModels(): Promise[]> { return models; } catch (error) { console.error("Failed to fetch Vercel AI Gateway models:", error); + if (generatorOptions.strict) throw error; return []; } } @@ -826,6 +866,7 @@ async function loadModelsDevData(): Promise[]> { try { console.log("Fetching models from models.dev API..."); const response = await fetch("https://models.dev/api.json"); + if (!response.ok) throw new Error(`models.dev API returned ${response.status}`); const data = await response.json(); const models: Model[] = []; @@ -1703,6 +1744,7 @@ async function loadModelsDevData(): Promise[]> { return models; } catch (error) { console.error("Failed to load models.dev data:", error); + if (generatorOptions.strict) throw error; return []; } } @@ -2236,85 +2278,110 @@ async function generateModels() { } } - // Generate TypeScript files: one catalog per provider plus an aggregator - const generatedHeader = `// This file is auto-generated by scripts/generate-models.ts + const sortedProviderIds = Object.keys(providers).sort(); + + if (!generatorOptions.jsonOnly) { + // Generate TypeScript files: one catalog per provider plus an aggregator + const generatedHeader = `// This file is auto-generated by scripts/generate-models.ts // Do not edit manually - run 'npm run generate-models' to update `; - const catalogConstName = (providerId: string) => `${providerId.toUpperCase().replace(/[^A-Z0-9]+/g, "_")}_MODELS`; + const catalogConstName = (providerId: string) => + `${providerId.toUpperCase().replace(/[^A-Z0-9]+/g, "_")}_MODELS`; - function emitModel(model: Model, indent: string): string { - let output = `${indent}"${model.id}": {\n`; - output += `${indent}\tid: "${model.id}",\n`; - output += `${indent}\tname: "${model.name}",\n`; - output += `${indent}\tapi: "${model.api}",\n`; - output += `${indent}\tprovider: "${model.provider}",\n`; - if (model.baseUrl !== undefined) { - output += `${indent}\tbaseUrl: "${model.baseUrl}",\n`; + function emitModel(model: Model, indent: string): string { + let output = `${indent}"${model.id}": {\n`; + output += `${indent}\tid: "${model.id}",\n`; + output += `${indent}\tname: "${model.name}",\n`; + output += `${indent}\tapi: "${model.api}",\n`; + output += `${indent}\tprovider: "${model.provider}",\n`; + if (model.baseUrl !== undefined) { + output += `${indent}\tbaseUrl: "${model.baseUrl}",\n`; + } + if (model.headers) { + output += `${indent}\theaders: ${JSON.stringify(model.headers)},\n`; + } + if (model.compat) { + output += `${indent}\tcompat: ${JSON.stringify(model.compat)},\n`; + } + output += `${indent}\treasoning: ${model.reasoning},\n`; + if (model.thinkingLevelMap) { + output += `${indent}\tthinkingLevelMap: ${JSON.stringify(model.thinkingLevelMap)},\n`; + } + output += `${indent}\tinput: [${model.input.map(i => `"${i}"`).join(", ")}],\n`; + output += `${indent}\tcost: {\n`; + output += `${indent}\t\tinput: ${model.cost.input},\n`; + output += `${indent}\t\toutput: ${model.cost.output},\n`; + output += `${indent}\t\tcacheRead: ${model.cost.cacheRead},\n`; + output += `${indent}\t\tcacheWrite: ${model.cost.cacheWrite},\n`; + if (model.cost.tiers) { + output += `${indent}\t\ttiers: ${JSON.stringify(model.cost.tiers)},\n`; + } + output += `${indent}\t},\n`; + output += `${indent}\tcontextWindow: ${model.contextWindow},\n`; + output += `${indent}\tmaxTokens: ${model.maxTokens},\n`; + output += `${indent}} satisfies Model<"${model.api}">,\n`; + return output; } - if (model.headers) { - output += `${indent}\theaders: ${JSON.stringify(model.headers)},\n`; - } - if (model.compat) { - output += `${indent}\tcompat: ${JSON.stringify(model.compat)},\n`; - } - output += `${indent}\treasoning: ${model.reasoning},\n`; - if (model.thinkingLevelMap) { - output += `${indent}\tthinkingLevelMap: ${JSON.stringify(model.thinkingLevelMap)},\n`; - } - output += `${indent}\tinput: [${model.input.map(i => `"${i}"`).join(", ")}],\n`; - output += `${indent}\tcost: {\n`; - output += `${indent}\t\tinput: ${model.cost.input},\n`; - output += `${indent}\t\toutput: ${model.cost.output},\n`; - output += `${indent}\t\tcacheRead: ${model.cost.cacheRead},\n`; - output += `${indent}\t\tcacheWrite: ${model.cost.cacheWrite},\n`; - if (model.cost.tiers) { - output += `${indent}\t\ttiers: ${JSON.stringify(model.cost.tiers)},\n`; - } - output += `${indent}\t},\n`; - output += `${indent}\tcontextWindow: ${model.contextWindow},\n`; - output += `${indent}\tmaxTokens: ${model.maxTokens},\n`; - output += `${indent}} satisfies Model<"${model.api}">,\n`; - return output; - } - const sortedProviderIds = Object.keys(providers).sort(); - const providersDir = join(packageRoot, "src/providers"); + const providersDir = join(packageRoot, "src/providers"); - // Remove stale per-provider catalogs - for (const entry of readdirSync(providersDir)) { - if (entry.endsWith(".models.ts")) { - rmSync(join(providersDir, entry)); + // Remove stale per-provider catalogs + for (const entry of readdirSync(providersDir)) { + if (entry.endsWith(".models.ts")) { + rmSync(join(providersDir, entry)); + } } - } - // Per-provider catalogs (sorted for deterministic output) - for (const providerId of sortedProviderIds) { - const models = providers[providerId]; + // Per-provider catalogs (sorted for deterministic output) + for (const providerId of sortedProviderIds) { + const models = providers[providerId]; + let output = generatedHeader; + output += `import type { Model } from "../types.ts";\n\n`; + output += `export const ${catalogConstName(providerId)} = {\n`; + const sortedModelIds = Object.keys(models).sort(); + for (const modelId of sortedModelIds) { + output += emitModel(models[modelId], "\t"); + } + output += `} as const;\n`; + writeFileSync(join(providersDir, `${providerId}.models.ts`), output); + } + console.log(`Generated ${sortedProviderIds.length} catalogs under src/providers/`); + + // Aggregator let output = generatedHeader; - output += `import type { Model } from "../types.ts";\n\n`; - output += `export const ${catalogConstName(providerId)} = {\n`; - const sortedModelIds = Object.keys(models).sort(); - for (const modelId of sortedModelIds) { - output += emitModel(models[modelId], "\t"); + for (const providerId of sortedProviderIds) { + output += `import { ${catalogConstName(providerId)} } from "./providers/${providerId}.models.ts";\n`; + } + output += `\nexport const MODELS = {\n`; + for (const providerId of sortedProviderIds) { + output += `\t${JSON.stringify(providerId)}: ${catalogConstName(providerId)},\n`; } output += `} as const;\n`; - writeFileSync(join(providersDir, `${providerId}.models.ts`), output); + writeFileSync(join(packageRoot, "src/models.generated.ts"), output); + console.log("Generated src/models.generated.ts"); } - console.log(`Generated ${sortedProviderIds.length} catalogs under src/providers/`); - // Aggregator - let output = generatedHeader; - for (const providerId of sortedProviderIds) { - output += `import { ${catalogConstName(providerId)} } from "./providers/${providerId}.models.ts";\n`; + if (generatorOptions.jsonOutputDir) { + const jsonProviders: Record>> = {}; + for (const providerId of sortedProviderIds) { + jsonProviders[providerId] = {}; + for (const modelId of Object.keys(providers[providerId]).sort()) { + jsonProviders[providerId][modelId] = providers[providerId][modelId]; + } + } + + const providerOutputDir = join(generatorOptions.jsonOutputDir, "providers"); + rmSync(generatorOptions.jsonOutputDir, { recursive: true, force: true }); + mkdirSync(providerOutputDir, { recursive: true }); + const writeJson = (path: string, value: unknown) => writeFileSync(path, `${JSON.stringify(value)}\n`); + writeJson(join(generatorOptions.jsonOutputDir, "models.json"), jsonProviders); + writeJson(join(generatorOptions.jsonOutputDir, "providers.json"), sortedProviderIds); + for (const providerId of sortedProviderIds) { + writeJson(join(providerOutputDir, `${providerId}.json`), jsonProviders[providerId]); + } + console.log(`Generated JSON model catalog under ${generatorOptions.jsonOutputDir}`); } - output += `\nexport const MODELS = {\n`; - for (const providerId of sortedProviderIds) { - output += `\t${JSON.stringify(providerId)}: ${catalogConstName(providerId)},\n`; - } - output += `} as const;\n`; - writeFileSync(join(packageRoot, "src/models.generated.ts"), output); - console.log("Generated src/models.generated.ts"); // Print statistics const totalModels = allModels.length; @@ -2330,4 +2397,7 @@ async function generateModels() { } // Run the generator -generateModels().catch(console.error); +generateModels().catch((error) => { + console.error(error); + process.exitCode = 1; +}); diff --git a/scripts/publish-model-catalog.mjs b/scripts/publish-model-catalog.mjs new file mode 100644 index 00000000..a8a30d43 --- /dev/null +++ b/scripts/publish-model-catalog.mjs @@ -0,0 +1,295 @@ +#!/usr/bin/env node + +import { createHash } from "node:crypto"; +import { + mkdtempSync, + readFileSync, + readdirSync, + rmSync, + writeFileSync, +} from "node:fs"; +import { tmpdir } from "node:os"; +import { join, resolve } from "node:path"; +import { spawnSync } from "node:child_process"; +import { isDeepStrictEqual } from "node:util"; + +const CATALOG_SCHEMA_VERSION = 1; +const CATALOG_PREFIX = `models/v${CATALOG_SCHEMA_VERSION}`; +const CATALOG_INDEX_KEY = `${CATALOG_PREFIX}/index.json`; +// Bump this only when generated model metadata requires behavior unavailable in older pi clients. +const MINIMUM_PI_VERSION = "0.80.7"; +const JSON_CONTENT_TYPE = "application/json; charset=utf-8"; +const IMMUTABLE_CACHE_CONTROL = "public, max-age=31536000, immutable"; +const INDEX_CACHE_CONTROL = "no-store"; +const REQUIRED_PROVIDERS = ["anthropic", "openai", "openrouter"]; +const MINIMUM_MODEL_COUNT = 500; + +function parseArgs(args) { + const options = { + input: undefined, + bucket: undefined, + endpoint: undefined, + sourceCommit: undefined, + dryRun: false, + }; + + for (let index = 0; index < args.length; index++) { + const arg = args[index]; + if (arg === "--dry-run") { + options.dryRun = true; + continue; + } + if (arg === "--input" || arg === "--bucket" || arg === "--endpoint" || arg === "--source-commit") { + const value = args[++index]; + if (!value) throw new Error(`${arg} requires a value`); + options[ + { + "--input": "input", + "--bucket": "bucket", + "--endpoint": "endpoint", + "--source-commit": "sourceCommit", + }[arg] + ] = value; + continue; + } + throw new Error(`Unknown argument: ${arg}`); + } + + if (!options.input) throw new Error("--input is required"); + if (!options.dryRun && !options.bucket) throw new Error("--bucket is required when publishing"); + if (!options.dryRun && !options.endpoint) throw new Error("--endpoint is required when publishing"); + return options; +} + +function readJson(path) { + return JSON.parse(readFileSync(path, "utf8")); +} + +function isRecord(value) { + return typeof value === "object" && value !== null && !Array.isArray(value); +} + +function validateBundle(inputDir) { + const modelsPath = join(inputDir, "models.json"); + const providerIndexPath = join(inputDir, "providers.json"); + const providersDir = join(inputDir, "providers"); + const modelsBytes = readFileSync(modelsPath); + const models = JSON.parse(modelsBytes.toString("utf8")); + const providerIds = readJson(providerIndexPath); + + if (!isRecord(models)) throw new Error("models.json must contain an object"); + if (!Array.isArray(providerIds) || !providerIds.every((value) => typeof value === "string")) { + throw new Error("providers.json must contain an array of provider IDs"); + } + + const expectedProviderIds = Object.keys(models).sort(); + if (!isDeepStrictEqual(providerIds, expectedProviderIds)) { + throw new Error("providers.json does not match the sorted providers in models.json"); + } + for (const providerId of REQUIRED_PROVIDERS) { + if (!Object.hasOwn(models, providerId)) throw new Error(`Required provider is missing: ${providerId}`); + } + + let modelCount = 0; + for (const providerId of providerIds) { + const providerModels = models[providerId]; + if (!isRecord(providerModels)) throw new Error(`Provider catalog must be an object: ${providerId}`); + const providerFile = readJson(join(providersDir, `${providerId}.json`)); + if (!isDeepStrictEqual(providerFile, providerModels)) { + throw new Error(`Provider shard does not match models.json: ${providerId}`); + } + for (const [modelId, model] of Object.entries(providerModels)) { + if (!isRecord(model) || model.id !== modelId || model.provider !== providerId) { + throw new Error(`Invalid model entry: ${providerId}/${modelId}`); + } + modelCount++; + } + } + + const shardFiles = readdirSync(providersDir).filter((name) => name.endsWith(".json")).sort(); + const expectedShardFiles = providerIds.map((providerId) => `${providerId}.json`).sort(); + if (!isDeepStrictEqual(shardFiles, expectedShardFiles)) { + throw new Error("Provider shard files do not match providers.json"); + } + if (modelCount < MINIMUM_MODEL_COUNT) { + throw new Error(`Refusing to publish only ${modelCount} models; expected at least ${MINIMUM_MODEL_COUNT}`); + } + + const digest = createHash("sha256").update(modelsBytes).digest("hex"); + return { + modelsPath, + providerIndexPath, + providersDir, + providerIds, + providerCount: providerIds.length, + modelCount, + revision: `sha256-${digest}`, + }; +} + +function gitSourceCommit() { + const result = spawnSync("git", ["rev-parse", "HEAD"], { encoding: "utf8" }); + if (result.status !== 0) throw new Error(`Unable to determine source commit: ${result.stderr.trim()}`); + return result.stdout.trim(); +} + +function aws(args, { allowNotFound = false } = {}) { + const result = spawnSync("aws", args, { + encoding: "utf8", + env: { + ...process.env, + AWS_DEFAULT_REGION: process.env.AWS_DEFAULT_REGION || "auto", + AWS_EC2_METADATA_DISABLED: "true", + }, + }); + if (result.error) throw result.error; + if (result.status === 0) return true; + const message = `${result.stdout}\n${result.stderr}`.trim(); + if (allowNotFound && /(?:404|NoSuchKey|Not Found)/i.test(message)) return false; + throw new Error(`aws ${args.slice(0, 2).join(" ")} failed:\n${message}`); +} + +function downloadIndex(bucket, endpoint, outputPath) { + return aws( + [ + "s3", + "cp", + `s3://${bucket}/${CATALOG_INDEX_KEY}`, + outputPath, + "--endpoint-url", + endpoint, + "--only-show-errors", + ], + { allowNotFound: true }, + ); +} + +function uploadJson(bucket, endpoint, sourcePath, key, cacheControl) { + aws([ + "s3", + "cp", + sourcePath, + `s3://${bucket}/${key}`, + "--endpoint-url", + endpoint, + "--content-type", + JSON_CONTENT_TYPE, + "--cache-control", + cacheControl, + "--only-show-errors", + ]); +} + +function validateIndex(index) { + if (!isRecord(index) || index.schemaVersion !== CATALOG_SCHEMA_VERSION) { + throw new Error(`Existing ${CATALOG_INDEX_KEY} has an unsupported schema`); + } + if (!Array.isArray(index.catalogs)) throw new Error(`Existing ${CATALOG_INDEX_KEY} has no catalogs array`); + for (const catalog of index.catalogs) { + if ( + !isRecord(catalog) || + typeof catalog.minimumPiVersion !== "string" || + typeof catalog.revision !== "string" + ) { + throw new Error(`Existing ${CATALOG_INDEX_KEY} contains an invalid catalog entry`); + } + } + return index; +} + +function comparePiVersions(left, right) { + const leftParts = left.split(".").map(Number); + const rightParts = right.split(".").map(Number); + for (let index = 0; index < Math.max(leftParts.length, rightParts.length); index++) { + const difference = (leftParts[index] || 0) - (rightParts[index] || 0); + if (difference !== 0) return difference; + } + return left.localeCompare(right); +} + +function buildIndex(existingIndex, publication) { + const entry = { + minimumPiVersion: MINIMUM_PI_VERSION, + revision: publication.revision, + sourceCommit: publication.sourceCommit, + publishedAt: new Date().toISOString(), + providerCount: publication.providerCount, + modelCount: publication.modelCount, + }; + const catalogs = (existingIndex?.catalogs || []) + .filter((catalog) => catalog.minimumPiVersion !== MINIMUM_PI_VERSION) + .concat(entry) + .sort((left, right) => comparePiVersions(left.minimumPiVersion, right.minimumPiVersion)); + return { + schemaVersion: CATALOG_SCHEMA_VERSION, + defaultRevision: publication.revision, + catalogs, + }; +} + +async function main() { + const options = parseArgs(process.argv.slice(2)); + const inputDir = resolve(options.input); + const bundle = validateBundle(inputDir); + const publication = { + schemaVersion: CATALOG_SCHEMA_VERSION, + minimumPiVersion: MINIMUM_PI_VERSION, + revision: bundle.revision, + sourceCommit: options.sourceCommit || gitSourceCommit(), + providerCount: bundle.providerCount, + modelCount: bundle.modelCount, + }; + writeFileSync(join(inputDir, "publication.json"), `${JSON.stringify(publication, null, 2)}\n`); + + console.log(JSON.stringify(publication, null, 2)); + if (options.dryRun) { + console.log(`Validated model catalog at ${inputDir}; no objects uploaded.`); + return; + } + + const temporaryDir = mkdtempSync(join(tmpdir(), "pi-model-catalog-")); + try { + const currentIndexPath = join(temporaryDir, "index-current.json"); + const hasCurrentIndex = downloadIndex(options.bucket, options.endpoint, currentIndexPath); + const currentIndex = hasCurrentIndex ? validateIndex(readJson(currentIndexPath)) : undefined; + const currentEntry = currentIndex?.catalogs.find( + (catalog) => catalog.minimumPiVersion === MINIMUM_PI_VERSION, + ); + if (currentIndex?.defaultRevision === bundle.revision && currentEntry?.revision === bundle.revision) { + console.log(`Model catalog ${bundle.revision} is already current; no objects uploaded.`); + return; + } + + const revisionPrefix = `${CATALOG_PREFIX}/revisions/${bundle.revision}`; + uploadJson(options.bucket, options.endpoint, bundle.modelsPath, `${revisionPrefix}/models.json`, IMMUTABLE_CACHE_CONTROL); + uploadJson( + options.bucket, + options.endpoint, + bundle.providerIndexPath, + `${revisionPrefix}/providers.json`, + IMMUTABLE_CACHE_CONTROL, + ); + for (const providerId of bundle.providerIds) { + uploadJson( + options.bucket, + options.endpoint, + join(bundle.providersDir, `${providerId}.json`), + `${revisionPrefix}/providers/${providerId}.json`, + IMMUTABLE_CACHE_CONTROL, + ); + } + + const nextIndex = buildIndex(currentIndex, publication); + const nextIndexPath = join(temporaryDir, "index-next.json"); + writeFileSync(nextIndexPath, `${JSON.stringify(nextIndex, null, 2)}\n`); + uploadJson(options.bucket, options.endpoint, nextIndexPath, CATALOG_INDEX_KEY, INDEX_CACHE_CONTROL); + console.log(`Published ${bundle.revision} to s3://${options.bucket}/${revisionPrefix}`); + } finally { + rmSync(temporaryDir, { recursive: true, force: true }); + } +} + +main().catch((error) => { + console.error(error instanceof Error ? error.message : error); + process.exitCode = 1; +});