From ce48d9b4ef72a4e16c868ef92a9c47c002dfea52 Mon Sep 17 00:00:00 2001 From: David Brailovsky Date: Fri, 17 Jul 2026 12:38:12 +0200 Subject: [PATCH] fix(ai): preserve GitHub Copilot long-context pricing tiers, closes #6668 --- package.json | 1 + packages/ai/CHANGELOG.md | 1 + packages/ai/scripts/generate-models.ts | 53 ++++++-- scripts/diff-model-catalog.mjs | 175 +++++++++++++++++++++++++ 4 files changed, 222 insertions(+), 8 deletions(-) create mode 100644 scripts/diff-model-catalog.mjs diff --git a/package.json b/package.json index 6329c159..c919d33e 100644 --- a/package.json +++ b/package.json @@ -21,6 +21,7 @@ "check:ts-imports": "node scripts/check-ts-relative-imports.mjs", "generate:models": "npm --prefix packages/ai run generate-models && npm --prefix packages/ai run generate-image-models", "generate:model-catalog": "npm --prefix packages/ai run generate-model-catalog", + "diff:model-catalog": "node scripts/diff-model-catalog.mjs", "check:model-catalog": "node scripts/publish-model-catalog.mjs --input .artifacts/model-catalog --dry-run", "profile:tui": "node scripts/profile-coding-agent-node.mjs --mode tui", "profile:rpc": "node scripts/profile-coding-agent-node.mjs --mode rpc", diff --git a/packages/ai/CHANGELOG.md b/packages/ai/CHANGELOG.md index 46ce4ed6..694c828b 100644 --- a/packages/ai/CHANGELOG.md +++ b/packages/ai/CHANGELOG.md @@ -4,6 +4,7 @@ ### Fixed +- Fixed GitHub Copilot long-context pricing tiers in generated model metadata ([#6668](https://github.com/earendil-works/pi/issues/6668)). - Fixed Kimi Coding subscription models to report API-equivalent implied costs when models.dev reports zero pricing. - Fixed OpenAI Responses early stream endings to be classified as retryable provider errors ([#6727](https://github.com/earendil-works/pi/issues/6727)). diff --git a/packages/ai/scripts/generate-models.ts b/packages/ai/scripts/generate-models.ts index 36e8fcfd..53d5a00a 100644 --- a/packages/ai/scripts/generate-models.ts +++ b/packages/ai/scripts/generate-models.ts @@ -14,6 +14,7 @@ import type { Api, KnownProvider, Model, + ModelCost, OpenAICompletionsCompat, OpenAIResponsesCompat, } from "../src/types.ts"; @@ -26,10 +27,12 @@ function readGeneratorOptions(args: string[]): { strict: boolean; jsonOnly: boolean; jsonOutputDir: string | undefined; + pretty: boolean; } { let strict = false; let jsonOnly = false; let jsonOutputDir: string | undefined; + let pretty = false; for (let index = 0; index < args.length; index++) { const arg = args[index]; @@ -41,6 +44,10 @@ function readGeneratorOptions(args: string[]): { jsonOnly = true; continue; } + if (arg === "--pretty") { + pretty = true; + continue; + } if (arg === "--json-output") { const value = args[++index]; if (!value) throw new Error("--json-output requires a directory"); @@ -51,7 +58,7 @@ function readGeneratorOptions(args: string[]): { } if (jsonOnly && !jsonOutputDir) throw new Error("--json-only requires --json-output"); - return { strict, jsonOnly, jsonOutputDir }; + return { strict, jsonOnly, jsonOutputDir, pretty }; } const generatorOptions = readGeneratorOptions(process.argv.slice(2)); @@ -70,6 +77,16 @@ interface ModelsDevModel { output?: number; cache_read?: number; cache_write?: number; + tiers?: { + input?: number; + output?: number; + cache_read?: number; + cache_write?: number; + tier?: { + type?: string; + size?: number; + }; + }[]; }; modalities?: { input?: string[]; @@ -753,6 +770,30 @@ function roundCost(value: number): number { return Number(value.toFixed(6)); } +function getModelsDevCost(cost: ModelsDevModel["cost"]): ModelCost { + const tiers = cost?.tiers?.flatMap((tier) => { + const context = tier.tier; + if (context?.type !== "context" || context.size === undefined) return []; + return [ + { + inputTokensAbove: context.size, + input: tier.input || 0, + output: tier.output || 0, + cacheRead: tier.cache_read || 0, + cacheWrite: tier.cache_write || 0, + }, + ]; + }); + + return { + input: cost?.input || 0, + output: cost?.output || 0, + cacheRead: cost?.cache_read || 0, + cacheWrite: cost?.cache_write || 0, + ...(tiers && tiers.length > 0 ? { tiers } : {}), + }; +} + async function fetchNvidiaNimModelIds(): Promise> { try { console.log("Fetching models from NVIDIA NIM API..."); @@ -1575,12 +1616,7 @@ async function loadModelsDevData(): Promise[]> { baseUrl: "https://api.individual.githubcopilot.com", reasoning: m.reasoning === true, input: m.modalities?.input?.includes("image") ? ["text", "image"] : ["text"], - cost: { - input: m.cost?.input || 0, - output: m.cost?.output || 0, - cacheRead: m.cost?.cache_read || 0, - cacheWrite: m.cost?.cache_write || 0, - }, + cost: getModelsDevCost(m.cost), contextWindow: m.limit?.context || 128000, maxTokens: m.limit?.output || 8192, headers: { ...COPILOT_STATIC_HEADERS }, @@ -2294,7 +2330,8 @@ async function generateModels() { jsonProviders[providerId][modelId] = providers[providerId][modelId]; } } - const writeJson = (path: string, value: unknown) => writeFileSync(path, `${JSON.stringify(value)}\n`); + const writeJson = (path: string, value: unknown) => + writeFileSync(path, `${JSON.stringify(value, null, generatorOptions.pretty ? 2 : undefined)}\n`); if (!generatorOptions.jsonOnly) { // Generate TypeScript structural catalogs and adjacent JSON values. diff --git a/scripts/diff-model-catalog.mjs b/scripts/diff-model-catalog.mjs new file mode 100644 index 00000000..8421daf5 --- /dev/null +++ b/scripts/diff-model-catalog.mjs @@ -0,0 +1,175 @@ +#!/usr/bin/env node + +import { existsSync, mkdtempSync, readFileSync, readdirSync, rmSync, symlinkSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { spawnSync } from "node:child_process"; + +function printUsage() { + console.log(`Usage: node scripts/diff-model-catalog.mjs [provider ...] + +Generates the model catalog at HEAD and in the current worktree, then shows +JSON differences. If providers are omitted, all providers are compared. + +Examples: + node scripts/diff-model-catalog.mjs github-copilot + npm run diff:model-catalog -- github-copilot +`); +} + +function run(command, args, options = {}) { + const result = spawnSync(command, args, { + cwd: options.cwd, + encoding: "utf8", + stdio: options.capture ? ["ignore", "pipe", "pipe"] : "inherit", + }); + if (result.status !== 0) { + const details = [result.stdout, result.stderr].filter(Boolean).join("\n"); + throw new Error(`Command failed: ${[command, ...args].join(" ")}\n${details}`); + } + return result.stdout ?? ""; +} + +function runDiff(args, cwd) { + return spawnSync("git", args, { + cwd, + encoding: "utf8", + stdio: ["ignore", "pipe", "pipe"], + }); +} + +const args = process.argv.slice(2); +if (args.includes("--help")) { + printUsage(); + process.exit(0); +} +if (args.some((arg) => arg.startsWith("-"))) { + printUsage(); + process.exit(1); +} + +const repoRoot = run("git", ["rev-parse", "--show-toplevel"], { capture: true }).trim(); +const temporaryRoot = mkdtempSync(join(tmpdir(), "pi-model-catalog-diff-")); +const baselineWorktree = join(temporaryRoot, "baseline-worktree"); +const baselineOutput = join(temporaryRoot, "before"); +const currentOutput = join(temporaryRoot, "after"); +let worktreeAdded = false; + +function generateCatalog(cwd, outputDir, pretty = false) { + const args = ["packages/ai/scripts/generate-models.ts", "--strict", "--json-only", "--json-output", outputDir]; + if (pretty) args.push("--pretty"); + run(process.execPath, args, { cwd, capture: true }); +} + +function formatProviderCatalogs(outputDir) { + const providersDir = join(outputDir, "providers"); + for (const entry of readdirSync(providersDir)) { + if (!entry.endsWith(".json")) continue; + const path = join(providersDir, entry); + const value = JSON.parse(readFileSync(path, "utf8")); + writeFileSync(path, `${JSON.stringify(value, null, 2)}\n`); + } +} + +function readProviderCatalog(outputDir, provider) { + const path = join(outputDir, "providers", `${provider}.json`); + return existsSync(path) ? JSON.parse(readFileSync(path, "utf8")) : undefined; +} + +function writeModelSnapshot(path, model) { + writeFileSync(path, model === undefined ? "" : `${JSON.stringify(model, null, 2)}\n`); +} + +function writeChangedLines(output) { + const changedLines = output.split("\n").filter((line) => { + const withoutColor = line.replace(/\u001b\[[0-9;]*m/g, ""); + return ( + (withoutColor.startsWith("+") && !withoutColor.startsWith("+++")) || + (withoutColor.startsWith("-") && !withoutColor.startsWith("---")) + ); + }); + if (changedLines.length > 0) process.stdout.write(`${changedLines.join("\n")}\n`); +} + +try { + run("git", ["worktree", "add", "--detach", baselineWorktree, "HEAD"], { cwd: repoRoot }); + worktreeAdded = true; + + const nodeModules = join(repoRoot, "node_modules"); + if (existsSync(nodeModules)) { + symlinkSync(nodeModules, join(baselineWorktree, "node_modules"), process.platform === "win32" ? "junction" : "dir"); + } + + console.log("Generating catalog from HEAD..."); + generateCatalog(baselineWorktree, baselineOutput); + formatProviderCatalogs(baselineOutput); + console.log("Generating catalog from the current worktree..."); + generateCatalog(repoRoot, currentOutput, true); + formatProviderCatalogs(currentOutput); + + const beforeProviders = JSON.parse(readFileSync(join(baselineOutput, "providers.json"), "utf8")); + const afterProviders = JSON.parse(readFileSync(join(currentOutput, "providers.json"), "utf8")); + const providers = args.length > 0 ? args : [...new Set([...beforeProviders, ...afterProviders])].sort(); + const beforeModelPath = "before-model.json"; + const afterModelPath = "after-model.json"; + const changedModels = []; + let differences = 0; + + for (const provider of providers) { + const beforeModels = readProviderCatalog(baselineOutput, provider); + const afterModels = readProviderCatalog(currentOutput, provider); + if (beforeModels === undefined && afterModels === undefined) { + throw new Error(`Unknown provider: ${provider}`); + } + + const modelIds = [...new Set([...Object.keys(beforeModels ?? {}), ...Object.keys(afterModels ?? {})])].sort(); + for (const modelId of modelIds) { + const beforeModel = beforeModels?.[modelId]; + const afterModel = afterModels?.[modelId]; + if (JSON.stringify(beforeModel) === JSON.stringify(afterModel)) continue; + + writeModelSnapshot(join(temporaryRoot, beforeModelPath), beforeModel); + writeModelSnapshot(join(temporaryRoot, afterModelPath), afterModel); + const result = runDiff( + [ + "diff", + "--no-index", + "--no-ext-diff", + "--color=always", + "--unified=0", + "--", + beforeModelPath, + afterModelPath, + ], + temporaryRoot, + ); + if (result.status === 1) { + const changedModel = `${provider}/${modelId}`; + console.log(`\n${changedModel}`); + writeChangedLines(result.stdout); + changedModels.push(changedModel); + differences++; + } else if (result.status !== 0) { + throw new Error(`Could not compare ${provider}/${modelId}: ${result.stderr || result.stdout}`); + } + } + } + + if (differences === 0) { + console.log(`No model catalog changes${args.length === 1 ? ` for ${args[0]}` : ""}.`); + } else { + console.log(`\n${differences} model catalog entr${differences === 1 ? "y" : "ies"} changed.`); + for (const changedModel of changedModels) { + console.log(`- ${changedModel}`); + } + } +} finally { + if (worktreeAdded) { + try { + run("git", ["worktree", "remove", "--force", baselineWorktree], { cwd: repoRoot }); + } catch (error) { + console.error(error instanceof Error ? error.message : String(error)); + } + } + rmSync(temporaryRoot, { recursive: true, force: true }); +}