From 1ae064099cee34302984c56ff637e63dad44a8c4 Mon Sep 17 00:00:00 2001 From: David Brailovsky Date: Wed, 22 Jul 2026 08:33:34 +0200 Subject: [PATCH] generate-models: use reasoning options from models.dev (#6928) * generate-models: use reasoning options from models.dev * fix tests --- packages/ai/scripts/generate-models.ts | 84 +++++++++++++++--- .../scripts/models-dev-reasoning-options.ts | 30 +++++++ packages/ai/test/reasoning-options.test.ts | 36 ++++++++ packages/ai/test/supports-xhigh.test.ts | 16 ++-- packages/ai/test/together-models.test.ts | 10 ++- scripts/diff-model-catalog.mjs | 86 ++++++++++++++++--- scripts/generate-thinking-capabilities.mjs | 30 +++++++ 7 files changed, 259 insertions(+), 33 deletions(-) create mode 100644 packages/ai/scripts/models-dev-reasoning-options.ts create mode 100644 packages/ai/test/reasoning-options.test.ts create mode 100644 scripts/generate-thinking-capabilities.mjs diff --git a/packages/ai/scripts/generate-models.ts b/packages/ai/scripts/generate-models.ts index 1892c11a..207ec1e0 100644 --- a/packages/ai/scripts/generate-models.ts +++ b/packages/ai/scripts/generate-models.ts @@ -3,6 +3,7 @@ import { existsSync, mkdirSync, mkdtempSync, readFileSync, readdirSync, renameSync, rmSync, writeFileSync } from "fs"; import { dirname, join, resolve } from "path"; import { fileURLToPath } from "url"; +import { getEffortThinkingLevelMap, type ModelsDevReasoningOption } from "./models-dev-reasoning-options.ts"; import { CLOUDFLARE_AI_GATEWAY_ANTHROPIC_BASE_URL, CLOUDFLARE_AI_GATEWAY_COMPAT_BASE_URL, @@ -83,6 +84,7 @@ interface ModelsDevModel { name: string; tool_call?: boolean; reasoning?: boolean; + reasoning_options?: ModelsDevReasoningOption[]; limit?: { context?: number; output?: number; @@ -112,6 +114,12 @@ interface ModelsDevModel { }; } +interface ModelsDevProvider { + models?: Record; +} + +type ModelsDevCatalog = Record; + interface NvidiaNimModelListItem { id: string; } @@ -257,15 +265,6 @@ const DEEPSEEK_V4_THINKING_LEVEL_MAP = { max: "max", } as const; -const KIMI_K3_THINKING_LEVEL_MAP = { - off: null, - minimal: null, - low: "low", - medium: null, - high: "high", - xhigh: null, - max: "max", -} as const; const KIMI_K3_MAX_TOKENS = 131072; const KIMI_K3_COST = { input: 3, @@ -398,6 +397,43 @@ function mergeThinkingLevelMap(model: Model, map: NonNullable["t model.thinkingLevelMap = { ...model.thinkingLevelMap, ...map }; } +const modelsDevReasoningOptions = new Map(); + +function getModelKey(model: Pick, "provider" | "id">): string { + return `${model.provider}:${model.id}`; +} + +function recordModelsDevReasoningOptions(provider: string, id: string, sourceModel: ModelsDevModel): void { + if (sourceModel.reasoning_options !== undefined) { + modelsDevReasoningOptions.set(`${provider}:${id}`, sourceModel.reasoning_options); + } +} + +function supportsDirectReasoningEffort(model: Model): boolean { + if (model.api === "anthropic-messages") return model.compat?.forceAdaptiveThinking === true; + if ( + model.api === "openai-responses" || + model.api === "azure-openai-responses" || + model.api === "openai-codex-responses" + ) { + return true; + } + if (model.api !== "openai-completions") return false; + + const compat = { + ...detectOpenAICompletionsCompat(model as Model<"openai-completions">), + ...(model.compat as OpenAICompletionsCompat | undefined), + }; + return compat.thinkingFormat === "openai" && compat.supportsReasoningEffort; +} + +function applyModelsDevReasoningOptionMetadata(model: Model): void { + const reasoningOptions = modelsDevReasoningOptions.get(getModelKey(model)); + if (!reasoningOptions || !supportsDirectReasoningEffort(model)) return; + const thinkingLevelMap = getEffortThinkingLevelMap(reasoningOptions); + if (thinkingLevelMap) mergeThinkingLevelMap(model, thinkingLevelMap); +} + function getTogetherCompat(modelId: string, reasoning: boolean): OpenAICompletionsCompat { if (!reasoning) return TOGETHER_BASE_COMPAT; if (TOGETHER_REASONING_EFFORT_MODELS.has(modelId)) return TOGETHER_REASONING_EFFORT_COMPAT; @@ -957,7 +993,7 @@ async function loadModelsDevData(): Promise[]> { console.log("Fetching models from models.dev API..."); const response = await fetch("https://models.dev/api.json"); if (!response.ok) throw new Error(`models.dev API returned ${response.status}`); - const data = await response.json(); + const data = (await response.json()) as ModelsDevCatalog; const models: Model[] = []; const nvidiaNimModelIds = data.nvidia?.models ? await fetchNvidiaNimModelIds() : new Map(); @@ -997,6 +1033,7 @@ async function loadModelsDevData(): Promise[]> { contextWindow: m.limit?.context || 4096, maxTokens: m.limit?.output || 4096, }); + recordModelsDevReasoningOptions("amazon-bedrock" as const, id, m); } } @@ -1023,6 +1060,7 @@ async function loadModelsDevData(): Promise[]> { contextWindow: m.limit?.context || 4096, maxTokens: m.limit?.output || 4096, }); + recordModelsDevReasoningOptions("anthropic", modelId, m); } } @@ -1056,6 +1094,7 @@ async function loadModelsDevData(): Promise[]> { contextWindow: source.limit?.context || 4096, maxTokens: source.limit?.output || 4096, }); + recordModelsDevReasoningOptions("google", modelId, source); } } @@ -1097,6 +1136,7 @@ async function loadModelsDevData(): Promise[]> { contextWindow: source.limit?.context || 4096, maxTokens: source.limit?.output || 4096, }); + recordModelsDevReasoningOptions("google-vertex", modelId, source); } } @@ -1125,6 +1165,7 @@ async function loadModelsDevData(): Promise[]> { contextWindow: m.limit?.context || 4096, maxTokens: m.limit?.output || 4096, }); + recordModelsDevReasoningOptions("openai", modelId, m); } } @@ -1151,6 +1192,7 @@ async function loadModelsDevData(): Promise[]> { contextWindow: m.limit?.context || 4096, maxTokens: m.limit?.output || 4096, }); + recordModelsDevReasoningOptions("groq", modelId, m); } } @@ -1177,6 +1219,7 @@ async function loadModelsDevData(): Promise[]> { contextWindow: m.limit?.context || 4096, maxTokens: m.limit?.output || 4096, }); + recordModelsDevReasoningOptions("cerebras", modelId, m); } } @@ -1204,6 +1247,7 @@ async function loadModelsDevData(): Promise[]> { maxTokens: m.limit?.output || 4096, compat: { sendSessionAffinityHeaders: true }, }); + recordModelsDevReasoningOptions("cloudflare-workers-ai", modelId, m); } } @@ -1260,6 +1304,7 @@ async function loadModelsDevData(): Promise[]> { maxTokens: m.limit?.output || 4096, ...(compat ? { compat } : {}), }); + recordModelsDevReasoningOptions("cloudflare-ai-gateway", id, m); } } @@ -1288,6 +1333,7 @@ async function loadModelsDevData(): Promise[]> { contextWindow: m.limit?.context || 4096, maxTokens: m.limit?.output || 4096, }); + recordModelsDevReasoningOptions("xai", modelId, m); } } @@ -1330,6 +1376,7 @@ async function loadModelsDevData(): Promise[]> { contextWindow: m.limit?.context || 4096, maxTokens: m.limit?.output || 4096, }); + recordModelsDevReasoningOptions(provider, modelId, m); } } } @@ -1357,6 +1404,7 @@ async function loadModelsDevData(): Promise[]> { contextWindow: m.limit?.context || 4096, maxTokens: m.limit?.output || 4096, }); + recordModelsDevReasoningOptions("mistral", modelId, m); } } @@ -1386,6 +1434,7 @@ async function loadModelsDevData(): Promise[]> { contextWindow: m.limit?.context || 4096, maxTokens: m.limit?.output || 4096, }); + recordModelsDevReasoningOptions("huggingface", modelId, m); } } @@ -1423,6 +1472,7 @@ async function loadModelsDevData(): Promise[]> { supportsLongCacheRetention: false, }, }); + recordModelsDevReasoningOptions("fireworks", modelId, m); } } @@ -1457,6 +1507,7 @@ async function loadModelsDevData(): Promise[]> { contextWindow: m.limit?.context || 4096, maxTokens: m.limit?.output || 4096, }); + recordModelsDevReasoningOptions("nvidia", liveModelId, m); } } @@ -1489,6 +1540,7 @@ async function loadModelsDevData(): Promise[]> { contextWindow: m.limit?.context || 4096, maxTokens: m.limit?.output || 4096, }); + recordModelsDevReasoningOptions("together", modelId, m); } } @@ -1596,6 +1648,7 @@ async function loadModelsDevData(): Promise[]> { contextWindow: m.limit?.context || 4096, maxTokens: m.limit?.output || 4096, }); + recordModelsDevReasoningOptions(variant.provider, modelId, m); } } @@ -1646,6 +1699,7 @@ async function loadModelsDevData(): Promise[]> { }; models.push(copilotModel); + recordModelsDevReasoningOptions("github-copilot", modelId, m); } } @@ -1679,6 +1733,7 @@ async function loadModelsDevData(): Promise[]> { contextWindow: m.limit?.context || 4096, maxTokens: m.limit?.output || 4096, }); + recordModelsDevReasoningOptions(provider, modelId, m); } } } @@ -1716,7 +1771,6 @@ async function loadModelsDevData(): Promise[]> { forceAdaptiveThinking: true, }, reasoning: isKimiK3 || m.reasoning === true, - ...(isKimiK3 ? { thinkingLevelMap: KIMI_K3_THINKING_LEVEL_MAP } : {}), input: m.modalities?.input?.includes("image") ? ["text", "image"] : ["text"], cost: { input: m.cost?.input || impliedCost?.input || 0, @@ -1727,6 +1781,7 @@ async function loadModelsDevData(): Promise[]> { contextWindow: m.limit?.context || 4096, maxTokens: m.limit?.output || 4096, }); + recordModelsDevReasoningOptions("kimi-coding", normalizedId, m); } } @@ -1771,7 +1826,6 @@ async function loadModelsDevData(): Promise[]> { provider, baseUrl, reasoning: isKimiK3 || m.reasoning === true, - ...(isKimiK3 ? { thinkingLevelMap: KIMI_K3_THINKING_LEVEL_MAP } : {}), input: m.modalities?.input?.includes("image") ? ["text", "image"] : ["text"], cost: { input: m.cost?.input || (isKimiK3 ? KIMI_K3_COST.input : 0), @@ -1783,6 +1837,7 @@ async function loadModelsDevData(): Promise[]> { maxTokens: m.limit?.output || 4096, compat, }); + recordModelsDevReasoningOptions(provider, modelId, m); } } @@ -1839,6 +1894,7 @@ async function loadModelsDevData(): Promise[]> { contextWindow: m.limit?.context || 4096, maxTokens: m.limit?.output || 4096, }); + recordModelsDevReasoningOptions(provider, modelId, m); } } @@ -1890,6 +1946,7 @@ async function loadModelsDevData(): Promise[]> { contextWindow: m.limit?.context || 4096, maxTokens: m.limit?.output || 4096, }); + recordModelsDevReasoningOptions(provider, modelId, m); } } @@ -2394,8 +2451,9 @@ async function generateModels() { allModels.push(...azureOpenAiModels); for (const model of allModels) { - applyThinkingLevelMetadata(model); applyOpenAICompletionsCompatMetadata(model); + applyModelsDevReasoningOptionMetadata(model); + applyThinkingLevelMetadata(model); applyOpenAIToolSearchMetadata(model); } diff --git a/packages/ai/scripts/models-dev-reasoning-options.ts b/packages/ai/scripts/models-dev-reasoning-options.ts new file mode 100644 index 00000000..9d071139 --- /dev/null +++ b/packages/ai/scripts/models-dev-reasoning-options.ts @@ -0,0 +1,30 @@ +import type { ThinkingLevel, ThinkingLevelMap } from "../src/types.ts"; + +export type ModelsDevReasoningOption = + | { type: "toggle" } + | { + type: "effort"; + values: Array<"none" | "minimal" | "low" | "medium" | "high" | "xhigh" | "max" | "default" | null>; + } + | { type: "budget_tokens"; min?: number; max?: number }; + +const THINKING_LEVELS: readonly ThinkingLevel[] = ["minimal", "low", "medium", "high", "xhigh", "max"]; + +/** + * Converts models.dev verified effort values into Pi's selectable thinking levels. + * Values without a Pi equivalent (`default` and JSON `null`) are intentionally + * omitted. + */ +export function getEffortThinkingLevelMap(options: readonly ModelsDevReasoningOption[]): ThinkingLevelMap | undefined { + const effortValues = options.flatMap((option) => (option.type === "effort" ? option.values : [])); + if (effortValues.length === 0) return undefined; + + const supported = new Set(effortValues); + if (!THINKING_LEVELS.some((level) => supported.has(level)) && !supported.has("none")) return undefined; + + const map: ThinkingLevelMap = { off: supported.has("none") ? "none" : null }; + for (const level of THINKING_LEVELS) { + map[level] = supported.has(level) ? level : null; + } + return map; +} diff --git a/packages/ai/test/reasoning-options.test.ts b/packages/ai/test/reasoning-options.test.ts new file mode 100644 index 00000000..fafa9fa5 --- /dev/null +++ b/packages/ai/test/reasoning-options.test.ts @@ -0,0 +1,36 @@ +import { describe, expect, it } from "vitest"; +import { getEffortThinkingLevelMap } from "../scripts/models-dev-reasoning-options.ts"; + +describe("getEffortThinkingLevelMap", () => { + it("exposes only verified effort values and none", () => { + expect( + getEffortThinkingLevelMap([{ type: "toggle" }, { type: "effort", values: ["none", "low", "high", "max"] }]), + ).toEqual({ + off: "none", + minimal: null, + low: "low", + medium: null, + high: "high", + xhigh: null, + max: "max", + }); + }); + + it("does not infer thinking-off from an effort list", () => { + expect(getEffortThinkingLevelMap([{ type: "effort", values: ["low", "high", "max"] }])).toEqual({ + off: null, + minimal: null, + low: "low", + medium: null, + high: "high", + xhigh: null, + max: "max", + }); + }); + + it("leaves toggle and budget controls for their adapter-specific implementations", () => { + expect(getEffortThinkingLevelMap([{ type: "toggle" }])).toBeUndefined(); + expect(getEffortThinkingLevelMap([{ type: "budget_tokens", min: 1024, max: 32000 }])).toBeUndefined(); + expect(getEffortThinkingLevelMap([{ type: "effort", values: [null, "default"] }])).toBeUndefined(); + }); +}); diff --git a/packages/ai/test/supports-xhigh.test.ts b/packages/ai/test/supports-xhigh.test.ts index b6988ba0..67279032 100644 --- a/packages/ai/test/supports-xhigh.test.ts +++ b/packages/ai/test/supports-xhigh.test.ts @@ -59,15 +59,7 @@ describe("getSupportedThinkingLevels", () => { (modelId) => { const model = getModel("openai", modelId); expect(model).toBeDefined(); - expect(getSupportedThinkingLevels(model!)).toEqual([ - "off", - "minimal", - "low", - "medium", - "high", - "xhigh", - "max", - ]); + expect(getSupportedThinkingLevels(model!)).toEqual(["off", "low", "medium", "high", "xhigh", "max"]); }, ); @@ -110,6 +102,12 @@ describe("getSupportedThinkingLevels", () => { } }); + it.each(["moonshotai", "moonshotai-cn"] as const)("uses the verified effort options for %s Kimi K3", (provider) => { + const model = getModel(provider, "kimi-k3"); + expect(model).toBeDefined(); + expect(getSupportedThinkingLevels(model!)).toEqual(["low", "high", "max"]); + }); + it("includes only low, high, max for Kimi Coding K3", () => { const model = getModel("kimi-coding", "k3"); expect(model).toBeDefined(); diff --git a/packages/ai/test/together-models.test.ts b/packages/ai/test/together-models.test.ts index 0d766d65..89c85525 100644 --- a/packages/ai/test/together-models.test.ts +++ b/packages/ai/test/together-models.test.ts @@ -44,7 +44,15 @@ describe("Together models", () => { it("models Together reasoning controls from the Together API surface", () => { const gptOss = getModel("together", "openai/gpt-oss-120b"); - expect(gptOss.thinkingLevelMap).toEqual({ off: null, minimal: null }); + expect(gptOss.thinkingLevelMap).toEqual({ + off: null, + minimal: null, + low: "low", + medium: "medium", + high: "high", + max: null, + xhigh: null, + }); expect(gptOss.compat).toMatchObject({ supportsReasoningEffort: true, thinkingFormat: "openai", diff --git a/scripts/diff-model-catalog.mjs b/scripts/diff-model-catalog.mjs index 8421daf5..e00563f7 100644 --- a/scripts/diff-model-catalog.mjs +++ b/scripts/diff-model-catalog.mjs @@ -1,19 +1,22 @@ #!/usr/bin/env node -import { existsSync, mkdtempSync, readFileSync, readdirSync, rmSync, symlinkSync, writeFileSync } from "node:fs"; +import { copyFileSync, existsSync, mkdtempSync, readFileSync, readdirSync, rmSync, symlinkSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { spawnSync } from "node:child_process"; function printUsage() { - console.log(`Usage: node scripts/diff-model-catalog.mjs [provider ...] + console.log(`Usage: node scripts/diff-model-catalog.mjs [--thinking] [provider ...] Generates the model catalog at HEAD and in the current worktree, then shows JSON differences. If providers are omitted, all providers are compared. +--thinking compares each worktree's effective thinking levels using that +worktree's getSupportedThinkingLevels() implementation. + Examples: node scripts/diff-model-catalog.mjs github-copilot - npm run diff:model-catalog -- github-copilot + npm run diff:model-catalog -- --thinking moonshotai kimi-coding `); } @@ -43,7 +46,9 @@ if (args.includes("--help")) { printUsage(); process.exit(0); } -if (args.some((arg) => arg.startsWith("-"))) { +const thinkingOnly = args.includes("--thinking"); +const requestedProviders = args.filter((arg) => arg !== "--thinking"); +if (requestedProviders.some((arg) => arg.startsWith("-"))) { printUsage(); process.exit(1); } @@ -53,6 +58,8 @@ const temporaryRoot = mkdtempSync(join(tmpdir(), "pi-model-catalog-diff-")); const baselineWorktree = join(temporaryRoot, "baseline-worktree"); const baselineOutput = join(temporaryRoot, "before"); const currentOutput = join(temporaryRoot, "after"); +const baselineThinkingOutput = join(temporaryRoot, "before-thinking"); +const currentThinkingOutput = join(temporaryRoot, "after-thinking"); let worktreeAdded = false; function generateCatalog(cwd, outputDir, pretty = false) { @@ -76,8 +83,54 @@ function readProviderCatalog(outputDir, provider) { return existsSync(path) ? JSON.parse(readFileSync(path, "utf8")) : undefined; } +function generateThinkingCatalog(cwd, catalogPath, outputDir) { + run(process.execPath, ["scripts/generate-thinking-capabilities.mjs", catalogPath, outputDir], { + cwd, + capture: true, + }); +} + +const THINKING_LEVEL_ORDER = ["off", "minimal", "low", "medium", "high", "xhigh", "max"]; +const THINKING_LEVEL_RANKS = new Map(THINKING_LEVEL_ORDER.map((key, index) => [key, index])); + +function sortJsonKeys(keys, parentKey) { + if (parentKey !== "thinkingLevelMap" && parentKey !== "values") return keys.sort(); + return keys.sort((left, right) => { + const leftRank = THINKING_LEVEL_RANKS.get(left) ?? Number.POSITIVE_INFINITY; + const rightRank = THINKING_LEVEL_RANKS.get(right) ?? Number.POSITIVE_INFINITY; + return leftRank - rightRank || left.localeCompare(right); + }); +} + +function canonicalizeJson(value, parentKey) { + if (Array.isArray(value)) return value.map((entry) => canonicalizeJson(entry)); + if (value === null || typeof value !== "object") return value; + + const result = {}; + for (const key of sortJsonKeys(Object.keys(value), parentKey)) { + result[key] = canonicalizeJson(value[key], key); + } + return result; +} + +function formatJsonForDiff(value, indent = "") { + if (Array.isArray(value)) { + if (value.length === 0) return "[]"; + const childIndent = `${indent} `; + return `[\n${value.map((entry) => `${childIndent}${formatJsonForDiff(entry, childIndent)},`).join("\n")}\n${indent}]`; + } + if (value === null || typeof value !== "object") return JSON.stringify(value); + + const entries = Object.entries(value); + if (entries.length === 0) return "{}"; + const childIndent = `${indent} `; + return `{\n${entries + .map(([key, entry]) => `${childIndent}${JSON.stringify(key)}: ${formatJsonForDiff(entry, childIndent)},`) + .join("\n")}\n${indent}}`; +} + function writeModelSnapshot(path, model) { - writeFileSync(path, model === undefined ? "" : `${JSON.stringify(model, null, 2)}\n`); + writeFileSync(path, model === undefined ? "" : `${formatJsonForDiff(canonicalizeJson(model))}\n`); } function writeChangedLines(output) { @@ -94,6 +147,10 @@ function writeChangedLines(output) { try { run("git", ["worktree", "add", "--detach", baselineWorktree, "HEAD"], { cwd: repoRoot }); worktreeAdded = true; + copyFileSync( + join(repoRoot, "scripts", "generate-thinking-capabilities.mjs"), + join(baselineWorktree, "scripts", "generate-thinking-capabilities.mjs"), + ); const nodeModules = join(repoRoot, "node_modules"); if (existsSync(nodeModules)) { @@ -107,17 +164,26 @@ try { generateCatalog(repoRoot, currentOutput, true); formatProviderCatalogs(currentOutput); + if (thinkingOnly) { + console.log("Computing effective thinking capabilities..."); + generateThinkingCatalog(baselineWorktree, join(baselineOutput, "models.json"), baselineThinkingOutput); + generateThinkingCatalog(repoRoot, join(currentOutput, "models.json"), currentThinkingOutput); + } + const beforeProviders = JSON.parse(readFileSync(join(baselineOutput, "providers.json"), "utf8")); const afterProviders = JSON.parse(readFileSync(join(currentOutput, "providers.json"), "utf8")); - const providers = args.length > 0 ? args : [...new Set([...beforeProviders, ...afterProviders])].sort(); + const providers = + requestedProviders.length > 0 ? requestedProviders : [...new Set([...beforeProviders, ...afterProviders])].sort(); + const beforeCatalogOutput = thinkingOnly ? baselineThinkingOutput : baselineOutput; + const currentCatalogOutput = thinkingOnly ? currentThinkingOutput : currentOutput; const beforeModelPath = "before-model.json"; const afterModelPath = "after-model.json"; const changedModels = []; let differences = 0; for (const provider of providers) { - const beforeModels = readProviderCatalog(baselineOutput, provider); - const afterModels = readProviderCatalog(currentOutput, provider); + const beforeModels = readProviderCatalog(beforeCatalogOutput, provider); + const afterModels = readProviderCatalog(currentCatalogOutput, provider); if (beforeModels === undefined && afterModels === undefined) { throw new Error(`Unknown provider: ${provider}`); } @@ -126,7 +192,7 @@ try { for (const modelId of modelIds) { const beforeModel = beforeModels?.[modelId]; const afterModel = afterModels?.[modelId]; - if (JSON.stringify(beforeModel) === JSON.stringify(afterModel)) continue; + if (JSON.stringify(canonicalizeJson(beforeModel)) === JSON.stringify(canonicalizeJson(afterModel))) continue; writeModelSnapshot(join(temporaryRoot, beforeModelPath), beforeModel); writeModelSnapshot(join(temporaryRoot, afterModelPath), afterModel); @@ -156,7 +222,7 @@ try { } if (differences === 0) { - console.log(`No model catalog changes${args.length === 1 ? ` for ${args[0]}` : ""}.`); + console.log(`No model catalog changes${requestedProviders.length === 1 ? ` for ${requestedProviders[0]}` : ""}.`); } else { console.log(`\n${differences} model catalog entr${differences === 1 ? "y" : "ies"} changed.`); for (const changedModel of changedModels) { diff --git a/scripts/generate-thinking-capabilities.mjs b/scripts/generate-thinking-capabilities.mjs new file mode 100644 index 00000000..fe7cd641 --- /dev/null +++ b/scripts/generate-thinking-capabilities.mjs @@ -0,0 +1,30 @@ +#!/usr/bin/env node + +import { mkdirSync, readFileSync, writeFileSync } from "node:fs"; +import { join } from "node:path"; +import { getSupportedThinkingLevels } from "../packages/ai/src/models.ts"; + +const [catalogPath, outputDir] = process.argv.slice(2); +if (!catalogPath || !outputDir) { + throw new Error("Usage: node scripts/generate-thinking-capabilities.mjs "); +} + +const catalog = JSON.parse(readFileSync(catalogPath, "utf8")); +const providersDir = join(outputDir, "providers"); +mkdirSync(providersDir, { recursive: true }); + +for (const [provider, models] of Object.entries(catalog)) { + const capabilities = Object.fromEntries( + Object.entries(models).map(([id, model]) => { + const levels = getSupportedThinkingLevels(model); + const values = Object.fromEntries( + levels.flatMap((level) => { + const value = model.thinkingLevelMap?.[level]; + return value !== undefined && value !== level ? [[level, value]] : []; + }), + ); + return [id, Object.keys(values).length > 0 ? { levels, values } : { levels }]; + }), + ); + writeFileSync(join(providersDir, `${provider}.json`), JSON.stringify(capabilities)); +}