generate-models: use reasoning options from models.dev (#6928)

* generate-models: use reasoning options from models.dev

* fix tests

packages/ai/scripts/generate-models.ts

@@ -3,6 +3,7 @@
 import { existsSync, mkdirSync, mkdtempSync, readFileSync, readdirSync, renameSync, rmSync, writeFileSync } from "fs";
 import { dirname, join, resolve } from "path";
 import { fileURLToPath } from "url";
+import { getEffortThinkingLevelMap, type ModelsDevReasoningOption } from "./models-dev-reasoning-options.ts";
 import {
 	CLOUDFLARE_AI_GATEWAY_ANTHROPIC_BASE_URL,
 	CLOUDFLARE_AI_GATEWAY_COMPAT_BASE_URL,
@@ -83,6 +84,7 @@ interface ModelsDevModel {
 	name: string;
 	tool_call?: boolean;
 	reasoning?: boolean;
+	reasoning_options?: ModelsDevReasoningOption[];
 	limit?: {
 		context?: number;
 		output?: number;
@@ -112,6 +114,12 @@ interface ModelsDevModel {
 	};
 }
 
+interface ModelsDevProvider {
+	models?: Record<string, ModelsDevModel>;
+}
+
+type ModelsDevCatalog = Record<string, ModelsDevProvider>;
+
 interface NvidiaNimModelListItem {
 	id: string;
 }
@@ -257,15 +265,6 @@ const DEEPSEEK_V4_THINKING_LEVEL_MAP = {
 	max: "max",
 } as const;
 
-const KIMI_K3_THINKING_LEVEL_MAP = {
-	off: null,
-	minimal: null,
-	low: "low",
-	medium: null,
-	high: "high",
-	xhigh: null,
-	max: "max",
-} as const;
 const KIMI_K3_MAX_TOKENS = 131072;
 const KIMI_K3_COST = {
 	input: 3,
@@ -398,6 +397,43 @@ function mergeThinkingLevelMap(model: Model<any>, map: NonNullable<Model<any>["t
 	model.thinkingLevelMap = { ...model.thinkingLevelMap, ...map };
 }
 
+const modelsDevReasoningOptions = new Map<string, ModelsDevReasoningOption[]>();
+
+function getModelKey(model: Pick<Model<Api>, "provider" | "id">): string {
+	return `${model.provider}:${model.id}`;
+}
+
+function recordModelsDevReasoningOptions(provider: string, id: string, sourceModel: ModelsDevModel): void {
+	if (sourceModel.reasoning_options !== undefined) {
+		modelsDevReasoningOptions.set(`${provider}:${id}`, sourceModel.reasoning_options);
+	}
+}
+
+function supportsDirectReasoningEffort(model: Model<Api>): boolean {
+	if (model.api === "anthropic-messages") return model.compat?.forceAdaptiveThinking === true;
+	if (
+		model.api === "openai-responses" ||
+		model.api === "azure-openai-responses" ||
+		model.api === "openai-codex-responses"
+	) {
+		return true;
+	}
+	if (model.api !== "openai-completions") return false;
+
+	const compat = {
+		...detectOpenAICompletionsCompat(model as Model<"openai-completions">),
+		...(model.compat as OpenAICompletionsCompat | undefined),
+	};
+	return compat.thinkingFormat === "openai" && compat.supportsReasoningEffort;
+}
+
+function applyModelsDevReasoningOptionMetadata(model: Model<Api>): void {
+	const reasoningOptions = modelsDevReasoningOptions.get(getModelKey(model));
+	if (!reasoningOptions || !supportsDirectReasoningEffort(model)) return;
+	const thinkingLevelMap = getEffortThinkingLevelMap(reasoningOptions);
+	if (thinkingLevelMap) mergeThinkingLevelMap(model, thinkingLevelMap);
+}
+
 function getTogetherCompat(modelId: string, reasoning: boolean): OpenAICompletionsCompat {
 	if (!reasoning) return TOGETHER_BASE_COMPAT;
 	if (TOGETHER_REASONING_EFFORT_MODELS.has(modelId)) return TOGETHER_REASONING_EFFORT_COMPAT;
@@ -957,7 +993,7 @@ async function loadModelsDevData(): Promise<Model<any>[]> {
 		console.log("Fetching models from models.dev API...");
 		const response = await fetch("https://models.dev/api.json");
 		if (!response.ok) throw new Error(`models.dev API returned ${response.status}`);
-		const data = await response.json();
+		const data = (await response.json()) as ModelsDevCatalog;
 
 		const models: Model<any>[] = [];
 		const nvidiaNimModelIds = data.nvidia?.models ? await fetchNvidiaNimModelIds() : new Map<string, string>();
@@ -997,6 +1033,7 @@ async function loadModelsDevData(): Promise<Model<any>[]> {
 					contextWindow: m.limit?.context || 4096,
 					maxTokens: m.limit?.output || 4096,
 				});
+				recordModelsDevReasoningOptions("amazon-bedrock" as const, id, m);
 			}
 		}
 
@@ -1023,6 +1060,7 @@ async function loadModelsDevData(): Promise<Model<any>[]> {
 					contextWindow: m.limit?.context || 4096,
 					maxTokens: m.limit?.output || 4096,
 				});
+				recordModelsDevReasoningOptions("anthropic", modelId, m);
 			}
 		}
 
@@ -1056,6 +1094,7 @@ async function loadModelsDevData(): Promise<Model<any>[]> {
 					contextWindow: source.limit?.context || 4096,
 					maxTokens: source.limit?.output || 4096,
 				});
+				recordModelsDevReasoningOptions("google", modelId, source);
 			}
 		}
 
@@ -1097,6 +1136,7 @@ async function loadModelsDevData(): Promise<Model<any>[]> {
 					contextWindow: source.limit?.context || 4096,
 					maxTokens: source.limit?.output || 4096,
 				});
+				recordModelsDevReasoningOptions("google-vertex", modelId, source);
 			}
 		}
 
@@ -1125,6 +1165,7 @@ async function loadModelsDevData(): Promise<Model<any>[]> {
 					contextWindow: m.limit?.context || 4096,
 					maxTokens: m.limit?.output || 4096,
 				});
+				recordModelsDevReasoningOptions("openai", modelId, m);
 			}
 		}
 
@@ -1151,6 +1192,7 @@ async function loadModelsDevData(): Promise<Model<any>[]> {
 					contextWindow: m.limit?.context || 4096,
 					maxTokens: m.limit?.output || 4096,
 				});
+				recordModelsDevReasoningOptions("groq", modelId, m);
 			}
 		}
 
@@ -1177,6 +1219,7 @@ async function loadModelsDevData(): Promise<Model<any>[]> {
 					contextWindow: m.limit?.context || 4096,
 					maxTokens: m.limit?.output || 4096,
 				});
+				recordModelsDevReasoningOptions("cerebras", modelId, m);
 			}
 		}
 
@@ -1204,6 +1247,7 @@ async function loadModelsDevData(): Promise<Model<any>[]> {
 					maxTokens: m.limit?.output || 4096,
 					compat: { sendSessionAffinityHeaders: true },
 				});
+				recordModelsDevReasoningOptions("cloudflare-workers-ai", modelId, m);
 			}
 		}
 
@@ -1260,6 +1304,7 @@ async function loadModelsDevData(): Promise<Model<any>[]> {
 					maxTokens: m.limit?.output || 4096,
 					...(compat ? { compat } : {}),
 				});
+				recordModelsDevReasoningOptions("cloudflare-ai-gateway", id, m);
 			}
 		}
 
@@ -1288,6 +1333,7 @@ async function loadModelsDevData(): Promise<Model<any>[]> {
 					contextWindow: m.limit?.context || 4096,
 					maxTokens: m.limit?.output || 4096,
 				});
+				recordModelsDevReasoningOptions("xai", modelId, m);
 			}
 		}
 
@@ -1330,6 +1376,7 @@ async function loadModelsDevData(): Promise<Model<any>[]> {
 						contextWindow: m.limit?.context || 4096,
 						maxTokens: m.limit?.output || 4096,
 					});
+					recordModelsDevReasoningOptions(provider, modelId, m);
 				}
 			}
 		}
@@ -1357,6 +1404,7 @@ async function loadModelsDevData(): Promise<Model<any>[]> {
 					contextWindow: m.limit?.context || 4096,
 					maxTokens: m.limit?.output || 4096,
 				});
+				recordModelsDevReasoningOptions("mistral", modelId, m);
 			}
 		}
 
@@ -1386,6 +1434,7 @@ async function loadModelsDevData(): Promise<Model<any>[]> {
 					contextWindow: m.limit?.context || 4096,
 					maxTokens: m.limit?.output || 4096,
 				});
+				recordModelsDevReasoningOptions("huggingface", modelId, m);
 			}
 		}
 
@@ -1423,6 +1472,7 @@ async function loadModelsDevData(): Promise<Model<any>[]> {
 						supportsLongCacheRetention: false,
 					},
 				});
+				recordModelsDevReasoningOptions("fireworks", modelId, m);
 			}
 		}
 
@@ -1457,6 +1507,7 @@ async function loadModelsDevData(): Promise<Model<any>[]> {
 					contextWindow: m.limit?.context || 4096,
 					maxTokens: m.limit?.output || 4096,
 				});
+				recordModelsDevReasoningOptions("nvidia", liveModelId, m);
 			}
 		}
 
@@ -1489,6 +1540,7 @@ async function loadModelsDevData(): Promise<Model<any>[]> {
 					contextWindow: m.limit?.context || 4096,
 					maxTokens: m.limit?.output || 4096,
 				});
+				recordModelsDevReasoningOptions("together", modelId, m);
 			}
 		}
 
@@ -1596,6 +1648,7 @@ async function loadModelsDevData(): Promise<Model<any>[]> {
 					contextWindow: m.limit?.context || 4096,
 					maxTokens: m.limit?.output || 4096,
 				});
+				recordModelsDevReasoningOptions(variant.provider, modelId, m);
 			}
 		}
 
@@ -1646,6 +1699,7 @@ async function loadModelsDevData(): Promise<Model<any>[]> {
 				};
 
 				models.push(copilotModel);
+				recordModelsDevReasoningOptions("github-copilot", modelId, m);
 			}
 		}
 
@@ -1679,6 +1733,7 @@ async function loadModelsDevData(): Promise<Model<any>[]> {
 						contextWindow: m.limit?.context || 4096,
 						maxTokens: m.limit?.output || 4096,
 					});
+					recordModelsDevReasoningOptions(provider, modelId, m);
 				}
 			}
 		}
@@ -1716,7 +1771,6 @@ async function loadModelsDevData(): Promise<Model<any>[]> {
 						forceAdaptiveThinking: true,
 					},
 					reasoning: isKimiK3 || m.reasoning === true,
-					...(isKimiK3 ? { thinkingLevelMap: KIMI_K3_THINKING_LEVEL_MAP } : {}),
 					input: m.modalities?.input?.includes("image") ? ["text", "image"] : ["text"],
 					cost: {
 						input: m.cost?.input || impliedCost?.input || 0,
@@ -1727,6 +1781,7 @@ async function loadModelsDevData(): Promise<Model<any>[]> {
 					contextWindow: m.limit?.context || 4096,
 					maxTokens: m.limit?.output || 4096,
 				});
+				recordModelsDevReasoningOptions("kimi-coding", normalizedId, m);
 			}
 		}
 
@@ -1771,7 +1826,6 @@ async function loadModelsDevData(): Promise<Model<any>[]> {
 					provider,
 					baseUrl,
 					reasoning: isKimiK3 || m.reasoning === true,
-					...(isKimiK3 ? { thinkingLevelMap: KIMI_K3_THINKING_LEVEL_MAP } : {}),
 					input: m.modalities?.input?.includes("image") ? ["text", "image"] : ["text"],
 					cost: {
 						input: m.cost?.input || (isKimiK3 ? KIMI_K3_COST.input : 0),
@@ -1783,6 +1837,7 @@ async function loadModelsDevData(): Promise<Model<any>[]> {
 					maxTokens: m.limit?.output || 4096,
 					compat,
 				});
+				recordModelsDevReasoningOptions(provider, modelId, m);
 			}
 		}
 
@@ -1839,6 +1894,7 @@ async function loadModelsDevData(): Promise<Model<any>[]> {
 					contextWindow: m.limit?.context || 4096,
 					maxTokens: m.limit?.output || 4096,
 				});
+				recordModelsDevReasoningOptions(provider, modelId, m);
 			}
 		}
 
@@ -1890,6 +1946,7 @@ async function loadModelsDevData(): Promise<Model<any>[]> {
 					contextWindow: m.limit?.context || 4096,
 					maxTokens: m.limit?.output || 4096,
 				});
+				recordModelsDevReasoningOptions(provider, modelId, m);
 			}
 		}
 
@@ -2394,8 +2451,9 @@ async function generateModels() {
 	allModels.push(...azureOpenAiModels);
 
 	for (const model of allModels) {
-		applyThinkingLevelMetadata(model);
 		applyOpenAICompletionsCompatMetadata(model);
+		applyModelsDevReasoningOptionMetadata(model);
+		applyThinkingLevelMetadata(model);
 		applyOpenAIToolSearchMetadata(model);
 	}
 

packages/ai/scripts/models-dev-reasoning-options.ts

@@ -0,0 +1,30 @@
+import type { ThinkingLevel, ThinkingLevelMap } from "../src/types.ts";
+
+export type ModelsDevReasoningOption =
+	| { type: "toggle" }
+	| {
+			type: "effort";
+			values: Array<"none" | "minimal" | "low" | "medium" | "high" | "xhigh" | "max" | "default" | null>;
+	  }
+	| { type: "budget_tokens"; min?: number; max?: number };
+
+const THINKING_LEVELS: readonly ThinkingLevel[] = ["minimal", "low", "medium", "high", "xhigh", "max"];
+
+/**
+ * Converts models.dev verified effort values into Pi's selectable thinking levels.
+ * Values without a Pi equivalent (`default` and JSON `null`) are intentionally
+ * omitted.
+ */
+export function getEffortThinkingLevelMap(options: readonly ModelsDevReasoningOption[]): ThinkingLevelMap | undefined {
+	const effortValues = options.flatMap((option) => (option.type === "effort" ? option.values : []));
+	if (effortValues.length === 0) return undefined;
+
+	const supported = new Set(effortValues);
+	if (!THINKING_LEVELS.some((level) => supported.has(level)) && !supported.has("none")) return undefined;
+
+	const map: ThinkingLevelMap = { off: supported.has("none") ? "none" : null };
+	for (const level of THINKING_LEVELS) {
+		map[level] = supported.has(level) ? level : null;
+	}
+	return map;
+}

packages/ai/test/reasoning-options.test.ts

@@ -0,0 +1,36 @@
+import { describe, expect, it } from "vitest";
+import { getEffortThinkingLevelMap } from "../scripts/models-dev-reasoning-options.ts";
+
+describe("getEffortThinkingLevelMap", () => {
+	it("exposes only verified effort values and none", () => {
+		expect(
+			getEffortThinkingLevelMap([{ type: "toggle" }, { type: "effort", values: ["none", "low", "high", "max"] }]),
+		).toEqual({
+			off: "none",
+			minimal: null,
+			low: "low",
+			medium: null,
+			high: "high",
+			xhigh: null,
+			max: "max",
+		});
+	});
+
+	it("does not infer thinking-off from an effort list", () => {
+		expect(getEffortThinkingLevelMap([{ type: "effort", values: ["low", "high", "max"] }])).toEqual({
+			off: null,
+			minimal: null,
+			low: "low",
+			medium: null,
+			high: "high",
+			xhigh: null,
+			max: "max",
+		});
+	});
+
+	it("leaves toggle and budget controls for their adapter-specific implementations", () => {
+		expect(getEffortThinkingLevelMap([{ type: "toggle" }])).toBeUndefined();
+		expect(getEffortThinkingLevelMap([{ type: "budget_tokens", min: 1024, max: 32000 }])).toBeUndefined();
+		expect(getEffortThinkingLevelMap([{ type: "effort", values: [null, "default"] }])).toBeUndefined();
+	});
+});

packages/ai/test/supports-xhigh.test.ts

@@ -59,15 +59,7 @@ describe("getSupportedThinkingLevels", () => {
 		(modelId) => {
 			const model = getModel("openai", modelId);
 			expect(model).toBeDefined();
-			expect(getSupportedThinkingLevels(model!)).toEqual([
+			expect(getSupportedThinkingLevels(model!)).toEqual(["off", "low", "medium", "high", "xhigh", "max"]);
-				"off",
-				"minimal",
-				"low",
-				"medium",
-				"high",
-				"xhigh",
-				"max",
-			]);
 		},
 	);
 
@@ -110,6 +102,12 @@ describe("getSupportedThinkingLevels", () => {
 		}
 	});
 
+	it.each(["moonshotai", "moonshotai-cn"] as const)("uses the verified effort options for %s Kimi K3", (provider) => {
+		const model = getModel(provider, "kimi-k3");
+		expect(model).toBeDefined();
+		expect(getSupportedThinkingLevels(model!)).toEqual(["low", "high", "max"]);
+	});
+
 	it("includes only low, high, max for Kimi Coding K3", () => {
 		const model = getModel("kimi-coding", "k3");
 		expect(model).toBeDefined();

packages/ai/test/together-models.test.ts

@@ -44,7 +44,15 @@ describe("Together models", () => {
 
 	it("models Together reasoning controls from the Together API surface", () => {
 		const gptOss = getModel("together", "openai/gpt-oss-120b");
-		expect(gptOss.thinkingLevelMap).toEqual({ off: null, minimal: null });
+		expect(gptOss.thinkingLevelMap).toEqual({
+			off: null,
+			minimal: null,
+			low: "low",
+			medium: "medium",
+			high: "high",
+			max: null,
+			xhigh: null,
+		});
 		expect(gptOss.compat).toMatchObject({
 			supportsReasoningEffort: true,
 			thinkingFormat: "openai",

scripts/diff-model-catalog.mjs

@@ -1,19 +1,22 @@
 #!/usr/bin/env node
 
-import { existsSync, mkdtempSync, readFileSync, readdirSync, rmSync, symlinkSync, writeFileSync } from "node:fs";
+import { copyFileSync, existsSync, mkdtempSync, readFileSync, readdirSync, rmSync, symlinkSync, writeFileSync } from "node:fs";
 import { tmpdir } from "node:os";
 import { join } from "node:path";
 import { spawnSync } from "node:child_process";
 
 function printUsage() {
-	console.log(`Usage: node scripts/diff-model-catalog.mjs [provider ...]
+	console.log(`Usage: node scripts/diff-model-catalog.mjs [--thinking] [provider ...]
 
 Generates the model catalog at HEAD and in the current worktree, then shows
 JSON differences. If providers are omitted, all providers are compared.
 
+--thinking compares each worktree's effective thinking levels using that
+worktree's getSupportedThinkingLevels() implementation.
+
 Examples:
   node scripts/diff-model-catalog.mjs github-copilot
-  npm run diff:model-catalog -- github-copilot
+  npm run diff:model-catalog -- --thinking moonshotai kimi-coding
 `);
 }
 
@@ -43,7 +46,9 @@ if (args.includes("--help")) {
 	printUsage();
 	process.exit(0);
 }
-if (args.some((arg) => arg.startsWith("-"))) {
+const thinkingOnly = args.includes("--thinking");
+const requestedProviders = args.filter((arg) => arg !== "--thinking");
+if (requestedProviders.some((arg) => arg.startsWith("-"))) {
 	printUsage();
 	process.exit(1);
 }
@@ -53,6 +58,8 @@ const temporaryRoot = mkdtempSync(join(tmpdir(), "pi-model-catalog-diff-"));
 const baselineWorktree = join(temporaryRoot, "baseline-worktree");
 const baselineOutput = join(temporaryRoot, "before");
 const currentOutput = join(temporaryRoot, "after");
+const baselineThinkingOutput = join(temporaryRoot, "before-thinking");
+const currentThinkingOutput = join(temporaryRoot, "after-thinking");
 let worktreeAdded = false;
 
 function generateCatalog(cwd, outputDir, pretty = false) {
@@ -76,8 +83,54 @@ function readProviderCatalog(outputDir, provider) {
 	return existsSync(path) ? JSON.parse(readFileSync(path, "utf8")) : undefined;
 }
 
+function generateThinkingCatalog(cwd, catalogPath, outputDir) {
+	run(process.execPath, ["scripts/generate-thinking-capabilities.mjs", catalogPath, outputDir], {
+		cwd,
+		capture: true,
+	});
+}
+
+const THINKING_LEVEL_ORDER = ["off", "minimal", "low", "medium", "high", "xhigh", "max"];
+const THINKING_LEVEL_RANKS = new Map(THINKING_LEVEL_ORDER.map((key, index) => [key, index]));
+
+function sortJsonKeys(keys, parentKey) {
+	if (parentKey !== "thinkingLevelMap" && parentKey !== "values") return keys.sort();
+	return keys.sort((left, right) => {
+		const leftRank = THINKING_LEVEL_RANKS.get(left) ?? Number.POSITIVE_INFINITY;
+		const rightRank = THINKING_LEVEL_RANKS.get(right) ?? Number.POSITIVE_INFINITY;
+		return leftRank - rightRank || left.localeCompare(right);
+	});
+}
+
+function canonicalizeJson(value, parentKey) {
+	if (Array.isArray(value)) return value.map((entry) => canonicalizeJson(entry));
+	if (value === null || typeof value !== "object") return value;
+
+	const result = {};
+	for (const key of sortJsonKeys(Object.keys(value), parentKey)) {
+		result[key] = canonicalizeJson(value[key], key);
+	}
+	return result;
+}
+
+function formatJsonForDiff(value, indent = "") {
+	if (Array.isArray(value)) {
+		if (value.length === 0) return "[]";
+		const childIndent = `${indent}  `;
+		return `[\n${value.map((entry) => `${childIndent}${formatJsonForDiff(entry, childIndent)},`).join("\n")}\n${indent}]`;
+	}
+	if (value === null || typeof value !== "object") return JSON.stringify(value);
+
+	const entries = Object.entries(value);
+	if (entries.length === 0) return "{}";
+	const childIndent = `${indent}  `;
+	return `{\n${entries
+		.map(([key, entry]) => `${childIndent}${JSON.stringify(key)}: ${formatJsonForDiff(entry, childIndent)},`)
+		.join("\n")}\n${indent}}`;
+}
+
 function writeModelSnapshot(path, model) {
-	writeFileSync(path, model === undefined ? "" : `${JSON.stringify(model, null, 2)}\n`);
+	writeFileSync(path, model === undefined ? "" : `${formatJsonForDiff(canonicalizeJson(model))}\n`);
 }
 
 function writeChangedLines(output) {
@@ -94,6 +147,10 @@ function writeChangedLines(output) {
 try {
 	run("git", ["worktree", "add", "--detach", baselineWorktree, "HEAD"], { cwd: repoRoot });
 	worktreeAdded = true;
+	copyFileSync(
+		join(repoRoot, "scripts", "generate-thinking-capabilities.mjs"),
+		join(baselineWorktree, "scripts", "generate-thinking-capabilities.mjs"),
+	);
 
 	const nodeModules = join(repoRoot, "node_modules");
 	if (existsSync(nodeModules)) {
@@ -107,17 +164,26 @@ try {
 	generateCatalog(repoRoot, currentOutput, true);
 	formatProviderCatalogs(currentOutput);
 
+	if (thinkingOnly) {
+		console.log("Computing effective thinking capabilities...");
+		generateThinkingCatalog(baselineWorktree, join(baselineOutput, "models.json"), baselineThinkingOutput);
+		generateThinkingCatalog(repoRoot, join(currentOutput, "models.json"), currentThinkingOutput);
+	}
+
 	const beforeProviders = JSON.parse(readFileSync(join(baselineOutput, "providers.json"), "utf8"));
 	const afterProviders = JSON.parse(readFileSync(join(currentOutput, "providers.json"), "utf8"));
-	const providers = args.length > 0 ? args : [...new Set([...beforeProviders, ...afterProviders])].sort();
+	const providers =
+		requestedProviders.length > 0 ? requestedProviders : [...new Set([...beforeProviders, ...afterProviders])].sort();
+	const beforeCatalogOutput = thinkingOnly ? baselineThinkingOutput : baselineOutput;
+	const currentCatalogOutput = thinkingOnly ? currentThinkingOutput : currentOutput;
 	const beforeModelPath = "before-model.json";
 	const afterModelPath = "after-model.json";
 	const changedModels = [];
 	let differences = 0;
 
 	for (const provider of providers) {
-		const beforeModels = readProviderCatalog(baselineOutput, provider);
+		const beforeModels = readProviderCatalog(beforeCatalogOutput, provider);
-		const afterModels = readProviderCatalog(currentOutput, provider);
+		const afterModels = readProviderCatalog(currentCatalogOutput, provider);
 		if (beforeModels === undefined && afterModels === undefined) {
 			throw new Error(`Unknown provider: ${provider}`);
 		}
@@ -126,7 +192,7 @@ try {
 		for (const modelId of modelIds) {
 			const beforeModel = beforeModels?.[modelId];
 			const afterModel = afterModels?.[modelId];
-			if (JSON.stringify(beforeModel) === JSON.stringify(afterModel)) continue;
+			if (JSON.stringify(canonicalizeJson(beforeModel)) === JSON.stringify(canonicalizeJson(afterModel))) continue;
 
 			writeModelSnapshot(join(temporaryRoot, beforeModelPath), beforeModel);
 			writeModelSnapshot(join(temporaryRoot, afterModelPath), afterModel);
@@ -156,7 +222,7 @@ try {
 	}
 
 	if (differences === 0) {
-		console.log(`No model catalog changes${args.length === 1 ? ` for ${args[0]}` : ""}.`);
+		console.log(`No model catalog changes${requestedProviders.length === 1 ? ` for ${requestedProviders[0]}` : ""}.`);
 	} else {
 		console.log(`\n${differences} model catalog entr${differences === 1 ? "y" : "ies"} changed.`);
 		for (const changedModel of changedModels) {

scripts/generate-thinking-capabilities.mjs

@@ -0,0 +1,30 @@
+#!/usr/bin/env node
+
+import { mkdirSync, readFileSync, writeFileSync } from "node:fs";
+import { join } from "node:path";
+import { getSupportedThinkingLevels } from "../packages/ai/src/models.ts";
+
+const [catalogPath, outputDir] = process.argv.slice(2);
+if (!catalogPath || !outputDir) {
+	throw new Error("Usage: node scripts/generate-thinking-capabilities.mjs <catalog-path> <output-dir>");
+}
+
+const catalog = JSON.parse(readFileSync(catalogPath, "utf8"));
+const providersDir = join(outputDir, "providers");
+mkdirSync(providersDir, { recursive: true });
+
+for (const [provider, models] of Object.entries(catalog)) {
+	const capabilities = Object.fromEntries(
+		Object.entries(models).map(([id, model]) => {
+			const levels = getSupportedThinkingLevels(model);
+			const values = Object.fromEntries(
+				levels.flatMap((level) => {
+					const value = model.thinkingLevelMap?.[level];
+					return value !== undefined && value !== level ? [[level, value]] : [];
+				}),
+			);
+			return [id, Object.keys(values).length > 0 ? { levels, values } : { levels }];
+		}),
+	);
+	writeFileSync(join(providersDir, `${provider}.json`), JSON.stringify(capabilities));
+}