ton/pi_harness
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

generate-models: use reasoning options from models.dev (#6928)

Browse Source 
* generate-models: use reasoning options from models.dev

* fix tests
This commit is contained in:
David Brailovsky
2026-07-22 08:33:34 +02:00
committed by GitHub
parent db647e421f
commit 1ae064099c
7 changed files with 259 additions and 33 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/diff-model-catalog.mjs
+76 -10
View File
@@ -1,19 +1,22 @@
#!/usr/bin/env node
import { existsSync, mkdtempSync, readFileSync, readdirSync, rmSync, symlinkSync, writeFileSync } from "node:fs";
import { copyFileSync, existsSync, mkdtempSync, readFileSync, readdirSync, rmSync, symlinkSync, writeFileSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { spawnSync } from "node:child_process";
function printUsage() {
console.log(`Usage: node scripts/diff-model-catalog.mjs [provider ...]
console.log(`Usage: node scripts/diff-model-catalog.mjs [--thinking] [provider ...]
Generates the model catalog at HEAD and in the current worktree, then shows
JSON differences. If providers are omitted, all providers are compared.
--thinking compares each worktree's effective thinking levels using that
worktree's getSupportedThinkingLevels() implementation.
Examples:
node scripts/diff-model-catalog.mjs github-copilot
npm run diff:model-catalog -- github-copilot
npm run diff:model-catalog -- --thinking moonshotai kimi-coding
`);
}
@@ -43,7 +46,9 @@ if (args.includes("--help")) {
printUsage();
process.exit(0);
}
if (args.some((arg) => arg.startsWith("-"))) {
const thinkingOnly = args.includes("--thinking");
const requestedProviders = args.filter((arg) => arg !== "--thinking");
if (requestedProviders.some((arg) => arg.startsWith("-"))) {
printUsage();
process.exit(1);
}
@@ -53,6 +58,8 @@ const temporaryRoot = mkdtempSync(join(tmpdir(), "pi-model-catalog-diff-"));
const baselineWorktree = join(temporaryRoot, "baseline-worktree");
const baselineOutput = join(temporaryRoot, "before");
const currentOutput = join(temporaryRoot, "after");
const baselineThinkingOutput = join(temporaryRoot, "before-thinking");
const currentThinkingOutput = join(temporaryRoot, "after-thinking");
let worktreeAdded = false;
function generateCatalog(cwd, outputDir, pretty = false) {
@@ -76,8 +83,54 @@ function readProviderCatalog(outputDir, provider) {
return existsSync(path) ? JSON.parse(readFileSync(path, "utf8")) : undefined;
}
function generateThinkingCatalog(cwd, catalogPath, outputDir) {
run(process.execPath, ["scripts/generate-thinking-capabilities.mjs", catalogPath, outputDir], {
cwd,
capture: true,
});
}
const THINKING_LEVEL_ORDER = ["off", "minimal", "low", "medium", "high", "xhigh", "max"];
const THINKING_LEVEL_RANKS = new Map(THINKING_LEVEL_ORDER.map((key, index) => [key, index]));
function sortJsonKeys(keys, parentKey) {
if (parentKey !== "thinkingLevelMap" && parentKey !== "values") return keys.sort();
return keys.sort((left, right) => {
const leftRank = THINKING_LEVEL_RANKS.get(left) ?? Number.POSITIVE_INFINITY;
const rightRank = THINKING_LEVEL_RANKS.get(right) ?? Number.POSITIVE_INFINITY;
return leftRank - rightRank || left.localeCompare(right);
});
}
function canonicalizeJson(value, parentKey) {
if (Array.isArray(value)) return value.map((entry) => canonicalizeJson(entry));
if (value === null || typeof value !== "object") return value;
const result = {};
for (const key of sortJsonKeys(Object.keys(value), parentKey)) {
result[key] = canonicalizeJson(value[key], key);
}
return result;
}
function formatJsonForDiff(value, indent = "") {
if (Array.isArray(value)) {
if (value.length === 0) return "[]";
const childIndent = `${indent} `;
return `[\n${value.map((entry) => `${childIndent}${formatJsonForDiff(entry, childIndent)},`).join("\n")}\n${indent}]`;
}
if (value === null || typeof value !== "object") return JSON.stringify(value);
const entries = Object.entries(value);
if (entries.length === 0) return "{}";
const childIndent = `${indent} `;
return `{\n${entries
.map(([key, entry]) => `${childIndent}${JSON.stringify(key)}: ${formatJsonForDiff(entry, childIndent)},`)
.join("\n")}\n${indent}}`;
}
function writeModelSnapshot(path, model) {
writeFileSync(path, model === undefined ? "" : `${JSON.stringify(model, null, 2)}\n`);
writeFileSync(path, model === undefined ? "" : `${formatJsonForDiff(canonicalizeJson(model))}\n`);
}
function writeChangedLines(output) {
@@ -94,6 +147,10 @@ function writeChangedLines(output) {
try {
run("git", ["worktree", "add", "--detach", baselineWorktree, "HEAD"], { cwd: repoRoot });
worktreeAdded = true;
copyFileSync(
join(repoRoot, "scripts", "generate-thinking-capabilities.mjs"),
join(baselineWorktree, "scripts", "generate-thinking-capabilities.mjs"),
);
const nodeModules = join(repoRoot, "node_modules");
if (existsSync(nodeModules)) {
@@ -107,17 +164,26 @@ try {
generateCatalog(repoRoot, currentOutput, true);
formatProviderCatalogs(currentOutput);
if (thinkingOnly) {
console.log("Computing effective thinking capabilities...");
generateThinkingCatalog(baselineWorktree, join(baselineOutput, "models.json"), baselineThinkingOutput);
generateThinkingCatalog(repoRoot, join(currentOutput, "models.json"), currentThinkingOutput);
}
const beforeProviders = JSON.parse(readFileSync(join(baselineOutput, "providers.json"), "utf8"));
const afterProviders = JSON.parse(readFileSync(join(currentOutput, "providers.json"), "utf8"));
const providers = args.length > 0 ? args : [...new Set([...beforeProviders, ...afterProviders])].sort();
const providers =
requestedProviders.length > 0 ? requestedProviders : [...new Set([...beforeProviders, ...afterProviders])].sort();
const beforeCatalogOutput = thinkingOnly ? baselineThinkingOutput : baselineOutput;
const currentCatalogOutput = thinkingOnly ? currentThinkingOutput : currentOutput;
const beforeModelPath = "before-model.json";
const afterModelPath = "after-model.json";
const changedModels = [];
let differences = 0;
for (const provider of providers) {
const beforeModels = readProviderCatalog(baselineOutput, provider);
const afterModels = readProviderCatalog(currentOutput, provider);
const beforeModels = readProviderCatalog(beforeCatalogOutput, provider);
const afterModels = readProviderCatalog(currentCatalogOutput, provider);
if (beforeModels === undefined && afterModels === undefined) {
throw new Error(`Unknown provider: ${provider}`);
}
@@ -126,7 +192,7 @@ try {
for (const modelId of modelIds) {
const beforeModel = beforeModels?.[modelId];
const afterModel = afterModels?.[modelId];
if (JSON.stringify(beforeModel) === JSON.stringify(afterModel)) continue;
if (JSON.stringify(canonicalizeJson(beforeModel)) === JSON.stringify(canonicalizeJson(afterModel))) continue;
writeModelSnapshot(join(temporaryRoot, beforeModelPath), beforeModel);
writeModelSnapshot(join(temporaryRoot, afterModelPath), afterModel);
@@ -156,7 +222,7 @@ try {
}
if (differences === 0) {
console.log(`No model catalog changes${args.length === 1 ? ` for ${args[0]}` : ""}.`);
console.log(`No model catalog changes${requestedProviders.length === 1 ? ` for ${requestedProviders[0]}` : ""}.`);
} else {
console.log(`\n${differences} model catalog entr${differences === 1 ? "y" : "ies"} changed.`);
for (const changedModel of changedModels) {
scripts/generate-thinking-capabilities.mjs
+30
View File
@@ -0,0 +1,30 @@
#!/usr/bin/env node
import { mkdirSync, readFileSync, writeFileSync } from "node:fs";
import { join } from "node:path";
import { getSupportedThinkingLevels } from "../packages/ai/src/models.ts";
const [catalogPath, outputDir] = process.argv.slice(2);
if (!catalogPath || !outputDir) {
throw new Error("Usage: node scripts/generate-thinking-capabilities.mjs <catalog-path> <output-dir>");
}
const catalog = JSON.parse(readFileSync(catalogPath, "utf8"));
const providersDir = join(outputDir, "providers");
mkdirSync(providersDir, { recursive: true });
for (const [provider, models] of Object.entries(catalog)) {
const capabilities = Object.fromEntries(
Object.entries(models).map(([id, model]) => {
const levels = getSupportedThinkingLevels(model);
const values = Object.fromEntries(
levels.flatMap((level) => {
const value = model.thinkingLevelMap?.[level];
return value !== undefined && value !== level ? [[level, value]] : [];
}),
);
return [id, Object.keys(values).length > 0 ? { levels, values } : { levels }];
}),
);
writeFileSync(join(providersDir, `${provider}.json`), JSON.stringify(capabilities));
}