ton/pi_harness
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(ai): add input-based pricing tiers

Browse Source
This commit is contained in:
Armin Ronacher
2026-07-09 22:43:19 +02:00
parent 6c735db060
commit a9ecf301fb
14 changed files with 241 additions and 60 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/ai/CHANGELOG.md
+2
View File
@@ -5,10 +5,12 @@
### Added
- Added a separate opt-in `max` thinking level, including native `xhigh` and `max` support for GPT-5.6 and Anthropic adaptive-thinking effort metadata matching Anthropic's documentation: `max` on all adaptive Claude models, native `xhigh` on Opus 4.7/4.8, Sonnet 5, and Fable 5 only.
- Added request-wide input-token pricing tiers to model cost metadata and usage cost calculation.
### Fixed
- Fixed post-compaction output-token budgeting to ignore stale assistant usage from before the compaction boundary ([#6464](https://github.com/earendil-works/pi/issues/6464)).
- Fixed GPT-5.6 metadata to keep direct OpenAI requests in the 272K short-context tier while exposing the Codex backend's 372K context window with long-context pricing.
## [0.80.5] - 2026-07-09
packages/ai/scripts/generate-models.ts
+54 -17
View File
@@ -194,6 +194,23 @@ const ANT_LING_RING_THINKING_LEVEL_MAP = {
} as const;
const MODELS_DEV_OPENAI_UNSUPPORTED_MODEL_IDS = new Set(["gpt-5.6"]);
const OPENAI_LONG_CONTEXT_INPUT_THRESHOLD = 272000;
const OPENAI_GPT_56_MODEL_IDS = new Set(["gpt-5.6-sol", "gpt-5.6-terra", "gpt-5.6-luna"]);
function withOpenAiLongContextPricing(cost: Model<Api>["cost"]): Model<Api>["cost"] {
return {
...cost,
tiers: [
{
inputTokensAbove: OPENAI_LONG_CONTEXT_INPUT_THRESHOLD,
input: cost.input * 2,
output: cost.output * 1.5,
cacheRead: cost.cacheRead * 2,
cacheWrite: cost.cacheWrite * 2,
},
],
};
}
const OPENAI_RESPONSES_NONE_REASONING_MODELS = new Set([
"gpt-5.1",
@@ -1678,9 +1695,16 @@ async function generateModels() {
candidate.contextWindow = 272000;
candidate.maxTokens = 128000;
}
if (candidate.provider === "openai" && (candidate.id === "gpt-5.4" || candidate.id === "gpt-5.5")) {
candidate.contextWindow = 272000;
// Keep direct OpenAI requests in the short-context pricing tier.
if (
candidate.provider === "openai" &&
(candidate.id === "gpt-5.4" || candidate.id === "gpt-5.5" || OPENAI_GPT_56_MODEL_IDS.has(candidate.id))
) {
candidate.contextWindow = OPENAI_LONG_CONTEXT_INPUT_THRESHOLD;
candidate.maxTokens = 128000;
if (OPENAI_GPT_56_MODEL_IDS.has(candidate.id)) {
candidate.cost = withOpenAiLongContextPricing(candidate.cost);
}
}
// models.dev reports gpt-5-pro output as 272000 (a duplicate of the input sub-limit);
// the actual max output is 128000. Also propagates to the derived Azure clone.
@@ -1724,8 +1748,8 @@ async function generateModels() {
provider: "openai",
reasoning: true,
input: ["text", "image"],
cost: { input: 5, output: 30, cacheRead: 0.5, cacheWrite: 0 },
contextWindow: 1050000,
cost: withOpenAiLongContextPricing({ input: 5, output: 30, cacheRead: 0.5, cacheWrite: 6.25 }),
contextWindow: OPENAI_LONG_CONTEXT_INPUT_THRESHOLD,
maxTokens: 128000,
},
{
@@ -1736,8 +1760,8 @@ async function generateModels() {
provider: "openai",
reasoning: true,
input: ["text", "image"],
cost: { input: 2.5, output: 15, cacheRead: 0.25, cacheWrite: 0 },
contextWindow: 1050000,
cost: withOpenAiLongContextPricing({ input: 2.5, output: 15, cacheRead: 0.25, cacheWrite: 3.125 }),
contextWindow: OPENAI_LONG_CONTEXT_INPUT_THRESHOLD,
maxTokens: 128000,
},
{
@@ -1748,8 +1772,8 @@ async function generateModels() {
provider: "openai",
reasoning: true,
input: ["text", "image"],
cost: { input: 1, output: 6, cacheRead: 0.1, cacheWrite: 0 },
contextWindow: 1050000,
cost: withOpenAiLongContextPricing({ input: 1, output: 6, cacheRead: 0.1, cacheWrite: 1.25 }),
contextWindow: OPENAI_LONG_CONTEXT_INPUT_THRESHOLD,
maxTokens: 128000,
},
{
@@ -1899,9 +1923,10 @@ async function generateModels() {
// OpenAI Codex (ChatGPT OAuth) models
// NOTE: These are not fetched from models.dev; we keep a small, explicit list to avoid aliases.
// Context window is based on observed server limits (400s above ~272k), not marketing numbers.
// Older model limits are based on observed server behavior; GPT-5.6 follows Codex's 372k catalog limit.
const CODEX_BASE_URL = "https://chatgpt.com/backend-api";
const CODEX_CONTEXT = 272000;
const CODEX_GPT_56_CONTEXT = 372000;
const CODEX_SPARK_CONTEXT = 128000;
const CODEX_MAX_TOKENS = 128000;
const codexModels: Model<"openai-codex-responses">[] = [
@@ -1961,8 +1986,8 @@ async function generateModels() {
baseUrl: CODEX_BASE_URL,
reasoning: true,
input: ["text", "image"],
cost: { input: 1, output: 6, cacheRead: 0.1, cacheWrite: 0 },
contextWindow: CODEX_CONTEXT,
cost: withOpenAiLongContextPricing({ input: 1, output: 6, cacheRead: 0.1, cacheWrite: 1.25 }),
contextWindow: CODEX_GPT_56_CONTEXT,
maxTokens: CODEX_MAX_TOKENS,
},
{
@@ -1973,8 +1998,8 @@ async function generateModels() {
baseUrl: CODEX_BASE_URL,
reasoning: true,
input: ["text", "image"],
cost: { input: 5, output: 30, cacheRead: 0.5, cacheWrite: 0 },
contextWindow: CODEX_CONTEXT,
cost: withOpenAiLongContextPricing({ input: 5, output: 30, cacheRead: 0.5, cacheWrite: 6.25 }),
contextWindow: CODEX_GPT_56_CONTEXT,
maxTokens: CODEX_MAX_TOKENS,
},
{
@@ -1985,8 +2010,8 @@ async function generateModels() {
baseUrl: CODEX_BASE_URL,
reasoning: true,
input: ["text", "image"],
cost: { input: 2.5, output: 15, cacheRead: 0.25, cacheWrite: 0 },
contextWindow: CODEX_CONTEXT,
cost: withOpenAiLongContextPricing({ input: 2.5, output: 15, cacheRead: 0.25, cacheWrite: 3.125 }),
contextWindow: CODEX_GPT_56_CONTEXT,
maxTokens: CODEX_MAX_TOKENS,
},
];
@@ -2112,11 +2137,14 @@ async function generateModels() {
});
}
// Azure Foundry deploys these with larger context windows than OpenAI's own API,
// which caps gpt-5.4/gpt-5.5 at 272k. See models-sold-directly-by-azure docs.
// Azure Foundry deploys these with larger context windows than OpenAI's own short-tier defaults.
// See models-sold-directly-by-azure docs.
const AZURE_CONTEXT_WINDOW_OVERRIDES: Record<string, number> = {
"gpt-5.4": 1050000,
"gpt-5.5": 1050000,
"gpt-5.6-luna": 1050000,
"gpt-5.6-sol": 1050000,
"gpt-5.6-terra": 1050000,
};
const azureOpenAiModels: Model<Api>[] = allModels
.filter((model) => model.provider === "openai" && model.api === "openai-responses")
@@ -2125,6 +2153,12 @@ async function generateModels() {
api: "azure-openai-responses",
provider: "azure-openai-responses",
baseUrl: "",
cost: {
input: model.cost.input,
output: model.cost.output,
cacheRead: model.cost.cacheRead,
cacheWrite: model.cost.cacheWrite,
},
contextWindow: AZURE_CONTEXT_WINDOW_OVERRIDES[model.id] ?? model.contextWindow,
}));
allModels.push(...azureOpenAiModels);
@@ -2179,6 +2213,9 @@ async function generateModels() {
output += `${indent}\t\toutput: ${model.cost.output},\n`;
output += `${indent}\t\tcacheRead: ${model.cost.cacheRead},\n`;
output += `${indent}\t\tcacheWrite: ${model.cost.cacheWrite},\n`;
if (model.cost.tiers) {
output += `${indent}\t\ttiers: ${JSON.stringify(model.cost.tiers)},\n`;
}
output += `${indent}\t},\n`;
output += `${indent}\tcontextWindow: ${model.contextWindow},\n`;
output += `${indent}\tmaxTokens: ${model.maxTokens},\n`;
packages/ai/src/api/openai-responses-shared.ts
+8 -4
View File
@@ -360,13 +360,17 @@ export async function processResponsesStream<TApi extends Api>(
output.responseId = response.id;
}
if (response?.usage) {
const cachedTokens = response.usage.input_tokens_details?.cached_tokens || 0;
const inputDetails = response.usage.input_tokens_details as
| { cached_tokens?: number; cache_write_tokens?: number }
| undefined;
const cachedTokens = inputDetails?.cached_tokens || 0;
const cacheWriteTokens = inputDetails?.cache_write_tokens || 0;
output.usage = {
// OpenAI includes cached tokens in input_tokens, so subtract to get non-cached input
input: (response.usage.input_tokens || 0) - cachedTokens,
// OpenAI includes cached and cache-write tokens in input_tokens, so subtract both.
input: Math.max(0, (response.usage.input_tokens || 0) - cachedTokens - cacheWriteTokens),
output: response.usage.output_tokens || 0,
cacheRead: cachedTokens,
cacheWrite: 0,
cacheWrite: cacheWriteTokens,
reasoning: response.usage.output_tokens_details?.reasoning_tokens || 0,
totalTokens: response.usage.total_tokens || 0,
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 },
packages/ai/src/models.ts
+15 -4
View File
@@ -10,6 +10,7 @@ import type {
AssistantMessageEventStream,
Context,
Model,
ModelCostRates,
ModelThinkingLevel,
ProviderHeaders,
ProviderStreams,
@@ -383,13 +384,23 @@ export function hasApi<TApi extends Api>(model: Model<Api>, api: TApi): model is
}
export function calculateCost<TApi extends Api>(model: Model<TApi>, usage: Usage): Usage["cost"] {
const inputTokens = usage.input + usage.cacheRead + usage.cacheWrite;
let rates: ModelCostRates = model.cost;
let matchedThreshold = -1;
for (const tier of model.cost.tiers ?? []) {
if (inputTokens > tier.inputTokensAbove && tier.inputTokensAbove > matchedThreshold) {
rates = tier;
matchedThreshold = tier.inputTokensAbove;
}
}
// Anthropic charges 2x base input for 1h cache writes.
const longWrite = usage.cacheWrite1h ?? 0;
const shortWrite = usage.cacheWrite - longWrite;
usage.cost.input = (model.cost.input / 1000000) * usage.input;
usage.cost.output = (model.cost.output / 1000000) * usage.output;
usage.cost.cacheRead = (model.cost.cacheRead / 1000000) * usage.cacheRead;
usage.cost.cacheWrite = (model.cost.cacheWrite * shortWrite + model.cost.input * 2 * longWrite) / 1000000;
usage.cost.input = (rates.input / 1000000) * usage.input;
usage.cost.output = (rates.output / 1000000) * usage.output;
usage.cost.cacheRead = (rates.cacheRead / 1000000) * usage.cacheRead;
usage.cost.cacheWrite = (rates.cacheWrite * shortWrite + rates.input * 2 * longWrite) / 1000000;
usage.cost.total = usage.cost.input + usage.cost.output + usage.cost.cacheRead + usage.cost.cacheWrite;
return usage.cost;
}
packages/ai/src/providers/openai-codex.models.ts
+9 -6
View File
@@ -89,9 +89,10 @@ export const OPENAI_CODEX_MODELS = {
input: 1,
output: 6,
cacheRead: 0.1,
cacheWrite: 0,
cacheWrite: 1.25,
tiers: [{"inputTokensAbove":272000,"input":2,"output":9,"cacheRead":0.2,"cacheWrite":2.5}],
},
contextWindow: 272000,
contextWindow: 372000,
maxTokens: 128000,
} satisfies Model<"openai-codex-responses">,
"gpt-5.6-sol": {
@@ -107,9 +108,10 @@ export const OPENAI_CODEX_MODELS = {
input: 5,
output: 30,
cacheRead: 0.5,
cacheWrite: 0,
cacheWrite: 6.25,
tiers: [{"inputTokensAbove":272000,"input":10,"output":45,"cacheRead":1,"cacheWrite":12.5}],
},
contextWindow: 272000,
contextWindow: 372000,
maxTokens: 128000,
} satisfies Model<"openai-codex-responses">,
"gpt-5.6-terra": {
@@ -125,9 +127,10 @@ export const OPENAI_CODEX_MODELS = {
input: 2.5,
output: 15,
cacheRead: 0.25,
cacheWrite: 0,
cacheWrite: 3.125,
tiers: [{"inputTokensAbove":272000,"input":5,"output":22.5,"cacheRead":0.5,"cacheWrite":6.25}],
},
contextWindow: 272000,
contextWindow: 372000,
maxTokens: 128000,
} satisfies Model<"openai-codex-responses">,
} as const;
packages/ai/src/providers/openai.models.ts
+6 -3
View File
@@ -620,8 +620,9 @@ export const OPENAI_MODELS = {
output: 6,
cacheRead: 0.1,
cacheWrite: 1.25,
tiers: [{"inputTokensAbove":272000,"input":2,"output":9,"cacheRead":0.2,"cacheWrite":2.5}],
},
contextWindow: 1050000,
contextWindow: 272000,
maxTokens: 128000,
} satisfies Model<"openai-responses">,
"gpt-5.6-sol": {
@@ -638,8 +639,9 @@ export const OPENAI_MODELS = {
output: 30,
cacheRead: 0.5,
cacheWrite: 6.25,
tiers: [{"inputTokensAbove":272000,"input":10,"output":45,"cacheRead":1,"cacheWrite":12.5}],
},
contextWindow: 1050000,
contextWindow: 272000,
maxTokens: 128000,
} satisfies Model<"openai-responses">,
"gpt-5.6-terra": {
@@ -656,8 +658,9 @@ export const OPENAI_MODELS = {
output: 15,
cacheRead: 0.25,
cacheWrite: 3.125,
tiers: [{"inputTokensAbove":272000,"input":5,"output":22.5,"cacheRead":0.5,"cacheWrite":6.25}],
},
contextWindow: 1050000,
contextWindow: 272000,
maxTokens: 128000,
} satisfies Model<"openai-responses">,
"o1": {
packages/ai/src/types.ts
+18 -6
View File
@@ -662,6 +662,23 @@ export interface VercelGatewayRouting {
order?: string[];
}
export interface ModelCostRates {
input: number; // $/million tokens
output: number; // $/million tokens
cacheRead: number; // $/million tokens
cacheWrite: number; // $/million tokens
}
export interface ModelCostTier extends ModelCostRates {
/** Use this tier for requests whose total input usage exceeds this token count. */
inputTokensAbove: number;
}
export interface ModelCost extends ModelCostRates {
/** Request-wide pricing tiers. The highest matching input threshold applies to the full request. */
tiers?: ModelCostTier[];
}
// Model interface for the unified model system
export interface Model<TApi extends Api> {
id: string;
@@ -676,12 +693,7 @@ export interface Model<TApi extends Api> {
*/
thinkingLevelMap?: ThinkingLevelMap;
input: ("text" | "image")[];
cost: {
input: number; // $/million tokens
output: number; // $/million tokens
cacheRead: number; // $/million tokens
cacheWrite: number; // $/million tokens
};
cost: ModelCost;
contextWindow: number;
maxTokens: number;
headers?: Record<string, string>;
packages/ai/test/models-runtime.test.ts
+38 -2
View File
@@ -1,8 +1,8 @@
import { describe, expect, it } from "vitest";
import { InMemoryCredentialStore } from "../src/auth/credential-store.ts";
import type { ApiKeyAuth, CredentialStore, OAuthAuth, ProviderAuth } from "../src/auth/types.ts";
import { createModels, hasApi, type Provider } from "../src/models.ts";
import type { Api, AssistantMessage, Context, Model, SimpleStreamOptions, StreamOptions } from "../src/types.ts";
import { calculateCost, createModels, hasApi, type Provider } from "../src/models.ts";
import type { Api, AssistantMessage, Context, Model, SimpleStreamOptions, StreamOptions, Usage } from "../src/types.ts";
import { AssistantMessageEventStream } from "../src/utils/event-stream.ts";
function testModel(provider: string, id: string): Model<Api> {
@@ -106,6 +106,42 @@ function testOAuth(overrides?: Partial<OAuthAuth>): OAuthAuth {
}
describe("Models runtime", () => {
it("applies request-wide pricing tiers above the configured input threshold", () => {
const model = testModel("openai", "gpt-5.6-sol");
model.cost = {
input: 5,
output: 30,
cacheRead: 0.5,
cacheWrite: 6.25,
tiers: [
{
inputTokensAbove: 272000,
input: 10,
output: 45,
cacheRead: 1,
cacheWrite: 12.5,
},
],
};
const createUsage = (cacheWrite: number): Usage => ({
input: 200000,
output: 100000,
cacheRead: 72000,
cacheWrite,
totalTokens: 372000 + cacheWrite,
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 },
});
const short = calculateCost(model, createUsage(0));
expect(short).toMatchObject({ input: 1, output: 3, cacheRead: 0.036, cacheWrite: 0 });
const long = calculateCost(model, createUsage(1));
expect(long.input).toBe(2);
expect(long.output).toBe(4.5);
expect(long.cacheRead).toBe(0.072);
expect(long.cacheWrite).toBe(0.0000125);
});
it("registers, replaces, and deletes providers", () => {
const models = createModels();
models.setProvider(testProvider({ id: "p1" }));
packages/ai/test/openai-responses-terminal-event.test.ts
+4 -4
View File
@@ -118,10 +118,10 @@ async function* createCompletedEvents(): AsyncIterable<ResponseStreamEvent> {
input_tokens: 20,
output_tokens: 7,
total_tokens: 27,
input_tokens_details: { cached_tokens: 2 },
input_tokens_details: { cached_tokens: 2, cache_write_tokens: 3 },
},
},
} as ResponseStreamEvent;
} as unknown as ResponseStreamEvent;
}
async function* createIncompleteEvents(): AsyncIterable<ResponseStreamEvent> {
@@ -195,10 +195,10 @@ describe("OpenAI Responses terminal event handling", () => {
expect(output.responseId).toBe("resp_completed");
expect(output.stopReason).toBe("stop");
expect(output.usage).toMatchObject({
input: 18,
input: 15,
output: 7,
cacheRead: 2,
cacheWrite: 0,
cacheWrite: 3,
totalTokens: 27,
});
});