diff --git a/packages/ai/CHANGELOG.md b/packages/ai/CHANGELOG.md index 646e20af..12e876c9 100644 --- a/packages/ai/CHANGELOG.md +++ b/packages/ai/CHANGELOG.md @@ -4,11 +4,14 @@ ### Added +- Added shared token estimation helpers for `Context` and `Message[]` in `pi-ai`. +- Added `maxTokensSharesContextWindow` to `AnthropicMessagesCompat` for Anthropic-compatible providers that count `input + max_tokens` against a single shared context budget. - Added an optional `reasoning` field to `Usage` reporting reasoning/thinking token counts as a subset of `output`. Populated for Anthropic (`output_tokens_details.thinking_tokens`), OpenAI Responses/Codex/Azure (`output_tokens_details.reasoning_tokens`), OpenAI Completions (`completion_tokens_details.reasoning_tokens`), and Google Generative AI / Vertex (`thoughtsTokenCount`). Bedrock Converse and Mistral are not populated because those APIs do not return a reasoning token breakdown ([#6057](https://github.com/earendil-works/pi/issues/6057)). ### Fixed - Fixed OpenAI Responses streams to preserve reasoning replay state when output items finish out of order ([#6009](https://github.com/earendil-works/pi/issues/6009)). +- Fixed MiniMax (`minimax`, `minimax-cn`) Anthropic-compatible requests failing on long conversations with `unknown error, 999` or `context window exceeds limit (2013)` by clamping `max_tokens` for shared-budget models ([#6061](https://github.com/earendil-works/pi/issues/6061)). - Fixed retry classification for provider errors that explicitly tell callers to retry the request ([#6019](https://github.com/earendil-works/pi/issues/6019)). ## [0.80.2] - 2026-06-23 diff --git a/packages/ai/scripts/generate-models.ts b/packages/ai/scripts/generate-models.ts index 3b0677b6..8bfb285e 100644 --- a/packages/ai/scripts/generate-models.ts +++ b/packages/ai/scripts/generate-models.ts @@ -1424,6 +1424,7 @@ async function loadModelsDevData(): Promise[]> { provider, // MiniMax's Anthropic-compatible API - SDK appends /v1/messages baseUrl, + compat: { maxTokensSharesContextWindow: true }, reasoning: m.reasoning === true, input: m.modalities?.input?.includes("image") ? ["text", "image"] : ["text"], cost: { diff --git a/packages/ai/src/api/anthropic-messages.ts b/packages/ai/src/api/anthropic-messages.ts index 39fe4eea..222ab641 100644 --- a/packages/ai/src/api/anthropic-messages.ts +++ b/packages/ai/src/api/anthropic-messages.ts @@ -29,6 +29,7 @@ import type { ToolCall, ToolResultMessage, } from "../types.ts"; +import { estimateContextTokens } from "../utils/estimate.ts"; import { AssistantMessageEventStream } from "../utils/event-stream.ts"; import { headersToRecord } from "../utils/headers.ts"; import { parseJsonWithRepair, parseStreamingJson } from "../utils/json-parse.ts"; @@ -177,9 +178,12 @@ function getAnthropicCompat( supportsCacheControlOnTools: model.compat?.supportsCacheControlOnTools ?? true, supportsTemperature: model.compat?.supportsTemperature ?? true, allowEmptySignature: model.compat?.allowEmptySignature ?? false, + maxTokensSharesContextWindow: model.compat?.maxTokensSharesContextWindow ?? false, }; } +const SHARED_BUDGET_MIN_OUTPUT_TOKENS = 1024; + export interface AnthropicOptions extends StreamOptions { /** * Enable extended thinking. @@ -998,6 +1002,18 @@ function buildParams( } } + if (compat.maxTokensSharesContextWindow && model.contextWindow > 0) { + const estimatedInput = estimateContextTokens(context).tokens; + const available = Math.max(SHARED_BUDGET_MIN_OUTPUT_TOKENS, model.contextWindow - estimatedInput); + const clamped = Math.min(params.max_tokens, available); + if (clamped < params.max_tokens) { + params.max_tokens = clamped; + if (params.thinking?.type === "enabled" && params.thinking.budget_tokens >= clamped) { + params.thinking.budget_tokens = Math.max(0, clamped - SHARED_BUDGET_MIN_OUTPUT_TOKENS); + } + } + } + return params; } diff --git a/packages/ai/src/index.ts b/packages/ai/src/index.ts index 57c448cc..375c8f6a 100644 --- a/packages/ai/src/index.ts +++ b/packages/ai/src/index.ts @@ -27,6 +27,7 @@ export * from "./providers/faux.ts"; export * from "./session-resources.ts"; export * from "./types.ts"; export * from "./utils/diagnostics.ts"; +export * from "./utils/estimate.ts"; export * from "./utils/event-stream.ts"; export * from "./utils/json-parse.ts"; export type { diff --git a/packages/ai/src/types.ts b/packages/ai/src/types.ts index e4f263f4..0af069a7 100644 --- a/packages/ai/src/types.ts +++ b/packages/ai/src/types.ts @@ -573,6 +573,14 @@ export interface AnthropicMessagesCompat { forceAdaptiveThinking?: boolean; /** Whether to replay empty thinking signatures as `signature: ""` instead of converting thinking to text. Default: false. */ allowEmptySignature?: boolean; + /** + * Whether the provider counts `input + max_tokens` against a single shared + * budget (the model's `contextWindow`) instead of Anthropic's independent + * input and output limits. When true, Anthropic Messages requests clamp + * `max_tokens` to leave room for the estimated context tokens. + * Default: false. + */ + maxTokensSharesContextWindow?: boolean; } /** diff --git a/packages/ai/src/utils/estimate.ts b/packages/ai/src/utils/estimate.ts new file mode 100644 index 00000000..0f660c86 --- /dev/null +++ b/packages/ai/src/utils/estimate.ts @@ -0,0 +1,127 @@ +import type { AssistantMessage, Context, ImageContent, Message, TextContent, Usage } from "../types.ts"; + +export interface ContextUsageEstimate { + /** Estimated total context tokens. */ + tokens: number; + /** Tokens reported by the most recent assistant usage block. */ + usageTokens: number; + /** Estimated tokens after the most recent assistant usage block. */ + trailingTokens: number; + /** Index of the message that provided usage, or null when none exists. */ + lastUsageIndex: number | null; +} + +const CHARS_PER_TOKEN = 4; +const ESTIMATED_IMAGE_CHARS = 4800; + +export function calculateContextTokens(usage: Usage): number { + return usage.totalTokens || usage.input + usage.output + usage.cacheRead + usage.cacheWrite; +} + +function safeJsonStringify(value: unknown): string { + try { + return JSON.stringify(value) ?? "undefined"; + } catch { + return "[unserializable]"; + } +} + +function estimateTextAndImageContentChars(content: string | Array): number { + if (typeof content === "string") return content.length; + + let chars = 0; + for (const block of content) { + chars += block.type === "text" ? block.text.length : ESTIMATED_IMAGE_CHARS; + } + return chars; +} + +export function estimateTextTokens(text: string): number { + return Math.ceil(text.length / CHARS_PER_TOKEN); +} + +export function estimateTextAndImageContentTokens(content: string | Array): number { + return Math.ceil(estimateTextAndImageContentChars(content) / CHARS_PER_TOKEN); +} + +export function estimateMessageTokens(message: Message): number { + let chars = 0; + + switch (message.role) { + case "user": + return estimateTextAndImageContentTokens(message.content); + case "assistant": + for (const block of message.content) { + if (block.type === "text") { + chars += block.text.length; + } else if (block.type === "thinking") { + chars += block.thinking.length; + } else { + chars += block.name.length + safeJsonStringify(block.arguments).length; + } + } + break; + case "toolResult": + chars = estimateTextAndImageContentChars(message.content); + break; + } + + return Math.ceil(chars / CHARS_PER_TOKEN); +} + +function getLastAssistantUsageInfo(messages: readonly Message[]): { usage: Usage; index: number } | undefined { + for (let i = messages.length - 1; i >= 0; i--) { + const message = messages[i]; + if (message.role !== "assistant") continue; + const assistant = message as AssistantMessage; + if (assistant.stopReason === "aborted" || assistant.stopReason === "error") continue; + if (calculateContextTokens(assistant.usage) > 0) return { usage: assistant.usage, index: i }; + } + return undefined; +} + +function estimateMessages(messages: readonly Message[]): ContextUsageEstimate { + const usageInfo = getLastAssistantUsageInfo(messages); + + if (!usageInfo) { + let estimated = 0; + for (const message of messages) estimated += estimateMessageTokens(message); + return { tokens: estimated, usageTokens: 0, trailingTokens: estimated, lastUsageIndex: null }; + } + + const usageTokens = calculateContextTokens(usageInfo.usage); + let trailingTokens = 0; + for (let i = usageInfo.index + 1; i < messages.length; i++) { + trailingTokens += estimateMessageTokens(messages[i]); + } + + return { + tokens: usageTokens + trailingTokens, + usageTokens, + trailingTokens, + lastUsageIndex: usageInfo.index, + }; +} + +function isMessageArray(value: Context | readonly Message[]): value is readonly Message[] { + return Array.isArray(value); +} + +export function estimateContextTokens(context: Context | readonly Message[]): ContextUsageEstimate { + if (isMessageArray(context)) return estimateMessages(context); + + const estimate = estimateMessages(context.messages); + if (estimate.lastUsageIndex !== null) return estimate; + + let prefixTokens = context.systemPrompt ? estimateTextTokens(context.systemPrompt) : 0; + if (context.tools && context.tools.length > 0) { + prefixTokens += estimateTextTokens(safeJsonStringify(context.tools)); + } + + return { + tokens: estimate.tokens + prefixTokens, + usageTokens: estimate.usageTokens, + trailingTokens: estimate.trailingTokens + prefixTokens, + lastUsageIndex: estimate.lastUsageIndex, + }; +} diff --git a/packages/ai/test/anthropic-shared-budget-max-tokens.test.ts b/packages/ai/test/anthropic-shared-budget-max-tokens.test.ts new file mode 100644 index 00000000..24032ff7 --- /dev/null +++ b/packages/ai/test/anthropic-shared-budget-max-tokens.test.ts @@ -0,0 +1,125 @@ +import { describe, expect, it } from "vitest"; +import { getModel, streamSimple } from "../src/compat.ts"; +import type { Context, Model, SimpleStreamOptions, Usage } from "../src/types.ts"; + +interface AnthropicPayload { + max_tokens: number; + thinking?: { type: string; budget_tokens?: number }; +} + +class PayloadCaptured extends Error { + constructor() { + super("payload captured"); + this.name = "PayloadCaptured"; + } +} + +function usage(totalTokens: number): Usage { + return { + input: totalTokens, + output: 0, + cacheRead: 0, + cacheWrite: 0, + totalTokens, + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 }, + }; +} + +function makeModel(compat?: Model<"anthropic-messages">["compat"]): Model<"anthropic-messages"> { + return { + id: "vendor--minimax-m2.7", + name: "Vendor MiniMax M2.7", + api: "anthropic-messages", + provider: "vendor-proxy", + baseUrl: "http://127.0.0.1:9", + reasoning: true, + input: ["text"], + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, + contextWindow: 204800, + maxTokens: 131072, + compat, + }; +} + +async function capturePayload( + model: Model<"anthropic-messages">, + context: Context, + options?: SimpleStreamOptions, +): Promise { + let capturedPayload: AnthropicPayload | undefined; + const stream = streamSimple(model, context, { + ...options, + apiKey: "fake-key", + onPayload: (payload) => { + capturedPayload = payload as AnthropicPayload; + throw new PayloadCaptured(); + }, + }); + + await stream.result(); + if (!capturedPayload) throw new Error("Expected payload to be captured before request failure"); + return capturedPayload; +} + +describe("Anthropic shared-budget max_tokens compatibility", () => { + it("clamps max_tokens for shared-budget providers", async () => { + const context: Context = { + messages: [{ role: "user", content: "x".repeat(600_000), timestamp: Date.now() }], + }; + + const payload = await capturePayload(makeModel({ maxTokensSharesContextWindow: true }), context); + + expect(payload.max_tokens).toBe(204800 - Math.ceil(600_000 / 4)); + }); + + it("uses provider usage baseline before clamping", async () => { + const context: Context = { + messages: [ + { role: "user", content: "x".repeat(800_000), timestamp: Date.now() }, + { + role: "assistant", + content: [{ type: "text", text: "done" }], + api: "anthropic-messages", + provider: "vendor-proxy", + model: "vendor--minimax-m2.7", + usage: usage(50_000), + stopReason: "stop", + timestamp: Date.now(), + }, + { role: "user", content: "x".repeat(400), timestamp: Date.now() }, + ], + }; + + const payload = await capturePayload(makeModel({ maxTokensSharesContextWindow: true }), context); + + expect(payload.max_tokens).toBe(131072); + }); + + it("does not clamp Anthropic-style independent input/output budgets", async () => { + const context: Context = { + messages: [{ role: "user", content: "x".repeat(600_000), timestamp: Date.now() }], + }; + + const payload = await capturePayload(makeModel(), context); + + expect(payload.max_tokens).toBe(131072); + }); + + it("keeps thinking budget below clamped max_tokens", async () => { + const context: Context = { + messages: [{ role: "user", content: "x".repeat(800_000), timestamp: Date.now() }], + }; + + const payload = await capturePayload(makeModel({ maxTokensSharesContextWindow: true }), context, { + reasoning: "high", + }); + + expect(payload.thinking?.type).toBe("enabled"); + expect(payload.thinking?.budget_tokens).toBeLessThan(payload.max_tokens); + }); + + it("marks built-in MiniMax models as shared-budget", () => { + expect(getModel("minimax", "MiniMax-M2.7-highspeed").compat?.maxTokensSharesContextWindow).toBe(true); + expect(getModel("minimax-cn", "MiniMax-M2.7-highspeed").compat?.maxTokensSharesContextWindow).toBe(true); + }); +}); diff --git a/packages/ai/test/estimate.test.ts b/packages/ai/test/estimate.test.ts new file mode 100644 index 00000000..b60f30a0 --- /dev/null +++ b/packages/ai/test/estimate.test.ts @@ -0,0 +1,74 @@ +import { describe, expect, it } from "vitest"; +import { + type AssistantMessage, + type Context, + calculateContextTokens, + estimateContextTokens, + estimateMessageTokens, + type Usage, +} from "../src/index.ts"; + +function usage(totalTokens: number): Usage { + return { + input: totalTokens, + output: 0, + cacheRead: 0, + cacheWrite: 0, + totalTokens, + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 }, + }; +} + +function assistant(totalTokens: number, stopReason: AssistantMessage["stopReason"] = "stop"): AssistantMessage { + return { + role: "assistant", + content: [{ type: "text", text: "done" }], + api: "anthropic-messages", + provider: "test-provider", + model: "test-model", + usage: usage(totalTokens), + stopReason, + timestamp: Date.now(), + }; +} + +describe("token estimation", () => { + it("calculates context tokens from usage", () => { + expect(calculateContextTokens({ ...usage(0), input: 10, output: 20, cacheRead: 3, cacheWrite: 4 })).toBe(37); + expect(calculateContextTokens(usage(123))).toBe(123); + }); + + it("estimates message tokens from content", () => { + expect(estimateMessageTokens({ role: "user", content: "x".repeat(400), timestamp: 0 })).toBe(100); + }); + + it("uses the latest valid assistant usage as baseline and estimates only trailing messages", () => { + const context: Context = { + systemPrompt: "system text that should already be counted by provider usage", + messages: [ + { role: "user", content: "x".repeat(10_000), timestamp: 0 }, + assistant(50_000), + { role: "user", content: "x".repeat(400), timestamp: 0 }, + ], + }; + + expect(estimateContextTokens(context)).toMatchObject({ + tokens: 50_100, + usageTokens: 50_000, + trailingTokens: 100, + lastUsageIndex: 1, + }); + }); + + it("ignores error/aborted assistant usage", () => { + const context: Context = { + messages: [assistant(50_000, "error"), { role: "user", content: "x".repeat(400), timestamp: 0 }], + }; + + expect(estimateContextTokens(context)).toMatchObject({ + tokens: 101, + usageTokens: 0, + lastUsageIndex: null, + }); + }); +}); diff --git a/packages/coding-agent/CHANGELOG.md b/packages/coding-agent/CHANGELOG.md index 3dd5d2e1..b48d9716 100644 --- a/packages/coding-agent/CHANGELOG.md +++ b/packages/coding-agent/CHANGELOG.md @@ -4,6 +4,7 @@ ### Fixed +- Fixed MiniMax (`minimax`, `minimax-cn`) sessions failing on long conversations with `unknown error, 999` or `context window exceeds limit (2013)` by clamping `max_tokens` for shared-budget MiniMax models ([#6061](https://github.com/earendil-works/pi/issues/6061)). - Fixed auto-retry for provider stream errors that explicitly tell callers to retry the request ([#6019](https://github.com/earendil-works/pi/issues/6019)). ## [0.80.2] - 2026-06-23