fix(ai): revert minimax max token clamp
This commit is contained in:
Mario Zechner
@@ -4,14 +4,11 @@
|
||||
|
||||
### Added
|
||||
|
||||
- Added shared token estimation helpers for `Context` and `Message[]` in `pi-ai`.
|
||||
- Added `maxTokensSharesContextWindow` to `AnthropicMessagesCompat` for Anthropic-compatible providers that count `input + max_tokens` against a single shared context budget.
|
||||
- Added an optional `reasoning` field to `Usage` reporting reasoning/thinking token counts as a subset of `output`. Populated for Anthropic (`output_tokens_details.thinking_tokens`), OpenAI Responses/Codex/Azure (`output_tokens_details.reasoning_tokens`), OpenAI Completions (`completion_tokens_details.reasoning_tokens`), and Google Generative AI / Vertex (`thoughtsTokenCount`). Bedrock Converse and Mistral are not populated because those APIs do not return a reasoning token breakdown ([#6057](https://github.com/earendil-works/pi/issues/6057)).
|
||||
|
||||
### Fixed
|
||||
|
||||
- Fixed OpenAI Responses streams to preserve reasoning replay state when output items finish out of order ([#6009](https://github.com/earendil-works/pi/issues/6009)).
|
||||
- Fixed MiniMax (`minimax`, `minimax-cn`) Anthropic-compatible requests failing on long conversations with `unknown error, 999` or `context window exceeds limit (2013)` by clamping `max_tokens` for shared-budget models ([#6061](https://github.com/earendil-works/pi/issues/6061)).
|
||||
- Fixed retry classification for provider errors that explicitly tell callers to retry the request ([#6019](https://github.com/earendil-works/pi/issues/6019)).
|
||||
|
||||
## [0.80.2] - 2026-06-23
|
||||
|
||||
@@ -1424,7 +1424,6 @@ async function loadModelsDevData(): Promise<Model<any>[]> {
|
||||
provider,
|
||||
// MiniMax's Anthropic-compatible API - SDK appends /v1/messages
|
||||
baseUrl,
|
||||
compat: { maxTokensSharesContextWindow: true },
|
||||
reasoning: m.reasoning === true,
|
||||
input: m.modalities?.input?.includes("image") ? ["text", "image"] : ["text"],
|
||||
cost: {
|
||||
|
||||
@@ -29,7 +29,6 @@ import type {
|
||||
ToolCall,
|
||||
ToolResultMessage,
|
||||
} from "../types.ts";
|
||||
import { estimateContextTokens } from "../utils/estimate.ts";
|
||||
import { AssistantMessageEventStream } from "../utils/event-stream.ts";
|
||||
import { headersToRecord } from "../utils/headers.ts";
|
||||
import { parseJsonWithRepair, parseStreamingJson } from "../utils/json-parse.ts";
|
||||
@@ -178,12 +177,9 @@ function getAnthropicCompat(
|
||||
supportsCacheControlOnTools: model.compat?.supportsCacheControlOnTools ?? true,
|
||||
supportsTemperature: model.compat?.supportsTemperature ?? true,
|
||||
allowEmptySignature: model.compat?.allowEmptySignature ?? false,
|
||||
maxTokensSharesContextWindow: model.compat?.maxTokensSharesContextWindow ?? false,
|
||||
};
|
||||
}
|
||||
|
||||
const SHARED_BUDGET_MIN_OUTPUT_TOKENS = 1024;
|
||||
|
||||
export interface AnthropicOptions extends StreamOptions {
|
||||
/**
|
||||
* Enable extended thinking.
|
||||
@@ -1002,18 +998,6 @@ function buildParams(
|
||||
}
|
||||
}
|
||||
|
||||
if (compat.maxTokensSharesContextWindow && model.contextWindow > 0) {
|
||||
const estimatedInput = estimateContextTokens(context).tokens;
|
||||
const available = Math.max(SHARED_BUDGET_MIN_OUTPUT_TOKENS, model.contextWindow - estimatedInput);
|
||||
const clamped = Math.min(params.max_tokens, available);
|
||||
if (clamped < params.max_tokens) {
|
||||
params.max_tokens = clamped;
|
||||
if (params.thinking?.type === "enabled" && params.thinking.budget_tokens >= clamped) {
|
||||
params.thinking.budget_tokens = Math.max(0, clamped - SHARED_BUDGET_MIN_OUTPUT_TOKENS);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return params;
|
||||
}
|
||||
|
||||
|
||||
@@ -27,7 +27,6 @@ export * from "./providers/faux.ts";
|
||||
export * from "./session-resources.ts";
|
||||
export * from "./types.ts";
|
||||
export * from "./utils/diagnostics.ts";
|
||||
export * from "./utils/estimate.ts";
|
||||
export * from "./utils/event-stream.ts";
|
||||
export * from "./utils/json-parse.ts";
|
||||
export type {
|
||||
|
||||
@@ -10,7 +10,6 @@ export const MINIMAX_CN_MODELS = {
|
||||
api: "anthropic-messages",
|
||||
provider: "minimax-cn",
|
||||
baseUrl: "https://api.minimaxi.com/anthropic",
|
||||
compat: {"maxTokensSharesContextWindow":true},
|
||||
reasoning: true,
|
||||
input: ["text"],
|
||||
cost: {
|
||||
@@ -28,7 +27,6 @@ export const MINIMAX_CN_MODELS = {
|
||||
api: "anthropic-messages",
|
||||
provider: "minimax-cn",
|
||||
baseUrl: "https://api.minimaxi.com/anthropic",
|
||||
compat: {"maxTokensSharesContextWindow":true},
|
||||
reasoning: true,
|
||||
input: ["text"],
|
||||
cost: {
|
||||
@@ -46,7 +44,6 @@ export const MINIMAX_CN_MODELS = {
|
||||
api: "anthropic-messages",
|
||||
provider: "minimax-cn",
|
||||
baseUrl: "https://api.minimaxi.com/anthropic",
|
||||
compat: {"maxTokensSharesContextWindow":true},
|
||||
reasoning: true,
|
||||
input: ["text", "image"],
|
||||
cost: {
|
||||
|
||||
@@ -10,7 +10,6 @@ export const MINIMAX_MODELS = {
|
||||
api: "anthropic-messages",
|
||||
provider: "minimax",
|
||||
baseUrl: "https://api.minimax.io/anthropic",
|
||||
compat: {"maxTokensSharesContextWindow":true},
|
||||
reasoning: true,
|
||||
input: ["text"],
|
||||
cost: {
|
||||
@@ -28,7 +27,6 @@ export const MINIMAX_MODELS = {
|
||||
api: "anthropic-messages",
|
||||
provider: "minimax",
|
||||
baseUrl: "https://api.minimax.io/anthropic",
|
||||
compat: {"maxTokensSharesContextWindow":true},
|
||||
reasoning: true,
|
||||
input: ["text"],
|
||||
cost: {
|
||||
@@ -46,7 +44,6 @@ export const MINIMAX_MODELS = {
|
||||
api: "anthropic-messages",
|
||||
provider: "minimax",
|
||||
baseUrl: "https://api.minimax.io/anthropic",
|
||||
compat: {"maxTokensSharesContextWindow":true},
|
||||
reasoning: true,
|
||||
input: ["text", "image"],
|
||||
cost: {
|
||||
|
||||
@@ -573,14 +573,6 @@ export interface AnthropicMessagesCompat {
|
||||
forceAdaptiveThinking?: boolean;
|
||||
/** Whether to replay empty thinking signatures as `signature: ""` instead of converting thinking to text. Default: false. */
|
||||
allowEmptySignature?: boolean;
|
||||
/**
|
||||
* Whether the provider counts `input + max_tokens` against a single shared
|
||||
* budget (the model's `contextWindow`) instead of Anthropic's independent
|
||||
* input and output limits. When true, Anthropic Messages requests clamp
|
||||
* `max_tokens` to leave room for the estimated context tokens.
|
||||
* Default: false.
|
||||
*/
|
||||
maxTokensSharesContextWindow?: boolean;
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
@@ -1,127 +0,0 @@
|
||||
import type { AssistantMessage, Context, ImageContent, Message, TextContent, Usage } from "../types.ts";
|
||||
|
||||
export interface ContextUsageEstimate {
|
||||
/** Estimated total context tokens. */
|
||||
tokens: number;
|
||||
/** Tokens reported by the most recent assistant usage block. */
|
||||
usageTokens: number;
|
||||
/** Estimated tokens after the most recent assistant usage block. */
|
||||
trailingTokens: number;
|
||||
/** Index of the message that provided usage, or null when none exists. */
|
||||
lastUsageIndex: number | null;
|
||||
}
|
||||
|
||||
const CHARS_PER_TOKEN = 4;
|
||||
const ESTIMATED_IMAGE_CHARS = 4800;
|
||||
|
||||
export function calculateContextTokens(usage: Usage): number {
|
||||
return usage.totalTokens || usage.input + usage.output + usage.cacheRead + usage.cacheWrite;
|
||||
}
|
||||
|
||||
function safeJsonStringify(value: unknown): string {
|
||||
try {
|
||||
return JSON.stringify(value) ?? "undefined";
|
||||
} catch {
|
||||
return "[unserializable]";
|
||||
}
|
||||
}
|
||||
|
||||
function estimateTextAndImageContentChars(content: string | Array<TextContent | ImageContent>): number {
|
||||
if (typeof content === "string") return content.length;
|
||||
|
||||
let chars = 0;
|
||||
for (const block of content) {
|
||||
chars += block.type === "text" ? block.text.length : ESTIMATED_IMAGE_CHARS;
|
||||
}
|
||||
return chars;
|
||||
}
|
||||
|
||||
export function estimateTextTokens(text: string): number {
|
||||
return Math.ceil(text.length / CHARS_PER_TOKEN);
|
||||
}
|
||||
|
||||
export function estimateTextAndImageContentTokens(content: string | Array<TextContent | ImageContent>): number {
|
||||
return Math.ceil(estimateTextAndImageContentChars(content) / CHARS_PER_TOKEN);
|
||||
}
|
||||
|
||||
export function estimateMessageTokens(message: Message): number {
|
||||
let chars = 0;
|
||||
|
||||
switch (message.role) {
|
||||
case "user":
|
||||
return estimateTextAndImageContentTokens(message.content);
|
||||
case "assistant":
|
||||
for (const block of message.content) {
|
||||
if (block.type === "text") {
|
||||
chars += block.text.length;
|
||||
} else if (block.type === "thinking") {
|
||||
chars += block.thinking.length;
|
||||
} else {
|
||||
chars += block.name.length + safeJsonStringify(block.arguments).length;
|
||||
}
|
||||
}
|
||||
break;
|
||||
case "toolResult":
|
||||
chars = estimateTextAndImageContentChars(message.content);
|
||||
break;
|
||||
}
|
||||
|
||||
return Math.ceil(chars / CHARS_PER_TOKEN);
|
||||
}
|
||||
|
||||
function getLastAssistantUsageInfo(messages: readonly Message[]): { usage: Usage; index: number } | undefined {
|
||||
for (let i = messages.length - 1; i >= 0; i--) {
|
||||
const message = messages[i];
|
||||
if (message.role !== "assistant") continue;
|
||||
const assistant = message as AssistantMessage;
|
||||
if (assistant.stopReason === "aborted" || assistant.stopReason === "error") continue;
|
||||
if (calculateContextTokens(assistant.usage) > 0) return { usage: assistant.usage, index: i };
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
function estimateMessages(messages: readonly Message[]): ContextUsageEstimate {
|
||||
const usageInfo = getLastAssistantUsageInfo(messages);
|
||||
|
||||
if (!usageInfo) {
|
||||
let estimated = 0;
|
||||
for (const message of messages) estimated += estimateMessageTokens(message);
|
||||
return { tokens: estimated, usageTokens: 0, trailingTokens: estimated, lastUsageIndex: null };
|
||||
}
|
||||
|
||||
const usageTokens = calculateContextTokens(usageInfo.usage);
|
||||
let trailingTokens = 0;
|
||||
for (let i = usageInfo.index + 1; i < messages.length; i++) {
|
||||
trailingTokens += estimateMessageTokens(messages[i]);
|
||||
}
|
||||
|
||||
return {
|
||||
tokens: usageTokens + trailingTokens,
|
||||
usageTokens,
|
||||
trailingTokens,
|
||||
lastUsageIndex: usageInfo.index,
|
||||
};
|
||||
}
|
||||
|
||||
function isMessageArray(value: Context | readonly Message[]): value is readonly Message[] {
|
||||
return Array.isArray(value);
|
||||
}
|
||||
|
||||
export function estimateContextTokens(context: Context | readonly Message[]): ContextUsageEstimate {
|
||||
if (isMessageArray(context)) return estimateMessages(context);
|
||||
|
||||
const estimate = estimateMessages(context.messages);
|
||||
if (estimate.lastUsageIndex !== null) return estimate;
|
||||
|
||||
let prefixTokens = context.systemPrompt ? estimateTextTokens(context.systemPrompt) : 0;
|
||||
if (context.tools && context.tools.length > 0) {
|
||||
prefixTokens += estimateTextTokens(safeJsonStringify(context.tools));
|
||||
}
|
||||
|
||||
return {
|
||||
tokens: estimate.tokens + prefixTokens,
|
||||
usageTokens: estimate.usageTokens,
|
||||
trailingTokens: estimate.trailingTokens + prefixTokens,
|
||||
lastUsageIndex: estimate.lastUsageIndex,
|
||||
};
|
||||
}
|
||||
@@ -1,125 +0,0 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { getModel, streamSimple } from "../src/compat.ts";
|
||||
import type { Context, Model, SimpleStreamOptions, Usage } from "../src/types.ts";
|
||||
|
||||
interface AnthropicPayload {
|
||||
max_tokens: number;
|
||||
thinking?: { type: string; budget_tokens?: number };
|
||||
}
|
||||
|
||||
class PayloadCaptured extends Error {
|
||||
constructor() {
|
||||
super("payload captured");
|
||||
this.name = "PayloadCaptured";
|
||||
}
|
||||
}
|
||||
|
||||
function usage(totalTokens: number): Usage {
|
||||
return {
|
||||
input: totalTokens,
|
||||
output: 0,
|
||||
cacheRead: 0,
|
||||
cacheWrite: 0,
|
||||
totalTokens,
|
||||
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 },
|
||||
};
|
||||
}
|
||||
|
||||
function makeModel(compat?: Model<"anthropic-messages">["compat"]): Model<"anthropic-messages"> {
|
||||
return {
|
||||
id: "vendor--minimax-m2.7",
|
||||
name: "Vendor MiniMax M2.7",
|
||||
api: "anthropic-messages",
|
||||
provider: "vendor-proxy",
|
||||
baseUrl: "http://127.0.0.1:9",
|
||||
reasoning: true,
|
||||
input: ["text"],
|
||||
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 },
|
||||
contextWindow: 204800,
|
||||
maxTokens: 131072,
|
||||
compat,
|
||||
};
|
||||
}
|
||||
|
||||
async function capturePayload(
|
||||
model: Model<"anthropic-messages">,
|
||||
context: Context,
|
||||
options?: SimpleStreamOptions,
|
||||
): Promise<AnthropicPayload> {
|
||||
let capturedPayload: AnthropicPayload | undefined;
|
||||
const stream = streamSimple(model, context, {
|
||||
...options,
|
||||
apiKey: "fake-key",
|
||||
onPayload: (payload) => {
|
||||
capturedPayload = payload as AnthropicPayload;
|
||||
throw new PayloadCaptured();
|
||||
},
|
||||
});
|
||||
|
||||
await stream.result();
|
||||
if (!capturedPayload) throw new Error("Expected payload to be captured before request failure");
|
||||
return capturedPayload;
|
||||
}
|
||||
|
||||
describe("Anthropic shared-budget max_tokens compatibility", () => {
|
||||
it("clamps max_tokens for shared-budget providers", async () => {
|
||||
const context: Context = {
|
||||
messages: [{ role: "user", content: "x".repeat(600_000), timestamp: Date.now() }],
|
||||
};
|
||||
|
||||
const payload = await capturePayload(makeModel({ maxTokensSharesContextWindow: true }), context);
|
||||
|
||||
expect(payload.max_tokens).toBe(204800 - Math.ceil(600_000 / 4));
|
||||
});
|
||||
|
||||
it("uses provider usage baseline before clamping", async () => {
|
||||
const context: Context = {
|
||||
messages: [
|
||||
{ role: "user", content: "x".repeat(800_000), timestamp: Date.now() },
|
||||
{
|
||||
role: "assistant",
|
||||
content: [{ type: "text", text: "done" }],
|
||||
api: "anthropic-messages",
|
||||
provider: "vendor-proxy",
|
||||
model: "vendor--minimax-m2.7",
|
||||
usage: usage(50_000),
|
||||
stopReason: "stop",
|
||||
timestamp: Date.now(),
|
||||
},
|
||||
{ role: "user", content: "x".repeat(400), timestamp: Date.now() },
|
||||
],
|
||||
};
|
||||
|
||||
const payload = await capturePayload(makeModel({ maxTokensSharesContextWindow: true }), context);
|
||||
|
||||
expect(payload.max_tokens).toBe(131072);
|
||||
});
|
||||
|
||||
it("does not clamp Anthropic-style independent input/output budgets", async () => {
|
||||
const context: Context = {
|
||||
messages: [{ role: "user", content: "x".repeat(600_000), timestamp: Date.now() }],
|
||||
};
|
||||
|
||||
const payload = await capturePayload(makeModel(), context);
|
||||
|
||||
expect(payload.max_tokens).toBe(131072);
|
||||
});
|
||||
|
||||
it("keeps thinking budget below clamped max_tokens", async () => {
|
||||
const context: Context = {
|
||||
messages: [{ role: "user", content: "x".repeat(800_000), timestamp: Date.now() }],
|
||||
};
|
||||
|
||||
const payload = await capturePayload(makeModel({ maxTokensSharesContextWindow: true }), context, {
|
||||
reasoning: "high",
|
||||
});
|
||||
|
||||
expect(payload.thinking?.type).toBe("enabled");
|
||||
expect(payload.thinking?.budget_tokens).toBeLessThan(payload.max_tokens);
|
||||
});
|
||||
|
||||
it("marks built-in MiniMax models as shared-budget", () => {
|
||||
expect(getModel("minimax", "MiniMax-M2.7-highspeed").compat?.maxTokensSharesContextWindow).toBe(true);
|
||||
expect(getModel("minimax-cn", "MiniMax-M2.7-highspeed").compat?.maxTokensSharesContextWindow).toBe(true);
|
||||
});
|
||||
});
|
||||
@@ -1,74 +0,0 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import {
|
||||
type AssistantMessage,
|
||||
type Context,
|
||||
calculateContextTokens,
|
||||
estimateContextTokens,
|
||||
estimateMessageTokens,
|
||||
type Usage,
|
||||
} from "../src/index.ts";
|
||||
|
||||
function usage(totalTokens: number): Usage {
|
||||
return {
|
||||
input: totalTokens,
|
||||
output: 0,
|
||||
cacheRead: 0,
|
||||
cacheWrite: 0,
|
||||
totalTokens,
|
||||
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 },
|
||||
};
|
||||
}
|
||||
|
||||
function assistant(totalTokens: number, stopReason: AssistantMessage["stopReason"] = "stop"): AssistantMessage {
|
||||
return {
|
||||
role: "assistant",
|
||||
content: [{ type: "text", text: "done" }],
|
||||
api: "anthropic-messages",
|
||||
provider: "test-provider",
|
||||
model: "test-model",
|
||||
usage: usage(totalTokens),
|
||||
stopReason,
|
||||
timestamp: Date.now(),
|
||||
};
|
||||
}
|
||||
|
||||
describe("token estimation", () => {
|
||||
it("calculates context tokens from usage", () => {
|
||||
expect(calculateContextTokens({ ...usage(0), input: 10, output: 20, cacheRead: 3, cacheWrite: 4 })).toBe(37);
|
||||
expect(calculateContextTokens(usage(123))).toBe(123);
|
||||
});
|
||||
|
||||
it("estimates message tokens from content", () => {
|
||||
expect(estimateMessageTokens({ role: "user", content: "x".repeat(400), timestamp: 0 })).toBe(100);
|
||||
});
|
||||
|
||||
it("uses the latest valid assistant usage as baseline and estimates only trailing messages", () => {
|
||||
const context: Context = {
|
||||
systemPrompt: "system text that should already be counted by provider usage",
|
||||
messages: [
|
||||
{ role: "user", content: "x".repeat(10_000), timestamp: 0 },
|
||||
assistant(50_000),
|
||||
{ role: "user", content: "x".repeat(400), timestamp: 0 },
|
||||
],
|
||||
};
|
||||
|
||||
expect(estimateContextTokens(context)).toMatchObject({
|
||||
tokens: 50_100,
|
||||
usageTokens: 50_000,
|
||||
trailingTokens: 100,
|
||||
lastUsageIndex: 1,
|
||||
});
|
||||
});
|
||||
|
||||
it("ignores error/aborted assistant usage", () => {
|
||||
const context: Context = {
|
||||
messages: [assistant(50_000, "error"), { role: "user", content: "x".repeat(400), timestamp: 0 }],
|
||||
};
|
||||
|
||||
expect(estimateContextTokens(context)).toMatchObject({
|
||||
tokens: 101,
|
||||
usageTokens: 0,
|
||||
lastUsageIndex: null,
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -4,7 +4,6 @@
|
||||
|
||||
### Fixed
|
||||
|
||||
- Fixed MiniMax (`minimax`, `minimax-cn`) sessions failing on long conversations with `unknown error, 999` or `context window exceeds limit (2013)` by clamping `max_tokens` for shared-budget MiniMax models ([#6061](https://github.com/earendil-works/pi/issues/6061)).
|
||||
- Fixed disk BMP image files to be detected, converted to PNG, and attached through `read` and CLI `@file` inputs ([#6047](https://github.com/earendil-works/pi/issues/6047)).
|
||||
- Fixed auto-retry for provider stream errors that explicitly tell callers to retry the request ([#6019](https://github.com/earendil-works/pi/issues/6019)).
|
||||
|
||||
|
||||
Reference in New Issue
Block a user