ton/pi_harness
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

new_pull

Browse Source
This commit is contained in:
narawat
2026-07-26 14:02:37 +07:00
parent bc56546b49
commit 367ebc1c7f
171 changed files with 4617 additions and 10402 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/ai/src/api/anthropic-messages.ts
+37 -8
View File
@@ -34,8 +34,10 @@ import { AssistantMessageEventStream } from "../utils/event-stream.ts";
import { headersToRecord } from "../utils/headers.ts";
import { parseJsonWithRepair, parseStreamingJson } from "../utils/json-parse.ts";
import { getProviderEnvValue } from "../utils/provider-env.ts";
import { retryProviderRequest } from "../utils/provider-retry.ts";
import { sanitizeSurrogates } from "../utils/sanitize-unicode.ts";
import { resolveJsonSchemaStrictSampling } from "./constrained-sampling.ts";
import { buildCopilotDynamicHeaders, hasCopilotVisionInput } from "./github-copilot-headers.ts";
import { adjustMaxTokensForThinking, buildBaseOptions, clampMaxTokensToContext } from "./simple-options.ts";
import { transformMessages } from "./transform-messages.ts";
@@ -178,6 +180,7 @@ function getAnthropicCompat(
supportsCacheControlOnTools: model.compat?.supportsCacheControlOnTools ?? true,
supportsTemperature: model.compat?.supportsTemperature ?? true,
allowEmptySignature: model.compat?.allowEmptySignature ?? false,
supportsStrictTools: model.compat?.supportsStrictTools ?? false,
supportsToolReferences: model.compat?.supportsToolReferences ?? defaultSupportsToolReferences(model),
};
}
@@ -550,9 +553,16 @@ export const stream: StreamFunction<"anthropic-messages", AnthropicOptions> = (
const requestOptions = {
...(options?.signal ? { signal: options.signal } : {}),
...(options?.timeoutMs !== undefined ? { timeout: options.timeoutMs } : {}),
maxRetries: options?.maxRetries ?? 0,
maxRetries: 0,
};
const response = await client.messages.create({ ...params, stream: true }, requestOptions).asResponse();
const response = await retryProviderRequest(
() => client.messages.create({ ...params, stream: true }, requestOptions).asResponse(),
{
maxRetries: options?.maxRetries,
maxRetryDelayMs: options?.maxRetryDelayMs,
signal: options?.signal,
},
);
await options?.onResponse?.({ status: response.status, headers: headersToRecord(response.headers) }, model);
stream.push({ type: "start", partial: output });
@@ -991,9 +1001,17 @@ function buildParams(
immediateTools,
isOAuthToken,
compat.supportsEagerToolInputStreaming,
compat.supportsStrictTools,
compat.supportsCacheControlOnTools ? cacheControl : undefined,
),
...convertTools(deferredTools, isOAuthToken, compat.supportsEagerToolInputStreaming, undefined, true),
...convertTools(
deferredTools,
isOAuthToken,
compat.supportsEagerToolInputStreaming,
compat.supportsStrictTools,
undefined,
true,
),
];
}
@@ -1261,23 +1279,34 @@ function convertTools(
tools: Tool[],
isOAuthToken: boolean,
supportsEagerToolInputStreaming: boolean,
supportsStrictTools: boolean,
cacheControl?: CacheControlEphemeral,
deferLoading = false,
): Anthropic.Messages.Tool[] {
if (!tools) return [];
return tools.map((tool, index) => {
const strict = resolveJsonSchemaStrictSampling(tool, supportsStrictTools);
const schema = tool.parameters as { properties?: unknown; required?: string[] };
const legacyInputSchema = {
type: "object" as const,
properties: schema.properties ?? {},
required: schema.required ?? [],
};
const inputSchema =
strict === true
? {
...(tool.parameters as Record<string, unknown>),
...legacyInputSchema,
}
: legacyInputSchema;
return {
name: isOAuthToken ? toClaudeCodeName(tool.name) : tool.name,
description: tool.description,
...(supportsEagerToolInputStreaming ? { eager_input_streaming: true } : {}),
input_schema: {
type: "object",
properties: schema.properties ?? {},
required: schema.required ?? [],
},
...(strict === true ? { strict: true } : {}),
input_schema: inputSchema,
...(deferLoading ? { defer_loading: true } : {}),
...(cacheControl && index === tools.length - 1 ? { cache_control: cacheControl } : {}),
};
packages/ai/src/api/azure-openai-responses.ts
+30 -7
View File
@@ -14,6 +14,8 @@ import { formatProviderError, normalizeProviderError } from "../utils/error-body
import { AssistantMessageEventStream } from "../utils/event-stream.ts";
import { headersToRecord } from "../utils/headers.ts";
import { getProviderEnvValue } from "../utils/provider-env.ts";
import { retryProviderRequest } from "../utils/provider-retry.ts";
import { createGrammarToolInputProperties } from "./constrained-sampling.ts";
import { clampOpenAIPromptCacheKey } from "./openai-prompt-cache.ts";
import { convertResponsesMessages, convertResponsesTools, processResponsesStream } from "./openai-responses-shared.ts";
import { buildBaseOptions } from "./simple-options.ts";
@@ -99,7 +101,11 @@ export const stream: StreamFunction<"azure-openai-responses", AzureOpenAIRespons
throw new Error(`No API key for provider: ${model.provider}`);
}
const client = createClient(model, apiKey, options);
let params = buildParams(model, context, options, deploymentName);
const grammarToolInputProperties = createGrammarToolInputProperties(
context.tools,
model.compat?.supportsOpenAIGrammarTools ?? false,
);
let params = buildParams(model, context, options, deploymentName, grammarToolInputProperties);
const nextParams = await options?.onPayload?.(params, model);
if (nextParams !== undefined) {
params = nextParams as ResponseCreateParamsStreaming;
@@ -107,13 +113,20 @@ export const stream: StreamFunction<"azure-openai-responses", AzureOpenAIRespons
const requestOptions = {
...(options?.signal ? { signal: options.signal } : {}),
...(options?.timeoutMs !== undefined ? { timeout: options.timeoutMs } : {}),
maxRetries: options?.maxRetries ?? 0,
maxRetries: 0,
};
const { data: openaiStream, response } = await client.responses.create(params, requestOptions).withResponse();
const { data: openaiStream, response } = await retryProviderRequest(
() => client.responses.create(params, requestOptions).withResponse(),
{
maxRetries: options?.maxRetries,
maxRetryDelayMs: options?.maxRetryDelayMs,
signal: options?.signal,
},
);
await options?.onResponse?.({ status: response.status, headers: headersToRecord(response.headers) }, model);
stream.push({ type: "start", partial: output });
await processResponsesStream(openaiStream, output, stream, model);
await processResponsesStream(openaiStream, output, stream, model, { grammarToolInputProperties });
if (options?.signal?.aborted) {
throw new Error("Request was aborted");
@@ -128,8 +141,9 @@ export const stream: StreamFunction<"azure-openai-responses", AzureOpenAIRespons
} catch (error) {
for (const block of output.content) {
delete (block as { index?: number }).index;
// partialJson is only a streaming scratch buffer; never persist it.
// Streaming scratch buffers are only used during parsing; never persist them.
delete (block as { partialJson?: string }).partialJson;
delete (block as { customInput?: unknown }).customInput;
}
output.stopReason = options?.signal?.aborted ? "aborted" : "error";
output.errorMessage = formatAzureOpenAIError(error);
@@ -254,8 +268,14 @@ function buildParams(
context: Context,
options: AzureOpenAIResponsesOptions | undefined,
deploymentName: string,
grammarToolInputProperties: ReadonlyMap<string, string> = createGrammarToolInputProperties(
context.tools,
model.compat?.supportsOpenAIGrammarTools ?? false,
),
) {
const messages = convertResponsesMessages(model, context, AZURE_TOOL_CALL_PROVIDERS);
const messages = convertResponsesMessages(model, context, AZURE_TOOL_CALL_PROVIDERS, {
grammarToolInputProperties,
});
const params: ResponseCreateParamsStreaming = {
model: deploymentName,
@@ -274,7 +294,10 @@ function buildParams(
}
if (context.tools && context.tools.length > 0) {
params.tools = convertResponsesTools(context.tools);
params.tools = convertResponsesTools(context.tools, {
supportsStrictMode: model.compat?.supportsStrictMode ?? true,
supportsOpenAIGrammarTools: model.compat?.supportsOpenAIGrammarTools ?? false,
});
}
if (model.reasoning) {
packages/ai/src/api/bedrock-converse-stream.ts
+25 -12
View File
@@ -54,6 +54,7 @@ import { parseStreamingJson } from "../utils/json-parse.ts";
import { resolveHttpProxyUrlForTarget } from "../utils/node-http-proxy.ts";
import { getProviderEnvValue } from "../utils/provider-env.ts";
import { sanitizeSurrogates } from "../utils/sanitize-unicode.ts";
import { resolveJsonSchemaStrictSampling } from "./constrained-sampling.ts";
import {
adjustMaxTokensForThinking,
buildBaseOptions,
@@ -228,7 +229,7 @@ export const stream: StreamFunction<"bedrock-converse-stream", BedrockOptions> =
...(inferenceMaxTokens !== undefined && { maxTokens: inferenceMaxTokens }),
...(options.temperature !== undefined && { temperature: options.temperature }),
},
toolConfig: convertToolConfig(context.tools, options.toolChoice),
toolConfig: convertToolConfig(context.tools, options.toolChoice, model.compat?.supportsStrictMode ?? false),
additionalModelRequestFields: buildAdditionalModelRequestFields(model, options),
...(options.requestMetadata !== undefined && { requestMetadata: options.requestMetadata }),
};
@@ -581,6 +582,7 @@ function supportsAdaptiveThinking(modelId: string, modelName?: string): boolean
s.includes("opus-4-6") ||
s.includes("opus-4-7") ||
s.includes("opus-4-8") ||
s.includes("opus-5") ||
s.includes("sonnet-4-6") ||
s.includes("sonnet-5") ||
s.includes("fable-5"),
@@ -590,7 +592,12 @@ function supportsAdaptiveThinking(modelId: string, modelName?: string): boolean
function supportsNativeXhighEffort(model: Model<"bedrock-converse-stream">): boolean {
const candidates = getModelMatchCandidates(model.id, model.name);
return candidates.some(
(s) => s.includes("opus-4-7") || s.includes("opus-4-8") || s.includes("sonnet-5") || s.includes("fable-5"),
(s) =>
s.includes("opus-4-7") ||
s.includes("opus-4-8") ||
s.includes("opus-5") ||
s.includes("sonnet-5") ||
s.includes("fable-5"),
);
}
@@ -669,8 +676,8 @@ function supportsPromptCaching(model: Model<"bedrock-converse-stream">, env?: Pr
if (getProviderEnvValue("AWS_BEDROCK_FORCE_CACHE", env) === "1") return true;
return false;
}
// Claude 5 models (fable-5, sonnet-5)
if (candidates.some((s) => s.includes("fable-5") || s.includes("sonnet-5"))) return true;
// Claude 5 models (fable-5, opus-5, sonnet-5)
if (candidates.some((s) => s.includes("fable-5") || s.includes("opus-5") || s.includes("sonnet-5"))) return true;
// Claude 4.x models (opus-4, sonnet-4, haiku-4)
if (candidates.some((s) => s.includes("-4-"))) return true;
// Claude 3.7 Sonnet
@@ -908,16 +915,22 @@ function convertMessages(
function convertToolConfig(
tools: Tool[] | undefined,
toolChoice: BedrockOptions["toolChoice"],
supportsStrictMode: boolean,
): ToolConfiguration | undefined {
if (!tools?.length || toolChoice === "none") return undefined;
if (!tools?.length) return undefined;
if (toolChoice === "none") return undefined;
const bedrockTools: BedrockTool[] = tools.map((tool) => ({
toolSpec: {
name: tool.name,
description: tool.description,
inputSchema: { json: tool.parameters as unknown as DocumentType },
},
}));
const bedrockTools: BedrockTool[] = tools.map((tool) => {
const strict = resolveJsonSchemaStrictSampling(tool, supportsStrictMode);
return {
toolSpec: {
name: tool.name,
description: tool.description,
inputSchema: { json: tool.parameters as unknown as DocumentType },
...(strict === true ? { strict: true } : {}),
},
};
});
let bedrockToolChoice: ToolChoice | undefined;
switch (toolChoice) {
packages/ai/src/api/constrained-sampling.ts
+148
View File
@@ -0,0 +1,148 @@
import type { Tool } from "../types.ts";
interface JsonSchemaObject {
type?: unknown;
properties?: Record<string, JsonSchemaObject | undefined>;
required?: unknown;
}
export interface GrammarConstrainedSampling {
format: "lark" | "regex";
definition: string;
inputProperty: string;
}
export interface GrammarToolInputJsonBuffer {
input: string;
started: boolean;
closed: boolean;
}
export function getGrammarToolInput(
toolName: string,
arguments_: Record<string, unknown>,
inputProperty: string,
): string {
const input = arguments_[inputProperty];
if (typeof input !== "string") {
throw new Error(`Grammar tool call "${toolName}" requires argument "${inputProperty}" to be a string.`);
}
return input;
}
export function appendGrammarToolInputJsonDelta(
buffer: GrammarToolInputJsonBuffer,
inputProperty: string,
nextInput: string,
close: boolean,
): string | undefined {
if (buffer.closed) {
if (close && nextInput === buffer.input) return undefined;
throw new Error(`grammar tool input for property "${inputProperty}" changed after it was closed`);
}
if (!nextInput.startsWith(buffer.input)) {
throw new Error(`grammar tool input for property "${inputProperty}" changed non-monotonically`);
}
const inputDelta = nextInput.slice(buffer.input.length);
if (!close && inputDelta.length === 0) return undefined;
let delta = "";
if (!buffer.started) {
delta += `{${JSON.stringify(inputProperty)}:"`;
buffer.started = true;
}
delta += JSON.stringify(inputDelta).slice(1, -1);
buffer.input = nextInput;
if (close) {
delta += '"}';
buffer.closed = true;
}
return delta;
}
function inferGrammarInputProperty(tool: Tool): string {
const schema = tool.parameters as JsonSchemaObject;
if (schema.type !== "object") {
throw new Error("grammar constrained sampling requires an object parameter schema");
}
if (!Array.isArray(schema.required) || schema.required.length !== 1 || typeof schema.required[0] !== "string") {
throw new Error("grammar constrained sampling requires exactly one required string property");
}
const inputProperty = schema.required[0];
if (!schema.properties?.[inputProperty]) {
throw new Error(`grammar constrained sampling requires a properties entry for ${inputProperty}`);
}
if (schema.properties[inputProperty]?.type !== "string") {
throw new Error(`grammar constrained sampling property ${inputProperty} must have type string`);
}
return inputProperty;
}
export function resolveJsonSchemaStrictSampling(tool: Tool, supportsStrictMode: boolean): boolean | undefined {
const config = tool.constrainedSampling;
if (!config || config.type !== "json_schema") {
return undefined;
}
if (supportsStrictMode) {
return true;
}
if (config.strict === "require") {
throw new Error(
`Tool "${tool.name}" requires JSON-schema constrained sampling, but strict tools are unsupported.`,
);
}
return undefined;
}
export function resolveGrammarConstrainedSampling(
tool: Tool,
supportsOpenAIGrammarTools: boolean,
): GrammarConstrainedSampling | undefined {
const config = tool.constrainedSampling;
if (!config || config.type !== "grammar") {
return undefined;
}
if (!supportsOpenAIGrammarTools) {
return undefined;
}
const larkDefinition = config.variants.openai_lark;
const regexDefinition = config.variants.openai_regex;
const hasLarkDefinition = typeof larkDefinition === "string" && larkDefinition.trim().length > 0;
const hasRegexDefinition = typeof regexDefinition === "string" && regexDefinition.trim().length > 0;
if (!hasLarkDefinition && !hasRegexDefinition) {
throw new Error(
`Tool "${tool.name}" cannot use grammar constrained sampling: no supported grammar variant was provided.`,
);
}
try {
return {
format: hasLarkDefinition ? "lark" : "regex",
definition: hasLarkDefinition ? larkDefinition : regexDefinition!,
inputProperty: inferGrammarInputProperty(tool),
};
} catch (error) {
const message = error instanceof Error ? error.message : String(error);
throw new Error(`Tool "${tool.name}" cannot use grammar constrained sampling: ${message}.`);
}
}
export function createGrammarToolInputProperties(
tools: Tool[] | undefined,
supportsOpenAIGrammarTools: boolean,
): ReadonlyMap<string, string> {
const properties = new Map<string, string>();
for (const tool of tools ?? []) {
const grammar = resolveGrammarConstrainedSampling(tool, supportsOpenAIGrammarTools);
if (grammar) {
properties.set(tool.name, grammar.inputProperty);
}
}
return properties;
}
packages/ai/src/api/google-generative-ai.ts
+8 -11
View File
@@ -30,8 +30,9 @@ import {
convertTools,
isThinkingPart,
mapStopReason,
mapToolChoice,
resolveGoogleFunctionCallingMode,
retainThoughtSignature,
supportsGoogleStrictToolSampling,
} from "./google-shared.ts";
import { buildBaseOptions } from "./simple-options.ts";
@@ -355,22 +356,18 @@ function buildParams(
generationConfig.maxOutputTokens = options.maxTokens;
}
const functionCallingMode = context.tools?.length
? resolveGoogleFunctionCallingMode(context.tools, options.toolChoice, supportsGoogleStrictToolSampling(model.id))
: undefined;
const config: GenerateContentConfig = {
...(Object.keys(generationConfig).length > 0 && generationConfig),
...(context.systemPrompt && { systemInstruction: sanitizeSurrogates(context.systemPrompt) }),
...(context.tools && context.tools.length > 0 && { tools: convertTools(context.tools) }),
...(functionCallingMode !== undefined && {
toolConfig: { functionCallingConfig: { mode: functionCallingMode } },
}),
};
if (context.tools && context.tools.length > 0 && options.toolChoice) {
config.toolConfig = {
functionCallingConfig: {
mode: mapToolChoice(options.toolChoice),
},
};
} else {
config.toolConfig = undefined;
}
if (options.thinking?.enabled && model.reasoning) {
const thinkingConfig: ThinkingConfig = { includeThoughts: true };
if (options.thinking.level !== undefined) {
packages/ai/src/api/google-shared.ts
+23 -3
View File
@@ -5,6 +5,7 @@
import { type Content, FinishReason, FunctionCallingConfigMode, type Part } from "@google/genai";
import type { Context, ImageContent, Model, StopReason, TextContent, Tool } from "../types.ts";
import { sanitizeSurrogates } from "../utils/sanitize-unicode.ts";
import { resolveJsonSchemaStrictSampling } from "./constrained-sampling.ts";
import { transformMessages } from "./transform-messages.ts";
type GoogleApiType = "google-generative-ai" | "google-vertex";
@@ -287,9 +288,13 @@ export function convertTools(
];
}
/**
* Map tool choice string to Gemini FunctionCallingConfigMode.
*/
/** Gemini 3+ enforces required function parameters in validated tool-calling modes. */
export function supportsGoogleStrictToolSampling(modelId: string): boolean {
const majorVersion = getGeminiMajorVersion(modelId);
return majorVersion !== undefined && majorVersion >= 3;
}
/** Map tool choice string to Gemini FunctionCallingConfigMode. */
export function mapToolChoice(choice: string): FunctionCallingConfigMode {
switch (choice) {
case "auto":
@@ -303,6 +308,21 @@ export function mapToolChoice(choice: string): FunctionCallingConfigMode {
}
}
export function resolveGoogleFunctionCallingMode(
tools: Tool[],
toolChoice: string | undefined,
supportsStrictMode: boolean,
): FunctionCallingConfigMode | undefined {
const useStrictMode = tools.some((tool) => resolveJsonSchemaStrictSampling(tool, supportsStrictMode) === true);
if (toolChoice === "none" || toolChoice === "any") {
return mapToolChoice(toolChoice);
}
if (useStrictMode) {
return FunctionCallingConfigMode.VALIDATED;
}
return toolChoice ? mapToolChoice(toolChoice) : undefined;
}
/**
* Map Gemini FinishReason to our StopReason.
*/
packages/ai/src/api/google-vertex.ts
+8 -11
View File
@@ -35,8 +35,9 @@ import {
convertTools,
isThinkingPart,
mapStopReason,
mapToolChoice,
resolveGoogleFunctionCallingMode,
retainThoughtSignature,
supportsGoogleStrictToolSampling,
} from "./google-shared.ts";
import { buildBaseOptions } from "./simple-options.ts";
@@ -454,22 +455,18 @@ function buildParams(
generationConfig.maxOutputTokens = options.maxTokens;
}
const functionCallingMode = context.tools?.length
? resolveGoogleFunctionCallingMode(context.tools, options.toolChoice, supportsGoogleStrictToolSampling(model.id))
: undefined;
const config: GenerateContentConfig = {
...(Object.keys(generationConfig).length > 0 && generationConfig),
...(context.systemPrompt && { systemInstruction: sanitizeSurrogates(context.systemPrompt) }),
...(context.tools && context.tools.length > 0 && { tools: convertTools(context.tools) }),
...(functionCallingMode !== undefined && {
toolConfig: { functionCallingConfig: { mode: functionCallingMode } },
}),
};
if (context.tools && context.tools.length > 0 && options.toolChoice) {
config.toolConfig = {
functionCallingConfig: {
mode: mapToolChoice(options.toolChoice),
},
};
} else {
config.toolConfig = undefined;
}
if (options.thinking?.enabled && model.reasoning) {
const thinkingConfig: ThinkingConfig = { includeThoughts: true };
if (options.thinking.level !== undefined) {
packages/ai/src/api/mistral-conversations.ts
+13 -9
View File
@@ -25,6 +25,7 @@ import { AssistantMessageEventStream } from "../utils/event-stream.ts";
import { shortHash } from "../utils/hash.ts";
import { parseStreamingJson } from "../utils/json-parse.ts";
import { sanitizeSurrogates } from "../utils/sanitize-unicode.ts";
import { resolveJsonSchemaStrictSampling } from "./constrained-sampling.ts";
import { buildBaseOptions } from "./simple-options.ts";
import { transformMessages } from "./transform-messages.ts";
@@ -483,15 +484,18 @@ async function consumeChatStream(
}
function toFunctionTools(tools: Tool[]): Array<FunctionTool & { type: "function" }> {
return tools.map((tool) => ({
type: "function",
function: {
name: tool.name,
description: tool.description,
parameters: stripSymbolKeys(tool.parameters) as Record<string, unknown>,
strict: false,
},
}));
return tools.map((tool) => {
const strict = resolveJsonSchemaStrictSampling(tool, true);
return {
type: "function",
function: {
name: tool.name,
description: tool.description,
parameters: stripSymbolKeys(tool.parameters) as Record<string, unknown>,
strict: strict ?? false,
},
};
});
}
function stripSymbolKeys(value: unknown): unknown {
packages/ai/src/api/openai-codex-responses.ts
+61 -20
View File
@@ -47,6 +47,7 @@ import { AssistantMessageEventStream } from "../utils/event-stream.ts";
import { headersToRecord } from "../utils/headers.ts";
import { resolveHttpProxyUrlForTarget } from "../utils/node-http-proxy.ts";
import { uuidv7 } from "../utils/uuid.ts";
import { createGrammarToolInputProperties } from "./constrained-sampling.ts";
import { clampOpenAIPromptCacheKey } from "./openai-prompt-cache.ts";
import { convertResponsesMessages, convertResponsesTools, processResponsesStream } from "./openai-responses-shared.ts";
import { buildBaseOptions } from "./simple-options.ts";
@@ -158,9 +159,16 @@ function getRetryAfterDelayMs(headers: Headers): number | undefined {
return undefined;
}
function capRetryDelayMs(delayMs: number, options?: StreamOptions): number {
class RetryDelayExceededError extends Error {}
function validateRetryDelayMs(delayMs: number, options?: StreamOptions): number {
const maxRetryDelayMs = options?.maxRetryDelayMs ?? DEFAULT_MAX_RETRY_DELAY_MS;
return maxRetryDelayMs > 0 ? Math.min(delayMs, maxRetryDelayMs) : delayMs;
if (maxRetryDelayMs > 0 && delayMs > maxRetryDelayMs) {
throw new RetryDelayExceededError(
`Server requested ${Math.ceil(delayMs / 1000)}s retry delay (max: ${Math.ceil(maxRetryDelayMs / 1000)}s)`,
);
}
return delayMs;
}
function sleep(ms: number, signal?: AbortSignal): Promise<void> {
@@ -255,12 +263,17 @@ export const stream: StreamFunction<"openai-codex-responses", OpenAICodexRespons
}
const accountId = extractAccountId(apiKey);
let body = buildRequestBody(model, context, options);
const grammarToolInputProperties = createGrammarToolInputProperties(
context.tools,
model.compat?.supportsOpenAIGrammarTools ?? false,
);
const cacheSessionId = options?.cacheRetention === "none" ? undefined : options?.sessionId;
const codexSessionId = clampOpenAIPromptCacheKey(cacheSessionId);
let body = buildRequestBody(model, context, options, codexSessionId, grammarToolInputProperties);
const nextBody = await options?.onPayload?.(body, model);
if (nextBody !== undefined) {
body = nextBody as RequestBody;
}
const codexSessionId = clampOpenAIPromptCacheKey(options?.sessionId);
const websocketRequestId = codexSessionId || uuidv7();
const sseHeaders = buildSSEHeaders(model.headers, options?.headers, accountId, apiKey, codexSessionId);
const websocketHeaders = buildWebSocketHeaders(
@@ -275,9 +288,9 @@ export const stream: StreamFunction<"openai-codex-responses", OpenAICodexRespons
const websocketConnectTimeoutMs = normalizeTimeoutMs(options?.websocketConnectTimeoutMs);
const transport = options?.transport || "auto";
let startEmitted = false;
const websocketDisabledForSession = transport !== "sse" && isWebSocketSseFallbackActive(options?.sessionId);
const websocketDisabledForSession = transport !== "sse" && isWebSocketSseFallbackActive(cacheSessionId);
if (websocketDisabledForSession) {
recordWebSocketSseFallback(options?.sessionId);
recordWebSocketSseFallback(cacheSessionId);
}
if (transport !== "sse" && !websocketDisabledForSession) {
@@ -303,6 +316,8 @@ export const stream: StreamFunction<"openai-codex-responses", OpenAICodexRespons
},
httpTimeoutMs,
websocketConnectTimeoutMs,
cacheSessionId,
grammarToolInputProperties,
options,
);
@@ -341,11 +356,11 @@ export const stream: StreamFunction<"openai-codex-responses", OpenAICodexRespons
requestBytes: new TextEncoder().encode(bodyJson).byteLength,
}),
);
recordWebSocketFailure(options?.sessionId, error);
recordWebSocketFailure(cacheSessionId, error);
if (websocketStarted) {
throw error;
}
recordWebSocketSseFallback(options?.sessionId);
recordWebSocketSseFallback(cacheSessionId);
break;
}
}
@@ -404,9 +419,7 @@ export const stream: StreamFunction<"openai-codex-responses", OpenAICodexRespons
const delayMs =
retryAfterDelayMs === undefined
? BASE_DELAY_MS * 2 ** attempt
: response.status === 429
? capRetryDelayMs(retryAfterDelayMs, options)
: retryAfterDelayMs;
: validateRetryDelayMs(retryAfterDelayMs, options);
await sleep(delayMs, options?.signal);
continue;
@@ -427,7 +440,11 @@ export const stream: StreamFunction<"openai-codex-responses", OpenAICodexRespons
}
lastError = error instanceof Error ? error : new Error(String(error));
// Network errors are retryable
if (attempt < maxRetries && !lastError.message.includes("usage limit")) {
if (
attempt < maxRetries &&
!(lastError instanceof RetryDelayExceededError) &&
!lastError.message.includes("usage limit")
) {
const delayMs = BASE_DELAY_MS * 2 ** attempt;
await sleep(delayMs, options?.signal);
continue;
@@ -448,7 +465,7 @@ export const stream: StreamFunction<"openai-codex-responses", OpenAICodexRespons
startEmitted = true;
stream.push({ type: "start", partial: output });
}
await processStream(response, output, stream, model, options);
await processStream(response, output, stream, model, grammarToolInputProperties, options);
if (options?.signal?.aborted) {
throw new Error("Request was aborted");
@@ -458,8 +475,9 @@ export const stream: StreamFunction<"openai-codex-responses", OpenAICodexRespons
stream.end();
} catch (error) {
for (const block of output.content) {
// partialJson is only a streaming scratch buffer; never persist it.
// Streaming scratch buffers are only used during parsing; never persist them.
delete (block as { partialJson?: string }).partialJson;
delete (block as { customInput?: unknown }).customInput;
}
output.stopReason = options?.signal?.aborted ? "aborted" : "error";
output.errorMessage = formatProviderError(normalizeProviderError(error));
@@ -498,12 +516,25 @@ export const streamSimple: StreamFunction<"openai-codex-responses", SimpleStream
function buildRequestBody(
model: Model<"openai-codex-responses">,
context: Context,
options?: OpenAICodexResponsesOptions,
options: OpenAICodexResponsesOptions | undefined,
cacheSessionId: string | undefined,
grammarToolInputProperties: ReadonlyMap<string, string> = createGrammarToolInputProperties(
context.tools,
model.compat?.supportsOpenAIGrammarTools ?? false,
),
): RequestBody {
const supportsStrictMode = model.compat?.supportsStrictMode ?? true;
const supportsOpenAIGrammarTools = model.compat?.supportsOpenAIGrammarTools ?? false;
const toolPlacement = splitDeferredTools(context, model.compat?.supportsToolSearch ?? false);
const messages = convertResponsesMessages(model, context, CODEX_TOOL_CALL_PROVIDERS, {
includeSystemPrompt: false,
grammarToolInputProperties,
deferredTools: toolPlacement.deferred,
toolOptions: {
strict: null,
supportsStrictMode,
supportsOpenAIGrammarTools,
},
});
const body: RequestBody = {
@@ -514,7 +545,7 @@ function buildRequestBody(
input: messages,
text: { verbosity: options?.textVerbosity || "low" },
include: ["reasoning.encrypted_content"],
prompt_cache_key: clampOpenAIPromptCacheKey(options?.sessionId),
prompt_cache_key: cacheSessionId,
tool_choice: options?.toolChoice ?? "auto",
parallel_tool_calls: true,
};
@@ -528,7 +559,11 @@ function buildRequestBody(
}
if (toolPlacement.immediate.length > 0) {
body.tools = convertResponsesTools(toolPlacement.immediate, { strict: null });
body.tools = convertResponsesTools(toolPlacement.immediate, {
strict: null,
supportsStrictMode,
supportsOpenAIGrammarTools,
});
}
if (options?.reasoningEffort !== undefined) {
@@ -610,10 +645,12 @@ async function processStream(
output: AssistantMessage,
stream: AssistantMessageEventStream,
model: Model<"openai-codex-responses">,
grammarToolInputProperties: ReadonlyMap<string, string>,
options?: OpenAICodexResponsesOptions,
): Promise<void> {
await processResponsesStream(mapCodexEvents(parseSSE(response, options?.signal)), output, stream, model, {
serviceTier: options?.serviceTier,
grammarToolInputProperties,
resolveServiceTier: resolveCodexServiceTier,
applyServiceTierPricing: (usage, serviceTier) => applyServiceTierPricing(usage, serviceTier, model),
});
@@ -1399,12 +1436,14 @@ async function processWebSocketStream(
onStart: () => void,
idleTimeoutMs: number | undefined,
websocketConnectTimeoutMs: number | undefined,
cacheSessionId: string | undefined,
grammarToolInputProperties: ReadonlyMap<string, string>,
options?: OpenAICodexResponsesOptions,
): Promise<void> {
const { socket, entry, reused, release } = await acquireWebSocket(
url,
headers,
options?.sessionId,
cacheSessionId,
options?.signal,
websocketConnectTimeoutMs,
options?.env,
@@ -1415,7 +1454,7 @@ async function processWebSocketStream(
// WebSocket continuation still works via connection-scoped previous_response_id state.
const fullBody = body;
const requestBody = useCachedContext && entry ? buildCachedWebSocketRequestBody(entry, fullBody) : fullBody;
const stats = options?.sessionId ? getOrCreateWebSocketDebugStats(options.sessionId) : undefined;
const stats = cacheSessionId ? getOrCreateWebSocketDebugStats(cacheSessionId) : undefined;
if (stats) {
stats.requests++;
if (reused) stats.connectionsReused++;
@@ -1445,6 +1484,7 @@ async function processWebSocketStream(
model,
{
serviceTier: options?.serviceTier,
grammarToolInputProperties,
resolveServiceTier: resolveCodexServiceTier,
applyServiceTierPricing: (usage, serviceTier) => applyServiceTierPricing(usage, serviceTier, model),
},
@@ -1454,7 +1494,8 @@ async function processWebSocketStream(
} else if (useCachedContext && entry && output.responseId) {
const responseItems = convertResponsesMessages(model, { messages: [output] }, CODEX_TOOL_CALL_PROVIDERS, {
includeSystemPrompt: false,
}).filter((item) => item.type !== "function_call_output");
grammarToolInputProperties,
}).filter((item) => item.type !== "function_call_output" && item.type !== "custom_tool_call_output");
entry.continuation = {
lastRequestBody: fullBody,
lastResponseId: output.responseId,
packages/ai/src/api/openai-completions.ts
+169 -33
View File
@@ -7,6 +7,7 @@ import type {
ChatCompletionContentPartText,
ChatCompletionDeveloperMessageParam,
ChatCompletionMessageParam,
ChatCompletionMessageToolCall,
ChatCompletionSystemMessageParam,
ChatCompletionToolMessageParam,
} from "openai/resources/chat/completions.js";
@@ -38,7 +39,16 @@ import { shortHash } from "../utils/hash.ts";
import { headersToRecord } from "../utils/headers.ts";
import { parseStreamingJson } from "../utils/json-parse.ts";
import { getProviderEnvValue } from "../utils/provider-env.ts";
import { retryProviderRequest } from "../utils/provider-retry.ts";
import { sanitizeSurrogates } from "../utils/sanitize-unicode.ts";
import {
appendGrammarToolInputJsonDelta,
createGrammarToolInputProperties,
type GrammarToolInputJsonBuffer,
getGrammarToolInput,
resolveGrammarConstrainedSampling,
resolveJsonSchemaStrictSampling,
} from "./constrained-sampling.ts";
import { buildCopilotDynamicHeaders, hasCopilotVisionInput } from "./github-copilot-headers.ts";
import { clampOpenAIPromptCacheKey } from "./openai-prompt-cache.ts";
import { buildBaseOptions } from "./simple-options.ts";
@@ -129,10 +139,14 @@ function isEncryptedReasoningDetail(detail: unknown): detail is OpenAIEncryptedR
}
export interface OpenAICompletionsOptions extends StreamOptions {
toolChoice?: "auto" | "none" | "required" | { type: "function"; function: { name: string } };
toolChoice?: OpenAI.Chat.Completions.ChatCompletionToolChoiceOption;
reasoningEffort?: "minimal" | "low" | "medium" | "high" | "xhigh" | "max";
}
export interface ConvertCompletionsMessagesOptions {
grammarToolInputProperties?: ReadonlyMap<string, string>;
}
interface OpenAICompatCacheControl {
type: "ephemeral";
ttl?: string;
@@ -208,10 +222,14 @@ export const stream: StreamFunction<"openai-completions", OpenAICompletionsOptio
try {
const apiKey = getClientApiKey(model.provider, options?.apiKey, options?.headers);
const compat = getCompat(model);
const grammarToolInputProperties = createGrammarToolInputProperties(
context.tools,
compat.supportsOpenAIGrammarTools,
);
const cacheRetention = resolveCacheRetention(options?.cacheRetention, options?.env);
const cacheSessionId = cacheRetention === "none" ? undefined : options?.sessionId;
const client = createClient(model, context, apiKey, options?.headers, cacheSessionId, compat);
let params = buildParams(model, context, options, compat, cacheRetention);
let params = buildParams(model, context, options, compat, cacheRetention, grammarToolInputProperties);
const nextParams = await options?.onPayload?.(params, model);
if (nextParams !== undefined) {
params = nextParams as OpenAI.Chat.Completions.ChatCompletionCreateParamsStreaming;
@@ -219,20 +237,35 @@ export const stream: StreamFunction<"openai-completions", OpenAICompletionsOptio
const requestOptions = {
...(options?.signal ? { signal: options.signal } : {}),
...(options?.timeoutMs !== undefined ? { timeout: options.timeoutMs } : {}),
maxRetries: options?.maxRetries ?? 0,
maxRetries: 0,
};
const { data: openaiStream, response } = await client.chat.completions
.create(params, requestOptions)
.withResponse();
const { data: openaiStream, response } = await retryProviderRequest(
() => client.chat.completions.create(params, requestOptions).withResponse(),
{
maxRetries: options?.maxRetries,
maxRetryDelayMs: options?.maxRetryDelayMs,
signal: options?.signal,
},
);
await options?.onResponse?.({ status: response.status, headers: headersToRecord(response.headers) }, model);
stream.push({ type: "start", partial: output });
interface StreamingToolCallBlock extends ToolCall {
partialArgs?: string;
customInput?: {
property: string;
jsonBuffer: GrammarToolInputJsonBuffer;
};
streamIndex?: number;
}
type StreamingBlock = TextContent | ThinkingContent | StreamingToolCallBlock;
type StreamingToolCallDelta = NonNullable<ChatCompletionChunk.Choice.Delta["tool_calls"]>[number];
type StreamingToolCallDelta = {
index?: number;
id?: string;
type?: string;
function?: { name?: string; arguments?: string };
custom?: { name?: string; input?: string };
};
let textBlock: TextContent | null = null;
let thinkingBlock: ThinkingContent | null = null;
@@ -242,6 +275,28 @@ export const stream: StreamFunction<"openai-completions", OpenAICompletionsOptio
const pendingReasoningDetailsByToolCallId = new Map<string, string>();
const blocks = output.content as StreamingBlock[];
const getContentIndex = (block: StreamingBlock) => blocks.indexOf(block);
const getCustomToolCallInput = (block: StreamingToolCallBlock): string => {
const property = block.customInput?.property;
if (property === undefined) return "";
const value = block.arguments[property];
return typeof value === "string" ? value : "";
};
const appendCustomToolCallInput = (
block: StreamingToolCallBlock,
nextInput: string,
close: boolean,
): string | undefined => {
const customInput = block.customInput;
if (!customInput) return undefined;
const delta = appendGrammarToolInputJsonDelta(
customInput.jsonBuffer,
customInput.property,
nextInput,
close,
);
block.arguments = { [customInput.property]: nextInput };
return delta;
};
const finishBlock = (block: StreamingBlock) => {
const contentIndex = getContentIndex(block);
if (contentIndex === -1) {
@@ -262,10 +317,23 @@ export const stream: StreamFunction<"openai-completions", OpenAICompletionsOptio
partial: output,
});
} else if (block.type === "toolCall") {
block.arguments = parseStreamingJson(block.partialArgs);
if (block.customInput) {
const delta = appendCustomToolCallInput(block, getCustomToolCallInput(block), true);
if (delta !== undefined) {
stream.push({
type: "toolcall_delta",
contentIndex,
delta,
partial: output,
});
}
} else {
block.arguments = parseStreamingJson(block.partialArgs);
}
// Finalize in-place and strip the scratch buffers so replay only
// carries parsed arguments.
delete block.partialArgs;
delete block.customInput;
delete block.streamIndex;
stream.push({
type: "toolcall_end",
@@ -307,17 +375,27 @@ export const stream: StreamFunction<"openai-completions", OpenAICompletionsOptio
};
const ensureToolCallBlock = (toolCall: StreamingToolCallDelta) => {
const streamIndex = typeof toolCall.index === "number" ? toolCall.index : undefined;
const name = toolCall.function?.name ?? toolCall.custom?.name ?? "";
let block = streamIndex !== undefined ? toolCallBlocksByIndex.get(streamIndex) : undefined;
if (!block && toolCall.id) {
block = toolCallBlocksById.get(toolCall.id);
}
if (!block) {
// Note: the "input" fallback here should/must not be taken. in case the LLM makes up
// a tool we don't knwo about, we at least have a place to stash our stuff.
const customInputProperty = toolCall.custom
? (grammarToolInputProperties.get(name) ?? "input")
: undefined;
const hasCustomInput = customInputProperty !== undefined;
block = {
type: "toolCall",
id: toolCall.id || "",
name: toolCall.function?.name || "",
arguments: {},
partialArgs: "",
name,
arguments: hasCustomInput ? { [customInputProperty]: "" } : {},
partialArgs: hasCustomInput ? undefined : "",
customInput: hasCustomInput
? { property: customInputProperty, jsonBuffer: { input: "", started: false, closed: false } }
: undefined,
streamIndex,
};
if (streamIndex !== undefined) {
@@ -340,6 +418,18 @@ export const stream: StreamFunction<"openai-completions", OpenAICompletionsOptio
if (toolCall.id) {
toolCallBlocksById.set(toolCall.id, block);
}
if (!block.name && name) {
block.name = name;
}
if (toolCall.custom && !block.customInput) {
const customInputProperty = grammarToolInputProperties.get(block.name) ?? "input";
block.arguments = { [customInputProperty]: "" };
block.customInput = {
property: customInputProperty,
jsonBuffer: { input: "", started: false, closed: false },
};
delete block.partialArgs;
}
applyPendingReasoningDetail(block);
return block;
};
@@ -425,14 +515,15 @@ export const stream: StreamFunction<"openai-completions", OpenAICompletionsOptio
}
if (choice?.delta?.tool_calls) {
for (const toolCall of choice.delta.tool_calls) {
for (const toolCall of choice.delta.tool_calls as StreamingToolCallDelta[]) {
const block = ensureToolCallBlock(toolCall);
if (!block.id && toolCall.id) {
block.id = toolCall.id;
toolCallBlocksById.set(toolCall.id, block);
}
if (!block.name && toolCall.function?.name) {
block.name = toolCall.function.name;
const name = toolCall.function?.name ?? toolCall.custom?.name;
if (!block.name && name) {
block.name = name;
}
let delta = "";
@@ -440,6 +531,9 @@ export const stream: StreamFunction<"openai-completions", OpenAICompletionsOptio
delta = toolCall.function.arguments;
block.partialArgs = (block.partialArgs ?? "") + toolCall.function.arguments;
block.arguments = parseStreamingJson(block.partialArgs);
} else if (toolCall.custom?.input) {
const nextInput = getCustomToolCallInput(block) + toolCall.custom.input;
delta = appendCustomToolCallInput(block, nextInput, false) ?? "";
}
stream.push({
type: "toolcall_delta",
@@ -491,6 +585,7 @@ export const stream: StreamFunction<"openai-completions", OpenAICompletionsOptio
delete (block as { index?: number }).index;
// Streaming scratch buffers are only used during parsing; never persist them.
delete (block as { partialArgs?: string }).partialArgs;
delete (block as { customInput?: unknown }).customInput;
delete (block as { streamIndex?: number }).streamIndex;
}
output.stopReason = options?.signal?.aborted ? "aborted" : "error";
@@ -579,8 +674,12 @@ function buildParams(
options?: OpenAICompletionsOptions,
compat: ResolvedOpenAICompletionsCompat = getCompat(model),
cacheRetention: CacheRetention = resolveCacheRetention(options?.cacheRetention, options?.env),
grammarToolInputProperties: ReadonlyMap<string, string> = createGrammarToolInputProperties(
context.tools,
compat.supportsOpenAIGrammarTools,
),
) {
const messages = convertMessages(model, context, compat);
const messages = convertMessages(model, context, compat, { grammarToolInputProperties });
const cacheControl = getCompatCacheControl(compat, cacheRetention);
const params: OpenAI.Chat.Completions.ChatCompletionCreateParamsStreaming = {
@@ -889,6 +988,7 @@ export function convertMessages(
model: Model<"openai-completions">,
context: Context,
compat: ResolvedOpenAICompletionsCompat,
options?: ConvertCompletionsMessagesOptions,
): ChatCompletionMessageParam[] {
const params: ChatCompletionMessageParam[] = [];
@@ -1026,14 +1126,27 @@ export function convertMessages(
const toolCalls = msg.content.filter(isToolCallBlock);
if (toolCalls.length > 0) {
assistantMsg.tool_calls = toolCalls.map((tc) => ({
id: tc.id,
type: "function" as const,
function: {
name: tc.name,
arguments: JSON.stringify(tc.arguments),
},
}));
assistantMsg.tool_calls = toolCalls.map((tc): ChatCompletionMessageToolCall => {
const customInputProperty = options?.grammarToolInputProperties?.get(tc.name);
if (customInputProperty !== undefined) {
return {
id: tc.id,
type: "custom",
custom: {
name: tc.name,
input: sanitizeSurrogates(getGrammarToolInput(tc.name, tc.arguments, customInputProperty)),
},
};
}
return {
id: tc.id,
type: "function",
function: {
name: tc.name,
arguments: JSON.stringify(tc.arguments),
},
};
});
const reasoningDetails = toolCalls
.filter((tc) => tc.thoughtSignature)
.map((tc) => {
@@ -1166,16 +1279,37 @@ function convertTools(
tools: Tool[],
compat: ResolvedOpenAICompletionsCompat,
): OpenAI.Chat.Completions.ChatCompletionTool[] {
return tools.map((tool) => ({
type: "function",
function: {
name: tool.name,
description: tool.description,
parameters: tool.parameters as any, // TypeBox already generates JSON Schema
// Only include strict if provider supports it. Some reject unknown fields.
...(compat.supportsStrictMode !== false && { strict: false }),
},
}));
return tools.map((tool) => {
const grammar = resolveGrammarConstrainedSampling(tool, compat.supportsOpenAIGrammarTools);
if (grammar) {
return {
type: "custom",
custom: {
name: tool.name,
description: tool.description,
format: {
type: "grammar",
grammar: {
syntax: grammar.format,
definition: grammar.definition,
},
},
},
};
}
const strict = resolveJsonSchemaStrictSampling(tool, compat.supportsStrictMode !== false);
return {
type: "function",
function: {
name: tool.name,
description: tool.description,
parameters: tool.parameters as Record<string, unknown>, // TypeBox already generates JSON Schema
// Only include strict if provider supports it. Some reject unknown fields.
...(compat.supportsStrictMode !== false && { strict: strict ?? false }),
},
};
});
}
function parseChunkUsage(
@@ -1318,6 +1452,7 @@ function detectCompat(model: Model<"openai-completions">): ResolvedOpenAIComplet
chatTemplateKwargs: {},
zaiToolStream: false,
supportsStrictMode: !isMoonshot && !isTogether && !isCloudflareAiGateway && !isNvidia,
supportsOpenAIGrammarTools: false,
cacheControlFormat,
sendSessionAffinityHeaders: false,
deferredToolsMode: undefined,
@@ -1359,6 +1494,7 @@ function getCompat(model: Model<"openai-completions">): ResolvedOpenAICompletion
chatTemplateKwargs: model.compat.chatTemplateKwargs ?? detected.chatTemplateKwargs,
zaiToolStream: model.compat.zaiToolStream ?? detected.zaiToolStream,
supportsStrictMode: model.compat.supportsStrictMode ?? detected.supportsStrictMode,
supportsOpenAIGrammarTools: model.compat.supportsOpenAIGrammarTools ?? detected.supportsOpenAIGrammarTools,
cacheControlFormat: model.compat.cacheControlFormat ?? detected.cacheControlFormat,
sendSessionAffinityHeaders: model.compat.sendSessionAffinityHeaders ?? detected.sendSessionAffinityHeaders,
deferredToolsMode: model.compat.deferredToolsMode ?? detected.deferredToolsMode,
packages/ai/src/api/openai-responses-shared.ts
+205 -72
View File
@@ -2,7 +2,6 @@ import type OpenAI from "openai";
import type {
Tool as OpenAITool,
ResponseCreateParamsStreaming,
ResponseFunctionCallOutputItemList,
ResponseInput,
ResponseInputContent,
ResponseInputImage,
@@ -33,6 +32,13 @@ import type { AssistantMessageEventStream } from "../utils/event-stream.ts";
import { shortHash } from "../utils/hash.ts";
import { parseStreamingJson } from "../utils/json-parse.ts";
import { sanitizeSurrogates } from "../utils/sanitize-unicode.ts";
import {
appendGrammarToolInputJsonDelta,
type GrammarToolInputJsonBuffer,
getGrammarToolInput,
resolveGrammarConstrainedSampling,
resolveJsonSchemaStrictSampling,
} from "./constrained-sampling.ts";
import { transformMessages } from "./transform-messages.ts";
// =============================================================================
@@ -65,8 +71,40 @@ function parseTextSignature(
return { id: signature };
}
type ToolResultOutputContent = Array<ResponseInputText | ResponseInputImage>;
function convertToolResultOutput<TApi extends Api>(
model: Model<TApi>,
content: readonly (TextContent | ImageContent)[],
): string | ToolResultOutputContent {
const textResult = content
.filter((c): c is TextContent => c.type === "text")
.map((c) => c.text)
.join("\n");
const images = content.filter((c): c is ImageContent => c.type === "image");
const hasText = textResult.length > 0;
if (images.length === 0 || !model.input.includes("image")) {
return sanitizeSurrogates(hasText ? textResult : images.length > 0 ? "(see attached image)" : "(no tool output)");
}
const output: ToolResultOutputContent = [];
if (hasText) {
output.push({ type: "input_text", text: sanitizeSurrogates(textResult) });
}
for (const image of images) {
output.push({
type: "input_image",
detail: "auto",
image_url: `data:${image.mimeType};base64,${image.data}`,
});
}
return output;
}
export interface OpenAIResponsesStreamOptions {
serviceTier?: ResponseCreateParamsStreaming["service_tier"];
grammarToolInputProperties?: ReadonlyMap<string, string>;
resolveServiceTier?: (
responseServiceTier: ResponseCreateParamsStreaming["service_tier"] | undefined,
requestServiceTier: ResponseCreateParamsStreaming["service_tier"] | undefined,
@@ -79,16 +117,18 @@ export interface OpenAIResponsesStreamOptions {
export interface ConvertResponsesMessagesOptions {
includeSystemPrompt?: boolean;
grammarToolInputProperties?: ReadonlyMap<string, string>;
deferredTools?: ReadonlyMap<string, Tool>;
toolOptions?: ConvertResponsesToolsOptions;
}
export interface ConvertResponsesToolsOptions {
strict?: boolean | null;
supportsStrictMode?: boolean;
supportsOpenAIGrammarTools?: boolean;
deferLoading?: boolean;
}
type OpenAIFunctionTool = Extract<OpenAITool, { type: "function" }>;
// =============================================================================
// Message conversion
// =============================================================================
@@ -206,67 +246,62 @@ export function convertResponsesMessages<TApi extends Api>(
} else if (block.type === "toolCall") {
const toolCall = block as ToolCall;
const [callId, itemIdRaw] = toolCall.id.split("|");
const customInputProperty = options?.grammarToolInputProperties?.get(toolCall.name);
let itemId: string | undefined = itemIdRaw;
// For different-model messages, set id to undefined to avoid pairing validation.
// OpenAI tracks which fc_xxx IDs were paired with rs_xxx reasoning items.
// By omitting the id, we avoid triggering that validation (like cross-provider does).
if (isDifferentModel && itemId?.startsWith("fc_")) {
// When replaying custom-tool calls as a function_call, also drop non-fc_* ids such as
// ctc_* custom-tool ids because function_call item ids must be fc_*.
if (
(isDifferentModel && itemId?.startsWith("fc_")) ||
(customInputProperty === undefined && !itemId?.startsWith("fc_"))
) {
itemId = undefined;
}
output.push({
type: "function_call",
id: itemId,
call_id: callId,
name: toolCall.name,
arguments: JSON.stringify(toolCall.arguments),
});
if (customInputProperty !== undefined) {
output.push({
type: "custom_tool_call",
id: itemId,
call_id: callId,
name: toolCall.name,
input: sanitizeSurrogates(
getGrammarToolInput(toolCall.name, toolCall.arguments, customInputProperty),
),
} satisfies ResponseOutputItem);
} else {
output.push({
type: "function_call",
id: itemId,
call_id: callId,
name: toolCall.name,
arguments: JSON.stringify(toolCall.arguments),
});
}
}
}
if (output.length === 0) continue;
messages.push(...output);
} else if (msg.role === "toolResult") {
const textResult = msg.content
.filter((c): c is TextContent => c.type === "text")
.map((c) => c.text)
.join("\n");
const hasImages = msg.content.some((c): c is ImageContent => c.type === "image");
const hasText = textResult.length > 0;
const [callId] = msg.toolCallId.split("|");
const output = convertToolResultOutput(model, msg.content);
let output: string | ResponseFunctionCallOutputItemList;
if (hasImages && model.input.includes("image")) {
const contentParts: ResponseFunctionCallOutputItemList = [];
if (hasText) {
contentParts.push({
type: "input_text",
text: sanitizeSurrogates(textResult),
});
}
for (const block of msg.content) {
if (block.type === "image") {
contentParts.push({
type: "input_image",
detail: "auto",
image_url: `data:${block.mimeType};base64,${block.data}`,
});
}
}
output = contentParts;
if (options?.grammarToolInputProperties?.has(msg.toolName)) {
messages.push({
type: "custom_tool_call_output",
call_id: callId,
output,
});
} else {
output = sanitizeSurrogates(hasText ? textResult : hasImages ? "(see attached image)" : "(no tool output)");
messages.push({
type: "function_call_output",
call_id: callId,
output,
});
}
messages.push({
type: "function_call_output",
call_id: callId,
output,
});
const deferredTools: Tool[] = [];
for (const name of msg.addedToolNames ?? []) {
const tool = options?.deferredTools?.get(name);
@@ -289,7 +324,10 @@ export function convertResponsesMessages<TApi extends Api>(
call_id: searchCallId,
execution: "client",
status: "completed",
tools: convertResponsesTools(deferredTools, { deferLoading: true }),
tools: convertResponsesTools(deferredTools, {
...options?.toolOptions,
deferLoading: true,
}),
} satisfies ResponseToolSearchOutputItemParam);
}
}
@@ -304,30 +342,77 @@ export function convertResponsesMessages<TApi extends Api>(
// =============================================================================
export function convertResponsesTools(tools: readonly Tool[], options?: ConvertResponsesToolsOptions): OpenAITool[] {
const strict = options?.strict === undefined ? false : options.strict;
return tools.map(
(tool): OpenAIFunctionTool => ({
const defaultStrict = options?.strict === undefined ? false : options.strict;
const supportsStrictMode = options?.supportsStrictMode ?? true;
const supportsOpenAIGrammarTools = options?.supportsOpenAIGrammarTools ?? false;
return tools.map((tool) => {
const grammar = resolveGrammarConstrainedSampling(tool, supportsOpenAIGrammarTools);
if (grammar) {
return {
type: "custom",
name: tool.name,
description: tool.description,
format: {
type: "grammar",
syntax: grammar.format,
definition: grammar.definition,
},
...(options?.deferLoading ? { defer_loading: true } : {}),
} satisfies OpenAITool;
}
const constrainedStrict = resolveJsonSchemaStrictSampling(tool, supportsStrictMode);
const functionTool: Omit<Extract<OpenAITool, { type: "function" }>, "strict"> & {
strict?: Extract<OpenAITool, { type: "function" }>["strict"];
} = {
type: "function",
name: tool.name,
description: tool.description,
parameters: tool.parameters as Record<string, unknown>, // TypeBox already generates JSON Schema
strict,
...(options?.deferLoading ? { defer_loading: true } : {}),
}),
);
};
if (supportsStrictMode) {
functionTool.strict = constrainedStrict ?? defaultStrict;
}
return functionTool as OpenAITool;
});
}
// =============================================================================
// Stream processing
// =============================================================================
type StreamingToolCall = ToolCall & { partialJson: string };
type StreamingToolCall = ToolCall & {
partialJson?: string;
customInput?: {
property: string;
jsonBuffer: GrammarToolInputJsonBuffer;
};
};
function getCustomToolCallInput(block: StreamingToolCall): string {
const property = block.customInput?.property;
if (property === undefined) return "";
const value = block.arguments[property];
return typeof value === "string" ? value : "";
}
function appendCustomToolCallInput(block: StreamingToolCall, nextInput: string, close: boolean): string | undefined {
const customInput = block.customInput;
if (!customInput) return undefined;
const delta = appendGrammarToolInputJsonDelta(customInput.jsonBuffer, customInput.property, nextInput, close);
block.arguments = { [customInput.property]: nextInput };
return delta;
}
type ResponsesOutputSlot =
| { type: "thinking"; block: ThinkingContent; contentIndex: number }
| { type: "text"; block: TextContent; contentIndex: number }
| { type: "toolCall"; block: StreamingToolCall; contentIndex: number };
type ToolCallOutputSlot = Extract<ResponsesOutputSlot, { type: "toolCall" }>;
export async function processResponsesStream<TApi extends Api>(
openaiStream: AsyncIterable<ResponseStreamEvent>,
output: AssistantMessage,
@@ -345,6 +430,15 @@ export async function processResponsesStream<TApi extends Api>(
const slot = outputSlots.get(outputIndex);
return slot?.type === type ? (slot as Extract<ResponsesOutputSlot, { type: TType }>) : undefined;
};
const pushToolCallDelta = (slot: ToolCallOutputSlot, delta: string | undefined): void => {
if (delta === undefined) return;
stream.push({
type: "toolcall_delta",
contentIndex: slot.contentIndex,
delta,
partial: output,
});
};
const createSlot = (outputIndex: number, item: ResponseOutputItem): ResponsesOutputSlot | undefined => {
if (item.type === "reasoning") {
const block: ThinkingContent = { type: "thinking", thinking: "" };
@@ -384,6 +478,29 @@ export async function processResponsesStream<TApi extends Api>(
stream.push({ type: "toolcall_start", contentIndex: slot.contentIndex, partial: output });
return slot;
}
if (item.type === "custom_tool_call") {
const inputProperty = options?.grammarToolInputProperties?.get(item.name) ?? "input";
const input = item.input || "";
const block: StreamingToolCall = {
type: "toolCall",
id: `${item.call_id}|${item.id}`,
name: item.name,
arguments: { [inputProperty]: input },
customInput: {
property: inputProperty,
jsonBuffer: { input: "", started: false, closed: false },
},
};
output.content.push(block);
const slot = {
type: "toolCall",
block,
contentIndex: output.content.length - 1,
} satisfies ResponsesOutputSlot;
outputSlots.set(outputIndex, slot);
stream.push({ type: "toolcall_start", contentIndex: slot.contentIndex, partial: output });
return slot;
}
return undefined;
};
const getOrCreateSlot = (outputIndex: number, item: ResponseOutputItem): ResponsesOutputSlot | undefined => {
@@ -503,33 +620,32 @@ export async function processResponsesStream<TApi extends Api>(
});
} else if (event.type === "response.function_call_arguments.delta") {
const slot = getSlot(event.output_index, "toolCall");
if (!slot) continue;
if (!slot || slot.block.partialJson === undefined) continue;
slot.block.partialJson += event.delta;
slot.block.arguments = parseStreamingJson(slot.block.partialJson);
stream.push({
type: "toolcall_delta",
contentIndex: slot.contentIndex,
delta: event.delta,
partial: output,
});
pushToolCallDelta(slot, event.delta);
} else if (event.type === "response.function_call_arguments.done") {
const slot = getSlot(event.output_index, "toolCall");
if (!slot) continue;
if (!slot || slot.block.partialJson === undefined) continue;
const previousPartialJson = slot.block.partialJson;
slot.block.partialJson = event.arguments;
slot.block.arguments = parseStreamingJson(slot.block.partialJson);
if (event.arguments.startsWith(previousPartialJson)) {
const delta = event.arguments.slice(previousPartialJson.length);
if (delta.length > 0) {
stream.push({
type: "toolcall_delta",
contentIndex: slot.contentIndex,
delta,
partial: output,
});
}
if (delta.length > 0) pushToolCallDelta(slot, delta);
}
} else if (event.type === "response.custom_tool_call_input.delta") {
const slot = getSlot(event.output_index, "toolCall");
if (!slot || !slot.block.customInput) continue;
pushToolCallDelta(
slot,
appendCustomToolCallInput(slot.block, getCustomToolCallInput(slot.block) + event.delta, false),
);
} else if (event.type === "response.custom_tool_call_input.done") {
const slot = getSlot(event.output_index, "toolCall");
if (!slot || !slot.block.customInput) continue;
pushToolCallDelta(slot, appendCustomToolCallInput(slot.block, event.input, true));
} else if (event.type === "response.output_item.done") {
const item = event.item;
const slot = getOrCreateSlot(event.output_index, item);
@@ -557,11 +673,28 @@ export async function processResponsesStream<TApi extends Api>(
partial: output,
});
outputSlots.delete(event.output_index);
} else if (item.type === "function_call" && slot?.type === "toolCall") {
} else if (
item.type === "function_call" &&
slot?.type === "toolCall" &&
slot.block.partialJson !== undefined
) {
slot.block.arguments = parseStreamingJson(item.arguments || slot.block.partialJson || "{}");
// Finalize in-place and strip the scratch buffer so replay only
// carries parsed arguments.
delete (slot.block as { partialJson?: string }).partialJson;
delete slot.block.partialJson;
stream.push({
type: "toolcall_end",
contentIndex: slot.contentIndex,
toolCall: slot.block,
partial: output,
});
outputSlots.delete(event.output_index);
} else if (item.type === "custom_tool_call" && slot?.type === "toolCall" && slot.block.customInput) {
pushToolCallDelta(
slot,
appendCustomToolCallInput(slot.block, item.input ?? getCustomToolCallInput(slot.block), true),
);
delete slot.block.customInput;
stream.push({
type: "toolcall_end",
contentIndex: slot.contentIndex,
packages/ai/src/api/openai-responses.ts
+45 -8
View File
@@ -20,6 +20,8 @@ import { formatProviderError, normalizeProviderError } from "../utils/error-body
import { AssistantMessageEventStream } from "../utils/event-stream.ts";
import { headersToRecord } from "../utils/headers.ts";
import { getProviderEnvValue } from "../utils/provider-env.ts";
import { retryProviderRequest } from "../utils/provider-retry.ts";
import { createGrammarToolInputProperties } from "./constrained-sampling.ts";
import { buildCopilotDynamicHeaders, hasCopilotVisionInput } from "./github-copilot-headers.ts";
import { clampOpenAIPromptCacheKey } from "./openai-prompt-cache.ts";
import { convertResponsesMessages, convertResponsesTools, processResponsesStream } from "./openai-responses-shared.ts";
@@ -67,7 +69,10 @@ function getCompat(model: Model<"openai-responses">): Required<OpenAIResponsesCo
supportsDeveloperRole: model.compat?.supportsDeveloperRole ?? true,
sessionAffinityFormat: model.compat?.sessionAffinityFormat ?? detectSessionAffinityFormat(model),
supportsLongCacheRetention: model.compat?.supportsLongCacheRetention ?? true,
supportsStrictMode: model.compat?.supportsStrictMode ?? false,
supportsOpenAIGrammarTools: model.compat?.supportsOpenAIGrammarTools ?? false,
supportsToolSearch: model.compat?.supportsToolSearch ?? false,
supportsExplicitPromptCacheMode: model.compat?.supportsExplicitPromptCacheMode ?? false,
};
}
@@ -125,8 +130,13 @@ export const stream: StreamFunction<"openai-responses", OpenAIResponsesOptions>
const apiKey = getClientApiKey(model.provider, options?.apiKey, options?.headers);
const cacheRetention = resolveCacheRetention(options?.cacheRetention, options?.env);
const cacheSessionId = cacheRetention === "none" ? undefined : options?.sessionId;
const compat = getCompat(model);
const grammarToolInputProperties = createGrammarToolInputProperties(
context.tools,
compat.supportsOpenAIGrammarTools,
);
const client = createClient(model, context, apiKey, options?.headers, cacheSessionId);
let params = buildParams(model, context, options);
let params = buildParams(model, context, options, compat, grammarToolInputProperties);
const nextParams = await options?.onPayload?.(params, model);
if (nextParams !== undefined) {
params = nextParams as ResponseCreateParamsStreaming;
@@ -134,14 +144,22 @@ export const stream: StreamFunction<"openai-responses", OpenAIResponsesOptions>
const requestOptions = {
...(options?.signal ? { signal: options.signal } : {}),
...(options?.timeoutMs !== undefined ? { timeout: options.timeoutMs } : {}),
maxRetries: options?.maxRetries ?? 0,
maxRetries: 0,
};
const { data: openaiStream, response } = await client.responses.create(params, requestOptions).withResponse();
const { data: openaiStream, response } = await retryProviderRequest(
() => client.responses.create(params, requestOptions).withResponse(),
{
maxRetries: options?.maxRetries,
maxRetryDelayMs: options?.maxRetryDelayMs,
signal: options?.signal,
},
);
await options?.onResponse?.({ status: response.status, headers: headersToRecord(response.headers) }, model);
stream.push({ type: "start", partial: output });
await processResponsesStream(openaiStream, output, stream, model, {
serviceTier: options?.serviceTier,
grammarToolInputProperties,
applyServiceTierPricing: (usage, serviceTier) => applyServiceTierPricing(usage, serviceTier, model),
});
@@ -158,8 +176,9 @@ export const stream: StreamFunction<"openai-responses", OpenAIResponsesOptions>
} catch (error) {
for (const block of output.content) {
delete (block as { index?: number }).index;
// partialJson is only a streaming scratch buffer; never persist it.
// Streaming scratch buffers are only used during parsing; never persist them.
delete (block as { partialJson?: string }).partialJson;
delete (block as { customInput?: unknown }).customInput;
}
output.stopReason = options?.signal?.aborted ? "aborted" : "error";
output.errorMessage = formatOpenAIResponsesError(error);
@@ -230,20 +249,35 @@ function createClient(
});
}
function buildParams(model: Model<"openai-responses">, context: Context, options?: OpenAIResponsesOptions) {
const compat = getCompat(model);
function buildParams(
model: Model<"openai-responses">,
context: Context,
options: OpenAIResponsesOptions | undefined,
compat: Required<OpenAIResponsesCompat> = getCompat(model),
grammarToolInputProperties: ReadonlyMap<string, string> = createGrammarToolInputProperties(
context.tools,
compat.supportsOpenAIGrammarTools,
),
) {
const toolPlacement = splitDeferredTools(context, compat.supportsToolSearch);
const messages = convertResponsesMessages(model, context, OPENAI_TOOL_CALL_PROVIDERS, {
grammarToolInputProperties,
deferredTools: toolPlacement.deferred,
toolOptions: {
supportsStrictMode: compat.supportsStrictMode,
supportsOpenAIGrammarTools: compat.supportsOpenAIGrammarTools,
},
});
const cacheRetention = resolveCacheRetention(options?.cacheRetention, options?.env);
const params: ResponseCreateParamsStreaming = {
const disableImplicitPromptCache = cacheRetention === "none" && compat.supportsExplicitPromptCacheMode;
const params: ResponseCreateParamsStreaming & { prompt_cache_options?: { mode: "explicit" } } = {
model: model.id,
input: messages,
stream: true,
prompt_cache_key: cacheRetention === "none" ? undefined : clampOpenAIPromptCacheKey(options?.sessionId),
prompt_cache_retention: getPromptCacheRetention(compat, cacheRetention),
prompt_cache_options: disableImplicitPromptCache ? { mode: "explicit" } : undefined,
store: false,
};
@@ -260,7 +294,10 @@ function buildParams(model: Model<"openai-responses">, context: Context, options
}
if (toolPlacement.immediate.length > 0) {
params.tools = convertResponsesTools(toolPlacement.immediate);
params.tools = convertResponsesTools(toolPlacement.immediate, {
supportsStrictMode: compat.supportsStrictMode,
supportsOpenAIGrammarTools: compat.supportsOpenAIGrammarTools,
});
}
if (options?.toolChoice !== undefined) {
packages/ai/src/api/openrouter-images.ts
+13 -4
View File
@@ -18,6 +18,7 @@ import type {
} from "../types.ts";
import { formatProviderError, normalizeProviderError } from "../utils/error-body.ts";
import { headersToRecord, providerHeadersToRecord } from "../utils/headers.ts";
import { retryProviderRequest } from "../utils/provider-retry.ts";
import { sanitizeSurrogates } from "../utils/sanitize-unicode.ts";
interface OpenRouterGeneratedImage {
@@ -64,11 +65,19 @@ export const generateImages: ImagesFunction<"openrouter-images", ImagesOptions>
const requestOptions = {
...(options?.signal ? { signal: options.signal } : {}),
...(options?.timeoutMs !== undefined ? { timeout: options.timeoutMs } : {}),
maxRetries: options?.maxRetries ?? 0,
maxRetries: 0,
};
const { data: response, response: rawResponse } = await client.chat.completions
.create(params as unknown as ChatCompletionCreateParamsNonStreaming, requestOptions)
.withResponse();
const { data: response, response: rawResponse } = await retryProviderRequest(
() =>
client.chat.completions
.create(params as unknown as ChatCompletionCreateParamsNonStreaming, requestOptions)
.withResponse(),
{
maxRetries: options?.maxRetries,
maxRetryDelayMs: options?.maxRetryDelayMs,
signal: options?.signal,
},
);
await options?.onResponse?.({ status: rawResponse.status, headers: headersToRecord(rawResponse.headers) }, model);
const imageResponse = response as OpenRouterImageGenerationResponse;
packages/ai/src/auth/oauth/load.ts
+1 -1
View File
@@ -47,7 +47,7 @@ export const loadOpenRouterOAuth = async (): Promise<OAuthAuth> => {
if (bundledLoaders) return bundledLoaders.openrouter();
return ((await importOAuthModule("./openrouter.ts")) as { openRouterOAuth: OAuthAuth }).openRouterOAuth;
};
export const loadKimiCodingOAuth = async (): Promise<OAuthAuth> => {
if (bundledLoaders) return bundledLoaders.kimiCoding();
return ((await importOAuthModule("./kimi-coding.ts")) as { kimiCodingOAuth: OAuthAuth }).kimiCodingOAuth;
packages/ai/src/auth/oauth/radius.ts
+44 -46
View File
@@ -1,8 +1,9 @@
/**
* Radius gateway OAuth flow.
*
* Radius is a pi-messages gateway. OAuth endpoints are discovered from the
* gateway (`/v1/oauth`); model catalog loading is owned by the Radius provider.
* Radius is a pi-messages gateway. OAuth client APIs live on the configured
* gateway; only the interactive browser authorization endpoint is discovered.
* Model catalog loading is owned by the Radius provider.
*
* NOTE: This module uses node:http for the OAuth callback server.
* It is only intended for CLI use, not browser environments.
@@ -29,29 +30,23 @@ const REDIRECT_URI = `http://${CALLBACK_HOST}:${CALLBACK_PORT}${CALLBACK_PATH}`;
const TOKEN_EXPIRY_SKEW_MS = 60_000;
const LOGIN_METHOD_BROWSER = "browser";
const LOGIN_METHOD_DEVICE_CODE = "device-code";
const OAUTH_CLIENT_ID = "pi-gateway";
const OAUTH_SCOPE = "gateway offline_access";
const OAUTH_DEVICE_CODE_GRANT_TYPE = "urn:ietf:params:oauth:grant-type:device_code";
type RadiusOAuthConfig = {
issuer: string;
type RadiusOAuthDiscovery = {
authorizationEndpoint: string;
tokenEndpoint: string;
deviceAuthorizationEndpoint: string;
deviceAuthorizationEventsEndpoint: string;
verificationEndpoint: string;
clientId: string;
scope: string;
deviceCodeGrantType: string;
};
type DeviceAuthorizationResponse = {
device_code: string;
user_code: string;
verification_uri?: string;
verification_uri_complete?: string;
verification_uri: string;
expires_in: number;
interval?: number;
};
async function loadRadiusOAuthConfig(gateway: string): Promise<RadiusOAuthConfig> {
async function loadRadiusOAuthDiscovery(gateway: string): Promise<RadiusOAuthDiscovery> {
const response = await fetch(new URL("/v1/oauth", gateway), {
headers: { accept: "application/json" },
});
@@ -62,7 +57,11 @@ async function loadRadiusOAuthConfig(gateway: string): Promise<RadiusOAuthConfig
);
}
return (await response.json()) as RadiusOAuthConfig;
const discovery = (await response.json()) as Partial<RadiusOAuthDiscovery>;
if (typeof discovery.authorizationEndpoint !== "string") {
throw new Error(`Invalid Radius OAuth config from ${gateway}`);
}
return { authorizationEndpoint: discovery.authorizationEndpoint };
}
class OAuthResponseError extends Error {
@@ -100,13 +99,13 @@ async function readOAuthResponseError(response: Response, message: string): Prom
}
async function requestOAuthToken(
oauth: RadiusOAuthConfig,
gateway: string,
body: URLSearchParams,
signal?: AbortSignal,
): Promise<OAuthCredential> {
let response: Response;
try {
response = await fetch(oauth.tokenEndpoint, {
response = await fetch(new URL("/v1/oauth/token", gateway), {
method: "POST",
headers: { accept: "application/json", "content-type": "application/x-www-form-urlencoded" },
body,
@@ -220,15 +219,19 @@ function startOAuthCallbackServer(
});
}
async function loginWithBrowser(oauth: RadiusOAuthConfig, interaction: AuthInteraction): Promise<OAuthCredential> {
async function loginWithBrowser(
gateway: string,
authorizationEndpoint: string,
interaction: AuthInteraction,
): Promise<OAuthCredential> {
const { verifier, challenge } = await generatePKCE();
const state = crypto.randomUUID();
const authorizeUrl = new URL(oauth.authorizationEndpoint);
const authorizeUrl = new URL(authorizationEndpoint);
authorizeUrl.search = new URLSearchParams({
response_type: "code",
client_id: oauth.clientId,
client_id: OAUTH_CLIENT_ID,
redirect_uri: REDIRECT_URI,
scope: oauth.scope,
scope: OAUTH_SCOPE,
code_challenge: challenge,
code_challenge_method: "S256",
handoff: "url",
@@ -252,10 +255,10 @@ async function loginWithBrowser(oauth: RadiusOAuthConfig, interaction: AuthInter
throw new Error("OAuth callback did not complete.");
}
return await requestOAuthToken(
oauth,
gateway,
new URLSearchParams({
grant_type: "authorization_code",
client_id: oauth.clientId,
client_id: OAUTH_CLIENT_ID,
redirect_uri: REDIRECT_URI,
code,
code_verifier: verifier,
@@ -268,15 +271,15 @@ async function loginWithBrowser(oauth: RadiusOAuthConfig, interaction: AuthInter
}
async function requestDeviceAuthorization(
oauth: RadiusOAuthConfig,
gateway: string,
signal: AbortSignal | undefined,
): Promise<DeviceAuthorizationResponse> {
let response: Response;
try {
response = await fetch(oauth.deviceAuthorizationEndpoint, {
response = await fetch(new URL("/v1/oauth/device", gateway), {
method: "POST",
headers: { accept: "application/json", "content-type": "application/x-www-form-urlencoded" },
body: new URLSearchParams({ client_id: oauth.clientId, scope: oauth.scope }),
body: new URLSearchParams({ client_id: OAUTH_CLIENT_ID, scope: OAUTH_SCOPE }),
signal,
});
} catch (error) {
@@ -291,7 +294,7 @@ async function requestDeviceAuthorization(
}
const data = (await response.json()) as Partial<DeviceAuthorizationResponse>;
if (!data.device_code || !data.user_code || !data.expires_in) {
if (!data.device_code || !data.user_code || !data.verification_uri || !data.expires_in) {
throw new Error("Radius OAuth device authorization response is missing required fields");
}
@@ -299,18 +302,17 @@ async function requestDeviceAuthorization(
device_code: data.device_code,
user_code: data.user_code,
verification_uri: data.verification_uri,
verification_uri_complete: data.verification_uri_complete,
expires_in: data.expires_in,
interval: data.interval,
};
}
async function loginWithDeviceCode(oauth: RadiusOAuthConfig, interaction: AuthInteraction): Promise<OAuthCredential> {
const device = await requestDeviceAuthorization(oauth, interaction.signal);
async function loginWithDeviceCode(gateway: string, interaction: AuthInteraction): Promise<OAuthCredential> {
const device = await requestDeviceAuthorization(gateway, interaction.signal);
interaction.notify({
type: "device_code",
userCode: device.user_code,
verificationUri: device.verification_uri || oauth.verificationEndpoint,
verificationUri: device.verification_uri,
intervalSeconds: device.interval,
expiresInSeconds: device.expires_in,
});
@@ -322,10 +324,10 @@ async function loginWithDeviceCode(oauth: RadiusOAuthConfig, interaction: AuthIn
poll: async () => {
try {
const credentials = await requestOAuthToken(
oauth,
gateway,
new URLSearchParams({
grant_type: oauth.deviceCodeGrantType,
client_id: oauth.clientId,
grant_type: OAUTH_DEVICE_CODE_GRANT_TYPE,
client_id: OAUTH_CLIENT_ID,
device_code: device.device_code,
}),
interaction.signal,
@@ -364,7 +366,6 @@ export function createRadiusOAuth(options: RadiusOAuthOptions): OAuthAuth {
name: options.name,
async login(interaction): Promise<OAuthCredential> {
const oauth = await loadRadiusOAuthConfig(gateway);
const loginMethod = await interaction.prompt({
type: "select",
message: `Sign in to ${options.name}:`,
@@ -377,25 +378,22 @@ export function createRadiusOAuth(options: RadiusOAuthOptions): OAuthAuth {
],
});
let credential: OAuthCredential;
if (loginMethod === LOGIN_METHOD_DEVICE_CODE) {
credential = await loginWithDeviceCode(oauth, interaction);
} else if (loginMethod === LOGIN_METHOD_BROWSER) {
credential = await loginWithBrowser(oauth, interaction);
} else {
throw new Error(`Unknown ${options.name} sign-in method: ${loginMethod}`);
return loginWithDeviceCode(gateway, interaction);
}
return credential;
if (loginMethod === LOGIN_METHOD_BROWSER) {
const discovery = await loadRadiusOAuthDiscovery(gateway);
return loginWithBrowser(gateway, discovery.authorizationEndpoint, interaction);
}
throw new Error(`Unknown ${options.name} sign-in method: ${loginMethod}`);
},
async refresh(credential, signal): Promise<OAuthCredential> {
const oauth = await loadRadiusOAuthConfig(gateway);
const refreshed = await requestOAuthToken(
oauth,
gateway,
new URLSearchParams({
grant_type: "refresh_token",
client_id: oauth.clientId,
client_id: OAUTH_CLIENT_ID,
refresh_token: credential.refresh,
}),
signal,
packages/ai/src/auth/resolve.ts
+10 -1
View File
@@ -1,4 +1,5 @@
import type { ProviderEnv } from "../types.ts";
import { formatThrownValue } from "../utils/diagnostics.ts";
import type {
ApiKeyAuth,
ApiKeyCredential,
@@ -22,12 +23,20 @@ export class ModelsError extends Error {
readonly code: ModelsErrorCode;
constructor(code: ModelsErrorCode, message: string, options?: { cause?: unknown }) {
super(message, options);
super(withCauseDetail(message, options?.cause), options);
this.name = "ModelsError";
this.code = code;
}
}
/** Callers surface `error.message` only, so keep the underlying reason in it. */
function withCauseDetail(message: string, cause: unknown): string {
if (cause === undefined || cause === null) return message;
const detail = formatThrownValue(cause).trim();
if (!detail || message.includes(detail)) return message;
return `${message}: ${detail}`;
}
/**
* Auth resolution shared by the `Models` and `ImagesModels` collections.
* A stored credential owns the provider: ambient/env is consulted only when
packages/ai/src/env-api-keys.ts
+9 -3
View File
@@ -26,6 +26,10 @@ if (typeof process !== "undefined" && (process.versions?.node || process.version
import type { KnownProvider, ProviderEnv } from "./types.ts";
import { getProviderEnvValue } from "./utils/provider-env.ts";
export const ANTHROPIC_AUTH_TOKEN_ENV = "ANTHROPIC_AUTH_TOKEN";
export const ANTHROPIC_OAUTH_TOKEN_ENV = "ANTHROPIC_OAUTH_TOKEN";
export const ANTHROPIC_API_KEY_ENV = "ANTHROPIC_API_KEY";
let cachedVertexAdcCredentialsExists: boolean | null = null;
function hasVertexAdcCredentials(env?: ProviderEnv): boolean {
@@ -66,9 +70,10 @@ function getApiKeyEnvVars(provider: string): readonly string[] | undefined {
return ["COPILOT_GITHUB_TOKEN"];
}
// ANTHROPIC_OAUTH_TOKEN takes precedence over ANTHROPIC_API_KEY
// ANTHROPIC_AUTH_TOKEN participates in env discovery/status, but
// getEnvApiKey() skips it because requests must pass it as Authorization: Bearer.
if (provider === "anthropic") {
return ["ANTHROPIC_OAUTH_TOKEN", "ANTHROPIC_API_KEY"];
return [ANTHROPIC_AUTH_TOKEN_ENV, ANTHROPIC_OAUTH_TOKEN_ENV, ANTHROPIC_API_KEY_ENV];
}
const envMap: Record<string, string> = {
@@ -139,7 +144,8 @@ export function getEnvApiKey(provider: string, env?: ProviderEnv): string | unde
export function getEnvApiKey(provider: string, env?: ProviderEnv): string | undefined {
const envKeys = findEnvKeys(provider, env);
if (envKeys?.[0]) {
return getProviderEnvValue(envKeys[0], env);
const apiKeyEnv = provider === "anthropic" ? envKeys.find((key) => key !== ANTHROPIC_AUTH_TOKEN_ENV) : envKeys[0];
if (apiKeyEnv) return getProviderEnvValue(apiKeyEnv, env);
}
// Vertex AI supports either an explicit API key or Application Default Credentials.
packages/ai/src/image-models.generated.ts
+15
View File
@@ -230,6 +230,21 @@ export const IMAGE_MODELS = {
cacheWrite: 0,
},
} satisfies ImagesModel<"openrouter-images">,
"microsoft/mai-image-2.5-pro": {
id: "microsoft/mai-image-2.5-pro",
name: "Microsoft: MAI-Image-2.5 Pro",
api: "openrouter-images",
provider: "openrouter",
baseUrl: "https://openrouter.ai/api/v1",
input: ["text", "image"],
output: ["image"],
cost: {
input: 5,
output: 0,
cacheRead: 0,
cacheWrite: 0,
},
} satisfies ImagesModel<"openrouter-images">,
"openai/gpt-5-image": {
id: "openai/gpt-5-image",
name: "OpenAI: GPT-5 Image",
packages/ai/src/models-store.ts
+5
View File
@@ -6,6 +6,11 @@ export interface ModelsStoreEntry {
lastModified?: number;
/** Unix timestamp of the last completed remote check. */
checkedAt?: number;
/**
* Opaque validator from the remote catalog's ETag header, stored verbatim
* (quotes included) and echoed back as If-None-Match.
*/
etag?: string;
}
/** Persistent model catalogs keyed by provider ID. */
packages/ai/src/providers/all.ts
+5 -3
View File
@@ -9,6 +9,7 @@ import { azureOpenAIResponsesProvider } from "./azure-openai-responses.ts";
import { cerebrasProvider } from "./cerebras.ts";
import { cloudflareAIGatewayProvider } from "./cloudflare-ai-gateway.ts";
import { cloudflareWorkersAIProvider } from "./cloudflare-workers-ai.ts";
import modelDataManifest from "./data/.manifest.json" with { type: "json" };
import { deepseekProvider } from "./deepseek.ts";
import { fireworksProvider } from "./fireworks.ts";
import { githubCopilotProvider } from "./github-copilot.ts";
@@ -67,9 +68,10 @@ export function getBuiltinProviders(): BuiltinProvider[] {
return Object.keys(MODELS) as BuiltinProvider[];
}
/** URL of a generated provider catalog, used to compare its mtime with remote catalogs during development. */
export function getBuiltinModelDataUrl(provider: BuiltinProvider): URL {
return new URL(`./data/${provider}.json`, import.meta.url);
/** Generation timestamp shared by all built-in provider catalogs. */
export function getBuiltinModelDataGeneratedAt(): number | undefined {
const generatedAt = Date.parse(modelDataManifest.generatedAt);
return Number.isNaN(generatedAt) ? undefined : generatedAt;
}
export function getBuiltinModels<TProvider extends BuiltinProvider>(
packages/ai/src/providers/anthropic.ts
+33 -3
View File
@@ -1,17 +1,47 @@
import { anthropicMessagesApi } from "../api/anthropic-messages.lazy.ts";
import { envApiKeyAuth, lazyOAuth } from "../auth/helpers.ts";
import { lazyOAuth } from "../auth/helpers.ts";
import { loadAnthropicOAuth } from "../auth/oauth/load.ts";
import type { ApiKeyAuth } from "../auth/types.ts";
import { ANTHROPIC_API_KEY_ENV, ANTHROPIC_AUTH_TOKEN_ENV, ANTHROPIC_OAUTH_TOKEN_ENV } from "../env-api-keys.ts";
import { createProvider, type Provider } from "../models.ts";
import { ANTHROPIC_MODELS } from "./anthropic.models.ts";
function anthropicApiKeyAuth(): ApiKeyAuth {
return {
name: "Anthropic API key",
login: async (interaction) => ({
type: "api_key",
key: await interaction.prompt({ type: "secret", message: "Enter Anthropic API key" }),
}),
resolve: async ({ ctx, credential }) => {
if (credential?.key) {
return { auth: { apiKey: credential.key }, env: credential.env, source: "stored credential" };
}
const authToken = await ctx.env(ANTHROPIC_AUTH_TOKEN_ENV);
if (authToken) {
return {
auth: { headers: { Authorization: `Bearer ${authToken}` } },
source: ANTHROPIC_AUTH_TOKEN_ENV,
};
}
for (const envVar of [ANTHROPIC_OAUTH_TOKEN_ENV, ANTHROPIC_API_KEY_ENV]) {
const apiKey = await ctx.env(envVar);
if (apiKey) return { auth: { apiKey }, source: envVar };
}
return undefined;
},
};
}
export function anthropicProvider(): Provider<"anthropic-messages"> {
return createProvider({
id: "anthropic",
name: "Anthropic",
baseUrl: "https://api.anthropic.com",
auth: {
// ANTHROPIC_OAUTH_TOKEN takes precedence over ANTHROPIC_API_KEY
apiKey: envApiKeyAuth("Anthropic API key", ["ANTHROPIC_OAUTH_TOKEN", "ANTHROPIC_API_KEY"]),
apiKey: anthropicApiKeyAuth(),
oauth: lazyOAuth({ name: "Anthropic (Claude Pro/Max)", load: loadAnthropicOAuth }),
},
models: Object.values(ANTHROPIC_MODELS),
packages/ai/src/types.ts
+43 -2
View File
@@ -445,10 +445,33 @@ export interface AssistantImages {
import type { TSchema } from "typebox";
/** OpenAI grammar variants for constrained sampling. */
export type GrammarFormat = "openai_lark" | "openai_regex";
export type GrammarVariants = Partial<Record<GrammarFormat, string>>;
/**
* Optional provider-side constrained sampling configs for a tool.
*
* The `json_schema` value roughly maps to the concept of `strict` in APIs which is
* implemented as json-schema constrained sampling by APIs. Grammar variants let
* callers provide provider-specific encodings of the same intended language.
*/
export type ConstrainedSamplingConfig =
| {
type: "json_schema";
strict: "prefer" | "require";
}
| {
type: "grammar";
variants: GrammarVariants;
};
export interface Tool<TParameters extends TSchema = TSchema> {
name: string;
description: string;
parameters: TParameters;
constrainedSampling?: false | ConstrainedSamplingConfig;
}
export interface Context {
@@ -522,6 +545,8 @@ export interface OpenAICompletionsCompat {
vercelGatewayRouting?: VercelGatewayRouting;
/** Whether z.ai supports top-level `tool_stream: true` for streaming tool call deltas. Default: false. */
zaiToolStream?: boolean;
/** Whether the provider supports OpenAI custom tools with Lark/regex grammar formats. When false, grammar-constrained tools fall back to normal function tools. Default: false; the generated model catalog enables it for capable models. */
supportsOpenAIGrammarTools?: boolean;
/** Whether the provider supports the `strict` field in tool definitions. Default: true. */
supportsStrictMode?: boolean;
/** Cache control convention for prompt caching. "anthropic" applies Anthropic-style `cache_control` markers to the system prompt, last tool definition, and last user, assistant, or tool-result text content. */
@@ -544,8 +569,14 @@ export interface OpenAIResponsesCompat {
sessionAffinityFormat?: SessionAffinityFormat;
/** Whether the provider supports `prompt_cache_retention: "24h"`. Default: true. */
supportsLongCacheRetention?: boolean;
/** Whether the provider supports strict JSON-schema function tools. Defaults are API-specific; generated OpenAI models enable it explicitly. */
supportsStrictMode?: boolean;
/** Whether to emit OpenAI custom tools with Lark/regex grammar formats. When false, grammar-constrained tools fall back to normal function tools. Default: false; the generated model catalog enables it for capable models. */
supportsOpenAIGrammarTools?: boolean;
/** Whether the model supports client-executed tool search for deferred tools. Default: false. */
supportsToolSearch?: boolean;
/** Whether the model accepts `prompt_cache_options` (OpenAI GPT-5.6+ explicit prompt caching). Older OpenAI models reject the parameter. Default: false. */
supportsExplicitPromptCacheMode?: boolean;
}
/** Compatibility settings for Anthropic Messages-compatible APIs. */
@@ -594,6 +625,8 @@ export interface AnthropicMessagesCompat {
forceAdaptiveThinking?: boolean;
/** Whether to replay empty thinking signatures as `signature: ""` instead of converting thinking to text. Default: false. */
allowEmptySignature?: boolean;
/** Whether the provider supports Anthropic strict tool schemas. Default: false; generated Anthropic models enable it explicitly. */
supportsStrictTools?: boolean;
/**
* Whether the provider supports deferred tools loaded by `tool_reference`
* blocks in tool results. Default: true for first-party Anthropic models
@@ -602,6 +635,12 @@ export interface AnthropicMessagesCompat {
supportsToolReferences?: boolean;
}
/** Compatibility settings for Amazon Bedrock models. */
export interface BedrockCompat {
/** Whether the model supports Bedrock strict tool schemas. Default: false. */
supportsStrictMode?: boolean;
}
/**
* OpenRouter provider routing preferences.
* Controls which upstream providers OpenRouter routes requests to.
@@ -727,11 +766,13 @@ export interface Model<TApi extends Api> {
/** Compatibility overrides for OpenAI-compatible APIs. If not set, auto-detected from baseUrl. */
compat?: TApi extends "openai-completions"
? OpenAICompletionsCompat
: TApi extends "openai-responses" | "openai-codex-responses"
: TApi extends "openai-responses" | "azure-openai-responses" | "openai-codex-responses"
? OpenAIResponsesCompat
: TApi extends "anthropic-messages"
? AnthropicMessagesCompat
: never;
: TApi extends "bedrock-converse-stream"
? BedrockCompat
: never;
}
export interface ImagesModel<TApi extends ImagesApi>
packages/ai/src/utils/error-body.ts
+8 -3
View File
@@ -69,9 +69,9 @@ function extractStatus(error: SdkErrorShape): number | undefined {
/**
* Probe the raw body reason, first usable hit wins, in SDK-field order:
* `body` string (Mistral) → `error` parsed JSON body object (`openai` SDK's
* `this.error`) → `$response.body` (Bedrock). Empty objects are treated as no
* body so an empty parsed body does not surface as `"{}"`. The chosen body is
* truncated to the cap.
* `this.error`) → `$response.body` (Bedrock). Empty objects and unread response
* streams are treated as no body so they do not surface as `"{}"` or serialized
* stream internals. The chosen body is truncated to the cap.
*/
function extractBody(error: SdkErrorShape): string | undefined {
const bodyText = pickBodyText(error);
@@ -86,10 +86,15 @@ function pickBodyText(error: SdkErrorShape): string | undefined {
if (isNonEmptyObject(error.error)) return safeJsonStringify(error.error);
const responseBody = error.$response?.body;
if (typeof responseBody === "string") return responseBody;
if (isReadableStreamLike(responseBody)) return undefined;
if (isNonEmptyObject(responseBody)) return safeJsonStringify(responseBody);
return undefined;
}
function isReadableStreamLike(value: unknown): boolean {
return typeof value === "object" && value !== null && "pipe" in value && typeof value.pipe === "function";
}
function isNonEmptyObject(value: unknown): boolean {
return typeof value === "object" && value !== null && Object.keys(value).length > 0;
}
packages/ai/src/utils/provider-retry.ts
+125
View File
@@ -0,0 +1,125 @@
const DEFAULT_MAX_RETRY_DELAY_MS = 60_000;
interface ProviderRetryOptions {
maxRetries?: number;
maxRetryDelayMs?: number;
signal?: AbortSignal;
}
interface ProviderError extends Error {
status: number | undefined;
headers: Headers | undefined;
}
function isProviderError(error: unknown): error is ProviderError {
if (!(error instanceof Error) || !("status" in error) || !("headers" in error)) return false;
return (
(error.status === undefined || typeof error.status === "number") &&
(error.headers === undefined || error.headers instanceof Headers)
);
}
/** Mirrors the pinned OpenAI/Anthropic SDK retry policy; review when either SDK is upgraded. */
function isRetryableProviderError(error: ProviderError): boolean {
const shouldRetry = error.headers?.get("x-should-retry");
if (shouldRetry === "true") return true;
if (shouldRetry === "false") return false;
if (error.status === undefined) return true;
return (
error.status === 408 ||
error.status === 409 ||
error.status === 429 ||
(typeof error.status === "number" && error.status >= 500)
);
}
function validateServerRetryDelayMs(
delayMs: number,
maxRetryDelayMs: number | undefined,
providerErrorMessage: string,
): number {
const maxDelayMs = maxRetryDelayMs ?? DEFAULT_MAX_RETRY_DELAY_MS;
if (maxDelayMs > 0 && delayMs > maxDelayMs) {
throw new Error(
`Server requested ${Math.ceil(delayMs / 1000)}s retry delay (max: ${Math.ceil(maxDelayMs / 1000)}s). ${providerErrorMessage}`,
);
}
return delayMs;
}
function getRetryDelayMs(error: ProviderError, retryIndex: number, maxRetryDelayMs: number | undefined): number {
const retryAfterMs = error.headers?.get("retry-after-ms");
if (retryAfterMs) {
const value = Number.parseFloat(retryAfterMs);
if (!Number.isNaN(value)) return validateServerRetryDelayMs(value, maxRetryDelayMs, error.message);
}
const retryAfter = error.headers?.get("retry-after");
if (retryAfter) {
const seconds = Number.parseFloat(retryAfter);
const delayMs = Number.isNaN(seconds) ? Date.parse(retryAfter) - Date.now() : seconds * 1000;
return validateServerRetryDelayMs(delayMs, maxRetryDelayMs, error.message);
}
const exponentialDelay = Math.min(0.5 * 2 ** retryIndex, 8) * 1000;
return exponentialDelay * (1 - Math.random() * 0.25);
}
function createAbortError(): Error {
const error = new Error("Request aborted");
error.name = "AbortError";
return error;
}
function abortableSleep(ms: number, signal?: AbortSignal): Promise<void> {
return new Promise((resolve, reject) => {
if (signal?.aborted) {
reject(createAbortError());
return;
}
const onAbort = () => {
clearTimeout(timeout);
reject(createAbortError());
};
const timeout = setTimeout(
() => {
signal?.removeEventListener("abort", onAbort);
resolve();
},
Math.max(0, ms),
);
signal?.addEventListener("abort", onAbort, { once: true });
});
}
/**
* Reproduce the retry behavior used by the OpenAI and Anthropic SDKs while making
* their backoff sleep interruptible. Their built-in retry timers ignore the
* request AbortSignal, so callers must invoke the SDK with `maxRetries: 0` and
* wrap the request with this helper. Provider-requested delays above
* `maxRetryDelayMs` fail immediately (60 seconds by default); set it to zero to
* disable the limit.
*/
export async function retryProviderRequest<T>(
request: () => Promise<T>,
options: ProviderRetryOptions = {},
): Promise<T> {
const maxRetries = options.maxRetries ?? 0;
let retriesRemaining = maxRetries;
for (;;) {
try {
// Each retry is a fresh SDK request, so X-Stainless-Retry-Count remains zero.
return await request();
} catch (error) {
if (options.signal?.aborted) throw createAbortError();
if (retriesRemaining <= 0 || !isProviderError(error) || !isRetryableProviderError(error)) throw error;
const retryIndex = maxRetries - retriesRemaining;
retriesRemaining--;
await abortableSleep(getRetryDelayMs(error, retryIndex, options.maxRetryDelayMs), options.signal);
}
}
}