ton/pi_harness
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(ai): support message-anchored tool loading (#6474)

Browse Source 
This adds cache-friendly dynamic tool loading anchored to tool results. Purely additive active-tool changes are recorded with `addedToolNames`, allowing supported Anthropic and OpenAI Responses models to load tool definitions at the point they become available instead of placing them in the cached prompt prefix.

It retains safe fallback behavior for unsupported models and non-additive changes but it will wipe caches.
This commit is contained in:
Armin Ronacher
2026-07-10 23:52:54 +02:00
committed by GitHub
parent d7a48d30a0
commit 3d8f74357c
19 changed files with 894 additions and 63 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/ai/CHANGELOG.md
+1
View File
@@ -10,6 +10,7 @@
### Added
- Added cache-friendly dynamic tool loading. `ToolResultMessage.addedToolNames` marks where tools from `Context.tools` became available; Anthropic and OpenAI Responses use native deferred loading so late tools stay out of the cached prefix, while other providers continue using `Context.tools` normally.
- Added a separate opt-in `max` thinking level, including native `xhigh` and `max` support for GPT-5.6 and Anthropic adaptive-thinking effort metadata matching Anthropic's documentation: `max` on all adaptive Claude models, native `xhigh` on Opus 4.7/4.8, Sonnet 5, and Fable 5 only.
- Added request-wide input-token pricing tiers to model cost metadata and usage cost calculation.
packages/ai/scripts/generate-models.ts
+28 -1
View File
@@ -9,7 +9,14 @@ import {
CLOUDFLARE_AI_GATEWAY_OPENAI_BASE_URL,
CLOUDFLARE_WORKERS_AI_BASE_URL,
} from "../src/api/cloudflare.ts";
import type { AnthropicMessagesCompat, Api, KnownProvider, Model, OpenAICompletionsCompat } from "../src/types.ts";
import type {
AnthropicMessagesCompat,
Api,
KnownProvider,
Model,
OpenAICompletionsCompat,
OpenAIResponsesCompat,
} from "../src/types.ts";
const __filename = fileURLToPath(import.meta.url);
const __dirname = dirname(__filename);
@@ -194,6 +201,15 @@ const ANT_LING_RING_THINKING_LEVEL_MAP = {
} as const;
const MODELS_DEV_OPENAI_UNSUPPORTED_MODEL_IDS = new Set(["gpt-5.6"]);
const OPENAI_TOOL_SEARCH_MODEL_IDS = new Set([
"gpt-5.4",
"gpt-5.4-mini",
"gpt-5.4-pro",
"gpt-5.5",
"gpt-5.6-sol",
"gpt-5.6-terra",
"gpt-5.6-luna",
]);
const OPENAI_LONG_CONTEXT_INPUT_THRESHOLD = 272000;
const OPENAI_SHORT_CONTEXT_CAPPED_MODEL_IDS = new Set([
"gpt-5.4",
@@ -482,6 +498,16 @@ function applyOpenAICompletionsCompatMetadata(model: Model<Api>): void {
}
}
function applyOpenAIToolSearchMetadata(model: Model<Api>): void {
const isOpenAIResponses = model.provider === "openai" && model.api === "openai-responses";
const isOpenAICodex = model.provider === "openai-codex" && model.api === "openai-codex-responses";
if (!(isOpenAIResponses || isOpenAICodex) || !OPENAI_TOOL_SEARCH_MODEL_IDS.has(model.id)) return;
model.compat = {
...(model.compat as OpenAIResponsesCompat | undefined),
supportsToolSearch: true,
};
}
function isGemini3ProModel(modelId: string): boolean {
return /gemini-3(?:\.\d+)?-pro/.test(modelId.toLowerCase());
}
@@ -2178,6 +2204,7 @@ async function generateModels() {
for (const model of allModels) {
applyThinkingLevelMetadata(model);
applyOpenAICompletionsCompatMetadata(model);
applyOpenAIToolSearchMetadata(model);
}
// Group by provider and deduplicate by model ID
packages/ai/src/api/anthropic-messages.ts
+104 -35
View File
@@ -29,6 +29,7 @@ import type {
ToolCall,
ToolResultMessage,
} from "../types.ts";
import { splitDeferredTools } from "../utils/deferred-tools.ts";
import { AssistantMessageEventStream } from "../utils/event-stream.ts";
import { headersToRecord } from "../utils/headers.ts";
import { parseJsonWithRepair, parseStreamingJson } from "../utils/json-parse.ts";
@@ -177,9 +178,24 @@ function getAnthropicCompat(
supportsCacheControlOnTools: model.compat?.supportsCacheControlOnTools ?? true,
supportsTemperature: model.compat?.supportsTemperature ?? true,
allowEmptySignature: model.compat?.allowEmptySignature ?? false,
supportsToolReferences: model.compat?.supportsToolReferences ?? defaultSupportsToolReferences(model),
};
}
/**
* Default for `supportsToolReferences`: first-party Anthropic models except
* Haiku (rejects client-side tool_reference blocks) and models that predate
* tool search (Claude 3.x, Opus/Sonnet 4.0, Opus 4.1).
*/
function defaultSupportsToolReferences(model: Model<"anthropic-messages">): boolean {
if (model.provider !== "anthropic" || model.id.includes("haiku")) return false;
const version = model.id.match(/^claude-(?:opus|sonnet|fable)-(\d+)(?:-(\d+))?(?:-|$)/);
if (!version) return false;
const major = Number(version[1]);
const minor = version[2] && version[2].length < 8 ? Number(version[2]) : 0;
return major > 4 || (major === 4 && minor >= 5);
}
export interface AnthropicOptions extends StreamOptions {
/**
* Enable extended thinking.
@@ -907,9 +923,30 @@ function buildParams(
): MessageCreateParamsStreaming {
const { cacheControl } = getCacheControl(model, options?.cacheRetention, options?.env);
const compat = getAnthropicCompat(model);
const transformedMessages = transformMessages(context.messages, model, normalizeToolCallId);
const normalizeToolName = isOAuthToken ? toClaudeCodeName : (name: string) => name;
const toolPlacement = splitDeferredTools(
{ ...context, messages: transformedMessages },
compat.supportsToolReferences,
normalizeToolName,
);
let immediateTools = toolPlacement.immediate;
let deferredTools = [...toolPlacement.deferred.values()];
if (immediateTools.length === 0 && deferredTools.length > 0) {
immediateTools = deferredTools;
deferredTools = [];
}
const deferredToolNames = new Set(deferredTools.map((tool) => normalizeToolName(tool.name)));
const params: MessageCreateParamsStreaming = {
model: model.id,
messages: convertMessages(context.messages, model, isOAuthToken, cacheControl, compat.allowEmptySignature),
messages: convertMessages(
transformedMessages,
isOAuthToken,
cacheControl,
compat.allowEmptySignature,
deferredToolNames,
normalizeToolName,
),
max_tokens: options?.maxTokens ?? model.maxTokens,
stream: true,
};
@@ -946,13 +983,16 @@ function buildParams(
params.temperature = options.temperature;
}
if (context.tools && context.tools.length > 0) {
params.tools = convertTools(
context.tools,
isOAuthToken,
compat.supportsEagerToolInputStreaming,
compat.supportsCacheControlOnTools ? cacheControl : undefined,
);
if (immediateTools.length > 0 || deferredTools.length > 0) {
params.tools = [
...convertTools(
immediateTools,
isOAuthToken,
compat.supportsEagerToolInputStreaming,
compat.supportsCacheControlOnTools ? cacheControl : undefined,
),
...convertTools(deferredTools, isOAuthToken, compat.supportsEagerToolInputStreaming, undefined, true),
];
}
// Configure thinking mode: adaptive, budget-based, or explicitly disabled.
@@ -1009,17 +1049,51 @@ function normalizeToolCallId(id: string): string {
return id.replace(/[^a-zA-Z0-9_-]/g, "_").slice(0, 64);
}
function convertToolResult(
msg: ToolResultMessage,
isOAuthToken: boolean,
deferredToolNames: ReadonlySet<string>,
loadedToolNames: Set<string>,
normalizeToolName: (name: string) => string,
): { toolResult: ContentBlockParam; siblingContent: ContentBlockParam[] } {
const references: Array<{ type: "tool_reference"; tool_name: string }> = [];
for (const name of msg.addedToolNames ?? []) {
const normalizedName = normalizeToolName(name);
if (!deferredToolNames.has(normalizedName) || loadedToolNames.has(normalizedName)) continue;
loadedToolNames.add(normalizedName);
references.push({
type: "tool_reference",
tool_name: isOAuthToken ? toClaudeCodeName(name) : name,
});
}
const convertedContent = convertContentBlocks(msg.content);
// Anthropic rejects tool references mixed with ordinary tool-result content.
return {
toolResult: {
type: "tool_result",
tool_use_id: msg.toolCallId,
content: references.length > 0 ? references : convertedContent,
is_error: msg.isError,
},
siblingContent:
references.length === 0
? []
: typeof convertedContent === "string"
? [{ type: "text", text: convertedContent }]
: convertedContent,
};
}
function convertMessages(
messages: Message[],
model: Model<"anthropic-messages">,
transformedMessages: Message[],
isOAuthToken: boolean,
cacheControl?: CacheControlEphemeral,
allowEmptySignature = false,
deferredToolNames: ReadonlySet<string> = new Set(),
normalizeToolName: (name: string) => string = (name) => name,
): MessageParam[] {
const params: MessageParam[] = [];
// Transform messages for cross-provider compatibility
const transformedMessages = transformMessages(messages, model, normalizeToolCallId);
const loadedToolNames = new Set<string>();
for (let i = 0; i < transformedMessages.length; i++) {
const msg = transformedMessages[i];
@@ -1122,37 +1196,30 @@ function convertMessages(
content: blocks,
});
} else if (msg.role === "toolResult") {
// Collect all consecutive toolResult messages, needed for z.ai Anthropic endpoint
// Collect all consecutive toolResult messages, needed for z.ai Anthropic endpoint.
const toolResults: ContentBlockParam[] = [];
// Add the current tool result
toolResults.push({
type: "tool_result",
tool_use_id: msg.toolCallId,
content: convertContentBlocks(msg.content),
is_error: msg.isError,
});
// Look ahead for consecutive toolResult messages
let j = i + 1;
const siblingContent: ContentBlockParam[] = [];
let j = i;
while (j < transformedMessages.length && transformedMessages[j].role === "toolResult") {
const nextMsg = transformedMessages[j] as ToolResultMessage; // We know it's a toolResult
toolResults.push({
type: "tool_result",
tool_use_id: nextMsg.toolCallId,
content: convertContentBlocks(nextMsg.content),
is_error: nextMsg.isError,
});
const converted = convertToolResult(
transformedMessages[j] as ToolResultMessage,
isOAuthToken,
deferredToolNames,
loadedToolNames,
normalizeToolName,
);
toolResults.push(converted.toolResult);
siblingContent.push(...converted.siblingContent);
j++;
}
// Skip the messages we've already processed
// Skip the messages we've already processed.
i = j - 1;
// Add a single user message with all tool results
// Displaced reference-bearing results must follow every tool_result block.
params.push({
role: "user",
content: toolResults,
content: [...toolResults, ...siblingContent],
});
}
}
@@ -1193,6 +1260,7 @@ function convertTools(
isOAuthToken: boolean,
supportsEagerToolInputStreaming: boolean,
cacheControl?: CacheControlEphemeral,
deferLoading = false,
): Anthropic.Messages.Tool[] {
if (!tools) return [];
@@ -1208,6 +1276,7 @@ function convertTools(
properties: schema.properties ?? {},
required: schema.required ?? [],
},
...(deferLoading ? { defer_loading: true } : {}),
...(cacheControl && index === tools.length - 1 ? { cache_control: cacheControl } : {}),
};
});
packages/ai/src/api/openai-codex-responses.ts
+5 -2
View File
@@ -36,6 +36,7 @@ import type {
Usage,
} from "../types.ts";
import { combineAbortSignals } from "../utils/abort-signals.ts";
import { splitDeferredTools } from "../utils/deferred-tools.ts";
import {
appendAssistantMessageDiagnostic,
createAssistantMessageDiagnostic,
@@ -481,8 +482,10 @@ function buildRequestBody(
context: Context,
options?: OpenAICodexResponsesOptions,
): RequestBody {
const toolPlacement = splitDeferredTools(context, model.compat?.supportsToolSearch ?? false);
const messages = convertResponsesMessages(model, context, CODEX_TOOL_CALL_PROVIDERS, {
includeSystemPrompt: false,
deferredTools: toolPlacement.deferred,
});
const body: RequestBody = {
@@ -506,8 +509,8 @@ function buildRequestBody(
body.service_tier = options.serviceTier;
}
if (context.tools && context.tools.length > 0) {
body.tools = convertResponsesTools(context.tools, { strict: null });
if (toolPlacement.immediate.length > 0) {
body.tools = convertResponsesTools(toolPlacement.immediate, { strict: null });
}
if (options?.reasoningEffort !== undefined) {
packages/ai/src/api/openai-responses-shared.ts
+44 -8
View File
@@ -6,11 +6,13 @@ import type {
ResponseInput,
ResponseInputContent,
ResponseInputImage,
ResponseInputItem,
ResponseInputText,
ResponseOutputItem,
ResponseOutputMessage,
ResponseReasoningItem,
ResponseStreamEvent,
ResponseToolSearchOutputItemParam,
} from "openai/resources/responses/responses.js";
import { calculateCost } from "../models.ts";
import type {
@@ -77,12 +79,16 @@ export interface OpenAIResponsesStreamOptions {
export interface ConvertResponsesMessagesOptions {
includeSystemPrompt?: boolean;
deferredTools?: ReadonlyMap<string, Tool>;
}
export interface ConvertResponsesToolsOptions {
strict?: boolean | null;
deferLoading?: boolean;
}
type OpenAIFunctionTool = Extract<OpenAITool, { type: "function" }>;
// =============================================================================
// Message conversion
// =============================================================================
@@ -94,6 +100,7 @@ export function convertResponsesMessages<TApi extends Api>(
options?: ConvertResponsesMessagesOptions,
): ResponseInput {
const messages: ResponseInput = [];
const loadedToolNames = new Set<string>();
const normalizeIdPart = (part: string): string => {
const sanitized = part.replace(/[^a-zA-Z0-9_-]/g, "_");
@@ -259,6 +266,32 @@ export function convertResponsesMessages<TApi extends Api>(
call_id: callId,
output,
});
const deferredTools: Tool[] = [];
for (const name of msg.addedToolNames ?? []) {
const tool = options?.deferredTools?.get(name);
if (!tool || loadedToolNames.has(name)) continue;
loadedToolNames.add(name);
deferredTools.push(tool);
}
if (deferredTools.length > 0) {
const names = deferredTools.map((tool) => tool.name);
const searchCallId = `pi_tool_load_${shortHash(`${msg.toolCallId}:${names.join(",")}`)}`;
messages.push({
type: "tool_search_call",
call_id: searchCallId,
execution: "client",
status: "completed",
arguments: { query: names.join(" "), limit: names.length },
} satisfies ResponseInputItem);
messages.push({
type: "tool_search_output",
call_id: searchCallId,
execution: "client",
status: "completed",
tools: convertResponsesTools(deferredTools, { deferLoading: true }),
} satisfies ResponseToolSearchOutputItemParam);
}
}
msgIndex++;
}
@@ -270,15 +303,18 @@ export function convertResponsesMessages<TApi extends Api>(
// Tool conversion
// =============================================================================
export function convertResponsesTools(tools: Tool[], options?: ConvertResponsesToolsOptions): OpenAITool[] {
export function convertResponsesTools(tools: readonly Tool[], options?: ConvertResponsesToolsOptions): OpenAITool[] {
const strict = options?.strict === undefined ? false : options.strict;
return tools.map((tool) => ({
type: "function",
name: tool.name,
description: tool.description,
parameters: tool.parameters as any, // TypeBox already generates JSON Schema
strict,
}));
return tools.map(
(tool): OpenAIFunctionTool => ({
type: "function",
name: tool.name,
description: tool.description,
parameters: tool.parameters as Record<string, unknown>, // TypeBox already generates JSON Schema
strict,
...(options?.deferLoading ? { defer_loading: true } : {}),
}),
);
}
// =============================================================================
packages/ai/src/api/openai-responses.ts
+9 -4
View File
@@ -15,6 +15,7 @@ import type {
StreamOptions,
Usage,
} from "../types.ts";
import { splitDeferredTools } from "../utils/deferred-tools.ts";
import { formatProviderError, normalizeProviderError } from "../utils/error-body.ts";
import { AssistantMessageEventStream } from "../utils/event-stream.ts";
import { headersToRecord } from "../utils/headers.ts";
@@ -62,6 +63,7 @@ function getCompat(model: Model<"openai-responses">): Required<OpenAIResponsesCo
supportsDeveloperRole: model.compat?.supportsDeveloperRole ?? true,
sendSessionIdHeader: model.compat?.sendSessionIdHeader ?? true,
supportsLongCacheRetention: model.compat?.supportsLongCacheRetention ?? true,
supportsToolSearch: model.compat?.supportsToolSearch ?? false,
};
}
@@ -220,10 +222,13 @@ function createClient(
}
function buildParams(model: Model<"openai-responses">, context: Context, options?: OpenAIResponsesOptions) {
const messages = convertResponsesMessages(model, context, OPENAI_TOOL_CALL_PROVIDERS);
const compat = getCompat(model);
const toolPlacement = splitDeferredTools(context, compat.supportsToolSearch);
const messages = convertResponsesMessages(model, context, OPENAI_TOOL_CALL_PROVIDERS, {
deferredTools: toolPlacement.deferred,
});
const cacheRetention = resolveCacheRetention(options?.cacheRetention, options?.env);
const compat = getCompat(model);
const params: ResponseCreateParamsStreaming = {
model: model.id,
input: messages,
@@ -245,8 +250,8 @@ function buildParams(model: Model<"openai-responses">, context: Context, options
params.service_tier = options.serviceTier;
}
if (context.tools && context.tools.length > 0) {
params.tools = convertResponsesTools(context.tools);
if (toolPlacement.immediate.length > 0) {
params.tools = convertResponsesTools(toolPlacement.immediate);
}
if (model.reasoning) {
packages/ai/src/providers/openai-codex.models.ts
+6
View File
@@ -28,6 +28,7 @@ export const OPENAI_CODEX_MODELS = {
api: "openai-codex-responses",
provider: "openai-codex",
baseUrl: "https://chatgpt.com/backend-api",
compat: {"supportsToolSearch":true},
reasoning: true,
thinkingLevelMap: {"xhigh":"xhigh","minimal":"low"},
input: ["text", "image"],
@@ -47,6 +48,7 @@ export const OPENAI_CODEX_MODELS = {
api: "openai-codex-responses",
provider: "openai-codex",
baseUrl: "https://chatgpt.com/backend-api",
compat: {"supportsToolSearch":true},
reasoning: true,
thinkingLevelMap: {"xhigh":"xhigh","minimal":"low"},
input: ["text", "image"],
@@ -65,6 +67,7 @@ export const OPENAI_CODEX_MODELS = {
api: "openai-codex-responses",
provider: "openai-codex",
baseUrl: "https://chatgpt.com/backend-api",
compat: {"supportsToolSearch":true},
reasoning: true,
thinkingLevelMap: {"xhigh":"xhigh","minimal":"low"},
input: ["text", "image"],
@@ -84,6 +87,7 @@ export const OPENAI_CODEX_MODELS = {
api: "openai-codex-responses",
provider: "openai-codex",
baseUrl: "https://chatgpt.com/backend-api",
compat: {"supportsToolSearch":true},
reasoning: true,
thinkingLevelMap: {"xhigh":"xhigh","max":"max","minimal":"low"},
input: ["text", "image"],
@@ -103,6 +107,7 @@ export const OPENAI_CODEX_MODELS = {
api: "openai-codex-responses",
provider: "openai-codex",
baseUrl: "https://chatgpt.com/backend-api",
compat: {"supportsToolSearch":true},
reasoning: true,
thinkingLevelMap: {"xhigh":"xhigh","max":"max","minimal":"low"},
input: ["text", "image"],
@@ -122,6 +127,7 @@ export const OPENAI_CODEX_MODELS = {
api: "openai-codex-responses",
provider: "openai-codex",
baseUrl: "https://chatgpt.com/backend-api",
compat: {"supportsToolSearch":true},
reasoning: true,
thinkingLevelMap: {"xhigh":"xhigh","max":"max","minimal":"low"},
input: ["text", "image"],
packages/ai/src/providers/openai.models.ts
+7
View File
@@ -504,6 +504,7 @@ export const OPENAI_MODELS = {
api: "openai-responses",
provider: "openai",
baseUrl: "https://api.openai.com/v1",
compat: {"supportsToolSearch":true},
reasoning: true,
thinkingLevelMap: {"off":"none","xhigh":"xhigh"},
input: ["text", "image"],
@@ -523,6 +524,7 @@ export const OPENAI_MODELS = {
api: "openai-responses",
provider: "openai",
baseUrl: "https://api.openai.com/v1",
compat: {"supportsToolSearch":true},
reasoning: true,
thinkingLevelMap: {"off":"none","xhigh":"xhigh"},
input: ["text", "image"],
@@ -559,6 +561,7 @@ export const OPENAI_MODELS = {
api: "openai-responses",
provider: "openai",
baseUrl: "https://api.openai.com/v1",
compat: {"supportsToolSearch":true},
reasoning: true,
thinkingLevelMap: {"off":null,"xhigh":"xhigh"},
input: ["text", "image"],
@@ -578,6 +581,7 @@ export const OPENAI_MODELS = {
api: "openai-responses",
provider: "openai",
baseUrl: "https://api.openai.com/v1",
compat: {"supportsToolSearch":true},
reasoning: true,
thinkingLevelMap: {"off":"none","xhigh":"xhigh","minimal":null},
input: ["text", "image"],
@@ -616,6 +620,7 @@ export const OPENAI_MODELS = {
api: "openai-responses",
provider: "openai",
baseUrl: "https://api.openai.com/v1",
compat: {"supportsToolSearch":true},
reasoning: true,
thinkingLevelMap: {"off":"none","xhigh":"xhigh","max":"max"},
input: ["text", "image"],
@@ -635,6 +640,7 @@ export const OPENAI_MODELS = {
api: "openai-responses",
provider: "openai",
baseUrl: "https://api.openai.com/v1",
compat: {"supportsToolSearch":true},
reasoning: true,
thinkingLevelMap: {"off":"none","xhigh":"xhigh","max":"max"},
input: ["text", "image"],
@@ -654,6 +660,7 @@ export const OPENAI_MODELS = {
api: "openai-responses",
provider: "openai",
baseUrl: "https://api.openai.com/v1",
compat: {"supportsToolSearch":true},
reasoning: true,
thinkingLevelMap: {"off":"none","xhigh":"xhigh","max":"max"},
input: ["text", "image"],
packages/ai/src/types.ts
+15 -1
View File
@@ -401,6 +401,12 @@ export interface ToolResultMessage<TDetails = any> {
toolName: string;
content: (TextContent | ImageContent)[]; // Supports text and images
details?: TDetails;
/**
* Names from `Context.tools` that became available after this result.
* Providers with native deferred tool loading use this as the load point;
* other providers ignore it and use `Context.tools` normally.
*/
addedToolNames?: string[];
isError: boolean;
timestamp: number; // Unix timestamp in milliseconds
}
@@ -525,6 +531,8 @@ export interface OpenAIResponsesCompat {
sendSessionIdHeader?: boolean;
/** Whether the provider supports `prompt_cache_retention: "24h"`. Default: true. */
supportsLongCacheRetention?: boolean;
/** Whether the model supports client-executed tool search for deferred tools. Default: false. */
supportsToolSearch?: boolean;
}
/** Compatibility settings for Anthropic Messages-compatible APIs. */
@@ -573,6 +581,12 @@ export interface AnthropicMessagesCompat {
forceAdaptiveThinking?: boolean;
/** Whether to replay empty thinking signatures as `signature: ""` instead of converting thinking to text. Default: false. */
allowEmptySignature?: boolean;
/**
* Whether the provider supports deferred tools loaded by `tool_reference`
* blocks in tool results. Default: true for first-party Anthropic models
* except Haiku and models older than Claude 4.5; false for other providers.
*/
supportsToolReferences?: boolean;
}
/**
@@ -700,7 +714,7 @@ export interface Model<TApi extends Api> {
/** Compatibility overrides for OpenAI-compatible APIs. If not set, auto-detected from baseUrl. */
compat?: TApi extends "openai-completions"
? OpenAICompletionsCompat
: TApi extends "openai-responses"
: TApi extends "openai-responses" | "openai-codex-responses"
? OpenAIResponsesCompat
: TApi extends "anthropic-messages"
? AnthropicMessagesCompat
packages/ai/src/utils/deferred-tools.ts
+39
View File
@@ -0,0 +1,39 @@
import type { Context, Tool } from "../types.ts";
type ToolNameNormalizer = (name: string) => string;
const identityToolName: ToolNameNormalizer = (name) => name;
/** Split current tools into prefix and transcript-loaded definitions. */
export function splitDeferredTools(
context: Context,
enabled: boolean,
normalizeName: ToolNameNormalizer = identityToolName,
): { immediate: Tool[]; deferred: Map<string, Tool> } {
const uniqueTools = new Map<string, Tool>();
for (const tool of context.tools ?? []) uniqueTools.set(normalizeName(tool.name), tool);
if (!enabled) return { immediate: [...uniqueTools.values()], deferred: new Map() };
const deferredNames = new Set<string>();
const usedNames = new Set<string>();
for (const message of context.messages) {
if (message.role === "assistant") {
for (const block of message.content) {
if (block.type === "toolCall") usedNames.add(normalizeName(block.name));
}
} else if (message.role === "toolResult") {
for (const name of message.addedToolNames ?? []) {
const normalizedName = normalizeName(name);
if (!usedNames.has(normalizedName)) deferredNames.add(normalizedName);
}
}
}
const immediate: Tool[] = [];
const deferred = new Map<string, Tool>();
for (const [name, tool] of uniqueTools) {
if (deferredNames.has(name)) deferred.set(name, tool);
else immediate.push(tool);
}
return { immediate, deferred };
}
packages/ai/src/utils/estimate.ts
+23 -6
View File
@@ -1,4 +1,4 @@
import type { AssistantMessage, Context, ImageContent, Message, TextContent, Usage } from "../types.ts";
import type { AssistantMessage, Context, ImageContent, Message, TextContent, Tool, Usage } from "../types.ts";
export interface ContextUsageEstimate {
/** Estimated total context tokens. */
@@ -102,6 +102,11 @@ function estimateMessages(messages: readonly Message[]): ContextUsageEstimate {
return { tokens, usageTokens: 0, trailingTokens: tokens, lastUsageIndex: null };
}
function estimateToolsTokens(tools: readonly Tool[] | undefined): number {
if (!tools || tools.length === 0) return 0;
return estimateTextTokens(safeJsonStringify(tools));
}
function isMessageArray(value: Context | readonly Message[]): value is readonly Message[] {
return Array.isArray(value);
}
@@ -110,13 +115,25 @@ export function estimateContextTokens(context: Context | readonly Message[]): Co
if (isMessageArray(context)) return estimateMessages(context);
const estimate = estimateMessages(context.messages);
if (estimate.lastUsageIndex !== null) return estimate;
let prefixTokens = context.systemPrompt ? estimateTextTokens(context.systemPrompt) : 0;
if (context.tools && context.tools.length > 0) {
prefixTokens += estimateTextTokens(safeJsonStringify(context.tools));
if (estimate.lastUsageIndex !== null) {
const addedNames = new Set(
context.messages
.slice(estimate.lastUsageIndex + 1)
.filter((message) => message.role === "toolResult")
.flatMap((message) => message.addedToolNames ?? []),
);
const addedToolTokens = estimateToolsTokens(context.tools?.filter((tool) => addedNames.has(tool.name)));
return {
tokens: estimate.tokens + addedToolTokens,
usageTokens: estimate.usageTokens,
trailingTokens: estimate.trailingTokens + addedToolTokens,
lastUsageIndex: estimate.lastUsageIndex,
};
}
const prefixTokens =
(context.systemPrompt ? estimateTextTokens(context.systemPrompt) : 0) + estimateToolsTokens(context.tools);
return {
tokens: estimate.tokens + prefixTokens,
usageTokens: estimate.usageTokens,
packages/ai/test/deferred-tools.test.ts
+387
View File
@@ -0,0 +1,387 @@
import { Type } from "typebox";
import { describe, expect, it } from "vitest";
import { getModel, streamSimple } from "../src/compat.ts";
import type { Api, AssistantMessage, Context, Model, Tool, ToolResultMessage, UserMessage } from "../src/types.ts";
import { estimateContextTokens } from "../src/utils/estimate.ts";
interface AnthropicToolPayload {
name: string;
description?: string;
defer_loading?: boolean;
}
interface AnthropicContentBlock {
type: string;
text?: string;
tool_use_id?: string;
content?: string | Array<{ type: string; tool_name?: string }>;
source?: {
type: string;
media_type: string;
data: string;
};
}
interface AnthropicPayload {
tools?: AnthropicToolPayload[];
messages: Array<{
content: string | AnthropicContentBlock[];
}>;
}
interface OpenAIToolSearchCall {
type: "tool_search_call";
call_id?: string | null;
execution?: string;
status?: string | null;
}
interface OpenAIToolSearchOutput {
type: "tool_search_output";
call_id?: string | null;
execution?: string;
status?: string | null;
tools: Array<{ type: string; name: string; defer_loading?: boolean }>;
}
interface OpenAIPayload {
tools?: Array<{ name?: string; function?: { name: string } }>;
input?: Array<OpenAIToolSearchCall | OpenAIToolSearchOutput | { type?: string }>;
}
class PayloadCaptured extends Error {}
function makeTool(name: string): Tool {
return {
name,
description: `The ${name} tool`,
parameters: Type.Object({ value: Type.String() }),
};
}
function makeUserMessage(timestamp: number): UserMessage {
return { role: "user", content: "Hello", timestamp };
}
function makeAssistantToolCall(): AssistantMessage {
return {
role: "assistant",
content: [{ type: "toolCall", id: "call_1", name: "base_tool", arguments: {} }],
api: "anthropic-messages",
provider: "anthropic",
model: "claude-opus-4-6",
usage: {
input: 0,
output: 0,
cacheRead: 0,
cacheWrite: 0,
totalTokens: 0,
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 },
},
stopReason: "toolUse",
timestamp: 2,
};
}
function makeToolResult(addedToolNames: string[]): ToolResultMessage {
return {
role: "toolResult",
toolCallId: "call_1",
toolName: "base_tool",
content: [{ type: "text", text: "done" }],
addedToolNames,
isError: false,
timestamp: 3,
};
}
function makeContext(tools: Tool[], addedToolNames = ["late_tool"]): Context {
return {
messages: [makeUserMessage(1), makeAssistantToolCall(), makeToolResult(addedToolNames), makeUserMessage(4)],
tools,
};
}
async function capturePayload<T>(model: Model<Api>, context: Context, apiKey = "fake-key"): Promise<T> {
let captured: T | undefined;
const stream = streamSimple({ ...model, baseUrl: "http://127.0.0.1:9" }, context, {
apiKey,
onPayload: (payload) => {
captured = payload as T;
throw new PayloadCaptured();
},
});
await stream.result();
if (!captured) throw new Error("Expected payload capture");
return captured;
}
function findAnthropicToolResultContent(payload: AnthropicPayload): AnthropicContentBlock[] {
for (const message of payload.messages) {
if (typeof message.content !== "string" && message.content.some((block) => block.type === "tool_result")) {
return message.content;
}
}
throw new Error("No tool result in payload");
}
function findAnthropicToolResult(payload: AnthropicPayload): AnthropicContentBlock {
const result = findAnthropicToolResultContent(payload).find((block) => block.type === "tool_result");
if (!result) throw new Error("No tool result in payload");
return result;
}
function openAIToolNames(payload: OpenAIPayload): string[] {
return (payload.tools ?? []).map((tool) => tool.name ?? tool.function?.name ?? "");
}
function makeCodexToken(): string {
return `header.${btoa(JSON.stringify({ "https://api.openai.com/auth": { chatgpt_account_id: "account" } }))}.signature`;
}
describe("deferred tools", () => {
it("loads an Anthropic tool at its tool-result marker", async () => {
const context = makeContext([makeTool("base_tool"), makeTool("late_tool")]);
const payload = await capturePayload<AnthropicPayload>(getModel("anthropic", "claude-opus-4-6"), context);
expect(payload.tools).toMatchObject([{ name: "base_tool" }, { name: "late_tool", defer_loading: true }]);
expect(findAnthropicToolResult(payload).content).toEqual([{ type: "tool_reference", tool_name: "late_tool" }]);
});
it("preserves tool output as sibling content after emitting references", async () => {
const context = makeContext([makeTool("base_tool"), makeTool("late_tool")]);
const assistant = context.messages[1] as AssistantMessage;
assistant.content = [
{ type: "toolCall", id: "call_1", name: "base_tool", arguments: {} },
{ type: "toolCall", id: "call_2", name: "base_tool", arguments: {} },
];
const firstResult = context.messages[2] as ToolResultMessage;
firstResult.content = [
{ type: "text", text: "work completed" },
{ type: "image", mimeType: "image/png", data: "aW1hZ2U=" },
];
context.messages.splice(3, 0, {
...makeToolResult([]),
toolCallId: "call_2",
content: [{ type: "text", text: "second result" }],
});
const payload = await capturePayload<AnthropicPayload>(getModel("anthropic", "claude-opus-4-6"), context);
expect(findAnthropicToolResultContent(payload)).toMatchObject([
{
type: "tool_result",
tool_use_id: "call_1",
content: [{ type: "tool_reference", tool_name: "late_tool" }],
},
{ type: "tool_result", tool_use_id: "call_2", content: "second result" },
{ type: "text", text: "work completed" },
{
type: "image",
source: { type: "base64", media_type: "image/png", data: "aW1hZ2U=" },
},
]);
});
it("loads a tool introduced by OpenAI history after switching to Anthropic", async () => {
const context = makeContext([makeTool("base_tool"), makeTool("late_tool")]);
const assistant = context.messages[1] as AssistantMessage;
assistant.api = "openai-responses";
assistant.provider = "openai";
assistant.model = "gpt-5.4";
const payload = await capturePayload<AnthropicPayload>(getModel("anthropic", "claude-opus-4-8"), context);
expect(payload.tools).toMatchObject([{ name: "base_tool" }, { name: "late_tool", defer_loading: true }]);
expect(findAnthropicToolResult(payload).content).toEqual([{ type: "tool_reference", tool_name: "late_tool" }]);
});
it("does not resurrect a marked tool missing from Context.tools", async () => {
const context = makeContext([makeTool("base_tool")]);
const payload = await capturePayload<AnthropicPayload>(getModel("anthropic", "claude-opus-4-6"), context);
expect(payload.tools?.map((tool) => tool.name)).toEqual(["base_tool"]);
const content = findAnthropicToolResult(payload).content;
expect(Array.isArray(content) && content.some((block) => block.type === "tool_reference")).toBe(false);
});
it("keeps a tool immediate when it was used before its marker", async () => {
const context = makeContext([makeTool("base_tool"), makeTool("late_tool")]);
const assistant = context.messages[1] as AssistantMessage;
assistant.content = [{ type: "toolCall", id: "call_1", name: "late_tool", arguments: {} }];
const payload = await capturePayload<AnthropicPayload>(getModel("anthropic", "claude-opus-4-6"), context);
expect(payload.tools?.map((tool) => tool.name)).toEqual(["base_tool", "late_tool"]);
expect(payload.tools?.every((tool) => !tool.defer_loading)).toBe(true);
});
it("normalizes OAuth names before checking prior tool usage", async () => {
const context = makeContext([makeTool("base_tool"), makeTool("read")], ["read"]);
const assistant = context.messages[1] as AssistantMessage;
assistant.content = [{ type: "toolCall", id: "call_1", name: "Read", arguments: {} }];
const payload = await capturePayload<AnthropicPayload>(
getModel("anthropic", "claude-opus-4-6"),
context,
"sk-ant-oat-fake",
);
expect(payload.tools?.map((tool) => tool.name)).toEqual(["base_tool", "Read"]);
expect(payload.tools?.every((tool) => !tool.defer_loading)).toBe(true);
const content = findAnthropicToolResult(payload).content;
expect(Array.isArray(content) && content.some((block) => block.type === "tool_reference")).toBe(false);
});
it("matches OAuth-canonicalized markers to active tools", async () => {
const context = makeContext([makeTool("base_tool"), makeTool("read")], ["Read"]);
const payload = await capturePayload<AnthropicPayload>(
getModel("anthropic", "claude-opus-4-6"),
context,
"sk-ant-oat-fake",
);
expect(payload.tools).toMatchObject([{ name: "base_tool" }, { name: "Read", defer_loading: true }]);
const content = findAnthropicToolResult(payload).content;
expect(
Array.isArray(content) &&
content.some((block) => block.type === "tool_reference" && block.tool_name === "Read"),
).toBe(true);
});
it("deduplicates active tools after OAuth canonicalization", async () => {
const context: Context = {
messages: [makeUserMessage(1)],
tools: [makeTool("read"), { ...makeTool("Read"), description: "Canonical definition" }],
};
const payload = await capturePayload<AnthropicPayload>(
getModel("anthropic", "claude-opus-4-6"),
context,
"sk-ant-oat-fake",
);
expect(payload.tools).toMatchObject([{ name: "Read", description: "Canonical definition" }]);
});
it("uses the normal tool list when Anthropic tool references are unsupported", async () => {
const context = makeContext([makeTool("base_tool"), makeTool("late_tool")]);
const models: Model<"anthropic-messages">[] = [
getModel("anthropic", "claude-haiku-4-5"),
{ ...getModel("anthropic", "claude-opus-4-6"), id: "claude-sonnet-4-20250514" },
];
for (const model of models) {
const payload = await capturePayload<AnthropicPayload>(model, context);
expect(payload.tools?.map((tool) => tool.name)).toEqual(["base_tool", "late_tool"]);
expect(payload.tools?.every((tool) => !tool.defer_loading)).toBe(true);
}
});
it("keeps one immediate Anthropic tool when every current tool is marked", async () => {
const context = makeContext([makeTool("late_tool")]);
const payload = await capturePayload<AnthropicPayload>(getModel("anthropic", "claude-opus-4-6"), context);
expect(payload.tools).toMatchObject([{ name: "late_tool" }]);
expect(payload.tools?.[0]?.defer_loading).toBeUndefined();
const content = findAnthropicToolResult(payload).content;
expect(Array.isArray(content) && content.some((block) => block.type === "tool_reference")).toBe(false);
});
it("supports explicit Anthropic compatibility overrides", async () => {
const model: Model<"anthropic-messages"> = {
...getModel("anthropic", "claude-opus-4-6"),
provider: "anthropic-proxy",
compat: { supportsToolReferences: true },
};
const context = makeContext([makeTool("base_tool"), makeTool("late_tool")]);
const payload = await capturePayload<AnthropicPayload>(model, context);
expect(payload.tools?.find((tool) => tool.name === "late_tool")?.defer_loading).toBe(true);
});
it("loads an OpenAI Responses tool through client tool search", async () => {
const context = makeContext([makeTool("base_tool"), makeTool("late_tool")]);
const payload = await capturePayload<OpenAIPayload>(getModel("openai", "gpt-5.4"), context);
const searchCall = payload.input?.find((item): item is OpenAIToolSearchCall => item.type === "tool_search_call");
const searchOutput = payload.input?.find(
(item): item is OpenAIToolSearchOutput => item.type === "tool_search_output",
);
expect(openAIToolNames(payload)).toEqual(["base_tool"]);
expect(searchCall).toMatchObject({ execution: "client", status: "completed" });
expect(searchOutput?.call_id).toBe(searchCall?.call_id);
expect(searchOutput?.tools).toMatchObject([{ type: "function", name: "late_tool", defer_loading: true }]);
});
it.each(["gpt-5.2", "gpt-5.4-nano", "gpt-5.5-pro"] as const)(
"uses the normal tool list for unsupported OpenAI model %s",
async (modelId) => {
const context = makeContext([makeTool("base_tool"), makeTool("late_tool")]);
const payload = await capturePayload<OpenAIPayload>(getModel("openai", modelId), context);
expect(openAIToolNames(payload)).toEqual(["base_tool", "late_tool"]);
expect(payload.input?.some((item) => item.type === "tool_search_output")).toBe(false);
},
);
it("uses the normal tool list when OpenAI tool search is explicitly disabled", async () => {
const model: Model<"openai-responses"> = {
...getModel("openai", "gpt-5.4"),
provider: "openai-proxy",
compat: { supportsToolSearch: false },
};
const context = makeContext([makeTool("base_tool"), makeTool("late_tool")]);
const payload = await capturePayload<OpenAIPayload>(model, context);
expect(openAIToolNames(payload)).toEqual(["base_tool", "late_tool"]);
expect(payload.input?.some((item) => item.type === "tool_search_output")).toBe(false);
});
it("uses tool search only for supported Codex models", async () => {
const context = makeContext([makeTool("base_tool"), makeTool("late_tool")]);
const supported = await capturePayload<OpenAIPayload>(
getModel("openai-codex", "gpt-5.4"),
context,
makeCodexToken(),
);
const unsupported = await capturePayload<OpenAIPayload>(
getModel("openai-codex", "gpt-5.3-codex-spark"),
context,
makeCodexToken(),
);
expect(openAIToolNames(supported)).toEqual(["base_tool"]);
expect(supported.input?.some((item) => item.type === "tool_search_output")).toBe(true);
expect(openAIToolNames(unsupported)).toEqual(["base_tool", "late_tool"]);
expect(unsupported.input?.some((item) => item.type === "tool_search_output")).toBe(false);
});
it("leaves providers without deferred loading unchanged", async () => {
const context = makeContext([makeTool("base_tool"), makeTool("late_tool")]);
const payload = await capturePayload<OpenAIPayload>(getModel("groq", "llama-3.3-70b-versatile"), context);
expect(openAIToolNames(payload)).toEqual(["base_tool", "late_tool"]);
});
it("counts definitions marked after the latest usage checkpoint", () => {
const assistant: AssistantMessage = {
...makeAssistantToolCall(),
content: [{ type: "text", text: "done" }],
usage: {
input: 50,
output: 50,
cacheRead: 0,
cacheWrite: 0,
totalTokens: 100,
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 },
},
stopReason: "stop",
};
const plain = estimateContextTokens({ messages: [assistant, makeUserMessage(4)], tools: [] });
const lateTool = { ...makeTool("late_tool"), description: "x".repeat(4000) };
const marked = estimateContextTokens({
messages: [assistant, makeToolResult(["late_tool"])],
tools: [lateTool],
});
expect(marked.tokens).toBeGreaterThan(plain.tokens + 500);
expect(marked.trailingTokens).toBeGreaterThan(plain.trailingTokens + 500);
});
});