ton/pi_harness
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(ai): support message-anchored tool loading (#6474)

Browse Source 
This adds cache-friendly dynamic tool loading anchored to tool results. Purely additive active-tool changes are recorded with `addedToolNames`, allowing supported Anthropic and OpenAI Responses models to load tool definitions at the point they become available instead of placing them in the cached prompt prefix.

It retains safe fallback behavior for unsupported models and non-additive changes but it will wipe caches.
This commit is contained in:
Armin Ronacher
2026-07-10 23:52:54 +02:00
committed by GitHub
parent d7a48d30a0
commit 3d8f74357c
19 changed files with 894 additions and 63 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/ai/src/api/anthropic-messages.ts
+104 -35
View File
@@ -29,6 +29,7 @@ import type {
ToolCall,
ToolResultMessage,
} from "../types.ts";
import { splitDeferredTools } from "../utils/deferred-tools.ts";
import { AssistantMessageEventStream } from "../utils/event-stream.ts";
import { headersToRecord } from "../utils/headers.ts";
import { parseJsonWithRepair, parseStreamingJson } from "../utils/json-parse.ts";
@@ -177,9 +178,24 @@ function getAnthropicCompat(
supportsCacheControlOnTools: model.compat?.supportsCacheControlOnTools ?? true,
supportsTemperature: model.compat?.supportsTemperature ?? true,
allowEmptySignature: model.compat?.allowEmptySignature ?? false,
supportsToolReferences: model.compat?.supportsToolReferences ?? defaultSupportsToolReferences(model),
};
}
/**
* Default for `supportsToolReferences`: first-party Anthropic models except
* Haiku (rejects client-side tool_reference blocks) and models that predate
* tool search (Claude 3.x, Opus/Sonnet 4.0, Opus 4.1).
*/
function defaultSupportsToolReferences(model: Model<"anthropic-messages">): boolean {
if (model.provider !== "anthropic" || model.id.includes("haiku")) return false;
const version = model.id.match(/^claude-(?:opus|sonnet|fable)-(\d+)(?:-(\d+))?(?:-|$)/);
if (!version) return false;
const major = Number(version[1]);
const minor = version[2] && version[2].length < 8 ? Number(version[2]) : 0;
return major > 4 || (major === 4 && minor >= 5);
}
export interface AnthropicOptions extends StreamOptions {
/**
* Enable extended thinking.
@@ -907,9 +923,30 @@ function buildParams(
): MessageCreateParamsStreaming {
const { cacheControl } = getCacheControl(model, options?.cacheRetention, options?.env);
const compat = getAnthropicCompat(model);
const transformedMessages = transformMessages(context.messages, model, normalizeToolCallId);
const normalizeToolName = isOAuthToken ? toClaudeCodeName : (name: string) => name;
const toolPlacement = splitDeferredTools(
{ ...context, messages: transformedMessages },
compat.supportsToolReferences,
normalizeToolName,
);
let immediateTools = toolPlacement.immediate;
let deferredTools = [...toolPlacement.deferred.values()];
if (immediateTools.length === 0 && deferredTools.length > 0) {
immediateTools = deferredTools;
deferredTools = [];
}
const deferredToolNames = new Set(deferredTools.map((tool) => normalizeToolName(tool.name)));
const params: MessageCreateParamsStreaming = {
model: model.id,
messages: convertMessages(context.messages, model, isOAuthToken, cacheControl, compat.allowEmptySignature),
messages: convertMessages(
transformedMessages,
isOAuthToken,
cacheControl,
compat.allowEmptySignature,
deferredToolNames,
normalizeToolName,
),
max_tokens: options?.maxTokens ?? model.maxTokens,
stream: true,
};
@@ -946,13 +983,16 @@ function buildParams(
params.temperature = options.temperature;
}
if (context.tools && context.tools.length > 0) {
params.tools = convertTools(
context.tools,
isOAuthToken,
compat.supportsEagerToolInputStreaming,
compat.supportsCacheControlOnTools ? cacheControl : undefined,
);
if (immediateTools.length > 0 || deferredTools.length > 0) {
params.tools = [
...convertTools(
immediateTools,
isOAuthToken,
compat.supportsEagerToolInputStreaming,
compat.supportsCacheControlOnTools ? cacheControl : undefined,
),
...convertTools(deferredTools, isOAuthToken, compat.supportsEagerToolInputStreaming, undefined, true),
];
}
// Configure thinking mode: adaptive, budget-based, or explicitly disabled.
@@ -1009,17 +1049,51 @@ function normalizeToolCallId(id: string): string {
return id.replace(/[^a-zA-Z0-9_-]/g, "_").slice(0, 64);
}
function convertToolResult(
msg: ToolResultMessage,
isOAuthToken: boolean,
deferredToolNames: ReadonlySet<string>,
loadedToolNames: Set<string>,
normalizeToolName: (name: string) => string,
): { toolResult: ContentBlockParam; siblingContent: ContentBlockParam[] } {
const references: Array<{ type: "tool_reference"; tool_name: string }> = [];
for (const name of msg.addedToolNames ?? []) {
const normalizedName = normalizeToolName(name);
if (!deferredToolNames.has(normalizedName) || loadedToolNames.has(normalizedName)) continue;
loadedToolNames.add(normalizedName);
references.push({
type: "tool_reference",
tool_name: isOAuthToken ? toClaudeCodeName(name) : name,
});
}
const convertedContent = convertContentBlocks(msg.content);
// Anthropic rejects tool references mixed with ordinary tool-result content.
return {
toolResult: {
type: "tool_result",
tool_use_id: msg.toolCallId,
content: references.length > 0 ? references : convertedContent,
is_error: msg.isError,
},
siblingContent:
references.length === 0
? []
: typeof convertedContent === "string"
? [{ type: "text", text: convertedContent }]
: convertedContent,
};
}
function convertMessages(
messages: Message[],
model: Model<"anthropic-messages">,
transformedMessages: Message[],
isOAuthToken: boolean,
cacheControl?: CacheControlEphemeral,
allowEmptySignature = false,
deferredToolNames: ReadonlySet<string> = new Set(),
normalizeToolName: (name: string) => string = (name) => name,
): MessageParam[] {
const params: MessageParam[] = [];
// Transform messages for cross-provider compatibility
const transformedMessages = transformMessages(messages, model, normalizeToolCallId);
const loadedToolNames = new Set<string>();
for (let i = 0; i < transformedMessages.length; i++) {
const msg = transformedMessages[i];
@@ -1122,37 +1196,30 @@ function convertMessages(
content: blocks,
});
} else if (msg.role === "toolResult") {
// Collect all consecutive toolResult messages, needed for z.ai Anthropic endpoint
// Collect all consecutive toolResult messages, needed for z.ai Anthropic endpoint.
const toolResults: ContentBlockParam[] = [];
// Add the current tool result
toolResults.push({
type: "tool_result",
tool_use_id: msg.toolCallId,
content: convertContentBlocks(msg.content),
is_error: msg.isError,
});
// Look ahead for consecutive toolResult messages
let j = i + 1;
const siblingContent: ContentBlockParam[] = [];
let j = i;
while (j < transformedMessages.length && transformedMessages[j].role === "toolResult") {
const nextMsg = transformedMessages[j] as ToolResultMessage; // We know it's a toolResult
toolResults.push({
type: "tool_result",
tool_use_id: nextMsg.toolCallId,
content: convertContentBlocks(nextMsg.content),
is_error: nextMsg.isError,
});
const converted = convertToolResult(
transformedMessages[j] as ToolResultMessage,
isOAuthToken,
deferredToolNames,
loadedToolNames,
normalizeToolName,
);
toolResults.push(converted.toolResult);
siblingContent.push(...converted.siblingContent);
j++;
}
// Skip the messages we've already processed
// Skip the messages we've already processed.
i = j - 1;
// Add a single user message with all tool results
// Displaced reference-bearing results must follow every tool_result block.
params.push({
role: "user",
content: toolResults,
content: [...toolResults, ...siblingContent],
});
}
}
@@ -1193,6 +1260,7 @@ function convertTools(
isOAuthToken: boolean,
supportsEagerToolInputStreaming: boolean,
cacheControl?: CacheControlEphemeral,
deferLoading = false,
): Anthropic.Messages.Tool[] {
if (!tools) return [];
@@ -1208,6 +1276,7 @@ function convertTools(
properties: schema.properties ?? {},
required: schema.required ?? [],
},
...(deferLoading ? { defer_loading: true } : {}),
...(cacheControl && index === tools.length - 1 ? { cache_control: cacheControl } : {}),
};
});