fix(ai): cache OpenRouter tool results

closes #6940
This commit is contained in:
Matthew
2026-07-22 02:48:33 -07:00
committed by GitHub
parent 6f95a33888
commit bc41f612da
2 changed files with 47 additions and 4 deletions
+3 -2
View File
@@ -809,7 +809,7 @@ function addCacheControlToLastConversationMessage(
): void { ): void {
for (let i = messages.length - 1; i >= 0; i--) { for (let i = messages.length - 1; i >= 0; i--) {
const message = messages[i]; const message = messages[i];
if (message.role === "user" || message.role === "assistant") { if (message.role === "user" || message.role === "assistant" || message.role === "tool") {
if (addCacheControlToMessage(message, cacheControl)) { if (addCacheControlToMessage(message, cacheControl)) {
return; return;
} }
@@ -840,7 +840,7 @@ function addCacheControlToMessage(
message: ChatCompletionMessageParam, message: ChatCompletionMessageParam,
cacheControl: OpenAICompatCacheControl, cacheControl: OpenAICompatCacheControl,
): boolean { ): boolean {
if (message.role === "user" || message.role === "assistant") { if (message.role === "user" || message.role === "assistant" || message.role === "tool") {
return addCacheControlToTextContent(message, cacheControl); return addCacheControlToTextContent(message, cacheControl);
} }
return false; return false;
@@ -850,6 +850,7 @@ function addCacheControlToTextContent(
message: message:
| ChatCompletionInstructionMessageParam | ChatCompletionInstructionMessageParam
| ChatCompletionAssistantMessageParam | ChatCompletionAssistantMessageParam
| ChatCompletionToolMessageParam
| Extract<ChatCompletionMessageParam, { role: "user" }>, | Extract<ChatCompletionMessageParam, { role: "user" }>,
cacheControl: OpenAICompatCacheControl, cacheControl: OpenAICompatCacheControl,
): boolean { ): boolean {
@@ -2,7 +2,7 @@ import { Type } from "typebox";
import { beforeEach, describe, expect, it, vi } from "vitest"; import { beforeEach, describe, expect, it, vi } from "vitest";
import { stream as streamOpenAICompletions } from "../src/api/openai-completions.ts"; import { stream as streamOpenAICompletions } from "../src/api/openai-completions.ts";
import { getModel } from "../src/compat.ts"; import { getModel } from "../src/compat.ts";
import type { Model } from "../src/types.ts"; import type { Message, Model } from "../src/types.ts";
interface CacheControl { interface CacheControl {
type: "ephemeral"; type: "ephemeral";
@@ -74,6 +74,7 @@ vi.mock("openai", () => {
async function capturePayload( async function capturePayload(
model: Model<"openai-completions">, model: Model<"openai-completions">,
options?: { cacheRetention?: "none" | "short" | "long" }, options?: { cacheRetention?: "none" | "short" | "long" },
messages?: Message[],
): Promise<CapturedParams> { ): Promise<CapturedParams> {
const timestamp = Date.now(); const timestamp = Date.now();
@@ -81,7 +82,7 @@ async function capturePayload(
model, model,
{ {
systemPrompt: "System prompt", systemPrompt: "System prompt",
messages: [{ role: "user", content: "Hello", timestamp }], messages: messages ?? [{ role: "user", content: "Hello", timestamp }],
tools: [ tools: [
{ {
name: "read", name: "read",
@@ -158,6 +159,47 @@ describe("openai-completions cacheControlFormat", () => {
expectAnthropicCacheMarkers(params); expectAnthropicCacheMarkers(params);
}); });
it("moves the conversation cache marker to a tool result", async () => {
const model = getModel("openrouter", "anthropic/claude-sonnet-4");
const timestamp = Date.now();
const params = await capturePayload(model, undefined, [
{ role: "user", content: "Read the file", timestamp },
{
role: "assistant",
content: [{ type: "toolCall", id: "call_1", name: "read", arguments: { path: "README.md" } }],
api: "openai-completions",
provider: "openrouter",
model: model.id,
usage: {
input: 0,
output: 0,
cacheRead: 0,
cacheWrite: 0,
totalTokens: 0,
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 },
},
stopReason: "toolUse",
timestamp,
},
{
role: "toolResult",
toolCallId: "call_1",
toolName: "read",
content: [{ type: "text", text: "file contents" }],
isError: false,
timestamp,
},
]);
const userMessage = params.messages.find((message) => message.role === "user");
expect(userMessage?.content).toBe("Read the file");
const toolMessage = params.messages[params.messages.length - 1];
expect(toolMessage.role).toBe("tool");
expect(Array.isArray(toolMessage.content)).toBe(true);
expect((toolMessage.content as TextPart[])[0]?.cache_control).toEqual({ type: "ephemeral" });
});
it("omits Anthropic-style cache markers when cacheRetention is none", async () => { it("omits Anthropic-style cache markers when cacheRetention is none", async () => {
const model: Model<"openai-completions"> = { const model: Model<"openai-completions"> = {
id: "custom-qwen", id: "custom-qwen",