@@ -809,7 +809,7 @@ function addCacheControlToLastConversationMessage(
|
|||||||
): void {
|
): void {
|
||||||
for (let i = messages.length - 1; i >= 0; i--) {
|
for (let i = messages.length - 1; i >= 0; i--) {
|
||||||
const message = messages[i];
|
const message = messages[i];
|
||||||
if (message.role === "user" || message.role === "assistant") {
|
if (message.role === "user" || message.role === "assistant" || message.role === "tool") {
|
||||||
if (addCacheControlToMessage(message, cacheControl)) {
|
if (addCacheControlToMessage(message, cacheControl)) {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
@@ -840,7 +840,7 @@ function addCacheControlToMessage(
|
|||||||
message: ChatCompletionMessageParam,
|
message: ChatCompletionMessageParam,
|
||||||
cacheControl: OpenAICompatCacheControl,
|
cacheControl: OpenAICompatCacheControl,
|
||||||
): boolean {
|
): boolean {
|
||||||
if (message.role === "user" || message.role === "assistant") {
|
if (message.role === "user" || message.role === "assistant" || message.role === "tool") {
|
||||||
return addCacheControlToTextContent(message, cacheControl);
|
return addCacheControlToTextContent(message, cacheControl);
|
||||||
}
|
}
|
||||||
return false;
|
return false;
|
||||||
@@ -850,6 +850,7 @@ function addCacheControlToTextContent(
|
|||||||
message:
|
message:
|
||||||
| ChatCompletionInstructionMessageParam
|
| ChatCompletionInstructionMessageParam
|
||||||
| ChatCompletionAssistantMessageParam
|
| ChatCompletionAssistantMessageParam
|
||||||
|
| ChatCompletionToolMessageParam
|
||||||
| Extract<ChatCompletionMessageParam, { role: "user" }>,
|
| Extract<ChatCompletionMessageParam, { role: "user" }>,
|
||||||
cacheControl: OpenAICompatCacheControl,
|
cacheControl: OpenAICompatCacheControl,
|
||||||
): boolean {
|
): boolean {
|
||||||
|
|||||||
@@ -2,7 +2,7 @@ import { Type } from "typebox";
|
|||||||
import { beforeEach, describe, expect, it, vi } from "vitest";
|
import { beforeEach, describe, expect, it, vi } from "vitest";
|
||||||
import { stream as streamOpenAICompletions } from "../src/api/openai-completions.ts";
|
import { stream as streamOpenAICompletions } from "../src/api/openai-completions.ts";
|
||||||
import { getModel } from "../src/compat.ts";
|
import { getModel } from "../src/compat.ts";
|
||||||
import type { Model } from "../src/types.ts";
|
import type { Message, Model } from "../src/types.ts";
|
||||||
|
|
||||||
interface CacheControl {
|
interface CacheControl {
|
||||||
type: "ephemeral";
|
type: "ephemeral";
|
||||||
@@ -74,6 +74,7 @@ vi.mock("openai", () => {
|
|||||||
async function capturePayload(
|
async function capturePayload(
|
||||||
model: Model<"openai-completions">,
|
model: Model<"openai-completions">,
|
||||||
options?: { cacheRetention?: "none" | "short" | "long" },
|
options?: { cacheRetention?: "none" | "short" | "long" },
|
||||||
|
messages?: Message[],
|
||||||
): Promise<CapturedParams> {
|
): Promise<CapturedParams> {
|
||||||
const timestamp = Date.now();
|
const timestamp = Date.now();
|
||||||
|
|
||||||
@@ -81,7 +82,7 @@ async function capturePayload(
|
|||||||
model,
|
model,
|
||||||
{
|
{
|
||||||
systemPrompt: "System prompt",
|
systemPrompt: "System prompt",
|
||||||
messages: [{ role: "user", content: "Hello", timestamp }],
|
messages: messages ?? [{ role: "user", content: "Hello", timestamp }],
|
||||||
tools: [
|
tools: [
|
||||||
{
|
{
|
||||||
name: "read",
|
name: "read",
|
||||||
@@ -158,6 +159,47 @@ describe("openai-completions cacheControlFormat", () => {
|
|||||||
expectAnthropicCacheMarkers(params);
|
expectAnthropicCacheMarkers(params);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it("moves the conversation cache marker to a tool result", async () => {
|
||||||
|
const model = getModel("openrouter", "anthropic/claude-sonnet-4");
|
||||||
|
const timestamp = Date.now();
|
||||||
|
const params = await capturePayload(model, undefined, [
|
||||||
|
{ role: "user", content: "Read the file", timestamp },
|
||||||
|
{
|
||||||
|
role: "assistant",
|
||||||
|
content: [{ type: "toolCall", id: "call_1", name: "read", arguments: { path: "README.md" } }],
|
||||||
|
api: "openai-completions",
|
||||||
|
provider: "openrouter",
|
||||||
|
model: model.id,
|
||||||
|
usage: {
|
||||||
|
input: 0,
|
||||||
|
output: 0,
|
||||||
|
cacheRead: 0,
|
||||||
|
cacheWrite: 0,
|
||||||
|
totalTokens: 0,
|
||||||
|
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 },
|
||||||
|
},
|
||||||
|
stopReason: "toolUse",
|
||||||
|
timestamp,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
role: "toolResult",
|
||||||
|
toolCallId: "call_1",
|
||||||
|
toolName: "read",
|
||||||
|
content: [{ type: "text", text: "file contents" }],
|
||||||
|
isError: false,
|
||||||
|
timestamp,
|
||||||
|
},
|
||||||
|
]);
|
||||||
|
|
||||||
|
const userMessage = params.messages.find((message) => message.role === "user");
|
||||||
|
expect(userMessage?.content).toBe("Read the file");
|
||||||
|
|
||||||
|
const toolMessage = params.messages[params.messages.length - 1];
|
||||||
|
expect(toolMessage.role).toBe("tool");
|
||||||
|
expect(Array.isArray(toolMessage.content)).toBe(true);
|
||||||
|
expect((toolMessage.content as TextPart[])[0]?.cache_control).toEqual({ type: "ephemeral" });
|
||||||
|
});
|
||||||
|
|
||||||
it("omits Anthropic-style cache markers when cacheRetention is none", async () => {
|
it("omits Anthropic-style cache markers when cacheRetention is none", async () => {
|
||||||
const model: Model<"openai-completions"> = {
|
const model: Model<"openai-completions"> = {
|
||||||
id: "custom-qwen",
|
id: "custom-qwen",
|
||||||
|
|||||||
Reference in New Issue
Block a user