diff --git a/packages/ai/CHANGELOG.md b/packages/ai/CHANGELOG.md index 1051de9d..33314368 100644 --- a/packages/ai/CHANGELOG.md +++ b/packages/ai/CHANGELOG.md @@ -11,6 +11,7 @@ - Fixed Amazon Bedrock prompt-cache points for Claude Fable 5 and Claude Sonnet 5 ([#6235](https://github.com/earendil-works/pi/issues/6235)). - Fixed DS4 server context overflow detection for `Prompt has ... tokens, but the configured context size is ... tokens` errors ([#6262](https://github.com/earendil-works/pi/issues/6262)). - Fixed OpenAI Codex WebSocket sessions to rotate cached connections before the backend's 60-minute limit, avoiding connection-limit failures on long sessions ([#6268](https://github.com/earendil-works/pi/issues/6268)). +- Fixed OpenAI Completions and Responses providers to send `(no tool output)` instead of `(see attached image)` when a tool result has empty text and no image content, preventing the model from hallucinating image attachments. - Fixed OpenAI Responses and Azure OpenAI Responses requests to avoid sending `max_output_tokens` values below the provider minimum ([#6265](https://github.com/earendil-works/pi/issues/6265)). - Fixed retry classification for Cloudflare 524 timeout responses ([#6239](https://github.com/earendil-works/pi/issues/6239)). diff --git a/packages/ai/src/api/openai-completions.ts b/packages/ai/src/api/openai-completions.ts index b7cbb00e..c1d9f933 100644 --- a/packages/ai/src/api/openai-completions.ts +++ b/packages/ai/src/api/openai-completions.ts @@ -1033,10 +1033,11 @@ export function convertMessages( // Always send tool result with text (or placeholder if only images) const hasText = textResult.length > 0; + const toolResultText = hasText ? textResult : hasImages ? "(see attached image)" : "(no tool output)"; // Some providers require the 'name' field in tool results const toolResultMsg: ChatCompletionToolMessageParam = { role: "tool", - content: sanitizeSurrogates(hasText ? textResult : "(see attached image)"), + content: sanitizeSurrogates(toolResultText), tool_call_id: toolMsg.toolCallId, }; if (compat.requiresToolResultName && toolMsg.toolName) { diff --git a/packages/ai/src/api/openai-responses-shared.ts b/packages/ai/src/api/openai-responses-shared.ts index 9e58046e..527dea5c 100644 --- a/packages/ai/src/api/openai-responses-shared.ts +++ b/packages/ai/src/api/openai-responses-shared.ts @@ -251,7 +251,7 @@ export function convertResponsesMessages( output = contentParts; } else { - output = sanitizeSurrogates(hasText ? textResult : "(see attached image)"); + output = sanitizeSurrogates(hasText ? textResult : hasImages ? "(see attached image)" : "(no tool output)"); } messages.push({ diff --git a/packages/ai/test/openai-completions-tool-result-images.test.ts b/packages/ai/test/openai-completions-tool-result-images.test.ts index c8500792..31be1667 100644 --- a/packages/ai/test/openai-completions-tool-result-images.test.ts +++ b/packages/ai/test/openai-completions-tool-result-images.test.ts @@ -54,6 +54,17 @@ function buildToolResult(toolCallId: string, timestamp: number): ToolResultMessa }; } +function buildEmptyToolResult(toolCallId: string, timestamp: number): ToolResultMessage { + return { + role: "toolResult", + toolCallId, + toolName: "bash", + content: [{ type: "text", text: "" }], + isError: false, + timestamp, + }; +} + describe("openai-completions convertMessages", () => { it("batches tool-result images after consecutive tool results", () => { const { compat: _compat, ...baseModel } = getModel("openai", "gpt-4o-mini"); @@ -100,4 +111,39 @@ describe("openai-completions convertMessages", () => { ); expect(imageParts.length).toBe(2); }); + + it("uses '(no tool output)' placeholder for empty tool results without images", () => { + const { compat: _compat, ...baseModel } = getModel("openai", "gpt-4o-mini"); + const model: Model<"openai-completions"> = { + ...baseModel, + api: "openai-completions", + input: ["text", "image"], + }; + + const now = Date.now(); + const assistantMessage: AssistantMessage = { + role: "assistant", + content: [{ type: "toolCall", id: "tool-1", name: "bash", arguments: { command: "true" } }], + api: model.api, + provider: model.provider, + model: model.id, + usage: emptyUsage, + stopReason: "toolUse", + timestamp: now, + }; + + const context: Context = { + messages: [ + { role: "user", content: "Run the command", timestamp: now - 1 }, + assistantMessage, + buildEmptyToolResult("tool-1", now + 1), + ], + }; + + const messages = convertMessages(model, context, compat); + const toolMessage = messages.find((m) => m.role === "tool") as { role: "tool"; content: string } | undefined; + expect(toolMessage).toBeTruthy(); + expect(toolMessage?.content).toBe("(no tool output)"); + expect(toolMessage?.content).not.toContain("see attached image"); + }); }); diff --git a/packages/ai/test/openai-responses-empty-tool-result.test.ts b/packages/ai/test/openai-responses-empty-tool-result.test.ts new file mode 100644 index 00000000..0db91237 --- /dev/null +++ b/packages/ai/test/openai-responses-empty-tool-result.test.ts @@ -0,0 +1,58 @@ +import { describe, expect, it } from "vitest"; +import { convertResponsesMessages } from "../src/api/openai-responses-shared.ts"; +import { getModel } from "../src/compat.ts"; +import type { AssistantMessage, Context, ToolResultMessage, Usage } from "../src/types.ts"; + +const usage: Usage = { + input: 0, + output: 0, + cacheRead: 0, + cacheWrite: 0, + totalTokens: 0, + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 }, +}; + +function buildEmptyToolResult(toolCallId: string, timestamp: number): ToolResultMessage { + return { + role: "toolResult", + toolCallId, + toolName: "bash", + content: [{ type: "text", text: "" }], + isError: false, + timestamp, + }; +} + +describe("OpenAI Responses convertResponsesMessages empty tool result", () => { + it("uses '(no tool output)' placeholder for empty tool results without images", () => { + const model = getModel("openai", "gpt-4o-mini"); + const now = Date.now(); + const assistant: AssistantMessage = { + role: "assistant", + content: [{ type: "toolCall", id: "tool-1", name: "bash", arguments: { command: "true" } }], + api: model.api, + provider: model.provider, + model: model.id, + usage, + stopReason: "toolUse", + timestamp: now, + }; + + const context: Context = { + messages: [ + { role: "user", content: "Run the command", timestamp: now - 1 }, + assistant, + buildEmptyToolResult("tool-1", now + 1), + ], + }; + + const input = convertResponsesMessages(model, context, new Set(["openai", "openai-codex", "opencode"])); + const functionCallOutput = input.find((item) => item.type === "function_call_output") as + | { type: "function_call_output"; output: string } + | undefined; + + expect(functionCallOutput).toBeTruthy(); + expect(functionCallOutput?.output).toBe("(no tool output)"); + expect(functionCallOutput?.output).not.toContain("see attached image"); + }); +});