From 279f53b098bc64789a1ad310a63d3473a48eba7b Mon Sep 17 00:00:00 2001 From: Samwise Wang Date: Tue, 7 Jul 2026 03:42:56 +0800 Subject: [PATCH] fix(ai): use "(no tool output)" placeholder for empty tool results without images (#6290) OpenAI Completions and Responses providers unconditionally replaced empty tool result text with "(see attached image)", even when the result had no image content. This caused the model to hallucinate image attachments for commands that produce no output (e.g. curl -s with SSL errors, grep with no matches, true/false). Now matches the Google provider behavior: "(see attached image)" is only used when images are actually present; empty results get "(no tool output)". Co-authored-by: tzwm Co-authored-by: Mario Zechner --- packages/ai/CHANGELOG.md | 1 + packages/ai/src/api/openai-completions.ts | 3 +- .../ai/src/api/openai-responses-shared.ts | 2 +- ...nai-completions-tool-result-images.test.ts | 46 +++++++++++++++ ...openai-responses-empty-tool-result.test.ts | 58 +++++++++++++++++++ 5 files changed, 108 insertions(+), 2 deletions(-) create mode 100644 packages/ai/test/openai-responses-empty-tool-result.test.ts diff --git a/packages/ai/CHANGELOG.md b/packages/ai/CHANGELOG.md index 1051de9d..33314368 100644 --- a/packages/ai/CHANGELOG.md +++ b/packages/ai/CHANGELOG.md @@ -11,6 +11,7 @@ - Fixed Amazon Bedrock prompt-cache points for Claude Fable 5 and Claude Sonnet 5 ([#6235](https://github.com/earendil-works/pi/issues/6235)). - Fixed DS4 server context overflow detection for `Prompt has ... tokens, but the configured context size is ... tokens` errors ([#6262](https://github.com/earendil-works/pi/issues/6262)). - Fixed OpenAI Codex WebSocket sessions to rotate cached connections before the backend's 60-minute limit, avoiding connection-limit failures on long sessions ([#6268](https://github.com/earendil-works/pi/issues/6268)). +- Fixed OpenAI Completions and Responses providers to send `(no tool output)` instead of `(see attached image)` when a tool result has empty text and no image content, preventing the model from hallucinating image attachments. - Fixed OpenAI Responses and Azure OpenAI Responses requests to avoid sending `max_output_tokens` values below the provider minimum ([#6265](https://github.com/earendil-works/pi/issues/6265)). - Fixed retry classification for Cloudflare 524 timeout responses ([#6239](https://github.com/earendil-works/pi/issues/6239)). diff --git a/packages/ai/src/api/openai-completions.ts b/packages/ai/src/api/openai-completions.ts index b7cbb00e..c1d9f933 100644 --- a/packages/ai/src/api/openai-completions.ts +++ b/packages/ai/src/api/openai-completions.ts @@ -1033,10 +1033,11 @@ export function convertMessages( // Always send tool result with text (or placeholder if only images) const hasText = textResult.length > 0; + const toolResultText = hasText ? textResult : hasImages ? "(see attached image)" : "(no tool output)"; // Some providers require the 'name' field in tool results const toolResultMsg: ChatCompletionToolMessageParam = { role: "tool", - content: sanitizeSurrogates(hasText ? textResult : "(see attached image)"), + content: sanitizeSurrogates(toolResultText), tool_call_id: toolMsg.toolCallId, }; if (compat.requiresToolResultName && toolMsg.toolName) { diff --git a/packages/ai/src/api/openai-responses-shared.ts b/packages/ai/src/api/openai-responses-shared.ts index 9e58046e..527dea5c 100644 --- a/packages/ai/src/api/openai-responses-shared.ts +++ b/packages/ai/src/api/openai-responses-shared.ts @@ -251,7 +251,7 @@ export function convertResponsesMessages( output = contentParts; } else { - output = sanitizeSurrogates(hasText ? textResult : "(see attached image)"); + output = sanitizeSurrogates(hasText ? textResult : hasImages ? "(see attached image)" : "(no tool output)"); } messages.push({ diff --git a/packages/ai/test/openai-completions-tool-result-images.test.ts b/packages/ai/test/openai-completions-tool-result-images.test.ts index c8500792..31be1667 100644 --- a/packages/ai/test/openai-completions-tool-result-images.test.ts +++ b/packages/ai/test/openai-completions-tool-result-images.test.ts @@ -54,6 +54,17 @@ function buildToolResult(toolCallId: string, timestamp: number): ToolResultMessa }; } +function buildEmptyToolResult(toolCallId: string, timestamp: number): ToolResultMessage { + return { + role: "toolResult", + toolCallId, + toolName: "bash", + content: [{ type: "text", text: "" }], + isError: false, + timestamp, + }; +} + describe("openai-completions convertMessages", () => { it("batches tool-result images after consecutive tool results", () => { const { compat: _compat, ...baseModel } = getModel("openai", "gpt-4o-mini"); @@ -100,4 +111,39 @@ describe("openai-completions convertMessages", () => { ); expect(imageParts.length).toBe(2); }); + + it("uses '(no tool output)' placeholder for empty tool results without images", () => { + const { compat: _compat, ...baseModel } = getModel("openai", "gpt-4o-mini"); + const model: Model<"openai-completions"> = { + ...baseModel, + api: "openai-completions", + input: ["text", "image"], + }; + + const now = Date.now(); + const assistantMessage: AssistantMessage = { + role: "assistant", + content: [{ type: "toolCall", id: "tool-1", name: "bash", arguments: { command: "true" } }], + api: model.api, + provider: model.provider, + model: model.id, + usage: emptyUsage, + stopReason: "toolUse", + timestamp: now, + }; + + const context: Context = { + messages: [ + { role: "user", content: "Run the command", timestamp: now - 1 }, + assistantMessage, + buildEmptyToolResult("tool-1", now + 1), + ], + }; + + const messages = convertMessages(model, context, compat); + const toolMessage = messages.find((m) => m.role === "tool") as { role: "tool"; content: string } | undefined; + expect(toolMessage).toBeTruthy(); + expect(toolMessage?.content).toBe("(no tool output)"); + expect(toolMessage?.content).not.toContain("see attached image"); + }); }); diff --git a/packages/ai/test/openai-responses-empty-tool-result.test.ts b/packages/ai/test/openai-responses-empty-tool-result.test.ts new file mode 100644 index 00000000..0db91237 --- /dev/null +++ b/packages/ai/test/openai-responses-empty-tool-result.test.ts @@ -0,0 +1,58 @@ +import { describe, expect, it } from "vitest"; +import { convertResponsesMessages } from "../src/api/openai-responses-shared.ts"; +import { getModel } from "../src/compat.ts"; +import type { AssistantMessage, Context, ToolResultMessage, Usage } from "../src/types.ts"; + +const usage: Usage = { + input: 0, + output: 0, + cacheRead: 0, + cacheWrite: 0, + totalTokens: 0, + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 }, +}; + +function buildEmptyToolResult(toolCallId: string, timestamp: number): ToolResultMessage { + return { + role: "toolResult", + toolCallId, + toolName: "bash", + content: [{ type: "text", text: "" }], + isError: false, + timestamp, + }; +} + +describe("OpenAI Responses convertResponsesMessages empty tool result", () => { + it("uses '(no tool output)' placeholder for empty tool results without images", () => { + const model = getModel("openai", "gpt-4o-mini"); + const now = Date.now(); + const assistant: AssistantMessage = { + role: "assistant", + content: [{ type: "toolCall", id: "tool-1", name: "bash", arguments: { command: "true" } }], + api: model.api, + provider: model.provider, + model: model.id, + usage, + stopReason: "toolUse", + timestamp: now, + }; + + const context: Context = { + messages: [ + { role: "user", content: "Run the command", timestamp: now - 1 }, + assistant, + buildEmptyToolResult("tool-1", now + 1), + ], + }; + + const input = convertResponsesMessages(model, context, new Set(["openai", "openai-codex", "opencode"])); + const functionCallOutput = input.find((item) => item.type === "function_call_output") as + | { type: "function_call_output"; output: string } + | undefined; + + expect(functionCallOutput).toBeTruthy(); + expect(functionCallOutput?.output).toBe("(no tool output)"); + expect(functionCallOutput?.output).not.toContain("see attached image"); + }); +});