diff --git a/packages/coding-agent/CHANGELOG.md b/packages/coding-agent/CHANGELOG.md index b48d9716..2ffc8c2e 100644 --- a/packages/coding-agent/CHANGELOG.md +++ b/packages/coding-agent/CHANGELOG.md @@ -5,6 +5,7 @@ ### Fixed - Fixed MiniMax (`minimax`, `minimax-cn`) sessions failing on long conversations with `unknown error, 999` or `context window exceeds limit (2013)` by clamping `max_tokens` for shared-budget MiniMax models ([#6061](https://github.com/earendil-works/pi/issues/6061)). +- Fixed disk BMP image files to be detected, converted to PNG, and attached through `read` and CLI `@file` inputs ([#6047](https://github.com/earendil-works/pi/issues/6047)). - Fixed auto-retry for provider stream errors that explicitly tell callers to retry the request ([#6019](https://github.com/earendil-works/pi/issues/6019)). ## [0.80.2] - 2026-06-23 diff --git a/packages/coding-agent/src/cli/file-processor.ts b/packages/coding-agent/src/cli/file-processor.ts index e1da052a..4b3bf0e2 100644 --- a/packages/coding-agent/src/cli/file-processor.ts +++ b/packages/coding-agent/src/cli/file-processor.ts @@ -7,7 +7,7 @@ import type { ImageContent } from "@earendil-works/pi-ai"; import chalk from "chalk"; import { resolve } from "path"; import { resolveReadPath } from "../core/tools/path-utils.ts"; -import { formatDimensionNote, resizeImage } from "../utils/image-resize.ts"; +import { processImage } from "../utils/image-process.ts"; import { detectSupportedImageMimeTypeFromFile } from "../utils/mime.ts"; export interface ProcessedFiles { @@ -50,35 +50,23 @@ export async function processFileArguments(fileArgs: string[], options?: Process if (mimeType) { // Handle image file const content = await readFile(absolutePath); + const processed = await processImage(content, mimeType, { autoResizeImages }); - let attachment: ImageContent; - let dimensionNote: string | undefined; - - if (autoResizeImages) { - const resized = await resizeImage(content, mimeType); - if (!resized) { - text += `[Image omitted: could not be resized below the inline image size limit.]\n`; - continue; - } - dimensionNote = formatDimensionNote(resized); - attachment = { - type: "image", - mimeType: resized.mimeType, - data: resized.data, - }; - } else { - attachment = { - type: "image", - mimeType, - data: content.toString("base64"), - }; + if (!processed.ok) { + text += `${processed.message}\n`; + continue; } + const attachment: ImageContent = { + type: "image", + mimeType: processed.mimeType, + data: processed.data, + }; images.push(attachment); - // Add text reference to image with optional dimension note - if (dimensionNote) { - text += `${dimensionNote}\n`; + // Add text reference to image with optional processing hints + if (processed.hints.length > 0) { + text += `${processed.hints.join("\n")}\n`; } else { text += `\n`; } diff --git a/packages/coding-agent/src/core/tools/read.ts b/packages/coding-agent/src/core/tools/read.ts index 52e87b1d..3a4dcecc 100644 --- a/packages/coding-agent/src/core/tools/read.ts +++ b/packages/coding-agent/src/core/tools/read.ts @@ -8,7 +8,7 @@ import { type Static, Type } from "typebox"; import { getReadmePath } from "../../config.ts"; import { keyHint, keyText } from "../../modes/interactive/components/keybinding-hints.ts"; import { getLanguageFromPath, highlightCode, type Theme } from "../../modes/interactive/theme/theme.ts"; -import { formatDimensionNote, resizeImage } from "../../utils/image-resize.ts"; +import { processImage } from "../../utils/image-process.ts"; import { detectSupportedImageMimeTypeFromFile } from "../../utils/mime.ts"; import { formatPathRelativeToCwdOrAbsolute } from "../../utils/paths.ts"; import type { ToolDefinition, ToolRenderResultOptions } from "../extensions/types.ts"; @@ -209,7 +209,7 @@ export function createReadToolDefinition( return { name: "read", label: "read", - description: `Read the contents of a file. Supports text files and images (jpg, png, gif, webp). Images are sent as attachments. For text files, output is truncated to ${DEFAULT_MAX_LINES} lines or ${DEFAULT_MAX_BYTES / 1024}KB (whichever is hit first). Use offset/limit for large files. When you need the full file, continue with offset until complete.`, + description: `Read the contents of a file. Supports text files and images (jpg, png, gif, webp, bmp). Images are sent as attachments. For text files, output is truncated to ${DEFAULT_MAX_LINES} lines or ${DEFAULT_MAX_BYTES / 1024}KB (whichever is hit first). Use offset/limit for large files. When you need the full file, continue with offset until complete.`, promptSnippet: "Read file contents", promptGuidelines: ["Use read to examine files instead of cat or sed."], parameters: readSchema, @@ -247,29 +247,18 @@ export function createReadToolDefinition( if (mimeType) { // Read image as binary. const buffer = await ops.readFile(absolutePath); - if (autoResizeImages) { - // Resize image if needed before sending it back to the model. - const resized = await resizeImage(buffer, mimeType); - if (!resized) { - let textNote = `Read image file [${mimeType}]\n[Image omitted: could not be resized below the inline image size limit.]`; - if (nonVisionImageNote) textNote += `\n${nonVisionImageNote}`; - content = [{ type: "text", text: textNote }]; - } else { - const dimensionNote = formatDimensionNote(resized); - let textNote = `Read image file [${resized.mimeType}]`; - if (dimensionNote) textNote += `\n${dimensionNote}`; - if (nonVisionImageNote) textNote += `\n${nonVisionImageNote}`; - content = [ - { type: "text", text: textNote }, - { type: "image", data: resized.data, mimeType: resized.mimeType }, - ]; - } + const processed = await processImage(buffer, mimeType, { autoResizeImages }); + if (!processed.ok) { + let textNote = `Read image file [${mimeType}]\n${processed.message}`; + if (nonVisionImageNote) textNote += `\n${nonVisionImageNote}`; + content = [{ type: "text", text: textNote }]; } else { - let textNote = `Read image file [${mimeType}]`; + let textNote = `Read image file [${processed.mimeType}]`; + if (processed.hints.length > 0) textNote += `\n${processed.hints.join("\n")}`; if (nonVisionImageNote) textNote += `\n${nonVisionImageNote}`; content = [ { type: "text", text: textNote }, - { type: "image", data: buffer.toString("base64"), mimeType }, + { type: "image", data: processed.data, mimeType: processed.mimeType }, ]; } } else { diff --git a/packages/coding-agent/src/utils/image-convert.ts b/packages/coding-agent/src/utils/image-convert.ts index 4d5f0204..f781d53d 100644 --- a/packages/coding-agent/src/utils/image-convert.ts +++ b/packages/coding-agent/src/utils/image-convert.ts @@ -1,6 +1,28 @@ import { applyExifOrientation } from "./exif-orientation.ts"; import { loadPhoton } from "./photon.ts"; +export async function convertImageBytesToPng(bytes: Uint8Array): Promise { + const photon = await loadPhoton(); + if (!photon) { + // Photon not available, can't convert + return null; + } + + try { + const rawImage = photon.PhotonImage.new_from_byteslice(bytes); + const image = applyExifOrientation(photon, rawImage, bytes); + if (image !== rawImage) rawImage.free(); + try { + return new Uint8Array(image.get_bytes()); + } finally { + image.free(); + } + } catch { + // Conversion failed + return null; + } +} + /** * Convert image to PNG format for terminal display. * Kitty graphics protocol requires PNG format (f=100). @@ -14,28 +36,14 @@ export async function convertToPng( return { data: base64Data, mimeType }; } - const photon = await loadPhoton(); - if (!photon) { - // Photon not available, can't convert + const bytes = new Uint8Array(Buffer.from(base64Data, "base64")); + const pngBytes = await convertImageBytesToPng(bytes); + if (!pngBytes) { return null; } - try { - const bytes = new Uint8Array(Buffer.from(base64Data, "base64")); - const rawImage = photon.PhotonImage.new_from_byteslice(bytes); - const image = applyExifOrientation(photon, rawImage, bytes); - if (image !== rawImage) rawImage.free(); - try { - const pngBuffer = image.get_bytes(); - return { - data: Buffer.from(pngBuffer).toString("base64"), - mimeType: "image/png", - }; - } finally { - image.free(); - } - } catch { - // Conversion failed - return null; - } + return { + data: Buffer.from(pngBytes).toString("base64"), + mimeType: "image/png", + }; } diff --git a/packages/coding-agent/src/utils/image-process.ts b/packages/coding-agent/src/utils/image-process.ts new file mode 100644 index 00000000..a461c8b2 --- /dev/null +++ b/packages/coding-agent/src/utils/image-process.ts @@ -0,0 +1,119 @@ +import { convertImageBytesToPng } from "./image-convert.ts"; +import { formatDimensionNote, type ImageResizeOptions, resizeImage } from "./image-resize.ts"; + +export interface ProcessImageOptions { + /** Whether to resize images to inline provider limits. Default: true */ + autoResizeImages?: boolean; + /** Optional resize overrides. Uses resizeImage defaults when omitted. */ + resizeOptions?: ImageResizeOptions; +} + +export type ProcessImageResult = + | { + ok: true; + data: string; + mimeType: string; + hints: string[]; + } + | { + ok: false; + message: string; + }; + +interface NormalizedImage { + bytes: Uint8Array; + mimeType: string; + convertedFrom?: string; +} + +function baseMimeType(mimeType: string): string { + return mimeType.split(";")[0]?.trim().toLowerCase() ?? mimeType.toLowerCase(); +} + +function normalizeSupportedImageMimeType(mimeType: string): string | null { + switch (baseMimeType(mimeType)) { + case "image/png": + return "image/png"; + case "image/jpeg": + case "image/jpg": + return "image/jpeg"; + case "image/gif": + return "image/gif"; + case "image/webp": + return "image/webp"; + default: + return null; + } +} + +async function normalizeImage(bytes: Uint8Array, mimeType: string): Promise { + const normalizedMimeType = normalizeSupportedImageMimeType(mimeType); + if (normalizedMimeType) { + return { bytes, mimeType: normalizedMimeType }; + } + + const pngBytes = await convertImageBytesToPng(bytes); + if (!pngBytes) { + return null; + } + + return { + bytes: pngBytes, + mimeType: "image/png", + convertedFrom: baseMimeType(mimeType), + }; +} + +function conversionHint(from: string | undefined, to: string): string | undefined { + if (!from || from === to) return undefined; + return `[Image converted from ${from} to ${to}.]`; +} + +export async function processImage( + bytes: Uint8Array, + mimeType: string, + options?: ProcessImageOptions, +): Promise { + const autoResizeImages = options?.autoResizeImages ?? true; + const normalized = await normalizeImage(bytes, mimeType); + if (!normalized) { + return { + ok: false, + message: "[Image omitted: could not be converted to a supported inline image format.]", + }; + } + + if (autoResizeImages) { + const resized = await resizeImage(normalized.bytes, normalized.mimeType, options?.resizeOptions); + if (!resized) { + return { + ok: false, + message: "[Image omitted: could not be resized below the inline image size limit.]", + }; + } + + const hints: string[] = []; + const convertedHint = conversionHint(normalized.convertedFrom, resized.mimeType); + if (convertedHint) hints.push(convertedHint); + const dimensionNote = formatDimensionNote(resized); + if (dimensionNote) hints.push(dimensionNote); + + return { + ok: true, + data: resized.data, + mimeType: resized.mimeType, + hints, + }; + } + + const hints: string[] = []; + const convertedHint = conversionHint(normalized.convertedFrom, normalized.mimeType); + if (convertedHint) hints.push(convertedHint); + + return { + ok: true, + data: Buffer.from(normalized.bytes).toString("base64"), + mimeType: normalized.mimeType, + hints, + }; +} diff --git a/packages/coding-agent/src/utils/mime.ts b/packages/coding-agent/src/utils/mime.ts index 7381d370..c68f378d 100644 --- a/packages/coding-agent/src/utils/mime.ts +++ b/packages/coding-agent/src/utils/mime.ts @@ -16,6 +16,9 @@ export function detectSupportedImageMimeType(buffer: Uint8Array): string | null if (startsWithAscii(buffer, 0, "RIFF") && startsWithAscii(buffer, 8, "WEBP")) { return "image/webp"; } + if (startsWithAscii(buffer, 0, "BM") && isBmp(buffer)) { + return "image/bmp"; + } return null; } @@ -51,6 +54,36 @@ function isAnimatedPng(buffer: Uint8Array): boolean { return false; } +function isBmp(buffer: Uint8Array): boolean { + if (buffer.length < 26) return false; + + const declaredFileSize = readUint32LE(buffer, 2); + const pixelDataOffset = readUint32LE(buffer, 10); + const dibHeaderSize = readUint32LE(buffer, 14); + if (declaredFileSize !== 0 && declaredFileSize < 26) return false; + if (pixelDataOffset < 14 + dibHeaderSize) return false; + if (declaredFileSize !== 0 && pixelDataOffset >= declaredFileSize) return false; + + let colorPlanes: number; + let bitsPerPixel: number; + if (dibHeaderSize === 12) { + colorPlanes = readUint16LE(buffer, 22); + bitsPerPixel = readUint16LE(buffer, 24); + } else if (dibHeaderSize >= 40 && dibHeaderSize <= 124) { + if (buffer.length < 30) return false; + colorPlanes = readUint16LE(buffer, 26); + bitsPerPixel = readUint16LE(buffer, 28); + } else { + return false; + } + + return colorPlanes === 1 && [1, 4, 8, 16, 24, 32].includes(bitsPerPixel); +} + +function readUint16LE(buffer: Uint8Array, offset: number): number { + return (buffer[offset] ?? 0) + ((buffer[offset + 1] ?? 0) << 8); +} + function readUint32BE(buffer: Uint8Array, offset: number): number { return ( (buffer[offset] ?? 0) * 0x1000000 + @@ -60,6 +93,15 @@ function readUint32BE(buffer: Uint8Array, offset: number): number { ); } +function readUint32LE(buffer: Uint8Array, offset: number): number { + return ( + (buffer[offset] ?? 0) + + ((buffer[offset + 1] ?? 0) << 8) + + ((buffer[offset + 2] ?? 0) << 16) + + (buffer[offset + 3] ?? 0) * 0x1000000 + ); +} + function startsWith(buffer: Uint8Array, bytes: number[]): boolean { if (buffer.length < bytes.length) return false; return bytes.every((byte, index) => buffer[index] === byte); diff --git a/packages/coding-agent/test/block-images.test.ts b/packages/coding-agent/test/block-images.test.ts index 19b43459..935d854d 100644 --- a/packages/coding-agent/test/block-images.test.ts +++ b/packages/coding-agent/test/block-images.test.ts @@ -10,6 +10,22 @@ import { createReadTool } from "../src/core/tools/read.ts"; const TINY_PNG_BASE64 = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8DwHwAFBQIAX8jx0gAAAABJRU5ErkJggg=="; +function createTinyBmp1x1Red24bpp(): Buffer { + const buffer = Buffer.alloc(58); + buffer.write("BM", 0, "ascii"); + buffer.writeUInt32LE(buffer.length, 2); + buffer.writeUInt32LE(54, 10); + buffer.writeUInt32LE(40, 14); + buffer.writeInt32LE(1, 18); + buffer.writeInt32LE(1, 22); + buffer.writeUInt16LE(1, 26); + buffer.writeUInt16LE(24, 28); + buffer.writeUInt32LE(0, 30); + buffer.writeUInt32LE(4, 34); + buffer[56] = 0xff; + return buffer; +} + describe("blockImages setting", () => { describe("SettingsManager", () => { it("should default blockImages to false", () => { @@ -106,6 +122,18 @@ describe("blockImages setting", () => { expect(result.images[0].type).toBe("image"); }); + it("should process BMP images from disk as PNG attachments", async () => { + const imagePath = join(testDir, "test.bmp"); + writeFileSync(imagePath, createTinyBmp1x1Red24bpp()); + + const result = await processFileArguments([imagePath]); + + expect(result.images).toHaveLength(1); + expect(result.images[0].type).toBe("image"); + expect(result.images[0].mimeType).toBe("image/png"); + expect(result.text).toContain("[Image converted from image/bmp to image/png.]"); + }); + it("should process text files normally", async () => { // Create test text file const textPath = join(testDir, "test.txt"); diff --git a/packages/coding-agent/test/image-process.test.ts b/packages/coding-agent/test/image-process.test.ts new file mode 100644 index 00000000..e65ebfb7 --- /dev/null +++ b/packages/coding-agent/test/image-process.test.ts @@ -0,0 +1,53 @@ +import { describe, expect, it } from "vitest"; +import { processImage } from "../src/utils/image-process.ts"; +import { detectSupportedImageMimeType } from "../src/utils/mime.ts"; + +function createTinyBmp1x1Red24bpp(): Buffer { + const buffer = Buffer.alloc(58); + buffer.write("BM", 0, "ascii"); + buffer.writeUInt32LE(buffer.length, 2); + buffer.writeUInt32LE(54, 10); + buffer.writeUInt32LE(40, 14); + buffer.writeInt32LE(1, 18); + buffer.writeInt32LE(1, 22); + buffer.writeUInt16LE(1, 26); + buffer.writeUInt16LE(24, 28); + buffer.writeUInt32LE(0, 30); + buffer.writeUInt32LE(4, 34); + buffer[56] = 0xff; + return buffer; +} + +function expectPngMagic(base64Data: string): void { + const buffer = Buffer.from(base64Data, "base64"); + expect(buffer[0]).toBe(0x89); + expect(buffer[1]).toBe(0x50); + expect(buffer[2]).toBe(0x4e); + expect(buffer[3]).toBe(0x47); +} + +describe("image processing pipeline", () => { + it("detects BMP files from magic bytes", () => { + expect(detectSupportedImageMimeType(createTinyBmp1x1Red24bpp())).toBe("image/bmp"); + }); + + it("converts BMP files to PNG attachments when auto-resize is disabled", async () => { + const result = await processImage(createTinyBmp1x1Red24bpp(), "image/bmp", { autoResizeImages: false }); + + expect(result.ok).toBe(true); + if (!result.ok) return; + expect(result.mimeType).toBe("image/png"); + expect(result.hints).toContain("[Image converted from image/bmp to image/png.]"); + expectPngMagic(result.data); + }); + + it("converts BMP files before auto-resizing", async () => { + const result = await processImage(createTinyBmp1x1Red24bpp(), "image/bmp"); + + expect(result.ok).toBe(true); + if (!result.ok) return; + expect(result.mimeType).toBe("image/png"); + expect(result.hints).toContain("[Image converted from image/bmp to image/png.]"); + expectPngMagic(result.data); + }); +}); diff --git a/packages/coding-agent/test/tools.test.ts b/packages/coding-agent/test/tools.test.ts index f8b09c90..63b7f626 100644 --- a/packages/coding-agent/test/tools.test.ts +++ b/packages/coding-agent/test/tools.test.ts @@ -34,6 +34,22 @@ function getTextOutput(result: any): string { ); } +function createTinyBmp1x1Red24bpp(): Buffer { + const buffer = Buffer.alloc(58); + buffer.write("BM", 0, "ascii"); + buffer.writeUInt32LE(buffer.length, 2); + buffer.writeUInt32LE(54, 10); + buffer.writeUInt32LE(40, 14); + buffer.writeInt32LE(1, 18); + buffer.writeInt32LE(1, 22); + buffer.writeUInt16LE(1, 26); + buffer.writeUInt16LE(24, 28); + buffer.writeUInt32LE(0, 30); + buffer.writeUInt32LE(4, 34); + buffer[56] = 0xff; + return buffer; +} + describe("Coding Agent Tools", () => { let testDir: string; @@ -191,6 +207,24 @@ describe("Coding Agent Tools", () => { expect((imageBlock?.data ?? "").length).toBeGreaterThan(0); }); + it("should read BMP files from disk as PNG image attachments", async () => { + const testFile = join(testDir, "image.bmp"); + writeFileSync(testFile, createTinyBmp1x1Red24bpp()); + + const result = await readTool.execute("test-call-img-bmp", { path: testFile }); + + expect(result.content[0]?.type).toBe("text"); + expect(getTextOutput(result)).toContain("Read image file [image/png]"); + expect(getTextOutput(result)).toContain("[Image converted from image/bmp to image/png.]"); + + const imageBlock = result.content.find( + (c): c is { type: "image"; mimeType: string; data: string } => c.type === "image", + ); + expect(imageBlock).toBeDefined(); + expect(imageBlock?.mimeType).toBe("image/png"); + expect(Buffer.from(imageBlock?.data ?? "", "base64")[0]).toBe(0x89); + }); + it("should treat files with image extension but non-image content as text", async () => { const testFile = join(testDir, "not-an-image.png"); writeFileSync(testFile, "definitely not a png");