fix(coding-agent): process BMP images from disk

closes #6047
2026-06-25 12:51:21 +02:00
parent 6ca7ba7c05
commit 4cc339f58d
9 changed files with 329 additions and 67 deletions
@@ -1,6 +1,28 @@
 import { applyExifOrientation } from "./exif-orientation.ts";
 import { loadPhoton } from "./photon.ts";

+export async function convertImageBytesToPng(bytes: Uint8Array): Promise<Uint8Array | null> {
+	const photon = await loadPhoton();
+	if (!photon) {
+		// Photon not available, can't convert
+		return null;
+	}
+
+	try {
+		const rawImage = photon.PhotonImage.new_from_byteslice(bytes);
+		const image = applyExifOrientation(photon, rawImage, bytes);
+		if (image !== rawImage) rawImage.free();
+		try {
+			return new Uint8Array(image.get_bytes());
+		} finally {
+			image.free();
+		}
+	} catch {
+		// Conversion failed
+		return null;
+	}
+}
+
 /**
 * Convert image to PNG format for terminal display.
 * Kitty graphics protocol requires PNG format (f=100).
@@ -14,28 +36,14 @@ export async function convertToPng(
 		return { data: base64Data, mimeType };
 	}

-	const photon = await loadPhoton();
-	if (!photon) {
-		// Photon not available, can't convert
+	const bytes = new Uint8Array(Buffer.from(base64Data, "base64"));
+	const pngBytes = await convertImageBytesToPng(bytes);
+	if (!pngBytes) {
 		return null;
 	}

-	try {
-		const bytes = new Uint8Array(Buffer.from(base64Data, "base64"));
-		const rawImage = photon.PhotonImage.new_from_byteslice(bytes);
-		const image = applyExifOrientation(photon, rawImage, bytes);
-		if (image !== rawImage) rawImage.free();
-		try {
-			const pngBuffer = image.get_bytes();
-			return {
-				data: Buffer.from(pngBuffer).toString("base64"),
-				mimeType: "image/png",
-			};
-		} finally {
-			image.free();
-		}
-	} catch {
-		// Conversion failed
-		return null;
-	}
+	return {
+		data: Buffer.from(pngBytes).toString("base64"),
+		mimeType: "image/png",
+	};
 }
@@ -0,0 +1,119 @@
+import { convertImageBytesToPng } from "./image-convert.ts";
+import { formatDimensionNote, type ImageResizeOptions, resizeImage } from "./image-resize.ts";
+
+export interface ProcessImageOptions {
+	/** Whether to resize images to inline provider limits. Default: true */
+	autoResizeImages?: boolean;
+	/** Optional resize overrides. Uses resizeImage defaults when omitted. */
+	resizeOptions?: ImageResizeOptions;
+}
+
+export type ProcessImageResult =
+	| {
+			ok: true;
+			data: string;
+			mimeType: string;
+			hints: string[];
+	  }
+	| {
+			ok: false;
+			message: string;
+	  };
+
+interface NormalizedImage {
+	bytes: Uint8Array;
+	mimeType: string;
+	convertedFrom?: string;
+}
+
+function baseMimeType(mimeType: string): string {
+	return mimeType.split(";")[0]?.trim().toLowerCase() ?? mimeType.toLowerCase();
+}
+
+function normalizeSupportedImageMimeType(mimeType: string): string | null {
+	switch (baseMimeType(mimeType)) {
+		case "image/png":
+			return "image/png";
+		case "image/jpeg":
+		case "image/jpg":
+			return "image/jpeg";
+		case "image/gif":
+			return "image/gif";
+		case "image/webp":
+			return "image/webp";
+		default:
+			return null;
+	}
+}
+
+async function normalizeImage(bytes: Uint8Array, mimeType: string): Promise<NormalizedImage | null> {
+	const normalizedMimeType = normalizeSupportedImageMimeType(mimeType);
+	if (normalizedMimeType) {
+		return { bytes, mimeType: normalizedMimeType };
+	}
+
+	const pngBytes = await convertImageBytesToPng(bytes);
+	if (!pngBytes) {
+		return null;
+	}
+
+	return {
+		bytes: pngBytes,
+		mimeType: "image/png",
+		convertedFrom: baseMimeType(mimeType),
+	};
+}
+
+function conversionHint(from: string | undefined, to: string): string | undefined {
+	if (!from || from === to) return undefined;
+	return `[Image converted from ${from} to ${to}.]`;
+}
+
+export async function processImage(
+	bytes: Uint8Array,
+	mimeType: string,
+	options?: ProcessImageOptions,
+): Promise<ProcessImageResult> {
+	const autoResizeImages = options?.autoResizeImages ?? true;
+	const normalized = await normalizeImage(bytes, mimeType);
+	if (!normalized) {
+		return {
+			ok: false,
+			message: "[Image omitted: could not be converted to a supported inline image format.]",
+		};
+	}
+
+	if (autoResizeImages) {
+		const resized = await resizeImage(normalized.bytes, normalized.mimeType, options?.resizeOptions);
+		if (!resized) {
+			return {
+				ok: false,
+				message: "[Image omitted: could not be resized below the inline image size limit.]",
+			};
+		}
+
+		const hints: string[] = [];
+		const convertedHint = conversionHint(normalized.convertedFrom, resized.mimeType);
+		if (convertedHint) hints.push(convertedHint);
+		const dimensionNote = formatDimensionNote(resized);
+		if (dimensionNote) hints.push(dimensionNote);
+
+		return {
+			ok: true,
+			data: resized.data,
+			mimeType: resized.mimeType,
+			hints,
+		};
+	}
+
+	const hints: string[] = [];
+	const convertedHint = conversionHint(normalized.convertedFrom, normalized.mimeType);
+	if (convertedHint) hints.push(convertedHint);
+
+	return {
+		ok: true,
+		data: Buffer.from(normalized.bytes).toString("base64"),
+		mimeType: normalized.mimeType,
+		hints,
+	};
+}
@@ -16,6 +16,9 @@ export function detectSupportedImageMimeType(buffer: Uint8Array): string | null
 	if (startsWithAscii(buffer, 0, "RIFF") && startsWithAscii(buffer, 8, "WEBP")) {
 		return "image/webp";
 	}
+	if (startsWithAscii(buffer, 0, "BM") && isBmp(buffer)) {
+		return "image/bmp";
+	}
 	return null;
 }

@@ -51,6 +54,36 @@ function isAnimatedPng(buffer: Uint8Array): boolean {
 	return false;
 }

+function isBmp(buffer: Uint8Array): boolean {
+	if (buffer.length < 26) return false;
+
+	const declaredFileSize = readUint32LE(buffer, 2);
+	const pixelDataOffset = readUint32LE(buffer, 10);
+	const dibHeaderSize = readUint32LE(buffer, 14);
+	if (declaredFileSize !== 0 && declaredFileSize < 26) return false;
+	if (pixelDataOffset < 14 + dibHeaderSize) return false;
+	if (declaredFileSize !== 0 && pixelDataOffset >= declaredFileSize) return false;
+
+	let colorPlanes: number;
+	let bitsPerPixel: number;
+	if (dibHeaderSize === 12) {
+		colorPlanes = readUint16LE(buffer, 22);
+		bitsPerPixel = readUint16LE(buffer, 24);
+	} else if (dibHeaderSize >= 40 && dibHeaderSize <= 124) {
+		if (buffer.length < 30) return false;
+		colorPlanes = readUint16LE(buffer, 26);
+		bitsPerPixel = readUint16LE(buffer, 28);
+	} else {
+		return false;
+	}
+
+	return colorPlanes === 1 && [1, 4, 8, 16, 24, 32].includes(bitsPerPixel);
+}
+
+function readUint16LE(buffer: Uint8Array, offset: number): number {
+	return (buffer[offset] ?? 0) + ((buffer[offset + 1] ?? 0) << 8);
+}
+
 function readUint32BE(buffer: Uint8Array, offset: number): number {
 	return (
 		(buffer[offset] ?? 0) * 0x1000000 +
@@ -60,6 +93,15 @@ function readUint32BE(buffer: Uint8Array, offset: number): number {
 	);
 }

+function readUint32LE(buffer: Uint8Array, offset: number): number {
+	return (
+		(buffer[offset] ?? 0) +
+		((buffer[offset + 1] ?? 0) << 8) +
+		((buffer[offset + 2] ?? 0) << 16) +
+		(buffer[offset + 3] ?? 0) * 0x1000000
+	);
+}
+
 function startsWith(buffer: Uint8Array, bytes: number[]): boolean {
 	if (buffer.length < bytes.length) return false;
 	return bytes.every((byte, index) => buffer[index] === byte);