import type { ImageContent } from "@mariozechner/pi-ai"; export interface ImageResizeOptions { maxWidth?: number; // Default: 2000 maxHeight?: number; // Default: 2000 maxBytes?: number; // Default: 4.5MB (below Anthropic's 5MB limit) jpegQuality?: number; // Default: 80 } export interface ResizedImage { data: string; // base64 mimeType: string; originalWidth: number; originalHeight: number; width: number; height: number; wasResized: boolean; } // 4.5MB - provides headroom below Anthropic's 5MB limit const DEFAULT_MAX_BYTES = 4.5 * 1024 * 1024; const DEFAULT_OPTIONS: Required = { maxWidth: 2000, maxHeight: 2000, maxBytes: DEFAULT_MAX_BYTES, jpegQuality: 80, }; /** Helper to pick the smaller of two buffers */ function pickSmaller( a: { buffer: Buffer; mimeType: string }, b: { buffer: Buffer; mimeType: string }, ): { buffer: Buffer; mimeType: string } { return a.buffer.length <= b.buffer.length ? a : b; } /** * Resize an image to fit within the specified max dimensions and file size. * Returns the original image if it already fits within the limits. * * Uses sharp for image processing. If sharp is not available (e.g., in some * environments), returns the original image unchanged. * * Strategy for staying under maxBytes: * 1. First resize to maxWidth/maxHeight * 2. Try both PNG and JPEG formats, pick the smaller one * 3. If still too large, try JPEG with decreasing quality * 4. If still too large, progressively reduce dimensions */ export async function resizeImage(img: ImageContent, options?: ImageResizeOptions): Promise { const opts = { ...DEFAULT_OPTIONS, ...options }; const buffer = Buffer.from(img.data, "base64"); let sharp: typeof import("sharp") | undefined; try { sharp = (await import("sharp")).default; } catch { // Sharp not available - return original image // We can't get dimensions without sharp, so return 0s return { data: img.data, mimeType: img.mimeType, originalWidth: 0, originalHeight: 0, width: 0, height: 0, wasResized: false, }; } const sharpImg = sharp(buffer); const metadata = await sharpImg.metadata(); const originalWidth = metadata.width ?? 0; const originalHeight = metadata.height ?? 0; const format = metadata.format ?? img.mimeType?.split("/")[1] ?? "png"; // Check if already within all limits (dimensions AND size) const originalSize = buffer.length; if (originalWidth <= opts.maxWidth && originalHeight <= opts.maxHeight && originalSize <= opts.maxBytes) { return { data: img.data, mimeType: img.mimeType ?? `image/${format}`, originalWidth, originalHeight, width: originalWidth, height: originalHeight, wasResized: false, }; } // Calculate initial dimensions respecting max limits let targetWidth = originalWidth; let targetHeight = originalHeight; if (targetWidth > opts.maxWidth) { targetHeight = Math.round((targetHeight * opts.maxWidth) / targetWidth); targetWidth = opts.maxWidth; } if (targetHeight > opts.maxHeight) { targetWidth = Math.round((targetWidth * opts.maxHeight) / targetHeight); targetHeight = opts.maxHeight; } // Helper to resize and encode in both formats, returning the smaller one async function tryBothFormats( width: number, height: number, jpegQuality: number, ): Promise<{ buffer: Buffer; mimeType: string }> { const resized = await sharp!(buffer) .resize(width, height, { fit: "inside", withoutEnlargement: true }) .toBuffer(); const [pngBuffer, jpegBuffer] = await Promise.all([ sharp!(resized).png({ compressionLevel: 9 }).toBuffer(), sharp!(resized).jpeg({ quality: jpegQuality }).toBuffer(), ]); return pickSmaller({ buffer: pngBuffer, mimeType: "image/png" }, { buffer: jpegBuffer, mimeType: "image/jpeg" }); } // Try to produce an image under maxBytes const qualitySteps = [85, 70, 55, 40]; const scaleSteps = [1.0, 0.75, 0.5, 0.35, 0.25]; let best: { buffer: Buffer; mimeType: string }; let finalWidth = targetWidth; let finalHeight = targetHeight; // First attempt: resize to target dimensions, try both formats best = await tryBothFormats(targetWidth, targetHeight, opts.jpegQuality); if (best.buffer.length <= opts.maxBytes) { return { data: best.buffer.toString("base64"), mimeType: best.mimeType, originalWidth, originalHeight, width: finalWidth, height: finalHeight, wasResized: true, }; } // Still too large - try JPEG with decreasing quality (and compare to PNG each time) for (const quality of qualitySteps) { best = await tryBothFormats(targetWidth, targetHeight, quality); if (best.buffer.length <= opts.maxBytes) { return { data: best.buffer.toString("base64"), mimeType: best.mimeType, originalWidth, originalHeight, width: finalWidth, height: finalHeight, wasResized: true, }; } } // Still too large - reduce dimensions progressively for (const scale of scaleSteps) { finalWidth = Math.round(targetWidth * scale); finalHeight = Math.round(targetHeight * scale); // Skip if dimensions are too small if (finalWidth < 100 || finalHeight < 100) { break; } for (const quality of qualitySteps) { best = await tryBothFormats(finalWidth, finalHeight, quality); if (best.buffer.length <= opts.maxBytes) { return { data: best.buffer.toString("base64"), mimeType: best.mimeType, originalWidth, originalHeight, width: finalWidth, height: finalHeight, wasResized: true, }; } } } // Last resort: return smallest version we produced even if over limit // (the API will reject it, but at least we tried everything) return { data: best.buffer.toString("base64"), mimeType: best.mimeType, originalWidth, originalHeight, width: finalWidth, height: finalHeight, wasResized: true, }; } /** * Format a dimension note for resized images. * This helps the model understand the coordinate mapping. */ export function formatDimensionNote(result: ResizedImage): string | undefined { if (!result.wasResized) { return undefined; } const scale = result.originalWidth / result.width; return `[Image: original ${result.originalWidth}x${result.originalHeight}, displayed at ${result.width}x${result.height}. Multiply coordinates by ${scale.toFixed(2)} to map to original image.]`; }