From 966042dcce544fe5a04c6aed1c468aea2eb3e40b Mon Sep 17 00:00:00 2001 From: Joe Date: Wed, 12 Aug 2026 18:54:56 +0200 Subject: [PATCH] feat(converter): add HEIC/HEIF image format support --- bun.lock | 8 ++ package.json | 2 + src/extensions/converter/index.ts | 2 + src/extensions/converter/ocr.ts | 100 +++++++++++++++++- .../converter/skills/converter/SKILL.md | 2 +- 5 files changed, 108 insertions(+), 6 deletions(-) diff --git a/bun.lock b/bun.lock index f6eacfc..970bd26 100644 --- a/bun.lock +++ b/bun.lock @@ -18,6 +18,7 @@ "elysia": "^1.4.29", "elysia-rate-limit": "^4.6.2", "file-type": "^22.0.1", + "heic-decode": "^2.1.0", "just-bash": "3.1.0", "logging": "^4.2.0", "nanoid": "^6.0.1", @@ -30,6 +31,7 @@ "devDependencies": { "@biomejs/biome": "^2.5.8", "@types/bun": "1.3.14", + "@types/heic-decode": "^2.0.0", "@types/node-telegram-bot-api": "^0.64.15", "@types/turndown": "^5.0.6", "drizzle-kit": "^0.31.10", @@ -393,6 +395,8 @@ "@types/caseless": ["@types/caseless@0.12.5", "", {}, "sha512-hWtVTC2q7hc7xZ/RLbxapMvDMgUnDvKvMOpKal4DrMyfGBUfB1oKaZlIRr6mJL+If3bAP6sV/QneGzF6tJjZDg=="], + "@types/heic-decode": ["@types/heic-decode@2.0.0", "", {}, "sha512-Zj/zngTwbbIE84GsJ9hBI3VmvQRGOe65QV/xVdmLEAGB9bswOBxTMhQVcqNLcxnQhzOmAro5KBg7trrLaYe5Jg=="], + "@types/node": ["@types/node@25.6.0", "", { "dependencies": { "undici-types": "~7.19.0" } }, "sha512-+qIYRKdNYJwY3vRCZMdJbPLJAtGjQBudzZzdzwQYkEPQd+PJGixUL5QfvCLDaULoLv+RhT3LDkwEfKaAkgSmNQ=="], "@types/node-telegram-bot-api": ["@types/node-telegram-bot-api@0.64.15", "", { "dependencies": { "@types/node": "*", "@types/request": "*" } }, "sha512-Ac1ZgSSHcO6P++7dsLSm3tjzInDLE0WrDTXHoUPKn/sUfDyR3f+IDtIUaSp/kLu+gn/jplGgGDY5CD95BEVk+g=="], @@ -625,6 +629,8 @@ "hasown": ["hasown@2.0.3", "", { "dependencies": { "function-bind": "^1.1.2" } }, "sha512-ej4AhfhfL2Q2zpMmLo7U1Uv9+PyhIZpgQLGT1F9miIGmiCJIoCgSmczFdrc97mWT4kVY72KA+WnnhJ5pghSvSg=="], + "heic-decode": ["heic-decode@2.1.0", "", { "dependencies": { "libheif-js": "^1.19.8" } }, "sha512-0fB3O3WMk38+PScbHLVp66jcNhsZ/ErtQ6u2lMYu/YxXgbBtl+oKOhGQHa4RpvE68k8IzbWkABzHnyAIjR758A=="], + "hono": ["hono@4.12.14", "", {}, "sha512-am5zfg3yu6sqn5yjKBNqhnTX7Cv+m00ox+7jbaKkrLMRJ4rAdldd1xPd/JzbBWspqaQv6RSTrgFN95EsfhC+7w=="], "html-escaper": ["html-escaper@3.0.3", "", {}, "sha512-RuMffC89BOWQoY0WKGpIhn5gX3iI54O6nRA0yC124NYVtzjmFWBIiFd8M0x+ZdX0P9R4lADg1mgP8C7PxGOWuQ=="], @@ -671,6 +677,8 @@ "jws": ["jws@4.0.1", "", { "dependencies": { "jwa": "^2.0.1", "safe-buffer": "^5.0.1" } }, "sha512-EKI/M/yqPncGUUh44xz0PxSidXFr/+r0pA70+gIYhjv+et7yxM+s29Y+VGDkovRofQem0fs7Uvf4+YmAdyRduA=="], + "libheif-js": ["libheif-js@1.19.8", "", {}, "sha512-vQJWusIxO7wavpON1dusciL8Go9jsIQ+EUrckauFYAiSTjcmLAsuJh3SszLpvkwPci3JcL41ek2n+LUZGFpPIQ=="], + "linkedom": ["linkedom@0.18.12", "", { "dependencies": { "css-select": "^5.1.0", "cssom": "^0.5.0", "html-escaper": "^3.0.3", "htmlparser2": "^10.0.0", "uhyphen": "^0.2.0" }, "peerDependencies": { "canvas": ">= 2" }, "optionalPeers": ["canvas"] }, "sha512-jalJsOwIKuQJSeTvsgzPe9iJzyfVaEJiEXl+25EkKevsULHvMJzpNqwvj1jOESWdmgKDiXObyjOYwlUqG7wo1Q=="], "logging": ["logging@4.2.0", "", { "dependencies": { "chalk": "^5.4.1", "debug": "^4.4.1", "nicely-format": "^1.1.0" } }, "sha512-OPfCRjNr43uM4/g+Qzv+F/PwdyClJR+eSb6Zb2Ha204NEdNCu6amfOX5WEwBEH5l3xx5Gl7fHKwbsnBUHEZlaw=="], diff --git a/package.json b/package.json index c442d81..c2ef2c6 100644 --- a/package.json +++ b/package.json @@ -13,6 +13,7 @@ "elysia": "^1.4.29", "elysia-rate-limit": "^4.6.2", "file-type": "^22.0.1", + "heic-decode": "^2.1.0", "just-bash": "3.1.0", "logging": "^4.2.0", "nanoid": "^6.0.1", @@ -43,6 +44,7 @@ "devDependencies": { "@biomejs/biome": "^2.5.8", "@types/bun": "1.3.14", + "@types/heic-decode": "^2.0.0", "@types/node-telegram-bot-api": "^0.64.15", "@types/turndown": "^5.0.6", "drizzle-kit": "^0.31.10", diff --git a/src/extensions/converter/index.ts b/src/extensions/converter/index.ts index 828b73c..467b5bd 100644 --- a/src/extensions/converter/index.ts +++ b/src/extensions/converter/index.ts @@ -86,6 +86,8 @@ const MIME_TO_EXT: Record = { "image/gif": ".gif", "image/bmp": ".bmp", "image/tiff": ".tiff", + "image/heic": ".heic", + "image/heif": ".heif", }; /** diff --git a/src/extensions/converter/ocr.ts b/src/extensions/converter/ocr.ts index 6244125..1803071 100644 --- a/src/extensions/converter/ocr.ts +++ b/src/extensions/converter/ocr.ts @@ -4,6 +4,7 @@ */ import path from "node:path"; +import decode from "heic-decode"; import { pdf } from "pdf-to-img"; /** System prompt for OCR processing. */ @@ -28,16 +29,99 @@ const MIME_TYPES: Record = { ".gif": "image/gif", ".bmp": "image/bmp", ".tiff": "image/tiff", + ".heic": "image/heic", + ".heif": "image/heif", }; +/** Extensions that require HEIC decoding before processing. */ +const HEIC_EXTENSIONS = new Set([".heic", ".heif"]); + +/** + * Encodes raw RGBA pixel data as an uncompressed 32-bit BMP buffer. + * BMP stores rows bottom-to-top, so the row order is flipped. + * + * @param data - RGBA pixel data (4 bytes per pixel) + * @param width - Image width in pixels + * @param height - Image height in pixels + * @returns A Buffer containing a valid BMP file + */ +function rgbaToBmp(data: Uint8ClampedArray, width: number, height: number): Buffer { + const rowSize = width * 4; + const pixelDataSize = rowSize * height; + const headerSize = 14 + 108; // BMP file header (14) + BITMAPV4HEADER (108) + const fileSize = headerSize + pixelDataSize; + + const buf = Buffer.alloc(fileSize); + + // -- BMP File Header (14 bytes) -- + buf.write("BM", 0); // Signature + buf.writeUInt32LE(fileSize, 2); // File size + buf.writeUInt32LE(0, 6); // Reserved + buf.writeUInt32LE(headerSize, 10); // Pixel data offset + + // -- BITMAPV4HEADER (108 bytes) - supports alpha channel -- + buf.writeUInt32LE(108, 14); // Header size + buf.writeInt32LE(width, 18); // Width + buf.writeInt32LE(-height, 22); // Height (negative = top-down row order) + buf.writeUInt16LE(1, 26); // Color planes + buf.writeUInt16LE(32, 28); // Bits per pixel + buf.writeUInt32LE(3, 30); // Compression: BI_BITFIELDS + buf.writeUInt32LE(pixelDataSize, 34); // Image size + buf.writeInt32LE(2835, 38); // X pixels per meter (~72 DPI) + buf.writeInt32LE(2835, 42); // Y pixels per meter + buf.writeUInt32LE(0, 46); // Colors used + buf.writeUInt32LE(0, 50); // Important colors + + // Channel masks (RGBA): R=0x00FF0000, G=0x0000FF00, B=0x000000FF, A=0xFF000000 + buf.writeUInt32LE(0x00ff0000, 54); // Red mask + buf.writeUInt32LE(0x0000ff00, 58); // Green mask + buf.writeUInt32LE(0x000000ff, 62); // Blue mask + buf.writeUInt32LE(0xff000000, 66); // Alpha mask + + // Color space type: LCS_sRGB + buf.writeUInt32LE(0x73524742, 70); + // Remaining V4 fields (endpoints + gamma) are zero-filled by Buffer.alloc + + // -- Pixel data (BGRA order for BMP) -- + for (let i = 0; i < data.length; i += 4) { + const offset = headerSize + i; + buf[offset] = data[i + 2]!; // B + buf[offset + 1] = data[i + 1]!; // G + buf[offset + 2] = data[i]!; // R + buf[offset + 3] = data[i + 3]!; // A + } + + return buf; +} + +/** + * Decodes a HEIC/HEIF file to raw RGBA pixel data using heic-decode, + * then converts it to a PNG buffer via Bun.Image. + * + * @param filePath - Absolute path to the HEIC/HEIF file + * @param imageSize - Maximum dimension to resize the output to + * @returns Base64-encoded PNG string + */ +async function decodeHeicToPngBase64(filePath: string, imageSize: number): Promise { + const fileBuffer = await Bun.file(filePath).arrayBuffer(); + const { width, height, data } = await decode({ buffer: new Uint8Array(fileBuffer) }); + + // heic-decode returns raw RGBA pixel data - encode as BMP so Bun.Image can read it + const bmpBuffer = rgbaToBmp(data, width, height); + return new Bun.Image(bmpBuffer).resize(imageSize).png().toBase64(); +} + /** * Reads a file and returns base64-encoded image content parts suitable * for injection into an {@link AgentMessage}. * - * For PDFs, each page is converted to a PNG buffer via `pdf-to-img`. - * For images, the raw bytes are read and base64-encoded directly. + * All output is encoded as PNG (lossless) to maximize OCR accuracy. + * For PDFs, each page is rasterized via `pdf-to-img` and re-encoded as PNG. + * For HEIC/HEIF, raw pixels are decoded via heic-decode and encoded as PNG. + * For other images, the file is resized and re-encoded as PNG. * * @param filePath - Absolute path to the image or PDF file + * @param imageSize - Maximum dimension to resize images to * @returns Array of image content parts * @throws If the file cannot be read or the PDF cannot be parsed */ @@ -52,7 +136,7 @@ export async function buildImageParts( const doc = await pdf(filePath); for await (const pageBuffer of doc) { - const data = await new Bun.Image(pageBuffer).resize(imageSize).jpeg().toBase64(); + const data = await new Bun.Image(pageBuffer).resize(imageSize).png().toBase64(); parts.push({ type: "image", data, @@ -67,6 +151,12 @@ export async function buildImageParts( throw new Error(`Unsupported file type: ${ext}`); } - const data = await new Bun.Image(filePath).resize(imageSize).jpeg().toBase64(); - return [{ type: "image", data, mimeType }]; + // HEIC/HEIF files need explicit decoding since Bun.Image cannot read them directly + if (HEIC_EXTENSIONS.has(ext)) { + const data = await decodeHeicToPngBase64(filePath, imageSize); + return [{ type: "image", data, mimeType: "image/png" }]; + } + + const data = await new Bun.Image(filePath).resize(imageSize).png().toBase64(); + return [{ type: "image", data, mimeType: "image/png" }]; } diff --git a/src/extensions/converter/skills/converter/SKILL.md b/src/extensions/converter/skills/converter/SKILL.md index 36f6d2e..7c1b550 100644 --- a/src/extensions/converter/skills/converter/SKILL.md +++ b/src/extensions/converter/skills/converter/SKILL.md @@ -11,7 +11,7 @@ The converter skill provides a `convert` shell command that transforms files int Supported file types: - PDF documents (each page is converted to an image and processed) -- Images: PNG, JPEG, WebP, GIF, BMP, TIFF +- Images: PNG, JPEG, WebP, GIF, BMP, TIFF, HEIC, HEIF File types are detected by reading magic bytes from the file header, not by file extension.