From 397d4b64913ddcb8d8105278e9e8b42a24a3377d Mon Sep 17 00:00:00 2001 From: ework-agent Date: Wed, 23 Sep 2026 01:46:44 +0800 Subject: [PATCH] feat: image pre-compression routing/downsample + image_full restore channel (#353) --- src/compress-tools.ts | 41 +++ src/config.ts | 30 ++ src/image-compress.ts | 552 +++++++++++++++++++++++++++++++ src/index.ts | 37 +++ src/persist/state-merge.ts | 2 + src/state.ts | 11 +- src/types.ts | 47 +++ tests/image-compress.test.ts | 609 +++++++++++++++++++++++++++++++++++ tests/persist.test.ts | 9 +- 9 files changed, 1336 insertions(+), 2 deletions(-) create mode 100644 src/image-compress.ts create mode 100644 tests/image-compress.test.ts diff --git a/src/compress-tools.ts b/src/compress-tools.ts index 916c9ab..26adb5e 100644 --- a/src/compress-tools.ts +++ b/src/compress-tools.ts @@ -22,6 +22,7 @@ export const SEARCH_CONTEXT_TOOL_NAME = "search_context"; export const ACP_STATUS_TOOL_NAME = "acp_status"; export const ACP_CACHE_TOOL_NAME = "acp_cache"; export const ABSORB_TOOL_NAME = "absorb"; +export const IMAGE_FULL_TOOL_NAME = "image_full"; /** Text-protocol trigger tags. The model emits these in its text output to * request compression (used when host client tools cannot coexist with a @@ -696,3 +697,43 @@ export const ABSORB_TOOL_GOOGLE = { description: ABSORB_TOOL_DESCRIPTION, parameters: ABSORB_PARAMETERS, }; + +/** Opt-in image_full tool (restore original-resolution images for a previously + * downscaled message). Inject/register only when + * config.imageCompression.enabled — NOT part of ACP_TOOLS_* arrays. */ +export const IMAGE_FULL_TOOL_DESCRIPTION = + 'Restore original-resolution images for a previously downscaled message. Call when you cannot read details (text, colors, alignment) in a reduced image: pass the message ref ("mNNNNN") from the [Downscaled screenshots] note. Full resolution applies for the rest of this session.'; + +const IMAGE_FULL_PARAMETERS = { + type: "object" as const, + properties: { + ref: { + type: "string", + description: + "mNNNNN ref of the message whose image(s) should be restored to full resolution", + }, + }, + required: ["ref"], +}; + +export const IMAGE_FULL_TOOL = { + name: IMAGE_FULL_TOOL_NAME, + description: IMAGE_FULL_TOOL_DESCRIPTION, + input_schema: IMAGE_FULL_PARAMETERS, +}; + +export const IMAGE_FULL_TOOL_OPENAI = { + type: "function" as const, + function: { + name: IMAGE_FULL_TOOL_NAME, + description: IMAGE_FULL_TOOL_DESCRIPTION, + parameters: IMAGE_FULL_PARAMETERS, + }, +}; + +export const IMAGE_FULL_TOOL_RESPONSES = { + type: "function" as const, + name: IMAGE_FULL_TOOL_OPENAI.function.name, + description: IMAGE_FULL_TOOL_OPENAI.function.description, + parameters: IMAGE_FULL_TOOL_OPENAI.function.parameters, +}; diff --git a/src/config.ts b/src/config.ts index 365a0b2..90ad95d 100644 --- a/src/config.ts +++ b/src/config.ts @@ -43,6 +43,13 @@ export function defaultConfig( enabled: false, minReduction: 0.1, }, + imageCompression: { + enabled: false, + minTokens: 512, + maxDimension: 1280, + quality: 80, + format: "webp", + }, }; return { ...base, @@ -55,6 +62,9 @@ export function defaultConfig( ? { ...base.absorb, ...overrides.absorb } : base.absorb, crush: overrides.crush ? { ...base.crush, ...overrides.crush } : base.crush, + imageCompression: overrides.imageCompression + ? { ...base.imageCompression, ...overrides.imageCompression } + : base.imageCompression, }; } @@ -163,5 +173,25 @@ export function validateConfig(config: Config): string[] { } } } + if (config.imageCompression) { + const ic = config.imageCompression; + if (ic.minTokens !== undefined && (!Number.isFinite(ic.minTokens) || ic.minTokens < 0)) { + errors.push("imageCompression.minTokens must be finite and >= 0"); + } + if (ic.maxDimension !== undefined && (!Number.isInteger(ic.maxDimension) || ic.maxDimension < 16)) { + errors.push("imageCompression.maxDimension must be an integer >= 16"); + } + if (ic.quality !== undefined && (!Number.isFinite(ic.quality) || ic.quality < 1 || ic.quality > 100)) { + errors.push("imageCompression.quality must be in [1, 100]"); + } + if ( + ic.format !== undefined && + ic.format !== "webp" && + ic.format !== "jpeg" && + ic.format !== "png" + ) { + errors.push('imageCompression.format must be "webp", "jpeg", or "png"'); + } + } return errors; } diff --git a/src/image-compress.ts b/src/image-compress.ts new file mode 100644 index 0000000..be2fa99 --- /dev/null +++ b/src/image-compress.ts @@ -0,0 +1,552 @@ +import { rawForRef } from "./refs.js"; +import { IMAGE_FULL_TOOL_NAME } from "./compress-tools.js"; +import type { + Config, + CompressionState, + ImageBillingMode, + ImageCompressionConfig, + ImageFormat, + ImageShrinkRecord, +} from "./types.js"; + +/** + * Image pre-compression (#353, filed from billion-context#1095 analysis): + * screenshot-like tool-result images are downscaled ONCE at arrival before + * entering the wire; non-screenshot originals pass through untouched; the + * lossy step is backstopped by the image_full restore channel. + * + * Kernel scope (pure, zero native deps): routing decision + recipe, billing- + * aware token estimation (canonical copy — moved from billion-context + * src/image-tokens.ts), image_full contract (arg parsing, sticky-restore + * state machine, result strings, system-note fragment), shrink records and + * stats counters on session state. + * + * Host scope (NOT here): executing the recipe with a real encoder + * (deterministic resize fit-inside withoutEnlargement → flatten → + * webp{quality} / jpeg{quality, mozjpeg} / png{compressionLevel:9}, keeping + * the smaller of original vs encoded), sidecar file I/O under the #1097 + * store, wire-body substitution across protocols, plugin-manifest + * registration, logging. + * + * Determinism is load-bearing: the downsampled bytes become the standing + * wire content after arrival-time substitution; a non-deterministic recipe + * would invalidate the provider prefix cache mid-session (same invariant as + * #1097's placeholder determinism). + */ + +export const DEFAULT_IMAGE_COMPRESSION_CONFIG: Required = { + enabled: false, + minTokens: 512, + maxDimension: 1280, + quality: 80, + format: "webp", +}; + +/** Sub-field merge over defaults; hosts layer global → provider → model + * exactly like absorb (three levels). */ +export function resolveImageCompressionConfig( + config: Config, +): Required { + return { ...DEFAULT_IMAGE_COMPRESSION_CONFIG, ...(config.imageCompression ?? {}) }; +} + +// --------------------------------------------------------------------------- +// Billing-aware image token estimation (canonical copy; ported from +// billion-context src/image-tokens.ts — the kernel now owns the math, hosts +// keep only the body-scanning wrapper that feeds ImageMeta in). +// --------------------------------------------------------------------------- + +/** Pixels-mode fallback when dimensions cannot be parsed header-only: the + * worst-case high-detail bill. */ +export const PIXEL_IMAGE_FALLBACK_TOKENS = 16_384; + +const TILE_EDGE = 512; +const TILE_SHORT_SIDE = 768; +const TILE_MAX_EDGE = 2048; + +export interface ImageDimensions { + width: number; + height: number; +} + +// NoUncheckedIndexedAccess: out-of-range reads are impossible after each +// section's length guard, but the type system cannot see that — coalesce. +function byteAt(b: Uint8Array, off: number): number { + return b[off] ?? 0; +} + +function u16be(b: Uint8Array, off: number): number { + return (byteAt(b, off) << 8) | byteAt(b, off + 1); +} + +function u16le(b: Uint8Array, off: number): number { + return byteAt(b, off) | (byteAt(b, off + 1) << 8); +} + +function u32be(b: Uint8Array, off: number): number { + return ( + (byteAt(b, off) << 24) | + (byteAt(b, off + 1) << 16) | + (byteAt(b, off + 2) << 8) | + byteAt(b, off + 3) + ) >>> 0; +} + +function i32le(b: Uint8Array, off: number): number { + return ( + byteAt(b, off) | + (byteAt(b, off + 1) << 8) | + (byteAt(b, off + 2) << 16) | + (byteAt(b, off + 3) << 24) + ); +} + +/** Header-only dimension parse: reads just enough leading bytes for container + * headers. Supports PNG / GIF / BMP / WebP (VP8X, VP8L, VP8 lossy) / JPEG + * (SOF segment scan). Unknown formats or truncated headers → undefined. */ +export function parseImageDimensions( + bytes: Uint8Array, +): ImageDimensions | undefined { + const b = bytes; + if (b.length < 10) return undefined; + // PNG: 8-byte signature, IHDR holds width/height (BE u32) at offset 16. + if (b[0] === 0x89 && b[1] === 0x50 && b[2] === 0x4e && b[3] === 0x47) { + if (b.length < 24) return undefined; + const w = u32be(b, 16); + const h = u32be(b, 20); + return w > 0 && h > 0 ? { width: w, height: h } : undefined; + } + // GIF: "GIF87a"/"GIF89a", logical screen size LE u16 at offsets 6/8. + if (b[0] === 0x47 && b[1] === 0x49 && b[2] === 0x46) { + const w = u16le(b, 6); + const h = u16le(b, 8); + return w > 0 && h > 0 ? { width: w, height: h } : undefined; + } + // BMP: "BM", BITMAPINFOHEADER at 14, width/height signed LE i32 at 18/22 + // (top-down bitmaps carry negative height). + if (b.length >= 26 && b[0] === 0x42 && b[1] === 0x4d) { + const w = i32le(b, 18); + const h = Math.abs(i32le(b, 22)); + return w > 0 && h > 0 ? { width: w, height: h } : undefined; + } + // WebP: RIFF....WEBP + VP8X (extended canvas) / VP8L (lossless) / VP8 (lossy). + // Each chunk carries its own required length (VP8L packs end at byte 24). + if ( + b.length >= 20 && + b[0] === 0x52 && + b[1] === 0x49 && + b[2] === 0x46 && + b[3] === 0x46 && + b[8] === 0x57 && + b[9] === 0x45 && + b[10] === 0x42 && + b[11] === 0x50 + ) { + if (b[12] === 0x56 && b[13] === 0x50 && b[14] === 0x38 && b[15] === 0x58) { + if (b.length < 30) return undefined; + // VP8X: 4-byte flags field, then canvas width-1 / height-1 24-bit LE. + const w = byteAt(b, 24) | (byteAt(b, 25) << 8) | (byteAt(b, 26) << 16); + const h = byteAt(b, 27) | (byteAt(b, 28) << 8) | (byteAt(b, 29) << 16); + return w > 0 && h > 0 ? { width: w + 1, height: h + 1 } : undefined; + } + if (b[12] === 0x56 && b[13] === 0x50 && b[14] === 0x38 && b[15] === 0x4c) { + if (b.length < 25) return undefined; + // VP8L: signature byte 0x2F, then a 4-byte pack: width-1 (14 bits), + // height-1 (14 bits), version (4 bits). + if (b[20] !== 0x2f) return undefined; + const w = ((byteAt(b, 22) & 0x3f) << 8 | byteAt(b, 21)) + 1; + const h = + ((byteAt(b, 24) & 0x0f) << 10) | (byteAt(b, 23) << 2) | ((byteAt(b, 22) & 0xc0) >> 6); + return w > 0 && h > 0 ? { width: w, height: h + 1 } : undefined; + } + if (b[12] === 0x56 && b[13] === 0x50 && b[14] === 0x38 && b[15] === 0x20) { + if (b.length < 30) return undefined; + // VP8 lossy: sync code 9D 01 2A, then actual width/height 14-bit fields + // (no -1 encoding, unlike VP8X/VP8L). + if (b[23] !== 0x9d || b[24] !== 0x01 || b[25] !== 0x2a) return undefined; + const w = u16le(b, 26) & 0x3fff; + const h = u16le(b, 28) & 0x3fff; + return w > 0 && h > 0 ? { width: w, height: h } : undefined; + } + return undefined; + } + // JPEG: walk segments until an SOFn marker (C0-CF excluding C4/C8/CC). + if (b[0] === 0xff && b[1] === 0xd8) { + let off = 2; + while (off + 4 <= b.length) { + if (b[off] !== 0xff) { + off += 1; + continue; + } + const marker = byteAt(b, off + 1); + if (marker === 0xd8 || marker === 0x01 || (marker >= 0xd0 && marker <= 0xd9)) { + off += 2; + continue; + } + if (marker === 0xda) break; // SOS with no SOF before it — malformed + const segLen = u16be(b, off + 2); + if (segLen < 2) break; + if ( + marker >= 0xc0 && + marker <= 0xcf && + marker !== 0xc4 && + marker !== 0xc8 && + marker !== 0xcc + ) { + if (off + 9 > b.length) break; + const h = u16be(b, off + 5); + const w = u16be(b, off + 7); + if (w > 0 && h > 0) return { width: w, height: h }; + break; + } + off += 2 + segLen; + } + } + return undefined; +} + +const HEADER_SCAN_CHARS = 64; +const JPEG_SCAN_CHARS = 350_000; + +/** Base64 convenience entry: decodes only the leading slice — enough for + * container headers (PNG/GIF/BMP/WebP within 64 chars; JPEG SOF within ~350K + * chars of EXIF bloat). Node-targeted (Buffer). */ +export function parseImageDimensionsFromBase64( + b64: string, +): ImageDimensions | undefined { + if (typeof b64 !== "string" || b64.length === 0) return undefined; + const head = parseImageDimensions(Buffer.from(b64.slice(0, HEADER_SCAN_CHARS), "base64")); + if (head) return head; + if (b64.startsWith("/9j/") && b64.length > HEADER_SCAN_CHARS) { + return parseImageDimensions(Buffer.from(b64.slice(0, JPEG_SCAN_CHARS), "base64")); + } + return undefined; +} + +/** OpenAI high-detail tile model: scale so the short side is at least 768 + * (providers bill small images enlarged), cap the long side at 2048, then + * 85 + 170 per 512×512 tile. Bounds ≈ [765, 2805]. */ +export function pixelTileEstimate(width: number, height: number): number { + if (!Number.isFinite(width) || !Number.isFinite(height) || width <= 0 || height <= 0) { + return PIXEL_IMAGE_FALLBACK_TOKENS; + } + let sw = width; + let sh = height; + const shortSide = Math.min(sw, sh); + if (shortSide > 0 && shortSide < TILE_SHORT_SIDE) { + const s = TILE_SHORT_SIDE / shortSide; + sw *= s; + sh *= s; + } + const longSide = Math.max(sw, sh); + if (longSide > TILE_MAX_EDGE) { + const s = TILE_MAX_EDGE / longSide; + sw *= s; + sh *= s; + } + const tiles = Math.ceil(sw / TILE_EDGE) * Math.ceil(sh / TILE_EDGE); + return 85 + 170 * tiles; +} + +/** One image attachment's metadata as seen by the routing decision. */ +export interface ImageMeta { + mediaType: string; + /** Length of the base64 payload (characters). */ + base64Length: number; + width?: number; + height?: number; + /** Upstream billing mode for this image. Default "bytes" (conservative). */ + billing?: ImageBillingMode; + /** Optional base64 payload; only its leading decode budget is read, and + * only when width/height are missing in pixels mode. */ + base64?: string; +} + +/** Billing-aware estimate of an image's token cost. Pixels mode uses the tile + * model on real dimensions (header-parsed from the payload head when absent); + * bytes mode counts ceil(base64len/4). Unknown dims in pixels mode fall back + * to PIXEL_IMAGE_FALLBACK_TOKENS. */ +export function estimateImageTokens(meta: ImageMeta): number { + const billing = meta.billing ?? "bytes"; + if (billing === "bytes") return Math.ceil(meta.base64Length / 4); + const w = meta.width; + const h = meta.height; + if (w && h) return pixelTileEstimate(w, h); + if (meta.base64) { + const dims = parseImageDimensionsFromBase64(meta.base64); + if (dims) return pixelTileEstimate(dims.width, dims.height); + } + return PIXEL_IMAGE_FALLBACK_TOKENS; +} + +// --------------------------------------------------------------------------- +// Routing decision +// --------------------------------------------------------------------------- + +/** Pluggable screenshot classifier. v1 ships the heuristic below; a trained + * router slots in here later without pipeline changes. */ +export interface ScreenshotClassifier { + readonly name: string; + isScreenshotLike(meta: ImageMeta): boolean; +} + +export interface HeuristicClassifierOptions { + /** Low end of the long/short aspect band. Default 1.4. */ + aspectRatioMin?: number; + /** High end of the long/short aspect band. Default 2.6. */ + aspectRatioMax?: number; + /** Short-side floor (px): below it even band-shaped images pass — small + * crops would be upscaled by the tile model and billed larger. Default 720. */ + minShortSide?: number; +} + +/** v1 heuristic: long/short aspect ratio within [1.4, 2.6] AND short side ≥ + * 720 px. Missing dimensions → false (conservative: never route what cannot + * be classified). Known limitation: 3:2 photos (ratio 1.5) false-positive as + * screenshots until a trained router slots into the classifier interface. */ +export function createHeuristicClassifier( + options: HeuristicClassifierOptions = {}, +): ScreenshotClassifier { + const aspectRatioMin = options.aspectRatioMin ?? 1.4; + const aspectRatioMax = options.aspectRatioMax ?? 2.6; + const minShortSide = options.minShortSide ?? 720; + return { + name: "heuristic-v1", + isScreenshotLike(meta) { + const w = meta.width; + const h = meta.height; + if (!w || !h || w <= 0 || h <= 0) return false; + const shortSide = Math.min(w, h); + if (shortSide < minShortSide) return false; + const ratio = Math.max(w, h) / shortSide; + return ratio >= aspectRatioMin && ratio <= aspectRatioMax; + }, + }; +} + +export const DEFAULT_SCREENSHOT_CLASSIFIER: ScreenshotClassifier = + createHeuristicClassifier(); + +export type ImageRouteReason = + | "disabled" + | "not-an-image" + | "below-min-tokens" + | "not-screenshot-like" + | "downsample"; + +/** Downsample parameters the host executes with its own encoder (sharp etc.). */ +export interface DownsampleRecipe { + /** Longest side (px) after resize. */ + maxDimension: number; + quality: number; + format: ImageFormat; +} + +export interface ImageRoutingDecision { + action: "pass" | "downsample"; + reason: ImageRouteReason; + /** Token estimate of the ORIGINAL payload (host log lines). */ + estimatedTokens: number; + recipe?: DownsampleRecipe; +} + +/** Pure + deterministic: same input + config → identical decision, always. + * The host executes the returned recipe exactly once, at arrival time. */ +export function decideImageRoute( + meta: ImageMeta, + config: Config, + classifier: ScreenshotClassifier = DEFAULT_SCREENSHOT_CLASSIFIER, +): ImageRoutingDecision { + const cfg = resolveImageCompressionConfig(config); + const estimatedTokens = estimateImageTokens(meta); + if (!cfg.enabled) return { action: "pass", reason: "disabled", estimatedTokens }; + if (!meta.mediaType.startsWith("image/")) { + return { action: "pass", reason: "not-an-image", estimatedTokens }; + } + if (estimatedTokens < cfg.minTokens) { + return { action: "pass", reason: "below-min-tokens", estimatedTokens }; + } + if (!classifier.isScreenshotLike(meta)) { + return { action: "pass", reason: "not-screenshot-like", estimatedTokens }; + } + return { + action: "downsample", + reason: "downsample", + estimatedTokens, + recipe: { maxDimension: cfg.maxDimension, quality: cfg.quality, format: cfg.format }, + }; +} + +// --------------------------------------------------------------------------- +// image_full restore channel +// --------------------------------------------------------------------------- + +/** Failure-prefix convention (mirrors absorb's marker style): any resultText + * starting with this marks a rejected call. */ +export const IMAGE_FULL_FAILURE_MARKER = "[image_full FAILED:"; + +/** Byte-stable system-prompt fragment; count = images downscaled so far in + * this session. Hosts inject it while the feature is enabled and shrinks + * exist. */ +export function buildImageFullSystemNote(count: number): string { + return `[Downscaled screenshots: ${count} image(s) were reduced before entering context. If you cannot read details (text, colors, alignment) in a reduced image, call ${IMAGE_FULL_TOOL_NAME} with that message's ref ("mNNNNN") to restore the original resolution.]`; +} + +/** Content-store entry-kind contract for image originals (#1097 store layout: + * binary sidecar entries under the same mNNNNN key scheme, append-once, + * drop-on-fold). The kernel owns this shape; file I/O stays host-side. */ +export interface ImageStoreEntry { + kind: "image"; + ref: string; + rawMessageId: string; + mediaType: string; + originalBytes: number; + createdAt: number; +} + +export interface ParsedImageFull { + ref: string; + callId?: string; +} + +/** Lenient arg parser mirroring parseAbsorbInput: accepts object or JSON- + * string payloads, key aliases (ref/messageId/of), trims whitespace. */ +export function parseImageFullInput( + input: unknown, + callId?: string, + onWarn?: (message: string) => void, +): ParsedImageFull | null { + let obj: Record | null = null; + if (typeof input === "string") { + try { + const parsed: unknown = JSON.parse(input); + if (parsed && typeof parsed === "object") obj = parsed as Record; + } catch { + obj = null; + } + } else if (input && typeof input === "object") { + obj = input as Record; + } + if (!obj) { + onWarn?.(`[acp-image-full-input] rejected: not an object (${typeof input})`); + return null; + } + let ref: string | undefined; + for (const key of ["ref", "messageId", "of"]) { + const value = obj[key]; + if (typeof value === "string") { + ref = value; + break; + } + } + if (typeof ref !== "string") { + onWarn?.( + `[acp-image-full-input] rejected: need ref (string); keys: ${Object.keys(obj).join(",")}`, + ); + return null; + } + const trimmed = ref.trim(); + if (!trimmed) { + onWarn?.("[acp-image-full-input] rejected: ref is empty"); + return null; + } + return { ref: trimmed, ...(callId ? { callId } : {}) }; +} + +/** Append one shrink record and bump cumulative stats. Pure: returns new state. */ +export function recordImageShrink( + state: CompressionState, + record: ImageShrinkRecord, +): CompressionState { + const bytesSaved = Math.max(0, record.originalBytes - record.shrunkBytes); + const tokensSaved = Math.max(0, record.tokensBefore - record.tokensAfter); + return { + ...state, + imageShrinks: [...(state.imageShrinks ?? []), record], + stats: { + ...state.stats, + imagesShrunk: (state.stats.imagesShrunk ?? 0) + 1, + imageBytesSaved: (state.stats.imageBytesSaved ?? 0) + bytesSaved, + imageTokensSaved: (state.stats.imageTokensSaved ?? 0) + tokensSaved, + }, + }; +} + +export function isImageFullRestored(state: CompressionState, ref: string): boolean { + return (state.imageFullRestored ?? []).includes(ref); +} + +export function imageShrinksForRef( + state: CompressionState, + ref: string, +): ImageShrinkRecord[] { + return (state.imageShrinks ?? []).filter((r) => r.ref === ref); +} + +export interface ImageFullInput { + ref: string; + state: CompressionState; + config: Config; +} + +export interface ImageFullOutcome { + state: CompressionState; + ok: boolean; + resultText: string; +} + +/** Sticky-restore state machine. Success marks the ref restored for the rest + * of the session (idempotent re-calls report "already restored"). Failure + * modes return ok:false with an [image_full FAILED: ...] resultText. Pure: + * returns new state. */ +export function applyImageFull(input: ImageFullInput): ImageFullOutcome { + const cfg = resolveImageCompressionConfig(input.config); + const ref = input.ref.trim(); + if (!cfg.enabled) { + return { + state: input.state, + ok: false, + resultText: `${IMAGE_FULL_FAILURE_MARKER} image compression is disabled in this session — no image was ever downscaled]`, + }; + } + const rawId = rawForRef(input.state.messageRefs, ref); + if (!rawId) { + return { + state: input.state, + ok: false, + resultText: `${IMAGE_FULL_FAILURE_MARKER} ref ${ref} does not exist in this session]`, + }; + } + if (isImageFullRestored(input.state, ref)) { + return { + state: input.state, + ok: true, + resultText: `already restored (${ref}) — full resolution stays in effect for the rest of this session.`, + }; + } + const shrinks = imageShrinksForRef(input.state, ref); + if (shrinks.length === 0) { + return { + state: input.state, + ok: false, + resultText: `${IMAGE_FULL_FAILURE_MARKER} no downscaled image is recorded for ${ref} — it was passed through untouched or never carried an image]`, + }; + } + return { + state: { + ...input.state, + imageFullRestored: [...(input.state.imageFullRestored ?? []), ref], + }, + ok: true, + resultText: `restored original-resolution image(s) for ${ref} (${shrinks.length} image${shrinks.length === 1 ? "" : "s"}); full resolution applies for the rest of this session.`, + }; +} + +/** Drop ref-keyed image_full state after a compaction reset (native-compaction + * archive, fork recovery, session restart). Refs are re-issued from m00001 + * after a state reset, so stale entries would misattribute. Cumulative stats + * survive — they remain valid aggregates. Hosts MUST call this on every + * state reset. */ +export function resetImageFullState(state: CompressionState): CompressionState { + return { ...state, imageFullRestored: [], imageShrinks: [] }; +} diff --git a/src/index.ts b/src/index.ts index 1dcb67a..318f922 100644 --- a/src/index.ts +++ b/src/index.ts @@ -145,6 +145,10 @@ export { ABSORB_TOOL_NAME, ABSORB_TOOL, ABSORB_TOOL_OPENAI, + IMAGE_FULL_TOOL_NAME, + IMAGE_FULL_TOOL, + IMAGE_FULL_TOOL_OPENAI, + IMAGE_FULL_TOOL_RESPONSES, } from "./compress-tools.js"; export { ABSORB_PROMPT_MARKER, @@ -209,6 +213,39 @@ export type { EvaluateCrushInput, ApplyCrushResult, } from "./crush.js"; +export { + DEFAULT_IMAGE_COMPRESSION_CONFIG, + resolveImageCompressionConfig, + PIXEL_IMAGE_FALLBACK_TOKENS, + parseImageDimensions, + parseImageDimensionsFromBase64, + pixelTileEstimate, + estimateImageTokens, + createHeuristicClassifier, + DEFAULT_SCREENSHOT_CLASSIFIER, + decideImageRoute, + buildImageFullSystemNote, + parseImageFullInput, + recordImageShrink, + isImageFullRestored, + imageShrinksForRef, + applyImageFull, + resetImageFullState, + IMAGE_FULL_FAILURE_MARKER, +} from "./image-compress.js"; +export type { + ImageDimensions, + ImageMeta, + ScreenshotClassifier, + HeuristicClassifierOptions, + ImageRouteReason, + DownsampleRecipe, + ImageRoutingDecision, + ParsedImageFull, + ImageStoreEntry, + ImageFullInput, + ImageFullOutcome, +} from "./image-compress.js"; export { rebuildCompressionState } from "./rebuild.js"; export type { RebuildResult, RebuildPorts } from "./rebuild.js"; export { parseCompressArgs } from "./parse-compress-input.js"; diff --git a/src/persist/state-merge.ts b/src/persist/state-merge.ts index cd70113..454808d 100644 --- a/src/persist/state-merge.ts +++ b/src/persist/state-merge.ts @@ -23,5 +23,7 @@ export function mergeCompressionState(parsed: CompressionState): CompressionStat rules: parsed.rules ?? fresh.rules, nextRuleId: parsed.nextRuleId ?? fresh.nextRuleId, terminalStreak: parsed.terminalStreak, + imageFullRestored: parsed.imageFullRestored ?? fresh.imageFullRestored, + imageShrinks: parsed.imageShrinks ?? fresh.imageShrinks, }; } diff --git a/src/state.ts b/src/state.ts index 698b722..f9c068c 100644 --- a/src/state.ts +++ b/src/state.ts @@ -12,10 +12,19 @@ export function createInitialState(): CompressionState { anchors: {}, lastShownByTier: {}, }, - stats: { tokensCompressed: 0, compressionCount: 0, absorbedTokens: 0 }, + stats: { + tokensCompressed: 0, + compressionCount: 0, + absorbedTokens: 0, + imagesShrunk: 0, + imageBytesSaved: 0, + imageTokensSaved: 0, + }, absorbed: [], rules: [], nextRuleId: 1, + imageFullRestored: [], + imageShrinks: [], nextBlockId: 1, nextRunId: 1, }; diff --git a/src/types.ts b/src/types.ts index 95961b9..4ace8c4 100644 --- a/src/types.ts +++ b/src/types.ts @@ -92,6 +92,12 @@ export interface CompressionStats { compressionCount: number; /** Cumulative tokens reclaimed by absorb (instant tool-result hiding). Optional for pre-absorb persisted states. */ absorbedTokens?: number; + /** Cumulative images downscaled at arrival (image pre-compression, #353). Optional for pre-feature persisted states. */ + imagesShrunk?: number; + /** Cumulative decoded-byte savings from image downsampling. */ + imageBytesSaved?: number; + /** Cumulative estimated-token savings from image downsampling. */ + imageTokensSaved?: number; } /** One instant tool-result absorption: the original tool-call + tool-result @@ -174,6 +180,41 @@ export interface CrushConfig { strategies?: Record; } +export type ImageFormat = "webp" | "jpeg" | "png"; + +/** How upstream bills this image's payload: pixel-tile model (Anthropic / OpenAI high-detail) or raw byte counting (some relays). */ +export type ImageBillingMode = "pixels" | "bytes"; + +/** Image pre-compression routing/downsample + image_full restore (#353). Hosts perform the three-level (global → provider → model) sub-field merge exactly like absorb. Default off: disabled ⇒ byte-identical pass-through. */ +export interface ImageCompressionConfig { + enabled: boolean; + /** Only route images whose token estimate >= this. Default 512. */ + minTokens?: number; + /** Longest side (px) of the downsample recipe. Default 1280. */ + maxDimension?: number; + /** Lossy encode quality (1-100) of the downsample recipe. Default 80. */ + quality?: number; + /** Encode format of the downsample recipe. Default "webp". */ + format?: ImageFormat; +} + +/** One arrival-time image downsample, reported by the host after executing the kernel's recipe. Drives image_full validation and session stats. */ +export interface ImageShrinkRecord { + ref: string; + rawMessageId: string; + mediaType: string; + format: ImageFormat; + /** Decoded byte length of the original payload. */ + originalBytes: number; + /** Decoded byte length after encoding. */ + shrunkBytes: number; + /** Kernel token estimate of the original payload. */ + tokensBefore: number; + /** Kernel token estimate of the shrunk payload. */ + tokensAfter: number; + createdAt: number; +} + export interface CompressionState { blocks: CompressionBlock[]; messageRefs: MessageRefMap; @@ -200,6 +241,10 @@ export interface CompressionState { * issued id is never re-issued with different content. Optional: defaults * to 1 when absent on older persisted states. */ nextRuleId?: number; + /** image_full sticky-restore: refs whose images were restored to full resolution for the rest of the session. MUST be cleared on compaction reset (resetImageFullState) — refs are re-issued after state reset, so stale entries would misattribute. Optional: pre-feature persisted states lack it. */ + imageFullRestored?: string[]; + /** Arrival-time downsample records (#353). Optional: pre-feature persisted states lack it. */ + imageShrinks?: ImageShrinkRecord[]; nextBlockId: number; nextRunId: number; } @@ -291,6 +336,8 @@ export interface Config { rules?: RulesConfig; /** Deterministic tool-result compression, tier 1 of the absorb gate (see crush.ts). Requires absorb enabled. */ crush?: CrushConfig; + /** Image pre-compression routing/downsample + image_full restore (see image-compress.ts). Absent/disabled = feature off. */ + imageCompression?: ImageCompressionConfig; messageFilters?: import("./filter/types.js").MessageFiltersConfig; } diff --git a/tests/image-compress.test.ts b/tests/image-compress.test.ts new file mode 100644 index 0000000..556aafa --- /dev/null +++ b/tests/image-compress.test.ts @@ -0,0 +1,609 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { + DEFAULT_IMAGE_COMPRESSION_CONFIG, + DEFAULT_SCREENSHOT_CLASSIFIER, + IMAGE_FULL_FAILURE_MARKER, + PIXEL_IMAGE_FALLBACK_TOKENS, + applyImageFull, + buildImageFullSystemNote, + createHeuristicClassifier, + decideImageRoute, + estimateImageTokens, + imageShrinksForRef, + isImageFullRestored, + parseImageDimensions, + parseImageDimensionsFromBase64, + parseImageFullInput, + pixelTileEstimate, + recordImageShrink, + resetImageFullState, + resolveImageCompressionConfig, +} from "../src/image-compress.js"; +import type { ImageMeta } from "../src/image-compress.js"; +import { + IMAGE_FULL_TOOL, + IMAGE_FULL_TOOL_NAME, + IMAGE_FULL_TOOL_OPENAI, + IMAGE_FULL_TOOL_RESPONSES, +} from "../src/compress-tools.js"; +import { defaultConfig, validateConfig } from "../src/config.js"; +import { createInitialState } from "../src/state.js"; +import { mergeCompressionState } from "../src/persist/state-merge.js"; +import type { CompressionState, ImageShrinkRecord } from "../src/types.js"; + +function u16beAt(b: Uint8Array, off: number, v: number): void { + b[off] = (v >> 8) & 0xff; + b[off + 1] = v & 0xff; +} + +function u16leAt(b: Uint8Array, off: number, v: number): void { + b[off] = v & 0xff; + b[off + 1] = (v >> 8) & 0xff; +} + +function u32beAt(b: Uint8Array, off: number, v: number): void { + b[off] = (v >>> 24) & 0xff; + b[off + 1] = (v >>> 16) & 0xff; + b[off + 2] = (v >>> 8) & 0xff; + b[off + 3] = v & 0xff; +} + +function u32leAt(b: Uint8Array, off: number, v: number): void { + b[off] = v & 0xff; + b[off + 1] = (v >> 8) & 0xff; + b[off + 2] = (v >> 16) & 0xff; + b[off + 3] = (v >>> 24) & 0xff; +} + +function pngBytes(w: number, h: number): Uint8Array { + const b = new Uint8Array(33); + b.set([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a], 0); + u32beAt(b, 8, 13); + b.set([0x49, 0x48, 0x44, 0x52], 12); + u32beAt(b, 16, w); + u32beAt(b, 20, h); + return b; +} + +function gifBytes(w: number, h: number): Uint8Array { + const b = new Uint8Array(13); + b.set([0x47, 0x49, 0x46, 0x38, 0x39, 0x61], 0); + u16leAt(b, 6, w); + u16leAt(b, 8, h); + return b; +} + +function bmpBytes(w: number, h: number): Uint8Array { + const b = new Uint8Array(26); + b.set([0x42, 0x4d], 0); + u32leAt(b, 14, 40); + u32leAt(b, 18, w | 0); + u32leAt(b, 22, h | 0); + return b; +} + +function webpVp8xBytes(w: number, h: number): Uint8Array { + const b = new Uint8Array(30); + b.set([0x52, 0x49, 0x46, 0x46], 0); + u32leAt(b, 4, 22); + b.set([0x57, 0x45, 0x42, 0x50], 8); + b.set([0x56, 0x50, 0x38, 0x58], 12); + u32leAt(b, 16, 10); + b[24] = (w - 1) & 0xff; + b[25] = ((w - 1) >> 8) & 0xff; + b[26] = ((w - 1) >> 16) & 0xff; + b[27] = (h - 1) & 0xff; + b[28] = ((h - 1) >> 8) & 0xff; + b[29] = ((h - 1) >> 16) & 0xff; + return b; +} + +function webpVp8lBytes(w: number, h: number): Uint8Array { + const b = new Uint8Array(25); + b.set([0x52, 0x49, 0x46, 0x46], 0); + u32leAt(b, 4, 22); + b.set([0x57, 0x45, 0x42, 0x50], 8); + b.set([0x56, 0x50, 0x38, 0x4c], 12); + u32leAt(b, 16, 10); + b[20] = 0x2f; + const wm = w - 1; + const hm = h - 1; + b[21] = wm & 0xff; + b[22] = ((wm >> 8) & 0x3f) | ((hm & 0x3) << 6); + b[23] = (hm >> 2) & 0xff; + b[24] = (hm >> 10) & 0x0f; + return b; +} + +function webpVp8LossyBytes(w: number, h: number): Uint8Array { + const b = new Uint8Array(30); + b.set([0x52, 0x49, 0x46, 0x46], 0); + u32leAt(b, 4, 22); + b.set([0x57, 0x45, 0x42, 0x50], 8); + b.set([0x56, 0x50, 0x38, 0x20], 12); + u32leAt(b, 16, 12); + b[23] = 0x9d; + b[24] = 0x01; + b[25] = 0x2a; + b[26] = w & 0xff; + b[27] = (w >> 8) & 0x3f; + b[28] = h & 0xff; + b[29] = (h >> 8) & 0x3f; + return b; +} + +function jpegBytes(w: number, h: number, app1Pad = 0): Uint8Array { + const b = new Uint8Array(14 + app1Pad); + b[0] = 0xff; + b[1] = 0xd8; + let off = 2; + if (app1Pad > 0) { + b[2] = 0xff; + b[3] = 0xe1; + u16beAt(b, 4, app1Pad); + off += 2 + app1Pad; + } + b[off] = 0xff; + b[off + 1] = 0xc0; + u16beAt(b, off + 2, 10); + b[off + 4] = 0x08; + u16beAt(b, off + 5, h); + u16beAt(b, off + 7, w); + b[off + 9] = 0x01; + return b; +} + +const OFF_CFG = () => defaultConfig(200_000); +const ON_CFG = () => defaultConfig(200_000, { imageCompression: { enabled: true } }); + +const PHONE: ImageMeta = { + mediaType: "image/png", + base64Length: 2_000_000, + width: 1568, + height: 2248, + billing: "pixels", +}; +const DESKTOP: ImageMeta = { + mediaType: "image/jpeg", + base64Length: 1_800_000, + width: 1920, + height: 1080, + billing: "pixels", +}; +const SQUARE_PHOTO: ImageMeta = { + mediaType: "image/jpeg", + base64Length: 3_000_000, + width: 2000, + height: 2000, + billing: "pixels", +}; +const BANNER: ImageMeta = { + mediaType: "image/png", + base64Length: 900_000, + width: 4000, + height: 1000, + billing: "pixels", +}; + +function shrinkRecord( + ref: string, + originalBytes: number, + shrunkBytes: number, + tokensBefore: number, + tokensAfter: number, +): ImageShrinkRecord { + return { + ref, + rawMessageId: `raw-${ref}`, + mediaType: "image/png", + format: "webp", + originalBytes, + shrunkBytes, + tokensBefore, + tokensAfter, + createdAt: 1, + }; +} + +function stateWithRefOnly(): CompressionState { + const state = createInitialState(); + state.messageRefs.byRaw["raw-1"] = "m00001"; + state.messageRefs.byRef["m00001"] = "raw-1"; + return state; +} + +function stateWithShrink(): CompressionState { + const state = stateWithRefOnly(); + return recordImageShrink(state, shrinkRecord("m00001", 2_000_000, 180_000, 2125, 765)); +} + +test("pixelTileEstimate: known values and bounds", () => { + assert.equal(pixelTileEstimate(1568, 2248), 2125); + assert.equal(pixelTileEstimate(1920, 1080), 2125); + assert.equal(pixelTileEstimate(512, 512), 765); + assert.equal(pixelTileEstimate(100, 100), 765); + assert.equal(pixelTileEstimate(3000, 3000), 2805); + assert.equal(pixelTileEstimate(400, 5000), 765); + assert.equal(pixelTileEstimate(0, 100), PIXEL_IMAGE_FALLBACK_TOKENS); + assert.equal(pixelTileEstimate(Number.NaN, 100), PIXEL_IMAGE_FALLBACK_TOKENS); +}); + +test("estimateImageTokens: bytes mode counts ceil(base64len/4)", () => { + assert.equal(estimateImageTokens({ mediaType: "image/png", base64Length: 1003 }), 251); + assert.equal( + estimateImageTokens({ mediaType: "image/png", base64Length: 1003, billing: "bytes" }), + 251, + ); +}); + +test("estimateImageTokens: pixels mode uses the tile model on known dims", () => { + assert.equal( + estimateImageTokens({ + mediaType: "image/png", + base64Length: 0, + width: 1568, + height: 2248, + billing: "pixels", + }), + 2125, + ); +}); + +test("estimateImageTokens: pixels mode parses dims from the base64 head when absent", () => { + const b64 = Buffer.from(pngBytes(1280, 960)).toString("base64"); + assert.equal( + estimateImageTokens({ + mediaType: "image/png", + base64Length: b64.length, + base64: b64, + billing: "pixels", + }), + pixelTileEstimate(1280, 960), + ); +}); + +test("estimateImageTokens: pixels mode falls back to 16384 for unknown dims", () => { + assert.equal(PIXEL_IMAGE_FALLBACK_TOKENS, 16_384); + assert.equal( + estimateImageTokens({ + mediaType: "image/png", + base64Length: 999_999, + base64: "bm90YW4gaW1hZ2U=", + billing: "pixels", + }), + PIXEL_IMAGE_FALLBACK_TOKENS, + ); +}); + +test("parseImageDimensions: PNG / GIF / BMP / WebP variants", () => { + assert.deepEqual(parseImageDimensions(pngBytes(1280, 960)), { width: 1280, height: 960 }); + assert.deepEqual(parseImageDimensions(gifBytes(320, 200)), { width: 320, height: 200 }); + assert.deepEqual(parseImageDimensions(bmpBytes(1024, 768)), { width: 1024, height: 768 }); + assert.deepEqual(parseImageDimensions(bmpBytes(1024, -768)), { width: 1024, height: 768 }); + assert.deepEqual(parseImageDimensions(webpVp8xBytes(640, 480)), { width: 640, height: 480 }); + assert.deepEqual(parseImageDimensions(webpVp8lBytes(100, 200)), { width: 100, height: 200 }); + assert.deepEqual(parseImageDimensions(webpVp8LossyBytes(800, 600)), { width: 800, height: 600 }); +}); + +test("parseImageDimensions: JPEG SOF scan incl. APP1 skip", () => { + assert.deepEqual(parseImageDimensions(jpegBytes(1280, 960)), { width: 1280, height: 960 }); + assert.deepEqual(parseImageDimensions(jpegBytes(1920, 1080, 60)), { width: 1920, height: 1080 }); +}); + +test("parseImageDimensions: unknown or truncated inputs → undefined", () => { + assert.equal(parseImageDimensions(new Uint8Array([1, 2, 3, 4, 5, 6, 7, 8, 9, 10])), undefined); + assert.equal(parseImageDimensions(new Uint8Array([0x89, 0x50, 0x4e, 0x47])), undefined); + assert.equal(parseImageDimensions(new Uint8Array([])), undefined); +}); + +test("parseImageDimensionsFromBase64: round-trips headers; JPEG needs the extended scan", () => { + const png = Buffer.from(pngBytes(1568, 2248)).toString("base64"); + assert.deepEqual(parseImageDimensionsFromBase64(png), { width: 1568, height: 2248 }); + const jpeg = Buffer.from(jpegBytes(1920, 1080, 60)).toString("base64"); + assert.deepEqual(parseImageDimensionsFromBase64(jpeg), { width: 1920, height: 1080 }); + assert.equal(parseImageDimensionsFromBase64(""), undefined); + assert.equal(parseImageDimensionsFromBase64("bm90YW4gaW1hZ2U="), undefined); +}); + +test("heuristic classifier: aspect band + short-side floor", () => { + const c = DEFAULT_SCREENSHOT_CLASSIFIER; + assert.equal(c.name, "heuristic-v1"); + assert.equal(c.isScreenshotLike(PHONE), true); + assert.equal(c.isScreenshotLike(DESKTOP), true); + assert.equal(c.isScreenshotLike(SQUARE_PHOTO), false); + assert.equal(c.isScreenshotLike(BANNER), false); + assert.equal(c.isScreenshotLike({ ...PHONE, width: 640, height: 960 }), false); + assert.equal(c.isScreenshotLike({ mediaType: "image/png", base64Length: 1000 }), false); +}); + +test("heuristic classifier: custom options", () => { + const c = createHeuristicClassifier({ aspectRatioMin: 2.0, aspectRatioMax: 3.0, minShortSide: 100 }); + assert.equal( + c.isScreenshotLike({ mediaType: "image/png", base64Length: 0, width: 100, height: 300 }), + true, + ); + assert.equal( + c.isScreenshotLike({ mediaType: "image/png", base64Length: 0, width: 800, height: 1200 }), + false, + ); +}); + +test("decideImageRoute: disabled ⇒ byte-identical pass-through battery", () => { + const metas: ImageMeta[] = [ + PHONE, + DESKTOP, + SQUARE_PHOTO, + BANNER, + { mediaType: "application/pdf", base64Length: 5_000_000 }, + { mediaType: "image/png", base64Length: 100 }, + ]; + for (const meta of metas) { + const d = decideImageRoute(meta, OFF_CFG()); + assert.equal(d.action, "pass"); + assert.equal(d.reason, "disabled"); + assert.equal(d.recipe, undefined); + } +}); + +test("decideImageRoute: screenshot-like ≥ minTokens ⇒ downsample with recipe", () => { + const d = decideImageRoute(PHONE, ON_CFG()); + assert.equal(d.action, "downsample"); + assert.equal(d.reason, "downsample"); + assert.equal(d.estimatedTokens, 2125); + assert.deepEqual(d.recipe, { maxDimension: 1280, quality: 80, format: "webp" }); +}); + +test("decideImageRoute: below-min-tokens passes untouched", () => { + const meta: ImageMeta = { + mediaType: "image/png", + base64Length: 2000, + width: 1568, + height: 2248, + billing: "bytes", + }; + const d = decideImageRoute(meta, ON_CFG()); + assert.equal(d.action, "pass"); + assert.equal(d.reason, "below-min-tokens"); + assert.equal(d.estimatedTokens, 500); +}); + +test("decideImageRoute: non-screenshot-like and non-image pass untouched", () => { + assert.equal(decideImageRoute(SQUARE_PHOTO, ON_CFG()).reason, "not-screenshot-like"); + assert.equal(decideImageRoute(BANNER, ON_CFG()).reason, "not-screenshot-like"); + assert.equal( + decideImageRoute({ mediaType: "application/pdf", base64Length: 5_000_000 }, ON_CFG()).reason, + "not-an-image", + ); + assert.equal( + decideImageRoute( + { mediaType: "image/png", base64Length: 1_000_000, billing: "pixels" }, + ON_CFG(), + ).reason, + "not-screenshot-like", + ); +}); + +test("decideImageRoute: custom config values flow into the recipe", () => { + const cfg = defaultConfig(200_000, { + imageCompression: { enabled: true, minTokens: 100, maxDimension: 1000, quality: 60, format: "jpeg" }, + }); + const d = decideImageRoute(PHONE, cfg); + assert.deepEqual(d.recipe, { maxDimension: 1000, quality: 60, format: "jpeg" }); +}); + +test("decideImageRoute: pluggable classifier injection", () => { + const alwaysTrue = { name: "always", isScreenshotLike: () => true }; + const alwaysFalse = { name: "never", isScreenshotLike: () => false }; + assert.equal(decideImageRoute(SQUARE_PHOTO, ON_CFG(), alwaysTrue).action, "downsample"); + assert.equal(decideImageRoute(PHONE, ON_CFG(), alwaysFalse).action, "pass"); + assert.equal(decideImageRoute(PHONE, ON_CFG(), alwaysFalse).reason, "not-screenshot-like"); +}); + +test("decideImageRoute: deterministic across repeated calls", () => { + const metas: ImageMeta[] = [ + PHONE, + DESKTOP, + SQUARE_PHOTO, + BANNER, + { mediaType: "image/png", base64Length: 12_345, billing: "pixels" }, + ]; + for (const meta of metas) { + const first = decideImageRoute(meta, ON_CFG()); + for (let i = 0; i < 50; i++) { + assert.deepEqual(decideImageRoute(meta, ON_CFG()), first); + } + } +}); + +test("recordImageShrink: appends record and bumps cumulative stats", () => { + let state = createInitialState(); + state = recordImageShrink(state, shrinkRecord("m00001", 2_000_000, 180_000, 2125, 765)); + assert.equal(state.stats.imagesShrunk, 1); + assert.equal(state.stats.imageBytesSaved, 1_820_000); + assert.equal(state.stats.imageTokensSaved, 1360); + assert.equal(state.imageShrinks?.length, 1); + state = recordImageShrink(state, shrinkRecord("m00002", 1_000_000, 900_000, 2125, 765)); + assert.equal(state.stats.imagesShrunk, 2); + assert.equal(state.stats.imageBytesSaved, 1_920_000); +}); + +test("recordImageShrink: negative deltas clamp to zero; works on pre-feature state shape", () => { + const legacy = createInitialState(); + delete legacy.imageShrinks; + delete legacy.imageFullRestored; + delete legacy.stats.imagesShrunk; + const grown = recordImageShrink(legacy, shrinkRecord("m00001", 1000, 1200, 700, 900)); + assert.equal(grown.stats.imageBytesSaved, 0); + assert.equal(grown.stats.imageTokensSaved, 0); + assert.equal(grown.imageShrinks?.length, 1); +}); + +test("imageShrinksForRef filters by ref", () => { + let state = stateWithShrink(); + state = recordImageShrink(state, shrinkRecord("m00002", 1000, 500, 100, 50)); + assert.equal(imageShrinksForRef(state, "m00001").length, 1); + assert.equal(imageShrinksForRef(state, "m00002").length, 1); + assert.equal(imageShrinksForRef(state, "m00003").length, 0); +}); + +test("applyImageFull: happy path marks sticky restore", () => { + const state = stateWithShrink(); + const out = applyImageFull({ ref: "m00001", state, config: ON_CFG() }); + assert.equal(out.ok, true); + assert.match(out.resultText, /^restored original-resolution image\(s\) for m00001 \(1 image\)/); + assert.deepEqual(out.state.imageFullRestored, ["m00001"]); + assert.equal(isImageFullRestored(out.state, "m00001"), true); +}); + +test("applyImageFull: idempotent second call", () => { + const first = applyImageFull({ ref: "m00001", state: stateWithShrink(), config: ON_CFG() }); + const second = applyImageFull({ ref: "m00001", state: first.state, config: ON_CFG() }); + assert.equal(second.ok, true); + assert.match(second.resultText, /^already restored \(m00001\)/); + assert.deepEqual(second.state.imageFullRestored, ["m00001"]); +}); + +test("applyImageFull: failure modes use the FAILED marker", () => { + const state = stateWithShrink(); + + const unknown = applyImageFull({ ref: "m00099", state, config: ON_CFG() }); + assert.equal(unknown.ok, false); + assert.ok(unknown.resultText.startsWith(IMAGE_FULL_FAILURE_MARKER)); + assert.match(unknown.resultText, /ref m00099 does not exist/); + + const noShrink = applyImageFull({ ref: "m00001", state: stateWithRefOnly(), config: ON_CFG() }); + assert.equal(noShrink.ok, false); + assert.ok(noShrink.resultText.startsWith(IMAGE_FULL_FAILURE_MARKER)); + assert.match(noShrink.resultText, /no downscaled image/); + + const disabled = applyImageFull({ ref: "m00001", state, config: OFF_CFG() }); + assert.equal(disabled.ok, false); + assert.ok(disabled.resultText.startsWith(IMAGE_FULL_FAILURE_MARKER)); + assert.match(disabled.resultText, /image compression is disabled/); + assert.equal(disabled.state, state); +}); + +test("resetImageFullState: clears ref-keyed entries, keeps stats and blocks", () => { + const state = stateWithShrink(); + const restored = applyImageFull({ ref: "m00001", state, config: ON_CFG() }).state; + const reset = resetImageFullState(restored); + assert.deepEqual(reset.imageFullRestored, []); + assert.deepEqual(reset.imageShrinks, []); + assert.equal(reset.stats.imagesShrunk, 1); + assert.equal(reset.stats.imageBytesSaved, 1_820_000); + assert.equal(isImageFullRestored(reset, "m00001"), false); +}); + +test("buildImageFullSystemNote: byte-stable, count-aware, mentions the contract", () => { + const a = buildImageFullSystemNote(3); + assert.equal(a, buildImageFullSystemNote(3)); + assert.notEqual(a, buildImageFullSystemNote(4)); + assert.match(a, /\[Downscaled screenshots: 3 image\(s\)/); + assert.match(a, /image_full/); + assert.match(a, /"mNNNNN"/); +}); + +test("parseImageFullInput: object, JSON string, aliases, trim", () => { + assert.deepEqual(parseImageFullInput({ ref: "m00001" }), { ref: "m00001" }); + assert.deepEqual(parseImageFullInput('{"ref":"m00002"}'), { ref: "m00002" }); + assert.deepEqual(parseImageFullInput({ messageId: "m00003" }), { ref: "m00003" }); + assert.deepEqual(parseImageFullInput({ of: "m00004" }), { ref: "m00004" }); + assert.deepEqual(parseImageFullInput({ ref: " m00005 " }), { ref: "m00005" }); + assert.deepEqual(parseImageFullInput({ ref: "m00001" }, "call-9"), { + ref: "m00001", + callId: "call-9", + }); +}); + +test("parseImageFullInput: rejects malformed payloads with warnings", () => { + const warnings: string[] = []; + const warn = (m: string) => warnings.push(m); + assert.equal(parseImageFullInput(42, undefined, warn), null); + assert.equal(parseImageFullInput(null, undefined, warn), null); + assert.equal(parseImageFullInput("not json", undefined, warn), null); + assert.equal(parseImageFullInput({ summary: "x" }, undefined, warn), null); + assert.equal(parseImageFullInput({ ref: " " }, undefined, warn), null); + assert.equal(warnings.length, 5); + assert.match(warnings[0], /^\[acp-image-full-input\] rejected:/); +}); + +test("image_full tool schemas: three wire shapes, required ref", () => { + assert.equal(IMAGE_FULL_TOOL_NAME, "image_full"); + assert.equal(IMAGE_FULL_TOOL.name, "image_full"); + assert.deepEqual(IMAGE_FULL_TOOL.input_schema.required, ["ref"]); + assert.equal(IMAGE_FULL_TOOL.input_schema.properties.ref.type, "string"); + assert.equal(IMAGE_FULL_TOOL_OPENAI.type, "function"); + assert.equal(IMAGE_FULL_TOOL_OPENAI.function.name, "image_full"); + assert.deepEqual(IMAGE_FULL_TOOL_OPENAI.function.parameters.required, ["ref"]); + assert.equal(IMAGE_FULL_TOOL_RESPONSES.type, "function"); + assert.deepEqual(IMAGE_FULL_TOOL_RESPONSES.parameters.required, ["ref"]); +}); + +test("defaultConfig: imageCompression defaults are off with issue spec values", () => { + assert.deepEqual(defaultConfig(200_000).imageCompression, { + enabled: false, + minTokens: 512, + maxDimension: 1280, + quality: 80, + format: "webp", + }); + assert.deepEqual(DEFAULT_IMAGE_COMPRESSION_CONFIG, defaultConfig(200_000).imageCompression); +}); + +test("defaultConfig: sub-field merge preserves other defaults (absorb-style)", () => { + const cfg = defaultConfig(200_000, { imageCompression: { enabled: true } }); + assert.deepEqual(cfg.imageCompression, { + enabled: true, + minTokens: 512, + maxDimension: 1280, + quality: 80, + format: "webp", + }); + assert.equal(resolveImageCompressionConfig(cfg).enabled, true); +}); + +test("validateConfig: rejects invalid imageCompression fields", () => { + const good = validateConfig( + defaultConfig(200_000, { + imageCompression: { enabled: true, minTokens: 0, maxDimension: 16, quality: 1, format: "png" }, + }), + ); + assert.deepEqual(good.filter((e) => e.startsWith("imageCompression")), []); + const bad = validateConfig( + defaultConfig(200_000, { + imageCompression: { enabled: true, minTokens: -1, maxDimension: 8, quality: 0, format: "gif" }, + }), + ); + assert.deepEqual(bad, [ + "imageCompression.minTokens must be finite and >= 0", + "imageCompression.maxDimension must be an integer >= 16", + "imageCompression.quality must be in [1, 100]", + 'imageCompression.format must be "webp", "jpeg", or "png"', + ]); +}); + +test("createInitialState seeds image counters and empty sticky arrays", () => { + const s = createInitialState(); + assert.deepEqual(s.stats, { + tokensCompressed: 0, + compressionCount: 0, + absorbedTokens: 0, + imagesShrunk: 0, + imageBytesSaved: 0, + imageTokensSaved: 0, + }); + assert.deepEqual(s.imageFullRestored, []); + assert.deepEqual(s.imageShrinks, []); +}); + +test("mergeCompressionState carries image fields through persist/load", () => { + const state = stateWithShrink(); + const restored = applyImageFull({ ref: "m00001", state, config: ON_CFG() }).state; + const merged = mergeCompressionState(restored); + assert.deepEqual(merged.imageFullRestored, ["m00001"]); + assert.equal(merged.imageShrinks?.length, 1); + const freshMerged = mergeCompressionState({ blocks: [] } as CompressionState); + assert.deepEqual(freshMerged.imageFullRestored, []); + assert.deepEqual(freshMerged.imageShrinks, []); +}); diff --git a/tests/persist.test.ts b/tests/persist.test.ts index 591448d..d84a4c8 100644 --- a/tests/persist.test.ts +++ b/tests/persist.test.ts @@ -370,7 +370,14 @@ test("mergeCompressionState keeps parsed values over defaults", () => { }; const merged = mergeCompressionState(parsed); assert.equal(merged.nextBlockId, 41); - assert.deepEqual(merged.stats, { tokensCompressed: 1234, compressionCount: 7, absorbedTokens: 0 }); + assert.deepEqual(merged.stats, { + tokensCompressed: 1234, + compressionCount: 7, + absorbedTokens: 0, + imagesShrunk: 0, + imageBytesSaved: 0, + imageTokensSaved: 0, + }); }); test("legacy hook adopts pre-envelope records on loadAll and loadSync", async () => {