diff --git a/AGENTS.md b/AGENTS.md index c0e46e705..567a8d472 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -70,6 +70,7 @@ app/api/ default-cwd/route.ts POST create ~/omp-cwd-YYYYMMDD files/[...path]/route.ts GET file contents for viewer github-repo/route.ts GET ?cwd= — GitHub owner/repo of the checkout (for #N links) + media/[hash]/route.ts GET tool-result image by sha256 (?thumb=1 → 480px WebP preview) home/route.ts GET user home directory models/route.ts GET { models, modelList, defaultModel } models-config/route.ts GET/PUT — read/write ~/.omp/agent/models.yml @@ -95,6 +96,7 @@ lib/ btw.ts /btw side-question records + pure frame/snapshot merge (order-safe) draft-store.ts local draft persistence helpers file-access.ts allowed file roots for /api/files and worktrees + media-cache.ts tool-result images → /api/media URLs; media copies + thumbnails file-paths.ts client/server path encoding helpers github-refs.ts remark plugin linking #N / owner/repo#N + GithubRepoContext git-clone.ts pure clone helpers: URL→directory name (https/ssh only), \r-aware progress log @@ -245,6 +247,21 @@ wait for that commit: - Live entries are cleared on `agent_start`, terminal `agent_end`, prompt send/settlement failure — a tool must never leak into the next run. +### Tool-result images (`lib/media-cache.ts`, `/api/media/[hash]`) +- Tool-result images never reach the browser as base64. History (`deferMedia`) + and live frames (`AgentSessionWrapper.emit`, which also feeds the replay + snapshot) replace each image with `{type:"image", mimeType, url:"/api/media/"}`. + User-message and custom-message images stay inline. +- The hash is omp's blob address (sha256 of the bytes): blob refs map directly + (a missing blob becomes the `[image unavailable …]` text), inline images are + hashed and copied to `/media/`. omp writes blobs in place, so the + route serves the copy until the blob has the copy's size. A daily sweep (run + on the next media write) deletes copies omp now holds and files older than + 30 days. +- `ToolCallBlock` shows `?thumb=1` (480px WebP made once with `sharp` and cached + next to the copies; full image if sharp is unavailable) and opens the full + image in the lightbox. The route serves only PNG/JPEG/GIF/WebP by magic bytes. + ### Event protocol differences vs pi omp emits no `prompt_done` / `prompt_error` / `compaction_start` / `compaction_end` events. Completion is `agent_end` (`isTerminal !== false`), diff --git a/CHANGELOG.md b/CHANGELOG.md index 5d1739611..1c41006af 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -32,6 +32,7 @@ All notable changes to **omp-web** (`@kahme247/ompweb`) are documented in this f ### Fixes & Improvements +- Show images from tool results again after reopening or reloading a session. Since tool-result images were dropped from the initial history load, screenshots and image reads showed only `[1 tool result image omitted from initial history payload …]`, with a wrong byte count. Expanding the tool row now shows a small preview; click it for the full image. Previews are 480px WebP files generated once and kept in omp-web's data folder, and live tool output also arrives as a link instead of inline base64, so many large screenshots no longer slow the chat down or fill browser memory. - Balance the composer icons on phones and narrow toolbars with a slightly smaller effort-level icon and a larger Slow-mode snail, while keeping menu icon sizes unchanged. - Keep the model picker on the left and reasoning effort, Fast, Slow, microphone, and Send/Stop controls in a right-aligned composer group, including wrapped mobile rows. - The Provider Usage panel now shows each account's email instead of omp's redacted form (`an*`); accounts without an email show as **Account N**. Only the email, plan and limits reach the browser. diff --git a/app/api/media/[hash]/route.ts b/app/api/media/[hash]/route.ts new file mode 100644 index 000000000..61dbfda90 --- /dev/null +++ b/app/api/media/[hash]/route.ts @@ -0,0 +1,53 @@ +import { readFile } from "node:fs/promises"; +import { createThumbnail, existingThumbnail, imageMimeType, mediaFilePath } from "@/lib/media-cache"; + +export const dynamic = "force-dynamic"; + +function image(body: Buffer, type: string, immutable = true): Response { + // A view, not a copy: these buffers come from readFile/sharp, never a SharedArrayBuffer. + return new Response(new Uint8Array(body.buffer as ArrayBuffer, body.byteOffset, body.byteLength), { + headers: { + "Content-Type": type, + // Content-addressed: the bytes behind a hash never change. + "Cache-Control": immutable ? "private, max-age=31536000, immutable" : "private, no-cache", + "X-Content-Type-Options": "nosniff", + }, + }); +} + +async function readIfPresent(file: string): Promise { + try { + return await readFile(file); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") return null; + throw error; + } +} + +/** GET /api/media/[?thumb=1] — a tool-result image, or its small WebP preview. */ +export async function GET(req: Request, { params }: { params: Promise<{ hash: string }> }) { + const { hash } = await params; + const thumb = new URL(req.url).searchParams.has("thumb"); + const cached = thumb ? existingThumbnail(hash) : null; + const cachedBytes = cached ? await readIfPresent(cached) : null; + if (cachedBytes) return image(cachedBytes, "image/webp"); + + // The sweep can delete our copy between lookup and read once omp's blob is + // complete; one more lookup then finds the blob. + let bytes: Buffer | null = null; + for (let attempt = 0; attempt < 2 && !bytes; attempt++) { + const file = mediaFilePath(hash); + if (!file) break; + bytes = await readIfPresent(file); + } + if (!bytes) return new Response("Not found", { status: 404 }); + const type = imageMimeType(bytes); + if (!type) return new Response("Not an image", { status: 415 }); + if (thumb) { + const preview = await createThumbnail(hash, bytes); + if (preview) return image(preview, "image/webp"); + // Full image standing in for a preview: not cached, so a later preview can replace it. + return image(bytes, type, false); + } + return image(bytes, type); +} diff --git a/components/ImageLightbox.tsx b/components/ImageLightbox.tsx index 1eeb6bcba..db117f66a 100644 --- a/components/ImageLightbox.tsx +++ b/components/ImageLightbox.tsx @@ -11,6 +11,8 @@ const ZOOM_STEP = 0.25; interface ClickableImageProps extends Omit, "src"> { /** Image source: string URL (data:, http(s):, blob:, /api/files/...) or Blob. */ src: ImgHTMLAttributes["src"]; + /** Larger image for the lightbox when `src` is only a preview. */ + fullSrc?: string; } /** @@ -22,7 +24,7 @@ interface ClickableImageProps extends Omit, * object URL that is revoked when the source changes or the component unmounts * — the same mechanism React's experimental `enableSrcObject` will use natively. */ -export function ClickableImage({ src, alt, ...imgProps }: ClickableImageProps) { +export function ClickableImage({ src, fullSrc, alt, ...imgProps }: ClickableImageProps) { const { t } = useI18n(); const [open, setOpen] = useState(false); const [objectUrl, setObjectUrl] = useState(null); @@ -67,7 +69,7 @@ export function ClickableImage({ src, alt, ...imgProps }: ClickableImageProps) { renders when `open`, which is strictly client-side — the short-circuit keeps `document.body` off the SSR path. */} - {open && createPortal( setOpen(false)} />, document.body)} + {open && createPortal( setOpen(false)} />, document.body)} ); } diff --git a/components/MessageView.tsx b/components/MessageView.tsx index 7b88c88ec..e8cce87d8 100644 --- a/components/MessageView.tsx +++ b/components/MessageView.tsx @@ -1197,7 +1197,8 @@ const ToolCallBlock = memo(function ToolCallBlock({ {resultImages.map((img, i) => ( diff --git a/lib/media-cache.ts b/lib/media-cache.ts new file mode 100644 index 000000000..c18d9fdbc --- /dev/null +++ b/lib/media-cache.ts @@ -0,0 +1,225 @@ +import { createHash, randomUUID } from "node:crypto"; +import { existsSync, mkdirSync, readdirSync, renameSync, rmSync, statSync, unlinkSync, writeFileSync } from "node:fs"; +import path from "node:path"; +import { getBlobsDir, getDiagnosticsDir } from "./omp/paths"; +import { isRecord } from "./type-guards"; + +/** + * Tool-result images reach the browser as `/api/media/` URLs instead of + * inline base64, so big screenshots never sit in the chat's React state. The + * hash is omp's own blob address (sha256 of the image bytes): history entries + * already carry it as `blob:sha256:`, and live output is hashed here and + * copied into omp-web's media directory until omp writes its own blob. + */ + +const HASH_RE = /^[a-f0-9]{64}$/; +const BLOB_PREFIX = "blob:sha256:"; +/** Twice the 240px tool-result preview, for high-density screens. */ +const THUMB_PX = 480; +const SWEEP_EVERY_MS = 24 * 60 * 60 * 1000; +const MAX_AGE_MS = 30 * 24 * 60 * 60 * 1000; + +declare global { + var __ompWebMediaSweptAt: number | undefined; +} + +function mediaDir(): string { + return path.join(getDiagnosticsDir(), "media"); +} + +function fileSize(file: string): number | undefined { + return statSync(file, { throwIfNoEntry: false })?.size; +} + +function writeAtomic(file: string, data: Buffer): void { + sweepMediaCache(); + mkdirSync(path.dirname(file), { recursive: true }); + const tmp = `${file}.${randomUUID()}.tmp`; + try { + writeFileSync(tmp, data); + renameSync(tmp, file); + } catch (error) { + rmSync(tmp, { force: true }); + throw error; + } +} + +/** Daily: drop copies omp now holds as complete blobs, and anything older than 30 days (previews regenerate on demand). */ +function sweepMediaCache(): void { + const now = Date.now(); + if (now - (globalThis.__ompWebMediaSweptAt ?? 0) < SWEEP_EVERY_MS) return; + globalThis.__ompWebMediaSweptAt = now; + let names: string[]; + try { + names = readdirSync(mediaDir()); + } catch { + return; + } + for (const name of names) { + const file = path.join(mediaDir(), name); + try { + const stat = statSync(file); + const hasBlob = HASH_RE.test(name) && fileSize(path.join(getBlobsDir(), name)) === stat.size; + if (hasBlob || now - stat.mtimeMs > MAX_AGE_MS) unlinkSync(file); + } catch { + // Removed concurrently or unreadable: the next sweep retries. + } + } +} + +// Live tool output resends the full accumulated result on every update; this +// keeps one screenshot from being decoded and hashed once per frame. Each key +// is a whole base64 image, so the memo is capped by size. +const recentHashes = new Map(); +const RECENT_MAX_CHARS = 16 * 1024 * 1024; +let recentChars = 0; + +/** Store a base64 image under its content address and return the hash. */ +export function storeInlineImage(base64: string): string { + // The memo skips only decoding and hashing: the files are rechecked every + // time, because the sweep may have removed the copy since. + let hash = recentHashes.get(base64); + if (!hash) { + hash = createHash("sha256").update(Buffer.from(base64, "base64")).digest("hex"); + if (base64.length <= RECENT_MAX_CHARS) { + recentHashes.set(base64, hash); + recentChars += base64.length; + while (recentChars > RECENT_MAX_CHARS) { + const oldest = recentHashes.keys().next().value!; + recentChars -= oldest.length; + recentHashes.delete(oldest); + } + } + } + const size = Buffer.byteLength(base64, "base64"); + const copy = path.join(mediaDir(), hash); + // omp writes blobs in place: keep our own copy unless its blob is already complete. + if (fileSize(path.join(getBlobsDir(), hash)) !== size && fileSize(copy) !== size) writeAtomic(copy, Buffer.from(base64, "base64")); + return hash; +} + +function imageBlockAsUrl(block: unknown): unknown { + if (!isRecord(block) || block.type !== "image") return block; + let data: unknown; + let mimeType: unknown; + if (typeof block.data === "string") { + data = block.data; + mimeType = block.mimeType; + } else if (isRecord(block.source) && block.source.type === "base64") { + data = block.source.data; + mimeType = block.source.media_type; + } + if (typeof data !== "string" || data === "") return block; + let hash: string; + if (data.startsWith(BLOB_PREFIX)) { + hash = data.slice(BLOB_PREFIX.length); + if (!HASH_RE.test(hash)) return block; + if (!existsSync(path.join(getBlobsDir(), hash)) && !existsSync(path.join(mediaDir(), hash))) { + return { type: "text", text: `[image unavailable: blob ${hash.slice(0, 12)}… not found]` }; + } + } else { + try { + hash = storeInlineImage(data); + } catch { + return block; // Disk trouble: keep showing the inline image. + } + } + return { type: "image", ...(typeof mimeType === "string" ? { mimeType } : {}), url: `/api/media/${hash}` }; +} + +/** Tool-result content with every image replaced by a media URL. */ +export function toolResultContentAsUrls(content: T): T { + if (!Array.isArray(content) || !content.some((block) => isRecord(block) && block.type === "image")) return content; + return content.map(imageBlockAsUrl) as T; +} + +function withContentAsUrls(value: unknown): unknown { + if (!isRecord(value)) return value; + const content = toolResultContentAsUrls(value.content); + return content === value.content ? value : { ...value, content }; +} + +function toolResultMessageAsUrls(message: unknown): unknown { + return isRecord(message) && message.role === "toolResult" ? withContentAsUrls(message) : message; +} + +function mapMessages(value: unknown): unknown { + if (!Array.isArray(value)) return value; + const mapped = value.map(toolResultMessageAsUrls); + return mapped.some((message, i) => message !== value[i]) ? mapped : value; +} + +/** An omp agent event with its tool-result images as media URLs (same object when it has none). */ +export function eventWithToolResultImageUrls(event: E): E { + const swap = (key: string, value: unknown): E => (value === event[key] ? event : { ...event, [key]: value }); + switch (event.type) { + case "tool_execution_update": return swap("partialResult", withContentAsUrls(event.partialResult)); + case "tool_execution_end": return swap("result", withContentAsUrls(event.result)); + case "message_start": + case "message_end": return swap("message", toolResultMessageAsUrls(event.message)); + case "turn_end": return swap("toolResults", mapMessages(event.toolResults)); + case "agent_end": return swap("messages", mapMessages(event.messages)); + case "subagent_event": { + // The child's raw event rides in `payload.event`. + const payload = event.payload; + if (!isRecord(payload) || !isRecord(payload.event) || typeof payload.event.type !== "string") return event; + const inner = eventWithToolResultImageUrls(payload.event as { type: string }); + return inner === payload.event ? event : swap("payload", { ...payload, event: inner }); + } + default: return event; + } +} + +/** + * File holding a stored image: omp's blob, else omp-web's copy. omp writes + * blobs in place, so a blob that differs in size from our copy is still being + * written and the copy is served instead. + */ +export function mediaFilePath(hash: string): string | null { + if (!HASH_RE.test(hash)) return null; + const blob = path.join(getBlobsDir(), hash); + const copy = path.join(mediaDir(), hash); + const blobSize = fileSize(blob); + const copySize = fileSize(copy); + if (blobSize !== undefined && (copySize === undefined || blobSize === copySize)) return blob; + return copySize === undefined ? null : copy; +} + +/** Raster image type from magic bytes; null for anything else (the blob store also holds non-images). */ +export function imageMimeType(bytes: Buffer): string | null { + if (bytes.subarray(0, 8).equals(Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]))) return "image/png"; + if (bytes[0] === 0xff && bytes[1] === 0xd8 && bytes[2] === 0xff) return "image/jpeg"; + if (bytes.subarray(0, 4).toString("latin1") === "GIF8") return "image/gif"; + if (bytes.subarray(0, 4).toString("latin1") === "RIFF" && bytes.subarray(8, 12).toString("latin1") === "WEBP") return "image/webp"; + return null; +} + +/** Existing thumbnail for `hash`, if one was generated before. */ +export function existingThumbnail(hash: string): string | null { + const thumb = path.join(mediaDir(), `${hash}.thumb.webp`); + return HASH_RE.test(hash) && existsSync(thumb) ? thumb : null; +} + +/** Generate and cache a small WebP preview; null when sharp is unavailable or fails. */ +export async function createThumbnail(hash: string, image: Buffer): Promise { + let thumb: Buffer; + try { + // Dynamic on purpose: sharp's native binary is platform-specific and may be + // missing, and this module is also loaded by the session reader and RPC + // manager, which must not fail or load libvips just to rewrite URLs. + const { default: sharp } = await import("sharp"); + thumb = await sharp(image) + .rotate() // apply EXIF orientation; WebP output drops the tag + .resize(THUMB_PX, THUMB_PX, { fit: "inside", withoutEnlargement: true }) + .webp({ quality: 80 }) + .toBuffer(); + } catch { + return null; + } + try { + writeAtomic(path.join(mediaDir(), `${hash}.thumb.webp`), thumb); + } catch { + // Not cached this time; the preview is still served. + } + return thumb; +} diff --git a/lib/media-route.test.mjs b/lib/media-route.test.mjs new file mode 100644 index 000000000..32958a5f6 --- /dev/null +++ b/lib/media-route.test.mjs @@ -0,0 +1,143 @@ +import assert from "node:assert/strict"; +import { createHash } from "node:crypto"; +import { existsSync, mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import test, { after } from "node:test"; +import { fileURLToPath } from "node:url"; +import { createJiti } from "jiti"; +import sharp from "sharp"; + +// Real GET /api/media/[hash] against a throwaway agent dir. +const agentDir = mkdtempSync(join(tmpdir(), "omp-web-media-route-")); +const previousAgentDir = process.env.PI_CODING_AGENT_DIR; +process.env.PI_CODING_AGENT_DIR = agentDir; +after(() => { + if (previousAgentDir === undefined) delete process.env.PI_CODING_AGENT_DIR; + else process.env.PI_CODING_AGENT_DIR = previousAgentDir; + rmSync(agentDir, { recursive: true, force: true }); +}); + +const jiti = createJiti(import.meta.url, { + tryNative: false, + alias: { "@/": fileURLToPath(new URL("../", import.meta.url)) }, +}); +const route = await jiti.import("../app/api/media/[hash]/route.ts"); +const { eventWithToolResultImageUrls } = await jiti.import("./media-cache.ts"); + +const blobs = join(agentDir, "blobs"); +const media = join(agentDir, "omp-web", "media"); +mkdirSync(blobs, { recursive: true }); +const sha = (bytes) => createHash("sha256").update(bytes).digest("hex"); +const get = (hash, query = "") => route.GET(new Request(`http://localhost/api/media/${hash}${query}`), { params: Promise.resolve({ hash }) }); + +const screenshot = await sharp({ create: { width: 1600, height: 1000, channels: 3, background: "#336699" } }).png().toBuffer(); +const screenshotHash = sha(screenshot); +writeFileSync(join(blobs, screenshotHash), screenshot); + +test("serves the full image with its real type, and a small cached WebP preview", async () => { + const full = await get(screenshotHash); + assert.equal(full.status, 200); + assert.equal(full.headers.get("content-type"), "image/png"); + assert.deepEqual(Buffer.from(await full.arrayBuffer()), screenshot); + + const thumb = await get(screenshotHash, "?thumb=1"); + assert.equal(thumb.headers.get("content-type"), "image/webp"); + const bytes = Buffer.from(await thumb.arrayBuffer()); + const { width, height } = await sharp(bytes).metadata(); + assert.deepEqual([width, height], [480, 300]); + assert.ok(bytes.length < screenshot.length); + assert.ok(existsSync(join(media, `${screenshotHash}.thumb.webp`)), "preview is generated once and kept"); +}); + +test("previews are reused and follow EXIF orientation", async () => { + // Cached: served even after the source is gone. + const portrait = await sharp({ create: { width: 800, height: 400, channels: 3, background: "#123456" } }) + .jpeg().withMetadata({ orientation: 6 }).toBuffer(); + const hash = sha(portrait); + writeFileSync(join(blobs, hash), portrait); + const first = Buffer.from(await (await get(hash, "?thumb=1")).arrayBuffer()); + const { width, height } = await sharp(first).metadata(); + assert.deepEqual([width, height], [240, 480], "a phone photo stays upright in the preview"); + rmSync(join(blobs, hash)); + const again = await get(hash, "?thumb=1"); + assert.equal(again.status, 200); + assert.deepEqual(Buffer.from(await again.arrayBuffer()), first); +}); + +test("never serves non-images or files outside the store", async () => { + const text = Buffer.from("data:text/html,"); + writeFileSync(join(blobs, sha(text)), text); + assert.equal((await get(sha(text))).status, 415); + assert.equal((await get(sha(text), "?thumb=1")).status, 415); + // A real image one level up must stay unreachable however the name is spelled. + writeFileSync(join(agentDir, "secret.png"), screenshot); + for (const name of ["../secret.png", "..%2Fsecret.png", `../blobs/${screenshotHash}`, `${screenshotHash}.png`, screenshotHash.toUpperCase()]) { + assert.equal((await get(name)).status, 404, name); + } +}); + +test("live tool output is served from omp-web's copy until omp has written the whole blob", async () => { + const live = await sharp({ create: { width: 64, height: 64, channels: 3, background: "#ff0000" } }).png().toBuffer(); + const hash = sha(live); + const event = { + type: "tool_execution_end", + toolCallId: "t1", + result: { content: [{ type: "text", text: "Screenshot captured" }, { type: "image", data: live.toString("base64"), mimeType: "image/png" }] }, + }; + const rewritten = eventWithToolResultImageUrls(event); + assert.deepEqual(rewritten.result.content[1], { type: "image", mimeType: "image/png", url: `/api/media/${hash}` }); + assert.equal(event.result.content[1].data, live.toString("base64"), "omp's frame is not mutated"); + assert.deepEqual(Buffer.from(await (await get(hash)).arrayBuffer()), live); + + // omp writes blobs in place: a half-written blob must not be served (as immutable). + writeFileSync(join(blobs, hash), live.subarray(0, 40)); + assert.deepEqual(Buffer.from(await (await get(hash)).arrayBuffer()), live); + + writeFileSync(join(blobs, hash), live); + assert.deepEqual(Buffer.from(await (await get(hash)).arrayBuffer()), live); + // The daily sweep drops the copy omp now holds; it runs on the next write. + globalThis.__ompWebMediaSweptAt = 0; + eventWithToolResultImageUrls({ type: "tool_execution_end", result: { content: [{ type: "image", data: Buffer.from("other").toString("base64") }] } }); + assert.equal(existsSync(join(media, hash)), false); +}); + +test("a live image arriving while omp is still writing its blob is kept and served whole", async () => { + const live = await sharp({ create: { width: 48, height: 48, channels: 3, background: "#00ff00" } }).png().toBuffer(); + const hash = sha(live); + writeFileSync(join(blobs, hash), live.subarray(0, 30)); + eventWithToolResultImageUrls({ type: "tool_execution_end", result: { content: [{ type: "image", data: live.toString("base64") }] } }); + assert.deepEqual(Buffer.from(await (await get(hash)).arrayBuffer()), live); +}); + +test("an image seen again after its copy was swept is stored again", async () => { + const live = await sharp({ create: { width: 40, height: 40, channels: 3, background: "#0000ff" } }).png().toBuffer(); + const hash = sha(live); + const frame = () => eventWithToolResultImageUrls({ type: "tool_execution_end", result: { content: [{ type: "image", data: live.toString("base64") }] } }); + frame(); + rmSync(join(media, hash)); + frame(); + assert.deepEqual(Buffer.from(await (await get(hash)).arrayBuffer()), live); +}); + +test("tool-result images become URLs in every event that carries them, and only there", () => { + const image = { type: "image", data: Buffer.from("event image").toString("base64"), mimeType: "image/png" }; + const url = `/api/media/${sha(Buffer.from("event image"))}`; + const toolResult = { role: "toolResult", toolCallId: "t", content: [image] }; + const user = { role: "user", content: [image] }; + const asUrl = [{ type: "image", mimeType: "image/png", url }]; + + assert.deepEqual(eventWithToolResultImageUrls({ type: "message_start", message: toolResult }).message.content, asUrl); + assert.deepEqual(eventWithToolResultImageUrls({ type: "message_end", message: user }).message, user, "user images stay inline"); + const turn = eventWithToolResultImageUrls({ type: "turn_end", message: user, toolResults: [toolResult] }); + assert.deepEqual(turn.toolResults[0].content, asUrl); + assert.equal(turn.message, user); + const end = eventWithToolResultImageUrls({ type: "agent_end", messages: [user, toolResult] }); + assert.equal(end.messages[0], user); + assert.deepEqual(end.messages[1].content, asUrl); + const sub = eventWithToolResultImageUrls({ type: "subagent_event", payload: { id: "Child", event: { type: "tool_execution_end", result: { content: [image] } } } }); + assert.equal(sub.payload.id, "Child"); + assert.deepEqual(sub.payload.event.result.content, asUrl, "a subagent's screenshot is not streamed as base64 either"); + const plain = { type: "agent_end", messages: [user] }; + assert.equal(eventWithToolResultImageUrls(plain), plain, "events without tool images are passed through as-is"); +}); diff --git a/lib/omp/session-files.ts b/lib/omp/session-files.ts index 598aab3bd..675b0bd76 100644 --- a/lib/omp/session-files.ts +++ b/lib/omp/session-files.ts @@ -344,8 +344,8 @@ export function containsBlobRef(value: unknown): boolean { } export interface ResolveBlobOptions { - /** Leave blob refs inside toolResult messages unresolved (the caller is about - * to omit those images from the payload anyway). */ + /** Leave blob refs inside toolResult messages unresolved (the caller turns + * those images into /api/media URLs instead of inlining them). */ skipToolResultImages?: boolean; } diff --git a/lib/rpc-manager.test.mjs b/lib/rpc-manager.test.mjs index 88b64dd09..e2e90e923 100644 --- a/lib/rpc-manager.test.mjs +++ b/lib/rpc-manager.test.mjs @@ -1,5 +1,8 @@ import assert from "node:assert/strict"; import { readFile } from "node:fs/promises"; +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; import test from "node:test"; import { createJiti } from "jiti"; import childProcess from "node:child_process"; @@ -813,6 +816,29 @@ test("live snapshots restore quiet partial messages and tools, with ordered even assert.deepEqual(wrapper.getStreamSnapshot().toolEvents, []); }); +test("live tool-result images reach tabs and the replay snapshot as media URLs", async (t) => { + const agentDir = mkdtempSync(join(tmpdir(), "omp-web-live-media-")); + const previous = process.env.PI_CODING_AGENT_DIR; + process.env.PI_CODING_AGENT_DIR = agentDir; + t.after(() => { + if (previous === undefined) delete process.env.PI_CODING_AGENT_DIR; + else process.env.PI_CODING_AGENT_DIR = previous; + rmSync(agentDir, { recursive: true, force: true }); + }); + const { wrapper, emit } = snapshotSession(t); + const events = []; + wrapper.onEvent((event) => events.push(event)); + const image = { type: "image", data: Buffer.from("png bytes").toString("base64"), mimeType: "image/png" }; + emit({ type: "tool_execution_start", toolCallId: "shot", toolName: "eval" }); + emit({ type: "tool_execution_update", toolCallId: "shot", partialResult: { content: [image] } }); + const url = events[1].partialResult.content[0].url; + assert.match(url, /^\/api\/media\/[a-f0-9]{64}$/); + assert.equal(wrapper.getStreamSnapshot().toolEvents[0].partialResult.content[0].url, url, "reconnecting tabs get the URL too"); + emit({ type: "message_end", message: { role: "toolResult", toolCallId: "shot", content: [{ type: "text", text: "done" }, image] } }); + assert.deepEqual(events[2].message.content[1], { type: "image", mimeType: "image/png", url }); + assert.equal(JSON.stringify(events).includes(image.data), false, "no base64 reaches a tab"); +}); + test("terminal failures, new runs, aborts and idle reconciliation cannot revive old partial output", async (t) => { let failPrompt = false; const { wrapper, emit } = snapshotSession(t, async (command) => { diff --git a/lib/rpc-manager.ts b/lib/rpc-manager.ts index 9bc9a42f7..1529cbe3c 100644 --- a/lib/rpc-manager.ts +++ b/lib/rpc-manager.ts @@ -22,6 +22,7 @@ import { samePath } from "./paths"; import { isRecord } from "./type-guards"; import { parseSkillDiagnosticsSnapshot, type SkillDiagnosticsSnapshot } from "./skill-diagnostics"; import { isReservedLaunchArg, loadProjectRegistry } from "./project-registry"; +import { eventWithToolResultImageUrls } from "./media-cache"; import type { BashResultInfo, OmpModel, @@ -990,7 +991,10 @@ export class AgentSessionWrapper { this.pendingHostUris.clear(); } - private emit(event: UnsequencedAgentEvent): void { + private emit(nativeEvent: UnsequencedAgentEvent): void { + // Tool-result images go out (and into the replay snapshot) as media URLs: + // big screenshots must never sit in the browser's chat state as base64. + const event = eventWithToolResultImageUrls(nativeEvent); // `web` belongs to this wrapper, never to native/extension-supplied frames. // Strip it before caching tool snapshots as well as before wire emission. delete event.web; diff --git a/lib/session-reader.test.mjs b/lib/session-reader.test.mjs index 30d10b88f..2b72af261 100644 --- a/lib/session-reader.test.mjs +++ b/lib/session-reader.test.mjs @@ -1,5 +1,6 @@ import assert from "node:assert/strict"; -import { appendFileSync, closeSync, mkdirSync, mkdtempSync, openSync, renameSync, rmSync, statSync, truncateSync, unlinkSync, utimesSync, writeFileSync, writeSync } from "node:fs"; +import { appendFileSync, closeSync, mkdirSync, mkdtempSync, openSync, readFileSync, renameSync, rmSync, statSync, truncateSync, unlinkSync, utimesSync, writeFileSync, writeSync } from "node:fs"; +import { createHash } from "node:crypto"; import { tmpdir } from "node:os"; import { join } from "node:path"; import test from "node:test"; @@ -308,7 +309,7 @@ test("deferThinking tolerates string-content assistant entries without crashing" assert.equal(context.messages[1].content, "legacy plain-text answer"); }); -test("defers only base64 images from historical tool results", () => { +test("deferred history sends tool-result images as media URLs and keeps user images inline", () => withAgentDir(() => { const userImage = { type: "image", source: { type: "base64", media_type: "image/png", data: "QUJDRA==" }, @@ -321,11 +322,13 @@ test("defers only base64 images from historical tool results", () => { type: "image", source: { type: "url", url: "https://example.com/result.png" }, }; - const flatToolImage = { - type: "image", - data: "QUJDRA==", - mimeType: "image/png", - }; + const flatToolImage = { type: "image", data: "QUJDRA==", mimeType: "image/png" }; + const blobHash = "a".repeat(64); + const blobToolImage = { type: "image", data: `blob:sha256:${blobHash}`, mimeType: "image/webp" }; + mkdirSync(join(process.env.PI_CODING_AGENT_DIR, "blobs"), { recursive: true }); + writeFileSync(join(process.env.PI_CODING_AGENT_DIR, "blobs", blobHash), "webp bytes"); + const missingHash = "b".repeat(64); + const missingToolImage = { type: "image", data: `blob:sha256:${missingHash}`, mimeType: "image/webp" }; const entries = [ userEntry("u1", null, [{ type: "text", text: "inspect this" }, userImage]), assistantEntry("a1", "u1", "reading"), @@ -337,27 +340,30 @@ test("defers only base64 images from historical tool results", () => { message: { role: "toolResult", toolCallId: "call1", - content: [ - { type: "text", text: "Read image file" }, - toolImage, - flatToolImage, - toolUrlImage, - ], + content: [{ type: "text", text: "Read image file" }, toolImage, flatToolImage, blobToolImage, missingToolImage, toolUrlImage], }, }, ]; + // omp's blob address: sha256 of the decoded bytes ("ABCD"). + const abcd = createHash("sha256").update("ABCD").digest("hex"); const deferred = buildSessionContext(entries, undefined, { deferToolResultImages: true }); assert.deepEqual(deferred.messages[0].content[1], userImage); - assert.deepEqual(deferred.messages[2].content[0], { type: "text", text: "Read image file" }); - assert.deepEqual(deferred.messages[2].content[1], toolUrlImage); - assert.match(deferred.messages[2].content[2].text, /2 tool result images omitted.*image\/jpeg, image\/png.*~8 bytes/); + assert.deepEqual(deferred.messages[2].content, [ + { type: "text", text: "Read image file" }, + { type: "image", mimeType: "image/jpeg", url: `/api/media/${abcd}` }, + { type: "image", mimeType: "image/png", url: `/api/media/${abcd}` }, + { type: "image", mimeType: "image/webp", url: `/api/media/${blobHash}` }, + // Same placeholder as the non-deferred path, instead of a URL that 404s. + { type: "text", text: `[image unavailable: blob ${missingHash.slice(0, 12)}… not found]` }, + toolUrlImage, + ]); + // Inline bytes omp never externalized are kept for the media route. + assert.equal(readFileSync(join(process.env.PI_CODING_AGENT_DIR, "omp-web", "media", abcd), "utf8"), "ABCD"); const full = buildSessionContext(entries); - assert.deepEqual(full.messages[2].content[1], toolImage); - assert.deepEqual(full.messages[2].content[2], flatToolImage); - assert.deepEqual(full.messages[2].content[3], toolUrlImage); -}); + assert.deepEqual(full.messages[2].content.slice(1, 3), [toolImage, flatToolImage]); +})); test("preserves hidden custom messages so the UI can render them collapsed", () => { const entries = [ diff --git a/lib/session-reader.ts b/lib/session-reader.ts index 8d2147ab9..c365d5b1c 100644 --- a/lib/session-reader.ts +++ b/lib/session-reader.ts @@ -25,6 +25,7 @@ import type { } from "./types"; import { normalizeToolCalls } from "./normalize"; import { isRecord } from "./type-guards"; +import { toolResultContentAsUrls } from "./media-cache"; import { taskResultRetryFailure, taskResultStructuredOutput, taskResultUsageCost } from "./task-result-details"; import type { TodoPhase } from "./pi-types"; import { projectIdentityKey, sessionPathKey } from "./paths"; @@ -466,7 +467,7 @@ function toDisplayEntries(entries: SessionEntry[], options: ResolveBlobOptions): entry.type === "message" && ((entry.message as { role?: string } | null | undefined)?.role === "toolResult") ) { - // Caller omits these images anyway — share the entry without blob work. + // Caller turns these images into media URLs — share the entry without blob work. out[i] = entry; continue; } @@ -940,24 +941,6 @@ function withNormalizedTimestamp(message: AgentMessage): AgentMessage { return timestamp === raw ? message : { ...message, timestamp }; } -function base64ImageInfo(block: unknown): { bytes: number; mime?: string } | null { - if (!isRecord(block) || block.type !== "image") return null; - - let data: string | undefined; - let mime: string | undefined; - if (typeof block.data === "string") { - data = block.data; - mime = typeof block.mimeType === "string" ? block.mimeType : undefined; - } else if (isRecord(block.source) && block.source.type === "base64" && typeof block.source.data === "string") { - data = block.source.data; - mime = typeof block.source.media_type === "string" ? block.source.media_type : undefined; - } - if (!data) return null; - - const padding = data.endsWith("==") ? 2 : data.endsWith("=") ? 1 : 0; - return { bytes: Math.max(0, Math.floor(data.length * 3 / 4) - padding), mime }; -} - /** * toolResult `details` is provider/tool-internal metadata that dominates real * history payloads (measured 643KB of a 1.34MB context on a 533-entry session), @@ -1128,32 +1111,10 @@ function stripToolResultDetails(message: AgentMessage): AgentMessage { return rest; } -function omitToolResultBase64Images(message: AgentMessage): AgentMessage { +function toolResultImagesAsUrls(message: AgentMessage): AgentMessage { if (message.role !== "toolResult") return message; - // Shape-malformed-but-JSON-valid files can carry a string content here - // (import accepts arbitrary content); the loader tolerates such lines, so - // the converter must too instead of crashing the whole session view. - if (!Array.isArray(message.content)) return message; - - let omitted = 0; - let bytes = 0; - const mimes = new Set(); - const content = message.content.filter((block) => { - const image = base64ImageInfo(block); - if (!image) return true; - omitted += 1; - bytes += image.bytes; - if (image.mime) mimes.add(image.mime); - return false; - }); - if (omitted === 0) return message; - - const mimeText = mimes.size > 0 ? `: ${[...mimes].join(", ")}` : ""; - content.push({ - type: "text", - text: `[${omitted} tool result image${omitted === 1 ? "" : "s"} omitted from initial history payload${mimeText}, ~${bytes} bytes]`, - }); - return { ...message, content }; + const content = toolResultContentAsUrls(message.content); + return content === message.content ? message : { ...message, content }; } function compactionUiMessage(entry: CompactionEntry, active: boolean): CustomMessage { @@ -1226,7 +1187,7 @@ export function entryToUiMessage( }; } const normalized = options.deferToolResultImages - ? omitToolResultBase64Images(normalizeToolCalls(raw)) + ? toolResultImagesAsUrls(normalizeToolCalls(raw)) : normalizeToolCalls(raw); const message = stripToolResultDetails(normalized); if (!options.deferThinking || message.role !== "assistant") return withNormalizedTimestamp(message); diff --git a/lib/types.ts b/lib/types.ts index cc9a2deee..469f6a3d3 100644 --- a/lib/types.ts +++ b/lib/types.ts @@ -36,12 +36,15 @@ export interface TextContent { /** * Image block. omp persists the flat {data, mimeType} shape (data may be a * `blob:sha256:` reference until resolved); legacy pi entries used the nested - * Anthropic-style `source` shape. The UI handles both. + * Anthropic-style `source` shape. The UI handles both. Tool-result images + * reach the browser as `url` (`/api/media/`, `?thumb=1` for a small + * preview) instead of inline base64. */ export interface ImageContent { type: "image"; data?: string; mimeType?: string; + url?: string; source?: { type: "base64" | "url"; media_type?: string; diff --git a/package-lock.json b/package-lock.json index 3d209ff19..7639d4201 100644 --- a/package-lock.json +++ b/package-lock.json @@ -28,6 +28,7 @@ "rehype-sanitize": "^6.0.0", "remark-gfm": "^4.0.1", "remark-math": "^6.0.0", + "sharp": "^0.35.4", "undici": "8.10.1", "yaml": "^2" }, @@ -1006,7 +1007,6 @@ "resolved": "https://registry.npmjs.org/@img/colour/-/colour-1.1.0.tgz", "integrity": "sha512-Td76q7j57o/tLVdgS746cYARfSyxk8iEfRxewL9h4OMzYhbW4TAcppl0mT4eyqXddh6L/jwoM75mo7ixa/pCeQ==", "license": "MIT", - "optional": true, "engines": { "node": ">=18" } @@ -5152,7 +5152,6 @@ "version": "2.1.2", "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-2.1.2.tgz", "integrity": "sha512-Btj2BOOO83o3WyH59e8MgXsxEQVcarkUOpEYrubB0urwnN10yQ364rsiByU11nZlqWYZm05i/of7io4mzihBtQ==", - "devOptional": true, "license": "Apache-2.0", "engines": { "node": ">=8" @@ -11061,7 +11060,6 @@ "resolved": "https://registry.npmjs.org/sharp/-/sharp-0.35.4.tgz", "integrity": "sha512-n++8XWcj+jCOr2IOl7h8LbKnGBDY4aPbmprMONBNFdn0ImXqpGVv5zliDs0V9HbmbCQLpbuo2ej9rAoOQTvMDA==", "license": "Apache-2.0", - "optional": true, "dependencies": { "@img/colour": "^1.1.0", "detect-libc": "^2.1.2", @@ -11111,7 +11109,6 @@ "resolved": "https://registry.npmjs.org/semver/-/semver-7.8.5.tgz", "integrity": "sha512-Y7/KDsb8LjooZpwaqGyulO6DQlksgCncchHGk+sZIY4SBvUocMBEFH5Ur1fI4dV+Jvl0w6cjvucaIi40puRioA==", "license": "ISC", - "optional": true, "bin": { "semver": "bin/semver.js" }, diff --git a/package.json b/package.json index 73f7c9307..9cbd90aa9 100644 --- a/package.json +++ b/package.json @@ -62,6 +62,7 @@ "@base-ui/react": "^1.7.0", "cmdk": "^1.1.1", "dbus-next": "^0.10.2", + "jiti": "^2.7.0", "katex": "^0.16.47", "lucide-react": "^1.39.0", "mammoth": "^1.12.2", @@ -77,9 +78,9 @@ "rehype-sanitize": "^6.0.0", "remark-gfm": "^4.0.1", "remark-math": "^6.0.0", + "sharp": "^0.35.4", "undici": "8.10.1", - "yaml": "^2", - "jiti": "^2.7.0" + "yaml": "^2" }, "devDependencies": { "@tailwindcss/postcss": "^4.3.3",