diff --git a/src/preflight.ts b/src/preflight.ts index 88a0004ec1..ae89e704a9 100644 --- a/src/preflight.ts +++ b/src/preflight.ts @@ -243,8 +243,10 @@ export interface PreflightDeps { proxyUrl?: string; signal?: AbortSignal; log: (level: string, msg: string) => void; - /** #1843: the image reserve for this request — the billed cost of the images riding the payload verbatim (#488). Folding never removes them, so every TEXT-channel decision compares the text estimate against `limit − reserve` instead of adding the reserve to each total; an image-estimate error can no longer move a fold decision. Kernel-facing quantities keep the total view (reserve added back), which is the conservative direction. */ + /** #1843: the image reserve — the billed cost of the images riding the payload verbatim (#488). Every TEXT-channel decision compares the text estimate against `limit − reserve` instead of adding the reserve to each total, so an image-estimate error can no longer move a fold decision; kernel-facing quantities keep the total view (reserve added back), which is the conservative direction. This field is the INITIAL wire view; on lanes where folds archive images off the wire (`measureImageReserve`), the working copy is re-measured from the post-fold view every round (#4932). */ imageReserve?: number; + /** Rebuild the protocol image view after folds; archived images no longer consume the window. */ + measureImageReserve?: (messages: CoreMessage[]) => number; /** Constant wire overhead for this request (system prompt + tool definitions, #470). Folding never removes it, so every fit decision must add it — otherwise the loop stops with "text fits" while the billed input still overflows. */ wireOverhead?: number; /** #553: the caller knows this session's input size is unmeasured AND its @@ -1634,11 +1636,11 @@ export async function preflightCompress(deps: PreflightDeps, messages: CoreMessa // text estimate vs `target − imageReserve`. Algebraically the same firing // set as the old total-view check (max(B, T+R) >= C <=> max(max(0,B−R), T) // >= max(0,C−R)), but an image-estimate error can no longer start or stop - // folding. Images ride the payload verbatim through every round, so the - // reserve is constant for this invocation. Kernel-facing quantities below + // folding for a fixed wire view. Archived images leave that view, so the host + // refreshes their reserve after each fold. Kernel-facing quantities below // keep the total view (reserve added back): conservative direction, and the // kernel's own truncation/absorb behavior stays byte-identical. - const imageReserve = deps.imageReserve ?? 0; + let imageReserve = deps.imageReserve ?? 0; const wireOverhead = deps.wireOverhead ?? 0; // #1933 F1: capture k̂ once — it only mutates on usage settlement (outside // this invocation), while every fit judgment in this loop must stay on @@ -1657,7 +1659,25 @@ export async function preflightCompress(deps: PreflightDeps, messages: CoreMessa // for k̂=1 / absent / mismatched-origin (applyEstimateCalibration → raw), so // the uncalibrated #1841 path stays byte-identical. const calibrateFoldMass = (rawMass: number): number => applyEstimateCalibration(rawMass, kFactor, kOrigin, deps.upstreamOrigin); - let textTarget = Math.max(0, Math.min(limit, deps.compressionTarget ?? limit) - imageReserve); + let foldTarget = Math.min(limit, deps.compressionTarget ?? limit); + let textTarget = Math.max(0, foldTarget - imageReserve); + const refreshImages = (view: CoreMessage[]): void => { + if (deps.measureImageReserve) imageReserve = deps.measureImageReserve(view); + textTarget = Math.max(0, foldTarget - imageReserve); + }; + // Removing every nominated source gives an upper bound on image savings. + // This is only a futility bound: the real post-fold view decides fit. + const imageSavingBound = (ranges: Array<{ startRef: string; endRef: string }>, view: CoreMessage[]): number => { + if (!deps.measureImageReserve || imageReserve <= 0) return 0; + const { refToIdx } = refMaps(messages, deps.session.state); + const removed = new Set(); + for (const range of ranges) { + const start = refToIdx.get(range.startRef), end = refToIdx.get(range.endRef); + if (start === undefined || end === undefined || start > end) continue; + for (let i = start; i <= end; i++) removed.add(messages[i]!.id); + } + return Math.max(0, imageReserve - deps.measureImageReserve(view.filter(message => !removed.has(message.id)))); + }; const result: PreflightResult = { compressedRanges: 0, savedTokens: 0, payloadEstimate: applyEstimateCalibration(estimateCoreMessages(messages) + wireOverhead, kFactor, kOrigin, deps.upstreamOrigin) + imageReserve, rangesRemaining: 0, fitsWindow: true, hadFoldableCandidates: false }; // #4288: set when the round-top fit exit fired — the walk ended in a // forward decision, and no later verdict may override it (see the final @@ -1844,6 +1864,7 @@ export async function preflightCompress(deps: PreflightDeps, messages: CoreMessa deps.session.state = turn.state; adoptContentStore(deps.session, turn.contentStore); const view = applyAbsorbView(turn.messages, turn.state, activeConfig, currentTokens); + refreshImages(view); const est = applyEstimateCalibration(estimateCoreMessages(view) + wireOverhead, kFactor, kOrigin, deps.upstreamOrigin) + imageReserve; result.payloadEstimate = est; // #2952 review: keep the char-count UPPER bound fresh through recovery. @@ -2081,6 +2102,7 @@ export async function preflightCompress(deps: PreflightDeps, messages: CoreMessa const baselineFloor = messages.length > 0 ? ((deps.session.stats.lastInputTokensSource === "usage" || deps.session.stats.lastInputTokensSource === "overflow-arm") ? deps.session.stats.lastInputTokens : 0) : deps.session.stats.lastInputTokens; + refreshImages(turn.messages); const rawRoundText = estimateCoreMessages(turn.messages) + wireOverhead; // #1933 F1: same calibrated caliber as the gate's trigger/fit checks — // per-round exit and final fit must judge this payload identically to @@ -2221,7 +2243,7 @@ export async function preflightCompress(deps: PreflightDeps, messages: CoreMessa }, 0); let potential = resolvableMass(ranges); const deficit = result.payloadEstimate - limit; - if (potential > 0 && deficit > 0 && calibrateFoldMass(potential) * FUTILITY_SLACK < deficit) { + if (potential > 0 && deficit > 0 && imageSavingBound(ranges, turn.messages) === 0 && calibrateFoldMass(potential) * FUTILITY_SLACK < deficit) { if (!relaxed) { // The soft-protected recent zone is not in `ranges` yet; #330 // makes it foldable on relax. Probe the relaxed view (CPU-only, @@ -2239,10 +2261,11 @@ export async function preflightCompress(deps: PreflightDeps, messages: CoreMessa // #2638: structural verdicts hold under relax too (coverage-based, zone-independent) — dead mass cannot close the gap either. const relaxedRanges = filterDeadRanges(deps.session, viableRanges(probe.nudge?.compressibleRanges ?? []).filter((r) => minChars <= 0 || (r.chars ?? r.tokens * 4) >= minChars)); const relaxedPotential = resolvableMass(relaxedRanges); - if (calibrateFoldMass(relaxedPotential) * FUTILITY_SLACK >= deficit) { + if (imageSavingBound(relaxedRanges, probe.messages) > 0 || calibrateFoldMass(relaxedPotential) * FUTILITY_SLACK >= deficit) { activeConfig = relaxedConfig(deps.config); relaxed = true; - textTarget = Math.max(0, limit - imageReserve); + foldTarget = limit; + textTarget = Math.max(0, foldTarget - imageReserve); summaryCalls = 0; budgetHit = false; transientRetryBudget = TRANSIENT_EMPTY_RETRY_BUDGET; @@ -2283,7 +2306,8 @@ export async function preflightCompress(deps: PreflightDeps, messages: CoreMessa if (!relaxed && result.payloadEstimate >= limit) { activeConfig = relaxedConfig(deps.config); relaxed = true; - textTarget = Math.max(0, limit - imageReserve); + foldTarget = limit; + textTarget = Math.max(0, foldTarget - imageReserve); // #575-merge: the summarization budget counts per protection // regime — reset it on relax, else bad summaries burned under // normal protection can starve the relaxed walk entirely and @@ -2658,7 +2682,8 @@ export async function preflightCompress(deps: PreflightDeps, messages: CoreMessa if (si !== undefined && ei !== undefined && si <= ei) remaining += rj.tokens; } const deficit = result.payloadEstimate - limit; - if (remaining > 0 && deficit > 0 && calibrateFoldMass(remaining) * FUTILITY_SLACK < deficit) { + const remainingRanges = ordered.slice(oi).filter(r => !skipSet.has(`${r.startRef}:${r.endRef}`)); + if (remaining > 0 && deficit > 0 && imageSavingBound(remainingRanges, turn.messages) === 0 && calibrateFoldMass(remaining) * FUTILITY_SLACK < deficit) { futileBail = true; bailRemaining = calibrateFoldMass(remaining); bailDeficit = deficit; diff --git a/src/responses-tool-output.ts b/src/responses-tool-output.ts index 155f82fda0..a31e7f140e 100644 --- a/src/responses-tool-output.ts +++ b/src/responses-tool-output.ts @@ -3,6 +3,11 @@ import { coreToResponses, patchResponsesInput, responsesToCore, type BiliMessage import { isRenderedSummaryMessage } from "acp-kernel"; import { systemToUser } from "./util.js"; +const responsesPartBinding = Symbol("responsesPartBinding"); +type PartBoundMessage = CoreMessage & { + [responsesPartBinding]?: { source: ResponseInputItem; part: ResponseContentPart; sourceId: string }; +}; + export function responsesToolImageParts(item: unknown): ResponseContentPart[] | undefined { if (typeof item !== "object" || item === null) return undefined; const raw = item as Record; @@ -27,6 +32,23 @@ function outputText(parts: ResponseContentPart[]): string { export function responsesToCoreWithToolImages(body: ResponsesRequestBody): ResponsesProjection { const projection = responsesToCore(body); + // The layout supplies exact source ownership, including duplicate images. + // Keep it on the core while it survives rendering; never guess ownership + // later from text equality or the first image in a multipart carrier. + const byId = new Map(projection.msgs.map(message => [message.id, message])); + for (const slot of projection.layout) { + const source = slot.original; + if (source.role !== "user" || !Array.isArray(source.content)) continue; + const parts = source.content as ResponseContentPart[]; + if (!parts.every(part => part.type === "input_image" || part.type === "input_text" || part.type === "output_text")) continue; + const emitted = parts.filter(part => part.type === "input_image" || (typeof part.text === "string" && part.text.length > 0)); + const ids = slot.coreIds ?? (slot.coreId ? [slot.coreId] : []); + if (ids.length !== emitted.length) continue; + ids.forEach((id, index) => { + const message = byId.get(id) as PartBoundMessage | undefined; + if (message) message[responsesPartBinding] = { source, part: emitted[index]!, sourceId: id }; + }); + } for (const message of projection.msgs) { const parts = responsesToolImageParts(message.rawResponsesItem); // Keep the codec's original content-derived id so persisted refs still @@ -104,9 +126,59 @@ export function mergeAdjacentConfigurationUpdates(items: readonly ResponseInputI // (isRenderedSummaryMessage) — client-origin mid system/developer cores are // conversation data and re-emit verbatim through their rawResponsesItem // sidecar; they are NOT rewritten here. +function boundCarrier(messages: PartBoundMessage[]): ResponseInputItem { + const source = messages[0]![responsesPartBinding]!.source; + const parts = messages.map(message => { + const part = message[responsesPartBinding]!.part; + return part.type === "input_image" || part.text === message.text ? part : { ...part, text: message.text ?? "" }; + }); + const original = source.content; + const unchanged = Array.isArray(original) && original.length === parts.length + && parts.every((part, index) => part === original[index]); + return unchanged ? source : { type: "message", role: "user", content: parts }; +} + +function boundProjection(projection: ResponsesProjection, messages: CoreMessage[]): ResponsesProjection { + const runs = new Map(); + let previous: PartBoundMessage | undefined; + for (const message of messages) { + const bound = message as PartBoundMessage; + const binding = bound[responsesPartBinding]; + if (!binding || message.role !== "user" || isRenderedSummaryMessage(message)) { + previous = undefined; + continue; + } + const groups = runs.get(binding.source) ?? []; + if (previous?.[responsesPartBinding]?.source === binding.source) groups.at(-1)!.push(bound); + else groups.push([bound]); + runs.set(binding.source, groups); + previous = bound; + } + const sources = new Map(projection.msgs.map(message => [message.id, message])); + const replaced = new Set(); + const rebound: BiliMessage[] = []; + const layout = projection.layout.flatMap(slot => { + const groups = runs.get(slot.original); + if (!groups) return [slot]; + for (const id of slot.coreIds ?? (slot.coreId ? [slot.coreId] : [])) replaced.add(id); + return groups.map(group => { + for (const message of group) { + const source = sources.get(message[responsesPartBinding]!.sourceId); + if (source) rebound.push({ ...source, id: message.id, text: message.text }); + } + const ids = group.map(message => message.id); + return { original: boundCarrier(group), coreId: ids[0], ...(ids.length > 1 ? { coreIds: ids } : {}) }; + }); + }); + // The virtual layout retains untracked wire items, but gives each surviving + // part its actual current ID. Separate runs leave an insertion anchor for + // a block summary between parts from the same original carrier. + return { ...projection, layout, msgs: [...projection.msgs.filter(message => !replaced.has(message.id)), ...rebound] }; +} + export function patchResponsesInputWithToolImages(projection: ResponsesProjection, messages: CoreMessage[]): string | ResponseInputItem[] { const voiced = systemToUser(messages, (m) => isRenderedSummaryMessage(m)); - const input = patchResponsesInput(projection, voiced); + const input = patchResponsesInput(boundProjection(projection, voiced), voiced); if (typeof input === "string") return input; const byOriginal = new Map(); const byCall = new Map(); @@ -122,8 +194,24 @@ export function patchResponsesInputWithToolImages(projection: ResponsesProjectio // #1999: same pre-codec re-voicing as patchResponsesInputWithToolImages above. // #4166: carriers only — client-origin mid system/developer stays verbatim. export function coreToResponsesWithToolImages(messages: CoreMessage[], customToolCallIds: Set = new Set()): ResponseInputItem[] { - return systemToUser(messages, (m) => isRenderedSummaryMessage(m)).flatMap((message) => { + const out: ResponseInputItem[] = []; + let pending: PartBoundMessage[] = []; + const flush = (): void => { + if (pending.length === 0) return; + out.push(boundCarrier(pending)); + pending = []; + }; + for (const message of systemToUser(messages, (m) => isRenderedSummaryMessage(m))) { + const bound = message as PartBoundMessage; + if (bound[responsesPartBinding] && message.role === "user" && !isRenderedSummaryMessage(message)) { + if (pending.length && pending[0]![responsesPartBinding]!.source !== bound[responsesPartBinding].source) flush(); + pending.push(bound); + continue; + } + flush(); const rebuilt = rebuildToolImages(message); - return rebuilt ? [rebuilt] : coreToResponses([message], customToolCallIds); - }); + out.push(...(rebuilt ? [rebuilt] : coreToResponses([message], customToolCallIds))); + } + flush(); + return out; } diff --git a/src/server.ts b/src/server.ts index a8cbd89bf6..961dd26635 100644 --- a/src/server.ts +++ b/src/server.ts @@ -3881,9 +3881,10 @@ export async function preflightCompressIfNeeded( // only the original tokenCount reads as "nothing happened" even when 16 // folds removed hundreds of thousands of tokens. foldedTokens/rangesLeft // stay undefined when no fold ran, so the original phrasing holds there. + let remainingImageTokens = imageTokens; const failFast = (status: number, detail: string, retryable: boolean, foldedTokens?: number, rangesLeft?: number): PreflightFailFast => { - const imageNote = imageTokens >= limit - ? ` Images alone account for ~${imageTokens} tokens (≥ window ${limit}); compression cannot remove them — shrink or remove the images, or raise the window.` + const imageNote = remainingImageTokens >= limit + ? ` Images alone account for ~${remainingImageTokens} estimated tokens (≥ window ${limit}); reduce the remaining image payload or raise the window.` : ""; const sizeClause = foldedTokens !== undefined && foldedTokens < tokenCount ? `context ~${foldedTokens} tokens (down from ~${tokenCount} before preflight) exceeds the model window ${limit}` @@ -3993,6 +3994,7 @@ export async function preflightCompressIfNeeded( // #3226: re-sign the summarization egress over its own body (same arm as the // main forward); undefined for unsigned traffic. const preflightResignCtx = inboundResignCredential(req.headers, opts, upstreamUrl); + const responsesProjection = prepared.responsesProjection; let result: PreflightResult; try { result = await preflightCompress( @@ -4020,6 +4022,9 @@ export async function preflightCompressIfNeeded( signal: clientAbort.signal, log, imageReserve: imageTokens, + measureImageReserve: prepared.protocol === "responses" && responsesProjection + ? (view) => imageReserveFor(session, "responses", { input: patchResponsesInput(responsesProjection, view) }, opts, upstreamUrl) + : undefined, wireOverhead: overheadEstimate, unknownBaseline, upstreamOrigin: currentOrigin, @@ -4048,6 +4053,7 @@ export async function preflightCompressIfNeeded( // sessions keep the loop's own upper-bound judgment (result.fitsWindow, // #553) — the optimistic re-estimate is exactly what that regime distrusts. let outbound: Prepared = prepared; + let reducedImageWire = false; let fits: boolean; let foldedCalibratedSize: number | undefined; if (result.compressedRanges > 0) { @@ -4070,6 +4076,9 @@ export async function preflightCompressIfNeeded( // raw body rides; mirror outboundPayloadBreakdown's fallback). const rebuiltMsgs = rebuilt.processedMessages.length > 0 ? rebuilt.processedMessages : rebuilt.originalMessages; const rebuiltTextSize = estimateCoreMessages(rebuiltMsgs) + overheadEstimate; + const rebuiltImageTokens = imageReserveFor(session, rebuilt.protocol, rebuilt.body, opts, upstreamUrl); + remainingImageTokens = rebuiltImageTokens; + reducedImageWire = result.compressedRanges > 0 && rebuiltImageTokens < imageTokens; if (rebuiltTextSize > session.stats.lastInputTokens) { session.stats.lastInputTokens = rebuiltTextSize; session.stats.lastInputTokensSource = "estimate"; @@ -4079,7 +4088,7 @@ export async function preflightCompressIfNeeded( // The baseline anchor above stays raw deliberately: it is a floor for // future meters, and a deflated (k̂ < 1) value would only delay the // next trigger, never advance it. - const rebuiltCalibrated = applyEstimateCalibration(rebuiltTextSize, kFactor, kOrigin, currentOrigin) + imageTokens; + const rebuiltCalibrated = applyEstimateCalibration(rebuiltTextSize, kFactor, kOrigin, currentOrigin) + rebuiltImageTokens; // #3645: hoist the calibrated post-fold figure so the fail-fast message // quotes the same scale as this fit and the pre-fold tokenCount — quoting // the raw size there claimed a sub-window number exceeds the window when k̂>1. @@ -4093,16 +4102,16 @@ export async function preflightCompressIfNeeded( // already-at-window context. Raw caliber (images included, no k̂ // deflation) mirrors the fit-gate quantity; lifetime is bounded (see // session.ts) so never-reporting upstreams fall back to legacy sizing. - setPostRebuildAnchor(session, rebuiltTextSize + imageTokens); + setPostRebuildAnchor(session, rebuiltTextSize + rebuiltImageTokens); // [#3444] exhausted honest pass: the chain folded everything it could // (no failure) yet the rebuilt payload still sits at/above the growth // target — the target is physically unreachable for this session. // Record the measured floor so the next trigger arms on real growth // past it instead of re-firing every request; latest measurement wins. - if (growthArmed && result.failure === undefined && rebuiltTextSize + imageTokens >= compressionTarget) { - session.metadata.autoFoldFloorTokens = rebuiltTextSize + imageTokens; + if (growthArmed && result.failure === undefined && rebuiltTextSize + rebuiltImageTokens >= compressionTarget) { + session.metadata.autoFoldFloorTokens = rebuiltTextSize + rebuiltImageTokens; markDirty(session); - log("warn", `[${session.id}] auto-fold target ${compressionTarget} unreachable — the session's irreducible baseline measures ~${rebuiltTextSize + imageTokens} tokens (system prompt + tools + live tail); effective target raised to the floor until the gap closes (#3444). Raise compress.externalSummary.autoFoldTargetTokens above ${rebuiltTextSize + imageTokens} for headroom`); + log("warn", `[${session.id}] auto-fold target ${compressionTarget} unreachable — the session's irreducible baseline measures ~${rebuiltTextSize + rebuiltImageTokens} tokens (system prompt + tools + live tail); effective target raised to the floor until the gap closes (#3444). Raise compress.externalSummary.autoFoldTargetTokens above ${rebuiltTextSize + rebuiltImageTokens} for headroom`); } } else { fits = unknownBaseline @@ -4207,13 +4216,16 @@ export async function preflightCompressIfNeeded( // upstream overflow evidence. Forward for the upstream to arbitrate billing // instead of fail-fasting a payload whose real bill likely fits (#496). The text // portion was already folded above when foldable; we do NOT re-loop. - if (imageArbitration && !localRefusalHere) { + const imageArbitrationAfterFold = reducedImageWire && resolveImageBilling(opts.imageBilling, upstreamUrl) === "pixels" + && session.stats.lastInputTokensSource !== "overflow-arm"; + if ((imageArbitration || imageArbitrationAfterFold) && !localRefusalHere) { // Same fallback as the fit-gate measurement above: processedMessages // empty ⇒ kernel transform failure ⇒ the raw body rides. const outMsgs = outbound.processedMessages.length > 0 ? outbound.processedMessages : outbound.originalMessages; const outText = estimateCoreMessages(outMsgs); - if (outText + overheadEstimate < limit && outText + overheadEstimate + imageTokens >= limit) { - log("info", `[${session.id}] preflight folded ${result.compressedRanges} range(s) but images alone (~${imageTokens} tokens) keep the estimate over window ${limit} with no upstream overflow evidence — forwarding for the upstream to arbitrate billing (#496/#1800)`); + const remainingImages = imageReserveFor(session, outbound.protocol, outbound.body, opts, upstreamUrl); + if (outText + overheadEstimate < limit && outText + overheadEstimate + remainingImages >= limit) { + log("info", `[${session.id}] preflight folded ${result.compressedRanges} range(s); remaining images (~${remainingImages} estimated tokens) still exceed window ${limit}; forwarding the reduced wire for upstream billing arbitration (#496/#1800)`); return outbound; } } diff --git a/tests/issue857-image-baseline.test.ts b/tests/issue857-image-baseline.test.ts index 013712c69b..bcfec0714e 100644 --- a/tests/issue857-image-baseline.test.ts +++ b/tests/issue857-image-baseline.test.ts @@ -22,9 +22,8 @@ import { listSessions } from "../src/session.ts"; // // Fix under test (still load-bearing after #987 removed the window learner): // C1 preflight write-back persists text+overhead only (image floor excluded) -// — note: since the #1841 futility gate, a doomed image-over-window round -// is skipped BEFORE any fold, so C1's write-back (which only runs when a -// round folds ≥1 range) is no longer exercised by scenario C below. +// — image-bearing ranges may shrink the wire. Scenario C supplies no +// acceptable summary, so no fold write-back runs in that control. // C2 stats.lastInputTokensSource tags every baseline raise ("usage"|"estimate") // C4 #496 hatch: an untrusted (estimate-derived / legacy-unmarked) baseline // is not overflow evidence — only usage-grounded evidence closes it @@ -219,7 +218,7 @@ test("#857 B: genuine overflow evidence (learned limit == configured window) sti } }); -test("#857 C: usage-grounded over-window baseline — hatch stays closed AND the #1841 futility skip leaves the baseline untouched", async () => { +test("#857 C: unusable image-fold summaries leave the grounded baseline and forwarding guard intact", async () => { const { server: upstream, port: uport, stats } = await startMockUpstream(); // No route model entry: the window is a pure fallback, so the upward // self-heal (which requires nativeFromFallback) can confirm it. @@ -232,35 +231,28 @@ test("#857 C: usage-grounded over-window baseline — hatch stays closed AND the assert.equal(r1.status, 200); await r1.text(); - // A REAL usage report put the baseline above the window (armed, - // usage-grade evidence) — but the image floor exceeds even the window, - // so compression cannot make the payload fit. + // A usage report put the baseline above the window. Historical images + // can now leave the wire, so the reserve alone cannot prove futility. const s = sess("s857c"); s.stats.lastInputTokens = FAKE_CONFIRMED; s.stats.lastInputTokensSource = "usage"; - // Filler history gives the kernel foldable candidates. Under the #1841 - // futility gate this round is provably doomed BEFORE any walk — the - // image floor alone (~1.1M) exceeds the window (~999K) while the whole - // foldable pool (~4.4K of filler) cannot close the ~110K gap — so the - // round is skipped with zero summarization calls: exactly the waste - // #1841 removes (old code walked both ranges, burned the calls, and - // still failed here). + // The mock cannot supply an acceptable summary. The bounded image-fold + // attempts must therefore fail without forwarding the original wire or + // replacing the measured baseline with an image estimate. const r2 = await fetch(url, { method: "POST", headers, body: body(FALLBACK_MODEL, "s857c", [...fillerHistory(16), ...imageInput()]) }); - // #3461: over-window per bili's own meter with no foldable mass left → 413. - assert.equal(r2.status, 413, "usage evidence keeps the hatch closed"); + assert.equal(r2.status, 502, "unusable summaries keep the hatch closed"); const err2 = JSON.parse(await r2.text()) as { error?: { code?: string; message?: string } }; assert.equal(err2.error?.code, "preflight_compress_failed"); - assert.match(err2.error?.message ?? "", /futile round/, `the skipped round names its futility verdict (got: ${err2.error?.message})`); assert.match(err2.error?.message ?? "", /Images alone account for/, "the image remedy wording survives the futility verdict"); assert.equal(stats.streamingForwards, 1); - assert.equal(stats.summaryCalls, 0, "#1841: a provably futile round spends zero summarization calls"); + assert.ok(stats.summaryCalls > 0, "image-bearing candidates receive bounded summary attempts"); // No fold ran, so nothing in the turn may touch the baseline: the armed // usage value stays exactly where the test put it (far below the image // floor — nothing to re-poison) with its grounded provenance intact. const after = sess("s857c"); - assert.equal(after.stats.lastInputTokens, FAKE_CONFIRMED, "skipped round leaves the baseline untouched"); + assert.equal(after.stats.lastInputTokens, FAKE_CONFIRMED, "failed folds leave the baseline untouched"); assert.equal(after.stats.lastInputTokensSource, "usage", "grounded provenance preserved"); assert.equal(confirmedOf("s857c"), undefined, "#987: no window is ever learned from session traffic"); } finally { diff --git a/tests/preflight-image-budget.test.ts b/tests/preflight-image-budget.test.ts new file mode 100644 index 0000000000..5ea0323abc --- /dev/null +++ b/tests/preflight-image-budget.test.ts @@ -0,0 +1,128 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import http from "node:http"; +import { once } from "node:events"; +import { randomBytes } from "node:crypto"; +import { deflateSync } from "node:zlib"; +import { readFileSync } from "node:fs"; +import { createCore, defaultConfig } from "acp-kernel"; +import { startServer, type ProxyOptions } from "../src/server.ts"; +import { SessionStore, _setStoreForTest } from "../src/persist.ts"; +import { listSessions, _resetSessionsForTest } from "../src/session.ts"; +import { _setForTest as setRegistryForTest } from "../src/registry.ts"; +import { resolveDecompress } from "../src/decompress-shared.ts"; + +// An actual PNG: deterministic dimensions, incompressible pixels, and valid CRCs. +function png(): Buffer { + const chunk = (type: string, data: Buffer): Buffer => { + const body = Buffer.concat([Buffer.from(type), data]); + let crc = 0xffffffff; + for (const byte of body) { + crc ^= byte; + for (let bit = 0; bit < 8; bit++) crc = (crc >>> 1) ^ ((crc & 1) ? 0xedb88320 : 0); + } + const header = Buffer.alloc(4), tail = Buffer.alloc(4); + header.writeUInt32BE(data.length); tail.writeUInt32BE((crc ^ 0xffffffff) >>> 0); + return Buffer.concat([header, body, tail]); + }; + const header = Buffer.alloc(13); + header.writeUInt32BE(256, 0); header.writeUInt32BE(256, 4); header[8] = 8; header[9] = 2; + const rows = Array.from({ length: 256 }, () => Buffer.concat([Buffer.from([0]), randomBytes(256 * 3)])); + return Buffer.concat([Buffer.from("89504e470d0a1a0a", "hex"), chunk("IHDR", header), chunk("IDAT", deflateSync(Buffer.concat(rows))), chunk("IEND", Buffer.alloc(0))]); +} + +for (const mode of ["all-images", "remaining-images", "upstream-overflow"] as const) { +test(`preflight image budget: ${mode}`, async () => { + _setStoreForTest(new SessionStore({ enabled: false })); + _resetSessionsForTest(); + setRegistryForTest({}); + const pixels = png(), base64 = pixels.toString("base64"); + const partial = mode !== "all-images"; + const limit = partial ? 8_000 : 12_000; + const liveImage = partial ? `data:image/png;base64,${png().toString("base64")}` : undefined; + const forwards: string[] = [], summaries: string[] = []; + const summary = "The historical screenshot was inspected. Keep its image retrieval reference, the accepted decisions, and the current task constraints when continuing the work."; + const upstream = http.createServer((req, res) => { + const chunks: Buffer[] = []; + req.on("data", (chunk: Buffer) => chunks.push(chunk)); + req.on("end", () => { + const raw = Buffer.concat(chunks).toString("utf8"); + if (JSON.parse(raw).stream === false) { + summaries.push(raw); + res.writeHead(200, { "content-type": "application/json" }); + res.end(JSON.stringify({ output_text: summary })); + } else { + forwards.push(raw); + res.writeHead(200, { "content-type": "text/event-stream" }); + res.end(`event: response.completed\ndata: ${JSON.stringify({ response: { id: "done", status: "completed", output: [], usage: { input_tokens: 1000, output_tokens: 5, total_tokens: 1005 } } })}\n\n`); + } + }); + }); + upstream.listen(0, "127.0.0.1"); + await once(upstream, "listening"); + const address = upstream.address(); assert.ok(address && typeof address !== "string"); + const origin = `http://127.0.0.1:${address.port}`; + const config = defaultConfig(limit); + config.preserveRecentMessages = 2; config.preserveRecentTokens = 500; + const opts: ProxyOptions = { + port: 0, host: "127.0.0.1", upstream: origin, + routes: { [origin]: { models: { test: { context: limit } } } }, + modelContextLimit: limit, kernelConfig: config, imageBilling: partial ? "pixels" : "bytes", + compress: { injectTool: true, injectNudge: true }, promptCache: { routing: "auto" }, sessionHeader: "x-acp-session", + log: false, debug: false, passthrough: false, passthroughSource: null, autoUpdate: false, autoRestartOnUpdate: false, + updateTag: "latest", advisoryCheck: false, releaseNotesCheck: false, compat: { roles: {} }, streamErrorShape: "protocol", mitm: { enabled: false, domains: [] }, + }; + const proxy = await startServer(opts); + await once(proxy, "listening"); + const proxyAddress = proxy.address(); assert.ok(proxyAddress && typeof proxyAddress !== "string"); + try { + const url = `http://127.0.0.1:${proxyAddress.port}/bili/${origin}/responses`; + const seed = await fetch(url, { method: "POST", headers: { "content-type": "application/json" }, body: JSON.stringify({ model: "test", stream: true, session_id: "image-budget", input: [{ type: "message", role: "user", content: "Begin the screenshot investigation." }] }) }); + assert.equal(seed.status, 200); await seed.text(); + const seeded = listSessions().find(s => s.meta.label === "image-budget"); assert.ok(seeded); + seeded.stats.lastInputTokens = 70_000; + seeded.stats.lastInputTokensSource = mode === "upstream-overflow" ? "overflow-arm" : "usage"; + seeded.stats.lastInputTokensOrigin = origin; + if (partial) seeded.metadata.learnedImageCosts = { "127.0.0.1": { cost: 20_000, seen: 1, ts: Date.now(), fp: "pixels:0" } }; + forwards.length = 0; + const input: unknown[] = [ + { type: "message", role: "user", content: "Begin the screenshot investigation." }, + { type: "message", role: "assistant", content: "The historical evidence and decisions follow." }, + { type: "message", role: "user", content: [{ type: "input_text", text: "Inspect this historical screenshot. " + "evidence details ".repeat(180) }, { type: "input_image", image_url: `data:image/png;base64,${base64}`, detail: "high" }] }, + ]; + for (let i = 0; i < 12; i++) input.push({ type: "message", role: i % 2 ? "user" : "assistant", content: `Historical turn ${i}. ` + "preserve decisions and constraints ".repeat(40) }); + input.push({ type: "message", role: "user", content: partial ? [ + { type: "input_text", text: "Continue using these current screenshots and the accepted findings." }, + { type: "input_image", image_url: liveImage, detail: "high" }, + { type: "input_image", image_url: liveImage, detail: "high" }, + ] : "Continue using the accepted findings." }); + const response = await fetch(url, { + method: "POST", headers: { "content-type": "application/json" }, + body: JSON.stringify({ model: "test", stream: true, session_id: "image-budget", input }), + }); + const body = await response.text(); + assert.ok(summaries.length > 0, "folding must not be declared futile from text savings alone"); + if (mode === "upstream-overflow") { + assert.equal(response.status, 502, body); + assert.equal(forwards.length, 0, "confirmed upstream overflow must not be bypassed by image arbitration"); + return; + } + assert.equal(response.status, 200, body); + assert.equal(forwards.length, 1); + assert.ok(!forwards[0]!.includes(base64), "the folded image must leave the actual outbound wire"); + assert.ok(forwards[0]!.includes(summary)); + assert.equal((forwards[0]!.match(/"input_image"/g) ?? []).length, partial ? 2 : 0, "only the protected live images remain"); + const ref = summaries.join("\n").match(/\[image: png 256x256 · (m\d+)\]/)?.[1]; + assert.ok(ref, "summary input retains a resolvable image reference"); + const session = listSessions().find(s => s.meta.label === "image-budget"); assert.ok(session); + const restored = resolveDecompress({ imageRef: ref }, { core: createCore(), config, messages: [], session, log: () => {} }); + assert.equal(restored.outcome, "success", restored.text); + const file = restored.text.match(/\n (.+\.png)/)?.[1]; assert.ok(file, restored.text); + assert.deepEqual(readFileSync(file), pixels, "retrieval restores the original pixels exactly"); + } finally { + const proxyClosed = once(proxy, "close"), upstreamClosed = once(upstream, "close"); + proxy.close(); proxy.closeAllConnections(); upstream.close(); upstream.closeAllConnections(); + await Promise.all([proxyClosed, upstreamClosed]); + } +}); +} diff --git a/tests/responses-block-wire.test.ts b/tests/responses-block-wire.test.ts new file mode 100644 index 0000000000..57797e70fb --- /dev/null +++ b/tests/responses-block-wire.test.ts @@ -0,0 +1,77 @@ +import assert from "node:assert/strict"; +import test from "node:test"; +import { createCore, createInitialState, defaultConfig } from "acp-kernel"; +import type { ResponseInputItem } from "acp-kernel/wire"; +import { responsesToCoreWithToolImages as project, patchResponsesInputWithToolImages as patch, coreToResponsesWithToolImages as encode } from "../src/responses-tool-output.ts"; + +function images(input: unknown): unknown[] { + if (Array.isArray(input)) return input.flatMap(images); + if (!input || typeof input !== "object") return []; + const item = input as Record; + return item.type === "input_image" ? [item] : Object.values(item).flatMap(images); +} + +for (const n of [1, 2, 12, 17, 26]) { + test(`host projection preserves ${n} images when current core IDs differ from layout IDs`, () => { + const input: ResponseInputItem[] = [{ type: "message", role: "user", content: Array.from({ length: n }, (_, i) => [ + { type: "input_text", text: `caption ${i}`, annotations: [`source-${i}`] }, + { type: "input_image", image_url: `https://example.invalid/${i}.png`, detail: i % 2 ? "low" : "high" }, + ]).flat() }]; + const projection = project({ input }); + assert.deepEqual(patch(projection, projection.msgs), input); + assert.deepEqual(encode(projection.msgs), input); + const remapped = projection.msgs.map(message => ({ ...message, id: `fresh_${message.id}` })); + assert.deepEqual(images(patch(projection, remapped)), images(input), "each surviving part must be emitted exactly once"); + }); +} + +test("partial block view keeps B, its detail and caption metadata, without resurrecting A", () => { + const a = { type: "input_image", image_url: "https://example.invalid/A.png", detail: "low" }; + const b = { type: "input_image", image_url: "https://example.invalid/B.png", detail: "high" }; + const caption = { type: "input_text", text: "retained B caption", annotations: ["keep"] }; + const projection = project({ input: [{ type: "message", role: "user", content: [a, caption, b] }] }); + const partial = projection.msgs.slice(1); + assert.deepEqual(images(patch(projection, partial)), [b]); + assert.deepEqual((patch(projection, partial) as ResponseInputItem[])[0]!.content, [caption, b]); + const between = { id: "inserted-summary", role: "user" as const, contentType: "text" as const, text: "Retained block summary at its original position." }; + const separated = patch(projection, [projection.msgs[0]!, between, projection.msgs[2]!]) as ResponseInputItem[]; + assert.deepEqual(images(separated[0]), [a]); + assert.ok(JSON.stringify(separated[1]).includes(between.text), "the summary stays between retained source parts"); + assert.deepEqual(images(separated[2]), [b]); +}); + +test("a real block replaces its originals on resends and preserves genuinely new identical instances once", () => { + const image = "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg=="; + const a = { type: "input_image", image_url: image, detail: "low" }; + const b = { type: "input_image", image_url: image, detail: "high" }; + const input: ResponseInputItem[] = [ + { type: "message", role: "user", content: "independent opening" }, + { type: "message", role: "assistant", content: "ready" }, + { type: "message", role: "user", content: [{ type: "input_text", text: "Historical A facts and constraints. ".repeat(200) }, a, + { type: "input_text", text: "Retained B facts." }, b] }, + { type: "message", role: "assistant", content: "Continue from the retained decisions." }, + ]; + const projection = project({ input }); + const core = createCore(); + const config = { ...defaultConfig(100_000), preserveRecentMessages: 0, preserveRecentTokens: 0 }; + const initial = core.processTurn({ messages: projection.msgs, state: createInitialState(), config, tokenCount: 3000, renderTags: "none" }); + const summary = "A facts established the archived screenshot's constraints; retain those decisions while continuing from screenshot B."; + const folded = core.applyCompression({ messages: projection.msgs, state: initial.state, config, ranges: [{ + startRef: initial.state.messageRefs.byRaw[projection.msgs[2]!.id]!, endRef: initial.state.messageRefs.byRaw[projection.msgs[3]!.id]!, summary, + }] }); + assert.equal(folded.result.blocksCreated, 1, JSON.stringify(folded.result)); + for (const prior of [initial.state.lastPassIds ?? [], []]) { + const turn = core.processTurn({ messages: projection.msgs, state: { ...structuredClone(folded.state), lastPassIds: prior }, config, tokenCount: 3000, renderTags: "none" }); + const wire = patch(projection, turn.messages); + if (prior.length > 0) { + assert.ok(turn.state.blocks.some(block => block.active), "the compression block remains active"); + assert.ok(JSON.stringify(wire).includes(summary), "the block summary replaces the original facts"); + assert.deepEqual(images(wire), [b], "archived A must stay out while B remains exactly once"); + } else { + // The kernel deliberately treats absent-prior content as a new + // identical instance (#1476), rather than hiding new work under an + // old fold. The host must emit that current view without n² replay. + assert.deepEqual(images(wire), [a, b]); + } + } +}); diff --git a/tests/responses-tool-images.test.ts b/tests/responses-tool-images.test.ts index 819072a5ba..39a775f530 100644 --- a/tests/responses-tool-images.test.ts +++ b/tests/responses-tool-images.test.ts @@ -2,7 +2,10 @@ import assert from "node:assert/strict"; import http from "node:http"; import { once } from "node:events"; import test from "node:test"; -import { defaultConfig } from "acp-kernel"; +import { createCore, defaultConfig } from "acp-kernel"; +import { mkdtempSync, readFileSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; import { responsesToCore as kernelResponsesToCore, type ResponseInputItem } from "acp-kernel/wire"; import { responsesToCoreWithToolImages as responsesToCore, patchResponsesInputWithToolImages as patchResponsesInput, coreToResponsesWithToolImages as coreToResponses } from "../src/responses-tool-output.ts"; import { imageTokensInParsedBody } from "../src/image-tokens.ts"; @@ -11,6 +14,8 @@ import { createResponsesAdapter } from "../src/loop/adapter-responses.ts"; import { startServer } from "../src/server.ts"; import type { ProxyOptions } from "../src/config.ts"; import { SessionStore, _setStoreForTest } from "../src/persist.ts"; +import { getSession, listSessions, _resetSessionsForTest } from "../src/session.ts"; +import { resolveDecompress } from "../src/decompress-shared.ts"; import { _setForTest as setRegistryForTest } from "../src/registry.ts"; process.env.NODE_ENV = "test"; @@ -28,10 +33,105 @@ function pngUrl(width: number, height: number, fill = 0): string { const imageUrl = pngUrl(2048, 1152); const parts = [{ type: "input_text", text: "The screenshot shows a failure." }, { type: "input_image", image_url: imageUrl, detail: "high" }]; +function userImageBody(n: number): { input: ResponseInputItem[] } { + return { input: [{ type: "message", role: "user", content: Array.from({ length: n }, (_, i) => [ + { type: "input_text", text: `caption ${i}`, annotations: [`keep-${i}`] }, + { type: "input_image", image_url: `https://example.test/image-${i}.png`, detail: i % 2 ? "low" : "high" }, + ]).flat() }] }; +} + +test("full encoding binds each multipart image to its source, including partial and duplicate retention", () => { + const first = { type: "message", role: "user", content: [ + { type: "input_text", text: "first source", custom: "keep-text-metadata" }, + { type: "input_image", image_url: imageUrl, detail: "low" }, + { type: "input_image", image_url: imageUrl, detail: "high" }, + ] }; + const second = { type: "message", role: "user", content: [ + { type: "input_text", text: "second source" }, + { type: "input_image", image_url: pngUrl(128, 256), detail: "auto" }, + ] }; + const projection = responsesToCore({ input: [first, second] }); + assert.deepEqual(coreToResponses(projection.msgs), [first, second], "complete carriers round-trip once each"); + const partial = projection.msgs.filter((_, index) => index !== 1); + const rebuilt = coreToResponses(partial); + assert.deepEqual(rebuilt[0]!.content, [first.content[0], first.content[2]], "retained duplicate keeps its own detail rather than taking the first image"); + assert.deepEqual(rebuilt[1], second, "a neighboring source keeps its own image"); + const summary = { id: "summary-carrier", role: "system" as const, contentType: "text" as const, text: "A new summary between retained image parts." }; + const split = coreToResponses([projection.msgs[1]!, summary, projection.msgs[4]!]); + assert.deepEqual(split[0]!.content, [first.content[1]]); + assert.equal(split[1]!.role, "developer", "ordinary system item stays between the image sources"); + assert.deepEqual(split[2]!.content, [second.content[1]]); +}); + function toolBody(type: string, output: unknown = parts): { input: ResponseInputItem[] } { return { input: [{ type, call_id: "screenshot-call", output }] }; } +test("#4767: full encoding preserves one multi-image carrier instead of repeating it per core", () => { + for (const n of [26, 2, 1]) { + const body = userImageBody(n); + const projection = responsesToCore(body); + assert.deepEqual(coreToResponses(projection.msgs, projection.customToolCallIds), body.input); + for (const textProtocol of [false, true]) { + const request = createResponsesAdapter(textProtocol).buildRequest(projection.msgs, "", body); + assert.deepEqual((request.input as ResponseInputItem[]).filter((item) => item.role === "user"), body.input); + } + } +}); + +test("#4767: ordinary multipart runs stay intact on both sides of image tool outputs", () => { + for (const type of ["function_call_output", "custom_tool_call_output"]) { + const before = userImageBody(2).input; + const tool = { type, call_id: "image-call", output: parts }; + const after = userImageBody(3).input; + const body = { input: [...before, tool, ...after] }; + const projection = responsesToCore(body); + const edited = projection.msgs.map((m) => m.role === "tool" ? { ...m, text: "changed" } : m); + assert.deepEqual(coreToResponses(edited, projection.customToolCallIds), [ + ...before, { ...tool, output: [{ ...parts[0], text: "changed" }, parts[1]] }, ...after, + ]); + } +}); + +test("#4767: full encoding preserves repeated identical image instances", () => { + const body = userImageBody(2); + const first = body.input[0]; + assert.ok(Array.isArray(first.content)); + first.content[3] = { ...first.content[1] }; + const projection = responsesToCore(body); + assert.deepEqual(coreToResponses(projection.msgs, projection.customToolCallIds), body.input); +}); + +test("#4767: batching adjacent source carriers must not lose the later carrier's images", () => { + const later = userImageBody(3).input; + const body = { input: [...userImageBody(2).input, ...later] }; + const projection = responsesToCore(body); + const rebuilt = coreToResponses(projection.msgs, projection.customToolCallIds); + assert.deepEqual(rebuilt, body.input, "each adjacent source keeps its own parts without duplication"); +}); + +test("#4767: full encoding preserves empty history and surrounding plain history", () => { + assert.deepEqual(coreToResponses([]), []); + const body = { input: [ + { type: "message", role: "assistant", content: "plain before" }, + ...userImageBody(2).input, + { type: "message", role: "assistant", content: "plain after" }, + ] }; + const projection = responsesToCore(body); + assert.deepEqual(coreToResponses(projection.msgs, projection.customToolCallIds), body.input); +}); + +test("#4767: summary carriers stay separate from a neighboring multipart source", () => { + const body = userImageBody(2); + const projection = responsesToCore(body); + const summary = { id: "acp_summary_b1", role: "system" as const, contentType: "text" as const, text: "folded history" }; + const summaryInput = [{ type: "message", role: "user", content: "folded history" }]; + assert.deepEqual(coreToResponses([summary, ...projection.msgs, summary], projection.customToolCallIds), [ + ...summaryInput, ...body.input, ...summaryInput, + ]); +}); + + test("Responses tool output images are charged by pixels or bytes, not ignored", () => { for (const type of ["function_call_output", "custom_tool_call_output"]) { const body = toolBody(type); @@ -109,7 +209,7 @@ test("ordinary JSON tool outputs retain their existing projection and are not ch type RequestBody = { stream?: boolean; input?: ResponseInputItem[] }; -async function testProxy(window: number, run: (url: string, forwarded: RequestBody[], summaries: RequestBody[]) => Promise): Promise { +async function testProxy(window: number, run: (url: string, forwarded: RequestBody[], summaries: RequestBody[]) => Promise, store = new SessionStore({ enabled: false })): Promise { const forwarded: RequestBody[] = []; const summaries: RequestBody[] = []; const upstream = http.createServer((req, res) => { @@ -131,7 +231,7 @@ async function testProxy(window: number, run: (url: string, forwarded: RequestBo upstream.listen(0, "127.0.0.1"); await once(upstream, "listening"); const upstreamPort = (upstream.address() as { port: number }).port; - _setStoreForTest(new SessionStore({ enabled: false })); + _setStoreForTest(store); setRegistryForTest({}); const proxy = await startServer({ port: 0, host: "127.0.0.1", upstream: "http://127.0.0.1", @@ -156,6 +256,82 @@ async function testProxy(window: number, run: (url: string, forwarded: RequestBo } } +test("a persisted partial compression block keeps its summary, correct live image and archived original on raw resends", async () => { + const dir = mkdtempSync(join(tmpdir(), "bili-partial-block-")); + let store = new SessionStore({ enabled: true, dir }); + const png = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg=="; + const a = { type: "input_image", image_url: `data:image/png;base64,${png}`, detail: "low" }; + const b = { type: "input_image", image_url: `data:image/png;base64,${png}`, detail: "high" }; + const opaque = { type: "compaction", encrypted_content: "opaque-client-state" }; + const input: ResponseInputItem[] = [ + { type: "message", role: "user", content: "Preserve independent opening." }, + { type: "message", role: "assistant", content: "Ready." }, + { type: "message", role: "user", content: [{ type: "input_text", text: "Archived A decisions and constraints. ".repeat(200) }, a, + { type: "input_text", text: "Current B facts." }, b] }, + opaque, { type: "message", role: "assistant", content: "Continue." }, + ]; + const config = { ...defaultConfig(100_000), preserveRecentMessages: 0, preserveRecentTokens: 0 }; + const summary = "A established the accepted decisions and constraints. Those findings remain authoritative while continuing from B."; + try { + await testProxy(100_000, async (url, forwarded) => { + const send = async (history: ResponseInputItem[] = input): Promise => { + const response = await fetch(url, { method: "POST", headers: { "content-type": "application/json" }, + body: JSON.stringify({ model: "image-test", stream: true, session_id: "persisted-partial-block", input: history }) }); + assert.equal(response.status, 200, await response.text()); + await new Promise(resolve => setImmediate(resolve)); + }; + await send(); + const session = listSessions().find(item => item.meta.label === "persisted-partial-block"); + assert.ok(session); + const projection = responsesToCore({ input }); + const folded = createCore().applyCompression({ messages: projection.msgs, state: session.state, config, + ranges: [{ startRef: session.state.messageRefs.byRaw[projection.msgs[2]!.id]!, + endRef: session.state.messageRefs.byRaw[projection.msgs[3]!.id]!, summary }] }); + assert.equal(folded.result.blocksCreated, 1, JSON.stringify(folded.result)); + session.state = folded.state; + const blockId = session.state.blocks.at(-1)!.blockId; + const check = (): void => { + const wire = forwarded.at(-1)!.input!; + const pictures = wire.flatMap(item => Array.isArray(item.content) ? item.content.filter(part => part.type === "input_image") : []); + assert.deepEqual(pictures, [b], "A stays folded; B remains exactly once with its own detail"); + assert.equal(JSON.stringify(wire).split(summary).length - 1, 1, "one block has one visible summary"); + assert.deepEqual(wire.find(item => item.type === "compaction"), opaque, "untracked client state survives reconstruction"); + assert.ok(getSession(session.id).state.blocks.some(block => block.blockId === blockId && block.active)); + }; + await send(); check(); + // A temporary short view must not make the subsequent full echo + // resurrect the originals hidden by this same compression block. + await send([input[0]!, input[1]!, opaque, input.at(-1)!]); + await send(); check(); + // Conversely, a new identical upload outside the covered span is + // current user content, not an echo to suppress by pixel hash. + await send([...input, { type: "message", role: "user", content: [a] }]); + const withNewImage = forwarded.at(-1)!.input!.flatMap(item => Array.isArray(item.content) + ? item.content.filter(part => part.type === "input_image") : []); + assert.deepEqual(withNewImage, [b, a], "the genuinely new identical image remains once"); + await send(); check(); + await store.writeNow(getSession(session.id)); + await store.closeQuietlyAsync(); + _resetSessionsForTest(); + store = new SessionStore({ enabled: true, dir }); + _setStoreForTest(store); + await send(); check(); + const restoredSession = getSession(session.id); + const imageRef = restoredSession.state.messageRefs.byRaw[projection.msgs[3]!.id]; + assert.ok(imageRef && imageRef !== "BLOCKED"); + const restored = resolveDecompress({ imageRef }, { core: createCore(), config, messages: [], session: restoredSession, log: () => {} }); + assert.equal(restored.outcome, "success", restored.text); + const path = restored.text.match(/\n (.+\.png)/)?.[1]; + assert.ok(path, restored.text); + assert.deepEqual(readFileSync(path), Buffer.from(png, "base64")); + await send(); check(); + }, store); + } finally { + await store.closeQuietlyAsync(); + rmSync(dir, { recursive: true, force: true }); + } +}); + function screenshotItems(): ResponseInputItem[] { return [ { type: "message", role: "user", content: "Investigate the screenshot failure." },