Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
17 commits
Select commit Hold shift + click to select a range
affc3fb
fix: preserve multipart Responses user runs in host encoding
ReineLan Oct 25, 2026
e4a33dd
Merge remote-tracking branch 'origin/ci-fixes' into pr-4770-review
Oct 25, 2026
37f483e
merge ci-fixes into 2026-10-26_responses-image-rebuild (#4599 help-me…
Oct 26, 2026
bf07d48
Merge remote-tracking branch 'origin/ci-fixes' into sweep-wip
Oct 26, 2026
a4c1959
fix(responses): retain exact multipart source ownership
Oct 26, 2026
ba2067e
fix(responses): bind surviving parts to their original carrier
Oct 26, 2026
6220eec
fix(preflight): account for images removed by folding
Oct 26, 2026
0ee6cb2
fix(responses): preserve block parts through projection rebuilds
Oct 26, 2026
0b45f01
fix(responses): preserve block parts through projection rebuilds
Oct 26, 2026
54ae0fd
Merge remote-tracking branch 'origin/master' into 2026-10-27_response…
Oct 27, 2026
ad521e4
Merge remote-tracking branch 'origin/master' into 2026-10-27_image-pr…
Oct 27, 2026
3de24c0
Merge branch '2026-10-27_responses-part-ownership' into 2026-10-27_im…
Oct 27, 2026
8a57bd6
docs(preflight): restore the image-reserve rationale thinned by the b…
Oct 27, 2026
346c816
Merge remote-tracking branch 'origin/master' into wt-4932-merge-guard-r1
Oct 29, 2026
bd7ff1e
Merge remote-tracking branch 'origin/ci-fixes' into wt-4932-ci-r1
Oct 30, 2026
5b6f1f1
Merge remote-tracking branch 'origin/master' into wt-4932-ci-r1
Oct 30, 2026
071d05f
Merge remote-tracking branch 'origin/master' into wt-4932-merge-guard-r1
Oct 31, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
45 changes: 35 additions & 10 deletions src/preflight.ts
Original file line number Diff line number Diff line change
Expand Up @@ -243,8 +243,10 @@ export interface PreflightDeps {
proxyUrl?: string;
signal?: AbortSignal;
log: (level: string, msg: string) => void;
/** #1843: the image reserve for this request — the billed cost of the images riding the payload verbatim (#488). Folding never removes them, so every TEXT-channel decision compares the text estimate against `limit − reserve` instead of adding the reserve to each total; an image-estimate error can no longer move a fold decision. Kernel-facing quantities keep the total view (reserve added back), which is the conservative direction. */
/** #1843: the image reserve — the billed cost of the images riding the payload verbatim (#488). Every TEXT-channel decision compares the text estimate against `limit − reserve` instead of adding the reserve to each total, so an image-estimate error can no longer move a fold decision; kernel-facing quantities keep the total view (reserve added back), which is the conservative direction. This field is the INITIAL wire view; on lanes where folds archive images off the wire (`measureImageReserve`), the working copy is re-measured from the post-fold view every round (#4932). */
imageReserve?: number;
/** Rebuild the protocol image view after folds; archived images no longer consume the window. */
measureImageReserve?: (messages: CoreMessage[]) => number;
/** Constant wire overhead for this request (system prompt + tool definitions, #470). Folding never removes it, so every fit decision must add it — otherwise the loop stops with "text fits" while the billed input still overflows. */
wireOverhead?: number;
/** #553: the caller knows this session's input size is unmeasured AND its
Expand Down Expand Up @@ -1634,11 +1636,11 @@ export async function preflightCompress(deps: PreflightDeps, messages: CoreMessa
// text estimate vs `target − imageReserve`. Algebraically the same firing
// set as the old total-view check (max(B, T+R) >= C <=> max(max(0,B−R), T)
// >= max(0,C−R)), but an image-estimate error can no longer start or stop
// folding. Images ride the payload verbatim through every round, so the
// reserve is constant for this invocation. Kernel-facing quantities below
// folding for a fixed wire view. Archived images leave that view, so the host
// refreshes their reserve after each fold. Kernel-facing quantities below
// keep the total view (reserve added back): conservative direction, and the
// kernel's own truncation/absorb behavior stays byte-identical.
const imageReserve = deps.imageReserve ?? 0;
let imageReserve = deps.imageReserve ?? 0;
const wireOverhead = deps.wireOverhead ?? 0;
// #1933 F1: capture k̂ once — it only mutates on usage settlement (outside
// this invocation), while every fit judgment in this loop must stay on
Expand All @@ -1657,7 +1659,25 @@ export async function preflightCompress(deps: PreflightDeps, messages: CoreMessa
// for k̂=1 / absent / mismatched-origin (applyEstimateCalibration → raw), so
// the uncalibrated #1841 path stays byte-identical.
const calibrateFoldMass = (rawMass: number): number => applyEstimateCalibration(rawMass, kFactor, kOrigin, deps.upstreamOrigin);
let textTarget = Math.max(0, Math.min(limit, deps.compressionTarget ?? limit) - imageReserve);
let foldTarget = Math.min(limit, deps.compressionTarget ?? limit);
let textTarget = Math.max(0, foldTarget - imageReserve);
const refreshImages = (view: CoreMessage[]): void => {
if (deps.measureImageReserve) imageReserve = deps.measureImageReserve(view);
textTarget = Math.max(0, foldTarget - imageReserve);
};
// Removing every nominated source gives an upper bound on image savings.
// This is only a futility bound: the real post-fold view decides fit.
const imageSavingBound = (ranges: Array<{ startRef: string; endRef: string }>, view: CoreMessage[]): number => {
if (!deps.measureImageReserve || imageReserve <= 0) return 0;
const { refToIdx } = refMaps(messages, deps.session.state);
const removed = new Set<string>();
for (const range of ranges) {
const start = refToIdx.get(range.startRef), end = refToIdx.get(range.endRef);
if (start === undefined || end === undefined || start > end) continue;
for (let i = start; i <= end; i++) removed.add(messages[i]!.id);
}
return Math.max(0, imageReserve - deps.measureImageReserve(view.filter(message => !removed.has(message.id))));
};
const result: PreflightResult = { compressedRanges: 0, savedTokens: 0, payloadEstimate: applyEstimateCalibration(estimateCoreMessages(messages) + wireOverhead, kFactor, kOrigin, deps.upstreamOrigin) + imageReserve, rangesRemaining: 0, fitsWindow: true, hadFoldableCandidates: false };
// #4288: set when the round-top fit exit fired — the walk ended in a
// forward decision, and no later verdict may override it (see the final
Expand Down Expand Up @@ -1844,6 +1864,7 @@ export async function preflightCompress(deps: PreflightDeps, messages: CoreMessa
deps.session.state = turn.state;
adoptContentStore(deps.session, turn.contentStore);
const view = applyAbsorbView(turn.messages, turn.state, activeConfig, currentTokens);
refreshImages(view);
const est = applyEstimateCalibration(estimateCoreMessages(view) + wireOverhead, kFactor, kOrigin, deps.upstreamOrigin) + imageReserve;
result.payloadEstimate = est;
// #2952 review: keep the char-count UPPER bound fresh through recovery.
Expand Down Expand Up @@ -2081,6 +2102,7 @@ export async function preflightCompress(deps: PreflightDeps, messages: CoreMessa
const baselineFloor = messages.length > 0
? ((deps.session.stats.lastInputTokensSource === "usage" || deps.session.stats.lastInputTokensSource === "overflow-arm") ? deps.session.stats.lastInputTokens : 0)
: deps.session.stats.lastInputTokens;
refreshImages(turn.messages);
const rawRoundText = estimateCoreMessages(turn.messages) + wireOverhead;
// #1933 F1: same calibrated caliber as the gate's trigger/fit checks —
// per-round exit and final fit must judge this payload identically to
Expand Down Expand Up @@ -2221,7 +2243,7 @@ export async function preflightCompress(deps: PreflightDeps, messages: CoreMessa
}, 0);
let potential = resolvableMass(ranges);
const deficit = result.payloadEstimate - limit;
if (potential > 0 && deficit > 0 && calibrateFoldMass(potential) * FUTILITY_SLACK < deficit) {
if (potential > 0 && deficit > 0 && imageSavingBound(ranges, turn.messages) === 0 && calibrateFoldMass(potential) * FUTILITY_SLACK < deficit) {
if (!relaxed) {
// The soft-protected recent zone is not in `ranges` yet; #330
// makes it foldable on relax. Probe the relaxed view (CPU-only,
Expand All @@ -2239,10 +2261,11 @@ export async function preflightCompress(deps: PreflightDeps, messages: CoreMessa
// #2638: structural verdicts hold under relax too (coverage-based, zone-independent) — dead mass cannot close the gap either.
const relaxedRanges = filterDeadRanges(deps.session, viableRanges(probe.nudge?.compressibleRanges ?? []).filter((r) => minChars <= 0 || (r.chars ?? r.tokens * 4) >= minChars));
const relaxedPotential = resolvableMass(relaxedRanges);
if (calibrateFoldMass(relaxedPotential) * FUTILITY_SLACK >= deficit) {
if (imageSavingBound(relaxedRanges, probe.messages) > 0 || calibrateFoldMass(relaxedPotential) * FUTILITY_SLACK >= deficit) {
activeConfig = relaxedConfig(deps.config);
relaxed = true;
textTarget = Math.max(0, limit - imageReserve);
foldTarget = limit;
textTarget = Math.max(0, foldTarget - imageReserve);
summaryCalls = 0;
budgetHit = false;
transientRetryBudget = TRANSIENT_EMPTY_RETRY_BUDGET;
Expand Down Expand Up @@ -2283,7 +2306,8 @@ export async function preflightCompress(deps: PreflightDeps, messages: CoreMessa
if (!relaxed && result.payloadEstimate >= limit) {
activeConfig = relaxedConfig(deps.config);
relaxed = true;
textTarget = Math.max(0, limit - imageReserve);
foldTarget = limit;
textTarget = Math.max(0, foldTarget - imageReserve);
// #575-merge: the summarization budget counts per protection
// regime — reset it on relax, else bad summaries burned under
// normal protection can starve the relaxed walk entirely and
Expand Down Expand Up @@ -2658,7 +2682,8 @@ export async function preflightCompress(deps: PreflightDeps, messages: CoreMessa
if (si !== undefined && ei !== undefined && si <= ei) remaining += rj.tokens;
}
const deficit = result.payloadEstimate - limit;
if (remaining > 0 && deficit > 0 && calibrateFoldMass(remaining) * FUTILITY_SLACK < deficit) {
const remainingRanges = ordered.slice(oi).filter(r => !skipSet.has(`${r.startRef}:${r.endRef}`));
if (remaining > 0 && deficit > 0 && imageSavingBound(remainingRanges, turn.messages) === 0 && calibrateFoldMass(remaining) * FUTILITY_SLACK < deficit) {
futileBail = true;
bailRemaining = calibrateFoldMass(remaining);
bailDeficit = deficit;
Expand Down
96 changes: 92 additions & 4 deletions src/responses-tool-output.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,11 @@ import { coreToResponses, patchResponsesInput, responsesToCore, type BiliMessage
import { isRenderedSummaryMessage } from "acp-kernel";
import { systemToUser } from "./util.js";

const responsesPartBinding = Symbol("responsesPartBinding");
type PartBoundMessage = CoreMessage & {
[responsesPartBinding]?: { source: ResponseInputItem; part: ResponseContentPart; sourceId: string };
};

export function responsesToolImageParts(item: unknown): ResponseContentPart[] | undefined {
if (typeof item !== "object" || item === null) return undefined;
const raw = item as Record<string, unknown>;
Expand All @@ -27,6 +32,23 @@ function outputText(parts: ResponseContentPart[]): string {

export function responsesToCoreWithToolImages(body: ResponsesRequestBody): ResponsesProjection {
const projection = responsesToCore(body);
// The layout supplies exact source ownership, including duplicate images.
// Keep it on the core while it survives rendering; never guess ownership
// later from text equality or the first image in a multipart carrier.
const byId = new Map(projection.msgs.map(message => [message.id, message]));
for (const slot of projection.layout) {
const source = slot.original;
if (source.role !== "user" || !Array.isArray(source.content)) continue;
const parts = source.content as ResponseContentPart[];
if (!parts.every(part => part.type === "input_image" || part.type === "input_text" || part.type === "output_text")) continue;
const emitted = parts.filter(part => part.type === "input_image" || (typeof part.text === "string" && part.text.length > 0));
const ids = slot.coreIds ?? (slot.coreId ? [slot.coreId] : []);
if (ids.length !== emitted.length) continue;
ids.forEach((id, index) => {
const message = byId.get(id) as PartBoundMessage | undefined;
if (message) message[responsesPartBinding] = { source, part: emitted[index]!, sourceId: id };
});
}
for (const message of projection.msgs) {
const parts = responsesToolImageParts(message.rawResponsesItem);
// Keep the codec's original content-derived id so persisted refs still
Expand Down Expand Up @@ -104,9 +126,59 @@ export function mergeAdjacentConfigurationUpdates(items: readonly ResponseInputI
// (isRenderedSummaryMessage) — client-origin mid system/developer cores are
// conversation data and re-emit verbatim through their rawResponsesItem
// sidecar; they are NOT rewritten here.
function boundCarrier(messages: PartBoundMessage[]): ResponseInputItem {
const source = messages[0]![responsesPartBinding]!.source;
const parts = messages.map(message => {
const part = message[responsesPartBinding]!.part;
return part.type === "input_image" || part.text === message.text ? part : { ...part, text: message.text ?? "" };
});
const original = source.content;
const unchanged = Array.isArray(original) && original.length === parts.length
&& parts.every((part, index) => part === original[index]);
return unchanged ? source : { type: "message", role: "user", content: parts };
}

function boundProjection(projection: ResponsesProjection, messages: CoreMessage[]): ResponsesProjection {
const runs = new Map<ResponseInputItem, PartBoundMessage[][]>();
let previous: PartBoundMessage | undefined;
for (const message of messages) {
const bound = message as PartBoundMessage;
const binding = bound[responsesPartBinding];
if (!binding || message.role !== "user" || isRenderedSummaryMessage(message)) {
previous = undefined;
continue;
}
const groups = runs.get(binding.source) ?? [];
if (previous?.[responsesPartBinding]?.source === binding.source) groups.at(-1)!.push(bound);
else groups.push([bound]);
runs.set(binding.source, groups);
previous = bound;
}
const sources = new Map(projection.msgs.map(message => [message.id, message]));
const replaced = new Set<string>();
const rebound: BiliMessage[] = [];
const layout = projection.layout.flatMap(slot => {
const groups = runs.get(slot.original);
if (!groups) return [slot];
for (const id of slot.coreIds ?? (slot.coreId ? [slot.coreId] : [])) replaced.add(id);
return groups.map(group => {
for (const message of group) {
const source = sources.get(message[responsesPartBinding]!.sourceId);
if (source) rebound.push({ ...source, id: message.id, text: message.text });
}
const ids = group.map(message => message.id);
return { original: boundCarrier(group), coreId: ids[0], ...(ids.length > 1 ? { coreIds: ids } : {}) };
});
});
// The virtual layout retains untracked wire items, but gives each surviving
// part its actual current ID. Separate runs leave an insertion anchor for
// a block summary between parts from the same original carrier.
return { ...projection, layout, msgs: [...projection.msgs.filter(message => !replaced.has(message.id)), ...rebound] };
}

export function patchResponsesInputWithToolImages(projection: ResponsesProjection, messages: CoreMessage[]): string | ResponseInputItem[] {
const voiced = systemToUser(messages, (m) => isRenderedSummaryMessage(m));
const input = patchResponsesInput(projection, voiced);
const input = patchResponsesInput(boundProjection(projection, voiced), voiced);
if (typeof input === "string") return input;
const byOriginal = new Map<unknown, ResponseInputItem>();
const byCall = new Map<string, ResponseInputItem>();
Expand All @@ -122,8 +194,24 @@ export function patchResponsesInputWithToolImages(projection: ResponsesProjectio
// #1999: same pre-codec re-voicing as patchResponsesInputWithToolImages above.
// #4166: carriers only — client-origin mid system/developer stays verbatim.
export function coreToResponsesWithToolImages(messages: CoreMessage[], customToolCallIds: Set<string> = new Set()): ResponseInputItem[] {
return systemToUser(messages, (m) => isRenderedSummaryMessage(m)).flatMap((message) => {
const out: ResponseInputItem[] = [];
let pending: PartBoundMessage[] = [];
const flush = (): void => {
if (pending.length === 0) return;
out.push(boundCarrier(pending));
pending = [];
};
for (const message of systemToUser(messages, (m) => isRenderedSummaryMessage(m))) {
const bound = message as PartBoundMessage;
if (bound[responsesPartBinding] && message.role === "user" && !isRenderedSummaryMessage(message)) {
if (pending.length && pending[0]![responsesPartBinding]!.source !== bound[responsesPartBinding].source) flush();
pending.push(bound);
continue;
}
flush();
const rebuilt = rebuildToolImages(message);
return rebuilt ? [rebuilt] : coreToResponses([message], customToolCallIds);
});
out.push(...(rebuilt ? [rebuilt] : coreToResponses([message], customToolCallIds)));
}
flush();
return out;
}
Loading
Loading