Skip to content
1 change: 1 addition & 0 deletions reference/architecture.md
Original file line number Diff line number Diff line change
Expand Up @@ -304,6 +304,7 @@ billion-context/
│ ├── remote-notices.ts # Remote announcement feed watcher: fail-open polling, last-known-good snapshot (#5548)
│ ├── request-membership.ts # Settle-side membership probe: is this usage report ours (#4805)
│ ├── resolver-wedge.ts # Persistent local DNS resolver wedge detector + alarm (#5221)
│ ├── responses-system-sections.ts # Single source of truth for the Responses-wire developer-section join: steady prepare + round-2 rebuild share one separator (#6922)
│ ├── responses-tool-output.ts # Responses tool-output image parts: projection + patch helpers
│ ├── rollout.ts # Feature rollout: per-install uuid, basis-point buckets, nested cohorts (#3062)
│ ├── runaway-guard.ts # Intrinsic stream terminator for runaway tag/codepoint enumeration (#2346)
Expand Down
6 changes: 3 additions & 3 deletions src/decompress-shared.ts
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ import { existsSync, mkdirSync, unlinkSync, writeFileSync } from "node:fs";
import { dirname, join, resolve as resolvePath } from "node:path";
import { tmpdir } from "node:os";
import { fileURLToPath } from "node:url";
import { markDirty, preCompactionArchiveOf, peekSession, findSessionByCanonicalId, type Session } from "./session.js";
import { blockContentSnapshot, markDirty, preCompactionArchiveOf, peekSession, findSessionByCanonicalId, type Session } from "./session.js";
import { getStore } from "./persist.js";
import { ccrEnabled, contentStoreOf } from "./store.js";
import { safePrefix } from "./text-safe.js";
Expand Down Expand Up @@ -218,7 +218,7 @@ export function resolveDecompress(
}

const full = args.full === true || args.full === "true";
const cached = ctx.session.blockContents.get(blockId);
const cached = blockContentSnapshot(ctx.session.blockContents, block);
let body: string;
let count: number;
if (cached) {
Expand Down Expand Up @@ -591,7 +591,7 @@ function resolveDerivedDecompress(
return toolFail(`[decompress FAILED: range restore (startId/endId) of derived-parent blocks is not supported — decompress "${blockId}" without range args (#1333)]`);
}
const full = args.full === true || args.full === "true";
const cached = anc.blockContents.get(blockId);
const cached = blockContentSnapshot(anc.blockContents, block);
const view = cached ? (full ? cached.full : (cached.one ?? cached.full)) : undefined;
const body = view?.text || block.summary;
const count = view?.count ?? 0;
Expand Down
10 changes: 5 additions & 5 deletions src/export.ts
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
import { existsSync, mkdirSync, writeFileSync } from "node:fs";
import path from "node:path";
import { indexToRef, matchSession as matchSessionKernel, parseStoredPlaceholder, refForRaw, renderHandoff as renderHandoffKernel, renderMessage, type CompressionBlock, type MessageContentStore, type CoreMessage } from "acp-kernel";
import type { Session } from "./session.js";
import { blockContentSnapshot, type Session } from "./session.js";
import { SessionStore } from "./persist.js";
import { SqliteBackend, backendSelectionNotice, listBackendNames, openBackend, resolveBackendName, resolveMysqlDsn, selectedBackendRaw, type StorageBackend } from "./storage/backend.js";
import { fileStorageBackend, fileStorageMysqlDsn } from "./knobs.js";
Expand Down Expand Up @@ -64,7 +64,7 @@ function blockLedgerLine(s: Session, b: CompressionBlock): string {
}
}
}
const orig = s.blockContents.get(b.blockId)?.full.text.length;
const orig = blockContentSnapshot(s.blockContents, b)?.full.text.length;
const status = !b.active ? "consumed" : b.retired ? "retired" : "active";
return `messages ${range} · original ${orig !== undefined ? `${orig} chars` : "not cached"} · summary ${b.summary.length} chars · ${status}`;
}
Expand Down Expand Up @@ -196,7 +196,7 @@ export function renderHandoff(s: Session, full: boolean): string {
lines.push("");
lines.push(b.summary.trim());
lines.push("");
const content = s.blockContents.get(b.blockId);
const content = blockContentSnapshot(s.blockContents, b);
if (full && content) {
lines.push(`### Original messages (${content.full.count})`);
lines.push("");
Expand Down Expand Up @@ -242,7 +242,7 @@ function blockSummariesSection(s: Session, includeOriginals: boolean, priorDoc:
lines.push(summary);
}
lines.push("");
const content = includeOriginals ? s.blockContents.get(b.blockId) : undefined;
const content = includeOriginals ? blockContentSnapshot(s.blockContents, b) : undefined;
if (content) {
lines.push(`#### Original messages (${content.full.count})`);
lines.push("");
Expand Down Expand Up @@ -392,7 +392,7 @@ export function renderSessionJsonlFromSession(s: Session, full: boolean, content
lines.push(JSON.stringify({ type: "message", seq: i, rawId: m.id, role: m.role, message: out }));
}
for (const [i, b] of s.state.blocks.entries()) {
const orig = s.blockContents.get(b.blockId)?.full.text.length;
const orig = blockContentSnapshot(s.blockContents, b)?.full.text.length;
lines.push(
JSON.stringify({
type: "block",
Expand Down
3 changes: 2 additions & 1 deletion src/external-summary-compress.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
import { collectBlockContent } from "acp-kernel";
import { blockContentSnapshot } from "./session.js";
import { buildCompressSystemPrompt, parseCompressInput, type ParsedRange } from "./compress-tool.js";
import { messageTimestampSections } from "./external-summary-surface.js";
import { configuredSummaryPlan } from "./external-summary-runtime.js";
Expand Down Expand Up @@ -84,7 +85,7 @@ export async function applyConfiguredCompression(input: unknown, ctx: RewriteCtx
const collected = collectBlockContent(state, block, source, { full: false }).text;
if (collected.trim()) return collected;
// An inline-restored block may no longer have originals in the wire view.
const cached = ctx.session.blockContents.get(block.blockId);
const cached = blockContentSnapshot(ctx.session.blockContents, block);
return cached?.one?.text ?? cached?.full?.text ?? "";
}).join("\n\n");
if (!content.trim()) continue;
Expand Down
16 changes: 16 additions & 0 deletions src/session.ts
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,22 @@ export type BlockContent = {
full: BlockView;
};

/** The compression-time snapshot of a block, or undefined when there is none.
* The unified index stores a placeholder for blocks whose legacy record had no
* snapshot ({ one: null, full: { text: block.summary, count: 0 } }, see
* completeBlockContents in src/storage/ingest.ts) so every block row has a view;
* replay serves it back. It is not a snapshot: readers must treat it like a
* missing entry (decompress collects from the live messages, the web view says
* "no snapshot") or a migrated session returns the summary as the block's content. */
export function blockContentSnapshot(
contents: { get(blockId: string): BlockContent | undefined },
block: { blockId: string; summary?: string },
): BlockContent | undefined {
const content = contents.get(block.blockId);
if (content !== undefined && content.one === null && content.full.count === 0 && content.full.text === block.summary) return undefined;
return content;
}

/** #3050 R3 ②b: block-contents map that folds away to (almost) nothing once
* a save has captured it, and re-materializes from the store on the first
* access. Idle sessions stop carrying every folded block's originals; the
Expand Down
6 changes: 3 additions & 3 deletions src/web/sessions-data.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
import { defaultCountTokens, orderedRefPair, type CompressionBlock } from "acp-kernel";
import { listSessions, displayContextBest, type ContextBest, type Session, type BlockContent } from "../session.js";
import { blockContentSnapshot, listSessions, displayContextBest, type ContextBest, type Session, type BlockContent } from "../session.js";
import { conflictClientOf, conflictEventsOf } from "../conflict-watch.js";
import { isDisplayOnlyConflictDetail, isSiblingConflictDetail } from "../thirdparty-scan.js";
import { fileNameMatchesId, getStore, isValidRecord, _resetStoreForTest, relPathFor } from "../persist.js";
Expand Down Expand Up @@ -1729,7 +1729,7 @@ function snapshotRefIndexFor(session: Session, block: CompressionBlock): Map<str
if (direct.length === 0) return null;
let content: BlockContent | undefined;
try {
content = session.blockContents.get(block.blockId);
content = blockContentSnapshot(session.blockContents, block);
} catch {
return null;
}
Expand Down Expand Up @@ -1883,7 +1883,7 @@ export async function buildBlockContent(id: string, blockId: string, view: Block
if (!block) return { ok: false, reason: "unknown-block" };
let content: BlockContent | undefined;
try {
content = session.blockContents.get(blockId);
content = blockContentSnapshot(session.blockContents, block);
} catch (error) {
// #3917: a corrupt/unreadable content store must not 500 the panel —
// report it as "no snapshot" and let the UI explain.
Expand Down
66 changes: 40 additions & 26 deletions tests/issue2929-codex-fold-double-miss.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -34,16 +34,17 @@ type Item = Record<string, unknown>;

const SUMMARY_MARKER = "[Compressed conversation section]";
const NUDGE_MARKER = "Compressible ranges (";
// Indices verified against the kernel's nudge gate with nudgeGrowthTokens=2000
// and ~9KiB filler turns. Pre-#4521 (master) the injections landed on
// requests {7,8,9}; since #4521 the armed-but-silent status line rides an
// ephemeral carrier on the gated turns and its shown-marker feeds the nudge
// gate back, so the schedule shifts to {4,6,8,9} — the fold/proxy-tool-round
// mechanics under test are unchanged, only this probe's anchor re-calibrates
// to the next nudge-carrying request pair (6 → tool round, 7 → fold round).
const PROXY_TOOL_AT = 6; // upstream request index whose response = acp_status call (round 1)
const FOLD_AT = 7; // upstream request index whose response = compress call (round 2)
// FOLD_AT+1 = post-fold refreshed re-request (round 3); FOLD_AT+2 = next steady client turn.
// The fold turn is materialized SCHEDULE-INDEPENDENTLY: the fake upstream
// arms on the first request carrying the ephemeral nudge (with >=11 tagged
// refs to fold) and folds on the re-request that follows the acp_status
// round. A hardcoded request-index anchor broke every time the nudge gate's
// token schedule shifted — #4521's status-line feedback ({7,8,9} → {4,6,8,9}),
// #6928's status-line removal, and #4313's timestamp-rendering default flip
// (the suite preload pin BILI_MESSAGE_TIMESTAMPS="0" changes the wire shape
// and hence the schedule: OFF {4,13} vs ON {3,5,11,14} — direct local runs
// without the preload diverge from CI). The probe also pins
// compress.messageTimestamps:false so its wire shape is hermetic regardless
// of how the suite is launched.
const TOTAL_TURNS = 13;

const FILLER = (seed: number, kb: number): string => {
Expand Down Expand Up @@ -120,22 +121,35 @@ test("#2929: nudge + proxy-tool-round fold keeps re-request/next-turn byte-stabl

try {
const captured: Array<{ url: string; body: string }> = [];
// Object (not bare lets) so the outer-scope reads aren't narrowed to
// the initial "idle" — the mutations happen inside the request callback.
const fold = {
state: "idle" as "idle" | "armed" | "folded",
armIdx: -1, // upstream request index that carried the nudge (acp_status round)
foldIdx: -1, // upstream request index that carried the compress call
};
upstream = http.createServer((req, res) => {
const chunks: Buffer[] = [];
req.on("data", (c: Buffer) => chunks.push(c));
req.on("end", () => {
const idx = captured.length;
const body = Buffer.concat(chunks).toString("utf8");
captured.push({ url: req.url ?? "", body });
if (idx === PROXY_TOOL_AT) {
// Round 1 of the fold turn: model asks for proxy status — the
// loop executes it locally and re-requests (round 2).
const input = (JSON.parse(body) as { input?: Item[] }).input ?? [];
if (fold.state === "idle" && hasNudge(input) && refsOf(input).length >= 11) {
// Round 1 of the fold turn: the first nudge-carrying request
// with enough refs — model asks for proxy status, the loop
// executes it locally and re-requests (round 2).
fold.state = "armed";
fold.armIdx = idx;
functionCallSse(res, `resp_${idx}`, "acp_status", "{}", `call_st_${idx}`);
return;
}
if (idx === FOLD_AT) {
const prev = (JSON.parse(captured[idx - 1]!.body) as { input?: Item[] }).input ?? [];
const uniq = refsOf(prev);
if (fold.state === "armed") {
// Round 2: the re-request after the acp_status round — fold.
fold.state = "folded";
fold.foldIdx = idx;
const uniq = refsOf(input);
assert.ok(uniq.length >= 11, `fold round: expected >=11 tagged refs, saw ${uniq.length}`);
const args = JSON.stringify({ content: [
{ startId: uniq[3]!, endId: uniq[5]!, topic: "F1A", summary: `F1A summary: ${FILLER(1, 1)}` },
Expand All @@ -144,10 +158,9 @@ test("#2929: nudge + proxy-tool-round fold keeps re-request/next-turn byte-stabl
functionCallSse(res, `resp_${idx}`, "compress", args, `call_cmp_${idx}`);
return;
}
const parsed = JSON.parse(body) as { input?: Item[] };
let label = "0";
for (let i = (parsed.input ?? []).length - 1; i >= 0; i--) {
const it = parsed.input![i]!;
for (let i = input.length - 1; i >= 0; i--) {
const it = input[i]!;
if (it.type === "message" && it.role === "user" && typeof it.content === "string") {
const m = (it.content as string).match(/Turn (\d+):/);
if (m) { label = m[1]!; break; }
Expand All @@ -169,7 +182,7 @@ test("#2929: nudge + proxy-tool-round fold keeps re-request/next-turn byte-stabl
routes: { [`http://127.0.0.1:${upstreamPort}`]: { models: { "gpt-test": { context: 200_000 } } } },
modelContextLimit: 200_000,
kernelConfig: defaultConfig(200_000),
compress: { injectTool: true, injectNudge: true, nudgeGrowthTokens: 2000 },
compress: { injectTool: true, injectNudge: true, nudgeGrowthTokens: 2000, messageTimestamps: false },
promptCache: { routing: "auto" },
sessionHeader: "x-acp-session",
log: true,
Expand Down Expand Up @@ -223,25 +236,26 @@ test("#2929: nudge + proxy-tool-round fold keeps re-request/next-turn byte-stabl
}

const inputs = captured.map((c) => (JSON.parse(c.body) as { input?: Item[] }).input ?? []);
assert.ok(inputs.length >= FOLD_AT + 3, `expected >= ${FOLD_AT + 3} upstream requests, got ${inputs.length}`);
assert.ok(fold.state === "folded", `scenario miss: the nudge never fired on a request with >=11 refs — probe cannot materialize the #2929 trigger; adjust nudgeGrowthTokens/filler`);
assert.ok(inputs.length >= fold.foldIdx + 3, `expected >= ${fold.foldIdx + 3} upstream requests, got ${inputs.length}`);

// (0) Scenario materialization: the nudge must have been injected into
// the PRE-FOLD original request (production shape: nudge armed -> model
// the arming (pre-fold) request (production shape: nudge armed -> model
// calls proxy tool -> compress). Without it this probe does not cover
// the #2929 trigger.
assert.ok(hasNudge(inputs[PROXY_TOOL_AT]!), `scenario miss: no ephemeral nudge in the pre-fold request (inputs[${PROXY_TOOL_AT}]) — probe does not cover the production trigger; adjust nudgeGrowthTokens/filler`);
console.log(`probe: nudge confirmed in pre-fold request; total upstream requests=${inputs.length}`);
assert.ok(hasNudge(inputs[fold.armIdx]!), `scenario miss: no ephemeral nudge in the arming request (inputs[${fold.armIdx}]) — probe does not cover the production trigger`);
console.log(`probe: nudge at inputs[${fold.armIdx}], fold at inputs[${fold.foldIdx}]; total upstream requests=${inputs.length}`);

// (1) The post-fold re-request carries the in-place summary(s).
const rPost = inputs[FOLD_AT + 1]!;
const rPost = inputs[fold.foldIdx + 1]!;
const postSums = summaryIndices(rPost);
assert.ok(postSums.length >= 1, "post-fold re-request must carry the in-place summary");

// (2) THE invariant: next steady client turn shares identical bytes with
// the post-fold re-request up to and including the summary anchor. In
// production this is where the chain breaks (next turn hits only the
// instructions baseline).
const next = inputs[FOLD_AT + 2]!;
const next = inputs[fold.foldIdx + 2]!;
const nextSums = summaryIndices(next);
assert.ok(nextSums.length >= 1, `next steady turn lost the summary entirely (summaries: ${nextSums.length}) — fold did not survive to the client turn`);
assert.equal(nextSums[0], postSums[0], `summary anchor drifted: re-request item[${postSums[0]}] vs next turn item[${nextSums[0]}]`);
Expand Down
Loading
Loading