From 7606a19ad572f888942e33523aece45eec4b02e4 Mon Sep 17 00:00:00 2001 From: ci-check Date: Mon, 16 Nov 2026 21:32:59 +0800 Subject: [PATCH 1/5] kernel: fold plain-text developer items carrying Codex metadata fields MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Codex Desktop stamps mid-history developer items with top-level id + internal_chat_message_metadata_passthrough (per-message bookkeeping: turn_id, content_item_kinds, provenance). The #4166 plain-text check only admitted type/role/content, so those plain-text items were classified unrecoverable, got a BLOCKED ref and were permanently un-foldable; their BLOCKED rows also poisoned the acp_status range view (billion-context #6949). Admit the two known-harmless keys with strict shape checks (id must be a string, the passthrough must be a plain object); any other unknown top-level key still pins. The plain-text rewrite path (wire/responses.ts) already re-emits plain-text items without these fields, so the kernel already treated them as disposable for plain-text items. kernel-change: #6949 — isPlainTextResponsesSystemItem over-rejected plain-text developer items carrying Codex id + metadata passthrough; admit the two known-harmless keys, regression test kernel/tests/issue6949-codex-developer-metadata.test.ts. --- kernel/package.json | 2 +- kernel/src/protected.ts | 27 ++- ...issue6949-codex-developer-metadata.test.ts | 204 ++++++++++++++++++ 3 files changed, 230 insertions(+), 3 deletions(-) create mode 100644 kernel/tests/issue6949-codex-developer-metadata.test.ts diff --git a/kernel/package.json b/kernel/package.json index a58d31360b..f65ce78fdd 100644 --- a/kernel/package.json +++ b/kernel/package.json @@ -1,6 +1,6 @@ { "name": "acp-kernel", - "version": "0.0.198", + "version": "0.0.199", "description": "Framework-agnostic context-compression engine (model-driven, 3-tier LSM). Pure core: no host dependency.", "license": "MIT", "author": "ranxianglei", diff --git a/kernel/src/protected.ts b/kernel/src/protected.ts index 27fd22fa3b..7d83926880 100644 --- a/kernel/src/protected.ts +++ b/kernel/src/protected.ts @@ -764,15 +764,38 @@ export function isPlainTextSystemMessage(raw: unknown): boolean { ); } +/** #6949: top-level keys a responses message item may carry beyond + * type/role/content without becoming structural. `id` is the standard item + * id — the plain-text rewrite path (wire/responses.ts) already re-emits + * plain-text items without it. `internal_chat_message_metadata_passthrough` + * is Codex's per-message bookkeeping (turn_id, content_item_kinds, + * provenance): the model never reads it and it dies with the message on + * fold. Strict shape checks — a different value shape still pins, and any + * OTHER unknown key keeps the conservative #4166 behavior. */ +function isHarmlessResponsesItemKey(key: string, value: unknown): boolean { + if (key === "id") return typeof value === "string"; + if (key === "internal_chat_message_metadata_passthrough") + return ( + typeof value === "object" && value !== null && !Array.isArray(value) + ); + return false; +} + /** #4166: plain-text responses system/developer item — content is a * string or an all-text parts array (input_text/output_text), so its bytes - * live entirely in `text` and folding loses nothing structural. */ + * live entirely in `text` and folding loses nothing structural. #6949: + * known-harmless top-level metadata keys (isHarmlessResponsesItemKey) + * don't make the item structural either. */ function isPlainTextResponsesSystemItem(raw: unknown): boolean { const m = asObj(raw); if ( !m || Object.keys(m).some( - (k) => k !== "type" && k !== "role" && k !== "content", + (k) => + k !== "type" && + k !== "role" && + k !== "content" && + !isHarmlessResponsesItemKey(k, m[k]), ) ) return false; diff --git a/kernel/tests/issue6949-codex-developer-metadata.test.ts b/kernel/tests/issue6949-codex-developer-metadata.test.ts new file mode 100644 index 0000000000..206813af0f --- /dev/null +++ b/kernel/tests/issue6949-codex-developer-metadata.test.ts @@ -0,0 +1,204 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { responsesToCore, coreToResponses } from "../src/wire/responses.js"; +import type { ResponsesRequestBody } from "../src/wire/responses.js"; +import { hasUnrecoverableMediaPayload } from "../src/protected.js"; +import { assignRefs, emptyRefMap, BLOCKED_REF } from "../src/refs.js"; +import type { CoreMessage } from "../src/types.js"; + +// #6949: Codex Desktop stamps mid-history developer items with top-level +// `id` + `internal_chat_message_metadata_passthrough` (per-message +// bookkeeping: turn_id, content_item_kinds, provenance). The #4166 +// plain-text check only allowed type/role/content, so those plain-text +// items were classified as unrecoverable → BLOCKED ref → permanently +// un-foldable, and their BLOCKED rows poisoned the acp_status range view. +// The fix admits the two known-harmless keys (strict shape checks); every +// other unknown top-level key still pins. + +const METADATA = { + turn_id: "turn_example", + content_item_kinds: [ + "additional_content.codex_apps_open_page_instructions", + ], + content_item_metadata: [{ provenance: { type: "harness", hook: false } }], +} as const; + +function midSystemCore(input: ResponsesRequestBody["input"]) { + const flat = responsesToCore({ model: "m", input } as ResponsesRequestBody); + const mid = flat.msgs.find((m) => m.role === "system"); + assert.ok(mid, "mid-history item must become a positioned sidecar core"); + return { flat, mid }; +} + +test("#6949 fixed: developer item with id + metadata passthrough folds (was BLOCKED)", () => { + const { mid } = midSystemCore([ + { type: "message", role: "user", content: "turn 1" }, + { + type: "message", + id: "msg_example", + role: "developer", + content: [ + { + type: "input_text", + text: "page rules", + }, + ], + internal_chat_message_metadata_passthrough: METADATA, + }, + { type: "message", role: "user", content: "turn 2" }, + ]); + assert.equal(mid.originalRole, "developer"); + assert.equal(hasUnrecoverableMediaPayload(mid), false); +}); + +test("#6949 fixed: string-content variant with both fields folds", () => { + const { mid } = midSystemCore([ + { type: "message", role: "user", content: "q" }, + { + type: "message", + id: "msg_example", + role: "developer", + content: "time context", + internal_chat_message_metadata_passthrough: METADATA, + }, + ]); + assert.equal(hasUnrecoverableMediaPayload(mid), false); +}); + +test("#6949: id alone (standard item id) does not pin", () => { + const { mid } = midSystemCore([ + { type: "message", role: "user", content: "q" }, + { + type: "message", + id: "msg_example", + role: "developer", + content: "plain policy text", + }, + ]); + assert.equal(hasUnrecoverableMediaPayload(mid), false); +}); + +test("#6949 negative: any OTHER unknown top-level key still pins", () => { + const { mid } = midSystemCore([ + { type: "message", role: "user", content: "q" }, + { + type: "message", + role: "developer", + content: "plain policy text", + some_future_field: { structural: true }, + } as never, + ]); + assert.equal(hasUnrecoverableMediaPayload(mid), true); +}); + +test("#6949 negative: wrong value shapes still pin", () => { + const nonStringId = midSystemCore([ + { type: "message", role: "user", content: "q" }, + { + type: "message", + id: 42, + role: "developer", + content: "plain policy text", + } as never, + ]).mid; + assert.equal(hasUnrecoverableMediaPayload(nonStringId), true); + + const stringMetadata = midSystemCore([ + { type: "message", role: "user", content: "q" }, + { + type: "message", + role: "developer", + content: "plain policy text", + internal_chat_message_metadata_passthrough: "not-an-object", + } as never, + ]).mid; + assert.equal(hasUnrecoverableMediaPayload(stringMetadata), true); + + const arrayMetadata = midSystemCore([ + { type: "message", role: "user", content: "q" }, + { + type: "message", + role: "developer", + content: "plain policy text", + internal_chat_message_metadata_passthrough: [METADATA], + } as never, + ]).mid; + assert.equal(hasUnrecoverableMediaPayload(arrayMetadata), true); +}); + +test("#6949: structured content with the harmless keys still pins (content check unchanged)", () => { + const { mid } = midSystemCore([ + { type: "message", role: "user", content: "q" }, + { + type: "message", + id: "msg_example", + role: "developer", + content: [ + { type: "input_text", text: "declared tools:" }, + { type: "tool_addition", name: "web_search" } as never, + ], + internal_chat_message_metadata_passthrough: METADATA, + }, + ]); + assert.equal(hasUnrecoverableMediaPayload(mid), true); +}); + +test("#6949: un-folded item re-emits byte-exact INCLUDING the metadata fields", () => { + const input = [ + { type: "message", role: "user", content: "turn 1" }, + { + type: "message", + id: "msg_example", + role: "developer", + content: [ + { + type: "input_text", + text: "page rules", + }, + ], + internal_chat_message_metadata_passthrough: METADATA, + }, + { type: "message", role: "user", content: "turn 2" }, + ] as ResponsesRequestBody["input"]; + const flat = responsesToCore({ model: "m", input } as ResponsesRequestBody); + const rebuilt = coreToResponses(flat.msgs, flat.customToolCallIds); + const sysDev = rebuilt.filter( + (i) => + (i as { type?: string }).type === "message" && + (i as { role?: string }).role === "developer", + ); + assert.equal(sysDev.length, 1); + assert.deepEqual( + sysDev[0], + input[1], + "verbatim sidecar round-trip keeps every field", + ); +}); + +test("#6949: assignRefs mints a numeric ref (not BLOCKED) for the item", () => { + const { mid } = midSystemCore([ + { type: "message", role: "user", content: "turn 1" }, + { + type: "message", + id: "msg_example", + role: "developer", + content: [ + { + type: "input_text", + text: "page rules", + }, + ], + internal_chat_message_metadata_passthrough: METADATA, + }, + { type: "message", role: "user", content: "turn 2" }, + ]); + const result = assignRefs([mid as CoreMessage], { + existing: emptyRefMap(), + nextIndex: 1, + isProtected: (m) => hasUnrecoverableMediaPayload(m), + }); + const ref = result.map.byRaw[mid.id]; + assert.ok(ref, "message must carry a ref"); + assert.notEqual(ref, BLOCKED_REF, "no longer BLOCKED"); + assert.match(ref, /^m\d{5}$/, "numeric mNNNNN ref"); +}); From 6356ee8dfb231f35573ce63f0ad69a2362684d45 Mon Sep 17 00:00:00 2001 From: ci-check Date: Mon, 16 Nov 2026 23:07:18 +0800 Subject: [PATCH 2/5] fix(kernel): sync kernel/package-lock.json version 0.0.198 -> 0.0.199 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The 0.0.199 bump in kernel/package.json (commit 7606a19) left the kernel lockfile at 0.0.198, so the lock-sync lane (npm install --package-lock-only + git diff --exit-code) fails on every run of this head and — because the test matrix job needs lock-sync — the full test suite never runs on this PR. Small in-pass fix found during PR review: two version fields, no other content. Verified idempotent: kernel/ npm install --package-lock-only makes no further change. --- kernel/package-lock.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/kernel/package-lock.json b/kernel/package-lock.json index 9437e48cf1..3e25df6ca0 100644 --- a/kernel/package-lock.json +++ b/kernel/package-lock.json @@ -1,12 +1,12 @@ { "name": "acp-kernel", - "version": "0.0.198", + "version": "0.0.199", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "acp-kernel", - "version": "0.0.198", + "version": "0.0.199", "license": "MIT", "devDependencies": { "@types/node": "^22.0.0", From 5c69c8ac24722119ca585620ff397b83f1ab3d30 Mon Sep 17 00:00:00 2001 From: ci-check Date: Mon, 16 Nov 2026 23:43:00 +0800 Subject: [PATCH 3/5] fix: sync root package-lock.json kernel entry version 0.0.198 -> 0.0.199 Second half of the lock-sync fix (6356ee8dfb covered kernel/package-lock.json): the root lockfile records the file: dependency's version under the 'kernel' folder key, which the 0.0.199 bump in kernel/package.json left stale. Same lock-sync lane, same root cause (version bump without lockfile sync). Verified: root npm install --package-lock-only produces exactly this one-line change and nothing else; kernel/ side already idempotent. --- package-lock.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package-lock.json b/package-lock.json index d12b556eb1..006bda1e14 100644 --- a/package-lock.json +++ b/package-lock.json @@ -51,7 +51,7 @@ }, "kernel": { "name": "acp-kernel", - "version": "0.0.198", + "version": "0.0.199", "dev": true, "license": "MIT", "devDependencies": { From 053b5a66b7b29ec673ed71b65a296d0c7894b738 Mon Sep 17 00:00:00 2001 From: ci-check Date: Tue, 17 Nov 2026 10:49:05 +0800 Subject: [PATCH 4/5] fix(logging): closeLogger waits for replaced-stream drains before resolving MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit closeLogger resolved when the FINAL stream ended, but rotated-out streams are end()-ed asynchronously and their flush can still be in flight. On a loaded Windows runner a slow replaced flush let the shutdown read a sealed generation before its tail was written, losing a rotated record (logger-rotate 'successive rotations retain distinct payloads' red on windows-22 shard 2, PR #6952). Track the in-flight replaced streams with a drain-waiter list: settle() already decrements pendingRetentionDrains and now notifies waiters at zero, and closeLogger waits for BOTH the final stream and pendingRetentionDrains === 0, honoring its documented contract ('resolves once every buffered line has drained to disk'). 'close' always fires, so the counter always settles — no new hang path. --- src/logger.ts | 39 ++++++++++++++++++++++++++++++++++----- 1 file changed, 34 insertions(+), 5 deletions(-) diff --git a/src/logger.ts b/src/logger.ts index b54c5ba9be..b861a95fae 100644 --- a/src/logger.ts +++ b/src/logger.ts @@ -208,6 +208,19 @@ export function setLogCapture(fn: ((level: string, msg: string) => void) | null) } let pendingRetentionDrains = 0; +// closeLogger() waiters, invoked when the last replaced stream drains so a +// shutdown can prove EVERY buffered line is on disk (not just the final +// stream's). Cleared on fire so a later close does not re-run stale callbacks. +let drainWaiters: Array<() => void> = []; + +function notifyDrainsSettled(): void { + if (pendingRetentionDrains !== 0) return; + const waiters = drainWaiters; + drainWaiters = []; + for (const w of waiters) { + try { w(); } catch { /* best-effort */ } + } +} function openStream(file: string, deferRetention = false): WriteStream { mkdirSync(path.dirname(file), { recursive: true }); @@ -332,6 +345,7 @@ function getStream(): WriteStream | undefined { replaced.removeListener("close", settle); pendingRetentionDrains--; if (pendingRetentionDrains === 0 && replacedPath) applyLogRetention(replacedPath); + notifyDrainsSettled(); }; replaced.once("close", settle); closeQuietly(replaced, settle); @@ -466,23 +480,38 @@ export const log: Logger = (level, msg) => { }; /** Flush + close the log file. Call on shutdown; resolves once every buffered - * line has drained to disk, so awaiting it proves the durable record is complete. */ + * line has drained to disk, so awaiting it proves the durable record is complete. + * That includes lines still in flight on REPLACED streams (rotated out before + * shutdown): each is end()-ed asynchronously, so we wait for the drain counter + * to settle as well as the final stream, or a slow replaced flush could resolve + * us with its tail still unflushed (observed: a rotated record lost on Windows). */ export function closeLogger(): Promise { const s = stream; stream = undefined; streamFd = undefined; writeProven = false; bytesWritten = 0; - if (!s) return Promise.resolve(); return new Promise((resolve) => { let done = false; - const finish = (): void => { + let streamEnded = !s; + const tryFinish = (): void => { if (done) return; + if (!streamEnded || pendingRetentionDrains !== 0) return; done = true; resolve(); }; - s.once("error", finish); - try { s.end(finish); } catch { finish(); } + if (s) { + const onEnd = (): void => { + streamEnded = true; + tryFinish(); + }; + s.once("error", onEnd); + try { s.end(onEnd); } catch { onEnd(); } + } + // settle() re-invokes tryFinish() as each replaced stream drains; the + // trailing call covers the already-settled case (no waiter registered). + if (pendingRetentionDrains > 0) drainWaiters.push(tryFinish); + tryFinish(); }); } From 8345b28863d66cf70c6161757c0b05080ddbae98 Mon Sep 17 00:00:00 2001 From: ci-check Date: Tue, 17 Nov 2026 22:45:14 +0800 Subject: [PATCH 5/5] kernel: add CLI metadata-variant + stale-BLOCKED migration tests (#6949) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit kernel-change: #6949 — acceptance follow-up requested in the issue thread (field audit, Codex CLI 0.169.0): pin the three observed plain-text classes (host_skills.instructions / tools.deferred_namespaces / generic.turn_aborted) with valid id + metadata object, and pin the stale-BLOCKED migration so pre-existing mislabeled sessions recover on re-assignment without a new session. --- ...issue6949-codex-developer-metadata.test.ts | 84 +++++++++++++++++++ 1 file changed, 84 insertions(+) diff --git a/kernel/tests/issue6949-codex-developer-metadata.test.ts b/kernel/tests/issue6949-codex-developer-metadata.test.ts index 206813af0f..6dbc736937 100644 --- a/kernel/tests/issue6949-codex-developer-metadata.test.ts +++ b/kernel/tests/issue6949-codex-developer-metadata.test.ts @@ -202,3 +202,87 @@ test("#6949: assignRefs mints a numeric ref (not BLOCKED) for the item", () => { assert.notEqual(ref, BLOCKED_REF, "no longer BLOCKED"); assert.match(ref, /^m\d{5}$/, "numeric mNNNNN ref"); }); + +// Acceptance follow-up (field audit on Codex CLI 0.169.0, #6949 thread): +// the three CLI plain-text classes that were observed pinned — host skill +// instructions, deferred tool namespaces, turn-aborted notices — all carry +// the same shape (valid id + metadata object + input_text parts) and must +// fold. The predicate is shape-based, so one row per class pins the real +// metadata variants. +test("#6949 acceptance: CLI's three plain-text classes fold (skills / deferred namespaces / turn-aborted)", () => { + const classes = [ + { + kind: "additional_content.host_skills.instructions", + text: "host skill instructions: before entering a review loop, read its SKILL.md and follow the loop steps", + }, + { + kind: "additional_content.tools.deferred_namespaces", + text: "deferred tool namespaces: mcp__search, mcp__files (loaded on demand)", + }, + { + kind: "additional_content.generic.turn_aborted", + text: "the previous turn was aborted by the user before a final answer was produced", + }, + ]; + for (const { kind, text } of classes) { + const { mid } = midSystemCore([ + { type: "message", role: "user", content: "q" }, + { + type: "message", + id: `msg_cli_${kind}`, + role: "developer", + content: [{ type: "input_text", text }], + internal_chat_message_metadata_passthrough: { + turn_id: "turn_cli", + content_item_kinds: [kind], + content_item_metadata: [{ provenance: { type: "harness", hook: false } }], + }, + }, + ]); + assert.equal(hasUnrecoverableMediaPayload(mid), false, `${kind} must fold`); + } +}); + +// Acceptance follow-up (field audit, #6949 thread): sessions that ALREADY +// carry a stale BLOCKED ref for one of these items must recover on the next +// ref-assignment pass after the upgrade — no new session required. This pins +// the #2607 stale-BLOCKED migration on the exact shape that produced it. +test("#6949 acceptance: pre-existing BLOCKED ref migrates to a numeric ref on re-assignment", () => { + const flat = responsesToCore({ + model: "m", + input: [ + { type: "message", role: "user", content: "turn 1" }, + { + type: "message", + id: "msg_example", + role: "developer", + content: [ + { + type: "input_text", + text: "page rules", + }, + ], + internal_chat_message_metadata_passthrough: METADATA, + }, + { type: "message", role: "user", content: "turn 2" }, + ], + } as ResponsesRequestBody); + const [u1, dev, u2] = flat.msgs; + assert.equal(dev?.role, "system"); + // Pre-fix session state: neighbors hold numeric refs, the developer item + // was pinned BLOCKED. + const existing = { + byRaw: { [u1!.id]: "m00001", [dev!.id]: BLOCKED_REF, [u2!.id]: "m00002" }, + byRef: { m00001: u1!.id, m00002: u2!.id }, + }; + const result = assignRefs(flat.msgs as CoreMessage[], { + existing, + nextIndex: 3, + isProtected: (m) => hasUnrecoverableMediaPayload(m), + }); + assert.deepEqual(result.migratedIds, [dev!.id]); + assert.equal(result.map.byRaw[dev!.id], "m00003", "stale BLOCKED mints a fresh numeric ref"); + assert.equal(result.map.byRef["m00003"], dev!.id); + assert.equal(result.map.byRaw[u1!.id], "m00001", "neighbors keep their refs"); + assert.equal(result.map.byRaw[u2!.id], "m00002", "neighbors keep their refs"); +});