diff --git a/kernel/package-lock.json b/kernel/package-lock.json index bb7bd588bb..8a2c4491d7 100644 --- a/kernel/package-lock.json +++ b/kernel/package-lock.json @@ -1,12 +1,12 @@ { "name": "acp-kernel", - "version": "0.0.200", + "version": "0.0.201", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "acp-kernel", - "version": "0.0.200", + "version": "0.0.201", "license": "MIT", "devDependencies": { "@types/node": "^22.0.0", diff --git a/kernel/package.json b/kernel/package.json index d20ac004ac..5ac694d897 100644 --- a/kernel/package.json +++ b/kernel/package.json @@ -1,6 +1,6 @@ { "name": "acp-kernel", - "version": "0.0.200", + "version": "0.0.201", "description": "Framework-agnostic context-compression engine (model-driven, 3-tier LSM). Pure core: no host dependency.", "license": "MIT", "author": "ranxianglei", diff --git a/kernel/src/protected.ts b/kernel/src/protected.ts index 27fd22fa3b..7d83926880 100644 --- a/kernel/src/protected.ts +++ b/kernel/src/protected.ts @@ -764,15 +764,38 @@ export function isPlainTextSystemMessage(raw: unknown): boolean { ); } +/** #6949: top-level keys a responses message item may carry beyond + * type/role/content without becoming structural. `id` is the standard item + * id — the plain-text rewrite path (wire/responses.ts) already re-emits + * plain-text items without it. `internal_chat_message_metadata_passthrough` + * is Codex's per-message bookkeeping (turn_id, content_item_kinds, + * provenance): the model never reads it and it dies with the message on + * fold. Strict shape checks — a different value shape still pins, and any + * OTHER unknown key keeps the conservative #4166 behavior. */ +function isHarmlessResponsesItemKey(key: string, value: unknown): boolean { + if (key === "id") return typeof value === "string"; + if (key === "internal_chat_message_metadata_passthrough") + return ( + typeof value === "object" && value !== null && !Array.isArray(value) + ); + return false; +} + /** #4166: plain-text responses system/developer item — content is a * string or an all-text parts array (input_text/output_text), so its bytes - * live entirely in `text` and folding loses nothing structural. */ + * live entirely in `text` and folding loses nothing structural. #6949: + * known-harmless top-level metadata keys (isHarmlessResponsesItemKey) + * don't make the item structural either. */ function isPlainTextResponsesSystemItem(raw: unknown): boolean { const m = asObj(raw); if ( !m || Object.keys(m).some( - (k) => k !== "type" && k !== "role" && k !== "content", + (k) => + k !== "type" && + k !== "role" && + k !== "content" && + !isHarmlessResponsesItemKey(k, m[k]), ) ) return false; diff --git a/kernel/tests/issue6949-codex-developer-metadata.test.ts b/kernel/tests/issue6949-codex-developer-metadata.test.ts new file mode 100644 index 0000000000..6dbc736937 --- /dev/null +++ b/kernel/tests/issue6949-codex-developer-metadata.test.ts @@ -0,0 +1,288 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { responsesToCore, coreToResponses } from "../src/wire/responses.js"; +import type { ResponsesRequestBody } from "../src/wire/responses.js"; +import { hasUnrecoverableMediaPayload } from "../src/protected.js"; +import { assignRefs, emptyRefMap, BLOCKED_REF } from "../src/refs.js"; +import type { CoreMessage } from "../src/types.js"; + +// #6949: Codex Desktop stamps mid-history developer items with top-level +// `id` + `internal_chat_message_metadata_passthrough` (per-message +// bookkeeping: turn_id, content_item_kinds, provenance). The #4166 +// plain-text check only allowed type/role/content, so those plain-text +// items were classified as unrecoverable → BLOCKED ref → permanently +// un-foldable, and their BLOCKED rows poisoned the acp_status range view. +// The fix admits the two known-harmless keys (strict shape checks); every +// other unknown top-level key still pins. + +const METADATA = { + turn_id: "turn_example", + content_item_kinds: [ + "additional_content.codex_apps_open_page_instructions", + ], + content_item_metadata: [{ provenance: { type: "harness", hook: false } }], +} as const; + +function midSystemCore(input: ResponsesRequestBody["input"]) { + const flat = responsesToCore({ model: "m", input } as ResponsesRequestBody); + const mid = flat.msgs.find((m) => m.role === "system"); + assert.ok(mid, "mid-history item must become a positioned sidecar core"); + return { flat, mid }; +} + +test("#6949 fixed: developer item with id + metadata passthrough folds (was BLOCKED)", () => { + const { mid } = midSystemCore([ + { type: "message", role: "user", content: "turn 1" }, + { + type: "message", + id: "msg_example", + role: "developer", + content: [ + { + type: "input_text", + text: "page rules", + }, + ], + internal_chat_message_metadata_passthrough: METADATA, + }, + { type: "message", role: "user", content: "turn 2" }, + ]); + assert.equal(mid.originalRole, "developer"); + assert.equal(hasUnrecoverableMediaPayload(mid), false); +}); + +test("#6949 fixed: string-content variant with both fields folds", () => { + const { mid } = midSystemCore([ + { type: "message", role: "user", content: "q" }, + { + type: "message", + id: "msg_example", + role: "developer", + content: "time context", + internal_chat_message_metadata_passthrough: METADATA, + }, + ]); + assert.equal(hasUnrecoverableMediaPayload(mid), false); +}); + +test("#6949: id alone (standard item id) does not pin", () => { + const { mid } = midSystemCore([ + { type: "message", role: "user", content: "q" }, + { + type: "message", + id: "msg_example", + role: "developer", + content: "plain policy text", + }, + ]); + assert.equal(hasUnrecoverableMediaPayload(mid), false); +}); + +test("#6949 negative: any OTHER unknown top-level key still pins", () => { + const { mid } = midSystemCore([ + { type: "message", role: "user", content: "q" }, + { + type: "message", + role: "developer", + content: "plain policy text", + some_future_field: { structural: true }, + } as never, + ]); + assert.equal(hasUnrecoverableMediaPayload(mid), true); +}); + +test("#6949 negative: wrong value shapes still pin", () => { + const nonStringId = midSystemCore([ + { type: "message", role: "user", content: "q" }, + { + type: "message", + id: 42, + role: "developer", + content: "plain policy text", + } as never, + ]).mid; + assert.equal(hasUnrecoverableMediaPayload(nonStringId), true); + + const stringMetadata = midSystemCore([ + { type: "message", role: "user", content: "q" }, + { + type: "message", + role: "developer", + content: "plain policy text", + internal_chat_message_metadata_passthrough: "not-an-object", + } as never, + ]).mid; + assert.equal(hasUnrecoverableMediaPayload(stringMetadata), true); + + const arrayMetadata = midSystemCore([ + { type: "message", role: "user", content: "q" }, + { + type: "message", + role: "developer", + content: "plain policy text", + internal_chat_message_metadata_passthrough: [METADATA], + } as never, + ]).mid; + assert.equal(hasUnrecoverableMediaPayload(arrayMetadata), true); +}); + +test("#6949: structured content with the harmless keys still pins (content check unchanged)", () => { + const { mid } = midSystemCore([ + { type: "message", role: "user", content: "q" }, + { + type: "message", + id: "msg_example", + role: "developer", + content: [ + { type: "input_text", text: "declared tools:" }, + { type: "tool_addition", name: "web_search" } as never, + ], + internal_chat_message_metadata_passthrough: METADATA, + }, + ]); + assert.equal(hasUnrecoverableMediaPayload(mid), true); +}); + +test("#6949: un-folded item re-emits byte-exact INCLUDING the metadata fields", () => { + const input = [ + { type: "message", role: "user", content: "turn 1" }, + { + type: "message", + id: "msg_example", + role: "developer", + content: [ + { + type: "input_text", + text: "page rules", + }, + ], + internal_chat_message_metadata_passthrough: METADATA, + }, + { type: "message", role: "user", content: "turn 2" }, + ] as ResponsesRequestBody["input"]; + const flat = responsesToCore({ model: "m", input } as ResponsesRequestBody); + const rebuilt = coreToResponses(flat.msgs, flat.customToolCallIds); + const sysDev = rebuilt.filter( + (i) => + (i as { type?: string }).type === "message" && + (i as { role?: string }).role === "developer", + ); + assert.equal(sysDev.length, 1); + assert.deepEqual( + sysDev[0], + input[1], + "verbatim sidecar round-trip keeps every field", + ); +}); + +test("#6949: assignRefs mints a numeric ref (not BLOCKED) for the item", () => { + const { mid } = midSystemCore([ + { type: "message", role: "user", content: "turn 1" }, + { + type: "message", + id: "msg_example", + role: "developer", + content: [ + { + type: "input_text", + text: "page rules", + }, + ], + internal_chat_message_metadata_passthrough: METADATA, + }, + { type: "message", role: "user", content: "turn 2" }, + ]); + const result = assignRefs([mid as CoreMessage], { + existing: emptyRefMap(), + nextIndex: 1, + isProtected: (m) => hasUnrecoverableMediaPayload(m), + }); + const ref = result.map.byRaw[mid.id]; + assert.ok(ref, "message must carry a ref"); + assert.notEqual(ref, BLOCKED_REF, "no longer BLOCKED"); + assert.match(ref, /^m\d{5}$/, "numeric mNNNNN ref"); +}); + +// Acceptance follow-up (field audit on Codex CLI 0.169.0, #6949 thread): +// the three CLI plain-text classes that were observed pinned — host skill +// instructions, deferred tool namespaces, turn-aborted notices — all carry +// the same shape (valid id + metadata object + input_text parts) and must +// fold. The predicate is shape-based, so one row per class pins the real +// metadata variants. +test("#6949 acceptance: CLI's three plain-text classes fold (skills / deferred namespaces / turn-aborted)", () => { + const classes = [ + { + kind: "additional_content.host_skills.instructions", + text: "host skill instructions: before entering a review loop, read its SKILL.md and follow the loop steps", + }, + { + kind: "additional_content.tools.deferred_namespaces", + text: "deferred tool namespaces: mcp__search, mcp__files (loaded on demand)", + }, + { + kind: "additional_content.generic.turn_aborted", + text: "the previous turn was aborted by the user before a final answer was produced", + }, + ]; + for (const { kind, text } of classes) { + const { mid } = midSystemCore([ + { type: "message", role: "user", content: "q" }, + { + type: "message", + id: `msg_cli_${kind}`, + role: "developer", + content: [{ type: "input_text", text }], + internal_chat_message_metadata_passthrough: { + turn_id: "turn_cli", + content_item_kinds: [kind], + content_item_metadata: [{ provenance: { type: "harness", hook: false } }], + }, + }, + ]); + assert.equal(hasUnrecoverableMediaPayload(mid), false, `${kind} must fold`); + } +}); + +// Acceptance follow-up (field audit, #6949 thread): sessions that ALREADY +// carry a stale BLOCKED ref for one of these items must recover on the next +// ref-assignment pass after the upgrade — no new session required. This pins +// the #2607 stale-BLOCKED migration on the exact shape that produced it. +test("#6949 acceptance: pre-existing BLOCKED ref migrates to a numeric ref on re-assignment", () => { + const flat = responsesToCore({ + model: "m", + input: [ + { type: "message", role: "user", content: "turn 1" }, + { + type: "message", + id: "msg_example", + role: "developer", + content: [ + { + type: "input_text", + text: "page rules", + }, + ], + internal_chat_message_metadata_passthrough: METADATA, + }, + { type: "message", role: "user", content: "turn 2" }, + ], + } as ResponsesRequestBody); + const [u1, dev, u2] = flat.msgs; + assert.equal(dev?.role, "system"); + // Pre-fix session state: neighbors hold numeric refs, the developer item + // was pinned BLOCKED. + const existing = { + byRaw: { [u1!.id]: "m00001", [dev!.id]: BLOCKED_REF, [u2!.id]: "m00002" }, + byRef: { m00001: u1!.id, m00002: u2!.id }, + }; + const result = assignRefs(flat.msgs as CoreMessage[], { + existing, + nextIndex: 3, + isProtected: (m) => hasUnrecoverableMediaPayload(m), + }); + assert.deepEqual(result.migratedIds, [dev!.id]); + assert.equal(result.map.byRaw[dev!.id], "m00003", "stale BLOCKED mints a fresh numeric ref"); + assert.equal(result.map.byRef["m00003"], dev!.id); + assert.equal(result.map.byRaw[u1!.id], "m00001", "neighbors keep their refs"); + assert.equal(result.map.byRaw[u2!.id], "m00002", "neighbors keep their refs"); +}); diff --git a/package-lock.json b/package-lock.json index dd0c2f32d2..72519f7f77 100644 --- a/package-lock.json +++ b/package-lock.json @@ -51,7 +51,7 @@ }, "kernel": { "name": "acp-kernel", - "version": "0.0.200", + "version": "0.0.201", "dev": true, "license": "MIT", "devDependencies": { diff --git a/src/logger.ts b/src/logger.ts index b54c5ba9be..b861a95fae 100644 --- a/src/logger.ts +++ b/src/logger.ts @@ -208,6 +208,19 @@ export function setLogCapture(fn: ((level: string, msg: string) => void) | null) } let pendingRetentionDrains = 0; +// closeLogger() waiters, invoked when the last replaced stream drains so a +// shutdown can prove EVERY buffered line is on disk (not just the final +// stream's). Cleared on fire so a later close does not re-run stale callbacks. +let drainWaiters: Array<() => void> = []; + +function notifyDrainsSettled(): void { + if (pendingRetentionDrains !== 0) return; + const waiters = drainWaiters; + drainWaiters = []; + for (const w of waiters) { + try { w(); } catch { /* best-effort */ } + } +} function openStream(file: string, deferRetention = false): WriteStream { mkdirSync(path.dirname(file), { recursive: true }); @@ -332,6 +345,7 @@ function getStream(): WriteStream | undefined { replaced.removeListener("close", settle); pendingRetentionDrains--; if (pendingRetentionDrains === 0 && replacedPath) applyLogRetention(replacedPath); + notifyDrainsSettled(); }; replaced.once("close", settle); closeQuietly(replaced, settle); @@ -466,23 +480,38 @@ export const log: Logger = (level, msg) => { }; /** Flush + close the log file. Call on shutdown; resolves once every buffered - * line has drained to disk, so awaiting it proves the durable record is complete. */ + * line has drained to disk, so awaiting it proves the durable record is complete. + * That includes lines still in flight on REPLACED streams (rotated out before + * shutdown): each is end()-ed asynchronously, so we wait for the drain counter + * to settle as well as the final stream, or a slow replaced flush could resolve + * us with its tail still unflushed (observed: a rotated record lost on Windows). */ export function closeLogger(): Promise { const s = stream; stream = undefined; streamFd = undefined; writeProven = false; bytesWritten = 0; - if (!s) return Promise.resolve(); return new Promise((resolve) => { let done = false; - const finish = (): void => { + let streamEnded = !s; + const tryFinish = (): void => { if (done) return; + if (!streamEnded || pendingRetentionDrains !== 0) return; done = true; resolve(); }; - s.once("error", finish); - try { s.end(finish); } catch { finish(); } + if (s) { + const onEnd = (): void => { + streamEnded = true; + tryFinish(); + }; + s.once("error", onEnd); + try { s.end(onEnd); } catch { onEnd(); } + } + // settle() re-invokes tryFinish() as each replaced stream drains; the + // trailing call covers the already-settled case (no waiter registered). + if (pendingRetentionDrains > 0) drainWaiters.push(tryFinish); + tryFinish(); }); }