Skip to content
Open
12 changes: 5 additions & 7 deletions src/session.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1889,13 +1889,11 @@ export function preCompactionArchiveOf(session: Session): PreCompactionArchive {
// byRaw/byRef are pruned to liveRawIds (stops the #390 additive leak).
// nextIndex is left alone so a freed ref slot is never re-allocated onto a
// retained tail's live tag.
/** Synthetic byRaw key pinning a session's ref high-water mark. Its value is
* the highest ref this session must never re-allocate (refs the parent used
* before a fork/resume, or numbers freed by an archive prune); the kernel
* cursor (highestUsedIndex+1) then starts above it. Never a real message id
* (those are SHA-256 hex), so it matches no incoming message and has no
* byRef entry. */
export const REF_FLOOR_RAW_ID = "bili:ref-floor";
// #5589: defined in state-hydration.js so the storage rebuild seam can skip
// the floor row without importing the session layer; re-exported for the
// existing test importers.
export { REF_FLOOR_RAW_ID } from "./state-hydration.js";
import { REF_FLOOR_RAW_ID } from "./state-hydration.js";

/** Raise `session`'s ref floor to at least `index` (no-op when its map
* already reaches that high). */
Expand Down
12 changes: 12 additions & 0 deletions src/state-hydration.ts
Original file line number Diff line number Diff line change
Expand Up @@ -109,3 +109,15 @@ export const COMPRESSION_STATE_FIELDS: readonly (keyof CompressionState)[] = (()
probe.timestampsHighWaterRefSeq = 0; // conditionally-spread; undefined on a fresh state
return Object.keys(mergeState(probe)) as (keyof CompressionState)[];
})();

/** Synthetic byRaw key pinning a session's ref high-water mark (#2592/F5). Its
* value is the highest ref this session must never re-allocate (refs the
* parent used before a fork/resume, or numbers freed by an archive prune);
* the kernel cursor (highestUsedIndex+1) then starts above it. Never a real
* message id (those are SHA-256 hex), so it matches no incoming message and
* has NO byRef entry — consumers that invert the ref ledger back into byRef
* must skip it exactly like "BLOCKED" (#5589: inverting the floor row minted
* a phantom entry the live map never has → permanent DIGEST DRIFT). Lives
* here, not in session.ts, so the storage rebuild seam can consume it without
* importing the session layer. */
export const REF_FLOOR_RAW_ID = "bili:ref-floor";
20 changes: 13 additions & 7 deletions src/storage/ingest.ts
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@ import { splitMediaContent, identityOfStoredContent } from "./identity.js";
* (src/state-hydration.ts), not hand-kept: the hand list had already drifted
* (#4192). The loader rebuilds the state object in exactly this order.
*/
import { COMPRESSION_STATE_FIELDS as MERGE_STATE_FIELDS } from "../state-hydration.js";
import { COMPRESSION_STATE_FIELDS as MERGE_STATE_FIELDS, REF_FLOOR_RAW_ID } from "../state-hydration.js";

/** Fields owned by dedicated tables/columns — never inside kstate_json. */
const TABLE_OWNED_FIELDS = new Set(["blocks", "messageRefs", "deadRefs", "nextBlockId", "nextRunId"]);
Expand Down Expand Up @@ -482,8 +482,11 @@ function finishPlan(

// #4216/#4242: the orphan journal — blockContents entries whose block is
// gone from state.blocks (#2695 reap keeps the contents for decompress).
// Rides every plan; the store merges it into sessions.orphans_json in the
// same transaction as the rest of the session. This is what makes the
// Rides every plan; the store records it authoritatively in
// sessions.orphans_json — REPLACE semantics (#5589-C): each save stores
// the live set verbatim, so an authoritative clear resets the journal
// (the pre-fix merge kept ghosts forever and froze the envelope). Same
// transaction as the rest of the session. This is what makes the
// reap shape REPRESENTABLE, so record() no longer skips the ingest (the
// freeze in #4216: sessions stuck at hours-old rows).
const knownBlockIds = new Set(session.state.blocks.map((b) => b.blockId));
Expand Down Expand Up @@ -664,11 +667,14 @@ export class LegacyViewBuilder {
const byRef: Record<string, string> = {};
for (const r of refs) {
byRaw[r.rawId] = r.ref;
// "BLOCKED" is a byRaw-only token: the in-memory map keeps the
// inverse projection strictly mNNNNN→rawId. Admitting it into byRef
// "BLOCKED" and the ref floor (#2592/F5, REF_FLOOR_RAW_ID) are
// byRaw-only tokens: the in-memory map keeps the inverse
// projection strictly mNNNNN→rawId. Admitting either into byRef
// mints a key the persisted state never had (#2671 soak: sub-agent
// sessions with protected messages drifted on every save).
if (r.ref !== "BLOCKED" && byRef[r.ref] === undefined) byRef[r.ref] = r.rawId;
// sessions with protected messages drifted on every save; #5589:
// the floor row drifted every Codex native-compaction session on
// every sampled verify → watermark drop + envelope freeze).
if (r.ref !== "BLOCKED" && r.rawId !== REF_FLOOR_RAW_ID && byRef[r.ref] === undefined) byRef[r.ref] = r.rawId;
}

const state = this.state;
Expand Down
8 changes: 5 additions & 3 deletions src/storage/mysql.ts
Original file line number Diff line number Diff line change
Expand Up @@ -722,8 +722,8 @@ export class MysqlBackend implements StorageBackend {
await c.query(
`INSERT INTO sessions (session_id, parent_session_id, fork_point_seq, protocol, label, title, host_title, namespace, origin,
status, created_at, updated_at, next_block_id, next_run_id,
meta_json, stats_json, metadata_json, kstate_json, tail_json, ccr_json, last_ingested_seq)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
meta_json, stats_json, metadata_json, kstate_json, tail_json, ccr_json, last_ingested_seq, orphans_json)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
ON DUPLICATE KEY UPDATE
fork_point_seq = IF(parent_session_id IS NULL, VALUES(fork_point_seq), fork_point_seq),
parent_session_id = COALESCE(parent_session_id, VALUES(parent_session_id)),
Expand All @@ -743,7 +743,8 @@ export class MysqlBackend implements StorageBackend {
kstate_json = VALUES(kstate_json),
tail_json = VALUES(tail_json),
ccr_json = VALUES(ccr_json),
last_ingested_seq = VALUES(last_ingested_seq)`,
last_ingested_seq = VALUES(last_ingested_seq),
orphans_json = VALUES(orphans_json)`,
[
s.sessionId,
s.parentSessionId ?? null,
Expand All @@ -766,6 +767,7 @@ export class MysqlBackend implements StorageBackend {
s.tailJson,
s.ccrJson ?? null,
watermark,
s.orphansJson ?? null,
],
);
await c.query(
Expand Down
50 changes: 44 additions & 6 deletions src/storage/shadow.ts
Original file line number Diff line number Diff line change
Expand Up @@ -464,7 +464,8 @@ export class UnifiedShadow {
// #4216/#4242: the reap shape (blockContents without their
// state.blocks entries) is now REPRESENTABLE — planIngest
// journals the orphan contents into sessions.orphans_json and
// the store merges it in the same transaction, while the blocks
// the store records it authoritatively in the same transaction
// (#5589-C: replacement, not merge), while the blocks
// stale-sweep drops the reaped rows. The old early-return here
// froze every subsequent save for such sessions (rows stuck at
// hours-old state, #4216). Normal ingest for everyone.
Expand Down Expand Up @@ -1534,7 +1535,7 @@ interface DriftDiagnostics {
* converges the tables it owns; the sub-key names the actual residue. */
stateKeys?: string[];
/** First divergent lastMessages slot (or the length-mismatch index). */
tailSlot?: { index: number; expect: DriftSlotFingerprint; got: DriftSlotFingerprint; /** #5677: top-level field names whose values disagree (names only). */ fields?: string[] };
tailSlot?: { index: number; expect: DriftSlotFingerprint; got: DriftSlotFingerprint; /** #5677: top-level field names whose values disagree (names only). */ fields?: string[]; /** #5589: for object-typed divergent fields, the inner keys that disagree (wire schema names only, capped). */ nestedFields?: Record<string, string[]> };
}

/** Compare the replay by component, retaining only message fingerprints across batches. */
Expand Down Expand Up @@ -1657,9 +1658,10 @@ class BatchedViewVerification {
this.changed(key);
if (key === "lastMessages" && expected && !this.tailSlot) {
const fields = diffFieldNames(expected[wantIndex], rec.raw);
const nestedFields = diffNestedFieldKeys(expected[wantIndex], rec.raw, fields);
this.tailSlot = fields === undefined
? { index, expect: want.slot, got: rec.sig.slot }
: { index, expect: want.slot, got: rec.sig.slot, fields };
: { index, expect: want.slot, got: rec.sig.slot, fields, ...(nestedFields ? { nestedFields } : {}) };
}
}
index++;
Expand Down Expand Up @@ -1742,6 +1744,39 @@ function diffFieldNames(a: unknown, b: unknown): string[] | undefined {
return out.length > 0 ? out : undefined;
}

/** #5589: one level deeper than diffFieldNames — for each divergent top-level
* field whose value is a PLAIN OBJECT on both legs, name the inner keys whose
* canonicalized values disagree. Wire schema names only — values never leave
* this function (same privacy contract as the #5677 field names). Bounded:
* at most MAX_DRIFT_NESTED_KEYS inner keys per field, sorted. Fields absent on
* one leg are NOT drilled (the top-level name already says "the sidecar
* appeared/disappeared"); arrays are not drilled (index-level diffs are a
* different shape). undefined when no field qualifies. */
const MAX_DRIFT_NESTED_KEYS = 5;
function diffNestedFieldKeys(a: unknown, b: unknown, fields: string[] | undefined): Record<string, string[]> | undefined {
if (!fields || !a || typeof a !== "object" || Array.isArray(a) || !b || typeof b !== "object" || Array.isArray(b)) return undefined;
const ao = a as Record<string, unknown>;
const bo = b as Record<string, unknown>;
const ser = (v: unknown): string | null => (v === undefined ? null : JSON.stringify(canonicalize(v)));
const out: Record<string, string[]> = {};
for (const f of fields) {
const av = ao[f];
const bv = bo[f];
if (av === null || typeof av !== "object" || Array.isArray(av) || bv === null || typeof bv !== "object" || Array.isArray(bv)) continue;
const aobj = av as Record<string, unknown>;
const bobj = bv as Record<string, unknown>;
const inner: string[] = [];
for (const k of [...new Set([...Object.keys(aobj), ...Object.keys(bobj)])].sort()) {
if (ser(aobj[k]) !== ser(bobj[k])) {
inner.push(k);
if (inner.length >= MAX_DRIFT_NESTED_KEYS) break;
}
}
if (inner.length > 0) out[f] = inner;
}
return Object.keys(out).length > 0 ? out : undefined;
}

// #3919: which dimensions disagree when the sampled verify detects drift —
// the digest is a single hash and leaves classification to guesswork. Pure;
// runs only on detected drift (rare). Fingerprints carry id/role/length/
Expand Down Expand Up @@ -1794,9 +1829,10 @@ export function describeViewDivergence(expectView: unknown, gotView: unknown): D
const ea = index < et.length ? et[index] : null;
const ga = index < gt.length ? gt[index] : null;
const fields = diffFieldNames(ea, ga);
const nestedFields = diffNestedFieldKeys(ea, ga, fields);
diagnostics.tailSlot = fields === undefined
? { index, expect: fp(ea), got: fp(ga) }
: { index, expect: fp(ea), got: fp(ga), fields };
: { index, expect: fp(ea), got: fp(ga), fields, ...(nestedFields ? { nestedFields } : {}) };
}
}
return diagnostics;
Expand All @@ -1807,8 +1843,10 @@ export function formatDriftDiagnostics(d: DriftDiagnostics): string {
const parts = [`diverged: ${d.keys.length > 0 ? d.keys.join(",") : "<unknown>"}`];
if (d.stateKeys) parts.push(`state: ${d.stateKeys.join(",")}`);
if (d.tailSlot) {
const { index, expect: ex, got: go, fields } = d.tailSlot;
parts.push(`tail slot ${index}: expect ${ex.id}/${ex.role} len=${ex.length} sha=${ex.sha8}, got ${go.id}/${go.role} len=${go.length} sha=${go.sha8}${fields ? `; fields: ${fields.join(",")}` : ""}`);
const { index, expect: ex, got: go, fields, nestedFields } = d.tailSlot;
let suffix = fields ? `; fields: ${fields.join(",")}` : "";
if (nestedFields) suffix += `; ${Object.entries(nestedFields).map(([f, ks]) => `${f}: ${ks.join(",")}`).join("; ")}`;
parts.push(`tail slot ${index}: expect ${ex.id}/${ex.role} len=${ex.length} sha=${ex.sha8}, got ${go.id}/${go.role} len=${go.length} sha=${go.sha8}${suffix}`);
}
return " [" + parts.join("; ") + "]";
}
58 changes: 23 additions & 35 deletions src/storage/store.ts
Original file line number Diff line number Diff line change
Expand Up @@ -347,9 +347,10 @@ export interface IngestSessionInput {
/** Serialized orphan map (Record<blockId, {one, full}>) — blockContents
* entries whose block is absent from state.blocks (#4216/#4242: the reap
* shape is REPRESENTABLE; rides the same transaction so the session row,
* blocks and tail keep advancing). MERGE semantics on write: existing
* journal entries survive, because LazyBlockContents means a live
* session's in-memory map may not carry entries trimmed earlier. */
* blocks and tail keep advancing). REPLACE semantics on write (#5589-C):
* finishPlan iterates the full live map (LazyBlockContents re-materializes
* on iteration), so this set is authoritative — an empty set resets the
* journal after an authoritative clear (native-compaction rebase). */
orphansJson?: string | null;
}

Expand Down Expand Up @@ -872,38 +873,26 @@ export class UnifiedStore {
}

/** refcount-aware release; the CAS body itself never leaves. */
/** #4242: merge the incoming orphan map over the journal's current
* contents. Incoming entries win on key collision (same id, refreshed
* view); existing entries the live session no longer carries (trimmed
* from memory by LazyBlockContents) survive. */
private mergeOrphansJson(sessionId: string, incomingJson: string | null): string | null {
if (incomingJson === null) {
// No orphans in this save — keep whatever the journal already
// holds (a previous reap's trimmed-away contents must survive).
const row = this.db
.prepare("SELECT orphans_json FROM sessions WHERE session_id = ?")
.get(sessionId) as { orphans_json: string | null } | undefined;
return row?.orphans_json ?? null;
}
let merged: Record<string, unknown> = {};
try {
const existing = this.db
.prepare("SELECT orphans_json FROM sessions WHERE session_id = ?")
.get(sessionId) as { orphans_json: string | null } | undefined;
if (existing?.orphans_json) {
const parsed = JSON.parse(existing.orphans_json) as Record<string, unknown>;
if (parsed && typeof parsed === "object") merged = parsed;
}
} catch {
// corrupt journal → start fresh (loadLegacyView skips bad entries too)
}
/** #4242/#5589-C: the plan's orphan set is AUTHORITATIVE — finishPlan
* iterates the full live map (LazyBlockContents re-materializes on
* iteration), so the journal REPLACES instead of merging. Merge
* semantics latched pre-clear ghosts forever: after a native-compaction
* rebase (resetSessionCompression clears blockContents) the live side
* can never carry the old ids again, yet every sampled verify flagged
* them ([diverged: blockContents]) until escalation froze the envelope
* with no save path able to clear the journal (field evidence S6,
* v0.1.212 Codex Desktop). */
private writeOrphansJson(incomingJson: string | null): string | null {
if (incomingJson === null) return null;
try {
const incoming = JSON.parse(incomingJson) as Record<string, unknown>;
if (incoming && typeof incoming === "object") merged = { ...merged, ...incoming };
if (incoming && typeof incoming === "object" && !Array.isArray(incoming)) {
return Object.keys(incoming).length > 0 ? JSON.stringify(incoming) : null;
}
} catch {
// corrupt incoming → keep existing only
// corrupt authoritative input → treat as an empty set
}
return Object.keys(merged).length > 0 ? JSON.stringify(merged) : null;
return null;
}

private releaseContent(hash: string | null): void {
Expand Down Expand Up @@ -1148,10 +1137,9 @@ export class UnifiedStore {
tailJson: s.tailJson,
ccrJson: s.ccrJson,
lastIngestedSeq: watermark,
// #4216/#4242: orphan journal rides the normal upsert with
// MERGE semantics (existing entries survive — LazyBlockContents
// means the live map may not carry trimmed entries).
orphansJson: this.mergeOrphansJson(s.sessionId, s.orphansJson ?? null),
// #4216/#4242/#5589-C: the plan carries the exact current
// orphan set; the upsert replaces the journal wholesale.
orphansJson: this.writeOrphansJson(s.orphansJson ?? null),
});

// Identity index (#2960 R2.1): claim (namespace, conversation_id)
Expand Down
Loading
Loading