Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
15 commits
Select commit Hold shift + click to select a range
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
26 changes: 26 additions & 0 deletions hermes-plugin/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -356,6 +356,28 @@ def read_registry_entries() -> List[Dict[str, Any]]:
return entries


def _find_spawned_in_registry(child_pid: Optional[int]) -> Optional[Dict[str, Any]]:
"""#4656 / Python twin of #2954 (src/launcher.ts): locate OUR spawned child among the
per-instance registry markers by pid. The single proxy-origin file is last-writer-wins
machine-wide, so a concurrent bring-up registering AFTER our child overwrites it with a
foreign launchToken and starves the shared-pointer match for the whole budget — but the
registry omits the launcher-private token and is keyed per instance, so it cannot be
clobbered that way; the pid WE spawned is the reliable identity proof (a concurrent writer
cannot fake it). If several markers carry that pid — a stale one from a previous run whose
dead pid the OS later reused for our child — the newest registration wins: our child's own
marker always postdates the stale one (the TS twin closes the same corner with a
responder-pid check, src/launcher.ts #2954). The caller still health-probes the returned
origin before trusting it."""
if not isinstance(child_pid, int) or child_pid <= 1:
return None
best: Optional[Dict[str, Any]] = None
for entry in read_registry_entries():
if entry.get("pid") == child_pid and entry.get("origin"):
if best is None or entry.get("started_at", 0) > best.get("started_at", 0):
best = entry
return best


def pid_alive(pid: Optional[int]) -> bool:
if not isinstance(pid, int) or pid <= 1:
return False
Expand Down Expand Up @@ -990,6 +1012,10 @@ def ensure_origin(sidecar: Dict[str, str]) -> Optional[str]:
if inst and inst.get("launch_token") == token and probe_proxy(inst["origin"]):
_state.update(origin=inst["origin"], mode="spawn")
return inst["origin"]
mine = _find_spawned_in_registry(child.pid)
if mine and probe_proxy(mine["origin"]):
_state.update(origin=mine["origin"], mode="spawn", instance_id=mine["instance_id"])
return mine["origin"]
if child.poll() is not None:
logger.warning("billion-context: proxy child exited before becoming healthy "
"(code %s); log: %s", child.returncode,
Expand Down
5 changes: 4 additions & 1 deletion src/dsh-channel.ts
Original file line number Diff line number Diff line change
Expand Up @@ -407,7 +407,10 @@ export function planDshSpawn(
"Set BILI_DSH_BIN to the dsh executable, or reword the argument.",
);
});
const comspec = env.COMSPEC?.trim() || "cmd.exe";
// #4599: sanitized env copies keep original key casing (`ComSpec`) — an
// exact `env.COMSPEC` probe misses it on plain objects.
const comspecRaw = env.COMSPEC ?? env.ComSpec ?? env.comspec;
const comspec = comspecRaw?.trim() || "cmd.exe";
const line = `"${[command, ...args].map(quoteWinToken).join(" ")}"`;
return { command: comspec, args: ["/d", "/s", "/c", line], windowsVerbatimArguments: true };
}
Expand Down
7 changes: 6 additions & 1 deletion src/launcher.ts
Original file line number Diff line number Diff line change
Expand Up @@ -5418,7 +5418,12 @@ export function planClientSpawn(
"Set BILI_CLIENT_BIN to the native executable (e.g. the real codex.exe) or to a script entry run under node, or reword the argument.",
);
});
const comspec = nonEmpty(env.COMSPEC) ? env.COMSPEC : "cmd.exe";
// #4599: Windows env vars are case-insensitive, but sanitized copies
// (spread objects, e.g. stripInheritedProxy's inlineEnv) keep the original
// key casing — `ComSpec` — where an exact `env.COMSPEC` probe misses it
// and silently degrades to the bare-name fallback. Probe every spelling.
const comspecRaw = env.COMSPEC ?? env.ComSpec ?? env.comspec;
const comspec = nonEmpty(comspecRaw) ? comspecRaw : "cmd.exe";
return {
command: comspec,
args: ["/d", "/s", "/c", buildWindowsCommandLine(cmd, args)],
Expand Down
24 changes: 24 additions & 0 deletions src/persist.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1610,6 +1610,30 @@ export class SessionStore {
if (outcome.superseded.size > 0) {
session.restampSuperseded = outcome.superseded;
}
// The token snapshot is keyed by ref (#3488 render-refs): renumbered
// refs would otherwise inherit the stale count of whichever message
// held the new number pre-alignment — the wire tag then disagrees
// with the legacy-restart leg beyond citation renumbering (storage
// engine-proofs wire-parity, #4599). Two-pass moves: stage all
// (oldRef → newRef → count) first, then delete, then write — a
// single-pass swap would lose entries whose newRef is another
// move's oldRef.
const snap = (state as { tokenSnapshot?: Record<string, number> }).tokenSnapshot;
if (snap && outcome.superseded.size > 0) {
const byRawNow = state.messageRefs.byRaw;
const moves: Array<[oldRef: string, newRef: string, count: number]> = [];
for (const [oldRef, rawId] of outcome.superseded) {
const count = snap[oldRef];
const newRef = byRawNow[rawId];
if (count === undefined || newRef === undefined || newRef === oldRef) continue;
moves.push([oldRef, newRef, count]);
}
if (moves.length > 0) {
for (const [oldRef] of moves) delete snap[oldRef];
for (const [, newRef, count] of moves) snap[newRef] = count;
this.log("info", `[persist] remapped ${moves.length} token-snapshot entries onto ledger-aligned refs for ${session.id} (#4599)`);
}
}
state.refLedgerAligned = true;
this.log("info", `[persist] ledger-aligned refs for ${session.id}: kept=${outcome.kept} restamped=${outcome.restamped} rehomed=${outcome.rehomed} blocked=${outcome.blocked} superseded=${outcome.superseded.size} (#4316 PR3/3)`);
this.scheduleSave(session);
Expand Down
11 changes: 7 additions & 4 deletions src/web/blogfeed.ts
Original file line number Diff line number Diff line change
Expand Up @@ -78,14 +78,17 @@ export function _setBlogRootForTest(root: string | null): void {
function parseFrontmatter(text: string): { fm: Record<string, string>; body: string } {
const fm: Record<string, string> = {};
let body = text;
if (text.startsWith("---\n")) {
const end = text.indexOf("\n---\n", 4);
// Git-for-Windows autocrlf checks .md posts out with CRLF; normalize so
// the exact-match fences below survive Windows checkouts (#4599).
const src = text.startsWith("---\r\n") ? text.replace(/\r\n/g, "\n") : text;
if (src.startsWith("---\n")) {
const end = src.indexOf("\n---\n", 4);
if (end !== -1) {
for (const line of text.slice(4, end).split("\n")) {
for (const line of src.slice(4, end).split("\n")) {
const j = line.indexOf(":");
if (j > 0) fm[line.slice(0, j).trim()] = line.slice(j + 1).trim().replace(/^["']|["']$/g, "");
}
body = text.slice(end + 5);
body = src.slice(end + 5);
}
}
return { fm, body };
Expand Down
75 changes: 73 additions & 2 deletions tests/corrupt-index-child.ts
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,7 @@ import type { ProxyOptions } from "../src/config.ts";
import { startServer } from "../src/server.ts";
import { SessionStore, _setStoreForTest } from "../src/persist.ts";
import { loadLegacyView, toLegacyLike } from "../src/storage/ingest.ts";
import { canonicalize, sha256 } from "./golden-canonical.ts";
import { canonicalize, sha256, sortObjectKeys } from "./golden-canonical.ts";

// ---------------------------------------------------------------------------
// Shared with the parent test files (moved verbatim, single definition).
Expand Down Expand Up @@ -81,6 +81,73 @@ export function viewStateDigest(view: ReturnType<typeof loadLegacyView>): string
})));
}

/** #4316 layer 3 PR3/3 (380ef4dac): a unified full-mode cold load performs a
* ONE-TIME ledger alignment — citation numbers are re-homed onto ledger
* sequence (kernel restampRefsToLedger) and state.refLedgerAligned is stamped.
* Cross-mode / cross-read-path "identical state" proofs must therefore compare
* the state modulo that one-time renumbering: this strips the alignment stamp
* and re-homes every ref number to a canonical order (k-th smallest raw id ->
* m0000k), so two views differing ONLY in which numbering scheme produced
* their refs hash identically under viewDigest/viewStateDigest. Object key
* order is normalized too: different hydration paths build identical content
* under different insertion orders, and canonicalize preserves them. Pure
* function — returns a new object, input untouched. */
export function stripOneTimeAlignment<T>(view: T): T {
const v = view as { state?: unknown; blockContents?: unknown };
if (!v.state || typeof v.state !== "object") return view;
const byRaw = (v.state as { messageRefs?: { byRaw?: Record<string, string> } }).messageRefs?.byRaw;
const remap = new Map<string, string>();
if (byRaw) {
for (const [raw, ref] of Object.entries(byRaw).sort((a, b) => (a[0] < b[0] ? -1 : 1))) {
remap.set(ref, `m${String(remap.size + 1).padStart(5, "0")}`);
}
}
// The JSON pass below flattens Maps — pull blockContents out while it is
// still a Map, renumber its keys afterwards, rebuild it. Everything else
// in these views is JSON-native.
const bcIsMap = v.blockContents instanceof Map;
let bcEntries: Array<[string, string]> | null = null;
let source: unknown = view;
if (bcIsMap) {
bcEntries = [...(v.blockContents as Map<string, string>).entries()];
source = { ...(v as Record<string, unknown>), blockContents: undefined };
}
let json = JSON.stringify(canonicalize(source));
if (remap.size > 0) {
// Single-pass alternation: sequential per-token replacement would chain
// (a target that is also another source gets re-mapped on the next pass).
const re = new RegExp(`\\b(${[...remap.keys()].join("|")})\\b`, "g");
json = json.replace(re, (tok) => remap.get(tok)!);
}
const out = JSON.parse(json) as (T & { state?: Record<string, unknown>; blockContents?: unknown }) & Record<string, unknown>;
if (out.state && "refLedgerAligned" in out.state) delete out.state["refLedgerAligned"];
// cacheLedger line `bd` digests hash the RAW outbound bodies — bodies whose
// only cross-leg delta is this same citation renumbering. The bodies are
// compared directly (per-turn shaCiteNorm), so masking the derived digests
// hides nothing; keeping them would fail every aligned comparison.
const ledger = (out.metadata as { cacheLedger?: { lines?: Array<Record<string, unknown>> } } | undefined)?.cacheLedger;
// Segment bookkeeping, not content, in the aligned view (#4599): a restart
// appends a fresh open tail line and re-anchors the closed-segment byte
// boundaries (be/bl/bs), so no-restart and restart legs legitimately differ
// there. Keep the closed segments' content; drop the unfinalized tail and
// the bookkeeping trio everywhere.
if (Array.isArray(ledger?.lines)) {
while (ledger.lines.length > 0 && ledger.lines[ledger.lines.length - 1]?.bl === undefined) ledger.lines.pop();
for (const line of ledger.lines) { delete line["bd"]; delete line["be"]; delete line["bl"]; delete line["bs"]; delete line["prov"]; delete line["seam"]; }
}
// agg is the same experience telemetry summed: a restart re-partitions the
// provider-cache misses into seam buckets (seamMissed/seamSuspects vs
// providerSideMissed), so legs legitimately disagree. Per-line content
// fields stay; only the derived aggregate goes.
if (ledger) delete (ledger as { agg?: unknown }).agg;
// snapshotShadow is the shadow-copier's OWN health telemetry — it reports
// the ingest pipeline's experience of building this view (errors, watermark
// drops), which is inherently load-path-dependent, not conversation state.
if (out.metadata && "snapshotShadow" in (out.metadata as Record<string, unknown>)) delete (out.metadata as Record<string, unknown>)["snapshotShadow"];
if (bcEntries) out.blockContents = new Map(bcEntries.map(([k, val]) => [remap.get(k) ?? k, val]));
return sortObjectKeys(out) as T;
}

export interface StorageWire {
requested: string;
effective: string;
Expand Down Expand Up @@ -277,8 +344,12 @@ async function engproofScenario(argv: string[], run: CheckRun): Promise<void> {
// Regression (#2781): the failed open must have CLOSED its handle.
await runGuard(run, ENGPROOF_CHECK_NAMES.midFlowGuard, dir, "-moved");

// #4316 PR3/3: the clean full-mode read carried the one-time ledger
// alignment; the legacy fallback over the corrupt index did not. The
// parent passes its digest already modulo that alignment (single
// definition), so strip it here too before comparing.
await run.run(ENGPROOF_CHECK_NAMES.stateDigest, () =>
assert.equal(viewStateDigest(toLegacyLike(reloaded)), expectedStateDigest,
assert.equal(viewStateDigest(stripOneTimeAlignment(toLegacyLike(reloaded))), expectedStateDigest,
"corrupt-index fallback state digest drift vs clean full-mode read"));

// Writes must keep working (legacy path) — no throw, file refreshed.
Expand Down
16 changes: 16 additions & 0 deletions tests/golden-canonical.ts
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,22 @@
export { canonicalize, sha256 } from "../src/storage/canonical.ts";
import { canonicalize, sha256 } from "../src/storage/canonical.ts";

/** Deterministic key order for cross-read-path digest comparisons: two views
* built by different hydration paths carry identical content under different
* map insertion orders, and canonicalize preserves insertion order — sorting
* here makes the digests path-independent. Maps/Sets pass through untouched
* (callers that need them iterable keep them intact; canonicalize flattens
* them downstream). */
export function sortObjectKeys(v: unknown): unknown {
if (Array.isArray(v)) return v.map(sortObjectKeys);
if (v && typeof v === "object" && !(v instanceof Map) && !(v instanceof Set)) {
const out: Record<string, unknown> = {};
for (const k of Object.keys(v as Record<string, unknown>).sort()) out[k] = sortObjectKeys((v as Record<string, unknown>)[k]);
return out;
}
return v;
}

/** Canonical JSON → sha256, with a final pass of STRING-level environment
* normalization (ports, session ids embedded inside text fields). */
function digestOf(value: unknown, extra?: { normalize?: (s: string) => string }): string {
Expand Down
46 changes: 34 additions & 12 deletions tests/golden-storage-drive.ts
Original file line number Diff line number Diff line change
Expand Up @@ -16,10 +16,11 @@ import http from "node:http";
import os from "node:os";
import path from "node:path";
import { once } from "node:events";
import { canonicalize, sha256 } from "./golden-canonical.ts";
import { canonicalize, sha256, sortObjectKeys } from "./golden-canonical.ts";
import { defaultConfig } from "acp-kernel";
import { startServer } from "../src/server.ts";
import { SessionStore, _setStoreForTest } from "../src/persist.ts";
import { stripOneTimeAlignment } from "./corrupt-index-child.ts";
import { flushAllSessions, _resetSessionsForTest } from "../src/session.ts";
import type { ProxyOptions } from "../src/config.ts";

Expand Down Expand Up @@ -110,14 +111,15 @@ const GROW_TURNS = 20; // hard cap for the grow loop
const SUMMARY_MARKER = "[Compressed conversation section]";
const EDIT_SENTINEL = "GOLDEN-EDIT-SENTINEL";

interface GoldenTurn { i: number; sha: string; bytes: number }
interface GoldenTurn { i: number; sha: string; bytes: number; shaCiteNorm: string }
interface GoldenRecord {
wire: Wire;
turns: GoldenTurn[];
foldAt: number; // judge-side body index where compress was issued
editedAt: number; // body index after the mid-history edit
summaryFrom: number; // first body index carrying the fold summary
stateDigest: string; // sha256 of the canonicalized persisted session
stateDigestAligned: string; // #4316 PR3/3: same, modulo the one-time ledger alignment
sectionDigests: Record<string, string>; // drift localization (same normalization)
stateKeyDigests: Record<string, string>; // per-top-level-state-key digests
stateShape: { // human-readable facts baked into the snapshot
Expand Down Expand Up @@ -169,6 +171,18 @@ function canonicalBody(body: string): string {
.replace(/msg-fix-[0-9a-f]+/g, "msg-fix-<MINT>");
}

// #4316 PR3/3 (380ef4dac): a full-mode cold load re-homes citation numbers
// once (one-time ledger alignment) — that renumbering is the ONLY permitted
// wire delta vs the legacy restart. Wire-parity proofs therefore pin a second
// digest per turn with every citation number masked to <REF>: equality says
// "only the citation numbers may differ", everything else stays byte-exact.
// All m\d{5} tokens are masked (not just tag payloads) because summaries cite
// numbers in prose too; user content is identical across legs, so the wider
// mask cannot hide a real divergence.
function citeNormBody(body: string): string {
return body.replace(/\bm\d{5}\b/g, "<REF>");
}

/** Timestamps (ms-epoch magnitude) and UUIDs become placeholders; everything
* else — refs, block ids, coverage sets, stats — must be byte-stable. */
// canonicalize/sha256 live in ./golden-canonical.ts (shared with the storage-corpus pins).
Expand Down Expand Up @@ -585,15 +599,7 @@ export async function driveWire(wire: Wire, sessionId = `golden-${wire}`, opts:
// order, so a harmless field-reorder commit (#3093) red all four
// goldens. Deep-sort object keys harness-side only; array order is
// preserved (foldAnchorOrder & co are ordered contracts).
const sortObjectKeys = (v: unknown): unknown => {
if (Array.isArray(v)) return v.map(sortObjectKeys);
if (v && typeof v === "object") {
const o: Record<string, unknown> = {};
for (const k of Object.keys(v as Record<string, unknown>).sort()) o[k] = sortObjectKeys((v as Record<string, unknown>)[k]);
return o;
}
return v;
};

const lastView = JSON.stringify(canonicalize(sortObjectKeys({
state: session.state,
blockContents: [...session.blockContents.entries()],
Expand All @@ -606,6 +612,18 @@ export async function driveWire(wire: Wire, sessionId = `golden-${wire}`, opts:
.replace(/127\.0\.0\.1:\d+/g, "127.0.0.1:<PORT>")
.split(JSON.stringify(sessionId).slice(1, -1)).join("<SID>");
const stateDigest = sha256(lastView);
// #4316 PR3/3: parallel digest modulo the one-time ledger alignment —
// cross-mode proofs compare states that may differ only in ref numbering.
const stateDigestAligned = sha256(
JSON.stringify(canonicalize(sortObjectKeys(stripOneTimeAlignment({
state: session.state,
blockContents: [...session.blockContents.entries()],
stats: session.stats,
metadata: session.metadata,
}))))
.replace(/127\.0\.0\.1:\d+/g, "127.0.0\.1:<PORT>")
.split(JSON.stringify(sessionId).slice(1, -1)).join("<SID>"),
);
// Localization aid for cross-machine digest drift: per-section and
// per-top-level-state-key digests under the SAME normalization, so a
// CI failure message pinpoints the divergent subtree without needing
Expand All @@ -629,11 +647,15 @@ export async function driveWire(wire: Wire, sessionId = `golden-${wire}`, opts:
const active = st.blocks?.filter((b) => b.active) ?? [];
return {
wire,
turns: bodies.map((b, i) => ({ i, sha: sha256(canonicalBody(b)), bytes: Buffer.byteLength(b, "utf8") })),
turns: bodies.map((b, i) => {
const cb = canonicalBody(b);
return { i, sha: sha256(cb), bytes: Buffer.byteLength(b, "utf8"), shaCiteNorm: sha256(canonicalBody(citeNormBody(b))) };
}),
foldAt,
editedAt,
summaryFrom: bodies.findIndex((b) => b.includes(SUMMARY_MARKER)),
stateDigest,
stateDigestAligned,
sectionDigests,
stateKeyDigests,
stateShape: {
Expand Down
Loading
Loading