Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 4 additions & 1 deletion src/dsh-channel.ts
Original file line number Diff line number Diff line change
Expand Up @@ -407,7 +407,10 @@ export function planDshSpawn(
"Set BILI_DSH_BIN to the dsh executable, or reword the argument.",
);
});
const comspec = env.COMSPEC?.trim() || "cmd.exe";
// #4599: sanitized env copies keep original key casing (`ComSpec`) — an
// exact `env.COMSPEC` probe misses it on plain objects.
const comspecRaw = env.COMSPEC ?? env.ComSpec ?? env.comspec;
const comspec = comspecRaw?.trim() || "cmd.exe";
const line = `"${[command, ...args].map(quoteWinToken).join(" ")}"`;
return { command: comspec, args: ["/d", "/s", "/c", line], windowsVerbatimArguments: true };
}
Expand Down
7 changes: 6 additions & 1 deletion src/launcher.ts
Original file line number Diff line number Diff line change
Expand Up @@ -5418,7 +5418,12 @@ export function planClientSpawn(
"Set BILI_CLIENT_BIN to the native executable (e.g. the real codex.exe) or to a script entry run under node, or reword the argument.",
);
});
const comspec = nonEmpty(env.COMSPEC) ? env.COMSPEC : "cmd.exe";
// #4599: Windows env vars are case-insensitive, but sanitized copies
// (spread objects, e.g. stripInheritedProxy's inlineEnv) keep the original
// key casing — `ComSpec` — where an exact `env.COMSPEC` probe misses it
// and silently degrades to the bare-name fallback. Probe every spelling.
const comspecRaw = env.COMSPEC ?? env.ComSpec ?? env.comspec;
const comspec = nonEmpty(comspecRaw) ? comspecRaw : "cmd.exe";
return {
command: comspec,
args: ["/d", "/s", "/c", buildWindowsCommandLine(cmd, args)],
Expand Down
35 changes: 30 additions & 5 deletions src/persist.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1610,6 +1610,30 @@ export class SessionStore {
if (outcome.superseded.size > 0) {
session.restampSuperseded = outcome.superseded;
}
// The token snapshot is keyed by ref (#3488 render-refs): renumbered
// refs would otherwise inherit the stale count of whichever message
// held the new number pre-alignment — the wire tag then disagrees
// with the legacy-restart leg beyond citation renumbering (storage
// engine-proofs wire-parity, #4599). Two-pass moves: stage all
// (oldRef → newRef → count) first, then delete, then write — a
// single-pass swap would lose entries whose newRef is another
// move's oldRef.
const snap = (state as { tokenSnapshot?: Record<string, number> }).tokenSnapshot;
if (snap && outcome.superseded.size > 0) {
const byRawNow = state.messageRefs.byRaw;
const moves: Array<[oldRef: string, newRef: string, count: number]> = [];
for (const [oldRef, rawId] of outcome.superseded) {
const count = snap[oldRef];
const newRef = byRawNow[rawId];
if (count === undefined || newRef === undefined || newRef === oldRef) continue;
moves.push([oldRef, newRef, count]);
}
if (moves.length > 0) {
for (const [oldRef] of moves) delete snap[oldRef];
for (const [, newRef, count] of moves) snap[newRef] = count;
this.log("info", `[persist] remapped ${moves.length} token-snapshot entries onto ledger-aligned refs for ${session.id} (#4599)`);
}
}
state.refLedgerAligned = true;
this.log("info", `[persist] ledger-aligned refs for ${session.id}: kept=${outcome.kept} restamped=${outcome.restamped} rehomed=${outcome.rehomed} blocked=${outcome.blocked} superseded=${outcome.superseded.size} (#4316 PR3/3)`);
this.scheduleSave(session);
Expand Down Expand Up @@ -1687,13 +1711,14 @@ export class SessionStore {
}
}

/** #4316 PR3/3: live (non-retired) row count for a session — the fork
* endpoint passes count - 1 as the inclusive throughSeq. -1 when the
* unified store is unavailable (kernel-only fork). */
unifiedLiveRowCount(id: string): number {
/** #4316 PR3/3 + #4730: highest live (non-retired) row seq for a session —
* the fork endpoint passes it directly as the inclusive throughSeq. -1 when
* the unified store is unavailable or the parent has no live rows
* (kernel-only fork). */
unifiedLastLiveSeq(id: string): number {
if (!this.shadow) return -1;
try {
return this.shadow.liveRowCount(id);
return this.shadow.lastLiveSeq(id);
} catch {
return -1;
}
Expand Down
9 changes: 7 additions & 2 deletions src/plugin.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1534,8 +1534,13 @@ export async function handlePluginFork(payload: string, res: ServerResponse): Pr
// fail for a storage nicety.
try {
const store = getStore();
const throughSeq = store.unifiedLiveRowCount(parent.id);
if (throughSeq > 0 && !store.unifiedCloneSessionRows(parent.id, child.id, throughSeq - 1)) {
// #4730: cut at the parent's HIGHEST live row seq, not its live
// row COUNT — seq is append-only, so a retired row makes COUNT
// undershoot the newest position and the child silently loses
// tail messages. unifiedLastLiveSeq returns -1 when there is
// nothing to clone (kernel-only fork).
const lastLiveSeq = store.unifiedLastLiveSeq(parent.id);
if (lastLiveSeq >= 0 && !store.unifiedCloneSessionRows(parent.id, child.id, lastLiveSeq)) {
loggerLog("debug", `fork ${request.childConversationId}: no unified rows to clone for ${parent.id} (kernel-only fork)`);
}
} catch (err) {
Expand Down
8 changes: 4 additions & 4 deletions src/storage/backend.ts
Original file line number Diff line number Diff line change
Expand Up @@ -67,8 +67,8 @@ export interface StorageBackend {

// --- reads ------------------------------------------------------------
getSessionRow(sessionId: string): Promise<SessionRow | null>;
/** #4316 PR3/3: live (non-retired) row count. */
liveRowCount(sessionId: string): Promise<number>;
/** #4316 PR3/3 + #4730: highest live (non-retired) row seq (-1 when none). */
lastLiveSeq(sessionId: string): Promise<number>;
/** #4316 PR3/3: raw id → canonical seq (single-row ids only). */
seqByRaw(sessionId: string): Promise<Map<string, number> | null>;
/** #4316 PR3/3: fork adoption row clone (idempotent). */
Expand Down Expand Up @@ -176,8 +176,8 @@ export class SqliteBackend implements StorageBackend {
getSessionRow(sessionId: string): Promise<SessionRow | null> {
return Promise.resolve(this.store.getSessionRow(sessionId));
}
liveRowCount(sessionId: string): Promise<number> {
return Promise.resolve(this.store.liveRowCount(sessionId));
lastLiveSeq(sessionId: string): Promise<number> {
return Promise.resolve(this.store.lastLiveSeq(sessionId));
}
seqByRaw(sessionId: string): Promise<Map<string, number> | null> {
return Promise.resolve(this.store.seqByRaw(sessionId));
Expand Down
10 changes: 6 additions & 4 deletions src/storage/mysql.ts
Original file line number Diff line number Diff line change
Expand Up @@ -896,13 +896,15 @@ export class MysqlBackend implements StorageBackend {
}));
}

/** #4316 PR3/3 — MySQL twin of UnifiedStore.liveRowCount. */
async liveRowCount(sessionId: string): Promise<number> {
/** #4316 PR3/3 + #4730 — MySQL twin of UnifiedStore.lastLiveSeq (MAX(seq)
* of live rows, -1 when none). */
async lastLiveSeq(sessionId: string): Promise<number> {
const [rows] = (await this.pool.query(
"SELECT COUNT(*) AS n FROM messages WHERE session_id = ? AND retired_at IS NULL",
"SELECT MAX(seq) AS m FROM messages WHERE session_id = ? AND retired_at IS NULL",
[sessionId],
)) as unknown as Row[][];
return num(rows[0].n);
const m = rows[0]?.m;
return typeof m === "number" && Number.isFinite(m) ? m : -1;
}

/** #4316 PR3/3 — MySQL twin of UnifiedStore.seqByRaw. */
Expand Down
6 changes: 3 additions & 3 deletions src/storage/shadow.ts
Original file line number Diff line number Diff line change
Expand Up @@ -788,10 +788,10 @@ export class UnifiedShadow {
return this.sync.seqByRaw(sessionId);
}

/** #4316 PR3/3: live (non-retired) message row count. */
liveRowCount(sessionId: string): number {
/** #4316 PR3/3 + #4730: highest live (non-retired) message row seq (-1 when none). */
lastLiveSeq(sessionId: string): number {
this.assertLive();
if (this.sync !== null) return this.sync.liveRowCount(sessionId);
if (this.sync !== null) return this.sync.lastLiveSeq(sessionId);
return -1;
}

Expand Down
15 changes: 9 additions & 6 deletions src/storage/store.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2176,13 +2176,16 @@ export class UnifiedStore {
* edit left both bytes in the log) has no single "the" seq — picking one
* would silently re-bind the citation to the wrong generation. Those
* stay on their legacy refs (kept honestly instead of re-bound wrongly). */
/** #4316 PR3/3: live (non-retired) row count — fork endpoint's
* throughSeq source (count - 1, inclusive). */
liveRowCount(sessionId: string): number {
/** #4316 PR3/3 + #4730: highest live (non-retired) row seq for a session —
* the fork endpoint's inclusive throughSeq source. seq is append-only
* (remap/delta anchor at MAX(seq)+1), so a COUNT of live rows understates
* the newest position once any row is retired; MAX(seq) is the true cut.
* -1 when the session has no live rows. */
lastLiveSeq(sessionId: string): number {
const row = this.db
.prepare("SELECT COUNT(*) AS n FROM messages WHERE session_id = ? AND retired_at IS NULL")
.prepare("SELECT MAX(seq) AS m FROM messages WHERE session_id = ? AND retired_at IS NULL")
.get(sessionId) as Record<string, unknown>;
return (row.n as number) ?? 0;
return (row.m as number | null) ?? -1;
}

seqByRaw(sessionId: string): Map<string, number> {
Expand All @@ -2204,7 +2207,7 @@ export class UnifiedStore {
* session row records the lineage; content/media CAS refcounts are
* bumped per copied row. Idempotent: replaying the same clone is a no-op.
* throughSeq is inclusive (store seq is 0-based; the endpoint passes the
* parent's live row count - 1). */
* parent's highest live row seq, #4730). */
cloneSessionRows(parentId: string, childId: string, throughSeq: number): void {
this.db.transaction(() => {
const parent = this.getSessionRow(parentId);
Expand Down
11 changes: 7 additions & 4 deletions src/web/blogfeed.ts
Original file line number Diff line number Diff line change
Expand Up @@ -78,14 +78,17 @@ export function _setBlogRootForTest(root: string | null): void {
function parseFrontmatter(text: string): { fm: Record<string, string>; body: string } {
const fm: Record<string, string> = {};
let body = text;
if (text.startsWith("---\n")) {
const end = text.indexOf("\n---\n", 4);
// Git-for-Windows autocrlf checks .md posts out with CRLF; normalize so
// the exact-match fences below survive Windows checkouts (#4599).
const src = text.startsWith("---\r\n") ? text.replace(/\r\n/g, "\n") : text;
if (src.startsWith("---\n")) {
const end = src.indexOf("\n---\n", 4);
if (end !== -1) {
for (const line of text.slice(4, end).split("\n")) {
for (const line of src.slice(4, end).split("\n")) {
const j = line.indexOf(":");
if (j > 0) fm[line.slice(0, j).trim()] = line.slice(j + 1).trim().replace(/^["']|["']$/g, "");
}
body = text.slice(end + 5);
body = src.slice(end + 5);
}
}
return { fm, body };
Expand Down
75 changes: 73 additions & 2 deletions tests/corrupt-index-child.ts
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,7 @@ import type { ProxyOptions } from "../src/config.ts";
import { startServer } from "../src/server.ts";
import { SessionStore, _setStoreForTest } from "../src/persist.ts";
import { loadLegacyView, toLegacyLike } from "../src/storage/ingest.ts";
import { canonicalize, sha256 } from "./golden-canonical.ts";
import { canonicalize, sha256, sortObjectKeys } from "./golden-canonical.ts";

// ---------------------------------------------------------------------------
// Shared with the parent test files (moved verbatim, single definition).
Expand Down Expand Up @@ -81,6 +81,73 @@ export function viewStateDigest(view: ReturnType<typeof loadLegacyView>): string
})));
}

/** #4316 layer 3 PR3/3 (380ef4dac): a unified full-mode cold load performs a
* ONE-TIME ledger alignment — citation numbers are re-homed onto ledger
* sequence (kernel restampRefsToLedger) and state.refLedgerAligned is stamped.
* Cross-mode / cross-read-path "identical state" proofs must therefore compare
* the state modulo that one-time renumbering: this strips the alignment stamp
* and re-homes every ref number to a canonical order (k-th smallest raw id ->
* m0000k), so two views differing ONLY in which numbering scheme produced
* their refs hash identically under viewDigest/viewStateDigest. Object key
* order is normalized too: different hydration paths build identical content
* under different insertion orders, and canonicalize preserves them. Pure
* function — returns a new object, input untouched. */
export function stripOneTimeAlignment<T>(view: T): T {
const v = view as { state?: unknown; blockContents?: unknown };
if (!v.state || typeof v.state !== "object") return view;
const byRaw = (v.state as { messageRefs?: { byRaw?: Record<string, string> } }).messageRefs?.byRaw;
const remap = new Map<string, string>();
if (byRaw) {
for (const [raw, ref] of Object.entries(byRaw).sort((a, b) => (a[0] < b[0] ? -1 : 1))) {
remap.set(ref, `m${String(remap.size + 1).padStart(5, "0")}`);
}
}
// The JSON pass below flattens Maps — pull blockContents out while it is
// still a Map, renumber its keys afterwards, rebuild it. Everything else
// in these views is JSON-native.
const bcIsMap = v.blockContents instanceof Map;
let bcEntries: Array<[string, string]> | null = null;
let source: unknown = view;
if (bcIsMap) {
bcEntries = [...(v.blockContents as Map<string, string>).entries()];
source = { ...(v as Record<string, unknown>), blockContents: undefined };
}
let json = JSON.stringify(canonicalize(source));
if (remap.size > 0) {
// Single-pass alternation: sequential per-token replacement would chain
// (a target that is also another source gets re-mapped on the next pass).
const re = new RegExp(`\\b(${[...remap.keys()].join("|")})\\b`, "g");
json = json.replace(re, (tok) => remap.get(tok)!);
}
const out = JSON.parse(json) as (T & { state?: Record<string, unknown>; blockContents?: unknown }) & Record<string, unknown>;
if (out.state && "refLedgerAligned" in out.state) delete out.state["refLedgerAligned"];
// cacheLedger line `bd` digests hash the RAW outbound bodies — bodies whose
// only cross-leg delta is this same citation renumbering. The bodies are
// compared directly (per-turn shaCiteNorm), so masking the derived digests
// hides nothing; keeping them would fail every aligned comparison.
const ledger = (out.metadata as { cacheLedger?: { lines?: Array<Record<string, unknown>> } } | undefined)?.cacheLedger;
// Segment bookkeeping, not content, in the aligned view (#4599): a restart
// appends a fresh open tail line and re-anchors the closed-segment byte
// boundaries (be/bl/bs), so no-restart and restart legs legitimately differ
// there. Keep the closed segments' content; drop the unfinalized tail and
// the bookkeeping trio everywhere.
if (Array.isArray(ledger?.lines)) {
while (ledger.lines.length > 0 && ledger.lines[ledger.lines.length - 1]?.bl === undefined) ledger.lines.pop();
for (const line of ledger.lines) { delete line["bd"]; delete line["be"]; delete line["bl"]; delete line["bs"]; delete line["prov"]; delete line["seam"]; }
}
// agg is the same experience telemetry summed: a restart re-partitions the
// provider-cache misses into seam buckets (seamMissed/seamSuspects vs
// providerSideMissed), so legs legitimately disagree. Per-line content
// fields stay; only the derived aggregate goes.
if (ledger) delete (ledger as { agg?: unknown }).agg;
// snapshotShadow is the shadow-copier's OWN health telemetry — it reports
// the ingest pipeline's experience of building this view (errors, watermark
// drops), which is inherently load-path-dependent, not conversation state.
if (out.metadata && "snapshotShadow" in (out.metadata as Record<string, unknown>)) delete (out.metadata as Record<string, unknown>)["snapshotShadow"];
if (bcEntries) out.blockContents = new Map(bcEntries.map(([k, val]) => [remap.get(k) ?? k, val]));
return sortObjectKeys(out) as T;
}

export interface StorageWire {
requested: string;
effective: string;
Expand Down Expand Up @@ -277,8 +344,12 @@ async function engproofScenario(argv: string[], run: CheckRun): Promise<void> {
// Regression (#2781): the failed open must have CLOSED its handle.
await runGuard(run, ENGPROOF_CHECK_NAMES.midFlowGuard, dir, "-moved");

// #4316 PR3/3: the clean full-mode read carried the one-time ledger
// alignment; the legacy fallback over the corrupt index did not. The
// parent passes its digest already modulo that alignment (single
// definition), so strip it here too before comparing.
await run.run(ENGPROOF_CHECK_NAMES.stateDigest, () =>
assert.equal(viewStateDigest(toLegacyLike(reloaded)), expectedStateDigest,
assert.equal(viewStateDigest(stripOneTimeAlignment(toLegacyLike(reloaded))), expectedStateDigest,
"corrupt-index fallback state digest drift vs clean full-mode read"));

// Writes must keep working (legacy path) — no throw, file refreshed.
Expand Down
16 changes: 16 additions & 0 deletions tests/golden-canonical.ts
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,22 @@
export { canonicalize, sha256 } from "../src/storage/canonical.ts";
import { canonicalize, sha256 } from "../src/storage/canonical.ts";

/** Deterministic key order for cross-read-path digest comparisons: two views
* built by different hydration paths carry identical content under different
* map insertion orders, and canonicalize preserves insertion order — sorting
* here makes the digests path-independent. Maps/Sets pass through untouched
* (callers that need them iterable keep them intact; canonicalize flattens
* them downstream). */
export function sortObjectKeys(v: unknown): unknown {
if (Array.isArray(v)) return v.map(sortObjectKeys);
if (v && typeof v === "object" && !(v instanceof Map) && !(v instanceof Set)) {
const out: Record<string, unknown> = {};
for (const k of Object.keys(v as Record<string, unknown>).sort()) out[k] = sortObjectKeys((v as Record<string, unknown>)[k]);
return out;
}
return v;
}

/** Canonical JSON → sha256, with a final pass of STRING-level environment
* normalization (ports, session ids embedded inside text fields). */
function digestOf(value: unknown, extra?: { normalize?: (s: string) => string }): string {
Expand Down
Loading
Loading