Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
96 changes: 96 additions & 0 deletions src/storage/canonical.ts
Original file line number Diff line number Diff line change
Expand Up @@ -71,12 +71,108 @@ function canonicalizeWithContext(value: unknown, key: string | undefined): unkno
}
return out;
}
return canonicalScalar(value, key);
}

function canonicalScalar(value: unknown, key: string | undefined): unknown {
if (key === "biliVersion" && typeof value === "string") return "__VER__";
if (typeof value === "number" && Number.isFinite(value) && value > 1_600_000_000_000) return "__TS__";
if (typeof value === "string" && /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i.test(value)) return "__UUID__";
return value;
}

// Spec array index: "0" .. 2^32-2, no leading zeros.
function isArrayIndexKey(k: string): boolean {
return /^(0|[1-9]\d*)$/.test(k) && Number(k) <= 4_294_967_294;
}

export function sha256(s: string): string {
return createHash("sha256").update(s, "utf8").digest("hex");
}

/** Hash a storage view without materializing its canonical copy or JSON string. */
export function canonicalDigest(value: unknown): string {
if (value === undefined || typeof value === "function" || typeof value === "symbol") return sha256(JSON.stringify(value));
const hash = createHash("sha256");
const chunkSize = 64 * 1024;
let pending = "";
function write(text: string): void {
pending += text;
if (pending.length >= chunkSize) {
hash.update(pending, "utf8");
pending = "";
}
}
function writeString(text: string): void {
write('"');
for (let start = 0; start < text.length;) {
let end = Math.min(start + chunkSize, text.length);
const last = text.charCodeAt(end - 1);
// Keep a surrogate pair together so JSON escaping and UTF-8 stay exact.
if (end < text.length && last >= 0xd800 && last <= 0xdbff) end--;
write(JSON.stringify(text.slice(start, end)).slice(1, -1));
start = end;
}
write('"');
}
function visit(current: unknown, key?: string): void {
if (Array.isArray(current)) {
write("[");
for (let i = 0; i < current.length; i++) {
if (i) write(",");
visit(current[i]);
}
write("]");
} else if (current instanceof Map) {
write('{"__map__":[');
let separator = "";
for (const [k, v] of current) {
write(separator + "[");
// canonicalize leaves Map keys untouched.
write(JSON.stringify(k) ?? "null");
write(",");
visit(v);
write("]");
separator = ",";
}
write("]}");
} else if (current instanceof Set) {
write('{"__set__":[');
let separator = "";
for (const v of current) {
write(separator);
visit(v);
separator = ",";
}
write("]}");
} else if (current && typeof current === "object") {
write("{");
let separator = "";
// Effective order of JSON.stringify(canonicalize(v)): array-index keys
// first, ascending numeric (spec enumeration overrides canonicalize's
// lexicographic sort), then the remaining keys lexicographic.
const keys = Object.keys(current);
const orderedKeys = keys.filter(isArrayIndexKey).sort((a, b) => Number(a) - Number(b))
.concat(keys.filter((k) => !isArrayIndexKey(k)).sort());
for (const k of orderedKeys) {
// Assigning __proto__ to canonicalize's plain output creates no JSON property.
if (Object.hasOwn(DIGEST_SKIPPED_KEYS, k) || k === "__proto__") continue;
const v = (current as Record<string, unknown>)[k];
if (v === undefined || typeof v === "function" || typeof v === "symbol") continue;
write(separator);
writeString(k);
write(":");
visit(v, k);
separator = ",";
}
write("}");
} else {
const normalized = canonicalScalar(current, key);
if (typeof normalized === "string") writeString(normalized);
else write(JSON.stringify(normalized) ?? "null");
}
}
visit(value);
hash.update(pending, "utf8");
return hash.digest("hex");
}
4 changes: 2 additions & 2 deletions src/storage/shadow.ts
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@ import { UnifiedStore, StorageIntegrityError, type CheckpointRow, type SessionSu
import { renameSync } from "node:fs";
import path from "node:path";
import { resolveBackendName, backendSelectionNotice, selectedBackendRaw, openBackend, type StorageBackend } from "./backend.ts";
import { canonicalize, sha256 } from "./canonical.ts";
import { canonicalDigest, canonicalize, sha256 } from "./canonical.ts";
import { recordSidecarInstance, recordSidecarOpenFailure } from "./runtime-status.ts";
import type { SqliteEngineName, SqliteEngineSource } from "./driver.ts";
import type { MessageRow, MessageRefContentRow } from "./store.js";
Expand Down Expand Up @@ -1243,7 +1243,7 @@ function warnTailGaps(id: string, view: { tailGaps?: number }): void {
}

function shadowDigest(view: unknown): string {
return sha256(JSON.stringify(canonicalize(view)));
return canonicalDigest(view);
}

// #3407: the parity digest must compare blockContents CONTENTS, not insertion order —
Expand Down
9 changes: 5 additions & 4 deletions src/web/blogfeed.ts
Original file line number Diff line number Diff line change
Expand Up @@ -106,10 +106,11 @@ export function _setBlogRootForTest(root: string | null): void {
/** Mirror of the website tools' frontmatter parser (key: value between --- fences). */
function parseFrontmatter(text: string): { fm: Record<string, string>; body: string } {
const fm: Record<string, string> = {};
let body = text;
// Git-for-Windows autocrlf checks .md posts out with CRLF; normalize so
// the exact-match fences below survive Windows checkouts (#4599).
const src = text.startsWith("---\r\n") ? text.replace(/\r\n/g, "\n") : text;
// Git-for-Windows autocrlf checks .md posts out with CRLF; normalize
// UNCONDITIONALLY so the exact-match fences survive Windows checkouts
// (#4599) and non-frontmatter bodies cannot leak \r into rendered HTML.
const src = text.replace(/\r\n/g, "\n");
let body = src;
if (src.startsWith("---\n")) {
const end = src.indexOf("\n---\n", 4);
if (end !== -1) {
Expand Down
66 changes: 50 additions & 16 deletions tests/issue4982-draining-twin-ladder.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -228,37 +228,68 @@ async function waitForStatus(port: number, timeoutMs: number): Promise<boolean>

async function killChild(child: ChildProcess): Promise<void> {
if (child.exitCode !== null || child.signalCode !== null) return;
child.kill("SIGTERM");
await Promise.race([
new Promise<void>((resolve) => child.once("exit", () => resolve())),
new Promise<void>((resolve) => setTimeout(resolve, 10_000)),
]);
if (child.exitCode === null && child.signalCode === null) child.kill("SIGKILL");
await new Promise<void>((resolve) => {
const timer = setTimeout(() => child.kill("SIGKILL"), 10_000);
child.once("close", () => {
clearTimeout(timer);
resolve();
});
child.kill("SIGTERM");
});
}

/** A TCP listener that accepts connections and then says nothing — invisible
* to /__bili/health probes (they time out as "no evidence") but still owns
* the socket, so binders get EADDRINUSE. Deterministically forces the LATE
* EADDRINUSE decision path instead of the pre-bind foreign-owner ladder. */
function silentTcpListener(port: number): Promise<net.Server> {
function silentTcpListener(port: number): Promise<{ close(): Promise<void> }> {
return new Promise((resolve, reject) => {
const sockets = new Set<net.Socket>();
const server = net.createServer((sock) => {
sockets.add(sock);
sock.once("close", () => sockets.delete(sock));
// accept and hang — never answer HTTP; drain the probe's bytes so peer
// EOF is actually delivered (a paused socket buffers them and never sees
// EOF) and release the connection, or a dead prober pins server.close()
// with CLOSE-WAIT sockets
sock.resume();
// A killed prober (Windows kill() is a hard terminate) RSTs us mid-probe;
// without an error listener that surfaces as an uncaughtException AFTER
// the test ends and fails the whole file (win-only #5050 follow-up).
sock.on("error", () => {});
sock.on("end", () => sock.destroy());
// An aborted health probe may reset instead of ending the connection,
// or a killed prober (Windows kill() is a hard terminate) RSTs mid-probe;
// without an error listener that surfaces as an uncaughtException AFTER
// the test ends and fails the whole file (#5050). Destroying keeps the
// fixture clean and owns the socket.
sock.on("error", () => sock.destroy());
});
server.once("error", reject);
server.listen(port, "127.0.0.1", () => resolve(server));
server.listen(port, "127.0.0.1", () => resolve({
close: () => new Promise<void>((done) => {
server.close(() => done());
for (const socket of sockets) socket.destroy();
}),
}));
});
}

test("silent TCP fixture survives an aborted health probe", { timeout: 5_000 }, async () => {
const port = await freePort();
const fixture = await silentTcpListener(port);
const client = net.connect({ host: "127.0.0.1", port });
try {
await new Promise<void>((resolve, reject) => {
client.once("connect", resolve);
client.once("error", reject);
});
client.write("GET /__bili/health HTTP/1.1\r\nHost: 127.0.0.1\r\n\r\n");
await new Promise((resolve) => setTimeout(resolve, 25));
client.resetAndDestroy();
await new Promise((resolve) => setTimeout(resolve, 25));
} finally {
client.destroy();
await fixture.close();
}
});

function tcpRefused(port: number): Promise<boolean> {
return new Promise((resolve) => {
const s = net.connect({ port, host: "127.0.0.1" });
Expand Down Expand Up @@ -323,6 +354,9 @@ function spawnBili(sandbox: string, port: number): { child: ChildProcess; out: {
async function waitUntilLadderedHealthy(stateHome: string, child: ChildProcess, label: string): Promise<number> {
let port = 0;
await waitUntil(async () => {
if (child.exitCode !== null || child.signalCode !== null) {
throw new Error(`${label} exited before becoming healthy`);
}
const m = readMarkers(stateHome).find((e) => e.pid === child.pid);
if (!m || typeof m.port !== "number") return false;
port = m.port as number;
Expand All @@ -339,7 +373,7 @@ test("#4982 e2e: draining same-lane predecessor + squatter → newcomer ladders
let p1Ref: ReturnType<typeof spawnBili> | null = null;
let p2Ref: ReturnType<typeof spawnBili> | null = null;
let heldRef: net.Socket | null = null;
let squatterRef: net.Server | null = null;
let squatterRef: Awaited<ReturnType<typeof silentTcpListener>> | null = null;
try {
// P1: the serving proxy that will enter graceful shutdown.
const p1 = spawnBili(sandbox, port);
Expand Down Expand Up @@ -415,7 +449,7 @@ test("#4982 e2e: draining same-lane predecessor + squatter → newcomer ladders
heldRef?.destroy();
if (p1Ref) await killChild(p1Ref.child);
const squatter = squatterRef;
if (squatter) await new Promise<void>((resolve) => squatter.close(() => resolve()));
if (squatter) await squatter.close();
fs.rmSync(sandbox, { recursive: true, force: true });
}
});
Expand All @@ -426,7 +460,7 @@ test("#4982 e2e: unverified registry twin claim (unstamped marker) → ladder, n
const port = await freePort();
await assertPortDead(port, { label: "4982 unverified-twin port" });
let p2Ref: ReturnType<typeof spawnBili> | null = null;
let squatterRef: net.Server | null = null;
let squatterRef: Awaited<ReturnType<typeof silentTcpListener>> | null = null;
try {
// A registry marker claiming `port` with a LIVE pid and full identity
// match — but NO portReleasedAt stamp (pre-fix build, or a failed
Expand Down Expand Up @@ -476,7 +510,7 @@ test("#4982 e2e: unverified registry twin claim (unstamped marker) → ladder, n
} finally {
if (p2Ref) await killChild(p2Ref.child);
const squatter = squatterRef;
if (squatter) await new Promise<void>((resolve) => squatter.close(() => resolve()));
if (squatter) await squatter.close();
fs.rmSync(sandbox, { recursive: true, force: true });
}
});
Loading
Loading