From 9ce97cac88a21f633a47f00ab36a000bef575fd3 Mon Sep 17 00:00:00 2001 From: anuzzolilli-oss Date: Sat, 14 Nov 2026 19:00:55 -0500 Subject: [PATCH 1/2] fix(zcode): store-drift repair re-checks the store's owner before each write (#63) repairSharedStoreDrift read the shared store's origin once, awaited up to two health probes and replacement discovery, then wrote without looking again. A route another process wrote during those awaits could be overwritten with this process's origin or a replacement, or reverted to direct. Each of the three writes is now allowed only while the store still points at the dead origin first read. The two route writes ask under the config lock (the existing shouldWrite hook); the revert re-checks immediately before it, which also stops a declined route write from falling through into a revert. New outcome "owner-changed" for the stand-down. Assisted-by: Cindy (https://cindy.app) --- src/zcode/native.ts | 17 ++++- tests/zcode-native.test.ts | 131 +++++++++++++++++++++++++++++++++++++ 2 files changed, 146 insertions(+), 2 deletions(-) diff --git a/src/zcode/native.ts b/src/zcode/native.ts index 6fd646bf24..e1c5bce973 100644 --- a/src/zcode/native.ts +++ b/src/zcode/native.ts @@ -528,6 +528,7 @@ type StoreDriftOutcome = | "unmanaged" | "self" | "foreign-live" + | "owner-changed" | "repointed-self" | "repointed-replacement" | "reverted-direct"; @@ -565,8 +566,14 @@ export async function repairSharedStoreDrift(opts: StoreDriftOptions): Promise detectCurrentZcodeOrigin(dataDir, env) === current; if (await probe(opts.selfOrigin)) { - const routed = await routeZcodeConfig({ origin: opts.selfOrigin, env, dataDir: opts.dataDir, log, policy }); + const routed = await routeZcodeConfig({ origin: opts.selfOrigin, env, dataDir: opts.dataDir, log, policy, shouldWrite: stillDead }); if (routed) { await activateZcodePluginMode(routed, { env, dataDir: opts.dataDir, log }); log(`store repaired → ${opts.selfOrigin}`); @@ -575,7 +582,7 @@ export async function repairSharedStoreDrift(opts: StoreDriftOptions): Promise void | Promise, logs: string[] = []) { + return repairSharedStoreDrift({ + selfOrigin: DRIFT_SELF, + dataDir: dir, + env: {}, + log: (m) => logs.push(m), + probe: async (origin) => { + if (origin !== DRIFT_SELF) return false; + if (writePath !== "self") return false; + await during(); + return true; + }, + findReplacement: async () => { + await during(); + return writePath === "replacement" ? { origin: DRIFT_REPLACEMENT } : undefined; + }, + }); +} + +for (const writePath of ["self", "replacement", "revert"] as const) { + test(`repairSharedStoreDrift leaves a route another owner took during its awaits (write path: ${writePath}) (#63)`, async () => { + const dir = dataDir(); + try { + await routeZcodeConfig({ origin: DRIFT_DEAD, dataDir: dir, env: {}, log: NO_LOG }); + let taken = ""; + const logs: string[] = []; + const outcome = await repairDownPath( + dir, + writePath, + async () => { + await routeZcodeConfig({ origin: DRIFT_NEW_OWNER, dataDir: dir, env: {}, log: NO_LOG }); + taken = readFileSync(storeFile(dir), "utf8"); + }, + logs, + ); + assert.equal(outcome, "owner-changed"); + assert.equal(detectCurrentZcodeOrigin(dir, {}), DRIFT_NEW_OWNER); + assert.equal(readFileSync(storeFile(dir), "utf8"), taken, "the new owner's store is byte-identical"); + assert.ok(logs.some((m) => /no longer points at/.test(m)), "the repair says why it stood down"); + assert.ok(!logs.some((m) => /store repaired|reverted to direct/.test(m)), "and claims no repair"); + } finally { + rmrf(dir); + } + }); + + test(`control: repairSharedStoreDrift still writes when nothing changed during its awaits (write path: ${writePath}) (#63)`, async () => { + const dir = dataDir(); + try { + await routeZcodeConfig({ origin: DRIFT_DEAD, dataDir: dir, env: {}, log: NO_LOG }); + const outcome = await repairDownPath(dir, writePath, () => {}); + if (writePath === "self") { + assert.equal(outcome, "repointed-self"); + assert.equal(detectCurrentZcodeOrigin(dir, {}), DRIFT_SELF); + } else if (writePath === "replacement") { + assert.equal(outcome, "repointed-replacement"); + assert.equal(detectCurrentZcodeOrigin(dir, {}), DRIFT_REPLACEMENT); + } else { + assert.equal(outcome, "reverted-direct"); + assert.doesNotMatch(readFileSync(storeFile(dir), "utf8"), /\/bili\//); + } + } finally { + rmrf(dir); + } + }); +} + +for (const writePath of ["self", "replacement"] as const) { + test(`repairSharedStoreDrift leaves a store that went direct during its awaits (write path: ${writePath}) (#63)`, async () => { + const dir = dataDir(); + try { + await routeZcodeConfig({ origin: DRIFT_DEAD, dataDir: dir, env: {}, log: NO_LOG }); + let direct = ""; + const outcome = await repairDownPath(dir, writePath, () => { + unrouteZcode({ dataDir: dir, env: {}, log: NO_LOG }); + direct = readFileSync(storeFile(dir), "utf8"); + }); + assert.equal(outcome, "owner-changed"); + assert.doesNotMatch(direct, /\/bili\//); + assert.equal(readFileSync(storeFile(dir), "utf8"), direct, "a store reverted to direct is not re-wrapped"); + } finally { + rmrf(dir); + } + }); + + // The check has to sit inside the locked write: here the store still + // points at the dead origin when the route call starts, and changes owner + // only while that call is waiting for the config lock. + test(`repairSharedStoreDrift leaves a route taken while it waited for the config lock (write path: ${writePath}) (#63)`, async () => { + const dir = dataDir(); + const lockDir = path.join(dir, "v2", ".bili-config.lock"); + try { + await routeZcodeConfig({ origin: DRIFT_DEAD, dataDir: dir, env: {}, log: NO_LOG }); + let taken = ""; + let heldWhenTaken = false; + const outcome = await repairDownPath(dir, writePath, () => { + // Another live process holds the lock, writes its route, releases. + mkdirSync(lockDir); + writeFileSync(path.join(lockDir, "pid"), String(process.pid)); + setTimeout(() => { + heldWhenTaken = existsSync(lockDir); + taken = readFileSync(storeFile(dir), "utf8").split(DRIFT_DEAD).join(DRIFT_NEW_OWNER); + writeFileSync(storeFile(dir), taken); + rmrf(lockDir); + }, 400); + }); + assert.ok(heldWhenTaken, "the route changed while the lock was held by the other writer"); + assert.equal(outcome, "owner-changed"); + assert.equal(detectCurrentZcodeOrigin(dir, {}), DRIFT_NEW_OWNER); + assert.equal(readFileSync(storeFile(dir), "utf8"), taken); + assert.ok(!existsSync(lockDir), "the repair released the lock it took"); + } finally { + rmrf(dir); + } + }); +} + test("handoffZcodeRoutingOnExit leaves a foreign pointer untouched (#1623)", async () => { const dir = dataDir(); try { From 00890c27f9e6bd8a7e7725ba7cf4c5f13f67fefc Mon Sep 17 00:00:00 2001 From: anuzzolilli-oss Date: Sat, 14 Nov 2026 19:22:50 -0500 Subject: [PATCH 2/2] test(zcode): drift-repair tests report runtime info to servers they own (#63) A successful repair calls activateZcodePluginMode, which posts runtime info to the origin it routed to. The new tests used fixed ports for the self and replacement origins, so a bili instance listening there could have received that post. Both origins are now loopback servers the test starts on allocated ports. Assisted-by: Cindy (https://cindy.app) --- tests/zcode-native.test.ts | 51 ++++++++++++++++++++++---------------- 1 file changed, 30 insertions(+), 21 deletions(-) diff --git a/tests/zcode-native.test.ts b/tests/zcode-native.test.ts index e16f7fda94..14d3e5a5ea 100644 --- a/tests/zcode-native.test.ts +++ b/tests/zcode-native.test.ts @@ -656,42 +656,51 @@ test("repairSharedStoreDrift reverts to direct when nothing live remains (#1623) // a replacement, revert to direct. The positive cases above are the controls. const DRIFT_DEAD = "http://127.0.0.1:18787"; -const DRIFT_SELF = "http://127.0.0.1:28787"; -const DRIFT_REPLACEMENT = "http://127.0.0.1:48787"; const DRIFT_NEW_OWNER = "http://127.0.0.1:38787"; type DriftPath = "self" | "replacement" | "revert"; +interface DriftOrigins { + readonly self: string; + readonly replacement: string; +} + +/** A successful repair reports runtime info to the origin it routed to, so the + * two possible destinations are servers this test owns, on allocated ports. + * The dead origin and the new owner are never contacted. */ +function withDriftOrigins(fn: (origins: DriftOrigins) => Promise): Promise { + return withHealthServer((self) => withHealthServer((replacement) => fn({ self, replacement }))); +} /** Runs the repair down one write path; `during` runs inside the last await - * before that path's write. Nothing is listening anywhere: liveness is - * entirely the probe seam's answer. */ -function repairDownPath(dir: string, writePath: DriftPath, during: () => void | Promise, logs: string[] = []) { + * before that path's write. Liveness is entirely the probe seam's answer. */ +function repairDownPath(o: DriftOrigins, dir: string, writePath: DriftPath, during: () => void | Promise, logs: string[] = []) { return repairSharedStoreDrift({ - selfOrigin: DRIFT_SELF, + selfOrigin: o.self, dataDir: dir, env: {}, log: (m) => logs.push(m), probe: async (origin) => { - if (origin !== DRIFT_SELF) return false; + if (origin !== o.self) return false; if (writePath !== "self") return false; await during(); return true; }, findReplacement: async () => { await during(); - return writePath === "replacement" ? { origin: DRIFT_REPLACEMENT } : undefined; + return writePath === "replacement" ? { origin: o.replacement } : undefined; }, }); } for (const writePath of ["self", "replacement", "revert"] as const) { - test(`repairSharedStoreDrift leaves a route another owner took during its awaits (write path: ${writePath}) (#63)`, async () => { + test(`repairSharedStoreDrift leaves a route another owner took during its awaits (write path: ${writePath}) (#63)`, () => withDriftOrigins(async (o) => { const dir = dataDir(); try { await routeZcodeConfig({ origin: DRIFT_DEAD, dataDir: dir, env: {}, log: NO_LOG }); let taken = ""; const logs: string[] = []; const outcome = await repairDownPath( + o, dir, writePath, async () => { @@ -708,19 +717,19 @@ for (const writePath of ["self", "replacement", "revert"] as const) { } finally { rmrf(dir); } - }); + })); - test(`control: repairSharedStoreDrift still writes when nothing changed during its awaits (write path: ${writePath}) (#63)`, async () => { + test(`control: repairSharedStoreDrift still writes when nothing changed during its awaits (write path: ${writePath}) (#63)`, () => withDriftOrigins(async (o) => { const dir = dataDir(); try { await routeZcodeConfig({ origin: DRIFT_DEAD, dataDir: dir, env: {}, log: NO_LOG }); - const outcome = await repairDownPath(dir, writePath, () => {}); + const outcome = await repairDownPath(o, dir, writePath, () => {}); if (writePath === "self") { assert.equal(outcome, "repointed-self"); - assert.equal(detectCurrentZcodeOrigin(dir, {}), DRIFT_SELF); + assert.equal(detectCurrentZcodeOrigin(dir, {}), o.self); } else if (writePath === "replacement") { assert.equal(outcome, "repointed-replacement"); - assert.equal(detectCurrentZcodeOrigin(dir, {}), DRIFT_REPLACEMENT); + assert.equal(detectCurrentZcodeOrigin(dir, {}), o.replacement); } else { assert.equal(outcome, "reverted-direct"); assert.doesNotMatch(readFileSync(storeFile(dir), "utf8"), /\/bili\//); @@ -728,16 +737,16 @@ for (const writePath of ["self", "replacement", "revert"] as const) { } finally { rmrf(dir); } - }); + })); } for (const writePath of ["self", "replacement"] as const) { - test(`repairSharedStoreDrift leaves a store that went direct during its awaits (write path: ${writePath}) (#63)`, async () => { + test(`repairSharedStoreDrift leaves a store that went direct during its awaits (write path: ${writePath}) (#63)`, () => withDriftOrigins(async (o) => { const dir = dataDir(); try { await routeZcodeConfig({ origin: DRIFT_DEAD, dataDir: dir, env: {}, log: NO_LOG }); let direct = ""; - const outcome = await repairDownPath(dir, writePath, () => { + const outcome = await repairDownPath(o, dir, writePath, () => { unrouteZcode({ dataDir: dir, env: {}, log: NO_LOG }); direct = readFileSync(storeFile(dir), "utf8"); }); @@ -747,19 +756,19 @@ for (const writePath of ["self", "replacement"] as const) { } finally { rmrf(dir); } - }); + })); // The check has to sit inside the locked write: here the store still // points at the dead origin when the route call starts, and changes owner // only while that call is waiting for the config lock. - test(`repairSharedStoreDrift leaves a route taken while it waited for the config lock (write path: ${writePath}) (#63)`, async () => { + test(`repairSharedStoreDrift leaves a route taken while it waited for the config lock (write path: ${writePath}) (#63)`, () => withDriftOrigins(async (o) => { const dir = dataDir(); const lockDir = path.join(dir, "v2", ".bili-config.lock"); try { await routeZcodeConfig({ origin: DRIFT_DEAD, dataDir: dir, env: {}, log: NO_LOG }); let taken = ""; let heldWhenTaken = false; - const outcome = await repairDownPath(dir, writePath, () => { + const outcome = await repairDownPath(o, dir, writePath, () => { // Another live process holds the lock, writes its route, releases. mkdirSync(lockDir); writeFileSync(path.join(lockDir, "pid"), String(process.pid)); @@ -778,7 +787,7 @@ for (const writePath of ["self", "replacement"] as const) { } finally { rmrf(dir); } - }); + })); } test("handoffZcodeRoutingOnExit leaves a foreign pointer untouched (#1623)", async () => {