Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 15 additions & 2 deletions src/zcode/native.ts
Original file line number Diff line number Diff line change
Expand Up @@ -528,6 +528,7 @@ type StoreDriftOutcome =
| "unmanaged"
| "self"
| "foreign-live"
| "owner-changed"
| "repointed-self"
| "repointed-replacement"
| "reverted-direct";
Expand Down Expand Up @@ -565,8 +566,14 @@ export async function repairSharedStoreDrift(opts: StoreDriftOptions): Promise<S
if (await probe(current)) return "foreign-live";
log(`shared store points at dead instance ${current} — repairing`);
const policy = zcodePolicyFromEnv(env);
// #63: the probes and replacement discovery await, and another process may
// re-route the store meanwhile. Every write below is allowed only while the
// store still points at the dead origin this repair set out to replace; a
// store that now points anywhere else (or went direct) is someone else's
// decision and stays untouched. The route writes ask under the config lock.
const stillDead = () => detectCurrentZcodeOrigin(dataDir, env) === current;
if (await probe(opts.selfOrigin)) {
const routed = await routeZcodeConfig({ origin: opts.selfOrigin, env, dataDir: opts.dataDir, log, policy });
const routed = await routeZcodeConfig({ origin: opts.selfOrigin, env, dataDir: opts.dataDir, log, policy, shouldWrite: stillDead });
if (routed) {
await activateZcodePluginMode(routed, { env, dataDir: opts.dataDir, log });
log(`store repaired → ${opts.selfOrigin}`);
Expand All @@ -575,14 +582,20 @@ export async function repairSharedStoreDrift(opts: StoreDriftOptions): Promise<S
} else {
const replacement = await (opts.findReplacement ?? defaultFindReplacement)();
if (replacement) {
const routed = await routeZcodeConfig({ origin: replacement.origin, env, dataDir: opts.dataDir, log, policy });
const routed = await routeZcodeConfig({ origin: replacement.origin, env, dataDir: opts.dataDir, log, policy, shouldWrite: stillDead });
if (routed) {
await activateZcodePluginMode(routed, { env, dataDir: opts.dataDir, log });
log(`store repaired → ${replacement.origin}`);
return "repointed-replacement";
}
}
}
// Also reached when a route write above was declined: without this check a
// declined overwrite would become a revert of the new owner's route.
if (!stillDead()) {
log(`store drift repair: the store no longer points at ${current} — leaving it untouched`);
return "owner-changed";
}
unrouteZcode({ env, dataDir: opts.dataDir, log });
log(`store reverted to direct upstream (${current} is dead)`);
return "reverted-direct";
Expand Down
140 changes: 140 additions & 0 deletions tests/zcode-native.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -650,6 +650,146 @@ test("repairSharedStoreDrift reverts to direct when nothing live remains (#1623)
}
});

// #63: the repair reads the store's owner once, then awaits health probes and
// replacement discovery. A route another process writes during those awaits
// must survive all three of the repair's writes: re-point to self, re-point to
// a replacement, revert to direct. The positive cases above are the controls.

const DRIFT_DEAD = "http://127.0.0.1:18787";
const DRIFT_NEW_OWNER = "http://127.0.0.1:38787";

type DriftPath = "self" | "replacement" | "revert";
interface DriftOrigins {
readonly self: string;
readonly replacement: string;
}

/** A successful repair reports runtime info to the origin it routed to, so the
* two possible destinations are servers this test owns, on allocated ports.
* The dead origin and the new owner are never contacted. */
function withDriftOrigins<T>(fn: (origins: DriftOrigins) => Promise<T>): Promise<T> {
return withHealthServer((self) => withHealthServer((replacement) => fn({ self, replacement })));
}

/** Runs the repair down one write path; `during` runs inside the last await
* before that path's write. Liveness is entirely the probe seam's answer. */
function repairDownPath(o: DriftOrigins, dir: string, writePath: DriftPath, during: () => void | Promise<void>, logs: string[] = []) {
return repairSharedStoreDrift({
selfOrigin: o.self,
dataDir: dir,
env: {},
log: (m) => logs.push(m),
probe: async (origin) => {
if (origin !== o.self) return false;
if (writePath !== "self") return false;
await during();
return true;
},
findReplacement: async () => {
await during();
return writePath === "replacement" ? { origin: o.replacement } : undefined;
},
});
}

for (const writePath of ["self", "replacement", "revert"] as const) {
test(`repairSharedStoreDrift leaves a route another owner took during its awaits (write path: ${writePath}) (#63)`, () => withDriftOrigins(async (o) => {
const dir = dataDir();
try {
await routeZcodeConfig({ origin: DRIFT_DEAD, dataDir: dir, env: {}, log: NO_LOG });
let taken = "";
const logs: string[] = [];
const outcome = await repairDownPath(
o,
dir,
writePath,
async () => {
await routeZcodeConfig({ origin: DRIFT_NEW_OWNER, dataDir: dir, env: {}, log: NO_LOG });
taken = readFileSync(storeFile(dir), "utf8");
},
logs,
);
assert.equal(outcome, "owner-changed");
assert.equal(detectCurrentZcodeOrigin(dir, {}), DRIFT_NEW_OWNER);
assert.equal(readFileSync(storeFile(dir), "utf8"), taken, "the new owner's store is byte-identical");
assert.ok(logs.some((m) => /no longer points at/.test(m)), "the repair says why it stood down");
assert.ok(!logs.some((m) => /store repaired|reverted to direct/.test(m)), "and claims no repair");
} finally {
rmrf(dir);
}
}));

test(`control: repairSharedStoreDrift still writes when nothing changed during its awaits (write path: ${writePath}) (#63)`, () => withDriftOrigins(async (o) => {
const dir = dataDir();
try {
await routeZcodeConfig({ origin: DRIFT_DEAD, dataDir: dir, env: {}, log: NO_LOG });
const outcome = await repairDownPath(o, dir, writePath, () => {});
if (writePath === "self") {
assert.equal(outcome, "repointed-self");
assert.equal(detectCurrentZcodeOrigin(dir, {}), o.self);
} else if (writePath === "replacement") {
assert.equal(outcome, "repointed-replacement");
assert.equal(detectCurrentZcodeOrigin(dir, {}), o.replacement);
} else {
assert.equal(outcome, "reverted-direct");
assert.doesNotMatch(readFileSync(storeFile(dir), "utf8"), /\/bili\//);
}
} finally {
rmrf(dir);
}
}));
}

for (const writePath of ["self", "replacement"] as const) {
test(`repairSharedStoreDrift leaves a store that went direct during its awaits (write path: ${writePath}) (#63)`, () => withDriftOrigins(async (o) => {
const dir = dataDir();
try {
await routeZcodeConfig({ origin: DRIFT_DEAD, dataDir: dir, env: {}, log: NO_LOG });
let direct = "";
const outcome = await repairDownPath(o, dir, writePath, () => {
unrouteZcode({ dataDir: dir, env: {}, log: NO_LOG });
direct = readFileSync(storeFile(dir), "utf8");
});
assert.equal(outcome, "owner-changed");
assert.doesNotMatch(direct, /\/bili\//);
assert.equal(readFileSync(storeFile(dir), "utf8"), direct, "a store reverted to direct is not re-wrapped");
} finally {
rmrf(dir);
}
}));

// The check has to sit inside the locked write: here the store still
// points at the dead origin when the route call starts, and changes owner
// only while that call is waiting for the config lock.
test(`repairSharedStoreDrift leaves a route taken while it waited for the config lock (write path: ${writePath}) (#63)`, () => withDriftOrigins(async (o) => {
const dir = dataDir();
const lockDir = path.join(dir, "v2", ".bili-config.lock");
try {
await routeZcodeConfig({ origin: DRIFT_DEAD, dataDir: dir, env: {}, log: NO_LOG });
let taken = "";
let heldWhenTaken = false;
const outcome = await repairDownPath(o, dir, writePath, () => {
// Another live process holds the lock, writes its route, releases.
mkdirSync(lockDir);
writeFileSync(path.join(lockDir, "pid"), String(process.pid));
setTimeout(() => {
heldWhenTaken = existsSync(lockDir);
taken = readFileSync(storeFile(dir), "utf8").split(DRIFT_DEAD).join(DRIFT_NEW_OWNER);
writeFileSync(storeFile(dir), taken);
rmrf(lockDir);
}, 400);
});
assert.ok(heldWhenTaken, "the route changed while the lock was held by the other writer");
assert.equal(outcome, "owner-changed");
assert.equal(detectCurrentZcodeOrigin(dir, {}), DRIFT_NEW_OWNER);
assert.equal(readFileSync(storeFile(dir), "utf8"), taken);
assert.ok(!existsSync(lockDir), "the repair released the lock it took");
} finally {
rmrf(dir);
}
}));
}

test("handoffZcodeRoutingOnExit leaves a foreign pointer untouched (#1623)", async () => {
const dir = dataDir();
try {
Expand Down
Loading