Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
33 changes: 33 additions & 0 deletions src/instance.ts
Original file line number Diff line number Diff line change
Expand Up @@ -248,6 +248,12 @@ export function classifySameLanePeer(
for (const e of entries) {
if (e.port !== port || e.lane !== lane) continue;
if (!isPidAlive(e.pid)) continue;
// #4982: a draining instance stamped portReleasedAt when its listener
// closed — its pid still lives but the socket no longer does, so it
// cannot be the EADDRINUSE holder even though every other identity cell
// matches. This is exactly the stale-twin misclassification that killed
// #4982's CI child while a cross-lane instance owned the freed port.
if (e.portReleasedAt !== undefined) continue;
if (e.codeFingerprint !== ownFingerprint) continue;
return { kind: sameShape(e) ? "twin" : "drifted", entry: e };
}
Expand Down Expand Up @@ -482,6 +488,14 @@ export interface RegistryEntry {
modelMaxOutputs?: Record<string, number>;
lane?: string;
codeFingerprint?: string;
/** #4982: set when this instance released its listening socket (graceful
* shutdown start). The process can stay alive for seconds-to-minutes while
* it drains keep-alive connections and flushes state — during that window
* it CANNOT be the EADDRINUSE holder of `port` anymore, so consumers that
* judge PORT OWNERSHIP (classifySameLanePeer) must skip marked entries,
* while consumers that wait for the process to EXIT (findSameLanePredecessor
* → wait-predecessor) keep counting them. */
portReleasedAt?: number;
}

/** Cross-instance liveness registry (#394/#527): one marker file per instance
Expand Down Expand Up @@ -549,6 +563,7 @@ function coerceEntry(value: unknown): RegistryEntry | undefined {
...(Object.keys(maxOutputs).length > 0 ? { modelMaxOutputs: maxOutputs } : {}),
...(typeof o.lane === "string" && o.lane !== "" ? { lane: o.lane } : {}),
...(typeof o.codeFingerprint === "string" && o.codeFingerprint !== "" ? { codeFingerprint: o.codeFingerprint } : {}),
...(typeof o.portReleasedAt === "number" && Number.isFinite(o.portReleasedAt) ? { portReleasedAt: o.portReleasedAt } : {}),
};
}

Expand Down Expand Up @@ -704,3 +719,21 @@ export function unregisterInstance(instanceId: string): void {
fs.unlinkSync(registryEntryFile(instanceId));
} catch {}
}

/** #4982: withdraw the port-ownership claim the moment the listening socket
* closes (graceful shutdown start). The drain can outlive the release by
* minutes (keep-alive pins, slow flushes), and during that window a stale
* "I hold port N" marker let a same-lane newcomer misclassify us as its twin
* and die with the adopt signal while a cross-lane instance owned the freed
* socket (#4982). Rewrites ONLY our own marker file with the stamp —
* per-instance files mean no cross-process read-modify-write. Best-effort:
* a failed mark degrades to the pre-#4982 behavior, where the health
* verification in the EADDRINUSE path still catches the stale claim. */
export function markPortReleased(instanceId: string): void {
try {
const file = registryEntryFile(instanceId);
const entry = coerceEntry(safeReadJson(file));
if (!entry || entry.portReleasedAt !== undefined) return;
atomicWriteJson({ ...entry, portReleasedAt: Date.now() }, file);
} catch {}
}
30 changes: 30 additions & 0 deletions src/port-owner.ts
Original file line number Diff line number Diff line change
Expand Up @@ -102,6 +102,36 @@ export async function findForeignOwner(
return undefined;
}

/** #4982: does the listener on `origin` answer health AS the registry entry's
* claimed owner (exact pid + instance identity)? A registry marker is a CLAIM
* written at announce time, not proof: between the claim and our bind the
* socket can move on — the claimed owner enters graceful shutdown (listener
* closed, process still draining) and a different instance binds the freed
* port. Declaring a twin collision on an unverified claim kills the newcomer
* with the adopt signal over a port it could have laddered off (#4982).
* An ANSWERED probe decides immediately (match or not); only a silent or
* non-bili responder is inconclusive and retried within the standard probe
* budget — a saturated-but-alive twin must not be mistaken for a moved
* socket either. */
export async function verifyEntryOwnsPort(
origin: string,
expected: { pid: number; instanceId: string },
opts: { attempts?: number; intervalMs?: number; timeoutMs?: number } & ProbeDeps = {},
): Promise<boolean> {
const attempts = Math.max(1, opts.attempts ?? DEFAULT_ATTEMPTS);
const intervalMs = opts.intervalMs ?? DEFAULT_INTERVAL_MS;
const timeoutMs = opts.timeoutMs ?? DEFAULT_TIMEOUT_MS;
const fetchImpl = opts.fetchImpl ?? probeFetch;
const sleepImpl = opts.sleepImpl ?? ((ms: number) => new Promise((r) => setTimeout(r, ms)));
for (let i = 0; i < attempts; i++) {
if (i > 0) await sleepImpl(intervalMs);
const info = await probeOwnerInfo(origin, fetchImpl, timeoutMs);
if (info === undefined) continue; // inconclusive — retry
return info.pid === expected.pid && info.instanceId === expected.instanceId;
}
return false;
}

type PortClaimOutcome =
| { kind: "claimed" }
| { kind: "ladder"; nextPort: number; foreign: OwnerProbeInfo }
Expand Down
Loading
Loading