diff --git a/packages/cli/src/__tests__/runtime-host-installed-update-activator.test.ts b/packages/cli/src/__tests__/runtime-host-installed-update-activator.test.ts index fbc95536b1..5a301f490d 100644 --- a/packages/cli/src/__tests__/runtime-host-installed-update-activator.test.ts +++ b/packages/cli/src/__tests__/runtime-host-installed-update-activator.test.ts @@ -19,6 +19,7 @@ import assert from 'node:assert/strict'; import test from 'node:test'; +import { LocalHostDeploymentAuthorityError } from '@maka/runtime-host/operator'; import { INTERACTIVE_RUNTIME_HOST_COMPOSITION_ID, RUNTIME_HOST_COMPATIBILITY_EPOCH, @@ -26,7 +27,11 @@ import { RUNTIME_HOST_REGISTRATION_SCHEMA_VERSION, type HostRegistration, } from '@maka/runtime-host/protocol'; -import { runRuntimeHostInstalledUpdateActivator } from '../runtime-host-installed-update-activator.js'; +import { + RuntimeHostDurableSettlementError, + runRuntimeHostInstalledUpdateActivator, + settleTargetFromDurableAuthority, +} from '../runtime-host-installed-update-activator.js'; const ROOT_ID = 'a'.repeat(64); @@ -176,13 +181,195 @@ test('fails closed through the authenticated connection when its coordinator cha retirement = { hostEpoch: connection.hostEpoch, mode }; return { kind: 'prepared', pid: 84 }; }, + readRecord: async () => undefined, }, ), - /lost its coordinator channel/u, + /lost its coordinator before ownership committed/u, ); assert.deepEqual(retirement, { hostEpoch: 'target-host', mode: 'interrupt_active_work' }); }); +test('durable committed ownership releases the launch barrier after an ambiguous record read', async () => { + const events: string[] = []; + let reads = 0; + const settlement = await settleTargetFromDurableAuthority( + { + connection: {} as never, + expectedRootId: ROOT_ID, + ownerInstallationId: 'npm-global:slot', + targetVersion: '2.0.0', + targetIntegrity: `sha512-${Buffer.alloc(64, 4).toString('base64')}`, + ownsCandidate: true, + launchBarrier: { + connect: async () => assert.fail('settlement must not connect'), + pause: () => events.push('pause'), + retireExcept: async () => { + events.push('retire'); + }, + resume: () => events.push('resume'), + release: () => events.push('release'), + }, + retireTarget: async () => assert.fail('an owned committed target must not retire'), + readRecord: async () => { + reads += 1; + if (reads === 1) throw new Error('fsync confirmation unavailable'); + return { + schemaVersion: 1, + rootId: ROOT_ID, + revision: '00000000-0000-4000-8000-000000000000', + state: { + kind: 'owned', + owner: { kind: 'cli', installationId: 'npm-global:slot' }, + selected: { + kind: 'npm_registry', + version: '2.0.0', + integrity: `sha512-${Buffer.alloc(64, 4).toString('base64')}`, + }, + }, + }; + }, + }, + { + retryRead: async () => { + events.push('retry-read'); + }, + }, + ); + + assert.equal(settlement, 'committed'); + assert.equal(reads, 2); + assert.deepEqual(events, ['retry-read', 'release']); +}); + +test('permanent invalid durable ownership fails closed without retrying', async () => { + let retries = 0; + await assert.rejects( + settleTargetFromDurableAuthority( + { + connection: {} as never, + expectedRootId: ROOT_ID, + ownerInstallationId: 'npm-global:slot', + targetVersion: '2.0.0', + targetIntegrity: `sha512-${Buffer.alloc(64, 4).toString('base64')}`, + ownsCandidate: true, + launchBarrier: { + connect: async () => assert.fail('settlement must not connect'), + pause: () => assert.fail('an unreadable record must not guess retirement'), + retireExcept: async () => assert.fail('an unreadable record must not guess retirement'), + resume: () => assert.fail('an unreadable record must not resume admission'), + release: () => assert.fail('an unreadable record must not release admission'), + }, + retireTarget: async () => assert.fail('an unreadable record must not guess retirement'), + readRecord: async () => { + throw new LocalHostDeploymentAuthorityError( + 'invalid_record', + 'The owner record is malformed', + ); + }, + }, + { + retryRead: async () => { + retries += 1; + }, + }, + ), + (error: unknown) => + error instanceof RuntimeHostDurableSettlementError && error.code === 'invalid_record', + ); + assert.equal(retries, 0); +}); + +test('persistent durable authority I/O fails closed after its bounded deadline', async () => { + let now = 1_000; + let reads = 0; + const delays: number[] = []; + await assert.rejects( + settleTargetFromDurableAuthority( + { + connection: {} as never, + expectedRootId: ROOT_ID, + ownerInstallationId: 'npm-global:slot', + targetVersion: '2.0.0', + targetIntegrity: `sha512-${Buffer.alloc(64, 4).toString('base64')}`, + ownsCandidate: true, + launchBarrier: { + connect: async () => assert.fail('settlement must not connect'), + pause: () => assert.fail('an unreadable record must not guess retirement'), + retireExcept: async () => assert.fail('an unreadable record must not guess retirement'), + resume: () => assert.fail('an unreadable record must not resume admission'), + release: () => assert.fail('an unreadable record must not release admission'), + }, + retireTarget: async () => assert.fail('an unreadable record must not guess retirement'), + readRecord: async () => { + reads += 1; + throw new LocalHostDeploymentAuthorityError( + 'authority_io_failed', + 'The owner record cannot be read', + ); + }, + }, + { + now: () => now, + retryRead: async (delayMs) => { + delays.push(delayMs); + now += delayMs; + }, + timeoutMs: 250, + }, + ), + (error: unknown) => + error instanceof RuntimeHostDurableSettlementError && error.code === 'authority_unavailable', + ); + assert.equal(reads, 4); + assert.deepEqual(delays, [100, 100, 50]); +}); + +test('a readable uncommitted handoff retires the guarded target', async () => { + const events: string[] = []; + const settlement = await settleTargetFromDurableAuthority({ + connection: {} as never, + expectedRootId: ROOT_ID, + ownerInstallationId: 'npm-global:slot', + targetVersion: '2.0.0', + targetIntegrity: `sha512-${Buffer.alloc(64, 4).toString('base64')}`, + ownsCandidate: true, + launchBarrier: { + connect: async () => assert.fail('settlement must not connect'), + pause: () => events.push('pause'), + retireExcept: async () => { + events.push('retire'); + }, + resume: () => events.push('resume'), + release: () => events.push('release'), + }, + retireTarget: async () => assert.fail('the launch barrier owns this candidate'), + readRecord: async () => ({ + schemaVersion: 1, + rootId: ROOT_ID, + revision: '00000000-0000-4000-8000-000000000000', + state: { + kind: 'handoff', + transactionId: 'transaction', + from: { kind: 'cli', installationId: 'npm-global:slot' }, + to: { kind: 'cli', installationId: 'npm-global:slot' }, + selected: { + kind: 'npm_registry', + version: '1.0.0', + integrity: `sha512-${Buffer.alloc(64, 3).toString('base64')}`, + }, + target: { + kind: 'npm_registry', + version: '2.0.0', + integrity: `sha512-${Buffer.alloc(64, 4).toString('base64')}`, + }, + }, + }), + }); + + assert.equal(settlement, 'retired'); + assert.deepEqual(events, ['pause', 'retire']); +}); + function registration(overrides: Partial = {}): HostRegistration { return { kind: 'maka-runtime-host', diff --git a/packages/cli/src/__tests__/runtime-host-installed-update-coordinator.test.ts b/packages/cli/src/__tests__/runtime-host-installed-update-coordinator.test.ts index 778c302dd5..f4a3ece19a 100644 --- a/packages/cli/src/__tests__/runtime-host-installed-update-coordinator.test.ts +++ b/packages/cli/src/__tests__/runtime-host-installed-update-coordinator.test.ts @@ -134,8 +134,8 @@ test('retires with the current package, activates with the target, then switches assert.deepEqual(input.target, target); return { kind: 'ready', - settle: async (outcome) => { - events.push(`settle-target:${outcome}`); + settle: async () => { + events.push('settle-target'); }, }; }, @@ -183,7 +183,7 @@ test('retires with the current package, activates with the target, then switches 'switch-global-package', 'verify-new-installation', 'commit-owner', - 'settle-target:committed', + 'settle-target', ]); }); @@ -253,8 +253,8 @@ test('crash-retry observes its own staged target and never retires or re-activat events.push('activate-target'); return { kind: 'ready', - settle: async (outcome) => { - events.push(`settle-target:${outcome}`); + settle: async () => { + events.push('settle-target'); }, }; }, @@ -300,7 +300,7 @@ test('crash-retry observes its own staged target and never retires or re-activat 'activate-target', 'switch-global-package', 'commit-owner', - 'settle-target:committed', + 'settle-target', ]); }); @@ -357,8 +357,8 @@ test('crash-retry with the global package already switched skips the second inst events.push('activate-target'); return { kind: 'ready', - settle: async (outcome) => { - events.push(`settle-target:${outcome}`); + settle: async () => { + events.push('settle-target'); }, }; }, @@ -399,11 +399,11 @@ test('crash-retry with the global package already switched skips the second inst 'close-observed-target', 'activate-target', 'commit-owner', - 'settle-target:committed', + 'settle-target', ]); }); -test('aborts the short-lived target activator when durable ownership cannot commit', async () => { +test('asks the target activator to adjudicate an uncertain durable commit', async () => { const events: string[] = []; const installation = { owner: OWNER, @@ -444,8 +444,8 @@ test('aborts the short-lived target activator when durable ownership cannot comm connectExisting: async () => ({ kind: 'unavailable', reason: 'not_registered' }), activateTarget: async () => ({ kind: 'ready', - settle: async (outcome) => { - events.push(`settle-target:${outcome}`); + settle: async () => { + events.push('settle-target'); }, }), reconcile: (async (_request: unknown, lifecycle: RuntimeHostLocalProcessLifecycleAdapter) => { @@ -468,7 +468,7 @@ test('aborts the short-lived target activator when durable ownership cannot comm }, ); assert.equal(exitCode, 1); - assert.deepEqual(events, ['settle-target:abort']); + assert.deepEqual(events, ['settle-target']); }); test('rejects extended tar headers before the final global npm switch can spawn', async (t) => { diff --git a/packages/cli/src/__tests__/runtime-host-local-handoff.test.ts b/packages/cli/src/__tests__/runtime-host-local-handoff.test.ts index 559d60f564..d1d3eaf0c6 100644 --- a/packages/cli/src/__tests__/runtime-host-local-handoff.test.ts +++ b/packages/cli/src/__tests__/runtime-host-local-handoff.test.ts @@ -333,25 +333,14 @@ test('explicit npm-global restart claims an exact staged legacy takeover', async resolveCandidate: async () => TARGET, withPackage: async (_candidate, use) => use(sourcePackageRoot), prepareDeployment: prepareRuntimeHostPackageDeployment, - connectOrSpawn: async (input) => { - launchedEntrypoint = String(input.candidateEntrypoint); - const registration = hostRegistration({ - hostEpoch: 'new-host', - pid: 84, - generation: input.generation, - }); + connectExisting: async () => incompatibleHost(hostRegistration()), + activateTarget: async (input) => { + launchedEntrypoint = input.staged.candidateEntrypoint; return { - kind: 'connected', - registration, - spawnedProcess: { - pid: 84, - exited: new Promise(() => undefined), + kind: 'ready', + settle: async () => { + closed += 1; }, - connection: { - close: async () => { - closed += 1; - }, - } as never, }; }, }, @@ -390,20 +379,10 @@ test('legacy restart reports active work without claiming deployment authority', resolveCandidate: async () => TARGET, withPackage: async (_candidate, use) => use(sourcePackageRoot), prepareDeployment: prepareRuntimeHostPackageDeployment, - connectOrSpawn: async () => ({ - kind: 'incompatible', - registration: observed, - handshake: { - kind: 'incompatible', - hostEpoch: observed.hostEpoch, - protocolMin: 0, - protocolMax: 0, - compatibilityEpoch: observed.compatibilityEpoch, - compositionId: observed.compositionId, - compositionRevision: observed.compositionRevision, - state: 'ready', - replacement: 'blocked_by_residency', - }, + connectExisting: async () => incompatibleHost(observed), + activateTarget: async () => ({ + kind: 'active_work', + settle: async () => undefined, }), }, ); @@ -412,6 +391,341 @@ test('legacy restart reports active work without claiming deployment authority', assert.equal(await readLocalHostDeploymentRecord(ROOT_ID, { authorityRoot }), undefined); }); +test('external npm replacement retires through the exact source package and commits the target', async (t) => { + const base = await mkdtemp(join(tmpdir(), 'maka-local-external-reconcile-')); + t.after(() => rm(base, { recursive: true, force: true })); + const sourcePackageRoot = await selfContainedPackage(base, PREVIOUS.version, { + sourceRetirementHelper: true, + }); + const targetPackageRoot = await selfContainedPackage(base, TARGET.version); + const authorityRoot = join(base, 'authority'); + await stageSelectedPackage(base, sourcePackageRoot); + const claimed = await applyLocalHostDeploymentTransition( + ROOT_ID, + { kind: 'claim', owner: CLI_OWNER, selected: PREVIOUS }, + { authorityRoot }, + ); + assert.equal(claimed.kind, 'applied'); + const events: string[] = []; + + const result = await restartRuntimeHostNpmGlobalDeployment( + { + rootPath: join(base, 'root'), + registration: hostRegistration(), + deploymentPathOptions: { platform: 'linux', homeDir: join(base, 'home') }, + activeWorkPolicy: 'interrupt_active_work', + }, + { authorityRoot }, + { + resolveInstallation: async () => ({ + owner: CLI_OWNER, + observedRelease: { + version: TARGET.version, + packageRoot: targetPackageRoot, + cliPath: join(targetPackageRoot, 'dist', 'cli.js'), + }, + }), + resolveCandidate: async () => TARGET, + withPackage: async (candidate, use) => { + assert.equal(candidate.version, TARGET.version); + return use(targetPackageRoot); + }, + prepareDeployment: prepareRuntimeHostPackageDeployment, + connectExisting: async () => incompatibleHost(hostRegistration()), + retireSource: async (input) => { + assert.match(input.sourceCliPath, /registry-[a-f0-9]{64}\/dist\/cli\.js$/u); + assert.equal(input.expectedHostEpoch, 'old-host'); + assert.equal(input.activeWorkPolicy, 'interrupt_active_work'); + events.push('source-retired'); + return 'prepared'; + }, + activateTarget: async (input) => { + assert.equal(input.target.version, TARGET.version); + events.push('target-ready'); + return { + kind: 'ready', + settle: async () => { + events.push('settled'); + }, + }; + }, + }, + ); + + assert.equal(result.kind, 'completed'); + assert.deepEqual(events, ['source-retired', 'target-ready', 'settled']); + const record = await readLocalHostDeploymentRecord(ROOT_ID, { authorityRoot }); + assert.equal(record?.state.kind, 'owned'); + assert.deepEqual(record?.state.selected, TARGET); +}); + +test('external source active work rolls back without launching the target', async (t) => { + const base = await mkdtemp(join(tmpdir(), 'maka-local-external-active-')); + t.after(() => rm(base, { recursive: true, force: true })); + const sourcePackageRoot = await selfContainedPackage(base, PREVIOUS.version, { + sourceRetirementHelper: true, + }); + const targetPackageRoot = await selfContainedPackage(base, TARGET.version); + const authorityRoot = join(base, 'authority'); + await stageSelectedPackage(base, sourcePackageRoot); + await applyLocalHostDeploymentTransition( + ROOT_ID, + { kind: 'claim', owner: CLI_OWNER, selected: PREVIOUS }, + { authorityRoot }, + ); + + const result = await restartRuntimeHostNpmGlobalDeployment( + { + rootPath: join(base, 'root'), + registration: hostRegistration(), + deploymentPathOptions: { platform: 'linux', homeDir: join(base, 'home') }, + }, + { authorityRoot }, + { + resolveInstallation: async () => ({ + owner: CLI_OWNER, + observedRelease: { + version: TARGET.version, + packageRoot: targetPackageRoot, + cliPath: join(targetPackageRoot, 'dist', 'cli.js'), + }, + }), + resolveCandidate: async () => TARGET, + withPackage: async (candidate, use) => { + assert.equal(candidate.version, TARGET.version); + return use(targetPackageRoot); + }, + prepareDeployment: prepareRuntimeHostPackageDeployment, + connectExisting: async () => incompatibleHost(hostRegistration()), + retireSource: async () => 'active_work', + activateTarget: async () => assert.fail('active source work must prevent target activation'), + }, + ); + + assert.equal(result.kind, 'active_work'); + const record = await readLocalHostDeploymentRecord(ROOT_ID, { authorityRoot }); + assert.deepEqual(record?.state, { kind: 'owned', owner: CLI_OWNER, selected: PREVIOUS }); +}); + +test('pre-helper source release keeps the bounded idle-only takeover path', async (t) => { + const base = await mkdtemp(join(tmpdir(), 'maka-local-external-pre-helper-')); + t.after(() => rm(base, { recursive: true, force: true })); + const sourcePackageRoot = await selfContainedPackage(base, PREVIOUS.version); + const targetPackageRoot = await selfContainedPackage(base, TARGET.version); + const authorityRoot = join(base, 'authority'); + await stageSelectedPackage(base, sourcePackageRoot); + await applyLocalHostDeploymentTransition( + ROOT_ID, + { kind: 'claim', owner: CLI_OWNER, selected: PREVIOUS }, + { authorityRoot }, + ); + let takeoverHostEpoch: string | undefined; + + const result = await restartRuntimeHostNpmGlobalDeployment( + { + rootPath: join(base, 'root'), + registration: hostRegistration(), + deploymentPathOptions: { platform: 'linux', homeDir: join(base, 'home') }, + }, + { authorityRoot }, + { + resolveInstallation: async () => ({ + owner: CLI_OWNER, + observedRelease: { + version: TARGET.version, + packageRoot: targetPackageRoot, + cliPath: join(targetPackageRoot, 'dist', 'cli.js'), + }, + }), + resolveCandidate: async () => TARGET, + withPackage: async (candidate, use) => { + assert.equal(candidate.version, TARGET.version); + return use(targetPackageRoot); + }, + prepareDeployment: prepareRuntimeHostPackageDeployment, + connectExisting: async () => incompatibleHost(hostRegistration()), + retireSource: async () => assert.fail('a pre-helper source must not run the new helper'), + activateTarget: async (input) => { + takeoverHostEpoch = input.takeoverHostEpoch; + return { kind: 'ready', settle: async () => undefined }; + }, + }, + ); + + assert.equal(result.kind, 'completed'); + assert.equal(takeoverHostEpoch, 'old-host'); +}); + +test('source helper failure preserves the durable handoff for recovery', async (t) => { + const base = await mkdtemp(join(tmpdir(), 'maka-local-external-helper-failure-')); + t.after(() => rm(base, { recursive: true, force: true })); + const sourcePackageRoot = await selfContainedPackage(base, PREVIOUS.version, { + sourceRetirementHelper: true, + }); + const targetPackageRoot = await selfContainedPackage(base, TARGET.version); + const authorityRoot = join(base, 'authority'); + await stageSelectedPackage(base, sourcePackageRoot); + await applyLocalHostDeploymentTransition( + ROOT_ID, + { kind: 'claim', owner: CLI_OWNER, selected: PREVIOUS }, + { authorityRoot }, + ); + + const result = await restartRuntimeHostNpmGlobalDeployment( + { + rootPath: join(base, 'root'), + registration: hostRegistration(), + deploymentPathOptions: { platform: 'linux', homeDir: join(base, 'home') }, + }, + { authorityRoot }, + { + resolveInstallation: async () => ({ + owner: CLI_OWNER, + observedRelease: { + version: TARGET.version, + packageRoot: targetPackageRoot, + cliPath: join(targetPackageRoot, 'dist', 'cli.js'), + }, + }), + resolveCandidate: async () => TARGET, + withPackage: async (candidate, use) => { + assert.equal(candidate.version, TARGET.version); + return use(targetPackageRoot); + }, + prepareDeployment: prepareRuntimeHostPackageDeployment, + connectExisting: async () => incompatibleHost(hostRegistration()), + retireSource: async () => { + throw new Error('source helper exited'); + }, + activateTarget: async () => assert.fail('a failed source helper must not launch the target'), + }, + ); + + assert.equal(result.kind, 'recovery_required'); + assert.equal( + result.kind === 'recovery_required' ? result.phase : undefined, + 'prepare_host_cutover', + ); + assert.equal( + (await readLocalHostDeploymentRecord(ROOT_ID, { authorityRoot }))?.state.kind, + 'handoff', + ); +}); + +test('a Host epoch change after confirmation cannot retire the replacement process', async (t) => { + const base = await mkdtemp(join(tmpdir(), 'maka-local-external-host-race-')); + t.after(() => rm(base, { recursive: true, force: true })); + const sourcePackageRoot = await selfContainedPackage(base, PREVIOUS.version, { + sourceRetirementHelper: true, + }); + const targetPackageRoot = await selfContainedPackage(base, TARGET.version); + const authorityRoot = join(base, 'authority'); + await stageSelectedPackage(base, sourcePackageRoot); + await applyLocalHostDeploymentTransition( + ROOT_ID, + { kind: 'claim', owner: CLI_OWNER, selected: PREVIOUS }, + { authorityRoot }, + ); + + const result = await restartRuntimeHostNpmGlobalDeployment( + { + rootPath: join(base, 'root'), + registration: hostRegistration(), + deploymentPathOptions: { platform: 'linux', homeDir: join(base, 'home') }, + }, + { authorityRoot }, + { + resolveInstallation: async () => ({ + owner: CLI_OWNER, + observedRelease: { + version: TARGET.version, + packageRoot: targetPackageRoot, + cliPath: join(targetPackageRoot, 'dist', 'cli.js'), + }, + }), + resolveCandidate: async () => TARGET, + withPackage: async (_candidate, use) => use(targetPackageRoot), + prepareDeployment: prepareRuntimeHostPackageDeployment, + connectExisting: async () => + incompatibleHost(hostRegistration({ hostEpoch: 'replacement-host' })), + retireSource: async () => assert.fail('an unconfirmed replacement Host must not retire'), + activateTarget: async () => assert.fail('an unconfirmed replacement Host must remain'), + }, + ); + + assert.equal(result.kind, 'recovery_required'); + assert.equal( + result.kind === 'recovery_required' ? result.phase : undefined, + 'prepare_host_cutover', + ); + assert.equal( + (await readLocalHostDeploymentRecord(ROOT_ID, { authorityRoot }))?.state.kind, + 'handoff', + ); +}); + +test('external reconciliation asks the activator to adjudicate when npm changes again', async (t) => { + const base = await mkdtemp(join(tmpdir(), 'maka-local-external-installation-race-')); + t.after(() => rm(base, { recursive: true, force: true })); + const sourcePackageRoot = await selfContainedPackage(base, PREVIOUS.version, { + sourceRetirementHelper: true, + }); + const targetPackageRoot = await selfContainedPackage(base, TARGET.version); + const authorityRoot = join(base, 'authority'); + await stageSelectedPackage(base, sourcePackageRoot); + await applyLocalHostDeploymentTransition( + ROOT_ID, + { kind: 'claim', owner: CLI_OWNER, selected: PREVIOUS }, + { authorityRoot }, + ); + let installationReads = 0; + let settlementRequested = false; + + const result = await restartRuntimeHostNpmGlobalDeployment( + { + rootPath: join(base, 'root'), + registration: hostRegistration(), + deploymentPathOptions: { platform: 'linux', homeDir: join(base, 'home') }, + }, + { authorityRoot }, + { + resolveInstallation: async () => { + installationReads += 1; + return { + owner: CLI_OWNER, + observedRelease: { + version: installationReads === 1 ? TARGET.version : '3.0.0', + packageRoot: targetPackageRoot, + cliPath: join(targetPackageRoot, 'dist', 'cli.js'), + }, + }; + }, + resolveCandidate: async () => TARGET, + withPackage: async (candidate, use) => { + assert.equal(candidate.version, TARGET.version); + return use(targetPackageRoot); + }, + prepareDeployment: prepareRuntimeHostPackageDeployment, + connectExisting: async () => incompatibleHost(hostRegistration()), + retireSource: async () => 'prepared', + activateTarget: async () => ({ + kind: 'ready', + settle: async () => { + settlementRequested = true; + }, + }), + }, + ); + + assert.equal(result.kind, 'recovery_required'); + assert.equal(result.kind === 'recovery_required' ? result.phase : undefined, 'finalize_target'); + assert.equal(settlementRequested, true); + assert.equal( + (await readLocalHostDeploymentRecord(ROOT_ID, { authorityRoot }))?.state.kind, + 'handoff', + ); +}); + test('local restart keeps service Hosts under operator authority', async () => { assert.deepEqual( await restartRuntimeHostNpmGlobalDeployment({ @@ -463,7 +777,97 @@ test('committed target conflicting with the observed Host fails closed', async ( ); }); -async function selfContainedPackage(base: string, version: string): Promise { +test('external npm replacement rejects a different durable source owner before staging', async (t) => { + const base = await mkdtemp(join(tmpdir(), 'maka-local-external-owner-mismatch-')); + t.after(() => rm(base, { recursive: true, force: true })); + const authorityRoot = join(base, 'authority'); + await applyLocalHostDeploymentTransition( + ROOT_ID, + { kind: 'claim', owner: DESKTOP_OWNER, selected: PREVIOUS }, + { authorityRoot }, + ); + + await assert.rejects( + restartRuntimeHostNpmGlobalDeployment( + { rootPath: join(base, 'root'), registration: hostRegistration() }, + { authorityRoot }, + { + resolveInstallation: async () => ({ + owner: CLI_OWNER, + observedRelease: { + version: TARGET.version, + packageRoot: base, + cliPath: join(base, 'dist', 'cli.js'), + }, + }), + resolveCandidate: async () => assert.fail('cross-owner replacement must not resolve'), + withPackage: async () => assert.fail('cross-owner replacement must not stage a target'), + connectExisting: async () => assert.fail('cross-owner replacement must not observe a Host'), + }, + ), + (error: unknown) => + error instanceof RuntimeHostLocalHandoffError && error.code === 'source_owner_mismatch', + ); +}); + +test('external npm downgrade is rejected before staging or retirement', async (t) => { + const base = await mkdtemp(join(tmpdir(), 'maka-local-external-downgrade-')); + t.after(() => rm(base, { recursive: true, force: true })); + const authorityRoot = join(base, 'authority'); + await applyLocalHostDeploymentTransition( + ROOT_ID, + { kind: 'claim', owner: CLI_OWNER, selected: PREVIOUS }, + { authorityRoot }, + ); + const downgrade = { + kind: 'npm_registry' as const, + version: '0.9.0', + integrity: `sha512-${Buffer.alloc(64, 9).toString('base64')}`, + }; + + await assert.rejects( + restartRuntimeHostNpmGlobalDeployment( + { rootPath: join(base, 'root'), registration: hostRegistration() }, + { authorityRoot }, + { + resolveInstallation: async () => ({ + owner: CLI_OWNER, + observedRelease: { + version: downgrade.version, + packageRoot: base, + cliPath: join(base, 'dist', 'cli.js'), + }, + }), + resolveCandidate: async () => downgrade, + withPackage: async () => assert.fail('a downgrade must not be staged'), + }, + ), + (error: unknown) => + error instanceof RuntimeHostLocalHandoffError && error.code === 'unsupported_downgrade', + ); +}); + +async function stageSelectedPackage(base: string, sourcePackageRoot: string): Promise { + await stageRuntimeHostNpmGlobalDeploymentTarget( + { + rootId: ROOT_ID, + owner: CLI_OWNER, + target: PREVIOUS, + transactionId: 'selected-source', + }, + { platform: 'linux', homeDir: join(base, 'home') }, + { + withPackage: async (_candidate, use) => use(sourcePackageRoot), + prepareDeployment: prepareRuntimeHostPackageDeployment, + }, + ); +} + +async function selfContainedPackage( + base: string, + version: string, + options: { readonly sourceRetirementHelper?: boolean } = {}, +): Promise { const root = join(base, `source-${version}`); const runtimeHostRoot = join(root, 'node_modules', '@maka', 'runtime-host'); await Promise.all([ @@ -473,6 +877,9 @@ async function selfContainedPackage(base: string, version: string): Promise = {}): HostRegist ...overrides, }; } + +function incompatibleHost(registration: HostRegistration) { + return { + kind: 'incompatible' as const, + registration, + handshake: { + kind: 'incompatible' as const, + hostEpoch: registration.hostEpoch, + protocolMin: 0, + protocolMax: 0, + compatibilityEpoch: registration.compatibilityEpoch, + compositionId: registration.compositionId, + compositionRevision: registration.compositionRevision, + state: 'ready' as const, + replacement: 'blocked_by_residency' as const, + }, + }; +} diff --git a/packages/cli/src/__tests__/runtime-host-local-source-retirement.test.ts b/packages/cli/src/__tests__/runtime-host-local-source-retirement.test.ts new file mode 100644 index 0000000000..9ba930b4c2 --- /dev/null +++ b/packages/cli/src/__tests__/runtime-host-local-source-retirement.test.ts @@ -0,0 +1,141 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import assert from 'node:assert/strict'; +import test from 'node:test'; +import { + INTERACTIVE_RUNTIME_HOST_COMPOSITION_ID, + RUNTIME_HOST_COMPATIBILITY_EPOCH, + RUNTIME_HOST_PROTOCOL_VERSION, + RUNTIME_HOST_REGISTRATION_SCHEMA_VERSION, + type HostRegistration, +} from '@maka/runtime-host/protocol'; +import { runRuntimeHostLocalSourceRetirement } from '../runtime-host-local-source-retirement.js'; + +const ROOT_ID = 'a'.repeat(64); +const INPUT = { + rootPath: '/state', + expectedRootId: ROOT_ID, + expectedHostEpoch: 'source-host', + activeWorkPolicy: 'refuse_active_work' as const, +}; + +test('retires only the exact ephemeral Host through the source package client', async () => { + let closed = false; + let observedMode = ''; + const result = await runRuntimeHostLocalSourceRetirement(INPUT, { + connectExisting: async () => ({ + kind: 'connected', + registration: registration(), + connection: { + close: async () => { + closed = true; + }, + } as never, + }), + prepareRetirement: async (_connection, mode) => { + observedMode = mode; + return { kind: 'prepared', pid: 42 }; + }, + }); + assert.equal(result, 0); + assert.equal(observedMode, 'refuse_active_work'); + assert.equal(closed, true); +}); + +test('preserves active work unless the parent explicitly selected interruption', async () => { + const result = await runRuntimeHostLocalSourceRetirement( + { ...INPUT, activeWorkPolicy: 'interrupt_active_work' }, + { + connectExisting: async () => ({ + kind: 'connected', + registration: registration(), + connection: { close: async () => undefined } as never, + }), + prepareRetirement: async (_connection, mode) => { + assert.equal(mode, 'interrupt_active_work'); + return { kind: 'active_tasks', tasks: [] }; + }, + }, + ); + assert.equal(result, 2); +}); + +test('fails closed when the observed Host changes before retirement', async () => { + let closed = false; + await assert.rejects( + runRuntimeHostLocalSourceRetirement(INPUT, { + connectExisting: async () => ({ + kind: 'connected', + registration: registration({ hostEpoch: 'replacement-host' }), + connection: { + close: async () => { + closed = true; + }, + } as never, + }), + prepareRetirement: async () => assert.fail('a changed Host must not be retired'), + }), + /changed before source-package retirement/u, + ); + assert.equal(closed, true); +}); + +test('keeps service Hosts under operator authority', async () => { + assert.equal( + await runRuntimeHostLocalSourceRetirement(INPUT, { + connectExisting: async () => ({ + kind: 'connected', + registration: registration({ lifecycleMode: 'service' }), + connection: { close: async () => undefined } as never, + }), + prepareRetirement: async () => assert.fail('a service Host must not be retired'), + }), + 4, + ); +}); + +test('does not misclassify an unavailable source Host as operator-owned', async () => { + await assert.rejects( + runRuntimeHostLocalSourceRetirement(INPUT, { + connectExisting: async () => ({ kind: 'unavailable', reason: 'not_registered' }), + }), + /cannot control the observed Runtime Host/u, + ); +}); + +function registration(overrides: Partial = {}): HostRegistration { + return { + kind: 'maka-runtime-host', + schemaVersion: RUNTIME_HOST_REGISTRATION_SCHEMA_VERSION, + rootId: ROOT_ID, + hostEpoch: 'source-host', + endpoint: '/tmp/maka.sock', + protocolMin: RUNTIME_HOST_PROTOCOL_VERSION, + protocolMax: RUNTIME_HOST_PROTOCOL_VERSION, + compatibilityEpoch: RUNTIME_HOST_COMPATIBILITY_EPOCH, + compositionId: INTERACTIVE_RUNTIME_HOST_COMPOSITION_ID, + compositionRevision: 'revision', + lifecycleMode: 'ephemeral', + state: 'ready', + pid: 42, + createdAt: new Date(0).toISOString(), + ...overrides, + }; +} diff --git a/packages/cli/src/__tests__/runtime-host-update-reconciliation.test.ts b/packages/cli/src/__tests__/runtime-host-update-reconciliation.test.ts index 0a8f9e28ce..7315d24017 100644 --- a/packages/cli/src/__tests__/runtime-host-update-reconciliation.test.ts +++ b/packages/cli/src/__tests__/runtime-host-update-reconciliation.test.ts @@ -104,6 +104,38 @@ describe('managed Runtime Host update reconciliation', () => { } }); + it('parses the exact source-package retirement helper contract', () => { + assert.deepEqual( + parseRuntimeHostCommand([ + 'local-source-retire', + '--root', + '/srv/maka', + '--expected-root-id', + TARGET.rootId, + '--expected-host-epoch', + 'source-host', + '--allow-interrupt-active-tasks', + ]), + { + kind: 'runtime-host-local-source-retire', + rootPath: '/srv/maka', + expectedRootId: TARGET.rootId, + expectedHostEpoch: 'source-host', + allowInterruptActiveTasks: true, + }, + ); + assert.equal( + parseRuntimeHostCommand([ + 'local-source-retire', + '--root', + '/srv/maka', + '--expected-root-id', + TARGET.rootId, + ]).kind, + 'error', + ); + }); + it('parses update policy and reconciliation commands against an optional expected target', () => { assert.deepEqual( parseRuntimeHostCommand([ diff --git a/packages/cli/src/cli-core.ts b/packages/cli/src/cli-core.ts index 65eca3ac6f..ba852c55df 100644 --- a/packages/cli/src/cli-core.ts +++ b/packages/cli/src/cli-core.ts @@ -403,6 +403,19 @@ export async function runMakaCli( ...(command.awaitCoordinatorCommit ? { inheritableAuthorityLeaseFd: 4 } : {}), }); } + case 'runtime-host-local-source-retire': { + const { runRuntimeHostLocalSourceRetirement } = await import( + './runtime-host-local-source-retirement.js' + ); + return runRuntimeHostLocalSourceRetirement({ + rootPath: command.rootPath, + expectedRootId: command.expectedRootId, + expectedHostEpoch: command.expectedHostEpoch, + activeWorkPolicy: command.allowInterruptActiveTasks + ? 'interrupt_active_work' + : 'refuse_active_work', + }); + } case 'runtime-host-setup': { const { runRuntimeHostSetupCli } = await import('./runtime-host-setup-command.js'); const { RUNTIME_HOST_SETUP_SOURCE_PACKAGE_INTEGRITY_ENV } = await import( diff --git a/packages/cli/src/runtime-host-cli.ts b/packages/cli/src/runtime-host-cli.ts index 74ae8bedab..fe7392e5f0 100644 --- a/packages/cli/src/runtime-host-cli.ts +++ b/packages/cli/src/runtime-host-cli.ts @@ -87,6 +87,13 @@ export type RuntimeHostCliCommand = targetVersion?: string; targetIntegrity?: string; } + | { + kind: 'runtime-host-local-source-retire'; + rootPath: string; + expectedRootId: string; + expectedHostEpoch: string; + allowInterruptActiveTasks: boolean; + } | { kind: 'runtime-host-serve'; rootPath?: string; @@ -340,6 +347,7 @@ export function parseRuntimeHostCommand(argv: string[]): RuntimeHostCliCommand { } if (argv[0] === 'local-update-apply') return parseLocalUpdateApply(argv.slice(1)); if (argv[0] === 'local-update-activate') return parseLocalUpdateActivate(argv.slice(1)); + if (argv[0] === 'local-source-retire') return parseLocalSourceRetire(argv.slice(1)); if (argv[0] === 'serve') return parseServeCommand(argv.slice(1)); if (argv[0] === 'setup') return parseSetupCommand(argv.slice(1)); if (argv[0] === 'service') return parseServiceManagementCommand(argv.slice(1)); @@ -577,6 +585,45 @@ function parseLocalUpdateActivate(argv: string[]): RuntimeHostCliCommand { }; } +function parseLocalSourceRetire(argv: string[]): RuntimeHostCliCommand { + const values = new Map(); + let allowInterruptActiveTasks = false; + const options = new Set(['--root', '--expected-root-id', '--expected-host-epoch']); + for (let index = 0; index < argv.length; index += 1) { + const argument = argv[index]; + if (argument === '--allow-interrupt-active-tasks') { + if (allowInterruptActiveTasks) return error(`Duplicate ${argument}`); + allowInterruptActiveTasks = true; + continue; + } + if (!argument || !options.has(argument)) return error(`Unexpected argument: ${argument ?? ''}`); + if (values.has(argument)) return error(`Duplicate ${argument}`); + const parsed = optionValue(argv, index, argument); + if (typeof parsed !== 'string') return parsed; + values.set(argument, parsed); + index += 1; + } + const rootPath = values.get('--root'); + const expectedRootId = values.get('--expected-root-id'); + const expectedHostEpoch = values.get('--expected-host-epoch'); + if (!rootPath || !expectedRootId || !expectedHostEpoch) { + return error('runtime-host local-source-retire requires its exact source Host identity'); + } + if (!isSafeAbsolutePath(rootPath)) { + return error('runtime-host local-source-retire root path must be absolute'); + } + if (!/^[a-f0-9]{64}$/u.test(expectedRootId) || !isSafeIdentity(expectedHostEpoch)) { + return error('runtime-host local-source-retire Host identity is invalid'); + } + return { + kind: 'runtime-host-local-source-retire', + rootPath, + expectedRootId, + expectedHostEpoch, + allowInterruptActiveTasks, + }; +} + function isSafeIdentity(value: string): boolean { return value.length <= 512 && !/[\u0000-\u001f\u007f]/u.test(value); } diff --git a/packages/cli/src/runtime-host-installed-update-activator.ts b/packages/cli/src/runtime-host-installed-update-activator.ts index 579e2e8841..24b16c28ac 100644 --- a/packages/cli/src/runtime-host-installed-update-activator.ts +++ b/packages/cli/src/runtime-host-installed-update-activator.ts @@ -26,12 +26,29 @@ import { type RuntimeHostCandidateLaunchBarrier, type RuntimeHostConnection, } from '@maka/runtime-host/client'; -import { readLocalHostDeploymentRecord } from '@maka/runtime-host/operator'; +import { + LocalHostDeploymentAuthorityError, + readLocalHostDeploymentRecord, +} from '@maka/runtime-host/operator'; import { INTERACTIVE_RUNTIME_HOST_COMPOSITION_ID, RUNTIME_HOST_PROTOCOL_VERSION, } from '@maka/runtime-host/protocol'; +const DURABLE_SETTLEMENT_RETRY_MS = 100; +const DURABLE_SETTLEMENT_TIMEOUT_MS = 10_000; + +export class RuntimeHostDurableSettlementError extends Error { + constructor( + readonly code: 'invalid_record' | 'authority_unavailable', + message: string, + options?: ErrorOptions, + ) { + super(message, options); + this.name = 'RuntimeHostDurableSettlementError'; + } +} + export async function runRuntimeHostInstalledUpdateActivator( input: { readonly rootPath: string; @@ -39,7 +56,7 @@ export async function runRuntimeHostInstalledUpdateActivator( readonly generation: string; readonly candidateEntrypoint: string; readonly takeoverHostEpoch?: string; - /** The coordinator keeps this short-lived activator alive through durable commit. */ + /** The short-lived activator adjudicates the durable owner record before exiting. */ readonly awaitCoordinatorCommit?: boolean; readonly expectedOwnerInstallationId?: string; readonly targetVersion?: string; @@ -72,7 +89,7 @@ export async function runRuntimeHostInstalledUpdateActivator( }); if (result.kind === 'connected') { let exactTarget = false; - let commitWaitOwnsAbort = false; + let durableSettlementOwnsRetirement = false; try { if ( result.registration.rootId !== input.expectedRootId || @@ -87,9 +104,8 @@ export async function runRuntimeHostInstalledUpdateActivator( if (!input.expectedOwnerInstallationId || !input.targetVersion || !input.targetIntegrity) { throw new Error('The activator is missing its exact durable commit expectation'); } - commitWaitOwnsAbort = true; + durableSettlementOwnsRetirement = true; await (overrides.awaitCoordinatorCommit ?? awaitCoordinatorCommit)({ - registration: result.registration, connection: result.connection, expectedRootId: input.expectedRootId, ownerInstallationId: input.expectedOwnerInstallationId, @@ -105,7 +121,7 @@ export async function runRuntimeHostInstalledUpdateActivator( } return 0; } catch (error) { - if (exactTarget && !commitWaitOwnsAbort) { + if (exactTarget && !durableSettlementOwnsRetirement) { await retireUncommittedTarget({ connection: result.connection, ownsCandidate: result.spawnedProcess !== undefined, @@ -127,7 +143,6 @@ export async function runRuntimeHostInstalledUpdateActivator( } interface CoordinatorCommitWaitInput { - readonly registration: { readonly pid: number }; readonly connection: RuntimeHostConnection; readonly expectedRootId: string; readonly ownerInstallationId: string; @@ -153,8 +168,13 @@ type UncommittedTargetInput = Pick< */ async function awaitCoordinatorCommit(input: CoordinatorCommitWaitInput): Promise { if (typeof process.send !== 'function' || !process.connected) { - await retireUncommittedTarget(input); - throw new Error('The installed update activator lost its coordinator channel'); + const settlement = await settleTargetFromDurableAuthority(input); + if (settlement === 'retired') { + throw new Error( + 'The installed update activator lost its coordinator before ownership committed', + ); + } + return; } await new Promise((resolve, reject) => { let settled = false; @@ -170,33 +190,19 @@ async function awaitCoordinatorCommit(input: CoordinatorCommitWaitInput): Promis }; const onMessage = (message: unknown) => { if (!isCoordinatorMessage(message)) return; - if (message.kind === 'committed') { - settle(async () => { - if (!(await isCommittedTarget(input))) { - await retireUncommittedTarget(input); - throw new Error( - 'The target activation was acknowledged before durable ownership committed', - ); - } - input.launchBarrier.release(); - }); - } - if (message.kind === 'abort') { - settle(async () => { - await retireUncommittedTarget(input); - throw new Error( - 'The installed update coordinator aborted before durable ownership committed', - ); - }); - } + settle(async () => { + const settlement = await settleTargetFromDurableAuthority(input); + if (settlement === 'retired') { + throw new Error('The installed update coordinator settled without durable ownership'); + } + }); }; const onDisconnect = () => { settle(async () => { - if (await isCommittedTarget(input)) return; - await retireUncommittedTarget(input); - throw new Error( - 'The installed update coordinator exited before durable ownership committed', - ); + const settlement = await settleTargetFromDurableAuthority(input); + if (settlement === 'retired') { + throw new Error('The installed update coordinator exited before ownership committed'); + } }); }; process.on('message', onMessage); @@ -205,18 +211,73 @@ async function awaitCoordinatorCommit(input: CoordinatorCommitWaitInput): Promis }); } -function isCoordinatorMessage(value: unknown): value is { readonly kind: 'committed' | 'abort' } { +function isCoordinatorMessage(value: unknown): value is { readonly kind: 'settle' } { return ( - typeof value === 'object' && - value !== null && - (value as { kind?: unknown }).kind !== undefined && - ((value as { kind?: unknown }).kind === 'committed' || - (value as { kind?: unknown }).kind === 'abort') + typeof value === 'object' && value !== null && (value as { kind?: unknown }).kind === 'settle' ); } -async function isCommittedTarget(input: CoordinatorCommitWaitInput): Promise { - const record = await input.readRecord(input.expectedRootId); +/** + * Makes the activator the only authority for target release versus retirement. + * Read failures are ambiguous, so retain the launch barrier and inherited lease + * while retrying transient uncertainty. Permanent corruption and a bounded + * transient-read timeout fail closed: the activator exits without releasing the + * barrier, and the launch-owner guard retires the inaccessible candidate. + */ +export async function settleTargetFromDurableAuthority( + input: CoordinatorCommitWaitInput, + options: { + readonly now?: () => number; + readonly retryRead?: (delayMs: number) => Promise; + readonly timeoutMs?: number; + } = {}, +): Promise<'committed' | 'retired'> { + const now = options.now ?? (() => performance.now()); + const retryRead = + options.retryRead ?? + ((delayMs: number) => new Promise((resolve) => setTimeout(resolve, delayMs))); + const deadline = now() + (options.timeoutMs ?? DURABLE_SETTLEMENT_TIMEOUT_MS); + let record: Awaited>; + for (;;) { + try { + record = await input.readRecord(input.expectedRootId); + break; + } catch (error) { + if ( + error instanceof LocalHostDeploymentAuthorityError && + error.code !== 'authority_io_failed' + ) { + throw new RuntimeHostDurableSettlementError( + error.code === 'invalid_record' ? 'invalid_record' : 'authority_unavailable', + error.code === 'invalid_record' + ? 'The local Runtime Host update requires recovery because its durable owner record is invalid' + : 'The local Runtime Host update requires recovery because its durable owner record cannot be read safely', + { cause: error }, + ); + } + const remainingMs = deadline - now(); + if (remainingMs <= 0) { + throw new RuntimeHostDurableSettlementError( + 'authority_unavailable', + 'The local Runtime Host update requires recovery because its durable owner record remained unavailable', + { cause: error }, + ); + } + await retryRead(Math.min(DURABLE_SETTLEMENT_RETRY_MS, remainingMs)); + } + } + if (isCommittedTarget(record, input)) { + input.launchBarrier.release(); + return 'committed'; + } + await retireUncommittedTarget(input); + return 'retired'; +} + +function isCommittedTarget( + record: Awaited>, + input: CoordinatorCommitWaitInput, +): boolean { return ( record?.state.kind === 'owned' && record.state.owner.kind === 'cli' && diff --git a/packages/cli/src/runtime-host-installed-update-coordinator.ts b/packages/cli/src/runtime-host-installed-update-coordinator.ts index 7b083528a0..fac9d753a4 100644 --- a/packages/cli/src/runtime-host-installed-update-coordinator.ts +++ b/packages/cli/src/runtime-host-installed-update-coordinator.ts @@ -38,7 +38,6 @@ import { resolveStorageRoot } from '@maka/storage/root-authority'; import { prepareRuntimeHostNpmGlobalStagedDeployment, reconcilePreparedRuntimeHostNpmGlobalDeployment, - type RuntimeHostLocalStagedDeployment, } from './runtime-host-local-handoff.js'; import { resolveRuntimeHostNpmGlobalInstallation, @@ -49,6 +48,11 @@ import { assertRuntimeHostArchiveExpansionBudget, withVerifiedRuntimeHostUpdateArchive, } from './runtime-host-update-package.js'; +import { + launchRuntimeHostTargetActivator, + type RuntimeHostTargetActivation, + type RuntimeHostTargetActivationInput, +} from './runtime-host-local-target-activation.js'; const NPM_TIMEOUT_MS = 5 * 60_000; const NPM_OUTPUT_MAX_BYTES = 64 * 1024; @@ -69,22 +73,6 @@ interface RuntimeHostInstalledUpdateCoordinatorDeps { readonly installArchive: typeof installRuntimeHostNpmGlobalArchive; } -interface RuntimeHostTargetActivationInput { - readonly rootPath: string; - readonly rootId: string; - readonly staged: RuntimeHostLocalStagedDeployment; - readonly ownerInstallationId: string; - readonly target: RuntimeHostUpdateCandidate; - readonly takeoverHostEpoch?: string; - readonly inheritableAuthorityLeaseFd: number; -} - -interface RuntimeHostTargetActivation { - readonly kind: 'ready' | 'active_work' | 'operator_required'; - /** Releases the short-lived child only after durable owner settlement. */ - settle(outcome: 'committed' | 'abort'): Promise; -} - export interface RuntimeHostInstalledUpdateCoordinatorInput { readonly rootPath: string; readonly archivePath: string; @@ -109,7 +97,7 @@ export async function runRuntimeHostInstalledUpdateCoordinator( withArchive: withVerifiedRuntimeHostUpdateArchive, prepareStaged: prepareRuntimeHostNpmGlobalStagedDeployment, reconcile: reconcilePreparedRuntimeHostNpmGlobalDeployment, - activateTarget: launchTargetActivator, + activateTarget: launchRuntimeHostTargetActivator, installArchive: installRuntimeHostNpmGlobalArchive, ...overrides, }; @@ -209,7 +197,7 @@ export async function runRuntimeHostInstalledUpdateCoordinator( const unreachable = async (): Promise => { throw new Error('The exact target activator must settle local Host cutover'); }; - let result: Awaited>; + let result: Awaited> | undefined; try { result = await deps.reconcile( { @@ -245,13 +233,14 @@ export async function runRuntimeHostInstalledUpdateCoordinator( }, authorityOptions, ); - if (result.kind === 'completed' && targetActivator) { - await targetActivator.settle('committed'); - targetActivator = undefined; - } } finally { - if (targetActivator) await targetActivator.settle('abort').catch(() => undefined); + if (targetActivator) { + const settlement = targetActivator.settle(); + if (result?.kind === 'completed') await settlement; + else await settlement.catch(() => undefined); + } } + if (!result) throw new Error('The installed update transaction produced no result'); if (result.kind === 'completed') { process.stdout.write(`Updated Maka to ${input.target.version}.\n`); return 0; @@ -365,93 +354,6 @@ export async function installRuntimeHostNpmGlobalArchive( }); } -function launchTargetActivator( - input: RuntimeHostTargetActivationInput, -): Promise { - const args = [ - input.staged.cliPath, - 'runtime-host', - 'local-update-activate', - '--root', - input.rootPath, - '--expected-root-id', - input.rootId, - '--generation', - input.staged.launchGeneration, - '--candidate-entrypoint', - input.staged.candidateEntrypoint, - '--await-coordinator-commit', - 'true', - '--expected-owner-installation-id', - input.ownerInstallationId, - '--target-version', - input.target.version, - '--target-integrity', - input.target.integrity, - ...(input.takeoverHostEpoch ? ['--takeover-host-epoch', input.takeoverHostEpoch] : []), - ]; - return new Promise((resolve, reject) => { - const child = spawn(process.execPath, args, { - // fd 3 is the coordinator channel; fd 4 is the inherited authority - // lease. The activator, not the long-lived target, owns that lease. - stdio: ['inherit', 'inherit', 'inherit', 'ipc', input.inheritableAuthorityLeaseFd], - windowsHide: false, - }); - let ready = false; - let settled = false; - const closed = new Promise<{ code: number | null; signal: NodeJS.Signals | null }>( - (resolveClosed) => { - child.once('close', (code, signal) => resolveClosed({ code, signal })); - }, - ); - const closeError = async (): Promise => { - const { code, signal } = await closed; - if (signal) throw new Error(`Maka target activator exited on ${signal}`); - if (code === 3) throw new Error('The activated Runtime Host still owns active work'); - if (code === 4) throw new Error('The observed Runtime Host requires its operator'); - throw new Error('The exact Maka target could not be activated'); - }; - const settle = async (outcome: 'committed' | 'abort'): Promise => { - if (settled) return; - settled = true; - if (child.connected) { - await new Promise((resolveSent, rejectSent) => { - child.send({ kind: outcome }, (error) => (error ? rejectSent(error) : resolveSent())); - }); - } - const exited = await closed; - if (outcome === 'committed' && (exited.signal || exited.code !== 0)) { - if (exited.signal) throw new Error(`Maka target activator exited on ${exited.signal}`); - throw new Error('The exact Maka target activator did not confirm durable ownership'); - } - }; - child.once('error', reject); - child.on('message', (message: unknown) => { - if (ready || !isTargetActivatorReadyMessage(message)) return; - ready = true; - resolve({ kind: 'ready', settle }); - }); - void closed.then(({ code, signal }) => { - if (ready) return; - if (signal) { - reject(new Error(`Maka target activator exited on ${signal}`)); - } else if (code === 3) { - resolve({ kind: 'active_work', settle: async () => undefined }); - } else if (code === 4) { - resolve({ kind: 'operator_required', settle: async () => undefined }); - } else { - void closeError().catch(reject); - } - }); - }); -} - -function isTargetActivatorReadyMessage(value: unknown): value is { readonly kind: 'ready' } { - return ( - typeof value === 'object' && value !== null && (value as { kind?: unknown }).kind === 'ready' - ); -} - function updateTransactionId( rootId: string, installation: RuntimeHostNpmGlobalInstallation, diff --git a/packages/cli/src/runtime-host-local-handoff.ts b/packages/cli/src/runtime-host-local-handoff.ts index f1695ea08c..12961a7998 100644 --- a/packages/cli/src/runtime-host-local-handoff.ts +++ b/packages/cli/src/runtime-host-local-handoff.ts @@ -20,7 +20,7 @@ import { createHash, randomUUID } from 'node:crypto'; import { realpath, stat } from 'node:fs/promises'; import { homedir } from 'node:os'; -import { join, posix, resolve, win32 } from 'node:path'; +import { dirname, join, posix, resolve, win32 } from 'node:path'; import { claimLocalHostProcessDeployment, handoffLocalHostProcessDeployment, @@ -32,7 +32,8 @@ import { type LocalHostProcessDeploymentHandoffResult, type RuntimeHostInstallationOwner, } from '@maka/runtime-host/operator'; -import { connectOrSpawnRuntimeHost, runtimeHostStartupError } from '@maka/runtime-host/client'; +import { compareProductReleaseVersions } from '@maka/runtime-host/operator/update-package-evidence'; +import { connectExistingRuntimeHost } from '@maka/runtime-host/client'; import { INTERACTIVE_RUNTIME_HOST_COMPOSITION_ID, RUNTIME_HOST_PROTOCOL_VERSION, @@ -43,7 +44,9 @@ import { type RuntimeHostNpmGlobalInstallation, } from './runtime-host-cli-installation.js'; import { + openRuntimeHostPackageDeployment, prepareRuntimeHostPackageDeployment, + resolveRuntimeHostPackageCliPath, type RuntimeHostPackageDeployment, } from './runtime-host-package-deployment.js'; import { @@ -52,6 +55,12 @@ import { } from './runtime-host-update-package.js'; import type { RuntimeHostUpdateCandidate } from './runtime-host-update-discovery.js'; import { resolveRuntimeHostRegistryUpdateCandidate } from './runtime-host-update-discovery.js'; +import { + launchRuntimeHostTargetActivator, + type RuntimeHostTargetActivation, + type RuntimeHostTargetActivationInput, +} from './runtime-host-local-target-activation.js'; +import { launchRuntimeHostLocalSourceRetirement } from './runtime-host-local-source-retirement.js'; const ROOT_ID = /^[a-f0-9]{64}$/u; const CANDIDATE_RELATIVE_PATH = [ @@ -73,7 +82,9 @@ export class RuntimeHostLocalHandoffError extends Error { readonly code: | 'installed_release_mismatch' | 'root_changed' - | 'selected_target_observation_conflict', + | 'selected_target_observation_conflict' + | 'source_owner_mismatch' + | 'unsupported_downgrade', message: string, ) { super(message); @@ -97,7 +108,11 @@ interface RuntimeHostLocalHandoffDeps { interface RuntimeHostLocalRestartDeps extends RuntimeHostLocalHandoffDeps { readonly resolveCandidate: typeof resolveRuntimeHostRegistryUpdateCandidate; - readonly connectOrSpawn: typeof connectOrSpawnRuntimeHost; + readonly connectExisting: typeof connectExistingRuntimeHost; + readonly activateTarget: ( + input: RuntimeHostTargetActivationInput, + ) => Promise; + readonly retireSource: typeof launchRuntimeHostLocalSourceRetirement; } export type RuntimeHostLocalProcessLifecycleAdapter = Omit< @@ -128,8 +143,9 @@ export type RuntimeHostNpmGlobalRestartResult = /** * Explicitly restarts one local ephemeral Host from the exact artifact matching - * the installed npm-global CLI. The released-Host takeover is a bounded adapter: - * it can replace only the observed exact Host when that Host reports true idle. + * the installed npm-global CLI. A durable selected source may retire its exact + * Host through its own compatible client; released Hosts without that helper + * retain only the bounded true-idle takeover adapter. */ export async function restartRuntimeHostNpmGlobalDeployment( input: { @@ -137,6 +153,7 @@ export async function restartRuntimeHostNpmGlobalDeployment( readonly registration: HostRegistration; readonly installationOptions?: Parameters[0]; readonly deploymentPathOptions?: RuntimeHostLocalDeploymentPathOptions; + readonly activeWorkPolicy?: 'refuse_active_work' | 'interrupt_active_work'; }, authorityOptions: LocalHostDeploymentAuthorityOptions = {}, overrides: Partial = {}, @@ -155,15 +172,24 @@ export async function restartRuntimeHostNpmGlobalDeployment( claim: claimLocalHostProcessDeployment, handoff: handoffLocalHostProcessDeployment, resolveCandidate: resolveRuntimeHostRegistryUpdateCandidate, - connectOrSpawn: connectOrSpawnRuntimeHost, + connectExisting: connectExistingRuntimeHost, + activateTarget: launchRuntimeHostTargetActivator, + retireSource: launchRuntimeHostLocalSourceRetirement, ...overrides, }; const installation = await deps.resolveInstallation(input.installationOptions); + const current = await deps.readRecord(input.registration.rootId, authorityOptions); + const sourceOwner = current?.state.kind === 'handoff' ? current.state.from : current?.state.owner; + if (sourceOwner && !sameOwner(sourceOwner, installation.owner)) { + throw new RuntimeHostLocalHandoffError( + 'source_owner_mismatch', + 'External npm replacement can reconcile only the same durable npm-global installation owner', + ); + } const target = await deps.resolveCandidate({ kind: 'exact', version: installation.observedRelease.version, }); - const current = await deps.readRecord(input.registration.rootId, authorityOptions); if ( current?.state.kind === 'owned' && current.state.owner.kind === installation.owner.kind && @@ -177,13 +203,66 @@ export async function restartRuntimeHostNpmGlobalDeployment( 'The active Runtime Host conflicts with the deployment already committed for this installation', ); } + if ( + current && + compareProductReleaseVersions(target.version, current.state.selected.version) < 0 + ) { + throw new RuntimeHostLocalHandoffError( + 'unsupported_downgrade', + `Downgrading the local Runtime Host from ${current.state.selected.version} to ${target.version} is not supported`, + ); + } const transactionId = restartTransactionId(input.registration.rootId, installation.owner, target); - let connectedTarget: - | Extract>, { kind: 'connected' }> - | undefined; + const activeWorkPolicy = input.activeWorkPolicy ?? 'refuse_active_work'; + const staged = await stageRuntimeHostNpmGlobalDeploymentTarget( + { + rootId: input.registration.rootId, + owner: installation.owner, + target, + transactionId, + }, + input.deploymentPathOptions, + deps, + ); + const selected = current?.state.selected; + const sourceIdentity = selected && !sameDeployment(selected, target) ? selected : undefined; + const source = sourceIdentity + ? await openRuntimeHostNpmGlobalStagedDeployment( + { + rootId: input.registration.rootId, + owner: installation.owner, + target: sourceIdentity, + transactionId: `${transactionId}:source`, + }, + input.deploymentPathOptions, + ) + : undefined; + const sourceSupportsRetirement = source ? await hasSourceRetirementHelper(source) : false; + let targetActivator: RuntimeHostTargetActivation | undefined; + const activateExactTarget = async ( + rootId: string, + stagedTarget: RuntimeHostLocalStagedDeployment, + inheritableAuthorityLeaseFd: number, + takeoverHostEpoch?: string, + ): Promise<'target_present' | 'active_work' | 'operator_required'> => { + const activated = await deps.activateTarget({ + rootPath: input.rootPath, + rootId, + staged: stagedTarget, + ownerInstallationId: installation.owner.installationId, + target, + inheritableAuthorityLeaseFd, + ...(takeoverHostEpoch ? { takeoverHostEpoch } : {}), + }); + if (activated.kind !== 'ready') return activated.kind; + targetActivator = activated; + return 'target_present'; + }; const prepare = async ( rootId: string, - staged: RuntimeHostLocalStagedDeployment, + selectedDeployment: RuntimeHostUpdateCandidate | undefined, + stagedTarget: RuntimeHostLocalStagedDeployment, + inheritableAuthorityLeaseFd: number, ): Promise<{ readonly kind: 'target_present' | 'active_work' }> => { if (rootId !== input.registration.rootId) { throw new RuntimeHostLocalHandoffError( @@ -191,74 +270,146 @@ export async function restartRuntimeHostNpmGlobalDeployment( 'The local Runtime Host State Root changed before restart', ); } - const result = await deps.connectOrSpawn({ + const observed = await deps.connectExisting({ rootPath: input.rootPath, protocol: { min: RUNTIME_HOST_PROTOCOL_VERSION, max: RUNTIME_HOST_PROTOCOL_VERSION }, compositionId: INTERACTIVE_RUNTIME_HOST_COMPOSITION_ID, - generation: staged.launchGeneration, - takeoverHostEpoch: input.registration.hostEpoch, clientInstanceId: randomUUID(), - candidateEntrypoint: staged.candidateEntrypoint, }); - if (result.kind === 'connected') { - if ( - result.registration.rootId !== rootId || - result.registration.generation !== staged.launchGeneration || - (result.spawnedProcess !== undefined && - result.spawnedProcess.pid !== result.registration.pid) - ) { - await result.connection.close().catch(() => undefined); - throw new Error('The restarted Runtime Host does not match the exact staged process'); + const registration = observed.registration; + if (registration?.rootId !== undefined && registration.rootId !== rootId) { + if (observed.kind === 'connected') await observed.connection.close().catch(() => undefined); + throw new RuntimeHostLocalHandoffError( + 'root_changed', + 'The local Runtime Host State Root changed during restart', + ); + } + if (observed.kind === 'connected') await observed.connection.close(); + if (!registration) { + const activated = await activateExactTarget( + rootId, + stagedTarget, + inheritableAuthorityLeaseFd, + ); + if (activated !== 'target_present') { + throw new Error('The exact target could not start after the source Host disappeared'); + } + return { kind: 'target_present' }; + } + if (registration.lifecycleMode !== 'ephemeral') { + throw new Error('The observed Runtime Host requires its operator to perform replacement'); + } + if (registration.generation === stagedTarget.launchGeneration) { + const activated = await activateExactTarget( + rootId, + stagedTarget, + inheritableAuthorityLeaseFd, + ); + if (activated !== 'target_present') { + throw new Error('The exact staged Runtime Host could not be reattached for recovery'); } - connectedTarget = result; return { kind: 'target_present' }; } + if (registration.hostEpoch !== input.registration.hostEpoch) { + throw new Error('The observed Runtime Host changed after restart confirmation'); + } if ( - (result.kind === 'incompatible' || result.kind === 'upgrade_required') && - result.registration.hostEpoch === input.registration.hostEpoch + source && + sourceSupportsRetirement && + selectedDeployment && + sourceIdentity && + sameDeployment(selectedDeployment, sourceIdentity) ) { - return { kind: 'active_work' }; + const retired = await deps.retireSource({ + sourceCliPath: source.cliPath, + rootPath: input.rootPath, + expectedRootId: rootId, + expectedHostEpoch: registration.hostEpoch, + activeWorkPolicy, + inheritableAuthorityLeaseFd, + }); + if (retired === 'active_work') return { kind: 'active_work' }; + if (retired === 'operator_required') { + throw new Error('The observed Runtime Host requires its operator to perform replacement'); + } + const activated = await activateExactTarget( + rootId, + stagedTarget, + inheritableAuthorityLeaseFd, + registration.hostEpoch, + ); + if (activated !== 'target_present') { + throw new Error('The exact target did not activate after source retirement began'); + } + return { kind: 'target_present' }; } - if (result.kind === 'failed') { - throw runtimeHostStartupError(result.reason, result.diagnostic); + const activated = await activateExactTarget( + rootId, + stagedTarget, + inheritableAuthorityLeaseFd, + registration.hostEpoch, + ); + if (activated === 'active_work') return { kind: 'active_work' }; + if (activated === 'operator_required') { + throw new Error('The observed Runtime Host requires its operator to perform replacement'); } - throw new Error('The observed Runtime Host changed before exact local restart completed'); + return { kind: 'target_present' }; }; const unreachable = async (): Promise => { - throw new Error('Legacy local restart must converge during exact takeover'); + throw new Error('Local restart must converge through its exact target activator'); }; + let result: + | LocalHostProcessDeploymentClaimResult + | LocalHostProcessDeploymentHandoffResult + | undefined; try { - return await reconcileRuntimeHostNpmGlobalDeployment( + result = await reconcilePreparedRuntimeHostNpmGlobalDeployment( { rootId: input.registration.rootId, transactionId, target, - activeWorkPolicy: 'refuse_active_work', - ...(input.installationOptions ? { installationOptions: input.installationOptions } : {}), + activeWorkPolicy, + installation, + staged, ...(input.deploymentPathOptions ? { deploymentPathOptions: input.deploymentPathOptions } : {}), }, { - prepareUnownedHostCutover: (rootId, _target, staged) => prepare(rootId, staged), - prepareHostCutover: (rootId, _selected, _target, staged) => prepare(rootId, staged), + prepareUnownedHostCutover: (rootId, _target, stagedTarget, _policy, leaseFd) => + prepare(rootId, undefined, stagedTarget, leaseFd), + prepareHostCutover: (rootId, selectedDeployment, _target, stagedTarget, _policy, leaseFd) => + prepare(rootId, selectedDeployment, stagedTarget, leaseFd), observeWriterRelease: unreachable, activateTarget: unreachable, - async verifyTargetReady(rootId, _target, staged) { + async verifyTargetReady() { + if (targetActivator?.kind !== 'ready') { + throw new Error('Exact restarted Runtime Host Ready evidence is unavailable'); + } + }, + async finalizeTarget() { + const observedInstallation = await deps.resolveInstallation(input.installationOptions); if ( - connectedTarget?.registration.rootId !== rootId || - connectedTarget.registration.generation !== staged.launchGeneration + observedInstallation.owner.installationId !== installation.owner.installationId || + observedInstallation.observedRelease.version !== target.version ) { - throw new Error('Exact restarted Runtime Host Ready evidence is unavailable'); + throw new RuntimeHostLocalHandoffError( + 'installed_release_mismatch', + 'The installed Maka release changed again before local Host ownership committed', + ); } - await connectedTarget.connection.close(); }, }, authorityOptions, deps, ); + return result; } finally { - await connectedTarget?.connection.close().catch(() => undefined); + if (targetActivator) { + const settlement = targetActivator.settle(); + if (result?.kind === 'completed') await settlement; + else await settlement.catch(() => undefined); + } } } @@ -419,6 +570,37 @@ export async function prepareRuntimeHostNpmGlobalStagedDeployment( }; } +async function openRuntimeHostNpmGlobalStagedDeployment( + input: { + readonly rootId: string; + readonly owner: RuntimeHostInstallationOwner & { readonly kind: 'cli' }; + readonly target: RuntimeHostUpdateCandidate; + readonly transactionId: string; + }, + pathOptions: RuntimeHostLocalDeploymentPathOptions = {}, +): Promise { + const deploymentRoot = resolveRuntimeHostLocalCliDeploymentRoot( + input.rootId, + input.owner, + pathOptions, + ); + const staged = await openRuntimeHostPackageDeployment({ + deploymentRoot, + cliPath: resolveRuntimeHostPackageCliPath( + deploymentRoot, + input.target.version, + input.target.integrity, + ), + version: input.target.version, + }); + const candidateEntrypoint = await requireCandidateEntrypoint(staged.packageRoot); + return { + ...staged, + candidateEntrypoint, + launchGeneration: launchGeneration(input.transactionId, input.target), + }; +} + export function resolveRuntimeHostLocalCliDeploymentRoot( rootId: string, owner: RuntimeHostInstallationOwner & { readonly kind: 'cli' }, @@ -458,6 +640,35 @@ async function requireCandidateEntrypoint(packageRoot: string): Promise return candidate; } +async function hasSourceRetirementHelper( + source: RuntimeHostLocalStagedDeployment, +): Promise { + try { + return ( + await stat(join(dirname(source.cliPath), 'runtime-host-local-source-retirement.js')) + ).isFile(); + } catch (error) { + if (isNodeError(error, 'ENOENT')) return false; + throw error; + } +} + +function sameDeployment( + left: RuntimeHostUpdateCandidate, + right: RuntimeHostUpdateCandidate, +): boolean { + return ( + left.kind === right.kind && left.version === right.version && left.integrity === right.integrity + ); +} + +function sameOwner( + left: RuntimeHostInstallationOwner, + right: RuntimeHostInstallationOwner, +): boolean { + return left.kind === right.kind && left.installationId === right.installationId; +} + function launchGeneration(transactionId: string, target: RuntimeHostUpdateCandidate): string { return `npm-global-handoff:${createHash('sha256') .update(transactionId) @@ -489,3 +700,7 @@ function restartTransactionId( function invalidStagedPackage(message: string, cause?: unknown): RuntimeHostUpdatePackageError { return new RuntimeHostUpdatePackageError('invalid_package', message, { cause }); } + +function isNodeError(error: unknown, code: string): error is NodeJS.ErrnoException { + return error instanceof Error && 'code' in error && error.code === code; +} diff --git a/packages/cli/src/runtime-host-local-source-retirement.ts b/packages/cli/src/runtime-host-local-source-retirement.ts new file mode 100644 index 0000000000..390d6b0059 --- /dev/null +++ b/packages/cli/src/runtime-host-local-source-retirement.ts @@ -0,0 +1,129 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { spawn } from 'node:child_process'; +import { + connectExistingRuntimeHost, + prepareConnectedRuntimeHostRetirement, +} from '@maka/runtime-host/client'; +import { + INTERACTIVE_RUNTIME_HOST_COMPOSITION_ID, + RUNTIME_HOST_PROTOCOL_VERSION, +} from '@maka/runtime-host/protocol'; + +const SOURCE_RETIREMENT_TIMEOUT_MS = 60_000; + +interface RuntimeHostLocalSourceRetirementDeps { + readonly connectExisting: typeof connectExistingRuntimeHost; + readonly prepareRetirement: typeof prepareConnectedRuntimeHostRetirement; +} + +export interface RuntimeHostLocalSourceRetirementInput { + readonly rootPath: string; + readonly expectedRootId: string; + readonly expectedHostEpoch: string; + readonly activeWorkPolicy: 'refuse_active_work' | 'interrupt_active_work'; +} + +/** + * Runs inside the exact selected source package so its client protocol remains + * compatible with the Host that package launched. The parent owns deployment + * authority; this helper only exercises the authenticated retirement contract. + */ +export async function runRuntimeHostLocalSourceRetirement( + input: RuntimeHostLocalSourceRetirementInput, + overrides: Partial = {}, +): Promise { + const connectExisting = overrides.connectExisting ?? connectExistingRuntimeHost; + const prepareRetirement = overrides.prepareRetirement ?? prepareConnectedRuntimeHostRetirement; + const connected = await connectExisting({ + rootPath: input.rootPath, + protocol: { min: RUNTIME_HOST_PROTOCOL_VERSION, max: RUNTIME_HOST_PROTOCOL_VERSION }, + compositionId: INTERACTIVE_RUNTIME_HOST_COMPOSITION_ID, + }); + if (connected.kind !== 'connected') { + if (connected.registration && connected.registration.lifecycleMode !== 'ephemeral') return 4; + throw new Error('The selected source package cannot control the observed Runtime Host'); + } + try { + if ( + connected.registration.rootId !== input.expectedRootId || + connected.registration.hostEpoch !== input.expectedHostEpoch + ) { + throw new Error('The Runtime Host changed before source-package retirement'); + } + if (connected.registration.lifecycleMode !== 'ephemeral') return 4; + const prepared = await prepareRetirement(connected.connection, input.activeWorkPolicy); + if (prepared.kind === 'active_tasks') return 2; + if (prepared.pid !== connected.registration.pid) { + throw new Error('The Runtime Host process changed while source retirement was prepared'); + } + return 0; + } finally { + await connected.connection.close().catch(() => undefined); + } +} + +export function launchRuntimeHostLocalSourceRetirement(input: { + readonly sourceCliPath: string; + readonly rootPath: string; + readonly expectedRootId: string; + readonly expectedHostEpoch: string; + readonly activeWorkPolicy: 'refuse_active_work' | 'interrupt_active_work'; + readonly inheritableAuthorityLeaseFd: number; +}): Promise<'prepared' | 'active_work' | 'operator_required'> { + const args = [ + input.sourceCliPath, + 'runtime-host', + 'local-source-retire', + '--root', + input.rootPath, + '--expected-root-id', + input.expectedRootId, + '--expected-host-epoch', + input.expectedHostEpoch, + ...(input.activeWorkPolicy === 'interrupt_active_work' + ? ['--allow-interrupt-active-tasks'] + : []), + ]; + return new Promise((resolve, reject) => { + const child = spawn(process.execPath, args, { + // fd 4 inherits the existing owner-authority lease. The source helper + // holds it only while the authenticated retirement request is in flight. + stdio: ['ignore', 'ignore', 'inherit', 'ignore', input.inheritableAuthorityLeaseFd], + timeout: SOURCE_RETIREMENT_TIMEOUT_MS, + killSignal: 'SIGKILL', + windowsHide: false, + }); + child.once('error', reject); + child.once('close', (code, signal) => { + if (signal) { + reject(new Error(`Maka source retirement helper exited on ${signal}`)); + } else if (code === 0) { + resolve('prepared'); + } else if (code === 2) { + resolve('active_work'); + } else if (code === 4) { + resolve('operator_required'); + } else { + reject(new Error('The selected Maka source package could not retire its Runtime Host')); + } + }); + }); +} diff --git a/packages/cli/src/runtime-host-local-target-activation.ts b/packages/cli/src/runtime-host-local-target-activation.ts new file mode 100644 index 0000000000..c321d0c194 --- /dev/null +++ b/packages/cli/src/runtime-host-local-target-activation.ts @@ -0,0 +1,144 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { spawn } from 'node:child_process'; +import type { RuntimeHostLocalStagedDeployment } from './runtime-host-local-handoff.js'; +import type { RuntimeHostUpdateCandidate } from './runtime-host-registry-update.js'; + +export interface RuntimeHostTargetActivationInput { + readonly rootPath: string; + readonly rootId: string; + readonly staged: RuntimeHostLocalStagedDeployment; + readonly ownerInstallationId: string; + readonly target: RuntimeHostUpdateCandidate; + readonly takeoverHostEpoch?: string; + readonly inheritableAuthorityLeaseFd: number; +} + +export interface RuntimeHostTargetActivation { + readonly kind: 'ready' | 'active_work' | 'operator_required'; + /** Asks the short-lived child to adjudicate the durable owner record. */ + settle(): Promise; +} + +export class RuntimeHostTargetActivationError extends Error { + constructor( + readonly code: 'recovery_required', + message: string, + options?: ErrorOptions, + ) { + super(message, options); + this.name = 'RuntimeHostTargetActivationError'; + } +} + +/** + * Launches the exact staged target through its own release and keeps the + * existing owner-authority lease in the activator until durable commit. + */ +export function launchRuntimeHostTargetActivator( + input: RuntimeHostTargetActivationInput, +): Promise { + const args = [ + input.staged.cliPath, + 'runtime-host', + 'local-update-activate', + '--root', + input.rootPath, + '--expected-root-id', + input.rootId, + '--generation', + input.staged.launchGeneration, + '--candidate-entrypoint', + input.staged.candidateEntrypoint, + '--await-coordinator-commit', + 'true', + '--expected-owner-installation-id', + input.ownerInstallationId, + '--target-version', + input.target.version, + '--target-integrity', + input.target.integrity, + ...(input.takeoverHostEpoch ? ['--takeover-host-epoch', input.takeoverHostEpoch] : []), + ]; + return new Promise((resolve, reject) => { + const child = spawn(process.execPath, args, { + // fd 3 is the coordinator channel; fd 4 is the inherited authority + // lease. The activator, not the long-lived target, owns that lease. + stdio: ['inherit', 'inherit', 'inherit', 'ipc', input.inheritableAuthorityLeaseFd], + windowsHide: false, + }); + let ready = false; + let settled = false; + const closed = new Promise<{ code: number | null; signal: NodeJS.Signals | null }>( + (resolveClosed) => { + child.once('close', (code, signal) => resolveClosed({ code, signal })); + }, + ); + const closeError = async (): Promise => { + const { code, signal } = await closed; + if (signal) throw new Error(`Maka target activator exited on ${signal}`); + if (code === 3) throw new Error('The activated Runtime Host still owns active work'); + if (code === 4) throw new Error('The observed Runtime Host requires its operator'); + throw new Error('The exact Maka target could not be activated'); + }; + const settle = async (): Promise => { + if (settled) return; + settled = true; + if (child.connected) { + await new Promise((resolveSent, rejectSent) => { + child.send({ kind: 'settle' }, (error) => (error ? rejectSent(error) : resolveSent())); + }); + } + const exited = await closed; + if (exited.signal || exited.code !== 0) { + throw new RuntimeHostTargetActivationError( + 'recovery_required', + exited.signal + ? `The local Runtime Host update requires recovery because its target activator exited on ${exited.signal}` + : 'The local Runtime Host update requires recovery because its target activator could not confirm durable ownership', + ); + } + }; + child.once('error', reject); + child.on('message', (message: unknown) => { + if (ready || !isTargetActivatorReadyMessage(message)) return; + ready = true; + resolve({ kind: 'ready', settle }); + }); + void closed.then(({ code, signal }) => { + if (ready) return; + if (signal) { + reject(new Error(`Maka target activator exited on ${signal}`)); + } else if (code === 3) { + resolve({ kind: 'active_work', settle: async () => undefined }); + } else if (code === 4) { + resolve({ kind: 'operator_required', settle: async () => undefined }); + } else { + void closeError().catch(reject); + } + }); + }); +} + +function isTargetActivatorReadyMessage(value: unknown): value is { readonly kind: 'ready' } { + return ( + typeof value === 'object' && value !== null && (value as { kind?: unknown }).kind === 'ready' + ); +} diff --git a/packages/runtime-host/src/__tests__/fixtures/owned-authority-launcher.ts b/packages/runtime-host/src/__tests__/fixtures/owned-authority-launcher.ts index 2d85f37d26..6e67dcd363 100644 --- a/packages/runtime-host/src/__tests__/fixtures/owned-authority-launcher.ts +++ b/packages/runtime-host/src/__tests__/fixtures/owned-authority-launcher.ts @@ -20,9 +20,11 @@ import { openSync } from 'node:fs'; import { launchOwnedRuntimeHostCandidate } from '../../client/launcher.js'; -const [rootPath, expectedRootId, leasePath] = process.argv.slice(2); -if (!rootPath || !expectedRootId || !leasePath) { - throw new Error('usage: owned-authority-launcher '); +const [rootPath, expectedRootId, leasePath, clientInstanceId] = process.argv.slice(2); +if (!rootPath || !expectedRootId || !leasePath || !clientInstanceId) { + throw new Error( + 'usage: owned-authority-launcher ', + ); } const leaseFd = openSync(leasePath, 'a+'); @@ -32,6 +34,10 @@ const attempt = await launchOwnedRuntimeHostCandidate({ entrypoint: new URL('../../execution-candidate-main.js', import.meta.url), idleGraceMs: 10_000, inheritableAuthorityLeaseFd: leaseFd, + launchOwnerClientInstanceId: clientInstanceId, }).spawned; process.send?.({ type: 'launched', pid: attempt.pid }); +process.on('message', (message) => { + if (message === 'release') attempt.releaseToEnvironment(); +}); await new Promise(() => undefined); diff --git a/packages/runtime-host/src/__tests__/host-kernel.test.ts b/packages/runtime-host/src/__tests__/host-kernel.test.ts index 3cb9781e0a..e7c3499bcd 100644 --- a/packages/runtime-host/src/__tests__/host-kernel.test.ts +++ b/packages/runtime-host/src/__tests__/host-kernel.test.ts @@ -1845,22 +1845,35 @@ describe('non-serving Runtime Host kernel', () => { test('an authority-supervised Candidate exits if its launch owner is killed', async () => { await withHostPaths(async (paths) => { const capability = await resolveStorageRoot({ path: paths.root, kind: 'interactive' }); + const launchOwnerClientInstanceId = 'authority-launch-owner'; const launcher = paths.resources.trackChild( fork( new URL('./fixtures/owned-authority-launcher.js', import.meta.url), - [paths.root, capability.rootId, join(paths.base, 'authority-lease-probe')], + [ + paths.root, + capability.rootId, + join(paths.base, 'authority-lease-probe'), + launchOwnerClientInstanceId, + ], { stdio: ['ignore', 'ignore', 'inherit', 'ipc'] }, ), ); const launchedPid = paths.resources.trackPid(await waitForLaunch(launcher)); - let connected = await retryConnect(paths, CURRENT_PROTOCOL); - if (connected.kind !== 'connected') { - connected = await retryConnect(paths, CURRENT_PROTOCOL); - } + const connected = await retryConnect(paths, CURRENT_PROTOCOL, { + clientInstanceId: launchOwnerClientInstanceId, + }); assert.equal(connected.kind, 'connected'); if (connected.kind !== 'connected') return; assert.equal(connected.registration.pid, launchedPid); + const ordinary = await connectRuntimeHost({ + ...paths, + rootPath: paths.root, + protocol: CURRENT_PROTOCOL, + clientInstanceId: 'ordinary-client-during-finalization', + }); + assert.equal(ordinary.kind, 'draining'); + launcher.kill('SIGKILL'); await waitForExit(launcher); await withTimeout( @@ -1873,6 +1886,58 @@ describe('non-serving Runtime Host kernel', () => { }); }); + test('a committed authority-supervised Candidate admits ordinary Clients after release', async () => { + await withHostPaths(async (paths) => { + const capability = await resolveStorageRoot({ path: paths.root, kind: 'interactive' }); + const launchOwnerClientInstanceId = 'committed-authority-launch-owner'; + const launcher = paths.resources.trackChild( + fork( + new URL('./fixtures/owned-authority-launcher.js', import.meta.url), + [ + paths.root, + capability.rootId, + join(paths.base, 'committed-authority-lease-probe'), + launchOwnerClientInstanceId, + ], + { stdio: ['ignore', 'ignore', 'inherit', 'ipc'] }, + ), + ); + const launchedPid = paths.resources.trackPid(await waitForLaunch(launcher)); + const owner = await retryConnect(paths, CURRENT_PROTOCOL, { + clientInstanceId: launchOwnerClientInstanceId, + }); + assert.equal(owner.kind, 'connected'); + if (owner.kind !== 'connected') return; + + const beforeCommit = await connectRuntimeHost({ + ...paths, + rootPath: paths.root, + protocol: CURRENT_PROTOCOL, + clientInstanceId: 'ordinary-client-before-commit', + }); + assert.equal(beforeCommit.kind, 'draining'); + + launcher.send('release'); + const ordinary = await retryConnect(paths, CURRENT_PROTOCOL, { + clientInstanceId: 'ordinary-client-after-commit', + }); + assert.equal(ordinary.kind, 'connected'); + if (ordinary.kind === 'connected') { + assert.equal( + (await ordinary.connection.request('host.diagnostics.query', {})).pid, + launchedPid, + ); + await ordinary.connection.close(); + } + await owner.connection.close(); + launcher.kill('SIGKILL'); + await waitForExit(launcher); + terminateProcess(launchedPid); + await waitForProcessExit(launchedPid); + paths.resources.forgetPid(launchedPid); + }); + }); + test('a detached Candidate survives writing stderr after its launcher exits', async () => { await withHostPaths(async (paths) => { const capability = await resolveStorageRoot({ path: paths.root, kind: 'interactive' }); @@ -3283,7 +3348,11 @@ function isConnectedClientMessage( async function retryConnect( paths: HostPaths, protocol: { min: number; max: number }, - options?: { livenessIntervalMs?: number; onLivenessProbe?: () => void }, + options?: { + livenessIntervalMs?: number; + onLivenessProbe?: () => void; + clientInstanceId?: string; + }, ) { const deadline = Date.now() + 5_000; let result = await connectRuntimeHost({ diff --git a/packages/runtime-host/src/candidate-entry.ts b/packages/runtime-host/src/candidate-entry.ts index ba46642f1b..c3e6854c72 100644 --- a/packages/runtime-host/src/candidate-entry.ts +++ b/packages/runtime-host/src/candidate-entry.ts @@ -68,7 +68,12 @@ export async function runExecutionCandidateEntry( const { startupAttemptId: parsedStartupAttemptId, ...options } = parsed; startupAttemptId = parsedStartupAttemptId; result = await startExecutionRuntimeHostCandidate( - hooks.overrideOptions ? hooks.overrideOptions(options) : options, + { + ...(hooks.overrideOptions ? hooks.overrideOptions(options) : options), + ...(launchOwnerGuard?.admission + ? { initialClientAdmission: launchOwnerGuard.admission } + : {}), + }, { ...hooks.dependencies, processLaunch: { diff --git a/packages/runtime-host/src/candidate-launch-owner-guard.ts b/packages/runtime-host/src/candidate-launch-owner-guard.ts index 45dbf4f3fd..a9653ac0b0 100644 --- a/packages/runtime-host/src/candidate-launch-owner-guard.ts +++ b/packages/runtime-host/src/candidate-launch-owner-guard.ts @@ -21,13 +21,19 @@ import { closeSync } from 'node:fs'; export const RUNTIME_HOST_LAUNCH_OWNER_LEASE_FD_ENV = 'MAKA_RUNTIME_HOST_LAUNCH_OWNER_LEASE_FD'; export const RUNTIME_HOST_LAUNCH_OWNER_GUARD_ENV = 'MAKA_RUNTIME_HOST_LAUNCH_OWNER_GUARD'; +export const RUNTIME_HOST_LAUNCH_OWNER_CLIENT_ID_ENV = 'MAKA_RUNTIME_HOST_LAUNCH_OWNER_CLIENT_ID'; export const RUNTIME_HOST_LAUNCH_OWNER_RELEASE_KIND = 'runtime-host-launch-owner-release'; export interface RuntimeHostLaunchOwnerGuard { + readonly admission?: RuntimeHostLaunchOwnerAdmission; bind(closeHost: () => Promise): void; dispose(): Promise; } +export interface RuntimeHostLaunchOwnerAdmission { + isClientAdmitted(clientInstanceId: string): boolean; +} + /** * Closes a launcher-owned Host if its launcher disappears. An optional updater * authority lease stays inside the Candidate until the launcher explicitly @@ -46,6 +52,18 @@ export function createRuntimeHostLaunchOwnerGuard( throw new Error('Runtime Host launch-owner authority descriptor is invalid'); } } + const clientInstanceId = env[RUNTIME_HOST_LAUNCH_OWNER_CLIENT_ID_ENV]; + if (leaseFd !== undefined && !clientInstanceId) { + throw new Error('Runtime Host launch-owner Client identity is missing'); + } + const admission = + leaseFd === undefined + ? undefined + : { + isClientAdmitted(candidateClientInstanceId: string) { + return state === 'released' || candidateClientInstanceId === clientInstanceId; + }, + }; let state: 'owned' | 'released' | 'lost' = process.connected ? 'owned' : 'lost'; let closeHost: (() => Promise) | undefined; @@ -88,6 +106,7 @@ export function createRuntimeHostLaunchOwnerGuard( process.once('disconnect', onDisconnect); return { + ...(admission ? { admission } : {}), bind(close) { closeHost = close; settleLoss(); diff --git a/packages/runtime-host/src/client/connect-or-spawn.ts b/packages/runtime-host/src/client/connect-or-spawn.ts index 7a992708d2..23274e7915 100644 --- a/packages/runtime-host/src/client/connect-or-spawn.ts +++ b/packages/runtime-host/src/client/connect-or-spawn.ts @@ -467,7 +467,10 @@ export async function connectOrSpawnRuntimeHostWithDependencies( ...(input.onExit === undefined ? {} : { onExit: input.onExit }), ...(input.inheritableAuthorityLeaseFd === undefined ? {} - : { inheritableAuthorityLeaseFd: input.inheritableAuthorityLeaseFd }), + : { + inheritableAuthorityLeaseFd: input.inheritableAuthorityLeaseFd, + launchOwnerClientInstanceId: clientInstanceId, + }), ...(input.closeOnLauncherExit === undefined ? {} : { closeOnLauncherExit: input.closeOnLauncherExit }), diff --git a/packages/runtime-host/src/client/launcher.ts b/packages/runtime-host/src/client/launcher.ts index a8bdd15ce3..2b80ee1266 100644 --- a/packages/runtime-host/src/client/launcher.ts +++ b/packages/runtime-host/src/client/launcher.ts @@ -27,6 +27,7 @@ import { } from '../candidate-startup-failure.js'; import { RUNTIME_HOST_LAUNCH_OWNER_GUARD_ENV, + RUNTIME_HOST_LAUNCH_OWNER_CLIENT_ID_ENV, RUNTIME_HOST_LAUNCH_OWNER_LEASE_FD_ENV, runtimeHostLaunchOwnerReleaseMessage, } from '../candidate-launch-owner-guard.js'; @@ -56,6 +57,8 @@ export interface DetachedCandidateInput { inheritableAuthorityLeaseFd?: number; /** Keep this Candidate bound to the launcher process for its whole lifetime. */ closeOnLauncherExit?: boolean; + /** Opaque Client identity admitted while the launch-owner lease remains held. */ + launchOwnerClientInstanceId?: string; /** Called with the candidate's exit details; the embedder owns the sink. */ readonly onExit?: (details: CandidateExitDetails) => void; } @@ -106,6 +109,9 @@ export function launchOwnedRuntimeHostCandidate(input: DetachedCandidateInput): const startupAttemptId = randomUUID(); const guarded = input.inheritableAuthorityLeaseFd !== undefined || input.closeOnLauncherExit === true; + if (input.inheritableAuthorityLeaseFd !== undefined && !input.launchOwnerClientInstanceId) { + throw new Error('A launch-owner-supervised Candidate requires its Client identity'); + } const child = spawnCandidate(input, false, startupAttemptId, guarded); const exited = observeCandidateExit(child); notifyCandidateExit(child, exited, input.onExit); @@ -185,7 +191,10 @@ function spawnCandidate( ...(guarded ? { [RUNTIME_HOST_LAUNCH_OWNER_GUARD_ENV]: '1' } : {}), ...(childLeaseFd === undefined ? {} - : { [RUNTIME_HOST_LAUNCH_OWNER_LEASE_FD_ENV]: String(childLeaseFd) }), + : { + [RUNTIME_HOST_LAUNCH_OWNER_LEASE_FD_ENV]: String(childLeaseFd), + [RUNTIME_HOST_LAUNCH_OWNER_CLIENT_ID_ENV]: input.launchOwnerClientInstanceId!, + }), }, }); const stderr = child.stderr as (NodeJS.ReadableStream & { unref?: () => void }) | null; diff --git a/packages/runtime-host/src/server/candidate.ts b/packages/runtime-host/src/server/candidate.ts index d31afe5555..16e79335da 100644 --- a/packages/runtime-host/src/server/candidate.ts +++ b/packages/runtime-host/src/server/candidate.ts @@ -39,6 +39,10 @@ export interface InteractiveRuntimeHostCandidateOptions { handshakeTimeoutMs?: number; generation?: string; managedLaunchClaim?: RuntimeHostManagedLaunchClaim; + /** Limits pre-commit admission for a launch-owner-supervised Candidate. */ + initialClientAdmission?: { + isClientAdmitted(clientInstanceId: string): boolean; + }; } export interface InteractiveRuntimeHostCandidateDependencies { @@ -91,6 +95,9 @@ export async function startInteractiveRuntimeHostCandidate( handshakeTimeoutMs: options.handshakeTimeoutMs, generation: options.generation, composition, + ...(options.initialClientAdmission + ? { initialClientAdmission: options.initialClientAdmission } + : {}), ...(accessAuthority ? { accessAuthority } : {}), ...(websocket && accessAuthority ? { diff --git a/packages/runtime-host/src/server/host-kernel.ts b/packages/runtime-host/src/server/host-kernel.ts index 664c46ec61..e86705ce28 100644 --- a/packages/runtime-host/src/server/host-kernel.ts +++ b/packages/runtime-host/src/server/host-kernel.ts @@ -160,6 +160,10 @@ interface RuntimeHostKernelCommonOptions { listenerSetFactory?: RuntimeHostListenerSetFactory; accessAuthority?: RuntimeHostAccessAuthority; peerMesh?: PeerMeshNode; + /** Ephemeral launch gate used until a supervised Candidate durably commits. */ + initialClientAdmission?: { + isClientAdmitted(clientInstanceId: string): boolean; + }; } export type RuntimeHostLifecycleMode = 'ephemeral' | 'service'; @@ -477,6 +481,18 @@ export class RuntimeHostKernel { compositionRevision: this.compositionDescriptor.revision, }; } + const initialClientAdmission = this.#options.initialClientAdmission; + if ( + initialClientAdmission && + !initialClientAdmission.isClientAdmitted(hello.clientInstanceId) + ) { + return { + kind: 'draining', + hostEpoch: this.hostEpoch, + compositionId: this.compositionDescriptor.id, + compositionRevision: this.compositionDescriptor.revision, + }; + } if (authority.clientInstanceId && authority.clientInstanceId !== hello.clientInstanceId) { throw new Error('Runtime Host access credential belongs to another Client'); }