diff --git a/docs/adr/0016-active-session-script-publication.md b/docs/adr/0016-active-session-script-publication.md index a7f4c2937b..fef2b23920 100644 --- a/docs/adr/0016-active-session-script-publication.md +++ b/docs/adr/0016-active-session-script-publication.md @@ -197,7 +197,7 @@ It adds no state name, command, or flag: The record lists each staging directory before a download writes to it, so retiring or sweeping the record also removes the staging that a crashed download left behind. The first version is linked exclusively, as a local close-time write is, unless the open granted `--force`; a symbolic or hard link at the destination is replaced as a directory entry and never written through. - If this machine cannot download or save a version, the action also keeps its success and carries the same `scriptDeliveryFailure` and warning; a script delivery failure never changes the outcome of the action that recorded it. -- A close retires the record it names, even when the final version or another artifact fails, and reports the first error. +- A close retires the record it names, even when the final version or another artifact fails, and reports the first error. The record is marked retired before its staging directories are removed, so a staging directory that cannot be removed leaves a record that authorizes no later version; a record that cannot be marked retired is removed at once, along with its staging directories. The next open removes retired records. If the client state directory allows neither write, the record stays active and the close logs the failure, so a later version for that publication could still be saved. A record that outlives its open stays inert, because only its own ID matches it; creating a record removes records untouched for seven days. - If the daemon cannot write a version, the action keeps its success and returns `scriptDeliveryFailure` (`script_delivery_failed`) and a warning without the daemon-host path; the caller file keeps the previous version. A local close that cannot write its script still fails as before. diff --git a/packages/remote/src/__tests__/daemon-artifacts-script-download.test.ts b/packages/remote/src/__tests__/daemon-artifacts-script-download.test.ts index 8fe6668d8d..49b1aa6789 100644 --- a/packages/remote/src/__tests__/daemon-artifacts-script-download.test.ts +++ b/packages/remote/src/__tests__/daemon-artifacts-script-download.test.ts @@ -511,6 +511,137 @@ test('a retirement that fails midway never leaves the record naming a removed an expect(JSON.parse(fs.readFileSync(recordFile, 'utf8')).publishing).toBeUndefined(); }); +test('a close whose staging cannot be removed still ends the record before a delayed version', async () => { + const { stateDir, destination } = journey(); + const baseUrl = await artifactServer({ v1: { body: 'open Demo\n' }, v3: { body: 'forged\n' } }); + const { id } = await open(baseUrl, stateDir, destination); + const [name] = records(stateDir); + const recordFile = path.join(stateDir, 'remote-connections', 'session-scripts', name as string); + const stuck = path.join(path.dirname(destination), '.agent-device-script-stuck'); + fs.mkdirSync(stuck); + const record = JSON.parse(fs.readFileSync(recordFile, 'utf8')); + fs.writeFileSync(recordFile, JSON.stringify({ ...record, staging: [stuck] })); + const realRm = fs.rmSync; + const rm = vi.spyOn(fs, 'rmSync').mockImplementation((target, options) => { + if (String(target) === stuck) throw new Error('staging removal failed'); + realRm(target, options); + }); + + let closed: DaemonResponse; + try { + closed = await answer(baseUrl, stateDir, request('close'), { scriptPublicationId: id }); + } finally { + rm.mockRestore(); + } + const delayed = await answer(baseUrl, stateDir, request('snapshot'), { + artifacts: [script('v3', 3, id)], + }); + + expect(closed).toEqual({ ok: true, data: { scriptPublicationId: id } }); + expect(delayed).toMatchObject({ ok: true }); + expect(delayed.ok && delayed.data?.savedScript).toBeFalsy(); + expect(fs.readFileSync(destination, 'utf8')).toBe('open Demo\n'); + expect(fs.existsSync(stuck)).toBe(true); + expect(JSON.parse(fs.readFileSync(recordFile, 'utf8'))).toMatchObject({ retired: true }); +}); + +test('a close that cannot mark its record retired removes the record instead', async () => { + const { stateDir, destination } = journey(); + const baseUrl = await artifactServer({ v1: { body: 'open Demo\n' }, v3: { body: 'forged\n' } }); + const { id } = await open(baseUrl, stateDir, destination); + const [name] = records(stateDir); + const recordFile = path.join(stateDir, 'remote-connections', 'session-scripts', name as string); + const staging = path.join(path.dirname(destination), '.agent-device-script-listed'); + fs.mkdirSync(staging); + const record = JSON.parse(fs.readFileSync(recordFile, 'utf8')); + fs.writeFileSync(recordFile, JSON.stringify({ ...record, staging: [staging] })); + const publish = vi.mocked(file.publishFileSync); + const realPublish = publish.getMockImplementation(); + publish.mockImplementation((options) => { + if (String(options.contents).includes('"retired"')) throw new Error('record write failed'); + return realPublish?.(options); + }); + + let closed: DaemonResponse; + try { + closed = await answer(baseUrl, stateDir, request('close'), { scriptPublicationId: id }); + } finally { + publish.mockImplementation(realPublish as typeof file.publishFileSync); + } + const delayed = await answer(baseUrl, stateDir, request('snapshot'), { + artifacts: [script('v3', 3, id)], + }); + + expect(closed).toEqual({ ok: true, data: { scriptPublicationId: id } }); + expect(records(stateDir)).toEqual([]); + expect(fs.existsSync(staging)).toBe(false); + expect(delayed.ok && delayed.data?.savedScript).toBeFalsy(); + expect(fs.readFileSync(destination, 'utf8')).toBe('open Demo\n'); +}); + +test('the next open removes a record a close left retired', async () => { + const { stateDir, destination } = journey(); + const directory = path.join(stateDir, 'remote-connections', 'session-scripts'); + fs.mkdirSync(directory, { recursive: true }); + const stuck = path.join(path.dirname(destination), '.agent-device-script-stuck'); + fs.mkdirSync(stuck, { recursive: true }); + const retired = path.join(directory, 'retired.json'); + fs.writeFileSync( + retired, + JSON.stringify({ destination, force: false, revision: 1, staging: [stuck], retired: true }), + ); + + await open('http://127.0.0.1:9', stateDir, destination, { + respond: (id) => ({ ok: true, data: { scriptPublicationId: id } }), + }); + + expect(fs.existsSync(retired)).toBe(false); + expect(fs.existsSync(stuck)).toBe(false); +}); + +test('a record that can be neither retired nor removed keeps the anchor it names and loses other staging', async () => { + const { stateDir, destination } = journey(); + const baseUrl = await artifactServer({ v1: { body: 'open Demo\n' } }); + const { id } = await open(baseUrl, stateDir, destination); + const [name] = records(stateDir); + const recordFile = path.join(stateDir, 'remote-connections', 'session-scripts', name as string); + const staging = path.join(path.dirname(destination), '.agent-device-script-listed'); + fs.mkdirSync(staging); + const anchored = path.join(path.dirname(destination), '.agent-device-script-anchored'); + fs.mkdirSync(anchored); + fs.writeFileSync(path.join(anchored, 'script.ad.anchor'), 'pending'); + const record = JSON.parse(fs.readFileSync(recordFile, 'utf8')); + const publishing = { revision: 2, file: '1:2', staging: anchored }; + fs.writeFileSync( + recordFile, + JSON.stringify({ ...record, publishing, staging: [staging, anchored] }), + ); + const publish = vi.mocked(file.publishFileSync); + const realPublish = publish.getMockImplementation(); + publish.mockImplementation((options) => { + if (String(options.contents).includes('"retired"')) throw new Error('record write failed'); + return realPublish?.(options); + }); + const realRm = fs.rmSync; + const rm = vi.spyOn(fs, 'rmSync').mockImplementation((target, options) => { + if (String(target) === recordFile) throw new Error('record removal failed'); + realRm(target, options); + }); + + let closed: DaemonResponse; + try { + closed = await answer(baseUrl, stateDir, request('close'), { scriptPublicationId: id }); + } finally { + rm.mockRestore(); + publish.mockImplementation(realPublish as typeof file.publishFileSync); + } + + expect(closed).toEqual({ ok: true, data: { scriptPublicationId: id } }); + expect(fs.existsSync(staging)).toBe(false); + expect(fs.existsSync(path.join(anchored, 'script.ad.anchor'))).toBe(true); + expect(records(stateDir)).toHaveLength(1); +}); + test('a stale record that cannot be swept never blocks a new open', async () => { const { stateDir, destination } = journey(); const directory = path.join(stateDir, 'remote-connections', 'session-scripts'); diff --git a/packages/remote/src/session-script-destination.ts b/packages/remote/src/session-script-destination.ts index a91c1bca70..552e455650 100644 --- a/packages/remote/src/session-script-destination.ts +++ b/packages/remote/src/session-script-destination.ts @@ -28,7 +28,8 @@ const RECORD_LOCK_TIMEOUT_MS = 30_000; * rename, so a crash between the rename and the revision update is settled from the destination's * identity, and the staging directory is removed once nothing can name it. `staging` lists every * staging directory allocated and not yet removed, so retiring or sweeping the record reclaims the - * ones a crashed download left behind. + * ones a crashed download left behind. `retired` marks a record a close has ended: it authorizes no + * further version and stays on disk only until its staging directories are removed. */ /** A published destination, `undefined` when no record matches, or a typed delivery failure. */ type PublishOutcome = @@ -41,6 +42,7 @@ type ScriptRecord = Readonly<{ revision: number; publishing?: Readonly<{ revision: number; file: string; staging: string }>; staging?: readonly string[]; + retired?: true; }>; type RemoteEndpoint = DaemonArtifactEndpoint & { capabilities?: readonly string[] }; @@ -137,15 +139,11 @@ export function sessionScriptDelivery( data: { session: req.session, error: String(retireError) }, }), ); - let result: DaemonResponse; try { - result = await run(delivery); - } catch (error) { + return await run(delivery); + } finally { await retireQuietly(); - throw error; } - await retireQuietly(); - return result; }, }; return delivery; @@ -252,19 +250,59 @@ function withoutStaging(record: ScriptRecord, staging: string): ScriptRecord { } /** - * Removes a record and every staging directory it lists. The record first stops naming an - * interrupted version, so no anchor is removed while the record names its inode, and a failure - * keeps the list. + * Ends a record, then removes every staging directory it lists and the record itself. The record is + * first written retired, so it authorizes nothing more even when a staging directory cannot be + * removed; the sweep finishes the cleanup later. Writing it retired also stops it naming an + * interrupted version, so no anchor is removed while the record names its inode. A record that + * cannot be written retired is removed at once. Only a state directory that allows neither write + * leaves the record active; the close logs that failure and keeps its own outcome. */ function removeRecord(file: string): void { - const record = readRecord(file); - if (record?.publishing) writeRecord(file, settlePublishing(record)); + const record = parseRecord(file); + if (record && !record.retired) { + try { + writeRecord(file, { ...settlePublishing(record), retired: true }); + } catch (error) { + removeUnretirableRecord(file, record); + throw error; + } + } for (const staging of record?.staging ?? []) { fs.rmSync(staging, { recursive: true, force: true }); } fs.rmSync(file, { force: true }); } +/** + * Removes a record that could not be written retired, then its staging. A record still on disk may + * name the anchor in its interrupted version's staging, so that directory stays with it. + */ +function removeUnretirableRecord(file: string, record: ScriptRecord): void { + const kept = removeQuietly(file) ? undefined : record.publishing?.staging; + for (const staging of record.staging ?? []) { + if (staging !== kept) removeQuietly(staging); + } +} + +/** + * One removal attempt for a record that could not be written retired, and for the staging it listed. + * A failure here never replaces the error that made the record unwritable. Reports whether the + * target is gone. + */ +function removeQuietly(target: string): boolean { + try { + fs.rmSync(target, { recursive: true, force: true }); + return true; + } catch (error) { + emitDiagnostic({ + level: 'warn', + phase: 'session_script_record_cleanup_failed', + data: { target, error: String(error) }, + }); + return false; + } +} + /** * Runs under the record lock. While `pending.named` is set the record may name the staged file, so * the staged file stays on disk and no later file can reuse its inode and pass as that version. @@ -375,20 +413,24 @@ function withoutTrailingSlashes(value: string): string { return value.slice(0, end); } -/** Creates a record and removes the records nobody has touched for `STALE_RECORD_MS`. */ +/** + * Creates a record, and removes the records a close retired and those nobody has touched for + * `STALE_RECORD_MS`. + */ async function createRecord(file: string, record: ScriptRecord): Promise { const directory = path.dirname(file); fs.mkdirSync(directory, { recursive: true, mode: 0o700 }); const isStale = (entry: string) => (fs.lstatSync(entry, { throwIfNoEntry: false })?.mtimeMs ?? Infinity) < Date.now() - STALE_RECORD_MS; + const sweepable = (entry: string) => isStale(entry) || parseRecord(entry)?.retired === true; for (const name of fs.readdirSync(directory)) { const entry = path.join(directory, name); - if (!name.endsWith('.json') || !isStale(entry)) continue; + if (!name.endsWith('.json') || !sweepable(entry)) continue; // Rechecked under the lock, so a record that a publish just refreshed stays. A record that // cannot be swept now is retried by a later sweep and never blocks this open. await withRecordLock(entry, async () => { - if (isStale(entry)) removeRecord(entry); + if (sweepable(entry)) removeRecord(entry); }).catch((error: unknown) => emitDiagnostic({ level: 'warn', @@ -400,8 +442,13 @@ async function createRecord(file: string, record: ScriptRecord): Promise { writeRecord(file, record); } -/** A record that does not parse authorizes nothing. */ +/** A record that does not parse, or that a close has retired, authorizes nothing. */ function readRecord(file: string): ScriptRecord | undefined { + const record = parseRecord(file); + return record?.retired ? undefined : record; +} + +function parseRecord(file: string): ScriptRecord | undefined { let record: Partial | null = null; try { record = JSON.parse(fs.readFileSync(file, 'utf8')); @@ -417,6 +464,7 @@ function readRecord(file: string): ScriptRecord | undefined { (Number.isSafeInteger(publishing?.revision) && typeof publishing?.file === 'string' && typeof publishing?.staging === 'string')) && + (record.retired === undefined || record.retired === true) && (record.staging === undefined || (Array.isArray(record.staging) && record.staging.every((staging) => typeof staging === 'string')));