diff --git a/changelog.d/watcher-holds-root.breaking.md b/changelog.d/watcher-holds-root.breaking.md new file mode 100644 index 000000000..79658ba3c --- /dev/null +++ b/changelog.d/watcher-holds-root.breaking.md @@ -0,0 +1,13 @@ +- **Operators:** on Linux, a workspace mount's watcher holds the mount's root + directory open while it watches it, so it can tell whether that directory + has been removed whatever has since been made at its path. + - **What stops working:** `umount` of the file system under a watched + root fails with "target is busy" while the host watches it. Stop the host + first, or use `umount -l`, which still detaches it. + - **What gets slower:** a root removed, or replaced by a rename over it, is + found at the watcher's next root poll (`watchRootPollMs`, 2 s by default) + rather than at once. + - **Unchanged:** where the watcher can't hold the root (macOS, Windows, or + on Linux a root it can't open as a directory), it holds nothing open, as + before. On WSL 2, Windows programs can still rename or remove a watched + folder on a Windows drive. diff --git a/changelog.d/watcher-root-name-renames.fixed.md b/changelog.d/watcher-root-name-renames.fixed.md new file mode 100644 index 000000000..3ce421960 --- /dev/null +++ b/changelog.d/watcher-root-name-renames.fixed.md @@ -0,0 +1,12 @@ +- On Linux, a workspace mount's watcher no longer re-attaches, rescanning + the whole mount and running its catch-up pass, when a file or directory + with the mount root's own name is made, saved over or removed anywhere in + the mount, as a program a build names after its directory is on every + build. libuv reports such a rename under the same name as the root's own + removal, and the watcher took every one for the root's. It now holds its + root open (see Breaking) and checks whether the root is still the directory + it holds. Where it can't hold the root, as on macOS and Windows or without + read permission, such a rename still re-attaches it, as before. +- A workspace mount's watcher whose re-attach fails to close its old chokidar + instance now reports that and attaches again. It used to stop watching the + mount until the root's identity changed. diff --git a/src/modules/workspace/watcher.ts b/src/modules/workspace/watcher.ts index 346d5af27..8a89780b7 100644 --- a/src/modules/workspace/watcher.ts +++ b/src/modules/workspace/watcher.ts @@ -6,7 +6,7 @@ */ import { watch, type FSWatcher } from 'chokidar'; -import { mkdirSync, statSync } from 'node:fs'; +import { closeSync, constants, fstatSync, mkdirSync, openSync, statSync, type Stats } from 'node:fs'; import { basename, resolve, sep } from 'node:path'; import { passedOverAt } from './observe.js'; import { type MountConfig } from './types.js'; @@ -43,6 +43,8 @@ interface RootIdentity { birthtimeMs: number; } +const identityOf = (stat: Stats): RootIdentity => ({ dev: stat.dev, ino: stat.ino, birthtimeMs: stat.birthtimeMs }); + /** * Manages filesystem watching for a single mount. * Handles debouncing, ignore patterns, and write suppression. @@ -58,13 +60,15 @@ export class MountWatcher { private suppressedPaths = new Set(); private suppressionTimers = new Map>(); private rootIdentity: RootIdentity | null = null; + // The root as last attached, held open on Linux (see holdRoot). + private rootFd: number | null = null; private detached = false; private attaching = false; private stopped = true; private readonly debounceMs: number; private readonly config: MountConfig; /** - * The root as the watcher makes, watches and checks it, and as each + * The root as the watcher makes, holds, watches and checks it, and as each * event's path is read against: the configured path resolved (a link in it * kept), since chokidar hands back the paths it reports normalized, and * those of a root configured as `./m` matched none of them. @@ -173,13 +177,19 @@ export class MountWatcher { this.watcher.on('error', (err) => { this.reportError(err); }); - this.watcher.on('raw', (event: string, rawPath: string, _details: unknown) => { - if (event !== 'rename') return; - // A rename on the watched root's path/basename is the one signal that - // survives inode reuse on Linux, where unlinkDir/error never fire. - if (this.isRootPath(rawPath) || basename(rawPath) === basename(this.root)) { - void this.checkRootLiveness({ force: true }); - } + this.watcher.on('raw', (event: string, rawPath: string | null) => { + // libuv reports the root's own move as a rename under the root's name, + // and its removal too once nothing holds it open. An entry beneath the + // root with that name comes the same way, as does, on macOS, an + // FSEvents event for the root that carries no modification flags, such + // as its making just before the watch. A held root tells these apart + // (checkRootLiveness), so the rename runs the check at once. A root + // that isn't held can be made again with the removed one's identity, + // so there the rename forces a re-attach. A rename without a name, as + // libuv gives for a file whose path it can't read, is passed over. + if (event !== 'rename' || typeof rawPath !== 'string') return; + if (basename(rawPath) !== basename(this.root)) return; + void this.checkRootLiveness({ force: this.rootFd === null }); }); this.captureRootIdentity(); @@ -204,6 +214,8 @@ export class MountWatcher { this.suppressionTimers.clear(); this.suppressedPaths.clear(); this.pendingChanges.clear(); + // Before the close, so a close that fails still lets go of the root. + this.releaseRoot(); if (this.watcher) { await this.watcher.close(); @@ -299,6 +311,15 @@ export class MountWatcher { this.rootPollTimer.unref(); } + /** + * Whether the root is still the directory the watcher attached to, checked + * every `watchRootPollMs` and at a rename under the root's name. A root + * replaced at its path, or whose held directory has been removed, is + * attached to again, and with `force` so is one that looks unchanged. A + * root that's gone is let go of: a read-write mount makes it again and + * attaches to it, and a read-only one waits, detached, until a check finds + * it back. + */ private async checkRootLiveness(opts: { force?: boolean } = {}): Promise { if (this.stopped || this.attaching) return; @@ -306,8 +327,10 @@ export class MountWatcher { try { current = this.readRootIdentity(); } catch (err) { - if (this.isErrno(err, 'ENOENT')) { + // A held root that has been removed is gone, whatever the path's error. + if (this.isErrno(err, 'ENOENT') || this.heldRootRemoved()) { this.detached = true; + this.releaseRoot(); if (this.config.mode === 'read-write') { await this.reattach(); } @@ -327,7 +350,7 @@ export class MountWatcher { } if (!opts.force && !this.detached && this.rootIdentity - && this.sameIdentity(current, this.rootIdentity)) { + && this.sameIdentity(current, this.rootIdentity) && !this.heldRootRemoved()) { return; } @@ -341,8 +364,15 @@ export class MountWatcher { try { const oldWatcher = this.watcher; this.watcher = null; + this.releaseRoot(); if (oldWatcher) { - await oldWatcher.close(); + try { + await oldWatcher.close(); + } catch (err) { + // chokidar removes its listeners before it closes, so the old + // instance reports nothing more, and the root is attached to again. + this.reportError(err); + } } if (this.stopped) return; @@ -359,9 +389,11 @@ export class MountWatcher { private captureRootIdentity(): void { try { - this.rootIdentity = this.readRootIdentity(); + this.rootIdentity = this.holdRoot(); this.detached = false; } catch (err) { + // What holdRoot opened before it failed isn't held. + this.releaseRoot(); if (this.isErrno(err, 'ENOENT')) { this.rootIdentity = null; this.detached = true; @@ -373,14 +405,79 @@ export class MountWatcher { } } + /** + * The root's identity, with the root held open on Linux until the watcher + * lets go of it: at a re-attach, when the root is found gone, or at the + * stop. The held directory then says itself whether it has been removed + * (heldRootRemoved). A removed directory's inode number can otherwise go + * to the next directory made, as ext4 gives it, and one held open isn't + * given out. Elsewhere, or where the root can't be opened as a directory, + * as without read permission, at the process's file limit or when the + * path isn't a directory, it isn't held, and its identity is read from its + * path. So too where the root's link count reads 0 as it's opened: if + * that's the file system's count for a directory in place, a later check + * would take the root for removed and re-attach at every poll. + */ + private holdRoot(): RootIdentity { + if (process.platform !== 'linux') return this.readRootIdentity(); + let fd: number; + try { + // A directory only, so a FIFO at the path fails at once rather than + // blocking until a writer opens it. + fd = openSync(this.root, constants.O_RDONLY | constants.O_DIRECTORY); + } catch { + return this.readRootIdentity(); + } + this.rootFd = fd; + const stats = fstatSync(fd); + if (stats.nlink === 0) { + this.releaseRoot(); + return this.readRootIdentity(); + } + return identityOf(stats); + } + + /** + * Whether the directory the watcher holds has been removed, which its link + * count says whatever has since been made at its path. On a file system + * whose inode numbers a server gives out, the hold can't keep a root made + * again from taking the removed one's number, and a removal by another + * client shows once this one revalidates the directory's cached + * attributes. A held directory that can't be read, as a stale one on a + * network file system, counts as removed. + */ + private heldRootRemoved(): boolean { + if (this.rootFd === null) return false; + try { + return fstatSync(this.rootFd).nlink === 0; + } catch { + return true; + } + } + + private releaseRoot(): void { + if (this.rootFd === null) return; + const fd = this.rootFd; + this.rootFd = null; + try { + closeSync(fd); + } catch (err) { + // Linux frees the descriptor whatever close returns, so the error is + // reported and what called this goes on. + this.reportError(err); + } + } + private readRootIdentity(): RootIdentity { - const stat = statSync(this.root); - return { dev: stat.dev, ino: stat.ino, birthtimeMs: stat.birthtimeMs }; + return identityOf(statSync(this.root)); } private sameIdentity(a: RootIdentity, b: RootIdentity): boolean { - // birthtimeMs is only a secondary tripwire: sub-millisecond replacements - // can reuse both the inode and timestamp, so raw rename remains essential. + // birthtimeMs tells a root made again from a removed one whose inode + // number it took, unless both were made within one tick of the clock or + // the file system keeps no birth time. A held root's removal shows in its + // link count instead, and where the root isn't held the rename's force + // covers those. return a.dev === b.dev && a.ino === b.ino && a.birthtimeMs === b.birthtimeMs; } diff --git a/test/workspace-three-way.test.ts b/test/workspace-three-way.test.ts index f7ac690aa..8d0586249 100644 --- a/test/workspace-three-way.test.ts +++ b/test/workspace-three-way.test.ts @@ -1424,7 +1424,8 @@ describe('the catch-up pass at start', () => { * re-attach the watcher reports, since a re-attached watcher's ready runs its own pass (here always long before its * own bound). A watcher can re-attach just after its first ready. On macOS, by libuv's src/unix/fsevents.c, an * FSEvents event for the watched directory itself, when it carries no modification flags, reaches fs.watch as a - * rename under the directory's own name, and the watcher's raw handler re-attaches on any rename named like its root. + * rename under the directory's own name, and there, holding no root, the watcher re-attaches at any rename named + * like its root. */ const assertPassesOnceReady = (env: Env, passes: number, expected: number) => { const reattaches = env.events.filter((e) => e.type === 'workspace:watcher-reattached').length; @@ -1452,17 +1453,19 @@ describe('the catch-up pass at start', () => { test('a re-attached watcher runs its pass at its own ready, so a change after that pass reaches the store', { skip: process.platform !== 'linux' }, async (t) => { const env = new Env(t); env.writeDisk('gate.json', 'v1'); - // The root poll is past the test's deadline, so only the root's rename event re-attaches the watcher. + // The root poll is past the test's deadline, so only the re-attach below happens. const m = await env.open({ watch: 'always', watchDebounceMs: 50, watchRootPollMs: 60_000 }); const mount = (m as any).mounts.get('work'); await until(() => !!mount.watcherReadyAt && storedText(env, 'gate.json') === 'v1', 'the first pass took gate.json in'); - // The root is moved away, its file changed, and moved back, so the watcher re-attaches to the same directory, - // which the module still agrees with. chokidar's directory reads each wait a second meanwhile, so the - // re-attached watcher's scan reaches gate.json long after a pass started at the re-attach would have run. + // The watcher re-attaches to the same directory, which the module still agrees with, as it does at a rename + // named like a root it can't hold open. gate.json changes once the old chokidar instance has begun to close, which + // takes its listeners first, so no watcher reports the change. chokidar's directory reads each wait a second + // meanwhile, so the re-attached watcher's scan reaches gate.json long after a pass started at the re-attach would + // have run. await withChokidarReaddir(() => new Promise((r) => setTimeout(r, 1000)), async (reads) => { - renameSync(env.dir, `${env.dir}.away`); - writeFileSync(join(`${env.dir}.away`, 'gate.json'), 'v2'); - renameSync(`${env.dir}.away`, env.dir); + const reattaching: Promise = (m as any).watchers.get('work').reattach(); + env.writeDisk('gate.json', 'v2'); + await reattaching; await until(() => reattached(env), 'the watcher re-attached'); await until(() => storedText(env, 'gate.json') === 'v2', 'gate.json\'s v2 was taken in', 15_000); env.writeDisk('gate.json', 'v3'); @@ -1529,8 +1532,9 @@ describe('the catch-up pass at start', () => { const m = await env.open({ watch: 'always', watchDebounceMs: 50 }, {}, (m) => { counted = countPasses(m); }); const mount = (m as any).mounts.get('work'); await until(() => !!mount.watcherReadyAt && storedText(env, 'gate.json') === 'v1', 'the pass ran at ready'); - // The rename macOS reports for an FSEvents event on the root itself, sent to the watcher's chokidar instance. - (m as any).watchers.get('work').watcher.emit('raw', 'rename', 'work', {}); + // The watcher re-attaches, as it does on macOS, where it holds no root, at the rename an FSEvents event on the root + // itself arrives as. + await (m as any).watchers.get('work').reattach(); await until(() => counted.passes >= 2, "the re-attached watcher's ready ran its pass"); await elapse(200); assert.ok(reattached(env), 'the watcher re-attached'); diff --git a/test/workspace-watcher.test.ts b/test/workspace-watcher.test.ts index 7da062fbc..1a77899db 100644 --- a/test/workspace-watcher.test.ts +++ b/test/workspace-watcher.test.ts @@ -1,6 +1,7 @@ import { describe, it, beforeEach, afterEach } from 'node:test'; import assert from 'node:assert'; -import { existsSync, mkdtempSync, mkdirSync, writeFileSync, unlinkSync, rmSync } from 'node:fs'; +import fs, { existsSync, mkdtempSync, mkdirSync, readdirSync, readlinkSync, realpathSync, renameSync, rmdirSync, statSync, symlinkSync, writeFileSync, unlinkSync, rmSync } from 'node:fs'; +import { syncBuiltinESMExports } from 'node:module'; import { tmpdir } from 'node:os'; import { basename, dirname, join } from 'node:path'; @@ -54,18 +55,41 @@ function collect( return { watcher, batches }; } -function withChokidarBackend(usePolling: boolean, start: () => T): T { +/** Sets chokidar's backend for the instances made next, and returns what puts it back. */ +function useChokidarBackend(usePolling: boolean): () => void { const previousUsePolling = process.env.CHOKIDAR_USEPOLLING; const previousInterval = process.env.CHOKIDAR_INTERVAL; process.env.CHOKIDAR_USEPOLLING = String(usePolling); if (usePolling) process.env.CHOKIDAR_INTERVAL = '25'; - try { - return start(); - } finally { + return () => { if (previousUsePolling === undefined) delete process.env.CHOKIDAR_USEPOLLING; else process.env.CHOKIDAR_USEPOLLING = previousUsePolling; if (previousInterval === undefined) delete process.env.CHOKIDAR_INTERVAL; else process.env.CHOKIDAR_INTERVAL = previousInterval; + }; +} + +/** The backend for the instance `start` makes; a re-attach's later instance takes the default. */ +function withChokidarBackend(usePolling: boolean, start: () => T): T { + const restore = useChokidarBackend(usePolling); + try { + return start(); + } finally { + restore(); + } +} + +/** + * chokidar's polling backend for every instance made while `fn` runs, + * re-attaches' included. It sends no raw rename, so only the watcher's own + * checks re-attach it. + */ +async function withPollingThroughout(fn: () => Promise): Promise { + const restore = useChokidarBackend(true); + try { + return await fn(); + } finally { + restore(); } } @@ -179,10 +203,77 @@ async function changesFlow(root: string, batches: FsChange[][], dirs: readonly s * system timing in it. */ function emitChokidar(watcher: MountWatcher, events: Array<['add' | 'change' | 'unlink', string]>): void { - const chokidar = (watcher as unknown as { watcher: { emit(event: string, path: string): boolean } }).watcher; + const chokidar = chokidarOf(watcher); for (const [event, path] of events) chokidar.emit(event, join(tmp, path)); } +/** + * The watcher's chokidar instance, an EventEmitter, so a test can hand it an + * event as chokidar emits one. A re-attach replaces it, and while one closes + * the old instance there is none. + */ +function chokidarOf(watcher: MountWatcher): { emit(event: string, ...args: unknown[]): boolean; close(): Promise } { + return (watcher as unknown as { watcher: { emit(event: string, ...args: unknown[]): boolean; close(): Promise } }).watcher; +} + +/** Makes `watcher`'s chokidar instance fail its close, once it has closed. */ +function failClose(watcher: MountWatcher): void { + const chokidar = chokidarOf(watcher); + const close = chokidar.close.bind(chokidar); + chokidar.close = async () => { + await close(); + throw new Error('close failed'); + }; +} + +/** + * The process's open descriptors of `path`, a real path, as Linux's /proc names + * them: the path, or the path and " (deleted)" once it has been removed. + */ +function descriptorsOf(path: string): string[] { + return readdirSync('/proc/self/fd').flatMap((fd) => { + let target: string; + try { + target = readlinkSync(`/proc/self/fd/${fd}`); + } catch { + return []; // closed since the listing, as the listing's own is + } + return target === path || target === `${path} (deleted)` ? [target] : []; + }); +} + +/** + * Runs `fn` with node:fs's `name` replaced by what `make` makes of the + * original. The watcher imports it by name, so the binding is synced both + * ways, and the test's own imports of it get the replacement too. + */ +async function withFs( + name: 'openSync' | 'statSync' | 'fstatSync' | 'closeSync', + make: (original: (...args: any[]) => any) => (...args: any[]) => any, + fn: () => Promise, +): Promise { + const target = fs as unknown as Record any>; + const original = target[name]!; + target[name] = make(original); + syncBuiltinESMExports(); + try { + return await fn(); + } finally { + target[name] = original; + syncBuiltinESMExports(); + } +} + +const fsError = (code: string, syscall: string, path: string) => Object.assign(new Error(`${code}: refused by the test, ${syscall} '${path}'`), { code, syscall, path }); + +/** Runs `fn` with `openSync` refusing `path`, as at the process's file limit. */ +function withOpenRefused(path: string, fn: () => Promise): Promise { + return withFs('openSync', (openSync) => (file: unknown, ...rest: unknown[]) => { + if (String(file) === path) throw fsError('EMFILE', 'open', path); + return openSync(file, ...rest); + }, fn); +} + describe('MountWatcher mergeOp', () => { // Each merge rule, from chokidar's events handed in within one debounce // window: what the batch says once the window closes. Nothing else changes @@ -334,19 +425,16 @@ describe('MountWatcher lifecycle', () => { }, async () => { let readyCount = 0; let reattachCount = 0; - const { watcher, batches } = collectWithNative( - { - onReady: () => { readyCount++; }, - onReattach: () => { reattachCount++; }, - }, - // The root poll is past the test's deadline, so within it only the raw - // rename event can re-attach. - { ...makeConfig(), watchRootPollMs: EVENT_MS * 6 }, - ); + const { watcher, batches } = collectWithNative({ + onReady: () => { readyCount++; }, + onReattach: () => { reattachCount++; }, + }); try { await waitFor(() => readyCount > 0, 'expected initial watcher ready'); + // The watcher holds its root open, so the root's removal sends no rename + // until the hold ends, and the root poll finds the root made again. rmSync(tmp, { recursive: true, force: true }); mkdirSync(tmp, { recursive: true }); @@ -434,7 +522,12 @@ describe('MountWatcher lifecycle', () => { try { await waitFor(() => readyCount > 0, 'expected initial watcher ready'); + const root = realpathSync(tmp); + assert.deepStrictEqual(descriptorsOf(root), [root], 'the watcher holds its root'); rmSync(tmp, { recursive: true, force: true }); + // A poll that finds the root gone lets go of it, so its file system + // isn't held busy while the mount waits for it. + await waitFor(() => descriptorsOf(root).length === 0, 'expected the watcher to let go of its removed root'); // Root polls run through this window and find the root gone. A mount // that watched the missing root again would fire onReady again, which // is what this test counts: the root stays absent either way, and @@ -447,6 +540,597 @@ describe('MountWatcher lifecycle', () => { await watcher.stop(); } }); + + // libuv reports a rename of an entry named like the root under that name, + // as it reports the root's own removal or move. On Linux the watcher holds + // its root, whose identity tells them apart. + it("doesn't re-attach for an entry named like its held root, made, saved over or removed anywhere beneath it", { + skip: process.platform !== 'linux', + }, async () => { + const root = join(tmp, 'work'); + mkdirSync(join(root, 'sub'), { recursive: true }); + let ready = false; + let reattachCount = 0; + const { watcher, batches } = collectWithNative( + { onReady: () => { ready = true; }, onReattach: () => { reattachCount++; } }, + // The root poll is past the test's deadline, so within it only a rename + // could re-attach. + { ...makeConfig(), path: root, watchRootPollMs: EVENT_MS * 6 }, + ); + // Each step waits for the batch its change comes in, which comes after + // the change's rename and the check that rename runs, or for a re-attach, + // which loses the change's events. + const step = async (what: string, change: () => void, path: string, op?: FsChange['op']) => { + batches.length = 0; + change(); + const reported = () => batches.flat().some((c) => c.path === path && (op === undefined || c.op === op)); + await waitFor(() => reported() || reattachCount > 0, `expected ${op ? `a ${op} entry` : 'a change'} for ${path}`); + assert.strictEqual(reattachCount, 0, `${what} re-attached the watcher`); + }; + + try { + await waitFor(() => ready, 'expected initial watcher ready'); + await step('a file named like the root, made', () => writeFileSync(join(root, 'work'), 'v1'), 'work', 'created'); + await step('a file named like the root, made in a subdirectory', () => writeFileSync(join(root, 'sub', 'work'), 'v1'), 'sub/work', 'created'); + await step('a file named like the root, saved over by a rename', () => { + writeFileSync(join(root, '.work.tmp'), 'v2'); + renameSync(join(root, '.work.tmp'), join(root, 'work')); + }, 'work'); + await step('a file named like the root, removed', () => unlinkSync(join(root, 'work')), 'work', 'deleted'); + await step('a file named like the root, removed from a subdirectory', () => unlinkSync(join(root, 'sub', 'work')), 'sub/work', 'deleted'); + // A directory's own making isn't reported, so this step waits for a + // file made in it after it. + await step('a directory named like the root, made', () => { + mkdirSync(join(root, 'work')); + writeFileSync(join(root, 'work', 'x.md'), 'x'); + }, 'work/x.md', 'created'); + } finally { + await watcher.stop(); + } + }); + + // On Linux a removed directory's inode number can go to the next directory + // made, as ext4 gives it, so a root made again could look unchanged. The + // watcher holds its root open, and a number held isn't given out. + it('holds its root open while it watches, so a root made again takes another inode number', { + skip: process.platform !== 'linux', + }, async (t) => { + // Whether this disk gives a removed directory's number to the next one + // made. tmpfs doesn't, and there the test has nothing to show. Another + // process can take the number first on a busy disk, so it's tried a few + // times. + const scratch = join(tmp, 'scratch'); + const givesNumberBack = (): boolean => { + mkdirSync(scratch); + const removed = statSync(scratch).ino; + rmdirSync(scratch); + mkdirSync(scratch); + const again = statSync(scratch).ino === removed; + rmdirSync(scratch); + return again; + }; + if (![1, 2, 3, 4, 5].some(givesNumberBack)) { + t.skip("this disk didn't give a removed directory's inode number to the next one made, in five tries"); + return; + } + const root = join(realpathSync(tmp), 'work'); + mkdirSync(root); + let readyCount = 0; + let reattachCount = 0; + const { watcher } = collectWithNative( + { onReady: () => { readyCount++; }, onReattach: () => { reattachCount++; } }, + { ...makeConfig(), path: root }, + ); + + try { + await waitFor(() => readyCount > 0, 'expected initial watcher ready'); + assert.deepStrictEqual(descriptorsOf(root), [root], 'the watcher holds its root'); + const held = statSync(root).ino; + rmdirSync(root); + mkdirSync(root); + assert.notStrictEqual(statSync(root).ino, held, "the root made again took the held root's inode number"); + await waitFor(() => reattachCount > 0 && readyCount > 1, 'expected the watcher to re-attach to the root made again'); + assert.deepStrictEqual(descriptorsOf(root), [root], 'the watcher holds the root made again, having let the removed one go'); + } finally { + await watcher.stop(); + } + assert.deepStrictEqual(descriptorsOf(root), [], 'a stopped watcher holds nothing of its root'); + }); + + // The identity is read from the descriptor held, so it's the held + // directory's even when the root is replaced between the open and the read. + // Here the replacement happens inside the open, and a poll finds it. + it('reads the held root identity from what it holds, so a root replaced as it was opened is found', { + skip: process.platform !== 'linux', + }, async () => { + const root = join(realpathSync(tmp), 'work'); + mkdirSync(root); + let replaced = false; + await withFs('openSync', (openSync) => (file: unknown, ...rest: unknown[]) => { + const fd = openSync(file, ...rest); + if (String(file) === root && !replaced) { + replaced = true; + renameSync(root, `${root}.away`); + mkdirSync(root); + } + return fd; + }, async () => { + let readyCount = 0; + let reattachCount = 0; + const { watcher } = collectWithNative( + { onReady: () => { readyCount++; }, onReattach: () => { reattachCount++; } }, + { ...makeConfig(), path: root }, + ); + try { + await waitFor(() => reattachCount > 0 && readyCount > 1, 'expected a poll to find the root replaced as it was opened'); + assert.deepStrictEqual(descriptorsOf(root), [root], 'the watcher holds the root at its path'); + assert.deepStrictEqual(descriptorsOf(`${root}.away`), [], 'the watcher still holds the directory moved away'); + } finally { + await watcher.stop(); + } + }); + }); + + // The watcher holds, and checks, the root it watches: the configured path + // resolved. A link followed by `..` in that path makes the directory the + // kernel opens for the path as written another one, and a root held or + // checked there would read as replaced at every poll. + it('holds and checks the root it watches when a link and .. in its configured path lead elsewhere', { + skip: process.platform !== 'linux', + }, async () => { + const base = realpathSync(tmp); + const root = join(base, 'work'); + mkdirSync(root); + mkdirSync(join(base, 'elsewhere', 'inner'), { recursive: true }); + mkdirSync(join(base, 'elsewhere', 'work')); + symlinkSync(join(base, 'elsewhere', 'inner'), join(base, 'link')); + let ready = false; + let reattachCount = 0; + const { watcher } = collectWithNative( + { onReady: () => { ready = true; }, onReattach: () => { reattachCount++; } }, + { ...makeConfig(), path: `${base}/link/../work` }, + ); + // Counts the root's checks, so the test waits for polls, not for a time. + const internals = watcher as unknown as { checkRootLiveness(opts?: { force?: boolean }): Promise }; + const check = internals.checkRootLiveness.bind(watcher); + let checks = 0; + internals.checkRootLiveness = (opts) => { + checks++; + return check(opts); + }; + try { + await waitFor(() => ready, 'expected initial watcher ready'); + assert.deepStrictEqual(descriptorsOf(root), [root], 'the watcher holds the root it watches'); + const polled = checks; + await waitFor(() => checks >= polled + 3, 'expected three root polls'); + assert.strictEqual(reattachCount, 0, 'the watcher re-attached to the root it watches'); + } finally { + await watcher.stop(); + } + }); + + it('lets go of its root at a stop whose close of chokidar fails', { + skip: process.platform !== 'linux', + }, async () => { + const root = join(realpathSync(tmp), 'work'); + mkdirSync(root); + let ready = false; + const { watcher } = collectWithNative({ onReady: () => { ready = true; } }, { ...makeConfig(), path: root }); + let stopped = false; + try { + await waitFor(() => ready, 'expected initial watcher ready'); + assert.deepStrictEqual(descriptorsOf(root), [root], 'the watcher holds its root'); + failClose(watcher); + stopped = true; + await assert.rejects(watcher.stop(), /close failed/); + assert.deepStrictEqual(descriptorsOf(root), [], 'the stopped watcher still holds its root'); + } finally { + // A watcher left running keeps the test process alive. + if (!stopped) await watcher.stop(); + } + }); + + it('lets go of its removed root at a re-attach whose close of chokidar fails', { + skip: process.platform !== 'linux', + }, async () => { + const root = join(realpathSync(tmp), 'work'); + mkdirSync(root); + let ready = false; + const errors: Error[] = []; + const { watcher } = collectWithNative( + { onReady: () => { ready = true; }, onError: (err) => { errors.push(err); } }, + { ...makeConfig(), path: root }, + ); + try { + await waitFor(() => ready, 'expected initial watcher ready'); + assert.deepStrictEqual(descriptorsOf(root), [root], 'the watcher holds its root'); + failClose(watcher); + rmdirSync(root); + mkdirSync(root); + await waitFor(() => errors.some((e) => e.message === 'close failed'), "expected the re-attach to report its close's failure"); + // A later poll may have attached to the root made again by now, so only + // the removed one is looked for. + assert.ok(!descriptorsOf(root).includes(`${root} (deleted)`), 'the watcher still holds its removed root'); + } finally { + await watcher.stop(); + } + }); + + // chokidar removes its listeners before it closes, so a re-attach whose + // close of the old instance fails reports that and attaches again, here to + // a root that hasn't changed, which no poll would re-attach. + it('attaches again after a re-attach whose close of chokidar fails', async () => { + await withPollingThroughout(async () => { + let readyCount = 0; + const errors: Error[] = []; + const { watcher, batches } = collect( + { onReady: () => { readyCount++; }, onError: (err) => { errors.push(err); } }, + ); + try { + await waitFor(() => readyCount === 1, 'expected initial watcher ready'); + failClose(watcher); + await (watcher as unknown as { reattach(): Promise }).reattach(); + assert.ok(errors.some((e) => e.message === 'close failed'), "the re-attach reported its close's failure"); + await waitFor(() => readyCount === 2, 'expected the watcher to attach again'); + writeFileSync(join(tmp, 'after.md'), 'x'); + await changeFor(batches, 'after.md', 'created'); + } finally { + await watcher.stop(); + } + }); + }); + + // close(2) frees a descriptor on Linux whatever it returns, so an error + // closing the held root is reported and the stop goes on. + it('reports an error closing its root, and still closes chokidar at the stop', { + skip: process.platform !== 'linux', + }, async () => { + const root = join(realpathSync(tmp), 'work'); + mkdirSync(root); + let ready = false; + const errors: Error[] = []; + const { watcher } = collectWithNative( + { onReady: () => { ready = true; }, onError: (err) => { errors.push(err); } }, + { ...makeConfig(), path: root }, + ); + try { + await waitFor(() => ready, 'expected initial watcher ready'); + } catch (err) { + // A watcher left running keeps the test process alive. + await watcher.stop(); + throw err; + } + const chokidar = chokidarOf(watcher); + try { + await withFs('closeSync', (closeSync) => (fd: unknown) => { + closeSync(fd); + throw fsError('EIO', 'close', root); + }, () => watcher.stop()); + } catch (err) { + // A stop that failed before closing chokidar leaves it running. + await watcher.stop(); + throw err; + } + assert.ok(errors.some((e) => /EIO/.test(e.message)), "the close's error was reported"); + assert.strictEqual(chokidar.emit('add', join(root, 'late.md')), false, 'the stopped watcher still listens to chokidar'); + }); + + // A root the watcher doesn't hold has its identity read from its path, + // which a root made again can share with the removed one, so a rename named + // like it forces a re-attach, as it always did. On Linux that's a root it + // can't open, as without read permission or at the process's file limit; + // elsewhere it holds no root. chokidar polls throughout, so the renames are + // the ones the test emits, and the root put in the root's place can only be + // found by a poll. + it("reads a root it doesn't hold from its path: a poll finds it replaced, and a rename named like it re-attaches it", async () => { + const root = join(tmp, 'work'); + mkdirSync(root); + const unheld = (fn: () => Promise): Promise => (process.platform === 'linux' ? withOpenRefused(root, fn) : fn()); + await withPollingThroughout(() => unheld(async () => { + let readyCount = 0; + let reattachCount = 0; + const { watcher } = collect( + { onReady: () => { readyCount++; }, onReattach: () => { reattachCount++; } }, + { ...makeConfig(), path: root }, + ); + // Counts the root's checks, so the test waits for polls, not for a time. + const internals = watcher as unknown as { checkRootLiveness(opts?: { force?: boolean }): Promise }; + const check = internals.checkRootLiveness.bind(watcher); + let checks = 0; + internals.checkRootLiveness = (opts) => { + checks++; + return check(opts); + }; + try { + await waitFor(() => readyCount > 0, 'expected initial watcher ready'); + const first = chokidarOf(watcher); + // A re-attach takes the instance away at once, so these need no wait. + first.emit('raw', 'rename', 'other.md', { watchedPath: root }); + assert.strictEqual(chokidarOf(watcher), first, 'a rename named otherwise re-attached the watcher'); + const polled = checks; + await waitFor(() => checks >= polled + 3, 'expected three root polls'); + assert.strictEqual(chokidarOf(watcher), first, 'a poll re-attached the watcher to its unchanged root'); + // The rename libuv reports for the root's own removal, as chokidar emits it. + first.emit('raw', 'rename', 'work', { watchedPath: root }); + await waitFor(() => reattachCount === 1 && readyCount === 2, 'expected the rename to re-attach the watcher'); + // A directory put in the root's place, the root moved aside so the new + // one can't take its number, is found by a poll. + renameSync(root, `${root}.aside`); + mkdirSync(root); + await waitFor(() => reattachCount === 2, "expected a poll to re-attach the watcher to the root put in the root's place"); + } finally { + await watcher.stop(); + } + })); + }); + + // libuv gives a rename without a name for a file whose path it can't read + // (its kqueue backend, macOS's for files). + it('passes over a rename that comes without a name', async () => { + let ready = false; + const { watcher } = withPollingBackend(() => collect({ onReady: () => { ready = true; } })); + try { + await waitFor(() => ready, 'expected initial watcher ready'); + const first = chokidarOf(watcher); + first.emit('raw', 'rename', null, { watchedPath: join(tmp, 'a.md') }); + // A re-attach takes the instance away at once. + assert.strictEqual(chokidarOf(watcher), first, 'a rename without a name re-attached the watcher'); + } finally { + await watcher.stop(); + } + }); + + // On a file system whose inode numbers a server gives out, a root made again + // can take the removed one's number, held or not. The held directory's link + // count still says it was removed. Here the root's path reads as the + // removed root, as such a file system can show it. + it('re-attaches when its held root is removed, though the root made again reads as the removed one', { + skip: process.platform !== 'linux', + }, async () => { + const root = join(realpathSync(tmp), 'work'); + mkdirSync(root); + const removed = statSync(root); + await withFs('statSync', (stat) => (file: unknown, ...rest: unknown[]) => (String(file) === root ? removed : stat(file, ...rest)), async () => { + let readyCount = 0; + let reattachCount = 0; + const { watcher } = collectWithNative( + { onReady: () => { readyCount++; }, onReattach: () => { reattachCount++; } }, + { ...makeConfig(), path: root }, + ); + try { + await waitFor(() => readyCount > 0, 'expected initial watcher ready'); + rmdirSync(root); + mkdirSync(root); + await waitFor(() => reattachCount > 0, 'expected the watcher to re-attach to the root made again'); + } finally { + await watcher.stop(); + } + }); + }); + + // A held directory that can't be read any more, as a stale one on a network + // file system, counts as removed. + it("re-attaches when it can't read the root it holds", { + skip: process.platform !== 'linux', + }, async () => { + const root = join(realpathSync(tmp), 'work'); + mkdirSync(root); + let readyCount = 0; + let reattachCount = 0; + const { watcher } = collectWithNative( + { onReady: () => { readyCount++; }, onReattach: () => { reattachCount++; } }, + { ...makeConfig(), path: root }, + ); + try { + await waitFor(() => readyCount > 0, 'expected initial watcher ready'); + const held = (watcher as unknown as { rootFd: number | null }).rootFd; + assert.notStrictEqual(held, null, 'the watcher holds its root'); + let refused = false; + await withFs('fstatSync', (fstat) => (fd: unknown, ...rest: unknown[]) => { + if (fd === held && !refused) { + refused = true; + throw fsError('ESTALE', 'fstat', root); + } + return fstat(fd, ...rest); + }, () => waitFor(() => reattachCount > 0, 'expected the watcher to re-attach')); + } finally { + await watcher.stop(); + } + }); + + // A held root that has been removed is let go of though its path fails with + // another error than ENOENT, as when one of its parents becomes a file. + it('lets go of its removed root when its path fails with another error', { + skip: process.platform !== 'linux', + }, async () => { + const root = join(realpathSync(tmp), 'work'); + mkdirSync(root); + let ready = false; + const { watcher } = collectWithNative( + { onReady: () => { ready = true; } }, + { ...makeConfig(), path: root, mode: 'read-only' }, + ); + try { + await waitFor(() => ready, 'expected initial watcher ready'); + assert.deepStrictEqual(descriptorsOf(root), [root], 'the watcher holds its root'); + await withFs('statSync', (stat) => (file: unknown, ...rest: unknown[]) => { + if (String(file) === root) throw fsError('ENOTDIR', 'stat', root); + return stat(file, ...rest); + }, async () => { + rmdirSync(root); + await waitFor(() => descriptorsOf(root).length === 0, 'expected the watcher to let go of its removed root'); + }); + } finally { + await watcher.stop(); + } + }); + + // A path that fails with another error than ENOENT can't say whether the + // root is still there, as when a parent loses search permission, so a held + // root that hasn't been removed stays held and attached. + it('keeps a root in place held when its path fails with another error', { + skip: process.platform !== 'linux', + }, async () => { + const root = join(realpathSync(tmp), 'work'); + mkdirSync(root); + let ready = false; + let reattachCount = 0; + const { watcher } = collectWithNative( + { onReady: () => { ready = true; }, onReattach: () => { reattachCount++; } }, + { ...makeConfig(), path: root }, + ); + // Counts the root's checks, so the test waits for polls, not for a time. + const internals = watcher as unknown as { checkRootLiveness(opts?: { force?: boolean }): Promise }; + const check = internals.checkRootLiveness.bind(watcher); + let checks = 0; + internals.checkRootLiveness = (opts) => { + checks++; + return check(opts); + }; + try { + await waitFor(() => ready, 'expected initial watcher ready'); + await withFs('statSync', (stat) => (file: unknown, ...rest: unknown[]) => { + if (String(file) === root) throw fsError('EACCES', 'stat', root); + return stat(file, ...rest); + }, async () => { + const polled = checks; + await waitFor(() => checks >= polled + 3, 'expected three root polls'); + }); + assert.deepStrictEqual(descriptorsOf(root), [root], 'the watcher let go of its root in place'); + assert.strictEqual(reattachCount, 0, 'the watcher re-attached to its root in place'); + } finally { + await watcher.stop(); + } + }); + + // What the watcher opened of a root whose identity it then can't read isn't + // held: the attach reports the error, and the next check attaches again. + it("holds nothing of a root it opened but couldn't read", { + skip: process.platform !== 'linux', + }, async () => { + const root = join(realpathSync(tmp), 'work'); + mkdirSync(root); + const errors: Error[] = []; + await withFs('fstatSync', () => () => { + throw fsError('EIO', 'fstat', root); + }, async () => { + // The root poll is past the test, so no check attaches again. + const { watcher } = collectWithNative({ onError: (err) => { errors.push(err); } }, { ...makeConfig(), path: root, watchRootPollMs: EVENT_MS * 6 }); + try { + // The attach ran in start(), before chokidar's own reads begin. + assert.ok(errors.some((e) => /EIO/.test(e.message)), "the attach reported the root's failed read"); + assert.deepStrictEqual(descriptorsOf(root), [], 'the watcher holds the root it could not read'); + } finally { + await watcher.stop(); + } + }); + }); + + // A root whose link count reads 0 as it's opened isn't held: if that's the + // file system's count for a directory in place, every later check would + // take the root for removed. Its polls leave it be. + it('holds no root whose link count reads 0, and its polls leave it be', { + skip: process.platform !== 'linux', + }, async () => { + const root = join(realpathSync(tmp), 'work'); + mkdirSync(root); + await withFs('fstatSync', (fstat) => (fd: unknown, ...rest: unknown[]) => Object.assign(fstat(fd, ...rest), { nlink: 0 }), async () => { + let readyCount = 0; + let reattachCount = 0; + const { watcher } = collectWithNative( + { onReady: () => { readyCount++; }, onReattach: () => { reattachCount++; } }, + { ...makeConfig(), path: root }, + ); + // Counts the root's checks, so the test waits for polls, not for a time. + const internals = watcher as unknown as { checkRootLiveness(opts?: { force?: boolean }): Promise }; + const check = internals.checkRootLiveness.bind(watcher); + let checks = 0; + internals.checkRootLiveness = (opts) => { + checks++; + return check(opts); + }; + try { + await waitFor(() => readyCount > 0, 'expected initial watcher ready'); + assert.deepStrictEqual(descriptorsOf(root), [], 'the watcher holds a root whose link count reads 0'); + const polled = checks; + await waitFor(() => checks >= polled + 3, 'expected three root polls'); + assert.strictEqual(reattachCount, 0, 'a poll re-attached the watcher to its root in place'); + } finally { + await watcher.stop(); + } + }); + }); + + // A FIFO at the root's path would block an open until a writer came, so the + // watcher opens its root only as a directory. A file there isn't held either. + it("holds nothing at a root path that isn't a directory", { + skip: process.platform !== 'linux', + }, async () => { + const root = join(realpathSync(tmp), 'work'); + writeFileSync(root, 'x'); + const watcher = new MountWatcher({ ...makeConfig(), path: root, mode: 'read-only', watchRootPollMs: EVENT_MS * 6 }, () => {}); + watcher.start(); + try { + assert.deepStrictEqual(descriptorsOf(root), [], 'the watcher holds a file at its root path'); + } finally { + await watcher.stop(); + } + }); + + // A root moved away and back before its renames are handled is the + // directory the watcher holds, so the checks find it unchanged, and its + // watches go on reporting. Both moves come before any event is handled. + it('goes on reporting a root moved away and back at once', { + skip: process.platform !== 'linux', + }, async () => { + const root = join(tmp, 'work'); + mkdirSync(root); + let ready = false; + let reattachCount = 0; + const { watcher, batches } = collectWithNative( + { onReady: () => { ready = true; }, onReattach: () => { reattachCount++; } }, + { ...makeConfig(), path: root, watchRootPollMs: EVENT_MS * 6 }, + ); + try { + await waitFor(() => ready, 'expected initial watcher ready'); + renameSync(root, `${root}.away`); + renameSync(`${root}.away`, root); + writeFileSync(join(root, 'after.md'), 'x'); + await changeFor(batches, 'after.md', 'created'); + assert.strictEqual(reattachCount, 0, 'the watcher re-attached to the root moved back'); + } finally { + await watcher.stop(); + } + }); + + // A root moved away is found at its rename, which comes for a move even + // while the root is held: the check finds nothing at the root's path. + it('re-attaches at the rename when its root is moved away', { + skip: process.platform !== 'linux', + }, async () => { + const root = join(tmp, 'work'); + mkdirSync(root); + let readyCount = 0; + let reattachCount = 0; + const { watcher, batches } = collectWithNative( + { onReady: () => { readyCount++; }, onReattach: () => { reattachCount++; } }, + // The root poll is past the test's deadline, so within it only the + // rename re-attaches. + { ...makeConfig(), path: root, watchRootPollMs: EVENT_MS * 6 }, + ); + + try { + await waitFor(() => readyCount > 0, 'expected initial watcher ready'); + renameSync(root, `${root}.away`); + await waitFor(() => reattachCount > 0 && readyCount > 1, 'expected the watcher to re-attach at the rename'); + // A read-write mount makes its root again, and the watcher reports what's + // written there. + writeFileSync(join(root, 'after.md'), 'x'); + await changeFor(batches, 'after.md', 'created'); + } finally { + await watcher.stop(); + } + }); }); describe('MountWatcher ignores what a walk passes over by design (connectome-host #278)', () => {