diff --git a/changelog.d/workspace-direct-mount.added.md b/changelog.d/workspace-direct-mount.added.md new file mode 100644 index 000000000..887daaedf --- /dev/null +++ b/changelog.d/workspace-direct-mount.added.md @@ -0,0 +1,7 @@ +- **A direct workspace mount, for a checkout an agent reads:** a mount whose config sets `backend: 'direct'` (the default is `'chronicle'`, the store-backed kind every mount has been) is read straight from disk. `read`, `read_image`, `ls`, `glob` and `grep` give its files as they are when they run, with no scan, watcher or pass, so no call waits behind one, and none of its files go into the store. It's read-only: the workspace refuses a direct mount with any `mode` but `'read-only'`, and one that sets `watch` other than `'never'`, `watchDebounceMs`, `watchRootPollMs`, a `wakeOnChange` that would wake, or `autoMaterialize: true`, since none of them would ever act there. Its paths, links, `include`, other mounts' directories, size limit and walk cap hold as on a store-backed mount. Where it differs: + - **What its listings say:** each file with its `size`, and no state or MIME type, or, for a link the mount doesn't follow or another entry it doesn't take, with why as `note`. A listing looks at each entry with `lstat`, so it lists a file it can't open with its size, and reading that file says why. `grep`'s results say `version: 'disk'`, and it lists under `skipped` a binary file, one over the size limit and one it doesn't read, each with why. `read` refuses a binary file and one over the size limit, as a store-backed mount's `read` refuses one it never took in from disk. + - **Its ignore list:** with none in its config, it takes `node_modules`, `dist`, `.git`, `build`, `coverage`, `target`, `__pycache__` and `.venv`. Walks of the whole mount pass over what the list covers, and a path a listing, glob or grep names, or its pattern's directory, is walked all the same, as a sync naming a path walks it on a store-backed mount. `ls` without a path, and `status`, give a mount configured direct its `backend`, as it runs this session, and the list it goes by. A direct mount's `status` gives only its `path`, `mode`, `backend` and `ignore`, since it keeps nothing to count. + - **Its root:** it reads beneath whatever directory is at its path now, with no check against the one a store-backed run of it last agreed with. Where its root now lies elsewhere than the last start placed it, as when a link it's configured through is repointed, it nests what a start would nest there now: a mount configured beneath its path, where that path leads, and a mount whose root the last start placed beneath where its root lies now, or whose configured path lies beneath a directory that start found there. Their files are those mounts'. A mount configured elsewhere whose own root has moved since, and the other mounts' own placements, wait for the next start, so a direct root moved into another mount's directory is still that mount's directory too. + - **`sync`:** passes over it, listing it under `skipped`. `materialize` passes over it too, naming it under `skipped`, as it does any read-only mount. + - **A mount the store holds changes on that disk lacks** (a draft, a workspace deletion, a conflict, or an intent it can't read, on the branch selected at a start, where its `include` covers it) runs store-backed from that start, so they stay in view. It runs unwatched, and with the direct default ignore list where its config names none. The workspace says so on stderr and in one ops alert (`workspace-mount-not-direct`). The alert first gives the two ways to start such a mount direct: give each change up with a sync of its path, or configure the mount `'chronicle'` and read-write, materialize it, follow materialize's answer, and restore its direct config; its next start then runs it direct. Where an intent it can't read is held, it says to sync that path. Then it names every such mount and what's held there, counting an intent it can't read as one. A mount that ran direct and runs store-backed again takes in what disk changed meanwhile as its listings, searches and syncs reach it, as an unwatched store-backed mount does. + - **Tool descriptions:** `ls`, `glob`, `grep` and `sync` gain a sentence about the direct kind only on a host with a mount that runs direct; elsewhere they're unchanged. diff --git a/src/modules/workspace/index.ts b/src/modules/workspace/index.ts index 99d8448d1..623e2acb3 100644 --- a/src/modules/workspace/index.ts +++ b/src/modules/workspace/index.ts @@ -40,7 +40,7 @@ import { MountWatcher, type FsChange } from './watcher.js'; import { hashContent, DEFAULT_MAX_FILE_SIZE } from './sync.js'; import { DiskAgreement, sameRootIdentity } from './disk-agreement.js'; import { type BranchIntent, BranchIntents } from './branch-intent.js'; -import { IGNORED, NONBLOCK, OTHER_MOUNT, OUTSIDE_INCLUDE, ROOT_REPLACED, ROOT_UNAVAILABLE, beginsWithDir, checkDir, checkRoot, coversLanding, diskSpelling, filesystemTrustsCtime, foldsCase, includePatternReadable, inOtherMount, insideMount, leadsInto, linkOnTheWay, linkRefusal, linkWords, locateRoot, namesHolder, namesNestedPlace, nestedAtReal, nestedHolder, type CheckedDir, type MountView, type NestedMount, parsedSpelling, passedOverAt, pathSegments, readPrefix, shownLeads, withinIncludes, type LinkFound, type WorkspaceEntries } from './observe.js'; +import { IGNORED, NONBLOCK, OTHER_MOUNT, OUTSIDE_INCLUDE, ROOT_REPLACED, ROOT_UNAVAILABLE, beginsWithDir, checkDir, checkRoot, coversLanding, diskSpelling, filesystemTrustsCtime, foldsCase, includePatternReadable, inOtherMount, insideMount, leadsInto, linkOnTheWay, linkRefusal, linkWords, locateRoot, looksBinary, namesHolder, namesNestedPlace, nestedAtReal, nestedHolder, observePath, parsedSpelling, passedOverAt, pathSegments, readBytes, readPrefix, shownLeads, sniffImageMime, walkScope, withinIncludes, type CheckedDir, type LinkFound, type MountView, type NestedMount, type WorkspaceEntries } from './observe.js'; import { TOOL_BATCH_HOOK_TIMEOUT_MS, ensureStateRegistered } from '../../module-registry.js'; import { DEFAULT_TOOL_RESULT_INLINE_MAX_CHARS } from '../../tool-result-history.js'; import { type EditSpec, type FilePlan, countBudget, decodeForEdit, hintBudget, planFile } from './edit-plan.js'; @@ -76,6 +76,18 @@ const CATCH_UP_BOUND_MS = 30_000; const NOT_RUNNING = 'The workspace has stopped: it takes no more work until it starts again'; /** The workspace's own deadline must end before the framework's hook bound. */ const AGENT_ACTION_SCAN_DEADLINE_MAX_MS = TOOL_BATCH_HOOK_TIMEOUT_MS - 1_000; +/** A direct mount's ignore list when its config names none (MountConfig.ignore). */ +const DIRECT_MOUNT_IGNORE: readonly string[] = ['node_modules', 'dist', '.git', 'build', 'coverage', 'target', '__pycache__', '.venv']; +/** + * How many files a direct mount's grep reads, and its listings look at, at + * once. Over a layout of seven checkouts, 5,929 files, a direct grep reading + * 16 at a time on Node's default threadpool had a median time below the + * store-backed grep's over the same files at every load tried, and one + * reading them in turn took 2 to 2.4 times as long. + */ +const DIRECT_READS_AT_ONCE = 16; +/** Why sync passes over a direct mount, as its result's `skipped` says. */ +const DIRECT_SYNC_SKIPPED = "a direct mount: its tools read disk as it is, so there's nothing to take in"; export type { WorkspaceConfig, @@ -322,7 +334,10 @@ function matchesPartialPrefix(buffer: Buffer, prefix: Buffer): boolean { * among them, the code of a link the mount doesn't follow where it stands: * `symlink` at the path itself, as any other link there gets, and * `parent_symlink` among its directories, judged on the parsed path - * (`relativePath`), whatever spelling the caller gave. + * (`relativePath`), whatever spelling the caller gave. Where the link leads is + * read beneath `rootReal`, the root the check that found it resolved, so the + * link and its target are judged beneath one root, by that root's nesting + * (`nested`). */ async function linkReadError( mount: { config: { name: string; path: string; include?: readonly string[] } }, @@ -330,12 +345,12 @@ async function linkReadError( mountPrefixedPath: string, relativePath: string, nested: readonly NestedMount[], + rootReal: string, ): Promise { if (!('link' in found)) { return new WorkspaceReadError('not_found', `Unable to read file: ${mountPrefixedPath}: ${linkWords(mount.config.name, found, null)}`, 'readlink', found.code); } - const rootReal = await realpath(mount.config.path).catch(() => resolve(mount.config.path)); - const leads = await shownLeads(mount.config.path, rootReal, found, mount.config.include); + const leads = await shownLeads(rootReal, rootReal, found, mount.config.include); const reason = `${mountPrefixedPath}: ${linkWords(mount.config.name, found, leads, nested, mount.config.include)}`; const holder = leads?.kind === 'inside' ? nestedHolder(leads.target, nested) : undefined; const code = leads?.kind === 'outside' ? 'escape' @@ -357,6 +372,48 @@ function nestedDir(from: string, to: string): string | undefined { return rel.split(sep).join('/'); } +/** Each directory above `path`, innermost first, to the filesystem's root. */ +function ancestorsOf(path: string): string[] { + const dirs: string[] = []; + for (let dir = dirname(path); ; dir = dirname(dir)) { + dirs.push(dir); + if (dirname(dir) === dir) return dirs; + } +} + +/** + * Where a mount is nested in an outer mount, as a start places it: at its + * directory as configured beneath the outer mount's configured path + * (`outerPath`), beneath each directory above its configured path that is + * the outer root on disk (`above`, each with where it lies on disk), and + * where its real root (`real`) lies beneath the outer root (`outerReal`), + * each also as disk spells it beneath `spellAt`. Its real root is claimed + * where it doesn't hold the outer root, which it would otherwise claim + * whole. Undefined where it's nested nowhere. + */ +async function placementOf( + outerPath: string, + outerReal: string, + innerPath: string, + real: string, + above: ReadonlyArray<{ dir: string; real: string }>, + spellAt: string, +): Promise<{ configured: string | undefined; fromRoots: Array<{ dir: string; root: string }>; spellings: Array<{ dir: string; root: string }>; claims: boolean } | undefined> { + const configured = nestedDir(outerPath, innerPath); + const throughRoot = above.filter((a) => a.real === outerReal).map((a) => nestedDir(a.dir, innerPath)); + const fromRoots: Array<{ dir: string; root: string }> = []; + for (const dir of [configured, ...throughRoot, nestedDir(outerReal, real)]) { + if (dir !== undefined && !fromRoots.some((s) => s.dir === dir)) fromRoots.push({ dir, root: join(outerPath, ...dir.split('/')) }); + } + if (fromRoots.length === 0) return undefined; + const spellings = [...fromRoots]; + for (const found of fromRoots) { + const dir = await diskSpelling(spellAt, found.dir); + if (!spellings.some((s) => s.dir === dir)) spellings.push({ dir, root: join(outerPath, ...dir.split('/')) }); + } + return { configured, fromRoots, spellings, claims: !isContainedPath(real, outerReal) }; +} + /** A mount nested in another, as the outer mount's otherMounts holds it. */ interface NestedMountEntry { name: string; @@ -372,6 +429,22 @@ interface NestedMountEntry { spellings: Array<{ dir: string; root: string }>; } +/** + * A path as parsePath reads it: its mount, and its mount-relative and + * mount-prefixed paths as the parse spells them; with the root the parse + * judged it beneath, a direct mount's as the parse resolved it and any + * other's as configured, and, where a direct mount's root lies elsewhere + * than the last start placed it, the mounts nested in it there + * (movedNesting), null otherwise. + */ +interface ParsedPath { + mount: MountState; + relativePath: string; + path: string; + root: string; + moved: NestedMount[] | null; +} + /** How many links realBeneath follows to a place not made yet before it stops, as the kernel's limit does. */ const LINKS_FOLLOWED = 40; @@ -878,6 +951,10 @@ export class WorkspaceModule implements Module { private otherMounts = new Map(); /** Whether each mount's root folds case (foldsCase), where disk has it, as the last start read it. */ private rootFoldsCase = new Map(); + /** Where the last start placed each mount's root (readNesting's `reals`). */ + private placedRoots = new Map(); + /** Each directory above each mount's configured path, with where the last start found it on disk (readNesting's `above`). */ + private placedAbove = new Map>(); /** * Whether a file's mode is observed, taken in and applied (MountRuntime's * `modesObserved`): not on Windows, which reports no POSIX modes. A test @@ -889,6 +966,12 @@ export class WorkspaceModule implements Module { * mounts, so restoration must be second-callback-safe like watcher * startup already is (issue #72). Cleared after application. */ private savedState: WorkspaceModuleState | null = null; + /** + * The mounts configured direct that run as 'chronicle' this session, with + * the changes the store held there that disk lacked (chooseKinds), for + * ensureRunning to say once both halves of the lifecycle have run. + */ + private notDirect = new Map>(); constructor(config: WorkspaceConfig) { for (const mount of config.mounts) { @@ -898,6 +981,14 @@ export class WorkspaceModule implements Module { `[workspace] Mount "${mount.name}": walkCap must be a positive integer no larger than ${Number.MAX_SAFE_INTEGER}, not ${shownValue(cap)}`, ); } + const backend: unknown = mount.backend; + if (backend !== undefined && backend !== 'chronicle' && backend !== 'direct') { + throw new Error(`[workspace] Mount "${mount.name}": backend must be 'chronicle' or 'direct', not ${shownValue(backend)}`); + } + const refused = backend === 'direct' ? directRefusals(mount) : []; + if (refused.length > 0) { + throw new Error(`[workspace] Mount "${mount.name}" is direct, which reads disk as it is with no store, watcher or push: ${refused.join('; ')}`); + } const include: unknown = mount.include; if (include !== undefined) { // A pattern the matcher can't read would match nothing, and take every @@ -960,23 +1051,34 @@ export class WorkspaceModule implements Module { // before the call, knowing no mount, and a later initStore starts clean. const mounts = new Map(); const intents = new Map(); + // A direct mount's files never go into the store, so its states aren't + // made. One that ran as 'chronicle' before has them, and they're read + // below for what the store holds there that disk lacks. + const made = this.config.mounts.some((m) => m.backend === 'direct') ? new Set(store.listStates().map((s) => s.id)) : null; // Register tree states for each mount: the workspace tree, and the // branch-local intent that qualifies it. for (const mount of this.config.mounts) { const treeStateId = `workspace/${mount.name}/tree`; const intentTreeStateId = `workspace/${mount.name}/intent`; - for (const id of [treeStateId, intentTreeStateId]) { - ensureStateRegistered(store, { - id, - strategy: 'tree', - deltaSnapshotEvery: this.config.deltaSnapshotEvery ?? 50, - fullSnapshotEvery: this.config.fullSnapshotEvery ?? 10, - }); + const direct = mount.backend === 'direct'; + // A mount configured direct that ran as 'chronicle' runs direct only + // once chooseKinds finds nothing held there. + const ranStoreBacked = direct && made!.has(treeStateId) && made!.has(intentTreeStateId); + if (!direct) { + for (const id of [treeStateId, intentTreeStateId]) { + ensureStateRegistered(store, { + id, + strategy: 'tree', + deltaSnapshotEvery: this.config.deltaSnapshotEvery ?? 50, + fullSnapshotEvery: this.config.fullSnapshotEvery ?? 10, + }); + } } mounts.set(mount.name, { config: mount, + direct: direct && !ranStoreBacked, treeStateId, intentTreeStateId, lastMaterializedSeq: 0, @@ -986,7 +1088,9 @@ export class WorkspaceModule implements Module { watcherReadyAt: null, watcherError: null, }); - intents.set(mount.name, new BranchIntents(store, intentTreeStateId)); + if (!direct || ranStoreBacked) { + intents.set(mount.name, new BranchIntents(store, intentTreeStateId)); + } } this.store = store; @@ -994,14 +1098,53 @@ export class WorkspaceModule implements Module { for (const [name, intent] of intents) this.intents.set(name, intent); // What disk last agreed with, per mount and path: global records, so a - // branch switch never rewinds it. + // branch switch never rewinds it. A mount configured direct has its + // evidence read only where a 'chronicle' run left its states, for what + // the store holds there that disk lacks (chooseKinds); one that never ran + // so has none to read, and none is begun for it. this.agreement = new DiskAgreement(store, { modesObserved: () => this.modesObserved }); - this.agreement.open(this.config.mounts.map((m) => ({ name: m.name, root: resolve(m.path) }))); + this.agreement.open(this.config.mounts.filter((m) => intents.has(m.name)).map((m) => ({ name: m.name, root: resolve(m.path) }))); + this.chooseKinds(); this.applySavedState(); this.ensureRunning(); } + /** + * Whether each mount configured direct runs direct: not while the store + * holds changes there that disk lacks, on the branch selected then, so + * they stay in view (MountConfig.backend); it runs as 'chronicle' instead, + * and ensureRunning says so. Decided when initStore gives the store, and + * again at each start, which knows how the mounts nest, so a start that + * finds none held starts the mount direct. A path held more than one way + * is named for the last way set here. + */ + private chooseKinds(): void { + // Every mount is decided before any is set, so a read of the store that + // throws leaves the decisions as they were. + const decided = new Map>(); + for (const [name, state] of this.mounts) { + if (state.config.backend !== 'direct') continue; + // One with no states from a 'chronicle' run holds nothing. + if (!this.intents.has(name)) { + decided.set(name, []); + continue; + } + const held = this.heldChanges(state); + const what = new Map(); + for (const path of held.owed) what.set(path, 'changed in the workspace'); + for (const path of held.unreadable) what.set(path, UNREADABLE_INTENT); + for (const path of held.deletions) what.set(path, 'deleted in the workspace'); + for (const path of held.conflicts) what.set(path, 'in conflict'); + decided.set(name, [...what].sort(([a], [b]) => a.localeCompare(b)).map(([path, label]) => ({ path, what: label }))); + } + this.notDirect = new Map(); + for (const [name, held] of decided) { + this.mounts.get(name)!.direct = held.length === 0; + if (held.length > 0) this.notDirect.set(name, held); + } + } + // ========================================================================== // Reconciliation plumbing // ========================================================================== @@ -1030,7 +1173,7 @@ export class WorkspaceModule implements Module { * A mount as observation reads it, with the root identity the workspace * recorded for it, if any: what a pass reads beneath (runtime), with * rootIdentity's, and what a disk read checks its root against (readRoot), - * with rootIdentityForRead's. + * with rootIdentityForRead's. A direct mount's has none (diskView). */ private viewOf(mount: MountState, rootIdentity: MountView['rootIdentity']): MountView { return { @@ -1039,13 +1182,42 @@ export class WorkspaceModule implements Module { rootIdentity, followSymlinks: mount.config.followSymlinks === true, maxFileSize: mount.config.maxFileSize ?? DEFAULT_MAX_FILE_SIZE, - ignore: mount.config.ignore ?? [], + ignore: this.ignoreOf(mount), ...(mount.config.include ? { include: mount.config.include } : {}), others: this.othersOf(mount.config.name), nested: this.nestedOf(mount.config.name), }; } + /** + * A mount's ignore list: its config's, or, for a mount configured direct + * that names none, the direct kind's default (MountConfig.ignore), whichever + * kind it runs as this session, as its watch mode goes by its config + * (watchOf). + */ + private ignoreOf(mount: MountState): string[] { + return mount.config.ignore ?? (mount.config.backend === 'direct' ? [...DIRECT_MOUNT_IGNORE] : []); + } + + /** + * A direct mount as its tools read it: its view, with no root identity, + * since it promises what's at its root now and keeps no evidence to check + * that against (MountConfig.backend), and its root as realpath resolves it, + * or as given where that fails, as a pass's runtime takes it. The view + * reads beneath that root, so a walk, a listing and grep's reads judge + * and open their files beneath one root, however its configured path is + * repointed meanwhile. A read of one file goes through openContainedFile + * instead, which refuses it once its root is no longer the directory it + * checked. + */ + private async diskView(mount: MountState): Promise<{ view: MountView; rootReal: string; moved: NestedMount[] | null }> { + const root = resolve(mount.config.path); + const rootReal = await realpath(root).catch(() => root); + const view = { ...this.viewOf(mount, undefined), root: rootReal }; + const moved = await this.movedNesting(mount, rootReal); + return { view: moved === null ? view : { ...view, nested: moved, others: moved.map((n) => n.dir) }, rootReal, moved }; + } + /** The view reconciliation needs of a mount, refreshed per pass. */ private async runtime(mount: MountState): Promise { const root = resolve(mount.config.path); @@ -1065,7 +1237,7 @@ export class WorkspaceModule implements Module { intents: this.intents.get(mount.config.name)!, readOnly: mount.config.mode === 'read-only', walkCap: mount.config.walkCap ?? WALK_CAP, - catchUpScan: (mount.config.watch ?? 'always') === 'always', + catchUpScan: watchOf(mount.config) === 'always', modesObserved: this.modesObserved, }; } @@ -1079,15 +1251,18 @@ export class WorkspaceModule implements Module { * under that spelling beside their own. So the start stops before a segment * spelled any other way, not there, or not a directory, and before a link, * which a walk of the whole mount doesn't go through either. It lists only - * directories it has checked lie inside the mount, and none the mount - * passes over by design, which the walk then answers for. + * directories it has checked lie inside the mount, and, on a 'chronicle' + * mount, none it passes over by design, which the walk then answers for. + * A direct mount walks a path its tools name whatever its ignore list or + * an editor's file name covers there (MountConfig.ignore), so there it + * lists such directories too, and only another mount's it doesn't. */ private async walkStart(mount: MountState, base: string, beneath: string): Promise { if (beneath === '') return base; - const { view, rootReal } = await this.runtime(mount); + const { view, rootReal } = mount.direct ? await this.diskView(mount) : await this.runtime(mount); let start = base; for (const segment of beneath.split('/')) { - if (start !== '' && passedOverAt(start, view.ignore, view.others) !== undefined) break; + if (start !== '' && (mount.direct ? inOtherMount(start, view.others) : passedOverAt(start, view.ignore, view.others) !== undefined)) break; const here = await checkDir(view, rootReal, start); if (here.kind !== 'checked') break; const entry = (await readdir(here.dir.real, { withFileTypes: true }).catch(() => [])).find((e) => e.name === segment); @@ -1146,12 +1321,14 @@ export class WorkspaceModule implements Module { // before they're known. A stop begun meanwhile stands, and this start then // sets nothing. const stops = this.stopsBegun; - const { nesting, reals } = await this.readNesting(); + const { nesting, reals, above } = await this.readNesting(); const rootFoldsCase = new Map(); for (const mount of this.config.mounts) rootFoldsCase.set(mount.name, await foldsCase(reals.get(mount.name)!)); if (this.stopsBegun !== stops) return; this.otherMounts = nesting; this.rootFoldsCase = rootFoldsCase; + this.placedRoots = reals; + this.placedAbove = above; this.ctx = ctx; this.stopped = false; @@ -1291,16 +1468,19 @@ export class WorkspaceModule implements Module { /** * Start chokidar watchers and emit workspace:mounted events. * Idempotent: safe to call from both start() and initStore(), fires once - * per mount regardless of which runs second. + * per mount regardless of which runs second. Once both have run, it + * decides again which mounts configured direct run direct, now that the + * mounts' nesting is known, and says which don't, once for this start. */ private ensureRunning(): void { if (!this.ctx || this.mounts.size === 0) return; + this.chooseKinds(); + if (this.notDirect.size > 0) this.sayNotDirect(); for (const [name, mount] of this.mounts) { if (this.watchers.has(name)) continue; - const watchMode = mount.config.watch ?? 'always'; - if (watchMode === 'always') { + if (watchOf(mount.config) === 'always') { // Chokidar is started with ignoreInitial:true, so changes made while // the session was down would be invisible. A catch-up pass takes them // in under the three-way rule: only real differences produce tree @@ -1384,6 +1564,51 @@ export class WorkspaceModule implements Module { } } + /** + * Say, for whoever runs the host, which mounts configured direct run as + * 'chronicle' this session (chooseKinds), what the workspace holds there + * that keeps them so, and the two ways to start one direct: on stderr, + * and as one ops alert for them all (the framework's failures log, its + * `ops:alert` trace and, where one is set, its webhook, which posts one + * alert of a kind per cooldown and keeps only the start and end of a long + * one). So the words of fixed length, what's happening and the ways out, + * come first, and the mounts and what they hold after them. + */ + private sayNotDirect(): void { + const mounts = [...this.notDirect]; + const one = mounts.length === 1; + // An intent the workspace can't read may hold a change or may not, so + // it's counted as what it is. + const counted = (paths: Array<{ what: string }>) => { + const u = paths.filter((h) => h.what === UNREADABLE_INTENT).length; + const c = paths.length - u; + return [ + c > 0 ? `${c} change${c === 1 ? '' : 's'} that disk lacks` : '', + u > 0 ? `${u} intent${u === 1 ? '' : 's'} it can't read` : '', + ].filter((part) => part !== '').join(' and '); + }; + const each = mounts.map(([name, paths]) => `"${name}" (${counted(paths)})`); + const which = one + ? `Mount "${mounts[0]![0]}" holds ${counted(mounts[0]![1])}` + : `Mounts ${each.slice(0, -1).join(', ')} and ${each[each.length - 1]} hold`; + const held = mounts.flatMap(([name, paths]) => paths.map((h) => `"${name}/${h.path}" (${h.what})`)); + // Materialize settles an intent the workspace can't read only where the + // workspace holds a copy there, and says nothing of one it doesn't, so + // the ways out say what does wherever one is held, past the ten paths + // named too. + const intents = mounts.some(([, paths]) => paths.some((h) => h.what === UNREADABLE_INTENT)) + ? "Sync the path of an intent it can't read. " + : ''; + const message = (one + ? "A mount configured direct runs as 'chronicle' this session. To start it direct, settle each held change, then start it again: " + : "Mounts configured direct run as 'chronicle' this session. To start a mount direct, settle each change held at it, then start it again: ") + + "sync its path to take disk's copy, or configure the mount 'chronicle' and read-write, materialize it, " + + `follow the answer, and restore its direct config. ${intents}${which}: ${held.slice(0, 10).join(', ')}` + + `${held.length > 10 ? `, and ${held.length - 10} more` : ''}.`; + console.warn(`[workspace] ${message}`); + this.ctx?.notifyOps?.('workspace-mount-not-direct', 'framework', message, { held: Object.fromEntries(mounts.map(([name, paths]) => [name, paths.length])) }); + } + private async initialScan(mountName: string): Promise { const store = this.store; const mount = this.mounts.get(mountName); @@ -1468,6 +1693,19 @@ export class WorkspaceModule implements Module { // ========================================================================== getTools(): ToolDefinition[] { + // Where a mount runs direct, four descriptions say what that changes. + // A host with none keeps them as they were, and its residents' caches. + const direct = [...this.mounts.values()].some((m) => m.direct); + const lsDirect = direct + ? ' A direct mount (ls without a path says which, with its ignore list) lists disk\'s files as they are now: each with ' + + 'its size and no state, or a note where the mount doesn\'t take it. A path its ignore list covers is listed when you name it.' + : ''; + const grepDirect = direct + ? ' On a direct mount, grep searches disk\'s files as they are now (version disk), and lists as skipped, with why, ' + + 'those it doesn\'t: binary, over the size limit, or not read, such as a link the mount doesn\'t follow.' + : ''; + const globDirect = direct ? ' On a direct mount, a match carries its size, or a note where the mount doesn\'t take it, and no state.' : ''; + const syncDirect = direct ? ' A direct mount is listed under `skipped`: its tools read disk as it is, so there\'s nothing to take in.' : ''; return [ { name: 'read', @@ -1592,7 +1830,7 @@ export class WorkspaceModule implements Module { + 'interrupted) — sync the path to take disk, or materialize that path with force to take the workspace, ' + 'with applyDeletions too where the workspace deleted it; ' + 'disk-missing-provenance-unknown; unverified (disk could not be checked). `incomplete` names regions ' - + 'the listing could not see.', + + 'the listing could not see.' + lsDirect, inputSchema: { type: 'object' as const, properties: { @@ -1603,7 +1841,7 @@ export class WorkspaceModule implements Module { }, { name: 'glob', - description: 'Find files matching a glob pattern, checked against disk first; each match carries its state, as ls describes.', + description: 'Find files matching a glob pattern, checked against disk first; each match carries its state, as ls describes.' + globDirect, inputSchema: { type: 'object' as const, properties: { @@ -1619,7 +1857,7 @@ export class WorkspaceModule implements Module { + 'which version matched: the workspace\'s (with its state); the disk version kept with a conflict ' + '(conflicting-disk); or, once disk has changed since the conflict was recorded, that recorded version ' + '(recorded-disk). Disk versions it could not search are listed as skipped: disk-only files (binary or ' - + 'over the size limit), and a conflict\'s disk version that is binary, oversize, or newer than the one recorded.', + + 'over the size limit), and a conflict\'s disk version that is binary, oversize, or newer than the one recorded.' + grepDirect, inputSchema: { type: 'object' as const, properties: { @@ -1689,7 +1927,7 @@ export class WorkspaceModule implements Module { + 'in another mount\'s directory isn\'t synced even when you name it: use the mount that holds it. Naming ' + 'an ignored directory sets aside the patterns that cover it or a directory above it; one that ' + 'covers only something inside it still applies, and what it covers is passed over, as are editors\' ' - + 'files inside it.', + + 'files inside it.' + syncDirect, inputSchema: { type: 'object' as const, properties: { @@ -1722,7 +1960,7 @@ export class WorkspaceModule implements Module { const mountName = slash < 0 ? path : path.slice(0, slash); const mount = this.config.mounts.find(m => m.name === mountName); if (!mount) throw new Error(`Unknown mount: "${mountName}"`); - // Against the resolved root, as parsePath judges it. + // Against the resolved root, as parsePathAsGiven's traversal guard judges it. const root = resolve(mount.path); const resolved = resolve(root, slash < 0 ? '' : path.slice(slash + 1)); if (!isContainedPath(root, resolved)) throw new Error(`Path traversal detected: "${path}"`); @@ -1892,6 +2130,10 @@ export class WorkspaceModule implements Module { * path for resolution-only purposes (write targets, nonexistent paths, * display); for reading file content, use {@link readFileFromDisk}, which * enforces the mount's `followSymlinks` policy and canonical containment. + * It judges other mounts' directories by where the last start placed them, + * so for a direct mount whose root has moved since, it can differ from + * `readFileFromDisk`, which takes the nesting where the root lies now + * (movedNesting), about which paths lie in another mount's directory. * * @deprecated for direct filesystem reads — call `readFileFromDisk()`. */ @@ -2016,8 +2258,10 @@ export class WorkspaceModule implements Module { /** * Read a file's raw bytes from a mount (through the Chronicle tree, synced - * first like the `read` tool). Public API for the framework's synthesized - * `read_image` tool and other peer callers that need binary content. + * first like the `read` tool; on a direct mount, from disk as + * readBinaryFromDisk reads it, at most the mount's `maxFileSize`). Public + * API for the framework's synthesized `read_image` tool and other peer + * callers that need binary content. */ async readBinary( mountPrefixedPath: string, @@ -2036,6 +2280,8 @@ export class WorkspaceModule implements Module { } catch (error) { return { error: error instanceof Error ? error.message : String(error) }; } + // A direct mount's bytes are disk's, as read takes them, at most its size limit. + if (mount.direct) return this.bytesFromDisk(mount, relativePath, mountPrefixedPath, mount.config.maxFileSize ?? DEFAULT_MAX_FILE_SIZE); const store = this.getStore(); return this.withMount(mount, async () => { await this.ensureSynced(mount, relativePath); @@ -2081,8 +2327,17 @@ export class WorkspaceModule implements Module { } catch (error) { return { error: error instanceof Error ? error.message : String(error) }; } - const absolutePath = resolve(mount.config.path, relativePath); - const maxBytes = opts.maxBytes; + const read = await this.bytesFromDisk(mount, relativePath, mountPrefixedPath, opts.maxBytes); + return 'error' in read ? read : { data: read.data, absolutePath: resolve(mount.config.path, relativePath) }; + } + + /** A file's bytes as readBinaryFromDisk reads them, once its path is parsed. */ + private async bytesFromDisk( + mount: MountState, + relativePath: string, + mountPrefixedPath: string, + maxBytes: number | undefined, + ): Promise<{ data: Buffer } | { error: string }> { let tooLarge: string | undefined; try { const { handle, fileStat } = await this.openContainedFile(mount, relativePath, mountPrefixedPath, (fileStat) => { @@ -2092,14 +2347,14 @@ export class WorkspaceModule implements Module { } }); try { - if (maxBytes === undefined) return { data: await handle.readFile(), absolutePath }; + if (maxBytes === undefined) return { data: await handle.readFile() }; // No more than maxBytes is loaded: a file grown past it since the // fstat above is refused here, as one already over it was there. Its // size is an fstat's after the read, and at least one more than the // bytes kept, so the refusal's size is always past maxBytes. const { bytes, more } = await readPrefix(handle, maxBytes, fileStat.size); if (more) return { error: `File too large: ${Math.max((await handle.stat()).size, bytes.length + 1)} > ${maxBytes} bytes` }; - return { data: bytes, absolutePath }; + return { data: bytes }; } finally { await handle.close(); } @@ -2142,9 +2397,11 @@ export class WorkspaceModule implements Module { * as configured still names it as configured (resolveAbsolutePath). A * directory not on disk keeps the spellings the roots give alone. Each * mount's real root comes back beside the nesting (`reals`), for what - * start reads there. + * start reads there, and each directory above each configured path with + * where it lies on disk (`above`), for a direct mount whose root moves + * (movedNesting). */ - private async readNesting(): Promise<{ nesting: Map; reals: Map }> { + private async readNesting(): Promise<{ nesting: Map; reals: Map; above: Map> }> { const reals = new Map(); // Each directory above a mount's configured path, with where it lies on // disk, so a path that reaches another mount's root through a link, as @@ -2154,10 +2411,7 @@ export class WorkspaceModule implements Module { const given = resolve(mount.path); reals.set(mount.name, await realBeneath(given)); const dirs: Array<{ dir: string; real: string }> = []; - for (let dir = dirname(given); ; dir = dirname(dir)) { - dirs.push({ dir, real: await realBeneath(dir) }); - if (dirname(dir) === dir) break; - } + for (const dir of ancestorsOf(given)) dirs.push({ dir, real: await realBeneath(dir) }); above.set(mount.name, dirs); } const nesting = new Map(); @@ -2167,23 +2421,9 @@ export class WorkspaceModule implements Module { for (const inner of this.config.mounts) { if (outer === inner) continue; const real = reals.get(inner.name)!; - const innerPath = resolve(inner.path); - const configured = nestedDir(outerPath, innerPath); - const fromRoots: Array<{ dir: string; root: string }> = []; - // Its directory as configured, read against this mount's root as - // configured and from each directory above it that is this mount's - // root on disk, and its real root against this mount's. - const throughRoot = above.get(inner.name)!.filter((a) => a.real === outerReal).map((a) => nestedDir(a.dir, innerPath)); - for (const dir of [configured, ...throughRoot, nestedDir(outerReal, real)]) { - if (dir !== undefined && !fromRoots.some((s) => s.dir === dir)) fromRoots.push({ dir, root: join(outerPath, ...dir.split('/')) }); - } - if (fromRoots.length === 0) continue; - const spellings = [...fromRoots]; - for (const found of fromRoots) { - const dir = await diskSpelling(outerPath, found.dir); - if (!spellings.some((s) => s.dir === dir)) spellings.push({ dir, root: join(outerPath, ...dir.split('/')) }); - } - const claims = !isContainedPath(real, outerReal); + const at = await placementOf(outerPath, outerReal, resolve(inner.path), real, above.get(inner.name)!, outerPath); + if (at === undefined) continue; + const { configured, fromRoots, spellings, claims } = at; nesting.set(outer.name, [...(nesting.get(outer.name) ?? []), { name: inner.name, ...(claims ? { real } : {}), fromRoots, spellings }]); const placed = fromRoots.filter((s) => s.dir !== configured); if (placed.length > 0) { @@ -2194,7 +2434,7 @@ export class WorkspaceModule implements Module { } } } - return { nesting, reals }; + return { nesting, reals, above }; } /** @@ -2206,6 +2446,108 @@ export class WorkspaceModule implements Module { o.spellings.map((s) => ({ dir: s.dir, name: o.name, real: o.real, include: this.mounts.get(o.name)?.config.include }))); } + /** + * The mounts nested in `mount` as a call of it that resolved its root to + * `rootReal` sees them: movedNesting where that differs from where the + * last start placed it, the start's placement (nestedOf) otherwise. + */ + private async nestedFor(mount: MountState, rootReal: string): Promise { + return (await this.movedNesting(mount, rootReal)) ?? this.nestedOf(mount.config.name); + } + + /** + * The mounts nested in a direct mount whose root a check resolved to + * `rootReal`, where that's elsewhere than the last start placed it, as when + * a link it's configured through is repointed; null where the start's + * placement holds, or where nothing is at `rootReal`: there the parse's + * checks keep the start's placement, and a read says the root is gone. A + * direct mount reads beneath whatever directory is at its root, so a check + * judges which mount a path is by the root it resolved. A tool's parse + * resolves it once, and its respelling and its checks of the path it's + * given (parseFilePath's, refusesAsOutside, grep's) read disk beneath that + * root and judge by the nesting there. A read and a walk each resolve it + * for themselves, and judge the include and the nesting again beneath it. + * So a root repointed during a call can change which answer or refusal a + * path gets, but nothing a read or a walk serves is a file the include + * leaves out, nothing a walk serves lies in another mount's directory as + * its own nesting places it, and nothing a read serves lies in the + * directory of a mount its own nesting claims by real root. Whoever writes + * the disk can race a read in two ways, as a rename during a call can on + * any mount. On a disk that folds names, a read judges the include by the + * path as its parse spelled it, so a root repointed between the two can + * have it serve a file the new root spells otherwise (sameLevels). And a + * mount nested by its directories alone, since its real root holds this + * one, is judged by the parse's nesting, so a root repointed between a + * read's parse and its open can have it serve a path in that mount's + * directory. Each mount is placed as a start places it (placementOf), from + * where its real root and the directories above its configured path lie on + * disk. For a mount configured at this mount's own path or beneath it, + * that's where they lie now: beneath `rootReal` where they lie beneath this + * mount's path, and the directories above this mount's path as resolving + * that path finds them. For any other mount, it's where the last start + * found them. Each carries its own include, as nestedOf gives it. It + * resolves nothing but the directories on this mount's own configured path + * and those beneath its root, so a share elsewhere that doesn't answer + * holds only its own calls, unless such a directory is a link into it, + * where that mount's files lie. What it leaves to the next start: a mount + * configured elsewhere whose own root has moved since, as when a link on + * its configured path is repointed, this mount's own reached through + * another link among them, that mount's spellings through other links, and + * the other mounts' own placements, so a direct root moved into another + * mount's directory is that mount's too, as a nested mount's root moved + * while running stays placed. + */ + private async movedNesting(mount: MountState, rootReal: string): Promise { + // One start's placements, whatever a start in the meantime sets. + const { placedRoots, placedAbove } = this; + const placed = placedRoots.get(mount.config.name); + if (!mount.direct || placed === undefined || placed === rootReal) return null; + const there = await stat(rootReal).then(() => true, (err: unknown) => { + const code = errnoOf(err); + return code !== 'ENOENT' && code !== 'ENOTDIR'; + }); + if (!there) return null; + const outerPath = resolve(mount.config.path); + const nested: NestedMount[] = []; + // The directories above this mount's own path, as they lie on disk now, + // resolved as resolving the path itself does. + let ownAbove: Array<{ dir: string; real: string }> | undefined; + const aboveOwn = async () => ownAbove ??= await Promise.all(ancestorsOf(outerPath).map(async (dir) => ({ dir, real: await realBeneath(dir) }))); + for (const inner of this.config.mounts) { + if (inner.name === mount.config.name) continue; + const innerPath = resolve(inner.path); + const configured = nestedDir(outerPath, innerPath); + let real: string | undefined; + let above: ReadonlyArray<{ dir: string; real: string }>; + if (configured !== undefined || innerPath === outerPath) { + // A mount configured on this mount's own path, or beneath it, lies + // where that path now leads: its real root, and each directory above + // it, resolved now, beneath this root where they lie beneath it. + const segments = configured?.split('/') ?? []; + real = await realBeneath(join(rootReal, ...segments)); + const beneath: Array<{ dir: string; real: string }> = []; + for (let i = segments.length - 1; i >= 1; i--) { + beneath.push({ dir: join(outerPath, ...segments.slice(0, i)), real: await realBeneath(join(rootReal, ...segments.slice(0, i))) }); + } + // This mount's own path, which is the root, would add only the + // configured directory, which placementOf adds anyway. + above = [...beneath, ...await aboveOwn()]; + } else { + // Any other mount lies where the last start found it and the + // directories above its path. One leading here only since, as + // through this mount's own link, waits for the next start. + real = placedRoots.get(inner.name); + if (real === undefined) continue; + above = placedAbove.get(inner.name) ?? []; + } + const at = await placementOf(outerPath, rootReal, innerPath, real, above, rootReal); + if (at === undefined) continue; + const include = this.mounts.get(inner.name)?.config.include; + for (const s of at.spellings) nested.push({ dir: s.dir, name: inner.name, ...(at.claims ? { real } : {}), include }); + } + return nested; + } + /** * The innermost mount nested in `mountName` whose directory holds a path * of that mount, judged lexically where the path lands (`.`, `..` and @@ -2219,17 +2561,21 @@ export class WorkspaceModule implements Module { * names apart, that takes a directory of this mount's own spelled like the * nested one in another way as that mount's too. */ - private holderOf(mountName: string, relativePath: string, asGiven = false): { name: string; use: string } | undefined { + private holderOf(mountName: string, relativePath: string, asGiven = false, moved: NestedMount[] | null = null): { name: string; use: string } | undefined { const mount = this.mounts.get(mountName); if (!mount) return undefined; const root = resolve(mount.config.path); const resolved = resolve(root, relativePath); if (!isContainedPath(root, resolved)) return undefined; const segments = resolved === root ? [] : relative(root, resolved).split(sep); - const holder = (this.otherMounts.get(mountName) ?? []) - .flatMap((o) => o.spellings - .filter((s) => beginsWithDir(segments, s.dir.split('/'), asGiven)) - .map((s) => ({ name: o.name, depth: s.dir.split('/').length }))) + // A direct mount whose root a call found moved is judged by that call's + // nesting (movedNesting), each directory beneath its configured path. + const dirs = moved !== null + ? moved.map((n) => ({ name: n.name, dir: n.dir })) + : (this.otherMounts.get(mountName) ?? []).flatMap((o) => o.spellings.map((s) => ({ name: o.name, dir: s.dir }))); + const holder = dirs + .filter((s) => beginsWithDir(segments, s.dir.split('/'), asGiven)) + .map((s) => ({ name: s.name, depth: s.dir.split('/').length })) .sort((a, b) => b.depth - a.depth)[0]; if (!holder) return undefined; const rest = segments.slice(holder.depth).join('/'); @@ -2241,8 +2587,8 @@ export class WorkspaceModule implements Module { * mount and the path to use: for a sync or a push, which of them leaves * it; for a listing, whose the path is. */ - private otherMountNote(mountName: string, relativePath: string, by: 'sync' | 'push' | 'list'): string { - const holder = this.holderOf(mountName, relativePath); + private otherMountNote(mountName: string, relativePath: string, by: 'sync' | 'push' | 'list', moved: NestedMount[] | null = null): string { + const holder = this.holderOf(mountName, relativePath, false, moved); const what = by === 'sync' ? ', which no sync of this mount takes in' : by === 'push' ? ', which no push of this mount writes' : ''; return holder ? `it lies in mount "${holder.name}", nested in this one${what}: use "${holder.use}"` @@ -2261,13 +2607,17 @@ export class WorkspaceModule implements Module { * directory on one side and a file on the other. */ private async parseFilePath(path: string): Promise<{ mount: MountState; relativePath: string; path: string }> { - const parsed = this.refuseNotHeld(await this.parsePath(path), path); + const spelled = await this.parsePath(path); + // Both checks judge by the nesting the parse found, and the second reads + // disk beneath the root the parse resolved. + const { mount, root, moved } = spelled; + const parsed = this.refuseNotHeld(spelled, path, false, moved); // A path can also reach a nested mount's directory, or a file this // mount's own include leaves out, through a symlink on its way. No push // of this mount writes it there and no read reads it, so it's refused // here, before anything is stored, and the checks where disk is acted on // refuse it again. - const refusal = await this.landingRefusal(parsed.mount, parsed.relativePath, parsed.path); + const refusal = await this.landingRefusal(mount, parsed.relativePath, parsed.path, moved ?? this.nestedOf(mount.config.name), root); if (refusal) throw refusal; return parsed; } @@ -2284,14 +2634,17 @@ export class WorkspaceModule implements Module { * leaves this one, in the disk readers' words, as it does where it lands * on a file this mount's include leaves out. A mount with nothing nested, * and without both an include and links followed, has nothing to find, so - * it pays nothing. + * it pays nothing. The caller gives the nesting and the root to judge by + * (`nested`, `root`). A direct mount's file operation gives those its + * parse found, so where the path lands, and any link on its way, is judged + * beneath the root that parse resolved; any other caller gives the start's + * nesting and the configured path. */ - private async landingRefusal(mount: MountState, relativePath: string, path: string): Promise { - const nested = this.nestedOf(mount.config.name); + private async landingRefusal(mount: MountState, relativePath: string, path: string, nested: readonly NestedMount[], root: string): Promise { const judged = mount.config.followSymlinks === true && mount.config.include !== undefined; if (nested.length === 0 && !judged) return null; const unresolved: { code?: string } = {}; - const real = await realBeneath(resolve(mount.config.path, relativePath), 0, unresolved); + const real = await realBeneath(resolve(root, relativePath), 0, unresolved); // Where realpath can't resolve the path, as for a loop or a directory // this process can't search, where it lands can't be judged, and a // promise about content fails closed: the path is refused, by every file @@ -2301,13 +2654,14 @@ export class WorkspaceModule implements Module { const holder = nestedAtReal(real, nested); if (!holder) { if (!judged) return null; - const rootReal = await realBeneath(resolve(mount.config.path)); + const rootReal = await realBeneath(root); if (!isContainedPath(rootReal, real) || await stat(real).then((info) => info.isDirectory(), () => false)) return null; return await coversLanding({ include: mount.config.include }, rootReal, real, relativePath) ? null : new WorkspaceReadError('escape', `Symlink escape detected: ${path}`, 'realpath'); } if (mount.config.followSymlinks !== true) { - const found = await linkOnTheWay({ root: mount.config.path, followSymlinks: false }, relativePath); - if (found !== null) return await linkReadError(mount, found, path, relativePath, nested); + const rootReal = await realBeneath(root); + const found = await linkOnTheWay({ root: rootReal, followSymlinks: false }, relativePath); + if (found !== null) return await linkReadError(mount, found, path, relativePath, nested, rootReal); } // On a mount with an include, the path to use there names each directory // on the place's way as that mount's, so a place beneath anything else @@ -2329,8 +2683,8 @@ export class WorkspaceModule implements Module { * folds where disk does). The include's refusal shows the path as given * (`given`), since it answers the path the caller asked about. */ - private refuseNotHeld

(parsed: P, given: string, asGiven = false): P { - const holder = this.holderOf(parsed.mount.config.name, parsed.relativePath, asGiven); + private refuseNotHeld

(parsed: P, given: string, asGiven = false, moved: NestedMount[] | null = null): P { + const holder = this.holderOf(parsed.mount.config.name, parsed.relativePath, asGiven, moved); if (holder) throw new NestedMountError(parsed.path, parsed.mount.config.name, holder.name, holder.use); if (!insideMount(parsed.relativePath, parsed.mount.config.include)) { throw new OutsideIncludeError(given, parsed.mount.config); @@ -2345,12 +2699,16 @@ export class WorkspaceModule implements Module { * as a directory only when neither side has a file there, and filters what * it finds beneath it by the mount's include. A path the include doesn't * cover names no file of the mount's, whatever disk or history holds - * there, so one refusesAsOutside lets through is a directory. + * there, so one refusesAsOutside lets through is a directory. A direct + * mount has no history: what a run of it as 'chronicle' left in the store + * isn't its own, so disk alone decides. Disk is read beneath `root`: a + * direct mount's is the root its parse resolved, a store-backed mount's + * its configured path. */ - private async namesFile(mount: MountState, relativePath: string): Promise { + private async namesFile(mount: MountState, relativePath: string, root: string): Promise { return relativePath !== '' && insideMount(relativePath, mount.config.include) && ( - this.getStore().treeGet(mount.treeStateId, relativePath) !== null || - await lstat(join(mount.config.path, relativePath)).then((s) => !s.isDirectory(), () => false) + (!mount.direct && this.getStore().treeGet(mount.treeStateId, relativePath) !== null) || + await lstat(join(root, relativePath)).then((s) => !s.isDirectory(), () => false) ); } @@ -2372,19 +2730,21 @@ export class WorkspaceModule implements Module { * mount. A * path in another mount's directory nested in this one is that mount's, * whatever this mount's include says, so it isn't refused here: each tool - * names the mount that holds it, as the file operations do. + * names the mount that holds it, as the file operations do. A direct mount + * has no history, as namesFile says, and is judged beneath the root its + * parse resolved (`root`), by the nesting there (`moved`, movedNesting). */ - private async refusesAsOutside(mount: MountState, relativePath: string): Promise { - if (this.holderOf(mount.config.name, relativePath)) return false; + private async refusesAsOutside(mount: MountState, relativePath: string, moved: NestedMount[] | null, root: string): Promise { + if (this.holderOf(mount.config.name, relativePath, false, moved)) return false; if (insideMount(relativePath, mount.config.include)) return false; const own = { include: mount.config.include, nested: this.nestedOf(mount.config.name) }; - if (this.getStore().treeList(mount.treeStateId, relativePath + '/') + if (!mount.direct && this.getStore().treeList(mount.treeStateId, relativePath + '/') .some((entry) => withinIncludes(own, entry.path))) return false; // A directory, and each directory above it to the mount's root, as the // walk reaches it: through no link, since the walk descends none. const segments = relativePath.split('/'); for (let i = 1; i <= segments.length; i++) { - const entry = await lstat(join(mount.config.path, ...segments.slice(0, i))).catch(() => null); + const entry = await lstat(join(root, ...segments.slice(0, i))).catch(() => null); if (!entry?.isDirectory()) return true; } return false; @@ -2407,7 +2767,8 @@ export class WorkspaceModule implements Module { * counts as an entry here, so the calls that settle it reach it. The * workspace's entries (entriesOf) stand in for disk where it has nothing * yet, so a draft has one name too. Neither applies to a path that disk's - * spelling puts in a nested mount's directory. Past a nested mount's + * spelling puts in a nested mount's directory. A direct mount holds no + * entries, so its paths take disk's spelling alone. Past a nested mount's * directory the path stays as given, since what lies there is that mount's * to name. Every answer shows `path`, a name the include leaves out * included, since how disk spells a name the caller gave is no content of @@ -2418,33 +2779,42 @@ export class WorkspaceModule implements Module { * calling this, since this awaits disk. The entries are read when the step * needs them, some after those awaits, from the branch current then. */ - private async parsePath(path: string): Promise<{ mount: MountState; relativePath: string; path: string }> { + private async parsePath(path: string): Promise { const given = this.parsePathAsGiven(path); - const spelled = await diskSpelling(resolve(given.mount.config.path), given.relativePath, this.entriesOf(given.mount)); + // A direct mount's root is resolved once here, so the respelling and the + // checks of the path that follow read disk beneath one root and judge by + // the nesting there (movedNesting). + const configured = resolve(given.mount.config.path); + const root = given.mount.direct ? await realpath(configured).catch(() => configured) : configured; + const moved = given.mount.direct ? await this.movedNesting(given.mount, root) : null; + const spelled = await diskSpelling(root, given.relativePath, given.mount.direct ? undefined : this.entriesOf(given.mount)); // Past a nested mount's directory the path stays as given: what lies // there is that mount's, whose own include decides what a tool may name, // so this mount doesn't respell it. // The directory itself takes disk's spelling, as a listing names it. // The respelling keeps each segment's place, so the given path's rest // follows the directory's segments. - const depth = this.nestedDepth(given.mount.config.name, spelled); + const depth = this.nestedDepth(given.mount.config.name, spelled, moved); const relativePath = depth === undefined ? spelled : [...spelled.split('/').slice(0, depth), ...given.relativePath.split('/').slice(depth)].join('/'); return { mount: given.mount, relativePath, path: relativePath === given.relativePath ? path : `${given.mount.config.name}/${relativePath}`, + root, + moved, }; } /** * How many segments the innermost nested mount's directory that holds - * `relativePath` has, in a spelling it's known by (othersOf); undefined - * when no nested mount's directory holds it. + * `relativePath` has, in a spelling it's known by (othersOf), or, for a + * direct mount whose root has moved, in the nesting there (`moved`); + * undefined when no nested mount's directory holds it. */ - private nestedDepth(mountName: string, relativePath: string): number | undefined { + private nestedDepth(mountName: string, relativePath: string, moved: NestedMount[] | null = null): number | undefined { let depth: number | undefined; - for (const dir of this.othersOf(mountName)) { + for (const dir of moved?.map((n) => n.dir) ?? this.othersOf(mountName)) { if (relativePath !== dir && !relativePath.startsWith(`${dir}/`)) continue; const segments = dir.split('/').length; if (depth === undefined || segments > depth) depth = segments; @@ -2605,6 +2975,8 @@ export class WorkspaceModule implements Module { * which answers while the evidence awaits reopening; where even it can't * be read, as when the evidence was never read back whole, the read is * refused too, with an errno only where no directory is at the root's path. + * A direct mount's root is checked against no identity, and its read reads + * no evidence: it promises what's at its root now (diskView). */ private async readRoot(mount: MountState): Promise<{ real: string; dir: CheckedDir }> { const name = mount.config.name; @@ -2616,13 +2988,15 @@ export class WorkspaceModule implements Module { throw new WorkspaceReadError('mount_unavailable', `Mount unavailable: ${name}`, 'realpath_root', errnoOf(err)); } let rootIdentity: MountView['rootIdentity']; - try { - rootIdentity = this.agreementOrThrow().rootIdentityForRead(name); - } catch (err) { - // The evidence can't be read back, so neither can the check be made. - // Where no directory is at the root's path, its errno still says so. - throw new WorkspaceReadError('mount_unavailable', - `Mount unavailable: ${name}: ${err instanceof Error ? err.message : String(err)}`, 'root', await rootAbsence(root)); + if (!mount.direct) { + try { + rootIdentity = this.agreementOrThrow().rootIdentityForRead(name); + } catch (err) { + // The evidence can't be read back, so neither can the check be made. + // Where no directory is at the root's path, its errno still says so. + throw new WorkspaceReadError('mount_unavailable', + `Mount unavailable: ${name}: ${err instanceof Error ? err.message : String(err)}`, 'root', await rootAbsence(root)); + } } const checked = await checkRoot(this.viewOf(mount, rootIdentity), real); if ('dir' in checked) return { real, dir: checked.dir }; @@ -2638,11 +3012,13 @@ export class WorkspaceModule implements Module { * with the code and errno readRoot gives what is at its path now (its stage * and words are this check's): gone, or not a directory, with ENOENT or * ENOTDIR, so a caller reads a root gone mid-read as one gone before it; or - * another directory, with none. + * another directory, with none. A direct mount reads whatever directory is + * at its root (MountConfig.backend), so for it, another directory there is + * one the next read reads. */ private async rootMoved(mount: MountState): Promise { const errno = await rootAbsence(resolve(mount.config.path)); - const reason = errno === undefined ? ROOT_REPLACED : ROOT_UNAVAILABLE; + const reason = errno !== undefined ? ROOT_UNAVAILABLE : mount.direct ? DIRECT_ROOT_REPLACED : ROOT_REPLACED; return new WorkspaceReadError('mount_unavailable', `Mount unavailable: ${mount.config.name}: ${reason}`, 'root', errno); } @@ -2718,9 +3094,14 @@ export class WorkspaceModule implements Module { // mount that doesn't follow symlinks goes through none among the // directories on the way, wherever it leads. Asked again of the path's // resolution after the open, below. - const linkView = { root: mount.config.path, followSymlinks: follow }; + // The checks below judge by the root step 0 checked, and read the links + // on the path's way beneath it. The lstat and the open go through the + // configured path, and step 5 refuses a read whose root is no longer the + // directory step 0 checked, so what's served is judged beneath one root. + const linkView = { root: realMountRoot, followSymlinks: follow }; + const nested = await this.nestedFor(mount, realMountRoot); const onTheWay = await linkOnTheWay(linkView, relativePath, 'parents'); - if (onTheWay !== null) throw await linkReadError(mount, onTheWay, mountPrefixedPath, relativePath, this.nestedOf(mount.config.name)); + if (onTheWay !== null) throw await linkReadError(mount, onTheWay, mountPrefixedPath, relativePath, nested, realMountRoot); let fileInfo: Awaited>; try { @@ -2812,10 +3193,10 @@ export class WorkspaceModule implements Module { // leading to the opened file itself, passes both: a race this doesn't // close. const resolvedThrough = await linkRefusal(linkView, realMountRoot, relativePath, realFilePath); - if (resolvedThrough !== null) throw await linkReadError(mount, resolvedThrough, mountPrefixedPath, relativePath, this.nestedOf(mount.config.name)); + if (resolvedThrough !== null) throw await linkReadError(mount, resolvedThrough, mountPrefixedPath, relativePath, nested, realMountRoot); const inside = isContainedPath(realMountRoot, realFilePath); - const holder = inside ? nestedAtReal(realFilePath, this.nestedOf(mount.config.name)) : undefined; + const holder = inside ? nestedAtReal(realFilePath, nested) : undefined; // A place in a nested mount's directory that its include leaves out is // no mount's, so the path leaves this one (namesNestedPlace), and so // does one reaching a file this mount's own include leaves out @@ -2875,7 +3256,9 @@ export class WorkspaceModule implements Module { * - Once the workspace has recorded the mount root's identity, with the * first evidence it gathers there, a root that is another directory is * an unavailable mount: a replaced directory, a link swapped in for it, - * or an unmounted drive's mountpoint. In any mount, recorded or not, a + * or an unmounted drive's mountpoint. A direct mount's reads check no + * recorded identity: they read beneath whatever directory is at its + * root (MountConfig.backend). In any mount, recorded or not, a * root that stops being the directory the read checked until the file * has been opened and checked is unavailable too: with ENOENT or * ENOTDIR where no directory is there, as before the read, and with no @@ -3322,6 +3705,63 @@ export class WorkspaceModule implements Module { } } + /** + * A file's bytes for read, from where the mount keeps what it reads: the + * workspace's copy, taken in from disk first where it holds none + * (ensureSynced); or, on a direct mount, disk's file as it is now + * (bytesOnDisk). An error says why there are none to read. + */ + private async bytesOf(mount: MountState, relativePath: string, path: string): Promise<{ bytes: Buffer } | { error: string }> { + if (mount.direct) return this.bytesOnDisk(mount, relativePath, path); + const store = this.getStore(); + return this.withMount(mount, async () => { + const report = await this.ensureSynced(mount, relativePath); + const entry = store.treeGet(mount.treeStateId, relativePath); + if (!entry) return { error: this.notInWorkspace(path, report) }; + const blob = store.getBlob(entry.blobHash); + if (!blob) return { error: `Blob not found for: ${path}` }; + return { bytes: blob }; + }); + } + + /** + * A direct mount's file for read: its bytes on disk as they are now, + * through the mount's boundary as readFileFromDisk reaches them + * (openContainedFile), whose refusals are the answer. What a 'chronicle' + * mount never stores, it doesn't show either: a file over the mount's + * size limit, and a binary one, with a NUL in its first 8 KB, which + * read_image reads where it's an image. + */ + private async bytesOnDisk(mount: MountState, relativePath: string, path: string): Promise<{ bytes: Buffer } | { error: string }> { + const maxSize = mount.config.maxFileSize ?? DEFAULT_MAX_FILE_SIZE; + const over = `${path} is over the size limit (${maxSize} bytes), so read doesn't show it.`; + let opened: Awaited>; + try { + opened = await this.openContainedFile(mount, relativePath, path, (fileStat) => { + if (fileStat.size > maxSize) throw new Error(over); + }); + } catch (err) { + // A refusal that names only the path says why with its errno, as a + // failed read below does. + const bare = err instanceof WorkspaceReadError && err.errno !== undefined && err.message === `Unable to read file: ${path}`; + return { error: bare ? `${err.message} (${err.errno})` : err instanceof Error ? err.message : String(err) }; + } + try { + // No more than the limit is loaded, as for a file over it at its fstat. + const { bytes, more } = await readPrefix(opened.handle, maxSize, opened.fileStat.size); + if (more) return { error: over }; + if (looksBinary(bytes)) { + return { error: `${path} is binary, so read doesn't show it as text${sniffImageMime(bytes) ? '; read it with read_image.' : '.'}` }; + } + return { bytes }; + } catch (err) { + const errno = errnoOf(err); + return { error: `Unable to read file: ${path}${errno ? ` (${errno})` : ''}` }; + } finally { + await opened.handle.close(); + } + } + // ========================================================================== // Tool Handlers // ========================================================================== @@ -3339,17 +3779,9 @@ export class WorkspaceModule implements Module { if (content === undefined) { const { mount, relativePath, path } = await this.parseFilePath(input.path); input = { ...input, path }; - const store = this.getStore(); - const found = await this.withMount(mount, async (): Promise<{ text: string } | { error: string }> => { - const report = await this.ensureSynced(mount, relativePath); - const entry = store.treeGet(mount.treeStateId, relativePath); - if (!entry) return { error: this.notInWorkspace(input.path, report) }; - const blob = store.getBlob(entry.blobHash); - if (!blob) return { error: `Blob not found for: ${input.path}` }; - return { text: blob.toString('utf-8') }; - }); + const found = await this.bytesOf(mount, relativePath, path); if ('error' in found) return { success: false, error: found.error, isError: true }; - content = found.text; + content = found.bytes.toString('utf-8'); } if (characterPaging) { const start = Math.min(offsetChars, content.length); @@ -3443,48 +3875,23 @@ export class WorkspaceModule implements Module { const maxSize = mount.config.maxFileSize ?? DEFAULT_MAX_FILE_SIZE; let treeError: WorkspaceImageReadError | null = null; - try { - const image = this.tryReadImageFromTree(mount, relativePath, input.path, maxSize); - if (image) { - return { - success: true, - data: [ - { - type: 'text', - text: `Path: ${input.path}\nMIME: ${image.mimeType}\nBytes: ${image.bytes.byteLength}`, - }, - { - type: 'image', - data: image.bytes.toString('base64'), - mimeType: image.mimeType, - }, - ], - }; - } - } catch (err) { - if (err instanceof WorkspaceImageReadError) { - treeError = err; - } else { - throw err; + // The workspace's copy first, where it holds one, then disk's. A direct + // mount holds none, so its image is disk's. + if (!mount.direct) { + try { + const image = this.tryReadImageFromTree(mount, relativePath, input.path, maxSize); + if (image) return imageResult(input.path, image); + } catch (err) { + if (err instanceof WorkspaceImageReadError) { + treeError = err; + } else { + throw err; + } } } try { - const image = await this.readImageFromFilesystem(mount, relativePath, input.path, maxSize); - return { - success: true, - data: [ - { - type: 'text', - text: `Path: ${input.path}\nMIME: ${image.mimeType}\nBytes: ${image.bytes.byteLength}`, - }, - { - type: 'image', - data: image.bytes.toString('base64'), - mimeType: image.mimeType, - }, - ], - }; + return imageResult(input.path, await this.readImageFromFilesystem(mount, relativePath, input.path, maxSize)); } catch (err) { if (err instanceof WorkspaceImageReadError) { if (err.code === 'not_found' && treeError) { @@ -4004,52 +4411,91 @@ export class WorkspaceModule implements Module { } private async handleLs(input: LsInput): Promise { - const store = this.getStore(); - if (!input.path) { - // List all mounts, each with its include where it has one, so the + // List all mounts, with the kind a mount configured direct runs as + // this session and the ignore list it walks by, whose default no + // config shows, and each with its include where it has one, so the // files a mount holds show before a path outside them is refused. const mounts = [...this.mounts.entries()].map(([name, m]) => ({ name, path: m.config.path, mode: m.config.mode, + ...(m.config.backend === 'direct' ? { backend: m.direct ? 'direct' : 'chronicle', ignore: this.ignoreOf(m) } : {}), ...(m.config.include ? { include: [...m.config.include] } : {}), })); return { success: true, data: { mounts } }; } - const { mount, relativePath, path } = await this.parsePath(input.path); + const { mount, relativePath, path, moved, root } = await this.parsePath(input.path); // A path outside the mount's include that's no scope (refusesAsOutside) // is refused, as glob refuses it, and shown as given. - if (await this.refusesAsOutside(mount, relativePath)) { + if (await this.refusesAsOutside(mount, relativePath, moved, root)) { return { success: false, error: new OutsideIncludeError(input.path, mount.config).message, isError: true }; } input = { ...input, path }; const recursive = input.recursive === true; // Every listing checks the paths it shows against disk first, under the - // three-way rule, whatever the mount's watch mode. - return this.withMount(mount, async (): Promise => { - const pass = await this.passUnlocked(mount, { kind: 'dir', dir: relativePath, recursive }); - const reports = [...pass.reports.values()].sort((a, b) => a.path.localeCompare(b.path)); - const incomplete = pass.incomplete.length > 0 ? { incomplete: pass.incomplete } : {}; + // three-way rule, whatever the mount's watch mode; a direct mount's + // shows disk's. + const listing = await this.listingOf(mount, relativePath, recursive); + const incomplete = listing.incomplete.length > 0 ? { incomplete: listing.incomplete } : {}; + if (recursive) { + return { + success: true, + data: { + path: input.path, + entries: listing.files, + count: listing.files.length, + ...incomplete, + }, + }; + } - if (recursive) { - return { - success: true, - data: { - path: input.path, - entries: reports.map((r) => ({ path: r.path, ...stateView(r) })), - count: reports.length, - ...incomplete, - }, - }; - } + // Non-recursive: the scope's files, and the directories beneath it. + const prefix = relativePath ? relativePath + '/' : ''; + const children: Array> = [ + ...listing.dirs.map((d) => d.slice(prefix.length)).sort().map((name) => ({ name, type: 'directory' })), + ...listing.files.map(({ path: file, ...view }) => ({ name: file.slice(prefix.length), type: 'file', ...view })), + ]; + return { + success: true, + data: { + path: input.path, + entries: children, + count: children.length, + ...incomplete, + }, + }; + } - // Non-recursive: the scope's files, and the directories beneath it that - // disk or the workspace has. - const prefix = relativePath ? relativePath + '/' : ''; - const dirNames = new Set(pass.dirs.map((d) => d.slice(prefix.length))); + /** + * What a listing shows of a directory scope, from where the mount keeps + * what it reads: the store-backed kind's pass (listingFromStore), or, on a + * direct mount, disk (listingFromDisk). `keep` chooses the files shown. + */ + private listingOf(mount: MountState, dir: string, recursive: boolean, keep?: (path: string) => boolean): Promise

{ + return mount.direct ? this.listingFromDisk(mount, dir, recursive, keep) : this.listingFromStore(mount, dir, recursive, keep); + } + + /** + * A 'chronicle' mount's listing: a pass over the scope under the three-way + * rule, each path it decided with its state, and the directories beneath + * the scope that disk or the workspace has. + */ + private listingFromStore(mount: MountState, dir: string, recursive: boolean, keep?: (path: string) => boolean): Promise { + const store = this.getStore(); + return this.withMount(mount, async (): Promise => { + const pass = await this.passUnlocked(mount, { kind: 'dir', dir, recursive }); + const files = [...pass.reports.values()] + .sort((a, b) => a.path.localeCompare(b.path)) + .filter((report) => keep === undefined || keep(report.path)) + .map((report) => ({ path: report.path, ...stateView(report) })); + if (recursive) return { files, dirs: [], incomplete: pass.incomplete }; + // Read with the pass's reports, on the branch it decided on, before + // anything else takes the mount's turn. + const dirs = new Set(pass.dirs); + const prefix = dir ? dir + '/' : ''; // History implies a directory only where it's the mount's own to name // (withinIncludes): not history in a nested mount's directory that // that mount's include leaves out. @@ -4057,24 +4503,60 @@ export class WorkspaceModule implements Module { for (const entry of store.treeList(mount.treeStateId, prefix || undefined)) { if (!withinIncludes(own, entry.path)) continue; const slash = entry.path.indexOf('/', prefix.length); - if (slash >= 0) dirNames.add(entry.path.slice(prefix.length, slash)); + if (slash >= 0) dirs.add(entry.path.slice(0, slash)); } - const children: Array> = [ - ...[...dirNames].sort().map((name) => ({ name, type: 'directory' })), - ...reports.map((r) => ({ name: r.path.slice(prefix.length), type: 'file', ...stateView(r) })), - ]; - return { - success: true, - data: { - path: input.path, - entries: children, - count: children.length, - ...incomplete, - }, - }; + return { files, dirs: [...dirs], incomplete: pass.incomplete }; }); } + /** + * A direct mount's listing: its walk of the scope (diskWalk), with each + * entry it took as disk has it now, observed as a pass observes it: a + * file's size, or why the mount doesn't take it (a link it doesn't follow, + * one that leaves the mount, what isn't a regular file). A file gone since + * the walk listed it is no longer there to show. + */ + private async listingFromDisk(mount: MountState, dir: string, recursive: boolean, keep?: (path: string) => boolean): Promise { + const { view, rootReal, taken, dirs, incomplete } = await this.diskWalk(mount, dir, recursive); + const files: Listing['files'] = []; + await inOrder(taken.filter((path) => keep === undefined || keep(path)), DIRECT_READS_AT_ONCE, (path) => observePath(view, rootReal, path), (seen, path) => { + if (seen.kind === 'file') files.push({ path, size: seen.size }); + else if (seen.kind === 'unobserved') files.push({ path, note: seen.reason }); + }); + return { files, dirs, incomplete }; + } + + /** + * A direct mount's walk of a scope, from its view (diskView): the entries + * it took that aren't directories (files, links and the like), in path + * order; the directories directly beneath a scope it doesn't descend; and + * the regions it couldn't see that a pass reports beside no tracked file: + * what it couldn't list, what the cap left, and a scope in another mount's + * directory, said as a listing says it. A path the walk names is walked + * whatever the mount's ignore list or an editor's file name covers there + * (MountConfig.ignore), as a path sync's walk is. + */ + private async diskWalk(mount: MountState, dir: string, recursive: boolean): Promise<{ + view: MountView; rootReal: string; taken: string[]; dirs: string[]; incomplete: Array<{ path: string; reason: string }>; + }> { + const { view, rootReal, moved } = await this.diskView(mount); + const walk = await walkScope(view, rootReal, dir, recursive, mount.config.walkCap ?? WALK_CAP, { named: true }); + const incomplete = walk.incomplete + .filter((region) => region.kind === 'error' || region.kind === 'cap' || (region.kind === 'ignored' && region.path === dir && dir !== '')) + .map((region) => ({ + path: region.path, + reason: region.reason === OTHER_MOUNT ? this.otherMountNote(mount.config.name, region.path, 'list', moved) : region.reason, + })); + return { + view, + rootReal, + taken: [...walk.files, ...walk.othersTaken].sort((a, b) => a.localeCompare(b)), + // A walk that doesn't descend sees the scope's own subdirectories alone. + dirs: recursive ? [] : [...walk.dirs], + incomplete, + }; + } + private async handleGlob(input: GlobInput): Promise { const regex = globToRegex(input.pattern); const matches: Array> = []; @@ -4083,13 +4565,13 @@ export class WorkspaceModule implements Module { // Search across mounts const mountsToSearch = input.path ? [await this.parsePath(input.path)] - : [...this.mounts.values()].map(m => ({ mount: m, relativePath: '' })); + : [...this.mounts.values()].map(m => ({ mount: m, relativePath: '', moved: null, root: resolve(m.config.path) })); // The path is a directory to search in. One outside the mount's include // that's no scope (refusesAsOutside) is refused, as grep refuses it, // rather than searched and reported as matching nothing. if (input.path) { - const { mount, relativePath } = mountsToSearch[0]!; - if (await this.refusesAsOutside(mount, relativePath)) { + const { mount, relativePath, moved, root } = mountsToSearch[0]!; + if (await this.refusesAsOutside(mount, relativePath, moved, root)) { return { success: false, error: new OutsideIncludeError(input.path, mount.config).message, isError: true }; } } @@ -4100,15 +4582,10 @@ export class WorkspaceModule implements Module { const patternDir = literalDirOf(input.pattern); for (const { mount, relativePath } of mountsToSearch) { const start = await this.walkStart(mount, relativePath, patternDir); - const pass = await this.withMount(mount, () => - this.passUnlocked(mount, { kind: 'dir', dir: start, recursive: true })); - for (const report of [...pass.reports.values()].sort((a, b) => a.path.localeCompare(b.path))) { - const testPath = relativePath ? report.path.slice(relativePath.length + 1) : report.path; - if (regex.test(testPath)) { - matches.push({ path: `${mount.config.name}/${report.path}`, ...stateView(report) }); - } - } - for (const region of pass.incomplete) { + const matched = (path: string) => regex.test(relativePath ? path.slice(relativePath.length + 1) : path); + const listing = await this.listingOf(mount, start, true, matched); + for (const { path, ...view } of listing.files) matches.push({ path: `${mount.config.name}/${path}`, ...view }); + for (const region of listing.incomplete) { incomplete.push({ path: `${mount.config.name}/${region.path}`, reason: region.reason }); } } @@ -4125,7 +4602,6 @@ export class WorkspaceModule implements Module { } private async handleGrep(input: GrepInput): Promise { - const store = this.getStore(); let regex: RegExp; try { regex = new RegExp(input.pattern); @@ -4144,19 +4620,19 @@ export class WorkspaceModule implements Module { const mountsToSearch = input.path ? [await this.parsePath(input.path)] - : [...this.mounts.values()].map(m => ({ mount: m, relativePath: '' })); + : [...this.mounts.values()].map(m => ({ mount: m, relativePath: '', moved: null, root: resolve(m.config.path) })); // A path outside the mount's include that's no scope is refused, as read // refuses it, rather than searched and reported as holding no match. if (input.path) { - const { mount, relativePath } = mountsToSearch[0]!; - if (await this.refusesAsOutside(mount, relativePath)) { + const { mount, relativePath, moved, root } = mountsToSearch[0]!; + if (await this.refusesAsOutside(mount, relativePath, moved, root)) { return { success: false, error: new OutsideIncludeError(input.path, mount.config).message, isError: true }; } } type GrepMatch = { line: number; text: string; context?: string[] }; - const results: Array<{ file: string; state: string; version: 'workspace' | 'conflicting-disk' | 'recorded-disk'; matches: GrepMatch[] }> = []; - const skipped: Array<{ file: string; reason: string; size?: number; mimeType?: string }> = []; + const results: Array<{ file: string; state?: string; version: GrepText['version']; matches: GrepMatch[] }> = []; + const skipped: GrepSkip[] = []; const incomplete: Array<{ path: string; reason: string }> = []; const search = (content: string): GrepMatch[] => { @@ -4176,12 +4652,13 @@ export class WorkspaceModule implements Module { return found; }; - for (const { mount, relativePath } of mountsToSearch) { + for (const { mount, relativePath, moved, root } of mountsToSearch) { // A path named in another mount's directory, as holderOf resolves it, // is that mount's: grep searches nothing of this mount's there, and - // says whose it is. - if (this.holderOf(mount.config.name, relativePath)) { - incomplete.push({ path: `${mount.config.name}/${relativePath}`, reason: this.otherMountNote(mount.config.name, relativePath, 'list') }); + // says whose it is. A direct mount's is judged by the nesting its parse + // found where its root lies now (movedNesting). + if (this.holderOf(mount.config.name, relativePath, false, moved)) { + incomplete.push({ path: `${mount.config.name}/${relativePath}`, reason: this.otherMountNote(mount.config.name, relativePath, 'list', moved) }); continue; } // A `path` and a filter whose directories don't overlap leave no file @@ -4189,109 +4666,16 @@ export class WorkspaceModule implements Module { const deeper = deeperOf(relativePath, filterDir); if (deeper === null) continue; const start = await this.walkStart(mount, relativePath, deeper === relativePath ? '' : deeper.slice(relativePath ? relativePath.length + 1 : 0)); - await this.withMount(mount, async () => { - // A `path` naming a single file greps just that file; otherwise it is a - // directory prefix. (A file path once produced the prefix "notes.md/" - // and silently matched nothing.) - const fileScope = await this.namesFile(mount, relativePath); - // What follows reads the tree and branch intent to go with the pass's - // reports, synchronously. They must be the branch the pass decided - // on: one selected in between would label its content with the other - // branch's states. So a pass whose branch is no longer selected runs - // again, and a branch that keeps changing leaves the scope unsearched. - // - // A copy held at a path that now leads into a nested mount's directory - // through a link is that mount's file too, and one that now lands on a - // file this mount's include leaves out is no file of this mount's: - // read refuses either (parseFilePath), so grep searches none of it and - // says why. The pass - // can't check such a path's disk, so only an unverified one can be - // one. Asking awaits disk, so it's done before the branch is - // compared, and a branch selected meanwhile runs the pass again. - let pass: PassResult | null = null; - let refused = new Map(); - for (let attempt = 0; attempt < 3 && pass === null; attempt++) { - const next = await this.passUnlocked(mount, fileScope - ? { kind: 'paths', paths: [relativePath] } - : { kind: 'dir', dir: start, recursive: true }); - const asked = new Map(); - for (const report of next.reports.values()) { - if (report.state !== 'unverified') continue; - const refusal = await this.landingRefusal(mount, report.path, `${mount.config.name}/${report.path}`); - if (refusal) asked.set(report.path, refusal.message); - } - if (next.branchId === undefined || next.branchId === store.currentBranch().id) { - pass = next; - refused = asked; - } - } - if (pass === null) { - incomplete.push({ - path: `${mount.config.name}/${relativePath}`, - reason: 'the selected branch kept changing while grep ran; nothing here was searched — grep again', - }); - return; - } - for (const region of pass.incomplete) { - incomplete.push({ path: `${mount.config.name}/${region.path}`, reason: region.reason }); - } - - const intents = this.intents.get(mount.config.name)!; - for (const report of [...pass.reports.values()].sort((a, b) => a.path.localeCompare(b.path))) { - if (fileGlob && !fileGlob.test(report.path)) continue; - const file = `${mount.config.name}/${report.path}`; - if (report.state === 'disk-only') { - skipped.push({ - file, - reason: 'disk-only: binary or over the size limit, never stored', - ...(report.size !== undefined ? { size: report.size } : {}), - ...(report.mimeType ? { mimeType: report.mimeType } : {}), - }); - continue; - } - // A path in another mount's directory is that mount's: grep gives - // none of this mount's copy of it, as read refuses it. Beneath a - // scope it's passed over by design, like the directory. - if (inOtherMount(report.path, this.othersOf(mount.config.name))) continue; - const refusal = refused.get(report.path); - if (refusal !== undefined) { - skipped.push({ file, reason: refusal }); - continue; - } - // The workspace's version, labelled with the path's state. - const entry = store.treeGet(mount.treeStateId, report.path); - const blob = entry ? store.getBlob(entry.blobHash) : null; - if (blob) { - const found = search(blob.toString('utf-8')); - if (found.length > 0) results.push({ file, state: report.state, version: 'workspace', matches: found }); - } else if (entry) { - skipped.push({ file, reason: 'the workspace copy is missing from the store, so it was not searched' }); - } - if (report.state !== 'conflict') continue; - // A conflict's disk side. The version recorded with the conflict is - // disk's only while disk still holds it; once disk has changed - // since, it is labelled as recorded, and a file disk holds now - // (never stored) is listed as not searched. A binary or oversize - // disk version was never stored either. - const disk = intents.get(report.path)?.conflict?.disk; - const changed = report.conflict?.diskChangedSinceRecorded === true; - const diskBlob = disk?.stored ? store.getBlob(disk.stored) : null; - if (diskBlob) { - const found = search(diskBlob.toString('utf-8')); - if (found.length > 0) results.push({ file, state: report.state, version: changed ? 'recorded-disk' : 'conflicting-disk', matches: found }); - } - if (disk?.stored && !diskBlob) { - skipped.push({ file, reason: 'conflict: the disk version recorded with the conflict is missing from the store, so it was not searched' }); - } - if (changed) { - if (report.diskNow === 'file') { - skipped.push({ file, reason: 'conflict: disk changed since the conflict was recorded; grep searches stored text, so the file disk holds now was not searched' }); - } - } else if (disk && !disk.stored) { - skipped.push({ file, reason: 'conflict: the disk version is binary or over the size limit, never stored', size: disk.size }); - } - } - }); + const searched = (text: GrepText): void => { + const matches = search(text.text); + if (matches.length > 0) results.push({ file: text.file, ...(text.state !== undefined ? { state: text.state } : {}), version: text.version, matches }); + }; + const kept = (path: string): boolean => fileGlob === null || fileGlob.test(path); + const found = mount.direct + ? await this.textsFromDisk(mount, relativePath, root, start, kept, searched) + : await this.textsFromStore(mount, relativePath, start, kept, searched); + skipped.push(...found.skipped); + incomplete.push(...found.incomplete); } return { @@ -4306,6 +4690,169 @@ export class WorkspaceModule implements Module { }; } + /** + * The texts a 'chronicle' mount's grep searches beneath `start`, or in the + * single file `relativePath` names, those `keep` lets through, each handed + * to `each` in path order: a pass over the scope, then each path's + * workspace copy, labelled with its state, and a conflict's disk side. + * What it doesn't search is in `skipped`, and what it couldn't see in + * `incomplete`, each mount-prefixed. + */ + private async textsFromStore( + mount: MountState, + relativePath: string, + start: string, + keep: (path: string) => boolean, + each: (text: GrepText) => void, + ): Promise<{ skipped: GrepSkip[]; incomplete: Array<{ path: string; reason: string }> }> { + const store = this.getStore(); + const skipped: GrepSkip[] = []; + const incomplete: Array<{ path: string; reason: string }> = []; + await this.withMount(mount, async () => { + // A `path` naming a single file greps just that file; otherwise it is a + // directory prefix. (A file path once produced the prefix "notes.md/" + // and silently matched nothing.) + const fileScope = await this.namesFile(mount, relativePath, resolve(mount.config.path)); + // What follows reads the tree and branch intent to go with the pass's + // reports, synchronously. They must be the branch the pass decided + // on: one selected in between would label its content with the other + // branch's states. So a pass whose branch is no longer selected runs + // again, and a branch that keeps changing leaves the scope unsearched. + // + // A copy held at a path that now leads into a nested mount's directory + // through a link is that mount's file too, and one that now lands on a + // file this mount's include leaves out is no file of this mount's: + // read refuses either (parseFilePath), so grep searches none of it and + // says why. The pass + // can't check such a path's disk, so only an unverified one can be + // one. Asking awaits disk, so it's done before the branch is + // compared, and a branch selected meanwhile runs the pass again. + let pass: PassResult | null = null; + let refused = new Map(); + for (let attempt = 0; attempt < 3 && pass === null; attempt++) { + const next = await this.passUnlocked(mount, fileScope + ? { kind: 'paths', paths: [relativePath] } + : { kind: 'dir', dir: start, recursive: true }); + const asked = new Map(); + for (const report of next.reports.values()) { + if (report.state !== 'unverified') continue; + const refusal = await this.landingRefusal(mount, report.path, `${mount.config.name}/${report.path}`, this.nestedOf(mount.config.name), resolve(mount.config.path)); + if (refusal) asked.set(report.path, refusal.message); + } + if (next.branchId === undefined || next.branchId === store.currentBranch().id) { + pass = next; + refused = asked; + } + } + if (pass === null) { + incomplete.push({ + path: `${mount.config.name}/${relativePath}`, + reason: 'the selected branch kept changing while grep ran; nothing here was searched — grep again', + }); + return; + } + for (const region of pass.incomplete) { + incomplete.push({ path: `${mount.config.name}/${region.path}`, reason: region.reason }); + } + + const intents = this.intents.get(mount.config.name)!; + for (const report of [...pass.reports.values()].sort((a, b) => a.path.localeCompare(b.path))) { + if (!keep(report.path)) continue; + const file = `${mount.config.name}/${report.path}`; + if (report.state === 'disk-only') { + skipped.push({ + file, + reason: 'disk-only: binary or over the size limit, never stored', + ...(report.size !== undefined ? { size: report.size } : {}), + ...(report.mimeType ? { mimeType: report.mimeType } : {}), + }); + continue; + } + // A path in another mount's directory is that mount's: grep gives + // none of this mount's copy of it, as read refuses it. Beneath a + // scope it's passed over by design, like the directory. + if (inOtherMount(report.path, this.othersOf(mount.config.name))) continue; + const refusal = refused.get(report.path); + if (refusal !== undefined) { + skipped.push({ file, reason: refusal }); + continue; + } + // The workspace's version, labelled with the path's state. + const entry = store.treeGet(mount.treeStateId, report.path); + const blob = entry ? store.getBlob(entry.blobHash) : null; + if (blob) { + each({ file, state: report.state, version: 'workspace', text: blob.toString('utf-8') }); + } else if (entry) { + skipped.push({ file, reason: 'the workspace copy is missing from the store, so it was not searched' }); + } + if (report.state !== 'conflict') continue; + // A conflict's disk side. The version recorded with the conflict is + // disk's only while disk still holds it; once disk has changed + // since, it is labelled as recorded, and a file disk holds now + // (never stored) is listed as not searched. A binary or oversize + // disk version was never stored either. + const disk = intents.get(report.path)?.conflict?.disk; + const changed = report.conflict?.diskChangedSinceRecorded === true; + const diskBlob = disk?.stored ? store.getBlob(disk.stored) : null; + if (diskBlob) { + each({ file, state: report.state, version: changed ? 'recorded-disk' : 'conflicting-disk', text: diskBlob.toString('utf-8') }); + } + if (disk?.stored && !diskBlob) { + skipped.push({ file, reason: 'conflict: the disk version recorded with the conflict is missing from the store, so it was not searched' }); + } + if (changed) { + if (report.diskNow === 'file') { + skipped.push({ file, reason: 'conflict: disk changed since the conflict was recorded; grep searches stored text, so the file disk holds now was not searched' }); + } + } else if (disk && !disk.stored) { + skipped.push({ file, reason: 'conflict: the disk version is binary or over the size limit, never stored', size: disk.size }); + } + } + }); + return { skipped, incomplete }; + } + + /** + * The texts a direct mount's grep searches beneath `start`, or in the + * single file `relativePath` names, as namesFile judges it beneath `root`, + * the root grep's parse resolved, those `keep` lets through, each handed + * to `each` in path order: each file its walk took (diskWalk), read from + * disk as it is now, DIRECT_READS_AT_ONCE at a time. A file over the size + * limit, a binary one and one the mount doesn't take are in `skipped`, + * each with why, as a 'chronicle' mount's grep lists what it never stored; + * one gone since the walk listed it isn't there to search. + */ + private async textsFromDisk( + mount: MountState, + relativePath: string, + root: string, + start: string, + keep: (path: string) => boolean, + each: (text: GrepText) => void, + ): Promise<{ skipped: GrepSkip[]; incomplete: Array<{ path: string; reason: string }> }> { + const name = mount.config.name; + const fileScope = await this.namesFile(mount, relativePath, root); + const { view, rootReal, taken, incomplete } = fileScope + ? { ...(await this.diskView(mount)), taken: [relativePath], incomplete: [] } + : await this.diskWalk(mount, start, true); + const maxSize = mount.config.maxFileSize ?? DEFAULT_MAX_FILE_SIZE; + const skipped: GrepSkip[] = []; + await inOrder(taken.filter(keep), DIRECT_READS_AT_ONCE, (path) => readBytes(view, rootReal, path, maxSize), (read, path) => { + const file = `${name}/${path}`; + if (read.kind === 'absent') return; + if (read.kind === 'unobserved') { + skipped.push({ file, reason: read.reason }); + return; + } + const mimeType = sniffImageMime(read.head); + const what = { size: read.size, ...(mimeType ? { mimeType } : {}) }; + if (read.bytes === null) skipped.push({ file, reason: `over the size limit (${maxSize} bytes)`, ...what }); + else if (looksBinary(read.head)) skipped.push({ file, reason: 'binary: a NUL in its first 8 KB', ...what }); + else each({ file, version: 'disk', text: read.bytes.toString('utf-8') }); + }); + return { skipped, incomplete: incomplete.map((region) => ({ path: `${name}/${region.path}`, reason: region.reason })) }; + } + private async handleStatus(input: StatusInput): Promise { const store = this.getStore(); const status: Record = {}; @@ -4315,32 +4862,30 @@ export class WorkspaceModule implements Module { for (const [name, mount] of this.mounts) { if (input.mount !== undefined && name !== input.mount) continue; + if (mount.direct) { + // A direct mount keeps nothing to count: its kind, and the ignore + // list its walks go by, whose default no config shows. + status[name] = { path: mount.config.path, mode: mount.config.mode, backend: 'direct', ignore: this.ignoreOf(mount) }; + continue; + } // What the workspace's history holds outside the mount's include isn't the mount's, // and nor is what it holds in a nested mount's directory that mount's include leaves // out (withinIncludes), which no listing names either. - const inside = (p: string): boolean => insideMount(p, mount.config.include); const listable = { include: mount.config.include, nested: this.nestedOf(name) }; const entries = store.treeList(mount.treeStateId).filter((e) => withinIncludes(listable, e.path)); const currentSeq = store.currentSequence(); - const own = this.ownPaths(mount); - const intents = this.intents.get(name)!.list().filter(([p]) => inside(p) && own(p)); - const conflictPaths = intents.filter(([, bi]) => bi.conflict).map(([p]) => p); - // A workspace deletion is pending until a push confirms it on disk: - // left there without applyDeletions, or unlinked but unconfirmed. A - // tombstone beside an entry that's still there, left by a deletion or - // a write that stopped partway (commitToolChanges), is no deletion. - const deletionPaths = intents.filter(([p, bi]) => bi.tombstone && !store.treeGet(mount.treeStateId, p)).map(([p]) => p); - // Pending is what disk owes by the evidence: every entry it doesn't - // show on disk (a refused or failed push stays owed), conflicts and - // deletions. An entry from before the evidence isn't counted until a - // listing or a materialize checks it: unchecked isn't unpushed. - const pending = new Set([...conflictPaths, ...deletionPaths, ...this.entriesByEvidence(mount).owed]); + const { conflicts: conflictPaths, deletions: deletionPaths, owed } = this.heldChanges(mount); + const pending = new Set([...conflictPaths, ...deletionPaths, ...owed]); const currentBranch = store.currentBranch(); status[name] = { path: mount.config.path, mode: mount.config.mode, - watch: mount.config.watch ?? 'always', + // A mount configured direct runs as 'chronicle' this session + // (chooseKinds), by the direct kind's ignore list where its config + // names none. + ...(mount.config.backend === 'direct' ? { backend: 'chronicle', ignore: this.ignoreOf(mount) } : {}), + watch: watchOf(mount.config), fileCount: entries.length, lastMaterializedSeq: mount.lastMaterializedSeq, currentSeq, @@ -4358,6 +4903,37 @@ export class WorkspaceModule implements Module { return { success: true, data: status }; } + /** + * What the store holds of a mount that disk lacks, on the selected branch, + * among the mount's own paths. Status counts the first three as pending: + * recorded conflicts; workspace deletions, pending until a push confirms + * them on disk (left there without applyDeletions, or unlinked but + * unconfirmed); and what disk owes by the evidence (entriesByEvidence's + * `owed`: every entry it doesn't show on disk, a refused or failed push's + * included). An entry from before the evidence isn't among them until a + * listing or a materialize checks it: unchecked isn't unpushed. A + * tombstone beside an entry that's still there, left by a deletion or a + * write that stopped partway (commitToolChanges), is no deletion: it holds + * no change disk lacks, and its entry is judged by the evidence as any + * other is. The fourth, intents that can't be read, may hold a deletion + * or a conflict, so a mount configured direct counts them too (initStore). + * Each lies where the mount's include covers, as status counts them: what + * the workspace's history holds outside it is no change of the mount's, + * and no push writes it to disk (pushUnlocked). + */ + private heldChanges(mount: MountState): { conflicts: string[]; deletions: string[]; owed: string[]; unreadable: string[] } { + const store = this.getStore(); + const own = this.ownPaths(mount); + // What the workspace's history holds outside the mount's include isn't the mount's. + const intents = this.intents.get(mount.config.name)!.list().filter(([p]) => insideMount(p, mount.config.include) && own(p)); + return { + conflicts: intents.filter(([, bi]) => bi.conflict).map(([p]) => p), + deletions: intents.filter(([p, bi]) => bi.tombstone && !store.treeGet(mount.treeStateId, p)).map(([p]) => p), + owed: this.entriesByEvidence(mount).owed, + unreadable: intents.filter(([, bi]) => bi.unreadable !== undefined).map(([p]) => p), + }; + } + /** * What a materialize of `mount` should consider: a named file, or every * tracked path beneath a named directory; otherwise what the evidence says @@ -4456,11 +5032,12 @@ export class WorkspaceModule implements Module { let explicit: { mount: MountState; relativePath: string } | null = null; if (input.path) { - explicit = await this.parsePath(input.path); + const parsed = await this.parsePath(input.path); + explicit = parsed; // A path outside the mount's include that's no scope (refusesAsOutside) // is refused; beneath a directory, the push leaves out what the include // doesn't cover. - if (await this.refusesAsOutside(explicit.mount, explicit.relativePath)) { + if (await this.refusesAsOutside(parsed.mount, parsed.relativePath, parsed.moved, parsed.root)) { return { success: false, error: new OutsideIncludeError(input.path, explicit.mount.config).message, isError: true }; } } @@ -4657,11 +5234,11 @@ export class WorkspaceModule implements Module { // A path names its own mount; otherwise the given mount, or every mount. let targets: Array<{ name: string; mount: MountState; relativePath: string }>; if (input.path) { - const { mount, relativePath } = await this.parsePath(input.path); + const { mount, relativePath, moved, root } = await this.parsePath(input.path); // A path outside the mount's include that's no scope (refusesAsOutside) // is refused; beneath a directory, the walk passes over what the include // doesn't cover. - if (await this.refusesAsOutside(mount, relativePath)) { + if (await this.refusesAsOutside(mount, relativePath, moved, root)) { return { success: false, error: new OutsideIncludeError(input.path, mount.config).message, isError: true }; } targets = [{ name: mount.config.name, mount, relativePath }]; @@ -4676,6 +5253,12 @@ export class WorkspaceModule implements Module { } for (const { name, mount, relativePath } of targets) { + // A direct mount's tools read disk as it is, so a sync has nothing to + // take in there, and says it didn't look rather than that nothing changed. + if (mount.direct) { + allSkipped.push({ mount: name, path: relativePath, reason: DIRECT_SYNC_SKIPPED }); + continue; + } const pass = await this.withMount(mount, async () => { if (input.acceptRoot && await this.acceptRootUnlocked(mount)) rootsAccepted.push(name); if (!relativePath) { @@ -4748,7 +5331,7 @@ export class WorkspaceModule implements Module { // without a path passes over it, so nothing else keeps it current // (agent-framework #276). A path in another mount's directory isn't // synced at all, and the note says so. - const passedOver = passedOverAt(relativePath, mount.config.ignore ?? [], this.othersOf(name)); + const passedOver = passedOverAt(relativePath, this.ignoreOf(mount), this.othersOf(name)); if (passedOver !== undefined) { allIgnored.push({ mount: name, @@ -4840,7 +5423,7 @@ export class WorkspaceModule implements Module { async onToolBatchComplete(_agentName?: string): Promise { // Every agent's workspace is the store's current branch, so the agent // that ran the batch doesn't choose what is scanned. - const mounts = [...this.mounts.values()].filter((m) => m.config.watch === 'on-agent-action'); + const mounts = [...this.mounts.values()].filter((m) => watchOf(m.config) === 'on-agent-action'); if (mounts.length === 0 || !this.store || !this.running()) return; const deadlineMs = this.agentActionScanDeadlineMs(); await Promise.all(mounts.map(async (mount) => { @@ -4989,6 +5572,72 @@ export class WorkspaceModule implements Module { } +/** + * A text grep searches (textsFromStore, textsFromDisk): its mount-prefixed + * file, which copy it is, and on a 'chronicle' mount, the path's state. + */ +interface GrepText { + file: string; + state?: string; + version: 'workspace' | 'conflicting-disk' | 'recorded-disk' | 'disk'; + text: string; +} + +/** A file, or a copy of one, grep doesn't search, with why. */ +type GrepSkip = { file: string; reason: string; size?: number; mimeType?: string }; + +/** What a listing shows of a directory scope (listingOf). */ +interface Listing { + /** Each file it shows, by mount-relative path, in order, with what a listing says of it beyond its name. */ + files: Array<{ path: string } & Record>; + /** For a listing that doesn't descend, the directories directly beneath the scope, mount-relative; none for one that does. */ + dirs: string[]; + /** The regions it couldn't see, mount-relative, each with why. */ + incomplete: Array<{ path: string; reason: string }>; +} + +/** + * Run `work` on each item, at most `width` at once, and hand each result to + * `each` in the items' order, as soon as it and those before it are done. + * A result handed on is let go, so what's held at once is at most `width` + * results, as a direct grep's file contents are. A `work` that rejects + * rejects this when its turn comes; any started after it finish unheard. + * Exported for its tests. + */ +export async function inOrder(items: readonly T[], width: number, work: (item: T) => Promise, each: (result: R, item: T) => void): Promise { + const waiting: Array> = []; + let started = 0; + for (let i = 0; i < items.length; i++) { + while (started < items.length && started < i + width) { + const running = work(items[started]!); + // Handled from the start, so a rejection that comes before its turn + // isn't an unhandled one; awaiting it below still throws it. + running.catch(() => {}); + waiting.push(running); + started++; + } + each(await waiting.shift()!, items[i]!); + } +} + +/** read_image's answer for an image, wherever it was read from. */ +function imageResult(path: string, image: { bytes: Buffer; mimeType: SupportedImageMimeType }): ToolResult { + return { + success: true, + data: [ + { + type: 'text', + text: `Path: ${path}\nMIME: ${image.mimeType}\nBytes: ${image.bytes.byteLength}`, + }, + { + type: 'image', + data: image.bytes.toString('base64'), + mimeType: image.mimeType, + }, + ], + }; +} + /** What a listing shows of a path beyond its name. */ function stateView(report: PathReport): Record { return { @@ -5007,6 +5656,49 @@ function stateView(report: PathReport): Record { /** A lone UTF-16 surrogate, which UTF-8 can't store: in `u` mode, a pair is one code point. */ const LONE_SURROGATE = /\p{Cs}/u; +/** Why a direct mount's read stopped when it found another directory put at its root as it opened the file. */ +const DIRECT_ROOT_REPLACED = 'the mount root was replaced as the file was opened: reading it again reads the directory there now'; + +/** + * How the alert that a mount configured direct runs as 'chronicle' names an + * intent the workspace can't read. A sync of its path settles it, with + * disk's copy, or with none where disk has none, and where disk's file + * can't be read it skips the path and says why. Materialize settles one + * only where the workspace holds a copy there: a plain one skips it and says + * to sync, a forced one writes the copy, and where the workspace holds none + * it neither selects nor names it. + */ +const UNREADABLE_INTENT = "its workspace intent can't be read; a sync of the path settles it or says why not"; + +/** + * A mount's watch mode as it runs. A mount configured direct watches + * nothing, whichever kind it runs as this session: its config takes no + * other mode (directRefusals), and leaving it out means 'never' there. + */ +function watchOf(config: MountConfig): 'always' | 'on-agent-action' | 'never' { + return config.watch ?? (config.backend === 'direct' ? 'never' : 'always'); +} + +/** + * What a direct mount's config sets that would never act there, each with + * the way out (MountConfig.backend): a direct mount is read-only, and has no + * watcher to tune, no change event to wake on and no push to make. + */ +function directRefusals(mount: MountConfig): string[] { + const refused: string[] = []; + if (mount.mode !== 'read-only') { + refused.push(mount.mode === undefined ? "mode must be 'read-only' (left out, it's 'read-write')" : `mode must be 'read-only', not ${shownValue(mount.mode)}`); + } + if (mount.watch !== undefined && mount.watch !== 'never') refused.push(`watch ${shownValue(mount.watch)} would watch nothing (leave it out, or set 'never')`); + for (const key of ['watchDebounceMs', 'watchRootPollMs'] as const) { + if (mount[key] !== undefined) refused.push(`${key} would tune no watcher (leave it out)`); + } + const wake = mount.wakeOnChange; + if (wake === true || (Array.isArray(wake) && wake.length > 0)) refused.push('wakeOnChange would never wake (leave it out)'); + if (mount.autoMaterialize === true) refused.push('autoMaterialize would never push (leave it out)'); + return refused; +} + /** A change commitToolChanges makes to one path's tree entry. */ type ToolChange = { kind: 'set'; blobHash: string; size: number; mode?: number } | { kind: 'remove'; previousHash: string }; diff --git a/src/modules/workspace/observe.ts b/src/modules/workspace/observe.ts index 23568c403..a19794623 100644 --- a/src/modules/workspace/observe.ts +++ b/src/modules/workspace/observe.ts @@ -49,7 +49,11 @@ import { basename, dirname, isAbsolute, join, posix, relative, resolve, sep } fr import { sameRootIdentity, type Fingerprint, type RootIdentity } from './disk-agreement.js'; export interface MountView { - /** Mount root as configured. */ + /** + * The directory its readers read beneath: the mount's root as configured, + * or, for a direct mount, the root its view resolved (diskView), so a + * walk opens what it judges beneath one root. + */ root: string; /** The mount's name, which a tool puts before a path in the mount. */ name: string; @@ -1425,6 +1429,66 @@ export async function readPath( } } +/** A file's bytes as readBytes read them, none hashed. */ +export interface BytesResult { + kind: 'bytes'; + /** + * Its size in bytes: as read, for a file within the limit; as its fstat + * says, for one over it, and at least one past the limit where the reads + * found it grown past it. + */ + size: number; + /** The whole file, when it is at most the limit both at its fstat and as read; null otherwise. */ + bytes: Buffer | null; + /** Its first bytes (at most HEAD_BYTES), for binary detection and MIME sniffing. */ + head: Buffer; +} + +/** + * A mount-relative file's bytes, through the same located descriptor as + * readPath, for a direct mount's tools, which keep nothing: at most `limit` + * of them, and none hashed. A file over `limit` at its fstat has only its + * head read, and one that grew past it since is read no further than it. + * What the open refuses gets the reason a listing gives it, such as a link + * to a directory, except a regular file it couldn't open, which keeps the + * open's. + */ +export async function readBytes( + view: MountView, + rootReal: string, + relativePath: string, + limit: number, +): Promise> { + const opened = await openContained(view, rootReal, relativePath); + if (!('handle' in opened)) { + // A walk takes a link to a directory as an entry, where the mount follows + // links, and the open may refuse it as a directory or, where it can't + // be opened, by its errno: either way it's named as a listing names it + // (observePath). + if (opened.kind === 'unobserved') { + const seen = await observePath(view, rootReal, relativePath); + return seen.kind === 'file' ? opened : seen; + } + return opened; + } + const { handle, info } = opened; + try { + const size = Number(info.size); + if (size > limit) { + const head = Buffer.alloc(Math.min(HEAD_BYTES, size)); + const { bytesRead } = await handle.read(head, 0, head.length, 0); + return { kind: 'bytes', size, bytes: null, head: head.subarray(0, bytesRead) }; + } + const { bytes, more } = await readPrefix(handle, limit, size); + if (more) return { kind: 'bytes', size: Math.max(Number((await handle.stat({ bigint: true })).size), bytes.length + 1), bytes: null, head: bytes.subarray(0, HEAD_BYTES) }; + return { kind: 'bytes', size: bytes.byteLength, bytes, head: bytes.subarray(0, HEAD_BYTES) }; + } catch (err) { + return { kind: 'unobserved', reason: `cannot read (${errno(err) ?? String(err)})` }; + } finally { + await handle.close(); + } +} + /** * The first bytes of a mount-relative file and its size, through the same * located descriptor as readPath — for listing a file nothing tracks, which diff --git a/src/modules/workspace/types.ts b/src/modules/workspace/types.ts index d2718c280..82a28c27b 100644 --- a/src/modules/workspace/types.ts +++ b/src/modules/workspace/types.ts @@ -16,6 +16,50 @@ export interface MountConfig { path: string; /** Access mode */ mode: 'read-write' | 'read-only'; + /** + * Where the mount's tools get what they read (default: 'chronicle'). + * - 'chronicle': the workspace keeps the mount's files in its store, where + * writes and edits land first, and takes in disk's changes under the + * three-way rule as its listings, searches, syncs and scans reach them. + * A read of a path the store doesn't hold takes in a new file there; + * otherwise it reads the store's copy. + * - 'direct': for a checkout an agent reads, not one it changes. Every + * read, listing and search is disk's own copy as it is when it runs, + * and none of its files go into the store. No scan or watcher + * runs, so no call waits behind one, and `sync`, `materialize` and the + * watch settings have nothing to act on. Only with `mode: 'read-only'`, + * and not with `watch` other than 'never', `watchDebounceMs`, + * `watchRootPollMs`, a `wakeOnChange` that would wake (true, or a + * non-empty list) or `autoMaterialize: true`: each of those would never + * act, and the workspace refuses the mount. Its `ignore` + * has a default (see there). Paths, links, its `include`, other mounts' + * directories, size limits and the walk cap hold as on a 'chronicle' + * mount. Its tools read beneath whatever directory is at its + * root, with no check against the one a 'chronicle' run of it last + * agreed with. Where that root now lies elsewhere than the last start + * placed it, each call nests in it, as a start would now, every mount + * configured beneath its path, where that path leads beneath its root, + * and every other mount whose root, as that start placed it, lies + * beneath where its root lies now, or whose configured path lies + * beneath a directory that start found there. A mount configured + * elsewhere whose own root has moved since, and the other mounts' own + * placements, wait for the next start, so a direct root moved into + * another mount's directory is still that mount's too. + * + * A mount configured 'direct' runs as 'chronicle' from a start that finds + * the store holding changes there that disk lacks, on the branch selected + * then: a draft, a workspace deletion or a conflict, as `status` counts + * them pending, or an intent the workspace can't read, which may hold one + * of those. They stay in view, and the workspace says so on stderr and + * through the framework's ops alerts, naming them and the ways to start it + * direct. It runs unwatched, and walks by the direct kind's default + * ignore list where its config names none; `ls` without a path, and + * `status`, give its `backend` as 'chronicle' and that list. A mount that + * ran direct and runs as 'chronicle' again takes in what disk changed + * meanwhile as its listings, searches and syncs reach it, as an unwatched + * 'chronicle' mount does after any time away. + */ + backend?: 'chronicle' | 'direct'; /** * Watch mode for filesystem changes: * - 'always': chokidar watches continuously; each event re-observes its paths @@ -25,8 +69,8 @@ export interface MountConfig { * the round goes on. Each scan walks from the mount's start, so on a * mount over its `walkCap` it covers the first `walkCap` entries every * time, and files past them come in as `walkCap` describes - * - 'never': no background reads; listings and lazy reads still observe - * the paths they show + * - 'never': no background reads; listings still observe the paths they + * show, and a read observes a path the store doesn't hold */ watch?: 'always' | 'on-agent-action' | 'never'; /** Debounce window in ms for watch: 'always' mode (default: 300) */ @@ -43,6 +87,16 @@ export interface MountConfig { * says, a directory with what lies beneath it: `.name.swp` and `.name.swx` * (the whole name), and names ending in `~` or in `.subl…tmp`, matched by * `^\..+\.sw[px]$`, `~$` and `\.subl.*\.tmp$`. + * + * A mount configured direct (`backend`) whose config names no list takes + * `node_modules`, `dist`, `.git`, `build`, `coverage`, `target`, + * `__pycache__` and `.venv`, each matching that name at any depth, + * whichever kind it runs as. A direct mount's walks of the whole mount, + * and of any path no pattern covers, pass over what the list covers. A + * listing, glob or grep whose path, or whose pattern's directory, the + * list covers is walked all the same, the patterns covering it set aside + * and the rest applying beneath it, as a sync that names a path walks it + * on a 'chronicle' mount. */ ignore?: string[]; /** @@ -98,7 +152,9 @@ export interface MountConfig { * include doesn't cover, and a listing that isn't recursive names each * directory directly beneath its path that disk has, and each that a * covered entry in history implies, with only the files the include - * covers. `ignore` still applies to what `include` covers. + * covers. A mount running direct has no history (`backend`), so there + * disk alone decides each of these. `ignore` still applies to what + * `include` covers. * * On a disk that folds names, the parse reads a path in disk's spelling, * as the walk names what it lists, so a covered name whose file disk @@ -126,7 +182,8 @@ export interface MountConfig { * link on a path's way is refused, as on any such mount (followSymlinks). * A nested mount's root is placed where disk has it at each start, and its * directory is matched by spelling, so four cases fall outside this, as - * for any nested mount: on a disk that folds names, a mount configured + * for any nested mount, and a fifth on a direct one: on a disk that folds + * names, a mount configured * through a link that spells a directory otherwise than disk does isn't * found nested, and this mount holds its files as its own; on such a disk, * a path through a link that spells a nested mount's directory otherwise @@ -139,7 +196,11 @@ export interface MountConfig { * spelling the link's target gives, so a refusal can name a file disk * spells outside that include, or leave a covered one unnamed; and a * nested mount whose root moves while running, as `acceptRoot` takes, - * stays placed where it was until the next start. + * stays placed where it was until the next start; and where a direct + * mount's root is repointed between a read's parse and its open, a race + * left to whoever writes the disk, a mount nested in it by its + * directories alone, since its real root holds that root, can have the + * read serve a path in its directory. * * Text the host registers for a path (`registerGeneratedTextFile`) is * served there whatever the include says, since it isn't one of the @@ -271,6 +332,13 @@ export interface WorkspaceConfig { export interface MountState { /** The mount config */ config: MountConfig; + /** + * Whether the mount runs direct (`MountConfig.backend`), as the workspace + * decides when initStore gives the store and again at each start: a mount + * configured direct runs as 'chronicle' while the store holds changes + * there that disk lacks. + */ + direct: boolean; /** Tree state ID in Chronicle */ treeStateId: string; /** Branch-local intent (tombstones, store origin, conflicts) for this mount's paths */ diff --git a/test/workspace-direct.test.ts b/test/workspace-direct.test.ts new file mode 100644 index 000000000..9c5e25f5a --- /dev/null +++ b/test/workspace-direct.test.ts @@ -0,0 +1,2401 @@ +/** + * A read-only direct mount (MountConfig.backend 'direct'): its tools read + * disk as it is when they run, record nothing in the store, and give the + * answers a 'chronicle' mount's tools give where the two should agree. And + * the switch between the kinds, both ways. + */ + +import test, { type TestContext } from 'node:test'; +import assert from 'node:assert/strict'; +import { appendFileSync, chmodSync, existsSync, mkdirSync, mkdtempSync, realpathSync, renameSync, rmSync, statSync, symlinkSync, truncateSync, unlinkSync, writeFileSync } from 'node:fs'; +import { createRequire, syncBuiltinESMExports } from 'node:module'; +import { tmpdir } from 'node:os'; +import { basename, dirname, join } from 'node:path'; +import { setFlagsFromString } from 'node:v8'; +import { runInNewContext } from 'node:vm'; +import { JsStore } from '@animalabs/chronicle'; +// A namespace, so this file loads where the module lacks inOrder, and only +// the tests that call it fail there. +import * as workspaceModule from '../src/modules/workspace/index.js'; +import { WorkspaceModule, WorkspaceReadError } from '../src/modules/workspace/index.js'; +import { BranchIntents } from '../src/modules/workspace/branch-intent.js'; +import type { MountConfig } from '../src/modules/workspace/types.js'; +import type { ModuleContext } from '../src/types/module.js'; +import type { ProcessEvent, ToolResult } from '../src/types/events.js'; +import { changedAfterFstat, countingLoads, withFileHandle } from './helpers/file-handle.js'; + +const IS_ROOT = typeof process.getuid === 'function' && process.getuid() === 0; + +const PNG = Buffer.from( + 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR4nGP4////fwAJ+wP9KobjigAAAABJRU5ErkJggg==', + 'base64', +); +const GIF = Buffer.from('R0lGODdhAQABAIEAAP///wAAAAAAAAAAACwAAAAAAQABAAAIBAABBAQAOw==', 'base64'); + +type Alert = { kind: string; agent: string; message: string; data?: Record }; + +class Env { + readonly root = mkdtempSync(join(tmpdir(), 'af-ws-direct-')); + readonly ref = join(this.root, 'ref'); + readonly storePath = join(this.root, 'ws.chronicle'); + store: JsStore; + events: ProcessEvent[] = []; + alerts: Alert[] = []; + private live: WorkspaceModule[] = []; + + constructor(t: TestContext) { + mkdirSync(this.ref, { recursive: true }); + this.store = JsStore.openOrCreate({ path: this.storePath }); + t.after(async () => { + for (const m of this.live) await m.stop().catch(() => {}); + if (!this.store.isClosed()) this.store.close(); + rmSync(this.root, { recursive: true, force: true }); + }); + } + + file(rel: string, content: string | Buffer): void { + mkdirSync(dirname(join(this.ref, rel)), { recursive: true }); + writeFileSync(join(this.ref, rel), content); + } + + /** A module whose `ref` mount is direct and read-only unless `mount` says otherwise, with `more` mounts beside it. */ + async open(mount: Partial = {}, more: MountConfig[] = []): Promise { + const m = new WorkspaceModule({ + mounts: [{ name: 'ref', path: this.ref, mode: 'read-only', backend: 'direct', ...mount }, ...more], + }); + m.initStore(this.store); + await m.start(this.context()); + this.live.push(m); + return m; + } + + /** A module context as a host gives one, recording events and ops alerts. */ + context(): ModuleContext { + return { + isRestart: false, + getState: () => null, + setState: () => {}, + pushEvent: (e: ProcessEvent) => this.events.push(e), + notifyOps: (kind: string, agent: string, message: string, data?: Record) => this.alerts.push({ kind, agent, message, data }), + } as unknown as ModuleContext; + } + + /** Stop the module and start it again, with the store it has. */ + async restart(m: WorkspaceModule): Promise { + await m.stop(); + await m.start(this.context()); + } + + /** The same mount as a 'chronicle' one, read-write and unwatched. */ + openChronicle(): Promise { + return this.open({ backend: 'chronicle', mode: 'read-write', watch: 'never' }); + } + + async close(m: WorkspaceModule): Promise { + await m.stop(); + this.live = this.live.filter((x) => x !== m); + } + + stateIds(): string[] { + return this.store.listStates().map((s) => s.id).sort(); + } +} + +/** The mounts a module watches. */ +function watched(m: WorkspaceModule): string[] { + return [...(m as unknown as { watchers: Map }).watchers.keys()]; +} + +/** Run `fn` with `node:fs/promises`'s `name`, as the module's imports see it, replaced by what `make` makes of it. */ +async function withFsPromise(name: 'lstat' | 'open' | 'realpath' | 'stat', make: (original: (...args: any[]) => Promise) => (...args: any[]) => Promise, fn: () => Promise): Promise { + const fsp = createRequire(import.meta.url)('node:fs/promises') as Record Promise>; + const original = fsp[name]!; + fsp[name] = make(original); + syncBuiltinESMExports(); + try { + return await fn(); + } finally { + fsp[name] = original; + syncBuiltinESMExports(); + } +} + +/** + * The most calls of `node:fs/promises`'s `name` on the files `counted` names + * under way at once while `fn` runs. The first are held until `width` of + * them are under way, and half a second longer, in which a reader wider than + * `width` would start more, so a reader `width` wide shows as `width` however + * fast the file system answers each step before the call. A narrower one is + * let go after 5 seconds, so it fails rather than hangs. + */ +async function mostAtOnce(name: 'lstat' | 'open', width: number, counted: RegExp, fn: () => Promise): Promise { + let now = 0; + let most = 0; + let release!: () => void; + const released = new Promise((resolve) => { release = resolve; }); + const fallback = setTimeout(release, 5000); + let settle: NodeJS.Timeout | undefined; + try { + await withFsPromise(name, (original) => async (...args: any[]) => { + if (!counted.test(String(args[0]))) return original(...args); + now++; + most = Math.max(most, now); + if (now === width && settle === undefined) settle = setTimeout(release, 500); + try { + await released; + return await original(...args); + } finally { + now--; + } + }, fn); + } finally { + clearTimeout(fallback); + clearTimeout(settle); + } + return most; +} + +function call(m: WorkspaceModule, name: string, input: Record = {}): Promise { + return m.handleToolCall({ id: name, name, input }); +} + +async function data>(m: WorkspaceModule, name: string, input: Record = {}): Promise { + const result = await call(m, name, input); + assert.equal(result.success, true, `${name} ${JSON.stringify(input)}: ${result.error}`); + return result.data as T; +} + +async function failure(m: WorkspaceModule, name: string, input: Record): Promise { + const result = await call(m, name, input); + assert.equal(result.success, false, `${name} ${JSON.stringify(input)} succeeded: ${JSON.stringify(result.data)}`); + return result.error!; +} + +test('a direct mount reads disk as it is now, and records nothing in the store from its start on', async (t) => { + const env = new Env(t); + env.file('src/a.ts', 'one\n'); + env.file('img/dot.png', PNG); + const states = env.stateIds(); + const sequence = env.store.currentSequence(); + const m = await env.open(); + + assert.equal((await data<{ content: string }>(m, 'read', { path: 'ref/src/a.ts' })).content, ' 1\tone'); + appendFileSync(join(env.ref, 'src/a.ts'), 'two\n'); + assert.equal((await data<{ content: string }>(m, 'read', { path: 'ref/src/a.ts' })).content, ' 1\tone\n 2\ttwo'); + await data(m, 'ls', { path: 'ref', recursive: true }); + await data(m, 'ls', { path: 'ref/src' }); + await data(m, 'glob', { pattern: '**/*.ts' }); + await data(m, 'grep', { pattern: 'two' }); + await data(m, 'read_image', { path: 'ref/img/dot.png' }); + await data(m, 'status'); + await data(m, 'sync'); + assert.deepEqual('data' in (await m.readBinary('ref/src/a.ts')), true); + await m.onToolBatchComplete(); + + assert.deepEqual(env.stateIds(), states, 'no state was made'); + assert.equal(env.store.currentSequence(), sequence, 'nothing was recorded, its disk evidence included'); + assert.deepEqual(env.events.filter((e) => e.type !== 'workspace:mounted').map((e) => e.type), [], 'no change was evented'); + assert.deepEqual(watched(m), [], 'no watcher runs, though watch is left out'); +}); + +test("read on a direct mount refuses what a 'chronicle' mount never stores: a binary file, and one over the size limit", async (t) => { + const env = new Env(t); + env.file('bin.dat', Buffer.from([0x61, 0x00, 0x62])); + env.file('dot.png', PNG); + env.file('big.txt', 'x'.repeat(9)); + env.file('fits.txt', 'x'.repeat(8)); + const m = await env.open({ maxFileSize: 8 }); + assert.equal(await failure(m, 'read', { path: 'ref/bin.dat' }), "ref/bin.dat is binary, so read doesn't show it as text."); + assert.equal(await failure(m, 'read', { path: 'ref/dot.png' }), "ref/dot.png is over the size limit (8 bytes), so read doesn't show it."); + assert.equal(await failure(m, 'read', { path: 'ref/big.txt' }), "ref/big.txt is over the size limit (8 bytes), so read doesn't show it."); + assert.equal((await data<{ content: string }>(m, 'read', { path: 'ref/fits.txt' })).content, ' 1\txxxxxxxx'); + + const roomy = await env.open({ maxFileSize: 1024 }); + assert.equal(await failure(roomy, 'read', { path: 'ref/dot.png' }), "ref/dot.png is binary, so read doesn't show it as text; read it with read_image."); +}); + +test("read on a direct mount answers with the disk reader's refusals", async (t) => { + const env = new Env(t); + env.file('src/a.ts', 'a\n'); + symlinkSync(join(env.ref, 'src/a.ts'), join(env.ref, 'link.ts')); + symlinkSync(join(env.ref, 'src'), join(env.ref, 'srclink')); + const m = await env.open(); + assert.equal(await failure(m, 'read', { path: 'ref/missing.ts' }), 'File not found: ref/missing.ts'); + assert.equal(await failure(m, 'read', { path: 'ref/src' }), 'Path is a directory: ref/src'); + assert.equal(await failure(m, 'read', { path: 'ref/link.ts' }), 'Symlinks are not allowed: ref/link.ts'); + assert.equal( + await failure(m, 'read', { path: 'ref/srclink/a.ts' }), + 'ref/srclink/a.ts: "ref/srclink" is a symlink to "ref/src", which this mount does not follow; the mount\'s host can set followSymlinks: true', + ); + assert.match(await failure(m, 'read', { path: 'ref/../escape.txt' }), /^Path traversal detected/); +}); + +test("read_image on a direct mount never shows the copy a 'chronicle' run stored", async (t) => { + const env = new Env(t); + const chronicle = await env.open({ backend: 'chronicle', mode: 'read-write', watch: 'never', autoMaterialize: true }); + assert.equal((await chronicle.writeBinary('ref/dot.img', PNG, 'image/png')).success, true); + await env.close(chronicle); + env.file('dot.img', GIF); + const direct = await env.open(); + assert.deepEqual(env.alerts, [], 'it starts direct'); + const image = await data>(direct, 'read_image', { path: 'ref/dot.img' }); + assert.deepEqual([image[1]!.mimeType, image[1]!.data], ['image/gif', GIF.toString('base64')]); +}); + +test("read_image on a direct mount reads disk's image as it is now", async (t) => { + const env = new Env(t); + env.file('dot.img', PNG); + const m = await env.open(); + const first = await data>(m, 'read_image', { path: 'ref/dot.img' }); + assert.equal(first[0]!.text, `Path: ref/dot.img\nMIME: image/png\nBytes: ${PNG.byteLength}`); + assert.equal(first[1]!.data, PNG.toString('base64')); + env.file('dot.img', GIF); + const second = await data>(m, 'read_image', { path: 'ref/dot.img' }); + assert.equal(second[1]!.mimeType, 'image/gif'); + assert.equal(second[1]!.data, GIF.toString('base64')); +}); + +test('ls on a direct mount lists disk\'s entries with their sizes and no state, and a link with why it isn\'t taken', async (t) => { + const env = new Env(t); + env.file('a.txt', 'abc'); + env.file('src/b.ts', 'bb'); + symlinkSync(join(env.ref, 'a.txt'), join(env.ref, 'link.txt')); + const m = await env.open(); + assert.deepEqual((await data(m, 'ls', { path: 'ref' })).entries, [ + { name: 'src', type: 'directory' }, + { name: 'a.txt', type: 'file', size: 3 }, + { name: 'link.txt', type: 'file', note: 'a symlink, which this mount does not follow' }, + ]); + assert.deepEqual(await data(m, 'ls', { path: 'ref', recursive: true }), { + path: 'ref', + entries: [ + { path: 'a.txt', size: 3 }, + { path: 'link.txt', note: 'a symlink, which this mount does not follow' }, + { path: 'src/b.ts', size: 2 }, + ], + count: 3, + }); + + const following = await env.open({ followSymlinks: true }); + assert.deepEqual((await data(following, 'ls', { path: 'ref' })).entries, [ + { name: 'src', type: 'directory' }, + { name: 'a.txt', type: 'file', size: 3 }, + { name: 'link.txt', type: 'file', size: 3 }, + ]); +}); + +test('the default ignore list keeps whole walks out of what it covers, and a path a tool names is walked', async (t) => { + const env = new Env(t); + env.file('src/a.js', 'needle\n'); + env.file('node_modules/pkg/index.js', 'needle\n'); + env.file('node_modules/pkg/dist/built.js', 'needle\n'); + env.file('dist/out.js', 'needle\n'); + const m = await env.open(); + // A whole walk passes over what the list covers, and doesn't count it as + // a region it couldn't reach. + assert.deepEqual(await data(m, 'ls', { path: 'ref' }), { path: 'ref', entries: [{ name: 'src', type: 'directory' }], count: 1 }); + assert.deepEqual(await data(m, 'glob', { pattern: '**/*.js' }), { pattern: '**/*.js', matches: [{ path: 'ref/src/a.js', size: 7 }], count: 1 }); + const grep = await data<{ results: Array<{ file: string }>; incomplete?: unknown }>(m, 'grep', { pattern: 'needle' }); + assert.deepEqual(grep.results.map((r) => r.file), ['ref/src/a.js']); + assert.equal(grep.incomplete, undefined); + // Named, by a path or a pattern's directory: walked, the rest of the list + // applying beneath it. + assert.deepEqual((await data(m, 'ls', { path: 'ref/node_modules/pkg' })).entries, [{ name: 'index.js', type: 'file', size: 7 }]); + assert.deepEqual((await data<{ matches: unknown[] }>(m, 'glob', { pattern: 'node_modules/pkg/*.js' })).matches, [{ path: 'ref/node_modules/pkg/index.js', size: 7 }]); + assert.deepEqual( + (await data<{ results: Array<{ file: string }> }>(m, 'grep', { pattern: 'needle', path: 'ref/node_modules' })).results.map((r) => r.file), + ['ref/node_modules/pkg/index.js'], + ); + assert.deepEqual( + (await data<{ results: Array<{ file: string }> }>(m, 'grep', { pattern: 'needle', glob: 'node_modules/**' })).results.map((r) => r.file), + ['ref/node_modules/pkg/index.js'], + ); + // A list the config names replaces it. + const own = await env.open({ ignore: ['dist'] }); + assert.deepEqual( + (await data<{ matches: Array<{ path: string }> }>(own, 'glob', { pattern: '**/*.js' })).matches.map((x) => x.path), + ['ref/node_modules/pkg/index.js', 'ref/src/a.js'], + ); +}); + +test('glob and grep on a direct mount find a file the moment disk has it, as disk has it', async (t) => { + const env = new Env(t); + env.file('src/a.ts', 'alpha\n'); + env.file('src/notes.md', 'notes\n'); + const m = await env.open(); + assert.equal((await data<{ count: number }>(m, 'glob', { pattern: 'src/*.ts' })).count, 1); + env.file('src/b.ts', 'beta\nalpha again\n'); + assert.deepEqual((await data<{ matches: unknown[] }>(m, 'glob', { pattern: 'src/*.ts' })).matches, [ + { path: 'ref/src/a.ts', size: 6 }, + { path: 'ref/src/b.ts', size: 17 }, + ]); + assert.deepEqual(await data(m, 'grep', { pattern: 'alpha', path: 'ref/src', contextAfter: 1 }), { + pattern: 'alpha', + results: [ + { file: 'ref/src/a.ts', version: 'disk', matches: [{ line: 1, text: 'alpha', context: ['alpha'] }] }, + { file: 'ref/src/b.ts', version: 'disk', matches: [{ line: 2, text: 'alpha again', context: ['alpha again'] }] }, + ], + totalMatches: 2, + }); + assert.deepEqual((await data<{ results: unknown[] }>(m, 'grep', { pattern: 'beta', path: 'ref/src/b.ts' })).results, [ + { file: 'ref/src/b.ts', version: 'disk', matches: [{ line: 1, text: 'beta' }] }, + ]); + assert.deepEqual( + (await data<{ results: Array<{ file: string }> }>(m, 'grep', { pattern: 'alpha', glob: 'src/b.*' })).results.map((r) => r.file), + ['ref/src/b.ts'], + ); +}); + +test("a direct mount's walks keep the cap, start where their pattern names, and say what they couldn't reach", async (t) => { + const env = new Env(t); + for (const name of ['1', '2', '3']) env.file(`node_modules/aaa/${name}.js`, 'x\n'); + env.file('node_modules/pkg/index.js', 'x\n'); + const m = await env.open({ walkCap: 2 }); + // The start descends into the directory the pattern names, past what the + // cap would leave of a walk from above it. + assert.deepEqual((await data<{ matches: unknown[] }>(m, 'glob', { pattern: 'node_modules/pkg/*.js' })).matches, [{ path: 'ref/node_modules/pkg/index.js', size: 2 }]); + const capped = [ + { path: 'node_modules/aaa', reason: 'the walk cap (2) was reached before everything here was taken' }, + { path: 'node_modules/pkg', reason: 'not visited: the walk cap (2) was reached' }, + ]; + assert.deepEqual(await data(m, 'ls', { path: 'ref/node_modules', recursive: true }), { + path: 'ref/node_modules', + entries: [{ path: 'node_modules/aaa/1.js', size: 2 }, { path: 'node_modules/aaa/2.js', size: 2 }], + count: 2, + incomplete: capped, + }); + assert.deepEqual( + (await data<{ incomplete: unknown[] }>(m, 'grep', { pattern: 'x', path: 'ref/node_modules' })).incomplete, + capped.map((region) => ({ ...region, path: `ref/${region.path}` })), + ); +}); + +test("a direct listing names a directory it couldn't list", { skip: IS_ROOT ? 'root lists every directory' : false }, async (t) => { + const env = new Env(t); + env.file('open/a.ts', 'a\n'); + env.file('shut/b.ts', 'b\n'); + const m = await env.open(); + chmodSync(join(env.ref, 'shut'), 0o000); + try { + assert.deepEqual(await data(m, 'ls', { path: 'ref', recursive: true }), { + path: 'ref', + entries: [{ path: 'open/a.ts', size: 2 }], + count: 1, + incomplete: [{ path: 'shut', reason: 'cannot list (EACCES)' }], + }); + } finally { + chmodSync(join(env.ref, 'shut'), 0o755); + } +}); + +test("a direct mount's read loads none of a file over the size limit and grep only its head, and both refuse one that grows past it as it's read", async (t) => { + const env = new Env(t); + env.file('grows.txt', 'x'.repeat(8)); + env.file('huge.txt', 'y'.repeat(100_000)); + const m = await env.open({ maxFileSize: 10 }); + const over = (path: string) => `${path} is over the size limit (10 bytes), so read doesn't show it.`; + // Over the limit at its open: read loads none of it, and grep its head. + const read = await countingLoads(() => call(m, 'read', { path: 'ref/huge.txt' })); + assert.deepEqual([read.result.error, read.loaded], [over('ref/huge.txt'), 0]); + const grep = await countingLoads(() => call(m, 'grep', { pattern: 'y', path: 'ref/huge.txt' })); + assert.deepEqual((grep.result.data as { skipped: unknown[] }).skipped, [{ file: 'ref/huge.txt', reason: 'over the size limit (10 bytes)', size: 100_000 }]); + assert.equal(grep.loaded, 8192); + // Grown past it after its open's fstat: no more than the limit and a byte. + const grows = join(env.ref, 'grows.txt'); + const grown = await changedAfterFstat(grows, () => appendFileSync(grows, 'x'.repeat(1000)), + () => countingLoads(() => call(m, 'read', { path: 'ref/grows.txt' }))); + assert.equal(grown.result.error, over('ref/grows.txt')); + assert.ok(grown.loaded <= 11, `loaded ${grown.loaded}`); + writeFileSync(grows, 'x'.repeat(8)); + const grownGrep = await changedAfterFstat(grows, () => appendFileSync(grows, 'x'.repeat(1000)), + () => countingLoads(() => call(m, 'grep', { pattern: 'x', path: 'ref/grows.txt' }))); + assert.deepEqual((grownGrep.result.data as { skipped: unknown[] }).skipped, [{ file: 'ref/grows.txt', reason: 'over the size limit (10 bytes)', size: 1008 }]); + assert.ok(grownGrep.loaded <= 11, `loaded ${grownGrep.loaded}`); +}); + +test("a direct mount's read and grep say so where reading a file fails", async (t) => { + const env = new Env(t); + env.file('a.txt', 'needle\n'); + const m = await env.open(); + const failing = { read: () => async function () { throw Object.assign(new Error('EIO: i/o error, read'), { code: 'EIO' }); } }; + assert.equal((await withFileHandle(failing, () => call(m, 'read', { path: 'ref/a.txt' }))).error, 'Unable to read file: ref/a.txt (EIO)'); + assert.deepEqual((await withFileHandle(failing, () => call(m, 'grep', { pattern: 'needle' }))).data, { + pattern: 'needle', + results: [], + totalMatches: 0, + skipped: [{ file: 'ref/a.txt', reason: 'cannot read (EIO)' }], + }); +}); + +test("read on a direct mount says why it couldn't open a file", { skip: IS_ROOT ? 'root opens every file' : false }, async (t) => { + const env = new Env(t); + env.file('noperm.txt', 'secret\n'); + env.file('shut/b.ts', 'b\n'); + const m = await env.open(); + chmodSync(join(env.ref, 'noperm.txt'), 0o000); + chmodSync(join(env.ref, 'shut'), 0o000); + try { + assert.equal(await failure(m, 'read', { path: 'ref/noperm.txt' }), 'Unable to read file: ref/noperm.txt (EACCES)'); + assert.equal(await failure(m, 'read', { path: 'ref/shut/b.ts' }), 'Unable to read file: ref/shut/b.ts (EACCES)'); + } finally { + chmodSync(join(env.ref, 'noperm.txt'), 0o644); + chmodSync(join(env.ref, 'shut'), 0o755); + } +}); + +test('a file gone between the walk that listed it and its read is neither listed nor searched', async (t) => { + const env = new Env(t); + const names = Array.from({ length: 40 }, (_, i) => `f${String(i).padStart(2, '0')}.txt`); + for (const name of names) env.file(name, 'needle\n'); + const m = await env.open(); + const gone = join(env.ref, 'f30.txt'); + const left = names.filter((name) => name !== 'f30.txt'); + // grep reads 16 at a time, in order: the 31st file is opened only once the + // 15th is done, so after the 2nd file's open, when it's removed. + const grep = await changedAfterFstat(join(env.ref, 'f01.txt'), () => unlinkSync(gone), + () => data<{ results: Array<{ file: string }>; skipped?: unknown }>(m, 'grep', { pattern: 'needle' })); + assert.deepEqual(grep.results.map((r) => r.file), left.map((name) => `ref/${name}`)); + assert.equal(grep.skipped, undefined); + // A listing looks at each file by its path, where this one is removed + // first. The look is known by the file's name, since the listing reads + // beneath the root it resolved, which on macOS isn't spelled as `env.ref` + // is (/var there is a link to /private/var). + env.file('f30.txt', 'needle\n'); + const listed = await withFsPromise('lstat', (lstat) => async (...args: any[]) => { + if (basename(String(args[0])) === 'f30.txt') rmSync(gone, { force: true }); + return lstat(...args); + }, () => data<{ entries: Array<{ path: string }> }>(m, 'ls', { path: 'ref', recursive: true })); + assert.deepEqual(listed.entries.map((e) => e.path), left); +}); + +test('grep on a direct mount lists what it doesn\'t search, with why', async (t) => { + const env = new Env(t); + env.file('text.txt', 'needle\n'); + env.file('bin.dat', Buffer.concat([Buffer.from('needle'), Buffer.from([0])])); + env.file('dot.png', PNG); + env.file('big.txt', 'needle '.repeat(40)); + symlinkSync(join(env.ref, 'text.txt'), join(env.ref, 'link.txt')); + const m = await env.open({ maxFileSize: 100 }); + assert.deepEqual(await data(m, 'grep', { pattern: 'needle' }), { + pattern: 'needle', + results: [{ file: 'ref/text.txt', version: 'disk', matches: [{ line: 1, text: 'needle' }] }], + totalMatches: 1, + skipped: [ + { file: 'ref/big.txt', reason: 'over the size limit (100 bytes)', size: 280 }, + { file: 'ref/bin.dat', reason: 'binary: a NUL in its first 8 KB', size: 7 }, + { file: 'ref/dot.png', reason: 'binary: a NUL in its first 8 KB', size: PNG.byteLength, mimeType: 'image/png' }, + { file: 'ref/link.txt', reason: 'a symlink, which this mount does not follow' }, + ], + }); +}); + +test('grep on a direct mount that follows links names a link to a directory as its listing does', async (t) => { + const env = new Env(t); + env.file('dir/x.ts', 'needle\n'); + symlinkSync(join(env.ref, 'dir'), join(env.ref, 'linkdir')); + const m = await env.open({ followSymlinks: true }); + const listed = await data<{ entries: Array<{ path: string }> }>(m, 'ls', { path: 'ref', recursive: true }); + assert.deepEqual(listed.entries.find((e) => e.path === 'linkdir'), { path: 'linkdir', note: 'a symlink to a directory, not walked' }); + assert.deepEqual((await data<{ skipped?: unknown[] }>(m, 'grep', { pattern: 'needle' })).skipped, [ + { file: 'ref/linkdir', reason: 'a symlink to a directory, not walked' }, + ]); +}); + +test("grep on a direct mount names a link to a directory it can't open as its listing does, and a file it can't open by the open's reason", { skip: IS_ROOT ? 'root opens everything' : false }, async (t) => { + const env = new Env(t); + env.file('shut/x.ts', 'needle\n'); + env.file('noperm.txt', 'needle\n'); + symlinkSync(join(env.ref, 'shut'), join(env.ref, 'linkshut')); + const m = await env.open({ followSymlinks: true, ignore: ['shut'] }); + chmodSync(join(env.ref, 'shut'), 0o311); + chmodSync(join(env.ref, 'noperm.txt'), 0o000); + try { + const listed = await data<{ entries: Array<{ path: string }> }>(m, 'ls', { path: 'ref', recursive: true }); + assert.deepEqual(listed.entries.find((e) => e.path === 'linkshut'), { path: 'linkshut', note: 'a symlink to a directory, not walked' }); + assert.deepEqual((await data<{ skipped?: unknown[] }>(m, 'grep', { pattern: 'needle' })).skipped, [ + { file: 'ref/linkshut', reason: 'a symlink to a directory, not walked' }, + { file: 'ref/noperm.txt', reason: 'cannot open (EACCES)' }, + ]); + } finally { + chmodSync(join(env.ref, 'shut'), 0o755); + chmodSync(join(env.ref, 'noperm.txt'), 0o644); + } +}); + +test("grep's results come in path order, however its reads finish", async (t) => { + const env = new Env(t); + const names = Array.from({ length: 40 }, (_, i) => `f${String(i).padStart(2, '0')}.txt`); + for (const name of [...names].reverse()) env.file(`many/${name}`, `${name} needle\n`.repeat(1 + (names.indexOf(name) % 7) * 500)); + const m = await env.open(); + // The first file's reads wait until the other fifteen files read beside + // it have been read, so it finishes after them. + const first = String(statSync(join(env.ref, 'many', names[0]!), { bigint: true }).ino); + const slow = new WeakSet(); + const read = new WeakSet(); + let othersRead = 0; + let fifteenRead!: () => void; + const fifteen = new Promise((resolve) => { fifteenRead = resolve; }); + let readBeforeFirst = -1; + const result = await withFileHandle({ + stat: (stat) => async function (this: object, ...args: unknown[]) { + const seen = await stat.apply(this, args); + if (String(seen.ino) === first) slow.add(this); + return seen; + }, + read: (original) => async function (this: object, ...args: unknown[]) { + if (slow.has(this)) { + let timer: ReturnType | undefined; + await Promise.race([fifteen, new Promise((resolve) => { timer = setTimeout(resolve, 5000); })]); + clearTimeout(timer); + readBeforeFirst = othersRead; + return original.apply(this, args); + } + const answer = await original.apply(this, args); + if (!read.has(this)) { + read.add(this); + if (++othersRead === 15) fifteenRead(); + } + return answer; + }, + }, () => data<{ results: Array<{ file: string }> }>(m, 'grep', { pattern: 'needle' })); + assert.ok(readBeforeFirst >= 15, `${readBeforeFirst} other files were read before the first`); + assert.deepEqual(result.results.map((r) => r.file), names.map((name) => `ref/many/${name}`)); +}); + +test('a direct grep and listing read 16 files at a time', async (t) => { + const env = new Env(t); + for (let i = 0; i < 40; i++) env.file(`f${String(i).padStart(2, '0')}.txt`, 'needle\n'); + const m = await env.open(); + // The files' own calls: a walk's are on directories. + const files = /[\\/]f\d\d\.txt$/; + assert.equal(await mostAtOnce('open', 16, files, () => data(m, 'grep', { pattern: 'needle' })), 16); + assert.equal(await mostAtOnce('lstat', 16, files, () => data(m, 'ls', { path: 'ref', recursive: true })), 16); +}); + +test("a direct mount's reads close every file they open, whatever they answer", async (t) => { + const env = new Env(t); + env.file('a.txt', 'needle\n'); + env.file('bin.dat', Buffer.from([0x61, 0x00, 0x62])); + env.file('big.txt', 'x'.repeat(100)); + env.file('dot.png', PNG); + const m = await env.open({ maxFileSize: 90 }); + // Each handle a read opens is fstat'd; a closed one's fd reads -1. + const opened = new Set<{ fd: number }>(); + await withFileHandle({ + stat: (stat) => async function (this: { fd: number }, ...args: unknown[]) { + opened.add(this); + return stat.apply(this, args); + }, + }, async () => { + for (const path of ['ref/a.txt', 'ref/bin.dat', 'ref/big.txt']) await call(m, 'read', { path }); + await call(m, 'grep', { pattern: 'needle' }); + await call(m, 'read_image', { path: 'ref/dot.png' }); + await m.readBinary('ref/bin.dat'); + }); + assert.ok(opened.size >= 9, `opened ${opened.size}`); + assert.deepEqual([...opened].map((handle) => handle.fd).filter((fd) => fd !== -1), []); +}); + +test("grep gives a skipped file's size as its reads found it, when the file changes as it's read", async (t) => { + const env = new Env(t); + env.file('grows.txt', 'x'.repeat(8)); + env.file('bin.dat', Buffer.from('a\0bcdefg')); + const m = await env.open({ maxFileSize: 10 }); + // Grown past the limit after its open's fstat, then cut short before its + // size is read again: at least a byte past the limit, as its reads found. + const grows = join(env.ref, 'grows.txt'); + let stats = 0; + const grown = await withFileHandle({ + stat: (stat) => async function (this: object, ...args: unknown[]) { + stats++; + if (stats === 2) truncateSync(grows, 5); + const seen = await stat.apply(this, args); + if (stats === 1) appendFileSync(grows, 'x'.repeat(1000)); + return seen; + }, + }, () => data<{ skipped: unknown[] }>(m, 'grep', { pattern: 'x', path: 'ref/grows.txt' })); + assert.deepEqual(grown.skipped, [{ file: 'ref/grows.txt', reason: 'over the size limit (10 bytes)', size: 11 }]); + // Cut short after its fstat, within the limit: the bytes read. + const bin = join(env.ref, 'bin.dat'); + const shrunk = await changedAfterFstat(bin, () => truncateSync(bin, 4), () => data<{ skipped: unknown[] }>(m, 'grep', { pattern: 'b', path: 'ref/bin.dat' })); + assert.deepEqual(shrunk.skipped, [{ file: 'ref/bin.dat', reason: 'binary: a NUL in its first 8 KB', size: 4 }]); +}); + +test('a direct read that finds its root replaced as it opens the file says to read again', async (t) => { + const env = new Env(t); + env.file('a.txt', 'a\n'); + const m = await env.open(); + const error = await changedAfterFstat(join(env.ref, 'a.txt'), () => { + renameSync(env.ref, `${env.ref}-before`); + mkdirSync(env.ref); + }, () => failure(m, 'read', { path: 'ref/a.txt' })); + assert.equal(error, 'Mount unavailable: ref: the mount root was replaced as the file was opened: reading it again reads the directory there now'); + assert.equal(await failure(m, 'read', { path: 'ref/a.txt' }), 'File not found: ref/a.txt'); +}); + +test("a direct mount takes each spelling of a path as a 'chronicle' mount does", async (t) => { + const env = new Env(t); + env.file('a.ts', 'alpha\n'); + env.file('dir/x.ts', 'alpha x\n'); + env.file('dot.png', PNG); + const reads = ['ref/./a.ts', 'ref/dir/../a.ts', 'ref/dir//x.ts', 'ref/a.ts/', 'ref//a.ts', 'ref/dir/..', 'ref/../outside.txt', 'ref/dir/../../ref/a.ts']; + const calls: Array<[string, Record]> = [ + ...reads.map((path): [string, Record] => ['read', { path }]), + ['read_image', { path: 'ref/./dot.png' }], + ['ls', { path: 'ref/./dir' }], + ['ls', { path: 'ref/dir//' }], + ['glob', { pattern: '*.ts', path: 'ref/dir/..' }], + ['grep', { pattern: 'alpha', path: 'ref/./dir' }], + ]; + const hostPaths = ['ref/a.ts/', 'ref/./dir/x.ts', 'ref/dir/../a.ts']; + const direct = await env.open(); + const fromDisk = await answersOf(direct, calls, hostPaths); + // Each spelling reaches what it names, and one that climbs out of the + // mount is refused, by the parse every mount shares. + assert.equal((fromDisk[0] as [string, unknown, { content: string }])[2].content, ' 1\talpha'); + assert.match(await failure(direct, 'read', { path: 'ref/../outside.txt' }), /climbs above the root of mount "ref"/); + await env.close(direct); + assert.deepEqual(fromDisk, await answersOf(await env.openChronicle(), calls, hostPaths)); +}); + +/** + * What a 'chronicle' mount and a direct one both answer, kind by kind alike: + * whether a call is refused, a file's text, and the names a listing, glob or + * grep gives. A refusal's words are each kind's own, except where they come + * from what the kinds share, which the tests check by name. + */ +function alike(result: ToolResult): unknown { + if (!result.success) return { refused: true }; + const d = result.data as Record; + if (Array.isArray(d)) return { text: d[0].text.split('\n')[0] }; + return { + path: d.path, + content: d.content, + entries: d.entries?.map((e: { path?: string; name: string }) => e.path ?? e.name).sort(), + matches: d.matches?.map((m: { path: string }) => m.path).sort(), + results: d.results?.map((r: { file: string }) => r.file).sort(), + }; +} + +/** Each call's answer as `alike` gives it, with the host's reads of each of `hostPaths` beside them. */ +async function answersOf(m: WorkspaceModule, calls: Array<[string, Record]>, hostPaths: string[]): Promise { + const out: unknown[] = []; + for (const [name, input] of calls) out.push([name, input, alike(await call(m, name, input))]); + for (const path of hostPaths) { + const binary = await m.readBinary(path); + const fromDisk = await m.readBinaryFromDisk(path); + const text = await m.readFileFromDisk(path).then((read) => read.bytes.toString(), (err: unknown) => (err instanceof WorkspaceReadError ? `refused: ${err.code}` : 'threw')); + out.push([path, 'error' in binary, 'error' in fromDisk, text, m.resolveAbsolutePath(path) !== null]); + } + return out; +} + +test("a direct mount's include holds as a 'chronicle' mount's does", async (t) => { + const env = new Env(t); + for (const path of ['gate.json', 'gate.json.bak', 'notes.md', 'other.txt', 'sub/gate.json', 'sub/x.md', 'sub/y.txt', 'docs/a/b.txt', 'docsx/c.txt']) { + env.file(path, `${path} needle\n`); + } + env.file('pic.png', PNG); + env.file('docs/pic.png', PNG); + const include = ['gate.json', '*.md', 'docs/**']; + const calls: Array<[string, Record]> = [ + ['ls', { path: 'ref', recursive: true }], + ['ls', { path: 'ref/sub' }], + ['ls', { path: 'ref/docsx' }], + ['ls', { path: 'ref/other.txt' }], + ['glob', { pattern: '**/*' }], + ['glob', { pattern: '*.txt', path: 'ref/docs' }], + ['glob', { pattern: '*', path: 'ref/other.txt' }], + ['grep', { pattern: 'needle' }], + ['grep', { pattern: 'needle', path: 'ref/docsx' }], + ['grep', { pattern: 'needle', path: 'ref/sub/y.txt' }], + ['read', { path: 'ref/notes.md' }], + ['read', { path: 'ref/other.txt' }], + ['read', { path: 'ref/sub/y.txt' }], + ['read', { path: 'ref/docsx' }], + ['read_image', { path: 'ref/other.txt' }], + ['read_image', { path: 'ref/pic.png' }], + ['read_image', { path: 'ref/docs/pic.png' }], + ]; + const hostPaths = ['ref/notes.md', 'ref/other.txt', 'ref/docsx/c.txt', 'ref/pic.png']; + const direct = await env.open({ include }); + const fromDisk = await answersOf(direct, calls, hostPaths); + // The include's refusal is the shared parse's, in the same words, and `ls` + // without a path lists it beside the kind. + const said = /^"ref\/other\.txt" is outside the mount's include: mount "ref" holds only gate\.json, \*\.md, docs\/\*\*\./; + assert.match(await failure(direct, 'read', { path: 'ref/other.txt' }), said); + // An image the include leaves out is refused in the include's words too, + // and one it covers is read. + assert.match(await failure(direct, 'read_image', { path: 'ref/pic.png' }), /^"ref\/pic\.png" is outside the mount's include: mount "ref" holds only /); + assert.ok((await call(direct, 'read_image', { path: 'ref/docs/pic.png' })).success); + assert.deepEqual((await data<{ mounts: Array> }>(direct, 'ls')).mounts[0]!.include, include); + assert.deepEqual((fromDisk[0] as [string, unknown, { entries: string[] }])[2].entries, ['docs/a/b.txt', 'docs/pic.png', 'gate.json', 'notes.md', 'sub/gate.json', 'sub/x.md']); + await env.close(direct); + assert.deepEqual(fromDisk, await answersOf(await env.open({ backend: 'chronicle', mode: 'read-write', watch: 'never', include }), calls, hostPaths)); +}); + +test("on a direct mount that follows links, a covered link that lands on a file its include leaves out is no file of the mount's, as on a 'chronicle' mount", async (t) => { + const env = new Env(t); + env.file('other.txt', 'needle\n'); + env.file('notes.md', 'needle\n'); + symlinkSync(join(env.ref, 'other.txt'), join(env.ref, 'link.md')); + symlinkSync(join(env.ref, 'notes.md'), join(env.ref, 'also.md')); + const include = ['*.md']; + const calls: Array<[string, Record]> = [ + ['read', { path: 'ref/link.md' }], + ['read', { path: 'ref/also.md' }], + ['ls', { path: 'ref', recursive: true }], + ['grep', { pattern: 'needle' }], + ]; + const direct = await env.open({ include, followSymlinks: true }); + const fromDisk = await answersOf(direct, calls, ['ref/link.md']); + assert.match(await failure(direct, 'read', { path: 'ref/link.md' }), /Symlink escape detected: ref\/link\.md/); + assert.deepEqual((fromDisk[3] as [string, unknown, { results: string[] }])[2].results, ['ref/also.md', 'ref/notes.md']); + await env.close(direct); + assert.deepEqual(fromDisk, await answersOf(await env.open({ backend: 'chronicle', mode: 'read-write', watch: 'never', include, followSymlinks: true }), calls, ['ref/link.md'])); +}); + +test("a path in a mount nested in a direct one is that mount's, whatever the direct mount's include says, as on a 'chronicle' mount", async (t) => { + const env = new Env(t); + env.file('inner/x.json', '{}\n'); + env.file('inner/y.md', 'y\n'); + env.file('top.md', 't\n'); + const inner: MountConfig = { name: 'inner', path: join(env.ref, 'inner'), mode: 'read-only', watch: 'never', include: ['*.json'] }; + const calls: Array<[string, Record]> = [ + ['read', { path: 'ref/inner/x.json' }], + ['read', { path: 'ref/inner/y.md' }], + ['ls', { path: 'ref', recursive: true }], + ['ls', { path: 'ref/inner' }], + ]; + const direct = await env.open({ include: ['*.md'] }, [inner]); + const fromDisk = await answersOf(direct, calls, ['ref/inner/x.json']); + assert.match(await failure(direct, 'read', { path: 'ref/inner/y.md' }), /lies in mount "inner", nested in mount "ref"/); + await env.close(direct); + assert.deepEqual(fromDisk, await answersOf(await env.open({ backend: 'chronicle', mode: 'read-write', watch: 'never', include: ['*.md'] }, [inner]), calls, ['ref/inner/x.json'])); +}); + +test("what a mount's history holds outside its include keeps no mount configured direct from starting direct", async (t) => { + const env = new Env(t); + env.file('notes.md', 'disk\n'); + // A 'chronicle' run with no include left a draft of a file the include + // now leaves out. + const chronicle = await env.openChronicle(); + await data(chronicle, 'write', { path: 'ref/src/b.ts', content: 'draft\n' }); + await env.close(chronicle); + // And an intent it can't read, there too. + env.store.treeSet('workspace/ref/intent', 'src/zz.ts', { blobHash: 'f'.repeat(64), size: 2, mode: 0 }); + const m = await env.open({ include: ['*.md'] }); + assert.deepEqual(env.alerts, []); + assert.equal((await data>>(m, 'status')).ref!.backend, 'direct'); +}); + +test("a direct mount judges a path's scope by disk, not by history a 'chronicle' run of it left", async (t) => { + const env = new Env(t); + env.file('x.md', 'x\n'); + env.file('gone/a.md', 'a\n'); + // A 'chronicle' run took in x.md as a file and gone/a.md. + const chronicle = await env.openChronicle(); + await data(chronicle, 'read', { path: 'ref/x.md' }); + await data(chronicle, 'read', { path: 'ref/gone/a.md' }); + await env.close(chronicle); + // Since then x.md has become a directory, and gone/ has gone. + rmSync(join(env.ref, 'x.md')); + env.file('x.md/inner.md', 'needle\n'); + rmSync(join(env.ref, 'gone'), { recursive: true }); + const m = await env.open({ include: ['*.md'] }); + assert.equal((await data>>(m, 'status')).ref!.backend, 'direct'); + // grep takes x.md as the directory disk has, and searches what's in it. + assert.deepEqual((await data<{ results: Array<{ file: string }> }>(m, 'grep', { pattern: 'needle', path: 'ref/x.md' })).results.map((r) => r.file), ['ref/x.md/inner.md']); + // gone, which the include doesn't cover, is no scope on disk, and history + // the 'chronicle' run left doesn't make it one. + assert.match(await failure(m, 'ls', { path: 'ref/gone' }), /outside the mount's include/); +}); + +test("a direct mount whose root is repointed through a link reads the mounts where they lie now", async (t) => { + const env = new Env(t); + // `ref` reaches `a` through a link, and `work` lies at `b/work` with a + // draft disk lacks. + const a = join(env.root, 'a'); + const b = join(env.root, 'b'); + mkdirSync(join(a, 'work'), { recursive: true }); + mkdirSync(join(b, 'work', 'sub'), { recursive: true }); + writeFileSync(join(a, 'work', 'file.txt'), 'a\n'); + writeFileSync(join(b, 'work', 'file.txt'), 'disk\n'); + const link = join(env.root, 'link'); + symlinkSync(a, link); + const m = await env.open({ path: link }, [{ name: 'work', path: join(b, 'work'), mode: 'read-write', watch: 'never' }]); + await data(m, 'write', { path: 'work/file.txt', content: 'draft\n' }); + assert.equal((await data<{ content: string }>(m, 'read', { path: 'ref/work/file.txt' })).content, ' 1\ta'); + + // Repointed to `b`, `ref/work` is `work`'s directory, for every read. + rmSync(link); + symlinkSync(b, link); + const all = await data<{ results: Array<{ file: string }> }>(m, 'grep', { pattern: 'disk' }); + assert.deepEqual(all.results.map((r) => r.file).filter((file) => file.startsWith('ref/')), []); + assert.match(await failure(m, 'read', { path: 'ref/work/file.txt' }), /lies in mount "work", nested in mount "ref"/); + await assert.rejects(m.readFileFromDisk('ref/work/file.txt'), (err: unknown) => err instanceof WorkspaceReadError && err.code === 'other_mount'); + assert.deepEqual((await data(m, 'ls', { path: 'ref', recursive: true })).entries, []); + assert.deepEqual((await data<{ results: unknown[] }>(m, 'grep', { pattern: 'disk', path: 'ref' })).results, []); + const named = await data<{ results: unknown[]; incomplete: Array<{ path: string; reason: string }> }>(m, 'grep', { pattern: 'disk', path: 'ref/work' }); + assert.deepEqual([named.results, named.incomplete], [[], [{ path: 'ref/work', reason: 'it lies in mount "work", nested in this one: use "work"' }]]); + assert.deepEqual((await data(m, 'ls', { path: 'ref/work' })).incomplete, [{ path: 'work', reason: 'it lies in mount "work", nested in this one: use "work"' }]); + assert.equal((await data<{ content: string }>(m, 'read', { path: 'work/file.txt' })).content, ' 1\tdraft'); + + // Pointed back at `a`, `ref/work` is `ref`'s own again. + rmSync(link); + symlinkSync(a, link); + assert.equal((await data<{ content: string }>(m, 'read', { path: 'ref/work/file.txt' })).content, ' 1\ta'); + + // Repointed into `work`'s directory, `ref` reads what's there, and `work` + // keeps the start's placement until the next start: the directory is + // still its own too. + rmSync(link); + symlinkSync(join(b, 'work', 'sub'), link); + writeFileSync(join(b, 'work', 'sub', 'inside.txt'), 'i\n'); + assert.equal((await data<{ content: string }>(m, 'read', { path: 'ref/inside.txt' })).content, ' 1\ti'); + const listed = await data<{ entries: Array<{ path: string }> }>(m, 'ls', { path: 'work', recursive: true }); + assert.ok(listed.entries.some((e) => e.path === 'sub/inside.txt'), JSON.stringify(listed.entries)); +}); + +test("a direct mount moved away from a mount nested in it reads what's there as its own, in read, the host's read, ls and grep", async (t) => { + const env = new Env(t); + // `ref` starts at `b` through a link, where `work` lies nested at `b/work`. + const a = join(env.root, 'a'); + const b = join(env.root, 'b'); + mkdirSync(join(a, 'work'), { recursive: true }); + mkdirSync(join(b, 'work'), { recursive: true }); + writeFileSync(join(a, 'work', 'file.txt'), 'mine\n'); + writeFileSync(join(b, 'work', 'file.txt'), 'theirs\n'); + const link = join(env.root, 'link'); + symlinkSync(b, link); + const m = await env.open({ path: link }, [{ name: 'work', path: join(b, 'work'), mode: 'read-write', watch: 'never' }]); + assert.match(await failure(m, 'read', { path: 'ref/work/file.txt' }), /lies in mount "work", nested in mount "ref"/); + + // Repointed to `a`, nothing lies nested in `ref`, so `a/work` is its own. + rmSync(link); + symlinkSync(a, link); + assert.equal((await data<{ content: string }>(m, 'read', { path: 'ref/work/file.txt' })).content, ' 1\tmine'); + assert.equal((await m.readFileFromDisk('ref/work/file.txt')).bytes.toString(), 'mine\n'); + assert.deepEqual((await data<{ entries: Array<{ path: string }> }>(m, 'ls', { path: 'ref', recursive: true })).entries.map((e) => e.path), ['work/file.txt']); + assert.deepEqual((await data<{ results: Array<{ file: string }> }>(m, 'grep', { pattern: 'mine', path: 'ref/work' })).results.map((r) => r.file), ['ref/work/file.txt']); +}); + +test("a mount configured beneath a direct mount's path is nested where its moved root now puts it, as a start there nests it", async (t) => { + const env = new Env(t); + // `ref` is configured through `link`, and `work` beneath it at + // `link/work`, as a deploy's `current` link and a mount inside it are. In + // `b`, a link to `work` and one to its file. + const a = join(env.root, 'a'); + const b = join(env.root, 'b'); + mkdirSync(join(a, 'work'), { recursive: true }); + mkdirSync(join(b, 'work'), { recursive: true }); + writeFileSync(join(a, 'work', 'file.txt'), 'old\n'); + writeFileSync(join(b, 'work', 'file.txt'), 'new\n'); + writeFileSync(join(b, 'top.txt'), 'top\n'); + symlinkSync('work', join(b, 'alias')); + symlinkSync(join('work', 'file.txt'), join(b, 'alias.txt')); + const link = join(env.root, 'link'); + symlinkSync(a, link); + const mounts: [Partial, MountConfig[]] = [{ path: link, followSymlinks: true }, [{ name: 'work', path: join(link, 'work'), mode: 'read-write', watch: 'never' }]]; + const moved = await env.open(...mounts); + + // Repointed to `b`, `work`'s path leads to `b/work`, which is still + // `work`'s: by its directory, and by where a link into it lands. + rmSync(link); + symlinkSync(b, link); + const started = await new Env(t).open(...mounts); + const answers = async (m: WorkspaceModule) => ({ + read: await failure(m, 'read', { path: 'ref/work/file.txt' }), + throughLink: await failure(m, 'read', { path: 'ref/alias/file.txt' }), + fileLink: await call(m, 'read', { path: 'ref/alias.txt' }), + disk: await m.readFileFromDisk('ref/work/file.txt').then(() => 'read', (err: unknown) => (err instanceof WorkspaceReadError ? err.code : String(err))), + listed: (await data<{ entries: unknown[] }>(m, 'ls', { path: 'ref', recursive: true })).entries, + found: await data(m, 'grep', { pattern: 'new', path: 'ref' }), + }); + const fromStart = await answers(started); + assert.match(fromStart.read, /lies in mount "work", nested in mount "ref"/); + assert.match(fromStart.throughLink, /"ref\/alias\/file\.txt" leads into mount "work", nested in mount "ref"/); + assert.equal(fromStart.disk, 'other_mount'); + assert.deepEqual((fromStart.found as { results: unknown[] }).results, []); + assert.deepEqual(await answers(moved), fromStart); +}); + +test("a mount configured beneath a moved direct root, at a directory that's a link there, is nested as a start there nests it", async (t) => { + const env = new Env(t); + // `work` is configured at `link/work`, and in `b` that's a link to + // `real-work`, a directory beside it. + const a = join(env.root, 'a'); + const b = join(env.root, 'b'); + mkdirSync(join(a, 'work'), { recursive: true }); + mkdirSync(join(b, 'real-work'), { recursive: true }); + symlinkSync('real-work', join(b, 'work')); + writeFileSync(join(b, 'real-work', 'file.txt'), 'b work\n'); + writeFileSync(join(b, 'own.txt'), 'own\n'); + const link = join(env.root, 'link'); + symlinkSync(a, link); + const mounts: [Partial, MountConfig[]] = [{ path: link }, [{ name: 'work', path: join(link, 'work'), mode: 'read-write', watch: 'never' }]]; + const moved = await env.open(...mounts); + rmSync(link); + symlinkSync(b, link); + const started = await new Env(t).open(...mounts); + const answers = async (m: WorkspaceModule) => ({ + read: await failure(m, 'read', { path: 'ref/real-work/file.txt' }), + disk: await m.readFileFromDisk('ref/real-work/file.txt').then(() => 'read', (err: unknown) => (err instanceof WorkspaceReadError ? err.code : String(err))), + own: (await data<{ content: string }>(m, 'read', { path: 'ref/own.txt' })).content, + listed: (await data<{ entries: Array<{ path: string }> }>(m, 'ls', { path: 'ref', recursive: true })).entries.map((e) => e.path), + found: (await data<{ results: Array<{ file: string }> }>(m, 'grep', { pattern: 'b work', path: 'ref' })).results.map((r) => r.file), + }); + const fromStart = await answers(started); + assert.match(fromStart.read, /lies in mount "work", nested in mount "ref"/); + assert.deepEqual([fromStart.disk, fromStart.found], ['other_mount', []]); + assert.deepEqual(await answers(moved), fromStart); +}); + +test("a mount configured beneath a moved direct root, at a link that leads above it, is nested by its directory alone, as a start there nests it", async (t) => { + const env = new Env(t); + // In `b`, `work` leads to `b`'s parent, which holds `b`: through it, + // `work` would claim every path of `ref`'s. + const a = join(env.root, 'a'); + const b = join(env.root, 'b'); + mkdirSync(join(a, 'work'), { recursive: true }); + mkdirSync(b, { recursive: true }); + symlinkSync('..', join(b, 'work')); + writeFileSync(join(b, 'own.txt'), 'own\n'); + const link = join(env.root, 'link'); + symlinkSync(a, link); + const mounts: [Partial, MountConfig[]] = [{ path: link }, [{ name: 'work', path: join(link, 'work'), mode: 'read-only', backend: 'direct' }]]; + const moved = await env.open(...mounts); + rmSync(link); + symlinkSync(b, link); + const started = await new Env(t).open(...mounts); + const answers = async (m: WorkspaceModule) => ({ + own: (await data<{ content: string }>(m, 'read', { path: 'ref/own.txt' })).content, + listed: (await data<{ entries: Array<{ path: string }> }>(m, 'ls', { path: 'ref', recursive: true })).entries.map((e) => e.path), + }); + const fromStart = await answers(started); + assert.equal(fromStart.own, ' 1\town'); + assert.deepEqual(await answers(moved), fromStart); +}); + +test("a file tool's parse judges where a link on the way leads beneath the root it resolved, though the root moves after", async (t) => { + const env = new Env(t); + // At `b`, `alias` is a link into `work`'s directory; at `a`, a directory of + // `ref`'s own. + const a = join(env.root, 'a'); + const b = join(env.root, 'b'); + mkdirSync(join(a, 'alias'), { recursive: true }); + writeFileSync(join(a, 'alias', 'file.txt'), 'mine\n'); + mkdirSync(join(b, 'work'), { recursive: true }); + writeFileSync(join(b, 'work', 'file.txt'), 'theirs\n'); + symlinkSync('work', join(b, 'alias')); + const link = join(env.root, 'link'); + symlinkSync(b, link); + const m = await env.open({ path: link }, [{ name: 'work', path: join(b, 'work'), mode: 'read-write', watch: 'never' }]); + const atB = await m.readFileFromDisk('ref/alias/file.txt').then(() => null, (e: unknown) => e); + assert.ok(atB instanceof WorkspaceReadError, String(atB)); + // The parse resolves the root at `b`, and the link is repointed to `a` as + // that answers: the parse's check of where the path leads is made beneath + // `b`, as its nesting is, the link on the way included, so the read + // answers as one made at `b` does. + let repointed = false; + const err = await withFsPromise('realpath', (original) => async (...args: any[]) => { + const real = await original(...args); + if (!repointed && args[0] === link) { + repointed = true; + rmSync(link); + symlinkSync(a, link); + } + return real; + }, () => m.readFileFromDisk('ref/alias/file.txt').then(() => null, (e: unknown) => e)); + assert.ok(repointed); + assert.ok(err instanceof WorkspaceReadError, String(err)); + assert.deepEqual([err.code, err.stage, err.message], [atB.code, atB.stage, atB.message]); + assert.deepEqual([err.code, err.stage], ['other_mount', 'readlink']); +}); + +test("a file tool's parse judges where a followed link lands against the include beneath the root it resolved, though the root moves after", async (t) => { + const env = new Env(t); + // At `b`, `link.md` leads to a file the include leaves out; at `a`, it's a + // covered file of its own. + const a = join(env.root, 'a'); + const b = join(env.root, 'b'); + mkdirSync(a, { recursive: true }); + mkdirSync(b, { recursive: true }); + writeFileSync(join(a, 'link.md'), 'a link.md\n'); + writeFileSync(join(b, 'other.txt'), 'b other\n'); + symlinkSync('other.txt', join(b, 'link.md')); + const link = join(env.root, 'link'); + symlinkSync(b, link); + const m = await env.open({ path: link, followSymlinks: true, include: ['*.md'] }); + // The parse resolves the root at `b`, and the link is repointed to `a` as + // that answers: the parse's check of where the path lands is made beneath + // `b`, where it lands on a file the include leaves out. + let repointed = false; + const err = await withFsPromise('realpath', (original) => async (...args: any[]) => { + const real = await original(...args); + if (!repointed && args[0] === link) { + repointed = true; + rmSync(link); + symlinkSync(a, link); + } + return real; + }, () => m.readFileFromDisk('ref/link.md').then(() => null, (e: unknown) => e)); + assert.ok(repointed); + assert.ok(err instanceof WorkspaceReadError, String(err)); + assert.deepEqual([err.code, err.message], ['escape', 'Symlink escape detected: ref/link.md']); +}); + +test("a file tool's parse reads a link on the way beneath the root it resolved, so a root moved after names nothing another mount's include leaves out", async (t) => { + const env = new Env(t); + // At `a`, `alias` leads into `old`, nested there; at `b`, an `alias` leads + // to a file `private`'s include leaves out. + const a = join(env.root, 'a'); + const b = join(env.root, 'b'); + mkdirSync(join(a, 'old'), { recursive: true }); + writeFileSync(join(a, 'old', 'public.json'), '{}\n'); + symlinkSync(join('old', 'public.json'), join(a, 'alias')); + mkdirSync(join(b, 'private'), { recursive: true }); + writeFileSync(join(b, 'private', 'secret.env'), 'S=1\n'); + symlinkSync(join('private', 'secret.env'), join(b, 'alias')); + const link = join(env.root, 'link'); + symlinkSync(a, link); + const m = await env.open({ path: link }, [ + { name: 'old', path: join(link, 'old'), mode: 'read-only', backend: 'direct' }, + { name: 'private', path: join(b, 'private'), mode: 'read-only', backend: 'direct', include: ['*.json'] }, + ]); + // The parse resolves the root at `a`, and the link is repointed to `b` as + // that answers: the refusal reads `alias` beneath `a`, where it leads into + // `old`. + let repointed = false; + const err = await withFsPromise('realpath', (original) => async (...args: any[]) => { + const real = await original(...args); + if (!repointed && args[0] === link) { + repointed = true; + rmSync(link); + symlinkSync(b, link); + } + return real; + }, () => m.readFileFromDisk('ref/alias').then(() => null, (e: unknown) => e)); + assert.ok(repointed); + assert.ok(err instanceof WorkspaceReadError, String(err)); + assert.equal(err.code, 'other_mount'); + assert.match(err.message, /old/); + assert.doesNotMatch(err.message, /private|secret/); +}); + +test("a mount configured in a moved direct root's new directory, through a link to that directory, is nested as a start there nests it", async (t) => { + const env = new Env(t); + // `work` is configured at `b/work`, a link to `b` itself, so a start with + // `ref` at `b` nests `work` at `work`, by its configured path alone, and + // not at `b/work`, which `b` holds too, as no directory above `b` is `b`. + const a = join(env.root, 'a'); + const b = join(env.root, 'b'); + mkdirSync(a, { recursive: true }); + mkdirSync(join(b, 'b', 'work'), { recursive: true }); + writeFileSync(join(b, 'file.txt'), 'b file\n'); + writeFileSync(join(b, 'b', 'work', 'own.txt'), 'b own\n'); + symlinkSync('.', join(b, 'work')); + const link = join(env.root, 'link'); + symlinkSync(a, link); + const mounts: [Partial, MountConfig[]] = [{ path: link, followSymlinks: true }, [{ name: 'work', path: join(b, 'work'), mode: 'read-only', backend: 'direct' }]]; + const moved = await env.open(...mounts); + rmSync(link); + symlinkSync(b, link); + const started = await new Env(t).open(...mounts); + const answers = async (m: WorkspaceModule) => ({ + read: await call(m, 'read', { path: 'ref/work/file.txt' }), + disk: await m.readFileFromDisk('ref/work/file.txt').then(() => 'read', (err: unknown) => (err instanceof WorkspaceReadError ? err.code : String(err))), + own: (await data<{ content: string }>(m, 'read', { path: 'ref/file.txt' })).content, + alike: (await data<{ content: string }>(m, 'read', { path: 'ref/b/work/own.txt' })).content, + found: (await data<{ results: Array<{ file: string }> }>(m, 'grep', { pattern: 'b file', path: 'ref' })).results.map((r) => r.file), + }); + const fromStart = await answers(started); + assert.match(fromStart.read.error ?? '', /lies in mount "work", nested in mount "ref"/); + assert.equal(fromStart.disk, 'other_mount'); + assert.deepEqual(await answers(moved), fromStart); +}); + +test("a mount configured at a link into a moved direct root's new directory is nested where the link leads, as a start there nests it", async (t) => { + const env = new Env(t); + // `work` is configured at `w`, a link to `b/deep/work`, so a start with + // `ref` at `b` nests it by its real root alone: no directory above `w` + // is `b`. + const a = join(env.root, 'a'); + const b = join(env.root, 'b'); + mkdirSync(a, { recursive: true }); + mkdirSync(join(b, 'deep', 'work'), { recursive: true }); + writeFileSync(join(b, 'deep', 'work', 'file.txt'), 'work file\n'); + writeFileSync(join(b, 'own.txt'), 'b own file\n'); + symlinkSync(join(b, 'deep', 'work'), join(env.root, 'w')); + const link = join(env.root, 'link'); + symlinkSync(a, link); + const mounts: [Partial, MountConfig[]] = [{ path: link }, [{ name: 'work', path: join(env.root, 'w'), mode: 'read-only', backend: 'direct' }]]; + const moved = await env.open(...mounts); + rmSync(link); + symlinkSync(b, link); + const started = await new Env(t).open(...mounts); + const answers = async (m: WorkspaceModule) => ({ + read: await call(m, 'read', { path: 'ref/deep/work/file.txt' }), + disk: await m.readFileFromDisk('ref/deep/work/file.txt').then(() => 'read', (err: unknown) => (err instanceof WorkspaceReadError ? err.code : String(err))), + found: (await data<{ results: Array<{ file: string }> }>(m, 'grep', { pattern: 'file', path: 'ref' })).results.map((r) => r.file), + }); + const fromStart = await answers(started); + assert.match(fromStart.read.error ?? '', /lies in mount "work", nested in mount "ref"/); + assert.equal(fromStart.disk, 'other_mount'); + assert.deepEqual(fromStart.found, ['ref/own.txt']); + assert.deepEqual(await answers(moved), fromStart); +}); + +test("a mount configured beneath a moved direct root, through a directory there that's the root itself, is nested beneath that directory too, as a start there nests it", async (t) => { + const env = new Env(t); + // `work` is configured at `link/sub/work`. At `b`, `sub` and `work` are + // links to `.`, so a start with `ref` at `b` nests `work` at `sub/work` + // and, beneath `sub`, which is `b` itself, at `work`. + const a = join(env.root, 'a'); + const b = join(env.root, 'b'); + mkdirSync(join(a, 'sub', 'work'), { recursive: true }); + mkdirSync(b, { recursive: true }); + writeFileSync(join(b, 'file.txt'), 'b file\n'); + symlinkSync('.', join(b, 'sub')); + symlinkSync('.', join(b, 'work')); + const link = join(env.root, 'link'); + symlinkSync(a, link); + const mounts: [Partial, MountConfig[]] = [{ path: link, followSymlinks: true }, [{ name: 'work', path: join(link, 'sub', 'work'), mode: 'read-only', backend: 'direct' }]]; + const moved = await env.open(...mounts); + rmSync(link); + symlinkSync(b, link); + const started = await new Env(t).open(...mounts); + const answers = async (m: WorkspaceModule) => ({ + read: await call(m, 'read', { path: 'ref/work/file.txt' }), + disk: await m.readFileFromDisk('ref/work/file.txt').then(() => 'read', (err: unknown) => (err instanceof WorkspaceReadError ? err.code : String(err))), + ls: await call(m, 'ls', { path: 'ref/work' }), + }); + const fromStart = await answers(started); + assert.match(fromStart.read.error ?? '', /lies in mount "work", nested in mount "ref"/); + assert.equal(fromStart.disk, 'other_mount'); + assert.deepEqual(await answers(moved), fromStart); +}); + +test("a mount configured beneath a moved direct root is nested beneath a directory on the way that's the root though it's not the first, and beneath none that isn't", async (t) => { + const env = new Env(t); + // `work` is configured at `link/x/y/work`. At `b`, `x` is a directory and + // `x/y` leads to `..`, `b` itself, and `work` to `.`: a start with `ref` at + // `b` nests `work` at `x/y/work` and, beneath `x/y`, at `work`, and not + // beneath `x`, which isn't `b`, so `y/work` is `ref`'s own. + const a = join(env.root, 'a'); + const b = join(env.root, 'b'); + mkdirSync(join(a, 'x', 'y', 'work'), { recursive: true }); + mkdirSync(join(b, 'x'), { recursive: true }); + mkdirSync(join(b, 'y', 'work'), { recursive: true }); + writeFileSync(join(b, 'file.txt'), 'b file\n'); + writeFileSync(join(b, 'y', 'work', 'own.txt'), 'b own\n'); + symlinkSync('..', join(b, 'x', 'y')); + symlinkSync('.', join(b, 'work')); + const link = join(env.root, 'link'); + symlinkSync(a, link); + const mounts: [Partial, MountConfig[]] = [{ path: link, followSymlinks: true }, [{ name: 'work', path: join(link, 'x', 'y', 'work'), mode: 'read-only', backend: 'direct' }]]; + const moved = await env.open(...mounts); + rmSync(link); + symlinkSync(b, link); + const started = await new Env(t).open(...mounts); + const answers = async (m: WorkspaceModule) => ({ + read: await call(m, 'read', { path: 'ref/work/file.txt' }), + own: await call(m, 'read', { path: 'ref/y/work/own.txt' }), + }); + const fromStart = await answers(started); + assert.match(fromStart.read.error ?? '', /lies in mount "work", nested in mount "ref"/); + assert.equal((fromStart.own.data as { content: string }).content, ' 1\tb own'); + assert.deepEqual(await answers(moved), fromStart); +}); + +test("a direct root moved to a directory beneath its own configured path nests no mount by where the last start found a directory beneath that path", async (t) => { + const env = new Env(t); + // `work` is configured at `link/sub/work`. The start, with `link` at `a`, + // finds `link/sub` at `a/sub`; then `link` leads to `a/sub`, where + // `sub/work` is `work`'s directory and `work` is `ref`'s own. + const a = join(env.root, 'a'); + mkdirSync(join(a, 'sub', 'sub', 'work'), { recursive: true }); + mkdirSync(join(a, 'sub', 'work'), { recursive: true }); + writeFileSync(join(a, 'sub', 'work', 'own.txt'), 'sub own\n'); + const link = join(env.root, 'link'); + symlinkSync(a, link); + const mounts: [Partial, MountConfig[]] = [{ path: link }, [{ name: 'work', path: join(link, 'sub', 'work'), mode: 'read-only', backend: 'direct' }]]; + const moved = await env.open(...mounts); + rmSync(link); + symlinkSync(join(a, 'sub'), link); + const started = await new Env(t).open(...mounts); + const answers = async (m: WorkspaceModule) => ({ + own: await call(m, 'read', { path: 'ref/work/own.txt' }), + nested: await call(m, 'read', { path: 'ref/sub/work/x.txt' }), + }); + const fromStart = await answers(started); + assert.equal((fromStart.own.data as { content: string }).content, ' 1\tsub own'); + assert.match(fromStart.nested.error ?? '', /lies in mount "work", nested in mount "ref"/); + assert.deepEqual(await answers(moved), fromStart); +}); + +test("a mount configured beneath a direct root moved to a directory above its configured path is nested beneath that directory too, as a start there nests it", async (t) => { + const env = new Env(t); + // `ref` is configured at `b/link`, which leads first to `a` and then to + // `.`, `b` itself; `work` is configured at `b/link/work`. A start with + // `ref` at `b` nests `work` at `work` and, beneath `b`, at `link/work`. + const a = join(env.root, 'a'); + const b = join(env.root, 'b'); + mkdirSync(join(a, 'work'), { recursive: true }); + mkdirSync(join(b, 'work'), { recursive: true }); + writeFileSync(join(b, 'work', 'f.txt'), 'work f\n'); + const link = join(b, 'link'); + symlinkSync(a, link); + const mounts: [Partial, MountConfig[]] = [{ path: link, followSymlinks: true }, [{ name: 'work', path: join(link, 'work'), mode: 'read-only', backend: 'direct' }]]; + const moved = await env.open(...mounts); + rmSync(link); + symlinkSync('.', link); + const started = await new Env(t).open(...mounts); + const answers = async (m: WorkspaceModule) => ({ + read: await call(m, 'read', { path: 'ref/link/work/f.txt' }), + disk: await m.readFileFromDisk('ref/link/work/f.txt').then(() => 'read', (err: unknown) => (err instanceof WorkspaceReadError ? err.code : String(err))), + }); + const fromStart = await answers(started); + assert.match(fromStart.read.error ?? '', /lies in mount "work", nested in mount "ref"/); + assert.equal(fromStart.disk, 'other_mount'); + assert.deepEqual(await answers(moved), fromStart); +}); + +test("a mount configured at a moved direct root's own path is nested nowhere in it, as a start there nests it", async (t) => { + const env = new Env(t); + // `ref` and `twin` are both configured at `link`. It leads first to `a`, + // which lies in `b` as `sub`, and then to `b`, where a start nests + // neither mount in the other. + const b = join(env.root, 'b'); + const a = join(b, 'sub'); + mkdirSync(a, { recursive: true }); + writeFileSync(join(a, 'f.txt'), 'sub f\n'); + const link = join(env.root, 'link'); + symlinkSync(a, link); + const mounts: [Partial, MountConfig[]] = [{ path: link }, [{ name: 'twin', path: link, mode: 'read-only', backend: 'direct' }]]; + const moved = await env.open(...mounts); + rmSync(link); + symlinkSync(b, link); + const started = await new Env(t).open(...mounts); + const answers = async (m: WorkspaceModule) => ({ + read: await call(m, 'read', { path: 'ref/sub/f.txt' }), + disk: await m.readFileFromDisk('ref/sub/f.txt').then(() => 'read', (err: unknown) => (err instanceof WorkspaceReadError ? err.code : String(err))), + }); + const fromStart = await answers(started); + assert.equal((fromStart.read.data as { content: string }).content, ' 1\tsub f'); + assert.equal(fromStart.disk, 'read'); + assert.deepEqual(await answers(moved), fromStart); +}); + +test("a mount configured at a moved direct root's own path is nested beneath a directory above that path that's the root now, as a start there nests it", async (t) => { + const env = new Env(t); + // `ref` and `twin` are both configured at `b/link`, which leads first to + // `b/sub` and then to `.`, `b` itself: a start there nests `twin` in `ref` + // at `link`, beneath `b`, the directory above `b/link`. + const b = join(env.root, 'b'); + mkdirSync(join(b, 'sub'), { recursive: true }); + writeFileSync(join(b, 'f.txt'), 'b f\n'); + const link = join(b, 'link'); + symlinkSync(join(b, 'sub'), link); + const mounts: [Partial, MountConfig[]] = [{ path: link, followSymlinks: true }, [{ name: 'twin', path: link, mode: 'read-only', backend: 'direct' }]]; + const moved = await env.open(...mounts); + rmSync(link); + symlinkSync('.', link); + const started = await new Env(t).open(...mounts); + const answers = async (m: WorkspaceModule) => ({ + read: await call(m, 'read', { path: 'ref/link/f.txt' }), + own: (await data<{ content: string }>(m, 'read', { path: 'ref/f.txt' })).content, + }); + const fromStart = await answers(started); + assert.match(fromStart.read.error ?? '', /lies in mount "twin", nested in mount "ref"/); + assert.deepEqual(await answers(moved), fromStart); +}); + +test("a mount configured beneath a direct root whose path's own directory above leads elsewhere now is nested as a start there nests it", async (t) => { + const env = new Env(t); + // `ref` is configured at `current/ref` and `work` at `current/ref/work`. + // `current` leads first to `a` and then to `b`, where `ref` and `work` lead + // to `.`: a start there finds `current` at `b`, the root, and nests `work` + // at `work` and, beneath `current`, at `ref/work`. + const a = join(env.root, 'a'); + const b = join(env.root, 'b'); + mkdirSync(join(a, 'ref', 'work'), { recursive: true }); + mkdirSync(b, { recursive: true }); + writeFileSync(join(b, 'file.txt'), 'b file\n'); + symlinkSync('.', join(b, 'ref')); + symlinkSync('.', join(b, 'work')); + const current = join(env.root, 'current'); + symlinkSync(a, current); + const mounts: [Partial, MountConfig[]] = [{ path: join(current, 'ref'), followSymlinks: true }, [{ name: 'work', path: join(current, 'ref', 'work'), mode: 'read-only', backend: 'direct' }]]; + const moved = await env.open(...mounts); + rmSync(current); + symlinkSync(b, current); + const started = await new Env(t).open(...mounts); + const answers = async (m: WorkspaceModule) => ({ + through: await call(m, 'read', { path: 'ref/ref/work/file.txt' }), + read: await call(m, 'read', { path: 'ref/work/file.txt' }), + }); + const fromStart = await answers(started); + assert.match(fromStart.through.error ?? '', /lies in mount "work", nested in mount "ref"/); + assert.deepEqual(await answers(moved), fromStart); +}); + +test("a direct mount whose moved root can't be looked at for a reason other than absence nests what a start there would, not what the last start placed", async (t) => { + const env = new Env(t); + // `work` lies at `b/work`, outside `a`, so the start with `ref` at `a` + // nests nothing; at `b`, it's nested at `work`. + const a = join(env.root, 'a'); + const b = join(env.root, 'b'); + mkdirSync(a, { recursive: true }); + mkdirSync(join(b, 'work'), { recursive: true }); + writeFileSync(join(b, 'work', 'file.txt'), 'work file\n'); + const link = join(env.root, 'link'); + symlinkSync(a, link); + const m = await env.open({ path: link }, [{ name: 'work', path: join(b, 'work'), mode: 'read-only', backend: 'direct' }]); + rmSync(link); + symlinkSync(b, link); + // A look at the moved root fails, as on an I/O error, though everything + // else reads it: the nesting is still judged beneath it. + const realB = realpathSync(b); + const answer = await withFsPromise('stat', (original) => async (...args: any[]) => { + if (args[0] === realB) throw Object.assign(new Error(`EIO: i/o error, stat '${realB}'`), { code: 'EIO' }); + return original(...args); + }, () => call(m, 'read', { path: 'ref/work/file.txt' })); + assert.match(answer.error ?? '', /lies in mount "work", nested in mount "ref"/); +}); + +test("a direct grep of a named file reads it beneath the root its read resolved, though the root is repointed before the open", async (t) => { + const env = new Env(t); + // `b` lies in `a` as `sub`, and at `b`, `file.txt` is a link into `work`'s + // directory there, at a file `work`'s include leaves out. + const a = join(env.root, 'a'); + const b = join(a, 'sub'); + mkdirSync(join(a, 'work'), { recursive: true }); + mkdirSync(join(b, 'work'), { recursive: true }); + writeFileSync(join(a, 'file.txt'), 'a needle\n'); + writeFileSync(join(b, 'work', 'secret.env'), 'S=needle\n'); + symlinkSync(join('work', 'secret.env'), join(b, 'file.txt')); + const link = join(env.root, 'link'); + symlinkSync(a, link); + const m = await env.open({ path: link, followSymlinks: true }, [{ name: 'work', path: join(link, 'work'), mode: 'read-only', backend: 'direct', include: ['*.json'] }]); + // The parse resolves the root at `a`, and the read resolves it again; the + // link is repointed to `b` as that answers, before the file is opened. + let resolved = 0; + const found = await withFsPromise('realpath', (original) => async (...args: any[]) => { + const real = await original(...args); + if (args[0] === link && ++resolved === 2) { + rmSync(link); + symlinkSync(b, link); + } + return real; + }, () => data<{ results: Array<{ file: string; matches: Array<{ text: string }> }> }>(m, 'grep', { pattern: 'needle', path: 'ref/file.txt' })); + assert.equal(resolved, 2); + assert.deepEqual(found.results.map((r) => [r.file, r.matches.map((x) => x.text)]), [['ref/file.txt', ['a needle']]]); +}); + +test("a direct grep judges whether a path it's given names a file beneath the root its parse resolved, though the root moves after", async (t) => { + const env = new Env(t); + // At `a`, `notes` is a file; at `b`, a directory. + const a = join(env.root, 'a'); + const b = join(env.root, 'b'); + mkdirSync(a, { recursive: true }); + mkdirSync(join(b, 'notes'), { recursive: true }); + writeFileSync(join(a, 'notes'), 'a needle\n'); + writeFileSync(join(b, 'notes', 'inner.md'), 'b needle\n'); + const link = join(env.root, 'link'); + symlinkSync(a, link); + const m = await env.open({ path: link }); + // The parse resolves the root at `a`, and the link is repointed to `b` as + // that answers: the path is judged a file beneath `a`, and the read, + // which resolves the root again, finds a directory there. + let repointed = false; + const found = await withFsPromise('realpath', (original) => async (...args: any[]) => { + const real = await original(...args); + if (!repointed && args[0] === link) { + repointed = true; + rmSync(link); + symlinkSync(b, link); + } + return real; + }, () => data<{ results: unknown[]; skipped: Array<{ file: string; reason: string }> }>(m, 'grep', { pattern: 'needle', path: 'ref/notes' })); + assert.ok(repointed); + assert.deepEqual([found.results, found.skipped], [[], [{ file: 'ref/notes', reason: 'a directory' }]]); +}); + +test("a direct listing judges whether a path its include leaves out is a directory beneath the root its parse resolved, though the root moves after", async (t) => { + const env = new Env(t); + // The include covers `*.md` alone. At `a`, `dir` is a directory, which a + // listing may name; at `b`, a file it leaves out. + const a = join(env.root, 'a'); + const b = join(env.root, 'b'); + mkdirSync(join(a, 'dir'), { recursive: true }); + mkdirSync(b, { recursive: true }); + writeFileSync(join(b, 'dir'), 'b dir\n'); + const link = join(env.root, 'link'); + symlinkSync(a, link); + const m = await env.open({ path: link, include: ['*.md'] }); + assert.match(await failure(await new Env(t).open({ path: b, include: ['*.md'] }), 'ls', { path: 'ref/dir' }), /is outside the mount's include/); + let repointed = false; + const listed = await withFsPromise('realpath', (original) => async (...args: any[]) => { + const real = await original(...args); + if (!repointed && args[0] === link) { + repointed = true; + rmSync(link); + symlinkSync(b, link); + } + return real; + }, () => call(m, 'ls', { path: 'ref/dir' })); + assert.ok(repointed); + assert.doesNotMatch(listed.error ?? '', /is outside the mount's include/); +}); + +test("a direct mount whose root moves between a read's parse and its open names the mount a link on the way leads into, by the root it opens under", async (t) => { + const env = new Env(t); + const a = join(env.root, 'a'); + const b = join(env.root, 'b'); + mkdirSync(a, { recursive: true }); + mkdirSync(join(b, 'work'), { recursive: true }); + writeFileSync(join(b, 'work', 'file.txt'), 'disk\n'); + symlinkSync('work', join(b, 'alias')); + const link = join(env.root, 'link'); + symlinkSync(a, link); + const m = await env.open({ path: link }, [{ name: 'work', path: join(b, 'work'), mode: 'read-write', watch: 'never' }]); + // The parse resolves the root at `a`, which has no `alias`; the open finds + // `b`, where `alias` is a link into `work`'s directory. + let repointed = false; + const err = await withFsPromise('realpath', (original) => async (...args: any[]) => { + const real = await original(...args); + if (!repointed && args[0] === link) { + repointed = true; + rmSync(link); + symlinkSync(b, link); + } + return real; + }, () => m.readFileFromDisk('ref/alias/file.txt').then(() => null, (e: unknown) => e)); + assert.ok(repointed); + assert.ok(err instanceof WorkspaceReadError, String(err)); + assert.match(err.message, /lies in mount "work", nested in mount "ref"/); +}); + +test("a direct mount whose root has moved resolves no other mount's root but those configured beneath its path", async (t) => { + const env = new Env(t); + // `ref` reaches `a` through `link`, and `x` is configured through a link + // of its own, elsewhere. + const a = join(env.root, 'a'); + const b = join(env.root, 'b'); + const c = join(env.root, 'c'); + mkdirSync(a, { recursive: true }); + mkdirSync(b, { recursive: true }); + mkdirSync(join(c, 'x'), { recursive: true }); + writeFileSync(join(b, 'own.txt'), 'own\n'); + const link = join(env.root, 'link'); + const xlink = join(env.root, 'xlink'); + symlinkSync(a, link); + symlinkSync(c, xlink); + const m = await env.open({ path: link }, [{ name: 'x', path: join(xlink, 'x'), mode: 'read-only', backend: 'direct' }]); + rmSync(link); + symlinkSync(b, link); + const resolved: string[] = []; + await withFsPromise('realpath', (original) => async (...args: any[]) => { + resolved.push(String(args[0])); + return original(...args); + }, async () => { + assert.equal((await data<{ content: string }>(m, 'read', { path: 'ref/own.txt' })).content, ' 1\town'); + await data(m, 'ls', { path: 'ref', recursive: true }); + await data(m, 'grep', { pattern: 'own', path: 'ref' }); + await m.readFileFromDisk('ref/own.txt'); + }); + assert.ok(resolved.some((path) => path === link), JSON.stringify(resolved)); + assert.deepEqual(resolved.filter((path) => path.startsWith(xlink) || path.startsWith(c)), []); +}); + +test("a direct mount whose root hasn't moved keeps the start's placement, of a mount configured beneath it whose directory has moved since too", async (t) => { + const env = new Env(t); + // `work` is configured at `ref/work`, a link to `x1` at the start. + env.file('x1/f.txt', 'x1\n'); + env.file('x2/f.txt', 'x2\n'); + symlinkSync('x1', join(env.ref, 'work')); + const m = await env.open({}, [{ name: 'work', path: join(env.ref, 'work'), mode: 'read-only', backend: 'direct' }]); + assert.match(await failure(m, 'read', { path: 'ref/x1/f.txt' }), /lies in mount "work", nested in mount "ref"/); + // Repointed to `x2`, `work`'s files are there now, and `ref`'s root is + // where the start placed it, so `x1` stays `work`'s until the next start. + rmSync(join(env.ref, 'work')); + symlinkSync('x2', join(env.ref, 'work')); + assert.match(await failure(m, 'read', { path: 'ref/x1/f.txt' }), /lies in mount "work", nested in mount "ref"/); + assert.equal((await data<{ content: string }>(m, 'read', { path: 'ref/x2/f.txt' })).content, ' 1\tx2'); +}); + +/** Whether the temp directory folds case, as macOS's default disk does. */ +const FOLDS_CASE = (() => { + const dir = mkdtempSync(join(tmpdir(), 'workspace-direct-folds-')); + try { + writeFileSync(join(dir, 'Fold'), ''); + return existsSync(join(dir, 'FOLD')); + } finally { + rmSync(dir, { recursive: true, force: true }); + } +})(); + +test("on a disk that folds case, a mount configured beneath a moved direct root, spelled otherwise than disk spells it, is nested as a start there nests it", { + skip: FOLDS_CASE ? false : "the temp directory tells case apart, so a path spelled in another case reaches nothing: this runs on a disk that folds case, such as macOS's default", +}, async (t) => { + const env = new Env(t); + const a = join(env.root, 'a'); + const b = join(env.root, 'b'); + mkdirSync(join(a, 'work'), { recursive: true }); + mkdirSync(join(b, 'work'), { recursive: true }); + writeFileSync(join(b, 'work', 'file.txt'), 'new\n'); + writeFileSync(join(b, 'top.txt'), 'top\n'); + const link = join(env.root, 'link'); + symlinkSync(a, link); + // Configured as `Work`, which disk spells `work`. + const mounts: [Partial, MountConfig[]] = [{ path: link }, [{ name: 'work', path: join(link, 'Work'), mode: 'read-only', backend: 'direct' }]]; + const moved = await env.open(...mounts); + rmSync(link); + symlinkSync(b, link); + const started = await new Env(t).open(...mounts); + const answers = async (m: WorkspaceModule) => ({ + asDisk: await call(m, 'read', { path: 'ref/work/file.txt' }), + otherwise: await call(m, 'read', { path: 'ref/WORK/file.txt' }), + listed: (await data<{ entries: unknown[] }>(m, 'ls', { path: 'ref', recursive: true })).entries, + found: await data(m, 'grep', { pattern: 'new', path: 'ref' }), + }); + const fromStart = await answers(started); + assert.equal(fromStart.asDisk.success, false); + assert.deepEqual(await answers(moved), fromStart); +}); + +test("on a disk that folds case, a path past the directory of a mount nested in a moved direct root keeps the caller's spelling there, as a start there keeps it", { + skip: FOLDS_CASE ? false : "the temp directory tells case apart, so a path spelled in another case reaches nothing: this runs on a disk that folds case, such as macOS's default", +}, async (t) => { + const env = new Env(t); + // `sub` is configured at `b/sub`, which lies in `ref` once `ref` is moved to `b`. + const a = join(env.root, 'a'); + const b = join(env.root, 'b'); + mkdirSync(a, { recursive: true }); + mkdirSync(join(b, 'sub'), { recursive: true }); + writeFileSync(join(b, 'sub', 'file.txt'), 'sub\n'); + const link = join(env.root, 'link'); + symlinkSync(a, link); + const mounts: [Partial, MountConfig[]] = [{ path: link }, [{ name: 'sub', path: join(b, 'sub'), mode: 'read-only', backend: 'direct' }]]; + const moved = await env.open(...mounts); + rmSync(link); + symlinkSync(b, link); + const started = await new Env(t).open(...mounts); + const answers = async (m: WorkspaceModule) => await failure(m, 'read', { path: 'ref/SUB/File.TXT' }); + const fromStart = await answers(started); + assert.match(fromStart, /"ref\/sub\/File\.TXT" lies in mount "sub", nested in mount "ref".*use "sub\/File\.TXT"/); + assert.equal(await answers(moved), fromStart); +}); + +test("a direct mount whose root is gone, beneath a link, keeps the start's placement, as one beneath no link does", async (t) => { + const env = new Env(t); + // `ref` is configured beneath `link`, so its root as the start placed it + // is `a/ref`, spelled otherwise than its configured path. + const a = join(env.root, 'a'); + mkdirSync(join(a, 'ref', 'work'), { recursive: true }); + const link = join(env.root, 'link'); + symlinkSync(a, link); + const m = await env.open({ path: join(link, 'ref') }, [{ name: 'work', path: join(a, 'ref', 'work'), mode: 'read-only', backend: 'direct' }]); + assert.match(await failure(m, 'read', { path: 'ref/work/file.txt' }), /lies in mount "work", nested in mount "ref"/); + rmSync(join(a, 'ref'), { recursive: true, force: true }); + assert.match(await failure(m, 'read', { path: 'ref/work/file.txt' }), /lies in mount "work", nested in mount "ref"/); +}); + +test("a direct mount whose root now leads through a file keeps the start's placement, as one that's gone does", async (t) => { + const env = new Env(t); + // The start, with `link` at `a`, nests `work` at `work`; then `link` leads + // beneath a file, where nothing can be. + const a = join(env.root, 'a'); + mkdirSync(join(a, 'work'), { recursive: true }); + writeFileSync(join(env.root, 'plain'), 'a file\n'); + const link = join(env.root, 'link'); + symlinkSync(a, link); + const m = await env.open({ path: link }, [{ name: 'work', path: join(a, 'work'), mode: 'read-only', backend: 'direct' }]); + assert.match(await failure(m, 'read', { path: 'ref/work/file.txt' }), /lies in mount "work", nested in mount "ref"/); + rmSync(link); + symlinkSync(join(env.root, 'plain', 'sub'), link); + assert.match(await failure(m, 'read', { path: 'ref/work/file.txt' }), /lies in mount "work", nested in mount "ref"/); +}); + +test("a direct mount whose root moves between a read's parse and its open judges the file by the root it opens under", async (t) => { + const env = new Env(t); + const a = join(env.root, 'a'); + const b = join(env.root, 'b'); + mkdirSync(join(a, 'work'), { recursive: true }); + mkdirSync(join(b, 'work'), { recursive: true }); + writeFileSync(join(a, 'work', 'file.txt'), 'a\n'); + writeFileSync(join(b, 'work', 'file.txt'), 'disk\n'); + const link = join(env.root, 'link'); + symlinkSync(a, link); + const m = await env.open({ path: link }, [{ name: 'work', path: join(b, 'work'), mode: 'read-write', watch: 'never' }]); + // The parse resolves the root at `a`, where nothing is nested, and the + // link is repointed to `b` as that answers, so the open finds `b`, where + // `work` lies. + let repointed = false; + const err = await withFsPromise('realpath', (original) => async (...args: any[]) => { + const real = await original(...args); + if (!repointed && args[0] === link) { + repointed = true; + rmSync(link); + symlinkSync(b, link); + } + return real; + }, () => m.readFileFromDisk('ref/work/file.txt').then(() => null, (e: unknown) => e)); + assert.ok(repointed); + assert.ok(err instanceof WorkspaceReadError, String(err)); + assert.deepEqual([err.code, err.stage], ['other_mount', 'realpath']); +}); + +test("a link inside a moved direct root that leads into a mount nested there is refused as that mount's, at a path only its draft has", async (t) => { + const env = new Env(t); + const a = join(env.root, 'a'); + const b = join(env.root, 'b'); + mkdirSync(a, { recursive: true }); + mkdirSync(join(b, 'work'), { recursive: true }); + symlinkSync('work', join(b, 'alias')); + const link = join(env.root, 'link'); + symlinkSync(a, link); + const m = await env.open({ path: link, followSymlinks: true }, [{ name: 'work', path: join(b, 'work'), mode: 'read-write', watch: 'never' }]); + await data(m, 'write', { path: 'work/new.txt', content: 'draft\n' }); + rmSync(link); + symlinkSync(b, link); + assert.match(await failure(m, 'read', { path: 'ref/alias/new.txt' }), /"ref\/alias\/new\.txt" leads into mount "work", nested in mount "ref"/); +}); + +test("a mount nested in a moved direct root is judged by its own include there, as a start there judges it", async (t) => { + const env = new Env(t); + // In `b`, a link to a file `work`'s include leaves out, and one to a file + // it covers. + const a = join(env.root, 'a'); + const b = join(env.root, 'b'); + mkdirSync(join(a, 'work'), { recursive: true }); + mkdirSync(join(b, 'work'), { recursive: true }); + writeFileSync(join(b, 'work', 'x.md'), 'needle x\n'); + writeFileSync(join(b, 'work', 'y.json'), 'needle y\n'); + symlinkSync(join('work', 'x.md'), join(b, 'l.md')); + symlinkSync(join('work', 'y.json'), join(b, 'k.json')); + const link = join(env.root, 'link'); + symlinkSync(a, link); + const mounts: [Partial, MountConfig[]] = [ + { path: link, followSymlinks: true }, + [{ name: 'work', path: join(link, 'work'), mode: 'read-write', watch: 'never', include: ['*.json'] }], + ]; + const moved = await env.open(...mounts); + rmSync(link); + symlinkSync(b, link); + const started = await new Env(t).open(...mounts); + const answers = async (m: WorkspaceModule) => ({ + outside: await failure(m, 'read', { path: 'ref/l.md' }), + covered: await failure(m, 'read', { path: 'ref/k.json' }), + disk: await m.readFileFromDisk('ref/l.md').then(() => 'read', (err: unknown) => (err instanceof WorkspaceReadError ? err.code : String(err))), + listed: (await data<{ entries: unknown[] }>(m, 'ls', { path: 'ref', recursive: true })).entries, + found: await data(m, 'grep', { pattern: 'needle', path: 'ref' }), + }); + const fromStart = await answers(started); + // A place `work`'s include leaves out is no mount's, so the link leaves + // `ref`; one it covers is `work`'s. + assert.equal(fromStart.outside, 'Symlink escape detected: ref/l.md'); + assert.match(fromStart.covered, /"ref\/k\.json" leads into mount "work", nested in mount "ref"/); + assert.equal(fromStart.disk, 'escape'); + assert.deepEqual(await answers(moved), fromStart); +}); + +test("a direct mount moved away from a mount nested in it judges a path its include leaves out by disk there, as a start there does", async (t) => { + const env = new Env(t); + // `ref` starts at `b`, where `work` lies nested at `b/work`; in `a`, `work` + // is a file the include leaves out. + const a = join(env.root, 'a'); + const b = join(env.root, 'b'); + mkdirSync(a, { recursive: true }); + mkdirSync(join(b, 'work'), { recursive: true }); + writeFileSync(join(a, 'work'), 'needle\n'); + writeFileSync(join(a, 'notes.md'), 'needle\n'); + const link = join(env.root, 'link'); + symlinkSync(b, link); + const mounts: [Partial, MountConfig[]] = [ + { path: link, include: ['*.md'] }, + [{ name: 'work', path: join(b, 'work'), mode: 'read-write', watch: 'never' }], + ]; + const moved = await env.open(...mounts); + rmSync(link); + symlinkSync(a, link); + const started = await new Env(t).open(...mounts); + const answers = async (m: WorkspaceModule) => [ + await call(m, 'ls', { path: 'ref/work' }), + await call(m, 'glob', { pattern: '*', path: 'ref/work' }), + await call(m, 'grep', { pattern: 'needle', path: 'ref/work' }), + await call(m, 'read', { path: 'ref/work' }), + await call(m, 'sync', { path: 'ref/work' }), + await call(m, 'materialize', { path: 'ref/work' }), + ]; + const fromStart = await answers(started); + // Nothing is nested in `ref` at `a`, so `work` is a file of `ref`'s that + // its include leaves out, and no scope. + for (const result of fromStart) assert.match(result.error ?? '', /^"ref\/work" is outside the mount's include/); + assert.deepEqual(await answers(moved), fromStart); +}); + +test("ls without a path and status give a direct mount's kind and ignore list, and a 'chronicle' mount's as before", async (t) => { + const env = new Env(t); + const work = join(env.root, 'work'); + mkdirSync(work); + const m = await env.open({}, [{ name: 'work', path: work, mode: 'read-write', watch: 'never' }]); + const ignore = ['node_modules', 'dist', '.git', 'build', 'coverage', 'target', '__pycache__', '.venv']; + assert.deepEqual((await data(m, 'ls')).mounts, [ + { name: 'ref', path: env.ref, mode: 'read-only', backend: 'direct', ignore }, + { name: 'work', path: work, mode: 'read-write' }, + ]); + const status = await data>>(m, 'status'); + assert.deepEqual(status.ref, { path: env.ref, mode: 'read-only', backend: 'direct', ignore }); + assert.deepEqual(Object.keys(status.work!).sort(), [ + 'canMaterialize', 'conflicts', 'currentBranch', 'currentSeq', 'fileCount', 'initialSyncDone', 'lastMaterializedBranch', + 'lastMaterializedSeq', 'mode', 'path', 'pendingChanges', 'pendingWorkspaceDeletions', 'watch', + ]); + const named = await env.open({ ignore: ['vendor'] }); + assert.deepEqual((await data>>(named, 'status')).ref!.ignore, ['vendor']); +}); + +test('a direct mount refuses every tool that writes, edit_many and materialize included, as read-only', async (t) => { + const env = new Env(t); + env.file('a.ts', 'alpha\n'); + const m = await env.open(); + const states = env.stateIds(); + const calls: Array<[string, Record]> = [ + ['write', { path: 'ref/a.ts', content: 'beta\n' }], + ['edit', { path: 'ref/a.ts', oldString: 'alpha', newString: 'beta' }], + ['edit_many', { edits: [{ path: 'ref/a.ts', oldString: 'alpha', newString: 'beta' }] }], + ['delete', { path: 'ref/a.ts' }], + ]; + for (const [name, input] of calls) assert.match(await failure(m, name, input), /Mount "ref" is read-only/, name); + // materialize names a read-only mount among what it skipped, as on any. + const materialized = await data<{ count: number; skipped: unknown[] }>(m, 'materialize', { mount: 'ref' }); + assert.deepEqual([materialized.count, materialized.skipped], [0, [{ mount: 'ref', reason: 'a read-only mount: materialize writes nothing to it' }]]); + assert.equal((await data<{ content: string }>(m, 'read', { path: 'ref/a.ts' })).content, ' 1\talpha'); + assert.deepEqual(env.stateIds(), states, 'no state was made'); +}); + +test('sync passes over a direct mount, and says so rather than that nothing changed', async (t) => { + const env = new Env(t); + env.file('src/a.ts', 'a\n'); + const m = await env.open(); + const reason = "a direct mount: its tools read disk as it is, so there's nothing to take in"; + assert.deepEqual(await data(m, 'sync'), { results: [], totalSynced: 0, totalConflicts: 0, skipped: [{ mount: 'ref', path: '', reason }] }); + assert.deepEqual((await data(m, 'sync', { mount: 'ref', acceptRoot: true })).skipped, [{ mount: 'ref', path: '', reason }]); + assert.deepEqual((await data(m, 'sync', { path: 'ref/src' })).skipped, [{ mount: 'ref', path: 'src', reason }]); + assert.deepEqual((await data(m, 'materialize', { mount: 'ref' })).skipped, [{ mount: 'ref', reason: 'a read-only mount: materialize writes nothing to it' }]); +}); + +test("readBinary reads a direct mount's disk, a binary file included, up to the mount's size limit", async (t) => { + const env = new Env(t); + env.file('bin.dat', Buffer.from([1, 0, 2])); + env.file('big.dat', Buffer.alloc(9)); + const m = await env.open({ maxFileSize: 8 }); + assert.deepEqual(await m.readBinary('ref/bin.dat'), { data: Buffer.from([1, 0, 2]) }); + assert.deepEqual(await m.readBinary('ref/big.dat'), { error: 'File too large: 9 > 8 bytes' }); + assert.deepEqual(await m.readBinary('ref/none.dat'), { error: 'File not found on disk: ref/none.dat' }); +}); + +test('a direct mount reads beneath whatever directory is at its root now', async (t) => { + const env = new Env(t); + env.file('a.txt', 'old\n'); + // A 'chronicle' run records the root it agreed with. + const chronicle = await env.openChronicle(); + assert.equal((await data<{ content: string }>(chronicle, 'read', { path: 'ref/a.txt' })).content, ' 1\told'); + await env.close(chronicle); + // The checkout is cloned again: another directory at the same path. + renameSync(env.ref, join(env.root, 'ref-before')); + env.file('a.txt', 'new\n'); + env.file('sub/x.txt', 'x\n'); + // A cap of one entry, so a search's start that the old root's identity + // stopped short would leave its directory unvisited. + const direct = await env.open({ walkCap: 1 }); + assert.equal((await data<{ content: string }>(direct, 'read', { path: 'ref/a.txt' })).content, ' 1\tnew'); + assert.deepEqual((await data<{ matches: unknown[] }>(direct, 'glob', { pattern: '*.txt' })).matches, [{ path: 'ref/a.txt', size: 4 }]); + assert.deepEqual((await data<{ matches: unknown[] }>(direct, 'glob', { pattern: 'sub/*.txt' })).matches, [{ path: 'ref/sub/x.txt', size: 2 }]); + await env.close(direct); + // A 'chronicle' run checks the root against the one it agreed with. + const again = await env.openChronicle(); + assert.match(JSON.stringify((await data(again, 'ls', { path: 'ref' })).incomplete), /not the directory disk last agreed with/); +}); + +test('a direct mount whose config sets what would never act there is refused, naming the mount and each setting', () => { + const base: MountConfig = { name: 'ref', path: '/tmp/af-ws-direct-none', mode: 'read-only', backend: 'direct' }; + const refused = (mount: Partial): string => { + try { + new WorkspaceModule({ mounts: [{ ...base, ...mount }] }); + } catch (err) { + return (err as Error).message; + } + assert.fail(`accepted ${JSON.stringify(mount)}`); + }; + const lead = '[workspace] Mount "ref" is direct, which reads disk as it is with no store, watcher or push: '; + assert.equal(refused({ mode: 'read-write' }), `${lead}mode must be 'read-only', not "read-write"`); + assert.equal(refused({ mode: undefined }), `${lead}mode must be 'read-only' (left out, it's 'read-write')`); + assert.equal(refused({ watch: 'always' }), `${lead}watch "always" would watch nothing (leave it out, or set 'never')`); + assert.equal(refused({ watch: 'on-agent-action' }), `${lead}watch "on-agent-action" would watch nothing (leave it out, or set 'never')`); + assert.equal(refused({ watchDebounceMs: 100 }), `${lead}watchDebounceMs would tune no watcher (leave it out)`); + assert.equal(refused({ watchRootPollMs: 100 }), `${lead}watchRootPollMs would tune no watcher (leave it out)`); + assert.equal(refused({ wakeOnChange: true }), `${lead}wakeOnChange would never wake (leave it out)`); + assert.equal(refused({ wakeOnChange: ['created'] }), `${lead}wakeOnChange would never wake (leave it out)`); + assert.equal(refused({ autoMaterialize: true }), `${lead}autoMaterialize would never push (leave it out)`); + assert.equal( + refused({ mode: 'read-write', autoMaterialize: true }), + `${lead}mode must be 'read-only', not "read-write"; autoMaterialize would never push (leave it out)`, + ); + for (const accepted of [{}, { watch: 'never' as const }, { wakeOnChange: false }, { wakeOnChange: [] }, { autoMaterialize: false }]) { + new WorkspaceModule({ mounts: [{ ...base, ...accepted }] }); + } + assert.throws( + () => new WorkspaceModule({ mounts: [{ ...base, backend: 'disk' as MountConfig['backend'] }] }), + { message: `[workspace] Mount "ref": backend must be 'chronicle' or 'direct', not "disk"` }, + ); +}); + +test('tool descriptions mention the direct kind only on a host with a direct mount', async (t) => { + const env = new Env(t); + const descriptions = (m: WorkspaceModule) => Object.fromEntries(m.getTools().map((tool) => [tool.name, tool.description])); + const chronicle = descriptions(await env.openChronicle()); + for (const [name, description] of Object.entries(chronicle)) assert.doesNotMatch(description, /\bdirect\b/, name); + // A host with a writable 'chronicle' mount beside a direct one, as one + // coding in a checkout with its neighbours mounted to read. + const work = join(env.root, 'work'); + mkdirSync(work); + for (const direct of [descriptions(await env.open()), descriptions(await env.open({}, [{ name: 'work', path: work, mode: 'read-write', watch: 'never' }]))]) { + for (const name of Object.keys(chronicle)) { + if (name === 'ls' || name === 'glob' || name === 'grep' || name === 'sync') { + assert.ok(direct[name]!.startsWith(chronicle[name]!) && direct[name]!.length > chronicle[name]!.length, name); + assert.match(direct[name]!, /direct mount/, name); + } else { + assert.equal(direct[name], chronicle[name], name); + } + } + } +}); + +test("a mount the store holds changes on that disk lacks runs as 'chronicle', says so, and starts direct once they're given up", async (t) => { + const env = new Env(t); + env.file('drafted.ts', 'disk\n'); + env.file('deleted.ts', 'kept on disk\n'); + env.file('both.ts', 'v1\n'); + env.file('dot.img', GIF); + const chronicle = await env.openChronicle(); + await data(chronicle, 'ls', { path: 'ref' }); + await data(chronicle, 'write', { path: 'ref/drafted.ts', content: 'draft\n' }); + await data(chronicle, 'delete', { path: 'ref/deleted.ts' }); + await data(chronicle, 'write', { path: 'ref/both.ts', content: 'v2\n' }); + assert.equal((await chronicle.writeBinary('ref/dot.img', PNG, 'image/png')).success, true); + env.file('both.ts', 'v3\n'); + assert.equal((await data<{ entries: Array<{ name: string; state: string }> }>(chronicle, 'ls', { path: 'ref' })).entries.find((e) => e.name === 'both.ts')!.state, 'conflict'); + await env.close(chronicle); + + const warned = t.mock.method(console, 'warn', () => {}); + const held = await env.open({ walkCap: 1 }); + warned.mock.restore(); + const message = "A mount configured direct runs as 'chronicle' this session. To start it direct, settle each held change, then " + + "start it again: sync its path to take disk's copy, or configure the mount 'chronicle' and read-write, materialize it, " + + "follow the answer, and restore its direct config. " + + 'Mount "ref" holds 4 changes that disk lacks: "ref/both.ts" (in conflict), "ref/deleted.ts" (deleted in the workspace), ' + + '"ref/dot.img" (in conflict), "ref/drafted.ts" (changed in the workspace).'; + assert.deepEqual(env.alerts, [{ kind: 'workspace-mount-not-direct', agent: 'framework', message, data: { held: { ref: 4 } } }]); + assert.deepEqual(warned.mock.calls.map((c) => c.arguments), [[`[workspace] ${message}`]]); + // As a 'chronicle' mount, unwatched, with no catch-up scan to take turns + // with its syncs, its changes in view, and saying what it runs as and the + // ignore list it walks by. + const ignore = ['node_modules', 'dist', '.git', 'build', 'coverage', 'target', '__pycache__', '.venv']; + assert.deepEqual((await data(held, 'ls')).mounts, [{ name: 'ref', path: env.ref, mode: 'read-only', backend: 'chronicle', ignore }]); + const status = await data>>(held, 'status'); + assert.equal(status.ref!.backend, 'chronicle'); + assert.deepEqual(status.ref!.ignore, ignore); + assert.equal(status.ref!.watch, 'never'); + assert.deepEqual(watched(held), []); + assert.equal(status.ref!.pendingChanges, 4); + const capped = (await data<{ incomplete: Array<{ path: string; reason: string }> }>(held, 'sync')).incomplete.map((region) => region.reason); + assert.ok(capped.some((reason) => /^the walk cap \(1\) was reached: the next sync of the mount resumes after ref\//.test(reason)), JSON.stringify(capped)); + assert.ok(held.getTools().every((tool) => !/\bdirect\b/.test(tool.description))); + await env.close(held); + + // Every tool reads the workspace's copy there, the draft and the image. + env.alerts = []; + const again = await env.open(); + assert.equal(env.alerts.length, 1, 'each start that finds them says so'); + assert.equal((await data<{ content: string }>(again, 'read', { path: 'ref/drafted.ts' })).content, ' 1\tdraft'); + assert.equal((await data<{ entries: Array<{ name: string; state?: string }> }>(again, 'ls', { path: 'ref' })).entries.find((e) => e.name === 'drafted.ts')?.state, 'workspace-draft'); + assert.deepEqual((await data<{ matches: Array<{ path: string; state?: string }> }>(again, 'glob', { pattern: 'drafted.ts' })).matches.map((x) => [x.path, x.state]), [['ref/drafted.ts', 'workspace-draft']]); + assert.deepEqual((await data<{ results: Array<{ file: string; version: string }> }>(again, 'grep', { pattern: 'draft' })).results.map((r) => [r.file, r.version]), [['ref/drafted.ts', 'workspace']]); + assert.equal((await data>(again, 'read_image', { path: 'ref/dot.img' }))[1]!.mimeType, 'image/png'); + assert.deepEqual(await again.readBinary('ref/dot.img'), { data: PNG }); + + for (const path of ['ref/drafted.ts', 'ref/deleted.ts', 'ref/both.ts', 'ref/dot.img']) await data(again, 'sync', { path }); + await env.close(again); + env.alerts = []; + const direct = await env.open(); + assert.deepEqual(env.alerts, []); + assert.equal((await data>>(direct, 'status')).ref!.backend, 'direct'); + assert.equal((await data<{ content: string }>(direct, 'read', { path: 'ref/drafted.ts' })).content, ' 1\tdisk'); +}); + +test("the changes that keep a mount from starting direct are named ten at a time, an intent that can't be read among them", async (t) => { + const env = new Env(t); + env.file('d00.ts', 'disk\n'); + const chronicle = await env.openChronicle(); + await data(chronicle, 'ls', { path: 'ref' }); + for (let i = 0; i < 11; i++) await data(chronicle, 'write', { path: `ref/d${String(i).padStart(2, '0')}.ts`, content: `${i}\n` }); + await env.close(chronicle); + // Recorded intents whose blob the store doesn't hold: one alone, and one at + // a changed file, which is named for the intent. + for (const path of ['aa.ts', 'd00.ts']) env.store.treeSet('workspace/ref/intent', path, { blobHash: 'f'.repeat(64), size: 2, mode: 0 }); + + await env.open(); + assert.equal(env.alerts.length, 1); + const unreadable = (path: string) => `"ref/${path}" (its workspace intent can't be read; a sync of the path settles it or says why not)`; + const shown = [unreadable('aa.ts'), unreadable('d00.ts'), ...Array.from({ length: 8 }, (_, i) => `"ref/d0${i + 1}.ts" (changed in the workspace)`)]; + // An intent it can't read is counted as one, not as a change. + assert.ok(env.alerts[0]!.message.endsWith(`Mount "ref" holds 10 changes that disk lacks and 2 intents it can't read: ${shown.join(', ')}, and 2 more.`), env.alerts[0]!.message); + assert.deepEqual(env.alerts[0]!.data, { held: { ref: 12 } }); +}); + +test("the ways out say to sync the path of an intent it can't read, where one is held past the ten paths named too", async (t) => { + const env = new Env(t); + const chronicle = await env.openChronicle(); + for (let i = 0; i < 11; i++) await data(chronicle, 'write', { path: `ref/d${String(i).padStart(2, '0')}.ts`, content: `${i}\n` }); + await env.close(chronicle); + const sentence = "and restore its direct config. Sync the path of an intent it can't read. Mount \"ref\" holds"; + + await env.close(await env.open()); + assert.ok(!env.alerts[0]!.message.includes('Sync the path'), env.alerts[0]!.message); + // One that sorts after every change, so it isn't among the ten named. + env.store.treeSet('workspace/ref/intent', 'zz.ts', { blobHash: 'f'.repeat(64), size: 2, mode: 0 }); + env.alerts = []; + await env.open(); + assert.ok(!env.alerts[0]!.message.includes('"ref/zz.ts"'), env.alerts[0]!.message); + assert.ok(env.alerts[0]!.message.includes(sentence), env.alerts[0]!.message); +}); + +test("an intent it can't read where the workspace holds no copy is left by materialize and settled by a sync of its path, as the alert says", async (t) => { + const env = new Env(t); + env.file('u.ts', 'disk\n'); + const chronicle = await env.openChronicle(); + env.store.treeSet('workspace/ref/intent', 'u.ts', { blobHash: 'f'.repeat(64), size: 2, mode: 0 }); + await env.close(chronicle); + await env.close(await env.open()); + assert.match(env.alerts[0]!.message, /Sync the path of an intent it can't read\./); + + // The alert's materialize route, forced too, neither settles it nor names it. + const again = await env.openChronicle(); + for (const input of [{ mount: 'ref' }, { mount: 'ref', force: true }]) { + assert.deepEqual(await data(again, 'materialize', input), { materialized: [], count: 0 }); + } + assert.notEqual(env.store.treeGet('workspace/ref/intent', 'u.ts'), null); + // A sync of its path settles it, and the next start runs direct. + assert.deepEqual((await data<{ totalSynced: number }>(again, 'sync', { path: 'ref/u.ts' })).totalSynced, 1); + assert.equal(env.store.treeGet('workspace/ref/intent', 'u.ts'), null); + await env.close(again); + env.alerts = []; + const direct = await env.open(); + assert.deepEqual([env.alerts.length, (await data>>(direct, 'status')).ref!.backend], [0, 'direct']); +}); + +test('each kind of change the store holds keeps a mount configured direct from starting direct on its own, until a sync of its path settles it', async (t) => { + const cases: Array<[string, (env: Env) => Promise, string, string]> = [ + ['a workspace deletion', async (env) => { + env.file('gone.ts', 'kept on disk\n'); + const chronicle = await env.openChronicle(); + await data(chronicle, 'ls', { path: 'ref' }); + await data(chronicle, 'delete', { path: 'ref/gone.ts' }); + await env.close(chronicle); + }, '1 change that disk lacks', '"ref/gone.ts" (deleted in the workspace)'], + ['a conflict', async (env) => { + await env.close(await env.openChronicle()); + new BranchIntents(env.store, 'workspace/ref/intent').put('c.ts', { conflict: { kind: 'both-changed', at: 0, disk: null } }); + }, '1 change that disk lacks', '"ref/c.ts" (in conflict)'], + ["an intent that can't be read", async (env) => { + await env.close(await env.openChronicle()); + env.store.treeSet('workspace/ref/intent', 'u.ts', { blobHash: 'f'.repeat(64), size: 2, mode: 0 }); + }, "1 intent it can't read", `"ref/u.ts" (its workspace intent can't be read; a sync of the path settles it or says why not)`], + ]; + for (const [kind, hold, counted, named] of cases) { + const env = new Env(t); + await hold(env); + const m = await env.open(); + assert.equal(env.alerts.length, 1, kind); + assert.ok(env.alerts[0]!.message.endsWith(`Mount "ref" holds ${counted}: ${named}.`), env.alerts[0]!.message); + assert.equal((await data>>(m, 'status')).ref!.backend, 'chronicle', kind); + + // The alert's way out: a sync of the path, then the next start is direct. + await data(m, 'sync', { path: named.slice(1, named.indexOf('"', 1)) }); + await env.close(m); + env.alerts = []; + const again = await env.open(); + assert.deepEqual(env.alerts, [], kind); + assert.equal((await data>>(again, 'status')).ref!.backend, 'direct', kind); + } +}); + +test("a tombstone beside an entry that's still there keeps no mount configured direct from starting direct, and holds it once the entry is gone", async (t) => { + const env = new Env(t); + env.file('k.ts', 'kept\n'); + const chronicle = await env.openChronicle(); + await data(chronicle, 'ls', { path: 'ref' }); + await env.close(chronicle); + // A deletion whose removal failed after its tombstone was written, and + // whose put-back failed too, leaves its tombstone beside the entry + // (commitToolChanges); it's given here directly. Disk agreed with the + // entry, which is still there, so the store holds no change disk lacks. + const entry = env.store.treeGet('workspace/ref/tree', 'k.ts'); + assert.ok(entry); + const intents = new BranchIntents(env.store, 'workspace/ref/intent'); + intents.put('k.ts', { tombstone: { hash: entry.blobHash } }); + const m = await env.open(); + assert.deepEqual(env.alerts, []); + assert.equal((await data>>(m, 'status')).ref!.backend, 'direct'); + assert.deepEqual(intents.get('k.ts'), { tombstone: { hash: entry.blobHash } }, 'a direct start leaves the store as it was'); + + // With the entry gone, the same tombstone is a workspace deletion disk + // lacks, and the mount runs as 'chronicle'. + await env.close(m); + env.store.treeRemove('workspace/ref/tree', 'k.ts'); + env.alerts = []; + const held = await env.open(); + assert.equal(env.alerts.length, 1); + assert.ok(env.alerts[0]!.message.endsWith('Mount "ref" holds 1 change that disk lacks: "ref/k.ts" (deleted in the workspace).'), env.alerts[0]!.message); + assert.equal((await data>>(held, 'status')).ref!.backend, 'chronicle'); +}); + +test('mounts configured direct that the store holds changes on are named in one alert', async (t) => { + const env = new Env(t); + const other = join(env.root, 'other'); + mkdirSync(other); + const chronicle = await env.open({ backend: 'chronicle', mode: 'read-write', watch: 'never' }, [{ name: 'other', path: other, mode: 'read-write', watch: 'never' }]); + await data(chronicle, 'write', { path: 'ref/a.ts', content: 'a\n' }); + await data(chronicle, 'write', { path: 'other/b.ts', content: 'b\n' }); + await data(chronicle, 'write', { path: 'other/c.ts', content: 'c\n' }); + await env.close(chronicle); + + const both = await env.open({}, [{ name: 'other', path: other, mode: 'read-only', backend: 'direct' }]); + assert.equal(env.alerts.length, 1); + assert.equal(env.alerts[0]!.message, "Mounts configured direct run as 'chronicle' this session. To start a mount direct, settle each change " + + "held at it, then start it again: sync its path to take disk's copy, or configure the mount 'chronicle' and read-write, " + + "materialize it, follow the answer, and restore its direct config. " + + 'Mounts "ref" (1 change that disk lacks) and "other" (2 changes that disk lacks) hold: "ref/a.ts" (changed in the workspace), ' + + '"other/b.ts" (changed in the workspace), "other/c.ts" (changed in the workspace).'); + assert.deepEqual(env.alerts[0]!.data, { held: { ref: 1, other: 2 } }); + await env.close(both); + + // An intent it can't read, in either mount, is counted as one. + env.store.treeSet('workspace/other/intent', 'u.ts', { blobHash: 'f'.repeat(64), size: 2, mode: 0 }); + env.alerts = []; + await env.open({}, [{ name: 'other', path: other, mode: 'read-only', backend: 'direct' }]); + assert.ok(env.alerts[0]!.message.includes("Mounts \"ref\" (1 change that disk lacks) and \"other\" (2 changes that disk lacks and 1 intent it can't read) hold: "), env.alerts[0]!.message); +}); + +test("the alert's ways out come within the start a webhook keeps of it, however many mounts it names and however long their names", async (t) => { + const env = new Env(t); + const extra = ['another-checkout-whose-name-is-rather-long', 'a-third-checkout-with-a-long-name-too'].map((name) => { + mkdirSync(join(env.root, name)); + return { name, path: join(env.root, name) }; + }); + const chronicle = await env.open({ backend: 'chronicle', mode: 'read-write', watch: 'never' }, extra.map((m) => ({ ...m, mode: 'read-write' as const, watch: 'never' as const }))); + for (const name of ['ref', ...extra.map((m) => m.name)]) await data(chronicle, 'write', { path: `${name}/draft.ts`, content: 'draft\n' }); + await env.close(chronicle); + // An intent it can't read too, whose sentence is the last of the ways out. + env.store.treeSet('workspace/ref/intent', 'u.ts', { blobHash: 'f'.repeat(64), size: 2, mode: 0 }); + + await env.open({}, extra.map((m) => ({ ...m, mode: 'read-only' as const, backend: 'direct' as const }))); + const message = env.alerts[0]!.message; + // The framework's webhook keeps the first 346 characters of an alert over + // 500, with its end (framework.ts, OPS_WEBHOOK_MESSAGE_MAX_CHARS and + // boundFailureText). + assert.ok(message.length > 500, `${message.length}`); + const last = "Sync the path of an intent it can't read."; + const at = message.indexOf(last); + assert.ok(at >= 0 && at + last.length <= 340, `${at}`); +}); + +test('a start of the same module decides again whether a mount configured direct runs direct', async (t) => { + const env = new Env(t); + env.file('drafted.ts', 'disk\n'); + const chronicle = await env.openChronicle(); + await data(chronicle, 'write', { path: 'ref/drafted.ts', content: 'draft\n' }); + await env.close(chronicle); + + const m = await env.open(); + assert.equal(env.alerts.length, 1); + await env.restart(m); + assert.equal(env.alerts.length, 2, 'a start that finds the change still held says so again'); + await data(m, 'sync', { path: 'ref/drafted.ts' }); + await env.restart(m); + assert.equal(env.alerts.length, 2, 'the start that finds none says nothing'); + assert.equal((await data>>(m, 'status')).ref!.backend, 'direct'); + assert.equal((await data<{ content: string }>(m, 'read', { path: 'ref/drafted.ts' })).content, ' 1\tdisk'); +}); + +test('whichever of start and initStore comes first, a mount configured direct is decided once the store is given, and said once', async (t) => { + const env = new Env(t); + env.file('drafted.ts', 'disk\n'); + const chronicle = await env.openChronicle(); + await data(chronicle, 'write', { path: 'ref/drafted.ts', content: 'draft\n' }); + await env.close(chronicle); + const config = { mounts: [{ name: 'ref', path: env.ref, mode: 'read-only' as const, backend: 'direct' as const }] }; + const status = async (m: WorkspaceModule) => (await data>>(m, 'status')).ref!.backend; + + // The host's order: start, then initStore. + const host = new WorkspaceModule(config); + try { + await host.start(env.context()); + assert.deepEqual(env.alerts, []); + host.initStore(env.store); + assert.equal(env.alerts.length, 1); + assert.equal(await status(host), 'chronicle'); + assert.equal((await data<{ content: string }>(host, 'read', { path: 'ref/drafted.ts' })).content, ' 1\tdraft'); + } finally { + await host.stop(); + } + + // The store first: decided before start, so a read between the two reads + // the change, and start says so once. + env.alerts = []; + const early = new WorkspaceModule(config); + try { + early.initStore(env.store); + assert.equal(await status(early), 'chronicle'); + assert.equal((await data<{ content: string }>(early, 'read', { path: 'ref/drafted.ts' })).content, ' 1\tdraft'); + await early.start(env.context()); + assert.equal(env.alerts.length, 1); + await data(early, 'sync', { path: 'ref/drafted.ts' }); + } finally { + await early.stop(); + } + + // With nothing held, a mount that ran as 'chronicle' is direct from + // initStore on. + const settled = new WorkspaceModule(config); + try { + settled.initStore(env.store); + assert.equal(await status(settled), 'direct'); + } finally { + await settled.stop(); + } +}); + +test("a decision that can't read the store leaves a mount that ran as 'chronicle' as it was, never direct", async (t) => { + const env = new Env(t); + env.file('drafted.ts', 'disk\n'); + const chronicle = await env.openChronicle(); + await data(chronicle, 'write', { path: 'ref/drafted.ts', content: 'draft\n' }); + await env.close(chronicle); + type Deciding = { heldChanges: () => never }; + const unreadable = () => { throw new Error('the evidence could not be read'); }; + + // At a later start, the earlier decision stands, and the start fails. + const m = await env.open(); + const atStart = t.mock.method(m as unknown as Deciding, 'heldChanges', unreadable); + await assert.rejects(env.restart(m), /the evidence could not be read/); + atStart.mock.restore(); + assert.equal((await data>>(m, 'status')).ref!.backend, 'chronicle'); + assert.equal((await data<{ content: string }>(m, 'read', { path: 'ref/drafted.ts' })).content, ' 1\tdraft'); + + // At initStore, before any decision has been made, it isn't direct. + const fresh = new WorkspaceModule({ mounts: [{ name: 'ref', path: env.ref, mode: 'read-only', backend: 'direct' }] }); + const atInit = t.mock.method(fresh as unknown as Deciding, 'heldChanges', unreadable); + assert.throws(() => fresh.initStore(env.store), /the evidence could not be read/); + atInit.mock.restore(); + assert.equal((await data<{ content: string }>(fresh, 'read', { path: 'ref/drafted.ts' })).content, ' 1\tdraft'); +}); + +test('a decision that fails at a later mount sets none of the mounts it decided before it', async (t) => { + const env = new Env(t); + const other = join(env.root, 'other'); + mkdirSync(other); + env.file('drafted.ts', 'disk\n'); + const chronicle = await env.open({ backend: 'chronicle', mode: 'read-write', watch: 'never' }, [{ name: 'other', path: other, mode: 'read-write', watch: 'never' }]); + await data(chronicle, 'write', { path: 'ref/drafted.ts', content: 'draft\n' }); + await data(chronicle, 'ls', { path: 'other' }); + await env.close(chronicle); + + const m = await env.open({}, [{ name: 'other', path: other, mode: 'read-only', backend: 'direct' }]); + const status = async () => { + const all = await data>>(m, 'status'); + return [all.ref!.backend, all.other!.backend]; + }; + assert.deepEqual(await status(), ['chronicle', 'direct']); + // "ref" is settled, so a decision would start it direct, but reading + // "other", decided after it, fails. + await data(m, 'sync', { path: 'ref/drafted.ts' }); + type Deciding = { heldChanges: (state: { config: { name: string } }) => unknown }; + const original = (m as unknown as Deciding).heldChanges; + const failing = t.mock.method(m as unknown as Deciding, 'heldChanges', function (this: unknown, state: { config: { name: string } }) { + if (state.config.name === 'other') throw new Error('the evidence could not be read'); + return original.call(this, state); + }); + await assert.rejects(env.restart(m), /the evidence could not be read/); + failing.mock.restore(); + assert.deepEqual(await status(), ['chronicle', 'direct']); + await env.restart(m); + assert.deepEqual(await status(), ['direct', 'direct']); +}); + +test('changes held on a branch not selected at the start keep no mount from starting direct', async (t) => { + const env = new Env(t); + env.file('drafted.ts', 'disk\n'); + const main = env.store.currentBranch().name; + env.store.createBranch('side', main); + env.store.switchBranch('side'); + const chronicle = await env.openChronicle(); + await data(chronicle, 'write', { path: 'ref/drafted.ts', content: 'draft\n' }); + await env.close(chronicle); + + env.store.switchBranch(main); + const m = await env.open(); + assert.deepEqual(env.alerts, [], 'the branch selected holds none'); + assert.equal((await data>>(m, 'status')).ref!.backend, 'direct'); + // The same module, started again on the branch that holds it, runs + // store-backed and says so. + await m.stop(); + env.store.switchBranch('side'); + await m.start(env.context()); + assert.equal(env.alerts.length, 1, 'the branch that holds it keeps the mount store-backed'); + assert.equal((await data>>(m, 'status')).ref!.backend, 'chronicle'); + assert.equal((await data<{ content: string }>(m, 'read', { path: 'ref/drafted.ts' })).content, ' 1\tdraft'); +}); + +test("a mount configured direct with no ignore list walks by the direct kind's default while it runs as 'chronicle'", async (t) => { + const env = new Env(t); + env.file('src/a.ts', 'a\n'); + env.file('node_modules/p/i.js', 'i\n'); + env.file('.git/HEAD', 'ref: refs/heads/main\n'); + const chronicle = await env.openChronicle(); + await data(chronicle, 'write', { path: 'ref/drafted.ts', content: 'draft\n' }); + await env.close(chronicle); + + const held = await env.open(); + assert.equal(env.alerts.length, 1, "it runs as 'chronicle'"); + const listed = (await data<{ entries: Array<{ path: string }> }>(held, 'ls', { path: 'ref', recursive: true })).entries.map((e) => e.path); + assert.deepEqual(listed.filter((path) => !['drafted.ts', 'src/a.ts'].includes(path)), []); + assert.deepEqual(env.store.treeList('workspace/ref/tree').map((e) => e.path).sort(), ['drafted.ts', 'src/a.ts']); +}); + +test("a mount that ran direct takes in what disk changed meanwhile when it runs as 'chronicle' again", async (t) => { + const env = new Env(t); + env.file('changed.ts', 'before\n'); + env.file('removed.ts', 'gone soon\n'); + const first = await env.openChronicle(); + assert.equal((await data<{ count: number }>(first, 'ls', { path: 'ref', recursive: true })).count, 2); + await env.close(first); + + const direct = await env.open(); + env.file('changed.ts', 'after\n'); + unlinkSync(join(env.ref, 'removed.ts')); + env.file('added.ts', 'new\n'); + assert.equal((await data<{ content: string }>(direct, 'read', { path: 'ref/changed.ts' })).content, ' 1\tafter'); + await env.close(direct); + + const again = await env.openChronicle(); + assert.deepEqual((await data(again, 'ls', { path: 'ref', recursive: true })).entries, [ + { path: 'added.ts', state: 'synced', size: 4 }, + { path: 'changed.ts', state: 'synced', size: 6 }, + ]); + assert.equal((await data<{ content: string }>(again, 'read', { path: 'ref/changed.ts' })).content, ' 1\tafter'); + const status = (await data>>(again, 'status')).ref!; + assert.deepEqual([status.pendingChanges, status.conflicts], [0, 0]); +}); + +test("a direct mount's listings pass over another mount's directory, naming the mount to use", async (t) => { + const env = new Env(t); + env.file('inner/x.ts', 'x\n'); + env.file('top.ts', 't\n'); + const m = await env.open({}, [{ name: 'inner', path: join(env.ref, 'inner'), mode: 'read-only', watch: 'never' }]); + assert.deepEqual((await data(m, 'ls', { path: 'ref' })).entries, [{ name: 'top.ts', type: 'file', size: 2 }]); + assert.deepEqual(await data(m, 'ls', { path: 'ref/inner' }), { + path: 'ref/inner', + entries: [], + count: 0, + incomplete: [{ path: 'inner', reason: 'it lies in mount "inner", nested in this one: use "inner"' }], + }); + assert.match(await failure(m, 'read', { path: 'ref/inner/x.ts' }), /lies in mount "inner", nested in mount "ref"/); +}); + +test('inOrder hands each result on in order, at most `width` at once', async () => { + const settle: Array<() => void> = []; + const settled: number[] = []; + const order: number[] = []; + let running = 0; + let most = 0; + const done = workspaceModule.inOrder([0, 1, 2, 3, 4, 5, 6, 7], 3, (i) => { + running++; + most = Math.max(most, running); + return new Promise((resolve) => { + settle[i] = () => { + running--; + settled.push(i); + resolve(i * 10); + }; + }); + }, (result, i) => { + assert.equal(result, i * 10); + order.push(i); + }); + // Each window's works settle last to first, so each settles before the + // ones ahead of it are handed on. + const turn = () => new Promise((resolve) => setImmediate(resolve)); + for (const window of [[2, 1, 0], [5, 4, 3], [7, 6]]) { + await turn(); + for (const i of window) { + settle[i]!(); + await turn(); + } + } + await done; + assert.deepEqual(settled, [2, 1, 0, 5, 4, 3, 7, 6]); + assert.deepEqual(order, [0, 1, 2, 3, 4, 5, 6, 7]); + assert.equal(most, 3); +}); + +test('inOrder lets each result go once it has handed it on', async () => { + setFlagsFromString('--expose_gc'); + const gc = runInNewContext('gc') as () => void; + const handed: Array> = []; + let firstKept: boolean | undefined; + // Each work ends in a later task than the one before it, so a WeakRef made + // in one is no longer held for its task when gc runs in another. + await workspaceModule.inOrder(Array.from({ length: 10 }, (_, i) => i), 2, async (i) => { + await new Promise((resolve) => setImmediate(resolve)); + return { i, bytes: Buffer.alloc(64 * 1024) }; + }, (result, i) => { + handed.push(new WeakRef(result)); + if (i === 9) { + gc(); + firstKept = handed[0]!.deref() !== undefined; + } + }); + assert.equal(firstKept, false); +}); + +test('inOrder rejects in turn for a work that rejects before its turn, leaving no rejection unhandled', async () => { + const unhandled: unknown[] = []; + const record = (reason: unknown) => { unhandled.push(reason); }; + process.on('unhandledRejection', record); + try { + let second!: () => void; + const secondRejected = new Promise((resolve) => { second = resolve; }); + const seen: number[] = []; + await assert.rejects(workspaceModule.inOrder([0, 1, 2], 3, async (i) => { + if (i === 1) { + queueMicrotask(second); + throw new Error('the second'); + } + // The first finishes only after the second has rejected and a turn of + // the event loop has passed, when Node reports a rejection unhandled. + if (i === 0) { + await secondRejected; + await new Promise((resolve) => setImmediate(resolve)); + } + return i; + }, (_, i) => { seen.push(i); }), { message: 'the second' }); + await new Promise((resolve) => setImmediate(resolve)); + assert.deepEqual(seen, [0]); + assert.deepEqual(unhandled, []); + } finally { + process.off('unhandledRejection', record); + } +});