fix(workspace): a per-mount walk cap, and syncs and scans that resume past it (#272)

changelog.d/272-walk-cap.fixed.md

@@ -0,0 +1,5 @@
+- A workspace mount with more files than one walk takes is now covered by repeated syncs and scans (#272). Each mount sets its cap with `walkCap`, a positive integer no larger than `Number.MAX_SAFE_INTEGER` that defaults to 5000; any other value is refused when the module is created, naming the mount. A bare `sync`, a `sync` of a directory, an agent-action scan, or a watched mount's catch-up scan (at start and when its watcher reattaches) that reaches the cap lists what it didn't reach under `incomplete`, and the next one of the same scope resumes after the last entry it took, then wraps to the start. The place is kept with the mount's disk evidence, so a restart resumes there too. Before, every walk stopped at the same 5000 files, so the files past the cap were never scanned, and a change to one reached the workspace only through a `sync` of its path. A listing (`ls`, `glob`, `grep`) still takes up to the cap from its scope's start.
+  - A walk takes each directory's entries that aren't directories by name, then its subdirectories by name, so each walk can resume where the last stopped, whatever order the platform lists a directory in. A link, or another entry that isn't a directory, counts against the cap as a file does, since the pass reads each one it takes, through the link on a mount that follows them.
+  - Each directory's listing is read whole, so a file deleted on disk is seen wherever the walk lists, past the cap included. Before, a directory holding more files than the cap never proved a deletion.
+  - The `sync` tool's description now says that a sync takes up to the mount's walk cap and that the next sync of the same mount or directory resumes where it stopped, so the first request after the upgrade that carries the workspace's tools misses the prompt cache from its tool block on.
+  - Going back to 0.22: its walks stop at the same 5000 files as before. It leaves the place alone, so coming forward again resumes there, unless 0.22 wrote a checkpoint meanwhile, which drops every place and starts each scope at its top.

src/modules/workspace/disk-agreement.ts

@@ -92,16 +92,30 @@ type JournalEntry =
   /** The root's identity; with `reset`, an accepted new root, whose evidence starts empty. */
   | { t: 'root'; mount: string; identity: RootIdentity; reset?: true }
   | { t: 'set'; mount: string; path: string; value: Physical }
-  | { t: 'forget'; mount: string; path: string };
+  | { t: 'forget'; mount: string; path: string }
+  /**
+   * Where a capped walk of `scope` stopped (the last entry it took), or null
+   * once a walk covered the scope. It carries no `path` field: agent-framework
+   * 0.22 replays an entry it doesn't know as a `forget` of `entry.path`, and
+   * so leaves the evidence alone when there is none.
+   */
+  | { t: 'walk'; mount: string; scope: string; after: string | null };
 
 interface MountEvidence {
   root: string;
   identity?: RootIdentity;
   paths: Map<string, Physical>;
+  /** By scope ('' for the mount root): where the next walk of it resumes. */
+  walks: Map<string, string>;
 }
 
 interface Snapshot {
-  mounts: Record<string, { root: string; identity?: RootIdentity; paths: Record<string, Physical> }>;
+  mounts: Record<string, {
+    root: string;
+    identity?: RootIdentity;
+    paths: Record<string, Physical>;
+    walks?: Record<string, string>;
+  }>;
 }
 
 export const DISK_AGREEMENT_RECORD_TYPE = 'workspace/disk-agreement';
@@ -183,27 +197,27 @@ export class DiskAgreement {
         root: mount.root,
         ...(mount.identity ? { identity: mount.identity } : {}),
         paths: new Map(Object.entries(mount.paths)),
+        walks: new Map(Object.entries(mount.walks ?? {})),
       });
     }
     for (const { entry } of entries) {
       if (entry.t === 'mount') {
         const existing = replayed.get(entry.mount);
-        if (!existing || !sameRootPath(existing.root, entry.root)) replayed.set(entry.mount, { root: entry.root, paths: new Map() });
+        if (!existing || !sameRootPath(existing.root, entry.root)) {
+          replayed.set(entry.mount, { root: entry.root, paths: new Map(), walks: new Map() });
+        }
         continue;
       }
       const mount = replayed.get(entry.mount);
-      if (!mount) continue;
-      if (entry.t === 'root') applyRoot(mount, entry);
-      else if (entry.t === 'set') mount.paths.set(entry.path, entry.value);
-      else mount.paths.delete(entry.path);
+      if (mount) apply(mount, entry);
     }
     this.mounts.clear();
     for (const { name, root } of this.configured) {
       const found = replayed.get(name);
       if (found && sameRootPath(found.root, root)) {
         this.mounts.set(name, found);
       } else {
-        this.mounts.set(name, { root, paths: new Map() });
+        this.mounts.set(name, { root, paths: new Map(), walks: new Map() });
         this.journal.append({ t: 'mount', mount: name, root });
       }
     }
@@ -250,6 +264,27 @@ export class DiskAgreement {
     return out;
   }
 
+  /**
+   * Where the next walk of `scope` ('' for the mount root) resumes: the last
+   * entry a walk of it took before its cap stopped it, if one did.
+   */
+  walkResumesAfter(mount: string, scope: string): string | undefined {
+    this.ensureReconciled();
+    return this.mounts.get(mount)?.walks.get(scope);
+  }
+
+  /**
+   * Record where a walk of `scope` stopped (`after`), or that one covered it
+   * (null). Kept with the evidence so a restart resumes there too. A place
+   * that's lost costs coverage, never a decision: an append that didn't land
+   * loses one, and a checkpoint written by a version that doesn't keep them
+   * (0.22) loses them all, and each scope's walks start at its top again.
+   */
+  recordWalk(mount: string, scope: string, after: string | null): void {
+    if ((this.walkResumesAfter(mount, scope) ?? null) === after) return;
+    this.write({ t: 'walk', mount, scope, after }, {});
+  }
+
   /**
    * Record an intent. `durable` makes it (and every append before it) stable
    * before the caller performs the effect; batches pass it on their last one.
@@ -288,10 +323,7 @@ export class DiskAgreement {
     this.journal.append(entry, opts);
     if (opts.durable) this.unsynced = false; // synced after this append, and so everything before it
     const mount = this.mounts.get((entry as { mount: string }).mount);
-    if (!mount) return;
-    if (entry.t === 'root') applyRoot(mount, entry);
-    else if (entry.t === 'set') mount.paths.set(entry.path, entry.value);
-    else if (entry.t === 'forget') mount.paths.delete(entry.path);
+    if (mount) apply(mount, entry);
   }
 
   /** Checkpoint when the tail has outgrown the live evidence. */
@@ -307,6 +339,7 @@ export class DiskAgreement {
         root: mount.root,
         ...(mount.identity ? { identity: mount.identity } : {}),
         paths: Object.fromEntries(mount.paths),
+        ...(mount.walks.size > 0 ? { walks: Object.fromEntries(mount.walks) } : {}),
       };
     }
     this.journal.checkpoint(snapshot);
@@ -319,7 +352,23 @@ export class DiskAgreement {
   }
 }
 
+/** Apply one entry to a mount's evidence; one of a kind this version doesn't know changes nothing. */
+function apply(mount: MountEvidence, entry: JournalEntry): void {
+  if (entry.t === 'root') applyRoot(mount, entry);
+  else if (entry.t === 'set') mount.paths.set(entry.path, entry.value);
+  else if (entry.t === 'forget') mount.paths.delete(entry.path);
+  else if (entry.t === 'walk') {
+    if (entry.after === null) mount.walks.delete(entry.scope);
+    else mount.walks.set(entry.scope, entry.after);
+  }
+}
+
 function applyRoot(mount: MountEvidence, entry: Extract<JournalEntry, { t: 'root' }>): void {
   mount.identity = entry.identity;
-  if (entry.reset) mount.paths.clear();
+  if (entry.reset) {
+    // Evidence about another directory, and where its walks stopped, say
+    // nothing about this one.
+    mount.paths.clear();
+    mount.walks.clear();
+  }
 }

src/modules/workspace/index.ts

@@ -50,6 +50,7 @@ import {
   type PushResult,
   type Scope,
   type TreeOp,
+  WALK_CAP,
   pushPaths,
   reconcilePass,
   settleAdoptionBeforeMutation,
@@ -651,6 +652,16 @@ export class WorkspaceModule implements Module {
   private savedState: WorkspaceModuleState | null = null;
 
   constructor(config: WorkspaceConfig) {
+    for (const mount of config.mounts) {
+      const cap: unknown = mount.walkCap;
+      if (cap !== undefined && !(Number.isSafeInteger(cap) && (cap as number) > 0)) {
+        // Numbers as JavaScript writes them (JSON would say null for NaN), anything else as JSON.
+        const shown = typeof cap === 'number' ? String(cap) : typeof cap === 'bigint' ? `${cap}n` : JSON.stringify(cap) ?? String(cap);
+        throw new Error(
+          `[workspace] Mount "${mount.name}": walkCap must be a positive integer no larger than ${Number.MAX_SAFE_INTEGER}, not ${shown}`,
+        );
+      }
+    }
     // Detect overlapping mount paths: if mount A contains mount B,
     // auto-add an ignore rule on A for B's path to prevent syncing
     // the sub-mount's directory through the super-mount.
@@ -779,6 +790,7 @@ export class WorkspaceModule implements Module {
       treeStateId: mount.treeStateId,
       intents: this.intents.get(mount.config.name)!,
       readOnly: mount.config.mode === 'read-only',
+      walkCap: mount.config.walkCap ?? WALK_CAP,
     };
   }
 
@@ -1004,7 +1016,7 @@ export class WorkspaceModule implements Module {
     try {
       // A full pass under the three-way rule: deletions on disk are confirmed
       // where nothing newer is pending, and nothing unvisited is touched.
-      await this.withMount(mount, () => this.passUnlocked(mount, { kind: 'dir', dir: '', recursive: true }));
+      await this.withMount(mount, () => this.passUnlocked(mount, { kind: 'dir', dir: '', recursive: true }, { resume: true }));
     } catch (err) {
       const msg = err instanceof Error ? err.message : String(err);
       this.ctx?.pushEvent({
@@ -1211,7 +1223,10 @@ export class WorkspaceModule implements Module {
       {
         name: 'sync',
         description: 'Bring disk changes into the workspace. Without a path, every mount (or the given one) is '
-          + 'rechecked in full and nothing pending in the workspace is discarded. With a path (file or directory), '
+          + 'rechecked and nothing pending in the workspace is discarded. A sync takes up to the mount\'s walk '
+          + 'cap of entries, files, links and the like (5000 unless configured); past it, what this one didn\'t reach is listed under '
+          + '`incomplete`, and the next sync of the same mount or directory resumes where it stopped. With a '
+          + 'path (file or directory), '
           + 'the workspace takes disk\'s state there explicitly: it restores a workspace-deleted file, resolves '
           + 'conflicts toward disk, replaces a workspace draft, and removes a workspace file disk does not have. '
           + 'Each workspace change it gives up is listed under `discarded`, with the state it had and, as `op`, '
@@ -2851,7 +2866,8 @@ export class WorkspaceModule implements Module {
         if (input.acceptRoot && await this.acceptRootUnlocked(mount)) rootsAccepted.push(name);
         if (!relativePath) {
           // A full recheck: every file rehashed, nothing pending discarded.
-          return this.passUnlocked(mount, { kind: 'dir', dir: '', recursive: true }, { rehash: true });
+          // Past the mount's walkCap, it resumes where the last one stopped.
+          return this.passUnlocked(mount, { kind: 'dir', dir: '', recursive: true }, { rehash: true, resume: true });
         }
         // An explicit path: the workspace takes disk's state there.
         const prefix = relativePath + '/';
@@ -2864,7 +2880,7 @@ export class WorkspaceModule implements Module {
         // beneath it (a directory scope's walk; a path scope doesn't walk).
         const opts: PassOptions = { rehash: true, adopt: () => true, branchId: askedOn, named: true };
         if (!isDir) return this.passUnlocked(mount, { kind: 'paths', paths: [relativePath] }, opts);
-        const beneath = await this.passUnlocked(mount, { kind: 'dir', dir: relativePath, recursive: true }, opts);
+        const beneath = await this.passUnlocked(mount, { kind: 'dir', dir: relativePath, recursive: true }, { ...opts, resume: true });
         // Something at the path itself is no part of a directory scope: a file
         // disk has where the workspace holds a directory, or a workspace file
         // where disk has a directory. Its own pass takes disk's state there
@@ -3057,7 +3073,7 @@ export class WorkspaceModule implements Module {
     const scan: AgentActionScan = { late: false, settled: Promise.resolve() };
     const pass = this.withMount(mount, () => {
       if (mount.agentActionScanWaiting === scan) mount.agentActionScanWaiting = undefined;
-      return this.passUnlocked(mount, { kind: 'dir', dir: '', recursive: true });
+      return this.passUnlocked(mount, { kind: 'dir', dir: '', recursive: true }, { resume: true });
     });
     scan.settled = pass.then(
       (result) => {