fix(workspace): restore contents the store lost from disk's matching copy

changelog.d/workspace-restore-lost-contents.fixed.md

@@ -0,0 +1,3 @@
+- A workspace file whose stored contents the store has lost, or can't read, is now read from disk's copy when that copy has exactly those bytes (the same size and SHA-256), by `read`, `edit`, `edit_many`, `grep`, `read_image` and `readBinary`. The contents are stored again, and whoever runs the host gets an ops alert: `workspace-contents-restored`, `workspace-contents-unstored` when the store doesn't keep them, or `workspace-contents-lost` when no copy with their hash can be read. Before, `read`, `edit`, `edit_many` and `readBinary` answered "Blob not found for:", `grep` left the file unsearched, and a blob it couldn't read failed the whole `grep`. Where nothing can restore them, the answer says what the store and disk hold and what the agent can do instead.
+- `read_image` serving disk's copy of a file whose workspace copy is lost now says so in its text.
+- `materialize` lists a workspace copy the store can't read as skipped, saying so, rather than failing the whole call, and a branch intent whose stored record can't be read leaves its path unverified with that reason rather than failing the listing.

src/modules/workspace/branch-intent.ts

@@ -19,6 +19,7 @@
  */
 
 import type { JsStore } from '@animalabs/chronicle';
+import { storedBytes } from './sync.js';
 
 export type ConflictKind =
   /** Disk, workspace and the last agreement all differ. */
@@ -56,9 +57,11 @@ export interface BranchIntent {
   origin?: 'store';
   conflict?: ConflictRecord;
   /**
-   * Why a recorded intent can't be read (its blob missing from the store, or
-   * not an intent): a tombstone or conflict it held is unknown, never absent.
-   * Never written; a put replaces the damaged record.
+   * Why a recorded intent can't be read (its blob missing from the store,
+   * a blob chronicle can't read, or not an intent): a tombstone or conflict
+   * it held is unknown, never absent.
+   * Never written; a put replaces the damaged record, unless it puts the same
+   * contents over a blob file chronicle can't read, which chronicle keeps.
    */
   unreadable?: string;
 }
@@ -70,8 +73,10 @@ function isEmpty(intent: BranchIntent): boolean {
 
 /** A recorded intent's blob, read; a damaged one says why it can't be. */
 function readIntent(store: JsStore, blobHash: string): BranchIntent {
-  const blob = store.getBlob(blobHash);
-  if (!blob) return { unreadable: "its record's blob is missing from the store" };
+  const stored = storedBytes(store, blobHash);
+  if ('missing' in stored) return { unreadable: "its record's blob is missing from the store" };
+  if ('unreadable' in stored) return { unreadable: `its record's blob can't be read from the store (${stored.unreadable})` };
+  const blob = stored.bytes;
   try {
     const parsed = JSON.parse(blob.toString('utf-8')) as unknown;
     if (parsed && typeof parsed === 'object' && !Array.isArray(parsed)) return parsed as BranchIntent;

src/modules/workspace/observe.ts

@@ -149,7 +149,7 @@ export const ROOT_UNAVAILABLE = 'the mount root is unavailable';
 export const ROOT_REPLACED =
   'the mount root is not the directory disk last agreed with (an unmounted drive, or a replaced directory): ' +
   'reconnect it; if it was replaced on purpose, sync the mount with acceptRoot to take it as it is now';
-const CHANGED = 'it changed while it was observed';
+export const CHANGED = 'it changed while it was observed';
 const CHANGED_WALK = 'the directory changed while the walk ran';
 const BENEATH_SYMLINK = 'beneath a symlink that does not lead to a directory inside the mount';
 

src/modules/workspace/reconcile.ts

@@ -53,6 +53,7 @@ import {
   sameRootIdentity,
 } from './disk-agreement.js';
 import type { BranchIntent, BranchIntents, ConflictKind, ConflictRecord, DiskCounterpart } from './branch-intent.js';
+import { storedBytes, type StoredBytes } from './sync.js';
 import {
   IGNORED,
   type MountView,
@@ -1386,12 +1387,12 @@ export async function pushPaths(
         continue;
       }
       const priorP = p && (p.kind === 'absent' || p.kind === 'content') ? p : null;
-      const blob = store.getBlob(s.hash);
+      const stored = storedBytes(store, s.hash);
       const forcedMode = mount.modesObserved && s.mode !== undefined && s.mode !== (d.kind === 'file' ? d.mode : undefined) ? s.mode : undefined;
       const agreedMode = priorP?.kind === 'content' ? priorP.mode : undefined;
-      if (blob) {
+      if ('bytes' in stored) {
         plans.push({
-          path, kind: 'write', blob, hash: s.hash, prior: priorP, before: agreement.get(mount.name, path), expect,
+          path, kind: 'write', blob: stored.bytes, hash: s.hash, prior: priorP, before: agreement.get(mount.name, path), expect,
           ...(forcedMode !== undefined ? { apply: forcedMode } : {}),
           ...(forcedMode !== undefined && d.kind === 'file' && agreedMode !== d.mode ? { overDisk: true as const } : {}),
           ...(s.mode !== undefined ? { storeMode: s.mode } : {}),
@@ -1400,7 +1401,7 @@ export async function pushPaths(
           ...(agreedMode !== undefined ? { priorMode: agreedMode } : {}),
           ...(d.kind === 'file' && d.mode !== undefined ? { seenMode: d.mode } : {}),
         });
-      } else result.skipped.push({ path, reason: 'the workspace copy is missing from the store' });
+      } else result.skipped.push({ path, reason: unstoredCopy(stored) });
       continue;
     }
     if (v.mode === 'adopt' && s && d.kind === 'file') {
@@ -1551,11 +1552,12 @@ export async function pushPaths(
       plans.push({ path, kind: 'unlink', prior, before: agreement.get(mount.name, path), expect });
       continue;
     }
-    const blob = store.getBlob(s.hash);
-    if (!blob) {
-      result.skipped.push({ path, reason: 'the workspace copy is missing from the store' });
+    const stored = storedBytes(store, s.hash);
+    if (!('bytes' in stored)) {
+      result.skipped.push({ path, reason: unstoredCopy(stored) });
       continue;
     }
+    const blob = stored.bytes;
     // The store's mode goes to disk with its bytes where the file is new,
     // the push is forced, or disk still has the mode they last agreed on. A
     // push that completes our own unconfirmed one, whose bytes landed, is a
@@ -1774,3 +1776,7 @@ export async function pushPaths(
   agreement.maybeCheckpoint();
   return result;
 }
+
+function unstoredCopy(stored: Exclude<StoredBytes, { bytes: Buffer }>): string {
+  return 'missing' in stored ? 'the workspace copy is missing from the store' : `the workspace copy is unreadable in the store (${stored.unreadable})`;
+}

src/modules/workspace/sync.ts

@@ -6,6 +6,7 @@
  */
 
 import { createHash } from 'node:crypto';
+import type { JsStore } from '@animalabs/chronicle';
 
 export const DEFAULT_MAX_FILE_SIZE = 5 * 1024 * 1024; // 5MB
 
@@ -18,3 +19,18 @@ export function hashContent(content: string | Buffer): string {
   hash.update(content);
   return hash.digest('hex');
 }
+
+export type StoredBytes = { bytes: Buffer } | { missing: true } | { unreadable: string };
+
+export function storedBytes(store: JsStore, hash: string): StoredBytes {
+  try {
+    const blob = store.getBlob(hash);
+    return blob ? { bytes: blob } : { missing: true };
+  } catch (error) {
+    if (store.isClosed()) throw error;
+    // Chronicle's getBlob throws, rather than answering null, for a blob file
+    // it can't read whole or whose format, checksum or hash doesn't check out,
+    // such as one a write cut short left truncated.
+    return { unreadable: error instanceof Error ? error.message : String(error) };
+  }
+}

test/workspace-edits.test.ts

@@ -978,6 +978,7 @@ test("edit_many refuses a file whose stored copy is missing from the store, and
   const { store, workspace, call, stored, file } = setup(t);
   await file('a.txt', 'alpha\n');
   await file('b.txt', 'beta\n');
+  assert.equal((await call('write', { path: 'work/b.txt', content: 'beta, a draft\n' })).success, true);
   const missing = store.treeGet(treeStateOf(workspace), 'b.txt')!.blobHash;
   const s = store as unknown as Record<string, (...args: unknown[]) => unknown>;
   const getBlob = store.getBlob;
@@ -988,7 +989,7 @@ test("edit_many refuses a file whose stored copy is missing from the store, and
         { path: 'work/a.txt', oldString: 'alpha', newString: 'ALPHA' },
         { path: 'work/b.txt', oldString: 'beta', newString: 'BETA' },
       ],
-    })), 'The edit_many call stored none of its edits: 1 problem.\n\nedit 2: Blob not found for: work/b.txt');
+    })), "The edit_many call stored none of its edits: 1 problem.\n\nedit 2: The workspace's copy of work/b.txt is missing from the store, and disk's copy holds other contents, so it can't be restored, and edit 2 changed nothing. Sync the path to keep disk's copy and give up the workspace's, or write the file again.");
   } finally {
     delete s.getBlob;
   }