feat(memorypack): tarball-aware appends (closes 0.4/0.6 directory-only limits)

packages/memorypack/CHANGELOG.md

@@ -2,6 +2,45 @@
 
 All notable changes to `@clude/memorypack` are documented here. The package follows [Semantic Versioning](https://semver.org/).
 
+## [0.7.0] — 2026-04-29
+
+`appendRevocations` and `appendRevocationAnchors` now accept `.tar.zst` paths transparently. Operators with tarball packs no longer have to extract / append / re-tarball by hand.
+
+### Added
+
+- Tarball-aware code path in both append functions: when the input path is a tarball, the function extracts to a temp directory, runs the directory-mode append against the inner pack, repacks atomically, and cleans up.
+- Atomic re-tarball: writes to `<original>.new-<pid>-<ts>` then renames into place. A failed extract / append / repack leaves the **original tarball untouched** — your audit trail never enters a half-written state.
+
+### Behaviour
+
+- Empty `revocations` / `anchors` input is a no-op (early return) — does NOT touch the tarball file. Mtime + bytes preserved.
+- Tarballs that decompress to multiple top-level directories are rejected (matches the reader's contract).
+- Tarballs whose extension is `.tar.zst` are routed through the tarball path. Other file paths are also routed through (in case someone renames a pack), then handled by the tar binary; truly malformed inputs fail at extraction.
+- Directory-mode behaviour is unchanged — covered by an explicit regression test.
+
+### Concurrency
+
+Concurrent appends to the same tarball are NOT safe — last writer wins. Same constraint as concurrent writes to a directory pack; callers needing multi-process coordination must layer their own locking. Single-producer flows (the common case) are unaffected.
+
+### Tests
+
+9 new tests, 92 total in this package, all green:
+
+- Append revocations to a tarball → read-back via `readMemoryPack` confirms presence
+- Append revocation anchors to a tarball → read-back confirms paired anchor
+- `streamMemoryPack` surfaces the appended anchor on tarballs too
+- Multiple appends stack across calls
+- Empty input is a no-op (mtime + bytes unchanged on the tarball file)
+- Successful append leaves no orphan staging files in the workdir
+- Tarball with multiple top-level dirs is rejected
+- Directory-mode regression guard
+
+### Limitations (deferred to v0.8)
+
+- @solana/web3.js test mocks for `verifyChainAnchors` / `verifyRevocationAnchors`.
+- Backdating-detection (`maxClockSkew`) on revocation anchors.
+- Symbol.asyncDispose for the streaming reader.
+
 ## [0.6.0] — 2026-04-29
 
 Chain-anchored revocations. Pin the `revoked_at` of a soft-deleted record to a Solana transaction so a producer can't backdate a deletion claim.

packages/memorypack/README.md

@@ -211,6 +211,6 @@ Post-v0.2 (tracked in the [main repo](https://github.com/sebbsssss/clude)):
 - Production IPFS / Arweave content anchoring
 - Multi-chain anchors (Ethereum L2, Bitcoin OP_RETURN)
 - True streaming through tar (today the reader extracts to a temp dir first)
-- Tarball-aware `appendRevocations` / `appendRevocationAnchors` (today both are directory-only)
 - @solana/web3.js test mocks for `verifyChainAnchors` / `verifyRevocationAnchors`
 - Backdating detection (compare on-chain block timestamp to signed `revoked_at`)
+- Symbol.asyncDispose for the streaming reader (clean tarball temp dirs on early break)

packages/memorypack/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@clude/memorypack",
-  "version": "0.6.0",
+  "version": "0.7.0",
   "description": "Reference reader/writer for the MemoryPack spec \u2014 open, signed, chain-anchorable file format for portable AI agent memory.",
   "license": "MIT",
   "homepage": "https://github.com/sebbsssss/clude/blob/main/docs/memorypack.md",

packages/memorypack/src/__tests__/revocation-anchors.test.ts

@@ -132,12 +132,12 @@ describe('appendRevocationAnchors', () => {
     expect(JSON.parse(lines[1]).tx).toBe('t2');
   });
 
-  it('throws on tarball pack', () => {
+  it('throws when tarball file does not exist', () => {
     expect(() =>
-      appendRevocationAnchors(join(dir, 'pack.tar.zst'), [{
+      appendRevocationAnchors(join(dir, 'nonexistent-pack.tar.zst'), [{
         record_hash: 'sha256:abc', revoked_at: '2026-04-29T00:00:00Z', chain: 'solana-mainnet', tx: 't',
       }]),
-    ).toThrow(/tarball/i);
+    ).toThrow(/not found/i);
   });
 
   it('throws when manifest.json missing', () => {

packages/memorypack/src/__tests__/revocations.test.ts

@@ -170,14 +170,14 @@ describe('appendRevocations', () => {
     expect(readFileSync(join(dir, 'manifest.json')).equals(manifestBytes)).toBe(true);
   });
 
-  it('throws on tarball pack', () => {
+  it('throws when tarball file does not exist', () => {
     expect(() =>
       appendRevocations(
-        join(dir, 'pack.tar.zst'),
+        join(dir, 'nonexistent-pack.tar.zst'),
         [{ record_hash: 'sha256:abc' }],
         { secretKey: new Uint8Array(64), publicKey: 'fake' },
       ),
-    ).toThrow(/tarball/i);
+    ).toThrow(/not found/i);
   });
 
   it('throws when packDir is missing manifest.json', () => {

packages/memorypack/src/__tests__/tarball-appends.test.ts

@@ -0,0 +1,276 @@
+// Tarball-aware appends — appendRevocations + appendRevocationAnchors
+// now operate on .tar.zst files transparently.
+//
+// Coverage:
+//   - Append revocations to a tarball, read back, verify presence
+//   - Append revocation anchors to a tarball, read back
+//   - Atomic-rename safety: a successful append leaves no orphan
+//     `<tarball>.new-...` siblings
+//   - Tarball with multiple top-level dirs is rejected
+//   - Empty input early-returns without re-tarballing
+//     (tarball mtime / bytes unchanged)
+//   - Append + verify round-trip on tarballs uses streamMemoryPack too
+
+import { afterEach, beforeEach, describe, expect, it } from 'vitest';
+import {
+  existsSync,
+  mkdtempSync,
+  readFileSync,
+  readdirSync,
+  rmSync,
+  statSync,
+  writeFileSync,
+} from 'fs';
+import { tmpdir } from 'os';
+import { join } from 'path';
+import { spawnSync } from 'child_process';
+import nacl from 'tweetnacl';
+// @ts-ignore — bs58 is ESM-only, works at runtime via Node CJS/ESM interop
+import * as bs58Module from 'bs58';
+const bs58: { encode: (b: Uint8Array) => string; decode: (s: string) => Uint8Array } =
+  (bs58Module as any).default || bs58Module;
+import {
+  appendRevocationAnchors,
+  appendRevocations,
+  hashRecordLine,
+  readMemoryPack,
+  streamMemoryPack,
+  writeMemoryPack,
+} from '../index.js';
+import { FIXTURE_RECORDS, FIXTURE_CLOCK } from './fixtures.js';
+
+let workdir: string;
+beforeEach(() => { workdir = mkdtempSync(join(tmpdir(), 'mp-tar-append-')); });
+afterEach(() => { rmSync(workdir, { recursive: true, force: true }); });
+
+function buildSignedTarball(target: string) {
+  const kp = nacl.sign.keyPair();
+  writeMemoryPack(target, FIXTURE_RECORDS, {
+    producer: { name: 'clude', version: '0.7.0', public_key: bs58.encode(kp.publicKey) },
+    record_schema: 'clude-memory-v3',
+    secretKey: kp.secretKey,
+    clock: FIXTURE_CLOCK,
+    format: 'tarball',
+  });
+  return kp;
+}
+
+function readPackHash(target: string): string {
+  // Pull the canonical hash of FIXTURE_RECORDS[0] from the tarball.
+  // Extract to a temp dir, hash records.jsonl line 1, clean up.
+  const tmp = mkdtempSync(join(tmpdir(), 'mp-hash-'));
+  try {
+    spawnSync('tar', ['--zstd', '-xf', target, '-C', tmp]);
+    const inner = join(tmp, readdirSync(tmp)[0]);
+    const lines = readFileSync(join(inner, 'records.jsonl'), 'utf-8')
+      .split('\n')
+      .filter((l) => l.length > 0);
+    return hashRecordLine(lines[0]);
+  } finally {
+    rmSync(tmp, { recursive: true, force: true });
+  }
+}
+
+// ────────────────────────────────────────────────────────────────────
+// Revocations on tarballs
+// ────────────────────────────────────────────────────────────────────
+
+describe('appendRevocations on tarball packs', () => {
+  it('round-trips: append → read → revocation present', () => {
+    const tarball = join(workdir, 'pack.tar.zst');
+    const kp = buildSignedTarball(tarball);
+    const hash = readPackHash(tarball);
+
+    const written = appendRevocations(
+      tarball,
+      [{ record_hash: hash, reason: 'gdpr-tarball' }],
+      {
+        secretKey: kp.secretKey,
+        publicKey: bs58.encode(kp.publicKey),
+        clock: () => '2026-04-29T12:00:00.000Z',
+      },
+    );
+    expect(written).toHaveLength(1);
+
+    // Re-read the tarball and confirm the revocation made it in.
+    const result = readMemoryPack(tarball);
+    expect(result.revocations).toHaveLength(1);
+    expect(result.revocations[0].reason).toBe('gdpr-tarball');
+    expect(result.revokedRecordHashes.has(hash)).toBe(true);
+  });
+
+  it('successful append leaves no orphan staging files in the workdir', () => {
+    const tarball = join(workdir, 'pack.tar.zst');
+    const kp = buildSignedTarball(tarball);
+    const hash = readPackHash(tarball);
+
+    appendRevocations(tarball, [{ record_hash: hash }], {
+      secretKey: kp.secretKey,
+      publicKey: bs58.encode(kp.publicKey),
+    });
+
+    const siblings = readdirSync(workdir);
+    expect(siblings).toEqual(['pack.tar.zst']);
+  });
+
+  it('multiple appends to the same tarball stack', () => {
+    const tarball = join(workdir, 'pack.tar.zst');
+    const kp = buildSignedTarball(tarball);
+    const hash = readPackHash(tarball);
+    const opts = {
+      secretKey: kp.secretKey,
+      publicKey: bs58.encode(kp.publicKey),
+      clock: (() => {
+        let i = 0;
+        return () => `2026-04-29T12:00:0${i++}.000Z`;
+      })(),
+    };
+
+    appendRevocations(tarball, [{ record_hash: hash, reason: 'first' }], opts);
+    appendRevocations(tarball, [{ record_hash: hash, reason: 'second' }], opts);
+
+    const result = readMemoryPack(tarball);
+    expect(result.revocations).toHaveLength(2);
+    expect(result.revocations.map((r) => r.reason)).toEqual(['first', 'second']);
+  });
+
+  it('empty input is a no-op — does not re-tarball', () => {
+    const tarball = join(workdir, 'pack.tar.zst');
+    buildSignedTarball(tarball);
+    const before = readFileSync(tarball);
+
+    const written = appendRevocations(tarball, [], {
+      secretKey: new Uint8Array(64),
+      publicKey: 'fake',
+    });
+    expect(written).toHaveLength(0);
+
+    const after = readFileSync(tarball);
+    expect(after.equals(before)).toBe(true);
+  });
+
+  it('rejects tarball with multiple top-level dirs', () => {
+    // Hand-build a malformed tarball with two top-level directories.
+    const stagingDir = mkdtempSync(join(tmpdir(), 'mp-bad-tar-'));
+    try {
+      const a = join(stagingDir, 'pack-a');
+      const b = join(stagingDir, 'pack-b');
+      writeMemoryPack(a, FIXTURE_RECORDS, {
+        producer: { name: 'clude', version: '0.7.0' },
+        record_schema: 'clude-memory-v3',
+      });
+      writeMemoryPack(b, FIXTURE_RECORDS, {
+        producer: { name: 'clude', version: '0.7.0' },
+        record_schema: 'clude-memory-v3',
+      });
+      const target = join(workdir, 'malformed.tar.zst');
+      const r = spawnSync('tar', ['--zstd', '-cf', target, '-C', stagingDir, 'pack-a', 'pack-b']);
+      expect(r.status).toBe(0);
+
+      expect(() =>
+        appendRevocations(target, [{ record_hash: 'sha256:abc' }], {
+          secretKey: new Uint8Array(64),
+          publicKey: 'fake',
+        }),
+      ).toThrow(/single top-level dir/);
+    } finally {
+      rmSync(stagingDir, { recursive: true, force: true });
+    }
+  });
+});
+
+// ────────────────────────────────────────────────────────────────────
+// Revocation anchors on tarballs
+// ────────────────────────────────────────────────────────────────────
+
+describe('appendRevocationAnchors on tarball packs', () => {
+  it('round-trips: append revocation + anchor → read → both present', () => {
+    const tarball = join(workdir, 'pack.tar.zst');
+    const kp = buildSignedTarball(tarball);
+    const hash = readPackHash(tarball);
+
+    appendRevocations(tarball, [{ record_hash: hash }], {
+      secretKey: kp.secretKey,
+      publicKey: bs58.encode(kp.publicKey),
+      clock: () => '2026-04-29T12:00:00.000Z',
+    });
+    const written = appendRevocationAnchors(tarball, [{
+      record_hash: hash,
+      revoked_at: '2026-04-29T12:00:00.000Z',
+      chain: 'solana-mainnet',
+      tx: 'tarball-tx',
+    }]);
+    expect(written).toHaveLength(1);
+
+    const result = readMemoryPack(tarball);
+    expect(result.revocations).toHaveLength(1);
+    expect(result.revocationAnchors).toHaveLength(1);
+    expect(result.revocationAnchors[0].tx).toBe('tarball-tx');
+  });
+
+  it('streamMemoryPack also surfaces the appended anchor', async () => {
+    const tarball = join(workdir, 'pack.tar.zst');
+    const kp = buildSignedTarball(tarball);
+    const hash = readPackHash(tarball);
+
+    appendRevocations(tarball, [{ record_hash: hash }], {
+      secretKey: kp.secretKey,
+      publicKey: bs58.encode(kp.publicKey),
+      clock: () => '2026-04-29T13:00:00.000Z',
+    });
+    appendRevocationAnchors(tarball, [{
+      record_hash: hash,
+      revoked_at: '2026-04-29T13:00:00.000Z',
+      chain: 'solana-mainnet',
+      tx: 'streamed-tarball-tx',
+    }]);
+
+    const { revocationAnchors, records } = await streamMemoryPack(tarball);
+    expect(revocationAnchors).toHaveLength(1);
+    expect(revocationAnchors[0].tx).toBe('streamed-tarball-tx');
+
+    let count = 0;
+    for await (const _ of records) count++;
+    expect(count).toBe(FIXTURE_RECORDS.length);
+  });
+
+  it('empty input is a no-op — does not re-tarball', () => {
+    const tarball = join(workdir, 'pack.tar.zst');
+    buildSignedTarball(tarball);
+    const before = readFileSync(tarball);
+
+    const written = appendRevocationAnchors(tarball, []);
+    expect(written).toHaveLength(0);
+
+    const after = readFileSync(tarball);
+    expect(after.equals(before)).toBe(true);
+  });
+});
+
+// ────────────────────────────────────────────────────────────────────
+// Sanity: directory packs still work (regression guard)
+// ────────────────────────────────────────────────────────────────────
+
+describe('directory-mode appends still work after refactor', () => {
+  it('appendRevocations on directory unchanged', () => {
+    const dir = join(workdir, 'pack');
+    const kp = nacl.sign.keyPair();
+    writeMemoryPack(dir, FIXTURE_RECORDS, {
+      producer: { name: 'clude', version: '0.7.0', public_key: bs58.encode(kp.publicKey) },
+      record_schema: 'clude-memory-v3',
+      secretKey: kp.secretKey,
+      clock: FIXTURE_CLOCK,
+    });
+    const lines = readFileSync(join(dir, 'records.jsonl'), 'utf-8')
+      .split('\n').filter((l) => l.length > 0);
+    const hash = hashRecordLine(lines[0]);
+
+    appendRevocations(dir, [{ record_hash: hash, reason: 'directory-mode' }], {
+      secretKey: kp.secretKey,
+      publicKey: bs58.encode(kp.publicKey),
+    });
+    const result = readMemoryPack(dir);
+    expect(result.revocations).toHaveLength(1);
+    expect(result.revocations[0].reason).toBe('directory-mode');
+  });
+});