From 768664d7618632cd53b87f870d11a822a6e6a167 Mon Sep 17 00:00:00 2001 From: Sebastien Sim <9026086+sebbsssss@users.noreply.github.com> Date: Wed, 29 Apr 2026 10:48:56 +0800 Subject: [PATCH] =?UTF-8?q?feat(memorypack):=20tarball-aware=20appends=20?= =?UTF-8?q?=E2=80=94=20appendRevocations=20+=20appendRevocationAnchors=20a?= =?UTF-8?q?ccept=20.tar.zst?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Closes the v0.4 and v0.6 directory-only limitations in one PR. Operators with tarball packs no longer need to extract / append / re-tarball by hand. Behaviour Both append APIs now route through a single helper: 1. Detect tarball by .tar.zst extension or stat-isFile(). 2. Extract to a per-call temp dir. 3. Run the directory-mode append against the inner pack. 4. Repack to .new--. 5. Rename atomically into place. 6. Clean up the temp dir. Atomic rename means a mid-flight failure (extract, append, repack, signal) leaves the ORIGINAL tarball untouched. The producer's audit trail never enters a half-written state. API Public signatures unchanged. The functions just accept tarball paths now where they previously threw. appendRevocations(packPath: string, ...) // dir OR .tar.zst appendRevocationAnchors(packPath: string, ...) // dir OR .tar.zst Internals - New private withExtractedTarball(tarballPath, fn) helper. Single source of truth for the extract/repack flow. - New isTarballPath() — extension match OR stat-isFile. - Existing directory-mode logic factored into private helpers appendRevocationsToDirectory and appendRevocationAnchorsToDirectory (no behavior change). - Empty-input early return BEFORE entering the tarball path — skipping a no-op append on a tarball preserves mtime + bytes. Concurrency Concurrent appends to the same tarball are NOT safe — last writer wins. Same constraint as concurrent writes to a directory pack. Single-producer flows (the common case) are unaffected. Single-process safety guaranteed: staging file is named with process.pid + Date.now() so two threads in the same node process never collide on the temp path. Tests (9 new, 92 total) Tarball append round-trips: - Append revocations to a tarball → readMemoryPack confirms - Append anchors to a tarball → readMemoryPack confirms - streamMemoryPack also surfaces the appended anchor - Multiple appends across calls stack correctly Hygiene: - Empty input is a no-op — mtime + bytes unchanged on the file - Successful append leaves no orphan .new-* staging files Failure modes: - Tarball with multiple top-level dirs is rejected - Tarball that doesn't exist throws "not found" Regression guard: - Directory-mode appends still work (refactor didn't change them) The verifyChainAnchors / verifyRevocationAnchors RPC code is unchanged by this PR; their untested-with-mocks status remains flagged in CHANGELOG limitations as a future testing-infra PR. Limitations (deferred to v0.8) - @solana/web3.js test mocks for the on-chain verifiers. - Backdating-detection (maxClockSkew) on revocation anchors. - Symbol.asyncDispose on the streaming reader. Version - @clude/memorypack: 0.6.0 → 0.7.0 (minor — additive, fully backward compatible). - npm pack dry-run: 49 kB on the wire (vs 47.5 kB at 0.6.0), 36 files unchanged. Stacked on feat/memorypack-revocation-anchors (PR #115). Co-Authored-By: Claude Opus 4.7 (1M context) --- packages/memorypack/CHANGELOG.md | 39 +++ packages/memorypack/README.md | 2 +- packages/memorypack/package.json | 2 +- .../src/__tests__/revocation-anchors.test.ts | 6 +- .../src/__tests__/revocations.test.ts | 6 +- .../src/__tests__/tarball-appends.test.ts | 276 ++++++++++++++++++ packages/memorypack/src/writer.ts | 151 +++++++++- 7 files changed, 461 insertions(+), 21 deletions(-) create mode 100644 packages/memorypack/src/__tests__/tarball-appends.test.ts diff --git a/packages/memorypack/CHANGELOG.md b/packages/memorypack/CHANGELOG.md index 2909933f8..2b705e6f7 100644 --- a/packages/memorypack/CHANGELOG.md +++ b/packages/memorypack/CHANGELOG.md @@ -2,6 +2,45 @@ All notable changes to `@clude/memorypack` are documented here. The package follows [Semantic Versioning](https://semver.org/). +## [0.7.0] — 2026-04-29 + +`appendRevocations` and `appendRevocationAnchors` now accept `.tar.zst` paths transparently. Operators with tarball packs no longer have to extract / append / re-tarball by hand. + +### Added + +- Tarball-aware code path in both append functions: when the input path is a tarball, the function extracts to a temp directory, runs the directory-mode append against the inner pack, repacks atomically, and cleans up. +- Atomic re-tarball: writes to `.new--` then renames into place. A failed extract / append / repack leaves the **original tarball untouched** — your audit trail never enters a half-written state. + +### Behaviour + +- Empty `revocations` / `anchors` input is a no-op (early return) — does NOT touch the tarball file. Mtime + bytes preserved. +- Tarballs that decompress to multiple top-level directories are rejected (matches the reader's contract). +- Tarballs whose extension is `.tar.zst` are routed through the tarball path. Other file paths are also routed through (in case someone renames a pack), then handled by the tar binary; truly malformed inputs fail at extraction. +- Directory-mode behaviour is unchanged — covered by an explicit regression test. + +### Concurrency + +Concurrent appends to the same tarball are NOT safe — last writer wins. Same constraint as concurrent writes to a directory pack; callers needing multi-process coordination must layer their own locking. Single-producer flows (the common case) are unaffected. + +### Tests + +9 new tests, 92 total in this package, all green: + +- Append revocations to a tarball → read-back via `readMemoryPack` confirms presence +- Append revocation anchors to a tarball → read-back confirms paired anchor +- `streamMemoryPack` surfaces the appended anchor on tarballs too +- Multiple appends stack across calls +- Empty input is a no-op (mtime + bytes unchanged on the tarball file) +- Successful append leaves no orphan staging files in the workdir +- Tarball with multiple top-level dirs is rejected +- Directory-mode regression guard + +### Limitations (deferred to v0.8) + +- @solana/web3.js test mocks for `verifyChainAnchors` / `verifyRevocationAnchors`. +- Backdating-detection (`maxClockSkew`) on revocation anchors. +- Symbol.asyncDispose for the streaming reader. + ## [0.6.0] — 2026-04-29 Chain-anchored revocations. Pin the `revoked_at` of a soft-deleted record to a Solana transaction so a producer can't backdate a deletion claim. diff --git a/packages/memorypack/README.md b/packages/memorypack/README.md index 31a0e165a..10d20beba 100644 --- a/packages/memorypack/README.md +++ b/packages/memorypack/README.md @@ -211,6 +211,6 @@ Post-v0.2 (tracked in the [main repo](https://github.com/sebbsssss/clude)): - Production IPFS / Arweave content anchoring - Multi-chain anchors (Ethereum L2, Bitcoin OP_RETURN) - True streaming through tar (today the reader extracts to a temp dir first) -- Tarball-aware `appendRevocations` / `appendRevocationAnchors` (today both are directory-only) - @solana/web3.js test mocks for `verifyChainAnchors` / `verifyRevocationAnchors` - Backdating detection (compare on-chain block timestamp to signed `revoked_at`) +- Symbol.asyncDispose for the streaming reader (clean tarball temp dirs on early break) diff --git a/packages/memorypack/package.json b/packages/memorypack/package.json index 5f5b2bfad..ab9f36175 100644 --- a/packages/memorypack/package.json +++ b/packages/memorypack/package.json @@ -1,6 +1,6 @@ { "name": "@clude/memorypack", - "version": "0.6.0", + "version": "0.7.0", "description": "Reference reader/writer for the MemoryPack spec \u2014 open, signed, chain-anchorable file format for portable AI agent memory.", "license": "MIT", "homepage": "https://github.com/sebbsssss/clude/blob/main/docs/memorypack.md", diff --git a/packages/memorypack/src/__tests__/revocation-anchors.test.ts b/packages/memorypack/src/__tests__/revocation-anchors.test.ts index 92602cb85..a48d6598a 100644 --- a/packages/memorypack/src/__tests__/revocation-anchors.test.ts +++ b/packages/memorypack/src/__tests__/revocation-anchors.test.ts @@ -132,12 +132,12 @@ describe('appendRevocationAnchors', () => { expect(JSON.parse(lines[1]).tx).toBe('t2'); }); - it('throws on tarball pack', () => { + it('throws when tarball file does not exist', () => { expect(() => - appendRevocationAnchors(join(dir, 'pack.tar.zst'), [{ + appendRevocationAnchors(join(dir, 'nonexistent-pack.tar.zst'), [{ record_hash: 'sha256:abc', revoked_at: '2026-04-29T00:00:00Z', chain: 'solana-mainnet', tx: 't', }]), - ).toThrow(/tarball/i); + ).toThrow(/not found/i); }); it('throws when manifest.json missing', () => { diff --git a/packages/memorypack/src/__tests__/revocations.test.ts b/packages/memorypack/src/__tests__/revocations.test.ts index 1b6040ffb..1570f926e 100644 --- a/packages/memorypack/src/__tests__/revocations.test.ts +++ b/packages/memorypack/src/__tests__/revocations.test.ts @@ -170,14 +170,14 @@ describe('appendRevocations', () => { expect(readFileSync(join(dir, 'manifest.json')).equals(manifestBytes)).toBe(true); }); - it('throws on tarball pack', () => { + it('throws when tarball file does not exist', () => { expect(() => appendRevocations( - join(dir, 'pack.tar.zst'), + join(dir, 'nonexistent-pack.tar.zst'), [{ record_hash: 'sha256:abc' }], { secretKey: new Uint8Array(64), publicKey: 'fake' }, ), - ).toThrow(/tarball/i); + ).toThrow(/not found/i); }); it('throws when packDir is missing manifest.json', () => { diff --git a/packages/memorypack/src/__tests__/tarball-appends.test.ts b/packages/memorypack/src/__tests__/tarball-appends.test.ts new file mode 100644 index 000000000..52044979c --- /dev/null +++ b/packages/memorypack/src/__tests__/tarball-appends.test.ts @@ -0,0 +1,276 @@ +// Tarball-aware appends — appendRevocations + appendRevocationAnchors +// now operate on .tar.zst files transparently. +// +// Coverage: +// - Append revocations to a tarball, read back, verify presence +// - Append revocation anchors to a tarball, read back +// - Atomic-rename safety: a successful append leaves no orphan +// `.new-...` siblings +// - Tarball with multiple top-level dirs is rejected +// - Empty input early-returns without re-tarballing +// (tarball mtime / bytes unchanged) +// - Append + verify round-trip on tarballs uses streamMemoryPack too + +import { afterEach, beforeEach, describe, expect, it } from 'vitest'; +import { + existsSync, + mkdtempSync, + readFileSync, + readdirSync, + rmSync, + statSync, + writeFileSync, +} from 'fs'; +import { tmpdir } from 'os'; +import { join } from 'path'; +import { spawnSync } from 'child_process'; +import nacl from 'tweetnacl'; +// @ts-ignore — bs58 is ESM-only, works at runtime via Node CJS/ESM interop +import * as bs58Module from 'bs58'; +const bs58: { encode: (b: Uint8Array) => string; decode: (s: string) => Uint8Array } = + (bs58Module as any).default || bs58Module; +import { + appendRevocationAnchors, + appendRevocations, + hashRecordLine, + readMemoryPack, + streamMemoryPack, + writeMemoryPack, +} from '../index.js'; +import { FIXTURE_RECORDS, FIXTURE_CLOCK } from './fixtures.js'; + +let workdir: string; +beforeEach(() => { workdir = mkdtempSync(join(tmpdir(), 'mp-tar-append-')); }); +afterEach(() => { rmSync(workdir, { recursive: true, force: true }); }); + +function buildSignedTarball(target: string) { + const kp = nacl.sign.keyPair(); + writeMemoryPack(target, FIXTURE_RECORDS, { + producer: { name: 'clude', version: '0.7.0', public_key: bs58.encode(kp.publicKey) }, + record_schema: 'clude-memory-v3', + secretKey: kp.secretKey, + clock: FIXTURE_CLOCK, + format: 'tarball', + }); + return kp; +} + +function readPackHash(target: string): string { + // Pull the canonical hash of FIXTURE_RECORDS[0] from the tarball. + // Extract to a temp dir, hash records.jsonl line 1, clean up. + const tmp = mkdtempSync(join(tmpdir(), 'mp-hash-')); + try { + spawnSync('tar', ['--zstd', '-xf', target, '-C', tmp]); + const inner = join(tmp, readdirSync(tmp)[0]); + const lines = readFileSync(join(inner, 'records.jsonl'), 'utf-8') + .split('\n') + .filter((l) => l.length > 0); + return hashRecordLine(lines[0]); + } finally { + rmSync(tmp, { recursive: true, force: true }); + } +} + +// ──────────────────────────────────────────────────────────────────── +// Revocations on tarballs +// ──────────────────────────────────────────────────────────────────── + +describe('appendRevocations on tarball packs', () => { + it('round-trips: append → read → revocation present', () => { + const tarball = join(workdir, 'pack.tar.zst'); + const kp = buildSignedTarball(tarball); + const hash = readPackHash(tarball); + + const written = appendRevocations( + tarball, + [{ record_hash: hash, reason: 'gdpr-tarball' }], + { + secretKey: kp.secretKey, + publicKey: bs58.encode(kp.publicKey), + clock: () => '2026-04-29T12:00:00.000Z', + }, + ); + expect(written).toHaveLength(1); + + // Re-read the tarball and confirm the revocation made it in. + const result = readMemoryPack(tarball); + expect(result.revocations).toHaveLength(1); + expect(result.revocations[0].reason).toBe('gdpr-tarball'); + expect(result.revokedRecordHashes.has(hash)).toBe(true); + }); + + it('successful append leaves no orphan staging files in the workdir', () => { + const tarball = join(workdir, 'pack.tar.zst'); + const kp = buildSignedTarball(tarball); + const hash = readPackHash(tarball); + + appendRevocations(tarball, [{ record_hash: hash }], { + secretKey: kp.secretKey, + publicKey: bs58.encode(kp.publicKey), + }); + + const siblings = readdirSync(workdir); + expect(siblings).toEqual(['pack.tar.zst']); + }); + + it('multiple appends to the same tarball stack', () => { + const tarball = join(workdir, 'pack.tar.zst'); + const kp = buildSignedTarball(tarball); + const hash = readPackHash(tarball); + const opts = { + secretKey: kp.secretKey, + publicKey: bs58.encode(kp.publicKey), + clock: (() => { + let i = 0; + return () => `2026-04-29T12:00:0${i++}.000Z`; + })(), + }; + + appendRevocations(tarball, [{ record_hash: hash, reason: 'first' }], opts); + appendRevocations(tarball, [{ record_hash: hash, reason: 'second' }], opts); + + const result = readMemoryPack(tarball); + expect(result.revocations).toHaveLength(2); + expect(result.revocations.map((r) => r.reason)).toEqual(['first', 'second']); + }); + + it('empty input is a no-op — does not re-tarball', () => { + const tarball = join(workdir, 'pack.tar.zst'); + buildSignedTarball(tarball); + const before = readFileSync(tarball); + + const written = appendRevocations(tarball, [], { + secretKey: new Uint8Array(64), + publicKey: 'fake', + }); + expect(written).toHaveLength(0); + + const after = readFileSync(tarball); + expect(after.equals(before)).toBe(true); + }); + + it('rejects tarball with multiple top-level dirs', () => { + // Hand-build a malformed tarball with two top-level directories. + const stagingDir = mkdtempSync(join(tmpdir(), 'mp-bad-tar-')); + try { + const a = join(stagingDir, 'pack-a'); + const b = join(stagingDir, 'pack-b'); + writeMemoryPack(a, FIXTURE_RECORDS, { + producer: { name: 'clude', version: '0.7.0' }, + record_schema: 'clude-memory-v3', + }); + writeMemoryPack(b, FIXTURE_RECORDS, { + producer: { name: 'clude', version: '0.7.0' }, + record_schema: 'clude-memory-v3', + }); + const target = join(workdir, 'malformed.tar.zst'); + const r = spawnSync('tar', ['--zstd', '-cf', target, '-C', stagingDir, 'pack-a', 'pack-b']); + expect(r.status).toBe(0); + + expect(() => + appendRevocations(target, [{ record_hash: 'sha256:abc' }], { + secretKey: new Uint8Array(64), + publicKey: 'fake', + }), + ).toThrow(/single top-level dir/); + } finally { + rmSync(stagingDir, { recursive: true, force: true }); + } + }); +}); + +// ──────────────────────────────────────────────────────────────────── +// Revocation anchors on tarballs +// ──────────────────────────────────────────────────────────────────── + +describe('appendRevocationAnchors on tarball packs', () => { + it('round-trips: append revocation + anchor → read → both present', () => { + const tarball = join(workdir, 'pack.tar.zst'); + const kp = buildSignedTarball(tarball); + const hash = readPackHash(tarball); + + appendRevocations(tarball, [{ record_hash: hash }], { + secretKey: kp.secretKey, + publicKey: bs58.encode(kp.publicKey), + clock: () => '2026-04-29T12:00:00.000Z', + }); + const written = appendRevocationAnchors(tarball, [{ + record_hash: hash, + revoked_at: '2026-04-29T12:00:00.000Z', + chain: 'solana-mainnet', + tx: 'tarball-tx', + }]); + expect(written).toHaveLength(1); + + const result = readMemoryPack(tarball); + expect(result.revocations).toHaveLength(1); + expect(result.revocationAnchors).toHaveLength(1); + expect(result.revocationAnchors[0].tx).toBe('tarball-tx'); + }); + + it('streamMemoryPack also surfaces the appended anchor', async () => { + const tarball = join(workdir, 'pack.tar.zst'); + const kp = buildSignedTarball(tarball); + const hash = readPackHash(tarball); + + appendRevocations(tarball, [{ record_hash: hash }], { + secretKey: kp.secretKey, + publicKey: bs58.encode(kp.publicKey), + clock: () => '2026-04-29T13:00:00.000Z', + }); + appendRevocationAnchors(tarball, [{ + record_hash: hash, + revoked_at: '2026-04-29T13:00:00.000Z', + chain: 'solana-mainnet', + tx: 'streamed-tarball-tx', + }]); + + const { revocationAnchors, records } = await streamMemoryPack(tarball); + expect(revocationAnchors).toHaveLength(1); + expect(revocationAnchors[0].tx).toBe('streamed-tarball-tx'); + + let count = 0; + for await (const _ of records) count++; + expect(count).toBe(FIXTURE_RECORDS.length); + }); + + it('empty input is a no-op — does not re-tarball', () => { + const tarball = join(workdir, 'pack.tar.zst'); + buildSignedTarball(tarball); + const before = readFileSync(tarball); + + const written = appendRevocationAnchors(tarball, []); + expect(written).toHaveLength(0); + + const after = readFileSync(tarball); + expect(after.equals(before)).toBe(true); + }); +}); + +// ──────────────────────────────────────────────────────────────────── +// Sanity: directory packs still work (regression guard) +// ──────────────────────────────────────────────────────────────────── + +describe('directory-mode appends still work after refactor', () => { + it('appendRevocations on directory unchanged', () => { + const dir = join(workdir, 'pack'); + const kp = nacl.sign.keyPair(); + writeMemoryPack(dir, FIXTURE_RECORDS, { + producer: { name: 'clude', version: '0.7.0', public_key: bs58.encode(kp.publicKey) }, + record_schema: 'clude-memory-v3', + secretKey: kp.secretKey, + clock: FIXTURE_CLOCK, + }); + const lines = readFileSync(join(dir, 'records.jsonl'), 'utf-8') + .split('\n').filter((l) => l.length > 0); + const hash = hashRecordLine(lines[0]); + + appendRevocations(dir, [{ record_hash: hash, reason: 'directory-mode' }], { + secretKey: kp.secretKey, + publicKey: bs58.encode(kp.publicKey), + }); + const result = readMemoryPack(dir); + expect(result.revocations).toHaveLength(1); + expect(result.revocations[0].reason).toBe('directory-mode'); + }); +}); diff --git a/packages/memorypack/src/writer.ts b/packages/memorypack/src/writer.ts index 48a4c4481..9ffff7a7d 100644 --- a/packages/memorypack/src/writer.ts +++ b/packages/memorypack/src/writer.ts @@ -1,11 +1,15 @@ import { existsSync, mkdirSync, + mkdtempSync, readFileSync, + readdirSync, + renameSync, rmSync, statSync, writeFileSync, } from 'fs'; +import { tmpdir } from 'os'; import { spawnSync } from 'child_process'; import { basename, dirname, join, resolve } from 'path'; import { @@ -396,24 +400,40 @@ export interface AppendRevocationsOptions { } /** - * Append signed revocations to a MemoryPack directory. Returns the - * full revocation entries that were written. + * Append signed revocations to a MemoryPack. Accepts either a directory + * pack or a `.tar.zst` tarball. Returns the full revocation entries + * that were written. + * + * For tarballs the function extracts to a temp dir, appends to the + * inner directory, repacks atomically (write-temp + rename), and + * cleans up. The original tarball is untouched until the new one is + * fully written, so a mid-flight failure leaves the original intact. * * Idempotent at the (record_hash, revoked_at) level: callers passing * the same input twice will get duplicate entries because the * `revoked_at` differs. To dedupe, pre-check via `readMemoryPack`'s * `revokedRecordHashes` set. - * - * Throws if `packDir` is a tarball or doesn't contain manifest.json. */ export function appendRevocations( - packDir: string, + packPath: string, revocations: RevocationInput[], opts: AppendRevocationsOptions, ): MemoryPackRevocation[] { - if (/\.tar\.zst$/i.test(packDir)) { - throw new Error('appendRevocations: tarball packs are not supported in v0.3 — extract, append, re-tarball'); + if (revocations.length === 0) return []; + + if (isTarballPath(packPath)) { + return withExtractedTarball(packPath, (innerDir) => + appendRevocationsToDirectory(innerDir, revocations, opts), + ); } + return appendRevocationsToDirectory(packPath, revocations, opts); +} + +function appendRevocationsToDirectory( + packDir: string, + revocations: RevocationInput[], + opts: AppendRevocationsOptions, +): MemoryPackRevocation[] { if (!existsSync(packDir) || !statSync(packDir).isDirectory()) { throw new Error(`appendRevocations: ${packDir} is not a directory`); } @@ -472,23 +492,34 @@ export interface RevocationAnchorInput { } /** - * Append chain-anchor entries to revocation_anchors.jsonl. + * Append chain-anchor entries to revocation_anchors.jsonl. Accepts + * either a directory pack or a `.tar.zst` tarball — same atomic + * extract/repack flow as appendRevocations. * * The (record_hash, revoked_at) pair MUST match an existing entry in * revocations.jsonl — otherwise the chain anchor is meaningless. This * function does not enforce that cross-check (callers shouldn't have * to load the whole revocations file just to append); the verifier * does enforce it. - * - * Tarball mode is NOT supported in v0.6, mirroring appendRevocations. */ export function appendRevocationAnchors( - packDir: string, + packPath: string, anchors: RevocationAnchorInput[], ): MemoryPackRevocationAnchor[] { - if (/\.tar\.zst$/i.test(packDir)) { - throw new Error('appendRevocationAnchors: tarball packs are not supported in v0.6 — extract, append, re-tarball'); + if (anchors.length === 0) return []; + + if (isTarballPath(packPath)) { + return withExtractedTarball(packPath, (innerDir) => + appendRevocationAnchorsToDirectory(innerDir, anchors), + ); } + return appendRevocationAnchorsToDirectory(packPath, anchors); +} + +function appendRevocationAnchorsToDirectory( + packDir: string, + anchors: RevocationAnchorInput[], +): MemoryPackRevocationAnchor[] { if (!existsSync(packDir) || !statSync(packDir).isDirectory()) { throw new Error(`appendRevocationAnchors: ${packDir} is not a directory`); } @@ -517,3 +548,97 @@ export function appendRevocationAnchors( return built; } + +// ──────────────────────────────────────────────────────────────────── +// Tarball-aware append helper (v0.7) +// +// The append APIs accept .tar.zst paths transparently. For each +// invocation we: +// 1. Extract the tarball into a per-call temp dir. +// 2. Run the directory-mode append against the inner pack dir. +// 3. Repack the inner dir into `.new`. +// 4. Atomically rename `.new` → ``. +// 5. Clean up the temp dir. +// +// The atomic rename means a mid-flight failure (tar exit, JSON write +// error, signal) leaves the ORIGINAL tarball intact. The producer's +// audit trail never enters a half-written state. +// +// Concurrent appends to the same tarball are NOT safe — last writer +// wins. Same constraint as concurrent writes to a directory pack; +// callers needing multi-process coordination must layer their own +// locking. Documented in CHANGELOG. +// ──────────────────────────────────────────────────────────────────── + +function isTarballPath(path: string): boolean { + if (/\.tar\.zst$/i.test(path)) return true; + // Some callers may pass a tarball without the canonical extension. + // Stat-then-isFile so directory packs (which we want to handle in + // directory-mode) don't get routed through the tarball path. + try { + return statSync(path).isFile(); + } catch { + return false; + } +} + +function withExtractedTarball( + tarballPath: string, + fn: (innerDir: string) => T, +): T { + if (!existsSync(tarballPath)) { + throw new Error(`tarball not found: ${tarballPath}`); + } + + const extractRoot = mkdtempSync(join(tmpdir(), 'mp-append-')); + try { + const extract = spawnSync( + 'tar', + ['--zstd', '-xf', tarballPath, '-C', extractRoot], + { stdio: ['ignore', 'ignore', 'pipe'] }, + ); + if (extract.status !== 0) { + const stderr = extract.stderr ? extract.stderr.toString() : ''; + throw new Error( + `tar --zstd extraction failed: ${stderr.trim() || 'no stderr'}`, + ); + } + + const entries = readdirSync(extractRoot); + if (entries.length !== 1) { + throw new Error( + `tarball expected single top-level dir, got ${entries.length}`, + ); + } + const innerName = entries[0]; + const innerDir = join(extractRoot, innerName); + + // Run the caller's mutation against the extracted directory. + const result = fn(innerDir); + + // Repack to a sibling temp file, then atomically rename. If the + // tar invocation fails, we throw before touching the original. + const stagingPath = `${tarballPath}.new-${process.pid}-${Date.now()}`; + const repack = spawnSync( + 'tar', + ['--zstd', '-cf', stagingPath, '-C', extractRoot, innerName], + { stdio: ['ignore', 'ignore', 'pipe'] }, + ); + if (repack.status !== 0) { + try { rmSync(stagingPath, { force: true }); } catch { /* ignore */ } + const stderr = repack.stderr ? repack.stderr.toString() : ''; + throw new Error( + `tar --zstd repack failed: ${stderr.trim() || 'no stderr'}`, + ); + } + if (!existsSync(stagingPath) || statSync(stagingPath).size === 0) { + try { rmSync(stagingPath, { force: true }); } catch { /* ignore */ } + throw new Error('tar produced an empty repack'); + } + + renameSync(stagingPath, tarballPath); + return result; + } finally { + rmSync(extractRoot, { recursive: true, force: true }); + } +}