diff --git a/CHANGELOG.md b/CHANGELOG.md index b7dea7e..0322a9c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,16 @@ independent of the on-disk **format** version (`format` in the manifest). ## [Unreleased] +### Added + +- **Format 1.1.0** — the manifest gains four optional fields (spec §3.1): `specURL`, + `coverage` (`{from, to}` — earliest/latest episode `eventTime`), `scopes` (sorted + context ids the records reference) and `bundleDigest` (sha256 of the exact `CHECKSUMS` + bytes — one hash for the whole archive). The exporter emits them; the validator + recomputes `bundleDigest` when present. 1.0 bundles remain valid — the shipped 1.0 + fixtures double as backward-compatibility tests. `MemCoverage` is exported from the + package. + ## [0.2.0] - 2026-09-14 `pip install portable-memory` now installs the **`mem` command line** — paste an diff --git a/Schemas/manifest.schema.json b/Schemas/manifest.schema.json index 2053fe6..e79a417 100644 --- a/Schemas/manifest.schema.json +++ b/Schemas/manifest.schema.json @@ -2,7 +2,7 @@ "$schema": "https://json-schema.org/draft/2020-12/schema", "$id": "https://macpaw.github.io/portable-memory/schemas/manifest.schema.json", "title": "Portable Memory manifest", - "description": "manifest.json — declares format version, capabilities, model tags, counts, and per-file integrity for a .mem bundle.", + "description": "manifest.json — declares format version, capabilities, model tags, counts, per-file integrity and (format 1.1) coverage, scopes and an archive-level digest for a .mem bundle.", "type": "object", "required": ["format", "generator", "conformanceLevel", "createdAt", "exportMode", "schemaVersion", "embeddingModel", "embeddingDim", "embeddingsIncluded", "capabilities", "counts", "files"], "properties": { @@ -12,6 +12,19 @@ "createdAt": { "type": "string", "format": "date-time" }, "exportMode": { "type": "string", "enum": ["full", "incremental"] }, "since": { "type": "string", "format": "date-time", "description": "present for incremental exports" }, + "specURL": { "type": "string", "format": "uri", "description": "format 1.1 — URL of the specification the bundle follows" }, + "coverage": { + "type": "object", + "description": "format 1.1 — earliest and latest episode eventTime in the bundle; absent when there are no episodes", + "required": ["from", "to"], + "properties": { + "from": { "type": "string", "format": "date-time" }, + "to": { "type": "string", "format": "date-time" } + }, + "additionalProperties": false + }, + "scopes": { "type": "array", "items": { "type": "string" }, "uniqueItems": true, "description": "format 1.1 — sorted scope (context) ids the exported records reference; absent when none" }, + "bundleDigest": { "type": "string", "pattern": "^[0-9a-f]{64}$", "description": "format 1.1 — sha256 of the exact CHECKSUMS bytes: one hash for the whole archive" }, "schemaVersion": { "type": "integer" }, "embeddingModel": { "type": "string" }, "embeddingDim": { "type": "integer", "minimum": 0 }, diff --git a/Spec/portable-memory-spec.md b/Spec/portable-memory-spec.md index 14b0e9f..8271427 100644 --- a/Spec/portable-memory-spec.md +++ b/Spec/portable-memory-spec.md @@ -1,7 +1,7 @@ -# Portable Memory — Specification v1.0 +# Portable Memory — Specification v1.1 > An open, vendor-neutral format for AI memory portability — proposed and stewarded in the open by MacPaw. -> Status: **v1.0**. Reference SDKs: Swift (github.com/MacPaw/portable-memory-swift) and Python (github.com/MacPaw/portable-memory). +> Status: **v1.1** (format `1.1.0`; 1.0 bundles remain valid). Reference SDKs: Swift (github.com/MacPaw/portable-memory-swift) and Python (github.com/MacPaw/portable-memory). > Paper: [Memory Belongs to the User: Portable Memory, an Open Standard Proposal for Cross-Vendor AI Memory](https://research.macpaw.com/publications/portable-memory) (MacPaw Research, 2026). **Goal.** Define an open, vendor-neutral format and protocol for carrying AI memory @@ -221,18 +221,22 @@ is. Embeddings are an optional accelerator. ```jsonc { - "format": "1.0.0", + "format": "1.1.0", + "specURL": "https://github.com/MacPaw/portable-memory/blob/main/Spec/portable-memory-spec.md", "generator": "your-app/1.0", "conformanceLevel": "L2", "createdAt": "2026-06-30T00:00:00Z", "exportMode": "full", // or "incremental" with "since" + "coverage": { "from": "2024-01-03T09:12:00Z", "to": "2026-06-29T18:20:00Z" }, // earliest/latest episode eventTime + "scopes": ["ctx_personal", "ctx_team-platform"], // context ids the records reference "schemaVersion": 12, "embeddingModel": "bge-m3", "embeddingDim": 1024, "embeddingsIncluded": false, // false ⇒ receiver re-embeds from source text "capabilities": ["bitemporal", "tombstones", "redaction", "evidence-pack"], "counts": { "episode": 128, "edge": 64, "tombstone": 3 }, - "files": [ { "path": "items/episode.jsonl", "sha256": "…", "bytes": 40213 } ] + "files": [ { "path": "items/episode.jsonl", "sha256": "…", "bytes": 40213 } ], + "bundleDigest": "…" // sha256 of the CHECKSUMS file — one hash for the whole archive } ``` @@ -240,6 +244,29 @@ is. Embeddings are an optional accelerator. capability and **preserve unknown kinds verbatim** + **foreign episode fields via `ext`** on round-trip (§10), so a newer or other-vendor bundle never loses data here. +### 3.1 Format 1.1 additions (normative) + +Format `1.1.0` adds four **optional** manifest fields so a bundle answers "what does +this archive cover, in which scopes, and is it whole?" without opening a stream — the +questions a procurement or erasure audit asks first. A producer at format ≥ 1.1 SHOULD +emit them; a reader MUST accept their absence (every 1.0 bundle is a valid 1.1 bundle). + +- **`specURL`** — the URL of the specification the bundle follows. +- **`coverage`** — `{ "from", "to" }`: the earliest and latest `eventTime` among the + episodes in the bundle (RFC 3339, whole-second UTC). Omitted when the bundle carries + no episodes. For an incremental export it describes the episodes *in this bundle*, not + the source store. +- **`scopes`** — the sorted (by Unicode code point), de-duplicated set of scope + identifiers the exported records reference: every episode `contextID` plus the `id` + of every exported `context` record. Omitted when empty. Scopes are how a host expresses + org / team / personal partitioning (§2); a reader can enumerate them before import. +- **`bundleDigest`** — the lowercase-hex SHA-256 of the exact bytes of `CHECKSUMS` + (§1.2). Because `CHECKSUMS` lists every data file's digest, this is a single hash for + the whole archive; a validator that finds it present MUST recompute it from the + on-disk `CHECKSUMS` and report a mismatch as an integrity failure. It complements — + not replaces — the optional manifest signature (§1.3), which is what makes the hash + *authentic* rather than merely *consistent*. + --- ## 4. Operations — export & import / merge @@ -397,6 +424,9 @@ tombstones* is a real, testable feature — not an aspiration. ## 9. Versioning & capability negotiation +- **Format history** — `1.0.0` (2026-06): initial release. `1.1.0` (2026-09): adds the + optional manifest fields `specURL`, `coverage`, `scopes`, `bundleDigest` (§3.1). A + minor bump: 1.0 readers ignore the new fields; 1.1 readers accept their absence. - `format` is semver in the manifest. `capabilities[]` declares features (e.g. `bitemporal`, `tombstones`, `redaction`, `evidence-pack`, `embeddings:`). - Importers negotiate by capability and **preserve unknown kinds verbatim** and diff --git a/portable_memory/__init__.py b/portable_memory/__init__.py index 069d6ca..71f266f 100644 --- a/portable_memory/__init__.py +++ b/portable_memory/__init__.py @@ -19,6 +19,7 @@ BundlePath, ConformanceLevel, ExportMode, + MemCoverage, MemFileEntry, MemFormat, MemKind, @@ -64,7 +65,7 @@ __all__ = [ "canonical_json", "format_timestamp", "parse_timestamp", "to_wire", "from_wire", "to_line", "MemFormat", "MemKind", "IMPORT_ORDER", "ExportMode", "ConformanceLevel", - "MemFileEntry", "MemManifest", "MemLimits", "BundlePath", + "MemFileEntry", "MemManifest", "MemCoverage", "MemLimits", "BundlePath", "sha256_hex", "sha256_hex_str", "embedding_cache_key", "KNOWN_EPISODE_KEYS", "extract_episode_ext", "merge_episode_ext", "PortableEpisode", "PortableEntity", "PortableEdge", "PortableFact", "PortableFactLink", diff --git a/portable_memory/exporter.py b/portable_memory/exporter.py index 1d4db6d..3c2641d 100644 --- a/portable_memory/exporter.py +++ b/portable_memory/exporter.py @@ -18,6 +18,7 @@ from .format import ( ConformanceLevel, ExportMode, + MemCoverage, MemFileEntry, MemFormat, MemKind, @@ -125,8 +126,9 @@ def keep(ts: datetime) -> bool: counts[MemKind.PROCEDURE.value] = self._write_jsonl( store.export_procedures(), "items/procedure.jsonl", root, files ) + contexts = store.export_contexts() counts[MemKind.CONTEXT.value] = self._write_jsonl( - store.export_contexts(), "items/context.jsonl", root, files + contexts, "items/context.jsonl", root, files ) counts[MemKind.COMMUNITY.value] = self._write_jsonl( store.export_communities(), "items/community.jsonl", root, files @@ -166,7 +168,7 @@ def keep(ts: datetime) -> bool: audit = store.export_audit_log(cutoff) counts["audit"] = self._write_jsonl(audit, "audit/log.jsonl", root, files) - self._write_checksums(files, root) + checksums = self._write_checksums(files, root) capabilities = [ "bitemporal", "tombstones", @@ -185,6 +187,9 @@ def keep(ts: datetime) -> bool: counts=counts, files=files, capabilities=capabilities, + coverage=_coverage_of(episodes), + scopes=_scopes_of(episodes, contexts), + bundle_digest=sha256_hex(checksums), ) self._write_manifest(manifest, root, signing_key) return manifest @@ -214,7 +219,7 @@ def export_evidence_pack( counts["provenanceEdge"] = self._write_jsonl( store.export_edges(), "provenance/edges.jsonl", root, files ) - self._write_checksums(files, root) + checksums = self._write_checksums(files, root) capabilities = ["evidence-pack", "proof-of-deletion", "audit", "provenance"] if signing_key is not None: capabilities.append("signed") @@ -226,6 +231,7 @@ def export_evidence_pack( counts=counts, files=files, capabilities=capabilities, + bundle_digest=sha256_hex(checksums), ) self._write_manifest(manifest, root, signing_key) return manifest @@ -272,9 +278,16 @@ def _make_manifest( counts: dict[str, int], files: list[MemFileEntry], capabilities: list[str], + coverage: MemCoverage | None = None, + scopes: list[str] | None = None, + bundle_digest: str | None = None, ) -> MemManifest: return MemManifest( format=MemFormat.VERSION, + spec_url=MemFormat.SPEC_URL, + coverage=coverage, + scopes=scopes, + bundle_digest=bundle_digest, generator=info.generator, conformance_level=level, # Aware UTC — a naive datetime.now() is LOCAL time, and the codec would @@ -292,13 +305,15 @@ def _make_manifest( files=sorted(files, key=lambda f: f.path), ) - def _write_checksums(self, files: list[MemFileEntry], root: str) -> None: + def _write_checksums(self, files: list[MemFileEntry], root: str) -> bytes: """Write ``CHECKSUMS`` in ``sha256sum`` format (`` ``), sorted by path - with a trailing newline. Two spaces separate hash and path per the tool's format.""" + with a trailing newline. Two spaces separate hash and path per the tool's format. + Returns the exact bytes written — the manifest's ``bundleDigest`` hashes them.""" ordered = sorted(files, key=lambda f: f.path) lines = [f"{f.sha256} {f.path}" for f in ordered] data = ("\n".join(lines) + "\n").encode("utf-8") self._write_bytes(data, "CHECKSUMS", root) + return data def _write_episodes( self, @@ -358,3 +373,22 @@ def _rmtree(path: str) -> None: else: os.remove(child) os.rmdir(path) + + +# ── Format 1.1 manifest summaries (spec §3.1) ───────────────────────────────────────── + +def _coverage_of(episodes: list) -> MemCoverage | None: + """Earliest and latest ``eventTime`` among the episodes *in this bundle*; None when + the bundle carries no episodes.""" + if not episodes: + return None + times = [e.event_time for e in episodes] + return MemCoverage(from_=min(times), to=max(times)) + + +def _scopes_of(episodes: list, contexts: list) -> list[str] | None: + """Sorted (by code point), de-duplicated scope ids: every episode ``contextID`` plus + the id of every exported ``context`` record; None when empty.""" + ids = {e.context_id for e in episodes if e.context_id} + ids.update(c.id for c in contexts) + return sorted(ids) or None diff --git a/portable_memory/format.py b/portable_memory/format.py index 02e58d9..555ccd6 100644 --- a/portable_memory/format.py +++ b/portable_memory/format.py @@ -15,7 +15,11 @@ class MemFormat: #: Semantic version of the on-disk format. Importers negotiate by this + capabilities. - VERSION = "1.0.0" + #: 1.1.0 added the optional manifest fields ``specURL``, ``coverage``, ``scopes`` and + #: ``bundleDigest``; 1.0 bundles remain valid (same major). + VERSION = "1.1.0" + #: Where the specification this bundle follows lives (``manifest.specURL``). + SPEC_URL = "https://github.com/MacPaw/portable-memory/blob/main/Spec/portable-memory-spec.md" #: Conventional bundle directory suffix. BUNDLE_SUFFIX = "mem" @@ -67,6 +71,15 @@ class MemFileEntry: bytes: int = field(metadata={"json": "bytes"}) +@dataclass(kw_only=True) +class MemCoverage: + """The time span of the memories in a bundle — the earliest and latest episode + ``eventTime`` (format 1.1). Lets a reader answer "what period does this archive + cover?" without opening a stream.""" + from_: datetime = field(metadata={"json": "from"}) + to: datetime = field(metadata={"json": "to"}) + + @dataclass(kw_only=True) class MemManifest: """Declares everything an importer needs to negotiate capabilities and verify integrity.""" @@ -83,6 +96,16 @@ class MemManifest: counts: dict[str, int] = field(metadata={"json": "counts"}) files: list[MemFileEntry] = field(metadata={"json": "files"}) since: datetime | None = field(default=None, metadata={"json": "since"}) + # ── Format 1.1 additions. Optional on read: a 1.0 bundle has none of them. ── + #: URL of the specification the bundle follows. + spec_url: str | None = field(default=None, metadata={"json": "specURL"}) + #: Earliest/latest episode ``eventTime`` in the bundle; absent when it has no episodes. + coverage: MemCoverage | None = field(default=None, metadata={"json": "coverage"}) + #: Sorted, unique scope (context) ids the exported records reference; absent when none. + scopes: list[str] | None = field(default=None, metadata={"json": "scopes"}) + #: Lowercase-hex SHA-256 of the exact ``CHECKSUMS`` bytes — one hash for the whole + #: archive. Validators recompute it when present. + bundle_digest: str | None = field(default=None, metadata={"json": "bundleDigest"}) class MemLimits: diff --git a/portable_memory/validator.py b/portable_memory/validator.py index ae74367..3d7fd99 100644 --- a/portable_memory/validator.py +++ b/portable_memory/validator.py @@ -178,6 +178,22 @@ def validate( issues.append(f"malformed {kind.value} record") break + # ── Archive-level digest (format 1.1, spec §3.1): bundleDigest = sha256(CHECKSUMS). ── + if manifest.bundle_digest is not None: + cpath = BundlePath.safe_path("CHECKSUMS", bundle) + if cpath is None or not os.path.isfile(cpath): + issues.append("bundleDigest declared but CHECKSUMS is missing") + else: + size = MemLimits.file_size(cpath) + if size is None or size > MemLimits.max_file_bytes: + issues.append("CHECKSUMS exceeds size limit") + else: + with open(cpath, "rb") as fh: + if sha256_hex(fh.read()) != manifest.bundle_digest: + issues.append( + "bundleDigest mismatch: CHECKSUMS does not hash to the manifest's bundleDigest" + ) + return ValidationResult(manifest=manifest, issues=issues) @staticmethod diff --git a/tests/test_manifest_v11.py b/tests/test_manifest_v11.py new file mode 100644 index 0000000..c34d1e6 --- /dev/null +++ b/tests/test_manifest_v11.py @@ -0,0 +1,133 @@ +"""Format 1.1 manifest fields — ``specURL``, ``coverage``, ``scopes``, ``bundleDigest`` +(spec §3.1) — plus backward compatibility with the shipped 1.0 fixtures.""" +from __future__ import annotations + +import json +from datetime import datetime, timezone +from pathlib import Path + +import pytest + +from portable_memory import ( + BundleExporter, BundleImporter, BundleValidator, ExportMode, MemFormat, MemManifest, + PortableContext, from_wire, sha256_hex, to_line, +) +from portable_memory.adapters.transfer import TransferTextAdapter +from portable_memory.inmemory import InMemoryStore + +FIXED = datetime.fromtimestamp(1_700_000_000, tz=timezone.utc) # 2023-11-14T22:13:20Z +ROOT = Path(__file__).resolve().parents[1] +SAMPLE = (ROOT / "Conformance" / "fixtures" / "transfer" / "sample-export.txt").read_text(encoding="utf-8") +#: sha256 of the CHECKSUMS file of the transfer-fixture bundle — identical in the Swift suite. +PARITY_DIGEST = "1f7f0cdce93ef223537e6106b4713af03ff725e5ec07ab984cc805f719d6e401" + + +def utc(*a): + return datetime(*a, tzinfo=timezone.utc) + + +def _store(): + s = InMemoryStore(generator="test/1.0") + for e in TransferTextAdapter.parse_episodes(SAMPLE, source="chatgpt", now=FIXED): + s.import_episode(e, None) + return s + + +class ScopedStore(InMemoryStore): + def __init__(self): + super().__init__(generator="test/1.0") + self.contexts: list[PortableContext] = [] + + def export_contexts(self): + return list(self.contexts) + + +def test_manifest_carries_format_1_1_fields(tmp_path): + out = tmp_path / "b.mem" + m = BundleExporter().export(_store(), out) + assert m.format == "1.1.0" == MemFormat.VERSION + assert m.spec_url == MemFormat.SPEC_URL + # coverage = earliest/latest eventTime: undated entries take `now`; the latest dated entry is [2026-03-01]. + assert m.coverage.from_ == FIXED and m.coverage.to == utc(2026, 3, 1) + assert m.scopes is None # transfer entries reference no scopes + checksums = (out / "CHECKSUMS").read_bytes() + assert m.bundle_digest == sha256_hex(checksums) == PARITY_DIGEST + + on_disk = json.loads((out / "manifest.json").read_bytes()) + assert on_disk["format"] == "1.1.0" and on_disk["specURL"] == MemFormat.SPEC_URL + assert on_disk["coverage"] == {"from": "2023-11-14T22:13:20Z", "to": "2026-03-01T00:00:00Z"} + assert "scopes" not in on_disk # absent optional fields are omitted, never null + assert on_disk["bundleDigest"] == m.bundle_digest + assert BundleValidator().validate(out).ok + assert from_wire(MemManifest, on_disk).coverage.to == utc(2026, 3, 1) # round-trips through the codec + + +def test_scopes_enumerate_context_ids_sorted_and_unique(tmp_path): + s = ScopedStore() + for i, ctx in enumerate(["ctx_team", "ctx_personal", "ctx_team", None, ""]): + (e,) = TransferTextAdapter.parse_episodes(f"[2026-01-0{i + 1}] - m{i}", now=FIXED) + e.context_id = ctx + s.import_episode(e, None) + s.contexts.append(PortableContext(id="ctx_org", label="Org", archived=False, created_at=FIXED, parent_id=None)) + m = BundleExporter().export(s, tmp_path / "s.mem") + assert m.scopes == ["ctx_org", "ctx_personal", "ctx_team"] # sorted, de-duplicated, empty/None skipped + assert m.coverage.from_ == utc(2026, 1, 1) and m.coverage.to == utc(2026, 1, 5) + assert json.loads((tmp_path / "s.mem" / "manifest.json").read_bytes())["scopes"] == m.scopes + + +def test_incremental_coverage_describes_only_the_bundle(tmp_path): + s = InMemoryStore(generator="test/1.0") + old, new = TransferTextAdapter.parse_episodes("[2024-01-01] - old\n[2026-06-01] - new\n", now=FIXED) + old.ingestion_time, new.ingestion_time = utc(2024, 1, 2), utc(2026, 6, 2) + s.import_episode(old, None) + s.import_episode(new, None) + m = BundleExporter().export(s, tmp_path / "i.mem", mode=ExportMode.INCREMENTAL, since=utc(2025, 1, 1)) + assert m.counts.get("episode") == 1 + assert m.coverage.from_ == utc(2026, 6, 1) and m.coverage.to == utc(2026, 6, 1) + + +def test_empty_bundle_omits_coverage_and_scopes_but_has_digest(tmp_path): + out = tmp_path / "e.mem" + m = BundleExporter().export(InMemoryStore(generator="test/1.0"), out) + assert m.coverage is None and m.scopes is None + assert m.bundle_digest == sha256_hex((out / "CHECKSUMS").read_bytes()) + assert BundleValidator().validate(out).ok + + +def test_validator_verifies_bundle_digest(tmp_path): + out = tmp_path / "t.mem" + BundleExporter().export(_store(), out) + with open(out / "CHECKSUMS", "ab") as fh: # a line smuggled into CHECKSUMS + fh.write(b"0" * 64 + b" items/evil.jsonl\n") + issues = BundleValidator().validate(out).issues + assert any("bundleDigest mismatch" in i for i in issues), issues + + BundleExporter().export(_store(), out) + m = from_wire(MemManifest, json.loads((out / "manifest.json").read_bytes())) + m.bundle_digest = "f" * 64 # manifest claims a digest that isn't true + (out / "manifest.json").write_bytes(to_line(m).encode("utf-8")) + assert any("bundleDigest mismatch" in i for i in BundleValidator().validate(out).issues) + + BundleExporter().export(_store(), out) + (out / "CHECKSUMS").unlink() + assert any("CHECKSUMS is missing" in i for i in BundleValidator().validate(out).issues) + + +def test_1_0_bundles_remain_valid_and_importable(): + """The shipped 1.0.0 fixtures have none of the 1.1 fields — a 1.1 reader must accept them.""" + fixture = ROOT / "Conformance" / "fixtures" / "sample.mem" + res = BundleValidator().validate(fixture) + assert res.ok, res.issues + assert res.manifest.format.startswith("1.0") and res.manifest.spec_url is None + assert res.manifest.coverage is None and res.manifest.scopes is None and res.manifest.bundle_digest is None + store = InMemoryStore() + report = BundleImporter().import_bundle(store, fixture, reembed=False) + assert len(store.episodes) == res.manifest.counts.get("episode", 0) - report.skipped_tombstoned or len(store.episodes) > 0 + + +def test_manifest_matches_json_schema(tmp_path): + jsonschema = pytest.importorskip("jsonschema") + out = tmp_path / "v.mem" + BundleExporter().export(_store(), out) + schema = json.loads((ROOT / "Schemas" / "manifest.schema.json").read_text(encoding="utf-8")) + jsonschema.Draft202012Validator(schema).validate(json.loads((out / "manifest.json").read_bytes()))