From c1e0cd50363933a6f6a0099aa5d69580cc8df894 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Mon, 31 Aug 2026 20:12:29 +0900 Subject: [PATCH 01/39] feat: add deterministic provenance archive envelope --- backend/services/tenant_provenance_bundle.py | 276 ++++++++++++++++++ .../tests/test_tenant_provenance_bundle.py | 218 ++++++++++++++ 2 files changed, 494 insertions(+) create mode 100644 backend/services/tenant_provenance_bundle.py create mode 100644 backend/tests/test_tenant_provenance_bundle.py diff --git a/backend/services/tenant_provenance_bundle.py b/backend/services/tenant_provenance_bundle.py new file mode 100644 index 000000000..da385c8c0 --- /dev/null +++ b/backend/services/tenant_provenance_bundle.py @@ -0,0 +1,276 @@ +"""Deterministic, bounded BagIt/RO-Crate envelopes for tenant provenance.""" + +from __future__ import annotations + +import hashlib +import io +import json +import math +import stat +import zipfile +from collections.abc import Mapping +from typing import Any + + +ARCHIVE_MAX_BYTES = 64 * 1024 * 1024 +ARCHIVE_MAX_ENTRIES = 64 +ENTRY_MAX_BYTES = 32 * 1024 * 1024 +MAX_COMPRESSION_RATIO = 100 +JSON_MAX_DEPTH = 64 +_FIXED_TIMESTAMP = (1980, 1, 1, 0, 0, 0) +_PAYLOAD_NAME = "data/records.json" +_EXPECTED_ENTRIES = frozenset( + { + "bagit.txt", + "bag-info.txt", + "manifest-sha512.txt", + "tagmanifest-sha512.txt", + "ro-crate-metadata.json", + _PAYLOAD_NAME, + } +) + + +class ProvenanceArchiveError(ValueError): + """Raised when a provenance envelope is malformed or outside this profile.""" + + +def _fail() -> None: + raise ProvenanceArchiveError("Invalid provenance archive") + + +def _validate_json_value(value: object, depth: int = 0) -> None: + if depth > JSON_MAX_DEPTH: + _fail() + if value is None or isinstance(value, (str, bool, int)): + return + if isinstance(value, float): + if not math.isfinite(value): + _fail() + return + if isinstance(value, list): + for item in value: + _validate_json_value(item, depth + 1) + return + if isinstance(value, Mapping): + for key, item in value.items(): + if not isinstance(key, str): + _fail() + _validate_json_value(item, depth + 1) + return + _fail() + + +def _canonical_json(value: object) -> bytes: + _validate_json_value(value) + try: + return json.dumps( + value, + allow_nan=False, + ensure_ascii=False, + separators=(",", ":"), + sort_keys=True, + ).encode("utf-8") + except (TypeError, ValueError, UnicodeEncodeError) as exc: + raise ProvenanceArchiveError("Invalid provenance archive") from exc + + +def _records_bundle_uid(records: Mapping[str, object]) -> str: + if records.get("profile") != "naruon-tenant-provenance/v1" or records.get( + "schema_version" + ) != 1: + _fail() + bundle_uid = records.get("bundle_uid") + if not isinstance(bundle_uid, str) or not bundle_uid: + _fail() + return bundle_uid + + +def _bag_info(bundle_uid: str) -> bytes: + return ( + "Bag-Software-Agent: naruon\n" + "Bagging-Date: 1980-01-01\n" + f"External-Identifier: {bundle_uid}\n" + ).encode("utf-8") + + +def _ro_crate(records: Mapping[str, object], payload_digest: str) -> bytes: + activity = records.get("export_activity") + activity_uid = activity.get("activity_uid") if isinstance(activity, Mapping) else None + if not isinstance(activity_uid, str) or not activity_uid: + _fail() + crate = { + "@context": "https://w3id.org/ro/crate/1.3/context", + "@graph": [ + { + "@id": "./", + "@type": "Dataset", + "conformsTo": "naruon-tenant-provenance/v1", + "hasPart": {"@id": _PAYLOAD_NAME}, + "name": "Naruon tenant provenance bundle", + }, + { + "@id": _PAYLOAD_NAME, + "@type": "File", + "encodingFormat": "application/json", + "sha512": payload_digest, + }, + { + "@id": f"#{activity_uid}", + "@type": "CreateAction", + "instrument": {"@id": "#naruon"}, + "object": {"@id": _PAYLOAD_NAME}, + }, + { + "@id": "#naruon", + "@type": "SoftwareApplication", + "name": "Naruon", + }, + ], + } + return _canonical_json(crate) + + +def _manifest(entries: Mapping[str, bytes], names: tuple[str, ...]) -> bytes: + return b"".join( + f"{hashlib.sha512(entries[name]).hexdigest()} {name}\n".encode("ascii") + for name in names + ) + + +def _archive_entries(records: Mapping[str, object]) -> dict[str, bytes]: + payload = _canonical_json(records) + bundle_uid = _records_bundle_uid(records) + entries = { + "bagit.txt": b"BagIt-Version: 1.0\nTag-File-Character-Encoding: UTF-8\n", + "bag-info.txt": _bag_info(bundle_uid), + _PAYLOAD_NAME: payload, + "ro-crate-metadata.json": _ro_crate(records, hashlib.sha512(payload).hexdigest()), + } + entries["manifest-sha512.txt"] = _manifest(entries, (_PAYLOAD_NAME,)) + entries["tagmanifest-sha512.txt"] = _manifest( + entries, + ("bag-info.txt", "bagit.txt", "manifest-sha512.txt", "ro-crate-metadata.json"), + ) + return entries + + +def _zip_info(name: str) -> zipfile.ZipInfo: + info = zipfile.ZipInfo(name, date_time=_FIXED_TIMESTAMP) + info.create_system = 3 + info.external_attr = (stat.S_IFREG | 0o644) << 16 + info.compress_type = zipfile.ZIP_DEFLATED + return info + + +def build_provenance_archive(records: Mapping[str, object]) -> bytes: + """Build the fixed deterministic ZIP envelope for a validated record payload.""" + if not isinstance(records, Mapping): + _fail() + entries = _archive_entries(records) + output = io.BytesIO() + with zipfile.ZipFile(output, "w", compression=zipfile.ZIP_DEFLATED, compresslevel=9) as archive: + for name in sorted(entries): + archive.writestr(_zip_info(name), entries[name], compress_type=zipfile.ZIP_DEFLATED, compresslevel=9) + archive_bytes = output.getvalue() + if len(archive_bytes) > ARCHIVE_MAX_BYTES: + _fail() + return archive_bytes + + +def _is_unsafe_member(info: zipfile.ZipInfo) -> bool: + name = info.filename + parts = name.split("/") + return ( + not name + or "\\" in name + or "\x00" in name + or name.startswith("/") + or any(part in {"", ".", ".."} for part in parts) + or parts[0].endswith(":") + or info.is_dir() + or stat.S_ISLNK(info.external_attr >> 16) + or bool(info.flag_bits & 0x1) + ) + + +def _read_entry(archive: zipfile.ZipFile, info: zipfile.ZipInfo) -> bytes: + if info.file_size > ENTRY_MAX_BYTES: + _fail() + if info.file_size and (not info.compress_size or info.file_size / info.compress_size > MAX_COMPRESSION_RATIO): + _fail() + data = bytearray() + try: + with archive.open(info, "r") as source: + while chunk := source.read(min(64 * 1024, ENTRY_MAX_BYTES + 1 - len(data))): + data.extend(chunk) + if len(data) > ENTRY_MAX_BYTES: + _fail() + except (OSError, RuntimeError, zipfile.BadZipFile) as exc: + raise ProvenanceArchiveError("Invalid provenance archive") from exc + if len(data) != info.file_size: + _fail() + return bytes(data) + + +def _reject_duplicate_json_keys(pairs: list[tuple[str, Any]]) -> dict[str, Any]: + value: dict[str, Any] = {} + for key, item in pairs: + if key in value: + _fail() + value[key] = item + return value + + +def _parse_records(data: bytes) -> dict[str, object]: + try: + records = json.loads( + data.decode("utf-8"), + object_pairs_hook=_reject_duplicate_json_keys, + parse_constant=lambda _value: _fail(), + ) + except (UnicodeDecodeError, json.JSONDecodeError, TypeError, ValueError) as exc: + raise ProvenanceArchiveError("Invalid provenance archive") from exc + if not isinstance(records, dict): + _fail() + if _canonical_json(records) != data: + _fail() + return records + + +def parse_provenance_archive(archive_bytes: bytes) -> dict[str, object]: + """Validate a bounded fixed envelope and return its canonical record payload.""" + if not isinstance(archive_bytes, (bytes, bytearray)) or len(archive_bytes) > ARCHIVE_MAX_BYTES: + _fail() + try: + with zipfile.ZipFile(io.BytesIO(archive_bytes), "r") as archive: + infos = archive.infolist() + if ( + len(infos) > ARCHIVE_MAX_ENTRIES + or sum(info.file_size for info in infos) > ARCHIVE_MAX_BYTES + or any( + _is_unsafe_member(info) + or info.date_time != _FIXED_TIMESTAMP + or info.create_system != 3 + or (info.external_attr >> 16) & 0o777 != 0o644 + or info.compress_type != zipfile.ZIP_DEFLATED + for info in infos + ) + ): + _fail() + names = [info.filename for info in infos] + if ( + names != sorted(names) + or len(set(names)) != len(names) + or set(names) != _EXPECTED_ENTRIES + ): + _fail() + entries = {info.filename: _read_entry(archive, info) for info in infos} + except (OSError, RuntimeError, zipfile.BadZipFile) as exc: + raise ProvenanceArchiveError("Invalid provenance archive") from exc + + records = _parse_records(entries[_PAYLOAD_NAME]) + expected = _archive_entries(records) + if entries != expected: + _fail() + return records diff --git a/backend/tests/test_tenant_provenance_bundle.py b/backend/tests/test_tenant_provenance_bundle.py new file mode 100644 index 000000000..02d40306b --- /dev/null +++ b/backend/tests/test_tenant_provenance_bundle.py @@ -0,0 +1,218 @@ +import hashlib +import io +import json +import warnings +import zipfile + +import pytest + +from services.tenant_provenance_bundle import ( + ARCHIVE_MAX_BYTES, + ARCHIVE_MAX_ENTRIES, + ProvenanceArchiveError, + build_provenance_archive, + parse_provenance_archive, +) + + +RECORDS = { + "profile": "naruon-tenant-provenance/v1", + "schema_version": 1, + "bundle_uid": "bundle-01HZZ", + "source_scope": {"organization_uid": "org-01", "workspace_uid": "ws-01"}, + "export_activity": {"activity_uid": "activity-01"}, + "emails": [{"email_uid": "email-01", "subject": "Evidence"}], + "attachments": [], + "content_nodes": [], + "content_segments": [], + "structural_edges": [], + "project_objects": [], + "project_edges": [], + "corrections": [], +} + +EXPECTED_ENTRIES = ( + "bag-info.txt", + "bagit.txt", + "data/records.json", + "manifest-sha512.txt", + "ro-crate-metadata.json", + "tagmanifest-sha512.txt", +) + + +def _canonical_json(value): + return json.dumps(value, ensure_ascii=False, allow_nan=False, separators=(",", ":"), sort_keys=True).encode( + "utf-8" + ) + + +def _replace_entries(archive, replacements, *, extra=()): + with zipfile.ZipFile(io.BytesIO(archive), "r") as source: + entries = { + info.filename: source.read(info) + for info in source.infolist() + if not info.is_dir() + } + entries.update(replacements) + return _archive_with_entries(entries, extra=extra) + + +def _rebuild_manifests(entries): + payload_names = ("data/records.json",) + entries["manifest-sha512.txt"] = b"".join( + f"{hashlib.sha512(entries[name]).hexdigest()} {name}\n".encode("ascii") + for name in payload_names + ) + tag_names = ("bag-info.txt", "bagit.txt", "manifest-sha512.txt", "ro-crate-metadata.json") + entries["tagmanifest-sha512.txt"] = b"".join( + f"{hashlib.sha512(entries[name]).hexdigest()} {name}\n".encode("ascii") + for name in tag_names + ) + return entries + + +def _archive_with_entries(entries, *, extra=(), fixed_metadata=True): + output = io.BytesIO() + with zipfile.ZipFile(output, "w", compression=zipfile.ZIP_DEFLATED) as target: + for name, content in sorted(entries.items()): + if fixed_metadata: + info = zipfile.ZipInfo(name, date_time=(1980, 1, 1, 0, 0, 0)) + info.create_system = 3 + info.external_attr = 0o100644 << 16 + info.compress_type = zipfile.ZIP_DEFLATED + target.writestr(info, content, compress_type=zipfile.ZIP_DEFLATED, compresslevel=9) + else: + target.writestr(name, content) + for name, content in extra: + with warnings.catch_warnings(): + warnings.simplefilter("ignore", UserWarning) + target.writestr(name, content) + return output.getvalue() + + +def test_build_is_deterministic_and_has_exact_fixed_entries(): + first = build_provenance_archive(RECORDS) + second = build_provenance_archive(dict(RECORDS)) + + assert first == second + with zipfile.ZipFile(io.BytesIO(first), "r") as archive: + assert tuple(sorted(archive.namelist())) == EXPECTED_ENTRIES + assert all(info.date_time == (1980, 1, 1, 0, 0, 0) for info in archive.infolist()) + assert all((info.external_attr >> 16) & 0o777 == 0o644 for info in archive.infolist()) + assert archive.read("data/records.json") == _canonical_json(RECORDS) + + +def test_parse_round_trips_records_and_verifies_ro_crate_metadata(): + archive = build_provenance_archive(RECORDS) + + assert parse_provenance_archive(archive) == RECORDS + with zipfile.ZipFile(io.BytesIO(archive), "r") as source: + crate = json.loads(source.read("ro-crate-metadata.json")) + assert crate["@context"] == "https://w3id.org/ro/crate/1.3/context" + assert {node["@type"] for node in crate["@graph"]} >= {"Dataset", "File", "CreateAction", "SoftwareApplication"} + + +def test_parse_rejects_payload_tampering(): + archive = build_provenance_archive(RECORDS) + tampered = _replace_entries(archive, {"data/records.json": b'{"tampered":true}'}) + + with pytest.raises(ProvenanceArchiveError): + parse_provenance_archive(tampered) + + +@pytest.mark.parametrize("name", ("../data/records.json", "data\\records.json")) +def test_parse_rejects_unsafe_paths(name): + archive = build_provenance_archive(RECORDS) + unsafe = _replace_entries(archive, {}, extra=((name, b"{}"),)) + + with pytest.raises(ProvenanceArchiveError): + parse_provenance_archive(unsafe) + + +def test_parse_rejects_colliding_paths(): + archive = build_provenance_archive(RECORDS) + collision = _replace_entries(archive, {}, extra=(("data/records.json", b"{}"),)) + + with pytest.raises(ProvenanceArchiveError): + parse_provenance_archive(collision) + + +@pytest.mark.parametrize("missing_or_extra", ("bagit.txt", "unexpected.txt")) +def test_parse_rejects_missing_or_extra_entries(missing_or_extra): + archive = build_provenance_archive(RECORDS) + with zipfile.ZipFile(io.BytesIO(archive), "r") as source: + entries = {info.filename: source.read(info) for info in source.infolist()} + if missing_or_extra in entries: + del entries[missing_or_extra] + else: + entries[missing_or_extra] = b"unexpected" + + with pytest.raises(ProvenanceArchiveError): + parse_provenance_archive(_archive_with_entries(entries)) + + +def test_parse_rejects_duplicate_json_keys_even_with_valid_manifests(): + archive = build_provenance_archive(RECORDS) + with zipfile.ZipFile(io.BytesIO(archive), "r") as source: + entries = {info.filename: source.read(info) for info in source.infolist()} + entries["data/records.json"] = b'{"profile":"first","profile":"second"}' + + with pytest.raises(ProvenanceArchiveError): + parse_provenance_archive(_archive_with_entries(_rebuild_manifests(entries))) + + +def test_build_rejects_non_finite_json_numbers(): + records = {**RECORDS, "export_activity": {"score": float("nan")}} + + with pytest.raises(ProvenanceArchiveError): + build_provenance_archive(records) + + +def test_build_rejects_an_unknown_profile(): + with pytest.raises(ProvenanceArchiveError): + build_provenance_archive({**RECORDS, "profile": "unknown"}) + + +def test_parse_rejects_nonfixed_zip_metadata(): + archive = build_provenance_archive(RECORDS) + with zipfile.ZipFile(io.BytesIO(archive), "r") as source: + entries = {info.filename: source.read(info) for info in source.infolist()} + + with pytest.raises(ProvenanceArchiveError): + parse_provenance_archive(_archive_with_entries(entries, fixed_metadata=False)) + + +def test_parse_enforces_total_uncompressed_bound(monkeypatch): + archive = build_provenance_archive({**RECORDS, "padding": "x" * 10_000}) + monkeypatch.setattr("services.tenant_provenance_bundle.ARCHIVE_MAX_BYTES", len(archive) + 1) + + with pytest.raises(ProvenanceArchiveError): + parse_provenance_archive(archive) + + +def test_parse_rejects_archive_bounds(monkeypatch): + monkeypatch.setattr("services.tenant_provenance_bundle.ARCHIVE_MAX_BYTES", 1) + with pytest.raises(ProvenanceArchiveError): + parse_provenance_archive(build_provenance_archive(RECORDS)) + + monkeypatch.setattr("services.tenant_provenance_bundle.ARCHIVE_MAX_BYTES", ARCHIVE_MAX_BYTES) + monkeypatch.setattr("services.tenant_provenance_bundle.ENTRY_MAX_BYTES", 1) + with pytest.raises(ProvenanceArchiveError): + parse_provenance_archive(build_provenance_archive(RECORDS)) + + +def test_parse_rejects_entry_count_and_compression_ratio(monkeypatch): + archive = build_provenance_archive(RECORDS) + with zipfile.ZipFile(io.BytesIO(archive), "r") as source: + entries = {info.filename: source.read(info) for info in source.infolist()} + + monkeypatch.setattr("services.tenant_provenance_bundle.ARCHIVE_MAX_ENTRIES", 1) + with pytest.raises(ProvenanceArchiveError): + parse_provenance_archive(_archive_with_entries(entries)) + + monkeypatch.setattr("services.tenant_provenance_bundle.ARCHIVE_MAX_ENTRIES", ARCHIVE_MAX_ENTRIES) + records = _canonical_json({**RECORDS, "padding": "x" * 10_000}) + entries["data/records.json"] = records + with pytest.raises(ProvenanceArchiveError): + parse_provenance_archive(_archive_with_entries(_rebuild_manifests(entries))) From 55da6e21f599db964c89a8320d2c27e84d10f257 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Mon, 31 Aug 2026 20:32:28 +0900 Subject: [PATCH 02/39] fix: harden provenance archive validation --- backend/services/tenant_provenance_bundle.py | 119 +++++++++--- .../tests/test_tenant_provenance_bundle.py | 176 ++++++++++++++++-- 2 files changed, 256 insertions(+), 39 deletions(-) diff --git a/backend/services/tenant_provenance_bundle.py b/backend/services/tenant_provenance_bundle.py index da385c8c0..3b32d46bf 100644 --- a/backend/services/tenant_provenance_bundle.py +++ b/backend/services/tenant_provenance_bundle.py @@ -5,7 +5,6 @@ import hashlib import io import json -import math import stat import zipfile from collections.abc import Mapping @@ -17,6 +16,8 @@ ENTRY_MAX_BYTES = 32 * 1024 * 1024 MAX_COMPRESSION_RATIO = 100 JSON_MAX_DEPTH = 64 +JSON_SAFE_INTEGER_MAX = 2**53 - 1 +_MAX_IDENTIFIER_LENGTH = 256 _FIXED_TIMESTAMP = (1980, 1, 1, 0, 0, 0) _PAYLOAD_NAME = "data/records.json" _EXPECTED_ENTRIES = frozenset( @@ -42,12 +43,14 @@ def _fail() -> None: def _validate_json_value(value: object, depth: int = 0) -> None: if depth > JSON_MAX_DEPTH: _fail() - if value is None or isinstance(value, (str, bool, int)): + if value is None or isinstance(value, (str, bool)): return - if isinstance(value, float): - if not math.isfinite(value): + if type(value) is int: + if not -JSON_SAFE_INTEGER_MAX <= value <= JSON_SAFE_INTEGER_MAX: _fail() return + if isinstance(value, float): + _fail() if isinstance(value, list): for item in value: _validate_json_value(item, depth + 1) @@ -56,6 +59,8 @@ def _validate_json_value(value: object, depth: int = 0) -> None: for key, item in value.items(): if not isinstance(key, str): _fail() + if not key.isascii(): + _fail() _validate_json_value(item, depth + 1) return _fail() @@ -76,14 +81,25 @@ def _canonical_json(value: object) -> bytes: def _records_bundle_uid(records: Mapping[str, object]) -> str: - if records.get("profile") != "naruon-tenant-provenance/v1" or records.get( - "schema_version" - ) != 1: + schema_version = records.get("schema_version") + if ( + records.get("profile") != "naruon-tenant-provenance/v1" + or type(schema_version) is not int + or schema_version != 1 + ): _fail() - bundle_uid = records.get("bundle_uid") - if not isinstance(bundle_uid, str) or not bundle_uid: + return _safe_identifier(records.get("bundle_uid")) + + +def _safe_identifier(value: object) -> str: + if ( + not isinstance(value, str) + or not value + or len(value) > _MAX_IDENTIFIER_LENGTH + or any(ord(character) < 32 or ord(character) == 127 for character in value) + ): _fail() - return bundle_uid + return value def _bag_info(bundle_uid: str) -> bytes: @@ -96,16 +112,27 @@ def _bag_info(bundle_uid: str) -> bytes: def _ro_crate(records: Mapping[str, object], payload_digest: str) -> bytes: activity = records.get("export_activity") - activity_uid = activity.get("activity_uid") if isinstance(activity, Mapping) else None - if not isinstance(activity_uid, str) or not activity_uid: + if not isinstance(activity, Mapping): _fail() + activity_uid = _safe_identifier(activity.get("activity_uid")) + date_published = _safe_identifier(activity.get("date_published")) crate = { - "@context": "https://w3id.org/ro/crate/1.3/context", + "@context": [ + "https://w3id.org/ro/crate/1.3/context", + {"prov": "http://www.w3.org/ns/prov#"}, + ], "@graph": [ + { + "@id": "ro-crate-metadata.json", + "@type": "CreativeWork", + "about": {"@id": "./"}, + "conformsTo": {"@id": "https://w3id.org/ro/crate/1.3"}, + }, { "@id": "./", "@type": "Dataset", "conformsTo": "naruon-tenant-provenance/v1", + "datePublished": date_published, "hasPart": {"@id": _PAYLOAD_NAME}, "name": "Naruon tenant provenance bundle", }, @@ -117,13 +144,15 @@ def _ro_crate(records: Mapping[str, object], payload_digest: str) -> bytes: }, { "@id": f"#{activity_uid}", - "@type": "CreateAction", + "@type": ["CreateAction", "prov:Activity"], "instrument": {"@id": "#naruon"}, "object": {"@id": _PAYLOAD_NAME}, + "prov:used": {"@id": "./"}, + "prov:wasAssociatedWith": {"@id": "#naruon"}, }, { "@id": "#naruon", - "@type": "SoftwareApplication", + "@type": ["SoftwareApplication", "prov:SoftwareAgent"], "name": "Naruon", }, ], @@ -145,7 +174,9 @@ def _archive_entries(records: Mapping[str, object]) -> dict[str, bytes]: "bagit.txt": b"BagIt-Version: 1.0\nTag-File-Character-Encoding: UTF-8\n", "bag-info.txt": _bag_info(bundle_uid), _PAYLOAD_NAME: payload, - "ro-crate-metadata.json": _ro_crate(records, hashlib.sha512(payload).hexdigest()), + "ro-crate-metadata.json": _ro_crate( + records, hashlib.sha512(payload).hexdigest() + ), } entries["manifest-sha512.txt"] = _manifest(entries, (_PAYLOAD_NAME,)) entries["tagmanifest-sha512.txt"] = _manifest( @@ -158,8 +189,16 @@ def _archive_entries(records: Mapping[str, object]) -> dict[str, bytes]: def _zip_info(name: str) -> zipfile.ZipInfo: info = zipfile.ZipInfo(name, date_time=_FIXED_TIMESTAMP) info.create_system = 3 + info.create_version = 20 + info.extract_version = 20 + info.reserved = 0 + info.flag_bits = 0 + info.volume = 0 + info.internal_attr = 0 info.external_attr = (stat.S_IFREG | 0o644) << 16 info.compress_type = zipfile.ZIP_DEFLATED + info.extra = b"" + info.comment = b"" return info @@ -169,12 +208,20 @@ def build_provenance_archive(records: Mapping[str, object]) -> bytes: _fail() entries = _archive_entries(records) output = io.BytesIO() - with zipfile.ZipFile(output, "w", compression=zipfile.ZIP_DEFLATED, compresslevel=9) as archive: + with zipfile.ZipFile( + output, "w", compression=zipfile.ZIP_DEFLATED, compresslevel=9 + ) as archive: for name in sorted(entries): - archive.writestr(_zip_info(name), entries[name], compress_type=zipfile.ZIP_DEFLATED, compresslevel=9) + archive.writestr( + _zip_info(name), + entries[name], + compress_type=zipfile.ZIP_DEFLATED, + compresslevel=9, + ) archive_bytes = output.getvalue() if len(archive_bytes) > ARCHIVE_MAX_BYTES: _fail() + parse_provenance_archive(archive_bytes) return archive_bytes @@ -194,10 +241,30 @@ def _is_unsafe_member(info: zipfile.ZipInfo) -> bool: ) +def _has_fixed_metadata(info: zipfile.ZipInfo) -> bool: + return ( + info.date_time == _FIXED_TIMESTAMP + and info.create_system == 3 + and info.create_version == 20 + and info.extract_version == 20 + and info.reserved == 0 + and info.flag_bits == 0 + and info.volume == 0 + and info.internal_attr == 0 + and info.external_attr == (stat.S_IFREG | 0o644) << 16 + and info.compress_type == zipfile.ZIP_DEFLATED + and info.extra == b"" + and info.comment == b"" + ) + + def _read_entry(archive: zipfile.ZipFile, info: zipfile.ZipInfo) -> bytes: if info.file_size > ENTRY_MAX_BYTES: _fail() - if info.file_size and (not info.compress_size or info.file_size / info.compress_size > MAX_COMPRESSION_RATIO): + if info.file_size and ( + not info.compress_size + or info.file_size / info.compress_size > MAX_COMPRESSION_RATIO + ): _fail() data = bytearray() try: @@ -240,20 +307,20 @@ def _parse_records(data: bytes) -> dict[str, object]: def parse_provenance_archive(archive_bytes: bytes) -> dict[str, object]: """Validate a bounded fixed envelope and return its canonical record payload.""" - if not isinstance(archive_bytes, (bytes, bytearray)) or len(archive_bytes) > ARCHIVE_MAX_BYTES: + if ( + not isinstance(archive_bytes, (bytes, bytearray)) + or len(archive_bytes) > ARCHIVE_MAX_BYTES + ): _fail() try: with zipfile.ZipFile(io.BytesIO(archive_bytes), "r") as archive: infos = archive.infolist() if ( - len(infos) > ARCHIVE_MAX_ENTRIES + archive.comment + or len(infos) > ARCHIVE_MAX_ENTRIES or sum(info.file_size for info in infos) > ARCHIVE_MAX_BYTES or any( - _is_unsafe_member(info) - or info.date_time != _FIXED_TIMESTAMP - or info.create_system != 3 - or (info.external_attr >> 16) & 0o777 != 0o644 - or info.compress_type != zipfile.ZIP_DEFLATED + _is_unsafe_member(info) or not _has_fixed_metadata(info) for info in infos ) ): diff --git a/backend/tests/test_tenant_provenance_bundle.py b/backend/tests/test_tenant_provenance_bundle.py index 02d40306b..aa331313c 100644 --- a/backend/tests/test_tenant_provenance_bundle.py +++ b/backend/tests/test_tenant_provenance_bundle.py @@ -9,6 +9,9 @@ from services.tenant_provenance_bundle import ( ARCHIVE_MAX_BYTES, ARCHIVE_MAX_ENTRIES, + ENTRY_MAX_BYTES, + JSON_SAFE_INTEGER_MAX, + MAX_COMPRESSION_RATIO, ProvenanceArchiveError, build_provenance_archive, parse_provenance_archive, @@ -20,7 +23,10 @@ "schema_version": 1, "bundle_uid": "bundle-01HZZ", "source_scope": {"organization_uid": "org-01", "workspace_uid": "ws-01"}, - "export_activity": {"activity_uid": "activity-01"}, + "export_activity": { + "activity_uid": "activity-01", + "date_published": "1980-01-01T00:00:00Z", + }, "emails": [{"email_uid": "email-01", "subject": "Evidence"}], "attachments": [], "content_nodes": [], @@ -42,9 +48,13 @@ def _canonical_json(value): - return json.dumps(value, ensure_ascii=False, allow_nan=False, separators=(",", ":"), sort_keys=True).encode( - "utf-8" - ) + return json.dumps( + value, + ensure_ascii=False, + allow_nan=False, + separators=(",", ":"), + sort_keys=True, + ).encode("utf-8") def _replace_entries(archive, replacements, *, extra=()): @@ -64,7 +74,12 @@ def _rebuild_manifests(entries): f"{hashlib.sha512(entries[name]).hexdigest()} {name}\n".encode("ascii") for name in payload_names ) - tag_names = ("bag-info.txt", "bagit.txt", "manifest-sha512.txt", "ro-crate-metadata.json") + tag_names = ( + "bag-info.txt", + "bagit.txt", + "manifest-sha512.txt", + "ro-crate-metadata.json", + ) entries["tagmanifest-sha512.txt"] = b"".join( f"{hashlib.sha512(entries[name]).hexdigest()} {name}\n".encode("ascii") for name in tag_names @@ -81,7 +96,9 @@ def _archive_with_entries(entries, *, extra=(), fixed_metadata=True): info.create_system = 3 info.external_attr = 0o100644 << 16 info.compress_type = zipfile.ZIP_DEFLATED - target.writestr(info, content, compress_type=zipfile.ZIP_DEFLATED, compresslevel=9) + target.writestr( + info, content, compress_type=zipfile.ZIP_DEFLATED, compresslevel=9 + ) else: target.writestr(name, content) for name, content in extra: @@ -91,6 +108,25 @@ def _archive_with_entries(entries, *, extra=(), fixed_metadata=True): return output.getvalue() +def _archive_with_zip_metadata( + entries, *, archive_comment=b"", member_extra=b"", member_comment=b"" +): + output = io.BytesIO() + with zipfile.ZipFile(output, "w", compression=zipfile.ZIP_DEFLATED) as target: + for name, content in sorted(entries.items()): + info = zipfile.ZipInfo(name, date_time=(1980, 1, 1, 0, 0, 0)) + info.create_system = 3 + info.external_attr = 0o100644 << 16 + info.compress_type = zipfile.ZIP_DEFLATED + info.extra = member_extra + info.comment = member_comment + target.writestr( + info, content, compress_type=zipfile.ZIP_DEFLATED, compresslevel=9 + ) + target.comment = archive_comment + return output.getvalue() + + def test_build_is_deterministic_and_has_exact_fixed_entries(): first = build_provenance_archive(RECORDS) second = build_provenance_archive(dict(RECORDS)) @@ -98,8 +134,12 @@ def test_build_is_deterministic_and_has_exact_fixed_entries(): assert first == second with zipfile.ZipFile(io.BytesIO(first), "r") as archive: assert tuple(sorted(archive.namelist())) == EXPECTED_ENTRIES - assert all(info.date_time == (1980, 1, 1, 0, 0, 0) for info in archive.infolist()) - assert all((info.external_attr >> 16) & 0o777 == 0o644 for info in archive.infolist()) + assert all( + info.date_time == (1980, 1, 1, 0, 0, 0) for info in archive.infolist() + ) + assert all( + (info.external_attr >> 16) & 0o777 == 0o644 for info in archive.infolist() + ) assert archive.read("data/records.json") == _canonical_json(RECORDS) @@ -109,8 +149,20 @@ def test_parse_round_trips_records_and_verifies_ro_crate_metadata(): assert parse_provenance_archive(archive) == RECORDS with zipfile.ZipFile(io.BytesIO(archive), "r") as source: crate = json.loads(source.read("ro-crate-metadata.json")) - assert crate["@context"] == "https://w3id.org/ro/crate/1.3/context" - assert {node["@type"] for node in crate["@graph"]} >= {"Dataset", "File", "CreateAction", "SoftwareApplication"} + assert crate["@context"] == [ + "https://w3id.org/ro/crate/1.3/context", + {"prov": "http://www.w3.org/ns/prov#"}, + ] + nodes = {node["@id"]: node for node in crate["@graph"]} + assert nodes["ro-crate-metadata.json"] == { + "@id": "ro-crate-metadata.json", + "@type": "CreativeWork", + "about": {"@id": "./"}, + "conformsTo": {"@id": "https://w3id.org/ro/crate/1.3"}, + } + assert nodes["./"]["datePublished"] == RECORDS["export_activity"]["date_published"] + assert nodes["#activity-01"]["prov:wasAssociatedWith"] == {"@id": "#naruon"} + assert "prov:SoftwareAgent" in nodes["#naruon"]["@type"] def test_parse_rejects_payload_tampering(): @@ -121,6 +173,20 @@ def test_parse_rejects_payload_tampering(): parse_provenance_archive(tampered) +@pytest.mark.parametrize( + "name", + ("manifest-sha512.txt", "tagmanifest-sha512.txt", "ro-crate-metadata.json"), +) +def test_parse_rejects_direct_tag_and_metadata_tampering(name): + archive = build_provenance_archive(RECORDS) + with zipfile.ZipFile(io.BytesIO(archive), "r") as source: + entries = {info.filename: source.read(info) for info in source.infolist()} + entries[name] += b"x" + + with pytest.raises(ProvenanceArchiveError): + parse_provenance_archive(_archive_with_entries(entries)) + + @pytest.mark.parametrize("name", ("../data/records.json", "data\\records.json")) def test_parse_rejects_unsafe_paths(name): archive = build_provenance_archive(RECORDS) @@ -169,11 +235,60 @@ def test_build_rejects_non_finite_json_numbers(): build_provenance_archive(records) +@pytest.mark.parametrize( + "value", + (-0.0, 1e-7, 1.0, JSON_SAFE_INTEGER_MAX + 1), +) +def test_build_rejects_values_outside_the_stdlib_jcs_subset(value): + with pytest.raises(ProvenanceArchiveError): + build_provenance_archive({**RECORDS, "unsupported": value}) + + +def test_build_rejects_non_ascii_object_keys(): + with pytest.raises(ProvenanceArchiveError): + build_provenance_archive({**RECORDS, "\ufffd": "bmp", "\U0001f600": "non-bmp"}) + + def test_build_rejects_an_unknown_profile(): with pytest.raises(ProvenanceArchiveError): build_provenance_archive({**RECORDS, "profile": "unknown"}) +def test_profile_rejects_boolean_schema_version_and_tag_injection(): + with pytest.raises(ProvenanceArchiveError): + build_provenance_archive({**RECORDS, "schema_version": True}) + with pytest.raises(ProvenanceArchiveError): + build_provenance_archive({**RECORDS, "bundle_uid": "bundle\nInjected: value"}) + with pytest.raises(ProvenanceArchiveError): + build_provenance_archive( + { + **RECORDS, + "export_activity": { + **RECORDS["export_activity"], + "activity_uid": "bad\ruid", + }, + } + ) + + +@pytest.mark.parametrize( + "records_json", + ( + b'{ "profile":"naruon-tenant-provenance/v1"}', + b'{"profile":"naruon-tenant-provenance/v1","schema_version":NaN}', + _canonical_json({**RECORDS, "profile": "unknown"}), + ), +) +def test_parse_rejects_noncanonical_nonfinite_and_unknown_profile(records_json): + archive = build_provenance_archive(RECORDS) + with zipfile.ZipFile(io.BytesIO(archive), "r") as source: + entries = {info.filename: source.read(info) for info in source.infolist()} + entries["data/records.json"] = records_json + + with pytest.raises(ProvenanceArchiveError): + parse_provenance_archive(_archive_with_entries(entries)) + + def test_parse_rejects_nonfixed_zip_metadata(): archive = build_provenance_archive(RECORDS) with zipfile.ZipFile(io.BytesIO(archive), "r") as source: @@ -183,9 +298,40 @@ def test_parse_rejects_nonfixed_zip_metadata(): parse_provenance_archive(_archive_with_entries(entries, fixed_metadata=False)) +@pytest.mark.parametrize( + "metadata", + ( + {"archive_comment": b"archive-comment"}, + {"member_extra": b"\x01\x00\x00\x00"}, + {"member_comment": b"member-comment"}, + ), +) +def test_parse_rejects_zip_comments_and_extra_fields(metadata): + archive = build_provenance_archive(RECORDS) + with zipfile.ZipFile(io.BytesIO(archive), "r") as source: + entries = {info.filename: source.read(info) for info in source.infolist()} + + with pytest.raises(ProvenanceArchiveError): + parse_provenance_archive(_archive_with_zip_metadata(entries, **metadata)) + + +def test_production_archive_limits_are_fixed(): + assert ARCHIVE_MAX_BYTES == 64 * 1024 * 1024 + assert ARCHIVE_MAX_ENTRIES == 64 + assert ENTRY_MAX_BYTES == 32 * 1024 * 1024 + assert MAX_COMPRESSION_RATIO == 100 + + +def test_build_output_always_passes_parser_bounds(): + with pytest.raises(ProvenanceArchiveError): + build_provenance_archive({**RECORDS, "padding": "x" * 1_000_000}) + + def test_parse_enforces_total_uncompressed_bound(monkeypatch): archive = build_provenance_archive({**RECORDS, "padding": "x" * 10_000}) - monkeypatch.setattr("services.tenant_provenance_bundle.ARCHIVE_MAX_BYTES", len(archive) + 1) + monkeypatch.setattr( + "services.tenant_provenance_bundle.ARCHIVE_MAX_BYTES", len(archive) + 1 + ) with pytest.raises(ProvenanceArchiveError): parse_provenance_archive(archive) @@ -196,7 +342,9 @@ def test_parse_rejects_archive_bounds(monkeypatch): with pytest.raises(ProvenanceArchiveError): parse_provenance_archive(build_provenance_archive(RECORDS)) - monkeypatch.setattr("services.tenant_provenance_bundle.ARCHIVE_MAX_BYTES", ARCHIVE_MAX_BYTES) + monkeypatch.setattr( + "services.tenant_provenance_bundle.ARCHIVE_MAX_BYTES", ARCHIVE_MAX_BYTES + ) monkeypatch.setattr("services.tenant_provenance_bundle.ENTRY_MAX_BYTES", 1) with pytest.raises(ProvenanceArchiveError): parse_provenance_archive(build_provenance_archive(RECORDS)) @@ -211,7 +359,9 @@ def test_parse_rejects_entry_count_and_compression_ratio(monkeypatch): with pytest.raises(ProvenanceArchiveError): parse_provenance_archive(_archive_with_entries(entries)) - monkeypatch.setattr("services.tenant_provenance_bundle.ARCHIVE_MAX_ENTRIES", ARCHIVE_MAX_ENTRIES) + monkeypatch.setattr( + "services.tenant_provenance_bundle.ARCHIVE_MAX_ENTRIES", ARCHIVE_MAX_ENTRIES + ) records = _canonical_json({**RECORDS, "padding": "x" * 10_000}) entries["data/records.json"] = records with pytest.raises(ProvenanceArchiveError): From dfc2caf387f7ebc8068faf6d998d27fe05990d92 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Mon, 31 Aug 2026 20:47:17 +0900 Subject: [PATCH 03/39] fix: enforce provenance archive profile bounds --- backend/services/tenant_provenance_bundle.py | 99 +++++++++++++++++-- .../tests/test_tenant_provenance_bundle.py | 94 ++++++++++++++++++ 2 files changed, 184 insertions(+), 9 deletions(-) diff --git a/backend/services/tenant_provenance_bundle.py b/backend/services/tenant_provenance_bundle.py index 3b32d46bf..42d44e530 100644 --- a/backend/services/tenant_provenance_bundle.py +++ b/backend/services/tenant_provenance_bundle.py @@ -6,8 +6,10 @@ import io import json import stat +import struct import zipfile from collections.abc import Mapping +from datetime import date, datetime from typing import Any @@ -20,6 +22,9 @@ _MAX_IDENTIFIER_LENGTH = 256 _FIXED_TIMESTAMP = (1980, 1, 1, 0, 0, 0) _PAYLOAD_NAME = "data/records.json" +_LOCAL_FILE_SIGNATURE = b"PK\x03\x04" +_EOCD_SIGNATURE = b"PK\x05\x06" +_EOCD_SIZE = 22 _EXPECTED_ENTRIES = frozenset( { "bagit.txt", @@ -102,6 +107,18 @@ def _safe_identifier(value: object) -> str: return value +def _iso8601_date_or_datetime(value: object) -> str: + date_published = _safe_identifier(value) + try: + date.fromisoformat(date_published) + except ValueError: + try: + datetime.fromisoformat(date_published) + except ValueError: + _fail() + return date_published + + def _bag_info(bundle_uid: str) -> bytes: return ( "Bag-Software-Agent: naruon\n" @@ -115,7 +132,7 @@ def _ro_crate(records: Mapping[str, object], payload_digest: str) -> bytes: if not isinstance(activity, Mapping): _fail() activity_uid = _safe_identifier(activity.get("activity_uid")) - date_published = _safe_identifier(activity.get("date_published")) + date_published = _iso8601_date_or_datetime(activity.get("date_published")) crate = { "@context": [ "https://w3id.org/ro/crate/1.3/context", @@ -258,12 +275,64 @@ def _has_fixed_metadata(info: zipfile.ZipInfo) -> bool: ) +def _within_archive_bounds( + *, + archive_bytes: int, + entry_count: int, + total_bytes: int, + entry_bytes: int, + compressed_bytes: int, +) -> bool: + return ( + 0 <= archive_bytes <= ARCHIVE_MAX_BYTES + and 0 <= entry_count <= ARCHIVE_MAX_ENTRIES + and 0 <= total_bytes <= ARCHIVE_MAX_BYTES + and 0 <= entry_bytes <= ENTRY_MAX_BYTES + and compressed_bytes >= 0 + and ( + entry_bytes == 0 + or 0 < compressed_bytes + and entry_bytes <= compressed_bytes * MAX_COMPRESSION_RATIO + ) + ) + + +def _has_profile_container_framing(archive_bytes: bytes | bytearray) -> bool: + if ( + len(archive_bytes) < _EOCD_SIZE + or not archive_bytes.startswith(_LOCAL_FILE_SIGNATURE) + or archive_bytes[-_EOCD_SIZE:-18] != _EOCD_SIGNATURE + ): + return False + ( + _, + disk_number, + directory_disk, + entries_on_disk, + entries, + directory_size, + directory_offset, + comment_size, + ) = struct.unpack("<4s4H2LH", archive_bytes[-_EOCD_SIZE:]) + return ( + disk_number == 0 + and directory_disk == 0 + and entries_on_disk == entries + and entries != 0xFFFF + and directory_size != 0xFFFFFFFF + and directory_offset != 0xFFFFFFFF + and comment_size == 0 + and directory_offset + directory_size == len(archive_bytes) - _EOCD_SIZE + ) + + def _read_entry(archive: zipfile.ZipFile, info: zipfile.ZipInfo) -> bytes: - if info.file_size > ENTRY_MAX_BYTES: - _fail() - if info.file_size and ( - not info.compress_size - or info.file_size / info.compress_size > MAX_COMPRESSION_RATIO + if not _within_archive_bounds( + archive_bytes=0, + entry_count=0, + total_bytes=0, + entry_bytes=info.file_size, + compressed_bytes=info.compress_size, ): _fail() data = bytearray() @@ -309,7 +378,14 @@ def parse_provenance_archive(archive_bytes: bytes) -> dict[str, object]: """Validate a bounded fixed envelope and return its canonical record payload.""" if ( not isinstance(archive_bytes, (bytes, bytearray)) - or len(archive_bytes) > ARCHIVE_MAX_BYTES + or not _within_archive_bounds( + archive_bytes=len(archive_bytes), + entry_count=0, + total_bytes=0, + entry_bytes=0, + compressed_bytes=0, + ) + or not _has_profile_container_framing(archive_bytes) ): _fail() try: @@ -317,8 +393,13 @@ def parse_provenance_archive(archive_bytes: bytes) -> dict[str, object]: infos = archive.infolist() if ( archive.comment - or len(infos) > ARCHIVE_MAX_ENTRIES - or sum(info.file_size for info in infos) > ARCHIVE_MAX_BYTES + or not _within_archive_bounds( + archive_bytes=len(archive_bytes), + entry_count=len(infos), + total_bytes=sum(info.file_size for info in infos), + entry_bytes=0, + compressed_bytes=0, + ) or any( _is_unsafe_member(info) or not _has_fixed_metadata(info) for info in infos diff --git a/backend/tests/test_tenant_provenance_bundle.py b/backend/tests/test_tenant_provenance_bundle.py index aa331313c..ce5700bef 100644 --- a/backend/tests/test_tenant_provenance_bundle.py +++ b/backend/tests/test_tenant_provenance_bundle.py @@ -13,6 +13,7 @@ JSON_SAFE_INTEGER_MAX, MAX_COMPRESSION_RATIO, ProvenanceArchiveError, + _within_archive_bounds, build_provenance_archive, parse_provenance_archive, ) @@ -165,6 +166,29 @@ def test_parse_round_trips_records_and_verifies_ro_crate_metadata(): assert "prov:SoftwareAgent" in nodes["#naruon"]["@type"] +@pytest.mark.parametrize("date_published", ("not-a-date", "2026-02-30")) +def test_build_rejects_non_iso_ro_crate_date_published(date_published): + with pytest.raises(ProvenanceArchiveError): + build_provenance_archive( + { + **RECORDS, + "export_activity": { + **RECORDS["export_activity"], + "date_published": date_published, + }, + } + ) + + +def test_build_preserves_valid_iso_ro_crate_date_published(): + archive = build_provenance_archive(RECORDS) + + assert ( + parse_provenance_archive(archive)["export_activity"]["date_published"] + == "1980-01-01T00:00:00Z" + ) + + def test_parse_rejects_payload_tampering(): archive = build_provenance_archive(RECORDS) tampered = _replace_entries(archive, {"data/records.json": b'{"tampered":true}'}) @@ -315,6 +339,34 @@ def test_parse_rejects_zip_comments_and_extra_fields(metadata): parse_provenance_archive(_archive_with_zip_metadata(entries, **metadata)) +@pytest.mark.parametrize( + "container_bytes", (b"leading-", b"trailing-unvalidated-bytes") +) +def test_parse_rejects_leading_and_trailing_container_bytes(container_bytes): + archive = build_provenance_archive(RECORDS) + candidate = ( + container_bytes + archive + if container_bytes == b"leading-" + else archive + container_bytes + ) + + with pytest.raises(ProvenanceArchiveError): + parse_provenance_archive(candidate) + + +@pytest.mark.parametrize( + "archive", + ( + lambda value: value[:-1], + lambda value: value[:-22] + b"truncated", + lambda value: value[:-22] + b"gap" + value[-22:], + ), +) +def test_parse_rejects_truncated_or_malformed_eocd(archive): + with pytest.raises(ProvenanceArchiveError): + parse_provenance_archive(archive(build_provenance_archive(RECORDS))) + + def test_production_archive_limits_are_fixed(): assert ARCHIVE_MAX_BYTES == 64 * 1024 * 1024 assert ARCHIVE_MAX_ENTRIES == 64 @@ -322,6 +374,48 @@ def test_production_archive_limits_are_fixed(): assert MAX_COMPRESSION_RATIO == 100 +@pytest.mark.parametrize( + ( + "archive_bytes", + "entry_count", + "total_bytes", + "entry_bytes", + "compressed_bytes", + "expected", + ), + ( + (ARCHIVE_MAX_BYTES - 1, 0, 0, 0, 0, True), + (ARCHIVE_MAX_BYTES, 0, 0, 0, 0, True), + (ARCHIVE_MAX_BYTES + 1, 0, 0, 0, 0, False), + (0, ARCHIVE_MAX_ENTRIES - 1, 0, 0, 0, True), + (0, ARCHIVE_MAX_ENTRIES, 0, 0, 0, True), + (0, ARCHIVE_MAX_ENTRIES + 1, 0, 0, 0, False), + (0, 0, ARCHIVE_MAX_BYTES - 1, 0, 0, True), + (0, 0, ARCHIVE_MAX_BYTES, 0, 0, True), + (0, 0, ARCHIVE_MAX_BYTES + 1, 0, 0, False), + (0, 0, 0, ENTRY_MAX_BYTES - 1, ENTRY_MAX_BYTES, True), + (0, 0, 0, ENTRY_MAX_BYTES, ENTRY_MAX_BYTES, True), + (0, 0, 0, ENTRY_MAX_BYTES + 1, ENTRY_MAX_BYTES + 1, False), + (0, 0, 0, MAX_COMPRESSION_RATIO - 1, 1, True), + (0, 0, 0, MAX_COMPRESSION_RATIO, 1, True), + (0, 0, 0, MAX_COMPRESSION_RATIO + 1, 1, False), + ), +) +def test_production_archive_limits_boundary( + archive_bytes, entry_count, total_bytes, entry_bytes, compressed_bytes, expected +): + assert ( + _within_archive_bounds( + archive_bytes=archive_bytes, + entry_count=entry_count, + total_bytes=total_bytes, + entry_bytes=entry_bytes, + compressed_bytes=compressed_bytes, + ) + is expected + ) + + def test_build_output_always_passes_parser_bounds(): with pytest.raises(ProvenanceArchiveError): build_provenance_archive({**RECORDS, "padding": "x" * 1_000_000}) From 80f2fe36b5f533ce92eb321d7750bc51c92a0fa0 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Mon, 31 Aug 2026 20:54:38 +0900 Subject: [PATCH 04/39] fix: reject provenance archive interior gaps --- backend/services/tenant_provenance_bundle.py | 15 +++++++++++++++ backend/tests/test_tenant_provenance_bundle.py | 14 ++++++++++++++ 2 files changed, 29 insertions(+) diff --git a/backend/services/tenant_provenance_bundle.py b/backend/services/tenant_provenance_bundle.py index 42d44e530..3b716b6a4 100644 --- a/backend/services/tenant_provenance_bundle.py +++ b/backend/services/tenant_provenance_bundle.py @@ -326,6 +326,20 @@ def _has_profile_container_framing(archive_bytes: bytes | bytearray) -> bool: ) +def _has_contiguous_member_data(infos: list[zipfile.ZipInfo], start_dir: int) -> bool: + expected_offset = 0 + for info in infos: + if info.header_offset != expected_offset: + return False + expected_offset += ( + zipfile.sizeFileHeader + + len(info.filename.encode("ascii")) + + len(info.extra) + + info.compress_size + ) + return expected_offset == start_dir + + def _read_entry(archive: zipfile.ZipFile, info: zipfile.ZipInfo) -> bytes: if not _within_archive_bounds( archive_bytes=0, @@ -411,6 +425,7 @@ def parse_provenance_archive(archive_bytes: bytes) -> dict[str, object]: names != sorted(names) or len(set(names)) != len(names) or set(names) != _EXPECTED_ENTRIES + or not _has_contiguous_member_data(infos, archive.start_dir) ): _fail() entries = {info.filename: _read_entry(archive, info) for info in infos} diff --git a/backend/tests/test_tenant_provenance_bundle.py b/backend/tests/test_tenant_provenance_bundle.py index ce5700bef..17b2ef7bd 100644 --- a/backend/tests/test_tenant_provenance_bundle.py +++ b/backend/tests/test_tenant_provenance_bundle.py @@ -1,6 +1,7 @@ import hashlib import io import json +import struct import warnings import zipfile @@ -367,6 +368,19 @@ def test_parse_rejects_truncated_or_malformed_eocd(archive): parse_provenance_archive(archive(build_provenance_archive(RECORDS))) +def test_parse_rejects_gap_before_central_directory(): + archive = build_provenance_archive(RECORDS) + assert parse_provenance_archive(archive) == RECORDS + with zipfile.ZipFile(io.BytesIO(archive), "r") as source: + start_dir = source.start_dir + gap = b"unaccounted-gap" + candidate = bytearray(archive[:start_dir] + gap + archive[start_dir:]) + struct.pack_into(" Date: Mon, 31 Aug 2026 22:04:12 +0900 Subject: [PATCH 05/39] feat: round trip tenant provenance closure --- backend/services/tenant_provenance_bundle.py | 1331 ++++++++++++++++- .../tests/test_tenant_provenance_bundle.py | 552 +++++++ 2 files changed, 1882 insertions(+), 1 deletion(-) diff --git a/backend/services/tenant_provenance_bundle.py b/backend/services/tenant_provenance_bundle.py index 3b716b6a4..7b1714609 100644 --- a/backend/services/tenant_provenance_bundle.py +++ b/backend/services/tenant_provenance_bundle.py @@ -5,13 +5,30 @@ import hashlib import io import json +import math import stat import struct import zipfile from collections.abc import Mapping -from datetime import date, datetime +from dataclasses import dataclass +from datetime import date, datetime, timezone from typing import Any +from sqlalchemy import select +from sqlalchemy.exc import DataError, IntegrityError, StatementError +from sqlalchemy.ext.asyncio import AsyncSession + +from db.models import ( + Attachment, + ContentNodeRecord, + ContentSegmentRecord, + Email, + KnowledgeGraphEdgeRecord, + ProjectGraphCorrectionRecord, + ProjectGraphEdgeRecord, + ProjectGraphObjectRecord, +) + ARCHIVE_MAX_BYTES = 64 * 1024 * 1024 ARCHIVE_MAX_ENTRIES = 64 @@ -35,12 +52,166 @@ _PAYLOAD_NAME, } ) +_COLLECTIONS = ( + "emails", + "attachments", + "content_nodes", + "content_segments", + "structural_edges", + "project_objects", + "project_edges", + "corrections", +) +_TEXTUAL_PARSER_KEYS = frozenset( + {"plain_text", "html", "markdown", "json", "csv", "xml", "calendar", "pdf"} +) +_RECORD_KEYS = { + "emails": frozenset( + { + "email_uid", + "thread_uid", + "fingerprint", + "sender", + "reply_to", + "recipients", + "subject", + "in_reply_to", + "references", + "date", + "body", + "is_read", + } + ), + "attachments": frozenset( + { + "attachment_uid", + "email_uid", + "filename", + "content", + "content_type", + "parse_status", + "parse_content_type", + "parser_key", + "parse_error_code", + } + ), + "content_nodes": frozenset( + { + "content_node_uid", + "email_uid", + "attachment_uid", + "source_kind", + "source_record_uid", + "parent_node_uid", + "node_kind", + "node_path", + "ordinal_index", + "display_label", + "safe_text_content", + "content_hash", + } + ), + "content_segments": frozenset( + { + "content_segment_uid", + "email_uid", + "attachment_uid", + "content_node_uid", + "source_kind", + "source_record_uid", + "segment_kind", + "segment_path", + "ordinal_index", + "heading_path", + "safe_text_content", + "content_hash", + "word_count", + } + ), + "structural_edges": frozenset( + { + "edge_uid", + "email_uid", + "attachment_uid", + "source_node_uid", + "target_node_uid", + "source_segment_uid", + "target_segment_uid", + "source_kind", + "source_record_uid", + "edge_kind", + "edge_path", + "ordinal_index", + } + ), + "project_objects": frozenset( + { + "object_uid", + "email_uid", + "attachment_uid", + "primary_content_segment_uid", + "object_type", + "title", + "summary", + "status_code", + "confidence", + "source_segment_uids", + "attributes_json", + "extractor_name", + "extractor_version", + } + ), + "project_edges": frozenset( + { + "edge_uid", + "source_uid", + "target_uid", + "edge_type", + "confidence", + "source_segment_uids", + "source_object_uid", + "target_object_uid", + "primary_content_segment_uid", + } + ), + "corrections": frozenset( + { + "correction_uid", + "object_uid", + "correction_action", + "before_json", + "after_json", + "rationale", + "source_segment_uids", + "created_at", + } + ), +} class ProvenanceArchiveError(ValueError): """Raised when a provenance envelope is malformed or outside this profile.""" +@dataclass(frozen=True) +class TenantProvenanceScope: + """Signed-session authority used for every source read and target write.""" + + user_id: str + organization_id: str | None + workspace_id: str + + +@dataclass(frozen=True) +class ImportReceipt: + """Verified import outcome without exposing persistence identifiers.""" + + bundle_uid: str + manifest_digest: str + created: dict[str, int] + skipped: dict[str, int] + + def _fail() -> None: raise ProvenanceArchiveError("Invalid provenance archive") @@ -437,3 +608,1161 @@ def parse_provenance_archive(archive_bytes: bytes) -> dict[str, object]: if entries != expected: _fail() return records + + +def _scope_filters(model: Any, scope: TenantProvenanceScope, *, workspace: bool): + organization_filter = ( + model.organization_id == scope.organization_id + if scope.organization_id is not None + else model.organization_id.is_(None) + ) + filters = [model.user_id == scope.user_id, organization_filter] + if workspace: + filters.append(model.workspace_id == scope.workspace_id) + return filters + + +def _utc_text(value: datetime) -> str: + if value.tzinfo is None: + _fail() + return value.astimezone(timezone.utc).isoformat().replace("+00:00", "Z") + + +def _parse_datetime(value: object) -> datetime: + text_value = _safe_identifier(value) + try: + parsed = datetime.fromisoformat(text_value) + except ValueError as exc: + raise ProvenanceArchiveError("Invalid provenance archive") from exc + if parsed.tzinfo is None: + _fail() + return parsed + + +def _confidence(value: float) -> str: + if not math.isfinite(value): + _fail() + return repr(value) + + +def _parse_confidence(value: object) -> float: + if not isinstance(value, str): + _fail() + try: + parsed = float(value) + except ValueError as exc: + raise ProvenanceArchiveError("Invalid provenance archive") from exc + if not math.isfinite(parsed): + _fail() + return parsed + + +def _attachment_core(attachment: Attachment, email_uid: str) -> dict[str, object]: + return { + "email_uid": email_uid, + "filename": attachment.filename, + "content": attachment.content, + "content_type": attachment.content_type, + "parse_status": attachment.parse_status, + "parse_content_type": attachment.parse_content_type, + "parser_key": attachment.parser_key, + "parse_error_code": attachment.parse_error_code, + } + + +def _is_admitted_attachment(attachment: Attachment) -> bool: + return ( + attachment.parse_status == "parsed" + and attachment.parser_key in _TEXTUAL_PARSER_KEYS + ) + + +def _attachment_records( + attachments: list[Attachment], email_uids: Mapping[int, str] +) -> tuple[list[dict[str, object]], dict[int, str]]: + admitted = [ + attachment + for attachment in attachments + if attachment.email_id in email_uids and _is_admitted_attachment(attachment) + ] + admitted.sort( + key=lambda item: ( + _canonical_json(_attachment_core(item, email_uids[item.email_id])), + item.id, + ) + ) + occurrence: dict[bytes, int] = {} + records: list[dict[str, object]] = [] + uid_by_id: dict[int, str] = {} + for attachment in admitted: + core = _attachment_core(attachment, email_uids[attachment.email_id]) + canonical = _canonical_json(core) + occurrence[canonical] = occurrence.get(canonical, 0) + 1 + uid = ( + "attachment-" + + hashlib.sha256( + canonical + b":" + str(occurrence[canonical]).encode("ascii") + ).hexdigest() + ) + uid_by_id[attachment.id] = uid + records.append({"attachment_uid": uid, **core}) + return sorted(records, key=lambda item: item["attachment_uid"]), uid_by_id + + +def _email_record(email: Email) -> dict[str, object]: + return { + "email_uid": email.message_id, + "thread_uid": email.thread_id, + "fingerprint": email.fingerprint, + "sender": email.sender, + "reply_to": email.reply_to, + "recipients": email.recipients, + "subject": email.subject, + "in_reply_to": email.in_reply_to, + "references": email.references, + "date": _utc_text(email.date), + "body": email.body, + "is_read": email.is_read, + } + + +def _node_record( + node: ContentNodeRecord, + email_uids: Mapping[int, str], + attachment_uids: Mapping[int, str], +) -> dict[str, object]: + return { + "content_node_uid": node.content_node_uid, + "email_uid": email_uids[node.email_id], + "attachment_uid": attachment_uids.get(node.attachment_id), + "source_kind": node.source_kind, + "source_record_uid": node.source_record_uid, + "parent_node_uid": node.parent_node_uid, + "node_kind": node.node_kind, + "node_path": node.node_path, + "ordinal_index": node.ordinal_index, + "display_label": node.display_label, + "safe_text_content": node.safe_text_content, + "content_hash": node.content_hash, + } + + +def _segment_record( + segment: ContentSegmentRecord, + email_uids: Mapping[int, str], + attachment_uids: Mapping[int, str], + node_uids: Mapping[int, str], +) -> dict[str, object]: + return { + "content_segment_uid": segment.content_segment_uid, + "email_uid": email_uids[segment.email_id], + "attachment_uid": attachment_uids.get(segment.attachment_id), + "content_node_uid": node_uids[segment.content_node_id], + "source_kind": segment.source_kind, + "source_record_uid": segment.source_record_uid, + "segment_kind": segment.segment_kind, + "segment_path": segment.segment_path, + "ordinal_index": segment.ordinal_index, + "heading_path": segment.heading_path, + "safe_text_content": segment.safe_text_content, + "content_hash": segment.content_hash, + "word_count": segment.word_count, + } + + +def _structural_edge_record( + edge: KnowledgeGraphEdgeRecord, + email_uids: Mapping[int, str], + attachment_uids: Mapping[int, str], + node_uids: Mapping[int, str], + segment_uids: Mapping[int, str], +) -> dict[str, object]: + return { + "edge_uid": edge.edge_uid, + "email_uid": email_uids[edge.email_id], + "attachment_uid": attachment_uids.get(edge.attachment_id), + "source_node_uid": node_uids.get(edge.source_node_id), + "target_node_uid": node_uids.get(edge.target_node_id), + "source_segment_uid": segment_uids.get(edge.source_segment_id), + "target_segment_uid": segment_uids.get(edge.target_segment_id), + "source_kind": edge.source_kind, + "source_record_uid": edge.source_record_uid, + "edge_kind": edge.edge_kind, + "edge_path": edge.edge_path, + "ordinal_index": edge.ordinal_index, + } + + +def _project_object_record( + project_object: ProjectGraphObjectRecord, + email_uids: Mapping[int, str], + attachment_uids: Mapping[int, str], + segment_uids: Mapping[int, str], +) -> dict[str, object]: + return { + "object_uid": project_object.object_uid, + "email_uid": email_uids[project_object.email_id], + "attachment_uid": attachment_uids.get(project_object.attachment_id), + "primary_content_segment_uid": segment_uids[ + project_object.primary_content_segment_id + ], + "object_type": project_object.object_type, + "title": project_object.title, + "summary": project_object.summary, + "status_code": project_object.status_code, + "confidence": _confidence(project_object.confidence), + "source_segment_uids": sorted(project_object.source_segment_uids), + "attributes_json": project_object.attributes_json, + "extractor_name": project_object.extractor_name, + "extractor_version": project_object.extractor_version, + } + + +def _project_edge_record( + edge: ProjectGraphEdgeRecord, + object_uids: Mapping[int, str], + segment_uids: Mapping[int, str], +) -> dict[str, object]: + return { + "edge_uid": edge.edge_uid, + "source_uid": edge.source_uid, + "target_uid": edge.target_uid, + "edge_type": edge.edge_type, + "confidence": _confidence(edge.confidence), + "source_segment_uids": sorted(edge.source_segment_uids), + "source_object_uid": object_uids.get(edge.source_object_id), + "target_object_uid": object_uids.get(edge.target_object_id), + "primary_content_segment_uid": segment_uids[edge.primary_content_segment_id], + } + + +def _correction_record( + correction: ProjectGraphCorrectionRecord, + object_uids: Mapping[int, str], +) -> dict[str, object]: + return { + "correction_uid": correction.correction_uid, + "object_uid": object_uids[correction.project_graph_object_id], + "correction_action": correction.correction_action, + "before_json": correction.before_json, + "after_json": correction.after_json, + "rationale": correction.rationale, + "source_segment_uids": sorted(correction.source_segment_uids), + "created_at": _utc_text(correction.created_at), + } + + +async def export_tenant_provenance( + session: AsyncSession, scope: TenantProvenanceScope +) -> bytes: + """Export the exact signed-scope project-evidence closure.""" + _validate_scope(scope) + project_objects = list( + ( + await session.scalars( + select(ProjectGraphObjectRecord) + .where(*_scope_filters(ProjectGraphObjectRecord, scope, workspace=True)) + .order_by(ProjectGraphObjectRecord.object_uid) + ) + ).all() + ) + email_ids = {record.email_id for record in project_objects} + emails = list( + ( + await session.scalars( + select(Email) + .where( + Email.id.in_(email_ids), + *_scope_filters(Email, scope, workspace=False), + ) + .order_by(Email.message_id) + ) + ).all() + ) + if len(emails) != len(email_ids): + _fail() + email_uids = {email.id: email.message_id for email in emails} + + async def descendants(model: Any, order_column: Any) -> list[Any]: + return list( + ( + await session.scalars( + select(model) + .where(model.email_id.in_(email_ids)) + .order_by(order_column) + ) + ).all() + ) + + attachments = await descendants(Attachment, Attachment.id) + nodes = await descendants(ContentNodeRecord, ContentNodeRecord.content_node_uid) + segments = await descendants( + ContentSegmentRecord, ContentSegmentRecord.content_segment_uid + ) + structural_edges = await descendants( + KnowledgeGraphEdgeRecord, KnowledgeGraphEdgeRecord.edge_uid + ) + project_edges = list( + ( + await session.scalars( + select(ProjectGraphEdgeRecord) + .where(*_scope_filters(ProjectGraphEdgeRecord, scope, workspace=True)) + .order_by(ProjectGraphEdgeRecord.edge_uid) + ) + ).all() + ) + corrections = list( + ( + await session.scalars( + select(ProjectGraphCorrectionRecord) + .where( + *_scope_filters(ProjectGraphCorrectionRecord, scope, workspace=True) + ) + .order_by(ProjectGraphCorrectionRecord.correction_uid) + ) + ).all() + ) + + attachment_records, attachment_uids = _attachment_records(attachments, email_uids) + node_uids = {node.content_node_id: node.content_node_uid for node in nodes} + segment_uids = { + segment.content_segment_id: segment.content_segment_uid for segment in segments + } + object_uids = { + project_object.project_graph_object_id: project_object.object_uid + for project_object in project_objects + } + payload = { + "source_scope": { + "organization_uid": scope.organization_id or "unscoped", + "workspace_uid": scope.workspace_id, + }, + "emails": [_email_record(email) for email in emails], + "attachments": attachment_records, + "content_nodes": [ + _node_record(node, email_uids, attachment_uids) for node in nodes + ], + "content_segments": [ + _segment_record(segment, email_uids, attachment_uids, node_uids) + for segment in segments + ], + "structural_edges": [ + _structural_edge_record( + edge, email_uids, attachment_uids, node_uids, segment_uids + ) + for edge in structural_edges + ], + "project_objects": [ + _project_object_record( + project_object, email_uids, attachment_uids, segment_uids + ) + for project_object in project_objects + ], + "project_edges": [ + _project_edge_record(edge, object_uids, segment_uids) + for edge in project_edges + ], + "corrections": [ + _correction_record(correction, object_uids) for correction in corrections + ], + } + content_digest = hashlib.sha256(_canonical_json(payload)).hexdigest() + records = { + "profile": "naruon-tenant-provenance/v1", + "schema_version": 1, + "bundle_uid": f"bundle-{content_digest}", + **payload, + "export_activity": { + "activity_uid": f"export-{content_digest}", + "date_published": "1980-01-01T00:00:00Z", + }, + } + _validate_record_graph(records) + return build_provenance_archive(records) + + +def _validate_scope(scope: TenantProvenanceScope) -> None: + if not isinstance(scope, TenantProvenanceScope): + _fail() + _safe_identifier(scope.user_id) + if scope.organization_id is not None: + _safe_identifier(scope.organization_id) + _safe_identifier(scope.workspace_id) + + +def _record_mapping(value: object, expected_keys: frozenset[str]) -> dict[str, object]: + if not isinstance(value, dict) or set(value) != expected_keys: + _fail() + return value + + +def _required_text(record: Mapping[str, object], key: str) -> str: + value = record.get(key) + if not isinstance(value, str): + _fail() + return value + + +def _optional_text(record: Mapping[str, object], key: str) -> str | None: + value = record.get(key) + if value is not None and not isinstance(value, str): + _fail() + return value + + +def _uid( + record: Mapping[str, object], key: str, *, optional: bool = False +) -> str | None: + value = _optional_text(record, key) if optional else _required_text(record, key) + return None if value is None else _safe_identifier(value) + + +def _integer(record: Mapping[str, object], key: str, *, minimum: int = 0) -> int: + value = record.get(key) + if type(value) is not int or value < minimum: + _fail() + return value + + +def _uid_list(record: Mapping[str, object], key: str) -> list[str]: + value = record.get(key) + if not isinstance(value, list): + _fail() + validated = [_safe_identifier(item) for item in value] + if validated != sorted(set(validated)): + _fail() + return validated + + +def _collection_records( + records: Mapping[str, object], collection: str, uid_key: str +) -> tuple[list[dict[str, object]], dict[str, dict[str, object]]]: + values = records.get(collection) + if not isinstance(values, list): + _fail() + validated = [_record_mapping(value, _RECORD_KEYS[collection]) for value in values] + keys = [_safe_identifier(value.get(uid_key)) for value in validated] + if keys != sorted(set(keys)): + _fail() + return validated, dict(zip(keys, validated, strict=True)) + + +def _validate_record_scalars( + collections: Mapping[str, list[dict[str, object]]], +) -> None: + for email in collections["emails"]: + _uid(email, "email_uid") + for key in ( + "thread_uid", + "fingerprint", + "reply_to", + "recipients", + "subject", + "in_reply_to", + "references", + ): + _optional_text(email, key) + for key in ("sender", "body"): + _required_text(email, key) + _parse_datetime(email.get("date")) + if type(email.get("is_read")) is not bool: + _fail() + + for attachment in collections["attachments"]: + _uid(attachment, "attachment_uid") + _uid(attachment, "email_uid") + for key in ( + "filename", + "content", + "content_type", + "parse_status", + "parse_content_type", + "parser_key", + ): + _required_text(attachment, key) + _optional_text(attachment, "parse_error_code") + if ( + attachment["parse_status"] != "parsed" + or attachment["parser_key"] not in _TEXTUAL_PARSER_KEYS + ): + _fail() + + for node in collections["content_nodes"]: + for key in ("content_node_uid", "email_uid"): + _uid(node, key) + for key in ("attachment_uid", "parent_node_uid"): + _uid(node, key, optional=True) + for key in ( + "source_kind", + "source_record_uid", + "node_kind", + "node_path", + "safe_text_content", + "content_hash", + ): + _required_text(node, key) + _optional_text(node, "display_label") + _integer(node, "ordinal_index") + + for segment in collections["content_segments"]: + for key in ("content_segment_uid", "email_uid", "content_node_uid"): + _uid(segment, key) + _uid(segment, "attachment_uid", optional=True) + for key in ( + "source_kind", + "source_record_uid", + "segment_kind", + "segment_path", + "safe_text_content", + "content_hash", + ): + _required_text(segment, key) + _optional_text(segment, "heading_path") + _integer(segment, "ordinal_index") + _integer(segment, "word_count") + + for edge in collections["structural_edges"]: + for key in ("edge_uid", "email_uid"): + _uid(edge, key) + for key in ( + "attachment_uid", + "source_node_uid", + "target_node_uid", + "source_segment_uid", + "target_segment_uid", + ): + _uid(edge, key, optional=True) + for key in ("source_kind", "source_record_uid", "edge_kind", "edge_path"): + _required_text(edge, key) + _integer(edge, "ordinal_index") + + for project_object in collections["project_objects"]: + for key in ( + "object_uid", + "email_uid", + "primary_content_segment_uid", + ): + _uid(project_object, key) + _uid(project_object, "attachment_uid", optional=True) + for key in ( + "object_type", + "title", + "summary", + "status_code", + "extractor_name", + "extractor_version", + ): + _required_text(project_object, key) + _parse_confidence(project_object.get("confidence")) + _uid_list(project_object, "source_segment_uids") + if not isinstance(project_object.get("attributes_json"), dict): + _fail() + + for edge in collections["project_edges"]: + for key in ( + "edge_uid", + "source_uid", + "target_uid", + "primary_content_segment_uid", + ): + _uid(edge, key) + for key in ("source_object_uid", "target_object_uid"): + _uid(edge, key, optional=True) + _required_text(edge, "edge_type") + _parse_confidence(edge.get("confidence")) + _uid_list(edge, "source_segment_uids") + + for correction in collections["corrections"]: + for key in ("correction_uid", "object_uid"): + _uid(correction, key) + _required_text(correction, "correction_action") + _optional_text(correction, "rationale") + _uid_list(correction, "source_segment_uids") + if not isinstance(correction.get("before_json"), dict) or not isinstance( + correction.get("after_json"), dict + ): + _fail() + _parse_datetime(correction.get("created_at")) + + +def _validate_record_graph(records: Mapping[str, object]) -> None: + expected_top_level = { + "profile", + "schema_version", + "bundle_uid", + "source_scope", + "export_activity", + *_COLLECTIONS, + } + if set(records) != expected_top_level: + _fail() + _records_bundle_uid(records) + source_scope = records.get("source_scope") + if not isinstance(source_scope, dict) or set(source_scope) != { + "organization_uid", + "workspace_uid", + }: + _fail() + _safe_identifier(source_scope.get("organization_uid")) + _safe_identifier(source_scope.get("workspace_uid")) + activity = records.get("export_activity") + if not isinstance(activity, dict) or set(activity) != { + "activity_uid", + "date_published", + }: + _fail() + _safe_identifier(activity.get("activity_uid")) + _iso8601_date_or_datetime(activity.get("date_published")) + + uid_keys = { + "emails": "email_uid", + "attachments": "attachment_uid", + "content_nodes": "content_node_uid", + "content_segments": "content_segment_uid", + "structural_edges": "edge_uid", + "project_objects": "object_uid", + "project_edges": "edge_uid", + "corrections": "correction_uid", + } + collections: dict[str, list[dict[str, object]]] = {} + indexed: dict[str, dict[str, dict[str, object]]] = {} + for collection, uid_key in uid_keys.items(): + collections[collection], indexed[collection] = _collection_records( + records, collection, uid_key + ) + _validate_record_scalars(collections) + + email_uids = set(indexed["emails"]) + attachment_uids = set(indexed["attachments"]) + node_uids = set(indexed["content_nodes"]) + segment_uids = set(indexed["content_segments"]) + object_uids = set(indexed["project_objects"]) + attachment_email = { + record["attachment_uid"]: record["email_uid"] + for record in collections["attachments"] + } + attachment_groups: dict[bytes, list[str]] = {} + for record in collections["attachments"]: + canonical = _canonical_json( + {key: value for key, value in record.items() if key != "attachment_uid"} + ) + attachment_groups.setdefault(canonical, []).append(record["attachment_uid"]) + for canonical, actual_uids in attachment_groups.items(): + expected_uids = { + "attachment-" + + hashlib.sha256( + canonical + b":" + str(occurrence).encode("ascii") + ).hexdigest() + for occurrence in range(1, len(actual_uids) + 1) + } + if set(actual_uids) != expected_uids: + _fail() + node_email = { + record["content_node_uid"]: record["email_uid"] + for record in collections["content_nodes"] + } + segment_email = { + record["content_segment_uid"]: record["email_uid"] + for record in collections["content_segments"] + } + + def require_reference(value: object, available: set[str]) -> None: + if value is not None and value not in available: + _fail() + + for record in collections["attachments"]: + require_reference(record["email_uid"], email_uids) + for record in collections["content_nodes"]: + email_uid = record["email_uid"] + require_reference(email_uid, email_uids) + require_reference(record["attachment_uid"], attachment_uids) + require_reference(record["parent_node_uid"], node_uids) + if ( + record["attachment_uid"] is not None + and attachment_email[record["attachment_uid"]] != email_uid + ): + _fail() + if ( + record["parent_node_uid"] is not None + and node_email[record["parent_node_uid"]] != email_uid + ): + _fail() + for record in collections["content_segments"]: + email_uid = record["email_uid"] + require_reference(email_uid, email_uids) + require_reference(record["attachment_uid"], attachment_uids) + require_reference(record["content_node_uid"], node_uids) + if node_email[record["content_node_uid"]] != email_uid: + _fail() + if ( + record["attachment_uid"] is not None + and attachment_email[record["attachment_uid"]] != email_uid + ): + _fail() + for record in collections["structural_edges"]: + email_uid = record["email_uid"] + require_reference(email_uid, email_uids) + require_reference(record["attachment_uid"], attachment_uids) + for key in ("source_node_uid", "target_node_uid"): + require_reference(record[key], node_uids) + if record[key] is not None and node_email[record[key]] != email_uid: + _fail() + for key in ("source_segment_uid", "target_segment_uid"): + require_reference(record[key], segment_uids) + if record[key] is not None and segment_email[record[key]] != email_uid: + _fail() + for record in collections["project_objects"]: + require_reference(record["email_uid"], email_uids) + require_reference(record["attachment_uid"], attachment_uids) + require_reference(record["primary_content_segment_uid"], segment_uids) + if segment_email[record["primary_content_segment_uid"]] != record["email_uid"]: + _fail() + for segment_uid in record["source_segment_uids"]: + require_reference(segment_uid, segment_uids) + for record in collections["project_edges"]: + require_reference(record["source_object_uid"], object_uids) + require_reference(record["target_object_uid"], object_uids) + require_reference(record["primary_content_segment_uid"], segment_uids) + for segment_uid in record["source_segment_uids"]: + require_reference(segment_uid, segment_uids) + for record in collections["corrections"]: + require_reference(record["object_uid"], object_uids) + for segment_uid in record["source_segment_uids"]: + require_reference(segment_uid, segment_uids) + + +async def _matching_models( + session: AsyncSession, + model: Any, + column: Any, + values: set[str], + key_attribute: str, +) -> dict[str, Any]: + if not values: + return {} + rows = list((await session.scalars(select(model).where(column.in_(values)))).all()) + return {getattr(row, key_attribute): row for row in rows} + + +async def _preflight_existing( + session: AsyncSession, + scope: TenantProvenanceScope, + records: Mapping[str, object], +) -> dict[str, dict[str, Any]]: + payload = { + collection: { + record[ + { + "emails": "email_uid", + "attachments": "attachment_uid", + "content_nodes": "content_node_uid", + "content_segments": "content_segment_uid", + "structural_edges": "edge_uid", + "project_objects": "object_uid", + "project_edges": "edge_uid", + "corrections": "correction_uid", + }[collection] + ]: record + for record in records[collection] + } + for collection in _COLLECTIONS + } + email_values = set(payload["emails"]) + email_rows = list( + ( + await session.scalars( + select(Email).where( + Email.message_id.in_(email_values), + *_scope_filters(Email, scope, workspace=False), + ) + ) + ).all() + ) + models: dict[str, dict[str, Any]] = { + "emails": {row.message_id: row for row in email_rows} + } + email_uids = {row.id: row.message_id for row in email_rows} + attachment_rows = ( + list( + ( + await session.scalars( + select(Attachment).where(Attachment.email_id.in_(email_uids)) + ) + ).all() + ) + if email_uids + else [] + ) + attachment_records, attachment_uids = _attachment_records( + attachment_rows, email_uids + ) + attachments_by_id = {row.id: row for row in attachment_rows} + models["attachments"] = { + uid: attachments_by_id[attachment_id] + for attachment_id, uid in attachment_uids.items() + if uid in payload["attachments"] + } + existing_serialized: dict[str, dict[str, dict[str, object]]] = { + "emails": {uid: _email_record(row) for uid, row in models["emails"].items()}, + "attachments": { + record["attachment_uid"]: record + for record in attachment_records + if record["attachment_uid"] in payload["attachments"] + }, + } + + models["content_nodes"] = await _matching_models( + session, + ContentNodeRecord, + ContentNodeRecord.content_node_uid, + set(payload["content_nodes"]), + "content_node_uid", + ) + node_uids = { + row.content_node_id: row.content_node_uid + for row in models["content_nodes"].values() + } + models["content_segments"] = await _matching_models( + session, + ContentSegmentRecord, + ContentSegmentRecord.content_segment_uid, + set(payload["content_segments"]), + "content_segment_uid", + ) + segment_uids = { + row.content_segment_id: row.content_segment_uid + for row in models["content_segments"].values() + } + models["structural_edges"] = await _matching_models( + session, + KnowledgeGraphEdgeRecord, + KnowledgeGraphEdgeRecord.edge_uid, + set(payload["structural_edges"]), + "edge_uid", + ) + models["project_objects"] = await _matching_models( + session, + ProjectGraphObjectRecord, + ProjectGraphObjectRecord.object_uid, + set(payload["project_objects"]), + "object_uid", + ) + object_uids = { + row.project_graph_object_id: row.object_uid + for row in models["project_objects"].values() + } + models["project_edges"] = await _matching_models( + session, + ProjectGraphEdgeRecord, + ProjectGraphEdgeRecord.edge_uid, + set(payload["project_edges"]), + "edge_uid", + ) + models["corrections"] = await _matching_models( + session, + ProjectGraphCorrectionRecord, + ProjectGraphCorrectionRecord.correction_uid, + set(payload["corrections"]), + "correction_uid", + ) + + try: + existing_serialized["content_nodes"] = { + uid: _node_record(row, email_uids, attachment_uids) + for uid, row in models["content_nodes"].items() + } + existing_serialized["content_segments"] = { + uid: _segment_record(row, email_uids, attachment_uids, node_uids) + for uid, row in models["content_segments"].items() + } + existing_serialized["structural_edges"] = { + uid: _structural_edge_record( + row, email_uids, attachment_uids, node_uids, segment_uids + ) + for uid, row in models["structural_edges"].items() + } + existing_serialized["project_objects"] = { + uid: _project_object_record(row, email_uids, attachment_uids, segment_uids) + for uid, row in models["project_objects"].items() + } + existing_serialized["project_edges"] = { + uid: _project_edge_record(row, object_uids, segment_uids) + for uid, row in models["project_edges"].items() + } + existing_serialized["corrections"] = { + uid: _correction_record(row, object_uids) + for uid, row in models["corrections"].items() + } + except KeyError: + _fail() + + scoped_models = ( + models["project_objects"].values(), + models["project_edges"].values(), + models["corrections"].values(), + ) + for rows in scoped_models: + for row in rows: + if ( + row.user_id != scope.user_id + or row.organization_id != scope.organization_id + or row.workspace_id != scope.workspace_id + ): + _fail() + if any( + row.actor_user_id != scope.user_id for row in models["corrections"].values() + ): + _fail() + for collection in _COLLECTIONS: + for uid, existing in existing_serialized[collection].items(): + if existing != payload[collection].get(uid): + _fail() + return models + + +async def _insert_records( + session: AsyncSession, + scope: TenantProvenanceScope, + records: Mapping[str, object], + models: dict[str, dict[str, Any]], + created: dict[str, int], +) -> None: + for record in records["emails"]: + uid = record["email_uid"] + if uid in models["emails"]: + continue + email = Email( + user_id=scope.user_id, + organization_id=scope.organization_id, + message_id=uid, + thread_id=record["thread_uid"], + fingerprint=record["fingerprint"], + sender=record["sender"], + reply_to=record["reply_to"], + recipients=record["recipients"], + subject=record["subject"], + in_reply_to=record["in_reply_to"], + references=record["references"], + date=_parse_datetime(record["date"]), + body=record["body"], + is_read=record["is_read"], + embedding=None, + ) + session.add(email) + models["emails"][uid] = email + created["emails"] += 1 + await session.flush() + + for record in records["attachments"]: + uid = record["attachment_uid"] + if uid in models["attachments"]: + continue + attachment = Attachment( + email=models["emails"][record["email_uid"]], + filename=record["filename"], + content=record["content"], + content_type=record["content_type"], + parse_status=record["parse_status"], + parse_content_type=record["parse_content_type"], + parser_key=record["parser_key"], + parse_error_code=record["parse_error_code"], + embedding=None, + ) + session.add(attachment) + models["attachments"][uid] = attachment + created["attachments"] += 1 + await session.flush() + + for record in records["content_nodes"]: + uid = record["content_node_uid"] + if uid in models["content_nodes"]: + continue + node = ContentNodeRecord( + content_node_uid=uid, + email=models["emails"][record["email_uid"]], + attachment=( + models["attachments"].get(record["attachment_uid"]) + if record["attachment_uid"] is not None + else None + ), + source_kind=record["source_kind"], + source_record_uid=record["source_record_uid"], + parent_node_uid=record["parent_node_uid"], + node_kind=record["node_kind"], + node_path=record["node_path"], + ordinal_index=record["ordinal_index"], + display_label=record["display_label"], + safe_text_content=record["safe_text_content"], + content_hash=record["content_hash"], + ) + session.add(node) + models["content_nodes"][uid] = node + created["content_nodes"] += 1 + await session.flush() + + for record in records["content_segments"]: + uid = record["content_segment_uid"] + if uid in models["content_segments"]: + continue + segment = ContentSegmentRecord( + content_segment_uid=uid, + email=models["emails"][record["email_uid"]], + attachment=( + models["attachments"].get(record["attachment_uid"]) + if record["attachment_uid"] is not None + else None + ), + content_node=models["content_nodes"][record["content_node_uid"]], + source_kind=record["source_kind"], + source_record_uid=record["source_record_uid"], + segment_kind=record["segment_kind"], + segment_path=record["segment_path"], + ordinal_index=record["ordinal_index"], + heading_path=record["heading_path"], + safe_text_content=record["safe_text_content"], + content_hash=record["content_hash"], + word_count=record["word_count"], + ) + session.add(segment) + models["content_segments"][uid] = segment + created["content_segments"] += 1 + await session.flush() + + for record in records["structural_edges"]: + uid = record["edge_uid"] + if uid in models["structural_edges"]: + continue + edge = KnowledgeGraphEdgeRecord( + edge_uid=uid, + email=models["emails"][record["email_uid"]], + attachment=( + models["attachments"].get(record["attachment_uid"]) + if record["attachment_uid"] is not None + else None + ), + source_node=models["content_nodes"].get(record["source_node_uid"]), + target_node=models["content_nodes"].get(record["target_node_uid"]), + source_segment=models["content_segments"].get(record["source_segment_uid"]), + target_segment=models["content_segments"].get(record["target_segment_uid"]), + source_kind=record["source_kind"], + source_record_uid=record["source_record_uid"], + edge_kind=record["edge_kind"], + edge_path=record["edge_path"], + ordinal_index=record["ordinal_index"], + ) + session.add(edge) + models["structural_edges"][uid] = edge + created["structural_edges"] += 1 + await session.flush() + + for record in records["project_objects"]: + uid = record["object_uid"] + if uid in models["project_objects"]: + continue + project_object = ProjectGraphObjectRecord( + object_uid=uid, + user_id=scope.user_id, + organization_id=scope.organization_id, + workspace_id=scope.workspace_id, + email=models["emails"][record["email_uid"]], + attachment=( + models["attachments"].get(record["attachment_uid"]) + if record["attachment_uid"] is not None + else None + ), + primary_content_segment=models["content_segments"][ + record["primary_content_segment_uid"] + ], + object_type=record["object_type"], + title=record["title"], + summary=record["summary"], + status_code=record["status_code"], + confidence=_parse_confidence(record["confidence"]), + source_segment_uids=record["source_segment_uids"], + attributes_json=record["attributes_json"], + extractor_name=record["extractor_name"], + extractor_version=record["extractor_version"], + ) + session.add(project_object) + models["project_objects"][uid] = project_object + created["project_objects"] += 1 + await session.flush() + + for record in records["project_edges"]: + uid = record["edge_uid"] + if uid in models["project_edges"]: + continue + edge = ProjectGraphEdgeRecord( + edge_uid=uid, + user_id=scope.user_id, + organization_id=scope.organization_id, + workspace_id=scope.workspace_id, + source_uid=record["source_uid"], + target_uid=record["target_uid"], + edge_type=record["edge_type"], + confidence=_parse_confidence(record["confidence"]), + source_segment_uids=record["source_segment_uids"], + source_object=models["project_objects"].get(record["source_object_uid"]), + target_object=models["project_objects"].get(record["target_object_uid"]), + primary_content_segment=models["content_segments"][ + record["primary_content_segment_uid"] + ], + ) + session.add(edge) + models["project_edges"][uid] = edge + created["project_edges"] += 1 + await session.flush() + + for record in records["corrections"]: + uid = record["correction_uid"] + if uid in models["corrections"]: + continue + correction = ProjectGraphCorrectionRecord( + correction_uid=uid, + project_object=models["project_objects"][record["object_uid"]], + user_id=scope.user_id, + organization_id=scope.organization_id, + workspace_id=scope.workspace_id, + actor_user_id=scope.user_id, + correction_action=record["correction_action"], + before_json=record["before_json"], + after_json=record["after_json"], + rationale=record["rationale"], + source_segment_uids=record["source_segment_uids"], + created_at=_parse_datetime(record["created_at"]), + ) + session.add(correction) + models["corrections"][uid] = correction + created["corrections"] += 1 + await session.flush() + + +async def import_tenant_provenance( + session: AsyncSession, + scope: TenantProvenanceScope, + archive_bytes: bytes, +) -> ImportReceipt: + """Validate all closure and conflict rules before one transactional restore.""" + _validate_scope(scope) + records = parse_provenance_archive(archive_bytes) + _validate_record_graph(records) + if scope.organization_id is None and records["emails"]: + _fail() + created = {collection: 0 for collection in _COLLECTIONS} + try: + async with session.begin(): + models = await _preflight_existing(session, scope, records) + skipped = { + collection: len(models[collection]) for collection in _COLLECTIONS + } + await _insert_records(session, scope, records, models, created) + except ProvenanceArchiveError: + raise + except (DataError, IntegrityError, StatementError, TypeError, ValueError) as exc: + raise ProvenanceArchiveError("Invalid provenance archive") from exc + return ImportReceipt( + bundle_uid=_safe_identifier(records.get("bundle_uid")), + manifest_digest=hashlib.sha512(_canonical_json(records)).hexdigest(), + created=created, + skipped=skipped, + ) diff --git a/backend/tests/test_tenant_provenance_bundle.py b/backend/tests/test_tenant_provenance_bundle.py index 17b2ef7bd..af61e67fe 100644 --- a/backend/tests/test_tenant_provenance_bundle.py +++ b/backend/tests/test_tenant_provenance_bundle.py @@ -1,11 +1,32 @@ +import copy +import datetime import hashlib import io import json import struct +import uuid import warnings import zipfile +import asyncpg import pytest +import pytest_asyncio +from sqlalchemy import delete, func, select, text +from sqlalchemy.exc import OperationalError +from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine + +from core.config import settings +from db.models import ( + Attachment, + Base, + ContentNodeRecord, + ContentSegmentRecord, + Email, + KnowledgeGraphEdgeRecord, + ProjectGraphCorrectionRecord, + ProjectGraphEdgeRecord, + ProjectGraphObjectRecord, +) from services.tenant_provenance_bundle import ( ARCHIVE_MAX_BYTES, @@ -13,9 +34,13 @@ ENTRY_MAX_BYTES, JSON_SAFE_INTEGER_MAX, MAX_COMPRESSION_RATIO, + ImportReceipt, ProvenanceArchiveError, + TenantProvenanceScope, _within_archive_bounds, build_provenance_archive, + export_tenant_provenance, + import_tenant_provenance, parse_provenance_archive, ) @@ -474,3 +499,530 @@ def test_parse_rejects_entry_count_and_compression_ratio(monkeypatch): entries["data/records.json"] = records with pytest.raises(ProvenanceArchiveError): parse_provenance_archive(_archive_with_entries(_rebuild_manifests(entries))) + + +@pytest_asyncio.fixture +async def provenance_sessionmaker(): + engine = create_async_engine(settings.DATABASE_URL, echo=False) + try: + async with engine.begin() as connection: + await connection.execute(text("SELECT 1")) + await connection.execute(text("CREATE EXTENSION IF NOT EXISTS vector")) + await connection.run_sync(Base.metadata.create_all) + async with engine.connect() as connection: + transaction = await connection.begin() + try: + yield async_sessionmaker( + bind=connection, + expire_on_commit=False, + join_transaction_mode="create_savepoint", + ) + finally: + if transaction.is_active: + await transaction.rollback() + except ( + ConnectionRefusedError, + OSError, + OperationalError, + asyncpg.CannotConnectNowError, + asyncpg.InvalidAuthorizationSpecificationError, + asyncpg.InvalidCatalogNameError, + asyncpg.InvalidPasswordError, + ): + pytest.skip("PostgreSQL smoke path unavailable") + finally: + await engine.dispose() + + +async def _seed_provenance_closure( + session, + *, + scope: TenantProvenanceScope, + token: str, +) -> dict[str, object]: + now = datetime.datetime(2026, 8, 31, 12, 0, tzinfo=datetime.timezone.utc) + email = Email( + user_id=scope.user_id, + organization_id=scope.organization_id, + message_id=f"<{token}@example.com>", + thread_id=f"thread-{token}", + fingerprint=f"sha256:{token}", + sender="source@example.com", + reply_to="reply@example.com", + recipients="owner@example.com", + subject=f"Evidence {token}", + in_reply_to=None, + references=None, + date=now, + body="Grounded source evidence", + is_read=False, + embedding=[0.125] * 1536, + ) + session.add(email) + await session.flush() + + attachment = Attachment( + email=email, + filename="evidence.txt", + content="Parser-confirmed attachment evidence", + content_type="text/plain", + parse_status="parsed", + parse_content_type="text/plain", + parser_key="plain_text", + embedding=[0.25] * 1536, + ) + binary_attachment = Attachment( + email=email, + filename="source.pdf", + content="binary-payload-must-not-export", + content_type="application/pdf", + parse_status="pdf_dom_recognition_pending", + parse_content_type="application/pdf", + parser_key="pdf", + embedding=[0.5] * 1536, + ) + session.add_all([attachment, binary_attachment]) + await session.flush() + + node = ContentNodeRecord( + content_node_uid=f"node-{token}", + email=email, + attachment=attachment, + source_kind="attachment", + source_record_uid=f"attachment-source-{token}", + parent_node_uid=None, + node_kind="document", + node_path="/document[1]", + ordinal_index=1, + display_label="Evidence", + safe_text_content="Parser-confirmed attachment evidence", + content_hash=f"nodehash-{token}", + created_at=now, + ) + session.add(node) + await session.flush() + + segment = ContentSegmentRecord( + content_segment_uid=f"segment-{token}", + email=email, + attachment=attachment, + content_node=node, + source_kind="attachment", + source_record_uid=f"attachment-source-{token}", + segment_kind="paragraph", + segment_path="/document[1]/paragraph[1]", + ordinal_index=1, + heading_path="Evidence", + safe_text_content="Parser-confirmed attachment evidence", + content_hash=f"segmenthash-{token}", + word_count=3, + created_at=now, + ) + session.add(segment) + await session.flush() + + structural_edge = KnowledgeGraphEdgeRecord( + edge_uid=f"structural-edge-{token}", + email=email, + attachment=attachment, + source_node=node, + target_segment=segment, + source_kind="attachment", + source_record_uid=f"attachment-source-{token}", + edge_kind="contains", + edge_path="/document[1]->/document[1]/paragraph[1]", + ordinal_index=1, + created_at=now, + ) + source_object = ProjectGraphObjectRecord( + object_uid=f"project-object-source-{token}", + user_id=scope.user_id, + organization_id=scope.organization_id, + workspace_id=scope.workspace_id, + email=email, + attachment=attachment, + primary_content_segment=segment, + object_type="requirement", + title="Portable requirement", + summary="Grounded requirement", + status_code="accepted", + confidence=0.91, + source_segment_uids=[segment.content_segment_uid], + attributes_json={"rank_value": 1, "source_label": "mail"}, + extractor_name="test-extractor", + extractor_version="1", + created_at=now, + updated_at=now, + ) + target_object = ProjectGraphObjectRecord( + object_uid=f"project-object-target-{token}", + user_id=scope.user_id, + organization_id=scope.organization_id, + workspace_id=scope.workspace_id, + email=email, + attachment=attachment, + primary_content_segment=segment, + object_type="decision", + title="Portable decision", + summary="Grounded decision", + status_code="accepted", + confidence=0.87, + source_segment_uids=[segment.content_segment_uid], + attributes_json={"decision_state": "approved"}, + extractor_name="test-extractor", + extractor_version="1", + created_at=now, + updated_at=now, + ) + session.add_all([structural_edge, source_object, target_object]) + await session.flush() + + project_edge = ProjectGraphEdgeRecord( + edge_uid=f"project-edge-{token}", + user_id=scope.user_id, + organization_id=scope.organization_id, + workspace_id=scope.workspace_id, + source_uid=source_object.object_uid, + target_uid=target_object.object_uid, + edge_type="supports", + confidence=0.89, + source_segment_uids=[segment.content_segment_uid], + source_object=source_object, + target_object=target_object, + primary_content_segment=segment, + created_at=now, + ) + correction = ProjectGraphCorrectionRecord( + correction_uid=f"correction-{token}", + project_object=source_object, + user_id=scope.user_id, + organization_id=scope.organization_id, + workspace_id=scope.workspace_id, + actor_user_id=scope.user_id, + correction_action="accept", + before_json={"status_code": "candidate", "rank_value": 0}, + after_json={"status_code": "accepted", "rank_value": 1}, + rationale="Verified against source", + source_segment_uids=[segment.content_segment_uid], + created_at=now, + ) + session.add_all([project_edge, correction]) + await session.commit() + return { + "email_id": email.id, + "attachment_id": attachment.id, + "node_id": node.content_node_id, + "segment_id": segment.content_segment_id, + "project_object_id": source_object.project_graph_object_id, + "email_uid": email.message_id, + "object_uids": [source_object.object_uid, target_object.object_uid], + } + + +async def _delete_exported_closure(session, records: dict[str, object]) -> None: + emails = records["emails"] + email_uids = [record["email_uid"] for record in emails] + email_ids = list( + ( + await session.scalars( + select(Email.id).where(Email.message_id.in_(email_uids)) + ) + ).all() + ) + await session.execute( + delete(ProjectGraphCorrectionRecord).where( + ProjectGraphCorrectionRecord.correction_uid.in_( + [record["correction_uid"] for record in records["corrections"]] + ) + ) + ) + await session.execute( + delete(ProjectGraphEdgeRecord).where( + ProjectGraphEdgeRecord.edge_uid.in_( + [record["edge_uid"] for record in records["project_edges"]] + ) + ) + ) + await session.execute( + delete(ProjectGraphObjectRecord).where( + ProjectGraphObjectRecord.object_uid.in_( + [record["object_uid"] for record in records["project_objects"]] + ) + ) + ) + await session.execute( + delete(KnowledgeGraphEdgeRecord).where( + KnowledgeGraphEdgeRecord.email_id.in_(email_ids) + ) + ) + await session.execute( + delete(ContentSegmentRecord).where(ContentSegmentRecord.email_id.in_(email_ids)) + ) + await session.execute( + delete(ContentNodeRecord).where(ContentNodeRecord.email_id.in_(email_ids)) + ) + await session.execute(delete(Attachment).where(Attachment.email_id.in_(email_ids))) + await session.execute(delete(Email).where(Email.id.in_(email_ids))) + await session.commit() + + +def _scope(token: str) -> TenantProvenanceScope: + return TenantProvenanceScope( + user_id=f"user-{token}", + organization_id=f"org-{token}", + workspace_id=f"workspace-{token}", + ) + + +@pytest.mark.asyncio +@pytest.mark.postgres +async def test_postgres_round_trip_preserves_stable_evidence_with_fresh_keys( + provenance_sessionmaker, +): + token = uuid.uuid4().hex[:12] + source_scope = _scope(f"source-{token}") + target_scope = _scope(f"target-{token}") + async with provenance_sessionmaker() as session: + source_keys = await _seed_provenance_closure( + session, scope=source_scope, token=token + ) + async with provenance_sessionmaker() as session: + archive = await export_tenant_provenance(session, source_scope) + source_records = parse_provenance_archive(archive) + async with provenance_sessionmaker() as session: + await _delete_exported_closure(session, source_records) + async with provenance_sessionmaker() as session: + receipt = await import_tenant_provenance(session, target_scope, archive) + + assert isinstance(receipt, ImportReceipt) + assert receipt.created == { + "emails": 1, + "attachments": 1, + "content_nodes": 1, + "content_segments": 1, + "structural_edges": 1, + "project_objects": 2, + "project_edges": 1, + "corrections": 1, + } + async with provenance_sessionmaker() as session: + restored_archive = await export_tenant_provenance(session, target_scope) + restored_email = await session.scalar( + select(Email).where(Email.message_id == source_keys["email_uid"]) + ) + restored_object = await session.scalar( + select(ProjectGraphObjectRecord).where( + ProjectGraphObjectRecord.object_uid == source_keys["object_uids"][0] + ) + ) + restored_correction = await session.scalar( + select(ProjectGraphCorrectionRecord).where( + ProjectGraphCorrectionRecord.correction_uid == f"correction-{token}" + ) + ) + restored_attachment = await session.scalar( + select(Attachment).where(Attachment.email_id == restored_email.id) + ) + restored_node = await session.scalar( + select(ContentNodeRecord).where( + ContentNodeRecord.content_node_uid == f"node-{token}" + ) + ) + restored_segment = await session.scalar( + select(ContentSegmentRecord).where( + ContentSegmentRecord.content_segment_uid == f"segment-{token}" + ) + ) + restored_records = parse_provenance_archive(restored_archive) + for collection in ( + "emails", + "attachments", + "content_nodes", + "content_segments", + "structural_edges", + "project_objects", + "project_edges", + "corrections", + ): + assert restored_records[collection] == source_records[collection] + assert restored_email.id != source_keys["email_id"] + assert restored_attachment.id != source_keys["attachment_id"] + assert restored_node.content_node_id != source_keys["node_id"] + assert restored_segment.content_segment_id != source_keys["segment_id"] + assert restored_email.user_id == target_scope.user_id + assert restored_email.organization_id == target_scope.organization_id + assert restored_object.project_graph_object_id != source_keys["project_object_id"] + assert restored_object.user_id == target_scope.user_id + assert restored_object.organization_id == target_scope.organization_id + assert restored_object.workspace_id == target_scope.workspace_id + assert restored_correction.actor_user_id == target_scope.user_id + assert restored_correction.before_json == { + "status_code": "candidate", + "rank_value": 0, + } + assert restored_correction.after_json == { + "status_code": "accepted", + "rank_value": 1, + } + + +@pytest.mark.asyncio +@pytest.mark.postgres +async def test_export_is_exact_workspace_scoped(provenance_sessionmaker): + token = uuid.uuid4().hex[:12] + shared_scope = _scope(f"shared-{token}") + other_scope = TenantProvenanceScope( + user_id=shared_scope.user_id, + organization_id=shared_scope.organization_id, + workspace_id=f"other-workspace-{token}", + ) + async with provenance_sessionmaker() as session: + expected = await _seed_provenance_closure( + session, scope=shared_scope, token=f"selected-{token}" + ) + excluded = await _seed_provenance_closure( + session, scope=other_scope, token=f"excluded-{token}" + ) + async with provenance_sessionmaker() as session: + records = parse_provenance_archive( + await export_tenant_provenance(session, shared_scope) + ) + + assert [record["email_uid"] for record in records["emails"]] == [ + expected["email_uid"] + ] + assert excluded["email_uid"] not in json.dumps(records) + + +@pytest.mark.asyncio +@pytest.mark.postgres +async def test_import_dangling_reference_rolls_back_without_mutation( + provenance_sessionmaker, +): + token = uuid.uuid4().hex[:12] + source_scope = _scope(f"source-{token}") + target_scope = _scope(f"target-{token}") + async with provenance_sessionmaker() as session: + await _seed_provenance_closure(session, scope=source_scope, token=token) + async with provenance_sessionmaker() as session: + source_archive = await export_tenant_provenance(session, source_scope) + records = copy.deepcopy(parse_provenance_archive(source_archive)) + records["project_objects"][0]["source_segment_uids"] = ["missing-segment"] + invalid_archive = build_provenance_archive(records) + async with provenance_sessionmaker() as session: + await _delete_exported_closure(session, records) + async with provenance_sessionmaker() as session: + with pytest.raises(ProvenanceArchiveError): + await import_tenant_provenance(session, target_scope, invalid_archive) + async with provenance_sessionmaker() as session: + assert ( + await session.scalar( + select(func.count()) + .select_from(Email) + .where( + *Email.owner_filters( + target_scope.user_id, target_scope.organization_id + ) + ) + ) + == 0 + ) + + +@pytest.mark.asyncio +@pytest.mark.postgres +async def test_import_late_conflict_fails_before_creating_email( + provenance_sessionmaker, +): + token = uuid.uuid4().hex[:12] + source_scope = _scope(f"source-{token}") + target_scope = _scope(f"target-{token}") + async with provenance_sessionmaker() as session: + await _seed_provenance_closure(session, scope=source_scope, token=token) + async with provenance_sessionmaker() as session: + archive = await export_tenant_provenance(session, source_scope) + records = parse_provenance_archive(archive) + imported_email_uid = records["emails"][0]["email_uid"] + conflict_uid = records["project_objects"][0]["object_uid"] + async with provenance_sessionmaker() as session: + await _delete_exported_closure(session, records) + target_seed = await _seed_provenance_closure( + session, scope=target_scope, token=f"conflict-{token}" + ) + existing = await session.scalar( + select(ProjectGraphObjectRecord).where( + ProjectGraphObjectRecord.object_uid == target_seed["object_uids"][0] + ) + ) + existing.object_uid = conflict_uid + existing.title = "Conflicting target record" + await session.commit() + + async with provenance_sessionmaker() as session: + with pytest.raises(ProvenanceArchiveError): + await import_tenant_provenance(session, target_scope, archive) + async with provenance_sessionmaker() as session: + assert ( + await session.scalar( + select(func.count()) + .select_from(Email) + .where(Email.message_id == imported_email_uid) + ) + == 0 + ) + + +@pytest.mark.asyncio +@pytest.mark.postgres +async def test_export_admits_only_parser_confirmed_text_and_omits_sensitive_fields( + provenance_sessionmaker, +): + token = uuid.uuid4().hex[:12] + scope = _scope(token) + async with provenance_sessionmaker() as session: + await _seed_provenance_closure(session, scope=scope, token=token) + async with provenance_sessionmaker() as session: + archive = await export_tenant_provenance(session, scope) + records = parse_provenance_archive(archive) + serialized = json.dumps(records, sort_keys=True) + + assert [record["filename"] for record in records["attachments"]] == ["evidence.txt"] + assert "binary-payload-must-not-export" not in serialized + for forbidden in ( + "embedding", + "credential", + "secret", + "api_key", + "provider_url", + "token", + "email_id", + "attachment_id", + "content_node_id", + "content_segment_id", + "project_graph_object_id", + ): + assert forbidden not in serialized + + +@pytest.mark.asyncio +@pytest.mark.postgres +async def test_import_is_idempotent_for_exact_target_records(provenance_sessionmaker): + token = uuid.uuid4().hex[:12] + source_scope = _scope(f"source-{token}") + target_scope = _scope(f"target-{token}") + async with provenance_sessionmaker() as session: + await _seed_provenance_closure(session, scope=source_scope, token=token) + async with provenance_sessionmaker() as session: + archive = await export_tenant_provenance(session, source_scope) + records = parse_provenance_archive(archive) + async with provenance_sessionmaker() as session: + await _delete_exported_closure(session, records) + async with provenance_sessionmaker() as session: + first = await import_tenant_provenance(session, target_scope, archive) + async with provenance_sessionmaker() as session: + second = await import_tenant_provenance(session, target_scope, archive) + + assert sum(first.created.values()) == 9 + assert sum(first.skipped.values()) == 0 + assert sum(second.created.values()) == 0 + assert second.skipped == first.created From 236423f5e15d2a792ce1d975185174d8427ef230 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Mon, 31 Aug 2026 22:21:30 +0900 Subject: [PATCH 06/39] fix: preserve provenance citation closure --- backend/services/tenant_provenance_bundle.py | 123 +++++++----- .../tests/test_tenant_provenance_bundle.py | 181 ++++++++++++++++++ 2 files changed, 259 insertions(+), 45 deletions(-) diff --git a/backend/services/tenant_provenance_bundle.py b/backend/services/tenant_provenance_bundle.py index 7b1714609..6eaba77ca 100644 --- a/backend/services/tenant_provenance_bundle.py +++ b/backend/services/tenant_provenance_bundle.py @@ -670,6 +670,39 @@ def _attachment_core(attachment: Attachment, email_uid: str) -> dict[str, object } +def _attachment_payload_core(record: Mapping[str, object]) -> bytes: + return _canonical_json( + {key: value for key, value in record.items() if key != "attachment_uid"} + ) + + +def _attachment_uid(canonical: bytes, occurrence: int) -> str: + return ( + "attachment-" + + hashlib.sha256(canonical + b":" + str(occurrence).encode("ascii")).hexdigest() + ) + + +def _attachments_in_occurrence_order( + records: list[dict[str, object]], +) -> list[dict[str, object]]: + groups: dict[bytes, list[dict[str, object]]] = {} + for record in records: + groups.setdefault(_attachment_payload_core(record), []).append(record) + ordered: list[dict[str, object]] = [] + for canonical, group in sorted(groups.items()): + occurrences = { + _attachment_uid(canonical, occurrence): occurrence + for occurrence in range(1, len(group) + 1) + } + if set(occurrences) != {record["attachment_uid"] for record in group}: + _fail() + ordered.extend( + sorted(group, key=lambda record: occurrences[record["attachment_uid"]]) + ) + return ordered + + def _is_admitted_attachment(attachment: Attachment) -> bool: return ( attachment.parse_status == "parsed" @@ -698,12 +731,7 @@ def _attachment_records( core = _attachment_core(attachment, email_uids[attachment.email_id]) canonical = _canonical_json(core) occurrence[canonical] = occurrence.get(canonical, 0) + 1 - uid = ( - "attachment-" - + hashlib.sha256( - canonical + b":" + str(occurrence[canonical]).encode("ascii") - ).hexdigest() - ) + uid = _attachment_uid(canonical, occurrence[canonical]) uid_by_id[attachment.id] = uid records.append({"attachment_uid": uid, **core}) return sorted(records, key=lambda item: item["attachment_uid"]), uid_by_id @@ -866,7 +894,48 @@ async def export_tenant_provenance( ) ).all() ) - email_ids = {record.email_id for record in project_objects} + project_edges = list( + ( + await session.scalars( + select(ProjectGraphEdgeRecord) + .where(*_scope_filters(ProjectGraphEdgeRecord, scope, workspace=True)) + .order_by(ProjectGraphEdgeRecord.edge_uid) + ) + ).all() + ) + corrections = list( + ( + await session.scalars( + select(ProjectGraphCorrectionRecord) + .where( + *_scope_filters(ProjectGraphCorrectionRecord, scope, workspace=True) + ) + .order_by(ProjectGraphCorrectionRecord.correction_uid) + ) + ).all() + ) + cited_segment_uids = { + segment_uid + for record in (*project_objects, *project_edges, *corrections) + for segment_uid in record.source_segment_uids + } + primary_segment_ids = { + record.primary_content_segment_id + for record in (*project_objects, *project_edges) + } + cited_segments = list( + ( + await session.scalars( + select(ContentSegmentRecord).where( + (ContentSegmentRecord.content_segment_uid.in_(cited_segment_uids)) + | (ContentSegmentRecord.content_segment_id.in_(primary_segment_ids)) + ) + ) + ).all() + ) + email_ids = {record.email_id for record in project_objects} | { + segment.email_id for segment in cited_segments + } emails = list( ( await session.scalars( @@ -902,27 +971,6 @@ async def descendants(model: Any, order_column: Any) -> list[Any]: structural_edges = await descendants( KnowledgeGraphEdgeRecord, KnowledgeGraphEdgeRecord.edge_uid ) - project_edges = list( - ( - await session.scalars( - select(ProjectGraphEdgeRecord) - .where(*_scope_filters(ProjectGraphEdgeRecord, scope, workspace=True)) - .order_by(ProjectGraphEdgeRecord.edge_uid) - ) - ).all() - ) - corrections = list( - ( - await session.scalars( - select(ProjectGraphCorrectionRecord) - .where( - *_scope_filters(ProjectGraphCorrectionRecord, scope, workspace=True) - ) - .order_by(ProjectGraphCorrectionRecord.correction_uid) - ) - ).all() - ) - attachment_records, attachment_uids = _attachment_records(attachments, email_uids) node_uids = {node.content_node_id: node.content_node_uid for node in nodes} segment_uids = { @@ -1241,22 +1289,7 @@ def _validate_record_graph(records: Mapping[str, object]) -> None: record["attachment_uid"]: record["email_uid"] for record in collections["attachments"] } - attachment_groups: dict[bytes, list[str]] = {} - for record in collections["attachments"]: - canonical = _canonical_json( - {key: value for key, value in record.items() if key != "attachment_uid"} - ) - attachment_groups.setdefault(canonical, []).append(record["attachment_uid"]) - for canonical, actual_uids in attachment_groups.items(): - expected_uids = { - "attachment-" - + hashlib.sha256( - canonical + b":" + str(occurrence).encode("ascii") - ).hexdigest() - for occurrence in range(1, len(actual_uids) + 1) - } - if set(actual_uids) != expected_uids: - _fail() + _attachments_in_occurrence_order(collections["attachments"]) node_email = { record["content_node_uid"]: record["email_uid"] for record in collections["content_nodes"] @@ -1553,7 +1586,7 @@ async def _insert_records( created["emails"] += 1 await session.flush() - for record in records["attachments"]: + for record in _attachments_in_occurrence_order(records["attachments"]): uid = record["attachment_uid"] if uid in models["attachments"]: continue diff --git a/backend/tests/test_tenant_provenance_bundle.py b/backend/tests/test_tenant_provenance_bundle.py index af61e67fe..de12d22c5 100644 --- a/backend/tests/test_tenant_provenance_bundle.py +++ b/backend/tests/test_tenant_provenance_bundle.py @@ -766,6 +766,66 @@ async def _delete_exported_closure(session, records: dict[str, object]) -> None: await session.commit() +async def _add_duplicate_text_attachments( + session, + *, + email_uid: str, + token: str, + count: int = 5, +) -> None: + email = await session.scalar(select(Email).where(Email.message_id == email_uid)) + now = datetime.datetime(2026, 8, 31, 12, 0, tzinfo=datetime.timezone.utc) + for index in range(1, count + 1): + attachment = Attachment( + email=email, + filename="evidence.txt", + content="Parser-confirmed attachment evidence", + content_type="text/plain", + parse_status="parsed", + parse_content_type="text/plain", + parser_key="plain_text", + embedding=[0.25] * 1536, + ) + session.add(attachment) + await session.flush() + node = ContentNodeRecord( + content_node_uid=f"duplicate-node-{index}-{token}", + email=email, + attachment=attachment, + source_kind="attachment", + source_record_uid=f"duplicate-source-{index}-{token}", + parent_node_uid=None, + node_kind="document", + node_path=f"/document[{index + 1}]", + ordinal_index=index + 1, + display_label=f"Evidence {index}", + safe_text_content=f"Duplicate evidence {index}", + content_hash=f"duplicate-node-hash-{index}-{token}", + created_at=now, + ) + session.add(node) + await session.flush() + session.add( + ContentSegmentRecord( + content_segment_uid=f"duplicate-segment-{index}-{token}", + email=email, + attachment=attachment, + content_node=node, + source_kind="attachment", + source_record_uid=f"duplicate-source-{index}-{token}", + segment_kind="paragraph", + segment_path=f"/document[{index + 1}]/paragraph[1]", + ordinal_index=index + 1, + heading_path=f"Evidence {index}", + safe_text_content=f"Duplicate evidence {index}", + content_hash=f"duplicate-segment-hash-{index}-{token}", + word_count=3, + created_at=now, + ) + ) + await session.commit() + + def _scope(token: str) -> TenantProvenanceScope: return TenantProvenanceScope( user_id=f"user-{token}", @@ -1026,3 +1086,124 @@ async def test_import_is_idempotent_for_exact_target_records(provenance_sessionm assert sum(first.skipped.values()) == 0 assert sum(second.created.values()) == 0 assert second.skipped == first.created + + +@pytest.mark.parametrize("citation_owner", ("object", "edge", "correction")) +@pytest.mark.asyncio +@pytest.mark.postgres +async def test_export_includes_cross_email_citation_source_closure( + provenance_sessionmaker, + citation_owner, +): + token = uuid.uuid4().hex[:12] + scope = _scope(f"source-{token}") + cited_scope = TenantProvenanceScope( + user_id=scope.user_id, + organization_id=scope.organization_id, + workspace_id=f"cited-workspace-{token}", + ) + async with provenance_sessionmaker() as session: + source = await _seed_provenance_closure(session, scope=scope, token=token) + cited = await _seed_provenance_closure( + session, + scope=cited_scope, + token=f"cited-{token}", + ) + cited_segment_uid = f"segment-cited-{token}" + if citation_owner == "object": + record = await session.scalar( + select(ProjectGraphObjectRecord).where( + ProjectGraphObjectRecord.object_uid == source["object_uids"][0] + ) + ) + elif citation_owner == "edge": + record = await session.scalar( + select(ProjectGraphEdgeRecord).where( + ProjectGraphEdgeRecord.edge_uid == f"project-edge-{token}" + ) + ) + else: + record = await session.scalar( + select(ProjectGraphCorrectionRecord).where( + ProjectGraphCorrectionRecord.correction_uid == f"correction-{token}" + ) + ) + record.source_segment_uids = sorted( + [*record.source_segment_uids, cited_segment_uid] + ) + await session.commit() + async with provenance_sessionmaker() as session: + records = parse_provenance_archive( + await export_tenant_provenance(session, scope) + ) + + assert {record["email_uid"] for record in records["emails"]} == { + source["email_uid"], + cited["email_uid"], + } + assert cited_segment_uid in { + record["content_segment_uid"] for record in records["content_segments"] + } + assert f"node-cited-{token}" in { + record["content_node_uid"] for record in records["content_nodes"] + } + assert any( + record["email_uid"] == cited["email_uid"] for record in records["attachments"] + ) + assert f"project-object-source-cited-{token}" not in { + record["object_uid"] for record in records["project_objects"] + } + + +@pytest.mark.asyncio +@pytest.mark.postgres +async def test_duplicate_canonical_attachments_keep_node_and_segment_identity( + provenance_sessionmaker, +): + token = uuid.uuid4().hex[:12] + source_scope = _scope(f"source-{token}") + target_scope = _scope(f"target-{token}") + async with provenance_sessionmaker() as session: + source = await _seed_provenance_closure( + session, scope=source_scope, token=token + ) + await _add_duplicate_text_attachments( + session, + email_uid=source["email_uid"], + token=token, + ) + async with provenance_sessionmaker() as session: + archive = await export_tenant_provenance(session, source_scope) + source_records = parse_provenance_archive(archive) + source_node_attachments = { + record["content_node_uid"]: record["attachment_uid"] + for record in source_records["content_nodes"] + if record["content_node_uid"].startswith("duplicate-node-") + } + source_segment_attachments = { + record["content_segment_uid"]: record["attachment_uid"] + for record in source_records["content_segments"] + if record["content_segment_uid"].startswith("duplicate-segment-") + } + assert list(source_node_attachments.values()) != sorted( + source_node_attachments.values() + ) + async with provenance_sessionmaker() as session: + await _delete_exported_closure(session, source_records) + async with provenance_sessionmaker() as session: + await import_tenant_provenance(session, target_scope, archive) + async with provenance_sessionmaker() as session: + restored = parse_provenance_archive( + await export_tenant_provenance(session, target_scope) + ) + + assert { + record["content_node_uid"]: record["attachment_uid"] + for record in restored["content_nodes"] + if record["content_node_uid"].startswith("duplicate-node-") + } == source_node_attachments + assert { + record["content_segment_uid"]: record["attachment_uid"] + for record in restored["content_segments"] + if record["content_segment_uid"].startswith("duplicate-segment-") + } == source_segment_attachments From 67be568b0328b94ec4b8543f639bb8d902c394e7 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Mon, 31 Aug 2026 23:18:47 +0900 Subject: [PATCH 07/39] fix: harden provenance bundle validation --- backend/services/tenant_provenance_bundle.py | 260 ++++++++- .../tests/test_tenant_provenance_bundle.py | 524 +++++++++++++++++- 2 files changed, 765 insertions(+), 19 deletions(-) diff --git a/backend/services/tenant_provenance_bundle.py b/backend/services/tenant_provenance_bundle.py index 6eaba77ca..ee0c94efc 100644 --- a/backend/services/tenant_provenance_bundle.py +++ b/backend/services/tenant_provenance_bundle.py @@ -65,6 +65,46 @@ _TEXTUAL_PARSER_KEYS = frozenset( {"plain_text", "html", "markdown", "json", "csv", "xml", "calendar", "pdf"} ) +_FORBIDDEN_METADATA_KEYS = frozenset( + { + "access_token", + "api_key", + "api_secret", + "attachment_id", + "auth_token", + "base_url", + "bearer_token", + "client_secret", + "connection_string", + "content_node_id", + "content_segment_id", + "credential", + "credentials", + "credentials_encrypted", + "database_id", + "database_url", + "db_id", + "dsn", + "email_id", + "id", + "knowledge_graph_edge_id", + "openai_api_key", + "password", + "primary_key", + "private_key", + "project_graph_correction_id", + "project_graph_edge_id", + "project_graph_object_id", + "provider_base_url", + "provider_endpoint", + "provider_url", + "refresh_token", + "row_id", + "secret", + "secrets", + "token", + } +) _RECORD_KEYS = { "emails": frozenset( { @@ -226,7 +266,9 @@ def _validate_json_value(value: object, depth: int = 0) -> None: _fail() return if isinstance(value, float): - _fail() + if not math.isfinite(value): + _fail() + return if isinstance(value, list): for item in value: _validate_json_value(item, depth + 1) @@ -242,6 +284,37 @@ def _validate_json_value(value: object, depth: int = 0) -> None: _fail() +def _validate_safe_metadata(value: object, depth: int = 0) -> None: + if depth > JSON_MAX_DEPTH: + _fail() + if isinstance(value, Mapping): + for key, item in value.items(): + if not isinstance(key, str): + _fail() + normalized_key = key.lower().replace("-", "_").replace(" ", "_") + if normalized_key in _FORBIDDEN_METADATA_KEYS: + _fail() + _validate_safe_metadata(item, depth + 1) + elif isinstance(value, list): + for item in value: + _validate_safe_metadata(item, depth + 1) + + +def _portable_metadata( + value: object, attachment_references: Mapping[str, str] +) -> object: + if isinstance(value, str): + return attachment_references.get(value, value) + if isinstance(value, list): + return [_portable_metadata(item, attachment_references) for item in value] + if isinstance(value, Mapping): + return { + key: _portable_metadata(item, attachment_references) + for key, item in value.items() + } + return value + + def _canonical_json(value: object) -> bytes: _validate_json_value(value) try: @@ -754,6 +827,19 @@ def _email_record(email: Email) -> dict[str, object]: } +def _portable_source_record_uid( + source_record_uid: str, + attachment_id: int | None, + attachment_uids: Mapping[int, str], +) -> str: + if attachment_id is None: + return source_record_uid + attachment_uid = attachment_uids.get(attachment_id) + if attachment_uid is None: + _fail() + return f"attachment:{attachment_uid}" + + def _node_record( node: ContentNodeRecord, email_uids: Mapping[int, str], @@ -764,7 +850,9 @@ def _node_record( "email_uid": email_uids[node.email_id], "attachment_uid": attachment_uids.get(node.attachment_id), "source_kind": node.source_kind, - "source_record_uid": node.source_record_uid, + "source_record_uid": _portable_source_record_uid( + node.source_record_uid, node.attachment_id, attachment_uids + ), "parent_node_uid": node.parent_node_uid, "node_kind": node.node_kind, "node_path": node.node_path, @@ -787,7 +875,9 @@ def _segment_record( "attachment_uid": attachment_uids.get(segment.attachment_id), "content_node_uid": node_uids[segment.content_node_id], "source_kind": segment.source_kind, - "source_record_uid": segment.source_record_uid, + "source_record_uid": _portable_source_record_uid( + segment.source_record_uid, segment.attachment_id, attachment_uids + ), "segment_kind": segment.segment_kind, "segment_path": segment.segment_path, "ordinal_index": segment.ordinal_index, @@ -814,7 +904,9 @@ def _structural_edge_record( "source_segment_uid": segment_uids.get(edge.source_segment_id), "target_segment_uid": segment_uids.get(edge.target_segment_id), "source_kind": edge.source_kind, - "source_record_uid": edge.source_record_uid, + "source_record_uid": _portable_source_record_uid( + edge.source_record_uid, edge.attachment_id, attachment_uids + ), "edge_kind": edge.edge_kind, "edge_path": edge.edge_path, "ordinal_index": edge.ordinal_index, @@ -826,7 +918,12 @@ def _project_object_record( email_uids: Mapping[int, str], attachment_uids: Mapping[int, str], segment_uids: Mapping[int, str], + attachment_references: Mapping[str, str], ) -> dict[str, object]: + attributes_json = _portable_metadata( + project_object.attributes_json, attachment_references + ) + _validate_safe_metadata(attributes_json) return { "object_uid": project_object.object_uid, "email_uid": email_uids[project_object.email_id], @@ -840,7 +937,7 @@ def _project_object_record( "status_code": project_object.status_code, "confidence": _confidence(project_object.confidence), "source_segment_uids": sorted(project_object.source_segment_uids), - "attributes_json": project_object.attributes_json, + "attributes_json": attributes_json, "extractor_name": project_object.extractor_name, "extractor_version": project_object.extractor_version, } @@ -867,13 +964,18 @@ def _project_edge_record( def _correction_record( correction: ProjectGraphCorrectionRecord, object_uids: Mapping[int, str], + attachment_references: Mapping[str, str], ) -> dict[str, object]: + before_json = _portable_metadata(correction.before_json, attachment_references) + after_json = _portable_metadata(correction.after_json, attachment_references) + _validate_safe_metadata(before_json) + _validate_safe_metadata(after_json) return { "correction_uid": correction.correction_uid, "object_uid": object_uids[correction.project_graph_object_id], "correction_action": correction.correction_action, - "before_json": correction.before_json, - "after_json": correction.after_json, + "before_json": before_json, + "after_json": after_json, "rationale": correction.rationale, "source_segment_uids": sorted(correction.source_segment_uids), "created_at": _utc_text(correction.created_at), @@ -972,6 +1074,10 @@ async def descendants(model: Any, order_column: Any) -> list[Any]: KnowledgeGraphEdgeRecord, KnowledgeGraphEdgeRecord.edge_uid ) attachment_records, attachment_uids = _attachment_records(attachments, email_uids) + attachment_references = { + f"attachment-{attachment_id}": f"attachment:{attachment_uid}" + for attachment_id, attachment_uid in attachment_uids.items() + } node_uids = {node.content_node_id: node.content_node_uid for node in nodes} segment_uids = { segment.content_segment_id: segment.content_segment_uid for segment in segments @@ -1002,7 +1108,11 @@ async def descendants(model: Any, order_column: Any) -> list[Any]: ], "project_objects": [ _project_object_record( - project_object, email_uids, attachment_uids, segment_uids + project_object, + email_uids, + attachment_uids, + segment_uids, + attachment_references, ) for project_object in project_objects ], @@ -1011,7 +1121,8 @@ async def descendants(model: Any, order_column: Any) -> list[Any]: for edge in project_edges ], "corrections": [ - _correction_record(correction, object_uids) for correction in corrections + _correction_record(correction, object_uids, attachment_references) + for correction in corrections ], } content_digest = hashlib.sha256(_canonical_json(payload)).hexdigest() @@ -1058,6 +1169,19 @@ def _optional_text(record: Mapping[str, object], key: str) -> str | None: return value +def _bounded_text( + record: Mapping[str, object], + key: str, + maximum: int, + *, + optional: bool = False, +) -> str | None: + value = _optional_text(record, key) if optional else _required_text(record, key) + if value is not None and len(value) > maximum: + _fail() + return value + + def _uid( record: Mapping[str, object], key: str, *, optional: bool = False ) -> str | None: @@ -1129,6 +1253,11 @@ def _validate_record_scalars( ): _required_text(attachment, key) _optional_text(attachment, "parse_error_code") + _bounded_text(attachment, "content_type", 120) + _bounded_text(attachment, "parse_status", 64) + _bounded_text(attachment, "parse_content_type", 120) + _bounded_text(attachment, "parser_key", 64) + _bounded_text(attachment, "parse_error_code", 120, optional=True) if ( attachment["parse_status"] != "parsed" or attachment["parser_key"] not in _TEXTUAL_PARSER_KEYS @@ -1150,6 +1279,14 @@ def _validate_record_scalars( ): _required_text(node, key) _optional_text(node, "display_label") + _bounded_text(node, "content_node_uid", 64) + _bounded_text(node, "source_kind", 64) + _bounded_text(node, "source_record_uid", 256) + _bounded_text(node, "parent_node_uid", 64, optional=True) + _bounded_text(node, "node_kind", 64) + _bounded_text(node, "node_path", 512) + _bounded_text(node, "display_label", 240, optional=True) + _bounded_text(node, "content_hash", 64) _integer(node, "ordinal_index") for segment in collections["content_segments"]: @@ -1166,6 +1303,13 @@ def _validate_record_scalars( ): _required_text(segment, key) _optional_text(segment, "heading_path") + _bounded_text(segment, "content_segment_uid", 64) + _bounded_text(segment, "source_kind", 64) + _bounded_text(segment, "source_record_uid", 256) + _bounded_text(segment, "segment_kind", 64) + _bounded_text(segment, "segment_path", 512) + _bounded_text(segment, "heading_path", 512, optional=True) + _bounded_text(segment, "content_hash", 64) _integer(segment, "ordinal_index") _integer(segment, "word_count") @@ -1182,6 +1326,11 @@ def _validate_record_scalars( _uid(edge, key, optional=True) for key in ("source_kind", "source_record_uid", "edge_kind", "edge_path"): _required_text(edge, key) + _bounded_text(edge, "edge_uid", 64) + _bounded_text(edge, "source_kind", 64) + _bounded_text(edge, "source_record_uid", 256) + _bounded_text(edge, "edge_kind", 64) + _bounded_text(edge, "edge_path", 512) _integer(edge, "ordinal_index") for project_object in collections["project_objects"]: @@ -1201,10 +1350,17 @@ def _validate_record_scalars( "extractor_version", ): _required_text(project_object, key) + _bounded_text(project_object, "object_uid", 96) + _bounded_text(project_object, "object_type", 64) + _bounded_text(project_object, "title", 240) + _bounded_text(project_object, "status_code", 64) + _bounded_text(project_object, "extractor_name", 120) + _bounded_text(project_object, "extractor_version", 64) _parse_confidence(project_object.get("confidence")) _uid_list(project_object, "source_segment_uids") if not isinstance(project_object.get("attributes_json"), dict): _fail() + _validate_safe_metadata(project_object["attributes_json"]) for edge in collections["project_edges"]: for key in ( @@ -1217,6 +1373,10 @@ def _validate_record_scalars( for key in ("source_object_uid", "target_object_uid"): _uid(edge, key, optional=True) _required_text(edge, "edge_type") + _bounded_text(edge, "edge_uid", 96) + _bounded_text(edge, "source_uid", 160) + _bounded_text(edge, "target_uid", 160) + _bounded_text(edge, "edge_type", 80) _parse_confidence(edge.get("confidence")) _uid_list(edge, "source_segment_uids") @@ -1224,12 +1384,16 @@ def _validate_record_scalars( for key in ("correction_uid", "object_uid"): _uid(correction, key) _required_text(correction, "correction_action") + _bounded_text(correction, "correction_uid", 96) + _bounded_text(correction, "correction_action", 64) _optional_text(correction, "rationale") _uid_list(correction, "source_segment_uids") if not isinstance(correction.get("before_json"), dict) or not isinstance( correction.get("after_json"), dict ): _fail() + _validate_safe_metadata(correction["before_json"]) + _validate_safe_metadata(correction["after_json"]) _parse_datetime(correction.get("created_at")) @@ -1298,6 +1462,10 @@ def _validate_record_graph(records: Mapping[str, object]) -> None: record["content_segment_uid"]: record["email_uid"] for record in collections["content_segments"] } + segment_attachment = { + record["content_segment_uid"]: record["attachment_uid"] + for record in collections["content_segments"] + } def require_reference(value: object, available: set[str]) -> None: if value is not None and value not in available: @@ -1315,6 +1483,11 @@ def require_reference(value: object, available: set[str]) -> None: and attachment_email[record["attachment_uid"]] != email_uid ): _fail() + if ( + record["attachment_uid"] is not None + and record["source_record_uid"] != f"attachment:{record['attachment_uid']}" + ): + _fail() if ( record["parent_node_uid"] is not None and node_email[record["parent_node_uid"]] != email_uid @@ -1332,10 +1505,25 @@ def require_reference(value: object, available: set[str]) -> None: and attachment_email[record["attachment_uid"]] != email_uid ): _fail() + if ( + record["attachment_uid"] is not None + and record["source_record_uid"] != f"attachment:{record['attachment_uid']}" + ): + _fail() for record in collections["structural_edges"]: email_uid = record["email_uid"] require_reference(email_uid, email_uids) require_reference(record["attachment_uid"], attachment_uids) + if ( + record["attachment_uid"] is not None + and attachment_email[record["attachment_uid"]] != email_uid + ): + _fail() + if ( + record["attachment_uid"] is not None + and record["source_record_uid"] != f"attachment:{record['attachment_uid']}" + ): + _fail() for key in ("source_node_uid", "target_node_uid"): require_reference(record[key], node_uids) if record[key] is not None and node_email[record[key]] != email_uid: @@ -1350,11 +1538,31 @@ def require_reference(value: object, available: set[str]) -> None: require_reference(record["primary_content_segment_uid"], segment_uids) if segment_email[record["primary_content_segment_uid"]] != record["email_uid"]: _fail() + if ( + record["attachment_uid"] is not None + and attachment_email[record["attachment_uid"]] != record["email_uid"] + ): + _fail() + if ( + segment_attachment[record["primary_content_segment_uid"]] + != record["attachment_uid"] + ): + _fail() for segment_uid in record["source_segment_uids"]: require_reference(segment_uid, segment_uids) for record in collections["project_edges"]: require_reference(record["source_object_uid"], object_uids) require_reference(record["target_object_uid"], object_uids) + if ( + record["source_object_uid"] is not None + and record["source_uid"] != record["source_object_uid"] + ): + _fail() + if ( + record["target_object_uid"] is not None + and record["target_uid"] != record["target_object_uid"] + ): + _fail() require_reference(record["primary_content_segment_uid"], segment_uids) for segment_uid in record["source_segment_uids"]: require_reference(segment_uid, segment_uids) @@ -1363,6 +1571,26 @@ def require_reference(value: object, available: set[str]) -> None: for segment_uid in record["source_segment_uids"]: require_reference(segment_uid, segment_uids) + rooted_email_uids = { + record["email_uid"] for record in collections["project_objects"] + } + for record in collections["project_objects"]: + rooted_email_uids.add(segment_email[record["primary_content_segment_uid"]]) + rooted_email_uids.update( + segment_email[segment_uid] for segment_uid in record["source_segment_uids"] + ) + for record in collections["project_edges"]: + rooted_email_uids.add(segment_email[record["primary_content_segment_uid"]]) + rooted_email_uids.update( + segment_email[segment_uid] for segment_uid in record["source_segment_uids"] + ) + for record in collections["corrections"]: + rooted_email_uids.update( + segment_email[segment_uid] for segment_uid in record["source_segment_uids"] + ) + if email_uids and rooted_email_uids != email_uids: + _fail() + async def _matching_models( session: AsyncSession, @@ -1429,6 +1657,10 @@ async def _preflight_existing( attachment_records, attachment_uids = _attachment_records( attachment_rows, email_uids ) + attachment_references = { + f"attachment-{attachment_id}": f"attachment:{attachment_uid}" + for attachment_id, attachment_uid in attachment_uids.items() + } attachments_by_id = {row.id: row for row in attachment_rows} models["attachments"] = { uid: attachments_by_id[attachment_id] @@ -1515,7 +1747,13 @@ async def _preflight_existing( for uid, row in models["structural_edges"].items() } existing_serialized["project_objects"] = { - uid: _project_object_record(row, email_uids, attachment_uids, segment_uids) + uid: _project_object_record( + row, + email_uids, + attachment_uids, + segment_uids, + attachment_references, + ) for uid, row in models["project_objects"].items() } existing_serialized["project_edges"] = { @@ -1523,7 +1761,7 @@ async def _preflight_existing( for uid, row in models["project_edges"].items() } existing_serialized["corrections"] = { - uid: _correction_record(row, object_uids) + uid: _correction_record(row, object_uids, attachment_references) for uid, row in models["corrections"].items() } except KeyError: diff --git a/backend/tests/test_tenant_provenance_bundle.py b/backend/tests/test_tenant_provenance_bundle.py index de12d22c5..7b1c32177 100644 --- a/backend/tests/test_tenant_provenance_bundle.py +++ b/backend/tests/test_tenant_provenance_bundle.py @@ -27,6 +27,7 @@ ProjectGraphEdgeRecord, ProjectGraphObjectRecord, ) +from services.project_graph.repository import ProjectGraphRepository from services.tenant_provenance_bundle import ( ARCHIVE_MAX_BYTES, @@ -278,20 +279,24 @@ def test_parse_rejects_duplicate_json_keys_even_with_valid_manifests(): parse_provenance_archive(_archive_with_entries(_rebuild_manifests(entries))) -def test_build_rejects_non_finite_json_numbers(): - records = {**RECORDS, "export_activity": {"score": float("nan")}} +@pytest.mark.parametrize("value", (float("nan"), float("inf"), float("-inf"))) +def test_build_rejects_non_finite_json_numbers(value): + records = {**RECORDS, "export_activity": {"score": value}} with pytest.raises(ProvenanceArchiveError): build_provenance_archive(records) -@pytest.mark.parametrize( - "value", - (-0.0, 1e-7, 1.0, JSON_SAFE_INTEGER_MAX + 1), -) -def test_build_rejects_values_outside_the_stdlib_jcs_subset(value): +def test_build_rejects_integer_outside_the_json_safe_range(): with pytest.raises(ProvenanceArchiveError): - build_provenance_archive({**RECORDS, "unsupported": value}) + build_provenance_archive({**RECORDS, "unsupported": JSON_SAFE_INTEGER_MAX + 1}) + + +@pytest.mark.parametrize("value", (-0.0, 1e-7, 1.0, 0.73)) +def test_build_round_trips_finite_json_floats(value): + records = {**RECORDS, "finite_value": value} + + assert parse_provenance_archive(build_provenance_archive(records)) == records def test_build_rejects_non_ascii_object_keys(): @@ -826,6 +831,73 @@ async def _add_duplicate_text_attachments( await session.commit() +async def _seed_email_graph_without_project_rows( + session, + *, + scope: TenantProvenanceScope, + token: str, +) -> dict[str, object]: + seeded = await _seed_provenance_closure(session, scope=scope, token=token) + await session.execute( + delete(ProjectGraphCorrectionRecord).where( + ProjectGraphCorrectionRecord.workspace_id == scope.workspace_id, + ProjectGraphCorrectionRecord.correction_uid == f"correction-{token}", + ) + ) + await session.execute( + delete(ProjectGraphEdgeRecord).where( + ProjectGraphEdgeRecord.workspace_id == scope.workspace_id, + ProjectGraphEdgeRecord.edge_uid == f"project-edge-{token}", + ) + ) + await session.execute( + delete(ProjectGraphObjectRecord).where( + ProjectGraphObjectRecord.object_uid.in_(seeded["object_uids"]) + ) + ) + await session.commit() + return seeded + + +async def _two_email_rooted_archive( + session, + *, + scope: TenantProvenanceScope, + token: str, +) -> bytes: + source = await _seed_provenance_closure(session, scope=scope, token=token) + cited = await _seed_email_graph_without_project_rows( + session, + scope=scope, + token=f"cited-{token}", + ) + project_object = await session.scalar( + select(ProjectGraphObjectRecord).where( + ProjectGraphObjectRecord.object_uid == source["object_uids"][0] + ) + ) + project_object.source_segment_uids = sorted( + [*project_object.source_segment_uids, f"segment-cited-{token}"] + ) + session.add( + Attachment( + email_id=source["email_id"], + filename="alternate-evidence.txt", + content="Alternate parser-confirmed attachment evidence", + content_type="text/plain", + parse_status="parsed", + parse_content_type="text/plain", + parser_key="plain_text", + embedding=None, + ) + ) + await session.commit() + archive = await export_tenant_provenance(session, scope) + records = parse_provenance_archive(archive) + assert cited["email_uid"] in {record["email_uid"] for record in records["emails"]} + return archive + + def _scope(token: str) -> TenantProvenanceScope: return TenantProvenanceScope( user_id=f"user-{token}", @@ -1207,3 +1279,439 @@ async def test_duplicate_canonical_attachments_keep_node_and_segment_identity( for record in restored["content_segments"] if record["content_segment_uid"].startswith("duplicate-segment-") } == source_segment_attachments + + +@pytest.mark.asyncio +@pytest.mark.postgres +async def test_recognized_pdf_omits_integer_keys_and_round_trips_graph_references( + provenance_sessionmaker, +): + token = uuid.uuid4().hex[:12] + source_scope = _scope(f"source-{token}") + target_scope = _scope(f"target-{token}") + async with provenance_sessionmaker() as session: + source = await _seed_provenance_closure( + session, scope=source_scope, token=token + ) + attachment = await session.get(Attachment, source["attachment_id"]) + attachment.content = "Recognized PDF evidence" + attachment.content_type = "application/pdf" + attachment.parse_status = "parsed" + attachment.parse_content_type = "application/pdf" + attachment.parser_key = "pdf" + legacy_source_uid = f"attachment-{attachment.id}" + node = await session.scalar( + select(ContentNodeRecord).where( + ContentNodeRecord.content_node_uid == f"node-{token}" + ) + ) + segment = await session.scalar( + select(ContentSegmentRecord).where( + ContentSegmentRecord.content_segment_uid == f"segment-{token}" + ) + ) + structural_edge = await session.scalar( + select(KnowledgeGraphEdgeRecord).where( + KnowledgeGraphEdgeRecord.edge_uid == f"structural-edge-{token}" + ) + ) + project_object = await session.scalar( + select(ProjectGraphObjectRecord).where( + ProjectGraphObjectRecord.object_uid == source["object_uids"][0] + ) + ) + node.source_record_uid = legacy_source_uid + segment.source_record_uid = legacy_source_uid + structural_edge.source_record_uid = legacy_source_uid + project_object.attributes_json = { + "source_record_uid": legacy_source_uid, + "source_object_uid": project_object.object_uid, + } + await session.commit() + async with provenance_sessionmaker() as session: + archive = await export_tenant_provenance(session, source_scope) + records = parse_provenance_archive(archive) + attachment_uid = records["attachments"][0]["attachment_uid"] + portable_source_uid = f"attachment:{attachment_uid}" + serialized = json.dumps(records, sort_keys=True) + + assert legacy_source_uid not in serialized + assert { + records["content_nodes"][0]["source_record_uid"], + records["content_segments"][0]["source_record_uid"], + records["structural_edges"][0]["source_record_uid"], + } == {portable_source_uid} + assert ( + records["project_objects"][0]["attributes_json"]["source_record_uid"] + == portable_source_uid + ) + + async with provenance_sessionmaker() as session: + await _delete_exported_closure(session, records) + async with provenance_sessionmaker() as session: + await import_tenant_provenance(session, target_scope, archive) + async with provenance_sessionmaker() as session: + restored = parse_provenance_archive( + await export_tenant_provenance(session, target_scope) + ) + restored_node = await session.scalar( + select(ContentNodeRecord).where( + ContentNodeRecord.content_node_uid == f"node-{token}" + ) + ) + + assert restored_node.source_record_uid == portable_source_uid + assert restored["content_nodes"][0]["attachment_uid"] == attachment_uid + assert restored["content_nodes"][0]["source_record_uid"] == portable_source_uid + + +@pytest.mark.asyncio +@pytest.mark.postgres +async def test_production_correction_float_round_trip(provenance_sessionmaker): + token = uuid.uuid4().hex[:12] + source_scope = _scope(f"source-{token}") + target_scope = _scope(f"target-{token}") + async with provenance_sessionmaker() as session: + source = await _seed_provenance_closure( + session, scope=source_scope, token=token + ) + correction = await ProjectGraphRepository(session).apply_correction( + object_uid=source["object_uids"][0], + user_id=source_scope.user_id, + organization_id=source_scope.organization_id, + workspace_id=source_scope.workspace_id, + actor_user_id=source_scope.user_id, + correction_action="adjust_confidence", + after_json={"confidence": 0.73}, + rationale="Production correction path", + ) + correction_uid = correction.correction_uid + await session.commit() + async with provenance_sessionmaker() as session: + archive = await export_tenant_provenance(session, source_scope) + records = parse_provenance_archive(archive) + correction_record = next( + record + for record in records["corrections"] + if record["correction_uid"] == correction_uid + ) + + assert correction_record["before_json"]["confidence"] == 0.91 + assert correction_record["after_json"]["confidence"] == 0.73 + assert isinstance(correction_record["before_json"]["confidence"], float) + assert isinstance(correction_record["after_json"]["confidence"], float) + + async with provenance_sessionmaker() as session: + await _delete_exported_closure(session, records) + async with provenance_sessionmaker() as session: + await import_tenant_provenance(session, target_scope, archive) + async with provenance_sessionmaker() as session: + restored = await session.scalar( + select(ProjectGraphCorrectionRecord).where( + ProjectGraphCorrectionRecord.correction_uid == correction_uid + ) + ) + + assert restored.before_json["confidence"] == 0.91 + assert restored.after_json["confidence"] == 0.73 + assert isinstance(restored.before_json["confidence"], float) + assert isinstance(restored.after_json["confidence"], float) + + +@pytest.mark.parametrize( + ("forbidden_key", "forbidden_value"), + ( + ("api_key", "key-material"), + ("provider_url", "https://provider.example/v1"), + ("access_token", "token-material"), + ("email_id", 123), + ), +) +@pytest.mark.asyncio +@pytest.mark.postgres +async def test_export_rejects_nested_sensitive_metadata( + provenance_sessionmaker, + forbidden_key, + forbidden_value, +): + token = uuid.uuid4().hex[:12] + scope = _scope(token) + async with provenance_sessionmaker() as session: + source = await _seed_provenance_closure(session, scope=scope, token=token) + project_object = await session.scalar( + select(ProjectGraphObjectRecord).where( + ProjectGraphObjectRecord.object_uid == source["object_uids"][0] + ) + ) + project_object.attributes_json = { + "source_object_uid": project_object.object_uid, + "source_segment_uids": [f"segment-{token}"], + "nested_metadata": {forbidden_key: forbidden_value}, + } + await session.commit() + async with provenance_sessionmaker() as session: + with pytest.raises(ProvenanceArchiveError): + await export_tenant_provenance(session, scope) + + +@pytest.mark.parametrize( + ("forbidden_key", "forbidden_value"), + ( + ("client_secret", "secret-material"), + ("provider_endpoint", "https://provider.example/v1"), + ("refresh_token", "token-material"), + ("attachment_id", 456), + ), +) +@pytest.mark.asyncio +@pytest.mark.postgres +async def test_import_rejects_nested_sensitive_metadata_before_flush( + provenance_sessionmaker, + monkeypatch, + forbidden_key, + forbidden_value, +): + token = uuid.uuid4().hex[:12] + source_scope = _scope(f"source-{token}") + target_scope = _scope(f"target-{token}") + async with provenance_sessionmaker() as session: + await _seed_provenance_closure(session, scope=source_scope, token=token) + async with provenance_sessionmaker() as session: + records = parse_provenance_archive( + await export_tenant_provenance(session, source_scope) + ) + records["corrections"][0]["after_json"] = { + "confidence": 1, + "source_object_uid": records["corrections"][0]["object_uid"], + "nested_metadata": {forbidden_key: forbidden_value}, + } + archive = build_provenance_archive(records) + async with provenance_sessionmaker() as session: + await _delete_exported_closure(session, records) + async with provenance_sessionmaker() as session: + flush_count = 0 + original_flush = session.flush + + async def counting_flush(*args, **kwargs): + nonlocal flush_count + flush_count += 1 + return await original_flush(*args, **kwargs) + + monkeypatch.setattr(session, "flush", counting_flush) + with pytest.raises(ProvenanceArchiveError): + await import_tenant_provenance(session, target_scope, archive) + assert flush_count == 0 + + +@pytest.mark.asyncio +@pytest.mark.postgres +async def test_unrooted_email_rejected_without_mutation_and_empty_allowed( + provenance_sessionmaker, + monkeypatch, +): + token = uuid.uuid4().hex[:12] + source_scope = _scope(f"source-{token}") + target_scope = _scope(f"target-{token}") + async with provenance_sessionmaker() as session: + await _seed_provenance_closure(session, scope=source_scope, token=token) + async with provenance_sessionmaker() as session: + rooted = parse_provenance_archive( + await export_tenant_provenance(session, source_scope) + ) + unrooted = copy.deepcopy(rooted) + unrooted["project_objects"] = [] + unrooted["project_edges"] = [] + unrooted["corrections"] = [] + unrooted_archive = build_provenance_archive(unrooted) + empty = copy.deepcopy(rooted) + for collection in ( + "emails", + "attachments", + "content_nodes", + "content_segments", + "structural_edges", + "project_objects", + "project_edges", + "corrections", + ): + empty[collection] = [] + empty_archive = build_provenance_archive(empty) + + async with provenance_sessionmaker() as session: + await _delete_exported_closure(session, rooted) + + async with provenance_sessionmaker() as session: + flush_count = 0 + original_flush = session.flush + + async def counting_flush(*args, **kwargs): + nonlocal flush_count + flush_count += 1 + return await original_flush(*args, **kwargs) + + monkeypatch.setattr(session, "flush", counting_flush) + with pytest.raises(ProvenanceArchiveError): + await import_tenant_provenance(session, target_scope, unrooted_archive) + assert flush_count == 0 + async with provenance_sessionmaker() as session: + receipt = await import_tenant_provenance(session, target_scope, empty_archive) + + assert sum(receipt.created.values()) == 0 + assert sum(receipt.skipped.values()) == 0 + + +@pytest.mark.parametrize( + "conflict_kind", + ( + "structural_attachment_email", + "object_attachment_email", + "object_attachment_primary", + "edge_source_endpoint", + "edge_target_endpoint", + ), +) +@pytest.mark.asyncio +@pytest.mark.postgres +async def test_cross_field_conflict_rejected_before_flush( + provenance_sessionmaker, + monkeypatch, + conflict_kind, +): + token = uuid.uuid4().hex[:12] + source_scope = _scope(f"source-{token}") + target_scope = _scope(f"target-{token}") + async with provenance_sessionmaker() as session: + archive = await _two_email_rooted_archive( + session, scope=source_scope, token=token + ) + records = parse_provenance_archive(archive) + source_email_uid = f"<{token}@example.com>" + foreign_attachment_uid = next( + record["attachment_uid"] + for record in records["attachments"] + if record["email_uid"] != source_email_uid + ) + same_email_attachment_uid = next( + record["attachment_uid"] + for record in records["attachments"] + if record["email_uid"] == source_email_uid + and record["attachment_uid"] != records["project_objects"][0]["attachment_uid"] + ) + if conflict_kind == "structural_attachment_email": + source_structural_edge = next( + record + for record in records["structural_edges"] + if record["email_uid"] == source_email_uid + ) + source_structural_edge["attachment_uid"] = foreign_attachment_uid + elif conflict_kind == "object_attachment_email": + records["project_objects"][0]["attachment_uid"] = foreign_attachment_uid + elif conflict_kind == "object_attachment_primary": + records["project_objects"][0]["attachment_uid"] = same_email_attachment_uid + elif conflict_kind == "edge_source_endpoint": + records["project_edges"][0]["source_uid"] = "mismatched-logical-endpoint" + else: + records["project_edges"][0]["target_uid"] = "mismatched-logical-endpoint" + invalid_archive = build_provenance_archive(records) + async with provenance_sessionmaker() as session: + await _delete_exported_closure(session, records) + async with provenance_sessionmaker() as session: + flush_count = 0 + original_flush = session.flush + + async def counting_flush(*args, **kwargs): + nonlocal flush_count + flush_count += 1 + return await original_flush(*args, **kwargs) + + monkeypatch.setattr(session, "flush", counting_flush) + with pytest.raises(ProvenanceArchiveError): + await import_tenant_provenance(session, target_scope, invalid_archive) + assert flush_count == 0 + + +@pytest.mark.parametrize("bound_kind", ("node_uid", "object_title", "edge_source")) +@pytest.mark.asyncio +@pytest.mark.postgres +async def test_column_bound_preflight_rejects_before_flush( + provenance_sessionmaker, + monkeypatch, + bound_kind, +): + token = uuid.uuid4().hex[:12] + source_scope = _scope(f"source-{token}") + target_scope = _scope(f"target-{token}") + async with provenance_sessionmaker() as session: + await _seed_provenance_closure(session, scope=source_scope, token=token) + async with provenance_sessionmaker() as session: + records = parse_provenance_archive( + await export_tenant_provenance(session, source_scope) + ) + if bound_kind == "node_uid": + old_node_uid = records["content_nodes"][0]["content_node_uid"] + oversized_node_uid = "n" * 65 + records["content_nodes"][0]["content_node_uid"] = oversized_node_uid + for segment in records["content_segments"]: + if segment["content_node_uid"] == old_node_uid: + segment["content_node_uid"] = oversized_node_uid + for edge in records["structural_edges"]: + if edge["source_node_uid"] == old_node_uid: + edge["source_node_uid"] = oversized_node_uid + if edge["target_node_uid"] == old_node_uid: + edge["target_node_uid"] = oversized_node_uid + elif bound_kind == "object_title": + records["project_objects"][0]["title"] = "t" * 241 + else: + records["project_edges"][0]["source_uid"] = "s" * 161 + invalid_archive = build_provenance_archive(records) + async with provenance_sessionmaker() as session: + await _delete_exported_closure(session, records) + async with provenance_sessionmaker() as session: + flush_count = 0 + original_flush = session.flush + + async def counting_flush(*args, **kwargs): + nonlocal flush_count + flush_count += 1 + return await original_flush(*args, **kwargs) + + monkeypatch.setattr(session, "flush", counting_flush) + with pytest.raises(ProvenanceArchiveError): + await import_tenant_provenance(session, target_scope, invalid_archive) + assert flush_count == 0 + + +@pytest.mark.parametrize("scope_dimension", ("user", "organization")) +@pytest.mark.asyncio +@pytest.mark.postgres +async def test_export_excludes_separate_user_and_organization_scope( + provenance_sessionmaker, + scope_dimension, +): + token = uuid.uuid4().hex[:12] + scope = _scope(f"selected-{token}") + rival_scope = TenantProvenanceScope( + user_id=f"rival-user-{token}" if scope_dimension == "user" else scope.user_id, + organization_id=( + f"rival-org-{token}" + if scope_dimension == "organization" + else scope.organization_id + ), + workspace_id=scope.workspace_id, + ) + async with provenance_sessionmaker() as session: + selected = await _seed_provenance_closure( + session, scope=scope, token=f"selected-{token}" + ) + rival = await _seed_provenance_closure( + session, scope=rival_scope, token=f"rival-{token}" + ) + async with provenance_sessionmaker() as session: + records = parse_provenance_archive( + await export_tenant_provenance(session, scope) + ) + + assert {record["email_uid"] for record in records["emails"]} == { + selected["email_uid"] + } + assert rival["email_uid"] not in json.dumps(records) From 832798ea3878d49fc0f28fff427b1bfd0dff3b85 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Mon, 31 Aug 2026 23:35:07 +0900 Subject: [PATCH 08/39] fix: classify compound provenance secrets --- backend/services/tenant_provenance_bundle.py | 21 ++++++- .../tests/test_tenant_provenance_bundle.py | 60 +++++++++++++++++++ 2 files changed, 79 insertions(+), 2 deletions(-) diff --git a/backend/services/tenant_provenance_bundle.py b/backend/services/tenant_provenance_bundle.py index ee0c94efc..30495c36f 100644 --- a/backend/services/tenant_provenance_bundle.py +++ b/backend/services/tenant_provenance_bundle.py @@ -105,6 +105,9 @@ "token", } ) +_SENSITIVE_METADATA_TOKENS = frozenset( + {"credential", "credentials", "password", "secret", "secrets", "token"} +) _RECORD_KEYS = { "emails": frozenset( { @@ -284,6 +287,21 @@ def _validate_json_value(value: object, depth: int = 0) -> None: _fail() +def _metadata_key_is_forbidden(key: str) -> bool: + normalized_key = key.lower().replace("-", "_").replace(" ", "_") + tokens = tuple(token for token in normalized_key.split("_") if token) + token_set = frozenset(tokens) + token_pairs = set(zip(tokens, tokens[1:])) + return ( + normalized_key in _FORBIDDEN_METADATA_KEYS + or bool(token_set & _SENSITIVE_METADATA_TOKENS) + or ("api", "key") in token_pairs + or ("provider" in token_set and bool({"endpoint", "url", "uri"} & token_set)) + or (bool(tokens) and tokens[-1] == "id") + or (bool({"database", "db"} & token_set) and bool({"id", "key"} & token_set)) + ) + + def _validate_safe_metadata(value: object, depth: int = 0) -> None: if depth > JSON_MAX_DEPTH: _fail() @@ -291,8 +309,7 @@ def _validate_safe_metadata(value: object, depth: int = 0) -> None: for key, item in value.items(): if not isinstance(key, str): _fail() - normalized_key = key.lower().replace("-", "_").replace(" ", "_") - if normalized_key in _FORBIDDEN_METADATA_KEYS: + if _metadata_key_is_forbidden(key): _fail() _validate_safe_metadata(item, depth + 1) elif isinstance(value, list): diff --git a/backend/tests/test_tenant_provenance_bundle.py b/backend/tests/test_tenant_provenance_bundle.py index 7b1c32177..ae171463d 100644 --- a/backend/tests/test_tenant_provenance_bundle.py +++ b/backend/tests/test_tenant_provenance_bundle.py @@ -1425,6 +1425,12 @@ async def test_production_correction_float_round_trip(provenance_sessionmaker): ("provider_url", "https://provider.example/v1"), ("access_token", "token-material"), ("email_id", 123), + ("smtp_password", "password-material"), + ("backup_api_key", "key-material"), + ("provider_auth_token", "token-material"), + ("mail_credentials_blob", "credential-material"), + ("backup_provider_endpoint_url", "https://provider.example/v1"), + ("legacy_attachment_id", 123), ), ) @pytest.mark.asyncio @@ -1461,6 +1467,12 @@ async def test_export_rejects_nested_sensitive_metadata( ("provider_endpoint", "https://provider.example/v1"), ("refresh_token", "token-material"), ("attachment_id", 456), + ("smtp_password", "password-material"), + ("backup_api_key", "key-material"), + ("provider_auth_token", "token-material"), + ("mail_credentials_blob", "credential-material"), + ("backup_provider_endpoint_url", "https://provider.example/v1"), + ("legacy_attachment_id", 456), ), ) @pytest.mark.asyncio @@ -1503,6 +1515,54 @@ async def counting_flush(*args, **kwargs): assert flush_count == 0 +@pytest.mark.asyncio +@pytest.mark.postgres +async def test_nested_benign_metadata_and_stable_uids_round_trip( + provenance_sessionmaker, +): + token = uuid.uuid4().hex[:12] + source_scope = _scope(f"source-{token}") + target_scope = _scope(f"target-{token}") + allowed_metadata = { + "confidence": 0.73, + "source_object_uid": f"project-object-source-{token}", + "source_segment_uids": [f"segment-{token}"], + "tokenization_strategy": "bounded", + "passwordless_mode": "enabled", + "credentialed_source": "oidc", + "secretary_note": "benign domain text", + "identifier_kind": "stable", + } + async with provenance_sessionmaker() as session: + source = await _seed_provenance_closure( + session, scope=source_scope, token=token + ) + project_object = await session.scalar( + select(ProjectGraphObjectRecord).where( + ProjectGraphObjectRecord.object_uid == source["object_uids"][0] + ) + ) + project_object.attributes_json = allowed_metadata + await session.commit() + async with provenance_sessionmaker() as session: + records = parse_provenance_archive( + await export_tenant_provenance(session, source_scope) + ) + archive = build_provenance_archive(records) + async with provenance_sessionmaker() as session: + await _delete_exported_closure(session, records) + async with provenance_sessionmaker() as session: + await import_tenant_provenance(session, target_scope, archive) + async with provenance_sessionmaker() as session: + restored = await session.scalar( + select(ProjectGraphObjectRecord).where( + ProjectGraphObjectRecord.object_uid == source["object_uids"][0] + ) + ) + + assert restored.attributes_json == allowed_metadata + + @pytest.mark.asyncio @pytest.mark.postgres async def test_unrooted_email_rejected_without_mutation_and_empty_allowed( From 9d78942cd152d355bcddb32dad896414d7c23cf4 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Mon, 31 Aug 2026 23:44:12 +0900 Subject: [PATCH 09/39] fix: tokenize provenance metadata keys --- backend/services/tenant_provenance_bundle.py | 11 +++++++++-- backend/tests/test_tenant_provenance_bundle.py | 16 ++++++++++++++++ 2 files changed, 25 insertions(+), 2 deletions(-) diff --git a/backend/services/tenant_provenance_bundle.py b/backend/services/tenant_provenance_bundle.py index 30495c36f..3d1a4cfb6 100644 --- a/backend/services/tenant_provenance_bundle.py +++ b/backend/services/tenant_provenance_bundle.py @@ -6,6 +6,7 @@ import io import json import math +import re import stat import struct import zipfile @@ -108,6 +109,10 @@ _SENSITIVE_METADATA_TOKENS = frozenset( {"credential", "credentials", "password", "secret", "secrets", "token"} ) +_METADATA_KEY_TOKEN_PATTERN = re.compile( + r"[A-Z]+(?=[A-Z][a-z]|[0-9]|[^A-Za-z0-9]|$)" + r"|[A-Z]?[a-z]+|[A-Z]+|[0-9]+" +) _RECORD_KEYS = { "emails": frozenset( { @@ -288,8 +293,10 @@ def _validate_json_value(value: object, depth: int = 0) -> None: def _metadata_key_is_forbidden(key: str) -> bool: - normalized_key = key.lower().replace("-", "_").replace(" ", "_") - tokens = tuple(token for token in normalized_key.split("_") if token) + tokens = tuple( + match.group(0).lower() for match in _METADATA_KEY_TOKEN_PATTERN.finditer(key) + ) + normalized_key = "_".join(tokens) token_set = frozenset(tokens) token_pairs = set(zip(tokens, tokens[1:])) return ( diff --git a/backend/tests/test_tenant_provenance_bundle.py b/backend/tests/test_tenant_provenance_bundle.py index ae171463d..9de712d23 100644 --- a/backend/tests/test_tenant_provenance_bundle.py +++ b/backend/tests/test_tenant_provenance_bundle.py @@ -1431,6 +1431,14 @@ async def test_production_correction_float_round_trip(provenance_sessionmaker): ("mail_credentials_blob", "credential-material"), ("backup_provider_endpoint_url", "https://provider.example/v1"), ("legacy_attachment_id", 123), + ("smtpPassword", "password-material"), + ("backupApiKey", "key-material"), + ("providerAuthToken", "token-material"), + ("provider.endpoint.url", "https://provider.example/v1"), + ("legacyAttachmentId", 123), + ("smtp/password", "password-material"), + ("APIKey", "key-material"), + ("OAuthToken", "token-material"), ), ) @pytest.mark.asyncio @@ -1473,6 +1481,14 @@ async def test_export_rejects_nested_sensitive_metadata( ("mail_credentials_blob", "credential-material"), ("backup_provider_endpoint_url", "https://provider.example/v1"), ("legacy_attachment_id", 456), + ("smtpPassword", "password-material"), + ("backupApiKey", "key-material"), + ("providerAuthToken", "token-material"), + ("provider.endpoint.url", "https://provider.example/v1"), + ("legacyAttachmentId", 456), + ("smtp/password", "password-material"), + ("APIKey", "key-material"), + ("OAuthToken", "token-material"), ), ) @pytest.mark.asyncio From aca3f3aee4dcc0402c0205226e04b5ac58f7edd0 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Tue, 1 Sep 2026 00:05:17 +0900 Subject: [PATCH 10/39] feat(data): add signed provenance bundle API --- backend/api/data.py | 68 ++++++++++++- backend/tests/test_data_api.py | 175 +++++++++++++++++++++++++++++++-- 2 files changed, 236 insertions(+), 7 deletions(-) diff --git a/backend/api/data.py b/backend/api/data.py index dccd85890..9528c0f70 100644 --- a/backend/api/data.py +++ b/backend/api/data.py @@ -7,7 +7,7 @@ import re from typing import Literal, NamedTuple -from fastapi import APIRouter, Depends, File, Form, HTTPException, UploadFile +from fastapi import APIRouter, Depends, File, Form, HTTPException, Request, Response, UploadFile from pydantic import BaseModel, ConfigDict, Field from sqlalchemy import and_, case, func, or_, select from sqlalchemy.engine import Row @@ -35,6 +35,13 @@ PDF_DOM_RECOGNITION_PENDING_STATUS, ) from services.ontology_service import ontology_service +from services.tenant_provenance_bundle import ( + ARCHIVE_MAX_BYTES, + ProvenanceArchiveError, + TenantProvenanceScope, + export_tenant_provenance, + import_tenant_provenance, +) from services.webdav_service import webdav_service router = APIRouter(prefix="/api/data", tags=["data"]) @@ -45,6 +52,7 @@ # would let a caller stash a pending document the configured sidecar will always # reject while the base64 copy inflates the database. _MAX_PDF_DOM_UPLOAD_BYTES = 20 * 1024 * 1024 +_PROVENANCE_ARCHIVE_MAX_BYTES = ARCHIVE_MAX_BYTES ATTACHMENT_PARSE_BREAKDOWN_EVIDENCE_SOURCE = ( "email_attachments.content_type, " "email_attachments.parse_content_type, " @@ -3160,6 +3168,64 @@ def _quality_checks( ] +def _provenance_scope(auth_context: AuthContext) -> TenantProvenanceScope: + return TenantProvenanceScope( + user_id=auth_context.user_id, + organization_id=auth_context.organization_id, + workspace_id=auth_context.workspace_id, + ) + + +async def _read_provenance_archive(request: Request) -> bytes: + archive = bytearray() + async for chunk in request.stream(): + archive.extend(chunk) + if len(archive) > _PROVENANCE_ARCHIVE_MAX_BYTES: + raise HTTPException(status_code=413, detail="Provenance archive too large") + return bytes(archive) + + +@router.get("/provenance-bundle") +async def download_provenance_bundle( + auth_context: AuthContext = Depends(get_auth_context), + db: AsyncSession = Depends(get_db), +) -> Response: + try: + archive = await export_tenant_provenance(db, _provenance_scope(auth_context)) + except ProvenanceArchiveError as exc: + raise HTTPException( + status_code=400, detail="Invalid provenance archive" + ) from exc + return Response( + content=archive, + media_type="application/zip", + headers={"Content-Disposition": 'attachment; filename="naruon-provenance.zip"'}, + ) + + +@router.post("/provenance-bundle/import") +async def upload_provenance_bundle( + request: Request, + auth_context: AuthContext = Depends(get_auth_context), + db: AsyncSession = Depends(get_db), +) -> dict[str, object]: + archive = await _read_provenance_archive(request) + try: + receipt = await import_tenant_provenance( + db, _provenance_scope(auth_context), archive + ) + except ProvenanceArchiveError as exc: + raise HTTPException( + status_code=400, detail="Invalid provenance archive" + ) from exc + return { + "bundle_uid": receipt.bundle_uid, + "manifest_digest": receipt.manifest_digest, + "created": receipt.created, + "skipped": receipt.skipped, + } + + @router.post("/documents", response_model=DataDocumentActionResponse) async def upload_data_document( request: DataDocumentUploadRequest, diff --git a/backend/tests/test_data_api.py b/backend/tests/test_data_api.py index cd0b7bf37..df6469d63 100644 --- a/backend/tests/test_data_api.py +++ b/backend/tests/test_data_api.py @@ -31,6 +31,7 @@ ) from db.session import get_db from main import app +from services.tenant_provenance_bundle import ImportReceipt, ProvenanceArchiveError TEST_SESSION_HMAC_SECRET = "data-quality-surface-hmac-material-32-bytes" # noqa: S105 @@ -363,6 +364,168 @@ def _expected_sample_key(prefix: str, value: str) -> str: return f"{prefix}_{digest[:16]}" +@pytest.mark.parametrize( + ("method", "path"), + [ + ("get", "/api/data/provenance-bundle"), + ("post", "/api/data/provenance-bundle/import"), + ], +) +def test_provenance_bundle_endpoints_require_signed_session(method, path, mock_db): + async def override_get_db(): + yield mock_db + + original_overrides = dict(app.dependency_overrides) + app.dependency_overrides[get_db] = override_get_db + app.dependency_overrides.pop(get_auth_context, None) + app.dependency_overrides.pop(get_current_user, None) + try: + with TestClient(app) as client: + response = client.request( + method.upper(), + path, + content=b"archive" if method == "post" else b"", + ) + finally: + app.dependency_overrides.clear() + app.dependency_overrides.update(original_overrides) + + assert response.status_code == 401 + assert response.json() == {"detail": "Authentication required"} + + +def test_provenance_bundle_download_uses_signed_scope_and_zip_response( + mock_db, monkeypatch +): + captured = {} + + async def fake_export(session, scope): + captured.update(session=session, scope=scope) + return b"PK\x03\x04bundle" + + monkeypatch.setattr(data_api, "export_tenant_provenance", fake_export) + token = _signed_session_token( + _valid_session_payload( + sub="signed-user", org="signed-org", workspace="signed-workspace" + ) + ) + client, previous_secret, original_overrides = _with_signed_auth(mock_db, token) + try: + response = client.get("/api/data/provenance-bundle") + finally: + client.close() + _restore_overrides(previous_secret, original_overrides) + + assert response.status_code == 200 + assert response.content == b"PK\x03\x04bundle" + assert response.headers["content-type"] == "application/zip" + assert response.headers["content-disposition"] == ( + 'attachment; filename="naruon-provenance.zip"' + ) + assert captured["session"] is mock_db + assert captured["scope"].user_id == "signed-user" + assert captured["scope"].organization_id == "signed-org" + assert captured["scope"].workspace_id == "signed-workspace" + + +def test_provenance_bundle_import_rewrites_target_scope_from_signed_session( + mock_db, monkeypatch +): + captured = {} + + async def fake_import(session, scope, archive_bytes): + captured.update(session=session, scope=scope, archive_bytes=archive_bytes) + return ImportReceipt( + bundle_uid="bundle_portable_1", + manifest_digest="a" * 128, + created={"emails": 1}, + skipped={"emails": 0}, + ) + + monkeypatch.setattr(data_api, "import_tenant_provenance", fake_import) + token = _signed_session_token( + _valid_session_payload( + sub="target-user", org="target-org", workspace="target-workspace" + ) + ) + client, previous_secret, original_overrides = _with_signed_auth(mock_db, token) + try: + response = client.post( + "/api/data/provenance-bundle/import", + content=b"PK\x03\x04portable", + headers={"Content-Type": "application/zip"}, + ) + finally: + client.close() + _restore_overrides(previous_secret, original_overrides) + + assert response.status_code == 200 + assert response.json() == { + "bundle_uid": "bundle_portable_1", + "manifest_digest": "a" * 128, + "created": {"emails": 1}, + "skipped": {"emails": 0}, + } + assert captured["session"] is mock_db + assert captured["archive_bytes"] == b"PK\x03\x04portable" + assert captured["scope"].user_id == "target-user" + assert captured["scope"].organization_id == "target-org" + assert captured["scope"].workspace_id == "target-workspace" + + +def test_provenance_bundle_import_rejects_oversize_before_service_mutation( + mock_db, monkeypatch +): + called = False + + async def fake_import(session, scope, archive_bytes): + nonlocal called + called = True + + monkeypatch.setattr(data_api, "_PROVENANCE_ARCHIVE_MAX_BYTES", 4) + monkeypatch.setattr(data_api, "import_tenant_provenance", fake_import) + token = _signed_session_token(_valid_session_payload()) + client, previous_secret, original_overrides = _with_signed_auth(mock_db, token) + try: + response = client.post( + "/api/data/provenance-bundle/import", + content=b"12345", + headers={"Content-Type": "application/zip"}, + ) + finally: + client.close() + _restore_overrides(previous_secret, original_overrides) + + assert response.status_code == 413 + assert response.json() == {"detail": "Provenance archive too large"} + assert called is False + + +@pytest.mark.parametrize("attacker_detail", ["bad local path", "signature secret"]) +def test_provenance_bundle_import_returns_fixed_safe_archive_errors( + attacker_detail, mock_db, monkeypatch +): + async def fake_import(session, scope, archive_bytes): + raise ProvenanceArchiveError(attacker_detail) + + monkeypatch.setattr(data_api, "import_tenant_provenance", fake_import) + token = _signed_session_token(_valid_session_payload()) + client, previous_secret, original_overrides = _with_signed_auth(mock_db, token) + try: + response = client.post( + "/api/data/provenance-bundle/import", + content=b"not-a-valid-archive", + headers={"Content-Type": "application/zip"}, + ) + finally: + client.close() + _restore_overrides(previous_secret, original_overrides) + + assert response.status_code == 400 + assert response.json() == {"detail": "Invalid provenance archive"} + assert attacker_detail not in response.text + + def _expected_acquisition_readiness_kpis(): return [ { @@ -2951,11 +3114,11 @@ async def _seed_smoke_test_data(conn, ids: dict): """ INSERT INTO email_records ( user_id, organization_id, message_id, thread_id, - fingerprint, sender, recipients, subject, "date", body + fingerprint, sender, recipients, subject, "date", body, is_read ) VALUES ( :user_id, :organization_id, :message_id, :thread_id, - :fingerprint, :sender, :recipients, :subject, now(), :body + :fingerprint, :sender, :recipients, :subject, now(), :body, true ) RETURNING id """ @@ -2977,11 +3140,11 @@ async def _seed_smoke_test_data(conn, ids: dict): """ INSERT INTO email_records ( user_id, organization_id, message_id, sender, recipients, - subject, "date", body + subject, "date", body, is_read ) VALUES ( :user_id, :organization_id, :message_id, :sender, - :recipients, :subject, now(), :body + :recipients, :subject, now(), :body, true ) RETURNING id """ @@ -3001,11 +3164,11 @@ async def _seed_smoke_test_data(conn, ids: dict): """ INSERT INTO email_records ( user_id, organization_id, message_id, thread_id, - fingerprint, sender, recipients, subject, "date", body + fingerprint, sender, recipients, subject, "date", body, is_read ) VALUES ( :user_id, :organization_id, :message_id, :thread_id, - :fingerprint, :sender, :recipients, :subject, now(), :body + :fingerprint, :sender, :recipients, :subject, now(), :body, true ) RETURNING id """ From b848a398b4c9868f389b128550c9a3483b7d56c9 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Tue, 1 Sep 2026 00:21:00 +0900 Subject: [PATCH 11/39] fix(data): require authoritative provenance scope --- backend/api/data.py | 26 +++- backend/tests/test_data_api.py | 240 ++++++++++++++++++++++++++++++--- 2 files changed, 245 insertions(+), 21 deletions(-) diff --git a/backend/api/data.py b/backend/api/data.py index 9528c0f70..573dff312 100644 --- a/backend/api/data.py +++ b/backend/api/data.py @@ -3176,12 +3176,32 @@ def _provenance_scope(auth_context: AuthContext) -> TenantProvenanceScope: ) +def _require_authoritative_provenance_scope(auth_context: AuthContext) -> None: + if auth_context.session_verifier == "hmac": + raise HTTPException( + status_code=403, + detail="Authoritative workspace membership is required for provenance bundles", + ) + + async def _read_provenance_archive(request: Request) -> bytes: + content_length = request.headers.get("content-length") + if content_length is not None: + if not content_length.isdigit(): + raise HTTPException(status_code=400, detail="Invalid Content-Length") + try: + declared_bytes = int(content_length) + except ValueError as exc: + raise HTTPException( + status_code=400, detail="Invalid Content-Length" + ) from exc + if declared_bytes > _PROVENANCE_ARCHIVE_MAX_BYTES: + raise HTTPException(status_code=413, detail="Provenance archive too large") archive = bytearray() async for chunk in request.stream(): - archive.extend(chunk) - if len(archive) > _PROVENANCE_ARCHIVE_MAX_BYTES: + if len(chunk) > _PROVENANCE_ARCHIVE_MAX_BYTES - len(archive): raise HTTPException(status_code=413, detail="Provenance archive too large") + archive.extend(chunk) return bytes(archive) @@ -3190,6 +3210,7 @@ async def download_provenance_bundle( auth_context: AuthContext = Depends(get_auth_context), db: AsyncSession = Depends(get_db), ) -> Response: + _require_authoritative_provenance_scope(auth_context) try: archive = await export_tenant_provenance(db, _provenance_scope(auth_context)) except ProvenanceArchiveError as exc: @@ -3209,6 +3230,7 @@ async def upload_provenance_bundle( auth_context: AuthContext = Depends(get_auth_context), db: AsyncSession = Depends(get_db), ) -> dict[str, object]: + _require_authoritative_provenance_scope(auth_context) archive = await _read_provenance_archive(request) try: receipt = await import_tenant_provenance( diff --git a/backend/tests/test_data_api.py b/backend/tests/test_data_api.py index df6469d63..2c6d7a587 100644 --- a/backend/tests/test_data_api.py +++ b/backend/tests/test_data_api.py @@ -9,6 +9,7 @@ import asyncpg import httpx import pytest +from fastapi import HTTPException, Request from fastapi.testclient import TestClient from cryptography.fernet import Fernet from pydantic import SecretStr @@ -17,7 +18,7 @@ from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine import api.data as data_api -from api.auth import get_auth_context, get_current_user +from api.auth import AuthContext, get_auth_context, get_current_user from core.config import settings from db.models import ( get_fernet, @@ -353,12 +354,28 @@ async def override_get_db(): return client, previous_secret, original_overrides +def _with_authoritative_auth(mock_db, auth_context: AuthContext): + async def override_get_db(): + yield mock_db + + original_overrides = dict(app.dependency_overrides) + app.dependency_overrides[get_db] = override_get_db + app.dependency_overrides[get_auth_context] = lambda: auth_context + app.dependency_overrides.pop(get_current_user, None) + return TestClient(app), original_overrides + + def _restore_overrides(previous_secret, original_overrides): settings.AUTH_SESSION_HMAC_SECRET = previous_secret app.dependency_overrides.clear() app.dependency_overrides.update(original_overrides) +def _restore_authoritative_overrides(original_overrides): + app.dependency_overrides.clear() + app.dependency_overrides.update(original_overrides) + + def _expected_sample_key(prefix: str, value: str) -> str: digest = hashlib.sha256(value.encode("utf-8")).hexdigest() return f"{prefix}_{digest[:16]}" @@ -394,6 +411,39 @@ async def override_get_db(): assert response.json() == {"detail": "Authentication required"} +@pytest.mark.parametrize( + ("method", "path"), + [ + ("get", "/api/data/provenance-bundle"), + ("post", "/api/data/provenance-bundle/import"), + ], +) +def test_provenance_bundle_endpoints_reject_hmac_workspace_claims_before_service( + method, path, mock_db, monkeypatch +): + async def forbidden_service(*args, **kwargs): + raise AssertionError("provenance service must not run") + + monkeypatch.setattr(data_api, "export_tenant_provenance", forbidden_service) + monkeypatch.setattr(data_api, "import_tenant_provenance", forbidden_service) + token = _signed_session_token(_valid_session_payload()) + client, previous_secret, original_overrides = _with_signed_auth(mock_db, token) + try: + response = client.request( + method.upper(), + path, + content=b"attacker archive" if method == "post" else b"", + ) + finally: + client.close() + _restore_overrides(previous_secret, original_overrides) + + assert response.status_code == 403 + assert response.json() == { + "detail": "Authoritative workspace membership is required for provenance bundles" + } + + def test_provenance_bundle_download_uses_signed_scope_and_zip_response( mock_db, monkeypatch ): @@ -404,17 +454,22 @@ async def fake_export(session, scope): return b"PK\x03\x04bundle" monkeypatch.setattr(data_api, "export_tenant_provenance", fake_export) - token = _signed_session_token( - _valid_session_payload( - sub="signed-user", org="signed-org", workspace="signed-workspace" - ) + client, original_overrides = _with_authoritative_auth( + mock_db, + AuthContext( + user_id="signed-user", + role="member", + organization_id="signed-org", + group_ids=(), + workspace_id="signed-workspace", + session_verifier="oidc", + ), ) - client, previous_secret, original_overrides = _with_signed_auth(mock_db, token) try: response = client.get("/api/data/provenance-bundle") finally: client.close() - _restore_overrides(previous_secret, original_overrides) + _restore_authoritative_overrides(original_overrides) assert response.status_code == 200 assert response.content == b"PK\x03\x04bundle" @@ -428,6 +483,34 @@ async def fake_export(session, scope): assert captured["scope"].workspace_id == "signed-workspace" +def test_provenance_bundle_download_returns_fixed_safe_archive_error( + mock_db, monkeypatch +): + async def fake_export(session, scope): + raise ProvenanceArchiveError("private export detail") + + monkeypatch.setattr(data_api, "export_tenant_provenance", fake_export) + client, original_overrides = _with_authoritative_auth( + mock_db, + AuthContext( + user_id="admin", + role="member", + organization_id="org-acme", + group_ids=(), + workspace_id="workspace-org-acme", + ), + ) + try: + response = client.get("/api/data/provenance-bundle") + finally: + client.close() + _restore_authoritative_overrides(original_overrides) + + assert response.status_code == 400 + assert response.json() == {"detail": "Invalid provenance archive"} + assert "private export detail" not in response.text + + def test_provenance_bundle_import_rewrites_target_scope_from_signed_session( mock_db, monkeypatch ): @@ -443,12 +526,17 @@ async def fake_import(session, scope, archive_bytes): ) monkeypatch.setattr(data_api, "import_tenant_provenance", fake_import) - token = _signed_session_token( - _valid_session_payload( - sub="target-user", org="target-org", workspace="target-workspace" - ) + client, original_overrides = _with_authoritative_auth( + mock_db, + AuthContext( + user_id="target-user", + role="member", + organization_id="target-org", + group_ids=(), + workspace_id="target-workspace", + session_verifier="server", + ), ) - client, previous_secret, original_overrides = _with_signed_auth(mock_db, token) try: response = client.post( "/api/data/provenance-bundle/import", @@ -457,7 +545,7 @@ async def fake_import(session, scope, archive_bytes): ) finally: client.close() - _restore_overrides(previous_secret, original_overrides) + _restore_authoritative_overrides(original_overrides) assert response.status_code == 200 assert response.json() == { @@ -484,8 +572,16 @@ async def fake_import(session, scope, archive_bytes): monkeypatch.setattr(data_api, "_PROVENANCE_ARCHIVE_MAX_BYTES", 4) monkeypatch.setattr(data_api, "import_tenant_provenance", fake_import) - token = _signed_session_token(_valid_session_payload()) - client, previous_secret, original_overrides = _with_signed_auth(mock_db, token) + client, original_overrides = _with_authoritative_auth( + mock_db, + AuthContext( + user_id="admin", + role="member", + organization_id="org-acme", + group_ids=("group-data",), + workspace_id="workspace-org-acme", + ), + ) try: response = client.post( "/api/data/provenance-bundle/import", @@ -494,7 +590,7 @@ async def fake_import(session, scope, archive_bytes): ) finally: client.close() - _restore_overrides(previous_secret, original_overrides) + _restore_authoritative_overrides(original_overrides) assert response.status_code == 413 assert response.json() == {"detail": "Provenance archive too large"} @@ -509,8 +605,16 @@ async def fake_import(session, scope, archive_bytes): raise ProvenanceArchiveError(attacker_detail) monkeypatch.setattr(data_api, "import_tenant_provenance", fake_import) - token = _signed_session_token(_valid_session_payload()) - client, previous_secret, original_overrides = _with_signed_auth(mock_db, token) + client, original_overrides = _with_authoritative_auth( + mock_db, + AuthContext( + user_id="admin", + role="member", + organization_id="org-acme", + group_ids=("group-data",), + workspace_id="workspace-org-acme", + ), + ) try: response = client.post( "/api/data/provenance-bundle/import", @@ -519,13 +623,111 @@ async def fake_import(session, scope, archive_bytes): ) finally: client.close() - _restore_overrides(previous_secret, original_overrides) + _restore_authoritative_overrides(original_overrides) assert response.status_code == 400 assert response.json() == {"detail": "Invalid provenance archive"} assert attacker_detail not in response.text +def _stream_request(*chunks: bytes, content_length: str | None = None) -> Request: + messages = [ + { + "type": "http.request", + "body": chunk, + "more_body": index < len(chunks) - 1, + } + for index, chunk in enumerate(chunks) + ] + + async def receive(): + return messages.pop(0) + + headers = [] + if content_length is not None: + headers.append((b"content-length", content_length.encode("ascii"))) + return Request( + { + "type": "http", + "http_version": "1.1", + "method": "POST", + "scheme": "http", + "path": "/api/data/provenance-bundle/import", + "raw_path": b"/api/data/provenance-bundle/import", + "query_string": b"", + "headers": headers, + "client": ("127.0.0.1", 1), + "server": ("testserver", 80), + }, + receive, + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("content_length", "status_code", "detail"), + [ + ("invalid", 400, "Invalid Content-Length"), + ("-1", 400, "Invalid Content-Length"), + ("5", 413, "Provenance archive too large"), + ], +) +async def test_provenance_archive_rejects_invalid_or_oversize_length_before_stream( + content_length, status_code, detail, monkeypatch +): + monkeypatch.setattr(data_api, "_PROVENANCE_ARCHIVE_MAX_BYTES", 4) + request = _stream_request(content_length=content_length) + + with pytest.raises(HTTPException) as exc_info: + await data_api._read_provenance_archive(request) + + assert exc_info.value.status_code == status_code + assert exc_info.value.detail == detail + + +@pytest.mark.asyncio +async def test_provenance_archive_checks_oversize_chunk_before_buffer_copy(monkeypatch): + class GuardedBuffer: + def __len__(self): + return 0 + + def extend(self, chunk): + raise AssertionError("oversize chunk was copied") + + monkeypatch.setattr(data_api, "_PROVENANCE_ARCHIVE_MAX_BYTES", 4) + monkeypatch.setattr(data_api, "bytearray", GuardedBuffer, raising=False) + + with pytest.raises(HTTPException) as exc_info: + await data_api._read_provenance_archive(_stream_request(b"12345")) + + assert exc_info.value.status_code == 413 + assert exc_info.value.detail == "Provenance archive too large" + + +@pytest.mark.asyncio +async def test_provenance_archive_accepts_absent_length_boundary_chunks(monkeypatch): + monkeypatch.setattr(data_api, "_PROVENANCE_ARCHIVE_MAX_BYTES", 4) + + archive = await data_api._read_provenance_archive( + _stream_request(b"12", b"34") + ) + + assert archive == b"1234" + + +@pytest.mark.asyncio +async def test_provenance_archive_rejects_body_larger_than_declared_length(monkeypatch): + monkeypatch.setattr(data_api, "_PROVENANCE_ARCHIVE_MAX_BYTES", 4) + + with pytest.raises(HTTPException) as exc_info: + await data_api._read_provenance_archive( + _stream_request(b"12", b"345", content_length="2") + ) + + assert exc_info.value.status_code == 413 + assert exc_info.value.detail == "Provenance archive too large" + + def _expected_acquisition_readiness_kpis(): return [ { From e1c9978781ba110ff51e1b6376d18ee1388d0736 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Tue, 1 Sep 2026 00:45:45 +0900 Subject: [PATCH 12/39] test: align bootstrap smoke with current schema --- backend/tests/test_bootstrap_db.py | 23 +++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/backend/tests/test_bootstrap_db.py b/backend/tests/test_bootstrap_db.py index 5af0540f0..d2d1cfae4 100644 --- a/backend/tests/test_bootstrap_db.py +++ b/backend/tests/test_bootstrap_db.py @@ -766,16 +766,23 @@ async def test_connector_signal_events_real_postgres_bootstrap_smoke(): text("DELETE FROM email_records WHERE user_id = :user_id"), {"user_id": smoke_user_id}, ) + await conn.execute( + text( + "CREATE TEMP TABLE emails (" + "user_id varchar, organization_id varchar, date timestamptz" + ") ON COMMIT DROP" + ) + ) email_result = await conn.execute( text(""" - INSERT INTO email_records ( - user_id, organization_id, message_id, sender, recipients, - subject, "date", body - ) - VALUES ( - :user_id, :organization_id, :message_id, :sender, - :recipients, :subject, now(), :body - ) + INSERT INTO email_records ( + user_id, organization_id, message_id, sender, recipients, + subject, "date", body, is_read + ) + VALUES ( + :user_id, :organization_id, :message_id, :sender, + :recipients, :subject, now(), :body, false + ) RETURNING id """), { From 7a19950bf9b9e0c50dac79666c661fd249d27127 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Tue, 1 Sep 2026 00:48:24 +0900 Subject: [PATCH 13/39] docs: record bounded provenance portability slice --- CHANGELOG.md | 2 + .../adr/0005-tenant-provenance-portability.md | 82 ++++++++++++++++ docs/adr/README.md | 1 + docs/planning/naruon-platform-plan.md | 18 ++-- docs/product-technical-gap-baseline.md | 4 +- .../2026-08-31-tenant-provenance-roundtrip.md | 80 ++++++++++++++++ ...8-31-tenant-provenance-roundtrip-design.md | 93 +++++++++++++++++++ 7 files changed, 270 insertions(+), 10 deletions(-) create mode 100644 docs/adr/0005-tenant-provenance-portability.md create mode 100644 docs/superpowers/plans/2026-08-31-tenant-provenance-roundtrip.md create mode 100644 docs/superpowers/specs/2026-08-31-tenant-provenance-roundtrip-design.md diff --git a/CHANGELOG.md b/CHANGELOG.md index 7ec84c36f..07b8f2566 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,6 @@ ## [Unreleased] +- 테넌트 portability의 첫 범위로, authoritative OIDC/server session이 현재 workspace의 프로젝트 근거 closure를 결정론적 BagIt 1.0/RO-Crate 1.3 ZIP으로 내보내고 다시 가져오는 `GET /api/data/provenance-bundle` 및 `POST /api/data/provenance-bundle/import`를 추가했습니다. 인용된 이메일, parser가 textual로 확인한 첨부, content/KG, project object/edge, correction만 stable UID로 보존하고 새 DB key로 한 transaction에서 복원합니다. 64 MiB archive/총 비압축 크기, 64 entries, entry당 32 MiB, 100:1 압축비를 검증하며 path/manifest/reference/중복/범위 오류는 mutation 전에 fail closed 합니다. HMAC fallback workspace claim은 거부하고 bundle scope가 target session authority를 덮어쓸 수 없습니다. 전체 tenant/mailbox, binary object, credential, provider/connector state, embedding, audit-history portability는 계속 미완료입니다(ADR-0005). +- 검증: 관련 Ruff, warnings-as-errors 결합 suite `151 passed`, 현재 schema에 맞춘 targeted PostgreSQL bootstrap smoke `1 passed`, 전체 backend suite `1971 passed, 3 skipped`. Bootstrap smoke fixture는 필수 `email_records.is_read`와 legacy-upgrade backfill이 전제하는 transaction-local `emails` index target을 명시해 단독 실행과 전체 실행의 순서 의존성을 제거했습니다. - 긴 이메일·첨부 본문을 의미 단위 청크로 임베딩한 뒤 기존 email/attachment 벡터 계약으로 평균화하고, 청크 요청·벡터 누적을 제한된 창으로 처리합니다. OpenAI `text-embedding-3-*`에는 저장 차원(`1536`)을 직접 요청하도록 보강했습니다. 합성 메일 fixture 5건(70청크)과 provider 요청 계약으로 1,536차원 벡터 경로를 검증했으며, 실행 시 선택한 임베딩 제공자에 본문·파싱된 첨부 텍스트를 전송할 수 있습니다. 회사 기밀 데이터는 fixture·commit·PR·log에 포함하지 않습니다. - EmailDetail 테스트가 지원하지 않는 스레드 병합/분리 버튼을 `textContent`뿐 아니라 `aria-label`과 `title` 접근 가능 이름으로도 검출하도록 바꿔, 아이콘 전용 버튼 회귀를 놓치지 않습니다. diff --git a/docs/adr/0005-tenant-provenance-portability.md b/docs/adr/0005-tenant-provenance-portability.md new file mode 100644 index 000000000..26748725b --- /dev/null +++ b/docs/adr/0005-tenant-provenance-portability.md @@ -0,0 +1,82 @@ +# ADR-0005: Package tenant provenance as a deterministic integrity envelope + +**Status:** Accepted for the workspace project-evidence closure +**Date:** 2026-08-31 +**Decision owner:** Naruon maintainers +**Scope:** Export and reimport of the email-derived provenance records cited by +one signed-session workspace. This decision does not claim full mailbox, +credential, connector, binary-object, or arbitrary multi-workspace portability. + +## Context + +GA-1 requires a buyer to leave, restore, or migrate without losing provenance. +Naruon persists email sources, attachment parse evidence, DOM nodes and +segments, structural edges, project objects and edges, and correction history, +but has no portable round-trip contract. Integer database keys cannot serve as +portable identity and credentials must never enter an export. + +`email_records` has owner and organization scope but no `workspace_id`. Exporting +all owner mail as a workspace bundle could therefore cross a same-organization +workspace boundary. The first safe closure begins with project graph rows scoped +to the exact workspace and includes only their cited source records. + +## Decision + +1. Package the exact project-evidence closure as a deterministic ZIP containing + a BagIt 1.0 envelope, `data/records.json`, and RO-Crate 1.3 JSON-LD metadata. +2. Use stable logical UIDs in the payload. Never serialize or restore sequential + database primary keys. Import resolves new keys in foreign-key order. +3. Canonical JSON uses UTF-8, no BOM, no insignificant whitespace, sorted object + keys, and pre-sorted set-like arrays. SHA-512 manifests cover exact bytes. +4. Reject unsafe or colliding ZIP paths, unlisted files, checksum mismatch, + unsupported profiles, dangling references, scope mismatch, duplicate logical + UIDs, and non-finite numbers before mutation. +5. Reimport is idempotent for target scope, bundle UID, and manifest digest. + Existing identical records are skipped; conflicting records fail closed. + Absence never means deletion and v1 has no tombstones. +6. Export plaintext evidence required for restoration, including email and + parser-confirmed textual attachment content. Exclude credentials, encrypted + secret rows, embeddings, raw binary documents, provider URLs or tokens, and + legacy unscoped audit details. Embeddings are regenerated after import. +7. Signed-session API routes use the current `user_id`, `organization_id`, and + `workspace_id`; bundle payload scope cannot override target authority. HMAC + fallback sessions are rejected because they do not prove workspace + membership; OIDC or server verification is required outside tests. +8. Full mailbox/customer-exit completion remains open until email ownership has + an explicit workspace dimension and binary object lifecycle is portable. + +## Alternatives rejected + +- **Database dump:** leaks implementation keys and secrets and couples versions. +- **RO-Crate alone:** describes data but is not a byte-integrity envelope. +- **BagIt alone:** verifies opaque bytes but does not describe provenance. +- **Data Transfer Project adapter first:** does not define an offline archive. +- **Export all owner emails:** unsafe while `email_records` lacks workspace scope. + +## Consequences + +- Buyers gain a verifiable first portability slice over evidence used for + project judgments and corrections. +- The importer validates the complete closure before one database transaction. +- Integrity hashes detect corruption but do not authenticate replacement; bundle + signing is a later profile revision. +- The product baseline keeps full tenant/mailbox, binary-object, credential, + provider/connector-state, and audit-history export/reimport open. + +## References (APA 7th) + +Kunze, J., Littman, J., Madden, L., Scancella, J., & Adams, C. (2018). *The +BagIt file packaging format (V1.0)* (RFC 8493). RFC Editor. +https://doi.org/10.17487/RFC8493 + +Lebo, T., Sahoo, S., & McGuinness, D. (Eds.). (2013). *PROV-O: The PROV +ontology*. World Wide Web Consortium. https://www.w3.org/TR/prov-o/ + +Rundgren, A., Jordan, B., & Erdtman, S. (2020). *JSON canonicalization scheme +(JCS)* (RFC 8785). RFC Editor. https://doi.org/10.17487/RFC8785 + +RO-Crate Community. (2026). *RO-Crate metadata specification 1.3*. +https://doi.org/10.5281/zenodo.20720080 + +The standards are cited and linked instead of copied; redistribution terms for +the complete referenced publications were not all established for this PR. diff --git a/docs/adr/README.md b/docs/adr/README.md index 4d461fff6..d0365c8af 100644 --- a/docs/adr/README.md +++ b/docs/adr/README.md @@ -13,6 +13,7 @@ govern implementation. | [ADR-0002](0002-fitted-topic-artifact-consumption.md) | Conditionally consume only a versioned fitted topic artifact through a fail-closed adapter | Proposed | Target `PLANNED`; runtime `BLOCKED-UPSTREAM` | | [ADR-0003](0003-separate-topic-measurement-from-agenda-generation.md) | Keep statistical measurement separate from agenda generation | Proposed | Target and future capability `PLANNED`; no implementation authorization | | [ADR-0004](0004-status-weighted-calendar-conflicts.md) | Evaluate CalDAV VEVENT overlaps by occupying status; cancelled does not occupy | Accepted | `ACCEPTED-NARUON-POLICY`; advisory evaluate API only | +| [ADR-0005](0005-tenant-provenance-portability.md) | Package workspace project evidence as a deterministic BagIt/RO-Crate provenance closure | Accepted | First portability slice; full tenant export remains incomplete | The complete topic-intelligence requirements, architecture, contract, UML, conceptual ERD, security, test, and operability graph is indexed at diff --git a/docs/planning/naruon-platform-plan.md b/docs/planning/naruon-platform-plan.md index 7e93ecbd2..45a88384f 100644 --- a/docs/planning/naruon-platform-plan.md +++ b/docs/planning/naruon-platform-plan.md @@ -119,7 +119,7 @@ KNOWLEDGE PLANE (dense semantic KG — the product) [PARTIAL] + typed, evidence-cited, confidence-weighted relations ``` -Scaffolding exists (`project_graph_objects` generic strings) but first-class Person/Event/Commitment entities are `[NEW]`; the semantic extractor is **test-only**, not wired into ingest. +Scaffolding exists (`project_graph_objects` generic strings) but first-class Person/Event/Commitment entities are `[NEW]`; email import invokes the semantic projection path when `PROJECT_GRAPH_EXTRACTION_ENABLED` is enabled (the default remains disabled). ### 4.3 Core entity model @@ -694,7 +694,7 @@ Hooks are **typed and ordered** (each point has a Pydantic input/output contract **Two tiers, evolving to three:** 1. **Content graph (LIVE)** — `content_nodes` + `content_segments` + `knowledge_graph_edges`: DOM decomposition of every email/attachment into structural nodes (`node_path`/`ordinal`) with **structural** edges (parent/child/sibling). Wired into ingest (alembic 0005–0007), built in one transaction with the email. -2. **Semantic/project graph (PARTIAL)** — `project_graph_objects` / `_edges` / `_corrections` (alembic 0009) with a full read/confirm/correct/traceability API. Today entities are generic `object_type` **strings** and the graph is **unpopulated in production** because `extract_project_semantics` / `persist_project_graph_projection` are **test-only** (verified: `callers_of(extract_project_semantics)` → not_found as a graph node; only test callers). Closing this is the single highest-leverage gap. +2. **Semantic/project graph (PARTIAL)** — `project_graph_objects` / `_edges` / `_corrections` (alembic 0009) with a full read/confirm/correct/traceability API. Today entities are generic `object_type` **strings**. Email import calls `persist_project_graph_projection` after content persistence when `PROJECT_GRAPH_EXTRACTION_ENABLED` is enabled; the default remains disabled, so deployment enablement and extraction-quality evidence remain open. 3. **Entity graph (TARGET)** — first-class typed entities and cross-entity relations, promoted from generic object strings. **Node & edge taxonomy.** Node types: `person`, `org`, `norm_group`, `project` (incl. Band), `thread`, `message`, `attachment`, `content_node`, `event`, `commitment`, `deliverable`, `requirement`, `wbs_item`, `erd_candidate`. Edge axes (density comes from many *simultaneous* relation axes): Social (`person—person`, `person—org`, `person—norm_group` **multi-membership**), Communication (`message—thread`, in_reply_to/references, sender/recipient), Temporal/event (`event—event` **enables/conflicts/unrelated**, resolved by density not asking; `event—commitment`), Commitment (status axis {confirmed|tentative|desired} + RSVP direction), Provenance (`object—content_segment` cited evidence, `object—extractor`, `correction—object`). Every semantic node/edge stores `confidence`, `extractor_name`, `extractor_version`, and cited `source_segment_uids` — auditable back to the exact DOM segment. @@ -734,7 +734,7 @@ Hooks are **typed and ordered** (each point has a Pydantic input/output contract Postgres is the system of record. **All new object names are 2+ word `snake_case`** (existing Camel/Pascal names left as-is). 38 SQLAlchemy tables today. -*Current core tables by domain:* **Identity/tenant** (`user_accounts`, `provider_accounts`, `organization_entities`, `organization_groups`, `scoped_role_assignments`, `workspace_entities`, `project_folders`, `tenant_configs`, `workspace_documents`); **Email live/legacy** (`email_records`, `email_attachments`); **Email normalized/in-progress** (`email_raws`, `email_messages`, `email_instances`, `email_threads`, `email_thread_edges` — not yet linked to `email_records`); **Content graph live** (`content_nodes`, `content_segments`, `knowledge_graph_edges`); **Project/semantic graph** (read-side live, extraction test-only: `project_graph_objects`, `project_graph_edges`, `project_graph_corrections`); **Tasks/workflow** (`ticket_tasks`, `workflow_definitions`, `agent_run_records`, `prompt_templates`); **Connectors/DAV/scheduling** (`caldav_accounts`, `webdav_accounts`, `calendar_writeback_sources`, `reply_trackers`, `sender_relationships`, `connector_signal_events`, `provider_writeback_retry_items`, `workspace_runner_configs`); **LLM/governance** (`llm_providers`, `audit_logs`, `security_audit_events`). +*Current core tables by domain:* **Identity/tenant** (`user_accounts`, `provider_accounts`, `organization_entities`, `organization_groups`, `scoped_role_assignments`, `workspace_entities`, `project_folders`, `tenant_configs`, `workspace_documents`); **Email live/legacy** (`email_records`, `email_attachments`); **Email normalized/in-progress** (`email_raws`, `email_messages`, `email_instances`, `email_threads`, `email_thread_edges` — not yet linked to `email_records`); **Content graph live** (`content_nodes`, `content_segments`, `knowledge_graph_edges`); **Project/semantic graph** (read-side live, feature-gated ingest projection: `project_graph_objects`, `project_graph_edges`, `project_graph_corrections`); **Tasks/workflow** (`ticket_tasks`, `workflow_definitions`, `agent_run_records`, `prompt_templates`); **Connectors/DAV/scheduling** (`caldav_accounts`, `webdav_accounts`, `calendar_writeback_sources`, `reply_trackers`, `sender_relationships`, `connector_signal_events`, `provider_writeback_retry_items`, `workspace_runner_configs`); **LLM/governance** (`llm_providers`, `audit_logs`, `security_audit_events`). *Proposed additions (TARGET, all 2+word snake_case):* @@ -789,11 +789,13 @@ This section is the honest reconciliation of vision against the code as it stand - Reply-SLA escalation → TicketTask; self-sent-note → TicketTask; sender-relationship ontology → next_action. - ABAC owner-scoping + `EncryptedString` secrets; Prometheus/OTEL flags + operational-signal API; audit + security-audit events. - Live screens (`/mail`, `/calendar`, `/tasks`, `/projects`, `/data`, `/search`, `/ai-hub`, `/prompt-studio`, `/tools`, `/security`, `/settings`) + the BFF proxy; `NetworkGraph.tsx` and `DecisionPointCard.tsx` components. +- Branch-local GA-1 portability slice: authoritative OIDC/server sessions can export or import a bounded deterministic BagIt/RO-Crate ZIP for the exact workspace's cited email, textual attachment, content/KG, project-graph, and correction closure. It is not full tenant portability and is not protected-branch evidence until merged. +- Exact branch verification: Ruff passed, the combined provenance service/API suite passed 151 tests, the repaired bootstrap PostgreSQL smoke passed in isolation, and the full backend suite passed 1,971 tests with 3 skipped. All Python runs promoted warnings to errors. ### 8.2 `[PARTIAL]` — exists but incomplete, **reconcile / wire** -- **Semantic project graph:** a full read/confirm/correct/traceability API over `project_graph_objects`/`_edges`/`_corrections` is live, but its extractor (`extract_project_semantics`/`persist_project_graph_projection`) is **test-only** — no ingest worker or API calls it, so the semantic graph is **unpopulated in production**. (Verified via `callers_of` → not_found + grep showing only test callers.) *Highest-leverage gap: the dense KG is built but empty.* +- **Semantic project graph:** a full read/confirm/correct/traceability API over `project_graph_objects`/`_edges`/`_corrections` is live. Email import calls `persist_project_graph_projection` after content-graph persistence when the feature flag is enabled; the default is disabled. Remaining gaps include deployment enablement, extraction-quality evidence, full portability, and complete multi-account ownership reconciliation. - **Multi-account model:** a normalized `email_messages/instances/raws/threads` model exists in parallel to the live `email_records` model and is **not reconciled**; live ingest still writes the legacy model. -- **Project-graph curation UI** (`/projects`) is read/confirm/correct only; extraction not wired. +- **Project-graph curation UI** (`/projects`) is read/confirm/correct only; ingest extraction is feature-gated and disabled by default. ### 8.3 `[NEW]` / `[PLANNED]` — net-new, **build** - First-class typed entities: **Person / NormGroup / Event / Commitment / Approval-outcome / DisclosureConsent** (today only generic `object_type` strings) and their typed, evidence-cited relations. @@ -813,9 +815,9 @@ This section is the honest reconciliation of vision against the code as it stand Ordered for maximum leverage against the vision while keeping each phase independently shippable. Every phase moves a concrete current-state gap toward the mission and preserves all five cross-cutting principles. -### Phase 0 — MVP: make the dense KG real (populate what's already built) -The single highest-leverage move: the semantic graph exists but is empty. -1. **Wire semantic extraction into ingest** — call `extract_project_semantics` / `persist_project_graph_projection` from the ingest worker so `project_graph_objects` populate in production (today they are test-only). +### Phase 0 — MVP: make the dense KG real (operate what's already built) +The single highest-leverage move is to enable and prove the existing semantic projection path on production-shaped data. +1. **Verify semantic extraction quality in ingest** — preserve the live `persist_project_graph_projection` call and add source-backed quality, correction, and portability evidence instead of describing production wiring as absent. 2. **Reconcile the multi-account email model** enough to have one source of truth for threads/messages feeding the graph. 3. **Extend hybrid search** to `content_segments` and populated `project_graph_objects` (search meaning, not just bodies). 4. Wire the existing `DecisionPointCard` to real synthesized thread cards (Epic 1 / UC-08) — first visible "judgment-ready structure." diff --git a/docs/product-technical-gap-baseline.md b/docs/product-technical-gap-baseline.md index 98bc17d2a..2e3cdb1f1 100644 --- a/docs/product-technical-gap-baseline.md +++ b/docs/product-technical-gap-baseline.md @@ -570,7 +570,7 @@ button, form, navigation, chart, or asynchronous data surface. |---|---|---|---|---| | Binary object lifecycle | large/deferred document bytes cannot remain an inline database strategy | S3-compatible implementation is Draft | #1076, #1364 | upload/read/recognize/retain/delete/backfill/orphan round trip with real integration | | Disaster recovery | a release is not enterprise-ready without restore evidence | HA evaluation exists; production WAL/PITR policy remains incomplete | #1428 | WAL archive/PITR, failover fencing, backup and clean restore rehearsal | -| Tenant export/reimport | customers need exit and migration without losing provenance | no single demonstrated full tenant round trip | #1428 | export → clean instance import preserving source, opaque IDs, history, evidence, policy | +| Tenant export/reimport | customers need exit and migration without losing provenance | Branch-local ADR-0005 slice provides authoritative-session GET/import APIs for a bounded deterministic workspace project-evidence closure: cited email, parser-admitted textual attachment, content/KG, project objects/edges, and corrections round-trip with fresh database keys. Ruff, 151 focused warnings-as-errors tests, the targeted PostgreSQL bootstrap smoke, and the full backend suite (`1971 passed, 3 skipped`) pass. The slice excludes uncited/mailbox-only mail, binary objects, credentials, provider/connector state, embeddings, and audit history. | #1428, `feat/tenant-provenance-roundtrip` | extend the bounded slice to full tenant/mailbox/binary/credential/provider/connector/audit portability; rehearse clean-instance export/import preserving source, opaque IDs, history, evidence, and policy | | Retention/legal hold/disposition | deletion and evidence preservation conflict unless modeled | partial security/key/retention work exists across repository history | #1428, #1364 | purpose-scoped retention, legal hold, verified disposition, object/DB reconciliation | | Attachment parser admission and unsupported formats | a file above 20 MiB can pass import transport but fail later at a hidden parser limit, while unsupported binaries are not searchable | Naruon import transport and generic deferred parser admission are bounded at 64 MiB; the NewsDOM `/parse` provider contract remains 20 MiB, so PDF bytes from 20–64 MiB are admitted and retained fail-closed but are not sent to NewsDOM; unsupported types remain metadata-only | #1427, #1469, #1353, #1419, NewsDOM #682/#707 | one documented bounded admission contract, provider-side PDF limit alignment or an explicit large-PDF fallback, parser/status evidence, deferred recognition, and object-backed retention before increasing the bound again | @@ -658,7 +658,7 @@ review approval, hidden manual database edits, or an unreleased sibling branch. 1. Implement typed temporal/multi-membership Person/Event/Commitment graph. 2. Complete status-weighted scheduling (#978/#988/#989/#990). 3. Complete evidence-based mail/document/media resolution (#1350). -4. Add tenant export/reimport and customer-exit evidence. +4. Extend the bounded workspace project-evidence bundle to full tenant export/reimport and customer-exit evidence. 5. Run the full buyer journey and failure/restore variants. ### Wave 3 — North-star platform diff --git a/docs/superpowers/plans/2026-08-31-tenant-provenance-roundtrip.md b/docs/superpowers/plans/2026-08-31-tenant-provenance-roundtrip.md new file mode 100644 index 000000000..87522f63d --- /dev/null +++ b/docs/superpowers/plans/2026-08-31-tenant-provenance-roundtrip.md @@ -0,0 +1,80 @@ +# Tenant Provenance Round-Trip Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Ship a signed deterministic export/reimport path preserving the current +workspace's email-derived project evidence and correction history. + +**Architecture:** Query the existing SQLAlchemy provenance graph by signed scope, +serialize stable identities into a bounded BagIt/RO-Crate ZIP, validate the full +closure, and restore it transactionally with fresh keys. The existing data router +exposes the operations. + +**Tech Stack:** Python 3.14, FastAPI, SQLAlchemy async, Python standard library, +PostgreSQL/pgvector, pytest. + +**Spec:** `docs/superpowers/specs/2026-08-31-tenant-provenance-roundtrip-design.md` + +## Global Constraints + +- Never inspect, copy, commit, or upload `tests/real_datasets`. +- Never serialize credentials, encrypted secrets, embeddings, integer keys, + binary documents, provider URLs or tokens, or legacy audit details. +- Source scope and target authority come from the signed session. +- Validate the bounded archive and reference graph before mutation. +- Add no dependency and no persistence table. +- Treat warning-class output as failure. + +--- + +### Task 1: Deterministic envelope and validator + +**Files:** Create `backend/services/tenant_provenance_bundle.py`; create +`backend/tests/test_tenant_provenance_bundle.py`. + +**Interfaces:** `build_provenance_archive(records) -> bytes` and +`parse_provenance_archive(archive_bytes) -> dict[str, object]`. + +- [ ] Write deterministic-byte and exact-entry tests; run and verify RED. +- [ ] Add RED tests for tampering, unsafe/colliding paths, bounds, missing or + extra entries, and duplicate JSON keys. +- [ ] Implement fixed ZIP metadata, canonical JSON, SHA-512 manifests, RO-Crate + metadata, bounded reads, and fail-closed validation with the standard library. +- [ ] Run the focused tests with `PYTHONWARNINGS=error` and require clean output. + +### Task 2: Scoped export and transactional reimport + +**Files:** Modify the service and focused test from Task 1. + +**Interfaces:** `TenantProvenanceScope`, `export_tenant_provenance`, +`import_tenant_provenance`, and `ImportReceipt`. + +- [ ] Seed Email through Correction in the PostgreSQL harness; write and run a + failing clean-target round-trip assertion over UIDs, hashes, and correction JSON. +- [ ] Add RED scope, dangling-reference rollback, conflict, text-admission, + secret/embedding absence, and idempotency tests. +- [ ] Implement scoped selects, stable records, pre-mutation validation, FK-order + restore maps, exact duplicate skips, and a single transaction boundary. +- [ ] Run the focused PostgreSQL suite with warnings promoted to errors. + +### Task 3: Signed data API + +**Files:** Modify `backend/api/data.py` and `backend/tests/test_data_api.py`. + +**Interfaces:** `GET /api/data/provenance-bundle` and +`POST /api/data/provenance-bundle/import`. + +- [ ] Add RED tests for authentication, response media/disposition, oversize + input, fixed safe errors, and target-scope rewrite. +- [ ] Add the two signed endpoints using bounded raw ZIP bytes and known errors. +- [ ] Run focused data API and bundle tests with warnings promoted to errors. + +### Task 4: Documentation truth and release evidence + +**Files:** Modify ADR index, product baseline, platform plan, and CHANGELOG. + +- [ ] Index ADR-0005, correct the stale test-only graph claim, and retain the full + portability gap after recording this bounded implemented slice. +- [ ] Run Ruff, focused suites, the full backend suite, and `git diff --check`. +- [ ] Confirm `git ls-files tests/real_datasets` is empty and inspect the complete + diff before committing and opening a PR. diff --git a/docs/superpowers/specs/2026-08-31-tenant-provenance-roundtrip-design.md b/docs/superpowers/specs/2026-08-31-tenant-provenance-roundtrip-design.md new file mode 100644 index 000000000..0779cf3f6 --- /dev/null +++ b/docs/superpowers/specs/2026-08-31-tenant-provenance-roundtrip-design.md @@ -0,0 +1,93 @@ +# Tenant Provenance Round-Trip Design + +**Status:** Accepted design for the first GA-1 portability slice. +**Decision:** [ADR-0005](../../adr/0005-tenant-provenance-portability.md) + +## Outcome + +A signed user can download a deterministic ZIP evidence bundle for the current +workspace's project-graph closure and import it into an authorized target scope. +A round trip preserves stable source, graph, citation, and correction identities +while allocating fresh database keys. + +This is not full tenant portability. Mail without workspace-grounded project +evidence, credentials, connector state, embeddings, and binary document objects +remain outside v1. + +## Contract + +```text +bagit.txt +bag-info.txt +manifest-sha512.txt +tagmanifest-sha512.txt +ro-crate-metadata.json +data/records.json +``` + +`data/records.json` carries the profile, schema version, opaque bundle UID, +source scope, export activity, and ordered arrays for emails, attachments, +content nodes, content segments, structural edges, project objects, project +edges, and corrections. Relationships use stable UIDs, never integer keys. +RO-Crate metadata describes the root Dataset, payload File, export Activity, and +software Agent using RO-Crate 1.3 and PROV terms. + +ZIP entries use fixed timestamps, mode, compression, and sorted names. SHA-512 +manifests cover exact bytes. Import validates archive structure, paths, byte +limits, manifests, JSON profile, scope closure, uniqueness, and every reference +before adding any ORM object. + +## Data flow + +```mermaid +flowchart LR + A[Signed AuthContext] --> B[Exact workspace project rows] + B --> C[Cited segment UIDs] + C --> D[Email provenance closure] + D --> E[Canonical records JSON] + E --> F[RO-Crate and BagIt manifests] + F --> G[Deterministic ZIP] + G --> H[Validate bytes and references] + H --> I[Remap stable UIDs to fresh DB keys] + I --> J[One transaction and scoped receipt] +``` + +## Scope and security invariants + +- Source reads match signed user and organization; project records additionally + match the signed workspace. +- The exported email closure is the set referenced by exact-workspace project + objects. Descendants are included only when owned by those emails. +- Import rewrites owner/workspace scope from the verified target session. +- Attachment content is accepted only for parser-confirmed textual media. +- Archive total, entry count, entry size, compression ratio, and JSON depth are + bounded (64 MiB archive/total uncompressed bytes, 64 entries, 32 MiB per + entry, and 100:1 compression ratio). No symlink, network fetch, external + context retrieval, or extraction. +- Errors use fixed codes and never echo record content or attacker paths. + +## Import behavior + +Import order is Email, Attachment, ContentNode, ContentSegment, +KnowledgeGraphEdge, ProjectGraphObject, ProjectGraphEdge, Correction. Each stage +maps stable identities to new keys. Dangling references or differing existing +identities abort the transaction. Exact duplicates are skipped. Absence never +deletes target data. + +## API + +- `GET /api/data/provenance-bundle` returns `application/zip`. +- `POST /api/data/provenance-bundle/import` accepts bounded raw ZIP bytes and + returns created/skipped counts plus a verified manifest digest. +- Both routes use the existing signed authentication dependency and reject the + HMAC fallback verifier because it is not authoritative workspace-membership + evidence. OIDC and server-verified contexts are accepted; dependency override + is test-only evidence. + +## Verification + +A real PostgreSQL test seeds the eight-model closure, exports it, imports into a +clean target scope, and compares stable UIDs, hashes, citations, and correction +history. Unit tests cover deterministic bytes, tampering, unsafe paths, dangling +references, cross-scope exclusion, secret/embedding absence, idempotent retry, +and rollback. API tests cover signed-session and target-scope enforcement. From 1809fc64496f5e6fd8fcf846ed653d4a04480b7e Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Tue, 1 Sep 2026 01:07:39 +0900 Subject: [PATCH 14/39] fix: skip legacy email index on fresh bootstrap --- .../versions/0001_initial_control_plane.py | 5 +- backend/scripts/bootstrap_db.py | 24 +++++---- backend/tests/test_alembic_migrations.py | 2 +- backend/tests/test_bootstrap_db.py | 51 +++++++++++++------ 4 files changed, 53 insertions(+), 29 deletions(-) diff --git a/backend/alembic/versions/0001_initial_control_plane.py b/backend/alembic/versions/0001_initial_control_plane.py index cc14ce39b..31b187f55 100644 --- a/backend/alembic/versions/0001_initial_control_plane.py +++ b/backend/alembic/versions/0001_initial_control_plane.py @@ -9,7 +9,7 @@ from sqlalchemy import text from db.models import Base -from scripts.bootstrap_db import schema_backfill_sql +from scripts.bootstrap_db import execute_schema_backfill revision = "0001_initial_control_plane" down_revision = None @@ -19,8 +19,7 @@ def upgrade() -> None: connection = op.get_bind() connection.execute(text("CREATE EXTENSION IF NOT EXISTS vector")) Base.metadata.create_all(connection) - for statement in schema_backfill_sql(): - connection.execute(statement) + execute_schema_backfill(connection) def downgrade() -> None: diff --git a/backend/scripts/bootstrap_db.py b/backend/scripts/bootstrap_db.py index 1047103e8..5f76237a6 100644 --- a/backend/scripts/bootstrap_db.py +++ b/backend/scripts/bootstrap_db.py @@ -2,13 +2,17 @@ import os from collections.abc import Sequence -from sqlalchemy import Executable, text +from sqlalchemy import Executable, inspect, text from sqlalchemy.engine import Connection from db.models import Base from db.session import engine INVALID_EMAIL_BACKFILL_OWNER_IDS = {None, "", "default"} +LEGACY_EMAILS_INDEX = text( + "CREATE INDEX IF NOT EXISTS ix_emails_owner_date " + "ON emails (user_id, organization_id, date)" +) def _static_bootstrap_sql(statement: str) -> Executable: @@ -93,8 +97,7 @@ def _get_add_columns_statements() -> list[Executable]: "ADD COLUMN IF NOT EXISTS organization_id varchar" ), _static_bootstrap_sql( - "ALTER TABLE prompt_templates " - "ADD COLUMN IF NOT EXISTS workspace_id varchar" + "ALTER TABLE prompt_templates ADD COLUMN IF NOT EXISTS workspace_id varchar" ), ] @@ -186,10 +189,7 @@ def _get_create_indexes_statements() -> list[Executable]: "CREATE INDEX IF NOT EXISTS ix_email_records_owner_date " "ON email_records (user_id, organization_id, date)" ), - text( - "CREATE INDEX IF NOT EXISTS ix_emails_owner_date " - "ON emails (user_id, organization_id, date)" - ), + LEGACY_EMAILS_INDEX, text( "CREATE INDEX IF NOT EXISTS ix_sender_relationships_owner_source " "ON sender_relationships " @@ -527,8 +527,14 @@ def schema_backfill_sql() -> list[Executable]: return statements -def _execute_statements(conn: Connection, statements: Sequence[Executable]) -> None: +def execute_schema_backfill( + conn: Connection, statements: Sequence[Executable] | None = None +) -> None: + statements = schema_backfill_sql() if statements is None else statements + legacy_emails_exists = inspect(conn).has_table("emails") for statement in statements: + if statement is LEGACY_EMAILS_INDEX and not legacy_emails_exists: + continue conn.execute(statement) @@ -536,7 +542,7 @@ async def bootstrap_db() -> None: async with engine.begin() as conn: await conn.execute(text("CREATE EXTENSION IF NOT EXISTS vector")) await conn.run_sync(Base.metadata.create_all) - await conn.run_sync(_execute_statements, schema_backfill_sql()) + await conn.run_sync(execute_schema_backfill) if __name__ == "__main__": diff --git a/backend/tests/test_alembic_migrations.py b/backend/tests/test_alembic_migrations.py index f8f3ffeae..4994f9a6d 100644 --- a/backend/tests/test_alembic_migrations.py +++ b/backend/tests/test_alembic_migrations.py @@ -32,7 +32,7 @@ def test_initial_alembic_revision_records_current_schema_path(): assert "down_revision = None" in revision_text assert "CREATE EXTENSION IF NOT EXISTS vector" in revision_text assert "Base.metadata.create_all" in revision_text - assert "schema_backfill_sql" in revision_text + assert "execute_schema_backfill" in revision_text def test_provider_writeback_retry_queue_has_incremental_revision(): diff --git a/backend/tests/test_bootstrap_db.py b/backend/tests/test_bootstrap_db.py index d2d1cfae4..47c8e88e0 100644 --- a/backend/tests/test_bootstrap_db.py +++ b/backend/tests/test_bootstrap_db.py @@ -6,7 +6,11 @@ from core.config import settings from db.models import Base -from scripts.bootstrap_db import schema_backfill_sql +from scripts.bootstrap_db import ( + LEGACY_EMAILS_INDEX, + execute_schema_backfill, + schema_backfill_sql, +) from db.models import ( AgentRunRecord, CalendarWritebackSource, @@ -30,8 +34,7 @@ def _get_schema_statements(monkeypatch): def _execute_schema_backfill(sync_conn): - for statement in schema_backfill_sql(): - sync_conn.execute(statement) + execute_schema_backfill(sync_conn) def test_schema_backfill_adds_email_columns(monkeypatch): @@ -204,13 +207,11 @@ def test_schema_backfill_adds_prompt_template_scope_columns_and_indexes(monkeypa for statement in statements ) assert any( - "alter table prompt_templates alter column prompt_uid set not null" - in statement + "alter table prompt_templates alter column prompt_uid set not null" in statement for statement in statements ) assert any( - "create unique index if not exists uq_prompt_templates_prompt_uid" - in statement + "create unique index if not exists uq_prompt_templates_prompt_uid" in statement for statement in statements ) assert any( @@ -298,8 +299,7 @@ def test_schema_backfill_creates_ai_hub_workflow_tables(monkeypatch): "ix_agent_run_records_scope_time" in statement for statement in statements ) assert any( - "ix_agent_run_records_workflow_uid" in statement - and "workflow_uid" in statement + "ix_agent_run_records_workflow_uid" in statement and "workflow_uid" in statement for statement in statements ) assert any( @@ -742,6 +742,32 @@ def test_schema_backfill_creates_connector_signal_events(): ) +@pytest.mark.asyncio +@pytest.mark.postgres +async def test_schema_backfill_creates_legacy_emails_index_when_table_exists(): + engine = create_async_engine(settings.DATABASE_URL) + try: + async with engine.begin() as conn: + await conn.execute( + text( + "CREATE TEMP TABLE emails (" + "user_id varchar, organization_id varchar, date timestamptz" + ") ON COMMIT DROP" + ) + ) + await conn.run_sync(execute_schema_backfill, [LEGACY_EMAILS_INDEX]) + result = await conn.execute( + text( + "SELECT indexname FROM pg_indexes " + "WHERE tablename = 'emails' " + "AND indexname = 'ix_emails_owner_date'" + ) + ) + assert result.scalar_one() == "ix_emails_owner_date" + finally: + await engine.dispose() + + @pytest.mark.asyncio @pytest.mark.postgres async def test_connector_signal_events_real_postgres_bootstrap_smoke(): @@ -766,13 +792,6 @@ async def test_connector_signal_events_real_postgres_bootstrap_smoke(): text("DELETE FROM email_records WHERE user_id = :user_id"), {"user_id": smoke_user_id}, ) - await conn.execute( - text( - "CREATE TEMP TABLE emails (" - "user_id varchar, organization_id varchar, date timestamptz" - ") ON COMMIT DROP" - ) - ) email_result = await conn.execute( text(""" INSERT INTO email_records ( From eaade6ea145a625fb66a9b3052ee23a450f10c08 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Tue, 1 Sep 2026 01:08:49 +0900 Subject: [PATCH 15/39] docs: record fresh bootstrap verification --- CHANGELOG.md | 2 +- docs/planning/naruon-platform-plan.md | 2 +- docs/product-technical-gap-baseline.md | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 07b8f2566..85657e07b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,6 @@ ## [Unreleased] - 테넌트 portability의 첫 범위로, authoritative OIDC/server session이 현재 workspace의 프로젝트 근거 closure를 결정론적 BagIt 1.0/RO-Crate 1.3 ZIP으로 내보내고 다시 가져오는 `GET /api/data/provenance-bundle` 및 `POST /api/data/provenance-bundle/import`를 추가했습니다. 인용된 이메일, parser가 textual로 확인한 첨부, content/KG, project object/edge, correction만 stable UID로 보존하고 새 DB key로 한 transaction에서 복원합니다. 64 MiB archive/총 비압축 크기, 64 entries, entry당 32 MiB, 100:1 압축비를 검증하며 path/manifest/reference/중복/범위 오류는 mutation 전에 fail closed 합니다. HMAC fallback workspace claim은 거부하고 bundle scope가 target session authority를 덮어쓸 수 없습니다. 전체 tenant/mailbox, binary object, credential, provider/connector state, embedding, audit-history portability는 계속 미완료입니다(ADR-0005). -- 검증: 관련 Ruff, warnings-as-errors 결합 suite `151 passed`, 현재 schema에 맞춘 targeted PostgreSQL bootstrap smoke `1 passed`, 전체 backend suite `1971 passed, 3 skipped`. Bootstrap smoke fixture는 필수 `email_records.is_read`와 legacy-upgrade backfill이 전제하는 transaction-local `emails` index target을 명시해 단독 실행과 전체 실행의 순서 의존성을 제거했습니다. +- 검증: 관련 Ruff, warnings-as-errors 결합 suite `151 passed`, fresh-schema 및 legacy-table PostgreSQL bootstrap 경로 `2 passed`, 전체 backend suite `1972 passed, 3 skipped`. Bootstrap 실행기는 legacy `emails` 테이블이 실제로 존재할 때만 그 전용 인덱스를 만들며, fresh schema에서는 해당 문장만 건너뜁니다. - 긴 이메일·첨부 본문을 의미 단위 청크로 임베딩한 뒤 기존 email/attachment 벡터 계약으로 평균화하고, 청크 요청·벡터 누적을 제한된 창으로 처리합니다. OpenAI `text-embedding-3-*`에는 저장 차원(`1536`)을 직접 요청하도록 보강했습니다. 합성 메일 fixture 5건(70청크)과 provider 요청 계약으로 1,536차원 벡터 경로를 검증했으며, 실행 시 선택한 임베딩 제공자에 본문·파싱된 첨부 텍스트를 전송할 수 있습니다. 회사 기밀 데이터는 fixture·commit·PR·log에 포함하지 않습니다. - EmailDetail 테스트가 지원하지 않는 스레드 병합/분리 버튼을 `textContent`뿐 아니라 `aria-label`과 `title` 접근 가능 이름으로도 검출하도록 바꿔, 아이콘 전용 버튼 회귀를 놓치지 않습니다. diff --git a/docs/planning/naruon-platform-plan.md b/docs/planning/naruon-platform-plan.md index 45a88384f..f2e127dc3 100644 --- a/docs/planning/naruon-platform-plan.md +++ b/docs/planning/naruon-platform-plan.md @@ -790,7 +790,7 @@ This section is the honest reconciliation of vision against the code as it stand - ABAC owner-scoping + `EncryptedString` secrets; Prometheus/OTEL flags + operational-signal API; audit + security-audit events. - Live screens (`/mail`, `/calendar`, `/tasks`, `/projects`, `/data`, `/search`, `/ai-hub`, `/prompt-studio`, `/tools`, `/security`, `/settings`) + the BFF proxy; `NetworkGraph.tsx` and `DecisionPointCard.tsx` components. - Branch-local GA-1 portability slice: authoritative OIDC/server sessions can export or import a bounded deterministic BagIt/RO-Crate ZIP for the exact workspace's cited email, textual attachment, content/KG, project-graph, and correction closure. It is not full tenant portability and is not protected-branch evidence until merged. -- Exact branch verification: Ruff passed, the combined provenance service/API suite passed 151 tests, the repaired bootstrap PostgreSQL smoke passed in isolation, and the full backend suite passed 1,971 tests with 3 skipped. All Python runs promoted warnings to errors. +- Exact branch verification: Ruff passed, the combined provenance service/API suite passed 151 tests, fresh-schema and legacy-table PostgreSQL bootstrap paths passed 2 tests, and the full backend suite passed 1,972 tests with 3 skipped. All Python runs promoted warnings to errors. ### 8.2 `[PARTIAL]` — exists but incomplete, **reconcile / wire** - **Semantic project graph:** a full read/confirm/correct/traceability API over `project_graph_objects`/`_edges`/`_corrections` is live. Email import calls `persist_project_graph_projection` after content-graph persistence when the feature flag is enabled; the default is disabled. Remaining gaps include deployment enablement, extraction-quality evidence, full portability, and complete multi-account ownership reconciliation. diff --git a/docs/product-technical-gap-baseline.md b/docs/product-technical-gap-baseline.md index 2e3cdb1f1..48d2faaa3 100644 --- a/docs/product-technical-gap-baseline.md +++ b/docs/product-technical-gap-baseline.md @@ -570,7 +570,7 @@ button, form, navigation, chart, or asynchronous data surface. |---|---|---|---|---| | Binary object lifecycle | large/deferred document bytes cannot remain an inline database strategy | S3-compatible implementation is Draft | #1076, #1364 | upload/read/recognize/retain/delete/backfill/orphan round trip with real integration | | Disaster recovery | a release is not enterprise-ready without restore evidence | HA evaluation exists; production WAL/PITR policy remains incomplete | #1428 | WAL archive/PITR, failover fencing, backup and clean restore rehearsal | -| Tenant export/reimport | customers need exit and migration without losing provenance | Branch-local ADR-0005 slice provides authoritative-session GET/import APIs for a bounded deterministic workspace project-evidence closure: cited email, parser-admitted textual attachment, content/KG, project objects/edges, and corrections round-trip with fresh database keys. Ruff, 151 focused warnings-as-errors tests, the targeted PostgreSQL bootstrap smoke, and the full backend suite (`1971 passed, 3 skipped`) pass. The slice excludes uncited/mailbox-only mail, binary objects, credentials, provider/connector state, embeddings, and audit history. | #1428, `feat/tenant-provenance-roundtrip` | extend the bounded slice to full tenant/mailbox/binary/credential/provider/connector/audit portability; rehearse clean-instance export/import preserving source, opaque IDs, history, evidence, and policy | +| Tenant export/reimport | customers need exit and migration without losing provenance | Branch-local ADR-0005 slice provides authoritative-session GET/import APIs for a bounded deterministic workspace project-evidence closure: cited email, parser-admitted textual attachment, content/KG, project objects/edges, and corrections round-trip with fresh database keys. Ruff, 151 focused warnings-as-errors tests, 2 fresh/legacy bootstrap tests, and the full backend suite (`1972 passed, 3 skipped`) pass. The slice excludes uncited/mailbox-only mail, binary objects, credentials, provider/connector state, embeddings, and audit history. | #1428, `feat/tenant-provenance-roundtrip` | extend the bounded slice to full tenant/mailbox/binary/credential/provider/connector/audit portability; rehearse clean-instance export/import preserving source, opaque IDs, history, evidence, and policy | | Retention/legal hold/disposition | deletion and evidence preservation conflict unless modeled | partial security/key/retention work exists across repository history | #1428, #1364 | purpose-scoped retention, legal hold, verified disposition, object/DB reconciliation | | Attachment parser admission and unsupported formats | a file above 20 MiB can pass import transport but fail later at a hidden parser limit, while unsupported binaries are not searchable | Naruon import transport and generic deferred parser admission are bounded at 64 MiB; the NewsDOM `/parse` provider contract remains 20 MiB, so PDF bytes from 20–64 MiB are admitted and retained fail-closed but are not sent to NewsDOM; unsupported types remain metadata-only | #1427, #1469, #1353, #1419, NewsDOM #682/#707 | one documented bounded admission contract, provider-side PDF limit alignment or an explicit large-PDF fallback, parser/status evidence, deferred recognition, and object-backed retention before increasing the bound again | From 63ec9f3b975e868016ad5fa9fcd59c2c7c9eddeb Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Tue, 1 Sep 2026 02:05:40 +0900 Subject: [PATCH 16/39] fix(provenance): scope portable identity mappings --- .../0018_provenance_identity_mappings.py | 55 +++ backend/db/models.py | 39 ++ backend/services/tenant_provenance_bundle.py | 362 +++++++++++++++++- backend/tests/test_data_api.py | 2 +- .../tests/test_tenant_provenance_bundle.py | 281 +++++++++++++- .../adr/0005-tenant-provenance-portability.md | 5 + ...8-31-tenant-provenance-roundtrip-design.md | 8 + 7 files changed, 741 insertions(+), 11 deletions(-) create mode 100644 backend/alembic/versions/0018_provenance_identity_mappings.py diff --git a/backend/alembic/versions/0018_provenance_identity_mappings.py b/backend/alembic/versions/0018_provenance_identity_mappings.py new file mode 100644 index 000000000..2bf731ddd --- /dev/null +++ b/backend/alembic/versions/0018_provenance_identity_mappings.py @@ -0,0 +1,55 @@ +"""Add scoped portable-to-database provenance identity mappings.""" + +from alembic import op +import sqlalchemy as sa + + +revision = "0018_provenance_identity" +down_revision = "0017_merge_newsdom_carddav_heads" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.create_table( + "provenance_identity_mappings", + sa.Column("provenance_identity_id", sa.Integer(), primary_key=True), + sa.Column("target_user_id", sa.String(), nullable=False), + sa.Column("target_organization_id", sa.String(), nullable=False), + sa.Column("target_workspace_id", sa.String(), nullable=False), + sa.Column("source_user_uid", sa.String(length=64), nullable=False), + sa.Column("source_organization_uid", sa.String(), nullable=False), + sa.Column("source_workspace_uid", sa.String(), nullable=False), + sa.Column("entity_kind", sa.String(length=64), nullable=False), + sa.Column("portable_uid", sa.String(length=256), nullable=False), + sa.Column("target_database_uid", sa.String(length=96), nullable=False), + sa.UniqueConstraint( + "target_user_id", + "target_organization_id", + "target_workspace_id", + "source_user_uid", + "source_organization_uid", + "source_workspace_uid", + "entity_kind", + "portable_uid", + name="uq_provenance_identity_source_target", + ), + sa.UniqueConstraint( + "entity_kind", + "target_database_uid", + name="uq_provenance_identity_target_uid", + ), + ) + op.create_index( + "ix_provenance_identity_target_scope", + "provenance_identity_mappings", + ["target_user_id", "target_organization_id", "target_workspace_id"], + ) + + +def downgrade() -> None: + op.drop_index( + "ix_provenance_identity_target_scope", + table_name="provenance_identity_mappings", + ) + op.drop_table("provenance_identity_mappings") diff --git a/backend/db/models.py b/backend/db/models.py index 98e17eef2..714588836 100644 --- a/backend/db/models.py +++ b/backend/db/models.py @@ -1293,6 +1293,45 @@ class ProjectGraphCorrectionRecord(Base): ) +class ProvenanceIdentityMapping(Base): + __tablename__ = "provenance_identity_mappings" + __table_args__ = ( + UniqueConstraint( + "target_user_id", + "target_organization_id", + "target_workspace_id", + "source_user_uid", + "source_organization_uid", + "source_workspace_uid", + "entity_kind", + "portable_uid", + name="uq_provenance_identity_source_target", + ), + UniqueConstraint( + "entity_kind", + "target_database_uid", + name="uq_provenance_identity_target_uid", + ), + Index( + "ix_provenance_identity_target_scope", + "target_user_id", + "target_organization_id", + "target_workspace_id", + ), + ) + + provenance_identity_id: Mapped[int] = mapped_column(primary_key=True) + target_user_id: Mapped[str] = mapped_column(String, nullable=False) + target_organization_id: Mapped[str] = mapped_column(String, nullable=False) + target_workspace_id: Mapped[str] = mapped_column(String, nullable=False) + source_user_uid: Mapped[str] = mapped_column(String(64), nullable=False) + source_organization_uid: Mapped[str] = mapped_column(String, nullable=False) + source_workspace_uid: Mapped[str] = mapped_column(String, nullable=False) + entity_kind: Mapped[str] = mapped_column(String(64), nullable=False) + portable_uid: Mapped[str] = mapped_column(String(256), nullable=False) + target_database_uid: Mapped[str] = mapped_column(String(96), nullable=False) + + class TenantConfig(Base): __tablename__ = "tenant_configs" diff --git a/backend/services/tenant_provenance_bundle.py b/backend/services/tenant_provenance_bundle.py index 3d1a4cfb6..1a7ae6394 100644 --- a/backend/services/tenant_provenance_bundle.py +++ b/backend/services/tenant_provenance_bundle.py @@ -2,6 +2,7 @@ from __future__ import annotations +import copy import hashlib import io import json @@ -28,6 +29,7 @@ ProjectGraphCorrectionRecord, ProjectGraphEdgeRecord, ProjectGraphObjectRecord, + ProvenanceIdentityMapping, ) @@ -63,6 +65,30 @@ "project_edges", "corrections", ) +_REMAPPED_COLLECTIONS = ( + "content_nodes", + "content_segments", + "structural_edges", + "project_objects", + "project_edges", + "corrections", +) +_UID_KEYS = { + "content_nodes": "content_node_uid", + "content_segments": "content_segment_uid", + "structural_edges": "edge_uid", + "project_objects": "object_uid", + "project_edges": "edge_uid", + "corrections": "correction_uid", +} +_UID_PREFIXES = { + "content_nodes": "tpn-", + "content_segments": "tps-", + "structural_edges": "tpk-", + "project_objects": "tpo-", + "project_edges": "tpe-", + "corrections": "tpc-", +} _TEXTUAL_PARSER_KEYS = frozenset( {"plain_text", "html", "markdown", "json", "csv", "xml", "calendar", "pdf"} ) @@ -339,6 +365,33 @@ def _portable_metadata( return value +def _source_user_uid(user_id: str) -> str: + return hashlib.sha256(user_id.encode("utf-8")).hexdigest() + + +def _target_database_uid( + scope: TenantProvenanceScope, + source_scope: Mapping[str, str], + collection: str, + portable_uid: str, +) -> str: + digest = hashlib.sha256( + _canonical_json( + { + "collection": collection, + "portable_uid": portable_uid, + "source_scope": dict(source_scope), + "target_scope": { + "organization_uid": scope.organization_id, + "user_uid": scope.user_id, + "workspace_uid": scope.workspace_id, + }, + } + ) + ).hexdigest() + return f"{_UID_PREFIXES[collection]}{digest[:60]}" + + def _canonical_json(value: object) -> bytes: _validate_json_value(value) try: @@ -353,6 +406,57 @@ def _canonical_json(value: object) -> bytes: raise ProvenanceArchiveError("Invalid provenance archive") from exc +def _translate_identity_records( + records: Mapping[str, object], maps: Mapping[str, Mapping[str, str]] +) -> dict[str, object]: + translated = copy.deepcopy(records) + + def replace(record: dict[str, object], field: str, collection: str) -> None: + value = record[field] + if value is not None: + record[field] = maps[collection].get(value, value) + + def replace_list(record: dict[str, object], field: str, collection: str) -> None: + record[field] = sorted( + maps[collection].get(value, value) for value in record[field] + ) + + for record in translated["content_nodes"]: + replace(record, "content_node_uid", "content_nodes") + replace(record, "parent_node_uid", "content_nodes") + for record in translated["content_segments"]: + replace(record, "content_segment_uid", "content_segments") + replace(record, "content_node_uid", "content_nodes") + for record in translated["structural_edges"]: + replace(record, "edge_uid", "structural_edges") + for field in ("source_node_uid", "target_node_uid"): + replace(record, field, "content_nodes") + for field in ("source_segment_uid", "target_segment_uid"): + replace(record, field, "content_segments") + for record in translated["project_objects"]: + replace(record, "object_uid", "project_objects") + replace(record, "primary_content_segment_uid", "content_segments") + replace_list(record, "source_segment_uids", "content_segments") + for record in translated["project_edges"]: + replace(record, "edge_uid", "project_edges") + for field in ( + "source_uid", + "target_uid", + "source_object_uid", + "target_object_uid", + ): + replace(record, field, "project_objects") + replace(record, "primary_content_segment_uid", "content_segments") + replace_list(record, "source_segment_uids", "content_segments") + for record in translated["corrections"]: + replace(record, "correction_uid", "corrections") + replace(record, "object_uid", "project_objects") + replace_list(record, "source_segment_uids", "content_segments") + for collection in _REMAPPED_COLLECTIONS: + translated[collection].sort(key=lambda record: record[_UID_KEYS[collection]]) + return translated + + def _records_bundle_uid(records: Mapping[str, object]) -> str: schema_version = records.get("schema_version") if ( @@ -737,7 +841,7 @@ def _parse_datetime(value: object) -> datetime: def _confidence(value: float) -> str: - if not math.isfinite(value): + if not math.isfinite(value) or not 0.0 <= value <= 1.0: _fail() return repr(value) @@ -749,7 +853,7 @@ def _parse_confidence(value: object) -> float: parsed = float(value) except ValueError as exc: raise ProvenanceArchiveError("Invalid provenance archive") from exc - if not math.isfinite(parsed): + if not math.isfinite(parsed) or not 0.0 <= parsed <= 1.0: _fail() return parsed @@ -1097,6 +1201,122 @@ async def descendants(model: Any, order_column: Any) -> list[Any]: structural_edges = await descendants( KnowledgeGraphEdgeRecord, KnowledgeGraphEdgeRecord.edge_uid ) + candidate_uids = { + *(row.content_node_uid for row in nodes), + *(row.content_segment_uid for row in segments), + *(row.edge_uid for row in structural_edges), + *(row.object_uid for row in project_objects), + *(row.edge_uid for row in project_edges), + *(row.correction_uid for row in corrections), + } + candidate_identity_rows = list( + ( + await session.scalars( + select(ProvenanceIdentityMapping).where( + ProvenanceIdentityMapping.target_database_uid.in_(candidate_uids) + ) + ) + ).all() + ) + identity_rows = [ + row + for row in candidate_identity_rows + if row.target_user_id == scope.user_id + and row.target_organization_id == scope.organization_id + and row.target_workspace_id == scope.workspace_id + ] + mapped_uids = { + collection: { + row.target_database_uid + for row in candidate_identity_rows + if row.entity_kind == collection + } + for collection in _REMAPPED_COLLECTIONS + } + target_mapped_uids = { + collection: { + row.target_database_uid + for row in identity_rows + if row.entity_kind == collection + } + for collection in _REMAPPED_COLLECTIONS + } + if identity_rows: + nodes = [ + row + for row in nodes + if row.content_node_uid in target_mapped_uids["content_nodes"] + ] + segments = [ + row + for row in segments + if row.content_segment_uid in target_mapped_uids["content_segments"] + ] + structural_edges = [ + row + for row in structural_edges + if row.edge_uid in target_mapped_uids["structural_edges"] + ] + else: + nodes = [ + row + for row in nodes + if row.content_node_uid not in mapped_uids["content_nodes"] + ] + segments = [ + row + for row in segments + if row.content_segment_uid not in mapped_uids["content_segments"] + ] + structural_edges = [ + row + for row in structural_edges + if row.edge_uid not in mapped_uids["structural_edges"] + ] + database_uids = { + "content_nodes": {row.content_node_uid for row in nodes}, + "content_segments": {row.content_segment_uid for row in segments}, + "structural_edges": {row.edge_uid for row in structural_edges}, + "project_objects": {row.object_uid for row in project_objects}, + "project_edges": {row.edge_uid for row in project_edges}, + "corrections": {row.correction_uid for row in corrections}, + } + relevant_identity_rows = [ + row + for row in identity_rows + if row.entity_kind in database_uids + and row.target_database_uid in database_uids[row.entity_kind] + ] + reverse_maps = {collection: {} for collection in _REMAPPED_COLLECTIONS} + source_scopes = { + ( + row.source_user_uid, + row.source_organization_uid, + row.source_workspace_uid, + ) + for row in relevant_identity_rows + } + if relevant_identity_rows: + if len(source_scopes) != 1: + _fail() + for collection in _REMAPPED_COLLECTIONS: + collection_rows = [ + row for row in relevant_identity_rows if row.entity_kind == collection + ] + if {row.target_database_uid for row in collection_rows} != database_uids[ + collection + ]: + _fail() + reverse_maps[collection] = { + row.target_database_uid: row.portable_uid for row in collection_rows + } + source_user_uid, source_organization_uid, source_workspace_uid = next( + iter(source_scopes) + ) + else: + source_user_uid = _source_user_uid(scope.user_id) + source_organization_uid = scope.organization_id or "unscoped" + source_workspace_uid = scope.workspace_id attachment_records, attachment_uids = _attachment_records(attachments, email_uids) attachment_references = { f"attachment-{attachment_id}": f"attachment:{attachment_uid}" @@ -1112,8 +1332,9 @@ async def descendants(model: Any, order_column: Any) -> list[Any]: } payload = { "source_scope": { - "organization_uid": scope.organization_id or "unscoped", - "workspace_uid": scope.workspace_id, + "user_uid": source_user_uid, + "organization_uid": source_organization_uid, + "workspace_uid": source_workspace_uid, }, "emails": [_email_record(email) for email in emails], "attachments": attachment_records, @@ -1149,6 +1370,8 @@ async def descendants(model: Any, order_column: Any) -> list[Any]: for correction in corrections ], } + if relevant_identity_rows: + payload = _translate_identity_records(payload, reverse_maps) content_digest = hashlib.sha256(_canonical_json(payload)).hexdigest() records = { "profile": "naruon-tenant-provenance/v1", @@ -1435,10 +1658,12 @@ def _validate_record_graph(records: Mapping[str, object]) -> None: _records_bundle_uid(records) source_scope = records.get("source_scope") if not isinstance(source_scope, dict) or set(source_scope) != { + "user_uid", "organization_uid", "workspace_uid", }: _fail() + _safe_identifier(source_scope.get("user_uid")) _safe_identifier(source_scope.get("organization_uid")) _safe_identifier(source_scope.get("workspace_uid")) activity = records.get("export_activity") @@ -1629,6 +1854,126 @@ async def _matching_models( return {getattr(row, key_attribute): row for row in rows} +async def _prepare_identity_import( + session: AsyncSession, + scope: TenantProvenanceScope, + records: Mapping[str, object], +) -> tuple[dict[str, object], list[ProvenanceIdentityMapping]]: + source_scope = records["source_scope"] + same_scope = ( + source_scope["user_uid"] == _source_user_uid(scope.user_id) + and source_scope["organization_uid"] == (scope.organization_id or "unscoped") + and source_scope["workspace_uid"] == scope.workspace_id + ) + if same_scope: + return copy.deepcopy(records), [] + + scoped_collisions: list[Any] = [] + for model, column, collection in ( + ( + ProjectGraphObjectRecord, + ProjectGraphObjectRecord.object_uid, + "project_objects", + ), + (ProjectGraphEdgeRecord, ProjectGraphEdgeRecord.edge_uid, "project_edges"), + ( + ProjectGraphCorrectionRecord, + ProjectGraphCorrectionRecord.correction_uid, + "corrections", + ), + ): + portable_uids = { + record[_UID_KEYS[collection]] for record in records[collection] + } + if portable_uids: + scoped_collisions.extend( + ( + await session.scalars( + select(model).where(column.in_(portable_uids)) + ) + ).all() + ) + if not scoped_collisions: + return copy.deepcopy(records), [] + if any( + row.user_id == scope.user_id + and row.organization_id == scope.organization_id + and row.workspace_id == scope.workspace_id + for row in scoped_collisions + ): + return copy.deepcopy(records), [] + if any( + _source_user_uid(row.user_id) != source_scope["user_uid"] + or (row.organization_id or "unscoped") != source_scope["organization_uid"] + or row.workspace_id != source_scope["workspace_uid"] + for row in scoped_collisions + ): + _fail() + + forward_maps = {collection: {} for collection in _REMAPPED_COLLECTIONS} + for collection in _REMAPPED_COLLECTIONS: + for record in records[collection]: + portable_uid = record[_UID_KEYS[collection]] + forward_maps[collection][portable_uid] = _target_database_uid( + scope, source_scope, collection, portable_uid + ) + target_uids = { + database_uid + for collection_map in forward_maps.values() + for database_uid in collection_map.values() + } + existing_rows = list( + ( + await session.scalars( + select(ProvenanceIdentityMapping).where( + ProvenanceIdentityMapping.target_database_uid.in_(target_uids) + ) + ) + ).all() + ) + existing = {(row.entity_kind, row.portable_uid): row for row in existing_rows} + new_rows: list[ProvenanceIdentityMapping] = [] + for collection, collection_map in forward_maps.items(): + for portable_uid, database_uid in collection_map.items(): + row = existing.get((collection, portable_uid)) + expected = ( + scope.user_id, + scope.organization_id, + scope.workspace_id, + source_scope["user_uid"], + source_scope["organization_uid"], + source_scope["workspace_uid"], + database_uid, + ) + if row is not None: + actual = ( + row.target_user_id, + row.target_organization_id, + row.target_workspace_id, + row.source_user_uid, + row.source_organization_uid, + row.source_workspace_uid, + row.target_database_uid, + ) + if actual != expected: + _fail() + continue + new_rows.append( + ProvenanceIdentityMapping( + target_user_id=scope.user_id, + target_organization_id=scope.organization_id, + target_workspace_id=scope.workspace_id, + source_user_uid=source_scope["user_uid"], + source_organization_uid=source_scope["organization_uid"], + source_workspace_uid=source_scope["workspace_uid"], + entity_kind=collection, + portable_uid=portable_uid, + target_database_uid=database_uid, + ) + ) + return _translate_identity_records(records, forward_maps), new_rows + + async def _preflight_existing( session: AsyncSession, scope: TenantProvenanceScope, @@ -2046,11 +2391,16 @@ async def import_tenant_provenance( created = {collection: 0 for collection in _COLLECTIONS} try: async with session.begin(): - models = await _preflight_existing(session, scope, records) + database_records, identity_rows = await _prepare_identity_import( + session, scope, records + ) + models = await _preflight_existing(session, scope, database_records) skipped = { collection: len(models[collection]) for collection in _COLLECTIONS } - await _insert_records(session, scope, records, models, created) + await _insert_records(session, scope, database_records, models, created) + session.add_all(identity_rows) + await session.flush() except ProvenanceArchiveError: raise except (DataError, IntegrityError, StatementError, TypeError, ValueError) as exc: diff --git a/backend/tests/test_data_api.py b/backend/tests/test_data_api.py index 2c6d7a587..fa3ddcf9c 100644 --- a/backend/tests/test_data_api.py +++ b/backend/tests/test_data_api.py @@ -716,7 +716,7 @@ async def test_provenance_archive_accepts_absent_length_boundary_chunks(monkeypa @pytest.mark.asyncio -async def test_provenance_archive_rejects_body_larger_than_declared_length(monkeypatch): +async def test_provenance_archive_rejects_streamed_body_over_max_bytes(monkeypatch): monkeypatch.setattr(data_api, "_PROVENANCE_ARCHIVE_MAX_BYTES", 4) with pytest.raises(HTTPException) as exc_info: diff --git a/backend/tests/test_tenant_provenance_bundle.py b/backend/tests/test_tenant_provenance_bundle.py index 9de712d23..a47b9473c 100644 --- a/backend/tests/test_tenant_provenance_bundle.py +++ b/backend/tests/test_tenant_provenance_bundle.py @@ -26,6 +26,7 @@ ProjectGraphCorrectionRecord, ProjectGraphEdgeRecord, ProjectGraphObjectRecord, + ProvenanceIdentityMapping, ) from services.project_graph.repository import ProjectGraphRepository @@ -50,7 +51,11 @@ "profile": "naruon-tenant-provenance/v1", "schema_version": 1, "bundle_uid": "bundle-01HZZ", - "source_scope": {"organization_uid": "org-01", "workspace_uid": "ws-01"}, + "source_scope": { + "user_uid": "user-source-01", + "organization_uid": "org-01", + "workspace_uid": "ws-01", + }, "export_activity": { "activity_uid": "activity-01", "date_published": "1980-01-01T00:00:00Z", @@ -998,6 +1003,275 @@ async def test_postgres_round_trip_preserves_stable_evidence_with_fresh_keys( } +@pytest.mark.asyncio +@pytest.mark.postgres +async def test_same_database_cross_workspace_import_keeps_portable_identity( + provenance_sessionmaker, +): + token = uuid.uuid4().hex[:12] + source_scope = _scope(f"source-{token}") + target_scope = TenantProvenanceScope( + user_id=source_scope.user_id, + organization_id=source_scope.organization_id, + workspace_id=f"target-workspace-{token}", + ) + async with provenance_sessionmaker() as session: + source = await _seed_provenance_closure( + session, scope=source_scope, token=token + ) + async with provenance_sessionmaker() as session: + source_archive = await export_tenant_provenance(session, source_scope) + source_records = parse_provenance_archive(source_archive) + + async with provenance_sessionmaker() as session: + first = await import_tenant_provenance(session, target_scope, source_archive) + async with provenance_sessionmaker() as session: + second = await import_tenant_provenance(session, target_scope, source_archive) + async with provenance_sessionmaker() as session: + source_after = parse_provenance_archive( + await export_tenant_provenance(session, source_scope) + ) + target_after = parse_provenance_archive( + await export_tenant_provenance(session, target_scope) + ) + source_objects = list( + ( + await session.scalars( + select(ProjectGraphObjectRecord).where( + ProjectGraphObjectRecord.workspace_id + == source_scope.workspace_id + ) + ) + ).all() + ) + target_objects = list( + ( + await session.scalars( + select(ProjectGraphObjectRecord).where( + ProjectGraphObjectRecord.workspace_id + == target_scope.workspace_id + ) + ) + ).all() + ) + target_edge = await session.scalar( + select(ProjectGraphEdgeRecord).where( + ProjectGraphEdgeRecord.workspace_id == target_scope.workspace_id + ) + ) + target_correction = await session.scalar( + select(ProjectGraphCorrectionRecord).where( + ProjectGraphCorrectionRecord.workspace_id == target_scope.workspace_id + ) + ) + identity_mappings = list( + ( + await session.scalars( + select(ProvenanceIdentityMapping).where( + ProvenanceIdentityMapping.target_workspace_id + == target_scope.workspace_id + ) + ) + ).all() + ) + + assert first.created == { + "emails": 0, + "attachments": 0, + "content_nodes": 1, + "content_segments": 1, + "structural_edges": 1, + "project_objects": 2, + "project_edges": 1, + "corrections": 1, + } + assert sum(second.created.values()) == 0 + assert second.skipped == { + collection: len(source_records[collection]) + for collection in ( + "emails", + "attachments", + "content_nodes", + "content_segments", + "structural_edges", + "project_objects", + "project_edges", + "corrections", + ) + } + for collection in ( + "emails", + "attachments", + "content_nodes", + "content_segments", + "structural_edges", + "project_objects", + "project_edges", + "corrections", + ): + assert source_after[collection] == source_records[collection] + assert target_after[collection] == source_records[collection] + assert {record.object_uid for record in source_objects} == set( + source["object_uids"] + ) + assert {record.object_uid for record in source_objects}.isdisjoint( + {record.object_uid for record in target_objects} + ) + target_object_uids = {record.object_uid for record in target_objects} + assert target_edge.source_uid in target_object_uids + assert target_edge.target_uid in target_object_uids + assert target_edge.source_object_id in { + record.project_graph_object_id for record in target_objects + } + assert target_edge.target_object_id in { + record.project_graph_object_id for record in target_objects + } + assert target_correction.project_graph_object_id in { + record.project_graph_object_id for record in target_objects + } + assert all( + record.primary_content_segment_id == target_edge.primary_content_segment_id + for record in target_objects + ) + assert len(identity_mappings) == 7 + assert {row.entity_kind for row in identity_mappings} == { + "content_nodes", + "content_segments", + "structural_edges", + "project_objects", + "project_edges", + "corrections", + } + assert all( + record.source_segment_uids == [target_edge.source_segment_uids[0]] + for record in target_objects + ) + assert target_correction.source_segment_uids == target_edge.source_segment_uids + + +@pytest.mark.parametrize( + ("model", "value"), + ( + (ProjectGraphObjectRecord, -0.01), + (ProjectGraphObjectRecord, 1.01), + (ProjectGraphEdgeRecord, -0.01), + (ProjectGraphEdgeRecord, 1.01), + ), +) +@pytest.mark.asyncio +@pytest.mark.postgres +async def test_export_rejects_confidence_outside_unit_interval( + provenance_sessionmaker, + model, + value, +): + token = uuid.uuid4().hex[:12] + scope = _scope(f"confidence-export-{token}") + async with provenance_sessionmaker() as session: + await _seed_provenance_closure(session, scope=scope, token=token) + record = await session.scalar( + select(model).where(model.workspace_id == scope.workspace_id) + ) + record.confidence = value + await session.commit() + + async with provenance_sessionmaker() as session: + with pytest.raises(ProvenanceArchiveError): + await export_tenant_provenance(session, scope) + + +@pytest.mark.parametrize( + ("collection", "value"), + ( + ("project_objects", "-0.01"), + ("project_objects", "1.01"), + ("project_edges", "-0.01"), + ("project_edges", "1.01"), + ), +) +@pytest.mark.asyncio +@pytest.mark.postgres +async def test_import_rejects_confidence_outside_unit_interval_before_flush( + provenance_sessionmaker, + monkeypatch, + collection, + value, +): + token = uuid.uuid4().hex[:12] + source_scope = _scope(f"confidence-source-{token}") + target_scope = _scope(f"confidence-target-{token}") + async with provenance_sessionmaker() as session: + await _seed_provenance_closure(session, scope=source_scope, token=token) + async with provenance_sessionmaker() as session: + records = parse_provenance_archive( + await export_tenant_provenance(session, source_scope) + ) + records[collection][0]["confidence"] = value + invalid_archive = build_provenance_archive(records) + async with provenance_sessionmaker() as session: + await _delete_exported_closure(session, records) + + async with provenance_sessionmaker() as session: + flush_count = 0 + original_flush = session.flush + + async def counting_flush(*args, **kwargs): + nonlocal flush_count + flush_count += 1 + return await original_flush(*args, **kwargs) + + monkeypatch.setattr(session, "flush", counting_flush) + with pytest.raises(ProvenanceArchiveError): + await import_tenant_provenance(session, target_scope, invalid_archive) + assert flush_count == 0 + + +@pytest.mark.asyncio +@pytest.mark.postgres +async def test_confidence_unit_interval_boundaries_round_trip( + provenance_sessionmaker, +): + token = uuid.uuid4().hex[:12] + source_scope = _scope(f"confidence-source-{token}") + target_scope = _scope(f"confidence-target-{token}") + async with provenance_sessionmaker() as session: + await _seed_provenance_closure(session, scope=source_scope, token=token) + project_object = await session.scalar( + select(ProjectGraphObjectRecord).where( + ProjectGraphObjectRecord.workspace_id == source_scope.workspace_id + ) + ) + project_edge = await session.scalar( + select(ProjectGraphEdgeRecord).where( + ProjectGraphEdgeRecord.workspace_id == source_scope.workspace_id + ) + ) + project_object.confidence = 0.0 + project_edge.confidence = 1.0 + await session.commit() + async with provenance_sessionmaker() as session: + archive = await export_tenant_provenance(session, source_scope) + records = parse_provenance_archive(archive) + async with provenance_sessionmaker() as session: + await _delete_exported_closure(session, records) + async with provenance_sessionmaker() as session: + await import_tenant_provenance(session, target_scope, archive) + restored_object = await session.scalar( + select(ProjectGraphObjectRecord).where( + ProjectGraphObjectRecord.workspace_id == target_scope.workspace_id, + ProjectGraphObjectRecord.confidence == 0.0, + ) + ) + restored_edge = await session.scalar( + select(ProjectGraphEdgeRecord).where( + ProjectGraphEdgeRecord.workspace_id == target_scope.workspace_id + ) + ) + + assert restored_object is not None + assert restored_edge.confidence == 1.0 + + @pytest.mark.asyncio @pytest.mark.postgres async def test_export_is_exact_workspace_scoped(provenance_sessionmaker): @@ -1257,9 +1531,8 @@ async def test_duplicate_canonical_attachments_keep_node_and_segment_identity( for record in source_records["content_segments"] if record["content_segment_uid"].startswith("duplicate-segment-") } - assert list(source_node_attachments.values()) != sorted( - source_node_attachments.values() - ) + assert len(source_node_attachments) == 5 + assert len(set(source_node_attachments.values())) == 5 async with provenance_sessionmaker() as session: await _delete_exported_closure(session, source_records) async with provenance_sessionmaker() as session: diff --git a/docs/adr/0005-tenant-provenance-portability.md b/docs/adr/0005-tenant-provenance-portability.md index 26748725b..a65f24fbf 100644 --- a/docs/adr/0005-tenant-provenance-portability.md +++ b/docs/adr/0005-tenant-provenance-portability.md @@ -44,6 +44,11 @@ to the exact workspace and includes only their cited source records. membership; OIDC or server verification is required outside tests. 8. Full mailbox/customer-exit completion remains open until email ownership has an explicit workspace dimension and binary object lifecycle is portable. +9. Same-instance cross-workspace imports reuse identical owner-scoped Email and + Attachment rows. Globally unique graph UIDs are deterministically remapped + only while the source graph coexists, using a target-scope plus source-scope + identity table; export reverses the mapping to the stable portable UIDs. The + archive identifies the source user with a one-way digest. ## Alternatives rejected diff --git a/docs/superpowers/specs/2026-08-31-tenant-provenance-roundtrip-design.md b/docs/superpowers/specs/2026-08-31-tenant-provenance-roundtrip-design.md index 0779cf3f6..4da7c92da 100644 --- a/docs/superpowers/specs/2026-08-31-tenant-provenance-roundtrip-design.md +++ b/docs/superpowers/specs/2026-08-31-tenant-provenance-roundtrip-design.md @@ -74,6 +74,14 @@ maps stable identities to new keys. Dangling references or differing existing identities abort the transaction. Exact duplicates are skipped. Absence never deletes target data. +Email and Attachment rows are owner-and-organization sources without workspace +scope, so a same-owner transfer reuses identical rows. If the source graph still +exists in the same database, the importer records a scoped portable-to-database +identity mapping and deterministically remaps every graph UID and typed graph +reference. Export reverses that mapping, preserving the portable archive bytes. +The source-user component stored in the archive is a one-way digest rather than +an account identifier. + ## API - `GET /api/data/provenance-bundle` returns `application/zip`. From 75769779dfa7910afa8904656b23c6a13b2df933 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Tue, 1 Sep 2026 02:18:37 +0900 Subject: [PATCH 17/39] fix(provenance): harden scoped identity imports --- backend/services/tenant_provenance_bundle.py | 65 ++++++- .../tests/test_tenant_provenance_bundle.py | 176 +++++++++++++++++- .../adr/0005-tenant-provenance-portability.md | 6 +- ...8-31-tenant-provenance-roundtrip-design.md | 8 +- 4 files changed, 249 insertions(+), 6 deletions(-) diff --git a/backend/services/tenant_provenance_bundle.py b/backend/services/tenant_provenance_bundle.py index 1a7ae6394..1ff8a4703 100644 --- a/backend/services/tenant_provenance_bundle.py +++ b/backend/services/tenant_provenance_bundle.py @@ -16,7 +16,7 @@ from datetime import date, datetime, timezone from typing import Any -from sqlalchemy import select +from sqlalchemy import func, select from sqlalchemy.exc import DataError, IntegrityError, StatementError from sqlalchemy.ext.asyncio import AsyncSession @@ -89,6 +89,22 @@ "project_edges": "tpe-", "corrections": "tpc-", } +_TYPED_METADATA_UIDS = { + "object_uid": "project_objects", + "source_object_uid": "project_objects", + "target_object_uid": "project_objects", + "content_node_uid": "content_nodes", + "parent_node_uid": "content_nodes", + "source_node_uid": "content_nodes", + "target_node_uid": "content_nodes", + "content_segment_uid": "content_segments", + "segment_uid": "content_segments", + "source_segment_uid": "content_segments", + "target_segment_uid": "content_segments", + "primary_segment_uid": "content_segments", + "primary_content_segment_uid": "content_segments", +} +_TYPED_METADATA_UID_LISTS = {"source_segment_uids": "content_segments"} _TEXTUAL_PARSER_KEYS = frozenset( {"plain_text", "html", "markdown", "json", "csv", "xml", "calendar", "pdf"} ) @@ -421,6 +437,26 @@ def replace_list(record: dict[str, object], field: str, collection: str) -> None maps[collection].get(value, value) for value in record[field] ) + def translate_metadata(value: object) -> object: + if isinstance(value, dict): + translated_mapping = { + key: translate_metadata(item) for key, item in value.items() + } + for key, collection in _TYPED_METADATA_UIDS.items(): + item = translated_mapping.get(key) + if isinstance(item, str): + translated_mapping[key] = maps[collection].get(item, item) + for key, collection in _TYPED_METADATA_UID_LISTS.items(): + item = translated_mapping.get(key) + if isinstance(item, list) and all(isinstance(uid, str) for uid in item): + translated_mapping[key] = [ + maps[collection].get(uid, uid) for uid in item + ] + return translated_mapping + if isinstance(value, list): + return [translate_metadata(item) for item in value] + return value + for record in translated["content_nodes"]: replace(record, "content_node_uid", "content_nodes") replace(record, "parent_node_uid", "content_nodes") @@ -437,6 +473,7 @@ def replace_list(record: dict[str, object], field: str, collection: str) -> None replace(record, "object_uid", "project_objects") replace(record, "primary_content_segment_uid", "content_segments") replace_list(record, "source_segment_uids", "content_segments") + record["attributes_json"] = translate_metadata(record["attributes_json"]) for record in translated["project_edges"]: replace(record, "edge_uid", "project_edges") for field in ( @@ -452,6 +489,8 @@ def replace_list(record: dict[str, object], field: str, collection: str) -> None replace(record, "correction_uid", "corrections") replace(record, "object_uid", "project_objects") replace_list(record, "source_segment_uids", "content_segments") + record["before_json"] = translate_metadata(record["before_json"]) + record["after_json"] = translate_metadata(record["after_json"]) for collection in _REMAPPED_COLLECTIONS: translated[collection].sort(key=lambda record: record[_UID_KEYS[collection]]) return translated @@ -1663,7 +1702,12 @@ def _validate_record_graph(records: Mapping[str, object]) -> None: "workspace_uid", }: _fail() - _safe_identifier(source_scope.get("user_uid")) + source_user_uid = source_scope.get("user_uid") + if ( + not isinstance(source_user_uid, str) + or re.fullmatch(r"[0-9a-f]{64}", source_user_uid) is None + ): + _fail() _safe_identifier(source_scope.get("organization_uid")) _safe_identifier(source_scope.get("workspace_uid")) activity = records.get("export_activity") @@ -2391,6 +2435,23 @@ async def import_tenant_provenance( created = {collection: 0 for collection in _COLLECTIONS} try: async with session.begin(): + bind = session.get_bind() + if getattr(getattr(bind, "dialect", None), "name", None) == "postgresql": + lock_digest = hashlib.sha256( + _canonical_json( + { + "namespace": "tenant-provenance-import-v1", + "source_scope": records["source_scope"], + "target_scope": { + "user_uid": _source_user_uid(scope.user_id), + "organization_uid": scope.organization_id, + "workspace_uid": scope.workspace_id, + }, + } + ) + ).digest() + lock_key = int.from_bytes(lock_digest[:8], "big", signed=True) + await session.execute(select(func.pg_advisory_xact_lock(lock_key))) database_records, identity_rows = await _prepare_identity_import( session, scope, records ) diff --git a/backend/tests/test_tenant_provenance_bundle.py b/backend/tests/test_tenant_provenance_bundle.py index a47b9473c..86e8276a2 100644 --- a/backend/tests/test_tenant_provenance_bundle.py +++ b/backend/tests/test_tenant_provenance_bundle.py @@ -1,4 +1,5 @@ import copy +import asyncio import datetime import hashlib import io @@ -52,7 +53,7 @@ "schema_version": 1, "bundle_uid": "bundle-01HZZ", "source_scope": { - "user_uid": "user-source-01", + "user_uid": "0" * 64, "organization_uid": "org-01", "workspace_uid": "ws-01", }, @@ -1019,6 +1020,36 @@ async def test_same_database_cross_workspace_import_keeps_portable_identity( source = await _seed_provenance_closure( session, scope=source_scope, token=token ) + source_object = await session.scalar( + select(ProjectGraphObjectRecord).where( + ProjectGraphObjectRecord.object_uid == source["object_uids"][0] + ) + ) + source_correction = await session.scalar( + select(ProjectGraphCorrectionRecord).where( + ProjectGraphCorrectionRecord.workspace_id == source_scope.workspace_id + ) + ) + portable_object_uid = source["object_uids"][0] + portable_segment_uid = f"segment-{token}" + source_object.attributes_json = { + "nested": [ + { + "source_object_uid": portable_object_uid, + "source_segment_uids": [portable_segment_uid], + "plain_text": portable_object_uid, + } + ] + } + source_correction.before_json = { + "object_uid": portable_object_uid, + "source_segment_uid": portable_segment_uid, + } + source_correction.after_json = { + "target_object_uid": portable_object_uid, + "primary_segment_uid": portable_segment_uid, + } + await session.commit() async with provenance_sessionmaker() as session: source_archive = await export_tenant_provenance(session, source_scope) source_records = parse_provenance_archive(source_archive) @@ -1074,6 +1105,12 @@ async def test_same_database_cross_workspace_import_keeps_portable_identity( ) ).all() ) + target_segment = await session.scalar( + select(ContentSegmentRecord).where( + ContentSegmentRecord.content_segment_uid + == target_edge.source_segment_uids[0] + ) + ) assert first.created == { "emails": 0, @@ -1147,6 +1184,143 @@ async def test_same_database_cross_workspace_import_keeps_portable_identity( for record in target_objects ) assert target_correction.source_segment_uids == target_edge.source_segment_uids + target_object = next( + record + for record in target_objects + if record.object_uid == target_edge.source_uid + ) + nested_metadata = target_object.attributes_json["nested"][0] + assert nested_metadata["source_object_uid"] == target_object.object_uid + assert nested_metadata["source_segment_uids"] == [ + target_segment.content_segment_uid + ] + assert nested_metadata["plain_text"] == portable_object_uid + assert target_correction.before_json == { + "object_uid": target_object.object_uid, + "source_segment_uid": target_segment.content_segment_uid, + } + assert target_correction.after_json == { + "target_object_uid": target_object.object_uid, + "primary_segment_uid": target_segment.content_segment_uid, + } + + +@pytest.mark.parametrize( + "source_user_uid", + ("a" * 63, "A" * 64, "g" * 64, "0" * 65), +) +@pytest.mark.asyncio +async def test_import_rejects_invalid_source_user_uid_before_transaction( + source_user_uid, +): + records = copy.deepcopy(RECORDS) + records["source_scope"]["user_uid"] = source_user_uid + archive = build_provenance_archive(records) + flush_count = 0 + + class NoTransactionSession: + def begin(self): + raise AssertionError("transaction must not start") + + async def flush(self): + nonlocal flush_count + flush_count += 1 + + with pytest.raises(ProvenanceArchiveError): + await import_tenant_provenance( + NoTransactionSession(), + TenantProvenanceScope("target-user", "target-org", "target-workspace"), + archive, + ) + assert flush_count == 0 + + +@pytest.mark.asyncio +@pytest.mark.postgres +async def test_concurrent_identical_same_database_imports_are_idempotent(): + engine = create_async_engine(settings.DATABASE_URL, echo=False) + session_factory = async_sessionmaker(engine, expire_on_commit=False) + token = uuid.uuid4().hex[:12] + source_scope = _scope(f"concurrent-source-{token}") + target_scope = TenantProvenanceScope( + user_id=source_scope.user_id, + organization_id=source_scope.organization_id, + workspace_id=f"concurrent-target-{token}", + ) + try: + async with engine.begin() as connection: + await connection.run_sync(Base.metadata.create_all) + async with session_factory() as session: + await _seed_provenance_closure(session, scope=source_scope, token=token) + async with session_factory() as session: + archive = await export_tenant_provenance(session, source_scope) + + async def run_import(): + async with session_factory() as session: + return await import_tenant_provenance(session, target_scope, archive) + + first, second = await asyncio.gather(run_import(), run_import()) + created_totals = sorted( + (sum(first.created.values()), sum(second.created.values())) + ) + assert created_totals == [0, 7] + assert sorted((sum(first.skipped.values()), sum(second.skipped.values()))) == [ + 2, + 9, + ] + finally: + async with session_factory.begin() as session: + email = await session.scalar( + select(Email).where(Email.message_id == f"<{token}@example.com>") + ) + if email is not None: + await session.execute( + delete(ProjectGraphCorrectionRecord).where( + ProjectGraphCorrectionRecord.workspace_id.in_( + [source_scope.workspace_id, target_scope.workspace_id] + ) + ) + ) + await session.execute( + delete(ProjectGraphEdgeRecord).where( + ProjectGraphEdgeRecord.workspace_id.in_( + [source_scope.workspace_id, target_scope.workspace_id] + ) + ) + ) + await session.execute( + delete(ProjectGraphObjectRecord).where( + ProjectGraphObjectRecord.workspace_id.in_( + [source_scope.workspace_id, target_scope.workspace_id] + ) + ) + ) + await session.execute( + delete(ProvenanceIdentityMapping).where( + ProvenanceIdentityMapping.target_workspace_id + == target_scope.workspace_id + ) + ) + await session.execute( + delete(KnowledgeGraphEdgeRecord).where( + KnowledgeGraphEdgeRecord.email_id == email.id + ) + ) + await session.execute( + delete(ContentSegmentRecord).where( + ContentSegmentRecord.email_id == email.id + ) + ) + await session.execute( + delete(ContentNodeRecord).where( + ContentNodeRecord.email_id == email.id + ) + ) + await session.execute( + delete(Attachment).where(Attachment.email_id == email.id) + ) + await session.delete(email) + await engine.dispose() @pytest.mark.parametrize( diff --git a/docs/adr/0005-tenant-provenance-portability.md b/docs/adr/0005-tenant-provenance-portability.md index a65f24fbf..458dc6aca 100644 --- a/docs/adr/0005-tenant-provenance-portability.md +++ b/docs/adr/0005-tenant-provenance-portability.md @@ -48,7 +48,11 @@ to the exact workspace and includes only their cited source records. Attachment rows. Globally unique graph UIDs are deterministically remapped only while the source graph coexists, using a target-scope plus source-scope identity table; export reverses the mapping to the stable portable UIDs. The - archive identifies the source user with a one-way digest. + archive identifies the source user with a validated lowercase SHA-256 digest. + Known typed UID keys nested in graph metadata are translated recursively, + while arbitrary string values remain unchanged. PostgreSQL serializes only + identical source-to-target import scopes with a transaction advisory lock so + concurrent retries remain idempotent. ## Alternatives rejected diff --git a/docs/superpowers/specs/2026-08-31-tenant-provenance-roundtrip-design.md b/docs/superpowers/specs/2026-08-31-tenant-provenance-roundtrip-design.md index 4da7c92da..d559094e0 100644 --- a/docs/superpowers/specs/2026-08-31-tenant-provenance-roundtrip-design.md +++ b/docs/superpowers/specs/2026-08-31-tenant-provenance-roundtrip-design.md @@ -78,9 +78,13 @@ Email and Attachment rows are owner-and-organization sources without workspace scope, so a same-owner transfer reuses identical rows. If the source graph still exists in the same database, the importer records a scoped portable-to-database identity mapping and deterministically remaps every graph UID and typed graph -reference. Export reverses that mapping, preserving the portable archive bytes. +reference, including recognized UID keys nested in object and correction +metadata. Arbitrary strings are never value-matched or rewritten. Export +reverses that mapping, preserving the portable archive bytes. The source-user component stored in the archive is a one-way digest rather than -an account identifier. +an account identifier and must be exactly a lowercase SHA-256 hex digest. +Concurrent imports of the same source-to-target scope are serialized with a +PostgreSQL transaction advisory lock; other scopes remain independent. ## API From 4fbac3dc212186b9c4a275bcd79c6374d22bc6ab Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Tue, 1 Sep 2026 02:33:39 +0900 Subject: [PATCH 18/39] fix(provenance): preserve untyped metadata strings --- backend/services/tenant_provenance_bundle.py | 16 ++++++++++------ backend/tests/test_tenant_provenance_bundle.py | 14 +++++++++++++- 2 files changed, 23 insertions(+), 7 deletions(-) diff --git a/backend/services/tenant_provenance_bundle.py b/backend/services/tenant_provenance_bundle.py index 1ff8a4703..3f0b6277c 100644 --- a/backend/services/tenant_provenance_bundle.py +++ b/backend/services/tenant_provenance_bundle.py @@ -105,6 +105,9 @@ "primary_content_segment_uid": "content_segments", } _TYPED_METADATA_UID_LISTS = {"source_segment_uids": "content_segments"} +_TYPED_METADATA_ATTACHMENT_REFERENCES = frozenset( + {"attachment_uid", "source_record_uid"} +) _TEXTUAL_PARSER_KEYS = frozenset( {"plain_text", "html", "markdown", "json", "csv", "xml", "calendar", "pdf"} ) @@ -369,15 +372,16 @@ def _validate_safe_metadata(value: object, depth: int = 0) -> None: def _portable_metadata( value: object, attachment_references: Mapping[str, str] ) -> object: - if isinstance(value, str): - return attachment_references.get(value, value) if isinstance(value, list): return [_portable_metadata(item, attachment_references) for item in value] if isinstance(value, Mapping): - return { - key: _portable_metadata(item, attachment_references) - for key, item in value.items() - } + portable = {} + for key, item in value.items(): + if key in _TYPED_METADATA_ATTACHMENT_REFERENCES and isinstance(item, str): + portable[key] = attachment_references.get(item, item) + else: + portable[key] = _portable_metadata(item, attachment_references) + return portable return value diff --git a/backend/tests/test_tenant_provenance_bundle.py b/backend/tests/test_tenant_provenance_bundle.py index 86e8276a2..c58247b92 100644 --- a/backend/tests/test_tenant_provenance_bundle.py +++ b/backend/tests/test_tenant_provenance_bundle.py @@ -1773,6 +1773,10 @@ async def test_recognized_pdf_omits_integer_keys_and_round_trips_graph_reference project_object.attributes_json = { "source_record_uid": legacy_source_uid, "source_object_uid": project_object.object_uid, + "nested": { + "attachment_uid": legacy_source_uid, + "free_text": legacy_source_uid, + }, } await session.commit() async with provenance_sessionmaker() as session: @@ -1782,7 +1786,7 @@ async def test_recognized_pdf_omits_integer_keys_and_round_trips_graph_reference portable_source_uid = f"attachment:{attachment_uid}" serialized = json.dumps(records, sort_keys=True) - assert legacy_source_uid not in serialized + assert serialized.count(legacy_source_uid) == 1 assert { records["content_nodes"][0]["source_record_uid"], records["content_segments"][0]["source_record_uid"], @@ -1792,6 +1796,10 @@ async def test_recognized_pdf_omits_integer_keys_and_round_trips_graph_reference records["project_objects"][0]["attributes_json"]["source_record_uid"] == portable_source_uid ) + assert records["project_objects"][0]["attributes_json"]["nested"] == { + "attachment_uid": portable_source_uid, + "free_text": legacy_source_uid, + } async with provenance_sessionmaker() as session: await _delete_exported_closure(session, records) @@ -1810,6 +1818,10 @@ async def test_recognized_pdf_omits_integer_keys_and_round_trips_graph_reference assert restored_node.source_record_uid == portable_source_uid assert restored["content_nodes"][0]["attachment_uid"] == attachment_uid assert restored["content_nodes"][0]["source_record_uid"] == portable_source_uid + assert restored["project_objects"][0]["attributes_json"]["nested"] == { + "attachment_uid": portable_source_uid, + "free_text": legacy_source_uid, + } @pytest.mark.asyncio From 73f174032ae7fe84dc4902466260950b4dff1f93 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Tue, 1 Sep 2026 02:39:07 +0900 Subject: [PATCH 19/39] fix(provenance): require verified archive membership --- .../0018_provenance_identity_mappings.py | 19 +++++--- backend/api/data.py | 2 +- backend/tests/test_data_api.py | 48 ++++++++++++++++++- .../tests/test_tenant_provenance_bundle.py | 39 +++++++++++++++ .../2026-08-31-tenant-provenance-roundtrip.md | 4 +- 5 files changed, 103 insertions(+), 9 deletions(-) diff --git a/backend/alembic/versions/0018_provenance_identity_mappings.py b/backend/alembic/versions/0018_provenance_identity_mappings.py index 2bf731ddd..5227196b9 100644 --- a/backend/alembic/versions/0018_provenance_identity_mappings.py +++ b/backend/alembic/versions/0018_provenance_identity_mappings.py @@ -8,11 +8,14 @@ down_revision = "0017_merge_newsdom_carddav_heads" branch_labels = None depends_on = None +_MAPPING_TABLE = "provenance_identity_mappings" def upgrade() -> None: - op.create_table( - "provenance_identity_mappings", + inspector = sa.inspect(op.get_bind()) + if not inspector.has_table(_MAPPING_TABLE): + op.create_table( + _MAPPING_TABLE, sa.Column("provenance_identity_id", sa.Integer(), primary_key=True), sa.Column("target_user_id", sa.String(), nullable=False), sa.Column("target_organization_id", sa.String(), nullable=False), @@ -39,17 +42,21 @@ def upgrade() -> None: "target_database_uid", name="uq_provenance_identity_target_uid", ), - ) + ) op.create_index( "ix_provenance_identity_target_scope", - "provenance_identity_mappings", + _MAPPING_TABLE, ["target_user_id", "target_organization_id", "target_workspace_id"], + if_not_exists=True, ) def downgrade() -> None: + if not sa.inspect(op.get_bind()).has_table(_MAPPING_TABLE): + return op.drop_index( "ix_provenance_identity_target_scope", - table_name="provenance_identity_mappings", + table_name=_MAPPING_TABLE, + if_exists=True, ) - op.drop_table("provenance_identity_mappings") + op.drop_table(_MAPPING_TABLE) diff --git a/backend/api/data.py b/backend/api/data.py index 573dff312..ef7c0c1ea 100644 --- a/backend/api/data.py +++ b/backend/api/data.py @@ -3177,7 +3177,7 @@ def _provenance_scope(auth_context: AuthContext) -> TenantProvenanceScope: def _require_authoritative_provenance_scope(auth_context: AuthContext) -> None: - if auth_context.session_verifier == "hmac": + if auth_context.session_verifier != "oidc": raise HTTPException( status_code=403, detail="Authoritative workspace membership is required for provenance bundles", diff --git a/backend/tests/test_data_api.py b/backend/tests/test_data_api.py index fa3ddcf9c..22136e6ba 100644 --- a/backend/tests/test_data_api.py +++ b/backend/tests/test_data_api.py @@ -444,6 +444,49 @@ async def forbidden_service(*args, **kwargs): } +@pytest.mark.parametrize("session_verifier", ["server", "override"]) +@pytest.mark.parametrize( + ("method", "path"), + [ + ("get", "/api/data/provenance-bundle"), + ("post", "/api/data/provenance-bundle/import"), + ], +) +def test_provenance_bundle_endpoints_reject_non_oidc_contexts_before_service( + session_verifier, method, path, mock_db, monkeypatch +): + async def forbidden_service(*args, **kwargs): + raise AssertionError("provenance service must not run") + + monkeypatch.setattr(data_api, "export_tenant_provenance", forbidden_service) + monkeypatch.setattr(data_api, "import_tenant_provenance", forbidden_service) + client, original_overrides = _with_authoritative_auth( + mock_db, + AuthContext( + user_id="unverified-user", + role="member", + organization_id="unverified-org", + group_ids=(), + workspace_id="unverified-workspace", + session_verifier=session_verifier, + ), + ) + try: + response = client.request( + method.upper(), + path, + content=b"attacker archive" if method == "post" else b"", + ) + finally: + client.close() + _restore_authoritative_overrides(original_overrides) + + assert response.status_code == 403 + assert response.json() == { + "detail": "Authoritative workspace membership is required for provenance bundles" + } + + def test_provenance_bundle_download_uses_signed_scope_and_zip_response( mock_db, monkeypatch ): @@ -498,6 +541,7 @@ async def fake_export(session, scope): organization_id="org-acme", group_ids=(), workspace_id="workspace-org-acme", + session_verifier="oidc", ), ) try: @@ -534,7 +578,7 @@ async def fake_import(session, scope, archive_bytes): organization_id="target-org", group_ids=(), workspace_id="target-workspace", - session_verifier="server", + session_verifier="oidc", ), ) try: @@ -580,6 +624,7 @@ async def fake_import(session, scope, archive_bytes): organization_id="org-acme", group_ids=("group-data",), workspace_id="workspace-org-acme", + session_verifier="oidc", ), ) try: @@ -613,6 +658,7 @@ async def fake_import(session, scope, archive_bytes): organization_id="org-acme", group_ids=("group-data",), workspace_id="workspace-org-acme", + session_verifier="oidc", ), ) try: diff --git a/backend/tests/test_tenant_provenance_bundle.py b/backend/tests/test_tenant_provenance_bundle.py index c58247b92..0c945c1f4 100644 --- a/backend/tests/test_tenant_provenance_bundle.py +++ b/backend/tests/test_tenant_provenance_bundle.py @@ -2,16 +2,20 @@ import asyncio import datetime import hashlib +import importlib.util import io import json import struct import uuid import warnings import zipfile +from pathlib import Path import asyncpg import pytest import pytest_asyncio +from alembic.migration import MigrationContext +from alembic.operations import Operations from sqlalchemy import delete, func, select, text from sqlalchemy.exc import OperationalError from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine @@ -1323,6 +1327,41 @@ async def run_import(): await engine.dispose() +@pytest.mark.asyncio +@pytest.mark.postgres +async def test_final_migration_upgrade_is_safe_after_fresh_metadata_bootstrap( + provenance_sessionmaker, +): + revision_path = ( + Path(__file__).parents[1] + / "alembic" + / "versions" + / "0018_provenance_identity_mappings.py" + ) + spec = importlib.util.spec_from_file_location( + "provenance_identity_revision", revision_path + ) + assert spec is not None and spec.loader is not None + revision = importlib.util.module_from_spec(spec) + spec.loader.exec_module(revision) + original_op = revision.op + + def run_upgrade(sync_connection): + revision.op = Operations(MigrationContext.configure(sync_connection)) + revision.upgrade() + revision.upgrade() + assert sync_connection.dialect.has_table( + sync_connection, "provenance_identity_mappings" + ) + + try: + async with provenance_sessionmaker() as session: + connection = await session.connection() + await connection.run_sync(run_upgrade) + finally: + revision.op = original_op + + @pytest.mark.parametrize( ("model", "value"), ( diff --git a/docs/superpowers/plans/2026-08-31-tenant-provenance-roundtrip.md b/docs/superpowers/plans/2026-08-31-tenant-provenance-roundtrip.md index 87522f63d..be870d791 100644 --- a/docs/superpowers/plans/2026-08-31-tenant-provenance-roundtrip.md +++ b/docs/superpowers/plans/2026-08-31-tenant-provenance-roundtrip.md @@ -22,7 +22,9 @@ PostgreSQL/pgvector, pytest. binary documents, provider URLs or tokens, or legacy audit details. - Source scope and target authority come from the signed session. - Validate the bounded archive and reference graph before mutation. -- Add no dependency and no persistence table. +- Add no dependency. The coexistence-safe implementation adds one internal + portable-to-database identity mapping table; its multi-word sequential key + `provenance_identity_id` is never serialized or exposed through the API. - Treat warning-class output as failure. --- From 092e39383ec2e8578750456bc480f92f64e8ad0f Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Tue, 1 Sep 2026 02:39:26 +0900 Subject: [PATCH 20/39] fix(provenance): export mixed identity origins --- backend/services/tenant_provenance_bundle.py | 56 +++++++++++-- .../tests/test_tenant_provenance_bundle.py | 83 +++++++++++++++++++ .../adr/0005-tenant-provenance-portability.md | 6 +- ...8-31-tenant-provenance-roundtrip-design.md | 7 +- 4 files changed, 140 insertions(+), 12 deletions(-) diff --git a/backend/services/tenant_provenance_bundle.py b/backend/services/tenant_provenance_bundle.py index 3f0b6277c..3eb83a896 100644 --- a/backend/services/tenant_provenance_bundle.py +++ b/backend/services/tenant_provenance_bundle.py @@ -1285,20 +1285,56 @@ async def descendants(model: Any, order_column: Any) -> list[Any]: for collection in _REMAPPED_COLLECTIONS } if identity_rows: - nodes = [ - row + nodes_by_id = {row.content_node_id: row for row in nodes} + nodes_by_uid = {row.content_node_uid: row for row in nodes} + selected_node_uids = { + row.content_node_uid for row in nodes if row.content_node_uid in target_mapped_uids["content_nodes"] - ] + } | {nodes_by_id[row.content_node_id].content_node_uid for row in cited_segments} + pending_node_uids = list(selected_node_uids) + while pending_node_uids: + parent_uid = nodes_by_uid[pending_node_uids.pop()].parent_node_uid + if parent_uid is not None and parent_uid not in selected_node_uids: + selected_node_uids.add(parent_uid) + pending_node_uids.append(parent_uid) + cited_segment_uid_set = {row.content_segment_uid for row in cited_segments} + nodes = [row for row in nodes if row.content_node_uid in selected_node_uids] segments = [ row for row in segments if row.content_segment_uid in target_mapped_uids["content_segments"] + or row.content_segment_uid in cited_segment_uid_set ] + selected_node_ids = {row.content_node_id for row in nodes} + selected_segment_ids = {row.content_segment_id for row in segments} structural_edges = [ row for row in structural_edges if row.edge_uid in target_mapped_uids["structural_edges"] + or ( + (row.source_node_id is None or row.source_node_id in selected_node_ids) + and ( + row.target_node_id is None or row.target_node_id in selected_node_ids + ) + and ( + row.source_segment_id is None + or row.source_segment_id in selected_segment_ids + ) + and ( + row.target_segment_id is None + or row.target_segment_id in selected_segment_ids + ) + and any( + value is not None + for value in ( + row.source_node_id, + row.target_node_id, + row.source_segment_id, + row.target_segment_id, + ) + ) + ) ] else: nodes = [ @@ -1339,9 +1375,10 @@ async def descendants(model: Any, order_column: Any) -> list[Any]: ) for row in relevant_identity_rows } - if relevant_identity_rows: - if len(source_scopes) != 1: - _fail() + can_restore_source_identity = bool(relevant_identity_rows) and len( + source_scopes + ) == 1 + if can_restore_source_identity: for collection in _REMAPPED_COLLECTIONS: collection_rows = [ row for row in relevant_identity_rows if row.entity_kind == collection @@ -1349,10 +1386,13 @@ async def descendants(model: Any, order_column: Any) -> list[Any]: if {row.target_database_uid for row in collection_rows} != database_uids[ collection ]: - _fail() + can_restore_source_identity = False + break reverse_maps[collection] = { row.target_database_uid: row.portable_uid for row in collection_rows } + + if can_restore_source_identity: source_user_uid, source_organization_uid, source_workspace_uid = next( iter(source_scopes) ) @@ -1413,7 +1453,7 @@ async def descendants(model: Any, order_column: Any) -> list[Any]: for correction in corrections ], } - if relevant_identity_rows: + if can_restore_source_identity: payload = _translate_identity_records(payload, reverse_maps) content_digest = hashlib.sha256(_canonical_json(payload)).hexdigest() records = { diff --git a/backend/tests/test_tenant_provenance_bundle.py b/backend/tests/test_tenant_provenance_bundle.py index 0c945c1f4..a0482c31c 100644 --- a/backend/tests/test_tenant_provenance_bundle.py +++ b/backend/tests/test_tenant_provenance_bundle.py @@ -1209,6 +1209,89 @@ async def test_same_database_cross_workspace_import_keeps_portable_identity( } +@pytest.mark.asyncio +@pytest.mark.postgres +async def test_mixed_native_and_multiple_import_origins_export_as_target_scope( + provenance_sessionmaker, +): + token = uuid.uuid4().hex[:12] + target_scope = _scope(f"target-{token}") + source_scopes = [_scope(f"source-{index}-{token}") for index in range(2)] + async with provenance_sessionmaker() as session: + native = await _seed_provenance_closure( + session, scope=target_scope, token=f"native-{token}" + ) + native_node = await session.get(ContentNodeRecord, native["node_id"]) + parent_uid = f"parent-node-native-{token}" + session.add( + ContentNodeRecord( + content_node_uid=parent_uid, + email_id=native["email_id"], + attachment_id=native["attachment_id"], + source_kind="attachment", + source_record_uid=f"attachment-source-native-{token}", + parent_node_uid=None, + node_kind="document", + node_path="/", + ordinal_index=0, + display_label="Parent", + safe_text_content="Parent evidence", + content_hash=f"parenthash-{token}", + ) + ) + native_node.parent_node_uid = parent_uid + await session.commit() + sources = [ + await _seed_provenance_closure( + session, scope=scope, token=f"source-{index}-{token}" + ) + for index, scope in enumerate(source_scopes) + ] + archives = [] + async with provenance_sessionmaker() as session: + for scope in source_scopes: + archives.append(await export_tenant_provenance(session, scope)) + for archive in archives: + async with provenance_sessionmaker() as session: + await import_tenant_provenance(session, target_scope, archive) + async with provenance_sessionmaker() as session: + records = parse_provenance_archive( + await export_tenant_provenance(session, target_scope) + ) + + assert records["source_scope"] == { + "user_uid": hashlib.sha256(target_scope.user_id.encode()).hexdigest(), + "organization_uid": target_scope.organization_id, + "workspace_uid": target_scope.workspace_id, + } + assert { + collection: len(records[collection]) + for collection in ( + "emails", + "attachments", + "content_nodes", + "content_segments", + "structural_edges", + "project_objects", + "project_edges", + "corrections", + ) + } == { + "emails": 3, + "attachments": 3, + "content_nodes": 4, + "content_segments": 3, + "structural_edges": 3, + "project_objects": 6, + "project_edges": 3, + "corrections": 3, + } + serialized = json.dumps(records, sort_keys=True) + assert native["email_uid"] in serialized + assert parent_uid in serialized + assert all(source["email_uid"] in serialized for source in sources) + + @pytest.mark.parametrize( "source_user_uid", ("a" * 63, "A" * 64, "g" * 64, "0" * 65), diff --git a/docs/adr/0005-tenant-provenance-portability.md b/docs/adr/0005-tenant-provenance-portability.md index 458dc6aca..c57035058 100644 --- a/docs/adr/0005-tenant-provenance-portability.md +++ b/docs/adr/0005-tenant-provenance-portability.md @@ -41,7 +41,7 @@ to the exact workspace and includes only their cited source records. 7. Signed-session API routes use the current `user_id`, `organization_id`, and `workspace_id`; bundle payload scope cannot override target authority. HMAC fallback sessions are rejected because they do not prove workspace - membership; OIDC or server verification is required outside tests. + membership; OIDC verification is required outside tests. 8. Full mailbox/customer-exit completion remains open until email ownership has an explicit workspace dimension and binary object lifecycle is portable. 9. Same-instance cross-workspace imports reuse identical owner-scoped Email and @@ -52,7 +52,9 @@ to the exact workspace and includes only their cited source records. Known typed UID keys nested in graph metadata are translated recursively, while arbitrary string values remain unchanged. PostgreSQL serializes only identical source-to-target import scopes with a transaction advisory lock so - concurrent retries remain idempotent. + concurrent retries remain idempotent. A target containing native records or + multiple imported origins exports one target-scoped archive with its current + database UIDs; a single fully mapped origin still restores its portable UIDs. ## Alternatives rejected diff --git a/docs/superpowers/specs/2026-08-31-tenant-provenance-roundtrip-design.md b/docs/superpowers/specs/2026-08-31-tenant-provenance-roundtrip-design.md index d559094e0..fcf31fb56 100644 --- a/docs/superpowers/specs/2026-08-31-tenant-provenance-roundtrip-design.md +++ b/docs/superpowers/specs/2026-08-31-tenant-provenance-roundtrip-design.md @@ -81,6 +81,9 @@ identity mapping and deterministically remaps every graph UID and typed graph reference, including recognized UID keys nested in object and correction metadata. Arbitrary strings are never value-matched or rewritten. Export reverses that mapping, preserving the portable archive bytes. +If native records or multiple imported origins coexist, export instead emits a +single target-scoped closure with the target database UIDs so every record stays +exportable without pretending that the mixed archive has one original source. The source-user component stored in the archive is a one-way digest rather than an account identifier and must be exactly a lowercase SHA-256 hex digest. Concurrent imports of the same source-to-target scope are serialized with a @@ -93,8 +96,8 @@ PostgreSQL transaction advisory lock; other scopes remain independent. returns created/skipped counts plus a verified manifest digest. - Both routes use the existing signed authentication dependency and reject the HMAC fallback verifier because it is not authoritative workspace-membership - evidence. OIDC and server-verified contexts are accepted; dependency override - is test-only evidence. + evidence. Only OIDC-verified contexts are accepted; dependency override is + test-only evidence. ## Verification From 9da29fd433eda690be78dde663c448031ddeb824 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Tue, 1 Sep 2026 02:59:13 +0900 Subject: [PATCH 21/39] fix(provenance): reject foreign segment citations --- backend/services/tenant_provenance_bundle.py | 13 +++- .../tests/test_tenant_provenance_bundle.py | 61 +++++++++++++++++++ 2 files changed, 72 insertions(+), 2 deletions(-) diff --git a/backend/services/tenant_provenance_bundle.py b/backend/services/tenant_provenance_bundle.py index 3eb83a896..c65f3bcd5 100644 --- a/backend/services/tenant_provenance_bundle.py +++ b/backend/services/tenant_provenance_bundle.py @@ -1199,13 +1199,22 @@ async def export_tenant_provenance( cited_segments = list( ( await session.scalars( - select(ContentSegmentRecord).where( + select(ContentSegmentRecord) + .join(Email, ContentSegmentRecord.email_id == Email.id) + .where( (ContentSegmentRecord.content_segment_uid.in_(cited_segment_uids)) - | (ContentSegmentRecord.content_segment_id.in_(primary_segment_ids)) + | (ContentSegmentRecord.content_segment_id.in_(primary_segment_ids)), + *_scope_filters(Email, scope, workspace=False), ) ) ).all() ) + if not cited_segment_uids.issubset( + {segment.content_segment_uid for segment in cited_segments} + ) or not primary_segment_ids.issubset( + {segment.content_segment_id for segment in cited_segments} + ): + _fail() email_ids = {record.email_id for record in project_objects} | { segment.email_id for segment in cited_segments } diff --git a/backend/tests/test_tenant_provenance_bundle.py b/backend/tests/test_tenant_provenance_bundle.py index a0482c31c..51b81227f 100644 --- a/backend/tests/test_tenant_provenance_bundle.py +++ b/backend/tests/test_tenant_provenance_bundle.py @@ -2372,3 +2372,64 @@ async def test_export_excludes_separate_user_and_organization_scope( selected["email_uid"] } assert rival["email_uid"] not in json.dumps(records) + + +@pytest.mark.parametrize( + "reference_field", + ( + "object_source", + "object_primary", + "edge_source", + "edge_primary", + "correction_source", + ), +) +@pytest.mark.asyncio +@pytest.mark.postgres +async def test_export_rejects_cross_tenant_segment_references_before_email_closure( + provenance_sessionmaker, + reference_field, +): + token = uuid.uuid4().hex[:12] + scope = _scope(f"selected-{token}") + foreign_scope = _scope(f"foreign-{token}") + async with provenance_sessionmaker() as session: + selected = await _seed_provenance_closure( + session, scope=scope, token=f"selected-{token}" + ) + foreign = await _seed_provenance_closure( + session, scope=foreign_scope, token=f"foreign-{token}" + ) + foreign_segment = await session.scalar( + select(ContentSegmentRecord).where( + ContentSegmentRecord.email_id == foreign["email_id"] + ) + ) + if reference_field.startswith("object_"): + record = await session.scalar( + select(ProjectGraphObjectRecord).where( + ProjectGraphObjectRecord.project_graph_object_id + == selected["project_object_id"] + ) + ) + elif reference_field.startswith("edge_"): + record = await session.scalar( + select(ProjectGraphEdgeRecord).where( + ProjectGraphEdgeRecord.workspace_id == scope.workspace_id + ) + ) + else: + record = await session.scalar( + select(ProjectGraphCorrectionRecord).where( + ProjectGraphCorrectionRecord.workspace_id == scope.workspace_id + ) + ) + if reference_field.endswith("_primary"): + record.primary_content_segment_id = foreign_segment.content_segment_id + else: + record.source_segment_uids = [foreign_segment.content_segment_uid] + await session.commit() + + async with provenance_sessionmaker() as session: + with pytest.raises(ProvenanceArchiveError): + await export_tenant_provenance(session, scope) From 4bd6589cf695d3156498a9f68284f738ea6c2a15 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Tue, 1 Sep 2026 03:00:48 +0900 Subject: [PATCH 22/39] fix(provenance): compose import transactions --- .../0018_provenance_identity_mappings.py | 52 ++++++------- backend/services/tenant_provenance_bundle.py | 5 +- .../tests/test_tenant_provenance_bundle.py | 73 ++++++++++++++++++- 3 files changed, 102 insertions(+), 28 deletions(-) diff --git a/backend/alembic/versions/0018_provenance_identity_mappings.py b/backend/alembic/versions/0018_provenance_identity_mappings.py index 5227196b9..2d56498d1 100644 --- a/backend/alembic/versions/0018_provenance_identity_mappings.py +++ b/backend/alembic/versions/0018_provenance_identity_mappings.py @@ -16,32 +16,32 @@ def upgrade() -> None: if not inspector.has_table(_MAPPING_TABLE): op.create_table( _MAPPING_TABLE, - sa.Column("provenance_identity_id", sa.Integer(), primary_key=True), - sa.Column("target_user_id", sa.String(), nullable=False), - sa.Column("target_organization_id", sa.String(), nullable=False), - sa.Column("target_workspace_id", sa.String(), nullable=False), - sa.Column("source_user_uid", sa.String(length=64), nullable=False), - sa.Column("source_organization_uid", sa.String(), nullable=False), - sa.Column("source_workspace_uid", sa.String(), nullable=False), - sa.Column("entity_kind", sa.String(length=64), nullable=False), - sa.Column("portable_uid", sa.String(length=256), nullable=False), - sa.Column("target_database_uid", sa.String(length=96), nullable=False), - sa.UniqueConstraint( - "target_user_id", - "target_organization_id", - "target_workspace_id", - "source_user_uid", - "source_organization_uid", - "source_workspace_uid", - "entity_kind", - "portable_uid", - name="uq_provenance_identity_source_target", - ), - sa.UniqueConstraint( - "entity_kind", - "target_database_uid", - name="uq_provenance_identity_target_uid", - ), + sa.Column("provenance_identity_id", sa.Integer(), primary_key=True), + sa.Column("target_user_id", sa.String(), nullable=False), + sa.Column("target_organization_id", sa.String(), nullable=False), + sa.Column("target_workspace_id", sa.String(), nullable=False), + sa.Column("source_user_uid", sa.String(length=64), nullable=False), + sa.Column("source_organization_uid", sa.String(), nullable=False), + sa.Column("source_workspace_uid", sa.String(), nullable=False), + sa.Column("entity_kind", sa.String(length=64), nullable=False), + sa.Column("portable_uid", sa.String(length=256), nullable=False), + sa.Column("target_database_uid", sa.String(length=96), nullable=False), + sa.UniqueConstraint( + "target_user_id", + "target_organization_id", + "target_workspace_id", + "source_user_uid", + "source_organization_uid", + "source_workspace_uid", + "entity_kind", + "portable_uid", + name="uq_provenance_identity_source_target", + ), + sa.UniqueConstraint( + "entity_kind", + "target_database_uid", + name="uq_provenance_identity_target_uid", + ), ) op.create_index( "ix_provenance_identity_target_scope", diff --git a/backend/services/tenant_provenance_bundle.py b/backend/services/tenant_provenance_bundle.py index c65f3bcd5..1f0bb58ce 100644 --- a/backend/services/tenant_provenance_bundle.py +++ b/backend/services/tenant_provenance_bundle.py @@ -2487,7 +2487,10 @@ async def import_tenant_provenance( _fail() created = {collection: 0 for collection in _COLLECTIONS} try: - async with session.begin(): + transaction = ( + session.begin_nested() if session.in_transaction() else session.begin() + ) + async with transaction: bind = session.get_bind() if getattr(getattr(bind, "dialect", None), "name", None) == "postgresql": lock_digest = hashlib.sha256( diff --git a/backend/tests/test_tenant_provenance_bundle.py b/backend/tests/test_tenant_provenance_bundle.py index 51b81227f..e87a8833e 100644 --- a/backend/tests/test_tenant_provenance_bundle.py +++ b/backend/tests/test_tenant_provenance_bundle.py @@ -17,7 +17,7 @@ from alembic.migration import MigrationContext from alembic.operations import Operations from sqlalchemy import delete, func, select, text -from sqlalchemy.exc import OperationalError +from sqlalchemy.exc import IntegrityError, OperationalError from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine from core.config import settings @@ -34,6 +34,7 @@ ProvenanceIdentityMapping, ) from services.project_graph.repository import ProjectGraphRepository +from services import tenant_provenance_bundle as provenance_service from services.tenant_provenance_bundle import ( ARCHIVE_MAX_BYTES, @@ -1008,6 +1009,76 @@ async def test_postgres_round_trip_preserves_stable_evidence_with_fresh_keys( } +@pytest.mark.asyncio +@pytest.mark.postgres +async def test_import_composes_with_transaction_started_by_prior_select( + provenance_sessionmaker, +): + token = uuid.uuid4().hex[:12] + source_scope = _scope(f"active-source-{token}") + target_scope = _scope(f"active-target-{token}") + async with provenance_sessionmaker() as session: + await _seed_provenance_closure(session, scope=source_scope, token=token) + async with provenance_sessionmaker() as session: + records = parse_provenance_archive( + await export_tenant_provenance(session, source_scope) + ) + archive = build_provenance_archive(records) + async with provenance_sessionmaker() as session: + await _delete_exported_closure(session, records) + + async with provenance_sessionmaker() as session: + await session.execute(select(func.count()).select_from(Email)) + assert session.in_transaction() + receipt = await import_tenant_provenance(session, target_scope, archive) + assert sum(receipt.created.values()) == 9 + assert await session.scalar( + select(func.count()).select_from(ProjectGraphObjectRecord).where( + ProjectGraphObjectRecord.workspace_id == target_scope.workspace_id + ) + ) == 2 + + +@pytest.mark.asyncio +@pytest.mark.postgres +async def test_active_transaction_import_failure_rolls_back_only_import_savepoint( + provenance_sessionmaker, + monkeypatch, +): + token = uuid.uuid4().hex[:12] + source_scope = _scope(f"rollback-source-{token}") + target_scope = _scope(f"rollback-target-{token}") + async with provenance_sessionmaker() as session: + await _seed_provenance_closure(session, scope=source_scope, token=token) + async with provenance_sessionmaker() as session: + records = parse_provenance_archive( + await export_tenant_provenance(session, source_scope) + ) + archive = build_provenance_archive(records) + async with provenance_sessionmaker() as session: + await _delete_exported_closure(session, records) + + original_insert = provenance_service._insert_records + + async def fail_after_insert(*args, **kwargs): + await original_insert(*args, **kwargs) + raise IntegrityError("forced import failure", {}, RuntimeError()) + + monkeypatch.setattr(provenance_service, "_insert_records", fail_after_insert) + async with provenance_sessionmaker() as session: + baseline_count = await session.scalar(select(func.count()).select_from(Email)) + assert session.in_transaction() + with pytest.raises(ProvenanceArchiveError): + await import_tenant_provenance(session, target_scope, archive) + assert session.in_transaction() + assert await session.scalar(select(func.count()).select_from(Email)) == baseline_count + assert await session.scalar( + select(func.count()).select_from(ProjectGraphObjectRecord).where( + ProjectGraphObjectRecord.workspace_id == target_scope.workspace_id + ) + ) == 0 + + @pytest.mark.asyncio @pytest.mark.postgres async def test_same_database_cross_workspace_import_keeps_portable_identity( From 26c8513528fca71001de75cb53827794c0aa83c4 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Tue, 1 Sep 2026 03:11:21 +0900 Subject: [PATCH 23/39] fix(provenance): bind citations to selected records --- backend/services/tenant_provenance_bundle.py | 35 ++++++++++++ .../tests/test_tenant_provenance_bundle.py | 53 +++++++------------ 2 files changed, 55 insertions(+), 33 deletions(-) diff --git a/backend/services/tenant_provenance_bundle.py b/backend/services/tenant_provenance_bundle.py index 1f0bb58ce..3616bfaa7 100644 --- a/backend/services/tenant_provenance_bundle.py +++ b/backend/services/tenant_provenance_bundle.py @@ -1215,6 +1215,41 @@ async def export_tenant_provenance( {segment.content_segment_id for segment in cited_segments} ): _fail() + segments_by_uid = { + segment.content_segment_uid: segment for segment in cited_segments + } + segments_by_id = {segment.content_segment_id: segment for segment in cited_segments} + objects_by_id = { + record.project_graph_object_id: record for record in project_objects + } + for record in project_objects: + if segments_by_id[ + record.primary_content_segment_id + ].email_id != record.email_id or any( + segments_by_uid[segment_uid].email_id != record.email_id + for segment_uid in record.source_segment_uids + ): + _fail() + for record in project_edges: + source_object = objects_by_id.get(record.source_object_id) + target_object = objects_by_id.get(record.target_object_id) + if source_object is None or target_object is None: + _fail() + endpoint_email_ids = {source_object.email_id, target_object.email_id} + if segments_by_id[ + record.primary_content_segment_id + ].email_id not in endpoint_email_ids or any( + segments_by_uid[segment_uid].email_id not in endpoint_email_ids + for segment_uid in record.source_segment_uids + ): + _fail() + for record in corrections: + project_object = objects_by_id.get(record.project_graph_object_id) + if project_object is None or any( + segments_by_uid[segment_uid].email_id != project_object.email_id + for segment_uid in record.source_segment_uids + ): + _fail() email_ids = {record.email_id for record in project_objects} | { segment.email_id for segment in cited_segments } diff --git a/backend/tests/test_tenant_provenance_bundle.py b/backend/tests/test_tenant_provenance_bundle.py index e87a8833e..a21d83f49 100644 --- a/backend/tests/test_tenant_provenance_bundle.py +++ b/backend/tests/test_tenant_provenance_bundle.py @@ -877,19 +877,11 @@ async def _two_email_rooted_archive( token: str, ) -> bytes: source = await _seed_provenance_closure(session, scope=scope, token=token) - cited = await _seed_email_graph_without_project_rows( + cited = await _seed_provenance_closure( session, scope=scope, token=f"cited-{token}", ) - project_object = await session.scalar( - select(ProjectGraphObjectRecord).where( - ProjectGraphObjectRecord.object_uid == source["object_uids"][0] - ) - ) - project_object.source_segment_uids = sorted( - [*project_object.source_segment_uids, f"segment-cited-{token}"] - ) session.add( Attachment( email_id=source["email_id"], @@ -1804,7 +1796,7 @@ async def test_import_is_idempotent_for_exact_target_records(provenance_sessionm @pytest.mark.parametrize("citation_owner", ("object", "edge", "correction")) @pytest.mark.asyncio @pytest.mark.postgres -async def test_export_includes_cross_email_citation_source_closure( +async def test_export_rejects_cross_email_citation_source_closure( provenance_sessionmaker, citation_owner, ): @@ -1817,7 +1809,7 @@ async def test_export_includes_cross_email_citation_source_closure( ) async with provenance_sessionmaker() as session: source = await _seed_provenance_closure(session, scope=scope, token=token) - cited = await _seed_provenance_closure( + await _seed_provenance_closure( session, scope=cited_scope, token=f"cited-{token}", @@ -1846,26 +1838,8 @@ async def test_export_includes_cross_email_citation_source_closure( ) await session.commit() async with provenance_sessionmaker() as session: - records = parse_provenance_archive( + with pytest.raises(ProvenanceArchiveError): await export_tenant_provenance(session, scope) - ) - - assert {record["email_uid"] for record in records["emails"]} == { - source["email_uid"], - cited["email_uid"], - } - assert cited_segment_uid in { - record["content_segment_uid"] for record in records["content_segments"] - } - assert f"node-cited-{token}" in { - record["content_node_uid"] for record in records["content_nodes"] - } - assert any( - record["email_uid"] == cited["email_uid"] for record in records["attachments"] - ) - assert f"project-object-source-cited-{token}" not in { - record["object_uid"] for record in records["project_objects"] - } @pytest.mark.asyncio @@ -2453,17 +2427,24 @@ async def test_export_excludes_separate_user_and_organization_scope( "edge_source", "edge_primary", "correction_source", + "edge_source_object", + "edge_target_object", + "correction_object", ), ) @pytest.mark.asyncio @pytest.mark.postgres -async def test_export_rejects_cross_tenant_segment_references_before_email_closure( +async def test_export_rejects_cross_workspace_segment_references_before_email_closure( provenance_sessionmaker, reference_field, ): token = uuid.uuid4().hex[:12] scope = _scope(f"selected-{token}") - foreign_scope = _scope(f"foreign-{token}") + foreign_scope = TenantProvenanceScope( + user_id=scope.user_id, + organization_id=scope.organization_id, + workspace_id=f"workspace-foreign-{token}", + ) async with provenance_sessionmaker() as session: selected = await _seed_provenance_closure( session, scope=scope, token=f"selected-{token}" @@ -2495,7 +2476,13 @@ async def test_export_rejects_cross_tenant_segment_references_before_email_closu ProjectGraphCorrectionRecord.workspace_id == scope.workspace_id ) ) - if reference_field.endswith("_primary"): + if reference_field == "edge_source_object": + record.source_object_id = foreign["project_object_id"] + elif reference_field == "edge_target_object": + record.target_object_id = foreign["project_object_id"] + elif reference_field == "correction_object": + record.project_graph_object_id = foreign["project_object_id"] + elif reference_field.endswith("_primary"): record.primary_content_segment_id = foreign_segment.content_segment_id else: record.source_segment_uids = [foreign_segment.content_segment_uid] From 176540c0a080a13c9e33ddc63b1d350435446f52 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Tue, 1 Sep 2026 03:15:36 +0900 Subject: [PATCH 24/39] fix(provenance): allow anchored segment edges --- backend/services/tenant_provenance_bundle.py | 14 +++++--- .../tests/test_tenant_provenance_bundle.py | 35 ++++++++++++++++++- 2 files changed, 44 insertions(+), 5 deletions(-) diff --git a/backend/services/tenant_provenance_bundle.py b/backend/services/tenant_provenance_bundle.py index 3616bfaa7..de6511f6f 100644 --- a/backend/services/tenant_provenance_bundle.py +++ b/backend/services/tenant_provenance_bundle.py @@ -1231,11 +1231,17 @@ async def export_tenant_provenance( ): _fail() for record in project_edges: - source_object = objects_by_id.get(record.source_object_id) - target_object = objects_by_id.get(record.target_object_id) - if source_object is None or target_object is None: + endpoint_objects = [] + for endpoint_id in (record.source_object_id, record.target_object_id): + if endpoint_id is None: + continue + endpoint_object = objects_by_id.get(endpoint_id) + if endpoint_object is None: + _fail() + endpoint_objects.append(endpoint_object) + if not endpoint_objects: _fail() - endpoint_email_ids = {source_object.email_id, target_object.email_id} + endpoint_email_ids = {endpoint.email_id for endpoint in endpoint_objects} if segments_by_id[ record.primary_content_segment_id ].email_id not in endpoint_email_ids or any( diff --git a/backend/tests/test_tenant_provenance_bundle.py b/backend/tests/test_tenant_provenance_bundle.py index a21d83f49..9de7f9bc0 100644 --- a/backend/tests/test_tenant_provenance_bundle.py +++ b/backend/tests/test_tenant_provenance_bundle.py @@ -2419,6 +2419,35 @@ async def test_export_excludes_separate_user_and_organization_scope( assert rival["email_uid"] not in json.dumps(records) +@pytest.mark.asyncio +@pytest.mark.postgres +async def test_export_allows_segment_evidence_edge_with_nullable_source_object( + provenance_sessionmaker, +): + token = uuid.uuid4().hex[:12] + scope = _scope(f"segment-edge-{token}") + async with provenance_sessionmaker() as session: + selected = await _seed_provenance_closure(session, scope=scope, token=token) + edge = await session.scalar( + select(ProjectGraphEdgeRecord).where( + ProjectGraphEdgeRecord.workspace_id == scope.workspace_id + ) + ) + edge.source_uid = f"segment-{token}" + edge.source_object_id = None + await session.commit() + + async with provenance_sessionmaker() as session: + records = parse_provenance_archive( + await export_tenant_provenance(session, scope) + ) + + exported_edge = records["project_edges"][0] + assert exported_edge["source_uid"] == f"segment-{token}" + assert exported_edge["source_object_uid"] is None + assert exported_edge["target_object_uid"] in selected["object_uids"] + + @pytest.mark.parametrize( "reference_field", ( @@ -2430,6 +2459,7 @@ async def test_export_excludes_separate_user_and_organization_scope( "edge_source_object", "edge_target_object", "correction_object", + "edge_without_object_anchors", ), ) @pytest.mark.asyncio @@ -2476,7 +2506,10 @@ async def test_export_rejects_cross_workspace_segment_references_before_email_cl ProjectGraphCorrectionRecord.workspace_id == scope.workspace_id ) ) - if reference_field == "edge_source_object": + if reference_field == "edge_without_object_anchors": + record.source_object_id = None + record.target_object_id = None + elif reference_field == "edge_source_object": record.source_object_id = foreign["project_object_id"] elif reference_field == "edge_target_object": record.target_object_id = foreign["project_object_id"] From c53703ae8f04799630b4e7933a68c8e86579f781 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Tue, 1 Sep 2026 03:27:21 +0900 Subject: [PATCH 25/39] fix(provenance): serialize shared email imports --- backend/services/tenant_provenance_bundle.py | 9 +- .../tests/test_tenant_provenance_bundle.py | 100 ++++++++++++++++++ 2 files changed, 105 insertions(+), 4 deletions(-) diff --git a/backend/services/tenant_provenance_bundle.py b/backend/services/tenant_provenance_bundle.py index de6511f6f..e05b6be6c 100644 --- a/backend/services/tenant_provenance_bundle.py +++ b/backend/services/tenant_provenance_bundle.py @@ -2537,13 +2537,14 @@ async def import_tenant_provenance( lock_digest = hashlib.sha256( _canonical_json( { - "namespace": "tenant-provenance-import-v1", - "source_scope": records["source_scope"], - "target_scope": { + "namespace": "tenant-provenance-import-v2", + "target_owner": { "user_uid": _source_user_uid(scope.user_id), "organization_uid": scope.organization_id, - "workspace_uid": scope.workspace_id, }, + "email_uids": sorted( + record["email_uid"] for record in records["emails"] + ), } ) ).digest() diff --git a/backend/tests/test_tenant_provenance_bundle.py b/backend/tests/test_tenant_provenance_bundle.py index 9de7f9bc0..b3f12de68 100644 --- a/backend/tests/test_tenant_provenance_bundle.py +++ b/backend/tests/test_tenant_provenance_bundle.py @@ -1473,6 +1473,106 @@ async def run_import(): await engine.dispose() +@pytest.mark.asyncio +@pytest.mark.postgres +async def test_concurrent_imports_across_workspaces_reuse_owner_email(): + engine = create_async_engine(settings.DATABASE_URL, echo=False) + session_factory = async_sessionmaker(engine, expire_on_commit=False) + token = uuid.uuid4().hex[:12] + source_scope = _scope(f"parallel-workspaces-source-{token}") + target_scopes = tuple( + TenantProvenanceScope( + user_id=source_scope.user_id, + organization_id=source_scope.organization_id, + workspace_id=f"parallel-workspace-{index}-{token}", + ) + for index in range(2) + ) + try: + async with engine.begin() as connection: + await connection.run_sync(Base.metadata.create_all) + async with session_factory() as session: + await _seed_provenance_closure(session, scope=source_scope, token=token) + async with session_factory() as session: + archive = await export_tenant_provenance(session, source_scope) + + async def run_import(target_scope): + async with session_factory() as session: + return await import_tenant_provenance(session, target_scope, archive) + + receipts = await asyncio.gather( + *(run_import(target_scope) for target_scope in target_scopes) + ) + assert all(sum(receipt.created.values()) == 7 for receipt in receipts) + async with session_factory() as session: + assert ( + await session.scalar( + select(func.count()) + .select_from(Email) + .where( + Email.user_id == source_scope.user_id, + Email.organization_id == source_scope.organization_id, + Email.message_id == f"<{token}@example.com>", + ) + ) + == 1 + ) + finally: + async with session_factory.begin() as session: + email = await session.scalar( + select(Email).where( + Email.user_id == source_scope.user_id, + Email.organization_id == source_scope.organization_id, + Email.message_id == f"<{token}@example.com>", + ) + ) + if email is not None: + workspace_ids = [ + source_scope.workspace_id, + *(scope.workspace_id for scope in target_scopes), + ] + await session.execute( + delete(ProjectGraphCorrectionRecord).where( + ProjectGraphCorrectionRecord.workspace_id.in_(workspace_ids) + ) + ) + await session.execute( + delete(ProjectGraphEdgeRecord).where( + ProjectGraphEdgeRecord.workspace_id.in_(workspace_ids) + ) + ) + await session.execute( + delete(ProjectGraphObjectRecord).where( + ProjectGraphObjectRecord.workspace_id.in_(workspace_ids) + ) + ) + await session.execute( + delete(ProvenanceIdentityMapping).where( + ProvenanceIdentityMapping.target_workspace_id.in_(workspace_ids) + ) + ) + await session.execute( + delete(KnowledgeGraphEdgeRecord).where( + KnowledgeGraphEdgeRecord.email_id == email.id + ) + ) + await session.execute( + delete(ContentSegmentRecord).where( + ContentSegmentRecord.email_id == email.id + ) + ) + await session.execute( + delete(ContentNodeRecord).where( + ContentNodeRecord.email_id == email.id + ) + ) + await session.execute( + delete(Attachment).where(Attachment.email_id == email.id) + ) + await session.delete(email) + await engine.dispose() + + @pytest.mark.asyncio @pytest.mark.postgres async def test_final_migration_upgrade_is_safe_after_fresh_metadata_bootstrap( From 4b3ae1af37c961ec2946966a6a660d86ea0c3ddd Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Tue, 1 Sep 2026 03:28:23 +0900 Subject: [PATCH 26/39] fix: validate imported provenance citation anchors --- backend/services/tenant_provenance_bundle.py | 28 +++++ .../tests/test_tenant_provenance_bundle.py | 103 ++++++++++++++++++ 2 files changed, 131 insertions(+) diff --git a/backend/services/tenant_provenance_bundle.py b/backend/services/tenant_provenance_bundle.py index e05b6be6c..52d680574 100644 --- a/backend/services/tenant_provenance_bundle.py +++ b/backend/services/tenant_provenance_bundle.py @@ -1853,6 +1853,10 @@ def _validate_record_graph(records: Mapping[str, object]) -> None: record["content_segment_uid"]: record["attachment_uid"] for record in collections["content_segments"] } + object_email = { + record["object_uid"]: record["email_uid"] + for record in collections["project_objects"] + } def require_reference(value: object, available: set[str]) -> None: if value is not None and value not in available: @@ -1937,9 +1941,24 @@ def require_reference(value: object, available: set[str]) -> None: _fail() for segment_uid in record["source_segment_uids"]: require_reference(segment_uid, segment_uids) + if segment_email[segment_uid] != record["email_uid"]: + _fail() for record in collections["project_edges"]: require_reference(record["source_object_uid"], object_uids) require_reference(record["target_object_uid"], object_uids) + endpoint_uids = { + endpoint_uid + for endpoint_uid in ( + record["source_object_uid"], + record["target_object_uid"], + ) + if endpoint_uid is not None + } + if not endpoint_uids: + _fail() + endpoint_email_uids = { + object_email[endpoint_uid] for endpoint_uid in endpoint_uids + } if ( record["source_object_uid"] is not None and record["source_uid"] != record["source_object_uid"] @@ -1951,12 +1970,21 @@ def require_reference(value: object, available: set[str]) -> None: ): _fail() require_reference(record["primary_content_segment_uid"], segment_uids) + if ( + segment_email[record["primary_content_segment_uid"]] + not in endpoint_email_uids + ): + _fail() for segment_uid in record["source_segment_uids"]: require_reference(segment_uid, segment_uids) + if segment_email[segment_uid] not in endpoint_email_uids: + _fail() for record in collections["corrections"]: require_reference(record["object_uid"], object_uids) for segment_uid in record["source_segment_uids"]: require_reference(segment_uid, segment_uids) + if segment_email[segment_uid] != object_email[record["object_uid"]]: + _fail() rooted_email_uids = { record["email_uid"] for record in collections["project_objects"] diff --git a/backend/tests/test_tenant_provenance_bundle.py b/backend/tests/test_tenant_provenance_bundle.py index b3f12de68..047d65c3f 100644 --- a/backend/tests/test_tenant_provenance_bundle.py +++ b/backend/tests/test_tenant_provenance_bundle.py @@ -1685,6 +1685,109 @@ async def counting_flush(*args, **kwargs): assert flush_count == 0 +@pytest.mark.parametrize( + "reference_field", + ( + "object_source", + "object_primary", + "edge_source", + "edge_primary", + "correction_source", + "edge_without_object_anchors", + ), +) +@pytest.mark.asyncio +@pytest.mark.postgres +async def test_import_rejects_cross_email_project_references_before_flush( + provenance_sessionmaker, + monkeypatch, + reference_field, +): + token = uuid.uuid4().hex[:12] + source_scope = _scope(f"citation-source-{token}") + target_scope = _scope(f"citation-target-{token}") + async with provenance_sessionmaker() as session: + archive = await _two_email_rooted_archive( + session, + scope=source_scope, + token=token, + ) + records = parse_provenance_archive(archive) + object_email = { + record["object_uid"]: record["email_uid"] + for record in records["project_objects"] + } + segment_email = { + record["content_segment_uid"]: record["email_uid"] + for record in records["content_segments"] + } + if reference_field.startswith("object_"): + record = records["project_objects"][0] + anchor_email_uid = record["email_uid"] + elif reference_field.startswith("edge_"): + record = records["project_edges"][0] + anchor_email_uid = object_email[ + record["source_object_uid"] or record["target_object_uid"] + ] + else: + record = records["corrections"][0] + anchor_email_uid = object_email[record["object_uid"]] + foreign_segment_uid = next( + segment_uid + for segment_uid, email_uid in segment_email.items() + if email_uid != anchor_email_uid + ) + if reference_field == "edge_without_object_anchors": + record["source_object_uid"] = None + record["target_object_uid"] = None + elif reference_field.endswith("_primary"): + record["primary_content_segment_uid"] = foreign_segment_uid + else: + record["source_segment_uids"] = [foreign_segment_uid] + invalid_archive = build_provenance_archive(records) + + async with provenance_sessionmaker() as session: + flush_count = 0 + original_flush = session.flush + + async def counting_flush(*args, **kwargs): + nonlocal flush_count + flush_count += 1 + return await original_flush(*args, **kwargs) + + monkeypatch.setattr(session, "flush", counting_flush) + with pytest.raises(ProvenanceArchiveError): + await import_tenant_provenance(session, target_scope, invalid_archive) + assert flush_count == 0 + + +@pytest.mark.asyncio +@pytest.mark.postgres +async def test_import_allows_edge_with_one_nullable_object_endpoint( + provenance_sessionmaker, +): + token = uuid.uuid4().hex[:12] + source_scope = _scope(f"nullable-source-{token}") + target_scope = _scope(f"nullable-target-{token}") + async with provenance_sessionmaker() as session: + await _seed_provenance_closure(session, scope=source_scope, token=token) + async with provenance_sessionmaker() as session: + records = parse_provenance_archive( + await export_tenant_provenance(session, source_scope) + ) + records["project_edges"][0]["source_object_uid"] = None + records["project_edges"][0]["source_uid"] = f"segment-{token}" + + async with provenance_sessionmaker() as session: + receipt = await import_tenant_provenance( + session, + target_scope, + build_provenance_archive(records), + ) + + assert receipt.created["project_edges"] == 1 + + @pytest.mark.asyncio @pytest.mark.postgres async def test_confidence_unit_interval_boundaries_round_trip( From 03de2231132aae75d9a0690bb0fc17aac950826e Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Tue, 1 Sep 2026 03:33:41 +0900 Subject: [PATCH 27/39] fix(provenance): lock overlapping email imports --- backend/services/tenant_provenance_bundle.py | 33 +++-- .../tests/test_tenant_provenance_bundle.py | 130 +++++++++++++----- 2 files changed, 110 insertions(+), 53 deletions(-) diff --git a/backend/services/tenant_provenance_bundle.py b/backend/services/tenant_provenance_bundle.py index 52d680574..739ed17ed 100644 --- a/backend/services/tenant_provenance_bundle.py +++ b/backend/services/tenant_provenance_bundle.py @@ -2562,22 +2562,25 @@ async def import_tenant_provenance( async with transaction: bind = session.get_bind() if getattr(getattr(bind, "dialect", None), "name", None) == "postgresql": - lock_digest = hashlib.sha256( - _canonical_json( - { - "namespace": "tenant-provenance-import-v2", - "target_owner": { - "user_uid": _source_user_uid(scope.user_id), - "organization_uid": scope.organization_id, - }, - "email_uids": sorted( - record["email_uid"] for record in records["emails"] - ), - } + for email_uid in sorted( + record["email_uid"] for record in records["emails"] + ): + lock_digest = hashlib.sha256( + _canonical_json( + { + "namespace": "tenant-provenance-email-import-v1", + "target_owner": { + "user_uid": _source_user_uid(scope.user_id), + "organization_uid": scope.organization_id, + }, + "email_uid": email_uid, + } + ) + ).digest() + lock_key = int.from_bytes(lock_digest[:8], "big", signed=True) + await session.execute( + select(func.pg_advisory_xact_lock(lock_key)) ) - ).digest() - lock_key = int.from_bytes(lock_digest[:8], "big", signed=True) - await session.execute(select(func.pg_advisory_xact_lock(lock_key))) database_records, identity_rows = await _prepare_identity_import( session, scope, records ) diff --git a/backend/tests/test_tenant_provenance_bundle.py b/backend/tests/test_tenant_provenance_bundle.py index 047d65c3f..7505a0bd1 100644 --- a/backend/tests/test_tenant_provenance_bundle.py +++ b/backend/tests/test_tenant_provenance_bundle.py @@ -1480,10 +1480,12 @@ async def test_concurrent_imports_across_workspaces_reuse_owner_email(): session_factory = async_sessionmaker(engine, expire_on_commit=False) token = uuid.uuid4().hex[:12] source_scope = _scope(f"parallel-workspaces-source-{token}") + target_user_id = f"parallel-target-user-{token}" + target_organization_id = f"parallel-target-org-{token}" target_scopes = tuple( TenantProvenanceScope( - user_id=source_scope.user_id, - organization_id=source_scope.organization_id, + user_id=target_user_id, + organization_id=target_organization_id, workspace_id=f"parallel-workspace-{index}-{token}", ) for index in range(2) @@ -1492,65 +1494,117 @@ async def test_concurrent_imports_across_workspaces_reuse_owner_email(): async with engine.begin() as connection: await connection.run_sync(Base.metadata.create_all) async with session_factory() as session: - await _seed_provenance_closure(session, scope=source_scope, token=token) - async with session_factory() as session: - archive = await export_tenant_provenance(session, source_scope) + archive = await _two_email_rooted_archive( + session, scope=source_scope, token=token + ) + first_records = parse_provenance_archive(archive) + second_records = copy.deepcopy(first_records) + shared_email_uid, distinct_email_uid = sorted( + record["email_uid"] for record in first_records["emails"] + ) + replacement_email_uid = f"" + + def replace_email_uid(value): + if isinstance(value, dict): + return {key: replace_email_uid(item) for key, item in value.items()} + if isinstance(value, list): + return [replace_email_uid(item) for item in value] + return replacement_email_uid if value == distinct_email_uid else value + + second_records = replace_email_uid(second_records) + attachment_uid_replacements = {} + for attachment in second_records["attachments"]: + if attachment["email_uid"] != replacement_email_uid: + continue + canonical = provenance_service._attachment_payload_core(attachment) + attachment_uid_replacements[attachment["attachment_uid"]] = ( + provenance_service._attachment_uid(canonical, 1) + ) - async def run_import(target_scope): + def replace_attachment_uid(value): + if isinstance(value, dict): + return { + key: replace_attachment_uid(item) for key, item in value.items() + } + if isinstance(value, list): + return [replace_attachment_uid(item) for item in value] + if isinstance(value, str) and value.startswith("attachment:"): + attachment_uid = value.removeprefix("attachment:") + return "attachment:" + attachment_uid_replacements.get( + attachment_uid, attachment_uid + ) + return attachment_uid_replacements.get(value, value) + + second_records = replace_attachment_uid(second_records) + uid_keys = { + "emails": "email_uid", + "attachments": "attachment_uid", + **provenance_service._UID_KEYS, + } + for collection, uid_key in uid_keys.items(): + second_records[collection].sort(key=lambda record: record[uid_key]) + second_archive = build_provenance_archive(second_records) + + async def run_import(target_scope, target_archive): async with session_factory() as session: - return await import_tenant_provenance(session, target_scope, archive) + return await import_tenant_provenance( + session, target_scope, target_archive + ) receipts = await asyncio.gather( - *(run_import(target_scope) for target_scope in target_scopes) + run_import(target_scopes[0], archive), + run_import(target_scopes[1], second_archive), ) - assert all(sum(receipt.created.values()) == 7 for receipt in receipts) + assert all(sum(receipt.created.values()) > 0 for receipt in receipts) async with session_factory() as session: assert ( await session.scalar( select(func.count()) .select_from(Email) .where( - Email.user_id == source_scope.user_id, - Email.organization_id == source_scope.organization_id, - Email.message_id == f"<{token}@example.com>", + Email.user_id == target_user_id, + Email.organization_id == target_organization_id, + Email.message_id == shared_email_uid, ) ) == 1 ) finally: async with session_factory.begin() as session: - email = await session.scalar( - select(Email).where( - Email.user_id == source_scope.user_id, - Email.organization_id == source_scope.organization_id, - Email.message_id == f"<{token}@example.com>", + workspace_ids = [ + source_scope.workspace_id, + *(scope.workspace_id for scope in target_scopes), + ] + await session.execute( + delete(ProjectGraphCorrectionRecord).where( + ProjectGraphCorrectionRecord.workspace_id.in_(workspace_ids) ) ) - if email is not None: - workspace_ids = [ - source_scope.workspace_id, - *(scope.workspace_id for scope in target_scopes), - ] - await session.execute( - delete(ProjectGraphCorrectionRecord).where( - ProjectGraphCorrectionRecord.workspace_id.in_(workspace_ids) - ) + await session.execute( + delete(ProjectGraphEdgeRecord).where( + ProjectGraphEdgeRecord.workspace_id.in_(workspace_ids) ) - await session.execute( - delete(ProjectGraphEdgeRecord).where( - ProjectGraphEdgeRecord.workspace_id.in_(workspace_ids) - ) + ) + await session.execute( + delete(ProjectGraphObjectRecord).where( + ProjectGraphObjectRecord.workspace_id.in_(workspace_ids) ) - await session.execute( - delete(ProjectGraphObjectRecord).where( - ProjectGraphObjectRecord.workspace_id.in_(workspace_ids) - ) + ) + await session.execute( + delete(ProvenanceIdentityMapping).where( + ProvenanceIdentityMapping.target_workspace_id.in_(workspace_ids) ) - await session.execute( - delete(ProvenanceIdentityMapping).where( - ProvenanceIdentityMapping.target_workspace_id.in_(workspace_ids) + ) + emails = list( + ( + await session.scalars( + select(Email).where( + Email.user_id.in_([source_scope.user_id, target_user_id]) + ) ) - ) + ).all() + ) + for email in emails: await session.execute( delete(KnowledgeGraphEdgeRecord).where( KnowledgeGraphEdgeRecord.email_id == email.id From d855dc74fdc0c8d278fcee555b46860ea5f72a08 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Tue, 1 Sep 2026 03:45:46 +0900 Subject: [PATCH 28/39] fix: require canonical provenance timestamps --- backend/services/tenant_provenance_bundle.py | 2 +- .../tests/test_tenant_provenance_bundle.py | 54 +++++++++++++++++++ .../adr/0005-tenant-provenance-portability.md | 3 +- 3 files changed, 57 insertions(+), 2 deletions(-) diff --git a/backend/services/tenant_provenance_bundle.py b/backend/services/tenant_provenance_bundle.py index 739ed17ed..c91be1a5a 100644 --- a/backend/services/tenant_provenance_bundle.py +++ b/backend/services/tenant_provenance_bundle.py @@ -878,7 +878,7 @@ def _parse_datetime(value: object) -> datetime: parsed = datetime.fromisoformat(text_value) except ValueError as exc: raise ProvenanceArchiveError("Invalid provenance archive") from exc - if parsed.tzinfo is None: + if parsed.tzinfo is None or _utc_text(parsed) != text_value: _fail() return parsed diff --git a/backend/tests/test_tenant_provenance_bundle.py b/backend/tests/test_tenant_provenance_bundle.py index 7505a0bd1..eeb24e991 100644 --- a/backend/tests/test_tenant_provenance_bundle.py +++ b/backend/tests/test_tenant_provenance_bundle.py @@ -1739,6 +1739,60 @@ async def counting_flush(*args, **kwargs): assert flush_count == 0 +@pytest.mark.parametrize( + ("collection", "timestamp_field"), + (("emails", "date"), ("corrections", "created_at")), +) +@pytest.mark.asyncio +@pytest.mark.postgres +async def test_import_rejects_equivalent_offset_timestamps_before_flush_on_retry( + provenance_sessionmaker, + monkeypatch, + collection, + timestamp_field, +): + token = uuid.uuid4().hex[:12] + source_scope = _scope(f"timestamp-source-{token}") + target_scope = _scope(f"timestamp-target-{token}") + async with provenance_sessionmaker() as session: + await _seed_provenance_closure(session, scope=source_scope, token=token) + async with provenance_sessionmaker() as session: + records = parse_provenance_archive( + await export_tenant_provenance(session, source_scope) + ) + valid_archive = build_provenance_archive(records) + async with provenance_sessionmaker() as session: + await import_tenant_provenance(session, target_scope, valid_archive) + + offset_records = copy.deepcopy(records) + utc_timestamp = datetime.datetime.fromisoformat( + offset_records[collection][0][timestamp_field] + ) + offset_records[collection][0][timestamp_field] = utc_timestamp.astimezone( + datetime.timezone(datetime.timedelta(hours=9)) + ).isoformat() + offset_archive = build_provenance_archive(offset_records) + + async with provenance_sessionmaker() as session: + flush_count = 0 + original_flush = session.flush + + async def counting_flush(*args, **kwargs): + nonlocal flush_count + flush_count += 1 + return await original_flush(*args, **kwargs) + + monkeypatch.setattr(session, "flush", counting_flush) + for _ in range(2): + with pytest.raises(ProvenanceArchiveError): + await import_tenant_provenance( + session, + target_scope, + offset_archive, + ) + assert flush_count == 0 + + @pytest.mark.parametrize( "reference_field", ( diff --git a/docs/adr/0005-tenant-provenance-portability.md b/docs/adr/0005-tenant-provenance-portability.md index c57035058..c1b9cc01c 100644 --- a/docs/adr/0005-tenant-provenance-portability.md +++ b/docs/adr/0005-tenant-provenance-portability.md @@ -27,7 +27,8 @@ to the exact workspace and includes only their cited source records. 2. Use stable logical UIDs in the payload. Never serialize or restore sequential database primary keys. Import resolves new keys in foreign-key order. 3. Canonical JSON uses UTF-8, no BOM, no insignificant whitespace, sorted object - keys, and pre-sorted set-like arrays. SHA-512 manifests cover exact bytes. + keys, pre-sorted set-like arrays, and canonical UTC `Z` email and correction + timestamps. SHA-512 manifests cover exact bytes. 4. Reject unsafe or colliding ZIP paths, unlisted files, checksum mismatch, unsupported profiles, dangling references, scope mismatch, duplicate logical UIDs, and non-finite numbers before mutation. From 42795bdcbf160141c74a15089c047f202114bec8 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Tue, 1 Sep 2026 03:46:11 +0900 Subject: [PATCH 29/39] fix(provenance): reuse complete identity mappings --- backend/services/tenant_provenance_bundle.py | 55 +++++++++++++--- .../tests/test_tenant_provenance_bundle.py | 63 +++++++++++++++++++ 2 files changed, 111 insertions(+), 7 deletions(-) diff --git a/backend/services/tenant_provenance_bundle.py b/backend/services/tenant_provenance_bundle.py index c91be1a5a..25657b449 100644 --- a/backend/services/tenant_provenance_bundle.py +++ b/backend/services/tenant_provenance_bundle.py @@ -2034,6 +2034,54 @@ async def _prepare_identity_import( if same_scope: return copy.deepcopy(records), [] + forward_maps = {collection: {} for collection in _REMAPPED_COLLECTIONS} + for collection in _REMAPPED_COLLECTIONS: + for record in records[collection]: + portable_uid = record[_UID_KEYS[collection]] + forward_maps[collection][portable_uid] = _target_database_uid( + scope, source_scope, collection, portable_uid + ) + expected_mapping_keys = { + (collection, portable_uid) + for collection, collection_map in forward_maps.items() + for portable_uid in collection_map + } + organization_filter = ( + ProvenanceIdentityMapping.target_organization_id == scope.organization_id + if scope.organization_id is not None + else ProvenanceIdentityMapping.target_organization_id.is_(None) + ) + scoped_mapping_rows = list( + ( + await session.scalars( + select(ProvenanceIdentityMapping).where( + ProvenanceIdentityMapping.target_user_id == scope.user_id, + organization_filter, + ProvenanceIdentityMapping.target_workspace_id == scope.workspace_id, + ProvenanceIdentityMapping.source_user_uid + == source_scope["user_uid"], + ProvenanceIdentityMapping.source_organization_uid + == source_scope["organization_uid"], + ProvenanceIdentityMapping.source_workspace_uid + == source_scope["workspace_uid"], + ProvenanceIdentityMapping.entity_kind.in_(_REMAPPED_COLLECTIONS), + ) + ) + ).all() + ) + scoped_mappings = { + (row.entity_kind, row.portable_uid): row + for row in scoped_mapping_rows + if (row.entity_kind, row.portable_uid) in expected_mapping_keys + } + if scoped_mappings: + if set(scoped_mappings) != expected_mapping_keys: + _fail() + for (collection, portable_uid), row in scoped_mappings.items(): + if row.target_database_uid != forward_maps[collection][portable_uid]: + _fail() + return _translate_identity_records(records, forward_maps), [] + scoped_collisions: list[Any] = [] for model, column, collection in ( ( @@ -2076,13 +2124,6 @@ async def _prepare_identity_import( ): _fail() - forward_maps = {collection: {} for collection in _REMAPPED_COLLECTIONS} - for collection in _REMAPPED_COLLECTIONS: - for record in records[collection]: - portable_uid = record[_UID_KEYS[collection]] - forward_maps[collection][portable_uid] = _target_database_uid( - scope, source_scope, collection, portable_uid - ) target_uids = { database_uid for collection_map in forward_maps.values() diff --git a/backend/tests/test_tenant_provenance_bundle.py b/backend/tests/test_tenant_provenance_bundle.py index eeb24e991..40a0ad533 100644 --- a/backend/tests/test_tenant_provenance_bundle.py +++ b/backend/tests/test_tenant_provenance_bundle.py @@ -2104,6 +2104,69 @@ async def test_import_is_idempotent_for_exact_target_records(provenance_sessionm assert second.skipped == first.created +@pytest.mark.asyncio +@pytest.mark.postgres +async def test_cross_workspace_retry_uses_mappings_after_source_graph_deletion( + provenance_sessionmaker, +): + token = uuid.uuid4().hex[:12] + source_scope = _scope(f"deleted-source-{token}") + target_scope = TenantProvenanceScope( + user_id=source_scope.user_id, + organization_id=source_scope.organization_id, + workspace_id=f"retained-target-{token}", + ) + async with provenance_sessionmaker() as session: + await _seed_provenance_closure(session, scope=source_scope, token=token) + async with provenance_sessionmaker() as session: + archive = await export_tenant_provenance(session, source_scope) + async with provenance_sessionmaker() as session: + first = await import_tenant_provenance(session, target_scope, archive) + async with provenance_sessionmaker() as session: + await session.execute( + delete(ProjectGraphCorrectionRecord).where( + ProjectGraphCorrectionRecord.workspace_id == source_scope.workspace_id + ) + ) + await session.execute( + delete(ProjectGraphEdgeRecord).where( + ProjectGraphEdgeRecord.workspace_id == source_scope.workspace_id + ) + ) + await session.execute( + delete(ProjectGraphObjectRecord).where( + ProjectGraphObjectRecord.workspace_id == source_scope.workspace_id + ) + ) + await session.commit() + async with provenance_sessionmaker() as session: + second = await import_tenant_provenance(session, target_scope, archive) + target_counts = ( + await session.scalar( + select(func.count()).select_from(ProjectGraphObjectRecord).where( + ProjectGraphObjectRecord.workspace_id == target_scope.workspace_id + ) + ), + await session.scalar( + select(func.count()).select_from(ProjectGraphEdgeRecord).where( + ProjectGraphEdgeRecord.workspace_id == target_scope.workspace_id + ) + ), + await session.scalar( + select(func.count()).select_from(ProjectGraphCorrectionRecord).where( + ProjectGraphCorrectionRecord.workspace_id == target_scope.workspace_id + ) + ), + ) + + assert sum(first.created.values()) > 0 + assert sum(second.created.values()) == 0 + assert sum(second.skipped.values()) == sum(first.created.values()) + sum( + first.skipped.values() + ) + assert target_counts == (2, 1, 1) + + @pytest.mark.parametrize("citation_owner", ("object", "edge", "correction")) @pytest.mark.asyncio @pytest.mark.postgres From fe0ad999987636b68ec1d6d18680c3eddf8e0f4d Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Tue, 1 Sep 2026 03:50:19 +0900 Subject: [PATCH 30/39] fix: canonicalize provenance activity timestamps --- backend/services/tenant_provenance_bundle.py | 11 +--- .../tests/test_tenant_provenance_bundle.py | 59 ++++++++++++++++++- .../adr/0005-tenant-provenance-portability.md | 4 +- 3 files changed, 61 insertions(+), 13 deletions(-) diff --git a/backend/services/tenant_provenance_bundle.py b/backend/services/tenant_provenance_bundle.py index 25657b449..2f35bdd53 100644 --- a/backend/services/tenant_provenance_bundle.py +++ b/backend/services/tenant_provenance_bundle.py @@ -13,7 +13,7 @@ import zipfile from collections.abc import Mapping from dataclasses import dataclass -from datetime import date, datetime, timezone +from datetime import datetime, timezone from typing import Any from sqlalchemy import func, select @@ -524,14 +524,7 @@ def _safe_identifier(value: object) -> str: def _iso8601_date_or_datetime(value: object) -> str: date_published = _safe_identifier(value) - try: - date.fromisoformat(date_published) - except ValueError: - try: - datetime.fromisoformat(date_published) - except ValueError: - _fail() - return date_published + return _utc_text(_parse_datetime(date_published)) def _bag_info(bundle_uid: str) -> bytes: diff --git a/backend/tests/test_tenant_provenance_bundle.py b/backend/tests/test_tenant_provenance_bundle.py index 40a0ad533..54679f3c2 100644 --- a/backend/tests/test_tenant_provenance_bundle.py +++ b/backend/tests/test_tenant_provenance_bundle.py @@ -204,8 +204,18 @@ def test_parse_round_trips_records_and_verifies_ro_crate_metadata(): assert "prov:SoftwareAgent" in nodes["#naruon"]["@type"] -@pytest.mark.parametrize("date_published", ("not-a-date", "2026-02-30")) -def test_build_rejects_non_iso_ro_crate_date_published(date_published): +@pytest.mark.parametrize( + "date_published", + ( + "not-a-date", + "2026-02-30", + "1980-01-01", + "1980-01-01T09:00:00+09:00", + "1980-01-01T00:00:00+00:00", + "1980-01-01T00:00:00.000000Z", + ), +) +def test_build_rejects_noncanonical_ro_crate_date_published(date_published): with pytest.raises(ProvenanceArchiveError): build_provenance_archive( { @@ -218,6 +228,51 @@ def test_build_rejects_non_iso_ro_crate_date_published(date_published): ) +@pytest.mark.parametrize( + "date_published", + ( + "1980-01-01", + "1980-01-01T09:00:00+09:00", + "1980-01-01T00:00:00+00:00", + "1980-01-01T00:00:00.000000Z", + ), +) +@pytest.mark.asyncio +async def test_import_rejects_noncanonical_activity_timestamp_before_transaction( + date_published, +): + valid_archive = build_provenance_archive(RECORDS) + with zipfile.ZipFile(io.BytesIO(valid_archive), "r") as archive: + entries = {info.filename: archive.read(info) for info in archive.infolist()} + records = copy.deepcopy(RECORDS) + records["export_activity"]["date_published"] = date_published + payload = _canonical_json(records) + crate = json.loads(entries["ro-crate-metadata.json"]) + crate_nodes = {node["@id"]: node for node in crate["@graph"]} + crate_nodes["./"]["datePublished"] = date_published + crate_nodes["data/records.json"]["sha512"] = hashlib.sha512(payload).hexdigest() + entries["data/records.json"] = payload + entries["ro-crate-metadata.json"] = _canonical_json(crate) + invalid_archive = _archive_with_entries(_rebuild_manifests(entries)) + + class NoTransactionSession: + def in_transaction(self): + return False + + def begin(self): + raise AssertionError("transaction must not start") + + async def flush(self): + raise AssertionError("flush must not run") + + with pytest.raises(ProvenanceArchiveError): + await import_tenant_provenance( + NoTransactionSession(), + TenantProvenanceScope("target-user", "target-org", "target-workspace"), + invalid_archive, + ) + + def test_build_preserves_valid_iso_ro_crate_date_published(): archive = build_provenance_archive(RECORDS) diff --git a/docs/adr/0005-tenant-provenance-portability.md b/docs/adr/0005-tenant-provenance-portability.md index c1b9cc01c..d975b04ba 100644 --- a/docs/adr/0005-tenant-provenance-portability.md +++ b/docs/adr/0005-tenant-provenance-portability.md @@ -27,8 +27,8 @@ to the exact workspace and includes only their cited source records. 2. Use stable logical UIDs in the payload. Never serialize or restore sequential database primary keys. Import resolves new keys in foreign-key order. 3. Canonical JSON uses UTF-8, no BOM, no insignificant whitespace, sorted object - keys, pre-sorted set-like arrays, and canonical UTC `Z` email and correction - timestamps. SHA-512 manifests cover exact bytes. + keys, pre-sorted set-like arrays, and canonical UTC `Z` timestamps throughout + the archive. SHA-512 manifests cover exact bytes. 4. Reject unsafe or colliding ZIP paths, unlisted files, checksum mismatch, unsupported profiles, dangling references, scope mismatch, duplicate logical UIDs, and non-finite numbers before mutation. From 66443a98708edf94f1493b3eaedd7b2b3fce6d6c Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Tue, 1 Sep 2026 03:59:21 +0900 Subject: [PATCH 31/39] fix(provenance): remap typed edge endpoints --- backend/services/tenant_provenance_bundle.py | 23 ++++++--- .../tests/test_tenant_provenance_bundle.py | 50 ++++++++++++++++--- 2 files changed, 58 insertions(+), 15 deletions(-) diff --git a/backend/services/tenant_provenance_bundle.py b/backend/services/tenant_provenance_bundle.py index 2f35bdd53..cf4d4dcf9 100644 --- a/backend/services/tenant_provenance_bundle.py +++ b/backend/services/tenant_provenance_bundle.py @@ -441,6 +441,18 @@ def replace_list(record: dict[str, object], field: str, collection: str) -> None maps[collection].get(value, value) for value in record[field] ) + def replace_project_endpoint( + record: dict[str, object], field: str, object_field: str + ) -> None: + value = record[field] + if record[object_field] is not None: + record[field] = maps["project_objects"].get(value, value) + elif isinstance(value, str) and value.startswith("segment:"): + segment_uid = value.removeprefix("segment:") + record[field] = "segment:" + maps["content_segments"].get( + segment_uid, segment_uid + ) + def translate_metadata(value: object) -> object: if isinstance(value, dict): translated_mapping = { @@ -480,13 +492,10 @@ def translate_metadata(value: object) -> object: record["attributes_json"] = translate_metadata(record["attributes_json"]) for record in translated["project_edges"]: replace(record, "edge_uid", "project_edges") - for field in ( - "source_uid", - "target_uid", - "source_object_uid", - "target_object_uid", - ): - replace(record, field, "project_objects") + replace_project_endpoint(record, "source_uid", "source_object_uid") + replace_project_endpoint(record, "target_uid", "target_object_uid") + replace(record, "source_object_uid", "project_objects") + replace(record, "target_object_uid", "project_objects") replace(record, "primary_content_segment_uid", "content_segments") replace_list(record, "source_segment_uids", "content_segments") for record in translated["corrections"]: diff --git a/backend/tests/test_tenant_provenance_bundle.py b/backend/tests/test_tenant_provenance_bundle.py index 54679f3c2..034720de5 100644 --- a/backend/tests/test_tenant_provenance_bundle.py +++ b/backend/tests/test_tenant_provenance_bundle.py @@ -1926,29 +1926,63 @@ async def counting_flush(*args, **kwargs): @pytest.mark.asyncio @pytest.mark.postgres -async def test_import_allows_edge_with_one_nullable_object_endpoint( +async def test_import_round_trip_remaps_nullable_segment_evidence_endpoint( provenance_sessionmaker, ): token = uuid.uuid4().hex[:12] source_scope = _scope(f"nullable-source-{token}") - target_scope = _scope(f"nullable-target-{token}") + target_scope = TenantProvenanceScope( + user_id=source_scope.user_id, + organization_id=source_scope.organization_id, + workspace_id=f"nullable-target-workspace-{token}", + ) + portable_segment_uid = f"segment-{token}" async with provenance_sessionmaker() as session: await _seed_provenance_closure(session, scope=source_scope, token=token) - async with provenance_sessionmaker() as session: - records = parse_provenance_archive( - await export_tenant_provenance(session, source_scope) + source_edge = await session.scalar( + select(ProjectGraphEdgeRecord).where( + ProjectGraphEdgeRecord.workspace_id == source_scope.workspace_id + ) ) - records["project_edges"][0]["source_object_uid"] = None - records["project_edges"][0]["source_uid"] = f"segment-{token}" + source_edge.source_object_id = None + source_edge.source_uid = f"segment:{portable_segment_uid}" + await session.commit() + async with provenance_sessionmaker() as session: + archive = await export_tenant_provenance(session, source_scope) async with provenance_sessionmaker() as session: receipt = await import_tenant_provenance( session, target_scope, - build_provenance_archive(records), + archive, + ) + target_edge = await session.scalar( + select(ProjectGraphEdgeRecord).where( + ProjectGraphEdgeRecord.workspace_id == target_scope.workspace_id + ) + ) + segment_mapping = await session.scalar( + select(ProvenanceIdentityMapping).where( + ProvenanceIdentityMapping.target_workspace_id + == target_scope.workspace_id, + ProvenanceIdentityMapping.entity_kind == "content_segments", + ProvenanceIdentityMapping.portable_uid == portable_segment_uid, + ) + ) + target_source_uid = target_edge.source_uid + target_source_object_id = target_edge.source_object_id + async with provenance_sessionmaker() as session: + reexported = parse_provenance_archive( + await export_tenant_provenance(session, target_scope) ) assert receipt.created["project_edges"] == 1 + assert target_source_object_id is None + assert target_source_uid == f"segment:{segment_mapping.target_database_uid}" + assert reexported["project_edges"][0]["source_object_uid"] is None + assert reexported["project_edges"][0]["source_uid"] == ( + f"segment:{portable_segment_uid}" + ) @pytest.mark.asyncio From 473ab32f65ea92965d67765563853900d10642f5 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Tue, 1 Sep 2026 04:07:37 +0900 Subject: [PATCH 32/39] fix(provenance): validate nullable edge endpoints --- backend/services/tenant_provenance_bundle.py | 60 ++++++++++---- .../tests/test_tenant_provenance_bundle.py | 83 +++++++++++++++++-- 2 files changed, 122 insertions(+), 21 deletions(-) diff --git a/backend/services/tenant_provenance_bundle.py b/backend/services/tenant_provenance_bundle.py index cf4d4dcf9..233677281 100644 --- a/backend/services/tenant_provenance_bundle.py +++ b/backend/services/tenant_provenance_bundle.py @@ -426,6 +426,15 @@ def _canonical_json(value: object) -> bytes: raise ProvenanceArchiveError("Invalid provenance archive") from exc +def _segment_endpoint_uid(value: object) -> str: + if not isinstance(value, str) or not value.startswith("segment:"): + _fail() + segment_uid = value.removeprefix("segment:") + if not segment_uid: + _fail() + return segment_uid + + def _translate_identity_records( records: Mapping[str, object], maps: Mapping[str, Mapping[str, str]] ) -> dict[str, object]: @@ -446,12 +455,16 @@ def replace_project_endpoint( ) -> None: value = record[field] if record[object_field] is not None: - record[field] = maps["project_objects"].get(value, value) - elif isinstance(value, str) and value.startswith("segment:"): - segment_uid = value.removeprefix("segment:") - record[field] = "segment:" + maps["content_segments"].get( - segment_uid, segment_uid - ) + mapped_uid = maps["project_objects"].get(value) + if mapped_uid is None: + _fail() + record[field] = mapped_uid + return + segment_uid = _segment_endpoint_uid(value) + mapped_uid = maps["content_segments"].get(segment_uid) + if mapped_uid is None: + _fail() + record[field] = f"segment:{mapped_uid}" def translate_metadata(value: object) -> object: if isinstance(value, dict): @@ -1194,6 +1207,13 @@ async def export_tenant_provenance( for record in (*project_objects, *project_edges, *corrections) for segment_uid in record.source_segment_uids } + for record in project_edges: + for endpoint_uid, endpoint_object_id in ( + (record.source_uid, record.source_object_id), + (record.target_uid, record.target_object_id), + ): + if endpoint_object_id is None: + cited_segment_uids.add(_segment_endpoint_uid(endpoint_uid)) primary_segment_ids = { record.primary_content_segment_id for record in (*project_objects, *project_edges) @@ -1244,6 +1264,14 @@ async def export_tenant_provenance( if not endpoint_objects: _fail() endpoint_email_ids = {endpoint.email_id for endpoint in endpoint_objects} + for endpoint_uid, endpoint_object_id in ( + (record.source_uid, record.source_object_id), + (record.target_uid, record.target_object_id), + ): + if endpoint_object_id is None: + segment_uid = _segment_endpoint_uid(endpoint_uid) + if segments_by_uid[segment_uid].email_id not in endpoint_email_ids: + _fail() if segments_by_id[ record.primary_content_segment_id ].email_id not in endpoint_email_ids or any( @@ -1961,16 +1989,18 @@ def require_reference(value: object, available: set[str]) -> None: endpoint_email_uids = { object_email[endpoint_uid] for endpoint_uid in endpoint_uids } - if ( - record["source_object_uid"] is not None - and record["source_uid"] != record["source_object_uid"] + for field, object_field in ( + ("source_uid", "source_object_uid"), + ("target_uid", "target_object_uid"), ): - _fail() - if ( - record["target_object_uid"] is not None - and record["target_uid"] != record["target_object_uid"] - ): - _fail() + if record[object_field] is not None: + if record[field] != record[object_field]: + _fail() + continue + segment_uid = _segment_endpoint_uid(record[field]) + require_reference(segment_uid, segment_uids) + if segment_email[segment_uid] not in endpoint_email_uids: + _fail() require_reference(record["primary_content_segment_uid"], segment_uids) if ( segment_email[record["primary_content_segment_uid"]] diff --git a/backend/tests/test_tenant_provenance_bundle.py b/backend/tests/test_tenant_provenance_bundle.py index 034720de5..6591a28dc 100644 --- a/backend/tests/test_tenant_provenance_bundle.py +++ b/backend/tests/test_tenant_provenance_bundle.py @@ -1985,6 +1985,64 @@ async def test_import_round_trip_remaps_nullable_segment_evidence_endpoint( ) +@pytest.mark.parametrize("endpoint", ("source", "target")) +@pytest.mark.parametrize( + "invalid_kind", ("empty", "bare", "unknown", "object_uid", "cross_email") +) +@pytest.mark.asyncio +@pytest.mark.postgres +async def test_import_rejects_invalid_nullable_segment_endpoint_before_flush( + provenance_sessionmaker, + monkeypatch, + endpoint, + invalid_kind, +): + token = uuid.uuid4().hex[:12] + source_scope = _scope(f"nullable-invalid-source-{token}") + target_scope = _scope(f"nullable-invalid-target-{token}") + async with provenance_sessionmaker() as session: + archive = await _two_email_rooted_archive( + session, scope=source_scope, token=token + ) + records = parse_provenance_archive(archive) + edge = records["project_edges"][0] + anchor_object_uid = edge[f"{'target' if endpoint == 'source' else 'source'}_object_uid"] + object_email = { + record["object_uid"]: record["email_uid"] + for record in records["project_objects"] + } + anchor_email_uid = object_email[anchor_object_uid] + cross_email_segment_uid = next( + record["content_segment_uid"] + for record in records["content_segments"] + if record["email_uid"] != anchor_email_uid + ) + invalid_values = { + "empty": "segment:", + "bare": edge["source_segment_uids"][0], + "unknown": f"segment:unknown-{token}", + "object_uid": edge["source_object_uid"], + "cross_email": f"segment:{cross_email_segment_uid}", + } + edge[f"{endpoint}_object_uid"] = None + edge[f"{endpoint}_uid"] = invalid_values[invalid_kind] + invalid_archive = build_provenance_archive(records) + + async with provenance_sessionmaker() as session: + flush_count = 0 + original_flush = session.flush + + async def counting_flush(*args, **kwargs): + nonlocal flush_count + flush_count += 1 + return await original_flush(*args, **kwargs) + + monkeypatch.setattr(session, "flush", counting_flush) + with pytest.raises(ProvenanceArchiveError): + await import_tenant_provenance(session, target_scope, invalid_archive) + assert flush_count == 0 + + @pytest.mark.asyncio @pytest.mark.postgres async def test_confidence_unit_interval_boundaries_round_trip( @@ -2756,11 +2814,16 @@ async def test_cross_field_conflict_rejected_before_flush( for record in records["attachments"] if record["email_uid"] != source_email_uid ) + source_project_object = next( + record + for record in records["project_objects"] + if record["email_uid"] == source_email_uid + ) same_email_attachment_uid = next( record["attachment_uid"] for record in records["attachments"] if record["email_uid"] == source_email_uid - and record["attachment_uid"] != records["project_objects"][0]["attachment_uid"] + and record["attachment_uid"] != source_project_object["attachment_uid"] ) if conflict_kind == "structural_attachment_email": source_structural_edge = next( @@ -2770,9 +2833,9 @@ async def test_cross_field_conflict_rejected_before_flush( ) source_structural_edge["attachment_uid"] = foreign_attachment_uid elif conflict_kind == "object_attachment_email": - records["project_objects"][0]["attachment_uid"] = foreign_attachment_uid + source_project_object["attachment_uid"] = foreign_attachment_uid elif conflict_kind == "object_attachment_primary": - records["project_objects"][0]["attachment_uid"] = same_email_attachment_uid + source_project_object["attachment_uid"] = same_email_attachment_uid elif conflict_kind == "edge_source_endpoint": records["project_edges"][0]["source_uid"] = "mismatched-logical-endpoint" else: @@ -2896,7 +2959,7 @@ async def test_export_allows_segment_evidence_edge_with_nullable_source_object( ProjectGraphEdgeRecord.workspace_id == scope.workspace_id ) ) - edge.source_uid = f"segment-{token}" + edge.source_uid = f"segment:segment-{token}" edge.source_object_id = None await session.commit() @@ -2906,7 +2969,7 @@ async def test_export_allows_segment_evidence_edge_with_nullable_source_object( ) exported_edge = records["project_edges"][0] - assert exported_edge["source_uid"] == f"segment-{token}" + assert exported_edge["source_uid"] == f"segment:segment-{token}" assert exported_edge["source_object_uid"] is None assert exported_edge["target_object_uid"] in selected["object_uids"] @@ -2923,6 +2986,8 @@ async def test_export_allows_segment_evidence_edge_with_nullable_source_object( "edge_target_object", "correction_object", "edge_without_object_anchors", + "edge_source_segment_endpoint", + "edge_target_segment_endpoint", ), ) @pytest.mark.asyncio @@ -2969,7 +3034,13 @@ async def test_export_rejects_cross_workspace_segment_references_before_email_cl ProjectGraphCorrectionRecord.workspace_id == scope.workspace_id ) ) - if reference_field == "edge_without_object_anchors": + if reference_field == "edge_source_segment_endpoint": + record.source_object_id = None + record.source_uid = f"segment:{foreign_segment.content_segment_uid}" + elif reference_field == "edge_target_segment_endpoint": + record.target_object_id = None + record.target_uid = f"segment:{foreign_segment.content_segment_uid}" + elif reference_field == "edge_without_object_anchors": record.source_object_id = None record.target_object_id = None elif reference_field == "edge_source_object": From 8ae8bd5929710a371f60f132ccfd26486a07c87a Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Tue, 1 Sep 2026 04:25:43 +0900 Subject: [PATCH 33/39] fix: bind provenance bundle identities to content --- backend/services/tenant_provenance_bundle.py | 47 +++++++++++++++++-- .../tests/test_tenant_provenance_bundle.py | 28 +++++++++-- .../adr/0005-tenant-provenance-portability.md | 3 ++ 3 files changed, 71 insertions(+), 7 deletions(-) diff --git a/backend/services/tenant_provenance_bundle.py b/backend/services/tenant_provenance_bundle.py index 233677281..5f4ba26e4 100644 --- a/backend/services/tenant_provenance_bundle.py +++ b/backend/services/tenant_provenance_bundle.py @@ -533,6 +533,32 @@ def _records_bundle_uid(records: Mapping[str, object]) -> str: return _safe_identifier(records.get("bundle_uid")) +def _content_identity_digest(records: Mapping[str, object]) -> str: + activity = records.get("export_activity") + if not isinstance(activity, Mapping): + _fail() + identity = { + "profile": records.get("profile"), + "schema_version": records.get("schema_version"), + "source_scope": records.get("source_scope"), + **{collection: records.get(collection) for collection in _COLLECTIONS}, + "export_activity": {"date_published": activity.get("date_published")}, + } + return hashlib.sha256(_canonical_json(identity)).hexdigest() + + +def _validate_content_bound_identifiers(records: Mapping[str, object]) -> str: + content_digest = _content_identity_digest(records) + if _records_bundle_uid(records) != f"bundle-{content_digest}": + _fail() + activity = records.get("export_activity") + if not isinstance(activity, Mapping) or _safe_identifier( + activity.get("activity_uid") + ) != f"export-{content_digest}": + _fail() + return content_digest + + def _safe_identifier(value: object) -> str: if ( not isinstance(value, str) @@ -616,6 +642,7 @@ def _manifest(entries: Mapping[str, bytes], names: tuple[str, ...]) -> bytes: def _archive_entries(records: Mapping[str, object]) -> dict[str, bytes]: payload = _canonical_json(records) + _validate_content_bound_identifiers(records) bundle_uid = _records_bundle_uid(records) entries = { "bagit.txt": b"BagIt-Version: 1.0\nTag-File-Character-Encoding: UTF-8\n", @@ -653,7 +680,19 @@ def build_provenance_archive(records: Mapping[str, object]) -> bytes: """Build the fixed deterministic ZIP envelope for a validated record payload.""" if not isinstance(records, Mapping): _fail() - entries = _archive_entries(records) + activity = records.get("export_activity") + if not isinstance(activity, Mapping): + _fail() + _records_bundle_uid(records) + _safe_identifier(activity.get("activity_uid")) + bound_records = { + **records, + "export_activity": dict(activity), + } + content_digest = _content_identity_digest(bound_records) + bound_records["bundle_uid"] = f"bundle-{content_digest}" + bound_records["export_activity"]["activity_uid"] = f"export-{content_digest}" + entries = _archive_entries(bound_records) output = io.BytesIO() with zipfile.ZipFile( output, "w", compression=zipfile.ZIP_DEFLATED, compresslevel=9 @@ -1535,17 +1574,17 @@ async def descendants(model: Any, order_column: Any) -> list[Any]: } if can_restore_source_identity: payload = _translate_identity_records(payload, reverse_maps) - content_digest = hashlib.sha256(_canonical_json(payload)).hexdigest() records = { "profile": "naruon-tenant-provenance/v1", "schema_version": 1, - "bundle_uid": f"bundle-{content_digest}", **payload, "export_activity": { - "activity_uid": f"export-{content_digest}", "date_published": "1980-01-01T00:00:00Z", }, } + content_digest = _content_identity_digest(records) + records["bundle_uid"] = f"bundle-{content_digest}" + records["export_activity"]["activity_uid"] = f"export-{content_digest}" _validate_record_graph(records) return build_provenance_archive(records) diff --git a/backend/tests/test_tenant_provenance_bundle.py b/backend/tests/test_tenant_provenance_bundle.py index 6591a28dc..656abc04c 100644 --- a/backend/tests/test_tenant_provenance_bundle.py +++ b/backend/tests/test_tenant_provenance_bundle.py @@ -56,14 +56,14 @@ RECORDS = { "profile": "naruon-tenant-provenance/v1", "schema_version": 1, - "bundle_uid": "bundle-01HZZ", + "bundle_uid": "bundle-d046d1ed5d46714042ed93ee835abe93db3d56e548440e6b0eff69e5a808fc74", "source_scope": { "user_uid": "0" * 64, "organization_uid": "org-01", "workspace_uid": "ws-01", }, "export_activity": { - "activity_uid": "activity-01", + "activity_uid": "export-d046d1ed5d46714042ed93ee835abe93db3d56e548440e6b0eff69e5a808fc74", "date_published": "1980-01-01T00:00:00Z", }, "emails": [{"email_uid": "email-01", "subject": "Evidence"}], @@ -200,7 +200,8 @@ def test_parse_round_trips_records_and_verifies_ro_crate_metadata(): "conformsTo": {"@id": "https://w3id.org/ro/crate/1.3"}, } assert nodes["./"]["datePublished"] == RECORDS["export_activity"]["date_published"] - assert nodes["#activity-01"]["prov:wasAssociatedWith"] == {"@id": "#naruon"} + activity_uid = RECORDS["export_activity"]["activity_uid"] + assert nodes[f"#{activity_uid}"]["prov:wasAssociatedWith"] == {"@id": "#naruon"} assert "prov:SoftwareAgent" in nodes["#naruon"]["@type"] @@ -290,6 +291,27 @@ def test_parse_rejects_payload_tampering(): parse_provenance_archive(tampered) +def test_archive_entries_reject_reused_content_identifiers_for_different_records(): + rebuilt_records = copy.deepcopy(RECORDS) + rebuilt_records["emails"][0]["subject"] = "Different evidence" + + with pytest.raises(ProvenanceArchiveError): + provenance_service._archive_entries(rebuilt_records) + + +def test_build_rebinds_content_identifiers_for_different_records(): + rebuilt_records = copy.deepcopy(RECORDS) + rebuilt_records["emails"][0]["subject"] = "Different evidence" + + parsed = parse_provenance_archive(build_provenance_archive(rebuilt_records)) + + assert parsed["bundle_uid"] != RECORDS["bundle_uid"] + assert ( + parsed["export_activity"]["activity_uid"] + != RECORDS["export_activity"]["activity_uid"] + ) + + @pytest.mark.parametrize( "name", ("manifest-sha512.txt", "tagmanifest-sha512.txt", "ro-crate-metadata.json"), diff --git a/docs/adr/0005-tenant-provenance-portability.md b/docs/adr/0005-tenant-provenance-portability.md index d975b04ba..7f05e16c1 100644 --- a/docs/adr/0005-tenant-provenance-portability.md +++ b/docs/adr/0005-tenant-provenance-portability.md @@ -29,6 +29,9 @@ to the exact workspace and includes only their cited source records. 3. Canonical JSON uses UTF-8, no BOM, no insignificant whitespace, sorted object keys, pre-sorted set-like arrays, and canonical UTC `Z` timestamps throughout the archive. SHA-512 manifests cover exact bytes. + `bundle_uid` and the export PROV activity UID are SHA-256 identities over the + complete canonical record content except those two derived UID fields, so a + rebuilt archive cannot reuse either identity for different records. 4. Reject unsafe or colliding ZIP paths, unlisted files, checksum mismatch, unsupported profiles, dangling references, scope mismatch, duplicate logical UIDs, and non-finite numbers before mutation. From c7ca74d951536f27b3cec2e532a13f2e9f3ec57e Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Tue, 1 Sep 2026 04:27:27 +0900 Subject: [PATCH 34/39] fix(provenance): bound export row loading --- backend/services/tenant_provenance_bundle.py | 134 ++++++++++++++++-- .../tests/test_tenant_provenance_bundle.py | 50 +++++++ 2 files changed, 173 insertions(+), 11 deletions(-) diff --git a/backend/services/tenant_provenance_bundle.py b/backend/services/tenant_provenance_bundle.py index 5f4ba26e4..ea1b2e1c3 100644 --- a/backend/services/tenant_provenance_bundle.py +++ b/backend/services/tenant_provenance_bundle.py @@ -16,7 +16,7 @@ from datetime import datetime, timezone from typing import Any -from sqlalchemy import func, select +from sqlalchemy import Text, cast, func, select from sqlalchemy.exc import DataError, IntegrityError, StatementError from sqlalchemy.ext.asyncio import AsyncSession @@ -45,6 +45,58 @@ _LOCAL_FILE_SIGNATURE = b"PK\x03\x04" _EOCD_SIGNATURE = b"PK\x05\x06" _EOCD_SIZE = 22 + +_EXPORT_TEXT_COLUMNS = { + Email: ( + Email.message_id, + Email.thread_id, + Email.fingerprint, + Email.sender, + Email.reply_to, + Email.recipients, + Email.subject, + Email.in_reply_to, + Email.references, + Email.body, + ), + Attachment: ( + Attachment.filename, + Attachment.content, + Attachment.content_type, + Attachment.parse_status, + Attachment.parse_content_type, + Attachment.parser_key, + Attachment.parse_error_code, + ), + ContentNodeRecord: ( + ContentNodeRecord.source_record_uid, + ContentNodeRecord.parent_node_uid, + ContentNodeRecord.display_label, + ContentNodeRecord.safe_text_content, + ), + ContentSegmentRecord: ( + ContentSegmentRecord.source_record_uid, + ContentSegmentRecord.heading_path, + ContentSegmentRecord.safe_text_content, + ), + KnowledgeGraphEdgeRecord: ( + KnowledgeGraphEdgeRecord.source_record_uid, + KnowledgeGraphEdgeRecord.edge_path, + ), + ProjectGraphObjectRecord: ( + ProjectGraphObjectRecord.title, + ProjectGraphObjectRecord.summary, + ProjectGraphObjectRecord.source_segment_uids, + ProjectGraphObjectRecord.attributes_json, + ), + ProjectGraphEdgeRecord: (ProjectGraphEdgeRecord.source_segment_uids,), + ProjectGraphCorrectionRecord: ( + ProjectGraphCorrectionRecord.before_json, + ProjectGraphCorrectionRecord.after_json, + ProjectGraphCorrectionRecord.rationale, + ProjectGraphCorrectionRecord.source_segment_uids, + ), +} _EXPECTED_ENTRIES = frozenset( { "bagit.txt", @@ -920,6 +972,35 @@ def _scope_filters(model: Any, scope: TenantProvenanceScope, *, workspace: bool) return filters +async def _preflight_export_rows( + session: AsyncSession, + model: Any, + filters: tuple[Any, ...] | list[Any], + consumed_bytes: int, +) -> int: + bind = session.get_bind() + if getattr(getattr(bind, "dialect", None), "name", None) != "postgresql": + return consumed_bytes + text_bytes = sum( + ( + func.coalesce(func.octet_length(cast(column, Text)), 0) + for column in _EXPORT_TEXT_COLUMNS[model] + ), + start=0, + ) + row_bytes = func.pg_column_size(model.__table__.table_valued()) + text_bytes + stored_bytes = int( + await session.scalar( + select(func.coalesce(func.sum(row_bytes), 0)).where(*filters) + ) + or 0 + ) + total_bytes = consumed_bytes + stored_bytes + if total_bytes > ENTRY_MAX_BYTES: + _fail() + return total_bytes + + def _utc_text(value: datetime) -> str: if value.tzinfo is None: _fail() @@ -1212,6 +1293,18 @@ async def export_tenant_provenance( ) -> bytes: """Export the exact signed-scope project-evidence closure.""" _validate_scope(scope) + consumed_bytes = 0 + for model in ( + ProjectGraphObjectRecord, + ProjectGraphEdgeRecord, + ProjectGraphCorrectionRecord, + ): + consumed_bytes = await _preflight_export_rows( + session, + model, + _scope_filters(model, scope, workspace=True), + consumed_bytes, + ) project_objects = list( ( await session.scalars( @@ -1257,16 +1350,26 @@ async def export_tenant_provenance( record.primary_content_segment_id for record in (*project_objects, *project_edges) } + cited_size_filters = ( + (ContentSegmentRecord.content_segment_uid.in_(cited_segment_uids)) + | (ContentSegmentRecord.content_segment_id.in_(primary_segment_ids)), + ) + cited_segment_filters = ( + *cited_size_filters, + *_scope_filters(Email, scope, workspace=False), + ) + consumed_bytes = await _preflight_export_rows( + session, + ContentSegmentRecord, + cited_size_filters, + consumed_bytes, + ) cited_segments = list( ( await session.scalars( select(ContentSegmentRecord) .join(Email, ContentSegmentRecord.email_id == Email.id) - .where( - (ContentSegmentRecord.content_segment_uid.in_(cited_segment_uids)) - | (ContentSegmentRecord.content_segment_id.in_(primary_segment_ids)), - *_scope_filters(Email, scope, workspace=False), - ) + .where(*cited_segment_filters) ) ).all() ) @@ -1328,14 +1431,18 @@ async def export_tenant_provenance( email_ids = {record.email_id for record in project_objects} | { segment.email_id for segment in cited_segments } + email_filters = ( + Email.id.in_(email_ids), + *_scope_filters(Email, scope, workspace=False), + ) + consumed_bytes = await _preflight_export_rows( + session, Email, email_filters, consumed_bytes + ) emails = list( ( await session.scalars( select(Email) - .where( - Email.id.in_(email_ids), - *_scope_filters(Email, scope, workspace=False), - ) + .where(*email_filters) .order_by(Email.message_id) ) ).all() @@ -1345,11 +1452,16 @@ async def export_tenant_provenance( email_uids = {email.id: email.message_id for email in emails} async def descendants(model: Any, order_column: Any) -> list[Any]: + nonlocal consumed_bytes + descendant_filters = (model.email_id.in_(email_ids),) + consumed_bytes = await _preflight_export_rows( + session, model, descendant_filters, consumed_bytes + ) return list( ( await session.scalars( select(model) - .where(model.email_id.in_(email_ids)) + .where(*descendant_filters) .order_by(order_column) ) ).all() diff --git a/backend/tests/test_tenant_provenance_bundle.py b/backend/tests/test_tenant_provenance_bundle.py index 656abc04c..d2b6c9124 100644 --- a/backend/tests/test_tenant_provenance_bundle.py +++ b/backend/tests/test_tenant_provenance_bundle.py @@ -627,6 +627,23 @@ async def provenance_sessionmaker(): await engine.dispose() +async def _require_direct_postgres(engine) -> None: + try: + async with engine.connect() as connection: + await connection.execute(text("SELECT 1")) + except ( + ConnectionRefusedError, + OSError, + OperationalError, + asyncpg.CannotConnectNowError, + asyncpg.InvalidAuthorizationSpecificationError, + asyncpg.InvalidCatalogNameError, + asyncpg.InvalidPasswordError, + ): + await engine.dispose() + pytest.skip("PostgreSQL smoke path unavailable") + + async def _seed_provenance_closure( session, *, @@ -1078,6 +1095,37 @@ async def test_postgres_round_trip_preserves_stable_evidence_with_fresh_keys( } +@pytest.mark.asyncio +@pytest.mark.postgres +async def test_export_rejects_oversized_stored_text_before_descendant_orm_load( + provenance_sessionmaker, + monkeypatch, +): + token = uuid.uuid4().hex[:12] + scope = _scope(f"oversized-export-{token}") + async with provenance_sessionmaker() as session: + seeded = await _seed_provenance_closure(session, scope=scope, token=token) + attachment = await session.get(Attachment, seeded["attachment_id"]) + attachment.content = "x" * (ENTRY_MAX_BYTES + 1) + await session.commit() + + async with provenance_sessionmaker() as session: + original_scalars = session.scalars + + async def reject_attachment_orm_load(statement, *args, **kwargs): + entities = { + description.get("entity") + for description in statement.column_descriptions + } + if Attachment in entities: + raise AssertionError("oversized attachment ORM row was materialized") + return await original_scalars(statement, *args, **kwargs) + + monkeypatch.setattr(session, "scalars", reject_attachment_orm_load) + with pytest.raises(ProvenanceArchiveError): + await export_tenant_provenance(session, scope) + + @pytest.mark.asyncio @pytest.mark.postgres async def test_import_composes_with_transaction_started_by_prior_select( @@ -1466,6 +1514,7 @@ async def flush(self): @pytest.mark.postgres async def test_concurrent_identical_same_database_imports_are_idempotent(): engine = create_async_engine(settings.DATABASE_URL, echo=False) + await _require_direct_postgres(engine) session_factory = async_sessionmaker(engine, expire_on_commit=False) token = uuid.uuid4().hex[:12] source_scope = _scope(f"concurrent-source-{token}") @@ -1554,6 +1603,7 @@ async def run_import(): @pytest.mark.postgres async def test_concurrent_imports_across_workspaces_reuse_owner_email(): engine = create_async_engine(settings.DATABASE_URL, echo=False) + await _require_direct_postgres(engine) session_factory = async_sessionmaker(engine, expire_on_commit=False) token = uuid.uuid4().hex[:12] source_scope = _scope(f"parallel-workspaces-source-{token}") From 053b40600e2b7fec4f2e6198f5c6775239648980 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Tue, 1 Sep 2026 06:05:51 +0900 Subject: [PATCH 35/39] fix(provenance): preserve portable import origins --- backend/services/tenant_provenance_bundle.py | 87 +++++++++++--- .../tests/test_tenant_provenance_bundle.py | 112 ++++++++++++++++++ 2 files changed, 181 insertions(+), 18 deletions(-) diff --git a/backend/services/tenant_provenance_bundle.py b/backend/services/tenant_provenance_bundle.py index ea1b2e1c3..a70a2b027 100644 --- a/backend/services/tenant_provenance_bundle.py +++ b/backend/services/tenant_provenance_bundle.py @@ -1451,9 +1451,11 @@ async def export_tenant_provenance( _fail() email_uids = {email.id: email.message_id for email in emails} - async def descendants(model: Any, order_column: Any) -> list[Any]: + async def descendants( + model: Any, order_column: Any, *admission_filters: Any + ) -> list[Any]: nonlocal consumed_bytes - descendant_filters = (model.email_id.in_(email_ids),) + descendant_filters = (model.email_id.in_(email_ids), *admission_filters) consumed_bytes = await _preflight_export_rows( session, model, descendant_filters, consumed_bytes ) @@ -1467,7 +1469,12 @@ async def descendants(model: Any, order_column: Any) -> list[Any]: ).all() ) - attachments = await descendants(Attachment, Attachment.id) + attachments = await descendants( + Attachment, + Attachment.id, + Attachment.parse_status == "parsed", + Attachment.parser_key.in_(_TEXTUAL_PARSER_KEYS), + ) nodes = await descendants(ContentNodeRecord, ContentNodeRecord.content_node_uid) segments = await descendants( ContentSegmentRecord, ContentSegmentRecord.content_segment_uid @@ -2261,11 +2268,15 @@ async def _prepare_identity_import( if set(scoped_mappings) != expected_mapping_keys: _fail() for (collection, portable_uid), row in scoped_mappings.items(): - if row.target_database_uid != forward_maps[collection][portable_uid]: + if row.target_database_uid not in { + portable_uid, + forward_maps[collection][portable_uid], + }: _fail() + forward_maps[collection][portable_uid] = row.target_database_uid return _translate_identity_records(records, forward_maps), [] - scoped_collisions: list[Any] = [] + scoped_collisions: list[tuple[str, Any, str]] = [] for model, column, collection in ( ( ProjectGraphObjectRecord, @@ -2284,28 +2295,68 @@ async def _prepare_identity_import( } if portable_uids: scoped_collisions.extend( - ( - await session.scalars( - select(model).where(column.in_(portable_uids)) - ) + (collection, row, getattr(row, _UID_KEYS[collection])) + for row in ( + await session.scalars(select(model).where(column.in_(portable_uids))) ).all() ) if not scoped_collisions: - return copy.deepcopy(records), [] + return copy.deepcopy(records), [ + ProvenanceIdentityMapping( + target_user_id=scope.user_id, + target_organization_id=scope.organization_id, + target_workspace_id=scope.workspace_id, + source_user_uid=source_scope["user_uid"], + source_organization_uid=source_scope["organization_uid"], + source_workspace_uid=source_scope["workspace_uid"], + entity_kind=collection, + portable_uid=portable_uid, + target_database_uid=portable_uid, + ) + for collection in _REMAPPED_COLLECTIONS + for portable_uid in forward_maps[collection] + ] if any( row.user_id == scope.user_id and row.organization_id == scope.organization_id and row.workspace_id == scope.workspace_id - for row in scoped_collisions + for _, row, _ in scoped_collisions ): return copy.deepcopy(records), [] - if any( - _source_user_uid(row.user_id) != source_scope["user_uid"] - or (row.organization_id or "unscoped") != source_scope["organization_uid"] - or row.workspace_id != source_scope["workspace_uid"] - for row in scoped_collisions - ): - _fail() + native_source = all( + _source_user_uid(row.user_id) == source_scope["user_uid"] + and (row.organization_id or "unscoped") == source_scope["organization_uid"] + and row.workspace_id == source_scope["workspace_uid"] + for _, row, _ in scoped_collisions + ) + if not native_source: + collision_keys = { + (collection, portable_uid) + for collection, _, portable_uid in scoped_collisions + } + origin_rows = list( + ( + await session.scalars( + select(ProvenanceIdentityMapping).where( + ProvenanceIdentityMapping.target_database_uid.in_( + {portable_uid for _, portable_uid in collision_keys} + ) + ) + ) + ).all() + ) + origins = { + (row.entity_kind, row.target_database_uid): row + for row in origin_rows + if (row.entity_kind, row.target_database_uid) in collision_keys + } + if set(origins) != collision_keys or any( + row.source_user_uid != source_scope["user_uid"] + or row.source_organization_uid != source_scope["organization_uid"] + or row.source_workspace_uid != source_scope["workspace_uid"] + for row in origins.values() + ): + _fail() target_uids = { database_uid diff --git a/backend/tests/test_tenant_provenance_bundle.py b/backend/tests/test_tenant_provenance_bundle.py index d2b6c9124..d3d23073e 100644 --- a/backend/tests/test_tenant_provenance_bundle.py +++ b/backend/tests/test_tenant_provenance_bundle.py @@ -1126,6 +1126,38 @@ async def reject_attachment_orm_load(statement, *args, **kwargs): await export_tenant_provenance(session, scope) +@pytest.mark.asyncio +@pytest.mark.postgres +async def test_export_ignores_oversized_attachment_excluded_from_archive( + provenance_sessionmaker, +): + token = uuid.uuid4().hex[:12] + scope = _scope(f"excluded-oversized-{token}") + async with provenance_sessionmaker() as session: + seeded = await _seed_provenance_closure(session, scope=scope, token=token) + session.add( + Attachment( + email_id=seeded["email_id"], + filename="pending.bin", + content="x" * (ENTRY_MAX_BYTES + 1), + content_type="application/octet-stream", + parse_status="unsupported_content_type", + parser_key="unsupported_binary", + embedding=[0.0] * 1536, + ) + ) + await session.commit() + + async with provenance_sessionmaker() as session: + records = parse_provenance_archive( + await export_tenant_provenance(session, scope) + ) + + assert [row["filename"] for row in records["attachments"]] == [ + "evidence.txt" + ] + + @pytest.mark.asyncio @pytest.mark.postgres async def test_import_composes_with_transaction_started_by_prior_select( @@ -1397,6 +1429,86 @@ async def test_same_database_cross_workspace_import_keeps_portable_identity( } +@pytest.mark.asyncio +@pytest.mark.postgres +async def test_portable_archive_imports_into_two_scopes_after_source_deletion( + provenance_sessionmaker, +): + token = uuid.uuid4().hex[:12] + source_scope = _scope(f"deleted-source-{token}") + target_a = _scope(f"portable-target-a-{token}") + target_b = _scope(f"portable-target-b-{token}") + async with provenance_sessionmaker() as session: + await _seed_provenance_closure(session, scope=source_scope, token=token) + async with provenance_sessionmaker() as session: + archive = await export_tenant_provenance(session, source_scope) + source_records = parse_provenance_archive(archive) + + async with provenance_sessionmaker() as session: + email_id = await session.scalar( + select(Email.id).where(Email.message_id == source_records["emails"][0]["email_uid"]) + ) + assert email_id is not None + for model in ( + ProjectGraphCorrectionRecord, + ProjectGraphEdgeRecord, + ProjectGraphObjectRecord, + ): + await session.execute( + delete(model).where(model.workspace_id == source_scope.workspace_id) + ) + for model in ( + KnowledgeGraphEdgeRecord, + ContentSegmentRecord, + ContentNodeRecord, + Attachment, + ): + await session.execute(delete(model).where(model.email_id == email_id)) + await session.execute(delete(Email).where(Email.id == email_id)) + await session.commit() + + async with provenance_sessionmaker() as session: + await import_tenant_provenance(session, target_a, archive) + async with provenance_sessionmaker() as session: + await import_tenant_provenance(session, target_b, archive) + + async with provenance_sessionmaker() as session: + target_a_records = parse_provenance_archive( + await export_tenant_provenance(session, target_a) + ) + target_b_records = parse_provenance_archive( + await export_tenant_provenance(session, target_b) + ) + mappings = list( + ( + await session.scalars( + select(ProvenanceIdentityMapping).where( + ProvenanceIdentityMapping.target_workspace_id.in_( + (target_a.workspace_id, target_b.workspace_id) + ) + ) + ) + ).all() + ) + + for collection in ( + "emails", + "attachments", + "content_nodes", + "content_segments", + "structural_edges", + "project_objects", + "project_edges", + "corrections", + ): + assert target_a_records[collection] == source_records[collection] + assert target_b_records[collection] == source_records[collection] + assert {row.target_workspace_id for row in mappings} == { + target_a.workspace_id, + target_b.workspace_id, + } + + @pytest.mark.asyncio @pytest.mark.postgres async def test_mixed_native_and_multiple_import_origins_export_as_target_scope( From 69ab128bafcc30efc826e592dd71fc45caa663ff Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Tue, 1 Sep 2026 08:41:21 +0900 Subject: [PATCH 36/39] fix(provenance): serialize portable identity imports --- backend/services/tenant_provenance_bundle.py | 42 ++++++- backend/tests/test_bootstrap_db.py | 10 ++ .../tests/test_tenant_provenance_bundle.py | 104 ++++++++++++++++++ 3 files changed, 154 insertions(+), 2 deletions(-) diff --git a/backend/services/tenant_provenance_bundle.py b/backend/services/tenant_provenance_bundle.py index a70a2b027..4f1112dd6 100644 --- a/backend/services/tenant_provenance_bundle.py +++ b/backend/services/tenant_provenance_bundle.py @@ -1476,8 +1476,25 @@ async def descendants( Attachment.parser_key.in_(_TEXTUAL_PARSER_KEYS), ) nodes = await descendants(ContentNodeRecord, ContentNodeRecord.content_node_uid) - segments = await descendants( - ContentSegmentRecord, ContentSegmentRecord.content_segment_uid + consumed_bytes = await _preflight_export_rows( + session, + ContentSegmentRecord, + ( + ContentSegmentRecord.email_id.in_(email_ids), + ContentSegmentRecord.content_segment_id.not_in( + {segment.content_segment_id for segment in cited_segments} + ), + ), + consumed_bytes, + ) + segments = list( + ( + await session.scalars( + select(ContentSegmentRecord) + .where(ContentSegmentRecord.email_id.in_(email_ids)) + .order_by(ContentSegmentRecord.content_segment_uid) + ) + ).all() ) structural_edges = await descendants( KnowledgeGraphEdgeRecord, KnowledgeGraphEdgeRecord.edge_uid @@ -2837,6 +2854,27 @@ async def import_tenant_provenance( async with transaction: bind = session.get_bind() if getattr(getattr(bind, "dialect", None), "name", None) == "postgresql": + identity_lock_digest = hashlib.sha256( + _canonical_json( + { + "namespace": "tenant-provenance-portable-identities-v1", + "identities": { + collection: sorted( + record[_UID_KEYS[collection]] + for record in records[collection] + ) + for collection in _REMAPPED_COLLECTIONS + }, + } + ) + ).digest() + await session.execute( + select( + func.pg_advisory_xact_lock( + int.from_bytes(identity_lock_digest[:8], "big", signed=True) + ) + ) + ) for email_uid in sorted( record["email_uid"] for record in records["emails"] ): diff --git a/backend/tests/test_bootstrap_db.py b/backend/tests/test_bootstrap_db.py index 47c8e88e0..b8d5cbc02 100644 --- a/backend/tests/test_bootstrap_db.py +++ b/backend/tests/test_bootstrap_db.py @@ -764,6 +764,16 @@ async def test_schema_backfill_creates_legacy_emails_index_when_table_exists(): ) ) assert result.scalar_one() == "ix_emails_owner_date" + except ( + ConnectionRefusedError, + OSError, + OperationalError, + asyncpg.CannotConnectNowError, + asyncpg.InvalidAuthorizationSpecificationError, + asyncpg.InvalidCatalogNameError, + asyncpg.InvalidPasswordError, + ): + pytest.skip("PostgreSQL smoke path unavailable") finally: await engine.dispose() diff --git a/backend/tests/test_tenant_provenance_bundle.py b/backend/tests/test_tenant_provenance_bundle.py index d3d23073e..c3cf310e8 100644 --- a/backend/tests/test_tenant_provenance_bundle.py +++ b/backend/tests/test_tenant_provenance_bundle.py @@ -5,6 +5,7 @@ import importlib.util import io import json +import random import struct import uuid import warnings @@ -1158,6 +1159,29 @@ async def test_export_ignores_oversized_attachment_excluded_from_archive( ] +@pytest.mark.asyncio +@pytest.mark.postgres +async def test_export_counts_cited_segment_bytes_once(provenance_sessionmaker): + token = uuid.uuid4().hex[:12] + scope = _scope(f"large-cited-segment-{token}") + large_text = random.Random(0).randbytes( + ENTRY_MAX_BYTES // 8 + 2048 + ).hex() + async with provenance_sessionmaker() as session: + seeded = await _seed_provenance_closure(session, scope=scope, token=token) + segment = await session.get(ContentSegmentRecord, seeded["segment_id"]) + segment.safe_text_content = large_text + segment.content_hash = hashlib.sha256(large_text.encode()).hexdigest() + await session.commit() + + async with provenance_sessionmaker() as session: + records = parse_provenance_archive( + await export_tenant_provenance(session, scope) + ) + + assert records["content_segments"][0]["safe_text_content"] == large_text + + @pytest.mark.asyncio @pytest.mark.postgres async def test_import_composes_with_transaction_started_by_prior_select( @@ -1711,6 +1735,86 @@ async def run_import(): await engine.dispose() +@pytest.mark.asyncio +@pytest.mark.postgres +async def test_concurrent_cross_tenant_imports_remap_global_identities(): + engine = create_async_engine(settings.DATABASE_URL, echo=False) + await _require_direct_postgres(engine) + session_factory = async_sessionmaker(engine, expire_on_commit=False) + token = uuid.uuid4().hex[:12] + source_scope = _scope(f"concurrent-portable-source-{token}") + target_scopes = ( + _scope(f"concurrent-portable-target-a-{token}"), + _scope(f"concurrent-portable-target-b-{token}"), + ) + try: + async with engine.begin() as connection: + await connection.run_sync(Base.metadata.create_all) + async with session_factory() as session: + await _seed_provenance_closure(session, scope=source_scope, token=token) + async with session_factory() as session: + archive = await export_tenant_provenance(session, source_scope) + records = parse_provenance_archive(archive) + await _delete_exported_closure(session, records) + await session.commit() + + async def run_import(target_scope): + async with session_factory() as session: + return await import_tenant_provenance(session, target_scope, archive) + + receipts = await asyncio.gather( + *(run_import(target_scope) for target_scope in target_scopes) + ) + assert all(sum(receipt.created.values()) > 0 for receipt in receipts) + async with session_factory() as session: + assert ( + await session.scalar( + select(func.count()) + .select_from(ProjectGraphObjectRecord) + .where( + ProjectGraphObjectRecord.workspace_id.in_( + [scope.workspace_id for scope in target_scopes] + ) + ) + ) + == 4 + ) + finally: + async with session_factory.begin() as session: + workspace_ids = [scope.workspace_id for scope in target_scopes] + for model in ( + ProjectGraphCorrectionRecord, + ProjectGraphEdgeRecord, + ProjectGraphObjectRecord, + ): + await session.execute( + delete(model).where(model.workspace_id.in_(workspace_ids)) + ) + await session.execute( + delete(ProvenanceIdentityMapping).where( + ProvenanceIdentityMapping.target_workspace_id.in_(workspace_ids) + ) + ) + email_ids = list( + ( + await session.scalars( + select(Email.id).where( + Email.message_id == f"<{token}@example.com>" + ) + ) + ).all() + ) + for model in ( + KnowledgeGraphEdgeRecord, + ContentSegmentRecord, + ContentNodeRecord, + Attachment, + ): + await session.execute(delete(model).where(model.email_id.in_(email_ids))) + await session.execute(delete(Email).where(Email.id.in_(email_ids))) + await engine.dispose() + + @pytest.mark.asyncio @pytest.mark.postgres async def test_concurrent_imports_across_workspaces_reuse_owner_email(): From 003421385d0e110a017b2d01d5a75706a500664f Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Tue, 1 Sep 2026 09:44:12 +0900 Subject: [PATCH 37/39] fix(data): preserve incremental provenance imports --- backend/services/tenant_provenance_bundle.py | 30 +++++++++++++-- .../tests/test_tenant_provenance_bundle.py | 38 +++++++++++++++++++ 2 files changed, 65 insertions(+), 3 deletions(-) diff --git a/backend/services/tenant_provenance_bundle.py b/backend/services/tenant_provenance_bundle.py index 4f1112dd6..539edc4a3 100644 --- a/backend/services/tenant_provenance_bundle.py +++ b/backend/services/tenant_provenance_bundle.py @@ -45,6 +45,11 @@ _LOCAL_FILE_SIGNATURE = b"PK\x03\x04" _EOCD_SIGNATURE = b"PK\x05\x06" _EOCD_SIZE = 22 +_EXPORT_MEMORY_LOCK_KEY = int.from_bytes( + hashlib.sha256(b"naruon-tenant-provenance-export-memory-v1").digest()[:8], + "big", + signed=True, +) _EXPORT_TEXT_COLUMNS = { Email: ( @@ -1293,6 +1298,13 @@ async def export_tenant_provenance( ) -> bytes: """Export the exact signed-scope project-evidence closure.""" _validate_scope(scope) + bind = session.get_bind() + if getattr(getattr(bind, "dialect", None), "name", None) == "postgresql": + # ponytail: global lock bounds cross-worker archive memory; replace with + # file-backed streaming when the public export contract supports it. + await session.execute( + select(func.pg_advisory_xact_lock(_EXPORT_MEMORY_LOCK_KEY)) + ) consumed_bytes = 0 for model in ( ProjectGraphObjectRecord, @@ -2282,8 +2294,6 @@ async def _prepare_identity_import( if (row.entity_kind, row.portable_uid) in expected_mapping_keys } if scoped_mappings: - if set(scoped_mappings) != expected_mapping_keys: - _fail() for (collection, portable_uid), row in scoped_mappings.items(): if row.target_database_uid not in { portable_uid, @@ -2291,7 +2301,21 @@ async def _prepare_identity_import( }: _fail() forward_maps[collection][portable_uid] = row.target_database_uid - return _translate_identity_records(records, forward_maps), [] + missing_mapping_keys = expected_mapping_keys - set(scoped_mappings) + return _translate_identity_records(records, forward_maps), [ + ProvenanceIdentityMapping( + target_user_id=scope.user_id, + target_organization_id=scope.organization_id, + target_workspace_id=scope.workspace_id, + source_user_uid=source_scope["user_uid"], + source_organization_uid=source_scope["organization_uid"], + source_workspace_uid=source_scope["workspace_uid"], + entity_kind=collection, + portable_uid=portable_uid, + target_database_uid=forward_maps[collection][portable_uid], + ) + for collection, portable_uid in sorted(missing_mapping_keys) + ] scoped_collisions: list[tuple[str, Any, str]] = [] for model, column, collection in ( diff --git a/backend/tests/test_tenant_provenance_bundle.py b/backend/tests/test_tenant_provenance_bundle.py index c3cf310e8..91fe92d52 100644 --- a/backend/tests/test_tenant_provenance_bundle.py +++ b/backend/tests/test_tenant_provenance_bundle.py @@ -1533,6 +1533,44 @@ async def test_portable_archive_imports_into_two_scopes_after_source_deletion( } +@pytest.mark.asyncio +@pytest.mark.postgres +async def test_portable_archive_incrementally_adds_identity_mappings( + provenance_sessionmaker, +): + token = uuid.uuid4().hex[:12] + source_scope = _scope(f"incremental-source-{token}") + target_scope = _scope(f"incremental-target-{token}") + async with provenance_sessionmaker() as session: + await _seed_provenance_closure( + session, scope=source_scope, token=f"first-{token}" + ) + async with provenance_sessionmaker() as session: + first_archive = await export_tenant_provenance(session, source_scope) + async with provenance_sessionmaker() as session: + await import_tenant_provenance(session, target_scope, first_archive) + + async with provenance_sessionmaker() as session: + await _seed_provenance_closure( + session, scope=source_scope, token=f"second-{token}" + ) + async with provenance_sessionmaker() as session: + expanded_archive = await export_tenant_provenance(session, source_scope) + expanded_records = parse_provenance_archive(expanded_archive) + async with provenance_sessionmaker() as session: + receipt = await import_tenant_provenance( + session, target_scope, expanded_archive + ) + async with provenance_sessionmaker() as session: + target_records = parse_provenance_archive( + await export_tenant_provenance(session, target_scope) + ) + + assert sum(receipt.created.values()) > 0 + for collection in provenance_service._COLLECTIONS: + assert target_records[collection] == expanded_records[collection] + + @pytest.mark.asyncio @pytest.mark.postgres async def test_mixed_native_and_multiple_import_origins_export_as_target_scope( From e2968cdd96bf9faf4d263785f1a4d0bdee78dc83 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Tue, 1 Sep 2026 11:04:20 +0900 Subject: [PATCH 38/39] test(db): fail on post-connect backfill errors --- backend/tests/test_bootstrap_db.py | 26 +++++++++++++++----------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/backend/tests/test_bootstrap_db.py b/backend/tests/test_bootstrap_db.py index b8d5cbc02..cd33d87a1 100644 --- a/backend/tests/test_bootstrap_db.py +++ b/backend/tests/test_bootstrap_db.py @@ -747,7 +747,20 @@ def test_schema_backfill_creates_connector_signal_events(): async def test_schema_backfill_creates_legacy_emails_index_when_table_exists(): engine = create_async_engine(settings.DATABASE_URL) try: - async with engine.begin() as conn: + conn = await engine.connect() + except ( + ConnectionRefusedError, + OSError, + OperationalError, + asyncpg.CannotConnectNowError, + asyncpg.InvalidAuthorizationSpecificationError, + asyncpg.InvalidCatalogNameError, + asyncpg.InvalidPasswordError, + ): + await engine.dispose() + pytest.skip("PostgreSQL smoke path unavailable") + try: + async with conn.begin(): await conn.execute( text( "CREATE TEMP TABLE emails (" @@ -764,17 +777,8 @@ async def test_schema_backfill_creates_legacy_emails_index_when_table_exists(): ) ) assert result.scalar_one() == "ix_emails_owner_date" - except ( - ConnectionRefusedError, - OSError, - OperationalError, - asyncpg.CannotConnectNowError, - asyncpg.InvalidAuthorizationSpecificationError, - asyncpg.InvalidCatalogNameError, - asyncpg.InvalidPasswordError, - ): - pytest.skip("PostgreSQL smoke path unavailable") finally: + await conn.close() await engine.dispose() From 152d1998c4e8024be9dc7026c8789d343c884fd0 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Tue, 1 Sep 2026 11:24:51 +0900 Subject: [PATCH 39/39] test(db): dispose pool after close failures --- backend/tests/test_bootstrap_db.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/backend/tests/test_bootstrap_db.py b/backend/tests/test_bootstrap_db.py index cd33d87a1..6a48e46cd 100644 --- a/backend/tests/test_bootstrap_db.py +++ b/backend/tests/test_bootstrap_db.py @@ -778,8 +778,10 @@ async def test_schema_backfill_creates_legacy_emails_index_when_table_exists(): ) assert result.scalar_one() == "ix_emails_owner_date" finally: - await conn.close() - await engine.dispose() + try: + await conn.close() + finally: + await engine.dispose() @pytest.mark.asyncio