diff --git a/CHANGELOG.md b/CHANGELOG.md index 5cfc3246e..9fb8b6e28 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,11 @@ ## [Unreleased] +- Proposed: export and restore the project evidence cited in one verified + workspace, preserving source links and stable identities. This first slice + excludes full mailbox, binary documents, credentials, provider and connector + state, embeddings, and audit history. Archives remain bounded to 64 MiB + compressed and total uncompressed content, 64 entries, 32 MiB per entry, and + a 100:1 compression ratio; invalid archives are rejected before changes. + Full customer-exit portability and deployment verification remain open. - Proposed: accept manual PDF uploads up to 64MiB after the required processing service release is verified and pinned. Larger files are rejected before storage or processing with HTTP 413; split the file and upload it again. diff --git a/backend/alembic/versions/0018_provenance_identity_mappings.py b/backend/alembic/versions/0018_provenance_identity_mappings.py new file mode 100644 index 000000000..9cf25a927 --- /dev/null +++ b/backend/alembic/versions/0018_provenance_identity_mappings.py @@ -0,0 +1,59 @@ +"""Add scoped portable-to-database provenance identity mappings.""" + +from alembic import op +import sqlalchemy as sa + + +revision = "0018_provenance_identity" +down_revision = "0017_merge_newsdom_carddav_heads" +branch_labels = None +depends_on = None +_MAPPING_TABLE = "provenance_identity_mappings" + + +def upgrade() -> None: + inspector = sa.inspect(op.get_bind()) + if not inspector.has_table(_MAPPING_TABLE): + op.create_table( + _MAPPING_TABLE, + sa.Column("provenance_identity_id", sa.Integer(), primary_key=True), + sa.Column("target_user_id", sa.String(), nullable=False), + sa.Column("target_organization_id", sa.String(), nullable=False), + sa.Column("target_workspace_id", sa.String(), nullable=False), + sa.Column("source_user_uid", sa.String(length=64), nullable=False), + sa.Column("source_organization_uid", sa.String(), nullable=False), + sa.Column("source_workspace_uid", sa.String(), nullable=False), + sa.Column("entity_kind", sa.String(length=64), nullable=False), + sa.Column("portable_uid", sa.String(length=256), nullable=False), + sa.Column("target_database_uid", sa.String(length=96), nullable=False), + sa.UniqueConstraint( + "target_user_id", + "target_organization_id", + "target_workspace_id", + "source_user_uid", + "source_organization_uid", + "source_workspace_uid", + "entity_kind", + "portable_uid", + name="uq_provenance_identity_source_target", + ), + sa.UniqueConstraint( + "entity_kind", + "target_database_uid", + name="uq_provenance_identity_target_uid", + ), + ) + op.create_index( + "ix_provenance_identity_target_scope", + _MAPPING_TABLE, + ["target_user_id", "target_organization_id", "target_workspace_id"], + if_not_exists=True, + ) + + +def downgrade() -> None: + """Preserve imported identity mappings when application code is rolled back.""" + # The table may predate this revision via 0001's live metadata bootstrap. + # Its portable identity history is not rebuildable from remapped records. + # Destructive retirement requires a separate, explicitly governed operation. + return None diff --git a/backend/alembic/versions/0021_merge_provenance_workspace.py b/backend/alembic/versions/0021_merge_provenance_workspace.py new file mode 100644 index 000000000..df72fd114 --- /dev/null +++ b/backend/alembic/versions/0021_merge_provenance_workspace.py @@ -0,0 +1,16 @@ +"""Join provenance identity and workspace repair histories without rewriting either.""" + +revision = "0021_merge_provenance_workspace" +down_revision = ("0018_provenance_identity", "0020_search_trigram_storage") +branch_labels = None +depends_on = None + + +def upgrade() -> None: + """Require both existing branches before recording the unified head.""" + return None + + +def downgrade() -> None: + """Reopen the two revision heads without changing customer records.""" + return None diff --git a/backend/api/data.py b/backend/api/data.py index e7f84162f..5ca62889e 100644 --- a/backend/api/data.py +++ b/backend/api/data.py @@ -7,7 +7,7 @@ import re from typing import Literal, NamedTuple -from fastapi import APIRouter, Depends, File, Form, HTTPException, UploadFile +from fastapi import APIRouter, Depends, File, Form, HTTPException, Request, Response, UploadFile from pydantic import BaseModel, ConfigDict, Field from sqlalchemy import and_, case, func, or_, select from sqlalchemy.engine import Row @@ -35,6 +35,13 @@ PDF_DOM_RECOGNITION_PENDING_STATUS, ) from services.ontology_service import ontology_service +from services.tenant_provenance_bundle import ( + ARCHIVE_MAX_BYTES, + ProvenanceArchiveError, + TenantProvenanceScope, + export_tenant_provenance, + import_tenant_provenance, +) from services.webdav_service import webdav_service from services.workspace_scope import get_or_create_workspace @@ -46,6 +53,7 @@ # transport ceiling so large customer PDFs are not accepted by one path and # rejected by the next. _MAX_PDF_DOM_UPLOAD_BYTES = 64 * 1024 * 1024 +_PROVENANCE_ARCHIVE_MAX_BYTES = ARCHIVE_MAX_BYTES ATTACHMENT_PARSE_BREAKDOWN_EVIDENCE_SOURCE = ( "email_attachments.content_type, " "email_attachments.parse_content_type, " @@ -3192,6 +3200,86 @@ def _quality_checks( ] +def _provenance_scope(auth_context: AuthContext) -> TenantProvenanceScope: + return TenantProvenanceScope( + user_id=auth_context.user_id, + organization_id=auth_context.organization_id, + workspace_id=auth_context.workspace_id, + ) + + +def _require_authoritative_provenance_scope(auth_context: AuthContext) -> None: + if auth_context.session_verifier != "oidc": + raise HTTPException( + status_code=403, + detail="Authoritative workspace membership is required for provenance bundles", + ) + + +async def _read_provenance_archive(request: Request) -> bytes: + content_length = request.headers.get("content-length") + if content_length is not None: + if not content_length.isdigit(): + raise HTTPException(status_code=400, detail="Invalid Content-Length") + try: + declared_bytes = int(content_length) + except ValueError as exc: + raise HTTPException( + status_code=400, detail="Invalid Content-Length" + ) from exc + if declared_bytes > _PROVENANCE_ARCHIVE_MAX_BYTES: + raise HTTPException(status_code=413, detail="Provenance archive too large") + archive = bytearray() + async for chunk in request.stream(): + if len(chunk) > _PROVENANCE_ARCHIVE_MAX_BYTES - len(archive): + raise HTTPException(status_code=413, detail="Provenance archive too large") + archive.extend(chunk) + return bytes(archive) + + +@router.get("/provenance-bundle") +async def download_provenance_bundle( + auth_context: AuthContext = Depends(get_auth_context), + db: AsyncSession = Depends(get_db), +) -> Response: + _require_authoritative_provenance_scope(auth_context) + try: + archive = await export_tenant_provenance(db, _provenance_scope(auth_context)) + except ProvenanceArchiveError as exc: + raise HTTPException( + status_code=400, detail="Invalid provenance archive" + ) from exc + return Response( + content=archive, + media_type="application/zip", + headers={"Content-Disposition": 'attachment; filename="naruon-provenance.zip"'}, + ) + + +@router.post("/provenance-bundle/import") +async def upload_provenance_bundle( + request: Request, + auth_context: AuthContext = Depends(get_auth_context), + db: AsyncSession = Depends(get_db), +) -> dict[str, object]: + _require_authoritative_provenance_scope(auth_context) + archive = await _read_provenance_archive(request) + try: + receipt = await import_tenant_provenance( + db, _provenance_scope(auth_context), archive + ) + except ProvenanceArchiveError as exc: + raise HTTPException( + status_code=400, detail="Invalid provenance archive" + ) from exc + return { + "bundle_uid": receipt.bundle_uid, + "manifest_digest": receipt.manifest_digest, + "created": receipt.created, + "skipped": receipt.skipped, + } + + @router.post("/documents", response_model=DataDocumentActionResponse) async def upload_data_document( request: DataDocumentUploadRequest, diff --git a/backend/db/models.py b/backend/db/models.py index 98e17eef2..714588836 100644 --- a/backend/db/models.py +++ b/backend/db/models.py @@ -1293,6 +1293,45 @@ class ProjectGraphCorrectionRecord(Base): ) +class ProvenanceIdentityMapping(Base): + __tablename__ = "provenance_identity_mappings" + __table_args__ = ( + UniqueConstraint( + "target_user_id", + "target_organization_id", + "target_workspace_id", + "source_user_uid", + "source_organization_uid", + "source_workspace_uid", + "entity_kind", + "portable_uid", + name="uq_provenance_identity_source_target", + ), + UniqueConstraint( + "entity_kind", + "target_database_uid", + name="uq_provenance_identity_target_uid", + ), + Index( + "ix_provenance_identity_target_scope", + "target_user_id", + "target_organization_id", + "target_workspace_id", + ), + ) + + provenance_identity_id: Mapped[int] = mapped_column(primary_key=True) + target_user_id: Mapped[str] = mapped_column(String, nullable=False) + target_organization_id: Mapped[str] = mapped_column(String, nullable=False) + target_workspace_id: Mapped[str] = mapped_column(String, nullable=False) + source_user_uid: Mapped[str] = mapped_column(String(64), nullable=False) + source_organization_uid: Mapped[str] = mapped_column(String, nullable=False) + source_workspace_uid: Mapped[str] = mapped_column(String, nullable=False) + entity_kind: Mapped[str] = mapped_column(String(64), nullable=False) + portable_uid: Mapped[str] = mapped_column(String(256), nullable=False) + target_database_uid: Mapped[str] = mapped_column(String(96), nullable=False) + + class TenantConfig(Base): __tablename__ = "tenant_configs" diff --git a/backend/services/tenant_provenance_bundle.py b/backend/services/tenant_provenance_bundle.py new file mode 100644 index 000000000..539edc4a3 --- /dev/null +++ b/backend/services/tenant_provenance_bundle.py @@ -0,0 +1,2940 @@ +"""Deterministic, bounded BagIt/RO-Crate envelopes for tenant provenance.""" + +from __future__ import annotations + +import copy +import hashlib +import io +import json +import math +import re +import stat +import struct +import zipfile +from collections.abc import Mapping +from dataclasses import dataclass +from datetime import datetime, timezone +from typing import Any + +from sqlalchemy import Text, cast, func, select +from sqlalchemy.exc import DataError, IntegrityError, StatementError +from sqlalchemy.ext.asyncio import AsyncSession + +from db.models import ( + Attachment, + ContentNodeRecord, + ContentSegmentRecord, + Email, + KnowledgeGraphEdgeRecord, + ProjectGraphCorrectionRecord, + ProjectGraphEdgeRecord, + ProjectGraphObjectRecord, + ProvenanceIdentityMapping, +) + + +ARCHIVE_MAX_BYTES = 64 * 1024 * 1024 +ARCHIVE_MAX_ENTRIES = 64 +ENTRY_MAX_BYTES = 32 * 1024 * 1024 +MAX_COMPRESSION_RATIO = 100 +JSON_MAX_DEPTH = 64 +JSON_SAFE_INTEGER_MAX = 2**53 - 1 +_MAX_IDENTIFIER_LENGTH = 256 +_FIXED_TIMESTAMP = (1980, 1, 1, 0, 0, 0) +_PAYLOAD_NAME = "data/records.json" +_LOCAL_FILE_SIGNATURE = b"PK\x03\x04" +_EOCD_SIGNATURE = b"PK\x05\x06" +_EOCD_SIZE = 22 +_EXPORT_MEMORY_LOCK_KEY = int.from_bytes( + hashlib.sha256(b"naruon-tenant-provenance-export-memory-v1").digest()[:8], + "big", + signed=True, +) + +_EXPORT_TEXT_COLUMNS = { + Email: ( + Email.message_id, + Email.thread_id, + Email.fingerprint, + Email.sender, + Email.reply_to, + Email.recipients, + Email.subject, + Email.in_reply_to, + Email.references, + Email.body, + ), + Attachment: ( + Attachment.filename, + Attachment.content, + Attachment.content_type, + Attachment.parse_status, + Attachment.parse_content_type, + Attachment.parser_key, + Attachment.parse_error_code, + ), + ContentNodeRecord: ( + ContentNodeRecord.source_record_uid, + ContentNodeRecord.parent_node_uid, + ContentNodeRecord.display_label, + ContentNodeRecord.safe_text_content, + ), + ContentSegmentRecord: ( + ContentSegmentRecord.source_record_uid, + ContentSegmentRecord.heading_path, + ContentSegmentRecord.safe_text_content, + ), + KnowledgeGraphEdgeRecord: ( + KnowledgeGraphEdgeRecord.source_record_uid, + KnowledgeGraphEdgeRecord.edge_path, + ), + ProjectGraphObjectRecord: ( + ProjectGraphObjectRecord.title, + ProjectGraphObjectRecord.summary, + ProjectGraphObjectRecord.source_segment_uids, + ProjectGraphObjectRecord.attributes_json, + ), + ProjectGraphEdgeRecord: (ProjectGraphEdgeRecord.source_segment_uids,), + ProjectGraphCorrectionRecord: ( + ProjectGraphCorrectionRecord.before_json, + ProjectGraphCorrectionRecord.after_json, + ProjectGraphCorrectionRecord.rationale, + ProjectGraphCorrectionRecord.source_segment_uids, + ), +} +_EXPECTED_ENTRIES = frozenset( + { + "bagit.txt", + "bag-info.txt", + "manifest-sha512.txt", + "tagmanifest-sha512.txt", + "ro-crate-metadata.json", + _PAYLOAD_NAME, + } +) +_COLLECTIONS = ( + "emails", + "attachments", + "content_nodes", + "content_segments", + "structural_edges", + "project_objects", + "project_edges", + "corrections", +) +_REMAPPED_COLLECTIONS = ( + "content_nodes", + "content_segments", + "structural_edges", + "project_objects", + "project_edges", + "corrections", +) +_UID_KEYS = { + "content_nodes": "content_node_uid", + "content_segments": "content_segment_uid", + "structural_edges": "edge_uid", + "project_objects": "object_uid", + "project_edges": "edge_uid", + "corrections": "correction_uid", +} +_UID_PREFIXES = { + "content_nodes": "tpn-", + "content_segments": "tps-", + "structural_edges": "tpk-", + "project_objects": "tpo-", + "project_edges": "tpe-", + "corrections": "tpc-", +} +_TYPED_METADATA_UIDS = { + "object_uid": "project_objects", + "source_object_uid": "project_objects", + "target_object_uid": "project_objects", + "content_node_uid": "content_nodes", + "parent_node_uid": "content_nodes", + "source_node_uid": "content_nodes", + "target_node_uid": "content_nodes", + "content_segment_uid": "content_segments", + "segment_uid": "content_segments", + "source_segment_uid": "content_segments", + "target_segment_uid": "content_segments", + "primary_segment_uid": "content_segments", + "primary_content_segment_uid": "content_segments", +} +_TYPED_METADATA_UID_LISTS = {"source_segment_uids": "content_segments"} +_TYPED_METADATA_ATTACHMENT_REFERENCES = frozenset( + {"attachment_uid", "source_record_uid"} +) +_TEXTUAL_PARSER_KEYS = frozenset( + {"plain_text", "html", "markdown", "json", "csv", "xml", "calendar", "pdf"} +) +_FORBIDDEN_METADATA_KEYS = frozenset( + { + "access_token", + "api_key", + "api_secret", + "attachment_id", + "auth_token", + "base_url", + "bearer_token", + "client_secret", + "connection_string", + "content_node_id", + "content_segment_id", + "credential", + "credentials", + "credentials_encrypted", + "database_id", + "database_url", + "db_id", + "dsn", + "email_id", + "id", + "knowledge_graph_edge_id", + "openai_api_key", + "password", + "primary_key", + "private_key", + "project_graph_correction_id", + "project_graph_edge_id", + "project_graph_object_id", + "provider_base_url", + "provider_endpoint", + "provider_url", + "refresh_token", + "row_id", + "secret", + "secrets", + "token", + } +) +_SENSITIVE_METADATA_TOKENS = frozenset( + {"credential", "credentials", "password", "secret", "secrets", "token"} +) +_METADATA_KEY_TOKEN_PATTERN = re.compile( + r"[A-Z]+(?=[A-Z][a-z]|[0-9]|[^A-Za-z0-9]|$)" + r"|[A-Z]?[a-z]+|[A-Z]+|[0-9]+" +) +_RECORD_KEYS = { + "emails": frozenset( + { + "email_uid", + "thread_uid", + "fingerprint", + "sender", + "reply_to", + "recipients", + "subject", + "in_reply_to", + "references", + "date", + "body", + "is_read", + } + ), + "attachments": frozenset( + { + "attachment_uid", + "email_uid", + "filename", + "content", + "content_type", + "parse_status", + "parse_content_type", + "parser_key", + "parse_error_code", + } + ), + "content_nodes": frozenset( + { + "content_node_uid", + "email_uid", + "attachment_uid", + "source_kind", + "source_record_uid", + "parent_node_uid", + "node_kind", + "node_path", + "ordinal_index", + "display_label", + "safe_text_content", + "content_hash", + } + ), + "content_segments": frozenset( + { + "content_segment_uid", + "email_uid", + "attachment_uid", + "content_node_uid", + "source_kind", + "source_record_uid", + "segment_kind", + "segment_path", + "ordinal_index", + "heading_path", + "safe_text_content", + "content_hash", + "word_count", + } + ), + "structural_edges": frozenset( + { + "edge_uid", + "email_uid", + "attachment_uid", + "source_node_uid", + "target_node_uid", + "source_segment_uid", + "target_segment_uid", + "source_kind", + "source_record_uid", + "edge_kind", + "edge_path", + "ordinal_index", + } + ), + "project_objects": frozenset( + { + "object_uid", + "email_uid", + "attachment_uid", + "primary_content_segment_uid", + "object_type", + "title", + "summary", + "status_code", + "confidence", + "source_segment_uids", + "attributes_json", + "extractor_name", + "extractor_version", + } + ), + "project_edges": frozenset( + { + "edge_uid", + "source_uid", + "target_uid", + "edge_type", + "confidence", + "source_segment_uids", + "source_object_uid", + "target_object_uid", + "primary_content_segment_uid", + } + ), + "corrections": frozenset( + { + "correction_uid", + "object_uid", + "correction_action", + "before_json", + "after_json", + "rationale", + "source_segment_uids", + "created_at", + } + ), +} + + +class ProvenanceArchiveError(ValueError): + """Raised when a provenance envelope is malformed or outside this profile.""" + + +@dataclass(frozen=True) +class TenantProvenanceScope: + """Signed-session authority used for every source read and target write.""" + + user_id: str + organization_id: str | None + workspace_id: str + + +@dataclass(frozen=True) +class ImportReceipt: + """Verified import outcome without exposing persistence identifiers.""" + + bundle_uid: str + manifest_digest: str + created: dict[str, int] + skipped: dict[str, int] + + +def _fail() -> None: + raise ProvenanceArchiveError("Invalid provenance archive") + + +def _validate_json_value(value: object, depth: int = 0) -> None: + if depth > JSON_MAX_DEPTH: + _fail() + if value is None or isinstance(value, (str, bool)): + return + if type(value) is int: + if not -JSON_SAFE_INTEGER_MAX <= value <= JSON_SAFE_INTEGER_MAX: + _fail() + return + if isinstance(value, float): + if not math.isfinite(value): + _fail() + return + if isinstance(value, list): + for item in value: + _validate_json_value(item, depth + 1) + return + if isinstance(value, Mapping): + for key, item in value.items(): + if not isinstance(key, str): + _fail() + if not key.isascii(): + _fail() + _validate_json_value(item, depth + 1) + return + _fail() + + +def _metadata_key_is_forbidden(key: str) -> bool: + tokens = tuple( + match.group(0).lower() for match in _METADATA_KEY_TOKEN_PATTERN.finditer(key) + ) + normalized_key = "_".join(tokens) + token_set = frozenset(tokens) + token_pairs = set(zip(tokens, tokens[1:])) + return ( + normalized_key in _FORBIDDEN_METADATA_KEYS + or bool(token_set & _SENSITIVE_METADATA_TOKENS) + or ("api", "key") in token_pairs + or ("provider" in token_set and bool({"endpoint", "url", "uri"} & token_set)) + or (bool(tokens) and tokens[-1] == "id") + or (bool({"database", "db"} & token_set) and bool({"id", "key"} & token_set)) + ) + + +def _validate_safe_metadata(value: object, depth: int = 0) -> None: + if depth > JSON_MAX_DEPTH: + _fail() + if isinstance(value, Mapping): + for key, item in value.items(): + if not isinstance(key, str): + _fail() + if _metadata_key_is_forbidden(key): + _fail() + _validate_safe_metadata(item, depth + 1) + elif isinstance(value, list): + for item in value: + _validate_safe_metadata(item, depth + 1) + + +def _portable_metadata( + value: object, attachment_references: Mapping[str, str] +) -> object: + if isinstance(value, list): + return [_portable_metadata(item, attachment_references) for item in value] + if isinstance(value, Mapping): + portable = {} + for key, item in value.items(): + if key in _TYPED_METADATA_ATTACHMENT_REFERENCES and isinstance(item, str): + portable[key] = attachment_references.get(item, item) + else: + portable[key] = _portable_metadata(item, attachment_references) + return portable + return value + + +def _source_user_uid(user_id: str) -> str: + return hashlib.sha256(user_id.encode("utf-8")).hexdigest() + + +def _target_database_uid( + scope: TenantProvenanceScope, + source_scope: Mapping[str, str], + collection: str, + portable_uid: str, +) -> str: + digest = hashlib.sha256( + _canonical_json( + { + "collection": collection, + "portable_uid": portable_uid, + "source_scope": dict(source_scope), + "target_scope": { + "organization_uid": scope.organization_id, + "user_uid": scope.user_id, + "workspace_uid": scope.workspace_id, + }, + } + ) + ).hexdigest() + return f"{_UID_PREFIXES[collection]}{digest[:60]}" + + +def _canonical_json(value: object) -> bytes: + _validate_json_value(value) + try: + return json.dumps( + value, + allow_nan=False, + ensure_ascii=False, + separators=(",", ":"), + sort_keys=True, + ).encode("utf-8") + except (TypeError, ValueError, UnicodeEncodeError) as exc: + raise ProvenanceArchiveError("Invalid provenance archive") from exc + + +def _segment_endpoint_uid(value: object) -> str: + if not isinstance(value, str) or not value.startswith("segment:"): + _fail() + segment_uid = value.removeprefix("segment:") + if not segment_uid: + _fail() + return segment_uid + + +def _translate_identity_records( + records: Mapping[str, object], maps: Mapping[str, Mapping[str, str]] +) -> dict[str, object]: + translated = copy.deepcopy(records) + + def replace(record: dict[str, object], field: str, collection: str) -> None: + value = record[field] + if value is not None: + record[field] = maps[collection].get(value, value) + + def replace_list(record: dict[str, object], field: str, collection: str) -> None: + record[field] = sorted( + maps[collection].get(value, value) for value in record[field] + ) + + def replace_project_endpoint( + record: dict[str, object], field: str, object_field: str + ) -> None: + value = record[field] + if record[object_field] is not None: + mapped_uid = maps["project_objects"].get(value) + if mapped_uid is None: + _fail() + record[field] = mapped_uid + return + segment_uid = _segment_endpoint_uid(value) + mapped_uid = maps["content_segments"].get(segment_uid) + if mapped_uid is None: + _fail() + record[field] = f"segment:{mapped_uid}" + + def translate_metadata(value: object) -> object: + if isinstance(value, dict): + translated_mapping = { + key: translate_metadata(item) for key, item in value.items() + } + for key, collection in _TYPED_METADATA_UIDS.items(): + item = translated_mapping.get(key) + if isinstance(item, str): + translated_mapping[key] = maps[collection].get(item, item) + for key, collection in _TYPED_METADATA_UID_LISTS.items(): + item = translated_mapping.get(key) + if isinstance(item, list) and all(isinstance(uid, str) for uid in item): + translated_mapping[key] = [ + maps[collection].get(uid, uid) for uid in item + ] + return translated_mapping + if isinstance(value, list): + return [translate_metadata(item) for item in value] + return value + + for record in translated["content_nodes"]: + replace(record, "content_node_uid", "content_nodes") + replace(record, "parent_node_uid", "content_nodes") + for record in translated["content_segments"]: + replace(record, "content_segment_uid", "content_segments") + replace(record, "content_node_uid", "content_nodes") + for record in translated["structural_edges"]: + replace(record, "edge_uid", "structural_edges") + for field in ("source_node_uid", "target_node_uid"): + replace(record, field, "content_nodes") + for field in ("source_segment_uid", "target_segment_uid"): + replace(record, field, "content_segments") + for record in translated["project_objects"]: + replace(record, "object_uid", "project_objects") + replace(record, "primary_content_segment_uid", "content_segments") + replace_list(record, "source_segment_uids", "content_segments") + record["attributes_json"] = translate_metadata(record["attributes_json"]) + for record in translated["project_edges"]: + replace(record, "edge_uid", "project_edges") + replace_project_endpoint(record, "source_uid", "source_object_uid") + replace_project_endpoint(record, "target_uid", "target_object_uid") + replace(record, "source_object_uid", "project_objects") + replace(record, "target_object_uid", "project_objects") + replace(record, "primary_content_segment_uid", "content_segments") + replace_list(record, "source_segment_uids", "content_segments") + for record in translated["corrections"]: + replace(record, "correction_uid", "corrections") + replace(record, "object_uid", "project_objects") + replace_list(record, "source_segment_uids", "content_segments") + record["before_json"] = translate_metadata(record["before_json"]) + record["after_json"] = translate_metadata(record["after_json"]) + for collection in _REMAPPED_COLLECTIONS: + translated[collection].sort(key=lambda record: record[_UID_KEYS[collection]]) + return translated + + +def _records_bundle_uid(records: Mapping[str, object]) -> str: + schema_version = records.get("schema_version") + if ( + records.get("profile") != "naruon-tenant-provenance/v1" + or type(schema_version) is not int + or schema_version != 1 + ): + _fail() + return _safe_identifier(records.get("bundle_uid")) + + +def _content_identity_digest(records: Mapping[str, object]) -> str: + activity = records.get("export_activity") + if not isinstance(activity, Mapping): + _fail() + identity = { + "profile": records.get("profile"), + "schema_version": records.get("schema_version"), + "source_scope": records.get("source_scope"), + **{collection: records.get(collection) for collection in _COLLECTIONS}, + "export_activity": {"date_published": activity.get("date_published")}, + } + return hashlib.sha256(_canonical_json(identity)).hexdigest() + + +def _validate_content_bound_identifiers(records: Mapping[str, object]) -> str: + content_digest = _content_identity_digest(records) + if _records_bundle_uid(records) != f"bundle-{content_digest}": + _fail() + activity = records.get("export_activity") + if not isinstance(activity, Mapping) or _safe_identifier( + activity.get("activity_uid") + ) != f"export-{content_digest}": + _fail() + return content_digest + + +def _safe_identifier(value: object) -> str: + if ( + not isinstance(value, str) + or not value + or len(value) > _MAX_IDENTIFIER_LENGTH + or any(ord(character) < 32 or ord(character) == 127 for character in value) + ): + _fail() + return value + + +def _iso8601_date_or_datetime(value: object) -> str: + date_published = _safe_identifier(value) + return _utc_text(_parse_datetime(date_published)) + + +def _bag_info(bundle_uid: str) -> bytes: + return ( + "Bag-Software-Agent: naruon\n" + "Bagging-Date: 1980-01-01\n" + f"External-Identifier: {bundle_uid}\n" + ).encode("utf-8") + + +def _ro_crate(records: Mapping[str, object], payload_digest: str) -> bytes: + activity = records.get("export_activity") + if not isinstance(activity, Mapping): + _fail() + activity_uid = _safe_identifier(activity.get("activity_uid")) + date_published = _iso8601_date_or_datetime(activity.get("date_published")) + crate = { + "@context": [ + "https://w3id.org/ro/crate/1.3/context", + {"prov": "http://www.w3.org/ns/prov#"}, + ], + "@graph": [ + { + "@id": "ro-crate-metadata.json", + "@type": "CreativeWork", + "about": {"@id": "./"}, + "conformsTo": {"@id": "https://w3id.org/ro/crate/1.3"}, + }, + { + "@id": "./", + "@type": "Dataset", + "conformsTo": "naruon-tenant-provenance/v1", + "datePublished": date_published, + "hasPart": {"@id": _PAYLOAD_NAME}, + "name": "Naruon tenant provenance bundle", + }, + { + "@id": _PAYLOAD_NAME, + "@type": "File", + "encodingFormat": "application/json", + "sha512": payload_digest, + }, + { + "@id": f"#{activity_uid}", + "@type": ["CreateAction", "prov:Activity"], + "instrument": {"@id": "#naruon"}, + "object": {"@id": _PAYLOAD_NAME}, + "prov:used": {"@id": "./"}, + "prov:wasAssociatedWith": {"@id": "#naruon"}, + }, + { + "@id": "#naruon", + "@type": ["SoftwareApplication", "prov:SoftwareAgent"], + "name": "Naruon", + }, + ], + } + return _canonical_json(crate) + + +def _manifest(entries: Mapping[str, bytes], names: tuple[str, ...]) -> bytes: + return b"".join( + f"{hashlib.sha512(entries[name]).hexdigest()} {name}\n".encode("ascii") + for name in names + ) + + +def _archive_entries(records: Mapping[str, object]) -> dict[str, bytes]: + payload = _canonical_json(records) + _validate_content_bound_identifiers(records) + bundle_uid = _records_bundle_uid(records) + entries = { + "bagit.txt": b"BagIt-Version: 1.0\nTag-File-Character-Encoding: UTF-8\n", + "bag-info.txt": _bag_info(bundle_uid), + _PAYLOAD_NAME: payload, + "ro-crate-metadata.json": _ro_crate( + records, hashlib.sha512(payload).hexdigest() + ), + } + entries["manifest-sha512.txt"] = _manifest(entries, (_PAYLOAD_NAME,)) + entries["tagmanifest-sha512.txt"] = _manifest( + entries, + ("bag-info.txt", "bagit.txt", "manifest-sha512.txt", "ro-crate-metadata.json"), + ) + return entries + + +def _zip_info(name: str) -> zipfile.ZipInfo: + info = zipfile.ZipInfo(name, date_time=_FIXED_TIMESTAMP) + info.create_system = 3 + info.create_version = 20 + info.extract_version = 20 + info.reserved = 0 + info.flag_bits = 0 + info.volume = 0 + info.internal_attr = 0 + info.external_attr = (stat.S_IFREG | 0o644) << 16 + info.compress_type = zipfile.ZIP_DEFLATED + info.extra = b"" + info.comment = b"" + return info + + +def build_provenance_archive(records: Mapping[str, object]) -> bytes: + """Build the fixed deterministic ZIP envelope for a validated record payload.""" + if not isinstance(records, Mapping): + _fail() + activity = records.get("export_activity") + if not isinstance(activity, Mapping): + _fail() + _records_bundle_uid(records) + _safe_identifier(activity.get("activity_uid")) + bound_records = { + **records, + "export_activity": dict(activity), + } + content_digest = _content_identity_digest(bound_records) + bound_records["bundle_uid"] = f"bundle-{content_digest}" + bound_records["export_activity"]["activity_uid"] = f"export-{content_digest}" + entries = _archive_entries(bound_records) + output = io.BytesIO() + with zipfile.ZipFile( + output, "w", compression=zipfile.ZIP_DEFLATED, compresslevel=9 + ) as archive: + for name in sorted(entries): + archive.writestr( + _zip_info(name), + entries[name], + compress_type=zipfile.ZIP_DEFLATED, + compresslevel=9, + ) + archive_bytes = output.getvalue() + if len(archive_bytes) > ARCHIVE_MAX_BYTES: + _fail() + parse_provenance_archive(archive_bytes) + return archive_bytes + + +def _is_unsafe_member(info: zipfile.ZipInfo) -> bool: + name = info.filename + parts = name.split("/") + return ( + not name + or "\\" in name + or "\x00" in name + or name.startswith("/") + or any(part in {"", ".", ".."} for part in parts) + or parts[0].endswith(":") + or info.is_dir() + or stat.S_ISLNK(info.external_attr >> 16) + or bool(info.flag_bits & 0x1) + ) + + +def _has_fixed_metadata(info: zipfile.ZipInfo) -> bool: + return ( + info.date_time == _FIXED_TIMESTAMP + and info.create_system == 3 + and info.create_version == 20 + and info.extract_version == 20 + and info.reserved == 0 + and info.flag_bits == 0 + and info.volume == 0 + and info.internal_attr == 0 + and info.external_attr == (stat.S_IFREG | 0o644) << 16 + and info.compress_type == zipfile.ZIP_DEFLATED + and info.extra == b"" + and info.comment == b"" + ) + + +def _within_archive_bounds( + *, + archive_bytes: int, + entry_count: int, + total_bytes: int, + entry_bytes: int, + compressed_bytes: int, +) -> bool: + return ( + 0 <= archive_bytes <= ARCHIVE_MAX_BYTES + and 0 <= entry_count <= ARCHIVE_MAX_ENTRIES + and 0 <= total_bytes <= ARCHIVE_MAX_BYTES + and 0 <= entry_bytes <= ENTRY_MAX_BYTES + and compressed_bytes >= 0 + and ( + entry_bytes == 0 + or 0 < compressed_bytes + and entry_bytes <= compressed_bytes * MAX_COMPRESSION_RATIO + ) + ) + + +def _has_profile_container_framing(archive_bytes: bytes | bytearray) -> bool: + if ( + len(archive_bytes) < _EOCD_SIZE + or not archive_bytes.startswith(_LOCAL_FILE_SIGNATURE) + or archive_bytes[-_EOCD_SIZE:-18] != _EOCD_SIGNATURE + ): + return False + ( + _, + disk_number, + directory_disk, + entries_on_disk, + entries, + directory_size, + directory_offset, + comment_size, + ) = struct.unpack("<4s4H2LH", archive_bytes[-_EOCD_SIZE:]) + return ( + disk_number == 0 + and directory_disk == 0 + and entries_on_disk == entries + and entries != 0xFFFF + and directory_size != 0xFFFFFFFF + and directory_offset != 0xFFFFFFFF + and comment_size == 0 + and directory_offset + directory_size == len(archive_bytes) - _EOCD_SIZE + ) + + +def _has_contiguous_member_data(infos: list[zipfile.ZipInfo], start_dir: int) -> bool: + expected_offset = 0 + for info in infos: + if info.header_offset != expected_offset: + return False + expected_offset += ( + zipfile.sizeFileHeader + + len(info.filename.encode("ascii")) + + len(info.extra) + + info.compress_size + ) + return expected_offset == start_dir + + +def _read_entry(archive: zipfile.ZipFile, info: zipfile.ZipInfo) -> bytes: + if not _within_archive_bounds( + archive_bytes=0, + entry_count=0, + total_bytes=0, + entry_bytes=info.file_size, + compressed_bytes=info.compress_size, + ): + _fail() + data = bytearray() + try: + with archive.open(info, "r") as source: + while chunk := source.read(min(64 * 1024, ENTRY_MAX_BYTES + 1 - len(data))): + data.extend(chunk) + if len(data) > ENTRY_MAX_BYTES: + _fail() + except (OSError, RuntimeError, zipfile.BadZipFile) as exc: + raise ProvenanceArchiveError("Invalid provenance archive") from exc + if len(data) != info.file_size: + _fail() + return bytes(data) + + +def _reject_duplicate_json_keys(pairs: list[tuple[str, Any]]) -> dict[str, Any]: + value: dict[str, Any] = {} + for key, item in pairs: + if key in value: + _fail() + value[key] = item + return value + + +def _parse_records(data: bytes) -> dict[str, object]: + try: + records = json.loads( + data.decode("utf-8"), + object_pairs_hook=_reject_duplicate_json_keys, + parse_constant=lambda _value: _fail(), + ) + except (UnicodeDecodeError, json.JSONDecodeError, TypeError, ValueError) as exc: + raise ProvenanceArchiveError("Invalid provenance archive") from exc + if not isinstance(records, dict): + _fail() + if _canonical_json(records) != data: + _fail() + return records + + +def parse_provenance_archive(archive_bytes: bytes) -> dict[str, object]: + """Validate a bounded fixed envelope and return its canonical record payload.""" + if ( + not isinstance(archive_bytes, (bytes, bytearray)) + or not _within_archive_bounds( + archive_bytes=len(archive_bytes), + entry_count=0, + total_bytes=0, + entry_bytes=0, + compressed_bytes=0, + ) + or not _has_profile_container_framing(archive_bytes) + ): + _fail() + try: + with zipfile.ZipFile(io.BytesIO(archive_bytes), "r") as archive: + infos = archive.infolist() + if ( + archive.comment + or not _within_archive_bounds( + archive_bytes=len(archive_bytes), + entry_count=len(infos), + total_bytes=sum(info.file_size for info in infos), + entry_bytes=0, + compressed_bytes=0, + ) + or any( + _is_unsafe_member(info) or not _has_fixed_metadata(info) + for info in infos + ) + ): + _fail() + names = [info.filename for info in infos] + if ( + names != sorted(names) + or len(set(names)) != len(names) + or set(names) != _EXPECTED_ENTRIES + or not _has_contiguous_member_data(infos, archive.start_dir) + ): + _fail() + entries = {info.filename: _read_entry(archive, info) for info in infos} + except (OSError, RuntimeError, zipfile.BadZipFile) as exc: + raise ProvenanceArchiveError("Invalid provenance archive") from exc + + records = _parse_records(entries[_PAYLOAD_NAME]) + expected = _archive_entries(records) + if entries != expected: + _fail() + return records + + +def _scope_filters(model: Any, scope: TenantProvenanceScope, *, workspace: bool): + organization_filter = ( + model.organization_id == scope.organization_id + if scope.organization_id is not None + else model.organization_id.is_(None) + ) + filters = [model.user_id == scope.user_id, organization_filter] + if workspace: + filters.append(model.workspace_id == scope.workspace_id) + return filters + + +async def _preflight_export_rows( + session: AsyncSession, + model: Any, + filters: tuple[Any, ...] | list[Any], + consumed_bytes: int, +) -> int: + bind = session.get_bind() + if getattr(getattr(bind, "dialect", None), "name", None) != "postgresql": + return consumed_bytes + text_bytes = sum( + ( + func.coalesce(func.octet_length(cast(column, Text)), 0) + for column in _EXPORT_TEXT_COLUMNS[model] + ), + start=0, + ) + row_bytes = func.pg_column_size(model.__table__.table_valued()) + text_bytes + stored_bytes = int( + await session.scalar( + select(func.coalesce(func.sum(row_bytes), 0)).where(*filters) + ) + or 0 + ) + total_bytes = consumed_bytes + stored_bytes + if total_bytes > ENTRY_MAX_BYTES: + _fail() + return total_bytes + + +def _utc_text(value: datetime) -> str: + if value.tzinfo is None: + _fail() + return value.astimezone(timezone.utc).isoformat().replace("+00:00", "Z") + + +def _parse_datetime(value: object) -> datetime: + text_value = _safe_identifier(value) + try: + parsed = datetime.fromisoformat(text_value) + except ValueError as exc: + raise ProvenanceArchiveError("Invalid provenance archive") from exc + if parsed.tzinfo is None or _utc_text(parsed) != text_value: + _fail() + return parsed + + +def _confidence(value: float) -> str: + if not math.isfinite(value) or not 0.0 <= value <= 1.0: + _fail() + return repr(value) + + +def _parse_confidence(value: object) -> float: + if not isinstance(value, str): + _fail() + try: + parsed = float(value) + except ValueError as exc: + raise ProvenanceArchiveError("Invalid provenance archive") from exc + if not math.isfinite(parsed) or not 0.0 <= parsed <= 1.0: + _fail() + return parsed + + +def _attachment_core(attachment: Attachment, email_uid: str) -> dict[str, object]: + return { + "email_uid": email_uid, + "filename": attachment.filename, + "content": attachment.content, + "content_type": attachment.content_type, + "parse_status": attachment.parse_status, + "parse_content_type": attachment.parse_content_type, + "parser_key": attachment.parser_key, + "parse_error_code": attachment.parse_error_code, + } + + +def _attachment_payload_core(record: Mapping[str, object]) -> bytes: + return _canonical_json( + {key: value for key, value in record.items() if key != "attachment_uid"} + ) + + +def _attachment_uid(canonical: bytes, occurrence: int) -> str: + return ( + "attachment-" + + hashlib.sha256(canonical + b":" + str(occurrence).encode("ascii")).hexdigest() + ) + + +def _attachments_in_occurrence_order( + records: list[dict[str, object]], +) -> list[dict[str, object]]: + groups: dict[bytes, list[dict[str, object]]] = {} + for record in records: + groups.setdefault(_attachment_payload_core(record), []).append(record) + ordered: list[dict[str, object]] = [] + for canonical, group in sorted(groups.items()): + occurrences = { + _attachment_uid(canonical, occurrence): occurrence + for occurrence in range(1, len(group) + 1) + } + if set(occurrences) != {record["attachment_uid"] for record in group}: + _fail() + ordered.extend( + sorted(group, key=lambda record: occurrences[record["attachment_uid"]]) + ) + return ordered + + +def _is_admitted_attachment(attachment: Attachment) -> bool: + return ( + attachment.parse_status == "parsed" + and attachment.parser_key in _TEXTUAL_PARSER_KEYS + ) + + +def _attachment_records( + attachments: list[Attachment], email_uids: Mapping[int, str] +) -> tuple[list[dict[str, object]], dict[int, str]]: + admitted = [ + attachment + for attachment in attachments + if attachment.email_id in email_uids and _is_admitted_attachment(attachment) + ] + admitted.sort( + key=lambda item: ( + _canonical_json(_attachment_core(item, email_uids[item.email_id])), + item.id, + ) + ) + occurrence: dict[bytes, int] = {} + records: list[dict[str, object]] = [] + uid_by_id: dict[int, str] = {} + for attachment in admitted: + core = _attachment_core(attachment, email_uids[attachment.email_id]) + canonical = _canonical_json(core) + occurrence[canonical] = occurrence.get(canonical, 0) + 1 + uid = _attachment_uid(canonical, occurrence[canonical]) + uid_by_id[attachment.id] = uid + records.append({"attachment_uid": uid, **core}) + return sorted(records, key=lambda item: item["attachment_uid"]), uid_by_id + + +def _email_record(email: Email) -> dict[str, object]: + return { + "email_uid": email.message_id, + "thread_uid": email.thread_id, + "fingerprint": email.fingerprint, + "sender": email.sender, + "reply_to": email.reply_to, + "recipients": email.recipients, + "subject": email.subject, + "in_reply_to": email.in_reply_to, + "references": email.references, + "date": _utc_text(email.date), + "body": email.body, + "is_read": email.is_read, + } + + +def _portable_source_record_uid( + source_record_uid: str, + attachment_id: int | None, + attachment_uids: Mapping[int, str], +) -> str: + if attachment_id is None: + return source_record_uid + attachment_uid = attachment_uids.get(attachment_id) + if attachment_uid is None: + _fail() + return f"attachment:{attachment_uid}" + + +def _node_record( + node: ContentNodeRecord, + email_uids: Mapping[int, str], + attachment_uids: Mapping[int, str], +) -> dict[str, object]: + return { + "content_node_uid": node.content_node_uid, + "email_uid": email_uids[node.email_id], + "attachment_uid": attachment_uids.get(node.attachment_id), + "source_kind": node.source_kind, + "source_record_uid": _portable_source_record_uid( + node.source_record_uid, node.attachment_id, attachment_uids + ), + "parent_node_uid": node.parent_node_uid, + "node_kind": node.node_kind, + "node_path": node.node_path, + "ordinal_index": node.ordinal_index, + "display_label": node.display_label, + "safe_text_content": node.safe_text_content, + "content_hash": node.content_hash, + } + + +def _segment_record( + segment: ContentSegmentRecord, + email_uids: Mapping[int, str], + attachment_uids: Mapping[int, str], + node_uids: Mapping[int, str], +) -> dict[str, object]: + return { + "content_segment_uid": segment.content_segment_uid, + "email_uid": email_uids[segment.email_id], + "attachment_uid": attachment_uids.get(segment.attachment_id), + "content_node_uid": node_uids[segment.content_node_id], + "source_kind": segment.source_kind, + "source_record_uid": _portable_source_record_uid( + segment.source_record_uid, segment.attachment_id, attachment_uids + ), + "segment_kind": segment.segment_kind, + "segment_path": segment.segment_path, + "ordinal_index": segment.ordinal_index, + "heading_path": segment.heading_path, + "safe_text_content": segment.safe_text_content, + "content_hash": segment.content_hash, + "word_count": segment.word_count, + } + + +def _structural_edge_record( + edge: KnowledgeGraphEdgeRecord, + email_uids: Mapping[int, str], + attachment_uids: Mapping[int, str], + node_uids: Mapping[int, str], + segment_uids: Mapping[int, str], +) -> dict[str, object]: + return { + "edge_uid": edge.edge_uid, + "email_uid": email_uids[edge.email_id], + "attachment_uid": attachment_uids.get(edge.attachment_id), + "source_node_uid": node_uids.get(edge.source_node_id), + "target_node_uid": node_uids.get(edge.target_node_id), + "source_segment_uid": segment_uids.get(edge.source_segment_id), + "target_segment_uid": segment_uids.get(edge.target_segment_id), + "source_kind": edge.source_kind, + "source_record_uid": _portable_source_record_uid( + edge.source_record_uid, edge.attachment_id, attachment_uids + ), + "edge_kind": edge.edge_kind, + "edge_path": edge.edge_path, + "ordinal_index": edge.ordinal_index, + } + + +def _project_object_record( + project_object: ProjectGraphObjectRecord, + email_uids: Mapping[int, str], + attachment_uids: Mapping[int, str], + segment_uids: Mapping[int, str], + attachment_references: Mapping[str, str], +) -> dict[str, object]: + attributes_json = _portable_metadata( + project_object.attributes_json, attachment_references + ) + _validate_safe_metadata(attributes_json) + return { + "object_uid": project_object.object_uid, + "email_uid": email_uids[project_object.email_id], + "attachment_uid": attachment_uids.get(project_object.attachment_id), + "primary_content_segment_uid": segment_uids[ + project_object.primary_content_segment_id + ], + "object_type": project_object.object_type, + "title": project_object.title, + "summary": project_object.summary, + "status_code": project_object.status_code, + "confidence": _confidence(project_object.confidence), + "source_segment_uids": sorted(project_object.source_segment_uids), + "attributes_json": attributes_json, + "extractor_name": project_object.extractor_name, + "extractor_version": project_object.extractor_version, + } + + +def _project_edge_record( + edge: ProjectGraphEdgeRecord, + object_uids: Mapping[int, str], + segment_uids: Mapping[int, str], +) -> dict[str, object]: + return { + "edge_uid": edge.edge_uid, + "source_uid": edge.source_uid, + "target_uid": edge.target_uid, + "edge_type": edge.edge_type, + "confidence": _confidence(edge.confidence), + "source_segment_uids": sorted(edge.source_segment_uids), + "source_object_uid": object_uids.get(edge.source_object_id), + "target_object_uid": object_uids.get(edge.target_object_id), + "primary_content_segment_uid": segment_uids[edge.primary_content_segment_id], + } + + +def _correction_record( + correction: ProjectGraphCorrectionRecord, + object_uids: Mapping[int, str], + attachment_references: Mapping[str, str], +) -> dict[str, object]: + before_json = _portable_metadata(correction.before_json, attachment_references) + after_json = _portable_metadata(correction.after_json, attachment_references) + _validate_safe_metadata(before_json) + _validate_safe_metadata(after_json) + return { + "correction_uid": correction.correction_uid, + "object_uid": object_uids[correction.project_graph_object_id], + "correction_action": correction.correction_action, + "before_json": before_json, + "after_json": after_json, + "rationale": correction.rationale, + "source_segment_uids": sorted(correction.source_segment_uids), + "created_at": _utc_text(correction.created_at), + } + + +async def export_tenant_provenance( + session: AsyncSession, scope: TenantProvenanceScope +) -> bytes: + """Export the exact signed-scope project-evidence closure.""" + _validate_scope(scope) + bind = session.get_bind() + if getattr(getattr(bind, "dialect", None), "name", None) == "postgresql": + # ponytail: global lock bounds cross-worker archive memory; replace with + # file-backed streaming when the public export contract supports it. + await session.execute( + select(func.pg_advisory_xact_lock(_EXPORT_MEMORY_LOCK_KEY)) + ) + consumed_bytes = 0 + for model in ( + ProjectGraphObjectRecord, + ProjectGraphEdgeRecord, + ProjectGraphCorrectionRecord, + ): + consumed_bytes = await _preflight_export_rows( + session, + model, + _scope_filters(model, scope, workspace=True), + consumed_bytes, + ) + project_objects = list( + ( + await session.scalars( + select(ProjectGraphObjectRecord) + .where(*_scope_filters(ProjectGraphObjectRecord, scope, workspace=True)) + .order_by(ProjectGraphObjectRecord.object_uid) + ) + ).all() + ) + project_edges = list( + ( + await session.scalars( + select(ProjectGraphEdgeRecord) + .where(*_scope_filters(ProjectGraphEdgeRecord, scope, workspace=True)) + .order_by(ProjectGraphEdgeRecord.edge_uid) + ) + ).all() + ) + corrections = list( + ( + await session.scalars( + select(ProjectGraphCorrectionRecord) + .where( + *_scope_filters(ProjectGraphCorrectionRecord, scope, workspace=True) + ) + .order_by(ProjectGraphCorrectionRecord.correction_uid) + ) + ).all() + ) + cited_segment_uids = { + segment_uid + for record in (*project_objects, *project_edges, *corrections) + for segment_uid in record.source_segment_uids + } + for record in project_edges: + for endpoint_uid, endpoint_object_id in ( + (record.source_uid, record.source_object_id), + (record.target_uid, record.target_object_id), + ): + if endpoint_object_id is None: + cited_segment_uids.add(_segment_endpoint_uid(endpoint_uid)) + primary_segment_ids = { + record.primary_content_segment_id + for record in (*project_objects, *project_edges) + } + cited_size_filters = ( + (ContentSegmentRecord.content_segment_uid.in_(cited_segment_uids)) + | (ContentSegmentRecord.content_segment_id.in_(primary_segment_ids)), + ) + cited_segment_filters = ( + *cited_size_filters, + *_scope_filters(Email, scope, workspace=False), + ) + consumed_bytes = await _preflight_export_rows( + session, + ContentSegmentRecord, + cited_size_filters, + consumed_bytes, + ) + cited_segments = list( + ( + await session.scalars( + select(ContentSegmentRecord) + .join(Email, ContentSegmentRecord.email_id == Email.id) + .where(*cited_segment_filters) + ) + ).all() + ) + if not cited_segment_uids.issubset( + {segment.content_segment_uid for segment in cited_segments} + ) or not primary_segment_ids.issubset( + {segment.content_segment_id for segment in cited_segments} + ): + _fail() + segments_by_uid = { + segment.content_segment_uid: segment for segment in cited_segments + } + segments_by_id = {segment.content_segment_id: segment for segment in cited_segments} + objects_by_id = { + record.project_graph_object_id: record for record in project_objects + } + for record in project_objects: + if segments_by_id[ + record.primary_content_segment_id + ].email_id != record.email_id or any( + segments_by_uid[segment_uid].email_id != record.email_id + for segment_uid in record.source_segment_uids + ): + _fail() + for record in project_edges: + endpoint_objects = [] + for endpoint_id in (record.source_object_id, record.target_object_id): + if endpoint_id is None: + continue + endpoint_object = objects_by_id.get(endpoint_id) + if endpoint_object is None: + _fail() + endpoint_objects.append(endpoint_object) + if not endpoint_objects: + _fail() + endpoint_email_ids = {endpoint.email_id for endpoint in endpoint_objects} + for endpoint_uid, endpoint_object_id in ( + (record.source_uid, record.source_object_id), + (record.target_uid, record.target_object_id), + ): + if endpoint_object_id is None: + segment_uid = _segment_endpoint_uid(endpoint_uid) + if segments_by_uid[segment_uid].email_id not in endpoint_email_ids: + _fail() + if segments_by_id[ + record.primary_content_segment_id + ].email_id not in endpoint_email_ids or any( + segments_by_uid[segment_uid].email_id not in endpoint_email_ids + for segment_uid in record.source_segment_uids + ): + _fail() + for record in corrections: + project_object = objects_by_id.get(record.project_graph_object_id) + if project_object is None or any( + segments_by_uid[segment_uid].email_id != project_object.email_id + for segment_uid in record.source_segment_uids + ): + _fail() + email_ids = {record.email_id for record in project_objects} | { + segment.email_id for segment in cited_segments + } + email_filters = ( + Email.id.in_(email_ids), + *_scope_filters(Email, scope, workspace=False), + ) + consumed_bytes = await _preflight_export_rows( + session, Email, email_filters, consumed_bytes + ) + emails = list( + ( + await session.scalars( + select(Email) + .where(*email_filters) + .order_by(Email.message_id) + ) + ).all() + ) + if len(emails) != len(email_ids): + _fail() + email_uids = {email.id: email.message_id for email in emails} + + async def descendants( + model: Any, order_column: Any, *admission_filters: Any + ) -> list[Any]: + nonlocal consumed_bytes + descendant_filters = (model.email_id.in_(email_ids), *admission_filters) + consumed_bytes = await _preflight_export_rows( + session, model, descendant_filters, consumed_bytes + ) + return list( + ( + await session.scalars( + select(model) + .where(*descendant_filters) + .order_by(order_column) + ) + ).all() + ) + + attachments = await descendants( + Attachment, + Attachment.id, + Attachment.parse_status == "parsed", + Attachment.parser_key.in_(_TEXTUAL_PARSER_KEYS), + ) + nodes = await descendants(ContentNodeRecord, ContentNodeRecord.content_node_uid) + consumed_bytes = await _preflight_export_rows( + session, + ContentSegmentRecord, + ( + ContentSegmentRecord.email_id.in_(email_ids), + ContentSegmentRecord.content_segment_id.not_in( + {segment.content_segment_id for segment in cited_segments} + ), + ), + consumed_bytes, + ) + segments = list( + ( + await session.scalars( + select(ContentSegmentRecord) + .where(ContentSegmentRecord.email_id.in_(email_ids)) + .order_by(ContentSegmentRecord.content_segment_uid) + ) + ).all() + ) + structural_edges = await descendants( + KnowledgeGraphEdgeRecord, KnowledgeGraphEdgeRecord.edge_uid + ) + candidate_uids = { + *(row.content_node_uid for row in nodes), + *(row.content_segment_uid for row in segments), + *(row.edge_uid for row in structural_edges), + *(row.object_uid for row in project_objects), + *(row.edge_uid for row in project_edges), + *(row.correction_uid for row in corrections), + } + candidate_identity_rows = list( + ( + await session.scalars( + select(ProvenanceIdentityMapping).where( + ProvenanceIdentityMapping.target_database_uid.in_(candidate_uids) + ) + ) + ).all() + ) + identity_rows = [ + row + for row in candidate_identity_rows + if row.target_user_id == scope.user_id + and row.target_organization_id == scope.organization_id + and row.target_workspace_id == scope.workspace_id + ] + mapped_uids = { + collection: { + row.target_database_uid + for row in candidate_identity_rows + if row.entity_kind == collection + } + for collection in _REMAPPED_COLLECTIONS + } + target_mapped_uids = { + collection: { + row.target_database_uid + for row in identity_rows + if row.entity_kind == collection + } + for collection in _REMAPPED_COLLECTIONS + } + if identity_rows: + nodes_by_id = {row.content_node_id: row for row in nodes} + nodes_by_uid = {row.content_node_uid: row for row in nodes} + selected_node_uids = { + row.content_node_uid + for row in nodes + if row.content_node_uid in target_mapped_uids["content_nodes"] + } | {nodes_by_id[row.content_node_id].content_node_uid for row in cited_segments} + pending_node_uids = list(selected_node_uids) + while pending_node_uids: + parent_uid = nodes_by_uid[pending_node_uids.pop()].parent_node_uid + if parent_uid is not None and parent_uid not in selected_node_uids: + selected_node_uids.add(parent_uid) + pending_node_uids.append(parent_uid) + cited_segment_uid_set = {row.content_segment_uid for row in cited_segments} + nodes = [row for row in nodes if row.content_node_uid in selected_node_uids] + segments = [ + row + for row in segments + if row.content_segment_uid in target_mapped_uids["content_segments"] + or row.content_segment_uid in cited_segment_uid_set + ] + selected_node_ids = {row.content_node_id for row in nodes} + selected_segment_ids = {row.content_segment_id for row in segments} + structural_edges = [ + row + for row in structural_edges + if row.edge_uid in target_mapped_uids["structural_edges"] + or ( + (row.source_node_id is None or row.source_node_id in selected_node_ids) + and ( + row.target_node_id is None or row.target_node_id in selected_node_ids + ) + and ( + row.source_segment_id is None + or row.source_segment_id in selected_segment_ids + ) + and ( + row.target_segment_id is None + or row.target_segment_id in selected_segment_ids + ) + and any( + value is not None + for value in ( + row.source_node_id, + row.target_node_id, + row.source_segment_id, + row.target_segment_id, + ) + ) + ) + ] + else: + nodes = [ + row + for row in nodes + if row.content_node_uid not in mapped_uids["content_nodes"] + ] + segments = [ + row + for row in segments + if row.content_segment_uid not in mapped_uids["content_segments"] + ] + structural_edges = [ + row + for row in structural_edges + if row.edge_uid not in mapped_uids["structural_edges"] + ] + database_uids = { + "content_nodes": {row.content_node_uid for row in nodes}, + "content_segments": {row.content_segment_uid for row in segments}, + "structural_edges": {row.edge_uid for row in structural_edges}, + "project_objects": {row.object_uid for row in project_objects}, + "project_edges": {row.edge_uid for row in project_edges}, + "corrections": {row.correction_uid for row in corrections}, + } + relevant_identity_rows = [ + row + for row in identity_rows + if row.entity_kind in database_uids + and row.target_database_uid in database_uids[row.entity_kind] + ] + reverse_maps = {collection: {} for collection in _REMAPPED_COLLECTIONS} + source_scopes = { + ( + row.source_user_uid, + row.source_organization_uid, + row.source_workspace_uid, + ) + for row in relevant_identity_rows + } + can_restore_source_identity = bool(relevant_identity_rows) and len( + source_scopes + ) == 1 + if can_restore_source_identity: + for collection in _REMAPPED_COLLECTIONS: + collection_rows = [ + row for row in relevant_identity_rows if row.entity_kind == collection + ] + if {row.target_database_uid for row in collection_rows} != database_uids[ + collection + ]: + can_restore_source_identity = False + break + reverse_maps[collection] = { + row.target_database_uid: row.portable_uid for row in collection_rows + } + + if can_restore_source_identity: + source_user_uid, source_organization_uid, source_workspace_uid = next( + iter(source_scopes) + ) + else: + source_user_uid = _source_user_uid(scope.user_id) + source_organization_uid = scope.organization_id or "unscoped" + source_workspace_uid = scope.workspace_id + attachment_records, attachment_uids = _attachment_records(attachments, email_uids) + attachment_references = { + f"attachment-{attachment_id}": f"attachment:{attachment_uid}" + for attachment_id, attachment_uid in attachment_uids.items() + } + node_uids = {node.content_node_id: node.content_node_uid for node in nodes} + segment_uids = { + segment.content_segment_id: segment.content_segment_uid for segment in segments + } + object_uids = { + project_object.project_graph_object_id: project_object.object_uid + for project_object in project_objects + } + payload = { + "source_scope": { + "user_uid": source_user_uid, + "organization_uid": source_organization_uid, + "workspace_uid": source_workspace_uid, + }, + "emails": [_email_record(email) for email in emails], + "attachments": attachment_records, + "content_nodes": [ + _node_record(node, email_uids, attachment_uids) for node in nodes + ], + "content_segments": [ + _segment_record(segment, email_uids, attachment_uids, node_uids) + for segment in segments + ], + "structural_edges": [ + _structural_edge_record( + edge, email_uids, attachment_uids, node_uids, segment_uids + ) + for edge in structural_edges + ], + "project_objects": [ + _project_object_record( + project_object, + email_uids, + attachment_uids, + segment_uids, + attachment_references, + ) + for project_object in project_objects + ], + "project_edges": [ + _project_edge_record(edge, object_uids, segment_uids) + for edge in project_edges + ], + "corrections": [ + _correction_record(correction, object_uids, attachment_references) + for correction in corrections + ], + } + if can_restore_source_identity: + payload = _translate_identity_records(payload, reverse_maps) + records = { + "profile": "naruon-tenant-provenance/v1", + "schema_version": 1, + **payload, + "export_activity": { + "date_published": "1980-01-01T00:00:00Z", + }, + } + content_digest = _content_identity_digest(records) + records["bundle_uid"] = f"bundle-{content_digest}" + records["export_activity"]["activity_uid"] = f"export-{content_digest}" + _validate_record_graph(records) + return build_provenance_archive(records) + + +def _validate_scope(scope: TenantProvenanceScope) -> None: + if not isinstance(scope, TenantProvenanceScope): + _fail() + _safe_identifier(scope.user_id) + if scope.organization_id is not None: + _safe_identifier(scope.organization_id) + _safe_identifier(scope.workspace_id) + + +def _record_mapping(value: object, expected_keys: frozenset[str]) -> dict[str, object]: + if not isinstance(value, dict) or set(value) != expected_keys: + _fail() + return value + + +def _required_text(record: Mapping[str, object], key: str) -> str: + value = record.get(key) + if not isinstance(value, str): + _fail() + return value + + +def _optional_text(record: Mapping[str, object], key: str) -> str | None: + value = record.get(key) + if value is not None and not isinstance(value, str): + _fail() + return value + + +def _bounded_text( + record: Mapping[str, object], + key: str, + maximum: int, + *, + optional: bool = False, +) -> str | None: + value = _optional_text(record, key) if optional else _required_text(record, key) + if value is not None and len(value) > maximum: + _fail() + return value + + +def _uid( + record: Mapping[str, object], key: str, *, optional: bool = False +) -> str | None: + value = _optional_text(record, key) if optional else _required_text(record, key) + return None if value is None else _safe_identifier(value) + + +def _integer(record: Mapping[str, object], key: str, *, minimum: int = 0) -> int: + value = record.get(key) + if type(value) is not int or value < minimum: + _fail() + return value + + +def _uid_list(record: Mapping[str, object], key: str) -> list[str]: + value = record.get(key) + if not isinstance(value, list): + _fail() + validated = [_safe_identifier(item) for item in value] + if validated != sorted(set(validated)): + _fail() + return validated + + +def _collection_records( + records: Mapping[str, object], collection: str, uid_key: str +) -> tuple[list[dict[str, object]], dict[str, dict[str, object]]]: + values = records.get(collection) + if not isinstance(values, list): + _fail() + validated = [_record_mapping(value, _RECORD_KEYS[collection]) for value in values] + keys = [_safe_identifier(value.get(uid_key)) for value in validated] + if keys != sorted(set(keys)): + _fail() + return validated, dict(zip(keys, validated, strict=True)) + + +def _validate_record_scalars( + collections: Mapping[str, list[dict[str, object]]], +) -> None: + for email in collections["emails"]: + _uid(email, "email_uid") + for key in ( + "thread_uid", + "fingerprint", + "reply_to", + "recipients", + "subject", + "in_reply_to", + "references", + ): + _optional_text(email, key) + for key in ("sender", "body"): + _required_text(email, key) + _parse_datetime(email.get("date")) + if type(email.get("is_read")) is not bool: + _fail() + + for attachment in collections["attachments"]: + _uid(attachment, "attachment_uid") + _uid(attachment, "email_uid") + for key in ( + "filename", + "content", + "content_type", + "parse_status", + "parse_content_type", + "parser_key", + ): + _required_text(attachment, key) + _optional_text(attachment, "parse_error_code") + _bounded_text(attachment, "content_type", 120) + _bounded_text(attachment, "parse_status", 64) + _bounded_text(attachment, "parse_content_type", 120) + _bounded_text(attachment, "parser_key", 64) + _bounded_text(attachment, "parse_error_code", 120, optional=True) + if ( + attachment["parse_status"] != "parsed" + or attachment["parser_key"] not in _TEXTUAL_PARSER_KEYS + ): + _fail() + + for node in collections["content_nodes"]: + for key in ("content_node_uid", "email_uid"): + _uid(node, key) + for key in ("attachment_uid", "parent_node_uid"): + _uid(node, key, optional=True) + for key in ( + "source_kind", + "source_record_uid", + "node_kind", + "node_path", + "safe_text_content", + "content_hash", + ): + _required_text(node, key) + _optional_text(node, "display_label") + _bounded_text(node, "content_node_uid", 64) + _bounded_text(node, "source_kind", 64) + _bounded_text(node, "source_record_uid", 256) + _bounded_text(node, "parent_node_uid", 64, optional=True) + _bounded_text(node, "node_kind", 64) + _bounded_text(node, "node_path", 512) + _bounded_text(node, "display_label", 240, optional=True) + _bounded_text(node, "content_hash", 64) + _integer(node, "ordinal_index") + + for segment in collections["content_segments"]: + for key in ("content_segment_uid", "email_uid", "content_node_uid"): + _uid(segment, key) + _uid(segment, "attachment_uid", optional=True) + for key in ( + "source_kind", + "source_record_uid", + "segment_kind", + "segment_path", + "safe_text_content", + "content_hash", + ): + _required_text(segment, key) + _optional_text(segment, "heading_path") + _bounded_text(segment, "content_segment_uid", 64) + _bounded_text(segment, "source_kind", 64) + _bounded_text(segment, "source_record_uid", 256) + _bounded_text(segment, "segment_kind", 64) + _bounded_text(segment, "segment_path", 512) + _bounded_text(segment, "heading_path", 512, optional=True) + _bounded_text(segment, "content_hash", 64) + _integer(segment, "ordinal_index") + _integer(segment, "word_count") + + for edge in collections["structural_edges"]: + for key in ("edge_uid", "email_uid"): + _uid(edge, key) + for key in ( + "attachment_uid", + "source_node_uid", + "target_node_uid", + "source_segment_uid", + "target_segment_uid", + ): + _uid(edge, key, optional=True) + for key in ("source_kind", "source_record_uid", "edge_kind", "edge_path"): + _required_text(edge, key) + _bounded_text(edge, "edge_uid", 64) + _bounded_text(edge, "source_kind", 64) + _bounded_text(edge, "source_record_uid", 256) + _bounded_text(edge, "edge_kind", 64) + _bounded_text(edge, "edge_path", 512) + _integer(edge, "ordinal_index") + + for project_object in collections["project_objects"]: + for key in ( + "object_uid", + "email_uid", + "primary_content_segment_uid", + ): + _uid(project_object, key) + _uid(project_object, "attachment_uid", optional=True) + for key in ( + "object_type", + "title", + "summary", + "status_code", + "extractor_name", + "extractor_version", + ): + _required_text(project_object, key) + _bounded_text(project_object, "object_uid", 96) + _bounded_text(project_object, "object_type", 64) + _bounded_text(project_object, "title", 240) + _bounded_text(project_object, "status_code", 64) + _bounded_text(project_object, "extractor_name", 120) + _bounded_text(project_object, "extractor_version", 64) + _parse_confidence(project_object.get("confidence")) + _uid_list(project_object, "source_segment_uids") + if not isinstance(project_object.get("attributes_json"), dict): + _fail() + _validate_safe_metadata(project_object["attributes_json"]) + + for edge in collections["project_edges"]: + for key in ( + "edge_uid", + "source_uid", + "target_uid", + "primary_content_segment_uid", + ): + _uid(edge, key) + for key in ("source_object_uid", "target_object_uid"): + _uid(edge, key, optional=True) + _required_text(edge, "edge_type") + _bounded_text(edge, "edge_uid", 96) + _bounded_text(edge, "source_uid", 160) + _bounded_text(edge, "target_uid", 160) + _bounded_text(edge, "edge_type", 80) + _parse_confidence(edge.get("confidence")) + _uid_list(edge, "source_segment_uids") + + for correction in collections["corrections"]: + for key in ("correction_uid", "object_uid"): + _uid(correction, key) + _required_text(correction, "correction_action") + _bounded_text(correction, "correction_uid", 96) + _bounded_text(correction, "correction_action", 64) + _optional_text(correction, "rationale") + _uid_list(correction, "source_segment_uids") + if not isinstance(correction.get("before_json"), dict) or not isinstance( + correction.get("after_json"), dict + ): + _fail() + _validate_safe_metadata(correction["before_json"]) + _validate_safe_metadata(correction["after_json"]) + _parse_datetime(correction.get("created_at")) + + +def _validate_record_graph(records: Mapping[str, object]) -> None: + expected_top_level = { + "profile", + "schema_version", + "bundle_uid", + "source_scope", + "export_activity", + *_COLLECTIONS, + } + if set(records) != expected_top_level: + _fail() + _records_bundle_uid(records) + source_scope = records.get("source_scope") + if not isinstance(source_scope, dict) or set(source_scope) != { + "user_uid", + "organization_uid", + "workspace_uid", + }: + _fail() + source_user_uid = source_scope.get("user_uid") + if ( + not isinstance(source_user_uid, str) + or re.fullmatch(r"[0-9a-f]{64}", source_user_uid) is None + ): + _fail() + _safe_identifier(source_scope.get("organization_uid")) + _safe_identifier(source_scope.get("workspace_uid")) + activity = records.get("export_activity") + if not isinstance(activity, dict) or set(activity) != { + "activity_uid", + "date_published", + }: + _fail() + _safe_identifier(activity.get("activity_uid")) + _iso8601_date_or_datetime(activity.get("date_published")) + + uid_keys = { + "emails": "email_uid", + "attachments": "attachment_uid", + "content_nodes": "content_node_uid", + "content_segments": "content_segment_uid", + "structural_edges": "edge_uid", + "project_objects": "object_uid", + "project_edges": "edge_uid", + "corrections": "correction_uid", + } + collections: dict[str, list[dict[str, object]]] = {} + indexed: dict[str, dict[str, dict[str, object]]] = {} + for collection, uid_key in uid_keys.items(): + collections[collection], indexed[collection] = _collection_records( + records, collection, uid_key + ) + _validate_record_scalars(collections) + + email_uids = set(indexed["emails"]) + attachment_uids = set(indexed["attachments"]) + node_uids = set(indexed["content_nodes"]) + segment_uids = set(indexed["content_segments"]) + object_uids = set(indexed["project_objects"]) + attachment_email = { + record["attachment_uid"]: record["email_uid"] + for record in collections["attachments"] + } + _attachments_in_occurrence_order(collections["attachments"]) + node_email = { + record["content_node_uid"]: record["email_uid"] + for record in collections["content_nodes"] + } + segment_email = { + record["content_segment_uid"]: record["email_uid"] + for record in collections["content_segments"] + } + segment_attachment = { + record["content_segment_uid"]: record["attachment_uid"] + for record in collections["content_segments"] + } + object_email = { + record["object_uid"]: record["email_uid"] + for record in collections["project_objects"] + } + + def require_reference(value: object, available: set[str]) -> None: + if value is not None and value not in available: + _fail() + + for record in collections["attachments"]: + require_reference(record["email_uid"], email_uids) + for record in collections["content_nodes"]: + email_uid = record["email_uid"] + require_reference(email_uid, email_uids) + require_reference(record["attachment_uid"], attachment_uids) + require_reference(record["parent_node_uid"], node_uids) + if ( + record["attachment_uid"] is not None + and attachment_email[record["attachment_uid"]] != email_uid + ): + _fail() + if ( + record["attachment_uid"] is not None + and record["source_record_uid"] != f"attachment:{record['attachment_uid']}" + ): + _fail() + if ( + record["parent_node_uid"] is not None + and node_email[record["parent_node_uid"]] != email_uid + ): + _fail() + for record in collections["content_segments"]: + email_uid = record["email_uid"] + require_reference(email_uid, email_uids) + require_reference(record["attachment_uid"], attachment_uids) + require_reference(record["content_node_uid"], node_uids) + if node_email[record["content_node_uid"]] != email_uid: + _fail() + if ( + record["attachment_uid"] is not None + and attachment_email[record["attachment_uid"]] != email_uid + ): + _fail() + if ( + record["attachment_uid"] is not None + and record["source_record_uid"] != f"attachment:{record['attachment_uid']}" + ): + _fail() + for record in collections["structural_edges"]: + email_uid = record["email_uid"] + require_reference(email_uid, email_uids) + require_reference(record["attachment_uid"], attachment_uids) + if ( + record["attachment_uid"] is not None + and attachment_email[record["attachment_uid"]] != email_uid + ): + _fail() + if ( + record["attachment_uid"] is not None + and record["source_record_uid"] != f"attachment:{record['attachment_uid']}" + ): + _fail() + for key in ("source_node_uid", "target_node_uid"): + require_reference(record[key], node_uids) + if record[key] is not None and node_email[record[key]] != email_uid: + _fail() + for key in ("source_segment_uid", "target_segment_uid"): + require_reference(record[key], segment_uids) + if record[key] is not None and segment_email[record[key]] != email_uid: + _fail() + for record in collections["project_objects"]: + require_reference(record["email_uid"], email_uids) + require_reference(record["attachment_uid"], attachment_uids) + require_reference(record["primary_content_segment_uid"], segment_uids) + if segment_email[record["primary_content_segment_uid"]] != record["email_uid"]: + _fail() + if ( + record["attachment_uid"] is not None + and attachment_email[record["attachment_uid"]] != record["email_uid"] + ): + _fail() + if ( + segment_attachment[record["primary_content_segment_uid"]] + != record["attachment_uid"] + ): + _fail() + for segment_uid in record["source_segment_uids"]: + require_reference(segment_uid, segment_uids) + if segment_email[segment_uid] != record["email_uid"]: + _fail() + for record in collections["project_edges"]: + require_reference(record["source_object_uid"], object_uids) + require_reference(record["target_object_uid"], object_uids) + endpoint_uids = { + endpoint_uid + for endpoint_uid in ( + record["source_object_uid"], + record["target_object_uid"], + ) + if endpoint_uid is not None + } + if not endpoint_uids: + _fail() + endpoint_email_uids = { + object_email[endpoint_uid] for endpoint_uid in endpoint_uids + } + for field, object_field in ( + ("source_uid", "source_object_uid"), + ("target_uid", "target_object_uid"), + ): + if record[object_field] is not None: + if record[field] != record[object_field]: + _fail() + continue + segment_uid = _segment_endpoint_uid(record[field]) + require_reference(segment_uid, segment_uids) + if segment_email[segment_uid] not in endpoint_email_uids: + _fail() + require_reference(record["primary_content_segment_uid"], segment_uids) + if ( + segment_email[record["primary_content_segment_uid"]] + not in endpoint_email_uids + ): + _fail() + for segment_uid in record["source_segment_uids"]: + require_reference(segment_uid, segment_uids) + if segment_email[segment_uid] not in endpoint_email_uids: + _fail() + for record in collections["corrections"]: + require_reference(record["object_uid"], object_uids) + for segment_uid in record["source_segment_uids"]: + require_reference(segment_uid, segment_uids) + if segment_email[segment_uid] != object_email[record["object_uid"]]: + _fail() + + rooted_email_uids = { + record["email_uid"] for record in collections["project_objects"] + } + for record in collections["project_objects"]: + rooted_email_uids.add(segment_email[record["primary_content_segment_uid"]]) + rooted_email_uids.update( + segment_email[segment_uid] for segment_uid in record["source_segment_uids"] + ) + for record in collections["project_edges"]: + rooted_email_uids.add(segment_email[record["primary_content_segment_uid"]]) + rooted_email_uids.update( + segment_email[segment_uid] for segment_uid in record["source_segment_uids"] + ) + for record in collections["corrections"]: + rooted_email_uids.update( + segment_email[segment_uid] for segment_uid in record["source_segment_uids"] + ) + if email_uids and rooted_email_uids != email_uids: + _fail() + + +async def _matching_models( + session: AsyncSession, + model: Any, + column: Any, + values: set[str], + key_attribute: str, +) -> dict[str, Any]: + if not values: + return {} + rows = list((await session.scalars(select(model).where(column.in_(values)))).all()) + return {getattr(row, key_attribute): row for row in rows} + + +async def _prepare_identity_import( + session: AsyncSession, + scope: TenantProvenanceScope, + records: Mapping[str, object], +) -> tuple[dict[str, object], list[ProvenanceIdentityMapping]]: + source_scope = records["source_scope"] + same_scope = ( + source_scope["user_uid"] == _source_user_uid(scope.user_id) + and source_scope["organization_uid"] == (scope.organization_id or "unscoped") + and source_scope["workspace_uid"] == scope.workspace_id + ) + if same_scope: + return copy.deepcopy(records), [] + + forward_maps = {collection: {} for collection in _REMAPPED_COLLECTIONS} + for collection in _REMAPPED_COLLECTIONS: + for record in records[collection]: + portable_uid = record[_UID_KEYS[collection]] + forward_maps[collection][portable_uid] = _target_database_uid( + scope, source_scope, collection, portable_uid + ) + expected_mapping_keys = { + (collection, portable_uid) + for collection, collection_map in forward_maps.items() + for portable_uid in collection_map + } + organization_filter = ( + ProvenanceIdentityMapping.target_organization_id == scope.organization_id + if scope.organization_id is not None + else ProvenanceIdentityMapping.target_organization_id.is_(None) + ) + scoped_mapping_rows = list( + ( + await session.scalars( + select(ProvenanceIdentityMapping).where( + ProvenanceIdentityMapping.target_user_id == scope.user_id, + organization_filter, + ProvenanceIdentityMapping.target_workspace_id == scope.workspace_id, + ProvenanceIdentityMapping.source_user_uid + == source_scope["user_uid"], + ProvenanceIdentityMapping.source_organization_uid + == source_scope["organization_uid"], + ProvenanceIdentityMapping.source_workspace_uid + == source_scope["workspace_uid"], + ProvenanceIdentityMapping.entity_kind.in_(_REMAPPED_COLLECTIONS), + ) + ) + ).all() + ) + scoped_mappings = { + (row.entity_kind, row.portable_uid): row + for row in scoped_mapping_rows + if (row.entity_kind, row.portable_uid) in expected_mapping_keys + } + if scoped_mappings: + for (collection, portable_uid), row in scoped_mappings.items(): + if row.target_database_uid not in { + portable_uid, + forward_maps[collection][portable_uid], + }: + _fail() + forward_maps[collection][portable_uid] = row.target_database_uid + missing_mapping_keys = expected_mapping_keys - set(scoped_mappings) + return _translate_identity_records(records, forward_maps), [ + ProvenanceIdentityMapping( + target_user_id=scope.user_id, + target_organization_id=scope.organization_id, + target_workspace_id=scope.workspace_id, + source_user_uid=source_scope["user_uid"], + source_organization_uid=source_scope["organization_uid"], + source_workspace_uid=source_scope["workspace_uid"], + entity_kind=collection, + portable_uid=portable_uid, + target_database_uid=forward_maps[collection][portable_uid], + ) + for collection, portable_uid in sorted(missing_mapping_keys) + ] + + scoped_collisions: list[tuple[str, Any, str]] = [] + for model, column, collection in ( + ( + ProjectGraphObjectRecord, + ProjectGraphObjectRecord.object_uid, + "project_objects", + ), + (ProjectGraphEdgeRecord, ProjectGraphEdgeRecord.edge_uid, "project_edges"), + ( + ProjectGraphCorrectionRecord, + ProjectGraphCorrectionRecord.correction_uid, + "corrections", + ), + ): + portable_uids = { + record[_UID_KEYS[collection]] for record in records[collection] + } + if portable_uids: + scoped_collisions.extend( + (collection, row, getattr(row, _UID_KEYS[collection])) + for row in ( + await session.scalars(select(model).where(column.in_(portable_uids))) + ).all() + ) + if not scoped_collisions: + return copy.deepcopy(records), [ + ProvenanceIdentityMapping( + target_user_id=scope.user_id, + target_organization_id=scope.organization_id, + target_workspace_id=scope.workspace_id, + source_user_uid=source_scope["user_uid"], + source_organization_uid=source_scope["organization_uid"], + source_workspace_uid=source_scope["workspace_uid"], + entity_kind=collection, + portable_uid=portable_uid, + target_database_uid=portable_uid, + ) + for collection in _REMAPPED_COLLECTIONS + for portable_uid in forward_maps[collection] + ] + if any( + row.user_id == scope.user_id + and row.organization_id == scope.organization_id + and row.workspace_id == scope.workspace_id + for _, row, _ in scoped_collisions + ): + return copy.deepcopy(records), [] + native_source = all( + _source_user_uid(row.user_id) == source_scope["user_uid"] + and (row.organization_id or "unscoped") == source_scope["organization_uid"] + and row.workspace_id == source_scope["workspace_uid"] + for _, row, _ in scoped_collisions + ) + if not native_source: + collision_keys = { + (collection, portable_uid) + for collection, _, portable_uid in scoped_collisions + } + origin_rows = list( + ( + await session.scalars( + select(ProvenanceIdentityMapping).where( + ProvenanceIdentityMapping.target_database_uid.in_( + {portable_uid for _, portable_uid in collision_keys} + ) + ) + ) + ).all() + ) + origins = { + (row.entity_kind, row.target_database_uid): row + for row in origin_rows + if (row.entity_kind, row.target_database_uid) in collision_keys + } + if set(origins) != collision_keys or any( + row.source_user_uid != source_scope["user_uid"] + or row.source_organization_uid != source_scope["organization_uid"] + or row.source_workspace_uid != source_scope["workspace_uid"] + for row in origins.values() + ): + _fail() + + target_uids = { + database_uid + for collection_map in forward_maps.values() + for database_uid in collection_map.values() + } + existing_rows = list( + ( + await session.scalars( + select(ProvenanceIdentityMapping).where( + ProvenanceIdentityMapping.target_database_uid.in_(target_uids) + ) + ) + ).all() + ) + existing = {(row.entity_kind, row.portable_uid): row for row in existing_rows} + new_rows: list[ProvenanceIdentityMapping] = [] + for collection, collection_map in forward_maps.items(): + for portable_uid, database_uid in collection_map.items(): + row = existing.get((collection, portable_uid)) + expected = ( + scope.user_id, + scope.organization_id, + scope.workspace_id, + source_scope["user_uid"], + source_scope["organization_uid"], + source_scope["workspace_uid"], + database_uid, + ) + if row is not None: + actual = ( + row.target_user_id, + row.target_organization_id, + row.target_workspace_id, + row.source_user_uid, + row.source_organization_uid, + row.source_workspace_uid, + row.target_database_uid, + ) + if actual != expected: + _fail() + continue + new_rows.append( + ProvenanceIdentityMapping( + target_user_id=scope.user_id, + target_organization_id=scope.organization_id, + target_workspace_id=scope.workspace_id, + source_user_uid=source_scope["user_uid"], + source_organization_uid=source_scope["organization_uid"], + source_workspace_uid=source_scope["workspace_uid"], + entity_kind=collection, + portable_uid=portable_uid, + target_database_uid=database_uid, + ) + ) + return _translate_identity_records(records, forward_maps), new_rows + + +async def _preflight_existing( + session: AsyncSession, + scope: TenantProvenanceScope, + records: Mapping[str, object], +) -> dict[str, dict[str, Any]]: + payload = { + collection: { + record[ + { + "emails": "email_uid", + "attachments": "attachment_uid", + "content_nodes": "content_node_uid", + "content_segments": "content_segment_uid", + "structural_edges": "edge_uid", + "project_objects": "object_uid", + "project_edges": "edge_uid", + "corrections": "correction_uid", + }[collection] + ]: record + for record in records[collection] + } + for collection in _COLLECTIONS + } + email_values = set(payload["emails"]) + email_rows = list( + ( + await session.scalars( + select(Email).where( + Email.message_id.in_(email_values), + *_scope_filters(Email, scope, workspace=False), + ) + ) + ).all() + ) + models: dict[str, dict[str, Any]] = { + "emails": {row.message_id: row for row in email_rows} + } + email_uids = {row.id: row.message_id for row in email_rows} + attachment_rows = ( + list( + ( + await session.scalars( + select(Attachment).where(Attachment.email_id.in_(email_uids)) + ) + ).all() + ) + if email_uids + else [] + ) + attachment_records, attachment_uids = _attachment_records( + attachment_rows, email_uids + ) + attachment_references = { + f"attachment-{attachment_id}": f"attachment:{attachment_uid}" + for attachment_id, attachment_uid in attachment_uids.items() + } + attachments_by_id = {row.id: row for row in attachment_rows} + models["attachments"] = { + uid: attachments_by_id[attachment_id] + for attachment_id, uid in attachment_uids.items() + if uid in payload["attachments"] + } + existing_serialized: dict[str, dict[str, dict[str, object]]] = { + "emails": {uid: _email_record(row) for uid, row in models["emails"].items()}, + "attachments": { + record["attachment_uid"]: record + for record in attachment_records + if record["attachment_uid"] in payload["attachments"] + }, + } + + models["content_nodes"] = await _matching_models( + session, + ContentNodeRecord, + ContentNodeRecord.content_node_uid, + set(payload["content_nodes"]), + "content_node_uid", + ) + node_uids = { + row.content_node_id: row.content_node_uid + for row in models["content_nodes"].values() + } + models["content_segments"] = await _matching_models( + session, + ContentSegmentRecord, + ContentSegmentRecord.content_segment_uid, + set(payload["content_segments"]), + "content_segment_uid", + ) + segment_uids = { + row.content_segment_id: row.content_segment_uid + for row in models["content_segments"].values() + } + models["structural_edges"] = await _matching_models( + session, + KnowledgeGraphEdgeRecord, + KnowledgeGraphEdgeRecord.edge_uid, + set(payload["structural_edges"]), + "edge_uid", + ) + models["project_objects"] = await _matching_models( + session, + ProjectGraphObjectRecord, + ProjectGraphObjectRecord.object_uid, + set(payload["project_objects"]), + "object_uid", + ) + object_uids = { + row.project_graph_object_id: row.object_uid + for row in models["project_objects"].values() + } + models["project_edges"] = await _matching_models( + session, + ProjectGraphEdgeRecord, + ProjectGraphEdgeRecord.edge_uid, + set(payload["project_edges"]), + "edge_uid", + ) + models["corrections"] = await _matching_models( + session, + ProjectGraphCorrectionRecord, + ProjectGraphCorrectionRecord.correction_uid, + set(payload["corrections"]), + "correction_uid", + ) + + try: + existing_serialized["content_nodes"] = { + uid: _node_record(row, email_uids, attachment_uids) + for uid, row in models["content_nodes"].items() + } + existing_serialized["content_segments"] = { + uid: _segment_record(row, email_uids, attachment_uids, node_uids) + for uid, row in models["content_segments"].items() + } + existing_serialized["structural_edges"] = { + uid: _structural_edge_record( + row, email_uids, attachment_uids, node_uids, segment_uids + ) + for uid, row in models["structural_edges"].items() + } + existing_serialized["project_objects"] = { + uid: _project_object_record( + row, + email_uids, + attachment_uids, + segment_uids, + attachment_references, + ) + for uid, row in models["project_objects"].items() + } + existing_serialized["project_edges"] = { + uid: _project_edge_record(row, object_uids, segment_uids) + for uid, row in models["project_edges"].items() + } + existing_serialized["corrections"] = { + uid: _correction_record(row, object_uids, attachment_references) + for uid, row in models["corrections"].items() + } + except KeyError: + _fail() + + scoped_models = ( + models["project_objects"].values(), + models["project_edges"].values(), + models["corrections"].values(), + ) + for rows in scoped_models: + for row in rows: + if ( + row.user_id != scope.user_id + or row.organization_id != scope.organization_id + or row.workspace_id != scope.workspace_id + ): + _fail() + if any( + row.actor_user_id != scope.user_id for row in models["corrections"].values() + ): + _fail() + for collection in _COLLECTIONS: + for uid, existing in existing_serialized[collection].items(): + if existing != payload[collection].get(uid): + _fail() + return models + + +async def _insert_records( + session: AsyncSession, + scope: TenantProvenanceScope, + records: Mapping[str, object], + models: dict[str, dict[str, Any]], + created: dict[str, int], +) -> None: + for record in records["emails"]: + uid = record["email_uid"] + if uid in models["emails"]: + continue + email = Email( + user_id=scope.user_id, + organization_id=scope.organization_id, + message_id=uid, + thread_id=record["thread_uid"], + fingerprint=record["fingerprint"], + sender=record["sender"], + reply_to=record["reply_to"], + recipients=record["recipients"], + subject=record["subject"], + in_reply_to=record["in_reply_to"], + references=record["references"], + date=_parse_datetime(record["date"]), + body=record["body"], + is_read=record["is_read"], + embedding=None, + ) + session.add(email) + models["emails"][uid] = email + created["emails"] += 1 + await session.flush() + + for record in _attachments_in_occurrence_order(records["attachments"]): + uid = record["attachment_uid"] + if uid in models["attachments"]: + continue + attachment = Attachment( + email=models["emails"][record["email_uid"]], + filename=record["filename"], + content=record["content"], + content_type=record["content_type"], + parse_status=record["parse_status"], + parse_content_type=record["parse_content_type"], + parser_key=record["parser_key"], + parse_error_code=record["parse_error_code"], + embedding=None, + ) + session.add(attachment) + models["attachments"][uid] = attachment + created["attachments"] += 1 + await session.flush() + + for record in records["content_nodes"]: + uid = record["content_node_uid"] + if uid in models["content_nodes"]: + continue + node = ContentNodeRecord( + content_node_uid=uid, + email=models["emails"][record["email_uid"]], + attachment=( + models["attachments"].get(record["attachment_uid"]) + if record["attachment_uid"] is not None + else None + ), + source_kind=record["source_kind"], + source_record_uid=record["source_record_uid"], + parent_node_uid=record["parent_node_uid"], + node_kind=record["node_kind"], + node_path=record["node_path"], + ordinal_index=record["ordinal_index"], + display_label=record["display_label"], + safe_text_content=record["safe_text_content"], + content_hash=record["content_hash"], + ) + session.add(node) + models["content_nodes"][uid] = node + created["content_nodes"] += 1 + await session.flush() + + for record in records["content_segments"]: + uid = record["content_segment_uid"] + if uid in models["content_segments"]: + continue + segment = ContentSegmentRecord( + content_segment_uid=uid, + email=models["emails"][record["email_uid"]], + attachment=( + models["attachments"].get(record["attachment_uid"]) + if record["attachment_uid"] is not None + else None + ), + content_node=models["content_nodes"][record["content_node_uid"]], + source_kind=record["source_kind"], + source_record_uid=record["source_record_uid"], + segment_kind=record["segment_kind"], + segment_path=record["segment_path"], + ordinal_index=record["ordinal_index"], + heading_path=record["heading_path"], + safe_text_content=record["safe_text_content"], + content_hash=record["content_hash"], + word_count=record["word_count"], + ) + session.add(segment) + models["content_segments"][uid] = segment + created["content_segments"] += 1 + await session.flush() + + for record in records["structural_edges"]: + uid = record["edge_uid"] + if uid in models["structural_edges"]: + continue + edge = KnowledgeGraphEdgeRecord( + edge_uid=uid, + email=models["emails"][record["email_uid"]], + attachment=( + models["attachments"].get(record["attachment_uid"]) + if record["attachment_uid"] is not None + else None + ), + source_node=models["content_nodes"].get(record["source_node_uid"]), + target_node=models["content_nodes"].get(record["target_node_uid"]), + source_segment=models["content_segments"].get(record["source_segment_uid"]), + target_segment=models["content_segments"].get(record["target_segment_uid"]), + source_kind=record["source_kind"], + source_record_uid=record["source_record_uid"], + edge_kind=record["edge_kind"], + edge_path=record["edge_path"], + ordinal_index=record["ordinal_index"], + ) + session.add(edge) + models["structural_edges"][uid] = edge + created["structural_edges"] += 1 + await session.flush() + + for record in records["project_objects"]: + uid = record["object_uid"] + if uid in models["project_objects"]: + continue + project_object = ProjectGraphObjectRecord( + object_uid=uid, + user_id=scope.user_id, + organization_id=scope.organization_id, + workspace_id=scope.workspace_id, + email=models["emails"][record["email_uid"]], + attachment=( + models["attachments"].get(record["attachment_uid"]) + if record["attachment_uid"] is not None + else None + ), + primary_content_segment=models["content_segments"][ + record["primary_content_segment_uid"] + ], + object_type=record["object_type"], + title=record["title"], + summary=record["summary"], + status_code=record["status_code"], + confidence=_parse_confidence(record["confidence"]), + source_segment_uids=record["source_segment_uids"], + attributes_json=record["attributes_json"], + extractor_name=record["extractor_name"], + extractor_version=record["extractor_version"], + ) + session.add(project_object) + models["project_objects"][uid] = project_object + created["project_objects"] += 1 + await session.flush() + + for record in records["project_edges"]: + uid = record["edge_uid"] + if uid in models["project_edges"]: + continue + edge = ProjectGraphEdgeRecord( + edge_uid=uid, + user_id=scope.user_id, + organization_id=scope.organization_id, + workspace_id=scope.workspace_id, + source_uid=record["source_uid"], + target_uid=record["target_uid"], + edge_type=record["edge_type"], + confidence=_parse_confidence(record["confidence"]), + source_segment_uids=record["source_segment_uids"], + source_object=models["project_objects"].get(record["source_object_uid"]), + target_object=models["project_objects"].get(record["target_object_uid"]), + primary_content_segment=models["content_segments"][ + record["primary_content_segment_uid"] + ], + ) + session.add(edge) + models["project_edges"][uid] = edge + created["project_edges"] += 1 + await session.flush() + + for record in records["corrections"]: + uid = record["correction_uid"] + if uid in models["corrections"]: + continue + correction = ProjectGraphCorrectionRecord( + correction_uid=uid, + project_object=models["project_objects"][record["object_uid"]], + user_id=scope.user_id, + organization_id=scope.organization_id, + workspace_id=scope.workspace_id, + actor_user_id=scope.user_id, + correction_action=record["correction_action"], + before_json=record["before_json"], + after_json=record["after_json"], + rationale=record["rationale"], + source_segment_uids=record["source_segment_uids"], + created_at=_parse_datetime(record["created_at"]), + ) + session.add(correction) + models["corrections"][uid] = correction + created["corrections"] += 1 + await session.flush() + + +async def import_tenant_provenance( + session: AsyncSession, + scope: TenantProvenanceScope, + archive_bytes: bytes, +) -> ImportReceipt: + """Validate all closure and conflict rules before one transactional restore.""" + _validate_scope(scope) + records = parse_provenance_archive(archive_bytes) + _validate_record_graph(records) + if scope.organization_id is None and records["emails"]: + _fail() + created = {collection: 0 for collection in _COLLECTIONS} + try: + transaction = ( + session.begin_nested() if session.in_transaction() else session.begin() + ) + async with transaction: + bind = session.get_bind() + if getattr(getattr(bind, "dialect", None), "name", None) == "postgresql": + identity_lock_digest = hashlib.sha256( + _canonical_json( + { + "namespace": "tenant-provenance-portable-identities-v1", + "identities": { + collection: sorted( + record[_UID_KEYS[collection]] + for record in records[collection] + ) + for collection in _REMAPPED_COLLECTIONS + }, + } + ) + ).digest() + await session.execute( + select( + func.pg_advisory_xact_lock( + int.from_bytes(identity_lock_digest[:8], "big", signed=True) + ) + ) + ) + for email_uid in sorted( + record["email_uid"] for record in records["emails"] + ): + lock_digest = hashlib.sha256( + _canonical_json( + { + "namespace": "tenant-provenance-email-import-v1", + "target_owner": { + "user_uid": _source_user_uid(scope.user_id), + "organization_uid": scope.organization_id, + }, + "email_uid": email_uid, + } + ) + ).digest() + lock_key = int.from_bytes(lock_digest[:8], "big", signed=True) + await session.execute( + select(func.pg_advisory_xact_lock(lock_key)) + ) + database_records, identity_rows = await _prepare_identity_import( + session, scope, records + ) + models = await _preflight_existing(session, scope, database_records) + skipped = { + collection: len(models[collection]) for collection in _COLLECTIONS + } + await _insert_records(session, scope, database_records, models, created) + session.add_all(identity_rows) + await session.flush() + except ProvenanceArchiveError: + raise + except (DataError, IntegrityError, StatementError, TypeError, ValueError) as exc: + raise ProvenanceArchiveError("Invalid provenance archive") from exc + return ImportReceipt( + bundle_uid=_safe_identifier(records.get("bundle_uid")), + manifest_digest=hashlib.sha512(_canonical_json(records)).hexdigest(), + created=created, + skipped=skipped, + ) diff --git a/backend/tests/test_bootstrap_db.py b/backend/tests/test_bootstrap_db.py index 140ae56b4..b41b96106 100644 --- a/backend/tests/test_bootstrap_db.py +++ b/backend/tests/test_bootstrap_db.py @@ -207,13 +207,11 @@ def test_schema_backfill_adds_prompt_template_scope_columns_and_indexes(monkeypa for statement in statements ) assert any( - "alter table prompt_templates alter column prompt_uid set not null" - in statement + "alter table prompt_templates alter column prompt_uid set not null" in statement for statement in statements ) assert any( - "create unique index if not exists uq_prompt_templates_prompt_uid" - in statement + "create unique index if not exists uq_prompt_templates_prompt_uid" in statement for statement in statements ) assert any( @@ -301,8 +299,7 @@ def test_schema_backfill_creates_ai_hub_workflow_tables(monkeypatch): "ix_agent_run_records_scope_time" in statement for statement in statements ) assert any( - "ix_agent_run_records_workflow_uid" in statement - and "workflow_uid" in statement + "ix_agent_run_records_workflow_uid" in statement and "workflow_uid" in statement for statement in statements ) assert any( @@ -745,6 +742,51 @@ def test_schema_backfill_creates_connector_signal_events(): ) +@pytest.mark.asyncio +@pytest.mark.postgres +async def test_schema_backfill_creates_legacy_emails_index_when_table_exists(): + engine = create_async_engine(settings.DATABASE_URL) + try: + conn = await engine.connect() + except ( + ConnectionRefusedError, + OSError, + OperationalError, + asyncpg.CannotConnectNowError, + asyncpg.InvalidAuthorizationSpecificationError, + asyncpg.InvalidCatalogNameError, + asyncpg.InvalidPasswordError, + ): + await engine.dispose() + pytest.skip("PostgreSQL smoke path unavailable") + try: + async with conn.begin(): + await conn.execute(text("CREATE EXTENSION IF NOT EXISTS vector")) + await conn.run_sync(Base.metadata.create_all) + await conn.execute( + text( + "CREATE TEMP TABLE emails (" + "user_id varchar, organization_id varchar, date timestamptz" + ") ON COMMIT DROP" + ) + ) + await conn.run_sync(execute_schema_backfill) + await conn.run_sync(execute_schema_backfill) + result = await conn.execute( + text( + "SELECT indexname FROM pg_indexes " + "WHERE tablename = 'emails' " + "AND indexname = 'ix_emails_owner_date'" + ) + ) + assert result.scalar_one() == "ix_emails_owner_date" + finally: + try: + await conn.close() + finally: + await engine.dispose() + + @pytest.mark.asyncio @pytest.mark.postgres async def test_connector_signal_events_real_postgres_bootstrap_smoke(): @@ -771,14 +813,14 @@ async def test_connector_signal_events_real_postgres_bootstrap_smoke(): ) email_result = await conn.execute( text(""" - INSERT INTO email_records ( - user_id, organization_id, message_id, sender, recipients, - subject, "date", body, is_read - ) - VALUES ( - :user_id, :organization_id, :message_id, :sender, - :recipients, :subject, now(), :body, true - ) + INSERT INTO email_records ( + user_id, organization_id, message_id, sender, recipients, + subject, "date", body, is_read + ) + VALUES ( + :user_id, :organization_id, :message_id, :sender, + :recipients, :subject, now(), :body, false + ) RETURNING id """), { diff --git a/backend/tests/test_data_api.py b/backend/tests/test_data_api.py index dbacd69a6..04274cfe1 100644 --- a/backend/tests/test_data_api.py +++ b/backend/tests/test_data_api.py @@ -9,6 +9,7 @@ import asyncpg import httpx import pytest +from fastapi import HTTPException, Request from fastapi.testclient import TestClient from cryptography.fernet import Fernet from pydantic import SecretStr @@ -17,7 +18,7 @@ from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine import api.data as data_api -from api.auth import get_auth_context, get_current_user +from api.auth import AuthContext, get_auth_context, get_current_user from core.config import settings from db.models import ( get_fernet, @@ -32,6 +33,7 @@ ) from db.session import get_db from main import app +from services.tenant_provenance_bundle import ImportReceipt, ProvenanceArchiveError TEST_SESSION_HMAC_SECRET = "data-quality-surface-hmac-material-32-bytes" # noqa: S105 @@ -420,17 +422,426 @@ async def override_get_db(): return client, previous_secret, original_overrides +def _with_authoritative_auth(mock_db, auth_context: AuthContext): + async def override_get_db(): + yield mock_db + + original_overrides = dict(app.dependency_overrides) + app.dependency_overrides[get_db] = override_get_db + app.dependency_overrides[get_auth_context] = lambda: auth_context + app.dependency_overrides.pop(get_current_user, None) + return TestClient(app), original_overrides + + def _restore_overrides(previous_secret, original_overrides): settings.AUTH_SESSION_HMAC_SECRET = previous_secret app.dependency_overrides.clear() app.dependency_overrides.update(original_overrides) +def _restore_authoritative_overrides(original_overrides): + app.dependency_overrides.clear() + app.dependency_overrides.update(original_overrides) + + def _expected_sample_key(prefix: str, value: str) -> str: digest = hashlib.sha256(value.encode("utf-8")).hexdigest() return f"{prefix}_{digest[:16]}" +@pytest.mark.parametrize( + ("method", "path"), + [ + ("get", "/api/data/provenance-bundle"), + ("post", "/api/data/provenance-bundle/import"), + ], +) +def test_provenance_bundle_endpoints_require_signed_session(method, path, mock_db): + async def override_get_db(): + yield mock_db + + original_overrides = dict(app.dependency_overrides) + app.dependency_overrides[get_db] = override_get_db + app.dependency_overrides.pop(get_auth_context, None) + app.dependency_overrides.pop(get_current_user, None) + try: + with TestClient(app) as client: + response = client.request( + method.upper(), + path, + content=b"archive" if method == "post" else b"", + ) + finally: + app.dependency_overrides.clear() + app.dependency_overrides.update(original_overrides) + + assert response.status_code == 401 + assert response.json() == {"detail": "Authentication required"} + + +@pytest.mark.parametrize( + ("method", "path"), + [ + ("get", "/api/data/provenance-bundle"), + ("post", "/api/data/provenance-bundle/import"), + ], +) +def test_provenance_bundle_endpoints_reject_hmac_workspace_claims_before_service( + method, path, mock_db, monkeypatch +): + async def forbidden_service(*args, **kwargs): + raise AssertionError("provenance service must not run") + + monkeypatch.setattr(data_api, "export_tenant_provenance", forbidden_service) + monkeypatch.setattr(data_api, "import_tenant_provenance", forbidden_service) + token = _signed_session_token(_valid_session_payload()) + client, previous_secret, original_overrides = _with_signed_auth(mock_db, token) + try: + response = client.request( + method.upper(), + path, + content=b"attacker archive" if method == "post" else b"", + ) + finally: + client.close() + _restore_overrides(previous_secret, original_overrides) + + assert response.status_code == 403 + assert response.json() == { + "detail": "Authoritative workspace membership is required for provenance bundles" + } + + +@pytest.mark.parametrize("session_verifier", ["server", "override"]) +@pytest.mark.parametrize( + ("method", "path"), + [ + ("get", "/api/data/provenance-bundle"), + ("post", "/api/data/provenance-bundle/import"), + ], +) +def test_provenance_bundle_endpoints_reject_non_oidc_contexts_before_service( + session_verifier, method, path, mock_db, monkeypatch +): + async def forbidden_service(*args, **kwargs): + raise AssertionError("provenance service must not run") + + monkeypatch.setattr(data_api, "export_tenant_provenance", forbidden_service) + monkeypatch.setattr(data_api, "import_tenant_provenance", forbidden_service) + client, original_overrides = _with_authoritative_auth( + mock_db, + AuthContext( + user_id="unverified-user", + role="member", + organization_id="unverified-org", + group_ids=(), + workspace_id="unverified-workspace", + session_verifier=session_verifier, + ), + ) + try: + response = client.request( + method.upper(), + path, + content=b"attacker archive" if method == "post" else b"", + ) + finally: + client.close() + _restore_authoritative_overrides(original_overrides) + + assert response.status_code == 403 + assert response.json() == { + "detail": "Authoritative workspace membership is required for provenance bundles" + } + + +def test_provenance_bundle_download_uses_signed_scope_and_zip_response( + mock_db, monkeypatch +): + captured = {} + + async def fake_export(session, scope): + captured.update(session=session, scope=scope) + return b"PK\x03\x04bundle" + + monkeypatch.setattr(data_api, "export_tenant_provenance", fake_export) + client, original_overrides = _with_authoritative_auth( + mock_db, + AuthContext( + user_id="signed-user", + role="member", + organization_id="signed-org", + group_ids=(), + workspace_id="signed-workspace", + session_verifier="oidc", + ), + ) + try: + response = client.get("/api/data/provenance-bundle") + finally: + client.close() + _restore_authoritative_overrides(original_overrides) + + assert response.status_code == 200 + assert response.content == b"PK\x03\x04bundle" + assert response.headers["content-type"] == "application/zip" + assert response.headers["content-disposition"] == ( + 'attachment; filename="naruon-provenance.zip"' + ) + assert captured["session"] is mock_db + assert captured["scope"].user_id == "signed-user" + assert captured["scope"].organization_id == "signed-org" + assert captured["scope"].workspace_id == "signed-workspace" + + +def test_provenance_bundle_download_returns_fixed_safe_archive_error( + mock_db, monkeypatch +): + async def fake_export(session, scope): + raise ProvenanceArchiveError("private export detail") + + monkeypatch.setattr(data_api, "export_tenant_provenance", fake_export) + client, original_overrides = _with_authoritative_auth( + mock_db, + AuthContext( + user_id="admin", + role="member", + organization_id="org-acme", + group_ids=(), + workspace_id="workspace-org-acme", + session_verifier="oidc", + ), + ) + try: + response = client.get("/api/data/provenance-bundle") + finally: + client.close() + _restore_authoritative_overrides(original_overrides) + + assert response.status_code == 400 + assert response.json() == {"detail": "Invalid provenance archive"} + assert "private export detail" not in response.text + + +def test_provenance_bundle_import_rewrites_target_scope_from_signed_session( + mock_db, monkeypatch +): + captured = {} + + async def fake_import(session, scope, archive_bytes): + captured.update(session=session, scope=scope, archive_bytes=archive_bytes) + return ImportReceipt( + bundle_uid="bundle_portable_1", + manifest_digest="a" * 128, + created={"emails": 1}, + skipped={"emails": 0}, + ) + + monkeypatch.setattr(data_api, "import_tenant_provenance", fake_import) + client, original_overrides = _with_authoritative_auth( + mock_db, + AuthContext( + user_id="target-user", + role="member", + organization_id="target-org", + group_ids=(), + workspace_id="target-workspace", + session_verifier="oidc", + ), + ) + try: + response = client.post( + "/api/data/provenance-bundle/import", + content=b"PK\x03\x04portable", + headers={"Content-Type": "application/zip"}, + ) + finally: + client.close() + _restore_authoritative_overrides(original_overrides) + + assert response.status_code == 200 + assert response.json() == { + "bundle_uid": "bundle_portable_1", + "manifest_digest": "a" * 128, + "created": {"emails": 1}, + "skipped": {"emails": 0}, + } + assert captured["session"] is mock_db + assert captured["archive_bytes"] == b"PK\x03\x04portable" + assert captured["scope"].user_id == "target-user" + assert captured["scope"].organization_id == "target-org" + assert captured["scope"].workspace_id == "target-workspace" + + +def test_provenance_bundle_import_rejects_oversize_before_service_mutation( + mock_db, monkeypatch +): + called = False + + async def fake_import(session, scope, archive_bytes): + nonlocal called + called = True + + monkeypatch.setattr(data_api, "_PROVENANCE_ARCHIVE_MAX_BYTES", 4) + monkeypatch.setattr(data_api, "import_tenant_provenance", fake_import) + client, original_overrides = _with_authoritative_auth( + mock_db, + AuthContext( + user_id="admin", + role="member", + organization_id="org-acme", + group_ids=("group-data",), + workspace_id="workspace-org-acme", + session_verifier="oidc", + ), + ) + try: + response = client.post( + "/api/data/provenance-bundle/import", + content=b"12345", + headers={"Content-Type": "application/zip"}, + ) + finally: + client.close() + _restore_authoritative_overrides(original_overrides) + + assert response.status_code == 413 + assert response.json() == {"detail": "Provenance archive too large"} + assert called is False + + +@pytest.mark.parametrize("attacker_detail", ["bad local path", "signature secret"]) +def test_provenance_bundle_import_returns_fixed_safe_archive_errors( + attacker_detail, mock_db, monkeypatch +): + async def fake_import(session, scope, archive_bytes): + raise ProvenanceArchiveError(attacker_detail) + + monkeypatch.setattr(data_api, "import_tenant_provenance", fake_import) + client, original_overrides = _with_authoritative_auth( + mock_db, + AuthContext( + user_id="admin", + role="member", + organization_id="org-acme", + group_ids=("group-data",), + workspace_id="workspace-org-acme", + session_verifier="oidc", + ), + ) + try: + response = client.post( + "/api/data/provenance-bundle/import", + content=b"not-a-valid-archive", + headers={"Content-Type": "application/zip"}, + ) + finally: + client.close() + _restore_authoritative_overrides(original_overrides) + + assert response.status_code == 400 + assert response.json() == {"detail": "Invalid provenance archive"} + assert attacker_detail not in response.text + + +def _stream_request(*chunks: bytes, content_length: str | None = None) -> Request: + messages = [ + { + "type": "http.request", + "body": chunk, + "more_body": index < len(chunks) - 1, + } + for index, chunk in enumerate(chunks) + ] + + async def receive(): + return messages.pop(0) + + headers = [] + if content_length is not None: + headers.append((b"content-length", content_length.encode("ascii"))) + return Request( + { + "type": "http", + "http_version": "1.1", + "method": "POST", + "scheme": "http", + "path": "/api/data/provenance-bundle/import", + "raw_path": b"/api/data/provenance-bundle/import", + "query_string": b"", + "headers": headers, + "client": ("127.0.0.1", 1), + "server": ("testserver", 80), + }, + receive, + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("content_length", "status_code", "detail"), + [ + ("invalid", 400, "Invalid Content-Length"), + ("-1", 400, "Invalid Content-Length"), + ("5", 413, "Provenance archive too large"), + ], +) +async def test_provenance_archive_rejects_invalid_or_oversize_length_before_stream( + content_length, status_code, detail, monkeypatch +): + monkeypatch.setattr(data_api, "_PROVENANCE_ARCHIVE_MAX_BYTES", 4) + request = _stream_request(content_length=content_length) + + with pytest.raises(HTTPException) as exc_info: + await data_api._read_provenance_archive(request) + + assert exc_info.value.status_code == status_code + assert exc_info.value.detail == detail + + +@pytest.mark.asyncio +async def test_provenance_archive_checks_oversize_chunk_before_buffer_copy(monkeypatch): + class GuardedBuffer: + def __len__(self): + return 0 + + def extend(self, chunk): + raise AssertionError("oversize chunk was copied") + + monkeypatch.setattr(data_api, "_PROVENANCE_ARCHIVE_MAX_BYTES", 4) + monkeypatch.setattr(data_api, "bytearray", GuardedBuffer, raising=False) + + with pytest.raises(HTTPException) as exc_info: + await data_api._read_provenance_archive(_stream_request(b"12345")) + + assert exc_info.value.status_code == 413 + assert exc_info.value.detail == "Provenance archive too large" + + +@pytest.mark.asyncio +async def test_provenance_archive_accepts_absent_length_boundary_chunks(monkeypatch): + monkeypatch.setattr(data_api, "_PROVENANCE_ARCHIVE_MAX_BYTES", 4) + + archive = await data_api._read_provenance_archive( + _stream_request(b"12", b"34") + ) + + assert archive == b"1234" + + +@pytest.mark.asyncio +async def test_provenance_archive_rejects_streamed_body_over_max_bytes(monkeypatch): + monkeypatch.setattr(data_api, "_PROVENANCE_ARCHIVE_MAX_BYTES", 4) + + with pytest.raises(HTTPException) as exc_info: + await data_api._read_provenance_archive( + _stream_request(b"12", b"345", content_length="2") + ) + + assert exc_info.value.status_code == 413 + assert exc_info.value.detail == "Provenance archive too large" + + def _expected_acquisition_readiness_kpis(): return [ { diff --git a/backend/tests/test_email_read_state_migration_postgres.py b/backend/tests/test_email_read_state_migration_postgres.py index 782ee026c..5eb40b0ff 100644 --- a/backend/tests/test_email_read_state_migration_postgres.py +++ b/backend/tests/test_email_read_state_migration_postgres.py @@ -28,6 +28,7 @@ ContentSegmentRecord, Email, ProjectGraphObjectRecord, + ProvenanceIdentityMapping, ) pytestmark = pytest.mark.postgres @@ -164,6 +165,75 @@ async def test_upgrade_adds_is_read_to_a_historical_email_records_table( assert await _column_exists(fresh_database_url, "email_records", "is_read") is True +@pytest.mark.parametrize( + "starting_revision", + [ + "base", + "0017_merge_newsdom_carddav_heads", + "0018_provenance_identity", + "0019_email_read_state_repair", + ], +) +@pytest.mark.asyncio +async def test_provenance_upgrade_and_rollback_preserve_portable_identity( + fresh_database_url, starting_revision +): + """Catch ambiguous heads, skipped table creation, and destructive rollback.""" + mapping_table = ProvenanceIdentityMapping.__table__ + engine = create_async_engine(fresh_database_url) + try: + if starting_revision != "base": + _run_migrations(fresh_database_url, starting_revision) + if starting_revision != "0018_provenance_identity": + # Historical bootstrap predates this model. Force the incremental + # create-table path, not a pass from today's Base.create_all(). + async with engine.begin() as connection: + await connection.run_sync( + lambda sync_conn: mapping_table.drop(sync_conn, checkfirst=True) + ) + + _run_migrations(fresh_database_url) + async with engine.begin() as connection: + assert list( + await connection.scalars(text("SELECT version_num FROM alembic_version")) + ) == ["0021_merge_provenance_workspace"] + await connection.execute( + mapping_table.insert().values( + target_user_id="restore_target_user", + target_organization_id="restore_target_org", + target_workspace_id="restore_target_workspace", + source_user_uid="0" * 64, + source_organization_uid="restore_source_org", + source_workspace_uid="restore_source_workspace", + entity_kind="project_objects", + portable_uid="restore_source_object", + target_database_uid="restore_target_object", + ) + ) + + _run_migrations(fresh_database_url) + _run_downgrade(fresh_database_url, "0017_merge_newsdom_carddav_heads") + async with engine.connect() as connection: + assert await connection.run_sync( + lambda sync_conn: inspect(sync_conn).has_table(mapping_table.name) + ), "rollback destroyed imported portable-identity mappings" + assert ( + await connection.execute( + select(mapping_table.c.portable_uid, mapping_table.c.target_database_uid) + ) + ).all() == [("restore_source_object", "restore_target_object")] + + _run_migrations(fresh_database_url) + async with engine.connect() as connection: + assert ( + await connection.execute( + select(mapping_table.c.portable_uid, mapping_table.c.target_database_uid) + ) + ).all() == [("restore_source_object", "restore_target_object")] + finally: + await engine.dispose() + + @pytest.mark.asyncio async def test_upgrade_is_idempotent_against_a_legacy_emails_table_with_is_read( fresh_database_url, diff --git a/backend/tests/test_tenant_provenance_bundle.py b/backend/tests/test_tenant_provenance_bundle.py new file mode 100644 index 000000000..45976f8a7 --- /dev/null +++ b/backend/tests/test_tenant_provenance_bundle.py @@ -0,0 +1,3498 @@ +import copy +import asyncio +import datetime +import hashlib +import importlib.util +import io +import json +import random +import struct +import uuid +import warnings +import zipfile +from pathlib import Path + +import asyncpg +import pytest +import pytest_asyncio +from alembic.migration import MigrationContext +from alembic.operations import Operations +from sqlalchemy import delete, func, select, text +from sqlalchemy.exc import IntegrityError, OperationalError +from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine + +from core.config import settings +from db.models import ( + Attachment, + Base, + ContentNodeRecord, + ContentSegmentRecord, + Email, + KnowledgeGraphEdgeRecord, + ProjectGraphCorrectionRecord, + ProjectGraphEdgeRecord, + ProjectGraphObjectRecord, + ProvenanceIdentityMapping, +) +from services.project_graph.repository import ProjectGraphRepository +from services import tenant_provenance_bundle as provenance_service + +from services.tenant_provenance_bundle import ( + ARCHIVE_MAX_BYTES, + ARCHIVE_MAX_ENTRIES, + ENTRY_MAX_BYTES, + JSON_SAFE_INTEGER_MAX, + MAX_COMPRESSION_RATIO, + ImportReceipt, + ProvenanceArchiveError, + TenantProvenanceScope, + _within_archive_bounds, + build_provenance_archive, + export_tenant_provenance, + import_tenant_provenance, + parse_provenance_archive, +) + + +RECORDS = { + "profile": "naruon-tenant-provenance/v1", + "schema_version": 1, + "bundle_uid": "bundle-d046d1ed5d46714042ed93ee835abe93db3d56e548440e6b0eff69e5a808fc74", + "source_scope": { + "user_uid": "0" * 64, + "organization_uid": "org-01", + "workspace_uid": "ws-01", + }, + "export_activity": { + "activity_uid": "export-d046d1ed5d46714042ed93ee835abe93db3d56e548440e6b0eff69e5a808fc74", + "date_published": "1980-01-01T00:00:00Z", + }, + "emails": [{"email_uid": "email-01", "subject": "Evidence"}], + "attachments": [], + "content_nodes": [], + "content_segments": [], + "structural_edges": [], + "project_objects": [], + "project_edges": [], + "corrections": [], +} + +EXPECTED_ENTRIES = ( + "bag-info.txt", + "bagit.txt", + "data/records.json", + "manifest-sha512.txt", + "ro-crate-metadata.json", + "tagmanifest-sha512.txt", +) + + +def _canonical_json(value): + return json.dumps( + value, + ensure_ascii=False, + allow_nan=False, + separators=(",", ":"), + sort_keys=True, + ).encode("utf-8") + + +def _replace_entries(archive, replacements, *, extra=()): + with zipfile.ZipFile(io.BytesIO(archive), "r") as source: + entries = { + info.filename: source.read(info) + for info in source.infolist() + if not info.is_dir() + } + entries.update(replacements) + return _archive_with_entries(entries, extra=extra) + + +def _rebuild_manifests(entries): + payload_names = ("data/records.json",) + entries["manifest-sha512.txt"] = b"".join( + f"{hashlib.sha512(entries[name]).hexdigest()} {name}\n".encode("ascii") + for name in payload_names + ) + tag_names = ( + "bag-info.txt", + "bagit.txt", + "manifest-sha512.txt", + "ro-crate-metadata.json", + ) + entries["tagmanifest-sha512.txt"] = b"".join( + f"{hashlib.sha512(entries[name]).hexdigest()} {name}\n".encode("ascii") + for name in tag_names + ) + return entries + + +def _archive_with_entries(entries, *, extra=(), fixed_metadata=True): + output = io.BytesIO() + with zipfile.ZipFile(output, "w", compression=zipfile.ZIP_DEFLATED) as target: + for name, content in sorted(entries.items()): + if fixed_metadata: + info = zipfile.ZipInfo(name, date_time=(1980, 1, 1, 0, 0, 0)) + info.create_system = 3 + info.external_attr = 0o100644 << 16 + info.compress_type = zipfile.ZIP_DEFLATED + target.writestr( + info, content, compress_type=zipfile.ZIP_DEFLATED, compresslevel=9 + ) + else: + target.writestr(name, content) + for name, content in extra: + with warnings.catch_warnings(): + warnings.simplefilter("ignore", UserWarning) + target.writestr(name, content) + return output.getvalue() + + +def _archive_with_zip_metadata( + entries, *, archive_comment=b"", member_extra=b"", member_comment=b"" +): + output = io.BytesIO() + with zipfile.ZipFile(output, "w", compression=zipfile.ZIP_DEFLATED) as target: + for name, content in sorted(entries.items()): + info = zipfile.ZipInfo(name, date_time=(1980, 1, 1, 0, 0, 0)) + info.create_system = 3 + info.external_attr = 0o100644 << 16 + info.compress_type = zipfile.ZIP_DEFLATED + info.extra = member_extra + info.comment = member_comment + target.writestr( + info, content, compress_type=zipfile.ZIP_DEFLATED, compresslevel=9 + ) + target.comment = archive_comment + return output.getvalue() + + +def test_build_is_deterministic_and_has_exact_fixed_entries(): + first = build_provenance_archive(RECORDS) + second = build_provenance_archive(dict(RECORDS)) + + assert first == second + with zipfile.ZipFile(io.BytesIO(first), "r") as archive: + assert tuple(sorted(archive.namelist())) == EXPECTED_ENTRIES + assert all( + info.date_time == (1980, 1, 1, 0, 0, 0) for info in archive.infolist() + ) + assert all( + (info.external_attr >> 16) & 0o777 == 0o644 for info in archive.infolist() + ) + assert archive.read("data/records.json") == _canonical_json(RECORDS) + + +def test_parse_round_trips_records_and_verifies_ro_crate_metadata(): + archive = build_provenance_archive(RECORDS) + + assert parse_provenance_archive(archive) == RECORDS + with zipfile.ZipFile(io.BytesIO(archive), "r") as source: + crate = json.loads(source.read("ro-crate-metadata.json")) + assert crate["@context"] == [ + "https://w3id.org/ro/crate/1.3/context", + {"prov": "http://www.w3.org/ns/prov#"}, + ] + nodes = {node["@id"]: node for node in crate["@graph"]} + assert nodes["ro-crate-metadata.json"] == { + "@id": "ro-crate-metadata.json", + "@type": "CreativeWork", + "about": {"@id": "./"}, + "conformsTo": {"@id": "https://w3id.org/ro/crate/1.3"}, + } + assert nodes["./"]["datePublished"] == RECORDS["export_activity"]["date_published"] + activity_uid = RECORDS["export_activity"]["activity_uid"] + assert nodes[f"#{activity_uid}"]["prov:wasAssociatedWith"] == {"@id": "#naruon"} + assert "prov:SoftwareAgent" in nodes["#naruon"]["@type"] + + +@pytest.mark.parametrize( + "date_published", + ( + "not-a-date", + "2026-02-30", + "1980-01-01", + "1980-01-01T09:00:00+09:00", + "1980-01-01T00:00:00+00:00", + "1980-01-01T00:00:00.000000Z", + ), +) +def test_build_rejects_noncanonical_ro_crate_date_published(date_published): + with pytest.raises(ProvenanceArchiveError): + build_provenance_archive( + { + **RECORDS, + "export_activity": { + **RECORDS["export_activity"], + "date_published": date_published, + }, + } + ) + + +@pytest.mark.parametrize( + "date_published", + ( + "1980-01-01", + "1980-01-01T09:00:00+09:00", + "1980-01-01T00:00:00+00:00", + "1980-01-01T00:00:00.000000Z", + ), +) +@pytest.mark.asyncio +async def test_import_rejects_noncanonical_activity_timestamp_before_transaction( + date_published, +): + valid_archive = build_provenance_archive(RECORDS) + with zipfile.ZipFile(io.BytesIO(valid_archive), "r") as archive: + entries = {info.filename: archive.read(info) for info in archive.infolist()} + records = copy.deepcopy(RECORDS) + records["export_activity"]["date_published"] = date_published + payload = _canonical_json(records) + crate = json.loads(entries["ro-crate-metadata.json"]) + crate_nodes = {node["@id"]: node for node in crate["@graph"]} + crate_nodes["./"]["datePublished"] = date_published + crate_nodes["data/records.json"]["sha512"] = hashlib.sha512(payload).hexdigest() + entries["data/records.json"] = payload + entries["ro-crate-metadata.json"] = _canonical_json(crate) + invalid_archive = _archive_with_entries(_rebuild_manifests(entries)) + + class NoTransactionSession: + def in_transaction(self): + return False + + def begin(self): + raise AssertionError("transaction must not start") + + async def flush(self): + raise AssertionError("flush must not run") + + with pytest.raises(ProvenanceArchiveError): + await import_tenant_provenance( + NoTransactionSession(), + TenantProvenanceScope("target-user", "target-org", "target-workspace"), + invalid_archive, + ) + + +def test_build_preserves_valid_iso_ro_crate_date_published(): + archive = build_provenance_archive(RECORDS) + + assert ( + parse_provenance_archive(archive)["export_activity"]["date_published"] + == "1980-01-01T00:00:00Z" + ) + + +def test_parse_rejects_payload_tampering(): + archive = build_provenance_archive(RECORDS) + tampered = _replace_entries(archive, {"data/records.json": b'{"tampered":true}'}) + + with pytest.raises(ProvenanceArchiveError): + parse_provenance_archive(tampered) + + +def test_archive_entries_reject_reused_content_identifiers_for_different_records(): + rebuilt_records = copy.deepcopy(RECORDS) + rebuilt_records["emails"][0]["subject"] = "Different evidence" + + with pytest.raises(ProvenanceArchiveError): + provenance_service._archive_entries(rebuilt_records) + + +def test_build_rebinds_content_identifiers_for_different_records(): + rebuilt_records = copy.deepcopy(RECORDS) + rebuilt_records["emails"][0]["subject"] = "Different evidence" + + parsed = parse_provenance_archive(build_provenance_archive(rebuilt_records)) + + assert parsed["bundle_uid"] != RECORDS["bundle_uid"] + assert ( + parsed["export_activity"]["activity_uid"] + != RECORDS["export_activity"]["activity_uid"] + ) + + +@pytest.mark.parametrize( + "name", + ("manifest-sha512.txt", "tagmanifest-sha512.txt", "ro-crate-metadata.json"), +) +def test_parse_rejects_direct_tag_and_metadata_tampering(name): + archive = build_provenance_archive(RECORDS) + with zipfile.ZipFile(io.BytesIO(archive), "r") as source: + entries = {info.filename: source.read(info) for info in source.infolist()} + entries[name] += b"x" + + with pytest.raises(ProvenanceArchiveError): + parse_provenance_archive(_archive_with_entries(entries)) + + +@pytest.mark.parametrize("name", ("../data/records.json", "data\\records.json")) +def test_parse_rejects_unsafe_paths(name): + archive = build_provenance_archive(RECORDS) + unsafe = _replace_entries(archive, {}, extra=((name, b"{}"),)) + + with pytest.raises(ProvenanceArchiveError): + parse_provenance_archive(unsafe) + + +def test_parse_rejects_colliding_paths(): + archive = build_provenance_archive(RECORDS) + collision = _replace_entries(archive, {}, extra=(("data/records.json", b"{}"),)) + + with pytest.raises(ProvenanceArchiveError): + parse_provenance_archive(collision) + + +@pytest.mark.parametrize("missing_or_extra", ("bagit.txt", "unexpected.txt")) +def test_parse_rejects_missing_or_extra_entries(missing_or_extra): + archive = build_provenance_archive(RECORDS) + with zipfile.ZipFile(io.BytesIO(archive), "r") as source: + entries = {info.filename: source.read(info) for info in source.infolist()} + if missing_or_extra in entries: + del entries[missing_or_extra] + else: + entries[missing_or_extra] = b"unexpected" + + with pytest.raises(ProvenanceArchiveError): + parse_provenance_archive(_archive_with_entries(entries)) + + +def test_parse_rejects_duplicate_json_keys_even_with_valid_manifests(): + archive = build_provenance_archive(RECORDS) + with zipfile.ZipFile(io.BytesIO(archive), "r") as source: + entries = {info.filename: source.read(info) for info in source.infolist()} + entries["data/records.json"] = b'{"profile":"first","profile":"second"}' + + with pytest.raises(ProvenanceArchiveError): + parse_provenance_archive(_archive_with_entries(_rebuild_manifests(entries))) + + +@pytest.mark.parametrize("value", (float("nan"), float("inf"), float("-inf"))) +def test_build_rejects_non_finite_json_numbers(value): + records = {**RECORDS, "export_activity": {"score": value}} + + with pytest.raises(ProvenanceArchiveError): + build_provenance_archive(records) + + +def test_build_rejects_integer_outside_the_json_safe_range(): + with pytest.raises(ProvenanceArchiveError): + build_provenance_archive({**RECORDS, "unsupported": JSON_SAFE_INTEGER_MAX + 1}) + + +@pytest.mark.parametrize("value", (-0.0, 1e-7, 1.0, 0.73)) +def test_build_round_trips_finite_json_floats(value): + records = {**RECORDS, "finite_value": value} + + assert parse_provenance_archive(build_provenance_archive(records)) == records + + +def test_build_rejects_non_ascii_object_keys(): + with pytest.raises(ProvenanceArchiveError): + build_provenance_archive({**RECORDS, "\ufffd": "bmp", "\U0001f600": "non-bmp"}) + + +def test_build_rejects_an_unknown_profile(): + with pytest.raises(ProvenanceArchiveError): + build_provenance_archive({**RECORDS, "profile": "unknown"}) + + +def test_profile_rejects_boolean_schema_version_and_tag_injection(): + with pytest.raises(ProvenanceArchiveError): + build_provenance_archive({**RECORDS, "schema_version": True}) + with pytest.raises(ProvenanceArchiveError): + build_provenance_archive({**RECORDS, "bundle_uid": "bundle\nInjected: value"}) + with pytest.raises(ProvenanceArchiveError): + build_provenance_archive( + { + **RECORDS, + "export_activity": { + **RECORDS["export_activity"], + "activity_uid": "bad\ruid", + }, + } + ) + + +@pytest.mark.parametrize( + "records_json", + ( + b'{ "profile":"naruon-tenant-provenance/v1"}', + b'{"profile":"naruon-tenant-provenance/v1","schema_version":NaN}', + _canonical_json({**RECORDS, "profile": "unknown"}), + ), +) +def test_parse_rejects_noncanonical_nonfinite_and_unknown_profile(records_json): + archive = build_provenance_archive(RECORDS) + with zipfile.ZipFile(io.BytesIO(archive), "r") as source: + entries = {info.filename: source.read(info) for info in source.infolist()} + entries["data/records.json"] = records_json + + with pytest.raises(ProvenanceArchiveError): + parse_provenance_archive(_archive_with_entries(entries)) + + +def test_parse_rejects_nonfixed_zip_metadata(): + archive = build_provenance_archive(RECORDS) + with zipfile.ZipFile(io.BytesIO(archive), "r") as source: + entries = {info.filename: source.read(info) for info in source.infolist()} + + with pytest.raises(ProvenanceArchiveError): + parse_provenance_archive(_archive_with_entries(entries, fixed_metadata=False)) + + +@pytest.mark.parametrize( + "metadata", + ( + {"archive_comment": b"archive-comment"}, + {"member_extra": b"\x01\x00\x00\x00"}, + {"member_comment": b"member-comment"}, + ), +) +def test_parse_rejects_zip_comments_and_extra_fields(metadata): + archive = build_provenance_archive(RECORDS) + with zipfile.ZipFile(io.BytesIO(archive), "r") as source: + entries = {info.filename: source.read(info) for info in source.infolist()} + + with pytest.raises(ProvenanceArchiveError): + parse_provenance_archive(_archive_with_zip_metadata(entries, **metadata)) + + +@pytest.mark.parametrize( + "container_bytes", (b"leading-", b"trailing-unvalidated-bytes") +) +def test_parse_rejects_leading_and_trailing_container_bytes(container_bytes): + archive = build_provenance_archive(RECORDS) + candidate = ( + container_bytes + archive + if container_bytes == b"leading-" + else archive + container_bytes + ) + + with pytest.raises(ProvenanceArchiveError): + parse_provenance_archive(candidate) + + +@pytest.mark.parametrize( + "archive", + ( + lambda value: value[:-1], + lambda value: value[:-22] + b"truncated", + lambda value: value[:-22] + b"gap" + value[-22:], + ), +) +def test_parse_rejects_truncated_or_malformed_eocd(archive): + with pytest.raises(ProvenanceArchiveError): + parse_provenance_archive(archive(build_provenance_archive(RECORDS))) + + +def test_parse_rejects_gap_before_central_directory(): + archive = build_provenance_archive(RECORDS) + assert parse_provenance_archive(archive) == RECORDS + with zipfile.ZipFile(io.BytesIO(archive), "r") as source: + start_dir = source.start_dir + gap = b"unaccounted-gap" + candidate = bytearray(archive[:start_dir] + gap + archive[start_dir:]) + struct.pack_into(" None: + try: + async with engine.connect() as connection: + await connection.execute(text("SELECT 1")) + except ( + ConnectionRefusedError, + OSError, + OperationalError, + asyncpg.CannotConnectNowError, + asyncpg.InvalidAuthorizationSpecificationError, + asyncpg.InvalidCatalogNameError, + asyncpg.InvalidPasswordError, + ): + await engine.dispose() + pytest.skip("PostgreSQL smoke path unavailable") + + +async def _seed_provenance_closure( + session, + *, + scope: TenantProvenanceScope, + token: str, +) -> dict[str, object]: + now = datetime.datetime(2026, 8, 31, 12, 0, tzinfo=datetime.timezone.utc) + email = Email( + user_id=scope.user_id, + organization_id=scope.organization_id, + message_id=f"<{token}@example.com>", + thread_id=f"thread-{token}", + fingerprint=f"sha256:{token}", + sender="source@example.com", + reply_to="reply@example.com", + recipients="owner@example.com", + subject=f"Evidence {token}", + in_reply_to=None, + references=None, + date=now, + body="Grounded source evidence", + is_read=False, + embedding=[0.125] * 1536, + ) + session.add(email) + await session.flush() + + attachment = Attachment( + email=email, + filename="evidence.txt", + content="Parser-confirmed attachment evidence", + content_type="text/plain", + parse_status="parsed", + parse_content_type="text/plain", + parser_key="plain_text", + embedding=[0.25] * 1536, + ) + binary_attachment = Attachment( + email=email, + filename="source.pdf", + content="binary-payload-must-not-export", + content_type="application/pdf", + parse_status="pdf_dom_recognition_pending", + parse_content_type="application/pdf", + parser_key="pdf", + embedding=[0.5] * 1536, + ) + session.add_all([attachment, binary_attachment]) + await session.flush() + + node = ContentNodeRecord( + content_node_uid=f"node-{token}", + email=email, + attachment=attachment, + source_kind="attachment", + source_record_uid=f"attachment-source-{token}", + parent_node_uid=None, + node_kind="document", + node_path="/document[1]", + ordinal_index=1, + display_label="Evidence", + safe_text_content="Parser-confirmed attachment evidence", + content_hash=f"nodehash-{token}", + created_at=now, + ) + session.add(node) + await session.flush() + + segment = ContentSegmentRecord( + content_segment_uid=f"segment-{token}", + email=email, + attachment=attachment, + content_node=node, + source_kind="attachment", + source_record_uid=f"attachment-source-{token}", + segment_kind="paragraph", + segment_path="/document[1]/paragraph[1]", + ordinal_index=1, + heading_path="Evidence", + safe_text_content="Parser-confirmed attachment evidence", + content_hash=f"segmenthash-{token}", + word_count=3, + created_at=now, + ) + session.add(segment) + await session.flush() + + structural_edge = KnowledgeGraphEdgeRecord( + edge_uid=f"structural-edge-{token}", + email=email, + attachment=attachment, + source_node=node, + target_segment=segment, + source_kind="attachment", + source_record_uid=f"attachment-source-{token}", + edge_kind="contains", + edge_path="/document[1]->/document[1]/paragraph[1]", + ordinal_index=1, + created_at=now, + ) + source_object = ProjectGraphObjectRecord( + object_uid=f"project-object-source-{token}", + user_id=scope.user_id, + organization_id=scope.organization_id, + workspace_id=scope.workspace_id, + email=email, + attachment=attachment, + primary_content_segment=segment, + object_type="requirement", + title="Portable requirement", + summary="Grounded requirement", + status_code="accepted", + confidence=0.91, + source_segment_uids=[segment.content_segment_uid], + attributes_json={"rank_value": 1, "source_label": "mail"}, + extractor_name="test-extractor", + extractor_version="1", + created_at=now, + updated_at=now, + ) + target_object = ProjectGraphObjectRecord( + object_uid=f"project-object-target-{token}", + user_id=scope.user_id, + organization_id=scope.organization_id, + workspace_id=scope.workspace_id, + email=email, + attachment=attachment, + primary_content_segment=segment, + object_type="decision", + title="Portable decision", + summary="Grounded decision", + status_code="accepted", + confidence=0.87, + source_segment_uids=[segment.content_segment_uid], + attributes_json={"decision_state": "approved"}, + extractor_name="test-extractor", + extractor_version="1", + created_at=now, + updated_at=now, + ) + session.add_all([structural_edge, source_object, target_object]) + await session.flush() + + project_edge = ProjectGraphEdgeRecord( + edge_uid=f"project-edge-{token}", + user_id=scope.user_id, + organization_id=scope.organization_id, + workspace_id=scope.workspace_id, + source_uid=source_object.object_uid, + target_uid=target_object.object_uid, + edge_type="supports", + confidence=0.89, + source_segment_uids=[segment.content_segment_uid], + source_object=source_object, + target_object=target_object, + primary_content_segment=segment, + created_at=now, + ) + correction = ProjectGraphCorrectionRecord( + correction_uid=f"correction-{token}", + project_object=source_object, + user_id=scope.user_id, + organization_id=scope.organization_id, + workspace_id=scope.workspace_id, + actor_user_id=scope.user_id, + correction_action="accept", + before_json={"status_code": "candidate", "rank_value": 0}, + after_json={"status_code": "accepted", "rank_value": 1}, + rationale="Verified against source", + source_segment_uids=[segment.content_segment_uid], + created_at=now, + ) + session.add_all([project_edge, correction]) + await session.commit() + return { + "email_id": email.id, + "attachment_id": attachment.id, + "node_id": node.content_node_id, + "segment_id": segment.content_segment_id, + "project_object_id": source_object.project_graph_object_id, + "email_uid": email.message_id, + "object_uids": [source_object.object_uid, target_object.object_uid], + } + + +async def _delete_exported_closure(session, records: dict[str, object]) -> None: + emails = records["emails"] + email_uids = [record["email_uid"] for record in emails] + email_ids = list( + ( + await session.scalars( + select(Email.id).where(Email.message_id.in_(email_uids)) + ) + ).all() + ) + await session.execute( + delete(ProjectGraphCorrectionRecord).where( + ProjectGraphCorrectionRecord.correction_uid.in_( + [record["correction_uid"] for record in records["corrections"]] + ) + ) + ) + await session.execute( + delete(ProjectGraphEdgeRecord).where( + ProjectGraphEdgeRecord.edge_uid.in_( + [record["edge_uid"] for record in records["project_edges"]] + ) + ) + ) + await session.execute( + delete(ProjectGraphObjectRecord).where( + ProjectGraphObjectRecord.object_uid.in_( + [record["object_uid"] for record in records["project_objects"]] + ) + ) + ) + await session.execute( + delete(KnowledgeGraphEdgeRecord).where( + KnowledgeGraphEdgeRecord.email_id.in_(email_ids) + ) + ) + await session.execute( + delete(ContentSegmentRecord).where(ContentSegmentRecord.email_id.in_(email_ids)) + ) + await session.execute( + delete(ContentNodeRecord).where(ContentNodeRecord.email_id.in_(email_ids)) + ) + await session.execute(delete(Attachment).where(Attachment.email_id.in_(email_ids))) + await session.execute(delete(Email).where(Email.id.in_(email_ids))) + await session.commit() + + +async def _add_duplicate_text_attachments( + session, + *, + email_uid: str, + token: str, + count: int = 5, +) -> None: + email = await session.scalar(select(Email).where(Email.message_id == email_uid)) + now = datetime.datetime(2026, 8, 31, 12, 0, tzinfo=datetime.timezone.utc) + for index in range(1, count + 1): + attachment = Attachment( + email=email, + filename="evidence.txt", + content="Parser-confirmed attachment evidence", + content_type="text/plain", + parse_status="parsed", + parse_content_type="text/plain", + parser_key="plain_text", + embedding=[0.25] * 1536, + ) + session.add(attachment) + await session.flush() + node = ContentNodeRecord( + content_node_uid=f"duplicate-node-{index}-{token}", + email=email, + attachment=attachment, + source_kind="attachment", + source_record_uid=f"duplicate-source-{index}-{token}", + parent_node_uid=None, + node_kind="document", + node_path=f"/document[{index + 1}]", + ordinal_index=index + 1, + display_label=f"Evidence {index}", + safe_text_content=f"Duplicate evidence {index}", + content_hash=f"duplicate-node-hash-{index}-{token}", + created_at=now, + ) + session.add(node) + await session.flush() + session.add( + ContentSegmentRecord( + content_segment_uid=f"duplicate-segment-{index}-{token}", + email=email, + attachment=attachment, + content_node=node, + source_kind="attachment", + source_record_uid=f"duplicate-source-{index}-{token}", + segment_kind="paragraph", + segment_path=f"/document[{index + 1}]/paragraph[1]", + ordinal_index=index + 1, + heading_path=f"Evidence {index}", + safe_text_content=f"Duplicate evidence {index}", + content_hash=f"duplicate-segment-hash-{index}-{token}", + word_count=3, + created_at=now, + ) + ) + await session.commit() + + +async def _seed_email_graph_without_project_rows( + session, + *, + scope: TenantProvenanceScope, + token: str, +) -> dict[str, object]: + seeded = await _seed_provenance_closure(session, scope=scope, token=token) + await session.execute( + delete(ProjectGraphCorrectionRecord).where( + ProjectGraphCorrectionRecord.workspace_id == scope.workspace_id, + ProjectGraphCorrectionRecord.correction_uid == f"correction-{token}", + ) + ) + await session.execute( + delete(ProjectGraphEdgeRecord).where( + ProjectGraphEdgeRecord.workspace_id == scope.workspace_id, + ProjectGraphEdgeRecord.edge_uid == f"project-edge-{token}", + ) + ) + await session.execute( + delete(ProjectGraphObjectRecord).where( + ProjectGraphObjectRecord.object_uid.in_(seeded["object_uids"]) + ) + ) + await session.commit() + return seeded + + +async def _two_email_rooted_archive( + session, + *, + scope: TenantProvenanceScope, + token: str, +) -> bytes: + source = await _seed_provenance_closure(session, scope=scope, token=token) + cited = await _seed_provenance_closure( + session, + scope=scope, + token=f"cited-{token}", + ) + session.add( + Attachment( + email_id=source["email_id"], + filename="alternate-evidence.txt", + content="Alternate parser-confirmed attachment evidence", + content_type="text/plain", + parse_status="parsed", + parse_content_type="text/plain", + parser_key="plain_text", + embedding=None, + ) + ) + await session.commit() + archive = await export_tenant_provenance(session, scope) + records = parse_provenance_archive(archive) + assert cited["email_uid"] in {record["email_uid"] for record in records["emails"]} + return archive + + +def _scope(token: str) -> TenantProvenanceScope: + return TenantProvenanceScope( + user_id=f"user-{token}", + organization_id=f"org-{token}", + workspace_id=f"workspace-{token}", + ) + + +@pytest.mark.asyncio +@pytest.mark.postgres +async def test_postgres_round_trip_preserves_stable_evidence_with_fresh_keys( + provenance_sessionmaker, +): + token = uuid.uuid4().hex[:12] + source_scope = _scope(f"source-{token}") + target_scope = _scope(f"target-{token}") + async with provenance_sessionmaker() as session: + source_keys = await _seed_provenance_closure( + session, scope=source_scope, token=token + ) + async with provenance_sessionmaker() as session: + archive = await export_tenant_provenance(session, source_scope) + source_records = parse_provenance_archive(archive) + async with provenance_sessionmaker() as session: + await _delete_exported_closure(session, source_records) + async with provenance_sessionmaker() as session: + receipt = await import_tenant_provenance(session, target_scope, archive) + + assert isinstance(receipt, ImportReceipt) + assert receipt.created == { + "emails": 1, + "attachments": 1, + "content_nodes": 1, + "content_segments": 1, + "structural_edges": 1, + "project_objects": 2, + "project_edges": 1, + "corrections": 1, + } + async with provenance_sessionmaker() as session: + restored_archive = await export_tenant_provenance(session, target_scope) + restored_email = await session.scalar( + select(Email).where(Email.message_id == source_keys["email_uid"]) + ) + restored_object = await session.scalar( + select(ProjectGraphObjectRecord).where( + ProjectGraphObjectRecord.object_uid == source_keys["object_uids"][0] + ) + ) + restored_correction = await session.scalar( + select(ProjectGraphCorrectionRecord).where( + ProjectGraphCorrectionRecord.correction_uid == f"correction-{token}" + ) + ) + restored_attachment = await session.scalar( + select(Attachment).where(Attachment.email_id == restored_email.id) + ) + restored_node = await session.scalar( + select(ContentNodeRecord).where( + ContentNodeRecord.content_node_uid == f"node-{token}" + ) + ) + restored_segment = await session.scalar( + select(ContentSegmentRecord).where( + ContentSegmentRecord.content_segment_uid == f"segment-{token}" + ) + ) + restored_records = parse_provenance_archive(restored_archive) + for collection in ( + "emails", + "attachments", + "content_nodes", + "content_segments", + "structural_edges", + "project_objects", + "project_edges", + "corrections", + ): + assert restored_records[collection] == source_records[collection] + assert restored_email.id != source_keys["email_id"] + assert restored_attachment.id != source_keys["attachment_id"] + assert restored_node.content_node_id != source_keys["node_id"] + assert restored_segment.content_segment_id != source_keys["segment_id"] + assert restored_email.user_id == target_scope.user_id + assert restored_email.organization_id == target_scope.organization_id + assert restored_object.project_graph_object_id != source_keys["project_object_id"] + assert restored_object.user_id == target_scope.user_id + assert restored_object.organization_id == target_scope.organization_id + assert restored_object.workspace_id == target_scope.workspace_id + assert restored_correction.actor_user_id == target_scope.user_id + assert restored_correction.before_json == { + "status_code": "candidate", + "rank_value": 0, + } + assert restored_correction.after_json == { + "status_code": "accepted", + "rank_value": 1, + } + + +@pytest.mark.asyncio +@pytest.mark.postgres +async def test_export_rejects_oversized_stored_text_before_descendant_orm_load( + provenance_sessionmaker, + monkeypatch, +): + token = uuid.uuid4().hex[:12] + scope = _scope(f"oversized-export-{token}") + async with provenance_sessionmaker() as session: + seeded = await _seed_provenance_closure(session, scope=scope, token=token) + attachment = await session.get(Attachment, seeded["attachment_id"]) + attachment.content = "x" * (ENTRY_MAX_BYTES + 1) + await session.commit() + + async with provenance_sessionmaker() as session: + original_scalars = session.scalars + + async def reject_attachment_orm_load(statement, *args, **kwargs): + entities = { + description.get("entity") + for description in statement.column_descriptions + } + if Attachment in entities: + raise AssertionError("oversized attachment ORM row was materialized") + return await original_scalars(statement, *args, **kwargs) + + monkeypatch.setattr(session, "scalars", reject_attachment_orm_load) + with pytest.raises(ProvenanceArchiveError): + await export_tenant_provenance(session, scope) + + +@pytest.mark.asyncio +@pytest.mark.postgres +async def test_export_ignores_oversized_attachment_excluded_from_archive( + provenance_sessionmaker, +): + token = uuid.uuid4().hex[:12] + scope = _scope(f"excluded-oversized-{token}") + async with provenance_sessionmaker() as session: + seeded = await _seed_provenance_closure(session, scope=scope, token=token) + session.add( + Attachment( + email_id=seeded["email_id"], + filename="pending.bin", + content="x" * (ENTRY_MAX_BYTES + 1), + content_type="application/octet-stream", + parse_status="unsupported_content_type", + parser_key="unsupported_binary", + embedding=[0.0] * 1536, + ) + ) + await session.commit() + + async with provenance_sessionmaker() as session: + records = parse_provenance_archive( + await export_tenant_provenance(session, scope) + ) + + assert [row["filename"] for row in records["attachments"]] == [ + "evidence.txt" + ] + + +@pytest.mark.asyncio +@pytest.mark.postgres +async def test_export_counts_cited_segment_bytes_once(provenance_sessionmaker): + token = uuid.uuid4().hex[:12] + scope = _scope(f"large-cited-segment-{token}") + large_text = random.Random(0).randbytes( + ENTRY_MAX_BYTES // 8 + 2048 + ).hex() + async with provenance_sessionmaker() as session: + seeded = await _seed_provenance_closure(session, scope=scope, token=token) + segment = await session.get(ContentSegmentRecord, seeded["segment_id"]) + segment.safe_text_content = large_text + segment.content_hash = hashlib.sha256(large_text.encode()).hexdigest() + await session.commit() + + async with provenance_sessionmaker() as session: + records = parse_provenance_archive( + await export_tenant_provenance(session, scope) + ) + + assert records["content_segments"][0]["safe_text_content"] == large_text + + +@pytest.mark.asyncio +@pytest.mark.postgres +async def test_import_composes_with_transaction_started_by_prior_select( + provenance_sessionmaker, +): + token = uuid.uuid4().hex[:12] + source_scope = _scope(f"active-source-{token}") + target_scope = _scope(f"active-target-{token}") + async with provenance_sessionmaker() as session: + await _seed_provenance_closure(session, scope=source_scope, token=token) + async with provenance_sessionmaker() as session: + records = parse_provenance_archive( + await export_tenant_provenance(session, source_scope) + ) + archive = build_provenance_archive(records) + async with provenance_sessionmaker() as session: + await _delete_exported_closure(session, records) + + async with provenance_sessionmaker() as session: + await session.execute(select(func.count()).select_from(Email)) + assert session.in_transaction() + receipt = await import_tenant_provenance(session, target_scope, archive) + assert sum(receipt.created.values()) == 9 + assert await session.scalar( + select(func.count()).select_from(ProjectGraphObjectRecord).where( + ProjectGraphObjectRecord.workspace_id == target_scope.workspace_id + ) + ) == 2 + + +@pytest.mark.asyncio +@pytest.mark.postgres +async def test_active_transaction_import_failure_rolls_back_only_import_savepoint( + provenance_sessionmaker, + monkeypatch, +): + token = uuid.uuid4().hex[:12] + source_scope = _scope(f"rollback-source-{token}") + target_scope = _scope(f"rollback-target-{token}") + async with provenance_sessionmaker() as session: + await _seed_provenance_closure(session, scope=source_scope, token=token) + async with provenance_sessionmaker() as session: + records = parse_provenance_archive( + await export_tenant_provenance(session, source_scope) + ) + archive = build_provenance_archive(records) + async with provenance_sessionmaker() as session: + await _delete_exported_closure(session, records) + + original_insert = provenance_service._insert_records + + async def fail_after_insert(*args, **kwargs): + await original_insert(*args, **kwargs) + raise IntegrityError("forced import failure", {}, RuntimeError()) + + monkeypatch.setattr(provenance_service, "_insert_records", fail_after_insert) + async with provenance_sessionmaker() as session: + baseline_count = await session.scalar(select(func.count()).select_from(Email)) + assert session.in_transaction() + with pytest.raises(ProvenanceArchiveError): + await import_tenant_provenance(session, target_scope, archive) + assert session.in_transaction() + assert await session.scalar(select(func.count()).select_from(Email)) == baseline_count + assert await session.scalar( + select(func.count()).select_from(ProjectGraphObjectRecord).where( + ProjectGraphObjectRecord.workspace_id == target_scope.workspace_id + ) + ) == 0 + + +@pytest.mark.asyncio +@pytest.mark.postgres +async def test_same_database_cross_workspace_import_keeps_portable_identity( + provenance_sessionmaker, +): + token = uuid.uuid4().hex[:12] + source_scope = _scope(f"source-{token}") + target_scope = TenantProvenanceScope( + user_id=source_scope.user_id, + organization_id=source_scope.organization_id, + workspace_id=f"target-workspace-{token}", + ) + async with provenance_sessionmaker() as session: + source = await _seed_provenance_closure( + session, scope=source_scope, token=token + ) + source_object = await session.scalar( + select(ProjectGraphObjectRecord).where( + ProjectGraphObjectRecord.object_uid == source["object_uids"][0] + ) + ) + source_correction = await session.scalar( + select(ProjectGraphCorrectionRecord).where( + ProjectGraphCorrectionRecord.workspace_id == source_scope.workspace_id + ) + ) + portable_object_uid = source["object_uids"][0] + portable_segment_uid = f"segment-{token}" + source_object.attributes_json = { + "nested": [ + { + "source_object_uid": portable_object_uid, + "source_segment_uids": [portable_segment_uid], + "plain_text": portable_object_uid, + } + ] + } + source_correction.before_json = { + "object_uid": portable_object_uid, + "source_segment_uid": portable_segment_uid, + } + source_correction.after_json = { + "target_object_uid": portable_object_uid, + "primary_segment_uid": portable_segment_uid, + } + await session.commit() + async with provenance_sessionmaker() as session: + source_archive = await export_tenant_provenance(session, source_scope) + source_records = parse_provenance_archive(source_archive) + + async with provenance_sessionmaker() as session: + first = await import_tenant_provenance(session, target_scope, source_archive) + async with provenance_sessionmaker() as session: + second = await import_tenant_provenance(session, target_scope, source_archive) + async with provenance_sessionmaker() as session: + source_after = parse_provenance_archive( + await export_tenant_provenance(session, source_scope) + ) + target_after = parse_provenance_archive( + await export_tenant_provenance(session, target_scope) + ) + source_objects = list( + ( + await session.scalars( + select(ProjectGraphObjectRecord).where( + ProjectGraphObjectRecord.workspace_id + == source_scope.workspace_id + ) + ) + ).all() + ) + target_objects = list( + ( + await session.scalars( + select(ProjectGraphObjectRecord).where( + ProjectGraphObjectRecord.workspace_id + == target_scope.workspace_id + ) + ) + ).all() + ) + target_edge = await session.scalar( + select(ProjectGraphEdgeRecord).where( + ProjectGraphEdgeRecord.workspace_id == target_scope.workspace_id + ) + ) + target_correction = await session.scalar( + select(ProjectGraphCorrectionRecord).where( + ProjectGraphCorrectionRecord.workspace_id == target_scope.workspace_id + ) + ) + identity_mappings = list( + ( + await session.scalars( + select(ProvenanceIdentityMapping).where( + ProvenanceIdentityMapping.target_workspace_id + == target_scope.workspace_id + ) + ) + ).all() + ) + target_segment = await session.scalar( + select(ContentSegmentRecord).where( + ContentSegmentRecord.content_segment_uid + == target_edge.source_segment_uids[0] + ) + ) + + assert first.created == { + "emails": 0, + "attachments": 0, + "content_nodes": 1, + "content_segments": 1, + "structural_edges": 1, + "project_objects": 2, + "project_edges": 1, + "corrections": 1, + } + assert sum(second.created.values()) == 0 + assert second.skipped == { + collection: len(source_records[collection]) + for collection in ( + "emails", + "attachments", + "content_nodes", + "content_segments", + "structural_edges", + "project_objects", + "project_edges", + "corrections", + ) + } + for collection in ( + "emails", + "attachments", + "content_nodes", + "content_segments", + "structural_edges", + "project_objects", + "project_edges", + "corrections", + ): + assert source_after[collection] == source_records[collection] + assert target_after[collection] == source_records[collection] + assert {record.object_uid for record in source_objects} == set( + source["object_uids"] + ) + assert {record.object_uid for record in source_objects}.isdisjoint( + {record.object_uid for record in target_objects} + ) + target_object_uids = {record.object_uid for record in target_objects} + assert target_edge.source_uid in target_object_uids + assert target_edge.target_uid in target_object_uids + assert target_edge.source_object_id in { + record.project_graph_object_id for record in target_objects + } + assert target_edge.target_object_id in { + record.project_graph_object_id for record in target_objects + } + assert target_correction.project_graph_object_id in { + record.project_graph_object_id for record in target_objects + } + assert all( + record.primary_content_segment_id == target_edge.primary_content_segment_id + for record in target_objects + ) + assert len(identity_mappings) == 7 + assert {row.entity_kind for row in identity_mappings} == { + "content_nodes", + "content_segments", + "structural_edges", + "project_objects", + "project_edges", + "corrections", + } + assert all( + record.source_segment_uids == [target_edge.source_segment_uids[0]] + for record in target_objects + ) + assert target_correction.source_segment_uids == target_edge.source_segment_uids + target_object = next( + record + for record in target_objects + if record.object_uid == target_edge.source_uid + ) + nested_metadata = target_object.attributes_json["nested"][0] + assert nested_metadata["source_object_uid"] == target_object.object_uid + assert nested_metadata["source_segment_uids"] == [ + target_segment.content_segment_uid + ] + assert nested_metadata["plain_text"] == portable_object_uid + assert target_correction.before_json == { + "object_uid": target_object.object_uid, + "source_segment_uid": target_segment.content_segment_uid, + } + assert target_correction.after_json == { + "target_object_uid": target_object.object_uid, + "primary_segment_uid": target_segment.content_segment_uid, + } + + +@pytest.mark.asyncio +@pytest.mark.postgres +async def test_portable_archive_imports_into_two_scopes_after_source_deletion( + provenance_sessionmaker, +): + token = uuid.uuid4().hex[:12] + source_scope = _scope(f"deleted-source-{token}") + target_a = _scope(f"portable-target-a-{token}") + target_b = _scope(f"portable-target-b-{token}") + async with provenance_sessionmaker() as session: + await _seed_provenance_closure(session, scope=source_scope, token=token) + async with provenance_sessionmaker() as session: + archive = await export_tenant_provenance(session, source_scope) + source_records = parse_provenance_archive(archive) + + async with provenance_sessionmaker() as session: + email_id = await session.scalar( + select(Email.id).where(Email.message_id == source_records["emails"][0]["email_uid"]) + ) + assert email_id is not None + for model in ( + ProjectGraphCorrectionRecord, + ProjectGraphEdgeRecord, + ProjectGraphObjectRecord, + ): + await session.execute( + delete(model).where(model.workspace_id == source_scope.workspace_id) + ) + for model in ( + KnowledgeGraphEdgeRecord, + ContentSegmentRecord, + ContentNodeRecord, + Attachment, + ): + await session.execute(delete(model).where(model.email_id == email_id)) + await session.execute(delete(Email).where(Email.id == email_id)) + await session.commit() + + async with provenance_sessionmaker() as session: + await import_tenant_provenance(session, target_a, archive) + async with provenance_sessionmaker() as session: + await import_tenant_provenance(session, target_b, archive) + + async with provenance_sessionmaker() as session: + target_a_records = parse_provenance_archive( + await export_tenant_provenance(session, target_a) + ) + target_b_records = parse_provenance_archive( + await export_tenant_provenance(session, target_b) + ) + mappings = list( + ( + await session.scalars( + select(ProvenanceIdentityMapping).where( + ProvenanceIdentityMapping.target_workspace_id.in_( + (target_a.workspace_id, target_b.workspace_id) + ) + ) + ) + ).all() + ) + + for collection in ( + "emails", + "attachments", + "content_nodes", + "content_segments", + "structural_edges", + "project_objects", + "project_edges", + "corrections", + ): + assert target_a_records[collection] == source_records[collection] + assert target_b_records[collection] == source_records[collection] + assert {row.target_workspace_id for row in mappings} == { + target_a.workspace_id, + target_b.workspace_id, + } + + +@pytest.mark.asyncio +@pytest.mark.postgres +async def test_portable_archive_incrementally_adds_identity_mappings( + provenance_sessionmaker, +): + token = uuid.uuid4().hex[:12] + source_scope = _scope(f"incremental-source-{token}") + target_scope = _scope(f"incremental-target-{token}") + async with provenance_sessionmaker() as session: + await _seed_provenance_closure( + session, scope=source_scope, token=f"first-{token}" + ) + async with provenance_sessionmaker() as session: + first_archive = await export_tenant_provenance(session, source_scope) + async with provenance_sessionmaker() as session: + await import_tenant_provenance(session, target_scope, first_archive) + + async with provenance_sessionmaker() as session: + await _seed_provenance_closure( + session, scope=source_scope, token=f"second-{token}" + ) + async with provenance_sessionmaker() as session: + expanded_archive = await export_tenant_provenance(session, source_scope) + expanded_records = parse_provenance_archive(expanded_archive) + async with provenance_sessionmaker() as session: + receipt = await import_tenant_provenance( + session, target_scope, expanded_archive + ) + async with provenance_sessionmaker() as session: + target_records = parse_provenance_archive( + await export_tenant_provenance(session, target_scope) + ) + + assert sum(receipt.created.values()) > 0 + for collection in provenance_service._COLLECTIONS: + assert target_records[collection] == expanded_records[collection] + + +@pytest.mark.asyncio +@pytest.mark.postgres +async def test_mixed_native_and_multiple_import_origins_export_as_target_scope( + provenance_sessionmaker, +): + token = uuid.uuid4().hex[:12] + target_scope = _scope(f"target-{token}") + source_scopes = [_scope(f"source-{index}-{token}") for index in range(2)] + async with provenance_sessionmaker() as session: + native = await _seed_provenance_closure( + session, scope=target_scope, token=f"native-{token}" + ) + native_node = await session.get(ContentNodeRecord, native["node_id"]) + parent_uid = f"parent-node-native-{token}" + session.add( + ContentNodeRecord( + content_node_uid=parent_uid, + email_id=native["email_id"], + attachment_id=native["attachment_id"], + source_kind="attachment", + source_record_uid=f"attachment-source-native-{token}", + parent_node_uid=None, + node_kind="document", + node_path="/", + ordinal_index=0, + display_label="Parent", + safe_text_content="Parent evidence", + content_hash=f"parenthash-{token}", + ) + ) + native_node.parent_node_uid = parent_uid + await session.commit() + sources = [ + await _seed_provenance_closure( + session, scope=scope, token=f"source-{index}-{token}" + ) + for index, scope in enumerate(source_scopes) + ] + archives = [] + async with provenance_sessionmaker() as session: + for scope in source_scopes: + archives.append(await export_tenant_provenance(session, scope)) + for archive in archives: + async with provenance_sessionmaker() as session: + await import_tenant_provenance(session, target_scope, archive) + async with provenance_sessionmaker() as session: + records = parse_provenance_archive( + await export_tenant_provenance(session, target_scope) + ) + + assert records["source_scope"] == { + "user_uid": hashlib.sha256(target_scope.user_id.encode()).hexdigest(), + "organization_uid": target_scope.organization_id, + "workspace_uid": target_scope.workspace_id, + } + assert { + collection: len(records[collection]) + for collection in ( + "emails", + "attachments", + "content_nodes", + "content_segments", + "structural_edges", + "project_objects", + "project_edges", + "corrections", + ) + } == { + "emails": 3, + "attachments": 3, + "content_nodes": 4, + "content_segments": 3, + "structural_edges": 3, + "project_objects": 6, + "project_edges": 3, + "corrections": 3, + } + serialized = json.dumps(records, sort_keys=True) + assert native["email_uid"] in serialized + assert parent_uid in serialized + assert all(source["email_uid"] in serialized for source in sources) + + +@pytest.mark.parametrize( + "source_user_uid", + ("a" * 63, "A" * 64, "g" * 64, "0" * 65), +) +@pytest.mark.asyncio +async def test_import_rejects_invalid_source_user_uid_before_transaction( + source_user_uid, +): + records = copy.deepcopy(RECORDS) + records["source_scope"]["user_uid"] = source_user_uid + archive = build_provenance_archive(records) + flush_count = 0 + + class NoTransactionSession: + def begin(self): + raise AssertionError("transaction must not start") + + async def flush(self): + nonlocal flush_count + flush_count += 1 + + with pytest.raises(ProvenanceArchiveError): + await import_tenant_provenance( + NoTransactionSession(), + TenantProvenanceScope("target-user", "target-org", "target-workspace"), + archive, + ) + assert flush_count == 0 + + +@pytest.mark.asyncio +@pytest.mark.postgres +@pytest.mark.parametrize( + "incompatible", [False, True], ids=["identical", "incompatible"] +) +async def test_concurrent_same_database_imports_preserve_winner( + monkeypatch, incompatible +): + engine = create_async_engine(settings.DATABASE_URL, echo=False) + await _require_direct_postgres(engine) + session_factory = async_sessionmaker(engine, expire_on_commit=False) + token = uuid.uuid4().hex[:12] + source_scope = _scope(f"concurrent-source-{token}") + target_scope = TenantProvenanceScope( + user_id=source_scope.user_id, + organization_id=source_scope.organization_id, + workspace_id=f"concurrent-target-{token}", + ) + try: + async with engine.begin() as connection: + await connection.run_sync(Base.metadata.create_all) + async with session_factory() as session: + await _seed_provenance_closure(session, scope=source_scope, token=token) + async with session_factory() as session: + archive = await export_tenant_provenance(session, source_scope) + + records = parse_provenance_archive(archive) + second_records = copy.deepcopy(records) + if incompatible: + second_records["project_objects"][0]["title"] = "Conflicting requirement" + archives = (archive, build_provenance_archive(second_records)) + overlap = asyncio.Barrier(2) + + async def run_import(target_archive): + async with session_factory() as session: + execute = session.execute + first_statement = True + + async def overlapping_execute(*args, **kwargs): + nonlocal first_statement + if first_statement: + first_statement = False + # Both transactions must be live before either takes import locks. + await execute(select(1)) + await asyncio.wait_for(overlap.wait(), timeout=10) + return await execute(*args, **kwargs) + + monkeypatch.setattr(session, "execute", overlapping_execute) + return await import_tenant_provenance( + session, target_scope, target_archive + ) + + outcomes = await asyncio.wait_for( + asyncio.gather( + *(run_import(target_archive) for target_archive in archives), + return_exceptions=True, + ), + timeout=30, + ) + receipts = [result for result in outcomes if isinstance(result, ImportReceipt)] + if incompatible: + # Regression: a conflicting retry overwrites the winner or leaks partial rows. + assert len(receipts) == 1 + assert ( + sum(isinstance(result, ProvenanceArchiveError) for result in outcomes) + == 1 + ) + assert sum(receipts[0].created.values()) == 7 + assert sum(receipts[0].skipped.values()) == 2 + else: + assert len(receipts) == 2 + first, second = receipts + created_totals = sorted( + (sum(first.created.values()), sum(second.created.values())) + ) + assert created_totals == [0, 7] + assert sorted( + (sum(first.skipped.values()), sum(second.skipped.values())) + ) == [ + 2, + 9, + ] + winner_records = parse_provenance_archive( + archives[ + outcomes.index(next(r for r in receipts if sum(r.created.values()))) + ] + ) + graph_collections = [ + ("content_nodes", ContentNodeRecord, "content_node_uid"), + ("content_segments", ContentSegmentRecord, "content_segment_uid"), + ("structural_edges", KnowledgeGraphEdgeRecord, "edge_uid"), + ("project_objects", ProjectGraphObjectRecord, "object_uid"), + ("project_edges", ProjectGraphEdgeRecord, "edge_uid"), + ("corrections", ProjectGraphCorrectionRecord, "correction_uid"), + ] + async with session_factory() as session: + target_records = parse_provenance_archive( + await export_tenant_provenance(session, target_scope) + ) + for collection in ( + "emails", + "attachments", + *(c for c, _, _ in graph_collections), + ): + assert target_records[collection] == winner_records[collection] + email = ( + await session.scalars( + select(Email).where(Email.user_id == source_scope.user_id) + ) + ).one() + assert ( + await session.scalar( + select(func.count()) + .select_from(Attachment) + .where(Attachment.email_id == email.id) + ) + == 2 + ) # The source also retains its excluded binary attachment. + for collection, model, _ in graph_collections: + scope_filter = ( + model.email_id == email.id + if collection + in ("content_nodes", "content_segments", "structural_edges") + else model.workspace_id.in_( + [source_scope.workspace_id, target_scope.workspace_id] + ) + ) + assert await session.scalar( + select(func.count()).select_from(model).where(scope_filter) + ) == 2 * len(records[collection]) + mappings = list( + ( + await session.scalars( + select(ProvenanceIdentityMapping).where( + ProvenanceIdentityMapping.target_workspace_id + == target_scope.workspace_id + ) + ) + ).all() + ) + assert len(mappings) == 7 + assert {(row.entity_kind, row.portable_uid) for row in mappings} == { + (collection, record[uid_key]) + for collection, _, uid_key in graph_collections + for record in winner_records[collection] + } + finally: + async with session_factory.begin() as session: + email = await session.scalar( + select(Email).where(Email.message_id == f"<{token}@example.com>") + ) + if email is not None: + await session.execute( + delete(ProjectGraphCorrectionRecord).where( + ProjectGraphCorrectionRecord.workspace_id.in_( + [source_scope.workspace_id, target_scope.workspace_id] + ) + ) + ) + await session.execute( + delete(ProjectGraphEdgeRecord).where( + ProjectGraphEdgeRecord.workspace_id.in_( + [source_scope.workspace_id, target_scope.workspace_id] + ) + ) + ) + await session.execute( + delete(ProjectGraphObjectRecord).where( + ProjectGraphObjectRecord.workspace_id.in_( + [source_scope.workspace_id, target_scope.workspace_id] + ) + ) + ) + await session.execute( + delete(ProvenanceIdentityMapping).where( + ProvenanceIdentityMapping.target_workspace_id + == target_scope.workspace_id + ) + ) + await session.execute( + delete(KnowledgeGraphEdgeRecord).where( + KnowledgeGraphEdgeRecord.email_id == email.id + ) + ) + await session.execute( + delete(ContentSegmentRecord).where( + ContentSegmentRecord.email_id == email.id + ) + ) + await session.execute( + delete(ContentNodeRecord).where( + ContentNodeRecord.email_id == email.id + ) + ) + await session.execute( + delete(Attachment).where(Attachment.email_id == email.id) + ) + await session.delete(email) + await engine.dispose() + + +@pytest.mark.asyncio +@pytest.mark.postgres +async def test_concurrent_cross_tenant_imports_remap_global_identities(): + engine = create_async_engine(settings.DATABASE_URL, echo=False) + await _require_direct_postgres(engine) + session_factory = async_sessionmaker(engine, expire_on_commit=False) + token = uuid.uuid4().hex[:12] + source_scope = _scope(f"concurrent-portable-source-{token}") + target_scopes = ( + _scope(f"concurrent-portable-target-a-{token}"), + _scope(f"concurrent-portable-target-b-{token}"), + ) + try: + async with engine.begin() as connection: + await connection.run_sync(Base.metadata.create_all) + async with session_factory() as session: + await _seed_provenance_closure(session, scope=source_scope, token=token) + async with session_factory() as session: + archive = await export_tenant_provenance(session, source_scope) + records = parse_provenance_archive(archive) + await _delete_exported_closure(session, records) + await session.commit() + + async def run_import(target_scope): + async with session_factory() as session: + return await import_tenant_provenance(session, target_scope, archive) + + receipts = await asyncio.gather( + *(run_import(target_scope) for target_scope in target_scopes) + ) + assert all(sum(receipt.created.values()) > 0 for receipt in receipts) + async with session_factory() as session: + assert ( + await session.scalar( + select(func.count()) + .select_from(ProjectGraphObjectRecord) + .where( + ProjectGraphObjectRecord.workspace_id.in_( + [scope.workspace_id for scope in target_scopes] + ) + ) + ) + == 4 + ) + finally: + async with session_factory.begin() as session: + workspace_ids = [scope.workspace_id for scope in target_scopes] + for model in ( + ProjectGraphCorrectionRecord, + ProjectGraphEdgeRecord, + ProjectGraphObjectRecord, + ): + await session.execute( + delete(model).where(model.workspace_id.in_(workspace_ids)) + ) + await session.execute( + delete(ProvenanceIdentityMapping).where( + ProvenanceIdentityMapping.target_workspace_id.in_(workspace_ids) + ) + ) + email_ids = list( + ( + await session.scalars( + select(Email.id).where( + Email.message_id == f"<{token}@example.com>" + ) + ) + ).all() + ) + for model in ( + KnowledgeGraphEdgeRecord, + ContentSegmentRecord, + ContentNodeRecord, + Attachment, + ): + await session.execute(delete(model).where(model.email_id.in_(email_ids))) + await session.execute(delete(Email).where(Email.id.in_(email_ids))) + await engine.dispose() + + +@pytest.mark.asyncio +@pytest.mark.postgres +async def test_concurrent_imports_across_workspaces_reuse_owner_email(): + engine = create_async_engine(settings.DATABASE_URL, echo=False) + await _require_direct_postgres(engine) + session_factory = async_sessionmaker(engine, expire_on_commit=False) + token = uuid.uuid4().hex[:12] + source_scope = _scope(f"parallel-workspaces-source-{token}") + target_user_id = f"parallel-target-user-{token}" + target_organization_id = f"parallel-target-org-{token}" + target_scopes = tuple( + TenantProvenanceScope( + user_id=target_user_id, + organization_id=target_organization_id, + workspace_id=f"parallel-workspace-{index}-{token}", + ) + for index in range(2) + ) + try: + async with engine.begin() as connection: + await connection.run_sync(Base.metadata.create_all) + async with session_factory() as session: + archive = await _two_email_rooted_archive( + session, scope=source_scope, token=token + ) + first_records = parse_provenance_archive(archive) + second_records = copy.deepcopy(first_records) + shared_email_uid, distinct_email_uid = sorted( + record["email_uid"] for record in first_records["emails"] + ) + replacement_email_uid = f"" + + def replace_email_uid(value): + if isinstance(value, dict): + return {key: replace_email_uid(item) for key, item in value.items()} + if isinstance(value, list): + return [replace_email_uid(item) for item in value] + return replacement_email_uid if value == distinct_email_uid else value + + second_records = replace_email_uid(second_records) + attachment_uid_replacements = {} + for attachment in second_records["attachments"]: + if attachment["email_uid"] != replacement_email_uid: + continue + canonical = provenance_service._attachment_payload_core(attachment) + attachment_uid_replacements[attachment["attachment_uid"]] = ( + provenance_service._attachment_uid(canonical, 1) + ) + + def replace_attachment_uid(value): + if isinstance(value, dict): + return { + key: replace_attachment_uid(item) for key, item in value.items() + } + if isinstance(value, list): + return [replace_attachment_uid(item) for item in value] + if isinstance(value, str) and value.startswith("attachment:"): + attachment_uid = value.removeprefix("attachment:") + return "attachment:" + attachment_uid_replacements.get( + attachment_uid, attachment_uid + ) + return attachment_uid_replacements.get(value, value) + + second_records = replace_attachment_uid(second_records) + uid_keys = { + "emails": "email_uid", + "attachments": "attachment_uid", + **provenance_service._UID_KEYS, + } + for collection, uid_key in uid_keys.items(): + second_records[collection].sort(key=lambda record: record[uid_key]) + second_archive = build_provenance_archive(second_records) + + async def run_import(target_scope, target_archive): + async with session_factory() as session: + return await import_tenant_provenance( + session, target_scope, target_archive + ) + + receipts = await asyncio.gather( + run_import(target_scopes[0], archive), + run_import(target_scopes[1], second_archive), + ) + assert all(sum(receipt.created.values()) > 0 for receipt in receipts) + async with session_factory() as session: + assert ( + await session.scalar( + select(func.count()) + .select_from(Email) + .where( + Email.user_id == target_user_id, + Email.organization_id == target_organization_id, + Email.message_id == shared_email_uid, + ) + ) + == 1 + ) + finally: + async with session_factory.begin() as session: + workspace_ids = [ + source_scope.workspace_id, + *(scope.workspace_id for scope in target_scopes), + ] + await session.execute( + delete(ProjectGraphCorrectionRecord).where( + ProjectGraphCorrectionRecord.workspace_id.in_(workspace_ids) + ) + ) + await session.execute( + delete(ProjectGraphEdgeRecord).where( + ProjectGraphEdgeRecord.workspace_id.in_(workspace_ids) + ) + ) + await session.execute( + delete(ProjectGraphObjectRecord).where( + ProjectGraphObjectRecord.workspace_id.in_(workspace_ids) + ) + ) + await session.execute( + delete(ProvenanceIdentityMapping).where( + ProvenanceIdentityMapping.target_workspace_id.in_(workspace_ids) + ) + ) + emails = list( + ( + await session.scalars( + select(Email).where( + Email.user_id.in_([source_scope.user_id, target_user_id]) + ) + ) + ).all() + ) + for email in emails: + await session.execute( + delete(KnowledgeGraphEdgeRecord).where( + KnowledgeGraphEdgeRecord.email_id == email.id + ) + ) + await session.execute( + delete(ContentSegmentRecord).where( + ContentSegmentRecord.email_id == email.id + ) + ) + await session.execute( + delete(ContentNodeRecord).where( + ContentNodeRecord.email_id == email.id + ) + ) + await session.execute( + delete(Attachment).where(Attachment.email_id == email.id) + ) + await session.delete(email) + await engine.dispose() + + +@pytest.mark.asyncio +@pytest.mark.postgres +async def test_final_migration_upgrade_is_safe_after_fresh_metadata_bootstrap( + provenance_sessionmaker, +): + revision_path = ( + Path(__file__).parents[1] + / "alembic" + / "versions" + / "0018_provenance_identity_mappings.py" + ) + spec = importlib.util.spec_from_file_location( + "provenance_identity_revision", revision_path + ) + assert spec is not None and spec.loader is not None + revision = importlib.util.module_from_spec(spec) + spec.loader.exec_module(revision) + original_op = revision.op + + def run_upgrade(sync_connection): + revision.op = Operations(MigrationContext.configure(sync_connection)) + revision.upgrade() + revision.upgrade() + assert sync_connection.dialect.has_table( + sync_connection, "provenance_identity_mappings" + ) + + try: + async with provenance_sessionmaker() as session: + connection = await session.connection() + await connection.run_sync(run_upgrade) + finally: + revision.op = original_op + + +@pytest.mark.parametrize( + ("model", "value"), + ( + (ProjectGraphObjectRecord, -0.01), + (ProjectGraphObjectRecord, 1.01), + (ProjectGraphEdgeRecord, -0.01), + (ProjectGraphEdgeRecord, 1.01), + ), +) +@pytest.mark.asyncio +@pytest.mark.postgres +async def test_export_rejects_confidence_outside_unit_interval( + provenance_sessionmaker, + model, + value, +): + token = uuid.uuid4().hex[:12] + scope = _scope(f"confidence-export-{token}") + async with provenance_sessionmaker() as session: + await _seed_provenance_closure(session, scope=scope, token=token) + record = await session.scalar( + select(model).where(model.workspace_id == scope.workspace_id) + ) + record.confidence = value + await session.commit() + + async with provenance_sessionmaker() as session: + with pytest.raises(ProvenanceArchiveError): + await export_tenant_provenance(session, scope) + + +@pytest.mark.parametrize( + ("collection", "value"), + ( + ("project_objects", "-0.01"), + ("project_objects", "1.01"), + ("project_edges", "-0.01"), + ("project_edges", "1.01"), + ), +) +@pytest.mark.asyncio +@pytest.mark.postgres +async def test_import_rejects_confidence_outside_unit_interval_before_flush( + provenance_sessionmaker, + monkeypatch, + collection, + value, +): + token = uuid.uuid4().hex[:12] + source_scope = _scope(f"confidence-source-{token}") + target_scope = _scope(f"confidence-target-{token}") + async with provenance_sessionmaker() as session: + await _seed_provenance_closure(session, scope=source_scope, token=token) + async with provenance_sessionmaker() as session: + records = parse_provenance_archive( + await export_tenant_provenance(session, source_scope) + ) + records[collection][0]["confidence"] = value + invalid_archive = build_provenance_archive(records) + async with provenance_sessionmaker() as session: + await _delete_exported_closure(session, records) + + async with provenance_sessionmaker() as session: + flush_count = 0 + original_flush = session.flush + + async def counting_flush(*args, **kwargs): + nonlocal flush_count + flush_count += 1 + return await original_flush(*args, **kwargs) + + monkeypatch.setattr(session, "flush", counting_flush) + with pytest.raises(ProvenanceArchiveError): + await import_tenant_provenance(session, target_scope, invalid_archive) + assert flush_count == 0 + + +@pytest.mark.parametrize( + ("collection", "timestamp_field"), + (("emails", "date"), ("corrections", "created_at")), +) +@pytest.mark.asyncio +@pytest.mark.postgres +async def test_import_rejects_equivalent_offset_timestamps_before_flush_on_retry( + provenance_sessionmaker, + monkeypatch, + collection, + timestamp_field, +): + token = uuid.uuid4().hex[:12] + source_scope = _scope(f"timestamp-source-{token}") + target_scope = _scope(f"timestamp-target-{token}") + async with provenance_sessionmaker() as session: + await _seed_provenance_closure(session, scope=source_scope, token=token) + async with provenance_sessionmaker() as session: + records = parse_provenance_archive( + await export_tenant_provenance(session, source_scope) + ) + valid_archive = build_provenance_archive(records) + async with provenance_sessionmaker() as session: + await import_tenant_provenance(session, target_scope, valid_archive) + + offset_records = copy.deepcopy(records) + utc_timestamp = datetime.datetime.fromisoformat( + offset_records[collection][0][timestamp_field] + ) + offset_records[collection][0][timestamp_field] = utc_timestamp.astimezone( + datetime.timezone(datetime.timedelta(hours=9)) + ).isoformat() + offset_archive = build_provenance_archive(offset_records) + + async with provenance_sessionmaker() as session: + flush_count = 0 + original_flush = session.flush + + async def counting_flush(*args, **kwargs): + nonlocal flush_count + flush_count += 1 + return await original_flush(*args, **kwargs) + + monkeypatch.setattr(session, "flush", counting_flush) + for _ in range(2): + with pytest.raises(ProvenanceArchiveError): + await import_tenant_provenance( + session, + target_scope, + offset_archive, + ) + assert flush_count == 0 + + +@pytest.mark.parametrize( + "reference_field", + ( + "object_source", + "object_primary", + "edge_source", + "edge_primary", + "correction_source", + "edge_without_object_anchors", + ), +) +@pytest.mark.asyncio +@pytest.mark.postgres +async def test_import_rejects_cross_email_project_references_before_flush( + provenance_sessionmaker, + monkeypatch, + reference_field, +): + token = uuid.uuid4().hex[:12] + source_scope = _scope(f"citation-source-{token}") + target_scope = _scope(f"citation-target-{token}") + async with provenance_sessionmaker() as session: + archive = await _two_email_rooted_archive( + session, + scope=source_scope, + token=token, + ) + records = parse_provenance_archive(archive) + object_email = { + record["object_uid"]: record["email_uid"] + for record in records["project_objects"] + } + segment_email = { + record["content_segment_uid"]: record["email_uid"] + for record in records["content_segments"] + } + if reference_field.startswith("object_"): + record = records["project_objects"][0] + anchor_email_uid = record["email_uid"] + elif reference_field.startswith("edge_"): + record = records["project_edges"][0] + anchor_email_uid = object_email[ + record["source_object_uid"] or record["target_object_uid"] + ] + else: + record = records["corrections"][0] + anchor_email_uid = object_email[record["object_uid"]] + foreign_segment_uid = next( + segment_uid + for segment_uid, email_uid in segment_email.items() + if email_uid != anchor_email_uid + ) + if reference_field == "edge_without_object_anchors": + record["source_object_uid"] = None + record["target_object_uid"] = None + elif reference_field.endswith("_primary"): + record["primary_content_segment_uid"] = foreign_segment_uid + else: + record["source_segment_uids"] = [foreign_segment_uid] + invalid_archive = build_provenance_archive(records) + + async with provenance_sessionmaker() as session: + flush_count = 0 + original_flush = session.flush + + async def counting_flush(*args, **kwargs): + nonlocal flush_count + flush_count += 1 + return await original_flush(*args, **kwargs) + + monkeypatch.setattr(session, "flush", counting_flush) + with pytest.raises(ProvenanceArchiveError): + await import_tenant_provenance(session, target_scope, invalid_archive) + assert flush_count == 0 + + +@pytest.mark.asyncio +@pytest.mark.postgres +async def test_import_round_trip_remaps_nullable_segment_evidence_endpoint( + provenance_sessionmaker, +): + token = uuid.uuid4().hex[:12] + source_scope = _scope(f"nullable-source-{token}") + target_scope = TenantProvenanceScope( + user_id=source_scope.user_id, + organization_id=source_scope.organization_id, + workspace_id=f"nullable-target-workspace-{token}", + ) + portable_segment_uid = f"segment-{token}" + async with provenance_sessionmaker() as session: + await _seed_provenance_closure(session, scope=source_scope, token=token) + source_edge = await session.scalar( + select(ProjectGraphEdgeRecord).where( + ProjectGraphEdgeRecord.workspace_id == source_scope.workspace_id + ) + ) + source_edge.source_object_id = None + source_edge.source_uid = f"segment:{portable_segment_uid}" + await session.commit() + async with provenance_sessionmaker() as session: + archive = await export_tenant_provenance(session, source_scope) + + async with provenance_sessionmaker() as session: + receipt = await import_tenant_provenance( + session, + target_scope, + archive, + ) + target_edge = await session.scalar( + select(ProjectGraphEdgeRecord).where( + ProjectGraphEdgeRecord.workspace_id == target_scope.workspace_id + ) + ) + segment_mapping = await session.scalar( + select(ProvenanceIdentityMapping).where( + ProvenanceIdentityMapping.target_workspace_id + == target_scope.workspace_id, + ProvenanceIdentityMapping.entity_kind == "content_segments", + ProvenanceIdentityMapping.portable_uid == portable_segment_uid, + ) + ) + target_source_uid = target_edge.source_uid + target_source_object_id = target_edge.source_object_id + async with provenance_sessionmaker() as session: + reexported = parse_provenance_archive( + await export_tenant_provenance(session, target_scope) + ) + + assert receipt.created["project_edges"] == 1 + assert target_source_object_id is None + assert target_source_uid == f"segment:{segment_mapping.target_database_uid}" + assert reexported["project_edges"][0]["source_object_uid"] is None + assert reexported["project_edges"][0]["source_uid"] == ( + f"segment:{portable_segment_uid}" + ) + + +@pytest.mark.parametrize("endpoint", ("source", "target")) +@pytest.mark.parametrize( + "invalid_kind", ("empty", "bare", "unknown", "object_uid", "cross_email") +) +@pytest.mark.asyncio +@pytest.mark.postgres +async def test_import_rejects_invalid_nullable_segment_endpoint_before_flush( + provenance_sessionmaker, + monkeypatch, + endpoint, + invalid_kind, +): + token = uuid.uuid4().hex[:12] + source_scope = _scope(f"nullable-invalid-source-{token}") + target_scope = _scope(f"nullable-invalid-target-{token}") + async with provenance_sessionmaker() as session: + archive = await _two_email_rooted_archive( + session, scope=source_scope, token=token + ) + records = parse_provenance_archive(archive) + edge = records["project_edges"][0] + anchor_object_uid = edge[f"{'target' if endpoint == 'source' else 'source'}_object_uid"] + object_email = { + record["object_uid"]: record["email_uid"] + for record in records["project_objects"] + } + anchor_email_uid = object_email[anchor_object_uid] + cross_email_segment_uid = next( + record["content_segment_uid"] + for record in records["content_segments"] + if record["email_uid"] != anchor_email_uid + ) + invalid_values = { + "empty": "segment:", + "bare": edge["source_segment_uids"][0], + "unknown": f"segment:unknown-{token}", + "object_uid": edge["source_object_uid"], + "cross_email": f"segment:{cross_email_segment_uid}", + } + edge[f"{endpoint}_object_uid"] = None + edge[f"{endpoint}_uid"] = invalid_values[invalid_kind] + invalid_archive = build_provenance_archive(records) + + async with provenance_sessionmaker() as session: + flush_count = 0 + original_flush = session.flush + + async def counting_flush(*args, **kwargs): + nonlocal flush_count + flush_count += 1 + return await original_flush(*args, **kwargs) + + monkeypatch.setattr(session, "flush", counting_flush) + with pytest.raises(ProvenanceArchiveError): + await import_tenant_provenance(session, target_scope, invalid_archive) + assert flush_count == 0 + + +@pytest.mark.asyncio +@pytest.mark.postgres +async def test_confidence_unit_interval_boundaries_round_trip( + provenance_sessionmaker, +): + token = uuid.uuid4().hex[:12] + source_scope = _scope(f"confidence-source-{token}") + target_scope = _scope(f"confidence-target-{token}") + async with provenance_sessionmaker() as session: + await _seed_provenance_closure(session, scope=source_scope, token=token) + project_object = await session.scalar( + select(ProjectGraphObjectRecord).where( + ProjectGraphObjectRecord.workspace_id == source_scope.workspace_id + ) + ) + project_edge = await session.scalar( + select(ProjectGraphEdgeRecord).where( + ProjectGraphEdgeRecord.workspace_id == source_scope.workspace_id + ) + ) + project_object.confidence = 0.0 + project_edge.confidence = 1.0 + await session.commit() + async with provenance_sessionmaker() as session: + archive = await export_tenant_provenance(session, source_scope) + records = parse_provenance_archive(archive) + async with provenance_sessionmaker() as session: + await _delete_exported_closure(session, records) + async with provenance_sessionmaker() as session: + await import_tenant_provenance(session, target_scope, archive) + restored_object = await session.scalar( + select(ProjectGraphObjectRecord).where( + ProjectGraphObjectRecord.workspace_id == target_scope.workspace_id, + ProjectGraphObjectRecord.confidence == 0.0, + ) + ) + restored_edge = await session.scalar( + select(ProjectGraphEdgeRecord).where( + ProjectGraphEdgeRecord.workspace_id == target_scope.workspace_id + ) + ) + + assert restored_object is not None + assert restored_edge.confidence == 1.0 + + +@pytest.mark.asyncio +@pytest.mark.postgres +async def test_export_is_exact_workspace_scoped(provenance_sessionmaker): + token = uuid.uuid4().hex[:12] + shared_scope = _scope(f"shared-{token}") + other_scope = TenantProvenanceScope( + user_id=shared_scope.user_id, + organization_id=shared_scope.organization_id, + workspace_id=f"other-workspace-{token}", + ) + async with provenance_sessionmaker() as session: + expected = await _seed_provenance_closure( + session, scope=shared_scope, token=f"selected-{token}" + ) + excluded = await _seed_provenance_closure( + session, scope=other_scope, token=f"excluded-{token}" + ) + async with provenance_sessionmaker() as session: + records = parse_provenance_archive( + await export_tenant_provenance(session, shared_scope) + ) + + assert [record["email_uid"] for record in records["emails"]] == [ + expected["email_uid"] + ] + assert excluded["email_uid"] not in json.dumps(records) + + +@pytest.mark.asyncio +@pytest.mark.postgres +async def test_import_dangling_reference_rolls_back_without_mutation( + provenance_sessionmaker, +): + token = uuid.uuid4().hex[:12] + source_scope = _scope(f"source-{token}") + target_scope = _scope(f"target-{token}") + async with provenance_sessionmaker() as session: + await _seed_provenance_closure(session, scope=source_scope, token=token) + async with provenance_sessionmaker() as session: + source_archive = await export_tenant_provenance(session, source_scope) + records = copy.deepcopy(parse_provenance_archive(source_archive)) + records["project_objects"][0]["source_segment_uids"] = ["missing-segment"] + invalid_archive = build_provenance_archive(records) + async with provenance_sessionmaker() as session: + await _delete_exported_closure(session, records) + async with provenance_sessionmaker() as session: + with pytest.raises(ProvenanceArchiveError): + await import_tenant_provenance(session, target_scope, invalid_archive) + async with provenance_sessionmaker() as session: + assert ( + await session.scalar( + select(func.count()) + .select_from(Email) + .where( + *Email.owner_filters( + target_scope.user_id, target_scope.organization_id + ) + ) + ) + == 0 + ) + + +@pytest.mark.asyncio +@pytest.mark.postgres +async def test_import_late_conflict_fails_before_creating_email( + provenance_sessionmaker, +): + token = uuid.uuid4().hex[:12] + source_scope = _scope(f"source-{token}") + target_scope = _scope(f"target-{token}") + async with provenance_sessionmaker() as session: + await _seed_provenance_closure(session, scope=source_scope, token=token) + async with provenance_sessionmaker() as session: + archive = await export_tenant_provenance(session, source_scope) + records = parse_provenance_archive(archive) + imported_email_uid = records["emails"][0]["email_uid"] + conflict_uid = records["project_objects"][0]["object_uid"] + async with provenance_sessionmaker() as session: + await _delete_exported_closure(session, records) + target_seed = await _seed_provenance_closure( + session, scope=target_scope, token=f"conflict-{token}" + ) + existing = await session.scalar( + select(ProjectGraphObjectRecord).where( + ProjectGraphObjectRecord.object_uid == target_seed["object_uids"][0] + ) + ) + existing.object_uid = conflict_uid + existing.title = "Conflicting target record" + await session.commit() + + async with provenance_sessionmaker() as session: + with pytest.raises(ProvenanceArchiveError): + await import_tenant_provenance(session, target_scope, archive) + async with provenance_sessionmaker() as session: + assert ( + await session.scalar( + select(func.count()) + .select_from(Email) + .where(Email.message_id == imported_email_uid) + ) + == 0 + ) + + +@pytest.mark.asyncio +@pytest.mark.postgres +async def test_export_admits_only_parser_confirmed_text_and_omits_sensitive_fields( + provenance_sessionmaker, +): + token = uuid.uuid4().hex[:12] + scope = _scope(token) + async with provenance_sessionmaker() as session: + await _seed_provenance_closure(session, scope=scope, token=token) + async with provenance_sessionmaker() as session: + archive = await export_tenant_provenance(session, scope) + records = parse_provenance_archive(archive) + serialized = json.dumps(records, sort_keys=True) + + assert [record["filename"] for record in records["attachments"]] == ["evidence.txt"] + assert "binary-payload-must-not-export" not in serialized + for forbidden in ( + "embedding", + "credential", + "secret", + "api_key", + "provider_url", + "token", + "email_id", + "attachment_id", + "content_node_id", + "content_segment_id", + "project_graph_object_id", + ): + assert forbidden not in serialized + + +@pytest.mark.asyncio +@pytest.mark.postgres +async def test_import_is_idempotent_for_exact_target_records(provenance_sessionmaker): + token = uuid.uuid4().hex[:12] + source_scope = _scope(f"source-{token}") + target_scope = _scope(f"target-{token}") + async with provenance_sessionmaker() as session: + await _seed_provenance_closure(session, scope=source_scope, token=token) + async with provenance_sessionmaker() as session: + archive = await export_tenant_provenance(session, source_scope) + records = parse_provenance_archive(archive) + async with provenance_sessionmaker() as session: + await _delete_exported_closure(session, records) + async with provenance_sessionmaker() as session: + first = await import_tenant_provenance(session, target_scope, archive) + async with provenance_sessionmaker() as session: + second = await import_tenant_provenance(session, target_scope, archive) + + assert sum(first.created.values()) == 9 + assert sum(first.skipped.values()) == 0 + assert sum(second.created.values()) == 0 + assert second.skipped == first.created + + +@pytest.mark.asyncio +@pytest.mark.postgres +async def test_cross_workspace_retry_uses_mappings_after_source_graph_deletion( + provenance_sessionmaker, +): + token = uuid.uuid4().hex[:12] + source_scope = _scope(f"deleted-source-{token}") + target_scope = TenantProvenanceScope( + user_id=source_scope.user_id, + organization_id=source_scope.organization_id, + workspace_id=f"retained-target-{token}", + ) + async with provenance_sessionmaker() as session: + await _seed_provenance_closure(session, scope=source_scope, token=token) + async with provenance_sessionmaker() as session: + archive = await export_tenant_provenance(session, source_scope) + async with provenance_sessionmaker() as session: + first = await import_tenant_provenance(session, target_scope, archive) + async with provenance_sessionmaker() as session: + await session.execute( + delete(ProjectGraphCorrectionRecord).where( + ProjectGraphCorrectionRecord.workspace_id == source_scope.workspace_id + ) + ) + await session.execute( + delete(ProjectGraphEdgeRecord).where( + ProjectGraphEdgeRecord.workspace_id == source_scope.workspace_id + ) + ) + await session.execute( + delete(ProjectGraphObjectRecord).where( + ProjectGraphObjectRecord.workspace_id == source_scope.workspace_id + ) + ) + await session.commit() + async with provenance_sessionmaker() as session: + second = await import_tenant_provenance(session, target_scope, archive) + target_counts = ( + await session.scalar( + select(func.count()).select_from(ProjectGraphObjectRecord).where( + ProjectGraphObjectRecord.workspace_id == target_scope.workspace_id + ) + ), + await session.scalar( + select(func.count()).select_from(ProjectGraphEdgeRecord).where( + ProjectGraphEdgeRecord.workspace_id == target_scope.workspace_id + ) + ), + await session.scalar( + select(func.count()).select_from(ProjectGraphCorrectionRecord).where( + ProjectGraphCorrectionRecord.workspace_id == target_scope.workspace_id + ) + ), + ) + + assert sum(first.created.values()) > 0 + assert sum(second.created.values()) == 0 + assert sum(second.skipped.values()) == sum(first.created.values()) + sum( + first.skipped.values() + ) + assert target_counts == (2, 1, 1) + + +@pytest.mark.parametrize("citation_owner", ("object", "edge", "correction")) +@pytest.mark.asyncio +@pytest.mark.postgres +async def test_export_rejects_cross_email_citation_source_closure( + provenance_sessionmaker, + citation_owner, +): + token = uuid.uuid4().hex[:12] + scope = _scope(f"source-{token}") + cited_scope = TenantProvenanceScope( + user_id=scope.user_id, + organization_id=scope.organization_id, + workspace_id=f"cited-workspace-{token}", + ) + async with provenance_sessionmaker() as session: + source = await _seed_provenance_closure(session, scope=scope, token=token) + await _seed_provenance_closure( + session, + scope=cited_scope, + token=f"cited-{token}", + ) + cited_segment_uid = f"segment-cited-{token}" + if citation_owner == "object": + record = await session.scalar( + select(ProjectGraphObjectRecord).where( + ProjectGraphObjectRecord.object_uid == source["object_uids"][0] + ) + ) + elif citation_owner == "edge": + record = await session.scalar( + select(ProjectGraphEdgeRecord).where( + ProjectGraphEdgeRecord.edge_uid == f"project-edge-{token}" + ) + ) + else: + record = await session.scalar( + select(ProjectGraphCorrectionRecord).where( + ProjectGraphCorrectionRecord.correction_uid == f"correction-{token}" + ) + ) + record.source_segment_uids = sorted( + [*record.source_segment_uids, cited_segment_uid] + ) + await session.commit() + async with provenance_sessionmaker() as session: + with pytest.raises(ProvenanceArchiveError): + await export_tenant_provenance(session, scope) + + +@pytest.mark.asyncio +@pytest.mark.postgres +async def test_duplicate_canonical_attachments_keep_node_and_segment_identity( + provenance_sessionmaker, +): + token = uuid.uuid4().hex[:12] + source_scope = _scope(f"source-{token}") + target_scope = _scope(f"target-{token}") + async with provenance_sessionmaker() as session: + source = await _seed_provenance_closure( + session, scope=source_scope, token=token + ) + await _add_duplicate_text_attachments( + session, + email_uid=source["email_uid"], + token=token, + ) + async with provenance_sessionmaker() as session: + archive = await export_tenant_provenance(session, source_scope) + source_records = parse_provenance_archive(archive) + source_node_attachments = { + record["content_node_uid"]: record["attachment_uid"] + for record in source_records["content_nodes"] + if record["content_node_uid"].startswith("duplicate-node-") + } + source_segment_attachments = { + record["content_segment_uid"]: record["attachment_uid"] + for record in source_records["content_segments"] + if record["content_segment_uid"].startswith("duplicate-segment-") + } + assert len(source_node_attachments) == 5 + assert len(set(source_node_attachments.values())) == 5 + async with provenance_sessionmaker() as session: + await _delete_exported_closure(session, source_records) + async with provenance_sessionmaker() as session: + await import_tenant_provenance(session, target_scope, archive) + async with provenance_sessionmaker() as session: + restored = parse_provenance_archive( + await export_tenant_provenance(session, target_scope) + ) + + assert { + record["content_node_uid"]: record["attachment_uid"] + for record in restored["content_nodes"] + if record["content_node_uid"].startswith("duplicate-node-") + } == source_node_attachments + assert { + record["content_segment_uid"]: record["attachment_uid"] + for record in restored["content_segments"] + if record["content_segment_uid"].startswith("duplicate-segment-") + } == source_segment_attachments + + +@pytest.mark.asyncio +@pytest.mark.postgres +async def test_recognized_pdf_omits_integer_keys_and_round_trips_graph_references( + provenance_sessionmaker, +): + token = uuid.uuid4().hex[:12] + source_scope = _scope(f"source-{token}") + target_scope = _scope(f"target-{token}") + async with provenance_sessionmaker() as session: + source = await _seed_provenance_closure( + session, scope=source_scope, token=token + ) + attachment = await session.get(Attachment, source["attachment_id"]) + attachment.content = "Recognized PDF evidence" + attachment.content_type = "application/pdf" + attachment.parse_status = "parsed" + attachment.parse_content_type = "application/pdf" + attachment.parser_key = "pdf" + legacy_source_uid = f"attachment-{attachment.id}" + node = await session.scalar( + select(ContentNodeRecord).where( + ContentNodeRecord.content_node_uid == f"node-{token}" + ) + ) + segment = await session.scalar( + select(ContentSegmentRecord).where( + ContentSegmentRecord.content_segment_uid == f"segment-{token}" + ) + ) + structural_edge = await session.scalar( + select(KnowledgeGraphEdgeRecord).where( + KnowledgeGraphEdgeRecord.edge_uid == f"structural-edge-{token}" + ) + ) + project_object = await session.scalar( + select(ProjectGraphObjectRecord).where( + ProjectGraphObjectRecord.object_uid == source["object_uids"][0] + ) + ) + node.source_record_uid = legacy_source_uid + segment.source_record_uid = legacy_source_uid + structural_edge.source_record_uid = legacy_source_uid + project_object.attributes_json = { + "source_record_uid": legacy_source_uid, + "source_object_uid": project_object.object_uid, + "nested": { + "attachment_uid": legacy_source_uid, + "free_text": legacy_source_uid, + }, + } + await session.commit() + async with provenance_sessionmaker() as session: + archive = await export_tenant_provenance(session, source_scope) + records = parse_provenance_archive(archive) + attachment_uid = records["attachments"][0]["attachment_uid"] + portable_source_uid = f"attachment:{attachment_uid}" + serialized = json.dumps(records, sort_keys=True) + + assert serialized.count(legacy_source_uid) == 1 + assert { + records["content_nodes"][0]["source_record_uid"], + records["content_segments"][0]["source_record_uid"], + records["structural_edges"][0]["source_record_uid"], + } == {portable_source_uid} + assert ( + records["project_objects"][0]["attributes_json"]["source_record_uid"] + == portable_source_uid + ) + assert records["project_objects"][0]["attributes_json"]["nested"] == { + "attachment_uid": portable_source_uid, + "free_text": legacy_source_uid, + } + + async with provenance_sessionmaker() as session: + await _delete_exported_closure(session, records) + async with provenance_sessionmaker() as session: + await import_tenant_provenance(session, target_scope, archive) + async with provenance_sessionmaker() as session: + restored = parse_provenance_archive( + await export_tenant_provenance(session, target_scope) + ) + restored_node = await session.scalar( + select(ContentNodeRecord).where( + ContentNodeRecord.content_node_uid == f"node-{token}" + ) + ) + + assert restored_node.source_record_uid == portable_source_uid + assert restored["content_nodes"][0]["attachment_uid"] == attachment_uid + assert restored["content_nodes"][0]["source_record_uid"] == portable_source_uid + assert restored["project_objects"][0]["attributes_json"]["nested"] == { + "attachment_uid": portable_source_uid, + "free_text": legacy_source_uid, + } + + +@pytest.mark.asyncio +@pytest.mark.postgres +async def test_production_correction_float_round_trip(provenance_sessionmaker): + token = uuid.uuid4().hex[:12] + source_scope = _scope(f"source-{token}") + target_scope = _scope(f"target-{token}") + async with provenance_sessionmaker() as session: + source = await _seed_provenance_closure( + session, scope=source_scope, token=token + ) + correction = await ProjectGraphRepository(session).apply_correction( + object_uid=source["object_uids"][0], + user_id=source_scope.user_id, + organization_id=source_scope.organization_id, + workspace_id=source_scope.workspace_id, + actor_user_id=source_scope.user_id, + correction_action="adjust_confidence", + after_json={"confidence": 0.73}, + rationale="Production correction path", + ) + correction_uid = correction.correction_uid + await session.commit() + async with provenance_sessionmaker() as session: + archive = await export_tenant_provenance(session, source_scope) + records = parse_provenance_archive(archive) + correction_record = next( + record + for record in records["corrections"] + if record["correction_uid"] == correction_uid + ) + + assert correction_record["before_json"]["confidence"] == 0.91 + assert correction_record["after_json"]["confidence"] == 0.73 + assert isinstance(correction_record["before_json"]["confidence"], float) + assert isinstance(correction_record["after_json"]["confidence"], float) + + async with provenance_sessionmaker() as session: + await _delete_exported_closure(session, records) + async with provenance_sessionmaker() as session: + await import_tenant_provenance(session, target_scope, archive) + async with provenance_sessionmaker() as session: + restored = await session.scalar( + select(ProjectGraphCorrectionRecord).where( + ProjectGraphCorrectionRecord.correction_uid == correction_uid + ) + ) + + assert restored.before_json["confidence"] == 0.91 + assert restored.after_json["confidence"] == 0.73 + assert isinstance(restored.before_json["confidence"], float) + assert isinstance(restored.after_json["confidence"], float) + + +@pytest.mark.parametrize( + ("forbidden_key", "forbidden_value"), + ( + ("api_key", "key-material"), + ("provider_url", "https://provider.example/v1"), + ("access_token", "token-material"), + ("email_id", 123), + ("smtp_password", "password-material"), + ("backup_api_key", "key-material"), + ("provider_auth_token", "token-material"), + ("mail_credentials_blob", "credential-material"), + ("backup_provider_endpoint_url", "https://provider.example/v1"), + ("legacy_attachment_id", 123), + ("smtpPassword", "password-material"), + ("backupApiKey", "key-material"), + ("providerAuthToken", "token-material"), + ("provider.endpoint.url", "https://provider.example/v1"), + ("legacyAttachmentId", 123), + ("smtp/password", "password-material"), + ("APIKey", "key-material"), + ("OAuthToken", "token-material"), + ), +) +@pytest.mark.asyncio +@pytest.mark.postgres +async def test_export_rejects_nested_sensitive_metadata( + provenance_sessionmaker, + forbidden_key, + forbidden_value, +): + token = uuid.uuid4().hex[:12] + scope = _scope(token) + async with provenance_sessionmaker() as session: + source = await _seed_provenance_closure(session, scope=scope, token=token) + project_object = await session.scalar( + select(ProjectGraphObjectRecord).where( + ProjectGraphObjectRecord.object_uid == source["object_uids"][0] + ) + ) + project_object.attributes_json = { + "source_object_uid": project_object.object_uid, + "source_segment_uids": [f"segment-{token}"], + "nested_metadata": {forbidden_key: forbidden_value}, + } + await session.commit() + async with provenance_sessionmaker() as session: + with pytest.raises(ProvenanceArchiveError): + await export_tenant_provenance(session, scope) + + +@pytest.mark.parametrize( + ("forbidden_key", "forbidden_value"), + ( + ("client_secret", "secret-material"), + ("provider_endpoint", "https://provider.example/v1"), + ("refresh_token", "token-material"), + ("attachment_id", 456), + ("smtp_password", "password-material"), + ("backup_api_key", "key-material"), + ("provider_auth_token", "token-material"), + ("mail_credentials_blob", "credential-material"), + ("backup_provider_endpoint_url", "https://provider.example/v1"), + ("legacy_attachment_id", 456), + ("smtpPassword", "password-material"), + ("backupApiKey", "key-material"), + ("providerAuthToken", "token-material"), + ("provider.endpoint.url", "https://provider.example/v1"), + ("legacyAttachmentId", 456), + ("smtp/password", "password-material"), + ("APIKey", "key-material"), + ("OAuthToken", "token-material"), + ), +) +@pytest.mark.asyncio +@pytest.mark.postgres +async def test_import_rejects_nested_sensitive_metadata_before_flush( + provenance_sessionmaker, + monkeypatch, + forbidden_key, + forbidden_value, +): + token = uuid.uuid4().hex[:12] + source_scope = _scope(f"source-{token}") + target_scope = _scope(f"target-{token}") + async with provenance_sessionmaker() as session: + await _seed_provenance_closure(session, scope=source_scope, token=token) + async with provenance_sessionmaker() as session: + records = parse_provenance_archive( + await export_tenant_provenance(session, source_scope) + ) + records["corrections"][0]["after_json"] = { + "confidence": 1, + "source_object_uid": records["corrections"][0]["object_uid"], + "nested_metadata": {forbidden_key: forbidden_value}, + } + archive = build_provenance_archive(records) + async with provenance_sessionmaker() as session: + await _delete_exported_closure(session, records) + async with provenance_sessionmaker() as session: + flush_count = 0 + original_flush = session.flush + + async def counting_flush(*args, **kwargs): + nonlocal flush_count + flush_count += 1 + return await original_flush(*args, **kwargs) + + monkeypatch.setattr(session, "flush", counting_flush) + with pytest.raises(ProvenanceArchiveError): + await import_tenant_provenance(session, target_scope, archive) + assert flush_count == 0 + + +@pytest.mark.asyncio +@pytest.mark.postgres +async def test_nested_benign_metadata_and_stable_uids_round_trip( + provenance_sessionmaker, +): + token = uuid.uuid4().hex[:12] + source_scope = _scope(f"source-{token}") + target_scope = _scope(f"target-{token}") + allowed_metadata = { + "confidence": 0.73, + "source_object_uid": f"project-object-source-{token}", + "source_segment_uids": [f"segment-{token}"], + "tokenization_strategy": "bounded", + "passwordless_mode": "enabled", + "credentialed_source": "oidc", + "secretary_note": "benign domain text", + "identifier_kind": "stable", + } + async with provenance_sessionmaker() as session: + source = await _seed_provenance_closure( + session, scope=source_scope, token=token + ) + project_object = await session.scalar( + select(ProjectGraphObjectRecord).where( + ProjectGraphObjectRecord.object_uid == source["object_uids"][0] + ) + ) + project_object.attributes_json = allowed_metadata + await session.commit() + async with provenance_sessionmaker() as session: + records = parse_provenance_archive( + await export_tenant_provenance(session, source_scope) + ) + archive = build_provenance_archive(records) + async with provenance_sessionmaker() as session: + await _delete_exported_closure(session, records) + async with provenance_sessionmaker() as session: + await import_tenant_provenance(session, target_scope, archive) + async with provenance_sessionmaker() as session: + restored = await session.scalar( + select(ProjectGraphObjectRecord).where( + ProjectGraphObjectRecord.object_uid == source["object_uids"][0] + ) + ) + + assert restored.attributes_json == allowed_metadata + + +@pytest.mark.asyncio +@pytest.mark.postgres +async def test_unrooted_email_rejected_without_mutation_and_empty_allowed( + provenance_sessionmaker, + monkeypatch, +): + token = uuid.uuid4().hex[:12] + source_scope = _scope(f"source-{token}") + target_scope = _scope(f"target-{token}") + async with provenance_sessionmaker() as session: + await _seed_provenance_closure(session, scope=source_scope, token=token) + async with provenance_sessionmaker() as session: + rooted = parse_provenance_archive( + await export_tenant_provenance(session, source_scope) + ) + unrooted = copy.deepcopy(rooted) + unrooted["project_objects"] = [] + unrooted["project_edges"] = [] + unrooted["corrections"] = [] + unrooted_archive = build_provenance_archive(unrooted) + empty = copy.deepcopy(rooted) + for collection in ( + "emails", + "attachments", + "content_nodes", + "content_segments", + "structural_edges", + "project_objects", + "project_edges", + "corrections", + ): + empty[collection] = [] + empty_archive = build_provenance_archive(empty) + + async with provenance_sessionmaker() as session: + await _delete_exported_closure(session, rooted) + + async with provenance_sessionmaker() as session: + flush_count = 0 + original_flush = session.flush + + async def counting_flush(*args, **kwargs): + nonlocal flush_count + flush_count += 1 + return await original_flush(*args, **kwargs) + + monkeypatch.setattr(session, "flush", counting_flush) + with pytest.raises(ProvenanceArchiveError): + await import_tenant_provenance(session, target_scope, unrooted_archive) + assert flush_count == 0 + async with provenance_sessionmaker() as session: + receipt = await import_tenant_provenance(session, target_scope, empty_archive) + + assert sum(receipt.created.values()) == 0 + assert sum(receipt.skipped.values()) == 0 + + +@pytest.mark.parametrize( + "conflict_kind", + ( + "structural_attachment_email", + "object_attachment_email", + "object_attachment_primary", + "edge_source_endpoint", + "edge_target_endpoint", + ), +) +@pytest.mark.asyncio +@pytest.mark.postgres +async def test_cross_field_conflict_rejected_before_flush( + provenance_sessionmaker, + monkeypatch, + conflict_kind, +): + token = uuid.uuid4().hex[:12] + source_scope = _scope(f"source-{token}") + target_scope = _scope(f"target-{token}") + async with provenance_sessionmaker() as session: + archive = await _two_email_rooted_archive( + session, scope=source_scope, token=token + ) + records = parse_provenance_archive(archive) + source_email_uid = f"<{token}@example.com>" + foreign_attachment_uid = next( + record["attachment_uid"] + for record in records["attachments"] + if record["email_uid"] != source_email_uid + ) + source_project_object = next( + record + for record in records["project_objects"] + if record["email_uid"] == source_email_uid + ) + same_email_attachment_uid = next( + record["attachment_uid"] + for record in records["attachments"] + if record["email_uid"] == source_email_uid + and record["attachment_uid"] != source_project_object["attachment_uid"] + ) + if conflict_kind == "structural_attachment_email": + source_structural_edge = next( + record + for record in records["structural_edges"] + if record["email_uid"] == source_email_uid + ) + source_structural_edge["attachment_uid"] = foreign_attachment_uid + elif conflict_kind == "object_attachment_email": + source_project_object["attachment_uid"] = foreign_attachment_uid + elif conflict_kind == "object_attachment_primary": + source_project_object["attachment_uid"] = same_email_attachment_uid + elif conflict_kind == "edge_source_endpoint": + records["project_edges"][0]["source_uid"] = "mismatched-logical-endpoint" + else: + records["project_edges"][0]["target_uid"] = "mismatched-logical-endpoint" + invalid_archive = build_provenance_archive(records) + async with provenance_sessionmaker() as session: + await _delete_exported_closure(session, records) + async with provenance_sessionmaker() as session: + flush_count = 0 + original_flush = session.flush + + async def counting_flush(*args, **kwargs): + nonlocal flush_count + flush_count += 1 + return await original_flush(*args, **kwargs) + + monkeypatch.setattr(session, "flush", counting_flush) + with pytest.raises(ProvenanceArchiveError): + await import_tenant_provenance(session, target_scope, invalid_archive) + assert flush_count == 0 + + +@pytest.mark.parametrize("bound_kind", ("node_uid", "object_title", "edge_source")) +@pytest.mark.asyncio +@pytest.mark.postgres +async def test_column_bound_preflight_rejects_before_flush( + provenance_sessionmaker, + monkeypatch, + bound_kind, +): + token = uuid.uuid4().hex[:12] + source_scope = _scope(f"source-{token}") + target_scope = _scope(f"target-{token}") + async with provenance_sessionmaker() as session: + await _seed_provenance_closure(session, scope=source_scope, token=token) + async with provenance_sessionmaker() as session: + records = parse_provenance_archive( + await export_tenant_provenance(session, source_scope) + ) + if bound_kind == "node_uid": + old_node_uid = records["content_nodes"][0]["content_node_uid"] + oversized_node_uid = "n" * 65 + records["content_nodes"][0]["content_node_uid"] = oversized_node_uid + for segment in records["content_segments"]: + if segment["content_node_uid"] == old_node_uid: + segment["content_node_uid"] = oversized_node_uid + for edge in records["structural_edges"]: + if edge["source_node_uid"] == old_node_uid: + edge["source_node_uid"] = oversized_node_uid + if edge["target_node_uid"] == old_node_uid: + edge["target_node_uid"] = oversized_node_uid + elif bound_kind == "object_title": + records["project_objects"][0]["title"] = "t" * 241 + else: + records["project_edges"][0]["source_uid"] = "s" * 161 + invalid_archive = build_provenance_archive(records) + async with provenance_sessionmaker() as session: + await _delete_exported_closure(session, records) + async with provenance_sessionmaker() as session: + flush_count = 0 + original_flush = session.flush + + async def counting_flush(*args, **kwargs): + nonlocal flush_count + flush_count += 1 + return await original_flush(*args, **kwargs) + + monkeypatch.setattr(session, "flush", counting_flush) + with pytest.raises(ProvenanceArchiveError): + await import_tenant_provenance(session, target_scope, invalid_archive) + assert flush_count == 0 + + +@pytest.mark.parametrize("scope_dimension", ("user", "organization")) +@pytest.mark.asyncio +@pytest.mark.postgres +async def test_export_excludes_separate_user_and_organization_scope( + provenance_sessionmaker, + scope_dimension, +): + token = uuid.uuid4().hex[:12] + scope = _scope(f"selected-{token}") + rival_scope = TenantProvenanceScope( + user_id=f"rival-user-{token}" if scope_dimension == "user" else scope.user_id, + organization_id=( + f"rival-org-{token}" + if scope_dimension == "organization" + else scope.organization_id + ), + workspace_id=scope.workspace_id, + ) + async with provenance_sessionmaker() as session: + selected = await _seed_provenance_closure( + session, scope=scope, token=f"selected-{token}" + ) + rival = await _seed_provenance_closure( + session, scope=rival_scope, token=f"rival-{token}" + ) + async with provenance_sessionmaker() as session: + records = parse_provenance_archive( + await export_tenant_provenance(session, scope) + ) + + assert {record["email_uid"] for record in records["emails"]} == { + selected["email_uid"] + } + assert rival["email_uid"] not in json.dumps(records) + + +@pytest.mark.asyncio +@pytest.mark.postgres +async def test_export_allows_segment_evidence_edge_with_nullable_source_object( + provenance_sessionmaker, +): + token = uuid.uuid4().hex[:12] + scope = _scope(f"segment-edge-{token}") + async with provenance_sessionmaker() as session: + selected = await _seed_provenance_closure(session, scope=scope, token=token) + edge = await session.scalar( + select(ProjectGraphEdgeRecord).where( + ProjectGraphEdgeRecord.workspace_id == scope.workspace_id + ) + ) + edge.source_uid = f"segment:segment-{token}" + edge.source_object_id = None + await session.commit() + + async with provenance_sessionmaker() as session: + records = parse_provenance_archive( + await export_tenant_provenance(session, scope) + ) + + exported_edge = records["project_edges"][0] + assert exported_edge["source_uid"] == f"segment:segment-{token}" + assert exported_edge["source_object_uid"] is None + assert exported_edge["target_object_uid"] in selected["object_uids"] + + +@pytest.mark.parametrize( + "reference_field", + ( + "object_source", + "object_primary", + "edge_source", + "edge_primary", + "correction_source", + "edge_source_object", + "edge_target_object", + "correction_object", + "edge_without_object_anchors", + "edge_source_segment_endpoint", + "edge_target_segment_endpoint", + ), +) +@pytest.mark.asyncio +@pytest.mark.postgres +async def test_export_rejects_cross_workspace_segment_references_before_email_closure( + provenance_sessionmaker, + reference_field, +): + token = uuid.uuid4().hex[:12] + scope = _scope(f"selected-{token}") + foreign_scope = TenantProvenanceScope( + user_id=scope.user_id, + organization_id=scope.organization_id, + workspace_id=f"workspace-foreign-{token}", + ) + async with provenance_sessionmaker() as session: + selected = await _seed_provenance_closure( + session, scope=scope, token=f"selected-{token}" + ) + foreign = await _seed_provenance_closure( + session, scope=foreign_scope, token=f"foreign-{token}" + ) + foreign_segment = await session.scalar( + select(ContentSegmentRecord).where( + ContentSegmentRecord.email_id == foreign["email_id"] + ) + ) + if reference_field.startswith("object_"): + record = await session.scalar( + select(ProjectGraphObjectRecord).where( + ProjectGraphObjectRecord.project_graph_object_id + == selected["project_object_id"] + ) + ) + elif reference_field.startswith("edge_"): + record = await session.scalar( + select(ProjectGraphEdgeRecord).where( + ProjectGraphEdgeRecord.workspace_id == scope.workspace_id + ) + ) + else: + record = await session.scalar( + select(ProjectGraphCorrectionRecord).where( + ProjectGraphCorrectionRecord.workspace_id == scope.workspace_id + ) + ) + if reference_field == "edge_source_segment_endpoint": + record.source_object_id = None + record.source_uid = f"segment:{foreign_segment.content_segment_uid}" + elif reference_field == "edge_target_segment_endpoint": + record.target_object_id = None + record.target_uid = f"segment:{foreign_segment.content_segment_uid}" + elif reference_field == "edge_without_object_anchors": + record.source_object_id = None + record.target_object_id = None + elif reference_field == "edge_source_object": + record.source_object_id = foreign["project_object_id"] + elif reference_field == "edge_target_object": + record.target_object_id = foreign["project_object_id"] + elif reference_field == "correction_object": + record.project_graph_object_id = foreign["project_object_id"] + elif reference_field.endswith("_primary"): + record.primary_content_segment_id = foreign_segment.content_segment_id + else: + record.source_segment_uids = [foreign_segment.content_segment_uid] + await session.commit() + + async with provenance_sessionmaker() as session: + with pytest.raises(ProvenanceArchiveError): + await export_tenant_provenance(session, scope) diff --git a/docs/adr/0022-tenant-provenance-portability.md b/docs/adr/0022-tenant-provenance-portability.md new file mode 100644 index 000000000..8e9e1926c --- /dev/null +++ b/docs/adr/0022-tenant-provenance-portability.md @@ -0,0 +1,114 @@ +# ADR-0022: Package tenant provenance as a deterministic integrity envelope + +**Status:** Proposed for the workspace project-evidence closure +**Date:** 2026-08-31 +**Decision owner:** Naruon maintainers +**Scope:** Export and reimport of the email-derived provenance records cited by +one signed-session workspace. This decision does not claim full mailbox, +credential, connector, binary-object, or arbitrary multi-workspace portability. +**Former proposal ID:** ADR-0007 in #1497. The 2026-09-05 open-PR inventory +found unrelated ADR-0007 in #1361. The proposal is renumbered, not discarded +or accepted; its historical discussion and evidence remain attached to #1497. + +## Context + +GA-1 requires a buyer to leave, restore, or migrate without losing provenance. +Naruon persists email sources, attachment parse evidence, DOM nodes and +segments, structural edges, project objects and edges, and correction history, +but has no portable round-trip contract. Integer database keys cannot serve as +portable identity and credentials must never enter an export. + +`email_records` has owner and organization scope but no `workspace_id`. Exporting +all owner mail as a workspace bundle could therefore cross a same-organization +workspace boundary. The first safe closure begins with project graph rows scoped +to the exact workspace and includes only their cited source records. + +## Decision + +1. Package the exact project-evidence closure as a deterministic ZIP containing + a BagIt 1.0 envelope, `data/records.json`, and RO-Crate 1.3 JSON-LD metadata. +2. Use stable logical UIDs in the payload. Never serialize or restore sequential + database primary keys. Import resolves new keys in foreign-key order. +3. Canonical JSON uses UTF-8, no BOM, no insignificant whitespace, sorted object + keys, pre-sorted set-like arrays, and canonical UTC `Z` timestamps throughout + the archive. SHA-512 manifests cover exact bytes. + `bundle_uid` and the export PROV activity UID are SHA-256 identities over the + complete canonical record content except those two derived UID fields, so a + rebuilt archive cannot reuse either identity for different records. +4. Reject unsafe or colliding ZIP paths, unlisted files, checksum mismatch, + unsupported profiles, dangling references, scope mismatch, duplicate logical + UIDs, and non-finite numbers before mutation. +5. Reimport is idempotent for target scope, bundle UID, and manifest digest. + Existing identical records are skipped; conflicting records fail closed. + Absence never means deletion and v1 has no tombstones. +6. Export plaintext evidence required for restoration, including email and + parser-confirmed textual attachment content. Exclude credentials, encrypted + secret rows, embeddings, raw binary documents, provider URLs or tokens, and + legacy unscoped audit details. Embeddings are regenerated after import. +7. Signed-session API routes use the current `user_id`, `organization_id`, and + `workspace_id`; bundle payload scope cannot override target authority. HMAC + fallback sessions are rejected because they do not prove workspace + membership; OIDC verification is required outside tests. +8. Full mailbox/customer-exit completion remains open until email ownership has + an explicit workspace dimension and binary object lifecycle is portable. +9. Same-instance cross-workspace imports reuse identical owner-scoped Email and + Attachment rows. Globally unique graph UIDs are deterministically remapped + only while the source graph coexists, using a target-scope plus source-scope + identity table; export reverses the mapping to the stable portable UIDs. The + archive identifies the source user with a validated lowercase SHA-256 digest. + Known typed UID keys nested in graph metadata are translated recursively, + while arbitrary string values remain unchanged. PostgreSQL first takes a + transaction advisory lock keyed by the complete sorted portable-identity + sets, then locks each email UID in sorted order under the target user and + organization. The first lock is shared by equal identity sets even across + target scopes; it does not serialize every arbitrary partial overlap. The + email locks and existing conflict/uniqueness checks remain distinct guards. + Forced identical and incompatible imports verify retries and preservation of + the winning records without losing-writer rows or mappings. A target containing native records or + multiple imported origins exports one target-scoped archive with its current + database UIDs; a single fully mapped origin still restores its portable UIDs. + +## Alternatives rejected + +- **Database dump:** leaks implementation keys and secrets and couples versions. +- **RO-Crate alone:** describes data but is not a byte-integrity envelope. +- **BagIt alone:** verifies opaque bytes but does not describe provenance. +- **Data Transfer Project adapter first:** does not define an offline archive. +- **Export all owner emails:** unsafe while `email_records` lacks workspace scope. + +## Consequences + +- Buyers gain a verifiable first portability slice over evidence used for + project judgments and corrections. +- The importer validates the complete closure before one database transaction. +- Integrity hashes detect corruption but do not authenticate replacement; bundle + signing is a later profile revision. +- The product baseline keeps full tenant/mailbox, binary-object, credential, + provider/connector-state, and audit-history export/reimport open. +- Application rollback retains portable-identity mappings. They cannot be + reconstructed from remapped database records, and deleting them would change + the identity of later exports. Destructive retirement needs its own governed + operation rather than a downgrade side effect. +- Fresh and historical installs must traverse the actual combined Alembic + graph before application tests run. ORM-created tables omit migration-owned + indexes and cannot alone prove installable restore. The unchanged large cited + segment case exposed the search-storage prerequisite in #1572; its Proposed + ADR-0020 separately gates representative search and migration costs. + +## References (APA 7th) + +Kunze, J., Littman, J., Madden, L., Scancella, J., & Adams, C. (2018). *The +BagIt file packaging format (V1.0)* (RFC 8493). RFC Editor. +https://doi.org/10.17487/RFC8493 + +Lebo, T., Sahoo, S., & McGuinness, D. (Eds.). (2013). *PROV-O: The PROV +ontology*. World Wide Web Consortium. https://www.w3.org/TR/prov-o/ + +Rundgren, A., Jordan, B., & Erdtman, S. (2020). *JSON canonicalization scheme +(JCS)* (RFC 8785). RFC Editor. https://doi.org/10.17487/RFC8785 + +RO-Crate Community. (2026). *RO-Crate metadata specification 1.3*. +https://doi.org/10.5281/zenodo.20720080 + +The standards are cited and linked instead of copied; redistribution terms for +the complete referenced publications were not all established for this PR. diff --git a/docs/adr/README.md b/docs/adr/README.md index da8978de8..5e3864bde 100644 --- a/docs/adr/README.md +++ b/docs/adr/README.md @@ -15,6 +15,7 @@ govern implementation. | [ADR-0004](0004-status-weighted-calendar-conflicts.md) | Evaluate CalDAV VEVENT overlaps by occupying status; cancelled does not occupy | Accepted | `ACCEPTED-NARUON-POLICY`; advisory evaluate API only | | [ADR-0020](0020-full-document-trigram-storage.md) | Repair whole-document trigram persistence without changing ranking; measure latency before rollout | Proposed | Storage candidate; no protected integration or performance acceptance | | [ADR-0021](0021-bounded-pdf-dom-upload-contract.md) | Propose 64MiB direct PDF admission only after the required NewsDOM release is pinned | Proposed | `BLOCKED-UPSTREAM`; requires an immutable NewsDOM 64MiB release and exact Naruon pin | +| [ADR-0022](0022-tenant-provenance-portability.md) | Package workspace project evidence as a deterministic BagIt/RO-Crate provenance closure | Proposed | First portability slice; full tenant export remains incomplete | The complete topic-intelligence requirements, architecture, contract, UML, conceptual ERD, security, test, and operability graph is indexed at diff --git a/docs/doctoring/tenant_provenance_integration.md b/docs/doctoring/tenant_provenance_integration.md new file mode 100644 index 000000000..385f64ce7 --- /dev/null +++ b/docs/doctoring/tenant_provenance_integration.md @@ -0,0 +1,88 @@ +# Migrated tenant provenance integration + +## Source and acceptance boundary + +Proposed, observed 2026-09-05. The implementing #1497 PR records its exact +tested head and tree. No protected merge, immutable release, representative +search-performance acceptance, or signed browser restore is claimed here. + +Remote #1497 started at `705d8ece2c97edc8575ea59766fd8f68bf4cdb82`. +The first normal integration checkpoint +`7ee6e68c31b2e716210fc8b62e287a78b765062b` preserved that history and #1427 +`02366791b2a449b8b23b527dcc550996361c0f96`. Its actual migrated suite was +288 passed / 1 failed: the unchanged large cited-segment case exceeded the +GiST index-row limit. That checkpoint was not published as passing evidence. + +The current integration then normally merges #1427 +`cb08b1c3ea2aba8844fc29ef703c34368cc55e47`, inheriting #1572 search-schema +owner `cd8ff413d4ed8a5f2855c47a21a31db5661cd487` through #1468. No source +copy, force push, fake legacy table, revision stamping, index omission, or +smaller replacement of the original archive regression is used. + +## Repairs and preserved behavior + +- `0021_merge_provenance_workspace` joins `0018_provenance_identity` and + `0020_search_trigram_storage`. The local checkpoint's not-yet-pushed merge + revision was renamed before publication; inherited revision IDs stay intact. +- The unmerged provenance proposal's downgrade retains its identity mappings. + A real rollback test first reproduced their destruction. Historical upgrades, + already-stamped paths, retained records through downgrade, and re-upgrade + exercise the actual migration graph. +- Forced-overlap transactions test identical retries and incompatible input. + The latter must produce one successful import, one rejected writer, intact + winning records, and no losing rows or mappings. The API success tests still + use service overrides; they are not signed HTTP-to-database restore evidence. +- All four canonical search indexes remain present as GIN, with full-content + insert/update and tail-word score assertions. The inherited search candidate + preserves ranking SQL but does not supply GiST distance-order acceleration. + Representative p95 and migration build/lock/storage costs remain open gates + under [ADR-0020](../adr/0020-full-document-trigram-storage.md). +- [ADR-0022](../adr/0022-tenant-provenance-portability.md) retains the former + ADR-0007 proposal and #1497 discussion lineage. An all-open-PR file inventory + found another ADR-0007 in #1361. The PDF predecessor similarly preserves its + former 0005 proposal as ADR-0021; none is prematurely Accepted. + +## Reproduction + +Use a task-owned disposable PostgreSQL database, never customer data. Reuse the +isolated image/resource contract in +[search storage doctoring](search_trigram_storage.md), supply random test-only +credentials without printing them, and run from `backend/`: + +```sh +uv sync --locked +uv run --frozen python scripts/migrate_db.py +uv run --frozen python scripts/migrate_db.py +uv run --frozen python -m pytest -q -W error -ra --tb=short \ + tests/test_email_read_state_migration_postgres.py \ + tests/test_alembic_migrations.py tests/test_bootstrap_db.py \ + tests/test_data_api.py tests/test_legacy_document_scope_postgres.py \ + tests/test_workspace_document_migration.py \ + tests/test_container_dependency_pin_contract.py \ + tests/test_search.py tests/test_search_postgres.py \ + tests/test_search_answer.py tests/test_hybrid_retrieval_fusion.py \ + tests/test_tenant_provenance_bundle.py +``` + +The final terminal receipt is **345 passed, zero failed/skipped**, with +`-W error`, in 176.52 seconds. Fresh and repeated migration reach the single +`0021_merge_provenance_workspace` head. The unchanged +`test_export_counts_cited_segment_bytes_once` now passes, as do the retained +identity and forced incompatible-writer cases. The command and its exact +Compose cleanup both complete with exit zero. Ruff and 13 touched-document +file links also pass; no browser or performance pass is inferred. + +Local artifacts are under `/private/tmp/naruon-search-index-rca.VRmUrq`; they +are not hosted artifacts. `prop1497_tests.xml` has SHA-256 +`e17b09ba990b8d58a51dc00d4d7386474102e3b2996c65e9e79e07214e802eb5`; +`prop1497_migration.log` has SHA-256 +`df9f68f8245cff79e8e3f5a86339787d8d2abdfacedd30a061be2a35b0fd0fb5`. +Generated test records are regression inputs, not representative production +performance data. + +Historical release-note counts (`151 passed`, bootstrap `2 passed`, full +backend `1972 passed, 3 skipped`) describe earlier scoped runs. They did not +prove this combined migration graph and have been removed from customer-facing +release copy without deleting their historical record. BagIt 1.0, RO-Crate 1.3, +PROV, API authority, and the unchanged archive bounds remain specified in the +ADR and design contract, not exposed as implementation details in release copy. diff --git a/docs/planning/naruon-platform-plan.md b/docs/planning/naruon-platform-plan.md index 7e93ecbd2..f2e127dc3 100644 --- a/docs/planning/naruon-platform-plan.md +++ b/docs/planning/naruon-platform-plan.md @@ -119,7 +119,7 @@ KNOWLEDGE PLANE (dense semantic KG — the product) [PARTIAL] + typed, evidence-cited, confidence-weighted relations ``` -Scaffolding exists (`project_graph_objects` generic strings) but first-class Person/Event/Commitment entities are `[NEW]`; the semantic extractor is **test-only**, not wired into ingest. +Scaffolding exists (`project_graph_objects` generic strings) but first-class Person/Event/Commitment entities are `[NEW]`; email import invokes the semantic projection path when `PROJECT_GRAPH_EXTRACTION_ENABLED` is enabled (the default remains disabled). ### 4.3 Core entity model @@ -694,7 +694,7 @@ Hooks are **typed and ordered** (each point has a Pydantic input/output contract **Two tiers, evolving to three:** 1. **Content graph (LIVE)** — `content_nodes` + `content_segments` + `knowledge_graph_edges`: DOM decomposition of every email/attachment into structural nodes (`node_path`/`ordinal`) with **structural** edges (parent/child/sibling). Wired into ingest (alembic 0005–0007), built in one transaction with the email. -2. **Semantic/project graph (PARTIAL)** — `project_graph_objects` / `_edges` / `_corrections` (alembic 0009) with a full read/confirm/correct/traceability API. Today entities are generic `object_type` **strings** and the graph is **unpopulated in production** because `extract_project_semantics` / `persist_project_graph_projection` are **test-only** (verified: `callers_of(extract_project_semantics)` → not_found as a graph node; only test callers). Closing this is the single highest-leverage gap. +2. **Semantic/project graph (PARTIAL)** — `project_graph_objects` / `_edges` / `_corrections` (alembic 0009) with a full read/confirm/correct/traceability API. Today entities are generic `object_type` **strings**. Email import calls `persist_project_graph_projection` after content persistence when `PROJECT_GRAPH_EXTRACTION_ENABLED` is enabled; the default remains disabled, so deployment enablement and extraction-quality evidence remain open. 3. **Entity graph (TARGET)** — first-class typed entities and cross-entity relations, promoted from generic object strings. **Node & edge taxonomy.** Node types: `person`, `org`, `norm_group`, `project` (incl. Band), `thread`, `message`, `attachment`, `content_node`, `event`, `commitment`, `deliverable`, `requirement`, `wbs_item`, `erd_candidate`. Edge axes (density comes from many *simultaneous* relation axes): Social (`person—person`, `person—org`, `person—norm_group` **multi-membership**), Communication (`message—thread`, in_reply_to/references, sender/recipient), Temporal/event (`event—event` **enables/conflicts/unrelated**, resolved by density not asking; `event—commitment`), Commitment (status axis {confirmed|tentative|desired} + RSVP direction), Provenance (`object—content_segment` cited evidence, `object—extractor`, `correction—object`). Every semantic node/edge stores `confidence`, `extractor_name`, `extractor_version`, and cited `source_segment_uids` — auditable back to the exact DOM segment. @@ -734,7 +734,7 @@ Hooks are **typed and ordered** (each point has a Pydantic input/output contract Postgres is the system of record. **All new object names are 2+ word `snake_case`** (existing Camel/Pascal names left as-is). 38 SQLAlchemy tables today. -*Current core tables by domain:* **Identity/tenant** (`user_accounts`, `provider_accounts`, `organization_entities`, `organization_groups`, `scoped_role_assignments`, `workspace_entities`, `project_folders`, `tenant_configs`, `workspace_documents`); **Email live/legacy** (`email_records`, `email_attachments`); **Email normalized/in-progress** (`email_raws`, `email_messages`, `email_instances`, `email_threads`, `email_thread_edges` — not yet linked to `email_records`); **Content graph live** (`content_nodes`, `content_segments`, `knowledge_graph_edges`); **Project/semantic graph** (read-side live, extraction test-only: `project_graph_objects`, `project_graph_edges`, `project_graph_corrections`); **Tasks/workflow** (`ticket_tasks`, `workflow_definitions`, `agent_run_records`, `prompt_templates`); **Connectors/DAV/scheduling** (`caldav_accounts`, `webdav_accounts`, `calendar_writeback_sources`, `reply_trackers`, `sender_relationships`, `connector_signal_events`, `provider_writeback_retry_items`, `workspace_runner_configs`); **LLM/governance** (`llm_providers`, `audit_logs`, `security_audit_events`). +*Current core tables by domain:* **Identity/tenant** (`user_accounts`, `provider_accounts`, `organization_entities`, `organization_groups`, `scoped_role_assignments`, `workspace_entities`, `project_folders`, `tenant_configs`, `workspace_documents`); **Email live/legacy** (`email_records`, `email_attachments`); **Email normalized/in-progress** (`email_raws`, `email_messages`, `email_instances`, `email_threads`, `email_thread_edges` — not yet linked to `email_records`); **Content graph live** (`content_nodes`, `content_segments`, `knowledge_graph_edges`); **Project/semantic graph** (read-side live, feature-gated ingest projection: `project_graph_objects`, `project_graph_edges`, `project_graph_corrections`); **Tasks/workflow** (`ticket_tasks`, `workflow_definitions`, `agent_run_records`, `prompt_templates`); **Connectors/DAV/scheduling** (`caldav_accounts`, `webdav_accounts`, `calendar_writeback_sources`, `reply_trackers`, `sender_relationships`, `connector_signal_events`, `provider_writeback_retry_items`, `workspace_runner_configs`); **LLM/governance** (`llm_providers`, `audit_logs`, `security_audit_events`). *Proposed additions (TARGET, all 2+word snake_case):* @@ -789,11 +789,13 @@ This section is the honest reconciliation of vision against the code as it stand - Reply-SLA escalation → TicketTask; self-sent-note → TicketTask; sender-relationship ontology → next_action. - ABAC owner-scoping + `EncryptedString` secrets; Prometheus/OTEL flags + operational-signal API; audit + security-audit events. - Live screens (`/mail`, `/calendar`, `/tasks`, `/projects`, `/data`, `/search`, `/ai-hub`, `/prompt-studio`, `/tools`, `/security`, `/settings`) + the BFF proxy; `NetworkGraph.tsx` and `DecisionPointCard.tsx` components. +- Branch-local GA-1 portability slice: authoritative OIDC/server sessions can export or import a bounded deterministic BagIt/RO-Crate ZIP for the exact workspace's cited email, textual attachment, content/KG, project-graph, and correction closure. It is not full tenant portability and is not protected-branch evidence until merged. +- Exact branch verification: Ruff passed, the combined provenance service/API suite passed 151 tests, fresh-schema and legacy-table PostgreSQL bootstrap paths passed 2 tests, and the full backend suite passed 1,972 tests with 3 skipped. All Python runs promoted warnings to errors. ### 8.2 `[PARTIAL]` — exists but incomplete, **reconcile / wire** -- **Semantic project graph:** a full read/confirm/correct/traceability API over `project_graph_objects`/`_edges`/`_corrections` is live, but its extractor (`extract_project_semantics`/`persist_project_graph_projection`) is **test-only** — no ingest worker or API calls it, so the semantic graph is **unpopulated in production**. (Verified via `callers_of` → not_found + grep showing only test callers.) *Highest-leverage gap: the dense KG is built but empty.* +- **Semantic project graph:** a full read/confirm/correct/traceability API over `project_graph_objects`/`_edges`/`_corrections` is live. Email import calls `persist_project_graph_projection` after content-graph persistence when the feature flag is enabled; the default is disabled. Remaining gaps include deployment enablement, extraction-quality evidence, full portability, and complete multi-account ownership reconciliation. - **Multi-account model:** a normalized `email_messages/instances/raws/threads` model exists in parallel to the live `email_records` model and is **not reconciled**; live ingest still writes the legacy model. -- **Project-graph curation UI** (`/projects`) is read/confirm/correct only; extraction not wired. +- **Project-graph curation UI** (`/projects`) is read/confirm/correct only; ingest extraction is feature-gated and disabled by default. ### 8.3 `[NEW]` / `[PLANNED]` — net-new, **build** - First-class typed entities: **Person / NormGroup / Event / Commitment / Approval-outcome / DisclosureConsent** (today only generic `object_type` strings) and their typed, evidence-cited relations. @@ -813,9 +815,9 @@ This section is the honest reconciliation of vision against the code as it stand Ordered for maximum leverage against the vision while keeping each phase independently shippable. Every phase moves a concrete current-state gap toward the mission and preserves all five cross-cutting principles. -### Phase 0 — MVP: make the dense KG real (populate what's already built) -The single highest-leverage move: the semantic graph exists but is empty. -1. **Wire semantic extraction into ingest** — call `extract_project_semantics` / `persist_project_graph_projection` from the ingest worker so `project_graph_objects` populate in production (today they are test-only). +### Phase 0 — MVP: make the dense KG real (operate what's already built) +The single highest-leverage move is to enable and prove the existing semantic projection path on production-shaped data. +1. **Verify semantic extraction quality in ingest** — preserve the live `persist_project_graph_projection` call and add source-backed quality, correction, and portability evidence instead of describing production wiring as absent. 2. **Reconcile the multi-account email model** enough to have one source of truth for threads/messages feeding the graph. 3. **Extend hybrid search** to `content_segments` and populated `project_graph_objects` (search meaning, not just bodies). 4. Wire the existing `DecisionPointCard` to real synthesized thread cards (Epic 1 / UC-08) — first visible "judgment-ready structure." diff --git a/docs/product-technical-gap-baseline.md b/docs/product-technical-gap-baseline.md index 98bc17d2a..c5026834d 100644 --- a/docs/product-technical-gap-baseline.md +++ b/docs/product-technical-gap-baseline.md @@ -570,9 +570,9 @@ button, form, navigation, chart, or asynchronous data surface. |---|---|---|---|---| | Binary object lifecycle | large/deferred document bytes cannot remain an inline database strategy | S3-compatible implementation is Draft | #1076, #1364 | upload/read/recognize/retain/delete/backfill/orphan round trip with real integration | | Disaster recovery | a release is not enterprise-ready without restore evidence | HA evaluation exists; production WAL/PITR policy remains incomplete | #1428 | WAL archive/PITR, failover fencing, backup and clean restore rehearsal | -| Tenant export/reimport | customers need exit and migration without losing provenance | no single demonstrated full tenant round trip | #1428 | export → clean instance import preserving source, opaque IDs, history, evidence, policy | +| Tenant export/reimport | customers need exit and migration without losing provenance | Branch-local proposed ADR-0007 slice provides authoritative-session GET/import APIs for a bounded deterministic workspace project-evidence closure: cited email, parser-admitted textual attachment, content/KG, project objects/edges, and corrections round-trip with fresh database keys. Ruff, 151 focused warnings-as-errors tests, 2 fresh/legacy bootstrap tests, and the full backend suite (`1972 passed, 3 skipped`) pass. The slice excludes uncited/mailbox-only mail, binary objects, credentials, provider/connector state, embeddings, and audit history. | #1428, `feat/tenant-provenance-roundtrip` | extend the bounded slice to full tenant/mailbox/binary/credential/provider/connector/audit portability; rehearse clean-instance export/import preserving source, opaque IDs, history, evidence, and policy | | Retention/legal hold/disposition | deletion and evidence preservation conflict unless modeled | partial security/key/retention work exists across repository history | #1428, #1364 | purpose-scoped retention, legal hold, verified disposition, object/DB reconciliation | -| Attachment parser admission and unsupported formats | a file above 20 MiB can pass import transport but fail later at a hidden parser limit, while unsupported binaries are not searchable | Naruon import transport and generic deferred parser admission are bounded at 64 MiB; the NewsDOM `/parse` provider contract remains 20 MiB, so PDF bytes from 20–64 MiB are admitted and retained fail-closed but are not sent to NewsDOM; unsupported types remain metadata-only | #1427, #1469, #1353, #1419, NewsDOM #682/#707 | one documented bounded admission contract, provider-side PDF limit alignment or an explicit large-PDF fallback, parser/status evidence, deferred recognition, and object-backed retention before increasing the bound again | +| Attachment parser admission and unsupported formats | a file above 20 MiB can pass import transport but fail later at a hidden parser limit, while unsupported binaries are not searchable | Naruon upload and deferred parser paths are bounded at 64 MiB in the proposed stack; canonical NewsDOM owner PR #665 was reopened after an unmerged close and proposes the matching `/parse` bound, but no immutable owner release exists yet; unsupported types remain metadata-only | #1427, #1469, #1353, #1419, NewsDOM #665 | protect-merge and release the owner contract, pin the consumer to that immutable release, retain parser/status evidence and deferred recognition, and move large bytes to object-backed lifecycle before increasing the bound again | ### P0 — Evidence-based AI and document intelligence @@ -658,7 +658,7 @@ review approval, hidden manual database edits, or an unreleased sibling branch. 1. Implement typed temporal/multi-membership Person/Event/Commitment graph. 2. Complete status-weighted scheduling (#978/#988/#989/#990). 3. Complete evidence-based mail/document/media resolution (#1350). -4. Add tenant export/reimport and customer-exit evidence. +4. Extend the bounded workspace project-evidence bundle to full tenant export/reimport and customer-exit evidence. 5. Run the full buyer journey and failure/restore variants. ### Wave 3 — North-star platform diff --git a/docs/superpowers/plans/2026-08-31-tenant-provenance-roundtrip.md b/docs/superpowers/plans/2026-08-31-tenant-provenance-roundtrip.md new file mode 100644 index 000000000..998229210 --- /dev/null +++ b/docs/superpowers/plans/2026-08-31-tenant-provenance-roundtrip.md @@ -0,0 +1,82 @@ +# Tenant Provenance Round-Trip Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Ship a signed deterministic export/reimport path preserving the current +workspace's email-derived project evidence and correction history. + +**Architecture:** Query the existing SQLAlchemy provenance graph by signed scope, +serialize stable identities into a bounded BagIt/RO-Crate ZIP, validate the full +closure, and restore it transactionally with fresh keys. The existing data router +exposes the operations. + +**Tech Stack:** Python 3.14, FastAPI, SQLAlchemy async, Python standard library, +PostgreSQL/pgvector, pytest. + +**Spec:** `docs/superpowers/specs/2026-08-31-tenant-provenance-roundtrip-design.md` + +## Global Constraints + +- Never inspect, copy, commit, or upload `tests/real_datasets`. +- Never serialize credentials, encrypted secrets, embeddings, integer keys, + binary documents, provider URLs or tokens, or legacy audit details. +- Source scope and target authority come from the signed session. +- Validate the bounded archive and reference graph before mutation. +- Add no dependency. The coexistence-safe implementation adds one internal + portable-to-database identity mapping table; its multi-word sequential key + `provenance_identity_id` is never serialized or exposed through the API. +- Treat warning-class output as failure. + +--- + +### Task 1: Deterministic envelope and validator + +**Files:** Create `backend/services/tenant_provenance_bundle.py`; create +`backend/tests/test_tenant_provenance_bundle.py`. + +**Interfaces:** `build_provenance_archive(records) -> bytes` and +`parse_provenance_archive(archive_bytes) -> dict[str, object]`. + +- [ ] Write deterministic-byte and exact-entry tests; run and verify RED. +- [ ] Add RED tests for tampering, unsafe/colliding paths, bounds, missing or + extra entries, and duplicate JSON keys. +- [ ] Implement fixed ZIP metadata, canonical JSON, SHA-512 manifests, RO-Crate + metadata, bounded reads, and fail-closed validation with the standard library. +- [ ] Run the focused tests with `PYTHONWARNINGS=error` and require clean output. + +### Task 2: Scoped export and transactional reimport + +**Files:** Modify the service and focused test from Task 1. + +**Interfaces:** `TenantProvenanceScope`, `export_tenant_provenance`, +`import_tenant_provenance`, and `ImportReceipt`. + +- [ ] Seed Email through Correction in the PostgreSQL harness; write and run a + failing clean-target round-trip assertion over UIDs, hashes, and correction JSON. +- [ ] Add RED scope, dangling-reference rollback, conflict, text-admission, + secret/embedding absence, and idempotency tests. +- [ ] Implement scoped selects, stable records, pre-mutation validation, FK-order + restore maps, exact duplicate skips, and a single transaction boundary. +- [ ] Run the focused PostgreSQL suite with warnings promoted to errors. + +### Task 3: Signed data API + +**Files:** Modify `backend/api/data.py` and `backend/tests/test_data_api.py`. + +**Interfaces:** `GET /api/data/provenance-bundle` and +`POST /api/data/provenance-bundle/import`. + +- [ ] Add RED tests for authentication, response media/disposition, oversize + input, fixed safe errors, and target-scope rewrite. +- [ ] Add the two signed endpoints using bounded raw ZIP bytes and known errors. +- [ ] Run focused data API and bundle tests with warnings promoted to errors. + +### Task 4: Documentation truth and release evidence + +**Files:** Modify ADR index, product baseline, platform plan, and CHANGELOG. + +- [ ] Index ADR-0022 (formerly ADR-0007), correct the stale test-only graph claim, and retain the full + portability gap after recording this bounded implemented slice. +- [ ] Run Ruff, focused suites, the full backend suite, and `git diff --check`. +- [ ] Confirm `git ls-files tests/real_datasets` is empty and inspect the complete + diff before committing and opening a PR. diff --git a/docs/superpowers/specs/2026-08-31-tenant-provenance-roundtrip-design.md b/docs/superpowers/specs/2026-08-31-tenant-provenance-roundtrip-design.md new file mode 100644 index 000000000..d65689362 --- /dev/null +++ b/docs/superpowers/specs/2026-08-31-tenant-provenance-roundtrip-design.md @@ -0,0 +1,108 @@ +# Tenant Provenance Round-Trip Design + +**Status:** Accepted design for the first GA-1 portability slice. +**Decision:** [ADR-0022](../../adr/0022-tenant-provenance-portability.md) (formerly ADR-0007; proposal identity preserved in #1497) + +## Outcome + +A signed user can download a deterministic ZIP evidence bundle for the current +workspace's project-graph closure and import it into an authorized target scope. +A round trip preserves stable source, graph, citation, and correction identities +while allocating fresh database keys. + +This is not full tenant portability. Mail without workspace-grounded project +evidence, credentials, connector state, embeddings, and binary document objects +remain outside v1. + +## Contract + +```text +bagit.txt +bag-info.txt +manifest-sha512.txt +tagmanifest-sha512.txt +ro-crate-metadata.json +data/records.json +``` + +`data/records.json` carries the profile, schema version, opaque bundle UID, +source scope, export activity, and ordered arrays for emails, attachments, +content nodes, content segments, structural edges, project objects, project +edges, and corrections. Relationships use stable UIDs, never integer keys. +RO-Crate metadata describes the root Dataset, payload File, export Activity, and +software Agent using RO-Crate 1.3 and PROV terms. + +ZIP entries use fixed timestamps, mode, compression, and sorted names. SHA-512 +manifests cover exact bytes. Import validates archive structure, paths, byte +limits, manifests, JSON profile, scope closure, uniqueness, and every reference +before adding any ORM object. + +## Data flow + +```mermaid +flowchart LR + A[Signed AuthContext] --> B[Exact workspace project rows] + B --> C[Cited segment UIDs] + C --> D[Email provenance closure] + D --> E[Canonical records JSON] + E --> F[RO-Crate and BagIt manifests] + F --> G[Deterministic ZIP] + G --> H[Validate bytes and references] + H --> I[Remap stable UIDs to fresh DB keys] + I --> J[One transaction and scoped receipt] +``` + +## Scope and security invariants + +- Source reads match signed user and organization; project records additionally + match the signed workspace. +- The exported email closure is the set referenced by exact-workspace project + objects. Descendants are included only when owned by those emails. +- Import rewrites owner/workspace scope from the verified target session. +- Attachment content is accepted only for parser-confirmed textual media. +- Archive total, entry count, entry size, compression ratio, and JSON depth are + bounded (64 MiB archive/total uncompressed bytes, 64 entries, 32 MiB per + entry, and 100:1 compression ratio). No symlink, network fetch, external + context retrieval, or extraction. +- Errors use fixed codes and never echo record content or attacker paths. + +## Import behavior + +Import order is Email, Attachment, ContentNode, ContentSegment, +KnowledgeGraphEdge, ProjectGraphObject, ProjectGraphEdge, Correction. Each stage +maps stable identities to new keys. Dangling references or differing existing +identities abort the transaction. Exact duplicates are skipped. Absence never +deletes target data. + +Email and Attachment rows are owner-and-organization sources without workspace +scope, so a same-owner transfer reuses identical rows. If the source graph still +exists in the same database, the importer records a scoped portable-to-database +identity mapping and deterministically remaps every graph UID and typed graph +reference, including recognized UID keys nested in object and correction +metadata. Arbitrary strings are never value-matched or rewritten. Export +reverses that mapping, preserving the portable archive bytes. +If native records or multiple imported origins coexist, export instead emits a +single target-scoped closure with the target database UIDs so every record stays +exportable without pretending that the mixed archive has one original source. +The source-user component stored in the archive is a one-way digest rather than +an account identifier and must be exactly a lowercase SHA-256 hex digest. +Concurrent imports of the same source-to-target scope are serialized with a +PostgreSQL transaction advisory lock; other scopes remain independent. + +## API + +- `GET /api/data/provenance-bundle` returns `application/zip`. +- `POST /api/data/provenance-bundle/import` accepts bounded raw ZIP bytes and + returns created/skipped counts plus a verified manifest digest. +- Both routes use the existing signed authentication dependency and reject the + HMAC fallback verifier because it is not authoritative workspace-membership + evidence. Only OIDC-verified contexts are accepted; dependency override is + test-only evidence. + +## Verification + +A real PostgreSQL test seeds the eight-model closure, exports it, imports into a +clean target scope, and compares stable UIDs, hashes, citations, and correction +history. Unit tests cover deterministic bytes, tampering, unsafe paths, dangling +references, cross-scope exclusion, secret/embedding absence, idempotent retry, +and rollback. API tests cover signed-session and target-scope enforcement.