From be90ea860753f5c675ca3cbc4bf0d99e0089ef85 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 13 Aug 2026 17:16:33 +0900 Subject: [PATCH 01/29] feat(naruon): persist DiskSage file lineage ontology --- .../versions/0018_disksage_file_lineage.py | 80 +++++++ backend/api/disksage.py | 129 +++++++++++ backend/db/models.py | 55 +++++ backend/main.py | 2 + backend/services/disksage_file_lineage.py | 214 ++++++++++++++++++ backend/tests/test_disksage_file_lineage.py | 138 +++++++++++ .../disksage-file-lineage-handoff.md | 41 ++++ 7 files changed, 659 insertions(+) create mode 100644 backend/alembic/versions/0018_disksage_file_lineage.py create mode 100644 backend/api/disksage.py create mode 100644 backend/services/disksage_file_lineage.py create mode 100644 backend/tests/test_disksage_file_lineage.py create mode 100644 docs/architecture/disksage-file-lineage-handoff.md diff --git a/backend/alembic/versions/0018_disksage_file_lineage.py b/backend/alembic/versions/0018_disksage_file_lineage.py new file mode 100644 index 000000000..4f71c552a --- /dev/null +++ b/backend/alembic/versions/0018_disksage_file_lineage.py @@ -0,0 +1,80 @@ +"""persist scoped, encrypted DiskSage file lineage envelopes + +Revision ID: 0018_disksage_file_lineage +Revises: 0017_merge_newsdom_carddav_heads +Create Date: 2026-08-13 00:00:00.000000 + +The Rust DiskSage verifier remains authoritative for copy/provider proof. Naruon +stores the validated envelope for workspace-scoped provenance and exposes only +the redacted graph projection in list responses. +""" + +from alembic import op +import sqlalchemy as sa + + +revision = "0018_disksage_file_lineage" +down_revision = "0017_merge_newsdom_carddav_heads" +branch_labels = None +depends_on = None + +_TABLE = "disksage_file_lineage_records" + + +def upgrade() -> None: + connection = op.get_bind() + inspector = sa.inspect(connection) + if not inspector.has_table(_TABLE): + op.create_table( + _TABLE, + sa.Column("lineage_record_uid", sa.String(length=96), nullable=False), + sa.Column("user_id", sa.String(), nullable=False), + sa.Column("organization_id", sa.String(), nullable=True), + sa.Column("workspace_id", sa.String(), nullable=False), + sa.Column("lineage_fingerprint", sa.String(length=64), nullable=False), + sa.Column("envelope_sha256", sa.String(length=64), nullable=False), + sa.Column("schema_version", sa.Integer(), nullable=False), + sa.Column("schema_kind", sa.String(length=96), nullable=False), + sa.Column("source_kind", sa.String(length=64), nullable=False), + sa.Column("archive_kind", sa.String(length=64), nullable=False), + sa.Column("raw_content_sha256", sa.String(length=64), nullable=False), + sa.Column("raw_content_blake3", sa.String(length=64), nullable=False), + sa.Column("bytes", sa.BigInteger(), nullable=False), + sa.Column("ontology_class", sa.String(length=256), nullable=False), + sa.Column("ontology_relation_count", sa.Integer(), nullable=False), + sa.Column("ontology_predicates", sa.JSON(), nullable=False), + sa.Column("provider", sa.String(length=32), nullable=False), + sa.Column("provider_sync_confirmed", sa.Boolean(), nullable=False), + sa.Column("envelope_json_encrypted", sa.String(), nullable=False), + sa.Column("created_at", sa.DateTime(timezone=True), nullable=False), + sa.PrimaryKeyConstraint("lineage_record_uid"), + sa.UniqueConstraint( + "user_id", + "workspace_id", + "lineage_fingerprint", + name="uq_disksage_lineage_workspace_fingerprint", + ), + ) + + op.create_index( + "ix_disksage_lineage_scope_time", + _TABLE, + ["user_id", "organization_id", "workspace_id", "created_at"], + if_not_exists=True, + ) + op.create_index( + "ix_disksage_lineage_ontology_class", + _TABLE, + ["workspace_id", "ontology_class"], + if_not_exists=True, + ) + + +def downgrade() -> None: + op.drop_index( + "ix_disksage_lineage_ontology_class", table_name=_TABLE, if_exists=True + ) + op.drop_index("ix_disksage_lineage_scope_time", table_name=_TABLE, if_exists=True) + connection = op.get_bind() + if sa.inspect(connection).has_table(_TABLE): + op.drop_table(_TABLE) diff --git a/backend/api/disksage.py b/backend/api/disksage.py new file mode 100644 index 000000000..8079c269e --- /dev/null +++ b/backend/api/disksage.py @@ -0,0 +1,129 @@ +import datetime +import json +import uuid + +from fastapi import APIRouter, Depends, HTTPException, Query +from sqlalchemy import select +from sqlalchemy.ext.asyncio import AsyncSession + +from api.auth import AuthContext, get_auth_context +from db.models import DiskSageFileLineageRecord +from db.session import get_db +from services.disksage_file_lineage import ( + FileLineageEnvelope, + FileLineageSummary, + canonical_envelope_sha256, + ontology_predicates, +) + +router = APIRouter(prefix="/api/disksage", tags=["disksage"]) + + +def _summary(record: DiskSageFileLineageRecord) -> FileLineageSummary: + return FileLineageSummary( + lineage_record_uid=record.lineage_record_uid, + lineage_fingerprint=record.lineage_fingerprint, + schema_version=record.schema_version, + source_kind=record.source_kind, + archive_kind=record.archive_kind, + raw_content_sha256=record.raw_content_sha256, + raw_content_blake3=record.raw_content_blake3, + bytes=record.bytes, + ontology_class=record.ontology_class, + ontology_relation_count=record.ontology_relation_count, + ontology_predicates=list(record.ontology_predicates or []), + provider=record.provider, + provider_sync_confirmed=record.provider_sync_confirmed, + created_at=record.created_at.isoformat(), + ) + + +def _encrypted_envelope_json(envelope: FileLineageEnvelope) -> str: + return json.dumps( + envelope.model_dump(mode="json"), + ensure_ascii=False, + separators=(",", ":"), + sort_keys=True, + ) + + +@router.post("/file-lineage", response_model=FileLineageSummary, status_code=201) +async def ingest_file_lineage( + envelope: FileLineageEnvelope, + auth_context: AuthContext = Depends(get_auth_context), + db: AsyncSession = Depends(get_db), +) -> FileLineageSummary: + """Persist one validated DiskSage envelope without authorizing source eviction.""" + + envelope_sha256 = canonical_envelope_sha256(envelope) + lineage_fingerprint = envelope.cloud_copy.lineage_fingerprint + existing_result = await db.execute( + select(DiskSageFileLineageRecord).where( + DiskSageFileLineageRecord.user_id == auth_context.user_id, + DiskSageFileLineageRecord.workspace_id == auth_context.workspace_id, + DiskSageFileLineageRecord.lineage_fingerprint == lineage_fingerprint, + ) + ) + existing = existing_result.scalar_one_or_none() + if existing is not None: + if existing.envelope_sha256 != envelope_sha256: + raise HTTPException( + status_code=409, + detail="lineage fingerprint is already bound to different evidence", + ) + return _summary(existing) + + record = DiskSageFileLineageRecord( + lineage_record_uid=f"disksage_lineage_{uuid.uuid4().hex}", + user_id=auth_context.user_id, + organization_id=auth_context.organization_id, + workspace_id=auth_context.workspace_id, + lineage_fingerprint=lineage_fingerprint, + envelope_sha256=envelope_sha256, + schema_version=envelope.schema_version, + schema_kind=envelope.schema_kind, + source_kind=envelope.source_kind, + archive_kind=envelope.archive_kind, + raw_content_sha256=envelope.raw_content_sha256, + raw_content_blake3=envelope.raw_content_blake3, + bytes=envelope.bytes, + ontology_class=envelope.ontology_class, + ontology_relation_count=len(envelope.ontology_relations), + ontology_predicates=ontology_predicates(envelope), + provider=envelope.cloud_copy.provider, + provider_sync_confirmed=envelope.cloud_copy.provider_sync_confirmed, + envelope_json_encrypted=_encrypted_envelope_json(envelope), + created_at=datetime.datetime.now(datetime.timezone.utc), + ) + db.add(record) + try: + await db.commit() + await db.refresh(record) + except RuntimeError as error: + if "ENCRYPTION_KEY is required" not in str(error): + raise + raise HTTPException( + status_code=503, + detail="Server encryption key is not configured. Contact your workspace administrator.", + ) from error + return _summary(record) + + +@router.get("/file-lineage", response_model=list[FileLineageSummary]) +async def list_file_lineage( + limit: int = Query(default=50, ge=1, le=100), + auth_context: AuthContext = Depends(get_auth_context), + db: AsyncSession = Depends(get_db), +) -> list[FileLineageSummary]: + """List redacted lineage graph projections; encrypted envelope values stay server-side.""" + + result = await db.execute( + select(DiskSageFileLineageRecord) + .where( + DiskSageFileLineageRecord.user_id == auth_context.user_id, + DiskSageFileLineageRecord.workspace_id == auth_context.workspace_id, + ) + .order_by(DiskSageFileLineageRecord.created_at.desc()) + .limit(limit) + ) + return [_summary(record) for record in result.scalars().all()] diff --git a/backend/db/models.py b/backend/db/models.py index 98e17eef2..5193533d1 100644 --- a/backend/db/models.py +++ b/backend/db/models.py @@ -7,6 +7,7 @@ from cryptography.fernet import Fernet, InvalidToken from pgvector.sqlalchemy import Vector from sqlalchemy import ( + BigInteger, Boolean, DateTime, ForeignKey, @@ -1102,6 +1103,60 @@ class KnowledgeGraphEdgeRecord(Base): ) +class DiskSageFileLineageRecord(Base): + """Encrypted DiskSage provenance scoped to the authenticated workspace.""" + + __tablename__ = "disksage_file_lineage_records" + __table_args__ = ( + UniqueConstraint( + "user_id", + "workspace_id", + "lineage_fingerprint", + name="uq_disksage_lineage_workspace_fingerprint", + ), + Index( + "ix_disksage_lineage_scope_time", + "user_id", + "organization_id", + "workspace_id", + "created_at", + ), + Index("ix_disksage_lineage_ontology_class", "workspace_id", "ontology_class"), + ) + + lineage_record_uid: Mapped[str] = mapped_column(String(96), primary_key=True) + user_id: Mapped[str] = mapped_column(String, index=True, nullable=False) + organization_id: Mapped[str | None] = mapped_column( + String, index=True, nullable=True + ) + workspace_id: Mapped[str] = mapped_column(String, index=True, nullable=False) + lineage_fingerprint: Mapped[str] = mapped_column(String(64), nullable=False) + envelope_sha256: Mapped[str] = mapped_column(String(64), nullable=False) + schema_version: Mapped[int] = mapped_column(Integer, nullable=False) + schema_kind: Mapped[str] = mapped_column(String(96), nullable=False) + source_kind: Mapped[str] = mapped_column(String(64), nullable=False) + archive_kind: Mapped[str] = mapped_column(String(64), nullable=False) + raw_content_sha256: Mapped[str] = mapped_column(String(64), nullable=False) + raw_content_blake3: Mapped[str] = mapped_column(String(64), nullable=False) + bytes: Mapped[int] = mapped_column(BigInteger, nullable=False) + ontology_class: Mapped[str] = mapped_column(String(256), nullable=False) + ontology_relation_count: Mapped[int] = mapped_column(Integer, nullable=False) + ontology_predicates: Mapped[list[str]] = mapped_column( + JSON, default=list, nullable=False + ) + provider: Mapped[str] = mapped_column(String(32), nullable=False) + provider_sync_confirmed: Mapped[bool] = mapped_column(Boolean, nullable=False) + # Source paths and metadata evidence are intentionally not queryable plaintext. + envelope_json_encrypted: Mapped[str] = mapped_column( + EncryptedString, nullable=False + ) + created_at: Mapped[datetime.datetime] = mapped_column( + DateTime(timezone=True), + default=lambda: datetime.datetime.now(datetime.timezone.utc), + nullable=False, + ) + + class ProjectGraphObjectRecord(Base): __tablename__ = "project_graph_objects" __table_args__ = ( diff --git a/backend/main.py b/backend/main.py index 0ad7762a8..357450f8a 100644 --- a/backend/main.py +++ b/backend/main.py @@ -31,6 +31,7 @@ from api.ai_hub import router as ai_hub_router from api.projects import router as projects_router from api.session import router as auth_session_router +from api.disksage import router as disksage_router from core.config import canonical_origin, settings from core.telemetry import setup_telemetry from core.version import get_release_version @@ -237,6 +238,7 @@ async def add_security_headers(request: Request, call_next): app.include_router(ai_hub_router, dependencies=PRIVATE_API_DEPENDENCIES) app.include_router(projects_router, dependencies=PRIVATE_API_DEPENDENCIES) app.include_router(auth_session_router, dependencies=PRIVATE_API_DEPENDENCIES) +app.include_router(disksage_router, dependencies=PRIVATE_API_DEPENDENCIES) @app.get("/") diff --git a/backend/services/disksage_file_lineage.py b/backend/services/disksage_file_lineage.py new file mode 100644 index 000000000..e9f1bec64 --- /dev/null +++ b/backend/services/disksage_file_lineage.py @@ -0,0 +1,214 @@ +"""Strict, scope-neutral validation for DiskSage file-lineage envelopes. + +DiskSage performs the filesystem and provider proof work locally. Naruon only +accepts the resulting immutable envelope; it never treats a local File +Provider copy as a provider API write or as permission to evict the source. +""" + +from __future__ import annotations + +import hashlib +import json +from typing import Literal + +from pydantic import BaseModel, ConfigDict, Field, model_validator + + +HEX64_PATTERN = r"^[0-9a-fA-F]{64}$" +ONTOLOGY_CLASS_PATTERN = ( + r"^https://disksage\.app/ontology#[A-Za-z][A-Za-z0-9_-]{0,127}$" +) +PROVIDER_VALUES = Literal["icloud", "onedrive", "google-drive"] +ARCHIVE_KIND_VALUES = Literal[ + "document", + "media", + "archive", + "dataset", + "backup", + "creative", + "incomplete-download", +] +REVIEW_DISPOSITION_VALUES = Literal["approved", "held"] +SYNC_KIND_VALUES = Literal["provider-api", "provider-native-status"] + + +class _StrictModel(BaseModel): + model_config = ConfigDict(extra="forbid") + + +class FileLineageRelation(_StrictModel): + subject: str = Field(min_length=1, max_length=2048) + predicate: str = Field(min_length=1, max_length=256) + object: str = Field(min_length=1, max_length=2048) + source: str = Field(min_length=1, max_length=256) + + @model_validator(mode="after") + def reject_control_values(self) -> FileLineageRelation: + if any( + any(ord(character) < 32 for character in value) + for value in (self.subject, self.predicate, self.object, self.source) + ): + raise ValueError("lineage relation contains a control character") + return self + + +class FileMetadataEvidence(_StrictModel): + field: str = Field(min_length=1, max_length=128) + value: str = Field(max_length=2048) + source: str = Field(min_length=1, max_length=256) + confidence: Literal["high", "medium", "low", "unknown"] + + +class ProductionTimeLineage(_StrictModel): + selected_value_ms: int = Field(ge=0) + selected_source: str = Field(min_length=1, max_length=256) + confidence: Literal["high", "medium", "low", "unknown"] + evidence_precedence: list[str] = Field(min_length=1, max_length=8) + + +class FilesystemTimeLineage(_StrictModel): + created_at_ms: int = Field(ge=0) + modified_at_ms: int = Field(ge=0) + + +class ReviewLineage(_StrictModel): + candidate_fingerprint: str = Field(pattern=HEX64_PATTERN) + review_fingerprint: str = Field(pattern=HEX64_PATTERN) + requires_review: bool + reason_codes: list[str] = Field(max_length=64) + decision_id: str | None = Field(default=None, max_length=256) + disposition: REVIEW_DISPOSITION_VALUES | None = None + reviewed_at_ms: int | None = Field(default=None, ge=0) + reviewed_by: str | None = Field(default=None, max_length=256) + rationale: str | None = Field(default=None, max_length=2000) + + @model_validator(mode="after") + def bind_decision_fields(self) -> ReviewLineage: + if self.disposition is not None and ( + self.decision_id is None + or self.reviewed_at_ms is None + or not self.reviewed_by + or not self.rationale + ): + raise ValueError("review disposition is missing its decision evidence") + return self + + +class RemoteContentProof(_StrictModel): + object_id: str = Field(min_length=1, max_length=512) + revision: str = Field(min_length=1, max_length=512) + algorithm: Literal["sha256", "quick-xor"] + checksum: str = Field(min_length=1, max_length=256) + location_bound: bool + location_proof: str | None = Field(default=None, max_length=2048) + + +class CloudCopyLineage(_StrictModel): + receipt_id: str = Field(pattern=HEX64_PATTERN) + lineage_fingerprint: str = Field(pattern=HEX64_PATTERN) + provider: PROVIDER_VALUES + destination_account_scope: Literal["personal", "organization", "shared", "unknown"] + destination: str = Field(min_length=1, max_length=4096) + copied_at_ms: int = Field(ge=0) + copy_verification_method: Literal["copied-by-disk-sage", "adopted-existing"] + local_copy_verified: bool + provider_write_executed: bool + provider_sync_confirmed: bool + sync_evidence_record_id: str | None = Field(default=None, max_length=256) + sync_evidence_kind: SYNC_KIND_VALUES | None = None + sync_evidence_id: str | None = Field(default=None, max_length=512) + sync_confirmed_at_ms: int | None = Field(default=None, ge=0) + remote_object_id: str | None = Field(default=None, max_length=512) + remote_revision: str | None = Field(default=None, max_length=512) + remote_location_bound: bool | None = None + + @model_validator(mode="after") + def bind_provider_evidence(self) -> CloudCopyLineage: + if not self.local_copy_verified: + raise ValueError("lineage copy is not locally verified") + if self.provider_write_executed: + raise ValueError("Naruon cannot accept a provider-write claim") + evidence_fields = ( + self.sync_evidence_record_id, + self.sync_evidence_kind, + self.sync_evidence_id, + self.sync_confirmed_at_ms, + ) + if self.provider_sync_confirmed and any( + value is None for value in evidence_fields + ): + raise ValueError("confirmed sync is missing provider evidence") + if self.remote_location_bound is True and not self.remote_object_id: + raise ValueError("remote location binding is missing its object id") + return self + + +class FileLineageEnvelope(_StrictModel): + schema_version: Literal[1] + schema_kind: Literal["disksage.file-lineage"] + source_kind: Literal["file"] + archive_kind: ARCHIVE_KIND_VALUES + source_filename: str = Field(min_length=1, max_length=512) + source_relative_path: str = Field(min_length=1, max_length=4096) + source_context: str = Field(min_length=1, max_length=1024) + ontology_class: str = Field(pattern=ONTOLOGY_CLASS_PATTERN) + ontology_relations: list[FileLineageRelation] = Field(max_length=256) + raw_content_sha256: str = Field(pattern=HEX64_PATTERN) + raw_content_blake3: str = Field(pattern=HEX64_PATTERN) + bytes: int = Field(ge=0) + production_time: ProductionTimeLineage + filesystem_time: FilesystemTimeLineage + metadata_evidence: list[FileMetadataEvidence] = Field(max_length=128) + content_title: str | None = Field(default=None, max_length=1024) + content_authors: list[str] = Field(max_length=32) + content_context: list[str] = Field(max_length=64) + duration_ms: int | None = Field(default=None, ge=0) + review: ReviewLineage + cloud_copy: CloudCopyLineage + + @model_validator(mode="after") + def validate_path_and_relations(self) -> FileLineageEnvelope: + relative = self.source_relative_path.replace("\\", "/") + parts = relative.split("/") + if relative.startswith("/") or any(part in {"", ".", ".."} for part in parts): + raise ValueError("source relative path is not a normalized relative path") + if self.source_filename != parts[-1]: + raise ValueError("source filename does not match source relative path") + if any(ord(character) < 32 for character in relative): + raise ValueError("source relative path contains a control character") + return self + + +class FileLineageSummary(_StrictModel): + lineage_record_uid: str + lineage_fingerprint: str + schema_version: int + source_kind: str + archive_kind: str + raw_content_sha256: str + raw_content_blake3: str + bytes: int + ontology_class: str + ontology_relation_count: int + ontology_predicates: list[str] + provider: PROVIDER_VALUES + provider_sync_confirmed: bool + created_at: str + + +def canonical_envelope_sha256(envelope: FileLineageEnvelope) -> str: + """Hash the exact validated JSON payload for idempotent, tamper-evident ingest.""" + + encoded = json.dumps( + envelope.model_dump(mode="json"), + ensure_ascii=False, + separators=(",", ":"), + sort_keys=True, + ).encode("utf-8") + return hashlib.sha256(encoded).hexdigest() + + +def ontology_predicates(envelope: FileLineageEnvelope) -> list[str]: + """Return a deterministic public projection without path/object values.""" + + return sorted({relation.predicate for relation in envelope.ontology_relations}) diff --git a/backend/tests/test_disksage_file_lineage.py b/backend/tests/test_disksage_file_lineage.py new file mode 100644 index 000000000..c7e4e3504 --- /dev/null +++ b/backend/tests/test_disksage_file_lineage.py @@ -0,0 +1,138 @@ +import pytest +from pydantic import ValidationError + +from services.disksage_file_lineage import ( + FileLineageEnvelope, + canonical_envelope_sha256, + ontology_predicates, +) + + +def _envelope(**overrides: object) -> dict[str, object]: + payload: dict[str, object] = { + "schema_version": 1, + "schema_kind": "disksage.file-lineage", + "source_kind": "file", + "archive_kind": "media", + "source_filename": "Video 1.mov", + "source_relative_path": "DaVinci Resolve/Video 1.mov", + "source_context": "DaVinci Resolve", + "ontology_class": "https://disksage.app/ontology#Media", + "ontology_relations": [ + { + "subject": "/Users/example/Movies/Video 1.mov", + "predicate": "https://disksage.app/ontology#archivedTo", + "object": "/Users/example/iCloud/DiskSage Archive/Video 1.mov", + "source": "archive-destination-planner", + }, + { + "subject": "source", + "predicate": "https://disksage.app/ontology#archivedTo", + "object": "destination", + "source": "test", + }, + ], + "raw_content_sha256": "a" * 64, + "raw_content_blake3": "b" * 64, + "bytes": 160085038, + "production_time": { + "selected_value_ms": 1, + "selected_source": "embedded:exiftool:MediaCreateDate", + "confidence": "high", + "evidence_precedence": [ + "embedded_metadata", + "explicit_filename_date", + "filesystem_created_at", + "filesystem_modified_at", + ], + }, + "filesystem_time": {"created_at_ms": 2, "modified_at_ms": 3}, + "metadata_evidence": [ + { + "field": "production_time", + "value": "2026-04-28T00:00:00Z", + "source": "exiftool:MediaCreateDate", + "confidence": "high", + } + ], + "content_title": "Video 1", + "content_authors": [], + "content_context": ["DaVinci Resolve"], + "duration_ms": 60000, + "review": { + "candidate_fingerprint": "c" * 64, + "review_fingerprint": "d" * 64, + "requires_review": True, + "reason_codes": ["destination-account-scope-unknown"], + "decision_id": "decision-1", + "disposition": "approved", + "reviewed_at_ms": 4, + "reviewed_by": "human:local:test", + "rationale": "Account scope reviewed", + }, + "cloud_copy": { + "receipt_id": "e" * 64, + "lineage_fingerprint": "f" * 64, + "provider": "icloud", + "destination_account_scope": "unknown", + "destination": "/Users/example/iCloud/DiskSage Archive/Video 1.mov", + "copied_at_ms": 5, + "copy_verification_method": "copied-by-disk-sage", + "local_copy_verified": True, + "provider_write_executed": False, + "provider_sync_confirmed": False, + "sync_evidence_record_id": None, + "sync_evidence_kind": None, + "sync_evidence_id": None, + "sync_confirmed_at_ms": None, + "remote_object_id": None, + "remote_revision": None, + "remote_location_bound": None, + }, + } + payload.update(overrides) + return payload + + +def test_valid_envelope_keeps_graph_projection_deterministic(): + envelope = FileLineageEnvelope.model_validate(_envelope()) + + assert envelope.schema_kind == "disksage.file-lineage" + assert len(envelope.ontology_relations) == 2 + assert ontology_predicates(envelope) == ["https://disksage.app/ontology#archivedTo"] + assert len(canonical_envelope_sha256(envelope)) == 64 + + +@pytest.mark.parametrize( + ("field", "value"), + [ + ("source_relative_path", "/absolute/path.mov"), + ("source_relative_path", "../escape.mov"), + ("source_filename", "different.mov"), + ], +) +def test_path_binding_rejects_unsafe_or_mismatched_values(field: str, value: str): + with pytest.raises(ValidationError): + FileLineageEnvelope.model_validate(_envelope(**{field: value})) + + +def test_provider_write_claim_is_rejected_even_when_copy_is_verified(): + payload = _envelope() + payload["cloud_copy"] = { + **payload["cloud_copy"], # type: ignore[arg-type] + "provider_write_executed": True, + } + with pytest.raises(ValidationError): + FileLineageEnvelope.model_validate(payload) + + +def test_unknown_fields_are_rejected_at_the_handoff_boundary(): + payload = _envelope(untrusted_private_value="must-not-persist") + with pytest.raises(ValidationError): + FileLineageEnvelope.model_validate(payload) + + +def test_naruon_exposes_the_scoped_lineage_resource(): + from main import app + + assert "/api/disksage/file-lineage" in app.openapi()["paths"] diff --git a/docs/architecture/disksage-file-lineage-handoff.md b/docs/architecture/disksage-file-lineage-handoff.md new file mode 100644 index 000000000..2693f4776 --- /dev/null +++ b/docs/architecture/disksage-file-lineage-handoff.md @@ -0,0 +1,41 @@ +# DiskSage file-lineage handoff + +**Status**: Accepted +**Date**: 2026-08-13 + +## Context + +DiskSage can prove a local copy, metadata precedence, review decision, and +provider synchronization evidence. A File Provider placeholder is not itself +proof that a provider API write executed. Naruon already persists content and +knowledge graphs, but the current cloud-copy handoff is only a readiness +summary and cannot represent a general file's provenance or ontology edges. + +## Decision + +Naruon accepts `disksage.file-lineage` version 1 through +`POST /api/disksage/file-lineage`. The request boundary is strict and rejects +unknown fields, unsafe relative paths, unverified copies, and +`provider_write_executed=true` claims. The complete envelope is encrypted at +rest and scoped by authenticated user/workspace. `GET /api/disksage/file-lineage` +returns only hashes, byte count, ontology class, predicate projection, provider, +and sync status; it does not expose local paths or raw metadata values. + +The payload keeps explicit file → archive destination → provider/account and +review relations. These relation edges follow the same entity/provenance +separation as PROV-O; DiskSage's local ontology remains the domain vocabulary. +Semantic-data-portal and pg-erd-cloud are deferred until a shared catalog/ERD +boundary is actually needed; this table is the durable Naruon boundary first. + +## Consequences + +- Naruon can index provenance and ontology predicates without authorizing source + deletion or inventing provider writes. +- The encrypted envelope is not directly queryable for graph search; a later + scoped projection can be added when a catalog consumer exists. +- DiskSage remains the authority for copy, hash, review, and provider evidence. + +## References + +- [W3C PROV-O](https://www.w3.org/TR/prov-o/) +- [W3C OWL 2 overview](https://www.w3.org/TR/owl2-overview/) From d79dbf7e78ce296a1c7414526b0ccc8600a9df7f Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 13 Aug 2026 18:30:18 +0900 Subject: [PATCH 02/29] feat(naruon): preserve provider sync state in file lineage --- .../versions/0018_disksage_file_lineage.py | 6 +++ .../0019_disksage_provider_sync_state.py | 43 +++++++++++++++++++ backend/api/disksage.py | 2 + backend/db/models.py | 3 ++ backend/services/disksage_file_lineage.py | 20 +++++++++ backend/tests/test_disksage_file_lineage.py | 13 ++++++ .../disksage-file-lineage-handoff.md | 4 +- 7 files changed, 90 insertions(+), 1 deletion(-) create mode 100644 backend/alembic/versions/0019_disksage_provider_sync_state.py diff --git a/backend/alembic/versions/0018_disksage_file_lineage.py b/backend/alembic/versions/0018_disksage_file_lineage.py index 4f71c552a..e06f94b2a 100644 --- a/backend/alembic/versions/0018_disksage_file_lineage.py +++ b/backend/alembic/versions/0018_disksage_file_lineage.py @@ -45,6 +45,12 @@ def upgrade() -> None: sa.Column("ontology_predicates", sa.JSON(), nullable=False), sa.Column("provider", sa.String(length=32), nullable=False), sa.Column("provider_sync_confirmed", sa.Boolean(), nullable=False), + sa.Column( + "provider_sync_state", + sa.String(length=32), + nullable=False, + server_default="unknown", + ), sa.Column("envelope_json_encrypted", sa.String(), nullable=False), sa.Column("created_at", sa.DateTime(timezone=True), nullable=False), sa.PrimaryKeyConstraint("lineage_record_uid"), diff --git a/backend/alembic/versions/0019_disksage_provider_sync_state.py b/backend/alembic/versions/0019_disksage_provider_sync_state.py new file mode 100644 index 000000000..f3c7fdc33 --- /dev/null +++ b/backend/alembic/versions/0019_disksage_provider_sync_state.py @@ -0,0 +1,43 @@ +"""persist provider-native sync state for DiskSage lineage envelopes + +Revision ID: 0019_disksage_provider_sync_state +Revises: 0018_disksage_file_lineage +Create Date: 2026-08-13 00:00:00.000000 +""" + +from alembic import op +import sqlalchemy as sa + + +revision = "0019_disksage_provider_sync_state" +down_revision = "0018_disksage_file_lineage" +branch_labels = None +depends_on = None + +_TABLE = "disksage_file_lineage_records" + + +def upgrade() -> None: + inspector = sa.inspect(op.get_bind()) + if inspector.has_table(_TABLE) and not any( + column["name"] == "provider_sync_state" + for column in inspector.get_columns(_TABLE) + ): + op.add_column( + _TABLE, + sa.Column( + "provider_sync_state", + sa.String(length=32), + nullable=False, + server_default="unknown", + ), + ) + + +def downgrade() -> None: + inspector = sa.inspect(op.get_bind()) + if inspector.has_table(_TABLE) and any( + column["name"] == "provider_sync_state" + for column in inspector.get_columns(_TABLE) + ): + op.drop_column(_TABLE, "provider_sync_state") diff --git a/backend/api/disksage.py b/backend/api/disksage.py index 8079c269e..2e28bb616 100644 --- a/backend/api/disksage.py +++ b/backend/api/disksage.py @@ -34,6 +34,7 @@ def _summary(record: DiskSageFileLineageRecord) -> FileLineageSummary: ontology_predicates=list(record.ontology_predicates or []), provider=record.provider, provider_sync_confirmed=record.provider_sync_confirmed, + provider_sync_state=record.provider_sync_state, created_at=record.created_at.isoformat(), ) @@ -92,6 +93,7 @@ async def ingest_file_lineage( ontology_predicates=ontology_predicates(envelope), provider=envelope.cloud_copy.provider, provider_sync_confirmed=envelope.cloud_copy.provider_sync_confirmed, + provider_sync_state=envelope.cloud_copy.provider_sync_state or "unknown", envelope_json_encrypted=_encrypted_envelope_json(envelope), created_at=datetime.datetime.now(datetime.timezone.utc), ) diff --git a/backend/db/models.py b/backend/db/models.py index 5193533d1..513e15b4a 100644 --- a/backend/db/models.py +++ b/backend/db/models.py @@ -1146,6 +1146,9 @@ class DiskSageFileLineageRecord(Base): ) provider: Mapped[str] = mapped_column(String(32), nullable=False) provider_sync_confirmed: Mapped[bool] = mapped_column(Boolean, nullable=False) + provider_sync_state: Mapped[str] = mapped_column( + String(32), nullable=False, default="unknown", server_default="unknown" + ) # Source paths and metadata evidence are intentionally not queryable plaintext. envelope_json_encrypted: Mapped[str] = mapped_column( EncryptedString, nullable=False diff --git a/backend/services/disksage_file_lineage.py b/backend/services/disksage_file_lineage.py index e9f1bec64..7d1c1decb 100644 --- a/backend/services/disksage_file_lineage.py +++ b/backend/services/disksage_file_lineage.py @@ -30,6 +30,18 @@ ] REVIEW_DISPOSITION_VALUES = Literal["approved", "held"] SYNC_KIND_VALUES = Literal["provider-api", "provider-native-status"] +PROVIDER_SYNC_STATE_VALUES = Literal[ + "complete", + "pending-upload", + "not-ubiquitous", + "not-local-current", + "uploading", + "excluded-from-sync", + "sync-paused", + "remote-unavailable", + "content-mismatch", + "unknown", +] class _StrictModel(BaseModel): @@ -114,6 +126,9 @@ class CloudCopyLineage(_StrictModel): local_copy_verified: bool provider_write_executed: bool provider_sync_confirmed: bool + # Optional keeps version-1 envelopes backwards compatible; new DiskSage + # exports preserve provider-native states such as pending-upload. + provider_sync_state: PROVIDER_SYNC_STATE_VALUES | None = None sync_evidence_record_id: str | None = Field(default=None, max_length=256) sync_evidence_kind: SYNC_KIND_VALUES | None = None sync_evidence_id: str | None = Field(default=None, max_length=512) @@ -138,6 +153,10 @@ def bind_provider_evidence(self) -> CloudCopyLineage: value is None for value in evidence_fields ): raise ValueError("confirmed sync is missing provider evidence") + if self.provider_sync_state not in (None, "unknown") and ( + self.provider_sync_confirmed != (self.provider_sync_state == "complete") + ): + raise ValueError("provider sync state does not match confirmation") if self.remote_location_bound is True and not self.remote_object_id: raise ValueError("remote location binding is missing its object id") return self @@ -193,6 +212,7 @@ class FileLineageSummary(_StrictModel): ontology_predicates: list[str] provider: PROVIDER_VALUES provider_sync_confirmed: bool + provider_sync_state: PROVIDER_SYNC_STATE_VALUES created_at: str diff --git a/backend/tests/test_disksage_file_lineage.py b/backend/tests/test_disksage_file_lineage.py index c7e4e3504..12ac2f0b4 100644 --- a/backend/tests/test_disksage_file_lineage.py +++ b/backend/tests/test_disksage_file_lineage.py @@ -126,6 +126,19 @@ def test_provider_write_claim_is_rejected_even_when_copy_is_verified(): FileLineageEnvelope.model_validate(payload) +def test_provider_sync_state_preserves_pending_upload_without_eviction_claim(): + payload = _envelope() + payload["cloud_copy"] = { + **payload["cloud_copy"], # type: ignore[arg-type] + "provider_sync_state": "pending-upload", + } + + envelope = FileLineageEnvelope.model_validate(payload) + + assert envelope.cloud_copy.provider_sync_state == "pending-upload" + assert envelope.cloud_copy.provider_sync_confirmed is False + + def test_unknown_fields_are_rejected_at_the_handoff_boundary(): payload = _envelope(untrusted_private_value="must-not-persist") with pytest.raises(ValidationError): diff --git a/docs/architecture/disksage-file-lineage-handoff.md b/docs/architecture/disksage-file-lineage-handoff.md index 2693f4776..7b3ded834 100644 --- a/docs/architecture/disksage-file-lineage-handoff.md +++ b/docs/architecture/disksage-file-lineage-handoff.md @@ -19,7 +19,9 @@ unknown fields, unsafe relative paths, unverified copies, and `provider_write_executed=true` claims. The complete envelope is encrypted at rest and scoped by authenticated user/workspace. `GET /api/disksage/file-lineage` returns only hashes, byte count, ontology class, predicate projection, provider, -and sync status; it does not expose local paths or raw metadata values. +the provider-native sync state, and sync status; it does not expose local paths +or raw metadata values. A state such as `pending-upload` is retained as an +incomplete provider proof and never authorizes source eviction. The payload keeps explicit file → archive destination → provider/account and review relations. These relation edges follow the same entity/provenance From 5baa5d8ea6f6c87ca52dcfbd9a3150a15882dc05 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 13 Aug 2026 20:01:04 +0900 Subject: [PATCH 03/29] fix: accept DiskSage lineage schema v2 --- backend/services/disksage_file_lineage.py | 4 +++- backend/tests/test_disksage_file_lineage.py | 9 +++++++++ docs/architecture/disksage-file-lineage-handoff.md | 2 +- 3 files changed, 13 insertions(+), 2 deletions(-) diff --git a/backend/services/disksage_file_lineage.py b/backend/services/disksage_file_lineage.py index 7d1c1decb..7cd5af23f 100644 --- a/backend/services/disksage_file_lineage.py +++ b/backend/services/disksage_file_lineage.py @@ -163,7 +163,9 @@ def bind_provider_evidence(self) -> CloudCopyLineage: class FileLineageEnvelope(_StrictModel): - schema_version: Literal[1] + # DiskSage v2 adds attributed copy-approval evidence while retaining the + # v1 ontology envelope shape. + schema_version: Literal[1, 2] schema_kind: Literal["disksage.file-lineage"] source_kind: Literal["file"] archive_kind: ARCHIVE_KIND_VALUES diff --git a/backend/tests/test_disksage_file_lineage.py b/backend/tests/test_disksage_file_lineage.py index 12ac2f0b4..9046a800a 100644 --- a/backend/tests/test_disksage_file_lineage.py +++ b/backend/tests/test_disksage_file_lineage.py @@ -103,6 +103,15 @@ def test_valid_envelope_keeps_graph_projection_deterministic(): assert len(canonical_envelope_sha256(envelope)) == 64 +@pytest.mark.parametrize("schema_version", [1, 2]) +def test_accepts_disk_sage_v1_and_v2_envelopes(schema_version: int): + envelope = FileLineageEnvelope.model_validate( + _envelope(schema_version=schema_version) + ) + + assert envelope.schema_version == schema_version + + @pytest.mark.parametrize( ("field", "value"), [ diff --git a/docs/architecture/disksage-file-lineage-handoff.md b/docs/architecture/disksage-file-lineage-handoff.md index 7b3ded834..97b7be0cd 100644 --- a/docs/architecture/disksage-file-lineage-handoff.md +++ b/docs/architecture/disksage-file-lineage-handoff.md @@ -13,7 +13,7 @@ summary and cannot represent a general file's provenance or ontology edges. ## Decision -Naruon accepts `disksage.file-lineage` version 1 through +Naruon accepts `disksage.file-lineage` versions 1 and 2 through `POST /api/disksage/file-lineage`. The request boundary is strict and rejects unknown fields, unsafe relative paths, unverified copies, and `provider_write_executed=true` claims. The complete envelope is encrypted at From 39dff050e6f211096f9aa638d5f8a5a5147dc6c0 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 13 Aug 2026 20:04:02 +0900 Subject: [PATCH 04/29] fix: preserve DiskSage v2 copy approval lineage --- backend/services/disksage_file_lineage.py | 8 ++++++++ backend/tests/test_disksage_file_lineage.py | 19 +++++++++++++++++++ 2 files changed, 27 insertions(+) diff --git a/backend/services/disksage_file_lineage.py b/backend/services/disksage_file_lineage.py index 7cd5af23f..094f40b55 100644 --- a/backend/services/disksage_file_lineage.py +++ b/backend/services/disksage_file_lineage.py @@ -30,6 +30,7 @@ ] REVIEW_DISPOSITION_VALUES = Literal["approved", "held"] SYNC_KIND_VALUES = Literal["provider-api", "provider-native-status"] +COPY_APPROVAL_ACTION_VALUES = Literal["copy-only", "adopt-existing-copy"] PROVIDER_SYNC_STATE_VALUES = Literal[ "complete", "pending-upload", @@ -123,6 +124,13 @@ class CloudCopyLineage(_StrictModel): destination: str = Field(min_length=1, max_length=4096) copied_at_ms: int = Field(ge=0) copy_verification_method: Literal["copied-by-disk-sage", "adopted-existing"] + # DiskSage v2 binds the copy to one attributed human action. These remain + # optional so v1 and pre-approval receipts can still be ingested. + copy_approval_id: str | None = Field(default=None, pattern=HEX64_PATTERN) + copy_approval_action: COPY_APPROVAL_ACTION_VALUES | None = None + copy_approved_at_ms: int | None = Field(default=None, ge=0) + copy_approved_by: str | None = Field(default=None, max_length=256) + copy_approval_rationale: str | None = Field(default=None, max_length=2000) local_copy_verified: bool provider_write_executed: bool provider_sync_confirmed: bool diff --git a/backend/tests/test_disksage_file_lineage.py b/backend/tests/test_disksage_file_lineage.py index 9046a800a..c054f4fd9 100644 --- a/backend/tests/test_disksage_file_lineage.py +++ b/backend/tests/test_disksage_file_lineage.py @@ -112,6 +112,25 @@ def test_accepts_disk_sage_v1_and_v2_envelopes(schema_version: int): assert envelope.schema_version == schema_version +def test_v2_preserves_attributed_copy_approval_fields(): + payload = _envelope( + schema_version=2, + cloud_copy={ + **_envelope()["cloud_copy"], # type: ignore[arg-type] + "copy_approval_id": "1" * 64, + "copy_approval_action": "copy-only", + "copy_approved_at_ms": 6, + "copy_approved_by": "human:local:test", + "copy_approval_rationale": "approved after metadata review", + }, + ) + + envelope = FileLineageEnvelope.model_validate(payload) + + assert envelope.cloud_copy.copy_approval_action == "copy-only" + assert envelope.cloud_copy.copy_approval_id == "1" * 64 + + @pytest.mark.parametrize( ("field", "value"), [ From f5aa966876842d5ded7bef7af8d1715ebe2ecaca Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 13 Aug 2026 20:17:25 +0900 Subject: [PATCH 05/29] docs: add DiskSage lineage DBML projection --- .../disksage-file-lineage-handoff.md | 7 +++-- .../erd/disksage-file-lineage.dbml | 31 +++++++++++++++++++ 2 files changed, 36 insertions(+), 2 deletions(-) create mode 100644 docs/architecture/erd/disksage-file-lineage.dbml diff --git a/docs/architecture/disksage-file-lineage-handoff.md b/docs/architecture/disksage-file-lineage-handoff.md index 97b7be0cd..536df6a32 100644 --- a/docs/architecture/disksage-file-lineage-handoff.md +++ b/docs/architecture/disksage-file-lineage-handoff.md @@ -26,8 +26,11 @@ incomplete provider proof and never authorizes source eviction. The payload keeps explicit file → archive destination → provider/account and review relations. These relation edges follow the same entity/provenance separation as PROV-O; DiskSage's local ontology remains the domain vocabulary. -Semantic-data-portal and pg-erd-cloud are deferred until a shared catalog/ERD -boundary is actually needed; this table is the durable Naruon boundary first. +The path-free semantic catalog candidate batch is the handoff to +semantic-data-portal. The durable table has an explicit DBML projection at +`docs/architecture/erd/disksage-file-lineage.dbml`; pg-erd-cloud can convert it +to the same snapshot shape as a live database without granting it mutation +authority. The Alembic model remains the runtime source of truth. ## Consequences diff --git a/docs/architecture/erd/disksage-file-lineage.dbml b/docs/architecture/erd/disksage-file-lineage.dbml new file mode 100644 index 000000000..8738312e1 --- /dev/null +++ b/docs/architecture/erd/disksage-file-lineage.dbml @@ -0,0 +1,31 @@ +// Design projection for pg-erd-cloud. Runtime truth remains the Alembic model. +// Source paths and metadata values stay inside envelope_json_encrypted. +Table public.disksage_file_lineage_records { + lineage_record_uid varchar(96) [pk] + user_id varchar [not null] + organization_id varchar + workspace_id varchar [not null] + lineage_fingerprint varchar(64) [not null] + envelope_sha256 varchar(64) [not null] + schema_version integer [not null] + schema_kind varchar(96) [not null] + source_kind varchar(64) [not null] + archive_kind varchar(64) [not null] + raw_content_sha256 varchar(64) [not null] + raw_content_blake3 varchar(64) [not null] + bytes bigint [not null] + ontology_class varchar(256) [not null] + ontology_relation_count integer [not null] + ontology_predicates json [not null] + provider varchar(32) [not null] + provider_sync_confirmed boolean [not null] + provider_sync_state varchar(32) [not null, default: 'unknown'] + envelope_json_encrypted text [not null] + created_at timestamptz [not null] + + indexes { + (user_id, workspace_id, lineage_fingerprint) [unique] + (user_id, organization_id, workspace_id, created_at) + (workspace_id, ontology_class) + } +} From d287dada9fa16130a07496de9699d8e70f32ee04 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 13 Aug 2026 20:31:20 +0900 Subject: [PATCH 06/29] fix: harden DiskSage lineage persistence --- .../versions/0018_disksage_file_lineage.py | 4 +- .../0020_disksage_lineage_column_names.py | 39 ++++++++++++++++++ backend/api/disksage.py | 40 +++++++++++++------ backend/core/runtime_secrets.py | 8 +++- backend/db/models.py | 12 ++++-- backend/services/disksage_file_lineage.py | 19 +++++---- backend/tests/test_disksage_file_lineage.py | 22 ++++++++++ .../disksage-file-lineage-handoff.md | 10 +++-- .../erd/disksage-file-lineage.dbml | 4 +- 9 files changed, 125 insertions(+), 33 deletions(-) create mode 100644 backend/alembic/versions/0020_disksage_lineage_column_names.py diff --git a/backend/alembic/versions/0018_disksage_file_lineage.py b/backend/alembic/versions/0018_disksage_file_lineage.py index e06f94b2a..ae44860cf 100644 --- a/backend/alembic/versions/0018_disksage_file_lineage.py +++ b/backend/alembic/versions/0018_disksage_file_lineage.py @@ -39,11 +39,11 @@ def upgrade() -> None: sa.Column("archive_kind", sa.String(length=64), nullable=False), sa.Column("raw_content_sha256", sa.String(length=64), nullable=False), sa.Column("raw_content_blake3", sa.String(length=64), nullable=False), - sa.Column("bytes", sa.BigInteger(), nullable=False), + sa.Column("content_bytes", sa.BigInteger(), nullable=False), sa.Column("ontology_class", sa.String(length=256), nullable=False), sa.Column("ontology_relation_count", sa.Integer(), nullable=False), sa.Column("ontology_predicates", sa.JSON(), nullable=False), - sa.Column("provider", sa.String(length=32), nullable=False), + sa.Column("provider_name", sa.String(length=32), nullable=False), sa.Column("provider_sync_confirmed", sa.Boolean(), nullable=False), sa.Column( "provider_sync_state", diff --git a/backend/alembic/versions/0020_disksage_lineage_column_names.py b/backend/alembic/versions/0020_disksage_lineage_column_names.py new file mode 100644 index 000000000..2fb45625b --- /dev/null +++ b/backend/alembic/versions/0020_disksage_lineage_column_names.py @@ -0,0 +1,39 @@ +"""rename ambiguous DiskSage lineage column names + +Revision ID: 0020_disksage_lineage_column_names +Revises: 0019_disksage_provider_sync_state +Create Date: 2026-08-13 00:00:00.000000 +""" + +from alembic import op +import sqlalchemy as sa + + +revision = "0020_disksage_lineage_column_names" +down_revision = "0019_disksage_provider_sync_state" +branch_labels = None +depends_on = None + +_TABLE = "disksage_file_lineage_records" + + +def _rename_if_present(inspector: sa.Inspector, old: str, new: str) -> None: + columns = {column["name"] for column in inspector.get_columns(_TABLE)} + if old in columns and new not in columns: + op.alter_column(_TABLE, old, new_column_name=new) + + +def upgrade() -> None: + inspector = sa.inspect(op.get_bind()) + if inspector.has_table(_TABLE): + _rename_if_present(inspector, "bytes", "content_bytes") + inspector = sa.inspect(op.get_bind()) + _rename_if_present(inspector, "provider", "provider_name") + + +def downgrade() -> None: + inspector = sa.inspect(op.get_bind()) + if inspector.has_table(_TABLE): + _rename_if_present(inspector, "content_bytes", "bytes") + inspector = sa.inspect(op.get_bind()) + _rename_if_present(inspector, "provider_name", "provider") diff --git a/backend/api/disksage.py b/backend/api/disksage.py index 2e28bb616..59166f9ad 100644 --- a/backend/api/disksage.py +++ b/backend/api/disksage.py @@ -1,10 +1,10 @@ import datetime -import json import uuid from fastapi import APIRouter, Depends, HTTPException, Query from sqlalchemy import select from sqlalchemy.ext.asyncio import AsyncSession +from sqlalchemy.exc import IntegrityError from api.auth import AuthContext, get_auth_context from db.models import DiskSageFileLineageRecord @@ -12,9 +12,11 @@ from services.disksage_file_lineage import ( FileLineageEnvelope, FileLineageSummary, + canonical_envelope_json, canonical_envelope_sha256, ontology_predicates, ) +from core.runtime_secrets import EncryptionKeyMissingError router = APIRouter(prefix="/api/disksage", tags=["disksage"]) @@ -28,11 +30,11 @@ def _summary(record: DiskSageFileLineageRecord) -> FileLineageSummary: archive_kind=record.archive_kind, raw_content_sha256=record.raw_content_sha256, raw_content_blake3=record.raw_content_blake3, - bytes=record.bytes, + content_bytes=record.content_bytes, ontology_class=record.ontology_class, ontology_relation_count=record.ontology_relation_count, ontology_predicates=list(record.ontology_predicates or []), - provider=record.provider, + provider_name=record.provider_name, provider_sync_confirmed=record.provider_sync_confirmed, provider_sync_state=record.provider_sync_state, created_at=record.created_at.isoformat(), @@ -40,12 +42,7 @@ def _summary(record: DiskSageFileLineageRecord) -> FileLineageSummary: def _encrypted_envelope_json(envelope: FileLineageEnvelope) -> str: - return json.dumps( - envelope.model_dump(mode="json"), - ensure_ascii=False, - separators=(",", ":"), - sort_keys=True, - ) + return canonical_envelope_json(envelope) @router.post("/file-lineage", response_model=FileLineageSummary, status_code=201) @@ -87,11 +84,11 @@ async def ingest_file_lineage( archive_kind=envelope.archive_kind, raw_content_sha256=envelope.raw_content_sha256, raw_content_blake3=envelope.raw_content_blake3, - bytes=envelope.bytes, + content_bytes=envelope.bytes, ontology_class=envelope.ontology_class, ontology_relation_count=len(envelope.ontology_relations), ontology_predicates=ontology_predicates(envelope), - provider=envelope.cloud_copy.provider, + provider_name=envelope.cloud_copy.provider, provider_sync_confirmed=envelope.cloud_copy.provider_sync_confirmed, provider_sync_state=envelope.cloud_copy.provider_sync_state or "unknown", envelope_json_encrypted=_encrypted_envelope_json(envelope), @@ -101,9 +98,26 @@ async def ingest_file_lineage( try: await db.commit() await db.refresh(record) - except RuntimeError as error: - if "ENCRYPTION_KEY is required" not in str(error): + except IntegrityError: + await db.rollback() + replayed_result = await db.execute( + select(DiskSageFileLineageRecord).where( + DiskSageFileLineageRecord.user_id == auth_context.user_id, + DiskSageFileLineageRecord.workspace_id == auth_context.workspace_id, + DiskSageFileLineageRecord.lineage_fingerprint == lineage_fingerprint, + ) + ) + replayed = replayed_result.scalar_one_or_none() + if replayed is None: raise + if replayed.envelope_sha256 != envelope_sha256: + raise HTTPException( + status_code=409, + detail="lineage fingerprint is already bound to different evidence", + ) from None + return _summary(replayed) + except EncryptionKeyMissingError as error: + await db.rollback() raise HTTPException( status_code=503, detail="Server encryption key is not configured. Contact your workspace administrator.", diff --git a/backend/core/runtime_secrets.py b/backend/core/runtime_secrets.py index 1bf1496d1..d6c820117 100644 --- a/backend/core/runtime_secrets.py +++ b/backend/core/runtime_secrets.py @@ -18,6 +18,12 @@ ENCRYPTION_KEY_ID_PATTERN = re.compile(r"^[A-Za-z0-9][A-Za-z0-9_.-]{0,63}$") +class EncryptionKeyMissingError(RuntimeError): + """Raised when encryption is requested without the configured active key.""" + + error_code = "encryption-key-missing" + + @dataclass(frozen=True) class RuntimeEncryptionKey: key_id: str @@ -147,7 +153,7 @@ def build_encryption_keyring( previous_keys_value: str | None = None, ) -> EncryptionKeyRing: if active_key_value is None or not active_key_value.strip(): - raise RuntimeError( + raise EncryptionKeyMissingError( "ENCRYPTION_KEY is required. Refusing to encrypt without a configured key." ) diff --git a/backend/db/models.py b/backend/db/models.py index 513e15b4a..26bbc74fe 100644 --- a/backend/db/models.py +++ b/backend/db/models.py @@ -24,7 +24,11 @@ from sqlalchemy.types import TypeDecorator from core.config import settings -from core.runtime_secrets import EncryptionKeyRing, build_encryption_keyring +from core.runtime_secrets import ( + EncryptionKeyMissingError, + EncryptionKeyRing, + build_encryption_keyring, +) logger = logging.getLogger(__name__) @@ -34,7 +38,7 @@ def _validated_fernet_key() -> bytes: if settings.ENCRYPTION_KEY is None: - raise RuntimeError( + raise EncryptionKeyMissingError( "ENCRYPTION_KEY is required. Refusing to encrypt without a configured key." ) @@ -1138,13 +1142,13 @@ class DiskSageFileLineageRecord(Base): archive_kind: Mapped[str] = mapped_column(String(64), nullable=False) raw_content_sha256: Mapped[str] = mapped_column(String(64), nullable=False) raw_content_blake3: Mapped[str] = mapped_column(String(64), nullable=False) - bytes: Mapped[int] = mapped_column(BigInteger, nullable=False) + content_bytes: Mapped[int] = mapped_column(BigInteger, nullable=False) ontology_class: Mapped[str] = mapped_column(String(256), nullable=False) ontology_relation_count: Mapped[int] = mapped_column(Integer, nullable=False) ontology_predicates: Mapped[list[str]] = mapped_column( JSON, default=list, nullable=False ) - provider: Mapped[str] = mapped_column(String(32), nullable=False) + provider_name: Mapped[str] = mapped_column(String(32), nullable=False) provider_sync_confirmed: Mapped[bool] = mapped_column(Boolean, nullable=False) provider_sync_state: Mapped[str] = mapped_column( String(32), nullable=False, default="unknown", server_default="unknown" diff --git a/backend/services/disksage_file_lineage.py b/backend/services/disksage_file_lineage.py index 094f40b55..2e13f87e6 100644 --- a/backend/services/disksage_file_lineage.py +++ b/backend/services/disksage_file_lineage.py @@ -216,26 +216,31 @@ class FileLineageSummary(_StrictModel): archive_kind: str raw_content_sha256: str raw_content_blake3: str - bytes: int + content_bytes: int ontology_class: str ontology_relation_count: int ontology_predicates: list[str] - provider: PROVIDER_VALUES + provider_name: PROVIDER_VALUES provider_sync_confirmed: bool provider_sync_state: PROVIDER_SYNC_STATE_VALUES created_at: str -def canonical_envelope_sha256(envelope: FileLineageEnvelope) -> str: - """Hash the exact validated JSON payload for idempotent, tamper-evident ingest.""" +def canonical_envelope_json(envelope: FileLineageEnvelope) -> str: + """Serialize the validated envelope deterministically for hashing and storage.""" - encoded = json.dumps( + return json.dumps( envelope.model_dump(mode="json"), ensure_ascii=False, separators=(",", ":"), sort_keys=True, - ).encode("utf-8") - return hashlib.sha256(encoded).hexdigest() + ) + + +def canonical_envelope_sha256(envelope: FileLineageEnvelope) -> str: + """Hash the exact validated JSON payload for idempotent, tamper-evident ingest.""" + + return hashlib.sha256(canonical_envelope_json(envelope).encode("utf-8")).hexdigest() def ontology_predicates(envelope: FileLineageEnvelope) -> list[str]: diff --git a/backend/tests/test_disksage_file_lineage.py b/backend/tests/test_disksage_file_lineage.py index c054f4fd9..9b04dc38b 100644 --- a/backend/tests/test_disksage_file_lineage.py +++ b/backend/tests/test_disksage_file_lineage.py @@ -3,6 +3,7 @@ from services.disksage_file_lineage import ( FileLineageEnvelope, + canonical_envelope_json, canonical_envelope_sha256, ontology_predicates, ) @@ -101,6 +102,7 @@ def test_valid_envelope_keeps_graph_projection_deterministic(): assert len(envelope.ontology_relations) == 2 assert ontology_predicates(envelope) == ["https://disksage.app/ontology#archivedTo"] assert len(canonical_envelope_sha256(envelope)) == 64 + assert canonical_envelope_json(envelope).encode("utf-8") @pytest.mark.parametrize("schema_version", [1, 2]) @@ -167,6 +169,26 @@ def test_provider_sync_state_preserves_pending_upload_without_eviction_claim(): assert envelope.cloud_copy.provider_sync_confirmed is False +def test_provider_sync_confirmation_requires_evidence(): + payload = _envelope() + payload["cloud_copy"] = { + **payload["cloud_copy"], # type: ignore[arg-type] + "provider_sync_confirmed": True, + } + with pytest.raises(ValidationError): + FileLineageEnvelope.model_validate(payload) + + +def test_complete_provider_sync_state_requires_confirmation(): + payload = _envelope() + payload["cloud_copy"] = { + **payload["cloud_copy"], # type: ignore[arg-type] + "provider_sync_state": "complete", + } + with pytest.raises(ValidationError): + FileLineageEnvelope.model_validate(payload) + + def test_unknown_fields_are_rejected_at_the_handoff_boundary(): payload = _envelope(untrusted_private_value="must-not-persist") with pytest.raises(ValidationError): diff --git a/docs/architecture/disksage-file-lineage-handoff.md b/docs/architecture/disksage-file-lineage-handoff.md index 536df6a32..05ea96690 100644 --- a/docs/architecture/disksage-file-lineage-handoff.md +++ b/docs/architecture/disksage-file-lineage-handoff.md @@ -18,10 +18,12 @@ Naruon accepts `disksage.file-lineage` versions 1 and 2 through unknown fields, unsafe relative paths, unverified copies, and `provider_write_executed=true` claims. The complete envelope is encrypted at rest and scoped by authenticated user/workspace. `GET /api/disksage/file-lineage` -returns only hashes, byte count, ontology class, predicate projection, provider, -the provider-native sync state, and sync status; it does not expose local paths -or raw metadata values. A state such as `pending-upload` is retained as an -incomplete provider proof and never authorizes source eviction. +returns only `lineage_record_uid`, `lineage_fingerprint`, `schema_version`, +`source_kind`, `archive_kind`, content hashes, `content_bytes`, ontology class, +predicate projection, `provider_name`, provider-native sync state, sync status, +and `created_at`; it does not expose local paths or raw metadata values. A state +such as `pending-upload` is retained as an incomplete provider proof and never +authorizes source eviction. The payload keeps explicit file → archive destination → provider/account and review relations. These relation edges follow the same entity/provenance diff --git a/docs/architecture/erd/disksage-file-lineage.dbml b/docs/architecture/erd/disksage-file-lineage.dbml index 8738312e1..81740e3af 100644 --- a/docs/architecture/erd/disksage-file-lineage.dbml +++ b/docs/architecture/erd/disksage-file-lineage.dbml @@ -13,11 +13,11 @@ Table public.disksage_file_lineage_records { archive_kind varchar(64) [not null] raw_content_sha256 varchar(64) [not null] raw_content_blake3 varchar(64) [not null] - bytes bigint [not null] + content_bytes bigint [not null] ontology_class varchar(256) [not null] ontology_relation_count integer [not null] ontology_predicates json [not null] - provider varchar(32) [not null] + provider_name varchar(32) [not null] provider_sync_confirmed boolean [not null] provider_sync_state varchar(32) [not null, default: 'unknown'] envelope_json_encrypted text [not null] From bd01171922476bd93d3ce60115210adbfc3058ca Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 13 Aug 2026 20:44:37 +0900 Subject: [PATCH 07/29] fix: close lineage schema and approval validation gaps --- .../0020_disksage_lineage_column_names.py | 10 ++-- .../0021_disksage_lineage_scope_index.py | 39 +++++++++++++++ backend/api/disksage.py | 48 +++++++++--------- backend/db/models.py | 1 - backend/services/disksage_file_lineage.py | 36 +++++++++++-- backend/tests/test_disksage_file_lineage.py | 50 +++++++++++++++++++ 6 files changed, 151 insertions(+), 33 deletions(-) create mode 100644 backend/alembic/versions/0021_disksage_lineage_scope_index.py diff --git a/backend/alembic/versions/0020_disksage_lineage_column_names.py b/backend/alembic/versions/0020_disksage_lineage_column_names.py index 2fb45625b..f82a36e6b 100644 --- a/backend/alembic/versions/0020_disksage_lineage_column_names.py +++ b/backend/alembic/versions/0020_disksage_lineage_column_names.py @@ -32,8 +32,8 @@ def upgrade() -> None: def downgrade() -> None: - inspector = sa.inspect(op.get_bind()) - if inspector.has_table(_TABLE): - _rename_if_present(inspector, "content_bytes", "bytes") - inspector = sa.inspect(op.get_bind()) - _rename_if_present(inspector, "provider_name", "provider") + # This revision repairs legacy installations that still have the old + # single-word names. Revision 0019 and the canonical 0018 schema both + # use content_bytes/provider_name, so a downgrade must not reintroduce + # the legacy shape or leave the model and migration chain inconsistent. + pass diff --git a/backend/alembic/versions/0021_disksage_lineage_scope_index.py b/backend/alembic/versions/0021_disksage_lineage_scope_index.py new file mode 100644 index 000000000..eb074d0ba --- /dev/null +++ b/backend/alembic/versions/0021_disksage_lineage_scope_index.py @@ -0,0 +1,39 @@ +"""align the DiskSage lineage scope index with the list query""" + +from alembic import op +import sqlalchemy as sa + + +revision = "0021_disksage_lineage_scope_index" +down_revision = "0020_disksage_lineage_column_names" +branch_labels = None +depends_on = None + +_TABLE = "disksage_file_lineage_records" +_INDEX = "ix_disksage_lineage_scope_time" + + +def upgrade() -> None: + inspector = sa.inspect(op.get_bind()) + if not inspector.has_table(_TABLE): + return + op.drop_index(_INDEX, table_name=_TABLE, if_exists=True) + op.create_index( + _INDEX, + _TABLE, + ["user_id", "workspace_id", "created_at"], + if_not_exists=True, + ) + + +def downgrade() -> None: + inspector = sa.inspect(op.get_bind()) + if not inspector.has_table(_TABLE): + return + op.drop_index(_INDEX, table_name=_TABLE, if_exists=True) + op.create_index( + _INDEX, + _TABLE, + ["user_id", "organization_id", "workspace_id", "created_at"], + if_not_exists=True, + ) diff --git a/backend/api/disksage.py b/backend/api/disksage.py index 59166f9ad..1541af9ff 100644 --- a/backend/api/disksage.py +++ b/backend/api/disksage.py @@ -71,31 +71,31 @@ async def ingest_file_lineage( ) return _summary(existing) - record = DiskSageFileLineageRecord( - lineage_record_uid=f"disksage_lineage_{uuid.uuid4().hex}", - user_id=auth_context.user_id, - organization_id=auth_context.organization_id, - workspace_id=auth_context.workspace_id, - lineage_fingerprint=lineage_fingerprint, - envelope_sha256=envelope_sha256, - schema_version=envelope.schema_version, - schema_kind=envelope.schema_kind, - source_kind=envelope.source_kind, - archive_kind=envelope.archive_kind, - raw_content_sha256=envelope.raw_content_sha256, - raw_content_blake3=envelope.raw_content_blake3, - content_bytes=envelope.bytes, - ontology_class=envelope.ontology_class, - ontology_relation_count=len(envelope.ontology_relations), - ontology_predicates=ontology_predicates(envelope), - provider_name=envelope.cloud_copy.provider, - provider_sync_confirmed=envelope.cloud_copy.provider_sync_confirmed, - provider_sync_state=envelope.cloud_copy.provider_sync_state or "unknown", - envelope_json_encrypted=_encrypted_envelope_json(envelope), - created_at=datetime.datetime.now(datetime.timezone.utc), - ) - db.add(record) try: + record = DiskSageFileLineageRecord( + lineage_record_uid=f"disksage_lineage_{uuid.uuid4().hex}", + user_id=auth_context.user_id, + organization_id=auth_context.organization_id, + workspace_id=auth_context.workspace_id, + lineage_fingerprint=lineage_fingerprint, + envelope_sha256=envelope_sha256, + schema_version=envelope.schema_version, + schema_kind=envelope.schema_kind, + source_kind=envelope.source_kind, + archive_kind=envelope.archive_kind, + raw_content_sha256=envelope.raw_content_sha256, + raw_content_blake3=envelope.raw_content_blake3, + content_bytes=envelope.bytes, + ontology_class=envelope.ontology_class, + ontology_relation_count=len(envelope.ontology_relations), + ontology_predicates=ontology_predicates(envelope), + provider_name=envelope.cloud_copy.provider, + provider_sync_confirmed=envelope.cloud_copy.provider_sync_confirmed, + provider_sync_state=envelope.cloud_copy.provider_sync_state or "unknown", + envelope_json_encrypted=_encrypted_envelope_json(envelope), + created_at=datetime.datetime.now(datetime.timezone.utc), + ) + db.add(record) await db.commit() await db.refresh(record) except IntegrityError: diff --git a/backend/db/models.py b/backend/db/models.py index 26bbc74fe..c47b10ffb 100644 --- a/backend/db/models.py +++ b/backend/db/models.py @@ -1121,7 +1121,6 @@ class DiskSageFileLineageRecord(Base): Index( "ix_disksage_lineage_scope_time", "user_id", - "organization_id", "workspace_id", "created_at", ), diff --git a/backend/services/disksage_file_lineage.py b/backend/services/disksage_file_lineage.py index 2e13f87e6..1d639cdd0 100644 --- a/backend/services/disksage_file_lineage.py +++ b/backend/services/disksage_file_lineage.py @@ -10,6 +10,7 @@ import hashlib import json from typing import Literal +from pathlib import PureWindowsPath from pydantic import BaseModel, ConfigDict, Field, model_validator @@ -158,15 +159,33 @@ def bind_provider_evidence(self) -> CloudCopyLineage: self.sync_confirmed_at_ms, ) if self.provider_sync_confirmed and any( - value is None for value in evidence_fields + value is None + or (isinstance(value, str) and not value.strip()) + for value in evidence_fields ): raise ValueError("confirmed sync is missing provider evidence") if self.provider_sync_state not in (None, "unknown") and ( self.provider_sync_confirmed != (self.provider_sync_state == "complete") ): raise ValueError("provider sync state does not match confirmation") - if self.remote_location_bound is True and not self.remote_object_id: + if self.remote_location_bound is True and ( + self.remote_object_id is None or not self.remote_object_id.strip() + ): raise ValueError("remote location binding is missing its object id") + approval_fields = ( + self.copy_approval_id, + self.copy_approval_action, + self.copy_approved_at_ms, + self.copy_approved_by, + self.copy_approval_rationale, + ) + approval_present = [value is not None for value in approval_fields] + if any(approval_present) and not all(approval_present): + raise ValueError("copy approval evidence must be complete or absent") + if any(approval_present) and ( + not self.copy_approved_by.strip() or not self.copy_approval_rationale.strip() + ): + raise ValueError("copy approval attribution is missing") return self @@ -197,7 +216,9 @@ class FileLineageEnvelope(_StrictModel): @model_validator(mode="after") def validate_path_and_relations(self) -> FileLineageEnvelope: - relative = self.source_relative_path.replace("\\", "/") + relative = self.source_relative_path + if "\\" in relative or PureWindowsPath(relative).drive: + raise ValueError("source relative path is not normalized") parts = relative.split("/") if relative.startswith("/") or any(part in {"", ".", ".."} for part in parts): raise ValueError("source relative path is not a normalized relative path") @@ -205,6 +226,15 @@ def validate_path_and_relations(self) -> FileLineageEnvelope: raise ValueError("source filename does not match source relative path") if any(ord(character) < 32 for character in relative): raise ValueError("source relative path contains a control character") + approval_fields = ( + self.cloud_copy.copy_approval_id, + self.cloud_copy.copy_approval_action, + self.cloud_copy.copy_approved_at_ms, + self.cloud_copy.copy_approved_by, + self.cloud_copy.copy_approval_rationale, + ) + if self.schema_version == 1 and any(value is not None for value in approval_fields): + raise ValueError("schema version 1 cannot carry copy approval evidence") return self diff --git a/backend/tests/test_disksage_file_lineage.py b/backend/tests/test_disksage_file_lineage.py index 9b04dc38b..87152b59b 100644 --- a/backend/tests/test_disksage_file_lineage.py +++ b/backend/tests/test_disksage_file_lineage.py @@ -138,6 +138,8 @@ def test_v2_preserves_attributed_copy_approval_fields(): [ ("source_relative_path", "/absolute/path.mov"), ("source_relative_path", "../escape.mov"), + ("source_relative_path", "DaVinci\\Video 1.mov"), + ("source_relative_path", "C:/Video 1.mov"), ("source_filename", "different.mov"), ], ) @@ -189,6 +191,54 @@ def test_complete_provider_sync_state_requires_confirmation(): FileLineageEnvelope.model_validate(payload) +def test_copy_approval_evidence_is_all_or_none_and_versioned(): + partial = _envelope() + partial["cloud_copy"] = { + **partial["cloud_copy"], # type: ignore[arg-type] + "copy_approval_id": "1" * 64, + } + with pytest.raises(ValidationError): + FileLineageEnvelope.model_validate(partial) + + v1_complete = _envelope( + cloud_copy={ + **_envelope()["cloud_copy"], # type: ignore[arg-type] + "copy_approval_id": "1" * 64, + "copy_approval_action": "copy-only", + "copy_approved_at_ms": 6, + "copy_approved_by": "human:local:test", + "copy_approval_rationale": "approved", + } + ) + with pytest.raises(ValidationError): + FileLineageEnvelope.model_validate(v1_complete) + + +def test_confirmed_provider_evidence_rejects_blank_identifiers(): + payload = _envelope() + payload["cloud_copy"] = { + **payload["cloud_copy"], # type: ignore[arg-type] + "provider_sync_confirmed": True, + "sync_evidence_record_id": " " * 2, + "sync_evidence_kind": "provider-native-status", + "sync_evidence_id": "evidence-id", + "sync_confirmed_at_ms": 7, + } + with pytest.raises(ValidationError): + FileLineageEnvelope.model_validate(payload) + + +def test_location_bound_remote_proof_rejects_blank_object_id(): + payload = _envelope() + payload["cloud_copy"] = { + **payload["cloud_copy"], # type: ignore[arg-type] + "remote_location_bound": True, + "remote_object_id": " ", + } + with pytest.raises(ValidationError): + FileLineageEnvelope.model_validate(payload) + + def test_unknown_fields_are_rejected_at_the_handoff_boundary(): payload = _envelope(untrusted_private_value="must-not-persist") with pytest.raises(ValidationError): From 3437e44474d386976b2e922bf66a11e31c6e780b Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 13 Aug 2026 21:24:45 +0900 Subject: [PATCH 08/29] fix: migrate legacy DiskSage lineage schemas --- backend/services/disksage_file_lineage.py | 21 +++++++++++++++----- backend/tests/test_disksage_file_lineage.py | 22 +++++++++++++++++++-- 2 files changed, 36 insertions(+), 7 deletions(-) diff --git a/backend/services/disksage_file_lineage.py b/backend/services/disksage_file_lineage.py index 1d639cdd0..c58e6ffc7 100644 --- a/backend/services/disksage_file_lineage.py +++ b/backend/services/disksage_file_lineage.py @@ -44,6 +44,7 @@ "content-mismatch", "unknown", ] +UNKNOWN_ONTOLOGY_CLASS = "https://disksage.app/ontology#Unknown" class _StrictModel(BaseModel): @@ -190,17 +191,19 @@ def bind_provider_evidence(self) -> CloudCopyLineage: class FileLineageEnvelope(_StrictModel): - # DiskSage v2 adds attributed copy-approval evidence while retaining the - # v1 ontology envelope shape. - schema_version: Literal[1, 2] + # DiskSage v3 makes ontology projection and provider-native state explicit; + # v1/v2 remain readable with deterministic legacy defaults. + schema_version: Literal[1, 2, 3] schema_kind: Literal["disksage.file-lineage"] source_kind: Literal["file"] archive_kind: ARCHIVE_KIND_VALUES source_filename: str = Field(min_length=1, max_length=512) source_relative_path: str = Field(min_length=1, max_length=4096) source_context: str = Field(min_length=1, max_length=1024) - ontology_class: str = Field(pattern=ONTOLOGY_CLASS_PATTERN) - ontology_relations: list[FileLineageRelation] = Field(max_length=256) + ontology_class: str = Field( + default=UNKNOWN_ONTOLOGY_CLASS, pattern=ONTOLOGY_CLASS_PATTERN + ) + ontology_relations: list[FileLineageRelation] = Field(default_factory=list, max_length=256) raw_content_sha256: str = Field(pattern=HEX64_PATTERN) raw_content_blake3: str = Field(pattern=HEX64_PATTERN) bytes: int = Field(ge=0) @@ -214,6 +217,14 @@ class FileLineageEnvelope(_StrictModel): review: ReviewLineage cloud_copy: CloudCopyLineage + @model_validator(mode="before") + @classmethod + def require_v3_ontology_projection(cls, values: object) -> object: + if isinstance(values, dict) and values.get("schema_version") == 3: + if not values.get("ontology_class") or "ontology_relations" not in values: + raise ValueError("schema version 3 requires ontology projection") + return values + @model_validator(mode="after") def validate_path_and_relations(self) -> FileLineageEnvelope: relative = self.source_relative_path diff --git a/backend/tests/test_disksage_file_lineage.py b/backend/tests/test_disksage_file_lineage.py index 87152b59b..fd59f4297 100644 --- a/backend/tests/test_disksage_file_lineage.py +++ b/backend/tests/test_disksage_file_lineage.py @@ -105,8 +105,8 @@ def test_valid_envelope_keeps_graph_projection_deterministic(): assert canonical_envelope_json(envelope).encode("utf-8") -@pytest.mark.parametrize("schema_version", [1, 2]) -def test_accepts_disk_sage_v1_and_v2_envelopes(schema_version: int): +@pytest.mark.parametrize("schema_version", [1, 2, 3]) +def test_accepts_disk_sage_file_lineage_schema_versions(schema_version: int): envelope = FileLineageEnvelope.model_validate( _envelope(schema_version=schema_version) ) @@ -114,6 +114,24 @@ def test_accepts_disk_sage_v1_and_v2_envelopes(schema_version: int): assert envelope.schema_version == schema_version +def test_legacy_v2_without_new_ontology_projection_is_migrated(): + payload = _envelope(schema_version=2) + payload.pop("ontology_class") + payload.pop("ontology_relations") + + envelope = FileLineageEnvelope.model_validate(payload) + + assert envelope.ontology_class.endswith("#Unknown") + assert envelope.ontology_relations == [] + + +def test_v3_requires_ontology_projection(): + payload = _envelope(schema_version=3) + payload.pop("ontology_class") + with pytest.raises(ValidationError): + FileLineageEnvelope.model_validate(payload) + + def test_v2_preserves_attributed_copy_approval_fields(): payload = _envelope( schema_version=2, From c7b46d042eca44cb9fb0dad50228e6383464dde5 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 13 Aug 2026 21:36:39 +0900 Subject: [PATCH 09/29] fix: keep DiskSage Alembic revisions within storage limits --- .../alembic/versions/0019_disksage_provider_sync_state.py | 4 ++-- .../versions/0020_disksage_lineage_column_names.py | 8 ++++---- .../alembic/versions/0021_disksage_lineage_scope_index.py | 4 ++-- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/backend/alembic/versions/0019_disksage_provider_sync_state.py b/backend/alembic/versions/0019_disksage_provider_sync_state.py index f3c7fdc33..314a8b2ba 100644 --- a/backend/alembic/versions/0019_disksage_provider_sync_state.py +++ b/backend/alembic/versions/0019_disksage_provider_sync_state.py @@ -1,6 +1,6 @@ """persist provider-native sync state for DiskSage lineage envelopes -Revision ID: 0019_disksage_provider_sync_state +Revision ID: 0019_disksage_sync_state Revises: 0018_disksage_file_lineage Create Date: 2026-08-13 00:00:00.000000 """ @@ -9,7 +9,7 @@ import sqlalchemy as sa -revision = "0019_disksage_provider_sync_state" +revision = "0019_disksage_sync_state" down_revision = "0018_disksage_file_lineage" branch_labels = None depends_on = None diff --git a/backend/alembic/versions/0020_disksage_lineage_column_names.py b/backend/alembic/versions/0020_disksage_lineage_column_names.py index f82a36e6b..326a69452 100644 --- a/backend/alembic/versions/0020_disksage_lineage_column_names.py +++ b/backend/alembic/versions/0020_disksage_lineage_column_names.py @@ -1,7 +1,7 @@ """rename ambiguous DiskSage lineage column names -Revision ID: 0020_disksage_lineage_column_names -Revises: 0019_disksage_provider_sync_state +Revision ID: 0020_disksage_columns +Revises: 0019_disksage_sync_state Create Date: 2026-08-13 00:00:00.000000 """ @@ -9,8 +9,8 @@ import sqlalchemy as sa -revision = "0020_disksage_lineage_column_names" -down_revision = "0019_disksage_provider_sync_state" +revision = "0020_disksage_columns" +down_revision = "0019_disksage_sync_state" branch_labels = None depends_on = None diff --git a/backend/alembic/versions/0021_disksage_lineage_scope_index.py b/backend/alembic/versions/0021_disksage_lineage_scope_index.py index eb074d0ba..bfb6c33eb 100644 --- a/backend/alembic/versions/0021_disksage_lineage_scope_index.py +++ b/backend/alembic/versions/0021_disksage_lineage_scope_index.py @@ -4,8 +4,8 @@ import sqlalchemy as sa -revision = "0021_disksage_lineage_scope_index" -down_revision = "0020_disksage_lineage_column_names" +revision = "0021_disksage_scope_idx" +down_revision = "0020_disksage_columns" branch_labels = None depends_on = None From 5ea732500f65bee7a03a67cf363db5b582b6f4cd Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 13 Aug 2026 22:06:53 +0900 Subject: [PATCH 10/29] fix: keep Python 3.14 safety checks green --- backend/api/dav.py | 33 ++++++++++++++++++++++++++++++++- backend/services/text_safety.py | 4 +++- 2 files changed, 35 insertions(+), 2 deletions(-) diff --git a/backend/api/dav.py b/backend/api/dav.py index d618c25f3..5aa95e672 100644 --- a/backend/api/dav.py +++ b/backend/api/dav.py @@ -158,7 +158,38 @@ async def _handle_project_propfind( @router.api_route( "/{path:path}", - methods=["PROPFIND", "REPORT", "MKCOL", "GET", "PUT", "DELETE", "OPTIONS"], + methods=["OPTIONS"], + operation_id="dav_options", +) +@router.api_route( + "/{path:path}", + methods=["DELETE"], + operation_id="dav_delete", +) +@router.api_route( + "/{path:path}", + methods=["PUT"], + operation_id="dav_put", +) +@router.api_route( + "/{path:path}", + methods=["GET"], + operation_id="dav_get", +) +@router.api_route( + "/{path:path}", + methods=["REPORT"], + operation_id="dav_report", +) +@router.api_route( + "/{path:path}", + methods=["MKCOL"], + operation_id="dav_mkcol", +) +@router.api_route( + "/{path:path}", + methods=["PROPFIND"], + operation_id="dav_propfind", ) async def dav_handler( request: Request, diff --git a/backend/services/text_safety.py b/backend/services/text_safety.py index d468a7d2f..62e8ed881 100644 --- a/backend/services/text_safety.py +++ b/backend/services/text_safety.py @@ -460,7 +460,9 @@ def strip_html_markup(value: str) -> str: cleaned_lines = [] for line in text.splitlines(): cleaned_lines.append(_strip_tag_like_segments(line)) - text = "\n".join(cleaned_lines).strip() + # A malformed comment opener such as ```` can leave the closing marker + # as parser data. Never return that marker from an HTML-sanitized value. + text = "\n".join(cleaned_lines).replace("-->", "").strip() for token, original in placeholders.items(): text = text.replace(token, original) From b6167d8bb3c3664b74c6d3f91d93205fc19f558b Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 13 Aug 2026 22:51:14 +0900 Subject: [PATCH 11/29] docs: document current lineage schema versions --- docs/architecture/disksage-file-lineage-handoff.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/architecture/disksage-file-lineage-handoff.md b/docs/architecture/disksage-file-lineage-handoff.md index 05ea96690..7854eefdc 100644 --- a/docs/architecture/disksage-file-lineage-handoff.md +++ b/docs/architecture/disksage-file-lineage-handoff.md @@ -13,7 +13,7 @@ summary and cannot represent a general file's provenance or ontology edges. ## Decision -Naruon accepts `disksage.file-lineage` versions 1 and 2 through +Naruon accepts `disksage.file-lineage` versions 1, 2, and 3 through `POST /api/disksage/file-lineage`. The request boundary is strict and rejects unknown fields, unsafe relative paths, unverified copies, and `provider_write_executed=true` claims. The complete envelope is encrypted at From f724378642e82fb95a1fb5ad0127eca9c102cf83 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 13 Aug 2026 23:20:27 +0900 Subject: [PATCH 12/29] fix: hide DAV extensions and preserve comment text --- backend/api/dav.py | 3 +++ backend/services/text_safety.py | 12 +++++++----- backend/tests/test_dav_api.py | 27 +++++++++++++++++++++++++++ backend/tests/test_text_safety.py | 4 ++++ 4 files changed, 41 insertions(+), 5 deletions(-) diff --git a/backend/api/dav.py b/backend/api/dav.py index 5aa95e672..ef93ffb9d 100644 --- a/backend/api/dav.py +++ b/backend/api/dav.py @@ -180,16 +180,19 @@ async def _handle_project_propfind( "/{path:path}", methods=["REPORT"], operation_id="dav_report", + include_in_schema=False, ) @router.api_route( "/{path:path}", methods=["MKCOL"], operation_id="dav_mkcol", + include_in_schema=False, ) @router.api_route( "/{path:path}", methods=["PROPFIND"], operation_id="dav_propfind", + include_in_schema=False, ) async def dav_handler( request: Request, diff --git a/backend/services/text_safety.py b/backend/services/text_safety.py index 62e8ed881..e7a03305f 100644 --- a/backend/services/text_safety.py +++ b/backend/services/text_safety.py @@ -452,18 +452,20 @@ def strip_html_markup(value: str) -> str: decoded = _decode_entities(value) masked, placeholders = _mask_angle_emails(decoded) + # Python's HTMLParser can expose the tail of the malformed ```` opener + # as data on some versions. Make that opener a real ignored comment instead + # of deleting every legitimate ``-->`` sequence from the final text. + masked = masked.replace("", "`` can leave the closing marker - # as parser data. Never return that marker from an HTML-sanitized value. - text = "\n".join(cleaned_lines).replace("-->", "").strip() - + text = "\n".join(cleaned_lines).strip() + for token, original in placeholders.items(): text = text.replace(token, original) return text diff --git a/backend/tests/test_dav_api.py b/backend/tests/test_dav_api.py index 70d455ea9..298cddbf9 100644 --- a/backend/tests/test_dav_api.py +++ b/backend/tests/test_dav_api.py @@ -68,6 +68,33 @@ def test_dav_options(dev_auth_dependency_overrides): assert "calendar-access" in response.headers.get("DAV", "") +def test_dav_extension_methods_are_hidden_from_openapi_but_registered( + dev_auth_dependency_overrides, +): + with TestClient(app) as client: + schema = client.get("/openapi.json").json() + + dav_path = schema["paths"]["/dav/{path}"] + assert {"get", "put", "delete", "options"}.issubset(dav_path) + assert not {"report", "mkcol", "propfind"}.intersection(dav_path) + + +@pytest.mark.parametrize("method", ["REPORT", "MKCOL"]) +def test_dav_extension_methods_dispatch_fail_closed( + method, + dev_auth_dependency_overrides, +): + with TestClient(app) as client: + response = client.request( + method, + "/dav/user123/projects/file.ics", + headers=AUTH_HEADERS, + ) + + assert response.status_code == 501 + assert "Provider-backed DAV method is not implemented" in response.text + + def test_dav_rejects_different_user_path(dev_auth_dependency_overrides): with TestClient(app) as client: response = client.request( diff --git a/backend/tests/test_text_safety.py b/backend/tests/test_text_safety.py index 7ec93d55d..c6269eee4 100644 --- a/backend/tests/test_text_safety.py +++ b/backend/tests/test_text_safety.py @@ -35,6 +35,10 @@ def test_strip_html_markup_preserves_safe_angle_bracket_display_text(safe_text): assert strip_html_markup(safe_text) == safe_text +def test_strip_html_markup_preserves_literal_comment_close_text(): + assert strip_html_markup("

literal --> content

") == "literal --> content" + + def test_strip_html_markup_plain_text_fast_path_skips_parser(monkeypatch): class ParserThatShouldNotRun: def __init__(self): From a76ef92abfe3254bb343e10edb7519ca0339793e Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 13 Aug 2026 23:24:37 +0900 Subject: [PATCH 13/29] test: cover DAV GET dispatch --- backend/tests/test_dav_api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/tests/test_dav_api.py b/backend/tests/test_dav_api.py index 298cddbf9..52bc90772 100644 --- a/backend/tests/test_dav_api.py +++ b/backend/tests/test_dav_api.py @@ -79,7 +79,7 @@ def test_dav_extension_methods_are_hidden_from_openapi_but_registered( assert not {"report", "mkcol", "propfind"}.intersection(dav_path) -@pytest.mark.parametrize("method", ["REPORT", "MKCOL"]) +@pytest.mark.parametrize("method", ["GET", "REPORT", "MKCOL"]) def test_dav_extension_methods_dispatch_fail_closed( method, dev_auth_dependency_overrides, From d5bbed85d68a451bd5c8c281438944d693707a0c Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 14 Aug 2026 10:26:45 +0900 Subject: [PATCH 14/29] fix(naruon): enforce metadata-first lineage boundary --- backend/api/disksage.py | 22 +- backend/services/disksage_file_lineage.py | 79 ++++++- backend/tests/test_disksage_file_lineage.py | 105 ++++++++- .../tests/test_disksage_file_lineage_api.py | 211 ++++++++++++++++++ .../test_disksage_file_lineage_postgres.py | 130 +++++++++++ .../disksage-file-lineage-handoff.md | 6 + .../erd/disksage-file-lineage.dbml | 2 +- 7 files changed, 544 insertions(+), 11 deletions(-) create mode 100644 backend/tests/test_disksage_file_lineage_api.py create mode 100644 backend/tests/test_disksage_file_lineage_postgres.py diff --git a/backend/api/disksage.py b/backend/api/disksage.py index 1541af9ff..6cb52db5f 100644 --- a/backend/api/disksage.py +++ b/backend/api/disksage.py @@ -45,6 +45,19 @@ def _encrypted_envelope_json(envelope: FileLineageEnvelope) -> str: return canonical_envelope_json(envelope) +def _lineage_scope(auth_context: AuthContext) -> tuple[object, ...]: + organization_filter = ( + DiskSageFileLineageRecord.organization_id == auth_context.organization_id + if auth_context.organization_id is not None + else DiskSageFileLineageRecord.organization_id.is_(None) + ) + return ( + DiskSageFileLineageRecord.user_id == auth_context.user_id, + organization_filter, + DiskSageFileLineageRecord.workspace_id == auth_context.workspace_id, + ) + + @router.post("/file-lineage", response_model=FileLineageSummary, status_code=201) async def ingest_file_lineage( envelope: FileLineageEnvelope, @@ -57,8 +70,7 @@ async def ingest_file_lineage( lineage_fingerprint = envelope.cloud_copy.lineage_fingerprint existing_result = await db.execute( select(DiskSageFileLineageRecord).where( - DiskSageFileLineageRecord.user_id == auth_context.user_id, - DiskSageFileLineageRecord.workspace_id == auth_context.workspace_id, + *_lineage_scope(auth_context), DiskSageFileLineageRecord.lineage_fingerprint == lineage_fingerprint, ) ) @@ -102,8 +114,7 @@ async def ingest_file_lineage( await db.rollback() replayed_result = await db.execute( select(DiskSageFileLineageRecord).where( - DiskSageFileLineageRecord.user_id == auth_context.user_id, - DiskSageFileLineageRecord.workspace_id == auth_context.workspace_id, + *_lineage_scope(auth_context), DiskSageFileLineageRecord.lineage_fingerprint == lineage_fingerprint, ) ) @@ -136,8 +147,7 @@ async def list_file_lineage( result = await db.execute( select(DiskSageFileLineageRecord) .where( - DiskSageFileLineageRecord.user_id == auth_context.user_id, - DiskSageFileLineageRecord.workspace_id == auth_context.workspace_id, + *_lineage_scope(auth_context), ) .order_by(DiskSageFileLineageRecord.created_at.desc()) .limit(limit) diff --git a/backend/services/disksage_file_lineage.py b/backend/services/disksage_file_lineage.py index c58e6ffc7..455f49b94 100644 --- a/backend/services/disksage_file_lineage.py +++ b/backend/services/disksage_file_lineage.py @@ -9,8 +9,9 @@ import hashlib import json -from typing import Literal +from datetime import UTC, datetime from pathlib import PureWindowsPath +from typing import Literal from pydantic import BaseModel, ConfigDict, Field, model_validator @@ -45,6 +46,12 @@ "unknown", ] UNKNOWN_ONTOLOGY_CLASS = "https://disksage.app/ontology#Unknown" +EVIDENCE_PRECEDENCE = ( + "embedded_metadata", + "explicit_filename_date", + "filesystem_created_at", + "filesystem_modified_at", +) class _StrictModel(BaseModel): @@ -237,6 +244,7 @@ def validate_path_and_relations(self) -> FileLineageEnvelope: raise ValueError("source filename does not match source relative path") if any(ord(character) < 32 for character in relative): raise ValueError("source relative path contains a control character") + _validate_production_evidence(self.production_time, self.metadata_evidence) approval_fields = ( self.cloud_copy.copy_approval_id, self.cloud_copy.copy_approval_action, @@ -267,6 +275,75 @@ class FileLineageSummary(_StrictModel): created_at: str +def _date_value(epoch_ms: int) -> str: + try: + return datetime.fromtimestamp(epoch_ms / 1000, tz=UTC).date().isoformat() + except (OverflowError, OSError, ValueError) as error: + raise ValueError("production time is out of bounds") from error + + +def _production_source_class(source: str) -> str: + if source.startswith("embedded:"): + return "embedded_metadata" + return { + "filename:path-token": "explicit_filename_date", + "filesystem:created": "filesystem_created_at", + "filesystem:modified-fallback": "filesystem_modified_at", + }.get(source, "") + + +def _evidence_source_class(evidence: FileMetadataEvidence) -> str | None: + if evidence.field == "production-date" and evidence.source.startswith("embedded:"): + return "embedded_metadata" + return { + ("filename-date-hint", "filename:path-token"): "explicit_filename_date", + ("filesystem-created-date", "filesystem:created"): "filesystem_created_at", + ( + "filesystem-modified-date", + "filesystem:modified", + ): "filesystem_modified_at", + }.get((evidence.field, evidence.source)) + + +def _validate_production_evidence( + production: ProductionTimeLineage, + metadata: list[FileMetadataEvidence], +) -> None: + if production.evidence_precedence != list(EVIDENCE_PRECEDENCE): + raise ValueError("production evidence precedence is not canonical") + source_class = _production_source_class(production.selected_source) + if not source_class: + raise ValueError("production time source is unsupported") + if source_class != "embedded_metadata" and production.confidence != "low": + raise ValueError("non-embedded production evidence must be low confidence") + expected = { + "embedded_metadata": ("production-date", production.selected_source), + "explicit_filename_date": ("filename-date-hint", "filename:path-token"), + "filesystem_created_at": ("filesystem-created-date", "filesystem:created"), + "filesystem_modified_at": ( + "filesystem-modified-date", + "filesystem:modified", + ), + }[source_class] + selected_date = _date_value(production.selected_value_ms) + if not any( + evidence.field == expected[0] + and evidence.source == expected[1] + and evidence.value == selected_date + for evidence in metadata + ): + raise ValueError("selected production evidence is missing or mismatched") + selected_rank = EVIDENCE_PRECEDENCE.index(source_class) + if any( + EVIDENCE_PRECEDENCE.index(evidence_class) < selected_rank + for evidence_class in ( + _evidence_source_class(evidence) for evidence in metadata + ) + if evidence_class is not None + ): + raise ValueError("production evidence violates precedence") + + def canonical_envelope_json(envelope: FileLineageEnvelope) -> str: """Serialize the validated envelope deterministically for hashing and storage.""" diff --git a/backend/tests/test_disksage_file_lineage.py b/backend/tests/test_disksage_file_lineage.py index fd59f4297..6db631989 100644 --- a/backend/tests/test_disksage_file_lineage.py +++ b/backend/tests/test_disksage_file_lineage.py @@ -50,9 +50,9 @@ def _envelope(**overrides: object) -> dict[str, object]: "filesystem_time": {"created_at_ms": 2, "modified_at_ms": 3}, "metadata_evidence": [ { - "field": "production_time", - "value": "2026-04-28T00:00:00Z", - "source": "exiftool:MediaCreateDate", + "field": "production-date", + "value": "1970-01-01", + "source": "embedded:exiftool:MediaCreateDate", "confidence": "high", } ], @@ -189,6 +189,105 @@ def test_provider_sync_state_preserves_pending_upload_without_eviction_claim(): assert envelope.cloud_copy.provider_sync_confirmed is False +@pytest.mark.parametrize("provider_sync_state", ["pending-upload", "not-local-current"]) +def test_provider_sync_incomplete_states_remain_unconfirmed(provider_sync_state: str): + payload = _envelope( + cloud_copy={ + **_envelope()["cloud_copy"], # type: ignore[arg-type] + "provider_sync_state": provider_sync_state, + } + ) + + envelope = FileLineageEnvelope.model_validate(payload) + + assert envelope.cloud_copy.provider_sync_state == provider_sync_state + assert envelope.cloud_copy.provider_sync_confirmed is False + + +def test_filename_date_is_only_auxiliary_when_embedded_metadata_exists(): + payload = _envelope( + production_time={ + "selected_value_ms": 1, + "selected_source": "filename:path-token", + "confidence": "low", + "evidence_precedence": [ + "embedded_metadata", + "explicit_filename_date", + "filesystem_created_at", + "filesystem_modified_at", + ], + }, + metadata_evidence=[ + { + "field": "filename-date-hint", + "value": "1970-01-01", + "source": "filename:path-token", + "confidence": "low", + }, + { + "field": "production-date", + "value": "1970-01-01", + "source": "embedded:exiftool:MediaCreateDate", + "confidence": "high", + }, + ], + ) + with pytest.raises(ValidationError): + FileLineageEnvelope.model_validate(payload) + + +def test_filename_date_can_be_selected_only_without_higher_precedence_evidence(): + payload = _envelope( + production_time={ + "selected_value_ms": 1, + "selected_source": "filename:path-token", + "confidence": "low", + "evidence_precedence": [ + "embedded_metadata", + "explicit_filename_date", + "filesystem_created_at", + "filesystem_modified_at", + ], + }, + metadata_evidence=[ + { + "field": "filename-date-hint", + "value": "1970-01-01", + "source": "filename:path-token", + "confidence": "low", + } + ], + ) + envelope = FileLineageEnvelope.model_validate(payload) + assert envelope.production_time.selected_source == "filename:path-token" + + +def test_filesystem_modified_fallback_uses_modified_evidence_source(): + payload = _envelope( + production_time={ + "selected_value_ms": 1, + "selected_source": "filesystem:modified-fallback", + "confidence": "low", + "evidence_precedence": [ + "embedded_metadata", + "explicit_filename_date", + "filesystem_created_at", + "filesystem_modified_at", + ], + }, + metadata_evidence=[ + { + "field": "filesystem-modified-date", + "value": "1970-01-01", + "source": "filesystem:modified", + "confidence": "low", + } + ], + ) + envelope = FileLineageEnvelope.model_validate(payload) + assert envelope.production_time.selected_source == "filesystem:modified-fallback" + + def test_provider_sync_confirmation_requires_evidence(): payload = _envelope() payload["cloud_copy"] = { diff --git a/backend/tests/test_disksage_file_lineage_api.py b/backend/tests/test_disksage_file_lineage_api.py new file mode 100644 index 000000000..c0b824377 --- /dev/null +++ b/backend/tests/test_disksage_file_lineage_api.py @@ -0,0 +1,211 @@ +import datetime +from types import SimpleNamespace + +import pytest +from fastapi import HTTPException + +from api.auth import AuthContext +from api.disksage import ingest_file_lineage, list_file_lineage +from core.runtime_secrets import EncryptionKeyMissingError +from services.disksage_file_lineage import ( + FileLineageEnvelope, + canonical_envelope_sha256, +) + + +def _envelope() -> dict[str, object]: + return { + "schema_version": 1, + "schema_kind": "disksage.file-lineage", + "source_kind": "file", + "archive_kind": "media", + "source_filename": "Video 1.mov", + "source_relative_path": "DaVinci Resolve/Video 1.mov", + "source_context": "DaVinci Resolve", + "ontology_class": "https://disksage.app/ontology#Media", + "ontology_relations": [], + "raw_content_sha256": "a" * 64, + "raw_content_blake3": "b" * 64, + "bytes": 160085038, + "production_time": { + "selected_value_ms": 1, + "selected_source": "embedded:exiftool:MediaCreateDate", + "confidence": "high", + "evidence_precedence": [ + "embedded_metadata", + "explicit_filename_date", + "filesystem_created_at", + "filesystem_modified_at", + ], + }, + "filesystem_time": {"created_at_ms": 2, "modified_at_ms": 3}, + "metadata_evidence": [ + { + "field": "production-date", + "value": "1970-01-01", + "source": "embedded:exiftool:MediaCreateDate", + "confidence": "high", + } + ], + "content_authors": [], + "content_context": [], + "review": { + "candidate_fingerprint": "c" * 64, + "review_fingerprint": "d" * 64, + "requires_review": False, + "reason_codes": [], + }, + "cloud_copy": { + "receipt_id": "e" * 64, + "lineage_fingerprint": "f" * 64, + "provider": "icloud", + "destination_account_scope": "unknown", + "destination": "/Users/example/iCloud/Video 1.mov", + "copied_at_ms": 5, + "copy_verification_method": "copied-by-disk-sage", + "local_copy_verified": True, + "provider_write_executed": False, + "provider_sync_confirmed": False, + }, + } + + +class _Result: + def __init__(self, record=None, records=()): + self._record = record + self._records = tuple(records) + + def scalar_one_or_none(self): + return self._record + + def scalars(self): + return self + + def all(self): + return list(self._records) + + +class _Session: + def __init__(self, results, *, commit_error=None): + self._results = iter(results) + self.commit_error = commit_error + self.statements = [] + self.rollback_count = 0 + self.commit_count = 0 + + async def execute(self, statement): + self.statements.append(statement) + return next(self._results) + + def add(self, _record): + pass + + async def commit(self): + self.commit_count += 1 + if self.commit_error is not None: + raise self.commit_error + + async def refresh(self, _record): + pass + + async def rollback(self): + self.rollback_count += 1 + + +def _auth() -> AuthContext: + return AuthContext( + user_id="user-1", + role="member", + organization_id="org-1", + group_ids=(), + workspace_id="workspace-1", + ) + + +def _record(**overrides): + values = { + "lineage_record_uid": "disksage_lineage_1", + "lineage_fingerprint": "f" * 64, + "schema_version": 1, + "source_kind": "file", + "archive_kind": "media", + "raw_content_sha256": "a" * 64, + "raw_content_blake3": "b" * 64, + "content_bytes": 1, + "ontology_class": "https://disksage.app/ontology#Media", + "ontology_relation_count": 0, + "ontology_predicates": [], + "provider_name": "icloud", + "provider_sync_confirmed": False, + "provider_sync_state": "pending-upload", + "created_at": datetime.datetime.now(datetime.timezone.utc), + "envelope_sha256": "c" * 64, + } + values.update(overrides) + return SimpleNamespace(**values) + + +@pytest.mark.asyncio +async def test_replayed_lineage_is_idempotent_with_workspace_and_org_scope(): + envelope = FileLineageEnvelope.model_validate(_envelope()) + record = _record(envelope_sha256=canonical_envelope_sha256(envelope)) + session = _Session([_Result(record)]) + + result = await ingest_file_lineage( + envelope=envelope, + auth_context=_auth(), + db=session, + ) + + assert result.lineage_record_uid == record.lineage_record_uid + assert session.commit_count == 0 + statement = str(session.statements[0]) + assert "organization_id" in statement + assert "workspace_id" in statement + + +@pytest.mark.asyncio +async def test_conflicting_lineage_fingerprint_returns_conflict(): + envelope = FileLineageEnvelope.model_validate(_envelope()) + session = _Session([_Result(_record())]) + + with pytest.raises(HTTPException) as error: + await ingest_file_lineage( + envelope=envelope, + auth_context=_auth(), + db=session, + ) + + assert error.value.status_code == 409 + + +@pytest.mark.asyncio +async def test_missing_encryption_key_rolls_back_as_service_unavailable(): + envelope = FileLineageEnvelope.model_validate(_envelope()) + session = _Session( + [_Result(None)], + commit_error=EncryptionKeyMissingError("missing active key"), + ) + + with pytest.raises(HTTPException) as error: + await ingest_file_lineage( + envelope=envelope, + auth_context=_auth(), + db=session, + ) + + assert error.value.status_code == 503 + assert session.rollback_count == 1 + + +@pytest.mark.asyncio +async def test_list_lineage_query_is_tenant_scoped(): + session = _Session([_Result(records=[_record()])]) + + result = await list_file_lineage(limit=50, auth_context=_auth(), db=session) + + assert len(result) == 1 + statement = str(session.statements[0]) + assert "organization_id" in statement + assert "workspace_id" in statement + assert "user_id" in statement diff --git a/backend/tests/test_disksage_file_lineage_postgres.py b/backend/tests/test_disksage_file_lineage_postgres.py new file mode 100644 index 000000000..f526552ff --- /dev/null +++ b/backend/tests/test_disksage_file_lineage_postgres.py @@ -0,0 +1,130 @@ +import uuid + +import asyncpg +import pytest +from cryptography.fernet import Fernet +from pydantic import SecretStr +from sqlalchemy import delete, text +from sqlalchemy.exc import OperationalError +from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine + +from api.auth import AuthContext +from api.disksage import ingest_file_lineage, list_file_lineage +from core.config import settings +from db.models import Base, DiskSageFileLineageRecord +from services.disksage_file_lineage import FileLineageEnvelope + + +def _envelope() -> dict[str, object]: + return { + "schema_version": 1, + "schema_kind": "disksage.file-lineage", + "source_kind": "file", + "archive_kind": "media", + "source_filename": "Video 1.mov", + "source_relative_path": "DaVinci Resolve/Video 1.mov", + "source_context": "DaVinci Resolve", + "ontology_class": "https://disksage.app/ontology#Media", + "ontology_relations": [], + "raw_content_sha256": "a" * 64, + "raw_content_blake3": "b" * 64, + "bytes": 160085038, + "production_time": { + "selected_value_ms": 1, + "selected_source": "embedded:exiftool:MediaCreateDate", + "confidence": "high", + "evidence_precedence": [ + "embedded_metadata", + "explicit_filename_date", + "filesystem_created_at", + "filesystem_modified_at", + ], + }, + "filesystem_time": {"created_at_ms": 2, "modified_at_ms": 3}, + "metadata_evidence": [ + { + "field": "production-date", + "value": "1970-01-01", + "source": "embedded:exiftool:MediaCreateDate", + "confidence": "high", + } + ], + "content_authors": [], + "content_context": [], + "review": { + "candidate_fingerprint": "c" * 64, + "review_fingerprint": "d" * 64, + "requires_review": False, + "reason_codes": [], + }, + "cloud_copy": { + "receipt_id": "e" * 64, + "lineage_fingerprint": "f" * 64, + "provider": "icloud", + "destination_account_scope": "unknown", + "destination": "/Users/example/iCloud/Video 1.mov", + "copied_at_ms": 5, + "copy_verification_method": "copied-by-disk-sage", + "local_copy_verified": True, + "provider_write_executed": False, + "provider_sync_confirmed": False, + }, + } + + +@pytest.mark.postgres +@pytest.mark.asyncio +async def test_file_lineage_ingest_and_list_postgres_smoke(): + database_url = getattr(settings, "DATABASE_URL", None) + if not database_url: + pytest.skip("PostgreSQL smoke path unavailable: DATABASE_URL is not set") + + scope = uuid.uuid4().hex + auth_context = AuthContext( + user_id=f"disksage_smoke_user_{scope}", + role="member", + organization_id=f"disksage_smoke_org_{scope}", + group_ids=(), + workspace_id=f"disksage_smoke_workspace_{scope}", + ) + envelope = FileLineageEnvelope.model_validate(_envelope()) + engine = create_async_engine(database_url, echo=False) + session_factory = async_sessionmaker(engine, expire_on_commit=False) + previous_key = settings.ENCRYPTION_KEY + settings.ENCRYPTION_KEY = SecretStr(Fernet.generate_key().decode("ascii")) + record_uid = None + try: + try: + async with engine.begin() as connection: + await connection.execute(text("CREATE EXTENSION IF NOT EXISTS vector")) + await connection.run_sync(Base.metadata.create_all) + except (OperationalError, asyncpg.PostgresError, OSError): + pytest.skip("PostgreSQL smoke schema is unavailable") + + async with session_factory() as session: + summary = await ingest_file_lineage( + envelope=envelope, + auth_context=auth_context, + db=session, + ) + record_uid = summary.lineage_record_uid + + async with session_factory() as session: + summaries = await list_file_lineage( + limit=50, + auth_context=auth_context, + db=session, + ) + assert [item.lineage_record_uid for item in summaries] == [record_uid] + assert summaries[0].provider_sync_state == "unknown" + finally: + settings.ENCRYPTION_KEY = previous_key + if record_uid is not None: + async with session_factory() as session: + await session.execute( + delete(DiskSageFileLineageRecord).where( + DiskSageFileLineageRecord.lineage_record_uid == record_uid + ) + ) + await session.commit() + await engine.dispose() diff --git a/docs/architecture/disksage-file-lineage-handoff.md b/docs/architecture/disksage-file-lineage-handoff.md index 7854eefdc..688c4983f 100644 --- a/docs/architecture/disksage-file-lineage-handoff.md +++ b/docs/architecture/disksage-file-lineage-handoff.md @@ -25,6 +25,12 @@ and `created_at`; it does not expose local paths or raw metadata values. A state such as `pending-upload` is retained as an incomplete provider proof and never authorizes source eviction. +Production-time validation is repeated at the Naruon trust boundary with the +canonical order `embedded_metadata` > `explicit_filename_date` > +`filesystem_created_at` > `filesystem_modified_at`. Filename date tokens such +as `2026-04-28` or `251210` are auxiliary evidence; they cannot override an +embedded production date, and a non-embedded selection must be low confidence. + The payload keeps explicit file → archive destination → provider/account and review relations. These relation edges follow the same entity/provenance separation as PROV-O; DiskSage's local ontology remains the domain vocabulary. diff --git a/docs/architecture/erd/disksage-file-lineage.dbml b/docs/architecture/erd/disksage-file-lineage.dbml index 81740e3af..bbb56ff57 100644 --- a/docs/architecture/erd/disksage-file-lineage.dbml +++ b/docs/architecture/erd/disksage-file-lineage.dbml @@ -25,7 +25,7 @@ Table public.disksage_file_lineage_records { indexes { (user_id, workspace_id, lineage_fingerprint) [unique] - (user_id, organization_id, workspace_id, created_at) + (user_id, workspace_id, created_at) (workspace_id, ontology_class) } } From ab5201b7bc8c0f385c00087e30ab8e216a45065d Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 14 Aug 2026 18:17:28 +0900 Subject: [PATCH 15/29] fix(naruon): accept provider API lineage evidence --- backend/services/disksage_file_lineage.py | 4 +++- backend/tests/test_disksage_file_lineage.py | 14 ++++++++++++++ docs/architecture/disksage-file-lineage-handoff.md | 6 +++++- 3 files changed, 22 insertions(+), 2 deletions(-) diff --git a/backend/services/disksage_file_lineage.py b/backend/services/disksage_file_lineage.py index 455f49b94..9e97a6773 100644 --- a/backend/services/disksage_file_lineage.py +++ b/backend/services/disksage_file_lineage.py @@ -132,7 +132,9 @@ class CloudCopyLineage(_StrictModel): destination_account_scope: Literal["personal", "organization", "shared", "unknown"] destination: str = Field(min_length=1, max_length=4096) copied_at_ms: int = Field(ge=0) - copy_verification_method: Literal["copied-by-disk-sage", "adopted-existing"] + copy_verification_method: Literal[ + "copied-by-disk-sage", "copied-by-provider-api", "adopted-existing" + ] # DiskSage v2 binds the copy to one attributed human action. These remain # optional so v1 and pre-approval receipts can still be ingested. copy_approval_id: str | None = Field(default=None, pattern=HEX64_PATTERN) diff --git a/backend/tests/test_disksage_file_lineage.py b/backend/tests/test_disksage_file_lineage.py index 6db631989..f166eaeb0 100644 --- a/backend/tests/test_disksage_file_lineage.py +++ b/backend/tests/test_disksage_file_lineage.py @@ -176,6 +176,20 @@ def test_provider_write_claim_is_rejected_even_when_copy_is_verified(): FileLineageEnvelope.model_validate(payload) +def test_provider_api_copy_method_is_preserved_without_naruon_write_authority(): + payload = _envelope( + cloud_copy={ + **_envelope()["cloud_copy"], # type: ignore[arg-type] + "copy_verification_method": "copied-by-provider-api", + } + ) + + envelope = FileLineageEnvelope.model_validate(payload) + + assert envelope.cloud_copy.copy_verification_method == "copied-by-provider-api" + assert envelope.cloud_copy.provider_write_executed is False + + def test_provider_sync_state_preserves_pending_upload_without_eviction_claim(): payload = _envelope() payload["cloud_copy"] = { diff --git a/docs/architecture/disksage-file-lineage-handoff.md b/docs/architecture/disksage-file-lineage-handoff.md index 688c4983f..1108f1e13 100644 --- a/docs/architecture/disksage-file-lineage-handoff.md +++ b/docs/architecture/disksage-file-lineage-handoff.md @@ -16,7 +16,11 @@ summary and cannot represent a general file's provenance or ontology edges. Naruon accepts `disksage.file-lineage` versions 1, 2, and 3 through `POST /api/disksage/file-lineage`. The request boundary is strict and rejects unknown fields, unsafe relative paths, unverified copies, and -`provider_write_executed=true` claims. The complete envelope is encrypted at +`provider_write_executed=true` claims. `copy_verification_method` may be +`copied-by-disk-sage`, `copied-by-provider-api`, or `adopted-existing`; the +provider-API variant records DiskSage's authenticated write evidence while +Naruon remains a catalog projection and never becomes the write authority. The +complete envelope is encrypted at rest and scoped by authenticated user/workspace. `GET /api/disksage/file-lineage` returns only `lineage_record_uid`, `lineage_fingerprint`, `schema_version`, `source_kind`, `archive_kind`, content hashes, `content_bytes`, ontology class, From 801dbe53ebda4d21fad486a012d1d8bd4221ae06 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 14 Aug 2026 20:45:29 +0900 Subject: [PATCH 16/29] fix: fail closed on browser origin gaps --- backend/main.py | 8 +++++--- backend/tests/test_main.py | 11 +++++++++++ 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/backend/main.py b/backend/main.py index 357450f8a..902e92d80 100644 --- a/backend/main.py +++ b/backend/main.py @@ -129,7 +129,7 @@ def _origin_from_referer(header_value: str | None) -> str | None: def _is_trusted_browser_origin(origin: str | None) -> bool: if origin is None: - return True + return False return origin in set(settings.ALLOWED_CORS_ORIGINS_LIST) @@ -151,20 +151,22 @@ async def reject_cross_site_state_changing_api_requests(request: Request, call_n ) raw_origin = request.headers.get("origin") + raw_referer = request.headers.get("referer") + if not fetch_site and raw_origin is None and raw_referer is None: + return await call_next(request) origin = _normalized_origin(raw_origin) if raw_origin is not None and origin is None: return JSONResponse( status_code=403, content={"error_code": "csrf_origin_rejected"}, ) - if not _is_trusted_browser_origin(origin): + if origin is not None and not _is_trusted_browser_origin(origin): return JSONResponse( status_code=403, content={"error_code": "csrf_origin_rejected"}, ) if origin is None: - raw_referer = request.headers.get("referer") referer_origin = _origin_from_referer(raw_referer) if raw_referer is not None and referer_origin is None: return JSONResponse( diff --git a/backend/tests/test_main.py b/backend/tests/test_main.py index 1852823f5..ed573d119 100644 --- a/backend/tests/test_main.py +++ b/backend/tests/test_main.py @@ -82,6 +82,17 @@ def test_state_changing_api_rejects_untrusted_origin(): assert response.json() == {"error_code": "csrf_origin_rejected"} +def test_state_changing_api_rejects_missing_browser_origin_headers(): + response = client.put( + "/api/accounts/config", + headers={"Sec-Fetch-Site": "same-origin"}, + json={"smtp_server": "mail.example.com"}, + ) + + assert response.status_code == 403 + assert response.json() == {"error_code": "csrf_referer_rejected"} + + def test_state_changing_api_allows_trusted_origin_to_reach_auth_gate(): response = client.put( "/api/accounts/config", From f8f91e7baef04735f6cb76d28094e91ea0836f63 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 14 Aug 2026 21:23:39 +0900 Subject: [PATCH 17/29] fix: accept DiskSage timeliness lineage fields --- backend/services/disksage_file_lineage.py | 13 ++++++ backend/tests/test_disksage_file_lineage.py | 45 +++++++++++++++++++++ 2 files changed, 58 insertions(+) diff --git a/backend/services/disksage_file_lineage.py b/backend/services/disksage_file_lineage.py index 9e97a6773..3223e1067 100644 --- a/backend/services/disksage_file_lineage.py +++ b/backend/services/disksage_file_lineage.py @@ -45,6 +45,7 @@ "content-mismatch", "unknown", ] +PROVIDER_SYNC_TIMELINESS_VALUES = Literal["complete", "pending", "overdue"] UNKNOWN_ONTOLOGY_CLASS = "https://disksage.app/ontology#Unknown" EVIDENCE_PRECEDENCE = ( "embedded_metadata", @@ -155,6 +156,12 @@ class CloudCopyLineage(_StrictModel): remote_object_id: str | None = Field(default=None, max_length=512) remote_revision: str | None = Field(default=None, max_length=512) remote_location_bound: bool | None = None + # DiskSage's diagnostic timeliness projection is retained inside the encrypted envelope; + # it never grants source-eviction authority. + sync_timeliness: PROVIDER_SYNC_TIMELINESS_VALUES | None = None + sync_pending_age_ms: int | None = Field(default=None, ge=0) + sync_overdue_after_ms: int | None = Field(default=None, ge=0) + sync_reason_codes: list[str] = Field(default_factory=list, max_length=32) @model_validator(mode="after") def bind_provider_evidence(self) -> CloudCopyLineage: @@ -182,6 +189,12 @@ def bind_provider_evidence(self) -> CloudCopyLineage: self.remote_object_id is None or not self.remote_object_id.strip() ): raise ValueError("remote location binding is missing its object id") + if self.sync_timeliness == "complete" and ( + not self.provider_sync_confirmed or self.provider_sync_state != "complete" + ): + raise ValueError("complete sync timeliness is not provider-confirmed") + if self.sync_timeliness in {"pending", "overdue"} and self.provider_sync_confirmed: + raise ValueError("incomplete sync timeliness is provider-confirmed") approval_fields = ( self.copy_approval_id, self.copy_approval_action, diff --git a/backend/tests/test_disksage_file_lineage.py b/backend/tests/test_disksage_file_lineage.py index f166eaeb0..cf78d25b8 100644 --- a/backend/tests/test_disksage_file_lineage.py +++ b/backend/tests/test_disksage_file_lineage.py @@ -218,6 +218,51 @@ def test_provider_sync_incomplete_states_remain_unconfirmed(provider_sync_state: assert envelope.cloud_copy.provider_sync_confirmed is False +def test_provider_sync_timeliness_projection_is_preserved_without_eviction_authority(): + payload = _envelope( + cloud_copy={ + **_envelope()["cloud_copy"], # type: ignore[arg-type] + "provider_sync_state": "unknown", + "sync_timeliness": "pending", + "sync_pending_age_ms": 123, + "sync_overdue_after_ms": 456, + "sync_reason_codes": ["provider-sync-confirmation-pending"], + } + ) + + envelope = FileLineageEnvelope.model_validate(payload) + + assert envelope.cloud_copy.sync_timeliness == "pending" + assert envelope.cloud_copy.sync_pending_age_ms == 123 + assert envelope.cloud_copy.sync_overdue_after_ms == 456 + assert envelope.cloud_copy.sync_reason_codes == [ + "provider-sync-confirmation-pending" + ] + assert envelope.cloud_copy.provider_sync_confirmed is False + + +def test_complete_timeliness_requires_confirmed_complete_provider_state(): + payload = _envelope( + cloud_copy={ + **_envelope()["cloud_copy"], # type: ignore[arg-type] + "provider_sync_confirmed": True, + "provider_sync_state": "complete", + "sync_evidence_record_id": "1" * 64, + "sync_evidence_kind": "provider-native-status", + "sync_evidence_id": "icloud-uploaded-flag", + "sync_confirmed_at_ms": 7, + "sync_timeliness": "complete", + "sync_pending_age_ms": 0, + "sync_overdue_after_ms": 456, + "sync_reason_codes": [], + } + ) + + envelope = FileLineageEnvelope.model_validate(payload) + + assert envelope.cloud_copy.sync_timeliness == "complete" + + def test_filename_date_is_only_auxiliary_when_embedded_metadata_exists(): payload = _envelope( production_time={ From a7606645c8335e4a909ca4d52a25ea9a4144a3c2 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 15 Aug 2026 00:52:56 +0900 Subject: [PATCH 18/29] fix: accept DiskSage readiness schema v6 --- backend/scripts/disksage_copy_readiness_handoff.py | 6 +++--- backend/tests/test_disksage_copy_readiness_handoff.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/backend/scripts/disksage_copy_readiness_handoff.py b/backend/scripts/disksage_copy_readiness_handoff.py index 1036d1bb1..1d9ae7a86 100644 --- a/backend/scripts/disksage_copy_readiness_handoff.py +++ b/backend/scripts/disksage_copy_readiness_handoff.py @@ -61,10 +61,10 @@ READINESS_STATES = frozenset( {"no-candidates", "blocked", "partially-ready", "ready-without-new-review"} ) -# DiskSage schema v5 adds path-free provider-global-sync evidence while retaining the same -# success contract consumed by this handoff. Keep v3/v4 readable for already-issued evidence +# DiskSage schema v6 is the current path-free provider-global-sync envelope and retains the same +# success contract consumed by this handoff. Keep v3-v5 readable for already-issued evidence # records; newer envelopes must be added here deliberately and tested. -SUPPORTED_READINESS_SCHEMA_VERSIONS = frozenset({3, 4, 5}) +SUPPORTED_READINESS_SCHEMA_VERSIONS = frozenset({3, 4, 5, 6}) ERROR_CODE_PATTERN = re.compile(r"[a-z0-9]+(?:-[a-z0-9]+)*") diff --git a/backend/tests/test_disksage_copy_readiness_handoff.py b/backend/tests/test_disksage_copy_readiness_handoff.py index 471cc0090..e031fd4e4 100644 --- a/backend/tests/test_disksage_copy_readiness_handoff.py +++ b/backend/tests/test_disksage_copy_readiness_handoff.py @@ -111,7 +111,7 @@ def test_main_delegates_to_absolute_verifier_without_shell_env_or_input_read( assert not (tmp_path / "must-not-exist").exists() -@pytest.mark.parametrize("schema_version", [4, 5]) +@pytest.mark.parametrize("schema_version", [4, 5, 6]) def test_main_accepts_current_disksage_schema_versions(tmp_path, capsys, schema_version): payload = _success_payload() payload["schema_version"] = schema_version From d5a83c8d27c3a5df1a3defd7b6e6940c14663188 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 15 Aug 2026 01:04:32 +0900 Subject: [PATCH 19/29] fix: accept native readiness verifier summary --- .../disksage_copy_readiness_handoff.py | 46 ++++++++++++++++++- .../test_disksage_copy_readiness_handoff.py | 33 +++++++++++++ 2 files changed, 77 insertions(+), 2 deletions(-) diff --git a/backend/scripts/disksage_copy_readiness_handoff.py b/backend/scripts/disksage_copy_readiness_handoff.py index 1d9ae7a86..2604864fc 100644 --- a/backend/scripts/disksage_copy_readiness_handoff.py +++ b/backend/scripts/disksage_copy_readiness_handoff.py @@ -32,7 +32,7 @@ EXIT_VERIFIER_UNAVAILABLE = 66 EXIT_EXECUTION_FAILED = 70 -SUCCESS_FIELDS = frozenset( +BASE_SUCCESS_FIELDS = frozenset( { "ok", "schema_kind", @@ -49,6 +49,14 @@ "source_eviction_authorized", } ) +NATIVE_STATUS_FIELDS = frozenset( + { + "icloud_native_status_observed", + "icloud_native_sync_state", + "icloud_native_status_timed_out", + } +) +SUCCESS_FIELDS = BASE_SUCCESS_FIELDS | NATIVE_STATUS_FIELDS FAILURE_FIELDS = frozenset({"ok", "error_code"}) FALSE_CLAIM_FIELDS = ( "local_paths_included", @@ -66,6 +74,7 @@ # records; newer envelopes must be added here deliberately and tested. SUPPORTED_READINESS_SCHEMA_VERSIONS = frozenset({3, 4, 5, 6}) ERROR_CODE_PATTERN = re.compile(r"[a-z0-9]+(?:-[a-z0-9]+)*") +NATIVE_STATUS_TOKEN_PATTERN = re.compile(r"[A-Za-z0-9|_.-]{1,128}") class HandoffError(Exception): @@ -353,9 +362,15 @@ def _decode_protocol(result: VerifierResult) -> dict[str, object]: raise HandoffError("disksage-verifier-protocol-invalid") if result.returncode == 0: + schema_version = payload.get("schema_version") + fields_valid = ( + frozenset(payload) == SUCCESS_FIELDS + if schema_version == 6 + else frozenset(payload) in (BASE_SUCCESS_FIELDS, SUCCESS_FIELDS) + ) valid = ( not result.stderr - and frozenset(payload) == SUCCESS_FIELDS + and fields_valid and payload.get("ok") is True and payload.get("schema_kind") == "disksage.naruon.cloud-copy-readiness" and type(payload.get("schema_version")) is int @@ -368,6 +383,33 @@ def _decode_protocol(result: VerifierResult) -> dict[str, object]: and payload["candidate_bytes"] >= 0 and _is_lower_hex_64(payload.get("readiness_fingerprint_sha256")) and all(payload.get(field) is False for field in FALSE_CLAIM_FIELDS) + and ( + frozenset(payload) == BASE_SUCCESS_FIELDS + or ( + type(payload.get("icloud_native_status_observed")) is bool + or payload.get("icloud_native_status_observed") is None + ) + ) + and ( + frozenset(payload) == BASE_SUCCESS_FIELDS + or ( + type(payload.get("icloud_native_status_timed_out")) is bool + or payload.get("icloud_native_status_timed_out") is None + ) + ) + and ( + frozenset(payload) == BASE_SUCCESS_FIELDS + or ( + payload.get("icloud_native_sync_state") is None + or ( + type(payload.get("icloud_native_sync_state")) is str + and NATIVE_STATUS_TOKEN_PATTERN.fullmatch( + payload["icloud_native_sync_state"] + ) + is not None + ) + ) + ) ) elif result.returncode in (64, 65): error_code = payload.get("error_code") diff --git a/backend/tests/test_disksage_copy_readiness_handoff.py b/backend/tests/test_disksage_copy_readiness_handoff.py index e031fd4e4..ed7fc2ce8 100644 --- a/backend/tests/test_disksage_copy_readiness_handoff.py +++ b/backend/tests/test_disksage_copy_readiness_handoff.py @@ -36,6 +36,9 @@ def _success_payload() -> dict[str, object]: "raw_metadata_values_included": False, "cloud_write_executed": False, "source_eviction_authorized": False, + "icloud_native_status_observed": None, + "icloud_native_sync_state": None, + "icloud_native_status_timed_out": None, } @@ -121,6 +124,17 @@ def test_main_accepts_current_disksage_schema_versions(tmp_path, capsys, schema_ assert json.loads(capsys.readouterr().out) == payload +def test_main_accepts_legacy_success_protocol_for_prior_schema(tmp_path, capsys): + payload = _success_payload() + payload["schema_version"] = 5 + for field in handoff.NATIVE_STATUS_FIELDS: + payload.pop(field) + verifier = _json_verifier(tmp_path / "verifier", payload, 0) + + assert handoff.main(_handoff_args(verifier, tmp_path / "readiness.json")) == 0 + assert json.loads(capsys.readouterr().out) == payload + + @pytest.mark.parametrize("exit_code", [64, 65]) def test_main_preserves_valid_disksage_failure_protocol(tmp_path, capsys, exit_code): payload = { @@ -765,6 +779,25 @@ def test_protocol_decoder_rejects_invalid_or_ambiguous_transport(result): assert error.value.error_code == "disksage-verifier-protocol-invalid" +@pytest.mark.parametrize( + "field_value", + [ + {"icloud_native_status_observed": "yes"}, + {"icloud_native_status_timed_out": 1}, + {"icloud_native_sync_state": "/private/source"}, + ], +) +def test_protocol_decoder_rejects_invalid_native_status_summary(field_value): + payload = _success_payload() + payload.update(field_value) + with pytest.raises(handoff.HandoffError) as error: + handoff._decode_protocol( + handoff.VerifierResult(0, json.dumps(payload).encode(), b"") + ) + + assert error.value.error_code == "disksage-verifier-protocol-invalid" + + @pytest.mark.parametrize("stream_name", ["stdout", "stderr"]) def test_main_kills_oversized_output_without_echoing_it(tmp_path, capsys, stream_name): verifier = _python_verifier( From 8d2b9377c3c62fbcd96d20e1210e412db1406652 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 15 Aug 2026 09:39:18 +0900 Subject: [PATCH 20/29] test(security): cover DAV cross-site origin guard --- backend/tests/test_main.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/backend/tests/test_main.py b/backend/tests/test_main.py index ed573d119..48910a029 100644 --- a/backend/tests/test_main.py +++ b/backend/tests/test_main.py @@ -71,6 +71,17 @@ def test_state_changing_api_rejects_cross_site_fetch_metadata(): assert response.json() == {"error_code": "csrf_fetch_site_rejected"} +def test_state_changing_dav_rejects_cross_site_fetch_metadata(): + response = client.put( + "/dav/user-1/projects/example", + headers={"Sec-Fetch-Site": "cross-site"}, + content=b"calendar-data", + ) + + assert response.status_code == 403 + assert response.json() == {"error_code": "csrf_fetch_site_rejected"} + + def test_state_changing_api_rejects_untrusted_origin(): response = client.put( "/api/accounts/config", From 208f746728d7737833a2168f525dc4243d5f7354 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 15 Aug 2026 09:42:41 +0900 Subject: [PATCH 21/29] fix(security): enforce browser-origin guard on DAV mutations --- backend/main.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/backend/main.py b/backend/main.py index 902e92d80..2bf52fd50 100644 --- a/backend/main.py +++ b/backend/main.py @@ -134,9 +134,8 @@ def _is_trusted_browser_origin(origin: str | None) -> bool: def _requires_browser_origin_check(request: Request) -> bool: - return ( - request.method.upper() in STATE_CHANGING_API_METHODS - and request.url.path.startswith("/api/") + return request.method.upper() in STATE_CHANGING_API_METHODS and request.url.path.startswith( + ("/api/", "/dav/") ) From 7ae7176d842d4d33b3278ceae5225dcc0374a27d Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 19 Aug 2026 15:19:07 +0900 Subject: [PATCH 22/29] chore(naruon): keep lineage PR scoped --- backend/api/dav.py | 36 +------------------------------ backend/main.py | 13 ++++++----- backend/services/text_safety.py | 6 +++--- backend/tests/test_dav_api.py | 27 ----------------------- backend/tests/test_main.py | 22 ------------------- backend/tests/test_text_safety.py | 8 +++---- 6 files changed, 14 insertions(+), 98 deletions(-) diff --git a/backend/api/dav.py b/backend/api/dav.py index ef93ffb9d..d618c25f3 100644 --- a/backend/api/dav.py +++ b/backend/api/dav.py @@ -158,41 +158,7 @@ async def _handle_project_propfind( @router.api_route( "/{path:path}", - methods=["OPTIONS"], - operation_id="dav_options", -) -@router.api_route( - "/{path:path}", - methods=["DELETE"], - operation_id="dav_delete", -) -@router.api_route( - "/{path:path}", - methods=["PUT"], - operation_id="dav_put", -) -@router.api_route( - "/{path:path}", - methods=["GET"], - operation_id="dav_get", -) -@router.api_route( - "/{path:path}", - methods=["REPORT"], - operation_id="dav_report", - include_in_schema=False, -) -@router.api_route( - "/{path:path}", - methods=["MKCOL"], - operation_id="dav_mkcol", - include_in_schema=False, -) -@router.api_route( - "/{path:path}", - methods=["PROPFIND"], - operation_id="dav_propfind", - include_in_schema=False, + methods=["PROPFIND", "REPORT", "MKCOL", "GET", "PUT", "DELETE", "OPTIONS"], ) async def dav_handler( request: Request, diff --git a/backend/main.py b/backend/main.py index cf13874f4..0569673a6 100644 --- a/backend/main.py +++ b/backend/main.py @@ -130,13 +130,14 @@ def _origin_from_referer(header_value: str | None) -> str | None: def _is_trusted_browser_origin(origin: str | None) -> bool: if origin is None: - return False + return True return origin in set(settings.ALLOWED_CORS_ORIGINS_LIST) def _requires_browser_origin_check(request: Request) -> bool: - return request.method.upper() in STATE_CHANGING_API_METHODS and request.url.path.startswith( - ("/api/", "/dav/") + return ( + request.method.upper() in STATE_CHANGING_API_METHODS + and request.url.path.startswith("/api/") ) @@ -151,22 +152,20 @@ async def reject_cross_site_state_changing_api_requests(request: Request, call_n ) raw_origin = request.headers.get("origin") - raw_referer = request.headers.get("referer") - if not fetch_site and raw_origin is None and raw_referer is None: - return await call_next(request) origin = _normalized_origin(raw_origin) if raw_origin is not None and origin is None: return JSONResponse( status_code=403, content={"error_code": "csrf_origin_rejected"}, ) - if origin is not None and not _is_trusted_browser_origin(origin): + if not _is_trusted_browser_origin(origin): return JSONResponse( status_code=403, content={"error_code": "csrf_origin_rejected"}, ) if origin is None: + raw_referer = request.headers.get("referer") referer_origin = _origin_from_referer(raw_referer) if raw_referer is not None and referer_origin is None: return JSONResponse( diff --git a/backend/services/text_safety.py b/backend/services/text_safety.py index e7a03305f..451b8ca29 100644 --- a/backend/services/text_safety.py +++ b/backend/services/text_safety.py @@ -452,9 +452,9 @@ def strip_html_markup(value: str) -> str: decoded = _decode_entities(value) masked, placeholders = _mask_angle_emails(decoded) - # Python's HTMLParser can expose the tail of the malformed ```` opener - # as data on some versions. Make that opener a real ignored comment instead - # of deleting every legitimate ``-->`` sequence from the final text. + # HTMLParser can expose the tail of the malformed ```` opener as + # literal data. Normalize that opener into an ignored comment boundary + # without deleting legitimate ``-->`` text elsewhere in user content. masked = masked.replace("", " as text" + + @pytest.mark.parametrize( "safe_text", [ @@ -35,10 +39,6 @@ def test_strip_html_markup_preserves_safe_angle_bracket_display_text(safe_text): assert strip_html_markup(safe_text) == safe_text -def test_strip_html_markup_preserves_literal_comment_close_text(): - assert strip_html_markup("

literal --> content

") == "literal --> content" - - def test_strip_html_markup_plain_text_fast_path_skips_parser(monkeypatch): class ParserThatShouldNotRun: def __init__(self): From 32b00ce304a39af35f7c20980e965ad369461062 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 19 Aug 2026 15:41:42 +0900 Subject: [PATCH 23/29] fix: keep DAV protocol route out of OpenAPI --- backend/api/dav.py | 1 + 1 file changed, 1 insertion(+) diff --git a/backend/api/dav.py b/backend/api/dav.py index d618c25f3..b20ec7c0e 100644 --- a/backend/api/dav.py +++ b/backend/api/dav.py @@ -159,6 +159,7 @@ async def _handle_project_propfind( @router.api_route( "/{path:path}", methods=["PROPFIND", "REPORT", "MKCOL", "GET", "PUT", "DELETE", "OPTIONS"], + include_in_schema=False, ) async def dav_handler( request: Request, From 5c5803d41a10c502f7985d5df321e72df34ae799 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 19 Aug 2026 17:11:21 +0900 Subject: [PATCH 24/29] docs: record pg-erd DBML validation --- docs/architecture/disksage-file-lineage-handoff.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/docs/architecture/disksage-file-lineage-handoff.md b/docs/architecture/disksage-file-lineage-handoff.md index 1108f1e13..d49fc23da 100644 --- a/docs/architecture/disksage-file-lineage-handoff.md +++ b/docs/architecture/disksage-file-lineage-handoff.md @@ -42,7 +42,10 @@ The path-free semantic catalog candidate batch is the handoff to semantic-data-portal. The durable table has an explicit DBML projection at `docs/architecture/erd/disksage-file-lineage.dbml`; pg-erd-cloud can convert it to the same snapshot shape as a live database without granting it mutation -authority. The Alembic model remains the runtime source of truth. +authority. The projection was validated against pg-erd-cloud's DBML converter; +default and simple index preservation is tracked in +[pg-erd-cloud#931](https://github.com/ContextualWisdomLab/pg-erd-cloud/pull/931). +The Alembic model remains the runtime source of truth. ## Consequences From 87c54dbb473192993de80719384294c90b48dfda Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 19 Aug 2026 23:11:49 +0900 Subject: [PATCH 25/29] feat: persist DiskSage organization lineage --- .../0022_disksage_organization_lineage.py | 51 ++++++ backend/api/disksage.py | 131 +++++++++++++++- backend/db/models.py | 46 ++++++ .../services/disksage_organization_lineage.py | 89 +++++++++++ .../test_disksage_organization_lineage.py | 145 ++++++++++++++++++ .../disksage-file-lineage-handoff.md | 9 ++ .../erd/disksage-file-lineage.dbml | 21 +++ 7 files changed, 491 insertions(+), 1 deletion(-) create mode 100644 backend/alembic/versions/0022_disksage_organization_lineage.py create mode 100644 backend/services/disksage_organization_lineage.py create mode 100644 backend/tests/test_disksage_organization_lineage.py diff --git a/backend/alembic/versions/0022_disksage_organization_lineage.py b/backend/alembic/versions/0022_disksage_organization_lineage.py new file mode 100644 index 000000000..6df7adeee --- /dev/null +++ b/backend/alembic/versions/0022_disksage_organization_lineage.py @@ -0,0 +1,51 @@ +"""persist path-free DiskSage organization lineage batches""" + +from alembic import op +import sqlalchemy as sa + + +revision = "0022_disksage_org_lineage" +down_revision = "0021_disksage_scope_idx" +branch_labels = None +depends_on = None + +_TABLE = "disksage_organization_lineage_records" + + +def upgrade() -> None: + inspector = sa.inspect(op.get_bind()) + if inspector.has_table(_TABLE): + return + op.create_table( + _TABLE, + sa.Column("organization_lineage_record_uid", sa.String(length=96), nullable=False), + sa.Column("user_id", sa.String(), nullable=False), + sa.Column("organization_id", sa.String(), nullable=True), + sa.Column("workspace_id", sa.String(), nullable=False), + sa.Column("batch_fingerprint_sha256", sa.String(length=64), nullable=False), + sa.Column("envelope_sha256", sa.String(length=64), nullable=False), + sa.Column("schema_version", sa.Integer(), nullable=False), + sa.Column("item_count", sa.Integer(), nullable=False), + sa.Column("ontology_classes", sa.JSON(), nullable=False), + sa.Column("envelope_json_encrypted", sa.String(), nullable=False), + sa.Column("created_at", sa.DateTime(timezone=True), nullable=False), + sa.PrimaryKeyConstraint("organization_lineage_record_uid"), + sa.UniqueConstraint( + "user_id", + "workspace_id", + "batch_fingerprint_sha256", + name="uq_disksage_org_lineage_workspace_fingerprint", + ), + ) + op.create_index( + "ix_disksage_org_lineage_scope_time", + _TABLE, + ["user_id", "workspace_id", "created_at"], + if_not_exists=True, + ) + + +def downgrade() -> None: + op.drop_index("ix_disksage_org_lineage_scope_time", table_name=_TABLE, if_exists=True) + if sa.inspect(op.get_bind()).has_table(_TABLE): + op.drop_table(_TABLE) diff --git a/backend/api/disksage.py b/backend/api/disksage.py index 6cb52db5f..776dc408f 100644 --- a/backend/api/disksage.py +++ b/backend/api/disksage.py @@ -7,7 +7,10 @@ from sqlalchemy.exc import IntegrityError from api.auth import AuthContext, get_auth_context -from db.models import DiskSageFileLineageRecord +from db.models import ( + DiskSageFileLineageRecord, + DiskSageOrganizationLineageRecord, +) from db.session import get_db from services.disksage_file_lineage import ( FileLineageEnvelope, @@ -16,6 +19,12 @@ canonical_envelope_sha256, ontology_predicates, ) +from services.disksage_organization_lineage import ( + OrganizationLineageBatch, + OrganizationLineageSummary, + canonical_batch_json, + canonical_batch_sha256, +) from core.runtime_secrets import EncryptionKeyMissingError router = APIRouter(prefix="/api/disksage", tags=["disksage"]) @@ -58,6 +67,32 @@ def _lineage_scope(auth_context: AuthContext) -> tuple[object, ...]: ) +def _organization_lineage_scope(auth_context: AuthContext) -> tuple[object, ...]: + organization_filter = ( + DiskSageOrganizationLineageRecord.organization_id == auth_context.organization_id + if auth_context.organization_id is not None + else DiskSageOrganizationLineageRecord.organization_id.is_(None) + ) + return ( + DiskSageOrganizationLineageRecord.user_id == auth_context.user_id, + organization_filter, + DiskSageOrganizationLineageRecord.workspace_id == auth_context.workspace_id, + ) + + +def _organization_summary( + record: DiskSageOrganizationLineageRecord, +) -> OrganizationLineageSummary: + return OrganizationLineageSummary( + organization_lineage_record_uid=record.organization_lineage_record_uid, + batch_fingerprint_sha256=record.batch_fingerprint_sha256, + schema_version=record.schema_version, + item_count=record.item_count, + ontology_classes=list(record.ontology_classes or []), + created_at=record.created_at.isoformat(), + ) + + @router.post("/file-lineage", response_model=FileLineageSummary, status_code=201) async def ingest_file_lineage( envelope: FileLineageEnvelope, @@ -153,3 +188,97 @@ async def list_file_lineage( .limit(limit) ) return [_summary(record) for record in result.scalars().all()] + + +@router.post( + "/organization-lineage", + response_model=OrganizationLineageSummary, + status_code=201, +) +async def ingest_organization_lineage( + envelope: OrganizationLineageBatch, + auth_context: AuthContext = Depends(get_auth_context), + db: AsyncSession = Depends(get_db), +) -> OrganizationLineageSummary: + """Persist a path-free ontology organization plan without authorizing moves.""" + + envelope_sha256 = canonical_batch_sha256(envelope) + batch_fingerprint = envelope.batch_fingerprint_sha256 + existing_result = await db.execute( + select(DiskSageOrganizationLineageRecord).where( + *_organization_lineage_scope(auth_context), + DiskSageOrganizationLineageRecord.batch_fingerprint_sha256 + == batch_fingerprint, + ) + ) + existing = existing_result.scalar_one_or_none() + if existing is not None: + if existing.envelope_sha256 != envelope_sha256: + raise HTTPException( + status_code=409, + detail="organization lineage fingerprint is already bound to different evidence", + ) + return _organization_summary(existing) + + try: + record = DiskSageOrganizationLineageRecord( + organization_lineage_record_uid=f"disksage_org_lineage_{uuid.uuid4().hex}", + user_id=auth_context.user_id, + organization_id=auth_context.organization_id, + workspace_id=auth_context.workspace_id, + batch_fingerprint_sha256=batch_fingerprint, + envelope_sha256=envelope_sha256, + schema_version=envelope.version, + item_count=len(envelope.items), + ontology_classes=sorted({item.ontology_class for item in envelope.items}), + envelope_json_encrypted=canonical_batch_json(envelope), + created_at=datetime.datetime.now(datetime.timezone.utc), + ) + db.add(record) + await db.commit() + await db.refresh(record) + except IntegrityError: + await db.rollback() + replayed_result = await db.execute( + select(DiskSageOrganizationLineageRecord).where( + *_organization_lineage_scope(auth_context), + DiskSageOrganizationLineageRecord.batch_fingerprint_sha256 + == batch_fingerprint, + ) + ) + replayed = replayed_result.scalar_one_or_none() + if replayed is None: + raise + if replayed.envelope_sha256 != envelope_sha256: + raise HTTPException( + status_code=409, + detail="organization lineage fingerprint is already bound to different evidence", + ) from None + return _organization_summary(replayed) + except EncryptionKeyMissingError as error: + await db.rollback() + raise HTTPException( + status_code=503, + detail="Server encryption key is not configured. Contact your workspace administrator.", + ) from error + return _organization_summary(record) + + +@router.get( + "/organization-lineage", + response_model=list[OrganizationLineageSummary], +) +async def list_organization_lineage( + limit: int = Query(default=50, ge=1, le=100), + auth_context: AuthContext = Depends(get_auth_context), + db: AsyncSession = Depends(get_db), +) -> list[OrganizationLineageSummary]: + """List redacted organization lineage summaries; paths remain encrypted.""" + + result = await db.execute( + select(DiskSageOrganizationLineageRecord) + .where(*_organization_lineage_scope(auth_context)) + .order_by(DiskSageOrganizationLineageRecord.created_at.desc()) + .limit(limit) + ) + return [_organization_summary(record) for record in result.scalars().all()] diff --git a/backend/db/models.py b/backend/db/models.py index c47b10ffb..2db34ddb7 100644 --- a/backend/db/models.py +++ b/backend/db/models.py @@ -1163,6 +1163,52 @@ class DiskSageFileLineageRecord(Base): ) +class DiskSageOrganizationLineageRecord(Base): + """Encrypted, path-free local organization lineage scoped to a workspace.""" + + __tablename__ = "disksage_organization_lineage_records" + __table_args__ = ( + UniqueConstraint( + "user_id", + "workspace_id", + "batch_fingerprint_sha256", + name="uq_disksage_org_lineage_workspace_fingerprint", + ), + Index( + "ix_disksage_org_lineage_scope_time", + "user_id", + "workspace_id", + "created_at", + ), + ) + + organization_lineage_record_uid: Mapped[str] = mapped_column( + String(96), primary_key=True + ) + user_id: Mapped[str] = mapped_column(String, index=True, nullable=False) + organization_id: Mapped[str | None] = mapped_column( + String, index=True, nullable=True + ) + workspace_id: Mapped[str] = mapped_column(String, index=True, nullable=False) + batch_fingerprint_sha256: Mapped[str] = mapped_column( + String(64), nullable=False + ) + envelope_sha256: Mapped[str] = mapped_column(String(64), nullable=False) + schema_version: Mapped[int] = mapped_column(Integer, nullable=False) + item_count: Mapped[int] = mapped_column(Integer, nullable=False) + ontology_classes: Mapped[list[str]] = mapped_column( + JSON, default=list, nullable=False + ) + envelope_json_encrypted: Mapped[str] = mapped_column( + EncryptedString, nullable=False + ) + created_at: Mapped[datetime.datetime] = mapped_column( + DateTime(timezone=True), + default=lambda: datetime.datetime.now(datetime.timezone.utc), + nullable=False, + ) + + class ProjectGraphObjectRecord(Base): __tablename__ = "project_graph_objects" __table_args__ = ( diff --git a/backend/services/disksage_organization_lineage.py b/backend/services/disksage_organization_lineage.py new file mode 100644 index 000000000..feca8f302 --- /dev/null +++ b/backend/services/disksage_organization_lineage.py @@ -0,0 +1,89 @@ +"""Strict validation for DiskSage's path-free local organization handoff.""" + +from __future__ import annotations + +import hashlib +import json +from typing import Literal + +from pydantic import BaseModel, ConfigDict, Field, model_validator + + +HEX64_PATTERN = r"^[0-9a-f]{64}$" +ONTOLOGY_CLASS_PATTERN = ( + r"^https://disksage\.app/ontology#[A-Za-z][A-Za-z0-9_-]{0,127}$" +) +PRODUCTION_SOURCES = ( + "embedded:", + "filename:path-token", + "filesystem:created", + "filesystem:modified-fallback", +) + + +class _StrictModel(BaseModel): + model_config = ConfigDict(extra="forbid") + + +class OrganizationLineageItem(_StrictModel): + lineage_fingerprint: str = Field(pattern=HEX64_PATTERN) + source_size: int = Field(ge=0) + source_mtime_ms: int = Field(ge=0) + production_time_ms: int = Field(gt=0) + production_time_source: str = Field(min_length=1, max_length=256) + production_time_confidence: Literal["high", "medium", "low", "unknown"] + ontology_class: str = Field(pattern=ONTOLOGY_CLASS_PATTERN) + destination_relation: Literal["targetFolder"] + action: Literal["move"] + + @model_validator(mode="after") + def validate_production_source(self) -> OrganizationLineageItem: + if not any( + self.production_time_source == source + or self.production_time_source.startswith(source) + for source in PRODUCTION_SOURCES + ): + raise ValueError("organization lineage production source is unsupported") + if not self.production_time_source.isprintable(): + raise ValueError("organization lineage production source contains control characters") + return self + + +class OrganizationLineageBatch(_StrictModel): + schema_kind: Literal["disksage.organization-lineage-batch"] = Field(alias="schema") + version: Literal[1] + generated_at_ms: int = Field(gt=0) + complete: Literal[True] + batch_fingerprint_sha256: str = Field(pattern=HEX64_PATTERN) + items: list[OrganizationLineageItem] = Field(min_length=1, max_length=200) + + @model_validator(mode="after") + def validate_batch(self) -> OrganizationLineageBatch: + fingerprints = [item.lineage_fingerprint for item in self.items] + if len(set(fingerprints)) != len(fingerprints): + raise ValueError("organization lineage fingerprints must be unique") + if self.generated_at_ms > 253_402_300_799_999: + raise ValueError("organization lineage generated time is out of bounds") + return self + + +class OrganizationLineageSummary(_StrictModel): + organization_lineage_record_uid: str + batch_fingerprint_sha256: str + schema_version: int + item_count: int + ontology_classes: list[str] + created_at: str + + +def canonical_batch_json(batch: OrganizationLineageBatch) -> str: + return json.dumps( + batch.model_dump(mode="json", by_alias=True), + ensure_ascii=False, + sort_keys=True, + separators=(",", ":"), + ) + + +def canonical_batch_sha256(batch: OrganizationLineageBatch) -> str: + return hashlib.sha256(canonical_batch_json(batch).encode("utf-8")).hexdigest() diff --git a/backend/tests/test_disksage_organization_lineage.py b/backend/tests/test_disksage_organization_lineage.py new file mode 100644 index 000000000..220f6a987 --- /dev/null +++ b/backend/tests/test_disksage_organization_lineage.py @@ -0,0 +1,145 @@ +import datetime +from types import SimpleNamespace + +import pytest +from pydantic import ValidationError + +from api.auth import AuthContext +from api.disksage import ingest_organization_lineage, list_organization_lineage +from services.disksage_organization_lineage import ( + OrganizationLineageBatch, + canonical_batch_sha256, +) + + +def _batch() -> dict[str, object]: + return { + "schema": "disksage.organization-lineage-batch", + "version": 1, + "generated_at_ms": 1_000, + "complete": True, + "batch_fingerprint_sha256": "a" * 64, + "items": [ + { + "lineage_fingerprint": "b" * 64, + "source_size": 42, + "source_mtime_ms": 123, + "production_time_ms": 456, + "production_time_source": "embedded:exiftool:MediaCreateDate", + "production_time_confidence": "high", + "ontology_class": "https://disksage.app/ontology#Media", + "destination_relation": "targetFolder", + "action": "move", + } + ], + } + + +class _Result: + def __init__(self, record=None, records=()): + self._record = record + self._records = tuple(records) + + def scalar_one_or_none(self): + return self._record + + def scalars(self): + return self + + def all(self): + return list(self._records) + + +class _Session: + def __init__(self, results): + self._results = iter(results) + self.statements = [] + self.commit_count = 0 + + async def execute(self, statement): + self.statements.append(statement) + return next(self._results) + + def add(self, _record): + pass + + async def commit(self): + self.commit_count += 1 + + async def refresh(self, _record): + pass + + async def rollback(self): + pass + + +def _auth() -> AuthContext: + return AuthContext( + user_id="user-1", + role="member", + organization_id="org-1", + group_ids=(), + workspace_id="workspace-1", + ) + + +def _record(**overrides): + batch = OrganizationLineageBatch.model_validate(_batch()) + values = { + "organization_lineage_record_uid": "disksage_org_lineage_1", + "batch_fingerprint_sha256": batch.batch_fingerprint_sha256, + "envelope_sha256": canonical_batch_sha256(batch), + "schema_version": 1, + "item_count": 1, + "ontology_classes": ["https://disksage.app/ontology#Media"], + "created_at": datetime.datetime.now(datetime.timezone.utc), + } + values.update(overrides) + return SimpleNamespace(**values) + + +def test_contract_is_path_free_and_rejects_unknown_fields(): + envelope = OrganizationLineageBatch.model_validate(_batch()) + assert envelope.items[0].ontology_class.endswith("#Media") + unsafe = _batch() + unsafe["source_path"] = "/private/source/secret.mov" + with pytest.raises(ValidationError): + OrganizationLineageBatch.model_validate(unsafe) + + +def test_contract_rejects_duplicate_lineage_fingerprints(): + unsafe = _batch() + unsafe["items"] = [unsafe["items"][0], unsafe["items"][0]] + with pytest.raises(ValidationError): + OrganizationLineageBatch.model_validate(unsafe) + + +@pytest.mark.asyncio +async def test_replayed_batch_is_idempotent_and_workspace_scoped(): + envelope = OrganizationLineageBatch.model_validate(_batch()) + record = _record() + session = _Session([_Result(record)]) + + result = await ingest_organization_lineage( + envelope=envelope, + auth_context=_auth(), + db=session, + ) + + assert result.organization_lineage_record_uid == record.organization_lineage_record_uid + assert session.commit_count == 0 + statement = str(session.statements[0]) + assert "workspace_id" in statement + assert "user_id" in statement + assert "organization_id" in statement + + +@pytest.mark.asyncio +async def test_list_returns_redacted_summaries_only(): + session = _Session([_Result(records=[_record()])]) + + result = await list_organization_lineage(limit=50, auth_context=_auth(), db=session) + + assert len(result) == 1 + assert result[0].ontology_classes == ["https://disksage.app/ontology#Media"] + assert "envelope_json_encrypted" not in result[0].model_dump() diff --git a/docs/architecture/disksage-file-lineage-handoff.md b/docs/architecture/disksage-file-lineage-handoff.md index d49fc23da..6291d6170 100644 --- a/docs/architecture/disksage-file-lineage-handoff.md +++ b/docs/architecture/disksage-file-lineage-handoff.md @@ -47,6 +47,15 @@ default and simple index preservation is tracked in [pg-erd-cloud#931](https://github.com/ContextualWisdomLab/pg-erd-cloud/pull/931). The Alembic model remains the runtime source of truth. +Local ontology organization uses a separate path-free contract: +`disksage.organization-lineage-batch` version 1 through +`POST /api/disksage/organization-lineage`. It records only immutable lineage +fingerprints, file size/mtime, metadata-first production time, ontology class, +the `targetFolder` relation, and the planned `move` action. Absolute paths, +names, content, provider credentials, and OAuth tokens are never part of this +payload. The endpoint stores the batch encrypted and returns only a scoped +summary; it is an audit/catalog handoff and never executes a move. + ## Consequences - Naruon can index provenance and ontology predicates without authorizing source diff --git a/docs/architecture/erd/disksage-file-lineage.dbml b/docs/architecture/erd/disksage-file-lineage.dbml index bbb56ff57..ce879990c 100644 --- a/docs/architecture/erd/disksage-file-lineage.dbml +++ b/docs/architecture/erd/disksage-file-lineage.dbml @@ -29,3 +29,24 @@ Table public.disksage_file_lineage_records { (workspace_id, ontology_class) } } + +// Path-free local organization handoff. The encrypted payload contains no +// queryable source/destination paths or file names. +Table public.disksage_organization_lineage_records { + organization_lineage_record_uid varchar(96) [pk] + user_id varchar [not null] + organization_id varchar + workspace_id varchar [not null] + batch_fingerprint_sha256 varchar(64) [not null] + envelope_sha256 varchar(64) [not null] + schema_version integer [not null] + item_count integer [not null] + ontology_classes json [not null] + envelope_json_encrypted text [not null] + created_at timestamptz [not null] + + indexes { + (user_id, workspace_id, batch_fingerprint_sha256) [unique] + (user_id, workspace_id, created_at) + } +} From 34b0cafef6d029b749a695047bf05f447e8450dc Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 20 Aug 2026 10:20:39 +0900 Subject: [PATCH 26/29] ci: bound Playwright browser setup and smoke --- .github/workflows/app-ci.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/workflows/app-ci.yml b/.github/workflows/app-ci.yml index e8f445748..c3ce0fb39 100644 --- a/.github/workflows/app-ci.yml +++ b/.github/workflows/app-ci.yml @@ -129,9 +129,14 @@ jobs: cd frontend && pnpm run build - name: Install Playwright Chromium - run: cd frontend && pnpm exec playwright install --with-deps chromium + timeout-minutes: 10 + # The hosted Ubuntu runner already provides the system libraries needed by + # Chromium. Installing them again with --with-deps can block on apt for + # hours and turn a real frontend result into a cancelled check. + run: cd frontend && pnpm exec playwright install chromium - name: Run full product smoke + timeout-minutes: 15 env: NEXT_TELEMETRY_DISABLED: "1" NARUON_FULL_PRODUCT_BASE_URL: "http://127.0.0.1:3001" From 298a3f2fee76520dd9de04f4518aa6c129131c8d Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Tue, 25 Aug 2026 14:50:03 +0900 Subject: [PATCH 27/29] fix(disksage): map wrapped encryption key failures to 503 --- backend/api/disksage.py | 19 +++++++++++--- .../tests/test_disksage_file_lineage_api.py | 25 +++++++++++++++++++ 2 files changed, 41 insertions(+), 3 deletions(-) diff --git a/backend/api/disksage.py b/backend/api/disksage.py index 776dc408f..9da0e79bd 100644 --- a/backend/api/disksage.py +++ b/backend/api/disksage.py @@ -4,7 +4,7 @@ from fastapi import APIRouter, Depends, HTTPException, Query from sqlalchemy import select from sqlalchemy.ext.asyncio import AsyncSession -from sqlalchemy.exc import IntegrityError +from sqlalchemy.exc import IntegrityError, StatementError from api.auth import AuthContext, get_auth_context from db.models import ( @@ -93,6 +93,15 @@ def _organization_summary( ) +def _is_missing_encryption_key(error: BaseException) -> bool: + """Recognize direct and SQLAlchemy-wrapped missing-key failures.""" + + return isinstance(error, EncryptionKeyMissingError) or ( + isinstance(error, StatementError) + and isinstance(error.orig, EncryptionKeyMissingError) + ) + + @router.post("/file-lineage", response_model=FileLineageSummary, status_code=201) async def ingest_file_lineage( envelope: FileLineageEnvelope, @@ -162,7 +171,9 @@ async def ingest_file_lineage( detail="lineage fingerprint is already bound to different evidence", ) from None return _summary(replayed) - except EncryptionKeyMissingError as error: + except (EncryptionKeyMissingError, StatementError) as error: + if not _is_missing_encryption_key(error): + raise await db.rollback() raise HTTPException( status_code=503, @@ -255,7 +266,9 @@ async def ingest_organization_lineage( detail="organization lineage fingerprint is already bound to different evidence", ) from None return _organization_summary(replayed) - except EncryptionKeyMissingError as error: + except (EncryptionKeyMissingError, StatementError) as error: + if not _is_missing_encryption_key(error): + raise await db.rollback() raise HTTPException( status_code=503, diff --git a/backend/tests/test_disksage_file_lineage_api.py b/backend/tests/test_disksage_file_lineage_api.py index c0b824377..6a1a2ee15 100644 --- a/backend/tests/test_disksage_file_lineage_api.py +++ b/backend/tests/test_disksage_file_lineage_api.py @@ -3,6 +3,7 @@ import pytest from fastapi import HTTPException +from sqlalchemy.exc import StatementError from api.auth import AuthContext from api.disksage import ingest_file_lineage, list_file_lineage @@ -198,6 +199,30 @@ async def test_missing_encryption_key_rolls_back_as_service_unavailable(): assert session.rollback_count == 1 +@pytest.mark.asyncio +async def test_wrapped_missing_encryption_key_rolls_back_as_service_unavailable(): + envelope = FileLineageEnvelope.model_validate(_envelope()) + session = _Session( + [_Result(None)], + commit_error=StatementError( + "encrypted bind failed", + None, + None, + EncryptionKeyMissingError("missing active key"), + ), + ) + + with pytest.raises(HTTPException) as error: + await ingest_file_lineage( + envelope=envelope, + auth_context=_auth(), + db=session, + ) + + assert error.value.status_code == 503 + assert session.rollback_count == 1 + + @pytest.mark.asyncio async def test_list_lineage_query_is_tenant_scoped(): session = _Session([_Result(records=[_record()])]) From b130cc7aa66645bd444a382a4f33d8da8cd0b58d Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Tue, 25 Aug 2026 14:51:34 +0900 Subject: [PATCH 28/29] fix(disksage): reject cross-organization lineage races --- backend/api/disksage.py | 16 ++++++++++- .../tests/test_disksage_file_lineage_api.py | 27 ++++++++++++++++++- 2 files changed, 41 insertions(+), 2 deletions(-) diff --git a/backend/api/disksage.py b/backend/api/disksage.py index 9da0e79bd..2fd3b005f 100644 --- a/backend/api/disksage.py +++ b/backend/api/disksage.py @@ -67,6 +67,15 @@ def _lineage_scope(auth_context: AuthContext) -> tuple[object, ...]: ) +def _lineage_identity_scope(auth_context: AuthContext) -> tuple[object, ...]: + """Return the columns covered by the lineage uniqueness constraint.""" + + return ( + DiskSageFileLineageRecord.user_id == auth_context.user_id, + DiskSageFileLineageRecord.workspace_id == auth_context.workspace_id, + ) + + def _organization_lineage_scope(auth_context: AuthContext) -> tuple[object, ...]: organization_filter = ( DiskSageOrganizationLineageRecord.organization_id == auth_context.organization_id @@ -158,13 +167,18 @@ async def ingest_file_lineage( await db.rollback() replayed_result = await db.execute( select(DiskSageFileLineageRecord).where( - *_lineage_scope(auth_context), + *_lineage_identity_scope(auth_context), DiskSageFileLineageRecord.lineage_fingerprint == lineage_fingerprint, ) ) replayed = replayed_result.scalar_one_or_none() if replayed is None: raise + if replayed.organization_id != auth_context.organization_id: + raise HTTPException( + status_code=409, + detail="lineage fingerprint is already bound to a different organization", + ) from None if replayed.envelope_sha256 != envelope_sha256: raise HTTPException( status_code=409, diff --git a/backend/tests/test_disksage_file_lineage_api.py b/backend/tests/test_disksage_file_lineage_api.py index 6a1a2ee15..3005252dd 100644 --- a/backend/tests/test_disksage_file_lineage_api.py +++ b/backend/tests/test_disksage_file_lineage_api.py @@ -3,7 +3,7 @@ import pytest from fastapi import HTTPException -from sqlalchemy.exc import StatementError +from sqlalchemy.exc import IntegrityError, StatementError from api.auth import AuthContext from api.disksage import ingest_file_lineage, list_file_lineage @@ -126,6 +126,7 @@ def _auth() -> AuthContext: def _record(**overrides): values = { "lineage_record_uid": "disksage_lineage_1", + "organization_id": "org-1", "lineage_fingerprint": "f" * 64, "schema_version": 1, "source_kind": "file", @@ -223,6 +224,30 @@ async def test_wrapped_missing_encryption_key_rolls_back_as_service_unavailable( assert session.rollback_count == 1 +@pytest.mark.asyncio +async def test_racing_fingerprint_in_another_organization_returns_conflict(): + envelope = FileLineageEnvelope.model_validate(_envelope()) + record = _record( + envelope_sha256=canonical_envelope_sha256(envelope), + organization_id="org-2", + ) + session = _Session( + [_Result(None), _Result(record)], + commit_error=IntegrityError("unique violation", None, Exception("duplicate")), + ) + + with pytest.raises(HTTPException) as error: + await ingest_file_lineage( + envelope=envelope, + auth_context=_auth(), + db=session, + ) + + assert error.value.status_code == 409 + assert "different organization" in error.value.detail + assert session.rollback_count == 1 + + @pytest.mark.asyncio async def test_list_lineage_query_is_tenant_scoped(): session = _Session([_Result(records=[_record()])]) From dd947ed050652264c4b74d4e59ee081895d96a0a Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Tue, 25 Aug 2026 15:20:00 +0900 Subject: [PATCH 29/29] fix(disksage): handle cross-organization batch races --- backend/api/disksage.py | 16 ++++++++- .../test_disksage_organization_lineage.py | 36 +++++++++++++++++-- 2 files changed, 49 insertions(+), 3 deletions(-) diff --git a/backend/api/disksage.py b/backend/api/disksage.py index 2fd3b005f..f07451b20 100644 --- a/backend/api/disksage.py +++ b/backend/api/disksage.py @@ -89,6 +89,15 @@ def _organization_lineage_scope(auth_context: AuthContext) -> tuple[object, ...] ) +def _organization_lineage_identity_scope(auth_context: AuthContext) -> tuple[object, ...]: + """Return the columns covered by the organization-lineage uniqueness constraint.""" + + return ( + DiskSageOrganizationLineageRecord.user_id == auth_context.user_id, + DiskSageOrganizationLineageRecord.workspace_id == auth_context.workspace_id, + ) + + def _organization_summary( record: DiskSageOrganizationLineageRecord, ) -> OrganizationLineageSummary: @@ -266,7 +275,7 @@ async def ingest_organization_lineage( await db.rollback() replayed_result = await db.execute( select(DiskSageOrganizationLineageRecord).where( - *_organization_lineage_scope(auth_context), + *_organization_lineage_identity_scope(auth_context), DiskSageOrganizationLineageRecord.batch_fingerprint_sha256 == batch_fingerprint, ) @@ -274,6 +283,11 @@ async def ingest_organization_lineage( replayed = replayed_result.scalar_one_or_none() if replayed is None: raise + if replayed.organization_id != auth_context.organization_id: + raise HTTPException( + status_code=409, + detail="organization lineage fingerprint is already bound to a different organization", + ) from None if replayed.envelope_sha256 != envelope_sha256: raise HTTPException( status_code=409, diff --git a/backend/tests/test_disksage_organization_lineage.py b/backend/tests/test_disksage_organization_lineage.py index 220f6a987..c4425b9db 100644 --- a/backend/tests/test_disksage_organization_lineage.py +++ b/backend/tests/test_disksage_organization_lineage.py @@ -2,7 +2,9 @@ from types import SimpleNamespace import pytest +from fastapi import HTTPException from pydantic import ValidationError +from sqlalchemy.exc import IntegrityError from api.auth import AuthContext from api.disksage import ingest_organization_lineage, list_organization_lineage @@ -51,10 +53,12 @@ def all(self): class _Session: - def __init__(self, results): + def __init__(self, results, *, commit_error=None): self._results = iter(results) + self.commit_error = commit_error self.statements = [] self.commit_count = 0 + self.rollback_count = 0 async def execute(self, statement): self.statements.append(statement) @@ -65,12 +69,14 @@ def add(self, _record): async def commit(self): self.commit_count += 1 + if self.commit_error is not None: + raise self.commit_error async def refresh(self, _record): pass async def rollback(self): - pass + self.rollback_count += 1 def _auth() -> AuthContext: @@ -87,6 +93,7 @@ def _record(**overrides): batch = OrganizationLineageBatch.model_validate(_batch()) values = { "organization_lineage_record_uid": "disksage_org_lineage_1", + "organization_id": "org-1", "batch_fingerprint_sha256": batch.batch_fingerprint_sha256, "envelope_sha256": canonical_batch_sha256(batch), "schema_version": 1, @@ -114,6 +121,31 @@ def test_contract_rejects_duplicate_lineage_fingerprints(): OrganizationLineageBatch.model_validate(unsafe) +@pytest.mark.asyncio +async def test_racing_batch_in_another_organization_returns_conflict(): + envelope = OrganizationLineageBatch.model_validate(_batch()) + record = _record( + organization_id="org-2", + envelope_sha256=canonical_batch_sha256(envelope), + ) + session = _Session( + [_Result(None), _Result(record)], + commit_error=IntegrityError("unique violation", None, Exception("duplicate")), + ) + + with pytest.raises(HTTPException) as error: + await ingest_organization_lineage( + envelope=envelope, + auth_context=_auth(), + db=session, + ) + + assert error.value.status_code == 409 + assert "different organization" in error.value.detail + assert session.rollback_count == 1 + assert "organization_id" not in str(session.statements[1]).split("WHERE", 1)[1] + + @pytest.mark.asyncio async def test_replayed_batch_is_idempotent_and_workspace_scoped(): envelope = OrganizationLineageBatch.model_validate(_batch())