fix(scan): encrypted/corrupt files → scan_failures (#828 open-core)

.gitignore

@@ -301,6 +301,13 @@ __marimo__/
 /e agora*.txt
 /*.txt
 
+# OPSEC: operator-local paths — never `git add -A` (explicit `git add <path>` only).
+# docs/ops/today-mode/ hub files already on origin stay tracked; this blocks new untracked drafts.
+docs/ops/today-mode/
+scripts/beastie-ecosystem-sync.ps1
+scripts/beastie-ecosystem-sync.sh
+api/static/mascot/bata_boar_mascot_pot.png
+
 # Generated POC test corpus (on-demand, not tracked)
 tests/synthetic_corpus/
 .cursor/rules/dossier-update-on-evidence.mdc

connectors/filesystem_connector.py

@@ -10,13 +10,15 @@
 import stat
 import tempfile
 import zipfile
-from collections.abc import Iterator
+from collections.abc import Callable, Iterator
 from pathlib import Path
 from typing import Any
 
 from core.connector_registry import register
 from core.archives import (
     ArchiveUnsupportedError,
+    SCAN_FAILURE_REASON_ENCRYPTED_NO_PASSWORD,
+    SCAN_FAILURE_REASON_WRONG_PASSWORD,
     default_compressed_extensions,
     detect_archive_type,
     iter_archive_members,
@@ -287,6 +289,7 @@ def _read_text_sample(
     ocr_max_dimension: int = 2000,
     ocr_lang: str = "eng",
     scan_for_stego: bool = False,
+    on_read_barrier: Callable[[str, str], None] | None = None,
 ) -> str:
     """Extract text from file for sensitivity scan; return empty on error. No content stored after return.
 
@@ -346,8 +349,18 @@ def finalize(text: str) -> str:
             if reader.is_encrypted:
                 password = pw.get(ext) or pw.get("default")
                 if password:
-                    reader.decrypt(password)
+                    decrypt_status = reader.decrypt(password)
+                    if decrypt_status == 0:
+                        if on_read_barrier is not None:
+                            on_read_barrier(
+                                SCAN_FAILURE_REASON_WRONG_PASSWORD, str(path)
+                            )
+                        return finalize("")
                 else:
+                    if on_read_barrier is not None:
+                        on_read_barrier(
+                            SCAN_FAILURE_REASON_ENCRYPTED_NO_PASSWORD, str(path)
+                        )
                     return finalize("")
             return finalize(
                 " ".join(p.extract_text() or "" for p in reader.pages[:5])[:max_chars]
@@ -778,6 +791,10 @@ def run(self) -> None:
                 _size = _stat.st_size
             except OSError:  # noqa: BLE001
                 pass  # stat() best-effort for file-identity metadata
+
+            def _record_read_barrier(reason: str, details: str) -> None:
+                self.db_manager.save_failure(target_name, reason, details)
+
             content = _read_text_sample(
                 file_path,
                 effective_ext,
@@ -788,6 +805,7 @@ def run(self) -> None:
                 ocr_max_dimension=self.ocr_max_dimension,
                 ocr_lang=self.ocr_lang,
                 scan_for_stego=self.scan_for_stego,
+                on_read_barrier=_record_read_barrier,
             )
             res = self.scanner.scan_file_content(content, file_path)
             if res is None:
@@ -891,12 +909,17 @@ def scan_archive_at_path(
     except (TypeError, ValueError):
         max_size = DEFAULT_MAX_INNER_SIZE
     try:
+
+        def _on_member_read_failure(reason: str, member: str, details: str) -> None:
+            db_manager.save_failure(target_name, reason, details)
+
         for member_name, data in iter_archive_members(
             archive_path,
             archive_type,
             max_size,
             extensions,
             file_passwords,
+            on_member_read_failure=_on_member_read_failure,
         ):
             ext = Path(member_name).suffix.lower()
             tmp_path: Path | None = None
@@ -910,6 +933,16 @@ def scan_archive_at_path(
                 ext_use = choose_effective_rich_media_extension(
                     ext_use, use_content_type, tmp_path
                 )
+
+                def _on_read_barrier(
+                    reason: str, details: str, *, _member: str = member_name
+                ) -> None:
+                    db_manager.save_failure(
+                        target_name,
+                        reason,
+                        f"{archive_display_name}|{_member}: {details}",
+                    )
+
                 content = _read_text_sample(
                     tmp_path,
                     ext_use,
@@ -920,6 +953,7 @@ def scan_archive_at_path(
                     ocr_max_dimension=ocr_max_dimension,
                     ocr_lang=ocr_lang,
                     scan_for_stego=scan_for_stego,
+                    on_read_barrier=_on_read_barrier,
                 )
                 res = scanner.scan_file_content(content, member_name)
                 if res is None:

core/archives.py

@@ -9,11 +9,19 @@
 from __future__ import annotations
 
 import zipfile
+from collections.abc import Callable
 from pathlib import Path
 from typing import Iterable, Iterator
 
 import tarfile
 
+# scan_failures reasons for unreadable archive members (#828; shared taxonomy with SQL sampling_error).
+SCAN_FAILURE_REASON_ENCRYPTED_NO_PASSWORD = "encrypted_no_password"
+SCAN_FAILURE_REASON_WRONG_PASSWORD = "wrong_password"
+SCAN_FAILURE_REASON_ARCHIVE_READ_ERROR = "archive_read_error"
+
+MemberReadFailureCallback = Callable[[str, str, str], None]
+
 
 # Tier 1 – stdlib-backed formats (see PLAN_COMPRESSED_FILES.md)
 _TIER1_EXTENSIONS: set[str] = {
@@ -201,12 +209,55 @@ class ArchiveUnsupportedError(Exception):
     pass
 
 
+def classify_zip_member_read_failure(
+    exc: BaseException,
+    *,
+    encrypted: bool,
+    password_provided: bool,
+) -> str:
+    """Map zipfile member read errors to scan_failures reason codes (#828)."""
+    if encrypted and not password_provided:
+        return SCAN_FAILURE_REASON_ENCRYPTED_NO_PASSWORD
+    if encrypted and password_provided:
+        msg = str(exc).lower()
+        if (
+            "bad password" in msg
+            or "incorrect password" in msg
+            or "bad crc-32" in msg
+            or "bad crc" in msg
+        ):
+            return SCAN_FAILURE_REASON_WRONG_PASSWORD
+    return SCAN_FAILURE_REASON_ARCHIVE_READ_ERROR
+
+
+def classify_7z_member_read_failure(
+    exc: BaseException,
+    *,
+    password_provided: bool,
+) -> str:
+    """Map py7zr member read errors to scan_failures reason codes (#828)."""
+    try:
+        from py7zr import exceptions as py7zr_exc
+    except ImportError:
+        py7zr_exc = None  # type: ignore[misc, assignment]
+    if py7zr_exc is not None and isinstance(exc, py7zr_exc.PasswordRequired):
+        return SCAN_FAILURE_REASON_ENCRYPTED_NO_PASSWORD
+    if password_provided:
+        if py7zr_exc is not None and isinstance(exc, py7zr_exc.CrcError):
+            return SCAN_FAILURE_REASON_WRONG_PASSWORD
+        msg = str(exc).lower()
+        if "password" in msg or "wrong" in msg or "crc" in msg:
+            return SCAN_FAILURE_REASON_WRONG_PASSWORD
+    return SCAN_FAILURE_REASON_ARCHIVE_READ_ERROR
+
+
 def iter_archive_members(
     path: Path,
     archive_type: str,
     max_inner_size: int,
     allowed_extensions: Iterable[str] | set[str],
     file_passwords: dict[str, str] | None = None,
+    on_member_read_failure: MemberReadFailureCallback | None = None,
 ) -> Iterator[tuple[str, bytes]]:
     """
     Yield (member_name, content_bytes) for each file inside the archive whose
@@ -238,7 +289,19 @@ def iter_archive_members(
                         continue
                     try:
                         data = z.read(name)
-                    except (RuntimeError, zipfile.BadZipFile):
+                    except (RuntimeError, zipfile.BadZipFile) as e:
+                        if on_member_read_failure is not None:
+                            encrypted = bool(info.flag_bits & 0x1)
+                            reason = classify_zip_member_read_failure(
+                                e,
+                                encrypted=encrypted,
+                                password_provided=bool(pwd),
+                            )
+                            on_member_read_failure(
+                                reason,
+                                name,
+                                f"{path.name}|{name}: {type(e).__name__}: {e}",
+                            )
                         continue
                     yield (name, data)
         except (zipfile.BadZipFile, OSError):
@@ -278,6 +341,8 @@ def iter_archive_members(
                 "on a host without liblzma see issue #926"
             )
         pwd = pw.get(".7z") or pw.get("default") or None
+        from py7zr.exceptions import PasswordRequired
+
         try:
             with py7zr.SevenZipFile(path, "r", password=pwd) as archive:
                 # py7zr >= 0.22: use list(); older docs used files_list (removed in 1.x).
@@ -296,8 +361,25 @@ def iter_archive_members(
                         if bio is not None:
                             data = bio.read()
                             yield (member.filename, data)
-                    except (Exception, OSError):
+                    except (Exception, OSError) as e:
+                        if on_member_read_failure is not None:
+                            reason = classify_7z_member_read_failure(
+                                e, password_provided=bool(pwd)
+                            )
+                            on_member_read_failure(
+                                reason,
+                                member.filename,
+                                f"{path.name}|{member.filename}: {type(e).__name__}: {e}",
+                            )
                         continue
+        except PasswordRequired as e:
+            if on_member_read_failure is not None:
+                on_member_read_failure(
+                    SCAN_FAILURE_REASON_ENCRYPTED_NO_PASSWORD,
+                    path.name,
+                    f"{path.name}: {type(e).__name__}: {e}",
+                )
+            return
         except (py7zr.Bad7zFile, OSError):
             return
 

core/database.py

@@ -169,6 +169,17 @@ def failure_hint(reason: str) -> str:
             "evaluated — this is not a clean result. Check dialect-specific sampling notes, credentials, "
             "and the detailed message; fix the target or sampling plan before re-running."
         )
+    if r == "encrypted_no_password":
+        return (
+            "File or archive member is encrypted and no password was configured. This is not a clean "
+            "result — add the password under file_scan.file_passwords (or the matching extension key) "
+            "and re-run, or exclude the path if decryption is out of scope."
+        )
+    if r == "wrong_password":
+        return (
+            "Password was provided but decryption failed. Verify file_scan.file_passwords for this "
+            "extension; a wrong password is not a clean scan result."
+        )
     return (
         "Unexpected error. Review the detailed message and audit log, verify the target configuration "
         "(host, port, path, credentials) and test connectivity manually before re-running."