release: bound sdist and wheel verifier member enumeration

scripts/ci/verify_distribution.py

@@ -15,6 +15,7 @@
 import os
 import re
 import stat
+import struct
 import tarfile
 import tempfile
 import zipfile
@@ -33,6 +34,17 @@
 DISTRIBUTION_NAME = "egressweave"
 HASH_CHUNK_SIZE = 1024 * 1024
 MAX_DISTRIBUTION_BYTES = 256 * 1024 * 1024
+MAX_SDIST_MEMBERS = 4096
+MAX_WHEEL_MEMBERS = 4096
+ZIP_EOCD_SIGNATURE = b"PK\x05\x06"
+ZIP64_EOCD_SIGNATURE = b"PK\x06\x06"
+ZIP64_EOCD_LOCATOR_SIGNATURE = b"PK\x06\x07"
+ZIP64_EOCD_LOCATOR_SIZE = 20
+ZIP_CENTRAL_SIGNATURE = b"PK\x01\x02"
+ZIP_EOCD = struct.Struct("<4s4H2LH")
+ZIP64_EOCD_LOCATOR = struct.Struct("<4sLQL")
+ZIP64_EOCD_PREFIX = struct.Struct("<4sQ")
+ZIP_CENTRAL_HEADER = struct.Struct("<4s6H3L5H2L")
 CHANGELOG_RELEASE_PATTERN = re.compile(
     r"^## \[(?P<version>\d+\.\d+\.\d+)\] - (?P<date>\d{4}-\d{2}-\d{2})$",
     flags=re.MULTILINE,
@@ -199,25 +211,218 @@ def _select_archives(dist_dir: Path, name: str, version: str) -> tuple[Path, Pat
     return wheel_path, sdist_path
 
 
+def _admit_archive_name(name: str, seen: set[str]) -> None:
+    """Validate and retain one archive path without building an unbounded name list."""
+    pure_path = PurePosixPath(name)
+    if (
+        not name
+        or "\\" in name
+        or pure_path.is_absolute()
+        or any(part in {"", ".", ".."} for part in pure_path.parts)
+        or name in seen
+    ):
+        raise SystemExit(f"distribution contains an unsafe archive path: {name!r}")
+    seen.add(name)
+
+
 def _safe_archive_names(names: list[str]) -> set[str]:
     """Reject absolute, parent-traversing, duplicate, or backslash archive paths."""
     seen: set[str] = set()
     for name in names:
-        pure_path = PurePosixPath(name)
+        _admit_archive_name(name, seen)
+    return seen
+
+
+def _read_exact(stream: BinaryIO, size: int, error_message: str) -> bytes:
+    """Read exactly ``size`` stable-snapshot bytes or fail closed on truncation."""
+    payload = stream.read(size)
+    if len(payload) != size:
+        raise SystemExit(error_message)
+    return payload
+
+
+def _wheel_snapshot_size(stream: BinaryIO) -> int:
+    """Return one finite stable wheel snapshot size without reopening its path."""
+    try:
+        size = stream.seek(0, os.SEEK_END)
+    except OSError:
+        raise SystemExit("wheel is not a valid ZIP archive") from None
+    if (
+        isinstance(size, bool)
+        or not isinstance(size, int)
+        or size < 0
+        or size > MAX_DISTRIBUTION_BYTES
+    ):
+        raise SystemExit("wheel is not a valid ZIP archive")
+    return size
+
+
+def _find_wheel_eocd(stream: BinaryIO) -> tuple[int, tuple[int, ...]]:
+    """Locate one canonical single-disk ZIP end record with a bounded tail read."""
+    invalid = "wheel is not a valid ZIP archive"
+    archive_size = _wheel_snapshot_size(stream)
+    tail_size = min(archive_size, ZIP_EOCD.size + 65_535)
+    stream.seek(archive_size - tail_size)
+    tail = _read_exact(stream, tail_size, invalid)
+    candidate = tail.rfind(ZIP_EOCD_SIGNATURE)
+    while candidate >= 0:
+        if candidate + ZIP_EOCD.size <= len(tail):
+            record = ZIP_EOCD.unpack_from(tail, candidate)
+            if candidate + ZIP_EOCD.size + record[-1] == len(tail):
+                return archive_size - tail_size + candidate, record[1:]
+        candidate = tail.rfind(ZIP_EOCD_SIGNATURE, 0, candidate)
+    raise SystemExit(invalid)
+
+
+def _wheel_extra_uses_zip64(extra: bytes) -> bool:
+    """Return whether a central-directory extra field declares ZIP64 data."""
+    invalid = "wheel is not a valid ZIP archive"
+    cursor = 0
+    while cursor < len(extra):
+        if cursor + 4 > len(extra):
+            raise SystemExit(invalid)
+        field_id, field_size = struct.unpack_from("<HH", extra, cursor)
+        cursor += 4
+        if cursor + field_size > len(extra):
+            raise SystemExit(invalid)
+        if field_id == 0x0001:
+            return True
+        cursor += field_size
+    return False
+
+
+def _wheel_tail_before(stream: BinaryIO, offset: int, size: int) -> bytes:
+    """Read at most ``size`` bytes immediately before one ZIP structure."""
+    start = max(0, offset - size)
+    stream.seek(start)
+    return _read_exact(
+        stream,
+        offset - start,
+        "wheel is not a valid ZIP archive",
+    )
+
+
+def _wheel_zip64_locator_is_structural(stream: BinaryIO, eocd_offset: int) -> bool:
+    """Recognize a ZIP64 locator only when its pointer frames a ZIP64 end record."""
+    if eocd_offset < ZIP64_EOCD_LOCATOR_SIZE:
+        return False
+    locator = _wheel_tail_before(stream, eocd_offset, ZIP64_EOCD_LOCATOR_SIZE)
+    if len(locator) != ZIP64_EOCD_LOCATOR_SIZE:
+        return False
+    signature, disk_number, zip64_offset, total_disks = ZIP64_EOCD_LOCATOR.unpack(locator)
+    if (
+        signature != ZIP64_EOCD_LOCATOR_SIGNATURE
+        or disk_number != 0
+        or total_disks != 1
+    ):
+        return False
+    locator_offset = eocd_offset - ZIP64_EOCD_LOCATOR_SIZE
+    if zip64_offset > locator_offset - ZIP64_EOCD_PREFIX.size:
+        return False
+    stream.seek(zip64_offset)
+    prefix = _read_exact(
+        stream,
+        ZIP64_EOCD_PREFIX.size,
+        "wheel is not a valid ZIP archive",
+    )
+    record_signature, record_size = ZIP64_EOCD_PREFIX.unpack(prefix)
+    return (
+        record_signature == ZIP64_EOCD_SIGNATURE
+        and record_size >= 44
+        and zip64_offset + ZIP64_EOCD_PREFIX.size + record_size == locator_offset
+    )
+
+
+def _preflight_wheel_members(stream: BinaryIO) -> int | None:
+    """Bound ZIP entries and return a safe stdlib-locator mask offset if needed."""
+    invalid = "wheel is not a valid ZIP archive"
+    eocd_offset, fields = _find_wheel_eocd(stream)
+    disk_number, directory_disk, disk_entries, total_entries, size, offset, _ = fields
+    if (
+        disk_number != 0
+        or directory_disk != 0
+        or disk_entries != total_entries
+        or _wheel_zip64_locator_is_structural(stream, eocd_offset)
+    ):
+        raise SystemExit(invalid)
+    if (
+        total_entries == 0xFFFF
+        or size == 0xFFFFFFFF
+        or offset == 0xFFFFFFFF
+        or offset + size != eocd_offset
+    ):
+        raise SystemExit(invalid)
+    if total_entries > MAX_WHEEL_MEMBERS:
+        raise SystemExit("wheel member limit exceeded")
+
+    stream.seek(offset)
+    consumed = 0
+    actual_entries = 0
+    locator_comment_offset: int | None = None
+    while consumed < size:
+        fixed = _read_exact(stream, ZIP_CENTRAL_HEADER.size, invalid)
+        consumed += len(fixed)
+        values = ZIP_CENTRAL_HEADER.unpack(fixed)
+        if values[0] != ZIP_CENTRAL_SIGNATURE:
+            raise SystemExit(invalid)
+        compressed_size, uncompressed_size = values[8], values[9]
+        name_size, extra_size, comment_size = values[10], values[11], values[12]
+        start_disk, local_offset = values[13], values[16]
+        variable_size = name_size + extra_size + comment_size
+        if consumed + variable_size > size:
+            raise SystemExit(invalid)
+        variable = _read_exact(stream, variable_size, invalid)
+        consumed += variable_size
+        extra = variable[name_size : name_size + extra_size]
         if (
-            not name
-            or "\\" in name
-            or pure_path.is_absolute()
-            or any(part in {"", ".", ".."} for part in pure_path.parts)
-            or name in seen
+            start_disk != 0
+            or compressed_size == 0xFFFFFFFF
+            or uncompressed_size == 0xFFFFFFFF
+            or local_offset == 0xFFFFFFFF
+            or _wheel_extra_uses_zip64(extra)
         ):
-            raise SystemExit(f"distribution contains an unsafe archive path: {name!r}")
-        seen.add(name)
-    return seen
+            raise SystemExit(invalid)
+        if (
+            consumed == size
+            and comment_size >= ZIP64_EOCD_LOCATOR_SIZE
+            and variable[-ZIP64_EOCD_LOCATOR_SIZE:].startswith(
+                ZIP64_EOCD_LOCATOR_SIGNATURE
+            )
+        ):
+            locator_comment_offset = eocd_offset - ZIP64_EOCD_LOCATOR_SIZE
+        actual_entries += 1
+        if actual_entries > MAX_WHEEL_MEMBERS:
+            raise SystemExit("wheel member limit exceeded")
+    if consumed != size or actual_entries != total_entries:
+        raise SystemExit(invalid)
+    stream.seek(0)
+    return locator_comment_offset
+
+
+def _wheel_zipfile_compatible_snapshot(stream: BinaryIO, offset: int) -> BinaryIO:
+    """Mask one validated member-comment signature only in the stdlib parser view."""
+    invalid = "wheel is not a valid ZIP archive"
+    parser_snapshot = tempfile.TemporaryFile(mode="w+b")  # noqa: SIM115
+    try:
+        stream.seek(0)
+        while block := stream.read(HASH_CHUNK_SIZE):
+            parser_snapshot.write(block)
+        parser_snapshot.seek(offset)
+        if _read_exact(parser_snapshot, 4, invalid) != ZIP64_EOCD_LOCATOR_SIGNATURE:
+            raise SystemExit(invalid)
+        parser_snapshot.seek(offset)
+        parser_snapshot.write(b"EW64")
+        parser_snapshot.seek(0)
+        return parser_snapshot
+    except BaseException:
+        parser_snapshot.close()
+        raise
+    finally:
+        stream.seek(0)
 
 
 def _verify_wheel(wheel_path: Path, project: dict[str, object]) -> str:
-    """Verify wheel contents and return the digest of the exact parsed snapshot."""
+    """Verify wheel contents after bounded central-directory member admission."""
     version = str(project["version"])
     dist_info = f"{DISTRIBUTION_NAME}-{version}.dist-info"
     required_paths = {
@@ -233,14 +438,27 @@ def _verify_wheel(wheel_path: Path, project: dict[str, object]) -> str:
     with _open_stable_distribution(wheel_path) as wheel_file:
         wheel_digest = _sha256_stream(wheel_file)
         wheel_file.seek(0)
-        with zipfile.ZipFile(wheel_file) as wheel_archive:
-            names = _safe_archive_names(wheel_archive.namelist())
-            missing = required_paths - names
-            if missing:
-                raise SystemExit(f"wheel is missing required files: {sorted(missing)}")
-            metadata = BytesParser(policy=default).parsebytes(
-                wheel_archive.read(f"{dist_info}/METADATA")
+        locator_comment_offset = _preflight_wheel_members(wheel_file)
+        parser_snapshot: BinaryIO | None = None
+        parser_stream = wheel_file
+        if locator_comment_offset is not None:
+            parser_snapshot = _wheel_zipfile_compatible_snapshot(
+                wheel_file,
+                locator_comment_offset,
             )
+            parser_stream = parser_snapshot
+        try:
+            with zipfile.ZipFile(parser_stream) as wheel_archive:
+                names = _safe_archive_names(wheel_archive.namelist())
+                missing = required_paths - names
+                if missing:
+                    raise SystemExit(f"wheel is missing required files: {sorted(missing)}")
+                metadata = BytesParser(policy=default).parsebytes(
+                    wheel_archive.read(f"{dist_info}/METADATA")
+                )
+        finally:
+            if parser_snapshot is not None:
+                parser_snapshot.close()
 
     expected_metadata = {
         "Name": str(project["name"]),
@@ -260,7 +478,7 @@ def _verify_wheel(wheel_path: Path, project: dict[str, object]) -> str:
 
 
 def _verify_sdist(sdist_path: Path, project: dict[str, object]) -> str:
-    """Verify source-distribution paths and return its parsed snapshot digest."""
+    """Verify source-distribution paths with bounded streaming member admission."""
     version = str(project["version"])
     prefix = f"{DISTRIBUTION_NAME}-{version}"
     required_paths = {
@@ -281,11 +499,17 @@ def _verify_sdist(sdist_path: Path, project: dict[str, object]) -> str:
     with _open_stable_distribution(sdist_path) as sdist_file:
         sdist_digest = _sha256_stream(sdist_file)
         sdist_file.seek(0)
-        with tarfile.open(fileobj=sdist_file, mode="r:gz") as sdist_archive:
-            members = sdist_archive.getmembers()
-            names = _safe_archive_names([member.name for member in members])
-            if any(member.issym() or member.islnk() or member.isdev() for member in members):
-                raise SystemExit("source distribution contains a link or device entry")
+        names: set[str] = set()
+        with tarfile.open(fileobj=sdist_file, mode="r|gz") as sdist_archive:
+            for member_count, member in enumerate(sdist_archive, start=1):
+                if member_count > MAX_SDIST_MEMBERS:
+                    raise SystemExit("source distribution member limit exceeded")
+                _admit_archive_name(member.name, names)
+                if member.issym() or member.islnk() or member.isdev():
+                    raise SystemExit("source distribution contains a link or device entry")
+                # Python 3.10-3.14 retain yielded TarInfo objects in this public list.
+                # Links are rejected above, so no later member may need that cache.
+                sdist_archive.members.clear()
 
     missing = required_paths - names
     if missing: