diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml new file mode 100644 index 0000000..2939cfe --- /dev/null +++ b/.github/workflows/tests.yml @@ -0,0 +1,33 @@ +name: Tests + +on: + push: + pull_request: + workflow_dispatch: + +permissions: + contents: read + +jobs: + test: + name: Python ${{ matrix.python }} on ${{ matrix.os }} + runs-on: ${{ matrix.os }} + timeout-minutes: 10 + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, windows-latest, macos-latest] + python: ['3.10', '3.14'] + steps: + - name: Check out repository + uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 + with: + persist-credentials: false + - name: Set up Python + uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6 + with: + python-version: ${{ matrix.python }} + - name: Run local-only tests + run: python -m unittest -v + - name: Check CLI help + run: python oci_runtime_inventory.py --help diff --git a/README.md b/README.md index 104ad4a..21874e7 100644 --- a/README.md +++ b/README.md @@ -1,12 +1,129 @@ # oci-runtime-inventory -Optional offline image/SBOM metadata inventory for the checker family tracked in +Standalone Python 3.10+ standard-library inventory for explicit local OCI-layout +or Docker-save archives and supported SBOM JSON. It never runs Docker, executes +an image/binary/package, extracts archive paths onto the host, follows archive +links, opens a network connection or guesses runtime versions from tags/names. +This optional supporting tool addresses [node2shell-check #12](https://github.com/completetech-security/node2shell-check/issues/12). -The implementation is prepared for review in the initial implementation PR. -No production image, customer inventory or runtime assessment is included. +No exploit checker requires it at runtime. -The tool is not a mandatory dependency of any exploit checker. The approved -implementation will remain standalone Python 3.10+, standard library only, -without image execution, archive extraction or network access. +```sh +python oci_runtime_inventory.py --archive authorized-image.tar --output image-inventory.json +python oci_runtime_inventory.py --sbom examples/inventory.sbom.json +python oci_runtime_inventory.py --archive authorized-image.tar --consumer sharp --output sharp-input.json +python oci_runtime_inventory.py --archive authorized-image.tar --consumer node2shell --output node-input.json +python oci_runtime_inventory.py --sbom examples/inventory.sbom.json --consumer flowise --output flowise-input.json +python -m unittest -v +``` -See AGENTS.md, CONTRIBUTING.md, SECURITY.md, LICENSING.md and BRAND_ASSETS.md. +The checked-in SBOM is synthetic. Real images, inventories and reports must stay +outside version control. Only read images/evidence you are authorized to inspect. +`--output` refuses existing files unless `--force` is explicit, and always protects +inputs, aliases and the executable. No installed dependencies or companion files +are required when the Python entry point is copied alone. + +## Evidence and uncertainty + +Canonical JSON schema 1.0 records supported parsing coverage, limits/usage, +artifact identities, layer digests and logical file provenance. OCI identities +are verified manifest digests; Docker-save identities are config-byte digests +(`identity_kind` distinguishes them). An OCI manifest digest binds its declared layer descriptors. A Docker-save +config digest identifies only config bytes and does not bind the archive layers: +`rootfs.diff_ids` are not validated, and identical configs with different supplied +layers retain the same config digest. Use the separately recorded ordered layer +digests when comparing observations. Neither identity establishes publisher trust +or that an archive can run. Config Env/Cmd/hostnames and image +tags are not reported. SBOM digests identify the SBOM artifact, not an image. + +The virtual filesystem applies layers in order. Whiteouts remove lower-layer +entries; opaque whiteouts retain same-layer additions regardless of TAR ordering. +Replaced/deleted manifests cannot survive as current package observations. +Links and device entries are opaque replacement nodes, never followed or created. +Metadata under a nondirectory ancestor, or a selected metadata path that is a +link/device/directory, makes image coverage incomplete. + +Exact versions come only from selected package manifests (Sharp, Flowise and +flowise-components) or Node.js package-database records (dpkg/APK). Installed +package metadata does not attest deployed runtime use, bundled/global native +linkage, vendor patches or custom builds. No filename/tag/binary-string inference +is performed. All local exploitability remains `not_tested`; no vulnerability or +safety verdict is produced. No observation is proof of absence. + +`--consumer node2shell|sharp|flowise` emits actual input contracts for those +standalone checkers. Unverified provenance/library selection is retained, so +metadata alone produces unknown assessments. Qualified/oversized/control-bearing +versions are never normalized into patch claims. Incomplete images are excluded +from consumer records; absent supported records produce unknown placeholders, +not empty evidence presented as a clean scan. Canonical output retains the full +coverage/provenance record; consumer output intentionally carries only each +checker's accepted fields. Preserve the process exit code as well as the output. + +## Supported formats and bounds + +- Uncompressed outer USTAR/V7 TAR containing OCI image-layout 1.0 or Docker-save + manifest/config/layers. Regular GNU members work when they fit basic header + fields. No host filesystem extraction is used. SHA-256 OCI manifest/config/layer + descriptors and declared blob sizes are checked; descriptor URLs are never used. +- OCI plain-TAR or gzip layers; Docker-save plain-TAR layers. Extended PAX/GNU + long-name/sparse records, zstd, foreign layers, unsupported digests, outer gzip + and other archive formats are unresolved, not silently accepted as complete. +- JSON CycloneDX 1.4/1.5/1.6, including metadata.component and nested components; + SPDX 2.2/2.3 package metadata. This reads selected metadata fields, not full SBOM + conformance or deployment/relationship attestation. +- Default 512 MiB expanded layer bytes; `--max-expanded-mib` accepts 16-4096. + Other limits: 8 GiB outer archive, 1 GiB compressed layer reads, 100,000 archive + and layer members, 64 path components/512 path characters, 16 images, 256 + layers/image, 64 index descriptors and three nested indexes, 4 MiB metadata + files, 16 MiB aggregate captured metadata, 10,000 visited SBOM component entries (including irrelevant entries) and + 10,000 selected component records per image, + 100 SBOM inputs, bounded JSON depth/node count and at most 1 MiB TAR tail padding. + +Unsupported/malformed input, integrity mismatch or any limit marks coverage +incomplete with a symbolic reason and exits 2 while preserving completed images. +No components from a failed image are emitted. Exit 0 means supported inspection +completed, not that a runtime is safe. Interruption exits 130 and preserves +completed observations before output begins when possible. Once stdout or atomic +file publication starts, it is never retried; inspect interrupted output before +running again. Ordinary regular-file/path checks assume stable evidence copies +and are not a sandbox against concurrent filesystem mutation. + +## Validation and provenance + +Tests use synthetic archives only. They cover layer replacement/deletion, +whiteout ordering, links/devices, traversal, duplicate/sparse/extended records, +truncation/checksums/digests, expansion limits, metadata privacy, standalone +execution, output aliases and interruption. Real exported consumer payloads were +also accepted by the actual Node2Shell, Sharp and Flowise CLIs with unknown +assessments. Local checks and exact-commit hosted CI are separate evidence. +GitHub Copilot reviews published changes before merge. + +Format references reviewed 2026-10-25: +[OCI layout](https://github.com/opencontainers/image-spec/blob/main/image-layout.md), +[OCI layers/whiteouts](https://github.com/opencontainers/image-spec/blob/main/layer.md), +[CycloneDX schema](https://github.com/CycloneDX/specification/blob/master/schema/bom-1.6.schema.json), +[SPDX package metadata](https://spdx.github.io/spdx-spec/v2.3.1/package-information/). +No advisory fix table is embedded: this is supporting inventory, not an exploit +checker. Consumer schemas are pinned by the documented v1 contracts; no feed or +network dependency is introduced. + +[AGENTS](AGENTS.md), [contribution terms](CONTRIBUTING.md), [security policy](SECURITY.md), +[AGPL license](LICENSE), [dual licensing](LICENSING.md), [brand policy](BRAND_ASSETS.md). + +Consumer exports are bounded by the destination input contract: Sharp 1,000 +records/four MiB, Node2Shell 10,000 records/four MiB, Flowise 10,000 records/16 MiB. +An oversized merged export fails before publication; use canonical output or split +the source inputs explicitly. No records are silently truncated. CycloneDX subject +component provenance uses `metadata/component` without an invented array index. + +Layer application skips lower-snapshot traversal for empty layers and limits +aggregate lower-snapshot path visits to two million. Per-layer selected-file +ancestor validation examines only new file observations; earlier verified state +is not rescanned for empty layers. Layer trailing padding shares the outer +archive one-MiB cap. OCI indexes require schemaVersion 2. Held dpkg Node packages +retain installed-version metadata. Namespaced CycloneDX packages are not matched +as unscoped Sharp/Flowise/Node names. + +Layer reads share a 200,000-call budget, bounding per-byte gzip header processing as well as byte/expansion limits. OCI image planning errors produce an incomplete image record while independently supported sibling images retain their observations. Limits remain shared across images. + +Every candidate package.json is captured and strictly parsed before its package name is known. Irrelevant manifests still consume the shared capture budget; malformed or excessive candidates make that image incomplete. These deliberately conservative bounds can reject large valid images/SBOMs. OCI attestation manifests and unsupported layer media types are recorded as incomplete; platform labels are not collected. No-force publication requires hard-link support and fails closed otherwise. diff --git a/SHA256SUMS b/SHA256SUMS new file mode 100644 index 0000000..a86d025 --- /dev/null +++ b/SHA256SUMS @@ -0,0 +1 @@ +b6422d9849a64a4fe54a316a1d64f3fed4a20f52486697990b1d9bc08e6547f2 oci_runtime_inventory.py diff --git a/examples/inventory.sbom.json b/examples/inventory.sbom.json new file mode 100644 index 0000000..2156299 --- /dev/null +++ b/examples/inventory.sbom.json @@ -0,0 +1,21 @@ +{ + "bomFormat": "CycloneDX", + "specVersion": "1.6", + "components": [ + { + "type": "library", + "name": "sharp", + "version": "0.35.5" + }, + { + "type": "application", + "name": "flowise", + "version": "3.1.3" + }, + { + "type": "application", + "name": "nodejs", + "version": "24.22.1-vendor" + } + ] +} diff --git a/oci_runtime_inventory.py b/oci_runtime_inventory.py new file mode 100644 index 0000000..3fa7fa0 --- /dev/null +++ b/oci_runtime_inventory.py @@ -0,0 +1,655 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-CompleteTech-Commercial +# Copyright (c) 2026 CompleteTech LLC. See LICENSING.md. +"""Offline bounded OCI/Docker-save inventory. Never extract or execute images.""" +from __future__ import annotations + +import argparse +import gzip +import zlib +import hashlib +import io +import json +import os +from pathlib import Path +import re +import stat +import sys +import tempfile + +VERSION = "0.1.0" +SCHEMA_VERSION = "1.0" +MAX_ARCHIVE = 8 * 1024**3 +MAX_JSON = 4 * 1024**2 +MAX_MEMBERS = 100000 +MAX_IMAGES = 16 +MAX_LAYERS = 256 +MAX_CAPTURE = 16 * 1024**2 +TARGETS = {"sharp", "flowise", "flowise-components"} + + +class InventoryError(ValueError): + """Messages identify failure classes without exposing input contents.""" + + +def json_value(raw): + def unique(pairs): + result = {} + for key, value in pairs: + if key in result: + raise InventoryError("duplicate_json_key") + result[key] = value + return result + def reject(_): + raise InventoryError("nonfinite_json") + try: + data = json.loads(raw.decode("utf-8-sig"), object_pairs_hook=unique, parse_constant=reject) + except (UnicodeError, json.JSONDecodeError, RecursionError): + raise InventoryError("invalid_json") from None + pending, count = [(data, 0)], 0 + while pending: + value, depth = pending.pop() + count += 1 + if depth > 40 or count > 200000: + raise InventoryError("json_structure_limit") + if isinstance(value, dict): + pending.extend((v, depth + 1) for v in value.values()) + elif isinstance(value, list): + pending.extend((v, depth + 1) for v in value) + return data + + +def safe_path(value): + if not isinstance(value, str) or len(value) > 512 or "\\" in value or ":" in value or value.startswith("/") or any(ord(c) < 32 or ord(c) == 127 for c in value): + raise InventoryError("unsafe_archive_path") + while value.startswith("./"): + value = value[2:] + value = value.rstrip("/") + if not value or value.count("/") > 63 or any(part in {"", ".", ".."} for part in value.split("/")): + raise InventoryError("unsafe_archive_path") + return value + + +def octal(field): + value = field.strip(b" \0") + if not re.fullmatch(b"[0-7]*", value): + raise InventoryError("unsupported_tar_number") + return int(value or b"0", 8) + + +def header(block): + if len(block) != 512: + raise InventoryError("truncated_tar_header") + if octal(block[148:156]) != sum(block[:148]) + 8 * 32 + sum(block[156:]): + raise InventoryError("invalid_tar_checksum") + try: + name = block[:100].split(b"\0", 1)[0].decode("utf-8") + prefix = block[345:500].split(b"\0", 1)[0].decode("utf-8") if block[257:263] == b"ustar\0" else "" + except UnicodeError: + raise InventoryError("invalid_tar_name_encoding") from None + kind = block[156:157] + if kind not in {b"\0", b"0", b"1", b"2", b"3", b"4", b"5", b"6"}: + raise InventoryError("unsupported_tar_extension_or_sparse_member") + size = octal(block[124:136]) + if kind == b"\0" and name.endswith("/"): + kind = b"5" # Legacy V7 directory encoding. + if kind not in {b"\0", b"0"} and size: + raise InventoryError("special_member_has_data") + path = "" if kind == b"5" and name in {".", "./"} and not prefix else safe_path(prefix + "/" + name if prefix else name) + return path, size, "file" if kind in {b"\0", b"0"} else "dir" if kind == b"5" else "opaque" + + +def regular_input(path): + fd = os.open(path, os.O_RDONLY | getattr(os, "O_BINARY", 0) | getattr(os, "O_NONBLOCK", 0)) + stream = os.fdopen(fd, "rb") + info = os.fstat(stream.fileno()) + if not stat.S_ISREG(info.st_mode) or info.st_size > MAX_ARCHIVE: + stream.close() + raise InventoryError("input_not_bounded_regular_file") + return stream, info.st_size + + +def archive_index(stream, length): + """Index an uncompressed USTAR outer archive without extracting any member.""" + result, position, count = {}, 0, 0 + while position + 512 <= length: + stream.seek(position) + block = stream.read(512) + position += 512 + if block == b"\0" * 512: + if stream.read(512) != b"\0" * 512: + raise InventoryError("invalid_tar_terminator") + if length - stream.tell() > 1024**2: + raise InventoryError("excessive_tar_trailing_padding") + while True: + trailing = stream.read(65536) + if not trailing: + return result + if trailing.strip(b"\0"): + raise InventoryError("nonzero_tar_trailing_data") + path, size, kind = header(block) + count += 1 + if count > MAX_MEMBERS: + raise InventoryError("outer_member_limit") + if not path and kind == "dir": + continue + if path in result or len(result) >= MAX_MEMBERS: + raise InventoryError("duplicate_or_excessive_archive_members") + if kind == "opaque": + raise InventoryError("outer_archive_link_or_device") + padded = (size + 511) // 512 * 512 + if position + padded > length: + raise InventoryError("truncated_archive_member") + result[path] = (position, size, kind) + position += padded + raise InventoryError("missing_tar_terminator") + + +MAX_READ_CALLS = 200000 + + +class Budget: + def __init__(self, expanded=512 * 1024**2): + self.expanded_limit = expanded + self.expanded = self.raw = self.members = self.capture = self.work = self.read_calls = 0 + + def add(self, key, amount, maximum): + value = getattr(self, key) + amount + if value > maximum: + raise InventoryError(key + "_limit") + setattr(self, key, value) + + +class Segment: + def __init__(self, stream, offset, size, budget): + self.stream, self.position, self.remaining = stream, offset, size + self.budget, self.digest = budget, hashlib.sha256() + + def read(self, size=-1): + self.budget.add("read_calls", 1, MAX_READ_CALLS) + size = min(self.remaining, 65536 if size < 0 else size) + self.stream.seek(self.position) + raw = self.stream.read(size) + if len(raw) != size: + raise InventoryError("truncated_blob") + self.budget.add("raw", len(raw), 1024**3) + self.digest.update(raw) + self.position += len(raw) + self.remaining -= len(raw) + return raw + + +class Expanded: + def __init__(self, stream, budget): + self.stream, self.budget = stream, budget + + def read(self, size): + raw = self.stream.read(size) + self.budget.add("expanded", len(raw), self.budget.expanded_limit) + return raw + + def exact(self, size): + raw = self.read(size) + if len(raw) != size: + raise InventoryError("truncated_layer") + return raw + + def discard(self, size): + while size: + amount = min(size, 65536) + self.exact(amount) + size -= amount + + +def selected_file(path): + return path.endswith("/package.json") or path == "package.json" or path in {"var/lib/dpkg/status", "lib/apk/db/installed"} + + +def layer_entries(stream, budget, layer_digest): + reader = Expanded(stream, budget) + entries, removals, opaque_dirs = {}, set(), set() + while True: + block = reader.exact(512) + if block == b"\0" * 512: + if reader.exact(512) != b"\0" * 512: + raise InventoryError("invalid_layer_terminator") + padding = 0 + while True: + raw = reader.read(65536) + padding += len(raw) + if padding > 1024**2: + raise InventoryError("excessive_layer_trailing_padding") + if not raw: + return entries, removals, opaque_dirs + if raw.strip(b"\0"): + raise InventoryError("nonzero_layer_trailing_data") + path, size, kind = header(block) + budget.add("members", 1, MAX_MEMBERS) + if not path and kind == "dir": + continue + if path in entries: + raise InventoryError("duplicate_layer_member") + directory, _, basename = path.rpartition("/") + if basename.startswith(".wh."): + if kind != "file" or size: + raise InventoryError("invalid_whiteout") + if basename == ".wh..wh..opq": + opaque_dirs.add(directory) + else: + if not basename[4:]: + raise InventoryError("invalid_whiteout") + removals.add(safe_path((directory + "/" if directory else "") + basename[4:])) + # Track normalized whiteout names too, rejecting duplicate markers. + entries[path] = ("whiteout", None, layer_digest) + else: + content = None + if kind == "file" and selected_file(path): + if size > MAX_JSON: + raise InventoryError("selected_file_byte_limit") + budget.add("capture", size, MAX_CAPTURE) + content = reader.exact(size) + else: + reader.discard(size) + entries[path] = (kind, content, layer_digest) + reader.discard((-size) % 512) + + +def apply_layer(filesystem, entries, removals, opaque_dirs, budget=None): + # Whiteouts affect lower layers regardless of ordering within this layer. + replacing = {path for path, entry in entries.items() if entry[0] not in {"dir", "whiteout"}} + if "" in opaque_dirs: + filesystem.clear() + elif removals or opaque_dirs or replacing: + if budget is not None: + budget.add("work",len(filesystem),2000000) + for path in list(filesystem): + parts = path.split("/") + parents = ["/".join(parts[:i]) for i in range(1, len(parts))] + if path in removals or any(parent in removals or parent in opaque_dirs or parent in replacing for parent in parents): + del filesystem[path] + filesystem.update({path: entry for path, entry in entries.items() if entry[0] != "whiteout"}) + if len(filesystem) > MAX_MEMBERS: + raise InventoryError("filesystem_member_limit") + + +def descriptor_path(descriptor, index): + if not isinstance(descriptor, dict) or not re.fullmatch(r"sha256:[0-9a-f]{64}", str(descriptor.get("digest", ""))): + raise InventoryError("unsupported_blob_digest") + size = descriptor.get("size") + name = "blobs/sha256/" + descriptor["digest"][7:] + if type(size) is not int or size < 0 or name not in index or index[name][1] != size or index[name][2] != "file": + raise InventoryError("missing_or_mismatched_blob") + return name + + +def read_member(stream, index, name, budget=None): + if name not in index or index[name][2] != "file" or index[name][1] > MAX_JSON: + raise InventoryError("missing_or_excessive_manifest") + offset, size, _ = index[name] + if budget is not None: + budget.add("capture", size, MAX_CAPTURE) + stream.seek(offset) + raw = stream.read(size) + if len(raw) != size: + raise InventoryError("truncated_manifest") + return raw + + +def blob_json(stream, index, descriptor, budget): + name = descriptor_path(descriptor, index) + raw = read_member(stream, index, name, budget) + if hashlib.sha256(raw).hexdigest() != descriptor["digest"][7:]: + raise InventoryError("manifest_digest_mismatch") + return json_value(raw) + + +def image_plans(stream, index, budget): + plans = [] + if "oci-layout" in index: + if json_value(read_member(stream, index, "oci-layout", budget)) != {"imageLayoutVersion": "1.0.0"}: + raise InventoryError("unsupported_oci_layout") + root = json_value(read_member(stream, index, "index.json", budget)) + pending = [(root, 0)] + visits = 0 + while pending: + document, depth = pending.pop() + if not isinstance(document,dict) or type(document.get("schemaVersion")) is not int or document["schemaVersion"] != 2: + raise InventoryError("unsupported_index_schema_version") + descriptors = document.get("manifests") + if depth > 3 or not isinstance(descriptors, list) or len(descriptors) > MAX_IMAGES: + raise InventoryError("invalid_or_excessive_image_index") + for descriptor in descriptors: + visits += 1 + if visits > 64: + raise InventoryError("image_descriptor_limit") + try: + manifest = blob_json(stream, index, descriptor, budget) + if isinstance(manifest, dict) and "manifests" in manifest: + pending.append((manifest, depth + 1)) + continue + if not isinstance(manifest, dict) or type(manifest.get("schemaVersion")) is not int or manifest["schemaVersion"] != 2 or not isinstance(manifest.get("layers"), list): + raise InventoryError("invalid_image_manifest") + if len(manifest["layers"]) > MAX_LAYERS: + raise InventoryError("image_layer_limit") + # Config is verified as opaque bytes, not parsed for Env/Cmd/host data. + config = manifest.get("config") + config_name = descriptor_path(config, index) + if hashlib.sha256(read_member(stream, index, config_name, budget)).hexdigest() != config["digest"][7:]: + raise InventoryError("config_digest_mismatch") + layers = [] + for layer in manifest["layers"]: + name = descriptor_path(layer, index) + media = layer.get("mediaType", "") + if not isinstance(media, str) or media not in {"application/vnd.oci.image.layer.v1.tar", "application/vnd.oci.image.layer.v1.tar+gzip", "application/vnd.docker.image.rootfs.diff.tar.gzip"}: + raise InventoryError("unsupported_layer_media_type") + layers.append((name, media.endswith("gzip"), layer["digest"])) + plans.append((descriptor["digest"], layers, "oci_manifest", None)) + except InventoryError as exc: + identity=descriptor.get("digest", "unknown") if isinstance(descriptor,dict) else "unknown" + if not isinstance(identity,str) or re.fullmatch(r"sha256:[a-f0-9]{64}",identity) is None: + identity="unknown" + plans.append((identity, [], "oci_manifest", str(exc))) + if len(plans) > MAX_IMAGES: + raise InventoryError("image_count_limit") + elif "manifest.json" in index: + manifests = json_value(read_member(stream, index, "manifest.json", budget)) + if not isinstance(manifests, list) or not manifests or len(manifests) > MAX_IMAGES: + raise InventoryError("invalid_docker_manifest_list") + for manifest in manifests: + if not isinstance(manifest, dict) or not isinstance(manifest.get("Layers"), list) or len(manifest["Layers"]) > MAX_LAYERS: + raise InventoryError("invalid_docker_image_manifest") + config = safe_path(manifest.get("Config")) + digest = "sha256:" + hashlib.sha256(read_member(stream, index, config, budget)).hexdigest() + layers = [] + for layer in manifest["Layers"]: + name = safe_path(layer) + if name not in index or index[name][2] != "file": + raise InventoryError("missing_docker_layer") + layers.append((name, False, None)) + plans.append((digest, layers, "docker_config", None)) + else: + raise InventoryError("unsupported_archive_layout") + if not plans: + raise InventoryError("empty_image_index") + return plans + + +def validate_selected_ancestors(filesystem, candidates=None): + selected = filesystem if candidates is None else candidates + for path, (kind, content, _) in selected.items(): + if kind != "file" or content is None: + continue + parts = path.split("/") + if any(filesystem.get("/".join(parts[:i]), ("dir",))[0] != "dir" for i in range(1, len(parts))): + raise InventoryError("metadata_under_nondirectory_ancestor") + + +def component_rows(filesystem): + rows = [] + for path, (kind, content, layer) in sorted(filesystem.items()): + if selected_file(path) and kind != "file": + raise InventoryError("selected_metadata_not_regular_file") + if kind != "file" or content is None: + continue + parts = path.split("/") + if any(filesystem.get("/".join(parts[:i]), ("dir",))[0] != "dir" for i in range(1, len(parts))): + raise InventoryError("metadata_under_nondirectory_ancestor") + found = [] + if path.endswith("package.json"): + data = json_value(content) + if not isinstance(data, dict): + raise InventoryError("invalid_package_manifest") + if isinstance(data.get("name"), str) and data["name"] in TARGETS: + found.append((data["name"], "npm", data.get("version", ""))) + elif path == "var/lib/dpkg/status": + try: + text = content.decode("utf-8") + except UnicodeError: + raise InventoryError("invalid_package_database_encoding") from None + for stanza in re.split(r"\r?\n\r?\n", text): + pairs = re.findall(r"^(Package|Version|Status):[ \t]*(.*)$", stanza, re.M) + if len({key for key, _ in pairs}) != len(pairs): + raise InventoryError("duplicate_package_database_field") + fields = {key: value.rstrip("\r") for key, value in pairs} + if fields.get("Package") == "nodejs" and re.fullmatch(r"(?:install|hold|deinstall|purge|unknown) ok installed",fields.get("Status", "")) is not None: + found.append(("nodejs", "deb", fields.get("Version", ""))) + elif path == "lib/apk/db/installed": + try: + text = content.decode("utf-8") + except UnicodeError: + raise InventoryError("invalid_package_database_encoding") from None + for stanza in text.split("\n\n"): + pairs = re.findall(r"^([PV]):(.*)$", stanza, re.M) + if len({key for key, _ in pairs}) != len(pairs): + raise InventoryError("duplicate_package_database_field") + fields = {key: value.rstrip("\r") for key, value in pairs} + if fields.get("P") in {"nodejs", "nodejs-current"}: + found.append(("nodejs", "apk", fields.get("V", ""))) + for name, ecosystem, version in found: + if not isinstance(version, str) or len(version) > 512: + raise InventoryError("invalid_package_version_metadata") + rows.append(dict(name=name, ecosystem=ecosystem, version=version, build_source="container_image_unverified", + path=path, layer_digest=layer, evidence_type="package_metadata", exploitability="not_tested")) + if len(rows) > 10000: + raise InventoryError("component_count_limit") + return rows + + +def scan_archive(path, report, budget): + stream, length = regular_input(path) + with stream: + index = archive_index(stream, length) + plans = image_plans(stream, index, budget) + for image_digest, layers, identity_kind, planning_error in plans: + image = dict(digest=image_digest, identity_kind=identity_kind, layers=[], components=[], coverage_complete=False, notes=[]) + try: + if planning_error: + raise InventoryError(planning_error) + filesystem = {} + for name, compressed, expected in layers: + offset, size, _ = index[name] + raw = Segment(stream, offset, size, budget) + decoded = gzip.GzipFile(fileobj=raw, mode="rb") if compressed else raw + entries, removals, opaque_dirs = layer_entries(decoded, budget, expected or "unverified") + if raw.remaining: + raise InventoryError("unconsumed_layer_bytes") + digest = "sha256:" + raw.digest.hexdigest() + if expected and digest != expected: + raise InventoryError("layer_digest_mismatch") + entries = {p: (k, c, digest) for p, (k, c, _) in entries.items()} + apply_layer(filesystem, entries, removals, opaque_dirs, budget) + validate_selected_ancestors(filesystem, entries) + image["layers"].append(digest) + image["components"] = component_rows(filesystem) + image["coverage_complete"] = True + except InventoryError as exc: + image["notes"].append(str(exc)) + report["complete"] = False + except (OSError, EOFError, ValueError, zlib.error): + image["notes"].append("image_invalid_or_unavailable") + report["complete"] = False + report["images"].append(image) + + +def scan_sbom(path, report, budget): + stream, length = regular_input(path) + with stream: + if length > MAX_JSON: + raise InventoryError("sbom_byte_limit") + budget.add("capture", length, MAX_CAPTURE) + raw = stream.read(length) + data = json_value(raw) + if not isinstance(data, dict): + raise InventoryError("invalid_sbom") + pending = [] + version_key = "version" + if data.get("bomFormat") == "CycloneDX" and isinstance(data.get("specVersion"), str) and data["specVersion"] in {"1.4", "1.5", "1.6"}: + pending.append((data.get("components", []), "components")) + metadata = data.get("metadata", {}) + if not isinstance(metadata, dict): + raise InventoryError("invalid_sbom_metadata") + if "component" in metadata: + pending.append(([metadata["component"]], None)) + elif isinstance(data.get("spdxVersion"), str) and data["spdxVersion"] in {"SPDX-2.2", "SPDX-2.3"}: + pending.append((data.get("packages", []), "packages")) + version_key = "versionInfo" + else: + raise InventoryError("unsupported_sbom_format") + rows, count = [], 0 + while pending: + entries, prefix = pending.pop() + if not isinstance(entries, list): + raise InventoryError("invalid_sbom_entries") + for index, entry in enumerate(entries): + count += 1 + if count > 10000 or not isinstance(entry, dict): + raise InventoryError("invalid_or_excessive_sbom_components") + location = prefix + "/" + str(index) if prefix is not None else "metadata/component" + name = entry.get("name", "") + if not isinstance(name, str): + raise InventoryError("invalid_sbom_component_name") + group=entry.get("group","") if version_key == "version" else "" + if not isinstance(group,str): + raise InventoryError("invalid_sbom_component_namespace") + if not group and name in TARGETS | {"nodejs"}: + version = entry.get(version_key, "") + if not isinstance(version, str) or len(version) > 512: + raise InventoryError("invalid_sbom_version") + rows.append(dict(name=name, version=version, ecosystem="sbom", build_source="offline_metadata_unverified", + path=location, layer_digest="", evidence_type="sbom_metadata", exploitability="not_tested")) + if "components" in entry: + pending.append((entry["components"], location + "/components")) + report["sboms"].append(dict(artifact_digest="sha256:" + hashlib.sha256(raw).hexdigest(), components=rows, coverage_complete=True, + notes=["SBOM artifact digest is not an image or publisher identity; package metadata is not runtime attestation."])) + + +def export_inventory(report, consumer): + output = _export_inventory(report, consumer) + if consumer != "canonical": + records = output["records"] if consumer == "sharp" else output + maximum = 1000 if consumer == "sharp" else 10000 + byte_maximum = 16 * 1024**2 if consumer == "flowise" else 4 * 1024**2 + if len(records) > maximum or len(json.dumps(output, ensure_ascii=True, sort_keys=True, indent=2, allow_nan=False).encode("utf-8")) + 1 > byte_maximum: + raise InventoryError("consumer_export_limit_use_canonical_or_split_inputs") + return output + + +def _export_inventory(report, consumer): + rows = [row for image in [*report["images"], *report.get("sboms", [])] if image["coverage_complete"] for row in image["components"]] + def version(row, maximum=512): + value = row["version"] + return value if len(value) <= maximum and not any(ord(c) < 32 or ord(c) == 127 for c in value) else "" + if consumer == "canonical": + return report + if consumer == "sharp": + selected = [row for row in rows if row["name"] == "sharp"] + return {"schema_version": "1.0", "records": [{"asset_id": "metadata-" + str(i + 1), "sharp_versions": [version(r, 64)] if version(r, 64) else [], "library_source": "unknown", "sharp_build_source": "unknown"} for i, r in enumerate(selected)] or [{"asset_id": "metadata-coverage"}]} + if consumer == "node2shell": + selected = [row for row in rows if row["name"] == "nodejs"] + return [{"schema_version": 1, "node_version": version(r), "build_source": "offline_metadata_unverified", "label": "metadata-node-" + str(i + 1)} for i, r in enumerate(selected)] or [{"schema_version": 1, "node_version": "", "build_source": "offline_metadata_unverified", "label": "metadata-node-coverage"}] + selected = [row for row in rows if row["name"] in {"flowise", "flowise-components"}] + return [{"schema_version": 1, "package": r["name"], "version": version(r), "build_source": "offline_metadata_unverified", "label": "metadata-flowise-" + str(i + 1)} for i, r in enumerate(selected)] or [{"schema_version": 1, "package": name, "version": "", "build_source": "offline_metadata_unverified", "label": "metadata-coverage"} for name in ["flowise", "flowise-components"]] + + +def validate_output(output, inputs, force): + if output is None: + return + if not output.parent.is_dir(): + raise InventoryError("output_parent_missing") + for source in [Path(__file__), *inputs]: + if output.resolve() == source.resolve() or (output.exists() and source.exists() and os.path.samefile(output, source)): + raise InventoryError("output_aliases_protected_input") + if os.path.lexists(output) and not force: + raise InventoryError("output_exists") + if os.path.lexists(output) and (not output.exists() or not stat.S_ISREG(output.stat().st_mode)): + raise InventoryError("output_not_regular") + + +def publish(data, output, force, state): + text = json.dumps(data, ensure_ascii=True, sort_keys=True, indent=2, allow_nan=False) + "\n" + if output is None: + state["stdout_started"] = True + sys.stdout.write(text) + state["published"] = True + return + staged = None + interrupted = False + try: + with tempfile.NamedTemporaryFile("wb", dir=output.parent, prefix=".oci-report-", delete=False) as stream: + staged = Path(stream.name) + stream.write(text.encode("utf-8")) + stream.flush() + os.fsync(stream.fileno()) + state["publication_started"] = True + if force: + os.replace(staged, output) + else: + os.link(staged, output) + state["published"] = True + except KeyboardInterrupt: + interrupted = True + raise + finally: + if staged is not None and (not interrupted or state["published"] or not state.get("publication_started")): + staged.unlink(missing_ok=True) + + +def main(argv=None): + parser = argparse.ArgumentParser(description=__doc__, allow_abbrev=False) + parser.add_argument("--archive", type=Path, help="Existing uncompressed OCI-layout or Docker-save TAR archive") + parser.add_argument("--sbom", action="append", default=[], type=Path, help="Existing CycloneDX 1.4-1.6 or SPDX 2.2/2.3 JSON; repeatable") + parser.add_argument("--consumer", choices=("canonical", "node2shell", "sharp", "flowise"), default="canonical", help="Optional offline consumer inventory; provenance remains unverified") + parser.add_argument("--max-expanded-mib", type=int, default=512, help="Global expanded-byte budget, 16-4096 MiB (default 512)") + parser.add_argument("--output", type=Path, help="New JSON report path; default stdout") + parser.add_argument("--force", action="store_true", help="Replace an existing report, never a protected input") + parser.add_argument("--version", action="version", version=VERSION) + args = parser.parse_args(argv) + if not args.archive and not args.sbom: + parser.error("provide --archive or --sbom") + if len(args.sbom) > 100: + parser.error("at most 100 SBOM files") + if not 16 <= args.max_expanded_mib <= 4096: + parser.error("--max-expanded-mib must be 16 through 4096") + report = dict(schema_version=SCHEMA_VERSION, tool={"name": "oci-runtime-inventory", "version": VERSION}, complete=True, + images=[], sboms=[], notes=["Offline package metadata only; no runtime, package provenance, patch state, absence or exploitability is established."], + exploitability="not_tested") + report["limits"] = dict(archive_bytes=MAX_ARCHIVE, expanded_bytes=args.max_expanded_mib * 1024**2, compressed_bytes=1024**3, + members=MAX_MEMBERS, capture_bytes=MAX_CAPTURE, images=MAX_IMAGES, layers_per_image=MAX_LAYERS) + budget = Budget(args.max_expanded_mib * 1024**2) + state = {"validated": False, "published": False, "stdout_started": False, "publication_started": False} + code = 0 + try: + validate_output(args.output, ([args.archive] if args.archive else []) + args.sbom, args.force) + state["validated"] = True + try: + if args.archive: + scan_archive(args.archive, report, budget) + for path in args.sbom: + scan_sbom(path, report, budget) + except InventoryError as exc: + report["complete"] = False + report["notes"].append(str(exc)) + except (OSError, EOFError, ValueError, zlib.error): + report["complete"] = False + report["notes"].append("input_invalid_or_unavailable") + report["usage"] = {name: getattr(budget, name) for name in ["raw", "expanded", "members", "capture", "work"]} + code = 0 if report["complete"] else 2 + publish(export_inventory(report, args.consumer), args.output, args.force, state) + except KeyboardInterrupt: + if state["validated"] and not any(state.get(key) for key in ["published", "stdout_started", "publication_started"]): + report["complete"] = False + report["notes"].append("interrupted_completed_images_only") + try: + publish(export_inventory(report, args.consumer), args.output, args.force, state) + except (OSError, ValueError, KeyboardInterrupt): + pass + sys.stderr.write("Interrupted; output may be incomplete or publication indeterminate; no publication retry.\n") + return 130 + except (OSError, ValueError): + sys.stderr.write("Inventory failed: unavailable/invalid input or protected/unwritable output.\n") + return 2 + return code + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/test_inventory.py b/test_inventory.py new file mode 100644 index 0000000..b0598a8 --- /dev/null +++ b/test_inventory.py @@ -0,0 +1,359 @@ +"""Synthetic local archive fixtures; no Docker, extraction or native execution.""" +import contextlib +import gzip +import hashlib +import io +import json +import os +from pathlib import Path +import subprocess +import sys +import tarfile +import tempfile +import unittest +from unittest import mock + +import oci_runtime_inventory as c + + +def tar_bytes(entries, fmt=tarfile.USTAR_FORMAT): + out = io.BytesIO() + with tarfile.open(fileobj=out, mode="w", format=fmt) as archive: + for entry in entries: + name, data = entry[:2] + kind = entry[2] if len(entry) > 2 else tarfile.REGTYPE + info = tarfile.TarInfo(name) + info.type = kind + info.size = len(data) if kind in {tarfile.REGTYPE, tarfile.AREGTYPE} else 0 + if kind in {tarfile.SYMTYPE, tarfile.LNKTYPE}: + info.linkname = entry[3] if len(entry) > 3 else "../outside" + archive.addfile(info, io.BytesIO(data) if info.size else None) + return out.getvalue() + + +def package(name="sharp", version="0.35.5"): + return json.dumps({"name": name, "version": version, "scripts": {"postinstall": "NEVER-EXECUTE"}}).encode() + + +def docker_archive(layers, config=None): + config = config or b'{"Hostname":"PRIVATE-HOST","Env":["TOKEN=PRIVATE-TOKEN"],"Cmd":["NEVER-EXECUTE"]}' + manifest = [{"Config": "config.json", "RepoTags": ["PRIVATE-TAG:latest"], "Layers": [f"layer{i}.tar" for i in range(len(layers))]}] + return tar_bytes([("manifest.json", json.dumps(manifest).encode()), ("config.json", config), *[(f"layer{i}.tar", data) for i, data in enumerate(layers)]]) + + +def oci_archive(layers, compressed=False, bad_digest=False): + blobs = [] + def blob(data, media): + digest = hashlib.sha256(data).hexdigest() + blobs.append(("blobs/sha256/" + digest, data)) + return {"digest": "sha256:" + digest, "size": len(data), "mediaType": media} + config = blob(b'{"Env":["PRIVATE-TOKEN"]}', "application/vnd.oci.image.config.v1+json") + descriptors = [blob(gzip.compress(data, mtime=0) if compressed else data, "application/vnd.oci.image.layer.v1.tar" + ("+gzip" if compressed else "")) for data in layers] + manifest = {"schemaVersion": 2, "config": config, "layers": descriptors} + descriptor = blob(json.dumps(manifest).encode(), "application/vnd.oci.image.manifest.v1+json") + if bad_digest: + # Keep descriptor/name and mutate content, so verification must reject it. + name, raw = blobs[-1] + blobs[-1] = (name, raw + b" ") + descriptor["size"] += 1 + index = {"schemaVersion": 2, "manifests": [descriptor]} + return tar_bytes([("oci-layout", b'{"imageLayoutVersion":"1.0.0"}'), ("index.json", json.dumps(index).encode()), *blobs]) + + +class InventoryTests(unittest.TestCase): + def cli(self, *args): + with contextlib.redirect_stdout(io.StringIO()) as out, contextlib.redirect_stderr(io.StringIO()) as err: + code = c.main(list(args)) + return code, out.getvalue(), err.getvalue() + + def scan(self, archive, *args): + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / "image.tar" + path.write_bytes(archive) + code, out, err = self.cli("--archive", str(path), *args) + return code, json.loads(out), err + + def test_gzip_header_work_is_globally_bounded(self): + layer=tar_bytes([('app/package.json',package())]) + archive=oci_archive([layer],compressed=True) + with tarfile.open(fileobj=io.BytesIO(archive)) as stream: + members=[(m.name,stream.extractfile(m).read()) for m in stream if m.isfile()] + # Rewrite gzip FNAME and the affected blob/manifest/index descriptors. + compressed=gzip.compress(layer,mtime=0) + malicious=compressed[:3]+bytes([8])+compressed[4:10]+b'x'*1000+b'\0'+compressed[10:] + blobs=dict(members);index=json.loads(blobs['index.json']);old=index['manifests'][0] + manifest=json.loads(blobs['blobs/sha256/'+old['digest'][7:]]) + digest=hashlib.sha256(malicious).hexdigest();blobs['blobs/sha256/'+digest]=malicious + manifest['layers'][0].update(digest='sha256:'+digest,size=len(malicious)) + raw=json.dumps(manifest).encode();digest=hashlib.sha256(raw).hexdigest();blobs['blobs/sha256/'+digest]=raw + old.update(digest='sha256:'+digest,size=len(raw));blobs['index.json']=json.dumps(index).encode() + with mock.patch.object(c,'MAX_READ_CALLS',100): + code,report,_=self.scan(tar_bytes(list(blobs.items()))) + self.assertEqual(2,code);self.assertIn('read_calls_limit',report['images'][0]['notes']) + self.assertEqual([],report['images'][0]['components']) + + def test_unsupported_image_plan_preserves_valid_sibling(self): + archive=oci_archive([tar_bytes([('app/package.json',package())])]) + with tarfile.open(fileobj=io.BytesIO(archive)) as stream: + blobs={m.name:stream.extractfile(m).read() for m in stream if m.isfile()} + index=json.loads(blobs['index.json']);descriptor=index['manifests'][0] + bad=json.loads(blobs['blobs/sha256/'+descriptor['digest'][7:]]) + bad['layers'][0]['mediaType']='application/vnd.oci.image.layer.v1.tar+zstd' + raw=json.dumps(bad).encode();digest=hashlib.sha256(raw).hexdigest();blobs['blobs/sha256/'+digest]=raw + index['manifests'].append(dict(digest='sha256:'+digest,size=len(raw),mediaType='application/vnd.oci.image.manifest.v1+json')) + blobs['index.json']=json.dumps(index).encode() + for reverse in (False,True): + if reverse: + index['manifests'].reverse();blobs['index.json']=json.dumps(index).encode() + code,report,_=self.scan(tar_bytes(list(blobs.items()))) + self.assertEqual(2,code);self.assertEqual(2,len(report['images'])) + valid=next(image for image in report['images'] if image['coverage_complete']) + failed=next(image for image in report['images'] if not image['coverage_complete']) + self.assertEqual('sharp',valid['components'][0]['name']) + self.assertEqual([],failed['components']);self.assertIn('unsupported_layer_media_type',failed['notes']) + + def test_failed_publication_removes_sensitive_staging_artifacts(self): + with tempfile.TemporaryDirectory() as directory: + root=Path(directory);output=root/"report.json" + for force in (False,True): + if force:output.write_text("existing",encoding="utf-8") + state={"published":False,"publication_started":False} + with mock.patch.object(c.os,"replace" if force else "link",side_effect=OSError("simulated failure")): + with self.assertRaises(OSError):c.publish({"private":"synthetic"},output,force,state) + self.assertEqual([],list(root.glob(".oci-report-*"))) + if force:self.assertEqual("existing",output.read_text()) + + def test_review_resource_padding_v7_index_hold_and_namespace(self): + # No traversal of a lower filesystem for an empty layer. + class NoTraversal(dict): + def __iter__(self): raise AssertionError("empty layer must not traverse") + c.apply_layer(NoTraversal({"app":("dir",None,"")}),{},set(),set()) + raw=bytearray(tar_bytes([("app/",b"",tarfile.DIRTYPE),("app/package.json",package())])) + raw[156]=0;raw[148:156]=b" " + raw[148:156]=f"{sum(raw[:512]):06o}\0 ".encode() + code,report,_=self.scan(docker_archive([bytes(raw)])) + self.assertEqual(0,code);self.assertEqual("sharp",report["images"][0]["components"][0]["name"]) + for compressed in (False,True): + code,report,_=self.scan(oci_archive([tar_bytes([])+b"\0"*(1024**2+1)],compressed=compressed)) + self.assertEqual(2,code) + held=b"Package: nodejs\nVersion: 24.22.0\nStatus: hold ok installed\n" + code,report,_=self.scan(docker_archive([tar_bytes([("var/lib/dpkg/status",held)])])) + self.assertEqual("24.22.0",report["images"][0]["components"][0]["version"]) + with tempfile.TemporaryDirectory() as directory: + path=Path(directory)/"sbom.json" + path.write_text(json.dumps(dict(bomFormat="CycloneDX",specVersion="1.6",components=[dict(group="@vendor",name="sharp",version="0.35.5",components=[dict(name="flowise",version="3.1.3")])])),encoding="utf-8") + report=dict(sboms=[]);c.scan_sbom(path,report,c.Budget()) + self.assertEqual(["flowise"],[r["name"] for r in report["sboms"][0]["components"]]) + # Replace the outer index version while retaining valid manifest descriptors. + archive=oci_archive([tar_bytes([])]) + with tarfile.open(fileobj=io.BytesIO(archive)) as stream: + members=[(m.name,stream.extractfile(m).read()) for m in stream if m.isfile()] + members=[(name,data.replace(b'"schemaVersion": 2',b'"schemaVersion": 3') if name=="index.json" else data) for name,data in members] + self.assertEqual(2,self.scan(tar_bytes(members))[0]) + + def test_each_layer_rejects_inaccessible_metadata_and_serialized_unicode_limit(self): + code, report, _ = self.scan(docker_archive([ + tar_bytes([("app", b"", tarfile.SYMTYPE), ("app/package.json", package())]), + tar_bytes([("app", b"", tarfile.DIRTYPE)]), + ])) + self.assertEqual(2,code) + self.assertEqual([],report["images"][0]["components"]) + row=dict(name="nodejs",version="α"*512) + value=dict(images=[],sboms=[dict(coverage_complete=True,components=[row]*1400)]) + with self.assertRaises(c.InventoryError): c.export_inventory(value,"node2shell") + + def test_copilot_whiteout_deflate_provenance_and_consumer_bounds(self): + code, report, _ = self.scan(docker_archive([tar_bytes([("app/package.json", package())]), tar_bytes([("app/.wh.", b"")])])) + self.assertEqual(2, code) + self.assertEqual([], report["images"][0]["components"]) + raw = docker_archive([tar_bytes([("app/package.json", package())])]) + with mock.patch.object(c, "layer_entries", side_effect=c.zlib.error("bad DEFLATE")): + code, report, _ = self.scan(raw) + self.assertEqual(2, code) + self.assertEqual("image_invalid_or_unavailable", report["images"][0]["notes"][0]) + row = dict(name="sharp", version="0.35.5") + report = dict(images=[], sboms=[dict(coverage_complete=True, components=[row] * 1001)]) + with self.assertRaises(c.InventoryError): c.export_inventory(report, "sharp") + self.assertEqual(report, c.export_inventory(report, "canonical")) + report["sboms"][0]["components"] = [dict(name="nodejs", version="24.22.0")] * 10001 + with self.assertRaises(c.InventoryError): c.export_inventory(report, "node2shell") + with tempfile.TemporaryDirectory() as directory: + path=Path(directory)/"sbom.json" + path.write_text(json.dumps(dict(bomFormat="CycloneDX", specVersion="1.6", metadata=dict(component=dict(name="sharp", version="0.35.5")))),encoding="utf-8") + report=dict(images=[],sboms=[]) + c.scan_sbom(path,report,c.Budget(512*1024**2)) + self.assertEqual("metadata/component",report["sboms"][0]["components"][0]["path"]) + + def test_docker_and_oci_plain_gzip_layers_have_digest_layer_file_provenance(self): + layer = tar_bytes([("app/node_modules/sharp/package.json", package()), ("app/node_modules/flowise/package.json", package("flowise", "3.1.3"))]) + for archive in [docker_archive([layer]), oci_archive([layer]), oci_archive([layer], compressed=True)]: + with self.subTest(size=len(archive)): + code, report, err = self.scan(archive) + self.assertEqual((0, ""), (code, err)) + self.assertTrue(report["complete"]) + image = report["images"][0] + self.assertTrue(image["coverage_complete"]) + self.assertTrue(image["digest"].startswith("sha256:")) + self.assertEqual({"sharp", "flowise"}, {r["name"] for r in image["components"]}) + for row in image["components"]: + self.assertTrue(row["layer_digest"].startswith("sha256:")) + self.assertEqual("not_tested", row["exploitability"]) + for private in ["PRIVATE-HOST", "PRIVATE-TOKEN", "PRIVATE-TAG", "NEVER-EXECUTE"]: + self.assertNotIn(private, json.dumps(report)) + + def test_replaced_and_whiteout_deleted_metadata_does_not_survive(self): + path = "app/node_modules/sharp/package.json" + lower = tar_bytes([(path, package(version="0.35.4"))]) + upper = tar_bytes([(path, package(version="0.35.5"))]) + code, report, _ = self.scan(docker_archive([lower, upper])) + self.assertEqual(0, code) + self.assertEqual(["0.35.5"], [r["version"] for r in report["images"][0]["components"]]) + deletion = tar_bytes([("app/node_modules/sharp/.wh.package.json", b"")]) + _, report, _ = self.scan(docker_archive([lower, deletion])) + self.assertEqual([], report["images"][0]["components"]) + + def test_opaque_whiteouts_only_remove_lower_layer_entries_independent_of_order(self): + lower = tar_bytes([("app/node_modules/sharp/package.json", package()), ("app/node_modules/flowise/package.json", package("flowise"))]) + for entries in [ + [("app/node_modules/.wh..wh..opq", b""), ("app/node_modules/flowise/package.json", package("flowise", "3.1.2"))], + [("app/node_modules/flowise/package.json", package("flowise", "3.1.2")), ("app/node_modules/.wh..wh..opq", b"")], + ]: + _, report, _ = self.scan(docker_archive([lower, tar_bytes(entries)])) + self.assertEqual([("flowise", "3.1.2")], [(r["name"], r["version"]) for r in report["images"][0]["components"]]) + + def test_symlinks_hardlinks_devices_never_open_host_paths_or_retain_replaced_files(self): + path = "app/package.json" + lower = tar_bytes([(path, package())]) + for kind in [tarfile.SYMTYPE, tarfile.LNKTYPE, tarfile.CHRTYPE, tarfile.BLKTYPE, tarfile.FIFOTYPE]: + upper = tar_bytes([(path, b"", kind, "/private/host/package.json")]) + code, report, _ = self.scan(docker_archive([lower, upper])) + self.assertEqual(2, code) + self.assertEqual([], report["images"][0]["components"]) + self.assertFalse(report["images"][0]["coverage_complete"]) + self.assertIn("selected_metadata_not_regular_file",report["images"][0]["notes"]) + + def test_metadata_below_link_is_incomplete_not_followed(self): + layer = tar_bytes([("app", b"", tarfile.SYMTYPE, "/host"), ("app/package.json", package())]) + code, report, _ = self.scan(docker_archive([layer])) + self.assertEqual(2, code) + self.assertFalse(report["complete"]) + self.assertEqual([], report["images"][0]["components"]) + + def test_unsafe_duplicate_sparse_and_extended_members_fail_closed(self): + for layer in [ + tar_bytes([("../package.json", package())]), + tar_bytes([("/package.json", package())]), + tar_bytes([("C:/package.json", package())]), + tar_bytes([("app/package.json", package()), ("app/package.json", package(version="0.35.4"))]), + tar_bytes([("app/package.json", b"", tarfile.GNUTYPE_SPARSE)]), + tar_bytes([("app/" + "long" * 40 + "/package.json", package())], fmt=tarfile.PAX_FORMAT), + ]: + code, report, _ = self.scan(docker_archive([layer])) + self.assertEqual(2, code) + self.assertFalse(report["complete"]) + bad_outer = tar_bytes([("manifest.json", b"[]"), ("manifest.json", b"[]")]) + self.assertEqual(2, self.scan(bad_outer)[0]) + + def test_truncated_checksum_and_blob_digest_mismatch_are_incomplete(self): + valid = docker_archive([tar_bytes([("app/package.json", package())])]) + corrupt = bytearray(valid) + corrupt[0] ^= 1 + for data in [valid[:200], bytes(corrupt), oci_archive([tar_bytes([])], bad_digest=True)]: + code, report, _ = self.scan(data) + self.assertEqual(2, code) + self.assertFalse(report["complete"]) + + def test_limits_bound_gzip_expansion_and_selected_file_capture(self): + compressed = oci_archive([tar_bytes([("unselected.bin",b"\0"*(17*1024**2))])], compressed=True) + code, report, _ = self.scan(compressed, "--max-expanded-mib", "16") + self.assertEqual(2, code) + self.assertIn("expanded_limit", report["images"][0]["notes"]) + layer = tar_bytes([("app/package.json", b" " * (c.MAX_JSON + 1))]) + code, report, _ = self.scan(docker_archive([layer])) + self.assertEqual(2, code) + self.assertIn("selected_file_byte_limit", report["images"][0]["notes"]) + + def test_node_versions_only_come_from_package_database_not_tags_or_binary_names(self): + status = b"Package: nodejs\nStatus: install ok installed\nVersion: 24.22.1-1vendor\n\n" + layer = tar_bytes([("var/lib/dpkg/status", status), ("usr/bin/node-v99.1.0", b"not executed")]) + code, report, _ = self.scan(docker_archive([layer])) + self.assertEqual(0, code) + self.assertEqual(["24.22.1-1vendor"], [r["version"] for r in report["images"][0]["components"]]) + self.assertNotIn("99.1.0", json.dumps(report)) + inventory = c.export_inventory(report, "node2shell") + self.assertEqual("offline_metadata_unverified", inventory[0]["build_source"]) + + def test_consumer_exports_preserve_uncertainty_and_cover_empty_results(self): + layer = tar_bytes([("app/node_modules/sharp/package.json", package()), ("app/node_modules/flowise/package.json", package("flowise", "3.1.3"))]) + _, report, _ = self.scan(docker_archive([layer])) + sharp = c.export_inventory(report, "sharp") + self.assertEqual("unknown", sharp["records"][0]["library_source"]) + flowise = c.export_inventory(report, "flowise") + self.assertEqual("offline_metadata_unverified", flowise[0]["build_source"]) + empty = {"images": [], "sboms": [], "complete": False} + self.assertEqual("", c.export_inventory(empty, "node2shell")[0]["node_version"]) + self.assertEqual(["flowise", "flowise-components"], [r["package"] for r in c.export_inventory(empty, "flowise")]) + + def test_sbom_subject_components_spdx_and_malformed_types(self): + fixtures = [ + {"bomFormat": "CycloneDX", "specVersion": "1.6", "metadata": {"component": {"name": "sharp", "version": "0.35.5", "components": [{"name": "flowise", "version": "3.1.3"}]}}}, + {"spdxVersion": "SPDX-2.3", "packages": [{"name": "sharp", "versionInfo": "0.35.5"}, {"name": "flowise", "versionInfo": "3.1.3"}]}, + ] + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / "bom.json" + for data in fixtures: + path.write_text(json.dumps(data), encoding="utf-8") + code, out, _ = self.cli("--sbom", str(path)) + self.assertEqual(0, code) + report = json.loads(out) + self.assertEqual(2, len(report["sboms"][0]["components"])) + self.assertNotIn("digest", report["sboms"][0]) # No fabricated image identity. + for data in [{"spdxVersion": []}, {"bomFormat": "CycloneDX", "specVersion": "1.6", "components": [{"name": []}]}]: + path.write_text(json.dumps(data), encoding="utf-8") + code, out, _ = self.cli("--sbom", str(path)) + self.assertEqual(2, code) + self.assertFalse(json.loads(out)["complete"]) + + def test_output_protection_no_clobber_and_standalone_copy(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + source = root / "image.tar" + source.write_bytes(docker_archive([tar_bytes([("app/package.json", package())])])) + before = source.read_bytes() + self.assertEqual(2, self.cli("--archive", str(source), "--output", str(source), "--force")[0]) + alias = root / "alias.tar" + os.link(source, alias) + self.assertEqual(2, self.cli("--archive", str(source), "--output", str(alias), "--force")[0]) + self.assertEqual(before, source.read_bytes()) + output = root / "report.json" + output.write_text("preserve", encoding="utf-8") + self.assertEqual(2, self.cli("--archive", str(source), "--output", str(output))[0]) + self.assertEqual("preserve", output.read_text(encoding="utf-8")) + checker = root / "checker.py" + checker.write_bytes(Path(c.__file__).read_bytes()) + result = subprocess.run([sys.executable, str(checker), "--archive", str(source)], capture_output=True, text=True, check=True) + self.assertTrue(json.loads(result.stdout)["complete"]) + + def test_interruptions_do_not_append_second_report_or_retry_publication(self): + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / "image.tar" + path.write_bytes(docker_archive([tar_bytes([])])) + with mock.patch.object(c, "scan_archive", side_effect=KeyboardInterrupt): + code, out, _ = self.cli("--archive", str(path)) + self.assertEqual(130, code) + self.assertFalse(json.loads(out)["complete"]) + class Partial(io.StringIO): + writes = 0 + def write(self, text): + self.writes += 1 + super().write(text[:20]) + raise KeyboardInterrupt + output = Partial() + with contextlib.redirect_stdout(output), contextlib.redirect_stderr(io.StringIO()): + code = c.main(["--archive", str(path)]) + self.assertEqual(130, code) + self.assertEqual(1, output.writes) + + +if __name__ == "__main__": + unittest.main()