diff --git a/README.md b/README.md index 5dc7d4f..446c42f 100644 --- a/README.md +++ b/README.md @@ -14,6 +14,7 @@ python oci_runtime_inventory.py --sbom examples/inventory.sbom.json python oci_runtime_inventory.py --archive authorized-image.tar --consumer sharp --output sharp-input.json python oci_runtime_inventory.py --archive authorized-image.tar --consumer node2shell --output node-input.json python oci_runtime_inventory.py --sbom examples/inventory.sbom.json --consumer flowise --output flowise-input.json +python oci_runtime_inventory.py --sbom examples/inventory.sbom.json --consumer sharp --output sharp-input.json --receipt sharp-receipt.json python -m unittest -v ``` @@ -36,6 +37,24 @@ digests when comparing observations. Neither identity establishes publisher trus or that an archive can run. Config Env/Cmd/hostnames and image tags are not reported. SBOM digests identify the SBOM artifact, not an image. +Canonical reports also retain `generated_at` (UTC inventory-generation time), +`tool.version` and `tool.executable_sha256` (SHA-256 of the standalone file's +bytes, including when invoked through a symlink). The code has no separate +advisory/rule table: the executable digest identifies its parser and export +rules. Generation time is captured once per invocation and reused if an +interruption produces a partial report. It is not an artifact build time, +evidence-acquisition time, publisher attestation or deployed-state observation. +No such times are inferred from file/archive modification timestamps. + +`content_sha256` hashes the canonical JSON projection with `generated_at` and +`content_sha256` omitted, using sorted keys, ASCII escaping, two-space indentation +and a final newline in UTF-8. This permits stable content comparisons across +different generation times while retaining all tool, coverage and artifact +evidence. Existing artifact/config/layer/SBOM digests remain unchanged. For +deterministic Python fixtures, `main(argv, clock=...)` accepts a callable returning +an aware `datetime`; it is converted to UTC without altering content identities +or confidence. The public CLI uses the current UTC clock. + The virtual filesystem applies layers in order. Whiteouts remove lower-layer entries; opaque whiteouts retain same-layer additions regardless of TAR ordering. Replaced/deleted manifests cannot survive as current package observations. @@ -59,6 +78,21 @@ not empty evidence presented as a clean scan. Canonical output retains the full coverage/provenance record; consumer output intentionally carries only each checker's accepted fields. Preserve the process exit code as well as the output. +Consumer schemas do not accept the canonical generation/tool envelope, so these +fields are never inserted into their records. Use `--receipt PATH` with a consumer +export to retain the canonical inventory alongside it. The receipt includes +`consumer`, `export_sha256` (the SHA-256 of the exact exported UTF-8 JSON bytes, +including the final newline), and `inventory`. It contains no input/output path +or hostname. CLI stdout is written as UTF-8 bytes without platform newline +translation, so a redirected export has the same hash on Windows and POSIX. +The two output paths must be distinct and each uses the same +protected no-clobber/explicit-force policy. The receipt is published first; +publication of two files is not transactional. If interruption or output failure +leaves only the receipt, compare its `export_sha256` with the consumer bytes +actually received before associating them. Neither publication is retried after +either begins. A receipt for incomplete input retains `complete=false`; the +consumer retains its existing unknown placeholder policy and exit code. + ## Supported formats and bounds - Uncompressed outer USTAR/V7 TAR containing OCI image-layout 1.0 or Docker-save diff --git a/SHA256SUMS b/SHA256SUMS index acb345b..2bf0770 100644 --- a/SHA256SUMS +++ b/SHA256SUMS @@ -1 +1 @@ -e4bec83a0e16bdfb7935040844535a82164d113b16c3d9aea17c3284242cadd4 oci_runtime_inventory.py +5a9c40db3524f5934ef2d99b2735d77b04de323b0d56c12b60c2fa8c5d3572cb oci_runtime_inventory.py diff --git a/oci_runtime_inventory.py b/oci_runtime_inventory.py index 05cc2d9..7af60dd 100644 --- a/oci_runtime_inventory.py +++ b/oci_runtime_inventory.py @@ -5,6 +5,7 @@ from __future__ import annotations import argparse +from datetime import datetime, timezone import gzip import zlib import hashlib @@ -557,6 +558,28 @@ def export_inventory(report, consumer): return output +def serialized(data): + return json.dumps(data, ensure_ascii=True, sort_keys=True, indent=2, allow_nan=False) + "\n" + + +def content_identity(report): + """Hash canonical evidence, excluding generation time and this digest.""" + content = {key: value for key, value in report.items() if key not in {"generated_at", "content_sha256"}} + return hashlib.sha256(serialized(content).encode("utf-8")).hexdigest() + + +def generation_context(clock=None): + value = (clock or (lambda: datetime.now(timezone.utc)))() + if not isinstance(value, datetime) or value.tzinfo is None or value.utcoffset() is None: + raise InventoryError("generation_clock_requires_aware_datetime") + with regular_input(Path(__file__).resolve())[0] as stream: + # Source identity is bounded independently of potentially large input archives. + source = stream.read(MAX_JSON + 1) + if len(source) > MAX_JSON: + raise InventoryError("executable_identity_size_limit") + return value.astimezone(timezone.utc).isoformat().replace("+00:00", "Z"), hashlib.sha256(source).hexdigest() + + def _export_inventory(report, consumer): rows = [row for image in [*report["images"], *report.get("sboms", [])] if image["coverage_complete"] for row in image["components"]] def version(row, maximum=512): @@ -589,10 +612,17 @@ def validate_output(output, inputs, force): def publish(data, output, force, state): - text = json.dumps(data, ensure_ascii=True, sort_keys=True, indent=2, allow_nan=False) + "\n" + text = serialized(data) if output is None: state["stdout_started"] = True - sys.stdout.write(text) + binary = getattr(sys.stdout, "buffer", None) + if binary is not None: + # Receipt hashes cover these exact UTF-8 bytes, not platform text translation. + binary.write(text.encode("utf-8")) + binary.flush() + else: + # In-memory text streams used by embedders have no byte channel. + sys.stdout.write(text) state["published"] = True return staged = None @@ -617,13 +647,14 @@ def publish(data, output, force, state): staged.unlink(missing_ok=True) -def main(argv=None): +def main(argv=None, *, clock=None): parser = argparse.ArgumentParser(description=__doc__, allow_abbrev=False) parser.add_argument("--archive", type=Path, help="Existing uncompressed OCI-layout or Docker-save TAR archive") parser.add_argument("--sbom", action="append", default=[], type=Path, help="Existing CycloneDX 1.4-1.6 or SPDX 2.2/2.3 JSON; repeatable") parser.add_argument("--consumer", choices=("canonical", "node2shell", "sharp", "flowise"), default="canonical", help="Optional offline consumer inventory; provenance remains unverified") parser.add_argument("--max-expanded-mib", type=int, default=512, help="Global expanded-byte budget, 16-4096 MiB (default 512)") parser.add_argument("--output", type=Path, help="New JSON report path; default stdout") + parser.add_argument("--receipt", type=Path, help="Optional canonical receipt for a consumer export, linked by its exact SHA-256") parser.add_argument("--force", action="store_true", help="Replace an existing report, never a protected input") parser.add_argument("--version", action="version", version=VERSION) args = parser.parse_args(argv) @@ -633,16 +664,36 @@ def main(argv=None): parser.error("at most 100 SBOM files") if not 16 <= args.max_expanded_mib <= 4096: parser.error("--max-expanded-mib must be 16 through 4096") + if args.receipt and args.consumer == "canonical": + parser.error("--receipt accompanies --consumer node2shell, sharp or flowise") report = dict(schema_version=SCHEMA_VERSION, tool={"name": "oci-runtime-inventory", "version": VERSION}, complete=True, images=[], sboms=[], notes=["Offline package metadata only; no runtime, package provenance, patch state, absence or exploitability is established."], exploitability="not_tested") report["limits"] = dict(archive_bytes=MAX_ARCHIVE, expanded_bytes=args.max_expanded_mib * 1024**2, compressed_bytes=1024**3, members=MAX_MEMBERS, capture_bytes=MAX_CAPTURE, images=MAX_IMAGES, layers_per_image=MAX_LAYERS) budget = Budget(args.max_expanded_mib * 1024**2) - state = {"validated": False, "published": False, "stdout_started": False, "publication_started": False} + state = {"validated": False, "published": False, "stdout_started": False, + "publication_started": False, "pair_started": False} code = 0 + def publish_report(): + report["usage"] = {name: getattr(budget, name) for name in ["raw", "expanded", "members", "capture", "work"]} + report["content_sha256"] = content_identity(report) + exported = export_inventory(report, args.consumer) + if args.receipt: + receipt = dict(schema_version="1.0", consumer=args.consumer, + export_sha256=hashlib.sha256(serialized(exported).encode("utf-8")).hexdigest(), inventory=report) + # Either artifact may survive interruption; never retry a partially published pair. + state["pair_started"] = True + publish(receipt, args.receipt, args.force, {"published": False, "publication_started": False}) + publish(exported, args.output, args.force, state) try: validate_output(args.output, ([args.archive] if args.archive else []) + args.sbom, args.force) + validate_output(args.receipt, ([args.archive] if args.archive else []) + args.sbom, args.force) + if args.output and args.receipt: + validate_output(args.receipt, [args.output], args.force) + generated_at, executable_sha256 = generation_context(clock) + report["generated_at"] = generated_at + report["tool"]["executable_sha256"] = executable_sha256 state["validated"] = True try: if args.archive: @@ -655,15 +706,14 @@ def main(argv=None): except (OSError, EOFError, ValueError, zlib.error): report["complete"] = False report["notes"].append("input_invalid_or_unavailable") - report["usage"] = {name: getattr(budget, name) for name in ["raw", "expanded", "members", "capture", "work"]} code = 0 if report["complete"] else 2 - publish(export_inventory(report, args.consumer), args.output, args.force, state) + publish_report() except KeyboardInterrupt: - if state["validated"] and not any(state.get(key) for key in ["published", "stdout_started", "publication_started"]): + if state["validated"] and not any(state.get(key) for key in ["published", "stdout_started", "publication_started", "pair_started"]): report["complete"] = False report["notes"].append("interrupted_completed_images_only") try: - publish(export_inventory(report, args.consumer), args.output, args.force, state) + publish_report() except (OSError, ValueError, KeyboardInterrupt): pass sys.stderr.write("Interrupted; output may be incomplete or publication indeterminate; no publication retry.\n") diff --git a/test_provenance.py b/test_provenance.py new file mode 100644 index 0000000..4d51400 --- /dev/null +++ b/test_provenance.py @@ -0,0 +1,199 @@ +"""Deterministic synthetic provenance tests; no registry or runtime access.""" +import contextlib +from datetime import datetime, timezone +import hashlib +import io +import json +from pathlib import Path +import shutil +import subprocess +import sys +import tempfile +import unittest +from unittest import mock + +import oci_runtime_inventory as c + + +class ProvenanceTests(unittest.TestCase): + def run_cli(self, args, when): + with contextlib.redirect_stdout(io.StringIO()) as out, contextlib.redirect_stderr(io.StringIO()) as err: + code = c.main(args, clock=lambda: when) + return code, out.getvalue(), err.getvalue() + + def test_clock_changes_only_generation_time_not_content_or_artifact_identity(self): + source = Path(__file__).parent / "examples/inventory.sbom.json" + reports = [] + for when in (datetime(2026, 1, 2, tzinfo=timezone.utc), datetime(2026, 1, 3, tzinfo=timezone.utc)): + code, text, err = self.run_cli(["--sbom", str(source)], when) + self.assertEqual((0, ""), (code, err)) + reports.append(json.loads(text)) + first, second = reports + self.assertEqual("2026-01-02T00:00:00Z", first["generated_at"]) + self.assertNotEqual(first["generated_at"], second["generated_at"]) + self.assertEqual(first["content_sha256"], second["content_sha256"]) + self.assertEqual(first["sboms"], second["sboms"]) + self.assertEqual("not_tested", first["exploitability"]) + self.assertEqual(hashlib.sha256(Path(c.__file__).read_bytes()).hexdigest(), first["tool"]["executable_sha256"]) + self.assertEqual(first["content_sha256"], c.content_identity(first)) + + def test_empty_invalid_and_interrupted_reports_retain_single_run_context(self): + when = datetime(2026, 1, 2, tzinfo=timezone.utc) + with tempfile.TemporaryDirectory() as directory: + source = Path(directory) / "empty.json" + source.write_text('{"bomFormat":"CycloneDX","specVersion":"1.6","components":[]}', encoding="utf-8") + for mode in ("empty", "invalid", "interrupted"): + if mode == "invalid": + source.write_text("invalid", encoding="utf-8") + calls = [] + def clock(): + calls.append(1) + return when + with mock.patch.object(c, "scan_sbom", side_effect=KeyboardInterrupt) if mode == "interrupted" else contextlib.nullcontext(): + with contextlib.redirect_stdout(io.StringIO()) as out, contextlib.redirect_stderr(io.StringIO()): + code = c.main(["--sbom", str(source)], clock=clock) + report = json.loads(out.getvalue()) + self.assertEqual({"empty": 0, "invalid": 2, "interrupted": 130}[mode], code) + self.assertEqual([1], calls) + self.assertEqual("2026-01-02T00:00:00Z", report["generated_at"]) + self.assertEqual(report["content_sha256"], c.content_identity(report)) + self.assertEqual(mode == "empty", report["complete"]) + + def test_consumer_receipt_links_exact_export_without_changing_contract(self): + source = Path(__file__).parent / "examples/inventory.sbom.json" + when = datetime(2026, 1, 2, tzinfo=timezone.utc) + with tempfile.TemporaryDirectory() as directory: + for consumer in ("node2shell", "sharp", "flowise"): + receipt = Path(directory) / (consumer + ".json") + code, text, err = self.run_cli(["--sbom", str(source), "--consumer", consumer, "--receipt", str(receipt)], when) + self.assertEqual((0, ""), (code, err)) + envelope = json.loads(receipt.read_text(encoding="utf-8")) + report = envelope["inventory"] + self.assertEqual(consumer, envelope["consumer"]) + self.assertEqual(hashlib.sha256(text.encode("utf-8")).hexdigest(), envelope["export_sha256"]) + self.assertEqual(c.export_inventory(report, consumer), json.loads(text)) + self.assertNotIn("generated_at", text) + self.assertEqual("2026-01-02T00:00:00Z", report["generated_at"]) + self.assertNotIn(str(source), receipt.read_text(encoding="utf-8")) + # Neither an existing receipt nor an output alias may be overwritten. + receipt.write_text("preserve", encoding="utf-8") + code, text, _ = self.run_cli(["--sbom", str(source), "--consumer", "sharp", "--receipt", str(receipt)], when) + self.assertEqual((2, ""), (code, text)) + self.assertEqual("preserve", receipt.read_text(encoding="utf-8")) + output = Path(directory) / "alias.json" + code, text, _ = self.run_cli(["--sbom", str(source), "--consumer", "sharp", "--receipt", str(output), "--output", str(output)], when) + self.assertEqual((2, ""), (code, text)) + self.assertFalse(output.exists()) + + def test_copied_and_symlinked_standalone_report_executed_bytes(self): + with tempfile.TemporaryDirectory() as directory: + copied = Path(directory) / "inventory.py" + shutil.copyfile(c.__file__, copied) + copied.write_bytes(copied.read_bytes() + b"\n# synthetic copy\n") + source = Path(__file__).parent / "examples/inventory.sbom.json" + paths = [copied] + link = Path(directory) / "link.py" + try: + link.symlink_to(copied) + paths.append(link) + except OSError: + pass + for path in paths: + result = subprocess.run([sys.executable, str(path), "--sbom", str(source)], capture_output=True, text=True, encoding="utf-8", timeout=15) + self.assertEqual(0, result.returncode, result.stderr) + report = json.loads(result.stdout) + self.assertEqual(hashlib.sha256(copied.read_bytes()).hexdigest(), report["tool"]["executable_sha256"]) + + def test_partial_receipt_has_linked_export_and_publication_is_not_retried(self): + when = datetime(2026, 1, 2, tzinfo=timezone.utc) + with tempfile.TemporaryDirectory() as directory: + source = Path(directory) / "invalid.json" + source.write_text("invalid", encoding="utf-8") + receipt = Path(directory) / "receipt.json" + code, text, _ = self.run_cli(["--sbom", str(source), "--consumer", "sharp", "--receipt", str(receipt)], when) + self.assertEqual(2, code) + envelope = json.loads(receipt.read_text(encoding="utf-8")) + self.assertFalse(envelope["inventory"]["complete"]) + self.assertEqual(hashlib.sha256(text.encode()).hexdigest(), envelope["export_sha256"]) + receipt.unlink() + original_publish = c.publish + calls = [] + def interrupt_output(data, output, force, state): + calls.append(output) + if len(calls) == 2: + raise KeyboardInterrupt + return original_publish(data, output, force, state) + with mock.patch.object(c, "publish", side_effect=interrupt_output): + code, text, _ = self.run_cli(["--sbom", str(source), "--consumer", "sharp", "--receipt", str(receipt)], when) + self.assertEqual((130, ""), (code, text)) + self.assertEqual([receipt, None], calls) + self.assertTrue(receipt.is_file()) + + def test_clock_requires_an_aware_datetime(self): + source = Path(__file__).parent / "examples/inventory.sbom.json" + for invalid in (datetime(2026, 1, 2), "2026-01-02", None): + code, text, _ = self.run_cli(["--sbom", str(source)], invalid) + self.assertEqual((2, ""), (code, text)) + + def test_receipt_staging_interrupt_preserves_exit_and_removes_private_temporary(self): + source = Path(__file__).parent / "examples/inventory.sbom.json" + with tempfile.TemporaryDirectory() as directory: + receipt = Path(directory) / "receipt.json" + output = Path(directory) / "consumer.json" + output.write_text("existing evidence", encoding="utf-8") + with mock.patch.object(c.os, "fsync", side_effect=KeyboardInterrupt): + code, text, _ = self.run_cli(["--sbom", str(source), "--consumer", "sharp", "--receipt", str(receipt), + "--output", str(output), "--force"], datetime(2026, 1, 2, tzinfo=timezone.utc)) + self.assertEqual((130, ""), (code, text)) + self.assertFalse(receipt.exists()) + self.assertEqual("existing evidence", output.read_text(encoding="utf-8")) + self.assertEqual([], list(Path(directory).glob(".oci-report-*"))) + + def test_stdout_receipt_hash_matches_raw_utf8_bytes_despite_text_translation(self): + source = Path(__file__).parent / "examples/inventory.sbom.json" + with tempfile.TemporaryDirectory() as directory: + receipt = Path(directory) / "translated-receipt.json" + raw = io.BytesIO() + translated = io.TextIOWrapper(raw, encoding="utf-16", newline="\r\n") + with contextlib.redirect_stdout(translated), contextlib.redirect_stderr(io.StringIO()): + code = c.main(["--sbom", str(source), "--consumer", "sharp", "--receipt", str(receipt)]) + translated.flush() + output_bytes = raw.getvalue() + self.assertEqual(0, code) + self.assertEqual(hashlib.sha256(output_bytes).hexdigest(), json.loads(receipt.read_text(encoding="utf-8"))["export_sha256"]) + self.assertNotIn(b"\r", output_bytes) + self.assertIsInstance(json.loads(output_bytes.decode("utf-8")), dict) + # Real redirected process stdout is captured as bytes on every CI OS. + receipt = Path(directory) / "process-receipt.json" + result = subprocess.run([sys.executable, c.__file__, "--sbom", str(source), "--consumer", "sharp", "--receipt", str(receipt)], + capture_output=True, timeout=15) + self.assertEqual(0, result.returncode, result.stderr) + self.assertEqual(hashlib.sha256(result.stdout).hexdigest(), json.loads(receipt.read_text(encoding="utf-8"))["export_sha256"]) + + def test_consumer_staging_interrupt_cleans_owned_temporary_but_retains_receipt(self): + source = Path(__file__).parent / "examples/inventory.sbom.json" + for force in (False, True): + with self.subTest(force=force), tempfile.TemporaryDirectory() as directory: + receipt = Path(directory) / "receipt.json" + output = Path(directory) / "consumer.json" + args = ["--sbom", str(source), "--consumer", "sharp", "--receipt", str(receipt), "--output", str(output)] + if force: + output.write_bytes(b"previous consumer evidence") + args.append("--force") + with mock.patch.object(c.os, "fsync", side_effect=[None, KeyboardInterrupt]) as sync: + code, text, _ = self.run_cli(args, datetime(2026, 1, 2, tzinfo=timezone.utc)) + self.assertEqual((130, ""), (code, text)) + self.assertEqual(2, sync.call_count) # No second attempt to publish the pair. + self.assertTrue(receipt.is_file()) # The committed receipt remains useful evidence. + envelope = json.loads(receipt.read_text(encoding="utf-8")) + expected = c.serialized(c.export_inventory(envelope["inventory"], "sharp")).encode("utf-8") + self.assertEqual(hashlib.sha256(expected).hexdigest(), envelope["export_sha256"]) + self.assertEqual([], list(Path(directory).glob(".oci-report-*"))) + if force: + self.assertEqual(b"previous consumer evidence", output.read_bytes()) + else: + self.assertFalse(output.exists()) + + +if __name__ == "__main__": + unittest.main()