From 500d5815006c008fdd308bef2ebfe9d7a3de1aa9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?G=C3=A1bor=20Z=20Sink=C3=B3?= Date: Sun, 1 Nov 2026 22:47:18 +0100 Subject: [PATCH 1/6] test(release): freeze the envelope 3 conformance yardstick at its merge commit The digests of the reference verifier, the golden fixtures, the vectors and the planted-defect proof are listed as computed from the blobs of d2d8ec7 (not regenerated), and checked against the working tree and, where the history is present, against that commit. A compiler change cannot move the yardstick it is measured against without a red test. --- [signing-metadata] key = cic-my-sign-key signature = vault:v1:MEYCIQCxVh9enWULFDjboyKsf7Z/wzJyh8U6BPaNMMzmQyeY0QIhAI4nowsZ0fD1R5TQ1MaAaiDmgt67ZgqlGwtlBCqk2dzt hash-algorithm = sha256 digest = iRXaqJUMYnIh58/E+QwG4aeD1UeohBCh1sKWG0/z1I8= [certificate] -----BEGIN CERTIFICATE----- MIICBjCCAaygAwIBAgIUSnRMR6RPnEbg296XWPOqq/u5PCwwCgYIKoZIzj0EAwIw QzELMAkGA1UEBhMCSFUxGTAXBgNVBAoMEENlbnRyYWxJbmZyYUNvcmUxGTAXBgNV BAMMEENJQyBEZXZlbG9wZXIgQ0EwHhcNMjYwMzIwMTMyMjU5WhcNMjYxMjMxMTMy MjU5WjBFMQswCQYDVQQGEwJIVTEZMBcGA1UECgwQQ2VudHJhbEluZnJhQ29yZTEb MBkGA1UEAwwSR2Fib3IgWm9sdGFuIFNpbmtvMFkwEwYHKoZIzj0CAQYIKoZIzj0D AQcDQgAEIG2CVmTfmLB9pLLclj7YmP2eedAjklpy4LGrU2ijoiy6Xqpuybv7OgJe i+ez31s65NEV8+X/ByeX1cstR988z6N8MHowCQYDVR0TBAIwADAdBgNVHQ4EFgQU yZN6AIX/TNnIJ9GwAa/NRN3ujHAwHwYDVR0jBBgwFoAUXn6CHYzPUqU4JVP8g+OS WeDYjhcwDgYDVR0PAQH/BAQDAgeAMB0GA1UdJQQWMBQGCCsGAQUFBwMCBggrBgEF BQcDBDAKBggqhkjOPQQDAgNIADBFAiEA+bFzXRoJ4PCQbhAAtpkcMjt0vNj5rEW0 lOMBGDNyaWkCIB1vmM7PcZzv/c9bIrxF5kqv6QXomouhByUfeNUTbpKW -----END CERTIFICATE----- --- tests/fixtures/envelope3/FROZEN.sha256 | 11 +++++ tests/test_conformance_frozen.py | 56 ++++++++++++++++++++++++++ 2 files changed, 67 insertions(+) create mode 100644 tests/fixtures/envelope3/FROZEN.sha256 create mode 100644 tests/test_conformance_frozen.py diff --git a/tests/fixtures/envelope3/FROZEN.sha256 b/tests/fixtures/envelope3/FROZEN.sha256 new file mode 100644 index 0000000..52ee0fa --- /dev/null +++ b/tests/fixtures/envelope3/FROZEN.sha256 @@ -0,0 +1,11 @@ +6e9f14d99768aa28c734e258d58f6bc6d11438c3149e005bb5b20ad9d1742560 tools/release_record_conformance.py +def32d95e8c57cb2189153fc2284158d814a08efbae90b7603f8810ded34ac2e tests/e3_fixtures.py +dd409c96d6202ba84549e1c991ee97f0481bb234e1883ca57741c5eaa6b27860 tests/e3_vectors.py +5de8fa3d9d1d0855d87454b9c99f3f493b03898871813d0387c06fc9d5cdbf2a tests/e3_mutants.py +ab0960738a47b90c9e5ceebd8929c5bfdc552a66ba3b05c83780314ae4987e6f tests/e3_generate_golden.py +ff7e3cf1d9f0b2dcdee363ccaa298e11dfdaeecc39c9b0307e9298c71378fc05 tests/fixtures/envelope3/golden-record.yaml +0151a844797fb4f198c450363c63baa6c10f4ef273000dcf0c72aff470ae3741 tests/fixtures/envelope3/golden.canonical.json +c800f9189ea36851b63bc8667c1deaa574f7ebb3e25d42f3ca1b82ab7a27650b tests/fixtures/envelope3/golden.expected.json +11d338065060296fc84c2eb638dbf948d20788ad751648e1c4973c31fad37f2d tests/test_release_record_conformance.py +0d0a2e73fc3d00f895079736bdf024ddc638e327ba34bf169c3f5d96b46fd79d tests/test_release_record_conformance_planted.py +284e44bfcf62d79f782d6cae359c4c99ef2f5297f6f71dcea52a95d28b8c4456 release/cic-primitives-v0.1.5.yaml diff --git a/tests/test_conformance_frozen.py b/tests/test_conformance_frozen.py new file mode 100644 index 0000000..276dd7c --- /dev/null +++ b/tests/test_conformance_frozen.py @@ -0,0 +1,56 @@ +"""The yardstick may not move under the compiler. + +The reference verifier, the golden fixtures, the vectors and the planted-defect proof of the envelope 3 conformance (PR #25) are frozen at the state that was +accepted and merged, d2d8ec7. tests/fixtures/envelope3/FROZEN.sha256 lists their SHA-256 digests as they were computed from the git objects of that merge commit; +the list is NOT regenerated from the working tree. A change to any of them (to make a compiler change pass, for instance) is a red test here and has to be made +in a change of its own that updates the list, where a reviewer sees it. + +Two readings: the working tree against the list (always), and, where the history is present (the gate fetches all of it), the list against the blobs of the merge +commit itself, so the list cannot be edited together with a file without the mismatch showing. +""" +import hashlib +import os +import pathlib +import subprocess + +import pytest + +ROOT = pathlib.Path(__file__).resolve().parents[1] +LIST = ROOT / "tests" / "fixtures" / "envelope3" / "FROZEN.sha256" +MERGE_COMMIT = "d2d8ec7422490053644779d3cca9fce388bb2886" +REQUIRED = os.environ.get("CIC_REQUIRE_RELEASE_HISTORY") == "1" + + +def frozen(): + out = {} + for line in LIST.read_text(encoding="utf-8").splitlines(): + digest, _sp, path = line.partition(" ") + out[path] = digest + return out + + +def has_commit(commit): + return subprocess.run(["git", "-C", str(ROOT), "cat-file", "-e", commit + "^{commit}"], capture_output=True).returncode == 0 + + +def test_the_list_names_the_yardstick(): + names = set(frozen()) + assert {"tools/release_record_conformance.py", "tests/e3_vectors.py", "tests/e3_fixtures.py", "tests/fixtures/envelope3/golden-record.yaml", + "tests/fixtures/envelope3/golden.expected.json", "tests/fixtures/envelope3/golden.canonical.json", "tests/test_release_record_conformance.py", + "tests/test_release_record_conformance_planted.py", "tests/e3_mutants.py", "release/cic-primitives-v0.1.5.yaml"} <= names + assert all(len(d) == 64 for d in frozen().values()) + + +@pytest.mark.parametrize("path", sorted(frozen())) +def test_the_working_tree_matches_the_frozen_digest(path): + assert hashlib.sha256((ROOT / path).read_bytes()).hexdigest() == frozen()[path], ( + "%s differs from the state accepted in %s: change the yardstick in a change of its own, not together with the compiler" % (path, MERGE_COMMIT[:7])) + + +@pytest.mark.parametrize("path", sorted(frozen())) +def test_the_frozen_digest_is_that_of_the_merge_commit(path): + if not has_commit(MERGE_COMMIT): + assert not REQUIRED, "CIC_REQUIRE_RELEASE_HISTORY=1 but %s is not in this clone (fetch-depth: 0)" % MERGE_COMMIT + pytest.skip("the merge commit is not in this clone") + blob = subprocess.run(["git", "-C", str(ROOT), "show", "%s:%s" % (MERGE_COMMIT, path)], capture_output=True, check=True).stdout + assert hashlib.sha256(blob).hexdigest() == frozen()[path] From b8584bde7a1ff19f0059a32ce87d4b0f6e667d17 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?G=C3=A1bor=20Z=20Sink=C3=B3?= Date: Sun, 1 Nov 2026 22:47:18 +0100 Subject: [PATCH 2/6] feat(release): the compiler writes envelope 3 Mandatory source_commit; the producer's explicit release_artifacts.paths, read from the source commit and resolved against its git tree only; per-artifact sha256: digests of the blob bytes, sorted by UTF-8 bytes, with the manifest digest; unique spec source paths; the declaration is refused before Vault is asked for anything. build_hash, release.sign and cic_countersign are the existing mechanism. verify-release reads envelope 1, 2 and 3 and leaves the artifact list to the conformance check; neither imports the other. The record the compiler writes is read back by the reference verifier and put through the golden coverage vectors; tests were written first and shown red. Writer-side tests of envelope 2 now assert envelope 3. --- [signing-metadata] key = cic-my-sign-key signature = vault:v1:MEUCIQCRK1t2AEnbQcM9C8c9eVH5NrKBOHJQKvzW4TcfJmQ1ygIgR0aZLGq4U9DKVYMXEgYENrXwdr3/lomezAz0xdFyzks= hash-algorithm = sha256 digest = i8VLtf9HEgyfzMnvnfzovdG3Q4PN5+22bHAb4+Wmh88= [certificate] -----BEGIN CERTIFICATE----- MIICBjCCAaygAwIBAgIUSnRMR6RPnEbg296XWPOqq/u5PCwwCgYIKoZIzj0EAwIw QzELMAkGA1UEBhMCSFUxGTAXBgNVBAoMEENlbnRyYWxJbmZyYUNvcmUxGTAXBgNV BAMMEENJQyBEZXZlbG9wZXIgQ0EwHhcNMjYwMzIwMTMyMjU5WhcNMjYxMjMxMTMy MjU5WjBFMQswCQYDVQQGEwJIVTEZMBcGA1UECgwQQ2VudHJhbEluZnJhQ29yZTEb MBkGA1UEAwwSR2Fib3IgWm9sdGFuIFNpbmtvMFkwEwYHKoZIzj0CAQYIKoZIzj0D AQcDQgAEIG2CVmTfmLB9pLLclj7YmP2eedAjklpy4LGrU2ijoiy6Xqpuybv7OgJe i+ez31s65NEV8+X/ByeX1cstR988z6N8MHowCQYDVR0TBAIwADAdBgNVHQ4EFgQU yZN6AIX/TNnIJ9GwAa/NRN3ujHAwHwYDVR0jBBgwFoAUXn6CHYzPUqU4JVP8g+OS WeDYjhcwDgYDVR0PAQH/BAQDAgeAMB0GA1UdJQQWMBQGCCsGAQUFBwMCBggrBgEF BQcDBDAKBggqhkjOPQQDAgNIADBFAiEA+bFzXRoJ4PCQbhAAtpkcMjt0vNj5rEW0 lOMBGDNyaWkCIB1vmM7PcZzv/c9bIrxF5kqv6QXomouhByUfeNUTbpKW -----END CERTIFICATE----- --- docs/en/RELEASE-COMPILER-ENVELOPE3.md | 40 +++ docs/hu/RELEASE-COMPILER-ENVELOPE3.md | 12 + project.schema.yaml | 13 + release.schema.yaml | 59 +++- tests/e3_compiler_support.py | 146 ++++++++ tests/test_compiler.py | 83 ++--- tests/test_compiler_envelope3.py | 477 ++++++++++++++++++++++++++ tools/compiler.py | 204 ++++++++++- 8 files changed, 965 insertions(+), 69 deletions(-) create mode 100644 docs/en/RELEASE-COMPILER-ENVELOPE3.md create mode 100644 docs/hu/RELEASE-COMPILER-ENVELOPE3.md create mode 100644 tests/e3_compiler_support.py create mode 100644 tests/test_compiler_envelope3.py diff --git a/docs/en/RELEASE-COMPILER-ENVELOPE3.md b/docs/en/RELEASE-COMPILER-ENVELOPE3.md new file mode 100644 index 0000000..1edb8c3 --- /dev/null +++ b/docs/en/RELEASE-COMPILER-ENVELOPE3.md @@ -0,0 +1,40 @@ +# The compiler writes envelope 3 + +`python tools/compiler.py release` writes a `PrimitiveRelease` record of **envelope 3** (docs/en/RELEASE-RECORD-CONTRACT.md) and nothing else. Envelope 1 and 2 records are not written any more, and are not touched: `verify-release` still reads them exactly as before, and the real records `release/cic-primitives-v0.1.5.yaml` (envelope 1) and `release/cic-primitives-v0.2.0.yaml` (envelope 2) are test cases. No record is rewritten, re-signed or migrated. + +## What changes in a release + +1. `provenance.source_commit` is mandatory: `git rev-parse HEAD`, 40 lowercase hexadecimal digits. If git cannot say, the compiler stops; it no longer records `null`. +2. The producer declares the files it publishes in `project.yaml`: + + ```yaml + compiler_settings: + release_artifacts: + paths: # explicit, relative to the repository root; no glob, no include, no exclude + - LICENSE.md + - schemas/atomic/shape.yaml + ``` + + The declaration is read from **the source commit** (`project.yaml` as that commit holds it), and the paths are resolved against **the git tree of that commit**: not the working directory, not a checkout with its line-end filters, not a consumer's pin manifest. A path that is a directory, a symbolic link or a submodule is refused, as is an absolute path, a `..` or `.` segment, a control character, a repeated path, a missing path, an empty list, a key other than `paths`. The compiler stops with the reason (`declaration_missing`, `declaration_empty`, `declaration_unknown_key`, `declaration_path_form`, `declaration_path_duplicate`, `declared_path_absent`, `declared_path_not_regular`) **before** it asks Vault for anything. +3. `artifacts` is written: for each declared path `sha256:` and the hex SHA-256 of the blob bytes at the source commit, sorted by the UTF-8 bytes of the path, and `manifest_digest` over the sha256sum-format lines. `specs[].source_path` must be unique. +4. `build_hash`, `release.sign` and (outside the compiler) `cic_countersign` are the existing mechanism: `build_hash` covers the whole bundle except `release.sign`, `release.build_hash` and `cic_countersign`, so `artifacts` and `source_commit` are covered without any change to it. + +A repository whose `project.yaml` has no `release_artifacts` cannot cut a release: it stops with `declaration_missing`. Which files a repository publishes is the producer's decision and is made in `project.yaml`, in a change of its own. + +## How the output is held to the contract + +The compiler does not say that its own output is right. + +* The record it writes is read back by the reference verifier (`tools/release_record_conformance.py`), which does not import the compiler's new functions (and the compiler does not import it): `record`, `blobs` and `crypto` layers, and the `artifacts` block is compared with values recomputed outside this code base (`tests/fixtures/envelope3`). +* The 32 signature coverage vectors of the contract run on the record the compiler wrote, not only on the golden record; the structural vectors too. +* The declaration vectors run on the compiler's own functions, and on the reference's, and the two must agree. +* The yardstick of PR #25 (the reference verifier, the golden fixtures, the vectors, the planted-defect proof) is frozen at its merge commit: `tests/test_conformance_frozen.py` fails if any of it changes together with the compiler. +* 58 planted defects in `tools/compiler.py` (`tests/e3_compiler_mutants.py`): 50 have to turn one of the tests that target them red, 8 cannot change behaviour, are listed with the reason and have to survive. + +## `verify-release` + +Accepts envelope 3 (build hash, signature, counter-signature, as for envelope 2) and says what it leaves to the conformance check: the artifact list is **not** checked there. The two are separate: `verify-release` is the cryptographic verification, `tools/release_record_conformance.py` is the check against the envelope 3 contract. A release policy may require both; neither imports the other. + +## The counter-signature service + +Read-only audit of `cic-countersign` (`internal/countersign/handler.go`, `devel` 8ce2427): the service takes `build_hash` as an opaque, already computed digest ("trusted as given, never recomputed") and verifies the author's signature over it. The submitted `document` is carried through for the audit log only (`kind` and `version` are logged) and is not hashed or canonicalized. It reads nothing of the envelope or of `artifacts`. Nothing there depends on this change; the service was not modified. diff --git a/docs/hu/RELEASE-COMPILER-ENVELOPE3.md b/docs/hu/RELEASE-COMPILER-ENVELOPE3.md new file mode 100644 index 0000000..9129cc5 --- /dev/null +++ b/docs/hu/RELEASE-COMPILER-ENVELOPE3.md @@ -0,0 +1,12 @@ +# A compiler envelope 3-at ír + +A `python tools/compiler.py release` kizárólag **envelope 3** `PrimitiveRelease` rekordot ír (docs/en/RELEASE-RECORD-CONTRACT.md). Az envelope 1 és 2 rekordot nem ír többé, a meglévőket nem érinti: a `verify-release` változatlanul olvassa őket, a valódi `cic-primitives-v0.1.5.yaml` (envelope 1) és `cic-primitives-v0.2.0.yaml` (envelope 2) tesztesetek. Rekord átírása, újraaláírása vagy migrációja nincs. + +* **`source_commit` kötelező** (`git rev-parse HEAD`, 40 kisbetűs hex); ha a git nem tud felelni, a compiler megáll, nem ír `null`-t. +* **A termelő deklarálja a közzétett fájlokat** a `project.yaml`-ban (`compiler_settings.release_artifacts.paths`, explicit útvonalak, glob/`include`/`exclude` nélkül). A deklarációt a **forráscommit**ból olvassa, az útvonalakat a commit **Git-fájából** oldja fel; nem a munkakönyvtárból, nem szűrt checkoutból, nem fogyasztói pin-manifestből. Könyvtár, link, submodule, abszolút útvonal, `..`/`.`, vezérlőkarakter, duplikátum, hiányzó útvonal, üres lista, ismeretlen kulcs: leáll az okkal, **még a Vault hívása előtt**. +* **`artifacts`:** útvonalanként `sha256:` + a blob bájtjainak SHA-256-ja, az útvonal UTF-8 bájtjai szerint rendezve, plusz `manifest_digest`. A `specs[].source_path` egyedi. +* **Aláírás:** a meglévő mechanizmus (`build_hash` a teljes csomagra, `release.sign`, külső `cic_countersign`); az `artifacts` és a `source_commit` ezért lefedett. +* **Nem a compiler igazolja magát:** a kimenetet a referencia-verifier olvassa vissza (nem importálják egymást), a 32 lefedettségi vektor a compiler rekordján is lefut, a deklarációs vektorok a compiler saját függvényén, és a #25 mércéje befagyasztott (`tests/test_conformance_frozen.py`). 58 planted defect a `tools/compiler.py`-ban. +* **`verify-release`:** elfogadja az envelope 3-at, és kimondja, hogy az artifact-lista ellenőrzése nem az ő dolga (`release_record_conformance.py`); a kettő különálló, egyik sem importálja a másikat. +* **Ellenjegyző szolgáltatás (csak olvasó vizsgálat):** a `cic-countersign` a `build_hash`-t átlátszatlan, kész digestként kapja és nem számolja újra; a `document` csak naplóhoz; az envelope-ból és az `artifacts`-ból semmit nem olvas. Nem módosítottam. +* **Deklaráció nélküli repó nem adhat ki release-t** (`declaration_missing`); a közzétett fájlok listája a termelő döntése, külön változásban. diff --git a/project.schema.yaml b/project.schema.yaml index 9ab94e8..226bbb6 100644 --- a/project.schema.yaml +++ b/project.schema.yaml @@ -99,6 +99,19 @@ properties: vault_cert_path: type: string description: "Vault KV v2 path to the signing certificate. Format: 'mount/secret-name/key'. Overridden by VAULT_CERT env var." + release_artifacts: + type: object + description: "The files a release publishes (docs/en/RELEASE-RECORD-CONTRACT.md, section 6): explicit paths relative to the repository root, resolved only against the git tree of the source commit. No globs, no include or exclude. Read from the source commit, not from the working directory." + additionalProperties: false + required: + - paths + properties: + paths: + type: array + minItems: 1 + items: + type: string + minLength: 1 x-cic: type: object diff --git a/release.schema.yaml b/release.schema.yaml index 699f8a3..48bdf26 100644 --- a/release.schema.yaml +++ b/release.schema.yaml @@ -123,9 +123,12 @@ properties: enum: - 1 - 2 + - 3 description: "Which payload build_hash covers. 1 = the four members {createdBy, releasedBy, specs,\ \ validity} \u2014 legacy, leaves kind/version/timestamp unsigned. 2 = the whole bundle except\ - \ release.sign, release.build_hash and cic_countersign. Absent means 1." + \ release.sign, release.build_hash and cic_countersign. 3 = the same payload rule as 2, and\ + \ the bundle carries `artifacts` and a mandatory `provenance.source_commit` (docs/en/RELEASE-RECORD-CONTRACT.md).\ + \ Absent means 1." cic_countersign: type: object description: External CIC counter-signature over release.build_hash. Applied by CICSourceCA after @@ -184,3 +187,57 @@ properties: type: - string - 'null' + artifacts: + type: object + description: 'Envelope 3: a digest of every released file (docs/en/RELEASE-RECORD-CONTRACT.md, sections 6 to 8). sha256: and 64 + lowercase hexadecimal digits of the git blob bytes at provenance.source_commit; files sorted by the UTF-8 bytes of the path, unique; + manifest_digest over the sha256sum-format lines. This schema checks the shape only; tools/release_record_conformance.py is the reference check.' + required: + - manifest_digest + - files + properties: + manifest_digest: + type: string + pattern: ^sha256:[0-9a-f]{64}$ + files: + type: array + minItems: 1 + items: + type: object + required: + - path + - digest + properties: + path: + type: string + minLength: 1 + digest: + type: string + pattern: ^sha256:[0-9a-f]{64}$ +allOf: +- description: An envelope 3 record carries artifacts and a source_commit that is 40 lowercase hexadecimal digits (never null). + if: + required: + - release + properties: + release: + required: + - envelope + properties: + envelope: + const: 3 + then: + required: + - provenance + - artifacts + properties: + provenance: + required: + - envelope + - source_commit + properties: + envelope: + const: 3 + source_commit: + type: string + pattern: ^[0-9a-f]{40}$ diff --git a/tests/e3_compiler_support.py b/tests/e3_compiler_support.py new file mode 100644 index 0000000..7161b52 --- /dev/null +++ b/tests/e3_compiler_support.py @@ -0,0 +1,146 @@ +"""Support for the tests of the compiler's envelope 3 output (tests/test_compiler_envelope3.py). + +The compiler is loaded through `load_compiler`, which the planted-defect runs point (E3_COMPILER) at a mutated copy. Everything the vectors expect comes from the frozen +fixtures (tests/e3_fixtures.py, tests/e3_vectors.py, tests/fixtures/envelope3), which the compiler change does not touch (tests/test_conformance_frozen.py). +""" +import datetime +import importlib.util +import os +import pathlib +import subprocess +import types + +import yaml + +import e3_fixtures as fx + +ROOT = fx.ROOT +FIXED_NOW = datetime.datetime(2026, 11, 1, 12, 0, 0) + + +def load_compiler(): + override = os.environ.get("E3_COMPILER") + if override: + spec = importlib.util.spec_from_file_location("compiler_under_test", override) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + from tools import compiler + return compiler + + +class _FixedDatetime(datetime.datetime): + @classmethod + def now(cls, tz=None): + return cls(FIXED_NOW.year, FIXED_NOW.month, FIXED_NOW.day, FIXED_NOW.hour, tzinfo=tz or datetime.timezone.utc) + + +def fix_the_clock(monkeypatch, module): + """The compiler reads the clock for the release timestamp and for certificate and commitment dates; a historical record is checked as of a day it was valid.""" + monkeypatch.setattr(module, "datetime", types.SimpleNamespace( + datetime=_FixedDatetime, date=datetime.date, timezone=datetime.timezone, timedelta=datetime.timedelta)) + + +def project_yaml(paths): + return yaml.dump({"project": {"name": "cic-primitives", "owner": "Dev"}, + "compiler_settings": {"release_artifacts": {"paths": list(paths)}}}, sort_keys=False) + + +def checkout(path, files): + """A repository at `path` whose HEAD is one commit holding `files` [(path, bytes, mode)], and whose working directory holds the same files.""" + commit = fx.build_repo(path, files) + fx._git(path, "update-ref", "refs/heads/main", commit) + fx._git(path, "symbolic-ref", "HEAD", "refs/heads/main") + for name, content, mode in files: + if mode in ("100644", "100755"): + target = pathlib.Path(path) / name + target.parent.mkdir(parents=True, exist_ok=True) + target.write_bytes(content) + return commit + + +def golden_paths(): + return [p for p, _c, _m in fx.GOLDEN_FILES] + + +def commit_files(paths): + """The golden files plus the project.yaml that declares `paths`.""" + return list(fx.GOLDEN_FILES) + [("project.yaml", project_yaml(paths).encode("utf-8"), "100644")] + + +def release_sandbox(C, tmp_path, monkeypatch, version="0.1.6", files=None, paths=None): + """What tests/test_compiler.py::_release_sandbox did, as a real git checkout: run_release() resolves the artifacts from the git tree of HEAD, so the sandbox has one. + + Vault and the developer commitment are replaced; the issuer is the test issuer of the frozen fixtures, so a record this compiler writes can be re-authenticated by the + same test keys as the golden record. Returns (directory, version, source_commit, calls) where `calls` counts the Vault calls. + """ + import test_compiler as tc + paths = golden_paths() if paths is None else paths + files = commit_files(paths) if files is None else files + commit = checkout(tmp_path, files) + repo = ROOT + (tmp_path / "proposals").mkdir(exist_ok=True) + (tmp_path / "proposals" / "atom-grammar").symlink_to(repo / "proposals" / "atom-grammar", target_is_directory=True) + (tmp_path / "schemas" / "atomic").mkdir(parents=True) + (tmp_path / "schemas" / "atomic" / "shape.yaml").write_text(tc._META.format(name="Shape", version="v0.1.dev") + "spec:\n kind: AtomicPrimitive\n fields: {a: 1}\n") + (tmp_path / "schemas" / "examples").mkdir() + (tmp_path / "schemas" / "examples" / "c.yaml").write_text(tc._META.format(name="Example", version="v0.1.dev") + tc._GOOD_COMPOSITION.lstrip("\n")) + (tmp_path / "dependency.yaml").write_text("schema_version: '1'\n") + (tmp_path / "md.meta.schema.yaml").write_text((repo / "md.meta.schema.yaml").read_text()) + (tmp_path / "schemas" / "index.yaml").write_text((repo / "schemas" / "index.yaml").read_text()) + monkeypatch.chdir(tmp_path) + + calls = {"vault": 0} + issuer_cert = fx.certificate("issuer", "root", 12) + developer_cert = fx.certificate("developer", "root", 11) + validity = {"from": str(FIXED_NOW.date()), "until": "2099-01-01"} + commitment = {"kind": "DeveloperCommitment", "validity": validity, + "createdBy": {"name": "Dev", "email": "d@e", "certificate": developer_cert}} + + def vault_cert(*a, **k): + calls["vault"] += 1 + return issuer_cert + + def vault_sign(digest, *a, **k): + calls["vault"] += 1 + return fx.vault_signature("issuer", digest) + + settings = dict(tc._RELEASE_SETTINGS) + monkeypatch.setenv("VAULT_ADDR", "https://vault") + monkeypatch.setenv("VAULT_TOKEN", "token") + monkeypatch.setattr(C, "validate_release_prerequisites", lambda: (version, "primitives/")) + monkeypatch.setattr(C, "_load_valid_commitment", lambda require_signature=True: commitment) + monkeypatch.setattr(C, "_vault_get_cert", vault_cert) + monkeypatch.setattr(C, "_vault_sign", vault_sign) + # the settings of the WORKING DIRECTORY'S project.yaml: deliberately without the declaration, which has to come from the commit + monkeypatch.setattr(C, "load_project_config", lambda full_config=False: ( + {"project": {"name": "cic-primitives", "owner": "Dev"}, "compiler_settings": settings} if full_config else settings)) + monkeypatch.setattr(C, "CONFIG", settings) + return tmp_path, version, commit, calls + + +def written_record(directory, version): + return yaml.safe_load((pathlib.Path(directory) / "release" / ("cic-primitives-v%s.yaml" % version)).read_text(encoding="utf-8")) + + +def countersign(bundle): + """What the external service does: sign the build_hash it is given, with its own key, and return the block (tests only; the service is not touched).""" + bundle["cic_countersign"] = { + "authority": {"name": "Conformance Fixture Authority", "certificate": fx.certificate("authority", "root", 13), + "root_certificate": fx.certificate("root", "root", 10, ca=True)}, + "signed_payload": "build_hash", + "sign": fx.vault_signature("authority", bundle["release"]["build_hash"]), + } + return bundle + + +def git_tree_via_plumbing(repo, commit): + """An independent reading of the tree, for the comparison with the compiler's own listing.""" + out = subprocess.run(["git", "-C", str(repo), "ls-tree", "-r", "-t", "-z", "--full-tree", commit], capture_output=True, check=True).stdout + tree = {} + for record in out.split(b"\0"): + if record: + meta, _t, name = record.partition(b"\t") + mode, kind, oid = meta.decode().split(" ") + tree[name.decode("utf-8", "surrogateescape")] = (mode, kind, oid) + return tree diff --git a/tests/test_compiler.py b/tests/test_compiler.py index ea2af04..bd926c2 100644 --- a/tests/test_compiler.py +++ b/tests/test_compiler.py @@ -1,5 +1,6 @@ import pytest import pathlib +import re from tools import compiler from _helpers import cert_subject_name, make_test_key_and_cert, sign_hash @@ -858,18 +859,25 @@ def test_fingerprint_is_stable_and_distinguishing(): # Provenance block: what the signature covers about the build's origin # --------------------------------------------------------------------------- +def _git_checkout(path): + """A git repository at `path`: since envelope 3 the source commit is mandatory, so provenance is only collected inside one.""" + import e3_compiler_support as sup + import e3_fixtures as fx + sup.checkout(path, list(fx.GOLDEN_FILES)) + + def test_collect_provenance_records_the_source_and_digests(): p = compiler._collect_provenance() assert p["envelope"] == compiler.RELEASE_ENVELOPE_VERSION - # In a git checkout this is a real commit id; the point is that the field - # exists so the signature covers it either way. - assert "source_commit" in p + # Since envelope 3 it is always a real commit id (the compiler stops without one). + assert re.fullmatch(r"[0-9a-f]{40}", p["source_commit"]) for key in ("dependency_lock_sha256", "grammar_sha256", "grammar_schema_sha256"): assert key in p def test_collect_provenance_digest_follows_the_file(tmp_path, monkeypatch): """A changed dependency lock must change the digest the release signs.""" + _git_checkout(tmp_path) monkeypatch.chdir(tmp_path) (tmp_path / "dependency.yaml").write_text("a: 1\n") first = compiler._collect_provenance()["dependency_lock_sha256"] @@ -879,6 +887,7 @@ def test_collect_provenance_digest_follows_the_file(tmp_path, monkeypatch): def test_collect_provenance_survives_a_missing_file(tmp_path, monkeypatch): + _git_checkout(tmp_path) monkeypatch.chdir(tmp_path) p = compiler._collect_provenance() assert p["dependency_lock_sha256"] is None @@ -1013,79 +1022,45 @@ def test_vault_get_cert_rejects_a_malformed_path(): def _release_sandbox(tmp_path, monkeypatch, mocker, version="0.1.6"): - """A tree run_release() can complete in without Vault, git or a network. + """A tree run_release() can complete in without Vault or a network. Worth the setup: run_release had 104 of its 120 lines uncovered, and it is the function that decides what a signed artifact contains. Every property - asserted below — that the envelope is v2, that the hash covers the whole + asserted below — that the envelope is v3, that the hash covers the whole bundle, that provenance is inside it — was previously only checked by reading the code. - """ - repo = pathlib.Path(compiler.__file__).resolve().parent.parent - (tmp_path / "proposals").mkdir() - (tmp_path / "proposals" / "atom-grammar").symlink_to( - repo / "proposals" / "atom-grammar", target_is_directory=True) - (tmp_path / "schemas" / "atomic").mkdir(parents=True) - (tmp_path / "schemas" / "atomic" / "shape.yaml").write_text( - _META.format(name="Shape", version="v0.1.dev") - + "spec:\n kind: AtomicPrimitive\n fields: {a: 1}\n") - (tmp_path / "schemas" / "examples").mkdir() - (tmp_path / "schemas" / "examples" / "c.yaml").write_text( - _META.format(name="Example", version="v0.1.dev") + _GOOD_COMPOSITION.lstrip("\n")) - (tmp_path / "dependency.yaml").write_text("schema_version: '1'\n") - # run_release() runs the three older validations first; they need the two - # meta-schemas and a docs tree, so the sandbox carries the real ones. - repo_root = pathlib.Path(compiler.__file__).resolve().parent.parent - for name in ("md.meta.schema.yaml",): - (tmp_path / name).write_text((repo_root / name).read_text()) - (tmp_path / "schemas" / "index.yaml").write_text( - (repo_root / "schemas" / "index.yaml").read_text()) - monkeypatch.chdir(tmp_path) - dev_key, dev_cert = make_test_key_and_cert("Dev") - rel_key, rel_cert = make_test_key_and_cert("Releaser") - today = datetime.datetime.now(datetime.timezone.utc).date() - validity = {"from": str(today), "until": "2099-01-01"} - created_by = {"name": "Dev", "email": "d@e", "certificate": dev_cert} - commitment = { - "kind": "DeveloperCommitment", "validity": validity, "createdBy": created_by, - } - - monkeypatch.setenv("VAULT_ADDR", "https://vault") - monkeypatch.setenv("VAULT_TOKEN", "token") - mocker.patch("tools.compiler.validate_release_prerequisites", - return_value=(version, "primitives/")) - mocker.patch("tools.compiler._load_valid_commitment", return_value=commitment) - mocker.patch("tools.compiler._vault_get_cert", return_value=rel_cert) - mocker.patch("tools.compiler._vault_sign", - side_effect=lambda digest, *a, **k: sign_hash(rel_key, digest)) - mocker.patch("tools.compiler.load_project_config", side_effect=lambda full_config=False: ( - {"project": {"name": "cic-primitives", "owner": "Dev"}, - "compiler_settings": _RELEASE_SETTINGS} - if full_config else _RELEASE_SETTINGS)) - return tmp_path, version + Since envelope 3 the sandbox is a real git checkout: the artifacts are + resolved from the git tree of HEAD, and the source commit is not optional. + The set-up lives in tests/e3_compiler_support.py, where the envelope 3 tests + share it. + """ + import e3_compiler_support as sup + root, version, _commit, _calls = sup.release_sandbox(compiler, tmp_path, monkeypatch, version=version) + return root, version -def test_release_writes_an_envelope_v2_bundle(tmp_path, monkeypatch, mocker): +def test_release_writes_an_envelope_v3_bundle(tmp_path, monkeypatch, mocker): root, version = _release_sandbox(tmp_path, monkeypatch, mocker) compiler.run_release() artifact = root / "release" / f"cic-primitives-v{version}.yaml" assert artifact.is_file() bundle = yaml.safe_load(artifact.read_text()) assert bundle["kind"] == "PrimitiveRelease" - assert bundle["release"]["envelope"] == compiler.RELEASE_ENVELOPE_VERSION == 2 - assert bundle["provenance"]["envelope"] == 2 - assert "source_commit" in bundle["provenance"] + assert bundle["release"]["envelope"] == compiler.RELEASE_ENVELOPE_VERSION == 3 + assert bundle["provenance"]["envelope"] == 3 + assert re.fullmatch(r"[0-9a-f]{40}", bundle["provenance"]["source_commit"]) + assert bundle["artifacts"]["files"] def test_release_hash_covers_the_whole_bundle(tmp_path, monkeypatch, mocker): - """Recomputing the v2 payload over the written artifact must reproduce it.""" + """Recomputing the v3 payload (the v2 rule) over the written artifact must reproduce it.""" root, version = _release_sandbox(tmp_path, monkeypatch, mocker) compiler.run_release() bundle = yaml.safe_load( (root / "release" / f"cic-primitives-v{version}.yaml").read_text()) recomputed = compiler.get_sha256_b64(compiler.to_canonical_json( - compiler._release_hash_payload(bundle, 2))) + compiler._release_hash_payload(bundle, 3))) assert recomputed == bundle["release"]["build_hash"] diff --git a/tests/test_compiler_envelope3.py b/tests/test_compiler_envelope3.py new file mode 100644 index 0000000..5126b4f --- /dev/null +++ b/tests/test_compiler_envelope3.py @@ -0,0 +1,477 @@ +"""The compiler writes envelope 3 (docs/en/RELEASE-RECORD-CONTRACT.md), and is held to the frozen yardstick of PR #25. + +The compiler does not certify its own output here. What it writes is read back by the reference verifier (tools/release_record_conformance.py, which does not import +the compiler's new functions), compared with values that were recomputed outside this code base (tests/fixtures/envelope3), and put through the same vectors as the +golden record. The planted-defect proof that these tests can fail is tests/test_compiler_envelope3_planted.py. +""" +import ast +import copy +import datetime +import itertools +import json +import pathlib +import re +import subprocess +import sys + +import pytest +import yaml +from jsonschema import ValidationError, validate + +sys.path.insert(0, str(pathlib.Path(__file__).resolve().parent)) +import e3_compiler_support as sup # noqa: E402 +import e3_fixtures as fx # noqa: E402 +import e3_vectors as vec # noqa: E402 + +C = sup.load_compiler() +V = fx.load_verifier() +ROOT = fx.ROOT + + +@pytest.fixture(scope="module") +def repos(tmp_path_factory): + base = tmp_path_factory.mktemp("e3compiler") + golden = fx.build_repo(base / "golden", fx.GOLDEN_FILES) + special = fx.build_repo(base / "special", fx.SPECIAL_FILES) + return {"golden": (base / "golden", golden), "special": (base / "special", special)} + + +def run_release_in(directory, monkeypatch, C=C): + monkeypatch.chdir(directory) + C.run_release() + + +# ---------------------------------------------------------------- the compiler's reading of the git tree and its declaration rules + +def test_the_envelope_the_compiler_writes_is_3(): + assert C.RELEASE_ENVELOPE_VERSION == 3 + + +def test_the_hash_payload_of_envelope_3_is_that_of_envelope_2_and_unknown_ones_have_none(): + golden = fx.load_golden() + assert C._release_hash_payload(golden, 3) == C._release_hash_payload(golden, 2) + assert C._release_hash_payload(golden, 4) is None + assert C._release_hash_payload(golden, 0) is None + + +@pytest.mark.parametrize("which", ["golden", "special"]) +def test_the_compilers_tree_listing_is_the_references_and_the_plumbings(which, repos, monkeypatch): + path, commit = repos[which] + monkeypatch.chdir(path) + listing = C._git_tree(commit) + assert listing == V.load_tree(str(path), commit) == sup.git_tree_via_plumbing(path, commit) + + +@pytest.mark.parametrize("case", vec.DECLARATIONS, ids=lambda c: c[0]) +def test_the_compiler_refuses_a_declaration_for_the_reason_of_the_vector(case, repos, monkeypatch): + # the same table as the reference verifier's declaration vectors, run on the compiler's own function + _id, reason, project = case + path, commit = repos["special"] + monkeypatch.chdir(path) + refusal = C._validate_release_artifact_declaration(fx.expand(project, {}), C._git_tree(commit)) + assert (refusal[0] if refusal else None) == reason, refusal + + +def test_the_compiler_and_the_reference_agree_on_every_declaration_vector(repos, monkeypatch): + path, commit = repos["special"] + monkeypatch.chdir(path) + tree = C._git_tree(commit) + for _id, _reason, project in vec.DECLARATIONS: + project = fx.expand(project, {}) + ours = C._validate_release_artifact_declaration(project, tree) + theirs = V.validate_declaration(project, tree) + assert (ours and ours[0]) == (theirs and theirs[0]), _id + + +@pytest.mark.parametrize("order", list(itertools.permutations(range(3)))) +def test_the_same_declared_set_in_any_order_gives_the_same_artifacts_block(order, repos, monkeypatch): + path, commit = repos["golden"] + monkeypatch.chdir(path) + names = sup.golden_paths() + block = C._build_release_artifacts([names[i] for i in order], C._git_tree(commit)) + expected = fx.load_expected() + assert block == {"manifest_digest": expected["manifest_digest"], "files": expected["files"]} + # byte-identical on repeated runs + again = C._build_release_artifacts([names[i] for i in order], C._git_tree(commit)) + assert json.dumps(block, sort_keys=True) == json.dumps(again, sort_keys=True) + + +def test_the_listed_order_is_utf8_bytes_not_utf16_units(tmp_path, monkeypatch): + # U+FF5E (EF BD 9E) sorts before U+10000 (F0 90 80 80) in UTF-8; in UTF-16 the surrogate D800 would put U+10000 first + files = [(chr(0xFF5E) + ".txt", b"x\n", "100644"), (chr(0x10000) + ".txt", b"y\n", "100644")] + commit = fx.build_repo(tmp_path, files) + monkeypatch.chdir(tmp_path) + paths = [p for p, _c, _m in reversed(files)] + block = C._build_release_artifacts(paths, C._git_tree(commit)) + assert [f["path"] for f in block["files"]] == [chr(0xFF5E) + ".txt", chr(0x10000) + ".txt"] + record = {"artifacts": block} + assert V._check_order(record) is None + + +# ---------------------------------------------------------------- what run_release writes + +def test_run_release_writes_an_envelope_3_record_that_meets_the_contract(tmp_path, monkeypatch): + root, version, commit, calls = sup.release_sandbox(C, tmp_path, monkeypatch) + run_release_in(root, monkeypatch) + bundle = sup.written_record(root, version) + assert bundle["release"]["envelope"] == bundle["provenance"]["envelope"] == 3 + assert bundle["provenance"]["source_commit"] == commit + # the artifacts block equals the independently recomputed one (sha256sum of the worked example), not the output of another run of the compiler + expected = fx.load_expected() + assert bundle["artifacts"] == {"manifest_digest": expected["manifest_digest"], "files": expected["files"]} + report = V.verify(sup.countersign(bundle), str(root)) + assert report["record"].status == "PASS", (report["record"].reason, report["record"].detail) + assert report["blobs"].status == "PASS", report["blobs"].failures + assert report["crypto"]["status"] == "PASS" + assert V.first_reason(report) is None + + +def test_the_hash_the_compiler_wrote_is_the_independent_one(tmp_path, monkeypatch): + root, version, _commit, _calls = sup.release_sandbox(C, tmp_path, monkeypatch) + run_release_in(root, monkeypatch) + bundle = sup.written_record(root, version) + assert bundle["release"]["build_hash"] == fx.independent_build_hash(bundle) + + +def test_without_a_counter_signature_the_compilers_record_is_incomplete_not_passed(tmp_path, monkeypatch): + root, version, _commit, _calls = sup.release_sandbox(C, tmp_path, monkeypatch) + run_release_in(root, monkeypatch) + report = V.verify(sup.written_record(root, version), str(root)) + assert (report["record"].status, report["blobs"].status, report["crypto"]["status"]) == ("PASS", "PASS", "INCOMPLETE") + + +def test_the_digest_is_of_the_blob_bytes_not_of_the_working_directory(tmp_path, monkeypatch): + root, version, _commit, _calls = sup.release_sandbox(C, tmp_path, monkeypatch) + # the working copies are changed after the commit: CRLF, other bytes, a file removed + (root / "a.txt").write_bytes(b"alpha\r\n") + (root / "d" / "b.txt").write_bytes(b"another\n") + (root / (fx.E_ACUTE + ".txt")).unlink() + run_release_in(root, monkeypatch) + bundle = sup.written_record(root, version) + assert bundle["artifacts"]["files"] == fx.load_expected()["files"] + + +def test_a_filter_in_the_checkout_does_not_reach_the_digest(tmp_path, monkeypatch): + attributes = ("100644", b"*.txt text eol=crlf\n") + files = sup.commit_files(sup.golden_paths()) + [(".gitattributes", attributes[1], attributes[0])] + root, version, _commit, _calls = sup.release_sandbox(C, tmp_path, monkeypatch, files=files) + fx._git(root, "config", "core.autocrlf", "true") + run_release_in(root, monkeypatch) + assert sup.written_record(root, version)["artifacts"]["files"] == fx.load_expected()["files"] + + +def test_the_declaration_comes_from_the_commit_not_from_the_working_directory(tmp_path, monkeypatch): + root, version, _commit, _calls = sup.release_sandbox(C, tmp_path, monkeypatch) + # the working copy of project.yaml declares something else entirely; the commit's declaration is the one that counts + (root / "project.yaml").write_text(sup.project_yaml(["a.txt"])) + run_release_in(root, monkeypatch) + assert [f["path"] for f in sup.written_record(root, version)["artifacts"]["files"]] == sorted(sup.golden_paths(), key=lambda p: p.encode("utf-8")) + + +def test_a_declaration_in_the_working_directory_only_is_no_declaration(tmp_path, monkeypatch, capsys): + files = list(fx.GOLDEN_FILES) # the commit has no project.yaml + root, version, _commit, calls = sup.release_sandbox(C, tmp_path, monkeypatch, files=files) + (root / "project.yaml").write_text(sup.project_yaml(sup.golden_paths())) + with pytest.raises(SystemExit) as stop: + run_release_in(root, monkeypatch) + assert stop.value.code == 1 + assert "declaration_missing" in capsys.readouterr().out + assert not (root / "release").exists() and calls["vault"] == 0 + + +def test_the_working_directory_does_not_stand_in_for_the_commit_on_the_paths_either(tmp_path, monkeypatch, capsys): + # a file that is only in the working directory, not in the commit, is not an artifact + root, version, _commit, calls = sup.release_sandbox(C, tmp_path, monkeypatch, paths=sup.golden_paths() + ["untracked.txt"]) + (root / "untracked.txt").write_text("only here\n") + with pytest.raises(SystemExit) as stop: + run_release_in(root, monkeypatch) + assert stop.value.code == 1 + assert "declared_path_absent" in capsys.readouterr().out + assert calls["vault"] == 0 + + +@pytest.mark.parametrize("paths,reason", [ + ([], "declaration_empty"), + (["a.txt", "a.txt"], "declaration_path_duplicate"), + (["../a.txt"], "declaration_path_form"), + (["/a.txt"], "declaration_path_form"), + (["d"], "declared_path_not_regular"), + (["missing.txt"], "declared_path_absent"), +]) +def test_run_release_refuses_before_any_vault_call(paths, reason, tmp_path, monkeypatch, capsys): + root, _v, _commit, calls = sup.release_sandbox(C, tmp_path, monkeypatch, paths=paths) + with pytest.raises(SystemExit) as stop: + run_release_in(root, monkeypatch) + assert stop.value.code == 1 + assert reason in capsys.readouterr().out + assert calls["vault"] == 0 + assert not (root / "release").exists() + + +def test_run_release_without_any_declaration_stops_with_declaration_missing(tmp_path, monkeypatch, capsys): + plain = yaml.dump({"project": {"name": "x"}, "compiler_settings": {"source_dir": "schemas"}}).encode() + root, _v, _commit, calls = sup.release_sandbox(C, tmp_path, monkeypatch, files=list(fx.GOLDEN_FILES) + [("project.yaml", plain, "100644")]) + with pytest.raises(SystemExit): + run_release_in(root, monkeypatch) + assert "declaration_missing" in capsys.readouterr().out and calls["vault"] == 0 + + +def test_run_release_refuses_when_it_cannot_name_the_source_commit(tmp_path, monkeypatch, capsys): + root, _v, _commit, calls = sup.release_sandbox(C, tmp_path, monkeypatch) + real = subprocess.run + + def failing(command, *args, **kwargs): + if list(command[:3]) == ["git", "rev-parse", "HEAD"]: + raise subprocess.CalledProcessError(128, command) + return real(command, *args, **kwargs) + + monkeypatch.setattr(C.subprocess, "run", failing) + with pytest.raises(SystemExit) as stop: + run_release_in(root, monkeypatch) + assert stop.value.code == 1 + assert calls["vault"] == 0 and not (root / "release").exists() + + +def test_the_source_commit_is_never_null_and_is_forty_lowercase_hex(tmp_path, monkeypatch, capsys): + monkeypatch.chdir(tmp_path) # not a git repository + with pytest.raises(SystemExit) as stop: + C._source_commit() + assert stop.value.code == 1 + root = tmp_path / "r" + root.mkdir() + commit = sup.checkout(root, list(fx.GOLDEN_FILES)) + monkeypatch.chdir(root) + assert C._source_commit() == commit and re.fullmatch(r"[0-9a-f]{40}", commit) + assert C._collect_provenance()["source_commit"] == commit + + +@pytest.mark.parametrize("printed", ["", "HEAD", "abc", "a" * 39, "a" * 41, "A" * 40, "g" * 40, "a" * 64, "a" * 40 + "\n" + "b" * 40]) +def test_a_source_commit_that_is_not_forty_lowercase_hex_is_refused(printed, monkeypatch): + real = subprocess.run + + def said(command, *args, **kwargs): + if list(command[:3]) == ["git", "rev-parse", "HEAD"]: + return subprocess.CompletedProcess(command, 0, stdout=printed + "\n", stderr="") + return real(command, *args, **kwargs) + + monkeypatch.setattr(C.subprocess, "run", said) + with pytest.raises(SystemExit) as stop: + C._source_commit() + assert stop.value.code == 1 + + +PATH_VECTORS = [(x["id"], fx.expand(x["ops"][0]["value"], {})) for x in vec.PATHS + if x["ops"][0]["op"] == "set" and x["ops"][0]["at"] == "artifacts.files.0.path" and isinstance(x["ops"][0]["value"], str)] + + +@pytest.mark.parametrize("vid,path", PATH_VECTORS, ids=[p[0] for p in PATH_VECTORS]) +def test_the_compilers_path_rule_refuses_every_path_of_the_path_vectors(vid, path): + assert C._release_artifact_path_problem(path) is not None, vid + + +def test_the_compilers_path_rule_refuses_the_rest_of_the_path_vectors(): + for odd in (7, None, "", chr(0xD800) + ".txt"): + assert C._release_artifact_path_problem(odd) is not None, odd + + +@pytest.mark.parametrize("path", [p for p, _c, _m in fx.GOLDEN_FILES] + ["A.txt", "a b.txt", "d/e/f.g", ".hidden", "d/.hidden", "a..b", "-x"]) +def test_the_compilers_path_rule_accepts_ordinary_paths(path): + assert C._release_artifact_path_problem(path) is None + + +def test_run_release_checks_the_source_paths_of_the_specs_it_writes(tmp_path, monkeypatch): + root, version, _commit, _calls = sup.release_sandbox(C, tmp_path, monkeypatch) + seen = [] + real = C._check_unique_source_paths + monkeypatch.setattr(C, "_check_unique_source_paths", lambda specs: (seen.append(list(specs)), real(specs))[1]) + run_release_in(root, monkeypatch) + written = sup.written_record(root, version) + assert seen and [e["source_path"] for e in seen[0]] == [e["source_path"] for e in written["specs"]] + + +def test_two_runs_of_the_same_release_give_the_same_build_hash(tmp_path, monkeypatch): + root, version, _commit, _calls = sup.release_sandbox(C, tmp_path, monkeypatch) + sup.fix_the_clock(monkeypatch, C) + run_release_in(root, monkeypatch) + first = sup.written_record(root, version) + (root / "release" / ("cic-primitives-v%s.yaml" % version)).unlink() + run_release_in(root, monkeypatch) + second = sup.written_record(root, version) + assert first["release"]["build_hash"] == second["release"]["build_hash"] + assert fx.independent_canonical_bytes(first) == fx.independent_canonical_bytes(second) + + +def test_duplicate_spec_source_paths_are_refused(monkeypatch): + specs = [{"source_path": "schemas/atomic/a.yaml"}, {"source_path": "schemas/atomic/a.yaml"}] + with pytest.raises(SystemExit) as stop: + C._check_unique_source_paths(specs) + assert stop.value.code == 1 + C._check_unique_source_paths([{"source_path": "schemas/atomic/a.yaml"}, {"source_path": "schemas/atomic/b.yaml"}]) + + +# ---------------------------------------------------------------- the signature covers what the compiler wrote + +@pytest.fixture(scope="module") +def compiler_record(tmp_path_factory): + mp = pytest.MonkeyPatch() + try: + root = tmp_path_factory.mktemp("e3written") + directory, version, _commit, _calls = sup.release_sandbox(C, root, mp) + mp.chdir(directory) + C.run_release() + return sup.countersign(sup.written_record(directory, version)) + finally: + mp.undo() + + +@pytest.mark.parametrize("vector", vec.CRYPTO, ids=lambda x: x["id"]) +def test_the_coverage_vectors_on_the_record_the_compiler_wrote(vector, compiler_record): + bundle = fx.apply_ops(compiler_record, vector["ops"], {}) + if vector["reauth"] != "none": + fx.reauthenticate(bundle, vector["reauth"]) + crypto = V.check_crypto(bundle) + assert crypto["status"] == "FAIL" + assert tuple(crypto[p].reason or crypto[p].status for p in ("build_hash", "signature", "countersign")) == vector["parts"] + + +@pytest.mark.parametrize("name,ops", vec.COVERAGE_MUTATIONS, ids=[m[0] for m in vec.COVERAGE_MUTATIONS]) +def test_the_same_defect_re_authenticated_is_a_validly_signed_compiler_record(name, ops, compiler_record): + assert V.check_crypto(fx.reauthenticate(fx.apply_ops(compiler_record, ops, {})))["status"] == "PASS" + + +def test_the_record_vectors_hold_for_the_compilers_record_too(compiler_record): + # a structural defect applied to what the compiler wrote is refused for the reason of the vector, and the crypto layer of the re-authenticated record passes + applied = 0 + for vector in vec.ALL_RECORD: + # vectors that name golden-only content (the second spec's source path) or that are positive cases are the golden record's own + if vector["id"] in ("specs-share-a-source-path",) or vector["reason"] is None: + continue + applied += 1 + bundle = fx.apply_ops(compiler_record, vector["ops"], {}) + if vector["reauth"] != "none": + fx.reauthenticate(bundle, vector["reauth"]) + result = V.check_record(bundle) + assert (result.status, result.reason) == ("FAIL", vector["reason"]), vector["id"] + assert applied > 80 + + +# ---------------------------------------------------------------- the old envelopes keep verifying, on real historical records + +def verify_release(path, monkeypatch): + sup.fix_the_clock(monkeypatch, C) + monkeypatch.chdir(ROOT) + C.run_verify_release(str(path)) + + +def test_the_real_envelope_1_record_still_verifies(monkeypatch, capsys): + record = ROOT / "release" / "cic-primitives-v0.1.5.yaml" + bundle = yaml.safe_load(record.read_text()) + assert "provenance" not in bundle and "envelope" not in bundle["release"] + verify_release(record, monkeypatch) + out = capsys.readouterr().out + assert "release envelope: v1" in out and "build_hash verified" in out and "Release signature verified" in out + + +V020_RELEASE_COMMIT = "960ee0f048d969a85ad20819bc0f2366ad91fa40" +V020_RECORD_SHA256 = "955fab2d6d9638c773f29fbe97a3279f9f10b9539c4e3e79957df0acef9a57a6" + + +def test_the_real_envelope_2_record_still_verifies(tmp_path, monkeypatch, capsys): + import hashlib + import os + have = subprocess.run(["git", "-C", str(ROOT), "cat-file", "-e", V020_RELEASE_COMMIT + "^{commit}"], capture_output=True).returncode == 0 + if not have: + assert os.environ.get("CIC_REQUIRE_RELEASE_HISTORY") != "1", "CIC_REQUIRE_RELEASE_HISTORY=1 but the commit of release v0.2.0 is not in this clone" + pytest.skip("the history of release v0.2.0 is not in this clone") + blob = subprocess.run(["git", "-C", str(ROOT), "show", "%s:release/cic-primitives-v0.2.0.yaml" % V020_RELEASE_COMMIT], capture_output=True, check=True).stdout + assert hashlib.sha256(blob).hexdigest() == V020_RECORD_SHA256, "this is not the frozen record of v0.2.0" + record = tmp_path / "cic-primitives-v0.2.0.yaml" + record.write_bytes(blob) + bundle = yaml.safe_load(blob) + assert bundle["release"]["envelope"] == bundle["provenance"]["envelope"] == 2 + verify_release(record, monkeypatch) + out = capsys.readouterr().out + assert "release envelope: v2" in out and "build_hash verified" in out and "counter-signature verified" in out + + +def test_an_unknown_envelope_is_still_refused_by_verify_release(tmp_path, monkeypatch): + bundle = yaml.safe_load((ROOT / "release" / "cic-primitives-v0.1.5.yaml").read_text()) + bundle["release"]["envelope"] = 99 + record = tmp_path / "r.yaml" + record.write_text(yaml.dump(bundle, sort_keys=False)) + with pytest.raises(SystemExit) as stop: + verify_release(record, monkeypatch) + assert stop.value.code == 1 + + +def test_verify_release_reads_an_envelope_3_record_and_says_what_it_leaves_to_the_conformance_check(compiler_record, tmp_path, monkeypatch, capsys): + record = tmp_path / "r.yaml" + record.write_text(yaml.dump(compiler_record, sort_keys=False)) + verify_release(record, monkeypatch) + out = capsys.readouterr().out + assert "release envelope: v3" in out and "build_hash verified" in out + assert "NOT proven" in out and "release_record_conformance" in out + + +def test_verify_release_does_not_hand_the_verdict_to_the_reference_verifier(): + source = (ROOT / "tools" / "compiler.py").read_text(encoding="utf-8") + imports = [n for n in ast.walk(ast.parse(source)) if isinstance(n, (ast.Import, ast.ImportFrom))] + names = " ".join(getattr(n, "module", "") or " ".join(a.name for a in n.names) for n in imports) + assert "release_record_conformance" not in names + + +def test_the_reference_verifier_does_not_call_the_compilers_new_functions(): + source = (ROOT / "tools" / "release_record_conformance.py").read_text(encoding="utf-8") + for name in ("_git_tree", "_build_release_artifacts", "_validate_release_artifact_declaration", "_source_commit", "_declaration_from_commit", "_resolve_release_artifacts", + "_check_unique_source_paths"): + assert not re.search(r"(? 1) + return 'declaration_path_duplicate', repr(repeated) + for path in paths: + if path not in tree: + return 'declared_path_absent', f'{path!r} is not in the tree of the source commit' + for path in paths: + mode, kind, _oid = tree[path] + if kind != 'blob' or mode not in REGULAR_FILE_MODES: + return 'declared_path_not_regular', f'{path!r} is mode {mode}, type {kind}' + return None + + +def _declaration_from_commit(tree): + """The parsed project.yaml as the SOURCE COMMIT holds it ({} when it holds none, or none that reads).""" + entry = tree.get('project.yaml') + if entry is None or entry[1] != 'blob' or entry[0] not in REGULAR_FILE_MODES: + return {} + try: + project = yaml.safe_load(_git_blob(entry[2])) + except yaml.YAMLError: + return {} + return project if isinstance(project, dict) else {} + + +def _build_release_artifacts(paths, tree): + """The `artifacts` block: `sha256:` + hex of each blob's bytes, sorted by the UTF-8 bytes of the path, and the digest of the sha256sum-format manifest.""" + files = [] + for path in sorted(paths, key=lambda p: p.encode('utf-8')): + digest = 'sha256:' + get_sha256_hex(_git_blob(tree[path][2])) + files.append({'path': path, 'digest': digest}) + manifest = ''.join(f"{entry['digest'][len('sha256:'):]} {entry['path']}\n" for entry in files) + return {'manifest_digest': 'sha256:' + get_sha256_hex(manifest.encode('utf-8')), 'files': files} + + +def _resolve_release_artifacts(commit): + """Declaration -> refusal or the `artifacts` block, from the git tree of `commit` only.""" + tree = _git_tree(commit) + project = _declaration_from_commit(tree) + refusal = _validate_release_artifact_declaration(project, tree) + if refusal: + print(f"\033[91m✗ ERROR: release_artifacts: {refusal[0]}: {refusal[1]}\033[0m") + sys.exit(1) + return _build_release_artifacts(project['compiler_settings']['release_artifacts']['paths'], tree) + + +def _check_unique_source_paths(specs): + """Two `specs` entries must not share a `source_path` (section 7): a later one never overwrites an earlier one.""" + seen = set() + for entry in specs: + path = entry.get('source_path') + if path in seen: + print(f"\033[91m✗ ERROR: spec_source_path_duplicate: {path!r}\033[0m") + sys.exit(1) + seen.add(path) + + def get_reproducible_repo_hash(tree_id): """ Calculates a reproducible SHA256 hash of a given git tree object. @@ -698,6 +858,14 @@ def run_release(): release_version, component_name = validate_release_prerequisites() run_grammar_gate() + # The artifacts are fixed before anything is asked of Vault: a declaration that is + # refused must not cost a signature, and the commit that names them is the one the + # record will carry. + source_commit = _source_commit() + artifacts = _resolve_release_artifacts(source_commit) + print(f" \033[92m✓ {len(artifacts['files'])} release artifacts digested from the tree of " + f"{source_commit[:12]}\033[0m") + vault_addr = os.getenv('VAULT_ADDR') vault_token = os.getenv('VAULT_TOKEN') vault_cacert = os.getenv('VAULT_CACERT') @@ -772,6 +940,8 @@ def run_release(): }) print(f" - {schema_file}: {meta_hash[:16]}...") + _check_unique_source_paths(specs) + # 6. Build hash: {createdBy (developer), release.createdBy (issuer), specs, validity} # Both certificates are cryptographically bound before Vault signs. # Future: this will become a Merkle tree root over CI artifact layers. @@ -786,7 +956,8 @@ def run_release(): 'validity': validity, 'createdBy': created_by, 'specs': specs, - 'provenance': _collect_provenance(), + 'provenance': _collect_provenance(source_commit), + 'artifacts': artifacts, 'release': { 'envelope': RELEASE_ENVELOPE_VERSION, 'createdBy': release_created_by, @@ -1002,7 +1173,7 @@ def run_verify_release(artifact_path, strict=False, trust_root=None): recorded_hash = release_block.get('build_hash', '') envelope_version = release_block.get('envelope', 1) - if envelope_version not in (1, 2): + if envelope_version not in (1, 2, 3): print(f" \033[91m✗ unknown release envelope version: {envelope_version}\033[0m") sys.exit(1) hash_payload = _release_hash_payload(bundle, envelope_version) @@ -1250,6 +1421,11 @@ def run_verify_release(artifact_path, strict=False, trust_root=None): if not (bundle.get('provenance') or {}).get('source_commit'): unproven.append("no source_commit recorded, so the artifact cannot be tied " "back to a tree") + if envelope_version == 3: + unproven.append("the artifact list is not checked here (its digests, order, paths " + "and the git tree of source_commit): that is " + "tools/release_record_conformance.py, the reference check of " + "docs/en/RELEASE-RECORD-CONTRACT.md") print(f"\n--- What this proves ---") for claim in proven: From 5ccc0dc9816ab8f18b17678ce2518d50d9205261 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?G=C3=A1bor=20Z=20Sink=C3=B3?= Date: Sun, 1 Nov 2026 22:47:18 +0100 Subject: [PATCH 3/6] test(release): planted defects in the compiler's envelope 3 output; gate timeout 20 minutes 58 source breaks of tools/compiler.py, each run against the tests that target it (50 must turn one red, 8 documented equivalent must survive; each break is compiled before it is run). The gate timeout goes from 10 to 20 minutes because the suite now carries both planted-defect proofs. --- [signing-metadata] key = cic-my-sign-key signature = vault:v1:MEQCIB10mVJKzVMoC+LZik+vOAwvy/4eKFmlXq4Lrbtub1r0AiA2WhqAj0tk1Fn+VIFr//EfN/TGOrCpDwnNyvreAgNwmQ== hash-algorithm = sha256 digest = qVOVUxxTou+M08uKqAzGK5YGnR4GYbwhw5HUMwxZdAc= [certificate] -----BEGIN CERTIFICATE----- MIICBjCCAaygAwIBAgIUSnRMR6RPnEbg296XWPOqq/u5PCwwCgYIKoZIzj0EAwIw QzELMAkGA1UEBhMCSFUxGTAXBgNVBAoMEENlbnRyYWxJbmZyYUNvcmUxGTAXBgNV BAMMEENJQyBEZXZlbG9wZXIgQ0EwHhcNMjYwMzIwMTMyMjU5WhcNMjYxMjMxMTMy MjU5WjBFMQswCQYDVQQGEwJIVTEZMBcGA1UECgwQQ2VudHJhbEluZnJhQ29yZTEb MBkGA1UEAwwSR2Fib3IgWm9sdGFuIFNpbmtvMFkwEwYHKoZIzj0CAQYIKoZIzj0D AQcDQgAEIG2CVmTfmLB9pLLclj7YmP2eedAjklpy4LGrU2ijoiy6Xqpuybv7OgJe i+ez31s65NEV8+X/ByeX1cstR988z6N8MHowCQYDVR0TBAIwADAdBgNVHQ4EFgQU yZN6AIX/TNnIJ9GwAa/NRN3ujHAwHwYDVR0jBBgwFoAUXn6CHYzPUqU4JVP8g+OS WeDYjhcwDgYDVR0PAQH/BAQDAgeAMB0GA1UdJQQWMBQGCCsGAQUFBwMCBggrBgEF BQcDBDAKBggqhkjOPQQDAgNIADBFAiEA+bFzXRoJ4PCQbhAAtpkcMjt0vNj5rEW0 lOMBGDNyaWkCIB1vmM7PcZzv/c9bIrxF5kqv6QXomouhByUfeNUTbpKW -----END CERTIFICATE----- --- .github/workflows/gate.yml | 4 +- tests/e3_compiler_mutants.py | 94 ++++++++++++++++++++++++ tests/test_compiler_envelope3_planted.py | 61 +++++++++++++++ 3 files changed, 158 insertions(+), 1 deletion(-) create mode 100644 tests/e3_compiler_mutants.py create mode 100644 tests/test_compiler_envelope3_planted.py diff --git a/.github/workflows/gate.yml b/.github/workflows/gate.yml index 4f79ff6..2c00574 100644 --- a/.github/workflows/gate.yml +++ b/.github/workflows/gate.yml @@ -38,7 +38,9 @@ jobs: gate: name: gate runs-on: ubuntu-latest - timeout-minutes: 10 + # 20, not 10: the suite now includes the planted-defect proofs of the envelope 3 conformance and of the compiler's envelope 3 output (about 160 source breaks, + # each run against the tests that target it, four at a time); locally the whole gate takes about 3.5 minutes + timeout-minutes: 20 steps: - uses: actions/checkout@v4 diff --git a/tests/e3_compiler_mutants.py b/tests/e3_compiler_mutants.py new file mode 100644 index 0000000..af125a0 --- /dev/null +++ b/tests/e3_compiler_mutants.py @@ -0,0 +1,94 @@ +"""Planted defects for the compiler's envelope 3 output: a small, deliberate break in tools/compiler.py, and the test of tests/test_compiler_envelope3.py that has to turn +red because of it. Same rules as tests/e3_mutants.py: every `old` occurs exactly once; `kills` are substrings of test ids, one of which must be among the failures; +`equivalent` mutants change the source and no behaviour any input can show, carry the reason, and have to survive. `edits` is a list of (old, new) for a break that +needs more than one place. +""" + + +def m(id, edits, kills=(), equivalent=None): + if isinstance(edits, tuple): + edits = [edits] + return {"id": id, "edits": list(edits), "kills": list(kills), "equivalent": equivalent} + + +MUTANTS = [ + # the version and the hash rule + m("compiler-still-writes-envelope-2", ("RELEASE_ENVELOPE_VERSION = 3\n", "RELEASE_ENVELOPE_VERSION = 2\n"), ["envelope_the_compiler_writes_is_3", "writes_an_envelope_3_record"]), + m("hash-payload-of-envelope-3-missing", ("if envelope_version in (2, 3):", "if envelope_version == 2:"), ["hash_payload_of_envelope_3"]), + m("hash-payload-of-envelope-3-is-the-v1-rule", ("if envelope_version == 1:\n return {", "if envelope_version in (1, 3):\n return {"), ["hash_payload_of_envelope_3", "independent_one"]), + m("verify-release-does-not-read-envelope-3", ("if envelope_version not in (1, 2, 3):", "if envelope_version not in (1, 2):"), ["reads_an_envelope_3_record"]), + m("verify-release-says-nothing-of-the-artifact-list", (" if envelope_version == 3:\n unproven.append(", " if False:\n unproven.append("), ["reads_an_envelope_3_record"]), + # the source commit + m("source-commit-null-when-git-says-nothing", (" sys.exit(1)\n return commit\n", " return None\n return commit\n"), ["never_null", "cannot_name_the_source_commit"]), + m("source-commit-any-text", ("SOURCE_COMMIT_RE = re.compile(r'^[0-9a-f]{40}$')", "SOURCE_COMMIT_RE = re.compile(r'^.*$')"), ["not_forty_lowercase_hex"]), + m("source-commit-uppercase-allowed", ("SOURCE_COMMIT_RE = re.compile(r'^[0-9a-f]{40}$')", "SOURCE_COMMIT_RE = re.compile(r'^[0-9a-fA-F]{40}$')"), ["not_forty_lowercase_hex"]), + m("source-commit-length-open", ("SOURCE_COMMIT_RE = re.compile(r'^[0-9a-f]{40}$')", "SOURCE_COMMIT_RE = re.compile(r'^[0-9a-f]{40,64}$')"), ["not_forty_lowercase_hex"]), + m("source-commit-end-not-anchored", ("SOURCE_COMMIT_RE = re.compile(r'^[0-9a-f]{40}$')", "SOURCE_COMMIT_RE = re.compile(r'^[0-9a-f]{40}')"), ["not_forty_lowercase_hex"]), + m("provenance-ignores-the-commit-it-is-given", ("'source_commit': commit or _source_commit()}", "'source_commit': _source_commit()}"), [], + equivalent="the run asks git for HEAD, which is the commit run_release passed in; the two readings are the same value in every run"), + m("provenance-without-a-commit-is-null", ("'source_commit': commit or _source_commit()}", "'source_commit': commit}"), ["never_null", "records_the_source_and_digests"]), + # the git tree + m("tree-without-directories", ("['git', 'ls-tree', '-r', '-t', '-z', '--full-tree', commit]", "['git', 'ls-tree', '-r', '-z', '--full-tree', commit]"), + ["refuses_a_declaration_for_the_reason_of_the_vector", "tree_listing", "run_release_refuses_before_any_vault_call"]), + m("tree-flag-full-tree", ("'-t', '-z', '--full-tree', commit]", "'-t', '-z', commit]"), [], + equivalent="the compiler runs at the repository root, where the listing is the whole tree with or without --full-tree"), + m("tree-names-split-on-lines", ("for record in listing.stdout.split(b'\\0'):", "for record in listing.stdout.split(b'\\n'):"), ["tree_listing"]), + m("blob-from-the-working-directory", ("digest = 'sha256:' + get_sha256_hex(_git_blob(tree[path][2]))", "digest = 'sha256:' + get_sha256_hex(open(path, 'rb').read())"), + ["blob_bytes_not_of_the_working_directory", "a_filter_in_the_checkout", "any_order_gives_the_same"]), + m("declaration-from-the-working-directory", ("project = yaml.safe_load(_git_blob(entry[2]))", "project = yaml.safe_load(open('project.yaml', 'rb').read())"), + ["declaration_comes_from_the_commit", "declaration_in_the_working_directory_only"]), + m("declaration-found-without-the-commit-entry", (" if entry is None or entry[1] != 'blob' or entry[0] not in REGULAR_FILE_MODES:\n return {}\n", ""), + ["declaration_in_the_working_directory_only", "without_any_declaration", "refuses_before_any_vault_call"]), + # the declaration rules + m("declaration-unknown-key-accepted", (" if unknown:\n return 'declaration_unknown_key'", " if False:\n return 'declaration_unknown_key'"), ["vector"]), + m("declaration-not-a-list-accepted", (" if not isinstance(paths, list):\n return 'declaration_missing'", " if False:\n return 'declaration_missing'"), ["vector"]), + m("declaration-empty-accepted", (" if not paths:\n return 'declaration_empty'", " if False:\n return 'declaration_empty'"), ["vector", "refuses_before_any_vault_call"]), + m("declaration-form-not-checked", (" problem = _release_artifact_path_problem(path)\n if problem:\n return 'declaration_path_form'", " problem = None\n if problem:\n return 'declaration_path_form'"), ["vector", "refuses_before_any_vault_call"]), + m("declaration-duplicate-accepted", (" if len(set(paths)) != len(paths):", " if False:"), ["vector", "refuses_before_any_vault_call"]), + m("declaration-absent-path-accepted", (" if path not in tree:\n return 'declared_path_absent'", " if False:\n return 'declared_path_absent'"), ["vector", "working_directory_does_not_stand_in"]), + m("declaration-not-regular-accepted", (" if kind != 'blob' or mode not in REGULAR_FILE_MODES:\n return 'declared_path_not_regular'", " if False:\n return 'declared_path_not_regular'"), ["vector"]), + m("declaration-link-counted-regular", ("REGULAR_FILE_MODES = ('100644', '100755')", "REGULAR_FILE_MODES = ('100644', '100755', '120000')"), ["decl-symbolic-link"]), + m("declaration-submodule-mode-counted-regular", ("REGULAR_FILE_MODES = ('100644', '100755')", "REGULAR_FILE_MODES = ('100644', '100755', '160000')"), [], + equivalent="a submodule is a commit entry and the type test (kind != blob) still refuses it; the mode test and the type test each refuse it alone"), + m("declaration-only-directories-refused", (" if kind != 'blob' or mode not in REGULAR_FILE_MODES:\n return 'declared_path_not_regular'", " if mode == '040000':\n return 'declared_path_not_regular'"), ["decl-symbolic-link", "decl-submodule"]), + m("declaration-type-test-removed", (" if kind != 'blob' or mode not in REGULAR_FILE_MODES:\n return 'declared_path_not_regular'", " if mode not in REGULAR_FILE_MODES:\n return 'declared_path_not_regular'"), [], + equivalent="every non-blob tree entry (a tree 040000, a commit 160000) has a mode outside REGULAR_FILE_MODES, so the type test adds nothing the mode test does not already refuse"), + # the path rule + m("path-control-characters-allowed", (" if any(ord(ch) < 0x20 or ord(ch) == 0x7F for ch in path):", " if False:"), ["path_rule_refuses", "vector"]), + m("path-del-allowed", ("ord(ch) < 0x20 or ord(ch) == 0x7F", "ord(ch) < 0x20"), ["path-del", "path_rule_refuses"]), + m("path-line-feed-allowed", ("ord(ch) < 0x20 or ord(ch) == 0x7F", "ord(ch) < 0x0A or ord(ch) > 0x0A and ord(ch) < 0x20 or ord(ch) == 0x7F"), ["path-line-feed", "decl-line-feed"]), + m("path-backslash-allowed", (" if '\\\\' in path:", " if False:"), ["path-backslash", "decl-backslash"]), + m("path-dot-allowed", (" if segment in ('.', '..'):", " if segment == '..':"), ["path-dot", "decl-dot"]), + m("path-dot-dot-allowed", (" if segment in ('.', '..'):", " if segment == '.':"), ["path-dot-dot", "decl-dot-dot"]), + m("path-empty-segment-allowed", (" if segment == '':", " if False:"), ["path-empty-segment"]), + m("path-non-string-allowed", (" if not isinstance(path, str) or path == '':", " if path == '':"), ["path-a-number", "path_rule_refuses_the_rest", "decl-not-a-string"]), + m("path-utf8-check-removed", (" try:\n path.encode('utf-8')\n except UnicodeEncodeError:\n return 'not UTF-8'\n", ""), ["path_rule_refuses_the_rest", "lone-surrogate"]), + m("path-absolute-has-its-own-branch", (" if path.startswith('/'):\n return 'absolute'\n", ""), [], + equivalent="a leading slash also gives an empty first segment, refused with the same reason; only the detail text differs"), + m("path-trailing-slash-has-its-own-branch", (" if path.endswith('/'):\n return 'trailing slash'\n", ""), [], + equivalent="a trailing slash also gives an empty last segment, refused with the same reason; only the detail text differs"), + m("path-empty-has-its-own-branch", (" if not isinstance(path, str) or path == '':", " if not isinstance(path, str):"), [], + equivalent="the empty string splits into one empty segment, refused with the same reason"), + # the block + m("artifacts-not-sorted", (" for path in sorted(paths, key=lambda p: p.encode('utf-8')):", " for path in list(paths):"), ["any_order_gives_the_same", "the_same_declared_set"]), + m("artifacts-sorted-by-utf16", (" for path in sorted(paths, key=lambda p: p.encode('utf-8')):", " for path in sorted(paths, key=lambda p: p.encode('utf-16-be')):"), ["utf8_bytes_not_utf16"]), + m("artifacts-sorted-by-code-point", (" for path in sorted(paths, key=lambda p: p.encode('utf-8')):", " for path in sorted(paths):"), [], + equivalent="UTF-8 preserves code point order, so sorting by code point and by UTF-8 bytes agree on every input"), + m("artifacts-sorted-descending", (" for path in sorted(paths, key=lambda p: p.encode('utf-8')):", " for path in sorted(paths, key=lambda p: p.encode('utf-8'), reverse=True):"), ["any_order_gives_the_same", "utf8_bytes_not_utf16"]), + m("artifact-digest-without-prefix", ("digest = 'sha256:' + get_sha256_hex(_git_blob(tree[path][2]))", "digest = get_sha256_hex(_git_blob(tree[path][2]))"), ["any_order_gives_the_same", "writes_an_envelope_3_record"]), + m("artifact-digest-base64", ("digest = 'sha256:' + get_sha256_hex(_git_blob(tree[path][2]))", "digest = 'sha256:' + get_sha256_b64(_git_blob(tree[path][2]))"), ["any_order_gives_the_same"]), + m("artifact-digest-of-the-path", ("digest = 'sha256:' + get_sha256_hex(_git_blob(tree[path][2]))", "digest = 'sha256:' + get_sha256_hex(path.encode('utf-8'))"), ["any_order_gives_the_same"]), + m("manifest-single-space", ("f\"{entry['digest'][len('sha256:'):]} {entry['path']}\\n\"", "f\"{entry['digest'][len('sha256:'):]} {entry['path']}\\n\""), ["any_order_gives_the_same"]), + m("manifest-no-line-feed", ("f\"{entry['digest'][len('sha256:'):]} {entry['path']}\\n\"", "f\"{entry['digest'][len('sha256:'):]} {entry['path']}\""), ["any_order_gives_the_same"]), + m("manifest-without-prefix", ("return {'manifest_digest': 'sha256:' + get_sha256_hex(manifest.encode('utf-8')), 'files': files}", "return {'manifest_digest': get_sha256_hex(manifest.encode('utf-8')), 'files': files}"), ["any_order_gives_the_same"]), + m("manifest-of-the-reversed-list", (" manifest = ''.join(f\"{entry['digest'][len('sha256:'):]} {entry['path']}\\n\" for entry in files)", " manifest = ''.join(f\"{entry['digest'][len('sha256:'):]} {entry['path']}\\n\" for entry in reversed(files))"), ["any_order_gives_the_same"]), + # run_release + m("record-without-artifacts", (" 'artifacts': artifacts,\n", ""), ["writes_an_envelope_3_record", "coverage_vectors"]), + m("artifacts-added-after-the-hash", [(" 'artifacts': artifacts,\n", ""), (" bundle['release']['build_hash'] = build_hash\n", " bundle['artifacts'] = artifacts\n bundle['release']['build_hash'] = build_hash\n")], + ["the_hash_the_compiler_wrote", "coverage_vectors", "writes_an_envelope_3_record"]), + m("artifacts-resolved-after-vault", [(" artifacts = _resolve_release_artifacts(source_commit)\n", " artifacts = None\n"), (" 'artifacts': artifacts,\n", " 'artifacts': _resolve_release_artifacts(source_commit),\n")], + ["refuses_before_any_vault_call", "without_any_declaration", "declaration_in_the_working_directory_only", "working_directory_does_not_stand_in"]), + m("release-block-keeps-envelope-2", (" 'envelope': RELEASE_ENVELOPE_VERSION,\n 'createdBy': release_created_by,", " 'envelope': 2,\n 'createdBy': release_created_by,"), ["writes_an_envelope_3_record"]), + m("source-paths-not-checked-for-duplicates", (" _check_unique_source_paths(specs)\n\n # 6. Build hash", " # 6. Build hash"), ["checks_the_source_paths_of_the_specs"]), + m("duplicate-source-path-accepted", (" if path in seen:\n print(f\"\\033[91m✗ ERROR: spec_source_path_duplicate", " if False:\n print(f\"\\033[91m✗ ERROR: spec_source_path_duplicate"), ["duplicate_spec_source_paths"]), +] diff --git a/tests/test_compiler_envelope3_planted.py b/tests/test_compiler_envelope3_planted.py new file mode 100644 index 0000000..dae1e4a --- /dev/null +++ b/tests/test_compiler_envelope3_planted.py @@ -0,0 +1,61 @@ +"""Planted-defect proof for the compiler's envelope 3 output: small breaks in tools/compiler.py (tests/e3_compiler_mutants.py) are run against +tests/test_compiler_envelope3.py in a subprocess; a break meant to be caught has to turn one of the tests that target it red (a break that fails only an unrelated test +does not count, and one that is not valid Python is refused before it is run), and the breaks that cannot change behaviour are listed with the reason and have to survive. +""" +import concurrent.futures +import os +import pathlib +import subprocess +import sys + +import pytest + +sys.path.insert(0, str(pathlib.Path(__file__).resolve().parent)) +import e3_compiler_mutants as mutants # noqa: E402 +import e3_fixtures as fx # noqa: E402 + +SOURCE = fx.ROOT / "tools" / "compiler.py" + + +def run_mutant(mutant, tmp): + mutated = SOURCE.read_text(encoding="utf-8") + for old, new in mutant["edits"]: + assert mutated.count(old) == 1, "%s: the planted text occurs %d times: %r" % (mutant["id"], mutated.count(old), old[:60]) + mutated = mutated.replace(old, new) + compile(mutated, mutant["id"], "exec") + path = pathlib.Path(tmp) / ("c_%s.py" % mutant["id"]) + path.write_text(mutated, encoding="utf-8") + env = dict(os.environ, E3_COMPILER=str(path), PYTHONDONTWRITEBYTECODE="1") + select = ["-k", " or ".join(mutant["kills"])] if mutant["kills"] else [] + done = subprocess.run([sys.executable, "-m", "pytest", str(fx.ROOT / "tests" / "test_compiler_envelope3.py"), "-q", "--tb=no", "-rf", "-p", "no:cacheprovider"] + select, + capture_output=True, text=True, env=env, cwd=str(fx.ROOT)) + failed = [line for line in done.stdout.splitlines() if line.startswith("FAILED")] + assert done.returncode != 5 and "no tests ran" not in done.stdout, "%s: the targeted tests %s do not exist" % (mutant["id"], mutant["kills"]) + return done.returncode, failed + + +@pytest.fixture(scope="module") +def results(tmp_path_factory): + tmp = tmp_path_factory.mktemp("compiler_mutants") + with concurrent.futures.ThreadPoolExecutor(max_workers=4) as pool: + futures = {m["id"]: pool.submit(run_mutant, m, tmp) for m in mutants.MUTANTS} + return {k: f.result() for k, f in futures.items()} + + +def test_mutant_ids_are_unique(): + ids = [m["id"] for m in mutants.MUTANTS] + assert len(ids) == len(set(ids)) + + +@pytest.mark.parametrize("mutant", [m for m in mutants.MUTANTS if not m["equivalent"]], ids=lambda m: m["id"]) +def test_a_planted_defect_in_the_compiler_turns_the_targeted_test_red(mutant, results): + code, failed = results[mutant["id"]] + assert code != 0, "the planted defect %s was not detected by any test" % mutant["id"] + assert any(any(k in line for k in mutant["kills"]) for line in failed), "%s: failed %s, none of the targeted %s" % (mutant["id"], failed[:4], mutant["kills"]) + + +@pytest.mark.parametrize("mutant", [m for m in mutants.MUTANTS if m["equivalent"]], ids=lambda m: m["id"]) +def test_an_equivalent_mutant_of_the_compiler_survives_and_says_why(mutant, results): + code, failed = results[mutant["id"]] + assert len(mutant["equivalent"]) > 30 + assert code == 0, "%s is documented as equivalent but %s" % (mutant["id"], failed[:3]) From bb67999c1eb71fdec374dd3d65c64698d1603890 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?G=C3=A1bor=20Z=20Sink=C3=B3?= Date: Sun, 1 Nov 2026 23:16:01 +0100 Subject: [PATCH 4/6] feat(release): the compiler stops when the working directory is not the source commit The artifacts come from the git tree, but the specs, the legacy provenance digests, the issuer and the validity are read from the working directory, and a clean git status does not make those the commit's bytes (ignored files, assume-unchanged, skip-worktree, line-end filters and sparse checkouts leave it clean). Before asking Vault for anything, run_release compares every such input with the blob of the source commit and the schema files it found with the selection made on the commit's tree; a difference stops it with worktree_differs_from_source_commit. Each case is a test with the clean status asserted as its premise, and the guard has its own planted defects. The residual (the validation gates read the working directory too) is documented. --- [signing-metadata] key = cic-my-sign-key signature = vault:v1:MEYCIQD8xFArc+eqlHff+PwlhMezml/FO6PPj0zPucWreV5LNAIhANMSL1gVK5XhtwYa6ph5DDqTgDEVRrCGN2NQabYELmfo hash-algorithm = sha256 digest = kmlxGot1yu1P5qXIBecwLqJlhAsJYiqutpafla+Hg3E= [certificate] -----BEGIN CERTIFICATE----- MIICBjCCAaygAwIBAgIUSnRMR6RPnEbg296XWPOqq/u5PCwwCgYIKoZIzj0EAwIw QzELMAkGA1UEBhMCSFUxGTAXBgNVBAoMEENlbnRyYWxJbmZyYUNvcmUxGTAXBgNV BAMMEENJQyBEZXZlbG9wZXIgQ0EwHhcNMjYwMzIwMTMyMjU5WhcNMjYxMjMxMTMy MjU5WjBFMQswCQYDVQQGEwJIVTEZMBcGA1UECgwQQ2VudHJhbEluZnJhQ29yZTEb MBkGA1UEAwwSR2Fib3IgWm9sdGFuIFNpbmtvMFkwEwYHKoZIzj0CAQYIKoZIzj0D AQcDQgAEIG2CVmTfmLB9pLLclj7YmP2eedAjklpy4LGrU2ijoiy6Xqpuybv7OgJe i+ez31s65NEV8+X/ByeX1cstR988z6N8MHowCQYDVR0TBAIwADAdBgNVHQ4EFgQU yZN6AIX/TNnIJ9GwAa/NRN3ujHAwHwYDVR0jBBgwFoAUXn6CHYzPUqU4JVP8g+OS WeDYjhcwDgYDVR0PAQH/BAQDAgeAMB0GA1UdJQQWMBQGCCsGAQUFBwMCBggrBgEF BQcDBDAKBggqhkjOPQQDAgNIADBFAiEA+bFzXRoJ4PCQbhAAtpkcMjt0vNj5rEW0 lOMBGDNyaWkCIB1vmM7PcZzv/c9bIrxF5kqv6QXomouhByUfeNUTbpKW -----END CERTIFICATE----- --- docs/en/RELEASE-COMPILER-ENVELOPE3.md | 10 ++- docs/hu/RELEASE-COMPILER-ENVELOPE3.md | 3 +- tests/e3_compiler_mutants.py | 18 +++- tests/e3_compiler_support.py | 38 +++++--- tests/test_compiler.py | 15 ++-- tests/test_compiler_envelope3.py | 120 +++++++++++++++++++++++++- tools/compiler.py | 100 +++++++++++++++++---- 7 files changed, 259 insertions(+), 45 deletions(-) diff --git a/docs/en/RELEASE-COMPILER-ENVELOPE3.md b/docs/en/RELEASE-COMPILER-ENVELOPE3.md index 1edb8c3..15aa32b 100644 --- a/docs/en/RELEASE-COMPILER-ENVELOPE3.md +++ b/docs/en/RELEASE-COMPILER-ENVELOPE3.md @@ -21,6 +21,14 @@ A repository whose `project.yaml` has no `release_artifacts` cannot cut a release: it stops with `declaration_missing`. Which files a repository publishes is the producer's decision and is made in `project.yaml`, in a change of its own. +## The rest of the record is the source commit's too + +`artifacts` is read from the git tree. The rest of the record is not made from the tree: `specs` (their content and `meta_hash`) come from the schema files, the three legacy provenance digests from `dependency.yaml` and the two grammar files, the issuer and the validity from `project.yaml` and `commitment.yaml`. `validate_release_prerequisites` asks `git status --porcelain`, and a clean status does **not** make those the commit's bytes: an ignored file, a file marked assume-unchanged or skip-worktree, a line-end filter or a sparse checkout all leave it clean (each is a test, with the premise that the status is clean asserted first). + +So `run_release` compares, before it asks Vault for anything, every one of those inputs with the blob of the source commit, and the schema files it found with the selection made on the commit's tree. A difference (`worktree_differs_from_source_commit`, with the file and what differs) stops the release: a record made of two states of the source would contradict itself without anyone having lied. + +What this does not cover: the validation gates (`run_validation`, `run_primitive_validation`, the grammar gate) read the working directory too. Their verdict can concern a file that is not part of the record (an ignored example, for instance) and is not about the commit for such files; it does not enter the record. A release should be cut from a fresh clone of the commit. + ## How the output is held to the contract The compiler does not say that its own output is right. @@ -29,7 +37,7 @@ The compiler does not say that its own output is right. * The 32 signature coverage vectors of the contract run on the record the compiler wrote, not only on the golden record; the structural vectors too. * The declaration vectors run on the compiler's own functions, and on the reference's, and the two must agree. * The yardstick of PR #25 (the reference verifier, the golden fixtures, the vectors, the planted-defect proof) is frozen at its merge commit: `tests/test_conformance_frozen.py` fails if any of it changes together with the compiler. -* 58 planted defects in `tools/compiler.py` (`tests/e3_compiler_mutants.py`): 50 have to turn one of the tests that target them red, 8 cannot change behaviour, are listed with the reason and have to survive. +* 72 planted defects in `tools/compiler.py` (`tests/e3_compiler_mutants.py`): 63 have to turn one of the tests that target them red, 9 cannot change behaviour, are listed with the reason and have to survive. ## `verify-release` diff --git a/docs/hu/RELEASE-COMPILER-ENVELOPE3.md b/docs/hu/RELEASE-COMPILER-ENVELOPE3.md index 9129cc5..1bb1eed 100644 --- a/docs/hu/RELEASE-COMPILER-ENVELOPE3.md +++ b/docs/hu/RELEASE-COMPILER-ENVELOPE3.md @@ -6,7 +6,8 @@ A `python tools/compiler.py release` kizárólag **envelope 3** `PrimitiveReleas * **A termelő deklarálja a közzétett fájlokat** a `project.yaml`-ban (`compiler_settings.release_artifacts.paths`, explicit útvonalak, glob/`include`/`exclude` nélkül). A deklarációt a **forráscommit**ból olvassa, az útvonalakat a commit **Git-fájából** oldja fel; nem a munkakönyvtárból, nem szűrt checkoutból, nem fogyasztói pin-manifestből. Könyvtár, link, submodule, abszolút útvonal, `..`/`.`, vezérlőkarakter, duplikátum, hiányzó útvonal, üres lista, ismeretlen kulcs: leáll az okkal, **még a Vault hívása előtt**. * **`artifacts`:** útvonalanként `sha256:` + a blob bájtjainak SHA-256-ja, az útvonal UTF-8 bájtjai szerint rendezve, plusz `manifest_digest`. A `specs[].source_path` egyedi. * **Aláírás:** a meglévő mechanizmus (`build_hash` a teljes csomagra, `release.sign`, külső `cic_countersign`); az `artifacts` és a `source_commit` ezért lefedett. -* **Nem a compiler igazolja magát:** a kimenetet a referencia-verifier olvassa vissza (nem importálják egymást), a 32 lefedettségi vektor a compiler rekordján is lefut, a deklarációs vektorok a compiler saját függvényén, és a #25 mércéje befagyasztott (`tests/test_conformance_frozen.py`). 58 planted defect a `tools/compiler.py`-ban. +* **Nem a compiler igazolja magát:** a kimenetet a referencia-verifier olvassa vissza (nem importálják egymást), a 32 lefedettségi vektor a compiler rekordján is lefut, a deklarációs vektorok a compiler saját függvényén, és a #25 mércéje befagyasztott (`tests/test_conformance_frozen.py`). 72 planted defect a `tools/compiler.py`-ban. +* **A rekord többi része is a forráscommité.** Az `artifacts` a git-fából jön; a `specs` (tartalom, `meta_hash`), a három régi provenance-digest, a kiadó és az érvényesség viszont a munkakönyvtárból (séma-fájlok, `dependency.yaml`, két grammar-fájl, `project.yaml`, `commitment.yaml`). A tiszta `git status` ezt **nem** garantálja (ignorált fájl, assume-unchanged, skip-worktree, sorvég-szűrő, sparse checkout mind tiszta státuszt hagy). A `run_release` ezért a Vault előtt minden ilyen bemenetet a forráscommit blobjához hasonlít, és a talált séma-fájlokat a commit fáján végzett kiválasztáshoz; eltérés: `worktree_differs_from_source_commit`, leáll. Nem fedi: a validációs kapuk (séma-, primitive-, grammar-kapu) is a munkakönyvtárat olvassák; ítéletük olyan fájlra is szólhat, ami nem része a rekordnak. Release-t friss klónból érdemes vágni. * **`verify-release`:** elfogadja az envelope 3-at, és kimondja, hogy az artifact-lista ellenőrzése nem az ő dolga (`release_record_conformance.py`); a kettő különálló, egyik sem importálja a másikat. * **Ellenjegyző szolgáltatás (csak olvasó vizsgálat):** a `cic-countersign` a `build_hash`-t átlátszatlan, kész digestként kapja és nem számolja újra; a `document` csak naplóhoz; az envelope-ból és az `artifacts`-ból semmit nem olvas. Nem módosítottam. * **Deklaráció nélküli repó nem adhat ki release-t** (`declaration_missing`); a közzétett fájlok listája a termelő döntése, külön változásban. diff --git a/tests/e3_compiler_mutants.py b/tests/e3_compiler_mutants.py index af125a0..1d3499c 100644 --- a/tests/e3_compiler_mutants.py +++ b/tests/e3_compiler_mutants.py @@ -86,9 +86,25 @@ def m(id, edits, kills=(), equivalent=None): m("record-without-artifacts", (" 'artifacts': artifacts,\n", ""), ["writes_an_envelope_3_record", "coverage_vectors"]), m("artifacts-added-after-the-hash", [(" 'artifacts': artifacts,\n", ""), (" bundle['release']['build_hash'] = build_hash\n", " bundle['artifacts'] = artifacts\n bundle['release']['build_hash'] = build_hash\n")], ["the_hash_the_compiler_wrote", "coverage_vectors", "writes_an_envelope_3_record"]), - m("artifacts-resolved-after-vault", [(" artifacts = _resolve_release_artifacts(source_commit)\n", " artifacts = None\n"), (" 'artifacts': artifacts,\n", " 'artifacts': _resolve_release_artifacts(source_commit),\n")], + m("artifacts-resolved-after-vault", [(" artifacts = _resolve_release_artifacts(source_commit, tree)\n", " artifacts = None\n"), (" 'artifacts': artifacts,\n", " 'artifacts': _resolve_release_artifacts(source_commit, tree),\n")], ["refuses_before_any_vault_call", "without_any_declaration", "declaration_in_the_working_directory_only", "working_directory_does_not_stand_in"]), m("release-block-keeps-envelope-2", (" 'envelope': RELEASE_ENVELOPE_VERSION,\n 'createdBy': release_created_by,", " 'envelope': 2,\n 'createdBy': release_created_by,"), ["writes_an_envelope_3_record"]), m("source-paths-not-checked-for-duplicates", (" _check_unique_source_paths(specs)\n\n # 6. Build hash", " # 6. Build hash"), ["checks_the_source_paths_of_the_specs"]), m("duplicate-source-path-accepted", (" if path in seen:\n print(f\"\\033[91m✗ ERROR: spec_source_path_duplicate", " if False:\n print(f\"\\033[91m✗ ERROR: spec_source_path_duplicate"), ["duplicate_spec_source_paths"]), + # the rest of the record is the source commit's too + m("guard-not-called", (" _check_release_inputs_match_commit(tree, schema_files)\n print(", " print("), ["ignored_schema_file", "assume_unchanged", "line_end_filter", "skip_worktree", "provenance_or_the_issuer", "ignored_commitment", "working_directory_that_declares"]), + m("guard-ignores-the-bytes", (" if fh.read() != _git_blob(entry[2]):", " if False:"), ["assume_unchanged", "line_end_filter", "provenance_or_the_issuer", "working_directory_that_declares", "symbolic_link"]), + m("guard-ignores-files-not-in-the-commit", (" elif entry is None:\n problems.append((path, 'not in the source commit'))", " elif entry is None:\n pass"), ["ignored_schema_file", "ignored_commitment"]), + m("guard-ignores-missing-files", (" if not present:\n problems.append((path, 'missing from the working directory'))", " if not present:\n pass"), ["lacks_is_refused"]), + m("guard-ignores-the-commits-schema-selection", (" for path in sorted(set(_release_schema_files_in_tree(tree)) - {p.replace(os.sep, '/') for p in schema_files}):\n problems.append(", " for path in []:\n problems.append("), ["skip_worktree"]), + m("guard-accepts-a-link-in-the-commit", (" elif entry[1] != 'blob' or entry[0] not in REGULAR_FILE_MODES:\n problems.append((path, f'not a regular file", " elif False:\n problems.append((path, f'not a regular file"), [] + , equivalent="a schema file that is a link in the commit has a blob of the link target's name, which the byte comparison with the file it points to already refuses"), + m("guard-skips-project-yaml", (" read = list(schema_files) + ['project.yaml', 'commitment.yaml']", " read = list(schema_files) + ['commitment.yaml']"), ["provenance_or_the_issuer", "working_directory_that_declares"]), + m("guard-skips-commitment-yaml", (" read = list(schema_files) + ['project.yaml', 'commitment.yaml']", " read = list(schema_files) + ['project.yaml']"), ["ignored_commitment"]), + m("guard-skips-the-provenance-inputs", ("+ [path for _label, path in PROVENANCE_FILES]", "+ []"), ["provenance_or_the_issuer"]), + m("guard-skips-the-schema-files", (" read = list(schema_files) + ['project.yaml', 'commitment.yaml']", " read = ['project.yaml', 'commitment.yaml']"), ["ignored_schema_file", "assume_unchanged", "line_end_filter"]), + m("tree-selection-takes-hidden-files", (" if any(part.startswith('.') for part in path.split('/')):\n continue\n", ""), ["tree_selection_of_schema_files"]), + m("tree-selection-takes-empty-files", (" if len(_git_blob(oid)) > 0:\n selected.append(path)", " selected.append(path)"), ["tree_selection_of_schema_files"]), + m("tree-selection-takes-examples", (" if os.path.basename(path) == 'index.yaml' or '/examples/' in path:", " if os.path.basename(path) == 'index.yaml':"), ["tree_selection_of_schema_files"]), + m("tree-selection-takes-other-directories", (" if not path.startswith(source_dir + '/') or not path.endswith('.yaml'):", " if not path.endswith('.yaml'):"), ["tree_selection_of_schema_files"]), ] diff --git a/tests/e3_compiler_support.py b/tests/e3_compiler_support.py index 7161b52..1658798 100644 --- a/tests/e3_compiler_support.py +++ b/tests/e3_compiler_support.py @@ -51,6 +51,7 @@ def checkout(path, files): commit = fx.build_repo(path, files) fx._git(path, "update-ref", "refs/heads/main", commit) fx._git(path, "symbolic-ref", "HEAD", "refs/heads/main") + fx._git(path, "read-tree", "HEAD") # the index, so that `git status` means something in the tests that ask it for name, content, mode in files: if mode in ("100644", "100755"): target = pathlib.Path(path) / name @@ -68,7 +69,24 @@ def commit_files(paths): return list(fx.GOLDEN_FILES) + [("project.yaml", project_yaml(paths).encode("utf-8"), "100644")] -def release_sandbox(C, tmp_path, monkeypatch, version="0.1.6", files=None, paths=None): +def sandbox_files(extra=()): + """What run_release() reads from the working directory besides the artifacts: the schema files, project.yaml (added by the caller), dependency.yaml, the two + grammar files of the provenance digests. They are COMMITTED, because the compiler stops when the working directory and the source commit disagree.""" + import test_compiler as tc + repo = ROOT + files = [ + ("schemas/atomic/shape.yaml", (tc._META.format(name="Shape", version="v0.1.dev") + "spec:\n kind: AtomicPrimitive\n fields: {a: 1}\n").encode(), "100644"), + ("schemas/examples/c.yaml", (tc._META.format(name="Example", version="v0.1.dev") + tc._GOOD_COMPOSITION.lstrip("\n")).encode(), "100644"), + ("schemas/index.yaml", (repo / "schemas" / "index.yaml").read_bytes(), "100644"), + ("md.meta.schema.yaml", (repo / "md.meta.schema.yaml").read_bytes(), "100644"), + ("dependency.yaml", b"schema_version: '1'\n", "100644"), + ("proposals/atom-grammar/check_grammar.py", (repo / "proposals/atom-grammar/check_grammar.py").read_bytes(), "100644"), + ("proposals/atom-grammar/instance-grammar.schema.yaml", (repo / "proposals/atom-grammar/instance-grammar.schema.yaml").read_bytes(), "100644"), + ] + return files + list(extra) + + +def release_sandbox(C, tmp_path, monkeypatch, version="0.1.6", files=None, paths=None, extra_files=()): """What tests/test_compiler.py::_release_sandbox did, as a real git checkout: run_release() resolves the artifacts from the git tree of HEAD, so the sandbox has one. Vault and the developer commitment are replaced; the issuer is the test issuer of the frozen fixtures, so a record this compiler writes can be re-authenticated by the @@ -76,18 +94,8 @@ def release_sandbox(C, tmp_path, monkeypatch, version="0.1.6", files=None, paths """ import test_compiler as tc paths = golden_paths() if paths is None else paths - files = commit_files(paths) if files is None else files + files = (commit_files(paths) if files is None else list(files)) + sandbox_files(extra_files) commit = checkout(tmp_path, files) - repo = ROOT - (tmp_path / "proposals").mkdir(exist_ok=True) - (tmp_path / "proposals" / "atom-grammar").symlink_to(repo / "proposals" / "atom-grammar", target_is_directory=True) - (tmp_path / "schemas" / "atomic").mkdir(parents=True) - (tmp_path / "schemas" / "atomic" / "shape.yaml").write_text(tc._META.format(name="Shape", version="v0.1.dev") + "spec:\n kind: AtomicPrimitive\n fields: {a: 1}\n") - (tmp_path / "schemas" / "examples").mkdir() - (tmp_path / "schemas" / "examples" / "c.yaml").write_text(tc._META.format(name="Example", version="v0.1.dev") + tc._GOOD_COMPOSITION.lstrip("\n")) - (tmp_path / "dependency.yaml").write_text("schema_version: '1'\n") - (tmp_path / "md.meta.schema.yaml").write_text((repo / "md.meta.schema.yaml").read_text()) - (tmp_path / "schemas" / "index.yaml").write_text((repo / "schemas" / "index.yaml").read_text()) monkeypatch.chdir(tmp_path) calls = {"vault": 0} @@ -112,7 +120,6 @@ def vault_sign(digest, *a, **k): monkeypatch.setattr(C, "_load_valid_commitment", lambda require_signature=True: commitment) monkeypatch.setattr(C, "_vault_get_cert", vault_cert) monkeypatch.setattr(C, "_vault_sign", vault_sign) - # the settings of the WORKING DIRECTORY'S project.yaml: deliberately without the declaration, which has to come from the commit monkeypatch.setattr(C, "load_project_config", lambda full_config=False: ( {"project": {"name": "cic-primitives", "owner": "Dev"}, "compiler_settings": settings} if full_config else settings)) monkeypatch.setattr(C, "CONFIG", settings) @@ -144,3 +151,8 @@ def git_tree_via_plumbing(repo, commit): mode, kind, oid = meta.decode().split(" ") tree[name.decode("utf-8", "surrogateescape")] = (mode, kind, oid) return tree + + +def porcelain(path): + """What validate_release_prerequisites asks: `git status --porcelain` (untracked files included, ignored ones not).""" + return subprocess.run(["git", "-C", str(path), "status", "--porcelain"], capture_output=True, text=True, check=True).stdout diff --git a/tests/test_compiler.py b/tests/test_compiler.py index bd926c2..478f4af 100644 --- a/tests/test_compiler.py +++ b/tests/test_compiler.py @@ -1021,7 +1021,7 @@ def test_vault_get_cert_rejects_a_malformed_path(): } -def _release_sandbox(tmp_path, monkeypatch, mocker, version="0.1.6"): +def _release_sandbox(tmp_path, monkeypatch, mocker, version="0.1.6", extra_files=()): """A tree run_release() can complete in without Vault or a network. Worth the setup: run_release had 104 of its 120 lines uncovered, and it is @@ -1036,7 +1036,7 @@ def _release_sandbox(tmp_path, monkeypatch, mocker, version="0.1.6"): share it. """ import e3_compiler_support as sup - root, version, _commit, _calls = sup.release_sandbox(compiler, tmp_path, monkeypatch, version=version) + root, version, _commit, _calls = sup.release_sandbox(compiler, tmp_path, monkeypatch, version=version, extra_files=extra_files) return root, version @@ -1438,12 +1438,11 @@ def test_release_collects_specs_from_nested_directories(tmp_path, monkeypatch, m Without recursion the bundle silently omits deeper schemas — a release that claims to cover the primitive set while shipping part of it. """ - root, version = _release_sandbox(tmp_path, monkeypatch, mocker) - deep = root / "schemas" / "atomic" / "nested" - deep.mkdir() - (deep / "deep.yaml").write_text( - _META.format(name="Deep", version="v0.1.dev") - + "spec:\n kind: AtomicPrimitive\n fields: {a: 1}\n") + deep = (_META.format(name="Deep", version="v0.1.dev") + + "spec:\n kind: AtomicPrimitive\n fields: {a: 1}\n").encode() + # committed: since envelope 3 the compiler stops at a schema file the source commit does not hold + root, version = _release_sandbox(tmp_path, monkeypatch, mocker, + extra_files=[("schemas/atomic/nested/deep.yaml", deep, "100644")]) compiler.run_release() bundle = yaml.safe_load( (root / "release" / f"cic-primitives-v{version}.yaml").read_text()) diff --git a/tests/test_compiler_envelope3.py b/tests/test_compiler_envelope3.py index 5126b4f..8913b97 100644 --- a/tests/test_compiler_envelope3.py +++ b/tests/test_compiler_envelope3.py @@ -161,11 +161,21 @@ def test_a_filter_in_the_checkout_does_not_reach_the_digest(tmp_path, monkeypatc def test_the_declaration_comes_from_the_commit_not_from_the_working_directory(tmp_path, monkeypatch): - root, version, _commit, _calls = sup.release_sandbox(C, tmp_path, monkeypatch) - # the working copy of project.yaml declares something else entirely; the commit's declaration is the one that counts + root, _version, commit, _calls = sup.release_sandbox(C, tmp_path, monkeypatch) + # the working copy of project.yaml declares something else entirely; the commit's declaration is the one that is read (root / "project.yaml").write_text(sup.project_yaml(["a.txt"])) - run_release_in(root, monkeypatch) - assert [f["path"] for f in sup.written_record(root, version)["artifacts"]["files"]] == sorted(sup.golden_paths(), key=lambda p: p.encode("utf-8")) + project = C._declaration_from_commit(C._git_tree(commit)) + assert project["compiler_settings"]["release_artifacts"]["paths"] == sup.golden_paths() + + +def test_a_working_directory_that_declares_something_else_stops_the_release(tmp_path, monkeypatch, capsys): + root, _version, _commit, calls = sup.release_sandbox(C, tmp_path, monkeypatch) + (root / "project.yaml").write_text(sup.project_yaml(["a.txt"])) + with pytest.raises(SystemExit) as stop: + run_release_in(root, monkeypatch) + out = capsys.readouterr().out + assert stop.value.code == 1 and "worktree_differs_from_source_commit" in out and "project.yaml" in out + assert calls["vault"] == 0 and not (root / "release").exists() def test_a_declaration_in_the_working_directory_only_is_no_declaration(tmp_path, monkeypatch, capsys): @@ -309,6 +319,108 @@ def test_duplicate_spec_source_paths_are_refused(monkeypatch): C._check_unique_source_paths([{"source_path": "schemas/atomic/a.yaml"}, {"source_path": "schemas/atomic/b.yaml"}]) +# ---------------------------------------------------------------- the rest of the record is the source commit's too + +SHAPE = "schemas/atomic/shape.yaml" + + +def refused_for(root, monkeypatch, capsys, calls, *words): + with pytest.raises(SystemExit) as stop: + run_release_in(root, monkeypatch) + out = capsys.readouterr().out + assert stop.value.code == 1 and "worktree_differs_from_source_commit" in out, out + for word in words: + assert word in out, out + assert calls["vault"] == 0 and not (root / "release").exists() + + +def test_a_clean_sandbox_is_consistent_and_the_guard_names_what_it_checked(tmp_path, monkeypatch, capsys): + root, version, _commit, _calls = sup.release_sandbox(C, tmp_path, monkeypatch) + assert sup.porcelain(root) == "" + run_release_in(root, monkeypatch) + assert "are the source commit's bytes" in capsys.readouterr().out + + +def test_an_ignored_schema_file_leaves_git_status_clean_and_is_refused(tmp_path, monkeypatch, capsys): + root, _v, _c, calls = sup.release_sandbox(C, tmp_path, monkeypatch, extra_files=[(".gitignore", b"schemas/atomic/ignored.yaml\n", "100644")]) + import test_compiler as tc + (root / "schemas" / "atomic" / "ignored.yaml").write_text(tc._META.format(name="Ignored", version="v0.1.dev") + "spec:\n kind: AtomicPrimitive\n fields: {a: 1}\n") + assert sup.porcelain(root) == "", "the premise: the status the prerequisites ask is clean" + refused_for(root, monkeypatch, capsys, calls, "schemas/atomic/ignored.yaml", "not in the source commit") + + +def test_an_assume_unchanged_schema_file_with_other_bytes_is_refused(tmp_path, monkeypatch, capsys): + root, _v, _c, calls = sup.release_sandbox(C, tmp_path, monkeypatch) + fx._git(root, "update-index", "--assume-unchanged", SHAPE) + (root / SHAPE).write_text((root / SHAPE).read_text().replace("a: 1", "a: 2")) + assert sup.porcelain(root) == "" + refused_for(root, monkeypatch, capsys, calls, SHAPE, "differ from the source commit") + + +def test_a_line_end_filter_leaves_git_status_clean_and_is_refused(tmp_path, monkeypatch, capsys): + root, _v, _c, calls = sup.release_sandbox(C, tmp_path, monkeypatch, extra_files=[(".gitattributes", b"*.yaml text eol=crlf\n", "100644")]) + for yaml_file in (root / "schemas").rglob("*.yaml"): + yaml_file.write_bytes(yaml_file.read_bytes().replace(b"\n", b"\r\n")) + assert sup.porcelain(root) == "", "the premise: a CRLF working copy of a text=eol=crlf file is clean to git" + refused_for(root, monkeypatch, capsys, calls, "differ from the source commit") + + +def test_a_skip_worktree_schema_file_that_is_missing_is_refused(tmp_path, monkeypatch, capsys): + import test_compiler as tc + content = (tc._META.format(name="Sparse", version="v0.1.dev") + "spec:\n kind: AtomicPrimitive\n fields: {a: 1}\n").encode() + root, _v, _c, calls = sup.release_sandbox(C, tmp_path, monkeypatch, extra_files=[("schemas/atomic/sparse.yaml", content, "100644")]) + fx._git(root, "update-index", "--skip-worktree", "schemas/atomic/sparse.yaml") + (root / "schemas" / "atomic" / "sparse.yaml").unlink() + assert sup.porcelain(root) == "", "the premise: a sparse checkout leaves the status clean" + refused_for(root, monkeypatch, capsys, calls, "schemas/atomic/sparse.yaml", "does not offer") + + +@pytest.mark.parametrize("path", ["dependency.yaml", "proposals/atom-grammar/check_grammar.py", "proposals/atom-grammar/instance-grammar.schema.yaml", "project.yaml"]) +def test_an_input_of_the_provenance_or_the_issuer_with_other_bytes_is_refused(path, tmp_path, monkeypatch, capsys): + root, _v, _c, calls = sup.release_sandbox(C, tmp_path, monkeypatch) + fx._git(root, "update-index", "--assume-unchanged", path) + with open(root / path, "ab") as fh: + fh.write(b"\n# changed after the commit\n") + assert sup.porcelain(root) == "" + refused_for(root, monkeypatch, capsys, calls, path, "differ from the source commit") + + +def test_a_provenance_input_that_the_commit_holds_and_the_working_directory_lacks_is_refused(tmp_path, monkeypatch, capsys): + path = "dependency.yaml" # the grammar files are read by the release grammar gate first, which stops on its own + root, _v, _c, calls = sup.release_sandbox(C, tmp_path, monkeypatch) + fx._git(root, "update-index", "--skip-worktree", path) + (root / path).unlink() + assert sup.porcelain(root) == "" + refused_for(root, monkeypatch, capsys, calls, path, "missing from the working directory") + + +def test_an_ignored_commitment_file_that_the_commit_does_not_hold_is_refused(tmp_path, monkeypatch, capsys): + root, _v, _c, calls = sup.release_sandbox(C, tmp_path, monkeypatch, extra_files=[(".gitignore", b"commitment.yaml\n", "100644")]) + (root / "commitment.yaml").write_text("kind: DeveloperCommitment\n") + assert sup.porcelain(root) == "" + refused_for(root, monkeypatch, capsys, calls, "commitment.yaml", "not in the source commit") + + +def test_a_symbolic_link_for_a_schema_file_is_refused(tmp_path, monkeypatch, capsys): + root, _v, _c, calls = sup.release_sandbox(C, tmp_path, monkeypatch) + fx._git(root, "update-index", "--assume-unchanged", SHAPE) + (root / SHAPE).unlink() + (root / SHAPE).symlink_to(root / "schemas" / "examples" / "c.yaml") + refused_for(root, monkeypatch, capsys, calls, SHAPE) + + +def test_the_tree_selection_of_schema_files_is_the_glob_selection(tmp_path, monkeypatch): + import test_compiler as tc + meta = tc._META.format(name="X", version="v0.1.dev") + "spec:\n kind: AtomicPrimitive\n fields: {a: 1}\n" + extra = [("schemas/atomic/.hidden.yaml", meta.encode(), "100644"), ("schemas/atomic/empty.yaml", b"", "100644"), ("schemas/atomic/nested/deep.yaml", meta.encode(), "100644"), + ("schemas/aggregate/agg.yaml", meta.encode(), "100644"), ("schemas/examples/x.yaml", meta.encode(), "100644"), ("schemas/atomic/notes.txt", b"x", "100644"), + ("schemas/.dot/inside.yaml", meta.encode(), "100644"), ("other/atomic/outside.yaml", meta.encode(), "100644")] + root, _v, commit, _c = sup.release_sandbox(C, tmp_path, monkeypatch, extra_files=extra) + from_tree = C._release_schema_files_in_tree(C._git_tree(commit)) + assert from_tree == C._release_schema_files() + assert "schemas/atomic/nested/deep.yaml" in from_tree and "schemas/atomic/.hidden.yaml" not in from_tree and "schemas/atomic/empty.yaml" not in from_tree + + # ---------------------------------------------------------------- the signature covers what the compiler wrote @pytest.fixture(scope="module") diff --git a/tools/compiler.py b/tools/compiler.py index e38d8fe..4b4acbe 100644 --- a/tools/compiler.py +++ b/tools/compiler.py @@ -148,13 +148,17 @@ def _source_commit(): return commit +# The files whose bytes the three legacy provenance digests are taken from (the working directory; see _check_release_inputs_match_commit). +PROVENANCE_FILES = (('dependency_lock', 'dependency.yaml'), + ('grammar', 'proposals/atom-grammar/check_grammar.py'), + ('grammar_schema', 'proposals/atom-grammar/instance-grammar.schema.yaml')) + + def _collect_provenance(commit=None): """Facts about WHERE this build came from, so the signature can cover them.""" provenance = {'envelope': RELEASE_ENVELOPE_VERSION, 'source_commit': commit or _source_commit()} - for label, path in (('dependency_lock', 'dependency.yaml'), - ('grammar', 'proposals/atom-grammar/check_grammar.py'), - ('grammar_schema', 'proposals/atom-grammar/instance-grammar.schema.yaml')): + for label, path in PROVENANCE_FILES: try: with open(path, 'rb') as fh: provenance[f'{label}_sha256'] = get_sha256_b64(fh.read()) @@ -275,9 +279,9 @@ def _build_release_artifacts(paths, tree): return {'manifest_digest': 'sha256:' + get_sha256_hex(manifest.encode('utf-8')), 'files': files} -def _resolve_release_artifacts(commit): +def _resolve_release_artifacts(commit, tree=None): """Declaration -> refusal or the `artifacts` block, from the git tree of `commit` only.""" - tree = _git_tree(commit) + tree = _git_tree(commit) if tree is None else tree project = _declaration_from_commit(tree) refusal = _validate_release_artifact_declaration(project, tree) if refusal: @@ -286,6 +290,72 @@ def _resolve_release_artifacts(commit): return _build_release_artifacts(project['compiler_settings']['release_artifacts']['paths'], tree) +def _release_schema_files(): + """The schema files a release carries as `specs`, as found in the WORKING DIRECTORY: atomic + aggregate, no examples, no index.""" + source_dir = CONFIG.get('source_dir', 'schemas') + all_files = glob.glob(os.path.join(source_dir, '**', '*.yaml'), recursive=True) + return sorted([ + f for f in all_files + if os.path.basename(f) != 'index.yaml' + and not f.endswith('.gitkeep') + and '/examples/' not in f.replace(os.sep, '/') + and os.path.getsize(f) > 0 + ]) + + +def _release_schema_files_in_tree(tree): + """The same selection as _release_schema_files, made on the git tree of the source commit (glob skips names that start with a dot).""" + source_dir = CONFIG.get('source_dir', 'schemas').strip('/') + selected = [] + for path, (mode, kind, oid) in tree.items(): + if kind != 'blob' or mode not in REGULAR_FILE_MODES: + continue + if not path.startswith(source_dir + '/') or not path.endswith('.yaml'): + continue + if any(part.startswith('.') for part in path.split('/')): + continue + if os.path.basename(path) == 'index.yaml' or '/examples/' in path: + continue + if len(_git_blob(oid)) > 0: + selected.append(path) + return sorted(selected) + + +def _check_release_inputs_match_commit(tree, schema_files): + """Everything the record takes from the WORKING DIRECTORY must be byte-identical to the source commit's blob. + + The `artifacts` block is read from the git tree. The rest of the record is not: `specs` (their content and `meta_hash`) come from the schema files, the three + legacy provenance digests from dependency.yaml and the two grammar files, the issuer and the developer's validity from project.yaml and commitment.yaml. A clean + `git status` (validate_release_prerequisites) does not make those the commit's bytes: an ignored file, a file marked assume-unchanged or skip-worktree, a line-end + filter, a symbolic link or a sparse checkout all leave the status clean. A record whose `specs` and whose `artifacts` come from two different states of the + source would contradict itself without anyone having lied, so the compiler stops (reason `worktree_differs_from_source_commit`) before it asks Vault for anything. + """ + read = list(schema_files) + ['project.yaml', 'commitment.yaml'] + [path for _label, path in PROVENANCE_FILES] + problems = [] + for path in read: + entry = tree.get(path.replace(os.sep, '/')) + present = os.path.lexists(path) + if not present and entry is None: + continue # absent on both sides: nothing was read + if not present: + problems.append((path, 'missing from the working directory')) + elif entry is None: + problems.append((path, 'not in the source commit')) + elif entry[1] != 'blob' or entry[0] not in REGULAR_FILE_MODES: + problems.append((path, f'not a regular file in the source commit (mode {entry[0]}, type {entry[1]})')) + else: + with open(path, 'rb') as fh: + if fh.read() != _git_blob(entry[2]): + problems.append((path, 'its bytes differ from the source commit')) + for path in sorted(set(_release_schema_files_in_tree(tree)) - {p.replace(os.sep, '/') for p in schema_files}): + problems.append((path, 'a schema file of the source commit that the working directory does not offer')) + if problems: + print("\033[91m✗ ERROR: worktree_differs_from_source_commit: the record would be made of two different states of the source\033[0m") + for path, why in problems: + print(f" {path}: {why}") + sys.exit(1) + + def _check_unique_source_paths(specs): """Two `specs` entries must not share a `source_path` (section 7): a later one never overwrites an earlier one.""" seen = set() @@ -862,9 +932,15 @@ def run_release(): # refused must not cost a signature, and the commit that names them is the one the # record will carry. source_commit = _source_commit() - artifacts = _resolve_release_artifacts(source_commit) + tree = _git_tree(source_commit) + artifacts = _resolve_release_artifacts(source_commit, tree) print(f" \033[92m✓ {len(artifacts['files'])} release artifacts digested from the tree of " f"{source_commit[:12]}\033[0m") + # ... and nothing else of the record may come from a different state than that tree + schema_files = _release_schema_files() + _check_release_inputs_match_commit(tree, schema_files) + print(" \033[92m✓ the schema files, project.yaml, commitment.yaml and the provenance inputs " + "are the source commit's bytes\033[0m") vault_addr = os.getenv('VAULT_ADDR') vault_token = os.getenv('VAULT_TOKEN') @@ -911,17 +987,7 @@ def run_release(): run_primitive_validation() run_domain_compatibility_check() - # 4. Collect schema files: atomic + aggregate, no examples, no index - source_dir = CONFIG.get('source_dir', 'schemas') - all_files = glob.glob(os.path.join(source_dir, '**', '*.yaml'), recursive=True) - schema_files = sorted([ - f for f in all_files - if os.path.basename(f) != 'index.yaml' - and not f.endswith('.gitkeep') - and '/examples/' not in f.replace(os.sep, '/') - and os.path.getsize(f) > 0 - ]) - + # 4. The schema files were collected (and checked against the source commit) above print(f"--- Building release bundle ({len(schema_files)} schemas) ---") # 5. Build specs[] — meta_hash over raw file bytes, version placeholder injected in spec content From adde7db9629184d133317704f9789f0be2788c09 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?G=C3=A1bor=20Z=20Sink=C3=B3?= Date: Sun, 1 Nov 2026 23:18:17 +0100 Subject: [PATCH 5/6] ci(gate): keep the 10 minute timeout The gate with both planted-defect proofs takes under two minutes on the runner (41353832789, and the run of bb67999), so the raise to 20 minutes is not needed. --- [signing-metadata] key = cic-my-sign-key signature = vault:v1:MEUCICS3E2UVdOUCI4C5abqdISj99vGKKzoWygOjyXNMafqbAiEAqWlyV8H0E1KGq4j9wqRlhFF70Eww35lbfZAA1uyckHc= hash-algorithm = sha256 digest = 73RGsb/kOHJPszJwCVTMp8vu97nfWTCoVIB7I6XjQNQ= [certificate] -----BEGIN CERTIFICATE----- MIICBjCCAaygAwIBAgIUSnRMR6RPnEbg296XWPOqq/u5PCwwCgYIKoZIzj0EAwIw QzELMAkGA1UEBhMCSFUxGTAXBgNVBAoMEENlbnRyYWxJbmZyYUNvcmUxGTAXBgNV BAMMEENJQyBEZXZlbG9wZXIgQ0EwHhcNMjYwMzIwMTMyMjU5WhcNMjYxMjMxMTMy MjU5WjBFMQswCQYDVQQGEwJIVTEZMBcGA1UECgwQQ2VudHJhbEluZnJhQ29yZTEb MBkGA1UEAwwSR2Fib3IgWm9sdGFuIFNpbmtvMFkwEwYHKoZIzj0CAQYIKoZIzj0D AQcDQgAEIG2CVmTfmLB9pLLclj7YmP2eedAjklpy4LGrU2ijoiy6Xqpuybv7OgJe i+ez31s65NEV8+X/ByeX1cstR988z6N8MHowCQYDVR0TBAIwADAdBgNVHQ4EFgQU yZN6AIX/TNnIJ9GwAa/NRN3ujHAwHwYDVR0jBBgwFoAUXn6CHYzPUqU4JVP8g+OS WeDYjhcwDgYDVR0PAQH/BAQDAgeAMB0GA1UdJQQWMBQGCCsGAQUFBwMCBggrBgEF BQcDBDAKBggqhkjOPQQDAgNIADBFAiEA+bFzXRoJ4PCQbhAAtpkcMjt0vNj5rEW0 lOMBGDNyaWkCIB1vmM7PcZzv/c9bIrxF5kqv6QXomouhByUfeNUTbpKW -----END CERTIFICATE----- --- .github/workflows/gate.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/gate.yml b/.github/workflows/gate.yml index 2c00574..4f79ff6 100644 --- a/.github/workflows/gate.yml +++ b/.github/workflows/gate.yml @@ -38,9 +38,7 @@ jobs: gate: name: gate runs-on: ubuntu-latest - # 20, not 10: the suite now includes the planted-defect proofs of the envelope 3 conformance and of the compiler's envelope 3 output (about 160 source breaks, - # each run against the tests that target it, four at a time); locally the whole gate takes about 3.5 minutes - timeout-minutes: 20 + timeout-minutes: 10 steps: - uses: actions/checkout@v4 From 83e30eaaac0f4ec878e3b81b9f3e752f9358059f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?G=C3=A1bor=20Z=20Sink=C3=B3?= Date: Sun, 1 Nov 2026 23:42:40 +0100 Subject: [PATCH 6/6] feat(release): required inputs absent on both sides are named; every published schema must be listed A required input of the release (project.yaml, commitment.yaml, the two grammar files) that is neither in the working directory nor in the source commit is no longer passed over by the consistency check; each is an end-to-end test with the real commitment loader. Every specs[].source_path must be among artifacts.files (spec_not_in_artifacts, before any Vault call). Two characterization tests pin the boundary that remains: the validation gates read the working directory (a composition that only the commit holds is not judged, and an ignored bad example that only the working directory holds stops the release). The test of the required inputs no longer reads its requirement from the constant it checks. Ten more planted defects. --- [signing-metadata] key = cic-my-sign-key signature = vault:v1:MEQCIDS+LZ9cJGjMwAapjY768DYynDx3AvxOAcz4R6pUxkhIAiA9og5WE1PCWJZknuRNm+A4Qx5uCb0EHPOvkCnDAFjaGQ== hash-algorithm = sha256 digest = lMCF5xmp2gGvuxFvdQAdhPyDmgLK/wf+Kln4Kt8rAOg= [certificate] -----BEGIN CERTIFICATE----- MIICBjCCAaygAwIBAgIUSnRMR6RPnEbg296XWPOqq/u5PCwwCgYIKoZIzj0EAwIw QzELMAkGA1UEBhMCSFUxGTAXBgNVBAoMEENlbnRyYWxJbmZyYUNvcmUxGTAXBgNV BAMMEENJQyBEZXZlbG9wZXIgQ0EwHhcNMjYwMzIwMTMyMjU5WhcNMjYxMjMxMTMy MjU5WjBFMQswCQYDVQQGEwJIVTEZMBcGA1UECgwQQ2VudHJhbEluZnJhQ29yZTEb MBkGA1UEAwwSR2Fib3IgWm9sdGFuIFNpbmtvMFkwEwYHKoZIzj0CAQYIKoZIzj0D AQcDQgAEIG2CVmTfmLB9pLLclj7YmP2eedAjklpy4LGrU2ijoiy6Xqpuybv7OgJe i+ez31s65NEV8+X/ByeX1cstR988z6N8MHowCQYDVR0TBAIwADAdBgNVHQ4EFgQU yZN6AIX/TNnIJ9GwAa/NRN3ujHAwHwYDVR0jBBgwFoAUXn6CHYzPUqU4JVP8g+OS WeDYjhcwDgYDVR0PAQH/BAQDAgeAMB0GA1UdJQQWMBQGCCsGAQUFBwMCBggrBgEF BQcDBDAKBggqhkjOPQQDAgNIADBFAiEA+bFzXRoJ4PCQbhAAtpkcMjt0vNj5rEW0 lOMBGDNyaWkCIB1vmM7PcZzv/c9bIrxF5kqv6QXomouhByUfeNUTbpKW -----END CERTIFICATE----- --- docs/en/RELEASE-COMPILER-ENVELOPE3.md | 8 +- docs/hu/RELEASE-COMPILER-ENVELOPE3.md | 3 +- tests/e3_compiler_mutants.py | 16 ++- tests/e3_compiler_support.py | 36 +++++-- tests/test_compiler_envelope3.py | 139 ++++++++++++++++++++++++-- tools/compiler.py | 26 ++++- 6 files changed, 208 insertions(+), 20 deletions(-) diff --git a/docs/en/RELEASE-COMPILER-ENVELOPE3.md b/docs/en/RELEASE-COMPILER-ENVELOPE3.md index 15aa32b..0b12cd5 100644 --- a/docs/en/RELEASE-COMPILER-ENVELOPE3.md +++ b/docs/en/RELEASE-COMPILER-ENVELOPE3.md @@ -27,7 +27,11 @@ A repository whose `project.yaml` has no `release_artifacts` cannot cut a releas So `run_release` compares, before it asks Vault for anything, every one of those inputs with the blob of the source commit, and the schema files it found with the selection made on the commit's tree. A difference (`worktree_differs_from_source_commit`, with the file and what differs) stops the release: a record made of two states of the source would contradict itself without anyone having lied. -What this does not cover: the validation gates (`run_validation`, `run_primitive_validation`, the grammar gate) read the working directory too. Their verdict can concern a file that is not part of the record (an ignored example, for instance) and is not about the commit for such files; it does not enter the record. A release should be cut from a fresh clone of the commit. +An input that exists **neither** in the working directory nor in the source commit is not consistency either: `project.yaml`, `commitment.yaml` and the two grammar files are required inputs of the release and the check names them itself (`dependency.yaml` is the one optional input: without it the legacy digest is `null`, as before). Each is a test end to end with the real commitment loader. + +**Every published schema is listed.** Every `specs[].source_path` the record would publish must be among `artifacts.files` (`spec_not_in_artifacts`, naming the files, before any Vault call). A published schema cannot be missing from the artifact manifest. This is a producer-side refusal of the compiler; it is not a rule of the record contract (a record-level check would be a revision of the contract and of the frozen yardstick, and is not made here). + +What this does not cover: the validation gates (`run_validation`, `run_primitive_validation`, the grammar gate) read the working directory too. Their verdict can concern a file that is not part of the record (an ignored example, for instance) and is not about the commit for such files; it does not enter the record. A release should be cut from a fresh clone of the commit. The boundary is pinned by two characterization tests (not wishes): a composition that the commit holds and the grammar rejects, left out of the working directory with `git status` clean, passes the gate and the release is cut; and an ignored bad example that the commit does not hold stops a release of a good commit. If the boundary is ever closed, those tests change on purpose. ## How the output is held to the contract @@ -37,7 +41,7 @@ The compiler does not say that its own output is right. * The 32 signature coverage vectors of the contract run on the record the compiler wrote, not only on the golden record; the structural vectors too. * The declaration vectors run on the compiler's own functions, and on the reference's, and the two must agree. * The yardstick of PR #25 (the reference verifier, the golden fixtures, the vectors, the planted-defect proof) is frozen at its merge commit: `tests/test_conformance_frozen.py` fails if any of it changes together with the compiler. -* 72 planted defects in `tools/compiler.py` (`tests/e3_compiler_mutants.py`): 63 have to turn one of the tests that target them red, 9 cannot change behaviour, are listed with the reason and have to survive. +* 82 planted defects in `tools/compiler.py` (`tests/e3_compiler_mutants.py`): 72 have to turn one of the tests that target them red, 10 cannot change behaviour, are listed with the reason and have to survive. ## `verify-release` diff --git a/docs/hu/RELEASE-COMPILER-ENVELOPE3.md b/docs/hu/RELEASE-COMPILER-ENVELOPE3.md index 1bb1eed..352c622 100644 --- a/docs/hu/RELEASE-COMPILER-ENVELOPE3.md +++ b/docs/hu/RELEASE-COMPILER-ENVELOPE3.md @@ -6,8 +6,9 @@ A `python tools/compiler.py release` kizárólag **envelope 3** `PrimitiveReleas * **A termelő deklarálja a közzétett fájlokat** a `project.yaml`-ban (`compiler_settings.release_artifacts.paths`, explicit útvonalak, glob/`include`/`exclude` nélkül). A deklarációt a **forráscommit**ból olvassa, az útvonalakat a commit **Git-fájából** oldja fel; nem a munkakönyvtárból, nem szűrt checkoutból, nem fogyasztói pin-manifestből. Könyvtár, link, submodule, abszolút útvonal, `..`/`.`, vezérlőkarakter, duplikátum, hiányzó útvonal, üres lista, ismeretlen kulcs: leáll az okkal, **még a Vault hívása előtt**. * **`artifacts`:** útvonalanként `sha256:` + a blob bájtjainak SHA-256-ja, az útvonal UTF-8 bájtjai szerint rendezve, plusz `manifest_digest`. A `specs[].source_path` egyedi. * **Aláírás:** a meglévő mechanizmus (`build_hash` a teljes csomagra, `release.sign`, külső `cic_countersign`); az `artifacts` és a `source_commit` ezért lefedett. -* **Nem a compiler igazolja magát:** a kimenetet a referencia-verifier olvassa vissza (nem importálják egymást), a 32 lefedettségi vektor a compiler rekordján is lefut, a deklarációs vektorok a compiler saját függvényén, és a #25 mércéje befagyasztott (`tests/test_conformance_frozen.py`). 72 planted defect a `tools/compiler.py`-ban. +* **Nem a compiler igazolja magát:** a kimenetet a referencia-verifier olvassa vissza (nem importálják egymást), a 32 lefedettségi vektor a compiler rekordján is lefut, a deklarációs vektorok a compiler saját függvényén, és a #25 mércéje befagyasztott (`tests/test_conformance_frozen.py`). 82 planted defect a `tools/compiler.py`-ban. * **A rekord többi része is a forráscommité.** Az `artifacts` a git-fából jön; a `specs` (tartalom, `meta_hash`), a három régi provenance-digest, a kiadó és az érvényesség viszont a munkakönyvtárból (séma-fájlok, `dependency.yaml`, két grammar-fájl, `project.yaml`, `commitment.yaml`). A tiszta `git status` ezt **nem** garantálja (ignorált fájl, assume-unchanged, skip-worktree, sorvég-szűrő, sparse checkout mind tiszta státuszt hagy). A `run_release` ezért a Vault előtt minden ilyen bemenetet a forráscommit blobjához hasonlít, és a talált séma-fájlokat a commit fáján végzett kiválasztáshoz; eltérés: `worktree_differs_from_source_commit`, leáll. Nem fedi: a validációs kapuk (séma-, primitive-, grammar-kapu) is a munkakönyvtárat olvassák; ítéletük olyan fájlra is szólhat, ami nem része a rekordnak. Release-t friss klónból érdemes vágni. +* **Kötelező bemenet és a közzétett sémák:** ami sem a munkakönyvtárban, sem a commitban nincs, az nem konzisztencia: a `project.yaml`, a `commitment.yaml` és a két grammar-fájl kötelező bemenet, a guard maga nevezi meg (a `dependency.yaml` az egyetlen opcionális: nélküle a régi digest `null`). Minden `specs[].source_path` benne kell legyen az `artifacts.files`-ban (`spec_not_in_artifacts`, a Vault előtt); ez termelői oldali elutasítás, nem rekord-szerződési szabály. A fennmaradó határt két jellemző teszt rögzíti: a validációs kapuk a munkakönyvtárat ítélik (a commitban lévő, de a munkakönyvtárból kihagyott rossz példát a kapu nem látja, az ignorált rossz példa viszont megállítja a jó commit release-ét). * **`verify-release`:** elfogadja az envelope 3-at, és kimondja, hogy az artifact-lista ellenőrzése nem az ő dolga (`release_record_conformance.py`); a kettő különálló, egyik sem importálja a másikat. * **Ellenjegyző szolgáltatás (csak olvasó vizsgálat):** a `cic-countersign` a `build_hash`-t átlátszatlan, kész digestként kapja és nem számolja újra; a `document` csak naplóhoz; az envelope-ból és az `artifacts`-ból semmit nem olvas. Nem módosítottam. * **Deklaráció nélküli repó nem adhat ki release-t** (`declaration_missing`); a közzétett fájlok listája a termelő döntése, külön változásban. diff --git a/tests/e3_compiler_mutants.py b/tests/e3_compiler_mutants.py index 1d3499c..bd55896 100644 --- a/tests/e3_compiler_mutants.py +++ b/tests/e3_compiler_mutants.py @@ -89,10 +89,10 @@ def m(id, edits, kills=(), equivalent=None): m("artifacts-resolved-after-vault", [(" artifacts = _resolve_release_artifacts(source_commit, tree)\n", " artifacts = None\n"), (" 'artifacts': artifacts,\n", " 'artifacts': _resolve_release_artifacts(source_commit, tree),\n")], ["refuses_before_any_vault_call", "without_any_declaration", "declaration_in_the_working_directory_only", "working_directory_does_not_stand_in"]), m("release-block-keeps-envelope-2", (" 'envelope': RELEASE_ENVELOPE_VERSION,\n 'createdBy': release_created_by,", " 'envelope': 2,\n 'createdBy': release_created_by,"), ["writes_an_envelope_3_record"]), - m("source-paths-not-checked-for-duplicates", (" _check_unique_source_paths(specs)\n\n # 6. Build hash", " # 6. Build hash"), ["checks_the_source_paths_of_the_specs"]), + m("source-paths-not-checked-for-duplicates", (" _check_unique_source_paths(specs)\n _check_specs_in_artifacts(", " _check_specs_in_artifacts("), ["checks_the_source_paths_of_the_specs"]), m("duplicate-source-path-accepted", (" if path in seen:\n print(f\"\\033[91m✗ ERROR: spec_source_path_duplicate", " if False:\n print(f\"\\033[91m✗ ERROR: spec_source_path_duplicate"), ["duplicate_spec_source_paths"]), # the rest of the record is the source commit's too - m("guard-not-called", (" _check_release_inputs_match_commit(tree, schema_files)\n print(", " print("), ["ignored_schema_file", "assume_unchanged", "line_end_filter", "skip_worktree", "provenance_or_the_issuer", "ignored_commitment", "working_directory_that_declares"]), + m("guard-not-called", (" _check_release_inputs_match_commit(tree, schema_files)\n _check_specs_in_artifacts(schema_files, artifacts)\n print(", " _check_specs_in_artifacts(schema_files, artifacts)\n print("), ["ignored_schema_file", "assume_unchanged", "line_end_filter", "skip_worktree", "provenance_or_the_issuer", "ignored_commitment", "working_directory_that_declares"]), m("guard-ignores-the-bytes", (" if fh.read() != _git_blob(entry[2]):", " if False:"), ["assume_unchanged", "line_end_filter", "provenance_or_the_issuer", "working_directory_that_declares", "symbolic_link"]), m("guard-ignores-files-not-in-the-commit", (" elif entry is None:\n problems.append((path, 'not in the source commit'))", " elif entry is None:\n pass"), ["ignored_schema_file", "ignored_commitment"]), m("guard-ignores-missing-files", (" if not present:\n problems.append((path, 'missing from the working directory'))", " if not present:\n pass"), ["lacks_is_refused"]), @@ -107,4 +107,16 @@ def m(id, edits, kills=(), equivalent=None): m("tree-selection-takes-empty-files", (" if len(_git_blob(oid)) > 0:\n selected.append(path)", " selected.append(path)"), ["tree_selection_of_schema_files"]), m("tree-selection-takes-examples", (" if os.path.basename(path) == 'index.yaml' or '/examples/' in path:", " if os.path.basename(path) == 'index.yaml':"), ["tree_selection_of_schema_files"]), m("tree-selection-takes-other-directories", (" if not path.startswith(source_dir + '/') or not path.endswith('.yaml'):", " if not path.endswith('.yaml'):"), ["tree_selection_of_schema_files"]), + # required inputs and the published schemas + m("required-input-absent-on-both-sides-passes", (" if path in REQUIRED_RELEASE_INPUTS:\n problems.append(", " if False:\n problems.append("), ["names_every_required_input", "end_to_end"]), + m("commitment-not-required", ("REQUIRED_RELEASE_INPUTS = ('project.yaml', 'commitment.yaml',", "REQUIRED_RELEASE_INPUTS = ('project.yaml',"), ["names_every_required_input", "end_to_end"]), + m("project-yaml-not-required", ("REQUIRED_RELEASE_INPUTS = ('project.yaml', 'commitment.yaml',", "REQUIRED_RELEASE_INPUTS = ('commitment.yaml',"), ["names_every_required_input"]), + m("grammar-schema-not-required", (" 'proposals/atom-grammar/instance-grammar.schema.yaml')", " )"), ["names_every_required_input"]), + m("dependency-yaml-required", (" 'proposals/atom-grammar/instance-grammar.schema.yaml')", " 'proposals/atom-grammar/instance-grammar.schema.yaml', 'dependency.yaml')"), ["optional_input", "names_every_required_input"]), + m("specs-not-checked-against-the-artifacts", (" if missing:\n print(\"\\033[91m✗ ERROR: spec_not_in_artifacts", " if False:\n print(\"\\033[91m✗ ERROR: spec_not_in_artifacts"), ["stops_the_release", "nested_schema_that_is_missing", "forward_slash_paths"]), + m("spec-check-after-vault-only", (" _check_specs_in_artifacts(schema_files, artifacts)\n print(", " print("), ["stops_the_release", "nested_schema_that_is_missing"]), + m("spec-check-of-the-written-specs-missing", (" _check_specs_in_artifacts([entry['source_path'] for entry in specs], artifacts)\n\n", "\n"), [], + equivalent="the specs are built from the same schema files the early check already compared, in the same run; the second call repeats the first on the list that is actually written and cannot disagree with it"), + m("spec-check-lists-every-path", (" missing = sorted(path.replace(os.sep, '/') for path in source_paths if path.replace(os.sep, '/') not in listed)", " missing = sorted(path.replace(os.sep, '/') for path in source_paths)"), ["every_spec_source_path", "writes_an_envelope_3_record"]), + m("spec-check-wrong-side", (" listed = {entry['path'] for entry in artifacts['files']}\n missing = sorted(", " listed = {entry['path'] for entry in artifacts['files']} | set(source_paths)\n missing = sorted("), ["stops_the_release", "nested_schema_that_is_missing", "forward_slash_paths"]), ] diff --git a/tests/e3_compiler_support.py b/tests/e3_compiler_support.py index 1658798..c5c7eac 100644 --- a/tests/e3_compiler_support.py +++ b/tests/e3_compiler_support.py @@ -69,7 +69,7 @@ def commit_files(paths): return list(fx.GOLDEN_FILES) + [("project.yaml", project_yaml(paths).encode("utf-8"), "100644")] -def sandbox_files(extra=()): +def sandbox_files(extra=(), without=()): """What run_release() reads from the working directory besides the artifacts: the schema files, project.yaml (added by the caller), dependency.yaml, the two grammar files of the provenance digests. They are COMMITTED, because the compiler stops when the working directory and the source commit disagree.""" import test_compiler as tc @@ -80,21 +80,44 @@ def sandbox_files(extra=()): ("schemas/index.yaml", (repo / "schemas" / "index.yaml").read_bytes(), "100644"), ("md.meta.schema.yaml", (repo / "md.meta.schema.yaml").read_bytes(), "100644"), ("dependency.yaml", b"schema_version: '1'\n", "100644"), + ("commitment.yaml", b"kind: DeveloperCommitment\n", "100644"), # a placeholder: the signed pledge is replaced by the test, but the file is a required input ("proposals/atom-grammar/check_grammar.py", (repo / "proposals/atom-grammar/check_grammar.py").read_bytes(), "100644"), ("proposals/atom-grammar/instance-grammar.schema.yaml", (repo / "proposals/atom-grammar/instance-grammar.schema.yaml").read_bytes(), "100644"), ] - return files + list(extra) + return [f for f in files if f[0] not in without] + list(extra) -def release_sandbox(C, tmp_path, monkeypatch, version="0.1.6", files=None, paths=None, extra_files=()): +def spec_paths(extra=()): + """The schema files the sandbox's release carries as `specs` (the selection of the compiler: under schemas/, not an example, not the index, not hidden, not empty).""" + paths = ["schemas/atomic/shape.yaml"] + for name, content, _mode in extra: + parts = name.split("/") + if name.startswith("schemas/") and name.endswith(".yaml") and "examples" not in parts and parts[-1] != "index.yaml" and not any(p.startswith(".") for p in parts) and content: + paths.append(name) + return paths + + +def sandbox_paths(extra=()): + """The default declaration of the sandbox: the golden files and every schema the record publishes (a spec that is not listed is refused).""" + return golden_paths() + spec_paths(extra) + + +def expected_artifacts(paths, extra=()): + """The artifacts block of `paths`, computed here from the committed bytes with the independent implementation (sha256 of the bytes, sha256sum-format manifest).""" + content = {name: data for name, data, _m in list(fx.GOLDEN_FILES) + sandbox_files(extra)} + files = [{"path": p, "digest": "sha256:" + fx.sha256_hex(content[p])} for p in sorted(paths, key=lambda p: p.encode("utf-8"))] + return {"manifest_digest": fx.independent_manifest_digest(files), "files": files} + + +def release_sandbox(C, tmp_path, monkeypatch, version="0.1.6", files=None, paths=None, extra_files=(), without=(), real_commitment=False): """What tests/test_compiler.py::_release_sandbox did, as a real git checkout: run_release() resolves the artifacts from the git tree of HEAD, so the sandbox has one. Vault and the developer commitment are replaced; the issuer is the test issuer of the frozen fixtures, so a record this compiler writes can be re-authenticated by the same test keys as the golden record. Returns (directory, version, source_commit, calls) where `calls` counts the Vault calls. """ import test_compiler as tc - paths = golden_paths() if paths is None else paths - files = (commit_files(paths) if files is None else list(files)) + sandbox_files(extra_files) + paths = sandbox_paths(extra_files) if paths is None else paths + files = (commit_files(paths) if files is None else list(files)) + sandbox_files(extra_files, without) commit = checkout(tmp_path, files) monkeypatch.chdir(tmp_path) @@ -117,7 +140,8 @@ def vault_sign(digest, *a, **k): monkeypatch.setenv("VAULT_ADDR", "https://vault") monkeypatch.setenv("VAULT_TOKEN", "token") monkeypatch.setattr(C, "validate_release_prerequisites", lambda: (version, "primitives/")) - monkeypatch.setattr(C, "_load_valid_commitment", lambda require_signature=True: commitment) + if not real_commitment: + monkeypatch.setattr(C, "_load_valid_commitment", lambda require_signature=True: commitment) monkeypatch.setattr(C, "_vault_get_cert", vault_cert) monkeypatch.setattr(C, "_vault_sign", vault_sign) monkeypatch.setattr(C, "load_project_config", lambda full_config=False: ( diff --git a/tests/test_compiler_envelope3.py b/tests/test_compiler_envelope3.py index 8913b97..4110aac 100644 --- a/tests/test_compiler_envelope3.py +++ b/tests/test_compiler_envelope3.py @@ -117,8 +117,10 @@ def test_run_release_writes_an_envelope_3_record_that_meets_the_contract(tmp_pat assert bundle["release"]["envelope"] == bundle["provenance"]["envelope"] == 3 assert bundle["provenance"]["source_commit"] == commit # the artifacts block equals the independently recomputed one (sha256sum of the worked example), not the output of another run of the compiler - expected = fx.load_expected() - assert bundle["artifacts"] == {"manifest_digest": expected["manifest_digest"], "files": expected["files"]} + assert bundle["artifacts"] == sup.expected_artifacts(sup.sandbox_paths()) + # the three golden files in it are the worked example of the contract, digest for digest + golden = {f["path"]: f["digest"] for f in fx.load_expected()["files"]} + assert {f["path"]: f["digest"] for f in bundle["artifacts"]["files"] if f["path"] in golden} == golden report = V.verify(sup.countersign(bundle), str(root)) assert report["record"].status == "PASS", (report["record"].reason, report["record"].detail) assert report["blobs"].status == "PASS", report["blobs"].failures @@ -148,16 +150,16 @@ def test_the_digest_is_of_the_blob_bytes_not_of_the_working_directory(tmp_path, (root / (fx.E_ACUTE + ".txt")).unlink() run_release_in(root, monkeypatch) bundle = sup.written_record(root, version) - assert bundle["artifacts"]["files"] == fx.load_expected()["files"] + assert bundle["artifacts"] == sup.expected_artifacts(sup.sandbox_paths()) def test_a_filter_in_the_checkout_does_not_reach_the_digest(tmp_path, monkeypatch): attributes = ("100644", b"*.txt text eol=crlf\n") - files = sup.commit_files(sup.golden_paths()) + [(".gitattributes", attributes[1], attributes[0])] + files = sup.commit_files(sup.sandbox_paths()) + [(".gitattributes", attributes[1], attributes[0])] root, version, _commit, _calls = sup.release_sandbox(C, tmp_path, monkeypatch, files=files) fx._git(root, "config", "core.autocrlf", "true") run_release_in(root, monkeypatch) - assert sup.written_record(root, version)["artifacts"]["files"] == fx.load_expected()["files"] + assert sup.written_record(root, version)["artifacts"] == sup.expected_artifacts(sup.sandbox_paths()) def test_the_declaration_comes_from_the_commit_not_from_the_working_directory(tmp_path, monkeypatch): @@ -165,7 +167,7 @@ def test_the_declaration_comes_from_the_commit_not_from_the_working_directory(tm # the working copy of project.yaml declares something else entirely; the commit's declaration is the one that is read (root / "project.yaml").write_text(sup.project_yaml(["a.txt"])) project = C._declaration_from_commit(C._git_tree(commit)) - assert project["compiler_settings"]["release_artifacts"]["paths"] == sup.golden_paths() + assert project["compiler_settings"]["release_artifacts"]["paths"] == sup.sandbox_paths() def test_a_working_directory_that_declares_something_else_stops_the_release(tmp_path, monkeypatch, capsys): @@ -191,7 +193,7 @@ def test_a_declaration_in_the_working_directory_only_is_no_declaration(tmp_path, def test_the_working_directory_does_not_stand_in_for_the_commit_on_the_paths_either(tmp_path, monkeypatch, capsys): # a file that is only in the working directory, not in the commit, is not an artifact - root, version, _commit, calls = sup.release_sandbox(C, tmp_path, monkeypatch, paths=sup.golden_paths() + ["untracked.txt"]) + root, version, _commit, calls = sup.release_sandbox(C, tmp_path, monkeypatch, paths=sup.sandbox_paths() + ["untracked.txt"]) (root / "untracked.txt").write_text("only here\n") with pytest.raises(SystemExit) as stop: run_release_in(root, monkeypatch) @@ -395,12 +397,59 @@ def test_a_provenance_input_that_the_commit_holds_and_the_working_directory_lack def test_an_ignored_commitment_file_that_the_commit_does_not_hold_is_refused(tmp_path, monkeypatch, capsys): - root, _v, _c, calls = sup.release_sandbox(C, tmp_path, monkeypatch, extra_files=[(".gitignore", b"commitment.yaml\n", "100644")]) + root, _v, _c, calls = sup.release_sandbox(C, tmp_path, monkeypatch, without=("commitment.yaml",), extra_files=[(".gitignore", b"commitment.yaml\n", "100644")]) (root / "commitment.yaml").write_text("kind: DeveloperCommitment\n") assert sup.porcelain(root) == "" refused_for(root, monkeypatch, capsys, calls, "commitment.yaml", "not in the source commit") +WHO_NAMES_IT = { + # the step that stops the release for a required input that exists on neither side; the check of this module names commitment.yaml itself + "commitment.yaml": "a required input of the release", + "project.yaml": "declaration_missing", + "proposals/atom-grammar/check_grammar.py": "not found", + "proposals/atom-grammar/instance-grammar.schema.yaml": "cannot load the atom grammar", +} + + +@pytest.mark.parametrize("path", sorted(WHO_NAMES_IT)) +def test_a_required_input_that_is_absent_on_both_sides_stops_the_release_end_to_end(path, tmp_path, monkeypatch, capsys): + # with the REAL commitment loader: the input exists neither in the working directory nor in the source commit + root, _v, _c, calls = sup.release_sandbox(C, tmp_path, monkeypatch, without=(path,), real_commitment=True, + files=[f for f in sup.commit_files(sup.sandbox_paths()) if f[0] != path]) + assert not (root / path).exists() + with pytest.raises(SystemExit) as stop: + run_release_in(root, monkeypatch) + out = capsys.readouterr().out + assert stop.value.code == 1 and WHO_NAMES_IT[path] in out, out + assert calls["vault"] == 0 and not (root / "release").exists() + + +def test_the_real_commitment_loader_would_stop_on_a_missing_commitment_too(tmp_path, monkeypatch, capsys): + monkeypatch.chdir(tmp_path) + with pytest.raises(SystemExit): + C._load_valid_commitment() + assert "commitment.yaml not found" in capsys.readouterr().out + + +def test_the_check_names_every_required_input_absent_on_both_sides_itself(tmp_path, monkeypatch, capsys): + monkeypatch.chdir(tmp_path) + with pytest.raises(SystemExit): + C._check_release_inputs_match_commit({}, []) + out = capsys.readouterr().out + # the requirement is written here, not read from the constant it checks + for path in ("project.yaml", "commitment.yaml", "proposals/atom-grammar/check_grammar.py", "proposals/atom-grammar/instance-grammar.schema.yaml"): + assert path in out, path + assert "dependency.yaml" not in out + + +def test_dependency_yaml_is_the_one_optional_input(tmp_path, monkeypatch): + # absent on both sides: the legacy digest is null, as it always was, and the release goes on + root, version, _c, _calls = sup.release_sandbox(C, tmp_path, monkeypatch, without=("dependency.yaml",)) + run_release_in(root, monkeypatch) + assert sup.written_record(root, version)["provenance"]["dependency_lock_sha256"] is None + + def test_a_symbolic_link_for_a_schema_file_is_refused(tmp_path, monkeypatch, capsys): root, _v, _c, calls = sup.release_sandbox(C, tmp_path, monkeypatch) fx._git(root, "update-index", "--assume-unchanged", SHAPE) @@ -421,6 +470,80 @@ def test_the_tree_selection_of_schema_files_is_the_glob_selection(tmp_path, monk assert "schemas/atomic/nested/deep.yaml" in from_tree and "schemas/atomic/.hidden.yaml" not in from_tree and "schemas/atomic/empty.yaml" not in from_tree +# ---------------------------------------------------------------- every published schema is among the artifacts + +def test_a_schema_the_record_publishes_but_the_declaration_does_not_list_stops_the_release(tmp_path, monkeypatch, capsys): + root, _v, _c, calls = sup.release_sandbox(C, tmp_path, monkeypatch, paths=sup.golden_paths()) + with pytest.raises(SystemExit) as stop: + run_release_in(root, monkeypatch) + out = capsys.readouterr().out + assert stop.value.code == 1 and "spec_not_in_artifacts" in out and "schemas/atomic/shape.yaml" in out + assert calls["vault"] == 0 and not (root / "release").exists() + + +def test_a_nested_schema_that_is_missing_from_the_declaration_is_named(tmp_path, monkeypatch, capsys): + import test_compiler as tc + deep = (tc._META.format(name="Deep", version="v0.1.dev") + "spec:\n kind: AtomicPrimitive\n fields: {a: 1}\n").encode() + extra = [("schemas/atomic/nested/deep.yaml", deep, "100644")] + root, _v, _c, calls = sup.release_sandbox(C, tmp_path, monkeypatch, paths=sup.sandbox_paths(), extra_files=extra) + with pytest.raises(SystemExit): + run_release_in(root, monkeypatch) + out = capsys.readouterr().out + assert "spec_not_in_artifacts" in out and "schemas/atomic/nested/deep.yaml" in out and "schemas/atomic/shape.yaml" not in out + assert calls["vault"] == 0 + + +def test_every_spec_source_path_of_the_written_record_is_among_its_artifacts(tmp_path, monkeypatch): + root, version, _c, _calls = sup.release_sandbox(C, tmp_path, monkeypatch) + run_release_in(root, monkeypatch) + bundle = sup.written_record(root, version) + listed = {f["path"] for f in bundle["artifacts"]["files"]} + assert bundle["specs"] and all(spec["source_path"] in listed for spec in bundle["specs"]) + + +def test_the_spec_check_compares_forward_slash_paths(): + artifacts = {"files": [{"path": "schemas/atomic/a.yaml", "digest": "sha256:" + "0" * 64}]} + C._check_specs_in_artifacts(["schemas/atomic/a.yaml"], artifacts) + with pytest.raises(SystemExit): + C._check_specs_in_artifacts(["schemas/atomic/b.yaml"], artifacts) + + +# ---------------------------------------------------------------- the boundary that remains: the validation gates read the working directory + +def test_a_composition_only_in_the_commit_is_not_judged_by_the_gate_the_boundary(tmp_path, monkeypatch, capsys): + """CHARACTERIZATION, not a wish. The grammar gate reads `schemas/examples/*.yaml` from the WORKING DIRECTORY. The consistency check compares what enters the record + (the schemas, the provenance inputs, project.yaml, commitment.yaml), and an example is not part of the record, so a composition that the commit holds and the + grammar rejects can be left out of the working directory (sparse checkout, skip-worktree) while `git status` stays clean: the gate passes and the release is cut. + The gate's verdict is then about the working directory, not about the commit. If this behaviour is ever closed, this test is to be changed on purpose.""" + import test_compiler as tc + bad = ("schemas/examples/bad.yaml", (tc._META.format(name="Bad", version="v0.1.dev") + tc._BAD_COMPOSITION.lstrip("\n")).encode(), "100644") + root, version, commit, _calls = sup.release_sandbox(C, tmp_path, monkeypatch, extra_files=[bad]) + fx._git(root, "update-index", "--skip-worktree", bad[0]) + (root / bad[0]).unlink() + assert sup.porcelain(root) == "" + run_release_in(root, monkeypatch) # the release is cut + assert sup.written_record(root, version)["provenance"]["source_commit"] == commit + # ... although the commit it is cut from holds a composition the same gate rejects once it is given that file + (root / bad[0]).write_bytes(bad[1]) + capsys.readouterr() + with pytest.raises(SystemExit) as stop: + C.run_grammar_gate() + assert stop.value.code == 1 + assert "grammar finding" in capsys.readouterr().out + + +def test_a_composition_only_in_the_working_directory_decides_the_gate_the_other_direction(tmp_path, monkeypatch, capsys): + """The same boundary the other way: an ignored bad example that the commit does not hold stops a release of a commit that is fine.""" + import test_compiler as tc + root, _v, _c, calls = sup.release_sandbox(C, tmp_path, monkeypatch, extra_files=[(".gitignore", b"schemas/examples/ignored-bad.yaml\n", "100644")]) + (root / "schemas" / "examples" / "ignored-bad.yaml").write_text(tc._META.format(name="Bad", version="v0.1.dev") + tc._BAD_COMPOSITION.lstrip("\n")) + assert sup.porcelain(root) == "" + with pytest.raises(SystemExit) as stop: + run_release_in(root, monkeypatch) + assert stop.value.code == 1 and "grammar finding" in capsys.readouterr().out + assert calls["vault"] == 0 + + # ---------------------------------------------------------------- the signature covers what the compiler wrote @pytest.fixture(scope="module") diff --git a/tools/compiler.py b/tools/compiler.py index 4b4acbe..478d960 100644 --- a/tools/compiler.py +++ b/tools/compiler.py @@ -321,6 +321,14 @@ def _release_schema_files_in_tree(tree): return sorted(selected) +# Inputs the release path cannot do without. Absent from the working directory AND from the source commit is not consistency, it is a missing input: the check says +# so itself instead of leaving it to whichever later step happens to notice (commitment.yaml: _load_valid_commitment; the grammar files: run_grammar_gate; +# project.yaml: the import of this module). dependency.yaml is optional: without it the legacy digest is recorded as null, as it always was. +REQUIRED_RELEASE_INPUTS = ('project.yaml', 'commitment.yaml', + 'proposals/atom-grammar/check_grammar.py', + 'proposals/atom-grammar/instance-grammar.schema.yaml') + + def _check_release_inputs_match_commit(tree, schema_files): """Everything the record takes from the WORKING DIRECTORY must be byte-identical to the source commit's blob. @@ -336,7 +344,9 @@ def _check_release_inputs_match_commit(tree, schema_files): entry = tree.get(path.replace(os.sep, '/')) present = os.path.lexists(path) if not present and entry is None: - continue # absent on both sides: nothing was read + if path in REQUIRED_RELEASE_INPUTS: + problems.append((path, 'a required input of the release, absent from the working directory and from the source commit')) + continue # an optional input absent on both sides: nothing was read if not present: problems.append((path, 'missing from the working directory')) elif entry is None: @@ -356,6 +366,18 @@ def _check_release_inputs_match_commit(tree, schema_files): sys.exit(1) +def _check_specs_in_artifacts(source_paths, artifacts): + """Every schema the record publishes as a `spec` must also be among the digested artifacts: a published schema must not be missing from the artifact manifest.""" + listed = {entry['path'] for entry in artifacts['files']} + missing = sorted(path.replace(os.sep, '/') for path in source_paths if path.replace(os.sep, '/') not in listed) + if missing: + print("\033[91m✗ ERROR: spec_not_in_artifacts: the record would publish schemas that " + "release_artifacts.paths does not list:\033[0m") + for path in missing: + print(f" {path}") + sys.exit(1) + + def _check_unique_source_paths(specs): """Two `specs` entries must not share a `source_path` (section 7): a later one never overwrites an earlier one.""" seen = set() @@ -939,6 +961,7 @@ def run_release(): # ... and nothing else of the record may come from a different state than that tree schema_files = _release_schema_files() _check_release_inputs_match_commit(tree, schema_files) + _check_specs_in_artifacts(schema_files, artifacts) print(" \033[92m✓ the schema files, project.yaml, commitment.yaml and the provenance inputs " "are the source commit's bytes\033[0m") @@ -1007,6 +1030,7 @@ def run_release(): print(f" - {schema_file}: {meta_hash[:16]}...") _check_unique_source_paths(specs) + _check_specs_in_artifacts([entry['source_path'] for entry in specs], artifacts) # 6. Build hash: {createdBy (developer), release.createdBy (issuer), specs, validity} # Both certificates are cryptographically bound before Vault signs.