From ba95174677ca4c0f2d8cb1e39f2e2334eb02e884 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?G=C3=A1bor=20Z=20Sink=C3=B3?= Date: Sun, 18 Oct 2026 23:41:42 +0200 Subject: [PATCH 1/2] feat: assemble the conformance/drift verdict artifact in Rust and Go, with a differential corpus --- [signing-metadata] key = cic-my-sign-key signature = vault:v1:MEYCIQCaJU+Ywl9cqHVRvUUg1TydmrdkSvH8KC/yd+DagIipGQIhAJi+Pw1e3ul5M6wrOBh4PWCd3MDn/VNUW8tiTLxBt+Ec hash-algorithm = sha256 digest = F0YbnUHqn8NXdvaZXkmXMGhp/AzNSHaNZsRM7W2FdCg= [certificate] -----BEGIN CERTIFICATE----- MIICBjCCAaygAwIBAgIUSnRMR6RPnEbg296XWPOqq/u5PCwwCgYIKoZIzj0EAwIw QzELMAkGA1UEBhMCSFUxGTAXBgNVBAoMEENlbnRyYWxJbmZyYUNvcmUxGTAXBgNV BAMMEENJQyBEZXZlbG9wZXIgQ0EwHhcNMjYwMzIwMTMyMjU5WhcNMjYxMjMxMTMy MjU5WjBFMQswCQYDVQQGEwJIVTEZMBcGA1UECgwQQ2VudHJhbEluZnJhQ29yZTEb MBkGA1UEAwwSR2Fib3IgWm9sdGFuIFNpbmtvMFkwEwYHKoZIzj0CAQYIKoZIzj0D AQcDQgAEIG2CVmTfmLB9pLLclj7YmP2eedAjklpy4LGrU2ijoiy6Xqpuybv7OgJe i+ez31s65NEV8+X/ByeX1cstR988z6N8MHowCQYDVR0TBAIwADAdBgNVHQ4EFgQU yZN6AIX/TNnIJ9GwAa/NRN3ujHAwHwYDVR0jBBgwFoAUXn6CHYzPUqU4JVP8g+OS WeDYjhcwDgYDVR0PAQH/BAQDAgeAMB0GA1UdJQQWMBQGCCsGAQUFBwMCBggrBgEF BQcDBDAKBggqhkjOPQQDAgNIADBFAiEA+bFzXRoJ4PCQbhAAtpkcMjt0vNj5rEW0 lOMBGDNyaWkCIB1vmM7PcZzv/c9bIrxF5kqv6QXomouhByUfeNUTbpKW -----END CERTIFICATE----- --- README.md | 12 +- conformance/differential/README.md | 18 + .../expected.json | 13 + .../multi_key_collection_identity/input.json | 26 ++ .../verdict/oci_conformant/expected.json | 16 + .../verdict/oci_conformant/input.json | 40 ++ .../expected.json | 17 + .../input.json | 40 ++ .../verdict/oci_drift/expected.json | 16 + .../differential/verdict/oci_drift/input.json | 38 ++ .../verdict/oci_not_comparable/expected.json | 16 + .../verdict/oci_not_comparable/input.json | 38 ++ .../verdict/oci_unobserved/expected.json | 16 + .../verdict/oci_unobserved/input.json | 37 ++ .../expected.json | 7 + .../input.json | 93 +++++ .../expected.json | 7 + .../input.json | 93 +++++ .../expected.json | 7 + .../input.json | 93 +++++ .../expected.json | 7 + .../input.json | 97 +++++ .../expected.json | 16 + .../unknown_coverage_flows_through/input.json | 38 ++ docs/VERDICT-SCHEMA.md | 32 ++ engine/src/error.rs | 7 + engine/src/lib.rs | 5 + engine/src/verdict.rs | 353 +++++++++++++++++ engine/tests/differential.rs | 259 +++++++++++- go/plan/plan.go | 7 + go/verdict/differential_test.go | 371 ++++++++++++++++++ go/verdict/verdict.go | 168 ++++++++ go/verdict/verdict_test.go | 160 ++++++++ 33 files changed, 2158 insertions(+), 5 deletions(-) create mode 100644 conformance/differential/verdict/multi_key_collection_identity/expected.json create mode 100644 conformance/differential/verdict/multi_key_collection_identity/input.json create mode 100644 conformance/differential/verdict/oci_conformant/expected.json create mode 100644 conformance/differential/verdict/oci_conformant/input.json create mode 100644 conformance/differential/verdict/oci_desired_absent_is_conformant/expected.json create mode 100644 conformance/differential/verdict/oci_desired_absent_is_conformant/input.json create mode 100644 conformance/differential/verdict/oci_drift/expected.json create mode 100644 conformance/differential/verdict/oci_drift/input.json create mode 100644 conformance/differential/verdict/oci_not_comparable/expected.json create mode 100644 conformance/differential/verdict/oci_not_comparable/input.json create mode 100644 conformance/differential/verdict/oci_unobserved/expected.json create mode 100644 conformance/differential/verdict/oci_unobserved/input.json create mode 100644 conformance/differential/verdict/tampered_evidence_for_unclassified_path/expected.json create mode 100644 conformance/differential/verdict/tampered_evidence_for_unclassified_path/input.json create mode 100644 conformance/differential/verdict/tampered_missing_consumed_evidence/expected.json create mode 100644 conformance/differential/verdict/tampered_missing_consumed_evidence/input.json create mode 100644 conformance/differential/verdict/tampered_object_not_following_fields/expected.json create mode 100644 conformance/differential/verdict/tampered_object_not_following_fields/input.json create mode 100644 conformance/differential/verdict/tampered_planned_scalar_missing/expected.json create mode 100644 conformance/differential/verdict/tampered_planned_scalar_missing/input.json create mode 100644 conformance/differential/verdict/unknown_coverage_flows_through/expected.json create mode 100644 conformance/differential/verdict/unknown_coverage_flows_through/input.json create mode 100644 engine/src/verdict.rs create mode 100644 go/verdict/differential_test.go create mode 100644 go/verdict/verdict.go create mode 100644 go/verdict/verdict_test.go diff --git a/README.md b/README.md index b7788f5..d4aa5aa 100644 --- a/README.md +++ b/README.md @@ -101,7 +101,17 @@ correction to the pipeline framing this file used to carry. > correction below, had zero internal callers, and keeping it around > would have let the tree imply a scope the contract now denies; see > "What does not belong here" for why removal, not just naming the gap, -> was the right close. Nothing here is a stable API. +> was the right close. +> +> **The conformance/drift verdict artifact exists in both languages:** +> `engine/src/verdict.rs` and `go/verdict` assemble the five required fields +> of `docs/VERDICT-SCHEMA.md` (`verdict_schema_version: 1`) from `evaluate`'s +> result and the existing digest projections, refuse an inconsistent result +> (`E_INCONSISTENT_VERDICT`), and are cross-checked byte-for-byte through +> `conformance/differential/verdict/`, including a test that the corpus +> detects deliberate divergence. +> +> Nothing here is a stable API. --- diff --git a/conformance/differential/README.md b/conformance/differential/README.md index 01468ab..edac046 100644 --- a/conformance/differential/README.md +++ b/conformance/differential/README.md @@ -42,6 +42,7 @@ differential/ plan/ -- F8: plan.Evaluate digest/ -- F9: digestprojection.* materialized/ -- MaterializedObject::try_new / NewMaterializedObject + verdict/ -- the assembled five-field verdict artifact (build_verdict / verdict.Build) ``` One group per primitive, landed incrementally — matching how F6 @@ -203,3 +204,20 @@ is, only whether both sides agree on being numeric or not) — so no vector here depends on it — but a future vector author relying on exact-comparator behavior that distinguishes `16` from `16.0` by *spelling* would hit a real asymmetry this layer does not paper over. + +### `verdict/` + +```text +input.json: the `plan/` group's input (intent, observed, observation, plan), plus an + optional `tamper` list of edits to the evaluate result before the artifact + is built: {op: drop_consumed|add_consumed|drop_field|set_object, path|value} +expected.json: an artifact vector: {verdict: , canonical: } + a refusal vector: {error: {code, path}} +``` + +Both languages must produce the identical canonical bytes, and a success +vector's `object`/`fields` must equal the same-named `plan/` vector's pin. A +second test per language applies eight deliberate divergences to a correct +artifact and requires the comparison to catch each. + diff --git a/conformance/differential/verdict/multi_key_collection_identity/expected.json b/conformance/differential/verdict/multi_key_collection_identity/expected.json new file mode 100644 index 0000000..69317f1 --- /dev/null +++ b/conformance/differential/verdict/multi_key_collection_identity/expected.json @@ -0,0 +1,13 @@ +{ + "verdict": { + "conformance_plan_digest": "sha256:deb83850cc0f689d6f0c643694567125bae9b2268bdaa6cf5ac20dddf1094683", + "fields": { + "/nics/{name=nic-0,zone=eu}/subnet": "DRIFT" + }, + "object": "DRIFT", + "observation_digest": "sha256:66fe8f76ccc06b0722d74a23fcd893e4167dcd3e6942218e1e6e08a60f333a72", + "verdict_schema_version": 1 + }, + "canonical": "{\"conformance_plan_digest\":\"sha256:deb83850cc0f689d6f0c643694567125bae9b2268bdaa6cf5ac20dddf1094683\",\"fields\":{\"/nics/{name=nic-0,zone=eu}/subnet\":\"DRIFT\"},\"object\":\"DRIFT\",\"observation_digest\":\"sha256:66fe8f76ccc06b0722d74a23fcd893e4167dcd3e6942218e1e6e08a60f333a72\",\"verdict_schema_version\":1}", + "why": "a multi-key element identity is part of the evaluated path in fields" +} diff --git a/conformance/differential/verdict/multi_key_collection_identity/input.json b/conformance/differential/verdict/multi_key_collection_identity/input.json new file mode 100644 index 0000000..50fd904 --- /dev/null +++ b/conformance/differential/verdict/multi_key_collection_identity/input.json @@ -0,0 +1,26 @@ +{ + "intent": { + "nics": [ + {"name": "nic-0", "zone": "eu", "subnet": "prod-a"} + ] + }, + "observed": { + "nics": [ + {"name": "nic-0", "zone": "eu", "subnet": "prod-WRONG"} + ] + }, + "observation": [ + {"path": "/nics/{name=nic-0,zone=eu}/subnet", "coverage": "observed"} + ], + "plan": { + "scalars": [], + "collections": [ + { + "path": "/nics", + "topology": "map", + "keys": ["name", "zone"], + "elements": [{"path": "subnet", "compare": "exact"}] + } + ] + } +} diff --git a/conformance/differential/verdict/oci_conformant/expected.json b/conformance/differential/verdict/oci_conformant/expected.json new file mode 100644 index 0000000..2a356ca --- /dev/null +++ b/conformance/differential/verdict/oci_conformant/expected.json @@ -0,0 +1,16 @@ +{ + "verdict": { + "conformance_plan_digest": "sha256:d973c7d2e9cf02f49ad1f59889a0dd2aaabd98b41d30c07377c63fdafd387850", + "fields": { + "/memory_gb": "CONFORMANT", + "/network_interfaces/{name=nic-0}/subnet": "CONFORMANT", + "/network_interfaces/{name=nic-1}/subnet": "CONFORMANT", + "/shape": "CONFORMANT" + }, + "object": "CONFORMANT", + "observation_digest": "sha256:5e24b8bb061a32c1ee09f8ede05374d7b13cfd89798985eb1a426781c93c2fe5", + "verdict_schema_version": 1 + }, + "canonical": "{\"conformance_plan_digest\":\"sha256:d973c7d2e9cf02f49ad1f59889a0dd2aaabd98b41d30c07377c63fdafd387850\",\"fields\":{\"/memory_gb\":\"CONFORMANT\",\"/network_interfaces/{name=nic-0}/subnet\":\"CONFORMANT\",\"/network_interfaces/{name=nic-1}/subnet\":\"CONFORMANT\",\"/shape\":\"CONFORMANT\"},\"object\":\"CONFORMANT\",\"observation_digest\":\"sha256:5e24b8bb061a32c1ee09f8ede05374d7b13cfd89798985eb1a426781c93c2fe5\",\"verdict_schema_version\":1}", + "why": "every planned field matches: the artifact carries the CONFORMANT object verdict, the four evaluated paths, and the two digests" +} diff --git a/conformance/differential/verdict/oci_conformant/input.json b/conformance/differential/verdict/oci_conformant/input.json new file mode 100644 index 0000000..ce167a8 --- /dev/null +++ b/conformance/differential/verdict/oci_conformant/input.json @@ -0,0 +1,40 @@ +{ + "intent": { + "shape": "VM.Standard.E4.Flex", + "memory_gb": 16, + "network_interfaces": [ + {"name": "nic-0", "subnet": {"ref": "prod-subnet-a", "kind": "Subnet"}}, + {"name": "nic-1", "subnet": {"ref": "prod-subnet-b", "kind": "Subnet"}} + ] + }, + "observed": { + "shape": "VM.Standard.E4.Flex", + "memory_gb": 16.0, + "provider_id": "ocid1.instance.oc1..aaaa", + "lifecycle_state": "RUNNING", + "network_interfaces": [ + {"name": "nic-0", "subnet": {"ref": "prod-subnet-a", "kind": "Subnet"}, "private_ip": "10.0.1.17"}, + {"name": "nic-1", "subnet": {"ref": "prod-subnet-b", "kind": "Subnet"}, "private_ip": "10.0.2.9"} + ] + }, + "observation": [ + {"path": "/shape", "coverage": "observed"}, + {"path": "/memory_gb", "coverage": "observed"}, + {"path": "/network_interfaces/{name=nic-0}/subnet", "coverage": "observed"}, + {"path": "/network_interfaces/{name=nic-1}/subnet", "coverage": "observed"} + ], + "plan": { + "scalars": [ + {"path": "/shape", "compare": "exact"}, + {"path": "/memory_gb", "compare": "numeric"} + ], + "collections": [ + { + "path": "/network_interfaces", + "topology": "map", + "keys": ["name"], + "elements": [{"path": "subnet", "compare": "exact"}] + } + ] + } +} diff --git a/conformance/differential/verdict/oci_desired_absent_is_conformant/expected.json b/conformance/differential/verdict/oci_desired_absent_is_conformant/expected.json new file mode 100644 index 0000000..66b07d0 --- /dev/null +++ b/conformance/differential/verdict/oci_desired_absent_is_conformant/expected.json @@ -0,0 +1,17 @@ +{ + "verdict": { + "conformance_plan_digest": "sha256:928988250ac70e1fc043022a372c2d72b5aa3a3018f98ac84ad3faa36e086fc4", + "fields": { + "/boot_volume": "OBSERVED_ABSENT", + "/memory_gb": "CONFORMANT", + "/network_interfaces/{name=nic-0}/subnet": "CONFORMANT", + "/network_interfaces/{name=nic-1}/subnet": "CONFORMANT", + "/shape": "CONFORMANT" + }, + "object": "CONFORMANT", + "observation_digest": "sha256:449a85439d54f9c4497e80ec85070a927a8519f9e619728f8d83760d241885ab", + "verdict_schema_version": 1 + }, + "canonical": "{\"conformance_plan_digest\":\"sha256:928988250ac70e1fc043022a372c2d72b5aa3a3018f98ac84ad3faa36e086fc4\",\"fields\":{\"/boot_volume\":\"OBSERVED_ABSENT\",\"/memory_gb\":\"CONFORMANT\",\"/network_interfaces/{name=nic-0}/subnet\":\"CONFORMANT\",\"/network_interfaces/{name=nic-1}/subnet\":\"CONFORMANT\",\"/shape\":\"CONFORMANT\"},\"object\":\"CONFORMANT\",\"observation_digest\":\"sha256:449a85439d54f9c4497e80ec85070a927a8519f9e619728f8d83760d241885ab\",\"verdict_schema_version\":1}", + "why": "a field the intent does not declare and the observation finds absent is OBSERVED_ABSENT and does not drift the object" +} diff --git a/conformance/differential/verdict/oci_desired_absent_is_conformant/input.json b/conformance/differential/verdict/oci_desired_absent_is_conformant/input.json new file mode 100644 index 0000000..24fe2b8 --- /dev/null +++ b/conformance/differential/verdict/oci_desired_absent_is_conformant/input.json @@ -0,0 +1,40 @@ +{ + "intent": { + "shape": "VM.Standard.E4.Flex", + "memory_gb": 16, + "network_interfaces": [ + {"name": "nic-0", "subnet": {"ref": "prod-subnet-a", "kind": "Subnet"}}, + {"name": "nic-1", "subnet": {"ref": "prod-subnet-b", "kind": "Subnet"}} + ] + }, + "observed": { + "shape": "VM.Standard.E4.Flex", + "memory_gb": 16.0, + "network_interfaces": [ + {"name": "nic-0", "subnet": {"ref": "prod-subnet-a", "kind": "Subnet"}}, + {"name": "nic-1", "subnet": {"ref": "prod-subnet-b", "kind": "Subnet"}} + ] + }, + "observation": [ + {"path": "/shape", "coverage": "observed"}, + {"path": "/memory_gb", "coverage": "observed"}, + {"path": "/network_interfaces/{name=nic-0}/subnet", "coverage": "observed"}, + {"path": "/network_interfaces/{name=nic-1}/subnet", "coverage": "observed"}, + {"path": "/boot_volume", "coverage": "absent"} + ], + "plan": { + "scalars": [ + {"path": "/shape", "compare": "exact"}, + {"path": "/memory_gb", "compare": "numeric"}, + {"path": "/boot_volume", "compare": "exact"} + ], + "collections": [ + { + "path": "/network_interfaces", + "topology": "map", + "keys": ["name"], + "elements": [{"path": "subnet", "compare": "exact"}] + } + ] + } +} diff --git a/conformance/differential/verdict/oci_drift/expected.json b/conformance/differential/verdict/oci_drift/expected.json new file mode 100644 index 0000000..a06d063 --- /dev/null +++ b/conformance/differential/verdict/oci_drift/expected.json @@ -0,0 +1,16 @@ +{ + "verdict": { + "conformance_plan_digest": "sha256:d973c7d2e9cf02f49ad1f59889a0dd2aaabd98b41d30c07377c63fdafd387850", + "fields": { + "/memory_gb": "CONFORMANT", + "/network_interfaces/{name=nic-0}/subnet": "CONFORMANT", + "/network_interfaces/{name=nic-1}/subnet": "DRIFT", + "/shape": "CONFORMANT" + }, + "object": "DRIFT", + "observation_digest": "sha256:69ec2c422d32b89c8bd0d060296982e8172e8fd6084984aee65c8325dd47a39f", + "verdict_schema_version": 1 + }, + "canonical": "{\"conformance_plan_digest\":\"sha256:d973c7d2e9cf02f49ad1f59889a0dd2aaabd98b41d30c07377c63fdafd387850\",\"fields\":{\"/memory_gb\":\"CONFORMANT\",\"/network_interfaces/{name=nic-0}/subnet\":\"CONFORMANT\",\"/network_interfaces/{name=nic-1}/subnet\":\"DRIFT\",\"/shape\":\"CONFORMANT\"},\"object\":\"DRIFT\",\"observation_digest\":\"sha256:69ec2c422d32b89c8bd0d060296982e8172e8fd6084984aee65c8325dd47a39f\",\"verdict_schema_version\":1}", + "why": "a single element subnet mismatch drifts only that field; the artifact aggregates to DRIFT and still lists every evaluated path" +} diff --git a/conformance/differential/verdict/oci_drift/input.json b/conformance/differential/verdict/oci_drift/input.json new file mode 100644 index 0000000..c4cf285 --- /dev/null +++ b/conformance/differential/verdict/oci_drift/input.json @@ -0,0 +1,38 @@ +{ + "intent": { + "shape": "VM.Standard.E4.Flex", + "memory_gb": 16, + "network_interfaces": [ + {"name": "nic-0", "subnet": {"ref": "prod-subnet-a", "kind": "Subnet"}}, + {"name": "nic-1", "subnet": {"ref": "prod-subnet-b", "kind": "Subnet"}} + ] + }, + "observed": { + "shape": "VM.Standard.E4.Flex", + "memory_gb": 16.0, + "network_interfaces": [ + {"name": "nic-0", "subnet": {"ref": "prod-subnet-a", "kind": "Subnet"}}, + {"name": "nic-1", "subnet": {"ref": "prod-subnet-WRONG", "kind": "Subnet"}} + ] + }, + "observation": [ + {"path": "/shape", "coverage": "observed"}, + {"path": "/memory_gb", "coverage": "observed"}, + {"path": "/network_interfaces/{name=nic-0}/subnet", "coverage": "observed"}, + {"path": "/network_interfaces/{name=nic-1}/subnet", "coverage": "observed"} + ], + "plan": { + "scalars": [ + {"path": "/shape", "compare": "exact"}, + {"path": "/memory_gb", "compare": "numeric"} + ], + "collections": [ + { + "path": "/network_interfaces", + "topology": "map", + "keys": ["name"], + "elements": [{"path": "subnet", "compare": "exact"}] + } + ] + } +} diff --git a/conformance/differential/verdict/oci_not_comparable/expected.json b/conformance/differential/verdict/oci_not_comparable/expected.json new file mode 100644 index 0000000..764f00e --- /dev/null +++ b/conformance/differential/verdict/oci_not_comparable/expected.json @@ -0,0 +1,16 @@ +{ + "verdict": { + "conformance_plan_digest": "sha256:d973c7d2e9cf02f49ad1f59889a0dd2aaabd98b41d30c07377c63fdafd387850", + "fields": { + "/memory_gb": "NOT_COMPARABLE", + "/network_interfaces/{name=nic-0}/subnet": "CONFORMANT", + "/network_interfaces/{name=nic-1}/subnet": "CONFORMANT", + "/shape": "CONFORMANT" + }, + "object": "DRIFT", + "observation_digest": "sha256:10e27324b2fd39a6ed030ed9861e54e06e00634a159e16cceb346b510daf8e2b", + "verdict_schema_version": 1 + }, + "canonical": "{\"conformance_plan_digest\":\"sha256:d973c7d2e9cf02f49ad1f59889a0dd2aaabd98b41d30c07377c63fdafd387850\",\"fields\":{\"/memory_gb\":\"NOT_COMPARABLE\",\"/network_interfaces/{name=nic-0}/subnet\":\"CONFORMANT\",\"/network_interfaces/{name=nic-1}/subnet\":\"CONFORMANT\",\"/shape\":\"CONFORMANT\"},\"object\":\"DRIFT\",\"observation_digest\":\"sha256:10e27324b2fd39a6ed030ed9861e54e06e00634a159e16cceb346b510daf8e2b\",\"verdict_schema_version\":1}", + "why": "an incomparable field makes the object DRIFT; the artifact keeps NOT_COMPARABLE as the field verdict" +} diff --git a/conformance/differential/verdict/oci_not_comparable/input.json b/conformance/differential/verdict/oci_not_comparable/input.json new file mode 100644 index 0000000..e6320fc --- /dev/null +++ b/conformance/differential/verdict/oci_not_comparable/input.json @@ -0,0 +1,38 @@ +{ + "intent": { + "shape": "VM.Standard.E4.Flex", + "memory_gb": 16, + "network_interfaces": [ + {"name": "nic-0", "subnet": {"ref": "prod-subnet-a", "kind": "Subnet"}}, + {"name": "nic-1", "subnet": {"ref": "prod-subnet-b", "kind": "Subnet"}} + ] + }, + "observed": { + "shape": "VM.Standard.E4.Flex", + "memory_gb": "large", + "network_interfaces": [ + {"name": "nic-0", "subnet": {"ref": "prod-subnet-a", "kind": "Subnet"}}, + {"name": "nic-1", "subnet": {"ref": "prod-subnet-b", "kind": "Subnet"}} + ] + }, + "observation": [ + {"path": "/shape", "coverage": "observed"}, + {"path": "/memory_gb", "coverage": "observed"}, + {"path": "/network_interfaces/{name=nic-0}/subnet", "coverage": "observed"}, + {"path": "/network_interfaces/{name=nic-1}/subnet", "coverage": "observed"} + ], + "plan": { + "scalars": [ + {"path": "/shape", "compare": "exact"}, + {"path": "/memory_gb", "compare": "numeric"} + ], + "collections": [ + { + "path": "/network_interfaces", + "topology": "map", + "keys": ["name"], + "elements": [{"path": "subnet", "compare": "exact"}] + } + ] + } +} diff --git a/conformance/differential/verdict/oci_unobserved/expected.json b/conformance/differential/verdict/oci_unobserved/expected.json new file mode 100644 index 0000000..ab46a9e --- /dev/null +++ b/conformance/differential/verdict/oci_unobserved/expected.json @@ -0,0 +1,16 @@ +{ + "verdict": { + "conformance_plan_digest": "sha256:d973c7d2e9cf02f49ad1f59889a0dd2aaabd98b41d30c07377c63fdafd387850", + "fields": { + "/memory_gb": "UNOBSERVED", + "/network_interfaces/{name=nic-0}/subnet": "CONFORMANT", + "/network_interfaces/{name=nic-1}/subnet": "CONFORMANT", + "/shape": "CONFORMANT" + }, + "object": "INCOMPLETE", + "observation_digest": "sha256:7afeba3ff68ab2ce4f789c4dae85727371a75120e9f458a4c3b8c9c39ee692e3", + "verdict_schema_version": 1 + }, + "canonical": "{\"conformance_plan_digest\":\"sha256:d973c7d2e9cf02f49ad1f59889a0dd2aaabd98b41d30c07377c63fdafd387850\",\"fields\":{\"/memory_gb\":\"UNOBSERVED\",\"/network_interfaces/{name=nic-0}/subnet\":\"CONFORMANT\",\"/network_interfaces/{name=nic-1}/subnet\":\"CONFORMANT\",\"/shape\":\"CONFORMANT\"},\"object\":\"INCOMPLETE\",\"observation_digest\":\"sha256:7afeba3ff68ab2ce4f789c4dae85727371a75120e9f458a4c3b8c9c39ee692e3\",\"verdict_schema_version\":1}", + "why": "a planned field that was not observed makes the object INCOMPLETE; the observation digest changes with the coverage" +} diff --git a/conformance/differential/verdict/oci_unobserved/input.json b/conformance/differential/verdict/oci_unobserved/input.json new file mode 100644 index 0000000..70cd65c --- /dev/null +++ b/conformance/differential/verdict/oci_unobserved/input.json @@ -0,0 +1,37 @@ +{ + "intent": { + "shape": "VM.Standard.E4.Flex", + "memory_gb": 16, + "network_interfaces": [ + {"name": "nic-0", "subnet": {"ref": "prod-subnet-a", "kind": "Subnet"}}, + {"name": "nic-1", "subnet": {"ref": "prod-subnet-b", "kind": "Subnet"}} + ] + }, + "observed": { + "shape": "VM.Standard.E4.Flex", + "memory_gb": 16.0, + "network_interfaces": [ + {"name": "nic-0", "subnet": {"ref": "prod-subnet-a", "kind": "Subnet"}}, + {"name": "nic-1", "subnet": {"ref": "prod-subnet-b", "kind": "Subnet"}} + ] + }, + "observation": [ + {"path": "/shape", "coverage": "observed"}, + {"path": "/network_interfaces/{name=nic-0}/subnet", "coverage": "observed"}, + {"path": "/network_interfaces/{name=nic-1}/subnet", "coverage": "observed"} + ], + "plan": { + "scalars": [ + {"path": "/shape", "compare": "exact"}, + {"path": "/memory_gb", "compare": "numeric"} + ], + "collections": [ + { + "path": "/network_interfaces", + "topology": "map", + "keys": ["name"], + "elements": [{"path": "subnet", "compare": "exact"}] + } + ] + } +} diff --git a/conformance/differential/verdict/tampered_evidence_for_unclassified_path/expected.json b/conformance/differential/verdict/tampered_evidence_for_unclassified_path/expected.json new file mode 100644 index 0000000..3e20df4 --- /dev/null +++ b/conformance/differential/verdict/tampered_evidence_for_unclassified_path/expected.json @@ -0,0 +1,7 @@ +{ + "error": { + "code": "E_INCONSISTENT_VERDICT", + "path": "/ghost" + }, + "why": "a result that cannot be an evaluate result for the plan is refused, never digested into an artifact" +} diff --git a/conformance/differential/verdict/tampered_evidence_for_unclassified_path/input.json b/conformance/differential/verdict/tampered_evidence_for_unclassified_path/input.json new file mode 100644 index 0000000..c2580b9 --- /dev/null +++ b/conformance/differential/verdict/tampered_evidence_for_unclassified_path/input.json @@ -0,0 +1,93 @@ +{ + "intent": { + "shape": "VM.Standard.E4.Flex", + "memory_gb": 16, + "network_interfaces": [ + { + "name": "nic-0", + "subnet": { + "ref": "prod-subnet-a", + "kind": "Subnet" + } + }, + { + "name": "nic-1", + "subnet": { + "ref": "prod-subnet-b", + "kind": "Subnet" + } + } + ] + }, + "observed": { + "shape": "VM.Standard.E4.Flex", + "memory_gb": 16.0, + "network_interfaces": [ + { + "name": "nic-0", + "subnet": { + "ref": "prod-subnet-a", + "kind": "Subnet" + } + }, + { + "name": "nic-1", + "subnet": { + "ref": "prod-subnet-WRONG", + "kind": "Subnet" + } + } + ] + }, + "observation": [ + { + "path": "/shape", + "coverage": "observed" + }, + { + "path": "/memory_gb", + "coverage": "observed" + }, + { + "path": "/network_interfaces/{name=nic-0}/subnet", + "coverage": "observed" + }, + { + "path": "/network_interfaces/{name=nic-1}/subnet", + "coverage": "observed" + } + ], + "plan": { + "scalars": [ + { + "path": "/shape", + "compare": "exact" + }, + { + "path": "/memory_gb", + "compare": "numeric" + } + ], + "collections": [ + { + "path": "/network_interfaces", + "topology": "map", + "keys": [ + "name" + ], + "elements": [ + { + "path": "subnet", + "compare": "exact" + } + ] + } + ] + }, + "tamper": [ + { + "op": "add_consumed", + "path": "/ghost" + } + ] +} diff --git a/conformance/differential/verdict/tampered_missing_consumed_evidence/expected.json b/conformance/differential/verdict/tampered_missing_consumed_evidence/expected.json new file mode 100644 index 0000000..baefa7b --- /dev/null +++ b/conformance/differential/verdict/tampered_missing_consumed_evidence/expected.json @@ -0,0 +1,7 @@ +{ + "error": { + "code": "E_INCONSISTENT_VERDICT", + "path": "/shape" + }, + "why": "a result that cannot be an evaluate result for the plan is refused, never digested into an artifact" +} diff --git a/conformance/differential/verdict/tampered_missing_consumed_evidence/input.json b/conformance/differential/verdict/tampered_missing_consumed_evidence/input.json new file mode 100644 index 0000000..61685ed --- /dev/null +++ b/conformance/differential/verdict/tampered_missing_consumed_evidence/input.json @@ -0,0 +1,93 @@ +{ + "intent": { + "shape": "VM.Standard.E4.Flex", + "memory_gb": 16, + "network_interfaces": [ + { + "name": "nic-0", + "subnet": { + "ref": "prod-subnet-a", + "kind": "Subnet" + } + }, + { + "name": "nic-1", + "subnet": { + "ref": "prod-subnet-b", + "kind": "Subnet" + } + } + ] + }, + "observed": { + "shape": "VM.Standard.E4.Flex", + "memory_gb": 16.0, + "network_interfaces": [ + { + "name": "nic-0", + "subnet": { + "ref": "prod-subnet-a", + "kind": "Subnet" + } + }, + { + "name": "nic-1", + "subnet": { + "ref": "prod-subnet-WRONG", + "kind": "Subnet" + } + } + ] + }, + "observation": [ + { + "path": "/shape", + "coverage": "observed" + }, + { + "path": "/memory_gb", + "coverage": "observed" + }, + { + "path": "/network_interfaces/{name=nic-0}/subnet", + "coverage": "observed" + }, + { + "path": "/network_interfaces/{name=nic-1}/subnet", + "coverage": "observed" + } + ], + "plan": { + "scalars": [ + { + "path": "/shape", + "compare": "exact" + }, + { + "path": "/memory_gb", + "compare": "numeric" + } + ], + "collections": [ + { + "path": "/network_interfaces", + "topology": "map", + "keys": [ + "name" + ], + "elements": [ + { + "path": "subnet", + "compare": "exact" + } + ] + } + ] + }, + "tamper": [ + { + "op": "drop_consumed", + "path": "/shape" + } + ] +} diff --git a/conformance/differential/verdict/tampered_object_not_following_fields/expected.json b/conformance/differential/verdict/tampered_object_not_following_fields/expected.json new file mode 100644 index 0000000..734e697 --- /dev/null +++ b/conformance/differential/verdict/tampered_object_not_following_fields/expected.json @@ -0,0 +1,7 @@ +{ + "error": { + "code": "E_INCONSISTENT_VERDICT", + "path": "$" + }, + "why": "a result that cannot be an evaluate result for the plan is refused, never digested into an artifact" +} diff --git a/conformance/differential/verdict/tampered_object_not_following_fields/input.json b/conformance/differential/verdict/tampered_object_not_following_fields/input.json new file mode 100644 index 0000000..41e4842 --- /dev/null +++ b/conformance/differential/verdict/tampered_object_not_following_fields/input.json @@ -0,0 +1,93 @@ +{ + "intent": { + "shape": "VM.Standard.E4.Flex", + "memory_gb": 16, + "network_interfaces": [ + { + "name": "nic-0", + "subnet": { + "ref": "prod-subnet-a", + "kind": "Subnet" + } + }, + { + "name": "nic-1", + "subnet": { + "ref": "prod-subnet-b", + "kind": "Subnet" + } + } + ] + }, + "observed": { + "shape": "VM.Standard.E4.Flex", + "memory_gb": 16.0, + "network_interfaces": [ + { + "name": "nic-0", + "subnet": { + "ref": "prod-subnet-a", + "kind": "Subnet" + } + }, + { + "name": "nic-1", + "subnet": { + "ref": "prod-subnet-WRONG", + "kind": "Subnet" + } + } + ] + }, + "observation": [ + { + "path": "/shape", + "coverage": "observed" + }, + { + "path": "/memory_gb", + "coverage": "observed" + }, + { + "path": "/network_interfaces/{name=nic-0}/subnet", + "coverage": "observed" + }, + { + "path": "/network_interfaces/{name=nic-1}/subnet", + "coverage": "observed" + } + ], + "plan": { + "scalars": [ + { + "path": "/shape", + "compare": "exact" + }, + { + "path": "/memory_gb", + "compare": "numeric" + } + ], + "collections": [ + { + "path": "/network_interfaces", + "topology": "map", + "keys": [ + "name" + ], + "elements": [ + { + "path": "subnet", + "compare": "exact" + } + ] + } + ] + }, + "tamper": [ + { + "op": "set_object", + "value": "CONFORMANT" + } + ] +} diff --git a/conformance/differential/verdict/tampered_planned_scalar_missing/expected.json b/conformance/differential/verdict/tampered_planned_scalar_missing/expected.json new file mode 100644 index 0000000..e7d36fd --- /dev/null +++ b/conformance/differential/verdict/tampered_planned_scalar_missing/expected.json @@ -0,0 +1,7 @@ +{ + "error": { + "code": "E_INCONSISTENT_VERDICT", + "path": "/memory_gb" + }, + "why": "a result that cannot be an evaluate result for the plan is refused, never digested into an artifact" +} diff --git a/conformance/differential/verdict/tampered_planned_scalar_missing/input.json b/conformance/differential/verdict/tampered_planned_scalar_missing/input.json new file mode 100644 index 0000000..5e50f71 --- /dev/null +++ b/conformance/differential/verdict/tampered_planned_scalar_missing/input.json @@ -0,0 +1,97 @@ +{ + "intent": { + "shape": "VM.Standard.E4.Flex", + "memory_gb": 16, + "network_interfaces": [ + { + "name": "nic-0", + "subnet": { + "ref": "prod-subnet-a", + "kind": "Subnet" + } + }, + { + "name": "nic-1", + "subnet": { + "ref": "prod-subnet-b", + "kind": "Subnet" + } + } + ] + }, + "observed": { + "shape": "VM.Standard.E4.Flex", + "memory_gb": 16.0, + "network_interfaces": [ + { + "name": "nic-0", + "subnet": { + "ref": "prod-subnet-a", + "kind": "Subnet" + } + }, + { + "name": "nic-1", + "subnet": { + "ref": "prod-subnet-WRONG", + "kind": "Subnet" + } + } + ] + }, + "observation": [ + { + "path": "/shape", + "coverage": "observed" + }, + { + "path": "/memory_gb", + "coverage": "observed" + }, + { + "path": "/network_interfaces/{name=nic-0}/subnet", + "coverage": "observed" + }, + { + "path": "/network_interfaces/{name=nic-1}/subnet", + "coverage": "observed" + } + ], + "plan": { + "scalars": [ + { + "path": "/shape", + "compare": "exact" + }, + { + "path": "/memory_gb", + "compare": "numeric" + } + ], + "collections": [ + { + "path": "/network_interfaces", + "topology": "map", + "keys": [ + "name" + ], + "elements": [ + { + "path": "subnet", + "compare": "exact" + } + ] + } + ] + }, + "tamper": [ + { + "op": "drop_field", + "path": "/memory_gb" + }, + { + "op": "drop_consumed", + "path": "/memory_gb" + } + ] +} diff --git a/conformance/differential/verdict/unknown_coverage_flows_through/expected.json b/conformance/differential/verdict/unknown_coverage_flows_through/expected.json new file mode 100644 index 0000000..30219b6 --- /dev/null +++ b/conformance/differential/verdict/unknown_coverage_flows_through/expected.json @@ -0,0 +1,16 @@ +{ + "verdict": { + "conformance_plan_digest": "sha256:d973c7d2e9cf02f49ad1f59889a0dd2aaabd98b41d30c07377c63fdafd387850", + "fields": { + "/memory_gb": "UNOBSERVED", + "/network_interfaces/{name=nic-0}/subnet": "CONFORMANT", + "/network_interfaces/{name=nic-1}/subnet": "CONFORMANT", + "/shape": "CONFORMANT" + }, + "object": "INCOMPLETE", + "observation_digest": "sha256:367cbe5c5c79950f2bed53eafebd9ac1a07dba45df052f71f5d3af4c5ef95fdd", + "verdict_schema_version": 1 + }, + "canonical": "{\"conformance_plan_digest\":\"sha256:d973c7d2e9cf02f49ad1f59889a0dd2aaabd98b41d30c07377c63fdafd387850\",\"fields\":{\"/memory_gb\":\"UNOBSERVED\",\"/network_interfaces/{name=nic-0}/subnet\":\"CONFORMANT\",\"/network_interfaces/{name=nic-1}/subnet\":\"CONFORMANT\",\"/shape\":\"CONFORMANT\"},\"object\":\"INCOMPLETE\",\"observation_digest\":\"sha256:367cbe5c5c79950f2bed53eafebd9ac1a07dba45df052f71f5d3af4c5ef95fdd\",\"verdict_schema_version\":1}", + "why": "unknown coverage classifies like unobserved; its observation digest differs from the unobserved vector with the same fields" +} diff --git a/conformance/differential/verdict/unknown_coverage_flows_through/input.json b/conformance/differential/verdict/unknown_coverage_flows_through/input.json new file mode 100644 index 0000000..6ef34be --- /dev/null +++ b/conformance/differential/verdict/unknown_coverage_flows_through/input.json @@ -0,0 +1,38 @@ +{ + "intent": { + "shape": "VM.Standard.E4.Flex", + "memory_gb": 16, + "network_interfaces": [ + {"name": "nic-0", "subnet": {"ref": "prod-subnet-a", "kind": "Subnet"}}, + {"name": "nic-1", "subnet": {"ref": "prod-subnet-b", "kind": "Subnet"}} + ] + }, + "observed": { + "shape": "VM.Standard.E4.Flex", + "memory_gb": 16.0, + "network_interfaces": [ + {"name": "nic-0", "subnet": {"ref": "prod-subnet-a", "kind": "Subnet"}}, + {"name": "nic-1", "subnet": {"ref": "prod-subnet-b", "kind": "Subnet"}} + ] + }, + "observation": [ + {"path": "/shape", "coverage": "observed"}, + {"path": "/memory_gb", "coverage": "unknown"}, + {"path": "/network_interfaces/{name=nic-0}/subnet", "coverage": "observed"}, + {"path": "/network_interfaces/{name=nic-1}/subnet", "coverage": "observed"} + ], + "plan": { + "scalars": [ + {"path": "/shape", "compare": "exact"}, + {"path": "/memory_gb", "compare": "numeric"} + ], + "collections": [ + { + "path": "/network_interfaces", + "topology": "map", + "keys": ["name"], + "elements": [{"path": "subnet", "compare": "exact"}] + } + ] + } +} diff --git a/docs/VERDICT-SCHEMA.md b/docs/VERDICT-SCHEMA.md index a267e13..fcf86a4 100644 --- a/docs/VERDICT-SCHEMA.md +++ b/docs/VERDICT-SCHEMA.md @@ -200,3 +200,35 @@ document's full contract into one artifact** — nothing anywhere in together into the single required shape above. Go/Rust computation parity and digest parity are real and cross-checked; a first-class verdict artifact builder is not built yet, in either language. + +## Status update, 2026-10-18: the first-class artifact builder exists in both languages + +The gap the previous section names is closed. `engine/src/verdict.rs` +(`build_verdict`, `verdict_value`, `verdict_canonical_bytes`) and its Go +peer `go/verdict` (`Build`, `Value`, `CanonicalBytes`) assemble this +document's full contract into one artifact: all five required fields, with +`verdict_schema_version: 1`. + +* `object` and `fields` are `evaluate`'s/`Evaluate`'s own result, carried over + verbatim; `fields` therefore holds exactly the paths that were evaluated. +* `conformance_plan_digest` and `observation_digest` are the existing normative + projections (`conformance_plan_digest` / `observation_digest`, and their Go + peers), called, not recomputed. A test pins this in each language, and the + `multi_key_collection_identity` vector's plan digest is byte-for-byte the + one already pinned by `conformance/differential/digest/plan_digest_with_collection`. +* Before an artifact exists the builder refuses (`E_INCONSISTENT_VERDICT`, + `R-VERDICT-CONSISTENT`) an evaluate result that cannot be one: classified + paths and consumed evidence that do not match one to one, a planned scalar + missing from `fields`, or an `object` that does not follow from `fields`. + A real `evaluate` result always passes; this guards the "every field is + REQUIRED and complete" guarantee against hand-assembled or edited results. +* Canonical bytes are section A's canonicalization of the five-key tree, and + `conformance/differential/verdict/` pins them. Rust and Go read the same files + and must produce the same bytes. A second test in each language applies eight + deliberate divergences (version bump, changed object, changed/dropped/added + field, altered or swapped digests) and requires the comparison to catch each. + +Still not decided, unchanged: signing the artifact, and an intent-digest +field. This builder adds neither. The Relay's enforcement boundary and the +materialization receipt are untouched. + diff --git a/engine/src/error.rs b/engine/src/error.rs index 2763e8d..f378a67 100644 --- a/engine/src/error.rs +++ b/engine/src/error.rs @@ -133,6 +133,13 @@ pub mod code { /// shape. Covered by unit tests in `engine/tests/hand_built_values.rs` /// (not a YAML vector -- the reader cannot produce them). pub const MALFORMED_VALUE: &str = "E_MALFORMED_VALUE"; + + /// A conformance/drift verdict was asked of an `ObjectVerdict` that cannot + /// be an `evaluate` result for the plan: its classified paths and its + /// consumed observation evidence disagree, a planned scalar is missing, or + /// `object` does not follow from `fields` (`R-VERDICT-CONSISTENT`, + /// `docs/VERDICT-SCHEMA.md`). Raised by `verdict::build_verdict`. + pub const INCONSISTENT_VERDICT: &str = "E_INCONSISTENT_VERDICT"; } /// The single error type this crate raises. diff --git a/engine/src/lib.rs b/engine/src/lib.rs index 570e838..8d73477 100644 --- a/engine/src/lib.rs +++ b/engine/src/lib.rs @@ -226,6 +226,7 @@ pub mod error; pub mod materialized; pub mod plan; pub mod reader; +pub mod verdict; pub use canonical::{digest, to_canonical_json}; pub use collection::{Collection, CollectionTopology}; @@ -244,3 +245,7 @@ pub use plan::{ evaluate, CollectionPlan, ConformancePlan, ConsumedField, FieldPlan, ObjectConformance, ObjectVerdict, }; +pub use verdict::{ + build_verdict, verdict_canonical_bytes, verdict_value, ConformanceVerdict, + VERDICT_SCHEMA_VERSION, +}; diff --git a/engine/src/verdict.rs b/engine/src/verdict.rs new file mode 100644 index 0000000..5b6c8e6 --- /dev/null +++ b/engine/src/verdict.rs @@ -0,0 +1,353 @@ +//! The conformance/drift verdict artifact, assembled into the single shape +//! `docs/VERDICT-SCHEMA.md` commits to. +//! +//! `plan::evaluate` produces the semantic core (`object`, `fields`, and the +//! `consumed` map it recorded while classifying) and `digest_projection` +//! produces the two digests from a plan and from that `consumed` map. Nothing +//! bundled those into the one required five-field artifact until this module. +//! +//! This module decides nothing new: +//! +//! * `object` and `fields` are `evaluate`'s own result, carried over verbatim; +//! * `conformance_plan_digest` is [`conformance_plan_digest`] of the plan that +//! was executed and `observation_digest` is [`observation_digest`] of the +//! `consumed` map `evaluate` recorded -- the existing normative projections, +//! called, never recomputed or re-derived here; +//! * `verdict_schema_version` is the bare integer `1` (`VERDICT-SCHEMA.md`). +//! +//! # What is checked before an artifact exists +//! +//! The schema says `fields` holds exactly one entry per path `evaluate` +//! classified, and that every field is required. [`build_verdict`] therefore +//! refuses an [`ObjectVerdict`] that cannot be such a result, rather than +//! digesting it into an artifact that would look authoritative: +//! +//! * the `fields` and `consumed` path sets must be identical (every classified +//! path has its observation evidence, and no evidence is left over); +//! * every `plan.scalars` path must be in `fields`; +//! * `object` must be what [`aggregate`] gives for `fields`. +//! +//! A real `evaluate` result always passes; a result that was edited or +//! assembled by hand does not. Scope stays with the contract: signing the +//! artifact and an intent digest are explicitly "not decided" in the schema and +//! are not added here. + +use std::collections::BTreeMap; + +use crate::canonical::to_canonical_json; +use crate::conformance::FieldVerdict; +use crate::digest_projection::{conformance_plan_digest, observation_digest}; +use crate::error::{code, Error, Result, Stage}; +use crate::plan::{aggregate, ConformancePlan, ObjectConformance, ObjectVerdict}; +use crate::reader::{Map, Value}; + +/// The only `verdict_schema_version` this module produces. Incremented on any +/// backward-incompatible change to the verdict contract (`VERDICT-SCHEMA.md`). +pub const VERDICT_SCHEMA_VERSION: i64 = 1; + +/// The five required fields, in the order the schema lists them. There is no +/// optional member and no way to leave one out. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ConformanceVerdict { + pub verdict_schema_version: i64, + pub object: ObjectConformance, + pub fields: BTreeMap, + pub conformance_plan_digest: String, + pub observation_digest: String, +} + +/// Assembles the verdict artifact for `result`, the output of +/// [`crate::evaluate`] run against `plan`. +/// +/// # Errors +/// `E_INCONSISTENT_VERDICT` (`R-VERDICT-CONSISTENT`) if `result` cannot be an +/// `evaluate` result for `plan` (see the module doc); a canonicalization error +/// from the digest functions if a consumed value holds a non-finite float. +pub fn build_verdict(plan: &ConformancePlan, result: &ObjectVerdict) -> Result { + check_consistent(plan, result)?; + Ok(ConformanceVerdict { + verdict_schema_version: VERDICT_SCHEMA_VERSION, + object: result.object, + fields: result.fields.clone(), + conformance_plan_digest: conformance_plan_digest(plan)?, + observation_digest: observation_digest(&result.consumed)?, + }) +} + +fn inconsistent(path: &str, detail: String) -> Error { + Error::new( + code::INCONSISTENT_VERDICT, + "R-VERDICT-CONSISTENT", + Stage::Validate, + path, + detail, + ) +} + +fn check_consistent(plan: &ConformancePlan, result: &ObjectVerdict) -> Result<()> { + for path in result.fields.keys() { + if !result.consumed.contains_key(path) { + return Err(inconsistent( + path, + "classified path has no consumed observation evidence".to_string(), + )); + } + } + for path in result.consumed.keys() { + if !result.fields.contains_key(path) { + return Err(inconsistent( + path, + "consumed observation evidence for a path that was not classified".to_string(), + )); + } + } + for fp in &plan.scalars { + if !result.fields.contains_key(&fp.path) { + return Err(inconsistent( + &fp.path, + "planned scalar path is missing from fields".to_string(), + )); + } + } + let expected = aggregate(&result.fields); + if expected != result.object { + return Err(inconsistent( + "$", + format!( + "object {} does not follow from fields (expected {})", + result.object.as_str(), + expected.as_str() + ), + )); + } + Ok(()) +} + +/// The artifact as a value tree: exactly the five required keys. +#[must_use] +pub fn verdict_value(v: &ConformanceVerdict) -> Value { + let mut fields = Map::default(); + for (path, fv) in &v.fields { + fields.push(path.clone(), Value::Str(fv.as_str().to_string())); + } + let mut m = Map::default(); + m.push( + "verdict_schema_version", + Value::Int(v.verdict_schema_version), + ); + m.push("object", Value::Str(v.object.as_str().to_string())); + m.push("fields", Value::Map(fields)); + m.push( + "conformance_plan_digest", + Value::Str(v.conformance_plan_digest.clone()), + ); + m.push( + "observation_digest", + Value::Str(v.observation_digest.clone()), + ); + Value::Map(m) +} + +/// The artifact's canonical bytes (section A): the value tree of +/// [`verdict_value`] through [`to_canonical_json`]. +/// +/// # Errors +/// Never in practice (the tree holds only strings and one integer); the +/// signature stays fallible because canonicalization is. +pub fn verdict_canonical_bytes(v: &ConformanceVerdict) -> Result> { + to_canonical_json(&verdict_value(v)) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::collection::{Collection, CollectionTopology}; + use crate::conformance::{CompareType, Coverage, Observation}; + use crate::materialized::{ + Capability, FieldEvidence, IntentEvidence, MaterializedField, MaterializedObject, + }; + use crate::plan::{evaluate, ConsumedField, FieldPlan}; + use std::collections::BTreeSet; + + fn object(entries: &[(&str, FieldEvidence)]) -> MaterializedObject { + let mut fields = BTreeMap::new(); + let mut keys = BTreeSet::new(); + for (k, ev) in entries { + keys.insert((*k).to_string()); + fields.insert( + (*k).to_string(), + MaterializedField { + capability: Capability::Implemented, + evidence: ev.clone(), + }, + ); + } + MaterializedObject::try_new(fields, &keys).expect("fixture key set is complete") + } + + fn authored(v: Value) -> FieldEvidence { + FieldEvidence::Intent(IntentEvidence::Authored(Some(v))) + } + + fn observed(v: Value) -> FieldEvidence { + FieldEvidence::Observation(ConsumedField::Observed(v)) + } + + fn sample() -> (ConformancePlan, ObjectVerdict) { + let plan = ConformancePlan { + scalars: vec![ + FieldPlan { + path: "/shape".to_string(), + compare: CompareType::Exact, + }, + FieldPlan { + path: "/memory_gb".to_string(), + compare: CompareType::Numeric, + }, + ], + collections: vec![], + }; + let intent = object(&[ + ("shape", authored(Value::Str("E4".into()))), + ("memory_gb", authored(Value::Int(16))), + ]); + let observed_obj = object(&[ + ("shape", observed(Value::Str("E4".into()))), + ("memory_gb", observed(Value::Float(32.0))), + ]); + let mut obs = Observation::new(); + obs.set("/shape", Coverage::Observed); + obs.set("/memory_gb", Coverage::Observed); + let result = evaluate(&intent, &observed_obj, &obs, &plan); + (plan, result) + } + + #[test] + fn a_real_evaluate_result_builds_all_five_fields() { + let (plan, result) = sample(); + let v = build_verdict(&plan, &result).expect("consistent"); + assert_eq!(v.verdict_schema_version, 1); + assert_eq!(v.object, ObjectConformance::Drift); + assert_eq!(v.fields.len(), 2); + assert!(v.conformance_plan_digest.starts_with("sha256:")); + assert!(v.observation_digest.starts_with("sha256:")); + } + + #[test] + fn the_digests_are_the_existing_projections_not_a_new_computation() { + let (plan, result) = sample(); + let v = build_verdict(&plan, &result).unwrap(); + assert_eq!( + v.conformance_plan_digest, + conformance_plan_digest(&plan).unwrap() + ); + assert_eq!( + v.observation_digest, + observation_digest(&result.consumed).unwrap() + ); + } + + #[test] + fn fields_hold_exactly_the_evaluated_paths() { + let (plan, result) = sample(); + let v = build_verdict(&plan, &result).unwrap(); + let keys: Vec<&String> = v.fields.keys().collect(); + let evaluated: Vec<&String> = result.fields.keys().collect(); + assert_eq!(keys, evaluated); + } + + #[test] + fn the_value_tree_has_exactly_the_five_required_keys() { + let (plan, result) = sample(); + let v = build_verdict(&plan, &result).unwrap(); + let Value::Map(m) = verdict_value(&v) else { + panic!("a verdict is a mapping") + }; + let mut keys: Vec<&str> = m.keys(); + keys.sort_unstable(); + assert_eq!( + keys, + [ + "conformance_plan_digest", + "fields", + "object", + "observation_digest", + "verdict_schema_version" + ] + ); + assert_eq!(m.get("verdict_schema_version"), Some(&Value::Int(1))); + } + + #[test] + fn canonical_bytes_are_sorted_and_stable() { + let (plan, result) = sample(); + let v = build_verdict(&plan, &result).unwrap(); + let a = verdict_canonical_bytes(&v).unwrap(); + let b = verdict_canonical_bytes(&v).unwrap(); + assert_eq!(a, b); + let text = String::from_utf8(a).unwrap(); + assert!(text.starts_with("{\"conformance_plan_digest\":\"sha256:")); + assert!(text.ends_with("\"verdict_schema_version\":1}")); + } + + #[test] + fn a_classified_path_without_evidence_is_refused() { + let (plan, mut result) = sample(); + result.consumed.remove("/shape"); + let e = build_verdict(&plan, &result).unwrap_err(); + assert_eq!(e.code, code::INCONSISTENT_VERDICT); + assert_eq!(e.path, "/shape"); + } + + #[test] + fn evidence_for_an_unclassified_path_is_refused() { + let (plan, mut result) = sample(); + result + .consumed + .insert("/ghost".to_string(), ConsumedField::Absent); + let e = build_verdict(&plan, &result).unwrap_err(); + assert_eq!(e.code, code::INCONSISTENT_VERDICT); + assert_eq!(e.path, "/ghost"); + } + + #[test] + fn a_planned_scalar_missing_from_fields_is_refused() { + let (plan, mut result) = sample(); + result.fields.remove("/memory_gb"); + result.consumed.remove("/memory_gb"); + let e = build_verdict(&plan, &result).unwrap_err(); + assert_eq!(e.code, code::INCONSISTENT_VERDICT); + assert_eq!(e.path, "/memory_gb"); + } + + #[test] + fn an_object_verdict_that_does_not_follow_from_fields_is_refused() { + let (plan, mut result) = sample(); + result.object = ObjectConformance::Conformant; + let e = build_verdict(&plan, &result).unwrap_err(); + assert_eq!(e.code, code::INCONSISTENT_VERDICT); + assert_eq!(e.path, "$"); + } + + #[test] + fn a_collection_plan_does_not_change_the_checked_invariants() { + let plan = ConformancePlan { + scalars: vec![], + collections: vec![crate::plan::CollectionPlan { + path: "/nics".to_string(), + collection: Collection { + topology: CollectionTopology::Map, + keys: vec!["name".to_string()], + }, + elements: vec![], + }], + }; + let empty = ObjectVerdict { + object: ObjectConformance::Conformant, + fields: BTreeMap::new(), + consumed: BTreeMap::new(), + }; + // No element field was classified, so there is nothing to disagree about. + assert!(build_verdict(&plan, &empty).is_ok()); + } +} diff --git a/engine/tests/differential.rs b/engine/tests/differential.rs index 3c178a6..77b228a 100644 --- a/engine/tests/differential.rs +++ b/engine/tests/differential.rs @@ -9,10 +9,12 @@ use std::fs; use std::path::{Path, PathBuf}; use cic_primitive_engine::{ - classify_field_value, conformance_plan_digest, evaluate, materialized_object_digest, - observation_digest, reader, Capability, Collection, CollectionPlan, CollectionTopology, - CompareType, ConformancePlan, ConsumedField, Coverage, FieldEvidence, FieldPlan, - IntentEvidence, MaterializedField, MaterializedObject, Observation, Stage, + build_verdict, classify_field_value, conformance_plan_digest, evaluate, + materialized_object_digest, observation_digest, reader, to_canonical_json, + verdict_canonical_bytes, Capability, Collection, CollectionPlan, CollectionTopology, + CompareType, ConformancePlan, ConformanceVerdict, ConsumedField, Coverage, FieldEvidence, + FieldPlan, FieldVerdict, IntentEvidence, MaterializedField, MaterializedObject, + ObjectConformance, ObjectVerdict, Observation, Stage, }; use reader::Value; use std::collections::{BTreeMap, BTreeSet}; @@ -610,3 +612,252 @@ fn materialized_vectors() { failures.join("\n ") ); } + +// --------------------------------------------------------------------------- +// conformance/differential/verdict/ -- the assembled verdict artifact +// (`docs/VERDICT-SCHEMA.md`). Every vector runs `evaluate` on the same input +// the `plan` group uses, builds the five-field artifact, and compares its +// canonical bytes with the pinned ones. The Go peer (`go/verdict`) reads the +// identical files and must produce the identical bytes. +// --------------------------------------------------------------------------- + +fn parse_object(s: &str) -> ObjectConformance { + match s { + "CONFORMANT" => ObjectConformance::Conformant, + "DRIFT" => ObjectConformance::Drift, + "INCOMPLETE" => ObjectConformance::Incomplete, + other => panic!("unknown object verdict {other:?}"), + } +} + +/// A vector's optional `tamper` list edits the `evaluate` result before the +/// artifact is built, to reach states a real `evaluate` never produces. +fn apply_tamper(result: &mut ObjectVerdict, input: &Value) { + let Some(Value::Seq(ops)) = map_get(input, "tamper") else { + return; + }; + for op in ops { + match str_field(op, "op") { + "drop_consumed" => { + result.consumed.remove(str_field(op, "path")); + } + "add_consumed" => { + result + .consumed + .insert(str_field(op, "path").to_string(), ConsumedField::Absent); + } + "drop_field" => { + result.fields.remove(str_field(op, "path")); + } + "set_object" => result.object = parse_object(str_field(op, "value")), + other => panic!("unknown tamper op {other:?}"), + } + } +} + +fn evaluate_vector(v: &Vector) -> (ConformancePlan, ObjectVerdict) { + let intent = map_get(&v.input, "intent").unwrap_or(&Value::Null); + let observed = map_get(&v.input, "observed").unwrap_or(&Value::Null); + let obs = build_observation(&v.input); + let plan = build_plan(&v.input); + let mut result = evaluate( + &intent_object(intent), + &observed_object(observed, &obs), + &obs, + &plan, + ); + apply_tamper(&mut result, &v.input); + (plan, result) +} + +/// Empty means the artifact matches the pinned expectation exactly: the +/// canonical bytes equal the pinned string, and the pinned `verdict` tree +/// canonicalizes to those same bytes. +fn verdict_mismatches(got: &ConformanceVerdict, expected: &Value) -> Vec { + let mut out = Vec::new(); + let bytes = verdict_canonical_bytes(got).expect("a verdict canonicalizes"); + let want = str_field(expected, "canonical"); + if bytes != want.as_bytes() { + out.push(format!( + "canonical bytes differ\n got: {}\n want: {want}", + String::from_utf8_lossy(&bytes) + )); + } + let tree = map_get(expected, "verdict").expect("expected holds a verdict tree"); + let tree_bytes = to_canonical_json(tree).expect("pinned tree canonicalizes"); + if tree_bytes != bytes { + out.push( + "pinned verdict tree and pinned canonical string disagree with the artifact" + .to_string(), + ); + } + out +} + +#[test] +fn verdict_vectors() { + let vectors = load_vectors("verdict"); + assert!( + !vectors.is_empty(), + "differential/verdict has no vectors -- an empty corpus trivially passes" + ); + let plan_vectors = load_vectors("plan"); + + let mut failures = Vec::new(); + let mut successes = 0; + let mut refusals = 0; + for v in &vectors { + let (plan, result) = evaluate_vector(v); + let built = build_verdict(&plan, &result); + + if let Some(Value::Map(_)) = map_get(&v.expected, "error") { + refusals += 1; + let want = map_get(&v.expected, "error").expect("checked"); + match built { + Ok(_) => failures.push(format!("{}: expected a refusal, got an artifact", v.name)), + Err(e) => { + if e.code != str_field(want, "code") || e.path != str_field(want, "path") { + failures.push(format!( + "{}: refusal got {} at {}, want {} at {}", + v.name, + e.code, + e.path, + str_field(want, "code"), + str_field(want, "path") + )); + } + } + } + continue; + } + + successes += 1; + let got = match built { + Ok(g) => g, + Err(e) => { + failures.push(format!("{}: unexpected refusal: {e}", v.name)); + continue; + } + }; + for m in verdict_mismatches(&got, &v.expected) { + failures.push(format!("{}: {m}", v.name)); + } + // The artifact's `object`/`fields` are the plan group's own pinned + // expectation for the same input, not a second opinion. + if let Some(pv) = plan_vectors.iter().find(|p| p.name == v.name) { + let got_fields: BTreeMap = got + .fields + .iter() + .map(|(k, fv)| (k.clone(), fv.as_str().to_string())) + .collect(); + if got.object.as_str() != str_field(&pv.expected, "object") + || got_fields != fields_from_value(&pv.expected) + { + failures.push(format!( + "{}: object/fields differ from the plan group's pin", + v.name + )); + } + } + } + + assert!( + successes > 0 && refusals > 0, + "the corpus needs both artifacts and refusals" + ); + assert!( + failures.is_empty(), + "{} of {} verdict vectors failed:\n {}", + failures.len(), + vectors.len(), + failures.join("\n ") + ); +} + +/// The corpus must be able to fail: each deliberate divergence of an +/// otherwise correct artifact has to be caught by the very comparison the +/// vectors use. The Go peer runs the identical list of divergences. +#[test] +fn verdict_vectors_detect_deliberate_divergence() { + type Mutation = (&'static str, fn(&mut ConformanceVerdict)); + fn flip_hex(d: &str) -> String { + let mut chars: Vec = d.chars().collect(); + let last = chars.len() - 1; + chars[last] = if chars[last] == '0' { '1' } else { '0' }; + chars.into_iter().collect() + } + let mutations: [Mutation; 8] = [ + ("schema version bumped", |v| v.verdict_schema_version += 1), + ("object changed", |v| { + v.object = if v.object == ObjectConformance::Conformant { + ObjectConformance::Drift + } else { + ObjectConformance::Conformant + } + }), + ("one field verdict changed", |v| { + let key = v + .fields + .keys() + .next() + .expect("a vector has a field") + .clone(); + let cur = v.fields[&key]; + v.fields.insert( + key, + if cur == FieldVerdict::Drift { + FieldVerdict::Conformant + } else { + FieldVerdict::Drift + }, + ); + }), + ("one field dropped", |v| { + let key = v + .fields + .keys() + .next() + .expect("a vector has a field") + .clone(); + v.fields.remove(&key); + }), + ("one field added", |v| { + v.fields + .insert("/extra".to_string(), FieldVerdict::Conformant); + }), + ("plan digest altered", |v| { + v.conformance_plan_digest = flip_hex(&v.conformance_plan_digest) + }), + ("observation digest altered", |v| { + v.observation_digest = flip_hex(&v.observation_digest) + }), + ("plan and observation digests swapped", |v| { + std::mem::swap(&mut v.conformance_plan_digest, &mut v.observation_digest) + }), + ]; + + let mut checked = 0; + for v in load_vectors("verdict") { + if map_get(&v.expected, "canonical").is_none() { + continue; + } + let (plan, result) = evaluate_vector(&v); + let good = build_verdict(&plan, &result).expect("a success vector builds"); + assert!( + verdict_mismatches(&good, &v.expected).is_empty(), + "{}: the unmodified artifact must match first", + v.name + ); + for (what, mutate) in &mutations { + let mut bad = good.clone(); + mutate(&mut bad); + assert!( + !verdict_mismatches(&bad, &v.expected).is_empty(), + "{}: divergence not detected: {what}", + v.name + ); + checked += 1; + } + } + assert!(checked > 0, "no success vector was mutated"); +} diff --git a/go/plan/plan.go b/go/plan/plan.go index 2b098b0..891c775 100644 --- a/go/plan/plan.go +++ b/go/plan/plan.go @@ -446,6 +446,13 @@ func aggregate(fields map[string]conformance.FieldVerdict) ObjectConformance { } } +// Aggregate is aggregate, exported so the verdict artifact (go/verdict) +// can check that an ObjectVerdict's Object follows from its Fields -- +// the Go peer of plan.rs's own `pub fn aggregate`. +func Aggregate(fields map[string]conformance.FieldVerdict) ObjectConformance { + return aggregate(fields) +} + // elementKeys returns the sorted union of element identities of a // collection across the intent and observed lists, keyed by the // collection's topology -- the Go peer of plan.rs's own element_keys, diff --git a/go/verdict/differential_test.go b/go/verdict/differential_test.go new file mode 100644 index 0000000..67c2d62 --- /dev/null +++ b/go/verdict/differential_test.go @@ -0,0 +1,371 @@ +package verdict + +// Runs the verdict group of the language-independent differential corpus in +// conformance/differential/ -- the Go peer of engine/tests/differential.rs's +// verdict_vectors and verdict_vectors_detect_deliberate_divergence, reading +// the identical JSON files. See that directory's README for why the corpus +// is JSON-only. + +import ( + "bytes" + "encoding/json" + "errors" + "os" + "path/filepath" + "sort" + "testing" + + "github.com/CentralInfraCore/cic-primitive-engine/go/canonical" + "github.com/CentralInfraCore/cic-primitive-engine/go/collection" + "github.com/CentralInfraCore/cic-primitive-engine/go/conformance" + "github.com/CentralInfraCore/cic-primitive-engine/go/materialized" + "github.com/CentralInfraCore/cic-primitive-engine/go/plan" +) + +func corpusRoot(group string) string { + // go test runs in this package's directory (go/verdict/). + return filepath.Join("..", "..", "conformance", "differential", group) +} + +type fieldPlanInput struct { + Path string `json:"path"` + Compare string `json:"compare"` +} + +type tamperOp struct { + Op string `json:"op"` + Path string `json:"path"` + Value string `json:"value"` +} + +type vectorInput struct { + Intent map[string]interface{} `json:"intent"` + Observed map[string]interface{} `json:"observed"` + Observation []struct { + Path string `json:"path"` + Coverage string `json:"coverage"` + } `json:"observation"` + Plan struct { + Scalars []fieldPlanInput `json:"scalars"` + Collections []struct { + Path string `json:"path"` + Topology string `json:"topology"` + Keys []string `json:"keys"` + Elements []fieldPlanInput `json:"elements"` + } `json:"collections"` + } `json:"plan"` + Tamper []tamperOp `json:"tamper"` +} + +type vectorExpected struct { + Verdict json.RawMessage `json:"verdict"` + Canonical string `json:"canonical"` + Error *struct { + Code string `json:"code"` + Path string `json:"path"` + } `json:"error"` +} + +func readJSON(t *testing.T, path string, v interface{}) { + t.Helper() + b, err := os.ReadFile(path) + if err != nil { + t.Fatalf("%s: %v", path, err) + } + if err := json.Unmarshal(b, v); err != nil { + t.Fatalf("%s: %v", path, err) + } +} + +func vectorNames(t *testing.T, group string) []string { + t.Helper() + entries, err := os.ReadDir(corpusRoot(group)) + if err != nil { + t.Fatalf("%s: %v", group, err) + } + var names []string + for _, e := range entries { + if e.IsDir() { + names = append(names, e.Name()) + } + } + sort.Strings(names) + if len(names) == 0 { + t.Fatalf("%s: no vectors -- an empty corpus trivially passes", group) + } + return names +} + +// objectFromValue wraps a fixture map into a MaterializedObject, the same +// test-only convenience go/plan's own tests use. +func objectFromValue( + t *testing.T, + v map[string]interface{}, + evidenceOf func(key string, val interface{}) materialized.FieldEvidence, +) materialized.MaterializedObject { + t.Helper() + fields := make(map[string]materialized.MaterializedField, len(v)) + keys := make(map[string]struct{}, len(v)) + for k, val := range v { + keys[k] = struct{}{} + fields[k] = materialized.MaterializedField{ + Capability: materialized.CapabilityImplemented, + Evidence: evidenceOf(k, val), + } + } + obj, err := materialized.NewMaterializedObject(fields, keys) + if err != nil { + t.Fatalf("a fixture's own key set is trivially complete against itself: %v", err) + } + return obj +} + +func evaluateVector(t *testing.T, in vectorInput) (plan.ConformancePlan, plan.ObjectVerdict) { + t.Helper() + obs := conformance.NewObservation() + for _, o := range in.Observation { + obs.Set(o.Path, conformance.Coverage(o.Coverage)) + } + toPlan := func(fp fieldPlanInput) plan.FieldPlan { + return plan.FieldPlan{Path: fp.Path, Compare: conformance.CompareType(fp.Compare)} + } + scalars := make([]plan.FieldPlan, len(in.Plan.Scalars)) + for i, fp := range in.Plan.Scalars { + scalars[i] = toPlan(fp) + } + collections := make([]plan.CollectionPlan, len(in.Plan.Collections)) + for i, cp := range in.Plan.Collections { + elements := make([]plan.FieldPlan, len(cp.Elements)) + for j, fp := range cp.Elements { + elements[j] = toPlan(fp) + } + collections[i] = plan.CollectionPlan{ + Path: cp.Path, + Collection: collection.Collection{ + Topology: collection.CollectionTopology(cp.Topology), + Keys: cp.Keys, + }, + Elements: elements, + } + } + cp := plan.ConformancePlan{Scalars: scalars, Collections: collections} + + intent := objectFromValue(t, in.Intent, func(_ string, val interface{}) materialized.FieldEvidence { + return materialized.NewIntentFieldEvidence(materialized.NewAuthoredIntentEvidence(val, true)) + }) + observed := objectFromValue(t, in.Observed, func(k string, val interface{}) materialized.FieldEvidence { + if _, isContainer := val.([]interface{}); isContainer { + return materialized.NewObservationFieldEvidence(conformance.CoverageObserved, val) + } + return materialized.NewObservationFieldEvidence(obs.Coverage("/"+k), val) + }) + result := plan.Evaluate(intent, observed, obs, cp) + applyTamper(t, &result, in.Tamper) + return cp, result +} + +// applyTamper edits the Evaluate result before the artifact is built, to +// reach states a real Evaluate never produces. +func applyTamper(t *testing.T, result *plan.ObjectVerdict, ops []tamperOp) { + t.Helper() + for _, op := range ops { + switch op.Op { + case "drop_consumed": + delete(result.Consumed, op.Path) + case "add_consumed": + result.Consumed[op.Path] = plan.NewAbsentConsumedField() + case "drop_field": + delete(result.Fields, op.Path) + case "set_object": + result.Object = plan.ObjectConformance(op.Value) + default: + t.Fatalf("unknown tamper op %q", op.Op) + } + } +} + +// mismatches is empty when the artifact matches the pinned expectation +// exactly: the canonical bytes equal the pinned string, and the pinned +// verdict tree canonicalizes to those same bytes. +func mismatches(t *testing.T, got Verdict, exp vectorExpected) []string { + t.Helper() + var out []string + b, err := got.CanonicalBytes() + if err != nil { + t.Fatalf("a verdict canonicalizes: %v", err) + } + if string(b) != exp.Canonical { + out = append(out, "canonical bytes differ\n got: "+string(b)+"\n want: "+exp.Canonical) + } + dec := json.NewDecoder(bytes.NewReader(exp.Verdict)) + dec.UseNumber() + var tree interface{} + if err := dec.Decode(&tree); err != nil { + t.Fatalf("pinned verdict tree: %v", err) + } + treeBytes, err := canonical.ToCanonicalJSON(tree) + if err != nil { + t.Fatalf("pinned verdict tree canonicalizes: %v", err) + } + if !bytes.Equal(treeBytes, b) { + out = append(out, "pinned verdict tree and pinned canonical string disagree with the artifact") + } + return out +} + +type planExpected struct { + Object string `json:"object"` + Fields map[string]string `json:"fields"` +} + +func TestDifferentialVerdictVectors(t *testing.T) { + names := vectorNames(t, "verdict") + planNames := map[string]bool{} + for _, n := range vectorNames(t, "plan") { + planNames[n] = true + } + successes, refusals := 0, 0 + for _, name := range names { + dir := filepath.Join(corpusRoot("verdict"), name) + var in vectorInput + readJSON(t, filepath.Join(dir, "input.json"), &in) + var exp vectorExpected + readJSON(t, filepath.Join(dir, "expected.json"), &exp) + + cp, result := evaluateVector(t, in) + got, err := Build(cp, result) + + if exp.Error != nil { + refusals++ + var verr *Error + if err == nil { + t.Errorf("%s: expected a refusal, got an artifact", name) + } else if !errors.As(err, &verr) { + t.Errorf("%s: refusal is not a *Error: %v", name, err) + } else if verr.Code != exp.Error.Code || verr.Path != exp.Error.Path { + t.Errorf("%s: refusal got %s at %s, want %s at %s", name, verr.Code, verr.Path, exp.Error.Code, exp.Error.Path) + } + continue + } + + successes++ + if err != nil { + t.Errorf("%s: unexpected refusal: %v", name, err) + continue + } + for _, m := range mismatches(t, got, exp) { + t.Errorf("%s: %s", name, m) + } + // The artifact's Object/Fields are the plan group's own pinned + // expectation for the same input, not a second opinion. + if planNames[name] { + var pe planExpected + readJSON(t, filepath.Join(corpusRoot("plan"), name, "expected.json"), &pe) + gotFields := make(map[string]string, len(got.Fields)) + for k, v := range got.Fields { + gotFields[k] = string(v) + } + if string(got.Object) != pe.Object || len(gotFields) != len(pe.Fields) { + t.Errorf("%s: object/fields differ from the plan group's pin", name) + continue + } + for k, want := range pe.Fields { + if gotFields[k] != want { + t.Errorf("%s: field %s: got %s, want %s", name, k, gotFields[k], want) + } + } + } + } + if successes == 0 || refusals == 0 { + t.Fatalf("the corpus needs both artifacts and refusals (got %d / %d)", successes, refusals) + } +} + +func flipHex(d string) string { + b := []byte(d) + last := len(b) - 1 + if b[last] == '0' { + b[last] = '1' + } else { + b[last] = '0' + } + return string(b) +} + +func firstField(v Verdict) string { + keys := make([]string, 0, len(v.Fields)) + for k := range v.Fields { + keys = append(keys, k) + } + sort.Strings(keys) + return keys[0] +} + +// The corpus must be able to fail: each deliberate divergence of an +// otherwise correct artifact has to be caught by the very comparison the +// vectors use. Same list as the Rust peer's verdict_vectors_detect_deliberate_divergence. +func TestDifferentialVerdictVectorsDetectDeliberateDivergence(t *testing.T) { + mutations := []struct { + what string + mutate func(v *Verdict) + }{ + {"schema version bumped", func(v *Verdict) { v.SchemaVersion++ }}, + {"object changed", func(v *Verdict) { + if v.Object == plan.ObjectConformant { + v.Object = plan.ObjectDrift + } else { + v.Object = plan.ObjectConformant + } + }}, + {"one field verdict changed", func(v *Verdict) { + k := firstField(*v) + if v.Fields[k] == conformance.VerdictDrift { + v.Fields[k] = conformance.VerdictConformant + } else { + v.Fields[k] = conformance.VerdictDrift + } + }}, + {"one field dropped", func(v *Verdict) { delete(v.Fields, firstField(*v)) }}, + {"one field added", func(v *Verdict) { v.Fields["/extra"] = conformance.VerdictConformant }}, + {"plan digest altered", func(v *Verdict) { v.ConformancePlanDigest = flipHex(v.ConformancePlanDigest) }}, + {"observation digest altered", func(v *Verdict) { v.ObservationDigest = flipHex(v.ObservationDigest) }}, + {"plan and observation digests swapped", func(v *Verdict) { + v.ConformancePlanDigest, v.ObservationDigest = v.ObservationDigest, v.ConformancePlanDigest + }}, + } + + checked := 0 + for _, name := range vectorNames(t, "verdict") { + dir := filepath.Join(corpusRoot("verdict"), name) + var exp vectorExpected + readJSON(t, filepath.Join(dir, "expected.json"), &exp) + if exp.Error != nil { + continue + } + var in vectorInput + readJSON(t, filepath.Join(dir, "input.json"), &in) + cp, result := evaluateVector(t, in) + good, err := Build(cp, result) + if err != nil { + t.Fatalf("%s: a success vector builds: %v", name, err) + } + if m := mismatches(t, good, exp); len(m) != 0 { + t.Fatalf("%s: the unmodified artifact must match first: %v", name, m) + } + for _, mu := range mutations { + bad := good + bad.Fields = make(map[string]conformance.FieldVerdict, len(good.Fields)+1) + for k, v := range good.Fields { + bad.Fields[k] = v + } + mu.mutate(&bad) + if len(mismatches(t, bad, exp)) == 0 { + t.Errorf("%s: divergence not detected: %s", name, mu.what) + } + checked++ + } + } + if checked == 0 { + t.Fatal("no success vector was mutated") + } +} diff --git a/go/verdict/verdict.go b/go/verdict/verdict.go new file mode 100644 index 0000000..7005e65 --- /dev/null +++ b/go/verdict/verdict.go @@ -0,0 +1,168 @@ +// Package verdict assembles the conformance/drift verdict artifact into the +// single five-field shape docs/VERDICT-SCHEMA.md commits to -- the Go peer of +// this repo's own engine/src/verdict.rs. +// +// plan.Evaluate produces the semantic core (Object, Fields, and the Consumed +// map it recorded while classifying) and go/digestprojection produces the two +// digests from a plan and from that Consumed map. Nothing bundled those into +// the one required artifact until this package. +// +// This package decides nothing new: +// +// - Object and Fields are Evaluate's own result, carried over verbatim; +// - ConformancePlanDigest is digestprojection.ConformancePlanDigest of the +// plan that was executed and ObservationDigest is +// digestprojection.ObservationDigest of the Consumed map Evaluate +// recorded -- the existing normative projections, called, never +// recomputed or re-derived here; +// - SchemaVersion is the bare integer 1 (VERDICT-SCHEMA.md). +// +// # What is checked before an artifact exists +// +// The schema says Fields holds exactly one entry per path Evaluate +// classified, and that every field is required. Build therefore refuses an +// ObjectVerdict that cannot be such a result, rather than digesting it into +// an artifact that would look authoritative: +// +// - the Fields and Consumed path sets must be identical; +// - every plan scalar path must be in Fields; +// - Object must be what plan.Aggregate gives for Fields. +// +// A Consumed entry that is not a valid ConsumedField (Go's zero value, which +// no constructor produced) is refused the same way; Rust cannot construct +// that state at all, so this check has no Rust counterpart. +// +// The checks run in the same order as the Rust peer and walk paths sorted +// byte-wise, so both languages name the same offending path. +package verdict + +import ( + "fmt" + "sort" + + "github.com/CentralInfraCore/cic-primitive-engine/go/canonical" + "github.com/CentralInfraCore/cic-primitive-engine/go/conformance" + "github.com/CentralInfraCore/cic-primitive-engine/go/digestprojection" + "github.com/CentralInfraCore/cic-primitive-engine/go/plan" +) + +// SchemaVersion is the only verdict_schema_version this package produces. +// Incremented on any backward-incompatible change to the verdict contract. +const SchemaVersion int64 = 1 + +// CodeInconsistentVerdict is the rejection code engine/src/error.rs calls +// INCONSISTENT_VERDICT ("E_INCONSISTENT_VERDICT"). +const CodeInconsistentVerdict = "E_INCONSISTENT_VERDICT" + +// Error is a rejection with the same code/rule/path fields the Rust peer's +// Error carries. +type Error struct { + Code string + Rule string + Path string + Detail string +} + +func (e *Error) Error() string { + return fmt.Sprintf("%s (%s) at %s: %s", e.Code, e.Rule, e.Path, e.Detail) +} + +func inconsistent(path, detail string) *Error { + return &Error{Code: CodeInconsistentVerdict, Rule: "R-VERDICT-CONSISTENT", Path: path, Detail: detail} +} + +// Verdict holds the five required fields. There is no optional member. +type Verdict struct { + SchemaVersion int64 + Object plan.ObjectConformance + Fields map[string]conformance.FieldVerdict + ConformancePlanDigest string + ObservationDigest string +} + +func sortedPaths[V any](m map[string]V) []string { + keys := make([]string, 0, len(m)) + for k := range m { + keys = append(keys, k) + } + sort.Strings(keys) + return keys +} + +func checkConsistent(cp plan.ConformancePlan, result plan.ObjectVerdict) error { + for _, path := range sortedPaths(result.Fields) { + if _, ok := result.Consumed[path]; !ok { + return inconsistent(path, "classified path has no consumed observation evidence") + } + } + for _, path := range sortedPaths(result.Consumed) { + if _, ok := result.Fields[path]; !ok { + return inconsistent(path, "consumed observation evidence for a path that was not classified") + } + if !result.Consumed[path].Valid() { + return inconsistent(path, "consumed observation evidence is not a valid ConsumedField") + } + } + for _, fp := range cp.Scalars { + if _, ok := result.Fields[fp.Path]; !ok { + return inconsistent(fp.Path, "planned scalar path is missing from fields") + } + } + if want := plan.Aggregate(result.Fields); want != result.Object { + return inconsistent("$", fmt.Sprintf( + "object %s does not follow from fields (expected %s)", result.Object, want)) + } + return nil +} + +// Build assembles the verdict artifact for result, the output of +// plan.Evaluate run against cp. +// +// Returns an *Error with code E_INCONSISTENT_VERDICT if result cannot be an +// Evaluate result for cp (see the package doc), or the canonicalization error +// of the digest functions. +func Build(cp plan.ConformancePlan, result plan.ObjectVerdict) (Verdict, error) { + if err := checkConsistent(cp, result); err != nil { + return Verdict{}, err + } + planDigest, err := digestprojection.ConformancePlanDigest(cp) + if err != nil { + return Verdict{}, err + } + obsDigest, err := digestprojection.ObservationDigest(result.Consumed) + if err != nil { + return Verdict{}, err + } + fields := make(map[string]conformance.FieldVerdict, len(result.Fields)) + for k, v := range result.Fields { + fields[k] = v + } + return Verdict{ + SchemaVersion: SchemaVersion, + Object: result.Object, + Fields: fields, + ConformancePlanDigest: planDigest, + ObservationDigest: obsDigest, + }, nil +} + +// Value is the artifact as a value tree: exactly the five required keys. +func (v Verdict) Value() map[string]interface{} { + fields := make(map[string]interface{}, len(v.Fields)) + for path, fv := range v.Fields { + fields[path] = string(fv) + } + return map[string]interface{}{ + "verdict_schema_version": v.SchemaVersion, + "object": string(v.Object), + "fields": fields, + "conformance_plan_digest": v.ConformancePlanDigest, + "observation_digest": v.ObservationDigest, + } +} + +// CanonicalBytes is the artifact's canonical bytes (section A): the value +// tree of Value through canonical.ToCanonicalJSON. +func (v Verdict) CanonicalBytes() ([]byte, error) { + return canonical.ToCanonicalJSON(v.Value()) +} diff --git a/go/verdict/verdict_test.go b/go/verdict/verdict_test.go new file mode 100644 index 0000000..fc4174e --- /dev/null +++ b/go/verdict/verdict_test.go @@ -0,0 +1,160 @@ +package verdict + +import ( + "errors" + "strings" + "testing" + + "github.com/CentralInfraCore/cic-primitive-engine/go/conformance" + "github.com/CentralInfraCore/cic-primitive-engine/go/digestprojection" + "github.com/CentralInfraCore/cic-primitive-engine/go/materialized" + "github.com/CentralInfraCore/cic-primitive-engine/go/plan" +) + +// sample mirrors engine/src/verdict.rs's own test fixture: two planned +// scalars, one matching and one drifting. +func sample(t *testing.T) (plan.ConformancePlan, plan.ObjectVerdict) { + t.Helper() + cp := plan.ConformancePlan{Scalars: []plan.FieldPlan{ + {Path: "/shape", Compare: conformance.CompareExact}, + {Path: "/memory_gb", Compare: conformance.CompareNumeric}, + }} + obs := conformance.NewObservation() + obs.Set("/shape", conformance.CoverageObserved) + obs.Set("/memory_gb", conformance.CoverageObserved) + intent := objectFromValue(t, map[string]interface{}{"shape": "E4", "memory_gb": 16}, + func(_ string, val interface{}) materialized.FieldEvidence { + return materialized.NewIntentFieldEvidence(materialized.NewAuthoredIntentEvidence(val, true)) + }) + observed := objectFromValue(t, map[string]interface{}{"shape": "E4", "memory_gb": 32.0}, + func(k string, val interface{}) materialized.FieldEvidence { + return materialized.NewObservationFieldEvidence(obs.Coverage("/"+k), val) + }) + return cp, plan.Evaluate(intent, observed, obs, cp) +} + +func wantInconsistent(t *testing.T, err error, path string) { + t.Helper() + var verr *Error + if !errors.As(err, &verr) { + t.Fatalf("want *Error, got %v", err) + } + if verr.Code != CodeInconsistentVerdict || verr.Path != path { + t.Fatalf("got %s at %s, want %s at %s", verr.Code, verr.Path, CodeInconsistentVerdict, path) + } +} + +func TestARealEvaluateResultBuildsAllFiveFields(t *testing.T) { + cp, result := sample(t) + v, err := Build(cp, result) + if err != nil { + t.Fatal(err) + } + if v.SchemaVersion != 1 || v.Object != plan.ObjectDrift || len(v.Fields) != 2 { + t.Fatalf("unexpected artifact: %+v", v) + } + if !strings.HasPrefix(v.ConformancePlanDigest, "sha256:") || !strings.HasPrefix(v.ObservationDigest, "sha256:") { + t.Fatalf("digests are not sha256: %q %q", v.ConformancePlanDigest, v.ObservationDigest) + } +} + +func TestTheDigestsAreTheExistingProjectionsNotANewComputation(t *testing.T) { + cp, result := sample(t) + v, err := Build(cp, result) + if err != nil { + t.Fatal(err) + } + wantPlan, _ := digestprojection.ConformancePlanDigest(cp) + wantObs, _ := digestprojection.ObservationDigest(result.Consumed) + if v.ConformancePlanDigest != wantPlan || v.ObservationDigest != wantObs { + t.Fatal("verdict digests differ from the digestprojection functions") + } +} + +func TestFieldsHoldExactlyTheEvaluatedPathsAndDoNotAliasTheResult(t *testing.T) { + cp, result := sample(t) + v, err := Build(cp, result) + if err != nil { + t.Fatal(err) + } + if len(v.Fields) != len(result.Fields) { + t.Fatalf("fields %d, evaluated %d", len(v.Fields), len(result.Fields)) + } + result.Fields["/shape"] = conformance.VerdictDrift + if v.Fields["/shape"] != conformance.VerdictConformant { + t.Fatal("the artifact aliases the Evaluate result's map") + } +} + +func TestTheValueTreeHasExactlyTheFiveRequiredKeys(t *testing.T) { + cp, result := sample(t) + v, _ := Build(cp, result) + tree := v.Value() + want := []string{"conformance_plan_digest", "fields", "object", "observation_digest", "verdict_schema_version"} + if len(tree) != len(want) { + t.Fatalf("got %d keys, want %d", len(tree), len(want)) + } + for _, k := range want { + if _, ok := tree[k]; !ok { + t.Fatalf("missing key %s", k) + } + } + if tree["verdict_schema_version"] != int64(1) { + t.Fatalf("version is %v", tree["verdict_schema_version"]) + } +} + +func TestCanonicalBytesAreSortedAndStable(t *testing.T) { + cp, result := sample(t) + v, _ := Build(cp, result) + a, err := v.CanonicalBytes() + if err != nil { + t.Fatal(err) + } + b, _ := v.CanonicalBytes() + if string(a) != string(b) { + t.Fatal("canonical bytes are not stable") + } + s := string(a) + if !strings.HasPrefix(s, `{"conformance_plan_digest":"sha256:`) || !strings.HasSuffix(s, `"verdict_schema_version":1}`) { + t.Fatalf("unexpected canonical form: %s", s) + } +} + +func TestAClassifiedPathWithoutEvidenceIsRefused(t *testing.T) { + cp, result := sample(t) + delete(result.Consumed, "/shape") + _, err := Build(cp, result) + wantInconsistent(t, err, "/shape") +} + +func TestEvidenceForAnUnclassifiedPathIsRefused(t *testing.T) { + cp, result := sample(t) + result.Consumed["/ghost"] = plan.NewAbsentConsumedField() + _, err := Build(cp, result) + wantInconsistent(t, err, "/ghost") +} + +func TestAPlannedScalarMissingFromFieldsIsRefused(t *testing.T) { + cp, result := sample(t) + delete(result.Fields, "/memory_gb") + delete(result.Consumed, "/memory_gb") + _, err := Build(cp, result) + wantInconsistent(t, err, "/memory_gb") +} + +func TestAnObjectThatDoesNotFollowFromFieldsIsRefused(t *testing.T) { + cp, result := sample(t) + result.Object = plan.ObjectConformant + _, err := Build(cp, result) + wantInconsistent(t, err, "$") +} + +// Go-only: the zero ConsumedField is not one of the four states a +// constructor produces, and Rust cannot construct the equivalent at all. +func TestAZeroValueConsumedFieldIsRefusedNotDigested(t *testing.T) { + cp, result := sample(t) + result.Consumed["/shape"] = plan.ConsumedField{} + _, err := Build(cp, result) + wantInconsistent(t, err, "/shape") +} From d6fa5047314ac25d12b898d00d77d9ccd8a91b2b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?G=C3=A1bor=20Z=20Sink=C3=B3?= Date: Sun, 18 Oct 2026 23:51:57 +0200 Subject: [PATCH 2/2] fix: bind the verdict to its plan through evaluate_verdict, refuse paths the plan cannot produce, document the low-level builder's limits --- [signing-metadata] key = cic-my-sign-key signature = vault:v1:MEQCIGZSBvHYHFyFSWbL8NRiiQVezQ31+1bSXYK4ETmdO8ciAiAEA2x9nFCGnk3n05DLh8NG27zGDx6I1T6igM98gUR1oQ== hash-algorithm = sha256 digest = /7LnYrURshnUbEAx/j8JXxB4GWNcQCkpzRdqMyTJZVc= [certificate] -----BEGIN CERTIFICATE----- MIICBjCCAaygAwIBAgIUSnRMR6RPnEbg296XWPOqq/u5PCwwCgYIKoZIzj0EAwIw QzELMAkGA1UEBhMCSFUxGTAXBgNVBAoMEENlbnRyYWxJbmZyYUNvcmUxGTAXBgNV BAMMEENJQyBEZXZlbG9wZXIgQ0EwHhcNMjYwMzIwMTMyMjU5WhcNMjYxMjMxMTMy MjU5WjBFMQswCQYDVQQGEwJIVTEZMBcGA1UECgwQQ2VudHJhbEluZnJhQ29yZTEb MBkGA1UEAwwSR2Fib3IgWm9sdGFuIFNpbmtvMFkwEwYHKoZIzj0CAQYIKoZIzj0D AQcDQgAEIG2CVmTfmLB9pLLclj7YmP2eedAjklpy4LGrU2ijoiy6Xqpuybv7OgJe i+ez31s65NEV8+X/ByeX1cstR988z6N8MHowCQYDVR0TBAIwADAdBgNVHQ4EFgQU yZN6AIX/TNnIJ9GwAa/NRN3ujHAwHwYDVR0jBBgwFoAUXn6CHYzPUqU4JVP8g+OS WeDYjhcwDgYDVR0PAQH/BAQDAgeAMB0GA1UdJQQWMBQGCCsGAQUFBwMCBggrBgEF BQcDBDAKBggqhkjOPQQDAgNIADBFAiEA+bFzXRoJ4PCQbhAAtpkcMjt0vNj5rEW0 lOMBGDNyaWkCIB1vmM7PcZzv/c9bIrxF5kqv6QXomouhByUfeNUTbpKW -----END CERTIFICATE----- --- README.md | 3 + docs/VERDICT-SCHEMA.md | 17 +++ engine/src/lib.rs | 2 +- engine/src/verdict.rs | 224 +++++++++++++++++++++++++++++--- engine/tests/differential.rs | 30 ++++- go/verdict/differential_test.go | 39 +++++- go/verdict/verdict.go | 90 +++++++++++-- go/verdict/verdict_test.go | 118 +++++++++++++++++ 8 files changed, 492 insertions(+), 31 deletions(-) diff --git a/README.md b/README.md index d4aa5aa..97dfd67 100644 --- a/README.md +++ b/README.md @@ -110,6 +110,9 @@ correction to the pipeline framing this file used to carry. > (`E_INCONSISTENT_VERDICT`), and are cross-checked byte-for-byte through > `conformance/differential/verdict/`, including a test that the corpus > detects deliberate divergence. +> `evaluate_verdict`/`verdict.EvaluateAndBuild` is the primary entry point (plan and +> result are bound by construction); `build_verdict`/`verdict.Build` is the +> low-level assembler and checks structure only -- see `docs/VERDICT-SCHEMA.md`. > > Nothing here is a stable API. diff --git a/docs/VERDICT-SCHEMA.md b/docs/VERDICT-SCHEMA.md index fcf86a4..6e8383f 100644 --- a/docs/VERDICT-SCHEMA.md +++ b/docs/VERDICT-SCHEMA.md @@ -228,6 +228,23 @@ document's full contract into one artifact: all five required fields, with deliberate divergences (version bump, changed object, changed/dropped/added field, altered or swapped digests) and requires the comparison to catch each. +**Two entry points, with different guarantees.** `evaluate_verdict` / +`verdict.EvaluateAndBuild` run `evaluate` and build the artifact in one +operation, so the `conformance_plan_digest` written into the artifact is the +digest of the plan that actually produced the verdicts and the result cannot be +swapped or edited in between. `build_verdict` / `verdict.Build` is the +low-level assembler for a caller that already holds a plan and a result. It +checks structure only: `fields` and `consumed` name the same paths, every +`fields` path is one the plan can produce (a planned scalar, or a planned +collection element path of the right shape), every planned scalar is present, +and `object` follows from `fields`. It does **not** know which plan produced +the result (a result from one plan paired with another plan of the same paths +and a different comparator is accepted, with the second plan's digest), does +not re-evaluate (a self-consistent altered result is accepted), and cannot tell +a real element path from one that only has the right shape. It must not be +described as rejecting every tampered result; where the binding matters, use the +primary entry point. + Still not decided, unchanged: signing the artifact, and an intent-digest field. This builder adds neither. The Relay's enforcement boundary and the materialization receipt are untouched. diff --git a/engine/src/lib.rs b/engine/src/lib.rs index 8d73477..3ef4770 100644 --- a/engine/src/lib.rs +++ b/engine/src/lib.rs @@ -246,6 +246,6 @@ pub use plan::{ ObjectVerdict, }; pub use verdict::{ - build_verdict, verdict_canonical_bytes, verdict_value, ConformanceVerdict, + build_verdict, evaluate_verdict, verdict_canonical_bytes, verdict_value, ConformanceVerdict, VERDICT_SCHEMA_VERSION, }; diff --git a/engine/src/verdict.rs b/engine/src/verdict.rs index 5b6c8e6..a11804c 100644 --- a/engine/src/verdict.rs +++ b/engine/src/verdict.rs @@ -15,30 +15,48 @@ //! called, never recomputed or re-derived here; //! * `verdict_schema_version` is the bare integer `1` (`VERDICT-SCHEMA.md`). //! -//! # What is checked before an artifact exists +//! # Two entry points, and what each one guarantees //! -//! The schema says `fields` holds exactly one entry per path `evaluate` -//! classified, and that every field is required. [`build_verdict`] therefore -//! refuses an [`ObjectVerdict`] that cannot be such a result, rather than -//! digesting it into an artifact that would look authoritative: +//! **[`evaluate_verdict`] is the primary entry point.** It runs +//! [`evaluate`] and builds the artifact in one operation, so the plan whose +//! digest is written into the artifact is, by construction, the plan that +//! produced the verdicts, and the result cannot be edited in between. Use it +//! unless you only hold a finished result. //! -//! * the `fields` and `consumed` path sets must be identical (every classified -//! path has its observation evidence, and no evidence is left over); -//! * every `plan.scalars` path must be in `fields`; -//! * `object` must be what [`aggregate`] gives for `fields`. +//! [`build_verdict`] is the low-level assembler for a caller that already holds +//! a `plan` and an [`ObjectVerdict`]. It checks structure only, and it is +//! exactly as trustworthy as the caller who pairs the two: //! -//! A real `evaluate` result always passes; a result that was edited or -//! assembled by hand does not. Scope stays with the contract: signing the -//! artifact and an intent digest are explicitly "not decided" in the schema and -//! are not added here. +//! * it rejects a result that is not shaped like an `evaluate` result for this +//! plan: `fields` and `consumed` must name the same paths, every `fields` +//! path must be one the plan can produce (a planned scalar, or a planned +//! collection element path), every planned scalar must be present, and +//! `object` must be what [`aggregate`] gives for `fields`; +//! * it does **not** know which plan produced the result. A result evaluated +//! under one plan, paired with another plan that has the same paths and a +//! different comparator, is accepted, and the second plan's digest is written +//! next to the first plan's verdicts; +//! * it does **not** re-evaluate, so a self-consistent but altered result (a +//! changed field verdict with a matching `object`, a changed consumed value) +//! is accepted; +//! * it cannot tell a collection element path that `evaluate` produced from +//! one that only has the right shape. +//! +//! So `build_verdict` must never be described as rejecting every tampered +//! result. Where the binding matters, use [`evaluate_verdict`]. +//! +//! Scope stays with the contract: signing the artifact and an intent digest are +//! explicitly "not decided" in the schema and are not added here. use std::collections::BTreeMap; use crate::canonical::to_canonical_json; use crate::conformance::FieldVerdict; +use crate::conformance::Observation; use crate::digest_projection::{conformance_plan_digest, observation_digest}; use crate::error::{code, Error, Result, Stage}; -use crate::plan::{aggregate, ConformancePlan, ObjectConformance, ObjectVerdict}; +use crate::materialized::MaterializedObject; +use crate::plan::{aggregate, evaluate, ConformancePlan, ObjectConformance, ObjectVerdict}; use crate::reader::{Map, Value}; /// The only `verdict_schema_version` this module produces. Incremented on any @@ -56,8 +74,29 @@ pub struct ConformanceVerdict { pub observation_digest: String, } -/// Assembles the verdict artifact for `result`, the output of -/// [`crate::evaluate`] run against `plan`. +/// Runs [`evaluate`] and builds the verdict artifact in one operation: the +/// primary entry point. The plan digest in the artifact is the digest of the +/// plan that was actually executed, and the result cannot be swapped or edited +/// between evaluation and assembly. +/// +/// # Errors +/// As [`build_verdict`]; a result produced here is always consistent, so in +/// practice only a canonicalization error from the digest functions. +pub fn evaluate_verdict( + intent: &MaterializedObject, + observed: &MaterializedObject, + obs: &Observation, + plan: &ConformancePlan, +) -> Result { + let result = evaluate(intent, observed, obs, plan); + build_verdict(plan, &result) +} + +/// Assembles the verdict artifact for `result`, claimed to be the output of +/// [`evaluate`] run against `plan`. Low-level: it checks structure only and +/// cannot verify that `plan` is the plan that produced `result` -- see the +/// module doc for exactly what it does and does not reject, and prefer +/// [`evaluate_verdict`]. /// /// # Errors /// `E_INCONSISTENT_VERDICT` (`R-VERDICT-CONSISTENT`) if `result` cannot be an @@ -84,6 +123,22 @@ fn inconsistent(path: &str, detail: String) -> Error { ) } +/// Whether `evaluate` could have classified `path` under `plan`: a planned +/// scalar path, or a collection element path, which `evaluate` builds as +/// `{collection}/{identity}/{element}` with a non-empty identity. +fn plan_can_produce(plan: &ConformancePlan, path: &str) -> bool { + if plan.scalars.iter().any(|fp| fp.path == path) { + return true; + } + plan.collections.iter().any(|cp| { + let head = format!("{}/{{", cp.path); + cp.elements.iter().any(|ef| { + let tail = format!("}}/{}", ef.path); + path.len() > head.len() + tail.len() && path.starts_with(&head) && path.ends_with(&tail) + }) + }) +} + fn check_consistent(plan: &ConformancePlan, result: &ObjectVerdict) -> Result<()> { for path in result.fields.keys() { if !result.consumed.contains_key(path) { @@ -101,6 +156,15 @@ fn check_consistent(plan: &ConformancePlan, result: &ObjectVerdict) -> Result<() )); } } + for path in result.fields.keys() { + if !plan_can_produce(plan, path) { + return Err(inconsistent( + path, + "path is not one the plan can produce (no planned scalar or collection element matches)" + .to_string(), + )); + } + } for fp in &plan.scalars { if !result.fields.contains_key(&fp.path) { return Err(inconsistent( @@ -329,6 +393,134 @@ mod tests { assert_eq!(e.path, "$"); } + /// Counterexample 1 (review of PR #60): `/ghost` is in both `fields` and + /// `consumed`, so the two path sets agree and `object` follows, but the plan + /// never asked for it. + #[test] + fn counterexample_a_path_outside_the_plan_is_refused() { + let (plan, mut result) = sample(); + result + .fields + .insert("/ghost".to_string(), FieldVerdict::Conformant); + result + .consumed + .insert("/ghost".to_string(), ConsumedField::Absent); + let e = build_verdict(&plan, &result).expect_err("a path the plan never asked for"); + assert_eq!(e.code, code::INCONSISTENT_VERDICT); + assert_eq!(e.path, "/ghost"); + } + + fn tie_inputs() -> (MaterializedObject, MaterializedObject, Observation) { + let intent = object(&[("memory_gb", authored(Value::Int(16)))]); + let observed_obj = object(&[("memory_gb", observed(Value::Str("16".into())))]); + let mut obs = Observation::new(); + obs.set("/memory_gb", Coverage::Observed); + (intent, observed_obj, obs) + } + + fn memory_plan(compare: CompareType) -> ConformancePlan { + ConformancePlan { + scalars: vec![FieldPlan { + path: "/memory_gb".to_string(), + compare, + }], + collections: vec![], + } + } + + /// Counterexample 2 (review of PR #60): the result was produced under + /// `plan_a` but `build_verdict` is handed `plan_b` with the same paths and a + /// different comparator. The low-level builder cannot tell: it accepts and + /// writes `plan_b`'s digest next to `plan_a`'s verdicts. This is the + /// documented limit of `build_verdict`, pinned here so it cannot be + /// mistaken for a guarantee. + #[test] + fn counterexample_the_low_level_builder_does_not_bind_the_plan() { + let (intent, observed_obj, obs) = tie_inputs(); + let plan_a = memory_plan(CompareType::Numeric); + let plan_b = memory_plan(CompareType::Exact); + let result_a = evaluate(&intent, &observed_obj, &obs, &plan_a); + + let mixed = build_verdict(&plan_b, &result_a).expect("structure alone looks fine"); + assert_eq!( + mixed.conformance_plan_digest, + conformance_plan_digest(&plan_b).unwrap() + ); + assert_ne!( + mixed.conformance_plan_digest, + conformance_plan_digest(&plan_a).unwrap(), + "the artifact names a plan that did not produce its verdicts" + ); + } + + /// The answer to counterexample 2: through the primary entry point the + /// plan and the result cannot be paired independently, so each plan's + /// digest sits next to the verdicts that plan actually produced. + #[test] + fn counterexample_the_entry_point_binds_the_plan_to_its_result() { + let (intent, observed_obj, obs) = tie_inputs(); + let plan_a = memory_plan(CompareType::Numeric); + let plan_b = memory_plan(CompareType::Exact); + let a = evaluate_verdict(&intent, &observed_obj, &obs, &plan_a).unwrap(); + let b = evaluate_verdict(&intent, &observed_obj, &obs, &plan_b).unwrap(); + + assert_eq!( + a.conformance_plan_digest, + conformance_plan_digest(&plan_a).unwrap() + ); + assert_eq!( + b.conformance_plan_digest, + conformance_plan_digest(&plan_b).unwrap() + ); + assert_ne!(a.conformance_plan_digest, b.conformance_plan_digest); + // The comparator really changes the verdict, so the digest is not decoration. + assert_ne!(a.fields, b.fields); + // Each artifact is exactly what the low-level builder gives for the + // result of the SAME plan. + let direct = build_verdict(&plan_b, &evaluate(&intent, &observed_obj, &obs, &plan_b)); + assert_eq!(b, direct.unwrap()); + } + + #[test] + fn collection_element_paths_are_matched_by_shape_and_foreign_ones_refused() { + let plan = ConformancePlan { + scalars: vec![], + collections: vec![crate::plan::CollectionPlan { + path: "/nics".to_string(), + collection: Collection { + topology: CollectionTopology::Map, + keys: vec!["name".to_string()], + }, + elements: vec![FieldPlan { + path: "subnet".to_string(), + compare: CompareType::Exact, + }], + }], + }; + let with = |path: &str| { + let mut fields = BTreeMap::new(); + fields.insert(path.to_string(), FieldVerdict::Conformant); + let mut consumed = BTreeMap::new(); + consumed.insert(path.to_string(), ConsumedField::Absent); + ObjectVerdict { + object: ObjectConformance::Conformant, + fields, + consumed, + } + }; + assert!(build_verdict(&plan, &with("/nics/{name=a}/subnet")).is_ok()); + for foreign in [ + "/nics/{name=a}/other", + "/other/{name=a}/subnet", + "/nics/{}/subnet", + "/nics/subnet", + ] { + let e = build_verdict(&plan, &with(foreign)).unwrap_err(); + assert_eq!(e.code, code::INCONSISTENT_VERDICT, "{foreign}"); + assert_eq!(e.path, foreign); + } + } + #[test] fn a_collection_plan_does_not_change_the_checked_invariants() { let plan = ConformancePlan { diff --git a/engine/tests/differential.rs b/engine/tests/differential.rs index 77b228a..de8befa 100644 --- a/engine/tests/differential.rs +++ b/engine/tests/differential.rs @@ -9,7 +9,7 @@ use std::fs; use std::path::{Path, PathBuf}; use cic_primitive_engine::{ - build_verdict, classify_field_value, conformance_plan_digest, evaluate, + build_verdict, classify_field_value, conformance_plan_digest, evaluate, evaluate_verdict, materialized_object_digest, observation_digest, reader, to_canonical_json, verdict_canonical_bytes, Capability, Collection, CollectionPlan, CollectionTopology, CompareType, ConformancePlan, ConformanceVerdict, ConsumedField, Coverage, FieldEvidence, @@ -861,3 +861,31 @@ fn verdict_vectors_detect_deliberate_divergence() { } assert!(checked > 0, "no success vector was mutated"); } + +/// The primary entry point must give the pinned bytes too: every artifact +/// vector (the corpus is unchanged), evaluated and assembled in one +/// operation, matches the same pin the low-level builder is held to. +#[test] +fn verdict_vectors_through_the_entry_point() { + let mut checked = 0; + for v in load_vectors("verdict") { + if map_get(&v.expected, "canonical").is_none() { + continue; + } + let intent = map_get(&v.input, "intent").unwrap_or(&Value::Null); + let observed = map_get(&v.input, "observed").unwrap_or(&Value::Null); + let obs = build_observation(&v.input); + let plan = build_plan(&v.input); + let got = evaluate_verdict( + &intent_object(intent), + &observed_object(observed, &obs), + &obs, + &plan, + ) + .unwrap_or_else(|e| panic!("{}: {e}", v.name)); + let mismatches = verdict_mismatches(&got, &v.expected); + assert!(mismatches.is_empty(), "{}: {mismatches:?}", v.name); + checked += 1; + } + assert!(checked > 0, "no artifact vector was checked"); +} diff --git a/go/verdict/differential_test.go b/go/verdict/differential_test.go index 67c2d62..c1a3213 100644 --- a/go/verdict/differential_test.go +++ b/go/verdict/differential_test.go @@ -120,7 +120,7 @@ func objectFromValue( return obj } -func evaluateVector(t *testing.T, in vectorInput) (plan.ConformancePlan, plan.ObjectVerdict) { +func vectorObjects(t *testing.T, in vectorInput) (plan.ConformancePlan, materialized.MaterializedObject, materialized.MaterializedObject, *conformance.Observation) { t.Helper() obs := conformance.NewObservation() for _, o := range in.Observation { @@ -159,6 +159,12 @@ func evaluateVector(t *testing.T, in vectorInput) (plan.ConformancePlan, plan.Ob } return materialized.NewObservationFieldEvidence(obs.Coverage("/"+k), val) }) + return cp, intent, observed, obs +} + +func evaluateVector(t *testing.T, in vectorInput) (plan.ConformancePlan, plan.ObjectVerdict) { + t.Helper() + cp, intent, observed, obs := vectorObjects(t, in) result := plan.Evaluate(intent, observed, obs, cp) applyTamper(t, &result, in.Tamper) return cp, result @@ -369,3 +375,34 @@ func TestDifferentialVerdictVectorsDetectDeliberateDivergence(t *testing.T) { t.Fatal("no success vector was mutated") } } + +// The primary entry point must give the pinned bytes too: every artifact +// vector (the corpus is unchanged), evaluated and assembled in one +// operation, matches the same pin the low-level builder is held to. +func TestDifferentialVerdictVectorsThroughTheEntryPoint(t *testing.T) { + checked := 0 + for _, name := range vectorNames(t, "verdict") { + dir := filepath.Join(corpusRoot("verdict"), name) + var exp vectorExpected + readJSON(t, filepath.Join(dir, "expected.json"), &exp) + if exp.Error != nil { + continue + } + var in vectorInput + readJSON(t, filepath.Join(dir, "input.json"), &in) + in.Tamper = nil + cp, intent, observed, obs := vectorObjects(t, in) + got, err := EvaluateAndBuild(intent, observed, obs, cp) + if err != nil { + t.Errorf("%s: %v", name, err) + continue + } + for _, m := range mismatches(t, got, exp) { + t.Errorf("%s: %s", name, m) + } + checked++ + } + if checked == 0 { + t.Fatal("no artifact vector was checked") + } +} diff --git a/go/verdict/verdict.go b/go/verdict/verdict.go index 7005e65..efbbdc4 100644 --- a/go/verdict/verdict.go +++ b/go/verdict/verdict.go @@ -17,20 +17,39 @@ // recomputed or re-derived here; // - SchemaVersion is the bare integer 1 (VERDICT-SCHEMA.md). // -// # What is checked before an artifact exists +// # Two entry points, and what each one guarantees // -// The schema says Fields holds exactly one entry per path Evaluate -// classified, and that every field is required. Build therefore refuses an -// ObjectVerdict that cannot be such a result, rather than digesting it into -// an artifact that would look authoritative: +// EvaluateAndBuild is the primary entry point. It runs plan.Evaluate and +// builds the artifact in one operation, so the plan whose digest is written +// into the artifact is, by construction, the plan that produced the verdicts, +// and the result cannot be edited in between. Use it unless you only hold a +// finished result. // -// - the Fields and Consumed path sets must be identical; -// - every plan scalar path must be in Fields; -// - Object must be what plan.Aggregate gives for Fields. +// Build is the low-level assembler for a caller that already holds a plan and +// a plan.ObjectVerdict. It checks structure only, and it is exactly as +// trustworthy as the caller who pairs the two: +// +// - it rejects a result that is not shaped like an Evaluate result for this +// plan: Fields and Consumed must name the same paths, every Fields path +// must be one the plan can produce (a planned scalar, or a planned +// collection element path), every planned scalar must be present, and +// Object must be what plan.Aggregate gives for Fields; +// - it does NOT know which plan produced the result. A result evaluated +// under one plan, paired with another plan that has the same paths and a +// different comparator, is accepted, and the second plan's digest is +// written next to the first plan's verdicts; +// - it does NOT re-evaluate, so a self-consistent but altered result (a +// changed field verdict with a matching Object, a changed consumed value) +// is accepted; +// - it cannot tell a collection element path that Evaluate produced from one +// that only has the right shape. +// +// So Build must never be described as rejecting every tampered result. Where +// the binding matters, use EvaluateAndBuild. // // A Consumed entry that is not a valid ConsumedField (Go's zero value, which -// no constructor produced) is refused the same way; Rust cannot construct -// that state at all, so this check has no Rust counterpart. +// no constructor produced) is refused as well; Rust cannot construct that +// state at all, so this check has no Rust counterpart. // // The checks run in the same order as the Rust peer and walk paths sorted // byte-wise, so both languages name the same offending path. @@ -39,10 +58,12 @@ package verdict import ( "fmt" "sort" + "strings" "github.com/CentralInfraCore/cic-primitive-engine/go/canonical" "github.com/CentralInfraCore/cic-primitive-engine/go/conformance" "github.com/CentralInfraCore/cic-primitive-engine/go/digestprojection" + "github.com/CentralInfraCore/cic-primitive-engine/go/materialized" "github.com/CentralInfraCore/cic-primitive-engine/go/plan" ) @@ -89,6 +110,27 @@ func sortedPaths[V any](m map[string]V) []string { return keys } +// planCanProduce reports whether Evaluate could have classified path under cp: +// a planned scalar path, or a collection element path, which Evaluate builds +// as "{collection}/{identity}/{element}" with a non-empty identity. +func planCanProduce(cp plan.ConformancePlan, path string) bool { + for _, fp := range cp.Scalars { + if fp.Path == path { + return true + } + } + for _, coll := range cp.Collections { + head := coll.Path + "/{" + for _, ef := range coll.Elements { + tail := "}/" + ef.Path + if len(path) > len(head)+len(tail) && strings.HasPrefix(path, head) && strings.HasSuffix(path, tail) { + return true + } + } + } + return false +} + func checkConsistent(cp plan.ConformancePlan, result plan.ObjectVerdict) error { for _, path := range sortedPaths(result.Fields) { if _, ok := result.Consumed[path]; !ok { @@ -103,6 +145,11 @@ func checkConsistent(cp plan.ConformancePlan, result plan.ObjectVerdict) error { return inconsistent(path, "consumed observation evidence is not a valid ConsumedField") } } + for _, path := range sortedPaths(result.Fields) { + if !planCanProduce(cp, path) { + return inconsistent(path, "path is not one the plan can produce (no planned scalar or collection element matches)") + } + } for _, fp := range cp.Scalars { if _, ok := result.Fields[fp.Path]; !ok { return inconsistent(fp.Path, "planned scalar path is missing from fields") @@ -115,8 +162,27 @@ func checkConsistent(cp plan.ConformancePlan, result plan.ObjectVerdict) error { return nil } -// Build assembles the verdict artifact for result, the output of -// plan.Evaluate run against cp. +// EvaluateAndBuild runs plan.Evaluate and builds the verdict artifact in one +// operation: the primary entry point. The plan digest in the artifact is the +// digest of the plan that was actually executed, and the result cannot be +// swapped or edited between evaluation and assembly. +// +// Like plan.Evaluate it panics if cp is malformed. The error is as Build's; a +// result produced here is always consistent, so in practice only a +// canonicalization error from the digest functions. +func EvaluateAndBuild( + intent, observed materialized.MaterializedObject, + obs *conformance.Observation, + cp plan.ConformancePlan, +) (Verdict, error) { + return Build(cp, plan.Evaluate(intent, observed, obs, cp)) +} + +// Build assembles the verdict artifact for result, claimed to be the output +// of plan.Evaluate run against cp. Low-level: it checks structure only and +// cannot verify that cp is the plan that produced result -- see the package +// doc for exactly what it does and does not reject, and prefer +// EvaluateAndBuild. // // Returns an *Error with code E_INCONSISTENT_VERDICT if result cannot be an // Evaluate result for cp (see the package doc), or the canonicalization error diff --git a/go/verdict/verdict_test.go b/go/verdict/verdict_test.go index fc4174e..de2431d 100644 --- a/go/verdict/verdict_test.go +++ b/go/verdict/verdict_test.go @@ -5,6 +5,7 @@ import ( "strings" "testing" + "github.com/CentralInfraCore/cic-primitive-engine/go/collection" "github.com/CentralInfraCore/cic-primitive-engine/go/conformance" "github.com/CentralInfraCore/cic-primitive-engine/go/digestprojection" "github.com/CentralInfraCore/cic-primitive-engine/go/materialized" @@ -158,3 +159,120 @@ func TestAZeroValueConsumedFieldIsRefusedNotDigested(t *testing.T) { _, err := Build(cp, result) wantInconsistent(t, err, "/shape") } + +// Counterexample 1 (review of PR #60): /ghost is in both Fields and Consumed, +// so the two path sets agree and Object follows, but the plan never asked +// for it. +func TestCounterexampleAPathOutsideThePlanIsRefused(t *testing.T) { + cp, result := sample(t) + result.Fields["/ghost"] = conformance.VerdictConformant + result.Consumed["/ghost"] = plan.NewAbsentConsumedField() + _, err := Build(cp, result) + wantInconsistent(t, err, "/ghost") +} + +func tieInputs(t *testing.T) (materialized.MaterializedObject, materialized.MaterializedObject, *conformance.Observation) { + t.Helper() + obs := conformance.NewObservation() + obs.Set("/memory_gb", conformance.CoverageObserved) + intent := objectFromValue(t, map[string]interface{}{"memory_gb": 16}, + func(_ string, val interface{}) materialized.FieldEvidence { + return materialized.NewIntentFieldEvidence(materialized.NewAuthoredIntentEvidence(val, true)) + }) + observed := objectFromValue(t, map[string]interface{}{"memory_gb": "16"}, + func(k string, val interface{}) materialized.FieldEvidence { + return materialized.NewObservationFieldEvidence(obs.Coverage("/"+k), val) + }) + return intent, observed, obs +} + +func memoryPlan(c conformance.CompareType) plan.ConformancePlan { + return plan.ConformancePlan{Scalars: []plan.FieldPlan{{Path: "/memory_gb", Compare: c}}} +} + +// Counterexample 2 (review of PR #60): the result was produced under planA +// but Build is handed planB with the same paths and a different comparator. +// The low-level builder cannot tell: it accepts and writes planB's digest +// next to planA's verdicts. This is the documented limit of Build, pinned +// here so it cannot be mistaken for a guarantee. +func TestCounterexampleTheLowLevelBuilderDoesNotBindThePlan(t *testing.T) { + intent, observed, obs := tieInputs(t) + planA := memoryPlan(conformance.CompareNumeric) + planB := memoryPlan(conformance.CompareExact) + resultA := plan.Evaluate(intent, observed, obs, planA) + + mixed, err := Build(planB, resultA) + if err != nil { + t.Fatalf("structure alone looks fine: %v", err) + } + digestA, _ := digestprojection.ConformancePlanDigest(planA) + digestB, _ := digestprojection.ConformancePlanDigest(planB) + if mixed.ConformancePlanDigest != digestB { + t.Fatal("the artifact should carry the digest of the plan it was handed") + } + if mixed.ConformancePlanDigest == digestA { + t.Fatal("the artifact names the plan that produced its verdicts: not the counterexample") + } +} + +// The answer to counterexample 2: through the primary entry point the plan +// and the result cannot be paired independently, so each plan's digest sits +// next to the verdicts that plan actually produced. +func TestCounterexampleTheEntryPointBindsThePlanToItsResult(t *testing.T) { + intent, observed, obs := tieInputs(t) + planA := memoryPlan(conformance.CompareNumeric) + planB := memoryPlan(conformance.CompareExact) + a, err := EvaluateAndBuild(intent, observed, obs, planA) + if err != nil { + t.Fatal(err) + } + b, err := EvaluateAndBuild(intent, observed, obs, planB) + if err != nil { + t.Fatal(err) + } + digestA, _ := digestprojection.ConformancePlanDigest(planA) + digestB, _ := digestprojection.ConformancePlanDigest(planB) + if a.ConformancePlanDigest != digestA || b.ConformancePlanDigest != digestB || digestA == digestB { + t.Fatal("each artifact must carry the digest of the plan that produced it") + } + // The comparator really changes the verdict, so the digest is not decoration. + if a.Fields["/memory_gb"] == b.Fields["/memory_gb"] { + t.Fatalf("the two plans should classify differently, both gave %s", a.Fields["/memory_gb"]) + } + // Each artifact is exactly what the low-level builder gives for the + // result of the SAME plan. + direct, err := Build(planB, plan.Evaluate(intent, observed, obs, planB)) + if err != nil { + t.Fatal(err) + } + gotBytes, _ := b.CanonicalBytes() + wantBytes, _ := direct.CanonicalBytes() + if string(gotBytes) != string(wantBytes) { + t.Fatal("EvaluateAndBuild differs from Build over the same plan's result") + } +} + +func TestCollectionElementPathsAreMatchedByShapeAndForeignOnesRefused(t *testing.T) { + cp := plan.ConformancePlan{Collections: []plan.CollectionPlan{{ + Path: "/nics", + Collection: collection.Collection{ + Topology: collection.TopologyMap, + Keys: []string{"name"}, + }, + Elements: []plan.FieldPlan{{Path: "subnet", Compare: conformance.CompareExact}}, + }}} + with := func(path string) plan.ObjectVerdict { + return plan.ObjectVerdict{ + Object: plan.ObjectConformant, + Fields: map[string]conformance.FieldVerdict{path: conformance.VerdictConformant}, + Consumed: map[string]plan.ConsumedField{path: plan.NewAbsentConsumedField()}, + } + } + if _, err := Build(cp, with("/nics/{name=a}/subnet")); err != nil { + t.Fatalf("a path of the planned shape is accepted: %v", err) + } + for _, foreign := range []string{"/nics/{name=a}/other", "/other/{name=a}/subnet", "/nics/{}/subnet", "/nics/subnet"} { + _, err := Build(cp, with(foreign)) + wantInconsistent(t, err, foreign) + } +}