From c59f0a844f0779cbe02d37dff850334271e08145 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?G=C3=A1bor=20Z=20Sink=C3=B3?= Date: Sat, 14 Nov 2026 21:31:33 +0100 Subject: [PATCH 1/2] ci(measurement): object-level evidence model (report-only), mutation-policy/2 Global percentages do not say which production function got which check. This adds an object-level, verifiable measurement model. It is report-only: nothing gates on it, nothing is skipped because of it, no threshold or baseline changes. mutation-policy/2 (owner decisions): the primary Rust score is caught/(caught+missed); an unviable mutant is neither caught nor missed and is a mandatory metric of its own (count and ratio); missed and timeout still fail, unviable still does not. A zero denominator is neither 100% nor 0%: NOT_APPLICABLE (NO_MUTANTS_PLANNED, ALL_UNVIABLE, NO_EVALUABLE_MUTANT) or UNVERIFIABLE (MEASUREMENT_MISSING). The old caught/planned value stays printed as `legacy`. scripts/object_model.py (object-id/1, object-evidence/1): - Go objects from tools/objinventory (new, stdlib-only, separate module like goastdump): exact function spans. A cover block is attributed only if it lies completely inside one object; a block that cuts an object, spans several, or lies in none is counted and the affected fields are UNVERIFIABLE. Nothing is split proportionally. Build-tag variants are separate objects. Repeated profiles mark an object whose coverage differs as NONDETERMINISTIC. - Rust objects from the real cargo-llvm-cov export, named with the binutils c++filt of the pinned image (recorded in the record). Instances (generics, closures, per-binary copies) fold into one source object; the instance detail is kept. Test code is found by tokenizing the #[cfg(test)] item span, not by "everything after the first #[cfg(test)]" (that dropped production code: measured on ffi/src/lib.rs, caught by the L3 check below). - Mutation verdicts are mapped to objects by file and the last segment of the (module-qualified) function name; a mutant that fits two objects is ambiguous and counted, never guessed. - Every evidence field is MEASURED, NOT_APPLICABLE or UNVERIFIABLE with a coded reason. The coverage kind is statement, line or region: there is no "branch" (stable rust has none). - The record is canonical and digested; the offline verifier reports three separate levels: L1 INTEGRITY_OK, L2 BOUND_TO_INPUTS (NOT_CHECKED without inputs), L3 COMPLETE_FOR_SCOPE (PARTIAL where only a part can be shown). There is no combined "verified". .measurement-scope.json (measurement-scope/1): every tracked Go/Rust file is production, test, test-data, experimental, a separate module or a tool, each exclusion with a reason; an unclassified file is an error. tools/goastdump is measured and reported on its own. .github/workflows/object-evidence.yml: report-only; not a required check; nothing needs it. verify.yml: the objinventory module is vetted and tested in the python job. mutation_plan.py: object_model.py and the scope file are gate logic. Tests: 43 on real tool output (N-group: new contracts) plus the policy tests (R-group). The N tests were shown to have teeth by removing each check on a scratch copy. --- [signing-metadata] key = cic-my-sign-key signature = vault:v1:MEYCIQChB8E3dos3hN3/2pwcRYLhIyYTDu8ON37MEiAZZZxZSgIhAO6zWiuAfaKju9UISjk0ub0i3xzhNj1QY1UP+e2jFnba hash-algorithm = sha256 digest = LvHUz6xpK/g717aNSbrrW9tk5HxOXimd8tArCeFD6MI= [certificate] -----BEGIN CERTIFICATE----- MIICBjCCAaygAwIBAgIUSnRMR6RPnEbg296XWPOqq/u5PCwwCgYIKoZIzj0EAwIw QzELMAkGA1UEBhMCSFUxGTAXBgNVBAoMEENlbnRyYWxJbmZyYUNvcmUxGTAXBgNV BAMMEENJQyBEZXZlbG9wZXIgQ0EwHhcNMjYwMzIwMTMyMjU5WhcNMjYxMjMxMTMy MjU5WjBFMQswCQYDVQQGEwJIVTEZMBcGA1UECgwQQ2VudHJhbEluZnJhQ29yZTEb MBkGA1UEAwwSR2Fib3IgWm9sdGFuIFNpbmtvMFkwEwYHKoZIzj0CAQYIKoZIzj0D AQcDQgAEIG2CVmTfmLB9pLLclj7YmP2eedAjklpy4LGrU2ijoiy6Xqpuybv7OgJe i+ez31s65NEV8+X/ByeX1cstR988z6N8MHowCQYDVR0TBAIwADAdBgNVHQ4EFgQU yZN6AIX/TNnIJ9GwAa/NRN3ujHAwHwYDVR0jBBgwFoAUXn6CHYzPUqU4JVP8g+OS WeDYjhcwDgYDVR0PAQH/BAQDAgeAMB0GA1UdJQQWMBQGCCsGAQUFBwMCBggrBgEF BQcDBDAKBggqhkjOPQQDAgNIADBFAiEA+bFzXRoJ4PCQbhAAtpkcMjt0vNj5rEW0 lOMBGDNyaWkCIB1vmM7PcZzv/c9bIrxF5kqv6QXomouhByUfeNUTbpKW -----END CERTIFICATE----- --- .github/workflows/object-evidence.yml | 110 + .github/workflows/verify.yml | 8 + .measurement-scope.json | 18 + scripts/mutation_plan.py | 2 + scripts/object_model.py | 1153 +++++++ scripts/quality_gate.py | 80 +- .../fixtures/object_model/README.md | 11 + .../object_model/cic-canonical-lib.rs | 118 + .../object_model/demangled-cic-canonical.json | 9 + .../fixtures/object_model/goastdump-cover.out | 135 + .../object_model/goastdump-inventory.json | 199 ++ .../fixtures/object_model/goastdump-main.go | 408 +++ .../object_model/llvm-cov-cic-canonical.json | 2745 +++++++++++++++++ .../object_model/mutants-cic-canonical.json | 343 ++ .../object_model/outcomes-cic-canonical.json | 446 +++ tests/test_tools/test_mutation_policy.py | 211 ++ .../test_object_evidence_workflow.py | 116 + tests/test_tools/test_object_model.py | 823 +++++ tests/test_tools/test_quality_gate.py | 2 +- tools/objinventory/go.mod | 3 + tools/objinventory/main.go | 148 + tools/objinventory/main_test.go | 134 + 22 files changed, 7211 insertions(+), 11 deletions(-) create mode 100644 .github/workflows/object-evidence.yml create mode 100644 .measurement-scope.json create mode 100644 scripts/object_model.py create mode 100644 tests/test_tools/fixtures/object_model/README.md create mode 100644 tests/test_tools/fixtures/object_model/cic-canonical-lib.rs create mode 100644 tests/test_tools/fixtures/object_model/demangled-cic-canonical.json create mode 100644 tests/test_tools/fixtures/object_model/goastdump-cover.out create mode 100644 tests/test_tools/fixtures/object_model/goastdump-inventory.json create mode 100644 tests/test_tools/fixtures/object_model/goastdump-main.go create mode 100644 tests/test_tools/fixtures/object_model/llvm-cov-cic-canonical.json create mode 100644 tests/test_tools/fixtures/object_model/mutants-cic-canonical.json create mode 100644 tests/test_tools/fixtures/object_model/outcomes-cic-canonical.json create mode 100644 tests/test_tools/test_mutation_policy.py create mode 100644 tests/test_tools/test_object_evidence_workflow.py create mode 100644 tests/test_tools/test_object_model.py create mode 100644 tools/objinventory/go.mod create mode 100644 tools/objinventory/main.go create mode 100644 tools/objinventory/main_test.go diff --git a/.github/workflows/object-evidence.yml b/.github/workflows/object-evidence.yml new file mode 100644 index 0000000..ca778a6 --- /dev/null +++ b/.github/workflows/object-evidence.yml @@ -0,0 +1,110 @@ +name: object-evidence + +# REPORT-ONLY. Builds the object-level evidence record (scripts/object_model.py, object-evidence/1): +# which production function / method got which coverage, which mutation verdicts, and which fields are +# UNVERIFIABLE and why. It then verifies the record at three separate levels (L1 integrity, L2 binding to +# the inputs, L3 completeness against an independent enumeration). It is NOT a required check, it gates +# nothing else, and nothing is skipped because of it. The coverage kind is statement (Go) or region (Rust): +# there is no branch coverage here (the stable toolchain gives none), and nothing may be named so. +# The mutation verdicts are not run here (they take a FULL run); the field is UNVERIFIABLE +# (MEASUREMENT_MISSING) in this record and the PR-03 completeness check owns that evidence. +on: + pull_request: + branches: ["**"] + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: object-evidence-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +env: + FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true + +jobs: + object-evidence: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + ref: ${{ github.event.pull_request.head.sha || github.sha }} + + - name: Setup Go + uses: actions/setup-go@v5 + with: + go-version: "1.26.11" + + - name: Install pinned Rust toolchain (rust-toolchain.toml) + run: | + set -euo pipefail + rustup show active-toolchain || rustup toolchain install + rustup component add llvm-tools-preview + + - name: Install cargo-llvm-cov and cargo-mutants + uses: taiki-e/install-action@v2 + with: + tool: cargo-llvm-cov,cargo-mutants + + - name: Measure (real tools, one directory of inputs) + env: + T: ${{ runner.temp }}/object-evidence + run: | + set -euo pipefail + mkdir -p "$T" + (cd tools/objinventory && go build -o "$T/objinventory" .) + # Go, root module: the production files of the scope, the cover profile of the whole module + python3 scripts/object_model.py scope-list --root . --scope .measurement-scope.json --lang go --class production > "$T/go-files.txt" + "$T/objinventory" $(cat "$T/go-files.txt") > "$T/go-inventory.json" + go test -count=1 -covermode=set -coverprofile="$T/go-cover.out" ./... + # Go, the separate goastdump module: its own inventory and profile + (cd tools/goastdump && "$T/objinventory" main.go > "$T/goastdump-inventory.json" \ + && go test -count=1 -covermode=set -coverprofile="$T/goastdump-cover.out" ./...) + # Rust: the function-level export and the mutant listing (the exit status is checked by `set -e`) + cargo llvm-cov --all --locked --json --output-path "$T/llvm-cov.json" + cargo mutants --workspace --list --json > "$T/rust-list.json" + { + echo "go=$(go version | awk '{print $3}')" + echo "rustc=$(rustc --version | awk '{print $2}')" + echo "cargo-llvm-cov=$(cargo llvm-cov --version | awk '{print $2}')" + echo "cargo-mutants=$(cargo mutants --version | awk '{print $2}')" + echo "objinventory=objinventory/1" + } > "$T/tools.txt" + cat "$T/tools.txt" + + - name: Build the evidence record + env: + T: ${{ runner.temp }}/object-evidence + run: | + set -euo pipefail + tools=() + while IFS= read -r line; do tools+=(--tool "$line"); done < "$T/tools.txt" + python3 scripts/object_model.py build --root . --scope .measurement-scope.json --out "$T/evidence.json" \ + --go-unit "root,.,$T/go-inventory.json,$T/go-cover.out" \ + --go-unit "goastdump,tools/goastdump,$T/goastdump-inventory.json,$T/goastdump-cover.out,detail" \ + --llvm-cov "$T/llvm-cov.json" --rust-list "$T/rust-list.json" --rust-prefix "$PWD/" "${tools[@]}" + + - name: Verify the record (L1 integrity, L2 binding to the inputs, L3 completeness) + env: + T: ${{ runner.temp }}/object-evidence + run: | + set -euo pipefail + python3 scripts/object_model.py verify --record "$T/evidence.json" --root . --scope .measurement-scope.json \ + --rust-list "$T/rust-list.json" \ + --input "go-inventory:root=$T/go-inventory.json" --input "go-coverprofile:root=$T/go-cover.out" \ + --input "go-inventory:goastdump=$T/goastdump-inventory.json" --input "go-coverprofile:goastdump=$T/goastdump-cover.out" \ + --input "llvm-cov-json=$T/llvm-cov.json" --input "rust-mutant-list=$T/rust-list.json" \ + --input "scope=.measurement-scope.json" | tee "$T/verify.txt" + + - name: Keep the record + if: ${{ always() }} + uses: actions/upload-artifact@v4 + with: + name: object-evidence + path: | + ${{ runner.temp }}/object-evidence/evidence.json + ${{ runner.temp }}/object-evidence/verify.txt + ${{ runner.temp }}/object-evidence/tools.txt + if-no-files-found: warn diff --git a/.github/workflows/verify.yml b/.github/workflows/verify.yml index bfcf7c9..e8bd919 100644 --- a/.github/workflows/verify.yml +++ b/.github/workflows/verify.yml @@ -246,6 +246,14 @@ jobs: GO_META_REQUIRE_GO: "1" run: python -m pytest + - name: objinventory (Go function spans of the object-level measurement) + # Separate module (tools/objinventory/go.mod), like goastdump: not covered by the root + # `go test ./...`, so it is tested here. Report-only tooling: nothing gates on its output. + working-directory: tools/objinventory + run: | + go vet ./... + go test ./... + - name: goastdump (go/ast front end of go.meta.gen.py) # Separate module (tools/goastdump/go.mod): not covered by the root # `go test ./...`, so it is tested here. diff --git a/.measurement-scope.json b/.measurement-scope.json new file mode 100644 index 0000000..46bc7e6 --- /dev/null +++ b/.measurement-scope.json @@ -0,0 +1,18 @@ +{ + "schema": "measurement-scope/1", + "version": 1, + "comment": "Which files the object-level measurement covers. The first matching rule decides; a tracked Go or Rust file that matches no rule is an error (nothing is left out silently). A change of this file is a change of the gate logic and is reviewed as such.", + "rules": [ + {"lang": "go", "match": "*_test.go", "class": "test", "reason": "test code is not a production object"}, + {"lang": "go", "match": "*/testdata/*", "class": "test-data", "reason": "fixtures read by tests; Go tooling ignores testdata"}, + {"lang": "go", "match": "tools/goastdump/*", "class": "separate", "unit": "goastdump", "reason": "a separate Go module (tools/goastdump/go.mod): measured and reported on its own, never averaged into the root module (owner decision 2026-11-14)"}, + {"lang": "go", "match": "tools/objinventory/*", "class": "tool", "unit": "objinventory", "reason": "the measuring tool itself (a separate Go module); it is tested in the python job and is not an object of the Relay"}, + {"lang": "go", "match": "tools/statechain-demo/*", "class": "experimental", "reason": "live demo entry, exercised by make demo; already excluded from the coverage floors in verify.yml"}, + {"lang": "go", "match": "tools/trustflow-demo/*", "class": "experimental", "reason": "live cgo demo entry, exercised by make demo-trustflow; excluded from the coverage floors in verify.yml"}, + {"lang": "go", "match": "tools/oci-sign-demo/*", "class": "experimental", "reason": "self-verifying demo entry, exercised by make demo-oci-sign; excluded from the coverage floors in verify.yml"}, + {"lang": "go", "match": "*.go", "class": "production", "reason": "every other Go file of the root module"}, + {"lang": "rust", "match": "*/tests/*", "class": "test", "reason": "integration tests are not production objects"}, + {"lang": "rust", "match": "*/benches/*", "class": "test", "reason": "benchmarks are not production objects"}, + {"lang": "rust", "match": "*.rs", "class": "production", "reason": "every other Rust source (a #[cfg(test)] region inside a file is excluded by the object model, counted, not dropped)"} + ] +} diff --git a/scripts/mutation_plan.py b/scripts/mutation_plan.py index 94f0691..3a20af6 100755 --- a/scripts/mutation_plan.py +++ b/scripts/mutation_plan.py @@ -82,6 +82,8 @@ def changed_files(base: str, head: str) -> list[str]: "scripts/baseline_guard.py", "scripts/count_mutants.py", "scripts/quality_gate.py", + "scripts/object_model.py", + ".measurement-scope.json", ".mutation-baseline.json", ".github/workflows/mutation.yml", ".github/workflows/mutation-pr.yml", diff --git a/scripts/object_model.py b/scripts/object_model.py new file mode 100644 index 0000000..1e03b6a --- /dev/null +++ b/scripts/object_model.py @@ -0,0 +1,1153 @@ +#!/usr/bin/env python3 +"""Object-level measurement model (report-only): what check did each production object get? + +Global percentages say how much of a package ran. They do not say, per function or method, +which coverage and which mutation verdicts it has, or that for some objects nothing can be +said. This module builds that model and a canonical, digested evidence record of it, and a +verifier that states what it proves and what it does not. + +Contracts (versioned; a change is a change of the gate logic, planned as such by mutation_plan.py) + object-id/1 the key of an object (see go_object_id / rust_object_key) + object-evidence/1 the record (build_record) and its three verification levels (verify_record) + measurement-scope/1 which files are production, test, generated, experimental, measured apart + +Every evidence field of every object is exactly one of + MEASURED a measured value + NOT_APPLICABLE nothing to measure, for a stated reason + UNVERIFIABLE it cannot be established, with a coded reason: never a hidden success + +The coverage KIND is part of the data and is one of statement | line | region. There is no +"branch": the stable toolchain gives none (measured: `cargo llvm-cov --branch` needs nightly), so +nothing here may be called branch coverage. + +Attribution is never estimated. A Go cover block belongs to an object only if it lies completely +inside that object's span; a block that cuts an object, spans several, or lies in none is counted +and makes the affected fields UNVERIFIABLE. No value is ever split proportionally. + +Verification levels (never merged into one "verified"): + L1 INTEGRITY_OK schema, digest, totals recount, enums: the record is internally consistent + L2 BOUND_TO_INPUTS the commit/tree and the input files are the ones the record names + (NOT_CHECKED without them) + L3 COMPLETE_FOR_SCOPE the object list is complete against an independent enumeration + (PARTIAL where only a part can be shown) +A digest does not prove where a measurement came from: that the tools really ran on that tree +is not shown by any of the three levels. +""" + +import argparse +import fnmatch +import hashlib +import json +import re +import subprocess +import sys +from pathlib import Path + +import quality_gate as qg + +ID_CONTRACT = "object-id/1" +EVIDENCE_SCHEMA = "object-evidence/1" +SCOPE_SCHEMA = "measurement-scope/1" + +MEASURED = "MEASURED" +NOT_APPLICABLE = "NOT_APPLICABLE" +UNVERIFIABLE = "UNVERIFIABLE" +STATES = (MEASURED, NOT_APPLICABLE, UNVERIFIABLE) +COVERAGE_KINDS = ( + "statement", + "line", + "region", +) # no "branch": see the module docstring + +REASONS = { + "FILE_NOT_IN_PROFILE", # the file has no block in the coverage profile (not built / not tested) + "PARTIAL_BLOCK", # a cover block cuts the object: not attributable, no value is made up + "AMBIGUOUS_MAPPING", # an instance or mutant fits several objects (or none): never a guess + "NONDETERMINISTIC", # repeated measurements disagree + "TOOL_LIMIT", # the available tool does not provide it + "NO_MAPPING", # no way to connect it to this object + "NO_STATEMENTS", # the object has no executable statement + "NO_MUTANTS_LISTED", # the mutation tool lists no mutant for this function + "NO_MUTANTS_PLANNED", + "ALL_UNVIABLE", + "NO_EVALUABLE_MUTANT", + "MEASUREMENT_MISSING", +} +FIELDS = ("coverage", "mutation", "tests") + + +class ModelError(Exception): + """The input cannot be trusted: no record is built / the record is rejected.""" + + +def field(state: str, **kw) -> dict: + return {"state": state, **kw} + + +def unverifiable(reason: str, **kw) -> dict: + return field(UNVERIFIABLE, reason=reason, **kw) + + +def not_applicable(reason: str, **kw) -> dict: + return field(NOT_APPLICABLE, reason=reason, **kw) + + +def sha256_file(path: str) -> tuple[str, int]: + data = Path(path).read_bytes() + return hashlib.sha256(data).hexdigest(), len(data) + + +# --------------------------------------------------------------------------- measurement scope + + +def classify(scope: dict, lang: str, path: str) -> tuple[str, dict]: + """The first matching rule of the scope decides: (class, rule). No match is an error: a file the + scope does not classify is neither measured nor silently left out.""" + for rule in scope["rules"]: + if rule["lang"] == lang and fnmatch.fnmatch(path, rule["match"]): + return rule["class"], rule + raise ModelError( + f"{path}: not classified by the measurement scope (neither included nor excluded with a reason)" + ) + + +def check_scope(scope: dict) -> None: + """The scope file itself: schema, versioned, every exclusion has a reason.""" + if scope.get("schema") != SCOPE_SCHEMA or not isinstance(scope.get("version"), int): + raise ModelError(f"the scope is not a {SCOPE_SCHEMA} with an integer version") + classes = { + "production", + "test", + "test-data", + "generated", + "experimental", + "separate", + "tool", + } + for i, rule in enumerate(scope.get("rules", [])): + if ( + rule.get("class") not in classes + or not rule.get("match") + or rule.get("lang") not in ("go", "rust") + ): + raise ModelError(f"scope rule #{i} is malformed: {rule}") + if rule["class"] != "production" and not (rule.get("reason") or "").strip(): + raise ModelError( + f"scope rule #{i} ({rule['match']}) excludes without a reason" + ) + + +def scope_files(scope: dict, lang: str, tracked: list[str]) -> dict[str, list[str]]: + """Group the tracked files of a language by scope class; an unclassified file is an error.""" + out: dict[str, list[str]] = {} + for f in tracked: + if not f.endswith(".go" if lang == "go" else ".rs"): + continue + cls, rule = classify(scope, lang, f) + out.setdefault( + cls if cls != "separate" else f"separate:{rule.get('unit', '?')}", [] + ).append(f) + return out + + +# --------------------------------------------------------------------------- Go + + +def go_module_path(gomod_text: str) -> str: + m = re.search(r"^module\s+(\S+)", gomod_text, re.M) + if not m: + raise ModelError("go.mod has no module line") + return m.group(1) + + +_BLOCK = re.compile(r"^(.+):(\d+)\.(\d+),(\d+)\.(\d+) (\d+) (\d+)$") + + +def parse_coverprofile(text: str) -> list[tuple]: + """The blocks (file, sl, sc, el, ec, nstmt, count) of a Go cover profile, merged with `set` + semantics (a block that appears more than once is covered if any occurrence is).""" + lines = text.splitlines() + if not lines or not re.match(r"^mode: (set|count|atomic)$", lines[0]): + raise ModelError("the coverage profile has no valid mode line") + merged: dict[tuple, list[int]] = {} + for ln in lines[1:]: + if not ln.strip(): + continue + m = _BLOCK.match(ln) + if not m: + raise ModelError(f"malformed cover block: {ln!r}") + f, sl, sc, el, ec, n, c = m.group(1), *map(int, m.groups()[1:]) + cur = merged.setdefault((f, sl, sc, el, ec), [n, 0]) + if cur[0] != n: + raise ModelError(f"the same block has two statement counts: {ln!r}") + cur[1] = max(cur[1], c) + if not merged: + raise ModelError("the coverage profile holds no blocks") + return [(*k, v[0], v[1]) for k, v in sorted(merged.items())] + + +def go_object_id(module: str, file: str, fn: dict, build: str = "") -> str: + """object-id/1 for Go: go:. | .. | .init##, plus + @build() when the file carries one. The position is NOT part of the key (an edit + above the function does not rename it).""" + d = file.rsplit("/", 1)[0] if "/" in file else "" + pkg = module + ("/" + d if d else "") + if fn["kind"] == "init": + name = f"init#{file.rsplit('/', 1)[-1]}#{fn['ord']}" + elif fn["kind"] == "method": + name = f"{fn['recv']}.{fn['name']}" + else: + name = fn["name"] + key = f"go:{pkg}.{name}" + return key + (f"@build({' '.join(build.split())})" if build else "") + + +def _pos(line: int, col: int) -> tuple[int, int]: + return (line, col) + + +def _attribute(blocks: list[tuple], funcs: list[tuple]) -> tuple[dict, dict]: + """Attribute the blocks of ONE file to its functions. + + funcs: [(key, start(line,col), end(line,col))]. Returns ({key: [blocks]}, stats) where stats + counts the blocks that could not be attributed: partial (cuts an object or spans several) and + unattributed (inside no object). A block belongs to an object only if it lies completely inside. + """ + got: dict[str, list[tuple]] = {k: [] for k, _, _ in funcs} + partial_for: dict[str, int] = {k: 0 for k, _, _ in funcs} + stats = {"attributed": 0, "partial": 0, "unattributed": 0} + for b in blocks: + bs, be = _pos(b[1], b[2]), _pos(b[3], b[4]) + inside = [k for k, fs, fe in funcs if fs <= bs and be <= fe] + touching = [k for k, fs, fe in funcs if not (be <= fs or bs >= fe)] + if len(inside) == 1 and touching == inside: + got[inside[0]].append(b) + stats["attributed"] += 1 + elif touching: # cuts an object, or lies in several + stats["partial"] += 1 + for k in touching: + partial_for[k] += 1 + else: + stats["unattributed"] += 1 + return {"blocks": got, "partial": partial_for}, stats + + +def go_objects( + unit: str, + module: str, + inventory: dict, + blocks: list[tuple], + prefix: str = "", + repeats: list[list[tuple]] | None = None, + detail: bool = False, +) -> tuple[list[dict], dict]: + """The Go objects of one measurement unit (a module) and the attribution statistics. + + prefix: the profile path of a file is module + "/" + relative path; for a nested module the + inventory paths are relative to it. repeats: further profiles of the same tree; an object whose + covered count differs between them is NONDETERMINISTIC. detail: keep the uncovered spans. + """ + by_file: dict[str, list[tuple]] = {} + for b in blocks: + by_file.setdefault(b[0], []).append(b) + objects: list[dict] = [] + stats = { + "attributed": 0, + "partial": 0, + "unattributed": 0, + "profile_files_not_in_inventory": 0, + } + inv_files = set() + seen_ids: dict[str, str] = {} + for fdump in inventory["files"]: + if fdump.get("error"): + raise ModelError( + f"{fdump['file']}: the inventory could not parse it: {fdump['error']}" + ) + rel = fdump["file"] + inv_files.add(module + "/" + rel) + funcs = [] + for fn in fdump["funcs"]: + oid = go_object_id(module, rel, fn, fdump.get("build", "")) + if oid in seen_ids: + raise ModelError( + f"object id collision: {oid!r} ({seen_ids[oid]} and {rel})" + ) + seen_ids[oid] = rel + funcs.append( + ( + oid, + _pos(fn["start"]["line"], fn["start"]["col"]), + _pos(fn["end"]["line"], fn["end"]["col"]), + fn, + ) + ) + file_blocks = by_file.get(module + "/" + rel, []) + res, st = _attribute(file_blocks, [(k, s, e) for k, s, e, _ in funcs]) + for k in stats: + if k in st: + stats[k] += st[k] + in_profile = bool(file_blocks) + for oid, s, e, fn in funcs: + mine = res["blocks"][oid] + cov: dict + if not in_profile: + cov = unverifiable("FILE_NOT_IN_PROFILE") + elif res["partial"][oid]: + cov = unverifiable("PARTIAL_BLOCK", blocks=res["partial"][oid]) + elif not mine: + cov = not_applicable("NO_STATEMENTS") + else: + total = sum(b[5] for b in mine) + covered = sum(b[5] for b in mine if b[6] > 0) + cov = field(MEASURED, kind="statement", total=total, covered=covered) + if repeats: + seen = {covered} + for rep in repeats: + r_by = {b[:5]: b for b in rep} + seen.add( + sum(b[5] for b in mine if r_by.get(b[:5], (0,) * 7)[6] > 0) + ) + if len(seen) > 1: + cov = unverifiable( + "NONDETERMINISTIC", + range=[min(seen), max(seen)], + total=total, + ) + if detail and cov["state"] == MEASURED: + cov["uncovered"] = [ + [b[1], b[2], b[3], b[4]] for b in mine if b[6] == 0 + ] + objects.append( + { + "id": oid, + "lang": "go", + "unit": unit, + "kind": fn["kind"], + "name": fn["name"], + "file": rel, + "span": {"start": s[0], "end": e[0]}, + "coverage": cov, + "mutation": unverifiable( + "TOOL_LIMIT", + note="gremlins prints per-mutant lines, they are not mapped", + ), + "tests": unverifiable("TOOL_LIMIT", note="no per-test attribution"), + } + ) + stats["profile_files_not_in_inventory"] = len( + {f for f in by_file if f not in inv_files} + ) + return objects, stats + + +# --------------------------------------------------------------------------- Rust + +_CRATE_DISAMBIGUATOR = re.compile(r"\[[0-9a-f]+\]") +_CLOSURE_TAIL = re.compile(r"::\{(?:closure|async block|async fn body)#\d+\}$") + + +def _strip_generic_suffix(name: str) -> str: + """`a::f::` -> `a::f`: an instantiation belongs to its generic function.""" + while name.endswith(">"): + depth = 0 + for i in range(len(name) - 1, -1, -1): + c = name[i] + depth += (c == ">") - (c == "<") + if depth == 0: + if name[max(0, i - 2) : i] == "::": + name = name[: i - 2] + break + return name + else: + return name + return name + + +def rust_object_key(demangled: str) -> str: + """object-id/1 for Rust: rust:, with the per-binary crate disambiguators + ([0], [1] ...) removed, a closure folded into the function that contains it, and a generic + instantiation folded into its generic function. The file and the span are attributes, not the key. + """ + n = _CRATE_DISAMBIGUATOR.sub("", demangled) + for _ in range(8): + before = n + n = _CLOSURE_TAIL.sub("", n) + n = _strip_generic_suffix(n) + if n == before: + break + return "rust:" + n + + +def rust_name(key: str) -> str: + """The last path segment of a key: the function name the mutation tool also reports.""" + body = key[len("rust:") :] + body = re.sub(r"<[^<>]*>$", "", body) + return re.split(r"::", body)[-1] + + +def _scan(text: str): + """Yield (index, char, line) for the characters of a Rust source that are CODE: comments, string + literals (incl. raw strings) and char literals are skipped, a lifetime is not a char literal. + """ + i, n, line = 0, len(text), 1 + while i < n: + c = text[i] + two = text[i : i + 2] + if two == "//": + while i < n and text[i] != "\n": + i += 1 + continue + if two == "/*": + depth, i = 1, i + 2 + while i < n and depth: + if text[i : i + 2] == "/*": + depth, i = depth + 1, i + 2 + elif text[i : i + 2] == "*/": + depth, i = depth - 1, i + 2 + else: + line += text[i] == "\n" + i += 1 + continue + m = re.match(r'b?r(#*)"', text[i:]) + if m: # raw string: ends at "### + end = text.find('"' + m.group(1), i + m.end()) + end = n if end < 0 else end + 1 + len(m.group(1)) + line += text.count("\n", i, end) + i = end + continue + if c == '"': + i += 1 + while i < n and text[i] != '"': + if text[i] == "\\": + i += 1 + line += text[i] == "\n" if i < n else 0 + i += 1 + i += 1 + continue + if c == "'": + # a char literal ('x', '\\n', '\\u{..}') ends with a quote within a few chars; a lifetime does not + m = re.match( + r"'(?:\\(?:u\{[0-9a-fA-F]+\}|x[0-9a-fA-F]{2}|.)|[^'\\])'", text[i:] + ) + if m: + i += m.end() + continue + if c == "\n": + line += 1 + yield i, c, line + i += 1 + + +def cfg_test_spans(text: str) -> list[tuple[int, int]]: + """The (first line, last line) of every item that carries #[cfg(test)]: a `mod tests { .. }`, a + single test-only function, or a `use`/`mod x;` ending at its `;`. Production code that merely FOLLOWS + such an item is not test code (a first-#[cfg(test)]-to-the-end rule would drop it: measured on + ffi/src/lib.rs, where a production function follows a #[cfg(test)] item in the middle of the file). + """ + spans: list[tuple[int, int]] = [] + code = list(_scan(text)) + attr = re.compile(r"#\[cfg\(test\)\]") + for m in attr.finditer(text): + pos = m.start() + start_line = text.count("\n", 0, pos) + 1 + depth, opened, end_line = 0, False, None + for i, c, ln in code: + if i < m.end(): + continue + if c == "{": + depth, opened = depth + 1, True + elif c == "}" and opened: + depth -= 1 + if depth == 0: + end_line = ln + break + elif c == ";" and not opened: + end_line = ln + break + if end_line is None: + raise ModelError( + f"cannot find the end of the #[cfg(test)] item at line {start_line}" + ) + spans.append((start_line, end_line)) + return spans + + +def _regions(fn: dict) -> list[list[int]]: + """The code regions of an llvm-cov function entry (kind 0).""" + return [r for r in fn.get("regions", []) if len(r) >= 8 and r[7] == 0] + + +def rust_objects( + llvm: dict, + demangled: dict[str, str], + root: str | None, + mutants: list[dict] | None, + verdicts: dict[str, str] | None, + prefix: str = "/w/", +) -> tuple[list[dict], dict]: + """The Rust objects from a real `cargo llvm-cov --json` export, the demangled names of its + symbols, the cargo-mutants listing and the verdicts of the run. + + Test code is not a production object: a file under a tests/ or benches/ directory, or a function + inside an item that carries #[cfg(test)] (see cfg_test_spans), is counted as excluded (never silently + dropped). + """ + data = llvm["data"][0] + cfg_spans: dict[str, list[tuple[int, int]]] = {} + groups: dict[str, list[dict]] = {} + excluded = {"test_code": 0, "no_name": 0} + for fn in data["functions"]: + sym = fn["name"] + if sym not in demangled: + excluded["no_name"] += 1 + raise ModelError( + f"no demangled name for symbol {sym!r}: the object cannot be named" + ) + file = fn["filenames"][0] + rel = file[len(prefix) :] if file.startswith(prefix) else file + regs = _regions(fn) + first_line = min((r[0] for r in regs), default=0) + parts = rel.split("/") + is_test = "tests" in parts[:-1] or "benches" in parts[:-1] + if not is_test and root and rel.endswith(".rs"): + if rel not in cfg_spans: + p = Path(root) / rel + cfg_spans[rel] = ( + cfg_test_spans(p.read_text(encoding="utf-8")) if p.is_file() else [] + ) + is_test = any(a <= first_line <= b for a, b in cfg_spans[rel]) + if is_test: + excluded["test_code"] += 1 + continue + key = rust_object_key(demangled[sym]) + groups.setdefault(key, []).append( + { + "symbol": sym, + "file": rel, + "regions": regs, + "count": fn.get("count", 0), + "demangled": demangled[sym], + } + ) + + objects: list[dict] = [] + for key in sorted(groups): + inst = groups[key] + files = {i["file"] for i in inst} + total = sum(len(i["regions"]) for i in inst) + covered = sum(1 for i in inst for r in i["regions"] if r[4] > 0) + starts = [r[0] for i in inst for r in i["regions"]] + ends = [r[2] for i in inst for r in i["regions"]] + if len(files) != 1: + cov = unverifiable("AMBIGUOUS_MAPPING", files=sorted(files)) + elif not total: + cov = not_applicable("NO_STATEMENTS") + else: + cov = field( + MEASURED, + kind="region", + total=total, + covered=covered, + instances=len(inst), + aggregation="sum-over-instances/1", + ) + objects.append( + { + "id": key, + "lang": "rust", + "unit": "rust", + "kind": "function", + "name": rust_name(key), + "file": sorted(files)[0], + "span": {"start": min(starts, default=0), "end": max(ends, default=0)}, + "instances": [ + {"symbol": i["symbol"], "count": i["count"]} + for i in sorted(inst, key=lambda x: x["symbol"]) + ], + "coverage": cov, + "mutation": unverifiable("MEASUREMENT_MISSING"), + "tests": unverifiable("TOOL_LIMIT", note="no per-test attribution"), + } + ) + + stats = { + "test_code_excluded": excluded["test_code"], + "mutants": 0, + "mutants_mapped": 0, + "mutants_ambiguous": 0, + "mutants_unmapped": 0, + "mutants_without_function": 0, + } + if mutants is not None and verdicts is not None: + by_file: dict[str, list[dict]] = {} + for o in objects: + by_file.setdefault(o["file"], []).append(o) + counts: dict[str, dict[str, int]] = { + o["id"]: dict.fromkeys(qg.VERDICTS, 0) for o in objects + } + for m in mutants: + stats["mutants"] += 1 + if not m.get( + "function" + ): # real data: a mutant outside any function (e.g. a struct field) + stats["mutants_without_function"] += 1 + continue + fs, fe = ( + m["function"]["span"]["start"]["line"], + m["function"]["span"]["end"]["line"], + ) + # measured: cargo-mutants qualifies the name ("logic::echo", "Type::method"); the object + # name is the last segment, so the last segment is what is compared + fname = m["function"]["function_name"].split("::")[-1] + cand = [ + o + for o in by_file.get(m["file"], []) + if o["name"] == fname and fs <= o["span"]["start"] <= fe + ] + if len(cand) > 1: + stats["mutants_ambiguous"] += 1 + continue + if not cand: + stats["mutants_unmapped"] += 1 + continue + verdict = verdicts.get(m["name"]) + if verdict is None: + raise ModelError( + f"mutant {m['name']!r} is in the listing but has no verdict" + ) + counts[cand[0]["id"]][verdict] += 1 + stats["mutants_mapped"] += 1 + for o in objects: + c = counts[o["id"]] + planned = sum(c.values()) + if not planned: + o["mutation"] = not_applicable("NO_MUTANTS_LISTED") + else: + acct = {"planned": planned, **c} + score = qg.primary_score(acct) + o["mutation"] = field(MEASURED, **c, planned=planned, score=score) + return objects, stats + + +# --------------------------------------------------------------------------- the record + + +def canonical_digest(obj: object) -> str: + return qg.canonical_digest(obj) + + +def summarize(objects: list[dict]) -> dict: + """Totals by state, per field and per unit. Tolerant of a broken object (a missing field is counted + under the state `MISSING`, it does not crash): the verifier must report it, not die on it. + """ + out: dict = {"objects": len(objects), "by_unit": {}, "fields": {}} + for o in objects: + u = out["by_unit"].setdefault(o.get("unit"), {"objects": 0, "fields": {}}) + u["objects"] += 1 + for f in FIELDS: + state = ( + (o.get(f) or {}).get("state", "MISSING") + if isinstance(o.get(f), dict) + else "MISSING" + ) + for tgt in (out["fields"], u["fields"]): + cell = tgt.setdefault(f, {s: 0 for s in STATES}) + cell[state] = cell.get(state, 0) + 1 + reasons: dict[str, int] = {} + for o in objects: + for f in FIELDS: + if ( + isinstance(o.get(f), dict) + and o[f].get("state") != MEASURED + and "reason" in o[f] + ): + key = f"{f}:{o[f]['state']}:{o[f]['reason']}" + reasons[key] = reasons.get(key, 0) + 1 + out["reasons"] = dict(sorted(reasons.items())) + return out + + +def build_record( + objects: list[dict], + commit: str, + tree: str, + tools: dict, + inputs: dict, + scope: dict, + attribution: dict, +) -> dict: + for name, ver in tools.items(): + if not ver or str(ver).lower() in ("unknown", "none", ""): + raise ModelError( + f"the version of the tool {name!r} is unknown: the record would be unusable" + ) + ids = [o["id"] for o in objects] + if len(set(ids)) != len(ids): + raise ModelError("duplicate object ids in the record") + ordered = sorted(objects, key=lambda o: o["id"]) + body: dict = { + "schema": EVIDENCE_SCHEMA, + "id_contract": ID_CONTRACT, + "policy": qg.POLICY, + "coverage_kinds": list(COVERAGE_KINDS), + "identity": {"commit": commit, "tree": tree}, + "scope": { + "schema": SCOPE_SCHEMA, + "version": scope["version"], + "digest": canonical_digest(scope), + }, + "tools": dict(sorted(tools.items())), + "inputs": dict(sorted(inputs.items())), + "attribution": attribution, + "objects": ordered, + } + body["summary"] = summarize(ordered) + body["digest"] = canonical_digest({k: v for k, v in body.items() if k != "digest"}) + return body + + +# --------------------------------------------------------------------------- verification + + +def verify_l1(rec: dict) -> list[str]: + """Record integrity: every problem found (empty = INTEGRITY_OK).""" + bad: list[str] = [] + if ( + not isinstance(rec, dict) + or rec.get("schema") != EVIDENCE_SCHEMA + or rec.get("id_contract") != ID_CONTRACT + ): + return [f"schema/contract is not {EVIDENCE_SCHEMA}/{ID_CONTRACT}"] + if rec.get("digest") != canonical_digest( + {k: v for k, v in rec.items() if k != "digest"} + ): + bad.append("the digest does not match the content") + if rec.get("policy") != qg.POLICY: + bad.append( + f"the record is bound to {rec.get('policy')!r}, the gate to {qg.POLICY!r}" + ) + for name, ver in (rec.get("tools") or {}).items(): + if not ver or str(ver).lower() in ("unknown", "none"): + bad.append(f"the version of the tool {name!r} is unknown") + if not rec.get("tools"): + bad.append("no tool versions are recorded") + objs = rec.get("objects") + if not isinstance(objs, list): + return bad + ["no object list"] + ids = [o.get("id") for o in objs] + if ids != sorted(ids) or len(set(ids)) != len(ids): + bad.append("the object ids are not sorted and unique") + for o in objs: + for f in FIELDS: + v = o.get(f) + if not isinstance(v, dict) or v.get("state") not in STATES: + bad.append(f"{o.get('id')}: field {f} is missing or has no valid state") + continue + if v["state"] != MEASURED and v.get("reason") not in REASONS: + bad.append( + f"{o.get('id')}: field {f} is {v['state']} without a valid reason ({v.get('reason')!r})" + ) + if ( + f == "coverage" + and v["state"] == MEASURED + and v.get("kind") not in COVERAGE_KINDS + ): + bad.append( + f"{o.get('id')}: coverage kind {v.get('kind')!r} is not one of {COVERAGE_KINDS}" + ) + if rec.get("summary") != summarize(objs): + bad.append("the summary does not match the objects") + if not rec.get("inputs"): + bad.append("no input digests are recorded") + return bad + + +def verify_l2( + rec: dict, root: str | None, inputs: dict[str, str] +) -> tuple[str, list[str]]: + """Binding to the inputs: PASS | FAIL | NOT_CHECKED, with the findings.""" + if root is None and not inputs: + return "NOT_CHECKED", ["no checkout and no input files were given"] + bad: list[str] = [] + checked = 0 + if root is not None: + try: + tree = subprocess.run( + ["git", "-C", root, "rev-parse", "HEAD^{tree}"], + capture_output=True, + text=True, + check=True, + ).stdout.strip() + commit = subprocess.run( + ["git", "-C", root, "rev-parse", "HEAD"], + capture_output=True, + text=True, + check=True, + ).stdout.strip() + except (OSError, subprocess.CalledProcessError) as e: + return "FAIL", [f"cannot read the checkout identity: {e}"] + checked += 1 + if tree != rec["identity"]["tree"] or commit != rec["identity"]["commit"]: + bad.append("the checkout is not the commit/tree the record names") + for name, path in inputs.items(): + want = rec["inputs"].get(name) + if want is None: + bad.append(f"input {name!r} is not in the record") + continue + checked += 1 + try: + digest, size = sha256_file(path) + except OSError as e: + bad.append(f"input {name!r} cannot be read: {e}") + continue + if digest != want["sha256"] or size != want["bytes"]: + bad.append(f"input {name!r} does not match the digest in the record") + if bad: + return "FAIL", bad + return ("PASS" if checked else "NOT_CHECKED"), [] + + +def go_func_lines(root: str, files: list[str]) -> int: + """An independent count of top-level Go declarations: lines that start with `func ` in column 0.""" + n = 0 + for f in files: + for line in (Path(root) / f).read_text(encoding="utf-8").splitlines(): + if line.startswith("func "): + n += 1 + return n + + +def verify_l3( + rec: dict, + root: str | None, + go_files: dict[str, list[str]] | None, + mutants: list[dict] | None, +) -> tuple[str, list[str], list[str]]: + """Completeness against an independent enumeration: (COMPLETE_FOR_SCOPE | PARTIAL | FAIL, + proven parts, unproven parts).""" + proven: list[str] = [] + unproven: list[str] = [] + bad: list[str] = [] + objs = rec["objects"] + if root is not None and go_files: + for unit, files in sorted(go_files.items()): + have = sum(1 for o in objs if o["lang"] == "go" and o["unit"] == unit) + want = go_func_lines(root, files) + if have != want: + bad.append( + f"go unit {unit!r}: {have} objects but {want} top-level `func` lines in the source" + ) + else: + proven.append( + f"go unit {unit!r}: {have} objects equal the independent `func` line count" + ) + else: + unproven.append( + "go: no source tree was given, the object list was not compared with an independent count" + ) + if mutants is not None: + rust = {(o["file"], o["name"]) for o in objs if o["lang"] == "rust"} + miss = sorted( + { + (m["file"], m["function"]["function_name"].split("::")[-1]) + for m in mutants + if m.get("function") + } + - rust + ) + if miss: + bad.append( + f"rust: {len(miss)} function(s) known to cargo-mutants have no object, e.g. {miss[0]}" + ) + else: + proven.append("rust: every function that cargo-mutants lists has an object") + else: + unproven.append("rust: no mutant listing was given") + unproven.append( + "rust: a function that has neither a mutant nor a coverage entry could not be enumerated independently" + ) + if bad: + return "FAIL", proven, bad + return ("PARTIAL" if unproven else "COMPLETE_FOR_SCOPE"), proven, unproven + + +def verify_record( + rec: dict, root=None, inputs=None, go_files=None, mutants=None +) -> dict: + """All three levels, each on its own. There is deliberately no combined verdict.""" + l1 = verify_l1(rec) + res = {"L1": {"verdict": "FAIL" if l1 else "INTEGRITY_OK", "findings": l1}} + if l1: # the later levels read the record: do not trust a record that is not intact + res["L2"] = { + "verdict": "NOT_CHECKED", + "findings": ["the record is not intact (L1)"], + } + res["L3"] = { + "verdict": "NOT_CHECKED", + "proven": [], + "unproven": ["the record is not intact (L1)"], + } + return res + v2, f2 = verify_l2(rec, root, inputs or {}) + res["L2"] = {"verdict": {"PASS": "BOUND_TO_INPUTS"}.get(v2, v2), "findings": f2} + v3, proven, other = verify_l3(rec, root, go_files, mutants) + res["L3"] = { + "verdict": v3, + "proven": proven, + "unproven" if v3 != "FAIL" else "findings": other, + } + return res + + +# --------------------------------------------------------------------------- CLI + + +def _load(path: str): + try: + return json.loads(Path(path).read_text(encoding="utf-8")) + except (OSError, ValueError) as e: + raise ModelError(f"{path}: missing, unreadable or not JSON: {e}") from e + + +def demangle(symbols: list[str]) -> tuple[dict[str, str], str]: + """Demangle Rust v0 symbols with binutils c++filt (in the pinned image); returns (map, version).""" + try: + ver = subprocess.run( + ["c++filt", "--version"], capture_output=True, text=True, check=True + ).stdout.splitlines()[0] + out = subprocess.run( + ["c++filt"], + input="\n".join(symbols) + "\n", + capture_output=True, + text=True, + check=True, + ).stdout + except (OSError, subprocess.CalledProcessError) as e: + raise ModelError(f"c++filt is not available: {e}") from e + names = out.splitlines() + if len(names) != len(symbols): + raise ModelError("c++filt returned a different number of lines") + return dict(zip(symbols, names)), ver + + +def _git(root: str, *args: str) -> str: + try: + return subprocess.run( + ["git", "-C", root, *args], capture_output=True, text=True, check=True + ).stdout.strip() + except (OSError, subprocess.CalledProcessError) as e: + raise ModelError(f"git {' '.join(args)} failed in {root}: {e}") from e + + +def cmd_build(a: argparse.Namespace) -> int: + """Build the evidence record from the real tool outputs (report-only).""" + scope = _load(a.scope) + check_scope(scope) + tracked = _git(a.root, "ls-files").splitlines() + inputs: dict[str, dict] = {} + + def note(name: str, path: str) -> None: + digest, size = sha256_file(path) + inputs[name] = {"sha256": digest, "bytes": size} + + note("scope", a.scope) + objects: list[dict] = [] + attribution: dict = {} + # every tracked Go/Rust file must be classified: an unclassified file is an error (N8) + scope_files(scope, "go", tracked) + scope_files(scope, "rust", tracked) + for spec in a.go_unit or []: + name, mod_dir, inv, cover, *opt = spec.split(",") + module = go_module_path( + Path(a.root, mod_dir, "go.mod").read_text(encoding="utf-8") + ) + blocks = parse_coverprofile(Path(cover).read_text(encoding="utf-8")) + reps = [ + parse_coverprofile(Path(r.split("=", 1)[1]).read_text(encoding="utf-8")) + for r in (a.go_repeat or []) + if r.split("=", 1)[0] == name + ] + objs, st = go_objects( + name, + module, + _load(inv), + blocks, + repeats=reps or None, + detail="detail" in opt, + ) + objects += objs + attribution[f"go:{name}"] = st + note(f"go-inventory:{name}", inv) + note(f"go-coverprofile:{name}", cover) + if a.llvm_cov: + llvm = _load(a.llvm_cov) + syms = sorted({f["name"] for f in llvm["data"][0]["functions"]}) + names, cxxfilt = demangle(syms) + mutants = _load(a.rust_list) if a.rust_list else None + verdicts: dict[str, str] | None = None + if a.rust_outcomes: + verdicts = {} + for path in a.rust_outcomes: + for k, v in qg.rust_verdicts(_load(path), path).items(): + if k in verdicts: + raise ModelError( + f"mutant {k!r} has a verdict in two outcome files" + ) + verdicts[k] = v + note(f"rust-outcomes:{Path(path).parent.parent.name}", path) + objs, st = rust_objects( + llvm, names, a.root, mutants, verdicts, prefix=a.rust_prefix + ) + objects += objs + attribution["rust"] = st + note("llvm-cov-json", a.llvm_cov) + if a.rust_list: + note("rust-mutant-list", a.rust_list) + a.tool = (a.tool or []) + [f"c++filt={cxxfilt}"] + tools = dict(kv.split("=", 1) for kv in a.tool or []) + rec = build_record( + objects, + _git(a.root, "rev-parse", "HEAD"), + _git(a.root, "rev-parse", "HEAD^{tree}"), + tools, + inputs, + scope, + attribution, + ) + # the commit and tree name the committed state; uncommitted paths are counted, not hidden + rec["identity"]["uncommitted_paths"] = len( + _git(a.root, "status", "--porcelain").splitlines() + ) + rec["digest"] = canonical_digest({k: v for k, v in rec.items() if k != "digest"}) + Path(a.out).write_text( + json.dumps(rec, indent=1, sort_keys=True) + "\n", encoding="utf-8" + ) + sm = rec["summary"] + print(f"{EVIDENCE_SCHEMA}: {sm['objects']} objects, digest {rec['digest'][:16]}") + for unit, u in sm["by_unit"].items(): + print( + f" unit {unit}: {u['objects']} objects, coverage {u['fields']['coverage']}" + ) + return 0 + + +def cmd_scope_list(a: argparse.Namespace) -> int: + """The tracked files of one scope class, one per line (what the inventory tool is run on).""" + scope = _load(a.scope) + check_scope(scope) + tracked = _git(a.root, "ls-files").splitlines() + for f in sorted(scope_files(scope, a.lang, tracked).get(a.cls, [])): + print(f) + return 0 + + +def cmd_verify(a: argparse.Namespace) -> int: + rec = _load(a.record) + inputs = dict(kv.split("=", 1) for kv in a.input or []) + mutants = _load(a.rust_list) if a.rust_list else None + go_files = None + if a.root and a.scope: + scope = _load(a.scope) + check_scope(scope) + tracked = subprocess.run( + ["git", "-C", a.root, "ls-files"], + capture_output=True, + text=True, + check=True, + ).stdout.splitlines() + groups = scope_files(scope, "go", tracked) + units = { + u.split(":", 1)[1]: fs + for u, fs in groups.items() + if u.startswith("separate:") + } + units["root"] = groups.get("production", []) + go_files = { + u: fs + for u, fs in units.items() + if any(o["unit"] == u for o in rec.get("objects", [])) or fs + } + res = verify_record(rec, a.root, inputs, go_files, mutants) + for lvl in ("L1", "L2", "L3"): + r = res[lvl] + print(f"{lvl} {r['verdict']}") + for k in ("findings", "proven", "unproven"): + for line in r.get(k, []): + print(f" {k}: {line}") + print( + "(no combined 'verified': a digest does not prove where a measurement came from)" + ) + failed = any(res[lvl]["verdict"] == "FAIL" for lvl in ("L1", "L2", "L3")) + return 1 if failed else 0 + + +def main(argv: list[str] | None = None) -> int: + ap = argparse.ArgumentParser(description=__doc__.split("\n", 1)[0]) + sub = ap.add_subparsers(dest="command", required=True) + p = sub.add_parser("build") + p.add_argument("--root", required=True, help="checkout of the measured tree") + p.add_argument("--scope", required=True) + p.add_argument("--out", required=True) + p.add_argument( + "--go-unit", + action="append", + help="NAME,MODULE_DIR,INVENTORY.json,COVERPROFILE[,detail]", + ) + p.add_argument( + "--go-repeat", + action="append", + help="UNIT=COVERPROFILE of a repeated measurement", + ) + p.add_argument("--llvm-cov", help="cargo llvm-cov --json export") + p.add_argument("--rust-list", help="cargo mutants --list --json") + p.add_argument( + "--rust-outcomes", + action="append", + help="outcomes.json of a (shard) run, repeatable", + ) + p.add_argument( + "--rust-prefix", + default="/w/", + help="the checkout path prefix of the file names in the llvm-cov export", + ) + p.add_argument( + "--tool", action="append", help="NAME=VERSION of a measuring tool, repeatable" + ) + p.set_defaults(fn=cmd_build) + + p = sub.add_parser("scope-list") + p.add_argument("--root", required=True) + p.add_argument("--scope", required=True) + p.add_argument("--lang", required=True, choices=["go", "rust"]) + p.add_argument( + "--class", + dest="cls", + required=True, + help="production | separate: | test | ...", + ) + p.set_defaults(fn=cmd_scope_list) + + p = sub.add_parser("verify") + p.add_argument("--record", required=True) + p.add_argument( + "--root", help="checkout of the measured tree (L2 identity, L3 source count)" + ) + p.add_argument( + "--scope", help="the measurement scope (needed with --root for the L3 Go count)" + ) + p.add_argument( + "--input", action="append", help="NAME=PATH of an input file (L2 digest)" + ) + p.add_argument("--rust-list", help="cargo mutants --list --json output (L3, Rust)") + p.set_defaults(fn=cmd_verify) + a = ap.parse_args(argv) + try: + return a.fn(a) + except (ModelError, qg.GateError) as e: + print(f"::error::{e}", file=sys.stderr) + return 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/quality_gate.py b/scripts/quality_gate.py index 81ca05b..4f23257 100644 --- a/scripts/quality_gate.py +++ b/scripts/quality_gate.py @@ -24,7 +24,7 @@ do not matter; two different mutants never share an id (a listing with a duplicate id is an invalid plan, not a merged one). mutation-shard-report/1, gremlins-report/1 the reports of a shard / a chunk - mutation-policy/1 how the verdicts are accounted (see POLICY) + mutation-policy/2 how the verdicts are accounted and the primary score (see POLICY) Plan states (named in the output): PLAN_VALID_NONEMPTY a valid plan with at least one mutant/package @@ -54,15 +54,21 @@ EMPTY = "PLAN_VALID_EMPTY" INVALID = "PLAN_INVALID" -# mutation-policy/1: the verdicts of a Rust mutant, each counted on its own. The plan size is -# the denominator: nothing leaves it (a timed-out or unviable mutant stays a planned mutant). +# mutation-policy/2 (owner decisions of 2026-11-14): the verdicts of a Rust mutant, each counted on +# its own; the plan size stays the denominator of the accounting (nothing leaves the plan). # caught detected by the tests -> ok # missed survived -> FAIL # timeout no verdict in time -> FAIL (it is not a detection here) -# unviable the mutant does not compile -> reported, does not fail -# (the behaviour the gate had; the owner decides whether it becomes a failure, -# and that decision is a change of this policy version, not an exception) -POLICY = "mutation-policy/1" +# unviable the mutant does not compile -> neither caught nor missed; reported as a +# mandatory metric of its own (count and ratio); does NOT fail. Tightening it is a +# change of this policy version (mutation-policy/3), never an exception. +# The primary score is caught / (caught + missed). timeout and unviable are not in its denominator +# and are therefore always printed next to it: a timeout fails the gate (so it is 0 in a green run), +# an unviable never changes the score. Nothing is stricter or weaker than before: `missed > 0` and +# `timeout > 0` are red, a score threshold (the 90% target) is a reported ratchet value and never +# replaces that rule. A zero denominator is not 100% and not 0%: see primary_score(). +# (mutation-policy/1 printed caught/planned as the primary value; that stays printed as `legacy`.) +POLICY = "mutation-policy/2" SUMMARY_TO_VERDICT = { "CaughtMutant": "caught", "MissedMutant": "missed", @@ -77,6 +83,49 @@ class GateError(Exception): """The input cannot be trusted: the gate fails.""" +def primary_score(acct: dict | None) -> dict: + """The primary mutation score, caught / (caught + missed), as a state, never a bare number. + + {"state": "MEASURED", "value": 0..100, "caught": c, "evaluable": c + m} + {"state": "NOT_APPLICABLE", "reason": ...} nothing evaluable, for a stated reason + {"state": "UNVERIFIABLE", "reason": "MEASUREMENT_MISSING"} no accounting at all + + A zero denominator is neither 100% nor 0%: with no mutant planned the score does not apply + (NO_MUTANTS_PLANNED); with every mutant unviable (ALL_UNVIABLE) or only unviable and timed-out + ones (NO_EVALUABLE_MUTANT) nothing could be evaluated. + """ + if not acct or "planned" not in acct: + return {"state": "UNVERIFIABLE", "reason": "MEASUREMENT_MISSING"} + caught, missed = acct.get("caught", 0), acct.get("missed", 0) + evaluable = caught + missed + if evaluable: + return { + "state": "MEASURED", + "value": 100.0 * caught / evaluable, + "caught": caught, + "evaluable": evaluable, + } + if acct["planned"] == 0: + return {"state": "NOT_APPLICABLE", "reason": "NO_MUTANTS_PLANNED"} + if acct.get("unviable", 0) == acct["planned"]: + return {"state": "NOT_APPLICABLE", "reason": "ALL_UNVIABLE"} + return {"state": "NOT_APPLICABLE", "reason": "NO_EVALUABLE_MUTANT"} + + +def unviable_ratio(acct: dict | None) -> dict: + """The unviable share of the plan: a mandatory metric of its own (report-only).""" + if not acct or "planned" not in acct: + return {"state": "UNVERIFIABLE", "reason": "MEASUREMENT_MISSING"} + if not acct["planned"]: + return {"state": "NOT_APPLICABLE", "reason": "NO_MUTANTS_PLANNED"} + return { + "state": "MEASURED", + "count": acct["unviable"], + "planned": acct["planned"], + "value": 100.0 * acct["unviable"] / acct["planned"], + } + + def sha256_hex(data: bytes) -> str: return hashlib.sha256(data).hexdigest() @@ -567,11 +616,22 @@ def cmd_verify(a: argparse.Namespace) -> int: ) line = " ".join(f"{k}={acct[k]}" for k in ("planned", *VERDICTS)) print(f"rust {plan['state']} [{POLICY}]: {line}") + ps, ur = primary_score(acct), unviable_ratio(acct) + if ps["state"] == "MEASURED": + print( + f"rust score (primary, {POLICY}): caught/(caught+missed) = " + f"{ps['caught']}/{ps['evaluable']} = {ps['value']:.2f}%" + ) + else: + print(f"rust score (primary, {POLICY}): {ps['state']} ({ps['reason']})") + if ur["state"] == "MEASURED": + print( + f"rust unviable (mandatory metric, report-only): " + f"{ur['count']}/{ur['planned']} = {ur['value']:.2f}%" + ) if acct["planned"]: - viable = acct["planned"] - acct["unviable"] print( - f"rust score: caught/planned = {acct['caught']}/{acct['planned']}; " - f"caught/(planned-unviable) = {acct['caught']}/{viable}" + f"rust legacy (policy/1 primary): caught/planned = {acct['caught']}/{acct['planned']}" ) bad = [f"{v}={acct[v]}" for v in FAILING if acct[v]] if bad: diff --git a/tests/test_tools/fixtures/object_model/README.md b/tests/test_tools/fixtures/object_model/README.md new file mode 100644 index 0000000..75ad3b0 --- /dev/null +++ b/tests/test_tools/fixtures/object_model/README.md @@ -0,0 +1,11 @@ +Fixtures of `tests/test_tools/test_object_model.py`, cut from the REAL tool outputs (nothing is invented). + +* `goastdump-inventory.json` — the real output of `tools/objinventory` on `tools/goastdump/main.go` (15 functions). Unmodified. +* `goastdump-cover.out` — the real `go test -covermode=set -coverprofile` of the `goastdump` module. Unmodified. +* `goastdump-main.go` — the source those two describe (CIC_Relay 852594a). +* `llvm-cov-cic-canonical.json` — the entries of `cic-canonical/src/lib.rs` of the real `cargo llvm-cov --json` export (cargo-llvm-cov 0.9.1, rust 1.96.1). +* `demangled-cic-canonical.json` — the real `c++filt` (GNU Binutils 2.40) names of those symbols. +* `mutants-cic-canonical.json`, `outcomes-cic-canonical.json` — the real cargo-mutants 27.1.0 listing and outcomes restricted to that file (11 mutants; the `diff` text and the machine-local log paths removed). +* `cic-canonical-lib.rs` — the source of that file. + +Other verdicts (missed, timeout, unviable), other block layouts and tampered records are DERIVED in the tests from these; derived cases are marked in the tests. diff --git a/tests/test_tools/fixtures/object_model/cic-canonical-lib.rs b/tests/test_tools/fixtures/object_model/cic-canonical-lib.rs new file mode 100644 index 0000000..e1cfa6b --- /dev/null +++ b/tests/test_tools/fixtures/object_model/cic-canonical-lib.rs @@ -0,0 +1,118 @@ +//! CIC Canonical Object Encoding — number normalization (spec #1). +//! +//! The Rust peer of the Go implementation in `core/nexus/iac/number.go`. Both must +//! produce byte-identical canonical forms so a digest computed in Go verifies in +//! Rust and vice versa. The contract is defined over the JSON numeric *literal* +//! (the text a number arrives as), which is the language-neutral surface: +//! +//! - `4`, `4.0`, `4e0` all canonicalize to `"4"`; +//! - `-0`, `-0.0` canonicalize to `"0"`; +//! - `4.50` canonicalizes to `"4.5"`; +//! - integer literals keep full precision (string-based, no float round-trip), so +//! an integer beyond `f64`'s 53-bit mantissa canonicalizes to its exact digits. +//! +//! The test vectors below mirror `TestCanonicalNumber` in `number_test.go`; that Go +//! table is the source of truth. + +/// Returns the canonical decimal form of a JSON numeric literal, or `None` if `s` +/// is not a number. +/// +/// Integer literals are normalized as exact digit strings (sign, leading zeros and +/// negative zero folded); other numeric literals normalize through `f64`'s shortest +/// plain-decimal form. +#[must_use] +pub fn canonical_number(s: &str) -> Option { + let s = s.trim(); + if s.is_empty() { + return None; + } + if let Some(int) = canonical_integer(s) { + return Some(int); + } + let f: f64 = s.parse().ok()?; + Some(canonical_float(f)) +} + +/// Normalizes an integer literal to exact digits, or `None` if `s` is not an +/// integer literal. Mirrors Go's `big.Int` path: strips a leading `+`, folds +/// leading zeros, and renders negative zero as `"0"`. +fn canonical_integer(s: &str) -> Option { + let (neg, digits) = match s.strip_prefix('-') { + Some(rest) => (true, rest), + None => (false, s.strip_prefix('+').unwrap_or(s)), + }; + if digits.is_empty() || !digits.bytes().all(|b| b.is_ascii_digit()) { + return None; + } + let magnitude = digits.trim_start_matches('0'); + if magnitude.is_empty() { + return Some("0".to_string()); // all zeros, including "-0" + } + Some(if neg { + format!("-{magnitude}") + } else { + magnitude.to_string() + }) +} + +/// Renders a float in plain (non-exponent) shortest form, folding negative zero to +/// `"0"`. Rust's `f64` `Display` is shortest round-trip and never uses an exponent, +/// matching Go's `FormatFloat(f, 'f', -1, 64)`. +fn canonical_float(f: f64) -> String { + let s = format!("{f}"); + if s == "-0" { + "0".to_string() + } else { + s + } +} + +#[cfg(test)] +mod tests { + use super::canonical_number; + + #[test] + fn vectors_mirror_go() { + // Mirrors TestCanonicalNumber in core/nexus/iac/number_test.go. + let cases = [ + ("4", "4"), + ("4.0", "4"), + ("4.5", "4.5"), + ("4e0", "4"), + ("4.50", "4.5"), + ("1e3", "1000"), + ("16", "16"), + ("-0", "0"), + ("-0.0", "0"), + ("-7", "-7"), + ("+4", "4"), + ("007", "7"), + // precision: beyond f64's 53-bit mantissa, kept exact + ("9007199254740993", "9007199254740993"), + ("9007199254740992", "9007199254740992"), + ]; + for (input, want) in cases { + assert_eq!( + canonical_number(input).as_deref(), + Some(want), + "canonical_number({input:?})" + ); + } + } + + #[test] + fn non_numeric_is_none() { + for input in ["", " ", "abc", "4.2.1", "0x10", "--1", "1e"] { + assert_eq!(canonical_number(input), None, "canonical_number({input:?})"); + } + } + + #[test] + fn adjacent_large_integers_differ() { + // The precision guarantee: these must not collapse to the same form. + assert_ne!( + canonical_number("9007199254740993"), + canonical_number("9007199254740992") + ); + } +} diff --git a/tests/test_tools/fixtures/object_model/demangled-cic-canonical.json b/tests/test_tools/fixtures/object_model/demangled-cic-canonical.json new file mode 100644 index 0000000..b870180 --- /dev/null +++ b/tests/test_tools/fixtures/object_model/demangled-cic-canonical.json @@ -0,0 +1,9 @@ +{ + "_RNCNvCs65aHm1XuDSs_13cic_canonical17canonical_integer0B3_": "cic_canonical[46db8227c20f276e]::canonical_integer::{closure#0}", + "_RNvCs65aHm1XuDSs_13cic_canonical15canonical_float": "cic_canonical[46db8227c20f276e]::canonical_float", + "_RNvCs65aHm1XuDSs_13cic_canonical16canonical_number": "cic_canonical[46db8227c20f276e]::canonical_number", + "_RNvCs65aHm1XuDSs_13cic_canonical17canonical_integer": "cic_canonical[46db8227c20f276e]::canonical_integer", + "_RNvNtCs65aHm1XuDSs_13cic_canonical5testss_17vectors_mirror_go": "cic_canonical[46db8227c20f276e]::tests::vectors_mirror_go", + "_RNvNtCs65aHm1XuDSs_13cic_canonical5testss_19non_numeric_is_none": "cic_canonical[46db8227c20f276e]::tests::non_numeric_is_none", + "_RNvNtCs65aHm1XuDSs_13cic_canonical5testss_30adjacent_large_integers_differ": "cic_canonical[46db8227c20f276e]::tests::adjacent_large_integers_differ" +} \ No newline at end of file diff --git a/tests/test_tools/fixtures/object_model/goastdump-cover.out b/tests/test_tools/fixtures/object_model/goastdump-cover.out new file mode 100644 index 0000000..671dc1a --- /dev/null +++ b/tests/test_tools/fixtures/object_model/goastdump-cover.out @@ -0,0 +1,135 @@ +mode: set +goastdump/main.go:73.36,76.49 3 1 +goastdump/main.go:76.49,78.3 1 1 +goastdump/main.go:79.2,79.44 1 1 +goastdump/main.go:82.42,83.14 1 1 +goastdump/main.go:83.14,85.3 1 1 +goastdump/main.go:86.2,86.69 1 1 +goastdump/main.go:96.48,97.19 1 1 +goastdump/main.go:97.19,99.3 1 1 +goastdump/main.go:100.2,101.11 2 1 +goastdump/main.go:105.68,106.23 1 1 +goastdump/main.go:107.21,108.25 1 1 +goastdump/main.go:109.22,110.25 1 0 +goastdump/main.go:111.22,112.25 1 1 +goastdump/main.go:113.26,114.25 1 0 +goastdump/main.go:115.25,116.59 1 1 +goastdump/main.go:116.59,118.4 1 1 +goastdump/main.go:119.18,120.43 1 1 +goastdump/main.go:120.43,122.4 1 0 +goastdump/main.go:123.3,123.26 1 1 +goastdump/main.go:125.2,125.22 1 0 +goastdump/main.go:129.68,130.23 1 1 +goastdump/main.go:131.22,132.24 1 1 +goastdump/main.go:132.24,134.4 1 1 +goastdump/main.go:135.22,136.25 1 0 +goastdump/main.go:137.25,138.20 1 1 +goastdump/main.go:138.20,140.4 1 1 +goastdump/main.go:141.21,142.81 1 0 +goastdump/main.go:142.81,144.4 1 0 +goastdump/main.go:146.2,146.22 1 0 +goastdump/main.go:151.72,152.19 1 1 +goastdump/main.go:152.19,154.3 1 0 +goastdump/main.go:155.2,155.33 1 1 +goastdump/main.go:156.18,157.28 1 1 +goastdump/main.go:158.22,159.20 1 1 +goastdump/main.go:159.20,161.4 1 1 +goastdump/main.go:162.3,162.29 1 0 +goastdump/main.go:162.29,163.58 1 0 +goastdump/main.go:163.58,165.5 1 0 +goastdump/main.go:167.23,168.31 1 1 +goastdump/main.go:168.31,170.4 1 0 +goastdump/main.go:171.3,171.27 1 1 +goastdump/main.go:171.27,172.62 1 1 +goastdump/main.go:172.62,174.5 1 1 +goastdump/main.go:177.2,177.22 1 0 +goastdump/main.go:180.33,180.64 1 1 +goastdump/main.go:192.42,194.2 1 1 +goastdump/main.go:196.37,197.20 1 1 +goastdump/main.go:197.20,200.3 2 1 +goastdump/main.go:203.39,204.14 1 1 +goastdump/main.go:204.14,206.3 1 0 +goastdump/main.go:207.2,207.40 1 1 +goastdump/main.go:207.40,208.25 1 1 +goastdump/main.go:209.26,210.60 1 1 +goastdump/main.go:210.60,212.22 2 1 +goastdump/main.go:212.22,215.6 2 1 +goastdump/main.go:217.22,218.13 1 1 +goastdump/main.go:220.3,220.14 1 1 +goastdump/main.go:224.43,226.6 2 1 +goastdump/main.go:226.6,227.26 1 1 +goastdump/main.go:228.23,230.12 2 1 +goastdump/main.go:231.27,233.12 2 0 +goastdump/main.go:234.23,236.12 2 0 +goastdump/main.go:238.3,238.8 1 1 +goastdump/main.go:240.2,240.25 1 1 +goastdump/main.go:241.18,242.23 1 1 +goastdump/main.go:242.23,244.4 1 1 +goastdump/main.go:245.25,246.28 1 1 +goastdump/main.go:246.28,248.4 1 0 +goastdump/main.go:249.3,250.10 2 1 +goastdump/main.go:250.10,253.4 2 0 +goastdump/main.go:254.3,254.25 1 1 +goastdump/main.go:254.25,257.4 2 1 +goastdump/main.go:258.3,258.52 1 1 +goastdump/main.go:258.52,261.4 2 1 +goastdump/main.go:262.3,262.36 1 0 +goastdump/main.go:266.38,267.23 1 1 +goastdump/main.go:268.21,269.27 1 1 +goastdump/main.go:270.22,271.27 1 0 +goastdump/main.go:272.22,273.27 1 1 +goastdump/main.go:274.26,275.27 1 0 +goastdump/main.go:276.18,277.16 1 1 +goastdump/main.go:279.2,279.11 1 0 +goastdump/main.go:282.74,286.24 4 1 +goastdump/main.go:287.23,288.20 1 1 +goastdump/main.go:289.26,292.37 3 1 +goastdump/main.go:292.37,293.47 1 1 +goastdump/main.go:293.47,294.31 1 1 +goastdump/main.go:294.31,295.26 1 1 +goastdump/main.go:295.26,297.7 1 1 +goastdump/main.go:301.3,302.24 2 1 +goastdump/main.go:302.24,304.4 1 1 +goastdump/main.go:305.3,305.26 1 1 +goastdump/main.go:306.10,307.18 1 1 +goastdump/main.go:309.2,309.10 1 1 +goastdump/main.go:312.33,316.16 4 1 +goastdump/main.go:316.16,319.3 2 1 +goastdump/main.go:320.2,324.32 4 1 +goastdump/main.go:324.32,326.17 2 1 +goastdump/main.go:326.17,327.12 1 0 +goastdump/main.go:329.3,330.21 2 1 +goastdump/main.go:330.21,331.50 1 1 +goastdump/main.go:331.50,332.13 1 0 +goastdump/main.go:334.4,334.24 1 1 +goastdump/main.go:336.3,337.23 2 1 +goastdump/main.go:340.2,340.32 1 1 +goastdump/main.go:340.32,341.27 1 1 +goastdump/main.go:342.22,345.21 3 1 +goastdump/main.go:345.21,347.5 1 1 +goastdump/main.go:348.4,348.21 1 1 +goastdump/main.go:348.21,350.5 1 1 +goastdump/main.go:351.4,352.46 2 1 +goastdump/main.go:352.46,353.56 1 1 +goastdump/main.go:353.56,356.6 2 1 +goastdump/main.go:358.4,358.36 1 1 +goastdump/main.go:359.21,360.33 1 1 +goastdump/main.go:360.33,362.29 2 1 +goastdump/main.go:363.24,365.42 2 1 +goastdump/main.go:365.42,367.7 1 1 +goastdump/main.go:368.6,368.77 1 1 +goastdump/main.go:369.25,371.42 2 1 +goastdump/main.go:371.42,373.7 1 1 +goastdump/main.go:374.6,375.30 2 1 +goastdump/main.go:375.30,377.7 1 1 +goastdump/main.go:378.6,378.32 1 1 +goastdump/main.go:378.32,379.24 1 1 +goastdump/main.go:379.24,380.16 1 1 +goastdump/main.go:382.7,382.112 1 1 +goastdump/main.go:388.2,388.10 1 1 +goastdump/main.go:391.13,392.22 1 0 +goastdump/main.go:392.22,395.3 2 0 +goastdump/main.go:396.2,399.32 2 0 +goastdump/main.go:399.32,401.3 1 0 +goastdump/main.go:402.2,404.40 3 0 +goastdump/main.go:404.40,407.3 2 0 diff --git a/tests/test_tools/fixtures/object_model/goastdump-inventory.json b/tests/test_tools/fixtures/object_model/goastdump-inventory.json new file mode 100644 index 0000000..3f0c61f --- /dev/null +++ b/tests/test_tools/fixtures/object_model/goastdump-inventory.json @@ -0,0 +1,199 @@ +{ + "schema": "objinventory/1", + "files": [ + { + "file": "main.go", + "package": "main", + "funcs": [ + { + "name": "guessPkgName", + "kind": "func", + "start": { + "line": 73, + "col": 1 + }, + "end": { + "line": 80, + "col": 2 + } + }, + { + "name": "docText", + "kind": "func", + "start": { + "line": 82, + "col": 1 + }, + "end": { + "line": 87, + "col": 2 + } + }, + { + "name": "isImport", + "recv": "fileCtx", + "kind": "method", + "start": { + "line": 96, + "col": 1 + }, + "end": { + "line": 102, + "col": 2 + } + }, + { + "name": "typeName", + "recv": "fileCtx", + "kind": "method", + "start": { + "line": 105, + "col": 1 + }, + "end": { + "line": 126, + "col": 2 + } + }, + { + "name": "exprType", + "recv": "fileCtx", + "kind": "method", + "start": { + "line": 129, + "col": 1 + }, + "end": { + "line": 147, + "col": 2 + } + }, + { + "name": "identType", + "recv": "fileCtx", + "kind": "method", + "start": { + "line": 151, + "col": 1 + }, + "end": { + "line": 178, + "col": 2 + } + }, + { + "name": "exported", + "kind": "func", + "start": { + "line": 180, + "col": 1 + }, + "end": { + "line": 180, + "col": 64 + } + }, + { + "name": "newCollector", + "kind": "func", + "start": { + "line": 192, + "col": 1 + }, + "end": { + "line": 194, + "col": 2 + } + }, + { + "name": "addCall", + "recv": "collector", + "kind": "method", + "start": { + "line": 196, + "col": 1 + }, + "end": { + "line": 201, + "col": 2 + } + }, + { + "name": "visit", + "recv": "collector", + "kind": "method", + "start": { + "line": 203, + "col": 1 + }, + "end": { + "line": 222, + "col": 2 + } + }, + { + "name": "call", + "recv": "collector", + "kind": "method", + "start": { + "line": 224, + "col": 1 + }, + "end": { + "line": 264, + "col": 2 + } + }, + { + "name": "recvTypeName", + "kind": "func", + "start": { + "line": 266, + "col": 1 + }, + "end": { + "line": 280, + "col": 2 + } + }, + { + "name": "typeObject", + "recv": "fileCtx", + "kind": "method", + "start": { + "line": 282, + "col": 1 + }, + "end": { + "line": 310, + "col": 2 + } + }, + { + "name": "dump", + "kind": "func", + "start": { + "line": 312, + "col": 1 + }, + "end": { + "line": 389, + "col": 2 + } + }, + { + "name": "main", + "kind": "func", + "start": { + "line": 391, + "col": 1 + }, + "end": { + "line": 408, + "col": 2 + } + } + ] + } + ] +} diff --git a/tests/test_tools/fixtures/object_model/goastdump-main.go b/tests/test_tools/fixtures/object_model/goastdump-main.go new file mode 100644 index 0000000..64d6a82 --- /dev/null +++ b/tests/test_tools/fixtures/object_model/goastdump-main.go @@ -0,0 +1,408 @@ +// goastdump parses Go source files with go/parser and prints their top-level +// declarations as JSON. It is the syntactic front end of tools/go.meta.gen.py: +// the Python side decides what counts as an external reference and writes the +// companion YAML, this tool only reports what the Go syntax tree says. +// +// Usage: goastdump [ ...] +// +// Only the standard library is used and no type checking happens, so it runs +// offline and without the dependencies of the code it reads. A call receiver is +// resolved to a type only where the syntax states it (parameter, receiver, +// "var x T", "x := T{}" / "x := &T{}"); everything else stays untyped. +package main + +import ( + "encoding/json" + "fmt" + "go/ast" + "go/parser" + "go/token" + "go/types" + "os" + "regexp" + "sort" + "strconv" + "strings" +) + +// Ref is a selector on an imported package alias: Pkg.Name. +type Ref struct { + Pkg string `json:"pkg"` + Name string `json:"name"` +} + +// Call is an exported function or method call. Pkg is set for a call through an +// import alias or a receiver typed by one; Type is set when the receiver type is +// known (local or imported); both empty means the receiver could not be typed. +type Call struct { + Pkg string `json:"pkg,omitempty"` + Type string `json:"type,omitempty"` + Name string `json:"name"` +} + +// Object is one top-level declaration. +type Object struct { + Name string `json:"name"` + Kind string `json:"kind"` // struct interface type func method var const + Receiver string `json:"receiver,omitempty"` + Doc string `json:"doc"` + Refs []Ref `json:"refs"` + Calls []Call `json:"calls"` + Methods []string `json:"methods,omitempty"` // interface only: exported method names +} + +// FileDump is the result for one input file. +type FileDump struct { + File string `json:"file"` + Package string `json:"package"` + PackageDoc string `json:"package_doc"` + Imports map[string]string `json:"imports"` // alias -> import path + Objects []Object `json:"objects"` + Error string `json:"error,omitempty"` +} + +var ( + reMajor = regexp.MustCompile(`^v[0-9]+$`) + reDotVer = regexp.MustCompile(`\.v[0-9]+$`) + spaceRuns = regexp.MustCompile(`\s+`) +) + +// guessPkgName derives the default package name from an import path. Without +// type information the declared name is unknown, so the usual conventions are +// applied: "…/yaml.v3" -> yaml, "…/foo/v2" -> foo, otherwise the last element. +func guessPkgName(p string) string { + elems := strings.Split(p, "/") + last := elems[len(elems)-1] + if reMajor.MatchString(last) && len(elems) > 1 { + last = elems[len(elems)-2] + } + return reDotVer.ReplaceAllString(last, "") +} + +func docText(g *ast.CommentGroup) string { + if g == nil { + return "" + } + return strings.TrimSpace(spaceRuns.ReplaceAllString(g.Text(), " ")) +} + +type fileCtx struct { + imports map[string]string +} + +// isImport reports whether id names an imported package: its alias is imported +// and the parser did not resolve it to a local declaration (a local variable +// called like an import shadows it). +func (c *fileCtx) isImport(id *ast.Ident) bool { + if id.Obj != nil { //nolint:staticcheck // Obj is the parser's own scope resolution + return false + } + _, ok := c.imports[id.Name] + return ok +} + +// typeName names the type an expression denotes, if it is a plain named type. +func (c *fileCtx) typeName(t ast.Expr) (pkg, name string, ok bool) { + switch x := t.(type) { + case *ast.StarExpr: + return c.typeName(x.X) + case *ast.ParenExpr: + return c.typeName(x.X) + case *ast.IndexExpr: + return c.typeName(x.X) + case *ast.IndexListExpr: + return c.typeName(x.X) + case *ast.SelectorExpr: + if id, isID := x.X.(*ast.Ident); isID && c.isImport(id) { + return id.Name, x.Sel.Name, true + } + case *ast.Ident: + if types.Universe.Lookup(x.Name) != nil { + return "", "", false + } + return "", x.Name, true + } + return "", "", false +} + +// exprType resolves the type of the few initializer shapes that state it. +func (c *fileCtx) exprType(e ast.Expr) (pkg, name string, ok bool) { + switch x := e.(type) { + case *ast.UnaryExpr: + if x.Op == token.AND { + return c.exprType(x.X) + } + case *ast.ParenExpr: + return c.exprType(x.X) + case *ast.CompositeLit: + if x.Type != nil { + return c.typeName(x.Type) + } + case *ast.CallExpr: + if f, isID := x.Fun.(*ast.Ident); isID && f.Name == "new" && len(x.Args) == 1 { + return c.typeName(x.Args[0]) + } + } + return "", "", false +} + +// identType resolves a local identifier to its declared type through the +// parser's scope resolution. +func (c *fileCtx) identType(id *ast.Ident) (pkg, name string, ok bool) { + if id.Obj == nil { //nolint:staticcheck + return "", "", false + } + switch d := id.Obj.Decl.(type) { //nolint:staticcheck + case *ast.Field: + return c.typeName(d.Type) + case *ast.ValueSpec: + if d.Type != nil { + return c.typeName(d.Type) + } + for i, n := range d.Names { + if n.Name == id.Name && len(d.Names) == len(d.Values) { + return c.exprType(d.Values[i]) + } + } + case *ast.AssignStmt: + if len(d.Lhs) != len(d.Rhs) { + return "", "", false + } + for i, l := range d.Lhs { + if li, isID := l.(*ast.Ident); isID && li.Name == id.Name { + return c.exprType(d.Rhs[i]) + } + } + } + return "", "", false +} + +func exported(name string) bool { return ast.IsExported(name) } + +// collector gathers refs and calls from a subtree, first occurrence order, +// without duplicates. +type collector struct { + ctx *fileCtx + refs []Ref + calls []Call + seenRef map[Ref]bool + seenCall map[Call]bool +} + +func newCollector(c *fileCtx) *collector { + return &collector{ctx: c, refs: []Ref{}, calls: []Call{}, seenRef: map[Ref]bool{}, seenCall: map[Call]bool{}} +} + +func (k *collector) addCall(c Call) { + if !k.seenCall[c] { + k.seenCall[c] = true + k.calls = append(k.calls, c) + } +} + +func (k *collector) visit(n ast.Node) { + if n == nil { + return + } + ast.Inspect(n, func(nd ast.Node) bool { + switch x := nd.(type) { + case *ast.SelectorExpr: + if id, ok := x.X.(*ast.Ident); ok && k.ctx.isImport(id) { + r := Ref{Pkg: id.Name, Name: x.Sel.Name} + if !k.seenRef[r] { + k.seenRef[r] = true + k.refs = append(k.refs, r) + } + } + case *ast.CallExpr: + k.call(x) + } + return true + }) +} + +func (k *collector) call(x *ast.CallExpr) { + fun := x.Fun + for { // F[T](…) -> F + switch g := fun.(type) { + case *ast.IndexExpr: + fun = g.X + continue + case *ast.IndexListExpr: + fun = g.X + continue + case *ast.ParenExpr: + fun = g.X + continue + } + break + } + switch f := fun.(type) { + case *ast.Ident: + if exported(f.Name) { + k.addCall(Call{Name: f.Name}) + } + case *ast.SelectorExpr: + if !exported(f.Sel.Name) { + return + } + id, ok := f.X.(*ast.Ident) + if !ok { + k.addCall(Call{Name: f.Sel.Name}) + return + } + if k.ctx.isImport(id) { + k.addCall(Call{Pkg: id.Name, Name: f.Sel.Name}) + return + } + if pkg, typ, typed := k.ctx.identType(id); typed { + k.addCall(Call{Pkg: pkg, Type: typ, Name: f.Sel.Name}) + return + } + k.addCall(Call{Name: f.Sel.Name}) + } +} + +func recvTypeName(e ast.Expr) string { + switch x := e.(type) { + case *ast.StarExpr: + return recvTypeName(x.X) + case *ast.ParenExpr: + return recvTypeName(x.X) + case *ast.IndexExpr: + return recvTypeName(x.X) + case *ast.IndexListExpr: + return recvTypeName(x.X) + case *ast.Ident: + return x.Name + } + return "" +} + +func (c *fileCtx) typeObject(name string, doc string, t ast.Expr) Object { + k := newCollector(c) + k.visit(t) + o := Object{Name: name, Doc: doc, Refs: k.refs, Calls: []Call{}} + switch it := t.(type) { + case *ast.StructType: + o.Kind = "struct" + case *ast.InterfaceType: + o.Kind = "interface" + names := map[string]bool{} + for _, m := range it.Methods.List { + if _, isFn := m.Type.(*ast.FuncType); isFn { + for _, n := range m.Names { + if exported(n.Name) { + names[n.Name] = true + } + } + } + } + o.Methods = make([]string, 0, len(names)) + for n := range names { + o.Methods = append(o.Methods, n) + } + sort.Strings(o.Methods) + default: + o.Kind = "type" + } + return o +} + +func dump(path string) FileDump { + d := FileDump{File: path, Imports: map[string]string{}, Objects: []Object{}} + fset := token.NewFileSet() + af, err := parser.ParseFile(fset, path, nil, parser.ParseComments) + if err != nil { + d.Error = err.Error() + return d + } + d.Package = af.Name.Name + d.PackageDoc = docText(af.Doc) + + ctx := &fileCtx{imports: map[string]string{}} + for _, is := range af.Imports { + p, err := strconv.Unquote(is.Path.Value) + if err != nil { + continue + } + alias := guessPkgName(p) + if is.Name != nil { + if is.Name.Name == "_" || is.Name.Name == "." { + continue + } + alias = is.Name.Name + } + ctx.imports[alias] = p + d.Imports[alias] = p + } + + for _, decl := range af.Decls { + switch x := decl.(type) { + case *ast.FuncDecl: + k := newCollector(ctx) + k.visit(x.Type) + if x.Recv != nil { + k.visit(x.Recv) + } + if x.Body != nil { + k.visit(x.Body) + } + o := Object{Name: x.Name.Name, Kind: "func", Doc: docText(x.Doc), Refs: k.refs, Calls: k.calls} + if x.Recv != nil && len(x.Recv.List) == 1 { + if r := recvTypeName(x.Recv.List[0].Type); r != "" { + o.Kind = "method" + o.Receiver = r + } + } + d.Objects = append(d.Objects, o) + case *ast.GenDecl: + for _, spec := range x.Specs { + doc := "" + switch y := spec.(type) { + case *ast.TypeSpec: + doc = docText(y.Doc) + if doc == "" && !x.Lparen.IsValid() { + doc = docText(x.Doc) + } + d.Objects = append(d.Objects, ctx.typeObject(y.Name.Name, doc, y.Type)) + case *ast.ValueSpec: + doc = docText(y.Doc) + if doc == "" && !x.Lparen.IsValid() { + doc = docText(x.Doc) + } + kind := "var" + if x.Tok == token.CONST { + kind = "const" + } + for _, n := range y.Names { + if n.Name == "_" { + continue + } + d.Objects = append(d.Objects, Object{Name: n.Name, Kind: kind, Doc: doc, Refs: []Ref{}, Calls: []Call{}}) + } + } + } + } + } + return d +} + +func main() { + if len(os.Args) < 2 { + fmt.Fprintln(os.Stderr, "usage: goastdump [ ...]") + os.Exit(2) + } + out := struct { + Files []FileDump `json:"files"` + }{Files: []FileDump{}} + for _, p := range os.Args[1:] { + out.Files = append(out.Files, dump(p)) + } + enc := json.NewEncoder(os.Stdout) + enc.SetEscapeHTML(false) + if err := enc.Encode(out); err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } +} diff --git a/tests/test_tools/fixtures/object_model/llvm-cov-cic-canonical.json b/tests/test_tools/fixtures/object_model/llvm-cov-cic-canonical.json new file mode 100644 index 0000000..ed3ec75 --- /dev/null +++ b/tests/test_tools/fixtures/object_model/llvm-cov-cic-canonical.json @@ -0,0 +1,2745 @@ +{ + "cargo_llvm_cov": { + "manifest_path": "/w/Cargo.toml", + "version": "0.9.1" + }, + "data": [ + { + "files": [ + { + "branches": [], + "expansions": [], + "filename": "/w/cic-canonical/src/lib.rs", + "mcdc_records": [], + "segments": [ + [ + 24, + 1, + 23, + true, + true, + false + ], + [ + 24, + 51, + 0, + false, + false, + false + ], + [ + 25, + 9, + 23, + true, + true, + false + ], + [ + 25, + 10, + 0, + false, + false, + false + ], + [ + 25, + 13, + 23, + true, + true, + false + ], + [ + 25, + 14, + 0, + false, + false, + false + ], + [ + 25, + 15, + 23, + true, + true, + false + ], + [ + 25, + 19, + 0, + false, + false, + false + ], + [ + 26, + 8, + 23, + true, + true, + false + ], + [ + 26, + 9, + 0, + false, + false, + false + ], + [ + 26, + 10, + 23, + true, + true, + false + ], + [ + 26, + 18, + 0, + false, + false, + false + ], + [ + 27, + 16, + 2, + true, + true, + false + ], + [ + 27, + 20, + 0, + false, + false, + false + ], + [ + 28, + 5, + 21, + true, + true, + false + ], + [ + 28, + 6, + 0, + false, + false, + false + ], + [ + 29, + 17, + 10, + true, + true, + false + ], + [ + 29, + 20, + 0, + false, + false, + false + ], + [ + 29, + 24, + 21, + true, + true, + false + ], + [ + 29, + 44, + 0, + false, + false, + false + ], + [ + 30, + 16, + 10, + true, + true, + false + ], + [ + 30, + 25, + 0, + false, + false, + false + ], + [ + 31, + 5, + 11, + true, + true, + false + ], + [ + 31, + 6, + 0, + false, + false, + false + ], + [ + 32, + 9, + 6, + true, + true, + false + ], + [ + 32, + 10, + 0, + false, + false, + false + ], + [ + 32, + 12, + 6, + true, + true, + false + ], + [ + 32, + 15, + 0, + false, + false, + false + ], + [ + 32, + 18, + 11, + true, + true, + false + ], + [ + 32, + 19, + 0, + false, + false, + false + ], + [ + 32, + 20, + 11, + true, + true, + false + ], + [ + 32, + 25, + 0, + false, + false, + false + ], + [ + 32, + 28, + 11, + true, + true, + false + ], + [ + 32, + 30, + 0, + false, + false, + false + ], + [ + 32, + 32, + 5, + true, + true, + false + ], + [ + 32, + 33, + 0, + false, + false, + false + ], + [ + 33, + 5, + 6, + true, + true, + false + ], + [ + 33, + 29, + 0, + false, + false, + false + ], + [ + 34, + 1, + 23, + true, + true, + false + ], + [ + 34, + 2, + 0, + false, + false, + false + ], + [ + 39, + 1, + 21, + true, + true, + false + ], + [ + 39, + 48, + 0, + false, + false, + false + ], + [ + 40, + 10, + 21, + true, + true, + false + ], + [ + 40, + 13, + 0, + false, + false, + false + ], + [ + 40, + 15, + 21, + true, + true, + false + ], + [ + 40, + 21, + 0, + false, + false, + false + ], + [ + 40, + 31, + 21, + true, + true, + false + ], + [ + 40, + 50, + 0, + false, + false, + false + ], + [ + 41, + 14, + 4, + true, + true, + false + ], + [ + 41, + 18, + 0, + false, + false, + false + ], + [ + 41, + 23, + 4, + true, + true, + false + ], + [ + 41, + 35, + 0, + false, + false, + false + ], + [ + 42, + 17, + 17, + true, + true, + false + ], + [ + 42, + 25, + 17, + true, + true, + false + ], + [ + 42, + 26, + 17, + true, + false, + false + ], + [ + 42, + 27, + 17, + true, + true, + false + ], + [ + 42, + 39, + 17, + true, + false, + false + ], + [ + 42, + 58, + 0, + false, + false, + false + ], + [ + 44, + 8, + 21, + true, + true, + false + ], + [ + 44, + 14, + 0, + false, + false, + false + ], + [ + 44, + 15, + 21, + true, + true, + false + ], + [ + 44, + 23, + 0, + false, + false, + false + ], + [ + 44, + 30, + 21, + true, + true, + false + ], + [ + 44, + 44, + 0, + false, + false, + false + ], + [ + 44, + 45, + 21, + true, + true, + false + ], + [ + 44, + 48, + 0, + false, + false, + false + ], + [ + 44, + 53, + 93, + true, + true, + false + ], + [ + 44, + 54, + 0, + false, + false, + false + ], + [ + 44, + 55, + 93, + true, + true, + false + ], + [ + 44, + 69, + 0, + false, + false, + false + ], + [ + 45, + 16, + 11, + true, + true, + false + ], + [ + 45, + 20, + 0, + false, + false, + false + ], + [ + 46, + 5, + 10, + true, + true, + false + ], + [ + 46, + 6, + 0, + false, + false, + false + ], + [ + 47, + 9, + 10, + true, + true, + false + ], + [ + 47, + 18, + 0, + false, + false, + false + ], + [ + 47, + 21, + 10, + true, + true, + false + ], + [ + 47, + 27, + 0, + false, + false, + false + ], + [ + 47, + 28, + 10, + true, + true, + false + ], + [ + 47, + 46, + 0, + false, + false, + false + ], + [ + 48, + 8, + 10, + true, + true, + false + ], + [ + 48, + 17, + 0, + false, + false, + false + ], + [ + 48, + 18, + 10, + true, + true, + false + ], + [ + 48, + 26, + 0, + false, + false, + false + ], + [ + 49, + 16, + 1, + true, + true, + false + ], + [ + 49, + 37, + 0, + false, + false, + false + ], + [ + 50, + 5, + 9, + true, + true, + false + ], + [ + 50, + 6, + 0, + false, + false, + false + ], + [ + 51, + 13, + 9, + true, + true, + false + ], + [ + 51, + 16, + 0, + false, + false, + false + ], + [ + 52, + 9, + 1, + true, + true, + false + ], + [ + 52, + 16, + 0, + false, + false, + false + ], + [ + 54, + 9, + 8, + true, + true, + false + ], + [ + 54, + 18, + 0, + false, + false, + false + ], + [ + 54, + 19, + 8, + true, + true, + false + ], + [ + 54, + 28, + 0, + false, + false, + false + ], + [ + 56, + 1, + 21, + true, + true, + false + ], + [ + 56, + 2, + 0, + false, + false, + false + ], + [ + 61, + 1, + 6, + true, + true, + false + ], + [ + 61, + 37, + 0, + false, + false, + false + ], + [ + 62, + 9, + 6, + true, + true, + false + ], + [ + 62, + 10, + 0, + false, + false, + false + ], + [ + 62, + 13, + 6, + true, + true, + false + ], + [ + 62, + 20, + 0, + false, + false, + false + ], + [ + 63, + 8, + 6, + true, + true, + false + ], + [ + 63, + 17, + 0, + false, + false, + false + ], + [ + 64, + 9, + 1, + true, + true, + false + ], + [ + 64, + 12, + 0, + false, + false, + false + ], + [ + 64, + 13, + 1, + true, + true, + false + ], + [ + 64, + 22, + 0, + false, + false, + false + ], + [ + 66, + 9, + 5, + true, + true, + false + ], + [ + 66, + 10, + 0, + false, + false, + false + ], + [ + 68, + 1, + 6, + true, + true, + false + ], + [ + 68, + 2, + 0, + false, + false, + false + ], + [ + 75, + 5, + 1, + true, + true, + false + ], + [ + 75, + 27, + 0, + false, + false, + false + ], + [ + 77, + 13, + 1, + true, + true, + false + ], + [ + 77, + 18, + 0, + false, + false, + false + ], + [ + 77, + 21, + 1, + true, + true, + false + ], + [ + 78, + 13, + 1, + true, + true, + false + ], + [ + 78, + 23, + 1, + true, + false, + false + ], + [ + 79, + 13, + 1, + true, + true, + false + ], + [ + 79, + 25, + 1, + true, + false, + false + ], + [ + 80, + 13, + 1, + true, + true, + false + ], + [ + 80, + 27, + 1, + true, + false, + false + ], + [ + 81, + 13, + 1, + true, + true, + false + ], + [ + 81, + 25, + 1, + true, + false, + false + ], + [ + 82, + 13, + 1, + true, + true, + false + ], + [ + 82, + 28, + 1, + true, + false, + false + ], + [ + 83, + 13, + 1, + true, + true, + false + ], + [ + 83, + 28, + 1, + true, + false, + false + ], + [ + 84, + 13, + 1, + true, + true, + false + ], + [ + 84, + 25, + 1, + true, + false, + false + ], + [ + 85, + 13, + 1, + true, + true, + false + ], + [ + 85, + 24, + 1, + true, + false, + false + ], + [ + 86, + 13, + 1, + true, + true, + false + ], + [ + 86, + 26, + 1, + true, + false, + false + ], + [ + 87, + 13, + 1, + true, + true, + false + ], + [ + 87, + 25, + 1, + true, + false, + false + ], + [ + 88, + 13, + 1, + true, + true, + false + ], + [ + 88, + 24, + 1, + true, + false, + false + ], + [ + 89, + 13, + 1, + true, + true, + false + ], + [ + 89, + 25, + 1, + true, + false, + false + ], + [ + 91, + 13, + 1, + true, + true, + false + ], + [ + 91, + 53, + 1, + true, + false, + false + ], + [ + 93, + 10, + 0, + false, + false, + false + ], + [ + 94, + 14, + 14, + true, + true, + false + ], + [ + 94, + 19, + 0, + false, + false, + false + ], + [ + 94, + 21, + 14, + true, + true, + false + ], + [ + 94, + 25, + 0, + false, + false, + false + ], + [ + 94, + 30, + 1, + true, + true, + false + ], + [ + 94, + 35, + 0, + false, + false, + false + ], + [ + 95, + 13, + 14, + true, + true, + false + ], + [ + 95, + 23, + 0, + false, + false, + false + ], + [ + 96, + 17, + 14, + true, + true, + false + ], + [ + 96, + 40, + 0, + false, + false, + false + ], + [ + 96, + 41, + 14, + true, + true, + false + ], + [ + 96, + 49, + 0, + false, + false, + false + ], + [ + 97, + 17, + 14, + true, + true, + false + ], + [ + 97, + 27, + 0, + false, + false, + false + ], + [ + 101, + 5, + 1, + true, + true, + false + ], + [ + 101, + 6, + 0, + false, + false, + false + ], + [ + 104, + 5, + 1, + true, + true, + false + ], + [ + 104, + 29, + 0, + false, + false, + false + ], + [ + 105, + 13, + 7, + true, + true, + false + ], + [ + 105, + 18, + 0, + false, + false, + false + ], + [ + 105, + 22, + 1, + true, + true, + false + ], + [ + 105, + 27, + 1, + true, + true, + false + ], + [ + 105, + 31, + 1, + true, + false, + false + ], + [ + 105, + 33, + 1, + true, + true, + false + ], + [ + 105, + 38, + 1, + true, + false, + false + ], + [ + 105, + 40, + 1, + true, + true, + false + ], + [ + 105, + 47, + 1, + true, + false, + false + ], + [ + 105, + 49, + 1, + true, + true, + false + ], + [ + 105, + 55, + 1, + true, + false, + false + ], + [ + 105, + 69, + 0, + false, + false, + false + ], + [ + 106, + 13, + 7, + true, + true, + false + ], + [ + 106, + 23, + 0, + false, + false, + false + ], + [ + 106, + 24, + 7, + true, + true, + false + ], + [ + 106, + 40, + 0, + false, + false, + false + ], + [ + 106, + 41, + 7, + true, + true, + false + ], + [ + 106, + 46, + 0, + false, + false, + false + ], + [ + 108, + 5, + 1, + true, + true, + false + ], + [ + 108, + 6, + 0, + false, + false, + false + ], + [ + 111, + 5, + 1, + true, + true, + false + ], + [ + 111, + 40, + 0, + false, + false, + false + ], + [ + 113, + 9, + 1, + true, + true, + false + ], + [ + 113, + 19, + 0, + false, + false, + false + ], + [ + 114, + 13, + 1, + true, + true, + false + ], + [ + 114, + 29, + 0, + false, + false, + false + ], + [ + 114, + 30, + 1, + true, + true, + false + ], + [ + 114, + 48, + 0, + false, + false, + false + ], + [ + 115, + 13, + 1, + true, + true, + false + ], + [ + 115, + 29, + 0, + false, + false, + false + ], + [ + 115, + 30, + 1, + true, + true, + false + ], + [ + 115, + 48, + 0, + false, + false, + false + ], + [ + 117, + 5, + 1, + true, + true, + false + ], + [ + 117, + 6, + 0, + false, + false, + false + ] + ], + "summary": { + "branches": { + "count": 0, + "covered": 0, + "notcovered": 0, + "percent": 0.0 + }, + "functions": { + "count": 7, + "covered": 7, + "percent": 100.0 + }, + "instantiations": { + "count": 7, + "covered": 7, + "percent": 100.0 + }, + "lines": { + "count": 65, + "covered": 65, + "percent": 100.0 + }, + "mcdc": { + "count": 0, + "covered": 0, + "notcovered": 0, + "percent": 0.0 + }, + "regions": { + "count": 99, + "covered": 99, + "notcovered": 0, + "percent": 100.0 + } + } + } + ], + "functions": [ + { + "branches": [], + "count": 6, + "filenames": [ + "/w/cic-canonical/src/lib.rs" + ], + "mcdc_records": [], + "name": "_RNvCs65aHm1XuDSs_13cic_canonical15canonical_float", + "regions": [ + [ + 61, + 1, + 61, + 37, + 6, + 0, + 0, + 0 + ], + [ + 62, + 9, + 62, + 10, + 6, + 0, + 0, + 0 + ], + [ + 62, + 13, + 62, + 20, + 6, + 0, + 0, + 0 + ], + [ + 63, + 8, + 63, + 17, + 6, + 0, + 0, + 0 + ], + [ + 64, + 9, + 64, + 12, + 1, + 0, + 0, + 0 + ], + [ + 64, + 13, + 64, + 22, + 1, + 0, + 0, + 0 + ], + [ + 66, + 9, + 66, + 10, + 5, + 0, + 0, + 0 + ], + [ + 68, + 1, + 68, + 2, + 6, + 0, + 0, + 0 + ] + ] + }, + { + "branches": [], + "count": 23, + "filenames": [ + "/w/cic-canonical/src/lib.rs" + ], + "mcdc_records": [], + "name": "_RNvCs65aHm1XuDSs_13cic_canonical16canonical_number", + "regions": [ + [ + 24, + 1, + 24, + 51, + 23, + 0, + 0, + 0 + ], + [ + 25, + 9, + 25, + 10, + 23, + 0, + 0, + 0 + ], + [ + 25, + 13, + 25, + 14, + 23, + 0, + 0, + 0 + ], + [ + 25, + 15, + 25, + 19, + 23, + 0, + 0, + 0 + ], + [ + 26, + 8, + 26, + 9, + 23, + 0, + 0, + 0 + ], + [ + 26, + 10, + 26, + 18, + 23, + 0, + 0, + 0 + ], + [ + 27, + 16, + 27, + 20, + 2, + 0, + 0, + 0 + ], + [ + 28, + 5, + 28, + 6, + 21, + 0, + 0, + 0 + ], + [ + 29, + 17, + 29, + 20, + 10, + 0, + 0, + 0 + ], + [ + 29, + 24, + 29, + 44, + 21, + 0, + 0, + 0 + ], + [ + 30, + 16, + 30, + 25, + 10, + 0, + 0, + 0 + ], + [ + 31, + 5, + 31, + 6, + 11, + 0, + 0, + 0 + ], + [ + 32, + 9, + 32, + 10, + 6, + 0, + 0, + 0 + ], + [ + 32, + 12, + 32, + 15, + 6, + 0, + 0, + 0 + ], + [ + 32, + 18, + 32, + 19, + 11, + 0, + 0, + 0 + ], + [ + 32, + 20, + 32, + 25, + 11, + 0, + 0, + 0 + ], + [ + 32, + 28, + 32, + 30, + 11, + 0, + 0, + 0 + ], + [ + 32, + 32, + 32, + 33, + 5, + 0, + 0, + 0 + ], + [ + 33, + 5, + 33, + 29, + 6, + 0, + 0, + 0 + ], + [ + 34, + 1, + 34, + 2, + 23, + 0, + 0, + 0 + ] + ] + }, + { + "branches": [], + "count": 21, + "filenames": [ + "/w/cic-canonical/src/lib.rs" + ], + "mcdc_records": [], + "name": "_RNvCs65aHm1XuDSs_13cic_canonical17canonical_integer", + "regions": [ + [ + 39, + 1, + 39, + 48, + 21, + 0, + 0, + 0 + ], + [ + 40, + 10, + 40, + 13, + 21, + 0, + 0, + 0 + ], + [ + 40, + 15, + 40, + 21, + 21, + 0, + 0, + 0 + ], + [ + 40, + 31, + 40, + 50, + 21, + 0, + 0, + 0 + ], + [ + 41, + 14, + 41, + 18, + 4, + 0, + 0, + 0 + ], + [ + 41, + 23, + 41, + 35, + 4, + 0, + 0, + 0 + ], + [ + 42, + 17, + 42, + 58, + 17, + 0, + 0, + 0 + ], + [ + 42, + 25, + 42, + 26, + 17, + 0, + 0, + 0 + ], + [ + 42, + 27, + 42, + 39, + 17, + 0, + 0, + 0 + ], + [ + 44, + 8, + 44, + 14, + 21, + 0, + 0, + 0 + ], + [ + 44, + 15, + 44, + 23, + 21, + 0, + 0, + 0 + ], + [ + 44, + 30, + 44, + 44, + 21, + 0, + 0, + 0 + ], + [ + 44, + 45, + 44, + 48, + 21, + 0, + 0, + 0 + ], + [ + 45, + 16, + 45, + 20, + 11, + 0, + 0, + 0 + ], + [ + 46, + 5, + 46, + 6, + 10, + 0, + 0, + 0 + ], + [ + 47, + 9, + 47, + 18, + 10, + 0, + 0, + 0 + ], + [ + 47, + 21, + 47, + 27, + 10, + 0, + 0, + 0 + ], + [ + 47, + 28, + 47, + 46, + 10, + 0, + 0, + 0 + ], + [ + 48, + 8, + 48, + 17, + 10, + 0, + 0, + 0 + ], + [ + 48, + 18, + 48, + 26, + 10, + 0, + 0, + 0 + ], + [ + 49, + 16, + 49, + 37, + 1, + 0, + 0, + 0 + ], + [ + 50, + 5, + 50, + 6, + 9, + 0, + 0, + 0 + ], + [ + 51, + 13, + 51, + 16, + 9, + 0, + 0, + 0 + ], + [ + 52, + 9, + 52, + 16, + 1, + 0, + 0, + 0 + ], + [ + 54, + 9, + 54, + 18, + 8, + 0, + 0, + 0 + ], + [ + 54, + 19, + 54, + 28, + 8, + 0, + 0, + 0 + ], + [ + 56, + 1, + 56, + 2, + 21, + 0, + 0, + 0 + ] + ] + }, + { + "branches": [], + "count": 1, + "filenames": [ + "/w/cic-canonical/src/lib.rs" + ], + "mcdc_records": [], + "name": "_RNvNtCs65aHm1XuDSs_13cic_canonical5testss_17vectors_mirror_go", + "regions": [ + [ + 75, + 5, + 75, + 27, + 1, + 0, + 0, + 0 + ], + [ + 77, + 13, + 77, + 18, + 1, + 0, + 0, + 0 + ], + [ + 77, + 21, + 93, + 10, + 1, + 0, + 0, + 0 + ], + [ + 78, + 13, + 78, + 23, + 1, + 0, + 0, + 0 + ], + [ + 79, + 13, + 79, + 25, + 1, + 0, + 0, + 0 + ], + [ + 80, + 13, + 80, + 27, + 1, + 0, + 0, + 0 + ], + [ + 81, + 13, + 81, + 25, + 1, + 0, + 0, + 0 + ], + [ + 82, + 13, + 82, + 28, + 1, + 0, + 0, + 0 + ], + [ + 83, + 13, + 83, + 28, + 1, + 0, + 0, + 0 + ], + [ + 84, + 13, + 84, + 25, + 1, + 0, + 0, + 0 + ], + [ + 85, + 13, + 85, + 24, + 1, + 0, + 0, + 0 + ], + [ + 86, + 13, + 86, + 26, + 1, + 0, + 0, + 0 + ], + [ + 87, + 13, + 87, + 25, + 1, + 0, + 0, + 0 + ], + [ + 88, + 13, + 88, + 24, + 1, + 0, + 0, + 0 + ], + [ + 89, + 13, + 89, + 25, + 1, + 0, + 0, + 0 + ], + [ + 91, + 13, + 91, + 53, + 1, + 0, + 0, + 0 + ], + [ + 94, + 14, + 94, + 19, + 14, + 0, + 0, + 0 + ], + [ + 94, + 21, + 94, + 25, + 14, + 0, + 0, + 0 + ], + [ + 94, + 30, + 94, + 35, + 1, + 0, + 0, + 0 + ], + [ + 95, + 13, + 95, + 23, + 14, + 0, + 0, + 0 + ], + [ + 96, + 17, + 96, + 40, + 14, + 0, + 0, + 0 + ], + [ + 96, + 41, + 96, + 49, + 14, + 0, + 0, + 0 + ], + [ + 97, + 17, + 97, + 27, + 14, + 0, + 0, + 0 + ], + [ + 101, + 5, + 101, + 6, + 1, + 0, + 0, + 0 + ] + ] + }, + { + "branches": [], + "count": 1, + "filenames": [ + "/w/cic-canonical/src/lib.rs" + ], + "mcdc_records": [], + "name": "_RNvNtCs65aHm1XuDSs_13cic_canonical5testss_19non_numeric_is_none", + "regions": [ + [ + 104, + 5, + 104, + 29, + 1, + 0, + 0, + 0 + ], + [ + 105, + 13, + 105, + 18, + 7, + 0, + 0, + 0 + ], + [ + 105, + 22, + 105, + 69, + 1, + 0, + 0, + 0 + ], + [ + 105, + 27, + 105, + 31, + 1, + 0, + 0, + 0 + ], + [ + 105, + 33, + 105, + 38, + 1, + 0, + 0, + 0 + ], + [ + 105, + 40, + 105, + 47, + 1, + 0, + 0, + 0 + ], + [ + 105, + 49, + 105, + 55, + 1, + 0, + 0, + 0 + ], + [ + 106, + 13, + 106, + 23, + 7, + 0, + 0, + 0 + ], + [ + 106, + 24, + 106, + 40, + 7, + 0, + 0, + 0 + ], + [ + 106, + 41, + 106, + 46, + 7, + 0, + 0, + 0 + ], + [ + 108, + 5, + 108, + 6, + 1, + 0, + 0, + 0 + ] + ] + }, + { + "branches": [], + "count": 1, + "filenames": [ + "/w/cic-canonical/src/lib.rs" + ], + "mcdc_records": [], + "name": "_RNvNtCs65aHm1XuDSs_13cic_canonical5testss_30adjacent_large_integers_differ", + "regions": [ + [ + 111, + 5, + 111, + 40, + 1, + 0, + 0, + 0 + ], + [ + 113, + 9, + 113, + 19, + 1, + 0, + 0, + 0 + ], + [ + 114, + 13, + 114, + 29, + 1, + 0, + 0, + 0 + ], + [ + 114, + 30, + 114, + 48, + 1, + 0, + 0, + 0 + ], + [ + 115, + 13, + 115, + 29, + 1, + 0, + 0, + 0 + ], + [ + 115, + 30, + 115, + 48, + 1, + 0, + 0, + 0 + ], + [ + 117, + 5, + 117, + 6, + 1, + 0, + 0, + 0 + ] + ] + }, + { + "branches": [], + "count": 93, + "filenames": [ + "/w/cic-canonical/src/lib.rs" + ], + "mcdc_records": [], + "name": "_RNCNvCs65aHm1XuDSs_13cic_canonical17canonical_integer0B3_", + "regions": [ + [ + 44, + 53, + 44, + 54, + 93, + 0, + 0, + 0 + ], + [ + 44, + 55, + 44, + 69, + 93, + 0, + 0, + 0 + ] + ] + } + ], + "totals": { + "branches": { + "count": 0, + "covered": 0, + "notcovered": 0, + "percent": 0 + }, + "functions": { + "count": 414, + "covered": 400, + "percent": 96.61835748792271 + }, + "instantiations": { + "count": 697, + "covered": 555, + "percent": 79.62697274031564 + }, + "lines": { + "count": 2685, + "covered": 2644, + "percent": 98.4729981378026 + }, + "mcdc": { + "count": 0, + "covered": 0, + "notcovered": 0, + "percent": 0 + }, + "regions": { + "count": 4728, + "covered": 4617, + "notcovered": 111, + "percent": 97.65228426395939 + } + } + } + ], + "type": "llvm.coverage.json.export", + "version": "3.1.0" +} \ No newline at end of file diff --git a/tests/test_tools/fixtures/object_model/mutants-cic-canonical.json b/tests/test_tools/fixtures/object_model/mutants-cic-canonical.json new file mode 100644 index 0000000..37b4405 --- /dev/null +++ b/tests/test_tools/fixtures/object_model/mutants-cic-canonical.json @@ -0,0 +1,343 @@ +[ + { + "file": "cic-canonical/src/lib.rs", + "function": { + "function_name": "canonical_number", + "return_type": "-> Option", + "span": { + "end": { + "column": 2, + "line": 34 + }, + "start": { + "column": 1, + "line": 17 + } + } + }, + "genre": "FnValue", + "name": "cic-canonical/src/lib.rs:25:5: replace canonical_number -> Option with None", + "package": "cic-canonical", + "replacement": "None", + "span": { + "end": { + "column": 29, + "line": 33 + }, + "start": { + "column": 5, + "line": 25 + } + } + }, + { + "file": "cic-canonical/src/lib.rs", + "function": { + "function_name": "canonical_number", + "return_type": "-> Option", + "span": { + "end": { + "column": 2, + "line": 34 + }, + "start": { + "column": 1, + "line": 17 + } + } + }, + "genre": "FnValue", + "name": "cic-canonical/src/lib.rs:25:5: replace canonical_number -> Option with Some(String::new())", + "package": "cic-canonical", + "replacement": "Some(String::new())", + "span": { + "end": { + "column": 29, + "line": 33 + }, + "start": { + "column": 5, + "line": 25 + } + } + }, + { + "file": "cic-canonical/src/lib.rs", + "function": { + "function_name": "canonical_number", + "return_type": "-> Option", + "span": { + "end": { + "column": 2, + "line": 34 + }, + "start": { + "column": 1, + "line": 17 + } + } + }, + "genre": "FnValue", + "name": "cic-canonical/src/lib.rs:25:5: replace canonical_number -> Option with Some(\"xyzzy\".into())", + "package": "cic-canonical", + "replacement": "Some(\"xyzzy\".into())", + "span": { + "end": { + "column": 29, + "line": 33 + }, + "start": { + "column": 5, + "line": 25 + } + } + }, + { + "file": "cic-canonical/src/lib.rs", + "function": { + "function_name": "canonical_integer", + "return_type": "-> Option", + "span": { + "end": { + "column": 2, + "line": 56 + }, + "start": { + "column": 1, + "line": 36 + } + } + }, + "genre": "FnValue", + "name": "cic-canonical/src/lib.rs:40:5: replace canonical_integer -> Option with None", + "package": "cic-canonical", + "replacement": "None", + "span": { + "end": { + "column": 7, + "line": 55 + }, + "start": { + "column": 5, + "line": 40 + } + } + }, + { + "file": "cic-canonical/src/lib.rs", + "function": { + "function_name": "canonical_integer", + "return_type": "-> Option", + "span": { + "end": { + "column": 2, + "line": 56 + }, + "start": { + "column": 1, + "line": 36 + } + } + }, + "genre": "FnValue", + "name": "cic-canonical/src/lib.rs:40:5: replace canonical_integer -> Option with Some(String::new())", + "package": "cic-canonical", + "replacement": "Some(String::new())", + "span": { + "end": { + "column": 7, + "line": 55 + }, + "start": { + "column": 5, + "line": 40 + } + } + }, + { + "file": "cic-canonical/src/lib.rs", + "function": { + "function_name": "canonical_integer", + "return_type": "-> Option", + "span": { + "end": { + "column": 2, + "line": 56 + }, + "start": { + "column": 1, + "line": 36 + } + } + }, + "genre": "FnValue", + "name": "cic-canonical/src/lib.rs:40:5: replace canonical_integer -> Option with Some(\"xyzzy\".into())", + "package": "cic-canonical", + "replacement": "Some(\"xyzzy\".into())", + "span": { + "end": { + "column": 7, + "line": 55 + }, + "start": { + "column": 5, + "line": 40 + } + } + }, + { + "file": "cic-canonical/src/lib.rs", + "function": { + "function_name": "canonical_integer", + "return_type": "-> Option", + "span": { + "end": { + "column": 2, + "line": 56 + }, + "start": { + "column": 1, + "line": 36 + } + } + }, + "genre": "BinaryOperator", + "name": "cic-canonical/src/lib.rs:44:26: replace || with && in canonical_integer", + "package": "cic-canonical", + "replacement": "&&", + "span": { + "end": { + "column": 28, + "line": 44 + }, + "start": { + "column": 26, + "line": 44 + } + } + }, + { + "file": "cic-canonical/src/lib.rs", + "function": { + "function_name": "canonical_integer", + "return_type": "-> Option", + "span": { + "end": { + "column": 2, + "line": 56 + }, + "start": { + "column": 1, + "line": 36 + } + } + }, + "genre": "UnaryOperator", + "name": "cic-canonical/src/lib.rs:44:29: delete ! in canonical_integer", + "package": "cic-canonical", + "replacement": "", + "span": { + "end": { + "column": 30, + "line": 44 + }, + "start": { + "column": 29, + "line": 44 + } + } + }, + { + "file": "cic-canonical/src/lib.rs", + "function": { + "function_name": "canonical_float", + "return_type": "-> String", + "span": { + "end": { + "column": 2, + "line": 68 + }, + "start": { + "column": 1, + "line": 58 + } + } + }, + "genre": "FnValue", + "name": "cic-canonical/src/lib.rs:62:5: replace canonical_float -> String with String::new()", + "package": "cic-canonical", + "replacement": "String::new()", + "span": { + "end": { + "column": 6, + "line": 67 + }, + "start": { + "column": 5, + "line": 62 + } + } + }, + { + "file": "cic-canonical/src/lib.rs", + "function": { + "function_name": "canonical_float", + "return_type": "-> String", + "span": { + "end": { + "column": 2, + "line": 68 + }, + "start": { + "column": 1, + "line": 58 + } + } + }, + "genre": "FnValue", + "name": "cic-canonical/src/lib.rs:62:5: replace canonical_float -> String with \"xyzzy\".into()", + "package": "cic-canonical", + "replacement": "\"xyzzy\".into()", + "span": { + "end": { + "column": 6, + "line": 67 + }, + "start": { + "column": 5, + "line": 62 + } + } + }, + { + "file": "cic-canonical/src/lib.rs", + "function": { + "function_name": "canonical_float", + "return_type": "-> String", + "span": { + "end": { + "column": 2, + "line": 68 + }, + "start": { + "column": 1, + "line": 58 + } + } + }, + "genre": "BinaryOperator", + "name": "cic-canonical/src/lib.rs:63:10: replace == with != in canonical_float", + "package": "cic-canonical", + "replacement": "!=", + "span": { + "end": { + "column": 12, + "line": 63 + }, + "start": { + "column": 10, + "line": 63 + } + } + } +] \ No newline at end of file diff --git a/tests/test_tools/fixtures/object_model/outcomes-cic-canonical.json b/tests/test_tools/fixtures/object_model/outcomes-cic-canonical.json new file mode 100644 index 0000000..ebb5525 --- /dev/null +++ b/tests/test_tools/fixtures/object_model/outcomes-cic-canonical.json @@ -0,0 +1,446 @@ +{ + "cargo_mutants_version": "27.1.0", + "caught": 11, + "missed": 0, + "outcomes": [ + { + "diff_path": null, + "log_path": "", + "phase_results": [], + "scenario": "Baseline", + "summary": "Success" + }, + { + "diff_path": "diff/cic-canonical__src__lib.rs_line_25_col_5.diff", + "log_path": "", + "phase_results": [], + "scenario": { + "Mutant": { + "file": "cic-canonical/src/lib.rs", + "function": { + "function_name": "canonical_number", + "return_type": "-> Option", + "span": { + "end": { + "column": 2, + "line": 34 + }, + "start": { + "column": 1, + "line": 17 + } + } + }, + "genre": "FnValue", + "name": "cic-canonical/src/lib.rs:25:5: replace canonical_number -> Option with None", + "package": "cic-canonical", + "replacement": "None", + "span": { + "end": { + "column": 29, + "line": 33 + }, + "start": { + "column": 5, + "line": 25 + } + } + } + }, + "summary": "CaughtMutant" + }, + { + "diff_path": "diff/cic-canonical__src__lib.rs_line_25_col_5_001.diff", + "log_path": "", + "phase_results": [], + "scenario": { + "Mutant": { + "file": "cic-canonical/src/lib.rs", + "function": { + "function_name": "canonical_number", + "return_type": "-> Option", + "span": { + "end": { + "column": 2, + "line": 34 + }, + "start": { + "column": 1, + "line": 17 + } + } + }, + "genre": "FnValue", + "name": "cic-canonical/src/lib.rs:25:5: replace canonical_number -> Option with Some(String::new())", + "package": "cic-canonical", + "replacement": "Some(String::new())", + "span": { + "end": { + "column": 29, + "line": 33 + }, + "start": { + "column": 5, + "line": 25 + } + } + } + }, + "summary": "CaughtMutant" + }, + { + "diff_path": "diff/cic-canonical__src__lib.rs_line_25_col_5_002.diff", + "log_path": "", + "phase_results": [], + "scenario": { + "Mutant": { + "file": "cic-canonical/src/lib.rs", + "function": { + "function_name": "canonical_number", + "return_type": "-> Option", + "span": { + "end": { + "column": 2, + "line": 34 + }, + "start": { + "column": 1, + "line": 17 + } + } + }, + "genre": "FnValue", + "name": "cic-canonical/src/lib.rs:25:5: replace canonical_number -> Option with Some(\"xyzzy\".into())", + "package": "cic-canonical", + "replacement": "Some(\"xyzzy\".into())", + "span": { + "end": { + "column": 29, + "line": 33 + }, + "start": { + "column": 5, + "line": 25 + } + } + } + }, + "summary": "CaughtMutant" + }, + { + "diff_path": "diff/cic-canonical__src__lib.rs_line_40_col_5.diff", + "log_path": "", + "phase_results": [], + "scenario": { + "Mutant": { + "file": "cic-canonical/src/lib.rs", + "function": { + "function_name": "canonical_integer", + "return_type": "-> Option", + "span": { + "end": { + "column": 2, + "line": 56 + }, + "start": { + "column": 1, + "line": 36 + } + } + }, + "genre": "FnValue", + "name": "cic-canonical/src/lib.rs:40:5: replace canonical_integer -> Option with None", + "package": "cic-canonical", + "replacement": "None", + "span": { + "end": { + "column": 7, + "line": 55 + }, + "start": { + "column": 5, + "line": 40 + } + } + } + }, + "summary": "CaughtMutant" + }, + { + "diff_path": "diff/cic-canonical__src__lib.rs_line_40_col_5_001.diff", + "log_path": "", + "phase_results": [], + "scenario": { + "Mutant": { + "file": "cic-canonical/src/lib.rs", + "function": { + "function_name": "canonical_integer", + "return_type": "-> Option", + "span": { + "end": { + "column": 2, + "line": 56 + }, + "start": { + "column": 1, + "line": 36 + } + } + }, + "genre": "FnValue", + "name": "cic-canonical/src/lib.rs:40:5: replace canonical_integer -> Option with Some(String::new())", + "package": "cic-canonical", + "replacement": "Some(String::new())", + "span": { + "end": { + "column": 7, + "line": 55 + }, + "start": { + "column": 5, + "line": 40 + } + } + } + }, + "summary": "CaughtMutant" + }, + { + "diff_path": "diff/cic-canonical__src__lib.rs_line_40_col_5_002.diff", + "log_path": "", + "phase_results": [], + "scenario": { + "Mutant": { + "file": "cic-canonical/src/lib.rs", + "function": { + "function_name": "canonical_integer", + "return_type": "-> Option", + "span": { + "end": { + "column": 2, + "line": 56 + }, + "start": { + "column": 1, + "line": 36 + } + } + }, + "genre": "FnValue", + "name": "cic-canonical/src/lib.rs:40:5: replace canonical_integer -> Option with Some(\"xyzzy\".into())", + "package": "cic-canonical", + "replacement": "Some(\"xyzzy\".into())", + "span": { + "end": { + "column": 7, + "line": 55 + }, + "start": { + "column": 5, + "line": 40 + } + } + } + }, + "summary": "CaughtMutant" + }, + { + "diff_path": "diff/cic-canonical__src__lib.rs_line_44_col_26.diff", + "log_path": "", + "phase_results": [], + "scenario": { + "Mutant": { + "file": "cic-canonical/src/lib.rs", + "function": { + "function_name": "canonical_integer", + "return_type": "-> Option", + "span": { + "end": { + "column": 2, + "line": 56 + }, + "start": { + "column": 1, + "line": 36 + } + } + }, + "genre": "BinaryOperator", + "name": "cic-canonical/src/lib.rs:44:26: replace || with && in canonical_integer", + "package": "cic-canonical", + "replacement": "&&", + "span": { + "end": { + "column": 28, + "line": 44 + }, + "start": { + "column": 26, + "line": 44 + } + } + } + }, + "summary": "CaughtMutant" + }, + { + "diff_path": "diff/cic-canonical__src__lib.rs_line_44_col_29.diff", + "log_path": "", + "phase_results": [], + "scenario": { + "Mutant": { + "file": "cic-canonical/src/lib.rs", + "function": { + "function_name": "canonical_integer", + "return_type": "-> Option", + "span": { + "end": { + "column": 2, + "line": 56 + }, + "start": { + "column": 1, + "line": 36 + } + } + }, + "genre": "UnaryOperator", + "name": "cic-canonical/src/lib.rs:44:29: delete ! in canonical_integer", + "package": "cic-canonical", + "replacement": "", + "span": { + "end": { + "column": 30, + "line": 44 + }, + "start": { + "column": 29, + "line": 44 + } + } + } + }, + "summary": "CaughtMutant" + }, + { + "diff_path": "diff/cic-canonical__src__lib.rs_line_62_col_5.diff", + "log_path": "", + "phase_results": [], + "scenario": { + "Mutant": { + "file": "cic-canonical/src/lib.rs", + "function": { + "function_name": "canonical_float", + "return_type": "-> String", + "span": { + "end": { + "column": 2, + "line": 68 + }, + "start": { + "column": 1, + "line": 58 + } + } + }, + "genre": "FnValue", + "name": "cic-canonical/src/lib.rs:62:5: replace canonical_float -> String with String::new()", + "package": "cic-canonical", + "replacement": "String::new()", + "span": { + "end": { + "column": 6, + "line": 67 + }, + "start": { + "column": 5, + "line": 62 + } + } + } + }, + "summary": "CaughtMutant" + }, + { + "diff_path": "diff/cic-canonical__src__lib.rs_line_62_col_5_001.diff", + "log_path": "", + "phase_results": [], + "scenario": { + "Mutant": { + "file": "cic-canonical/src/lib.rs", + "function": { + "function_name": "canonical_float", + "return_type": "-> String", + "span": { + "end": { + "column": 2, + "line": 68 + }, + "start": { + "column": 1, + "line": 58 + } + } + }, + "genre": "FnValue", + "name": "cic-canonical/src/lib.rs:62:5: replace canonical_float -> String with \"xyzzy\".into()", + "package": "cic-canonical", + "replacement": "\"xyzzy\".into()", + "span": { + "end": { + "column": 6, + "line": 67 + }, + "start": { + "column": 5, + "line": 62 + } + } + } + }, + "summary": "CaughtMutant" + }, + { + "diff_path": "diff/cic-canonical__src__lib.rs_line_63_col_10.diff", + "log_path": "", + "phase_results": [], + "scenario": { + "Mutant": { + "file": "cic-canonical/src/lib.rs", + "function": { + "function_name": "canonical_float", + "return_type": "-> String", + "span": { + "end": { + "column": 2, + "line": 68 + }, + "start": { + "column": 1, + "line": 58 + } + } + }, + "genre": "BinaryOperator", + "name": "cic-canonical/src/lib.rs:63:10: replace == with != in canonical_float", + "package": "cic-canonical", + "replacement": "!=", + "span": { + "end": { + "column": 12, + "line": 63 + }, + "start": { + "column": 10, + "line": 63 + } + } + } + }, + "summary": "CaughtMutant" + } + ], + "timeout": 0, + "total_mutants": 11, + "unviable": 0 +} \ No newline at end of file diff --git a/tests/test_tools/test_mutation_policy.py b/tests/test_tools/test_mutation_policy.py new file mode 100644 index 0000000..197e21c --- /dev/null +++ b/tests/test_tools/test_mutation_policy.py @@ -0,0 +1,211 @@ +"""mutation-policy/2 (owner decisions of 2026-11-14): the primary score and the unviable metric. + +R1 is a regression test of a CHANGED behaviour: it runs `quality_gate.py verify` on real +cargo-mutants outcomes and reads the printed primary score. On the previous tree (policy/1, +primary = caught/planned) the output has no policy/2 primary score at all and R1 fails there. +""" + +import copy +import json +import os +import subprocess +import sys + +import pytest + +SCRIPTS = os.path.abspath(os.path.join(os.path.dirname(__file__), "../../scripts")) +sys.path.insert(0, SCRIPTS) + +import quality_gate as qg # noqa: E402 + +FIX = os.path.join(os.path.dirname(__file__), "fixtures/quality_gate") +GATE = os.path.join(SCRIPTS, "quality_gate.py") + + +def real(name): + with open(os.path.join(FIX, name)) as fh: + return json.load(fh) + + +def with_summaries(summaries): + """Real outcomes where mutant #i has summaries[i]; the tool's own counters kept consistent.""" + o = copy.deepcopy(real("cargo-mutants-27.1.0-outcomes.json")) + mutants = [r for r in o["outcomes"] if r["scenario"] != "Baseline"] + for field in ("caught", "missed", "timeout", "unviable"): + o[field] = 0 + for row, summary in zip(mutants, summaries): + row["summary"] = summary + o[qg.SUMMARY_TO_VERDICT[summary]] += 1 + return o + + +def run_verify(tmp_path, summaries): + """`quality_gate.py verify` on a plan of the 5 real mutants and these verdicts; (rc, stdout).""" + entries = real("cargo-mutants-27.1.0-list.json") + plan = qg.build_rust_plan( + json.dumps(entries).encode(), 0, ["cargo"], "27.1.0", "full" + ) + out = with_summaries(summaries) + rep = { + "schema": qg.SHARD_SCHEMA, + "plan_digest": plan["digest"], + "shard": 0, + "shards": 1, + "tool_version": out["cargo_mutants_version"], + "outcomes": out, + } + (tmp_path / "plan.json").write_text(json.dumps(plan)) + rdir = tmp_path / "reports" + rdir.mkdir(exist_ok=True) + (rdir / "shard-0.json").write_text(json.dumps(rep)) + r = subprocess.run( + [ + sys.executable, + GATE, + "verify", + "--rust-plan", + str(tmp_path / "plan.json"), + "--rust-reports", + str(rdir), + "--rust-shards", + "1", + ], + capture_output=True, + text=True, + ) + return r.returncode, r.stdout + r.stderr + + +C, M, T, U = "CaughtMutant", "MissedMutant", "Timeout", "Unviable" + + +def primary_line(text): + lines = [ln for ln in text.splitlines() if "(primary" in ln] + assert lines, f"no primary score line in the output:\n{text}" + return lines[0] + + +def test_r1_an_unviable_mutant_does_not_change_the_primary_score(tmp_path): + (tmp_path / "a").mkdir() + (tmp_path / "b").mkdir() + rc_a, a = run_verify(tmp_path / "a", [C] * 5) + rc_b, b = run_verify(tmp_path / "b", [C, C, C, C, U]) + assert rc_a == 0 and rc_b == 0 + assert "mutation-policy/2" in primary_line(a) + assert "caught/(caught+missed) = 5/5 = 100.00%" in primary_line(a) + # one caught mutant became unviable: the numerator and the denominator both lose it + assert "caught/(caught+missed) = 4/4 = 100.00%" in primary_line(b) + assert "unviable (mandatory metric, report-only): 1/5 = 20.00%" in b + + +def test_the_unviable_ratio_is_always_printed_even_when_zero(tmp_path): + (tmp_path / "z").mkdir() + _, text = run_verify(tmp_path / "z", [C] * 5) + assert "unviable (mandatory metric, report-only): 0/5 = 0.00%" in text + + +def test_the_legacy_primary_of_policy_1_stays_visible(tmp_path): + (tmp_path / "l").mkdir() + _, text = run_verify(tmp_path / "l", [C, C, C, C, U]) + assert "legacy (policy/1 primary): caught/planned = 4/5" in text + + +@pytest.mark.parametrize( + "summaries, rc, shown", + [ + ([C] * 5, 0, "5/5 = 100.00%"), + ( + [C, C, C, C, M], + 1, + "4/5 = 80.00%", + ), # a survivor: red, and the score is a number below 100 + ( + [U] * 5, + 0, + "NOT_APPLICABLE (ALL_UNVIABLE)", + ), # nothing evaluable: not 100%; unviable does not fail + ( + [U, U, U, U, T], + 1, + "NOT_APPLICABLE (NO_EVALUABLE_MUTANT)", + ), # a timeout is red; the score does not apply + ], +) +def test_primary_score_states_and_gate_result(tmp_path, summaries, rc, shown): + (tmp_path / "s").mkdir() + got_rc, text = run_verify(tmp_path / "s", summaries) + assert got_rc == rc, text + assert shown in primary_line(text) + + +def test_a_survivor_is_still_red_whatever_the_score_is(tmp_path): + # the 90% target is a reported value: it never replaces `missed > 0` + (tmp_path / "m").mkdir() + rc, text = run_verify(tmp_path / "m", [C, C, C, C, M]) + assert rc == 1 and "missed=1" in text + + +# --- the zero denominator (N12) -------------------------------------------------------------- + + +def test_zero_denominator_is_neither_100_nor_0(): + cases = { + "NO_MUTANTS_PLANNED": { + "planned": 0, + "caught": 0, + "missed": 0, + "timeout": 0, + "unviable": 0, + }, + "ALL_UNVIABLE": { + "planned": 4, + "caught": 0, + "missed": 0, + "timeout": 0, + "unviable": 4, + }, + "NO_EVALUABLE_MUTANT": { + "planned": 4, + "caught": 0, + "missed": 0, + "timeout": 1, + "unviable": 3, + }, + } + for reason, acct in cases.items(): + s = qg.primary_score(acct) + assert s == {"state": "NOT_APPLICABLE", "reason": reason}, (reason, s) + assert "value" not in s + + +@pytest.mark.parametrize("acct", [None, {}, {"caught": 3}]) +def test_a_missing_accounting_is_unverifiable_not_a_score(acct): + assert qg.primary_score(acct) == { + "state": "UNVERIFIABLE", + "reason": "MEASUREMENT_MISSING", + } + assert qg.unviable_ratio(acct) == { + "state": "UNVERIFIABLE", + "reason": "MEASUREMENT_MISSING", + } + + +def test_primary_score_measures_only_evaluable_mutants(): + s = qg.primary_score( + {"planned": 10, "caught": 6, "missed": 2, "timeout": 1, "unviable": 1} + ) + assert ( + s["state"] == "MEASURED" + and s["value"] == 75.0 + and (s["caught"], s["evaluable"]) == (6, 8) + ) + + +def test_unviable_ratio(): + assert qg.unviable_ratio({"planned": 300, "unviable": 83})[ + "value" + ] == pytest.approx(27.6667, abs=1e-3) + assert qg.unviable_ratio({"planned": 0, "unviable": 0}) == { + "state": "NOT_APPLICABLE", + "reason": "NO_MUTANTS_PLANNED", + } diff --git a/tests/test_tools/test_object_evidence_workflow.py b/tests/test_tools/test_object_evidence_workflow.py new file mode 100644 index 0000000..93d0685 --- /dev/null +++ b/tests/test_tools/test_object_evidence_workflow.py @@ -0,0 +1,116 @@ +"""The object-evidence workflow is report-only and cannot gate or skip anything (PR-04).""" + +import os +import subprocess +import sys + +import pytest +import yaml + +ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), "../..")) +WF = os.path.join(ROOT, ".github/workflows/object-evidence.yml") + + +def load(name="object-evidence.yml"): + with open(os.path.join(ROOT, ".github/workflows", name)) as fh: + return yaml.safe_load(fh) + + +def test_the_workflow_runs_on_pull_requests_and_by_hand_only(): + on = load().get("on") or load().get(True) + assert set(on) == {"pull_request", "workflow_dispatch"} + + +def test_it_is_read_only_and_never_hides_a_failure(): + doc = load() + assert doc["permissions"] == {"contents": "read"} + for job in doc["jobs"].values(): + assert not job.get("continue-on-error") + for st in job["steps"]: + assert not st.get("continue-on-error") + assert "REPORT-ONLY" in open(WF).read() + + +def test_nothing_waits_for_it_and_the_mutation_gate_is_unchanged(): + for name in os.listdir(os.path.join(ROOT, ".github/workflows")): + if name == "object-evidence.yml": + continue + for job in load(name)["jobs"].values(): + needs = job.get("needs", []) + needs = [needs] if isinstance(needs, str) else needs + assert "object-evidence" not in needs + assert set(load("mutation-pr.yml")["jobs"]["gate"]["needs"]) == { + "plan", + "go", + "rust", + "completeness", + "baseline-guard", + } + + +def test_the_record_is_verified_at_its_three_levels_against_the_checkout_and_the_inputs(): + steps = {s["name"]: s for s in load()["jobs"]["object-evidence"]["steps"]} + verify = next(v for k, v in steps.items() if k.startswith("Verify the record"))[ + "run" + ] + for needle in ( + "object_model.py verify", + "--root .", + "--scope .measurement-scope.json", + "--rust-list", + "--input", + ): + assert needle in verify + build = next( + v for k, v in steps.items() if k.startswith("Build the evidence record") + )["run"] + assert "--go-unit" in build and "goastdump" in build and "--llvm-cov" in build + # the verdicts of the mutation run are not produced here: the record says UNVERIFIABLE for them + assert "--rust-outcomes" not in build + + +def test_the_measuring_tools_are_the_real_ones_and_their_versions_are_recorded(): + run = next( + s + for s in load()["jobs"]["object-evidence"]["steps"] + if s["name"].startswith("Measure") + )["run"] + for needle in ( + "cargo llvm-cov", + "cargo mutants", + "go test", + "-covermode=set", + "objinventory", + "go version", + "rustc --version", + ): + assert needle in run + + +def test_scope_list_prints_the_files_of_a_class(): + out = subprocess.run( + [ + sys.executable, + os.path.join(ROOT, "scripts/object_model.py"), + "scope-list", + "--root", + ROOT, + "--scope", + os.path.join(ROOT, ".measurement-scope.json"), + "--lang", + "go", + "--class", + "separate:goastdump", + ], + capture_output=True, + text=True, + ) + assert out.returncode == 0 and out.stdout.split() == ["tools/goastdump/main.go"] + + +@pytest.mark.parametrize("path", ["scripts/object_model.py", ".measurement-scope.json"]) +def test_a_change_of_the_object_model_or_the_scope_is_gate_logic(path): + sys.path.insert(0, os.path.join(ROOT, "scripts")) + import mutation_plan + + assert path in mutation_plan.GATE_LOGIC_PATHS diff --git a/tests/test_tools/test_object_model.py b/tests/test_tools/test_object_model.py new file mode 100644 index 0000000..b9ac5d5 --- /dev/null +++ b/tests/test_tools/test_object_model.py @@ -0,0 +1,823 @@ +"""The object-level measurement model (scripts/object_model.py): the new contracts' tests (N-group). + +The inputs are real tool output (tests/test_tools/fixtures/object_model); derived cases are marked. +For an N test the previous state has nothing to run, so the evidence is: the good real input is +accepted, the bad input is rejected with the specific message, and the check is shown to have teeth +(see the report: the check removed on a scratch copy makes its test fail). +""" + +import copy +import json +import os +import shutil +import subprocess +import sys + +import pytest + +SCRIPTS = os.path.abspath(os.path.join(os.path.dirname(__file__), "../../scripts")) +sys.path.insert(0, SCRIPTS) + +import object_model as om # noqa: E402 +import quality_gate as qg # noqa: E402 + +FIX = os.path.join(os.path.dirname(__file__), "fixtures/object_model") +SCOPE_FILE = os.path.abspath( + os.path.join(os.path.dirname(__file__), "../../.measurement-scope.json") +) + + +def real(name): + with open(os.path.join(FIX, name)) as fh: + return json.load(fh) + + +def text(name): + with open(os.path.join(FIX, name)) as fh: + return fh.read() + + +def goastdump_objects(**kw): + blocks = om.parse_coverprofile(text("goastdump-cover.out")) + return om.go_objects( + "goastdump", + "goastdump", + real("goastdump-inventory.json"), + blocks, + detail=True, + **kw, + ) + + +@pytest.fixture() +def rust_root(tmp_path): + p = tmp_path / "cic-canonical/src" + p.mkdir(parents=True) + shutil.copy(os.path.join(FIX, "cic-canonical-lib.rs"), p / "lib.rs") + return str(tmp_path) + + +def rust_objects(root, summaries=None, **kw): + outcomes = real("outcomes-cic-canonical.json") + verdicts = qg.rust_verdicts(outcomes, "fixture") + if summaries: # DERIVED: another verdict for some mutants + verdicts.update(summaries) + return om.rust_objects( + real("llvm-cov-cic-canonical.json"), + real("demangled-cic-canonical.json"), + root, + real("mutants-cic-canonical.json"), + verdicts, + **kw, + ) + + +# --------------------------------------------------------------------------- scope (N8) + + +def test_the_repo_scope_is_valid_and_versioned(): + scope = json.load(open(SCOPE_FILE)) + om.check_scope(scope) + assert scope["schema"] == "measurement-scope/1" and isinstance( + scope["version"], int + ) + + +def test_every_tracked_go_and_rust_file_of_the_repo_is_classified(): + scope = json.load(open(SCOPE_FILE)) + root = os.path.abspath(os.path.join(SCRIPTS, "..")) + tracked = subprocess.run( + ["git", "-C", root, "ls-files"], capture_output=True, text=True, check=True + ).stdout.splitlines() + go, rs = om.scope_files(scope, "go", tracked), om.scope_files( + scope, "rust", tracked + ) + assert go["production"] and rs["production"] and "separate:goastdump" in go + + +def test_n8_a_file_the_scope_does_not_classify_is_an_error(): + scope = { + "schema": om.SCOPE_SCHEMA, + "version": 1, + "rules": [{"lang": "go", "match": "core/*", "class": "production"}], + } + with pytest.raises(om.ModelError, match="not classified by the measurement scope"): + om.scope_files(scope, "go", ["core/a.go", "tools/new/x.go"]) + + +def test_n8_an_exclusion_without_a_reason_is_rejected(): + scope = { + "schema": om.SCOPE_SCHEMA, + "version": 1, + "rules": [{"lang": "go", "match": "x/*", "class": "experimental"}], + } + with pytest.raises(om.ModelError, match="without a reason"): + om.check_scope(scope) + scope["rules"][0]["reason"] = " " + with pytest.raises(om.ModelError, match="without a reason"): + om.check_scope(scope) + + +def test_the_scope_needs_a_schema_and_an_integer_version(): + with pytest.raises(om.ModelError): + om.check_scope({"schema": "nope", "version": 1, "rules": []}) + with pytest.raises(om.ModelError): + om.check_scope({"schema": om.SCOPE_SCHEMA, "version": "1", "rules": []}) + + +# --------------------------------------------------------------------------- Go ids and the real goastdump unit + + +def test_go_object_ids(): + f = lambda **k: {"name": "N", "kind": "func", **k} # noqa: E731 + assert ( + om.go_object_id("m", "a/b.go", f()) == "go:m/a.N"[: len("go:m/a")] + "/b.N" + or True + ) + assert om.go_object_id("m", "a/b.go", f()) == "go:m/a.N" + assert om.go_object_id("m", "b.go", f()) == "go:m.N" + assert om.go_object_id("m", "a/b.go", f(kind="method", recv="T")) == "go:m/a.T.N" + assert ( + om.go_object_id("m", "a/b.go", f(kind="init", ord=2, name="init")) + == "go:m/a.init#b.go#2" + ) + assert ( + om.go_object_id("m", "a/b.go", f(), build="ffi_cgo") + == "go:m/a.N@build(ffi_cgo)" + ) + # the position is not part of the key + assert om.go_object_id( + "m", "a/b.go", f(start={"line": 1, "col": 1}) + ) == om.go_object_id("m", "a/b.go", f(start={"line": 99, "col": 5})) + + +def test_the_build_variants_of_one_name_are_two_objects_not_a_collision(): + inv = { + "files": [ + { + "file": "p/a_cgo.go", + "build": "ffi_cgo", + "funcs": [ + { + "name": "F", + "kind": "func", + "start": {"line": 1, "col": 1}, + "end": {"line": 2, "col": 2}, + } + ], + }, + { + "file": "p/a_stub.go", + "build": "!ffi_cgo", + "funcs": [ + { + "name": "F", + "kind": "func", + "start": {"line": 1, "col": 1}, + "end": {"line": 2, "col": 2}, + } + ], + }, + ] + } + blocks = [("m/p/a_stub.go", 1, 1, 2, 2, 1, 1)] + objs, _ = om.go_objects("root", "m", inv, blocks) + by = {o["id"]: o["coverage"] for o in objs} + assert by["go:m/p.F@build(!ffi_cgo)"]["state"] == om.MEASURED + # the variant that is not in the build has no block in the profile: UNVERIFIABLE, not 0% + assert by["go:m/p.F@build(ffi_cgo)"] == { + "state": "UNVERIFIABLE", + "reason": "FILE_NOT_IN_PROFILE", + } + + +def test_n1_two_different_objects_with_the_same_id_are_rejected(): + fn = { + "name": "F", + "kind": "func", + "start": {"line": 1, "col": 1}, + "end": {"line": 2, "col": 2}, + } + inv = { + "files": [{"file": "p/a.go", "funcs": [fn]}, {"file": "p/b.go", "funcs": [fn]}] + } # same package, same name, no build tag + with pytest.raises(om.ModelError, match="object id collision"): + om.go_objects("root", "m", inv, [("m/p/a.go", 1, 1, 2, 2, 1, 1)]) + + +def test_the_real_goastdump_unit(): + objs, st = goastdump_objects() + assert len(objs) == 15 and st["partial"] == 0 and st["unattributed"] == 0 + by = {o["name"]: o["coverage"] for o in objs} + assert (by["main"]["total"], by["main"]["covered"]) == ( + 11, + 0, + ) # the entry point is not run by any test + assert (by["call"]["total"], by["call"]["covered"]) == (26, 18) + assert by["call"]["kind"] == "statement" and by["call"]["uncovered"] + assert all(c["state"] == om.MEASURED for c in by.values()) + # the uncovered spans of an object lie inside that object's own span + for o in objs: + for sl, _, el, _ in o["coverage"].get("uncovered", []): + assert o["span"]["start"] <= sl and el <= o["span"]["end"] + + +# --------------------------------------------------------------------------- block attribution (N11), repeats (N9) + + +def fn(name, s, e, **kw): + return { + "name": name, + "kind": "func", + "start": {"line": s[0], "col": s[1]}, + "end": {"line": e[0], "col": e[1]}, + **kw, + } + + +def objs_for(funcs, blocks, **kw): + inv = {"files": [{"file": "a.go", "funcs": funcs}]} + o, st = om.go_objects("root", "m", inv, [("m/a.go", *b) for b in blocks], **kw) + return {x["name"]: x["coverage"] for x in o}, st + + +def test_n11_a_block_inside_one_object_is_attributed(): + cov, st = objs_for( + [fn("A", (1, 1), (5, 2))], [(2, 2, 3, 10, 2, 1), (3, 10, 4, 3, 1, 0)] + ) + assert cov["A"] == { + "state": om.MEASURED, + "kind": "statement", + "total": 3, + "covered": 2, + } + assert st["attributed"] == 2 and st["partial"] == 0 and st["unattributed"] == 0 + + +def test_n11_a_block_that_cuts_an_object_is_not_attributed_and_no_value_is_made_up(): + cov, st = objs_for( + [fn("A", (1, 1), (5, 2)), fn("B", (7, 1), (9, 2))], + [(2, 2, 3, 10, 2, 1), (4, 2, 8, 3, 4, 1)], + ) + assert st["partial"] == 1 + # the block (4,2)-(8,3) cuts A's end and B's start: neither gets a proportional share + assert cov["A"] == { + "state": om.UNVERIFIABLE, + "reason": "PARTIAL_BLOCK", + "blocks": 1, + } + assert cov["B"] == { + "state": om.UNVERIFIABLE, + "reason": "PARTIAL_BLOCK", + "blocks": 1, + } + + +def test_n11_a_block_that_spans_several_objects_is_not_attributed(): + cov, st = objs_for( + [fn("A", (1, 1), (3, 2)), fn("B", (5, 1), (7, 2))], [(1, 1, 7, 2, 9, 1)] + ) + assert ( + st["partial"] == 1 + and cov["A"]["reason"] == "PARTIAL_BLOCK" + and cov["B"]["reason"] == "PARTIAL_BLOCK" + ) + + +def test_n11_a_block_outside_every_object_is_counted_not_dropped(): + # e.g. a package-level `var f = func() {...}`: it belongs to no top-level function + cov, st = objs_for( + [fn("A", (10, 1), (12, 2))], [(2, 5, 4, 3, 2, 1), (11, 2, 11, 9, 1, 1)] + ) + assert st["unattributed"] == 1 and st["attributed"] == 1 + assert cov["A"]["total"] == 1 + + +def test_n11_a_function_literal_belongs_to_the_function_that_contains_it(): + cov, _ = objs_for( + [fn("Outer", (1, 1), (9, 2))], [(2, 2, 3, 3, 1, 1), (4, 10, 6, 4, 2, 0)] + ) # the second block is inside a literal + assert (cov["Outer"]["total"], cov["Outer"]["covered"]) == (3, 1) + + +def test_an_object_without_a_statement_is_not_applicable_not_zero(): + cov, _ = objs_for( + [fn("Empty", (1, 1), (1, 12)), fn("A", (3, 1), (5, 2))], [(4, 2, 4, 9, 1, 1)] + ) + assert cov["Empty"] == {"state": om.NOT_APPLICABLE, "reason": "NO_STATEMENTS"} + + +def test_a_block_that_appears_twice_is_covered_if_any_occurrence_is(): + blocks = om.parse_coverprofile( + "mode: set\nm/a.go:2.2,3.3 2 0\nm/a.go:2.2,3.3 2 1\n" + ) + assert len(blocks) == 1 and blocks[0][6] == 1 + with pytest.raises(om.ModelError, match="two statement counts"): + om.parse_coverprofile("mode: set\nm/a.go:2.2,3.3 2 0\nm/a.go:2.2,3.3 3 1\n") + + +def test_n9_an_object_whose_coverage_differs_between_runs_is_nondeterministic(): + base = [(2, 2, 3, 10, 2, 1), (3, 10, 4, 3, 1, 1)] + other = [ + (2, 2, 3, 10, 2, 1), + (3, 10, 4, 3, 1, 0), + ] # DERIVED: the second block was not covered in this run + inv = { + "files": [ + { + "file": "a.go", + "funcs": [fn("A", (1, 1), (5, 2)), fn("Stable", (7, 1), (9, 2))], + } + ] + } + blocks = [("m/a.go", *b) for b in base] + [("m/a.go", 8, 2, 8, 9, 1, 1)] + rep = [("m/a.go", *b) for b in other] + [("m/a.go", 8, 2, 8, 9, 1, 1)] + objs, _ = om.go_objects("root", "m", inv, blocks, repeats=[rep]) + by = {o["name"]: o["coverage"] for o in objs} + assert by["A"] == { + "state": om.UNVERIFIABLE, + "reason": "NONDETERMINISTIC", + "range": [2, 3], + "total": 3, + } + assert by["Stable"]["state"] == om.MEASURED + + +def test_a_failed_inventory_file_is_an_error(): + with pytest.raises(om.ModelError, match="could not parse"): + om.go_objects( + "root", + "m", + {"files": [{"file": "a.go", "funcs": [], "error": "boom"}]}, + [("m/a.go", 1, 1, 2, 2, 1, 1)], + ) + + +def test_a_profile_without_blocks_or_mode_is_an_error_not_zero(): + for bad in ( + "", + "mode: set\n", + "garbage\nm/a.go:1.1,2.2 1 1\n", + "mode: set\nm/a.go:1.1,2.2 1\n", + ): + with pytest.raises(om.ModelError): + om.parse_coverprofile(bad) + + +# --------------------------------------------------------------------------- Rust ids, test regions, instances (N10) + + +def test_rust_object_keys_fold_instances_closures_and_disambiguators(): + k = om.rust_object_key + assert ( + k("cic_canonical[0]::canonical_integer") + == "rust:cic_canonical::canonical_integer" + ) + assert k("cic_canonical[7]::canonical_integer") == k( + "cic_canonical[0]::canonical_integer" + ) # the same function in another test binary + assert k("cic_canonical[0]::canonical_integer::{closure#0}") == k( + "cic_canonical[0]::canonical_integer" + ) # a closure belongs to its function + assert ( + k("a[0]::f::") == "rust:a::f" + ) # an instantiation belongs to its generic function + assert ( + k("::with_header::") + == "rust:::with_header" + ) + assert ( + k(">::from") + == "rust:>::from" + ) + assert om.rust_name("rust:::with_header") == "with_header" + + +def test_n10_the_real_instances_of_one_function_are_one_object_with_the_instance_detail_kept( + rust_root, +): + objs, st = rust_objects(rust_root) + assert objs, "no objects" + ids = [o["id"] for o in objs] + assert len(ids) == len(set(ids)) + for o in objs: + c = o["coverage"] + assert ( + c["kind"] == "region" + and c["aggregation"] == "sum-over-instances/1" + and c["instances"] == len(o["instances"]) + ) + + +def test_cfg_test_spans_cover_exactly_the_marked_items(): + src = ( + "fn prod_a() {}\n" # 1 + "#[cfg(test)]\n" # 2 + "mod tests {\n" # 3 + " fn t() { let s = \"}{ not code\"; let c = '}'; /* } */ }\n" # 4 + "}\n" # 5 + "fn prod_b<'a>(x: &'a str) { let _ = x; }\n" # 6 + "#[cfg(test)]\n" # 7 + "fn only_test() {\n" # 8 + ' let _ = r#"}"#;\n' # 9 + "}\n" # 10 + "fn prod_c() {}\n" # 11 + "#[cfg(test)]\n" # 12 + "use std::fmt;\n" # 13 + "fn prod_d() {}\n" # 14 + ) + assert om.cfg_test_spans(src) == [(2, 5), (7, 10), (12, 13)] + + +def test_a_production_function_after_a_cfg_test_item_is_not_test_code(): + # measured on the real ffi/src/lib.rs: a #[cfg(test)] item in the middle, production code after it + src = "#[cfg(test)]\nfn helper() {\n}\n\npub fn production() {}\n" + spans = om.cfg_test_spans(src) + assert spans == [(1, 3)] and not any(a <= 5 <= b for a, b in spans) + + +def test_cfg_test_without_an_end_is_an_error(): + with pytest.raises(om.ModelError, match="cannot find the end"): + om.cfg_test_spans("#[cfg(test)]\nmod tests {\n") + + +def test_test_code_of_the_real_file_is_excluded_and_counted(rust_root): + objs, st = rust_objects(rust_root) + assert st["test_code_excluded"] > 0 + assert not any("tests" in o["id"].split("::") for o in objs) + + +# --------------------------------------------------------------------------- mutation per object, the zero denominator + + +def test_real_mutants_map_to_objects_by_the_last_segment_of_the_function_name( + rust_root, +): + objs, st = rust_objects(rust_root) + assert ( + st["mutants"] == 11 + and st["mutants_mapped"] == 11 + and st["mutants_unmapped"] == 0 + and st["mutants_ambiguous"] == 0 + ) + measured = [o for o in objs if o["mutation"]["state"] == om.MEASURED] + assert sum(o["mutation"]["caught"] for o in measured) == 11 + for o in measured: + assert o["mutation"]["score"] == { + "state": "MEASURED", + "value": 100.0, + "caught": o["mutation"]["caught"], + "evaluable": o["mutation"]["caught"], + } + + +def test_a_function_without_a_listed_mutant_is_not_applicable_not_a_perfect_score( + rust_root, +): + mutants = real("mutants-cic-canonical.json") + drop = next( + m for m in mutants if m["function"]["function_name"].endswith("canonical_float") + )["function"]["function_name"] + kept = [ + m for m in mutants if m["function"]["function_name"] != drop + ] # DERIVED: the tool lists no mutant for one function + verdicts = qg.rust_verdicts(real("outcomes-cic-canonical.json"), "fixture") + objs, _ = om.rust_objects( + real("llvm-cov-cic-canonical.json"), + real("demangled-cic-canonical.json"), + rust_root, + kept, + verdicts, + ) + o = next(x for x in objs if x["name"] == "canonical_float") + assert o["mutation"] == {"state": om.NOT_APPLICABLE, "reason": "NO_MUTANTS_LISTED"} + assert "score" not in o["mutation"] + + +def test_n12_an_object_whose_mutants_are_all_unviable_has_no_score(rust_root): + mutants = real("mutants-cic-canonical.json") + target = next( + m for m in mutants if m["function"]["function_name"].endswith("canonical_float") + ) + same = [ + m["name"] + for m in mutants + if m["function"]["function_name"] == target["function"]["function_name"] + ] + objs, _ = rust_objects(rust_root, {n: "unviable" for n in same}) # DERIVED + o = next(x for x in objs if x["name"] == "canonical_float") + assert o["mutation"]["score"] == { + "state": "NOT_APPLICABLE", + "reason": "ALL_UNVIABLE", + } + assert o["mutation"]["unviable"] == len(same) and o["mutation"]["caught"] == 0 + + +def test_a_mutant_that_fits_two_objects_is_ambiguous_not_guessed(rust_root): + mutants = real("mutants-cic-canonical.json") + m = copy.deepcopy(mutants[0]) + llvm = real("llvm-cov-cic-canonical.json") + demangled = real("demangled-cic-canonical.json") + # DERIVED: a second function of the same last name in the same file, inside the mutant's function span + twin = copy.deepcopy( + next( + f + for f in llvm["data"][0]["functions"] + if demangled[f["name"]].split("::")[-1] + == m["function"]["function_name"].split("::")[-1] + ) + ) + twin["name"] = "_RNvC13cic_canonical99twin_symbol" + demangled = { + **demangled, + twin["name"]: "cic_canonical[0]::other_mod::" + + m["function"]["function_name"].split("::")[-1], + } + llvm["data"][0]["functions"].append(twin) + verdicts = qg.rust_verdicts(real("outcomes-cic-canonical.json"), "fixture") + objs, st = om.rust_objects(llvm, demangled, rust_root, [m], verdicts) + assert st["mutants_ambiguous"] == 1 and st["mutants_mapped"] == 0 + assert all(o["mutation"].get("caught", 0) == 0 for o in objs) + + +def test_a_symbol_without_a_demangled_name_is_an_error_not_a_dropped_object(rust_root): + with pytest.raises(om.ModelError, match="no demangled name"): + om.rust_objects(real("llvm-cov-cic-canonical.json"), {}, rust_root, None, None) + + +# --------------------------------------------------------------------------- the record and its verifier + +TOOLS = { + "go": "go1.26.10", + "rustc": "1.96.1", + "cargo-mutants": "27.1.0", + "cargo-llvm-cov": "0.9.1", + "c++filt": "2.40", +} +SCOPE = json.load(open(SCOPE_FILE)) + + +def make_record(rust_root, tools=None): + go, _ = goastdump_objects() + rs, _ = rust_objects(rust_root) + inputs = {"fixture": {"sha256": "0" * 64, "bytes": 1}} + return om.build_record( + go + rs, + "c" * 40, + "t" * 40, + tools or dict(TOOLS), + inputs, + SCOPE, + {"go:goastdump": {"partial": 0}}, + ) + + +def reseal(rec): + rec["summary"] = om.summarize(rec["objects"]) + rec["digest"] = om.canonical_digest({k: v for k, v in rec.items() if k != "digest"}) + return rec + + +def test_the_record_of_the_real_fixtures_is_intact(rust_root): + rec = make_record(rust_root) + assert om.verify_l1(rec) == [] + assert rec["summary"]["by_unit"]["goastdump"]["objects"] == 15 + assert [o["id"] for o in rec["objects"]] == sorted(o["id"] for o in rec["objects"]) + + +def test_n4_a_changed_record_is_rejected(rust_root): + rec = make_record(rust_root) + t = copy.deepcopy(rec) + t["objects"][0]["name"] = "renamed" # content changed, digest kept + assert any("digest does not match" in b for b in om.verify_l1(t)) + t = copy.deepcopy(rec) + del t["objects"][3] # an object removed, digest kept + assert any("digest does not match" in b for b in om.verify_l1(t)) + t = copy.deepcopy(rec) + t["summary"][ + "objects" + ] += 1 # a summary that no longer matches its objects, digest recomputed + t["digest"] = om.canonical_digest({k: v for k, v in t.items() if k != "digest"}) + assert any("summary does not match" in b for b in om.verify_l1(t)) + t = copy.deepcopy(rec) + t["inputs"] = {} + t = reseal(t) + assert any("no input digests" in b for b in om.verify_l1(t)) + + +def test_n2_every_field_of_every_object_has_a_valid_state_and_a_coded_reason(rust_root): + rec = make_record(rust_root) + for o in rec["objects"]: + for f in om.FIELDS: + assert o[f]["state"] in om.STATES + if o[f]["state"] != om.MEASURED: + assert o[f]["reason"] in om.REASONS + t = copy.deepcopy(rec) + del t["objects"][0]["tests"] + assert any("field tests is missing" in b for b in om.verify_l1(reseal(t))) + t = copy.deepcopy(rec) + t["objects"][0]["tests"] = {"state": "UNVERIFIABLE"} # a missing reason + assert any("without a valid reason" in b for b in om.verify_l1(reseal(t))) + t = copy.deepcopy(rec) + t["objects"][0]["tests"] = {"state": "VERIFIED"} # a state that is not in the enum + assert any("no valid state" in b for b in om.verify_l1(reseal(t))) + + +def test_n5_an_unknown_tool_version_makes_the_record_unusable(rust_root): + with pytest.raises(om.ModelError, match="unknown"): + make_record(rust_root, tools={**TOOLS, "cargo-mutants": "unknown"}) + rec = make_record(rust_root) + rec["tools"]["go"] = "" + assert any( + "version of the tool 'go' is unknown" in b for b in om.verify_l1(reseal(rec)) + ) + rec = make_record(rust_root) + rec["tools"] = {} + assert any("no tool versions" in b for b in om.verify_l1(reseal(rec))) + + +def test_n6_a_coverage_kind_that_is_not_measured_cannot_be_branch(rust_root): + rec = make_record(rust_root) + assert "branch" not in om.COVERAGE_KINDS and rec["coverage_kinds"] == [ + "statement", + "line", + "region", + ] + o = next(o for o in rec["objects"] if o["coverage"]["state"] == om.MEASURED) + o["coverage"]["kind"] = "branch" + assert any( + "coverage kind 'branch' is not one of" in b for b in om.verify_l1(reseal(rec)) + ) + + +def test_n7_the_global_and_the_object_level_view_are_separate_and_unverifiable_is_counted( + rust_root, +): + rec = make_record(rust_root) + sm = rec["summary"] + assert set(sm["by_unit"]) == {"goastdump", "rust"} and "fields" in sm + # every object is unverifiable for the test link (no per-test attribution): the summary says so, there is no single "verified" + assert sm["fields"]["tests"][om.UNVERIFIABLE] == sm["objects"] + assert ( + "verified" not in json.dumps(sm).lower() + or "unverifiable" in json.dumps(sm).lower() + ) + assert not any(k.lower() == "verified" for k in rec) + + +def test_a_record_bound_to_another_policy_version_is_rejected(rust_root): + rec = make_record(rust_root) + rec["policy"] = "mutation-policy/1" + assert any("bound to 'mutation-policy/1'" in b for b in om.verify_l1(reseal(rec))) + + +def test_duplicate_object_ids_cannot_be_built(rust_root): + go, _ = goastdump_objects() + with pytest.raises(om.ModelError, match="duplicate object ids"): + om.build_record( + go + go[:1], + "c" * 40, + "t" * 40, + dict(TOOLS), + {"x": {"sha256": "0" * 64, "bytes": 1}}, + SCOPE, + {}, + ) + + +def git_repo(tmp_path): + env = { + **os.environ, + "GIT_AUTHOR_NAME": "t", + "GIT_AUTHOR_EMAIL": "t@x", + "GIT_COMMITTER_NAME": "t", + "GIT_COMMITTER_EMAIL": "t@x", + } + d = tmp_path / "checkout" + d.mkdir() + for args in (["init", "-q"],): + subprocess.run(["git", "-C", str(d), *args], check=True, env=env) + shutil.copy(os.path.join(FIX, "goastdump-main.go"), d / "main.go") + subprocess.run(["git", "-C", str(d), "add", "-A"], check=True, env=env) + subprocess.run( + [ + "git", + "-C", + str(d), + "-c", + "commit.gpgsign=false", + "-c", + "core.hooksPath=/dev/null", + "commit", + "-q", + "-m", + "x", + ], + check=True, + env=env, + ) + commit = subprocess.run( + ["git", "-C", str(d), "rev-parse", "HEAD"], + capture_output=True, + text=True, + check=True, + ).stdout.strip() + tree = subprocess.run( + ["git", "-C", str(d), "rev-parse", "HEAD^{tree}"], + capture_output=True, + text=True, + check=True, + ).stdout.strip() + return str(d), commit, tree + + +def test_n13_l2_is_not_checked_without_inputs_and_fails_on_a_changed_input( + tmp_path, rust_root +): + checkout, commit, tree = git_repo(tmp_path) + inp = tmp_path / "cover.out" + inp.write_text(text("goastdump-cover.out")) + digest, size = om.sha256_file(str(inp)) + go, _ = goastdump_objects() + rec = om.build_record( + go, + commit, + tree, + dict(TOOLS), + {"cover": {"sha256": digest, "bytes": size}}, + SCOPE, + {}, + ) + assert om.verify_l2(rec, None, {}) == ( + "NOT_CHECKED", + ["no checkout and no input files were given"], + ) + assert om.verify_l2(rec, checkout, {"cover": str(inp)}) == ("PASS", []) + inp.write_text(text("goastdump-cover.out") + "goastdump/main.go:1.1,2.2 1 1\n") + v, f = om.verify_l2(rec, checkout, {"cover": str(inp)}) + assert v == "FAIL" and "does not match the digest" in f[0] + rec["identity"]["tree"] = "0" * 40 + assert om.verify_l2(rec, checkout, {})[0] == "FAIL" + assert om.verify_l2(rec, checkout, {"unknown": str(inp)})[0] == "FAIL" + + +def test_n13_a_consistently_forged_record_passes_l1_but_not_l3(tmp_path): + checkout, commit, tree = git_repo(tmp_path) + go, _ = goastdump_objects() + inputs = {"x": {"sha256": "0" * 64, "bytes": 1}} + honest = om.build_record(go, commit, tree, dict(TOOLS), inputs, SCOPE, {}) + forged = copy.deepcopy(honest) + forged["objects"].pop(5) # an object is removed ... + forged = reseal( + forged + ) # ... and the digest and the summary are recomputed consistently + assert om.verify_l1(forged) == [] # L1 proves only the internal consistency + res = om.verify_record(forged, checkout, {}, {"goastdump": ["main.go"]}, None) + assert res["L1"]["verdict"] == "INTEGRITY_OK" and res["L3"]["verdict"] == "FAIL" + assert "14 objects but 15 top-level" in res["L3"]["findings"][0] + ok = om.verify_record(honest, checkout, {}, {"goastdump": ["main.go"]}, None) + assert ( + ok["L3"]["verdict"] == "PARTIAL" + ) # Go is proven, Rust could not be: said, not hidden + assert any("equal the independent" in p for p in ok["L3"]["proven"]) + + +def test_n13_a_record_that_is_not_intact_is_not_trusted_by_the_later_levels(rust_root): + rec = make_record(rust_root) + rec["digest"] = "0" * 64 + res = om.verify_record(rec) + assert res["L1"]["verdict"] == "FAIL" + assert ( + res["L2"]["verdict"] == "NOT_CHECKED" and res["L3"]["verdict"] == "NOT_CHECKED" + ) + + +def test_n13_the_verifier_output_has_three_levels_and_no_combined_verdict( + tmp_path, rust_root, capsys +): + rec = make_record(rust_root) + p = tmp_path / "rec.json" + p.write_text(json.dumps(rec)) + rc = om.main(["verify", "--record", str(p)]) + out = capsys.readouterr().out + assert rc == 0 + for lvl, word in (("L1", "INTEGRITY_OK"), ("L2", "NOT_CHECKED"), ("L3", "PARTIAL")): + assert f"{lvl} {word}" in out + assert "no combined 'verified'" in out and "VERIFIED" not in out + + +def test_the_rust_l3_names_a_function_that_has_no_object(rust_root): + rec = make_record(rust_root) + mutants = real("mutants-cic-canonical.json") + mutants.append( + { + "file": "cic-canonical/src/lib.rs", + "name": "x", + "function": { + "function_name": "m::ghost_function", + "span": {"start": {"line": 1}, "end": {"line": 2}}, + }, + } + ) + v, _, findings = om.verify_l3(rec, None, None, mutants) + assert v == "FAIL" and "ghost_function" in findings[0] diff --git a/tests/test_tools/test_quality_gate.py b/tests/test_tools/test_quality_gate.py index 822c5c1..1ab5634 100644 --- a/tests/test_tools/test_quality_gate.py +++ b/tests/test_tools/test_quality_gate.py @@ -249,7 +249,7 @@ def test_f2_what_fails_the_gate(tmp_path, summary, fails, capsys): text = capsys.readouterr().out assert (rc != 0) == fails, text assert ( - "mutation-policy/1" in text and "planned=5" in text + "mutation-policy/2" in text and "planned=5" in text ) # the accounting is always shown diff --git a/tools/objinventory/go.mod b/tools/objinventory/go.mod new file mode 100644 index 0000000..c05fae5 --- /dev/null +++ b/tools/objinventory/go.mod @@ -0,0 +1,3 @@ +module objinventory + +go 1.22 diff --git a/tools/objinventory/main.go b/tools/objinventory/main.go new file mode 100644 index 0000000..b49e0e6 --- /dev/null +++ b/tools/objinventory/main.go @@ -0,0 +1,148 @@ +// objinventory lists the top-level functions and methods of Go source files with +// their exact source spans, as JSON. It is the Go side of the object inventory of +// scripts/object_model.py (object-id/1): the Python side decides what an object is +// called and how coverage blocks are attributed, this tool only reports what the +// syntax tree says. +// +// Usage: objinventory [ ...] +// +// Only the standard library is used and nothing is type checked. A function literal +// is not an object of its own: it belongs to the top-level declaration whose span +// contains it (the documented attribution rule), so only FuncDecl spans are listed. +package main + +import ( + "encoding/json" + "fmt" + "go/ast" + "go/parser" + "go/token" + "io" + "os" + "strings" +) + +// Pos is a source position. Line and column are 1-based, as in a Go cover profile. +type Pos struct { + Line int `json:"line"` + Col int `json:"col"` +} + +// Func is one top-level function or method. +type Func struct { + Name string `json:"name"` + Recv string `json:"recv,omitempty"` // receiver base type, without * or type parameters + Kind string `json:"kind"` // func | method | init + // Ord is the 1-based ordinal among the `init` functions of the file (0 otherwise). + Ord int `json:"ord,omitempty"` + Start Pos `json:"start"` + End Pos `json:"end"` +} + +// File is the result for one input file. +type File struct { + File string `json:"file"` + Package string `json:"package"` + // Build is the //go:build constraint of the file, "" if it has none. + Build string `json:"build,omitempty"` + Funcs []Func `json:"funcs"` + Error string `json:"error,omitempty"` +} + +// Dump is the whole output. +type Dump struct { + Schema string `json:"schema"` + Files []File `json:"files"` +} + +// recvName returns the base type name of a receiver expression: T, *T, T[K], *T[K]. +func recvName(e ast.Expr) string { + switch x := e.(type) { + case *ast.StarExpr: + return recvName(x.X) + case *ast.ParenExpr: + return recvName(x.X) + case *ast.IndexExpr: + return recvName(x.X) + case *ast.IndexListExpr: + return recvName(x.X) + case *ast.Ident: + return x.Name + } + return "" +} + +// buildConstraint returns the //go:build line that precedes the package clause. +func buildConstraint(f *ast.File) string { + for _, g := range f.Comments { + if g.Pos() >= f.Package { + break + } + for _, c := range g.List { + if rest, ok := strings.CutPrefix(c.Text, "//go:build "); ok { + return strings.TrimSpace(rest) + } + } + } + return "" +} + +func dumpFile(path string) File { + fset := token.NewFileSet() + f, err := parser.ParseFile(fset, path, nil, parser.ParseComments) + if err != nil { + return File{File: path, Funcs: []Func{}, Error: err.Error()} + } + out := File{File: path, Package: f.Name.Name, Build: buildConstraint(f), Funcs: []Func{}} + inits := 0 + for _, d := range f.Decls { + fd, ok := d.(*ast.FuncDecl) + if !ok { + continue + } + s, e := fset.Position(fd.Pos()), fset.Position(fd.End()) + fn := Func{ + Name: fd.Name.Name, + Kind: "func", + Start: Pos{s.Line, s.Column}, + End: Pos{e.Line, e.Column}, + } + if fd.Recv != nil && len(fd.Recv.List) == 1 { + fn.Kind, fn.Recv = "method", recvName(fd.Recv.List[0].Type) + } else if fd.Name.Name == "init" && fd.Recv == nil { + inits++ + fn.Kind, fn.Ord = "init", inits + } + out.Funcs = append(out.Funcs, fn) + } + return out +} + +// run is main without the process: it returns the exit status. +func run(args []string, stdout, stderr io.Writer) int { + if len(args) < 1 { + fmt.Fprintln(stderr, "usage: objinventory [ ...]") + return 2 + } + d := Dump{Schema: "objinventory/1", Files: []File{}} + failed := false + for _, p := range args { + f := dumpFile(p) + failed = failed || f.Error != "" + d.Files = append(d.Files, f) + } + enc := json.NewEncoder(stdout) + enc.SetIndent("", " ") + if err := enc.Encode(d); err != nil { + fmt.Fprintln(stderr, err) + return 1 + } + if failed { // a file that cannot be parsed is reported in the output AND fails the run + return 1 + } + return 0 +} + +func main() { + os.Exit(run(os.Args[1:], os.Stdout, os.Stderr)) +} diff --git a/tools/objinventory/main_test.go b/tools/objinventory/main_test.go new file mode 100644 index 0000000..7260ecf --- /dev/null +++ b/tools/objinventory/main_test.go @@ -0,0 +1,134 @@ +package main + +import ( + "bytes" + "encoding/json" + "os" + "path/filepath" + "testing" +) + +const sample = `//go:build linux && !nocgo + +package demo + +type T struct{} +type G[K comparable] struct{} + +var hook = func() int { return 1 } // outside any FuncDecl + +func Top(a int) int { + f := func() int { return a } // a literal belongs to Top + return f() +} + +func (t *T) Ptr() {} +func (t T) Val() {} +func (g *G[K]) Gen() {} + +func init() {} +func init() {} +` + +func write(t *testing.T, name, src string) string { + t.Helper() + p := filepath.Join(t.TempDir(), name) + if err := os.WriteFile(p, []byte(src), 0o600); err != nil { + t.Fatal(err) + } + return p +} + +func TestDumpFile(t *testing.T) { + f := dumpFile(write(t, "a.go", sample)) + if f.Error != "" || f.Package != "demo" || f.Build != "linux && !nocgo" { + t.Fatalf("file header wrong: %+v", f) + } + type want struct { + name, recv, kind string + ord int + } + exp := []want{ + {"Top", "", "func", 0}, + {"Ptr", "T", "method", 0}, + {"Val", "T", "method", 0}, + {"Gen", "G", "method", 0}, + {"init", "", "init", 1}, + {"init", "", "init", 2}, + } + if len(f.Funcs) != len(exp) { + t.Fatalf("got %d funcs, want %d: %+v", len(f.Funcs), len(exp), f.Funcs) + } + for i, w := range exp { + g := f.Funcs[i] + if g.Name != w.name || g.Recv != w.recv || g.Kind != w.kind || g.Ord != w.ord { + t.Errorf("func %d = %+v, want %+v", i, g, w) + } + } +} + +func TestSpansAreExact(t *testing.T) { + f := dumpFile(write(t, "a.go", sample)) + top := f.Funcs[0] + // `func Top` is on line 10 of the sample and closes on line 13 + if top.Start != (Pos{10, 1}) || top.End != (Pos{13, 2}) { + t.Errorf("Top span = %+v..%+v", top.Start, top.End) + } +} + +func TestNoBuildConstraint(t *testing.T) { + f := dumpFile(write(t, "b.go", "package b\n\nfunc F() {}\n")) + if f.Build != "" || len(f.Funcs) != 1 { + t.Errorf("unexpected: %+v", f) + } +} + +func TestParseErrorIsReported(t *testing.T) { + f := dumpFile(write(t, "c.go", "package c\n\nfunc (\n")) + if f.Error == "" || f.Funcs == nil || len(f.Funcs) != 0 { + t.Errorf("a parse error must be reported with an empty (non-nil) list: %+v", f) + } +} + +func TestMissingFileIsReported(t *testing.T) { + if f := dumpFile(filepath.Join(t.TempDir(), "none.go")); f.Error == "" { + t.Error("a missing file must be an error") + } +} + +func TestRecvName(t *testing.T) { + f := dumpFile(write(t, "d.go", "package d\n\ntype T[K any] struct{}\n\nfunc (t *T[K]) A() {}\nfunc (T[K]) B() {}\n")) + for _, fn := range f.Funcs { + if fn.Recv != "T" { + t.Errorf("%s: recv = %q, want T", fn.Name, fn.Recv) + } + } +} + +func TestRunExitStatusAndOutput(t *testing.T) { + good := write(t, "g.go", "package g\n\nfunc F() {}\n") + bad := write(t, "b.go", "package b\n\nfunc (\n") + + var out, errw bytes.Buffer + if rc := run(nil, &out, &errw); rc != 2 || errw.Len() == 0 || out.Len() != 0 { + t.Errorf("no arguments: rc=%d out=%q err=%q", rc, out.String(), errw.String()) + } + + out.Reset() + errw.Reset() + if rc := run([]string{good}, &out, &errw); rc != 0 { + t.Fatalf("a good file must succeed, rc=%d err=%q", rc, errw.String()) + } + var d Dump + if err := json.Unmarshal(out.Bytes(), &d); err != nil || d.Schema != "objinventory/1" || len(d.Files) != 1 { + t.Errorf("unexpected output %q (%v)", out.String(), err) + } + + out.Reset() + if rc := run([]string{good, bad}, &out, &errw); rc != 1 { + t.Errorf("a file that cannot be parsed must fail the run, rc=%d", rc) + } + if err := json.Unmarshal(out.Bytes(), &d); err != nil || len(d.Files) != 2 || d.Files[1].Error == "" { + t.Errorf("the failing file must still be reported in the output: %q", out.String()) + } +} From 34043f790c2c313c4fcf74869cf6dced5f0435b3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?G=C3=A1bor=20Z=20Sink=C3=B3?= Date: Sat, 14 Nov 2026 21:43:05 +0100 Subject: [PATCH 2/2] fix(measurement): fixture sources are .txt, tests/ is test-data in the scope The first CI run of this change failed in `coverage` and in `go (chunk 4)`/`completeness`/ `mutation-pr`: the fixture `tests/test_tools/fixtures/object_model/goastdump-main.go` is a `package main` file, which the root module treats as a package of its own (centralrelay/tests/test_tools/fixtures/object_model). The coverage floor (0% < min 80%) and the mutation gate (0 of 37 mutants covered, efficacy 0.00) both failed on it. Not an environment problem and not pre-existing: the base has no such file; it is a regression of this change. The same mistake showed in the measurement scope: `*.go` classified the fixture as production, so the object-evidence run counted 479 root objects instead of 464 (the 15 functions of the fixture). - the Go and Rust fixture sources are stored as .txt (no tool sees source code in them) - .measurement-scope.json: tests/* is test-data for Go and Rust, with a reason - two guard tests: no .go/.rs file under tests/ outside testdata; a source file under tests/ is not production in the scope No blocking gate, threshold or baseline is touched. --- [signing-metadata] key = cic-my-sign-key signature = vault:v1:MEUCIQDz9d60E5aPqHDufCfNCgFmkm1pqVyKaRgR6+MK9wxqeAIgErThcuAT975EUDLfLUfJHtqar5RC0Bb287v7eEH14kI= hash-algorithm = sha256 digest = xuua4oLhoRGpKfSz4kPhgBci7dYy/qVTC1HcoWoeoI8= [certificate] -----BEGIN CERTIFICATE----- MIICBjCCAaygAwIBAgIUSnRMR6RPnEbg296XWPOqq/u5PCwwCgYIKoZIzj0EAwIw QzELMAkGA1UEBhMCSFUxGTAXBgNVBAoMEENlbnRyYWxJbmZyYUNvcmUxGTAXBgNV BAMMEENJQyBEZXZlbG9wZXIgQ0EwHhcNMjYwMzIwMTMyMjU5WhcNMjYxMjMxMTMy MjU5WjBFMQswCQYDVQQGEwJIVTEZMBcGA1UECgwQQ2VudHJhbEluZnJhQ29yZTEb MBkGA1UEAwwSR2Fib3IgWm9sdGFuIFNpbmtvMFkwEwYHKoZIzj0CAQYIKoZIzj0D AQcDQgAEIG2CVmTfmLB9pLLclj7YmP2eedAjklpy4LGrU2ijoiy6Xqpuybv7OgJe i+ez31s65NEV8+X/ByeX1cstR988z6N8MHowCQYDVR0TBAIwADAdBgNVHQ4EFgQU yZN6AIX/TNnIJ9GwAa/NRN3ujHAwHwYDVR0jBBgwFoAUXn6CHYzPUqU4JVP8g+OS WeDYjhcwDgYDVR0PAQH/BAQDAgeAMB0GA1UdJQQWMBQGCCsGAQUFBwMCBggrBgEF BQcDBDAKBggqhkjOPQQDAgNIADBFAiEA+bFzXRoJ4PCQbhAAtpkcMjt0vNj5rEW0 lOMBGDNyaWkCIB1vmM7PcZzv/c9bIrxF5kqv6QXomouhByUfeNUTbpKW -----END CERTIFICATE----- --- .measurement-scope.json | 91 ++++++++++++++++--- .../fixtures/object_model/README.md | 4 +- ...onical-lib.rs => cic-canonical-lib.rs.txt} | 0 ...oastdump-main.go => goastdump-main.go.txt} | 0 tests/test_tools/test_object_model.py | 29 +++++- 5 files changed, 109 insertions(+), 15 deletions(-) rename tests/test_tools/fixtures/object_model/{cic-canonical-lib.rs => cic-canonical-lib.rs.txt} (100%) rename tests/test_tools/fixtures/object_model/{goastdump-main.go => goastdump-main.go.txt} (100%) diff --git a/.measurement-scope.json b/.measurement-scope.json index 46bc7e6..2c4efcc 100644 --- a/.measurement-scope.json +++ b/.measurement-scope.json @@ -3,16 +3,85 @@ "version": 1, "comment": "Which files the object-level measurement covers. The first matching rule decides; a tracked Go or Rust file that matches no rule is an error (nothing is left out silently). A change of this file is a change of the gate logic and is reviewed as such.", "rules": [ - {"lang": "go", "match": "*_test.go", "class": "test", "reason": "test code is not a production object"}, - {"lang": "go", "match": "*/testdata/*", "class": "test-data", "reason": "fixtures read by tests; Go tooling ignores testdata"}, - {"lang": "go", "match": "tools/goastdump/*", "class": "separate", "unit": "goastdump", "reason": "a separate Go module (tools/goastdump/go.mod): measured and reported on its own, never averaged into the root module (owner decision 2026-11-14)"}, - {"lang": "go", "match": "tools/objinventory/*", "class": "tool", "unit": "objinventory", "reason": "the measuring tool itself (a separate Go module); it is tested in the python job and is not an object of the Relay"}, - {"lang": "go", "match": "tools/statechain-demo/*", "class": "experimental", "reason": "live demo entry, exercised by make demo; already excluded from the coverage floors in verify.yml"}, - {"lang": "go", "match": "tools/trustflow-demo/*", "class": "experimental", "reason": "live cgo demo entry, exercised by make demo-trustflow; excluded from the coverage floors in verify.yml"}, - {"lang": "go", "match": "tools/oci-sign-demo/*", "class": "experimental", "reason": "self-verifying demo entry, exercised by make demo-oci-sign; excluded from the coverage floors in verify.yml"}, - {"lang": "go", "match": "*.go", "class": "production", "reason": "every other Go file of the root module"}, - {"lang": "rust", "match": "*/tests/*", "class": "test", "reason": "integration tests are not production objects"}, - {"lang": "rust", "match": "*/benches/*", "class": "test", "reason": "benchmarks are not production objects"}, - {"lang": "rust", "match": "*.rs", "class": "production", "reason": "every other Rust source (a #[cfg(test)] region inside a file is excluded by the object model, counted, not dropped)"} + { + "lang": "go", + "match": "*_test.go", + "class": "test", + "reason": "test code is not a production object" + }, + { + "lang": "go", + "match": "*/testdata/*", + "class": "test-data", + "reason": "fixtures read by tests; Go tooling ignores testdata" + }, + { + "lang": "go", + "match": "tests/*", + "class": "test-data", + "reason": "test fixtures and the tests of the scripts: not part of the Go module's production code (a source file stored here would also become a package of the root module)" + }, + { + "lang": "go", + "match": "tools/goastdump/*", + "class": "separate", + "unit": "goastdump", + "reason": "a separate Go module (tools/goastdump/go.mod): measured and reported on its own, never averaged into the root module (owner decision 2026-11-14)" + }, + { + "lang": "go", + "match": "tools/objinventory/*", + "class": "tool", + "unit": "objinventory", + "reason": "the measuring tool itself (a separate Go module); it is tested in the python job and is not an object of the Relay" + }, + { + "lang": "go", + "match": "tools/statechain-demo/*", + "class": "experimental", + "reason": "live demo entry, exercised by make demo; already excluded from the coverage floors in verify.yml" + }, + { + "lang": "go", + "match": "tools/trustflow-demo/*", + "class": "experimental", + "reason": "live cgo demo entry, exercised by make demo-trustflow; excluded from the coverage floors in verify.yml" + }, + { + "lang": "go", + "match": "tools/oci-sign-demo/*", + "class": "experimental", + "reason": "self-verifying demo entry, exercised by make demo-oci-sign; excluded from the coverage floors in verify.yml" + }, + { + "lang": "go", + "match": "*.go", + "class": "production", + "reason": "every other Go file of the root module" + }, + { + "lang": "rust", + "match": "tests/*", + "class": "test-data", + "reason": "test fixtures and the tests of the scripts: not Rust workspace members" + }, + { + "lang": "rust", + "match": "*/tests/*", + "class": "test", + "reason": "integration tests are not production objects" + }, + { + "lang": "rust", + "match": "*/benches/*", + "class": "test", + "reason": "benchmarks are not production objects" + }, + { + "lang": "rust", + "match": "*.rs", + "class": "production", + "reason": "every other Rust source (a #[cfg(test)] region inside a file is excluded by the object model, counted, not dropped)" + } ] } diff --git a/tests/test_tools/fixtures/object_model/README.md b/tests/test_tools/fixtures/object_model/README.md index 75ad3b0..094c5fc 100644 --- a/tests/test_tools/fixtures/object_model/README.md +++ b/tests/test_tools/fixtures/object_model/README.md @@ -2,10 +2,10 @@ Fixtures of `tests/test_tools/test_object_model.py`, cut from the REAL tool outp * `goastdump-inventory.json` — the real output of `tools/objinventory` on `tools/goastdump/main.go` (15 functions). Unmodified. * `goastdump-cover.out` — the real `go test -covermode=set -coverprofile` of the `goastdump` module. Unmodified. -* `goastdump-main.go` — the source those two describe (CIC_Relay 852594a). +* `goastdump-main.go.txt` — the source those two describe (CIC_Relay 852594a). **Stored as `.txt` on purpose**: a `.go` file here would be a package of the root module (coverage floor and mutation gate fail on it: measured on the first CI run of this change). * `llvm-cov-cic-canonical.json` — the entries of `cic-canonical/src/lib.rs` of the real `cargo llvm-cov --json` export (cargo-llvm-cov 0.9.1, rust 1.96.1). * `demangled-cic-canonical.json` — the real `c++filt` (GNU Binutils 2.40) names of those symbols. * `mutants-cic-canonical.json`, `outcomes-cic-canonical.json` — the real cargo-mutants 27.1.0 listing and outcomes restricted to that file (11 mutants; the `diff` text and the machine-local log paths removed). -* `cic-canonical-lib.rs` — the source of that file. +* `cic-canonical-lib.rs.txt` — the source of that file (`.txt` for the same reason). Other verdicts (missed, timeout, unviable), other block layouts and tampered records are DERIVED in the tests from these; derived cases are marked in the tests. diff --git a/tests/test_tools/fixtures/object_model/cic-canonical-lib.rs b/tests/test_tools/fixtures/object_model/cic-canonical-lib.rs.txt similarity index 100% rename from tests/test_tools/fixtures/object_model/cic-canonical-lib.rs rename to tests/test_tools/fixtures/object_model/cic-canonical-lib.rs.txt diff --git a/tests/test_tools/fixtures/object_model/goastdump-main.go b/tests/test_tools/fixtures/object_model/goastdump-main.go.txt similarity index 100% rename from tests/test_tools/fixtures/object_model/goastdump-main.go rename to tests/test_tools/fixtures/object_model/goastdump-main.go.txt diff --git a/tests/test_tools/test_object_model.py b/tests/test_tools/test_object_model.py index b9ac5d5..1756dfb 100644 --- a/tests/test_tools/test_object_model.py +++ b/tests/test_tools/test_object_model.py @@ -53,7 +53,7 @@ def goastdump_objects(**kw): def rust_root(tmp_path): p = tmp_path / "cic-canonical/src" p.mkdir(parents=True) - shutil.copy(os.path.join(FIX, "cic-canonical-lib.rs"), p / "lib.rs") + shutil.copy(os.path.join(FIX, "cic-canonical-lib.rs.txt"), p / "lib.rs") return str(tmp_path) @@ -697,7 +697,7 @@ def git_repo(tmp_path): d.mkdir() for args in (["init", "-q"],): subprocess.run(["git", "-C", str(d), *args], check=True, env=env) - shutil.copy(os.path.join(FIX, "goastdump-main.go"), d / "main.go") + shutil.copy(os.path.join(FIX, "goastdump-main.go.txt"), d / "main.go") subprocess.run(["git", "-C", str(d), "add", "-A"], check=True, env=env) subprocess.run( [ @@ -821,3 +821,28 @@ def test_the_rust_l3_names_a_function_that_has_no_object(rust_root): ) v, _, findings = om.verify_l3(rec, None, None, mutants) assert v == "FAIL" and "ghost_function" in findings[0] + + +def test_no_go_or_rust_source_lives_under_tests_outside_testdata(): + # a `package main` file in a fixture directory becomes a package of the root module: the coverage floor + # (0% < 80%) and the mutation gate (0 of 37 mutants covered) both fail on it. Measured on this PR's own + # first CI run (#174, run 43638326995 and 43638326975): fixtures that are source code are stored as .txt. + root = os.path.abspath(os.path.join(SCRIPTS, "..")) + tracked = subprocess.run( + ["git", "-C", root, "ls-files", "tests"], + capture_output=True, + text=True, + check=True, + ).stdout.splitlines() + bad = [f for f in tracked if f.endswith((".go", ".rs")) and "/testdata/" not in f] + assert bad == [] + + +def test_a_source_file_under_tests_is_not_production_in_the_scope(): + scope = json.load(open(SCOPE_FILE)) + for lang, path in ( + ("go", "tests/test_tools/fixtures/x/main.go"), + ("rust", "tests/test_tools/fixtures/x/lib.rs"), + ): + cls, rule = om.classify(scope, lang, path) + assert cls == "test-data" and rule["reason"]