diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4fa882c..e58733a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -82,7 +82,8 @@ jobs: language-integration: runs-on: ubuntu-latest needs: [ test ] - if: github.event_name == 'push' || contains(github.event.pull_request.labels.*.name, 'test-languages') + # Runs on every PR. Parity is this project's entire claim, so a change that + # breaks it should fail before review, not after a merge to main. steps: - uses: actions/checkout@v6 with: @@ -106,7 +107,21 @@ jobs: node-version: '22' - name: Install Python pre-commit - run: pip install pre-commit + # Pinned to the line our version number claims parity with. The harness + # rejects any other line, so a bump here is a deliberate parity decision. + run: pip install 'pre-commit==4.6.2' - name: Run parity tests + # PARITY_REQUIRE turns "no Python found" and "zero checks ran" from a + # green skip into a failure. This job exists to make the comparison; a + # run that makes no comparison has not done its job. + env: + PARITY_REQUIRE: '1' run: go test -v -tags=integration -timeout=600s ./test/integration/ + + - name: Upload parity report + if: always() + uses: actions/upload-artifact@v4 + with: + name: parity-report + path: test/integration/parity_report.json diff --git a/test/integration/parity_test.go b/test/integration/parity_test.go index 12f6a0e..7c08ec1 100644 --- a/test/integration/parity_test.go +++ b/test/integration/parity_test.go @@ -6,6 +6,9 @@ // - Go binary buildable from the repo root // // Run with: go test -v -tags=integration -timeout=600s ./test/integration/ +// +//go:build integration + package integration import ( @@ -94,6 +97,14 @@ func TestMain(m *testing.M) { exitCode := m.Run() printParityReport() os.RemoveAll(tmp) + + // A run that measured nothing passed every check it ran, which is how a + // suite of skips renders as a green job. Under PARITY_REQUIRE that is the + // failure it looks like everywhere else. + if exitCode == 0 && len(parityReport.results) == 0 && os.Getenv("PARITY_REQUIRE") != "" { + fmt.Fprintln(os.Stderr, "PARITY_REQUIRE is set but zero parity checks ran") + exitCode = 1 + } os.Exit(exitCode) } @@ -121,6 +132,11 @@ func printParityReport() { fmt.Fprintln(w, " PRE-COMMIT PARITY REPORT") fmt.Fprintln(w, "================================================================================") fmt.Fprintf(w, " Generated: %s\n", time.Now().Format("2006-01-02 15:04:05")) + if pythonVer != "" { + fmt.Fprintf(w, " Measured against: %s (%s)\n", pythonVer, pythonPath) + } else { + fmt.Fprintf(w, " Measured against: NOTHING - no Python pre-commit was resolved\n") + } fmt.Fprintf(w, " Total checks: %d Pass: %d Fail: %d Parity: %.1f%%\n", total, pass, fail, pct) fmt.Fprintln(w, "================================================================================") @@ -158,20 +174,22 @@ func printParityReport() { repoRoot, _ := filepath.Abs(filepath.Join("..", "..")) reportPath := filepath.Join(repoRoot, "test", "integration", "parity_report.json") type jsonReport struct { - Generated string `json:"generated"` - Total int `json:"total"` - Pass int `json:"pass"` - Fail int `json:"fail"` - Parity string `json:"parity"` - Results []parityResult `json:"results"` + Generated string `json:"generated"` + MeasuredAgainst pythonInfo `json:"measured_against"` + Total int `json:"total"` + Pass int `json:"pass"` + Fail int `json:"fail"` + Parity string `json:"parity"` + Results []parityResult `json:"results"` } report := jsonReport{ - Generated: time.Now().Format(time.RFC3339), - Total: total, - Pass: pass, - Fail: fail, - Parity: fmt.Sprintf("%.1f%%", pct), - Results: parityReport.results, + Generated: time.Now().Format(time.RFC3339), + MeasuredAgainst: pythonInfo{Version: pythonVer}, + Total: total, + Pass: pass, + Fail: fail, + Parity: fmt.Sprintf("%.1f%%", pct), + Results: parityReport.results, } data, _ := json.MarshalIndent(report, "", " ") os.WriteFile(reportPath, data, 0o644) @@ -182,13 +200,117 @@ func printParityReport() { // Helpers // --------------------------------------------------------------------------- +// parityTarget is the Python pre-commit minor line this project claims parity +// with, and it is the meaning of our own version number: v4.6.x means "behaves +// like Python pre-commit 4.6.x". Comparing against any other line measures +// something we are not claiming, so resolvePython refuses to do it. +const parityTarget = "4.6" + +// pythonInfo describes the counterpart binary a parity run measured against. +// It goes into the JSON report because a parity percentage without the version +// it was taken against is not a claim anyone can check. +type pythonInfo struct { + Version string `json:"version"` +} + +var ( + pythonOnce sync.Once + pythonPath string + pythonVer string + pythonErr error +) + +// pythonPreCommit returns the path to the real Python pre-commit. +// +// The obvious implementation — LookPath("pre-commit") — is wrong here, and +// wrong in the most expensive way: this project installs a binary called +// pre-commit, so on any machine that has it, LookPath finds *us*. A parity +// suite that silently diffs the tool against an older build of itself reports +// a healthy-looking number that means nothing. (It did: the report committed +// before this change recorded py="pre-commit 4.5.0 (build v4.5.3)", which is +// this project's version format, not Python's.) +// +// So: walk every PATH entry rather than taking the first hit, and make each +// candidate prove it is Python by its --version output. func pythonPreCommit(t *testing.T) string { t.Helper() - path, err := exec.LookPath("pre-commit") - if err != nil { - t.Skip("Python pre-commit not found on PATH") + pythonOnce.Do(resolvePython) + if pythonErr != nil { + // In CI the whole point of the job is this comparison, so a missing or + // wrong counterpart is a failure. Locally it is a skip, because not + // every contributor wants a Python toolchain — but a loud one. + if os.Getenv("PARITY_REQUIRE") != "" { + t.Fatalf("PARITY_REQUIRE is set and no usable Python pre-commit was found: %v", pythonErr) + } + t.Skipf("skipping parity check: %v", pythonErr) + } + return pythonPath +} + +func resolvePython() { + var tried []string + + check := func(path string) bool { + out, err := exec.Command(path, "--version").CombinedOutput() + if err != nil { + tried = append(tried, fmt.Sprintf("%s (--version failed: %v)", path, err)) + return false + } + version := strings.TrimSpace(string(out)) + + // Our own binary stamps build metadata onto the version line; Python's + // prints "pre-commit X.Y.Z" and nothing else. That difference is the + // only reliable way to tell the two apart by execution alone. + if strings.Contains(version, "(build ") { + tried = append(tried, fmt.Sprintf("%s (this project, not Python: %q)", path, version)) + return false + } + + fields := strings.Fields(version) + if len(fields) < 2 || fields[0] != "pre-commit" { + tried = append(tried, fmt.Sprintf("%s (unrecognized --version output: %q)", path, version)) + return false + } + + if !strings.HasPrefix(fields[1], parityTarget+".") && fields[1] != parityTarget { + tried = append(tried, fmt.Sprintf("%s (version %s is not on the %s line this project claims parity with)", + path, fields[1], parityTarget)) + return false + } + + pythonPath, pythonVer = path, version + return true + } + + // An explicit override wins, and fails loudly if it is not what it claims — + // silently falling back to a search would hide the operator's mistake. + if override := os.Getenv("PARITY_PYTHON_PRE_COMMIT"); override != "" { + if check(override) { + return + } + pythonErr = fmt.Errorf("PARITY_PYTHON_PRE_COMMIT=%s is unusable: %s", override, strings.Join(tried, "; ")) + return + } + + for _, dir := range filepath.SplitList(os.Getenv("PATH")) { + if dir == "" { + continue + } + candidate := filepath.Join(dir, "pre-commit") + if info, err := os.Stat(candidate); err != nil || info.IsDir() { + continue + } + if check(candidate) { + return + } + } + + detail := "no pre-commit found on PATH" + if len(tried) > 0 { + detail = "rejected every candidate: " + strings.Join(tried, "; ") } - return path + pythonErr = fmt.Errorf("Python pre-commit %s.x is required (set PARITY_PYTHON_PRE_COMMIT to point at it); %s", + parityTarget, detail) } func runCmd(t *testing.T, dir, name string, args ...string) (combined string, exitCode int) {