Refuse a required context and a check name that disagree (#71)

.github/workflows/contexts.yml

@@ -0,0 +1,108 @@
+# The gate refers to check names, and this is what holds the two together.
+#
+# A required context is a string. A check name is a string in a workflow file.
+# Nothing on the platform ties them, so a job renamed in passing removes itself
+# from the required set while the tab still looks green, and a context required
+# under a name nothing reports blocks every merge without saying why on the pull
+# request. Both are failures of the same disagreement and this job refuses both.
+#
+# WHY THIS IS A WORKFLOW AND NOT A VERB OF THE RUNNER. Half the comparison is a
+# live setting on the hosting platform rather than a file in the tree, so it can
+# only be answered by asking the API. lab reads a checkout and opens no network
+# connection, and that claim is repeated in every document here, so the fetch
+# lives in the step below and the judgement lives in a command that is handed the
+# answer on standard input. The token is granted on this one job and nowhere
+# else, read-only, because reading a ruleset is all it is for.
+#
+# The check name below is what the required set refers to later, so it is written
+# here rather than left to the job id, and changing it is a change to the gate
+# rather than tidying. This job is the one place where that has a consequence it
+# can see: its own name is in the deliberate-absence list in
+# internal/contexts/contexts.go, so renaming it without moving that entry reddens
+# this check and the ordinary suite together.
+#
+# Hardened the way every other workflow in this tree is, because the audit job in
+# zizmor.yml refuses a new one that departs from it: permissions denied at the
+# top and granted per job, every action pinned to a commit with its version in a
+# comment, checkout without persisted credentials, and nothing expanded into a
+# run block.
+name: Required contexts
+
+on:
+  pull_request:
+    branches: ["**"]
+  push:
+    branches: [main]
+
+# Explicit deny-all at workflow level; the job below grants only what it needs.
+permissions: {}
+
+# Cancel a superseded run on the same ref. Nothing here publishes, so a run that
+# has been overtaken is safe to drop.
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  contexts:
+    name: required contexts
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    permissions:
+      # Read the tree, and read the ruleset. Both are reads and there is no
+      # third thing this job does.
+      contents: read
+    steps:
+      - name: Checkout Repository
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          persist-credentials: false
+
+      - name: Set up Go
+        uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0
+        with:
+          # The toolchain comes from go.mod, so the version this job uses moves
+          # when the module says so and never separately.
+          go-version-file: go.mod
+          # This module has no dependencies and therefore no go.sum for a cache
+          # key to be built from. Asking for a cache anyway makes the step
+          # depend on a file that is not there.
+          cache: false
+
+      - name: Read the contexts the ruleset requires
+        # The fetch is its own step so that a ruleset the run could not read is
+        # a different outcome from a ruleset that disagrees with the tree. A
+        # failed fetch here stops the job, and a job that had piped a failed
+        # fetch into the comparison would have compared the tree against an
+        # empty list and reported whatever that produced.
+        #
+        # An empty answer is not a failure. The ruleset on this board requires
+        # no status check today, and the comparison still has work to do in
+        # that state, which is why the count is printed rather than the run
+        # stopping.
+        #
+        # Every value reaches the script through the environment. Nothing an
+        # author controls is part of a command, and no expression is expanded
+        # inside a run block.
+        env:
+          GH_TOKEN: ${{ github.token }}
+          REPOSITORY: ${{ github.repository }}
+          DEFAULT_BRANCH: ${{ github.event.repository.default_branch }}
+        run: |
+          set -euo pipefail
+          if ! gh api "repos/${REPOSITORY}/rules/branches/${DEFAULT_BRANCH}" \
+                --jq '.[] | select(.type=="required_status_checks")
+                      | .parameters.required_status_checks[].context' > required.txt; then
+            echo "::error::The ruleset on ${DEFAULT_BRANCH} could not be read, so this run could not judge whether the gate and the tree agree. That is not the same as them agreeing."
+            exit 1
+          fi
+          echo "the ruleset on ${DEFAULT_BRANCH} requires $(wc -l < required.txt) context(s):"
+          cat required.txt
+
+      - name: Compare them against the check names this tree declares
+        # Built from this commit rather than downloaded, so a change that breaks
+        # a refusal is caught by the pull request that made it. The exit codes
+        # are the contract in docs/decisions/0011-the-exit-codes.md: 0 refused
+        # nothing, 1 refused something, 2 could not do the job. This step keys on
+        # all three by letting the command's own code become the job's.
+        run: go run ./cmd/contexts < required.txt

cmd/contexts/main.go

@@ -0,0 +1,135 @@
+// Command contexts compares the contexts the ruleset requires on the default
+// branch against the check names the workflows in this repository declare, and
+// refuses when the two disagree.
+//
+// WHY THIS IS NOT A VERB OF THE RUNNER. lab reads a checkout and opens no
+// network connection, which is a claim a downloader is asked to take on trust
+// and which cmd/lab's own suite holds it to. Half of this comparison is a live
+// setting on the hosting platform, so it can only be answered by asking the API,
+// and putting that inside lab would widen what an operator has to believe about
+// a binary they downloaded in exchange for a verb that is useless outside a
+// workflow. Keeping it here costs a third entry point under cmd/, which record
+// 0002 does not name, and that is the judgement in this file a reader should
+// check rather than accept. It is the same one cmd/pullrequest already made for
+// the same reason.
+//
+// THE API CALL IS NOT IN HERE EITHER. The required set arrives on standard
+// input, one context per line, so this command reads a checkout and a list and
+// nothing else. The workflow that runs it is where the token is, which keeps the
+// credential in one visible place and lets every rule below be proved against a
+// list a test wrote out in full.
+package main
+
+import (
+	"bufio"
+	"fmt"
+	"io"
+	"os"
+	"strings"
+
+	"github.com/Flowfin/lab/internal/contexts"
+)
+
+// The exit codes. Decision record 0011 is the contract, and this command returns
+// the same three the runner does, for the same reasons, which is why they are
+// written here with the record named rather than invented. The exit-code leg in
+// internal/invariants reads every exit-code constant in the tree and refuses a
+// code declared with two numbers, or a number declared under two codes, so keep
+// the convention these three follow.
+const (
+	// exitClean means the run completed and refused nothing. It does not mean
+	// the gate is intact, only that the two lists agree, and the report says
+	// what it compared so the two are not confused.
+	exitClean = 0
+
+	// exitRefused means the run completed and refused something. It is the
+	// only code that carries refusals.
+	exitRefused = 1
+
+	// exitCannot means the check could not do its job: no required set on
+	// standard input, no workflow directory to read, a file in a shape the
+	// reader was not built for. It is deliberately not the code a refusal
+	// returns, because a gate that treats a broken invocation like a violation
+	// reports one as the other and nobody investigates either.
+	exitCannot = 2
+)
+
+func main() {
+	os.Exit(run(os.Stdin, os.Stdout, os.Stderr, edges{
+		workflowsDir: contexts.WorkflowsDir,
+		absences:     contexts.Absences,
+		alsoReported: contexts.ReportedOutsideAWorkflowFile,
+	}))
+}
+
+// edges is everything this command reaches for that is not its own logic: where
+// the workflows are read from, which deliberate-absence list the comparison is
+// made against, and which check names arrive from something other than a job.
+// All three are parameters so that what the command prints and what it returns
+// can be read by a test against a tree written out in full, rather than against
+// whichever repository the suite happens to be running inside.
+type edges struct {
+	workflowsDir string
+	absences     []contexts.Absence
+	alsoReported []contexts.Declared
+}
+
+// run is main with its edges passed in.
+func run(in io.Reader, out, errOut io.Writer, e edges) int {
+	required, err := readRequired(in)
+	if err != nil {
+		fmt.Fprintf(errOut, "contexts: %v\n", err)
+		return exitCannot
+	}
+
+	declared, err := contexts.ReadWorkflows(e.workflowsDir)
+	if err != nil {
+		fmt.Fprintf(errOut, "contexts: %v\n", err)
+		return exitCannot
+	}
+	declared = append(declared, e.alsoReported...)
+
+	verdict := contexts.Judge(declared, required, e.absences)
+	fmt.Fprint(out, verdict.Report(declared, required, e.absences))
+	if len(verdict.Refusals) > 0 {
+		return exitRefused
+	}
+	return exitClean
+}
+
+// readRequired reads the contexts the ruleset requires, one per line.
+//
+// AN EMPTY LIST IS A LIST AND NOT A FAILURE. The ruleset on this board requires
+// no status check today, so a run that read nothing is the ordinary case rather
+// than a broken invocation, and the comparison still has work to do: every check
+// name the tree declares has to be written down as a deliberate absence, and an
+// absence naming nothing is still refused. A command that treated no input as a
+// reason to stop would switch the whole check off on exactly the board it was
+// written for.
+//
+// What it will not accept is a line that is not a context name. A required
+// context is a check-run name, and a line carrying a tab or a control character
+// is a fetch that returned something other than the list this expects, which is
+// a broken invocation rather than a gate that disagrees.
+func readRequired(in io.Reader) ([]string, error) {
+	var required []string
+	scanner := bufio.NewScanner(in)
+	for scanner.Scan() {
+		line := strings.TrimSpace(scanner.Text())
+		if line == "" {
+			continue
+		}
+		if strings.ContainsAny(line, "\t") || strings.ContainsFunc(line, isControl) {
+			return nil, fmt.Errorf("the required set carries the line %q, which is not a check-run name, so what arrived on standard input is not the list this expects", line)
+		}
+		required = append(required, line)
+	}
+	if err := scanner.Err(); err != nil {
+		return nil, fmt.Errorf("cannot read the required set: %w", err)
+	}
+	return required, nil
+}
+
+func isControl(r rune) bool {
+	return r < 0x20 || r == 0x7f
+}

cmd/contexts/main_test.go

@@ -0,0 +1,101 @@
+package main
+
+import (
+	"bytes"
+	"path/filepath"
+	"strings"
+	"testing"
+)
+
+// fixtures is a case directory from the package's own harness, reused here so
+// the command is proved against the same trees the rules are.
+const fixtures = "../../testdata/contexts"
+
+// TestTheCommandReturnsTheCodeItsVerdictEarns pins the contract record 0011
+// sets. Anything keyed on one of these codes is reading that record whether or
+// not anybody said so, and a gate that returned the same number for a
+// disagreement and for a broken fetch would report one as the other.
+func TestTheCommandReturnsTheCodeItsVerdictEarns(t *testing.T) {
+	for _, one := range []struct {
+		name     string
+		required string
+		dir      string
+		want     int
+	}{
+		{
+			name:     "the two lists agree",
+			required: "the first check\nthe second check\n",
+			dir:      filepath.Join(fixtures, "everything-agrees", "workflows"),
+			want:     exitClean,
+		},
+		{
+			name:     "a required context nothing reports",
+			required: "the first check\nthe second check\na check nobody wrote\n",
+			dir:      filepath.Join(fixtures, "a-required-context-nothing-reports", "workflows"),
+			want:     exitRefused,
+		},
+		{
+			name:     "there is no workflow directory to read",
+			required: "the first check\n",
+			dir:      filepath.Join(fixtures, "there-is-no-such-case", "workflows"),
+			want:     exitCannot,
+		},
+		{
+			name:     "what arrived is not a list of check names",
+			required: "the first check\tand a column of something else\n",
+			dir:      filepath.Join(fixtures, "everything-agrees", "workflows"),
+			want:     exitCannot,
+		},
+	} {
+		t.Run(one.name, func(t *testing.T) {
+			var out, errOut bytes.Buffer
+			got := run(strings.NewReader(one.required), &out, &errOut, edges{workflowsDir: one.dir})
+			if got != one.want {
+				t.Errorf("returned %d, want %d\nstdout: %s\nstderr: %s", got, one.want, out.String(), errOut.String())
+			}
+		})
+	}
+}
+
+// TestTheReportSaysWhatItCompared refuses a run whose whole output is a verdict.
+// A comparison of two empty lists and a comparison of two full ones both produce
+// no refusals, and the only thing that tells them apart is the count the run
+// printed.
+func TestTheReportSaysWhatItCompared(t *testing.T) {
+	var out, errOut bytes.Buffer
+	code := run(
+		strings.NewReader("the first check\nthe second check\n"),
+		&out, &errOut,
+		edges{workflowsDir: filepath.Join(fixtures, "everything-agrees", "workflows")},
+	)
+	if code != exitClean {
+		t.Fatalf("returned %d\n%s", code, errOut.String())
+	}
+	for _, want := range []string{
+		"required by the ruleset: 2",
+		"declared by the workflows: 2",
+		"written down as deliberately absent:",
+	} {
+		if !strings.Contains(out.String(), want) {
+			t.Errorf("the report does not say %q\n%s", want, out.String())
+		}
+	}
+}
+
+// TestAnEmptyRequiredSetIsCompared refuses a command that treats no required
+// contexts as nothing to do. That is the state of this board today, and a check
+// that switched itself off there would be a check that has never run.
+func TestAnEmptyRequiredSetIsCompared(t *testing.T) {
+	var out, errOut bytes.Buffer
+	code := run(
+		strings.NewReader(""),
+		&out, &errOut,
+		edges{workflowsDir: filepath.Join(fixtures, "everything-agrees", "workflows")},
+	)
+	if code != exitRefused {
+		t.Fatalf("returned %d, and with an empty required set every declared name is outside it\n%s%s", code, out.String(), errOut.String())
+	}
+	if !strings.Contains(out.String(), "required by the ruleset: 0") {
+		t.Errorf("the report does not say the required set was empty\n%s", out.String())
+	}
+}