diff --git a/.github/workflows/runtime-proof.yml b/.github/workflows/runtime-proof.yml index 941a9fe..8bb23d3 100644 --- a/.github/workflows/runtime-proof.yml +++ b/.github/workflows/runtime-proof.yml @@ -982,6 +982,10 @@ jobs: permissions: actions: read contents: read + # To comment on #125 when the criterion is met. Read-only here would make + # the announcement below a no-op that reports success, which is the shape + # of failure this whole change is about. + issues: write steps: - name: Harden the runner uses: step-security/harden-runner@e14015d583714f6e62063499dc959a02595150a1 # v2.21.1 @@ -989,6 +993,7 @@ jobs: egress-policy: audit - name: Count the streak, from the Actions history + id: count env: GH_TOKEN: ${{ github.token }} TARGET: '14' @@ -1032,6 +1037,25 @@ jobs: } >> "$GITHUB_STEP_SUMMARY" echo "streak=${streak}/${TARGET}" + echo "streak=${streak}" >> "$GITHUB_OUTPUT" + + # The summary above is where this said "the criterion is met" on + # 2026-09-21, and nobody read it. The streak broke the next night and the + # moment passed. A job log and a step summary are the two places nobody + # opens without already knowing there is a problem — the sentence the + # report job below already carries for red nights, never applied to the + # half that asks for action. + # + # So it says it on the issue, once. tools/ci/streak-report.sh holds the + # decision because a run: block cannot be executed outside Actions, and + # tools/ci/streak_report_test.go drives it with a stubbed gh. + - name: A met criterion says so on the issue + if: steps.count.outputs.streak != '' + env: + GH_TOKEN: ${{ github.token }} + GITHUB_REPOSITORY: ${{ github.repository }} + FEINT_RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + run: tools/ci/streak-report.sh --apply "${{ steps.count.outputs.streak }}" 14 125 # A red night tells somebody (#502). Ten scheduled reds in twelve nights # (#501) left no trace outside job logs and the summary above — the two diff --git a/tools/ci/streak-report.sh b/tools/ci/streak-report.sh new file mode 100755 index 0000000..f7b88b8 --- /dev/null +++ b/tools/ci/streak-report.sh @@ -0,0 +1,102 @@ +#!/usr/bin/env bash +# A met promotion criterion says so where somebody reads it (#125). +# +# runtime-proof.yml counts consecutive green scheduled runs and, at fourteen, +# writes into $GITHUB_STEP_SUMMARY: "The criterion is met. Move this workflow +# onto pull_request and close #125." +# +# It did exactly that on 2026-09-21. Nobody read it. The streak broke the next +# night — #740 armed `guard.sh verification` and a real broken claim reddened +# four nights running — and the moment passed unnoticed. +# +# The failure mode is the one the workflow's own report job already names, one +# sentence above this file's reason for existing: a job log and a step summary +# are "the two places nobody opens without already knowing there is a problem". +# That lesson was applied to red nights (#502) and not to a met criterion, so +# the half that asks for action was the half staying silent. +# +# So this posts a comment on the issue instead. Once — a comment carrying the +# marker below already on the issue means it has been said, and saying it again +# every night is how a notification becomes noise somebody mutes. +# +# The logic lives here rather than in a run: block, for the reason +# night-report.sh states: a run: block cannot be executed outside GitHub +# Actions, and this repository has paid for CI fixes described in comments and +# never executed. The controls on this file: +# +# - tools/ci/streak_report_test.go drives it with a stubbed `gh`; +# - tools/falsify/specs/a-met-criterion-is-announced.json replays those tests +# with each decision neutralised. +# +# Usage: streak-report.sh [--apply] +# +# Without --apply it prints what it would do and writes nothing. +set -euo pipefail + +apply=false +if [ "${1:-}" = "--apply" ]; then + apply=true + shift +fi + +streak="${1:?usage: streak-report.sh [--apply] }" +target="${2:?usage: streak-report.sh [--apply] }" +issue="${3:?usage: streak-report.sh [--apply] }" +repo="${GITHUB_REPOSITORY:-stephrobert/feint}" +run_url="${FEINT_RUN_URL:-}" + +# The marker is what makes this idempotent. It is invisible in the rendered +# comment and unique to this announcement, so a maintainer's own prose about +# the streak never counts as one. +marker="" + +if [ "$streak" -lt "$target" ]; then + echo "verdict: not met (${streak}/${target}), nothing to announce" + exit 0 +fi + +# Said once. `gh issue view --json comments` answers every comment, and the +# marker is searched in their bodies rather than in a title or an author: a +# reply quoting this comment must not count as the comment itself, so the +# marker sits on its own line and nothing else writes it. +already=false +if comments="$(gh issue view "${issue}" --repo "${repo}" --json comments \ + --jq '.comments[].body' 2>/dev/null)"; then + if printf '%s' "${comments}" | grep -qF "${marker}"; then + already=true + fi +fi + +if [ "$already" = true ]; then + echo "verdict: met (${streak}/${target}), already announced on #${issue}" + exit 0 +fi + +body_file="$(mktemp)" +trap 'rm -f "${body_file}"' EXIT +{ + echo "${marker}" + echo + echo "**The promotion criterion is met: ${streak} consecutive green scheduled runs, target ${target}.**" + echo + echo "This is the number this issue asks for, counted from the Actions history by the" + echo "\`Consecutive green scheduled runs\` job rather than from anybody's memory. Nothing" + echo "else has to be decided: the remaining work is to move \`runtime-proof.yml\` onto" + echo "\`pull_request\` and close this issue." + echo + if [ -n "${run_url}" ]; then + echo "The run that counted it: ${run_url}" + echo + fi + echo "Posted automatically, once. The criterion was met before — on 2026-09-21, with" + echo "fourteen greens — and the only trace was a step summary, which is why this comment" + echo "exists at all." +} > "${body_file}" + +echo "verdict: met (${streak}/${target}), announcing on #${issue}" +if [ "$apply" = true ]; then + gh issue comment "${issue}" --repo "${repo}" --body-file "${body_file}" +else + echo "--- the comment it would post ---" + cat "${body_file}" +fi diff --git a/tools/ci/streak_report_test.go b/tools/ci/streak_report_test.go new file mode 100644 index 0000000..aeef197 --- /dev/null +++ b/tools/ci/streak_report_test.go @@ -0,0 +1,249 @@ +package ci + +import ( + "os" + "os/exec" + "path/filepath" + "strconv" + "strings" + "testing" +) + +// A met promotion criterion says so where somebody reads it (#125). +// +// The streak job counts consecutive green scheduled runs and, at fourteen, +// writes the verdict into $GITHUB_STEP_SUMMARY. It did on 2026-09-21. Nobody +// read it, the streak broke the next night, and the moment passed — which is +// the failure the workflow's own report job already names: a job log and a step +// summary are the two places nobody opens without already knowing there is a +// problem. +// +// These tests drive the real script with a stubbed `gh`, so the writes land in +// a log instead of on the repository's issues. + +// streakStub answers the two calls the script makes and records every write. +// It dispatches on the subcommand rather than on the whole argument line: a +// body carrying the word "comment" must not be mistaken for the verb. +const streakStub = `#!/bin/sh +set -eu +case "$1" in + issue) + case "$2" in + view) + cat "${GH_STUB_COMMENTS}" + ;; + comment) + printf '%s\n' "$*" >>"${GH_STUB_LOG}" + prev="" + for a in "$@"; do + if [ "${prev}" = "--body-file" ]; then + cat "${a}" >>"${GH_STUB_LOG}" + fi + prev="${a}" + done + ;; + *) + echo "unexpected gh issue call: $*" >&2 + exit 64 + ;; + esac + ;; + *) + echo "unexpected gh call: $*" >&2 + exit 64 + ;; +esac +` + +// runStreak executes streak-report.sh and returns its exit code, its output, +// and the log of every gh write the stub received. comments is what +// `gh issue view --json comments --jq '.comments[].body'` would answer. +func runStreak(t *testing.T, streak, target int, comments string, apply bool) (int, string, string) { + t.Helper() + + stubDir := t.TempDir() + stub := filepath.Join(stubDir, "gh") + if err := os.WriteFile(stub, []byte(streakStub), 0o755); err != nil { //nolint:gosec // a stub this test runs + t.Fatal(err) + } + commentsFile := filepath.Join(stubDir, "comments.txt") + if err := os.WriteFile(commentsFile, []byte(comments), 0o600); err != nil { + t.Fatal(err) + } + log := filepath.Join(stubDir, "calls.log") + + args := []string{"streak-report.sh"} + if apply { + args = append(args, "--apply") + } + args = append(args, strconv.Itoa(streak), strconv.Itoa(target), "125") + + cmd := exec.Command("bash", args...) + cmd.Env = append(os.Environ(), + "PATH="+stubDir+":"+os.Getenv("PATH"), + "GH_STUB_COMMENTS="+commentsFile, + "GH_STUB_LOG="+log, + "GITHUB_REPOSITORY=stephrobert/feint", + "FEINT_RUN_URL=https://example.invalid/run/1", + ) + out, err := cmd.CombinedOutput() + code := 0 + var exit *exec.ExitError + if err != nil { + if ok := asExitError(err, &exit); ok { + code = exit.ExitCode() + } else { + t.Fatalf("run streak-report.sh: %v\n%s", err, out) + } + } + written, _ := os.ReadFile(log) //nolint:gosec // a path this test made + return code, string(out), string(written) +} + +// Below the target, nothing is said. +// +// A comment every night would be the shape #502 refuses for red nights: a +// notification that arrives whatever happens teaches its reader to skip it. +func TestAStreakBelowTheTargetAnnouncesNothing(t *testing.T) { + code, out, writes := runStreak(t, 13, 14, "", true) + if code != 0 { + t.Fatalf("exit %d for a streak below target: %s", code, out) + } + if writes != "" { + t.Errorf("it wrote to the issue at 13/14:\n%s", writes) + } + if !strings.Contains(out, "not met") { + t.Errorf("the verdict does not say the criterion is unmet: %s", out) + } +} + +// At the target, the issue gets the comment. +// +// This is the whole point: on 2026-09-21 the criterion was met and the only +// trace was a step summary. +func TestAMetCriterionIsAnnouncedOnTheIssue(t *testing.T) { + code, out, writes := runStreak(t, 14, 14, "", true) + if code != 0 { + t.Fatalf("exit %d when the criterion is met: %s", code, out) + } + if !strings.Contains(writes, "comment") || !strings.Contains(writes, "125") { + t.Fatalf("no comment was posted on #125 when the criterion was met:\n%s", writes) + } + // The number is in the comment, because "the criterion is met" without it + // asks the reader to go and count. + if !strings.Contains(writes, "14 consecutive green scheduled runs") { + t.Errorf("the comment does not carry the count that earns it:\n%s", writes) + } + // And what to do next, since the issue's own answer is a one-line action. + if !strings.Contains(writes, "pull_request") { + t.Errorf("the comment does not name the promotion it asks for:\n%s", writes) + } +} + +// A streak past the target still announces, once. +// +// Fourteen is a floor, not an equality: a run at fifteen must not fall through +// the condition and go silent. +func TestAStreakPastTheTargetStillAnnounces(t *testing.T) { + _, _, writes := runStreak(t, 21, 14, "", true) + if !strings.Contains(writes, "comment") { + t.Errorf("nothing was posted at 21/14: the criterion is a floor, not an equality:\n%s", writes) + } +} + +// It is said once, not every night. +// +// The marker is what makes it idempotent. Without this, a met criterion posts a +// comment every night until somebody acts, which is how a notification becomes +// noise somebody mutes — and then the next one is missed too. +func TestAnAnnouncedCriterionIsNotRepeated(t *testing.T) { + const posted = "\n\nThe promotion criterion is met." + code, out, writes := runStreak(t, 15, 14, posted, true) + if code != 0 { + t.Fatalf("exit %d on a second night: %s", code, out) + } + if writes != "" { + t.Errorf("it posted a second time:\n%s", writes) + } + if !strings.Contains(out, "already announced") { + t.Errorf("the verdict does not say it was already announced: %s", out) + } +} + +// A maintainer's own prose about the streak is not the announcement. +// +// The marker is searched, not the words: somebody writing "the criterion is +// met" in a discussion must not silence the mechanism. +func TestHumanProseDoesNotCountAsTheAnnouncement(t *testing.T) { + const prose = "I think the promotion criterion is met, should we move it onto pull_request?" + _, _, writes := runStreak(t, 14, 14, prose, true) + if !strings.Contains(writes, "comment") { + t.Errorf("a comment merely mentioning the criterion silenced the announcement:\n%s", writes) + } +} + +// Without --apply it writes nothing, so the script can be run against any past +// number and read. +func TestTheStreakReportWritesNothingWithoutApply(t *testing.T) { + _, out, writes := runStreak(t, 14, 14, "", false) + if writes != "" { + t.Errorf("it wrote without --apply:\n%s", writes) + } + if !strings.Contains(out, "the comment it would post") { + t.Errorf("a dry run does not show what it would post: %s", out) + } +} + +// The workflow actually calls the announcement. +// +// The script and its tests prove the decision; they say nothing about whether +// anything runs it. On 2026-09-21 the counting worked perfectly and the verdict +// reached nobody, so "the logic is correct" was already true when the failure +// happened. +func TestTheStreakJobAnnouncesOnTheIssue(t *testing.T) { + workflow := readWorkflow(t, "runtime-proof.yml") + + block := stepBlock(workflow, "A met criterion says so on the issue") + if block == "" { + t.Fatal("runtime-proof.yml counts the streak and never announces it: the verdict " + + "reaches a step summary and stops there, which is what happened on 2026-09-21") + } + + // The command the step RUNS, not a substring of the line. Asserting + // `Contains(…, "streak-report.sh --apply")` passes for + // `echo streak-report.sh --apply`, which announces nothing — falsify caught + // exactly that, and it is the third time in this repository that a Contains + // has been satisfied by a longer string that does the opposite. + run := commandOf(block) + if !strings.HasPrefix(run, "tools/ci/streak-report.sh --apply") { + t.Fatalf("the announcing step runs %q, not the announcement", run) + } + // Writing on an issue needs the permission; without it the step fails, or + // worse, reports success having posted nothing. + streakJob := workflow[strings.Index(workflow, " streak:"):] + if cut := strings.Index(streakJob, "\n report:"); cut > 0 { + streakJob = streakJob[:cut] + } + if !strings.Contains(streakJob, "issues: write") { + t.Error("the streak job cannot write on an issue: `issues: write` is missing, and a " + + "comment it cannot post is a silence that reports success") + } + // The issue it announces on is this one; a typo here posts the verdict + // somewhere nobody is waiting for it. + if !strings.Contains(block, " 125") { + t.Errorf("the announcement does not name issue 125:\n%s", block) + } +} + +// commandOf answers a step's `run:` command, or "" when it declares none. +// +// The value, so a check on it cannot be satisfied by a longer line that merely +// contains it. +func commandOf(block string) string { + for _, line := range strings.Split(block, "\n") { + if after, found := strings.CutPrefix(strings.TrimSpace(line), "run:"); found { + return strings.TrimSpace(after) + } + } + return "" +} diff --git a/tools/falsify/specs/a-met-criterion-is-announced.json b/tools/falsify/specs/a-met-criterion-is-announced.json new file mode 100644 index 0000000..5ce619f --- /dev/null +++ b/tools/falsify/specs/a-met-criterion-is-announced.json @@ -0,0 +1,47 @@ +{ + "package": "./tools/ci/", + "mutations": [ + { + "label": "a met criterion goes back to a step summary alone, which is where it was said on 2026-09-21 and where nobody read it", + "file": ".github/workflows/runtime-proof.yml", + "find": " run: tools/ci/streak-report.sh --apply \"${{ steps.count.outputs.streak }}\" 14 125", + "replace": " run: echo tools/ci/streak-report.sh --apply \"${{ steps.count.outputs.streak }}\" 14 125", + "test": "TestTheStreakJobAnnouncesOnTheIssue" + }, + { + "label": "the job cannot write on an issue, so the announcement is a silence that reports success", + "file": ".github/workflows/runtime-proof.yml", + "find": " issues: write\n steps:\n - name: Harden the runner", + "replace": " # issues: write\n issues: read\n steps:\n - name: Harden the runner", + "test": "TestTheStreakJobAnnouncesOnTheIssue" + }, + { + "label": "the criterion is read as an equality, so a streak past the target falls through and goes silent", + "file": "tools/ci/streak-report.sh", + "find": "if [ \"$streak\" -lt \"$target\" ]; then", + "replace": "if [ \"$streak\" -lt \"$target\" ] || [ \"$streak\" -gt \"$target\" ]; then", + "test": "TestAStreakPastTheTargetStillAnnounces" + }, + { + "label": "the announcement repeats every night, which is how a notification becomes noise somebody mutes", + "file": "tools/ci/streak-report.sh", + "find": "if [ \"$already\" = true ]; then", + "replace": "if [ \"$already\" = maybe ]; then", + "test": "TestAnAnnouncedCriterionIsNotRepeated" + }, + { + "label": "a maintainer writing about the criterion silences the mechanism, because the words are searched rather than the marker", + "file": "tools/ci/streak-report.sh", + "find": " if printf '%s' \"${comments}\" | grep -qF \"${marker}\"; then", + "replace": " if printf '%s' \"${comments}\" | grep -qF \"criterion is met\"; then", + "test": "TestHumanProseDoesNotCountAsTheAnnouncement" + }, + { + "label": "it announces below the target too, so the comment arrives whatever happens and stops meaning anything", + "file": "tools/ci/streak-report.sh", + "find": " echo \"verdict: not met (${streak}/${target}), nothing to announce\"\n exit 0", + "replace": " echo \"verdict: not met (${streak}/${target}), nothing to announce\"\n if [ -z \"$streak\" ]; then exit 0; fi\n streak=\"$target\"", + "test": "TestAStreakBelowTheTargetAnnouncesNothing" + } + ] +}