From d03c292b6f546e2290b35c13aa35fcc1faf1e288 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20ROBERT?= Date: Fri, 25 Sep 2026 12:43:01 +0200 Subject: [PATCH] fix(ci): a met promotion criterion says so where somebody reads it MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit runtime-proof.yml counts consecutive green scheduled runs and, at fourteen, writes into the step summary: The criterion is met. Move this workflow onto pull_request and close #125. It wrote exactly that on 2026-09-21, with fourteen greens from the 8th. Nobody read it. The streak broke the next night — #740 armed `guard.sh verification` and a real broken claim has reddened every night since — and the moment passed unnoticed. #125 is still open, and the number it asks for was reached four days ago. THE FAILURE IS THE ONE THIS WORKFLOW ALREADY NAMES Its own report job carries the sentence, about red nights: left no trace outside job logs and the summary above — the two places nobody opens without already knowing there is a problem That lesson was applied to failure (#502: one issue, opened, updated, closed) and never to the half that asks for action. So the mechanism that measures the criterion had no way to tell anyone it was met, and the streak job's verdict had exactly the reach the red nights had before #502. WHAT CHANGES The criterion, once met, is announced on the issue. Once: a marker in the comment body makes it idempotent, because a comment every night is how a notification becomes noise somebody mutes — which is the shape #502 refuses for red nights and would be no better here. The decision lives in tools/ci/streak-report.sh rather than in a run: block, for the reason night-report.sh states: a run: block cannot be executed outside Actions, and this repository has paid for CI fixes described in comments and never run. WHAT FALSIFY FOUND IN THIS WORK The workflow test asserted `Contains(workflow, "streak-report.sh --apply")`, which `echo streak-report.sh --apply` satisfies while announcing nothing. It reads the step's `run:` value now. That is the third time in this repository a Contains has been satisfied by a longer string doing the opposite — after `if: always() == false` and a step commented out — so the helper reads the whole value rather than looking for a substring. PROVEN - Seven tests driving the real script with a stubbed gh: below target, at target, past target, already announced, a maintainer's prose that must not silence it, a dry run, and the workflow actually calling it. - Six mutations, each compiling and each biting, including the two that matter most: the announcement reduced to a step summary again, and the job left without permission to write — a silence that reports success. - `mise run prepush` green. WHAT THIS DOES NOT DO It does not close #125. The streak is 0 and the criterion needs fourteen, so the issue closes when the nights earn it — which is the rule its header states. What this removes is the need for anyone to watch for the moment. Assisted-by: Claude Code (claude-opus-5) --- .github/workflows/runtime-proof.yml | 24 ++ tools/ci/streak-report.sh | 102 +++++++ tools/ci/streak_report_test.go | 249 ++++++++++++++++++ .../specs/a-met-criterion-is-announced.json | 47 ++++ 4 files changed, 422 insertions(+) create mode 100755 tools/ci/streak-report.sh create mode 100644 tools/ci/streak_report_test.go create mode 100644 tools/falsify/specs/a-met-criterion-is-announced.json diff --git a/.github/workflows/runtime-proof.yml b/.github/workflows/runtime-proof.yml index 941a9fe..8bb23d3 100644 --- a/.github/workflows/runtime-proof.yml +++ b/.github/workflows/runtime-proof.yml @@ -982,6 +982,10 @@ jobs: permissions: actions: read contents: read + # To comment on #125 when the criterion is met. Read-only here would make + # the announcement below a no-op that reports success, which is the shape + # of failure this whole change is about. + issues: write steps: - name: Harden the runner uses: step-security/harden-runner@e14015d583714f6e62063499dc959a02595150a1 # v2.21.1 @@ -989,6 +993,7 @@ jobs: egress-policy: audit - name: Count the streak, from the Actions history + id: count env: GH_TOKEN: ${{ github.token }} TARGET: '14' @@ -1032,6 +1037,25 @@ jobs: } >> "$GITHUB_STEP_SUMMARY" echo "streak=${streak}/${TARGET}" + echo "streak=${streak}" >> "$GITHUB_OUTPUT" + + # The summary above is where this said "the criterion is met" on + # 2026-09-21, and nobody read it. The streak broke the next night and the + # moment passed. A job log and a step summary are the two places nobody + # opens without already knowing there is a problem — the sentence the + # report job below already carries for red nights, never applied to the + # half that asks for action. + # + # So it says it on the issue, once. tools/ci/streak-report.sh holds the + # decision because a run: block cannot be executed outside Actions, and + # tools/ci/streak_report_test.go drives it with a stubbed gh. + - name: A met criterion says so on the issue + if: steps.count.outputs.streak != '' + env: + GH_TOKEN: ${{ github.token }} + GITHUB_REPOSITORY: ${{ github.repository }} + FEINT_RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + run: tools/ci/streak-report.sh --apply "${{ steps.count.outputs.streak }}" 14 125 # A red night tells somebody (#502). Ten scheduled reds in twelve nights # (#501) left no trace outside job logs and the summary above — the two diff --git a/tools/ci/streak-report.sh b/tools/ci/streak-report.sh new file mode 100755 index 0000000..f7b88b8 --- /dev/null +++ b/tools/ci/streak-report.sh @@ -0,0 +1,102 @@ +#!/usr/bin/env bash +# A met promotion criterion says so where somebody reads it (#125). +# +# runtime-proof.yml counts consecutive green scheduled runs and, at fourteen, +# writes into $GITHUB_STEP_SUMMARY: "The criterion is met. Move this workflow +# onto pull_request and close #125." +# +# It did exactly that on 2026-09-21. Nobody read it. The streak broke the next +# night — #740 armed `guard.sh verification` and a real broken claim reddened +# four nights running — and the moment passed unnoticed. +# +# The failure mode is the one the workflow's own report job already names, one +# sentence above this file's reason for existing: a job log and a step summary +# are "the two places nobody opens without already knowing there is a problem". +# That lesson was applied to red nights (#502) and not to a met criterion, so +# the half that asks for action was the half staying silent. +# +# So this posts a comment on the issue instead. Once — a comment carrying the +# marker below already on the issue means it has been said, and saying it again +# every night is how a notification becomes noise somebody mutes. +# +# The logic lives here rather than in a run: block, for the reason +# night-report.sh states: a run: block cannot be executed outside GitHub +# Actions, and this repository has paid for CI fixes described in comments and +# never executed. The controls on this file: +# +# - tools/ci/streak_report_test.go drives it with a stubbed `gh`; +# - tools/falsify/specs/a-met-criterion-is-announced.json replays those tests +# with each decision neutralised. +# +# Usage: streak-report.sh [--apply] +# +# Without --apply it prints what it would do and writes nothing. +set -euo pipefail + +apply=false +if [ "${1:-}" = "--apply" ]; then + apply=true + shift +fi + +streak="${1:?usage: streak-report.sh [--apply] }" +target="${2:?usage: streak-report.sh [--apply] }" +issue="${3:?usage: streak-report.sh [--apply] }" +repo="${GITHUB_REPOSITORY:-stephrobert/feint}" +run_url="${FEINT_RUN_URL:-}" + +# The marker is what makes this idempotent. It is invisible in the rendered +# comment and unique to this announcement, so a maintainer's own prose about +# the streak never counts as one. +marker="" + +if [ "$streak" -lt "$target" ]; then + echo "verdict: not met (${streak}/${target}), nothing to announce" + exit 0 +fi + +# Said once. `gh issue view --json comments` answers every comment, and the +# marker is searched in their bodies rather than in a title or an author: a +# reply quoting this comment must not count as the comment itself, so the +# marker sits on its own line and nothing else writes it. +already=false +if comments="$(gh issue view "${issue}" --repo "${repo}" --json comments \ + --jq '.comments[].body' 2>/dev/null)"; then + if printf '%s' "${comments}" | grep -qF "${marker}"; then + already=true + fi +fi + +if [ "$already" = true ]; then + echo "verdict: met (${streak}/${target}), already announced on #${issue}" + exit 0 +fi + +body_file="$(mktemp)" +trap 'rm -f "${body_file}"' EXIT +{ + echo "${marker}" + echo + echo "**The promotion criterion is met: ${streak} consecutive green scheduled runs, target ${target}.**" + echo + echo "This is the number this issue asks for, counted from the Actions history by the" + echo "\`Consecutive green scheduled runs\` job rather than from anybody's memory. Nothing" + echo "else has to be decided: the remaining work is to move \`runtime-proof.yml\` onto" + echo "\`pull_request\` and close this issue." + echo + if [ -n "${run_url}" ]; then + echo "The run that counted it: ${run_url}" + echo + fi + echo "Posted automatically, once. The criterion was met before — on 2026-09-21, with" + echo "fourteen greens — and the only trace was a step summary, which is why this comment" + echo "exists at all." +} > "${body_file}" + +echo "verdict: met (${streak}/${target}), announcing on #${issue}" +if [ "$apply" = true ]; then + gh issue comment "${issue}" --repo "${repo}" --body-file "${body_file}" +else + echo "--- the comment it would post ---" + cat "${body_file}" +fi diff --git a/tools/ci/streak_report_test.go b/tools/ci/streak_report_test.go new file mode 100644 index 0000000..aeef197 --- /dev/null +++ b/tools/ci/streak_report_test.go @@ -0,0 +1,249 @@ +package ci + +import ( + "os" + "os/exec" + "path/filepath" + "strconv" + "strings" + "testing" +) + +// A met promotion criterion says so where somebody reads it (#125). +// +// The streak job counts consecutive green scheduled runs and, at fourteen, +// writes the verdict into $GITHUB_STEP_SUMMARY. It did on 2026-09-21. Nobody +// read it, the streak broke the next night, and the moment passed — which is +// the failure the workflow's own report job already names: a job log and a step +// summary are the two places nobody opens without already knowing there is a +// problem. +// +// These tests drive the real script with a stubbed `gh`, so the writes land in +// a log instead of on the repository's issues. + +// streakStub answers the two calls the script makes and records every write. +// It dispatches on the subcommand rather than on the whole argument line: a +// body carrying the word "comment" must not be mistaken for the verb. +const streakStub = `#!/bin/sh +set -eu +case "$1" in + issue) + case "$2" in + view) + cat "${GH_STUB_COMMENTS}" + ;; + comment) + printf '%s\n' "$*" >>"${GH_STUB_LOG}" + prev="" + for a in "$@"; do + if [ "${prev}" = "--body-file" ]; then + cat "${a}" >>"${GH_STUB_LOG}" + fi + prev="${a}" + done + ;; + *) + echo "unexpected gh issue call: $*" >&2 + exit 64 + ;; + esac + ;; + *) + echo "unexpected gh call: $*" >&2 + exit 64 + ;; +esac +` + +// runStreak executes streak-report.sh and returns its exit code, its output, +// and the log of every gh write the stub received. comments is what +// `gh issue view --json comments --jq '.comments[].body'` would answer. +func runStreak(t *testing.T, streak, target int, comments string, apply bool) (int, string, string) { + t.Helper() + + stubDir := t.TempDir() + stub := filepath.Join(stubDir, "gh") + if err := os.WriteFile(stub, []byte(streakStub), 0o755); err != nil { //nolint:gosec // a stub this test runs + t.Fatal(err) + } + commentsFile := filepath.Join(stubDir, "comments.txt") + if err := os.WriteFile(commentsFile, []byte(comments), 0o600); err != nil { + t.Fatal(err) + } + log := filepath.Join(stubDir, "calls.log") + + args := []string{"streak-report.sh"} + if apply { + args = append(args, "--apply") + } + args = append(args, strconv.Itoa(streak), strconv.Itoa(target), "125") + + cmd := exec.Command("bash", args...) + cmd.Env = append(os.Environ(), + "PATH="+stubDir+":"+os.Getenv("PATH"), + "GH_STUB_COMMENTS="+commentsFile, + "GH_STUB_LOG="+log, + "GITHUB_REPOSITORY=stephrobert/feint", + "FEINT_RUN_URL=https://example.invalid/run/1", + ) + out, err := cmd.CombinedOutput() + code := 0 + var exit *exec.ExitError + if err != nil { + if ok := asExitError(err, &exit); ok { + code = exit.ExitCode() + } else { + t.Fatalf("run streak-report.sh: %v\n%s", err, out) + } + } + written, _ := os.ReadFile(log) //nolint:gosec // a path this test made + return code, string(out), string(written) +} + +// Below the target, nothing is said. +// +// A comment every night would be the shape #502 refuses for red nights: a +// notification that arrives whatever happens teaches its reader to skip it. +func TestAStreakBelowTheTargetAnnouncesNothing(t *testing.T) { + code, out, writes := runStreak(t, 13, 14, "", true) + if code != 0 { + t.Fatalf("exit %d for a streak below target: %s", code, out) + } + if writes != "" { + t.Errorf("it wrote to the issue at 13/14:\n%s", writes) + } + if !strings.Contains(out, "not met") { + t.Errorf("the verdict does not say the criterion is unmet: %s", out) + } +} + +// At the target, the issue gets the comment. +// +// This is the whole point: on 2026-09-21 the criterion was met and the only +// trace was a step summary. +func TestAMetCriterionIsAnnouncedOnTheIssue(t *testing.T) { + code, out, writes := runStreak(t, 14, 14, "", true) + if code != 0 { + t.Fatalf("exit %d when the criterion is met: %s", code, out) + } + if !strings.Contains(writes, "comment") || !strings.Contains(writes, "125") { + t.Fatalf("no comment was posted on #125 when the criterion was met:\n%s", writes) + } + // The number is in the comment, because "the criterion is met" without it + // asks the reader to go and count. + if !strings.Contains(writes, "14 consecutive green scheduled runs") { + t.Errorf("the comment does not carry the count that earns it:\n%s", writes) + } + // And what to do next, since the issue's own answer is a one-line action. + if !strings.Contains(writes, "pull_request") { + t.Errorf("the comment does not name the promotion it asks for:\n%s", writes) + } +} + +// A streak past the target still announces, once. +// +// Fourteen is a floor, not an equality: a run at fifteen must not fall through +// the condition and go silent. +func TestAStreakPastTheTargetStillAnnounces(t *testing.T) { + _, _, writes := runStreak(t, 21, 14, "", true) + if !strings.Contains(writes, "comment") { + t.Errorf("nothing was posted at 21/14: the criterion is a floor, not an equality:\n%s", writes) + } +} + +// It is said once, not every night. +// +// The marker is what makes it idempotent. Without this, a met criterion posts a +// comment every night until somebody acts, which is how a notification becomes +// noise somebody mutes — and then the next one is missed too. +func TestAnAnnouncedCriterionIsNotRepeated(t *testing.T) { + const posted = "\n\nThe promotion criterion is met." + code, out, writes := runStreak(t, 15, 14, posted, true) + if code != 0 { + t.Fatalf("exit %d on a second night: %s", code, out) + } + if writes != "" { + t.Errorf("it posted a second time:\n%s", writes) + } + if !strings.Contains(out, "already announced") { + t.Errorf("the verdict does not say it was already announced: %s", out) + } +} + +// A maintainer's own prose about the streak is not the announcement. +// +// The marker is searched, not the words: somebody writing "the criterion is +// met" in a discussion must not silence the mechanism. +func TestHumanProseDoesNotCountAsTheAnnouncement(t *testing.T) { + const prose = "I think the promotion criterion is met, should we move it onto pull_request?" + _, _, writes := runStreak(t, 14, 14, prose, true) + if !strings.Contains(writes, "comment") { + t.Errorf("a comment merely mentioning the criterion silenced the announcement:\n%s", writes) + } +} + +// Without --apply it writes nothing, so the script can be run against any past +// number and read. +func TestTheStreakReportWritesNothingWithoutApply(t *testing.T) { + _, out, writes := runStreak(t, 14, 14, "", false) + if writes != "" { + t.Errorf("it wrote without --apply:\n%s", writes) + } + if !strings.Contains(out, "the comment it would post") { + t.Errorf("a dry run does not show what it would post: %s", out) + } +} + +// The workflow actually calls the announcement. +// +// The script and its tests prove the decision; they say nothing about whether +// anything runs it. On 2026-09-21 the counting worked perfectly and the verdict +// reached nobody, so "the logic is correct" was already true when the failure +// happened. +func TestTheStreakJobAnnouncesOnTheIssue(t *testing.T) { + workflow := readWorkflow(t, "runtime-proof.yml") + + block := stepBlock(workflow, "A met criterion says so on the issue") + if block == "" { + t.Fatal("runtime-proof.yml counts the streak and never announces it: the verdict " + + "reaches a step summary and stops there, which is what happened on 2026-09-21") + } + + // The command the step RUNS, not a substring of the line. Asserting + // `Contains(…, "streak-report.sh --apply")` passes for + // `echo streak-report.sh --apply`, which announces nothing — falsify caught + // exactly that, and it is the third time in this repository that a Contains + // has been satisfied by a longer string that does the opposite. + run := commandOf(block) + if !strings.HasPrefix(run, "tools/ci/streak-report.sh --apply") { + t.Fatalf("the announcing step runs %q, not the announcement", run) + } + // Writing on an issue needs the permission; without it the step fails, or + // worse, reports success having posted nothing. + streakJob := workflow[strings.Index(workflow, " streak:"):] + if cut := strings.Index(streakJob, "\n report:"); cut > 0 { + streakJob = streakJob[:cut] + } + if !strings.Contains(streakJob, "issues: write") { + t.Error("the streak job cannot write on an issue: `issues: write` is missing, and a " + + "comment it cannot post is a silence that reports success") + } + // The issue it announces on is this one; a typo here posts the verdict + // somewhere nobody is waiting for it. + if !strings.Contains(block, " 125") { + t.Errorf("the announcement does not name issue 125:\n%s", block) + } +} + +// commandOf answers a step's `run:` command, or "" when it declares none. +// +// The value, so a check on it cannot be satisfied by a longer line that merely +// contains it. +func commandOf(block string) string { + for _, line := range strings.Split(block, "\n") { + if after, found := strings.CutPrefix(strings.TrimSpace(line), "run:"); found { + return strings.TrimSpace(after) + } + } + return "" +} diff --git a/tools/falsify/specs/a-met-criterion-is-announced.json b/tools/falsify/specs/a-met-criterion-is-announced.json new file mode 100644 index 0000000..5ce619f --- /dev/null +++ b/tools/falsify/specs/a-met-criterion-is-announced.json @@ -0,0 +1,47 @@ +{ + "package": "./tools/ci/", + "mutations": [ + { + "label": "a met criterion goes back to a step summary alone, which is where it was said on 2026-09-21 and where nobody read it", + "file": ".github/workflows/runtime-proof.yml", + "find": " run: tools/ci/streak-report.sh --apply \"${{ steps.count.outputs.streak }}\" 14 125", + "replace": " run: echo tools/ci/streak-report.sh --apply \"${{ steps.count.outputs.streak }}\" 14 125", + "test": "TestTheStreakJobAnnouncesOnTheIssue" + }, + { + "label": "the job cannot write on an issue, so the announcement is a silence that reports success", + "file": ".github/workflows/runtime-proof.yml", + "find": " issues: write\n steps:\n - name: Harden the runner", + "replace": " # issues: write\n issues: read\n steps:\n - name: Harden the runner", + "test": "TestTheStreakJobAnnouncesOnTheIssue" + }, + { + "label": "the criterion is read as an equality, so a streak past the target falls through and goes silent", + "file": "tools/ci/streak-report.sh", + "find": "if [ \"$streak\" -lt \"$target\" ]; then", + "replace": "if [ \"$streak\" -lt \"$target\" ] || [ \"$streak\" -gt \"$target\" ]; then", + "test": "TestAStreakPastTheTargetStillAnnounces" + }, + { + "label": "the announcement repeats every night, which is how a notification becomes noise somebody mutes", + "file": "tools/ci/streak-report.sh", + "find": "if [ \"$already\" = true ]; then", + "replace": "if [ \"$already\" = maybe ]; then", + "test": "TestAnAnnouncedCriterionIsNotRepeated" + }, + { + "label": "a maintainer writing about the criterion silences the mechanism, because the words are searched rather than the marker", + "file": "tools/ci/streak-report.sh", + "find": " if printf '%s' \"${comments}\" | grep -qF \"${marker}\"; then", + "replace": " if printf '%s' \"${comments}\" | grep -qF \"criterion is met\"; then", + "test": "TestHumanProseDoesNotCountAsTheAnnouncement" + }, + { + "label": "it announces below the target too, so the comment arrives whatever happens and stops meaning anything", + "file": "tools/ci/streak-report.sh", + "find": " echo \"verdict: not met (${streak}/${target}), nothing to announce\"\n exit 0", + "replace": " echo \"verdict: not met (${streak}/${target}), nothing to announce\"\n if [ -z \"$streak\" ]; then exit 0; fi\n streak=\"$target\"", + "test": "TestAStreakBelowTheTargetAnnouncesNothing" + } + ] +}