From 19b2ce8d0dfcd4776d3c82bca4646329751366ef Mon Sep 17 00:00:00 2001 From: Andy Anderson Date: Mon, 2 Nov 2026 11:00:32 -0500 Subject: [PATCH] =?UTF-8?q?=E2=9C=A8=20evidence:=20add=20fail-closed=20mon?= =?UTF-8?q?thly=20outcomes=20publication?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Andy Anderson --- .github/scripts/integration-governance.cjs | 2 +- .github/scripts/monthly-outcomes-pr.sh | 48 ++++ .github/workflows/monthly-outcomes.yml | 75 ++++++ changelog.d/added-monthly-outcomes.md | 1 + src/docs/benchmarks/README.md | 17 +- src/docs/benchmarks/monthly-automation.md | 125 +++++++++ src/scripts/monthly-outcomes.py | 291 +++++++++++++++++++++ src/scripts/test-monthly-outcomes.py | 228 ++++++++++++++++ 8 files changed, 782 insertions(+), 5 deletions(-) create mode 100644 .github/scripts/monthly-outcomes-pr.sh create mode 100644 .github/workflows/monthly-outcomes.yml create mode 100644 changelog.d/added-monthly-outcomes.md create mode 100644 src/docs/benchmarks/monthly-automation.md create mode 100644 src/scripts/monthly-outcomes.py create mode 100644 src/scripts/test-monthly-outcomes.py diff --git a/.github/scripts/integration-governance.cjs b/.github/scripts/integration-governance.cjs index 488d35bd1a..cf323ff045 100644 --- a/.github/scripts/integration-governance.cjs +++ b/.github/scripts/integration-governance.cjs @@ -9,7 +9,7 @@ const MAX_COMMITS = 1000; const MAX_AUTHOR_BATCH = 50; // This literal is inside the enrolled .github closure. A caller-supplied // preparedBy.sourceSHA must never choose a new executable verifier tree. -const TRUSTED_SCRIPTS_TREE = '308d64dd1ac3bf1061a4dced3071647236327e51'; +const TRUSTED_SCRIPTS_TREE = '8949d85a54a8ac40fce2fc31f869acd67ed4e032'; const SOURCE = 'https://github.com/kubernetes-sigs/kubebuilder-release-tools/tree/012269a88fa4c034a0acf1ba84c26b195c0dbab4'; const sha = value => crypto.createHash('sha256').update(value).digest('hex'); const git = (args, cwd = process.cwd()) => diff --git a/.github/scripts/monthly-outcomes-pr.sh b/.github/scripts/monthly-outcomes-pr.sh new file mode 100644 index 0000000000..a21eafee8e --- /dev/null +++ b/.github/scripts/monthly-outcomes-pr.sh @@ -0,0 +1,48 @@ +#!/usr/bin/env bash +# Only sanitized renderer outputs enter this branch; never stage an input tree. +set -euo pipefail +unset GITHUB_TOKEN +: "${MONTH:?missing validated month}" +: "${GITHUB_REPOSITORY:?missing repository}" +[[ "$MONTH" =~ ^[0-9]{4}-[0-9]{2}$ ]] || exit 1 +branch="automation/monthly-outcomes-$MONTH" +pr="$(gh pr list --repo "$GITHUB_REPOSITORY" --head "$branch" --state all --limit 2 --json number,state)" +if [ "$(printf '%s' "$pr" | jq length)" -gt 1 ] || printf '%s' "$pr" | jq -e 'any(.[]; .state != "OPEN")' >/dev/null; then + echo "::error::A closed/merged monthly PR already exists; resolve it explicitly rather than reposting." + exit 1 +fi +if git ls-remote --exit-code --heads origin "$branch" >/dev/null; then + git fetch origin "$branch" + git checkout -b "$branch" FETCH_HEAD +else + git checkout -b "$branch" +fi +dest="src/docs/benchmarks" +# Keep the original renderer revision when the approved input is unchanged, +# even if unrelated v5 commits landed while this month's PR was open. +provenance="outcomes-$MONTH.provenance.json" +if [ -f "$dest/$provenance" ] && + [ "$(jq -er .input_sha256 "$dest/$provenance")" = "$(jq -er .input_sha256 ".monthly-outcomes/$provenance")" ]; then + echo "Approved input unchanged; preserving the existing monthly PR without a repost." + exit 0 +fi +# Replace only this month's outputs, including obsolete cards. +git ls-files -z "$dest/outcomes-$MONTH.*" "$dest/assets/outcomes-$MONTH-*" | xargs -0 -r git rm -q -- +cp ".monthly-outcomes/outcomes-$MONTH.json" "$dest/" +cp ".monthly-outcomes/outcomes-$MONTH.md" "$dest/" +cp ".monthly-outcomes/outcomes-$MONTH.provenance.json" "$dest/" +cp .monthly-outcomes/assets/"outcomes-$MONTH-"* "$dest/assets/" +git add -- "$dest/outcomes-$MONTH.json" "$dest/outcomes-$MONTH.md" "$dest/outcomes-$MONTH.provenance.json" "$dest/assets/outcomes-$MONTH-"* +if git diff --cached --quiet; then + echo "No monthly report diff; no commit, push or stale PR repost." + exit 0 +fi +git config user.name 'github-actions[bot]' +git config user.email '41898282+github-actions[bot]@users.noreply.github.com' +git commit -s -m "📖 docs: publish outcomes for $MONTH" +git push origin "$branch" +if [ "$(printf '%s' "$pr" | jq length)" -eq 0 ]; then + gh pr create --repo "$GITHUB_REPOSITORY" --base v5 --head "$branch" \ + --title "📖 docs: monthly outcomes $MONTH" \ + --body "Owner-supplied, sanitized monthly aggregates. Review the cohort, observation time, cost denominator and provenance before merging. No raw evidence, stories or deployment identities are published. Collection completeness and publication approval are owner attestations, not independently verified by this workflow. CI and normal review gates must pass; this workflow does not merge." +fi diff --git a/.github/workflows/monthly-outcomes.yml b/.github/workflows/monthly-outcomes.yml new file mode 100644 index 0000000000..ec0df81b40 --- /dev/null +++ b/.github/workflows/monthly-outcomes.yml @@ -0,0 +1,75 @@ +name: Monthly outcomes + +on: + schedule: + # Prior month, after every merge has the existing 14-day maturation period. + - cron: '17 6 15 * *' + workflow_dispatch: + inputs: + month: + description: 'Completed UTC month (YYYY-MM); empty selects prior month' + type: string + default: '' + pull_request: + paths: + - '.github/workflows/monthly-outcomes.yml' + - '.github/scripts/monthly-outcomes-pr.sh' + - 'src/scripts/monthly-outcomes.py' + - 'src/scripts/test-monthly-outcomes.py' + +permissions: + contents: read + +concurrency: + group: monthly-outcomes-${{ github.event_name == 'pull_request' && github.event.pull_request.number || 'publication' }} + cancel-in-progress: false + +jobs: + selftest: + runs-on: ubuntu-latest + timeout-minutes: 20 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 + with: + go-version-file: src/go.mod + cache: false + - run: python3 src/scripts/test-monthly-outcomes.py + + publish: + if: github.event_name != 'pull_request' && github.repository == 'hivecommons/hive' + needs: selftest + runs-on: ubuntu-latest + timeout-minutes: 20 + steps: + - name: Require supplied input and existing publication credential + env: + SUPPLIED: ${{ secrets.MONTHLY_OUTCOMES_INPUT }} + PUSH_TOKEN: ${{ secrets.TOPUP_PUSH_TOKEN }} + run: | + if [ -z "$SUPPLIED" ] || [ -z "$PUSH_TOKEN" ]; then + echo "::error::Monthly outcomes is not operational: an owner must supply MONTHLY_OUTCOMES_INPUT and the existing TOPUP_PUSH_TOKEN. No deployment collection is configured." + exit 1 + fi + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: v5 + fetch-depth: 0 + token: ${{ secrets.TOPUP_PUSH_TOKEN }} + persist-credentials: true + - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 + with: + go-version-file: src/go.mod + cache: false + - name: Validate, sanitize and render supplied aggregate + id: generate + env: + MONTHLY_OUTCOMES_INPUT: ${{ secrets.MONTHLY_OUTCOMES_INPUT }} + MONTH: ${{ inputs.month }} + run: python3 src/scripts/monthly-outcomes.py render --month "$MONTH" --out "$GITHUB_WORKSPACE/.monthly-outcomes" + - name: Open or update the month's reviewed PR + if: steps.generate.outputs.changed == 'true' + env: + GH_TOKEN: ${{ secrets.TOPUP_PUSH_TOKEN }} + MONTH: ${{ steps.generate.outputs.month }} + run: bash .github/scripts/monthly-outcomes-pr.sh diff --git a/changelog.d/added-monthly-outcomes.md b/changelog.d/added-monthly-outcomes.md new file mode 100644 index 0000000000..1b293b1c0e --- /dev/null +++ b/changelog.d/added-monthly-outcomes.md @@ -0,0 +1 @@ +- Add a fail-closed monthly outcomes publication wrapper and reviewed-PR cadence for owner-supplied, sanitized evidence aggregates; deployment source access and monthly input remain explicit operator prerequisites. diff --git a/src/docs/benchmarks/README.md b/src/docs/benchmarks/README.md index bd27483705..5562d27f0e 100644 --- a/src/docs/benchmarks/README.md +++ b/src/docs/benchmarks/README.md @@ -38,16 +38,25 @@ Each report's "What the README shows and leaves out" section records, per card, ## Regenerate +For the fail-closed monthly schedule, owner-supplied input contract and reviewed +PR process, see [monthly automation](monthly-automation.md). Hosted runners +do not have the spoke's evidence; scheduling alone does not collect it. + ```sh # on a spoke, or anywhere with a copy of its evidence tree src/scripts/outcome-benchmark.sh --root /data/evidence --month 2026-10 \ - --spend cost-2026-10.json # saved GET /api/cost, for cost per merged PR - --stories stories-2026-10.json # curated caught → fixed → merged examples - --red-team red-team-2026-10.json # prompt-injection evaluation counts - --hub hub.json # saved GET /api/hub/evidence (admin session), optional + --now 2026-10-27T00:00:00Z \ + --spend docs/benchmarks/cost-2026-10.json \ + --stories docs/benchmarks/stories-2026-10.json \ + --red-team docs/benchmarks/red-team-2026-10.json \ --source "hive-oke spoke snapshot 2026-10-27" ``` +The example preserves the historical observation time, not a completed-month +snapshot. `--hub` is optional and requires a separately saved, authorized +`GET /api/hub/evidence` response. Saved input paths above are relative to `src/`, +where the wrapper runs; `--root` must point to the actual private evidence copy. + The script wraps `hivectl evidence benchmark`; see [hivectl.md](../hivectl.md#evidence--benchmark) for every flag. Redraw the committed report and legacy cards without the evidence tree using `src/scripts/outcome-benchmark.sh --rerender`. The wrapper rejects `--readme` with instructions for the separate README renderer. The underlying CLI still supports marked legacy documents, but the root README deliberately has no legacy outcomes markers. To take a read-only copy of a spoke's evidence: `kubectl exec deploy/hive -c hive -- tar czf - -C /data evidence | tar xzf -`. ## Metrics diff --git a/src/docs/benchmarks/monthly-automation.md b/src/docs/benchmarks/monthly-automation.md new file mode 100644 index 0000000000..fff6fe9f93 --- /dev/null +++ b/src/docs/benchmarks/monthly-automation.md @@ -0,0 +1,125 @@ +# Monthly outcomes publication + +## Status and source access + +This is a **supplied-input publication cadence, not a deployed data collector**. +The existing `outcome-benchmark.sh` runs `hivectl evidence benchmark` offline. It +reads schema-v1 files under the spoke's `/data/evidence`, optionally a saved +`GET /api/cost`, curated stories, red-team counts and saved hub evidence. It +does not call a dashboard or discover/authenticate to a spoke. Hub evidence +requires an admin session; the ordinary cost response also requires authorized +dashboard access. A hosted runner has neither that filesystem nor an assumed +dashboard session. No deployment credential, new service or private artifact +download is introduced here. + +An evidence owner must produce and approve a complete monthly aggregate in the +source environment. Until that owner supplies the input below and the existing +publication credential, the scheduled/manual workflow **fails visibly and +publishes nothing**. Its presence does not establish operational collection. +If source collection is later automated, its timer belongs with the deployment +that owns the evidence, not on a hosted runner with expanded credentials. + +## Cadence and cohort + +`.github/workflows/monthly-outcomes.yml` runs at **06:17 UTC on the 15th** and +supports manual dispatch with an optional `month: YYYY-MM`. Empty selects the +prior completed UTC month, including December at January rollover. GitHub may +delay scheduled runs. The merge cohort must be exactly `[month start, next +month start)`, and its snapshot must observe at least 14 days after month end, +matching the existing maturation rule. A manual run before that point fails. +Past months can be dispatched explicitly; current/future months cannot. + +The existing fold, rates, small-n suppression, Wilson intervals, lower medians, +coverage caveats and trend history are retained. The wrapper validates counts +and rate intervals; it does not invent confidence intervals for medians or +estimated spend. Cost keeps its **own** denominator and window, even if those +differ from the monthly merge cohort. The committed October snapshot remains +historical: approximately $6124.75 / 2869 hive-attributed merges from +2026-08-08 through 2026-10-27, an upper-bound/order-of-magnitude estimate, not +today's universal cost. It is not a completed-month automation input: it was +observed before October ended and contains 10 unreadable bundles. + +## Owner-supplied input + +1. In the evidence-owning environment, take an immutable, complete snapshot + using existing authorized access. Check collection completion independently: + zero unreadable files alone cannot prove a snapshot is complete. Retain the + raw snapshot privately and calculate its SHA-256. Do not copy it into this + public repository or upload it as an Actions artifact. +2. Record the exact generator Git revision. Use the existing generator against + that snapshot with explicit `--month`, `--now` (snapshot observation time) + and a private output directory. Optional cost input must be its matching + saved response, not an extrapolation from October's figures. Check any + `count_error`/partial or stale-count indicators in that saved response before + approving it: the computed Report cannot reconstruct those source errors. + Omit `Spend` entirely when its source is incomplete or unverifiable. Omit optional + hub, story and evaluation inputs; publication strips them regardless. +3. Wrap the computed `outcomes-YYYY-MM.json` in this envelope, in a private local + file. `report` is the complete existing `outcomebench.Report` object, not raw + bundles. The two attestations are the evidence owner's responsibility. + + ```json + { + "schema": 1, + "month": "YYYY-MM", + "collection_complete": true, + "publication_approved": true, + "snapshot_sha256": "<64 lowercase hex characters>", + "generator_revision": "<40 lowercase hex characters>", + "report": {} + } + ``` + +4. Sanitize and validate locally, then inspect the prepared file: + + ```sh + python3 src/scripts/monthly-outcomes.py prepare --month YYYY-MM \ + --input PRIVATE-ENVELOPE.json --out PREPARED-AGGREGATE.json + ``` + + The wrapper projects onto known aggregate fields. It replaces repository + identities with `repository-001` labels, replaces source prose with the + snapshot digest, allows only `agent/bot/human/unknown` author categories, + drops unknown report fields, and omits stories, issue/review/log text, + URLs, hub context and evaluation metadata. Dates and fixed cohort/category + names are the only surviving input strings. It never logs input or renderer + stderr. Aggregate counts can still be sensitive: explicit owner approval + and human PR review are required, not just string scrubbing. +5. A repository secret administrator sets `MONTHLY_OUTCOMES_INPUT` to that + **prepared** JSON (maximum 48000 UTF-8 bytes), using the existing GitHub + secret management process. Do not put JSON in dispatch fields, issue bodies + or logs. Replace the secret with a newly approved snapshot each month. A + missing/stale month, premature observation, unreadable/empty cohort, + invalid metric or unapproved/incomplete attestation fails closed. + The workflow also requires the **existing** `TOPUP_PUSH_TOKEN` publication + credential. Do not expand its scope or mint new privileges for this task; + if unavailable, publication remains blocked. It uses no write grant on + `GITHUB_TOKEN` and does not fall back to that token (which may suppress PR CI). + +For manual offline rendering, run `monthly-outcomes.py render` with the same +input/month and an **empty** `--out` directory. It calls the existing Go +`--from-json` renderer, emitting its usual report/JSON/SVG/PNG assets plus +`outcomes-YYYY-MM.provenance.json`. Provenance records the approved snapshot +digest, input generator revision, actual renderer revision, observation time +and sanitized input digest. Completeness is an attestation; hashes enable +reproduction by authorized owners, not independent public verification. + +## Review, reruns and failure handling + +Only this month's rendered outputs and provenance are staged. Raw/supplied +input is never staged or uploaded. Root README graphics and prior reports are +not refreshed. Identical already-merged input skips rendering; identical input +on the open monthly branch makes no commit/push/repost. A serialized +`automation/monthly-outcomes-YYYY-MM` branch has one PR against `v5`, updated +with ordinary signed-off commits (no force push). Obsolete cards for that +month are removed. A closed/declined or already-merged PR is not automatically +reopened/reposted; a human must resolve any proposed correction. + +The workflow never merges. Normal CI and authorized review gates apply, +including privacy, provenance and representativeness review. Input does not +prove that mediated and unmediated PRs are causally comparable. Required +source data/access and credential provisioning remain operator gates. +CI's focused fixtures exercise UTC selection, malformed/partial/stale input, +privacy projection, historical cost preservation, deterministic real rendering +and no-op/error handling, including local-Git/Mock-GitHub one-PR reruns and +closed-PR refusal; no fixture makes a live deployment/API request. diff --git a/src/scripts/monthly-outcomes.py b/src/scripts/monthly-outcomes.py new file mode 100644 index 0000000000..2a54c16e90 --- /dev/null +++ b/src/scripts/monthly-outcomes.py @@ -0,0 +1,291 @@ +#!/usr/bin/env python3 +"""Prepare and render owner-supplied monthly outcome aggregates; never collect data.""" + +import argparse +import copy +import datetime as dt +import hashlib +import json +import math +import os +from pathlib import Path +import re +import subprocess +import sys + +UTC = dt.timezone.utc +MATURATION_DAYS = 14 +MIN_N = 10 +MAX_INPUT_BYTES = 48_000 +ZERO_TIME = "0001-01-01T00:00:00Z" +ROOT = Path(__file__).resolve().parents[2] +REPORT_DIR = Path("src/docs/benchmarks") + + +class InputError(ValueError): + """Only static, non-sensitive validation messages belong here.""" + + +def require(condition, message): + if not condition: + raise InputError(message) + + +def timestamp(value): + require(isinstance(value, str), "timestamp must be RFC3339 UTC") + require(re.fullmatch(r"\d{4}-\d\d-\d\dT\d\d:\d\d:\d\d(?:\.\d+)?Z", value), + "timestamp must be RFC3339 UTC") + return dt.datetime.fromisoformat(value.replace("Z", "+00:00")) + + +def window(month): + require(isinstance(month, str) and re.fullmatch(r"\d{4}-\d\d", month), + "month must be YYYY-MM") + start = dt.datetime.strptime(month, "%Y-%m").replace(tzinfo=UTC) + end = (start.replace(day=28) + dt.timedelta(days=4)).replace(day=1) + return start, end + + +def select_month(month, now): + current = now.astimezone(UTC).replace(day=1, hour=0, minute=0, second=0, microsecond=0) + month = month or (current - dt.timedelta(days=1)).strftime("%Y-%m") + start, end = window(month) + require(end <= current, "requested month is not completed in UTC") + require(now >= end + dt.timedelta(days=MATURATION_DAYS), + "wait until 14 days after month end for outcome maturation") + return month, start, end + + +def count(value): + require(type(value) is int and value >= 0, "counts must be nonnegative integers") + return value + + +def number(value): + require(type(value) in (int, float) and math.isfinite(value) and value >= 0, + "metrics must be finite nonnegative numbers") + return value + + +def rate(value): + require(set(value) == {"K", "N", "Rate", "Lo", "Hi", "Reported"}, "invalid rate fields") + k, n = count(value["K"]), count(value["N"]) + require(k <= n, "rate numerator exceeds denominator") + require(type(value["Reported"]) is bool and value["Reported"] == (n >= MIN_N), + "invalid small-n reporting") + expected = [0, 0, 0] + if n >= MIN_N: + p, z = k / n, 1.96 + divisor = 1 + z * z / n + centre = (p + z * z / (2 * n)) / divisor + half = z * math.sqrt(p * (1 - p) / n + z * z / (4 * n * n)) / divisor + expected = [p, max(0, centre - half), min(1, centre + half)] + for key, wanted in zip(("Rate", "Lo", "Hi"), expected): + require(math.isclose(number(value[key]), wanted, abs_tol=1e-12), + "rate or Wilson interval does not match its counts") + return copy.deepcopy(value) + + +def median(value): + require(set(value) == {"Value", "N", "Reported"}, "invalid median fields") + n = count(value["N"]) + number(value["Value"]) + require(type(value["Reported"]) is bool and value["Reported"] == (n >= MIN_N), + "invalid median reporting") + require(n >= MIN_N or value["Value"] == 0, "small-n median must be suppressed") + return copy.deepcopy(value) + + +def metrics(value, counts, rates=(), medians=()): + require(set(value) == set(counts) | set(rates) | set(medians), "invalid metric fields") + return {**{k: count(value[k]) for k in counts}, + **{k: rate(value[k]) for k in rates}, + **{k: median(value[k]) for k in medians}} + + +def cohort(value, name): + require(value["Name"] == name, "cohort name mismatch") + kinds = value["ByAuthorKind"] + require(set(kinds) <= {"agent", "bot", "human", "unknown"}, "unknown author category") + clean = metrics({k: v for k, v in value.items() if k not in ("Name", "ByAuthorKind")}, + ("Merged", "Matured"), + ("RevertRate", "ChangeFailure", "CIFailureAtHead", "CICoverage", + "ChangesRequested"), ("TimeToMerge",)) + require(clean["Matured"] == clean["Merged"], + "completed monthly cohort must be fully matured at this observation time") + for key in ("RevertRate", "ChangeFailure"): + require(clean[key]["N"] == clean["Matured"], "matured denominator mismatch") + require(clean["CICoverage"]["N"] == clean["Merged"], "CI coverage denominator mismatch") + require(clean["CIFailureAtHead"]["N"] == clean["CICoverage"]["K"], + "CI failure denominator mismatch") + require(clean["TimeToMerge"]["N"] <= clean["Merged"], "time-to-merge denominator mismatch") + require(clean["ChangesRequested"]["N"] == clean["Merged"], + "review denominator mismatch") + require(name != "unmediated" or clean["ChangesRequested"]["K"] == 0, + "unmediated cohort cannot have recorded review objections") + clean.update(Name=name, ByAuthorKind={k: count(v) for k, v in kinds.items()}) + require(sum(clean["ByAuthorKind"].values()) == clean["Merged"], "author counts mismatch") + return clean + + +def sanitize(report, start, end, now, source_hash): + """Project onto aggregate fields: no raw prose, repository identities or links.""" + observed = timestamp(report["Now"]) + require(timestamp(report["Since"]) == start and timestamp(report["Until"]) == end, + "report window differs from requested completed month") + require(end + dt.timedelta(days=MATURATION_DAYS) <= observed <= now, + "snapshot is premature or from the future") + require(timestamp(report["GeneratedAt"]) == observed, "generation/observation mismatch") + clean = {k: report[k] for k in ("GeneratedAt", "Since", "Until", "Now")} + clean.update({k: count(report[k]) for k in ("Bundles", "Unreadable", "PRs", "Verdicts")}) + require(clean["Unreadable"] == 0, "partial collection: unreadable bundles") + require(clean["Bundles"] >= clean["PRs"] > 0, "empty or inconsistent evidence") + for name in ("mediated", "unmediated"): + clean[name.capitalize()] = cohort(report[name.capitalize()], name) + merged = clean["Mediated"]["Merged"] + clean["Unmediated"]["Merged"] + require(0 < merged <= clean["PRs"], "empty or inconsistent merged cohort") + repos = report["Repos"] + require(isinstance(repos, list) and len({r["Repo"] for r in repos}) == len(repos), + "duplicate repositories") + # Labels are deliberately independent of the private repository names. + clean["Repos"] = [{"Repo": f"repository-{i + 1:03d}", "Merged": count(r["Merged"])} + for i, r in enumerate(repos)] + require(sum(r["Merged"] for r in clean["Repos"]) == merged, "repository counts mismatch") + clean["Source"] = f"owner-supplied aggregate; evidence snapshot SHA-256 {source_hash}" + first = report["FirstMediatedMerge"] + require(timestamp(first) <= observed or first == ZERO_TIME, "invalid first mediated merge") + clean["FirstMediatedMerge"] = first + months = report["Months"] + require(isinstance(months, list) and months == sorted(set(months)), "invalid trend months") + for month in months: + require(window(month)[0] <= observed, "future trend month") + clean["Months"] = months + clean["Trend"] = {} + for name in ("mediated", "unmediated"): + points = report["Trend"][name] + require(len(points) == len(months), "partial trend") + clean["Trend"][name] = [] + for month, point in zip(months, points): + require(point["Month"] == month, "trend month mismatch") + fields = metrics({k: v for k, v in point.items() if k != "Month"}, + ("Merged",), ("RevertRate",), ("TimeToMerge",)) + require(fields["RevertRate"]["N"] <= fields["Merged"] and + fields["TimeToMerge"]["N"] <= fields["Merged"], "invalid trend denominator") + if window(month)[1] <= end: + require(fields["RevertRate"]["N"] == fields["Merged"], + "completed trend cohort must be fully matured") + clean["Trend"][name].append(dict(Month=month, **fields)) + clean["Followup"] = metrics( + report["Followup"], + ("SentBack", "NewHead", "MergedSameHead", "MergedSameHeadApproved", "Unresolved", + "CIPairs", "CIFixed"), ("FixedAndApproved", "CICoverage"), ("TimeToApproval",)) + followup = clean["Followup"] + require(followup["FixedAndApproved"]["N"] == followup["SentBack"] and + followup["FixedAndApproved"]["K"] <= followup["NewHead"] <= followup["SentBack"], + "follow-up denominator mismatch") + require(followup["CICoverage"]["N"] == followup["NewHead"] and + followup["CICoverage"]["K"] == followup["CIPairs"] and + followup["CIFixed"] <= followup["CIPairs"] and + followup["MergedSameHeadApproved"] <= followup["MergedSameHead"] <= followup["SentBack"] and + followup["TimeToApproval"]["N"] == followup["FixedAndApproved"]["K"], + "follow-up counts mismatch") + clean["Spend"] = None + if report.get("Spend") is not None: + spend = report["Spend"] + require(set(spend) == {"usd", "merged_prs", "window_start", "window_end", "basis"}, + "invalid spend fields (no cost confidence intervals)") + require(timestamp(spend["window_start"]) < timestamp(spend["window_end"]) <= observed, + "invalid spend observation window") + clean["Spend"] = dict(usd=number(spend["usd"]), merged_prs=count(spend["merged_prs"]), + window_start=spend["window_start"], window_end=spend["window_end"], + basis="hive-attributed") + require(spend["basis"] == "hive-attributed", "unsupported cost denominator basis") + clean["Hub"] = None + clean["ReadmeMinBase"] = count(report.get("ReadmeMinBase", 0)) + require(clean["ReadmeMinBase"] in (0, 200), "monthly publication keeps existing card threshold") + return clean + + +def digest(value): + return hashlib.sha256(json.dumps(value, sort_keys=True, separators=(",", ":")).encode()).hexdigest() + + +def validate(payload, month, now): + month, start, end = select_month(month, now) + require(set(payload) == {"schema", "month", "collection_complete", "publication_approved", + "snapshot_sha256", "generator_revision", "report"}, "invalid input envelope") + require(payload["schema"] == 1 and payload["month"] == month, "stale or unsupported input") + require(payload["collection_complete"] is True, "owner has not attested complete collection") + require(payload["publication_approved"] is True, "owner has not approved aggregate publication") + require(re.fullmatch(r"[a-f0-9]{64}", payload["snapshot_sha256"]), "invalid snapshot digest") + require(re.fullmatch(r"[a-f0-9]{40}", payload["generator_revision"]), "invalid generator revision") + clean = copy.deepcopy(payload) + clean["report"] = sanitize(payload["report"], start, end, now, payload["snapshot_sha256"]) + return clean + + +def read_input(path): + data = Path(path).read_bytes() if path else os.environ.get("MONTHLY_OUTCOMES_INPUT", "").encode() + require(data, "missing owner-supplied input: set MONTHLY_OUTCOMES_INPUT; no source access is configured") + require(len(data) <= MAX_INPUT_BYTES, "aggregate exceeds 48000-byte secret limit") + return json.loads(data) + + +def render(payload, directory, revision): + month = payload["month"] + provenance = { + "schema": 1, "month": month, "input_sha256": digest(payload), + "snapshot_sha256": payload["snapshot_sha256"], + "generator_revision": payload["generator_revision"], + "renderer_revision": revision, + "observed_at": payload["report"]["Now"], + "collection_complete": True, "publication_approved": True, + "privacy": "aggregates only; repository labels anonymized; no stories, hub, eval or raw evidence", + } + committed = ROOT / REPORT_DIR / f"outcomes-{month}.provenance.json" + if committed.exists() and json.loads(committed.read_text())["input_sha256"] == digest(payload): + return False + directory.mkdir(parents=True, exist_ok=True) + require(not any(directory.iterdir()), "render staging directory must be empty") + source = directory / "report-input.json" + source.write_text(json.dumps(payload["report"]) + "\n") + subprocess.run(["go", "run", "./cmd/hivectl", "evidence", "benchmark", + "--from-json", str(source.resolve()), "--out-dir", str(directory.resolve())], + cwd=ROOT / "src", check=True, stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL) + source.unlink() + (directory / f"outcomes-{month}.provenance.json").write_text( + json.dumps(provenance, indent=2) + "\n") + return True + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("command", choices=("prepare", "render")) + parser.add_argument("--input", help="envelope JSON (otherwise MONTHLY_OUTCOMES_INPUT)") + parser.add_argument("--month", default="", help="completed YYYY-MM; default prior UTC month") + parser.add_argument("--out", required=True, help="prepared envelope file or render staging directory") + args = parser.parse_args() + payload = validate(read_input(args.input), args.month, dt.datetime.now(UTC)) + if args.command == "prepare": + Path(args.out).write_text(json.dumps(payload, indent=2) + "\n") + else: + revision = subprocess.check_output(["git", "rev-parse", "HEAD"], cwd=ROOT, text=True).strip() + changed = render(payload, Path(args.out), revision) + if os.environ.get("GITHUB_OUTPUT"): + with open(os.environ["GITHUB_OUTPUT"], "a") as output: + output.write(f"month={payload['month']}\nchanged={str(changed).lower()}\n") + print(f"monthly outcomes {payload['month']}: {'generated' if changed else 'unchanged'}") + + +if __name__ == "__main__": + try: + main() + except InputError as error: + print(f"monthly outcomes failed: {error}", file=sys.stderr) + sys.exit(1) + except (ValueError, KeyError, TypeError, OSError, subprocess.SubprocessError): + # Source JSON and generator stderr can contain private paths/prose. + print("monthly outcomes failed: missing, invalid, premature or incomplete supplied input, " + "or renderer failure; see the aggregate input contract (input is never logged)", file=sys.stderr) + sys.exit(1) diff --git a/src/scripts/test-monthly-outcomes.py b/src/scripts/test-monthly-outcomes.py new file mode 100644 index 0000000000..fa7e284103 --- /dev/null +++ b/src/scripts/test-monthly-outcomes.py @@ -0,0 +1,228 @@ +#!/usr/bin/env python3 +"""Focused CI fixtures; never collect deployment evidence or contact GitHub.""" + +import datetime as dt +import importlib.util +import json +import math +import os +from pathlib import Path +import subprocess +import tempfile +import unittest +from unittest import mock + +SCRIPT = Path(__file__).with_name("monthly-outcomes.py") +SPEC = importlib.util.spec_from_file_location("monthly_outcomes", SCRIPT) +monthly = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(monthly) +NOW = dt.datetime(2026, 11, 15, 6, 17, tzinfo=monthly.UTC) + + +def fixture(): + # Historical counts are test data, not a new October observation. + report = json.loads((monthly.ROOT / monthly.REPORT_DIR / "outcomes-2026-10.json").read_text()) + report.update(Now="2026-11-15T00:00:00Z", GeneratedAt="2026-11-15T00:00:00Z", Unreadable=0) + for name in ("mediated", "unmediated"): + cohort = report[name.capitalize()] + cohort["Matured"] = cohort["Merged"] + for key in ("RevertRate", "ChangeFailure"): + rate = cohort[key] + k, n = rate["K"], cohort["Merged"] + p, z = k / n, 1.96 + divisor = 1 + z * z / n + centre = (p + z * z / (2 * n)) / divisor + half = z * math.sqrt(p * (1 - p) / n + z * z / (4 * n * n)) / divisor + rate.update(N=n, Rate=p, Lo=max(0, centre - half), Hi=min(1, centre + half), Reported=True) + report["Trend"][name][-1]["RevertRate"] = dict(cohort["RevertRate"]) + return dict(schema=1, month="2026-10", collection_complete=True, publication_approved=True, + snapshot_sha256="a" * 64, generator_revision="b" * 40, report=report) + + +class MonthlyOutcomes(unittest.TestCase): + def test_utc_boundary_rollover_and_maturation(self): + zone = dt.timezone(dt.timedelta(hours=-8)) + month, start, end = monthly.select_month( + "", dt.datetime(2027, 1, 15, 0, 0, tzinfo=zone)) + self.assertEqual(month, "2026-12") + self.assertEqual(start.isoformat(), "2026-12-01T00:00:00+00:00") + self.assertEqual(end.isoformat(), "2027-01-01T00:00:00+00:00") + for month, instant in [ + ("2026-11", NOW), ("2026-13", NOW), ("2026-1", NOW), + ("", dt.datetime(2027, 1, 1, tzinfo=monthly.UTC)), + ("2026-10", dt.datetime(2026, 11, 14, 23, 59, tzinfo=monthly.UTC))]: + with self.subTest(month=month, instant=instant), self.assertRaises(ValueError): + monthly.select_month(month, instant) + self.assertEqual(monthly.select_month("2026-02", NOW)[2].day, 1) + + def test_projection_preserves_cost_and_removes_private_fields(self): + payload = fixture() + payload["report"]["Source"] = "PRIVATE-NAME and credential" + payload["report"]["Stories"] = [{"Caught": "PRIVATE-BODY"}] + payload["report"]["Hub"] = {"private": "PRIVATE-HUB"} + payload["report"]["RedTeam"] = {"source": "PRIVATE-EVAL"} + payload["report"]["Repos"][0]["Repo"] = "PRIVATE-REPO" + payload["report"]["Unexpected"] = "PRIVATE-UNKNOWN" + clean = monthly.validate(payload, "", NOW) + text = json.dumps(clean) + self.assertNotIn("PRIVATE", text) + self.assertEqual(clean["report"]["Spend"], payload["report"]["Spend"]) + self.assertEqual(clean["report"]["Spend"]["merged_prs"], 2869) + self.assertEqual(clean["report"]["Spend"]["usd"], 6124.75144225) + self.assertEqual(clean["report"]["Spend"]["window_start"], "2026-08-08T00:00:00Z") + self.assertEqual(monthly.validate(clean, "", NOW), clean) + self.assertEqual(clean["report"]["Mediated"], payload["report"]["Mediated"]) + + def test_empty_partial_stale_unapproved_and_invalid_data_fail(self): + mutations = [ + lambda p: p.update(month="2026-09"), + lambda p: p.update(collection_complete=False), + lambda p: p.update(publication_approved=False), + lambda p: p.update(generator_revision="not-a-revision"), + lambda p: p["report"].update(Unreadable=1), + lambda p: p["report"].update(Bundles=0), + lambda p: p["report"].update(Until="2026-10-27T00:00:00Z"), + lambda p: p["report"].update(Now="2026-11-01T00:00:00Z"), + lambda p: p["report"].update(GeneratedAt="2026-11-16T00:00:00Z"), + lambda p: p["report"]["Mediated"].update(Matured=99999), + lambda p: p["report"]["Mediated"].update(Matured=43), + lambda p: p["report"]["Mediated"]["RevertRate"].update(Hi=0.95), + lambda p: p["report"]["Mediated"]["ByAuthorKind"].update(private_user=1), + lambda p: p["report"]["Trend"]["mediated"].pop(), + lambda p: p["report"]["Spend"].update(usd=float("nan")), + lambda p: p["report"]["Spend"].update(confidence_interval=[0, 1]), + lambda p: p["report"]["Followup"].update(CIPairs=999999), + ] + for index, mutate in enumerate(mutations): + payload = fixture() + mutate(payload) + with self.subTest(index=index), self.assertRaises((ValueError, KeyError)): + monthly.validate(payload, "", NOW) + + def test_missing_secret_fails_without_logging_input(self): + with mock.patch.dict(os.environ, {"MONTHLY_OUTCOMES_INPUT": ""}): + with self.assertRaisesRegex(ValueError, "missing owner-supplied input"): + monthly.read_input(None) + with tempfile.TemporaryDirectory(dir=monthly.ROOT) as directory: + path = Path(directory) / "invalid.json" + path.write_text('{"PRIVATE-SOURCE":') + result = subprocess.run( + ["python3", str(SCRIPT), "prepare", "--input", str(path), + "--out", str(Path(directory) / "output.json")], + text=True, capture_output=True, check=False) + self.assertNotEqual(result.returncode, 0) + self.assertNotIn("PRIVATE-SOURCE", result.stderr + result.stdout) + self.assertFalse((Path(directory) / "output.json").exists()) + + def test_idempotent_committed_input_skips_renderer(self): + payload = monthly.validate(fixture(), "", NOW) + with tempfile.TemporaryDirectory(dir=monthly.ROOT) as directory: + root = Path(directory) + provenance = root / monthly.REPORT_DIR / "outcomes-2026-10.provenance.json" + provenance.parent.mkdir(parents=True) + provenance.write_text(json.dumps({"input_sha256": monthly.digest(payload)})) + with mock.patch.object(monthly, "ROOT", root), mock.patch.object(monthly.subprocess, "run") as run: + self.assertFalse(monthly.render(payload, root / "stage", "c" * 40)) + run.assert_not_called() + self.assertFalse((root / "stage").exists()) + + def test_renderer_error_propagates(self): + with tempfile.TemporaryDirectory(dir=monthly.ROOT) as directory: + with mock.patch.object(monthly.subprocess, "run", side_effect=subprocess.CalledProcessError(1, "go")): + with self.assertRaises(subprocess.CalledProcessError): + monthly.render(monthly.validate(fixture(), "", NOW), Path(directory), "c" * 40) + + def test_real_renderer_retains_methodology_and_reproducibility(self): + payload = monthly.validate(fixture(), "", NOW) + with tempfile.TemporaryDirectory(dir=monthly.ROOT) as first, \ + tempfile.TemporaryDirectory(dir=monthly.ROOT) as second: + for directory in (first, second): + self.assertTrue(monthly.render(payload, Path(directory), "c" * 40)) + for path in Path(first).rglob("*"): + if path.is_file(): + self.assertEqual(path.read_bytes(), (Path(second) / path.relative_to(first)).read_bytes()) + text = (Path(first) / "outcomes-2026-10.md").read_text() + for expected in ("95% Wilson", "upper-bound order of magnitude", "n=2869 merged", + "2026-08-08 → 2026-10-27", "not a controlled comparison", + "Window: merged 2026-10-01 → 2026-11-01"): + self.assertIn(expected.lower(), text.lower()) + report = json.loads((Path(first) / "outcomes-2026-10.json").read_text()) + self.assertEqual(report["Spend"], payload["report"]["Spend"]) + self.assertFalse(report.get("Stories")) + self.assertFalse(report.get("Hub")) + self.assertFalse(report.get("RedTeam")) + provenance = json.loads((Path(first) / "outcomes-2026-10.provenance.json").read_text()) + self.assertEqual(provenance["input_sha256"], monthly.digest(payload)) + self.assertEqual(provenance["generator_revision"], "b" * 40) + self.assertEqual(provenance["renderer_revision"], "c" * 40) + + def test_publication_one_pr_rerun_no_diff_and_closed_pr(self): + publisher = monthly.ROOT / ".github/scripts/monthly-outcomes-pr.sh" + with tempfile.TemporaryDirectory(dir=monthly.ROOT) as directory: + root = Path(directory) + origin = root / "origin.git" + seed = root / "seed" + calls = root / "gh-calls" + binary = root / "bin" + binary.mkdir() + gh = binary / "gh" + gh.write_text( + "#!/usr/bin/env bash\nset -eu\n" + "if [ \"$2\" = list ]; then printf '%s' \"$MOCK_PRS\"; " + "else echo create >> \"$MOCK_CALLS\"; fi\n") + gh.chmod(0o755) + env = dict(os.environ, PATH=f"{binary}:{os.environ['PATH']}", + GH_TOKEN="not-a-real-credential", GITHUB_REPOSITORY="fixture/repo", + MONTH="2026-10", MOCK_PRS="[]", MOCK_CALLS=str(calls)) + + def git(*args, cwd=root): + return subprocess.check_output(["git", *args], cwd=cwd, text=True, + stderr=subprocess.DEVNULL).strip() + + git("init", "--bare", str(origin)) + git("init", "-b", "v5", str(seed)) + git("config", "user.name", "Fixture", cwd=seed) + git("config", "user.email", "fixture@example.invalid", cwd=seed) + dest = seed / monthly.REPORT_DIR + (dest / "assets").mkdir(parents=True) + (dest / "assets/keep.svg").write_text("unrelated") + (dest / "assets/outcomes-2026-10-card-obsolete.svg").write_text("stale") + (dest / "outcomes-2026-09.json").write_text("unrelated") + git("add", ".", cwd=seed) + git("commit", "-s", "-m", "fixture", cwd=seed) + git("remote", "add", "origin", str(origin), cwd=seed) + git("push", "origin", "v5", cwd=seed) + heads = [] + for attempt, prs in enumerate(("[]", '[{"number":1,"state":"OPEN"}]', + '[{"number":1,"state":"CLOSED"}]')): + checkout = root / f"checkout-{attempt}" + git("clone", "--branch", "v5", str(origin), str(checkout)) + stage = checkout / ".monthly-outcomes" + (stage / "assets").mkdir(parents=True) + for suffix in ("json", "md"): + (stage / f"outcomes-2026-10.{suffix}").write_text("same approved aggregate") + (stage / "outcomes-2026-10.provenance.json").write_text( + json.dumps({"input_sha256": "a" * 64, "renderer_revision": str(attempt)})) + (stage / "assets/outcomes-2026-10-bars.svg").write_text("same chart") + env["MOCK_PRS"] = prs + result = subprocess.run(["bash", str(publisher)], cwd=checkout, env=env, + text=True, capture_output=True, check=False) + if attempt < 2: + self.assertEqual(result.returncode, 0, result.stderr + result.stdout) + heads.append(git("rev-parse", "HEAD", cwd=checkout)) + self.assertEqual((checkout / monthly.REPORT_DIR / "outcomes-2026-09.json").read_text(), + "unrelated") + self.assertEqual((checkout / monthly.REPORT_DIR / "assets/keep.svg").read_text(), + "unrelated") + self.assertFalse((checkout / monthly.REPORT_DIR / + "assets/outcomes-2026-10-card-obsolete.svg").exists()) + self.assertIn("Signed-off-by:", git("log", "-1", "--format=%B", cwd=checkout)) + else: + self.assertNotEqual(result.returncode, 0) + self.assertIn("already exists", result.stdout) + self.assertEqual(heads[0], heads[1], "identical input must not create another commit") + self.assertEqual(calls.read_text().splitlines(), ["create"]) + + +if __name__ == "__main__": + unittest.main()