From 31b062449dd278dc0e8485835725d9edb8532b80 Mon Sep 17 00:00:00 2001 From: Milenko <39630670+MrMilenko@users.noreply.github.com> Date: Thu, 12 Nov 2026 12:57:31 -0500 Subject: [PATCH] Added a housekeeping report --- .github/workflows/ci.yml | 20 ++++ .github/workflows/housekeeping.yml | 100 ++++++++++++++++++ README.md | 39 +++++++ housekeeping/housekeeping.py | 162 +++++++++++++++++++++++++++++ tests/test_housekeeping.py | 99 ++++++++++++++++++ 5 files changed, 420 insertions(+) create mode 100644 .github/workflows/housekeeping.yml create mode 100755 housekeeping/housekeeping.py create mode 100644 tests/test_housekeeping.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index cc6c176..2a673ac 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -47,3 +47,23 @@ jobs: - name: pytest run: python -m pytest -q tests + + # The reusable workflows, run on this repository from this commit: the + # labels as a dry run, and a housekeeping report without a webhook. + labels-dry-run: + permissions: + contents: read + issues: write + uses: ./.github/workflows/labels.yml + with: + toolkit-ref: ${{ github.sha }} + + housekeeping-report: + permissions: + contents: read + pull-requests: read + uses: ./.github/workflows/housekeeping.yml + with: + toolkit-ref: ${{ github.sha }} + reminders: | + This is a test run. diff --git a/.github/workflows/housekeeping.yml b/.github/workflows/housekeeping.yml new file mode 100644 index 0000000..bd815f7 --- /dev/null +++ b/.github/workflows/housekeeping.yml @@ -0,0 +1,100 @@ +name: Housekeeping + +# Reusable: a report across repositories of merged branches (safe to delete), +# quiet branches and waiting pull requests. It only reports; nothing is +# deleted. The report goes in the run's summary, and to Discord when a +# webhook is given. + +on: + workflow_call: + inputs: + repos: + description: "Space- or newline-separated OWNER/NAME list (default: the calling repository)" + type: string + default: "" + keep: + description: "Space-separated branches that live on purpose (main and master always are)" + type: string + default: "" + upstreams: + description: "Newline-separated OWNER/NAME=UPSTREAM/NAME pairs: open pull requests there keep their branches" + type: string + default: "" + reminders: + description: "Newline-separated lines for an 'Also check' list" + type: string + default: "" + days: + description: "A branch without commits this many days is quiet" + type: number + default: 14 + pr-days: + description: "A pull request untouched this many days is waiting" + type: number + default: 7 + title: + description: "The Discord post's title" + type: string + default: Housekeeping + toolkit-ref: + description: "The toolkit version to run the script from" + type: string + default: v0 + secrets: + token: + description: "A token that can read every repository listed (default: the workflow's own token)" + required: false + webhook: + description: "A Discord webhook URL for the report (optional)" + required: false + +permissions: + contents: read + pull-requests: read + +jobs: + report: + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - uses: actions/checkout@v4 + with: + repository: Horrible-Dev/toolkit + ref: ${{ inputs.toolkit-ref }} + path: .toolkit + + - name: Report + env: + GH_TOKEN: ${{ secrets.token || github.token }} + REPOS: ${{ inputs.repos || github.repository }} + KEEP: ${{ inputs.keep }} + UPSTREAMS: ${{ inputs.upstreams }} + REMINDERS: ${{ inputs.reminders }} + DAYS: ${{ inputs.days }} + PR_DAYS: ${{ inputs.pr-days }} + run: | + set -- --days "$DAYS" --pr-days "$PR_DAYS" + for repo in $REPOS; do set -- "$@" --repo "$repo"; done + for branch in $KEEP; do set -- "$@" --keep "$branch"; done + while IFS= read -r pair; do + [ -n "$pair" ] && set -- "$@" --upstream "$pair" + done <<< "$UPSTREAMS" + while IFS= read -r line; do + [ -n "$line" ] && set -- "$@" --reminder "$line" + done <<< "$REMINDERS" + python3 .toolkit/housekeeping/housekeeping.py "$@" > "$RUNNER_TEMP/report.md" + cat "$RUNNER_TEMP/report.md" >> "$GITHUB_STEP_SUMMARY" + { + echo 'report<> "$GITHUB_ENV" + + - name: Post to Discord + uses: ./.toolkit/actions/discord-post + with: + webhook: ${{ secrets.webhook }} + title: ${{ inputs.title }} + url: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + description: ${{ env.report }} + color: "10070709" diff --git a/README.md b/README.md index 000d159..22a8cb5 100644 --- a/README.md +++ b/README.md @@ -125,6 +125,45 @@ python3 release_notes.py --config .github/release-notes.toml --version 1.2.0 \ For a test build, add `--testing --tag v1.2.0-testing`. The notes then start with the config's `[testing]` text, or a default one. +## Housekeeping + +[housekeeping/housekeeping.py](housekeeping/housekeeping.py) reports, across your repositories: + +- branches already merged into the default branch, so safe to delete +- branches with no commits in a while +- pull requests nobody has touched in a while + +It only reports. Nothing is deleted. The reusable workflow puts the report in the run's summary, and posts it to Discord when you give it a webhook. + +```yaml +# .github/workflows/housekeeping.yml +name: Housekeeping +on: + schedule: + - cron: "0 14 * * 1" + workflow_dispatch: +permissions: + contents: read + pull-requests: read +jobs: + report: + uses: Horrible-Dev/toolkit/.github/workflows/housekeeping.yml@v0 + with: + repos: | + OWNER/app + OWNER/website + keep: release-1.x + reminders: | + Remove old build folders on the build machine. + secrets: + token: ${{ secrets.READ_TOKEN }} + webhook: ${{ secrets.DISCORD_WEBHOOK }} +``` + +The workflow's own token can only read the calling repository. To cover others, pass a read-only token as `token`. `upstreams` takes `OWNER/NAME=UPSTREAM/NAME` lines: branches with an open pull request to that upstream are kept. + +Run the script by hand with `python3 housekeeping/housekeeping.py --repo OWNER/app --repo OWNER/website`. + ## Credits Parts of the toolkit are adapted from ChupathingyCE's release and housekeeping tools. diff --git a/housekeeping/housekeeping.py b/housekeeping/housekeeping.py new file mode 100755 index 0000000..a9a4b9c --- /dev/null +++ b/housekeeping/housekeeping.py @@ -0,0 +1,162 @@ +#!/usr/bin/env python3 +"""A housekeeping report across repositories: branches already merged into +the default branch (safe to delete), branches nobody has committed to in a +while, and pull requests waiting a while. It only reports; nothing is +deleted. + + housekeeping.py --repo OWNER/NAME [--repo ...] [--keep BRANCH ...] + [--upstream OWNER/NAME=UPSTREAM/NAME ...] [--reminder TEXT ...] + [--days N] [--pr-days N] + +Reads GitHub with gh (GH_TOKEN). Prints Markdown short enough for a Discord +embed (4096 characters). + +--keep names branches that live on purpose (main and master always are). +--upstream lists a repository whose open pull requests to another one come +from its branches: those branches are kept while the pull request is open. +--reminder adds a line to an "Also check" list at the end. +""" + +import argparse +import datetime +import json +import subprocess +import sys + +ALWAYS_KEPT = {"main", "master", "gh-pages"} +EMBED_LIMIT = 4096 + + +class ReadError(RuntimeError): + pass + + +def failure(what: str, stderr: str) -> ReadError: + """one short line for the report (gh's usage text never goes to Discord)""" + text = (stderr.strip().splitlines() or ["failed"])[0] + if "No common ancestor" in stderr: + text = "no history in common with the default branch" + elif "HTTP 403" in stderr or "HTTP 404" in stderr: + text = "the token can't read this repository" + return ReadError(f"{what}: {text[:200]}") + + +def gh(path: str): + result = subprocess.run(["gh", "api", path], capture_output=True, text=True) + if result.returncode != 0: + raise failure(path, result.stderr) + return json.loads(result.stdout) if result.stdout.strip() else None + + +def paged(path: str) -> list: + # (one item a line: --jq instead of --slurp, which older gh lacks) + result = subprocess.run(["gh", "api", "--paginate", path, "--jq", ".[]"], capture_output=True, text=True) + if result.returncode != 0: + raise failure(path, result.stderr) + return [json.loads(line) for line in result.stdout.splitlines() if line.strip()] + + +def age_days(timestamp: str, now: datetime.datetime) -> int: + when = datetime.datetime.fromisoformat(timestamp.replace("Z", "+00:00")) + return (now - when).days + + +def own_heads(pulls: list, repository: str) -> set: + return {pull["head"]["ref"] for pull in pulls + if pull["head"].get("repo") and pull["head"]["repo"]["full_name"] == repository} + + +def survey(repository: str, now: datetime.datetime, stale_days: int, pr_days: int, kept=(), upstream=None): + """(merged branches, stale branches, waiting pull requests)""" + default = gh(f"repos/{repository}")["default_branch"] + pulls = paged(f"repos/{repository}/pulls?state=open&per_page=100") + open_heads = own_heads(pulls, repository) + if upstream: + open_heads |= own_heads(paged(f"repos/{upstream}/pulls?state=open&per_page=100"), repository) + keep = ALWAYS_KEPT | set(kept) + merged, stale = [], [] + for branch in paged(f"repos/{repository}/branches?per_page=100"): + name = branch["name"] + if name in keep or name == default: + continue + try: + comparison = gh(f"repos/{repository}/compare/{default}...{branch['commit']['sha']}") + except ReadError as problem: + # (a data branch with its own history: skipped, not the whole repository) + if "no history in common" in str(problem): + continue + raise + if comparison["ahead_by"] == 0: + if name not in open_heads: + merged.append(name) + continue + days = age_days(comparison["commits"][-1]["commit"]["committer"]["date"], now) if comparison["commits"] else 0 + if days >= stale_days and name not in open_heads: + stale.append(f"{name} ({days} d)") + waiting = [] + for pull in pulls: + days = age_days(pull["updated_at"], now) + if days >= pr_days: + waiting.append(f"#{pull['number']} {pull['title'][:60]} ({days} d{', draft' if pull.get('draft') else ''})") + return merged, stale, waiting + + +def listed(items, limit=8) -> str: + shown = ", ".join(f"`{item}`" for item in items[:limit]) + return shown + (f" and {len(items) - limit} more" if len(items) > limit else "") + + +def report(repositories, now, days=14, pr_days=7, kept=(), upstreams=None, reminders=()): + """(the Markdown report, how many items it lists)""" + upstreams = upstreams or {} + lines, total = [], 0 + for repository in repositories: + short = repository.split("/")[1] + try: + merged, stale, waiting = survey(repository, now, days, pr_days, kept, upstreams.get(repository)) + except ReadError as error: + lines.append(f"**{short}**: not read ({error})") + continue + total += len(merged) + len(stale) + len(waiting) + if not (merged or stale or waiting): + lines.append(f"**{short}**: tidy") + continue + lines.append(f"**{short}**") + if merged: + lines.append(f"- Merged, safe to delete ({len(merged)}): {listed(merged)}") + if stale: + lines.append(f"- No commits in {days}+ days ({len(stale)}): {listed(stale)}") + if waiting: + lines.append(f"- PRs waiting {pr_days}+ days ({len(waiting)}): {listed(waiting, 5)}") + if reminders: + lines += ["", "**Also check:**"] + [f"- {reminder}" for reminder in reminders] + text = "\n".join(lines) + if len(text) > EMBED_LIMIT: + text = text[:EMBED_LIMIT - 20].rsplit("\n", 1)[0] + "\n(cut short)" + return text, total + + +def main(argv=None) -> int: + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("--repo", action="append", required=True, help="OWNER/NAME; repeat for more") + parser.add_argument("--keep", action="append", default=[], help="a branch that lives on purpose") + parser.add_argument("--upstream", action="append", default=[], help="OWNER/NAME=UPSTREAM/NAME") + parser.add_argument("--reminder", action="append", default=[], help="a line for the 'Also check' list") + parser.add_argument("--days", type=int, default=14, help="a branch without commits this long is quiet") + parser.add_argument("--pr-days", type=int, default=7, help="a pull request untouched this long is waiting") + options = parser.parse_args(argv) + upstreams = {} + for pair in options.upstream: + repository, sep, upstream = pair.partition("=") + if not sep or not upstream: + parser.error(f"--upstream takes OWNER/NAME=UPSTREAM/NAME, not {pair!r}") + upstreams[repository.strip()] = upstream.strip() + now = datetime.datetime.now(datetime.timezone.utc) + text, total = report(options.repo, now, options.days, options.pr_days, options.keep, upstreams, options.reminder) + print(text) + print(f"{total} item(s) to look at", file=sys.stderr) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/test_housekeeping.py b/tests/test_housekeeping.py new file mode 100644 index 0000000..06dfac5 --- /dev/null +++ b/tests/test_housekeeping.py @@ -0,0 +1,99 @@ +import datetime +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parent.parent +sys.path.insert(0, str(ROOT / "housekeeping")) +import housekeeping # noqa: E402 + +NOW = datetime.datetime(2026, 6, 30, tzinfo=datetime.timezone.utc) + + +def days_ago(days): + return (NOW - datetime.timedelta(days=days)).strftime("%Y-%m-%dT%H:%M:%SZ") + + +def pull(number, ref, repo, updated, title="A change", draft=False): + return {"number": number, "title": title, "draft": draft, "updated_at": updated, + "head": {"ref": ref, "repo": {"full_name": repo}}} + + +class FakeGitHub: + def __init__(self, branches, comparisons, pulls, upstream_pulls=(), unreadable=()): + self.branches = branches + self.comparisons = comparisons + self.pulls = pulls + self.upstream_pulls = list(upstream_pulls) + self.unreadable = set(unreadable) + + def gh(self, path): + repository = "/".join(path.split("/")[1:3]) + if repository in self.unreadable: + raise housekeeping.failure(path, "HTTP 404: Not Found") + if path.count("/") == 2: + return {"default_branch": "main"} + sha = path.split("...")[1] + if self.comparisons[sha] == "unrelated": + raise housekeeping.failure(path, "No common ancestor between main and x") + return self.comparisons[sha] + + def paged(self, path): + if "/branches" in path: + return [{"name": name, "commit": {"sha": name}} for name in self.branches] + if path.startswith("repos/up/"): + return self.upstream_pulls + return self.pulls + + +def install(monkeypatch, fake): + monkeypatch.setattr(housekeeping, "gh", fake.gh) + monkeypatch.setattr(housekeeping, "paged", fake.paged) + + +def comparison(ahead, days): + return {"ahead_by": ahead, "commits": [{"commit": {"committer": {"date": days_ago(days)}}}] if ahead else []} + + +def test_merged_stale_and_waiting(monkeypatch): + fake = FakeGitHub( + branches=["main", "done", "quiet", "fresh", "open-pr", "data", "keepme"], + comparisons={"done": comparison(0, 0), "quiet": comparison(3, 30), "fresh": comparison(1, 2), + "open-pr": comparison(2, 40), "data": "unrelated", "keepme": comparison(0, 0)}, + pulls=[pull(5, "open-pr", "o/r", days_ago(10), "Old work", draft=True), pull(6, "x", "fork/r", days_ago(1))]) + install(monkeypatch, fake) + merged, stale, waiting = housekeeping.survey("o/r", NOW, 14, 7, kept=["keepme"]) + assert merged == ["done"] + assert stale == ["quiet (30 d)"] + assert waiting == ["#5 Old work (10 d, draft)"] + + +def test_upstream_pull_requests_keep_their_branches(monkeypatch): + fake = FakeGitHub(branches=["sent"], comparisons={"sent": comparison(0, 0)}, pulls=[], + upstream_pulls=[pull(9, "sent", "o/r", days_ago(1))]) + install(monkeypatch, fake) + assert housekeeping.survey("o/r", NOW, 14, 7, upstream="up/r") == ([], [], []) + assert housekeeping.survey("o/r", NOW, 14, 7) == (["sent"], [], []) + + +def test_report_lines(monkeypatch): + fake = FakeGitHub(branches=["done"], comparisons={"done": comparison(0, 0)}, pulls=[], unreadable={"o/secret"}) + install(monkeypatch, fake) + text, total = housekeeping.report(["o/r", "o/secret"], NOW, reminders=["Clear old worktrees."]) + assert total == 1 + assert "**r**\n- Merged, safe to delete (1): `done`" in text + assert "**secret**: not read (" in text and "can't read this repository" in text + assert text.endswith("**Also check:**\n- Clear old worktrees.") + + +def test_tidy_and_long_reports(monkeypatch): + fake = FakeGitHub(branches=[], comparisons={}, pulls=[]) + install(monkeypatch, fake) + text, total = housekeeping.report(["o/r"], NOW) + assert (text, total) == ("**r**: tidy", 0) + text, _ = housekeeping.report([f"o/r{i}" for i in range(400)], NOW) + assert len(text) <= housekeeping.EMBED_LIMIT + assert text.endswith("(cut short)") + + +def test_listed_cuts_long_lists(): + assert housekeeping.listed(list("abc"), 2) == "`a`, `b` and 1 more"