diff --git a/.gitguardian.yaml b/.gitguardian.yaml new file mode 100644 index 0000000..e2adc70 --- /dev/null +++ b/.gitguardian.yaml @@ -0,0 +1,10 @@ +# ggshield settings (GitGuardian's command-line scanner, configuration version 2): +# https://docs.gitguardian.com/ggshield-docs/configuration +# The backfill scan of 2026-11-04 (https://github.com/colvenna/colvenna.github.io/issues/4), which also +# covered the Pages artifact, found nothing in the history, that artifact or the GitHub text, so nothing +# is ignored: no value, no path and no detector. Add an entry under secret.ignored_matches only for a +# value checked by hand, by its ignore SHA with a name that gives the file and the reason, reviewed in +# its pull request (README.md#secret-scanning); a real credential is revoked or rotated, never ignored. +version: 2 +# The pre-push hook scans at most this many commits of a push (ggshield's default is 50). +max_commits_for_hook: 1000 diff --git a/.github/workflows/ggshield.yml b/.github/workflows/ggshield.yml new file mode 100644 index 0000000..1527ff7 --- /dev/null +++ b/.github/workflows/ggshield.yml @@ -0,0 +1,52 @@ +name: Website secret scan + +on: + push: + pull_request: + workflow_dispatch: + +permissions: + contents: read + +jobs: + ggshield: + name: Scan for secrets with ggshield + runs-on: ubuntu-24.04-arm + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + fetch-depth: 0 + - name: Install hash-locked ggshield + run: | + python3 -m venv "$RUNNER_TEMP/ggshield" + "$RUNNER_TEMP/ggshield/bin/python" -m pip install --require-hashes --only-binary=:all: \ + -r scripts/ggshield-requirements.lock + - name: Scan with the repository's .gitguardian.yaml + env: + GITGUARDIAN_API_KEY: ${{ secrets.GITGUARDIAN_API_KEY }} + FROM_FORK: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != github.repository }} + # ggshield's commit range: a pull request's base branch, else the commits of the push + # (GITHUB_PUSH_BASE_SHA), else everything not on the default branch. After a force push the + # checkout lacks the replaced commit, which GitHub still serves by SHA: it is fetched, so only + # the new commits are scanned. A new branch (all zeros) falls back to the default branch. + GITHUB_PUSH_BASE_SHA: ${{ github.event.before }} + GITHUB_DEFAULT_BRANCH: origin/${{ github.event.repository.default_branch }} + EVENT: ${{ github.event_name }} + run: | + if [ -z "$GITGUARDIAN_API_KEY" ]; then + if [ "$FROM_FORK" = true ]; then + echo "::notice::A pull request from a fork gets no secrets: a maintainer scans it before merging (README.md#secret-scanning)." + exit 0 + fi + echo "::error::The GITGUARDIAN_API_KEY Actions secret is not set (README.md#secret-scanning)." + exit 1 + fi + if [ -n "$GITHUB_PUSH_BASE_SHA" ] && ! git cat-file -e "$GITHUB_PUSH_BASE_SHA^{commit}" 2>/dev/null \ + && ! git fetch --quiet --no-tags origin "$GITHUB_PUSH_BASE_SHA" 2>/dev/null; then + unset GITHUB_PUSH_BASE_SHA + fi + if [ "$EVENT" = workflow_dispatch ]; then + "$RUNNER_TEMP/ggshield/bin/ggshield" secret scan repo . + else + "$RUNNER_TEMP/ggshield/bin/ggshield" secret scan ci + fi diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..a873942 --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +# ggshield writes its cache, which holds the latest findings, here: never commit it. +.cache_ggshield diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..3586572 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,22 @@ +# Local hooks (https://pre-commit.com): `pre-commit install` once per clone installs both hook types. +# ggshield scans the staged changes (pre-commit) and the commits about to be pushed (pre-push; in a public +# repository a push publishes them) with .gitguardian.yaml. scripts/ggshield-hook.sh runs the ggshield on +# PATH only at the version scripts/ggshield-requirements.lock pins (install it from that file), and +# ggshield needs a GitGuardian API key (GITGUARDIAN_API_KEY, or `ggshield auth login`). +# See README.md#secret-scanning. +default_install_hook_types: [pre-commit, pre-push] +repos: + - repo: local + hooks: + - id: ggshield + name: ggshield (secret scan of the staged changes) + entry: scripts/ggshield-hook.sh pre-commit + language: system + pass_filenames: false + stages: [pre-commit] + - id: ggshield-push + name: ggshield (secret scan of the commits to push) + entry: scripts/ggshield-hook.sh pre-push + language: system + pass_filenames: false + stages: [pre-push] diff --git a/README.md b/README.md new file mode 100644 index 0000000..89e50af --- /dev/null +++ b/README.md @@ -0,0 +1,60 @@ +# Colvenna website + +This repository holds the Colvenna series website: the generated static pages (`index.html`, `site.js`, `styles.css`) and the brand assets. GitHub Pages publishes it from the root of `main`, at the domain in `CNAME`. + +The site files are generated by the series index's site build and copied here in a pull request. A sync replaces the site files only. This repository's own files are not part of that build, and a sync keeps them: this README, `.gitignore`, `.gitguardian.yaml`, `.pre-commit-config.yaml`, `scripts/` and `.github/`. + +## Secret scanning + +Nothing is made public before [ggshield](https://docs.gitguardian.com/ggshield-docs/getting-started), GitGuardian's command-line scanner, finds 0 incidents in it with this repository's [`.gitguardian.yaml`](.gitguardian.yaml). The repository is public, so a push publishes its commits, and a merge into `main` publishes the website. The local hooks below are a best-effort check before a push, and CI scans each push again. A ruleset on `main` requires a pull request whose secret scan check passes and whose branch is up to date with `main`, and blocks force pushes and deletion. Before merging, a maintainer also runs the full scan. ggshield is installed from a hash-locked file: + +```bash +python3 -m venv ~/.venvs/colvenna-ggshield +~/.venvs/colvenna-ggshield/bin/python -m pip install --require-hashes --only-binary=:all: \ + -r scripts/ggshield-requirements.lock +export PATH="$HOME/.venvs/colvenna-ggshield/bin:$PATH" GITGUARDIAN_API_KEY=... # or: ggshield auth login +``` + +- **Each pull request:** the [secret scan workflow](.github/workflows/ggshield.yml) scans the new commits of each push and pull request with the `GITGUARDIAN_API_KEY` Actions secret, which the repository owner sets. A manual run scans the whole history. + - Pull requests from forks get no secrets, so for them the job only notes this. A maintainer scans such a change before merging it. +- **Before merging a site sync**, a maintainer also runs the full scan on the sync's branch and records its summary in the pull request. Check the remaining API quota first with `ggshield quota`, because a full scan uses part of it. + + ```bash + scripts/ggshield-scan.sh + ``` + + It scans everything the public can fetch: + - an anonymous mirror clone (every branch, tag and `refs/pull/*`), plus each pull-request head that a force push replaced and GitHub still serves. The website is the repository's content, so this covers it; + - the GitHub text: issues, pull requests, comments, reviews, release notes and commit comments. + + It uses the `.gitguardian.yaml` of the given branch (`main` by default), and refuses any ggshield version other than the one in the lock file. ggshield's default exclusions (for example `vendor/`, `node_modules/`, `*.mdx` and `*.sops`) apply to every scan and cannot be turned off in 1.56.1, so any path in the history that they match makes the scan fail. It also scans the wiki when the wiki is enabled; an enabled wiki that cannot be cloned makes it fail. It exits 0 only when every scan is complete and finds 0 incidents; any file that ggshield skips, other than binary files, makes it fail. + + The reports in the directory contain the matched values, so keep the directory private. The summary has no values. +- **Each commit and each push (best effort):** `pre-commit install` enables the [ggshield hooks](.pre-commit-config.yaml). They use the `ggshield` on `PATH` and refuse any version other than the one in the lock file. + - The pre-commit hook scans the staged changes. + - The pre-push hook scans the commits about to be published. It covers at most `max_commits_for_hook` commits of a push (1000, set in `.gitguardian.yaml`), and ggshield lets a push through with only a warning when it cannot work out the commit range. When one push updates several refs (`git push --all`, or several branches or tags), pre-commit hands ggshield only the first, so push them one at a time. + - Without an API key, they stop with ggshield's error. + - `SKIP=ggshield,ggshield-push` or `git push --no-verify` skips them. The CI job and the full scan still apply. +- **A finding:** + - A real credential is revoked or rotated first, then the history is reviewed. It is never ignored. + - Any other finding is added to `.gitguardian.yaml` by its ignore SHA, with a name that gives the file and the reason. The addition is reviewed in its pull request. + - No path or detector is ignored. +- **Deployment values:** ggshield does not look for AWS account IDs, ARNs, endpoints or internal addresses, so before merging, a maintainer also searches what the full scan fetched: + - Each account that a deployment of yours used must appear 0 times. + - List the accounts named in ARNs, the AWS endpoint hosts and the private IPv4 addresses, and review each one. + + Documentation examples, such as `123456789012`, are expected: + + ```bash + ACCOUNT=; DIR= + content() { for git in "$DIR"/mirror.git "$DIR"/wiki.git; do if [ -d "$git" ]; then git -C "$git" log --all -p; fi; done + grep -rhI '' "$DIR/github-text"; } + content | grep -c "$ACCOUNT" # must be 0 + content | grep -oE 'arn:aws[a-z-]*:[a-z0-9-]*:[a-z0-9-]*:[0-9]{12}:' | grep -oE '[0-9]{12}' | sort | uniq -c + content | grep -oE '[a-z0-9.-]+\.(amazonaws\.com|cloudfront\.net|awsapps\.com)' | sort | uniq -c + content | grep -oE '\b(10\.[0-9]+|172\.(1[6-9]|2[0-9]|3[01])|192\.168)\.[0-9]+\.[0-9]+\b' | sort | uniq -c + ``` + +## License + +See [LICENSE](LICENSE). diff --git a/scripts/ggshield-hook.sh b/scripts/ggshield-hook.sh new file mode 100755 index 0000000..0a16d0a --- /dev/null +++ b/scripts/ggshield-hook.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash +# pre-commit hook: `ggshield-hook.sh pre-commit` scans the staged changes, `ggshield-hook.sh pre-push` the +# commits about to be pushed (in a public repository a push publishes them). Either runs the ggshield on +# PATH only at the version scripts/ggshield-requirements.lock pins (README.md#secret-scanning). +set -euo pipefail +mode="${1:-}" +[[ "$mode" == pre-commit || "$mode" == pre-push ]] || { echo "usage: ggshield-hook.sh {pre-commit,pre-push}" >&2; exit 2; } +shift +want="$(sed -n 's/^ggshield==\([^ ]*\) .*/\1/p' "$(dirname "$0")/ggshield-requirements.lock")" +have="$(ggshield --version 2>/dev/null | sed -n 's/^ggshield, version //p' || true)" +if [[ "$have" != "$want" ]]; then + echo "the ggshield hook needs ggshield $want on PATH, installed from scripts/ggshield-requirements.lock" \ + "(found: ${have:-none}); see README.md#secret-scanning" >&2 + exit 1 +fi +exec ggshield secret scan "$mode" "$@" diff --git a/scripts/ggshield-requirements.lock b/scripts/ggshield-requirements.lock new file mode 100644 index 0000000..8d1d66a --- /dev/null +++ b/scripts/ggshield-requirements.lock @@ -0,0 +1,62 @@ +# Secret scan (ggshield: scripts/ggshield-scan.sh, the pre-commit hook and CI): Python 3.12, Linux arm64. Downloaded wheel hashes. +annotated-types==0.8.0 --hash=sha256:f072f4d804ea359e4eaf198b1af7a8b0943881a87f31bb764f8bf219bb9419e0 +anyio==4.16.0 --hash=sha256:c66ae634e0b8228396b801741beb3294326d4b6102f7bba4041c0dfb58c29026 +certifi==2026.10.23 --hash=sha256:e9b54de76e530bb6cf0ccd036787103acb58fd42ba930ea8bf948ddbaa2f61d1 +cffi==2.1.1 --hash=sha256:68e62fe11f30d5ca8289242866f0a5291402d8529ca2178ab8afc5c9694ae890 +charset-normalizer==3.1.0 --hash=sha256:3d9098b479e78c85080c98e1e35ff40b4a31d8953102bb0fd7d1b6f8a2111a3d +click==8.5.0 --hash=sha256:255bc9599cf7748b4b1a446ccc735421bd08a2ae529a8b88597d3de5664ee360 +configupdater==3.2 --hash=sha256:0f65a041627d7693840b4dd743581db4c441c97195298a29d075f91b79539df2 +cryptography==50.0.2 --hash=sha256:87e9ce85beb6b328ba370cc6e6aea483c92617b4c95b1d33a49297eb662bfb04 +dnspython==2.9.2 --hash=sha256:af15f7d385a6ae2fb35a796fde2138004c8c42c7389090e2addc83e3b3340fd8 +email-validator==2.3.0 --hash=sha256:80f13f623413e6b197ae73bb10bf4eb0908faf509ad8362c5edeb0be7fd450b4 +filelock==4.2.8 --hash=sha256:c2be657e2c279cb032af8e4ea8ce2e20aa982e540be77fcaa34910d068ac2510 +ggshield==1.56.1 --hash=sha256:8361e5e953f5d1a96e5727baf14df3ad0e8f1b2a85918b50f13e0698d9b35475 +h11==0.16.0 --hash=sha256:63cf8bbe7522de3bf65932fda1d9c2772064ffb3dae62d55932da54b31cb6c86 +httpcore2==2.16.0 --hash=sha256:5c4358319f2602d984ad93182c9fee5160f22e9da7babdda870a5e7c4e2dc34f +httpx2==2.16.0 --hash=sha256:fd2c5d7909e8d86fe92d53020c310ca0e4e431b838c207c5754b9bbf0308d49a +id==1.6.1 --hash=sha256:f5ec41ed2629a508f5d0988eda142e190c9c6da971100612c4de9ad9f9b237ca +idna==3.20 --hash=sha256:ab7ae7122974553370f0bdb919e1a960b2cd1bc1ef0276416d896db81c14582c +jaraco-classes==3.4.0 --hash=sha256:f662826b6bed8cace05e7ff873ce0f9283b5c924470fe664fff1c2f00f581790 +jaraco-context==6.1.2 --hash=sha256:bf8150b79a2d5d91ae48629d8b427a8f7ba0e1097dd6202a9059f29a36379535 +jaraco-functools==4.6.0 --hash=sha256:99e3dc0060c5cbe8fcd1cdb36258e2a65ca40f1566b2033b12abb1bb44dd3c30 +jeepney==0.9.0 --hash=sha256:97e5714520c16fc0a45695e5365a2e11b81ea79bba796e26f9f1d178cb182683 +keyring==25.7.0 --hash=sha256:be4a0b195f149690c166e850609a477c532ddbfbaed96a404d4e43f8d5e2689f +loguru==0.6.0 --hash=sha256:4e2414d534a2ab57573365b3e6d0234dfb1d84b68b7f3b948e6fb743860a77c3 +markdown-it-py==4.2.0 --hash=sha256:9f7ebbcd14fe59494226453aed97c1070d83f8d24b6fc3a3bcf9a38092641c4a +marshmallow==4.4.0 --hash=sha256:03a882ff2eabb9a4e24b433c2949f359ef4d172a12fdc14cde10375fd55328b4 +marshmallow-dataclass==8.7.1 --hash=sha256:405cbaaad9cea56b3de2f85eff32a9880e3bf849f652e7f6de7395e4b1ddc072 +mdurl==0.1.2 --hash=sha256:84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8 +more-itertools==11.2.1 --hash=sha256:35a7377edd1dd6608dcb2cdf534ded55ea32d49448875ad042cd3f879fb1ded0 +mypy-extensions==1.2.0 --hash=sha256:993e7a20aa5671eb41a7132cf555af30062da99729601601b9a8382b11cae768 +notify-py==0.3.43 --hash=sha256:90da686b480bf05d033b21fb5118662bb7a4c68e68cb21d93a048ca7b867f9b4 +oauthlib==3.3.1 --hash=sha256:88119c938d2b8fb88561af5f6ee0eec8cc8d552b7bb1f712743136eb7523b7a1 +packaging==26.3 --hash=sha256:d7193f7c8e4e93f444fde0262bf90af30e16fa0ad0ad44cb553c87339b23cd1c +platformdirs==4.14.4 --hash=sha256:89c1bfa30434b2dcc08655bcfb1789e1a6f97602c11ff990615d09a4b823d6be +pycparser==3.12 --hash=sha256:a076f3f489b39b43ed13563a1dafe42447445bec2e294154ec5c57fcdda4c85e +pydantic==2.14.2 --hash=sha256:7efbf7507b937931cd32a5755acb00ea1cf5ccb1d4e8067a5850d5069e382118 +pydantic-core==2.50.2 --hash=sha256:43a8fd56f3c681f3cda2d5c842ad28e88241df742dfcbe8fd1dd437a3056e1f6 +pygitguardian==1.35.0 --hash=sha256:8c9969fc9aaf0e8b5d9c9ef8289e779393ad3b9fc64ff686643eedbda3eb70b6 +pygments==2.21.0 --hash=sha256:2363c69b61c4a97c838da3b130dcd6468f4848992b21a82f2a63ec34377137d9 +pyjwt==2.15.1 --hash=sha256:42d59d631f7768a1028a64c7ff581a9bf7519804daf91fc5b6c56e30eec5e193 +python-dotenv==1.2.4 --hash=sha256:42269a8a5b3fd54ffa6f3d84b18abed50064717576b4ecf03dc4a55d8aa04fdc +pyyaml==6.0.3 --hash=sha256:9149cad251584d5fb4981be1ecde53a1ca46c891a79788c0df828d2f166bda28 +requests==2.34.2 --hash=sha256:2a0d60c172f83ac6ab31e4554906c0f3b3588d37b5cb939b1c061f4907e278e0 +rfc3161-client==1.0.9 --hash=sha256:7b703b233dbee7228d117d5f1c118363c61d523fa6f018c9217c79e3c6684f86 +rfc8785==0.1.4 --hash=sha256:520d690b448ecf0703691c76e1a34a24ddcd4fc5bc41d589cb7c58ec651bcd48 +rich==13.9.4 --hash=sha256:6049d5e6ec054bf2779ab3358186963bac2ea89175919d699e378b99738c2a90 +ruamel-yaml==0.19.1 --hash=sha256:27592957fedf6e0b62f281e96effd28043345e0e66001f97683aa9a40c667c93 +secretstorage==3.5.0 --hash=sha256:0ce65888c0725fcb2c5bc0fdb8e5438eece02c523557ea40ce0703c266248137 +securesystemslib==1.6.0 --hash=sha256:1c1aa00f058f4851f038d409c8ca4a727db854a91f07ec3a0fa047bb93e43e0e +setuptools==84.2.1 --hash=sha256:ca140b45d5325a9478f880e351484f62fef15f336462e449ca0c03c873d589c2 +sigstore==4.5.1 --hash=sha256:02e000197b34980f1d382741d6f83c2fecbcef7debf7304f36a5253f3c892cd9 +sigstore-models==0.0.6 --hash=sha256:5201a68f4d7d0f8bec1e2f4378eb646b084c52609a4e31db8c385095fff68b2e +sigstore-rekor-types==0.0.18 --hash=sha256:b62bf38c5b1a62bc0d7fe0ee51a0709e49311d137c7880c329882a8f4b2d1d78 +tomlkit==0.13.3 --hash=sha256:c89c649d79ee40629a9fda55f8ace8c6a1b42deb912b2a8fd8d942ddadb606b0 +truststore==0.10.5 --hash=sha256:9aaaedaefaf06d8b206278cf8b5012bc897f485a874503501e12d776df78951c +tuf==7.0.1 --hash=sha256:d30434bda6e079ab303fb30d1b3006d939a10ca34783b1573d61cd9b802fa45c +typeguard==4.6.0 --hash=sha256:79878165bb86f2cf5d41d159a0ff1792a796cf496882d2fe1b1c6c7049b9cdd7 +typing-extensions==4.16.0 --hash=sha256:481caa481374e813c1b176ada14e97f1f67a4539ce9cfeb3f350d78d6370c2e8 +typing-inspect==0.9.0 --hash=sha256:9ee6fc59062311ef8547596ab6b955e1b8aa46242d854bfc78f4f6b0eff35f9f +typing-inspection==0.4.4 --hash=sha256:65b8397ba37ccbce054456aaccddfc91e6e3083c92824df348d96ca832f3f147 +unearth==0.19.0 --hash=sha256:f12c65341303ece9bee986db35ac8aa0d17df4fa000c728a76c8c3798b1f4cf1 +urllib3==2.8.0 --hash=sha256:0cf3cae568d36aa9576b28dfb35f11328f1cb974ca7647d9475ebb86c75ac6e3 diff --git a/scripts/ggshield-scan.sh b/scripts/ggshield-scan.sh new file mode 100755 index 0000000..c99e0a5 --- /dev/null +++ b/scripts/ggshield-scan.sh @@ -0,0 +1,179 @@ +#!/usr/bin/env bash +# Publication secret scan with ggshield (GitGuardian's CLI), required before anything here is made public +# (README.md#secret-scanning). It scans everything the public can fetch, with the .gitguardian.yaml of the +# scanned branch: +# - history: an anonymous `git clone --mirror` (every branch, tag and refs/pull/*), plus each pull-request +# head a force push replaced that GitHub still serves by its SHA. The website is the repository's +# content, so the history covers it; +# - GitHub text: issues and pull requests with their comments, reviews and review comments, release +# notes and commit comments; and the wiki, when it is enabled (an enabled wiki that cannot be cloned +# fails the scan). +# Reports with matched values stay in the output directory (keep it private); the summary holds none. +# Exit 0 only when every scan is complete and finds 0 incidents. It refuses a ggshield version other +# than the one scripts/ggshield-requirements.lock pins, because its checks read that version's messages. +# +# usage: scripts/ggshield-scan.sh [branch, default main] +# needs: git, gh (authenticated), python3, ggshield (or GGSHIELD=) with a GitGuardian API key +set -euo pipefail +out="${1:?usage: scripts/ggshield-scan.sh [branch]}" +branch="${2:-main}" +repo=colvenna/colvenna.github.io +gg="${GGSHIELD:-ggshield}" +mkdir -m 700 "$out" +export GIT_TERMINAL_PROMPT=0 # anonymous: never prompt, use no stored credential +anon=(git -c credential.helper=) +want="$(sed -n 's/^ggshield==\([^ ]*\) .*/\1/p' "$(dirname "$0")/ggshield-requirements.lock")" +have="$("$gg" --version 2>/dev/null | sed -n 's/^ggshield, version //p' || true)" +if [[ "$have" != "$want" ]]; then # the completeness checks read this version's messages + echo "the scan needs ggshield $want, installed from scripts/ggshield-requirements.lock (found: ${have:-none})" >&2 + exit 2 +fi +python="$(dirname "$(readlink -f "$(command -v "$gg")")")/python3" # ggshield's own environment +# ggshield 1.56.1 always applies its default exclusions (vendor/, node_modules/, .venv/, *.mdx, *.sops, ...) +# and says nothing about them; --ignore-default-excludes has no effect. excluded prints each NUL-separated +# path on stdin that they match, with ggshield's own matcher. +excluded() { + "$python" -c ' +import sys +from ggshield.cmd.secret.scan.secret_scan_common_options import IGNORED_DEFAULT_WILDCARDS +from ggshield.core.filter import init_exclusion_regexes +from ggshield.utils.files import is_path_excluded +regexes = init_exclusion_regexes(set(IGNORED_DEFAULT_WILDCARDS)) +for path in sorted({path.strip("\n") for path in sys.stdin.read().split("\0")} - {""}): + if is_path_excluded(path, regexes): + print(path)' +} +# Its path scans match the exclusions against the absolute path: refuse an output directory under one. +if [[ -n "$(printf '%s\0' "$(readlink -f "$out")/probe/file.txt" | excluded)" ]]; then + echo "$out is under a path ggshield's default exclusions skip (such as vendor/ or .venv/)" >&2; exit 2 +fi +quota() { "$gg" quota 2>/dev/null | head -1 || echo unavailable; } +quota_before="$(quota)" + +"${anon[@]}" clone --quiet --mirror "https://github.com/$repo.git" "$out/mirror.git" +git -C "$out/mirror.git" remote set-url --push origin no-push://read-only +# shellcheck disable=SC2016 # GraphQL variables, not shell ones +gh api graphql --paginate -F owner="${repo%/*}" -F name="${repo#*/}" -f query=' + query($owner: String!, $name: String!, $endCursor: String) { + repository(owner: $owner, name: $name) { + pullRequests(first: 50, after: $endCursor) { + pageInfo { hasNextPage endCursor } + nodes { timelineItems(first: 100, itemTypes: [HEAD_REF_FORCE_PUSHED_EVENT]) { + nodes { ... on HeadRefForcePushedEvent { beforeCommit { oid } } } } } } } }' \ + --jq '.data.repository.pullRequests.nodes[].timelineItems.nodes[].beforeCommit.oid // empty' \ + | sort -u > "$out/force-pushed-heads.txt" +fetched=0 reachable=0 missing=0 +while read -r sha; do + if git -C "$out/mirror.git" cat-file -e "$sha^{commit}" 2>/dev/null; then reachable=$((reachable + 1)) + elif "${anon[@]}" -C "$out/mirror.git" fetch --quiet origin "$sha:refs/force-pushed/$sha" 2>/dev/null; then + fetched=$((fetched + 1)) + else missing=$((missing + 1)); fi +done < "$out/force-pushed-heads.txt" + +head_sha="$(git -C "$out/mirror.git" rev-parse --verify --quiet "refs/heads/$branch")" \ + || { echo "the repository has no branch $branch" >&2; exit 2; } +git -C "$out/mirror.git" show "$head_sha:.gitguardian.yaml" > "$out/gitguardian.yaml" 2>/dev/null \ + || { echo "$branch has no .gitguardian.yaml: the scan uses the repository's own configuration" >&2; exit 2; } + +# GitHub text, one file per document: {name, text} objects, one a line. +mkdir "$out/github-text" +{ + gh api --paginate "repos/$repo/issues?state=all&per_page=100" --jq '.[] | {name: "\(if .pull_request + then "pull" else "issue" end)-\(.number).md", text: "\(.title)\n\n\(.body // "")"}' + gh api --paginate "repos/$repo/issues/comments?per_page=100" --jq '.[] | {name: "comment-\(.id).md", text: (.body // "")}' + gh api --paginate "repos/$repo/pulls/comments?per_page=100" \ + --jq '.[] | {name: "review-comment-\(.id).md", text: "\(.diff_hunk // "")\n\n\(.body // "")"}' + gh api --paginate "repos/$repo/releases?per_page=100" \ + --jq '.[] | {name: "release-\(.tag_name).md", text: "\(.name // "")\n\n\(.body // "")"}' + gh api --paginate "repos/$repo/comments?per_page=100" --jq '.[] | {name: "commit-comment-\(.id).md", text: (.body // "")}' + pulls="$(gh api --paginate "repos/$repo/pulls?state=all&per_page=100" --jq '.[].number')" # fails the script + for number in $pulls; do + gh api --paginate "repos/$repo/pulls/$number/reviews?per_page=100" \ + --jq ".[] | select(.body != \"\") | {name: \"pull-$number-review-\(.id).md\", text: .body}" + done +} > "$out/github-text.jsonl" +documents="$(python3 -c 'import json, pathlib, sys +lines = open(sys.argv[1]).read().splitlines() +for line in lines: + document = json.loads(line) + pathlib.Path(sys.argv[2], document["name"]).write_text(document["text"]) +print(len(lines))' "$out/github-text.jsonl" "$out/github-text")" + +status=0 +"$gg" --config-path "$out/gitguardian.yaml" secret scan repo "$out/mirror.git" --json \ + > "$out/history.json" 2> "$out/history.stderr" || status=$? +scopes=("history:$status") +has_wiki="$(gh api "repos/$repo" --jq .has_wiki)" # fails the script, rather than skipping the wiki +if [[ "$has_wiki" == true ]]; then + "${anon[@]}" clone --quiet --mirror "https://github.com/$repo.wiki.git" "$out/wiki.git" 2>/dev/null || { + echo "the wiki is enabled but has no clone to scan: disable it, or add a page and scan again" >&2; exit 2; } + wiki_status=0 + "$gg" --config-path "$out/gitguardian.yaml" secret scan repo "$out/wiki.git" --json \ + > "$out/wiki.json" 2> "$out/wiki.stderr" || wiki_status=$? + scopes+=("wiki:$wiki_status") +fi +text_status=0 +"$gg" --config-path "$out/gitguardian.yaml" secret scan path --recursive --yes "$out/github-text" --json \ + > "$out/github-text.json" 2> "$out/github-text.stderr" || text_status=$? + +# The paths of each history that ggshield's default exclusions skip, unquoted (core.quotePath=false), so +# that a non-ASCII path is matched as ggshield sees it; the summary fails on any. +for scope in "${scopes[@]%%:*}"; do + clone="$out/mirror.git"; [[ "$scope" == wiki ]] && clone="$out/wiki.git" + git -C "$clone" -c core.quotePath=false log -z --all --format= --name-only | excluded > "$out/$scope.unscanned" +done +refs="$(git -C "$out/mirror.git" for-each-ref --format='%(refname)')" +{ + echo "date: $(date -u +%Y-%m-%dT%H:%M:%SZ)" + echo "repository: $repo; configuration: .gitguardian.yaml of $branch $head_sha" + echo "ggshield: $("$gg" --version)" + echo "quota before: $quota_before" + echo "refs: $(grep -c '^refs/heads/' <<< "$refs") branches, $(grep -c '^refs/tags/' <<< "$refs") tags," \ + "$(grep -c '^refs/pull/' <<< "$refs") refs/pull; $(git -C "$out/mirror.git" rev-list --all --count) commits;" \ + "force-pushed heads: $fetched fetched, $reachable still reachable, $missing not served" + echo "GitHub text: $documents documents" + echo "quota after: $(quota)" + # A scan is clean when ggshield exits 0, reports no incident and no error, says nothing failed, and + # skips nothing but binary files (which it cannot decode). Findings are located, never shown. + python3 - "$out" "${scopes[@]}" "github-text:$text_status" <<'PY' +import json, sys +from pathlib import Path +out, clean = Path(sys.argv[1]), True +def errors(report): + return bool(report.get("errors")) or any(errors(inner) for inner in report.get("scans") or []) +for argument in sys.argv[2:]: + scope, returncode = argument.rsplit(":", 1) + try: + report = json.loads((out / f"{scope}.json").read_text()) + except json.JSONDecodeError: + report = {} + stderr = (out / f"{scope}.stderr").read_text() + if returncode not in ("0", "1") or "total_incidents" not in report: + print(f"{scope}: ggshield exited {returncode}") + clean = False + continue + if errors(report) or "Scanning failed" in stderr: + print(f"{scope}: incomplete, ggshield reported a failed scan (see its report)") + clean = False + print(f"{scope}: {report['total_incidents']} incidents") + clean = clean and returncode == "0" and report["total_incidents"] == 0 + for line in stderr.splitlines(): + if line.startswith("Skipped ") and not line.endswith(": can't detect encoding"): + print(f" - not scanned: {line[len('Skipped '):].split('/')[-1]}") + clean = False + unscanned = out / f"{scope}.unscanned" + for path in unscanned.read_text().split() if unscanned.exists() else []: + print(f" - not scanned: {path} (ggshield's default exclusions)") + clean = False + for commit in report.get("scans") or [report]: + where = f"commit {commit['id'][:12]}, " if "scans" in report else "" + for entity in commit.get("entities_with_incidents") or []: + for found in entity["incidents"]: + line = found["occurrences"][0]["line_start"] if found["occurrences"] else "?" + print(f" - {found['type']}, {where}{Path(entity['filename']).name}:{line}, " + f"ignore_sha {found['ignore_sha']}") +print("VERDICT: CLEAN" if clean else "VERDICT: FINDINGS") +sys.exit(0 if clean else 1) +PY +} | tee "$out/summary.txt" +grep -qx "VERDICT: CLEAN" "$out/summary.txt"