diff --git a/.github/workflows/benchmark_pages.yml b/.github/workflows/benchmark_pages.yml index ec358483b1..d9511854da 100644 --- a/.github/workflows/benchmark_pages.yml +++ b/.github/workflows/benchmark_pages.yml @@ -42,7 +42,8 @@ jobs: publish: needs: [preview] if: >- - always() && !cancelled() && (needs.preview.outputs.available == 'true' || + always() && !cancelled() && github.ref == 'refs/heads/main' && + (needs.preview.outputs.available == 'true' || github.event_name == 'workflow_dispatch' || (github.event.workflow_run.conclusion == 'success' && (github.event.workflow_run.name == 'Release' || @@ -57,12 +58,7 @@ jobs: permissions: actions: read contents: write - pull-requests: write - pages: write - id-token: write - environment: - name: github-pages - url: ${{ steps.deployment.outputs.page_url }} + pull-requests: read steps: # Only trusted publisher code executes; incoming benchmark artifacts are JSON data. - uses: actions/checkout@v7 @@ -140,7 +136,7 @@ jobs: run: | npm ci --ignore-scripts --no-audit --no-fund npm run build - - name: Refresh stable benchmark references and deploy + - name: Refresh and retain stable benchmark references env: GH_TOKEN: ${{ github.token }} PREVIEW: ${{ needs.preview.outputs.available }} @@ -172,30 +168,3 @@ jobs: export -f refresh_site python3 source/tools/publish_site.py --repo=site -- \ bash -c 'set -euo pipefail; refresh_site retry' - mkdir -p public - rsync --archive --exclude='.git' site/ public/ - python3 source/tools/site_artwork.py source/docs/assets public - python3 source/tools/compact_site.py public - - uses: actions/upload-pages-artifact@v5 - with: - path: public - - uses: actions/deploy-pages@v5 - id: deployment - - name: Verify live benchmark publication - env: - PAGE_URL: ${{ steps.deployment.outputs.page_url }} - run: | - python3 source/tools/verify_benchmark_publication.py \ - --index=public/benchmarks/index.html --url="${PAGE_URL}benchmarks/index.html" - - name: Update the current PR benchmark overview after publication - if: steps.preview.outputs.available == 'true' - env: - GH_TOKEN: ${{ github.token }} - PAGE_URL: ${{ steps.deployment.outputs.page_url }} - SOURCE_RUN: ${{ github.event.workflow_run.id }} - SOURCE_ATTEMPT: ${{ github.event.workflow_run.run_attempt }} - run: | - python3 source/tools/site_storage.py expand-benchmarks site/benchmarks/previews - python3 source/tools/benchmark_preview_ci.py comment --repository="${GITHUB_REPOSITORY}" \ - --run="${SOURCE_RUN}" --attempt="${SOURCE_ATTEMPT}" --root=site/benchmarks \ - --site-url="${PAGE_URL}benchmarks/" diff --git a/.github/workflows/coverage_pages.yml b/.github/workflows/coverage_pages.yml index eace6980d5..557c8a071f 100644 --- a/.github/workflows/coverage_pages.yml +++ b/.github/workflows/coverage_pages.yml @@ -12,8 +12,8 @@ on: workflows: [Release, Test] types: [completed] -# Updating the retained site and deploying it must be one serial operation: two runs restoring the -# same branch concurrently would otherwise overwrite whichever report was pushed first. +# Retain every source report serially: concurrent branch writers could overwrite one another. +# Deployment runs separately, so a Pages timeout never holds this ingestion lock. concurrency: group: coverage-pages queue: max @@ -23,16 +23,12 @@ permissions: read-all jobs: publish: + if: github.ref == 'refs/heads/main' runs-on: ubuntu-latest permissions: actions: read contents: write - id-token: write - pages: write pull-requests: read - environment: - name: github-pages - url: ${{ steps.deployment.outputs.page_url }} steps: # This workflow always comes from the trusted default branch. It consumes the report emitted # by an unprivileged test run; PR code never receives the token that updates Pages. @@ -177,13 +173,3 @@ jobs: git -C site commit -m "coverage: publish reports and refresh PR metadata" git -C site push fi - mkdir -p public - rsync --archive --exclude='.git' site/ public/ - python3 source/tools/site_artwork.py source/docs/assets public - python3 source/tools/compact_site.py public - - uses: actions/upload-pages-artifact@v5 - with: - path: public - - name: Deploy to GitHub Pages - id: deployment - uses: actions/deploy-pages@v5 diff --git a/.github/workflows/pages.yml b/.github/workflows/pages.yml index 8234d59fd6..62069b5b98 100644 --- a/.github/workflows/pages.yml +++ b/.github/workflows/pages.yml @@ -21,7 +21,7 @@ on: permissions: {} -# Share the lock and retained branch with coverage: every deployment includes both. +# Serialize retained-site updates with the other report producers. concurrency: group: coverage-pages queue: max @@ -30,16 +30,11 @@ concurrency: jobs: publish: if: >- - github.event_name != 'workflow_run' || - (github.event.workflow_run.conclusion == 'success' && github.event.workflow_run.event == 'push') + github.ref == 'refs/heads/main' && (github.event_name != 'workflow_run' || + (github.event.workflow_run.conclusion == 'success' && github.event.workflow_run.event == 'push')) runs-on: ubuntu-latest permissions: contents: write - pages: write - id-token: write - environment: - name: github-pages - url: ${{ steps.deployment.outputs.page_url }} steps: - uses: actions/checkout@v7 with: @@ -131,7 +126,7 @@ jobs: --repository "${GITHUB_REPOSITORY}" --tag "${RELEASE_TAG}" --latest "${LATEST_TAG}" \ --generated generated \ "${config_args[@]}" - - name: Retain and stage complete Pages tree + - name: Retain complete Pages tree env: RELEASE_TAG: ${{ steps.release.outputs.tag }} run: | @@ -145,14 +140,3 @@ jobs: git -C site commit -m "site: publish ${RELEASE_TAG}" git -C site push origin HEAD:coverage-pages fi - mkdir -p public - rsync --archive --exclude='.git' site/ public/ - python3 source/tools/site_artwork.py source/docs/assets public - python3 source/tools/compact_site.py public - - uses: actions/configure-pages@v6 - - uses: actions/upload-pages-artifact@v5 - with: - path: public - - name: Deploy complete site - id: deployment - uses: actions/deploy-pages@v5 diff --git a/.github/workflows/pages_deploy.yml b/.github/workflows/pages_deploy.yml new file mode 100644 index 0000000000..0809825774 --- /dev/null +++ b/.github/workflows/pages_deploy.yml @@ -0,0 +1,80 @@ +# SPDX-FileCopyrightText: Copyright (c) M. Boerger, the MBO Works authors +# SPDX-License-Identifier: Apache-2.0 +name: Deploy retained site + +on: + workflow_run: + workflows: [Publish coverage, Publish benchmarks, Publish release site] + types: [completed] + workflow_dispatch: {} + +permissions: {} + +jobs: + deploy: + if: >- + github.ref == 'refs/heads/main' && + (github.event_name == 'workflow_dispatch' || github.event.workflow_run.conclusion == 'success') + # Only deployment notifications coalesce. Producers retain every source report before this + # workflow starts; each deployment reads the newest retained branch at its own checkout. + concurrency: + group: retained-pages-deployment + queue: single + cancel-in-progress: false + runs-on: ubuntu-latest + timeout-minutes: 30 + permissions: + contents: read + actions: read + pages: write + id-token: write + pull-requests: write + environment: + name: github-pages + url: ${{ steps.deployment.outputs.page_url }} + steps: + # Execute trusted default-branch tools, never files from the retained site. + - uses: actions/checkout@v7 + with: + ref: main + path: source + persist-credentials: false + - uses: actions/checkout@v7 + with: + ref: coverage-pages + path: site + persist-credentials: false + - name: Record the immutable deployment snapshot + run: | + snapshot="$(git -C site rev-parse HEAD)" + echo "Retained site snapshot: ${snapshot}" >> "${GITHUB_STEP_SUMMARY}" + - name: Stage complete site + run: | + set -euo pipefail + mkdir -p public + rsync --archive --exclude='.git' site/ public/ + python3 source/tools/site_artwork.py source/docs/assets public + python3 source/tools/compact_site.py public + - uses: actions/configure-pages@v6 + - uses: actions/upload-pages-artifact@v5 + with: + path: public + - name: Deploy immutable snapshot + id: deployment + uses: actions/deploy-pages@v5 + - name: Verify live benchmark publication + if: hashFiles('public/benchmarks/index.html') != '' + env: + PAGE_URL: ${{ steps.deployment.outputs.page_url }} + run: | + python3 source/tools/verify_benchmark_publication.py \ + --index=public/benchmarks/index.html --url="${PAGE_URL}benchmarks/index.html" + - name: Update current PR overviews from the deployed snapshot + if: hashFiles('public/benchmarks/index.html') != '' + env: + GH_TOKEN: ${{ github.token }} + PAGE_URL: ${{ steps.deployment.outputs.page_url }} + run: | + python3 source/tools/site_storage.py expand-benchmarks site/benchmarks/previews + python3 source/tools/benchmark_preview_ci.py comment-retained --repository="${GITHUB_REPOSITORY}" \ + --root=site/benchmarks --site-url="${PAGE_URL}benchmarks/" diff --git a/BUILD.bazel b/BUILD.bazel index 376b29ef2c..dfd7ca47f4 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -85,6 +85,7 @@ filegroup( ".github/workflows/fuzz.yml", ".github/workflows/main.yml", ".github/workflows/pages.yml", + ".github/workflows/pages_deploy.yml", ".github/workflows/release.yml", ], visibility = ["//tools:__pkg__"], diff --git a/TODO.md b/TODO.md index 1ed8887c20..26886713ab 100644 --- a/TODO.md +++ b/TODO.md @@ -163,6 +163,14 @@ scope or declaring new flags implemented. See ## Pages deployment size +- [x] Separate retained-site writers from latest-snapshot deployment; coalesce deployment + notifications only, preserve every source ingestion, and update current PR comments only + after verifying the deployed snapshot. +- [ ] Verify the first separate Pages deployment and current-head PR comments on main. Recover + source events rejected by the full legacy publisher queue without canceling healthy ingestion + or dropping release/main observations. Old queued definitions retain their deployment steps. +- [ ] Reduce full historical rendering inside the writer lock; preserve all raw measurements and + coverage aggregate history when designing coalesced rendering or ingestion replay. - [x] Redirect byte-identical stable coverage HTML to preserved run archives in all publishers; check the complete staged payload before upload and verify the served benchmark index. - [ ] Decide long-term coverage history hosting or retention before immutable reports alone diff --git a/docs/benchmark-backfill.md b/docs/benchmark-backfill.md index 12c9458043..a256444b76 100644 --- a/docs/benchmark-backfill.md +++ b/docs/benchmark-backfill.md @@ -368,7 +368,8 @@ raw reports and the frozen host/build contract under batch fails. Binaries, source checkouts and build logs are not uploaded. There is no automatic upload from the measurement machine. -Publication is restricted to `main` by the Pages environment and uses trusted main code. +Retention and deployment explicitly require `main` and use trusted main code. The separate +deployment worker also uses the Pages environment; ingestion does not wait for Pages deployment. Every deployment fetches the public benchmark index and checks its digest, so a successful Pages response alone cannot mark stale content as successfully published. diff --git a/docs/benchmark-comparisons.md b/docs/benchmark-comparisons.md index 9fefcf8ecd..b1c9965a21 100644 --- a/docs/benchmark-comparisons.md +++ b/docs/benchmark-comparisons.md @@ -73,7 +73,9 @@ selection on that refresh; the original observations remain archived at their im The trusted publisher starts watching when PR CI starts. Once the combined Linux/macOS comparison job passes, it publishes both existing JSON results without waiting for unrelated tests. It does not -collect another set of measurements. Publication still waits for any active Pages deployment. The browser +collect another set of measurements. Retention waits for other site writers, independently of +deployment. A separate deployment worker publishes the newest retained snapshot after any active +deployment finishes. The browser loads public data from the site and needs no GitHub credentials. Preview publication does not mean the rest of CI passed. @@ -81,11 +83,15 @@ The publisher checks the current PR head, repository, run attempt, successful co artifact creation times, and the tested merge commit's parents. Reports preserve both the PR branch head and tested merge commit, the base parent, and artifact identity. Only code checked out from `main` executes with publication permissions. Uploaded HTML is ignored; bounded JSON is -validated and rendered again. Waiting happens outside the shared deployment lock. Preview updates +validated and rendered again. Waiting happens outside the shared writer lock. Preview updates reuse merged pages and their catalog instead of rebuilding every historical chart. Temporary GitHub read failures retry with bounded backoff; comment creation and updates are never blindly retried. +Deployment notifications may coalesce across several PRs. After verifying the deployed benchmark +index, the deployment worker reconciles comments for every current PR represented by its snapshot, +not just the notification that started it. It never comments from newer, undeployed retained data. + The compact **Performance overview** appears before the detailed tables, in the selected chart view, and in one bot comment updated on the PR after successful publication. It gives: diff --git a/docs/benchmark-history.md b/docs/benchmark-history.md index d3cca110d2..47a7c33ff5 100644 --- a/docs/benchmark-history.md +++ b/docs/benchmark-history.md @@ -49,8 +49,11 @@ as JSON so summaries can be reproduced. The publisher executes trusted main-branch code and treats measurement artifacts as JSON data. It checks the measured head against the source workflow SHA and derives run identity from GitHub. -It shares the `coverage-pages` deployment queue and retained branch with coverage and release -publication, preserving their pages. +It shares the `coverage-pages` writer queue and retained branch with coverage and release +publication, preserving their pages and every incoming measurement. After retention succeeds, +`Deploy retained site` reads and deploys the latest immutable branch snapshot. Only deployment +notifications coalesce; source-artifact ingestion remains serialized without cancellation. A Pages +timeout no longer holds the writer lock. Rendering remains inside the writer queue. Local uploads can advance the retained branch while a publisher is rendering it. A rejected push fetches and rebases onto that update, then performs a full benchmark refresh before retrying, with diff --git a/docs/release-website.md b/docs/release-website.md index 0704bd05c9..59c3624e9f 100644 --- a/docs/release-website.md +++ b/docs/release-website.md @@ -55,12 +55,30 @@ copied, including remote badges. Markdown conversion uses the browsing the result requires no Markdown renderer or CDN. After the Release workflow succeeds, `Publish release site` retains the snapshot on -`coverage-pages` and deploys the complete Pages tree. Coverage and site publication share a -concurrency group to preserve both trees. GitHub's latest stable release selects the root redirect; +`coverage-pages`. Coverage, benchmark, and release producers share a writer queue to preserve every +incoming report. A separate `Deploy retained site` workflow deploys the latest complete branch +snapshot after a producer succeeds. GitHub's latest stable release selects the root redirect; backfilling an older release does not make it latest. The workflow can also be dispatched with a published tag to retry publication. Enable GitHub Pages with **GitHub Actions** as its source, and set the repository's About website to `https://mboworks.github.io/xff/`. +Deployment notifications coalesce; report ingestion does not. An active deployment completes, and +one pending deployment reads the newest retained snapshot when it starts. The deployment never +updates the retained branch. Producers and deployment explicitly require `main`; removing the Pages +environment from producers does not relax that branch restriction. A Pages timeout therefore cannot +hold the writer queue or discard source measurements. The deployment verifies the served benchmark index before updating PR +comments from that same snapshot, rechecking each PR's current head and workflow attempt. +To retry deployment of already retained data without downloading or regenerating reports: + +```sh +gh workflow run pages_deploy.yml --repo mboworks/xff --ref main +``` + +This separation does not recover source events previously rejected by GitHub's full writer queue. +Those require a separate ingestion replay. Runs already queued with an older workflow definition +retain their original steps, including deployment; allow for that overlap during rollout. Rendering +still occurs inside each producer's writer lock and can remain the ingestion bottleneck. + The release site renders `NOTICE.md` as `NOTICE.html`, with the same navigation as other pages. The Markdown notice is generated from the all-extras binary via `--help=notice --help-format=markdown`. `LICENSE` links remain pinned to the tagged commit on GitHub, where readers can use GitHub's license analysis. @@ -145,7 +163,7 @@ The README uses the original transparent MBO Works logo in The favicon assets are derived from that same artwork: a 32-pixel PNG, a 180-pixel Apple Touch icon, and an ICO containing 16-, 32-, 48-, and 256-pixel PNG images. -All three Pages publishers (release, coverage, and benchmarks) run `tools/site_artwork.py` on the final deployment directory. +`Deploy retained site` runs `tools/site_artwork.py` on the final deployment directory. This supplies favicons to documentation, native HTML references, redirects, coverage reports, and benchmark reports, including existing releases. Relative links work at every nesting depth and under xff's project URL. Only the deployment copy receives these links; retained release files and diff --git a/tools/benchmark_history_test.py b/tools/benchmark_history_test.py index e1e3c2ba36..3c053f8afc 100644 --- a/tools/benchmark_history_test.py +++ b/tools/benchmark_history_test.py @@ -208,7 +208,7 @@ def test_workflow_keeps_measurement_unprivileged_and_publication_serialized(self self.assertIn("ref: main\n persist-credentials: false", waiter) self.assertNotIn(": write", waiter) self.assertIn("ref: main\n path: source", publish) - self.assertIn("Verify live benchmark publication", publish) + self.assertNotIn("actions/deploy-pages@", publish) self.assertIn('python3 source/tools/publish_site.py --repo=site', publish) self.assertIn("bash -c 'set -euo pipefail; refresh_site retry'", publish) self.assertIn('[ "${PREVIEW}" = \'true\' ] && [ "$1" = \'initial\' ]', publish) @@ -596,7 +596,7 @@ def test_release_refresh_does_not_download_measurement_artifacts(self): self.assertIn("workflow_dispatch: {}", publish) self.assertIn("fetch-depth: 0", publish) self.assertIn("uses: actions/download-artifact@v8\n if: github.event.workflow_run.name == 'Benchmarks'", publish) - refresh = publish.split(" - name: Refresh stable benchmark references and deploy", 1)[1].split( + refresh = publish.split(" - name: Refresh and retain stable benchmark references", 1)[1].split( " - uses: actions/upload-pages-artifact", 1)[0] self.assertNotIn("SOURCE_RUN", refresh) self.assertNotIn("download-artifact", refresh) diff --git a/tools/benchmark_preview_ci.py b/tools/benchmark_preview_ci.py index 16118688aa..c217bef8bf 100644 --- a/tools/benchmark_preview_ci.py +++ b/tools/benchmark_preview_ci.py @@ -201,19 +201,36 @@ def update_comment(api, run, attempt, root, site_url): return True +def update_retained_comments(api, root, site_url): + """Reconcile every current PR represented by this immutable deployed snapshot.""" + pulls = api.pages('pulls?state=open') + identities = set() + for path in benchmark_preview.selected(root, pulls): + source = json.loads(path.read_text())['source'] + identities.add((benchmark_preview.positive_id(source['id']), + benchmark_preview.positive_id(source['run_attempt']))) + return sum(update_comment(api, run, attempt, root, site_url) + for run, attempt in sorted(identities)) + + def main(): parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument('operation', choices=('wait', 'collect', 'comment')) + parser.add_argument('operation', choices=('wait', 'collect', 'comment', 'comment-retained')) parser.add_argument('--repository', required=True) - parser.add_argument('--run', type=int, required=True) - parser.add_argument('--attempt', type=int, required=True) + parser.add_argument('--run', type=int) + parser.add_argument('--attempt', type=int) parser.add_argument('--root', type=Path) parser.add_argument('--output', type=Path) parser.add_argument('--site-url') args = parser.parse_args() api = GitHub(args.repository) - benchmark_preview.positive_id(args.run) - benchmark_preview.positive_id(args.attempt) + if args.operation != 'comment-retained': + if args.run is None or args.attempt is None: + parser.error('--run and --attempt are required') + benchmark_preview.positive_id(args.run) + benchmark_preview.positive_id(args.attempt) + elif args.run is not None or args.attempt is not None: + parser.error('comment-retained selects source runs from the deployed snapshot') if args.operation == 'wait': available = wait(api, args.run, args.attempt) elif args.root is None: @@ -222,6 +239,8 @@ def main(): available = bool(collect(api, args.run, args.attempt, args.root)) elif not args.site_url: parser.error('--site-url is required') + elif args.operation == 'comment-retained': + available = bool(update_retained_comments(api, args.root, args.site_url)) else: available = update_comment(api, args.run, args.attempt, args.root, args.site_url) if args.output: diff --git a/tools/benchmark_preview_ci_test.py b/tools/benchmark_preview_ci_test.py index 213df71455..583dacb1a0 100644 --- a/tools/benchmark_preview_ci_test.py +++ b/tools/benchmark_preview_ci_test.py @@ -6,6 +6,7 @@ import json from pathlib import Path import subprocess +import sys import tempfile import unittest from unittest import mock @@ -39,6 +40,62 @@ def artifacts(): class BenchmarkPreviewCiTest(unittest.TestCase): + def test_coalesced_deployment_comments_on_every_current_pr_once(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + pulls = [pull(), dict(pull(), number=11)] + sources = {10: source(), 12: dict(source(), id=12, pull_requests=[{'number': 11}])} + for number, run in ((9, 10), (11, 12)): + for platform in ('linux', 'macos'): + folder = root / f'previews/{number}/{run}/2/{platform}' + folder.mkdir(parents=True) + record = {'kind': 'pr-preview', 'pull_number': number, 'platform': platform, + 'head': 'c' * 40, 'branch_head': 'b' * 40, 'base': 'a' * 40, + 'source': sources[run], + 'tool_comparisons': {'tasks': [{'shape': 'legacy'}]}} + (folder / 'report.json').write_text(json.dumps(record)) + api = mock.Mock(repository='owner/repo') + api.pages.side_effect = lambda path: pulls if path == 'pulls?state=open' else [] + + def request(path, *args): + if path.startswith('actions/runs/'): + return sources[int(path.rsplit('/', 1)[1])] + if path.startswith('pulls/'): + return next(pr for pr in pulls if pr['number'] == int(path.rsplit('/', 1)[1])) + return {} + + api.request.side_effect = request + url = 'https://example.org/project/benchmarks/' + self.assertEqual(ci.update_retained_comments(api, root, url), 2) + mutations = [call for call in api.request.call_args_list if len(call.args) > 1] + self.assertEqual([call.args[0] for call in mutations], ['issues/9/comments', 'issues/11/comments']) + for call, (number, run) in zip(mutations, ((9, 10), (11, 12))): + self.assertIn(f'{url}previews/{number}/{run}/2/linux/', call.args[1]['body']) + # A newer current attempt or a closed PR must never receive this snapshot's comment. + sources[10]['run_attempt'] = 3 + pulls[1]['state'] = 'closed' + api.request.reset_mock() + self.assertEqual(ci.update_retained_comments(api, root, url), 0) + self.assertFalse(any(len(call.args) > 1 for call in api.request.call_args_list)) + + def test_source_operations_still_require_both_run_and_attempt(self): + for operation in ('wait', 'collect', 'comment'): + for arguments in ([], ['--run=10'], ['--attempt=2']): + with self.subTest(operation=operation, arguments=arguments), \ + mock.patch.object(sys, 'argv', ['preview', operation, '--repository=owner/repo', *arguments]), \ + mock.patch.object(sys, 'stderr', io.StringIO()), self.assertRaises(SystemExit) as error: + ci.main() + self.assertEqual(error.exception.code, 2) + + def test_retained_comment_command_uses_snapshot_without_a_triggering_source_run(self): + with tempfile.TemporaryDirectory() as directory, \ + mock.patch.object(sys, 'argv', ['preview', 'comment-retained', '--repository=owner/repo', + '--root=' + directory, '--site-url=https://example.org/']), \ + mock.patch.object(ci, 'GitHub') as github, \ + mock.patch.object(ci, 'update_retained_comments', return_value=2) as update: + ci.main() + update.assert_called_once_with(github.return_value, Path(directory), 'https://example.org/') + def test_collect_validates_tested_merge_and_retains_exact_json_from_each_platform(self): api = mock.Mock(repository='owner/repo') sample = dict.fromkeys(ci.benchmark_preview.benchmark_compare.METRICS, 1) diff --git a/tools/compact_site_test.py b/tools/compact_site_test.py index 230abda33c..7f2bb7047a 100644 --- a/tools/compact_site_test.py +++ b/tools/compact_site_test.py @@ -24,13 +24,13 @@ def test_counts_entire_payload_excluding_git_database(self): self.assertEqual(report['sections']['site']['bytes'], 20) self.assertTrue(root.joinpath('site-storage.js').exists()) - def test_each_publisher_compacts_retained_branch_and_complete_deployment(self): + def test_each_producer_compacts_retained_branch_before_commit(self): for name in ('coverage_pages.yml', 'benchmark_pages.yml', 'pages.yml'): with self.subTest(workflow=name): source = repository_file('.github/workflows/' + name).read_text() self.assertLess(source.index('compact_site.py site'), source.index('git -C site commit')) - self.assertLess(source.index('site_artwork.py'), source.index('compact_site.py public')) - self.assertLess(source.index('compact_site.py public'), source.index('uses: actions/upload-pages-artifact@')) + self.assertNotIn('actions/deploy-pages@', source) + self.assertNotIn('actions/upload-pages-artifact@', source) self.assertIn('git -C site add --all', source) if name == 'pages.yml': self.assertIn('git fetch --no-tags --depth=1 origin', source) @@ -38,6 +38,11 @@ def test_each_publisher_compacts_retained_branch_and_complete_deployment(self): else: self.assertIn('fetch-depth: 0\n filter: blob:none', source) + def test_deployment_compacts_complete_snapshot_before_upload(self): + source = repository_file('.github/workflows/pages_deploy.yml').read_text() + self.assertLess(source.index('site_artwork.py'), source.index('compact_site.py public')) + self.assertLess(source.index('compact_site.py public'), source.index('uses: actions/upload-pages-artifact@')) + if __name__ == '__main__': unittest.main() diff --git a/tools/coverage_index_test.py b/tools/coverage_index_test.py index 8e53814766..59ee42bacc 100644 --- a/tools/coverage_index_test.py +++ b/tools/coverage_index_test.py @@ -27,6 +27,33 @@ def _summary(percent: float) -> dict: class CoverageIndexTest(unittest.TestCase): + def test_deployment_coalesces_notifications_without_discarding_source_reports(self): + deployment = repository_file('.github/workflows/pages_deploy.yml').read_text() + self.assertIn('workflows: [Publish coverage, Publish benchmarks, Publish release site]', deployment) + self.assertIn('types: [completed]', deployment) + self.assertIn("github.event.workflow_run.conclusion == 'success'", deployment) + self.assertIn("github.ref == 'refs/heads/main'", deployment) + self.assertIn(' concurrency:\n group: retained-pages-deployment\n queue: single\n' + ' cancel-in-progress: false', deployment) + self.assertIn('ref: coverage-pages\n path: site\n persist-credentials: false', deployment) + for forbidden in ('contents: write', 'git -C site push', 'publish_site.py', 'download-artifact@'): + self.assertNotIn(forbidden, deployment) + self.assertLess(deployment.index('actions/deploy-pages@'), deployment.index('Verify live benchmark publication')) + self.assertLess(deployment.index('Verify live benchmark publication'), deployment.index('comment-retained')) + self.assertIn('--index=public/benchmarks/index.html', deployment) + self.assertIn('--root=site/benchmarks', deployment) + self.assertNotIn('SOURCE_RUN', deployment) + for name in ('coverage_pages.yml', 'benchmark_pages.yml', 'pages.yml'): + producer = repository_file('.github/workflows/' + name).read_text() + with self.subTest(producer=name): + self.assertIn('queue: max', producer) + self.assertIn("github.ref == 'refs/heads/main'", producer) + self.assertNotIn('queue: single', producer) + self.assertNotIn('actions/deploy-pages@', producer) + self.assertNotIn('pages: write', producer) + self.assertNotIn('id-token: write', producer) + self.assertNotIn('pull-requests: write', producer) + def test_metadata_refresh_shares_the_report_publication_queue(self): workflow = repository_file(".github/workflows/coverage_pages.yml").read_text() pages = repository_file(".github/workflows/pages.yml").read_text() @@ -136,7 +163,8 @@ def test_merge_refresh_reorders_existing_report_without_replacing_measurements(s repository.mkdir() subprocess.run(["git", "init", "-q", str(repository)], check=True) subprocess.run(["git", "-C", str(repository), "-c", "user.name=Test", - "-c", "user.email=test@example.com", "commit", "--allow-empty", "-qm", "merge"], check=True) + "-c", "user.email=test@example.com", "-c", "commit.gpgsign=false", + "commit", "--allow-empty", "-qm", "merge"], check=True) sha = subprocess.check_output(["git", "-C", str(repository), "rev-parse", "HEAD"], text=True).strip() reports = root / "reports" original = {} diff --git a/tools/site_artwork_test.py b/tools/site_artwork_test.py index 82e9eeec98..fafc1ee703 100644 --- a/tools/site_artwork_test.py +++ b/tools/site_artwork_test.py @@ -11,15 +11,13 @@ class SiteArtworkTest(unittest.TestCase): - def test_every_pages_publisher_decorates_staged_site_before_upload(self): - for name in ("pages.yml", "coverage_pages.yml", "benchmark_pages.yml"): - with self.subTest(workflow=name): - workflow = repository_file(f".github/workflows/{name}").read_text() - stage = workflow.index("rsync --archive --exclude='.git' site/ public/") - decorate = workflow.index("python3 source/tools/site_artwork.py source/docs/assets public") - upload = workflow.index("uses: actions/upload-pages-artifact@") - self.assertLess(stage, decorate) - self.assertLess(decorate, upload) + def test_deployment_decorates_complete_snapshot_before_upload(self): + workflow = repository_file(".github/workflows/pages_deploy.yml").read_text() + stage = workflow.index("rsync --archive --exclude='.git' site/ public/") + decorate = workflow.index("python3 source/tools/site_artwork.py source/docs/assets public") + upload = workflow.index("uses: actions/upload-pages-artifact@") + self.assertLess(stage, decorate) + self.assertLess(decorate, upload) def test_nested_pages_and_idempotence(self): with tempfile.TemporaryDirectory() as temporary: