Add WordPress integration regressions and automated release checks

.github/workflows/deploy-wordpress.yml

@@ -25,8 +25,15 @@ on:
     tags:
       - 'wp-*' # Push events to matching wp-*, i.e. wp-2.0.8
 
+permissions:
+  contents: read
+
 jobs:
+  validation:
+    uses: ./.github/workflows/release-checks.yml
+
   deploy:
+    needs: validation
     runs-on: ubuntu-latest
     steps:
       - name: Checkout code

.github/workflows/integration-regressions.yml

@@ -0,0 +1,84 @@
+name: WordPress integration regressions
+
+on:
+  workflow_dispatch:
+    inputs:
+      plugin_ref:
+        description: 'Plugin branch, tag or commit under test; blank uses this workflow commit'
+        required: false
+        type: string
+      wordpress_version:
+        description: 'WordPress version (default: 6.9.4)'
+        default: '6.9.4'
+        type: string
+      php_version:
+        description: 'PHP version supported by Playground'
+        default: '8.3'
+        type: string
+
+permissions:
+  contents: read
+
+jobs:
+  regressions:
+    runs-on: ubuntu-latest
+    timeout-minutes: 25
+    steps:
+      - name: Explain test coverage
+        if: ${{ always() }}
+        run: |
+          cat >> "$GITHUB_STEP_SUMMARY" <<'SUMMARY'
+          ### WordPress integration regressions
+          - Environment: Chromium and Firefox; disposable WordPress multisite; Siteimprove mocked.
+          - Coverage: Regression catalog, browser contracts, roles, multisite, URL mapping, and settings.
+          - Report: integration-regressions-test-report
+          - Results: see the test steps below; this description does not indicate a pass.
+          SUMMARY
+      - uses: actions/checkout@v6
+        with:
+          persist-credentials: false
+      - uses: actions/checkout@v6
+        with:
+          ref: ${{ inputs.plugin_ref || github.sha }}
+          path: .plugin-under-test
+          sparse-checkout: siteimprove
+          persist-credentials: false
+      - name: Record the plugin commit under test
+        run: git -C .plugin-under-test rev-parse HEAD
+      - uses: actions/setup-node@v6
+        with:
+          node-version: '24'
+          cache: npm
+      - run: npm ci
+      - run: npx playwright install --with-deps chromium firefox
+      - name: Validate regression coverage catalog
+        run: npm run test:regressions:catalog
+      - name: Test content capture, rechecks, and highlighting handoff
+        run: npm test
+        env:
+          SITEIMPROVE_TEST_SCRIPT: ${{ github.workspace }}/.plugin-under-test/siteimprove/admin/js/siteimprove.js
+      - name: Allow Playground multisite to use the standard HTTP port
+        if: ${{ !cancelled() }}
+        run: sudo sysctl -w net.ipv4.ip_unprivileged_port_start=0
+      - run: npm run env:regressions:start
+        id: regression_environment
+        if: ${{ !cancelled() }}
+        env:
+          SITEIMPROVE_TEST_PLUGIN: .plugin-under-test/siteimprove
+          REGRESSION_WP_VERSION: ${{ inputs.wordpress_version }}
+          REGRESSION_PHP_VERSION: ${{ inputs.php_version }}
+      - name: Test roles, multisite, URL mapping, and settings
+        run: npm run test:regressions
+        if: ${{ !cancelled() && steps.regression_environment.outcome == 'success' }}
+      - uses: actions/upload-artifact@v4
+        if: ${{ !cancelled() }}
+        with:
+          name: integration-regressions-test-report
+          path: |
+            playwright-report/
+            playwright-integration-regressions-report/
+            test-results/integration-regressions/
+            test-results/integration-regressions-results.json
+          retention-days: 14
+      - run: npm run env:regressions:stop
+        if: ${{ always() }}

.github/workflows/pr-tests.yml

@@ -0,0 +1,113 @@
+name: WordPress plugin PR tests
+
+on:
+  workflow_call:
+  pull_request:
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+concurrency:
+  group: wordpress-pr-tests-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
+
+jobs:
+  browser:
+    name: Browser tests
+    runs-on: ubuntu-latest
+    timeout-minutes: 15
+    steps:
+      - uses: actions/checkout@v6
+        with:
+          persist-credentials: false
+      - name: Record the tested commit
+        run: |
+          printf 'Tested repository commit: `%s`\n' "$(git rev-parse HEAD)" >> "$GITHUB_STEP_SUMMARY"
+      - uses: actions/setup-node@v6
+        with:
+          node-version: '24'
+          cache: npm
+      - run: npm ci
+      - run: npx playwright install --with-deps chromium firefox
+      - name: Validate regression coverage catalog
+        run: npm run test:regressions:catalog
+      - name: Test live runner contracts and privacy safeguards
+        run: npm run test:live:contracts
+      - name: Test content capture, rechecks, and highlighting handoff
+        run: npm test
+      - name: Upload browser report
+        if: ${{ !cancelled() }}
+        uses: actions/upload-artifact@v4
+        with:
+          name: pr-browser-test-report
+          path: playwright-report/
+          retention-days: 14
+      - name: Explain coverage and reports
+        if: ${{ always() }}
+        run: |
+          cat >> "$GITHUB_STEP_SUMMARY" <<'SUMMARY'
+          ### Browser tests
+          Chromium and Firefox test content capture, recheck callbacks, and highlighting handoff on local fixtures. Catalog and live-runner privacy contracts are also checked. Siteimprove is mocked; no account credentials are used.
+
+          See the step results for outcomes and `pr-browser-test-report` in Artifacts for individual browser cases, when available.
+          SUMMARY
+
+  wordpress:
+    name: WordPress tests
+    runs-on: ubuntu-latest
+    timeout-minutes: 35
+    steps:
+      - uses: actions/checkout@v6
+        with:
+          persist-credentials: false
+      - name: Record the tested commit
+        run: |
+          printf 'Tested repository commit: `%s`\n' "$(git rev-parse HEAD)" >> "$GITHUB_STEP_SUMMARY"
+      - uses: actions/setup-node@v6
+        with:
+          node-version: '24'
+          cache: npm
+      - run: npm ci
+      - run: npx playwright install --with-deps chromium firefox
+      - name: Start disposable WordPress
+        id: wordpress_environment
+        run: npm run env:start
+      - name: Test drafts, previews, autosaves, and content capture
+        if: ${{ !cancelled() && steps.wordpress_environment.outcome == 'success' }}
+        run: npm run test:wordpress
+      - name: Stop disposable WordPress
+        if: ${{ always() }}
+        run: npm run env:stop
+      - name: Allow multisite to use the standard HTTP port
+        if: ${{ !cancelled() }}
+        run: sudo sysctl -w net.ipv4.ip_unprivileged_port_start=0
+      - name: Start disposable WordPress multisite
+        id: regression_environment
+        if: ${{ !cancelled() }}
+        run: npm run env:regressions:start
+      - name: Test roles, multisite, URL mapping, and settings
+        if: ${{ !cancelled() && steps.regression_environment.outcome == 'success' }}
+        run: npm run test:regressions
+      - name: Stop disposable WordPress multisite
+        if: ${{ always() }}
+        run: npm run env:regressions:stop
+      - name: Upload WordPress reports
+        if: ${{ !cancelled() }}
+        uses: actions/upload-artifact@v4
+        with:
+          name: pr-wordpress-test-reports
+          path: |
+            playwright-wordpress-report/
+            playwright-integration-regressions-report/
+            test-results/integration-regressions-results.json
+          retention-days: 14
+      - name: Explain coverage and reports
+        if: ${{ always() }}
+        run: |
+          cat >> "$GITHUB_STEP_SUMMARY" <<'SUMMARY'
+          ### WordPress tests
+          Chromium and Firefox test drafts, previews, autosaves, content capture, roles, multisite, URL mapping, and settings. Single-site and multisite fixtures run sequentially with one dependency/browser installation. Siteimprove is mocked; no account credentials are used.
+
+          See the step results for outcomes and `pr-wordpress-test-reports` in Artifacts for individual cases, when available. Failed steps remain failures even if later suites pass.
+          SUMMARY

.github/workflows/prepublish-live.yml

@@ -1,6 +1,7 @@
-name: Prepublish live Siteimprove test
+name: WordPress–Siteimprove live integration tests
 
 on:
+  workflow_call:
   workflow_dispatch:
 
 permissions:
@@ -13,18 +14,29 @@ concurrency:
 jobs:
   live:
     # Reviewed workflow code only. Environment approval is required separately.
-    if: github.repository == 'Siteimprove/CMS-plugin-Wordpress' && github.ref == 'refs/heads/master'
+    if: github.repository == 'Siteimprove/CMS-plugin-Wordpress' && github.event_name != 'pull_request'
     environment: siteimprove-test
     runs-on: ubuntu-latest
     timeout-minutes: 20
     steps:
+      - name: Explain test coverage
+        if: ${{ always() }}
+        run: |
+          cat >> "$GITHUB_STEP_SUMMARY" <<'SUMMARY'
+          ### WordPress–Siteimprove live integration tests
+          - Environment: Chromium; local WordPress connected to the real Siteimprove platform.
+          - Coverage: Configuration, entitlement, login, Live page data, fresh draft handoff, and loading-state exit.
+          - Skipped: Missing-title result assertion; scan-result correctness remains unverified.
+          - Report: No artifacts are uploaded for this authenticated suite.
+          - Results: see the test steps below; this description does not indicate a pass.
+          SUMMARY
       - uses: actions/checkout@v6
         with:
           persist-credentials: false
-      - name: Check out the reviewed PR 64 plugin
+      - name: Check out the plugin commit under test
         uses: actions/checkout@v6
         with:
-          ref: a23af8519861f674d700cbe4fe183817f47458dc
+          ref: ${{ github.sha }}
           path: .plugin-under-test
           sparse-checkout: siteimprove
           persist-credentials: false
@@ -33,11 +45,12 @@ jobs:
           node-version: '24'
           cache: npm
       - run: npm ci
-      - run: npm run test:live:contracts
+      - name: Test live runner contracts and privacy safeguards
+        run: npm run test:live:contracts
       - run: npx playwright install --with-deps chromium
       - run: node scripts/prepare-live-env.js
       - run: npm run env:start
-      - name: Verify Live page data and one fresh Prepublish issue
+      - name: Verify Live page data, draft handoff, and loading-state exit
         run: npm run test:live
         env:
           SITEIMPROVE_RUN_LIVE: '1'

.github/workflows/prepublish-test.yml

@@ -1,4 +1,4 @@
-name: Prepublish test
+name: WordPress plugin browser tests
 
 on:
   workflow_dispatch:
@@ -16,6 +16,16 @@ jobs:
     runs-on: ubuntu-latest
     timeout-minutes: 15
     steps:
+      - name: Explain test coverage
+        if: ${{ always() }}
+        run: |
+          cat >> "$GITHUB_STEP_SUMMARY" <<'SUMMARY'
+          ### WordPress plugin browser tests
+          - Environment: Chromium and Firefox; local browser fixtures; Siteimprove mocked.
+          - Coverage: Content capture, recheck callbacks, and highlighting handoff.
+          - Report: plugin-browser-test-report
+          - Results: see the test steps below; this description does not indicate a pass.
+          SUMMARY
       - uses: actions/checkout@v6
         with:
           persist-credentials: false
@@ -34,13 +44,14 @@ jobs:
           cache: npm
       - run: npm ci
       - run: npx playwright install --with-deps chromium firefox
-      - run: npm test
+      - name: Test content capture, rechecks, and highlighting handoff
+        run: npm test
         env:
           SITEIMPROVE_TEST_SCRIPT: ${{ github.workspace }}/.plugin-under-test/siteimprove/admin/js/siteimprove.js
       - uses: actions/upload-artifact@v4
         if: ${{ !cancelled() }}
         with:
-          name: prepublish-test-report
+          name: plugin-browser-test-report
           path: |
             playwright-report/
             test-results/browser/

.github/workflows/prepublish-wordpress.yml

@@ -1,4 +1,4 @@
-name: Prepublish WordPress environment
+name: WordPress plugin environment tests
 
 on:
   workflow_dispatch:
@@ -16,6 +16,16 @@ jobs:
     runs-on: ubuntu-latest
     timeout-minutes: 20
     steps:
+      - name: Explain test coverage
+        if: ${{ always() }}
+        run: |
+          cat >> "$GITHUB_STEP_SUMMARY" <<'SUMMARY'
+          ### WordPress plugin environment tests
+          - Environment: Chromium and Firefox; disposable local WordPress; Siteimprove mocked.
+          - Coverage: Drafts, previews, autosaves, nonces, and content capture.
+          - Report: wordpress-env-test-report
+          - Results: see the test steps below; this description does not indicate a pass.
+          SUMMARY
       - uses: actions/checkout@v6
         with:
           persist-credentials: false
@@ -37,11 +47,12 @@ jobs:
       - name: Point WordPress at the selected plugin
         run: node -e "require('node:fs').writeFileSync('.wp-env.override.json', JSON.stringify({plugins:['./.plugin-under-test/siteimprove']}))"
       - run: npm run env:start
-      - run: npm run test:wordpress
+      - name: Test WordPress drafts, previews, and content capture
+        run: npm run test:wordpress
       - uses: actions/upload-artifact@v4
         if: ${{ !cancelled() }}
         with:
-          name: prepublish-wordpress-report
+          name: wordpress-env-test-report
           path: |
             playwright-wordpress-report/
             test-results/wordpress/

.github/workflows/release-checks.yml

@@ -0,0 +1,49 @@
+name: Release checks
+
+on:
+  workflow_call:
+
+permissions:
+  contents: read
+
+jobs:
+  integration:
+    uses: ./.github/workflows/pr-tests.yml
+  coding-standard:
+    uses: ./.github/workflows/wpcs.yml
+  live:
+    needs: [integration, coding-standard]
+    uses: ./.github/workflows/prepublish-live.yml
+  package:
+    runs-on: ubuntu-latest
+    timeout-minutes: 15
+    steps:
+      - uses: actions/checkout@v6
+        with:
+          persist-credentials: false
+      - name: Build and test the release ZIP lifecycle
+        run: bash scripts/test-release-package.sh
+      - name: Retain the validated release ZIP
+        uses: actions/upload-artifact@v4
+        with:
+          name: validated-release-package
+          path: siteimprove.zip
+          retention-days: 14
+  passed:
+    name: All release checks passed
+    if: ${{ always() }}
+    needs: [integration, coding-standard, live, package]
+    runs-on: ubuntu-latest
+    steps:
+      - name: Require every check to succeed
+        env:
+          INTEGRATION: ${{ needs.integration.result }}
+          STANDARD: ${{ needs.coding-standard.result }}
+          LIVE: ${{ needs.live.result }}
+          PACKAGE: ${{ needs.package.result }}
+        run: |
+          test "$INTEGRATION" = success
+          test "$STANDARD" = success
+          test "$LIVE" = success
+          test "$PACKAGE" = success
+          printf '%s\n' 'All automated release checks passed for this commit.' >> "$GITHUB_STEP_SUMMARY"