From 744806f103b705f1155e8eef3e8b0746df788e47 Mon Sep 17 00:00:00 2001 From: MortenFriisSiteImprove Date: Tue, 15 Sep 2026 09:24:47 +0200 Subject: [PATCH] Consolidate test and release workflows and document their use --- .github/workflows/deploy-wordpress.yml | 52 ---- .github/workflows/integration-regressions.yml | 84 ------ .github/workflows/pr-tests.yml | 50 +++- .github/workflows/prepublish-live.yml | 2 +- .github/workflows/prepublish-test.yml | 58 ---- .github/workflows/prepublish-wordpress.yml | 61 ---- .github/workflows/release.yml | 66 +++- .github/workflows/wpcs.yml | 2 +- README.md | 4 + TESTING.md | 283 ++---------------- docs/workflows.md | 96 ++++++ tests/SCENARIOS.md | 2 +- tests/integration-regressions/README.md | 2 +- tests/live/README.md | 2 +- 14 files changed, 231 insertions(+), 533 deletions(-) delete mode 100644 .github/workflows/deploy-wordpress.yml delete mode 100644 .github/workflows/integration-regressions.yml delete mode 100644 .github/workflows/prepublish-test.yml delete mode 100644 .github/workflows/prepublish-wordpress.yml create mode 100644 docs/workflows.md diff --git a/.github/workflows/deploy-wordpress.yml b/.github/workflows/deploy-wordpress.yml deleted file mode 100644 index 064f974..0000000 --- a/.github/workflows/deploy-wordpress.yml +++ /dev/null @@ -1,52 +0,0 @@ -name: Deploy to WordPress Marketplace -on: - workflow_dispatch: - inputs: - version: - description: 'Plugin version to deploy' - required: true - default: '' - dry-run: - description: 'Perform dry run (no actual deployment)' - required: false - default: true - type: boolean - test-mode: - description: 'Run in test mode' - required: false - default: false - type: boolean - push: - branches: - - main - paths: - - 'siteimprove/**' - - '.github/workflows/deploy-wordpress.yml' - tags: - - 'wp-*' # Push events to matching wp-*, i.e. wp-2.0.8 - -permissions: - contents: read - -jobs: - validation: - uses: ./.github/workflows/release-checks.yml - - deploy: - needs: validation - runs-on: ubuntu-latest - steps: - - name: Checkout code - uses: actions/checkout@v4 - - - name: Deploy to WordPress Marketplace - uses: ./.github/actions/deploy-to-wordpress - with: - version: ${{ github.event.inputs.version || github.ref_name }} - plugin-slug: ${{ vars.WP_PLUGIN_SLUG || 'siteimprove' }} - svn-username: ${{ github.event.inputs.test-mode && secrets.TEST_SVN_USERNAME || secrets.WP_SVN_USERNAME }} - svn-password: ${{ github.event.inputs.test-mode && secrets.TEST_SVN_PASSWORD || secrets.WP_SVN_PASSWORD }} - svn-url: ${{ vars.WP_SVN_URL || 'https://plugins.svn.wordpress.org/siteimprove/' }} - assets-dir: ${{ vars.WP_ASSETS_DIR || 'wordpress-assets' }} - dry-run: ${{ github.event.inputs.dry-run || false }} - test-mode: ${{ github.event.inputs.test-mode || false }} diff --git a/.github/workflows/integration-regressions.yml b/.github/workflows/integration-regressions.yml deleted file mode 100644 index a6abd4a..0000000 --- a/.github/workflows/integration-regressions.yml +++ /dev/null @@ -1,84 +0,0 @@ -name: WordPress integration regressions - -on: - workflow_dispatch: - inputs: - plugin_ref: - description: 'Plugin branch, tag or commit under test; blank uses this workflow commit' - required: false - type: string - wordpress_version: - description: 'WordPress version (default: 6.9.4)' - default: '6.9.4' - type: string - php_version: - description: 'PHP version supported by Playground' - default: '8.3' - type: string - -permissions: - contents: read - -jobs: - regressions: - runs-on: ubuntu-latest - timeout-minutes: 25 - steps: - - name: Explain test coverage - if: ${{ always() }} - run: | - cat >> "$GITHUB_STEP_SUMMARY" <<'SUMMARY' - ### WordPress integration regressions - - Environment: Chromium and Firefox; disposable WordPress multisite; Siteimprove mocked. - - Coverage: Regression catalog, browser contracts, roles, multisite, URL mapping, and settings. - - Report: integration-regressions-test-report - - Results: see the test steps below; this description does not indicate a pass. - SUMMARY - - uses: actions/checkout@v6 - with: - persist-credentials: false - - uses: actions/checkout@v6 - with: - ref: ${{ inputs.plugin_ref || github.sha }} - path: .plugin-under-test - sparse-checkout: siteimprove - persist-credentials: false - - name: Record the plugin commit under test - run: git -C .plugin-under-test rev-parse HEAD - - uses: actions/setup-node@v6 - with: - node-version: '24' - cache: npm - - run: npm ci - - run: npx playwright install --with-deps chromium firefox - - name: Validate regression coverage catalog - run: npm run test:regressions:catalog - - name: Test content capture, rechecks, and highlighting handoff - run: npm test - env: - SITEIMPROVE_TEST_SCRIPT: ${{ github.workspace }}/.plugin-under-test/siteimprove/admin/js/siteimprove.js - - name: Allow Playground multisite to use the standard HTTP port - if: ${{ !cancelled() }} - run: sudo sysctl -w net.ipv4.ip_unprivileged_port_start=0 - - run: npm run env:regressions:start - id: regression_environment - if: ${{ !cancelled() }} - env: - SITEIMPROVE_TEST_PLUGIN: .plugin-under-test/siteimprove - REGRESSION_WP_VERSION: ${{ inputs.wordpress_version }} - REGRESSION_PHP_VERSION: ${{ inputs.php_version }} - - name: Test roles, multisite, URL mapping, and settings - run: npm run test:regressions - if: ${{ !cancelled() && steps.regression_environment.outcome == 'success' }} - - uses: actions/upload-artifact@v4 - if: ${{ !cancelled() }} - with: - name: integration-regressions-test-report - path: | - playwright-report/ - playwright-integration-regressions-report/ - test-results/integration-regressions/ - test-results/integration-regressions-results.json - retention-days: 14 - - run: npm run env:regressions:stop - if: ${{ always() }} diff --git a/.github/workflows/pr-tests.yml b/.github/workflows/pr-tests.yml index 16cb5c7..bf47319 100644 --- a/.github/workflows/pr-tests.yml +++ b/.github/workflows/pr-tests.yml @@ -1,9 +1,21 @@ -name: WordPress plugin PR tests +name: Plugin tests on: workflow_call: pull_request: workflow_dispatch: + inputs: + plugin_ref: + description: 'Plugin branch, tag or commit; blank tests the workflow commit' + type: string + wordpress_version: + description: 'WordPress version for single-site and multisite tests' + default: '6.9.4' + type: string + php_version: + description: 'PHP version supported by Playground' + default: '8.3' + type: string permissions: contents: read @@ -21,6 +33,16 @@ jobs: - uses: actions/checkout@v6 with: persist-credentials: false + - name: Check out the plugin under test + uses: actions/checkout@v6 + with: + ref: ${{ inputs.plugin_ref || github.sha }} + path: .plugin-under-test + sparse-checkout: siteimprove + persist-credentials: false + - name: Record the plugin commit + run: | + printf 'Tested plugin commit: `%s`\n' "$(git -C .plugin-under-test rev-parse HEAD)" >> "$GITHUB_STEP_SUMMARY" - name: Record the tested commit run: | printf 'Tested repository commit: `%s`\n' "$(git rev-parse HEAD)" >> "$GITHUB_STEP_SUMMARY" @@ -36,6 +58,8 @@ jobs: run: npm run test:live:contracts - name: Test content capture, rechecks, and highlighting handoff run: npm test + env: + SITEIMPROVE_TEST_SCRIPT: ${{ github.workspace }}/.plugin-under-test/siteimprove/admin/js/siteimprove.js - name: Upload browser report if: ${{ !cancelled() }} uses: actions/upload-artifact@v4 @@ -61,6 +85,16 @@ jobs: - uses: actions/checkout@v6 with: persist-credentials: false + - name: Check out the plugin under test + uses: actions/checkout@v6 + with: + ref: ${{ inputs.plugin_ref || github.sha }} + path: .plugin-under-test + sparse-checkout: siteimprove + persist-credentials: false + - name: Record the plugin commit + run: | + printf 'Tested plugin commit: `%s`\n' "$(git -C .plugin-under-test rev-parse HEAD)" >> "$GITHUB_STEP_SUMMARY" - name: Record the tested commit run: | printf 'Tested repository commit: `%s`\n' "$(git rev-parse HEAD)" >> "$GITHUB_STEP_SUMMARY" @@ -70,6 +104,16 @@ jobs: cache: npm - run: npm ci - run: npx playwright install --with-deps chromium firefox + - name: Configure the selected plugin and runtime + env: + WORDPRESS_VERSION: ${{ inputs.wordpress_version }} + PHP_VERSION: ${{ inputs.php_version || '8.3' }} + run: | + node - <<'NODE' + const config = {plugins: ['./.plugin-under-test/siteimprove'], phpVersion: process.env.PHP_VERSION}; + if (process.env.WORDPRESS_VERSION) config.core = `WordPress/WordPress#${process.env.WORDPRESS_VERSION}`; + require('node:fs').writeFileSync('.wp-env.override.json', JSON.stringify(config)); + NODE - name: Start disposable WordPress id: wordpress_environment run: npm run env:start @@ -86,6 +130,10 @@ jobs: id: regression_environment if: ${{ !cancelled() }} run: npm run env:regressions:start + env: + SITEIMPROVE_TEST_PLUGIN: .plugin-under-test/siteimprove + REGRESSION_WP_VERSION: ${{ inputs.wordpress_version || '6.9.4' }} + REGRESSION_PHP_VERSION: ${{ inputs.php_version || '8.3' }} - name: Test roles, multisite, URL mapping, and settings if: ${{ !cancelled() && steps.regression_environment.outcome == 'success' }} run: npm run test:regressions diff --git a/.github/workflows/prepublish-live.yml b/.github/workflows/prepublish-live.yml index b909b1a..3f2e109 100644 --- a/.github/workflows/prepublish-live.yml +++ b/.github/workflows/prepublish-live.yml @@ -1,4 +1,4 @@ -name: WordPress–Siteimprove live integration tests +name: Live Siteimprove test on: workflow_call: diff --git a/.github/workflows/prepublish-test.yml b/.github/workflows/prepublish-test.yml deleted file mode 100644 index 7311c68..0000000 --- a/.github/workflows/prepublish-test.yml +++ /dev/null @@ -1,58 +0,0 @@ -name: WordPress plugin browser tests - -on: - workflow_dispatch: - inputs: - plugin_ref: - description: 'Plugin branch, tag or commit to test (e.g. refs/pull/64/head); blank uses this workflow commit' - required: false - type: string - -permissions: - contents: read - -jobs: - browser-tests: - runs-on: ubuntu-latest - timeout-minutes: 15 - steps: - - name: Explain test coverage - if: ${{ always() }} - run: | - cat >> "$GITHUB_STEP_SUMMARY" <<'SUMMARY' - ### WordPress plugin browser tests - - Environment: Chromium and Firefox; local browser fixtures; Siteimprove mocked. - - Coverage: Content capture, recheck callbacks, and highlighting handoff. - - Report: plugin-browser-test-report - - Results: see the test steps below; this description does not indicate a pass. - SUMMARY - - uses: actions/checkout@v6 - with: - persist-credentials: false - - name: Check out the plugin version under test - uses: actions/checkout@v6 - with: - ref: ${{ inputs.plugin_ref || github.sha }} - path: .plugin-under-test - sparse-checkout: siteimprove - persist-credentials: false - - name: Record the plugin commit under test - run: git -C .plugin-under-test rev-parse HEAD - - uses: actions/setup-node@v6 - with: - node-version: '24' - cache: npm - - run: npm ci - - run: npx playwright install --with-deps chromium firefox - - name: Test content capture, rechecks, and highlighting handoff - run: npm test - env: - SITEIMPROVE_TEST_SCRIPT: ${{ github.workspace }}/.plugin-under-test/siteimprove/admin/js/siteimprove.js - - uses: actions/upload-artifact@v4 - if: ${{ !cancelled() }} - with: - name: plugin-browser-test-report - path: | - playwright-report/ - test-results/browser/ - retention-days: 14 diff --git a/.github/workflows/prepublish-wordpress.yml b/.github/workflows/prepublish-wordpress.yml deleted file mode 100644 index b9ec808..0000000 --- a/.github/workflows/prepublish-wordpress.yml +++ /dev/null @@ -1,61 +0,0 @@ -name: WordPress plugin environment tests - -on: - workflow_dispatch: - inputs: - plugin_ref: - description: 'Plugin branch, tag or commit to test (e.g. refs/pull/64/head); blank uses this workflow commit' - required: false - type: string - -permissions: - contents: read - -jobs: - wordpress: - runs-on: ubuntu-latest - timeout-minutes: 20 - steps: - - name: Explain test coverage - if: ${{ always() }} - run: | - cat >> "$GITHUB_STEP_SUMMARY" <<'SUMMARY' - ### WordPress plugin environment tests - - Environment: Chromium and Firefox; disposable local WordPress; Siteimprove mocked. - - Coverage: Drafts, previews, autosaves, nonces, and content capture. - - Report: wordpress-env-test-report - - Results: see the test steps below; this description does not indicate a pass. - SUMMARY - - uses: actions/checkout@v6 - with: - persist-credentials: false - - name: Check out the plugin version under test - uses: actions/checkout@v6 - with: - ref: ${{ inputs.plugin_ref || github.sha }} - path: .plugin-under-test - sparse-checkout: siteimprove - persist-credentials: false - - name: Record the plugin commit under test - run: git -C .plugin-under-test rev-parse HEAD - - uses: actions/setup-node@v6 - with: - node-version: '24' - cache: npm - - run: npm ci - - run: npx playwright install --with-deps chromium firefox - - name: Point WordPress at the selected plugin - run: node -e "require('node:fs').writeFileSync('.wp-env.override.json', JSON.stringify({plugins:['./.plugin-under-test/siteimprove']}))" - - run: npm run env:start - - name: Test WordPress drafts, previews, and content capture - run: npm run test:wordpress - - uses: actions/upload-artifact@v4 - if: ${{ !cancelled() }} - with: - name: wordpress-env-test-report - path: | - playwright-wordpress-report/ - test-results/wordpress/ - retention-days: 14 - - run: npm run env:stop - if: ${{ always() }} diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 42ec92f..62909fc 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -1,5 +1,12 @@ name: Create and deploy release on: + workflow_dispatch: + inputs: + mode: + description: 'Dry run prepares files only; test commits to TEST_SVN_URL; production publishes' + type: choice + default: dry-run + options: [dry-run, test, production] push: # Sequence of patterns matched against refs/tags tags: @@ -8,12 +15,57 @@ on: permissions: contents: read +concurrency: + group: wordpress-release + cancel-in-progress: false + jobs: + target: + runs-on: ubuntu-latest + outputs: + mode: ${{ steps.target.outputs.mode }} + steps: + - name: Require a version tag and resolve deployment mode + id: target + env: + REF_TYPE: ${{ github.ref_type }} + REF_NAME: ${{ github.ref_name }} + REQUESTED_MODE: ${{ inputs.mode }} + run: | + test "$REF_TYPE" = tag + [[ "$REF_NAME" =~ ^v[0-9]+\.[0-9]+\.[0-9]+(-test|-dry-run)?$ ]] || exit 1 + mode="$REQUESTED_MODE" + if [ -z "$mode" ]; then + case "$REF_NAME" in + *-dry-run) mode=dry-run ;; + *-test) mode=test ;; + *) mode=production ;; + esac + fi + case "$mode" in dry-run|test|production) ;; *) exit 1 ;; esac + if [ "$mode" = production ]; then + [[ "$REF_NAME" =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]] || exit 1 + fi + printf 'mode=%s\n' "$mode" >> "$GITHUB_OUTPUT" + printf 'Release tag: `%s` — mode: **%s**\n' "$REF_NAME" "$mode" >> "$GITHUB_STEP_SUMMARY" + + - name: Require a separate test destination + if: steps.target.outputs.mode == 'test' + env: + TEST_URL: ${{ vars.TEST_SVN_URL }} + PRODUCTION_URL: ${{ vars.WP_SVN_URL || 'https://plugins.svn.wordpress.org/siteimprove/' }} + run: | + test -n "$TEST_URL" + test "${TEST_URL%/}" != "${PRODUCTION_URL%/}" + test "${TEST_URL%/}" != 'https://plugins.svn.wordpress.org/siteimprove' + validation: + needs: target uses: ./.github/workflows/release-checks.yml build: - needs: validation + needs: [target, validation] + if: needs.target.outputs.mode != 'dry-run' permissions: contents: write runs-on: ubuntu-latest @@ -31,8 +83,8 @@ jobs: deploy-dry-run: runs-on: ubuntu-latest - needs: build - if: contains(github.ref_name, '-dry-run') + needs: [target, validation] + if: needs.target.outputs.mode == 'dry-run' steps: - uses: actions/checkout@v6 - name: Deploy to WordPress Marketplace (Dry Run) @@ -49,8 +101,8 @@ jobs: deploy-test: runs-on: ubuntu-latest - needs: build - if: contains(github.ref_name, '-test') + needs: [target, build] + if: needs.target.outputs.mode == 'test' steps: - uses: actions/checkout@v6 - name: Deploy to WordPress Marketplace (Test) @@ -67,8 +119,8 @@ jobs: deploy-production: runs-on: ubuntu-latest - needs: build - if: "!contains(github.ref_name, '-test') && !contains(github.ref_name, '-dry-run')" + needs: [target, build] + if: needs.target.outputs.mode == 'production' steps: - uses: actions/checkout@v6 - name: Deploy to WordPress Marketplace (Production) diff --git a/.github/workflows/wpcs.yml b/.github/workflows/wpcs.yml index 8cf61d6..85b21d7 100644 --- a/.github/workflows/wpcs.yml +++ b/.github/workflows/wpcs.yml @@ -1,4 +1,4 @@ -name: WPCS check +name: Code standards on: pull_request: diff --git a/README.md b/README.md index d30abbf..b0a111d 100644 --- a/README.md +++ b/README.md @@ -6,6 +6,10 @@ The code on this repository has to match the WordPress Coding Standards in order Every pull request will be checked against WPCS through GitHub Actions. +## Development and releases + +See [GitHub Actions: tests and publishing](docs/workflows.md) for workflow selection, reports, and release instructions. + ## Version History ### 2.1.4 * Bugfix - Prepublish now reads the page from the WordPress domain rather than the configured Public URL, so the check works on setups where Public URL points at a separate delivery domain diff --git a/TESTING.md b/TESTING.md index ec6d560..dad7269 100644 --- a/TESTING.md +++ b/TESTING.md @@ -107,33 +107,10 @@ by its command queue and callback contract; its actual UI is not exercised. ### GitHub Actions -The individual browser and WordPress workflows support manual dispatch. -`pr-tests.yml` runs both suites and integration regressions automatically on PRs. Commit the workflows, package files -(including the lockfile), Playwright configs, tests, and documentation to the PR -branch. A manually started **WordPress plugin browser tests** run attaches an HTML report plus -failure traces/screenshots. It has read-only repository permissions and does not deploy. -The manual **Run workflow** button is available once the workflow exists on the -repository's default branch. Both workflows accept `plugin_ref`: enter a branch, -tag or commit such as `refs/pull/64/head`. The workflow records the resolved -plugin commit and checks it out separately under `.plugin-under-test/`; the tests -and configuration come from the workflow branch. Blank means this workflow's -commit. Before PR #64 is merged, running against the unchanged default-branch -plugin is expected to fail the regression scenarios. Select PR #64 explicitly to -verify its fix. - -The WordPress workflow writes an ignored `.wp-env.override.json` to install the -selected plugin. It does not overwrite or commit production plugin files. - -The existing WPCS workflow still runs on pull requests. For a review-only upload, -`[skip ci]` in the head commit message skips GitHub Actions `push` and -`pull_request` runs for that commit, including WPCS. Required skipped checks may -remain pending and block merging. It does not block manual runs or other event -types. To prevent a workflow from running at all, a maintainer can disable it in -GitHub Actions. Push a feature branch without release tags; the existing release -and deployment workflows have their own triggers. - -See [GitHub skip instructions](https://docs.github.com/en/actions/how-tos/manage-workflow-runs/skip-workflow-runs) -and [disabling workflows](https://docs.github.com/en/actions/how-tos/manage-workflow-runs/disable-and-enable-workflows). +Use **Plugin tests** for automatic PR checks or manual plugin/runtime selection. +See [the workflow guide](docs/workflows.md) for inputs, tested commits, reports, +release procedures and credentials. The manual workflows have been consolidated; +all browser, single-site and multisite suites remain. ### Failure screenshots and readable reports @@ -145,8 +122,8 @@ evidence: a screenshot shows the visible page and may not show the captured DOM. After a GitHub run, open **Actions > workflow run > Artifacts** and download: -- `plugin-browser-test-report` for the browser regression tests (also includes failure traces). -- `wordpress-env-test-report` for the real WordPress tests (screenshots and HTML report; no traces or video). +- `pr-browser-test-report` for the browser regression tests (also includes failure traces). +- `pr-wordpress-test-reports` for the real WordPress tests (screenshots and HTML report; no traces or video). Extract the artifact and open the report's `index.html`, or use Playwright's report viewer on its report directory. Artifacts are retained for 14 days. @@ -231,13 +208,8 @@ HTTP API calls. This mode is for these credential-free integration tests only. `npm test` runs 46 browser checks: 30 capture checks and 16 integration regression checks. The additional multisite regression checks run with `npm run test:regressions`. -The manual **WordPress plugin environment tests** GitHub workflow starts a fresh -instance on the selected branch, runs the integration tests in both browsers and -stops it. The site -exists only on the runner while the job executes; GitHub does not host a lasting, -public test site. As with the other manual workflow, it must first be added to -the default branch, and the selected workflow branch must contain its supporting files. Use -`plugin_ref` to select a different plugin version, including PR #64. +The **Plugin tests** workflow starts and stops disposable single-site and multisite +instances. Use its manual inputs to select a plugin ref and WordPress/PHP versions. ### Why the draft and style tests were added @@ -345,230 +317,11 @@ make this test pass. References: [Playwright CI setup](https://playwright.dev/docs/ci-intro) and [PR #64](https://github.com/Siteimprove/CMS-plugin-Wordpress/pull/64). -# Testing WordPress Deployment Action +## Deployment validation -This guide covers how to test the unified WordPress deployment action safely. - -## 🧪 Testing Options - -### 1. **Local Testing Script (Recommended for Quick Tests)** - -The fastest way to test the deployment process locally: - -```bash -# Make script executable (if not already) -chmod +x scripts/test-deploy.sh - -# Run with default values -./scripts/test-deploy.sh - -# Run with custom values -./scripts/test-deploy.sh "2.0.9-test" "siteimprove" "wordpress-assets" - -# Clean up after testing -rm -rf deploy -``` - -**What it does:** -- Simulates the file preparation process (dry-run mode) -- Uses the same deployment script as GitHub Actions -- Shows exactly what files would be deployed -- No network connections or external dependencies -- Creates only the `deploy/` directory (no SVN operations) - -### 2. **GitHub Actions Dry Run (Recommended for CI/CD Testing)** - -Use the unified action with dry-run enabled: - -1. Go to **Actions** tab in your GitHub repository -2. Select **"Deploy to WordPress Marketplace"** -3. Click **"Run workflow"** -4. Set **dry-run** to `true` -5. Set **test-mode** to `true` (optional) -6. Enter test version (e.g., `2.0.8-test`) -7. Click **"Run workflow"** - -**What it does:** -- Runs the actual GitHub Action -- Performs all deployment steps except SVN commit -- Shows detailed preview of what would be deployed -- Uses the same logic as production deployment - -### 3. **Test SVN Repository (Recommended for Full Testing)** - -Set up a test SVN repository to test the complete deployment process: - -#### Option A: Use GitHub as SVN Repository -```bash -# Create a test repository on GitHub -# Then use GitHub's SVN interface -# URL: https://github.com/your-username/test-wordpress-deploy -``` - -#### Option B: Use a Local SVN Repository -```bash -# Install SVN locally -brew install subversion # macOS -sudo apt-get install subversion # Ubuntu - -# Create local SVN repository -svnadmin create test-svn-repo -svn mkdir file:///path/to/test-svn-repo/trunk -m "Create trunk" -svn mkdir file:///path/to/test-svn-repo/tags -m "Create tags" -svn mkdir file:///path/to/test-svn-repo/assets -m "Create assets" -``` - -#### Option C: Use a Cloud SVN Service -- **Assembla**: Free SVN hosting -- **Beanstalk**: SVN hosting with free tier -- **Visual Studio Team Services**: Free SVN repositories - -## 🔧 Configuration for Testing - -### GitHub Secrets for Testing -Add these to your repository secrets: - -``` -TEST_SVN_USERNAME=your-test-username -TEST_SVN_PASSWORD=your-test-password -``` - -### GitHub Variables for Testing -Add these to your repository variables: - -``` -TEST_SVN_URL=https://your-test-svn-repo.com/siteimprove-test/ -WP_PLUGIN_SLUG=siteimprove -WP_ASSETS_DIR=wordpress-assets -``` - -## 📋 Testing Checklist - -### Before Testing -- [ ] Plugin files are in the `siteimprove/` directory -- [ ] Main plugin file (`siteimprove.php`) has correct header -- [ ] Version number is properly formatted -- [ ] Assets directory exists (if using assets) -- [ ] No sensitive data in plugin files - -### During Testing -- [ ] Files are copied correctly -- [ ] Development files are removed -- [ ] Version is updated in main file -- [ ] Assets are copied (if present) -- [ ] No errors in the process - -### After Testing -- [ ] Review generated files -- [ ] Check file permissions -- [ ] Verify version numbers -- [ ] Test plugin functionality -- [ ] Clean up test files - -## 🚨 Common Issues and Solutions - -### Issue: "Permission denied" on script -```bash -chmod +x scripts/test-deploy.sh -``` - -### Issue: "SVN checkout failed" -- Check SVN credentials -- Verify SVN repository URL -- Ensure repository exists and is accessible - -### Issue: "Files not found" -- Run script from project root directory -- Ensure `siteimprove/` directory exists -- Check file paths in the action - -## 🔄 Testing Workflow - -### Recommended Testing Sequence: - -1. **Local Script Test** (5 minutes) - ```bash - ./scripts/test-deploy.sh - ``` - - Tests file preparation only (dry-run mode) - - Creates `deploy/` directory with prepared files - -2. **GitHub Actions Dry Run** (10 minutes) - - Run unified action with dry-run=true - - Review output and artifacts - - Tests complete deployment logic without SVN operations - -3. **Test SVN Repository** (30 minutes) - - Set up test SVN repo - - Run full deployment with test-mode=true, dry-run=false - - Verify complete deployment process - -4. **Production Deployment** (when ready) - - Update to real SVN credentials - - Run actual deployment with dry-run=false, test-mode=false - -## 📊 What to Look For - -### Successful Test Results: -- ✅ All plugin files copied -- ✅ Development files removed -- ✅ Version updated correctly -- ✅ No error messages -- ✅ Files are properly organized -- ✅ `deploy/` directory created with prepared files - -### Warning Signs: -- ❌ Missing files -- ❌ Development files included -- ❌ Wrong version numbers -- ❌ Permission issues -- ❌ `deploy/` directory not created - -## 🎯 Next Steps After Testing - -1. **Review Results**: Check all generated files -2. **Fix Issues**: Address any problems found -3. **Update Configuration**: Set real SVN credentials -4. **Test Again**: Run with real repository -5. **Deploy**: Use for actual releases - -## 🔧 Using the Unified Action - -The unified action supports three modes: - -### Dry Run Mode (Safe Testing) -```yaml -- name: Test Deployment - uses: ./.github/actions/deploy-to-wordpress - with: - version: '2.0.8-test' - dry-run: true - test-mode: true -``` - -### Test Mode (Test Repository) -```yaml -- name: Test Deployment - uses: ./.github/actions/deploy-to-wordpress - with: - version: '2.0.8-test' - test-mode: true - svn-url: 'https://your-test-svn-repo.com/siteimprove-test/' -``` - -### Production Mode (Live Deployment) -```yaml -- name: Deploy to Production - uses: ./.github/actions/deploy-to-wordpress - with: - version: '2.0.8' - dry-run: false - test-mode: false - svn-username: ${{ secrets.WP_SVN_USERNAME }} - svn-password: ${{ secrets.WP_SVN_PASSWORD }} -``` - -This simplified approach ensures your deployment process is reliable and safe before using it for production releases. +See [Create and deploy a release](docs/workflows.md#create-and-deploy-a-release) +for the supported tag formats, manual modes, destination settings, release checks, +and dry-run limitations. The separate Marketplace deployment workflow is retired. ### Workflow summary privacy @@ -582,15 +335,15 @@ GitHub still displays the account that triggered a run, and existing commit meta The browser suite runs once, alongside catalog validation and live-runner privacy contracts. The WordPress job installs dependencies and browsers once, then runs the single-site and multisite suites sequentially with separate disposable environments. Both jobs use Chromium and Firefox. Reports contain synthetic fixture data and exclude Git author metadata. -The existing individual test workflows remain available for manual investigation. The regression workflow no longer runs separately on branch pushes. WPCS remains its own PR check; the authenticated live workflow remains manual and protected. Branch-protection settings are not changed. Configure **Browser tests**, **WordPress tests**, and **WPCS** as required checks after their first successful GitHub run. Measure that run before setting a runtime target. +Manual investigation uses the same **Plugin tests** workflow with optional plugin +and runtime inputs. WPCS remains its own PR check. Branch-protection settings are +unchanged; the check names remain **Browser tests**, **WordPress tests**, and **WPCS**. ### Release validation -Both **Create and deploy release** (`v*` tags) and **Deploy to WordPress Marketplace** call `release-checks.yml` before publishing. That reusable workflow runs the two PR test jobs, WPCS, the protected live Siteimprove test, and a disposable-container ZIP lifecycle check. Every check must succeed, including the live check; missing credentials, failed checks, or skipped checks prevent publishing. Existing deployment triggers remain in place, including manual and `wp-*` releases. - -All checks use the triggering commit. The ZIP contains only tracked `siteimprove/` files. The package check installs and activates it in WordPress, verifies plugin bootstrap, then deactivates and deletes it. **Create and deploy release** downloads and publishes that validated ZIP; Marketplace deployment uses plugin sources from the same commit. The live suite uses credentials from the `siteimprove-test` environment, whose deployment rules must allow the intended release refs. No secrets are passed to the PR suites. - -The first GitHub run must establish package lifecycle and live-test success. Workflow parsing and local contract tests do not establish those results. Broader hosting compatibility and visual overlay inspection remain outside these automated checks. +**Create and deploy release** calls the shared `release-checks.yml` before any +publishing. Every check must succeed, including the protected live test and ZIP +lifecycle check. See [the workflow guide](docs/workflows.md) for the exact sequence. ### Live result assertion temporarily skipped diff --git a/docs/workflows.md b/docs/workflows.md new file mode 100644 index 0000000..6975bd9 --- /dev/null +++ b/docs/workflows.md @@ -0,0 +1,96 @@ +# GitHub Actions + +## Which workflow should I use? + +| Workflow | Trigger | Purpose | Publishes? | +| --- | --- | --- | --- | +| **Code standards** (`wpcs.yml`) | Pull requests; release checks | WordPress PHP coding standards. The check name stays **WPCS**. | No | +| **Plugin tests** (`pr-tests.yml`) | Pull requests; manual; release checks | Chromium and Firefox browser, WordPress draft/preview, and multisite regression tests. Siteimprove is mocked. | No | +| **Live Siteimprove test** (`prepublish-live.yml`) | Manual; release checks | Real account login, Live page data, draft handoff and loading-state exit. | No release; sends synthetic test content to Siteimprove | +| **Create and deploy release** (`release.yml`) | `v*` tags; manual on a version tag | Validate, create the GitHub release, and deploy to WordPress.org; dry-run and test modes are available. | Yes, except dry-run | + +`release-checks.yml` is a shared helper, not a fifth operation to run manually. +It requires Plugin tests, Code standards, the live test and the install/activate/ +deactivate/delete ZIP lifecycle check to pass before publishing. It requires +Docker on its GitHub runner. A skipped or failed prerequisite blocks publishing. + +## Check a pull request or another plugin version + +Pull requests automatically run **Plugin tests** and **Code standards**. +Keep **Browser tests**, **WordPress tests** and **WPCS** as the check names in +branch protection; this cleanup does not change repository protection settings. + +For manual investigation, open **Actions → Plugin tests → Run workflow**: + +- Select the branch containing the test code. +- Set `plugin_ref` to a plugin branch, tag, commit or `refs/pull/NUMBER/head`. + Blank uses the workflow commit. Only the plugin is taken from this ref; + test code comes from the workflow branch. +- Set `wordpress_version` and `php_version` if needed. Manual defaults are + WordPress 6.9.4 and PHP 8.3, applied to both WordPress environments. +- Automatic runs retain the existing defaults: latest WordPress for single-site, + WordPress 6.9.4 for multisite, and PHP 8.3. + +The summary records both the repository and plugin commits. Environments are +created on the runner and stopped after testing; they are not persistent sites. +No real Siteimprove credentials are used. + +Open the failed step first, then download **pr-browser-test-report** or +**pr-wordpress-test-reports** from the run's Artifacts. Reports are retained for +14 days and contain individual assertions and synthetic failure screenshots. +A startup failure may have no browser report. See [testing details](../TESTING.md) +and the [regression catalog](../tests/integration-regressions/COVERAGE.md). + +## Check the real Siteimprove connection + +Run **Live Siteimprove test** on reviewed code. It tests the triggering commit, +not an independently selectable plugin ref. The `siteimprove-test` environment +supplies credentials and applies its configured approval/ref restrictions. +It needs `SITEIMPROVE_PUBLIC_URL`, `SITEIMPROVE_CRAWLED_URL`, +`SITEIMPROVE_API_USERNAME`, `SITEIMPROVE_API_KEY`, `SITEIMPROVE_USERNAME` and +`SITEIMPROVE_PASSWORD`. Release tags must be allowed by the environment rules. + +Use fixed phase outcomes in the job log for diagnostics. No authenticated +screenshots, traces, account data or browser-state artifacts are uploaded. +**The missing-title result assertion is skipped:** a green result verifies the +implemented login/data/handoff checks, not scan-result correctness or visual +highlighting. See [live test coverage and setup](../tests/live/README.md). + +## Create and deploy a release + +1. Merge the reviewed change after all PR checks pass. Update the plugin version + and release notes before tagging. +2. Use a version tag such as `v2.1.5`. Pushing that tag automatically starts + production validation and publishing. Do not push a production tag merely + to try a dry run. +3. To rehearse first, push `v2.1.5-dry-run`. This runs release validation and file + preparation without creating a GitHub release or performing any SVN operation. + It does not verify SVN authentication or prove deployment will succeed. +4. For a test deployment, use a `v2.1.5-test` tag. It creates a GitHub release and + commits to the separate test SVN repository. It is a real write, not a simulation. +5. Manual runs must select an existing version tag using the GitHub UI or CLI + `--ref`; branch runs are rejected. Choose `dry-run` (default), `test` or + `production`. Production mode requires a plain `vMAJOR.MINOR.PATCH` tag. +6. After production, verify the GitHub ZIP and release, the WordPress.org version, + and installation of that version. Inspect existing release/SVN state before + retrying a partly completed deployment: an existing SVN tag can prevent retry. + +The validated GitHub ZIP contains tracked `siteimprove/` files only. SVN deployment +uses plugin sources from the same tag. Publishing waits for all release checks, +including the protected live test. Release runs are serialized to avoid concurrent +SVN writes. The deployed version drops the leading `v`; test/dry-run suffixes remain. + +Production uses `WP_SVN_USERNAME`, `WP_SVN_PASSWORD`, and optional `WP_SVN_URL` +(default `https://plugins.svn.wordpress.org/siteimprove/`). Test mode requires +`TEST_SVN_USERNAME`, `TEST_SVN_PASSWORD`, and `TEST_SVN_URL`, which must differ +from production. Shared optional variables are `WP_PLUGIN_SLUG` and `WP_ASSETS_DIR`. + +## Cleanup map + +The separate `prepublish-test.yml`, `prepublish-wordpress.yml`, and +`integration-regressions.yml` entry points are replaced by **Plugin tests**; +all suites remain, with manual plugin/runtime selection. +`deploy-wordpress.yml` is replaced by **Create and deploy release**. +The old `wp-*` tag and `main` branch deployment triggers are removed. Use `v*` +version tags and the release workflow's explicit modes. GitHub may retain old +workflow run history; historical entries do not indicate another current workflow. diff --git a/tests/SCENARIOS.md b/tests/SCENARIOS.md index 64a04ed..8510c83 100644 --- a/tests/SCENARIOS.md +++ b/tests/SCENARIOS.md @@ -6,7 +6,7 @@ them in its HTML report (`npm run test:report`). A failed step identifies which part of the scenario failed; technical assertion details remain available below it. Both suites attach failure screenshots. The WordPress HTML report is available with `npm run test:wordpress:report`. GitHub runs upload the reports as -`prepublish-test-report` and `prepublish-wordpress-report`, retained for 14 days. +`pr-browser-test-report` and `pr-wordpress-test-reports`, retained for 14 days. ## Automated today: browser capture diff --git a/tests/integration-regressions/README.md b/tests/integration-regressions/README.md index 78929cd..66049fe 100644 --- a/tests/integration-regressions/README.md +++ b/tests/integration-regressions/README.md @@ -64,7 +64,7 @@ an expendable package copy so they cannot delete the checkout. Reports go to ignored `playwright-report/` and `playwright-integration-regressions-report/`. Use `npm run test:regressions:report` to view the latter. The two-job PR workflow runs the regression suite automatically against the -PR merge commit. The individual regression workflow is manual only. +PR merge commit. Manual investigation uses the same **Plugin tests** workflow with plugin and runtime inputs. Manual runs can select another plugin revision once the workflow exists on the default branch. These runs do not deploy the plugin. Ordinary assertion failures remain failures. diff --git a/tests/live/README.md b/tests/live/README.md index b649743..f228f1d 100644 --- a/tests/live/README.md +++ b/tests/live/README.md @@ -57,7 +57,7 @@ browser document would not exercise the same path. ## Execution and approval -The workflow is `prepublish-live.yml` (**WordPress–Siteimprove live integration tests**). +The workflow is `prepublish-live.yml` (**Live Siteimprove test**). It supports manual dispatch and reuse by the release checks in this repository. It does not run on PR events and uses the `siteimprove-test` environment's approval and deployment rules. Configure that environment to allow the release