From 8c58eca7a7c2b75c1f992b54240e6c79ec90c941 Mon Sep 17 00:00:00 2001 From: MortenFriisSiteImprove Date: Thu, 10 Sep 2026 20:38:00 +0200 Subject: [PATCH 1/2] Add manual live Prepublish smoke test [skip ci] --- .github/workflows/prepublish-live.yml | 54 ++++++ TESTING.md | 11 +- package.json | 5 +- playwright.live-fixture.config.js | 8 + scripts/prepare-live-env.js | 9 + tests/SCENARIOS.md | 9 +- tests/live/README.md | 122 ++++++++++++++ tests/live/fixture.spec.js | 46 +++++ tests/live/observer.spec.js | 31 ++++ tests/live/run.js | 203 +++++++++++++++++++++++ tests/live/settings.js | 36 ++++ tests/live/settings.test.js | 41 +++++ tests/live/support/fixtures.php | 72 ++++++++ tests/live/support/templates/preview.php | 16 ++ 14 files changed, 655 insertions(+), 8 deletions(-) create mode 100644 .github/workflows/prepublish-live.yml create mode 100644 playwright.live-fixture.config.js create mode 100644 scripts/prepare-live-env.js create mode 100644 tests/live/README.md create mode 100644 tests/live/fixture.spec.js create mode 100644 tests/live/observer.spec.js create mode 100644 tests/live/run.js create mode 100644 tests/live/settings.js create mode 100644 tests/live/settings.test.js create mode 100644 tests/live/support/fixtures.php create mode 100644 tests/live/support/templates/preview.php diff --git a/.github/workflows/prepublish-live.yml b/.github/workflows/prepublish-live.yml new file mode 100644 index 0000000..52c51ba --- /dev/null +++ b/.github/workflows/prepublish-live.yml @@ -0,0 +1,54 @@ +name: Prepublish live Siteimprove test + +on: + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: siteimprove-live-test + cancel-in-progress: false + +jobs: + live: + # Reviewed workflow code only. Environment approval is required separately. + if: github.repository == 'Siteimprove/CMS-plugin-Wordpress' && github.ref == 'refs/heads/master' + environment: siteimprove-test + runs-on: ubuntu-latest + timeout-minutes: 20 + steps: + - uses: actions/checkout@v6 + with: + persist-credentials: false + - name: Check out the reviewed PR 64 plugin + uses: actions/checkout@v6 + with: + ref: a23af8519861f674d700cbe4fe183817f47458dc + path: .plugin-under-test + sparse-checkout: siteimprove + persist-credentials: false + - uses: actions/setup-node@v6 + with: + node-version: '24' + cache: npm + - run: npm ci + - run: npm run test:live:contracts + - run: npx playwright install --with-deps chromium + - run: node scripts/prepare-live-env.js + - run: npm run env:start + - name: Verify Live page data and one fresh Prepublish issue + run: npm run test:live + env: + SITEIMPROVE_RUN_LIVE: '1' + SITEIMPROVE_PUBLIC_URL: ${{ secrets.SITEIMPROVE_PUBLIC_URL }} + SITEIMPROVE_CRAWLED_URL: ${{ secrets.SITEIMPROVE_CRAWLED_URL }} + SITEIMPROVE_API_USERNAME: ${{ secrets.SITEIMPROVE_API_USERNAME }} + SITEIMPROVE_API_KEY: ${{ secrets.SITEIMPROVE_API_KEY }} + SITEIMPROVE_USERNAME: ${{ secrets.SITEIMPROVE_USERNAME }} + SITEIMPROVE_PASSWORD: ${{ secrets.SITEIMPROVE_PASSWORD }} + DEBUG: '' + PWDEBUG: '0' + - run: npm run env:stop + if: ${{ always() }} + # No screenshots, traces, browser state, server logs or account data uploads. diff --git a/TESTING.md b/TESTING.md index 481f80c..7706017 100644 --- a/TESTING.md +++ b/TESTING.md @@ -237,6 +237,11 @@ the missing-title result. This concerns page-content fidelity, not SDK UI stylin ### What remains for an actual Siteimprove end-to-end test +A first manual live workflow and runner are now prepared in +[tests/live/README.md](tests/live/README.md). They have not been authenticated or +run against Siteimprove. The following account and network requirements still +apply; PR #65’s existing tests continue to use no secrets. + We still need a Siteimprove test account with Prepublish access and a known crawled page accessible to both the browser user and API user. The plugin's credential validation compares **Public URL** with the sites available to the @@ -276,9 +281,9 @@ First configure and validate the real integration in that environment: of source files, logs, reports and uploaded traces. Local browser state belongs in the ignored `playwright/.auth/` directory. -The full scan test and its authenticated GitHub job have **not** been implemented -or run yet: the account, registered site and actual overlay/login flow are needed -to implement and verify those steps. The integration workflow must not be interpreted +The initial live runner and authenticated GitHub job are **prepared but unverified +against the account**. The first authorized run must verify the actual login, +site mapping and fresh scan results. The integration workflow must not be interpreted as proof that a Siteimprove scan passed. The revision fixture uses [WordPress’s autosave API](https://developer.wordpress.org/reference/functions/wp_create_post_autosave/) diff --git a/package.json b/package.json index 7590ad1..6dfc4df 100644 --- a/package.json +++ b/package.json @@ -8,7 +8,10 @@ "env:stop": "node scripts/wordpress-env.js stop", "env:status": "node scripts/wordpress-env.js status", "test:wordpress": "playwright test --config=playwright.wordpress.config.js", - "test:wordpress:report": "playwright show-report playwright-wordpress-report" + "test:wordpress:report": "playwright show-report playwright-wordpress-report", + "test:live": "node tests/live/run.js", + "test:live:contracts": "node --test tests/live/settings.test.js", + "test:live:fixture": "playwright test --config=playwright.live-fixture.config.js" }, "devDependencies": { "@playwright/test": "1.63.0", diff --git a/playwright.live-fixture.config.js b/playwright.live-fixture.config.js new file mode 100644 index 0000000..56fbe76 --- /dev/null +++ b/playwright.live-fixture.config.js @@ -0,0 +1,8 @@ +const { defineConfig } = require('@playwright/test'); +module.exports = defineConfig({ + testDir:'./tests/live', testMatch:['fixture.spec.js','observer.spec.js'], timeout:60000, workers:1, + outputDir:'./test-results/live-fixture', retries:0, forbidOnly:Boolean(process.env.CI), + reporter:[['list',{printSteps:true}]], + projects:[{name:'chromium',use:{browserName:'chromium'}},{name:'firefox',testIgnore:'observer.spec.js',use:{browserName:'firefox'}}], + use:{baseURL:'http://localhost:8888',trace:'off',screenshot:'off',video:'off'}, +}); diff --git a/scripts/prepare-live-env.js b/scripts/prepare-live-env.js new file mode 100644 index 0000000..1744f9e --- /dev/null +++ b/scripts/prepare-live-env.js @@ -0,0 +1,9 @@ +const fs = require('node:fs'); +// Contains paths and flags only; secret values are supplied to the runner later. +fs.writeFileSync('.wp-env.override.json', JSON.stringify({ + plugins:['./.plugin-under-test/siteimprove'], + config:{ WP_DEBUG:false, WP_DEBUG_LOG:false, WP_DEBUG_DISPLAY:false, + SITEIMPROVE_TEST_ENVIRONMENT:false, SITEIMPROVE_TEST_MOCK_SERVICE:false, + SITEIMPROVE_LIVE_TEST_ENVIRONMENT:true }, + mappings:{'wp-content/mu-plugins':'./tests/live/support'}, +}, null, 2)); diff --git a/tests/SCENARIOS.md b/tests/SCENARIOS.md index 7e0185d..64a04ed 100644 --- a/tests/SCENARIOS.md +++ b/tests/SCENARIOS.md @@ -99,10 +99,11 @@ is still unverified. The suite has 18 tests including readiness in each browser. ## Planned: authenticated Siteimprove checks -These scenarios are **not implemented or verified yet**. They require account -access, working authentication and URL mapping. The missing-title fixture also -still needs to be added. Run the small checks across selected WordPress setups; -the environment matrix is described in [TESTING.md](../TESTING.md). +An initial runner and missing-title fixture are now prepared in +[the live-test follow-up](live/README.md), but the authenticated flow is **not +verified against the account**. It requires an approved manual run to validate +login, URL mapping and real results. Start with one Chromium configuration; +additional live environments remain future coverage. ### Retrieve existing Live page data diff --git a/tests/live/README.md b/tests/live/README.md new file mode 100644 index 0000000..7db6eb0 --- /dev/null +++ b/tests/live/README.md @@ -0,0 +1,122 @@ +# Manual live Siteimprove test + +Status: the workflow and initial test implementation are prepared, but no +GitHub run, authenticated login or real Prepublish scan has been performed. +The first approved run must validate the account, network access and SDK UI +selectors. A failure is not converted into a skip or a pass. + +This follow-up builds on the credential-free infrastructure in PR #65. It adds +one Chromium smoke test with two outcomes: existing Live page data for the exact +crawled URL, followed by a newly completed Prepublish check reporting a missing +HTML title in the current local draft. It does not test SDK layout or highlighting. +Report-rerender styling remains a separate manual acceptance check. + +## Prerequisites + +The `siteimprove-test` GitHub Environment must contain: + +- `SITEIMPROVE_PUBLIC_URL`: the base URL configured in the WordPress plugin. +- `SITEIMPROVE_CRAWLED_URL`: one exact crawled page under that base URL. +- `SITEIMPROVE_API_USERNAME` and `SITEIMPROVE_API_KEY`. +- `SITEIMPROVE_USERNAME` and `SITEIMPROVE_PASSWORD`: a direct-login test user. + +The account needs existing Prepublish access, and both users must have access +to the mapped site. Initial terms acceptance must already be complete. The test +does not accept terms, activate a subscription or bypass MFA/CAPTCHA. It currently +uses English SDK labels; those labels and the missing-title issue name need +confirmation in the account during the first approved run. + +The runner must reach the Siteimprove API, SDK and identity services. The crawled +website itself is not visited: its URL is used as the plugin's normal mapping +context. Whether an unrelated existing crawled site can serve as that context +for this local fixture remains unverified with the real service. + +## What the test does + +1. Check that the API reports Prepublish as ready. +2. Sign into the disposable WordPress installation. +3. Configure Public URL, the latest SDK experience and API credentials using + the plugin's settings form and normal credential validation. +4. Create a local published control and an unpublished autosave with a unique + marker. The fixture's permalink matches the crawled page path; the plugin + performs its own Public URL transformation without request rewriting. +5. Confirm the preview contains the unique marker and exactly one empty title, + that the plugin reports the expected public URL, and that an anonymous + visitor cannot retrieve the draft content. +6. Sign into Siteimprove through the real plugin's popup using direct login. +7. Require an authenticated SDK polling response for the exact crawled URL, + with existing Live page data, and open Live page view. +8. Start a new check from Prepublish view. Observe its running state and the + real SDK's `contentcheck-flat-dom` message, without replacing the SDK queue + or altering the content. The message must contain this run's draft marker. +9. Require completion of that new check and the missing-title issue in + Prepublish view. Historical crawl results cannot satisfy this assertion. + +The title is removed in the server-rendered preview template, including the +plugin's capture iframe. Clearing only the editor title or mutating the outer +browser document would not exercise the same path. + +## Execution and approval + +The workflow is `prepublish-live.yml` (**Prepublish live Siteimprove test**). +It has only `workflow_dispatch`, runs only from `master` in this repository, +and uses the `siteimprove-test` environment's approval rules. It cannot run +from a PR branch. Workflow availability requires merging it into the default +branch; neither creating the PR nor adding secrets starts it. + +The plugin is pinned to reviewed PR #64 commit +`a23af8519861f674d700cbe4fe183817f47458dc`. There is deliberately no arbitrary +plugin-ref input in this secret-bearing job. Changing the plugin commit requires +a reviewed workflow change. The ordinary credential-free workflows retain their +flexible `plugin_ref` inputs. + +No workflow has been dispatched. Review this implementation before an authorized +maintainer starts and approves the first live run. + +## Diagnostics and secrets + +The live runner prints only fixed phase names and pass/fail outcomes. It catches +raw browser and API errors instead of printing URLs, field values, headers or +account content. Screenshots, traces, video, saved browser sessions and server-log +uploads are disabled. WordPress debug logging is disabled. Secrets are supplied +only to the live test step, after environment approval; no secret values are +written into tracked configuration files. + +Failures identify a phase such as API entitlement, login, URL mapping or new +Prepublish completion. The tradeoff is less detail for investigating SDK changes; +raw account diagnostics must not be published as public PR artifacts. This policy +is separate from the screenshots enabled for the synthetic tests in PR #65. + +## Credential-free validation + +`npm run test:live:contracts` verifies URL mapping, required configuration and +recognition of real Live page data versus login/missing-page responses. + +The PHP fixture can be checked locally without any Siteimprove credentials: + +```sh +npm run env:stop +node scripts/prepare-live-env.js +npm run env:start +npm run test:live:fixture +``` + +These fixture tests block external browser requests and use synthetic content. +They require the ignored `.plugin-under-test/siteimprove` checkout, as used by +the manual workflows. They verify root and nested paths, query preservation, +real WordPress autosaves, empty titles, inline styles and anonymous access checks. +The WordPress fixture runs in Chromium and Firefox. A synthetic cross-origin +observer check runs in Chromium, matching the live runner. The same observer +check failed in Firefox because iframe initialization did not install the +DOMContentLoaded observer on the final document; Firefox live instrumentation +is not validated. These checks do not validate live login, entitlement or scans. + +The live runner refuses to proceed unless `SITEIMPROVE_RUN_LIVE=1` and all six +secrets are supplied. Do not enable that flag merely to validate fixture code. + +## Sources for the initial integration + +- [WordPress plugin setup](https://help.siteimprove.com/support/solutions/articles/80001079368-new-wordpress-plugin) +- [Prepublish workflow](https://help.siteimprove.com/support/solutions/articles/80001077559-how-to-run-a-prepublish-check-with-the-new-plugin-ui) +- [Public SDK loader](https://cdn.siteimprove.net/cms/overlay-latest.js) — inspected version 2.1.3130.1 for iframe, polling and message contracts. +- The public identity form was inspected without entering credentials; its first step uses `loginId` and a Continue button. The subsequent password step still needs authenticated-flow verification. diff --git a/tests/live/fixture.spec.js b/tests/live/fixture.spec.js new file mode 100644 index 0000000..04ed865 --- /dev/null +++ b/tests/live/fixture.spec.js @@ -0,0 +1,46 @@ +const { test, expect } = require('@playwright/test'); +const { randomUUID } = require('node:crypto'); + +// Synthetic data only: validates the new PHP fixture, not a live Siteimprove scan. +for (const fixture of [{path:'/test-live/story.html',query:'lang=en'}, {path:'/',query:''}]) { + test(`The live fixture preserves ${fixture.path} and renders a private, styled preview with no title`, async ({ page, browser }) => { + await page.route('**/*', route => new URL(route.request().url()).hostname === 'localhost' ? route.continue() : route.abort()); + await page.goto('/wp-login.php'); + await expect(page.locator('#user_login')).toBeFocused(); + await page.locator('#user_login').fill('admin'); + await page.locator('#user_pass').fill('password'); + await page.locator('#wp-submit').click(); + await expect(page).toHaveURL(/\/wp-admin\//); + const marker = `SI-LIVE-${randomUUID()}`; + const urls = await test.step('Given a published control and a distinct unpublished autosave at the mapped local path', async () => { + await page.goto('/wp-admin/tools.php?page=siteimprove-live-fixture'); + await page.locator('[name=fixture_path]').fill(fixture.path); + await page.locator('[name=fixture_query]').fill(fixture.query); + await page.locator('[name=fixture_marker]').fill(marker); + await page.getByRole('button',{name:'Create live test fixture'}).click(); + return {preview:await page.locator('#live-preview').getAttribute('href'), published:await page.locator('#live-published').getAttribute('href')}; + }); + await test.step('Then the mapped path and query are preserved and the preview has an empty title', async () => { + expect(new URL(urls.published).pathname).toBe(fixture.path); + expect(new URL(urls.published).search.slice(1)).toBe(fixture.query); + await page.goto(urls.preview); + expect(new URL(page.url()).pathname).toBe(fixture.path); + await expect(page.locator('head > title')).toHaveCount(1); + await expect(page).toHaveTitle(''); + await expect(page.getByText(marker,{exact:true})).toBeVisible(); + await expect(page.getByText('SI-LIVE-PUBLISHED-CONTROL',{exact:true})).toHaveCount(0); + await expect(page.locator('#si-live-style')).toHaveCSS('color','rgb(20, 40, 60)'); + }); + await test.step('And an anonymous visitor sees only the published control', async () => { + const anonymous = await browser.newContext(); + try { + const published = await anonymous.request.get(urls.published); + expect(await published.text()).toContain('Live test published control'); + expect(await published.text()).toContain('SI-LIVE-PUBLISHED-CONTROL'); + expect(await published.text()).not.toContain(marker); + const preview = await anonymous.request.get(urls.preview); + expect(await preview.text()).not.toContain(marker); + } finally { await anonymous.close(); } + }); + }); +} diff --git a/tests/live/observer.spec.js b/tests/live/observer.spec.js new file mode 100644 index 0000000..92aef41 --- /dev/null +++ b/tests/live/observer.spec.js @@ -0,0 +1,31 @@ +const { test, expect } = require('@playwright/test'); +const { randomUUID } = require('node:crypto'); + +// The live runner uses Chromium; this validates its browser instrumentation. +test('Read-only observers recognize a captured preview and its cross-origin SDK handoff', async ({ browser }) => { + const { observeCapture } = require('./run'); + const context = await browser.newContext(); + const evidence = {}; + const marker = `SI-LIVE-${randomUUID()}`; + try { + await observeCapture(context, marker, evidence); + const page = await context.newPage(); + // Synthetic HTTP documents; no request reaches WordPress or Siteimprove. + await page.route('**/*', route => route.fulfill({contentType:'text/html',body:`${marker}`})); + await page.goto('http://localhost:8888/observer-parent'); + await page.evaluate(async () => { + for (const url of ['http://localhost:8888/observer-preview?si_preview_nonce=synthetic', 'https://contentassistant.eu.siteimprove.com/observer']) { + const frame = document.createElement('iframe'); + const loaded = new Promise(resolve => { frame.onload = resolve; }); + frame.src = url; + document.body.appendChild(frame); + await loaded; + } + }); + await expect.poll(() => evidence.preview).toEqual({kind:'preview',marker:true,emptyTitle:true,excludesPublished:true,excludesPlugin:true}); + await page.evaluate(marker => { + window.frames[1].postMessage({si:'contentcheck-flat-dom',data:{dom:{strings:[marker,'rgb(20, 40, 60)']}}},'https://contentassistant.eu.siteimprove.com'); + },marker); + await expect.poll(() => evidence.handoff).toEqual({kind:'handoff',marker:true,style:true,excludesPublished:true}); + } finally { await context.close(); } +}); diff --git a/tests/live/run.js b/tests/live/run.js new file mode 100644 index 0000000..4b47be6 --- /dev/null +++ b/tests/live/run.js @@ -0,0 +1,203 @@ +const { chromium } = require('playwright'); +const { randomUUID } = require('node:crypto'); +const fs = require('node:fs'); +const { loadSettings, isLivePageData } = require('./settings'); +const CMS = 'http://localhost:8888'; +const sleep = ms => new Promise(resolve => setTimeout(resolve, ms)); + +function requireCondition(condition) { if (!condition) throw new Error('Live test condition was not met'); } +async function until(condition, timeout = 30000) { + const end = Date.now() + timeout; + while (Date.now() < end) { if (await condition()) return; await sleep(200); } + throw new Error('Live test condition timed out'); +} +async function visibleOne(locators, timeout = 30000) { + let result; + await until(async () => { + for (const locator of locators) { + const count = await locator.count(); + for (let index = 0; index < count; index++) { + if (await locator.nth(index).isVisible()) { result = locator.nth(index); return true; } + } + } + return false; + }, timeout); + return result; +} + +// Read-only observers: no SDK queue replacement and no modification of submitted +// content or URLs. Only booleans leave the page; account data is never logged. +async function observeCapture(context, marker, evidence) { + await context.exposeBinding('__siLiveEvidence', (source, result) => { + const origin = new URL(source.frame.url()).origin; + if (result.kind === 'preview' && origin === CMS && source.frame.parentFrame()) evidence.preview = result; + if (result.kind === 'handoff' && /^https:\/\/contentassistant\.[a-z]+\.siteimprove\.com$/.test(origin)) evidence.handoff = result; + }); + await context.addInitScript(({ marker, cms }) => { + document.addEventListener('DOMContentLoaded', () => { + if (location.origin === cms && new URL(location.href).searchParams.has('si_preview_nonce')) { + window.__siLiveEvidence({ kind:'preview', marker:document.body.textContent.includes(marker), + emptyTitle:document.querySelectorAll('head > title').length === 1 && document.title.trim() === '', + excludesPublished:!document.body.textContent.includes('SI-LIVE-PUBLISHED-CONTROL'), + excludesPlugin:document.querySelectorAll('script[src*="siteimprove/admin/js/siteimprove.js"]').length === 0, + }); + } + }); + window.addEventListener('message', event => { + if (event.origin !== cms || event.source !== window.parent || event.data?.si !== 'contentcheck-flat-dom') return; + const serialized = JSON.stringify(event.data.data?.dom ?? null); + window.__siLiveEvidence({ kind:'handoff', marker:serialized.includes(marker), + style:serialized.includes('rgb(20, 40, 60)'), + excludesPublished:!serialized.includes('SI-LIVE-PUBLISHED-CONTROL') }); + }); + }, { marker, cms:CMS }); +} + +async function run() { + let browser; + let phase = 'configuration'; + const step = async (name, callback) => { + phase = name; + console.log(`START: ${name}`); + const result = await callback(); + console.log(`PASS: ${name}`); + return result; + }; + try { + requireCondition(process.env.SITEIMPROVE_RUN_LIVE === '1'); + const settings = loadSettings(process.env); + const marker = `SI-LIVE-${randomUUID()}`; + await step('API access and existing Prepublish entitlement', async () => { + const authorization = `Basic ${Buffer.from(`${settings.SITEIMPROVE_API_USERNAME}:${settings.SITEIMPROVE_API_KEY}`).toString('base64')}`; + const response = await fetch('https://api.siteimprove.com/v2/settings/content_checking', { + headers:{Authorization:authorization}, signal:AbortSignal.timeout(30000), redirect:'error', + }); + requireCondition(response.ok && (await response.json()).is_ready === true); + }); + browser = await step('Browser startup', () => chromium.launch()); + const context = await browser.newContext(); // No saved session, trace, video or screenshots. + context.setDefaultTimeout(30000); + const evidence = {}; + await observeCapture(context, marker, evidence); + const page = await context.newPage(); + // The internal crawled website is a mapping context, never a browser target. + await context.route('**/*', route => new URL(route.request().url()).origin === new URL(settings.fixture.crawledUrl).origin + ? route.abort() : route.continue()); + await step('WordPress administrator login', async () => { + await page.goto(`${CMS}/wp-login.php`); + await until(() => page.locator('#user_login').evaluate(element => element === document.activeElement)); + await page.locator('#user_login').fill('admin'); + await page.locator('#user_pass').fill('password'); + await page.locator('#wp-submit').click(); + await page.waitForURL(`${CMS}/wp-admin/**`); + }); + await step('Normal plugin configuration and credential validation', async () => { + await page.goto(`${CMS}/wp-admin/admin.php?page=siteimprove&devmode=1`); + await page.locator('#siteimprove_public_url_field').fill(settings.fixture.publicBase); + await page.locator('#siteimprove_ignore_path_segments_field').fill(''); + await page.locator('#siteimprove_disable_new_version_field').check(); + await page.locator('#siteimprove_overlayjs_file_field').fill('overlay-latest.js'); + if (!(await page.locator('#siteimprove_token_field').inputValue())) { + await page.locator('#siteimprove_token_request').click(); + await until(async () => Boolean(await page.locator('#siteimprove_token_field').inputValue())); + } + await page.locator('#submit').click(); + // Save the URL before API validation, which reads the stored Public URL. + await page.goto(`${CMS}/wp-admin/admin.php?page=siteimprove`); + await page.locator('#siteimprove_api_username_field').fill(settings.SITEIMPROVE_API_USERNAME); + await page.locator('#siteimprove_api_key_field').fill(settings.SITEIMPROVE_API_KEY); + await page.locator('#submit').click(); + await page.getByText('Prepublish feature is already enabled for the current website.', { exact:false }).waitFor(); + }); + const previewUrl = await step('Create a mapped unpublished preview with an empty HTML title', async () => { + await page.goto(`${CMS}/wp-admin/tools.php?page=siteimprove-live-fixture`); + await page.locator('[name=fixture_path]').fill(settings.fixture.path); + await page.locator('[name=fixture_query]').fill(settings.fixture.query); + await page.locator('[name=fixture_marker]').fill(marker); + await page.getByRole('button', {name:'Create live test fixture',exact:true}).click(); + return page.locator('#live-preview').getAttribute('href'); + }); + let liveDataReceived = false; + page.on('response', async response => { + try { + const url = new URL(response.url()); + if (/^contentassistant\.[a-z]+\.siteimprove\.com$/.test(url.hostname) && url.pathname === '/cms/poll' && + url.searchParams.get('url') === settings.fixture.crawledUrl && response.ok()) { + if (isLivePageData(await response.json())) liveDataReceived = true; + } + } catch { /* Login redirects and non-JSON responses cannot satisfy the check. */ } + }); + await step('Verify draft privacy and normal public URL mapping', async () => { + await page.goto(previewUrl); + requireCondition(await page.title() === ''); + requireCondition(await page.locator('head > title').count() === 1); + requireCondition((await page.locator('main').innerText()).includes(marker)); + const input = await page.evaluate(() => ({ url:window.siteimprove_input?.url, enabled:window.php_vars })); + requireCondition(input.url === settings.fixture.crawledUrl); + requireCondition(Number(input.enabled?.prepublish_allowed) === 1 && Number(input.enabled?.prepublish_enabled) === 1); + const anonymous = await browser.newContext(); + try { requireCondition(!(await (await anonymous.request.get(previewUrl)).text()).includes(marker)); } + finally { await anonymous.close(); } + }); + await step('Direct Siteimprove login through the plugin', async () => { + const popupPromise = page.waitForEvent('popup'); + await page.locator('.si-smallbox button.si-button').click(); + const popup = await popupPromise; + await popup.locator('input[name=loginId]').fill(settings.SITEIMPROVE_USERNAME); + await popup.getByRole('button', {name:'Continue',exact:true}).click(); + await popup.locator('input[type=password]').fill(settings.SITEIMPROVE_PASSWORD); + const submit = await visibleOne([popup.getByRole('button', {name:/^(Sign in|Log in|Continue)$/i})]); + const closed = popup.waitForEvent('close', {timeout:60000}); + await submit.click(); + await closed; + // Terms, MFA, CAPTCHA or entitlement failures stop here; never bypass them. + }); + let overlay; + await step('Existing Live page data arrives for the exact mapped URL', async () => { + await until(() => liveDataReceived, 60000); + await page.locator('.si-smallbox button.si-button').click(); + await page.locator('iframe.si-iframe-element').waitFor({state:'visible'}); + overlay = page.frameLocator('iframe.si-iframe-element'); + const liveTab = await visibleOne([overlay.getByRole('tab',{name:/Live page/i}), overlay.getByText('Live page',{exact:true})]); + await liveTab.click(); + }); + await step('Start a fresh Prepublish check of the current draft', async () => { + const prepublishTab = await visibleOne([overlay.getByRole('tab',{name:/Prepublish/i}), overlay.getByText('Prepublish view',{exact:true})]); + await prepublishTab.click(); + const start = await visibleOne([overlay.getByRole('button',{name:/^(Run content check|Recheck draft)$/i})]); + // Clear evidence only when starting this new check. + delete evidence.preview; + delete evidence.handoff; + await start.click(); + // Observe a running state so a previous result cannot satisfy completion. + await overlay.getByRole('button',{name:/Cancel content check/i}).waitFor({state:'visible',timeout:30000}); + await until(() => evidence.preview?.marker && evidence.handoff?.marker, 60000); + requireCondition(evidence.preview.emptyTitle && evidence.preview.excludesPublished && evidence.preview.excludesPlugin); + requireCondition(evidence.handoff.style && evidence.handoff.excludesPublished); + }); + await step('The new check completes and reports the missing title', async () => { + // A new submission was observed above. Require the terminal recheck control, + // no active cancellation control, and the expected issue in Prepublish view. + await overlay.getByRole('button',{name:/^Recheck draft$/i}).waitFor({state:'visible',timeout:180000}); + await until(async () => !(await overlay.getByRole('button',{name:/Cancel content check/i}).isVisible()),180000); + const missingTitle = await visibleOne([ + overlay.getByText(/^(Page has no title|Page title is missing|Missing page title|Page does not have a title|Page is missing a title)$/i), + ],30000); + requireCondition(await missingTitle.isVisible()); + }); + console.log('PASS: Live page data and fresh Prepublish missing-title check'); + if (process.env.GITHUB_STEP_SUMMARY) fs.appendFileSync(process.env.GITHUB_STEP_SUMMARY, + 'Live page data and a fresh Prepublish missing-title check passed. Page-report styling still requires manual inspection. No account data or screenshots were retained.\n'); + return 0; + } catch { + // Never print raw Playwright/API errors: they may include URLs, form values, + // account content, request headers or authentication tokens. + console.error(`FAIL: ${phase}. Account details and raw diagnostics were suppressed.`); + if (process.env.GITHUB_STEP_SUMMARY) fs.appendFileSync(process.env.GITHUB_STEP_SUMMARY, `Live test failed during: ${phase}. No sensitive diagnostics were retained.\n`); + return 1; + } finally { + if (browser) await browser.close().catch(() => {}); + } +} +if (require.main === module) run().then(code => { process.exitCode = code; }); +module.exports = { observeCapture, run }; diff --git a/tests/live/settings.js b/tests/live/settings.js new file mode 100644 index 0000000..9e60ee7 --- /dev/null +++ b/tests/live/settings.js @@ -0,0 +1,36 @@ +const SECRET_NAMES = [ + 'SITEIMPROVE_PUBLIC_URL', 'SITEIMPROVE_CRAWLED_URL', + 'SITEIMPROVE_API_USERNAME', 'SITEIMPROVE_API_KEY', + 'SITEIMPROVE_USERNAME', 'SITEIMPROVE_PASSWORD', +]; + +function fixtureLocation(baseValue, pageValue) { + let base, page; + try { base = new URL(baseValue); page = new URL(pageValue); } catch { throw new Error('Invalid test URL configuration'); } + if (![base, page].every(url => ['http:', 'https:'].includes(url.protocol) && !url.username && !url.password && !url.hash)) { + throw new Error('Invalid test URL configuration'); + } + const basePath = base.pathname.replace(/\/$/, ''); + if (base.search || base.origin !== page.origin || (page.pathname !== basePath && !page.pathname.startsWith(`${basePath}/`))) { + throw new Error('The crawled page must belong to the configured public base URL'); + } + const path = page.pathname.slice(basePath.length) || '/'; + if (path.startsWith('//') || /^\/wp-(admin|login\.php|json)(\/|$)/.test(path)) throw new Error('Unsupported fixture path'); + for (const key of ['preview', 'preview_id', 'preview_nonce', 'si_preview_nonce', 'p', 'page_id', 'rest_route']) { + if (page.searchParams.has(key)) throw new Error('Crawled URL uses a reserved WordPress preview parameter'); + } + return { path, query: page.search.slice(1), publicBase: `${base.origin}${basePath}`, crawledUrl: page.href }; +} + +function loadSettings(env) { + if (SECRET_NAMES.some(name => !env[name]?.trim())) throw new Error('All six Siteimprove secrets are required'); + return { ...Object.fromEntries(SECRET_NAMES.map(name => [name, env[name]])), + fixture: fixtureLocation(env.SITEIMPROVE_PUBLIC_URL, env.SITEIMPROVE_CRAWLED_URL) }; +} + +// Fields used by the current public overlay-latest.js SDK polling contract. +function isLivePageData(body) { + return body?.authed === true && body.error === 'None' && + Number.isFinite(body.issues) && body.issues >= 0 && typeof body.mainUrl === 'string' && body.mainUrl.length > 0; +} +module.exports = { SECRET_NAMES, fixtureLocation, loadSettings, isLivePageData }; diff --git a/tests/live/settings.test.js b/tests/live/settings.test.js new file mode 100644 index 0000000..b1a178e --- /dev/null +++ b/tests/live/settings.test.js @@ -0,0 +1,41 @@ +const { test } = require('node:test'); +const assert = require('node:assert/strict'); +const { fixtureLocation, loadSettings, isLivePageData } = require('./settings'); + +test('A crawled page maps to a local fixture without changing its query', () => { + assert.deepEqual(fixtureLocation('https://public.example.test/site/', 'https://public.example.test/site/articles/example.html?lang=en'), { + path:'/articles/example.html', query:'lang=en', publicBase:'https://public.example.test/site', + crawledUrl:'https://public.example.test/site/articles/example.html?lang=en', + }); +}); +test('A different host or a path outside the public base is rejected', () => { + for (const url of ['https://other.example.test/site/page', 'https://public.example.test/site-other/page']) { + assert.throws(() => fixtureLocation('https://public.example.test/site', url)); + } +}); +test('Credentials, fragments and reserved preview parameters are rejected', () => { + for (const url of ['https://user:pass@public.example.test/page', 'https://public.example.test/page#part', 'https://public.example.test/page?preview=true']) { + assert.throws(() => fixtureLocation('https://public.example.test', url)); + } +}); +test('Missing secrets fail before any browser or network activity', () => assert.throws(() => loadSettings({}))); +test('Zero issues still counts as existing Live page data', () => { + assert.equal(isLivePageData({authed:true,error:'None',issues:0,mainUrl:'https://contentassistant.eu.siteimprove.com/example'}),true); +}); +test('Unauthenticated, missing-page and incomplete responses cannot pass as Live page data', () => { + for (const body of [null, {}, {authed:false,error:'None',issues:3,mainUrl:'example'}, {authed:true,error:'NoPageFound',issues:0,mainUrl:'example'}, {authed:true,error:'None',issues:null,mainUrl:'example'}]) { + assert.equal(isLivePageData(body),false); + } +}); + +test('Configuration failures do not print supplied values or raw exception details', () => { + const { spawnSync } = require('node:child_process'); + const { SECRET_NAMES } = require('./settings'); + const env = { ...process.env, SITEIMPROVE_RUN_LIVE:'1' }; + delete env.GITHUB_STEP_SUMMARY; + for (const name of SECRET_NAMES) env[name] = 'private-sentinel-do-not-log'; + const result = spawnSync(process.execPath, [require.resolve('./run')], {env,encoding:'utf8',timeout:5000}); + assert.equal(result.status,1); + assert.match(result.stderr,/FAIL: configuration/); + assert(!`${result.stdout}${result.stderr}`.includes('private-sentinel-do-not-log')); +}); diff --git a/tests/live/support/fixtures.php b/tests/live/support/fixtures.php new file mode 100644 index 0000000..8266f8a --- /dev/null +++ b/tests/live/support/fixtures.php @@ -0,0 +1,72 @@ + 'page', 'post_status' => 'publish', + 'post_title' => 'Live test published control', 'post_author' => get_current_user_id(), + 'post_content' => '

SI-LIVE-PUBLISHED-CONTROL

', + ), true ); + if ( is_wp_error( $id ) || ! $id ) { wp_die( 'Fixture creation failed.' ); } + update_option( 'siteimprove_live_fixture', array( 'id' => $id, 'path' => $path, 'query' => $query ) ); + require_once ABSPATH . 'wp-admin/includes/post.php'; + $revision = wp_create_post_autosave( array( + 'post_ID' => $id, 'post_type' => 'page', 'post_title' => 'Live test unpublished preview', + 'post_content' => '

' . esc_html( $marker ) . '

Styled draft marker

', + 'post_excerpt' => '', + ) ); + if ( is_wp_error( $revision ) || ! $revision ) { wp_die( 'Autosave creation failed.' ); } + $preview = get_preview_post_link( $id, array( 'preview_id' => $id, 'preview_nonce' => wp_create_nonce( 'post_preview_' . $id ) ) ); + echo 'Open the test preview'; + echo 'Open the published control'; + } + echo '
'; + wp_nonce_field( 'siteimprove-live-fixture' ); + echo ''; + echo '
'; + } ); +} ); + +// Map the local fixture's permalink to the known crawled page's path. The plugin +// still performs its normal Public URL transformation; no SDK request is rewritten. +add_filter( 'page_link', function ( $link, $id ) { + $fixture = get_option( 'siteimprove_live_fixture' ); + if ( ! $fixture || (int) $id !== (int) $fixture['id'] ) { return $link; } + return home_url( $fixture['path'] ) . ( $fixture['query'] ? '?' . $fixture['query'] : '' ); +}, 10, 2 ); +add_filter( 'request', function ( $query ) { + $fixture = get_option( 'siteimprove_live_fixture' ); + $path = wp_parse_url( isset( $_SERVER['REQUEST_URI'] ) ? wp_unslash( $_SERVER['REQUEST_URI'] ) : '', PHP_URL_PATH ); + if ( $fixture && $path === $fixture['path'] ) { + $query = array( 'page_id' => $fixture['id'] ); + if ( isset( $_GET['preview'] ) ) { $query['preview'] = true; } + } + return $query; +} ); +add_filter( 'redirect_canonical', function ( $url ) { + $fixture = get_option( 'siteimprove_live_fixture' ); + return $fixture && get_queried_object_id() === (int) $fixture['id'] ? false : $url; +} ); +add_filter( 'template_include', function ( $template ) { + $fixture = get_option( 'siteimprove_live_fixture' ); + if ( $fixture && get_queried_object_id() === (int) $fixture['id'] ) { + remove_action( 'wp_head', '_wp_render_title_tag', 1 ); + remove_action( 'wp_head', '_block_template_render_title_tag', 1 ); + return __DIR__ . '/templates/preview.php'; + } + return $template; +} ); diff --git a/tests/live/support/templates/preview.php b/tests/live/support/templates/preview.php new file mode 100644 index 0000000..ec16ae4 --- /dev/null +++ b/tests/live/support/templates/preview.php @@ -0,0 +1,16 @@ + + + + + +<?php echo is_preview() ? '' : 'Live test published control'; ?> + + + + +

Prepublish test page

+ + + From f9cce54da22c043a708d68d9e71d06c97ef0c3c3 Mon Sep 17 00:00:00 2001 From: MortenFriisSiteImprove Date: Thu, 10 Sep 2026 21:10:36 +0200 Subject: [PATCH 2/2] Handle live login failures and guard account artifact privacy [skip ci] --- package.json | 2 +- tests/live/README.md | 5 +++- tests/live/run.js | 33 +++++++++++++++----------- tests/live/safety.test.js | 49 +++++++++++++++++++++++++++++++++++++++ 4 files changed, 73 insertions(+), 16 deletions(-) create mode 100644 tests/live/safety.test.js diff --git a/package.json b/package.json index 6dfc4df..e3dba0a 100644 --- a/package.json +++ b/package.json @@ -10,7 +10,7 @@ "test:wordpress": "playwright test --config=playwright.wordpress.config.js", "test:wordpress:report": "playwright show-report playwright-wordpress-report", "test:live": "node tests/live/run.js", - "test:live:contracts": "node --test tests/live/settings.test.js", + "test:live:contracts": "node --test tests/live/*.test.js", "test:live:fixture": "playwright test --config=playwright.live-fixture.config.js" }, "devDependencies": { diff --git a/tests/live/README.md b/tests/live/README.md index 7db6eb0..cbf95dc 100644 --- a/tests/live/README.md +++ b/tests/live/README.md @@ -90,7 +90,10 @@ is separate from the screenshots enabled for the synthetic tests in PR #65. ## Credential-free validation `npm run test:live:contracts` verifies URL mapping, required configuration and -recognition of real Live page data versus login/missing-page responses. +recognition of real Live page data versus login/missing-page responses. It also +checks both login failure paths for unhandled event rejections and guards against +adding screenshot/trace/video/session capture or artifact uploads to the current +live runner and workflow. The PHP fixture can be checked locally without any Siteimprove credentials: diff --git a/tests/live/run.js b/tests/live/run.js index 4b47be6..5922c0d 100644 --- a/tests/live/run.js +++ b/tests/live/run.js @@ -53,6 +53,23 @@ async function observeCapture(context, marker, evidence) { }, { marker, cms:CMS }); } +async function loginToSiteimprove(page, settings) { + // Attach handlers to both promises immediately, including when the click fails. + const [popup] = await Promise.all([ + page.waitForEvent('popup'), + page.locator('.si-smallbox button.si-button').click(), + ]); + await popup.locator('input[name=loginId]').fill(settings.SITEIMPROVE_USERNAME); + await popup.getByRole('button', {name:'Continue',exact:true}).click(); + await popup.locator('input[type=password]').fill(settings.SITEIMPROVE_PASSWORD); + const submit = await visibleOne([popup.getByRole('button', {name:/^(Sign in|Log in|Continue)$/i})]); + await Promise.all([ + popup.waitForEvent('close', {timeout:60000}), + submit.click(), + ]); + // Terms, MFA, CAPTCHA or entitlement failures stop here; never bypass them. +} + async function run() { let browser; let phase = 'configuration'; @@ -139,19 +156,7 @@ async function run() { try { requireCondition(!(await (await anonymous.request.get(previewUrl)).text()).includes(marker)); } finally { await anonymous.close(); } }); - await step('Direct Siteimprove login through the plugin', async () => { - const popupPromise = page.waitForEvent('popup'); - await page.locator('.si-smallbox button.si-button').click(); - const popup = await popupPromise; - await popup.locator('input[name=loginId]').fill(settings.SITEIMPROVE_USERNAME); - await popup.getByRole('button', {name:'Continue',exact:true}).click(); - await popup.locator('input[type=password]').fill(settings.SITEIMPROVE_PASSWORD); - const submit = await visibleOne([popup.getByRole('button', {name:/^(Sign in|Log in|Continue)$/i})]); - const closed = popup.waitForEvent('close', {timeout:60000}); - await submit.click(); - await closed; - // Terms, MFA, CAPTCHA or entitlement failures stop here; never bypass them. - }); + await step('Direct Siteimprove login through the plugin', () => loginToSiteimprove(page, settings)); let overlay; await step('Existing Live page data arrives for the exact mapped URL', async () => { await until(() => liveDataReceived, 60000); @@ -200,4 +205,4 @@ async function run() { } } if (require.main === module) run().then(code => { process.exitCode = code; }); -module.exports = { observeCapture, run }; +module.exports = { observeCapture, loginToSiteimprove, run }; diff --git a/tests/live/safety.test.js b/tests/live/safety.test.js new file mode 100644 index 0000000..3cb8120 --- /dev/null +++ b/tests/live/safety.test.js @@ -0,0 +1,49 @@ +const { test } = require('node:test'); +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const path = require('node:path'); +const { loginToSiteimprove } = require('./run'); +const pause = ms => new Promise(resolve => setTimeout(resolve, ms)); +const delayedFailure = () => new Promise((_, reject) => setTimeout(() => reject(new Error('Synthetic event failure')), 10)); +const settings = {SITEIMPROVE_USERNAME:'synthetic-user',SITEIMPROVE_PASSWORD:'synthetic-password'}; + +test('A failed login click also handles the later popup timeout', async () => { + const page = { + waitForEvent: delayedFailure, + locator: () => ({click:async () => { throw new Error('Synthetic click failure'); }}), + }; + await assert.rejects(loginToSiteimprove(page, settings), /Synthetic click failure/); + // node:test fails this test if the second promise becomes an unhandled rejection. + await pause(30); +}); + +test('A failed password submit also handles the later popup-close failure', async () => { + let clicks = 0; + const button = { + click:async () => { if (++clicks === 2) throw new Error('Synthetic submit failure'); }, + count:async () => 1, nth:() => button, isVisible:async () => true, + }; + const popup = { + locator:() => ({fill:async () => {}}), getByRole:() => button, + waitForEvent: delayedFailure, + }; + const page = {waitForEvent:async () => popup, locator:() => ({click:async () => {}})}; + await assert.rejects(loginToSiteimprove(page, settings), /Synthetic submit failure/); + await pause(30); +}); + +test('The live runner and workflow do not capture or upload account artifacts', () => { + const root = path.resolve(__dirname, '../..'); + const runner = fs.readFileSync(path.join(__dirname, 'run.js'), 'utf8'); + const workflow = fs.readFileSync(path.join(root, '.github/workflows/prepublish-live.yml'), 'utf8'); + // The authenticated runner uses raw Playwright, which has no automatic failure + // screenshots. Guard against introducing explicit capture or recording here. + assert.doesNotMatch(runner, /\.screenshot\s*\(|\.tracing\s*\.|\brecordVideo\s*:|\brecordHar\s*:|\.storageState\s*\(/); + assert.doesNotMatch(workflow, /uses:\s*[^\n]*(?:upload-artifact|upload-pages-artifact)/); + assert.match(workflow, /run: npm run test:live\s/); + assert.doesNotMatch(workflow, /run:\s*(?:npx playwright test|npm run test:live:fixture)/); + const fixtureConfig = require('../../playwright.live-fixture.config'); + assert.equal(fixtureConfig.use.screenshot, 'off'); + assert.equal(fixtureConfig.use.trace, 'off'); + assert.equal(fixtureConfig.use.video, 'off'); +});