From 502e2cb4f2696fbc9364da0ecaac2e8a8052e4ac Mon Sep 17 00:00:00 2001 From: MortenFriisSiteImprove Date: Wed, 16 Sep 2026 09:45:57 +0200 Subject: [PATCH] Verify completed Prepublish results when corrected issue groups disappear --- tests/browser/result-navigation.spec.js | 10 ++++++ tests/live/prepublish.spec.mjs | 15 +++++--- tests/live/result-view.mjs | 3 +- tests/live/safe-reporter.mjs | 2 +- tests/live/scan-results.mjs | 48 +++++++++++++++++++++++++ tests/unit/scan-results.test.cjs | 47 ++++++++++++++++++++++++ 6 files changed, 119 insertions(+), 6 deletions(-) create mode 100644 tests/live/scan-results.mjs create mode 100644 tests/unit/scan-results.test.cjs diff --git a/tests/browser/result-navigation.spec.js b/tests/browser/result-navigation.spec.js index 7f55b43..45a84c1 100644 --- a/tests/browser/result-navigation.spec.js +++ b/tests/browser/result-navigation.spec.js @@ -47,3 +47,13 @@ for (const detailed of [true, false]) { await expect(page.getByRole('button', { name: 'Back', exact: true })).toHaveCount(0); }); } + + +test('completed clean results can omit the Level A group', async ({ page }) => { + const { openAccessibilityResults } = await import('../live/result-view.mjs'); + await page.setContent(` + `); + await openAccessibilityResults(page, { levelRequired: false }); + await expect(page.locator('section')).toBeVisible(); + await expect(page.getByText('Image missing a text alternative', { exact: true })).toHaveCount(0); +}); diff --git a/tests/live/prepublish.spec.mjs b/tests/live/prepublish.spec.mjs index b128a81..8ed0e8c 100644 --- a/tests/live/prepublish.spec.mjs +++ b/tests/live/prepublish.spec.mjs @@ -1,14 +1,16 @@ import { test, expect } from '@playwright/test'; import { openLiveEditor } from './editor.mjs'; +import { observeScanResults } from './scan-results.mjs'; import { settings } from './settings.mjs'; import { observeDraft } from './prepublish.mjs'; import { openAccessibilityResults, openPrepublishOverview, resultViewState } from './result-view.mjs'; import { publicSdkAsset } from './diagnostics.mjs'; import { imageAlternativeRule } from './accessibility-rule.mjs'; -async function scan(page, evidence, marker) { +async function scan(page, evidence, marker, results, issueExpected) { const overlay = page.frameLocator('iframe.si-iframe-element'); const before = evidence[marker]; + const current = results.begin(marker); await test.step('live: start prepublish', async () => { await openPrepublishOverview(overlay); await overlay.getByRole('button', { name: /^(Run content check|Recheck draft)$/i }).click({ timeout: 30_000 }); @@ -23,6 +25,10 @@ async function scan(page, evidence, marker) { await expect(overlay.getByRole('button', { name: /Cancel content check/i })) .toBeHidden({ timeout: Math.max(1, deadline - Date.now()) }); }); + await test.step('live: completed scan result', async () => { + await expect.poll(() => current.result?.issuePresent, { timeout: 60_000 }).toBe(issueExpected); + }); + return current.result; } test('prepublish detects WCAG 1.1.1 image alternative issue and clears it after a saved fix', async ({ page, context }) => { @@ -32,6 +38,7 @@ test('prepublish detects WCAG 1.1.1 image alternative issue and clears it after const fixedMarker = process.env.CMS_DRAFT_FIXED_MARKER; expect(Boolean(marker && fixedMarker && marker !== fixedMarker)).toBe(true); const evidence = await observeDraft(context, config.cmsOrigin, [marker, fixedMarker]); + const results = observeScanResults(page, config.crawledUrl); const sdkAssets = new Set(); const progress = { firstIssueDetected: false, fixedDraftSaved: false, fixedIssueCleared: false }; page.on('response', response => { @@ -49,7 +56,7 @@ test('prepublish detects WCAG 1.1.1 image alternative issue and clears it after expect(published.ok()).toBe(true); expect(await published.text()).not.toContain(marker); await test.step('fresh draft is handed to the SDK and leaves the running state', async () => { - await scan(page, evidence, marker); + await scan(page, evidence, marker, results, true); await test.step('live: accessibility results', async () => { await openAccessibilityResults(page.frameLocator('iframe.si-iframe-element')); }); @@ -70,9 +77,9 @@ test('prepublish detects WCAG 1.1.1 image alternative issue and clears it after const panel = page.locator('iframe.si-iframe-element'); if (!await panel.isVisible()) await page.locator('.si-smallbox button.si-button').click(); await expect(panel).toBeVisible(); - await scan(page, evidence, fixedMarker); + const corrected = await scan(page, evidence, fixedMarker, results, false); await test.step('live: accessibility results', async () => { - await openAccessibilityResults(page.frameLocator('iframe.si-iframe-element')); + await openAccessibilityResults(page.frameLocator('iframe.si-iframe-element'), { levelRequired: corrected.levelAPresent }); }); await test.step('live: WCAG 1.1.1 issue cleared', async () => { const overlay = page.frameLocator('iframe.si-iframe-element'); diff --git a/tests/live/result-view.mjs b/tests/live/result-view.mjs index 47fe6ff..f8836b3 100644 --- a/tests/live/result-view.mjs +++ b/tests/live/result-view.mjs @@ -9,7 +9,7 @@ const levelA = overlay => overlay.getByRole('button', { name: /^Level A(?:\s|$)/ .or(overlay.getByRole('tab', { name: /^Level A(?:\s|$)/ })) .or(overlay.getByText('Level A', { exact: true })).filter({ visible: true }).first(); -export async function openAccessibilityResults(overlay) { +export async function openAccessibilityResults(overlay, { levelRequired = true } = {}) { const issue = overlay.getByText(imageAlternativeRule.label, { exact: true }).filter({ visible: true }); if (await issue.count()) return; if (!await levelA(overlay).isVisible()) { @@ -19,6 +19,7 @@ export async function openAccessibilityResults(overlay) { && await control.getAttribute('aria-selected') !== 'true') await control.click(); } const level = levelA(overlay); + if (!levelRequired && !await level.isVisible()) return; await expect(level).toBeVisible(); if (await level.getAttribute('aria-expanded') !== 'true' && await level.getAttribute('aria-selected') !== 'true') await level.click(); diff --git a/tests/live/safe-reporter.mjs b/tests/live/safe-reporter.mjs index 4dcc2a4..57e4b70 100644 --- a/tests/live/safe-reporter.mjs +++ b/tests/live/safe-reporter.mjs @@ -7,7 +7,7 @@ const stages = new Set(['live: entitlement', 'live: CMS login', 'live: public UR 'live: draft preview', 'live: open login popup', 'live: identity username', 'live: identity password', 'live: submit login', 'live: report panel', 'live: mapped report data', 'live: start prepublish', 'live: draft handoff', 'live: loading-state exit', - 'live: accessibility results', 'live: WCAG 1.1.1 issue detected', 'live: WCAG 1.1.1 issue cleared']); + 'live: completed scan result', 'live: accessibility results', 'live: WCAG 1.1.1 issue detected', 'live: WCAG 1.1.1 issue cleared']); export default class SafeReporter { results = []; diff --git a/tests/live/scan-results.mjs b/tests/live/scan-results.mjs new file mode 100644 index 0000000..c9d6284 --- /dev/null +++ b/tests/live/scan-results.mjs @@ -0,0 +1,48 @@ +import { imageAlternativeRule } from './accessibility-rule.mjs'; + +// Match the result structure rendered by the SDK. Never retain titles or report bodies. +export function summarizeResult(value) { + const data = value?.data; + if (value?.checkInProgress || !Array.isArray(data?.topics) || !Array.isArray(data?.categories) + || !Array.isArray(data?.elements) + || data.elements.some(element => typeof element?.content?.title !== 'string')) return null; + if (!data.topics.some(topic => topic.title === 'Accessibility')) return null; + return { + issuePresent: data.elements.some(element => element.content.title === imageAlternativeRule.label), + levelAPresent: data.categories.some(category => category.title === 'Level A'), + }; +} + +export function observeScanResults(page, crawledUrl) { + let active; + const requests = new WeakMap(); + const endpoint = request => { + try { + const url = new URL(request.url()); + if (!/^https:\/\/contentassistant\.[a-z]+\.siteimprove\.com$/.test(url.origin) + || url.searchParams.get('url') !== crawledUrl) return ''; + return url.pathname; + } catch { return ''; } + }; + page.on('request', request => { + const path = endpoint(request); + if (active && path === '/PrepublishCheck/Upload' && request.method() === 'POST' + && request.postData()?.includes(active.marker)) requests.set(request, { scan: active, upload: true }); + if (active?.accepted && path === '/PrepublishCheck/Results') requests.set(request, { scan: active }); + }); + page.on('response', async response => { + const match = requests.get(response.request()); + if (!match || response.status() !== 200) return; + try { + const value = await response.json(); + if (match.upload) match.scan.accepted = value?.success === true; + else { + const summary = summarizeResult(value); + if (summary) match.scan.result = summary; + } + } catch { /* Missing or unreadable results cannot satisfy the assertion. */ } + }); + return { + begin(marker) { active = { marker, accepted: false, result: null }; return active; }, + }; +} diff --git a/tests/unit/scan-results.test.cjs b/tests/unit/scan-results.test.cjs new file mode 100644 index 0000000..9d5c531 --- /dev/null +++ b/tests/unit/scan-results.test.cjs @@ -0,0 +1,47 @@ +const { test } = require('node:test'); +const assert = require('node:assert/strict'); +const { EventEmitter } = require('node:events'); +const report = issue => ({ checkInProgress: null, data: { + topics: [{ title: 'Accessibility' }], categories: issue ? [{ title: 'Level A' }] : [], + elements: issue ? [{ content: { title: 'Image missing a text alternative' } }] : [], +} }); + +test('scan summaries distinguish complete clean results from missing, running, and malformed results', async () => { + const { summarizeResult } = await import('../live/scan-results.mjs'); + assert.deepEqual(summarizeResult(report(true)), { issuePresent: true, levelAPresent: true }); + assert.deepEqual(summarizeResult(report(false)), { issuePresent: false, levelAPresent: false }); + for (const value of [null, {}, { ...report(false), checkInProgress: {} }, + { data: { topics: [], categories: [], elements: [] } }, + { data: { ...report(false).data, elements: [{}] } }]) assert.equal(summarizeResult(value), null); +}); + +test('scan observer requires a matching accepted upload and a subsequent result request', async () => { + const { observeScanResults } = await import('../live/scan-results.mjs'); + const page = new EventEmitter(); + const url = 'https://public.example/page/'; + const observer = observeScanResults(page, url); + const request = (path, body = '', origin = 'https://contentassistant.eu.siteimprove.com') => ({ + url: () => `${origin}${path}?url=${encodeURIComponent(url)}`, method: () => body ? 'POST' : 'GET', postData: () => body, + }); + const respond = async (req, value) => { + page.emit('response', { request: () => req, status: () => 200, json: async () => value }); + await new Promise(resolve => setImmediate(resolve)); + }; + const first = observer.begin('draft-one'); + const stale = request('/PrepublishCheck/Results'); page.emit('request', stale); + const wrong = request('/PrepublishCheck/Upload', 'draft-other'); page.emit('request', wrong); + await respond(wrong, { success: true }); assert.equal(first.accepted, false); + const upload = request('/PrepublishCheck/Upload', 'draft-one'); page.emit('request', upload); + await respond(upload, { success: true }); assert.equal(first.accepted, true); + await respond(stale, report(false)); assert.equal(first.result, null); + const fresh = request('/PrepublishCheck/Results'); page.emit('request', fresh); + await respond(fresh, report(true)); assert.equal(first.result.issuePresent, true); + const fixed = observer.begin('draft-two'); + await respond(fresh, report(false)); assert.equal(fixed.result, null); + const rejected = request('/PrepublishCheck/Upload', 'draft-two'); page.emit('request', rejected); + await respond(rejected, { success: false }); assert.equal(fixed.accepted, false); + const fixedUpload = request('/PrepublishCheck/Upload', 'draft-two'); page.emit('request', fixedUpload); + await respond(fixedUpload, { success: true }); + const fixedResults = request('/PrepublishCheck/Results'); page.emit('request', fixedResults); + await respond(fixedResults, report(false)); assert.deepEqual(fixed.result, { issuePresent: false, levelAPresent: false }); +});