Identify live smoke failures with safe setup stages

docs/live-testing.md

@@ -41,7 +41,7 @@ Existing report content may differ from the synthetic CMS page. The report smoke
 
 After this workflow is merged, choose **Actions → Live Siteimprove smoke test → Run workflow → main**. Complete the configured environment approval. The workflow runs functional and package/CMS tests before the live job; it neither creates a release nor publishes a package.
 
-Only `live-smoke-outcome/result.json` is uploaded from the live run. It contains fixed test names, statuses and durations. Traces, videos, screenshots, raw browser output, cookies, account URLs and report bodies are not uploaded. Host output is discarded in live mode, private browser output is removed, and the disposable database and identity keys are removed on normal completion or handled failure. An abrupt runner termination relies on disposal of the hosted runner.
+Only `live-smoke-outcome/result.json` is uploaded from the live run. It contains fixed test names, statuses, durations and the last allowlisted setup stage. Traces, videos, screenshots, raw browser output, cookies, account URLs and report bodies are not uploaded. Host output is discarded in live mode, private browser output is removed, and the disposable database and identity keys are removed on normal completion or handled failure. An abrupt runner termination relies on disposal of the hosted runner.
 
 The browser permits only localhost and HTTPS Siteimprove domains. A changed external login dependency therefore needs review. The CMS's live HTTP handler permits only token acquisition and entitlement reads, with redirects disabled. Automatic public-page rechecks remain disabled. The browser explicitly requests two prepublish scans of synthetic draft content; no public page is published, no remote settings are changed and no subscription is activated.
 

tests/live/editor.mjs

@@ -1,16 +1,18 @@
-import { expect } from '@playwright/test';
+import { test, expect } from '@playwright/test';
 import { settings, allowedRequest, isReport } from './settings.mjs';
 
 export async function openLiveEditor(page, context) {
   const config = settings(process.env);
   await context.route('**/*', route => allowedRequest(route.request().url()) ? route.continue() : route.abort());
   // Read existing entitlement. Never enable a subscription or trigger a recheck.
-  const entitlement = await fetch('https://api.siteimprove.com/v2/settings/content_checking', {
-    headers: { Authorization: `Basic ${Buffer.from(`${config.SITEIMPROVE_API_USERNAME}:${config.SITEIMPROVE_API_KEY}`).toString('base64')}` },
-    redirect: 'error', signal: AbortSignal.timeout(30_000),
+  await test.step('live: entitlement', async () => {
+    const entitlement = await fetch('https://api.siteimprove.com/v2/settings/content_checking', {
+      headers: { Authorization: `Basic ${Buffer.from(`${config.SITEIMPROVE_API_USERNAME}:${config.SITEIMPROVE_API_KEY}`).toString('base64')}` },
+      redirect: 'error', signal: AbortSignal.timeout(30_000),
+    });
+    expect(entitlement.ok).toBe(true);
+    expect((await entitlement.json()).is_ready).toBe(true);
   });
-  expect(entitlement.ok).toBe(true);
-  expect((await entitlement.json()).is_ready).toBe(true);
 
   let reportReceived = false;
   page.on('response', async response => {
@@ -21,39 +23,59 @@ export async function openLiveEditor(page, context) {
         && isReport(await response.json(), config.crawledUrl)) reportReceived = true;
     } catch { /* Invalid or failed responses cannot satisfy report readiness. */ }
   });
-  await page.goto('/episerver/cms');
-  await page.locator('input[name="Username"], input[name="UserName"]').fill('editor');
-  await page.locator('input[name="Password"]').fill(process.env.CMS_EDITOR_PASSWORD);
-  await page.getByRole('button', { name: /log in/i }).click();
-  await expect(page.locator('input[name="Password"]')).toHaveCount(0);
-  const routesResponse = await page.request.get('/test/routes');
-  expect(routesResponse.ok()).toBe(true);
-  const { plugin } = await routesResponse.json();
-  const targetResponse = await page.request.get('/test/live-target');
-  expect(targetResponse.ok()).toBe(true);
-  const { contentId } = await targetResponse.json();
-  const mapping = await page.request.get(`${plugin}/PageUrl?contentId=${contentId}&locale=en`);
-  expect(mapping.ok()).toBe(true);
-  expect((await mapping.json()).url).toBe(config.crawledUrl);
-  await page.goto(`/episerver/cms/#context=epi.cms.contentdata:///${contentId}`);
-  await expect(page.frameLocator('iframe[name="sitePreview"]').locator('h1')).toBeVisible();
+  await test.step('live: CMS login', async () => {
+    await page.goto('/episerver/cms');
+    await page.locator('input[name="Username"], input[name="UserName"]').fill('editor');
+    await page.locator('input[name="Password"]').fill(process.env.CMS_EDITOR_PASSWORD);
+    await page.getByRole('button', { name: /log in/i }).click();
+    await expect(page.locator('input[name="Password"]')).toHaveCount(0);
+  });
+  const contentId = await test.step('live: public URL mapping', async () => {
+    const routesResponse = await page.request.get('/test/routes');
+    expect(routesResponse.ok()).toBe(true);
+    const { plugin } = await routesResponse.json();
+    const targetResponse = await page.request.get('/test/live-target');
+    expect(targetResponse.ok()).toBe(true);
+    const { contentId } = await targetResponse.json();
+    const mapping = await page.request.get(`${plugin}/PageUrl?contentId=${contentId}&locale=en`);
+    expect(mapping.ok()).toBe(true);
+    expect((await mapping.json()).url).toBe(config.crawledUrl);
+    return contentId;
+  });
+  await test.step('live: draft preview', async () => {
+    await page.goto(`/episerver/cms/#context=epi.cms.contentdata:///${contentId}`);
+    await expect(page.frameLocator('iframe[name="sitePreview"]').locator('h1')).toBeVisible();
+  });
 
-  const [popup] = await Promise.all([
-    page.waitForEvent('popup'), page.locator('.si-smallbox button.si-button').click(),
-  ]);
-  await popup.waitForURL(url => url.origin === 'https://identity.siteimprove.com');
-  await popup.locator('input[name=loginId]').fill(config.SITEIMPROVE_USERNAME);
-  await popup.getByRole('button', { name: 'Continue', exact: true }).click();
-  await popup.locator('input[type=password]').waitFor();
-  expect(new URL(popup.url()).origin).toBe('https://identity.siteimprove.com');
-  await popup.locator('input[type=password]').fill(config.SITEIMPROVE_PASSWORD);
-  await Promise.all([
-    popup.waitForEvent('close', { timeout: 60_000 }),
-    popup.getByRole('button', { name: /^(Sign in|Log in|Continue)$/i }).click(),
-  ]);
-  const panel = page.locator('iframe.si-iframe-element');
-  if (!await panel.isVisible()) await page.locator('.si-smallbox button.si-button').click();
-  await expect(panel).toBeVisible();
-  await expect.poll(() => reportReceived, { timeout: 60_000 }).toBe(true);
+  const popup = await test.step('live: open login popup', async () => {
+    const [popup] = await Promise.all([
+      page.waitForEvent('popup'), page.locator('.si-smallbox button.si-button').click(),
+    ]);
+    return popup;
+  });
+  await test.step('live: identity username', async () => {
+    await popup.waitForURL(url => url.origin === 'https://identity.siteimprove.com');
+    await popup.locator('input[name=loginId]').fill(config.SITEIMPROVE_USERNAME);
+    await popup.getByRole('button', { name: 'Continue', exact: true }).click();
+  });
+  await test.step('live: identity password', async () => {
+    await popup.locator('input[type=password]').waitFor();
+    expect(new URL(popup.url()).origin).toBe('https://identity.siteimprove.com');
+    await popup.locator('input[type=password]').fill(config.SITEIMPROVE_PASSWORD);
+  });
+  await test.step('live: submit login', async () => {
+    await Promise.all([
+      popup.waitForEvent('close', { timeout: 60_000 }),
+      popup.getByRole('button', { name: /^(Sign in|Log in|Continue)$/i }).click(),
+    ]);
+  });
+  await test.step('live: report panel', async () => {
+    const panel = page.locator('iframe.si-iframe-element');
+    if (!await panel.isVisible()) await page.locator('.si-smallbox button.si-button').click();
+    await expect(panel).toBeVisible();
+  });
+  await test.step('live: mapped report data', async () => {
+    await expect.poll(() => reportReceived, { timeout: 60_000 }).toBe(true);
+  });
   return { config, contentId };
 }

tests/live/safe-reporter.mjs

@@ -2,10 +2,20 @@ import { mkdirSync, writeFileSync } from 'node:fs';
 
 // Live errors can contain credentials, cookies, URLs and report content.
 // Only fixed test names and statuses may leave the browser run.
+const stages = new Set(['live: entitlement', 'live: CMS login', 'live: public URL mapping',
+  'live: draft preview', 'live: open login popup', 'live: identity username',
+  'live: identity password', 'live: submit login', 'live: report panel', 'live: mapped report data']);
+
 export default class SafeReporter {
   results = [];
+  stages = new Map();
+  onStepBegin(test, result, step) {
+    if (step.category === 'test.step' && stages.has(step.title))
+      this.stages.set(result, step.title);
+  }
   onTestEnd(test, result) {
-    this.results.push({ test: test.title, status: result.status, durationMs: result.duration });
+    this.results.push({ test: test.title, status: result.status, durationMs: result.duration,
+      lastStage: this.stages.get(result) ?? null });
   }
   onError() {}
   onEnd(result) {

tests/unit/live.test.cjs

@@ -59,12 +59,17 @@ test('live reporting discards raw failures and attachments', async () => {
     process.chdir(temp);
     const reporter = new Reporter();
     reporter.onError(new Error('private-value'));
-    reporter.onTestEnd({ title: 'fixed smoke test' }, { status: 'failed', duration: 1,
-      error: { message: 'private-value' }, attachments: [{ body: 'private-value' }] });
+    const result = { status: 'failed', duration: 1,
+      error: { message: 'private-value' }, attachments: [{ body: 'private-value' }] };
+    reporter.onStepBegin({}, result, { category: 'test.step', title: 'live: CMS login' });
+    reporter.onStepBegin({}, result, { category: 'test.step', title: 'private-value' });
+    reporter.onStepBegin({}, result, { category: 'pw:api', title: 'private-value' });
+    reporter.onTestEnd({ title: 'fixed smoke test' }, result);
     reporter.onEnd({ status: 'failed' });
     const output = fs.readFileSync('artifacts/live/result.json', 'utf8');
     assert.equal(output.includes('private-value'), false);
     assert.equal(JSON.parse(output).tests[0].status, 'failed');
+    assert.equal(JSON.parse(output).tests[0].lastStage, 'live: CMS login');
   } finally { process.chdir(cwd); fs.rmSync(temp, { recursive: true, force: true }); }
 });