Wait for live report readiness before opening the panel

docs/live-testing.md

@@ -41,7 +41,7 @@ Existing report content may differ from the synthetic CMS page. The report smoke
 
 After this workflow is merged, choose **Actions → Live Siteimprove smoke test → Run workflow → main**. Complete the configured environment approval. The workflow runs functional and package/CMS tests before the live job; it neither creates a release nor publishes a package.
 
-Only `live-smoke-outcome/result.json` is uploaded from the live run. It contains fixed test names, statuses, durations and the last allowlisted setup stage. Traces, videos, screenshots, raw browser output, cookies, account URLs and report bodies are not uploaded. Host output is discarded in live mode, private browser output is removed, and the disposable database and identity keys are removed on normal completion or handled failure. An abrupt runner termination relies on disposal of the hosted runner.
+Only `live-smoke-outcome/result.json` is uploaded from the live run. It contains fixed test names, statuses, durations and the last allowlisted stage, HTTP status codes and known boolean readiness indicators. Traces, videos, screenshots, raw browser output, cookies, account URLs and report bodies are not uploaded. Host output is discarded in live mode, private browser output is removed, and the disposable database and identity keys are removed on normal completion or handled failure. An abrupt runner termination relies on disposal of the hosted runner.
 
 The browser permits only localhost and HTTPS Siteimprove domains. A changed external login dependency therefore needs review. The CMS's live HTTP handler permits only token acquisition and entitlement reads, with redirects disabled. Automatic public-page rechecks remain disabled. The browser explicitly requests two prepublish scans of synthetic draft content; no public page is published, no remote settings are changed and no subscription is activated.
 
@@ -52,3 +52,5 @@ Do not pass live credentials to `test:cms`. The shared runner rejects Siteimprov
 Unit tests verify configuration rejection, URL separation, report-readiness predicates, request-domain restrictions safe reporting, and marker isolation. Ordinary CMS CI validates six scenarios on each of the two locked CMS profiles, including persisted draft corrections and unchanged published content. Neither proves the real login selectors, live response schema, environment protections or the first live run. Those remain prerequisites before marking this smoke test operational or making it a release gate.
 
 The outcome JSON contains an explicit `skipped` entry for missing-image-alternative result validation. Passing smoke checks establish authentication, report lookup, fresh draft handoff and loading-state exit; they do not establish correct prepublish results. This follows the [WordPress live runner](https://github.com/Siteimprove/CMS-plugin-Wordpress/blob/master/tests/live/run.js).
+
+After login, the test waits for authenticated report data for the mapped URL before opening the report panel, matching the WordPress flow. Diagnostic fields are independently filtered by the reporter; they contain no raw response fields, account text or URLs.

tests/live/diagnostics.mjs

@@ -0,0 +1,12 @@
+const flags = ['entitlementReady', 'pollSeen', 'pollOk', 'pollAuthenticated', 'pollUrlMatches',
+  'pollIssueCountValid', 'pollErrorNone', 'panelFramePresent', 'panelVisible', 'launcherVisible',
+  'blockedExternalRequest'];
+
+// Account text and arbitrary response fields must never enter public artifacts.
+export function safeDiagnostics(value) {
+  const result = {};
+  for (const key of flags) if (typeof value?.[key] === 'boolean') result[key] = value[key];
+  for (const key of ['entitlementStatus', 'pollStatus'])
+    if (Number.isInteger(value?.[key]) && value[key] >= 100 && value[key] <= 599) result[key] = value[key];
+  return result;
+}

tests/live/editor.mjs

@@ -1,81 +1,108 @@
 import { test, expect } from '@playwright/test';
+import { safeDiagnostics } from './diagnostics.mjs';
 import { settings, allowedRequest, isReport } from './settings.mjs';
 
 export async function openLiveEditor(page, context) {
   const config = settings(process.env);
-  await context.route('**/*', route => allowedRequest(route.request().url()) ? route.continue() : route.abort());
-  // Read existing entitlement. Never enable a subscription or trigger a recheck.
-  await test.step('live: entitlement', async () => {
-    const entitlement = await fetch('https://api.siteimprove.com/v2/settings/content_checking', {
-      headers: { Authorization: `Basic ${Buffer.from(`${config.SITEIMPROVE_API_USERNAME}:${config.SITEIMPROVE_API_KEY}`).toString('base64')}` },
-      redirect: 'error', signal: AbortSignal.timeout(30_000),
+  const diagnostic = {};
+  try {
+    await context.route('**/*', route => {
+      if (allowedRequest(route.request().url())) return route.continue();
+      diagnostic.blockedExternalRequest = true;
+      return route.abort();
+    });
+    // Read existing entitlement. Never enable a subscription or trigger a recheck.
+    await test.step('live: entitlement', async () => {
+      const entitlement = await fetch('https://api.siteimprove.com/v2/settings/content_checking', {
+        headers: { Authorization: `Basic ${Buffer.from(`${config.SITEIMPROVE_API_USERNAME}:${config.SITEIMPROVE_API_KEY}`).toString('base64')}` },
+        redirect: 'error', signal: AbortSignal.timeout(30_000),
+      });
+      diagnostic.entitlementStatus = entitlement.status;
+      expect(entitlement.ok).toBe(true);
+      diagnostic.entitlementReady = (await entitlement.json()).is_ready === true;
+      expect(diagnostic.entitlementReady).toBe(true);
     });
-    expect(entitlement.ok).toBe(true);
-    expect((await entitlement.json()).is_ready).toBe(true);
-  });
 
-  let reportReceived = false;
-  page.on('response', async response => {
-    try {
-      const url = new URL(response.url());
-      if (/^https:\/\/contentassistant\.[a-z]+\.siteimprove\.com$/.test(url.origin)
-        && url.pathname === '/cms/poll' && url.searchParams.get('url') === config.crawledUrl && response.ok()
-        && isReport(await response.json(), config.crawledUrl)) reportReceived = true;
-    } catch { /* Invalid or failed responses cannot satisfy report readiness. */ }
-  });
-  await test.step('live: CMS login', async () => {
-    await page.goto('/episerver/cms');
-    await page.locator('input[name="Username"], input[name="UserName"]').fill('editor');
-    await page.locator('input[name="Password"]').fill(process.env.CMS_EDITOR_PASSWORD);
-    await page.getByRole('button', { name: /log in/i }).click();
-    await expect(page.locator('input[name="Password"]')).toHaveCount(0);
-  });
-  const contentId = await test.step('live: public URL mapping', async () => {
-    const routesResponse = await page.request.get('/test/routes');
-    expect(routesResponse.ok()).toBe(true);
-    const { plugin } = await routesResponse.json();
-    const targetResponse = await page.request.get('/test/live-target');
-    expect(targetResponse.ok()).toBe(true);
-    const { contentId } = await targetResponse.json();
-    const mapping = await page.request.get(`${plugin}/PageUrl?contentId=${contentId}&locale=en`);
-    expect(mapping.ok()).toBe(true);
-    expect((await mapping.json()).url).toBe(config.crawledUrl);
-    return contentId;
-  });
-  await test.step('live: draft preview', async () => {
-    await page.goto(`/episerver/cms/#context=epi.cms.contentdata:///${contentId}`);
-    await expect(page.frameLocator('iframe[name="sitePreview"]').locator('h1')).toBeVisible();
-  });
+    let reportReceived = false;
+    page.on('response', async response => {
+      try {
+        const url = new URL(response.url());
+        if (/^https:\/\/contentassistant\.[a-z]+\.siteimprove\.com$/.test(url.origin)
+          && url.pathname === '/cms/poll') {
+          diagnostic.pollSeen = true;
+          diagnostic.pollStatus = response.status();
+          diagnostic.pollOk = response.ok();
+          const body = await response.json();
+          diagnostic.pollAuthenticated = body?.authed === true;
+          diagnostic.pollUrlMatches = url.searchParams.get('url') === config.crawledUrl && body?.mainUrl === config.crawledUrl;
+          diagnostic.pollIssueCountValid = Number.isFinite(body?.issues) && body.issues >= 0;
+          diagnostic.pollErrorNone = body?.error === 'None';
+          if (diagnostic.pollUrlMatches && response.ok() && isReport(body, config.crawledUrl)) reportReceived = true;
+        }
+      } catch { /* Invalid or failed responses cannot satisfy report readiness. */ }
+    });
+    await test.step('live: CMS login', async () => {
+      await page.goto('/episerver/cms');
+      await page.locator('input[name="Username"], input[name="UserName"]').fill('editor');
+      await page.locator('input[name="Password"]').fill(process.env.CMS_EDITOR_PASSWORD);
+      await page.getByRole('button', { name: /log in/i }).click();
+      await expect(page.locator('input[name="Password"]')).toHaveCount(0);
+    });
+    const contentId = await test.step('live: public URL mapping', async () => {
+      const routesResponse = await page.request.get('/test/routes');
+      expect(routesResponse.ok()).toBe(true);
+      const { plugin } = await routesResponse.json();
+      const targetResponse = await page.request.get('/test/live-target');
+      expect(targetResponse.ok()).toBe(true);
+      const { contentId } = await targetResponse.json();
+      const mapping = await page.request.get(`${plugin}/PageUrl?contentId=${contentId}&locale=en`);
+      expect(mapping.ok()).toBe(true);
+      expect((await mapping.json()).url).toBe(config.crawledUrl);
+      return contentId;
+    });
+    await test.step('live: draft preview', async () => {
+      await page.goto(`/episerver/cms/#context=epi.cms.contentdata:///${contentId}`);
+      await expect(page.frameLocator('iframe[name="sitePreview"]').locator('h1')).toBeVisible();
+    });
 
-  const popup = await test.step('live: open login popup', async () => {
-    const [popup] = await Promise.all([
-      page.waitForEvent('popup'), page.locator('.si-smallbox button.si-button').click(),
-    ]);
-    return popup;
-  });
-  await test.step('live: identity username', async () => {
-    await popup.waitForURL(url => url.origin === 'https://identity.siteimprove.com');
-    await popup.locator('input[name=loginId]').fill(config.SITEIMPROVE_USERNAME);
-    await popup.getByRole('button', { name: 'Continue', exact: true }).click();
-  });
-  await test.step('live: identity password', async () => {
-    await popup.locator('input[type=password]').waitFor();
-    expect(new URL(popup.url()).origin).toBe('https://identity.siteimprove.com');
-    await popup.locator('input[type=password]').fill(config.SITEIMPROVE_PASSWORD);
-  });
-  await test.step('live: submit login', async () => {
-    await Promise.all([
-      popup.waitForEvent('close', { timeout: 60_000 }),
-      popup.getByRole('button', { name: /^(Sign in|Log in|Continue)$/i }).click(),
-    ]);
-  });
-  await test.step('live: report panel', async () => {
-    const panel = page.locator('iframe.si-iframe-element');
-    if (!await panel.isVisible()) await page.locator('.si-smallbox button.si-button').click();
-    await expect(panel).toBeVisible();
-  });
-  await test.step('live: mapped report data', async () => {
-    await expect.poll(() => reportReceived, { timeout: 60_000 }).toBe(true);
-  });
-  return { config, contentId };
+    const popup = await test.step('live: open login popup', async () => {
+      const [popup] = await Promise.all([
+        page.waitForEvent('popup'), page.locator('.si-smallbox button.si-button').click(),
+      ]);
+      return popup;
+    });
+    await test.step('live: identity username', async () => {
+      await popup.waitForURL(url => url.origin === 'https://identity.siteimprove.com');
+      await popup.locator('input[name=loginId]').fill(config.SITEIMPROVE_USERNAME);
+      await popup.getByRole('button', { name: 'Continue', exact: true }).click();
+    });
+    await test.step('live: identity password', async () => {
+      await popup.locator('input[type=password]').waitFor();
+      expect(new URL(popup.url()).origin).toBe('https://identity.siteimprove.com');
+      await popup.locator('input[type=password]').fill(config.SITEIMPROVE_PASSWORD);
+    });
+    await test.step('live: submit login', async () => {
+      await Promise.all([
+        popup.waitForEvent('close', { timeout: 60_000 }),
+        popup.getByRole('button', { name: /^(Sign in|Log in|Continue)$/i }).click(),
+      ]);
+    });
+    await test.step('live: mapped report data', async () => {
+      await expect.poll(() => reportReceived, { timeout: 60_000 }).toBe(true);
+    });
+    await test.step('live: report panel', async () => {
+      const panel = page.locator('iframe.si-iframe-element');
+      if (!await panel.isVisible()) await page.locator('.si-smallbox button.si-button').click();
+      await expect(panel).toBeVisible();
+    });
+    return { config, contentId };
+  } finally {
+    try {
+      const panel = page.locator('iframe.si-iframe-element');
+      diagnostic.panelFramePresent = await panel.count() > 0;
+      diagnostic.panelVisible = await panel.first().isVisible();
+      diagnostic.launcherVisible = await page.locator('.si-smallbox button.si-button').first().isVisible();
+    } catch { /* Diagnostics must not replace the original failure. */ }
+    test.info().annotations.push({ type: 'live-diagnostics', description: JSON.stringify(safeDiagnostics(diagnostic)) });
+  }
 }

tests/live/prepublish.spec.mjs

@@ -5,16 +5,22 @@ import { observeDraft } from './prepublish.mjs';
 
 async function scan(page, evidence, marker) {
   const overlay = page.frameLocator('iframe.si-iframe-element');
-  await overlay.getByRole('tab', { name: /Prepublish/i })
-    .or(overlay.getByText('Prepublish view', { exact: true })).first().click();
   const before = evidence[marker];
-  await overlay.getByRole('button', { name: /^(Run content check|Recheck draft)$/i }).click();
-  await expect(overlay.getByRole('button', { name: /Cancel content check/i })).toBeVisible();
-  await expect.poll(() => evidence[marker], { timeout: 60_000 }).toBeGreaterThan(before);
-  const deadline = Date.now() + 300_000;
-  await expect(overlay.getByRole('button', { name: /^Recheck draft$/i })).toBeVisible({ timeout: 300_000 });
-  await expect(overlay.getByRole('button', { name: /Cancel content check/i }))
-    .toBeHidden({ timeout: Math.max(1, deadline - Date.now()) });
+  await test.step('live: start prepublish', async () => {
+    await overlay.getByRole('tab', { name: /Prepublish/i })
+      .or(overlay.getByText('Prepublish view', { exact: true })).first().click();
+    await overlay.getByRole('button', { name: /^(Run content check|Recheck draft)$/i }).click();
+    await expect(overlay.getByRole('button', { name: /Cancel content check/i })).toBeVisible();
+  });
+  await test.step('live: draft handoff', async () => {
+    await expect.poll(() => evidence[marker], { timeout: 60_000 }).toBeGreaterThan(before);
+  });
+  await test.step('live: loading-state exit', async () => {
+    const deadline = Date.now() + 300_000;
+    await expect(overlay.getByRole('button', { name: /^Recheck draft$/i })).toBeVisible({ timeout: 300_000 });
+    await expect(overlay.getByRole('button', { name: /Cancel content check/i }))
+      .toBeHidden({ timeout: Math.max(1, deadline - Date.now()) });
+  });
 }
 
 test('prepublish hands off both saved draft revisions and exits the loading state', async ({ page, context }) => {

tests/live/safe-reporter.mjs

@@ -1,10 +1,12 @@
+import { safeDiagnostics } from './diagnostics.mjs';
 import { mkdirSync, writeFileSync } from 'node:fs';
 
 // Live errors can contain credentials, cookies, URLs and report content.
 // Only fixed test names and statuses may leave the browser run.
 const stages = new Set(['live: entitlement', 'live: CMS login', 'live: public URL mapping',
   'live: draft preview', 'live: open login popup', 'live: identity username',
-  'live: identity password', 'live: submit login', 'live: report panel', 'live: mapped report data']);
+  'live: identity password', 'live: submit login', 'live: report panel', 'live: mapped report data',
+  'live: start prepublish', 'live: draft handoff', 'live: loading-state exit']);
 
 export default class SafeReporter {
   results = [];
@@ -14,8 +16,13 @@ export default class SafeReporter {
       this.stages.set(result, step.title);
   }
   onTestEnd(test, result) {
+    let diagnostics = {};
+    for (const annotation of result.annotations ?? []) {
+      if (annotation.type !== 'live-diagnostics') continue;
+      try { diagnostics = safeDiagnostics(JSON.parse(annotation.description)); } catch {}
+    }
     this.results.push({ test: test.title, status: result.status, durationMs: result.duration,
-      lastStage: this.stages.get(result) ?? null });
+      lastStage: this.stages.get(result) ?? null, diagnostics });
   }
   onError() {}
   onEnd(result) {

tests/unit/live.test.cjs

@@ -60,6 +60,8 @@ test('live reporting discards raw failures and attachments', async () => {
     const reporter = new Reporter();
     reporter.onError(new Error('private-value'));
     const result = { status: 'failed', duration: 1,
+      annotations: [{ type: 'live-diagnostics', description: JSON.stringify({ entitlementStatus: 429,
+        panelVisible: false, pollStatus: 'private-value', url: 'private-value', token: 'private-value' }) }],
       error: { message: 'private-value' }, attachments: [{ body: 'private-value' }] };
     reporter.onStepBegin({}, result, { category: 'test.step', title: 'live: CMS login' });
     reporter.onStepBegin({}, result, { category: 'test.step', title: 'private-value' });
@@ -70,6 +72,7 @@ test('live reporting discards raw failures and attachments', async () => {
     assert.equal(output.includes('private-value'), false);
     assert.equal(JSON.parse(output).tests[0].status, 'failed');
     assert.equal(JSON.parse(output).tests[0].lastStage, 'live: CMS login');
+    assert.deepEqual(JSON.parse(output).tests[0].diagnostics, { panelVisible: false, entitlementStatus: 429 });
   } finally { process.chdir(cwd); fs.rmSync(temp, { recursive: true, force: true }); }
 });
 
@@ -93,3 +96,10 @@ test('draft evidence ignores messages outside the real SDK origin and tracks eac
   assert.equal(evidence['draft-one'], 0);
   assert.equal(evidence['draft-two'], 1);
 });
+
+test('live diagnostics accept only known booleans and HTTP status codes', async () => {
+  const { safeDiagnostics } = await import('../live/diagnostics.mjs');
+  assert.deepEqual(safeDiagnostics({ pollStatus: 200, pollAuthenticated: true, body: 'private-value' }),
+    { pollAuthenticated: true, pollStatus: 200 });
+  assert.deepEqual(safeDiagnostics({ pollStatus: 123456, entitlementStatus: 0, pollAuthenticated: 'private-value' }), {});
+});