Diagnose the live SDK result contract without account data

docs/live-testing.md

@@ -58,3 +58,5 @@ After login, the test waits for authenticated report data for the mapped URL bef
 Report identity is checked against the exact `url` parameter on the SDK polling request. Like WordPress, the response must be authenticated, error-free, include a nonnegative numeric issue count and a nonempty `mainUrl`; that response field is not assumed to equal the crawled page URL.
 
 The test opens the Accessibility results section before asserting the documented image-alternative rule. Safe result diagnostics report only whether the category, target issue, alert, running/recheck controls or nested frames are present/visible. Setup and result diagnostics are merged through the same allowlist; no result text or URLs are retained.
+
+To investigate the result UI contract, sanitized diagnostics may include SDK JavaScript bundle locations from the official contentassistant origin and known static bundle paths. Query strings, fragments, application routes and other origins are excluded. These are software asset locations, not account/report URLs; no bundle contents or account text are uploaded.

tests/live/diagnostics.mjs

@@ -1,13 +1,30 @@
 const flags = ['entitlementReady', 'pollSeen', 'pollOk', 'pollAuthenticated', 'pollUrlMatches',
   'pollMainUrlPresent', 'pollIssueCountValid', 'pollErrorNone', 'panelFramePresent', 'panelVisible', 'launcherVisible',
   'blockedExternalRequest', 'accessibilityCategoryVisible', 'imageIssuePresent', 'imageIssueVisible',
-  'resultAlertVisible', 'resultHasNestedFrame', 'resultRunning', 'resultRecheckVisible'];
+  'resultAlertVisible', 'resultHasNestedFrame', 'resultRunning', 'resultRecheckVisible',
+  'prepublishViewSelected', 'livePageViewSelected', 'firstDraftImagePresent',
+  'fixedDraftImagePresent', 'fixedDraftAlternativePresent'];
+
+// Only static SDK bundles; never application routes or query strings.
+export function publicSdkAsset(value) {
+  try {
+    const url = new URL(value);
+    if (!/^https:\/\/(?:contentassistant\.[a-z]+\.siteimprove\.com|cdn\.siteimprove\.net)$/.test(url.origin)
+      || url.username || url.password || url.hash) return null;
+    if (!/^\/(?:assets|js|scripts|dist|static|bundles|build|content|cms)\/(?:[a-z0-9_-]+\/){0,4}(?:app|main|index|runtime|vendor|vendors|cms|sdk|site|bundle|contentassistant)(?:[.-][a-z0-9_-]+)*\.js$/i.test(url.pathname)) return null;
+    return url.origin + url.pathname;
+  } catch { return null; }
+}
 
 // Account text and arbitrary response fields must never enter public artifacts.
 export function safeDiagnostics(value) {
   const result = {};
   for (const key of flags) if (typeof value?.[key] === 'boolean') result[key] = value[key];
   for (const key of ['entitlementStatus', 'pollStatus'])
     if (Number.isInteger(value?.[key]) && value[key] >= 100 && value[key] <= 599) result[key] = value[key];
+  if (Array.isArray(value?.sdkAssets)) {
+    const assets = value.sdkAssets.filter(item => typeof item === 'string').map(publicSdkAsset).filter(Boolean);
+    result.sdkAssets = [...new Set(assets)].sort().slice(0, 32);
+  }
   return result;
 }

tests/live/prepublish.mjs

@@ -3,16 +3,20 @@ export const isSdkOrigin = origin => /^https:\/\/contentassistant\.[a-z]+\.sitei
 // Observe messages sent by the real integration without replacing the overlay or invoking its queue.
 export async function observeDraft(context, cmsOrigin, markers) {
   const evidence = Object.fromEntries(markers.map(marker => [marker, 0]));
+  evidence.captures = Object.fromEntries(markers.map(marker => [marker, {}]));
   await context.exposeBinding('__cmsDraftEvidence', ({ frame }, value) => {
-    if (isSdkOrigin(new URL(frame.url()).origin) && markers.includes(value?.marker))
+    if (isSdkOrigin(new URL(frame.url()).origin) && markers.includes(value?.marker)) {
       evidence[value.marker]++;
+      evidence.captures[value.marker] = { imagePresent: value.imagePresent === true, fixedAlternativePresent: value.fixedAlternativePresent === true };
+    }
   });
   await context.addInitScript(({ cmsOrigin, markers }) => {
     window.addEventListener('message', event => {
       if (event.origin !== cmsOrigin || event.source !== window.parent || event.data?.si !== 'contentcheck-flat-dom') return;
       const dom = JSON.stringify(event.data.data?.dom ?? null);
       for (const marker of markers)
-        if (dom.includes(marker)) window.__cmsDraftEvidence({ marker });
+        if (dom.includes(marker)) window.__cmsDraftEvidence({ marker, imagePresent: dom.includes('live-test-image'),
+          fixedAlternativePresent: dom.includes('Blue square for the prepublish test') });
     });
   }, { cmsOrigin, markers });
   return evidence;

tests/live/prepublish.spec.mjs

@@ -3,6 +3,7 @@ import { openLiveEditor } from './editor.mjs';
 import { settings } from './settings.mjs';
 import { observeDraft } from './prepublish.mjs';
 import { openAccessibilityResults, resultViewState } from './result-view.mjs';
+import { publicSdkAsset } from './diagnostics.mjs';
 import { imageAlternativeRule } from './accessibility-rule.mjs';
 
 async function scan(page, evidence, marker) {
@@ -32,6 +33,11 @@ test('prepublish detects WCAG 1.1.1 image alternative issue and clears it after
   const fixedMarker = process.env.CMS_DRAFT_FIXED_MARKER;
   expect(Boolean(marker && fixedMarker && marker !== fixedMarker)).toBe(true);
   const evidence = await observeDraft(context, config.cmsOrigin, [marker, fixedMarker]);
+  const sdkAssets = new Set();
+  page.on('response', response => {
+    const asset = publicSdkAsset(response.url());
+    if (asset) sdkAssets.add(asset);
+  });
   await openLiveEditor(page, context);
   try {
     const preview = page.frameLocator('iframe[name="sitePreview"]');
@@ -76,6 +82,10 @@ test('prepublish detects WCAG 1.1.1 image alternative issue and clears it after
       expect(await (await page.request.get(publishedPath)).text()).not.toContain(fixedMarker);
     });
   } finally {
+    test.info().annotations.push({ type: 'live-diagnostics', description: JSON.stringify({ sdkAssets: [...sdkAssets],
+      firstDraftImagePresent: evidence.captures[marker].imagePresent,
+      fixedDraftImagePresent: evidence.captures[fixedMarker].imagePresent,
+      fixedDraftAlternativePresent: evidence.captures[fixedMarker].fixedAlternativePresent }) });
     try {
       const state = await resultViewState(page.frameLocator('iframe.si-iframe-element'));
       test.info().annotations.push({ type: 'live-diagnostics', description: JSON.stringify(state) });

tests/live/result-view.mjs

@@ -17,6 +17,8 @@ export async function openAccessibilityResults(overlay) {
 export async function resultViewState(overlay) {
   const issue = overlay.getByText(imageAlternativeRule.label, { exact: true });
   return {
+    prepublishViewSelected: await overlay.getByRole('tab', { name: /Prepublish/i, selected: true }).count() > 0,
+    livePageViewSelected: await overlay.getByRole('tab', { name: /Live page/i, selected: true }).count() > 0,
     accessibilityCategoryVisible: await category(overlay).isVisible(),
     imageIssuePresent: await issue.count() > 0,
     imageIssueVisible: await issue.filter({ visible: true }).count() > 0,

tests/unit/live.test.cjs

@@ -95,9 +95,10 @@ test('draft evidence ignores messages outside the real SDK origin and tracks eac
   receive({ frame: { url: () => 'https://unrelated.example.test/' } }, { marker: 'draft-one' });
   receive({ frame: { url: () => 'https://contentassistant.eu.siteimprove.com/' } }, { marker: 'unrelated' });
   assert.equal(evidence['draft-one'], 0);
-  receive({ frame: { url: () => 'https://contentassistant.eu.siteimprove.com/' } }, { marker: 'draft-two' });
+  receive({ frame: { url: () => 'https://contentassistant.eu.siteimprove.com/' } }, { marker: 'draft-two', imagePresent: true, fixedAlternativePresent: true });
   assert.equal(evidence['draft-one'], 0);
   assert.equal(evidence['draft-two'], 1);
+  assert.deepEqual(evidence.captures['draft-two'], { imagePresent: true, fixedAlternativePresent: true });
 });
 
 test('live diagnostics accept only known booleans and HTTP status codes', async () => {
@@ -106,3 +107,17 @@ test('live diagnostics accept only known booleans and HTTP status codes', async
     { pollAuthenticated: true, pollStatus: 200 });
   assert.deepEqual(safeDiagnostics({ pollStatus: 123456, entitlementStatus: 0, pollAuthenticated: 'private-value' }), {});
 });
+
+test('SDK asset diagnostics exclude account routes, origins, credentials and query strings', async () => {
+  const { publicSdkAsset, safeDiagnostics } = await import('../live/diagnostics.mjs');
+  const asset = 'https://contentassistant.eu.siteimprove.com/assets/index-abcd1234.js';
+  assert.equal(publicSdkAsset(asset + '?token=private-value'), asset);
+  for (const value of ['https://attacker.example/assets/index.js',
+    'https://user:private-value@contentassistant.eu.siteimprove.com/assets/index.js',
+    'https://contentassistant.eu.siteimprove.com/cms/private-value.js',
+    'https://contentassistant.eu.siteimprove.com/assets/private-value.js',
+    asset + '#private-value', 'not-a-url']) assert.equal(publicSdkAsset(value), null);
+  const result = safeDiagnostics({ sdkAssets: [asset, asset + '?token=private-value', 'private-value'], url: 'private-value' });
+  assert.deepEqual(result, { sdkAssets: [asset] });
+  assert.equal(JSON.stringify(result).includes('private-value'), false);
+});