From 2957c30b77f79e1ddc85007f716ef5416e2ed9fb Mon Sep 17 00:00:00 2001 From: Christopher <131207452+PetrefiedThunder@users.noreply.github.com> Date: Fri, 23 Oct 2026 22:28:42 -0700 Subject: [PATCH 1/5] test(visual): add sitemap crawler for broken links and dead ends (SEL-453) Signed-off-by: Christopher <131207452+PetrefiedThunder@users.noreply.github.com> --- frontend/playwright.visual.config.ts | 15 ++ frontend/tests/visual/README.md | 6 + frontend/tests/visual/crawl.spec.ts | 360 +++++++++++++++++++++++++++ 3 files changed, 381 insertions(+) create mode 100644 frontend/tests/visual/crawl.spec.ts diff --git a/frontend/playwright.visual.config.ts b/frontend/playwright.visual.config.ts index 022a63048..1ef8e88b4 100644 --- a/frontend/playwright.visual.config.ts +++ b/frontend/playwright.visual.config.ts @@ -90,6 +90,21 @@ export default defineConfig({ name: 'smoke', testMatch: ['smoke.spec.ts'], }, + { + name: 'crawl-desktop', + testMatch: ['crawl.spec.ts'], + use: { viewport: { width: 1440, height: 900 }, deviceScaleFactor: 1 }, + }, + { + name: 'crawl-mobile', + testMatch: ['crawl.spec.ts'], + use: { + viewport: { width: 390, height: 844 }, + isMobile: true, + hasTouch: true, + deviceScaleFactor: 1, + }, + }, /* Conditional projects: absent unless the env flag is set. */ ...(allowFormSubmit ? [ diff --git a/frontend/tests/visual/README.md b/frontend/tests/visual/README.md index ca40fe4a7..74814862e 100644 --- a/frontend/tests/visual/README.md +++ b/frontend/tests/visual/README.md @@ -19,6 +19,7 @@ workflow sets automatically (see below). | `priority-extras.visual.spec.ts` | Mobile nav-open, CTA focus ring, tablet home+pricing | | `content.spec.ts` | Copy rules (all `expect.soft`): no em/en dashes, no lone `-`, banned demo words, EPCIS→Preview, Live-label and trust rules | | `smoke.spec.ts` | Page/same-origin console errors, same-origin ≥400s, empty pages; third-party failures attached + annotated, never fail | +| `crawl.spec.ts` | Sitemap crawler: visits every live sitemap URL (rewritten to base origin), flags broken links / dead ends / page+console errors, checks routes.json coverage; writes `test-results-visual/crawl-report-.json` | | `preview-forms.spec.ts` | Blank-submit validation screenshots, login+signup (preview only) | | `signed-in.spec.ts` | Read-only signed-in checks (manual only, `VISUAL_AUTH=1`) | | `layout-boxes.json` | Expected header/nav boxes (empty: `_note` only — see below) | @@ -37,6 +38,9 @@ VISUAL_BASE_URL=https://regengine.co npx playwright test --config=playwright.vis # One project VISUAL_BASE_URL=https://regengine.co npx playwright test --config=playwright.visual.config.ts --project=smoke +# Sitemap crawler at both widths +VISUAL_BASE_URL=https://regengine.co npx playwright test --config=playwright.visual.config.ts --project=crawl-desktop --project=crawl-mobile + # View the last HTML report npx playwright show-report playwright-visual-report ``` @@ -62,6 +66,8 @@ npx playwright show-report playwright-visual-report | `priority-extras` | priority extras | mixed (set per test) | | `content` | content rules | default | | `smoke` | smoke | default | +| `crawl-desktop` | sitemap crawl | 1440×900 | +| `crawl-mobile` | sitemap crawl | 390×844, touch | | `preview-forms` | blank submits | default, only if `VISUAL_ALLOW_FORM_SUBMIT=1` | | `signed-in` | signed-in reads | default, only if `VISUAL_AUTH=1` | diff --git a/frontend/tests/visual/crawl.spec.ts b/frontend/tests/visual/crawl.spec.ts new file mode 100644 index 000000000..a102b6a98 --- /dev/null +++ b/frontend/tests/visual/crawl.spec.ts @@ -0,0 +1,360 @@ +/** + * Sitemap crawler (report-only, read-only GET visits, public pages only). + * + * Fetches /sitemap.xml at runtime (live data, no fixtures), + * rewrites each entry to the base URL's origin so preview deploys work, + * and visits every entry that is not auth-gated or excluded in + * routes.json. Flags broken same-origin links, dead ends, page errors, + * and same-origin console errors. Writes a summary JSON per project to + * test-results-visual/crawl-report-.json. + */ +import { expect, test, type Page } from '@playwright/test'; +import { mkdirSync, writeFileSync } from 'node:fs'; +import path from 'node:path'; +import { routes } from './helpers'; + +/** Full crawl visits every sitemap page sequentially; budget 20 minutes. */ +const CRAWL_TIMEOUT_MS = 1_200_000; + +const NAV_TIMEOUT_MS = 30_000; +const LINK_TIMEOUT_MS = 15_000; +/** Max concurrent link-check GETs; page visits are sequential. */ +const LINK_CONCURRENCY = 6; +const SKIP_EXACT = new Set(['/auth/sso/mock']); + +interface BrokenLink { + source: string; + target: string; + status: number; + error?: string; +} + +interface PageFinding { + path: string; + finalUrl: string; + status: number; + sameOriginLinks: number; + deadEnd: boolean; + deadEndReason: string | null; + pageErrors: string[]; + consoleErrors: string[]; +} + +function baseOriginOf(page: Page): string { + return new URL(test.info().project.use.baseURL ?? page.url()).origin; +} + +/** Extract values from a sitemap or sitemap-index document. */ +function parseLocs(xml: string): string[] { + const locs: string[] = []; + const re = /\s*([^<]+?)\s*<\/loc>/gi; + let match: RegExpExecArray | null; + while ((match = re.exec(xml)) !== null) locs.push(match[1].trim()); + return locs; +} + +function isSitemapUrl(url: string): boolean { + return /\.xml$/i.test(url.split(/[?#]/)[0]); +} + +/** + * Fetch the live sitemap (following a sitemap index one level) and return + * unique pathnames rewritten to the base URL's origin. + */ +async function fetchSitemapPaths(page: Page, baseOrigin: string): Promise { + const first = await page.request.get(`${baseOrigin}/sitemap.xml`, { + timeout: LINK_TIMEOUT_MS, + maxRedirects: 5, + }); + if (!first.ok()) throw new Error(`sitemap.xml GET failed: ${first.status()}`); + let locs = parseLocs(await first.text()); + if (locs.length > 0 && locs.every((loc) => isSitemapUrl(loc))) { + const childPaths: string[] = []; + for (const loc of locs) { + const response = await page.request.get(loc, { + timeout: LINK_TIMEOUT_MS, + maxRedirects: 5, + }); + if (!response.ok()) continue; + childPaths.push(...parseLocs(await response.text())); + } + locs = childPaths; + } + const paths = new Set(); + for (const loc of locs) { + let url: URL; + try { + url = new URL(loc); + } catch { + continue; + } + if (url.protocol !== 'http:' && url.protocol !== 'https:') continue; + // Rewrite to the base URL's origin so preview runs crawl the preview. + paths.add(url.pathname + (url.search || '')); + } + return [...paths].sort(); +} + +function isSkipped(pathname: string): boolean { + const bare = pathname.split('?')[0]; + return ( + SKIP_EXACT.has(bare) || + routes.authOnly.includes(bare) || + bare in routes.excludedReason + ); +} + +/** Normalize an href to a same-origin absolute URL, or null to skip it. */ +function normalizeLink(href: string, pageUrl: string, baseOrigin: string): string | null { + const trimmed = href.trim(); + if (trimmed === '' || trimmed.startsWith('#')) return null; + const lower = trimmed.toLowerCase(); + if (lower.startsWith('mailto:') || lower.startsWith('tel:')) return null; + if (/^[a-z][a-z0-9+.-]*:/i.test(trimmed) && !lower.startsWith('http')) return null; + let url: URL; + try { + url = new URL(trimmed, pageUrl); + } catch { + return null; + } + if (url.origin !== baseOrigin) return null; + if (SKIP_EXACT.has(url.pathname)) return null; + url.hash = ''; + return url.toString(); +} + +async function checkLinks( + page: Page, + targets: Map, +): Promise { + const broken: BrokenLink[] = []; + const entries = [...targets.entries()]; + for (let i = 0; i < entries.length; i += LINK_CONCURRENCY) { + const batch = entries.slice(i, i + LINK_CONCURRENCY); + const results = await Promise.all( + batch.map(async ([target, source]): Promise => { + try { + const response = await page.request.get(target, { + timeout: LINK_TIMEOUT_MS, + maxRedirects: 5, + }); + const status = response.status(); + return status >= 400 + ? { source, target, status } + : null; + } catch (error) { + return { + source, + target, + status: 0, + error: String(error).slice(0, 200), + }; + } + }), + ); + for (const result of results) if (result) broken.push(result); + } + return broken.sort((a, b) => + a.source.localeCompare(b.source) || a.target.localeCompare(b.target), + ); +} + +test('crawl sitemap pages for broken links, dead ends, and errors', async ({ + page, +}, testInfo) => { + test.setTimeout(CRAWL_TIMEOUT_MS); + const baseOrigin = baseOriginOf(page); + const sitemapPaths = await fetchSitemapPaths(page, baseOrigin); + const visitPaths = sitemapPaths.filter((p) => !isSkipped(p)); + const skippedPaths = sitemapPaths.filter((p) => isSkipped(p)); + + const linkSources = new Map(); + const findings: PageFinding[] = []; + + for (const crawlPath of visitPaths) { + const pageErrors: string[] = []; + const consoleErrors: string[] = []; + const onPageError = (error: Error): void => { + pageErrors.push(String(error).slice(0, 500)); + }; + const onConsole = (message: { + type(): string; + text(): string; + location(): { url: string }; + }): void => { + if (message.type() !== 'error') return; + const locationUrl = message.location().url; + if (!locationUrl) return; + try { + if (new URL(locationUrl).origin === baseOrigin) { + consoleErrors.push(message.text().slice(0, 500)); + } + } catch { + // Ignore unparseable location URLs. + } + }; + page.on('pageerror', onPageError); + page.on('console', onConsole); + let status = 0; + try { + const response = await page.goto(crawlPath, { + waitUntil: 'domcontentloaded', + timeout: NAV_TIMEOUT_MS, + }); + status = response?.status() ?? 0; + await page + .waitForLoadState('networkidle', { timeout: 15_000 }) + .catch(() => undefined); + } catch (error) { + pageErrors.push(`navigation failed: ${String(error).slice(0, 300)}`); + } finally { + page.off('pageerror', onPageError); + page.off('console', onConsole); + } + + const pageUrl = page.url(); + const hrefs = await page.locator('a[href]').evaluateAll((anchors) => + anchors.map((a) => (a as HTMLAnchorElement).getAttribute('href') ?? ''), + ); + const scopedHrefs = await page + .locator('main a[href], nav a[href], footer a[href]') + .evaluateAll((anchors) => + anchors.map((a) => (a as HTMLAnchorElement).getAttribute('href') ?? ''), + ); + let scopedSameOrigin = 0; + for (const href of scopedHrefs) { + if (normalizeLink(href, pageUrl, baseOrigin)) scopedSameOrigin += 1; + } + for (const href of hrefs) { + const target = normalizeLink(href, pageUrl, baseOrigin); + if (target && !linkSources.has(target)) linkSources.set(target, crawlPath); + } + + let deadEndReason: string | null = null; + if (status >= 400) { + deadEndReason = `HTTP ${status}`; + } else { + let finalPath = ''; + try { + finalPath = new URL(pageUrl).pathname; + } catch { + finalPath = ''; + } + const h1 = ((await page.locator('h1').first().textContent().catch(() => '')) ?? '').trim(); + if (finalPath === '/404' || finalPath === '/not-found' || h1 === '404') { + deadEndReason = `ends on 404 page (${finalPath || pageUrl})`; + } else if (scopedSameOrigin === 0) { + deadEndReason = 'zero same-origin outbound links in main/nav/footer'; + } + } + + findings.push({ + path: crawlPath, + finalUrl: pageUrl, + status, + sameOriginLinks: scopedSameOrigin, + deadEnd: deadEndReason !== null, + deadEndReason, + pageErrors, + consoleErrors, + }); + } + + const brokenLinks = await checkLinks(page, linkSources); + const deadEnds = findings.filter((f) => f.deadEnd); + const pagesWithPageErrors = findings.filter((f) => f.pageErrors.length > 0); + const pagesWithConsoleErrors = findings.filter((f) => f.consoleErrors.length > 0); + + const report = { + project: testInfo.project.name, + baseOrigin, + fetchedAt: new Date().toISOString(), + sitemapCount: sitemapPaths.length, + visitedCount: visitPaths.length, + skippedCount: skippedPaths.length, + skippedPaths, + uniqueLinksChecked: linkSources.size, + brokenLinks, + deadEnds: deadEnds.map((f) => ({ + path: f.path, + finalUrl: f.finalUrl, + status: f.status, + reason: f.deadEndReason, + })), + pagesWithPageErrors: pagesWithPageErrors.map((f) => ({ + path: f.path, + errors: f.pageErrors, + })), + pagesWithConsoleErrors: pagesWithConsoleErrors.map((f) => ({ + path: f.path, + errors: f.consoleErrors, + })), + pages: findings.map((f) => ({ + path: f.path, + status: f.status, + sameOriginLinks: f.sameOriginLinks, + deadEnd: f.deadEnd, + })), + }; + + const outDir = path.resolve(process.cwd(), 'test-results-visual'); + mkdirSync(outDir, { recursive: true }); + writeFileSync( + path.join(outDir, `crawl-report-${testInfo.project.name}.json`), + JSON.stringify(report, null, 2), + ); + + await testInfo.attach('broken-links', { + body: JSON.stringify(brokenLinks, null, 2), + contentType: 'application/json', + }); + await testInfo.attach('dead-ends', { + body: JSON.stringify(report.deadEnds, null, 2), + contentType: 'application/json', + }); + await testInfo.attach('crawl-page-errors', { + body: JSON.stringify( + { pagesWithPageErrors: report.pagesWithPageErrors, pagesWithConsoleErrors: report.pagesWithConsoleErrors }, + null, + 2, + ), + contentType: 'application/json', + }); + + expect.soft(brokenLinks, 'broken same-origin links (see broken-links attachment)').toEqual([]); + expect.soft( + report.deadEnds, + 'dead-end pages (see dead-ends attachment)', + ).toEqual([]); + expect.soft( + report.pagesWithPageErrors, + 'pages with page errors (see crawl-page-errors attachment)', + ).toEqual([]); + expect.soft( + report.pagesWithConsoleErrors, + 'pages with same-origin console errors (see crawl-page-errors attachment)', + ).toEqual([]); +}); + +test('sitemap paths are covered by routes.json', async ({ page }, testInfo) => { + const baseOrigin = baseOriginOf(page); + const sitemapPaths = await fetchSitemapPaths(page, baseOrigin); + const covered = new Set([ + ...routes.public, + ...routes.priority, + ...routes.skippedDynamic, + ...routes.expectRedirectOr404, + ...routes.authOnly, + ...Object.keys(routes.excludedReason), + ]); + const uncovered = sitemapPaths.filter((p) => !covered.has(p.split('?')[0])); + + await testInfo.attach('uncovered-sitemap-paths', { + body: JSON.stringify(uncovered, null, 2), + contentType: 'application/json', + }); + expect.soft( + uncovered, + 'sitemap paths missing from routes.json (see uncovered-sitemap-paths attachment)', + ).toEqual([]); +}); From af33fc1583e3f02af02ddc13f4f105acfe87bc73 Mon Sep 17 00:00:00 2001 From: Christopher <131207452+PetrefiedThunder@users.noreply.github.com> Date: Fri, 23 Oct 2026 22:39:36 -0700 Subject: [PATCH 2/5] fix(visual): harden sitemap crawler and make it opt-in (SEL-453) Signed-off-by: Christopher <131207452+PetrefiedThunder@users.noreply.github.com> --- frontend/playwright.visual.config.ts | 42 ++++++++------ frontend/tests/visual/README.md | 9 +-- frontend/tests/visual/crawl.spec.ts | 82 ++++++++++++++++++++++------ 3 files changed, 96 insertions(+), 37 deletions(-) diff --git a/frontend/playwright.visual.config.ts b/frontend/playwright.visual.config.ts index 1ef8e88b4..030623abe 100644 --- a/frontend/playwright.visual.config.ts +++ b/frontend/playwright.visual.config.ts @@ -2,9 +2,12 @@ * Report-only visual regression + page monitoring config. * * Read-only by design: every project in this file visits pages with GET only, - * except the two conditional projects below, which do not exist unless their - * env flag is set (so nothing ever shows as "skipped"): + * except the conditional preview-forms project below, which does not exist + * unless its env flag is set (so nothing ever shows as "skipped"). + * Conditional projects (absent unless the env flag is set): * + * - crawl-desktop / crawl-mobile: sitemap crawler (crawl.spec.ts). + * Opt-in via VISUAL_CRAWL=1. No workflow sets VISUAL_CRAWL. * - preview-forms: blank-submit validation screenshots on /login + /signup. * Only on preview deploys, via VISUAL_ALLOW_FORM_SUBMIT=1. Never fills * credentials, never creates an account. @@ -21,6 +24,7 @@ import { defineConfig } from '@playwright/test'; const allowFormSubmit = process.env.VISUAL_ALLOW_FORM_SUBMIT === '1'; const allowSignedIn = process.env.VISUAL_AUTH === '1'; +const allowCrawl = process.env.VISUAL_CRAWL === '1'; export default defineConfig({ testDir: './tests/visual', @@ -90,22 +94,26 @@ export default defineConfig({ name: 'smoke', testMatch: ['smoke.spec.ts'], }, - { - name: 'crawl-desktop', - testMatch: ['crawl.spec.ts'], - use: { viewport: { width: 1440, height: 900 }, deviceScaleFactor: 1 }, - }, - { - name: 'crawl-mobile', - testMatch: ['crawl.spec.ts'], - use: { - viewport: { width: 390, height: 844 }, - isMobile: true, - hasTouch: true, - deviceScaleFactor: 1, - }, - }, /* Conditional projects: absent unless the env flag is set. */ + ...(allowCrawl + ? [ + { + name: 'crawl-desktop', + testMatch: ['crawl.spec.ts'], + use: { viewport: { width: 1440, height: 900 }, deviceScaleFactor: 1 }, + }, + { + name: 'crawl-mobile', + testMatch: ['crawl.spec.ts'], + use: { + viewport: { width: 390, height: 844 }, + isMobile: true, + hasTouch: true, + deviceScaleFactor: 1, + }, + }, + ] + : []), ...(allowFormSubmit ? [ { diff --git a/frontend/tests/visual/README.md b/frontend/tests/visual/README.md index 74814862e..940b6b9c8 100644 --- a/frontend/tests/visual/README.md +++ b/frontend/tests/visual/README.md @@ -38,8 +38,8 @@ VISUAL_BASE_URL=https://regengine.co npx playwright test --config=playwright.vis # One project VISUAL_BASE_URL=https://regengine.co npx playwright test --config=playwright.visual.config.ts --project=smoke -# Sitemap crawler at both widths -VISUAL_BASE_URL=https://regengine.co npx playwright test --config=playwright.visual.config.ts --project=crawl-desktop --project=crawl-mobile +# Sitemap crawler at both widths (opt-in via VISUAL_CRAWL=1) +VISUAL_BASE_URL=https://regengine.co VISUAL_CRAWL=1 npx playwright test --config=playwright.visual.config.ts --project=crawl-desktop --project=crawl-mobile # View the last HTML report npx playwright show-report playwright-visual-report @@ -52,6 +52,7 @@ npx playwright show-report playwright-visual-report | `VISUAL_BASE_URL` | Base URL under test. Default `https://regengine.co` | | `VISUAL_ALLOW_FORM_SUBMIT` | `1` enables the `preview-forms` project (blank submits). Set only by `visual-preview.yml` | | `VISUAL_AUTH` | `1` enables the `signed-in` project. Set by hand for manual runs; no workflow sets it | +| `VISUAL_CRAWL` | `1` enables the `crawl-desktop` / `crawl-mobile` projects. Set by hand for manual runs; no workflow sets it | | `VISUAL_USER_EMAIL` | Expected session email for the signed-in project (asserted, not used to log in) | | `VISUAL_SESSION_TOKEN` | Opaque session cookie value for the signed-in project, provisioned out-of-band | | `VISUAL_UPDATE_BOXES` | `1` records header/nav boxes to `test-results-visual/layout-boxes.candidate.json` (JSONL) instead of asserting | @@ -66,8 +67,8 @@ npx playwright show-report playwright-visual-report | `priority-extras` | priority extras | mixed (set per test) | | `content` | content rules | default | | `smoke` | smoke | default | -| `crawl-desktop` | sitemap crawl | 1440×900 | -| `crawl-mobile` | sitemap crawl | 390×844, touch | +| `crawl-desktop` | sitemap crawl | 1440×900, only if `VISUAL_CRAWL=1` | +| `crawl-mobile` | sitemap crawl | 390×844, touch, only if `VISUAL_CRAWL=1` | | `preview-forms` | blank submits | default, only if `VISUAL_ALLOW_FORM_SUBMIT=1` | | `signed-in` | signed-in reads | default, only if `VISUAL_AUTH=1` | diff --git a/frontend/tests/visual/crawl.spec.ts b/frontend/tests/visual/crawl.spec.ts index a102b6a98..5034c1485 100644 --- a/frontend/tests/visual/crawl.spec.ts +++ b/frontend/tests/visual/crawl.spec.ts @@ -41,7 +41,45 @@ interface PageFinding { } function baseOriginOf(page: Page): string { - return new URL(test.info().project.use.baseURL ?? page.url()).origin; + const raw = + test.info().project.use.baseURL ?? + process.env.VISUAL_BASE_URL ?? + page.url(); + let origin: string; + try { + origin = new URL(raw).origin; + } catch { + throw new Error(`Cannot determine crawl base origin from "${raw}": invalid URL`); + } + if (origin === 'null' || origin === '') { + const fallback = process.env.VISUAL_BASE_URL ?? 'https://regengine.co'; + try { + const fallbackOrigin = new URL(fallback).origin; + if (fallbackOrigin !== 'null' && fallbackOrigin !== '') { + return fallbackOrigin; + } + } catch { + // Fall through to the clear error below. + } + throw new Error( + `Cannot determine crawl base origin from "${raw}": got opaque origin 'null'. ` + + 'Set VISUAL_BASE_URL to a valid http(s) URL.', + ); + } + return origin; +} + +/** Rewrite a sitemap URL to the base origin so preview runs crawl the preview. */ +function rewriteToBaseOrigin(loc: string, baseOrigin: string): string { + try { + const url = new URL(loc, baseOrigin); + const base = new URL(baseOrigin); + url.protocol = base.protocol; + url.host = base.host; + return url.toString(); + } catch { + return loc; + } } /** Extract values from a sitemap or sitemap-index document. */ @@ -71,7 +109,7 @@ async function fetchSitemapPaths(page: Page, baseOrigin: string): Promise 0 && locs.every((loc) => isSitemapUrl(loc))) { const childPaths: string[] = []; for (const loc of locs) { - const response = await page.request.get(loc, { + const response = await page.request.get(rewriteToBaseOrigin(loc, baseOrigin), { timeout: LINK_TIMEOUT_MS, maxRedirects: 5, }); @@ -196,6 +234,7 @@ test('crawl sitemap pages for broken links, dead ends, and errors', async ({ page.on('pageerror', onPageError); page.on('console', onConsole); let status = 0; + let navigationFailed = false; try { const response = await page.goto(crawlPath, { waitUntil: 'domcontentloaded', @@ -206,6 +245,7 @@ test('crawl sitemap pages for broken links, dead ends, and errors', async ({ .waitForLoadState('networkidle', { timeout: 15_000 }) .catch(() => undefined); } catch (error) { + navigationFailed = true; pageErrors.push(`navigation failed: ${String(error).slice(0, 300)}`); } finally { page.off('pageerror', onPageError); @@ -213,21 +253,25 @@ test('crawl sitemap pages for broken links, dead ends, and errors', async ({ } const pageUrl = page.url(); - const hrefs = await page.locator('a[href]').evaluateAll((anchors) => - anchors.map((a) => (a as HTMLAnchorElement).getAttribute('href') ?? ''), - ); - const scopedHrefs = await page - .locator('main a[href], nav a[href], footer a[href]') - .evaluateAll((anchors) => + // Scope links per page: a failed load keeps the previous page's DOM, + // so it must record no links rather than crediting them to this path. + let scopedSameOrigin = 0; + if (!navigationFailed) { + const hrefs = await page.locator('a[href]').evaluateAll((anchors) => anchors.map((a) => (a as HTMLAnchorElement).getAttribute('href') ?? ''), ); - let scopedSameOrigin = 0; - for (const href of scopedHrefs) { - if (normalizeLink(href, pageUrl, baseOrigin)) scopedSameOrigin += 1; - } - for (const href of hrefs) { - const target = normalizeLink(href, pageUrl, baseOrigin); - if (target && !linkSources.has(target)) linkSources.set(target, crawlPath); + const scopedHrefs = await page + .locator('main a[href], nav a[href], footer a[href]') + .evaluateAll((anchors) => + anchors.map((a) => (a as HTMLAnchorElement).getAttribute('href') ?? ''), + ); + for (const href of scopedHrefs) { + if (normalizeLink(href, pageUrl, baseOrigin)) scopedSameOrigin += 1; + } + for (const href of hrefs) { + const target = normalizeLink(href, pageUrl, baseOrigin); + if (target && !linkSources.has(target)) linkSources.set(target, crawlPath); + } } let deadEndReason: string | null = null; @@ -240,7 +284,13 @@ test('crawl sitemap pages for broken links, dead ends, and errors', async ({ } catch { finalPath = ''; } - const h1 = ((await page.locator('h1').first().textContent().catch(() => '')) ?? '').trim(); + const h1 = ( + (await page + .locator('h1') + .first() + .textContent({ timeout: 5_000 }) + .catch(() => '')) ?? '' + ).trim(); if (finalPath === '/404' || finalPath === '/not-found' || h1 === '404') { deadEndReason = `ends on 404 page (${finalPath || pageUrl})`; } else if (scopedSameOrigin === 0) { From 807fdbf13ac8f5db26ac79a996c1693185cb330e Mon Sep 17 00:00:00 2001 From: Christopher <131207452+PetrefiedThunder@users.noreply.github.com> Date: Sat, 24 Oct 2026 01:47:40 -0700 Subject: [PATCH 3/5] fix(visual): skip dead-end check on failed navigation, dedupe config header (SEL-453) --- frontend/playwright.visual.config.ts | 7 +++---- frontend/tests/visual/crawl.spec.ts | 6 ++++-- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/frontend/playwright.visual.config.ts b/frontend/playwright.visual.config.ts index 7ce801ece..6bd8c9eb2 100644 --- a/frontend/playwright.visual.config.ts +++ b/frontend/playwright.visual.config.ts @@ -1,10 +1,9 @@ /** * Report-only visual regression + page monitoring config. * - * Read-only by design: every project in this file visits pages with GET only, - * except the conditional preview-forms project below, which does not exist - * unless its env flag is set (so nothing ever shows as "skipped"). - * Conditional projects (absent unless the env flag is set): + * Read-only by design: every project in this file visits pages with GET only. + * Conditional projects below do not exist unless their env flag is set + * (so nothing ever shows as "skipped"): * * - crawl-desktop / crawl-mobile: sitemap crawler (crawl.spec.ts). * Opt-in via VISUAL_CRAWL=1. No workflow sets VISUAL_CRAWL. diff --git a/frontend/tests/visual/crawl.spec.ts b/frontend/tests/visual/crawl.spec.ts index 5034c1485..1c4771eff 100644 --- a/frontend/tests/visual/crawl.spec.ts +++ b/frontend/tests/visual/crawl.spec.ts @@ -275,9 +275,11 @@ test('crawl sitemap pages for broken links, dead ends, and errors', async ({ } let deadEndReason: string | null = null; - if (status >= 400) { + // A failed navigation leaves stale DOM behind and is already recorded + // in pageErrors, so skip dead-end classification entirely. + if (!navigationFailed && status >= 400) { deadEndReason = `HTTP ${status}`; - } else { + } else if (!navigationFailed) { let finalPath = ''; try { finalPath = new URL(pageUrl).pathname; From d830db243e81c14c9a885576d6911d2e0f38afd7 Mon Sep 17 00:00:00 2001 From: Christopher <131207452+PetrefiedThunder@users.noreply.github.com> Date: Sat, 24 Oct 2026 01:48:29 -0700 Subject: [PATCH 4/5] fix(visual): drop unconditional crawl projects reintroduced by main merge (SEL-453) --- frontend/playwright.visual.config.ts | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/frontend/playwright.visual.config.ts b/frontend/playwright.visual.config.ts index 6bd8c9eb2..8f31abf71 100644 --- a/frontend/playwright.visual.config.ts +++ b/frontend/playwright.visual.config.ts @@ -93,21 +93,6 @@ export default defineConfig({ name: 'smoke', testMatch: ['smoke.spec.ts'], }, - { - name: 'crawl-desktop', - testMatch: ['crawl.spec.ts'], - use: { viewport: { width: 1440, height: 900 }, deviceScaleFactor: 1 }, - }, - { - name: 'crawl-mobile', - testMatch: ['crawl.spec.ts'], - use: { - viewport: { width: 390, height: 844 }, - isMobile: true, - hasTouch: true, - deviceScaleFactor: 1, - }, - }, /* Conditional projects: absent unless the env flag is set. */ ...(allowCrawl ? [ From 3d0262afcac181557d445973bd4a8f3bbaa7e56d Mon Sep 17 00:00:00 2001 From: Christopher <131207452+PetrefiedThunder@users.noreply.github.com> Date: Sat, 24 Oct 2026 18:50:00 -0700 Subject: [PATCH 5/5] fix(crawl): fail on empty or partial sitemaps, decode loc entities, ignore self-links Signed-off-by: Christopher <131207452+PetrefiedThunder@users.noreply.github.com> --- frontend/tests/visual/crawl.spec.ts | 31 +++++++++++++++++++++++++---- 1 file changed, 27 insertions(+), 4 deletions(-) diff --git a/frontend/tests/visual/crawl.spec.ts b/frontend/tests/visual/crawl.spec.ts index 1c4771eff..ac3daba37 100644 --- a/frontend/tests/visual/crawl.spec.ts +++ b/frontend/tests/visual/crawl.spec.ts @@ -82,12 +82,29 @@ function rewriteToBaseOrigin(loc: string, baseOrigin: string): string { } } +/** Decode XML entities in a value; & is decoded last. */ +function decodeXmlEntities(value: string): string { + return value + .replace(/&#x([0-9A-Fa-f]+);/g, (_m: string, hex: string) => + String.fromCodePoint(parseInt(hex, 16)), + ) + .replace(/&#([0-9]+);/g, (_m: string, dec: string) => + String.fromCodePoint(parseInt(dec, 10)), + ) + .replace(/</g, '<') + .replace(/>/g, '>') + .replace(/"/g, '"') + .replace(/'/g, "'") + .replace(/&/g, '&'); +} + /** Extract values from a sitemap or sitemap-index document. */ function parseLocs(xml: string): string[] { const locs: string[] = []; const re = /\s*([^<]+?)\s*<\/loc>/gi; let match: RegExpExecArray | null; - while ((match = re.exec(xml)) !== null) locs.push(match[1].trim()); + while ((match = re.exec(xml)) !== null) + locs.push(decodeXmlEntities(match[1].trim())); return locs; } @@ -109,11 +126,13 @@ async function fetchSitemapPaths(page: Page, baseOrigin: string): Promise 0 && locs.every((loc) => isSitemapUrl(loc))) { const childPaths: string[] = []; for (const loc of locs) { - const response = await page.request.get(rewriteToBaseOrigin(loc, baseOrigin), { + const childUrl = rewriteToBaseOrigin(loc, baseOrigin); + const response = await page.request.get(childUrl, { timeout: LINK_TIMEOUT_MS, maxRedirects: 5, }); - if (!response.ok()) continue; + if (!response.ok()) + throw new Error(`child sitemap GET failed: ${childUrl} status ${response.status()}`); childPaths.push(...parseLocs(await response.text())); } locs = childPaths; @@ -130,6 +149,7 @@ async function fetchSitemapPaths(page: Page, baseOrigin: string): Promise anchors.map((a) => (a as HTMLAnchorElement).getAttribute('href') ?? ''), ); + const normalizedPageUrl = normalizeLink(pageUrl, pageUrl, baseOrigin); for (const href of scopedHrefs) { - if (normalizeLink(href, pageUrl, baseOrigin)) scopedSameOrigin += 1; + const scopedTarget = normalizeLink(href, pageUrl, baseOrigin); + if (scopedTarget && scopedTarget !== normalizedPageUrl) + scopedSameOrigin += 1; } for (const href of hrefs) { const target = normalizeLink(href, pageUrl, baseOrigin);