diff --git a/.github/workflows/temporary-smoke-fail-closed.yml b/.github/workflows/temporary-smoke-fail-closed.yml new file mode 100644 index 000000000..71608033d --- /dev/null +++ b/.github/workflows/temporary-smoke-fail-closed.yml @@ -0,0 +1,91 @@ +name: Temporary smoke fail-closed repair + +on: + push: + branches: + - codex/smoke-late-error-guard + +permissions: + contents: write + +jobs: + repair: + name: Repair fail-open mocked API boundary + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - name: Harden the runner (Audit all outbound calls) + uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0 + with: + egress-policy: audit + + - name: Checkout exact staging head + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: true + ref: ${{ github.sha }} + + - name: Set up Node.js + uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 + with: + node-version: "24" + + - name: Enable pnpm + run: corepack enable pnpm + + - name: Verify single-writer head + shell: bash + run: | + set -euo pipefail + remote_head="$(git ls-remote origin "refs/heads/${GITHUB_REF_NAME}" | awk '{print $1}')" + test "${remote_head}" = "${GITHUB_SHA}" + + - name: Apply bounded source repair + run: python scripts/ci/temporary_smoke_fail_closed.py + + - name: Verify bounded source delta + shell: bash + run: | + set -euo pipefail + git diff --check + git diff --name-only | sort > /tmp/actual-paths + printf '%s\n' \ + frontend/scripts/full-product-ui-smoke.mjs \ + frontend/scripts/full-product-ui-smoke.test.mjs \ + > /tmp/expected-paths + diff -u /tmp/expected-paths /tmp/actual-paths + + - name: Install frontend dependencies + run: cd frontend && pnpm install --frozen-lockfile + + - name: Run focused regression and lint + shell: bash + run: | + set -euo pipefail + cd frontend + pnpm exec vitest run scripts/full-product-ui-smoke.test.mjs + pnpm exec eslint scripts/full-product-ui-smoke.mjs scripts/full-product-ui-smoke.test.mjs --max-warnings 0 + + - name: Commit and push product delta non-force + shell: bash + env: + BRANCH_NAME: codex/smoke-late-error-guard + run: | + set -euo pipefail + remote_head="$(git ls-remote origin "refs/heads/${BRANCH_NAME}" | awk '{print $1}')" + if [[ "${remote_head}" != "${GITHUB_SHA}" ]]; then + echo "Remote head moved: expected ${GITHUB_SHA}, got ${remote_head}" >&2 + exit 1 + fi + git add -- frontend/scripts/full-product-ui-smoke.mjs frontend/scripts/full-product-ui-smoke.test.mjs + git diff --cached --check + git diff --cached --name-only | sort > /tmp/actual-staged-paths + printf '%s\n' \ + frontend/scripts/full-product-ui-smoke.mjs \ + frontend/scripts/full-product-ui-smoke.test.mjs \ + > /tmp/expected-staged-paths + diff -u /tmp/expected-staged-paths /tmp/actual-staged-paths + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + git commit -m "fix(smoke): fail closed on unregistered mocked APIs" + git push origin "HEAD:${BRANCH_NAME}" diff --git a/frontend/scripts/full-product-ui-smoke-screenshot-failure.test.mjs b/frontend/scripts/full-product-ui-smoke-screenshot-failure.test.mjs new file mode 100644 index 000000000..70904ac04 --- /dev/null +++ b/frontend/scripts/full-product-ui-smoke-screenshot-failure.test.mjs @@ -0,0 +1,80 @@ +import { mkdtemp, readdir, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import path from "node:path"; + +import { describe, expect, it } from "vitest"; + +import { FULL_PRODUCT_ROUTES, runRouteSmoke } from "./full-product-ui-smoke.mjs"; + +function createRoutePageThatCannotCaptureScreenshot() { + let evaluationCount = 0; + let screenshotAttempts = 0; + let closed = false; + + return { + page: { + on: () => {}, + route: async () => {}, + goto: async () => {}, + waitForLoadState: async () => {}, + locator: () => ({ waitFor: async () => {}, innerText: async () => "Naruon" }), + evaluate: async () => { + evaluationCount += 1; + if (evaluationCount === 1) return { duplicateIds: [], unnamedInteractive: [] }; + return { tagName: "BUTTON" }; + }, + keyboard: { press: async () => {} }, + waitForTimeout: async () => {}, + screenshot: async () => { + screenshotAttempts += 1; + throw new Error("screenshot-backend-unavailable"); + }, + close: async () => { + closed = true; + }, + }, + evidence: { + get screenshotAttempts() { + return screenshotAttempts; + }, + get closed() { + return closed; + }, + }, + }; +} + +describe("full-product screenshot evidence", () => { + it("fails the route after both screenshot attempts fail while still closing the page", async () => { + const screenshotDirectory = await mkdtemp(path.join(tmpdir(), "naruon-full-product-smoke-failure-")); + const { page, evidence } = createRoutePageThatCannotCaptureScreenshot(); + + try { + await expect( + runRouteSmoke( + { newPage: async () => page }, + FULL_PRODUCT_ROUTES[0], + { name: "desktop", width: 1440, height: 1024 }, + 1, + screenshotDirectory, + ), + ).rejects.toThrow("screenshot-backend-unavailable"); + expect(evidence.screenshotAttempts).toBe(2); + expect(evidence.closed).toBe(true); + expect(await readdir(screenshotDirectory)).toEqual([]); + } finally { + await rm(screenshotDirectory, { recursive: true, force: true }); + } + }); + + it("closes the page when navigation fails before capture", async () => { + const { page, evidence } = createRoutePageThatCannotCaptureScreenshot(); + page.goto = async () => { throw new Error("navigation-unavailable"); }; + await expect(runRouteSmoke( + { newPage: async () => page }, FULL_PRODUCT_ROUTES[0], + { name: "desktop", width: 1440, height: 1024 }, 1, tmpdir(), + )).rejects.toThrow("navigation-unavailable"); + expect(evidence.screenshotAttempts).toBe(0); + expect(evidence.closed).toBe(true); + }); +}); diff --git a/frontend/scripts/full-product-ui-smoke.mjs b/frontend/scripts/full-product-ui-smoke.mjs index 008d61267..9efdf2aaf 100644 --- a/frontend/scripts/full-product-ui-smoke.mjs +++ b/frontend/scripts/full-product-ui-smoke.mjs @@ -1,5 +1,5 @@ import { execFile, spawn } from "node:child_process"; -import { access, mkdtemp, writeFile } from "node:fs/promises"; +import { access, mkdtemp } from "node:fs/promises"; import net from "node:net"; import { tmpdir } from "node:os"; import path from "node:path"; @@ -187,7 +187,7 @@ function log(message) { process.stdout.write(`${message}\n`); } -async function captureSmokeScreenshot(page, screenshotPath, label) { +async function captureSmokeScreenshot(page, screenshotPath) { for (let attempt = 1; attempt <= 2; attempt += 1) { try { await page.screenshot({ path: screenshotPath, fullPage: false }); @@ -197,16 +197,8 @@ async function captureSmokeScreenshot(page, screenshotPath, label) { await page.waitForTimeout(250); continue; } - const diagnosticFileName = path.basename(screenshotPath).replace(/\.png$/u, ".screenshot-failed.txt"); - const diagnosticPath = resolveFullProductArtifactPath(path.dirname(screenshotPath), diagnosticFileName); - const reason = error instanceof Error ? error.message : String(error); - await writeFile( - diagnosticPath, - `screenshot_failed label=${label}\nreason=${reason}\n`, - "utf-8", - ); - log(`Screenshot capture failed for ${label}: ${reason}`); - return diagnosticPath; + log("Screenshot capture failed after both attempts"); + throw error; } } return screenshotPath; @@ -777,7 +769,7 @@ function routeJson(route, body, status = 200) { }); } -async function installRoutes(page) { +async function installRoutes(page, unhandledApiRequests = new Set()) { let emailSendCount = 0; let savedAccountConfig = { ...accountConfig }; let savedLlmProviders = [{ ...llmProvider }]; @@ -1080,7 +1072,8 @@ async function installRoutes(page) { if (endpoint.startsWith("/api/tools/")) return routeJson(route, { output: "ok", status: "success" }); if (endpoint === "/api/runtime-config") return routeJson(route, {}); - return routeJson(route, { ok: true }); + unhandledApiRequests.add(`${request.method()} ${endpoint}`); + return route.abort("failed"); }); } @@ -1555,45 +1548,56 @@ async function runAccessibilitySmoke(page, routeSpec) { return [`${routeSpec.name}:a11y-basics`]; } -async function runRouteSmoke(context, routeSpec, viewportSpec, viewportCount, screenshotDir) { +export async function runRouteSmoke(context, routeSpec, viewportSpec, viewportCount, screenshotDir) { const page = await context.newPage(); const consoleErrors = []; - page.on("console", (message) => { - if (message.type() === "error") { - consoleErrors.push(`${message.type()}: ${message.text()}`); + const unhandledApiRequests = new Set(); + let screenshotArtifact; + let interactionEvidence; + let accessibilityEvidence; + try { + page.on("console", (message) => { + if (message.type() === "error") { + consoleErrors.push(`${message.type()}: ${message.text()}`); + } + }); + page.on("pageerror", (error) => consoleErrors.push(`pageerror: ${error.message}`)); + await installRoutes(page, unhandledApiRequests); + await page.goto(new URL(routeSpec.path, baseUrl).href, { waitUntil: "domcontentloaded" }); + await page.waitForLoadState("networkidle", { timeout: 10_000 }).catch(() => {}); + await page.locator("body").waitFor({ state: "visible", timeout: 20_000 }); + const bodyText = await page.locator("body").innerText({ timeout: 10_000 }); + const expectedTexts = Array.isArray(routeSpec.expectedText) ? routeSpec.expectedText : [routeSpec.expectedText]; + if (!expectedTexts.some((expectedText) => bodyText.includes(expectedText))) { + const bodySnippet = bodyText.replace(/\s+/g, " ").trim().slice(0, 500); + throw new Error( + `Route ${routeSpec.path} did not render expected text: ${expectedTexts.join(" or ")}. Body snippet: ${bodySnippet}`, + ); } - }); - page.on("pageerror", (error) => consoleErrors.push(`pageerror: ${error.message}`)); - await installRoutes(page); - await page.goto(new URL(routeSpec.path, baseUrl).href, { waitUntil: "domcontentloaded" }); - await page.waitForLoadState("networkidle", { timeout: 10_000 }).catch(() => {}); - await page.locator("body").waitFor({ state: "visible", timeout: 20_000 }); - const bodyText = await page.locator("body").innerText({ timeout: 10_000 }); - const expectedTexts = Array.isArray(routeSpec.expectedText) ? routeSpec.expectedText : [routeSpec.expectedText]; - if (!expectedTexts.some((expectedText) => bodyText.includes(expectedText))) { - const bodySnippet = bodyText.replace(/\s+/g, " ").trim().slice(0, 500); - throw new Error( - `Route ${routeSpec.path} did not render expected text: ${expectedTexts.join(" or ")}. Body snippet: ${bodySnippet}`, + if (bodyText.includes("404") || bodyText.includes("This page could not be found")) { + throw new Error(`Route ${routeSpec.path} rendered a not-found page`); + } + if (consoleErrors.length > 0) { + throw new Error(`Route ${routeSpec.path} emitted console errors:\n${consoleErrors.join("\n")}`); + } + interactionEvidence = await runCriticalInteractionSmoke(page, routeSpec, viewportSpec); + accessibilityEvidence = await runAccessibilitySmoke(page, routeSpec); + const screenshotPath = resolveFullProductArtifactPath( + screenshotDir, + fullProductScreenshotName(routeSpec, viewportSpec, viewportCount), ); + screenshotArtifact = await captureSmokeScreenshot(page, screenshotPath); + } finally { + await page.close(); } - if (bodyText.includes("404") || bodyText.includes("This page could not be found")) { - throw new Error(`Route ${routeSpec.path} rendered a not-found page`); + if (unhandledApiRequests.size > 0) { + throw new Error( + `Route ${routeSpec.path} requested unregistered mocked APIs:\n${[...unhandledApiRequests].join("\n")}`, + ); } if (consoleErrors.length > 0) { throw new Error(`Route ${routeSpec.path} emitted console errors:\n${consoleErrors.join("\n")}`); } - const interactionEvidence = await runCriticalInteractionSmoke(page, routeSpec, viewportSpec); - const accessibilityEvidence = await runAccessibilitySmoke(page, routeSpec); - const screenshotPath = resolveFullProductArtifactPath( - screenshotDir, - fullProductScreenshotName(routeSpec, viewportSpec, viewportCount), - ); - const screenshotArtifact = await captureSmokeScreenshot( - page, - screenshotPath, - `${viewportSpec.name}:${routeSpec.path}`, - ); - await page.close(); return { screenshotPath: screenshotArtifact, interactionEvidence, accessibilityEvidence }; } diff --git a/frontend/scripts/full-product-ui-smoke.test.mjs b/frontend/scripts/full-product-ui-smoke.test.mjs index 4e373a399..cfdd12363 100644 --- a/frontend/scripts/full-product-ui-smoke.test.mjs +++ b/frontend/scripts/full-product-ui-smoke.test.mjs @@ -19,8 +19,92 @@ import { resolveFullProductChromePath, resolveFullProductScreenshotProfile, resolveFullProductViewportSpecs, + runRouteSmoke, } from "./full-product-ui-smoke.mjs"; +describe("route smoke API fixture boundary", () => { + it("fails closed when the product requests an unregistered mocked API", async () => { + let apiRouteHandler; + let abortReason; + let evaluationCount = 0; + const page = { + on: () => {}, + route: async (pattern, handler) => { + if (pattern === "**/api/**") apiRouteHandler = handler; + }, + goto: async () => { + expect(apiRouteHandler).toBeTypeOf("function"); + await apiRouteHandler({ + request: () => ({ + url: () => "http://127.0.0.1:3001/api/unregistered-smoke-endpoint", + method: () => "GET", + }), + abort: async (reason) => { + abortReason = reason; + }, + }); + }, + waitForLoadState: async () => {}, + locator: () => ({ waitFor: async () => {}, innerText: async () => "Naruon" }), + evaluate: async () => { + evaluationCount += 1; + if (evaluationCount === 1) return { duplicateIds: [], unnamedInteractive: [] }; + return { tagName: "BUTTON" }; + }, + keyboard: { press: async () => {} }, + screenshot: async () => {}, + close: async () => {}, + }; + + await expect(runRouteSmoke( + { newPage: async () => page }, + FULL_PRODUCT_ROUTES[0], + { name: "desktop", width: 1440, height: 1024 }, + 1, + path.join(tmpdir(), "naruon-full-product-smoke-unit"), + )).rejects.toThrow("GET /api/unregistered-smoke-endpoint"); + expect(abortReason).toBe("failed"); + }); +}); + +describe("route smoke late browser errors", () => { + it.each(["console", "pageerror"])("rejects %s errors emitted during page close", async (eventName) => { + const handlers = {}; + let evaluationCount = 0; + let closeCalled = false; + const page = { + on: (name, handler) => { handlers[name] = handler; }, + route: async () => {}, + goto: async () => {}, + waitForLoadState: async () => {}, + locator: () => ({ waitFor: async () => {}, innerText: async () => "Naruon" }), + evaluate: async () => { + evaluationCount += 1; + if (evaluationCount === 1) return { duplicateIds: [], unnamedInteractive: [] }; + return { tagName: "BUTTON" }; + }, + keyboard: { press: async () => {} }, + screenshot: async () => {}, + close: async () => { + closeCalled = true; + if (eventName === "console") { + handlers.console({ type: () => "error", text: () => "late-browser-failure" }); + } else { + handlers.pageerror(new Error("late-browser-failure")); + } + }, + }; + await expect(runRouteSmoke( + { newPage: async () => page }, + FULL_PRODUCT_ROUTES[0], + { name: "desktop", width: 1440, height: 1024 }, + 1, + path.join(tmpdir(), "naruon-full-product-smoke-unit"), + )).rejects.toThrow("late-browser-failure"); + expect(closeCalled).toBe(true); + }); +}); + describe("full product UI smoke base URL guard", () => { it("allows localhost full-product smoke targets", () => { expect(resolveFullProductBaseUrl("http://127.0.0.1:3001").hostname).toBe("127.0.0.1"); diff --git a/scripts/ci/temporary_smoke_fail_closed.py b/scripts/ci/temporary_smoke_fail_closed.py new file mode 100644 index 000000000..b99793eac --- /dev/null +++ b/scripts/ci/temporary_smoke_fail_closed.py @@ -0,0 +1,99 @@ +from pathlib import Path + +source_path = Path("frontend/scripts/full-product-ui-smoke.mjs") +test_path = Path("frontend/scripts/full-product-ui-smoke.test.mjs") +source = source_path.read_text(encoding="utf-8") +tests = test_path.read_text(encoding="utf-8") + +replacements = [ + ( + "async function installRoutes(page) {", + "async function installRoutes(page, unhandledApiRequests = new Set()) {", + ), + ( + " return routeJson(route, { ok: true });\n", + " unhandledApiRequests.add(`${request.method()} ${endpoint}`);\n" + " return route.abort(\"failed\");\n", + ), + ( + " const consoleErrors = [];\n let screenshotArtifact;", + " const consoleErrors = [];\n const unhandledApiRequests = new Set();\n let screenshotArtifact;", + ), + ( + " await installRoutes(page);", + " await installRoutes(page, unhandledApiRequests);", + ), + ( + " if (consoleErrors.length > 0) {\n throw new Error(`Route ${routeSpec.path} emitted console errors:\\n${consoleErrors.join(\"\\n\")}`);\n }\n return { screenshotPath: screenshotArtifact, interactionEvidence, accessibilityEvidence };", + " if (unhandledApiRequests.size > 0) {\n" + " throw new Error(\n" + " `Route ${routeSpec.path} requested unregistered mocked APIs:\\n${[...unhandledApiRequests].join(\"\\n\")}`,\n" + " );\n" + " }\n" + " if (consoleErrors.length > 0) {\n" + " throw new Error(`Route ${routeSpec.path} emitted console errors:\\n${consoleErrors.join(\"\\n\")}`);\n" + " }\n" + " return { screenshotPath: screenshotArtifact, interactionEvidence, accessibilityEvidence };", + ), +] + +for old, new in replacements: + count = source.count(old) + if count != 1: + raise SystemExit(f"expected one source match, found {count}: {old[:80]!r}") + source = source.replace(old, new, 1) + +marker = 'describe("route smoke late browser errors", () => {' +if tests.count(marker) != 1: + raise SystemExit("route-smoke test insertion marker changed") + +regression = r'''describe("route smoke API fixture boundary", () => { + it("fails closed when the product requests an unregistered mocked API", async () => { + let apiRouteHandler; + let abortReason; + let evaluationCount = 0; + const page = { + on: () => {}, + route: async (pattern, handler) => { + if (pattern === "**/api/**") apiRouteHandler = handler; + }, + goto: async () => { + expect(apiRouteHandler).toBeTypeOf("function"); + await apiRouteHandler({ + request: () => ({ + url: () => "http://127.0.0.1:3001/api/unregistered-smoke-endpoint", + method: () => "GET", + }), + abort: async (reason) => { + abortReason = reason; + }, + }); + }, + waitForLoadState: async () => {}, + locator: () => ({ waitFor: async () => {}, innerText: async () => "Naruon" }), + evaluate: async () => { + evaluationCount += 1; + if (evaluationCount === 1) return { duplicateIds: [], unnamedInteractive: [] }; + return { tagName: "BUTTON" }; + }, + keyboard: { press: async () => {} }, + screenshot: async () => {}, + close: async () => {}, + }; + + await expect(runRouteSmoke( + { newPage: async () => page }, + FULL_PRODUCT_ROUTES[0], + { name: "desktop", width: 1440, height: 1024 }, + 1, + path.join(tmpdir(), "naruon-full-product-smoke-unit"), + )).rejects.toThrow("GET /api/unregistered-smoke-endpoint"); + expect(abortReason).toBe("failed"); + }); +}); + +''' +tests = tests.replace(marker, regression + marker, 1) + +source_path.write_text(source, encoding="utf-8") +test_path.write_text(tests, encoding="utf-8")