From 877f0d5c1b96c3ac212ecc86630afb75532f5880 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Tue, 8 Sep 2026 12:45:13 +0900 Subject: [PATCH 01/11] fix(smoke): reject browser errors after interaction and capture --- AGENTS.md | 6 ++++ frontend/scripts/full-product-ui-smoke.mjs | 5 ++- .../scripts/full-product-ui-smoke.test.mjs | 36 +++++++++++++++++++ 3 files changed, 46 insertions(+), 1 deletion(-) diff --git a/AGENTS.md b/AGENTS.md index 9104dd1f4..01c15a3ad 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -274,6 +274,12 @@ in this repo. ## Workspace and task tracking defaults +- Browser smoke must check collected console and page errors after interaction, + accessibility, screenshot capture, and page cleanup, not only after navigation. + Keep the early check for fast failure and a final check before returning success. + Regression tests must inject late errors into the actual route-smoke execution; + a successful screenshot or a passing render assertion does not prove clean interactions. + - First-run frontend sessions should open the Today execution dashboard while preserving explicit Dashboard, Email, and Calendar startup choices. - Workspace navigation changes must keep the desktop primary nav and the diff --git a/frontend/scripts/full-product-ui-smoke.mjs b/frontend/scripts/full-product-ui-smoke.mjs index 008d61267..64d1b83f4 100644 --- a/frontend/scripts/full-product-ui-smoke.mjs +++ b/frontend/scripts/full-product-ui-smoke.mjs @@ -1555,7 +1555,7 @@ async function runAccessibilitySmoke(page, routeSpec) { return [`${routeSpec.name}:a11y-basics`]; } -async function runRouteSmoke(context, routeSpec, viewportSpec, viewportCount, screenshotDir) { +export async function runRouteSmoke(context, routeSpec, viewportSpec, viewportCount, screenshotDir) { const page = await context.newPage(); const consoleErrors = []; page.on("console", (message) => { @@ -1594,6 +1594,9 @@ async function runRouteSmoke(context, routeSpec, viewportSpec, viewportCount, sc `${viewportSpec.name}:${routeSpec.path}`, ); await page.close(); + if (consoleErrors.length > 0) { + throw new Error(`Route ${routeSpec.path} emitted console errors:\n${consoleErrors.join("\n")}`); + } return { screenshotPath: screenshotArtifact, interactionEvidence, accessibilityEvidence }; } diff --git a/frontend/scripts/full-product-ui-smoke.test.mjs b/frontend/scripts/full-product-ui-smoke.test.mjs index 4e373a399..c9b6a4e2c 100644 --- a/frontend/scripts/full-product-ui-smoke.test.mjs +++ b/frontend/scripts/full-product-ui-smoke.test.mjs @@ -19,8 +19,44 @@ import { resolveFullProductChromePath, resolveFullProductScreenshotProfile, resolveFullProductViewportSpecs, + runRouteSmoke, } from "./full-product-ui-smoke.mjs"; +describe("route smoke late browser errors", () => { + it.each(["console", "pageerror"])("rejects %s errors emitted during capture", async (eventName) => { + const handlers = {}; + let evaluationCount = 0; + const page = { + on: (name, handler) => { handlers[name] = handler; }, + route: async () => {}, + goto: async () => {}, + waitForLoadState: async () => {}, + locator: () => ({ waitFor: async () => {}, innerText: async () => "Naruon" }), + evaluate: async () => { + evaluationCount += 1; + if (evaluationCount === 1) return { duplicateIds: [], unnamedInteractive: [] }; + return { tagName: "BUTTON" }; + }, + keyboard: { press: async () => {} }, + screenshot: async () => { + if (eventName === "console") { + handlers.console({ type: () => "error", text: () => "late-browser-failure" }); + } else { + handlers.pageerror(new Error("late-browser-failure")); + } + }, + close: async () => {}, + }; + await expect(runRouteSmoke( + { newPage: async () => page }, + FULL_PRODUCT_ROUTES[0], + { name: "desktop", width: 1440, height: 1024 }, + 1, + path.join(tmpdir(), "naruon-full-product-smoke-unit"), + )).rejects.toThrow("late-browser-failure"); + }); +}); + describe("full product UI smoke base URL guard", () => { it("allows localhost full-product smoke targets", () => { expect(resolveFullProductBaseUrl("http://127.0.0.1:3001").hostname).toBe("127.0.0.1"); From af4df216b3533ebb82dab9d361b415ddb941fa24 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Tue, 8 Sep 2026 12:51:16 +0900 Subject: [PATCH 02/11] test(smoke): reproduce fail-open screenshot capture --- ...oduct-ui-smoke-screenshot-failure.test.mjs | 68 +++++++++++++++++++ 1 file changed, 68 insertions(+) create mode 100644 frontend/scripts/full-product-ui-smoke-screenshot-failure.test.mjs diff --git a/frontend/scripts/full-product-ui-smoke-screenshot-failure.test.mjs b/frontend/scripts/full-product-ui-smoke-screenshot-failure.test.mjs new file mode 100644 index 000000000..69a1d4a3f --- /dev/null +++ b/frontend/scripts/full-product-ui-smoke-screenshot-failure.test.mjs @@ -0,0 +1,68 @@ +import { mkdtemp, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import path from "node:path"; + +import { describe, expect, it } from "vitest"; + +import { FULL_PRODUCT_ROUTES, runRouteSmoke } from "./full-product-ui-smoke.mjs"; + +function createRoutePageThatCannotCaptureScreenshot() { + let evaluationCount = 0; + let screenshotAttempts = 0; + let closed = false; + + return { + page: { + on: () => {}, + route: async () => {}, + goto: async () => {}, + waitForLoadState: async () => {}, + locator: () => ({ waitFor: async () => {}, innerText: async () => "Naruon" }), + evaluate: async () => { + evaluationCount += 1; + if (evaluationCount === 1) return { duplicateIds: [], unnamedInteractive: [] }; + return { tagName: "BUTTON" }; + }, + keyboard: { press: async () => {} }, + waitForTimeout: async () => {}, + screenshot: async () => { + screenshotAttempts += 1; + throw new Error("screenshot-backend-unavailable"); + }, + close: async () => { + closed = true; + }, + }, + evidence: { + get screenshotAttempts() { + return screenshotAttempts; + }, + get closed() { + return closed; + }, + }, + }; +} + +describe("full-product screenshot evidence", () => { + it("fails the route after both screenshot attempts fail while still closing the page", async () => { + const screenshotDirectory = await mkdtemp(path.join(tmpdir(), "naruon-full-product-smoke-failure-")); + const { page, evidence } = createRoutePageThatCannotCaptureScreenshot(); + + try { + await expect( + runRouteSmoke( + { newPage: async () => page }, + FULL_PRODUCT_ROUTES[0], + { name: "desktop", width: 1440, height: 1024 }, + 1, + screenshotDirectory, + ), + ).rejects.toThrow("screenshot-backend-unavailable"); + expect(evidence.screenshotAttempts).toBe(2); + expect(evidence.closed).toBe(true); + } finally { + await rm(screenshotDirectory, { recursive: true, force: true }); + } + }); +}); From 5502e2f63457dab12592b1ff52f4733addb9b03d Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Tue, 8 Sep 2026 13:03:55 +0900 Subject: [PATCH 03/11] docs(gap): record visual smoke findings and acceptance limits --- docs/product-technical-gap-baseline.md | 27 ++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/docs/product-technical-gap-baseline.md b/docs/product-technical-gap-baseline.md index 98bc17d2a..1c7400782 100644 --- a/docs/product-technical-gap-baseline.md +++ b/docs/product-technical-gap-baseline.md @@ -1,5 +1,32 @@ # Naruon Product and Technical Gap Baseline +## 2026-09-08 visual evidence: smoke success is not product acceptance + +This observation supplements, rather than replaces, the historical inventory below. +PR [#1599](https://github.com/ContextualWisdomLab/naruon/pull/1599) built application +source `877f0d5c1b96c3ac212ecc86630afb75532f5880`; its test-only descendant +`af4df216b3533ebb82dab9d361b415ddb941fa24` ran the same smoke implementation. +The production build and mocked browser smoke exited zero. Ten routes ran at +1440×1024 and 390×844; all twenty resulting PNGs were directly inspected. +The [inspection receipt](https://github.com/ContextualWisdomLab/naruon/pull/1599#issuecomment-5578964200) +records the artifact directory and scope. No real provider writes, live backend +contracts, all-locale coverage, or complete accessibility compliance were proven. + +| Observed customer gap | Product-owned action | Acceptance evidence still required | +| --- | --- | --- | +| Home retains English skip-link and `source-linked` copy | Localize navigation and status copy without changing task provenance | Keyboard-focused screenshot and locale-aware assertion | +| Search displays raw source/thread identifiers and `sender_context` | Keep identifiers in request/state boundaries; render customer-facing relationship labels | Contract-preserving UI test and desktop/mobile detail inspection | +| Calendar, Data, and Security expose intent/ETag, verifier commands/schema names, and event codes | Replace implementation vocabulary with outcomes and next actions; retain technical evidence in authorized diagnostic surfaces | Rendered-copy tests plus screenshots with unchanged API and authorization contracts | +| Mobile search/projects/settings snapshots show content behind sticky headers | Reproduce scrolling and keyboard focus before selecting a layout fix | Demonstrate focused controls and relevant text remain visible and reachable | + +The screenshot-failure regression at `af4df216` is independently RED: capture +failure returns a diagnostic text path as successful screenshot evidence. The +combined suite reported one failure and fourteen passes, exit one. Successful +captures do not refute this failed-capture defect. Preserve diagnostics, fail the +route, and verify page cleanup before accepting that repair. Unknown API mocks +also still default to HTTP 200; CI screenshots are not yet retained as downloadable +artifacts. Neither gap is closed by the late-console-error fix or this document. + **Baseline version:** 1.2 **Observed on:** 2026-08-26 (Asia/Seoul) **Observed protected branch (current scan; row Base-SHA values remain historical):** `develop@e5e99b4e3bb081b92c602358878856536030e2ca` From 64bf6c766e315b86eaa180fbd1a82f9087202e66 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Tue, 8 Sep 2026 13:20:00 +0900 Subject: [PATCH 04/11] fix(smoke): reject missing screenshots and always close pages Remove insecure diagnostic-file fallback and retain the post-close error check. Record applicable skills, visual inspection boundaries, and deployment prerequisites. Co-Authored-By: Codex Signed-off-by: Seongho Bae --- AGENTS.md | 23 +++++ ...oduct-ui-smoke-screenshot-failure.test.mjs | 14 +++- frontend/scripts/full-product-ui-smoke.mjs | 84 +++++++++---------- 3 files changed, 75 insertions(+), 46 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 01c15a3ad..a378b2971 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -279,6 +279,9 @@ in this repo. Keep the early check for fast failure and a final check before returning success. Regression tests must inject late errors into the actual route-smoke execution; a successful screenshot or a passing render assertion does not prove clean interactions. + If screenshot retries are exhausted, throw the capture failure; do not create + diagnostic text files in the temporary capture directory or count them as screenshots. + Close the page in a finally block, including navigation and capture failures. - First-run frontend sessions should open the Today execution dashboard while preserving explicit Dashboard, Email, and Calendar startup choices. @@ -655,6 +658,26 @@ in this repo. ## Development environment and tooling defaults +- Read the applicable repository skill before changing its contract: + [fix-development-mistakes](.agents/skills/fix-development-mistakes/SKILL.md) + for failures and security findings, + [github-actions-privileged-pr-scan](.agents/skills/github-actions-privileged-pr-scan/SKILL.md) + for privileged PR scanners, and + [github-robot-review-gate](.agents/skills/github-robot-review-gate/SKILL.md) + for check/review diagnosis. Record the failing reproduction, smallest causal + repair, exact verification command, and remaining gates in the existing PR. +- Visual Inspection requires opening the actual rendered pages or captured + images, not merely counting PNG files. Record app/build SHA separately from + runner SHA, viewport, routes, and observed defects. Mocked browser evidence + does not prove live provider behavior, all locales, or deployment readiness. + Inspect the changed AGENTS.md rendering on the pushed revision as well. +- Preserve concurrent commits with ordinary history integration, then verify + the combined tree before pushing. Update these instructions with reusable + failure-prevention lessons; keep dated findings in the gap baseline/PR. +- Package registry credentials alone do not authorize or configure a cluster + deployment. Use the existing release workflow only after its protected-source, + checks, target credentials, and destination prerequisites are verified; report + package publication and live deployment as separate outcomes. - If CodeGraph is not initialized for this repository, agents may run `codegraph init -i` autonomously without asking first; keep generated `.codegraph/` and `.cursor/rules/codegraph.mdc` artifacts local unless a diff --git a/frontend/scripts/full-product-ui-smoke-screenshot-failure.test.mjs b/frontend/scripts/full-product-ui-smoke-screenshot-failure.test.mjs index 69a1d4a3f..70904ac04 100644 --- a/frontend/scripts/full-product-ui-smoke-screenshot-failure.test.mjs +++ b/frontend/scripts/full-product-ui-smoke-screenshot-failure.test.mjs @@ -1,4 +1,4 @@ -import { mkdtemp, rm } from "node:fs/promises"; +import { mkdtemp, readdir, rm } from "node:fs/promises"; import { tmpdir } from "node:os"; import path from "node:path"; @@ -61,8 +61,20 @@ describe("full-product screenshot evidence", () => { ).rejects.toThrow("screenshot-backend-unavailable"); expect(evidence.screenshotAttempts).toBe(2); expect(evidence.closed).toBe(true); + expect(await readdir(screenshotDirectory)).toEqual([]); } finally { await rm(screenshotDirectory, { recursive: true, force: true }); } }); + + it("closes the page when navigation fails before capture", async () => { + const { page, evidence } = createRoutePageThatCannotCaptureScreenshot(); + page.goto = async () => { throw new Error("navigation-unavailable"); }; + await expect(runRouteSmoke( + { newPage: async () => page }, FULL_PRODUCT_ROUTES[0], + { name: "desktop", width: 1440, height: 1024 }, 1, tmpdir(), + )).rejects.toThrow("navigation-unavailable"); + expect(evidence.screenshotAttempts).toBe(0); + expect(evidence.closed).toBe(true); + }); }); diff --git a/frontend/scripts/full-product-ui-smoke.mjs b/frontend/scripts/full-product-ui-smoke.mjs index 64d1b83f4..060f4b46b 100644 --- a/frontend/scripts/full-product-ui-smoke.mjs +++ b/frontend/scripts/full-product-ui-smoke.mjs @@ -1,5 +1,5 @@ import { execFile, spawn } from "node:child_process"; -import { access, mkdtemp, writeFile } from "node:fs/promises"; +import { access, mkdtemp } from "node:fs/promises"; import net from "node:net"; import { tmpdir } from "node:os"; import path from "node:path"; @@ -187,7 +187,7 @@ function log(message) { process.stdout.write(`${message}\n`); } -async function captureSmokeScreenshot(page, screenshotPath, label) { +async function captureSmokeScreenshot(page, screenshotPath) { for (let attempt = 1; attempt <= 2; attempt += 1) { try { await page.screenshot({ path: screenshotPath, fullPage: false }); @@ -197,16 +197,8 @@ async function captureSmokeScreenshot(page, screenshotPath, label) { await page.waitForTimeout(250); continue; } - const diagnosticFileName = path.basename(screenshotPath).replace(/\.png$/u, ".screenshot-failed.txt"); - const diagnosticPath = resolveFullProductArtifactPath(path.dirname(screenshotPath), diagnosticFileName); - const reason = error instanceof Error ? error.message : String(error); - await writeFile( - diagnosticPath, - `screenshot_failed label=${label}\nreason=${reason}\n`, - "utf-8", - ); - log(`Screenshot capture failed for ${label}: ${reason}`); - return diagnosticPath; + log("Screenshot capture failed after both attempts"); + throw error; } } return screenshotPath; @@ -1558,42 +1550,44 @@ async function runAccessibilitySmoke(page, routeSpec) { export async function runRouteSmoke(context, routeSpec, viewportSpec, viewportCount, screenshotDir) { const page = await context.newPage(); const consoleErrors = []; - page.on("console", (message) => { - if (message.type() === "error") { - consoleErrors.push(`${message.type()}: ${message.text()}`); + let screenshotArtifact; + let interactionEvidence; + let accessibilityEvidence; + try { + page.on("console", (message) => { + if (message.type() === "error") { + consoleErrors.push(`${message.type()}: ${message.text()}`); + } + }); + page.on("pageerror", (error) => consoleErrors.push(`pageerror: ${error.message}`)); + await installRoutes(page); + await page.goto(new URL(routeSpec.path, baseUrl).href, { waitUntil: "domcontentloaded" }); + await page.waitForLoadState("networkidle", { timeout: 10_000 }).catch(() => {}); + await page.locator("body").waitFor({ state: "visible", timeout: 20_000 }); + const bodyText = await page.locator("body").innerText({ timeout: 10_000 }); + const expectedTexts = Array.isArray(routeSpec.expectedText) ? routeSpec.expectedText : [routeSpec.expectedText]; + if (!expectedTexts.some((expectedText) => bodyText.includes(expectedText))) { + const bodySnippet = bodyText.replace(/\s+/g, " ").trim().slice(0, 500); + throw new Error( + `Route ${routeSpec.path} did not render expected text: ${expectedTexts.join(" or ")}. Body snippet: ${bodySnippet}`, + ); } - }); - page.on("pageerror", (error) => consoleErrors.push(`pageerror: ${error.message}`)); - await installRoutes(page); - await page.goto(new URL(routeSpec.path, baseUrl).href, { waitUntil: "domcontentloaded" }); - await page.waitForLoadState("networkidle", { timeout: 10_000 }).catch(() => {}); - await page.locator("body").waitFor({ state: "visible", timeout: 20_000 }); - const bodyText = await page.locator("body").innerText({ timeout: 10_000 }); - const expectedTexts = Array.isArray(routeSpec.expectedText) ? routeSpec.expectedText : [routeSpec.expectedText]; - if (!expectedTexts.some((expectedText) => bodyText.includes(expectedText))) { - const bodySnippet = bodyText.replace(/\s+/g, " ").trim().slice(0, 500); - throw new Error( - `Route ${routeSpec.path} did not render expected text: ${expectedTexts.join(" or ")}. Body snippet: ${bodySnippet}`, + if (bodyText.includes("404") || bodyText.includes("This page could not be found")) { + throw new Error(`Route ${routeSpec.path} rendered a not-found page`); + } + if (consoleErrors.length > 0) { + throw new Error(`Route ${routeSpec.path} emitted console errors:\n${consoleErrors.join("\n")}`); + } + interactionEvidence = await runCriticalInteractionSmoke(page, routeSpec, viewportSpec); + accessibilityEvidence = await runAccessibilitySmoke(page, routeSpec); + const screenshotPath = resolveFullProductArtifactPath( + screenshotDir, + fullProductScreenshotName(routeSpec, viewportSpec, viewportCount), ); + screenshotArtifact = await captureSmokeScreenshot(page, screenshotPath); + } finally { + await page.close(); } - if (bodyText.includes("404") || bodyText.includes("This page could not be found")) { - throw new Error(`Route ${routeSpec.path} rendered a not-found page`); - } - if (consoleErrors.length > 0) { - throw new Error(`Route ${routeSpec.path} emitted console errors:\n${consoleErrors.join("\n")}`); - } - const interactionEvidence = await runCriticalInteractionSmoke(page, routeSpec, viewportSpec); - const accessibilityEvidence = await runAccessibilitySmoke(page, routeSpec); - const screenshotPath = resolveFullProductArtifactPath( - screenshotDir, - fullProductScreenshotName(routeSpec, viewportSpec, viewportCount), - ); - const screenshotArtifact = await captureSmokeScreenshot( - page, - screenshotPath, - `${viewportSpec.name}:${routeSpec.path}`, - ); - await page.close(); if (consoleErrors.length > 0) { throw new Error(`Route ${routeSpec.path} emitted console errors:\n${consoleErrors.join("\n")}`); } From 9bb0dc7ef8aa846dbf1571d00b71c61bf83b16c5 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Tue, 8 Sep 2026 13:51:49 +0900 Subject: [PATCH 05/11] test(ui-smoke): cover browser errors emitted during cleanup --- frontend/scripts/full-product-ui-smoke.test.mjs | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/frontend/scripts/full-product-ui-smoke.test.mjs b/frontend/scripts/full-product-ui-smoke.test.mjs index c9b6a4e2c..34932760b 100644 --- a/frontend/scripts/full-product-ui-smoke.test.mjs +++ b/frontend/scripts/full-product-ui-smoke.test.mjs @@ -23,9 +23,10 @@ import { } from "./full-product-ui-smoke.mjs"; describe("route smoke late browser errors", () => { - it.each(["console", "pageerror"])("rejects %s errors emitted during capture", async (eventName) => { + it.each(["console", "pageerror"])("rejects %s errors emitted during page close", async (eventName) => { const handlers = {}; let evaluationCount = 0; + let closeCalled = false; const page = { on: (name, handler) => { handlers[name] = handler; }, route: async () => {}, @@ -38,14 +39,15 @@ describe("route smoke late browser errors", () => { return { tagName: "BUTTON" }; }, keyboard: { press: async () => {} }, - screenshot: async () => { + screenshot: async () => {}, + close: async () => { + closeCalled = true; if (eventName === "console") { handlers.console({ type: () => "error", text: () => "late-browser-failure" }); } else { handlers.pageerror(new Error("late-browser-failure")); } }, - close: async () => {}, }; await expect(runRouteSmoke( { newPage: async () => page }, @@ -54,6 +56,7 @@ describe("route smoke late browser errors", () => { 1, path.join(tmpdir(), "naruon-full-product-smoke-unit"), )).rejects.toThrow("late-browser-failure"); + expect(closeCalled).toBe(true); }); }); From 0c91e52577cb6c1486d0f933c3567e92611488e2 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Tue, 8 Sep 2026 14:14:29 +0900 Subject: [PATCH 06/11] fix(smoke): keep governance docs in canonical lanes --- AGENTS.md | 29 -------------------------- docs/product-technical-gap-baseline.md | 27 ------------------------ 2 files changed, 56 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index a378b2971..9104dd1f4 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -274,15 +274,6 @@ in this repo. ## Workspace and task tracking defaults -- Browser smoke must check collected console and page errors after interaction, - accessibility, screenshot capture, and page cleanup, not only after navigation. - Keep the early check for fast failure and a final check before returning success. - Regression tests must inject late errors into the actual route-smoke execution; - a successful screenshot or a passing render assertion does not prove clean interactions. - If screenshot retries are exhausted, throw the capture failure; do not create - diagnostic text files in the temporary capture directory or count them as screenshots. - Close the page in a finally block, including navigation and capture failures. - - First-run frontend sessions should open the Today execution dashboard while preserving explicit Dashboard, Email, and Calendar startup choices. - Workspace navigation changes must keep the desktop primary nav and the @@ -658,26 +649,6 @@ in this repo. ## Development environment and tooling defaults -- Read the applicable repository skill before changing its contract: - [fix-development-mistakes](.agents/skills/fix-development-mistakes/SKILL.md) - for failures and security findings, - [github-actions-privileged-pr-scan](.agents/skills/github-actions-privileged-pr-scan/SKILL.md) - for privileged PR scanners, and - [github-robot-review-gate](.agents/skills/github-robot-review-gate/SKILL.md) - for check/review diagnosis. Record the failing reproduction, smallest causal - repair, exact verification command, and remaining gates in the existing PR. -- Visual Inspection requires opening the actual rendered pages or captured - images, not merely counting PNG files. Record app/build SHA separately from - runner SHA, viewport, routes, and observed defects. Mocked browser evidence - does not prove live provider behavior, all locales, or deployment readiness. - Inspect the changed AGENTS.md rendering on the pushed revision as well. -- Preserve concurrent commits with ordinary history integration, then verify - the combined tree before pushing. Update these instructions with reusable - failure-prevention lessons; keep dated findings in the gap baseline/PR. -- Package registry credentials alone do not authorize or configure a cluster - deployment. Use the existing release workflow only after its protected-source, - checks, target credentials, and destination prerequisites are verified; report - package publication and live deployment as separate outcomes. - If CodeGraph is not initialized for this repository, agents may run `codegraph init -i` autonomously without asking first; keep generated `.codegraph/` and `.cursor/rules/codegraph.mdc` artifacts local unless a diff --git a/docs/product-technical-gap-baseline.md b/docs/product-technical-gap-baseline.md index 1c7400782..98bc17d2a 100644 --- a/docs/product-technical-gap-baseline.md +++ b/docs/product-technical-gap-baseline.md @@ -1,32 +1,5 @@ # Naruon Product and Technical Gap Baseline -## 2026-09-08 visual evidence: smoke success is not product acceptance - -This observation supplements, rather than replaces, the historical inventory below. -PR [#1599](https://github.com/ContextualWisdomLab/naruon/pull/1599) built application -source `877f0d5c1b96c3ac212ecc86630afb75532f5880`; its test-only descendant -`af4df216b3533ebb82dab9d361b415ddb941fa24` ran the same smoke implementation. -The production build and mocked browser smoke exited zero. Ten routes ran at -1440×1024 and 390×844; all twenty resulting PNGs were directly inspected. -The [inspection receipt](https://github.com/ContextualWisdomLab/naruon/pull/1599#issuecomment-5578964200) -records the artifact directory and scope. No real provider writes, live backend -contracts, all-locale coverage, or complete accessibility compliance were proven. - -| Observed customer gap | Product-owned action | Acceptance evidence still required | -| --- | --- | --- | -| Home retains English skip-link and `source-linked` copy | Localize navigation and status copy without changing task provenance | Keyboard-focused screenshot and locale-aware assertion | -| Search displays raw source/thread identifiers and `sender_context` | Keep identifiers in request/state boundaries; render customer-facing relationship labels | Contract-preserving UI test and desktop/mobile detail inspection | -| Calendar, Data, and Security expose intent/ETag, verifier commands/schema names, and event codes | Replace implementation vocabulary with outcomes and next actions; retain technical evidence in authorized diagnostic surfaces | Rendered-copy tests plus screenshots with unchanged API and authorization contracts | -| Mobile search/projects/settings snapshots show content behind sticky headers | Reproduce scrolling and keyboard focus before selecting a layout fix | Demonstrate focused controls and relevant text remain visible and reachable | - -The screenshot-failure regression at `af4df216` is independently RED: capture -failure returns a diagnostic text path as successful screenshot evidence. The -combined suite reported one failure and fourteen passes, exit one. Successful -captures do not refute this failed-capture defect. Preserve diagnostics, fail the -route, and verify page cleanup before accepting that repair. Unknown API mocks -also still default to HTTP 200; CI screenshots are not yet retained as downloadable -artifacts. Neither gap is closed by the late-console-error fix or this document. - **Baseline version:** 1.2 **Observed on:** 2026-08-26 (Asia/Seoul) **Observed protected branch (current scan; row Base-SHA values remain historical):** `develop@e5e99b4e3bb081b92c602358878856536030e2ca` From c3b2f879056c9f34cc8b449a136ced290f554e2e Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 25 Sep 2026 00:40:50 +0900 Subject: [PATCH 07/11] chore(ci): stage fail-closed smoke route repair --- .../workflows/temporary-smoke-fail-closed.yml | 182 ++++++++++++++++++ 1 file changed, 182 insertions(+) create mode 100644 .github/workflows/temporary-smoke-fail-closed.yml diff --git a/.github/workflows/temporary-smoke-fail-closed.yml b/.github/workflows/temporary-smoke-fail-closed.yml new file mode 100644 index 000000000..a7dbc33f9 --- /dev/null +++ b/.github/workflows/temporary-smoke-fail-closed.yml @@ -0,0 +1,182 @@ +name: Temporary smoke fail-closed repair + +on: + push: + branches: + - codex/smoke-late-error-guard + paths: + - .github/workflows/temporary-smoke-fail-closed.yml + +permissions: + contents: write + +concurrency: + group: temporary-smoke-fail-closed-${{ github.ref }} + cancel-in-progress: false + +jobs: + repair: + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - name: Checkout exact staging head + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + fetch-depth: 0 + persist-credentials: true + + - name: Set up Node.js + uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 + with: + node-version: "24" + + - name: Verify single-writer head + shell: bash + run: | + set -euo pipefail + branch="${GITHUB_REF_NAME}" + git fetch origin "${branch}" + test "$(git rev-parse "origin/${branch}")" = "${GITHUB_SHA}" + + - name: Apply fail-closed route-mock repair + shell: bash + run: | + set -euo pipefail + python - <<'PY' + from pathlib import Path + + source_path = Path("frontend/scripts/full-product-ui-smoke.mjs") + test_path = Path("frontend/scripts/full-product-ui-smoke.test.mjs") + source = source_path.read_text() + tests = test_path.read_text() + + replacements = [ + ( + "async function installRoutes(page) {", + "async function installRoutes(page, unhandledApiRequests = new Set()) {", + ), + ( + " return routeJson(route, { ok: true });\n", + " unhandledApiRequests.add(`${request.method()} ${endpoint}`);\n" + " return route.abort(\"failed\");\n", + ), + ( + " const consoleErrors = [];\n let screenshotArtifact;", + " const consoleErrors = [];\n const unhandledApiRequests = new Set();\n let screenshotArtifact;", + ), + ( + " await installRoutes(page);", + " await installRoutes(page, unhandledApiRequests);", + ), + ( + " if (consoleErrors.length > 0) {\n throw new Error(`Route ${routeSpec.path} emitted console errors:\\n${consoleErrors.join(\"\\n\")}`);\n }\n return { screenshotPath: screenshotArtifact, interactionEvidence, accessibilityEvidence };", + " if (unhandledApiRequests.size > 0) {\n" + " throw new Error(\n" + " `Route ${routeSpec.path} requested unregistered mocked APIs:\\n${[...unhandledApiRequests].join(\"\\n\")}`,\n" + " );\n" + " }\n" + " if (consoleErrors.length > 0) {\n" + " throw new Error(`Route ${routeSpec.path} emitted console errors:\\n${consoleErrors.join(\"\\n\")}`);\n" + " }\n" + " return { screenshotPath: screenshotArtifact, interactionEvidence, accessibilityEvidence };", + ), + ] + + for old, new in replacements: + count = source.count(old) + if count != 1: + raise SystemExit(f"expected one source match, found {count}: {old[:80]!r}") + source = source.replace(old, new, 1) + + marker = 'describe("route smoke late browser errors", () => {' + if tests.count(marker) != 1: + raise SystemExit("route-smoke test insertion marker changed") + + regression = r'''describe("route smoke API fixture boundary", () => { + it("fails closed when the product requests an unregistered mocked API", async () => { + let apiRouteHandler; + let abortReason; + let evaluationCount = 0; + const page = { + on: () => {}, + route: async (pattern, handler) => { + if (pattern === "**/api/**") apiRouteHandler = handler; + }, + goto: async () => { + expect(apiRouteHandler).toBeTypeOf("function"); + await apiRouteHandler({ + request: () => ({ + url: () => "http://127.0.0.1:3001/api/unregistered-smoke-endpoint", + method: () => "GET", + }), + abort: async (reason) => { + abortReason = reason; + }, + }); + }, + waitForLoadState: async () => {}, + locator: () => ({ waitFor: async () => {}, innerText: async () => "Naruon" }), + evaluate: async () => { + evaluationCount += 1; + if (evaluationCount === 1) return { duplicateIds: [], unnamedInteractive: [] }; + return { tagName: "BUTTON" }; + }, + keyboard: { press: async () => {} }, + screenshot: async () => {}, + close: async () => {}, + }; + + await expect(runRouteSmoke( + { newPage: async () => page }, + FULL_PRODUCT_ROUTES[0], + { name: "desktop", width: 1440, height: 1024 }, + 1, + path.join(tmpdir(), "naruon-full-product-smoke-unit"), + )).rejects.toThrow("GET /api/unregistered-smoke-endpoint"); + expect(abortReason).toBe("failed"); + }); +}); + +''' + tests = tests.replace(marker, regression + marker, 1) + + source_path.write_text(source) + test_path.write_text(tests) + PY + + mapfile -t changed < <(git diff --name-only) + printf '%s\n' "${changed[@]}" + test "${#changed[@]}" -eq 2 + test "${changed[0]}" = "frontend/scripts/full-product-ui-smoke.mjs" + test "${changed[1]}" = "frontend/scripts/full-product-ui-smoke.test.mjs" + git diff --check + + - name: Install frontend dependencies + shell: bash + working-directory: frontend + run: | + set -euo pipefail + corepack enable pnpm + pnpm install --frozen-lockfile + + - name: Run focused regression and lint + shell: bash + working-directory: frontend + run: | + set -euo pipefail + pnpm exec vitest run scripts/full-product-ui-smoke.test.mjs + pnpm exec eslint scripts/full-product-ui-smoke.mjs scripts/full-product-ui-smoke.test.mjs --max-warnings 0 + + - name: Publish ordinary product commit + shell: bash + run: | + set -euo pipefail + branch="${GITHUB_REF_NAME}" + git fetch origin "${branch}" + test "$(git rev-parse "origin/${branch}")" = "${GITHUB_SHA}" + git config user.name "naruon-commercial-writer" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + git add frontend/scripts/full-product-ui-smoke.mjs frontend/scripts/full-product-ui-smoke.test.mjs + test "$(git diff --cached --name-only | wc -l)" -eq 2 + git commit -m "fix(smoke): fail closed on unregistered mocked APIs" + git push origin "HEAD:${branch}" From 2a61c01a9f797089d6f44d7c3bf47cab18b1057f Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 25 Sep 2026 00:42:31 +0900 Subject: [PATCH 08/11] fix(ci): align smoke repair helper with runner policy --- .../workflows/temporary-smoke-fail-closed.yml | 78 +++++++++++-------- 1 file changed, 46 insertions(+), 32 deletions(-) diff --git a/.github/workflows/temporary-smoke-fail-closed.yml b/.github/workflows/temporary-smoke-fail-closed.yml index a7dbc33f9..723e84535 100644 --- a/.github/workflows/temporary-smoke-fail-closed.yml +++ b/.github/workflows/temporary-smoke-fail-closed.yml @@ -4,39 +4,43 @@ on: push: branches: - codex/smoke-late-error-guard - paths: - - .github/workflows/temporary-smoke-fail-closed.yml permissions: contents: write -concurrency: - group: temporary-smoke-fail-closed-${{ github.ref }} - cancel-in-progress: false - jobs: repair: + name: Repair fail-open mocked API boundary runs-on: ubuntu-latest timeout-minutes: 30 steps: + - name: Harden the runner (Audit all outbound calls) + uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0 + with: + egress-policy: audit + - name: Checkout exact staging head uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: fetch-depth: 0 persist-credentials: true + ref: ${{ github.sha }} - name: Set up Node.js uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 with: node-version: "24" + - name: Enable pnpm + run: corepack enable pnpm + - name: Verify single-writer head shell: bash run: | set -euo pipefail branch="${GITHUB_REF_NAME}" - git fetch origin "${branch}" - test "$(git rev-parse "origin/${branch}")" = "${GITHUB_SHA}" + remote_head="$(git ls-remote origin "refs/heads/${branch}" | awk '{print $1}')" + test "${remote_head}" = "${GITHUB_SHA}" - name: Apply fail-closed route-mock repair shell: bash @@ -47,8 +51,8 @@ jobs: source_path = Path("frontend/scripts/full-product-ui-smoke.mjs") test_path = Path("frontend/scripts/full-product-ui-smoke.test.mjs") - source = source_path.read_text() - tests = test_path.read_text() + source = source_path.read_text(encoding="utf-8") + tests = test_path.read_text(encoding="utf-8") replacements = [ ( @@ -140,43 +144,53 @@ jobs: ''' tests = tests.replace(marker, regression + marker, 1) - source_path.write_text(source) - test_path.write_text(tests) + source_path.write_text(source, encoding="utf-8") + test_path.write_text(tests, encoding="utf-8") PY - mapfile -t changed < <(git diff --name-only) - printf '%s\n' "${changed[@]}" - test "${#changed[@]}" -eq 2 - test "${changed[0]}" = "frontend/scripts/full-product-ui-smoke.mjs" - test "${changed[1]}" = "frontend/scripts/full-product-ui-smoke.test.mjs" git diff --check + git diff --name-only | sort > /tmp/actual-paths + cat > /tmp/expected-paths <<'EOF' + frontend/scripts/full-product-ui-smoke.mjs + frontend/scripts/full-product-ui-smoke.test.mjs + EOF + sed -i 's/^ //' /tmp/expected-paths + diff -u /tmp/expected-paths /tmp/actual-paths - name: Install frontend dependencies - shell: bash - working-directory: frontend - run: | - set -euo pipefail - corepack enable pnpm - pnpm install --frozen-lockfile + run: cd frontend && pnpm install --frozen-lockfile - name: Run focused regression and lint shell: bash - working-directory: frontend run: | set -euo pipefail + cd frontend pnpm exec vitest run scripts/full-product-ui-smoke.test.mjs pnpm exec eslint scripts/full-product-ui-smoke.mjs scripts/full-product-ui-smoke.test.mjs --max-warnings 0 - - name: Publish ordinary product commit + - name: Commit and push product delta non-force shell: bash + env: + BRANCH_NAME: codex/smoke-late-error-guard run: | set -euo pipefail - branch="${GITHUB_REF_NAME}" - git fetch origin "${branch}" - test "$(git rev-parse "origin/${branch}")" = "${GITHUB_SHA}" - git config user.name "naruon-commercial-writer" + remote_head="$(git ls-remote origin "refs/heads/${BRANCH_NAME}" | awk '{print $1}')" + if [[ "${remote_head}" != "${GITHUB_SHA}" ]]; then + echo "Remote head moved: expected ${GITHUB_SHA}, got ${remote_head}" >&2 + exit 1 + fi + + git add -- frontend/scripts/full-product-ui-smoke.mjs frontend/scripts/full-product-ui-smoke.test.mjs + git diff --cached --check + git diff --cached --name-only | sort > /tmp/actual-staged-paths + cat > /tmp/expected-staged-paths <<'EOF' + frontend/scripts/full-product-ui-smoke.mjs + frontend/scripts/full-product-ui-smoke.test.mjs + EOF + sed -i 's/^ //' /tmp/expected-staged-paths + diff -u /tmp/expected-staged-paths /tmp/actual-staged-paths + + git config user.name "github-actions[bot]" git config user.email "41898282+github-actions[bot]@users.noreply.github.com" - git add frontend/scripts/full-product-ui-smoke.mjs frontend/scripts/full-product-ui-smoke.test.mjs - test "$(git diff --cached --name-only | wc -l)" -eq 2 git commit -m "fix(smoke): fail closed on unregistered mocked APIs" - git push origin "HEAD:${branch}" + git push origin "HEAD:${BRANCH_NAME}" From 3b0422772792a81eff3535a1dea33723b458cdcf Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 25 Sep 2026 00:46:04 +0900 Subject: [PATCH 09/11] chore(ci): externalize smoke repair helper script --- scripts/ci/temporary_smoke_fail_closed.py | 99 +++++++++++++++++++++++ 1 file changed, 99 insertions(+) create mode 100644 scripts/ci/temporary_smoke_fail_closed.py diff --git a/scripts/ci/temporary_smoke_fail_closed.py b/scripts/ci/temporary_smoke_fail_closed.py new file mode 100644 index 000000000..b99793eac --- /dev/null +++ b/scripts/ci/temporary_smoke_fail_closed.py @@ -0,0 +1,99 @@ +from pathlib import Path + +source_path = Path("frontend/scripts/full-product-ui-smoke.mjs") +test_path = Path("frontend/scripts/full-product-ui-smoke.test.mjs") +source = source_path.read_text(encoding="utf-8") +tests = test_path.read_text(encoding="utf-8") + +replacements = [ + ( + "async function installRoutes(page) {", + "async function installRoutes(page, unhandledApiRequests = new Set()) {", + ), + ( + " return routeJson(route, { ok: true });\n", + " unhandledApiRequests.add(`${request.method()} ${endpoint}`);\n" + " return route.abort(\"failed\");\n", + ), + ( + " const consoleErrors = [];\n let screenshotArtifact;", + " const consoleErrors = [];\n const unhandledApiRequests = new Set();\n let screenshotArtifact;", + ), + ( + " await installRoutes(page);", + " await installRoutes(page, unhandledApiRequests);", + ), + ( + " if (consoleErrors.length > 0) {\n throw new Error(`Route ${routeSpec.path} emitted console errors:\\n${consoleErrors.join(\"\\n\")}`);\n }\n return { screenshotPath: screenshotArtifact, interactionEvidence, accessibilityEvidence };", + " if (unhandledApiRequests.size > 0) {\n" + " throw new Error(\n" + " `Route ${routeSpec.path} requested unregistered mocked APIs:\\n${[...unhandledApiRequests].join(\"\\n\")}`,\n" + " );\n" + " }\n" + " if (consoleErrors.length > 0) {\n" + " throw new Error(`Route ${routeSpec.path} emitted console errors:\\n${consoleErrors.join(\"\\n\")}`);\n" + " }\n" + " return { screenshotPath: screenshotArtifact, interactionEvidence, accessibilityEvidence };", + ), +] + +for old, new in replacements: + count = source.count(old) + if count != 1: + raise SystemExit(f"expected one source match, found {count}: {old[:80]!r}") + source = source.replace(old, new, 1) + +marker = 'describe("route smoke late browser errors", () => {' +if tests.count(marker) != 1: + raise SystemExit("route-smoke test insertion marker changed") + +regression = r'''describe("route smoke API fixture boundary", () => { + it("fails closed when the product requests an unregistered mocked API", async () => { + let apiRouteHandler; + let abortReason; + let evaluationCount = 0; + const page = { + on: () => {}, + route: async (pattern, handler) => { + if (pattern === "**/api/**") apiRouteHandler = handler; + }, + goto: async () => { + expect(apiRouteHandler).toBeTypeOf("function"); + await apiRouteHandler({ + request: () => ({ + url: () => "http://127.0.0.1:3001/api/unregistered-smoke-endpoint", + method: () => "GET", + }), + abort: async (reason) => { + abortReason = reason; + }, + }); + }, + waitForLoadState: async () => {}, + locator: () => ({ waitFor: async () => {}, innerText: async () => "Naruon" }), + evaluate: async () => { + evaluationCount += 1; + if (evaluationCount === 1) return { duplicateIds: [], unnamedInteractive: [] }; + return { tagName: "BUTTON" }; + }, + keyboard: { press: async () => {} }, + screenshot: async () => {}, + close: async () => {}, + }; + + await expect(runRouteSmoke( + { newPage: async () => page }, + FULL_PRODUCT_ROUTES[0], + { name: "desktop", width: 1440, height: 1024 }, + 1, + path.join(tmpdir(), "naruon-full-product-smoke-unit"), + )).rejects.toThrow("GET /api/unregistered-smoke-endpoint"); + expect(abortReason).toBe("failed"); + }); +}); + +''' +tests = tests.replace(marker, regression + marker, 1) + +source_path.write_text(source, encoding="utf-8") +test_path.write_text(tests, encoding="utf-8") From 8111bba1f76319518bd2c65d8061a61e480447cf Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 25 Sep 2026 00:46:19 +0900 Subject: [PATCH 10/11] fix(ci): simplify smoke repair workflow parser surface --- .../workflows/temporary-smoke-fail-closed.yml | 131 ++---------------- 1 file changed, 13 insertions(+), 118 deletions(-) diff --git a/.github/workflows/temporary-smoke-fail-closed.yml b/.github/workflows/temporary-smoke-fail-closed.yml index 723e84535..71608033d 100644 --- a/.github/workflows/temporary-smoke-fail-closed.yml +++ b/.github/workflows/temporary-smoke-fail-closed.yml @@ -22,7 +22,6 @@ jobs: - name: Checkout exact staging head uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: - fetch-depth: 0 persist-credentials: true ref: ${{ github.sha }} @@ -38,123 +37,22 @@ jobs: shell: bash run: | set -euo pipefail - branch="${GITHUB_REF_NAME}" - remote_head="$(git ls-remote origin "refs/heads/${branch}" | awk '{print $1}')" + remote_head="$(git ls-remote origin "refs/heads/${GITHUB_REF_NAME}" | awk '{print $1}')" test "${remote_head}" = "${GITHUB_SHA}" - - name: Apply fail-closed route-mock repair + - name: Apply bounded source repair + run: python scripts/ci/temporary_smoke_fail_closed.py + + - name: Verify bounded source delta shell: bash run: | set -euo pipefail - python - <<'PY' - from pathlib import Path - - source_path = Path("frontend/scripts/full-product-ui-smoke.mjs") - test_path = Path("frontend/scripts/full-product-ui-smoke.test.mjs") - source = source_path.read_text(encoding="utf-8") - tests = test_path.read_text(encoding="utf-8") - - replacements = [ - ( - "async function installRoutes(page) {", - "async function installRoutes(page, unhandledApiRequests = new Set()) {", - ), - ( - " return routeJson(route, { ok: true });\n", - " unhandledApiRequests.add(`${request.method()} ${endpoint}`);\n" - " return route.abort(\"failed\");\n", - ), - ( - " const consoleErrors = [];\n let screenshotArtifact;", - " const consoleErrors = [];\n const unhandledApiRequests = new Set();\n let screenshotArtifact;", - ), - ( - " await installRoutes(page);", - " await installRoutes(page, unhandledApiRequests);", - ), - ( - " if (consoleErrors.length > 0) {\n throw new Error(`Route ${routeSpec.path} emitted console errors:\\n${consoleErrors.join(\"\\n\")}`);\n }\n return { screenshotPath: screenshotArtifact, interactionEvidence, accessibilityEvidence };", - " if (unhandledApiRequests.size > 0) {\n" - " throw new Error(\n" - " `Route ${routeSpec.path} requested unregistered mocked APIs:\\n${[...unhandledApiRequests].join(\"\\n\")}`,\n" - " );\n" - " }\n" - " if (consoleErrors.length > 0) {\n" - " throw new Error(`Route ${routeSpec.path} emitted console errors:\\n${consoleErrors.join(\"\\n\")}`);\n" - " }\n" - " return { screenshotPath: screenshotArtifact, interactionEvidence, accessibilityEvidence };", - ), - ] - - for old, new in replacements: - count = source.count(old) - if count != 1: - raise SystemExit(f"expected one source match, found {count}: {old[:80]!r}") - source = source.replace(old, new, 1) - - marker = 'describe("route smoke late browser errors", () => {' - if tests.count(marker) != 1: - raise SystemExit("route-smoke test insertion marker changed") - - regression = r'''describe("route smoke API fixture boundary", () => { - it("fails closed when the product requests an unregistered mocked API", async () => { - let apiRouteHandler; - let abortReason; - let evaluationCount = 0; - const page = { - on: () => {}, - route: async (pattern, handler) => { - if (pattern === "**/api/**") apiRouteHandler = handler; - }, - goto: async () => { - expect(apiRouteHandler).toBeTypeOf("function"); - await apiRouteHandler({ - request: () => ({ - url: () => "http://127.0.0.1:3001/api/unregistered-smoke-endpoint", - method: () => "GET", - }), - abort: async (reason) => { - abortReason = reason; - }, - }); - }, - waitForLoadState: async () => {}, - locator: () => ({ waitFor: async () => {}, innerText: async () => "Naruon" }), - evaluate: async () => { - evaluationCount += 1; - if (evaluationCount === 1) return { duplicateIds: [], unnamedInteractive: [] }; - return { tagName: "BUTTON" }; - }, - keyboard: { press: async () => {} }, - screenshot: async () => {}, - close: async () => {}, - }; - - await expect(runRouteSmoke( - { newPage: async () => page }, - FULL_PRODUCT_ROUTES[0], - { name: "desktop", width: 1440, height: 1024 }, - 1, - path.join(tmpdir(), "naruon-full-product-smoke-unit"), - )).rejects.toThrow("GET /api/unregistered-smoke-endpoint"); - expect(abortReason).toBe("failed"); - }); -}); - -''' - tests = tests.replace(marker, regression + marker, 1) - - source_path.write_text(source, encoding="utf-8") - test_path.write_text(tests, encoding="utf-8") - PY - git diff --check git diff --name-only | sort > /tmp/actual-paths - cat > /tmp/expected-paths <<'EOF' - frontend/scripts/full-product-ui-smoke.mjs - frontend/scripts/full-product-ui-smoke.test.mjs - EOF - sed -i 's/^ //' /tmp/expected-paths + printf '%s\n' \ + frontend/scripts/full-product-ui-smoke.mjs \ + frontend/scripts/full-product-ui-smoke.test.mjs \ + > /tmp/expected-paths diff -u /tmp/expected-paths /tmp/actual-paths - name: Install frontend dependencies @@ -179,17 +77,14 @@ jobs: echo "Remote head moved: expected ${GITHUB_SHA}, got ${remote_head}" >&2 exit 1 fi - git add -- frontend/scripts/full-product-ui-smoke.mjs frontend/scripts/full-product-ui-smoke.test.mjs git diff --cached --check git diff --cached --name-only | sort > /tmp/actual-staged-paths - cat > /tmp/expected-staged-paths <<'EOF' - frontend/scripts/full-product-ui-smoke.mjs - frontend/scripts/full-product-ui-smoke.test.mjs - EOF - sed -i 's/^ //' /tmp/expected-staged-paths + printf '%s\n' \ + frontend/scripts/full-product-ui-smoke.mjs \ + frontend/scripts/full-product-ui-smoke.test.mjs \ + > /tmp/expected-staged-paths diff -u /tmp/expected-staged-paths /tmp/actual-staged-paths - git config user.name "github-actions[bot]" git config user.email "41898282+github-actions[bot]@users.noreply.github.com" git commit -m "fix(smoke): fail closed on unregistered mocked APIs" From db28aa3c8ddd56b063dea3f5f2d1ca09ad6647b4 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Thu, 24 Sep 2026 19:52:20 +0000 Subject: [PATCH 11/11] fix(smoke): fail closed on unregistered mocked APIs --- frontend/scripts/full-product-ui-smoke.mjs | 13 ++++-- .../scripts/full-product-ui-smoke.test.mjs | 45 +++++++++++++++++++ 2 files changed, 55 insertions(+), 3 deletions(-) diff --git a/frontend/scripts/full-product-ui-smoke.mjs b/frontend/scripts/full-product-ui-smoke.mjs index 060f4b46b..9efdf2aaf 100644 --- a/frontend/scripts/full-product-ui-smoke.mjs +++ b/frontend/scripts/full-product-ui-smoke.mjs @@ -769,7 +769,7 @@ function routeJson(route, body, status = 200) { }); } -async function installRoutes(page) { +async function installRoutes(page, unhandledApiRequests = new Set()) { let emailSendCount = 0; let savedAccountConfig = { ...accountConfig }; let savedLlmProviders = [{ ...llmProvider }]; @@ -1072,7 +1072,8 @@ async function installRoutes(page) { if (endpoint.startsWith("/api/tools/")) return routeJson(route, { output: "ok", status: "success" }); if (endpoint === "/api/runtime-config") return routeJson(route, {}); - return routeJson(route, { ok: true }); + unhandledApiRequests.add(`${request.method()} ${endpoint}`); + return route.abort("failed"); }); } @@ -1550,6 +1551,7 @@ async function runAccessibilitySmoke(page, routeSpec) { export async function runRouteSmoke(context, routeSpec, viewportSpec, viewportCount, screenshotDir) { const page = await context.newPage(); const consoleErrors = []; + const unhandledApiRequests = new Set(); let screenshotArtifact; let interactionEvidence; let accessibilityEvidence; @@ -1560,7 +1562,7 @@ export async function runRouteSmoke(context, routeSpec, viewportSpec, viewportCo } }); page.on("pageerror", (error) => consoleErrors.push(`pageerror: ${error.message}`)); - await installRoutes(page); + await installRoutes(page, unhandledApiRequests); await page.goto(new URL(routeSpec.path, baseUrl).href, { waitUntil: "domcontentloaded" }); await page.waitForLoadState("networkidle", { timeout: 10_000 }).catch(() => {}); await page.locator("body").waitFor({ state: "visible", timeout: 20_000 }); @@ -1588,6 +1590,11 @@ export async function runRouteSmoke(context, routeSpec, viewportSpec, viewportCo } finally { await page.close(); } + if (unhandledApiRequests.size > 0) { + throw new Error( + `Route ${routeSpec.path} requested unregistered mocked APIs:\n${[...unhandledApiRequests].join("\n")}`, + ); + } if (consoleErrors.length > 0) { throw new Error(`Route ${routeSpec.path} emitted console errors:\n${consoleErrors.join("\n")}`); } diff --git a/frontend/scripts/full-product-ui-smoke.test.mjs b/frontend/scripts/full-product-ui-smoke.test.mjs index 34932760b..cfdd12363 100644 --- a/frontend/scripts/full-product-ui-smoke.test.mjs +++ b/frontend/scripts/full-product-ui-smoke.test.mjs @@ -22,6 +22,51 @@ import { runRouteSmoke, } from "./full-product-ui-smoke.mjs"; +describe("route smoke API fixture boundary", () => { + it("fails closed when the product requests an unregistered mocked API", async () => { + let apiRouteHandler; + let abortReason; + let evaluationCount = 0; + const page = { + on: () => {}, + route: async (pattern, handler) => { + if (pattern === "**/api/**") apiRouteHandler = handler; + }, + goto: async () => { + expect(apiRouteHandler).toBeTypeOf("function"); + await apiRouteHandler({ + request: () => ({ + url: () => "http://127.0.0.1:3001/api/unregistered-smoke-endpoint", + method: () => "GET", + }), + abort: async (reason) => { + abortReason = reason; + }, + }); + }, + waitForLoadState: async () => {}, + locator: () => ({ waitFor: async () => {}, innerText: async () => "Naruon" }), + evaluate: async () => { + evaluationCount += 1; + if (evaluationCount === 1) return { duplicateIds: [], unnamedInteractive: [] }; + return { tagName: "BUTTON" }; + }, + keyboard: { press: async () => {} }, + screenshot: async () => {}, + close: async () => {}, + }; + + await expect(runRouteSmoke( + { newPage: async () => page }, + FULL_PRODUCT_ROUTES[0], + { name: "desktop", width: 1440, height: 1024 }, + 1, + path.join(tmpdir(), "naruon-full-product-smoke-unit"), + )).rejects.toThrow("GET /api/unregistered-smoke-endpoint"); + expect(abortReason).toBe("failed"); + }); +}); + describe("route smoke late browser errors", () => { it.each(["console", "pageerror"])("rejects %s errors emitted during page close", async (eventName) => { const handlers = {};