From 0377ec3994839763f89c9cdc6cb6d6e4aa176f73 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 14 Sep 2026 10:07:22 +0000 Subject: [PATCH 1/2] Report the login overlay test as a test, in its own job MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit It is an end-to-end test: it boots the app and drives a real BrowserWindow through a WebAuthn wait. Everything naming it said "check" โ€” the script, the file, the text it printed โ€” and it ran as a step inside the job called "๐Ÿงน Lint app", so a pull request's check list never said whether it had run at all. `npm run test:overlay`, `tools/test-login-overlay.js`, and a job named ๐Ÿงช End-to-end tests that `build` waits on. It costs a second npm ci and build, which run in parallel with lint, and buys a check line that goes red on its own. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_018XK1mi3rSPLzxu7XUhyC6a --- .github/workflows/ci.yaml | 41 +++++++++++++++---- AGENTS.md | 14 ++++--- package.json | 2 +- ...login-overlay.js => test-login-overlay.js} | 14 +++---- 4 files changed, 50 insertions(+), 21 deletions(-) rename tools/{check-login-overlay.js => test-login-overlay.js} (93%) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 55713aa..a1d6518 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -2,10 +2,11 @@ name: ๐Ÿ” Build # Validates the full release build (all platforms, including macOS signing and # notarization) on pull requests and on every push to main, so we can confirm -# everything works before drafting a release. Lints, then delegates the matrix -# build to build.yaml in "make" mode (signs, notarizes, zips, uploads -# artifacts; never tags or publishes). On pushes to main, also updates the -# draft release notes that release.yaml will pick up when published. +# everything works before drafting a release. Lints and runs the end-to-end +# tests that drive the real app, then delegates the matrix build to build.yaml in "make" +# mode (signs, notarizes, zips, uploads artifacts; never tags or publishes). On +# pushes to main, also updates the draft release notes that release.yaml will +# pick up when published. on: push: @@ -48,12 +49,38 @@ jobs: run: | npm run lint - - name: ๐Ÿ”‘ Check the login overlay + # Separate from linting, and named for what it is: these boot the real app + # under xvfb and drive it โ€” a login window against a WebAuthn request, and a + # whole refresh against a stubbed AWS SSO. They are the tests most likely to + # be the reason a pull request is red, so they report under their own name + # rather than inside a job called "lint". Do not fold them back in. + e2e: + name: ๐Ÿงช End-to-end tests + runs-on: ubuntu-latest + steps: + - name: ๐Ÿ“ฅ Checkout sources + uses: actions/checkout@v7 + + - name: โš™๏ธ Install Node.js and NPM + uses: actions/setup-node@v7 + with: + node-version: ${{ env.NODEJS_VERSION }} + cache: npm + + - name: ๐Ÿ“ฆ Install node modules + run: | + npm ci + + - name: ๐Ÿ”จ Build app + run: | + npm run build + + - name: ๐Ÿ”‘ Login window credential overlay run: | - xvfb-run -a npm run check:overlay -- --no-sandbox + xvfb-run -a npm run test:overlay -- --no-sandbox build: - needs: lint + needs: [lint, e2e] uses: ./.github/workflows/build.yaml # Read-only for pull request validation. The publish path declares its own # contents: write in release.yaml. diff --git a/AGENTS.md b/AGENTS.md index b23cfae..60fba12 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -88,13 +88,15 @@ npm only (`package-lock.json`; CI runs `npm ci`). Do not add a `yarn.lock`. - `npm run build` โ€” `tsc`, then copies the tray icons and `dashboard.html`. - `npm run lint` โ€” oxlint, configured by `.oxlintrc.json`. -- `npm run check:overlay` โ€” drives the login window's credential overlay +- `npm run test:overlay` โ€” drives the login window's credential overlay through a real WebAuthn wait. Needs `npm run build` first, and a display: - `xvfb-run -a npm run check:overlay -- --no-sandbox`. + `xvfb-run -a npm run test:overlay -- --no-sandbox`. - `npm start` / `npm run package` / `npm run make` โ€” Electron Forge. -All three run in CI. Build and lint alone do not prove the app launches; see -"Verification limits". +All three run in CI. `test:overlay` runs as its own job (**๐Ÿงช End-to-end +tests**) rather than inside the lint job: it boots the real app, so it reports +under a name that says so. Build and lint alone do not prove the app launches; +see "Verification limits". ## ESM @@ -161,7 +163,7 @@ when a credential request starts and before the account picker opens. Under automatic approval the window may not be on screen yet, and a modal sheet on a window nobody can see is a prompt nobody can answer. -`npm run check:overlay` is the regression test for all of that: it drives the +`npm run test:overlay` is the regression test for all of that: it drives the real `attachLoginIndicator()` on a real `BrowserWindow` against pages that ask for a key before and after `dom-ready`, and asserts the wait reached the main process. No key needed โ€” only the start of the request matters, and that is @@ -518,7 +520,7 @@ linux**. You *can* also launch it, given those same downloads and `xvfb`: `xvfb-run -a ./node_modules/electron/dist/electron --no-sandbox .` boots the -whole app, and `npm run check:overlay` uses that to drive a real +whole app, and `npm run test:overlay` uses that to drive a real `BrowserWindow`. That is how the overlay's document-start bug was found; build and lint could not have. What it does **not** give you is a real desktop: no tray interaction, no dock, no security key, no keychain, no macOS signing. Say diff --git a/package.json b/package.json index bf7d508..efe9866 100644 --- a/package.json +++ b/package.json @@ -17,7 +17,7 @@ "build": "tsc && npm run build:icons && npm run build:html && npm run build:overlay", "watch": "tsc -w", "lint": "oxlint src", - "check:overlay": "electron tools/check-login-overlay.js", + "test:overlay": "electron tools/test-login-overlay.js", "start": "electron-forge start", "package": "electron-forge package", "make": "electron-forge make" diff --git a/tools/check-login-overlay.js b/tools/test-login-overlay.js similarity index 93% rename from tools/check-login-overlay.js rename to tools/test-login-overlay.js index 259801b..fc91849 100644 --- a/tools/check-login-overlay.js +++ b/tools/test-login-overlay.js @@ -1,10 +1,10 @@ -// Self-check for the login window's credential overlay. +// End-to-end test for the login window's credential overlay. // -// npm run build && npx electron tools/check-login-overlay.js +// npm run build && npx electron tools/test-login-overlay.js // // Headless (CI, a container): wrap it in a display โ€” // -// xvfb-run -a npx electron --no-sandbox tools/check-login-overlay.js +// xvfb-run -a npx electron --no-sandbox tools/test-login-overlay.js // // Why this exists: every part of the overlay can look right and still show the // user nothing, and the failure is silent โ€” no error, no log line, just a login @@ -14,7 +14,7 @@ // key as it boots (Google's security-key challenge does) had already called // `navigator.credentials.get()` by the time there was anything to wrap. // -// So the check drives the real thing โ€” the real `attachLoginIndicator()` on a +// So the test drives the real thing โ€” the real `attachLoginIndicator()` on a // real BrowserWindow โ€” against both orderings, and asserts the wait actually // reached the main process. It needs no security key: only the *start* of the // request is interesting, and that is signalled the moment the page asks. @@ -167,13 +167,13 @@ async function check() { app.on("window-all-closed", () => {}); app.whenReady().then(async () => { - console.log(`login overlay self-check (Frost ${version})`); + console.log(`login overlay test (Frost ${version})`); try { await check(); - console.log("login overlay self-check OK"); + console.log("login overlay test passed"); app.exit(0); } catch (err) { - console.error(`login overlay self-check FAILED\n${err}`); + console.error(`login overlay test FAILED\n${err}`); app.exit(1); } }); From ce9c29ce5040de2e1549bfbe0d69fa7664cc4995 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 14 Sep 2026 10:08:53 +0000 Subject: [PATCH 2/2] End-to-end tests for automatic approval, against a stubbed AWS SSO MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Automatic approval is a script clicking buttons on pages Frost does not own, in a window the user cannot see. Every way it can be wrong is quiet: a button never found (the refresh hangs until the device code expires), a button that should not have been clicked (the request is denied), a hand-over that never happens (the user waits in front of nothing). None of that is visible to a type-check, and testing the matching rules alone misses what actually makes it work โ€” the whole path. So `npm run test:auto-approve` drives the real refresh(), the same entry point the tray, the hotkey and the timer use, and asserts on what the user would have seen. Four interceptions make that possible without the app knowing it is under test: - AWS_ENDPOINT_URL_SSO_OIDC / AWS_ENDPOINT_URL_SSO, an AWS SDK feature, point the SDK at a stub HTTP service. The device authorization, the polling and its AuthorizationPendingException are the real client speaking a real protocol, and the token only becomes redeemable when the stub's approval page is actually fetched. - session.protocol.handle("https", ...) serves the pages at their real names, so the renderer gets https://d-โ€ฆ.awsapps.com, a secure context, and a genuine cross-origin redirect to the identity provider. Served from localhost it would prove nothing โ€” the host rule is the point. - Notification.prototype.show and shell.openExternal are recorded rather than performed: what the user was told and where they were sent are the assertions, and a CI runner has neither a notification daemon nor a browser. - powerMonitor.getSystemIdleTime() answers whatever the scenario says. Frost only shows a login page to somebody who is there, so real idle time would make these tests depend on whether anyone had touched the keyboard. Nine scenarios, one per outcome, including the two that decide whether anybody is interrupted: an identity provider hop that the session carries through stays silent, and one that asks for a password brings the window up. HOME and the electron-store move to a temp directory, so a run touches nothing of the developer's. Each mutation fails exactly one scenario: the host rule returning false fails both approval scenarios and returning true fails the identity-provider one; dropping the refusal rule fails the unrecognised page; isUserPresent() returning true unconditionally fails the unattended one. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_018XK1mi3rSPLzxu7XUhyC6a --- .github/workflows/ci.yaml | 4 + AGENTS.md | 78 +++- package.json | 1 + tools/test-auto-approve.js | 881 +++++++++++++++++++++++++++++++++++++ 4 files changed, 951 insertions(+), 13 deletions(-) create mode 100644 tools/test-auto-approve.js diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index a1d6518..497dc14 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -79,6 +79,10 @@ jobs: run: | xvfb-run -a npm run test:overlay -- --no-sandbox + - name: โœ… Automatic approval, end to end + run: | + xvfb-run -a npm run test:auto-approve -- --no-sandbox + build: needs: [lint, e2e] uses: ./.github/workflows/build.yaml diff --git a/AGENTS.md b/AGENTS.md index 60fba12..c1feb7a 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -91,12 +91,16 @@ npm only (`package-lock.json`; CI runs `npm ci`). Do not add a `yarn.lock`. - `npm run test:overlay` โ€” drives the login window's credential overlay through a real WebAuthn wait. Needs `npm run build` first, and a display: `xvfb-run -a npm run test:overlay -- --no-sandbox`. +- `npm run test:auto-approve` โ€” drives a whole `refresh()` against a stubbed + AWS SSO, end to end. Same requirements, same shape: + `xvfb-run -a npm run test:auto-approve -- --no-sandbox`. - `npm start` / `npm run package` / `npm run make` โ€” Electron Forge. -All three run in CI. `test:overlay` runs as its own job (**๐Ÿงช End-to-end -tests**) rather than inside the lint job: it boots the real app, so it reports -under a name that says so. Build and lint alone do not prove the app launches; -see "Verification limits". +All of those run in CI. The two `test:` scripts run as their own job +(**๐Ÿงช End-to-end tests**) rather than inside the lint job: they boot the real +app and are the likeliest reason a pull request is red, so they report under a +name that says so. Build and lint alone do not prove the app launches; see +"Verification limits". ## ESM @@ -215,12 +219,58 @@ of them says the user is needed (issue #1). Keep these true: - The console signal is forgeable by the page, exactly like the overlay's, so it may only ever decide whether to show a window. -`approve-overlay.ts` is import-free browser code, so it is checked the same way -the overlay's drawing is: `new Function("window", source)` over the built -`dist/approve-overlay.js` with a stub `window` whose `document.querySelectorAll` -answers the two selectors it uses, asserting which stub controls were clicked -and what it logged. That covers every page shape โ€” confirm, allow, sign-in, -approved, unrecognised โ€” without a browser. +`npm run test:auto-approve` (`tools/test-auto-approve.js`) is the regression +test, and it is end to end: it drives the real `refresh()` โ€” the entry point +the tray, the hotkey and the timer all use โ€” against a stub of AWS SSO, and +asserts on what the user would have seen. Four interceptions make that possible +without the app knowing it is under test, and all four are worth keeping: + +- `AWS_ENDPOINT_URL_SSO_OIDC` / `AWS_ENDPOINT_URL_SSO` are an AWS SDK feature, + so the device authorization, the polling and its + AuthorizationPendingException are the real client talking a real protocol to + a stub service over HTTP. The token only becomes redeemable when the stub's + approval page is actually fetched, so nothing passes without a real click. +- `session.protocol.handle("https", ...)` serves the pages at their real names, + so the renderer sees `https://d-โ€ฆ.awsapps.com`, a secure context, and a + genuine cross-origin redirect to the identity provider. Served from localhost + it would prove nothing: the host rule is the point. +- `Notification.prototype.show` and `shell.openExternal` are recorded rather + than performed โ€” "what was the user told" and "where were they sent" are the + assertions, and a CI runner has neither a notification daemon nor a browser. + `Notification` itself is a non-configurable export, so the patch has to go on + the prototype. +- `powerMonitor.getSystemIdleTime()` answers whatever the scenario says. Frost + only puts a login page in front of somebody who is there, so real idle time + would make these depend on whether anyone had touched the keyboard: green on + a fresh CI runner, red on a desktop five minutes after you started them and + walked away. + +`HOME` and the electron-store move to a temp directory, so a run touches +nothing of yours. Nine scenarios, ~30s, one per outcome: + +| Scenario | What must be true | +| --- | --- | +| Portal session is live | Token collected, **no window ever shown**, no notification | +| Federated, IdP session is live | Same, and the cross-origin hop happened | +| Federated, IdP wants a password | Window shown; after the test signs in, the driver finishes the approval | +| Notify mode | Nothing opens until `triggerPendingAuth()`, even when the approval needs nobody | +| Default-browser mode | `openExternal` gets the verification URL, no window shown | +| Automatic approval off | Window visible from the start, nothing driven | +| IdP page with an "Allow access" button | Never clicked โ€” it is not our host | +| AWS page nothing recognises | Nothing clicked, including a refusal wearing `cli_login_button`'s id; window comes up | +| Nobody at the machine | The login is held hidden, and shown when presence returns | + +It is a real test, not a smoke test, and each mutation fails exactly one +scenario: the host rule returning `false` fails both approval scenarios and +returning `true` fails the identity-provider one; dropping the refusal rule +fails the unrecognised-page one; `isUserPresent()` returning `true` +unconditionally fails the unattended one. Confirm with a mutation before +trusting a change here. + +The matching rules alone can also be exercised without a browser โ€” +`approve-overlay.ts` is import-free, so `new Function("window", source)` over +the built file with a stub `window` runs them โ€” which is the quicker loop while +writing them. ## `~/.aws/config` ownership @@ -520,9 +570,11 @@ linux**. You *can* also launch it, given those same downloads and `xvfb`: `xvfb-run -a ./node_modules/electron/dist/electron --no-sandbox .` boots the -whole app, and `npm run test:overlay` uses that to drive a real -`BrowserWindow`. That is how the overlay's document-start bug was found; build -and lint could not have. What it does **not** give you is a real desktop: no +whole app, and `npm run test:overlay` and `npm run test:auto-approve` use that +to drive a real `BrowserWindow` โ€” the latter running a whole `refresh()` +against a stubbed AWS SSO, so the login path can be exercised end to end +without an AWS account. That is how the overlay's document-start bug was found; +build and lint could not have. What it does **not** give you is a real desktop: no tray interaction, no dock, no security key, no keychain, no macOS signing. Say so rather than claiming the app works. diff --git a/package.json b/package.json index efe9866..1ce2ccb 100644 --- a/package.json +++ b/package.json @@ -18,6 +18,7 @@ "watch": "tsc -w", "lint": "oxlint src", "test:overlay": "electron tools/test-login-overlay.js", + "test:auto-approve": "electron tools/test-auto-approve.js", "start": "electron-forge start", "package": "electron-forge package", "make": "electron-forge make" diff --git a/tools/test-auto-approve.js b/tools/test-auto-approve.js new file mode 100644 index 0000000..d3c6b0c --- /dev/null +++ b/tools/test-auto-approve.js @@ -0,0 +1,881 @@ +// End-to-end tests for automatic approval. +// +// npm run build && npx electron tools/test-auto-approve.js +// +// Headless (CI, a container): wrap it in a display โ€” +// +// xvfb-run -a npx electron --no-sandbox tools/test-auto-approve.js +// +// Why this exists: automatic approval is a script clicking buttons on pages +// Frost does not own, in a window the user cannot see. Every way it can be +// wrong is quiet โ€” a button that is never found (the refresh hangs until the +// device code expires), a button that should not have been clicked (the request +// is denied), a hand-over that never happens (the user waits in front of +// nothing). None of that shows up in a type-check or in a unit test of the +// matching rules, because what makes it work is the whole path: the real device +// flow, real pages at real AWS and identity provider origins, a real hidden +// window, and the real poll loop collecting the token afterwards. +// +// So this drives `refresh()` itself โ€” the same entry point the tray, the hotkey +// and the timer use โ€” against a stub of AWS SSO, and asserts on what the user +// would have seen. Three interceptions make that possible, none of which asks +// the app to know it is being tested: +// +// - `AWS_ENDPOINT_URL_SSO_OIDC` / `AWS_ENDPOINT_URL_SSO`, an AWS SDK feature, +// point the SDK clients at the stub's HTTP server. The device +// authorization, the polling and its AuthorizationPendingException are the +// real client talking a real protocol to a stub service. +// - `session.protocol.handle("https", ...)` serves the pages from memory at +// their real names. The renderer sees `https://d-1234567890.awsapps.com`, a +// secure context, and a genuine cross-origin redirect to the identity +// provider โ€” which is what the driver's host rule is written against. +// Served from localhost this would prove nothing. +// - `Notification` and `shell.openExternal` are recorded rather than +// performed, because "what was the user told" and "where were they sent" +// are the assertions, and neither a CI runner nor a developer's desktop +// should have to grow a notification daemon or a browser window for them. +// - `powerMonitor.getSystemIdleTime()` answers whatever the scenario says. +// Frost shows a login page only when somebody is there to see it, so real +// idle time would make these tests depend on whether anyone happened to +// touch the keyboard: green in CI on a fresh runner, red on a desktop five +// minutes after you started them and walked away. +// +// `HOME` and the electron-store move into a temp directory for the duration, so +// a run touches nothing of yours. + +import assert from "assert"; +import fs from "fs"; +import http from "http"; +import os from "os"; +import path from "path"; +import { createRequire } from "module"; +import { app, BrowserWindow, powerMonitor, session, shell } from "electron"; + +const require = createRequire(import.meta.url); +const { version } = require("../package.json"); + +/** The account portal, and the identity provider it federates to. */ +const PORTAL = "https://d-1234567890.awsapps.com"; +const IDP = "https://idp.example.test"; + +const USER_CODE = "ABCD-EFGH"; + +/** Long enough that nothing races the device code expiring. */ +const DEVICE_CODE_LIFETIME_SEC = 120; + +/** + * For scenarios that end with nobody completing the login: the run finishes + * when the code expires. Long enough to outlast the driver's 12s stall timer, + * which is what the unrecognised-page scenario is waiting for. + */ +const STALLING_DEVICE_CODE_LIFETIME_SEC = 16; + +/** + * For scenarios that hand over immediately and then have nothing left to wait + * for. Nothing here depends on the stall timer, so the code can be short. + */ +const SHORT_DEVICE_CODE_LIFETIME_SEC = 6; + +const CASE_TIMEOUT_MS = 60000; +const POLL_MS = 100; + +/** The app's own idea of "nobody is here", so these cannot drift apart. */ +const { AWAY_IDLE_SEC } = await import("../dist/schedule.js"); + +// โ”€โ”€ The pages โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ +// +// Close enough to the real ones to exercise the rules that matter: the device +// page carries its code already filled in (the thing that must *not* read as +// "the user has to type something"), and every page carries something the +// driver must leave alone. + +const STYLE = ""; + +function page(title, body) { + return `${title}${STYLE} +

${title}

${body}`; +} + +/** Step one: "Authorize request". The code arrives filled in from the URL. */ +const DEVICE_PAGE = page( + "Authorize request", + `
+ +
+ + ` +); + +/** Step two: "Allow access". Its neighbour is the one that must never be hit. */ +const ALLOW_PAGE = page( + "Allow access to your data?", + ` + ` +); + +const APPROVED_PAGE = page( + "Request approved", + "

You can close this window and return to Frost.

" +); + +/** The identity provider: the one page here that is the user's to answer. */ +const SIGNIN_PAGE = page( + "Sign in", + `
+ + + +
` +); + +/** + * An identity provider page wearing the portal's clothes: the very label the + * driver is looking for, on a host that is not AWS. Clicking it would be + * clicking a stranger's button. The password field beside it is what brings the + * window up, so the test does not have to wait out the stall timer to see the + * answer. + */ +const IDP_TRAP_PAGE = page( + "Sign in to continue", + ` +
+ +
` +); + +/** + * A page the driver has no business touching. Three traps: a label it does not + * know, a plain refusal, and โ€” the one that matters most โ€” a refusal wearing + * the id of the button it wants, which is what AWS reusing an id on an "are you + * sure?" page would look like. + */ +const UNKNOWN_PAGE = page( + "Something else entirely", + ` + + ` +); + +// โ”€โ”€ The stub โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ + +/** Reset for each scenario; the assertions read it afterwards. */ +let run = null; + +function newRun(name, options = {}) { + run = { + name, + // What the portal does once the request is confirmed: go straight to + // consent, or federate to the identity provider โ€” which either carries + // the user through on a session it already has, asks them to sign in, + // or tries to get Frost to click something of its own. + idp: options.idp || "none", + firstPage: options.firstPage || DEVICE_PAGE, + lifetimeSec: options.lifetimeSec || DEVICE_CODE_LIFETIME_SEC, + approved: false, + // Every page request, API call, notification and browser hand-off, in + // order: the assertions read it, and it is printed when one fails. + trail: [], + // Sampled rather than taken from the `show` event, so it holds whether + // the window was created hidden and shown later or created visible. + everVisible: false, + notifications: [], + browserOpens: [], + tokenPolls: 0, + }; + return run; +} + +function note(what) { + if (run) run.trail.push(what); +} + +function json(body, status = 200, headers = {}) { + return [ + status, + { "Content-Type": "application/json", ...headers }, + JSON.stringify(body), + ]; +} + +/** + * The AWS SSO-OIDC and SSO endpoints a refresh actually calls. Only the three + * device-flow operations and the account listing are needed: with no accounts + * there are no profiles, and with no profiles the EKS scan returns before it + * asks for a region. + */ +function handleApi(method, url) { + if (method === "POST" && url === "/client/register") { + note("oidc:register"); + return json({ + clientId: "frost-check-client", + clientSecret: "frost-check-secret", + clientIdIssuedAt: Math.floor(Date.now() / 1000), + clientSecretExpiresAt: Math.floor(Date.now() / 1000) + 3600, + }); + } + + if (method === "POST" && url === "/device_authorization") { + note("oidc:device-authorization"); + return json({ + deviceCode: "frost-check-device-code", + userCode: USER_CODE, + verificationUri: `${PORTAL}/start/#/device`, + verificationUriComplete: `${PORTAL}/start/?user_code=${USER_CODE}#/device`, + expiresIn: run.lifetimeSec, + // One second keeps the check quick without changing anything about + // the loop under test. + interval: 1, + }); + } + + if (method === "POST" && url === "/token") { + run.tokenPolls += 1; + if (!run.approved) { + note("oidc:token-pending"); + // The shape the SDK turns back into AuthorizationPendingException, + // which is what the poll loop is written against. + return json( + { + __type: "AuthorizationPendingException", + error: "authorization_pending", + error_description: "The request is pending approval", + }, + 400, + { "x-amzn-errortype": "AuthorizationPendingException:" } + ); + } + note("oidc:token-issued"); + return json({ + accessToken: "frost-check-access-token", + tokenType: "Bearer", + expiresIn: 28800, + }); + } + + // ListAccounts. An empty list is a complete, successful refresh with no + // profiles to write and no EKS scan to run. + if (method === "GET" && url.startsWith("/assignment/accounts")) { + note("sso:list-accounts"); + return json({ accountList: [] }); + } + + note(`api:unhandled ${method} ${url}`); + return json({ __type: "InternalServerException" }, 500); +} + +function startApiStub() { + return new Promise((resolve, reject) => { + const server = http.createServer((req, res) => { + let body = ""; + req.on("data", (chunk) => (body += chunk)); + req.on("end", () => { + const [status, headers, payload] = handleApi( + req.method, + req.url + ); + res.writeHead(status, headers); + res.end(payload); + }); + }); + server.on("error", reject); + server.listen(0, "127.0.0.1", () => + resolve({ server, port: server.address().port }) + ); + }); +} + +/** + * Serve the pages under their real names. The driver only clicks on the AWS + * portal's own hosts, so where these come from is part of what is being tested. + */ +function interceptPages() { + session.defaultSession.protocol.handle("https", (request) => { + const url = new URL(request.url); + note(`page:${url.origin}${url.pathname}`); + + const html = (body) => + new Response(body, { + status: 200, + headers: { + "Content-Type": "text/html", + "Cache-Control": "no-store", + }, + }); + + const redirect = (to) => + new Response(null, { status: 302, headers: { Location: to } }); + + if (url.origin === PORTAL) { + switch (url.pathname) { + case "/start/": + return html(run.firstPage); + case "/next": + // A confirmed request either goes straight to consent or + // federates out. The redirect is the real shape of that + // hop, and it moves the page to another origin โ€” and + // another renderer process. + return run.idp === "none" + ? html(ALLOW_PAGE) + : redirect(`${IDP}/signin`); + case "/allow": + return html(ALLOW_PAGE); + case "/approved": + // The moment that makes the device code redeemable, which + // is why no token appears without a real click. + run.approved = true; + return html(APPROVED_PAGE); + default: + return html(page("Unexpected", `

${url.pathname}

`)); + } + } + + if (url.origin === IDP) { + // Signed in; back to the portal for the consent step, which is the + // driver's again. + if (url.pathname === "/submit") return redirect(`${PORTAL}/allow`); + if (run.idp === "live") return redirect(`${PORTAL}/allow`); + if (run.idp === "trap") return html(IDP_TRAP_PAGE); + return html(SIGNIN_PAGE); + } + + return new Response("not found", { status: 404 }); + }); +} + +// โ”€โ”€ Watching what the user would have seen โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ + +let sampler = null; + +/** + * Sampled, not taken from the window's `show` event: "was anything ever put in + * front of the user" has to hold for a window created visible as much as for + * one revealed later, and the scenarios need both. + */ +function sampleWindows() { + sampler = setInterval(() => { + if (!run || run.everVisible) return; + const visible = BrowserWindow.getAllWindows().some( + (window) => !window.isDestroyed() && window.isVisible() + ); + if (visible) { + run.everVisible = true; + note("window:visible"); + } + }, POLL_MS); +} + +function visibleWindow() { + return BrowserWindow.getAllWindows().find( + (window) => !window.isDestroyed() && window.isVisible() + ); +} + +/** + * Record what the user is told instead of telling them. + * + * On the prototype, not by swapping the class: `Notification` is a + * non-configurable export of the electron module, and every `new + * Notification(...)` in the app reaches this either way. Nothing is passed + * through to the real `show()` โ€” a CI container has no notification daemon, + * and the assertion is that Frost said something, not that a desktop drew it. + */ +function recordNotifications() { + const { Notification } = require("electron"); + Notification.prototype.show = function () { + note(`notification:${this.title}`); + if (run) run.notifications.push({ title: this.title, body: this.body }); + }; + return typeof Notification.prototype.show === "function"; +} + +/** + * Whoever is at the machine, as the scenarios decide. Frost reads this through + * `powerMonitor.getSystemIdleTime()` whenever it matters โ€” a login started with + * nobody there can end with the user watching โ€” so one value covers a whole run + * and can change part-way through it. + */ +let idleSec = 0; + +function beHere() { + idleSec = 0; +} + +function beAway() { + idleSec = AWAY_IDLE_SEC * 2; +} + +function controlPresence() { + powerMonitor.getSystemIdleTime = () => idleSec; + return powerMonitor.getSystemIdleTime() === idleSec; +} + +/** Record where the user would have been sent, without opening a browser. */ +function recordBrowserOpens() { + shell.openExternal = async (url) => { + note(`browser:${url}`); + if (run) run.browserOpens.push(url); + }; +} + +// โ”€โ”€ Waiting โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ + +function describe() { + return `${run.name}\n trail: ${run.trail.join("\n ")}`; +} + +function fail(what) { + return new Error(`${what}\n ${describe()}`); +} + +function check(condition, what) { + assert.ok(condition, fail(what).message); +} + +async function waitFor(predicate, what, timeoutMs = CASE_TIMEOUT_MS) { + const deadline = Date.now() + timeoutMs; + for (;;) { + const value = predicate(); + if (value) return value; + if (Date.now() > deadline) throw fail(`timed out waiting for ${what}`); + await new Promise((resolve) => setTimeout(resolve, POLL_MS)); + } +} + +async function withTimeout(promise, what, timeoutMs = CASE_TIMEOUT_MS) { + let timer; + const timeout = new Promise((_resolve, reject) => { + timer = setTimeout( + () => reject(fail(`timed out waiting for ${what}`)), + timeoutMs + ); + }); + try { + return await Promise.race([promise, timeout]); + } finally { + clearTimeout(timer); + } +} + +/** Stand in for the user at the identity provider's sign-in form. */ +async function signInOnPage(window) { + await window.webContents.executeJavaScript(` + (function () { + var text = document.querySelector('input[type=text]'); + if (text) text.value = "someone@example.com"; + var password = document.querySelector('input[type=password]'); + if (password) password.value = "hunter2"; + document.querySelector('form').submit(); + })(); + `); +} + +/** + * Give up the way the user does. `close()`, not `destroy()`: closing is what + * tells the poll loop nobody is signing in, and it is the path that has to + * survive a remote page's `beforeunload`. + */ +function closeVisibleWindow() { + const window = visibleWindow(); + if (window) window.close(); +} + +// โ”€โ”€ Scenarios โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ + +/** + * The case the feature exists for: a portal session that carries the user + * through, and a refresh that finishes with nothing on screen. + */ +async function silentApproval(frost) { + newRun("a refresh nobody has to see"); + + await withTimeout(frost.refresh(), "the refresh to finish"); + + check(run.approved, "the request was never approved"); + check(!run.everVisible, "the login window was shown"); + check(frost.hasToken(), "no token was stored"); + check( + run.notifications.length === 0, + "the user was notified about a refresh that needed nothing from them" + ); +} + +/** + * The same, the long way round: AWS federates to the identity provider, which + * still has a session and hands the user straight back. A cross-origin hop is + * not by itself a reason to show anybody anything. + */ +async function silentApprovalThroughIdp(frost) { + newRun("a federated refresh nobody has to see", { idp: "live" }); + + await withTimeout(frost.refresh(), "the refresh to finish"); + + check( + run.trail.includes(`page:${IDP}/signin`), + "the identity provider was never reached" + ); + check(run.approved, "the request was never approved"); + check(!run.everVisible, "the login window was shown"); + check(frost.hasToken(), "no token was stored"); +} + +/** + * The identity provider wants a password. That is the user's to answer, so the + * window has to come up โ€” and the approval steps after they sign in are Frost's + * again. + */ +async function signInShowsTheWindow(frost) { + newRun("a federated refresh that needs the user", { idp: "signin" }); + + const refreshing = frost.refresh(); + const window = await waitFor(visibleWindow, "the login window to be shown"); + + check( + run.trail.includes(`page:${IDP}/signin`), + "the window came up before the sign-in page did" + ); + + await signInOnPage(window); + await withTimeout(refreshing, "the refresh to finish"); + + check(run.approved, "the request was never approved"); + check(frost.hasToken(), "no token was stored"); +} + +/** + * Notify mode is a promise not to put a login page in front of the user + * unannounced, and it keeps that promise by announcing every refresh before + * anything opens โ€” including this one, which the portal session would have + * carried through without asking anybody anything. + */ +async function notifyModeAsksFirst(frost) { + newRun("a refresh in notify mode that needs nobody", {}); + + const refreshing = frost.refresh(); + await waitFor(frost.hasPendingAuth, "Frost to ask before starting the login"); + + check( + run.notifications.length > 0, + "nothing was said before the refresh waited for the user" + ); + check( + !run.trail.some((entry) => entry.startsWith("page:")), + "the login page was opened before the go-ahead" + ); + + // The user presses the hotkey, or clicks the notification. + frost.triggerPendingAuth(); + await withTimeout(refreshing, "the refresh to finish"); + + // From here it is an ordinary silent approval. + check(frost.hasToken(), "no token was stored"); + check(!run.everVisible, "the login window was shown"); +} + +/** + * Someone who picked the default browser picked it because that is where their + * passkeys and saved passwords are. The silent attempt is only a probe: the + * moment it needs them, the browser gets the login and the probe goes away. + */ +async function defaultBrowserHandsOver(frost) { + newRun("a refresh that needs the user, in default-browser mode", { + idp: "signin", + lifetimeSec: SHORT_DEVICE_CODE_LIFETIME_SEC, + }); + + const refreshing = frost.refresh(); + await waitFor( + () => run.browserOpens.length > 0, + "the login to be handed to the browser" + ); + + check( + run.browserOpens[0].includes(USER_CODE), + `the browser was sent somewhere unexpected: ${run.browserOpens[0]}` + ); + check( + !run.everVisible, + "a window was shown to someone who asked for their browser" + ); + + // Nobody finishes it over there, so the run ends with the device code. + await withTimeout(refreshing, "the refresh to give up"); + check(!frost.hasToken(), "a token appeared from nowhere"); +} + +/** + * With the setting off, nothing is driven and nothing is hidden: the login + * window is on screen from the start, exactly as it was before the feature. + */ +async function settingOffShowsEverything(frost) { + newRun("a refresh with automatic approval switched off", { + lifetimeSec: SHORT_DEVICE_CODE_LIFETIME_SEC, + }); + + const refreshing = frost.refresh(); + await waitFor(visibleWindow, "the login window to be shown"); + + // Long enough that a driver, if one were attached, would have clicked. + await new Promise((resolve) => setTimeout(resolve, 2000)); + check( + !run.trail.includes(`page:${PORTAL}/next`), + "something confirmed the request with the setting off" + ); + check(!run.approved, "the request was approved anyway"); + + closeVisibleWindow(); + await withTimeout(refreshing, "the refresh to end after the window closed"); + check(!frost.hasToken(), "a token appeared from nowhere"); +} + +/** + * The identity provider's pages are not Frost's to drive, however familiar + * their buttons look. This one offers exactly the label the driver wants. + */ +async function identityProviderIsNotOursToClick(frost) { + newRun("an identity provider offering a button of our own name", { + idp: "trap", + lifetimeSec: SHORT_DEVICE_CODE_LIFETIME_SEC, + }); + + const refreshing = frost.refresh(); + await waitFor(visibleWindow, "the login window to be shown"); + + check( + !run.trail.some((entry) => entry.includes("/clicked-idp-allow")), + "the driver clicked a button on the identity provider" + ); + check(!run.approved, "the request was approved"); + + closeVisibleWindow(); + await withTimeout(refreshing, "the refresh to end after the window closed"); + check(!frost.hasToken(), "a token appeared from nowhere"); +} + +/** + * A page with nothing the driver recognises, and two refusals next to it โ€” one + * carrying the id of the button it wants. Clicking any of them denies the + * request; the right answer is to touch nothing and let the user look at it. + */ +async function unknownPageHandsOver(frost) { + newRun("a page the driver does not recognise", { + firstPage: UNKNOWN_PAGE, + lifetimeSec: STALLING_DEVICE_CODE_LIFETIME_SEC, + }); + + const refreshing = frost.refresh(); + await waitFor( + visibleWindow, + "the login window to be shown for a page nobody could drive" + ); + + check( + !run.trail.some((entry) => entry.includes("/clicked-")), + "the driver clicked something it should not have" + ); + check(!run.approved, "the request was approved"); + + closeVisibleWindow(); + await withTimeout(refreshing, "the refresh to end after the window closed"); + check(!frost.hasToken(), "a token appeared from nowhere"); +} + +/** + * Nobody is at the machine. A window shown now, or a tab opened in a browser + * nobody is looking at, is a login page that expires unseen โ€” so the attempt is + * held instead, hidden and still being driven, and handed over the moment + * somebody turns up. + */ +async function unattendedLoginIsParked(frost) { + newRun("a refresh that needs the user, with nobody at the machine", { + idp: "signin", + }); + beAway(); + + const refreshing = frost.refresh(); + await waitFor( + () => run.trail.includes(`page:${IDP}/signin`), + "the sign-in page to be reached" + ); + // Long enough that a hand-over, if one were coming, would have happened. + await new Promise((resolve) => setTimeout(resolve, 2000)); + + check(!run.everVisible, "a login window was shown to an empty room"); + check(run.notifications.length === 0, "an empty room was notified"); + + // The user turns up; the page they get is the one Frost has been holding. + beHere(); + const window = await waitFor( + visibleWindow, + "the held login to be shown once the user is back" + ); + await signInOnPage(window); + await withTimeout(refreshing, "the refresh to finish"); + + check(frost.hasToken(), "no token was stored"); +} + +const TESTS = [ + ["silent approval: token collected, nothing shown", silentApproval, {}], + [ + "federated session: the identity provider hop stays silent too", + silentApprovalThroughIdp, + {}, + ], + [ + "sign-in needed: window shown, approval finished afterwards", + signInShowsTheWindow, + {}, + ], + [ + "notify mode: nothing opens before the user says go", + notifyModeAsksFirst, + { refreshMode: "notify" }, + ], + [ + "default browser: handed over there, no window shown", + defaultBrowserHandsOver, + { loginMethod: "default_browser" }, + ], + [ + "setting off: window shown from the start, nothing driven", + settingOffShowsEverything, + { autoApprove: false }, + ], + [ + "identity provider: its buttons are never clicked", + identityProviderIsNotOursToClick, + {}, + ], + [ + "unrecognised page: nothing clicked, not even the id trap, window shown", + unknownPageHandsOver, + {}, + ], + [ + "nobody there: login held hidden, shown when the user comes back", + unattendedLoginIsParked, + {}, + ], +]; + +// โ”€โ”€ Wiring โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ + +async function main() { + console.log(`auto-approve end-to-end tests (Frost ${version})`); + + // Everything the app writes goes here, and is thrown away at the end. + const home = fs.mkdtempSync(path.join(os.tmpdir(), "frost-check-")); + process.env.HOME = home; + process.env.USERPROFILE = home; + app.setPath("userData", path.join(home, "userData")); + app.setPath("logs", path.join(home, "logs")); + + const { server, port } = await startApiStub(); + const endpoint = `http://127.0.0.1:${port}`; + process.env.AWS_ENDPOINT_URL_SSO_OIDC = endpoint; + process.env.AWS_ENDPOINT_URL_SSO = endpoint; + process.env.AWS_REGION = "us-east-1"; + // The device-flow calls are unauthenticated, but the SDK still resolves a + // credential chain; without these it would go looking for real ones. + process.env.AWS_ACCESS_KEY_ID = "frost-check"; + process.env.AWS_SECRET_ACCESS_KEY = "frost-check"; + + await app.whenReady(); + + // The app quits by default once the last window closes, and every scenario + // here closes one โ€” src/main.ts holds it open with the same handler. + app.on("window-all-closed", () => {}); + + interceptPages(); + sampleWindows(); + recordBrowserOpens(); + assert.ok( + controlPresence(), + "could not control the idle time; the tests would depend on whether anyone touched the keyboard" + ); + assert.ok( + recordNotifications(), + "could not record notifications; the tests cannot tell what the user was told" + ); + + // Imported after the paths are redirected and the recorders are in place: + // the store is constructed on import, and the app's modules capture + // `Notification` as they are evaluated. + const { config } = await import("../dist/config.js"); + const { refresh, cancelTokenRefresh, hasPendingAuth, triggerPendingAuth } = + await import("../dist/aws-sso.js"); + + const frost = { + refresh, + hasPendingAuth, + triggerPendingAuth, + hasToken: () => { + const expiresAt = config.get("expiresAt"); + return Boolean(expiresAt) && Date.parse(expiresAt) > Date.now(); + }, + reset: (behavior) => { + cancelTokenRefresh(); + config.set("isWorking", false); + config.delete("accessToken"); + config.delete("expiresAt"); + config.delete("ssoClient"); + config.set("userConfig", { + startUrl: `${PORTAL}/start`, + region: "us-east-1", + }); + config.set("behaviorConfig", { + refreshMode: "auto", + refreshHotkey: "CmdOrCtrl+Shift+R", + historyRetentionDays: 7, + loginMethod: "popup", + autoApprove: true, + ...behavior, + }); + }, + }; + + let failures = 0; + for (const [label, scenario, behavior] of TESTS) { + frost.reset(behavior); + beHere(); + const startedAt = Date.now(); + try { + await scenario(frost); + console.log( + ` ok ${label} (${((Date.now() - startedAt) / 1000).toFixed(1)}s)` + ); + } catch (err) { + failures += 1; + console.error(` FAIL ${label}\n ${err.message}`); + } + for (const window of BrowserWindow.getAllWindows()) { + if (!window.isDestroyed()) window.destroy(); + } + } + + clearInterval(sampler); + cancelTokenRefresh(); + server.close(); + fs.rmSync(home, { recursive: true, force: true }); + + if (failures) { + console.error(`\n${failures} of ${TESTS.length} failing`); + app.exit(1); + return; + } + console.log(`auto-approve end-to-end tests passed (${TESTS.length} scenarios)`); + app.exit(0); +} + +main().catch((err) => { + console.error(err); + app.exit(1); +});