WIP: Add E2E tests to run in BrowserStack

.gitignore

@@ -16,6 +16,7 @@ dist-ssr/
 .env.local
 .env.production
 .env.*.local
+tests/browserstack/.env.browserstack
 
 # Editor directories and files
 .vscode/*

tests/browserstack/.env.browserstack.prod.example

@@ -0,0 +1,6 @@
+STORMBOX_TARGET_ENV=prod
+STORMBOX_BASE_URL=https://webmail.thundermail.com/
+TBPRO_USERNAME="your Thundermail username"
+TBPRO_PASSWORD="your Thundermail password"
+BROWSERSTACK_USERNAME="your browserstack account user name"
+BROWSERSTACK_ACCESS_KEY="your corresponding browserstack access key"

tests/browserstack/.env.browserstack.stage.example

@@ -0,0 +1,6 @@
+STORMBOX_TARGET_ENV=stage
+STORMBOX_BASE_URL=https://webmail.stage-thundermail.com/
+TBPRO_USERNAME="your Thundermail username"
+TBPRO_PASSWORD="your Thundermail password"
+BROWSERSTACK_USERNAME="your browserstack account user name"
+BROWSERSTACK_ACCESS_KEY="your corresponding browserstack access key"

tests/browserstack/README.md

@@ -0,0 +1,61 @@
+# Stormbox BrowserStack E2E Tests
+
+This package contains UI-only Playwright tests for deployed Stormbox stage and production. These tests run against the public Stormbox web application on your local machine or in BrowserStack.
+
+These tests are not for the local Stormbox stack. The local-stack integration tests live in `../e2e`; those tests use JMAP helpers, database reads, local stack setup, and cache assertions. Use this package when you want to verify the deployed UI only.
+
+## Setup
+
+Run these commands directly from this directory (NOT from the Stormbox root and not inside the `thundermail-dev` container):
+
+```bash
+cd tests/browserstack
+npm install
+npx playwright install
+```
+
+Note: If installing the playwright browsers fails (hangs at 100%), this is due to a known playwright/test 1.59 issue and 
+Node 24:16+. The issue is fixed in playwright/test 1.60 however we cannot use 1.60 because it is not supported on BrowserStack. The workaround for this issue is to use Node 24:15 or BELOW.
+
+Choose a target environment, then copy the matching example file:
+
+```bash
+cp .env.browserstack.stage.example .env.browserstack
+# or
+cp .env.browserstack.prod.example .env.browserstack
+```
+
+Fill in these values in `.env.browserstack`:
+
+```bash
+TBPRO_USERNAME="your Thundermail username"
+TBPRO_PASSWORD="your Thundermail password"
+BROWSERSTACK_USERNAME="your browserstack account user name"
+BROWSERSTACK_ACCESS_KEY="your corresponding browserstack access key"
+```
+
+The `.env.browserstack` file contains credentials and must stay local.
+
+## Local Runs
+
+These commands run Playwright from your machine against the deployed `STORMBOX_BASE_URL`:
+
+```bash
+npm run e2e:desktop:firefox
+npm run e2e:desktop:chrome
+npm run e2e:desktop:safari
+npm run e2e:mobile:google:pixel:viewport
+```
+
+## BrowserStack Runs
+
+These commands run the same UI-only coverage in BrowserStack using the checked-in BrowserStack YAML files:
+
+```bash
+npm run e2e:browserstack:desktop:firefox
+npm run e2e:browserstack:desktop:chrome
+npm run e2e:browserstack:desktop:safari
+npm run e2e:browserstack:mobile:android:chrome
+```
+
+Desktop runs authenticate once in `tests/desktop/auth.desktop.ts` and save `test-results/.auth/user.json`. Mobile runs sign in through the UI for each test because BrowserStack mobile contexts cannot use the saved desktop auth state.

tests/browserstack/browserstack-desktop-nightly.yml

@@ -0,0 +1,106 @@
+# =============================
+# Set BrowserStack Credentials
+# =============================
+# In the terminal export BROWSERSTACK_USERNAME=<secret>
+# In the terminal export BROWSERSTACK_ACCESS_KEY=<secret>
+
+# ======================
+# BrowserStack Reporting
+# ======================
+# The following capabilities are used to set up reporting on BrowserStack:
+# Set 'projectName' to the name of your project. Example, Marketing Website
+projectName: Stormbox
+# Set `buildName` as the name of the job / testsuite being run
+buildName: Nightly Webmail E2E Tests (Desktop)
+# `buildIdentifier` is a unique id to differentiate every execution that gets appended to
+# buildName. Choose your buildIdentifier format from the available expressions:
+# ${BUILD_NUMBER} (Default): Generates an incremental counter with every execution
+# ${DATE_TIME}: Generates a Timestamp with every execution. Eg. 05-Nov-19:30
+# Read more about buildIdentifiers here -> https://www.browserstack.com/docs/automate/selenium/organize-tests
+buildIdentifier: '${DATE_TIME}'
+
+# =======================================
+# Platforms (Browsers / Devices to test)
+# =======================================
+# Platforms object contains all the browser / device combinations you want to test on.
+# For browser/os versions compatible with our playwright version see https://www.browserstack.com/docs/automate/playwright/browsers-and-os?fw-lang=nodejs
+platforms:
+  - os: OS X
+    osVersion: Sequoia
+    browserName: playwright-firefox
+    browserVersion: 139.0
+    playwrightConfigOptions:
+      name: Firefox-OSX
+      setup: 
+      - name: 'firefox_auth'
+        testMatch: 'auth.desktop.ts'
+      use:
+        storageState: 'test-results/.auth/user.json'
+
+  - os: OS X
+    osVersion: Sequoia
+    browserName: playwright-webkit
+    browserVersion: 18.5
+    playwrightConfigOptions:
+      name: Safari-OSX
+      setup: 
+      - name: 'safari_auth'
+        testMatch: 'auth.desktop.ts'
+      use:
+        storageState: 'test-results/.auth/user.json'
+
+  - os: Windows
+    osVersion: 11
+    browserName: playwright-chromium
+    browserVersion: 138.0
+    playwrightConfigOptions:
+      name: Chromium-Win11
+      setup: 
+      - name: 'chromium_auth'
+        testMatch: 'auth.desktop.ts'
+      use:
+        storageState: 'test-results/.auth/user.json'
+
+# =======================
+# Parallels per Platform
+# =======================
+# The number of parallel threads to be used for each platform set.
+# BrowserStack's SDK runner will select the best strategy based on the configured value
+#
+# Example 1 - If you have configured 3 platforms and set `parallelsPerPlatform` as 2, a total of 6 (2 * 3) parallel threads will be used on BrowserStack
+#
+# Example 2 - If you have configured 1 platform and set `parallelsPerPlatform` as 5, a total of 5 (1 * 5) parallel threads will be used on BrowserStack
+parallelsPerPlatform: 1
+
+# ==========================================
+# BrowserStack Local
+# (For localhost, staging/private websites)
+# ==========================================
+# Set browserStackLocal to true if your website under test is not accessible publicly over the internet
+# Learn more about how BrowserStack Local works here -> https://www.browserstack.com/docs/automate/selenium/local-testing-introduction
+browserstackLocal: false # <boolean> (Default false)
+# browserStackLocalOptions:
+# Options to be passed to BrowserStack local in-case of advanced configurations
+  # localIdentifier: # <string> (Default: null) Needed if you need to run multiple instances of local.
+  # forceLocal: true  # <boolean> (Default: false) Set to true if you need to resolve all your traffic via BrowserStack Local tunnel.
+  # Entire list of arguments available here -> https://www.browserstack.com/docs/automate/selenium/manage-incoming-connections
+
+# ===================
+# Debugging features
+# ===================
+debug: false # <boolean> # Set to true if you need screenshots for every selenium command ran
+networkLogs: false # <boolean> Set to true to enable HAR logs capturing; off as may contain sensitive info like login API requests
+consoleLogs: info # <string> Remote browser's console debug levels to be printed (`disable`, `errors`, `warnings`, `info`, or `verbose`)
+framework: playwright
+# the e2e tests use the same node.js project so the playwright version dependecy must work on both desktop
+# and mobile browsers; and therefore must be the same in all of our browserstack-*.yml config files
+# for playwright ver we need one compatible with our desktop and mobile browsers, see:
+# https://www.browserstack.com/docs/automate/playwright/browsers-and-os?fw-lang=nodejs
+browserstack.playwrightVersion: 1.59.1 # must match our client playwright version in package.json
+browserstack.playwrightLogs: false # disable playwright logs appearing on browserstack builds as may contain sensitive info like credentials
+# CUSTOM_TAG_<INT>: # <string> (Default: parent folder name of the test file) Custom tag for your test suite
+browserstack.maskBasicAuth: true # mask username and passwords from browserstack session logs
+browserstack.maskCommands: sendType, sendPress, setHTTPCredentials, setStorageState, setWebAuthnCredentials # prevent sensitive info appearing in browserstack session logs
+# Test Observability is an intelligent test reporting & debugging product. It collects data using the SDK. Read more about what data is collected at https://www.browserstack.com/docs/test-observability/references/terms-and-conditions
+# Visit observability.browserstack.com to see your test reports and insights. To disable test observability, specify `testObservability: false` in the key below.
+testObservability: false

tests/browserstack/browserstack-mobile-nightly.yml

@@ -0,0 +1,83 @@
+# =============================
+# Set BrowserStack Credentials
+# =============================
+# In the terminal export BROWSERSTACK_USERNAME=<secret>
+# In the terminal export BROWSERSTACK_ACCESS_KEY=<secret>
+
+# ======================
+# BrowserStack Reporting
+# ======================
+# The following capabilities are used to set up reporting on BrowserStack:
+# Set 'projectName' to the name of your project. Example, Marketing Website
+projectName: Stormbox
+# Set `buildName` as the name of the job / testsuite being run
+buildName: Nightly Webmail E2E Tests (Mobile)
+# `buildIdentifier` is a unique id to differentiate every execution that gets appended to
+# buildName. Choose your buildIdentifier format from the available expressions:
+# ${BUILD_NUMBER} (Default): Generates an incremental counter with every execution
+# ${DATE_TIME}: Generates a Timestamp with every execution. Eg. 05-Nov-19:30
+# Read more about buildIdentifiers here -> https://www.browserstack.com/docs/automate/selenium/organize-tests
+buildIdentifier: '${DATE_TIME}'
+
+# =======================================
+# Platforms (Browsers / Devices to test)
+# =======================================
+# Platforms object contains all the browser / device combinations you want to test on.
+# For browser/os versions compatible with our playwright version see https://www.browserstack.com/docs/automate/playwright/browsers-and-os?fw-lang=nodejs
+platforms:
+  - browserName: chrome
+    osVersion: 16.0
+    deviceName: Google Pixel 10
+    playwrightConfigOptions:
+      name: android-chrome
+    chrome:
+      chromeOptions:
+        prefs:
+          credentials_enable_service: false
+        profile:
+          password_manager_enabled: false
+
+# =======================
+# Parallels per Platform
+# =======================
+# The number of parallel threads to be used for each platform set.
+# BrowserStack's SDK runner will select the best strategy based on the configured value
+#
+# Example 1 - If you have configured 3 platforms and set `parallelsPerPlatform` as 2, a total of 6 (2 * 3) parallel threads will be used on BrowserStack
+#
+# Example 2 - If you have configured 1 platform and set `parallelsPerPlatform` as 5, a total of 5 (1 * 5) parallel threads will be used on BrowserStack
+parallelsPerPlatform: 1
+
+# ==========================================
+# BrowserStack Local
+# (For localhost, staging/private websites)
+# ==========================================
+# Set browserStackLocal to true if your website under test is not accessible publicly over the internet
+# Learn more about how BrowserStack Local works here -> https://www.browserstack.com/docs/automate/selenium/local-testing-introduction
+browserstackLocal: false # <boolean> (Default false)
+# browserStackLocalOptions:
+# Options to be passed to BrowserStack local in-case of advanced configurations
+  # localIdentifier: # <string> (Default: null) Needed if you need to run multiple instances of local.
+  # forceLocal: true  # <boolean> (Default: false) Set to true if you need to resolve all your traffic via BrowserStack Local tunnel.
+  # Entire list of arguments available here -> https://www.browserstack.com/docs/automate/selenium/manage-incoming-connections
+
+# ===================
+# Debugging features
+# ===================
+debug: false # <boolean> # Set to true if you need screenshots for every selenium command ran
+networkLogs: false # <boolean> Set to true to enable HAR logs capturing; off as may contain sensitive info like login API requests
+consoleLogs: info # <string> Remote browser's console debug levels to be printed (`disable`, `errors`, `warnings`, `info`, or `verbose`)
+framework: playwright
+idleTimeout: 500 # seeing if this helps rid BROWSERSTACK_IDLE_TIMEOUTs on ios
+# the e2e tests use the same node.js project so the playwright version dependecy must work on both desktop
+# and mobile browsers; and therefore must be the same in all of our browserstack-*.yml config files
+# for playwright ver we need one compatible with our desktop and mobile browsers, see:
+# https://www.browserstack.com/docs/automate/playwright/browsers-and-os?fw-lang=nodejs
+browserstack.playwrightVersion: 1.59.1 # must match ver in package.json
+browserstack.playwrightLogs: false # disable playwright logs appearing on browserstack builds as may contain sensitive info like credentials
+# CUSTOM_TAG_<INT>: # <string> (Default: parent folder name of the test file) Custom tag for your test suite
+browserstack.maskBasicAuth: true # mask username and passwords from browserstack session logs
+browserstack.maskCommands: sendType, sendPress, setHTTPCredentials, setStorageState, setWebAuthnCredentials # prevent sensitive info appearing in browserstack session logs
+# Test Observability is an intelligent test reporting & debugging product. It collects data using the SDK. Read more about what data is collected at https://www.browserstack.com/docs/test-observability/references/terms-and-conditions
+# Visit observability.browserstack.com to see your test reports and insights. To disable test observability, specify `testObservability: false` in the key below.
+testObservability: false

tests/browserstack/const/constants.ts

@@ -0,0 +1,14 @@
+export const STORMBOX_TARGET_ENV = String(process.env.STORMBOX_TARGET_ENV ?? '');
+export const STORMBOX_BASE_URL = String(process.env.STORMBOX_BASE_URL ?? '');
+
+export const TBPRO_USERNAME = String(process.env.TBPRO_USERNAME ?? '');
+export const TBPRO_PASSWORD = String(process.env.TBPRO_PASSWORD ?? '');
+
+export const PLAYWRIGHT_TAG_DESKTOP = '@stormbox-desktop';
+export const PLAYWRIGHT_TAG_MOBILE = '@stormbox-mobile';
+
+export const TIMEOUT_2_SECONDS = 2_000;
+export const TIMEOUT_5_SECONDS = 5_000;
+export const TIMEOUT_10_SECONDS = 10_000;
+export const TIMEOUT_30_SECONDS = 30_000;
+export const TIMEOUT_60_SECONDS = 60_000;