From 1455b791ab3577881d891919cba720f4f9bdd6d4 Mon Sep 17 00:00:00 2001 From: tomasvarga Date: Thu, 12 Nov 2026 12:35:52 +0100 Subject: [PATCH] Add e2e example: end-to-end tests on Android and in Chromium (JavaScript) examples/e2e-android-web-tests-js runs e2e tests in E2B sandboxes with the e2b() and e2bBrowsers() providers from @e2e-dev/e2b: 2048 on Android 16 and Wikipedia in Chromium, each with one scripted test and one agent test. No package-lock.json yet: @e2e-dev/e2b 0.0.1 is not on npm until e2e-inc/e2e#1590 is released. --- README.md | 6 ++ .../e2e-android-web-tests-js/.env.example | 11 +++ examples/e2e-android-web-tests-js/.gitignore | 4 + .../e2e-android-web-tests-js/.oxlintrc.json | 5 ++ examples/e2e-android-web-tests-js/README.md | 86 +++++++++++++++++++ .../e2e-android-web-tests-js/e2e.config.ts | 39 +++++++++ .../e2e-android-web-tests-js/package.json | 32 +++++++ .../scripts/build-template.ts | 14 +++ .../scripts/download-apk.ts | 22 +++++ .../tests/android/2048.e2e.ts | 39 +++++++++ .../tests/web/wikipedia.e2e.ts | 24 ++++++ .../e2e-android-web-tests-js/tsconfig.json | 15 ++++ tests/run-examples.ts | 5 ++ 13 files changed, 302 insertions(+) create mode 100644 examples/e2e-android-web-tests-js/.env.example create mode 100644 examples/e2e-android-web-tests-js/.gitignore create mode 100644 examples/e2e-android-web-tests-js/.oxlintrc.json create mode 100644 examples/e2e-android-web-tests-js/README.md create mode 100644 examples/e2e-android-web-tests-js/e2e.config.ts create mode 100644 examples/e2e-android-web-tests-js/package.json create mode 100644 examples/e2e-android-web-tests-js/scripts/build-template.ts create mode 100644 examples/e2e-android-web-tests-js/scripts/download-apk.ts create mode 100644 examples/e2e-android-web-tests-js/tests/android/2048.e2e.ts create mode 100644 examples/e2e-android-web-tests-js/tests/web/wikipedia.e2e.ts create mode 100644 examples/e2e-android-web-tests-js/tsconfig.json diff --git a/README.md b/README.md index 9dd23f5c..c04a70c4 100644 --- a/README.md +++ b/README.md @@ -312,6 +312,12 @@ Read more about E2B on the [E2B website](https://e2b.dev/?utm_source=github&utm_ Python - + + e2e + Run end-to-end tests on Android and in Chromium, one sandbox per target for each test worker + - + TypeScript + diff --git a/examples/e2e-android-web-tests-js/.env.example b/examples/e2e-android-web-tests-js/.env.example new file mode 100644 index 00000000..e0fe628e --- /dev/null +++ b/examples/e2e-android-web-tests-js/.env.example @@ -0,0 +1,11 @@ +# https://e2b.dev/dashboard +E2B_API_KEY= + +# Optional. Without it, the two agent tests are skipped. +# https://console.anthropic.com/settings/keys +ANTHROPIC_API_KEY= +# Optional, defaults to claude-haiku-5-5 +ANTHROPIC_MODEL= + +# Optional, defaults to e2e-chromium (the name that npm run build-template uses) +E2B_BROWSER_TEMPLATE= diff --git a/examples/e2e-android-web-tests-js/.gitignore b/examples/e2e-android-web-tests-js/.gitignore new file mode 100644 index 00000000..c900f4e3 --- /dev/null +++ b/examples/e2e-android-web-tests-js/.gitignore @@ -0,0 +1,4 @@ +node_modules/ +.env +.e2e/ +apps/ diff --git a/examples/e2e-android-web-tests-js/.oxlintrc.json b/examples/e2e-android-web-tests-js/.oxlintrc.json new file mode 100644 index 00000000..fbd4e928 --- /dev/null +++ b/examples/e2e-android-web-tests-js/.oxlintrc.json @@ -0,0 +1,5 @@ +{ + "$schema": "./node_modules/oxlint/configuration_schema.json", + "categories": { "correctness": "error", "suspicious": "warn" }, + "rules": { "curly": "error" } +} diff --git a/examples/e2e-android-web-tests-js/README.md b/examples/e2e-android-web-tests-js/README.md new file mode 100644 index 00000000..928d8525 --- /dev/null +++ b/examples/e2e-android-web-tests-js/README.md @@ -0,0 +1,86 @@ +# End-to-end tests on Android and in Chromium with e2e + E2B + +[e2e](https://github.com/e2e-inc/e2e) is an open-source runner for end-to-end tests. A test can use scripted steps, agent steps (a model works toward a goal), or both. This example runs tests in E2B sandboxes with the two providers from [`@e2e-dev/e2b`](https://github.com/e2e-inc/e2e/tree/main/packages/e2b): + +- `e2b()` leases Android 16 sandboxes. The example tests the game 2048. You do not need the Android SDK, an emulator, or `adb` on your machine. +- `e2bBrowsers()` leases Chromium sandboxes. The example tests Wikipedia. + +Each target has one scripted test and one agent test. The tests do not change when you move them from a local device or browser to E2B. + +``` +your machine E2B +e2e run ──┬─ target "android" ──tunnel──▶ Android 16 sandbox (template android-16) + │ 2048.apk installed by the provider + └─ target "web" ──────tunnel──▶ Chromium sandbox (template e2e-chromium) +``` + +The runner leases one sandbox per target for each worker and deletes them when the run ends. The sandboxes refuse public traffic: the runner reaches each one through a tunnel on `127.0.0.1` that adds the E2B access token. + +## Prerequisites + +- Node.js 22.22.3 or later (or 24.8.0 or later) +- An [E2B API key](https://e2b.dev/dashboard) + +## Run the Android tests + +```bash +cd examples/e2e-android-web-tests-js +npm install +cp .env.example .env # add E2B_API_KEY +npm run download-apk # saves apps/2048.apk from F-Droid and checks its SHA-256 +npm start +``` + +The Android target uses E2B's public `android-16` template, so there is nothing to build. The scripted test passes and the agent test is skipped. + +## Optional: run the agent tests + +Add an [Anthropic API key](https://console.anthropic.com/settings/keys) to `.env` as `ANTHROPIC_API_KEY` and run `npm start` again. The agent gets a goal ("start a new game, and swipe the board until the score is above 0"), and a scripted assertion checks the result. + +## Optional: run the Chromium tests + +The web target needs a sandbox template with Chromium. Build it one time for each E2B team: + +```bash +npm run build-template # builds the e2e-chromium template from the Playwright image +npm run test:web # only Chromium +npm run test:all # Android and Chromium +``` + +The image tag in `scripts/build-template.ts` must match the Playwright version that `@e2e-dev/web` uses. When you update `@e2e-dev/web`, update the tag and build the template again. + +## Results + +`npm run test:all` runs four tests. With an Anthropic key, all four pass. Without it, the two scripted tests pass and the two agent tests are skipped. A run takes about one minute. + +The report, the screenshots, and the agent step cache are in `.e2e/`. The runner can replay a passed agent step from the cache without model calls. It calls the model again when the screen changed or when a step used an action that cannot be replayed, such as the Android swipe tool. + +## Where to read the code + +- `e2e.config.ts`: the two targets, the two E2B providers, and the agent model. +- `tests/android/2048.e2e.ts`: scripted swipes on the board, then the same goal given to an agent. +- `tests/web/wikipedia.e2e.ts`: a scripted search, then the same goal given to an agent. +- `scripts/build-template.ts`: the Chromium template. +- `scripts/download-apk.ts`: the pinned download of the app under test. + +`npm run check` runs the typecheck and the lint (no network). + +## Use your own app + +- **Android**: set `app.appPath` to your `.apk`. Add `bundleId` (the package name of the app) if the runner must not read it from the `.apk`. +- **Web**: set `app.url` to your site. The browser runs in E2B's cloud, so the URL must be reachable from the internet. `localhost` on your machine is not. +- **More workers**: increase `workers`. Each worker gets its own sandbox for each target. + +See the [E2B integration guide](https://github.com/e2e-inc/e2e/blob/main/docs/integrations/e2b.mdx) in the e2e repository for all options. + +## Limitations + +- Android on E2B is x86_64 only, has no Google Play Services, and renders in software. Apps with native libraries need x86_64 builds. The provider installs an `.apk`, not an `.aab`. +- E2B has no iOS sandboxes. +- A sandbox is billed while it runs. The sandboxes run from the start of the run to its end. +- The default sandbox lifetime is 1 hour (`timeoutMs`). If the runner is killed before it deletes a sandbox, E2B deletes it when the lifetime ends. +- The web tests use the live Wikipedia site, so a change to that site can break them. + +Tested with e2e 0.23.1, `@e2e-dev/e2b` 0.0.1, E2B SDK 2.63.0 and Node.js 24. + +2048 is [Privacy Friendly 2048](https://github.com/SecUSo/privacy-friendly-2048) by SECUSO (GPL-3.0). The example downloads it from F-Droid and does not redistribute it. diff --git a/examples/e2e-android-web-tests-js/e2e.config.ts b/examples/e2e-android-web-tests-js/e2e.config.ts new file mode 100644 index 00000000..7657efb8 --- /dev/null +++ b/examples/e2e-android-web-tests-js/e2e.config.ts @@ -0,0 +1,39 @@ +import { existsSync } from 'node:fs'; +import { anthropic } from '@ai-sdk/anthropic'; +import { e2b } from '@e2e-dev/e2b'; +import { e2bBrowsers } from '@e2e-dev/e2b/web'; +import { mobile } from '@e2e-dev/mobile'; +import { mobileTools } from '@e2e-dev/mobile/tools'; +import { web } from '@e2e-dev/web'; +import type { E2EConfig } from 'e2e'; + +// e2e does not read .env, so load it before the providers look for E2B_API_KEY. +if (existsSync('.env')) { + process.loadEnvFile('.env'); +} + +// One Android 16 sandbox per worker. The provider installs the .apk and deletes the sandbox after the run. +const android = mobile({ platform: 'android', device: e2b() }); + +// One Chromium sandbox per worker, from the template that `npm run build-template` builds. +const chromium = web({ + browser: e2bBrowsers({ template: process.env.E2B_BROWSER_TEMPLATE || 'e2e-chromium' }), +}); + +export default { + targets: [ + { name: 'android', engine: android, app: { appPath: 'apps/2048.apk' } }, + // The browser runs in E2B's cloud, so the URL must be reachable from the internet. + { name: 'web', engine: chromium, app: { url: 'https://en.wikipedia.org' } }, + ], + // Each worker is one sandbox per target, and the sandboxes run for the whole run. + workers: 1, + agents: { + default: { + // Only the agent tests call the model. They are skipped without ANTHROPIC_API_KEY. + model: anthropic(process.env.ANTHROPIC_MODEL || 'claude-haiku-5-5'), + // Adds swipe, open_app and alert to the agent's taps and typing on Android. + tools: mobileTools(android), + }, + }, +} satisfies E2EConfig; diff --git a/examples/e2e-android-web-tests-js/package.json b/examples/e2e-android-web-tests-js/package.json new file mode 100644 index 00000000..f4be0f65 --- /dev/null +++ b/examples/e2e-android-web-tests-js/package.json @@ -0,0 +1,32 @@ +{ + "name": "e2e-android-web-tests", + "private": true, + "type": "module", + "engines": { + "node": "^22.22.3 || >=24.8.0" + }, + "scripts": { + "start": "e2e run --target android", + "test:web": "e2e run --target web", + "test:all": "e2e run", + "download-apk": "node scripts/download-apk.ts", + "build-template": "node --env-file-if-exists=.env scripts/build-template.ts", + "typecheck": "tsc --noEmit", + "lint": "oxlint", + "check": "npm run typecheck && npm run lint" + }, + "dependencies": { + "@ai-sdk/anthropic": "4.0.102", + "@e2e-dev/e2b": "0.0.1", + "@e2e-dev/mobile": "0.12.7", + "@e2e-dev/web": "0.16.0", + "ai": "7.0.170", + "e2b": "2.63.0", + "e2e": "0.23.1" + }, + "devDependencies": { + "@types/node": "24.19.1", + "oxlint": "1.93.0", + "typescript": "7.0.2" + } +} diff --git a/examples/e2e-android-web-tests-js/scripts/build-template.ts b/examples/e2e-android-web-tests-js/scripts/build-template.ts new file mode 100644 index 00000000..ae116284 --- /dev/null +++ b/examples/e2e-android-web-tests-js/scripts/build-template.ts @@ -0,0 +1,14 @@ +// Builds the Chromium sandbox template for the web target. Run it once per E2B team. +// The image tag must match the Playwright version that @e2e-dev/web uses (playwright-core in its package.json). +import { Template, defaultBuildLogger } from 'e2b'; + +const NAME = process.env.E2B_BROWSER_TEMPLATE || 'e2e-chromium'; + +const template = Template().fromImage('mcr.microsoft.com/playwright:v1.65.0-noble'); + +await Template.build(template, NAME, { + cpuCount: 4, + memoryMB: 4096, + onBuildLogs: defaultBuildLogger(), +}); +console.log(`Template ${NAME} is ready`); diff --git a/examples/e2e-android-web-tests-js/scripts/download-apk.ts b/examples/e2e-android-web-tests-js/scripts/download-apk.ts new file mode 100644 index 00000000..4c779f1a --- /dev/null +++ b/examples/e2e-android-web-tests-js/scripts/download-apk.ts @@ -0,0 +1,22 @@ +// Downloads the app under test: Privacy Friendly 2048 (GPL-3.0) from F-Droid, pinned by SHA-256. +import { createHash } from 'node:crypto'; +import { mkdir, writeFile } from 'node:fs/promises'; + +const URL = 'https://f-droid.org/repo/org.secuso.privacyfriendly2048_100.apk'; +const SHA256 = '02c799d3d582669daf2acf920093c68d2933f60aa937bb72fa2a805557233fe8'; +const FILE = 'apps/2048.apk'; + +const response = await fetch(URL); +if (!response.ok) { + throw new Error(`Download failed: ${response.status} ${URL}`); +} + +const apk = Buffer.from(await response.arrayBuffer()); +const sha256 = createHash('sha256').update(apk).digest('hex'); +if (sha256 !== SHA256) { + throw new Error(`Checksum mismatch for ${URL}: expected ${SHA256}, got ${sha256}`); +} + +await mkdir('apps', { recursive: true }); +await writeFile(FILE, apk); +console.log(`Saved ${FILE} (${(apk.length / 1e6).toFixed(1)} MB)`); diff --git a/examples/e2e-android-web-tests-js/tests/android/2048.e2e.ts b/examples/e2e-android-web-tests-js/tests/android/2048.e2e.ts new file mode 100644 index 00000000..d8aa6e6f --- /dev/null +++ b/examples/e2e-android-web-tests-js/tests/android/2048.e2e.ts @@ -0,0 +1,39 @@ +import { test } from '@e2e-dev/mobile'; +import { beforeEach, describe, expect } from 'e2e'; + +const APP_ID = 'org.secuso.privacyfriendly2048'; +const DIRECTIONS = ['left', 'down', 'right', 'up'] as const; +const MAX_SWIPES = 12; + +describe('2048 on Android', { platforms: ['android'] }, () => { + // Both tests share one sandbox, so each one starts from a fresh install state. + beforeEach(async ({ app }) => { + await app.clearState(); + }); + + test('scripted swipes change the score', async ({ app, screen }) => { + await screen.getByRole('button', 'SKIP').click(); + await screen.getByRole('button', 'START NEW GAME').click(); + + const board = screen.getByTestId(`${APP_ID}:id/touch_field`); + const points = screen.getByTestId(`${APP_ID}:id/points`); + await expect(points).toHaveText('0'); + + // The start tiles are random, so swipe until two tiles merge. + for (let swipe = 0; swipe < MAX_SWIPES && (await points.textContent()) === '0'; swipe++) { + await board.swipe({ direction: DIRECTIONS[swipe % DIRECTIONS.length]!, momentum: 'fast' }); + } + + await expect(points).not.toHaveText('0'); + await app.screenshot('2048-board'); + }); + + test('an agent plays from a goal', async ({ agent, screen }) => { + test.skip(!process.env.ANTHROPIC_API_KEY, 'set ANTHROPIC_API_KEY to run the agent tests'); + + await agent.act('Skip the tutorial, start a new game, and swipe the board until the score is above 0.'); + + // The agent acts, and a plain assertion checks the result. + await expect(screen.getByTestId(`${APP_ID}:id/points`)).not.toHaveText('0'); + }); +}); diff --git a/examples/e2e-android-web-tests-js/tests/web/wikipedia.e2e.ts b/examples/e2e-android-web-tests-js/tests/web/wikipedia.e2e.ts new file mode 100644 index 00000000..53981cb1 --- /dev/null +++ b/examples/e2e-android-web-tests-js/tests/web/wikipedia.e2e.ts @@ -0,0 +1,24 @@ +import { describe, expect, test } from 'e2e'; + +const ARTICLE = 'Firecracker (software)'; + +describe('Wikipedia in Chromium', { platforms: ['web'] }, () => { + test('a scripted search opens the article', async ({ app, screen }) => { + await app.open('/wiki/Main_Page'); + // A search for an exact title opens the article directly. + await screen.getByRole('searchbox', /search wikipedia/i).fill(ARTICLE); + await screen.getByRole('button', 'Search').click(); + + await expect(screen.getByRole('heading', { name: ARTICLE, level: 1 })).toBeVisible(); + }); + + test('an agent finds the article from a goal', async ({ agent, app, screen }) => { + test.skip(!process.env.ANTHROPIC_API_KEY, 'set ANTHROPIC_API_KEY to run the agent tests'); + + await app.open('/wiki/Main_Page'); + await agent.act(`Search Wikipedia for "${ARTICLE}" and open that article.`); + + // The agent acts, and a plain assertion checks the result. + await expect(screen.getByRole('heading', { name: ARTICLE, level: 1 })).toBeVisible(); + }); +}); diff --git a/examples/e2e-android-web-tests-js/tsconfig.json b/examples/e2e-android-web-tests-js/tsconfig.json new file mode 100644 index 00000000..1a9a98f0 --- /dev/null +++ b/examples/e2e-android-web-tests-js/tsconfig.json @@ -0,0 +1,15 @@ +{ + "compilerOptions": { + "target": "ES2023", + "module": "NodeNext", + "moduleResolution": "NodeNext", + "strict": true, + "noUncheckedIndexedAccess": true, + "allowImportingTsExtensions": true, + "erasableSyntaxOnly": true, + "noEmit": true, + "skipLibCheck": true, + "types": ["node"] + }, + "include": ["e2e.config.ts", "tests", "scripts"] +} diff --git a/tests/run-examples.ts b/tests/run-examples.ts index d28e20d5..83ba3847 100644 --- a/tests/run-examples.ts +++ b/tests/run-examples.ts @@ -164,6 +164,11 @@ const scripts: { // suite does not install Crabbox. Its unit tests cover command order and cleanup; // the real provider lifecycle remains a separate, explicitly approved smoke: // crabbox-e2b-python +// The e2e test runner leases its own Android and Chromium sandboxes, so running it +// here would nest live sandboxes inside this runner, and the web target needs the +// e2e-chromium template built on the account first. It is verified by a live run +// from a developer machine instead: +// e2e-android-web-tests-js // Runs TRL on the evaluator host and reuses E2B workers for isolated code rewards. // This runner would nest sandbox creation and has no training GPU. Core reward // checks run in .github/workflows/trl-grpo.yml; live smoke and training are separate: