diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 86d644b..5f2d150 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -134,6 +134,44 @@ jobs: - run: bash conformance/vm/e2e.sh "$USER@localhost" --no-firewall --no-ssh-hardening --no-swap --no-auto-updates + # nextship-loadtest for real: k6 has to accept the script the package writes and + # return a summary its report can read, which no unit test can show. Both ways of + # finding k6 are run, Docker first, while no k6 is installed to be preferred to it. + loadtest: + name: Load test command + runs-on: ubuntu-latest + defaults: + run: + shell: bash + steps: + - uses: actions/checkout@v4 + - uses: pnpm/action-setup@v6 + - uses: actions/setup-node@v4 + with: + node-version: '22' + cache: pnpm + + - run: pnpm install --frozen-lockfile + - run: pnpm build + + - name: Run it with k6 in Docker + working-directory: packages/loadtest + run: node scripts/e2e.mjs docker + + # The release K6_IMAGE in packages/loadtest/src/engine.ts names, checked against + # the checksums published with it. + - name: Install k6 + run: | + curl -fsSLO https://github.com/grafana/k6/releases/download/v2.3.0/k6-v2.3.0-linux-amd64.tar.gz + curl -fsSLO https://github.com/grafana/k6/releases/download/v2.3.0/k6-v2.3.0-checksums.txt + grep 'k6-v2.3.0-linux-amd64.tar.gz$' k6-v2.3.0-checksums.txt | sha256sum -c - + tar -xzf k6-v2.3.0-linux-amd64.tar.gz + echo "$PWD/k6-v2.3.0-linux-amd64" >> "$GITHUB_PATH" + + - name: Run it with k6 on PATH + working-directory: packages/loadtest + run: node scripts/e2e.mjs local + site: name: Site runs-on: ubuntu-latest diff --git a/CHANGELOG.md b/CHANGELOG.md index b5be91d..ac2ad07 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,20 @@ All notable changes to this repository. Attribution rules: `AGENTS.md` §1.1. ## [Unreleased] +### Added + +- **`nextship-loadtest`, a separate command that reports how many users at once a site + holds.** Give it an address and it runs k6 for you, the one on PATH or the official + Docker image, in up to five rising steps, then says which step the site stopped coping + at: a step is held when at most 1% of its requests fail and 95% finish within a second. + It works on any site, needs no nextship project, prints its plan and sends nothing + until `--yes`, and reports a target that never answers as an error rather than a result. + Asked for by a reader who wanted to know what an app can take before deploying it. In + `packages/loadtest`, with 39 unit tests and an end-to-end script that CI runs with Docker + and with k6 on PATH, and that passed on Windows with k6 on PATH and with Docker Desktop. + Not published to npm yet, and not run with Docker Desktop on macOS; both are in + `docs/roadmap.md`. (Gowtham) + ### Docs - **The evidence page compares nextship with Coolify, measured.** The same app deployed diff --git a/README.md b/README.md index f724f95..a7d5723 100644 --- a/README.md +++ b/README.md @@ -964,6 +964,17 @@ egress is $0.02 per GiB against Vercel's $0.15 and up, and App Platform needs no balancer, which was the line item that made the AWS path cost more than Vercel at small scale. +## Load testing + +`packages/loadtest` is a separate command, `nextship-loadtest`, that reports how many users +at once a site holds: give it an address and it runs [k6](https://k6.io) for you, in rising +steps, and says which step the site stopped coping at. It works on any site and does not +need nextship. It prints its plan and sends nothing until `--yes`, because a load test is +real traffic. + +It is in this repository and **not published to npm yet**. How to run it, what the numbers +mean and what has been run are in [`packages/loadtest/README.md`](./packages/loadtest/README.md). + ## Known limitations The full table with consequences and status is @@ -1084,6 +1095,8 @@ packages/ adapter/ Next.js Adapter API implementation, injected via NEXT_ADAPTER_PATH cli/ every command; runtime/ holds the files copied into a build, and runtime/vm/ the setup and watchdog scripts sent to a server + loadtest/ nextship-loadtest, a separate command that load tests any site + with k6; not published yet site/ the marketing site and documentation, exported as static files ``` diff --git a/docs/design.md b/docs/design.md index 6b848b3..a044399 100644 --- a/docs/design.md +++ b/docs/design.md @@ -1180,3 +1180,55 @@ here rather than lost: That claim depends entirely on the v2 correctness work (shared cache, distributed tag coordination, atomic HTML and RSC invalidation), because that is the one part of this problem no existing tool solves. Convenience is not a wedge. Correctness is. + +## 15. Load testing (Implemented, not published) + +`packages/loadtest` is a second command, `nextship-loadtest`, in its own package with no +dependency on the CLI in either direction. It answers one question, how many users at once +a site holds, for any site. It was asked for by a reader who wanted to know what an app can +take before deploying it. + +**k6 does the work.** Load generation and timing are k6's, the tool people already trust +for them; the package writes the script, runs k6 and reads the result. k6 is found on PATH, +and failing that run from the official image, pinned as `grafana/k6:2.3.0`, the release the +script and the report were written against. k6 is AGPL-3.0 and is only ever run as a +separate process, never bundled. + +**A staircase, not a flat load.** The users asked for are reached in up to five equal +steps, because the question is where a site stops coping and only a rising load shows the +step where it did. Each step is its own k6 scenario with thresholds that cannot fail: k6 +reports figures per scenario only for a scenario a threshold names. + +**The rule is stated with the result.** A step is held when at most 1% of its requests +fail and 95% finish within 1000 ms. The headline is the last step held before the first +that was not. Requests per second are counted from the length of each step, since k6's own +per-scenario rate divides by the whole run. + +**Plan first.** Like every nextship command it prints the plan and sends nothing until +`--yes`. Users are capped at 1000 and the run at 600 s. An address carrying credentials is +refused, because the plan would print them. The address reaches k6 through its +environment and the numbers come from a validated plan, so nothing typed is interpolated +into the script. + +**A target on the same machine.** A container has its own loopback address. On Linux the +container is given the host's network; Docker Desktop has none to give, so there the +address is requested as `host.docker.internal`, and the plan says so. + +**A target that never answers is an error.** Every request failing from the first step +says the address is wrong or the site is down, not how much load it takes, so no result is +printed. + +Verified by `packages/loadtest/scripts/e2e.mjs`, which runs the built command against a +server that answers one request at a time, and fails unless the run holds an early step, +fails a later one on response time and refuses a closed port. On Windows it passes with +k6 2.3.0 on PATH, and with Docker Desktop (Docker 29.7.2), where the server on the same +machine was reached as `host.docker.internal`. CI runs it on Linux with Docker and with k6 +on PATH. + +Known limitations: + +- **Not run with Docker Desktop on macOS.** It takes the path that passed on Windows, and + nobody has run it there. +- **One address, GET only.** No logins or journeys; those need a hand-written k6 script. +- **The test runs from the user's machine**, so the times include their network. +- **Not published.** The npm name and the release path are in `docs/roadmap.md`. diff --git a/docs/roadmap.md b/docs/roadmap.md index bee1592..b9f232c 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -267,6 +267,20 @@ Known from the Droplet runs, to fix or document here: `conformance/vm/e2e.sh` ne with no other app on it, and after adding swap `server add` prints `done swap already on`, the post-apply check's wording rather than what it did. +## Load testing (Built, not published) + +`nextship-loadtest` is in `packages/loadtest` and described in `design.md` §15. What is +left before anyone can install it: + +| Item | State | +|---|---| +| Run it with Docker Desktop on macOS, against a target on the same machine | Not run. Windows passed on 2026-10-09; macOS needs a Mac with Docker Desktop running | +| Choose the npm name and publish it | `nextship-loadtest` is free on npm and is the working name. Publishing needs its own trusted publisher on npm and a job in `release.yml`, which today publishes `packages/cli` only | +| A page for it on the site | After it is published, so the site never documents a command nobody can install | + +Not planned until someone asks: a step in `nextship deploy` that load tests the new +deployment before it takes traffic. + ## AWS: on demand, after the Lightsail streaming experiment Previously v1.1. Moved behind the VM target: an EC2 instance is a Linux server over SSH, which diff --git a/packages/loadtest/README.md b/packages/loadtest/README.md new file mode 100644 index 0000000..8bbcf50 --- /dev/null +++ b/packages/loadtest/README.md @@ -0,0 +1,109 @@ +# nextship-loadtest + +Find how many users at once a site holds, with one command. You give it an address; it +runs [k6](https://k6.io) for you and reports the result in one line. It works on any site, +whatever it is built with and wherever it is hosted. You do not need nextship to use it. + +**Not published to npm yet.** Until it is, run it from this repository (see +[Running it from the repository](#running-it-from-the-repository)). + +``` +nextship-loadtest https://example.com/ # shows the plan, sends nothing +nextship-loadtest https://example.com/ --yes # runs it +``` + +Only test a site you own or have permission to test. A load test is real traffic, and +against someone else's site it is an attack. That is why the command prints its plan and +sends nothing until you add `--yes`. + +## Requirements + +| Requirement | Why | +|---|---| +| Node.js 22 or newer | Runs the command | +| k6 on your PATH, or Docker running | k6 sends the requests and measures them. With no k6 installed, the official `grafana/k6:2.3.0` image is used | + +## What it does + +The test climbs in steps. With the defaults it holds 10 users at once, then 20, 30, 40 and +50, for 12 seconds each. Every user requests the address again as soon as it is answered, +so 50 users means 50 requests in flight at all times, which is far more than 50 people +browsing. + +``` +> Plan + target http://127.0.0.1:4599/ + load 5 step(s) of 4 s: 12, 24, 36, 48, 60 users at once + each user requests the address again as soon as it is answered + engine k6 2.3.0, installed on this machine + traffic real requests to the target for 20 s +> Running, for 20 s +> Result + users requests/s median 95% within failed + 12 30.5 390 ms 418 ms 0.0% + 24 28.0 775 ms 965 ms 0.0% + 36 25.0 1155 ms 1742 ms 0.0% + 48 22.3 1533 ms 2542 ms 0.0% + 60 19.0 2284 ms 3356 ms 0.0% +v Holds 24 users at once + With 36 users at once, 95% of requests took up to 1742 ms. + A step is held when at most 1.0% of its requests fail and 95% finish within 1000 ms. +``` + +That run is real: k6 2.3.0 against a test server built to answer one request at a time. + +A step is **held** when at most 1% of its requests fail and 95% of them finish within one +second. The headline is the last step held before the first one that was not. If every +step is held, it says so, and that the limit was not found: raise `--users`. + +If no request succeeds at all, you get an error instead of a result, because that says +the address is wrong or the site is down, not how much load it takes. + +## Options + +| Option | Meaning | +|---|---| +| `--users ` | The most users at once, reached in up to five steps. Default 50, at most 1000 | +| `--duration ` | The length of the whole test in seconds. Default 60, from 10 to 600 | +| `--yes` | Send the requests. Without it, only the plan is printed | +| `-h`, `--help` | Show the usage | +| `-v`, `--version` | Show the version | + +## Testing a site on your own machine + +`http://localhost:3000/` works. With k6 installed, nothing special happens. With Docker, +a container cannot reach your machine's `localhost` by that name, so on Linux the +container shares the host's network, and on macOS and Windows the address is requested as +`host.docker.internal`, which the plan shows before anything runs. + +## What the numbers do and do not mean + +- **The test runs from your machine.** The times include your network's path to the site. + Run it from a server near the site for figures about the site alone. +- **One address, requested with GET.** No logins, forms or user journeys. For those, write + a k6 script; this command is for the first question, not every question. +- **One run is one sample.** Run it more than once before you rely on a number. +- **It is not a stress tool.** 1000 users at once is the ceiling, since one machine running + k6 beside everything else cannot reliably generate more. + +## What has been run + +- With k6 2.3.0 on PATH, on Windows: `scripts/e2e.mjs local` passes. +- With Docker on Linux: `scripts/e2e.mjs docker` runs in this repository's CI. +- With Docker Desktop on Windows (Docker 29.7.2): `scripts/e2e.mjs docker` passes, with + the test server on the same machine requested as `host.docker.internal`. +- With Docker Desktop on macOS: **not run yet.** It takes the same path as Windows, which + unit tests cover, but nobody has run it there. + +## Running it from the repository + +``` +pnpm install +pnpm build +node packages/loadtest/dist/index.js https://example.com/ +``` + +## License + +Apache-2.0. k6 is a separate program under its own license (AGPL-3.0), run as a separate +process and never bundled or redistributed by this package. diff --git a/packages/loadtest/package.json b/packages/loadtest/package.json new file mode 100644 index 0000000..0942f82 --- /dev/null +++ b/packages/loadtest/package.json @@ -0,0 +1,41 @@ +{ + "name": "nextship-loadtest", + "private": true, + "version": "0.1.0", + "license": "Apache-2.0", + "author": "Gowtham ", + "contributors": [ + "Ragul D " + ], + "description": "Find how many users at once a site holds, with one command. Runs k6 for you. Not published yet.", + "keywords": [ + "load-testing", + "k6", + "performance", + "cli" + ], + "engines": { + "node": ">=22" + }, + "type": "module", + "bin": { + "nextship-loadtest": "dist/index.js" + }, + "files": [ + "dist", + "!dist/**/*.test.*" + ], + "scripts": { + "build": "tsc -p tsconfig.json", + "typecheck": "tsc -p tsconfig.json --noEmit", + "test": "node --test \"dist/**/*.test.js\"" + }, + "repository": { + "type": "git", + "url": "git+https://github.com/gowtham472/nextship.git" + }, + "homepage": "https://nextship.doodlebytestudio.in", + "bugs": { + "url": "https://github.com/gowtham472/nextship/issues" + } +} diff --git a/packages/loadtest/scripts/e2e.mjs b/packages/loadtest/scripts/e2e.mjs new file mode 100644 index 0000000..fbe2813 --- /dev/null +++ b/packages/loadtest/scripts/e2e.mjs @@ -0,0 +1,82 @@ +#!/usr/bin/env node +/** + * Runs the built command for real, against a server on this machine, and fails + * unless it finds the limit that server was built to have. + * + * The server answers one request at a time and takes 25 ms over each, so the time + * a request waits grows with the users waiting ahead of it: a few users are + * answered well inside a second, and sixty are not. A correct run therefore holds + * its first step, fails a later one on response time, and says which. The unit + * tests cover the arithmetic; this covers what they cannot, that the k6 found on + * this machine accepts the script and writes a summary the report can read. + * + * Usage: node scripts/e2e.mjs + * The argument is the engine the run must use, so a machine that silently fell + * back to the other one fails instead of passing for the wrong reason. + * + * Author: Gowtham + */ + +import { spawn } from 'node:child_process' +import { createServer } from 'node:http' +import path from 'node:path' +import { fileURLToPath } from 'node:url' + +const expected = process.argv[2] +if (expected !== 'local' && expected !== 'docker') { + console.error('usage: node scripts/e2e.mjs ') + process.exit(2) +} + +const cli = path.join(path.dirname(fileURLToPath(import.meta.url)), '..', 'dist', 'index.js') + +let queue = Promise.resolve() +const server = createServer((_request, response) => { + queue = queue.then(() => new Promise((done) => setTimeout(() => done(response.end('ok')), 25))) +}) +// Every interface, not loopback alone: a container on Docker Desktop reaches this +// machine through an address that is not 127.0.0.1. +await new Promise((resolve) => server.listen(0, '0.0.0.0', resolve)) +const url = `http://127.0.0.1:${server.address().port}/` + +function run(args) { + return new Promise((resolve) => { + const child = spawn(process.execPath, [cli, ...args], { stdio: ['ignore', 'pipe', 'pipe'] }) + let output = '' + child.stdout.on('data', (chunk) => (output += chunk)) + child.stderr.on('data', (chunk) => (output += chunk)) + child.on('close', (code) => resolve({ code, output })) + }) +} + +const failures = [] +const check = (what, passed, output) => { + console.log(`${passed ? 'ok ' : 'FAIL'} ${what}`) + if (!passed) failures.push(`${what}\n${output}`) +} + +const engine = expected === 'local' ? /engine k6 \d+\.\d+\.\d+, installed on this machine/ : /engine k6 in Docker/ + +const plan = await run([url, '--users', '60', '--duration', '20']) +check('the plan exits 0 and sends nothing', plan.code === 0 && /No request was sent/.test(plan.output), plan.output) +check(`the plan names the ${expected} engine`, engine.test(plan.output), plan.output) + +const test = await run([url, '--users', '60', '--duration', '20', '--yes']) +const held = /v Holds (\d+) users at once\n/.exec(test.output) +check('the run exits 0', test.code === 0, test.output) +check('the result has a row for each of the five steps', (test.output.match(/^\s+\d+\s+\d+\.\d\s+\d+ ms\s+\d+ ms\s+\d+\.\d%$/gm) ?? []).length === 5, test.output) +check('it holds a step below 60 users and says which', held !== null && Number(held[1]) < 60, test.output) +check('it says the step after failed on response time', /With \d+ users at once, 95% of requests took up to \d+ ms\./.test(test.output), test.output) + +server.close() +// Requests k6 abandoned at the end of a step are still queued in the server. +server.closeAllConnections() + +const closed = await run([url, '--users', '2', '--duration', '10', '--yes']) +check('a target that answers nothing is an error with no result', closed.code === 1 && /No request to .* succeeded/.test(closed.output) && !/Holds/.test(closed.output), closed.output) + +if (failures.length > 0) { + console.error(`\n${failures.length} check(s) failed:\n\n${failures.join('\n\n')}`) + process.exit(1) +} +console.log(`\nnextship-loadtest e2e: passed with the ${expected} engine`) diff --git a/packages/loadtest/src/engine.test.ts b/packages/loadtest/src/engine.test.ts new file mode 100644 index 0000000..1cefa7e --- /dev/null +++ b/packages/loadtest/src/engine.test.ts @@ -0,0 +1,70 @@ +/** + * nextship-loadtest: the command a test runs + * + * Which k6 runs, and how a container is pointed at this machine, are decided by a + * pure function, so each case is checked here without k6 or Docker installed. + * Running the command for real is `scripts/e2e.mjs`. + * + * Author: Gowtham + */ + +import test from 'node:test' +import assert from 'node:assert/strict' + +import { describeEngine, invocation, isLoopback, K6_IMAGE } from './engine.js' + +const local = { kind: 'local', version: '2.3.0' } as const +const docker = { kind: 'docker' } as const + +test('a local k6 reads the script from stdin and the target from its environment', () => { + assert.deepEqual(invocation(local, 'http://localhost:3000/', 'linux'), { + command: 'k6', + args: ['run', '--quiet', '-e', 'TARGET=http://localhost:3000/', '-'], + target: 'http://localhost:3000/', + }) +}) + +test('Docker runs the pinned image, removed afterwards, with stdin open', () => { + assert.deepEqual(invocation(docker, 'https://example.com/', 'linux'), { + command: 'docker', + args: ['run', '--rm', '-i', K6_IMAGE, 'run', '--quiet', '-e', 'TARGET=https://example.com/', '-'], + target: 'https://example.com/', + }) +}) + +test('on Linux a container reaches a target on this machine through the host network', () => { + const run = invocation(docker, 'http://127.0.0.1:3000/', 'linux') + assert.deepEqual(run.args.slice(0, 5), ['run', '--rm', '-i', '--network', 'host']) + assert.equal(run.target, 'http://127.0.0.1:3000/') +}) + +for (const platform of ['darwin', 'win32'] as const) { + test(`on ${platform} a target on this machine is requested as host.docker.internal`, () => { + const run = invocation(docker, 'http://localhost:3000/shop?page=2', platform) + assert.equal(run.target, 'http://host.docker.internal:3000/shop?page=2') + assert.ok(run.args.includes('TARGET=http://host.docker.internal:3000/shop?page=2')) + assert.ok(!run.args.includes('--network')) + }) +} + +test('a target elsewhere is never rewritten or given the host network', () => { + for (const platform of ['linux', 'darwin', 'win32'] as const) { + const run = invocation(docker, 'https://example.com/', platform) + assert.equal(run.target, 'https://example.com/') + assert.ok(!run.args.includes('--network')) + } +}) + +test('loopback is localhost, its subdomains, 127.0.0.0/8 and ::1, and nothing else', () => { + for (const hostname of ['localhost', 'app.localhost', '127.0.0.1', '127.8.9.10', '[::1]']) { + assert.equal(isLoopback(hostname), true, hostname) + } + for (const hostname of ['example.com', 'localhost.example.com', '128.0.0.1', '10.0.0.1', 'notlocalhost']) { + assert.equal(isLoopback(hostname), false, hostname) + } +}) + +test('the plan names the k6 that will run', () => { + assert.equal(describeEngine(local), 'k6 2.3.0, installed on this machine') + assert.equal(describeEngine(docker), `k6 in Docker (${K6_IMAGE}), pulled on first use`) +}) diff --git a/packages/loadtest/src/engine.ts b/packages/loadtest/src/engine.ts new file mode 100644 index 0000000..b5c7b66 --- /dev/null +++ b/packages/loadtest/src/engine.ts @@ -0,0 +1,113 @@ +/** + * nextship-loadtest: finding and running k6 + * + * k6 does the load generation and the measuring. This finds a k6 to run, the one + * on PATH if there is one and the official Docker image otherwise, so the command + * works on a machine that has only Docker, which every nextship user already has. + * The script goes in on stdin and the summary comes back on stdout, so nothing is + * written to disk on either side. + * + * A container has its own loopback address, so a target on this machine needs + * help to be reached from one. On Linux the container shares the host's network. + * Docker Desktop has no host network to share, and gives the host the name + * `host.docker.internal` instead, so the target is rewritten to that name there. + * + * Author: Gowtham + */ + +import { spawn } from 'node:child_process' +import { LoadtestError } from './errors.js' + +/** The k6 release the generated script and the report were written against. */ +export const K6_IMAGE = 'grafana/k6:2.3.0' + +export type Engine = { kind: 'local'; version: string } | { kind: 'docker' } + +export interface Invocation { + command: string + args: string[] + /** The target as k6 requests it, which differs from the one typed when it was rewritten. */ + target: string +} + +/** The k6 on PATH, else Docker with a running daemon, else a failure that says how to get either. */ +export async function findEngine(): Promise { + const local = await probe('k6', ['version']) + const version = local === null ? null : /v(\d+\.\d+\.\d+)/.exec(local) + if (version) return { kind: 'local', version: version[1] } + + // The server version, not the client's: a Docker CLI with no daemon cannot run anything. + if ((await probe('docker', ['version', '--format', '{{.Server.Version}}'])) !== null) return { kind: 'docker' } + + throw new LoadtestError( + 'Neither k6 nor a running Docker was found, and the test needs one of them.', + 'Start Docker, or install k6 from https://grafana.com/docs/k6/latest/set-up/install-k6/, then run the same command again.' + ) +} + +export function describeEngine(engine: Engine): string { + return engine.kind === 'local' + ? `k6 ${engine.version}, installed on this machine` + : `k6 in Docker (${K6_IMAGE}), pulled on first use` +} + +export function isLoopback(hostname: string): boolean { + return hostname === 'localhost' || hostname.endsWith('.localhost') || /^127\./.test(hostname) || hostname === '[::1]' +} + +/** The exact command for a test, kept apart from running it so the choices above can be tested. */ +export function invocation(engine: Engine, url: string, platform: NodeJS.Platform): Invocation { + const k6 = (target: string): string[] => ['run', '--quiet', '-e', `TARGET=${target}`, '-'] + if (engine.kind === 'local') return { command: 'k6', args: k6(url), target: url } + + const parsed = new URL(url) + if (!isLoopback(parsed.hostname)) return { command: 'docker', args: ['run', '--rm', '-i', K6_IMAGE, ...k6(url)], target: url } + + if (platform === 'linux') { + return { command: 'docker', args: ['run', '--rm', '-i', '--network', 'host', K6_IMAGE, ...k6(url)], target: url } + } + parsed.hostname = 'host.docker.internal' + return { command: 'docker', args: ['run', '--rm', '-i', K6_IMAGE, ...k6(parsed.href)], target: parsed.href } +} + +/** How much of k6's stderr is kept: a failing target makes k6 warn once per request. */ +const STDERR_KEPT = 2000 + +/** Runs the script and returns what k6 wrote to stdout, which is the JSON summary. */ +export async function runK6(run: Invocation, script: string): Promise { + const { code, stdout, stderr } = await new Promise<{ code: number | null; stdout: string; stderr: string }>( + (resolve, reject) => { + const child = spawn(run.command, run.args, { stdio: ['pipe', 'pipe', 'pipe'] }) + let out = '' + let err = '' + child.stdout.on('data', (chunk: Buffer) => (out += chunk.toString())) + child.stderr.on('data', (chunk: Buffer) => (err = (err + chunk.toString()).slice(-STDERR_KEPT))) + child.on('error', (error) => + reject(new LoadtestError(`Could not start \`${run.command}\`: ${error.message}`, 'Check the error above.')) + ) + child.on('close', (exitCode) => resolve({ code: exitCode, stdout: out, stderr: err })) + // k6 that exits before reading the script closes the pipe; its exit code says why. + child.stdin.on('error', () => {}) + child.stdin.end(script) + } + ) + + if (code !== 0) { + throw new LoadtestError( + `\`${run.command}\` ${code === null ? 'was stopped before the test finished' : `exited with code ${code}`}: ${stderr.trim() || 'no output'}`, + 'Check the error above. No result was produced, so nothing is reported.' + ) + } + return stdout +} + +/** Trimmed stdout of a command that answers a question, or null when it cannot run or fails. */ +async function probe(command: string, args: string[]): Promise { + return new Promise((resolve) => { + const child = spawn(command, args, { stdio: ['ignore', 'pipe', 'ignore'] }) + let stdout = '' + child.stdout.on('data', (chunk: Buffer) => (stdout += chunk.toString())) + child.on('error', () => resolve(null)) + child.on('close', (code) => resolve(code === 0 ? stdout.trim() : null)) + }) +} diff --git a/packages/loadtest/src/errors.ts b/packages/loadtest/src/errors.ts new file mode 100644 index 0000000..a7bacfb --- /dev/null +++ b/packages/loadtest/src/errors.ts @@ -0,0 +1,21 @@ +/** + * nextship-loadtest: typed error + * + * Every failure the user can cause carries a cause and a next action, so the CLI + * never asks the user to interpret a stack trace. Its own class rather than the + * CLI's, because this package is installed and run without nextship. + * + * Author: Gowtham + * Rules: ../../../AGENTS.md §4.2 + */ + +export class LoadtestError extends Error { + /** What the user should do next. Printed under the message by the entry point. */ + readonly action: string + + constructor(message: string, action: string) { + super(message) + this.name = 'LoadtestError' + this.action = action + } +} diff --git a/packages/loadtest/src/index.ts b/packages/loadtest/src/index.ts new file mode 100644 index 0000000..d25fe22 --- /dev/null +++ b/packages/loadtest/src/index.ts @@ -0,0 +1,128 @@ +#!/usr/bin/env node +/** + * nextship-loadtest: entry point + * + * One command: give it an address, and it reports how many users at once the + * site held. It prints the plan and sends nothing until `--yes`, because a load + * test is real traffic, and against a site that is not yours it is an attack. + * + * Author: Gowtham + */ + +import { describeEngine, findEngine, invocation, runK6 } from './engine.js' +import { LoadtestError } from './errors.js' +import { detail, fail, ok, step } from './log.js' +import { DEFAULT_SECONDS, DEFAULT_USERS, MAX_SECONDS, MAX_USERS, MIN_SECONDS, planFor, totalSeconds } from './plan.js' +import { assertAnswered, readSummary, report } from './report.js' +import { renderScript } from './script.js' +import { VERSION } from './version.js' + +const USAGE = `nextship-loadtest + +Usage + nextship-loadtest Show the plan for a load test of + nextship-loadtest --yes Run it, and report how many users at once it held + +Options + --users The most users at once, reached in steps (default ${DEFAULT_USERS}, at most ${MAX_USERS}) + --duration The length of the whole test in seconds (default ${DEFAULT_SECONDS}, ${MIN_SECONDS} to ${MAX_SECONDS}) + --yes Send the requests. Without it, only the plan is printed + -h, --help Show this message + -v, --version Show the version + +Runs k6, the one on PATH or the official Docker image. Only test a site you own +or have permission to test. +` + +interface Arguments { + url: string + users: string | null + duration: string | null + confirmed: boolean +} + +function parse(argv: string[]): Arguments { + let url: string | null = null + let users: string | null = null + let duration: string | null = null + let confirmed = false + + for (let index = 0; index < argv.length; index += 1) { + const token = argv[index] + if (token === '--yes') { + confirmed = true + } else if (token === '--users' || token === '--duration') { + const value = argv[index + 1] + if (value === undefined || value.startsWith('--')) { + throw new LoadtestError(`${token} needs a value.`, 'Run `nextship-loadtest --help`.') + } + if (token === '--users') users = value + else duration = value + index += 1 + } else if (token.startsWith('-')) { + throw new LoadtestError(`Unknown option \`${token}\`.`, 'Run `nextship-loadtest --help`.') + } else if (url === null) { + url = token + } else { + throw new LoadtestError(`Unexpected argument \`${token}\`.`, 'Give one address. Run `nextship-loadtest --help`.') + } + } + + if (url === null) throw new LoadtestError('No address was given.', 'Run `nextship-loadtest `, for example https://example.com/.') + return { url, users, duration, confirmed } +} + +async function main(argv: string[]): Promise { + if (argv.length === 0 || argv.includes('-h') || argv.includes('--help')) { + process.stdout.write(USAGE) + return + } + if (argv.includes('-v') || argv.includes('--version')) { + process.stdout.write(`${VERSION}\n`) + return + } + + const args = parse(argv) + const plan = planFor(args.url, { users: args.users, duration: args.duration }) + const engine = await findEngine() + const run = invocation(engine, plan.url, process.platform) + const seconds = totalSeconds(plan) + + step('Plan') + detail(`target ${plan.url}`) + if (run.target !== plan.url) detail(` requested as ${run.target}, the name a container reaches this machine by`) + detail( + `load ${plan.steps.length} step(s) of ${plan.steps[0].seconds} s: ${plan.steps.map((entry) => entry.users).join(', ')} users at once` + ) + detail(' each user requests the address again as soon as it is answered') + detail(`engine ${describeEngine(engine)}`) + detail(`traffic real requests to the target for ${seconds} s`) + + if (!args.confirmed) { + ok('This was a plan only. No request was sent.') + detail('Only test a site you own or have permission to test. Add --yes to run it.') + return + } + + step(`Running, for ${seconds} s`) + const results = readSummary(await runK6(run, renderScript(plan)), plan) + assertAnswered(results, run.target) + + step('Result') + const outcome = report(results) + for (const line of outcome.table) detail(line) + ok(outcome.headline) + for (const note of outcome.notes) detail(note) +} + +main(process.argv.slice(2)).catch((error: unknown) => { + if (error instanceof LoadtestError) { + fail(error.message, error.action) + } else { + fail( + error instanceof Error ? error.message : String(error), + 'This is unexpected. Please report it with the output above.' + ) + } + process.exitCode = 1 +}) diff --git a/packages/loadtest/src/log.ts b/packages/loadtest/src/log.ts new file mode 100644 index 0000000..449e5e3 --- /dev/null +++ b/packages/loadtest/src/log.ts @@ -0,0 +1,35 @@ +/** + * nextship-loadtest: terminal output + * + * The same visual format as every nextship command, so a load test reads like the + * deploy before it. Repeated here rather than imported, because this package is + * installed and run without nextship. + * + * Author: Gowtham + * Rules: ../../../AGENTS.md §4.3 + */ + +const color = process.stdout.isTTY && !process.env.NO_COLOR + +const paint = (code: string, text: string) => (color ? `\u001b[${code}m${text}\u001b[0m` : text) + +/** Announces a stage that is starting. */ +export function step(message: string): void { + process.stdout.write(`${paint('34', '>')} ${message}\n`) +} + +/** Reports a completed stage by its result, never by the effort it took. */ +export function ok(message: string): void { + process.stdout.write(`${paint('32', 'v')} ${message}\n`) +} + +/** Supporting detail under a step or result. */ +export function detail(message: string): void { + process.stdout.write(` ${paint('90', message)}\n`) +} + +/** A cause and the action that resolves it. */ +export function fail(message: string, action: string): void { + process.stderr.write(`${paint('31', 'x')} ${message}\n`) + process.stderr.write(` ${action}\n`) +} diff --git a/packages/loadtest/src/plan.test.ts b/packages/loadtest/src/plan.test.ts new file mode 100644 index 0000000..16a4ad0 --- /dev/null +++ b/packages/loadtest/src/plan.test.ts @@ -0,0 +1,91 @@ +/** + * nextship-loadtest: what a plan is built from + * + * The plan is the only place user input becomes numbers, so each test pins either + * the staircase a valid input produces or the refusal an invalid one gets. + * + * Author: Gowtham + */ + +import test from 'node:test' +import assert from 'node:assert/strict' + +import { LoadtestError } from './errors.js' +import { planFor, totalSeconds } from './plan.js' + +const defaults = { users: null, duration: null } + +test('the default plan climbs to 50 users in five steps over a minute', () => { + const plan = planFor('https://example.com', defaults) + assert.equal(plan.url, 'https://example.com/') + assert.deepEqual(plan.steps.map((entry) => entry.users), [10, 20, 30, 40, 50]) + assert.deepEqual(plan.steps.map((entry) => entry.seconds), [12, 12, 12, 12, 12]) + assert.equal(totalSeconds(plan), 60) +}) + +test('fewer users than steps gives one step per user', () => { + const plan = planFor('http://localhost:3000/', { users: '3', duration: '30' }) + assert.deepEqual(plan.steps, [ + { users: 1, seconds: 10 }, + { users: 2, seconds: 10 }, + { users: 3, seconds: 10 }, + ]) +}) + +test('the last step always reaches the users asked for, whatever the rounding', () => { + const plan = planFor('http://localhost:3000/', { users: '7', duration: '23' }) + assert.deepEqual(plan.steps.map((entry) => entry.users), [1, 3, 4, 6, 7]) + // 23 s does not divide by five, so the run is the 20 s the steps add up to. + assert.equal(totalSeconds(plan), 20) +}) + +test('the path and query of the address are kept', () => { + assert.equal(planFor('https://example.com/shop?page=2', defaults).url, 'https://example.com/shop?page=2') +}) + +for (const [users, why] of [ + ['0', 'zero'], + ['1001', 'over the limit'], + ['12.5', 'a fraction'], + ['-4', 'negative'], + ['many', 'not a number'], + ['1e2', 'an exponent'], +] as const) { + test(`--users ${users} is refused: ${why}`, () => { + assert.throws(() => planFor('https://example.com', { users, duration: null }), (error: unknown) => { + assert.ok(error instanceof LoadtestError) + assert.match(error.message, /--users must be a whole number from 1 to 1000/) + return true + }) + }) +} + +for (const duration of ['9', '601', '30s']) { + test(`--duration ${duration} is refused`, () => { + assert.throws(() => planFor('https://example.com', { users: null, duration }), (error: unknown) => { + assert.ok(error instanceof LoadtestError) + assert.match(error.message, /--duration must be a whole number from 10 to 600/) + return true + }) + }) +} + +test('something that is not a URL is refused with an example of one', () => { + assert.throws(() => planFor('example', defaults), (error: unknown) => { + assert.ok(error instanceof LoadtestError) + assert.match(error.action, /https:\/\/example\.com\//) + return true + }) +}) + +test('an address that is not http or https is refused', () => { + assert.throws(() => planFor('ftp://example.com', defaults), /not an http or https address/) +}) + +test('an address carrying credentials is refused, since the plan would print them', () => { + assert.throws(() => planFor('https://user:secret@example.com', defaults), (error: unknown) => { + assert.ok(error instanceof LoadtestError) + assert.doesNotMatch(error.message, /secret/) + return true + }) +}) diff --git a/packages/loadtest/src/plan.ts b/packages/loadtest/src/plan.ts new file mode 100644 index 0000000..9f9995e --- /dev/null +++ b/packages/loadtest/src/plan.ts @@ -0,0 +1,97 @@ +/** + * nextship-loadtest: the plan for one test + * + * Turns what the user typed into the load that will be sent: a target, and a + * staircase of steps, each holding more users at once than the last. A staircase + * rather than one flat load, because the question is where a site stops coping, + * and only a rising load shows the step where it did. + * + * Every number is checked here, so nothing unvalidated reaches the k6 script. + * + * Author: Gowtham + */ + +import { LoadtestError } from './errors.js' + +export const DEFAULT_USERS = 50 +export const DEFAULT_SECONDS = 60 + +/** + * One machine running k6 beside everything else cannot reliably hold more, and a + * number typed by mistake should not become a flood. + */ +export const MAX_USERS = 1000 +export const MIN_SECONDS = 10 +export const MAX_SECONDS = 600 + +/** Enough steps to see a trend, few enough that each lasts long enough to measure. */ +const STEP_COUNT = 5 + +export interface Step { + /** Users requesting at once during this step. */ + users: number + seconds: number +} + +export interface Plan { + /** The target as k6 will request it. */ + url: string + steps: Step[] +} + +/** The length of the whole run, which is what the user is told to expect. */ +export function totalSeconds(plan: Plan): number { + return plan.steps.reduce((sum, entry) => sum + entry.seconds, 0) +} + +export function planFor(target: string, options: { users: string | null; duration: string | null }): Plan { + const users = options.users === null ? DEFAULT_USERS : wholeNumber('--users', options.users, 1, MAX_USERS) + const seconds = + options.duration === null ? DEFAULT_SECONDS : wholeNumber('--duration', options.duration, MIN_SECONDS, MAX_SECONDS) + + const count = Math.min(STEP_COUNT, users) + const each = Math.floor(seconds / count) + const steps = Array.from({ length: count }, (_, index) => ({ + users: Math.round((users * (index + 1)) / count), + seconds: each, + })) + return { url: targetUrl(target), steps } +} + +function targetUrl(target: string): string { + let url: URL + try { + url = new URL(target) + } catch { + throw new LoadtestError( + `"${target}" is not a URL.`, + 'Give the full address, such as https://example.com/ or http://localhost:3000/.' + ) + } + if (url.protocol !== 'http:' && url.protocol !== 'https:') { + throw new LoadtestError( + `"${target}" is not an http or https address.`, + 'Give an address that starts with http:// or https://.' + ) + } + if (url.username !== '' || url.password !== '') { + throw new LoadtestError( + 'The address carries a username or password, which this command would print in its plan and pass on a command line.', + 'Test an address that needs no credentials in it.' + ) + } + return url.href +} + +function wholeNumber(flag: string, value: string, minimum: number, maximum: number): number { + const number = Number(value) + if (!/^\d+$/.test(value) || number < minimum || number > maximum) { + throw new LoadtestError( + `${flag} must be a whole number from ${minimum} to ${maximum}, not "${value}".`, + flag === '--users' + ? 'For a heavier test than that, run k6 directly on a machine sized for it.' + : 'Give the length of the whole test in seconds.' + ) + } + return number +} diff --git a/packages/loadtest/src/report.test.ts b/packages/loadtest/src/report.test.ts new file mode 100644 index 0000000..06aa73d --- /dev/null +++ b/packages/loadtest/src/report.test.ts @@ -0,0 +1,113 @@ +/** + * nextship-loadtest: from a k6 summary to a result + * + * The summaries here have the shape k6 2.3.0 wrote for a script this package + * rendered, with only the fields the report reads. The headline is the one line + * most people will act on, so every way it can come out has a test. + * + * Author: Gowtham + */ + +import test from 'node:test' +import assert from 'node:assert/strict' + +import { LoadtestError } from './errors.js' +import { planFor } from './plan.js' +import { assertAnswered, readSummary, report, struggle, type StepResult } from './report.js' + +/** A summary with one entry per step: requests, failed share, median and 95th percentile. */ +function summary(steps: Array<[number, number, number, number]>): string { + const metrics: Record = {} + steps.forEach(([count, failed, med, p95], index) => { + const tag = `{scenario:step${index + 1}}` + // k6 divides by the whole run, so its own rate is deliberately wrong here. + metrics[`http_reqs${tag}`] = { type: 'counter', values: { count, rate: 1 } } + metrics[`http_req_failed${tag}`] = { type: 'rate', values: { rate: failed, passes: 0, fails: count } } + metrics[`http_req_duration${tag}`] = { type: 'trend', values: { med, 'p(95)': p95 } } + }) + return JSON.stringify({ metrics, state: { testRunDurationMs: 1 } }) +} + +const twoSteps = planFor('https://example.com', { users: '2', duration: '20' }) + +const held = (users: number): StepResult => ({ users, requests: 500, perSecond: 50, failed: 0, medianMs: 20, p95Ms: 40 }) + +test('each step is read by its scenario, and requests per second come from its own length', () => { + const results = readSummary(summary([[300, 0, 31.4, 36.6], [450, 0.02, 40, 80]]), twoSteps) + assert.deepEqual(results, [ + { users: 1, requests: 300, perSecond: 30, failed: 0, medianMs: 31.4, p95Ms: 36.6 }, + { users: 2, requests: 450, perSecond: 45, failed: 0.02, medianMs: 40, p95Ms: 80 }, + ]) +}) + +test('a summary missing a step is an error, not a row of zeros', () => { + assert.throws(() => readSummary(summary([[300, 0, 31, 36]]), twoSteps), (error: unknown) => { + assert.ok(error instanceof LoadtestError) + assert.match(error.message, /no http_reqs count for step 2/) + return true + }) +}) + +test('output that is not a k6 summary is an error', () => { + for (const output of ['', 'not json', '{"metrics":null}', '[]']) { + assert.throws(() => readSummary(output, twoSteps), /without a summary this command can read/) + } +}) + +test('a step struggles past 1% failed, past a second at the 95th percentile, or with nothing finished', () => { + assert.equal(struggle(held(10)), null) + assert.equal(struggle({ ...held(10), failed: 0.01 }), null) + assert.equal(struggle({ ...held(10), p95Ms: 1000 }), null) + assert.equal(struggle({ ...held(10), failed: 0.042 }), '4.2% of requests failed') + assert.equal(struggle({ ...held(10), p95Ms: 1755.6 }), '95% of requests took up to 1756 ms') + assert.equal(struggle({ ...held(10), requests: 0 }), 'no request finished') +}) + +test('when every step is held, the headline says so and that the limit was not found', () => { + const outcome = report([held(10), held(20)]) + assert.equal(outcome.headline, 'Holds 20 users at once, the most this test tried') + assert.equal(outcome.notes.at(-1), 'Raise --users to look for the limit.') +}) + +test('the headline is the last step held, and the first note is why the next was not', () => { + const outcome = report([held(10), held(20), { ...held(30), p95Ms: 1756 }, { ...held(40), failed: 0.5 }]) + assert.equal(outcome.headline, 'Holds 20 users at once') + assert.equal(outcome.notes[0], 'With 30 users at once, 95% of requests took up to 1756 ms.') + assert.equal(outcome.notes[1], 'A step is held when at most 1.0% of its requests fail and 95% finish within 1000 ms.') +}) + +test('a first step that struggles means fewer than its users, with what to do next', () => { + const outcome = report([{ ...held(10), failed: 0.2 }, held(20)]) + assert.equal(outcome.headline, 'Holds fewer than 10 users at once') + assert.equal(outcome.notes[0], 'With 10 users at once, 20.0% of requests failed.') + assert.equal(outcome.notes.at(-1), 'Run again with a lower --users to find what it does hold.') +}) + +test('a single user that struggles is said plainly, with no lower number to try', () => { + const outcome = report([{ ...held(1), p95Ms: 4000 }]) + assert.equal(outcome.headline, 'Does not hold one user at a time') + assert.equal(outcome.notes[0], 'With 1 user at a time, 95% of requests took up to 4000 ms.') + assert.equal(outcome.notes.length, 2) +}) + +test('the table has a header and one aligned row per step', () => { + const { table } = report([held(5), { ...held(100), perSecond: 1234.56, medianMs: 250.4, p95Ms: 1999.5, failed: 0.125 }]) + assert.deepEqual(table, [ + 'users requests/s median 95% within failed', + ' 5 50.0 20 ms 40 ms 0.0%', + ' 100 1234.6 250 ms 2000 ms 12.5%', + ]) +}) + +test('a target that never answered is refused rather than reported', () => { + const dead = [{ ...held(1), failed: 1, medianMs: 0, p95Ms: 0 }, { ...held(2), requests: 0 }] + assert.throws(() => assertAnswered(dead, 'http://localhost:9/'), (error: unknown) => { + assert.ok(error instanceof LoadtestError) + assert.match(error.message, /No request to http:\/\/localhost:9\/ succeeded/) + return true + }) +}) + +test('a target that answered at first and failed under load is reported', () => { + assert.doesNotThrow(() => assertAnswered([held(1), { ...held(2), failed: 1 }], 'http://localhost:9/')) +}) diff --git a/packages/loadtest/src/report.ts b/packages/loadtest/src/report.ts new file mode 100644 index 0000000..784373d --- /dev/null +++ b/packages/loadtest/src/report.ts @@ -0,0 +1,136 @@ +/** + * nextship-loadtest: reading the result + * + * Turns the summary k6 wrote into one row per step, and one sentence: how many + * users at once the site held. A step is held when at most 1% of its requests + * failed and 95% of them finished within a second. Both lines are stated in the + * output, because a result means nothing without the rule that produced it. + * + * Requests per second are counted here from the length of each step. k6 gives a + * rate per scenario too, but divides by the length of the whole run, which + * understates every step. + * + * Author: Gowtham + */ + +import { LoadtestError } from './errors.js' +import type { Plan } from './plan.js' +import { scenarioName } from './script.js' + +/** The most requests of a step that may fail for the step to count as held. */ +export const FAILED_LIMIT = 0.01 +/** The time within which 95% of the requests of a step must finish. */ +export const P95_LIMIT_MS = 1000 + +export interface StepResult { + users: number + requests: number + perSecond: number + /** The share of requests that failed: an error status, or no answer at all. */ + failed: number + medianMs: number + p95Ms: number +} + +export interface Report { + /** A header row and one row per step, aligned. */ + table: string[] + /** The result in one sentence. */ + headline: string + notes: string[] +} + +export function readSummary(summary: string, plan: Plan): StepResult[] { + let metrics: Record } | undefined> + try { + metrics = (JSON.parse(summary) as { metrics: typeof metrics }).metrics + if (typeof metrics !== 'object' || metrics === null) throw new Error('no metrics') + } catch { + throw unreadable('k6 finished without a summary this command can read') + } + + return plan.steps.map((entry, index) => { + const name = scenarioName(index) + const value = (metric: string, field: string): number => { + const found = metrics[`${metric}{scenario:${name}}`]?.values?.[field] + if (typeof found !== 'number') throw unreadable(`the summary has no ${metric} ${field} for step ${index + 1}`) + return found + } + const requests = value('http_reqs', 'count') + return { + users: entry.users, + requests, + perSecond: requests / entry.seconds, + failed: value('http_req_failed', 'rate'), + medianMs: value('http_req_duration', 'med'), + p95Ms: value('http_req_duration', 'p(95)'), + } + }) +} + +/** Why a step was not held, or null when it was. */ +export function struggle(result: StepResult): string | null { + if (result.requests === 0) return 'no request finished' + if (result.failed > FAILED_LIMIT) return `${percent(result.failed)} of requests failed` + if (result.p95Ms > P95_LIMIT_MS) return `95% of requests took up to ${Math.round(result.p95Ms)} ms` + return null +} + +export function report(results: StepResult[]): Report { + const rows = [ + ['users', 'requests/s', 'median', '95% within', 'failed'], + ...results.map((result) => [ + String(result.users), + result.perSecond.toFixed(1), + `${Math.round(result.medianMs)} ms`, + `${Math.round(result.p95Ms)} ms`, + percent(result.failed), + ]), + ] + const widths = rows[0].map((_, column) => Math.max(...rows.map((row) => row[column].length))) + const table = rows.map((row) => row.map((cell, column) => cell.padStart(widths[column])).join(' ')) + + const rule = `A step is held when at most ${percent(FAILED_LIMIT)} of its requests fail and 95% finish within ${P95_LIMIT_MS} ms.` + const first = results.findIndex((result) => struggle(result) !== null) + if (first === -1) { + return { + table, + headline: `Holds ${atOnce(results[results.length - 1].users)}, the most this test tried`, + notes: [rule, 'Raise --users to look for the limit.'], + } + } + const failing = results[first] + const reason = `With ${atOnce(failing.users)}, ${struggle(failing)}.` + if (first > 0) return { table, headline: `Holds ${atOnce(results[first - 1].users)}`, notes: [reason, rule] } + if (failing.users === 1) return { table, headline: 'Does not hold one user at a time', notes: [reason, rule] } + return { + table, + headline: `Holds fewer than ${atOnce(failing.users)}`, + notes: [reason, rule, 'Run again with a lower --users to find what it does hold.'], + } +} + +/** + * Refuses to report on a target that never answered. Every request failing from + * the first step on says nothing about load: the address is wrong, the site is + * down, or it cannot be reached from where k6 ran. + */ +export function assertAnswered(results: StepResult[], target: string): void { + if (results.every((result) => result.requests === 0 || result.failed === 1)) { + throw new LoadtestError( + `No request to ${target} succeeded, so there is no result to report.`, + 'Check that the address is right and the site is up, then run the same command again.' + ) + } +} + +const atOnce = (users: number): string => (users === 1 ? '1 user at a time' : `${users} users at once`) + +const percent = (share: number): string => `${(share * 100).toFixed(1)}%` + +function unreadable(what: string): LoadtestError { + return new LoadtestError( + `The test ran, but ${what}.`, + 'This is a defect in nextship-loadtest, or a k6 version it was not written for. Please report it with the k6 version.' + ) +} diff --git a/packages/loadtest/src/script.test.ts b/packages/loadtest/src/script.test.ts new file mode 100644 index 0000000..d7a6110 --- /dev/null +++ b/packages/loadtest/src/script.test.ts @@ -0,0 +1,71 @@ +/** + * nextship-loadtest: the script k6 is given + * + * The report can only tell the steps apart if each is its own scenario with its + * own thresholds, and one step must start when the last ends. These tests read + * the options back out of the rendered script and check exactly that. + * + * Author: Gowtham + */ + +import test from 'node:test' +import assert from 'node:assert/strict' + +import { planFor } from './plan.js' +import { renderScript, scenarioName } from './script.js' + +interface Options { + scenarios: Record + thresholds: Record + summaryTrendStats: string[] +} + +function optionsOf(script: string): Options { + const match = /export const options = (\{[\s\S]*?\n\})\n/.exec(script) + assert.ok(match, 'the script declares its options') + return JSON.parse(match[1]) as Options +} + +test('each step is a scenario that starts when the one before it ends', () => { + const plan = planFor('https://example.com', { users: '30', duration: '30' }) + const { scenarios } = optionsOf(renderScript(plan)) + assert.deepEqual(Object.keys(scenarios), ['step1', 'step2', 'step3', 'step4', 'step5']) + assert.deepEqual( + Object.values(scenarios).map((scenario) => [scenario.vus, scenario.duration, scenario.startTime]), + [ + [6, '6s', '0s'], + [12, '6s', '6s'], + [18, '6s', '12s'], + [24, '6s', '18s'], + [30, '6s', '24s'], + ] + ) + for (const scenario of Object.values(scenarios)) { + assert.equal(scenario.executor, 'constant-vus') + assert.equal(scenario.gracefulStop, '0s') + } +}) + +test('every step has the three thresholds that make k6 report it on its own', () => { + const plan = planFor('https://example.com', { users: '2', duration: '10' }) + const { thresholds, summaryTrendStats } = optionsOf(renderScript(plan)) + for (const index of [0, 1]) { + const name = scenarioName(index) + assert.deepEqual(thresholds[`http_req_duration{scenario:${name}}`], ['max>=0']) + assert.deepEqual(thresholds[`http_req_failed{scenario:${name}}`], ['rate>=0']) + assert.deepEqual(thresholds[`http_reqs{scenario:${name}}`], ['count>=0']) + } + assert.equal(Object.keys(thresholds).length, 6) + assert.deepEqual(summaryTrendStats, ['med', 'p(95)']) +}) + +test('the address is never written into the script', () => { + const script = renderScript(planFor('https://example.com/a"b', { users: null, duration: null })) + assert.doesNotMatch(script, /example\.com/) + assert.match(script, /http\.get\(__ENV\.TARGET\)/) +}) + +test('the summary is handed back as JSON on stdout', () => { + const script = renderScript(planFor('https://example.com', { users: null, duration: null })) + assert.match(script, /export function handleSummary\(data\) \{\n {2}return \{ stdout: JSON\.stringify\(data\) \}/) +}) diff --git a/packages/loadtest/src/script.ts b/packages/loadtest/src/script.ts new file mode 100644 index 0000000..4e2005a --- /dev/null +++ b/packages/loadtest/src/script.ts @@ -0,0 +1,57 @@ +/** + * nextship-loadtest: the k6 script for a plan + * + * k6 runs a JavaScript file. This writes it, so nobody using the command has to. + * Each step of the plan is its own k6 scenario, started when the one before ends, + * because k6 reports figures per scenario only for a scenario a threshold names. + * The thresholds here can never fail: they exist to make k6 keep the figures of + * each step apart, which is the only way to see which step a site stopped coping at. + * + * The target is read from the environment and the numbers come from a validated + * plan, so nothing the user typed is ever interpolated into the script. + * + * Author: Gowtham + */ + +import type { Plan } from './plan.js' + +/** The scenario name of a step, which the report reads the figures back by. */ +export const scenarioName = (index: number): string => `step${index + 1}` + +export function renderScript(plan: Plan): string { + let startsAt = 0 + const scenarios: Record = {} + const thresholds: Record = {} + + plan.steps.forEach((entry, index) => { + const name = scenarioName(index) + scenarios[name] = { + executor: 'constant-vus', + vus: entry.users, + duration: `${entry.seconds}s`, + startTime: `${startsAt}s`, + // A request still in flight when a step ends would otherwise run into the + // next step and be counted against it. + gracefulStop: '0s', + } + thresholds[`http_req_duration{scenario:${name}}`] = ['max>=0'] + thresholds[`http_req_failed{scenario:${name}}`] = ['rate>=0'] + thresholds[`http_reqs{scenario:${name}}`] = ['count>=0'] + startsAt += entry.seconds + }) + + const options = { scenarios, thresholds, summaryTrendStats: ['med', 'p(95)'] } + return [ + "import http from 'k6/http'", + `export const options = ${JSON.stringify(options, null, 2)}`, + 'export default function () {', + ' http.get(__ENV.TARGET)', + '}', + // The summary goes to stdout as JSON for the report to read, in place of the + // text summary k6 prints by default. + 'export function handleSummary(data) {', + ' return { stdout: JSON.stringify(data) }', + '}', + '', + ].join('\n') +} diff --git a/packages/loadtest/src/version.ts b/packages/loadtest/src/version.ts new file mode 100644 index 0000000..4222f3a --- /dev/null +++ b/packages/loadtest/src/version.ts @@ -0,0 +1,13 @@ +/** + * nextship-loadtest: the installed version + * + * Read once from the package manifest, and reported by `--version`. + * + * Author: Gowtham + */ + +import { createRequire } from 'node:module' + +export const VERSION: string = ( + createRequire(import.meta.url)('../package.json') as { version: string } +).version diff --git a/packages/loadtest/tsconfig.json b/packages/loadtest/tsconfig.json new file mode 100644 index 0000000..6fcd57f --- /dev/null +++ b/packages/loadtest/tsconfig.json @@ -0,0 +1,5 @@ +{ + "extends": "../../tsconfig.base.json", + "compilerOptions": { "rootDir": "src", "outDir": "dist" }, + "include": ["src"] +} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index dacb6dc..fe73608 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -34,6 +34,8 @@ importers: specifier: ^6.0.6 version: 6.0.6 + packages/loadtest: {} + packages: '@emnapi/runtime@1.11.3':