From 8c7eeae6ad2acf02b5cb51bc6a617697621d2fdd Mon Sep 17 00:00:00 2001 From: Logan Lindquist Land Date: Sun, 13 Sep 2026 10:01:37 -0500 Subject: [PATCH] Add local performance benchmark harness --- app/Commands/SeedMessages.php | 137 ++++++++++++++++++ app/Config/RateLimit.php | 14 ++ bench/README.md | 53 +++++++ bench/lib/auth.js | 83 +++++++++++ bench/post-messages.js | 42 ++++++ bench/read-messages.js | 39 +++++ bench/websocket-fanout.js | 83 +++++++++++ docs/performance.md | 28 ++++ package.json | 3 + .../Integration/SeedMessagesCommandTest.php | 67 +++++++++ 10 files changed, 549 insertions(+) create mode 100644 app/Commands/SeedMessages.php create mode 100644 bench/README.md create mode 100644 bench/lib/auth.js create mode 100644 bench/post-messages.js create mode 100644 bench/read-messages.js create mode 100644 bench/websocket-fanout.js create mode 100644 docs/performance.md create mode 100644 tests/Tests/Integration/SeedMessagesCommandTest.php diff --git a/app/Commands/SeedMessages.php b/app/Commands/SeedMessages.php new file mode 100644 index 0000000..fa56256 --- /dev/null +++ b/app/Commands/SeedMessages.php @@ -0,0 +1,137 @@ +'; + protected $arguments = [ + 'count' => 'Number of benchmark messages to create (1-1,000,000).', + ]; + + public function run(array $params): int + { + if (ENVIRONMENT === 'production') { + CLI::error('Benchmark seeding is disabled in production.'); + + return 1; + } + + if (! filter_var(env('BENCHMARK_MODE', false), FILTER_VALIDATE_BOOL)) { + CLI::error('Set BENCHMARK_MODE=true to seed benchmark data.'); + + return 1; + } + + $count = self::parseCount($params[0] ?? null); + + if ($count === null) { + CLI::error('Count must be an integer between 1 and ' . number_format(self::MAX_MESSAGES) . '.'); + + return 1; + } + + $db = db_connect(); + + try { + $this->replaceBenchmarkData($db, $count); + Services::cache()->clean(); + } catch (Throwable $exception) { + CLI::error('Unable to seed benchmark messages: ' . $exception->getMessage()); + + return 1; + } + + CLI::write(sprintf( + 'Seeded %s messages and %d benchmark users.', + number_format($count), + self::BENCHMARK_USER_COUNT, + ), 'green'); + + return 0; + } + + public static function parseCount(mixed $value): ?int + { + if (! is_string($value) || ! ctype_digit($value)) { + return null; + } + + $count = (int) $value; + + return $count >= 1 && $count <= self::MAX_MESSAGES ? $count : null; + } + + private function replaceBenchmarkData(BaseConnection $db, int $count): void + { + $db->transException(true)->transStart(); + + $this->seedUsers($db); + + $messages = $db->table('messages'); + $messages->like('user', 'k6-user-', 'after')->delete(); + + for ($offset = 0; $offset < $count; $offset += self::BATCH_SIZE) { + $batchSize = min(self::BATCH_SIZE, $count - $offset); + $rows = []; + + for ($index = 0; $index < $batchSize; $index++) { + $sequence = $offset + $index + 1; + $rows[] = [ + 'user' => self::usernameFor($sequence), + 'msg' => sprintf('Benchmark message %08d', $sequence), + 'time' => 1_700_000_000 + $sequence, + ]; + } + + $messages->insertBatch($rows); + } + + $db->transComplete(); + } + + private function seedUsers(BaseConnection $db): void + { + $users = $db->table('users'); + $password = password_hash(self::BENCHMARK_PASSWORD, PASSWORD_DEFAULT); + $now = date('Y-m-d H:i:s'); + + for ($sequence = 1; $sequence <= self::BENCHMARK_USER_COUNT; $sequence++) { + $username = self::usernameFor($sequence); + $existing = $users->select('id')->where('username', $username)->get()->getRowArray(); + $row = [ + 'email' => sprintf('k6-user-%03d@example.test', $sequence), + 'password' => $password, + 'updated_at' => $now, + ]; + + if ($existing === null) { + $users->insert($row + [ + 'username' => $username, + 'created_at' => $now, + ]); + } else { + $users->where('id', $existing['id'])->update($row); + } + } + } + + private static function usernameFor(int $sequence): string + { + return sprintf('k6-user-%03d', (($sequence - 1) % self::BENCHMARK_USER_COUNT) + 1); + } +} diff --git a/app/Config/RateLimit.php b/app/Config/RateLimit.php index 9076db2..00bdef8 100644 --- a/app/Config/RateLimit.php +++ b/app/Config/RateLimit.php @@ -27,4 +27,18 @@ class RateLimit extends BaseConfig 'authenticated' => [30, 60], ], ]; + + public function __construct() + { + parent::__construct(); + + if (ENVIRONMENT !== 'production' && filter_var(env('BENCHMARK_MODE', false), FILTER_VALIDATE_BOOL)) { + foreach ($this->profiles as $profile => $limits) { + $this->profiles[$profile] = [ + 'anonymous' => [1_000_000, $limits['anonymous'][1]], + 'authenticated' => [1_000_000, $limits['authenticated'][1]], + ]; + } + } + } } diff --git a/bench/README.md b/bench/README.md new file mode 100644 index 0000000..e692a10 --- /dev/null +++ b/bench/README.md @@ -0,0 +1,53 @@ +# Local performance benchmarks + +These k6 scenarios exercise the application through its public HTTP and WebSocket interfaces. They are intentionally local-only and never run in CI. + +## Prerequisites + +- PHP and project dependencies from `composer install` +- [k6](https://grafana.com/docs/k6/latest/set-up/install-k6/) +- SQLite support, or a configured MySQL database +- The application migrations applied to the benchmark database + +Prepare a reproducible dataset. This replaces messages owned by the reserved `k6-user-*` accounts and creates 100 local benchmark users with password `Benchmark123!`: + +```shell +DB_DRIVER=SQLite3 php spark migrate --all +BENCHMARK_MODE=true DB_DRIVER=SQLite3 php spark seed:messages 10000 +``` + +The seed command refuses to run in production or without `BENCHMARK_MODE=true` because it creates known local-only credentials and replaces messages owned by the reserved accounts. + +Start the HTTP server with multiple workers and the local-only benchmark rate-limit profile. `BENCHMARK_MODE` is ignored in production: + +```shell +CI_ENVIRONMENT=development PHP_CLI_SERVER_WORKERS=8 BENCHMARK_MODE=true DB_DRIVER=SQLite3 app_baseURL=http://127.0.0.1:8080/ app_CSPEnabled=false cookie_secure=false php spark serve --host 127.0.0.1 --port 8080 +``` + +For the WebSocket fanout scenario, start the WebSocket server in another terminal: + +```shell +DB_DRIVER=SQLite3 php spark chat:websocket --port 8081 +``` + +Keep the WebSocket server stopped while measuring message posts. Post requests attempt a best-effort broadcast, so running the standalone server would mix WebSocket handshake behavior into the HTTP persistence measurement. The dedicated fanout scenario measures WebSocket delivery separately. + +## Run scenarios + +From the project root: + +```shell +pnpm bench:post +pnpm bench:read +pnpm bench:ws +``` + +Override endpoints or the deterministic password when needed: + +```shell +BASE_URL=http://127.0.0.1:8080 WS_URL=ws://127.0.0.1:8081 BENCH_PASSWORD='Benchmark123!' pnpm bench:ws +``` + +The post scenario runs 10 virtual users for 30 seconds and expects p95 message-post latency below 200 ms. The read scenario runs 50 polling users for 30 seconds and expects p99 read latency below 200 ms. The fanout scenario connects 100 users, lets every client broadcast once, and expects p95 delivery below 500 ms. + +Treat thresholds as local smoke targets, not universal service-level objectives. Compare results only on a similar machine, database, dataset, and server configuration. A failed check usually indicates authentication, CSRF, rate-limit, or server setup trouble; a failed duration threshold indicates a performance result worth investigating. diff --git a/bench/lib/auth.js b/bench/lib/auth.js new file mode 100644 index 0000000..8d0d12f --- /dev/null +++ b/bench/lib/auth.js @@ -0,0 +1,83 @@ +import http from 'k6/http'; +import { check, fail } from 'k6'; +import exec from 'k6/execution'; + +export const baseUrl = (__ENV.BASE_URL || 'http://127.0.0.1:8080').replace(/\/$/, ''); +export const websocketUrl = (__ENV.WS_URL || 'ws://127.0.0.1:8081').replace(/\/$/, ''); + +const password = __ENV.BENCH_PASSWORD || 'Benchmark123!'; + +export function login() { + return loginForUser(((exec.vu.idInTest - 1) % 100) + 1); +} + +export function loginForUser(sequence) { + const username = `k6-user-${String(sequence).padStart(3, '0')}`; + const loginPage = http.get(`${baseUrl}/auth/login`, { tags: { name: 'GET /auth/login' } }); + const loginCsrf = metaValue(loginPage.body, 'csrf-token'); + + if (!loginCsrf) { + fail(`Could not read the CSRF token from ${baseUrl}/auth/login`); + } + + const response = http.post(`${baseUrl}/auth/processLogin`, { + username, + password, + csrf_test_name: loginCsrf, + }, { + redirects: 5, + tags: { name: 'POST /auth/processLogin' }, + }); + const authenticated = check(response, { + 'login succeeds': (result) => result.status === 200 && result.url.includes('/chat'), + }); + + if (!authenticated) { + fail(`Login failed for ${username} with status ${response.status}`); + } + + const chatPage = http.get(`${baseUrl}/chat/vue`, { tags: { name: 'GET /chat/vue' } }); + const userId = scriptValue(chatPage.body, 'CURRENT_USER_ID'); + const websocketToken = scriptValue(chatPage.body, 'WEBSOCKET_TOKEN'); + + if (!userId || !websocketToken) { + fail(`Could not read WebSocket credentials for ${username}`); + } + + return { + username, + userId, + websocketToken, + }; +} + +export function csrfToken() { + const cookies = http.cookieJar().cookiesForURL(baseUrl); + const values = cookies.csrf_cookie_name || []; + + if (values.length === 0) { + fail('The authenticated session does not contain a CSRF cookie'); + } + + return values[0]; +} + +function metaValue(html, name) { + if (typeof html !== 'string') { + return ''; + } + + const match = html.match(new RegExp(`]+name=["']${name}["'][^>]+content=["']([^"']+)["']`, 'i')); + + return match ? match[1] : ''; +} + +function scriptValue(html, name) { + if (typeof html !== 'string') { + return ''; + } + + const match = html.match(new RegExp(`${name}\\s*=\\s*["']?([^"';\\s]+)`)); + + return match ? match[1] : ''; +} diff --git a/bench/post-messages.js b/bench/post-messages.js new file mode 100644 index 0000000..3a0d267 --- /dev/null +++ b/bench/post-messages.js @@ -0,0 +1,42 @@ +import http from 'k6/http'; +import { check, sleep } from 'k6'; +import { Rate, Trend } from 'k6/metrics'; +import { baseUrl, csrfToken, login } from './lib/auth.js'; + +const failures = new Rate('post_message_failures'); +const latency = new Trend('post_message_duration', true); +let session; + +export const options = { + noCookiesReset: true, + scenarios: { + post_messages: { + executor: 'constant-vus', + vus: 10, + duration: '30s', + }, + }, + thresholds: { + checks: ['rate>0.99'], + post_message_failures: ['rate<0.01'], + post_message_duration: ['p(95)<200'], + }, +}; + +export default function () { + session ||= login(); + const response = http.post(`${baseUrl}/api/v1/messages`, { + message: `k6 post ${session.username} ${Date.now()}`, + csrf_test_name: csrfToken(), + }, { + headers: { Accept: 'application/json' }, + tags: { name: 'POST /api/v1/messages' }, + }); + const succeeded = check(response, { + 'message post succeeds': (result) => result.status === 200 && result.json('success') === true, + }); + + failures.add(!succeeded); + latency.add(response.timings.duration); + sleep(0.5); +} diff --git a/bench/read-messages.js b/bench/read-messages.js new file mode 100644 index 0000000..524431e --- /dev/null +++ b/bench/read-messages.js @@ -0,0 +1,39 @@ +import http from 'k6/http'; +import { check, sleep } from 'k6'; +import { Rate, Trend } from 'k6/metrics'; +import { baseUrl, login } from './lib/auth.js'; + +const failures = new Rate('read_message_failures'); +const latency = new Trend('read_message_duration', true); +let session; + +export const options = { + noCookiesReset: true, + scenarios: { + read_messages: { + executor: 'constant-vus', + vus: 50, + duration: '30s', + }, + }, + thresholds: { + checks: ['rate>0.99'], + read_message_failures: ['rate<0.01'], + read_message_duration: ['p(99)<200'], + }, +}; + +export default function () { + session ||= login(); + const response = http.get(`${baseUrl}/api/v1/messages?page=1&per_page=50`, { + headers: { Accept: 'application/json' }, + tags: { name: 'GET /api/v1/messages' }, + }); + const succeeded = check(response, { + 'message read succeeds': (result) => result.status === 200 && Array.isArray(result.json('messages')), + }); + + failures.add(!succeeded); + latency.add(response.timings.duration); + sleep(0.5); +} diff --git a/bench/websocket-fanout.js b/bench/websocket-fanout.js new file mode 100644 index 0000000..bbd063a --- /dev/null +++ b/bench/websocket-fanout.js @@ -0,0 +1,83 @@ +import { check } from 'k6'; +import { Rate, Trend } from 'k6/metrics'; +import exec from 'k6/execution'; +import { WebSocket } from 'k6/websockets'; +import { loginForUser, websocketUrl } from './lib/auth.js'; + +const deliveries = new Rate('websocket_delivery_success'); +const latency = new Trend('websocket_delivery_duration', true); + +export const options = { + noCookiesReset: true, + setupTimeout: '2m', + scenarios: { + websocket_fanout: { + executor: 'per-vu-iterations', + vus: 100, + iterations: 1, + maxDuration: '30s', + }, + }, + thresholds: { + checks: ['rate>0.99'], + websocket_delivery_success: ['rate>0.99'], + websocket_delivery_duration: ['p(95)<500'], + }, +}; + +export function setup() { + const sessions = []; + + for (let sequence = 1; sequence <= 100; sequence += 1) { + sessions.push(loginForUser(sequence)); + } + + return sessions; +} + +export default function (sessions) { + const session = sessions[exec.vu.idInTest - 1]; + const marker = `k6 fanout ${session.username} ${Date.now()}`; + const socket = new WebSocket( + `${websocketUrl}?token=${encodeURIComponent(session.websocketToken)}&user_id=${session.userId}`, + ); + let sentAt = 0; + let delivered = false; + let deliveryTimeout; + + socket.addEventListener('open', () => { + setTimeout(() => { + sentAt = Date.now(); + socket.send(JSON.stringify({ + action: 'sendMessage', + username: session.username, + message: marker, + })); + }, 2_000); + }); + + socket.addEventListener('message', (event) => { + const payload = JSON.parse(event.data); + + if (payload.action === 'newMessage' && payload.data?.msg === marker) { + delivered = true; + deliveries.add(true); + latency.add(Date.now() - sentAt); + check(payload, { 'broadcast returns the sent message': () => true }); + clearTimeout(deliveryTimeout); + socket.close(); + } + }); + + socket.addEventListener('error', () => { + deliveries.add(false); + }); + + deliveryTimeout = setTimeout(() => { + if (!delivered) { + deliveries.add(false); + check(delivered, { 'broadcast returns the sent message': (value) => value }); + socket.close(); + } + }, 10_000); +} diff --git a/docs/performance.md b/docs/performance.md new file mode 100644 index 0000000..83f5467 --- /dev/null +++ b/docs/performance.md @@ -0,0 +1,28 @@ +# Performance baseline + +The k6 harness in `bench/` provides local comparison points for performance-sensitive changes. These numbers are not production capacity claims or CI gates. + +## Baseline environment + +- Application revision: `12b314cfd871c2380a725398802e551ac21e26c8` +- Date: September 13, 2026 +- Machine: Mac Studio, Apple M1 Ultra (20 cores), 64 GB memory +- Runtime: macOS 26.6.2, PHP 8.5.7, k6 2.2.0, SQLite 3.51.0 +- Server: PHP development server with 8 workers; Ratchet WebSocket server used only for the fanout scenario +- Dataset: 10,000 deterministic messages and 100 benchmark users + +The application revision identifies the endpoint and persistence code being measured. The benchmark harness itself was the issue #119 working tree based on that revision. + +## Results + +| Scenario | Load | Throughput | Latency | Failures | +| --- | --- | ---: | ---: | ---: | +| Message posts | 10 VUs for 30s | 17.94 iterations/s | p95 72.51 ms | 0/547 | +| Message reads | 50 VUs for 30s | 89.56 iterations/s | p99 94.15 ms | 0/2,752 | +| WebSocket fanout | 100 simultaneous clients | 100/100 deliveries | p95 95.05 ms | 0/100 | + +The WebSocket run established all 100 connections, sent one marked message per client, and observed 5,050 total received frames while measuring each sender's own broadcast round trip. Connection p95 was 40.28 ms. + +## Comparing later runs + +Re-run the same scenario on comparable hardware with the same dataset and server settings. Compare the custom duration and failure metrics, not the login/setup HTTP totals. Record a new baseline only for intentional performance work, and explain material configuration or environment differences alongside the numbers. diff --git a/package.json b/package.json index c83f465..f523f90 100644 --- a/package.json +++ b/package.json @@ -11,6 +11,9 @@ "dev": "vite", "build": "vite build", "preview": "vite preview", + "bench:post": "k6 run bench/post-messages.js", + "bench:read": "k6 run bench/read-messages.js", + "bench:ws": "k6 run bench/websocket-fanout.js", "e2e": "playwright test", "e2e:ui": "playwright test --ui", "e2e:codegen": "playwright codegen http://127.0.0.1:8085" diff --git a/tests/Tests/Integration/SeedMessagesCommandTest.php b/tests/Tests/Integration/SeedMessagesCommandTest.php new file mode 100644 index 0000000..2f9e43e --- /dev/null +++ b/tests/Tests/Integration/SeedMessagesCommandTest.php @@ -0,0 +1,67 @@ +setBenchmarkMode(null); + + parent::tearDown(); + } + + public function testItRequiresBenchmarkMode(): void + { + $this->setBenchmarkMode(null); + $command = new SeedMessages(service('logger'), service('commands')); + + $this->assertSame(1, $command->run(['1'])); + $this->assertSame(0, $this->db->table('users')->like('username', 'k6-user-', 'after')->countAllResults()); + } + + public function testItRejectsInvalidCounts(): void + { + $this->assertNull(SeedMessages::parseCount(null)); + $this->assertNull(SeedMessages::parseCount('0')); + $this->assertNull(SeedMessages::parseCount('-1')); + $this->assertNull(SeedMessages::parseCount('1000001')); + $this->assertSame(25, SeedMessages::parseCount('25')); + } + + public function testItReplacesBenchmarkMessagesAndCreatesLoadUsers(): void + { + $this->setBenchmarkMode('true'); + $command = new SeedMessages(service('logger'), service('commands')); + + $this->assertSame(0, $command->run(['3'])); + $this->assertSame(3, $this->db->table('messages')->like('user', 'k6-user-', 'after')->countAllResults()); + $this->assertSame(100, $this->db->table('users')->like('username', 'k6-user-', 'after')->countAllResults()); + + $this->assertSame(0, $command->run(['2'])); + $messages = $this->db->table('messages')->like('user', 'k6-user-', 'after')->orderBy('time')->get()->getResultArray(); + + $this->assertCount(2, $messages); + $this->assertSame('Benchmark message 00000001', $messages[0]['msg']); + $this->assertSame('1700000001', (string) $messages[0]['time']); + } + + private function setBenchmarkMode(?string $value): void + { + if ($value === null) { + putenv('BENCHMARK_MODE'); + unset($_ENV['BENCHMARK_MODE'], $_SERVER['BENCHMARK_MODE']); + + return; + } + + putenv('BENCHMARK_MODE=' . $value); + $_ENV['BENCHMARK_MODE'] = $value; + $_SERVER['BENCHMARK_MODE'] = $value; + } +}