From bae08d40eca72e845b28185e8bcf7eb4974f4e2f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Fri, 13 Nov 2026 11:00:39 +0100 Subject: [PATCH 1/2] ci(conformance): keep the Maestro version probe's failure evidence (#4455) The layer-3 pin check parsed `maestro --version` inside a pipeline that sent stderr to /dev/null and masked the pipeline exit with `|| true`. Its whole failure record was `expected= actual=`, so the November 9 run could not tell a CLI that never started from one that printed nothing parseable. Run the probe through scripts/maestro-conformance/verify-cli-version.mjs. It records the exit status, signal, spawn error and both streams, retains each stream's last 8 KiB newest-first, and reports one verdict: command failure, missing or unparseable version, mismatch, or match. The pin invariant is unchanged and still fails closed; a probe that did not complete cleanly is never accepted on the strength of its stdout. The fixture-executable suite covers a matching version inside notices, a wrong version, empty and unparseable stdout, a nonzero exit with matching-looking stdout, an absent executable, and bounded oversized output. --- .../workflows/conformance-differential.yml | 18 +- package.json | 2 +- scripts/maestro-conformance/README.md | 6 +- .../verify-cli-version.mjs | 200 ++++++++++++++++++ .../verify-cli-version.test.mjs | 194 +++++++++++++++++ 5 files changed, 406 insertions(+), 14 deletions(-) create mode 100644 scripts/maestro-conformance/verify-cli-version.mjs create mode 100644 scripts/maestro-conformance/verify-cli-version.test.mjs diff --git a/.github/workflows/conformance-differential.yml b/.github/workflows/conformance-differential.yml index 911060e845..fd04fe7c74 100644 --- a/.github/workflows/conformance-differential.yml +++ b/.github/workflows/conformance-differential.yml @@ -117,19 +117,13 @@ jobs: curl -fsSL "https://get.maestro.mobile.dev" | bash echo "$HOME/.maestro/bin" >> "$GITHUB_PATH" + # Asserts what `maestro --version` actually reports against the same pin layers 1-2 use. + # The probe lives in scripts/maestro-conformance so its failure keeps the command's exit, + # stdout and stderr as bounded diagnostics and names which of command failure, + # missing/unparseable version, or a genuine mismatch it saw (#4455). It fails closed: only a + # reported version equal to the pin gets the differential to run. - name: Verify the Maestro CLI matches the oracle pin - run: | - set -euo pipefail - EXPECTED=$(node -p "require('./scripts/maestro-conformance/pinned-upstream.json').version") - # `maestro --version` can print notices (e.g. the analytics banner) - # around the version, so match the semver line rather than the whole - # output. Empty ACTUAL fails the comparison below, as it should. - ACTUAL=$(maestro --version 2>/dev/null | grep -oE '[0-9]+\.[0-9]+\.[0-9]+' | tail -1 || true) - echo "expected=$EXPECTED actual=$ACTUAL" - if [ "$ACTUAL" != "$EXPECTED" ]; then - echo "::error::Layer-3 Maestro CLI is $ACTUAL but the oracle pins $EXPECTED. A differential against a different Maestro proves nothing about the pinned behavior." - exit 1 - fi + run: node scripts/maestro-conformance/verify-cli-version.mjs # --trace-root lets the runner find agent-device's replay-timing.ndjson and # assert engine-side invariants (e.g. the settle loop latches instead of diff --git a/package.json b/package.json index 31406a2dff..ff36b8809e 100644 --- a/package.json +++ b/package.json @@ -125,7 +125,7 @@ "ad": "node bin/agent-device.mjs", "bench:android:automation": "node scripts/bench-android-automation.mjs", "bench:help-conformance": "node scripts/help-conformance-bench.mjs", - "maestro:conformance": "node --experimental-strip-types scripts/node-test-tmpdir.ts --experimental-strip-types --test scripts/maestro-conformance/format-generated-json.test.mjs packages/maestro/test/conformance/verify.test.ts packages/maestro/test/conformance/differential/engine-process.test.ts packages/maestro/test/conformance/differential/report-output.test.ts packages/maestro/test/conformance/differential/run.test.ts packages/maestro/test/conformance/differential/invariants.test.ts", + "maestro:conformance": "node --experimental-strip-types scripts/node-test-tmpdir.ts --experimental-strip-types --test scripts/maestro-conformance/format-generated-json.test.mjs scripts/maestro-conformance/verify-cli-version.test.mjs packages/maestro/test/conformance/verify.test.ts packages/maestro/test/conformance/differential/engine-process.test.ts packages/maestro/test/conformance/differential/report-output.test.ts packages/maestro/test/conformance/differential/run.test.ts packages/maestro/test/conformance/differential/invariants.test.ts", "maestro:conformance:regenerate": "node --experimental-strip-types scripts/maestro-conformance/regenerate.mjs", "maestro:conformance:differential": "node --experimental-strip-types packages/maestro/test/conformance/differential/run.ts", "test:ios-snapshot-differential": "node --experimental-strip-types scripts/swift-toolchain-tmpdir.ts swift test --package-path apple/snapshot-presentation && node --experimental-strip-types scripts/node-test-tmpdir.ts --experimental-strip-types --test scripts/ios-snapshot-differential.test.ts", diff --git a/scripts/maestro-conformance/README.md b/scripts/maestro-conformance/README.md index 6ff716b573..27feb2b8ed 100644 --- a/scripts/maestro-conformance/README.md +++ b/scripts/maestro-conformance/README.md @@ -147,7 +147,11 @@ Runs scheduled on the `conformance-differential` workflow, and on demand via The workflow builds and installs the fixture app, verifies its bundle id, pins the Maestro CLI to the same version as layers 1-2, and passes `--maestro` so the -flow routes through the compat engine. +flow routes through the compat engine. The pin check runs +[`verify-cli-version.mjs`](./verify-cli-version.mjs): it fails unless +`maestro --version` exits 0 and its newest semver equals the pin, and it keeps the +probe's exit, stdout and stderr as bounded diagnostics so a failure names its cause +(#4455). The built `.app` is **cached** (keyed on the app's sources, its dependency graph, the iOS runtime, and the Xcode version), because building it costs ~22 minutes diff --git a/scripts/maestro-conformance/verify-cli-version.mjs b/scripts/maestro-conformance/verify-cli-version.mjs new file mode 100644 index 0000000000..fbd3327690 --- /dev/null +++ b/scripts/maestro-conformance/verify-cli-version.mjs @@ -0,0 +1,200 @@ +#!/usr/bin/env node +// Asserts that the installed Maestro CLI reports the version this oracle pins, and keeps the +// probe's exit status, stdout and stderr as the failure evidence (#4455). +// +// The workflow step this replaced parsed the version inside a shell pipeline that sent the CLI's +// stderr to /dev/null and masked the pipeline's exit with `|| true`. Its whole failure record was +// `expected= actual=`, which cannot tell apart the causes that need different fixes: the CLI +// never started, it started and printed nothing parseable, or a genuinely different Maestro is +// installed. The invariant stays strict — anything other than the pinned version fails the step — +// because a differential against an unproven CLI proves nothing about pinned behavior. What was +// missing is the cause, which is what this reports. +// +// A script rather than `run:` shell because retained evidence has to be bounded: a JVM startup +// failure prints a stack trace, and a step log is not the place to dump one unbounded. Each stream +// keeps its last MAX_DIAGNOSTIC_BYTES, newest first, which is also the half the parse wants — the +// notice-tolerant match takes the LAST semver of stdout, as the `grep | tail -1` it replaced did. +import { spawn } from 'node:child_process'; +import fs from 'node:fs'; +import path from 'node:path'; +import { fileURLToPath, pathToFileURL } from 'node:url'; + +const HERE = path.dirname(fileURLToPath(import.meta.url)); + +/** Per stream: how much of the probe's output the run retains and prints. */ +export const MAX_DIAGNOSTIC_BYTES = 8192; + +/** The shape the step grepped for. Tolerates notices (e.g. the analytics banner) around it. */ +const SEMVER_PATTERN = /\d+\.\d+\.\d+/g; + +/** Every verdict the probe can reach. Only `match` lets the differential proceed. */ +const PROBE_OUTCOMES = Object.freeze({ + match: 'match', + commandFailed: 'command-failed', + missingVersion: 'missing-version', + unparseableVersion: 'unparseable-version', + versionMismatch: 'version-mismatch', +}); + +/** Reads the pin layers 1-2 use, so the two cannot drift apart. */ +export function readPinnedVersion() { + const { version } = JSON.parse(fs.readFileSync(path.join(HERE, 'pinned-upstream.json'), 'utf8')); + if (typeof version !== 'string' || version === '') { + throw new Error(`pinned-upstream.json declares no version to check the Maestro CLI against`); + } + return version; +} + +/** The last semver in `output`, or null when it carries none. */ +export function parseVersion(output) { + return output.match(SEMVER_PATTERN)?.at(-1) ?? null; +} + +/** + * Runs the version probe and records what it did rather than what a pipeline survived: exit status, + * terminating signal, a spawn failure's code, and both output streams. + */ +function probeVersion({ command, args = ['--version'], env = process.env }) { + return new Promise((resolve) => { + const child = spawn(command, args, { env, stdio: ['ignore', 'pipe', 'pipe'] }); + const stdout = boundedCapture(); + const stderr = boundedCapture(); + child.stdout.on('data', (chunk) => stdout.append(chunk)); + child.stderr.on('data', (chunk) => stderr.append(chunk)); + + let settled = false; + child.on('close', (exitCode, signal) => { + if (settled) return; + settled = true; + resolve({ + command, + args, + spawnErrorCode: undefined, + exitCode, + signal, + stdout: stdout.read(), + stderr: stderr.read(), + }); + }); + // A failed spawn (an absent or non-executable `maestro`) reports here and never closes. + child.on('error', (error) => { + if (settled) return; + settled = true; + resolve({ + command, + args, + spawnErrorCode: error?.code, + exitCode: null, + signal: null, + stdout: stdout.read(), + stderr: stderr.read(), + }); + }); + }); +} + +/** + * Decides the step's verdict. A probe that did not complete cleanly is never accepted on the + * strength of its stdout: a CLI that prints the pinned version and still exits nonzero is a CLI + * whose behavior the differential cannot trust. + */ +export function evaluateVersionProbe(probe, expected) { + const version = parseVersion(probe.stdout.value); + const outcome = probeOutcome(probe, version, expected); + return { outcome, version, report: reportFor(probe, { outcome, version, expected }) }; +} + +function probeOutcome(probe, version, expected) { + if (probe.spawnErrorCode !== undefined || probe.signal !== null || probe.exitCode !== 0) { + return PROBE_OUTCOMES.commandFailed; + } + if (version !== null) { + return version === expected ? PROBE_OUTCOMES.match : PROBE_OUTCOMES.versionMismatch; + } + return probe.stdout.value === '' + ? PROBE_OUTCOMES.missingVersion + : PROBE_OUTCOMES.unparseableVersion; +} + +const OUTCOME_EXPLANATION = { + [PROBE_OUTCOMES.commandFailed]: 'did not complete successfully', + [PROBE_OUTCOMES.missingVersion]: 'printed no stdout at all', + [PROBE_OUTCOMES.unparseableVersion]: 'printed no version-shaped text', + [PROBE_OUTCOMES.versionMismatch]: 'reports a different Maestro version', + [PROBE_OUTCOMES.match]: 'reports the pinned version', +}; + +function reportFor(probe, { outcome, version, expected }) { + const facts = [ + `expected=${expected} actual=${version ?? ''}`, + `probe=${probe.command} ${probe.args.join(' ')}` + + ` exit=${probe.exitCode ?? '-'} signal=${probe.signal ?? '-'}` + + ` spawnError=${probe.spawnErrorCode ?? '-'}`, + `verdict=${outcome} (${OUTCOME_EXPLANATION[outcome]})`, + ]; + if (outcome === PROBE_OUTCOMES.match) { + return { stdout: facts, stderr: [] }; + } + const error = + `::error::Layer-3 Maestro CLI check failed with verdict ${outcome}: the installed ` + + `maestro ${OUTCOME_EXPLANATION[outcome]}. The oracle pins ${expected}; a differential ` + + 'against a different or unhealthy Maestro proves nothing about the pinned behavior.'; + return { + stdout: [...facts, error, describeStream(probe.stdout, 'stdout')], + stderr: [describeStream(probe.stderr, 'stderr')], + }; +} + +function describeStream(stream, name) { + const bound = + `(retained the last ${stream.bytes} of ${stream.totalBytes} bytes` + + (stream.droppedBytes > 0 ? `, dropped ${stream.droppedBytes}` : '') + + ')'; + return [`--- maestro --version ${name} ${bound} ---`, stream.value.trim() || '(empty)'].join( + '\n', + ); +} + +/** Retains at most `MAX_DIAGNOSTIC_BYTES` of a stream, newest first, and counts what it dropped. */ +function boundedCapture() { + let totalBytes = 0; + let retainedBytes = 0; + const chunks = []; + return { + append(chunk) { + const buffer = Buffer.isBuffer(chunk) ? chunk : Buffer.from(String(chunk)); + totalBytes += buffer.length; + retainedBytes += buffer.length; + chunks.push(buffer); + // One failing JVM can report its whole stack in a single write, so the newest chunk is + // trimmed too: bounding only whole chunks would keep whatever one write carried. + while (retainedBytes > MAX_DIAGNOSTIC_BYTES) { + const oldest = chunks[0]; + const excess = retainedBytes - MAX_DIAGNOSTIC_BYTES; + if (oldest.length <= excess) { + chunks.shift(); + retainedBytes -= oldest.length; + } else { + chunks[0] = oldest.subarray(excess); + retainedBytes -= excess; + } + } + }, + read() { + const value = Buffer.concat(chunks).toString('utf8'); + return { value, bytes: retainedBytes, totalBytes, droppedBytes: totalBytes - retainedBytes }; + }, + }; +} + +async function main({ command = 'maestro', env = process.env } = {}) { + const expected = readPinnedVersion(); + const { outcome, report } = evaluateVersionProbe(await probeVersion({ command, env }), expected); + for (const line of report.stdout) process.stdout.write(`${line}\n`); + for (const line of report.stderr) process.stderr.write(`${line}\n`); + return outcome === PROBE_OUTCOMES.match ? 0 : 1; +} + +if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { + process.exitCode = await main(); +} diff --git a/scripts/maestro-conformance/verify-cli-version.test.mjs b/scripts/maestro-conformance/verify-cli-version.test.mjs new file mode 100644 index 0000000000..fdad11a018 --- /dev/null +++ b/scripts/maestro-conformance/verify-cli-version.test.mjs @@ -0,0 +1,194 @@ +// Exercises the layer-3 Maestro version probe end to end: the script the conformance workflow step +// runs, against fixture executables on PATH, so each verdict is proven by a real subprocess exit +// code and real output rather than a stub of itself (#4455). + +import assert from 'node:assert/strict'; +import { spawnSync } from 'node:child_process'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import test from 'node:test'; +import { parse } from 'yaml'; +import { + MAX_DIAGNOSTIC_BYTES, + evaluateVersionProbe, + parseVersion, + readPinnedVersion, +} from './verify-cli-version.mjs'; + +const repoRoot = path.resolve(import.meta.dirname, '../..'); +const scriptPath = path.join(repoRoot, 'scripts/maestro-conformance/verify-cli-version.mjs'); +const workflowPath = path.join(repoRoot, '.github/workflows/conformance-differential.yml'); +const PIN = readPinnedVersion(); + +function scratchDirectory(prefix) { + const root = fs.mkdtempSync(path.join(os.tmpdir(), prefix)); + test.after(() => fs.rmSync(root, { force: true, recursive: true })); + return root; +} + +/** + * Runs the real probe script with a fixture `maestro` first on PATH — so it shadows any real CLI + * the host has — that prints `stdout` and `stderr` and exits `exitCode`. + */ +function runProbe({ exitCode = 0, stdout = '', stderr = '' } = {}) { + const root = scratchDirectory('maestro-version-probe-'); + const bin = path.join(root, 'bin'); + fs.mkdirSync(bin); + const stdoutFile = path.join(root, 'stdout.txt'); + const stderrFile = path.join(root, 'stderr.txt'); + fs.writeFileSync(stdoutFile, stdout); + fs.writeFileSync(stderrFile, stderr); + const fixture = path.join(bin, 'maestro'); + fs.writeFileSync( + fixture, + '#!/bin/sh\n' + + `cat ${JSON.stringify(stdoutFile)}\n` + + `cat ${JSON.stringify(stderrFile)} >&2\n` + + `exit ${exitCode}\n`, + ); + fs.chmodSync(fixture, 0o755); + const result = spawnSync(process.execPath, [scriptPath], { + cwd: repoRoot, + encoding: 'utf8', + // The fixture directory comes first, so it shadows a real `maestro` the host may have. + env: { ...process.env, PATH: `${bin}${path.delimiter}${process.env.PATH ?? ''}` }, + }); + return { ...result, output: `${result.stdout}${result.stderr}` }; +} + +function completedProbe(overrides) { + return { + command: 'maestro', + args: ['--version'], + spawnErrorCode: undefined, + exitCode: 0, + signal: null, + stdout: { value: '', bytes: 0, totalBytes: 0, droppedBytes: 0 }, + stderr: { value: '', bytes: 0, totalBytes: 0, droppedBytes: 0 }, + ...overrides, + }; +} + +function streamOf(value) { + return { value, bytes: value.length, totalBytes: value.length, droppedBytes: 0 }; +} + +test(`stdout notices around the pinned ${PIN} still match the pin`, () => { + const run = runProbe({ + stdout: + 'Thank you for using Maestro! Analytics is disabled.\n' + + `Maestro 2.0.0 installed 2026-01-02\n===\n${PIN}\n`, + }); + assert.equal(run.status, 0, run.output); + assert.match(run.stdout, new RegExp(`expected=${PIN} actual=${PIN}`)); + assert.match(run.stdout, /verdict=match \(reports the pinned version\)/); + assert.doesNotMatch(run.output, /::error::/); +}); + +test('a probe whose newest semver is not the pin is a mismatch naming both sides', () => { + const run = runProbe({ stdout: `${PIN}\n1.2.3\n` }); + assert.equal(run.status, 1); + // The last semver wins: the same half of the output `grep | tail -1` read. + assert.match(run.stdout, new RegExp(`expected=${PIN} actual=1\\.2\\.3`)); + assert.match(run.stdout, /verdict=version-mismatch \(reports a different Maestro version\)/); + assert.match( + run.stdout, + /::error::Layer-3 Maestro CLI check failed with verdict version-mismatch/, + ); +}); + +test('empty stdout is reported as a probe that printed nothing', () => { + const run = runProbe(); + assert.equal(run.status, 1); + assert.match(run.stdout, new RegExp(`expected=${PIN} actual=$`, 'm')); + assert.match(run.stdout, /verdict=missing-version \(printed no stdout at all\)/); + assert.match(run.stdout, /maestro --version stdout .*\n\(empty\)/s); +}); + +test('stdout without a version is reported as unparseable, not as a mismatch', () => { + const run = runProbe({ stdout: 'usage: maestro \n' }); + assert.equal(run.status, 1); + assert.match(run.stdout, /verdict=unparseable-version \(printed no version-shaped text\)/); + assert.match(run.stdout, /usage: maestro /); +}); + +test('a nonzero exit is a command failure even when stdout prints the pinned version', () => { + const stderr = 'Fatal error: could not start JVM\n'; + const run = runProbe({ exitCode: 1, stdout: `${PIN}\n`, stderr }); + assert.equal(run.status, 1); + assert.match(run.stdout, /verdict=command-failed \(did not complete successfully\)/); + // Exactly what the discarded-stderr masked pipeline lost: the exit code and the CLI's reason. + assert.match(run.stdout, /exit=1 signal=- spawnError=-/); + assert.match(run.output, /Fatal error: could not start JVM/); + assert.match( + run.stderr, + new RegExp( + `maestro --version stderr \\(retained the last ${stderr.length} of ${stderr.length} bytes\\)`, + ), + ); +}); + +test('an absent executable reports the spawn failure instead of a blank actual', () => { + const root = scratchDirectory('maestro-version-probe-absent-'); + const run = spawnSync(process.execPath, [scriptPath], { + cwd: repoRoot, + encoding: 'utf8', + // Only the system directories, and no `maestro` anywhere on them. + env: { ...process.env, PATH: path.join(root, 'bin') }, + }); + assert.equal(run.status, 1); + assert.match(run.stdout, /verdict=command-failed \(did not complete successfully\)/); + assert.match(run.stdout, /spawnError=ENOENT/); +}); + +test('oversized output is retained only up to its bounded tail and names what it dropped', () => { + const filler = 'x'.repeat(MAX_DIAGNOSTIC_BYTES * 3); + const run = runProbe({ exitCode: 1, stdout: `${PIN}\n`, stderr: `${filler}\nEND OF FAILURE\n` }); + assert.equal(run.status, 1); + assert.match(run.output, /END OF FAILURE/); + assert.match( + run.stderr, + new RegExp(`retained the last ${MAX_DIAGNOSTIC_BYTES} of \\d+ bytes, dropped \\d+`), + ); + assert.ok(run.output.length < MAX_DIAGNOSTIC_BYTES * 2, 'the step log stays bounded'); +}); + +test('a probe that did not complete cleanly is never accepted on the strength of its stdout', () => { + const stdout = streamOf(`${PIN}\n`); + assert.equal(evaluateVersionProbe(completedProbe({ stdout }), PIN).outcome, 'match'); + assert.equal( + evaluateVersionProbe(completedProbe({ stdout, exitCode: 7 }), PIN).outcome, + 'command-failed', + ); + assert.equal( + evaluateVersionProbe(completedProbe({ stdout, exitCode: null, signal: 'SIGTERM' }), PIN) + .outcome, + 'command-failed', + ); + assert.equal( + evaluateVersionProbe( + completedProbe({ stdout, exitCode: null, signal: null, spawnErrorCode: 'ENOENT' }), + PIN, + ).outcome, + 'command-failed', + ); +}); + +test('the version match takes the last semver and tolerates notices', () => { + assert.equal(parseVersion(`banner 1.0.0\n${PIN}\nsee 2.9.9 notes`), '2.9.9'); + assert.equal(parseVersion('no version here'), null); +}); + +test('the workflow step runs the probe script instead of a masked pipeline', () => { + const workflow = parse(fs.readFileSync(workflowPath, 'utf8')); + const steps = workflow.jobs['differential-ios'].steps; + const step = steps.find( + (candidate) => candidate.name === 'Verify the Maestro CLI matches the oracle pin', + ); + assert.ok(step, 'the pin check step must exist'); + assert.match(step.run, /scripts\/maestro-conformance\/verify-cli-version\.mjs/); + // The two edits that made the November 9 run undiagnosable must not come back. + assert.doesNotMatch(step.run, /2>\/dev\/null/); + assert.doesNotMatch(step.run, /\|\| true/); +}); From 94f427458ddb05f56f8bcee4f387e03aad34a989 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Fri, 13 Nov 2026 11:11:05 +0100 Subject: [PATCH 2/2] fix(conformance): parse probe versions before diagnostic truncation --- scripts/maestro-conformance/README.md | 2 +- .../verify-cli-version.mjs | 70 ++++++++++++++++--- .../verify-cli-version.test.mjs | 45 +++++++++++- 3 files changed, 105 insertions(+), 12 deletions(-) diff --git a/scripts/maestro-conformance/README.md b/scripts/maestro-conformance/README.md index 27feb2b8ed..c9870e7bd2 100644 --- a/scripts/maestro-conformance/README.md +++ b/scripts/maestro-conformance/README.md @@ -149,7 +149,7 @@ The workflow builds and installs the fixture app, verifies its bundle id, pins the Maestro CLI to the same version as layers 1-2, and passes `--maestro` so the flow routes through the compat engine. The pin check runs [`verify-cli-version.mjs`](./verify-cli-version.mjs): it fails unless -`maestro --version` exits 0 and its newest semver equals the pin, and it keeps the +`maestro --version` exits 0 and its last semver in stdout equals the pin, and it keeps the probe's exit, stdout and stderr as bounded diagnostics so a failure names its cause (#4455). diff --git a/scripts/maestro-conformance/verify-cli-version.mjs b/scripts/maestro-conformance/verify-cli-version.mjs index fbd3327690..40b26b6e7f 100644 --- a/scripts/maestro-conformance/verify-cli-version.mjs +++ b/scripts/maestro-conformance/verify-cli-version.mjs @@ -12,8 +12,8 @@ // // A script rather than `run:` shell because retained evidence has to be bounded: a JVM startup // failure prints a stack trace, and a step log is not the place to dump one unbounded. Each stream -// keeps its last MAX_DIAGNOSTIC_BYTES, newest first, which is also the half the parse wants — the -// notice-tolerant match takes the LAST semver of stdout, as the `grep | tail -1` it replaced did. +// keeps its last MAX_DIAGNOSTIC_BYTES. Version matching consumes the full stream separately, so +// truncating diagnostics cannot change which version is checked. import { spawn } from 'node:child_process'; import fs from 'node:fs'; import path from 'node:path'; @@ -24,9 +24,6 @@ const HERE = path.dirname(fileURLToPath(import.meta.url)); /** Per stream: how much of the probe's output the run retains and prints. */ export const MAX_DIAGNOSTIC_BYTES = 8192; -/** The shape the step grepped for. Tolerates notices (e.g. the analytics banner) around it. */ -const SEMVER_PATTERN = /\d+\.\d+\.\d+/g; - /** Every verdict the probe can reach. Only `match` lets the differential proceed. */ const PROBE_OUTCOMES = Object.freeze({ match: 'match', @@ -45,9 +42,58 @@ export function readPinnedVersion() { return version; } -/** The last semver in `output`, or null when it carries none. */ +/** The last semver in `output`, with oversized versions represented by a bounded failure marker. */ export function parseVersion(output) { - return output.match(SEMVER_PATTERN)?.at(-1) ?? null; + const capture = createVersionCapture(); + capture.append(output); + return capture.read(); +} + +const VERSION_STATES = { + search: { digit: 'major' }, + major: { digit: 'major', dot: 'major-dot' }, + 'major-dot': { digit: 'minor' }, + minor: { digit: 'minor', dot: 'minor-dot' }, + 'minor-dot': { digit: 'patch' }, + patch: { digit: 'patch' }, +}; + +function versionCharacterKind(byte) { + if (byte >= 48 && byte <= 57) return 'digit'; + if (byte === 46) return 'dot'; + return 'other'; +} + +function createVersionCapture() { + let state = 'search'; + let candidate = ''; + let oversized = false; + let lastVersion = null; + const currentVersion = () => + oversized ? `(version exceeds ${MAX_DIAGNOSTIC_BYTES} bytes)` : candidate; + const retain = (character) => { + if (candidate.length < MAX_DIAGNOSTIC_BYTES) candidate += character; + else oversized = true; + }; + return { + append(output) { + for (const byte of Buffer.isBuffer(output) ? output : Buffer.from(output)) { + const nextState = VERSION_STATES[state][versionCharacterKind(byte)]; + if (nextState !== undefined) { + state = nextState; + retain(String.fromCharCode(byte)); + continue; + } + if (state === 'patch') lastVersion = currentVersion(); + state = 'search'; + candidate = ''; + oversized = false; + } + }, + read() { + return state === 'patch' ? currentVersion() : lastVersion; + }, + }; } /** @@ -59,7 +105,11 @@ function probeVersion({ command, args = ['--version'], env = process.env }) { const child = spawn(command, args, { env, stdio: ['ignore', 'pipe', 'pipe'] }); const stdout = boundedCapture(); const stderr = boundedCapture(); - child.stdout.on('data', (chunk) => stdout.append(chunk)); + const version = createVersionCapture(); + child.stdout.on('data', (chunk) => { + stdout.append(chunk); + version.append(chunk); + }); child.stderr.on('data', (chunk) => stderr.append(chunk)); let settled = false; @@ -72,6 +122,7 @@ function probeVersion({ command, args = ['--version'], env = process.env }) { spawnErrorCode: undefined, exitCode, signal, + version: version.read(), stdout: stdout.read(), stderr: stderr.read(), }); @@ -86,6 +137,7 @@ function probeVersion({ command, args = ['--version'], env = process.env }) { spawnErrorCode: error?.code, exitCode: null, signal: null, + version: version.read(), stdout: stdout.read(), stderr: stderr.read(), }); @@ -99,7 +151,7 @@ function probeVersion({ command, args = ['--version'], env = process.env }) { * whose behavior the differential cannot trust. */ export function evaluateVersionProbe(probe, expected) { - const version = parseVersion(probe.stdout.value); + const version = probe.version; const outcome = probeOutcome(probe, version, expected); return { outcome, version, report: reportFor(probe, { outcome, version, expected }) }; } diff --git a/scripts/maestro-conformance/verify-cli-version.test.mjs b/scripts/maestro-conformance/verify-cli-version.test.mjs index fdad11a018..2fc1075a3e 100644 --- a/scripts/maestro-conformance/verify-cli-version.test.mjs +++ b/scripts/maestro-conformance/verify-cli-version.test.mjs @@ -5,9 +5,9 @@ import assert from 'node:assert/strict'; import { spawnSync } from 'node:child_process'; import fs from 'node:fs'; -import os from 'node:os'; import path from 'node:path'; import test from 'node:test'; +import { mkdtempForTestSync } from '@agent-device/test-kit'; import { parse } from 'yaml'; import { MAX_DIAGNOSTIC_BYTES, @@ -22,7 +22,7 @@ const workflowPath = path.join(repoRoot, '.github/workflows/conformance-differen const PIN = readPinnedVersion(); function scratchDirectory(prefix) { - const root = fs.mkdtempSync(path.join(os.tmpdir(), prefix)); + const root = mkdtempForTestSync(prefix); test.after(() => fs.rmSync(root, { force: true, recursive: true })); return root; } @@ -64,6 +64,7 @@ function completedProbe(overrides) { spawnErrorCode: undefined, exitCode: 0, signal: null, + version: parseVersion(overrides.stdout?.value ?? ''), stdout: { value: '', bytes: 0, totalBytes: 0, droppedBytes: 0 }, stderr: { value: '', bytes: 0, totalBytes: 0, droppedBytes: 0 }, ...overrides, @@ -154,6 +155,46 @@ test('oversized output is retained only up to its bounded tail and names what it assert.ok(run.output.length < MAX_DIAGNOSTIC_BYTES * 2, 'the step log stays bounded'); }); +test('a truncated number cannot become the pinned version at the retained boundary', () => { + for (const prefix of ['1', '9.']) { + const run = runProbe({ + stdout: `${prefix}${PIN}${'x'.repeat(MAX_DIAGNOSTIC_BYTES - PIN.length)}`, + }); + assert.equal(run.status, 1, `truncated ${prefix}${PIN} must not match ${PIN}: ${run.output}`); + assert.match(run.stdout, /verdict=version-mismatch/); + } +}); + +test('a complete version after a truncated prefix still determines the verdict', () => { + for (const [version, expectedStatus] of [ + [PIN, 0], + ['1.2.3', 1], + ]) { + const run = runProbe({ stdout: `${'9'.repeat(MAX_DIAGNOSTIC_BYTES)} ${version}\n` }); + assert.equal(run.status, expectedStatus, run.output); + assert.ok(run.stdout.includes(`actual=${version}\n`), run.output); + } +}); + +test('a version before oversized trailing notices remains authoritative', () => { + for (const [version, expectedStatus] of [ + [PIN, 0], + ['1.2.3', 1], + ]) { + const run = runProbe({ stdout: `${version}\n${'notice '.repeat(MAX_DIAGNOSTIC_BYTES)}\n` }); + assert.equal(run.status, expectedStatus, run.output); + assert.ok(run.stdout.includes(`actual=${version}\n`), run.output); + } +}); + +test('an oversized numeric version stays bounded and cannot match the pin', () => { + const run = runProbe({ stdout: `${'1'.repeat(MAX_DIAGNOSTIC_BYTES * 3)}.${PIN}\n` }); + assert.equal(run.status, 1, run.output); + assert.match(run.stdout, /verdict=version-mismatch/); + assert.match(run.stdout, /actual=\(version exceeds 8192 bytes\)/); + assert.ok(run.output.length < MAX_DIAGNOSTIC_BYTES * 2); +}); + test('a probe that did not complete cleanly is never accepted on the strength of its stdout', () => { const stdout = streamOf(`${PIN}\n`); assert.equal(evaluateVersionProbe(completedProbe({ stdout }), PIN).outcome, 'match');