Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -8,3 +8,5 @@ experiments/**/output/
app-icon.png
# Tauri regenerates these schemas on every build
examples/**/src-tauri/gen/
# the CLI's default test-root directory (release-qa designate/status use it when --root is not given)
.release-qa/
27 changes: 26 additions & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -11,12 +11,37 @@ The first targets are Tauri applications on Windows and Linux, with Dot X as the

## Status

Stage 0 (proving the assumptions) is under way; the tool itself is not built yet. Results so far:
Stage 0 (proving the assumptions) is complete. The runner (environment checks, scenario execution, the durable run
journal) and a first slice of the CLI (`doctor`, `designate`, `status`) exist; running an actual scenario from the
CLI (`run`, `resume`), the Tauri driver, the dashboard and the GitHub integration do not yet.

- [Native automation](docs/decisions/native-automation.md): unchanged packaged Tauri apps can be driven on Windows and Ubuntu. The Dot X feasibility check is still open.
- [GitHub merge gate](docs/decisions/github-gate.md): a no-service required check works, with documented design changes and unproven items.
- [Tool layout and defaults](docs/decisions/tool-layout.md): runtime, package manager, baselines and repository structure.

## CLI

Run it straight from a checkout; no build step (see [tool layout](docs/decisions/tool-layout.md)). `designate` and
`status` work as they stand, against any directory:

```sh
node packages/qa/src/cli/main.ts designate [--root <path>] [--json] # marks a directory safe to install and delete into
node packages/qa/src/cli/main.ts status [--root <path>] [--json] # reports what a designated root holds, dirty or clean
```

`doctor` needs a `qa/project.json` from a project that has one (this repository does not ship a sample yet — that
lands with the CLI's `run`/`resume` commands):

```sh
node packages/qa/src/cli/main.ts doctor --project path/to/qa/project.json --profile windows [--json]
```

`--root` defaults to `.release-qa` under the current directory (gitignored) when not given. Every command prints to
stdout on success and to stderr on failure; `--json` switches both to one line of machine-readable JSON. Exit codes:
`0` passed/ready, `1` a scenario the candidate failed (not reachable yet — no command runs a scenario), `2` this
machine does not meet a requested profile, `3` anything else that stopped the command (bad usage, an unreadable or
invalid project file, a profile the project does not declare).

## Development

Requires Node.js 22.18 or newer (the CLI needs no build step because Node strips types directly from that version on).
Expand Down
107 changes: 107 additions & 0 deletions packages/qa/src/cli/args.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,107 @@
/**
* Parses `process.argv.slice(2)`. Never throws: a malformed invocation is a `{ ok: false }` result the caller turns
* into an exit code, not an exception that would print a stack trace instead of a usable message. `--json` is
* decided by a single scan of the whole invocation, independent of where it appears and of everything else that
* might be wrong with it, so the caller can render even a parse failure as JSON when that is what was asked for.
*/

export interface DoctorCommand {
name: 'doctor';
project: string;
profile: string;
json: boolean;
}

export interface DesignateCommand {
name: 'designate';
/** Absent means the caller applies its own default; parsing does not know or guess a working directory. */
root: string | undefined;
json: boolean;
}

export interface StatusCommand {
name: 'status';
root: string | undefined;
json: boolean;
}

export type Command = DoctorCommand | DesignateCommand | StatusCommand;

export type ParsedArgs = { ok: true; command: Command } | { ok: false; error: string; json: boolean };

interface Spec {
/** Flags that must be given a value. */
required: readonly string[];
/** Flags that may be given a value but need not be. */
optional: readonly string[];
}

const SPECS: Record<Command['name'], Spec> = {
doctor: { required: ['project', 'profile'], optional: [] },
designate: { required: [], optional: ['root'] },
status: { required: [], optional: ['root'] },
};

const COMMAND_NAMES = Object.keys(SPECS) as Command['name'][];

export function parseArgs(argv: readonly string[]): ParsedArgs {
const json = countJson(argv) === 1;
const [name, ...rest] = argv;
if (name === undefined) return fail(`expected a command: ${COMMAND_NAMES.join(', ')}`, json);
if (!isCommandName(name)) return fail(`unknown command "${name}"; expected one of ${COMMAND_NAMES.join(', ')}`, json);
if (countJson(argv) > 1) return fail('--json was given more than once', true);

const flags = parseFlags(rest, SPECS[name]);
if (!flags.ok) return fail(flags.error, json);

switch (name) {
case 'doctor':
return { ok: true, command: { name, project: flags.values.project as string, profile: flags.values.profile as string, json } };
case 'designate':
case 'status':
return { ok: true, command: { name, root: flags.values.root as string | undefined, json } };
}
}

const countJson = (argv: readonly string[]): number => argv.filter((token) => token === '--json').length;

function isCommandName(value: string): value is Command['name'] {
return (COMMAND_NAMES as string[]).includes(value);
}

type FlagResult = { ok: true; values: Record<string, string | undefined> } | { ok: false; error: string };

/** `--json` is handled by the caller; every other token is a `--flag value` pair or an unexpected extra. */
function parseFlags(args: readonly string[], spec: Spec): FlagResult {
const known = new Set([...spec.required, ...spec.optional]);
const values: Record<string, string> = {};
const positional: string[] = [];

for (let i = 0; i < args.length; i += 1) {
const token = args[i];
if (token === undefined || token === '--json') continue;
if (!token.startsWith('--')) {
positional.push(token);
continue;
}
const flag = token.slice(2);
if (!known.has(flag)) return failFlags(`unknown flag "--${flag}"`);
if (Object.hasOwn(values, flag)) return failFlags(`--${flag} was given more than once`);
const value = args[i + 1];
if (value === undefined || value.startsWith('--')) return failFlags(`--${flag} needs a value`);
values[flag] = value;
i += 1;
}

if (positional.length > 0) return failFlags(`unexpected argument "${positional[0]}"`);
for (const flag of spec.required) if (values[flag] === undefined) return failFlags(`--${flag} is required`);
return { ok: true, values };
}

function failFlags(error: string): { ok: false; error: string } {
return { ok: false, error };
}

function fail(error: string, json: boolean): { ok: false; error: string; json: boolean } {
return { ok: false, error, json };
}
36 changes: 36 additions & 0 deletions packages/qa/src/cli/doctor.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,36 @@
import { inspectEnvironment, type EnvironmentProbes } from '../runner/environment.ts';
import type { DisplayKind } from '../runner/display.ts';
import type { Project } from '../model/project.ts';
import type { MeasuredEnvironment } from '../model/result.ts';

export interface DoctorReport {
/** Whether this machine is what the requested profile expects; not whether any particular suite can run. */
ok: boolean;
profile: string;
machine: MeasuredEnvironment;
display: { kind: DisplayKind; detail: string };
mismatch?: string;
}

export type DoctorResult = { ok: true; report: DoctorReport } | { ok: false; error: string };

/** Measures this machine against a profile the project declares. Never throws: an unknown profile is a result, not an exception. */
export async function runDoctor(project: Project, profileId: string, probes?: EnvironmentProbes): Promise<DoctorResult> {
const profile = project.profiles.find((p) => p.id === profileId);
if (profile === undefined) {
const known = project.profiles.map((p) => p.id).join(', ') || '(none declared)';
return { ok: false, error: `profile "${profileId}" is not defined by this project; known profiles: ${known}` };
}

const inspected = await inspectEnvironment(profile, probes);
return {
ok: true,
report: {
ok: inspected.profileMismatch === undefined,
profile: profileId,
machine: inspected.environment,
display: inspected.display,
...(inspected.profileMismatch === undefined ? {} : { mismatch: inspected.profileMismatch }),
},
};
}
50 changes: 50 additions & 0 deletions packages/qa/src/cli/environment-commands.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,50 @@
import { checkTestRoot, designateTestRoot, readDirty, readLedger, type OwnedResource, type TestRootCheck } from '../runner/resources.ts';

const message = (error: unknown): string => (error instanceof Error ? error.message : String(error));

/** `options.home` is a test seam only: production callers never pass it, so the real home directory is always used. */
export interface EnvironmentCommandOptions {
home?: string;
}

export type DesignateResult = { ok: true; root: string } | { ok: false; error: string };

/** Marks a directory as somewhere the runner may install, launch and delete. Never throws. */
export async function runDesignate(root: string, options: EnvironmentCommandOptions = {}): Promise<DesignateResult> {
try {
await designateTestRoot(root, options);
return { ok: true, root };
} catch (error) {
return { ok: false, error: message(error) };
}
}

export interface StatusReport {
root: string;
designated: boolean;
/** Present only when `designated` is false: which check failed. */
reason?: Exclude<TestRootCheck, { ok: true }>['reason'];
/** Present only when the environment was left dirty by an earlier run. */
dirty?: string;
owned: OwnedResource[];
}

export type StatusResult = { ok: true; report: StatusReport } | { ok: false; error: string };

/**
* Reports what this root is and holds, without changing anything. An undesignated or malformed root is a normal
* report (`designated: false`), not an error; only a root this cannot read at all is an error.
*/
export async function runStatus(root: string, options: EnvironmentCommandOptions = {}): Promise<StatusResult> {
try {
const check = await checkTestRoot(root, options);
if (!check.ok) return { ok: true, report: { root, designated: false, reason: check.reason, owned: [] } };

const [dirty, owned] = await Promise.all([readDirty(check.root), readLedger(check.root)]);
return { ok: true, report: { root: check.root, designated: true, owned, ...(dirty === undefined ? {} : { dirty }) } };
} catch (error) {
// Covers checkTestRoot too: it can throw if the root is removed between its own existence check and resolving
// the real path, a narrow race this function's "never throws" promise still needs to hold against.
return { ok: false, error: `could not read the state of ${root}: ${message(error)}` };
}
}
120 changes: 120 additions & 0 deletions packages/qa/src/cli/main.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,120 @@
import { join } from 'node:path';
import { pathToFileURL } from 'node:url';
import { parseArgs } from './args.ts';
import { runDoctor, type DoctorReport } from './doctor.ts';
import { runDesignate, runStatus, type StatusReport } from './environment-commands.ts';
import { loadProject } from './project.ts';
import type { ValidationIssue } from '../model/validate.ts';

/**
* `0` passed. `1` a scenario the candidate failed (not yet reachable: no command runs a scenario in this build).
* `2` a missing prerequisite the tester, not the tool, must resolve (e.g. this machine does not match a profile).
* `3` everything else that stops the command: bad usage, a file that cannot be read, an unknown profile.
*/
export const EXIT = { ok: 0, scenarioFailure: 1, missingPrerequisite: 2, infrastructure: 3 } as const;

export interface Io {
log(line: string): void;
error(line: string): void;
}

const defaultIo: Io = { log: (line) => console.log(line), error: (line) => console.error(line) };

/** Runs one CLI invocation and returns the process exit code; never throws and never touches `process` itself. */
export async function main(argv: readonly string[], io: Io = defaultIo, cwd: () => string = () => process.cwd()): Promise<number> {
const parsed = parseArgs(argv);
if (!parsed.ok) {
reportError(io, parsed.json, parsed.error);
return EXIT.infrastructure;
}
const { command } = parsed;

switch (command.name) {
case 'doctor': {
const loaded = await loadProject(command.project);
if (!loaded.ok) {
reportError(io, command.json, loaded.error, loaded.issues);
return EXIT.infrastructure;
}
const result = await runDoctor(loaded.project, command.profile);
if (!result.ok) {
reportError(io, command.json, result.error);
return EXIT.infrastructure;
}
printDoctor(io, command.json, result.report);
return result.report.ok ? EXIT.ok : EXIT.missingPrerequisite;
}

case 'designate': {
const root = command.root ?? defaultRoot(cwd);
const result = await runDesignate(root);
if (!result.ok) {
reportError(io, command.json, result.error);
return EXIT.infrastructure;
}
io.log(command.json ? JSON.stringify(result) : `designated ${result.root}`);
return EXIT.ok;
}

case 'status': {
const root = command.root ?? defaultRoot(cwd);
const result = await runStatus(root);
if (!result.ok) {
reportError(io, command.json, result.error);
return EXIT.infrastructure;
}
// What status finds (undesignated, dirty) is a fact it reports, never a failure of the status command itself.
printStatus(io, command.json, result.report);
return EXIT.ok;
}
}
}

const defaultRoot = (cwd: () => string): string => join(cwd(), '.release-qa');

/** `issues` is always present in JSON output, empty when there are none, so a consumer can key on it unconditionally. */
function reportError(io: Io, json: boolean, error: string, issues?: readonly ValidationIssue[]): void {
io.error(json ? JSON.stringify({ ok: false, error, issues: issues ?? [] }) : error);
}

function printDoctor(io: Io, json: boolean, report: DoctorReport): void {
if (json) {
io.log(JSON.stringify(report));
return;
}
io.log(
[
`profile: ${report.profile}`,
`ready: ${report.ok}`,
`os: ${report.machine.os} ${report.machine.osVersion}`,
`arch: ${report.machine.arch}`,
`capabilities: ${report.machine.capabilities.join(', ') || '(none)'}`,
`display: ${report.display.kind} (${report.display.detail})`,
...(report.mismatch === undefined ? [] : [`mismatch: ${report.mismatch}`]),
].join('\n'),
);
}

function printStatus(io: Io, json: boolean, report: StatusReport): void {
if (json) {
io.log(JSON.stringify(report));
return;
}
io.log(
[
`root: ${report.root}`,
`designated: ${report.designated}`,
...(report.reason === undefined ? [] : [`reason: ${report.reason}`]),
...(report.dirty === undefined ? [] : [`dirty: ${report.dirty}`]),
`owned: ${report.owned.length} resource(s)`,
].join('\n'),
);
}

// Runs only when this file is the process's entry point (`node packages/qa/src/cli/main.ts ...`), never when a test
// imports it as a module.
if (process.argv[1] !== undefined && pathToFileURL(process.argv[1]).href === import.meta.url) {
main(process.argv.slice(2)).then((code) => {
process.exitCode = code;
});
}
28 changes: 28 additions & 0 deletions packages/qa/src/cli/project.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,28 @@
import { readFile } from 'node:fs/promises';
import { parseProject, type Project } from '../model/project.ts';
import type { ValidationIssue } from '../model/validate.ts';

export type LoadedProject = { ok: true; project: Project; path: string } | { ok: false; error: string; issues?: readonly ValidationIssue[] };

const message = (error: unknown): string => (error instanceof Error ? error.message : String(error));

/** Reads and validates a `qa/project.json`. Never throws: every way the file can be wrong becomes a `{ ok: false }`. */
export async function loadProject(path: string): Promise<LoadedProject> {
let text: string;
try {
text = await readFile(path, 'utf8');
} catch (error) {
return { ok: false, error: `could not read ${path}: ${message(error)}` };
}

let parsed: unknown;
try {
parsed = JSON.parse(text);
} catch (error) {
return { ok: false, error: `${path} is not valid JSON: ${message(error)}` };
}

const result = parseProject(parsed);
if (!result.ok) return { ok: false, error: `${path}: ${result.error.message}`, issues: result.error.issues };
return { ok: true, project: result.value, path };
}
Loading
Loading