diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md new file mode 100644 index 0000000..4f97ab6 --- /dev/null +++ b/.github/pull_request_template.md @@ -0,0 +1,22 @@ +# Pull request + +## What changed + + + +## How it was verified + + + +## Checklist + +Tick each item, or replace it with one line saying why it does not apply. These +mirror the gates in `pr.yml` and the expectations in `CONTRIBUTING.md`. + +- [ ] Scenario catalog (`scenarios/catalog.yml`) updated, or this change adds or removes no scenarios. +- [ ] Matrix regenerated with `python3 tools/check-scenarios.py --write-matrix`, and the README section 5 embed matches it. +- [ ] Every command I changed in the documentation was executed, not just edited. +- [ ] Documentation updated for this change type, per the lifecycle table in [`docs/README.md`](../docs/README.md). +- [ ] Workflow changes under `.github/workflows/` are mirrored in `docs/ci-workflows.md` in this pull request. +- [ ] No credentials or tokens, private endpoints, unredacted traces or screenshots, or real user data, per the secrets and captures rules in `CONTRIBUTING.md`. +- [ ] ADR added under `docs/adr/`, or this decision is not durable enough to need one. diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index 14f8a59..ae54b53 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -66,6 +66,43 @@ jobs: - name: Lint Markdown run: git ls-files -z '*.md' | xargs -0 npx --yes markdownlint-cli2@0.23.0 + - name: Check Markdown links + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: >- + git ls-files -z '*.md' | xargs -0 + docker run --rm + -v "$PWD:/repo:ro" + -w /repo + -e GITHUB_TOKEN + lycheeverse/lychee:0.24.2@sha256:e2d19e57cf6ab037026f20b8e449a1f30d9d7f81eef4194763aab2eab20bd28d + --no-progress + --exclude-loopback + --max-retries 3 + --retry-wait-time 2 + --timeout 20 + + - name: Validate README fast start + run: python3 tools/check-fast-start.py + + - name: Require CI guide updates with workflow changes + if: >- + github.event_name == 'pull_request' && + !contains(github.event.pull_request.title, '[ci-guide-exempt]') + env: + BASE_SHA: ${{ github.event.pull_request.base.sha }} + HEAD_SHA: ${{ github.event.pull_request.head.sha }} + run: | + changed="$(git diff --name-only "$BASE_SHA"..."$HEAD_SHA")" + if printf '%s\n' "$changed" | grep -q '^\.github/workflows/' && + ! printf '%s\n' "$changed" | grep -qx 'docs/ci-workflows.md'; then + echo "::error file=docs/ci-workflows.md::This pull request changes .github/workflows/ but not docs/ci-workflows.md" + echo "docs/ci-workflows.md mirrors the workflows at job and step granularity, and nothing else keeps it honest." + echo "Update it in this pull request. If the change has no semantic effect on the guide, add [ci-guide-exempt] to the pull request title." + exit 1 + fi + echo "CI guide sync ok" + scenario-catalog: name: "Scenario catalog · drift check" runs-on: ubuntu-24.04 diff --git a/.lycheeignore b/.lycheeignore new file mode 100644 index 0000000..bcff584 --- /dev/null +++ b/.lycheeignore @@ -0,0 +1,17 @@ +# Exclusions for the link check in the repo-hygiene job of +# .github/workflows/pr.yml. One regular expression per line. +# +# Only genuinely unverifiable or volatile targets belong here. Everything else, +# including every relative link between documents, is checked strictly: a +# broken relative link must fail the pull request. + +# The public demo app under test. Its availability is upstream and outside this +# repository's control, so an outage there must never fail an unrelated pull +# request. The pinned container image, not this host, is what CI actually runs +# against. +https://the-internet\.herokuapp\.com + +# Workflow status badges. The badge endpoint answers per request and is prone to +# rate limiting and redirects, which would make the gate flaky. The Actions +# links rendered beside each badge point at the same workflows and are checked. +https://github\.com/jsugg/the-internet-tests/actions/workflows/[^/]+/badge\.svg diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..dc1df08 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,250 @@ +# Contributing + +This is a teaching repository: the tests, the catalog, and the CI wiring are the +product. A change is finished when someone reading the documentation can +reproduce it, not when it passes locally. Read [`README.md`](README.md) for what +the repo is and is not before proposing new scope. + +## Repository setup + +Start the demo app under test from the repository root. Every stack runs against +it on `http://localhost:7080`: + +```bash +docker compose -f docker/compose.yml up -d website +``` + +Set up only the stacks you are changing. Each stack is self-contained and runs +from its own directory: + +```bash +cd stacks/ts-playwright +npm ci +npx playwright install --with-deps chromium +``` + +```bash +cd stacks/python-playwright +python3 -m venv .venv +source .venv/bin/activate +python -m pip install --upgrade pip +python -m pip install -e ".[dev]" +python -m playwright install --with-deps chromium +``` + +The Java stack needs no install step; Maven resolves dependencies on first run, +and Selenium Manager provisions the browser drivers. + +Stop the app from the repository root when you are done: + +```bash +docker compose -f docker/compose.yml down +``` + +Each stack README is the authority on running that stack: +[Java](stacks/java-selenium-testng/README.md), +[TypeScript](stacks/ts-playwright/README.md), +[Python](stacks/python-playwright/README.md). + +## Fast validation loop + +Run these two from the repository root before every push. They are the same +checks the `repo-hygiene` and `scenario-catalog` PR jobs run, and they need no +browsers and no app: + +```bash +python3 tools/check-scenarios.py +git ls-files -z '*.md' | xargs -0 npx --yes markdownlint-cli2@0.23.0 +``` + +Then run the static checks for the stacks you touched: + +```bash +mvn -B -f stacks/java-selenium-testng/pom.xml clean test-compile +``` + +```bash +cd stacks/ts-playwright +npm run typecheck +npm run lint +npm run format:check +``` + +```bash +cd stacks/python-playwright +ruff check . +ruff format --check . +mypy src tests +``` + +If you changed a workflow, lint it with the same pinned image CI uses, from the +repository root: + +```bash +docker run --rm -v "$PWD:/repo:ro" -w /repo rhysd/actionlint:1.7.12 -color +``` + +## Adding or changing a scenario + +The catalog is the canonical list of scenarios; the tests reference it, and +`tools/check-scenarios.py` reconciles the two in both directions. A scenario that +is not in the catalog is not covered, no matter how many tests exist. Work in +this order: + +1. Add or edit the row in [`scenarios/catalog.yml`](scenarios/catalog.yml). Give + it a unique ID, a priority, a title, and a `coverage:` entry per stack. +2. Add the tests, referencing the scenario ID. Java requires the ID in the + TestNG `testName="…"` attribute; TypeScript and Python carry it in the test + title or marker. +3. Set `coverage:` to `true` only for the stacks that actually have a test. The + checker fails in both directions: a catalog row marked covered with no test, + and a test whose ID is missing or not marked covered. +4. Regenerate the matrix and sync the README embed from the repository root: + + ```bash + python3 tools/check-scenarios.py --write-matrix + ``` + +Never hand-edit [`docs/scenario-matrix.md`](docs/scenario-matrix.md); it is +generated, and the `scenario-catalog` job fails on any drift. + +## Commit conventions + +Commits are linted by commitlint in CI against +[`commitlint.config.cjs`](commitlint.config.cjs), which is the authority. The +rules: + +- Conventional commits: `type(scope): subject`. +- Type is one of `feat`, `fix`, `docs`, `ci`, `build`, `refactor`, `test`, + `chore`, `perf`, lower-case. +- Scope is optional, lower-case, and if present must be one of `java`, `ts`, + `py`, `catalog`, `docker`, `actions`, `deps`, `readme`. There is deliberately + no `docs` or `ci` scope: use a scopeless `docs: …` for governance docs and + `ci(actions): …` for workflow changes. +- The header is at most 72 characters, and the subject takes no trailing period. +- Body and footer lines are at most 100 characters. + +Explain why in the body, not what; the diff already shows what. Do not add +attribution trailers, and do not name tooling vendors or assistants in branch +names, commit messages, or pull requests. + +## Pull requests + +Fill in the [pull request template](.github/pull_request_template.md); it is the +change checklist, not a formality. Squash-merge is the norm, so the PR title +becomes the squashed subject and must satisfy the same commitlint rules. + +Every pull request runs the always-on fast gate (`pr.yml`): repository hygiene, +scenario-catalog drift, and a compile-plus-smoke slice per stack. Regression +workflows are path-filtered and only run when your change touches their stack. +See [`docs/ci-workflows.md`](docs/ci-workflows.md) for the full trigger fan-out. + +## Secrets and captures + +This is a teaching repository with no deployed service, no users, and no data of +its own, so the realistic risk here is not an exploit. It is committing something +that should never have been public, into a repository that is public on purpose. + +- **Never commit a credential or token.** Not an API key, a provider token, a + session cookie, or a private key — including expired and throwaway ones. Secret + scanning and push protection are enabled on this repository, so a detected + secret blocks the push. Treat that as the backstop, not the plan. +- **Point the stacks at the demo app, not at a real system.** The moment a stack + runs against something real, its traces and screenshots inherit that system's + secrets. Use the pinned container on `http://localhost:7080`. +- **Redact captures before attaching them.** A Playwright trace records request + headers in full. Open a trace, screenshot, or log before you put it in a pull + request. +- **Keep `artifacts/` out of Git.** It is git-ignored staging for CI upload, and + CI artifacts on a public repository are downloadable by anyone. +- **Keep remote-grid credentials outside the repository**, injected only through + a private runner or a maintainer-approved workflow secret, as the + [Java stack README](stacks/java-selenium-testng/README.md) says. No + checked-in workflow needs one. + +The demo app's `tomsmith` / `SuperSecretPassword!` login is a published fixture +of a public teaching app, not a secret. It belongs in the tests. + +## Documentation compliance + +A change is documentation-compliant when all five hold: + +1. A new contributor can reproduce the affected behavior from the documented + commands, without reading the diff. +2. Canonical documentation agrees with the executable configuration it + describes. +3. Cross-stack design changes are recorded, so the reasoning outlives the PR. +4. Documentation responsibility for the change is identifiable. +5. No required documentation validation fails. + +## Documentation expectations + +Update the documentation in the same pull request as the change. What to update +depends on what you changed. [`docs/README.md`](docs/README.md) carries the full +lifecycle table — every document's audience, update trigger, and review cadence — +plus the authority order to apply when two sources disagree, and the +documentation conventions to follow when writing: terminology, headings, dates, +shell assumptions, source references, diagrams, and generated files. + +| Change | Update | +| --- | --- | +| A workflow under `.github/workflows/` | [`docs/ci-workflows.md`](docs/ci-workflows.md), in the same pull request. CI enforces this. | +| A scenario or the catalog | Regenerate `docs/scenario-matrix.md` and sync the README §5 embed. | +| A stack's commands, runtime, or reports | That stack's README, and the root README stack matrix if the headline command changed. | +| Retry, tagging, quarantine, or artifact policy | [`docs/flakiness-guide.md`](docs/flakiness-guide.md). | +| A durable cross-stack, CI, or catalog decision | A new ADR under `docs/adr/`. | +| Scope, quick start, or supported stacks | [`README.md`](README.md). | + +## Waivers + +A waiver is a decision not to meet a requirement, written down so it stays a +decision instead of decaying into an oversight. The difference matters: an +undocumented gap looks identical to a mistake, and the next reader cannot tell +whether it was considered or forgotten. + +To record one, copy +[`docs/templates/waiver-template.md`](docs/templates/waiver-template.md), fill +every field, and add it to the register below. A waiver carries the requirement, +its scope, the reason, the owner, the approver, the revisit trigger, and a +tracking issue. + +Two rules keep this from becoming a way to opt out of anything inconvenient: + +- **The reason must say why the requirement is wrong *here*, not that meeting it + is expensive.** Cost alone is a backlog item, not a waiver. +- **The revisit trigger must be an observable event, not a date.** "First + sustained external contributor" is something anyone can check; "later" and + "Q3" are not. + +A waiver is not permanent, and it is not an argument. When its trigger fires, the +requirement is back on the table. + +### Recorded waivers + +Requirements this repository deliberately does not meet, with the condition that +would reopen each one. Both exist because this is a single-maintainer repository; +neither is an oversight. + +#### W-001 — `CODEOWNERS` omitted + +| Field | Value | +| --- | --- | +| Requirement | Code-owner review via a `CODEOWNERS` file | +| Scope | Whole repository | +| Reason | The only owner entry possible today is `* @jsugg`. Code-owner review would then demand an approval that the sole maintainer cannot give on their own pull requests, deadlocking every change. | +| Owner | @jsugg | +| Approver | @jsugg | +| Revisit trigger | First sustained external contributor | +| Tracking issue | None; repository issues are disabled, so the trigger above is the record | + +#### W-002 — Reviewer-assignment machinery omitted + +| Field | Value | +| --- | --- | +| Requirement | Owner maps, required reviewers, and automatic reviewer assignment | +| Scope | Whole repository | +| Reason | There is one maintainer and no reviewer pool to route to. Required reviewers would block every pull request for the same reason as W-001, and an owner map would encode a single name already visible in the repository metadata. | +| Owner | @jsugg | +| Approver | @jsugg | +| Revisit trigger | First sustained external contributor | +| Tracking issue | None; repository issues are disabled, so the trigger above is the record | diff --git a/README.md b/README.md index ed56999..3b12433 100644 --- a/README.md +++ b/README.md @@ -54,7 +54,9 @@ Then use the stack READMEs and guides for deeper runs: ## 5. Scenario matrix -Embedded from [`docs/scenario-matrix.md`](docs/scenario-matrix.md), generated by `tools/check-scenarios.py --write-matrix`. +Embedded from [`docs/scenario-matrix.md`](docs/scenario-matrix.md), generated by `tools/check-scenarios.py --write-matrix`. This copy is checked against the catalog in CI, so edit the catalog and regenerate rather than editing the table below. + +Legend: ✅ = covered by that stack · — = not covered. | ID | Priority | Scenario | Java | TS | Python | | --- | --- | --- | --- | --- | --- | @@ -137,3 +139,8 @@ CI stages report files under `artifacts////` before upload | TypeScript full gate | [![TypeScript](https://github.com/jsugg/the-internet-tests/actions/workflows/ts.yml/badge.svg?branch=master)](https://github.com/jsugg/the-internet-tests/actions/workflows/ts.yml?query=branch%3Amaster) | | Python full gate | [![Python](https://github.com/jsugg/the-internet-tests/actions/workflows/python.yml/badge.svg?branch=master)](https://github.com/jsugg/the-internet-tests/actions/workflows/python.yml?query=branch%3Amaster) | | Nightly matrix | [![Nightly](https://github.com/jsugg/the-internet-tests/actions/workflows/nightly.yml/badge.svg?branch=master)](https://github.com/jsugg/the-internet-tests/actions/workflows/nightly.yml?query=branch%3Amaster) | + +## 9. Contributing + +- [Contributing guide](CONTRIBUTING.md) — setup, the fast validation loop, the scenario-catalog workflow, commit and pull-request conventions, and the recorded waivers. +- [Documentation index](docs/README.md) — every document, its audience and update trigger, and which source of truth wins when two disagree. diff --git a/docs/README.md b/docs/README.md new file mode 100644 index 0000000..692bc31 --- /dev/null +++ b/docs/README.md @@ -0,0 +1,138 @@ +# Documentation index + +Every document in this repository, who it is for, whether it is authored or +generated, and what obliges someone to update it. If you are looking for where +to start reading rather than what exists, use +[`learning-paths.md`](learning-paths.md), which routes five audiences. + +## Documents + +Review cadence is a backstop, not a workflow: the update trigger is what +normally forces a change. Use Git history for versioning — no document carries a +version number or a "last updated" line, because both rot faster than the prose +they annotate. + +| Document | Audience | Status | Update trigger | Review cadence | +| --- | --- | --- | --- | --- | +| [`../README.md`](../README.md) | Everyone; first contact with the repo | Authored | Stack, quick-start, support, or scope change | Every 6 months | +| [`../CONTRIBUTING.md`](../CONTRIBUTING.md) | Contributors | Authored | Setup, validation loop, commit or pull-request convention, or waiver change | Every 6 months | +| [`../.github/pull_request_template.md`](../.github/pull_request_template.md) | Contributors | Authored | Any change to the gates or documentation expectations it lists | Every 6 months | +| [`README.md`](README.md) (this file) | Anyone navigating the documentation | Authored | Any document added, removed, or repurposed | Every 6 months | +| [`architecture.md`](architecture.md) | Contributors; reviewers | Authored | New stack, catalog model change, or CI topology change | Annually | +| [`ci-workflows.md`](ci-workflows.md) | Contributors; reviewers | Authored | Any workflow semantic change, in the same pull request | Every 3 months | +| [`flakiness-guide.md`](flakiness-guide.md) | Contributors touching tags or retries | Authored | Retry, tagging, quarantine, or artifact policy change | Every 6 months | +| [`framework-comparison.md`](framework-comparison.md) | Learners; reviewers choosing a stack | Authored | A stack is added or dropped, or a dated external claim goes stale | Every 6 months | +| [`interview-guide.md`](interview-guide.md) | Interview candidates | Authored | Scenario coverage or emphasis change | Every 6 months | +| [`learning-paths.md`](learning-paths.md) | Learners, by audience | Authored | A document is added or removed, or audience routing changes | Every 6 months | +| [`anti-patterns.md`](anti-patterns.md) | Legacy Selenium maintainers | Authored | A defect is rescued, or a referenced symbol moves or is renamed | Every 6 months | +| [`scenario-matrix.md`](scenario-matrix.md) | Everyone | **Generated** | Never edit. Regenerate with `python3 tools/check-scenarios.py --write-matrix` whenever `scenarios/catalog.yml` changes | None; CI fails on drift | +| [`runbooks/ci-failure-triage.md`](runbooks/ci-failure-triage.md) | Anyone facing a red check | Authored | A job, gate, or reproduction command changes | Every 6 months | +| [`adr/README.md`](adr/README.md) | Contributors; reviewers | Authored | An ADR is added or superseded | Annually | +| [`adr/0000-template.md`](adr/0000-template.md) | ADR authors | Authored | The ADR structure changes | Annually | +| [`adr/0001-shared-scenario-catalog.md`](adr/0001-shared-scenario-catalog.md) | Contributors; reviewers | Authored | Superseding decision only | None | +| [`adr/0002-ui-http-scenario-split.md`](adr/0002-ui-http-scenario-split.md) | Contributors; reviewers | Authored | Superseding decision only | None | +| [`adr/0003-three-framework-tracks.md`](adr/0003-three-framework-tracks.md) | Contributors; reviewers | Authored | Superseding decision only | None | +| [`adr/0004-flaky-demo-not-ci-isolation.md`](adr/0004-flaky-demo-not-ci-isolation.md) | Contributors; reviewers | Authored | Superseding decision only | None | +| [`adr/0005-independent-workflow-topology.md`](adr/0005-independent-workflow-topology.md) | Contributors; reviewers | Authored | Superseding decision only | None | +| [`templates/stack-readme-template.md`](templates/stack-readme-template.md) | Anyone adding a stack | Authored | The stack-README contract changes | Annually | +| [`templates/runbook-template.md`](templates/runbook-template.md) | Anyone writing a runbook | Authored | The runbook shape changes | Annually | +| [`templates/waiver-template.md`](templates/waiver-template.md) | Anyone recording a waiver | Authored | The waiver fields change | Annually | +| [`../stacks/java-selenium-testng/README.md`](../stacks/java-selenium-testng/README.md) | Java Selenium/TestNG track | Authored | Runtime, dependency, command, project, or report change | Every 6 months | +| [`../stacks/ts-playwright/README.md`](../stacks/ts-playwright/README.md) | TypeScript Playwright track | Authored | Runtime, dependency, command, project, or report change | Every 6 months | +| [`../stacks/python-playwright/README.md`](../stacks/python-playwright/README.md) | Python Playwright track | Authored | Runtime, dependency, command, project, or report change | Every 6 months | + +## Which source wins + +Documentation describes things that are themselves executable. When a document +and the thing it describes disagree, the executable artifact is right and the +document is a bug. Resolve conflicts in this order, highest authority first: + +1. **[`../.github/workflows/`](../.github/workflows/)** — what CI actually runs. Canonical for triggers, jobs, steps, matrices, and pins. +2. **[`../scenarios/catalog.yml`](../scenarios/catalog.yml)** — canonical for which scenarios exist, their IDs, priorities, and which stacks claim coverage. +3. **[`scenario-matrix.md`](scenario-matrix.md)** — generated from the catalog. Authoritative over prose, but never over the catalog: if they disagree, the matrix was not regenerated. +4. **Stack READMEs** — canonical for how to run one stack. +5. **Guides** — this directory. Explain why, and defer to all of the above for what. + +The root [`README.md`](../README.md) defers to every source above it for +specifics. It is a map, not an authority: its stack matrix and its embedded copy +of the scenario matrix are conveniences that must agree with the sources they +summarize. + +Two documents restate a source of truth and can therefore drift from it: + +- The root README section 5 scenario matrix is a copy of the generated matrix. +- [`ci-workflows.md`](ci-workflows.md) mirrors the workflows at step granularity. + +Never repair either by editing the copy alone. Regenerate the matrix from the +catalog, and update the CI guide in the same pull request as the workflow change +that invalidated it. + +## Documentation conventions + +How to write documentation here. These are conventions, not preferences: each +one exists because its absence caused a specific problem. + +### Terminology + +Six words carry weight in this repository. Use them as defined, and do not +introduce synonyms for them: + +| Term | Means | +| --- | --- | +| **Stack** | One of the three self-contained framework implementations under `stacks/`. | +| **Track** | A stack seen from its audience's side — the legacy-maintainer track, the flagship track, the Python track. Same three things; "stack" is the directory, "track" is who it is for. | +| **Scenario** | A row in `scenarios/catalog.yml` with an ID such as `UI-LOGIN-001`. The unit of coverage. | +| **Slice** | A subset of tests run and staged as one unit — a matrix leg such as `smoke`, `chromium`, or `mobile-emulation`. It is the `` in `artifacts////`. | +| **Gate** | A check that can block a merge. The five always-on `pr.yml` jobs are gates; the path-filtered regression workflows are not, because they do not run on every pull request. | +| **Lane** | A trigger family: lane A runs on code change, lane B on a schedule. See [`ci-workflows.md`](ci-workflows.md). | + +### Headings + +Sentence case: capitalize the first word and proper nouns, nothing else. "Run +locally", not "Run Locally". Product names keep their own capitalization — +"Java Selenium TestNG stack". + +### Dates and time-sensitive claims + +Use ISO dates (`YYYY-MM-DD`), never `07/08/2026`, which means two different days +depending on the reader. + +Any claim that depends on the outside world — tool popularity, a vendor's +current behavior, an ecosystem trend — must be dated and must cite its source: +"As of 2026-07-12, the latest published State of JavaScript results are the 2025 +edition", with the link. An undated external claim cannot be audited later; a +dated one is at worst honestly stale. Do not date claims about this repository: +those are enforced by CI, and a date on them just rots. + +### Commands and shells + +Commands are POSIX shell, and must work under `bash` and `zsh` without change. +Every command block states where it runs — the repository root or a specific +stack directory — because this repository has no root `pom.xml` or +`package.json`, so a stack command run from the root fails. Show the `cd` before +the first stack-local command, and keep repository-root commands labelled as +such. + +### Source references + +Never cite a bare line number. `UITest.java:64` is wrong the moment someone adds +an import above it, and it is wrong silently — the reference still looks precise. +Cite a file plus a stable anchor instead: a method, class, or test name, as in +`UITest.java` — `tearDown()`. If a line truly matters, link a permalink pinned to +a revision, which cannot drift. + +### Diagrams + +Every meaningful diagram carries an adjacent prose or table equivalent that +states the same relationships. A diagram is an accelerator for readers who can +see it, never the only copy of the information: it is invisible to a screen +reader, to `grep`, and in a terminal. Both Mermaid diagrams in this repository +follow this rule — check [`architecture.md`](architecture.md) for the pattern. + +### Generated files + +A generated document carries a machine-readable marker directly after its H1 +saying it is generated and naming the command that rebuilds it. Any table using +symbols for state carries a one-line legend defining them, because `✅` and `—` +are not self-describing and are announced unhelpfully by screen readers. See +[`scenario-matrix.md`](scenario-matrix.md). diff --git a/docs/adr/0000-template.md b/docs/adr/0000-template.md new file mode 100644 index 0000000..2492cd6 --- /dev/null +++ b/docs/adr/0000-template.md @@ -0,0 +1,25 @@ +# ADR-NNNN — short decision title + +- **Status:** Proposed | Accepted | Superseded by ADR-NNNN +- **Date:** YYYY-MM-DD + + + +## Context + + + +## Decision + + + +## Consequences + + diff --git a/docs/adr/0001-shared-scenario-catalog.md b/docs/adr/0001-shared-scenario-catalog.md new file mode 100644 index 0000000..ecea3a9 --- /dev/null +++ b/docs/adr/0001-shared-scenario-catalog.md @@ -0,0 +1,49 @@ +# ADR-0001 — Shared scenario catalog as the canonical model + +- **Status:** Accepted +- **Date:** 2026-07-08 + +## Context + +The repository exists to compare automation stacks against the same application. +That comparison is only meaningful if the stacks are answering the same +questions. Without a shared definition of "the scenarios", each stack would drift +into testing whatever its examples happened to cover, and any claim that one +stack covers more than another would be unfalsifiable — the coverage tables would +be prose, maintained by whoever last remembered. + +The obvious alternatives each fail differently. Deriving coverage from test names +alone makes the tests self-certifying: a stack covers whatever it says it covers, +and a missing test is invisible because nothing declares it should exist. Keeping +a hand-written coverage table in the README makes the table a second source of +truth that rots on the first merge. + +## Decision + +Make [`scenarios/catalog.yml`](../../scenarios/catalog.yml) the canonical list of +scenarios. Every scenario has a stable ID matching `(?:UI|HTTP)-[A-Z0-9-]+`, a +title, a priority, and a `coverage:` map with one boolean per stack. Tests +reference the ID; the catalog leads and the tests follow. + +Enforce it with [`tools/check-scenarios.py`](../../tools/check-scenarios.py), +which reconciles catalog and tests in **both** directions and fails on a catalog +row marked covered with no test, a test ID with no catalog row, or a test whose +row is not marked covered. The coverage matrix is generated from the catalog, not +written by hand. + +## Consequences + +- Coverage becomes a fact that CI checks, not a claim. Uneven coverage across + stacks is recorded honestly rather than hidden. +- Adding a scenario costs a catalog edit before the test. This friction is the + point: it forces the scenario to be named and prioritized before it is + implemented. +- The catalog is parsed with a line-oriented reader rather than a YAML library, + so the file's formatting is load-bearing. Keep new rows shaped like existing + ones. +- Reconciliation matches IDs by regular expression, so a green run proves the + catalog and the test sources agree about names — not that a test asserts + anything. See [ADR-0002](0002-ui-http-scenario-split.md) for the ID grammar and + [`../architecture.md`](../architecture.md) for the caveat in full. +- Revisit if the catalog outgrows a flat list — for example if scenarios need + hierarchy, per-stack parameters, or ownership metadata. diff --git a/docs/adr/0002-ui-http-scenario-split.md b/docs/adr/0002-ui-http-scenario-split.md new file mode 100644 index 0000000..99d93a9 --- /dev/null +++ b/docs/adr/0002-ui-http-scenario-split.md @@ -0,0 +1,43 @@ +# ADR-0002 — Separate UI scenarios from HTTP/resource scenarios + +- **Status:** Accepted +- **Date:** 2026-07-10 + +## Context + +Some of the demo application's behavior is not about rendering: status codes, +redirects, basic and digest authentication, slow resources, and broken images are +properties of HTTP responses. A browser can observe them, but driving a browser +to assert a status code is theatre — it is slower, flakier, and teaches the wrong +instinct. + +The cost showed up in the browser matrix. Running a status-code check in +Chromium, Firefox, and WebKit repeats an identical assertion three times and +proves nothing about the second and third: the response does not change because +the rendering engine did. + +## Decision + +Split the catalog by prefix. `UI-` scenarios drive a browser and assert rendered +behavior. `HTTP-` scenarios assert resource-layer behavior and make no rendering +claims. The prefix is part of the scenario ID, so the split is visible in the ID +grammar, the matrix, and every test name. + +Pin `HTTP-` scenarios to a single browser project in CI with the `@http` / `http` +tag, and invert them out of the other projects so they run once per run rather +than once per browser. + +## Consequences + +- The browser matrix stays honest: a matrix leg exists to test a rendering + engine, so only scenarios that depend on one fan out across it. +- Faster runs and fewer duplicate failures. One broken redirect fails once, not + once per browser. +- The Java track has no `HTTP-` coverage, which the matrix shows as a real gap + rather than an oversight. Direct resource checks in Java belong in a dedicated + `@http` suite if that track ever needs them. +- A new scenario now requires a judgment before it is written: is this about what + the browser renders, or about what the server returned? Getting it wrong is + visible — an `HTTP-` scenario that needs the DOM is misfiled. +- Revisit if a scenario legitimately spans both layers and the prefix stops + describing it. diff --git a/docs/adr/0003-three-framework-tracks.md b/docs/adr/0003-three-framework-tracks.md new file mode 100644 index 0000000..9ada368 --- /dev/null +++ b/docs/adr/0003-three-framework-tracks.md @@ -0,0 +1,52 @@ +# ADR-0003 — Maintain three framework tracks + +- **Status:** Accepted +- **Date:** 2026-07-11 + +## Context + +A comparison repository has to choose how many things to compare. Too few and +there is no contrast; too many and every scenario must be written several more +times, the coverage gaps multiply, and the repository becomes a museum of +half-finished stacks — which teaches the opposite of the intended lesson. + +Three audiences were worth serving: the maintainer inheriting a legacy Selenium +suite, the engineer learning the modern default, and the Python team that cannot +adopt a Node-only toolchain. Each additional stack costs a full implementation of +every scenario it claims, forever. + +## Decision + +Maintain exactly three tracks, each with its own toolchain, dependency manifest, +and README, sharing no code: + +| Track | Serves | +| --- | --- | +| Java Selenium/TestNG | The legacy-maintainer audience | +| TypeScript Playwright | The modern flagship | +| Python Playwright | Teams whose language is Python | + +Treat Cypress and WebdriverIO as **comparison topics only**, documented but not +implemented, for the reasons recorded in +[`../framework-comparison.md`](../framework-comparison.md): Cypress would add a +fourth overlapping browser stack, and WebdriverIO has low marginal teaching value +once Selenium and Playwright are both present. Argue their tradeoffs in prose +rather than paying their maintenance cost in tests. + +Deliberately do not extract a shared abstraction layer across the three tracks. +The differences between them are the subject matter; hiding those differences +behind a common wrapper would destroy the thing being taught. + +## Consequences + +- Every scenario can cost up to three implementations. Coverage is therefore + intentionally uneven — TypeScript carries the most, Java the legacy subset, + Python the P0 suite plus selected P1 work — and the generated matrix records + that rather than hiding it. +- Duplication across stacks is expected and correct here. Do not refactor it away. +- A reader can compare real, idiomatic code in each framework instead of reading + a claim about how they differ. +- Cypress and WebdriverIO opinions in the comparison are not backed by tests in + this repository, and should be read as reasoning, not evidence. +- Revisit if an audience appears that none of the three serves, or if a track + stops being maintained — an unmaintained track is worse than an absent one. diff --git a/docs/adr/0004-flaky-demo-not-ci-isolation.md b/docs/adr/0004-flaky-demo-not-ci-isolation.md new file mode 100644 index 0000000..ac9698a --- /dev/null +++ b/docs/adr/0004-flaky-demo-not-ci-isolation.md @@ -0,0 +1,49 @@ +# ADR-0004 — Isolate flaky-demo and not-ci tests from the gates + +- **Status:** Accepted +- **Date:** 2026-07-10 + +## Context + +This repository deliberately contains unstable tests. The flake lab exists to +study the patterns that make suites unreliable — races, animations, +nondeterministic copy, shifting layout — and those examples are only instructive +if they actually misbehave. A demo app page that randomizes its own content +cannot be made reliable by better waiting; that is the lesson. + +That creates a direct conflict with the gates. A suite that fails at random +trains people to ignore red, which is the exact habit the repository argues +against. The tempting resolutions are both wrong: deleting the unstable examples +removes the teaching material, and retrying until green teaches that flakiness is +a retry budget rather than a defect. + +## Decision + +Keep the unstable material and tag it out of the gates rather than removing it or +papering over it: + +- `@flaky-demo` / `flaky_demo` — deliberately unstable teaching examples. + Excluded from the pull-request gate and from nightly by default, and included + in nightly only when the `include-flaky-demo` dispatch input is true. +- `@not-ci` / `not_ci` — examples unsuitable for unattended automation. Excluded + everywhere in CI; run locally, on purpose. + +Everything unmarked is expected to be deterministic, and a failure there is a +real defect. Never widen a tag to silence a genuine failure: the triage order is +recorded in [`../flakiness-guide.md`](../flakiness-guide.md), and retagging is +the last step, taken only once the nondeterminism is understood and attributable +to the application rather than the test. + +## Consequences + +- The gates stay trustworthy: red means broken. That is what makes requiring the + checks on `master` reasonable at all. +- The flake lab remains runnable and studyable on demand, and nightly can opt in. +- The tags are load-bearing and abusable. `@flaky-demo` on a test that is merely + poorly written hides a real bug and converts a defect into a teaching exhibit; + the guide's triage checklist exists to make that misuse harder. +- Python declares its markers with `--strict-markers`, so a typo fails instead of + silently matching nothing — a mistyped tag would otherwise quietly re-admit an + unstable test to the gate. +- Revisit if unmarked tests start failing intermittently, which would mean the + boundary is in the wrong place. diff --git a/docs/adr/0005-independent-workflow-topology.md b/docs/adr/0005-independent-workflow-topology.md new file mode 100644 index 0000000..1e19583 --- /dev/null +++ b/docs/adr/0005-independent-workflow-topology.md @@ -0,0 +1,53 @@ +# ADR-0005 — Keep workflows independent and parallel + +- **Status:** Accepted +- **Date:** 2026-07-10 + +## Context + +Once the repository had a fast gate, three per-stack regressions, and a nightly +sweep, the obvious next move was to wire them together: make the regressions +depend on the smoke job, chain nightly off a successful run, add `concurrency:` +to cancel superseded runs. + +Every one of those adds a coupling that has to be understood before anything can +be predicted. `needs:` turns a five-minute lint failure into a blocker for +everything behind it. `workflow_run:` makes the trigger invisible from the +workflow that runs. `concurrency:` cancels runs for reasons that look like +flakiness at the moment someone is trying to debug a failure. For a repository +whose purpose is to be *read* as a CI reference, that cost is paid on every +reading. + +## Decision + +Use no `needs:`, `workflow_run:`, `workflow_call:`, or `concurrency:` anywhere in +the five workflows. Every trigger fans out to its matching workflows in parallel; +inside each workflow every job runs in parallel. The only sequential execution is +the ordered list of steps within one job. + +Accept the duplication this implies — each workflow checks out and sets up its +own toolchain — rather than factoring it into a called workflow. The duplication +is legible; the indirection would not be. + +Where fan-out would waste work, solve it inside the job instead of across jobs: +each Playwright slice is planned before it runs, and a slice matching no tests is +skipped rather than failed. + +## Consequences + +- Any workflow can be read top to bottom and understood alone. "What will this + change run?" is answered by the trigger and the path filter, nothing else. +- One failing leg never cancels another, so a run reports every independent + failure at once instead of the first one. `fail-fast` is disabled on the + matrices for the same reason. +- Redundant setup work runs in parallel across workflows, costing runner minutes + that a `needs:` chain would have saved. That cost is accepted deliberately. +- Superseded runs are not cancelled, so pushing repeatedly to a pull request + leaves runs in flight. +- Because nothing chains, no workflow can assume another has passed. The gate is + the required-checks list on `master`, not workflow ordering. +- The full topology, triggers, jobs, and matrices are documented in + [`../ci-workflows.md`](../ci-workflows.md), which is the mirror of the + workflows themselves. +- Revisit if runner cost becomes the binding constraint, or if a job genuinely + cannot run without another's artifact. diff --git a/docs/adr/README.md b/docs/adr/README.md new file mode 100644 index 0000000..861c1b8 --- /dev/null +++ b/docs/adr/README.md @@ -0,0 +1,43 @@ +# Architecture decision records + +Decisions that shaped this repository, with the reasoning that produced them. +The code shows what the repository does; these records exist for the part the +code cannot show — what was rejected, and why. + +## What warrants an ADR + +Write one when a decision is hard to reverse and affects any of: + +- More than one stack. +- The scenario catalog or its model. +- CI topology: what runs, when, and what gates a merge. +- Tagging semantics. +- Artifact contracts. +- Contributor workflow. + +Everything else is a pull request. A decision that is cheap to undo does not +need a record; a decision someone will otherwise re-litigate in six months does. +If you are unsure, ask whether a reader would be able to reconstruct the +reasoning from the diff alone. If not, write the ADR. + +## How to add one + +1. Copy [`0000-template.md`](0000-template.md) to `NNNN-short-title.md`, taking + the next free number. +2. Fill in Status, Date, Context, Decision, and Consequences. +3. Add it to the table below, and to the + [documentation index](../README.md). + +ADRs are immutable once accepted. A decision that no longer holds is not edited +and not deleted: write a new ADR and set the old one's status to **Superseded +by** the new number. The record of having been wrong is the useful part. + +## Records + +| ADR | Title | Status | Date | +| --- | --- | --- | --- | +| [0001](0001-shared-scenario-catalog.md) | Shared scenario catalog as the canonical model | Accepted | 2026-07-08 | +| [0002](0002-ui-http-scenario-split.md) | Separate UI scenarios from HTTP/resource scenarios | Accepted | 2026-07-10 | +| [0003](0003-three-framework-tracks.md) | Maintain three framework tracks | Accepted | 2026-07-11 | +| [0004](0004-flaky-demo-not-ci-isolation.md) | Isolate flaky-demo and not-ci tests from the gates | Accepted | 2026-07-10 | +| [0005](0005-independent-workflow-topology.md) | Keep workflows independent and parallel | Accepted | 2026-07-10 | diff --git a/docs/anti-patterns.md b/docs/anti-patterns.md index d83bae3..d072195 100644 --- a/docs/anti-patterns.md +++ b/docs/anti-patterns.md @@ -7,15 +7,22 @@ Use these examples when reviewing legacy browser automation code. | Anti-pattern | Example location | Safer pattern | | --- | --- | --- | -| Waiting on the wrong element before reading feedback | `stacks/java-selenium-testng/src/test/java/theinternetwebsite/ui/pageobjects/LoginFormPage.java:58` | Wait for the flash message itself, then read the returned visible element text. | -| Malformed regex character-class intersection | `stacks/java-selenium-testng/src/test/java/theinternetwebsite/ui/UITest.java:186` | Put the intersection inside one character class: `[\\p{Cntrl}&&[^\\r\\n\\t]]`. | -| Calling `close()` before `quit()` during teardown | `stacks/java-selenium-testng/src/test/java/theinternetwebsite/ui/UITest.java:64` | Quit the session once; guard for failed setup so teardown preserves the original error. | -| Constructor navigation repeated inside a helper | `stacks/java-selenium-testng/src/test/java/theinternetwebsite/ui/testcases/LoginTest.java:29` | Reuse the page object already loaded by the test when submitting the form. | -| Suite parameters declared but ignored by tests | `stacks/java-selenium-testng/src/test/java/theinternetwebsite/ui/testcases/LoginTest.java:43` | Pass TestNG parameters through to the action; keep invalid defaults only as local fallbacks. | -| URL strings assembled with an accidental double slash | `stacks/java-selenium-testng/src/test/java/theinternetwebsite/ui/pageobjects/JavascriptErrorPage.java:23` | Join route paths with exactly one slash before navigation. | +| Waiting on the wrong element before reading feedback | `stacks/java-selenium-testng/src/test/java/theinternetwebsite/ui/pageobjects/LoginFormPage.java` — `getErrorMessage()` | Wait for the flash message itself, then read the returned visible element text. | +| Malformed regex character-class intersection | `stacks/java-selenium-testng/src/test/java/theinternetwebsite/ui/support/TextContent.java` — `clean()` | Put the intersection inside one character class: `[\\p{Cntrl}&&[^\\r\\n\\t]]`. | +| Calling `close()` before `quit()` during teardown | `stacks/java-selenium-testng/src/test/java/theinternetwebsite/ui/UITest.java` — `tearDown()` | Quit the session once; guard for failed setup so teardown preserves the original error. | +| Constructor navigation repeated inside a helper | `stacks/java-selenium-testng/src/test/java/theinternetwebsite/ui/testcases/LoginTest.java` — `successfulLogin()` (`UI-LOGIN-001`) | Reuse the page object already loaded by the test when submitting the form. | +| Suite parameters declared but ignored by tests | `stacks/java-selenium-testng/src/test/java/theinternetwebsite/ui/testcases/LoginTest.java` — `invalidUsername()` (`UI-LOGIN-002`) | Pass TestNG parameters through to the action; keep invalid defaults only as local fallbacks. | +| URL strings assembled with an accidental double slash | `stacks/java-selenium-testng/src/test/java/theinternetwebsite/ui/pageobjects/JavascriptErrorPage.java` — the `JavascriptErrorPage(UITest)` constructor | Pass the route to `BasePage`, which joins it through `UITest.urlFor()` with exactly one slash before navigation. | ## Download strategy split fixed in P3.4 | Anti-pattern | Example location | Safer pattern | | --- | --- | --- | | Verifying a browser download by fetching the link with raw HTTP | `stacks/java-selenium-testng/src/test/java/theinternetwebsite/ui/pageobjects/DownloadPage.java` | UI download scenarios click in the browser and wait for Selenium-managed downloads or the local download directory; future direct resource checks belong in an `@http` suite. | + +## Related documentation + +- [Documentation index](README.md) — every document, and which source wins on conflict. +- [Java Selenium/TestNG stack](../stacks/java-selenium-testng/README.md) — how to run the suite these defects were found in. +- [Flakiness guide](flakiness-guide.md) — the triage order for a test that fails intermittently rather than wrongly. +- [Interview guide](interview-guide.md) — the same mistakes, phrased as things to avoid saying. diff --git a/docs/architecture.md b/docs/architecture.md new file mode 100644 index 0000000..c9b46b7 --- /dev/null +++ b/docs/architecture.md @@ -0,0 +1,200 @@ +# Architecture + +How the pieces fit: one catalog of scenarios, three independent stacks that +implement them, one pinned application they all test, and a checker that refuses +to let the first two disagree. This guide is for contributors and reviewers who +need the shape of the system before changing it. For what CI runs and when, read +[`ci-workflows.md`](ci-workflows.md); this guide does not restate it. + +## The shape + +```mermaid +flowchart LR + CAT["scenarios/catalog.yml
canonical scenario list"] + CHK["tools/check-scenarios.py
reconciler + generator"] + MTX["docs/scenario-matrix.md
generated"] + J["stacks/java-selenium-testng
Selenium + TestNG"] + T["stacks/ts-playwright
Playwright Test"] + P["stacks/python-playwright
pytest + Playwright"] + APP["the-internet demo app
pinned image, :7080"] + CAT --> CHK + CHK --> MTX + J -. "test IDs" .-> CHK + T -. "test IDs" .-> CHK + P -. "test IDs" .-> CHK + J --> APP + T --> APP + P --> APP +``` + +In prose: the catalog is the only list of scenarios. The checker reads the +catalog, scrapes scenario IDs out of all three stacks' test sources, compares the +two in both directions, and regenerates the matrix. The three stacks never +reference each other; their only shared dependencies are the catalog they answer +to and the demo application they drive. Solid arrows are the generation and +execution paths; dotted arrows are the IDs the checker scrapes back out of each +stack. + +## The scenario catalog is the canonical model + +[`../scenarios/catalog.yml`](../scenarios/catalog.yml) is the single list of what +this repository tests: 48 scenarios today, 20 `P0`, 17 `P1`, 11 `P2`. Each row +carries an ID, a title, a priority, optional tags, and a `coverage:` map with one +boolean per stack: + +```yaml + - id: UI-LOGIN-001 + title: Login succeeds with valid credentials + priority: P0 + tags: [smoke] + coverage: {java-selenium-testng: true, ts-playwright: true, python-playwright: true} +``` + +Scenario IDs match `(?:UI|HTTP)-[A-Z0-9-]+`. The `UI-` and `HTTP-` prefixes are +the one deliberate split in the model: `UI-` scenarios drive a browser, `HTTP-` +scenarios check status codes, redirects, auth, and slow resources at the +resource layer, without asserting rendering. That split is why the `HTTP-` +scenarios are pinned to a single browser project in CI — running them per +browser would repeat identical assertions. + +A scenario that is not in the catalog is not covered, however many tests exist. +The catalog leads; the tests follow. + +## Three stack boundaries + +Each stack is a self-contained project with its own toolchain, dependency +manifest, and README, and is runnable without the others. They deliberately +share no code — the repository's purpose is to compare them, so a shared +abstraction layer would destroy the thing being taught. + +| Stack | Directory | Carries the scenario ID in | Test sources the checker reads | +| --- | --- | --- | --- | +| Java Selenium/TestNG | `stacks/java-selenium-testng` | The TestNG `testName="…"` attribute | `src/test/java/theinternetwebsite/ui/testcases/*.java` | +| TypeScript Playwright | `stacks/ts-playwright` | The test title | `tests/**/*.ts` | +| Python Playwright | `stacks/python-playwright` | The test docstring | `tests/**/*.py` | + +Coverage is intentionally uneven: the TypeScript track is the flagship and +carries the most scenarios, Java carries the legacy-maintainer subset, and Python +carries the P0 suite plus selected P1 work. The matrix records that unevenness +rather than hiding it. + +## The application under test + +All three stacks drive the same target: the public +[The Internet](https://github.com/saucelabs/the-internet) demo app, run locally +and in CI from a container pinned by digest in +[`../docker/compose.yml`](../docker/compose.yml): + +| Property | Value | +| --- | --- | +| Image | `gprestes/the-internet:v2.6.5`, pinned by `sha256:205b8fc7…` | +| Port mapping | `7080:5000` — container listens on 5000, host reaches it on 7080 | +| Base URL | `http://localhost:7080`, passed to each stack via `THE_INTERNET_BASE_URL` (Java uses `-DwebAppAddress`) | + +The digest pin is what makes the suite honest. An unpinned demo app could change +under the tests and turn a real regression into a mystery, which is the exact +failure this repository teaches people to avoid. CI runs the identical image as a +service container, so a local failure and a CI failure mean the same thing. + +## Test-ID reconciliation + +[`../tools/check-scenarios.py`](../tools/check-scenarios.py) is the gate that +keeps the catalog and the tests from drifting apart. It parses the catalog, +extracts the ID set from each stack, and fails on any of these: + +- A duplicate ID in the catalog. +- A catalog row marked covered for a stack that has no matching test. +- A test ID with no catalog row. +- A test ID whose catalog row is not marked covered for that stack. + +Both directions matter: forgetting the test and forgetting the catalog row are +both errors. With `--write-matrix` the checker also regenerates +[`scenario-matrix.md`](scenario-matrix.md); CI regenerates and diffs it, so a +stale matrix fails the pull request. + +### Matching is regex-based, and that is a real caveat + +The checker greps; it does not parse. Each stack must carry the ID somewhere that +means "this test *is* this scenario", not merely somewhere in the file: + +| Stack | Where the ID must appear | Pattern | +| --- | --- | --- | +| Java | The TestNG `testName` attribute | `testName\s*=\s*"(ID)"` | +| TypeScript | A test title | `test(...)`, including `test.skip` and similar | +| Python | A test docstring, a marker argument, or a `parametrize` ID | `"""(ID)`, `@pytest.mark.x("(ID)")`, `parametrize(... "(ID)")` | + +Matching was originally unanchored for TypeScript and Python — any occurrence of +the ID counted, so `// UI-SLIDER-001 is not done yet` in a test file was enough +to satisfy the checker that the scenario was covered. It is now anchored to the +contexts above, so a mention in a comment no longer creates coverage. + +The caveat that remains: this is still a regular expression, not a parser. A test +whose title carries the right ID counts as covering that scenario even if its +body asserts nothing useful, and nothing checks that the assertions match the +scenario's stated title. Anchoring removed the accidental failure — an ID in a +comment — but not the deliberate one. + +Treat green as "every catalogued scenario has a test that claims to be it", not +"the behavior is tested". + +## Tag taxonomy + +Tags are how a scenario says which slices may run it. TypeScript uses +`@`-prefixed title tags that Playwright filters with `--grep`; Python uses the +equivalent pytest markers, declared with `--strict-markers` so a typo fails +rather than silently matching nothing. + +| Tag (TypeScript / Python) | Means | Why it exists | +| --- | --- | --- | +| `@http` / `http` | Resource-layer check, no rendering assertions | Runs once on the default browser instead of per browser | +| `@desktop` / `desktop` | Needs a mouse | Inverted out of mobile-emulation slices | +| `@mobile-emulation` / `mobile_emulation` | Needs a mobile device profile | Selects the `Mobile Chrome` and `Mobile Safari` projects | +| `@flaky-demo` / `flaky_demo` | Deliberately unstable teaching example | Kept out of every gate by default; see [`flakiness-guide.md`](flakiness-guide.md) | +| `@not-ci` / `not_ci` | Unsuitable for unattended runs | Local only, excluded everywhere in CI | +| `@smoke` / `smoke` | Fast subset | The always-on pull-request slice | + +`@flaky-demo` and `@not-ci` are the load-bearing ones: this repository +deliberately contains unstable tests as teaching material, and the tags are what +stop that material from touching the gates. + +No test currently carries `@mobile-emulation`, so the mobile-emulation slices +select nothing. That is why each Playwright slice is planned before it runs: the +plan step counts matching tests and skips the slice at zero rather than failing +it. An empty slice is a valid state here, not a bug. + +## Artifact flow + +Every stack stages its reports under `artifacts////` before +uploading, so outputs are grouped by stack, run, and slice rather than colliding +in one directory: + +- `` is `java`, `ts`, or `py`. +- `` is the GitHub run ID, which keeps reruns distinct. +- `` is the matrix leg, such as `smoke` or a browser project. + +`artifacts/` is git-ignored: it is a staging area for upload, never a committed +result. Uploads run under `if: always()`, so a failing test still produces its +trace — which is the only reason the artifacts are worth having. These artifacts +are public on a public repository; see the secrets and captures rules in +[`../CONTRIBUTING.md`](../CONTRIBUTING.md) for what must never end up in one. + +## CI fan-out + +Triggers fan out to independent workflows with no chaining: an always-on fast +gate on every change, three path-filtered per-stack regressions, and a nightly +cross-browser and Grid sweep. Nothing uses `needs:`, `workflow_run:`, +`workflow_call:`, or `concurrency:`, so no workflow orders or cancels another. + +That is the whole architectural claim. For the trigger map, the job and step +lists, the matrices, and where each job stages its reports, read +[`ci-workflows.md`](ci-workflows.md), which mirrors +[`../.github/workflows/`](../.github/workflows/) and is the only place those +tables live. + +## Canonical sources + +When two descriptions of this system disagree, resolve in the order recorded in +the [documentation index](README.md): the workflows, then the catalog, then the +generated matrix, then the stack READMEs, then the guides — with the root README +deferring to all of them. Every document in that order is executable except the +last two, which is the point: prose loses to the thing that runs. diff --git a/docs/ci-workflows.md b/docs/ci-workflows.md index a088a4f..8415ff8 100644 --- a/docs/ci-workflows.md +++ b/docs/ci-workflows.md @@ -61,8 +61,8 @@ Steps run top to bottom within each job. A step marked (always) carries `if: alw | Job (`id`) | Intent | Steps (top to bottom) | File | | --- | --- | --- | --- | -| `repo-hygiene` | Lint workflows, commit messages, and Markdown | 1. Checkout repository, 2. Set up Node 22, 3. Lint GitHub Actions workflows, 4. Lint pull request commits (conditional), 5. Lint latest commit (conditional), 6. Lint Markdown | [`pr.yml`](../.github/workflows/pr.yml) | -| `scenario-catalog` | Fail if the generated scenario matrix has drifted | 1. Checkout repository, 2. Check scenario catalog | [`pr.yml`](../.github/workflows/pr.yml) | +| `repo-hygiene` | Lint workflows, commit messages, and Markdown, check documentation links, validate the README fast start, and require CI-guide updates alongside workflow changes | 1. Checkout repository, 2. Set up Node 22, 3. Lint GitHub Actions workflows, 4. Lint pull request commits (conditional), 5. Lint latest commit (conditional), 6. Lint Markdown, 7. Check Markdown links, 8. Validate README fast start, 9. Require CI guide updates with workflow changes (conditional) | [`pr.yml`](../.github/workflows/pr.yml) | +| `scenario-catalog` | Fail if the generated scenario matrix or the README embed of it has drifted | 1. Checkout repository, 2. Check scenario catalog | [`pr.yml`](../.github/workflows/pr.yml) | | `java-smoke` | Compile the Java tests and run the Chrome smoke suite | 1. Checkout repository, 2. Set up JDK 25, 3. Compile Java tests, 4. Run Java smoke, 5. Stage Java reports (always), 6. Upload Surefire XML (always) | [`pr.yml`](../.github/workflows/pr.yml) | | `ts-smoke` | Type-check, lint, and format-check, then run the Chromium smoke suite | 1. Checkout repository, 2. Set up Node 22, 3. Install TypeScript stack dependencies, 4. Type-check TypeScript stack, 5. Lint TypeScript stack, 6. Check TypeScript stack formatting, 7. Install Chromium browser, 8. Run TypeScript Chromium smoke, 9. Upload Playwright artifacts (always) | [`pr.yml`](../.github/workflows/pr.yml) | | `py-smoke` | Lint, format-check, and type-check, then run the Chromium smoke suite | 1. Checkout repository, 2. Set up Python 3.14, 3. Install Python stack dependencies, 4. Lint Python stack, 5. Check Python stack formatting, 6. Type-check Python stack, 7. Install Chromium browser, 8. Run Python Chromium smoke, 9. Upload Playwright artifacts (always) | [`pr.yml`](../.github/workflows/pr.yml) | @@ -74,6 +74,14 @@ Steps run top to bottom within each job. A step marked (always) carries `if: alw | `typescript-nightly` | Plan and run each TypeScript nightly slice per matrix project | 1. Checkout repository, 2. Set up Node 22, 3. Install TypeScript stack dependencies, 4. Plan TypeScript nightly slice, 5. Install Playwright browser (conditional), 6. Run TypeScript nightly slice (conditional), 7. Upload Playwright artifacts (always) | [`nightly.yml`](../.github/workflows/nightly.yml) | | `python-nightly` | Plan and run each Python nightly slice per matrix project | 1. Checkout repository, 2. Set up Python 3.14, 3. Install Python stack dependencies, 4. Plan Python nightly slice, 5. Install Playwright browser (conditional), 6. Run Python nightly slice (conditional), 7. Upload Playwright artifacts (always) | [`nightly.yml`](../.github/workflows/nightly.yml) | +This guide is the one document CI checks against its own source. A pull request +that changes anything under `.github/workflows/` without touching this file fails +`repo-hygiene`, because a step-level mirror that nothing enforces is the highest- +risk documentation in the repository: it stays plausible long after it stops +being true. If a workflow change genuinely has no semantic effect here — a +whitespace or comment edit — put the literal token `[ci-guide-exempt]` in the +pull request title to skip the check. + ## Matrix and tags Each regression and nightly job fans out over a `strategy.matrix`, with `fail-fast` disabled so one failing leg never cancels the rest. @@ -105,3 +113,7 @@ Every stack stages its reports under `artifacts////` befor - **Python** (`artifacts/py/...`) — JUnit XML plus retained traces, screenshots, and videos, uploaded by **Upload Playwright artifacts** `if: always()`. Because the uploads run on `always()`, artifacts are available even when the test step fails. See the [`README.md`](../README.md) reports section for the matching per-stack local report paths. + +## When a check fails + +This guide describes what runs. For what to do when one of these jobs goes red — which artifact to open first, and how to reproduce each failure class locally — see [`runbooks/ci-failure-triage.md`](runbooks/ci-failure-triage.md). diff --git a/docs/flakiness-guide.md b/docs/flakiness-guide.md index fee00bc..e9b40ff 100644 --- a/docs/flakiness-guide.md +++ b/docs/flakiness-guide.md @@ -44,3 +44,10 @@ THE_INTERNET_BASE_URL=http://localhost:7080 npx playwright test --project=chromi 5. If variable, encode the allowed variants or move the scenario to `@flaky-demo`. 6. If unsuitable for CI, add `@not-ci` and document the manual learning value. 7. Never weaken catalog reconciliation, linting, typechecking, or CI gates to hide the failure. + +## Related documentation + +- [Documentation index](README.md) — every document, and which source wins on conflict. +- [ADR-0004 — Isolate flaky-demo and not-ci tests from the gates](adr/0004-flaky-demo-not-ci-isolation.md) — why the unstable examples exist at all. +- [CI workflows](ci-workflows.md) — where each tag is included or inverted. +- [Anti-patterns](anti-patterns.md) — defects that look like flakiness but are not. diff --git a/docs/framework-comparison.md b/docs/framework-comparison.md index d19e4be..968dc58 100644 --- a/docs/framework-comparison.md +++ b/docs/framework-comparison.md @@ -25,3 +25,10 @@ Use this guide to explain why the repository keeps three maintained tracks inste As of 2026-07-12, the latest published State of JavaScript results are the 2025 edition. Its testing section is the citation to refresh when discussing JavaScript testing tool popularity or sentiment: [State of JavaScript 2025 — Testing](https://2025.stateofjs.com/en-US/libraries/testing/). Use that citation as context, not as a mandate. This repository's implementation choices are driven by teaching coverage: Selenium for legacy/WebDriver literacy, TypeScript Playwright for the flagship modern stack, and Python Playwright for a second-language comparison. + +## Related documentation + +- [Documentation index](README.md) — every document, and which source wins on conflict. +- [ADR-0003 — Maintain three framework tracks](adr/0003-three-framework-tracks.md) — why Cypress and WebdriverIO are argued here but not implemented. +- [Architecture](architecture.md) — the stack boundaries this comparison depends on. +- [Learning paths](learning-paths.md) — where to go next, by audience. diff --git a/docs/interview-guide.md b/docs/interview-guide.md index 2027918..b17a0dc 100644 --- a/docs/interview-guide.md +++ b/docs/interview-guide.md @@ -53,3 +53,10 @@ Use the shared scenario catalog as the interview spine: identify the behavior, n - Marking flaky tests as normal smoke tests. - Hiding setup failures behind broad exception handling. - Changing the catalog without adding or removing matching test IDs. + +## Related documentation + +- [Documentation index](README.md) — every document, and which source wins on conflict. +- [Scenario coverage matrix](scenario-matrix.md) — which scenarios each stack implements. +- [Framework comparison](framework-comparison.md) — the tradeoffs to argue when asked to choose a stack. +- [Learning paths](learning-paths.md) — where to go next, by audience. diff --git a/docs/runbooks/ci-failure-triage.md b/docs/runbooks/ci-failure-triage.md new file mode 100644 index 0000000..86ce469 --- /dev/null +++ b/docs/runbooks/ci-failure-triage.md @@ -0,0 +1,209 @@ +# Runbook: CI failure triage + +A check went red. This is how to find out why without reading the whole workflow +file, and how to reproduce it locally so you are debugging your change rather +than the Actions UI. + +Rule zero: **never fix a red gate by weakening it.** Widening a tag, adding a +retry, or relaxing a lint to get green converts a defect into a permanent lie. +If a failure is genuinely not your change, say so in the pull request and fix the +cause. + +Every reproduction below runs from the repository root unless it says otherwise, +and assumes the demo app is up where the failing job had it: + +```bash +docker compose -f docker/compose.yml up -d website +``` + +## Failure classes + +| Failure class | Workflow · job | First diagnostic step | Artifacts | Reproduce locally | +| --- | --- | --- | --- | --- | +| Hygiene lint | `pr.yml` · `repo-hygiene` | Read the failing step's name: it says which linter | None; the log is the artifact | See [hygiene lint](#hygiene-lint) | +| Catalog drift | `pr.yml` · `scenario-catalog` | The diff in the log names the rows that disagree | None | `python3 tools/check-scenarios.py --write-matrix` | +| Per-stack static | `pr.yml` · `ts-smoke`, `py-smoke`, `java-smoke` | Type, lint, and format failures are deterministic and reproduce exactly | None | See [per-stack static](#per-stack-static-checks) | +| Smoke | `pr.yml` · `java-smoke`, `ts-smoke`, `py-smoke` | Open the uploaded trace or Surefire XML before rerunning | `artifacts///smoke/` | See [smoke](#smoke) | +| Regression matrix leg | `java.yml`, `ts.yml`, `python.yml` | Note **which leg** failed; one browser failing is a different bug from all of them failing | `artifacts////` | See [regression leg](#regression-matrix-leg) | +| Nightly grid | `nightly.yml` · `java-nightly-grid` | Check whether the local-driver leg passed on the same browser | `artifacts/java///surefire-reports/` | See [nightly grid](#nightly-grid) | + +Path-filtered workflows only run when your change touches their stack, so a +regression workflow that did not run is not a failure. See +[`../ci-workflows.md`](../ci-workflows.md) for the trigger map. + +## Hygiene lint + +Five linters share one job, so read the step name first. Each is runnable alone: + +```bash +docker run --rm -v "$PWD:/repo:ro" -w /repo rhysd/actionlint:1.7.12 -color +git ls-files -z '*.md' | xargs -0 npx --yes markdownlint-cli2@0.23.0 +``` + +Commit-message failures name the offending commit and rule. The rules, and the +exact scope list, are in [`../../CONTRIBUTING.md`](../../CONTRIBUTING.md); +`commitlint.config.cjs` is the authority. + +A link-check failure prints the file, the line, and the unreachable target. If +the target is genuinely unverifiable rather than wrong — an upstream host that +rate-limits — it belongs in `.lycheeignore` with a reason, not in a retry. + +**"This pull request changes `.github/workflows/` but not `docs/ci-workflows.md`"** +is not a lint failure: the CI guide mirrors the workflows and your change made it +untrue. Update the guide. Use `[ci-guide-exempt]` in the pull request title only +when the workflow edit has no semantic effect. + +## Catalog drift + +The job regenerates the matrix and diffs it. It fails when the committed matrix, +or the README copy of it, disagrees with `scenarios/catalog.yml`: + +```bash +python3 tools/check-scenarios.py --write-matrix +git diff --exit-code docs/scenario-matrix.md +``` + +The output tells you which of the two things happened: + +- A **diff in `docs/scenario-matrix.md`** means you changed the catalog and did + not regenerate. Commit the regenerated file. +- A **README embed diff** means the catalog moved and the README section 5 copy + did not. Copy the table out of `docs/scenario-matrix.md`. +- **`catalog … ids missing tests`** or **`test ids missing catalog rows`** means + the catalog and the tests genuinely disagree. Fix whichever is wrong — do not + flip a `coverage:` flag to silence it. + +## Per-stack static checks + +These are deterministic: if CI failed, your machine will too. + +```bash +cd stacks/ts-playwright +npm run typecheck +npm run lint +npm run format:check +``` + +```bash +cd stacks/python-playwright +ruff check . +ruff format --check . +mypy src tests +``` + +```bash +mvn -B -f stacks/java-selenium-testng/pom.xml clean test-compile +``` + +## Smoke + +The smoke jobs run one browser against the demo app. Read the artifact before +rerunning: a Playwright trace usually shows the cause in one pass, and a rerun +that passes without explanation has taught you nothing. + +```bash +cd stacks/ts-playwright +THE_INTERNET_BASE_URL=http://localhost:7080 npm run test:chromium:smoke +``` + +```bash +cd stacks/python-playwright +THE_INTERNET_BASE_URL=http://localhost:7080 pytest -m smoke --browser chromium +``` + +```bash +mvn -B -f stacks/java-selenium-testng/pom.xml -P CLI_Parameters test \ + -DsuiteXmlFile=src/test/resources/smoke.xml \ + -Dbrowser=chrome \ + -DheadlessBrowser=true \ + -DuseSeleniumGrid=false \ + -DwebAppAddress=http://localhost:7080 \ + -DtestRunnerAddress=http://localhost:7080 +``` + +If a smoke test fails intermittently rather than consistently, stop and use +[`../flakiness-guide.md`](../flakiness-guide.md). Do not retag it to get green. + +## Regression matrix leg + +`fail-fast` is disabled, so every leg reports independently. Which legs failed is +the diagnosis: + +- **One browser leg only** — a genuine engine-specific difference, or a locator + that only holds in one engine. +- **Every leg** — the change is broken everywhere; the matrix is noise. Debug the + default browser and ignore the rest. +- **Only the mobile-emulation legs** — a device-profile assumption, usually + viewport or touch. + +Reproduce one leg by naming its project. Substitute the failing leg's browser: + +```bash +cd stacks/ts-playwright +THE_INTERNET_BASE_URL=http://localhost:7080 npx playwright test --project=firefox +``` + +```bash +cd stacks/python-playwright +THE_INTERNET_BASE_URL=http://localhost:7080 pytest --browser firefox +``` + +```bash +mvn -B -f stacks/java-selenium-testng/pom.xml -P CLI_Parameters clean test \ + -DsuiteXmlFile=src/test/resources/regression.xml \ + -Dbrowser=firefox \ + -DheadlessBrowser=true \ + -DuseSeleniumGrid=false \ + -DwebAppAddress=http://localhost:7080 \ + -DtestRunnerAddress=http://localhost:7080 +``` + +## Nightly grid + +`nightly.yml` runs Java twice per browser: once with a local driver, once against +Selenium Grid. Compare them before anything else, because the pair localizes the +fault for free: + +- **Grid fails, local passes** — the fault is in the Grid path: session + capabilities, the node image, or a client/node version mismatch. It is not your + test logic. +- **Both fail** — an ordinary test failure that has nothing to do with Grid. + +The Grid services are pinned to the same Selenium image digests the nightly job +uses, so a local Grid reproduces it. Start the app plus Grid from the repository +root: + +```bash +docker compose -f docker/compose.yml -f docker/compose.grid.yml up -d +``` + +Then run against the Grid from the stack directory: + +```bash +cd stacks/java-selenium-testng +mvn -P CLI_Parameters test \ + -DsuiteXmlFile=src/test/resources/regression.xml \ + -Dbrowser=remote-chrome \ + -DheadlessBrowser=true \ + -DuseSeleniumGrid=true \ + -DtestRunnerAddress=http://website:5000 \ + -DwebAppAddress=http://localhost:7080 \ + -DseleniumGridAddress=http://localhost:4444/wd/hub +``` + +Stop everything from the repository root when finished: + +```bash +docker compose -f docker/compose.yml -f docker/compose.grid.yml down +``` + +A nightly failure gates nothing on its own, but it is the only signal for +browsers the pull-request gate never exercises. Do not let it stay red — a +permanently red nightly is the same as no nightly. + +## Related documentation + +- [Documentation index](../README.md) — every document, and which source wins on conflict. +- [CI workflows](../ci-workflows.md) — what runs, when, and where each job stages its reports. +- [Flakiness guide](../flakiness-guide.md) — for failures that are intermittent rather than wrong. +- [Architecture](../architecture.md) — the catalog, stacks, and artifact model behind these jobs. diff --git a/docs/scenario-matrix.md b/docs/scenario-matrix.md index 109b277..4c208ea 100644 --- a/docs/scenario-matrix.md +++ b/docs/scenario-matrix.md @@ -1,7 +1,11 @@ # Scenario coverage matrix + + Generated by `tools/check-scenarios.py --write-matrix`. +Legend: ✅ = covered by that stack · — = not covered. + | ID | Priority | Scenario | Java | TS | Python | | --- | --- | --- | --- | --- | --- | | UI-LOGIN-001 | P0 | Login succeeds with valid credentials | ✅ | ✅ | ✅ | diff --git a/docs/templates/runbook-template.md b/docs/templates/runbook-template.md new file mode 100644 index 0000000..2b71451 --- /dev/null +++ b/docs/templates/runbook-template.md @@ -0,0 +1,56 @@ +# Runbook: `` + + + +One sentence: what has gone wrong, and what this runbook gets you to. + + + +Preconditions: what must be running or set up before any step below works. + +```bash + +``` + +## Failure classes + + + +| Failure class | Where it surfaces | First diagnostic step | Artifacts | Reproduce locally | +| --- | --- | --- | --- | --- | +| `` | `` | `` | `` | `` | +| `` | `` | `` | `` | `` | + +## Failure class: `` + + + +What this failure actually indicates. + +```bash + +``` + +How to tell a real defect from an environmental failure, and what to do in each +case. + +## Failure class: `` + +What this failure actually indicates. + +```bash + +``` + +## Related documentation + + + +- [Documentation index](../README.md) — every document, and which source wins on conflict. +- `` — why a reader here would want it. diff --git a/docs/templates/stack-readme-template.md b/docs/templates/stack-readme-template.md new file mode 100644 index 0000000..828e17f --- /dev/null +++ b/docs/templates/stack-readme-template.md @@ -0,0 +1,125 @@ +# `` stack + + + +One sentence: which framework, and which slice of the catalog this stack claims. + +## Supported runtime versions + + + +| Component | Version | +| --- | --- | +| `` | `` | +| `` | `` | +| `` | `` | + +## Prerequisites + +- `` +- `` + +## Install + + + +State the working directory before the first stack-local command. + +```bash +cd stacks/ + +``` + +## Configuration + + + +| Variable | Default | Purpose | +| --- | --- | --- | +| `THE_INTERNET_BASE_URL` | `` | Base URL of the app under test | +| `` | `` | `` | + +## Fast validation + + + +```bash + +``` + +## Run locally + + + +Start the demo app from the repository root: + +```bash +docker compose -f docker/compose.yml up -d website +``` + +Run the suite from this stack directory: + +```bash +cd stacks/ + +``` + +Stop the app from the repository root: + +```bash +docker compose -f docker/compose.yml down +``` + +## Static checks + + + +```bash + +``` + +## CI mapping + + + +| Workflow | Job | Runs | +| --- | --- | --- | +| `pr.yml` | `` | `` | +| `.yml` | `` | `` | + +## Reports and artifacts + +| Output | Local path | CI path | +| --- | --- | --- | +| `` | `` | `artifacts////` | + +## Known limitations + + + +- `` + +## Adding or changing scenarios + + + +Where the ID goes in this stack, and the catalog-first workflow. + +## Troubleshooting + + + +| Symptom | Cause | Fix | +| --- | --- | --- | +| `` | `` | `` | + +## Ownership + +Who maintains this stack, and what triggers a review of this document. diff --git a/docs/templates/waiver-template.md b/docs/templates/waiver-template.md new file mode 100644 index 0000000..7c99228 --- /dev/null +++ b/docs/templates/waiver-template.md @@ -0,0 +1,23 @@ +# Waiver: `` + + + +| Field | Value | +| --- | --- | +| Requirement | `` | +| Scope | `` | +| Reason | `` — not "no time". A waiver records a considered decision; cost alone is a backlog item. | +| Owner | `` | +| Approver | `` — not the owner, unless the repository has a single maintainer and that fact is itself recorded. | +| Revisit trigger | `` — stated so anyone can tell whether it has happened. Prefer an observable event over a date. | +| Tracking issue | ``, or `None` plus the reason — for example, issues are disabled on this repository. | + +## Notes + + + +Anything a future reader needs that the table cannot hold. diff --git a/stacks/java-selenium-testng/README.md b/stacks/java-selenium-testng/README.md index 52f1da9..89804e3 100644 --- a/stacks/java-selenium-testng/README.md +++ b/stacks/java-selenium-testng/README.md @@ -14,19 +14,22 @@ Selenium Manager provisions browser drivers automatically through Selenium. WebD ## Run locally +Run the Maven commands in this guide from this stack directory. There is no root `pom.xml`, so `mvn test` fails from the repository root. Docker Compose commands are the exception and run from the repository root, as each step below states. + Start The Internet demo app from the repository root: ```bash docker compose -f docker/compose.yml up -d website ``` -Run the suite: +Run the suite from this stack directory: ```bash +cd stacks/java-selenium-testng mvn test ``` -Stop the app: +Stop the app from the repository root: ```bash docker compose -f docker/compose.yml down @@ -44,9 +47,10 @@ Start the app plus local Grid from the repository root: docker compose -f docker/compose.yml -f docker/compose.grid.yml up -d ``` -Run against the Grid: +Run against the Grid from this stack directory: ```bash +cd stacks/java-selenium-testng mvn -P CLI_Parameters test \ -DsuiteXmlFile=src/test/resources/regression.xml \ -Dbrowser=remote-chrome \ @@ -59,7 +63,7 @@ mvn -P CLI_Parameters test \ -DseleniumGridAddress=http://localhost:4444/wd/hub ``` -Stop all Compose services: +Stop all Compose services from the repository root: ```bash docker compose -f docker/compose.yml -f docker/compose.grid.yml down diff --git a/stacks/python-playwright/README.md b/stacks/python-playwright/README.md index be95ac3..1b56eb0 100644 --- a/stacks/python-playwright/README.md +++ b/stacks/python-playwright/README.md @@ -12,7 +12,13 @@ the Python P0 UI suite, selected P1 data-modeling coverage, and Chromium-only HT ## Install +Run the pip, Playwright, and pytest commands in this guide from this stack +directory, so that the virtual environment and `pyproject.toml` resolve. Docker +Compose commands are the exception and run from the repository root, as each +step below states. + ```bash +cd stacks/python-playwright python3 -m venv .venv source .venv/bin/activate python -m pip install --upgrade pip @@ -28,7 +34,7 @@ Start the demo app from the repository root: docker compose -f docker/compose.yml up -d website ``` -Run the smoke scenario: +Run the smoke scenario from this stack directory: ```bash THE_INTERNET_BASE_URL=http://localhost:7080 pytest -m smoke --browser chromium @@ -48,7 +54,7 @@ ruff format --check . mypy src tests ``` -Stop the demo app when finished: +Stop the demo app from the repository root when finished: ```bash docker compose -f docker/compose.yml down diff --git a/stacks/ts-playwright/README.md b/stacks/ts-playwright/README.md index bdbd121..38a2a46 100644 --- a/stacks/ts-playwright/README.md +++ b/stacks/ts-playwright/README.md @@ -11,7 +11,10 @@ Playwright coverage for the shared scenario catalog. This stack covers the TypeS ## Install +Run the npm and Playwright commands in this guide from this stack directory. Docker Compose commands are the exception and run from the repository root, as each step below states. + ```bash +cd stacks/ts-playwright npm ci npx playwright install --with-deps chromium ``` @@ -24,7 +27,7 @@ Start the demo app from the repository root: docker compose -f docker/compose.yml up -d website ``` -Run the smoke scenario: +Run the smoke scenario from this stack directory: ```bash THE_INTERNET_BASE_URL=http://localhost:7080 npm run test:chromium:smoke @@ -52,7 +55,7 @@ npm run lint npm run format:check ``` -Stop the demo app when finished: +Stop the demo app from the repository root when finished: ```bash docker compose -f docker/compose.yml down diff --git a/tools/check-fast-start.py b/tools/check-fast-start.py new file mode 100644 index 0000000..3ea4367 --- /dev/null +++ b/tools/check-fast-start.py @@ -0,0 +1,103 @@ +#!/usr/bin/env python3 +"""Statically validate the root README fast-start block. + +The fast start is the first thing a new contributor runs, so a stale command +there is the most expensive kind of documentation rot. This walks the block and +checks that everything it names still exists: the compose file parses, the +directories are real, and the npm scripts are defined. + +It never starts a browser, installs anything, or runs a test. It only checks +that the commands refer to things that exist. +""" +from __future__ import annotations + +import json, os, re, shutil, subprocess, sys +from pathlib import Path + +README = Path("README.md") +SECTION = "## 3. Fast start" +ENV_PREFIX = re.compile(r"^(?:[A-Z_][A-Z0-9_]*=\S*\s+)+") + +def fast_start_commands() -> list[str]: + lines = README.read_text().splitlines() + if SECTION not in lines: + sys.exit(f"{README}: section '{SECTION}' not found; the fast-start check is anchored to it") + out: list[str] = [] + fenced = False + for line in lines[lines.index(SECTION) + 1:]: + if line.startswith("## "): + break + if line.startswith("```"): + fenced = line.strip() == "```bash" + continue + if fenced and line.strip(): + out.append(line.strip()) + if not out: + sys.exit(f"{README}: no ```bash block under '{SECTION}'") + return out + +def compose_file(cwd: Path, cmd: str, errors: list[str]) -> None: + m = re.search(r"-f\s+(\S+)", cmd) + if not m: + errors.append(f"docker compose without -f, cannot verify: {cmd}") + return + path = Path(os.path.normpath(cwd / m.group(1))) + if not path.is_file(): + errors.append(f"compose file referenced by the fast start does not exist: {path}") + return + service = cmd.split()[-1] if cmd.rstrip().endswith(("website",)) else None + if service and f"{service}:" not in path.read_text(): + errors.append(f"compose service '{service}' not defined in {path}") + if shutil.which("docker") is None: + print(f" skip {path} parse check (docker CLI not available)") + return + proc = subprocess.run(["docker", "compose", "-f", str(path), "config", "-q"], + capture_output=True, text=True) + if proc.returncode != 0: + errors.append(f"{path} does not parse: {proc.stderr.strip()}") + else: + print(f" ok {path} parses") + +def npm_script(cwd: Path, script: str, errors: list[str]) -> None: + pkg = cwd / "package.json" + if not pkg.is_file(): + errors.append(f"'npm run {script}' has no package.json in {cwd}") + return + scripts = json.loads(pkg.read_text()).get("scripts", {}) + if script not in scripts: + errors.append(f"npm script '{script}' is not defined in {pkg} (fast start would fail)") + else: + print(f" ok npm run {script} -> {scripts[script]}") + +def main() -> None: + errors: list[str] = [] + cwd = Path(".") + for raw in fast_start_commands(): + cmd = ENV_PREFIX.sub("", raw) + if cmd.startswith("cd "): + cwd = Path(os.path.normpath(cwd / cmd[3:].strip())) + if not cwd.is_dir(): + errors.append(f"fast start does 'cd' into a directory that does not exist: {cwd}") + else: + print(f" ok cd {cwd}") + elif cmd.startswith("docker compose"): + compose_file(cwd, cmd, errors) + elif cmd.startswith("npm run "): + npm_script(cwd, cmd.split()[2], errors) + elif cmd.startswith("npm ci"): + for f in ("package.json", "package-lock.json"): + if not (cwd / f).is_file(): + errors.append(f"'npm ci' in {cwd} requires {f}, which does not exist") + print(f" ok npm ci in {cwd}") + elif cmd.startswith("npx playwright install"): + pkg = cwd / "package.json" + if pkg.is_file() and "playwright" not in pkg.read_text(): + errors.append(f"'npx playwright install' in {cwd}, but playwright is not a dependency there") + print(f" ok npx playwright install in {cwd}") + else: + errors.append(f"fast-start command not understood by this check: {cmd!r}") + if errors: + sys.exit("README fast start is stale:\n" + "\n".join(f" - {e}" for e in errors)) + print("readme fast start ok: every command refers to something that exists") + +main() diff --git a/tools/check-scenarios.py b/tools/check-scenarios.py index f1582b3..d957f67 100755 --- a/tools/check-scenarios.py +++ b/tools/check-scenarios.py @@ -1,16 +1,30 @@ #!/usr/bin/env python3 from __future__ import annotations -import argparse, re, sys +import argparse, difflib, re, sys from pathlib import Path +from typing import cast CATALOG = Path("scenarios/catalog.yml") JAVA = Path("stacks/java-selenium-testng/src/test/java/theinternetwebsite/ui/testcases") TS = Path("stacks/ts-playwright/tests") PY = Path("stacks/python-playwright/tests") MATRIX = Path("docs/scenario-matrix.md") +README = Path("README.md") STACKS = ("java-selenium-testng", "ts-playwright", "python-playwright") SCENARIO_ID = r"(?:UI|HTTP)-[A-Z0-9-]+" +# Each stack must carry the scenario ID somewhere that means "this test is this +# scenario", not merely somewhere in the file. Matching the ID anywhere would let +# a mention in a comment stand in for a test. +JAVA_PATTERNS = (rf'testName\s*=\s*"({SCENARIO_ID})"',) +TS_PATTERNS = (rf"\btest(?:\.\w+)*\(\s*['\"`]\s*({SCENARIO_ID})\b",) +PY_PATTERNS = ( + rf'"""\s*({SCENARIO_ID})\b', + rf'@pytest\.mark\.\w+\(\s*[\'"]\s*({SCENARIO_ID})\b', + rf'parametrize\([^)]*[\'"]\s*({SCENARIO_ID})\b', +) +HEADER = "| ID | Priority | Scenario | Java | TS | Python |" +SEPARATOR = "| --- | --- | --- | --- | --- | --- |" def catalog() -> list[dict[str, object]]: rows: list[dict[str, object]] = [] @@ -24,23 +38,30 @@ def catalog() -> list[dict[str, object]]: rows[-1]["coverage"] = dict(re.findall(r"([\w-]+): (true|false)", s)) return rows +def coverage(row: dict[str, object]) -> dict[str, str]: + return cast(dict[str, str], row["coverage"]) + +def ids(root: Path, glob: str, patterns: tuple[str, ...]) -> set[str]: + if not root.exists(): return set() + text = "\n".join(p.read_text() for p in root.glob(glob)) + found: set[str] = set() + for pattern in patterns: found |= set(re.findall(pattern, text)) + return found + def java_ids() -> set[str]: - text = "\n".join(p.read_text() for p in JAVA.glob("*.java")) - return set(re.findall(rf'testName\s*=\s*"({SCENARIO_ID})"', text)) + return ids(JAVA, "*.java", JAVA_PATTERNS) def ts_ids() -> set[str]: - text = "\n".join(p.read_text() for p in TS.glob("**/*.ts")) if TS.exists() else "" - return set(re.findall(rf"\b({SCENARIO_ID})\b", text)) + return ids(TS, "**/*.ts", TS_PATTERNS) def py_ids() -> set[str]: - text = "\n".join(p.read_text() for p in PY.glob("**/*.py")) if PY.exists() else "" - return set(re.findall(rf"\b({SCENARIO_ID})\b", text)) + return ids(PY, "**/*.py", PY_PATTERNS) def check(rows: list[dict[str, object]]) -> None: ids = [str(r["id"]) for r in rows]; all_ids = set(ids); java_tests = java_ids(); ts_tests = ts_ids(); py_tests = py_ids() - java_catalog = {str(r["id"]) for r in rows if dict(r["coverage"])["java-selenium-testng"] == "true"} - ts_catalog = {str(r["id"]) for r in rows if dict(r["coverage"])["ts-playwright"] == "true"} - py_catalog = {str(r["id"]) for r in rows if dict(r["coverage"])["python-playwright"] == "true"} + java_catalog = {str(r["id"]) for r in rows if coverage(r)["java-selenium-testng"] == "true"} + ts_catalog = {str(r["id"]) for r in rows if coverage(r)["ts-playwright"] == "true"} + py_catalog = {str(r["id"]) for r in rows if coverage(r)["python-playwright"] == "true"} errors = [] errors += [f"duplicate catalog ids: {sorted({i for i in ids if ids.count(i) > 1})}"] if len(ids) != len(all_ids) else [] errors += [f"catalog java ids missing tests: {sorted(java_catalog - java_tests)}"] if java_catalog - java_tests else [] @@ -54,14 +75,33 @@ def check(rows: list[dict[str, object]]) -> None: errors += [f"py test ids not marked covered: {sorted(py_tests - py_catalog)}"] if py_tests - py_catalog else [] if errors: sys.exit("\n".join(errors)) -def write_matrix(rows: list[dict[str, object]]) -> None: - out = ["# Scenario coverage matrix", "", "Generated by `tools/check-scenarios.py --write-matrix`.", "", "| ID | Priority | Scenario | Java | TS | Python |", "| --- | --- | --- | --- | --- | --- |"] +def table(rows: list[dict[str, object]]) -> list[str]: + out = [HEADER, SEPARATOR] for r in rows: - c = dict(r["coverage"]); marks = ["✅" if c[s] == "true" else "—" for s in STACKS] + c = coverage(r); marks = ["✅" if c[s] == "true" else "—" for s in STACKS] out.append(f"| {r['id']} | {r['priority']} | {r['title']} | {marks[0]} | {marks[1]} | {marks[2]} |") + return out + +MARKER = "" +LEGEND = "Legend: ✅ = covered by that stack · — = not covered." + +def write_matrix(rows: list[dict[str, object]]) -> None: + out = ["# Scenario coverage matrix", "", MARKER, "", "Generated by `tools/check-scenarios.py --write-matrix`.", "", LEGEND, ""] + table(rows) MATRIX.parent.mkdir(exist_ok=True); MATRIX.write_text("\n".join(out) + "\n") +def check_readme(rows: list[dict[str, object]]) -> None: + lines = README.read_text().splitlines() + if HEADER not in lines: + sys.exit(f"{README}: scenario-matrix embed not found; expected a table headed\n {HEADER}") + start = lines.index(HEADER); end = start + while end < len(lines) and lines[end].startswith("|"): end += 1 + embedded = lines[start:end]; expected = table(rows) + if embedded != expected: + diff = "\n".join(difflib.unified_diff(embedded, expected, f"{README} (embedded copy)", f"generated from {CATALOG}", lineterm="")) + sys.exit(f"{README} scenario-matrix embed does not match the catalog:\n{diff}\n\nfix: copy the table out of {MATRIX} into the scenario-matrix section of {README}") + parser = argparse.ArgumentParser(); parser.add_argument("--write-matrix", action="store_true") args = parser.parse_args(); rows = catalog(); check(rows) if args.write_matrix: write_matrix(rows) -print(f"scenario catalog ok: {len(rows)} rows, {len(java_ids())} java tests, {len(ts_ids())} ts tests, {len(py_ids())} py tests") +check_readme(rows) +print(f"scenario catalog ok: {len(rows)} rows, {len(java_ids())} java tests, {len(ts_ids())} ts tests, {len(py_ids())} py tests; readme embed in sync")