From 06e27169814095ffebd58e22be3fe4190bc1c91a Mon Sep 17 00:00:00 2001 From: Dominik Franczyk Date: Mon, 20 Jul 2026 15:25:26 +0200 Subject: [PATCH 1/2] feat: run existing pytest suites --- .github/workflows/ci.yml | 7 + .github/workflows/release.yml | 7 + CHANGELOG.md | 7 + README.md | 67 +++++-- docs/architecture.md | 21 +- docs/benchmarking.md | 13 ++ docs/benchmarks/results.md | 3 +- docs/for-llms.md | 2 + docs/getting-started.md | 29 ++- docs/guides/pytest-compatibility.md | 118 +++++++++++ docs/index.md | 38 ++-- docs/llms-full.txt | 296 +++++++++++++++++++++++++--- docs/llms.txt | 4 +- docs/performance-analysis.md | 11 +- docs/reference/cli.md | 41 +++- docs/reference/configuration.md | 4 + docs/roadmap.md | 4 +- llms-full.txt | 296 +++++++++++++++++++++++++--- llms.txt | 4 +- pyproject.toml | 6 +- scripts/generate_docs_assets.py | 12 +- src/testenix/cli.py | 52 ++++- src/testenix/pytest_adapter.py | 87 ++++++++ tests/test_pytest_adapter.py | 272 +++++++++++++++++++++++++ uv.lock | 9 +- 25 files changed, 1323 insertions(+), 87 deletions(-) create mode 100644 docs/guides/pytest-compatibility.md create mode 100644 src/testenix/pytest_adapter.py create mode 100644 tests/test_pytest_adapter.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2e46719..b0b09e5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -56,11 +56,18 @@ jobs: - run: uv run --no-editable ruff format --check . - run: uv run --no-editable mypy - run: uv run --no-editable pyright --pythonversion 3.11 src/testenix + - run: uv run --no-editable testenix pytest -q - run: uv build --no-sources - run: uv run twine check --strict dist/* - run: uv run check-wheel-contents dist/*.whl - run: uv run --isolated --no-project --with ./dist/testenix-*.whl testenix --version - run: uv run --isolated --no-project --with ./dist/testenix-*.whl testenix run examples/basic --no-history + - name: Verify pytest extra from wheel + shell: bash + run: | + wheel_path="$(find dist -maxdepth 1 -type f -name '*.whl' -print -quit)" + uv run --isolated --no-project --with "testenix[pytest] @ $wheel_path" testenix pytest --version + uv run --isolated --no-project --with "testenix[pytest] @ $wheel_path" --with "pytest==8.3.0" testenix pytest --version - run: uv run --isolated --no-project --with ./dist/testenix-*.tar.gz testenix --version - run: uv run --isolated --no-project --with ./dist/testenix-*.tar.gz testenix run examples/basic --no-history diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 1e53b87..d564190 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -40,6 +40,7 @@ jobs: fi - run: uv sync --locked --dev --no-editable - run: uv run --no-editable pytest + - run: uv run --no-editable testenix pytest -q - run: uv run --no-editable ruff check . - run: uv run --no-editable ruff format --check . - run: uv run --no-editable mypy @@ -49,6 +50,12 @@ jobs: - run: uv run check-wheel-contents dist/*.whl - run: uv run --isolated --no-project --with ./dist/testenix-*.whl testenix --version - run: uv run --isolated --no-project --with ./dist/testenix-*.whl testenix run examples/basic --no-history + - name: Verify pytest extra from wheel + shell: bash + run: | + wheel_path="$(find dist -maxdepth 1 -type f -name '*.whl' -print -quit)" + uv run --isolated --no-project --with "testenix[pytest] @ $wheel_path" testenix pytest --version + uv run --isolated --no-project --with "testenix[pytest] @ $wheel_path" --with "pytest==8.3.0" testenix pytest --version - run: uv run --isolated --no-project --with ./dist/testenix-*.tar.gz testenix --version - run: uv run --isolated --no-project --with ./dist/testenix-*.tar.gz testenix run examples/basic --no-history - name: Store distributions diff --git a/CHANGELOG.md b/CHANGELOG.md index 1b743ff..48fa92d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,13 @@ project intends to use Semantic Versioning once its public API reaches stability ## [Unreleased] +### Added + +- `testenix pytest [PYTEST_ARGS ...]` compatibility bridge for unchanged pytest suites, preserving + the real pytest collector, fixtures, parametrization, markers, plugins, output, and exit status. +- Optional `testenix[pytest]` installation extra and a documented native-versus-compatibility + capability matrix. + ## [0.1.0] - 2026-07-20 ### Added diff --git a/README.md b/README.md index c5d330f..55589c0 100644 --- a/README.md +++ b/README.md @@ -16,8 +16,9 @@ Testenix is an experimental native Python testing framework built around five gu 4. retries never erase previous failures; 5. every report is derived from one versioned, lossless result model. -The project is currently an alpha. Its runtime has no third-party dependencies and does not depend -on pytest. +The project is currently an alpha. Its native runtime has no third-party dependencies and does not +depend on pytest. An optional compatibility bridge can delegate unchanged pytest suites to the +pytest installation already present in a project. ## Installation @@ -27,6 +28,15 @@ Install the published package with: python -m pip install testenix ``` +For an existing pytest project, install the optional convenience extra: + +```bash +python -m pip install "testenix[pytest]" +``` + +If a supported pytest (`>=8.3,<10`) is already installed in the same environment, the base +`testenix` package is sufficient. + Testenix requires Python 3.11 or newer. The project is currently an alpha; pin the version before using it in CI. Until the first PyPI release is visible, use the GitHub installation below. @@ -42,9 +52,36 @@ You can also install the latest source directly from GitHub: ```bash python -m pip install "testenix @ git+https://github.com/polishdataengineer/testenix.git@main" +# include pytest when the project environment does not already provide it +python -m pip install "testenix[pytest] @ git+https://github.com/polishdataengineer/testenix.git@main" +``` + +## Run an existing pytest suite + +Yes. Testenix provides a transparent compatibility bridge for existing pytest projects: + +```bash +testenix pytest -q tests +``` + +Everything after `testenix pytest` is forwarded unchanged to the same interpreter as +`python -m pytest`. This preserves pytest collection, `conftest.py`, fixtures, parametrization, +markers, assertion rewriting, plugins, configuration, output, node IDs, and exit codes. + +```bash +testenix pytest tests/test_api.py::test_health -k smoke --maxfail=1 +testenix pytest -n auto tests # requires pytest-xdist in the same environment +testenix pytest --junitxml=reports/pytest.xml tests ``` -## Quick start +This command is a compatibility and migration bridge. It does not use the native Testenix +collector, scheduler, worker pool, retries, history, result model, or reporters. Its performance is +therefore pytest performance plus launcher and adapter overhead; the native benchmark results do +not apply to it. See the +[pytest compatibility guide](https://polishdataengineer.github.io/testenix/guides/pytest-compatibility/) +for the complete capability matrix and migration choices. + +## Native quick start Create `tests/test_multiplication.py` with this complete example: @@ -138,10 +175,13 @@ test. ## Where Testenix is deliberately different -Testenix is not yet a drop-in replacement for pytest. Its v0.1 value is a smaller, coherent native -stack: async tests and async fixtures need no plugin, parallel execution and duration-aware -scheduling need no xdist, every retry remains visible, and a worker crash cannot silently erase -tests that completed before it. The runtime has no third-party dependencies. +The `testenix run` engine is not a native drop-in reimplementation of pytest. Its v0.1 value is a +smaller, coherent native stack: async tests and async fixtures need no plugin, parallel execution +and duration-aware scheduling need no xdist, every retry remains visible, and a worker crash cannot +silently erase tests that completed before it. The native runtime has no third-party dependencies. + +For unchanged pytest semantics, use `testenix pytest`. That bridge intentionally delegates to +pytest instead of silently approximating fixtures, markers, plugins, or hook behavior. The trade-off is ecosystem maturity. pytest currently has much broader plugin, IDE, assertion rewriting, and migration support. Testenix should only claim to be better for the guarantees above @@ -161,10 +201,11 @@ can copy its own text or the complete project reference for an LLM. ## Benchmarks -In the checked-in M4 Pro/CPython 3.11 synthetic baseline, Testenix completed 100,000 empty tests -across 16 modules in a median 8.04 seconds, compared with 25.33 seconds for pytest and 21.30 seconds -for pytest-xdist. That is 3.15x the throughput of pytest for this specific workload, not a universal -performance promise. The result includes one warm-up and five measured, counterbalanced rounds. +In the checked-in M4 Pro/CPython 3.11 synthetic baseline, native `testenix run` completed 100,000 +empty tests across 16 modules in a median 8.04 seconds, compared with 25.33 seconds for pytest and +21.30 seconds for pytest-xdist. That is 3.15x the throughput of pytest for this specific workload, +not a universal performance promise. The result includes one warm-up and five measured, +counterbalanced rounds. It does not describe `testenix pytest`, which executes through pytest. See the [generated results and chart](https://polishdataengineer.github.io/testenix/benchmarks/results/), [raw JSON](https://github.com/polishdataengineer/testenix/tree/main/benchmarks), @@ -193,7 +234,9 @@ See the [generated results and chart](https://polishdataengineer.github.io/teste - On Windows, a script that calls the programmatic `run()`/`run_async()` API must use the standard `if __name__ == "__main__":` multiprocessing guard. The `testenix` CLI handles process startup itself. -- Test impact analysis, result caching, remote workers, and the pytest migration adapter are not +- The pytest bridge does not translate delegated outcomes into Testenix `RunResult`, JSON, history, + retry, timeout, or scheduling semantics. Use pytest's own flags and installed plugins in that mode. +- Test impact analysis, result caching, remote workers, and deep pytest-result aggregation are not part of version 0.1. ## Project status diff --git a/docs/architecture.md b/docs/architecture.md index cdf7893..35e18a2 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -4,6 +4,22 @@ Testenix is a native Python testing framework. Its core does not depend on pytes Compatibility adapters may translate foreign test frameworks into the same manifest and event contracts, but they are not part of native execution. +The first pytest compatibility bridge deliberately does not translate pytest internals. On POSIX, +`testenix pytest` uses `os.execv` to replace itself with `sys.executable -m pytest`. On Windows it +calls pytest's public `console_main` entry point in the existing process because the platform's +`exec` family does not provide equivalent replacement semantics. Both paths keep pytest in the +foreground CLI process with the same working directory, environment, terminal, streams, signal +handling, and exit status. This preserves semantics without making the native core depend on +pytest: + +```text +POSIX: testenix pytest ==exec==> Python -m pytest -> collector/plugins/executor -> output/status +Windows: testenix pytest =========> pytest.console_main -> collector/plugins/executor -> output/status +``` + +The bridge is a CLI infrastructure adapter, not a native collection adapter. It does not emit +Testenix events or construct a `RunResult` in version 0.1. + ## Product contract Testenix aims to be typed, async-native, parallel-first, deterministic, and lossless when reporting @@ -25,7 +41,7 @@ Authoring API -> supervised collection -> inert manifest -> affinity scheduler - - `events`, `aggregate`, and `scheduler` remain engine-independent. - `runner` is the application service connecting the native engine with execution policy. - reporters and storage consume completed domain results or versioned events. -- optional compatibility adapters depend inward; the core never imports them. +- optional compatibility adapters stay at the CLI boundary; the native core never imports pytest. ## Version 0.1 scope @@ -37,7 +53,8 @@ Authoring API -> supervised collection -> inert manifest -> affinity scheduler - - deterministic scheduling based on historical durations; - append-only JSONL events and a pure reducer; - console, JSON, and JUnit output plus local SQLite duration history; -- retries represented as immutable attempts and finalized as `FLAKY` when appropriate. +- retries represented as immutable attempts and finalized as `FLAKY` when appropriate; +- an optional platform-aware pytest handoff for unchanged legacy suites. Remote workers, distributed storage, result caching, automatic quarantine, and a stable third-party plugin SDK are deliberately outside version 0.1. diff --git a/docs/benchmarking.md b/docs/benchmarking.md index 5968aec..ca68faf 100644 --- a/docs/benchmarking.md +++ b/docs/benchmarking.md @@ -30,6 +30,19 @@ files. Correctness wins over speed: a run with a missing, duplicated, or incorrectly finalized result is invalid and excluded from performance comparisons. +## Pytest compatibility bridge + +Measurements of `testenix pytest` must be reported separately from native `testenix run` +measurements. The compatibility command hands the current interpreter to pytest through a POSIX +process overlay or pytest's in-process console entry point on Windows; it does not execute tests +through the Testenix engine. + +A compatibility-overhead comparison must use the same interpreter, working directory, +environment, pytest configuration, plugins, and arguments for both `python -m pytest ...` and +`testenix pytest ...`. Any difference measures adapter overhead only and must not be presented as a +Testenix execution speedup. Native comparisons continue to use `testenix run` and must validate +that both runners execute the same tests and produce equivalent outcomes. + Run the reproducible local harness with: ```bash diff --git a/docs/benchmarks/results.md b/docs/benchmarks/results.md index c821f76..ecd21c7 100644 --- a/docs/benchmarks/results.md +++ b/docs/benchmarks/results.md @@ -2,7 +2,8 @@ These tables are generated from the raw JSON committed in `benchmarks/`. They are development evidence for specific synthetic workloads, not a universal claim that Testenix is always faster -than pytest. +than pytest. `Testenix` in these results means the native `testenix run` engine. The +`testenix pytest` compatibility bridge delegates to pytest and is not represented here. ![Preliminary Testenix throughput ratios](../_static/benchmark-speedup.svg) diff --git a/docs/for-llms.md b/docs/for-llms.md index e757614..4b6d647 100644 --- a/docs/for-llms.md +++ b/docs/for-llms.md @@ -11,6 +11,7 @@ context. Every documentation page also includes **Copy this page** and | [`llms.txt`](https://polishdataengineer.github.io/testenix/llms.txt) | A model should first discover the project and choose relevant pages. | | [`llms-full.txt`](https://polishdataengineer.github.io/testenix/llms-full.txt) | You want one self-contained reference with guides, API, architecture, and benchmark context. | | [Python API reference](reference/api.md) | The task is specifically about authoring or embedding Testenix. | +| [Pytest compatibility](guides/pytest-compatibility.md) | A model must choose between delegation and native migration. | | [Benchmark results](benchmarks/results.md) | A model needs to evaluate or repeat performance claims. | `llms.txt` follows the emerging llms.txt proposal, but it should be treated as a convenience @@ -29,6 +30,7 @@ workload-specific and preserve all documented limitations. ## What the full reference contains - installation and first-run instructions; +- the pytest compatibility bridge, capability matrix, and migration boundary; - native tests, cases, tags, skips, expected failures, and fixtures; - parallelism, timeouts, retries, crash recovery, reports, and history; - CLI, configuration, and generated Python API reference; diff --git a/docs/getting-started.md b/docs/getting-started.md index 8c2e41f..31b951e 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -6,7 +6,8 @@ This guide takes a new project from installation to its first parallel Testenix - CPython 3.11 or newer - Linux, macOS, or Windows -- no runtime dependencies beyond the Python standard library +- no runtime dependencies beyond the Python standard library for native `testenix run` +- pytest in the same environment when using the optional compatibility bridge ## Install @@ -24,13 +25,36 @@ $ uv add --dev testenix $ uv run testenix --version ``` +For an existing pytest project, install the compatibility extra: + +```console +$ python -m pip install "testenix[pytest]" +# or +$ uv add --dev "testenix[pytest]" +``` + Until the first PyPI release is visible, install directly from the protected `main` branch: ```console $ python -m pip install "testenix @ git+https://github.com/polishdataengineer/testenix.git@main" +# include pytest when the project environment does not already provide it +$ python -m pip install "testenix[pytest] @ git+https://github.com/polishdataengineer/testenix.git@main" +``` + +## Choose an execution mode + +Run an unchanged pytest suite through its real engine: + +```console +$ testenix pytest -q tests ``` -## Create a test +Use `testenix run` for native Testenix tests and the built-in scheduler, retries, history, and +lossless reports. The two commands have deliberately separate semantics. See +[pytest compatibility](guides/pytest-compatibility.md) for the capability matrix and migration +boundary. + +## Create a native test Testenix collects ordinary functions whose names begin with `test_`. Decorators are optional for simple tests. @@ -110,6 +134,7 @@ collection and execution process trees before control returns to the caller. ## Next steps - [Write tests, cases, tags, skips, and expected failures](guides/writing-tests.md) +- [Run or migrate an existing pytest suite](guides/pytest-compatibility.md) - [Build fixture graphs](guides/fixtures.md) - [Understand process parallelism and timeouts](guides/parallelism.md) - [Produce JSON and JUnit reports](guides/reports.md) diff --git a/docs/guides/pytest-compatibility.md b/docs/guides/pytest-compatibility.md new file mode 100644 index 0000000..f0150c5 --- /dev/null +++ b/docs/guides/pytest-compatibility.md @@ -0,0 +1,118 @@ +--- +description: Run existing pytest suites unchanged through Testenix and understand the native migration boundary. +--- + +# Pytest compatibility + +Testenix can run an existing pytest suite without rewriting it. Use the compatibility bridge when +the suite depends on pytest fixtures, parametrization, markers, classes, plugins, hooks, or +configuration: + +```console +$ python -m pip install "testenix[pytest]" +$ testenix pytest -q tests +``` + +If a supported pytest (`>=8.3,<10`) is already installed in the same Python environment, installing +the base `testenix` package is enough. The extra is a convenience for environments that do not +already contain pytest. + +## What the command does + +Everything after `testenix pytest` is forwarded unchanged to: + +```console +$ python -m pytest [PYTEST_ARGS ...] +``` + +Testenix hands its current CLI process to pytest from the same Python interpreter. On POSIX it uses +a process overlay; on Windows it calls pytest's public console entry point in-process because the +platform does not provide the same overlay semantics. Both paths preserve the foreground process, +working directory, environment, terminal, standard streams, and pytest's signal handling. Pytest +therefore remains responsible for collection, execution, configuration, plugin loading, output, +descendants such as pytest-xdist workers, and exit status. + +```console +$ testenix pytest tests/test_api.py::test_health -k smoke --maxfail=1 +$ testenix pytest -m "unit and not slow" tests +$ testenix pytest -n auto tests +$ testenix pytest --junitxml=reports/pytest.xml tests +``` + +The `-n` example requires pytest-xdist in the same environment. Testenix does not translate its +native `--workers` option into an xdist option. + +## Capability matrix + +| Capability | `testenix pytest` | `testenix run` | +| --- | --- | --- | +| Plain module-level `test_*` functions | Yes, through pytest | Yes | +| Pytest classes and unittest-style tests | Yes | No | +| `pytest.fixture` and built-in fixtures | Yes | No | +| `conftest.py` fixture and hook discovery | Yes | No | +| `pytest.mark.parametrize` | Yes | No; use Testenix `@case` or `@cases` | +| Pytest skip, xfail, and custom markers | Yes | No; use Testenix decorators and tags | +| Pytest assertion rewriting | Yes | No | +| Pytest plugins and hooks | Yes, when installed in the same environment | No | +| `pytest.ini` and `[tool.pytest.ini_options]` | Yes | No | +| Pytest node IDs and CLI selectors | Yes | No | +| pytest-xdist | Yes, when installed and requested with `-n` | No; use native `--workers` | +| Async tests | According to the installed pytest plugins | Native, without a plugin | +| Testenix retries and `FLAKY` semantics | No | Yes | +| Testenix duration history and scheduling | No | Yes | +| Testenix lossless result model | No | Yes | +| Testenix JSON/JUnit reporters | No; use pytest options or plugins | Yes | +| Published native Testenix speedups | No | Only for the documented workloads | +| Exit status | Unchanged pytest or plugin status | Testenix exit-code contract | + +Some simple pytest-authored functions also happen to run with `testenix run`, but that overlap is +not a compatibility guarantee. In particular, the native collector does not interpret pytest +markers. Running a pytest suite through the native command can turn a skip or xfail into an +ordinary execution, collapse parametrized cases, miss class-based tests, or fail fixture setup. +Use the explicit compatibility command until a suite has been intentionally migrated. + +## Boundaries + +`testenix pytest` does not use `[tool.testenix]`, the native collector, worker pool, retries, +timeouts, tags, history, event model, JSON reporter, or JUnit reporter. Pass the corresponding +pytest or plugin options after the subcommand. For example, use pytest's `--junitxml`, not +Testenix's native `--junit`. + +Pytest and every required plugin must be installed beside the `testenix` executable in the same +interpreter environment. For uv-managed projects, prefer: + +```console +$ uv add --dev "testenix[pytest]" +$ uv run testenix pytest -q tests +``` + +An isolated `uv tool install testenix` environment does not automatically see pytest plugins from +the project environment. Install the required packages into the tool environment or invoke +Testenix through the project environment instead. + +After the handoff, pytest owns signal handling and returns any status chosen by pytest or a plugin, +unchanged. If pytest is missing, Testenix returns `2` with an installation hint before the handoff. +Failure to hand the process to pytest returns `3`. + +## Performance and benchmark interpretation + +Compatibility mode has pytest's execution performance plus launcher and adapter overhead, which has +not yet been measured separately. It is not expected to be faster than invoking `python -m pytest` +directly. The published Testenix benchmarks exercise the native `testenix run` engine and must not +be used to describe `testenix pytest`. + +## Migration path + +Use the bridge first to put Testenix in front of an unchanged suite without altering its semantics. +Migrate individual modules to native Testenix only when their pytest dependencies have explicit +equivalents: + +1. keep the whole suite green with `testenix pytest`; +2. replace pytest fixtures with Testenix `@fixture` providers; +3. replace `parametrize` with `@case` or `@cases`; +4. replace pytest markers with Testenix tags, `@skip`, and `@xfail`; +5. run migrated modules with `testenix run` and leave the remainder on the bridge; +6. compare behavior before making native performance claims. + +The current bridge does not convert pytest outcomes into a Testenix `RunResult`. Deeper event and +report aggregation is a future compatibility layer and requires explicit pytest hook integration. diff --git a/docs/index.md b/docs/index.md index 10300d2..680233f 100644 --- a/docs/index.md +++ b/docs/index.md @@ -8,11 +8,12 @@ description: Testenix is an async-native, parallel-first Python testing framewor
Python testing framework · Alpha
Fast tests. Clear results.

- Testenix unifies synchronous and asynchronous tests, fixtures, local parallelism, - retries, crash recovery, and machine-readable reports in one dependency-free runtime. + Testenix combines a dependency-free native runtime with a transparent bridge for + running existing pytest suites unchanged.

Start testing + Run pytest suites See the benchmarks Copy docs for an LLM
@@ -23,6 +24,7 @@ description: Testenix is an async-native, parallel-first Python testing framewor :maxdepth: 2 getting-started +guides/pytest-compatibility guides/writing-tests guides/fixtures guides/parallelism @@ -41,10 +43,10 @@ for-llms ## Why Testenix
-
0third-party runtime dependencies
+
0dependencies in the native runtime
12Python and OS combinations in CI
3console, JSON, and JUnit reports
-
3.15×100k synthetic throughput vs pytest
+
3.15×native testenix run on the 100k synthetic workload vs pytest
Testenix is deliberately built around a few strong guarantees: @@ -62,6 +64,17 @@ Testenix is deliberately built around a few strong guarantees: ## A complete first test +Already have a pytest suite? Keep its fixtures, parametrization, markers, classes, configuration, +and plugins: + +```console +$ python -m pip install "testenix[pytest]" +$ testenix pytest -q tests +``` + +[Read the compatibility contract](guides/pytest-compatibility/) before migrating individual +modules to the native engine. + ```python from collections.abc import AsyncIterator @@ -94,9 +107,10 @@ If the first PyPI release is not available yet, install the current source with ## Performance evidence, with context -The checked-in development baseline measured 100,000 empty tests across 16 generated modules on -an Apple M4 Pro and CPython 3.11. Testenix completed that specific workload in a median 8.04 -seconds, compared with 25.33 seconds for pytest and 21.30 seconds for pytest-xdist. +The checked-in development baseline measured native `testenix run` on 100,000 empty tests across +16 generated modules on an Apple M4 Pro and CPython 3.11. Native Testenix completed that specific +workload in a median 8.04 seconds, compared with 25.33 seconds for pytest and 21.30 seconds for +pytest-xdist. These measurements do not apply to the delegated `testenix pytest` command.
This is a preliminary synthetic result from one machine, not a promise that every project will be @@ -109,7 +123,9 @@ and limitations so that the claim can be evaluated rather than taken on trust. ## Project maturity -Testenix is alpha software and is not yet a drop-in pytest replacement. Pytest has a much broader -plugin ecosystem, richer IDE integration, and mature assertion rewriting. Testenix is a good fit -when its native async model, built-in parallel execution, explicit failure semantics, or -dependency-free runtime are more important than ecosystem compatibility. +Testenix is alpha software. The `testenix pytest` bridge preserves an existing suite by delegating +to real pytest, while `testenix run` is a distinct native engine rather than a drop-in pytest +reimplementation. Pytest still has a much broader plugin ecosystem, richer IDE integration, and +mature assertion rewriting. Choose the bridge for compatibility and the native engine when its +async model, built-in parallel execution, explicit failure semantics, or dependency-free core are +more important. diff --git a/docs/llms-full.txt b/docs/llms-full.txt index 5e8659e..6ec4b3f 100644 --- a/docs/llms-full.txt +++ b/docs/llms-full.txt @@ -22,11 +22,12 @@ description: Testenix is an async-native, parallel-first Python testing framewor
Python testing framework · Alpha
Fast tests. Clear results.

- Testenix unifies synchronous and asynchronous tests, fixtures, local parallelism, - retries, crash recovery, and machine-readable reports in one dependency-free runtime. + Testenix combines a dependency-free native runtime with a transparent bridge for + running existing pytest suites unchanged.

@@ -37,6 +38,7 @@ description: Testenix is an async-native, parallel-first Python testing framewor :maxdepth: 2 getting-started +guides/pytest-compatibility guides/writing-tests guides/fixtures guides/parallelism @@ -55,10 +57,10 @@ for-llms ## Why Testenix
-
0third-party runtime dependencies
+
0dependencies in the native runtime
12Python and OS combinations in CI
3console, JSON, and JUnit reports
-
3.15×100k synthetic throughput vs pytest
+
3.15×native testenix run on the 100k synthetic workload vs pytest
Testenix is deliberately built around a few strong guarantees: @@ -76,6 +78,17 @@ Testenix is deliberately built around a few strong guarantees: ## A complete first test +Already have a pytest suite? Keep its fixtures, parametrization, markers, classes, configuration, +and plugins: + +```console +$ python -m pip install "testenix[pytest]" +$ testenix pytest -q tests +``` + +[Read the compatibility contract](https://polishdataengineer.github.io/testenix/guides/pytest-compatibility/) before migrating individual +modules to the native engine. + ```python from collections.abc import AsyncIterator @@ -108,9 +121,10 @@ If the first PyPI release is not available yet, install the current source with ## Performance evidence, with context -The checked-in development baseline measured 100,000 empty tests across 16 generated modules on -an Apple M4 Pro and CPython 3.11. Testenix completed that specific workload in a median 8.04 -seconds, compared with 25.33 seconds for pytest and 21.30 seconds for pytest-xdist. +The checked-in development baseline measured native `testenix run` on 100,000 empty tests across +16 generated modules on an Apple M4 Pro and CPython 3.11. Native Testenix completed that specific +workload in a median 8.04 seconds, compared with 25.33 seconds for pytest and 21.30 seconds for +pytest-xdist. These measurements do not apply to the delegated `testenix pytest` command.
This is a preliminary synthetic result from one machine, not a promise that every project will be @@ -123,10 +137,12 @@ and limitations so that the claim can be evaluated rather than taken on trust. ## Project maturity -Testenix is alpha software and is not yet a drop-in pytest replacement. Pytest has a much broader -plugin ecosystem, richer IDE integration, and mature assertion rewriting. Testenix is a good fit -when its native async model, built-in parallel execution, explicit failure semantics, or -dependency-free runtime are more important than ecosystem compatibility. +Testenix is alpha software. The `testenix pytest` bridge preserves an existing suite by delegating +to real pytest, while `testenix run` is a distinct native engine rather than a drop-in pytest +reimplementation. Pytest still has a much broader plugin ecosystem, richer IDE integration, and +mature assertion rewriting. Choose the bridge for compatibility and the native engine when its +async model, built-in parallel execution, explicit failure semantics, or dependency-free core are +more important. --- @@ -143,7 +159,8 @@ This guide takes a new project from installation to its first parallel Testenix - CPython 3.11 or newer - Linux, macOS, or Windows -- no runtime dependencies beyond the Python standard library +- no runtime dependencies beyond the Python standard library for native `testenix run` +- pytest in the same environment when using the optional compatibility bridge ## Install @@ -161,13 +178,36 @@ $ uv add --dev testenix $ uv run testenix --version ``` +For an existing pytest project, install the compatibility extra: + +```console +$ python -m pip install "testenix[pytest]" +# or +$ uv add --dev "testenix[pytest]" +``` + Until the first PyPI release is visible, install directly from the protected `main` branch: ```console $ python -m pip install "testenix @ git+https://github.com/polishdataengineer/testenix.git@main" +# include pytest when the project environment does not already provide it +$ python -m pip install "testenix[pytest] @ git+https://github.com/polishdataengineer/testenix.git@main" +``` + +## Choose an execution mode + +Run an unchanged pytest suite through its real engine: + +```console +$ testenix pytest -q tests ``` -## Create a test +Use `testenix run` for native Testenix tests and the built-in scheduler, retries, history, and +lossless reports. The two commands have deliberately separate semantics. See +[pytest compatibility](https://polishdataengineer.github.io/testenix/guides/pytest-compatibility/) for the capability matrix and migration +boundary. + +## Create a native test Testenix collects ordinary functions whose names begin with `test_`. Decorators are optional for simple tests. @@ -247,6 +287,7 @@ collection and execution process trees before control returns to the caller. ## Next steps - [Write tests, cases, tags, skips, and expected failures](https://polishdataengineer.github.io/testenix/guides/writing-tests/) +- [Run or migrate an existing pytest suite](https://polishdataengineer.github.io/testenix/guides/pytest-compatibility/) - [Build fixture graphs](https://polishdataengineer.github.io/testenix/guides/fixtures/) - [Understand process parallelism and timeouts](https://polishdataengineer.github.io/testenix/guides/parallelism/) - [Produce JSON and JUnit reports](https://polishdataengineer.github.io/testenix/guides/reports/) @@ -254,6 +295,132 @@ collection and execution process trees before control returns to the caller. --- +# Document: Pytest compatibility + +Canonical URL: https://polishdataengineer.github.io/testenix/guides/pytest-compatibility/ +Source: docs/guides/pytest-compatibility.md + +--- +description: Run existing pytest suites unchanged through Testenix and understand the native migration boundary. +--- + +# Pytest compatibility + +Testenix can run an existing pytest suite without rewriting it. Use the compatibility bridge when +the suite depends on pytest fixtures, parametrization, markers, classes, plugins, hooks, or +configuration: + +```console +$ python -m pip install "testenix[pytest]" +$ testenix pytest -q tests +``` + +If a supported pytest (`>=8.3,<10`) is already installed in the same Python environment, installing +the base `testenix` package is enough. The extra is a convenience for environments that do not +already contain pytest. + +## What the command does + +Everything after `testenix pytest` is forwarded unchanged to: + +```console +$ python -m pytest [PYTEST_ARGS ...] +``` + +Testenix hands its current CLI process to pytest from the same Python interpreter. On POSIX it uses +a process overlay; on Windows it calls pytest's public console entry point in-process because the +platform does not provide the same overlay semantics. Both paths preserve the foreground process, +working directory, environment, terminal, standard streams, and pytest's signal handling. Pytest +therefore remains responsible for collection, execution, configuration, plugin loading, output, +descendants such as pytest-xdist workers, and exit status. + +```console +$ testenix pytest tests/test_api.py::test_health -k smoke --maxfail=1 +$ testenix pytest -m "unit and not slow" tests +$ testenix pytest -n auto tests +$ testenix pytest --junitxml=reports/pytest.xml tests +``` + +The `-n` example requires pytest-xdist in the same environment. Testenix does not translate its +native `--workers` option into an xdist option. + +## Capability matrix + +| Capability | `testenix pytest` | `testenix run` | +| --- | --- | --- | +| Plain module-level `test_*` functions | Yes, through pytest | Yes | +| Pytest classes and unittest-style tests | Yes | No | +| `pytest.fixture` and built-in fixtures | Yes | No | +| `conftest.py` fixture and hook discovery | Yes | No | +| `pytest.mark.parametrize` | Yes | No; use Testenix `@case` or `@cases` | +| Pytest skip, xfail, and custom markers | Yes | No; use Testenix decorators and tags | +| Pytest assertion rewriting | Yes | No | +| Pytest plugins and hooks | Yes, when installed in the same environment | No | +| `pytest.ini` and `[tool.pytest.ini_options]` | Yes | No | +| Pytest node IDs and CLI selectors | Yes | No | +| pytest-xdist | Yes, when installed and requested with `-n` | No; use native `--workers` | +| Async tests | According to the installed pytest plugins | Native, without a plugin | +| Testenix retries and `FLAKY` semantics | No | Yes | +| Testenix duration history and scheduling | No | Yes | +| Testenix lossless result model | No | Yes | +| Testenix JSON/JUnit reporters | No; use pytest options or plugins | Yes | +| Published native Testenix speedups | No | Only for the documented workloads | +| Exit status | Unchanged pytest or plugin status | Testenix exit-code contract | + +Some simple pytest-authored functions also happen to run with `testenix run`, but that overlap is +not a compatibility guarantee. In particular, the native collector does not interpret pytest +markers. Running a pytest suite through the native command can turn a skip or xfail into an +ordinary execution, collapse parametrized cases, miss class-based tests, or fail fixture setup. +Use the explicit compatibility command until a suite has been intentionally migrated. + +## Boundaries + +`testenix pytest` does not use `[tool.testenix]`, the native collector, worker pool, retries, +timeouts, tags, history, event model, JSON reporter, or JUnit reporter. Pass the corresponding +pytest or plugin options after the subcommand. For example, use pytest's `--junitxml`, not +Testenix's native `--junit`. + +Pytest and every required plugin must be installed beside the `testenix` executable in the same +interpreter environment. For uv-managed projects, prefer: + +```console +$ uv add --dev "testenix[pytest]" +$ uv run testenix pytest -q tests +``` + +An isolated `uv tool install testenix` environment does not automatically see pytest plugins from +the project environment. Install the required packages into the tool environment or invoke +Testenix through the project environment instead. + +After the handoff, pytest owns signal handling and returns any status chosen by pytest or a plugin, +unchanged. If pytest is missing, Testenix returns `2` with an installation hint before the handoff. +Failure to hand the process to pytest returns `3`. + +## Performance and benchmark interpretation + +Compatibility mode has pytest's execution performance plus launcher and adapter overhead, which has +not yet been measured separately. It is not expected to be faster than invoking `python -m pytest` +directly. The published Testenix benchmarks exercise the native `testenix run` engine and must not +be used to describe `testenix pytest`. + +## Migration path + +Use the bridge first to put Testenix in front of an unchanged suite without altering its semantics. +Migrate individual modules to native Testenix only when their pytest dependencies have explicit +equivalents: + +1. keep the whole suite green with `testenix pytest`; +2. replace pytest fixtures with Testenix `@fixture` providers; +3. replace `parametrize` with `@case` or `@cases`; +4. replace pytest markers with Testenix tags, `@skip`, and `@xfail`; +5. run migrated modules with `testenix run` and leave the remainder on the bridge; +6. compare behavior before making native performance claims. + +The current bridge does not convert pytest outcomes into a Testenix `RunResult`. Deeper event and +report aggregation is a future compatibility layer and requires explicit pytest hook integration. + +--- + # Document: Writing tests Canonical URL: https://polishdataengineer.github.io/testenix/guides/writing-tests/ @@ -687,14 +854,16 @@ Source: docs/reference/cli.md ## Top-level command ```text -testenix [-h] [--version] [--config PYPROJECT] {run} ... +testenix [-h] [--version] +testenix [--config PYPROJECT] run [RUN_ARGS ...] +testenix pytest [PYTEST_ARGS ...] ``` | Option | Description | | --- | --- | | `-h`, `--help` | Show command help. | | `--version` | Print the installed Testenix version. | -| `--config PATH` | Load `[tool.testenix]` from a specific pyproject file. | +| `--config PATH` | Load `[tool.testenix]` for native `testenix run`. | ## `testenix run` @@ -724,6 +893,33 @@ testenix run [PATH ...] CLI options override `[tool.testenix]` values for the current run. +## `testenix pytest` + +```text +testenix pytest [PYTEST_ARGS ...] +``` + +This compatibility command hands the current CLI process to pytest from the same interpreter and +forwards every argument without translation. It preserves pytest configuration, collection, +fixtures, parametrization, markers, classes, hooks, plugins, output, and normal exit status. + +```console +$ testenix pytest -q tests +$ testenix pytest tests/test_api.py::test_health -k smoke --maxfail=1 +$ testenix pytest -n auto tests +$ testenix pytest --junitxml=reports/pytest.xml tests +``` + +Pytest and its plugins must be installed in the same Python environment as Testenix. The optional +`testenix[pytest]` extra installs a supported pytest (`>=8.3,<10`) when needed. Testenix does not +consume a leading `--`: pytest receives it unchanged, including its normal end-of-options meaning. + +`[tool.testenix]` and native options such as `--workers`, `--retries`, `--timeout`, `--tag`, +`--json`, `--junit`, and `--history` do not affect this command. Pass pytest or plugin options +instead. In particular, `testenix --config PATH pytest ...` is rejected; `pytest` must immediately +follow `testenix`. See [pytest compatibility](https://polishdataengineer.github.io/testenix/guides/pytest-compatibility/) for the full +boundary. + ## Examples ```console @@ -733,10 +929,13 @@ $ testenix run --tag unit --tag fast $ testenix run --retries 1 --timeout 10 $ testenix run --json reports/run.json --junit reports/junit.xml $ testenix --config config/pyproject.toml run +$ testenix pytest -q tests ``` ## Exit codes +The native `testenix run` command uses these codes: + | Code | Meaning | | ---: | --- | | `0` | The run has no gating result. | @@ -745,6 +944,11 @@ $ testenix --config config/pyproject.toml run | `3` | Internal runner, reporter, or history failure. | | `130` | User interruption. | +`testenix pytest` hands the current CLI process to pytest, so pytest or plugin exit statuses are +returned unchanged. Standard pytest statuses are `0` success, `1` test failure, `2` interruption, +`3` internal error, `4` usage error, and `5` no tests collected. Before the handoff, Testenix +returns `2` when pytest is missing and `3` when pytest cannot take over the process. + --- # Document: Configuration reference @@ -756,6 +960,10 @@ Source: docs/reference/configuration.md Project defaults live in `[tool.testenix]` in `pyproject.toml`. +This table configures only the native `testenix run` engine. The `testenix pytest` compatibility +command delegates configuration to pytest and therefore uses `pytest.ini`, `pyproject.toml` +`[tool.pytest.ini_options]`, or arguments passed after the subcommand. + ```toml [tool.testenix] paths = ["tests"] @@ -960,7 +1168,8 @@ Source: docs/benchmarks/results.md These tables are generated from the raw JSON committed in `benchmarks/`. They are development evidence for specific synthetic workloads, not a universal claim that Testenix is always faster -than pytest. +than pytest. `Testenix` in these results means the native `testenix run` engine. The +`testenix pytest` compatibility bridge delegates to pytest and is not represented here. ![Preliminary Testenix throughput ratios](https://polishdataengineer.github.io/testenix/_static/benchmark-speedup.svg) @@ -1113,6 +1322,19 @@ files. Correctness wins over speed: a run with a missing, duplicated, or incorrectly finalized result is invalid and excluded from performance comparisons. +## Pytest compatibility bridge + +Measurements of `testenix pytest` must be reported separately from native `testenix run` +measurements. The compatibility command hands the current interpreter to pytest through a POSIX +process overlay or pytest's in-process console entry point on Windows; it does not execute tests +through the Testenix engine. + +A compatibility-overhead comparison must use the same interpreter, working directory, +environment, pytest configuration, plugins, and arguments for both `python -m pytest ...` and +`testenix pytest ...`. Any difference measures adapter overhead only and must not be presented as a +Testenix execution speedup. Native comparisons continue to use `testenix run` and must validate +that both runners execute the same tests and produce equivalent outcomes. + Run the reproducible local harness with: ```bash @@ -1147,7 +1369,7 @@ Source: docs/performance-analysis.md ## Executive summary -The optimized v0.1 runner is faster than pytest and pytest-xdist in the checked-in synthetic +The optimized native v0.1 `testenix run` engine is faster than pytest and pytest-xdist in the checked-in synthetic large-suite scenarios. The largest recorded comparison is 100,000 passing tests across 16 modules: Testenix completed the suite in a median 8.038 seconds, pytest in 25.333 seconds, and pytest-xdist in 21.300 seconds. Every measured command had to report the expected test count or the harness @@ -1158,6 +1380,10 @@ This is evidence for the tested workload and machine, not a universal claim abou project. Import-heavy suites, fixture-heavy suites, slow tests, failure output, different operating systems, and real repositories still need independent measurements. +These results do not apply to `testenix pytest`. The compatibility command delegates to pytest and +has pytest execution performance plus launcher and adapter overhead, which has not yet been +measured separately. + ## Environment and method - Apple M4 Pro, 14 logical CPUs, 24 GiB RAM; @@ -1179,7 +1405,7 @@ Testenix, pytest, and pytest-xdist versions. ## Results -| Scenario | pytest | pytest-xdist | Testenix | Testenix advantage | +| Scenario | pytest | pytest-xdist | Native Testenix | Native Testenix advantage | |---|---:|---:|---:|---:| | 10,000 no-op tests, 16 modules | 2.477 s | 2.106 s | 0.869 s | 2.85x vs pytest | | 10,000 uneven tests, 16 modules | 3.076 s | 2.138 s | 1.345 s | 2.29x vs pytest | @@ -1289,7 +1515,8 @@ Relevant upstream constraints are documented in the No universal “always faster than pytest” statement should be published until the real-project and cross-platform gates pass. The supported claim today is narrower: Testenix is materially faster in the -measured large passing-suite scenarios while retaining supervised isolation and complete results. +measured native large passing-suite scenarios while retaining supervised isolation and complete +results. --- @@ -1304,6 +1531,22 @@ Testenix is a native Python testing framework. Its core does not depend on pytes Compatibility adapters may translate foreign test frameworks into the same manifest and event contracts, but they are not part of native execution. +The first pytest compatibility bridge deliberately does not translate pytest internals. On POSIX, +`testenix pytest` uses `os.execv` to replace itself with `sys.executable -m pytest`. On Windows it +calls pytest's public `console_main` entry point in the existing process because the platform's +`exec` family does not provide equivalent replacement semantics. Both paths keep pytest in the +foreground CLI process with the same working directory, environment, terminal, streams, signal +handling, and exit status. This preserves semantics without making the native core depend on +pytest: + +```text +POSIX: testenix pytest ==exec==> Python -m pytest -> collector/plugins/executor -> output/status +Windows: testenix pytest =========> pytest.console_main -> collector/plugins/executor -> output/status +``` + +The bridge is a CLI infrastructure adapter, not a native collection adapter. It does not emit +Testenix events or construct a `RunResult` in version 0.1. + ## Product contract Testenix aims to be typed, async-native, parallel-first, deterministic, and lossless when reporting @@ -1325,7 +1568,7 @@ Authoring API -> supervised collection -> inert manifest -> affinity scheduler - - `events`, `aggregate`, and `scheduler` remain engine-independent. - `runner` is the application service connecting the native engine with execution policy. - reporters and storage consume completed domain results or versioned events. -- optional compatibility adapters depend inward; the core never imports them. +- optional compatibility adapters stay at the CLI boundary; the native core never imports pytest. ## Version 0.1 scope @@ -1337,7 +1580,8 @@ Authoring API -> supervised collection -> inert manifest -> affinity scheduler - - deterministic scheduling based on historical durations; - append-only JSONL events and a pure reducer; - console, JSON, and JUnit output plus local SQLite duration history; -- retries represented as immutable attempts and finalized as `FLAKY` when appropriate. +- retries represented as immutable attempts and finalized as `FLAKY` when appropriate; +- an optional platform-aware pytest handoff for unchanged legacy suites. Remote workers, distributed storage, result caching, automatic quarantine, and a stable third-party plugin SDK are deliberately outside version 0.1. @@ -1402,6 +1646,7 @@ Source: docs/roadmap.md - Local process workers with deterministic LPT scheduling. - Immutable attempts and lossless setup/call/teardown aggregation. - Console, JSON, JUnit XML, JSONL events, and SQLite duration history. +- Optional `testenix pytest` compatibility bridge preserving the real pytest engine and plugins. ## 0.2 — fast feedback @@ -1412,7 +1657,7 @@ Source: docs/roadmap.md ## 0.3 — adoption -- Optional pytest collection/execution adapter in a separate extra or distribution. +- Pytest hook adapter translating collection and outcomes into Testenix events and `RunResult`. - A migration checker and automated transformations for common fixtures and markers. - Versioned reporter and selector plugin interfaces. - IDE protocol and machine-readable collection manifest. @@ -1441,6 +1686,13 @@ project intends to use Semantic Versioning once its public API reaches stability ## [Unreleased] +### Added + +- `testenix pytest [PYTEST_ARGS ...]` compatibility bridge for unchanged pytest suites, preserving + the real pytest collector, fixtures, parametrization, markers, plugins, output, and exit status. +- Optional `testenix[pytest]` installation extra and a documented native-versus-compatibility + capability matrix. + ## [0.1.0] - 2026-07-20 ### Added diff --git a/docs/llms.txt b/docs/llms.txt index e5500ca..0446c50 100644 --- a/docs/llms.txt +++ b/docs/llms.txt @@ -1,7 +1,7 @@ # Testenix > Testenix is an alpha, async-native, parallel-first Python testing framework with a -> dependency-free runtime and a lossless result model. +> dependency-free native runtime, a lossless result model, and an optional pytest bridge. Canonical documentation: https://polishdataengineer.github.io/testenix/ Source repository: https://github.com/polishdataengineer/testenix @@ -11,6 +11,8 @@ Package index: https://pypi.org/project/testenix/ - [Overview](https://polishdataengineer.github.io/testenix/): project positioning, example, guarantees, and maturity. - [Getting started](https://polishdataengineer.github.io/testenix/getting-started/): installation and the first run. +- [Pytest compatibility](https://polishdataengineer.github.io/testenix/guides/pytest-compatibility/): run existing suites unchanged, + compare capabilities, and choose a migration boundary. - [Writing tests](https://polishdataengineer.github.io/testenix/guides/writing-tests/): tests, cases, tags, skips, xfail, and retries. - [Fixtures](https://polishdataengineer.github.io/testenix/guides/fixtures/): dependency graphs, cleanup, and scopes. - [Parallel execution](https://polishdataengineer.github.io/testenix/guides/parallelism/): workers, scheduling, crashes, and timeouts. diff --git a/docs/performance-analysis.md b/docs/performance-analysis.md index e952082..c8755e4 100644 --- a/docs/performance-analysis.md +++ b/docs/performance-analysis.md @@ -2,7 +2,7 @@ ## Executive summary -The optimized v0.1 runner is faster than pytest and pytest-xdist in the checked-in synthetic +The optimized native v0.1 `testenix run` engine is faster than pytest and pytest-xdist in the checked-in synthetic large-suite scenarios. The largest recorded comparison is 100,000 passing tests across 16 modules: Testenix completed the suite in a median 8.038 seconds, pytest in 25.333 seconds, and pytest-xdist in 21.300 seconds. Every measured command had to report the expected test count or the harness @@ -13,6 +13,10 @@ This is evidence for the tested workload and machine, not a universal claim abou project. Import-heavy suites, fixture-heavy suites, slow tests, failure output, different operating systems, and real repositories still need independent measurements. +These results do not apply to `testenix pytest`. The compatibility command delegates to pytest and +has pytest execution performance plus launcher and adapter overhead, which has not yet been +measured separately. + ## Environment and method - Apple M4 Pro, 14 logical CPUs, 24 GiB RAM; @@ -34,7 +38,7 @@ Testenix, pytest, and pytest-xdist versions. ## Results -| Scenario | pytest | pytest-xdist | Testenix | Testenix advantage | +| Scenario | pytest | pytest-xdist | Native Testenix | Native Testenix advantage | |---|---:|---:|---:|---:| | 10,000 no-op tests, 16 modules | 2.477 s | 2.106 s | 0.869 s | 2.85x vs pytest | | 10,000 uneven tests, 16 modules | 3.076 s | 2.138 s | 1.345 s | 2.29x vs pytest | @@ -144,4 +148,5 @@ Relevant upstream constraints are documented in the No universal “always faster than pytest” statement should be published until the real-project and cross-platform gates pass. The supported claim today is narrower: Testenix is materially faster in the -measured large passing-suite scenarios while retaining supervised isolation and complete results. +measured native large passing-suite scenarios while retaining supervised isolation and complete +results. diff --git a/docs/reference/cli.md b/docs/reference/cli.md index 3474a5a..d169d95 100644 --- a/docs/reference/cli.md +++ b/docs/reference/cli.md @@ -3,14 +3,16 @@ ## Top-level command ```text -testenix [-h] [--version] [--config PYPROJECT] {run} ... +testenix [-h] [--version] +testenix [--config PYPROJECT] run [RUN_ARGS ...] +testenix pytest [PYTEST_ARGS ...] ``` | Option | Description | | --- | --- | | `-h`, `--help` | Show command help. | | `--version` | Print the installed Testenix version. | -| `--config PATH` | Load `[tool.testenix]` from a specific pyproject file. | +| `--config PATH` | Load `[tool.testenix]` for native `testenix run`. | ## `testenix run` @@ -40,6 +42,33 @@ testenix run [PATH ...] CLI options override `[tool.testenix]` values for the current run. +## `testenix pytest` + +```text +testenix pytest [PYTEST_ARGS ...] +``` + +This compatibility command hands the current CLI process to pytest from the same interpreter and +forwards every argument without translation. It preserves pytest configuration, collection, +fixtures, parametrization, markers, classes, hooks, plugins, output, and normal exit status. + +```console +$ testenix pytest -q tests +$ testenix pytest tests/test_api.py::test_health -k smoke --maxfail=1 +$ testenix pytest -n auto tests +$ testenix pytest --junitxml=reports/pytest.xml tests +``` + +Pytest and its plugins must be installed in the same Python environment as Testenix. The optional +`testenix[pytest]` extra installs a supported pytest (`>=8.3,<10`) when needed. Testenix does not +consume a leading `--`: pytest receives it unchanged, including its normal end-of-options meaning. + +`[tool.testenix]` and native options such as `--workers`, `--retries`, `--timeout`, `--tag`, +`--json`, `--junit`, and `--history` do not affect this command. Pass pytest or plugin options +instead. In particular, `testenix --config PATH pytest ...` is rejected; `pytest` must immediately +follow `testenix`. See [pytest compatibility](../guides/pytest-compatibility.md) for the full +boundary. + ## Examples ```console @@ -49,10 +78,13 @@ $ testenix run --tag unit --tag fast $ testenix run --retries 1 --timeout 10 $ testenix run --json reports/run.json --junit reports/junit.xml $ testenix --config config/pyproject.toml run +$ testenix pytest -q tests ``` ## Exit codes +The native `testenix run` command uses these codes: + | Code | Meaning | | ---: | --- | | `0` | The run has no gating result. | @@ -60,3 +92,8 @@ $ testenix --config config/pyproject.toml run | `2` | Invalid CLI/configuration, collection error, or empty explicit tag selection. | | `3` | Internal runner, reporter, or history failure. | | `130` | User interruption. | + +`testenix pytest` hands the current CLI process to pytest, so pytest or plugin exit statuses are +returned unchanged. Standard pytest statuses are `0` success, `1` test failure, `2` interruption, +`3` internal error, `4` usage error, and `5` no tests collected. Before the handoff, Testenix +returns `2` when pytest is missing and `3` when pytest cannot take over the process. diff --git a/docs/reference/configuration.md b/docs/reference/configuration.md index 96a80fc..634553f 100644 --- a/docs/reference/configuration.md +++ b/docs/reference/configuration.md @@ -2,6 +2,10 @@ Project defaults live in `[tool.testenix]` in `pyproject.toml`. +This table configures only the native `testenix run` engine. The `testenix pytest` compatibility +command delegates configuration to pytest and therefore uses `pytest.ini`, `pyproject.toml` +`[tool.pytest.ini_options]`, or arguments passed after the subcommand. + ```toml [tool.testenix] paths = ["tests"] diff --git a/docs/roadmap.md b/docs/roadmap.md index 02ff2c2..57d6710 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -9,6 +9,7 @@ - Local process workers with deterministic LPT scheduling. - Immutable attempts and lossless setup/call/teardown aggregation. - Console, JSON, JUnit XML, JSONL events, and SQLite duration history. +- Optional `testenix pytest` compatibility bridge preserving the real pytest engine and plugins. ## 0.2 — fast feedback @@ -19,7 +20,7 @@ ## 0.3 — adoption -- Optional pytest collection/execution adapter in a separate extra or distribution. +- Pytest hook adapter translating collection and outcomes into Testenix events and `RunResult`. - A migration checker and automated transformations for common fixtures and markers. - Versioned reporter and selector plugin interfaces. - IDE protocol and machine-readable collection manifest. @@ -33,4 +34,3 @@ Result caching and impact selection will not become build-gating defaults until their false-negative rate is measured continuously against full-suite runs. - diff --git a/llms-full.txt b/llms-full.txt index 5e8659e..6ec4b3f 100644 --- a/llms-full.txt +++ b/llms-full.txt @@ -22,11 +22,12 @@ description: Testenix is an async-native, parallel-first Python testing framewor
Python testing framework · Alpha
Fast tests. Clear results.

- Testenix unifies synchronous and asynchronous tests, fixtures, local parallelism, - retries, crash recovery, and machine-readable reports in one dependency-free runtime. + Testenix combines a dependency-free native runtime with a transparent bridge for + running existing pytest suites unchanged.

@@ -37,6 +38,7 @@ description: Testenix is an async-native, parallel-first Python testing framewor :maxdepth: 2 getting-started +guides/pytest-compatibility guides/writing-tests guides/fixtures guides/parallelism @@ -55,10 +57,10 @@ for-llms ## Why Testenix
-
0third-party runtime dependencies
+
0dependencies in the native runtime
12Python and OS combinations in CI
3console, JSON, and JUnit reports
-
3.15×100k synthetic throughput vs pytest
+
3.15×native testenix run on the 100k synthetic workload vs pytest
Testenix is deliberately built around a few strong guarantees: @@ -76,6 +78,17 @@ Testenix is deliberately built around a few strong guarantees: ## A complete first test +Already have a pytest suite? Keep its fixtures, parametrization, markers, classes, configuration, +and plugins: + +```console +$ python -m pip install "testenix[pytest]" +$ testenix pytest -q tests +``` + +[Read the compatibility contract](https://polishdataengineer.github.io/testenix/guides/pytest-compatibility/) before migrating individual +modules to the native engine. + ```python from collections.abc import AsyncIterator @@ -108,9 +121,10 @@ If the first PyPI release is not available yet, install the current source with ## Performance evidence, with context -The checked-in development baseline measured 100,000 empty tests across 16 generated modules on -an Apple M4 Pro and CPython 3.11. Testenix completed that specific workload in a median 8.04 -seconds, compared with 25.33 seconds for pytest and 21.30 seconds for pytest-xdist. +The checked-in development baseline measured native `testenix run` on 100,000 empty tests across +16 generated modules on an Apple M4 Pro and CPython 3.11. Native Testenix completed that specific +workload in a median 8.04 seconds, compared with 25.33 seconds for pytest and 21.30 seconds for +pytest-xdist. These measurements do not apply to the delegated `testenix pytest` command.
This is a preliminary synthetic result from one machine, not a promise that every project will be @@ -123,10 +137,12 @@ and limitations so that the claim can be evaluated rather than taken on trust. ## Project maturity -Testenix is alpha software and is not yet a drop-in pytest replacement. Pytest has a much broader -plugin ecosystem, richer IDE integration, and mature assertion rewriting. Testenix is a good fit -when its native async model, built-in parallel execution, explicit failure semantics, or -dependency-free runtime are more important than ecosystem compatibility. +Testenix is alpha software. The `testenix pytest` bridge preserves an existing suite by delegating +to real pytest, while `testenix run` is a distinct native engine rather than a drop-in pytest +reimplementation. Pytest still has a much broader plugin ecosystem, richer IDE integration, and +mature assertion rewriting. Choose the bridge for compatibility and the native engine when its +async model, built-in parallel execution, explicit failure semantics, or dependency-free core are +more important. --- @@ -143,7 +159,8 @@ This guide takes a new project from installation to its first parallel Testenix - CPython 3.11 or newer - Linux, macOS, or Windows -- no runtime dependencies beyond the Python standard library +- no runtime dependencies beyond the Python standard library for native `testenix run` +- pytest in the same environment when using the optional compatibility bridge ## Install @@ -161,13 +178,36 @@ $ uv add --dev testenix $ uv run testenix --version ``` +For an existing pytest project, install the compatibility extra: + +```console +$ python -m pip install "testenix[pytest]" +# or +$ uv add --dev "testenix[pytest]" +``` + Until the first PyPI release is visible, install directly from the protected `main` branch: ```console $ python -m pip install "testenix @ git+https://github.com/polishdataengineer/testenix.git@main" +# include pytest when the project environment does not already provide it +$ python -m pip install "testenix[pytest] @ git+https://github.com/polishdataengineer/testenix.git@main" +``` + +## Choose an execution mode + +Run an unchanged pytest suite through its real engine: + +```console +$ testenix pytest -q tests ``` -## Create a test +Use `testenix run` for native Testenix tests and the built-in scheduler, retries, history, and +lossless reports. The two commands have deliberately separate semantics. See +[pytest compatibility](https://polishdataengineer.github.io/testenix/guides/pytest-compatibility/) for the capability matrix and migration +boundary. + +## Create a native test Testenix collects ordinary functions whose names begin with `test_`. Decorators are optional for simple tests. @@ -247,6 +287,7 @@ collection and execution process trees before control returns to the caller. ## Next steps - [Write tests, cases, tags, skips, and expected failures](https://polishdataengineer.github.io/testenix/guides/writing-tests/) +- [Run or migrate an existing pytest suite](https://polishdataengineer.github.io/testenix/guides/pytest-compatibility/) - [Build fixture graphs](https://polishdataengineer.github.io/testenix/guides/fixtures/) - [Understand process parallelism and timeouts](https://polishdataengineer.github.io/testenix/guides/parallelism/) - [Produce JSON and JUnit reports](https://polishdataengineer.github.io/testenix/guides/reports/) @@ -254,6 +295,132 @@ collection and execution process trees before control returns to the caller. --- +# Document: Pytest compatibility + +Canonical URL: https://polishdataengineer.github.io/testenix/guides/pytest-compatibility/ +Source: docs/guides/pytest-compatibility.md + +--- +description: Run existing pytest suites unchanged through Testenix and understand the native migration boundary. +--- + +# Pytest compatibility + +Testenix can run an existing pytest suite without rewriting it. Use the compatibility bridge when +the suite depends on pytest fixtures, parametrization, markers, classes, plugins, hooks, or +configuration: + +```console +$ python -m pip install "testenix[pytest]" +$ testenix pytest -q tests +``` + +If a supported pytest (`>=8.3,<10`) is already installed in the same Python environment, installing +the base `testenix` package is enough. The extra is a convenience for environments that do not +already contain pytest. + +## What the command does + +Everything after `testenix pytest` is forwarded unchanged to: + +```console +$ python -m pytest [PYTEST_ARGS ...] +``` + +Testenix hands its current CLI process to pytest from the same Python interpreter. On POSIX it uses +a process overlay; on Windows it calls pytest's public console entry point in-process because the +platform does not provide the same overlay semantics. Both paths preserve the foreground process, +working directory, environment, terminal, standard streams, and pytest's signal handling. Pytest +therefore remains responsible for collection, execution, configuration, plugin loading, output, +descendants such as pytest-xdist workers, and exit status. + +```console +$ testenix pytest tests/test_api.py::test_health -k smoke --maxfail=1 +$ testenix pytest -m "unit and not slow" tests +$ testenix pytest -n auto tests +$ testenix pytest --junitxml=reports/pytest.xml tests +``` + +The `-n` example requires pytest-xdist in the same environment. Testenix does not translate its +native `--workers` option into an xdist option. + +## Capability matrix + +| Capability | `testenix pytest` | `testenix run` | +| --- | --- | --- | +| Plain module-level `test_*` functions | Yes, through pytest | Yes | +| Pytest classes and unittest-style tests | Yes | No | +| `pytest.fixture` and built-in fixtures | Yes | No | +| `conftest.py` fixture and hook discovery | Yes | No | +| `pytest.mark.parametrize` | Yes | No; use Testenix `@case` or `@cases` | +| Pytest skip, xfail, and custom markers | Yes | No; use Testenix decorators and tags | +| Pytest assertion rewriting | Yes | No | +| Pytest plugins and hooks | Yes, when installed in the same environment | No | +| `pytest.ini` and `[tool.pytest.ini_options]` | Yes | No | +| Pytest node IDs and CLI selectors | Yes | No | +| pytest-xdist | Yes, when installed and requested with `-n` | No; use native `--workers` | +| Async tests | According to the installed pytest plugins | Native, without a plugin | +| Testenix retries and `FLAKY` semantics | No | Yes | +| Testenix duration history and scheduling | No | Yes | +| Testenix lossless result model | No | Yes | +| Testenix JSON/JUnit reporters | No; use pytest options or plugins | Yes | +| Published native Testenix speedups | No | Only for the documented workloads | +| Exit status | Unchanged pytest or plugin status | Testenix exit-code contract | + +Some simple pytest-authored functions also happen to run with `testenix run`, but that overlap is +not a compatibility guarantee. In particular, the native collector does not interpret pytest +markers. Running a pytest suite through the native command can turn a skip or xfail into an +ordinary execution, collapse parametrized cases, miss class-based tests, or fail fixture setup. +Use the explicit compatibility command until a suite has been intentionally migrated. + +## Boundaries + +`testenix pytest` does not use `[tool.testenix]`, the native collector, worker pool, retries, +timeouts, tags, history, event model, JSON reporter, or JUnit reporter. Pass the corresponding +pytest or plugin options after the subcommand. For example, use pytest's `--junitxml`, not +Testenix's native `--junit`. + +Pytest and every required plugin must be installed beside the `testenix` executable in the same +interpreter environment. For uv-managed projects, prefer: + +```console +$ uv add --dev "testenix[pytest]" +$ uv run testenix pytest -q tests +``` + +An isolated `uv tool install testenix` environment does not automatically see pytest plugins from +the project environment. Install the required packages into the tool environment or invoke +Testenix through the project environment instead. + +After the handoff, pytest owns signal handling and returns any status chosen by pytest or a plugin, +unchanged. If pytest is missing, Testenix returns `2` with an installation hint before the handoff. +Failure to hand the process to pytest returns `3`. + +## Performance and benchmark interpretation + +Compatibility mode has pytest's execution performance plus launcher and adapter overhead, which has +not yet been measured separately. It is not expected to be faster than invoking `python -m pytest` +directly. The published Testenix benchmarks exercise the native `testenix run` engine and must not +be used to describe `testenix pytest`. + +## Migration path + +Use the bridge first to put Testenix in front of an unchanged suite without altering its semantics. +Migrate individual modules to native Testenix only when their pytest dependencies have explicit +equivalents: + +1. keep the whole suite green with `testenix pytest`; +2. replace pytest fixtures with Testenix `@fixture` providers; +3. replace `parametrize` with `@case` or `@cases`; +4. replace pytest markers with Testenix tags, `@skip`, and `@xfail`; +5. run migrated modules with `testenix run` and leave the remainder on the bridge; +6. compare behavior before making native performance claims. + +The current bridge does not convert pytest outcomes into a Testenix `RunResult`. Deeper event and +report aggregation is a future compatibility layer and requires explicit pytest hook integration. + +--- + # Document: Writing tests Canonical URL: https://polishdataengineer.github.io/testenix/guides/writing-tests/ @@ -687,14 +854,16 @@ Source: docs/reference/cli.md ## Top-level command ```text -testenix [-h] [--version] [--config PYPROJECT] {run} ... +testenix [-h] [--version] +testenix [--config PYPROJECT] run [RUN_ARGS ...] +testenix pytest [PYTEST_ARGS ...] ``` | Option | Description | | --- | --- | | `-h`, `--help` | Show command help. | | `--version` | Print the installed Testenix version. | -| `--config PATH` | Load `[tool.testenix]` from a specific pyproject file. | +| `--config PATH` | Load `[tool.testenix]` for native `testenix run`. | ## `testenix run` @@ -724,6 +893,33 @@ testenix run [PATH ...] CLI options override `[tool.testenix]` values for the current run. +## `testenix pytest` + +```text +testenix pytest [PYTEST_ARGS ...] +``` + +This compatibility command hands the current CLI process to pytest from the same interpreter and +forwards every argument without translation. It preserves pytest configuration, collection, +fixtures, parametrization, markers, classes, hooks, plugins, output, and normal exit status. + +```console +$ testenix pytest -q tests +$ testenix pytest tests/test_api.py::test_health -k smoke --maxfail=1 +$ testenix pytest -n auto tests +$ testenix pytest --junitxml=reports/pytest.xml tests +``` + +Pytest and its plugins must be installed in the same Python environment as Testenix. The optional +`testenix[pytest]` extra installs a supported pytest (`>=8.3,<10`) when needed. Testenix does not +consume a leading `--`: pytest receives it unchanged, including its normal end-of-options meaning. + +`[tool.testenix]` and native options such as `--workers`, `--retries`, `--timeout`, `--tag`, +`--json`, `--junit`, and `--history` do not affect this command. Pass pytest or plugin options +instead. In particular, `testenix --config PATH pytest ...` is rejected; `pytest` must immediately +follow `testenix`. See [pytest compatibility](https://polishdataengineer.github.io/testenix/guides/pytest-compatibility/) for the full +boundary. + ## Examples ```console @@ -733,10 +929,13 @@ $ testenix run --tag unit --tag fast $ testenix run --retries 1 --timeout 10 $ testenix run --json reports/run.json --junit reports/junit.xml $ testenix --config config/pyproject.toml run +$ testenix pytest -q tests ``` ## Exit codes +The native `testenix run` command uses these codes: + | Code | Meaning | | ---: | --- | | `0` | The run has no gating result. | @@ -745,6 +944,11 @@ $ testenix --config config/pyproject.toml run | `3` | Internal runner, reporter, or history failure. | | `130` | User interruption. | +`testenix pytest` hands the current CLI process to pytest, so pytest or plugin exit statuses are +returned unchanged. Standard pytest statuses are `0` success, `1` test failure, `2` interruption, +`3` internal error, `4` usage error, and `5` no tests collected. Before the handoff, Testenix +returns `2` when pytest is missing and `3` when pytest cannot take over the process. + --- # Document: Configuration reference @@ -756,6 +960,10 @@ Source: docs/reference/configuration.md Project defaults live in `[tool.testenix]` in `pyproject.toml`. +This table configures only the native `testenix run` engine. The `testenix pytest` compatibility +command delegates configuration to pytest and therefore uses `pytest.ini`, `pyproject.toml` +`[tool.pytest.ini_options]`, or arguments passed after the subcommand. + ```toml [tool.testenix] paths = ["tests"] @@ -960,7 +1168,8 @@ Source: docs/benchmarks/results.md These tables are generated from the raw JSON committed in `benchmarks/`. They are development evidence for specific synthetic workloads, not a universal claim that Testenix is always faster -than pytest. +than pytest. `Testenix` in these results means the native `testenix run` engine. The +`testenix pytest` compatibility bridge delegates to pytest and is not represented here. ![Preliminary Testenix throughput ratios](https://polishdataengineer.github.io/testenix/_static/benchmark-speedup.svg) @@ -1113,6 +1322,19 @@ files. Correctness wins over speed: a run with a missing, duplicated, or incorrectly finalized result is invalid and excluded from performance comparisons. +## Pytest compatibility bridge + +Measurements of `testenix pytest` must be reported separately from native `testenix run` +measurements. The compatibility command hands the current interpreter to pytest through a POSIX +process overlay or pytest's in-process console entry point on Windows; it does not execute tests +through the Testenix engine. + +A compatibility-overhead comparison must use the same interpreter, working directory, +environment, pytest configuration, plugins, and arguments for both `python -m pytest ...` and +`testenix pytest ...`. Any difference measures adapter overhead only and must not be presented as a +Testenix execution speedup. Native comparisons continue to use `testenix run` and must validate +that both runners execute the same tests and produce equivalent outcomes. + Run the reproducible local harness with: ```bash @@ -1147,7 +1369,7 @@ Source: docs/performance-analysis.md ## Executive summary -The optimized v0.1 runner is faster than pytest and pytest-xdist in the checked-in synthetic +The optimized native v0.1 `testenix run` engine is faster than pytest and pytest-xdist in the checked-in synthetic large-suite scenarios. The largest recorded comparison is 100,000 passing tests across 16 modules: Testenix completed the suite in a median 8.038 seconds, pytest in 25.333 seconds, and pytest-xdist in 21.300 seconds. Every measured command had to report the expected test count or the harness @@ -1158,6 +1380,10 @@ This is evidence for the tested workload and machine, not a universal claim abou project. Import-heavy suites, fixture-heavy suites, slow tests, failure output, different operating systems, and real repositories still need independent measurements. +These results do not apply to `testenix pytest`. The compatibility command delegates to pytest and +has pytest execution performance plus launcher and adapter overhead, which has not yet been +measured separately. + ## Environment and method - Apple M4 Pro, 14 logical CPUs, 24 GiB RAM; @@ -1179,7 +1405,7 @@ Testenix, pytest, and pytest-xdist versions. ## Results -| Scenario | pytest | pytest-xdist | Testenix | Testenix advantage | +| Scenario | pytest | pytest-xdist | Native Testenix | Native Testenix advantage | |---|---:|---:|---:|---:| | 10,000 no-op tests, 16 modules | 2.477 s | 2.106 s | 0.869 s | 2.85x vs pytest | | 10,000 uneven tests, 16 modules | 3.076 s | 2.138 s | 1.345 s | 2.29x vs pytest | @@ -1289,7 +1515,8 @@ Relevant upstream constraints are documented in the No universal “always faster than pytest” statement should be published until the real-project and cross-platform gates pass. The supported claim today is narrower: Testenix is materially faster in the -measured large passing-suite scenarios while retaining supervised isolation and complete results. +measured native large passing-suite scenarios while retaining supervised isolation and complete +results. --- @@ -1304,6 +1531,22 @@ Testenix is a native Python testing framework. Its core does not depend on pytes Compatibility adapters may translate foreign test frameworks into the same manifest and event contracts, but they are not part of native execution. +The first pytest compatibility bridge deliberately does not translate pytest internals. On POSIX, +`testenix pytest` uses `os.execv` to replace itself with `sys.executable -m pytest`. On Windows it +calls pytest's public `console_main` entry point in the existing process because the platform's +`exec` family does not provide equivalent replacement semantics. Both paths keep pytest in the +foreground CLI process with the same working directory, environment, terminal, streams, signal +handling, and exit status. This preserves semantics without making the native core depend on +pytest: + +```text +POSIX: testenix pytest ==exec==> Python -m pytest -> collector/plugins/executor -> output/status +Windows: testenix pytest =========> pytest.console_main -> collector/plugins/executor -> output/status +``` + +The bridge is a CLI infrastructure adapter, not a native collection adapter. It does not emit +Testenix events or construct a `RunResult` in version 0.1. + ## Product contract Testenix aims to be typed, async-native, parallel-first, deterministic, and lossless when reporting @@ -1325,7 +1568,7 @@ Authoring API -> supervised collection -> inert manifest -> affinity scheduler - - `events`, `aggregate`, and `scheduler` remain engine-independent. - `runner` is the application service connecting the native engine with execution policy. - reporters and storage consume completed domain results or versioned events. -- optional compatibility adapters depend inward; the core never imports them. +- optional compatibility adapters stay at the CLI boundary; the native core never imports pytest. ## Version 0.1 scope @@ -1337,7 +1580,8 @@ Authoring API -> supervised collection -> inert manifest -> affinity scheduler - - deterministic scheduling based on historical durations; - append-only JSONL events and a pure reducer; - console, JSON, and JUnit output plus local SQLite duration history; -- retries represented as immutable attempts and finalized as `FLAKY` when appropriate. +- retries represented as immutable attempts and finalized as `FLAKY` when appropriate; +- an optional platform-aware pytest handoff for unchanged legacy suites. Remote workers, distributed storage, result caching, automatic quarantine, and a stable third-party plugin SDK are deliberately outside version 0.1. @@ -1402,6 +1646,7 @@ Source: docs/roadmap.md - Local process workers with deterministic LPT scheduling. - Immutable attempts and lossless setup/call/teardown aggregation. - Console, JSON, JUnit XML, JSONL events, and SQLite duration history. +- Optional `testenix pytest` compatibility bridge preserving the real pytest engine and plugins. ## 0.2 — fast feedback @@ -1412,7 +1657,7 @@ Source: docs/roadmap.md ## 0.3 — adoption -- Optional pytest collection/execution adapter in a separate extra or distribution. +- Pytest hook adapter translating collection and outcomes into Testenix events and `RunResult`. - A migration checker and automated transformations for common fixtures and markers. - Versioned reporter and selector plugin interfaces. - IDE protocol and machine-readable collection manifest. @@ -1441,6 +1686,13 @@ project intends to use Semantic Versioning once its public API reaches stability ## [Unreleased] +### Added + +- `testenix pytest [PYTEST_ARGS ...]` compatibility bridge for unchanged pytest suites, preserving + the real pytest collector, fixtures, parametrization, markers, plugins, output, and exit status. +- Optional `testenix[pytest]` installation extra and a documented native-versus-compatibility + capability matrix. + ## [0.1.0] - 2026-07-20 ### Added diff --git a/llms.txt b/llms.txt index e5500ca..0446c50 100644 --- a/llms.txt +++ b/llms.txt @@ -1,7 +1,7 @@ # Testenix > Testenix is an alpha, async-native, parallel-first Python testing framework with a -> dependency-free runtime and a lossless result model. +> dependency-free native runtime, a lossless result model, and an optional pytest bridge. Canonical documentation: https://polishdataengineer.github.io/testenix/ Source repository: https://github.com/polishdataengineer/testenix @@ -11,6 +11,8 @@ Package index: https://pypi.org/project/testenix/ - [Overview](https://polishdataengineer.github.io/testenix/): project positioning, example, guarantees, and maturity. - [Getting started](https://polishdataengineer.github.io/testenix/getting-started/): installation and the first run. +- [Pytest compatibility](https://polishdataengineer.github.io/testenix/guides/pytest-compatibility/): run existing suites unchanged, + compare capabilities, and choose a migration boundary. - [Writing tests](https://polishdataengineer.github.io/testenix/guides/writing-tests/): tests, cases, tags, skips, xfail, and retries. - [Fixtures](https://polishdataengineer.github.io/testenix/guides/fixtures/): dependency graphs, cleanup, and scopes. - [Parallel execution](https://polishdataengineer.github.io/testenix/guides/parallelism/): workers, scheduling, crashes, and timeouts. diff --git a/pyproject.toml b/pyproject.toml index 6b406de..9fc868c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -17,6 +17,7 @@ classifiers = [ "Development Status :: 3 - Alpha", "Environment :: Console", "Framework :: AsyncIO", + "Framework :: Pytest", "Intended Audience :: Developers", "Programming Language :: Python :: 3", "Programming Language :: Python :: 3.11", @@ -27,6 +28,9 @@ classifiers = [ "Typing :: Typed", ] +[project.optional-dependencies] +pytest = ["pytest>=8.3,<10"] + [project.urls] Homepage = "https://github.com/polishdataengineer/testenix" Documentation = "https://polishdataengineer.github.io/testenix/" @@ -42,7 +46,7 @@ testenix = "testenix.cli:main" dev = [ "mypy>=2.3,<3", "pyright>=1.1.411,<2", - "pytest>=8.3", + "pytest>=8.3,<10", "pytest-xdist>=3.6", "ruff>=0.9", "twine>=6.2,<7", diff --git a/scripts/generate_docs_assets.py b/scripts/generate_docs_assets.py index 2811fb7..fbc1e96 100644 --- a/scripts/generate_docs_assets.py +++ b/scripts/generate_docs_assets.py @@ -30,6 +30,11 @@ LLM_DOCUMENTS = ( ("Overview", Path("docs/index.md"), ""), ("Getting started", Path("docs/getting-started.md"), "getting-started/"), + ( + "Pytest compatibility", + Path("docs/guides/pytest-compatibility.md"), + "guides/pytest-compatibility/", + ), ("Writing tests", Path("docs/guides/writing-tests.md"), "guides/writing-tests/"), ("Fixtures", Path("docs/guides/fixtures.md"), "guides/fixtures/"), ("Parallel execution", Path("docs/guides/parallelism.md"), "guides/parallelism/"), @@ -222,7 +227,8 @@ def _render_benchmark_results(benchmarks: tuple[Benchmark, ...]) -> str: These tables are generated from the raw JSON committed in `benchmarks/`. They are development evidence for specific synthetic workloads, not a universal claim that Testenix is always faster -than pytest. +than pytest. `Testenix` in these results means the native `testenix run` engine. The +`testenix pytest` compatibility bridge delegates to pytest and is not represented here. ![Preliminary Testenix throughput ratios](../_static/benchmark-speedup.svg) @@ -452,7 +458,7 @@ def _render_llms_index() -> str: return f"""# Testenix > Testenix is an alpha, async-native, parallel-first Python testing framework with a -> dependency-free runtime and a lossless result model. +> dependency-free native runtime, a lossless result model, and an optional pytest bridge. Canonical documentation: {SITE_URL} Source repository: {REPOSITORY_URL} @@ -462,6 +468,8 @@ def _render_llms_index() -> str: - [Overview]({SITE_URL}): project positioning, example, guarantees, and maturity. - [Getting started]({SITE_URL}getting-started/): installation and the first run. +- [Pytest compatibility]({SITE_URL}guides/pytest-compatibility/): run existing suites unchanged, + compare capabilities, and choose a migration boundary. - [Writing tests]({SITE_URL}guides/writing-tests/): tests, cases, tags, skips, xfail, and retries. - [Fixtures]({SITE_URL}guides/fixtures/): dependency graphs, cleanup, and scopes. - [Parallel execution]({SITE_URL}guides/parallelism/): workers, scheduling, crashes, and timeouts. diff --git a/src/testenix/cli.py b/src/testenix/cli.py index 3c24cc4..34c9f83 100644 --- a/src/testenix/cli.py +++ b/src/testenix/cli.py @@ -34,7 +34,7 @@ def build_parser() -> argparse.ArgumentParser: "--config", dest="global_config", type=Path, - help="pyproject.toml containing [tool.testenix]", + help="pyproject.toml containing [tool.testenix] for the native run command", ) subparsers = parser.add_subparsers(dest="command", required=True) @@ -67,12 +67,26 @@ def build_parser() -> argparse.ArgumentParser: history_group.add_argument("--history", dest="history_path", type=Path, default=None) history_group.add_argument("--no-history", action="store_true") run_parser.set_defaults(handler=_run_command) + + # Pytest owns the complete argument grammar for this compatibility command. + # ``main`` intercepts it before argparse so flags such as ``-q`` and ``-k`` + # can pass through unchanged; this placeholder keeps top-level help useful. + pytest_parser = subparsers.add_parser( + "pytest", + add_help=False, + help="run an existing pytest suite through the compatibility adapter", + ) + pytest_parser.set_defaults(handler=_misplaced_pytest_command) return parser def main(argv: Sequence[str] | None = None) -> int: + raw_arguments = tuple(sys.argv[1:] if argv is None else argv) + if raw_arguments[:1] == ("pytest",): + return _pytest_command(raw_arguments[1:]) + parser = build_parser() - arguments = parser.parse_args(argv) + arguments = parser.parse_args(raw_arguments) try: return int(arguments.handler(arguments)) except ConfigError as error: @@ -132,6 +146,40 @@ def _call_runner(paths: Sequence[str], config: TestenixConfig) -> RunResult: return run(paths, config) +def _pytest_command(arguments: Sequence[str]) -> int: + from testenix.pytest_adapter import PytestInvocationError, PytestUnavailableError + + try: + return _call_pytest(tuple(arguments)) + except KeyboardInterrupt: + print("testenix: interrupted", file=sys.stderr) + return EXIT_INTERRUPTED + except PytestUnavailableError as error: + print(f"testenix: {error}", file=sys.stderr) + return EXIT_USAGE + except PytestInvocationError as error: + print(f"testenix: {error}", file=sys.stderr) + return EXIT_INTERNAL_ERROR + + +def _call_pytest(arguments: Sequence[str]) -> int: + # Kept behind this seam so the CLI contract can be tested without handing + # the current test process to pytest. + from testenix.pytest_adapter import run_pytest + + return run_pytest(arguments) + + +def _misplaced_pytest_command(arguments: argparse.Namespace) -> int: + del arguments + print( + "testenix: put 'pytest' immediately after 'testenix'; " + "Testenix --config applies only to the native run command", + file=sys.stderr, + ) + return EXIT_USAGE + + def _worker_count(value: str) -> int | str: if value == "auto": return value diff --git a/src/testenix/pytest_adapter.py b/src/testenix/pytest_adapter.py new file mode 100644 index 0000000..66c5f2b --- /dev/null +++ b/src/testenix/pytest_adapter.py @@ -0,0 +1,87 @@ +"""Compatibility adapter for existing pytest suites. + +The native Testenix engine deliberately does not import pytest. This adapter +preserves pytest semantics by handing the current CLI process to pytest, +without translating tests, arguments, plugins, configuration, output, signals, +or exit status. +""" + +from __future__ import annotations + +import importlib.util +import os +import sys +from collections.abc import Callable, Sequence +from typing import NoReturn, cast + +PYTEST_INSTALL_HINT = 'python -m pip install "testenix[pytest]"' + + +class PytestUnavailableError(RuntimeError): + """Raised when the compatibility command cannot find pytest.""" + + +class PytestInvocationError(RuntimeError): + """Raised when pytest cannot take over the current CLI process.""" + + +def run_pytest(arguments: Sequence[str] = ()) -> int: + """Run pytest in the current CLI process using the same interpreter. + + POSIX supports a true process overlay. Windows uses pytest's public console + entry point in-process because its ``exec`` family does not provide the same + replacement semantics. Both paths give pytest ownership of terminal + interaction, signals, output, and the final exit status. + """ + + if importlib.util.find_spec("pytest") is None: + raise PytestUnavailableError( + f"pytest is not installed in this Python environment; run: {PYTEST_INSTALL_HINT}" + ) + + if _is_windows(): + return _run_pytest_in_process(arguments) + _exec_pytest(arguments) + + +def _is_windows() -> bool: + return os.name == "nt" + + +def _run_pytest_in_process(arguments: Sequence[str]) -> int: + try: + pytest_module = importlib.import_module("pytest") + except ImportError as error: + raise PytestInvocationError(f"cannot import pytest: {error}") from error + + console_main = getattr(pytest_module, "console_main", None) + if not callable(console_main): + raise PytestInvocationError("installed pytest does not expose console_main") + typed_console_main = cast(Callable[[], int], console_main) + original_argv = sys.argv + sys.argv = [original_argv[0], *arguments] + try: + return int(typed_console_main()) + except (KeyboardInterrupt, SystemExit): + raise + except Exception as error: + raise PytestInvocationError(f"cannot run pytest: {error}") from error + finally: + sys.argv = original_argv + + +def _exec_pytest(arguments: Sequence[str]) -> NoReturn: + command = (sys.executable, "-m", "pytest", *tuple(arguments)) + try: + os.execv(sys.executable, command) + except OSError as error: + raise PytestInvocationError(f"cannot start pytest: {error}") from error + raise AssertionError("os.execv returned unexpectedly") # pragma: no cover + + +__all__ = [ + "PYTEST_INSTALL_HINT", + "PytestInvocationError", + "PytestUnavailableError", + "run_pytest", +] diff --git a/tests/test_pytest_adapter.py b/tests/test_pytest_adapter.py new file mode 100644 index 0000000..2b75f24 --- /dev/null +++ b/tests/test_pytest_adapter.py @@ -0,0 +1,272 @@ +from __future__ import annotations + +import os +import subprocess +import sys +import xml.etree.ElementTree as ET +from pathlib import Path + +import pytest + +from testenix.cli import EXIT_INTERNAL_ERROR, EXIT_INTERRUPTED, EXIT_USAGE, main +from testenix.pytest_adapter import ( + PytestInvocationError, + PytestUnavailableError, + run_pytest, +) + + +def test_pytest_adapter_overlays_exact_interpreter_command_on_posix( + monkeypatch: pytest.MonkeyPatch, +) -> None: + captured: dict[str, object] = {} + + class ProcessReplaced(Exception): + pass + + def fake_execv(executable: str, command: tuple[str, ...]) -> None: + captured["executable"] = executable + captured["command"] = command + raise ProcessReplaced + + monkeypatch.setattr("testenix.pytest_adapter.importlib.util.find_spec", lambda name: object()) + monkeypatch.setattr("testenix.pytest_adapter._is_windows", lambda: False) + monkeypatch.setattr("testenix.pytest_adapter.os.execv", fake_execv) + + with pytest.raises(ProcessReplaced): + run_pytest(("-q", "tests/test_api.py", "-k", "smoke")) + assert captured == { + "executable": sys.executable, + "command": ( + sys.executable, + "-m", + "pytest", + "-q", + "tests/test_api.py", + "-k", + "smoke", + ), + } + + +def test_pytest_adapter_uses_console_entry_point_in_process_on_windows( + monkeypatch: pytest.MonkeyPatch, +) -> None: + captured: list[str] = [] + + class FakePytest: + @staticmethod + def console_main() -> int: + captured.extend(sys.argv[1:]) + return 5 + + monkeypatch.setattr("testenix.pytest_adapter.importlib.util.find_spec", lambda name: object()) + monkeypatch.setattr("testenix.pytest_adapter._is_windows", lambda: True) + monkeypatch.setattr( + "testenix.pytest_adapter.importlib.import_module", lambda name: FakePytest() + ) + + original_argv = sys.argv + assert run_pytest(("--", "-q", "tests")) == 5 + assert captured == ["--", "-q", "tests"] + assert sys.argv is original_argv + + +def test_pytest_adapter_in_process_path_executes_real_pytest() -> None: + source_root = Path(__file__).parents[1] / "src" + python_path = os.pathsep.join(filter(None, (str(source_root), os.environ.get("PYTHONPATH")))) + completed = subprocess.run( + ( + sys.executable, + "-c", + "from testenix.pytest_adapter import _run_pytest_in_process; " + "raise SystemExit(_run_pytest_in_process(('--version',)))", + ), + check=False, + capture_output=True, + text=True, + env={**os.environ, "PYTHONPATH": python_path}, + ) + + assert completed.returncode == 0 + assert completed.stdout.startswith("pytest ") + + +def test_pytest_cli_forwards_options_without_argparse_interpreting_them( + monkeypatch: pytest.MonkeyPatch, +) -> None: + captured: list[tuple[str, ...]] = [] + + def fake_pytest(arguments: tuple[str, ...]) -> int: + captured.append(arguments) + return 5 + + monkeypatch.setattr("testenix.cli._call_pytest", fake_pytest) + + assert main(["pytest", "-q", "tests", "-k", "smoke", "--maxfail=1"]) == 5 + assert main(["pytest", "--help"]) == 5 + assert main(["pytest", "--", "-q", "tests"]) == 5 + assert captured == [ + ("-q", "tests", "-k", "smoke", "--maxfail=1"), + ("--help",), + ("--", "-q", "tests"), + ] + + +def test_pytest_cli_handles_interrupt_and_rejects_global_testenix_config( + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], +) -> None: + def interrupted(arguments: tuple[str, ...]) -> int: + raise KeyboardInterrupt + + monkeypatch.setattr("testenix.cli._call_pytest", interrupted) + assert main(["pytest", "tests"]) == EXIT_INTERRUPTED + assert "interrupted" in capsys.readouterr().err + + assert main(["--config", "pyproject.toml", "pytest"]) == EXIT_USAGE + assert "immediately after" in capsys.readouterr().err + + +def test_pytest_adapter_reports_missing_pytest( + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], +) -> None: + monkeypatch.setattr("testenix.pytest_adapter.importlib.util.find_spec", lambda name: None) + + with pytest.raises(PytestUnavailableError, match=r"testenix\[pytest\]"): + run_pytest() + + def missing_pytest(arguments: tuple[str, ...]) -> int: + raise PytestUnavailableError("pytest is missing") + + monkeypatch.setattr("testenix.cli._call_pytest", missing_pytest) + assert main(["pytest", "tests"]) == EXIT_USAGE + assert "pytest is missing" in capsys.readouterr().err + + +def test_pytest_adapter_reports_process_start_failure( + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], +) -> None: + monkeypatch.setattr("testenix.pytest_adapter.importlib.util.find_spec", lambda name: object()) + monkeypatch.setattr("testenix.pytest_adapter._is_windows", lambda: False) + + def cannot_start(executable: str, command: tuple[str, ...]) -> None: + raise OSError("process unavailable") + + monkeypatch.setattr("testenix.pytest_adapter.os.execv", cannot_start) + with pytest.raises(PytestInvocationError, match="process unavailable"): + run_pytest() + + def broken_pytest(arguments: tuple[str, ...]) -> int: + raise PytestInvocationError("cannot start pytest") + + monkeypatch.setattr("testenix.cli._call_pytest", broken_pytest) + assert main(["pytest"]) == EXIT_INTERNAL_ERROR + assert "cannot start pytest" in capsys.readouterr().err + + +def test_pytest_adapter_executes_existing_pytest_features(tmp_path: Path) -> None: + suite = tmp_path / "suite" + suite.mkdir() + teardown_marker = tmp_path / "fixture-closed" + process_marker = tmp_path / "pytest-process" + pytest_config = tmp_path / "pytest.ini" + pytest_config.write_text( + "[pytest]\nmarkers =\n smoke: compatibility acceptance tests\n", + encoding="utf-8", + ) + (suite / "conftest.py").write_text( + f""" +import os +from pathlib import Path + +import pytest + + +@pytest.fixture(scope="session", autouse=True) +def record_pytest_process(): + Path({str(process_marker)!r}).write_text(str(os.getpid()), encoding="utf-8") + + +@pytest.fixture(scope="module") +def factor(): + yield 3 + Path({str(teardown_marker)!r}).write_text("closed", encoding="utf-8") +""", + encoding="utf-8", + ) + (suite / "test_existing_pytest_suite.py").write_text( + """ +import os + +import pytest + + +@pytest.mark.smoke +@pytest.mark.parametrize( + "value, expected", + [(2, 6), (4, 12)], + ids=["small", "large"], +) +def test_multiplies_with_conftest_fixture(factor, value, expected): + assert factor * value == expected + + +@pytest.mark.smoke +class TestBuiltInFixtures: + def test_monkeypatch(self, monkeypatch): + monkeypatch.setenv("TESTENIX_PYTEST_BRIDGE", "works") + assert os.environ["TESTENIX_PYTEST_BRIDGE"] == "works" + + +@pytest.mark.smoke +@pytest.mark.skip(reason="compatibility skip") +def test_pytest_skip_semantics(): + raise AssertionError("pytest should not execute a skipped test") + + +@pytest.mark.smoke +@pytest.mark.xfail(reason="compatibility xfail", strict=True) +def test_pytest_xfail_semantics(): + assert False +""", + encoding="utf-8", + ) + report = tmp_path / "pytest.xml" + + command = ( + sys.executable, + "-m", + "testenix", + "pytest", + "-q", + "-p", + "no:cacheprovider", + "-c", + str(pytest_config), + "-m", + "smoke", + f"--junitxml={report}", + str(suite), + ) + source_root = Path(__file__).parents[1] / "src" + python_path = os.pathsep.join(filter(None, (str(source_root), os.environ.get("PYTHONPATH")))) + process = subprocess.Popen(command, env={**os.environ, "PYTHONPATH": python_path}) + try: + returncode = process.wait(timeout=60) + except subprocess.TimeoutExpired: + process.kill() + process.wait() + raise + + assert returncode == 0 + assert int(process_marker.read_text(encoding="utf-8")) == process.pid + assert teardown_marker.read_text(encoding="utf-8") == "closed" + test_suite = ET.parse(report).find(".//testsuite") + assert test_suite is not None + assert test_suite.attrib["tests"] == "5" + assert test_suite.attrib["failures"] == "0" + assert test_suite.attrib["errors"] == "0" + assert test_suite.attrib["skipped"] == "2" diff --git a/uv.lock b/uv.lock index 2d54135..afd708d 100644 --- a/uv.lock +++ b/uv.lock @@ -1354,6 +1354,11 @@ name = "testenix" version = "0.1.0" source = { editable = "." } +[package.optional-dependencies] +pytest = [ + { name = "pytest" }, +] + [package.dev-dependencies] dev = [ { name = "check-wheel-contents" }, @@ -1372,13 +1377,15 @@ docs = [ ] [package.metadata] +requires-dist = [{ name = "pytest", marker = "extra == 'pytest'", specifier = ">=8.3,<10" }] +provides-extras = ["pytest"] [package.metadata.requires-dev] dev = [ { name = "check-wheel-contents", specifier = ">=0.6.3,<0.7" }, { name = "mypy", specifier = ">=2.3,<3" }, { name = "pyright", specifier = ">=1.1.411,<2" }, - { name = "pytest", specifier = ">=8.3" }, + { name = "pytest", specifier = ">=8.3,<10" }, { name = "pytest-xdist", specifier = ">=3.6" }, { name = "ruff", specifier = ">=0.9" }, { name = "twine", specifier = ">=6.2,<7" }, From 631f1c2e4b75069188fe5e40d83a9fab9511c034 Mon Sep 17 00:00:00 2001 From: Dominik Franczyk Date: Mon, 20 Jul 2026 15:31:02 +0200 Subject: [PATCH 2/2] test: account for Windows Python launchers --- tests/test_pytest_adapter.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/test_pytest_adapter.py b/tests/test_pytest_adapter.py index 2b75f24..467b07b 100644 --- a/tests/test_pytest_adapter.py +++ b/tests/test_pytest_adapter.py @@ -262,7 +262,11 @@ def test_pytest_xfail_semantics(): raise assert returncode == 0 - assert int(process_marker.read_text(encoding="utf-8")) == process.pid + assert process_marker.read_text(encoding="utf-8").isdigit() + if os.name == "posix": + # POSIX exec preserves the PID. A Windows virtualenv executable may be + # a launcher whose PID differs from the in-process Python interpreter. + assert int(process_marker.read_text(encoding="utf-8")) == process.pid assert teardown_marker.read_text(encoding="utf-8") == "closed" test_suite = ET.parse(report).find(".//testsuite") assert test_suite is not None