diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 8c86f71..862a2e9 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -59,4 +59,4 @@ jobs: # Run e2e tests. # DEV: Enable this after remmoving requirement to test with Poetry and Pipenv. - # pytest tests/e2e_tests -s + # pytest tests/e2e_tests -s --include-core-tests diff --git a/README.md b/README.md index 457dabd..54a562d 100644 --- a/README.md +++ b/README.md @@ -79,4 +79,10 @@ $ pytest tests/e2e_tests -s The `-s` flag is not required, but since the tests takes a fairly long time it's helpful to see the output as it's running. You can get a good sense of which steps are working by watching the output as the test runs. +This will only run the integration tests for each new plugin in the e2e test. It will not run all of django-simple-deploy's tests for each new plugin. If you want to run the full set of tests, include an additional flag: + +```sh +$ pytest tests/e2e_tests -s --include-core-tests +``` + Currently, CI tests only run unit and integration tests. There's an open task in django-simple-deploy to remove the dependence on poetry and pipenv for running tests. When that is implemented, e2e tests can run much more easily in CI. diff --git a/tests/e2e_tests/conftest.py b/tests/e2e_tests/conftest.py new file mode 100644 index 0000000..35423e4 --- /dev/null +++ b/tests/e2e_tests/conftest.py @@ -0,0 +1,23 @@ +"""Configuration for e2e test runs.""" + +from dataclasses import dataclass + +import pytest + + +def pytest_addoption(parser): + parser.addoption( + "--include-core-tests", + action="store_true", + help="Run the full set of core django-simple-deploy tests for each new plugin.", + ) + +@dataclass +class CLIOptions: + include_core_tests: bool=False + +@pytest.fixture(scope="session") +def cli_options(request): + return CLIOptions( + include_core_tests=request.config.getoption("--include-core-tests"), + ) diff --git a/tests/e2e_tests/test_basic_plugin.py b/tests/e2e_tests/test_basic_plugin.py index 519bd7d..a147b4f 100644 --- a/tests/e2e_tests/test_basic_plugin.py +++ b/tests/e2e_tests/test_basic_plugin.py @@ -4,7 +4,7 @@ - Generates a new plugin. - Sets up a development environment for django-simple-deploy core. - Installs the new plugin to the development environment. -- Runs the initial set of tests against the plugin. +- Runs the plugin's integration tests. Notes: - This makes an editable install of both django-simple-deploy and the new plugin. @@ -18,33 +18,20 @@ from pathlib import Path import subprocess import shlex -import re import pytest from utils.plugin_config import PluginConfig +from tests.e2e_tests.utils import e2e_utils import generate_plugin as gp -def uv_available(): - """Ensure uv is available before running test.""" - cmd = "uv self version -q" - cmd_parts = shlex.split(cmd) - try: - subprocess.run(cmd_parts) - return True - except FileNotFoundError: - # This is the exception raised on macOS when the command uv is unavailable. - return False - - -### --- Test function --- - -@pytest.mark.skipif(not uv_available(), reason="uv must be installed in order to run e2e tests.") -def test_new_plugin_e2e(tmp_path_factory): +@pytest.mark.skipif(not e2e_utils.uv_available(), reason="uv must be installed in order to run e2e tests.") +def test_new_plugin_e2e(tmp_path_factory, cli_options): # Flag to temporarily disable running dsd and plugin tests. This is helpful when # examining this environment. Otherwise pytest runs so many tests, this one can't - # easily be found. + # easily be found. This is not a CLI arg, because it only needs to be modified when you're + # working on this test, not when you're just running it. run_core_plugin_tests = True # Build a new plugin in temp dir. @@ -67,7 +54,7 @@ def test_new_plugin_e2e(tmp_path_factory): # Clone django-simple-deploy in temp env. path_dsd = tmp_path / "django-simple-deploy" - cmd = f"git clone https://github.com/django-simple-deploy/django-simple-deploy.git {path_dsd.as_posix()}" + cmd = f"git clone https://github.com/django-simple-deploy/django-simple-deploy.git {path_dsd.as_posix()} --depth 1" cmd_parts = shlex.split(cmd) subprocess.run(cmd_parts) @@ -85,23 +72,7 @@ def test_new_plugin_e2e(tmp_path_factory): if run_core_plugin_tests: # Run core tests without a plugin installed. - tests_dir = path_dsd / "tests" - cmd = f"{path_to_python} -m pytest {tests_dir.as_posix()}" - cmd_parts = shlex.split(cmd) - output = subprocess.run(cmd_parts, capture_output=True) - stdout = output.stdout.decode() - - assert "test session starts" in stdout - assert "[100%]" in stdout - - # Check number of core tests that passed and skipped. - re_passed_skipped = r"""(\d*) passed, (\d*) skipped""" - m = re.findall(re_passed_skipped, stdout) - if m: - passed = int(m[0][0]) - skipped = int(m[0][1]) - assert passed >= 31 - assert skipped >= 22 + e2e_utils.run_core_tests(path_dsd, path_to_python, cli_options) # Install plugin editable to django-simple-deploy env. cmd = f'uv pip install --python {path_to_python} -e "{path_new_plugin.as_posix()}[dev]"' @@ -109,22 +80,8 @@ def test_new_plugin_e2e(tmp_path_factory): subprocess.run(cmd_parts) if run_core_plugin_tests: - # Run core tests **with** the new plugin installed. - tests_dir = path_dsd / "tests" - cmd = f"cd {path_dsd.as_posix()} && source .venv/bin/activate && pytest" - output = subprocess.run(cmd, capture_output=True,shell=True) - stdout = output.stdout.decode() - - assert "test session starts" in stdout - assert "[100%]" in stdout - - # Check number of core tests that passed and skipped. - m = re.findall(re_passed_skipped, stdout) - if m: - passed = int(m[0][0]) - skipped = int(m[0][1]) - assert passed >= 65 - assert skipped >= 6 + e2e_utils.run_core_plugin_tests(path_dsd, plugin_config, cli_options) + # Remove plugin, and test another one. # This is much faster than having a completely separate test. We lose some test @@ -156,19 +113,4 @@ def test_new_plugin_e2e(tmp_path_factory): subprocess.run(cmd_parts) if run_core_plugin_tests: - # Run core tests **with** the new plugin installed. - tests_dir = path_dsd / "tests" - cmd = f"cd {path_dsd.as_posix()} && source .venv/bin/activate && pytest" - output = subprocess.run(cmd, capture_output=True,shell=True) - stdout = output.stdout.decode() - - assert "test session starts" in stdout - assert "[100%]" in stdout - - # Check number of core tests that passed and skipped. - m = re.findall(re_passed_skipped, stdout) - if m: - passed = int(m[0][0]) - skipped = int(m[0][1]) - assert passed >= 65 - assert skipped >= 6 \ No newline at end of file + e2e_utils.run_core_plugin_tests(path_dsd, plugin_config, cli_options) diff --git a/tests/e2e_tests/utils/e2e_utils.py b/tests/e2e_tests/utils/e2e_utils.py new file mode 100644 index 0000000..e997a93 --- /dev/null +++ b/tests/e2e_tests/utils/e2e_utils.py @@ -0,0 +1,77 @@ +"""Utility functions for e2e tests.""" + +import shlex +import subprocess +import re + +from utils.generator_utils import get_platform_name_lower + + +def uv_available(): + """Ensure uv is available before running test.""" + cmd = "uv self version -q" + cmd_parts = shlex.split(cmd) + try: + subprocess.run(cmd_parts) + return True + except FileNotFoundError: + # This is the exception raised on macOS when the command uv is unavailable. + return False + + +def run_core_tests(path_dsd, path_to_python, cli_options): + """Run django-simple-deploy's test suite with no plugin installed.""" + tests_dir = path_dsd / "tests" + cmd = f"{path_to_python} -m pytest {tests_dir.as_posix()}" + cmd_parts = shlex.split(cmd) + output = subprocess.run(cmd_parts, capture_output=True) + stdout = output.stdout.decode() + + assert "[100%]" in stdout + check_core_plugin_tests(stdout, cli_options, core_only=True) + +def run_core_plugin_tests(path_dsd, plugin_config, cli_options): + """Run django-simple-deploy's test suite with a plugin installed.""" + tests_dir = path_dsd / "tests" + platform_name_lower = get_platform_name_lower(plugin_config.platform_name) + cmd = get_core_plugin_test_cmd(path_dsd, cli_options, platform_name_lower) + + output = subprocess.run(cmd, capture_output=True,shell=True) + stdout = output.stdout.decode() + + assert "[100%]" in stdout + check_core_plugin_tests(stdout, cli_options) + + +def get_core_plugin_test_cmd(path_dsd, cli_options, platform_name_lower): + """Get the command for running django-simple-deploy tests after a new plugin has been installed.""" + test_filename = f"test_{platform_name_lower}_config.py" + + if cli_options.include_core_tests: + # Run full set of core django-simple-deploy tests, and plugin integration tests. + cmd = f"cd {path_dsd.as_posix()} && source .venv/bin/activate && pytest" + else: + # Only run the new plugin's integration tests. + cmd = f"cd {path_dsd.as_posix()} && source .venv/bin/activate && pytest -k {test_filename}" + + return cmd + + +def check_core_plugin_tests(stdout, cli_options, core_only=False): + """Check number of core and plugin tests that passed and skipped.""" + # No assertions about number skipped, but helpful to know at times. + re_passed_skipped = r"""(\d*) passed, (\d*) skipped""" + m = re.findall(re_passed_skipped, stdout) + if m: + passed = int(m[0][0]) + skipped = int(m[0][1]) + + if core_only: + # Core tests, no plugin installed. + assert passed >= 31 + elif cli_options.include_core_tests: + # Core tests, and plugin's integrationtests. + assert passed >= 65 + else: + # Plugin's integration tests, no core tests. + assert passed >= 18