Refactor e2e tests

.github/workflows/test.yaml

@@ -59,4 +59,4 @@ jobs:
 
           # Run e2e tests.
           # DEV: Enable this after remmoving requirement to test with Poetry and Pipenv.
-          # pytest tests/e2e_tests -s
+          # pytest tests/e2e_tests -s --include-core-tests

README.md

@@ -79,4 +79,10 @@ $ pytest tests/e2e_tests -s
 
 The `-s` flag is not required, but since the tests takes a fairly long time it's helpful to see the output as it's running. You can get a good sense of which steps are working by watching the output as the test runs.
 
+This will only run the integration tests for each new plugin in the e2e test. It will not run all of django-simple-deploy's tests for each new plugin. If you want to run the full set of tests, include an additional flag:
+
+```sh
+$ pytest tests/e2e_tests -s --include-core-tests
+```
+
 Currently, CI tests only run unit and integration tests. There's an open task in django-simple-deploy to remove the dependence on poetry and pipenv for running tests. When that is implemented, e2e tests can run much more easily in CI.

tests/e2e_tests/conftest.py

@@ -0,0 +1,23 @@
+"""Configuration for e2e test runs."""
+
+from dataclasses import dataclass
+
+import pytest
+
+
+def pytest_addoption(parser):
+    parser.addoption(
+        "--include-core-tests",
+        action="store_true",
+        help="Run the full set of core django-simple-deploy tests for each new plugin.",
+    )
+
+@dataclass
+class CLIOptions:
+    include_core_tests: bool=False
+
+@pytest.fixture(scope="session")
+def cli_options(request):
+    return CLIOptions(
+        include_core_tests=request.config.getoption("--include-core-tests"),
+    )

tests/e2e_tests/test_basic_plugin.py

@@ -4,7 +4,7 @@
 - Generates a new plugin.
 - Sets up a development environment for django-simple-deploy core.
 - Installs the new plugin to the development environment.
-- Runs the initial set of tests against the plugin.
+- Runs the plugin's integration tests.
 
 Notes:
 - This makes an editable install of both django-simple-deploy and the new plugin.
@@ -18,33 +18,20 @@
 from pathlib import Path
 import subprocess
 import shlex
-import re
 
 import pytest
 
 from utils.plugin_config import PluginConfig
+from tests.e2e_tests.utils import e2e_utils
 import generate_plugin as gp
 
 
-def uv_available():
-    """Ensure uv is available before running test."""
-    cmd = "uv self version -q"
-    cmd_parts = shlex.split(cmd)
-    try:
-        subprocess.run(cmd_parts)
-        return True
-    except FileNotFoundError:
-        # This is the exception raised on macOS when the command uv is unavailable.
-        return False
-
-
-### --- Test function ---
-
-@pytest.mark.skipif(not uv_available(), reason="uv must be installed in order to run e2e tests.")
-def test_new_plugin_e2e(tmp_path_factory):
+@pytest.mark.skipif(not e2e_utils.uv_available(), reason="uv must be installed in order to run e2e tests.")
+def test_new_plugin_e2e(tmp_path_factory, cli_options):
     # Flag to temporarily disable running dsd and plugin tests. This is helpful when
     # examining this environment. Otherwise pytest runs so many tests, this one can't
-    # easily be found.
+    # easily be found. This is not a CLI arg, because it only needs to be modified when you're
+    # working on this test, not when you're just running it.
     run_core_plugin_tests = True
 
     # Build a new plugin in temp dir.
@@ -67,7 +54,7 @@ def test_new_plugin_e2e(tmp_path_factory):
 
     # Clone django-simple-deploy in temp env.
     path_dsd = tmp_path / "django-simple-deploy"
-    cmd = f"git clone https://github.com/django-simple-deploy/django-simple-deploy.git {path_dsd.as_posix()}"
+    cmd = f"git clone https://github.com/django-simple-deploy/django-simple-deploy.git {path_dsd.as_posix()} --depth 1"
     cmd_parts = shlex.split(cmd)
     subprocess.run(cmd_parts)
 
@@ -85,46 +72,16 @@ def test_new_plugin_e2e(tmp_path_factory):
 
     if run_core_plugin_tests:
         # Run core tests without a plugin installed.
-        tests_dir = path_dsd / "tests"
-        cmd = f"{path_to_python} -m pytest {tests_dir.as_posix()}"
-        cmd_parts = shlex.split(cmd)
-        output = subprocess.run(cmd_parts, capture_output=True)
-        stdout = output.stdout.decode()
-
-        assert "test session starts" in stdout
-        assert "[100%]" in stdout
-
-        # Check number of core tests that passed and skipped.
-        re_passed_skipped = r"""(\d*) passed, (\d*) skipped"""
-        m = re.findall(re_passed_skipped, stdout)
-        if m:
-            passed = int(m[0][0])
-            skipped = int(m[0][1])
-            assert passed >= 31
-            assert skipped >= 22
+        e2e_utils.run_core_tests(path_dsd, path_to_python, cli_options)
 
     # Install plugin editable to django-simple-deploy env.
     cmd = f'uv pip install --python {path_to_python} -e "{path_new_plugin.as_posix()}[dev]"'
     cmd_parts = shlex.split(cmd)
     subprocess.run(cmd_parts)
 
     if run_core_plugin_tests:
-        # Run core tests **with** the new plugin installed.
-        tests_dir = path_dsd / "tests"
-        cmd = f"cd {path_dsd.as_posix()} && source .venv/bin/activate && pytest"
-        output = subprocess.run(cmd, capture_output=True,shell=True)
-        stdout = output.stdout.decode()
-
-        assert "test session starts" in stdout
-        assert "[100%]" in stdout
-
-        # Check number of core tests that passed and skipped.
-        m = re.findall(re_passed_skipped, stdout)
-        if m:
-            passed = int(m[0][0])
-            skipped = int(m[0][1])
-            assert passed >= 65
-            assert skipped >= 6
+        e2e_utils.run_core_plugin_tests(path_dsd, plugin_config, cli_options)
+
 
     # Remove plugin, and test another one.
     # This is much faster than having a completely separate test. We lose some test
@@ -156,19 +113,4 @@ def test_new_plugin_e2e(tmp_path_factory):
     subprocess.run(cmd_parts)
 
     if run_core_plugin_tests:
-        # Run core tests **with** the new plugin installed.
-        tests_dir = path_dsd / "tests"
-        cmd = f"cd {path_dsd.as_posix()} && source .venv/bin/activate && pytest"
-        output = subprocess.run(cmd, capture_output=True,shell=True)
-        stdout = output.stdout.decode()
-
-        assert "test session starts" in stdout
-        assert "[100%]" in stdout
-
-        # Check number of core tests that passed and skipped.
-        m = re.findall(re_passed_skipped, stdout)
-        if m:
-            passed = int(m[0][0])
-            skipped = int(m[0][1])
-            assert passed >= 65
-            assert skipped >= 6
+        e2e_utils.run_core_plugin_tests(path_dsd, plugin_config, cli_options)

tests/e2e_tests/utils/e2e_utils.py

@@ -0,0 +1,77 @@
+"""Utility functions for e2e tests."""
+
+import shlex
+import subprocess
+import re
+
+from utils.generator_utils import get_platform_name_lower
+
+
+def uv_available():
+    """Ensure uv is available before running test."""
+    cmd = "uv self version -q"
+    cmd_parts = shlex.split(cmd)
+    try:
+        subprocess.run(cmd_parts)
+        return True
+    except FileNotFoundError:
+        # This is the exception raised on macOS when the command uv is unavailable.
+        return False
+
+
+def run_core_tests(path_dsd, path_to_python, cli_options):
+    """Run django-simple-deploy's test suite with no plugin installed."""
+    tests_dir = path_dsd / "tests"
+    cmd = f"{path_to_python} -m pytest {tests_dir.as_posix()}"
+    cmd_parts = shlex.split(cmd)
+    output = subprocess.run(cmd_parts, capture_output=True)
+    stdout = output.stdout.decode()
+
+    assert "[100%]" in stdout
+    check_core_plugin_tests(stdout, cli_options, core_only=True)
+
+def run_core_plugin_tests(path_dsd, plugin_config, cli_options):
+    """Run django-simple-deploy's test suite with a plugin installed."""
+    tests_dir = path_dsd / "tests"
+    platform_name_lower = get_platform_name_lower(plugin_config.platform_name)
+    cmd = get_core_plugin_test_cmd(path_dsd, cli_options, platform_name_lower)
+
+    output = subprocess.run(cmd, capture_output=True,shell=True)
+    stdout = output.stdout.decode()
+
+    assert "[100%]" in stdout
+    check_core_plugin_tests(stdout, cli_options)
+
+
+def get_core_plugin_test_cmd(path_dsd, cli_options, platform_name_lower):
+    """Get the command for running django-simple-deploy tests after a new plugin has been installed."""
+    test_filename = f"test_{platform_name_lower}_config.py"
+
+    if cli_options.include_core_tests:
+        # Run full set of core django-simple-deploy tests, and plugin integration tests.
+        cmd = f"cd {path_dsd.as_posix()} && source .venv/bin/activate && pytest"
+    else:
+        # Only run the new plugin's integration tests.
+        cmd = f"cd {path_dsd.as_posix()} && source .venv/bin/activate && pytest -k {test_filename}"
+
+    return cmd
+
+
+def check_core_plugin_tests(stdout, cli_options, core_only=False):
+    """Check number of core and plugin tests that passed and skipped."""
+    # No assertions about number skipped, but helpful to know at times.
+    re_passed_skipped = r"""(\d*) passed, (\d*) skipped"""
+    m = re.findall(re_passed_skipped, stdout)
+    if m:
+        passed = int(m[0][0])
+        skipped = int(m[0][1])
+
+        if core_only:
+            # Core tests, no plugin installed.
+            assert passed >= 31
+        elif cli_options.include_core_tests:
+            # Core tests, and plugin's integrationtests.
+            assert passed >= 65
+        else:
+            # Plugin's integration tests, no core tests.
+            assert passed >= 18