E2E Test Expansion

.github/workflows/e2e.yml

@@ -1,7 +1,5 @@
 name: E2E Tests
-
 on:
-
   pull_request:
   push:
     branches:
@@ -15,7 +13,38 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
-        uses: actions/checkout@v2
+        uses: actions/checkout@v7
+        with:
+          # full history so the PR base commit is available to diff against
+          fetch-depth: 0
+
+      - name: Select tests
+        id: select
+        # Per-episode tests run unless EVERY changed file is known not to affect episode pages.
+        #   Anything not listed in SAFE (Hugo version, config.toml, base layouts, CSS/JS, new paths...) runs them.
+        run: |
+          markers="not periodic"
+          if [ "${{ github.event_name }}" = "pull_request" ]; then
+            SAFE='^(docs/|archetypes/|scripts/|data/picks/|[^/]+\.md$|render\.yaml$'
+            SAFE+='|content/'
+            SAFE+='|themes/jb/layouts/(archive|calendar|community|contact|live|picks|sponsors|tags|taxonomy|people|show|section)/'
+            SAFE+='|themes/jb/layouts/(index|404)\.html$|themes/jb/layouts/partials/(live|community|show)/'
+            SAFE+='|themes/jb/assets/js/(live|jb-live)\.js$'
+            SAFE+='|test/|\.github/)'
+            # carve-outs from SAFE that do feed episode pages
+            UNSAFE='^(content/(show|people|sponsors|membership)/'
+            UNSAFE+='|test/(conftest\.py|e2e/test_episode\.py|e2e/episode_data\.py|Pipfile|Pipfile\.lock|docker-compose\.yml)$'
+            UNSAFE+='|\.github/workflows/e2e\.yml$)'
+            changed=$(git diff --name-only ${{ github.event.pull_request.base.sha }} ${{ github.sha }})
+            affecting=$(printf '%s\n' "$changed" | grep -v '^$' | grep -E "$UNSAFE" ; printf '%s\n' "$changed" | grep -v '^$' | grep -vE "$SAFE")
+            if [ -z "$affecting" ]; then
+              markers="not periodic and not episodes"
+            else
+              echo "Running episode tests; changed files that can affect episode pages:"
+              printf '%s\n' "$affecting" | sort -u | head -20
+            fi
+          fi
+          echo "markers=$markers" >> "$GITHUB_OUTPUT"
 
       # using docker compose, so we can re-use local Dockerfiles
       - name: starting up containers
@@ -30,17 +59,20 @@ jobs:
       - name: Run your tests
         working-directory: ./test/
         # https://github.com/actions/runner/issues/241#issuecomment-577360161
-        run: script -e -c "docker compose exec testing pytest -m 'not periodic' --base-url http://jb-com --junitxml report.xml"
+        run: script -e -c "docker compose exec testing pytest -m '${{ steps.select.outputs.markers }}' --base-url http://jb-com --junitxml report.xml"
 
       - name: Save screenshots
-        uses: actions/upload-artifact@v4
+        # screenshots are only taken on failure, so this has to run when the test step fails
+        if: always()
+        uses: actions/upload-artifact@v7
         with:
           name: screenshots
           path: screenshots/
           overwrite: true
+          if-no-files-found: ignore
 
       - name: Publish Test Report
-        uses: dorny/test-reporter@v1
+        uses: dorny/test-reporter@v3
         # TODO: potential solution in the future: https://stackoverflow.com/a/70448851
         if: always() && github.event_name != 'pull_request'
         with:

.github/workflows/periodic_test.yml

@@ -10,7 +10,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
-        uses: actions/checkout@v2
+        uses: actions/checkout@v7
 
       # using docker compose, so we can re-use local Dockerfiles
       - name: starting up containers
@@ -28,7 +28,7 @@ jobs:
         run: script -e -c "docker compose exec testing pytest -m periodic --base-url http://jb-com --junitxml report.xml"
 
       - name: Publish Test Report
-        uses: dorny/test-reporter@v1
+        uses: dorny/test-reporter@v3
         with:
           name: Periodic Tests
           path: './report.xml'

Makefile

@@ -18,6 +18,9 @@ dev: dev-build ## Run the development hugo container
 dev-metrics: dev-build ## Run the development hugo container with metrics
 	docker run --rm -it -v "$${PWD}":/app -w /app -p $${HOST_IP:-127.0.0.1}:1313:1313 jb_hugo:latest serve -D --bind 0.0.0.0 --templateMetrics --templateMetricsHints
 
+dev-tests: dev-build ## Run the development hugo container with every page pre-rendered (for `make tests` with parallel workers)
+	docker run --rm -it -v "$${PWD}":/app -w /app -p $${HOST_IP:-127.0.0.1}:1313:1313 jb_hugo:latest serve -D --bind 0.0.0.0 --disableFastRender
+
 deploy-prod: ## Deploy everything to production
 	docker build -t jb-jbcom .
 	docker compose -f ~/docker-compose.yml up -d --force-recreate jb-jbcom

config/_default/params.toml

@@ -1,7 +1,7 @@
 keywords = "Linux, FreeBSD, howto, Ubuntu, Arch, tech news, HD, app review, Systems, Network, Sysadmin, netsec, Politics, development, podcast, Linux Action Show, TechSNAP, SciByte, FauxShow, Coder Radio, LIVE stream, Bitcoin, PlanB, BSDNow"
 description = "Independent coverage of Linux, BSD, Open Source, System Administration, Information Security, News, Software Development and Discussion."
 author = "Jupiter Broadcasting"
-ogimage = "/images/jupiter-broadcasting.png" # Open Graph default image
+ogimage = "/images/logo.png" # Open Graph default image
 
 # itunes feed info
 feed.copyright = "Copyright 2022 Jupiter Broadcasting" #do not use markdown in this field; it is used in the feed

pytest.ini

@@ -2,8 +2,9 @@
 python_files = test_*
 python_classes = *Tests
 python_functions = test_*
-addopts = -ra -v
+addopts = -ra -v -n auto --color=yes --browser chromium --browser firefox --browser webkit --screenshot only-on-failure --output screenshots
 testpaths = test
 markers =
     dev: "the current test(s) you're working on for developing"
     periodic: "tests that need to run on a schedule instead of running when a PR is made"
+    episodes: "per-episode tests over the content sample; CI runs them only on PRs that touch episode content or templates"

test/conftest.py

@@ -1,11 +1,56 @@
 
+import re
 from pathlib import Path
-from typing import Dict, Generator, List, Tuple, Callable
-from pytest import fixture
+from typing import Dict, Generator, List, Tuple
+from pytest import fixture, hookimpl
 from _pytest.fixtures import SubRequest
 from pytest_base_url.plugin import base_url
-from playwright.sync_api import Playwright, APIRequestContext, Route, Page, Browser
-import json
+from playwright.sync_api import Playwright, APIRequestContext, Page, Browser
+
+# device -> the engine it really ships with (all iOS browsers are WebKit), so each device
+#   only runs on that browser instead of all three
+MOBILE_DEVICES = {
+    # phones: narrowest, current iPhone width, newest Pixel in Playwright 1.50
+    "Galaxy S9+": "chromium",
+    "iPhone 15": "webkit",
+    "Pixel 7": "chromium",
+    # tablets: 768 is Bulma's last mobile px, 834 is Bulma tablet,
+    #   1194 is desktop layout but still under $navbar-breakpoint (1220px) so the burger shows
+    "iPad Mini": "webkit",
+    "iPad Pro 11": "webkit",
+    "iPad Pro 11 landscape": "webkit",
+}
+
+@hookimpl(wrapper=True)
+def pytest_runtest_makereport(item, call):
+    # exposes the call-phase result to fixtures, so mobile_device_tuple can screenshot on failure
+    report = yield
+    if report.when == "call":
+        item.rep_call = report
+    return report
+
+
+def pytest_collection_modifyitems(config, items):
+    """
+    Deselect (rather than skip) browser copies of tests marked only_browser/skip_browser,
+    so the summary doesn't report hundreds of skips that aren't real
+    """
+    keep, drop = [], []
+    for item in items:
+        browser = getattr(item, "callspec", None) and item.callspec.params.get("browser_name")
+        only = {b for m in item.iter_markers("only_browser") for b in m.args}
+        device = getattr(item, "callspec", None) and item.callspec.params.get("mobile_device_tuple")
+        if device:
+            only = {MOBILE_DEVICES[device]}
+        skip = {b for m in item.iter_markers("skip_browser") for b in m.args}
+        if browser and ((only and browser not in only) or browser in skip):
+            drop.append(item)
+        else:
+            keep.append(item)
+    if drop:
+        config.hook.pytest_deselected(items=drop)
+        items[:] = keep
+
 
 @fixture
 def get_test_dir() -> Path:
@@ -28,14 +73,8 @@ def screenshot_dir() -> Path:
 # also should be able to view them here:
 # https://github.com/microsoft/playwright/blob/v1.25.2/packages/playwright-core/src/server/deviceDescriptorsSource.json
 @fixture(
-    # doing session so it doesn't have to be re-created for each usage
-    scope="session",
-    # some example phone sizes to use
-    params=[
-        "iPhone 13",
-        "Pixel 5",
-        "Galaxy S9+",
-    ]
+    # function scope so menu state from one test can't leak into the next
+    params=list(MOBILE_DEVICES),
 )
 def mobile_device_tuple(
     # how to access the params
@@ -49,17 +88,19 @@ def mobile_device_tuple(
     base_url: base_url,
 ) -> Tuple[Page,str]:
 
-    # based on here: https://playwright.dev/python/docs/emulation#devices
     context = browser.new_context(
         base_url=base_url,
-        # based on this info: https://playwright.dev/python/docs/emulation#devices
         **playwright.devices[request.param]
     )
+    page = context.new_page()
     try:
         # essentially a "return", but used with generators
-        yield (context.new_page(), request.param)
-    except Exception as excpt:
-        raise excpt
+        yield (page, request.param)
+        # pytest-playwright's --screenshot only covers its own page fixture, not contexts made here
+        report = getattr(request.node, "rep_call", None)
+        if report and report.failed:
+            slug = re.sub(r"[^\w.-]+", "-", request.node.nodeid).strip("-")
+            page.screenshot(path=Path(request.config.getoption("--output")) / slug / "screenshot.png")
     finally:
         # supposed to get rid of coookies/other stored info after generator is done
         # https://playwright.dev/python/docs/api/class-apirequestcontext#api-request-context-dispose
@@ -161,30 +202,3 @@ def expect_nav_items() -> List[Dict[str,str]]:
         # failing on tests here: https://github.com/JupiterBroadcasting/jupiterbroadcasting.com/runs/8254156209?check_suite_focus=true#step:9:26
         {'title': 'Contact', 'href': '/contact/'},
     ]
-
-@fixture
-def set_live() -> Callable:
-    return _replace_live_event
-
-@staticmethod
-def _replace_live_event(page: Page) -> None:
-    def handle_route(route: Route) -> None:
-        # fetch original response
-        response = page.request.fetch(route.request)
-        json_data = response.json()
-
-        # override response values
-        if json_data.get('data'):
-            json_data['data'][0]['isLive'] = True
-        json_data['total'] = 1
-
-        # setting live event
-        route.fulfill(
-            # Pass all fields from the response
-            response=response,
-            body=json.dumps(json_data),
-        )
-    page.route(
-        "https://jupiter.tube/api/v1/video-channels/live/**",
-        handle_route
-    )

test/e2e/episode_data.py

@@ -0,0 +1,81 @@
+import json
+import re
+from datetime import datetime, timezone
+from dataclasses import dataclass, field
+from pathlib import Path
+
+CONTENT: Path = Path(__file__).parents[2] / "content"
+CONTENT_SHOW: Path = CONTENT / "show"
+META_FILES: set[str] = {"_index.md", "subscribe.md"}
+ACTIVE_SAMPLE: int = 5  # the scraper can rewrite an active show's last 5 episodes
+ARCHIVED_SAMPLE: int = 2
+
+EXTRA_EPISODES: list[tuple[str, str]] = [
+    ("linux-unplugged", "434"),  # 3 hosts and 3 guests
+    ("coder-radio", "343"),  # no sponsors
+    ("coder-radio", "443"),  # tag with an apostrophe (issue #534)
+]
+
+
+@dataclass(frozen=True)
+class Episode:
+    show: str
+    number: str
+    url: str
+    data: dict = field(compare=False, hash=False, repr=False)
+
+
+def load_episode(path: Path) -> Episode:
+    data, _ = json.JSONDecoder().raw_decode(path.read_text().lstrip())
+    show = path.parent.name
+    number = str(data["episode"])
+    return Episode(show=show, number=number, url=f"/show/{show}/{str(data['slug']).lower()}/", data=data)
+
+
+def episodes_by_date(show_dir: Path) -> list[Episode]:
+    # filenames aren't chronological (e.g. the-launch T-0.md sorts after 0097.md)
+    episodes = [load_episode(p) for p in show_dir.glob("*.md") if p.name not in META_FILES]
+    return sorted(episodes, key=lambda e: _published(e.data["date"]))
+
+
+def _published(date: str) -> datetime:
+    # Hugo reads dates without an offset as UTC (no timeZone set in config.toml)
+    parsed = datetime.fromisoformat(date)
+    return parsed if parsed.tzinfo else parsed.replace(tzinfo=timezone.utc)
+
+
+def is_active(show_dir: Path) -> bool:
+    return re.search(r"^\s*active\s*=\s*true\b", (show_dir / "_index.md").read_text(), re.M) is not None
+
+
+def sample_episodes() -> list[Episode]:
+    """
+    Oldest + newest episodes of every show (ACTIVE_SAMPLE each for active shows, ARCHIVED_SAMPLE for archived),
+    plus EXTRA_EPISODES (known edge cases)
+    """
+    selected: list[Episode] = []
+    for show_dir in sorted(p for p in CONTENT_SHOW.iterdir() if p.is_dir()):
+        by_date = episodes_by_date(show_dir)
+        n = ACTIVE_SAMPLE if is_active(show_dir) else ARCHIVED_SAMPLE
+        selected += by_date[:n] + by_date[-n:]
+    selected += [load_episode(CONTENT_SHOW / show / f"{int(number):04d}.md") for show, number in EXTRA_EPISODES]
+
+    episodes: dict[str, Episode] = {}
+    for episode in selected:
+        episodes.setdefault(episode.url, episode)
+    return list(episodes.values())
+
+
+def person_type(username: str) -> str | None:
+    """
+    "host"/"guest" from content/people/<username>.md (YAML front matter), or None if there's no people page
+    """
+    path = CONTENT / "people" / f"{username}.md"
+    if not path.exists():
+        return None
+    match = re.search(r"^type: *[\"']?(\w+)", path.read_text(), re.M)
+    return match.group(1) if match else None
+
+
+def sponsor_exists(shortname: str) -> bool:
+    return (CONTENT / "sponsors" / f"{shortname}.md").exists()

test/e2e/test_404.py

@@ -0,0 +1,24 @@
+from uuid import uuid4
+
+from pytest import fixture
+from playwright.sync_api import Page, expect
+
+
+@fixture(autouse=True)
+def setup(page: Page):
+    """
+    Code here will run for every test in this file
+    """
+    response = page.goto(f"/this-page-does-not-exist-{uuid4()}/")
+    assert response.status == 404
+
+
+def test_404_contents(page: Page):
+    """
+    Ensure text "Page Not Found" and unicorn image is present on the 404 page
+    """
+    title = page.locator(".title.has-text-centered")
+    expect(title).to_contain_text("Page Not Found")
+
+    unicorn = page.locator("#img-404")
+    expect(unicorn).to_be_visible()