diff --git a/docs/skills/ci-ops/ops/references/cross-image-tag-skipping-bluefin-on-non-bluefin-images.md b/docs/skills/ci-ops/ops/references/cross-image-tag-skipping-bluefin-on-non-bluefin-images.md index 2acb269c5..2474541fc 100644 --- a/docs/skills/ci-ops/ops/references/cross-image-tag-skipping-bluefin-on-non-bluefin-images.md +++ b/docs/skills/ci-ops/ops/references/cross-image-tag-skipping-bluefin-on-non-bluefin-images.md @@ -10,24 +10,24 @@ metadata: ## Cross-image tag skipping: @bluefin on non-bluefin images -The common suite skips scenarios tagged `@bluefin` when `IMAGE` env var refers to a -non-bluefin image (e.g. dakota). This is implemented via `_is_bluefin_image()` in -`tests/common/features/environment.py`. +The common and smoke suites skip scenarios tagged `@bluefin` unless the image +under test ships Bluefin's common desktop layer. Both environments use +`tests/shared/image_family.py`. -**Pitfall**: match the image *name*, not the full URL. The org name `projectbluefin` -contains `"bluefin"`, so naively checking `"bluefin" in image_url.lower()` returns -`True` for `ghcr.io/projectbluefin/dakota:testing`. +Read `BASE_IMAGE` before `IMAGE`: composed runs replace `IMAGE` with +`ghcr.io//testsuite-e2e:run-`, which carries no family. Match only +the image-name component, not the org: `projectbluefin` contains `bluefin`, so a +URL-wide check would select Bluefin scenarios on Dakota. -Correct pattern: +Bluefin-family names are `bluefin`, `bazzite`, `utah`, and `common`. Common's PR +e2e image is named `common`, but it overlays common's `shared` and `bluefin` +trees onto its base image. Utah is Bluefin built on Fedora Hummingbird. Dakota +remains separate and selects `@dakota_only` scenarios. In the common suite, +`@skip_utah` excludes a `@bluefin` scenario on Utah only: it marks content that +Bluefin's common pin still ships but Utah's newer pin does not (the Searchlight +dconf scenario, until #1076 replaces it). -```python -def _is_bluefin_image(image: str) -> bool: - lower = image.lower() - name = lower.split("/")[-1].split(":")[0].split("@")[0] - return "bluefin" in name or "bazzite" in lower -``` - -This extracts the image name component (`bluefin`, `dakota`, etc.) before checking. - -The smoke suite environment replicates this same pattern so `@bluefin` tags are -respected in AT-SPI tests as well. +The smoke suite runs inside the GUI runner container, so it only sees +`BASE_IMAGE` if `e2e.yml` passes it into that container (`-e BASE_IMAGE` and +`-D image=`). Without them, composed smoke runs still report the run-scoped ref +and skip every `@bluefin` and `@dakota_only` scenario. diff --git a/docs/skills/test-authoring/suite-map/SKILL.md b/docs/skills/test-authoring/suite-map/SKILL.md index be1fa336f..9f5e2353d 100644 --- a/docs/skills/test-authoring/suite-map/SKILL.md +++ b/docs/skills/test-authoring/suite-map/SKILL.md @@ -139,8 +139,9 @@ Set `chunked_enabled: true` once `ghcr.io/projectbluefin/bluefin:latest` ships z | Tag | Meaning | |---|---| | `@smoke_suite` | Runs as part of the standard smoke suite | -| `@bluefin` | Smoke scenario runs only when the image name contains `bluefin`; smoke `environment.py` skips it elsewhere | -| `@dakota_only` | Scenario runs only when the image name contains `dakota`; the `smoke` and `common` `environment.py` files skip it elsewhere. Only the image **name** is matched, so the `projectbluefin` org name cannot false-positive | +| `@bluefin` | Scenario runs on Bluefin-family image names (`bluefin`, `bazzite`, `utah`, `common`); smoke and common read `BASE_IMAGE` before `IMAGE`, so composed runs keep their family | +| `@dakota_only` | Scenario runs only when the image name contains `dakota`; smoke and common read `BASE_IMAGE` before `IMAGE`. Only the image **name** is matched, so the `projectbluefin` org name cannot false-positive | +| `@skip_utah` | Common scenario skips when the image name contains `utah`: it asserts content Bluefin's common pin still ships but Utah's newer pin does not. Remove it when the scenario is replaced (#1076) | | `@dx_only` / `@developer_suite` | DX variant only | | `@nvidia_only` | NVIDIA variant only | | `@flatcar_suite` | Flatcar OS only | diff --git a/tests/common/features/common_dconf.feature b/tests/common/features/common_dconf.feature index 1cc00e034..df311f1d9 100644 --- a/tests/common/features/common_dconf.feature +++ b/tests/common/features/common_dconf.feature @@ -84,6 +84,8 @@ Feature: Bluefin common dconf and GSettings defaults * SSH command return code is "0" * SSH command output stripped "is" "'xdg-terminal-exec'" + # Bluefin's pinned common still ships Searchlight; Utah's newer pin does not (#1076). + @skip_utah Scenario: Searchlight extension is configured * Run SSH command: "dconf read /org/gnome/shell/extensions/search-light/shortcut-search" * SSH command return code is "0" diff --git a/tests/common/features/environment.py b/tests/common/features/environment.py index dfd4fb736..5387c39af 100644 --- a/tests/common/features/environment.py +++ b/tests/common/features/environment.py @@ -7,6 +7,12 @@ import os import shlex +from tests.shared.image_family import ( + image_under_test, + is_bluefin_image as _is_bluefin_image, + is_dakota_image as _is_dakota_image, + is_utah_image as _is_utah_image, +) from tests.shared.ssh_config import _first_value from tests.shared.ssh_steps import * # noqa: F401,F403 from tests.shared.ssh_steps import run_ssh @@ -21,31 +27,6 @@ def record_end(context, scenario): return None -def _is_bluefin_image(image: str) -> bool: - """Return True if the image reference looks like a Bluefin image. - - Matches the image name component only (e.g. "bluefin" in - "ghcr.io/projectbluefin/bluefin:testing") to avoid false-positives where - the org name "projectbluefin" would match a dakota image URL. - """ - lower = image.lower() - # Extract image name: last path segment before any tag/digest - name = lower.split("/")[-1].split(":")[0].split("@")[0] - return "bluefin" in name or "bazzite" in lower - - -def _is_dakota_image(image: str) -> bool: - """Return True if the image reference looks like a Dakota image. - - Matches the image name component only (e.g. "dakota" in - "ghcr.io/projectbluefin/dakota:testing") to avoid false-positives where - the org name "projectbluefin" would match a Bluefin image URL. - """ - lower = image.lower() - name = lower.split("/")[-1].split(":")[0].split("@")[0] - return "dakota" in name - - def _is_projectbluefin_image(image: str) -> bool: """Return True if the image reference is published under the projectbluefin org. @@ -138,19 +119,15 @@ def _has_custom_command_list(context) -> bool: def before_all(context): userdata = context.config.userdata - # When IMAGE env var is set (GHA runner), auto-detect image family so - # @bluefin scenarios can be skipped gracefully on non-Bluefin images. - # Prefer BASE_IMAGE: on composed runs IMAGE is the derived - # ghcr.io//testsuite-e2e:run- ref, which carries neither the - # family nor the org of the image under test (#907). - image_ref = ( - os.environ.get("BASE_IMAGE") - or os.environ.get("IMAGE") - or userdata.get("image", "") - ) + # Auto-detect the image family so @bluefin scenarios skip on non-Bluefin + # images. image_under_test() prefers BASE_IMAGE: on composed runs IMAGE is + # the derived ghcr.io//testsuite-e2e:run- ref, which carries + # neither the family nor the org of the image under test (#907). + image_ref = image_under_test(userdata) context.is_bluefin_image = _is_bluefin_image(image_ref) if image_ref else True context.is_dakota_image = _is_dakota_image(image_ref) if image_ref else False context.is_projectbluefin_image = _is_projectbluefin_image(image_ref) + context.is_utah_image = _is_utah_image(image_ref) if image_ref else False context.vm_ip = _first_value( userdata.get("vm_ip", ""), userdata.get("host", ""), @@ -236,6 +213,10 @@ def before_scenario(context, scenario): f"(IMAGE={os.environ.get('IMAGE', 'unknown')})" ) return + # Temporary: drop @skip_utah once #1076 replaces the scenarios it gates. + if "skip_utah" in scenario_tags and getattr(context, "is_utah_image", False): + scenario.skip("Skipping @skip_utah scenario: Utah's common pin no longer ships it") + return if "requires_brew" in scenario_tags and not _has_brew(context): scenario.skip("Homebrew not present on this image") return diff --git a/tests/shared/image_family.py b/tests/shared/image_family.py new file mode 100644 index 000000000..979a588fa --- /dev/null +++ b/tests/shared/image_family.py @@ -0,0 +1,37 @@ +"""Image-family selection for variant-tagged behave scenarios.""" +import os + +_BLUEFIN_FAMILY_NAMES = ("bluefin", "bazzite", "utah", "common") + + +def image_name(image: str) -> str: + """Return the image-name component, excluding registry, org, tag, and digest.""" + return image.lower().split("/")[-1].split(":")[0].split("@")[0] + + +def image_under_test(userdata=None) -> str: + """Return the input image, not a composed run-scoped testsuite image. + + Precedence is BASE_IMAGE, then IMAGE, then ``-D image=`` (userdata), as + the common suite has always resolved it. This differs from the software + suite, which checks ``-D image=`` before IMAGE. + """ + return ( + os.environ.get("BASE_IMAGE") + or os.environ.get("IMAGE") + or (userdata or {}).get("image", "") + ) + + +def is_bluefin_image(image: str) -> bool: + """Return True for images that ship Bluefin's common desktop layer.""" + name = image_name(image) + return any(family in name for family in _BLUEFIN_FAMILY_NAMES) + + +def is_dakota_image(image: str) -> bool: + return "dakota" in image_name(image) + + +def is_utah_image(image: str) -> bool: + return "utah" in image_name(image) diff --git a/tests/smoke/features/environment.py b/tests/smoke/features/environment.py index 39080a867..57cc70a78 100644 --- a/tests/smoke/features/environment.py +++ b/tests/smoke/features/environment.py @@ -17,6 +17,7 @@ import subprocess as _subprocess from tests.shared.failed_setup import record_failed_setup, write_failed_setup_marker +from tests.shared.image_family import image_under_test, is_bluefin_image, is_dakota_image from tests.shared.results_dir import resolve_results_dir try: @@ -313,6 +314,25 @@ def take_fastfetch_screenshot(): } +def _detect_image_family(context) -> None: + """Set is_bluefin_image / is_dakota_image for variant-tagged scenarios. + + Composed runs replace IMAGE with ghcr.io//testsuite-e2e:run-, + which carries no family, so image_under_test() checks BASE_IMAGE first, + then IMAGE, then ``-D image=``. The GUI runner container always sets + IMAGE, so ``-D image=`` only applies to local runs without it. Only the + image name is matched, so the "projectbluefin" org cannot false-positive. + """ + userdata = getattr(getattr(context, "config", None), "userdata", None) + image_ref = image_under_test(userdata if isinstance(userdata, dict) else None) + if image_ref: + context.is_bluefin_image = is_bluefin_image(image_ref) + context.is_dakota_image = is_dakota_image(image_ref) + else: + context.is_bluefin_image = True # default to Bluefin when no image is known + context.is_dakota_image = False + + def before_all(context) -> None: import time import subprocess @@ -506,18 +526,7 @@ def _nautilus_aliased_application(name, *args, **kwargs): else: print("WARNING: clock/system toggles not found after 15s — proceeding anyway", flush=True) - # Detect image family so variant-tagged scenarios can be skipped on the - # wrong image. Match only the image name component (last path segment before - # ':' or '@') — the org "projectbluefin" must not be treated as an image name. - image_ref = os.environ.get("IMAGE", "") - if image_ref: - _lower = image_ref.lower() - _name = _lower.split("/")[-1].split(":")[0].split("@")[0] - context.is_bluefin_image = "bluefin" in _name or "bazzite" in _name - context.is_dakota_image = "dakota" in _name - else: - context.is_bluefin_image = True # default to Bluefin when IMAGE is unset - context.is_dakota_image = False + _detect_image_family(context) try: context.optional_scenario_availability = { diff --git a/tests/unit/test_common_steps.py b/tests/unit/test_common_steps.py index 908c85c4d..659af7cf0 100644 --- a/tests/unit/test_common_steps.py +++ b/tests/unit/test_common_steps.py @@ -249,6 +249,26 @@ def test_allows_dakota_only_on_dakota_image(self): assert scenario.skip_message is None +class TestCommonEnvironmentSkipUtah: + def test_skips_skip_utah_on_utah_image(self): + m = _import_common_environment() + context = _ctx(is_bluefin_image=True, is_utah_image=True, has_brew=True) + scenario = _Scenario(["bluefin", "skip_utah"]) + + m.before_scenario(context, scenario) + + assert "Skipping @skip_utah scenario" in scenario.skip_message + + def test_allows_skip_utah_on_bluefin_image(self): + m = _import_common_environment() + context = _ctx(is_bluefin_image=True, is_utah_image=False, has_brew=True) + scenario = _Scenario(["bluefin", "skip_utah"]) + + m.before_scenario(context, scenario) + + assert scenario.skip_message is None + + class TestCommonEnvironmentRequiresToggleAction: def test_skips_when_recipe_lacks_action_value(self): m = _import_common_environment(run_ssh_returncode=1) @@ -501,6 +521,20 @@ def test_composed_ref_alone_has_no_family(self, monkeypatch): assert context.is_bluefin_image is False assert context.is_dakota_image is False + @pytest.mark.parametrize( + "base_image", + [ + "ghcr.io/projectbluefin/utah@sha256:abc123", + "ghcr.io/projectbluefin/common:e2e-pr-42-abc", + ], + ) + def test_common_layer_consumers_run_bluefin_scenarios(self, monkeypatch, base_image): + """Utah and common PR e2e ship common's bluefin tree; skipping them was a false green (#1073).""" + context = self._run(monkeypatch, base_image=base_image, image=self.COMPOSED) + + assert context.is_bluefin_image is True + assert context.is_dakota_image is False + class TestCommonEnvironmentBootcUnifiedStorage: def test_restarts_service_when_result_is_not_success(self): diff --git a/tests/unit/test_image_family.py b/tests/unit/test_image_family.py new file mode 100644 index 000000000..92779fe75 --- /dev/null +++ b/tests/unit/test_image_family.py @@ -0,0 +1,119 @@ +"""Unit tests for tests/shared/image_family.py (#1073).""" +import importlib +from types import SimpleNamespace + +import pytest + +from tests.shared.image_family import ( + image_name, + image_under_test, + is_bluefin_image, + is_dakota_image, + is_utah_image, +) + + +@pytest.fixture(autouse=True) +def _clear_image_env(monkeypatch): + monkeypatch.delenv("BASE_IMAGE", raising=False) + monkeypatch.delenv("IMAGE", raising=False) + + +@pytest.mark.parametrize( + "image", + [ + "ghcr.io/projectbluefin/bluefin:testing", + "ghcr.io/projectbluefin/bluefin-lts@sha256:abc", + "ghcr.io/ublue-os/bazzite:stable", + "ghcr.io/projectbluefin/utah@sha256:abc", + "ghcr.io/projectbluefin/common:e2e-pr-12-abc", + ], +) +def test_common_layer_consumers_select_bluefin_scenarios(image): + assert is_bluefin_image(image) + assert not is_dakota_image(image) + + +@pytest.mark.parametrize( + "image", + [ + "ghcr.io/projectbluefin/dakota:testing", + "ghcr.io/common/dakota:testing", + "ghcr.io/utah/dakota@sha256:abc", + ], +) +def test_org_name_never_selects_a_family(image): + assert is_dakota_image(image) + assert not is_bluefin_image(image) + + +def test_image_name_strips_registry_org_tag_and_digest(): + assert image_name("GHCR.io/ProjectBluefin/Utah:testing@sha256:abc") == "utah" + + +def test_composed_run_uses_base_image_family(monkeypatch): + monkeypatch.setenv("BASE_IMAGE", "ghcr.io/projectbluefin/dakota:testing") + monkeypatch.setenv("IMAGE", "ghcr.io/projectbluefin/testsuite-e2e:run-1") + + image = image_under_test({"image": "ghcr.io/projectbluefin/bluefin:testing"}) + + assert image == "ghcr.io/projectbluefin/dakota:testing" + + +def test_image_then_userdata_are_fallbacks(monkeypatch): + assert image_under_test({"image": "ghcr.io/projectbluefin/utah:testing"}) == ( + "ghcr.io/projectbluefin/utah:testing" + ) + monkeypatch.setenv("IMAGE", "ghcr.io/projectbluefin/bluefin:testing") + assert image_under_test({"image": "ghcr.io/projectbluefin/utah:testing"}) == ( + "ghcr.io/projectbluefin/bluefin:testing" + ) + assert image_under_test() == "ghcr.io/projectbluefin/bluefin:testing" + + +def _smoke_detect(userdata): + from tests.unit.test_smoke_environment import _setup_stubs + + _setup_stubs() + env = importlib.import_module("tests.smoke.features.environment") + context = SimpleNamespace(config=SimpleNamespace(userdata=userdata)) + env._detect_image_family(context) + return context + + +def test_smoke_composed_run_keeps_dakota_family(monkeypatch): + monkeypatch.setenv("BASE_IMAGE", "ghcr.io/projectbluefin/dakota:testing") + monkeypatch.setenv("IMAGE", "ghcr.io/projectbluefin/testsuite-e2e:run-1") + + context = _smoke_detect({}) + + assert context.is_dakota_image is True + assert context.is_bluefin_image is False + + +def test_smoke_reads_image_from_behave_userdata(monkeypatch): + context = _smoke_detect({"image": "ghcr.io/projectbluefin/utah@sha256:abc"}) + + assert context.is_bluefin_image is True + assert context.is_dakota_image is False + + +def test_smoke_defaults_to_bluefin_when_no_image_is_known(): + context = _smoke_detect({}) + + assert context.is_bluefin_image is True + assert context.is_dakota_image is False + + +@pytest.mark.parametrize( + ("image", "expected"), + [ + ("ghcr.io/projectbluefin/utah@sha256:abc", True), + ("ghcr.io/projectbluefin/utah:testing", True), + ("ghcr.io/projectbluefin/bluefin:testing", False), + ("ghcr.io/utah/bluefin:testing", False), + ("", False), + ], +) +def test_is_utah_image_matches_image_name_only(image, expected): + assert is_utah_image(image) is expected