Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -10,24 +10,24 @@ metadata:

## Cross-image tag skipping: @bluefin on non-bluefin images

The common suite skips scenarios tagged `@bluefin` when `IMAGE` env var refers to a
non-bluefin image (e.g. dakota). This is implemented via `_is_bluefin_image()` in
`tests/common/features/environment.py`.
The common and smoke suites skip scenarios tagged `@bluefin` unless the image
under test ships Bluefin's common desktop layer. Both environments use
`tests/shared/image_family.py`.

**Pitfall**: match the image *name*, not the full URL. The org name `projectbluefin`
contains `"bluefin"`, so naively checking `"bluefin" in image_url.lower()` returns
`True` for `ghcr.io/projectbluefin/dakota:testing`.
Read `BASE_IMAGE` before `IMAGE`: composed runs replace `IMAGE` with
`ghcr.io/<owner>/testsuite-e2e:run-<id>`, which carries no family. Match only
the image-name component, not the org: `projectbluefin` contains `bluefin`, so a
URL-wide check would select Bluefin scenarios on Dakota.

Correct pattern:
Bluefin-family names are `bluefin`, `bazzite`, `utah`, and `common`. Common's PR
e2e image is named `common`, but it overlays common's `shared` and `bluefin`
trees onto its base image. Utah is Bluefin built on Fedora Hummingbird. Dakota
remains separate and selects `@dakota_only` scenarios. In the common suite,
`@skip_utah` excludes a `@bluefin` scenario on Utah only: it marks content that
Bluefin's common pin still ships but Utah's newer pin does not (the Searchlight
dconf scenario, until #1076 replaces it).

```python
def _is_bluefin_image(image: str) -> bool:
lower = image.lower()
name = lower.split("/")[-1].split(":")[0].split("@")[0]
return "bluefin" in name or "bazzite" in lower
```

This extracts the image name component (`bluefin`, `dakota`, etc.) before checking.

The smoke suite environment replicates this same pattern so `@bluefin` tags are
respected in AT-SPI tests as well.
The smoke suite runs inside the GUI runner container, so it only sees
`BASE_IMAGE` if `e2e.yml` passes it into that container (`-e BASE_IMAGE` and
`-D image=`). Without them, composed smoke runs still report the run-scoped ref
and skip every `@bluefin` and `@dakota_only` scenario.
5 changes: 3 additions & 2 deletions docs/skills/test-authoring/suite-map/SKILL.md
Original file line number Diff line number Diff line change
Expand Up @@ -139,8 +139,9 @@ Set `chunked_enabled: true` once `ghcr.io/projectbluefin/bluefin:latest` ships z
| Tag | Meaning |
|---|---|
| `@smoke_suite` | Runs as part of the standard smoke suite |
| `@bluefin` | Smoke scenario runs only when the image name contains `bluefin`; smoke `environment.py` skips it elsewhere |
| `@dakota_only` | Scenario runs only when the image name contains `dakota`; the `smoke` and `common` `environment.py` files skip it elsewhere. Only the image **name** is matched, so the `projectbluefin` org name cannot false-positive |
| `@bluefin` | Scenario runs on Bluefin-family image names (`bluefin`, `bazzite`, `utah`, `common`); smoke and common read `BASE_IMAGE` before `IMAGE`, so composed runs keep their family |
| `@dakota_only` | Scenario runs only when the image name contains `dakota`; smoke and common read `BASE_IMAGE` before `IMAGE`. Only the image **name** is matched, so the `projectbluefin` org name cannot false-positive |
| `@skip_utah` | Common scenario skips when the image name contains `utah`: it asserts content Bluefin's common pin still ships but Utah's newer pin does not. Remove it when the scenario is replaced (#1076) |
| `@dx_only` / `@developer_suite` | DX variant only |
| `@nvidia_only` | NVIDIA variant only |
| `@flatcar_suite` | Flatcar OS only |
Expand Down
2 changes: 2 additions & 0 deletions tests/common/features/common_dconf.feature
Original file line number Diff line number Diff line change
Expand Up @@ -84,6 +84,8 @@ Feature: Bluefin common dconf and GSettings defaults
* SSH command return code is "0"
* SSH command output stripped "is" "'xdg-terminal-exec'"

# Bluefin's pinned common still ships Searchlight; Utah's newer pin does not (#1076).
@skip_utah
Scenario: Searchlight extension is configured
* Run SSH command: "dconf read /org/gnome/shell/extensions/search-light/shortcut-search"
* SSH command return code is "0"
Expand Down
51 changes: 16 additions & 35 deletions tests/common/features/environment.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,12 @@
import os
import shlex

from tests.shared.image_family import (
image_under_test,
is_bluefin_image as _is_bluefin_image,
is_dakota_image as _is_dakota_image,
is_utah_image as _is_utah_image,
)
from tests.shared.ssh_config import _first_value
from tests.shared.ssh_steps import * # noqa: F401,F403
from tests.shared.ssh_steps import run_ssh
Expand All @@ -21,31 +27,6 @@ def record_end(context, scenario):
return None


def _is_bluefin_image(image: str) -> bool:
"""Return True if the image reference looks like a Bluefin image.

Matches the image name component only (e.g. "bluefin" in
"ghcr.io/projectbluefin/bluefin:testing") to avoid false-positives where
the org name "projectbluefin" would match a dakota image URL.
"""
lower = image.lower()
# Extract image name: last path segment before any tag/digest
name = lower.split("/")[-1].split(":")[0].split("@")[0]
return "bluefin" in name or "bazzite" in lower


def _is_dakota_image(image: str) -> bool:
"""Return True if the image reference looks like a Dakota image.

Matches the image name component only (e.g. "dakota" in
"ghcr.io/projectbluefin/dakota:testing") to avoid false-positives where
the org name "projectbluefin" would match a Bluefin image URL.
"""
lower = image.lower()
name = lower.split("/")[-1].split(":")[0].split("@")[0]
return "dakota" in name


def _is_projectbluefin_image(image: str) -> bool:
"""Return True if the image reference is published under the projectbluefin org.

Expand Down Expand Up @@ -138,19 +119,15 @@ def _has_custom_command_list(context) -> bool:

def before_all(context):
userdata = context.config.userdata
# When IMAGE env var is set (GHA runner), auto-detect image family so
# @bluefin scenarios can be skipped gracefully on non-Bluefin images.
# Prefer BASE_IMAGE: on composed runs IMAGE is the derived
# ghcr.io/<owner>/testsuite-e2e:run-<id> ref, which carries neither the
# family nor the org of the image under test (#907).
image_ref = (
os.environ.get("BASE_IMAGE")
or os.environ.get("IMAGE")
or userdata.get("image", "")
)
# Auto-detect the image family so @bluefin scenarios skip on non-Bluefin
# images. image_under_test() prefers BASE_IMAGE: on composed runs IMAGE is
# the derived ghcr.io/<owner>/testsuite-e2e:run-<id> ref, which carries
# neither the family nor the org of the image under test (#907).
image_ref = image_under_test(userdata)
context.is_bluefin_image = _is_bluefin_image(image_ref) if image_ref else True
context.is_dakota_image = _is_dakota_image(image_ref) if image_ref else False
context.is_projectbluefin_image = _is_projectbluefin_image(image_ref)
context.is_utah_image = _is_utah_image(image_ref) if image_ref else False
context.vm_ip = _first_value(
userdata.get("vm_ip", ""),
userdata.get("host", ""),
Expand Down Expand Up @@ -236,6 +213,10 @@ def before_scenario(context, scenario):
f"(IMAGE={os.environ.get('IMAGE', 'unknown')})"
)
return
# Temporary: drop @skip_utah once #1076 replaces the scenarios it gates.
if "skip_utah" in scenario_tags and getattr(context, "is_utah_image", False):
scenario.skip("Skipping @skip_utah scenario: Utah's common pin no longer ships it")
return
if "requires_brew" in scenario_tags and not _has_brew(context):
scenario.skip("Homebrew not present on this image")
return
Expand Down
37 changes: 37 additions & 0 deletions tests/shared/image_family.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,37 @@
"""Image-family selection for variant-tagged behave scenarios."""
import os

_BLUEFIN_FAMILY_NAMES = ("bluefin", "bazzite", "utah", "common")


def image_name(image: str) -> str:
"""Return the image-name component, excluding registry, org, tag, and digest."""
return image.lower().split("/")[-1].split(":")[0].split("@")[0]


def image_under_test(userdata=None) -> str:
"""Return the input image, not a composed run-scoped testsuite image.

Precedence is BASE_IMAGE, then IMAGE, then ``-D image=`` (userdata), as
the common suite has always resolved it. This differs from the software
suite, which checks ``-D image=`` before IMAGE.
"""
return (
os.environ.get("BASE_IMAGE")
or os.environ.get("IMAGE")
or (userdata or {}).get("image", "")
)


def is_bluefin_image(image: str) -> bool:
"""Return True for images that ship Bluefin's common desktop layer."""
name = image_name(image)
return any(family in name for family in _BLUEFIN_FAMILY_NAMES)


def is_dakota_image(image: str) -> bool:
return "dakota" in image_name(image)


def is_utah_image(image: str) -> bool:
return "utah" in image_name(image)
33 changes: 21 additions & 12 deletions tests/smoke/features/environment.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@
import subprocess as _subprocess

from tests.shared.failed_setup import record_failed_setup, write_failed_setup_marker
from tests.shared.image_family import image_under_test, is_bluefin_image, is_dakota_image
from tests.shared.results_dir import resolve_results_dir

try:
Expand Down Expand Up @@ -313,6 +314,25 @@ def take_fastfetch_screenshot():
}


def _detect_image_family(context) -> None:
"""Set is_bluefin_image / is_dakota_image for variant-tagged scenarios.

Composed runs replace IMAGE with ghcr.io/<owner>/testsuite-e2e:run-<id>,
which carries no family, so image_under_test() checks BASE_IMAGE first,
then IMAGE, then ``-D image=``. The GUI runner container always sets
IMAGE, so ``-D image=`` only applies to local runs without it. Only the
image name is matched, so the "projectbluefin" org cannot false-positive.
"""
userdata = getattr(getattr(context, "config", None), "userdata", None)
image_ref = image_under_test(userdata if isinstance(userdata, dict) else None)
if image_ref:
context.is_bluefin_image = is_bluefin_image(image_ref)
context.is_dakota_image = is_dakota_image(image_ref)
else:
context.is_bluefin_image = True # default to Bluefin when no image is known
context.is_dakota_image = False


def before_all(context) -> None:
import time
import subprocess
Expand Down Expand Up @@ -506,18 +526,7 @@ def _nautilus_aliased_application(name, *args, **kwargs):
else:
print("WARNING: clock/system toggles not found after 15s — proceeding anyway", flush=True)

# Detect image family so variant-tagged scenarios can be skipped on the
# wrong image. Match only the image name component (last path segment before
# ':' or '@') — the org "projectbluefin" must not be treated as an image name.
image_ref = os.environ.get("IMAGE", "")
if image_ref:
_lower = image_ref.lower()
_name = _lower.split("/")[-1].split(":")[0].split("@")[0]
context.is_bluefin_image = "bluefin" in _name or "bazzite" in _name
context.is_dakota_image = "dakota" in _name
else:
context.is_bluefin_image = True # default to Bluefin when IMAGE is unset
context.is_dakota_image = False
_detect_image_family(context)

try:
context.optional_scenario_availability = {
Expand Down
34 changes: 34 additions & 0 deletions tests/unit/test_common_steps.py
Original file line number Diff line number Diff line change
Expand Up @@ -249,6 +249,26 @@ def test_allows_dakota_only_on_dakota_image(self):
assert scenario.skip_message is None


class TestCommonEnvironmentSkipUtah:
def test_skips_skip_utah_on_utah_image(self):
m = _import_common_environment()
context = _ctx(is_bluefin_image=True, is_utah_image=True, has_brew=True)
scenario = _Scenario(["bluefin", "skip_utah"])

m.before_scenario(context, scenario)

assert "Skipping @skip_utah scenario" in scenario.skip_message

def test_allows_skip_utah_on_bluefin_image(self):
m = _import_common_environment()
context = _ctx(is_bluefin_image=True, is_utah_image=False, has_brew=True)
scenario = _Scenario(["bluefin", "skip_utah"])

m.before_scenario(context, scenario)

assert scenario.skip_message is None


class TestCommonEnvironmentRequiresToggleAction:
def test_skips_when_recipe_lacks_action_value(self):
m = _import_common_environment(run_ssh_returncode=1)
Expand Down Expand Up @@ -501,6 +521,20 @@ def test_composed_ref_alone_has_no_family(self, monkeypatch):
assert context.is_bluefin_image is False
assert context.is_dakota_image is False

@pytest.mark.parametrize(
"base_image",
[
"ghcr.io/projectbluefin/utah@sha256:abc123",
"ghcr.io/projectbluefin/common:e2e-pr-42-abc",
],
)
def test_common_layer_consumers_run_bluefin_scenarios(self, monkeypatch, base_image):
"""Utah and common PR e2e ship common's bluefin tree; skipping them was a false green (#1073)."""
context = self._run(monkeypatch, base_image=base_image, image=self.COMPOSED)

assert context.is_bluefin_image is True
assert context.is_dakota_image is False


class TestCommonEnvironmentBootcUnifiedStorage:
def test_restarts_service_when_result_is_not_success(self):
Expand Down
Loading
Loading