diff --git a/README.md b/README.md index de04e21..411ada7 100644 --- a/README.md +++ b/README.md @@ -134,6 +134,11 @@ reason = "ConditionDelay not supported in v1.2" test = "scenario/sequencing/val_split_*.py::*" reason = "Sequencing not implemented" strict = true # optional, default false — if true, an unexpected pass is a failure + +[[xfail]] +test = "scenario/trajectories/val_interpolation.py::test_timed_polyline_trajectory_interpolation" +except = ["linear_position-*"] +reason = "Only linear position variants are expected to pass" ``` Each `[[xfail]]` entry requires: @@ -145,6 +150,7 @@ For example, `scenario/foo.py::test_bar` matches collected items such as `scenar Optional: - `strict` (bool, default `false`) — when `true`, the test is marked as failed if it unexpectedly passes +- `except` (list of strings) — patterns selected by `test` that should not be marked xfail. Patterns containing `::` match full pytest node IDs; bare patterns match only parametrization IDs inside `[...]`. ### Development checks diff --git a/docs/example_test_profile/test_profile.toml b/docs/example_test_profile/test_profile.toml index 0fd751b..fdb47bc 100644 --- a/docs/example_test_profile/test_profile.toml +++ b/docs/example_test_profile/test_profile.toml @@ -7,5 +7,6 @@ test = "scenario/sequencing/val_split_*.py::*" reason = "Sequencing not implemented by Tool v0.0.1" [[xfail]] -test = "scenario/trajectories/val_interpolation.py::test_timed_polyline_trajectory_interpolation[constant_acceleration_from_initial_speed-*]" -reason = "Linear position interpolation is expected to fail for this tool" +test = "scenario/trajectories/val_interpolation.py::test_timed_polyline_trajectory_interpolation" +except = ["linear_position-*"] +reason = "Only linear position interpolation variants are expected to pass for this tool" diff --git a/osc_validation/test_profile.py b/osc_validation/test_profile.py index ff28c94..a5952bf 100644 --- a/osc_validation/test_profile.py +++ b/osc_validation/test_profile.py @@ -37,17 +37,44 @@ class XFailEntry: test: str reason: str strict: bool = False + except_patterns: list[str] = field(default_factory=list) def matches(self, node_id: str) -> bool: """Return True if *node_id* matches this entry's test pattern.""" - return ( - node_id == self.test - or _unparameterized_node_id(node_id) == self.test - or fnmatch(node_id, self.test) - or fnmatch(node_id, _escape_parameter_bracket_glob(self.test)) + return _matches_node_id_pattern(node_id, self.test) and not any( + _matches_except_pattern(node_id, pattern) for pattern in self.except_patterns ) +def _matches_node_id_pattern(node_id: str, pattern: str) -> bool: + """Return True if *node_id* matches a full pytest node ID pattern.""" + return ( + node_id == pattern + or _unparameterized_node_id(node_id) == pattern + or fnmatch(node_id, pattern) + or fnmatch(node_id, _escape_parameter_bracket_glob(pattern)) + ) + + +def _parameter_id(node_id: str) -> str | None: + """Return the trailing pytest parameter ID, if present.""" + if not node_id.endswith("]"): + return None + base_node_id, separator, parameter_id = node_id.rpartition("[") + if separator and "::" in base_node_id: + return parameter_id[:-1] + return None + + +def _matches_except_pattern(node_id: str, pattern: str) -> bool: + """Return True if *pattern* exempts *node_id* from an xfail entry.""" + if "::" in pattern: + return _matches_node_id_pattern(node_id, pattern) + + parameter_id = _parameter_id(node_id) + return parameter_id is not None and fnmatch(parameter_id, pattern) + + def _unparameterized_node_id(node_id: str) -> str: """Return the base test node ID without a trailing pytest parameter suffix.""" if not node_id.endswith("]"): @@ -106,11 +133,19 @@ def load_test_profile(path: str | Path) -> Profile: raise ValueError( f"xfail entry {i} in '{path}' is missing required field 'reason'" ) + except_patterns = entry.get("except", []) + if not isinstance(except_patterns, list) or not all( + isinstance(pattern, str) for pattern in except_patterns + ): + raise ValueError( + f"xfail entry {i} in '{path}' field 'except' must be a list of strings" + ) xfails.append( XFailEntry( test=entry["test"], reason=entry["reason"], strict=entry.get("strict", False), + except_patterns=except_patterns, ) ) diff --git a/tests/test_test_profile.py b/tests/test_test_profile.py index bee9d95..edc58c8 100644 --- a/tests/test_test_profile.py +++ b/tests/test_test_profile.py @@ -67,6 +67,53 @@ def test_xfail_entry_strict_default(): assert entry.strict is False +def test_xfail_entry_bare_except_exempts_matching_parameter_id(): + entry = XFailEntry( + test="validation/scenario/foo.py::test_bar", + reason="r", + except_patterns=["data-set-1"], + ) + assert not entry.matches("validation/scenario/foo.py::test_bar[data-set-1]") + + +def test_xfail_entry_bare_except_supports_glob_parameter_id(): + entry = XFailEntry( + test="validation/scenario/foo.py::test_bar", + reason="r", + except_patterns=["constant_velocity_*"], + ) + assert not entry.matches( + "validation/scenario/foo.py::test_bar[constant_velocity_forward]" + ) + + +def test_xfail_entry_full_node_id_except_uses_node_id_matching(): + entry = XFailEntry( + test="validation/scenario/foo.py::test_bar", + reason="r", + except_patterns=["validation/scenario/foo.py::test_bar[case-*]"], + ) + assert not entry.matches("validation/scenario/foo.py::test_bar[case-1]") + + +def test_xfail_entry_non_excepted_parametrization_still_matches(): + entry = XFailEntry( + test="validation/scenario/foo.py::test_bar", + reason="r", + except_patterns=["passing-case"], + ) + assert entry.matches("validation/scenario/foo.py::test_bar[failing-case]") + + +def test_xfail_entry_unparameterized_item_not_exempted_by_bare_except(): + entry = XFailEntry( + test="validation/scenario/foo.py::test_bar", + reason="r", + except_patterns=["test_bar"], + ) + assert entry.matches("validation/scenario/foo.py::test_bar") + + # --------------------------------------------------------------------------- # TestProfile.xfail_for # --------------------------------------------------------------------------- @@ -131,6 +178,20 @@ def test_load_valid_profile(tmp_path): assert profile.xfails[1].strict is True +def test_load_valid_profile_with_except(tmp_path): + path = _write_toml( + tmp_path, + """\ + [[xfail]] + test = "validation/scenario/foo.py::test_bar" + except = ["passing-*"] + reason = "Known bug" + """, + ) + profile = load_test_profile(path) + assert profile.xfails[0].except_patterns == ["passing-*"] + + def test_load_empty_profile(tmp_path): path = _write_toml(tmp_path, "") profile = load_test_profile(path) @@ -173,6 +234,34 @@ def test_load_missing_reason_field(tmp_path): load_test_profile(path) +def test_load_rejects_except_when_not_list(tmp_path): + path = _write_toml( + tmp_path, + """\ + [[xfail]] + test = "validation/scenario/foo.py::test_bar" + except = "passing-*" + reason = "Known bug" + """, + ) + with pytest.raises(ValueError, match="field 'except' must be a list of strings"): + load_test_profile(path) + + +def test_load_rejects_except_when_list_contains_non_string(tmp_path): + path = _write_toml( + tmp_path, + """\ + [[xfail]] + test = "validation/scenario/foo.py::test_bar" + except = ["passing-*", 1] + reason = "Known bug" + """, + ) + with pytest.raises(ValueError, match="field 'except' must be a list of strings"): + load_test_profile(path) + + def test_load_accepts_string_path(tmp_path): path = _write_toml( tmp_path,