diff --git a/osc_validation/validation/conftest.py b/osc_validation/validation/conftest.py index 7ae0065..60e917f 100644 --- a/osc_validation/validation/conftest.py +++ b/osc_validation/validation/conftest.py @@ -1,3 +1,6 @@ +import html +import inspect + import pytest pytest_plugins = ["osc_validation.pytest_plugin"] @@ -6,3 +9,71 @@ @pytest.hookimpl(optionalhook=True) def pytest_html_report_title(report): report.title = "OSC Validation Report" + + +def _marker_value(item: pytest.Item, marker_name: str) -> str: + marker = item.get_closest_marker(marker_name) + if marker is None or not marker.args: + return "" + return str(marker.args[0]) + + +def _test_description(item: pytest.Item) -> str: + test_object = getattr(item, "obj", None) + if test_object is None: + return "" + return inspect.getdoc(test_object) or "" + + +@pytest.hookimpl(hookwrapper=True) +def pytest_runtest_makereport(item: pytest.Item, call): + outcome = yield + report = outcome.get_result() + + if report.when != "call": + return + + report.validation_category = _marker_value(item, "validation_category") + report.validation_feature = _marker_value(item, "validation_feature") + + description = _test_description(item) + + pytest_html = item.config.pluginmanager.getplugin("html") + if pytest_html is None: + return + + extras = getattr(report, "extras", []) + tmp_path = item.funcargs.get("tmp_path") + if tmp_path is not None: + resolved_tmp_path = tmp_path.resolve() + escaped_tmp_path = html.escape(str(resolved_tmp_path)) + escaped_tmp_path_uri = html.escape(resolved_tmp_path.as_uri(), quote=True) + extras.append( + pytest_html.extras.html( + "
Pytest temp directory
" + f'{escaped_tmp_path}
' + ) + ) + + if description: + escaped_description = html.escape(description).replace("\n", "
") + extras.append( + pytest_html.extras.html( + f"
Validation description
{escaped_description}
" + ) + ) + report.extras = extras + + +@pytest.hookimpl(optionalhook=True) +def pytest_html_results_table_header(cells): + cells.insert(2, "Validation Category") + cells.insert(3, "OpenSCENARIO Feature") + + +@pytest.hookimpl(optionalhook=True) +def pytest_html_results_table_row(report, cells): + category = html.escape(getattr(report, "validation_category", "")) + feature = html.escape(getattr(report, "validation_feature", "")) + cells.insert(2, f"{category}") + cells.insert(3, f"{feature}") diff --git a/osc_validation/validation/pytest.ini b/osc_validation/validation/pytest.ini index c40ad2f..1d45d2b 100644 --- a/osc_validation/validation/pytest.ini +++ b/osc_validation/validation/pytest.ini @@ -3,4 +3,5 @@ testpaths = scenario python_files = val_*.py filterwarnings = ignore::DeprecationWarning markers = - trajectory: trajectory-specific tests + validation_feature(name): OpenSCENARIO feature covered by the validation case + validation_category(name): Validation case category diff --git a/osc_validation/validation/scenario/init_actions/val_init_actions.py b/osc_validation/validation/scenario/init_actions/val_init_actions.py index 676cb14..d1fa0e7 100644 --- a/osc_validation/validation/scenario/init_actions/val_init_actions.py +++ b/osc_validation/validation/scenario/init_actions/val_init_actions.py @@ -59,11 +59,21 @@ def _compute_object_state_metric( ) +@pytest.mark.validation_category("init_action") +@pytest.mark.validation_feature("Init TeleportAction") def test_init_teleport_action_places_actor_at_expected_pose( odr_file: Path, generate_tool_trace: Callable, tmp_path: Path, ): + """ + OpenSCENARIO feature: Init actions using TeleportAction. + + Creates an oracle scenario where the actor is placed at a known position and + orientation during initialization. The resulting OSI trace must match the + expected position and full orientation. + """ + rate = 0.05 case_result = build_init_teleport_action_case( InitActionCaseSpec( @@ -110,11 +120,21 @@ def test_init_teleport_action_places_actor_at_expected_pose( assert metric_result.max_orientation_error < 0.01 +@pytest.mark.validation_category("init_action") +@pytest.mark.validation_feature("Init AddEntityAction") def test_init_add_entity_action_places_actor_at_expected_pose( odr_file: Path, generate_tool_trace: Callable, tmp_path: Path, ): + """ + OpenSCENARIO feature: Init actions using AddEntityAction. + + Creates an oracle scenario where an entity is added during initialization at a + known pose. The validation checks that the generated object state matches the + expected position and orientation. + """ + rate = 0.05 case_result = build_init_add_entity_action_case( InitActionCaseSpec( @@ -161,11 +181,21 @@ def test_init_add_entity_action_places_actor_at_expected_pose( assert metric_result.max_orientation_error < 0.01 +@pytest.mark.validation_category("init_action") +@pytest.mark.validation_feature("Init SpeedAction") def test_init_speed_action_sets_expected_motion( odr_file: Path, generate_tool_trace: Callable, tmp_path: Path, ): + """ + OpenSCENARIO feature: Init actions using SpeedAction. + + Initializes an actor with a configured speed and compares the generated motion + with an oracle reference trace. The validation focuses on the initialized + planar speed rather than complete trajectory replay. + """ + rate = 0.05 speed_mps = 5.0 case_result = build_init_speed_action_case( diff --git a/osc_validation/validation/scenario/object_state/val_vehicle_dimensions.py b/osc_validation/validation/scenario/object_state/val_vehicle_dimensions.py index 8952963..def47ec 100644 --- a/osc_validation/validation/scenario/object_state/val_vehicle_dimensions.py +++ b/osc_validation/validation/scenario/object_state/val_vehicle_dimensions.py @@ -26,7 +26,8 @@ def odr_file(request): return request.getfixturevalue("osi_trace").with_suffix(".xodr") -@pytest.mark.object_state +@pytest.mark.validation_category("object_state") +@pytest.mark.validation_feature("Vehicle dimensions") @pytest.mark.parametrize("moving_object_id", [1, 2]) def test_vehicle_dimensions_match_reference( osi_trace: Path, @@ -36,6 +37,14 @@ def test_vehicle_dimensions_match_reference( moving_object_id: int, tolerance: float = 0.1, ): + """ + OpenSCENARIO feature: vehicle/object geometry emitted from scenario entities. + + Converts a reference OSI trace to OpenSCENARIO and checks that replayed vehicle + dimensions match the source objects. The validation compares length, width, and + height for the tested moving objects. + """ + reference_trace_channel_spec = ChannelSpecification( osi_trace, message_type=MessageType.SENSOR_VIEW ) diff --git a/osc_validation/validation/scenario/sequencing/val_split_trajectory.py b/osc_validation/validation/scenario/sequencing/val_split_trajectory.py index 445576a..883bb0d 100644 --- a/osc_validation/validation/scenario/sequencing/val_split_trajectory.py +++ b/osc_validation/validation/scenario/sequencing/val_split_trajectory.py @@ -34,7 +34,8 @@ def odr_file(request): return request.getfixturevalue("osi_trace").with_suffix(".xodr") -@pytest.mark.trajectory +@pytest.mark.validation_category("sequencing") +@pytest.mark.validation_feature("Trajectory sequencing") @pytest.mark.parametrize( "sequencing_level", ["event", "maneuver", "maneuver_group", "act", "story"], @@ -52,6 +53,8 @@ def test_split_ego_trajectory_preserves_reference_motion( tolerance: float = 0.1, ): """ + OpenSCENARIO feature: equivalent trajectory fragments across sequencing levels. + Validates that splitting one actor trajectory into sequential OpenSCENARIO elements preserves the same replayed trajectory. """ diff --git a/osc_validation/validation/scenario/trajectories/val_follow_trajectory_future_time_reference.py b/osc_validation/validation/scenario/trajectories/val_follow_trajectory_future_time_reference.py index efcbdd8..11ff663 100644 --- a/osc_validation/validation/scenario/trajectories/val_follow_trajectory_future_time_reference.py +++ b/osc_validation/validation/scenario/trajectories/val_follow_trajectory_future_time_reference.py @@ -55,13 +55,16 @@ def _off_trajectory_init_pose() -> TrajectoryInterpolationVertex: ) -@pytest.mark.trajectory +@pytest.mark.validation_category("trajectory") +@pytest.mark.validation_feature("FollowTrajectoryAction TimeReference") def test_follow_trajectory_future_time_reference_continues_init_speed_until_first_vertex( odr_file: Path, generate_tool_trace: Callable, tmp_path: Path, ): """ + OpenSCENARIO feature: FollowTrajectoryAction, position mode, absolute future TimeReference. + Validates that a FollowTrajectoryAction with position mode and an absolute TimeReference at the beginning of the trajectory continues at the initial speed until the first trajectory vertex time is reached, even if the initial diff --git a/osc_validation/validation/scenario/trajectories/val_follow_trajectory_teleport.py b/osc_validation/validation/scenario/trajectories/val_follow_trajectory_teleport.py index 413c566..4b93951 100644 --- a/osc_validation/validation/scenario/trajectories/val_follow_trajectory_teleport.py +++ b/osc_validation/validation/scenario/trajectories/val_follow_trajectory_teleport.py @@ -60,7 +60,8 @@ def _off_trajectory_init_pose() -> TrajectoryInterpolationVertex: ) -@pytest.mark.trajectory +@pytest.mark.validation_category("trajectory") +@pytest.mark.validation_feature("FollowTrajectoryAction TimeReference") @pytest.mark.parametrize( ("case_name", "action_start_time_s"), [ @@ -76,6 +77,8 @@ def test_follow_trajectory_position_mode_teleports_to_current_trajectory_time( action_start_time_s: float, ): """ + OpenSCENARIO feature: FollowTrajectoryAction, position mode, absolute TimeReference. + Validates that a FollowTrajectoryAction with Position mode and an absolute TimeReference at the beginning of the trajectory teleports the actor to the correct trajectory position when the action starts later, even if the diff --git a/osc_validation/validation/scenario/trajectories/val_interpolation.py b/osc_validation/validation/scenario/trajectories/val_interpolation.py index 176e0b0..db2cced 100644 --- a/osc_validation/validation/scenario/trajectories/val_interpolation.py +++ b/osc_validation/validation/scenario/trajectories/val_interpolation.py @@ -54,7 +54,8 @@ def _actor() -> TrajectoryInterpolationActor: ) -@pytest.mark.trajectory +@pytest.mark.validation_category("trajectory") +@pytest.mark.validation_feature("FollowTrajectoryAction interpolation") @pytest.mark.parametrize( ("interpolation_mode", "initial_speed_mps"), [ @@ -70,6 +71,8 @@ def test_timed_polyline_trajectory_interpolation( initial_speed_mps: float, ): """ + OpenSCENARIO feature: FollowTrajectoryAction with timed vertices and interpolation. + Validates that a FollowTrajectoryAction with position mode correctly follows the trajectory and uses the configured interpolation mode. diff --git a/osc_validation/validation/scenario/trajectories/val_simple_trajectories.py b/osc_validation/validation/scenario/trajectories/val_simple_trajectories.py index 66e704c..0d99953 100644 --- a/osc_validation/validation/scenario/trajectories/val_simple_trajectories.py +++ b/osc_validation/validation/scenario/trajectories/val_simple_trajectories.py @@ -28,7 +28,8 @@ def odr_file(request): return request.getfixturevalue("osi_trace").with_suffix(".xodr") -@pytest.mark.trajectory +@pytest.mark.validation_category("trajectory") +@pytest.mark.validation_feature("FollowTrajectoryAction") @pytest.mark.parametrize("moving_object_id", [1, 2]) def test_trajectory_and_osi_compliance( osi_trace: Path, @@ -40,21 +41,13 @@ def test_trajectory_and_osi_compliance( tolerance=1e-1, ): """ + OpenSCENARIO feature: generated FollowTrajectoryAction replay from an OSI trace. + Validates that a tool-generated trajectory closely matches the original OSI trace for a given moving object, using similarity metrics within a specified tolerance. Also runs qc_osi_trace on the tool-generated trace and checks if it complies with the OSI 3.7.0 ruleset. - - Args: - osi_trace (Path): Path to the original OSI trace file (pytest module fixture). - odr_file (Path): Path to the OpenDRIVE (.odr) file (pytest module fixture). - generate_tool_trace (Callable): Function to generate an OSI trace from an OpenSCENARIO file (pytest session fixture). - tmp_path (Path): Temporary directory for intermediate files (built-in pytest fixture). - moving_object_id (int): ID of the moving object (in the reference trace) selected for trajectory comparison (pytest parameter). - tolerance (float, optional): Maximum allowed value for similarity metrics. Defaults to 1e-6. - Raises: - AssertionError: If any similarity metric exceeds the specified tolerance. """ logger = logging.getLogger() diff --git a/osc_validation/validation/scenario/triggers/val_condition_delay.py b/osc_validation/validation/scenario/triggers/val_condition_delay.py index 1c77d28..4976b55 100644 --- a/osc_validation/validation/scenario/triggers/val_condition_delay.py +++ b/osc_validation/validation/scenario/triggers/val_condition_delay.py @@ -25,13 +25,22 @@ def odr_file(request): return request.getfixturevalue("osi_trace").with_suffix(".xodr") -@pytest.mark.trajectory +@pytest.mark.validation_category("trigger") +@pytest.mark.validation_feature("Condition delay") def test_distance_longitudinal_start_trigger_respects_condition_delay( osi_trace: Path, odr_file: Path, generate_tool_trace: Callable, tmp_path: Path, ): + """ + OpenSCENARIO feature: trigger Condition delay on longitudinal distance conditions. + + Reuses the longitudinal distance trigger case with a one-second conditionDelay. + The target actor should activate only after the distance condition has held + through that delay. + """ + _run_distance_longitudinal_start_trigger_case( osi_trace=osi_trace, odr_file=odr_file, @@ -41,13 +50,22 @@ def test_distance_longitudinal_start_trigger_respects_condition_delay( ) -@pytest.mark.trajectory +@pytest.mark.validation_category("trigger") +@pytest.mark.validation_feature("Condition delay") def test_speed_start_trigger_respects_condition_delay( osi_trace: Path, odr_file: Path, generate_tool_trace: Callable, tmp_path: Path, ): + """ + OpenSCENARIO feature: trigger Condition delay on speed conditions. + + Reuses the speed start trigger setup with a one-second conditionDelay. The + target actor's trajectory should begin after the delayed activation time, not + immediately at the threshold crossing. + """ + _run_speed_start_trigger_case( osi_trace=osi_trace, odr_file=odr_file, diff --git a/osc_validation/validation/scenario/triggers/val_distance_euclidian_start_trigger.py b/osc_validation/validation/scenario/triggers/val_distance_euclidian_start_trigger.py index 96bf482..25c8e59 100644 --- a/osc_validation/validation/scenario/triggers/val_distance_euclidian_start_trigger.py +++ b/osc_validation/validation/scenario/triggers/val_distance_euclidian_start_trigger.py @@ -53,7 +53,8 @@ def odr_file(request): return request.getfixturevalue("osi_trace").with_suffix(".xodr") -@pytest.mark.trajectory +@pytest.mark.validation_category("trigger") +@pytest.mark.validation_feature("DistanceCondition euclidianDistance") @pytest.mark.parametrize("moving_object_id", [2]) @pytest.mark.parametrize("trigger_object_id", [1]) @pytest.mark.parametrize("trigger_distance_m", [10.0]) @@ -73,6 +74,14 @@ def test_distance_euclidian_start_trigger_activates_target_actor( rate: float, tolerance: float, ): + """ + OpenSCENARIO feature: DistanceCondition with relativeDistanceType euclidianDistance. + + Activates a target actor based on Euclidean distance from the trigger object to + a target position. The validation evaluates the trajectory after the trigger + edge because the exact first activation frame is not fully constrained. + """ + logger = logging.getLogger() logger.setLevel(logging.INFO) diff --git a/osc_validation/validation/scenario/triggers/val_distance_longitudinal_start_trigger.py b/osc_validation/validation/scenario/triggers/val_distance_longitudinal_start_trigger.py index 5c1351b..809d168 100644 --- a/osc_validation/validation/scenario/triggers/val_distance_longitudinal_start_trigger.py +++ b/osc_validation/validation/scenario/triggers/val_distance_longitudinal_start_trigger.py @@ -219,7 +219,8 @@ def _run_distance_longitudinal_start_trigger_case( assert mae < tolerance -@pytest.mark.trajectory +@pytest.mark.validation_category("trigger") +@pytest.mark.validation_feature("DistanceCondition longitudinal") def test_distance_longitudinal_start_trigger_activates_target_actor( osi_trace: Path, odr_file: Path, @@ -227,6 +228,14 @@ def test_distance_longitudinal_start_trigger_activates_target_actor( generate_tool_trace: Callable, tmp_path: Path, ): + """ + OpenSCENARIO feature: DistanceCondition with relativeDistanceType longitudinal. + + Activates a target actor when the ego object is within the configured + longitudinal distance of a target position. The target actor's post-trigger + trajectory is compared with the transformed reference trace. + """ + _run_distance_longitudinal_start_trigger_case( osi_trace=osi_trace, odr_file=odr_file, diff --git a/osc_validation/validation/scenario/triggers/val_simulation_time_start_trigger.py b/osc_validation/validation/scenario/triggers/val_simulation_time_start_trigger.py index 6baf20d..1a94520 100644 --- a/osc_validation/validation/scenario/triggers/val_simulation_time_start_trigger.py +++ b/osc_validation/validation/scenario/triggers/val_simulation_time_start_trigger.py @@ -36,7 +36,8 @@ def odr_file(request): return request.getfixturevalue("osi_trace").with_suffix(".xodr") -@pytest.mark.trajectory +@pytest.mark.validation_category("trigger") +@pytest.mark.validation_feature("SimulationTimeCondition") @pytest.mark.parametrize("moving_object_id", [1, 2]) @pytest.mark.parametrize("ego_object_id", [1]) @pytest.mark.parametrize("init_pose_object_id", [2]) @@ -61,6 +62,8 @@ def test_simulation_time_start_trigger_delays_actor_trajectory( tolerance: float, ): """ + OpenSCENARIO feature: StartTrigger using SimulationTimeCondition. + Validates delayed Event StartTrigger (SimulationTimeCondition) behavior using trajectory alignment similarity. """ diff --git a/osc_validation/validation/scenario/triggers/val_speed_start_trigger.py b/osc_validation/validation/scenario/triggers/val_speed_start_trigger.py index c61fe96..7152a5a 100644 --- a/osc_validation/validation/scenario/triggers/val_speed_start_trigger.py +++ b/osc_validation/validation/scenario/triggers/val_speed_start_trigger.py @@ -206,7 +206,8 @@ def _run_speed_start_trigger_case( assert mae < tolerance -@pytest.mark.trajectory +@pytest.mark.validation_category("trigger") +@pytest.mark.validation_feature("SpeedCondition") def test_speed_start_trigger_activates_target_actor( osi_trace: Path, odr_file: Path, @@ -214,6 +215,14 @@ def test_speed_start_trigger_activates_target_actor( generate_tool_trace: Callable, tmp_path: Path, ): + """ + OpenSCENARIO feature: StartTrigger using SpeedCondition with a rising edge. + + Activates a target actor when the trigger object's speed crosses the configured + threshold upward. The post-activation trajectory is compared with a reference + trace that includes calculated kinematics. + """ + _run_speed_start_trigger_case( osi_trace=osi_trace, odr_file=odr_file, @@ -224,6 +233,8 @@ def test_speed_start_trigger_activates_target_actor( ) +@pytest.mark.validation_category("trigger") +@pytest.mark.validation_feature("SpeedCondition") def test_speed_start_trigger_condition_edge_none_activates_when_initially_true( osi_trace: Path, odr_file: Path, @@ -231,6 +242,14 @@ def test_speed_start_trigger_condition_edge_none_activates_when_initially_true( generate_tool_trace: Callable, tmp_path: Path, ): + """ + OpenSCENARIO feature: SpeedCondition with conditionEdge set to none. + + Checks that a speed condition can activate when it is already true at the start + of the scenario. The trigger threshold is zero, so activation depends on the + condition state rather than an edge transition. + """ + _run_speed_start_trigger_case( osi_trace=osi_trace, odr_file=odr_file, @@ -239,8 +258,9 @@ def test_speed_start_trigger_condition_edge_none_activates_when_initially_true( trigger_speed_mps=0.0, condition_edge="none", ) - +@pytest.mark.validation_category("trigger") +@pytest.mark.validation_feature("SpeedCondition") def test_speed_start_trigger_condition_edge_falling_activates( osi_trace: Path, odr_file: Path, @@ -248,6 +268,14 @@ def test_speed_start_trigger_condition_edge_falling_activates( generate_tool_trace: Callable, tmp_path: Path, ): + """ + OpenSCENARIO feature: SpeedCondition with conditionEdge set to falling. + + Activates a target actor when the triggering object's speed crosses the + configured threshold downward. The resulting trajectory is compared with the + expected triggered reference behavior. + """ + _run_speed_start_trigger_case( osi_trace=osi_trace, odr_file=odr_file, diff --git a/osc_validation/validation/scenario/triggers/val_time_to_collision_start_trigger.py b/osc_validation/validation/scenario/triggers/val_time_to_collision_start_trigger.py index 2d53532..1ff3328 100644 --- a/osc_validation/validation/scenario/triggers/val_time_to_collision_start_trigger.py +++ b/osc_validation/validation/scenario/triggers/val_time_to_collision_start_trigger.py @@ -76,7 +76,8 @@ def _get_object_position_at_last_frame( ) -@pytest.mark.trajectory +@pytest.mark.validation_category("trigger") +@pytest.mark.validation_feature("TimeToCollisionCondition") @pytest.mark.parametrize("moving_object_id", [2]) @pytest.mark.parametrize("trigger_object_id", [1]) @pytest.mark.parametrize("trigger_ttc_s", [5.0]) @@ -96,6 +97,14 @@ def test_time_to_collision_start_trigger_activates_target_actor( rate: float, tolerance: float, ): + """ + OpenSCENARIO feature: TimeToCollisionCondition to a target position. + + Computes kinematics from the reference trace and activates the target actor + when time-to-collision falls below the configured threshold. The post-trigger + trajectory is compared with a transformed reference trace. + """ + logger = logging.getLogger() logger.setLevel(logging.INFO) diff --git a/osc_validation/validation/scenario/triggers/val_traveled_distance_start_trigger.py b/osc_validation/validation/scenario/triggers/val_traveled_distance_start_trigger.py index be1c9c1..644b9ec 100644 --- a/osc_validation/validation/scenario/triggers/val_traveled_distance_start_trigger.py +++ b/osc_validation/validation/scenario/triggers/val_traveled_distance_start_trigger.py @@ -51,7 +51,8 @@ def odr_file(request): return request.getfixturevalue("osi_trace").with_suffix(".xodr") -@pytest.mark.trajectory +@pytest.mark.validation_category("trigger") +@pytest.mark.validation_feature("TraveledDistanceCondition") @pytest.mark.parametrize("moving_object_id", [2]) # moving object to be evaluated for trigger activation and trajectory alignment @pytest.mark.parametrize("trigger_object_id", [1]) # moving object to be triggered for traveled distance trigger condition @pytest.mark.parametrize("trigger_distance_m", [55.0]) @@ -71,6 +72,14 @@ def test_traveled_distance_start_trigger_activates_target_actor( rate: float, tolerance: float, ): + """ + OpenSCENARIO feature: StartTrigger using TraveledDistanceCondition. + + Activates a target actor once the triggering object has traveled the configured + distance. The post-trigger trajectory is compared with a transformed reference + trace after the trigger edge. + """ + logger = logging.getLogger() logger.setLevel(logging.INFO)