diff --git a/examples/conditional_execution.md b/examples/conditional_execution.md index d58eaf68..5fa05644 100644 --- a/examples/conditional_execution.md +++ b/examples/conditional_execution.md @@ -119,6 +119,46 @@ Access the return value/result data from a specific step. Steps may return data **Note**: The `result` field accesses the return value/details from the step execution. Use `!= None` to check if a step returned any data. +## Continue on Error + +By default, when a step fails the scenario stops. Use `continue-on-error: true` to allow subsequent steps to execute even if the current step fails: + +```yaml +- id: validate_metrics + step: Verify Reported Metrics + continue-on-error: true # Scenario continues even if validation fails +``` + +This is different from `if: always()` — `continue-on-error` controls what happens **after** the step runs and fails, while `if` controls whether the step runs **at all**. + +### Combining with Conditions + +`continue-on-error` and `if` can be used together for robust workflows: + +```yaml +steps: + - id: run_tests + step: Run Tests + continue-on-error: true # Don't halt if tests fail + + - id: collect_results + step: Collect Test Results + description: Runs even if tests failed above + + - id: cleanup + step: Cleanup Resources + if: always() # Runs regardless of any previous failure +``` + +### When to Use Each + +| Scenario | Use | +|----------|-----| +| Step might fail but rest of workflow should continue | `continue-on-error: true` | +| Step should only run when a previous step failed | `if: failure()` | +| Step must always run (cleanup, teardown) | `if: always()` | +| Step should run only on success (default behavior) | No `if` needed, or `if: success()` | + ## Complete Example ```yaml diff --git a/examples/workflow_features.md b/examples/workflow_features.md index 3ae0e6dc..1e2a05d7 100644 --- a/examples/workflow_features.md +++ b/examples/workflow_features.md @@ -21,6 +21,16 @@ Complete guide to advanced workflow features including conditional execution and **Operators:** `==`, `!=`, `&&`, `||`, `not` +### Continue on Error + +```yaml +- id: step_name + step: Operation Name + continue-on-error: true # Scenario continues even if this step fails +``` + +By default, a failing step halts the scenario. Use `continue-on-error: true` to allow subsequent steps to execute regardless. This is useful for non-critical checks or when cleanup steps must always run. + ### Loops ```yaml @@ -46,6 +56,7 @@ Complete guide to advanced workflow features including conditional execution and | **Item Loop** | `loop: { items: [...] }` | Process list of files | | **While Loop** | `loop: { while: condition }` | Poll until ready | | **Conditional Step** | `if: expression` | Run only on success | +| **Continue on Error** | `continue-on-error: true` | Don't halt on failure | | **Step Status** | `steps.X.status` | Check if step passed | | **Step Result** | `steps.X.result` | Access return value | | **Loop Index** | `loop.index` | Get iteration number | diff --git a/scenarios/README.md b/scenarios/README.md index 896ad08c..2707a54b 100644 --- a/scenarios/README.md +++ b/scenarios/README.md @@ -108,6 +108,40 @@ steps: - Available status strings: `success`, `failure`, `running`, `skipped`. - You can use conditions like `always()`, `success()`, `failure()` and combined references (e.g., `steps.Upload Flight Declaration.status == 'success'`). +### Run Conditions (`if`) + +Control when a step executes using the `if` field: + +```yaml +steps: + - step: Cleanup Resources + if: always() # Always runs, even after failures + + - step: Deploy + if: success() # Runs only if all previous steps succeeded + + - step: Rollback + if: failure() # Runs only if a previous step failed + + - step: Custom Check + if: steps.test.status == 'success' # Check a specific step +``` + +### Continue on Error (`continue-on-error`) + +By default, a failing step halts the scenario. Set `continue-on-error: true` to allow subsequent steps to execute even if this step fails: + +```yaml +steps: + - step: Run Flaky Test + continue-on-error: true # Scenario continues even if this fails + + - step: Cleanup Resources + if: always() +``` + +This is useful for non-critical validation steps or cleanup flows where you want the scenario to keep going. + ## Validation To validate scenarios locally: diff --git a/scenarios/flight-declarations/add_flight_declaration.yaml b/scenarios/flight-declarations/add_flight_declaration.yaml index b10b3dca..13ee5681 100644 --- a/scenarios/flight-declarations/add_flight_declaration.yaml +++ b/scenarios/flight-declarations/add_flight_declaration.yaml @@ -14,3 +14,4 @@ steps: arguments: state: ENDED - step: Teardown Flight Declaration + if: always() diff --git a/scenarios/flight-declarations/add_flight_declaration_via_operational_intent.yaml b/scenarios/flight-declarations/add_flight_declaration_via_operational_intent.yaml index f3743f08..5a517092 100644 --- a/scenarios/flight-declarations/add_flight_declaration_via_operational_intent.yaml +++ b/scenarios/flight-declarations/add_flight_declaration_via_operational_intent.yaml @@ -14,3 +14,4 @@ steps: arguments: state: ENDED - step: Teardown Flight Declaration + if: always() diff --git a/scenarios/flight-declarations/bulk_add_flight_declarations.yaml b/scenarios/flight-declarations/bulk_add_flight_declarations.yaml index 73043f22..259f836f 100644 --- a/scenarios/flight-declarations/bulk_add_flight_declarations.yaml +++ b/scenarios/flight-declarations/bulk_add_flight_declarations.yaml @@ -4,9 +4,11 @@ steps: - step: Setup Two Flight Declarations - id: teardown_first_declaration step: Teardown Flight Declaration + if: always() arguments: flight_declaration_id: ${{ steps.Setup Two Flight Declarations.result.declarations[0].id }} - id: teardown_second_declaration step: Teardown Flight Declaration + if: always() arguments: flight_declaration_id: ${{ steps.Setup Two Flight Declarations.result.declarations[1].id }} diff --git a/scenarios/flight-declarations/bulk_add_flight_declarations_via_operational_intents.yaml b/scenarios/flight-declarations/bulk_add_flight_declarations_via_operational_intents.yaml index 08051798..b9384634 100644 --- a/scenarios/flight-declarations/bulk_add_flight_declarations_via_operational_intents.yaml +++ b/scenarios/flight-declarations/bulk_add_flight_declarations_via_operational_intents.yaml @@ -4,9 +4,11 @@ steps: - step: Setup Two Operational Intents - id: teardown_first_declaration step: Teardown Flight Declaration + if: always() arguments: flight_declaration_id: ${{ steps.Setup Two Operational Intents.result.declarations[0].id }} - id: teardown_second_declaration step: Teardown Flight Declaration + if: always() arguments: flight_declaration_id: ${{ steps.Setup Two Operational Intents.result.declarations[1].id }} diff --git a/scenarios/sdsp-f3623/sdsp_heartbeat.yaml b/scenarios/sdsp-f3623/sdsp_heartbeat.yaml index 43028bb4..1c75fe65 100644 --- a/scenarios/sdsp-f3623/sdsp_heartbeat.yaml +++ b/scenarios/sdsp-f3623/sdsp_heartbeat.yaml @@ -29,3 +29,4 @@ steps: arguments: action: STOP surveillance_session_id: ${{ steps.Generate UUID.result }} + if: always() diff --git a/scenarios/sdsp-f3623/sdsp_track.yaml b/scenarios/sdsp-f3623/sdsp_track.yaml index 9e5e69ab..b685fe04 100644 --- a/scenarios/sdsp-f3623/sdsp_track.yaml +++ b/scenarios/sdsp-f3623/sdsp_track.yaml @@ -33,3 +33,4 @@ steps: action: STOP needs: - stream_air_traffic + if: always() diff --git a/scenarios/sdsp-f3623/verify_sdsp_metrics.yaml b/scenarios/sdsp-f3623/verify_sdsp_metrics.yaml index 64de850e..eeeb096a 100644 --- a/scenarios/sdsp-f3623/verify_sdsp_metrics.yaml +++ b/scenarios/sdsp-f3623/verify_sdsp_metrics.yaml @@ -27,6 +27,7 @@ steps: - step: Verify Reported Metrics in Flight Blender needs: - stream_air_traffic + continue-on-error: true arguments: observations: ${{ steps.stream_air_traffic.result.observations }} # session_id: ${{ steps.generated_sdsp_session_id.result }} @@ -37,3 +38,4 @@ steps: action: STOP needs: - stream_air_traffic + if: always() diff --git a/scenarios/standard-scenarios/F1_flow_no_telemetry_with_user_input.yaml b/scenarios/standard-scenarios/F1_flow_no_telemetry_with_user_input.yaml index c2b07335..ee9ead76 100644 --- a/scenarios/standard-scenarios/F1_flow_no_telemetry_with_user_input.yaml +++ b/scenarios/standard-scenarios/F1_flow_no_telemetry_with_user_input.yaml @@ -16,3 +16,4 @@ steps: arguments: state: ENDED - step: Teardown Flight Declaration + if: always() diff --git a/scenarios/standard-scenarios/F1_happy_path.yaml b/scenarios/standard-scenarios/F1_happy_path.yaml index 0b60484f..751cfe71 100644 --- a/scenarios/standard-scenarios/F1_happy_path.yaml +++ b/scenarios/standard-scenarios/F1_happy_path.yaml @@ -22,4 +22,5 @@ steps: state: ENDED description: Marks the operation as ended after the flight is complete. - step: Teardown Flight Declaration + if: always() description: Cleans up the flight declaration and any associated resources. diff --git a/scenarios/standard-scenarios/F2_contingent_path.yaml b/scenarios/standard-scenarios/F2_contingent_path.yaml index 503754b3..15596a9d 100644 --- a/scenarios/standard-scenarios/F2_contingent_path.yaml +++ b/scenarios/standard-scenarios/F2_contingent_path.yaml @@ -18,3 +18,4 @@ steps: arguments: state: ENDED - step: Teardown Flight Declaration + if: always() diff --git a/scenarios/standard-scenarios/F3_non_conforming_path.yaml b/scenarios/standard-scenarios/F3_non_conforming_path.yaml index 9b2811a1..4bc6a77b 100644 --- a/scenarios/standard-scenarios/F3_non_conforming_path.yaml +++ b/scenarios/standard-scenarios/F3_non_conforming_path.yaml @@ -29,4 +29,5 @@ steps: arguments: state: ENDED - step: Teardown Flight Declaration + if: always() description: Cleanup. diff --git a/scenarios/standard-scenarios/F5_non_conforming_path.yaml b/scenarios/standard-scenarios/F5_non_conforming_path.yaml index 793cd08e..3cad647e 100644 --- a/scenarios/standard-scenarios/F5_non_conforming_path.yaml +++ b/scenarios/standard-scenarios/F5_non_conforming_path.yaml @@ -21,3 +21,4 @@ steps: arguments: state: ENDED - step: Teardown Flight Declaration + if: always() diff --git a/src/openutm_verification/core/execution/conditions.py b/src/openutm_verification/core/execution/conditions.py index aa9fc2c3..348038c2 100644 --- a/src/openutm_verification/core/execution/conditions.py +++ b/src/openutm_verification/core/execution/conditions.py @@ -32,9 +32,9 @@ def __init__(self, steps: dict[str, StepResult[Any]], loop_context: dict[str, An self.loop_context = loop_context or {} self.last_step_status: Status | None = None - # Determine last step status (excluding skipped steps) + # Determine last step status (excluding skipped steps and steps with continue_on_error that failed) if steps: - completed_steps = [s for s in steps.values() if s.status != Status.SKIP] + completed_steps = [s for s in steps.values() if s.status != Status.SKIP and not (s.continue_on_error and s.status == Status.FAIL)] if completed_steps: self.last_step_status = completed_steps[-1].status diff --git a/src/openutm_verification/core/execution/definitions.py b/src/openutm_verification/core/execution/definitions.py index 7737a3f4..12deb228 100644 --- a/src/openutm_verification/core/execution/definitions.py +++ b/src/openutm_verification/core/execution/definitions.py @@ -24,6 +24,11 @@ class StepDefinition(BaseModel): description: str | None = None if_condition: str | None = Field(default=None, alias="if", description="Conditional expression to determine if step should run") loop: LoopConfig | None = Field(default=None, description="Loop configuration for repeating the step") + continue_on_error: bool = Field( + default=False, + alias="continue-on-error", + description="If true, subsequent steps will still execute even if this step fails. Default is false (stop on failure).", + ) class GroupDefinition(BaseModel): diff --git a/src/openutm_verification/core/reporting/reporting_models.py b/src/openutm_verification/core/reporting/reporting_models.py index 5833ef30..7563e860 100644 --- a/src/openutm_verification/core/reporting/reporting_models.py +++ b/src/openutm_verification/core/reporting/reporting_models.py @@ -44,6 +44,7 @@ class StepResult(BaseModel, Generic[T]): result: T = None # type: ignore error_message: str | None = None logs: list[str] = [] + continue_on_error: bool = False class ScenarioResult(BaseModel): diff --git a/src/openutm_verification/server/router.py b/src/openutm_verification/server/router.py index df57866a..6674eca7 100644 --- a/src/openutm_verification/server/router.py +++ b/src/openutm_verification/server/router.py @@ -105,7 +105,7 @@ async def get_scenario(scenario: str): """Get the content of a specific scenario.""" try: scenario_def = load_yaml_scenario_definition(scenario) - return scenario_def.model_dump() + return scenario_def.model_dump(by_alias=True) except FileNotFoundError: raise HTTPException(status_code=404, detail="Scenario not found") except Exception as e: @@ -131,7 +131,7 @@ async def save_scenario(name: str, scenario: ScenarioDefinition): try: # Convert Pydantic model to dict, excluding None values to keep YAML clean - data = scenario.model_dump(exclude_none=True, exclude_defaults=True) + data = scenario.model_dump(by_alias=True, exclude_none=True, exclude_defaults=True) with open(file_path, "w") as f: yaml.dump(data, f, sort_keys=False, default_flow_style=False) diff --git a/src/openutm_verification/server/runner.py b/src/openutm_verification/server/runner.py index f6314d1a..2d131752 100644 --- a/src/openutm_verification/server/runner.py +++ b/src/openutm_verification/server/runner.py @@ -626,10 +626,17 @@ async def _execute_group( # If step failed and it's not allowed to fail, break the group if result.status == Status.FAIL: - logger.error(f"Group step '{group_step.id}' failed, stopping group execution") - # Mark remaining steps as SKIP - self._mark_remaining_group_steps_skipped(group, group_name, loop_context, executed_step_indices) - break + if group_step.continue_on_error: + logger.warning(f"Group step '{group_step.id}' failed, but continue-on-error=true — continuing group") + result.continue_on_error = True + if self.session_context: + with self.session_context: + self.session_context.update_result(result) + else: + logger.error(f"Group step '{group_step.id}' failed, stopping group execution") + # Mark remaining steps as SKIP + self._mark_remaining_group_steps_skipped(group, group_name, loop_context, executed_step_indices) + break return results @@ -663,6 +670,15 @@ def _mark_remaining_group_steps_skipped( with self.session_context: self.session_context.update_result(skipped_result) + def _mark_results_continue_on_error(self, results: List[StepResult]) -> None: + """Mark failed results as continue_on_error and update them in session context.""" + for result in results: + if result.status == Status.FAIL: + result.continue_on_error = True + if self.session_context: + with self.session_context: + self.session_context.update_result(result) + async def _wait_for_dependencies(self, step: StepDefinition) -> None: """Wait for any declared dependencies (by step ID) before executing a step.""" if not step.needs: @@ -779,19 +795,29 @@ async def run_scenario(self, scenario: ScenarioDefinition) -> List[StepResult]: # Wait for declared dependencies (useful for background steps) await self._wait_for_dependencies(step) + should_continue_on_error = step.continue_on_error + # Check if this step references a group if self._is_group_reference(step.step, scenario): # Handle loop execution for groups if step.loop: loop_results = await self._execute_loop_for_group(step, scenario) if loop_results and any(r.status == Status.FAIL for r in loop_results): - logger.error(f"Loop for group '{step.id}' failed, breaking scenario") - break + if should_continue_on_error: + logger.warning(f"Loop for group '{step.id}' failed, but continue-on-error=true — continuing") + self._mark_results_continue_on_error(loop_results) + else: + logger.error(f"Loop for group '{step.id}' failed, breaking scenario") + break else: group_results = await self._execute_group(step, scenario) if any(r.status == Status.FAIL for r in group_results): - logger.error(f"Group '{step.id}' failed, breaking scenario") - break + if should_continue_on_error: + logger.warning(f"Group '{step.id}' failed, but continue-on-error=true — continuing") + self._mark_results_continue_on_error(group_results) + else: + logger.error(f"Group '{step.id}' failed, breaking scenario") + break else: # Regular step execution # Handle loop execution @@ -799,10 +825,20 @@ async def run_scenario(self, scenario: ScenarioDefinition) -> List[StepResult]: loop_results = await self._execute_loop(step) # Check if any loop iteration failed if loop_results and any(r.status == Status.FAIL for r in loop_results): - logger.error(f"Loop for step '{step.id}' failed, breaking scenario") - break + if should_continue_on_error: + logger.warning(f"Loop for step '{step.id}' failed, but continue-on-error=true — continuing") + self._mark_results_continue_on_error(loop_results) + else: + logger.error(f"Loop for step '{step.id}' failed, breaking scenario") + break else: - await self.execute_single_step(step) + result = await self.execute_single_step(step) + if result.status == Status.FAIL and should_continue_on_error: + logger.warning(f"Step '{step.id}' failed, but continue-on-error=true — continuing") + result.continue_on_error = True + if self.session_context: + with self.session_context: + self.session_context.update_result(result) # An empty scenario (or one whose every step was skipped before any # `with self.session_context:` block ran) leaves session_context.state # unset; return an empty list rather than crashing. diff --git a/tests/test_conditions.py b/tests/test_conditions.py index f4f2203c..1dcf91b1 100644 --- a/tests/test_conditions.py +++ b/tests/test_conditions.py @@ -171,3 +171,42 @@ def test_loop_index_boundary_conditions(self): evaluator = ConditionEvaluator(steps, loop_context) assert evaluator.evaluate("loop.index < 100") is True assert evaluator.evaluate("loop.index >= 99") is True + + def test_success_ignores_continue_on_error_steps(self): + """Test success() ignores failed steps with continue_on_error=True.""" + steps = { + "step1": StepResult(id="step1", name="Step 1", status=Status.PASS, duration=1.0), + "step2": StepResult(id="step2", name="Step 2", status=Status.FAIL, duration=1.0, continue_on_error=True), + } + evaluator = ConditionEvaluator(steps) + # step2 failed but has continue_on_error, so success() should still be True + assert evaluator.evaluate("success()") is True + + def test_failure_ignores_continue_on_error_steps(self): + """Test failure() ignores failed steps with continue_on_error=True.""" + steps = { + "step1": StepResult(id="step1", name="Step 1", status=Status.PASS, duration=1.0), + "step2": StepResult(id="step2", name="Step 2", status=Status.FAIL, duration=1.0, continue_on_error=True), + } + evaluator = ConditionEvaluator(steps) + # step2 failed but has continue_on_error, so last_step_status is PASS (from step1) + assert evaluator.evaluate("failure()") is False + + def test_continue_on_error_step_status_still_queryable(self): + """Test that individual step status is still queryable even with continue_on_error.""" + steps = { + "step1": StepResult(id="step1", name="Step 1", status=Status.PASS, duration=1.0), + "step2": StepResult(id="step2", name="Step 2", status=Status.FAIL, duration=1.0, continue_on_error=True), + } + evaluator = ConditionEvaluator(steps) + # Direct step status queries should still reflect the actual status + assert evaluator.evaluate("steps.step2.status == 'failure'") is True + + def test_success_fails_with_non_continue_on_error_step(self): + """Test success() still returns False for failed steps without continue_on_error.""" + steps = { + "step1": StepResult(id="step1", name="Step 1", status=Status.PASS, duration=1.0), + "step2": StepResult(id="step2", name="Step 2", status=Status.FAIL, duration=1.0, continue_on_error=False), + } + evaluator = ConditionEvaluator(steps) + assert evaluator.evaluate("success()") is False diff --git a/tests/test_continue_on_error_integration.py b/tests/test_continue_on_error_integration.py new file mode 100644 index 00000000..8447c4e0 --- /dev/null +++ b/tests/test_continue_on_error_integration.py @@ -0,0 +1,231 @@ +"""Integration tests for continue-on-error runner behavior. + +Validates that the SessionManager.run_scenario properly continues +execution after failures when continue-on-error is set, and that +the failed StepResult is marked continue_on_error=True. +""" + +from unittest.mock import AsyncMock, patch + +import pytest + +from openutm_verification.core.execution.definitions import ( + GroupDefinition, + ScenarioDefinition, + StepDefinition, +) +from openutm_verification.core.execution.scenario_runner import ScenarioContext +from openutm_verification.core.reporting.reporting_models import Status, StepResult +from openutm_verification.server.runner import SessionManager + + +def _make_runner() -> SessionManager: + """Create a minimal SessionManager with mocked internals.""" + SessionManager._instance = None + runner = SessionManager.__new__(SessionManager) + runner._initialized = True + runner.config_path = None + runner.config = type( + "C", + (), + { + "suites": {}, + "reporting": type("R", (), {"output_dir": "/tmp/reports", "formats": []})(), + }, + )() + runner.client_map = {} + runner.session_stack = None + runner.session_resolver = None + runner.session_context = None + runner.session_tasks = {} + runner.current_start_time = None + runner.current_output_dir = None + runner.current_timestamp_str = None + runner.current_run_error = None + runner.data_files = None + runner._stop_event = None + return runner + + +def _setup_session(runner: SessionManager) -> None: + """Prepare a fresh ScenarioContext on the runner.""" + ctx = ScenarioContext() + ctx.__enter__() + runner.session_context = ctx + + +@pytest.fixture(autouse=True) +def _reset_singleton(): + """Ensure SessionManager singleton is reset between tests.""" + SessionManager._instance = None + yield + SessionManager._instance = None + + +# ── Regular step tests ───────────────────────────────────────────── + + +@pytest.mark.asyncio +async def test_continue_on_error_continues_after_failure(): + """Scenario continues executing steps after a continue-on-error failure.""" + runner = _make_runner() + _setup_session(runner) + + scenario = ScenarioDefinition( + name="test", + steps=[ + StepDefinition(step="step_a", id="step_a", **{"continue-on-error": True}), + StepDefinition(step="step_b", id="step_b"), + ], + ) + + call_order = [] + + async def fake_execute(step, loop_context=None): + call_order.append(step.id) + if step.id == "step_a": + result = StepResult(id=step.id, name=step.step, status=Status.FAIL, duration=0.1, error_message="boom") + else: + result = StepResult(id=step.id, name=step.step, status=Status.PASS, duration=0.1) + with runner.session_context: + runner.session_context.update_result(result) + return result + + with patch.object(runner, "execute_single_step", side_effect=fake_execute): + with patch.object(runner, "initialize_session", new_callable=AsyncMock): + with patch.object(runner, "_wait_for_dependencies", new_callable=AsyncMock): + results = await runner.run_scenario(scenario) + + assert call_order == ["step_a", "step_b"], "step_b should have executed after step_a failed with continue-on-error" + + step_a_result = next(r for r in results if r.id == "step_a") + assert step_a_result.status == Status.FAIL + assert step_a_result.continue_on_error is True + + step_b_result = next(r for r in results if r.id == "step_b") + assert step_b_result.status == Status.PASS + + +@pytest.mark.asyncio +async def test_without_continue_on_error_stops_after_failure(): + """Scenario stops executing steps after a failure without continue-on-error.""" + runner = _make_runner() + _setup_session(runner) + + scenario = ScenarioDefinition( + name="test", + steps=[ + StepDefinition(step="step_a", id="step_a"), + StepDefinition(step="step_b", id="step_b"), + ], + ) + + call_order = [] + + async def fake_execute(step, loop_context=None): + call_order.append(step.id) + if step.id == "step_a": + result = StepResult(id=step.id, name=step.step, status=Status.FAIL, duration=0.1, error_message="boom") + else: + result = StepResult(id=step.id, name=step.step, status=Status.PASS, duration=0.1) + with runner.session_context: + runner.session_context.update_result(result) + return result + + with patch.object(runner, "execute_single_step", side_effect=fake_execute): + with patch.object(runner, "initialize_session", new_callable=AsyncMock): + with patch.object(runner, "_wait_for_dependencies", new_callable=AsyncMock): + results = await runner.run_scenario(scenario) + + assert call_order == ["step_a"], "step_b should NOT have executed" + + +# ── Group step tests ─────────────────────────────────────────────── + + +@pytest.mark.asyncio +async def test_continue_on_error_inside_group(): + """A group step with continue-on-error allows subsequent group steps to run.""" + runner = _make_runner() + _setup_session(runner) + + scenario = ScenarioDefinition( + name="test", + groups={ + "my_group": GroupDefinition( + steps=[ + StepDefinition(step="g_step1", id="g_step1", **{"continue-on-error": True}), + StepDefinition(step="g_step2", id="g_step2"), + ], + ), + }, + steps=[ + StepDefinition(step="my_group", id="run_group"), + ], + ) + + call_order = [] + + async def fake_execute(step, loop_context=None): + call_order.append(step.id) + if step.step == "g_step1": + result = StepResult(id=step.id, name=step.step, status=Status.FAIL, duration=0.1, error_message="boom") + else: + result = StepResult(id=step.id, name=step.step, status=Status.PASS, duration=0.1) + with runner.session_context: + runner.session_context.update_result(result) + return result + + with patch.object(runner, "execute_single_step", side_effect=fake_execute): + with patch.object(runner, "initialize_session", new_callable=AsyncMock): + with patch.object(runner, "_wait_for_dependencies", new_callable=AsyncMock): + results = await runner.run_scenario(scenario) + + assert "g_step1" in call_order + assert "g_step2" in call_order, "g_step2 should execute because g_step1 has continue-on-error" + + g1 = next(r for r in results if "g_step1" in (r.id or "")) + assert g1.status == Status.FAIL + assert g1.continue_on_error is True + + +@pytest.mark.asyncio +async def test_group_stops_without_continue_on_error(): + """A group stops at a failed step when continue-on-error is not set.""" + runner = _make_runner() + _setup_session(runner) + + scenario = ScenarioDefinition( + name="test", + groups={ + "my_group": GroupDefinition( + steps=[ + StepDefinition(step="g_step1", id="g_step1"), + StepDefinition(step="g_step2", id="g_step2"), + ], + ), + }, + steps=[ + StepDefinition(step="my_group", id="run_group"), + ], + ) + + call_order = [] + + async def fake_execute(step, loop_context=None): + call_order.append(step.id) + if step.step == "g_step1": + result = StepResult(id=step.id, name=step.step, status=Status.FAIL, duration=0.1, error_message="boom") + else: + result = StepResult(id=step.id, name=step.step, status=Status.PASS, duration=0.1) + with runner.session_context: + runner.session_context.update_result(result) + return result + + with patch.object(runner, "execute_single_step", side_effect=fake_execute): + with patch.object(runner, "initialize_session", new_callable=AsyncMock): + with patch.object(runner, "_wait_for_dependencies", new_callable=AsyncMock): + results = await runner.run_scenario(scenario) + + assert "g_step1" in call_order + assert "g_step2" not in call_order, "g_step2 should NOT execute after g_step1 failure" diff --git a/tests/test_group_execution.py b/tests/test_group_execution.py index b408d36c..2536601d 100644 --- a/tests/test_group_execution.py +++ b/tests/test_group_execution.py @@ -181,3 +181,65 @@ async def test_empty_groups_section(): # Verify groups is empty dict assert scenario.groups == {} assert len(scenario.steps) == 1 + + +@pytest.mark.asyncio +async def test_continue_on_error_parsing(): + """Test that continue-on-error is correctly parsed from YAML.""" + yaml_content = """ +name: test_continue_on_error +description: Test scenario with continue-on-error + +steps: + - step: Setup Flight Declaration + - step: Validate Metrics + continue-on-error: true + - step: Teardown Flight Declaration +""" + + data = yaml.safe_load(yaml_content) + scenario = ScenarioDefinition.model_validate(data) + + assert scenario.steps[0].continue_on_error is False # default + assert scenario.steps[1].continue_on_error is True + assert scenario.steps[2].continue_on_error is False # default + + +@pytest.mark.asyncio +async def test_continue_on_error_default(): + """Test that continue_on_error defaults to False.""" + yaml_content = """ +name: test_default_error_action +description: Test default continue-on-error + +steps: + - step: Setup Flight Declaration +""" + + data = yaml.safe_load(yaml_content) + scenario = ScenarioDefinition.model_validate(data) + assert scenario.steps[0].continue_on_error is False + + +@pytest.mark.asyncio +async def test_continue_on_error_in_group(): + """Test that continue-on-error works inside groups.""" + yaml_content = """ +name: test_continue_on_error_group +description: Test scenario with continue-on-error inside a group + +groups: + my_group: + steps: + - step: Validate Metrics + continue-on-error: true + - step: Teardown Flight Declaration + +steps: + - step: my_group +""" + + data = yaml.safe_load(yaml_content) + scenario = ScenarioDefinition.model_validate(data) + assert scenario.groups["my_group"].steps[0].continue_on_error is True + assert scenario.groups["my_group"].steps[1].continue_on_error is False diff --git a/web-editor/README_GROUPS.md b/web-editor/README_GROUPS.md index 5d4f43d1..b19655e1 100644 --- a/web-editor/README_GROUPS.md +++ b/web-editor/README_GROUPS.md @@ -43,7 +43,7 @@ In the main scenario flow: - Fixed count: `count: 5` - Iterate items: `items: [...]` - While condition: `while: loop.index < 10` -- **Conditional groups**: Groups can have conditions: `if: ${{ always() }}` +- **Conditional groups**: Groups can have conditions: `if: always()` ### 5. UI Indicators diff --git a/web-editor/src/components/ScenarioEditor.tsx b/web-editor/src/components/ScenarioEditor.tsx index dcb1a3d7..1a337605 100644 --- a/web-editor/src/components/ScenarioEditor.tsx +++ b/web-editor/src/components/ScenarioEditor.tsx @@ -942,6 +942,49 @@ const ScenarioEditorContent = () => { }); }, [setNodes]); + const updateNodeContinueOnError = useCallback((nodeId: string, value: boolean) => { + setIsDirty(true); + setNodes((nds) => { + const targetNode = nds.find(n => n.id === nodeId); + // If this is a group child node, sync back to the groups state + if (targetNode?.parentId) { + const parentNode = nds.find(n => n.id === targetNode.parentId); + if (parentNode?.data.isGroupContainer) { + const groupName = parentNode.data.label.replace(/^📦\s*/, '').trim(); + const stepId = targetNode.data.stepId || targetNode.data.label; + setCurrentScenarioGroups(prev => { + const group = prev[groupName]; + if (!group) return prev; + return { + ...prev, + [groupName]: { + ...group, + steps: group.steps.map(s => + (s.id || s.step) === stepId + ? { ...s, 'continue-on-error': value || undefined } + : s + ), + }, + }; + }); + } + } + return nds.map((node) => ( + node.id === nodeId + ? { ...node, data: { ...node.data, continueOnError: value } } + : node + )); + }); + + setSelectedNode((prev) => { + if (!prev || prev.id !== nodeId) return prev; + return { + ...prev, + data: { ...prev.data, continueOnError: value }, + }; + }); + }, [setNodes]); + const updateNodeNeeds = useCallback((nodeId: string, needs: string[]) => { setIsDirty(true); const cleanedNeeds = needs.filter(Boolean); @@ -1226,6 +1269,7 @@ const ScenarioEditorContent = () => { onUpdateStepId={updateNodeStepId} onUpdateIfCondition={updateNodeIfCondition} onUpdateLoop={updateNodeLoop} + onUpdateContinueOnError={updateNodeContinueOnError} onUpdateGroupDescription={updateGroupDescription} /> ) : ( diff --git a/web-editor/src/components/ScenarioEditor/CustomNode.tsx b/web-editor/src/components/ScenarioEditor/CustomNode.tsx index ce45e9ac..54182fff 100644 --- a/web-editor/src/components/ScenarioEditor/CustomNode.tsx +++ b/web-editor/src/components/ScenarioEditor/CustomNode.tsx @@ -1,10 +1,29 @@ import { Handle, Position, type NodeProps, type Node } from '@xyflow/react'; -import { Box, CheckCircle, XCircle, AlertTriangle, Loader2, MinusCircle, RotateCw, GitBranch, Timer, Hourglass, Plane } from 'lucide-react'; +import { Box, CheckCircle, XCircle, AlertTriangle, Loader2, MinusCircle, RotateCw, GitBranch, Timer, Hourglass, Plane, ShieldAlert, CircleDot, Infinity as InfinityIcon } from 'lucide-react'; import styles from '../../styles/Node.module.css'; import { getPhaseColor, PHASE_LABELS } from '../../utils/phaseColors'; import type { NodeData } from '../../types/scenario'; +const ConditionIcon = ({ condition }: { condition: string }) => { + const trimmed = condition.trim(); + if (trimmed === 'success()') return ; + if (trimmed === 'failure()') return ; + if (trimmed === 'always()') return ; + return ; +}; + +const CONDITION_LABELS: Record = { + 'success()': 'Runs on success', + 'failure()': 'Runs on failure', + 'always()': 'Always runs', +}; + +const getConditionLabel = (condition: string): string => { + const trimmed = condition.trim(); + return CONDITION_LABELS[trimmed] || trimmed; +}; + const ModifierBadges = ({ data }: { data: NodeData }) => (
{data.runInBackground && ( @@ -25,6 +44,12 @@ const ModifierBadges = ({ data }: { data: NodeData }) => ( loop
)} + {data.continueOnError && ( +
+ + continue +
+ )} ); @@ -73,6 +98,12 @@ export const CustomNode = ({ data, selected }: NodeProps>) => { {data.label} + {data.ifCondition && data.ifCondition.trim() !== '' && data.ifCondition.trim() !== 'success()' && ( +
+ + {getConditionLabel(data.ifCondition)} +
+ )} ); } @@ -123,6 +154,12 @@ export const CustomNode = ({ data, selected }: NodeProps>) => { )} + {data.ifCondition && data.ifCondition.trim() !== '' && data.ifCondition.trim() !== 'success()' && ( +
+ + {getConditionLabel(data.ifCondition)} +
+ )} ); diff --git a/web-editor/src/components/ScenarioEditor/PropertiesPanel.tsx b/web-editor/src/components/ScenarioEditor/PropertiesPanel.tsx index af6120ce..4a898f65 100644 --- a/web-editor/src/components/ScenarioEditor/PropertiesPanel.tsx +++ b/web-editor/src/components/ScenarioEditor/PropertiesPanel.tsx @@ -1,4 +1,4 @@ -import React, { useState, useMemo } from 'react'; +import React, { useState, useMemo, useEffect } from 'react'; import { X, Link as LinkIcon, Unlink, Plane } from 'lucide-react'; import type { Node } from '@xyflow/react'; import layoutStyles from '../../styles/EditorLayout.module.css'; @@ -65,6 +65,7 @@ interface PropertiesPanelProps { onUpdateIfCondition: (nodeId: string, condition: string) => void; onUpdateLoop: (nodeId: string, loopConfig: { count?: number; items?: unknown[]; while?: string } | undefined) => void; onUpdateNeeds: (nodeId: string, needs: string[]) => void; + onUpdateContinueOnError: (nodeId: string, value: boolean) => void; onUpdateGroupDescription?: (groupName: string, description: string) => void; } @@ -78,7 +79,7 @@ const parseRefString = (value: unknown) => { }; }; -export const PropertiesPanel = ({ selectedNode, connectedNodes, allNodes, onClose, onUpdateParameter, onUpdateRunInBackground, onUpdateStepId, onUpdateIfCondition, onUpdateLoop, onUpdateNeeds, onUpdateGroupDescription }: PropertiesPanelProps) => { +export const PropertiesPanel = ({ selectedNode, connectedNodes, allNodes, onClose, onUpdateParameter, onUpdateRunInBackground, onUpdateStepId, onUpdateIfCondition, onUpdateLoop, onUpdateNeeds, onUpdateContinueOnError, onUpdateGroupDescription }: PropertiesPanelProps) => { const { sidebarWidth: width, isResizing, startResizing } = useSidebarResize(480, 300, 800); // Compute loop type from node data @@ -92,15 +93,14 @@ export const PropertiesPanel = ({ selectedNode, connectedNodes, allNodes, onClos // Local state for JSON items textarea to allow invalid JSON while typing const [jsonItemsText, setJsonItemsText] = useState(null); - const [prevNodeId, setPrevNodeId] = useState(selectedNode.id); - const [prevItems, setPrevItems] = useState(selectedNode.data.loop?.items); + // Local state to track when "Custom expression…" is explicitly selected + const [isCustomCondition, setIsCustomCondition] = useState(false); - // Sync local JSON text when node data changes externally - if (selectedNode.id !== prevNodeId || selectedNode.data.loop?.items !== prevItems) { - setPrevNodeId(selectedNode.id); - setPrevItems(selectedNode.data.loop?.items); + // Sync local state when node selection or loop items change + useEffect(() => { setJsonItemsText(null); - } + setIsCustomCondition(false); + }, [selectedNode.id, selectedNode.data.loop?.items]); const getItemsText = () => { if (jsonItemsText !== null) return jsonItemsText; @@ -215,6 +215,20 @@ export const PropertiesPanel = ({ selectedNode, connectedNodes, allNodes, onClos +
+ +
+ If checked, subsequent steps will still execute even if this step fails. +
+
+
@@ -252,20 +266,44 @@ export const PropertiesPanel = ({ selectedNode, connectedNodes, allNodes, onClos
- - Run Condition + + {(isCustomCondition || (selectedNode.data.ifCondition && !['', 'success()', 'failure()', 'always()'].includes(selectedNode.data.ifCondition.trim()))) && ( + onUpdateIfCondition(selectedNode.id, e.target.value)} + placeholder="steps.step1.status == 'success'" + /> + )}
- Conditional expression (GitHub Actions-style). Examples:
- • success() - run if previous step succeeded
- • failure() - run if previous step failed
- • always() - always run
- • steps.step1.status == 'pass' - check specific step + Controls when this step executes. Steps marked "Continue on Error" are excluded from success/failure evaluation. Custom expressions support status checks like steps.step_id.status == 'success'
diff --git a/web-editor/src/components/ScenarioEditor/__tests__/PropertiesPanel.test.tsx b/web-editor/src/components/ScenarioEditor/__tests__/PropertiesPanel.test.tsx index 0778de7b..e369b698 100644 --- a/web-editor/src/components/ScenarioEditor/__tests__/PropertiesPanel.test.tsx +++ b/web-editor/src/components/ScenarioEditor/__tests__/PropertiesPanel.test.tsx @@ -29,6 +29,7 @@ describe('PropertiesPanel', () => { onUpdateIfCondition: vi.fn(), onUpdateLoop: vi.fn(), onUpdateNeeds: vi.fn(), + onUpdateContinueOnError: vi.fn(), }; it('renders correctly', () => { diff --git a/web-editor/src/styles/Node.module.css b/web-editor/src/styles/Node.module.css index 721e2c80..1857ee4b 100644 --- a/web-editor/src/styles/Node.module.css +++ b/web-editor/src/styles/Node.module.css @@ -120,7 +120,8 @@ .loopBadge, .conditionBadge, -.backgroundBadge { +.backgroundBadge, +.continueOnErrorBadge { display: flex; align-items: center; gap: 3px; @@ -161,6 +162,37 @@ background-color: rgba(59, 130, 246, 0.28); } +.continueOnErrorBadge { + background-color: rgba(245, 158, 11, 0.2); + color: #f59e0b; + border: 1px solid rgba(245, 158, 11, 0.4); +} + +.continueOnErrorBadge:hover { + background-color: rgba(245, 158, 11, 0.3); +} + +.conditionText { + display: flex; + align-items: center; + gap: 4px; + font-size: 11px; + color: #22c55e; + padding: 2px 4px; + margin-top: -4px; + margin-bottom: 2px; + overflow: hidden; + text-overflow: ellipsis; + white-space: nowrap; + max-width: 100%; +} + +.conditionText span { + overflow: hidden; + text-overflow: ellipsis; + white-space: nowrap; +} + /* Group container label */ .groupContainerLabel { position: absolute; diff --git a/web-editor/src/types/scenario.ts b/web-editor/src/types/scenario.ts index ffcc7fbc..24691d3a 100644 --- a/web-editor/src/types/scenario.ts +++ b/web-editor/src/types/scenario.ts @@ -28,6 +28,7 @@ export interface NodeData extends Record { ifCondition?: string; loop?: LoopConfig; needs?: string[]; + continueOnError?: boolean; isGroupReference?: boolean; isGroupContainer?: boolean; isPhaseContainer?: boolean; @@ -61,6 +62,7 @@ export interface ScenarioStep { description?: string; if?: string; loop?: LoopConfig; + 'continue-on-error'?: boolean; } export interface ScenarioDefinition { diff --git a/web-editor/src/utils/__tests__/scenarioConversion.test.ts b/web-editor/src/utils/__tests__/scenarioConversion.test.ts index 441780a3..22a23acc 100644 --- a/web-editor/src/utils/__tests__/scenarioConversion.test.ts +++ b/web-editor/src/utils/__tests__/scenarioConversion.test.ts @@ -145,3 +145,58 @@ describe('YAML Conversion - Reference Normalization', () => { expect(scenario.groups?.my_group?.steps?.[1]?.arguments?.data).toBe('${{ group.fetch.result }}'); }); }); + +describe('YAML Conversion - continue-on-error', () => { + it('includes continue-on-error=true in YAML output', () => { + const node: Node = { + id: 'node1', + position: { x: 0, y: 0 }, + data: { + label: 'Validate Metrics', + stepId: 'validate', + operationId: 'validate_op', + parameters: [], + continueOnError: true + } + }; + + const scenario = convertGraphToYaml([node], [], [], 'test', 'test'); + const step = scenario.steps[0]; + expect(step['continue-on-error']).toBe(true); + }); + + it('omits continue-on-error when set to false (default)', () => { + const node: Node = { + id: 'node1', + position: { x: 0, y: 0 }, + data: { + label: 'Setup', + stepId: 'setup', + operationId: 'setup_op', + parameters: [], + continueOnError: false + } + }; + + const scenario = convertGraphToYaml([node], [], [], 'test', 'test'); + const step = scenario.steps[0]; + expect(step['continue-on-error']).toBeUndefined(); + }); + + it('omits continue-on-error when not set', () => { + const node: Node = { + id: 'node1', + position: { x: 0, y: 0 }, + data: { + label: 'Setup', + stepId: 'setup', + operationId: 'setup_op', + parameters: [] + } + }; + + const scenario = convertGraphToYaml([node], [], [], 'test', 'test'); + const step = scenario.steps[0]; + expect(step['continue-on-error']).toBeUndefined(); + }); +}); diff --git a/web-editor/src/utils/scenarioConversion.ts b/web-editor/src/utils/scenarioConversion.ts index 68655e2e..bfb3d86f 100644 --- a/web-editor/src/utils/scenarioConversion.ts +++ b/web-editor/src/utils/scenarioConversion.ts @@ -79,6 +79,7 @@ export const convertYamlToGraph = ( runInBackground: step.background, ifCondition: step.if, loop: step.loop, + continueOnError: step['continue-on-error'], isGroupReference: true, isGroupContainer: true }, @@ -124,7 +125,8 @@ export const convertYamlToGraph = ( operationId: groupOperation?.id, description: groupStep.description || groupOperation?.description || '', phase: groupOperation?.phase, - parameters: groupParameters + parameters: groupParameters, + continueOnError: groupStep['continue-on-error'], } }; @@ -187,6 +189,7 @@ export const convertYamlToGraph = ( ifCondition: step.if, loop: step.loop, needs: needsList, + continueOnError: step['continue-on-error'], isGroupReference: false } }; @@ -313,6 +316,10 @@ export const convertGraphToYaml = ( step.loop = node.data.loop; } + if (node.data.continueOnError) { + step['continue-on-error'] = true; + } + return step; } @@ -390,6 +397,10 @@ export const convertGraphToYaml = ( step.loop = node.data.loop; } + if (node.data.continueOnError) { + step['continue-on-error'] = true; + } + // Dependencies (needs) come from dotted edges into this node const deps = dependencyEdges .filter(e => e.target === node.id)