From 0ad395a96544b511dc2ec7d5f4d70f0c395d199e Mon Sep 17 00:00:00 2001 From: Attila Kobor Date: Fri, 20 Mar 2026 16:00:18 +0100 Subject: [PATCH 1/5] Add support for 'on_step_result_fail' in scenario steps and UI components - Introduced 'on_step_result_fail' property to control behavior on step failure. - Updated YAML conversion functions to include 'on_step_result_fail'. - Enhanced UI components to allow setting 'on_step_result_fail' for steps. - Added tests to verify the correct parsing and functionality of 'on_step_result_fail'. --- .../add_flight_declaration.yaml | 1 + ...ht_declaration_via_operational_intent.yaml | 1 + .../bulk_add_flight_declarations.yaml | 2 + ..._declarations_via_operational_intents.yaml | 2 + scenarios/sdsp-f3623/sdsp_heartbeat.yaml | 1 + scenarios/sdsp-f3623/sdsp_track.yaml | 1 + scenarios/sdsp-f3623/verify_sdsp_metrics.yaml | 2 + .../F1_flow_no_telemetry_with_user_input.yaml | 1 + .../standard-scenarios/F1_happy_path.yaml | 1 + .../F2_contingent_path.yaml | 1 + .../F3_non_conforming_path.yaml | 1 + .../F5_non_conforming_path.yaml | 1 + .../core/execution/conditions.py | 4 +- .../core/execution/definitions.py | 7 ++- .../core/reporting/reporting_models.py | 1 + src/openutm_verification/server/router.py | 2 +- src/openutm_verification/server/runner.py | 58 +++++++++++++---- tests/test_conditions.py | 39 ++++++++++++ tests/test_group_execution.py | 62 +++++++++++++++++++ web-editor/src/components/ScenarioEditor.tsx | 18 ++++++ .../components/ScenarioEditor/CustomNode.tsx | 28 ++++++++- .../ScenarioEditor/PropertiesPanel.tsx | 24 +++++-- .../__tests__/PropertiesPanel.test.tsx | 1 + web-editor/src/styles/Node.module.css | 34 +++++++++- web-editor/src/types/scenario.ts | 2 + .../__tests__/scenarioConversion.test.ts | 55 ++++++++++++++++ web-editor/src/utils/scenarioConversion.ts | 10 +++ 27 files changed, 339 insertions(+), 21 deletions(-) diff --git a/scenarios/flight-declarations/add_flight_declaration.yaml b/scenarios/flight-declarations/add_flight_declaration.yaml index b10b3dca..13ee5681 100644 --- a/scenarios/flight-declarations/add_flight_declaration.yaml +++ b/scenarios/flight-declarations/add_flight_declaration.yaml @@ -14,3 +14,4 @@ steps: arguments: state: ENDED - step: Teardown Flight Declaration + if: always() diff --git a/scenarios/flight-declarations/add_flight_declaration_via_operational_intent.yaml b/scenarios/flight-declarations/add_flight_declaration_via_operational_intent.yaml index f3743f08..5a517092 100644 --- a/scenarios/flight-declarations/add_flight_declaration_via_operational_intent.yaml +++ b/scenarios/flight-declarations/add_flight_declaration_via_operational_intent.yaml @@ -14,3 +14,4 @@ steps: arguments: state: ENDED - step: Teardown Flight Declaration + if: always() diff --git a/scenarios/flight-declarations/bulk_add_flight_declarations.yaml b/scenarios/flight-declarations/bulk_add_flight_declarations.yaml index 73043f22..259f836f 100644 --- a/scenarios/flight-declarations/bulk_add_flight_declarations.yaml +++ b/scenarios/flight-declarations/bulk_add_flight_declarations.yaml @@ -4,9 +4,11 @@ steps: - step: Setup Two Flight Declarations - id: teardown_first_declaration step: Teardown Flight Declaration + if: always() arguments: flight_declaration_id: ${{ steps.Setup Two Flight Declarations.result.declarations[0].id }} - id: teardown_second_declaration step: Teardown Flight Declaration + if: always() arguments: flight_declaration_id: ${{ steps.Setup Two Flight Declarations.result.declarations[1].id }} diff --git a/scenarios/flight-declarations/bulk_add_flight_declarations_via_operational_intents.yaml b/scenarios/flight-declarations/bulk_add_flight_declarations_via_operational_intents.yaml index 08051798..b9384634 100644 --- a/scenarios/flight-declarations/bulk_add_flight_declarations_via_operational_intents.yaml +++ b/scenarios/flight-declarations/bulk_add_flight_declarations_via_operational_intents.yaml @@ -4,9 +4,11 @@ steps: - step: Setup Two Operational Intents - id: teardown_first_declaration step: Teardown Flight Declaration + if: always() arguments: flight_declaration_id: ${{ steps.Setup Two Operational Intents.result.declarations[0].id }} - id: teardown_second_declaration step: Teardown Flight Declaration + if: always() arguments: flight_declaration_id: ${{ steps.Setup Two Operational Intents.result.declarations[1].id }} diff --git a/scenarios/sdsp-f3623/sdsp_heartbeat.yaml b/scenarios/sdsp-f3623/sdsp_heartbeat.yaml index 43028bb4..1c75fe65 100644 --- a/scenarios/sdsp-f3623/sdsp_heartbeat.yaml +++ b/scenarios/sdsp-f3623/sdsp_heartbeat.yaml @@ -29,3 +29,4 @@ steps: arguments: action: STOP surveillance_session_id: ${{ steps.Generate UUID.result }} + if: always() diff --git a/scenarios/sdsp-f3623/sdsp_track.yaml b/scenarios/sdsp-f3623/sdsp_track.yaml index 9e5e69ab..b685fe04 100644 --- a/scenarios/sdsp-f3623/sdsp_track.yaml +++ b/scenarios/sdsp-f3623/sdsp_track.yaml @@ -33,3 +33,4 @@ steps: action: STOP needs: - stream_air_traffic + if: always() diff --git a/scenarios/sdsp-f3623/verify_sdsp_metrics.yaml b/scenarios/sdsp-f3623/verify_sdsp_metrics.yaml index 64de850e..8db200cc 100644 --- a/scenarios/sdsp-f3623/verify_sdsp_metrics.yaml +++ b/scenarios/sdsp-f3623/verify_sdsp_metrics.yaml @@ -27,6 +27,7 @@ steps: - step: Verify Reported Metrics in Flight Blender needs: - stream_air_traffic + on_step_result_fail: continue arguments: observations: ${{ steps.stream_air_traffic.result.observations }} # session_id: ${{ steps.generated_sdsp_session_id.result }} @@ -37,3 +38,4 @@ steps: action: STOP needs: - stream_air_traffic + if: always() diff --git a/scenarios/standard-scenarios/F1_flow_no_telemetry_with_user_input.yaml b/scenarios/standard-scenarios/F1_flow_no_telemetry_with_user_input.yaml index c2b07335..ee9ead76 100644 --- a/scenarios/standard-scenarios/F1_flow_no_telemetry_with_user_input.yaml +++ b/scenarios/standard-scenarios/F1_flow_no_telemetry_with_user_input.yaml @@ -16,3 +16,4 @@ steps: arguments: state: ENDED - step: Teardown Flight Declaration + if: always() diff --git a/scenarios/standard-scenarios/F1_happy_path.yaml b/scenarios/standard-scenarios/F1_happy_path.yaml index 0b60484f..751cfe71 100644 --- a/scenarios/standard-scenarios/F1_happy_path.yaml +++ b/scenarios/standard-scenarios/F1_happy_path.yaml @@ -22,4 +22,5 @@ steps: state: ENDED description: Marks the operation as ended after the flight is complete. - step: Teardown Flight Declaration + if: always() description: Cleans up the flight declaration and any associated resources. diff --git a/scenarios/standard-scenarios/F2_contingent_path.yaml b/scenarios/standard-scenarios/F2_contingent_path.yaml index 503754b3..15596a9d 100644 --- a/scenarios/standard-scenarios/F2_contingent_path.yaml +++ b/scenarios/standard-scenarios/F2_contingent_path.yaml @@ -18,3 +18,4 @@ steps: arguments: state: ENDED - step: Teardown Flight Declaration + if: always() diff --git a/scenarios/standard-scenarios/F3_non_conforming_path.yaml b/scenarios/standard-scenarios/F3_non_conforming_path.yaml index 9b2811a1..4bc6a77b 100644 --- a/scenarios/standard-scenarios/F3_non_conforming_path.yaml +++ b/scenarios/standard-scenarios/F3_non_conforming_path.yaml @@ -29,4 +29,5 @@ steps: arguments: state: ENDED - step: Teardown Flight Declaration + if: always() description: Cleanup. diff --git a/scenarios/standard-scenarios/F5_non_conforming_path.yaml b/scenarios/standard-scenarios/F5_non_conforming_path.yaml index 793cd08e..3cad647e 100644 --- a/scenarios/standard-scenarios/F5_non_conforming_path.yaml +++ b/scenarios/standard-scenarios/F5_non_conforming_path.yaml @@ -21,3 +21,4 @@ steps: arguments: state: ENDED - step: Teardown Flight Declaration + if: always() diff --git a/src/openutm_verification/core/execution/conditions.py b/src/openutm_verification/core/execution/conditions.py index aa9fc2c3..5baf5885 100644 --- a/src/openutm_verification/core/execution/conditions.py +++ b/src/openutm_verification/core/execution/conditions.py @@ -32,9 +32,9 @@ def __init__(self, steps: dict[str, StepResult[Any]], loop_context: dict[str, An self.loop_context = loop_context or {} self.last_step_status: Status | None = None - # Determine last step status (excluding skipped steps) + # Determine last step status (excluding skipped steps and steps with continue_on_fail that failed) if steps: - completed_steps = [s for s in steps.values() if s.status != Status.SKIP] + completed_steps = [s for s in steps.values() if s.status != Status.SKIP and not (s.continue_on_fail and s.status == Status.FAIL)] if completed_steps: self.last_step_status = completed_steps[-1].status diff --git a/src/openutm_verification/core/execution/definitions.py b/src/openutm_verification/core/execution/definitions.py index 7737a3f4..9bd4d478 100644 --- a/src/openutm_verification/core/execution/definitions.py +++ b/src/openutm_verification/core/execution/definitions.py @@ -1,4 +1,4 @@ -from typing import Any, Dict, List +from typing import Any, Dict, List, Literal from pydantic import BaseModel, Field @@ -24,6 +24,11 @@ class StepDefinition(BaseModel): description: str | None = None if_condition: str | None = Field(default=None, alias="if", description="Conditional expression to determine if step should run") loop: LoopConfig | None = Field(default=None, description="Loop configuration for repeating the step") + on_step_result_fail: Literal["stop", "continue"] = Field( + default="stop", + alias="on_step_result_fail", + description="Action when this step fails: 'stop' halts the scenario (default), 'continue' allows subsequent steps to execute", + ) class GroupDefinition(BaseModel): diff --git a/src/openutm_verification/core/reporting/reporting_models.py b/src/openutm_verification/core/reporting/reporting_models.py index 5833ef30..c9a5ac76 100644 --- a/src/openutm_verification/core/reporting/reporting_models.py +++ b/src/openutm_verification/core/reporting/reporting_models.py @@ -44,6 +44,7 @@ class StepResult(BaseModel, Generic[T]): result: T = None # type: ignore error_message: str | None = None logs: list[str] = [] + continue_on_fail: bool = False class ScenarioResult(BaseModel): diff --git a/src/openutm_verification/server/router.py b/src/openutm_verification/server/router.py index df57866a..aef76090 100644 --- a/src/openutm_verification/server/router.py +++ b/src/openutm_verification/server/router.py @@ -105,7 +105,7 @@ async def get_scenario(scenario: str): """Get the content of a specific scenario.""" try: scenario_def = load_yaml_scenario_definition(scenario) - return scenario_def.model_dump() + return scenario_def.model_dump(by_alias=True) except FileNotFoundError: raise HTTPException(status_code=404, detail="Scenario not found") except Exception as e: diff --git a/src/openutm_verification/server/runner.py b/src/openutm_verification/server/runner.py index cc17e5ea..fe66c1e3 100644 --- a/src/openutm_verification/server/runner.py +++ b/src/openutm_verification/server/runner.py @@ -606,10 +606,17 @@ async def _execute_group( # If step failed and it's not allowed to fail, break the group if result.status == Status.FAIL: - logger.error(f"Group step '{group_step.id}' failed, stopping group execution") - # Mark remaining steps as SKIP - self._mark_remaining_group_steps_skipped(group, group_name, loop_context, executed_step_indices) - break + if step.on_step_result_fail == "continue": + logger.warning(f"Group step '{group_step.id}' failed, but on_step_result_fail=continue — continuing group") + result.continue_on_fail = True + if self.session_context: + with self.session_context: + self.session_context.update_result(result) + else: + logger.error(f"Group step '{group_step.id}' failed, stopping group execution") + # Mark remaining steps as SKIP + self._mark_remaining_group_steps_skipped(group, group_name, loop_context, executed_step_indices) + break return results @@ -643,6 +650,15 @@ def _mark_remaining_group_steps_skipped( with self.session_context: self.session_context.update_result(skipped_result) + def _mark_results_continue_on_fail(self, results: List[StepResult]) -> None: + """Mark failed results as continue_on_fail and update them in session context.""" + for result in results: + if result.status == Status.FAIL: + result.continue_on_fail = True + if self.session_context: + with self.session_context: + self.session_context.update_result(result) + async def _wait_for_dependencies(self, step: StepDefinition) -> None: """Wait for any declared dependencies (by step ID) before executing a step.""" if not step.needs: @@ -759,19 +775,29 @@ async def run_scenario(self, scenario: ScenarioDefinition) -> List[StepResult]: # Wait for declared dependencies (useful for background steps) await self._wait_for_dependencies(step) + should_continue_on_fail = step.on_step_result_fail == "continue" + # Check if this step references a group if self._is_group_reference(step.step, scenario): # Handle loop execution for groups if step.loop: loop_results = await self._execute_loop_for_group(step, scenario) if loop_results and any(r.status == Status.FAIL for r in loop_results): - logger.error(f"Loop for group '{step.id}' failed, breaking scenario") - break + if should_continue_on_fail: + logger.warning(f"Loop for group '{step.id}' failed, but on_step_result_fail=continue — continuing") + self._mark_results_continue_on_fail(loop_results) + else: + logger.error(f"Loop for group '{step.id}' failed, breaking scenario") + break else: group_results = await self._execute_group(step, scenario) if any(r.status == Status.FAIL for r in group_results): - logger.error(f"Group '{step.id}' failed, breaking scenario") - break + if should_continue_on_fail: + logger.warning(f"Group '{step.id}' failed, but on_step_result_fail=continue — continuing") + self._mark_results_continue_on_fail(group_results) + else: + logger.error(f"Group '{step.id}' failed, breaking scenario") + break else: # Regular step execution # Handle loop execution @@ -779,10 +805,20 @@ async def run_scenario(self, scenario: ScenarioDefinition) -> List[StepResult]: loop_results = await self._execute_loop(step) # Check if any loop iteration failed if loop_results and any(r.status == Status.FAIL for r in loop_results): - logger.error(f"Loop for step '{step.id}' failed, breaking scenario") - break + if should_continue_on_fail: + logger.warning(f"Loop for step '{step.id}' failed, but on_step_result_fail=continue — continuing") + self._mark_results_continue_on_fail(loop_results) + else: + logger.error(f"Loop for step '{step.id}' failed, breaking scenario") + break else: - await self.execute_single_step(step) + result = await self.execute_single_step(step) + if result.status == Status.FAIL and should_continue_on_fail: + logger.warning(f"Step '{step.id}' failed, but on_step_result_fail=continue — continuing") + result.continue_on_fail = True + if self.session_context: + with self.session_context: + self.session_context.update_result(result) return self.session_context.state.steps async def _execute_loop_for_group(self, step: StepDefinition, scenario: ScenarioDefinition) -> List[StepResult]: diff --git a/tests/test_conditions.py b/tests/test_conditions.py index f4f2203c..e5a0ddea 100644 --- a/tests/test_conditions.py +++ b/tests/test_conditions.py @@ -171,3 +171,42 @@ def test_loop_index_boundary_conditions(self): evaluator = ConditionEvaluator(steps, loop_context) assert evaluator.evaluate("loop.index < 100") is True assert evaluator.evaluate("loop.index >= 99") is True + + def test_success_ignores_continue_on_fail_steps(self): + """Test success() ignores failed steps with continue_on_fail=True.""" + steps = { + "step1": StepResult(id="step1", name="Step 1", status=Status.PASS, duration=1.0), + "step2": StepResult(id="step2", name="Step 2", status=Status.FAIL, duration=1.0, continue_on_fail=True), + } + evaluator = ConditionEvaluator(steps) + # step2 failed but has continue_on_fail, so success() should still be True + assert evaluator.evaluate("success()") is True + + def test_failure_ignores_continue_on_fail_steps(self): + """Test failure() ignores failed steps with continue_on_fail=True.""" + steps = { + "step1": StepResult(id="step1", name="Step 1", status=Status.PASS, duration=1.0), + "step2": StepResult(id="step2", name="Step 2", status=Status.FAIL, duration=1.0, continue_on_fail=True), + } + evaluator = ConditionEvaluator(steps) + # step2 failed but has continue_on_fail, so last_step_status is PASS (from step1) + assert evaluator.evaluate("failure()") is False + + def test_continue_on_fail_step_status_still_queryable(self): + """Test that individual step status is still queryable even with continue_on_fail.""" + steps = { + "step1": StepResult(id="step1", name="Step 1", status=Status.PASS, duration=1.0), + "step2": StepResult(id="step2", name="Step 2", status=Status.FAIL, duration=1.0, continue_on_fail=True), + } + evaluator = ConditionEvaluator(steps) + # Direct step status queries should still reflect the actual status + assert evaluator.evaluate("steps.step2.status == 'failure'") is True + + def test_success_fails_with_non_continue_on_fail_step(self): + """Test success() still returns False for failed steps without continue_on_fail.""" + steps = { + "step1": StepResult(id="step1", name="Step 1", status=Status.PASS, duration=1.0), + "step2": StepResult(id="step2", name="Step 2", status=Status.FAIL, duration=1.0, continue_on_fail=False), + } + evaluator = ConditionEvaluator(steps) + assert evaluator.evaluate("success()") is False diff --git a/tests/test_group_execution.py b/tests/test_group_execution.py index b408d36c..c4ea6f5e 100644 --- a/tests/test_group_execution.py +++ b/tests/test_group_execution.py @@ -181,3 +181,65 @@ async def test_empty_groups_section(): # Verify groups is empty dict assert scenario.groups == {} assert len(scenario.steps) == 1 + + +@pytest.mark.asyncio +async def test_on_step_result_fail_parsing(): + """Test that on_step_result_fail is correctly parsed from YAML.""" + yaml_content = """ +name: test_continue_on_fail +description: Test scenario with on_step_result_fail + +steps: + - step: Setup Flight Declaration + - step: Validate Metrics + on_step_result_fail: continue + - step: Teardown Flight Declaration +""" + + data = yaml.safe_load(yaml_content) + scenario = ScenarioDefinition.model_validate(data) + + assert scenario.steps[0].on_step_result_fail == "stop" # default + assert scenario.steps[1].on_step_result_fail == "continue" + assert scenario.steps[2].on_step_result_fail == "stop" # default + + +@pytest.mark.asyncio +async def test_on_step_result_fail_default(): + """Test that on_step_result_fail defaults to 'stop'.""" + yaml_content = """ +name: test_default_fail_action +description: Test default on_step_result_fail + +steps: + - step: Setup Flight Declaration +""" + + data = yaml.safe_load(yaml_content) + scenario = ScenarioDefinition.model_validate(data) + assert scenario.steps[0].on_step_result_fail == "stop" + + +@pytest.mark.asyncio +async def test_on_step_result_fail_in_group(): + """Test that on_step_result_fail works inside groups.""" + yaml_content = """ +name: test_continue_on_fail_group +description: Test scenario with on_step_result_fail inside a group + +groups: + my_group: + steps: + - step: Validate Metrics + on_step_result_fail: continue + - step: Teardown Flight Declaration + +steps: + - step: my_group +""" + + data = yaml.safe_load(yaml_content) + scenario = ScenarioDefinition.model_validate(data) + assert scenario.groups["my_group"].steps[0].on_step_result_fail == "continue" + assert scenario.groups["my_group"].steps[1].on_step_result_fail == "stop" diff --git a/web-editor/src/components/ScenarioEditor.tsx b/web-editor/src/components/ScenarioEditor.tsx index cae55b9c..7181624e 100644 --- a/web-editor/src/components/ScenarioEditor.tsx +++ b/web-editor/src/components/ScenarioEditor.tsx @@ -955,6 +955,23 @@ const ScenarioEditorContent = () => { }); }, [setNodes]); + const updateNodeOnStepResultFail = useCallback((nodeId: string, value: 'stop' | 'continue') => { + setIsDirty(true); + setNodes((nds) => nds.map((node) => ( + node.id === nodeId + ? { ...node, data: { ...node.data, onStepResultFail: value } } + : node + ))); + + setSelectedNode((prev) => { + if (!prev || prev.id !== nodeId) return prev; + return { + ...prev, + data: { ...prev.data, onStepResultFail: value }, + }; + }); + }, [setNodes]); + const updateNodeNeeds = useCallback((nodeId: string, needs: string[]) => { setIsDirty(true); const cleanedNeeds = needs.filter(Boolean); @@ -1239,6 +1256,7 @@ const ScenarioEditorContent = () => { onUpdateStepId={updateNodeStepId} onUpdateIfCondition={updateNodeIfCondition} onUpdateLoop={updateNodeLoop} + onUpdateOnStepResultFail={updateNodeOnStepResultFail} onUpdateGroupDescription={updateGroupDescription} /> ) : ( diff --git a/web-editor/src/components/ScenarioEditor/CustomNode.tsx b/web-editor/src/components/ScenarioEditor/CustomNode.tsx index ce45e9ac..95473c89 100644 --- a/web-editor/src/components/ScenarioEditor/CustomNode.tsx +++ b/web-editor/src/components/ScenarioEditor/CustomNode.tsx @@ -1,10 +1,18 @@ import { Handle, Position, type NodeProps, type Node } from '@xyflow/react'; -import { Box, CheckCircle, XCircle, AlertTriangle, Loader2, MinusCircle, RotateCw, GitBranch, Timer, Hourglass, Plane } from 'lucide-react'; +import { Box, CheckCircle, XCircle, AlertTriangle, Loader2, MinusCircle, RotateCw, GitBranch, Timer, Hourglass, Plane, ShieldAlert, CircleDot, Infinity as InfinityIcon } from 'lucide-react'; import styles from '../../styles/Node.module.css'; import { getPhaseColor, PHASE_LABELS } from '../../utils/phaseColors'; import type { NodeData } from '../../types/scenario'; +const ConditionIcon = ({ condition }: { condition: string }) => { + const trimmed = condition.trim(); + if (trimmed === 'success()') return ; + if (trimmed === 'failure()') return ; + if (trimmed === 'always()') return ; + return ; +}; + const ModifierBadges = ({ data }: { data: NodeData }) => (
{data.runInBackground && ( @@ -25,6 +33,12 @@ const ModifierBadges = ({ data }: { data: NodeData }) => ( loop
)} + {data.onStepResultFail === 'continue' && ( +
+ + continue +
+ )} ); @@ -73,6 +87,12 @@ export const CustomNode = ({ data, selected }: NodeProps>) => { {data.label} + {data.ifCondition && data.ifCondition.trim() !== '' && ( +
+ + if: {data.ifCondition} +
+ )} ); } @@ -123,6 +143,12 @@ export const CustomNode = ({ data, selected }: NodeProps>) => { )} + {data.ifCondition && data.ifCondition.trim() !== '' && ( +
+ + if: {data.ifCondition} +
+ )} ); diff --git a/web-editor/src/components/ScenarioEditor/PropertiesPanel.tsx b/web-editor/src/components/ScenarioEditor/PropertiesPanel.tsx index 2e532cdf..3c1b341d 100644 --- a/web-editor/src/components/ScenarioEditor/PropertiesPanel.tsx +++ b/web-editor/src/components/ScenarioEditor/PropertiesPanel.tsx @@ -65,6 +65,7 @@ interface PropertiesPanelProps { onUpdateIfCondition: (nodeId: string, condition: string) => void; onUpdateLoop: (nodeId: string, loopConfig: { count?: number; items?: unknown[]; while?: string } | undefined) => void; onUpdateNeeds: (nodeId: string, needs: string[]) => void; + onUpdateOnStepResultFail: (nodeId: string, value: 'stop' | 'continue') => void; onUpdateGroupDescription?: (groupName: string, description: string) => void; } @@ -78,7 +79,7 @@ const parseRefString = (value: unknown) => { }; }; -export const PropertiesPanel = ({ selectedNode, connectedNodes, allNodes, onClose, onUpdateParameter, onUpdateRunInBackground, onUpdateStepId, onUpdateIfCondition, onUpdateLoop, onUpdateNeeds, onUpdateGroupDescription }: PropertiesPanelProps) => { +export const PropertiesPanel = ({ selectedNode, connectedNodes, allNodes, onClose, onUpdateParameter, onUpdateRunInBackground, onUpdateStepId, onUpdateIfCondition, onUpdateLoop, onUpdateNeeds, onUpdateOnStepResultFail, onUpdateGroupDescription }: PropertiesPanelProps) => { const { sidebarWidth: width, isResizing, startResizing } = useSidebarResize(480, 300, 800); // Compute loop type from node data @@ -215,6 +216,20 @@ export const PropertiesPanel = ({ selectedNode, connectedNodes, allNodes, onClos +
+ +
+ If checked, subsequent steps will still execute even if this step fails. +
+
+
@@ -258,11 +273,12 @@ export const PropertiesPanel = ({ selectedNode, connectedNodes, allNodes, onClos className={styles.paramInput} value={selectedNode.data.ifCondition || ''} onChange={(e) => onUpdateIfCondition(selectedNode.id, e.target.value)} - placeholder="e.g., success() or steps.step1.status == 'pass'" + placeholder="success()" />
- Conditional expression (GitHub Actions-style). Examples:
- • success() - run if previous step succeeded
+ Default: success() — steps only run if no previous step failed.
+ GitHub Actions-style expressions:
+ • success() - run if previous step succeeded (default)
• failure() - run if previous step failed
• always() - always run
• steps.step1.status == 'pass' - check specific step diff --git a/web-editor/src/components/ScenarioEditor/__tests__/PropertiesPanel.test.tsx b/web-editor/src/components/ScenarioEditor/__tests__/PropertiesPanel.test.tsx index 0778de7b..fe2e1de5 100644 --- a/web-editor/src/components/ScenarioEditor/__tests__/PropertiesPanel.test.tsx +++ b/web-editor/src/components/ScenarioEditor/__tests__/PropertiesPanel.test.tsx @@ -29,6 +29,7 @@ describe('PropertiesPanel', () => { onUpdateIfCondition: vi.fn(), onUpdateLoop: vi.fn(), onUpdateNeeds: vi.fn(), + onUpdateOnStepResultFail: vi.fn(), }; it('renders correctly', () => { diff --git a/web-editor/src/styles/Node.module.css b/web-editor/src/styles/Node.module.css index 721e2c80..fd5e23c0 100644 --- a/web-editor/src/styles/Node.module.css +++ b/web-editor/src/styles/Node.module.css @@ -120,7 +120,8 @@ .loopBadge, .conditionBadge, -.backgroundBadge { +.backgroundBadge, +.continueOnFailBadge { display: flex; align-items: center; gap: 3px; @@ -161,6 +162,37 @@ background-color: rgba(59, 130, 246, 0.28); } +.continueOnFailBadge { + background-color: rgba(245, 158, 11, 0.2); + color: #f59e0b; + border: 1px solid rgba(245, 158, 11, 0.4); +} + +.continueOnFailBadge:hover { + background-color: rgba(245, 158, 11, 0.3); +} + +.conditionText { + display: flex; + align-items: center; + gap: 4px; + font-size: 11px; + color: #22c55e; + padding: 2px 4px; + margin-top: -4px; + margin-bottom: 2px; + overflow: hidden; + text-overflow: ellipsis; + white-space: nowrap; + max-width: 100%; +} + +.conditionText span { + overflow: hidden; + text-overflow: ellipsis; + white-space: nowrap; +} + /* Group container label */ .groupContainerLabel { position: absolute; diff --git a/web-editor/src/types/scenario.ts b/web-editor/src/types/scenario.ts index 5f7b83f8..df58cf8c 100644 --- a/web-editor/src/types/scenario.ts +++ b/web-editor/src/types/scenario.ts @@ -28,6 +28,7 @@ export interface NodeData extends Record { ifCondition?: string; loop?: LoopConfig; needs?: string[]; + onStepResultFail?: 'stop' | 'continue'; isGroupReference?: boolean; isGroupContainer?: boolean; isPhaseContainer?: boolean; @@ -61,6 +62,7 @@ export interface ScenarioStep { description?: string; if?: string; loop?: LoopConfig; + on_step_result_fail?: 'stop' | 'continue'; } export interface ScenarioDefinition { diff --git a/web-editor/src/utils/__tests__/scenarioConversion.test.ts b/web-editor/src/utils/__tests__/scenarioConversion.test.ts index 441780a3..697c1a07 100644 --- a/web-editor/src/utils/__tests__/scenarioConversion.test.ts +++ b/web-editor/src/utils/__tests__/scenarioConversion.test.ts @@ -145,3 +145,58 @@ describe('YAML Conversion - Reference Normalization', () => { expect(scenario.groups?.my_group?.steps?.[1]?.arguments?.data).toBe('${{ group.fetch.result }}'); }); }); + +describe('YAML Conversion - on_step_result_fail', () => { + it('includes on_step_result_fail=continue in YAML output', () => { + const node: Node = { + id: 'node1', + position: { x: 0, y: 0 }, + data: { + label: 'Validate Metrics', + stepId: 'validate', + operationId: 'validate_op', + parameters: [], + onStepResultFail: 'continue' + } + }; + + const scenario = convertGraphToYaml([node], [], [], 'test', 'test'); + const step = scenario.steps[0]; + expect(step.on_step_result_fail).toBe('continue'); + }); + + it('omits on_step_result_fail when set to stop (default)', () => { + const node: Node = { + id: 'node1', + position: { x: 0, y: 0 }, + data: { + label: 'Setup', + stepId: 'setup', + operationId: 'setup_op', + parameters: [], + onStepResultFail: 'stop' + } + }; + + const scenario = convertGraphToYaml([node], [], [], 'test', 'test'); + const step = scenario.steps[0]; + expect(step.on_step_result_fail).toBeUndefined(); + }); + + it('omits on_step_result_fail when not set', () => { + const node: Node = { + id: 'node1', + position: { x: 0, y: 0 }, + data: { + label: 'Setup', + stepId: 'setup', + operationId: 'setup_op', + parameters: [] + } + }; + + const scenario = convertGraphToYaml([node], [], [], 'test', 'test'); + const step = scenario.steps[0]; + expect(step.on_step_result_fail).toBeUndefined(); + }); +}); diff --git a/web-editor/src/utils/scenarioConversion.ts b/web-editor/src/utils/scenarioConversion.ts index 68655e2e..82395328 100644 --- a/web-editor/src/utils/scenarioConversion.ts +++ b/web-editor/src/utils/scenarioConversion.ts @@ -79,6 +79,7 @@ export const convertYamlToGraph = ( runInBackground: step.background, ifCondition: step.if, loop: step.loop, + onStepResultFail: step.on_step_result_fail, isGroupReference: true, isGroupContainer: true }, @@ -187,6 +188,7 @@ export const convertYamlToGraph = ( ifCondition: step.if, loop: step.loop, needs: needsList, + onStepResultFail: step.on_step_result_fail, isGroupReference: false } }; @@ -313,6 +315,10 @@ export const convertGraphToYaml = ( step.loop = node.data.loop; } + if (node.data.onStepResultFail === 'continue') { + step.on_step_result_fail = 'continue'; + } + return step; } @@ -390,6 +396,10 @@ export const convertGraphToYaml = ( step.loop = node.data.loop; } + if (node.data.onStepResultFail === 'continue') { + step.on_step_result_fail = 'continue'; + } + // Dependencies (needs) come from dotted edges into this node const deps = dependencyEdges .filter(e => e.target === node.id) From bd2a8dd350f77d7baa641e8f7a1a1dfe97ba5301 Mon Sep 17 00:00:00 2001 From: atti92 Date: Tue, 31 Mar 2026 00:20:29 +0200 Subject: [PATCH 2/5] resolve comments --- examples/conditional_execution.md | 40 ++++++++++++ examples/workflow_features.md | 11 ++++ scenarios/README.md | 34 ++++++++++ scenarios/sdsp-f3623/verify_sdsp_metrics.yaml | 2 +- .../core/execution/conditions.py | 4 +- .../core/execution/definitions.py | 10 +-- .../core/reporting/reporting_models.py | 2 +- src/openutm_verification/server/router.py | 2 +- src/openutm_verification/server/runner.py | 38 ++++++------ tests/test_conditions.py | 28 ++++----- tests/test_group_execution.py | 40 ++++++------ web-editor/README_GROUPS.md | 2 +- web-editor/src/components/ScenarioEditor.tsx | 8 +-- .../components/ScenarioEditor/CustomNode.tsx | 23 +++++-- .../ScenarioEditor/PropertiesPanel.tsx | 62 +++++++++++++------ .../__tests__/PropertiesPanel.test.tsx | 2 +- web-editor/src/styles/Node.module.css | 6 +- web-editor/src/types/scenario.ts | 4 +- .../__tests__/scenarioConversion.test.ts | 18 +++--- web-editor/src/utils/scenarioConversion.ts | 12 ++-- 20 files changed, 235 insertions(+), 113 deletions(-) diff --git a/examples/conditional_execution.md b/examples/conditional_execution.md index d58eaf68..5fa05644 100644 --- a/examples/conditional_execution.md +++ b/examples/conditional_execution.md @@ -119,6 +119,46 @@ Access the return value/result data from a specific step. Steps may return data **Note**: The `result` field accesses the return value/details from the step execution. Use `!= None` to check if a step returned any data. +## Continue on Error + +By default, when a step fails the scenario stops. Use `continue-on-error: true` to allow subsequent steps to execute even if the current step fails: + +```yaml +- id: validate_metrics + step: Verify Reported Metrics + continue-on-error: true # Scenario continues even if validation fails +``` + +This is different from `if: always()` — `continue-on-error` controls what happens **after** the step runs and fails, while `if` controls whether the step runs **at all**. + +### Combining with Conditions + +`continue-on-error` and `if` can be used together for robust workflows: + +```yaml +steps: + - id: run_tests + step: Run Tests + continue-on-error: true # Don't halt if tests fail + + - id: collect_results + step: Collect Test Results + description: Runs even if tests failed above + + - id: cleanup + step: Cleanup Resources + if: always() # Runs regardless of any previous failure +``` + +### When to Use Each + +| Scenario | Use | +|----------|-----| +| Step might fail but rest of workflow should continue | `continue-on-error: true` | +| Step should only run when a previous step failed | `if: failure()` | +| Step must always run (cleanup, teardown) | `if: always()` | +| Step should run only on success (default behavior) | No `if` needed, or `if: success()` | + ## Complete Example ```yaml diff --git a/examples/workflow_features.md b/examples/workflow_features.md index 3ae0e6dc..1e2a05d7 100644 --- a/examples/workflow_features.md +++ b/examples/workflow_features.md @@ -21,6 +21,16 @@ Complete guide to advanced workflow features including conditional execution and **Operators:** `==`, `!=`, `&&`, `||`, `not` +### Continue on Error + +```yaml +- id: step_name + step: Operation Name + continue-on-error: true # Scenario continues even if this step fails +``` + +By default, a failing step halts the scenario. Use `continue-on-error: true` to allow subsequent steps to execute regardless. This is useful for non-critical checks or when cleanup steps must always run. + ### Loops ```yaml @@ -46,6 +56,7 @@ Complete guide to advanced workflow features including conditional execution and | **Item Loop** | `loop: { items: [...] }` | Process list of files | | **While Loop** | `loop: { while: condition }` | Poll until ready | | **Conditional Step** | `if: expression` | Run only on success | +| **Continue on Error** | `continue-on-error: true` | Don't halt on failure | | **Step Status** | `steps.X.status` | Check if step passed | | **Step Result** | `steps.X.result` | Access return value | | **Loop Index** | `loop.index` | Get iteration number | diff --git a/scenarios/README.md b/scenarios/README.md index db3a03f9..c2a2247e 100644 --- a/scenarios/README.md +++ b/scenarios/README.md @@ -125,6 +125,40 @@ steps: - Available status strings: `success`, `failure`, `running`, `skipped`. - You can use conditions like `always()`, `success()`, `failure()` and combined references (e.g., `steps.Upload Flight Declaration.status == 'success'`). +### Run Conditions (`if`) + +Control when a step executes using the `if` field: + +```yaml +steps: + - step: Cleanup Resources + if: always() # Always runs, even after failures + + - step: Deploy + if: success() # Runs only if all previous steps succeeded + + - step: Rollback + if: failure() # Runs only if a previous step failed + + - step: Custom Check + if: steps.test.status == 'success' # Check a specific step +``` + +### Continue on Error (`continue-on-error`) + +By default, a failing step halts the scenario. Set `continue-on-error: true` to allow subsequent steps to execute even if this step fails: + +```yaml +steps: + - step: Run Flaky Test + continue-on-error: true # Scenario continues even if this fails + + - step: Cleanup Resources + if: always() +``` + +This is useful for non-critical validation steps or cleanup flows where you want the scenario to keep going. + ## Validation To validate scenarios locally: diff --git a/scenarios/sdsp-f3623/verify_sdsp_metrics.yaml b/scenarios/sdsp-f3623/verify_sdsp_metrics.yaml index 8db200cc..eeeb096a 100644 --- a/scenarios/sdsp-f3623/verify_sdsp_metrics.yaml +++ b/scenarios/sdsp-f3623/verify_sdsp_metrics.yaml @@ -27,7 +27,7 @@ steps: - step: Verify Reported Metrics in Flight Blender needs: - stream_air_traffic - on_step_result_fail: continue + continue-on-error: true arguments: observations: ${{ steps.stream_air_traffic.result.observations }} # session_id: ${{ steps.generated_sdsp_session_id.result }} diff --git a/src/openutm_verification/core/execution/conditions.py b/src/openutm_verification/core/execution/conditions.py index 5baf5885..348038c2 100644 --- a/src/openutm_verification/core/execution/conditions.py +++ b/src/openutm_verification/core/execution/conditions.py @@ -32,9 +32,9 @@ def __init__(self, steps: dict[str, StepResult[Any]], loop_context: dict[str, An self.loop_context = loop_context or {} self.last_step_status: Status | None = None - # Determine last step status (excluding skipped steps and steps with continue_on_fail that failed) + # Determine last step status (excluding skipped steps and steps with continue_on_error that failed) if steps: - completed_steps = [s for s in steps.values() if s.status != Status.SKIP and not (s.continue_on_fail and s.status == Status.FAIL)] + completed_steps = [s for s in steps.values() if s.status != Status.SKIP and not (s.continue_on_error and s.status == Status.FAIL)] if completed_steps: self.last_step_status = completed_steps[-1].status diff --git a/src/openutm_verification/core/execution/definitions.py b/src/openutm_verification/core/execution/definitions.py index 9bd4d478..12deb228 100644 --- a/src/openutm_verification/core/execution/definitions.py +++ b/src/openutm_verification/core/execution/definitions.py @@ -1,4 +1,4 @@ -from typing import Any, Dict, List, Literal +from typing import Any, Dict, List from pydantic import BaseModel, Field @@ -24,10 +24,10 @@ class StepDefinition(BaseModel): description: str | None = None if_condition: str | None = Field(default=None, alias="if", description="Conditional expression to determine if step should run") loop: LoopConfig | None = Field(default=None, description="Loop configuration for repeating the step") - on_step_result_fail: Literal["stop", "continue"] = Field( - default="stop", - alias="on_step_result_fail", - description="Action when this step fails: 'stop' halts the scenario (default), 'continue' allows subsequent steps to execute", + continue_on_error: bool = Field( + default=False, + alias="continue-on-error", + description="If true, subsequent steps will still execute even if this step fails. Default is false (stop on failure).", ) diff --git a/src/openutm_verification/core/reporting/reporting_models.py b/src/openutm_verification/core/reporting/reporting_models.py index c9a5ac76..7563e860 100644 --- a/src/openutm_verification/core/reporting/reporting_models.py +++ b/src/openutm_verification/core/reporting/reporting_models.py @@ -44,7 +44,7 @@ class StepResult(BaseModel, Generic[T]): result: T = None # type: ignore error_message: str | None = None logs: list[str] = [] - continue_on_fail: bool = False + continue_on_error: bool = False class ScenarioResult(BaseModel): diff --git a/src/openutm_verification/server/router.py b/src/openutm_verification/server/router.py index aef76090..6674eca7 100644 --- a/src/openutm_verification/server/router.py +++ b/src/openutm_verification/server/router.py @@ -131,7 +131,7 @@ async def save_scenario(name: str, scenario: ScenarioDefinition): try: # Convert Pydantic model to dict, excluding None values to keep YAML clean - data = scenario.model_dump(exclude_none=True, exclude_defaults=True) + data = scenario.model_dump(by_alias=True, exclude_none=True, exclude_defaults=True) with open(file_path, "w") as f: yaml.dump(data, f, sort_keys=False, default_flow_style=False) diff --git a/src/openutm_verification/server/runner.py b/src/openutm_verification/server/runner.py index fe66c1e3..14b225fd 100644 --- a/src/openutm_verification/server/runner.py +++ b/src/openutm_verification/server/runner.py @@ -606,9 +606,9 @@ async def _execute_group( # If step failed and it's not allowed to fail, break the group if result.status == Status.FAIL: - if step.on_step_result_fail == "continue": - logger.warning(f"Group step '{group_step.id}' failed, but on_step_result_fail=continue — continuing group") - result.continue_on_fail = True + if group_step.continue_on_error: + logger.warning(f"Group step '{group_step.id}' failed, but continue-on-error=true — continuing group") + result.continue_on_error = True if self.session_context: with self.session_context: self.session_context.update_result(result) @@ -650,11 +650,11 @@ def _mark_remaining_group_steps_skipped( with self.session_context: self.session_context.update_result(skipped_result) - def _mark_results_continue_on_fail(self, results: List[StepResult]) -> None: - """Mark failed results as continue_on_fail and update them in session context.""" + def _mark_results_continue_on_error(self, results: List[StepResult]) -> None: + """Mark failed results as continue_on_error and update them in session context.""" for result in results: if result.status == Status.FAIL: - result.continue_on_fail = True + result.continue_on_error = True if self.session_context: with self.session_context: self.session_context.update_result(result) @@ -775,7 +775,7 @@ async def run_scenario(self, scenario: ScenarioDefinition) -> List[StepResult]: # Wait for declared dependencies (useful for background steps) await self._wait_for_dependencies(step) - should_continue_on_fail = step.on_step_result_fail == "continue" + should_continue_on_error = step.continue_on_error # Check if this step references a group if self._is_group_reference(step.step, scenario): @@ -783,18 +783,18 @@ async def run_scenario(self, scenario: ScenarioDefinition) -> List[StepResult]: if step.loop: loop_results = await self._execute_loop_for_group(step, scenario) if loop_results and any(r.status == Status.FAIL for r in loop_results): - if should_continue_on_fail: - logger.warning(f"Loop for group '{step.id}' failed, but on_step_result_fail=continue — continuing") - self._mark_results_continue_on_fail(loop_results) + if should_continue_on_error: + logger.warning(f"Loop for group '{step.id}' failed, but continue-on-error=true — continuing") + self._mark_results_continue_on_error(loop_results) else: logger.error(f"Loop for group '{step.id}' failed, breaking scenario") break else: group_results = await self._execute_group(step, scenario) if any(r.status == Status.FAIL for r in group_results): - if should_continue_on_fail: - logger.warning(f"Group '{step.id}' failed, but on_step_result_fail=continue — continuing") - self._mark_results_continue_on_fail(group_results) + if should_continue_on_error: + logger.warning(f"Group '{step.id}' failed, but continue-on-error=true — continuing") + self._mark_results_continue_on_error(group_results) else: logger.error(f"Group '{step.id}' failed, breaking scenario") break @@ -805,17 +805,17 @@ async def run_scenario(self, scenario: ScenarioDefinition) -> List[StepResult]: loop_results = await self._execute_loop(step) # Check if any loop iteration failed if loop_results and any(r.status == Status.FAIL for r in loop_results): - if should_continue_on_fail: - logger.warning(f"Loop for step '{step.id}' failed, but on_step_result_fail=continue — continuing") - self._mark_results_continue_on_fail(loop_results) + if should_continue_on_error: + logger.warning(f"Loop for step '{step.id}' failed, but continue-on-error=true — continuing") + self._mark_results_continue_on_error(loop_results) else: logger.error(f"Loop for step '{step.id}' failed, breaking scenario") break else: result = await self.execute_single_step(step) - if result.status == Status.FAIL and should_continue_on_fail: - logger.warning(f"Step '{step.id}' failed, but on_step_result_fail=continue — continuing") - result.continue_on_fail = True + if result.status == Status.FAIL and should_continue_on_error: + logger.warning(f"Step '{step.id}' failed, but continue-on-error=true — continuing") + result.continue_on_error = True if self.session_context: with self.session_context: self.session_context.update_result(result) diff --git a/tests/test_conditions.py b/tests/test_conditions.py index e5a0ddea..1dcf91b1 100644 --- a/tests/test_conditions.py +++ b/tests/test_conditions.py @@ -172,41 +172,41 @@ def test_loop_index_boundary_conditions(self): assert evaluator.evaluate("loop.index < 100") is True assert evaluator.evaluate("loop.index >= 99") is True - def test_success_ignores_continue_on_fail_steps(self): - """Test success() ignores failed steps with continue_on_fail=True.""" + def test_success_ignores_continue_on_error_steps(self): + """Test success() ignores failed steps with continue_on_error=True.""" steps = { "step1": StepResult(id="step1", name="Step 1", status=Status.PASS, duration=1.0), - "step2": StepResult(id="step2", name="Step 2", status=Status.FAIL, duration=1.0, continue_on_fail=True), + "step2": StepResult(id="step2", name="Step 2", status=Status.FAIL, duration=1.0, continue_on_error=True), } evaluator = ConditionEvaluator(steps) - # step2 failed but has continue_on_fail, so success() should still be True + # step2 failed but has continue_on_error, so success() should still be True assert evaluator.evaluate("success()") is True - def test_failure_ignores_continue_on_fail_steps(self): - """Test failure() ignores failed steps with continue_on_fail=True.""" + def test_failure_ignores_continue_on_error_steps(self): + """Test failure() ignores failed steps with continue_on_error=True.""" steps = { "step1": StepResult(id="step1", name="Step 1", status=Status.PASS, duration=1.0), - "step2": StepResult(id="step2", name="Step 2", status=Status.FAIL, duration=1.0, continue_on_fail=True), + "step2": StepResult(id="step2", name="Step 2", status=Status.FAIL, duration=1.0, continue_on_error=True), } evaluator = ConditionEvaluator(steps) - # step2 failed but has continue_on_fail, so last_step_status is PASS (from step1) + # step2 failed but has continue_on_error, so last_step_status is PASS (from step1) assert evaluator.evaluate("failure()") is False - def test_continue_on_fail_step_status_still_queryable(self): - """Test that individual step status is still queryable even with continue_on_fail.""" + def test_continue_on_error_step_status_still_queryable(self): + """Test that individual step status is still queryable even with continue_on_error.""" steps = { "step1": StepResult(id="step1", name="Step 1", status=Status.PASS, duration=1.0), - "step2": StepResult(id="step2", name="Step 2", status=Status.FAIL, duration=1.0, continue_on_fail=True), + "step2": StepResult(id="step2", name="Step 2", status=Status.FAIL, duration=1.0, continue_on_error=True), } evaluator = ConditionEvaluator(steps) # Direct step status queries should still reflect the actual status assert evaluator.evaluate("steps.step2.status == 'failure'") is True - def test_success_fails_with_non_continue_on_fail_step(self): - """Test success() still returns False for failed steps without continue_on_fail.""" + def test_success_fails_with_non_continue_on_error_step(self): + """Test success() still returns False for failed steps without continue_on_error.""" steps = { "step1": StepResult(id="step1", name="Step 1", status=Status.PASS, duration=1.0), - "step2": StepResult(id="step2", name="Step 2", status=Status.FAIL, duration=1.0, continue_on_fail=False), + "step2": StepResult(id="step2", name="Step 2", status=Status.FAIL, duration=1.0, continue_on_error=False), } evaluator = ConditionEvaluator(steps) assert evaluator.evaluate("success()") is False diff --git a/tests/test_group_execution.py b/tests/test_group_execution.py index c4ea6f5e..2536601d 100644 --- a/tests/test_group_execution.py +++ b/tests/test_group_execution.py @@ -184,33 +184,33 @@ async def test_empty_groups_section(): @pytest.mark.asyncio -async def test_on_step_result_fail_parsing(): - """Test that on_step_result_fail is correctly parsed from YAML.""" +async def test_continue_on_error_parsing(): + """Test that continue-on-error is correctly parsed from YAML.""" yaml_content = """ -name: test_continue_on_fail -description: Test scenario with on_step_result_fail +name: test_continue_on_error +description: Test scenario with continue-on-error steps: - step: Setup Flight Declaration - step: Validate Metrics - on_step_result_fail: continue + continue-on-error: true - step: Teardown Flight Declaration """ data = yaml.safe_load(yaml_content) scenario = ScenarioDefinition.model_validate(data) - assert scenario.steps[0].on_step_result_fail == "stop" # default - assert scenario.steps[1].on_step_result_fail == "continue" - assert scenario.steps[2].on_step_result_fail == "stop" # default + assert scenario.steps[0].continue_on_error is False # default + assert scenario.steps[1].continue_on_error is True + assert scenario.steps[2].continue_on_error is False # default @pytest.mark.asyncio -async def test_on_step_result_fail_default(): - """Test that on_step_result_fail defaults to 'stop'.""" +async def test_continue_on_error_default(): + """Test that continue_on_error defaults to False.""" yaml_content = """ -name: test_default_fail_action -description: Test default on_step_result_fail +name: test_default_error_action +description: Test default continue-on-error steps: - step: Setup Flight Declaration @@ -218,21 +218,21 @@ async def test_on_step_result_fail_default(): data = yaml.safe_load(yaml_content) scenario = ScenarioDefinition.model_validate(data) - assert scenario.steps[0].on_step_result_fail == "stop" + assert scenario.steps[0].continue_on_error is False @pytest.mark.asyncio -async def test_on_step_result_fail_in_group(): - """Test that on_step_result_fail works inside groups.""" +async def test_continue_on_error_in_group(): + """Test that continue-on-error works inside groups.""" yaml_content = """ -name: test_continue_on_fail_group -description: Test scenario with on_step_result_fail inside a group +name: test_continue_on_error_group +description: Test scenario with continue-on-error inside a group groups: my_group: steps: - step: Validate Metrics - on_step_result_fail: continue + continue-on-error: true - step: Teardown Flight Declaration steps: @@ -241,5 +241,5 @@ async def test_on_step_result_fail_in_group(): data = yaml.safe_load(yaml_content) scenario = ScenarioDefinition.model_validate(data) - assert scenario.groups["my_group"].steps[0].on_step_result_fail == "continue" - assert scenario.groups["my_group"].steps[1].on_step_result_fail == "stop" + assert scenario.groups["my_group"].steps[0].continue_on_error is True + assert scenario.groups["my_group"].steps[1].continue_on_error is False diff --git a/web-editor/README_GROUPS.md b/web-editor/README_GROUPS.md index e29836da..f05f2c7f 100644 --- a/web-editor/README_GROUPS.md +++ b/web-editor/README_GROUPS.md @@ -43,7 +43,7 @@ In the main scenario flow: - Fixed count: `count: 5` - Iterate items: `items: [...]` - While condition: `while: loop.index < 10` -- **Conditional groups**: Groups can have conditions: `if: ${{ always() }}` +- **Conditional groups**: Groups can have conditions: `if: always()` ### 5. UI Indicators diff --git a/web-editor/src/components/ScenarioEditor.tsx b/web-editor/src/components/ScenarioEditor.tsx index 7181624e..5a3c9811 100644 --- a/web-editor/src/components/ScenarioEditor.tsx +++ b/web-editor/src/components/ScenarioEditor.tsx @@ -955,11 +955,11 @@ const ScenarioEditorContent = () => { }); }, [setNodes]); - const updateNodeOnStepResultFail = useCallback((nodeId: string, value: 'stop' | 'continue') => { + const updateNodeContinueOnError = useCallback((nodeId: string, value: boolean) => { setIsDirty(true); setNodes((nds) => nds.map((node) => ( node.id === nodeId - ? { ...node, data: { ...node.data, onStepResultFail: value } } + ? { ...node, data: { ...node.data, continueOnError: value } } : node ))); @@ -967,7 +967,7 @@ const ScenarioEditorContent = () => { if (!prev || prev.id !== nodeId) return prev; return { ...prev, - data: { ...prev.data, onStepResultFail: value }, + data: { ...prev.data, continueOnError: value }, }; }); }, [setNodes]); @@ -1256,7 +1256,7 @@ const ScenarioEditorContent = () => { onUpdateStepId={updateNodeStepId} onUpdateIfCondition={updateNodeIfCondition} onUpdateLoop={updateNodeLoop} - onUpdateOnStepResultFail={updateNodeOnStepResultFail} + onUpdateContinueOnError={updateNodeContinueOnError} onUpdateGroupDescription={updateGroupDescription} /> ) : ( diff --git a/web-editor/src/components/ScenarioEditor/CustomNode.tsx b/web-editor/src/components/ScenarioEditor/CustomNode.tsx index 95473c89..54182fff 100644 --- a/web-editor/src/components/ScenarioEditor/CustomNode.tsx +++ b/web-editor/src/components/ScenarioEditor/CustomNode.tsx @@ -13,6 +13,17 @@ const ConditionIcon = ({ condition }: { condition: string }) => { return ; }; +const CONDITION_LABELS: Record = { + 'success()': 'Runs on success', + 'failure()': 'Runs on failure', + 'always()': 'Always runs', +}; + +const getConditionLabel = (condition: string): string => { + const trimmed = condition.trim(); + return CONDITION_LABELS[trimmed] || trimmed; +}; + const ModifierBadges = ({ data }: { data: NodeData }) => (
{data.runInBackground && ( @@ -33,8 +44,8 @@ const ModifierBadges = ({ data }: { data: NodeData }) => ( loop
)} - {data.onStepResultFail === 'continue' && ( -
+ {data.continueOnError && ( +
continue
@@ -87,10 +98,10 @@ export const CustomNode = ({ data, selected }: NodeProps>) => { {data.label}
- {data.ifCondition && data.ifCondition.trim() !== '' && ( + {data.ifCondition && data.ifCondition.trim() !== '' && data.ifCondition.trim() !== 'success()' && (
- if: {data.ifCondition} + {getConditionLabel(data.ifCondition)}
)}
@@ -143,10 +154,10 @@ export const CustomNode = ({ data, selected }: NodeProps>) => {
)}
- {data.ifCondition && data.ifCondition.trim() !== '' && ( + {data.ifCondition && data.ifCondition.trim() !== '' && data.ifCondition.trim() !== 'success()' && (
- if: {data.ifCondition} + {getConditionLabel(data.ifCondition)}
)} diff --git a/web-editor/src/components/ScenarioEditor/PropertiesPanel.tsx b/web-editor/src/components/ScenarioEditor/PropertiesPanel.tsx index 3c1b341d..44d124cd 100644 --- a/web-editor/src/components/ScenarioEditor/PropertiesPanel.tsx +++ b/web-editor/src/components/ScenarioEditor/PropertiesPanel.tsx @@ -65,7 +65,7 @@ interface PropertiesPanelProps { onUpdateIfCondition: (nodeId: string, condition: string) => void; onUpdateLoop: (nodeId: string, loopConfig: { count?: number; items?: unknown[]; while?: string } | undefined) => void; onUpdateNeeds: (nodeId: string, needs: string[]) => void; - onUpdateOnStepResultFail: (nodeId: string, value: 'stop' | 'continue') => void; + onUpdateContinueOnError: (nodeId: string, value: boolean) => void; onUpdateGroupDescription?: (groupName: string, description: string) => void; } @@ -79,7 +79,7 @@ const parseRefString = (value: unknown) => { }; }; -export const PropertiesPanel = ({ selectedNode, connectedNodes, allNodes, onClose, onUpdateParameter, onUpdateRunInBackground, onUpdateStepId, onUpdateIfCondition, onUpdateLoop, onUpdateNeeds, onUpdateOnStepResultFail, onUpdateGroupDescription }: PropertiesPanelProps) => { +export const PropertiesPanel = ({ selectedNode, connectedNodes, allNodes, onClose, onUpdateParameter, onUpdateRunInBackground, onUpdateStepId, onUpdateIfCondition, onUpdateLoop, onUpdateNeeds, onUpdateContinueOnError, onUpdateGroupDescription }: PropertiesPanelProps) => { const { sidebarWidth: width, isResizing, startResizing } = useSidebarResize(480, 300, 800); // Compute loop type from node data @@ -93,14 +93,17 @@ export const PropertiesPanel = ({ selectedNode, connectedNodes, allNodes, onClos // Local state for JSON items textarea to allow invalid JSON while typing const [jsonItemsText, setJsonItemsText] = useState(null); + // Local state to track when "Custom expression…" is explicitly selected + const [isCustomCondition, setIsCustomCondition] = useState(false); const [prevNodeId, setPrevNodeId] = useState(selectedNode.id); const [prevItems, setPrevItems] = useState(selectedNode.data.loop?.items); - // Sync local JSON text when node data changes externally + // Sync local state when node selection changes if (selectedNode.id !== prevNodeId || selectedNode.data.loop?.items !== prevItems) { setPrevNodeId(selectedNode.id); setPrevItems(selectedNode.data.loop?.items); setJsonItemsText(null); + setIsCustomCondition(false); } const getItemsText = () => { @@ -220,8 +223,8 @@ export const PropertiesPanel = ({ selectedNode, connectedNodes, allNodes, onClos @@ -267,21 +270,44 @@ export const PropertiesPanel = ({ selectedNode, connectedNodes, allNodes, onClos
- - Run Condition + + {(isCustomCondition || (selectedNode.data.ifCondition && !['', 'success()', 'failure()', 'always()'].includes(selectedNode.data.ifCondition.trim()))) && ( + onUpdateIfCondition(selectedNode.id, e.target.value)} + placeholder="steps.step1.status == 'success'" + /> + )}
- Default: success() — steps only run if no previous step failed.
- GitHub Actions-style expressions:
- • success() - run if previous step succeeded (default)
- • failure() - run if previous step failed
- • always() - always run
- • steps.step1.status == 'pass' - check specific step + Controls when this step executes. Steps marked "Continue on Error" are excluded from success/failure evaluation. Custom expressions support status checks like steps.step_id.status == 'success'
diff --git a/web-editor/src/components/ScenarioEditor/__tests__/PropertiesPanel.test.tsx b/web-editor/src/components/ScenarioEditor/__tests__/PropertiesPanel.test.tsx index fe2e1de5..e369b698 100644 --- a/web-editor/src/components/ScenarioEditor/__tests__/PropertiesPanel.test.tsx +++ b/web-editor/src/components/ScenarioEditor/__tests__/PropertiesPanel.test.tsx @@ -29,7 +29,7 @@ describe('PropertiesPanel', () => { onUpdateIfCondition: vi.fn(), onUpdateLoop: vi.fn(), onUpdateNeeds: vi.fn(), - onUpdateOnStepResultFail: vi.fn(), + onUpdateContinueOnError: vi.fn(), }; it('renders correctly', () => { diff --git a/web-editor/src/styles/Node.module.css b/web-editor/src/styles/Node.module.css index fd5e23c0..1857ee4b 100644 --- a/web-editor/src/styles/Node.module.css +++ b/web-editor/src/styles/Node.module.css @@ -121,7 +121,7 @@ .loopBadge, .conditionBadge, .backgroundBadge, -.continueOnFailBadge { +.continueOnErrorBadge { display: flex; align-items: center; gap: 3px; @@ -162,13 +162,13 @@ background-color: rgba(59, 130, 246, 0.28); } -.continueOnFailBadge { +.continueOnErrorBadge { background-color: rgba(245, 158, 11, 0.2); color: #f59e0b; border: 1px solid rgba(245, 158, 11, 0.4); } -.continueOnFailBadge:hover { +.continueOnErrorBadge:hover { background-color: rgba(245, 158, 11, 0.3); } diff --git a/web-editor/src/types/scenario.ts b/web-editor/src/types/scenario.ts index df58cf8c..27af42e1 100644 --- a/web-editor/src/types/scenario.ts +++ b/web-editor/src/types/scenario.ts @@ -28,7 +28,7 @@ export interface NodeData extends Record { ifCondition?: string; loop?: LoopConfig; needs?: string[]; - onStepResultFail?: 'stop' | 'continue'; + continueOnError?: boolean; isGroupReference?: boolean; isGroupContainer?: boolean; isPhaseContainer?: boolean; @@ -62,7 +62,7 @@ export interface ScenarioStep { description?: string; if?: string; loop?: LoopConfig; - on_step_result_fail?: 'stop' | 'continue'; + 'continue-on-error'?: boolean; } export interface ScenarioDefinition { diff --git a/web-editor/src/utils/__tests__/scenarioConversion.test.ts b/web-editor/src/utils/__tests__/scenarioConversion.test.ts index 697c1a07..22a23acc 100644 --- a/web-editor/src/utils/__tests__/scenarioConversion.test.ts +++ b/web-editor/src/utils/__tests__/scenarioConversion.test.ts @@ -146,8 +146,8 @@ describe('YAML Conversion - Reference Normalization', () => { }); }); -describe('YAML Conversion - on_step_result_fail', () => { - it('includes on_step_result_fail=continue in YAML output', () => { +describe('YAML Conversion - continue-on-error', () => { + it('includes continue-on-error=true in YAML output', () => { const node: Node = { id: 'node1', position: { x: 0, y: 0 }, @@ -156,16 +156,16 @@ describe('YAML Conversion - on_step_result_fail', () => { stepId: 'validate', operationId: 'validate_op', parameters: [], - onStepResultFail: 'continue' + continueOnError: true } }; const scenario = convertGraphToYaml([node], [], [], 'test', 'test'); const step = scenario.steps[0]; - expect(step.on_step_result_fail).toBe('continue'); + expect(step['continue-on-error']).toBe(true); }); - it('omits on_step_result_fail when set to stop (default)', () => { + it('omits continue-on-error when set to false (default)', () => { const node: Node = { id: 'node1', position: { x: 0, y: 0 }, @@ -174,16 +174,16 @@ describe('YAML Conversion - on_step_result_fail', () => { stepId: 'setup', operationId: 'setup_op', parameters: [], - onStepResultFail: 'stop' + continueOnError: false } }; const scenario = convertGraphToYaml([node], [], [], 'test', 'test'); const step = scenario.steps[0]; - expect(step.on_step_result_fail).toBeUndefined(); + expect(step['continue-on-error']).toBeUndefined(); }); - it('omits on_step_result_fail when not set', () => { + it('omits continue-on-error when not set', () => { const node: Node = { id: 'node1', position: { x: 0, y: 0 }, @@ -197,6 +197,6 @@ describe('YAML Conversion - on_step_result_fail', () => { const scenario = convertGraphToYaml([node], [], [], 'test', 'test'); const step = scenario.steps[0]; - expect(step.on_step_result_fail).toBeUndefined(); + expect(step['continue-on-error']).toBeUndefined(); }); }); diff --git a/web-editor/src/utils/scenarioConversion.ts b/web-editor/src/utils/scenarioConversion.ts index 82395328..42bc4394 100644 --- a/web-editor/src/utils/scenarioConversion.ts +++ b/web-editor/src/utils/scenarioConversion.ts @@ -79,7 +79,7 @@ export const convertYamlToGraph = ( runInBackground: step.background, ifCondition: step.if, loop: step.loop, - onStepResultFail: step.on_step_result_fail, + continueOnError: step['continue-on-error'], isGroupReference: true, isGroupContainer: true }, @@ -188,7 +188,7 @@ export const convertYamlToGraph = ( ifCondition: step.if, loop: step.loop, needs: needsList, - onStepResultFail: step.on_step_result_fail, + continueOnError: step['continue-on-error'], isGroupReference: false } }; @@ -315,8 +315,8 @@ export const convertGraphToYaml = ( step.loop = node.data.loop; } - if (node.data.onStepResultFail === 'continue') { - step.on_step_result_fail = 'continue'; + if (node.data.continueOnError) { + step['continue-on-error'] = true; } return step; @@ -396,8 +396,8 @@ export const convertGraphToYaml = ( step.loop = node.data.loop; } - if (node.data.onStepResultFail === 'continue') { - step.on_step_result_fail = 'continue'; + if (node.data.continueOnError) { + step['continue-on-error'] = true; } // Dependencies (needs) come from dotted edges into this node From ff51208aa92bc2fbbee21578624d0ea501714094 Mon Sep 17 00:00:00 2001 From: atti92 Date: Fri, 10 Apr 2026 21:05:45 +0200 Subject: [PATCH 3/5] resolve comments --- tests/test_continue_on_error_integration.py | 227 ++++++++++++++++++ web-editor/src/components/ScenarioEditor.tsx | 36 ++- .../ScenarioEditor/PropertiesPanel.tsx | 12 +- web-editor/src/utils/scenarioConversion.ts | 3 +- 4 files changed, 264 insertions(+), 14 deletions(-) create mode 100644 tests/test_continue_on_error_integration.py diff --git a/tests/test_continue_on_error_integration.py b/tests/test_continue_on_error_integration.py new file mode 100644 index 00000000..cb5b907b --- /dev/null +++ b/tests/test_continue_on_error_integration.py @@ -0,0 +1,227 @@ +"""Integration tests for continue-on-error runner behavior. + +Validates that the SessionManager.run_scenario properly continues +execution after failures when continue-on-error is set, and that +the failed StepResult is marked continue_on_error=True. +""" + +from unittest.mock import AsyncMock, patch + +import pytest + +from openutm_verification.core.execution.definitions import ( + GroupDefinition, + ScenarioDefinition, + StepDefinition, +) +from openutm_verification.core.execution.scenario_runner import ScenarioContext +from openutm_verification.core.reporting.reporting_models import Status, StepResult +from openutm_verification.server.runner import SessionManager + + +def _make_runner() -> SessionManager: + """Create a minimal SessionManager with mocked internals.""" + SessionManager._instance = None + runner = SessionManager.__new__(SessionManager) + runner._initialized = True + runner.config_path = None + runner.config = type("C", (), { + "suites": {}, + "reporting": type("R", (), {"output_dir": "/tmp/reports", "formats": []})(), + })() + runner.client_map = {} + runner.session_stack = None + runner.session_resolver = None + runner.session_context = None + runner.session_tasks = {} + runner.current_start_time = None + runner.current_output_dir = None + runner.current_timestamp_str = None + runner.current_run_error = None + runner.data_files = None + runner._stop_event = None + return runner + + +def _setup_session(runner: SessionManager) -> None: + """Prepare a fresh ScenarioContext on the runner.""" + ctx = ScenarioContext() + ctx.__enter__() + runner.session_context = ctx + + +@pytest.fixture(autouse=True) +def _reset_singleton(): + """Ensure SessionManager singleton is reset between tests.""" + SessionManager._instance = None + yield + SessionManager._instance = None + + +# ── Regular step tests ───────────────────────────────────────────── + + +@pytest.mark.asyncio +async def test_continue_on_error_continues_after_failure(): + """Scenario continues executing steps after a continue-on-error failure.""" + runner = _make_runner() + _setup_session(runner) + + scenario = ScenarioDefinition( + name="test", + steps=[ + StepDefinition(step="step_a", id="step_a", **{"continue-on-error": True}), + StepDefinition(step="step_b", id="step_b"), + ], + ) + + call_order = [] + + async def fake_execute(step, loop_context=None): + call_order.append(step.id) + if step.id == "step_a": + result = StepResult(id=step.id, name=step.step, status=Status.FAIL, duration=0.1, error_message="boom") + else: + result = StepResult(id=step.id, name=step.step, status=Status.PASS, duration=0.1) + with runner.session_context: + runner.session_context.update_result(result) + return result + + with patch.object(runner, "execute_single_step", side_effect=fake_execute): + with patch.object(runner, "initialize_session", new_callable=AsyncMock): + with patch.object(runner, "_wait_for_dependencies", new_callable=AsyncMock): + results = await runner.run_scenario(scenario) + + assert call_order == ["step_a", "step_b"], "step_b should have executed after step_a failed with continue-on-error" + + step_a_result = next(r for r in results if r.id == "step_a") + assert step_a_result.status == Status.FAIL + assert step_a_result.continue_on_error is True + + step_b_result = next(r for r in results if r.id == "step_b") + assert step_b_result.status == Status.PASS + + +@pytest.mark.asyncio +async def test_without_continue_on_error_stops_after_failure(): + """Scenario stops executing steps after a failure without continue-on-error.""" + runner = _make_runner() + _setup_session(runner) + + scenario = ScenarioDefinition( + name="test", + steps=[ + StepDefinition(step="step_a", id="step_a"), + StepDefinition(step="step_b", id="step_b"), + ], + ) + + call_order = [] + + async def fake_execute(step, loop_context=None): + call_order.append(step.id) + if step.id == "step_a": + result = StepResult(id=step.id, name=step.step, status=Status.FAIL, duration=0.1, error_message="boom") + else: + result = StepResult(id=step.id, name=step.step, status=Status.PASS, duration=0.1) + with runner.session_context: + runner.session_context.update_result(result) + return result + + with patch.object(runner, "execute_single_step", side_effect=fake_execute): + with patch.object(runner, "initialize_session", new_callable=AsyncMock): + with patch.object(runner, "_wait_for_dependencies", new_callable=AsyncMock): + results = await runner.run_scenario(scenario) + + assert call_order == ["step_a"], "step_b should NOT have executed" + + +# ── Group step tests ─────────────────────────────────────────────── + + +@pytest.mark.asyncio +async def test_continue_on_error_inside_group(): + """A group step with continue-on-error allows subsequent group steps to run.""" + runner = _make_runner() + _setup_session(runner) + + scenario = ScenarioDefinition( + name="test", + groups={ + "my_group": GroupDefinition( + steps=[ + StepDefinition(step="g_step1", id="g_step1", **{"continue-on-error": True}), + StepDefinition(step="g_step2", id="g_step2"), + ], + ), + }, + steps=[ + StepDefinition(step="my_group", id="run_group"), + ], + ) + + call_order = [] + + async def fake_execute(step, loop_context=None): + call_order.append(step.id) + if step.step == "g_step1": + result = StepResult(id=step.id, name=step.step, status=Status.FAIL, duration=0.1, error_message="boom") + else: + result = StepResult(id=step.id, name=step.step, status=Status.PASS, duration=0.1) + with runner.session_context: + runner.session_context.update_result(result) + return result + + with patch.object(runner, "execute_single_step", side_effect=fake_execute): + with patch.object(runner, "initialize_session", new_callable=AsyncMock): + with patch.object(runner, "_wait_for_dependencies", new_callable=AsyncMock): + results = await runner.run_scenario(scenario) + + assert "g_step1" in call_order + assert "g_step2" in call_order, "g_step2 should execute because g_step1 has continue-on-error" + + g1 = next(r for r in results if "g_step1" in (r.id or "")) + assert g1.status == Status.FAIL + assert g1.continue_on_error is True + + +@pytest.mark.asyncio +async def test_group_stops_without_continue_on_error(): + """A group stops at a failed step when continue-on-error is not set.""" + runner = _make_runner() + _setup_session(runner) + + scenario = ScenarioDefinition( + name="test", + groups={ + "my_group": GroupDefinition( + steps=[ + StepDefinition(step="g_step1", id="g_step1"), + StepDefinition(step="g_step2", id="g_step2"), + ], + ), + }, + steps=[ + StepDefinition(step="my_group", id="run_group"), + ], + ) + + call_order = [] + + async def fake_execute(step, loop_context=None): + call_order.append(step.id) + if step.step == "g_step1": + result = StepResult(id=step.id, name=step.step, status=Status.FAIL, duration=0.1, error_message="boom") + else: + result = StepResult(id=step.id, name=step.step, status=Status.PASS, duration=0.1) + with runner.session_context: + runner.session_context.update_result(result) + return result + + with patch.object(runner, "execute_single_step", side_effect=fake_execute): + with patch.object(runner, "initialize_session", new_callable=AsyncMock): + with patch.object(runner, "_wait_for_dependencies", new_callable=AsyncMock): + results = await runner.run_scenario(scenario) + + assert "g_step1" in call_order + assert "g_step2" not in call_order, "g_step2 should NOT execute after g_step1 failure" diff --git a/web-editor/src/components/ScenarioEditor.tsx b/web-editor/src/components/ScenarioEditor.tsx index 5a3c9811..2cd46e6c 100644 --- a/web-editor/src/components/ScenarioEditor.tsx +++ b/web-editor/src/components/ScenarioEditor.tsx @@ -957,11 +957,37 @@ const ScenarioEditorContent = () => { const updateNodeContinueOnError = useCallback((nodeId: string, value: boolean) => { setIsDirty(true); - setNodes((nds) => nds.map((node) => ( - node.id === nodeId - ? { ...node, data: { ...node.data, continueOnError: value } } - : node - ))); + setNodes((nds) => { + const targetNode = nds.find(n => n.id === nodeId); + // If this is a group child node, sync back to the groups state + if (targetNode?.parentId) { + const parentNode = nds.find(n => n.id === targetNode.parentId); + if (parentNode?.data.isGroupContainer) { + const groupName = parentNode.data.label.replace(/^📦\s*/, '').trim(); + const stepId = targetNode.data.stepId || targetNode.data.label; + setCurrentScenarioGroups(prev => { + const group = prev[groupName]; + if (!group) return prev; + return { + ...prev, + [groupName]: { + ...group, + steps: group.steps.map(s => + (s.id || s.step) === stepId + ? { ...s, 'continue-on-error': value || undefined } + : s + ), + }, + }; + }); + } + } + return nds.map((node) => ( + node.id === nodeId + ? { ...node, data: { ...node.data, continueOnError: value } } + : node + )); + }); setSelectedNode((prev) => { if (!prev || prev.id !== nodeId) return prev; diff --git a/web-editor/src/components/ScenarioEditor/PropertiesPanel.tsx b/web-editor/src/components/ScenarioEditor/PropertiesPanel.tsx index 44d124cd..2fa3e54c 100644 --- a/web-editor/src/components/ScenarioEditor/PropertiesPanel.tsx +++ b/web-editor/src/components/ScenarioEditor/PropertiesPanel.tsx @@ -1,4 +1,4 @@ -import React, { useState, useMemo } from 'react'; +import React, { useState, useMemo, useEffect } from 'react'; import { X, Link as LinkIcon, Unlink, Plane } from 'lucide-react'; import type { Node } from '@xyflow/react'; import layoutStyles from '../../styles/EditorLayout.module.css'; @@ -95,16 +95,12 @@ export const PropertiesPanel = ({ selectedNode, connectedNodes, allNodes, onClos const [jsonItemsText, setJsonItemsText] = useState(null); // Local state to track when "Custom expression…" is explicitly selected const [isCustomCondition, setIsCustomCondition] = useState(false); - const [prevNodeId, setPrevNodeId] = useState(selectedNode.id); - const [prevItems, setPrevItems] = useState(selectedNode.data.loop?.items); - // Sync local state when node selection changes - if (selectedNode.id !== prevNodeId || selectedNode.data.loop?.items !== prevItems) { - setPrevNodeId(selectedNode.id); - setPrevItems(selectedNode.data.loop?.items); + // Sync local state when node selection or loop items change + useEffect(() => { setJsonItemsText(null); setIsCustomCondition(false); - } + }, [selectedNode.id, selectedNode.data.loop?.items]); const getItemsText = () => { if (jsonItemsText !== null) return jsonItemsText; diff --git a/web-editor/src/utils/scenarioConversion.ts b/web-editor/src/utils/scenarioConversion.ts index 42bc4394..bfb3d86f 100644 --- a/web-editor/src/utils/scenarioConversion.ts +++ b/web-editor/src/utils/scenarioConversion.ts @@ -125,7 +125,8 @@ export const convertYamlToGraph = ( operationId: groupOperation?.id, description: groupStep.description || groupOperation?.description || '', phase: groupOperation?.phase, - parameters: groupParameters + parameters: groupParameters, + continueOnError: groupStep['continue-on-error'], } }; From 5fca634de60a5a4e9dcfbb4ea5c971a975e4cb7a Mon Sep 17 00:00:00 2001 From: atti92 Date: Fri, 10 Apr 2026 21:05:46 +0200 Subject: [PATCH 4/5] update --- tests/test_continue_on_error_integration.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/tests/test_continue_on_error_integration.py b/tests/test_continue_on_error_integration.py index cb5b907b..8447c4e0 100644 --- a/tests/test_continue_on_error_integration.py +++ b/tests/test_continue_on_error_integration.py @@ -25,10 +25,14 @@ def _make_runner() -> SessionManager: runner = SessionManager.__new__(SessionManager) runner._initialized = True runner.config_path = None - runner.config = type("C", (), { - "suites": {}, - "reporting": type("R", (), {"output_dir": "/tmp/reports", "formats": []})(), - })() + runner.config = type( + "C", + (), + { + "suites": {}, + "reporting": type("R", (), {"output_dir": "/tmp/reports", "formats": []})(), + }, + )() runner.client_map = {} runner.session_stack = None runner.session_resolver = None From 1cc1abf0c51f255e26e61eae1291381cc75c86a8 Mon Sep 17 00:00:00 2001 From: Attila Kobor Date: Sat, 25 Apr 2026 19:57:31 +0200 Subject: [PATCH 5/5] fix unit test --- src/openutm_verification/server/runner.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/openutm_verification/server/runner.py b/src/openutm_verification/server/runner.py index c1b5de04..2d131752 100644 --- a/src/openutm_verification/server/runner.py +++ b/src/openutm_verification/server/runner.py @@ -839,7 +839,6 @@ async def run_scenario(self, scenario: ScenarioDefinition) -> List[StepResult]: if self.session_context: with self.session_context: self.session_context.update_result(result) - await self.execute_single_step(step) # An empty scenario (or one whose every step was skipped before any # `with self.session_context:` block ran) leaves session_context.state # unset; return an empty list rather than crashing.