diff --git a/README.rst b/README.rst index 5e154b4..5f1e6ca 100644 --- a/README.rst +++ b/README.rst @@ -262,6 +262,14 @@ during their execution too. These can be obtained as follows: Changelog --------- +Release 0.13.1 +~~~~~~~~~~~~~~ + +`26th May, 2026` + +* Fix run loop hanging when Nextflow exits with a config parse error (#14). + + Release 0.13.0 ~~~~~~~~~~~~~~ diff --git a/nextflow/command.py b/nextflow/command.py index 64071ad..c20cef2 100644 --- a/nextflow/command.py +++ b/nextflow/command.py @@ -94,7 +94,7 @@ def _run( params=params ) - execution, log_start = None, 0 + execution, log_start, rc_seen = None, 0, False while True: time.sleep(sleep) execution, diff = get_execution( @@ -102,9 +102,11 @@ def _run( ) log_start += diff if execution and poll: yield execution - if execution and execution.return_code and execution.finished: + if execution and execution.return_code and (execution.finished or rc_seen): if not poll: yield execution break + if execution and execution.return_code: + rc_seen = True def submit_execution( @@ -207,9 +209,10 @@ def make_nextflow_command(run_path, output_path, log_path, pipeline_path, resume profiles = make_nextflow_command_profiles_string(profiles) reports = make_reports_string(output_path, report, timeline, dag, trace) command = f"{env}{nf} {log}{configs}run {pipeline_path} {resume}{params} {profiles} {reports}" + prefix = (str(output_path) + os.path.sep) if output_path != run_path else "" + command = f":>{prefix}rc.txt; {command}" abspath = io.abspath if io else os.path.abspath if run_path != abspath("."): command = f"cd {run_path}; {command}" - prefix = (str(output_path) + os.path.sep) if output_path != run_path else "" command = command.rstrip() + f" >{prefix}" command += f"stdout.txt 2>{prefix}" command += f"stderr.txt; echo $? >{prefix}rc.txt" diff --git a/setup.py b/setup.py index aef2e35..1e13828 100644 --- a/setup.py +++ b/setup.py @@ -5,13 +5,13 @@ setup( name="nextflowpy", - version="0.13.0", + version="0.13.1", description="A Python wrapper around Nextflow.", long_description=long_description, long_description_content_type="text/x-rst", url="https://github.com/goodwright/nextflow.py", - author="Sam Ireland", - author_email="sam@goodwright.com", + author="Goodwright Ltd", + author_email="engineering@flow.bio", license="GPLv3+", classifiers=[ "Development Status :: 4 - Beta", diff --git a/tests/unit/test_command.py b/tests/unit/test_command.py index 21613fc..8f81008 100644 --- a/tests/unit/test_command.py +++ b/tests/unit/test_command.py @@ -43,13 +43,26 @@ def test_can_run_with_custom_values(self, mock_ex, mock_sleep, mock_submit): self.assertEqual(executions, [mock_executions[1]]) + @patch("nextflow.command.submit_execution") + @patch("time.sleep") + @patch("nextflow.command.get_execution") + def test_loop_terminates_when_return_code_set_but_finished_never_arrives(self, mock_ex, mock_sleep, mock_submit): + submission = Mock() + mock_submit.return_value = submission + execution = Mock(return_code="1", finished=None) + mock_ex.side_effect = [(execution, 100), (execution, 0)] + executions = list(_run("main.nf")) + self.assertEqual(executions, [execution]) + self.assertEqual(mock_ex.call_count, 2) + + @patch("nextflow.command.submit_execution") @patch("time.sleep") @patch("nextflow.command.get_execution") def test_can_run_and_poll(self, mock_ex, mock_sleep, mock_submit): submission = Mock() mock_submit.return_value = submission - mock_executions = [Mock(finished=False), Mock(finished=True)] + mock_executions = [Mock(return_code=""), Mock(return_code="0")] mock_ex.side_effect = [[None, 20], [mock_executions[0], 40], [mock_executions[1], 20]] executions = list(_run("main.nf", poll=True, output_path="/out")) mock_sleep.assert_called_with(1) @@ -181,7 +194,7 @@ def test_can_get_full_nextflow_command(self, mock_report, mock_prof, mock_params mock_params.assert_called_with({"param": "2"}) mock_prof.assert_called_with(["docker"]) mock_report.assert_called_with("/out", "report.html", "time.html", "dag.html", "trace.html") - self.assertEqual(command, "cd /exdir; A=B C=D nextflow -Duser.country=US -log '.nextflow.log' -c conf1 -c conf2 run main.nf -resume X --p1=10 --p2=20 -profile docker,test --dag.html >/out/stdout.txt 2>/out/stderr.txt; echo $? >/out/rc.txt") + self.assertEqual(command, "cd /exdir; :>/out/rc.txt; A=B C=D nextflow -Duser.country=US -log '.nextflow.log' -c conf1 -c conf2 run main.nf -resume X --p1=10 --p2=20 -profile docker,test --dag.html >/out/stdout.txt 2>/out/stderr.txt; echo $? >/out/rc.txt") @patch("nextflow.command.make_nextflow_command_env_string") @@ -208,7 +221,7 @@ def test_can_get_minimal_nextflow_command(self, mock_abspath, mock_report, mock_ mock_params.assert_called_with({"param": "2"}) mock_prof.assert_called_with(["docker"]) mock_report.assert_called_with("/exdir", None, None, None, None) - self.assertEqual(command, "nextflow -Duser.country=US run main.nf >stdout.txt 2>stderr.txt; echo $? >rc.txt") + self.assertEqual(command, ":>rc.txt; nextflow -Duser.country=US run main.nf >stdout.txt 2>stderr.txt; echo $? >rc.txt") @patch("nextflow.command.make_nextflow_command_env_string") @@ -235,7 +248,29 @@ def test_can_use_custom_io(self, mock_report, mock_prof, mock_params, mock_resum mock_params.assert_called_with({"param": "2"}) mock_prof.assert_called_with(["docker"]) mock_report.assert_called_with("/exdir", None, None, None, None) - self.assertEqual(command, "nextflow -Duser.country=US run main.nf >stdout.txt 2>stderr.txt; echo $? >rc.txt") + self.assertEqual(command, ":>rc.txt; nextflow -Duser.country=US run main.nf >stdout.txt 2>stderr.txt; echo $? >rc.txt") + + + @patch("nextflow.command.make_nextflow_command_env_string") + @patch("nextflow.command.make_nextflow_command_log_string") + @patch("nextflow.command.make_nextflow_command_config_string") + @patch("nextflow.command.make_nextflow_command_resume_string") + @patch("nextflow.command.make_nextflow_command_params_string") + @patch("nextflow.command.make_nextflow_command_profiles_string") + @patch("nextflow.command.make_reports_string") + def test_command_truncates_stale_rc_before_running(self, mock_report, mock_prof, mock_params, mock_resume, mock_conf, mock_log, mock_env): + mock_env.return_value = "" + mock_log.return_value = "" + mock_conf.return_value = "" + mock_resume.return_value = "" + mock_params.return_value = "" + mock_prof.return_value = "" + mock_report.return_value = "" + io = Mock() + io.abspath.return_value = "/exdir" + command = make_nextflow_command("/exdir", "/out", "/log", "main.nf", False, None, None, None, None, None, None, None, None, None, None, io) + self.assertIn(":>/out/rc.txt;", command) + self.assertLess(command.index(":>/out/rc.txt;"), command.index("nextflow"))