From 4f1e72ec483990e86f0172fefe6af51a8ba58057 Mon Sep 17 00:00:00 2001 From: andrew771027 Date: Wed, 16 Sep 2026 12:58:04 +0000 Subject: [PATCH 1/9] add runner v1.6.1 source code & test code (partial) --- main.py | 68 ++++++- runner/executor.py | 145 ++++++++------ runner/process.py | 120 ++++++++++++ .../test_integration_processs.py | 181 ++++++++++++++++++ tests/test_unit/test_process.py | 79 ++++++++ 5 files changed, 531 insertions(+), 62 deletions(-) create mode 100644 runner/process.py create mode 100644 tests/test_integration/test_integration_processs.py create mode 100644 tests/test_unit/test_process.py diff --git a/main.py b/main.py index e3a3244..758cf8a 100644 --- a/main.py +++ b/main.py @@ -1,5 +1,6 @@ import argparse import sys +import signal from pathlib import Path from runner.artifact import ArtifactManager @@ -9,6 +10,8 @@ from runner.failure import FailureClassifier from runner.reporter import JsonReporter from runner.runner import DeviceTestRunner +from runner.cancellation import CancellationToken +from runner.process import ProcessTerminator PROJECT_ROOT = Path(__file__).resolve().parent @@ -20,15 +23,52 @@ def parse_args() -> argparse.Namespace: return args +class SignalCancellationHandler: + def __init__(self, token: CancellationToken): + self.token = token + self._signal_count = 0 + + def handle_sigint(self, signum, frame): + self._signal_count += 1 + + if self._signal_count == 1: + print("\nCancellaiton requested. " + "Cleaning up..." + ) + + self.token.cancel() + + return + + # + # Optional: + # second Ctrl + C = immediate interruption + # + print( + "\nForce exit requested." + ) + + raise KeyboardInterrupt + def main(): args = parse_args() config = ConfigLoader().load(args.config) + failure_classifier = FailureClassifier() + + cancellation_token = CancellationToken() + + signal_handler = SignalCancellationHandler(cancellation_token) + + old_sigint_handler = signal_handler.signal(signal.SIGINT, signal_handler.handle_sigint) + + process_terminator = ProcessTerminator(grace_period_seconds=2.0) + runner = DeviceTestRunner( executor=SubprocessExecutor( - project_directory=PROJECT_ROOT, failure_classifier=failure_classifier + project_directory=PROJECT_ROOT, failure_classifier=failure_classifier, process_terminator=process_terminator ), artifact_manager=ArtifactManager(output_dir=config.artifact.output_dir), artifact_validator=ArtifactValidator(), @@ -36,10 +76,30 @@ def main(): reporter=JsonReporter(), ) - result = runner.run(config=config) + try: + + result = runner.run(config=config, cancellation_token=cancellation_token) + + except KeyboardInterrupt: + + # + # Second Ctrl + C + # + return 130 + + finally: + signal.signal(signal.SIGINT, old_sigint_handler) + + print(f"Status: {result.summary.status}") + + if result.summary.status == "CANCELLED": + return 130 + + if result.summary.status == "FAILED": + return 1 - print(result.summary.status) + return 0 if __name__ == "__main__": - main() + sys.exit(main()) diff --git a/runner/executor.py b/runner/executor.py index 56e48cf..d6f0d27 100644 --- a/runner/executor.py +++ b/runner/executor.py @@ -4,13 +4,13 @@ import threading import time from pathlib import Path -from typing import TextIO +from typing import Callable, TextIO from runner.artifact import StepLogWriter from runner.cancellation import CancellationToken from runner.failure import FailureClassifier from runner.models import FailureType, LifecycleStepContent, StepAttemptResult - +from runner.process import ProcessTerminator class SubprocessExecutor: POLL_INTERVAL_SECONDS = 0.1 @@ -19,9 +19,11 @@ def __init__( self, project_directory: str | Path, failure_classifier: FailureClassifier, + process_terminator: ProcessTerminator, ): self.project_directory = Path(project_directory).resolve() self.failure_classifier = failure_classifier + self.process_terminator = process_terminator def execute( self, @@ -46,13 +48,21 @@ def execute( process: subprocess.Popen[str] | None = None - error_message: str | None = None + stdout_thread: threading.Thread | None = None + stderr_thread: threading.Thread | None = None timed_out: bool = False - cancelled: bool = False + error_message: str | None = None + try: + # + # ---------------------------------------------- + # Start process group + # ---------------------------------------------- + # + process = subprocess.Popen( step.command, shell=True, @@ -64,11 +74,23 @@ def execute( encoding="utf-8", errors="replace", bufsize=1, + + # + # Important: + # this attempt gets its own session / + # process group + start_new_session=True ) if process.stdout is None or process.stderr is None: raise RuntimeError("Unable to open subprocess streams.") + # + # -------------------------------------------- + # stdout reader + # -------------------------------------------- + # + stdout_thread = threading.Thread( target=self._consume_stream, args=(process.stdout, log_writer.write_stdout), @@ -76,6 +98,12 @@ def execute( daemon=True, ) + # + # -------------------------------------------- + # stderr reader + # -------------------------------------------- + # + stderr_thread = threading.Thread( target=self._consume_stream, args=(process.stderr, log_writer.write_stderr), @@ -86,13 +114,22 @@ def execute( stdout_thread.start() stderr_thread.start() + # + # -------------------------------------------- + # Process monitoring loop + # -------------------------------------------- + # + while True: + # # 1. process 已經正常結束 + # if process.poll() is not None: break - + # # 2. 外部要求取消 + # if cancellation_token.is_cancelled: cancelled = True @@ -102,7 +139,9 @@ def execute( break + # # 3. timeout + # elapsed_seconds = time.perf_counter() - start_time if elapsed_seconds >= step.timeout_second: @@ -110,19 +149,39 @@ def execute( error_message = f"Command timeout after {step.timeout_second}" - self._stop_process(process) + self.process_terminator.terminate_process_group(process) break time.sleep(self.POLL_INTERVAL_SECONDS) - stdout_thread.join() - stderr_thread.join() + # + # ----------------------------------------- + # Make sure direct child has been reaped + # ----------------------------------------- + # - exit_code = process.returncode + if process.poll() is None: + process.wait() + + # + # ---------------------------------------- + # Drain stdout/stderr + # ---------------------------------------- + # + + self._join_reader_thread(stdout_thread) + + self._join_reader_thread(stderr_thread) duration_seconds = time.perf_counter() - start_time + # + # --------------------------------------- + # Classifiaction + # --------------------------------------- + # + if cancelled: success = False @@ -131,7 +190,7 @@ def execute( else: - success = not timed_out and exit_code == 0 + success = not timed_out and process.returncode == 0 failure_type = self.failure_classifier.classify_process_failure( process_success=success, @@ -146,66 +205,37 @@ def execute( failure_type=failure_type, timed_out=timed_out, cancelled=cancelled, - exit_code=exit_code, + exit_code=process.returncode, duration_seconds=duration_seconds, stdout=log_writer.stdout, stderr=log_writer.stderr, stdout_log_path=str(log_writer.stdout_path), stderr_log_path=str(log_writer.stderr_path), error=error_message, + artifact_validation_results=[], ) - except OSError as error: + except (OSError, RuntimeError) as error: duration_seconds = time.perf_counter() - start_time error_message = f"Unable to execute command: {error}" log_writer.write_stderr(f"{error_message}\n") - failure_type = self.failure_classifier.classify_process_failure( - process_success=False, - timed_out=False, - stderr=log_writer.stderr, - error=error_message, - ) - - if process is not None: - self._stop_process(process) - - return StepAttemptResult( - attempt=attempt, - success=False, - failure_type=failure_type, - timed_out=False, - cancelled=False, - exit_code=process.returncode if process is not None else None, - duration_seconds=duration_seconds, - stdout=log_writer.stdout, - stderr=log_writer.stderr, - stdout_log_path=str(log_writer.stdout_path), - stderr_log_path=str(log_writer.stderr_path), - error=error_message, - ) - - except RuntimeError as error: - duration_seconds = time.perf_counter() - start_time - error_message = str(error) + # + # Exception 發生時也不能留下 process + # - log_writer.write_stderr(f"{error_message}\n") + if process is not None and process.poll() is None: + self.process_terminator.terminate_process_group(process) - failure_type = self.failure_classifier.classify_process_failure( - process_success=False, - timed_out=False, - stderr=log_writer.stderr, - error=error_message, - ) + self._join_reader_thread(stdout_thread) - if process is not None: - self._stop_process(process) + self._join_reader_thread(stderr_thread) return StepAttemptResult( attempt=attempt, success=False, - failure_type=failure_type, + failure_type=FailureType.PROCESS_ERROR, timed_out=False, cancelled=False, exit_code=process.returncode if process is not None else None, @@ -215,8 +245,10 @@ def execute( stdout_log_path=str(log_writer.stdout_path), stderr_log_path=str(log_writer.stderr_path), error=error_message, + artifact_validation_results=[], ) + @staticmethod def _consume_stream(stream: TextIO, write_line) -> None: try: @@ -226,17 +258,14 @@ def _consume_stream(stream: TextIO, write_line) -> None: stream.close() @staticmethod - def _stop_process(process: subprocess.Popen[str]) -> None: - if process.poll() is not None: + def _join_reader_thread(thread: threading.Thread | None) -> None: + if thread is None: return - process.terminate() + thread.join(timeout=2) - try: - process.wait(timeout=2) - except subprocess.TimeoutExpired: - process.kill() - process.wait() + if thread.is_alive(): + raise RuntimeError("Output reader thread did not stop.") @staticmethod def _create_default_log_writer(stage: str, step_name: str) -> StepLogWriter: diff --git a/runner/process.py b/runner/process.py new file mode 100644 index 0000000..e22d7a2 --- /dev/null +++ b/runner/process.py @@ -0,0 +1,120 @@ +import os +import signal +import subprocess +import time +from dataclasses import dataclass + +@dataclass(frozen=True) +class ProcessTerminationResult: + terminated: bool + killed: bool + return_code: int | None + +class ProcessTerminator: + + def __init__(self, grace_period_seconds: float = 2.0, poll_interval_seconds: float = 0.05): + if grace_period_seconds < 0: + raise ValueError("grace_period_seconds must be >= 0") + + if poll_interval_seconds <= 0: + raise ValueError("poll_internal_seconds must be > 0") + + self.grace_period_seconds = grace_period_seconds + self.poll_internal_seconds = poll_interval_seconds + + def terminate_process_group(self, process: subprocess.Popen) -> ProcessTerminationResult: + + # + # Process 已經結束 + # + if process.poll() is not None: + return ProcessTerminationResult( + terminated=False, + killed=False, + return_code=process.returncode, + ) + + try: + + process_group_id = os.getpgid(process.pid) + + except ProcessLookupError: + return ProcessTerminationResult( + terminated=False, + killed=False, + return_code=process.poll(), + ) + + # + # -------------------------------------------------- + # Phase 1 + # Graceful termination + # -------------------------------------------------- + # + try: + + os.killpg(process_group_id, signal.SIGTERM) + + except ProcessLookupError: + pass + + # + # 給 process group 一段時間自己 cleanup + # + exited = self._wait_for_exit(process=process, timeout_seconds=self.grace_period_seconds) + + if exited: + return ProcessTerminationResult( + terminated=True, + killed=False, + return_code=process.returncode, + ) + + # + # ------------------------------------------------ + # Phase 3 + # Force kill + # ------------------------------------------------ + # + try: + + os.killpg(process_group_id, signal.SIGKILL) + + except ProcessLookupError: + pass + + # + # Reap direct child + # + try: + + process.wait(timeout=2) + + except subprocess.TimeoutExpired: + # + # SIGKILL 理論上應該會很快結束 + # 這裡再 poll，而不是無限 wait + # + pass + + return ProcessTerminationResult( + terminated=True, + killed=True, + return_code=process.poll() + ) + + def _wait_for_exit(self, process: subprocess.Popen, timeout_seconds: float) -> bool: + + deadline = time.monotonic() + timeout_seconds + + while True: + + if process.poll() is not None: + return True + + remaining = deadline - time.monotonic() + + if remaining <= 0: + return False + + time.sleep(min(self.poll_internal_seconds, remaining)) \ No newline at end of file diff --git a/tests/test_integration/test_integration_processs.py b/tests/test_integration/test_integration_processs.py new file mode 100644 index 0000000..306818c --- /dev/null +++ b/tests/test_integration/test_integration_processs.py @@ -0,0 +1,181 @@ +import os +import subprocess +import time +import threading +from pathlib import Path +from runner.process import ProcessTerminator +from runner.cancellation import CancellationToken +from runner.executor import SubprocessExecutor +from runner.failure import FailureClassifier +from runner.models import LifecycleStepContent +from runner.artifact import ArtifactManager + +PROJECT_ROOT = Path(__file__).resolve().parent + +def test_process_group_termination_cleans_child_processes(tmp_path: Path): + + child_pid_file = tmp_path / "child.pid" + + command = f""" + sleep 30 & + CHILD_PID=$! + echo $CHILD_PID > "{child_pid_file}" + wait $CHILD_PID +""" + + process = subprocess.Popen( + command, + shell=True, + start_new_session=True, + ) + + # + # 等child PID 寫出來 + # + deadline = time.monotonic() + 2 + + while not child_pid_file.exists(): + + if time.monotonic() >= deadline: + raise AssertionError( + "Child PID file was not created" + ) + + time.sleep(0.05) + + child_pid = int(child_pid_file.read_text().strip()) + + # + # Child 一開始應該存在 + # + os.kill(child_pid, 0) + + process_terminator = ProcessTerminator(grace_period_seconds=0.5) + + process_terminator.terminate_process_group(process) + + # + # 給 OS 一點時間完成 process cleanup + # + time.sleep(0.1) + + try: + + os.kill(child_pid, 0) + + child_alive = True + + except ProcessLookupError: + + child_alive = False + + assert child_alive is False + + +def test_stdout_reader_thread_finishes_after_process_termination(tmp_path: Path): + + token = CancellationToken() + + executor = SubprocessExecutor( + process_terminator=PROJECT_ROOT, + failure_classifier=FailureClassifier(), + process_terminator=ProcessTerminator(grace_period_seconds=0.5) + ) + + step = LifecycleStepContent( + name="streaming", + type="command", + command=( + "echo start; " \ + "sleep 30" + ), + timeout_second=60 + ) + + artifat_manager = ArtifactManager(tmp_path) + + run_dir = artifat_manager.create_run_directory("stream_test") + + writer = artifat_manager.create_step_log_writer(run_dir=run_dir, + stage="scenario", + step_name="streaming", + attemp=1, + show_console=False) + + def cancel_soon(): + time.sleep(0.2) + token.cancel() + + thread = threading.Thread(target=cancel_soon) + + thread.start() + + with writer: + + result = executor.execute( + step=step, + stage="scenario", + attempt=1, + log_writer=writer, + working_directory=run_dir, + cancellation_token=token + ) + + thread.join() + + assert result.cancelled is True + assert "start" in result.stdout + +def test_stderr_is_drained_after_cancellation(tmp_path: Path): + + token = CancellationToken() + + executor = SubprocessExecutor( + process_terminator=PROJECT_ROOT, + failure_classifier=FailureClassifier(), + process_terminator=ProcessTerminator(grace_period_seconds=0.5) + ) + + step = LifecycleStepContent( + name="stderr_test", + type="command", + command=( + "echo error-message >&2; " \ + "sleep 30" + ), + timeout_second=60 + ) + + artifat_manager = ArtifactManager(tmp_path) + + run_dir = artifat_manager.create_run_directory("stream_test") + + writer = artifat_manager.create_step_log_writer(run_dir=run_dir, + stage="scenario", + step_name="stderr_test", + attemp=1, + show_console=False) + + def cancel_soon(): + time.sleep(0.2) + token.cancel() + + thread = threading.Thread(target=cancel_soon) + + thread.start() + + with writer: + + result = executor.execute( + step=step, + stage="scenario", + attempt=1, + log_writer=writer, + working_directory=run_dir, + cancellation_token=token + ) + + thread.join() + + assert result.cancelled is True + assert "error-message" in result.stderr \ No newline at end of file diff --git a/tests/test_unit/test_process.py b/tests/test_unit/test_process.py new file mode 100644 index 0000000..84b390b --- /dev/null +++ b/tests/test_unit/test_process.py @@ -0,0 +1,79 @@ +import subprocess +import time +from runner.process import ProcessTerminator + +def test_terminated_process_does_not_need_cleanup(): + + process = subprocess.Popen( + ["true"], + start_new_session=True, + ) + + process.wait() + + process_terminator = ProcessTerminator() + + result = process_terminator.terminate_process_group(process) + + assert result.terminated is False + assert result.killed is False + assert result.return_code == 0 + +def test_process_group_terminates_gracefully(): + + process = subprocess.Popen( + [ + "python", + "-c", + ( + "import time; " + "time.sleep(30)" + ), + ], + start_new_session=True + ) + + process_terminator = ProcessTerminator(grace_period_seconds=1.0) + + result = process_terminator.terminate_process_group(process) + + assert result.terminated is True + assert result.killed is False + assert process.poll() is not None + +def test_process_is_killed_when_sigterm_is_ignored(): + + command = ( + "import signal, time;" + "signal.signal(" + "signal.SIGTERM, " + "signal.SIG_IGN" + "); " + "print('ready', flush=True);" + "time.sleep(30)" + ) + + process = subprocess.Popen( + [ + "python", + "-c", + command, + ], + + stdout=subprocess.PIPE, + text=True, + start_new_session=True + ) + + # + # 等child 確定已經install SIGTERM handler + # + assert process.stdout.readline().strip() == "ready" + + process_terminator = ProcessTerminator(grace_period_seconds=0.2) + + result = process_terminator.terminate_process_group(process) + + assert result.terminated is True + assert result.killed is True + assert process.poll() is not None \ No newline at end of file From 30295587e369a156798fb3658c4e48346f2dace8 Mon Sep 17 00:00:00 2001 From: Andrew Wang Date: Wed, 16 Sep 2026 22:49:06 +0800 Subject: [PATCH 2/9] add test code --- .github/workflows/ci.yml | 2 +- .../test_integration_cancellation.py | 169 ++++++++++++++++ .../test_integration_processs.py | 181 ------------------ tests/test_unit/test_runner.py | 69 +++++++ 4 files changed, 239 insertions(+), 182 deletions(-) delete mode 100644 tests/test_integration/test_integration_processs.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0b210b5..c2a3724 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,4 +1,4 @@ -name: Device Test Runner Ci +name: Device Test Runner CI on: push: diff --git a/tests/test_integration/test_integration_cancellation.py b/tests/test_integration/test_integration_cancellation.py index 1f4a3f3..944cacf 100644 --- a/tests/test_integration/test_integration_cancellation.py +++ b/tests/test_integration/test_integration_cancellation.py @@ -1,3 +1,5 @@ +import os +import subprocess import threading import time from pathlib import Path @@ -7,6 +9,7 @@ from runner.executor import SubprocessExecutor from runner.failure import FailureClassifier from runner.models import FailureType, LifecycleStepContent +from runner.process import ProcessTerminator PROJECT_ROOT = Path(__file__).resolve().parent @@ -70,3 +73,169 @@ def cancel_soon(): assert result.failure_type == FailureType.CANCELLED assert result.failure_type != FailureType.TIMEOUT assert "started" in result.stdout + + +def test_stdout_reader_thread_finishes_after_process_termination(tmp_path: Path): + + token = CancellationToken() + + executor = SubprocessExecutor( + process_terminator=PROJECT_ROOT, + failure_classifier=FailureClassifier(), + process_terminator=ProcessTerminator(grace_period_seconds=0.5), + ) + + step = LifecycleStepContent( + name="streaming", + type="command", + command=("echo start; " "sleep 30"), + timeout_second=60, + ) + + artifat_manager = ArtifactManager(tmp_path) + + run_dir = artifat_manager.create_run_directory("stream_test") + + writer = artifat_manager.create_step_log_writer( + run_dir=run_dir, + stage="scenario", + step_name="streaming", + attemp=1, + show_console=False, + ) + + def cancel_soon(): + time.sleep(0.2) + token.cancel() + + thread = threading.Thread(target=cancel_soon) + + thread.start() + + with writer: + + result = executor.execute( + step=step, + stage="scenario", + attempt=1, + log_writer=writer, + working_directory=run_dir, + cancellation_token=token, + ) + + thread.join() + + assert result.cancelled is True + assert "start" in result.stdout + + +def test_stderr_is_drained_after_cancellation(tmp_path: Path): + + token = CancellationToken() + + executor = SubprocessExecutor( + process_terminator=PROJECT_ROOT, + failure_classifier=FailureClassifier(), + process_terminator=ProcessTerminator(grace_period_seconds=0.5), + ) + + step = LifecycleStepContent( + name="stderr_test", + type="command", + command=("echo error-message >&2; " "sleep 30"), + timeout_second=60, + ) + + artifat_manager = ArtifactManager(tmp_path) + + run_dir = artifat_manager.create_run_directory("stream_test") + + writer = artifat_manager.create_step_log_writer( + run_dir=run_dir, + stage="scenario", + step_name="stderr_test", + attemp=1, + show_console=False, + ) + + def cancel_soon(): + time.sleep(0.2) + token.cancel() + + thread = threading.Thread(target=cancel_soon) + + thread.start() + + with writer: + + result = executor.execute( + step=step, + stage="scenario", + attempt=1, + log_writer=writer, + working_directory=run_dir, + cancellation_token=token, + ) + + thread.join() + + assert result.cancelled is True + assert "error-message" in result.stderr + + +def test_process_group_termination_cleans_child_processes(tmp_path: Path): + + child_pid_file = tmp_path / "child.pid" + + command = f""" + sleep 30 & + CHILD_PID=$! + echo $CHILD_PID > "{child_pid_file}" + wait $CHILD_PID +""" + + process = subprocess.Popen( + command, + shell=True, + start_new_session=True, + ) + + # + # 等child PID 寫出來 + # + deadline = time.monotonic() + 2 + + while not child_pid_file.exists(): + + if time.monotonic() >= deadline: + raise AssertionError("Child PID file was not created") + + time.sleep(0.05) + + child_pid = int(child_pid_file.read_text().strip()) + + # + # Child 一開始應該存在 + # + os.kill(child_pid, 0) + + process_terminator = ProcessTerminator(grace_period_seconds=0.5) + + process_terminator.terminate_process_group(process) + + # + # 給 OS 一點時間完成 process cleanup + # + time.sleep(0.1) + + try: + + os.kill(child_pid, 0) + + child_alive = True + + except ProcessLookupError: + + child_alive = False + + assert child_alive is False diff --git a/tests/test_integration/test_integration_processs.py b/tests/test_integration/test_integration_processs.py deleted file mode 100644 index 306818c..0000000 --- a/tests/test_integration/test_integration_processs.py +++ /dev/null @@ -1,181 +0,0 @@ -import os -import subprocess -import time -import threading -from pathlib import Path -from runner.process import ProcessTerminator -from runner.cancellation import CancellationToken -from runner.executor import SubprocessExecutor -from runner.failure import FailureClassifier -from runner.models import LifecycleStepContent -from runner.artifact import ArtifactManager - -PROJECT_ROOT = Path(__file__).resolve().parent - -def test_process_group_termination_cleans_child_processes(tmp_path: Path): - - child_pid_file = tmp_path / "child.pid" - - command = f""" - sleep 30 & - CHILD_PID=$! - echo $CHILD_PID > "{child_pid_file}" - wait $CHILD_PID -""" - - process = subprocess.Popen( - command, - shell=True, - start_new_session=True, - ) - - # - # 等child PID 寫出來 - # - deadline = time.monotonic() + 2 - - while not child_pid_file.exists(): - - if time.monotonic() >= deadline: - raise AssertionError( - "Child PID file was not created" - ) - - time.sleep(0.05) - - child_pid = int(child_pid_file.read_text().strip()) - - # - # Child 一開始應該存在 - # - os.kill(child_pid, 0) - - process_terminator = ProcessTerminator(grace_period_seconds=0.5) - - process_terminator.terminate_process_group(process) - - # - # 給 OS 一點時間完成 process cleanup - # - time.sleep(0.1) - - try: - - os.kill(child_pid, 0) - - child_alive = True - - except ProcessLookupError: - - child_alive = False - - assert child_alive is False - - -def test_stdout_reader_thread_finishes_after_process_termination(tmp_path: Path): - - token = CancellationToken() - - executor = SubprocessExecutor( - process_terminator=PROJECT_ROOT, - failure_classifier=FailureClassifier(), - process_terminator=ProcessTerminator(grace_period_seconds=0.5) - ) - - step = LifecycleStepContent( - name="streaming", - type="command", - command=( - "echo start; " \ - "sleep 30" - ), - timeout_second=60 - ) - - artifat_manager = ArtifactManager(tmp_path) - - run_dir = artifat_manager.create_run_directory("stream_test") - - writer = artifat_manager.create_step_log_writer(run_dir=run_dir, - stage="scenario", - step_name="streaming", - attemp=1, - show_console=False) - - def cancel_soon(): - time.sleep(0.2) - token.cancel() - - thread = threading.Thread(target=cancel_soon) - - thread.start() - - with writer: - - result = executor.execute( - step=step, - stage="scenario", - attempt=1, - log_writer=writer, - working_directory=run_dir, - cancellation_token=token - ) - - thread.join() - - assert result.cancelled is True - assert "start" in result.stdout - -def test_stderr_is_drained_after_cancellation(tmp_path: Path): - - token = CancellationToken() - - executor = SubprocessExecutor( - process_terminator=PROJECT_ROOT, - failure_classifier=FailureClassifier(), - process_terminator=ProcessTerminator(grace_period_seconds=0.5) - ) - - step = LifecycleStepContent( - name="stderr_test", - type="command", - command=( - "echo error-message >&2; " \ - "sleep 30" - ), - timeout_second=60 - ) - - artifat_manager = ArtifactManager(tmp_path) - - run_dir = artifat_manager.create_run_directory("stream_test") - - writer = artifat_manager.create_step_log_writer(run_dir=run_dir, - stage="scenario", - step_name="stderr_test", - attemp=1, - show_console=False) - - def cancel_soon(): - time.sleep(0.2) - token.cancel() - - thread = threading.Thread(target=cancel_soon) - - thread.start() - - with writer: - - result = executor.execute( - step=step, - stage="scenario", - attempt=1, - log_writer=writer, - working_directory=run_dir, - cancellation_token=token - ) - - thread.join() - - assert result.cancelled is True - assert "error-message" in result.stderr \ No newline at end of file diff --git a/tests/test_unit/test_runner.py b/tests/test_unit/test_runner.py index bf32d87..611f4b4 100644 --- a/tests/test_unit/test_runner.py +++ b/tests/test_unit/test_runner.py @@ -282,6 +282,75 @@ def execute( ) +class MockCleanupAwareExecutor: + + def __init__(self): + self.execute_count = 0 + self.previous_attempt_cleaned = True + + def execute( + self, + step: LifecycleStepContent, + stage: str, + attempt: int, + log_writer: StepLogWriter, + working_directory: str | Path, + cancellation_token: CancellationToken, + ) -> StepAttemptResult: + + # + # 下一個 attempt 啟動前 + # 上一次一定已經 cleaned + # + + assert self.previous_attempt_cleaned is True + + self.execute_count += 1 + + # + # 模擬這次 process lifecycle + # + self.previous_attempt_cleaned = False + + # + # Executor 在 return 前完成cleanup + # + self.previous_attempt_cleaned = True + + if attempt == 1: + return StepAttemptResult( + attempt=attempt, + success=False, + failure_type=FailureType.TIMEOUT, + timed_out=True, + cancelled=False, + exit_code=None, + duration_seconds=0.01, + stdout="", + stderr="", + stdout_log_path=log_writer.stdout_path, + stderr_log_path=log_writer.stderr_path, + error="timeout", + artifact_validation_results=[], + ) + + return StepAttemptResult( + attempt=attempt, + success=True, + failure_type=FailureType.NONE, + timed_out=False, + cancelled=False, + exit_code=0, + duration_seconds=0.01, + stdout="", + stderr="", + stdout_log_path=str(log_writer.stdout_path), + stderr_log_path=str(log_writer.stderr_path), + error="", + artifact_validation_results=[], + ) + + class MockRecordingArtifactValidator: def __init__(self): self.call_count = 0 From f3d73ee2a302b370af3732dacbc090e2d4b5d466 Mon Sep 17 00:00:00 2001 From: andrew771027 Date: Thu, 17 Sep 2026 13:03:34 +0000 Subject: [PATCH 3/9] add runner v1.6.1 test code --- tests/fixtures/__init__.py | 0 tests/fixtures/retry_process.py | 116 +++++++++ .../test_integration_cancellation.py | 2 + .../test_integration_retry.py | 245 ++++++++++++++++++ tests/test_unit/test_runner.py | 147 +++++++++-- 5 files changed, 493 insertions(+), 17 deletions(-) create mode 100644 tests/fixtures/__init__.py create mode 100644 tests/fixtures/retry_process.py diff --git a/tests/fixtures/__init__.py b/tests/fixtures/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/fixtures/retry_process.py b/tests/fixtures/retry_process.py new file mode 100644 index 0000000..cab471e --- /dev/null +++ b/tests/fixtures/retry_process.py @@ -0,0 +1,116 @@ +import os +import sys +import subprocess +import time +from pathlib import Path + +def main() -> int: + + run_dir = Path(os.environ["RUN_ARTIFACT_DIR"]) + + counter_file = run_dir / "retry_process_counter.txt" + + # + # 判斷現在是第幾次 attempt + # + if counter_file.exists(): + attempt = int(counter_file.read_text(encoding="utf-8").strip() + 1) + else: + attempt = 1 + + counter_file.write_text(str(attempt), encoding="utf-8") + + print(f"fixture attempt={attempt}", flush=True) + + # + # ----------------------------------------- + # Attempt 1 + # ----------------------------------------- + # + # 故意執行很久， + # 讓 Device Test Runner Timeout。 + # + if attempt == 1: + # + # Record fixture PID. + # + fixture_pid_file = run_dir / f"attempt_{attempt}.pid" + + fixture_pid_file.write_text(str(os.getpid()), encoding="utf-8") + + print(f"attempt {attempt} pid={os.getpid()}", flush=True) + + print(f"attempt {attempt} waiting for timeout", flush=True) + + # + # Start a child process. + # + child = subprocess.Popen( + [ + sys.executable, + "-c", + ( + "import time; " + "time.sleep(30)" + ), + ] + ) + + child_pid_file = ( + run_dir + / "attempt_1_child.pid" + ) + + child_pid_file.write_text( + str(child.pid), + encoding="utf-8", + ) + + print( + ( + f"attempt {attempt} pid=" + f"{os.getpid()}" + ), + flush=True, + ) + + print( + ( + f"attempt {attempt} child pid=" + f"{child.pid}" + ), + flush=True, + ) + + # + # Parent 也保持 running， + # 等 DTR timeout。 + # + time.sleep(30) + + # + # 正常情況不應該跑到這裡 + # + print(f"attempt {attempt} unexpectedly finished", flush=True) + + return 1 + + # ----------------------------------------- + # Attempt 2 + # ----------------------------------------- + # + # 第二次直接成功 + # + if attempt == 2: + print(f"attempt {attempt} pid={os.getpid()}", flush=True) + + print(f"attempt {attempt} success", flush=True) + + return 0 + + print(f"Unexpected attempt={attempt}", file=sys.stderr, flush=True) + + return 1 + +if __name__ == "__main__": + sys.exit(main()) \ No newline at end of file diff --git a/tests/test_integration/test_integration_cancellation.py b/tests/test_integration/test_integration_cancellation.py index 944cacf..fbf3bd3 100644 --- a/tests/test_integration/test_integration_cancellation.py +++ b/tests/test_integration/test_integration_cancellation.py @@ -1,4 +1,5 @@ import os + import subprocess import threading import time @@ -14,6 +15,7 @@ PROJECT_ROOT = Path(__file__).resolve().parent + def test_executor_cancels_running_process(tmp_path: Path): """Acceptance scenario. diff --git a/tests/test_integration/test_integration_retry.py b/tests/test_integration/test_integration_retry.py index 4eb561e..206cb7d 100644 --- a/tests/test_integration/test_integration_retry.py +++ b/tests/test_integration/test_integration_retry.py @@ -1,3 +1,6 @@ +import sys +import os +import time from pathlib import Path from runner.artifact import ArtifactManager @@ -8,8 +11,42 @@ from runner.reporter import JsonReporter from runner.runner import DeviceTestRunner +from runner.cancellation import ( + CancellationToken, +) +from runner.models import ( + ArtifactConfig, + DeviceInfo, + DeviceTestCase, + FailureType, + LifecycleConfig, + LifecycleStepContent, + LifecycleSteps, + RetryConfig, + RunnerConfig, +) +from runner.process import ( + ProcessTerminator, +) + + + PROJECT_ROOT = Path(__file__).resolve().parent +def is_process_alive(pid: int) -> bool: + try: + os.kill(pid, 0) + + except ProcessLookupError: + return False + + except PermissionError: + # + # Process 存在 + # 只是目前沒有權限 singnal + # + return True + return True def test_real_artifact_aware_retry(tmp_path: Path): """Acceptance scenario. @@ -130,3 +167,211 @@ def test_real_artifact_aware_retry(tmp_path: Path): assert second_attempt.artifact_validation_results[0].passed is True assert result.summary.status == "PASSED" + + +def test_retry_does_not_leave_previous_attempt_process(tmp_path: Path): + + # + # ------------------------------------------------ + # Project root + # ------------------------------------------------ + # + + project_root = Path(__file__).resolve().parents[2] + + fixture_script = project_root / "tests" / "fixture" / "retry_process.py" + + assert fixture_script.exists() + + # + # 使用目前 pytest 所使用的 python + # + # 比直接寫 python 更穩定 + command = ( + f'"{sys.executable}" ' + f'"{fixture_script}' + ) + + # + # ------------------------------------------ + # Config + # ------------------------------------------ + # + config = RunnerConfig( + test_case=DeviceTestCase( + id="retry_process_test", + name="retry_process_test", + description=( + "Verify process cleanup before retry" + ) + ), + device=DeviceInfo( + serial="device_001", + product="pixel", + build="build_001" + ), + retry=RetryConfig( + max_attempts=2, + delay_seconds=0.01, + retry_on=[ + FailureType.TIMEOUT + ] + ), + lifecycle=LifecycleConfig( + global_setup=LifecycleSteps(steps=[]), + setup=LifecycleSteps(steps=[]), + scenario=LifecycleConfig( + steps=[ + LifecycleStepContent( + name="retry_process", + type="command", + command=command, + + # + # Attempt 1 sleep 30 sec + # 所以一定會timeout + # + timeout_second=1 + ) + ] + ), + teardown=LifecycleSteps( + steps=[] + ), + global_teardown=LifecycleSteps( + steps=[] + ) + ), + artifact=ArtifactConfig( + output_dir=str(tmp_path) + ) + ) + + # + # -------------------------------------- + # Real Executor + # -------------------------------------- + # + + failure_classifier = FailureClassifier() + + process_terminator = ProcessTerminator(grace_period_seconds=0.5, poll_interval_seconds=0.05) + + executor = SubprocessExecutor( + project_directory=project_root, + failure_classifier=failure_classifier, + process_terminator=process_terminator + ) + + artifact_manager = ArtifactManager(tmp_path) + + runner = DeviceTestRunner( + executor=executor, + artifact_manager=artifact_manager, + artifact_validator=ArtifactValidator(), + failure_classifier=failure_classifier, + reporter=JsonReporter(), + show_console_output=False + ) + + # + # -------------------------------------------- + # Run + # -------------------------------------------- + # + + result = runner.run(config=config, cancellation_token=CancellationToken()) + + # + # -------------------------------------------- + # Find scenario result + # -------------------------------------------- + # + scenario_result = next( + step_result + for step_result + in result.step_results + if ( + step_result.name == "retry_process" + ) + ) + + # + # ------------------------------------------ + # Retry Assertions + # ------------------------------------------ + # + + assert scenario_result.attempts == 2 + assert scenario_result.success is True + assert scenario_result.cancelled is False + assert len(scenario_result.attempt_results) == 2 + + # + # Attempt 1 + # + + attempt_1 = scenario_result.attempt_results[0] + assert attempt_1.success is False + assert attempt_1.timed_out is True + assert attempt_1.cancelled is False + assert attempt_1.failure_type == FailureType.TIMEOUT + + # + # Attempt 2 + # + + attempt_2 = scenario_result.attempt_results[1] + assert attempt_2.success is True + assert attempt_2.timed_out is False + assert attempt_2.cancelled is False + assert attempt_2.failure_type == FailureType.NONE + + assert "attempt 2 success" in attempt_2.stdout + + # + # ----------------------------------------- + # Find run directory + # ----------------------------------------- + # + # 如果你的 RunResult metadata 已經有 + # run_dir，優先直接從 metadata 取得。 + # + # 以下依你的 ArtifactManager API 調整。 + # + + run_dir = Path( + result.metadata.run_dir + ) + + pid_file = ( + run_dir + / "attempt_1.pid" + ) + + assert pid_file.exists() + + attempt_1_pid = int( + pid_file + .read_text( + encoding="utf-8" + ) + .strip() + ) + + # + # ----------------------------------------- + # Critical assertion + # ----------------------------------------- + # + # Executor 在 Attempt 1 return 前 + # 應該已經把 process 收乾淨。 + # + + deadline = time.monotonic() + 1.0 + + while (is_process_alive(attempt_1_pid) and time.monotonic() < deadline): + + time.sleep(0.05) + + assert is_process_alive(attempt_1_pid) is False \ No newline at end of file diff --git a/tests/test_unit/test_runner.py b/tests/test_unit/test_runner.py index 611f4b4..7b5c74a 100644 --- a/tests/test_unit/test_runner.py +++ b/tests/test_unit/test_runner.py @@ -282,12 +282,11 @@ def execute( ) -class MockCleanupAwareExecutor: +class MockLifecycleTrackingExecutor: def __init__(self): - self.execute_count = 0 - self.previous_attempt_cleaned = True - + self.events: list[str] = [] + def execute( self, step: LifecycleStepContent, @@ -298,26 +297,40 @@ def execute( cancellation_token: CancellationToken, ) -> StepAttemptResult: + self.events.append(f"attempt_{attempt}_start") + # - # 下一個 attempt 啟動前 - # 上一次一定已經 cleaned + # 模擬 process 開始執行 # - assert self.previous_attempt_cleaned is True + self.events.append(f"attempt_{attempt}_process_running") + + if attempt == 1: - self.execute_count += 1 + # + # 模擬 Attempt 1 發生 Timeout + # - # - # 模擬這次 process lifecycle - # - self.previous_attempt_cleaned = False + self.events.append(f"attempt_{attempt}_timeout") - # - # Executor 在 return 前完成cleanup - # - self.previous_attempt_cleaned = True + # + # 這裡代表 Executor 內部： + # + # ProcessTerminator + # ↓ + # SIGTERM + # ↓ + # grace period + # ↓ + # SIGKILL (if needed) + # ↓ + # stdout/stderr join + # + + self.events.append(f"attempt_{attempt}_process_cleanup") + + self.events.append(f"attempt_{attempt}_retrun") - if attempt == 1: return StepAttemptResult( attempt=attempt, success=False, @@ -334,6 +347,16 @@ def execute( artifact_validation_results=[], ) + # + # Attempt 2 + # + + self.events.append(f"attempt_{attempt}_success") + + self.events.append(f"attempt_{attempt}_process_cleanup") + + self.events.append(f"attempt_{attempt}_return") + return StepAttemptResult( attempt=attempt, success=True, @@ -2151,6 +2174,10 @@ def test_cancel_before_run_only_runs_global_teardown(tmp_path: Path): result = runner.run(config=config, cancellation_token=token) + assert "global_setup" not in executor.executed_steps + assert "setup" not in executor.executed_steps + assert "scenario" not in executor.executed_steps + assert "teardown" not in executor.executed_steps assert executor.executed_steps == ["global_teardown"] @@ -2439,3 +2466,89 @@ def fake_monotonic(): assert result.step_results[0].attempts == 1 assert result.step_results[0].cancelled is True assert result.summary.status == "CANCELLED" + +def test_retry_starts_only_after_previous_attempt_cleanup(tmp_path: Path): + + config = RunnerConfig( + test_case=DeviceTestCase( + id="power_001", + name="power_001", + description="Description", + ), + device=DeviceInfo( + serial="device_001", + product="pixel", + build="build_001", + ), + retry=RetryConfig( + max_attempts=3, + delay_seconds=1, + retry_on=[ + FailureType.TIMEOUT + ] + ), + lifecycle=LifecycleConfig( + global_setup=LifecycleSteps(steps=[]), + setup=LifecycleSteps(steps=[]), + scenario=LifecycleSteps( + steps=[ + mock_step("scenario"), + ] + ), + teardown=LifecycleSteps(steps=[]), + global_teardown=LifecycleSteps(steps=[]), + ), + artifact=ArtifactConfig( + output_dir=str(tmp_path), + ), + ) + + executor = MockLifecycleTrackingExecutor() + + runner = DeviceTestRunner( + executor=executor, + artifact_manager=ArtifactManager(tmp_path), + artifact_validator=ArtifactValidator(), + failure_classifier=FailureClassifier(), + reporter=JsonReporter(), + show_console_output=False, + ) + + result = runner.run(config) + + scenario_result = next(step_result for step_result in result.step_results if step_result.name == "scenario") + + # + # Attempt 1 timeout + # -> retry + # -> Attempt 2 pass + + assert scenario_result.attempts == 2 + assert scenario_result.success is True + + # + # 最重要的 assertion: + # + # Attempt 1 cleanup 必須發生在 + # Attempt 2 start 之前 + # + + attempt_1_cleanup_index = executor.events.index("attempt_1_process_cleanup") + + attempt_2_start_index = executor.events.index("attempt_2_start") + + assert attempt_1_cleanup_index < attempt_2_start_index + + assert executor.events == [ + "attempt_1_start", + "attempt_1_process_running", + "attempt_1_timeout", + "attempt_1_process_cleanup", + "attempt_1_return", + + "attempt_2_start", + "attempt_2_process_running", + "attempt_2_success", + "attempt_2_process_cleanup", + "attempt_2_return", + ] \ No newline at end of file From 0dbdc51d9968a5f83c0c581607d71b31008a98c1 Mon Sep 17 00:00:00 2001 From: Andrew Wang Date: Thu, 17 Sep 2026 22:02:51 +0800 Subject: [PATCH 4/9] update runner 1.6.1 test code --- main.py | 23 +- runner/executor.py | 23 +- runner/models.py | 2 +- runner/process.py | 71 ++--- runner/runner.py | 2 +- tests/fixtures/process_tree.py | 71 +++++ tests/fixtures/retry_process.py | 48 ++-- tests/test_integration/test_integration.py | 15 +- .../test_integration_artifact_validation.py | 21 +- .../test_integration_cancellation.py | 257 +++++++++++++++++- .../test_integration_failure.py | 4 + .../test_integration_retry.py | 105 +++---- tests/test_unit/test_executor.py | 40 ++- tests/test_unit/test_process.py | 89 +++++- tests/test_unit/test_runner.py | 96 ++++--- tests/test_unit/test_signals.py | 35 +++ 16 files changed, 662 insertions(+), 240 deletions(-) create mode 100644 tests/fixtures/process_tree.py create mode 100644 tests/test_unit/test_signals.py diff --git a/main.py b/main.py index 758cf8a..f1161d5 100644 --- a/main.py +++ b/main.py @@ -1,17 +1,17 @@ import argparse -import sys import signal +import sys from pathlib import Path from runner.artifact import ArtifactManager from runner.artifact_validator import ArtifactValidator +from runner.cancellation import CancellationToken from runner.config import ConfigLoader from runner.executor import SubprocessExecutor from runner.failure import FailureClassifier +from runner.process import ProcessTerminator from runner.reporter import JsonReporter from runner.runner import DeviceTestRunner -from runner.cancellation import CancellationToken -from runner.process import ProcessTerminator PROJECT_ROOT = Path(__file__).resolve().parent @@ -32,9 +32,7 @@ def handle_sigint(self, signum, frame): self._signal_count += 1 if self._signal_count == 1: - print("\nCancellaiton requested. " - "Cleaning up..." - ) + print("\nCancellaiton requested. " "Cleaning up...") self.token.cancel() @@ -44,12 +42,11 @@ def handle_sigint(self, signum, frame): # Optional: # second Ctrl + C = immediate interruption # - print( - "\nForce exit requested." - ) + print("\nForce exit requested.") raise KeyboardInterrupt + def main(): args = parse_args() @@ -62,13 +59,15 @@ def main(): signal_handler = SignalCancellationHandler(cancellation_token) - old_sigint_handler = signal_handler.signal(signal.SIGINT, signal_handler.handle_sigint) - + old_sigint_handler = signal.signal(signal.SIGINT, signal_handler.handle_sigint) + process_terminator = ProcessTerminator(grace_period_seconds=2.0) runner = DeviceTestRunner( executor=SubprocessExecutor( - project_directory=PROJECT_ROOT, failure_classifier=failure_classifier, process_terminator=process_terminator + project_directory=PROJECT_ROOT, + failure_classifier=failure_classifier, + process_terminator=process_terminator, ), artifact_manager=ArtifactManager(output_dir=config.artifact.output_dir), artifact_validator=ArtifactValidator(), diff --git a/runner/executor.py b/runner/executor.py index d6f0d27..d7cd2d2 100644 --- a/runner/executor.py +++ b/runner/executor.py @@ -4,7 +4,7 @@ import threading import time from pathlib import Path -from typing import Callable, TextIO +from typing import TextIO from runner.artifact import StepLogWriter from runner.cancellation import CancellationToken @@ -12,6 +12,7 @@ from runner.models import FailureType, LifecycleStepContent, StepAttemptResult from runner.process import ProcessTerminator + class SubprocessExecutor: POLL_INTERVAL_SECONDS = 0.1 @@ -36,7 +37,9 @@ def execute( ) -> StepAttemptResult: if log_writer is None: - log_writer = self._create_default_log_writer(stage=stage, step_name=step.name) + log_writer = self._create_default_log_writer( + stage=stage, step_name=step.name + ) environment = os.environ.copy() @@ -74,18 +77,17 @@ def execute( encoding="utf-8", errors="replace", bufsize=1, - - # + # # Important: - # this attempt gets its own session / + # this attempt gets its own session / # process group - start_new_session=True + start_new_session=True, ) if process.stdout is None or process.stderr is None: raise RuntimeError("Unable to open subprocess streams.") - # + # # -------------------------------------------- # stdout reader # -------------------------------------------- @@ -98,7 +100,7 @@ def execute( daemon=True, ) - # + # # -------------------------------------------- # stderr reader # -------------------------------------------- @@ -122,7 +124,7 @@ def execute( while True: - # + # # 1. process 已經正常結束 # if process.poll() is not None: @@ -135,7 +137,7 @@ def execute( error_message = "Execution cancelled" - self._stop_process(process) + self.process_terminator.terminate_process_group(process) break @@ -248,7 +250,6 @@ def execute( artifact_validation_results=[], ) - @staticmethod def _consume_stream(stream: TextIO, write_line) -> None: try: diff --git a/runner/models.py b/runner/models.py index ff3a9a0..cf5cd34 100644 --- a/runner/models.py +++ b/runner/models.py @@ -146,7 +146,7 @@ class StepAttemptResult: failure_type: FailureType timed_out: bool cancelled: bool - + exit_code: Optional[int] duration_seconds: float diff --git a/runner/process.py b/runner/process.py index e22d7a2..c918009 100644 --- a/runner/process.py +++ b/runner/process.py @@ -4,38 +4,40 @@ import time from dataclasses import dataclass + @dataclass(frozen=True) class ProcessTerminationResult: terminated: bool killed: bool return_code: int | None + class ProcessTerminator: - + def __init__(self, grace_period_seconds: float = 2.0, poll_interval_seconds: float = 0.05): if grace_period_seconds < 0: raise ValueError("grace_period_seconds must be >= 0") - + if poll_interval_seconds <= 0: raise ValueError("poll_internal_seconds must be > 0") - + self.grace_period_seconds = grace_period_seconds self.poll_internal_seconds = poll_interval_seconds - + def terminate_process_group(self, process: subprocess.Popen) -> ProcessTerminationResult: # # Process 已經結束 - # + # if process.poll() is not None: return ProcessTerminationResult( terminated=False, killed=False, return_code=process.returncode, ) - + try: - + process_group_id = os.getpgid(process.pid) except ProcessLookupError: @@ -54,14 +56,18 @@ def terminate_process_group(self, process: subprocess.Popen) -> ProcessTerminati try: os.killpg(process_group_id, signal.SIGTERM) - + except ProcessLookupError: pass - - # + + # # 給 process group 一段時間自己 cleanup # - exited = self._wait_for_exit(process=process, timeout_seconds=self.grace_period_seconds) + exited = self._wait_for_exit( + process=process, + process_group_id=process_group_id, + timeout_seconds=self.grace_period_seconds, + ) if exited: return ProcessTerminationResult( @@ -69,7 +75,7 @@ def terminate_process_group(self, process: subprocess.Popen) -> ProcessTerminati killed=False, return_code=process.returncode, ) - + # # ------------------------------------------------ # Phase 3 @@ -82,39 +88,40 @@ def terminate_process_group(self, process: subprocess.Popen) -> ProcessTerminati except ProcessLookupError: pass - + # - # Reap direct child + # Reap the direct child and wait for descendants before allowing retry. # - try: + if not self._wait_for_exit( + process=process, process_group_id=process_group_id, timeout_seconds=2 + ): + raise RuntimeError("Process group did not exit after SIGKILL.") - process.wait(timeout=2) - - except subprocess.TimeoutExpired: - # - # SIGKILL 理論上應該會很快結束 - # 這裡再 poll，而不是無限 wait - # - pass - - return ProcessTerminationResult( - terminated=True, - killed=True, - return_code=process.poll() - ) + return ProcessTerminationResult(terminated=True, killed=True, return_code=process.poll()) - def _wait_for_exit(self, process: subprocess.Popen, timeout_seconds: float) -> bool: + def _wait_for_exit( + self, process: subprocess.Popen, process_group_id: int, timeout_seconds: float + ) -> bool: deadline = time.monotonic() + timeout_seconds while True: - if process.poll() is not None: + # poll() reaps the direct child, but its exit alone says nothing + # about descendants that may still hold stdout/stderr open. + process.poll() + try: + os.killpg(process_group_id, 0) + except ProcessLookupError: return True + except PermissionError: + # Lack of permission does not prove that the group is gone. + # Retry until it disappears or the bounded wait expires. + pass remaining = deadline - time.monotonic() if remaining <= 0: return False - time.sleep(min(self.poll_internal_seconds, remaining)) \ No newline at end of file + time.sleep(min(self.poll_internal_seconds, remaining)) diff --git a/runner/runner.py b/runner/runner.py index 222d129..d69c563 100644 --- a/runner/runner.py +++ b/runner/runner.py @@ -26,7 +26,7 @@ class DeviceTestRunner: - VERSION = "1.6.0" + VERSION = "1.6.1" def __init__( self, diff --git a/tests/fixtures/process_tree.py b/tests/fixtures/process_tree.py new file mode 100644 index 0000000..9bb85fd --- /dev/null +++ b/tests/fixtures/process_tree.py @@ -0,0 +1,71 @@ +import os +import signal +import subprocess +import sys +import time +from pathlib import Path + + +def main() -> int: + # + # ------------------------------------------ + # 讀取測試傳入的參數 + # ------------------------------------------ + # + # 同一支 script 會啟動自己，建立三層 process： + # 0 = parent -> 1 = child -> 2 = grandchild。 + # + run_dir = Path(sys.argv[1]) + process_level = int(sys.argv[2]) + descendant_behavior = sys.argv[3] + + # parent 正常接受 SIGTERM。 + # ignore 情境下，只有 child / grandchild 故意忽略 SIGTERM。 + if descendant_behavior == "ignore" and process_level > 0: + signal.signal(signal.SIGTERM, signal.SIG_IGN) + + pid_file = run_dir / f"{process_level}.pid" + pid_file.write_text(str(os.getpid()), encoding="utf-8") + + # + # ------------------------------------------ + # 啟動下一層 process + # ------------------------------------------ + # + if process_level < 2: + child_level = process_level + 1 + + subprocess.Popen( + [ + sys.executable, + __file__, + str(run_dir), + str(child_level), + descendant_behavior, + ] + ) + + # 等下一層準備好，自己才繼續。 + # 因此 parent 寫出 0.ready 時，代表三層都已經準備完成。 + child_ready_file = run_dir / f"{child_level}.ready" + + while not child_ready_file.exists(): + time.sleep(0.01) + + # + # ------------------------------------------ + # 輸出 log，然後等待測試終止 process + # ------------------------------------------ + # + print(f"stdout role={process_level}", flush=True) + print(f"stderr role={process_level}", file=sys.stderr, flush=True) + + ready_file = run_dir / f"{process_level}.ready" + ready_file.touch() + + time.sleep(30) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/fixtures/retry_process.py b/tests/fixtures/retry_process.py index cab471e..b46d3fc 100644 --- a/tests/fixtures/retry_process.py +++ b/tests/fixtures/retry_process.py @@ -1,10 +1,11 @@ import os -import sys import subprocess +import sys import time from pathlib import Path -def main() -> int: + +def main() -> int: run_dir = Path(os.environ["RUN_ARTIFACT_DIR"]) @@ -14,7 +15,7 @@ def main() -> int: # 判斷現在是第幾次 attempt # if counter_file.exists(): - attempt = int(counter_file.read_text(encoding="utf-8").strip() + 1) + attempt = int(counter_file.read_text(encoding="utf-8").strip()) + 1 else: attempt = 1 @@ -49,17 +50,11 @@ def main() -> int: [ sys.executable, "-c", - ( - "import time; " - "time.sleep(30)" - ), + ("import time; " "time.sleep(30)"), ] ) - child_pid_file = ( - run_dir - / "attempt_1_child.pid" - ) + child_pid_file = run_dir / "attempt_1_child.pid" child_pid_file.write_text( str(child.pid), @@ -67,18 +62,12 @@ def main() -> int: ) print( - ( - f"attempt {attempt} pid=" - f"{os.getpid()}" - ), + (f"attempt {attempt} pid=" f"{os.getpid()}"), flush=True, ) print( - ( - f"attempt {attempt} child pid=" - f"{child.pid}" - ), + (f"attempt {attempt} child pid=" f"{child.pid}"), flush=True, ) @@ -102,6 +91,24 @@ def main() -> int: # 第二次直接成功 # if attempt == 2: + # + # Attempt 2 一開始就檢查前一次的 parent / child。 + # 這裡不能先 sleep，否則無法確認 retry 開始前是否已經清乾淨。 + # + for filename in ("attempt_1.pid", "attempt_1_child.pid"): + pid = int((run_dir / filename).read_text(encoding="utf-8")) + try: + # signal 0 只檢查 process 是否存在。 + os.kill(pid, 0) + except ProcessLookupError: + # 前一次的 process 已經結束，繼續檢查下一個 PID。 + continue + + # 還找得到前一次的 process，代表 cleanup 沒有完成。 + print(f"previous attempt still alive: {filename}={pid}", file=sys.stderr) + return 1 + print("previous attempt tree cleaned before retry", flush=True) + print(f"attempt {attempt} pid={os.getpid()}", flush=True) print(f"attempt {attempt} success", flush=True) @@ -112,5 +119,6 @@ def main() -> int: return 1 + if __name__ == "__main__": - sys.exit(main()) \ No newline at end of file + sys.exit(main()) diff --git a/tests/test_integration/test_integration.py b/tests/test_integration/test_integration.py index 0aa020b..70e6a99 100644 --- a/tests/test_integration/test_integration.py +++ b/tests/test_integration/test_integration.py @@ -6,6 +6,7 @@ from runner.config import ConfigLoader from runner.executor import SubprocessExecutor from runner.failure import FailureClassifier +from runner.process import ProcessTerminator from runner.reporter import JsonReporter from runner.runner import DeviceTestRunner @@ -80,6 +81,7 @@ def test_yaml_to_result_json(tmp_path): executor=SubprocessExecutor( project_directory=PROJECT_ROOT, failure_classifier=FailureClassifier(), + process_terminator=ProcessTerminator(), ), artifact_manager=ArtifactManager(output_dir=config.artifact.output_dir), artifact_validator=ArtifactValidator(), @@ -146,7 +148,7 @@ def test_yaml_to_result_json(tmp_path): report = json.loads((run_dir / "result.json").read_text(encoding="utf-8")) - assert report["metadata"]["runner_version"] == "1.6.0" + assert report["metadata"]["runner_version"] == "1.6.1" assert report["summary"]["status"] == "PASSED" @@ -219,7 +221,9 @@ def test_integration_stops_after_failed_step(tmp_path): runner = DeviceTestRunner( executor=SubprocessExecutor( - project_directory=PROJECT_ROOT, failure_classifier=FailureClassifier() + project_directory=PROJECT_ROOT, + failure_classifier=FailureClassifier(), + process_terminator=ProcessTerminator(), ), artifact_manager=ArtifactManager(output_dir=config.artifact.output_dir), artifact_validator=ArtifactValidator(), @@ -289,7 +293,7 @@ def test_integration_stops_after_failed_step(tmp_path): report = json.loads((run_dir / "result.json").read_text(encoding="utf-8")) - assert report["metadata"]["runner_version"] == "1.6.0" + assert report["metadata"]["runner_version"] == "1.6.1" assert report["summary"]["status"] == "FAILED" @@ -365,6 +369,7 @@ def test_integration_steps_succeeds_after_party(tmp_path: Path): executor=SubprocessExecutor( project_directory=PROJECT_ROOT, failure_classifier=FailureClassifier(), + process_terminator=ProcessTerminator(), ), artifact_manager=ArtifactManager(output_dir=str(output_dir)), artifact_validator=ArtifactValidator(), @@ -435,7 +440,9 @@ def test_real_subprocess_fails_after_retry_exhausted(tmp_path: Path): config = ConfigLoader().load(str(config_file)) runner = DeviceTestRunner( executor=SubprocessExecutor( - project_directory=PROJECT_ROOT, failure_classifier=FailureClassifier() + project_directory=PROJECT_ROOT, + failure_classifier=FailureClassifier(), + process_terminator=ProcessTerminator(), ), artifact_manager=ArtifactManager(output_dir=str(output_dir)), artifact_validator=ArtifactValidator(), diff --git a/tests/test_integration/test_integration_artifact_validation.py b/tests/test_integration/test_integration_artifact_validation.py index 065e163..28c9a67 100644 --- a/tests/test_integration/test_integration_artifact_validation.py +++ b/tests/test_integration/test_integration_artifact_validation.py @@ -6,6 +6,7 @@ from runner.config import ConfigLoader from runner.executor import SubprocessExecutor from runner.failure import FailureClassifier +from runner.process import ProcessTerminator from runner.reporter import JsonReporter from runner.runner import DeviceTestRunner @@ -76,7 +77,9 @@ def test_command_creates_and_validates_artifact(tmp_path: Path): runner = DeviceTestRunner( executor=SubprocessExecutor( - project_directory=PROJECT_ROOT, failure_classifier=FailureClassifier() + project_directory=PROJECT_ROOT, + failure_classifier=FailureClassifier(), + process_terminator=ProcessTerminator(), ), artifact_manager=ArtifactManager(output_dir=config.artifact.output_dir), artifact_validator=ArtifactValidator(), @@ -161,7 +164,9 @@ def test_run_fails_when_command_does_not_create_artifact(tmp_path: Path): config = ConfigLoader().load(config_path) runner = DeviceTestRunner( executor=SubprocessExecutor( - project_directory=PROJECT_ROOT, failure_classifier=FailureClassifier() + project_directory=PROJECT_ROOT, + failure_classifier=FailureClassifier(), + process_terminator=ProcessTerminator(), ), artifact_manager=ArtifactManager(output_dir=config.artifact.output_dir), artifact_validator=ArtifactValidator(), @@ -234,7 +239,9 @@ def test_command_runs_inside_run_directory(tmp_path: Path): config = ConfigLoader().load(config_path) runner = DeviceTestRunner( executor=SubprocessExecutor( - project_directory=PROJECT_ROOT, failure_classifier=FailureClassifier() + project_directory=PROJECT_ROOT, + failure_classifier=FailureClassifier(), + process_terminator=ProcessTerminator(), ), artifact_manager=ArtifactManager(output_dir=config.artifact.output_dir), artifact_validator=ArtifactValidator(), @@ -328,6 +335,7 @@ def test_csv_and_json_content_validation(tmp_path: Path): executor=SubprocessExecutor( project_directory=PROJECT_ROOT, failure_classifier=FailureClassifier(), + process_terminator=ProcessTerminator(), ), artifact_manager=ArtifactManager(output_dir=config.artifact.output_dir), artifact_validator=ArtifactValidator(), @@ -353,7 +361,7 @@ def test_csv_and_json_content_validation(tmp_path: Path): (Path(result.artifact_dir) / "result.json").read_text(encoding="utf-8") ) - assert saved_report["metadata"]["runner_version"] == "1.6.0" + assert saved_report["metadata"]["runner_version"] == "1.6.1" assert saved_report["summary"]["status"] == "PASSED" @@ -412,6 +420,7 @@ def test_run_fails_when_csv_content_invalid(tmp_path: Path): executor=SubprocessExecutor( project_directory=PROJECT_ROOT, failure_classifier=FailureClassifier(), + process_terminator=ProcessTerminator(), ), artifact_manager=ArtifactManager(output_dir=config.artifact.output_dir), artifact_validator=ArtifactValidator(), @@ -495,7 +504,9 @@ def test_run_fails_when_json_status_invalid(tmp_path: Path): config = ConfigLoader().load(config_file) runner = DeviceTestRunner( executor=SubprocessExecutor( - project_directory=PROJECT_ROOT, failure_classifier=FailureClassifier() + project_directory=PROJECT_ROOT, + failure_classifier=FailureClassifier(), + process_terminator=ProcessTerminator(), ), artifact_manager=ArtifactManager(output_dir=config.artifact.output_dir), artifact_validator=ArtifactValidator(), diff --git a/tests/test_integration/test_integration_cancellation.py b/tests/test_integration/test_integration_cancellation.py index fbf3bd3..8a02634 100644 --- a/tests/test_integration/test_integration_cancellation.py +++ b/tests/test_integration/test_integration_cancellation.py @@ -1,10 +1,14 @@ import os - +import shlex +import signal import subprocess +import sys import threading import time from pathlib import Path +import pytest + from runner.artifact import ArtifactManager from runner.cancellation import CancellationToken from runner.executor import SubprocessExecutor @@ -15,7 +19,6 @@ PROJECT_ROOT = Path(__file__).resolve().parent - def test_executor_cancels_running_process(tmp_path: Path): """Acceptance scenario. @@ -45,7 +48,9 @@ def test_executor_cancels_running_process(tmp_path: Path): ) executor = SubprocessExecutor( - project_directory=PROJECT_ROOT, failure_classifier=FailureClassifier() + project_directory=PROJECT_ROOT, + failure_classifier=FailureClassifier(), + process_terminator=ProcessTerminator(), ) def cancel_soon(): @@ -82,7 +87,7 @@ def test_stdout_reader_thread_finishes_after_process_termination(tmp_path: Path) token = CancellationToken() executor = SubprocessExecutor( - process_terminator=PROJECT_ROOT, + project_directory=PROJECT_ROOT, failure_classifier=FailureClassifier(), process_terminator=ProcessTerminator(grace_period_seconds=0.5), ) @@ -102,7 +107,7 @@ def test_stdout_reader_thread_finishes_after_process_termination(tmp_path: Path) run_dir=run_dir, stage="scenario", step_name="streaming", - attemp=1, + attempt=1, show_console=False, ) @@ -136,7 +141,7 @@ def test_stderr_is_drained_after_cancellation(tmp_path: Path): token = CancellationToken() executor = SubprocessExecutor( - process_terminator=PROJECT_ROOT, + project_directory=PROJECT_ROOT, failure_classifier=FailureClassifier(), process_terminator=ProcessTerminator(grace_period_seconds=0.5), ) @@ -156,7 +161,7 @@ def test_stderr_is_drained_after_cancellation(tmp_path: Path): run_dir=run_dir, stage="scenario", step_name="stderr_test", - attemp=1, + attempt=1, show_console=False, ) @@ -241,3 +246,241 @@ def test_process_group_termination_cleans_child_processes(tmp_path: Path): child_alive = False assert child_alive is False + + +@pytest.mark.parametrize( + argnames="stop_reason, descendant_behavior", + argvalues=[ + ("timeout", "default"), + ("timeout", "ignore"), + ("cancel", "default"), + ("cancel", "ignore"), + ], +) +def test_executor_cleans_entire_tree_and_drains_readers( + tmp_path: Path, monkeypatch, stop_reason, descendant_behavior +): + # + # ------------------------------------------ + # 測試情境 + # ------------------------------------------ + # + # 啟動 parent -> child -> grandchild。 + # timeout 或 cancel 後，三個 process 都必須結束。 + # + # default: child / grandchild 正常接受 SIGTERM。 + # ignore: child / grandchild 忽略 SIGTERM，必須靠 SIGKILL 結束。 + # + fixture_script = Path(__file__).resolve().parents[1] / "fixtures" / "process_tree.py" + + # + # ------------------------------------------ + # 記錄真正啟動的 process 和 reader thread + # ------------------------------------------ + # + # 這裡沒有換成假的 process。 + # 保留原本的函式，呼叫時順便記錄物件，方便最後檢查及清理。 + # + started_processes = [] + reader_threads = [] + + original_popen = subprocess.Popen + original_join_reader = SubprocessExecutor._join_reader_thread + + def start_and_record_process(*args, **kwargs): + process = original_popen(*args, **kwargs) + started_processes.append(process) + return process + + def join_and_record_reader(thread): + if thread is not None: + reader_threads.append(thread) + + original_join_reader(thread) + + monkeypatch.setattr("runner.executor.subprocess.Popen", start_and_record_process) + + # _join_reader_thread 原本是 staticmethod，替換後也維持相同用法。 + monkeypatch.setattr( + SubprocessExecutor, + "_join_reader_thread", + staticmethod(join_and_record_reader), + ) + + # + # ------------------------------------------ + # 等待 process 準備好，再發出 cancel + # ------------------------------------------ + # + token = CancellationToken() + + # Event 可以讓兩個 thread 通知彼此。 + # 測試結束時設為 True，讓等待中的 cancellation thread 離開。 + stop_waiting = threading.Event() + tree_ready = threading.Event() + + def cancel_when_processes_are_ready(): + deadline = time.monotonic() + 5 + + while time.monotonic() < deadline: + if stop_waiting.is_set(): + return + + # fixture 確認三個 process 都已準備好，才會建立 0.ready。 + if (tmp_path / "0.ready").exists(): + tree_ready.set() + token.cancel() + return + + stop_waiting.wait(0.01) + + cancellation_thread = threading.Thread(target=cancel_when_processes_are_ready) + + # + # ------------------------------------------ + # Config / Executor + # ------------------------------------------ + # + artifact_manager = ArtifactManager(tmp_path) + + writer = artifact_manager.create_step_log_writer( + run_dir=tmp_path, + stage="scenario", + step_name="tree", + attempt=1, + show_console=False, + ) + + if stop_reason == "timeout": + timeout_seconds = 2 + else: + # cancel 情境保留較長 timeout，避免還沒取消就先逾時。 + timeout_seconds = 10 + + # shlex.join 會處理路徑中的空白及引號。 + # exec 讓 shell 直接換成 fixture process，方便記錄 process group。 + command = shlex.join( + ["exec", sys.executable, str(fixture_script), str(tmp_path), "0", descendant_behavior] + ) + + step = LifecycleStepContent( + name="tree", + type="command", + command=command, + timeout_second=timeout_seconds, + ) + + executor = SubprocessExecutor( + project_directory=PROJECT_ROOT, + failure_classifier=FailureClassifier(), + process_terminator=ProcessTerminator(grace_period_seconds=0.2), + ) + + try: + # + # ------------------------------------------ + # Run + # ------------------------------------------ + # + if stop_reason == "cancel": + cancellation_thread.start() + + with writer: + result = executor.execute( + step=step, + stage="scenario", + attempt=1, + log_writer=writer, + working_directory=tmp_path, + cancellation_token=token, + ) + + # + # ------------------------------------------ + # 確認取消與逾時沒有混在一起 + # ------------------------------------------ + # + assert (tmp_path / "0.ready").exists() + + if stop_reason == "cancel": + assert tree_ready.is_set() + assert result.cancelled is True + assert result.timed_out is False + assert result.failure_type == FailureType.CANCELLED + else: + assert result.cancelled is False + assert result.timed_out is True + assert result.failure_type == FailureType.TIMEOUT + + # + # ------------------------------------------ + # stdout / stderr reader 都必須結束 + # ------------------------------------------ + # + assert len(reader_threads) == 2 + + reader_names = [] + for thread in reader_threads: + reader_names.append(thread.name) + assert thread.is_alive() is False + + assert "scenario-tree-stdout" in reader_names + assert "scenario-tree-stderr" in reader_names + + # 記憶體中的輸出，也必須完整寫入 log 檔案。 + assert writer.stdout_path.read_text(encoding="utf-8") == result.stdout + assert writer.stderr_path.read_text(encoding="utf-8") == result.stderr + + # + # ------------------------------------------ + # 確認每一層 process 的輸出及 PID + # ------------------------------------------ + # + # 0 = parent，1 = child，2 = grandchild。 + # + for process_level in range(3): + assert f"stdout role={process_level}" in result.stdout + assert f"stderr role={process_level}" in result.stderr + + pid_file = tmp_path / f"{process_level}.pid" + process_pid = int(pid_file.read_text(encoding="utf-8")) + + # 給 OS 一點時間完成回收，最多等 2 秒。 + process_alive = True + deadline = time.monotonic() + 2 + + while time.monotonic() < deadline: + try: + # signal 0 只檢查 process 是否存在，不會終止它。 + os.kill(process_pid, 0) + except ProcessLookupError: + process_alive = False + break + + time.sleep(0.01) + + assert process_alive is False, f"Process 還沒結束：PID={process_pid}" + + finally: + # + # ------------------------------------------ + # 測試失敗時也要清理，避免留下背景 process + # ------------------------------------------ + # + stop_waiting.set() + + # ident 不是 None，代表這個 thread 曾經啟動過，才可以 join。 + if cancellation_thread.ident is not None: + cancellation_thread.join(timeout=2) + + for process in started_processes: + try: + os.killpg(process.pid, signal.SIGKILL) + except ProcessLookupError: + # 群組已經結束，不需要再處理。 + pass + + process.wait(timeout=2) + + for thread in reader_threads: + thread.join(timeout=2) diff --git a/tests/test_integration/test_integration_failure.py b/tests/test_integration/test_integration_failure.py index a8e23b3..e06abdb 100644 --- a/tests/test_integration/test_integration_failure.py +++ b/tests/test_integration/test_integration_failure.py @@ -5,6 +5,7 @@ from runner.executor import SubprocessExecutor from runner.failure import FailureClassifier from runner.models import FailureType, LifecycleStepContent +from runner.process import ProcessTerminator PROJECT_ROOT = Path(__file__).resolve().parent @@ -35,6 +36,7 @@ def test_executor_classifies_timeout(tmp_path: Path): executor = SubprocessExecutor( project_directory=tmp_path, failure_classifier=FailureClassifier(), + process_terminator=ProcessTerminator(), ) with writer: @@ -85,6 +87,7 @@ def test_executor_classifies_device_offline(tmp_path: Path): executor = SubprocessExecutor( project_directory=PROJECT_ROOT, failure_classifier=FailureClassifier(), + process_terminator=ProcessTerminator(), ) with writer: @@ -132,6 +135,7 @@ def test_executor_classifies_process_error(tmp_path: Path): executor = SubprocessExecutor( project_directory=PROJECT_ROOT, failure_classifier=FailureClassifier(), + process_terminator=ProcessTerminator(), ) with writer: diff --git a/tests/test_integration/test_integration_retry.py b/tests/test_integration/test_integration_retry.py index 206cb7d..119adb3 100644 --- a/tests/test_integration/test_integration_retry.py +++ b/tests/test_integration/test_integration_retry.py @@ -1,19 +1,14 @@ -import sys import os +import sys import time from pathlib import Path from runner.artifact import ArtifactManager from runner.artifact_validator import ArtifactValidator +from runner.cancellation import CancellationToken from runner.config import ConfigLoader from runner.executor import SubprocessExecutor from runner.failure import FailureClassifier -from runner.reporter import JsonReporter -from runner.runner import DeviceTestRunner - -from runner.cancellation import ( - CancellationToken, -) from runner.models import ( ArtifactConfig, DeviceInfo, @@ -25,14 +20,13 @@ RetryConfig, RunnerConfig, ) -from runner.process import ( - ProcessTerminator, -) - - +from runner.process import ProcessTerminator +from runner.reporter import JsonReporter +from runner.runner import DeviceTestRunner PROJECT_ROOT = Path(__file__).resolve().parent + def is_process_alive(pid: int) -> bool: try: os.kill(pid, 0) @@ -44,10 +38,11 @@ def is_process_alive(pid: int) -> bool: # # Process 存在 # 只是目前沒有權限 singnal - # + # return True return True + def test_real_artifact_aware_retry(tmp_path: Path): """Acceptance scenario. @@ -136,6 +131,7 @@ def test_real_artifact_aware_retry(tmp_path: Path): executor=SubprocessExecutor( project_directory=PROJECT_ROOT, failure_classifier=FailureClassifier(), + process_terminator=ProcessTerminator(), ), artifact_manager=ArtifactManager(output_dir=output_dir), artifact_validator=ArtifactValidator(), @@ -176,10 +172,10 @@ def test_retry_does_not_leave_previous_attempt_process(tmp_path: Path): # Project root # ------------------------------------------------ # - + project_root = Path(__file__).resolve().parents[2] - fixture_script = project_root / "tests" / "fixture" / "retry_process.py" + fixture_script = project_root / "tests" / "fixtures" / "retry_process.py" assert fixture_script.exists() @@ -187,10 +183,7 @@ def test_retry_does_not_leave_previous_attempt_process(tmp_path: Path): # 使用目前 pytest 所使用的 python # # 比直接寫 python 更穩定 - command = ( - f'"{sys.executable}" ' - f'"{fixture_script}' - ) + command = f'"{sys.executable}" "{fixture_script}"' # # ------------------------------------------ @@ -201,50 +194,31 @@ def test_retry_does_not_leave_previous_attempt_process(tmp_path: Path): test_case=DeviceTestCase( id="retry_process_test", name="retry_process_test", - description=( - "Verify process cleanup before retry" - ) - ), - device=DeviceInfo( - serial="device_001", - product="pixel", - build="build_001" - ), - retry=RetryConfig( - max_attempts=2, - delay_seconds=0.01, - retry_on=[ - FailureType.TIMEOUT - ] + description=("Verify process cleanup before retry"), ), + device=DeviceInfo(serial="device_001", product="pixel", build="build_001"), + retry=RetryConfig(max_attempts=2, delay_seconds=0.01, retry_on=[FailureType.TIMEOUT]), lifecycle=LifecycleConfig( global_setup=LifecycleSteps(steps=[]), setup=LifecycleSteps(steps=[]), - scenario=LifecycleConfig( + scenario=LifecycleSteps( steps=[ LifecycleStepContent( name="retry_process", type="command", command=command, - # # Attempt 1 sleep 30 sec # 所以一定會timeout # - timeout_second=1 + timeout_second=1, ) ] ), - teardown=LifecycleSteps( - steps=[] - ), - global_teardown=LifecycleSteps( - steps=[] - ) + teardown=LifecycleSteps(steps=[]), + global_teardown=LifecycleSteps(steps=[]), ), - artifact=ArtifactConfig( - output_dir=str(tmp_path) - ) + artifact=ArtifactConfig(output_dir=str(tmp_path)), ) # @@ -260,7 +234,7 @@ def test_retry_does_not_leave_previous_attempt_process(tmp_path: Path): executor = SubprocessExecutor( project_directory=project_root, failure_classifier=failure_classifier, - process_terminator=process_terminator + process_terminator=process_terminator, ) artifact_manager = ArtifactManager(tmp_path) @@ -271,7 +245,7 @@ def test_retry_does_not_leave_previous_attempt_process(tmp_path: Path): artifact_validator=ArtifactValidator(), failure_classifier=failure_classifier, reporter=JsonReporter(), - show_console_output=False + show_console_output=False, ) # @@ -288,12 +262,7 @@ def test_retry_does_not_leave_previous_attempt_process(tmp_path: Path): # -------------------------------------------- # scenario_result = next( - step_result - for step_result - in result.step_results - if ( - step_result.name == "retry_process" - ) + step_result for step_result in result.step_results if (step_result.name == "retry_process") ) # @@ -328,36 +297,24 @@ def test_retry_does_not_leave_previous_attempt_process(tmp_path: Path): assert attempt_2.failure_type == FailureType.NONE assert "attempt 2 success" in attempt_2.stdout + assert "previous attempt tree cleaned before retry" in attempt_2.stdout # # ----------------------------------------- # Find run directory # ----------------------------------------- # - # 如果你的 RunResult metadata 已經有 - # run_dir，優先直接從 metadata 取得。 - # - # 以下依你的 ArtifactManager API 調整。 + # RunResult.artifact_dir 是此次執行的結果目錄。 # - run_dir = Path( - result.metadata.run_dir - ) + assert result.artifact_dir is not None + run_dir = Path(result.artifact_dir) - pid_file = ( - run_dir - / "attempt_1.pid" - ) + pid_file = run_dir / "attempt_1.pid" assert pid_file.exists() - attempt_1_pid = int( - pid_file - .read_text( - encoding="utf-8" - ) - .strip() - ) + attempt_1_pid = int(pid_file.read_text(encoding="utf-8").strip()) # # ----------------------------------------- @@ -370,8 +327,8 @@ def test_retry_does_not_leave_previous_attempt_process(tmp_path: Path): deadline = time.monotonic() + 1.0 - while (is_process_alive(attempt_1_pid) and time.monotonic() < deadline): + while is_process_alive(attempt_1_pid) and time.monotonic() < deadline: time.sleep(0.05) - assert is_process_alive(attempt_1_pid) is False \ No newline at end of file + assert is_process_alive(attempt_1_pid) is False diff --git a/tests/test_unit/test_executor.py b/tests/test_unit/test_executor.py index 245dbac..a299c2d 100644 --- a/tests/test_unit/test_executor.py +++ b/tests/test_unit/test_executor.py @@ -1,3 +1,4 @@ +import signal import subprocess import sys from io import StringIO @@ -11,6 +12,7 @@ from runner.executor import SubprocessExecutor from runner.failure import FailureClassifier from runner.models import FailureType, LifecycleStepContent +from runner.process import ProcessTerminator PROJECT_ROOT = Path(__file__).resolve().parent @@ -41,7 +43,9 @@ def test_subprocess_executor_return_success(tmp_path, test_case_id, step, stage, Then success is true, cancellation and timeout are false, and stdout matches its log. """ executor = SubprocessExecutor( - project_directory=PROJECT_ROOT, failure_classifier=FailureClassifier() + project_directory=PROJECT_ROOT, + failure_classifier=FailureClassifier(), + process_terminator=ProcessTerminator(), ) token = CancellationToken() @@ -106,7 +110,9 @@ def test_subprocess_executor_failure(tmp_path, test_case_id, step, stage, attemp Then the unsuccessful result retains the exit code and captured logs. """ executor = SubprocessExecutor( - project_directory=PROJECT_ROOT, failure_classifier=FailureClassifier() + project_directory=PROJECT_ROOT, + failure_classifier=FailureClassifier(), + process_terminator=ProcessTerminator(), ) artifact_manager = ArtifactManager(tmp_path) @@ -182,7 +188,9 @@ def test_subprocess_executor_polls_completed_subprocess( monkeypatch.setattr("runner.executor.subprocess.Popen", mocked_popen) executor = SubprocessExecutor( - project_directory=PROJECT_ROOT, failure_classifier=FailureClassifier() + project_directory=PROJECT_ROOT, + failure_classifier=FailureClassifier(), + process_terminator=ProcessTerminator(), ) token = CancellationToken() @@ -224,7 +232,7 @@ def test_subprocess_executor_polls_completed_subprocess( assert kwargs["errors"] == "replace" assert kwargs["bufsize"] == 1 - mocked_process.poll.assert_called_once_with() + mocked_process.poll.assert_called_with() mocked_process.wait.assert_not_called() assert result.exit_code == 0 @@ -267,9 +275,19 @@ def test_subprocess_executor_raised_timeout_error( mocked_process.stdout = StringIO("") mocked_process.stderr = StringIO("") - # poll() 回傳 None 表示程序仍在執行；wait() 用於終止後回收程序。 + # 終止前 poll() 回傳 None；terminator 負責更新終止後的狀態。 mocked_process.poll.return_value = None - mocked_process.wait.return_value = 0 + + mocked_terminator = Mock(spec=ProcessTerminator) + + def terminate_process_group(process): + # 模擬 process 收到 SIGTERM 後結束。 + # 被 signal 終止時，returncode 會是負的 signal 編號。 + process.returncode = -signal.SIGTERM + process.poll.return_value = process.returncode + + # Executor 呼叫清理時，就執行上面的函式更新假 process 的狀態。 + mocked_terminator.terminate_process_group.side_effect = terminate_process_group # 依序模擬開始時間、逾時檢查時間與結束時間，避免實際等待。 monkeypatch.setattr( @@ -298,7 +316,9 @@ def test_subprocess_executor_raised_timeout_error( ) executor = SubprocessExecutor( - project_directory=PROJECT_ROOT, failure_classifier=FailureClassifier() + project_directory=PROJECT_ROOT, + failure_classifier=FailureClassifier(), + process_terminator=mocked_terminator, ) with log_writer: @@ -315,7 +335,7 @@ def test_subprocess_executor_raised_timeout_error( assert result.attempt == 1 assert result.success is False - assert result.exit_code is None + assert result.exit_code == -signal.SIGTERM assert result.passed is False assert log_writer.stdout_path.read_text(encoding="utf-8") == result.stdout @@ -327,6 +347,4 @@ def test_subprocess_executor_raised_timeout_error( assert result.cancelled is False assert result.failure_type == FailureType.TIMEOUT assert result.error == f"Command timeout after {step.timeout_second}" - mocked_process.terminate.assert_called_once_with() - mocked_process.wait.assert_called_once_with(timeout=2) - mocked_process.kill.assert_not_called() + mocked_terminator.terminate_process_group.assert_called_once_with(mocked_process) diff --git a/tests/test_unit/test_process.py b/tests/test_unit/test_process.py index 84b390b..a0c8b60 100644 --- a/tests/test_unit/test_process.py +++ b/tests/test_unit/test_process.py @@ -1,8 +1,11 @@ +import signal import subprocess -import time +import sys + from runner.process import ProcessTerminator -def test_terminated_process_does_not_need_cleanup(): + +def test_terminated_process_does_not_need_cleanup(monkeypatch): process = subprocess.Popen( ["true"], @@ -11,6 +14,13 @@ def test_terminated_process_does_not_need_cleanup(): process.wait() + signal_calls = [] + + def fake_killpg(process_group_id, signal_number): + signal_calls.append((process_group_id, signal_number)) + + monkeypatch.setattr("runner.process.os.killpg", fake_killpg) + process_terminator = ProcessTerminator() result = process_terminator.terminate_process_group(process) @@ -18,19 +28,18 @@ def test_terminated_process_does_not_need_cleanup(): assert result.terminated is False assert result.killed is False assert result.return_code == 0 + assert signal_calls == [] + def test_process_group_terminates_gracefully(): process = subprocess.Popen( [ - "python", + sys.executable, "-c", - ( - "import time; " - "time.sleep(30)" - ), + ("import time; " "time.sleep(30)"), ], - start_new_session=True + start_new_session=True, ) process_terminator = ProcessTerminator(grace_period_seconds=1.0) @@ -41,6 +50,7 @@ def test_process_group_terminates_gracefully(): assert result.killed is False assert process.poll() is not None + def test_process_is_killed_when_sigterm_is_ignored(): command = ( @@ -55,14 +65,13 @@ def test_process_is_killed_when_sigterm_is_ignored(): process = subprocess.Popen( [ - "python", + sys.executable, "-c", command, ], - stdout=subprocess.PIPE, text=True, - start_new_session=True + start_new_session=True, ) # @@ -76,4 +85,60 @@ def test_process_is_killed_when_sigterm_is_ignored(): assert result.terminated is True assert result.killed is True - assert process.poll() is not None \ No newline at end of file + assert process.poll() is not None + + +def test_group_probe_permission_error_does_not_abort_cleanup(monkeypatch): + + # + # 檢查 process group 時，暫時沒有權限不代表 process 已經結束。 + # 應該繼續等待，直到確認群組不存在。 + # + class FakeProcess: + def __init__(self): + self.pid = 123 + self.returncode = None + self.poll_count = 0 + + def poll(self): + self.poll_count += 1 + + # 前兩次檢查還在執行，第三次才結束。 + if self.poll_count >= 3: + self.returncode = 0 + + return self.returncode + + process = FakeProcess() + + def fake_getpgid(pid): + assert pid == process.pid + return 123 + + signal_calls = [] + + def fake_killpg(process_group_id, signal_number): + signal_calls.append((process_group_id, signal_number)) + + # 第一次送出 SIGTERM 成功，直接返回。 + # 第二次檢查暫時沒有權限，第三次確認群組已不存在。 + if len(signal_calls) == 2: + raise PermissionError() + + if len(signal_calls) == 3: + raise ProcessLookupError() + + monkeypatch.setattr("runner.process.os.getpgid", fake_getpgid) + monkeypatch.setattr("runner.process.os.killpg", fake_killpg) + + process_terminator = ProcessTerminator(grace_period_seconds=1) + + result = process_terminator.terminate_process_group(process) + + assert result.terminated is True + assert result.killed is False + assert signal_calls == [ + (123, signal.SIGTERM), + (123, 0), + (123, 0), + ] diff --git a/tests/test_unit/test_runner.py b/tests/test_unit/test_runner.py index 7b5c74a..3df3c83 100644 --- a/tests/test_unit/test_runner.py +++ b/tests/test_unit/test_runner.py @@ -286,7 +286,7 @@ class MockLifecycleTrackingExecutor: def __init__(self): self.events: list[str] = [] - + def execute( self, step: LifecycleStepContent, @@ -304,7 +304,7 @@ def execute( # self.events.append(f"attempt_{attempt}_process_running") - + if attempt == 1: # @@ -313,7 +313,7 @@ def execute( self.events.append(f"attempt_{attempt}_timeout") - # + # # 這裡代表 Executor 內部： # # ProcessTerminator @@ -329,7 +329,7 @@ def execute( self.events.append(f"attempt_{attempt}_process_cleanup") - self.events.append(f"attempt_{attempt}_retrun") + self.events.append(f"attempt_{attempt}_return") return StepAttemptResult( attempt=attempt, @@ -341,8 +341,8 @@ def execute( duration_seconds=0.01, stdout="", stderr="", - stdout_log_path=log_writer.stdout_path, - stderr_log_path=log_writer.stderr_path, + stdout_log_path=str(log_writer.stdout_path), + stderr_log_path=str(log_writer.stderr_path), error="timeout", artifact_validation_results=[], ) @@ -2350,7 +2350,7 @@ def test_cancellation_lifecycle_and_summary( report = json.loads(report_path.read_text(encoding="utf-8")) assert report["metadata"]["cancel_requested"] is True - assert report["metadata"]["runner_version"] == "1.6.0" + assert report["metadata"]["runner_version"] == "1.6.1" assert report["summary"]["status"] == "CANCELLED" assert report["summary"]["cancelled_steps"] == expected_cancelled_steps assert report["summary"]["failed_steps"] == 0 @@ -2467,41 +2467,36 @@ def fake_monotonic(): assert result.step_results[0].cancelled is True assert result.summary.status == "CANCELLED" + def test_retry_starts_only_after_previous_attempt_cleanup(tmp_path: Path): config = RunnerConfig( - test_case=DeviceTestCase( - id="power_001", - name="power_001", - description="Description", - ), - device=DeviceInfo( - serial="device_001", - product="pixel", - build="build_001", - ), - retry=RetryConfig( - max_attempts=3, - delay_seconds=1, - retry_on=[ - FailureType.TIMEOUT + test_case=DeviceTestCase( + id="power_001", + name="power_001", + description="Description", + ), + device=DeviceInfo( + serial="device_001", + product="pixel", + build="build_001", + ), + retry=RetryConfig(max_attempts=3, delay_seconds=1, retry_on=[FailureType.TIMEOUT]), + lifecycle=LifecycleConfig( + global_setup=LifecycleSteps(steps=[]), + setup=LifecycleSteps(steps=[]), + scenario=LifecycleSteps( + steps=[ + mock_step("scenario"), ] ), - lifecycle=LifecycleConfig( - global_setup=LifecycleSteps(steps=[]), - setup=LifecycleSteps(steps=[]), - scenario=LifecycleSteps( - steps=[ - mock_step("scenario"), - ] - ), - teardown=LifecycleSteps(steps=[]), - global_teardown=LifecycleSteps(steps=[]), - ), - artifact=ArtifactConfig( - output_dir=str(tmp_path), - ), - ) + teardown=LifecycleSteps(steps=[]), + global_teardown=LifecycleSteps(steps=[]), + ), + artifact=ArtifactConfig( + output_dir=str(tmp_path), + ), + ) executor = MockLifecycleTrackingExecutor() @@ -2516,7 +2511,9 @@ def test_retry_starts_only_after_previous_attempt_cleanup(tmp_path: Path): result = runner.run(config) - scenario_result = next(step_result for step_result in result.step_results if step_result.name == "scenario") + scenario_result = next( + step_result for step_result in result.step_results if step_result.name == "scenario" + ) # # Attempt 1 timeout @@ -2540,15 +2537,14 @@ def test_retry_starts_only_after_previous_attempt_cleanup(tmp_path: Path): assert attempt_1_cleanup_index < attempt_2_start_index assert executor.events == [ - "attempt_1_start", - "attempt_1_process_running", - "attempt_1_timeout", - "attempt_1_process_cleanup", - "attempt_1_return", - - "attempt_2_start", - "attempt_2_process_running", - "attempt_2_success", - "attempt_2_process_cleanup", - "attempt_2_return", - ] \ No newline at end of file + "attempt_1_start", + "attempt_1_process_running", + "attempt_1_timeout", + "attempt_1_process_cleanup", + "attempt_1_return", + "attempt_2_start", + "attempt_2_process_running", + "attempt_2_success", + "attempt_2_process_cleanup", + "attempt_2_return", + ] diff --git a/tests/test_unit/test_signals.py b/tests/test_unit/test_signals.py new file mode 100644 index 0000000..bfa59de --- /dev/null +++ b/tests/test_unit/test_signals.py @@ -0,0 +1,35 @@ +import signal + +import pytest + +from main import SignalCancellationHandler +from runner.cancellation import CancellationToken + + +def test_first_sigint_cancels_token(): + + token = CancellationToken() + handler = SignalCancellationHandler(token) + + # 第一次 Ctrl+C：通知 runner 取消，讓它有機會清理 process。 + # 直接呼叫 handler，避免真的把 pytest 中斷。 + handler.handle_sigint(signal.SIGINT, None) + + assert token.is_cancelled is True + + +def test_second_sigint_raises_keyboard_interrupt(): + + token = CancellationToken() + handler = SignalCancellationHandler(token) + + # 第一次只發出取消通知。 + handler.handle_sigint(signal.SIGINT, None) + + # 第二次必須拋出 KeyboardInterrupt。 + # pytest.raises 會確認有沒有出現指定的 exception。 + with pytest.raises(KeyboardInterrupt): + handler.handle_sigint(signal.SIGINT, None) + + # 第二次中斷後，token 仍然維持取消狀態。 + assert token.is_cancelled is True From 933aa09e4535f1409018f23d891e3b889c39e64b Mon Sep 17 00:00:00 2001 From: Andrew Wang Date: Thu, 17 Sep 2026 22:50:36 +0800 Subject: [PATCH 5/9] add runner v1.6.1 docs --- CHANGELOG.md | 32 ++ CommitManual.md | 30 +- README.md | 56 ++- .../acceptance_criteria_v1.0.0.md | 2 +- .../acceptance_criteria_v1.1.0.md | 2 +- .../acceptance_criteria_v1.2.0.md | 2 +- .../acceptance_criteria_v1.3.0.md | 2 +- .../acceptance_criteria_v1.3.5.md | 2 +- .../acceptance_criteria_v1.4.0.md | 2 +- .../acceptance_criteria_v1.4.1.md | 2 +- .../acceptance_criteria_v1.5.0.md | 2 +- .../acceptance_criteria_v1.5.1.md | 2 +- .../acceptance_criteria_v1.5.2.md | 2 +- .../acceptance_criteria_v1.5.3.md | 2 +- .../acceptance_criteria_v1.6.0.md | 2 +- .../acceptance_criteria_v1.6.1.md | 31 ++ docs/architecture/architecture_v1.1.0.md | 15 +- docs/architecture/architecture_v1.2.0.md | 24 +- docs/architecture/architecture_v1.3.0.md | 16 +- docs/architecture/architecture_v1.3.5.md | 33 +- docs/architecture/architecture_v1.4.0.md | 28 +- docs/architecture/architecture_v1.4.1.md | 28 +- docs/architecture/architecture_v1.6.1.md | 151 ++++++++ .../definition_of_done_v1.6.1.md | 51 +++ docs/process_lifecycle.md | 133 +++++++ docs/roadmap.md | 113 +----- docs/test_guide.md | 361 ++++++++++++++++++ docs/test_matrix/test_matrix_v1.0.0.md | 4 + docs/test_matrix/test_matrix_v1.1.0.md | 4 + docs/test_matrix/test_matrix_v1.2.0.md | 4 + docs/test_matrix/test_matrix_v1.3.0.md | 4 + docs/test_matrix/test_matrix_v1.3.5.md | 4 + docs/test_matrix/test_matrix_v1.4.0.md | 4 + docs/test_matrix/test_matrix_v1.4.1.md | 4 + docs/test_matrix/test_matrix_v1.5.0.md | 4 + docs/test_matrix/test_matrix_v1.5.1.md | 4 + docs/test_matrix/test_matrix_v1.5.2.md | 4 + docs/test_matrix/test_matrix_v1.5.3.md | 4 + docs/test_matrix/test_matrix_v1.6.0.md | 4 + docs/test_matrix/test_matrix_v1.6.1.md | 231 +++++++++++ .../test_integration_cancellation.py | 24 ++ .../test_integration_retry.py | 6 + tests/test_unit/test_process.py | 24 ++ tests/test_unit/test_runner.py | 6 + tests/test_unit/test_signals.py | 12 + 45 files changed, 1208 insertions(+), 269 deletions(-) create mode 100644 docs/acceptance_criteria/acceptance_criteria_v1.6.1.md create mode 100644 docs/architecture/architecture_v1.6.1.md create mode 100644 docs/definition_of_done/definition_of_done_v1.6.1.md create mode 100644 docs/process_lifecycle.md create mode 100644 docs/test_guide.md create mode 100644 docs/test_matrix/test_matrix_v1.6.1.md diff --git a/CHANGELOG.md b/CHANGELOG.md index ea780d4..c3b21b8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,38 @@ - Edited versioned documentation for clearer headings, shorter explanations and readable acceptance conditions. - Expanded the commit checklist with concrete documentation and release checks. +## [1.6.1] + +Version scope: Safe Process Termination. Runtime is 1.6.1; distribution remains 1.6.0. This entry describes source changes, not a verified release. + +### Added + +- ProcessTerminator sends SIGTERM to an attempt process group, then SIGKILL after the grace period when needed. +- First SIGINT requests cancellation; the second raises KeyboardInterrupt. CLI run results use exit codes 0, 1 and 130. +- Real parent/child/grandchild tests cover timeout, cancellation, ignored SIGTERM, reader completion and saved stdout/stderr. +- Retry fixtures verify previous parent and child PIDs are absent when attempt two starts. +- Dedicated SIGINT handler tests and v1.6.1 architecture, test matrix, acceptance criteria and definition of done. + +### Changed + +- Executor starts a separate session for each attempt and requires an injected ProcessTerminator. +- Each output reader join has a two-second limit; cleanup failures can report PROCESS_ERROR or propagate if recovery also fails. +- Runtime and report metadata identify 1.6.1; result schema is unchanged from 1.6.0. +- Added Given/When/Then docstrings to 12 tests without changing their executable AST. + +### Fixed + +- Cleanup waits for the process group rather than treating parent exit as proof that descendants stopped. +- PermissionError during group probing no longer aborts the wait immediately. +- Retry tests use the correct fixture path, command quoting, lifecycle type, integer counter and artifact directory field. +- Executor timeout tests model the injected terminator; runner test log paths serialize as strings. + +### Verification and limitations + +- Local evidence and exact commands are recorded in [v1.6.1 Definition of Done](docs/definition_of_done/definition_of_done_v1.6.1.md). +- Detached descendants, normal completion with surviving children, Linux CI and end-to-end CLI signals remain unverified. SIGTERM is not wired to cancellation. +- Package version synchronization, tag, issue/milestone status and GitHub Release publication remain pending. + ## [1.6.0] Version scope: Cancellation Foundation. Runtime implementation is present; release tag and publication are not verified. diff --git a/CommitManual.md b/CommitManual.md index 188bfaf..29adae1 100644 --- a/CommitManual.md +++ b/CommitManual.md @@ -14,6 +14,7 @@ - [ ] 在 [CHANGELOG](CHANGELOG.md) 記錄使用者可觀察的變更。 - [ ] 更新完成條件，區分已驗證與尚未完成的項目。 - [ ] 更新測試矩陣，讓需求能對應到測試。 +- [ ] 更新 [測試指南](docs/test_guide.md) 的 pytest 用法大綱與對應版本案例；AI 新增或修改測試所使用的技巧也需納入。 - [ ] 更新驗收條件，寫出可確認的預期行為。 - [ ] 更新 [Roadmap](docs/roadmap.md)，同步已實作功能與後續規劃。 - [ ] 保留舊版文件的歷史內容；新版本依既有語言、標題、檔名與 Markdown 格式撰寫。 @@ -49,7 +50,8 @@ Codex 必須先完整閱讀本清單，並核對下列文件。`` 代 | --- | --- | | `README.md` | 使用方式、範例與限制 | | `docs/architecture/architecture_v.md` | 架構、介面、資料流，以及符合該版本實作的 UML 類別圖與循序圖 | -| `docs/test_matrix/test_matrix_v.md` | 測試覆蓋與證據 | +| `docs/test_matrix/test_matrix_v.md` | 測試覆蓋、證據與 AI 協作新增或補強的案例，連結至測試指南的對應版本 | +| `docs/test_guide.md` | Pytest 用法大綱、工具分類、版本索引與各版實際案例 | | `docs/acceptance_criteria/acceptance_criteria_v.md` | 驗收條件與結論 | | `docs/definition_of_done/definition_of_done_v.md` | 完成狀態與待辦項目 | | `docs/roadmap.md` | 版本進度與後續規劃 | @@ -58,6 +60,32 @@ Codex 必須先完整閱讀本清單，並核對下列文件。`` 代 文件任務只在需要時修改測試說明，不應為了更新文件改變產品行為。功能、測試數量、結果、tag、release 與 issue 狀態都必須有證據。未確認的發佈項目保持未勾選；Git tag 的存在不能證明 GitHub Release 已發布。 +### 測試指南與 test matrix 的分工 + +[測試指南](docs/test_guide.md) 集中說明測試怎麼寫、使用哪些 pytest 工具；各版 test matrix 說明測試涵蓋哪些需求、修正與限制。教學內容放在指南，矩陣連結至指南的對應版本，避免重複維護。 + +AI 更新測試文件時，需同步核對以下項目： + +- 維護指南的用法大綱與版本索引，依實際使用情況整理函式、fixture、Mock／Fake、參數化與執行方式。 +- 各版說明依對應 Git tag、commit 或明確工作版本撰寫，不將最新版用法套回舊版；保留歷史結果與日期。 +- 共通用法集中說明，各版只記錄差異與實際測試案例，避免重複教學。 +- 區分框架提供的工具、專案輔助程式與示意範例；未使用的功能不列為已實作，範例不算測試證據。 +- 新增版本或測試技巧時更新指南；只為補充說明時，不改變產品與測試行為。 + +#### AI 協作案例與修正紀錄 + +- 將 AI 新增、補強或修正的案例納入矩陣，分別寫出測試名稱、原本缺口、主要斷言與仍未涵蓋的情境。有協作紀錄才標示來源，不憑程式風格推測作者。 +- 重要修正需說明「原本為何失敗 → 修改了什麼 → 哪個測試防止再次發生」。區分產品程式的問題與測試資料、Mock 設定或路徑錯誤。 +- 說明測試的驗證範圍與限制，具體做法記錄在對應版本的 test matrix。 +- 保留既有程式風格，以容易閱讀的註解與範例解釋必要概念；不為了補文件改寫測試行為。 + +#### 文件檢查 + +- [ ] 測試路徑、函式名稱與 API 用法已對照該版本原始碼。 +- [ ] 範例語法、Markdown code fence、連結與執行指令已檢查。 +- [ ] 執行指令與實際執行結果分開記錄；未執行的測試不寫成通過。 +- [ ] 已說明模擬與真實操作的邊界，沒有把單元測試擴大解讀成端到端保證。 + ### 驗證與提交 1. 修改前執行 `pytest -q`，確認基準測試通過。 diff --git a/README.md b/README.md index 8b78ce8..ece6592 100644 --- a/README.md +++ b/README.md @@ -6,11 +6,11 @@ Device Test Runner 使用 YAML 定義裝置測試流程，執行既有的 Bash、Python、ADB 等命令，並保存每次執行的輸出與 JSON 報告。它負責安排測試步驟、驗證輸出檔案，以及依設定重試失敗步驟；裝置操作仍由你的腳本處理。 -目前版本為 **v1.6.0**，支援五階段測試流程、步驟逾時、選擇性重試、必要與選用的輸出檔案驗證，以及 Python API 取消。Recorder 管理與遠端執行仍在規劃中。 +目前 runtime／報告版本為 **v1.6.1**，支援五階段測試流程、步驟逾時、選擇性重試、輸出檔案驗證、程序群組清理與 Ctrl+C 取消。`pyproject.toml` 仍為 `1.6.0`，套件版本同步與發佈確認列於 [完成條件](docs/definition_of_done/definition_of_done_v1.6.1.md)。Recorder 管理與遠端執行仍在規劃中。 ## 安裝 -需要 Python 3.10+ 與 Poetry 2.x。在專案目錄執行： +需要 Python 3.10+ 與 Poetry 2.x。程序群組清理使用 POSIX signal API；目前本機驗證為 macOS，沒有 Windows 支援驗證。在專案目錄執行： ```bash git clone git@github.com:andrew771027/DeviceTestRunner.git @@ -77,7 +77,7 @@ global_teardown | `teardown` | 記錄失敗但繼續執行該 stage 的剩餘 steps，之後執行 `global_teardown` | | `global_teardown` | 記錄失敗但繼續執行該 stage 的剩餘 steps | -上述為一般失敗路由。v1.6.0 進入 setup 區塊後，即使 setup／scenario 取消，仍執行兩個 cleanup stages；run 開始前或 global_setup 取消時只執行 global_teardown。如果 global_setup 成功後、進入 setup 區塊前已觀察到取消，也會跳過 teardown。Cleanup attempt 使用新的 token；這些是受控流程的 best effort，不涵蓋未處理 Python exception 或 KeyboardInterrupt。 +上述為一般失敗路由。進入 setup 區塊後，即使 setup／scenario 取消，仍執行兩個 cleanup stages；run 開始前或 global_setup 取消時只執行 global_teardown。如果 global_setup 成功後、進入 setup 區塊前已觀察到取消，也會跳過 teardown。Cleanup attempt 使用新的 token；這些是受控流程的 best effort，不涵蓋未處理 Python exception 或 KeyboardInterrupt。 最終 `summary.status` 以 `CANCELLED` 優先；沒有取消時，failed step、skipped step 或 required artifact failure 任一存在即為 `FAILED`，其餘為 `PASSED`。 @@ -160,11 +160,13 @@ artifact: min_rows: 1 ``` -`after_step` 將 validation rule 綁定到指定 step，runner 會在該 step 每次 command 成功後立即驗證。`required` 預設為 `true`：required rule 失敗會使 attempt 失敗，且只有 failure type 出現在 `retry.retry_on`、尚未達 `max_attempts` 時才重試；`required: false` 的失敗仍寫入 report，但不影響 step 或 run 狀態。Lifecycle 結束後會再次對所有規則執行 final validation，包括有 `after_step` 的規則。YAML 未設定 `retry_on` 時預設為空清單，因此不重試；`none`、`cancelled` 與未知值會被拒絕。直接使用 Python `RetryConfig()` 的預設清單不同，詳見 v1.6.0 Architecture。 +`after_step` 將 validation rule 綁定到指定 step，runner 會在該 step 每次 command 成功後立即驗證。`required` 預設為 `true`：required rule 失敗會使 attempt 失敗，且只有 failure type 出現在 `retry.retry_on`、尚未達 `max_attempts` 時才重試；`required: false` 的失敗仍寫入 report，但不影響 step 或 run 狀態。Lifecycle 結束後會再次對所有規則執行 final validation，包括有 `after_step` 的規則。YAML 未設定 `retry_on` 時預設為空清單，因此不重試；`none`、`cancelled` 與未知值會被拒絕。直接使用 Python `RetryConfig()` 的預設清單不同，詳見 [v1.6.0 的設定說明](docs/architecture/architecture_v1.6.0.md)。 ## 透過 Python API 取消 -取消由呼叫端持有 token 並呼叫 `cancel()`；`main.py` 尚未將 Ctrl+C／SIGTERM 轉為 token cancellation。以下示範沿用載入的 configuration，在兩秒後提出取消請求： +CLI 第一次 Ctrl+C／SIGINT 會呼叫 `token.cancel()`；第二次會拋出 `KeyboardInterrupt`，可能中斷 cleanup 與報告寫入。CLI 正常完成回傳 0、FAILED 回傳 1、CANCELLED 或 run 期間的 KeyboardInterrupt 回傳 130。SIGTERM 尚未接到 token。 + +Python API 由呼叫端持有 token 並呼叫 `cancel()`。以下示範沿用載入的 configuration，在兩秒後提出取消請求： ```python from pathlib import Path @@ -176,13 +178,18 @@ from runner.cancellation import CancellationToken from runner.config import ConfigLoader from runner.executor import SubprocessExecutor from runner.failure import FailureClassifier +from runner.process import ProcessTerminator from runner.reporter import JsonReporter from runner.runner import DeviceTestRunner config = ConfigLoader().load("configs/sample.yaml") classifier = FailureClassifier() runner = DeviceTestRunner( - executor=SubprocessExecutor(Path.cwd(), classifier), + executor=SubprocessExecutor( + project_directory=Path.cwd(), + failure_classifier=classifier, + process_terminator=ProcessTerminator(), + ), artifact_manager=ArtifactManager(config.artifact.output_dir), artifact_validator=ArtifactValidator(), failure_classifier=classifier, @@ -200,9 +207,11 @@ finally: timer.join() ``` -Executor 以 0.1 秒 polling 檢查取消與 timeout，對直接子程序 terminate、等待兩秒後必要時 kill。尚未處理整棵 process tree，後代程序持有 stdout／stderr pipe 時，完成時間可能延長；上述 timer 不是兩秒內返回的保證。Cancelled attempt 不做 attempt validation 或 retry，但 run 最後仍驗證所有 artifacts。 +Executor 每 0.1 秒檢查取消與 timeout。每次 attempt 建立獨立 session，清理時對 process group 送 SIGTERM，預設等待 2 秒；仍有程序則送 SIGKILL，再等最多 2 秒。stdout／stderr reader 各有 2 秒 join 上限。上述 timer 不是兩秒內返回的保證。 + +同群組的 child／grandchild 已有清理測試；自行脫離群組的程序，以及直接 process 正常退出後留下的背景程序，不在目前保證內。詳見 [Process Lifecycle](docs/process_lifecycle.md)。Cancelled attempt 不做 attempt validation 或 retry，但 run 最後仍驗證所有 artifacts。 -升級至 v1.6.0 時，Python 呼叫端需調整 `SubprocessExecutor.execute(..., cancellation_token=...)`，以及手動建構 result dataclasses 時的新欄位。`run(config)` 仍可不傳 token。完整差異見 [Architecture](docs/architecture/architecture_v1.6.0.md)。 +從 v1.6.0 升級時，`SubprocessExecutor` 建構子需新增 `process_terminator`。從更舊版本升級時，Python 呼叫端也需調整 `SubprocessExecutor.execute(..., cancellation_token=...)`，以及手動建構 result dataclasses 時的新欄位。`run(config)` 仍可不傳 token。完整差異見 [Architecture](docs/architecture/architecture_v1.6.1.md)。 ## 輸出檔案 @@ -258,7 +267,7 @@ artifacts/ "device_serial": "demo", "device_product": "demo", "device_build": "demo", - "runner_version": "1.6.0", + "runner_version": "1.6.1", "started_at": "2026-09-12T00:00:00+00:00", "finished_at": "2026-09-12T00:00:01+00:00", "cancel_requested": false @@ -333,22 +342,6 @@ DeviceTestRunner result.json / per-attempt logs / validation results ``` -Device Test Runner 的核心資料流： - -```text -Config - ↓ -Runner - ↓ -Executor - ↓ -StepResult - ↓ -ArtifactManager - ↓ -RunResult / result.json -``` - ## 執行專案測試 執行所有測試： @@ -383,6 +376,8 @@ poetry run pytest -m artifact 歷史驗證紀錄（2026-09-12，Python 3.14）：`.venv/bin/python -m pytest -q` → **153 passed in 39.39s**。150 個測試函式皆有 Given／When／Then 說明；參數化後共 153 個案例。 +本次 v1.6.1 驗證與 162 個測試函式／168 個案例的對照，見 [完成條件](docs/definition_of_done/definition_of_done_v1.6.1.md) 與 [測試矩陣](docs/test_matrix/test_matrix_v1.6.1.md)。 + ## 持續整合 [CI workflow](.github/workflows/ci.yml) 會在 push 至 `main` 或建立以 `main` 為目標的 pull request 時執行。流程安裝 Python、Poetry 與專案依賴後，執行 pytest。 @@ -406,7 +401,7 @@ Repository secret 名稱為 `OPENAI_API_KEY`，在 CLI invocation 映射成 `COD 目前先補齊單機執行與取消流程，再加入可重用設定與 recorder 管理： -1. v1.6.1：程序群組終止、輸出串流收尾與 CLI 取消訊號。 +1. v1.6.1：已實作程序群組終止、輸出串流收尾與 SIGINT handler；平台驗證、版本同步與發佈待完成。 2. v1.6.2：整次 run 的逾時設定。 3. v1.6.3：取消後的清理範圍、時間限制與部分結果保存。 4. v1.7.0～v1.7.2：YAML 靜態變數、環境變數與執行資訊。 @@ -419,9 +414,10 @@ Repository secret 名稱為 `OPENAI_API_KEY`，在 CLI invocation 映射成 `COD | 文件 | 用途 | | --- | --- | -| [架構 v1.6.0](docs/architecture/architecture_v1.6.0.md) | 元件、取消路由、報告欄位與相容性 | -| [測試矩陣 v1.6.0](docs/test_matrix/test_matrix_v1.6.0.md) | 功能對應的測試與覆蓋限制 | -| [驗收條件 v1.6.0](docs/acceptance_criteria/acceptance_criteria_v1.6.0.md) | 可觀察的預期行為 | -| [完成條件 v1.6.0](docs/definition_of_done/definition_of_done_v1.6.0.md) | 驗證紀錄與待完成的發佈項目 | +| [架構 v1.6.1](docs/architecture/architecture_v1.6.1.md) | 元件、取消路由、報告欄位與相容性 | +| [測試指南](docs/test_guide.md) | Pytest 工具、fixture、Mock 與各版用法 | +| [測試矩陣 v1.6.1](docs/test_matrix/test_matrix_v1.6.1.md) | 功能對應的測試與覆蓋限制 | +| [驗收條件 v1.6.1](docs/acceptance_criteria/acceptance_criteria_v1.6.1.md) | 可觀察的預期行為 | +| [完成條件 v1.6.1](docs/definition_of_done/definition_of_done_v1.6.1.md) | 驗證紀錄與待完成的發佈項目 | `docs/` 內的舊版文件保留當時的設計與介面，使用時請確認版本。 diff --git a/docs/acceptance_criteria/acceptance_criteria_v1.0.0.md b/docs/acceptance_criteria/acceptance_criteria_v1.0.0.md index 4e48610..dc7905d 100644 --- a/docs/acceptance_criteria/acceptance_criteria_v1.0.0.md +++ b/docs/acceptance_criteria/acceptance_criteria_v1.0.0.md @@ -6,7 +6,7 @@ Basic YAML runner. Version baseline: Git tag `v1.0.0` -This document preserves the recorded acceptance state for this version. Test results and release checks below are historical records; they have not been rerun or reverified by this formatting update. +Historical acceptance record. Tests and release checks were not rerun for this editorial update. ## Acceptance criteria diff --git a/docs/acceptance_criteria/acceptance_criteria_v1.1.0.md b/docs/acceptance_criteria/acceptance_criteria_v1.1.0.md index b9191a7..dcb24d8 100644 --- a/docs/acceptance_criteria/acceptance_criteria_v1.1.0.md +++ b/docs/acceptance_criteria/acceptance_criteria_v1.1.0.md @@ -6,7 +6,7 @@ Naming and model refactoring. Version baseline: Git tag `v1.1.0` -This document preserves the recorded acceptance state for this version. Test results and release checks below are historical records; they have not been rerun or reverified by this formatting update. +Historical acceptance record. Tests and release checks were not rerun for this editorial update. ## Acceptance criteria diff --git a/docs/acceptance_criteria/acceptance_criteria_v1.2.0.md b/docs/acceptance_criteria/acceptance_criteria_v1.2.0.md index a4dca75..a4d0d31 100644 --- a/docs/acceptance_criteria/acceptance_criteria_v1.2.0.md +++ b/docs/acceptance_criteria/acceptance_criteria_v1.2.0.md @@ -6,7 +6,7 @@ Artifact management. Version baseline: Git tag `v1.2.0` -This document preserves the recorded acceptance state for this version. Test results and release checks below are historical records; they have not been rerun or reverified by this formatting update. +Historical acceptance record. Tests and release checks were not rerun for this editorial update. ## Acceptance criteria diff --git a/docs/acceptance_criteria/acceptance_criteria_v1.3.0.md b/docs/acceptance_criteria/acceptance_criteria_v1.3.0.md index 37feeb1..d8cc5c0 100644 --- a/docs/acceptance_criteria/acceptance_criteria_v1.3.0.md +++ b/docs/acceptance_criteria/acceptance_criteria_v1.3.0.md @@ -6,7 +6,7 @@ Test lifecycle. Version baseline: Git tag `v1.3.0` -This document preserves the recorded acceptance state for this version. Test results and release checks below are historical records; they have not been rerun or reverified by this formatting update. +Historical acceptance record. Tests and release checks were not rerun for this editorial update. ## Acceptance criteria diff --git a/docs/acceptance_criteria/acceptance_criteria_v1.3.5.md b/docs/acceptance_criteria/acceptance_criteria_v1.3.5.md index c1af31a..c28e363 100644 --- a/docs/acceptance_criteria/acceptance_criteria_v1.3.5.md +++ b/docs/acceptance_criteria/acceptance_criteria_v1.3.5.md @@ -6,7 +6,7 @@ Command output streaming. Version baseline: Git tag `v1.3.5` -This document preserves the recorded acceptance state for this version. Test results and release checks below are historical records; they have not been rerun or reverified by this formatting update. +Historical acceptance record. Tests and release checks were not rerun for this editorial update. ## Acceptance criteria diff --git a/docs/acceptance_criteria/acceptance_criteria_v1.4.0.md b/docs/acceptance_criteria/acceptance_criteria_v1.4.0.md index 21b1385..e3b164b 100644 --- a/docs/acceptance_criteria/acceptance_criteria_v1.4.0.md +++ b/docs/acceptance_criteria/acceptance_criteria_v1.4.0.md @@ -6,7 +6,7 @@ Artifact validation. Version baseline: Git tag `v1.4.0` -This document preserves the recorded acceptance state for this version. Test results and release checks below are historical records; they have not been rerun or reverified by this formatting update. +Historical acceptance record. Tests and release checks were not rerun for this editorial update. ## Acceptance criteria diff --git a/docs/acceptance_criteria/acceptance_criteria_v1.4.1.md b/docs/acceptance_criteria/acceptance_criteria_v1.4.1.md index c5fb903..1507075 100644 --- a/docs/acceptance_criteria/acceptance_criteria_v1.4.1.md +++ b/docs/acceptance_criteria/acceptance_criteria_v1.4.1.md @@ -6,7 +6,7 @@ CSV and JSON validation. Version baseline: Git tag `v1.4.1` -This document preserves the recorded acceptance state for this version. Test results and release checks below are historical records; they have not been rerun or reverified by this formatting update. +Historical acceptance record. Tests and release checks were not rerun for this editorial update. ## Acceptance criteria diff --git a/docs/acceptance_criteria/acceptance_criteria_v1.5.0.md b/docs/acceptance_criteria/acceptance_criteria_v1.5.0.md index dd143ee..7525ef2 100644 --- a/docs/acceptance_criteria/acceptance_criteria_v1.5.0.md +++ b/docs/acceptance_criteria/acceptance_criteria_v1.5.0.md @@ -6,7 +6,7 @@ Retry policy. Version baseline: Git tag `v1.5.0` -This document preserves the recorded acceptance state for this version. Test results and release checks below are historical records; they have not been rerun or reverified by this formatting update. +Historical acceptance record. Tests and release checks were not rerun for this editorial update. ## Acceptance criteria diff --git a/docs/acceptance_criteria/acceptance_criteria_v1.5.1.md b/docs/acceptance_criteria/acceptance_criteria_v1.5.1.md index 72f5251..546a222 100644 --- a/docs/acceptance_criteria/acceptance_criteria_v1.5.1.md +++ b/docs/acceptance_criteria/acceptance_criteria_v1.5.1.md @@ -6,7 +6,7 @@ Artifact-aware retry. Historical baseline: the working tree after tag `v1.5.0` (release candidate) -This document preserves the recorded acceptance state for this version. Test results and release checks below are historical records; they have not been rerun or reverified by this formatting update. +Historical acceptance record. Tests and release checks were not rerun for this editorial update. ## Acceptance criteria diff --git a/docs/acceptance_criteria/acceptance_criteria_v1.5.2.md b/docs/acceptance_criteria/acceptance_criteria_v1.5.2.md index 1d22986..963f15f 100644 --- a/docs/acceptance_criteria/acceptance_criteria_v1.5.2.md +++ b/docs/acceptance_criteria/acceptance_criteria_v1.5.2.md @@ -6,7 +6,7 @@ Failure Classification. Version baseline: implementation and verification record described below. -This document preserves the recorded acceptance state for this version. Test results and release checks below are historical records; they have not been rerun or reverified by this formatting update. +Historical acceptance record. Tests and release checks were not rerun for this editorial update. ## Acceptance criteria diff --git a/docs/acceptance_criteria/acceptance_criteria_v1.5.3.md b/docs/acceptance_criteria/acceptance_criteria_v1.5.3.md index c9c0ffe..59e84a3 100644 --- a/docs/acceptance_criteria/acceptance_criteria_v1.5.3.md +++ b/docs/acceptance_criteria/acceptance_criteria_v1.5.3.md @@ -6,7 +6,7 @@ Selective Retry and Artifact Criticality. Version baseline: implementation and verification record described below. -This document preserves the recorded acceptance state for this version. Test results and release checks below are historical records; they have not been rerun or reverified by this formatting update. +Historical acceptance record. Tests and release checks were not rerun for this editorial update. ## Acceptance criteria diff --git a/docs/acceptance_criteria/acceptance_criteria_v1.6.0.md b/docs/acceptance_criteria/acceptance_criteria_v1.6.0.md index e1297ad..81cfdb1 100644 --- a/docs/acceptance_criteria/acceptance_criteria_v1.6.0.md +++ b/docs/acceptance_criteria/acceptance_criteria_v1.6.0.md @@ -6,7 +6,7 @@ Cancellation Foundation. Version baseline: implementation and verification record described below. -This document preserves the recorded acceptance state for this version. Test results and release checks below are historical records; they have not been rerun or reverified by this formatting update. +Historical acceptance record. Tests and release checks were not rerun for this editorial update. ## Acceptance criteria diff --git a/docs/acceptance_criteria/acceptance_criteria_v1.6.1.md b/docs/acceptance_criteria/acceptance_criteria_v1.6.1.md new file mode 100644 index 0000000..561f422 --- /dev/null +++ b/docs/acceptance_criteria/acceptance_criteria_v1.6.1.md @@ -0,0 +1,31 @@ +# Device Test Runner v1.6.1 Acceptance Criteria + +## Scope + +Safe Process Termination on POSIX. Runtime is `1.6.1`; distribution version synchronization and release verification remain pending. + +## Acceptance criteria + +| ID | Given | When | Then | +| --- | --- | --- | --- | +| AC-1 | a direct process already exited | termination is requested | no group signal is sent; its return code is preserved. | +| AC-2 | a live attempt group accepts SIGTERM | cleanup starts | the group exits during the grace period without SIGKILL. | +| AC-3 | a process or its descendants ignore SIGTERM | the grace period expires | SIGKILL stops the remaining group members. | +| AC-4 | parent, child and grandchild share the attempt group | timeout or cancellation is observed | the three PIDs disappear; timeout and cancellation remain distinct. | +| AC-5 | the processes have written stdout and stderr | executor cleanup completes | both reader threads stop and result output matches the saved log files. | +| AC-6 | a first attempt times out with a child process | the second attempt starts | it immediately confirms both previous PIDs are absent before succeeding. | +| AC-7 | a token is cancelled with retries available | runner handles cancellation | no cancelled retry occurs; reachable teardown/global_teardown use active execution tokens. | +| AC-8 | the SIGINT handler has an active token | it receives its first invocation | it cancels the token without raising KeyboardInterrupt. | +| AC-9 | the handler already received one invocation | it is invoked again | it raises KeyboardInterrupt and the token stays cancelled. | + +## Verification and decision + +- [x] AC-1 through AC-9 map to the [Test Matrix](../test_matrix/test_matrix_v1.6.1.md). +- [x] Local test results and test-docstring checks are recorded in [Definition of Done](../definition_of_done/definition_of_done_v1.6.1.md). +- [x] Architecture includes process-group and retry sequence diagrams. +- [ ] Distribution and runtime versions are synchronized. +- [ ] Real CLI SIGINT delivery, exit codes and handler restoration are verified end to end. +- [ ] Ubuntu CI results and GitHub issue/milestone state are verified. +- [ ] v1.6.1 tag and GitHub Release publication are verified. + +The tested process-group behavior is accepted within the documented scope. Full release readiness is not established. AC-7 uses runner mocks; AC-8/9 directly invoke the handler. Neither proves finalization after a second real Ctrl+C. diff --git a/docs/architecture/architecture_v1.1.0.md b/docs/architecture/architecture_v1.1.0.md index c5e6bc2..ac53108 100644 --- a/docs/architecture/architecture_v1.1.0.md +++ b/docs/architecture/architecture_v1.1.0.md @@ -1251,20 +1251,7 @@ RunResult ## v1.1 架構價值 -v1.1 的主要價值不是單純把 YAML 改成 List。 - -真正的架構變化是：`一個大 Script` - -被拆成：`多個有名稱、有 timeout、有結果的 WorkflowStep` - -因此系統開始具備 Workflow Orchestration 的基本能力： - -* 步驟有順序 -* 步驟可以獨立執行 -* 步驟可以獨立紀錄 -* 步驟可以獨立判斷成功或失敗 -* Runner 可以控制失敗後的行為 -* RunResult 可以顯示具體失敗位置 +v1.1 將流程拆成有名稱、timeout 與結果的 WorkflowStep。Runner 依序執行步驟、處理失敗，RunResult 保留各步驟的結果。 ## v1.1 架構摘要 diff --git a/docs/architecture/architecture_v1.2.0.md b/docs/architecture/architecture_v1.2.0.md index de9bd05..dfcaab4 100644 --- a/docs/architecture/architecture_v1.2.0.md +++ b/docs/architecture/architecture_v1.2.0.md @@ -1726,27 +1726,7 @@ report content matches RunResult ## v1.2 的架構價值 -v1.1 已經可以正確執行 Workflow，但比較像：`執行工具` - -v1.2 開始變成：`可追蹤的 Test Runner` - -因為一個成熟的 Test Runner 不只需要知道：`現在執行成功或失敗` - -還需要回答： - -```text -是哪一個 Test Case？ -在哪一台 Device？ -使用哪一個 Build？ -什麼時間執行？ -每個 Step 執行多久？ -哪一個 Step 失敗？ -stdout 是什麼？ -stderr 是什麼？ -結果檔案放在哪裡？ -``` - -ArtifactManager 與 JsonReporter 讓這些資訊可以被保存與重現。 +ArtifactManager 與 JsonReporter 保存 test case、裝置、build、執行時間、步驟結果及 stdout／stderr，供執行後追查。 ## v1.2 的責任分離 @@ -1859,5 +1839,3 @@ flowchart TD Device Test Runner v1.2 的核心架構可以濃縮成： > `DeviceTestRunner` 依序執行 `WorkflowStep`，由 `CommandStepExecutor` 產生 `StepResult`；`ArtifactManager` 將每個 Step 的原始 stdout 與 stderr 保存到獨立 Run Directory；`JsonReporter` 再將 `RunnerConfig`、`RunResult` 與執行 metadata 彙整成 `report.json`。 - -v1.2 是 Device Test Runner 從「可以執行 Workflow」走向「可以保存、追蹤與分析每一次 Test Run」的重要版本。 diff --git a/docs/architecture/architecture_v1.3.0.md b/docs/architecture/architecture_v1.3.0.md index bf7f4b5..d0bc754 100644 --- a/docs/architecture/architecture_v1.3.0.md +++ b/docs/architecture/architecture_v1.3.0.md @@ -2450,21 +2450,7 @@ flowchart TD ## v1.3 的架構價值 -v1.3 的價值不只是多了五個欄位。 - -真正的改變是 Runner 開始理解： - -```text -哪些步驟是準備 -哪些步驟是主要測試 -哪些步驟是清理 -失敗後哪些步驟應停止 -哪些清理步驟仍必須執行 -``` - -這代表 Device Test Runner 從：`Sequential Command Runner` - -進一步成為：`Lifecycle-aware Test Runner` +Runner 區分準備、測試與清理階段，依失敗位置決定停止哪些步驟，以及繼續哪些清理工作。 ## v1.3 架構摘要 diff --git a/docs/architecture/architecture_v1.3.5.md b/docs/architecture/architecture_v1.3.5.md index c9bbfd4..5296b71 100644 --- a/docs/architecture/architecture_v1.3.5.md +++ b/docs/architecture/architecture_v1.3.5.md @@ -2004,7 +2004,7 @@ Runner 不需要知道 Executor 內部用了： * terminate * kill -這是封裝的重要價值。 +這讓呼叫端不必處理檔案細節。 ```text Runner knows WHAT happened. @@ -2358,7 +2358,7 @@ Mock 需要模擬行為，而不只是資料。 因此 v1.3.5 的 Executor Test 本質上開始接近：`Process lifecycle simulation` -這也是 v1.3.5 比 v1.3 更重要的學習價值。 + ## v1.3 與 v1.3.5 比較 @@ -2384,34 +2384,7 @@ Mock 需要模擬行為，而不只是資料。 ## v1.3.5 的架構價值 -v1.3.5 的價值不是單純把： - -```python -subprocess.run() -``` - -換成： - -```python -subprocess.Popen() -``` - -真正的架構變化是 Runner 開始處理一個外部 Process 的完整執行生命週期： - -```text -建立 Process -監控 Process -同時讀取兩條輸出流 -即時保存執行紀錄 -處理 timeout -終止 Process -等待 Reader 完成 -建立執行結果 -``` - -這使 Device Test Runner 從：`Command Wrapper` - -進一步成為：`Process-aware Execution Engine` +Executor 建立並監看 process，同時讀取 stdout／stderr、保存 log、處理 timeout，等待 readers 結束後建立結果。 ## v1.3.5 架構摘要 diff --git a/docs/architecture/architecture_v1.4.0.md b/docs/architecture/architecture_v1.4.0.md index 1b44e46..88f77d1 100644 --- a/docs/architecture/architecture_v1.4.0.md +++ b/docs/architecture/architecture_v1.4.0.md @@ -1585,7 +1585,7 @@ Validation Result 應該提供清楚的 error。 或者：`Artifact size below minimum: expected >= 1024 bytes, actual=128 bytes` -這會讓 `report.json` 本身具備 Debug 價值。 +可直接從 `report.json` 查閱失敗原因。 ## Artifact Path Resolution @@ -2400,31 +2400,7 @@ Run status reflects artifact validity? ## v1.4.0 的設計價值 -v1.3.5 之前，Runner 大部分依賴：`Process-level truth` - -具體規則： - -```text -exit code -stdout -stderr -timeout -``` - -v1.4.0 開始引入：`Test-output-level truth` - -具體規則： - -```text -expected artifact -actual artifact -validation rule -validation result -``` - -這表示 Test Runner 開始不再完全相信：`Process 說自己成功` - -而是會確認：`成功的證據到底有沒有產生` +Runner 除了檢查 exit code，也依 validation rules 檢查預期的 artifact，將驗證結果納入 run 狀態。 ## v1.3.5 與 v1.4.0 比較 diff --git a/docs/architecture/architecture_v1.4.1.md b/docs/architecture/architecture_v1.4.1.md index 6df5694..2055bac 100644 --- a/docs/architecture/architecture_v1.4.1.md +++ b/docs/architecture/architecture_v1.4.1.md @@ -1868,7 +1868,7 @@ assert ( ## Run Status Tests -最重要的四個 case： +四個主要 case： ```python @pytest.mark.parametrize( @@ -1970,7 +1970,7 @@ RunResult.status = FAILED ## Integration Test：Execution FAIL / Artifact Exists -另一個很有價值的案例： +另一個案例： ```bash #!/bin/bash @@ -2253,29 +2253,7 @@ How do I make execution and validation results consistent? ## v1.4.1 的架構價值 -v1.4.0 最大的功能突破是：`Runner 不再只相信 exit code。` - -v1.4.1 的最大架構突破則是：`Runner 開始有清楚的 Result Model hierarchy。` - -具體規則： - -```text -StepResult - ↓ -ExecutionSummary - -ArtifactValidationResult - ↓ -ValidationSummary - -ExecutionSummary -+ -ValidationSummary - ↓ -RunResult.status -``` - -每一層負責自己的 truth。 +StepResult 彙整為 ExecutionSummary，ArtifactValidationResult 彙整為 ValidationSummary，兩者共同決定 RunResult.status。 ## Result Hierarchy diff --git a/docs/architecture/architecture_v1.6.1.md b/docs/architecture/architecture_v1.6.1.md new file mode 100644 index 0000000..191833c --- /dev/null +++ b/docs/architecture/architecture_v1.6.1.md @@ -0,0 +1,151 @@ +# Device Test Runner Architecture v1.6.1 — Safe Process Termination + +本文件定義 v1.6.1 的程序清理架構，涵蓋 timeout、cancel、retry 與 SIGINT。介面、流程與限制以 v1.6.1 實作為準；後續版本的變更另立版本文件。 + +## 版本與介面 + +Runtime 與 `result.json` 的版本為 `1.6.1`。Result dataclass 欄位延續 v1.6.0，沒有新增 termination result 欄位到 JSON。套件版本同步與發佈狀態記錄於 [Definition of Done](../definition_of_done/definition_of_done_v1.6.1.md)。 + +`SubprocessExecutor(project_directory, failure_classifier, process_terminator)` 必須注入 `ProcessTerminator`。`execute(..., cancellation_token)` 與 `run(config, cancellation_token=None)` 的 token 用法不變。 + +```python +executor = SubprocessExecutor( + project_directory=project_root, + failure_classifier=FailureClassifier(), + process_terminator=ProcessTerminator(grace_period_seconds=2.0), +) +``` + +## Process 與 process group + +Process 是一個 OS 程序；process group 是一組可以一起接收 signal 的程序。Executor 用 `Popen(shell=True, start_new_session=True)` 啟動每次 attempt。沒有自行脫離群組的 child／grandchild 會留在同一個群組。 + +Command 在 run directory 執行，環境包含 `DEVICE_TEST_RUNNER_ROOT` 與 `RUN_ARTIFACT_DIR`。兩個 reader threads 分別保存 stdout 與 stderr。 + +每輪 polling 間隔為 0.1 秒，依序檢查： + +1. 直接 process 是否已完成。 +2. token 是否取消。 +3. 是否超過 step 的 `timeout_second`。 + +因此已完成的程序優先；仍執行中且取消與 timeout 同時可見時，取消優先。正常完成不會主動呼叫群組清理。 + +## ProcessTerminator + +`terminate_process_group(process)` 的流程： + +1. 如果直接 process 已退出，直接回傳，不發出 signal。 +2. 取得 PGID，對群組送出 SIGTERM。 +3. 預設每 0.05 秒 `poll()` 回收直接子程序，並以 `killpg(PGID, 0)` 檢查群組；預設寬限時間為 2 秒。 +4. 群組仍存在就送出 SIGKILL，再等最多 2 秒。 +5. SIGKILL 後仍未確認群組消失，拋出 `RuntimeError`。 + +`ProcessLookupError` 表示群組不存在。探測時的 `PermissionError` 不當成已結束，會繼續等到 deadline。這不代表送出 SIGTERM／SIGKILL 的權限錯誤也會被忽略。 + +回傳的 `ProcessTerminationResult` 包含 `terminated`、`killed`、`return_code`。`terminated` 表示進入終止流程；Executor 並未把這個物件寫入報告。 + +## Reader、retry 與清理 + +Executor 在清理後依序 join stdout／stderr reader，每個 reader 最多等 2 秒。仍存活就拋出 `RuntimeError`，避免正常路徑無限等待。這些分段等待不是整次 run 的總時間上限。 + +成功處理取消時，attempt 為 `CANCELLED`，不做 attempt validation 或 retry。Timeout 為 `TIMEOUT`，只有 retry policy 允許且尚有次數才重試。Runner 同步等待 `execute()` 回傳後，才判斷驗證、artifact cleanup、delay 與下一次 attempt。 + +Executor 捕捉 `OSError`／`RuntimeError` 後，會嘗試清理仍存活的直接 process、join readers，再回傳 `PROCESS_ERROR`。如果這段補救再次拋出例外，例外仍會離開 executor；不能宣稱所有錯誤都會留下完整報告。 + +## CLI 與 lifecycle + +* 第一次 Ctrl+C／SIGINT 呼叫 `token.cancel()`，走原本的受控取消流程。 +* 第二次呼叫 handler 會拋出 `KeyboardInterrupt`；`main()` 在 run 的 try 區塊捕捉後回傳 130，finally 還原原本 SIGINT handler。 +* 一般結果的 CLI exit code：PASSED 為 0、FAILED 為 1、CANCELLED 為 130。 +* SIGTERM 沒有接到 token。第二次 Ctrl+C 可能中斷 cleanup 與 JSON 寫入。 + +取消路由沿用 v1.6.0：pre-cancel 或 global_setup 取消只到 global_teardown；進入 setup 區塊後取消，仍到 teardown 與 global_teardown。Cleanup attempt 使用新的 token；retry delay 仍讀原始 run token。Final validation 仍驗證全部 artifact rules，summary 以 CANCELLED 優先。 + +## UML 類別圖 + +```mermaid +classDiagram + class SignalCancellationHandler { + +handle_sigint(signum, frame) + } + class CancellationToken { + +cancel() + +bool is_cancelled + } + class DeviceTestRunner { + +run(config, cancellation_token) RunResult + } + class SubprocessExecutor { + +execute(step, stage, attempt, log_writer, working_directory, cancellation_token) StepAttemptResult + } + class ProcessTerminator { + +terminate_process_group(process) ProcessTerminationResult + } + class ProcessTerminationResult { + +bool terminated + +bool killed + +int return_code + } + class StepLogWriter + class StepAttemptResult + class JsonReporter + SignalCancellationHandler --> CancellationToken : cancels + DeviceTestRunner --> SubprocessExecutor : executes attempts + DeviceTestRunner ..> CancellationToken : checks state + DeviceTestRunner --> JsonReporter : saves RunResult + SubprocessExecutor ..> CancellationToken : polls + SubprocessExecutor --> ProcessTerminator : delegates cleanup + SubprocessExecutor ..> StepLogWriter : writes output + SubprocessExecutor ..> StepAttemptResult : returns + ProcessTerminator ..> ProcessTerminationResult : returns +``` + +## Timeout／cancel 循序圖 + +圖中是直接 process 仍在執行，且清理成功的路徑。例外與已退出的 early return 見上文。 + +```mermaid +sequenceDiagram + participant R as DeviceTestRunner + participant E as SubprocessExecutor + participant T as ProcessTerminator + participant G as Attempt process group + participant L as stdout and stderr readers + R->>E: execute(attempt, token) + E->>G: Popen(start_new_session=True) + E->>L: Start two readers + E->>E: Observe cancel or timeout while running + E->>T: terminate_process_group(process) + T->>G: SIGTERM + loop Until group disappears or grace expires + T->>T: poll direct process + T->>G: killpg(PGID, 0) + end + opt Group still exists + T->>G: SIGKILL + T->>T: Poll and probe group for up to 2 seconds + end + T-->>E: ProcessTerminationResult + E->>L: join each reader, timeout=2 + E-->>R: StepAttemptResult + alt Cancelled + R->>R: Skip attempt validation and retry + else Timeout and retry allowed + R->>R: Clean required artifact targets and wait retry delay + opt Run token still active + R->>E: execute(next attempt) + end + end + R->>R: Reachable cleanup, final validation, save report +``` + +## 支援範圍與限制 + +* 使用 POSIX 的 `getpgid`、`killpg`、SIGTERM／SIGKILL；v1.6.1 的本機驗證環境為 macOS，沒有 Windows 或 Linux CI 成功證據。 +* 同群組的三層程序已有真實測試；自行 `setsid()`／脫離群組的 daemon 不在管理範圍內。 +* 直接 process 在清理開始前已退出時會 early return。正常完成但留下背景後代的情境，不在 v1.6.1 的清理保證內。 +* Zombie 回收、權限、OS 排程會影響群組消失時間；尚未驗證各平台的完整邊界。 +* SIGINT 測試直接呼叫 handler，沒有證明終端送訊號、handler 安裝／還原與 CLI exit code 的端到端行為。 +* Lifecycle 取消路由以 mock 驗證；未處理例外與第二次 Ctrl+C 不保證完成 teardown／report。 + +測試依據見 [Test Matrix](../test_matrix/test_matrix_v1.6.1.md)，發佈待辦見 [Definition of Done](../definition_of_done/definition_of_done_v1.6.1.md)。 diff --git a/docs/definition_of_done/definition_of_done_v1.6.1.md b/docs/definition_of_done/definition_of_done_v1.6.1.md new file mode 100644 index 0000000..3eb3f27 --- /dev/null +++ b/docs/definition_of_done/definition_of_done_v1.6.1.md @@ -0,0 +1,51 @@ +# Device Test Runner v1.6.1 Definition of Done + +Version scope: Safe Process Termination. Source baseline: `0dbdc51`, compared with Git tag `v1.6.0`. + +## Product and architecture + +- [x] Attempts start in separate sessions; timeout and cancellation delegate to ProcessTerminator. +- [x] SIGTERM grace period checks the group; surviving members receive SIGKILL. +- [x] Each stdout/stderr reader has a bounded join and raises if still alive. +- [x] Real retry checks previous parent and child PIDs before the next attempt does work. +- [x] First SIGINT cancels the token; second handler invocation raises KeyboardInterrupt. +- [x] Runtime/report version is `1.6.1`; v1.6.0 result fields are preserved. +- [x] POSIX, early-exit, detached-process and exception-finalization limits are documented. + +## Quality and documentation + +- [x] Unit and integration coverage is mapped in the Test Matrix. +- [x] All 162 test functions have Given/When/Then docstrings; parametrization produces 168 cases. +- [x] This documentation update adds 12 missing descriptions; AST comparison confirms executable tests are unchanged. +- [x] README, CHANGELOG, Roadmap, Process Lifecycle and four v1.6.1 documents reflect source behavior. +- [x] Architecture contains Mermaid class and sequence diagrams. +- [x] Older versioned documents and their historical verification records are preserved. + +### Local verification — 2026-09-17 + +Environment: macOS (`Darwin x86_64`), Python 3.14.0, repository `.venv`. This is local evidence, not proof of a successful Ubuntu/Python 3.12 CI run. + +| Check | Command / method | Observed result | +| --- | --- | --- | +| Before documentation edits | `.venv/bin/python -m pytest -q` | 168 passed in 38.12s | +| After documentation edits | `.venv/bin/python -m pytest -q` | 168 passed in 38.46s | +| Test descriptions | AST docstring audit and workflow-style Given/When/Then line counts | 162 functions; 162 Given, 162 When, 162 Then | +| Test behavior | Compare AST with docstrings removed before/after editing | Unchanged for all test files | +| Documentation | Local relative links, JSON example parsing and Python example compilation | 8 documents, 28 links checked; examples parse/compile | +| Whitespace | `git diff --check` | Passed | + +The sample configuration was not rerun during this documentation update. Its prior failure record remains in the [v1.6.0 Definition of Done](definition_of_done_v1.6.0.md); it is not current-version passing evidence. + +## Release and follow-up + +- [ ] Synchronize `pyproject.toml` distribution version (`1.6.0`) with runtime (`1.6.1`). This documentation task did not change package metadata. +- [ ] Verify a passing sample configuration. +- [ ] Verify Ubuntu CI and platform-specific process/zombie cleanup behavior. +- [ ] Add end-to-end CLI signal, exit-code and handler-restoration evidence. +- [ ] Define handling for detached descendants and normal completion with surviving background children. +- [ ] Verify cleanup/report finalization on unexpected exceptions or second Ctrl+C. +- [ ] Verify related GitHub issues and milestone completion. +- [ ] Create or verify tag `v1.6.1`; absent from inspected local tag refs on this update. +- [ ] Verify GitHub Release publication; not checked through a remote service. + +Release readiness: the documented process-group behavior is implemented and locally tested. The unchecked release and boundary items remain open; no commit, push, tag or release was performed by this update. diff --git a/docs/process_lifecycle.md b/docs/process_lifecycle.md new file mode 100644 index 0000000..e98eabb --- /dev/null +++ b/docs/process_lifecycle.md @@ -0,0 +1,133 @@ +# Device Test Runner Process Lifecycle + +Version: v1.6.1 + +## Purpose + +本文件說明一次 attempt 如何啟動、停止與收尾，以及清理和 retry 的關係。閱讀順序是「程序 → 群組 → 輸出 → 下一次 attempt」。類別介面與 UML 見 [Architecture](architecture/architecture_v1.6.1.md)，測試案例見 [Test Matrix](test_matrix/test_matrix_v1.6.1.md)。 + +## Process model + +Executor 用 `Popen(shell=True, start_new_session=True)` 執行命令，讓每次 attempt 有自己的 session 與 process group。沒有自行脫離群組的 child／grandchild，可以一起接收終止訊號。 + +```text +Device Test Runner +└── Attempt process group + └── shell / test script（parent） + └── child + └── grandchild +``` + +Parent／child 是誰啟動誰的關係；process group 是一起接收 signal 的範圍。`killpg` 操作的是群組，不是沿著 parent／child 關係搜尋所有後代。 + +| 元件 | 責任 | +| --- | --- | +| `DeviceTestRunner` | 安排 lifecycle、判斷 retry、驗證 artifacts 與寫入報告 | +| `SubprocessExecutor` | 啟動一次 attempt，監看完成／取消／timeout，收集輸出 | +| `ProcessTerminator` | 對該 attempt 的程序群組送訊號，等待群組退出 | +| stdout／stderr reader threads | 讀取外部程序的兩條輸出串流，保存到結果與 log | + +## Normal completion, timeout and cancellation + +Executor 每輪依序檢查 process 完成、token 取消、step timeout。正常 polling 間隔為 0.1 秒。 + +| 情境 | Executor 行為 | 後續結果 | +| --- | --- | --- | +| 直接 process 已完成 | 離開監看迴圈，收尾 reader；不主動清理群組 | 依 exit code 分類；runner 再處理 artifact validation 與 retry policy | +| 程序仍執行且 token 已取消 | 呼叫群組清理，再收尾 reader | 清理成功時為 CANCELLED，不 retry | +| 程序仍執行且已超過 timeout | 呼叫群組清理，再收尾 reader | 清理成功時為 TIMEOUT，符合 retry policy 才重試 | + +已觀察到完成的程序優先。如果程序仍執行中，取消與 timeout 同時可見，取消優先。以下流程描述清理成功的路徑；清理例外見後面的錯誤處理說明。 + +## Cleanup sequence + +```mermaid +flowchart TD + A[觀察到 cancel 或 timeout] --> B{清理開始時直接 process 已退出?} + B -- Yes --> C[Terminator 直接返回，不送訊號] + B -- No --> D[取得 PGID，對群組送 SIGTERM] + D --> E[回收直接子程序並檢查群組] + E --> F{寬限時間內群組消失?} + F -- Yes --> G[Terminator 返回] + F -- No --> H[對群組送 SIGKILL，再等最多 2 秒] + H --> I{群組已消失?} + I -- Yes --> G + I -- No --> J[拋出 RuntimeError] + C --> K[Executor 收尾 stdout 與 stderr readers] + G --> K + K --> L[返回 attempt 結果，Runner 判斷後續流程] +``` + +取得 PGID 或送出 signal 時，如果程序／群組已不存在，程式會處理 `ProcessLookupError`；不要求每次檢查之間程序狀態都保持不變。 + +### 為什麼不能只看 parent？ + +例如 parent 收到 SIGTERM 就結束，但 child 故意忽略 SIGTERM。此時 `process.poll()` 已經回傳 exit code，child 卻仍可能工作。 + +先前清理只等待直接 process 結束，會過早返回。現在進入群組終止流程後，會持續用 `os.killpg(PGID, 0)` 檢查群組，必要時升級 SIGKILL。這是本版的重要修正。 + +`signal 0` 只檢查存在與權限，不會終止程序。`ProcessLookupError` 表示群組不存在；探測遇到 `PermissionError` 則不能宣告清理完成，會繼續等待到 deadline。這項處理僅針對探測，不表示送出 SIGTERM／SIGKILL 的權限錯誤也會被忽略。 + +### 等待時間 + +| 設定／階段 | 目前值 | 說明 | +| --- | --- | --- | +| Step timeout | YAML 的 `timeout_second` | 命令執行逾時門檻，不包含完整清理完成的承諾 | +| Executor polling | 0.1 秒 | 檢查完成、取消與 timeout 的間隔 | +| SIGTERM grace period | 預設 2 秒 | 可由 `ProcessTerminator(grace_period_seconds=...)` 調整 | +| 群組探測間隔 | 預設 0.05 秒 | 可由 `poll_interval_seconds` 調整 | +| SIGKILL 後等待 | 最多 2 秒 | 目前寫在實作中，不是 YAML 設定 | +| Reader join | 每個最多 2 秒 | 依序等 stdout、stderr，並非兩個合計 2 秒 | + +這些是分段等待，不是整次 run 的總 deadline。OS 排程、清理例外、lifecycle cleanup 與報告寫入也會影響完成時間。 + +## Output readers and retry + +外部程序透過 pipe 把 stdout／stderr 交給 runner。Reader 等到資料讀完、輸出端關閉後才會結束。Child 若仍持有 pipe，parent 即使已退出，reader 仍可能等待。 + +因此成功清理路徑的順序是： + +1. 停止群組並確認它消失。 +2. 等 stdout／stderr reader 結束，保留已讀取內容。 +3. Executor 返回 attempt 結果。 +4. Runner 判斷是否重試，處理 required artifact targets 與 retry delay。 +5. 符合條件才開始下一次 attempt。 + +真實 retry 測試在 Attempt 2 一開始就確認前次 parent 與 child PID 不存在，不會先 sleep 等它們消失。三層程序測試另外確認兩個 reader 已停止，且 stdout／stderr log 與結果一致。 + +程序清理與 lifecycle 的 `teardown` 是不同層級。前者停止一次 attempt 的 OS 程序；後者執行使用者設定的清理命令，例如還原裝置狀態。取消後仍依 runner 的 lifecycle 路由執行可到達的 teardown／global_teardown，各 cleanup attempt 使用新的 token。 + +## Cleanup errors + +| 失敗位置 | 目前行為 | +| --- | --- | +| SIGKILL 後群組仍未消失 | Terminator 拋出 `RuntimeError("Process group did not exit after SIGKILL.")` | +| Reader join 後仍存活 | Executor 拋出 `RuntimeError("Output reader thread did not stop.")` | +| Executor 捕捉到 `OSError`／`RuntimeError` | 記錄錯誤，嘗試終止仍存活的直接 process 所屬群組並 join readers；補救成功後回傳 PROCESS_ERROR | +| 補救流程再次拋出例外 | 例外可能離開 executor，不能保證後續 teardown 或 JSON 報告完成 | + +因此 timeout／cancel 觸發清理，不代表最終一定保留 TIMEOUT／CANCELLED 分類；清理失敗可能變成 PROCESS_ERROR，或直接以例外中斷流程。 + +## Ctrl+C and cancellation + +第一次 Ctrl+C／SIGINT 呼叫 token.cancel，讓 executor 與 runner 走受控取消流程。第二次 handler 呼叫拋出 KeyboardInterrupt，可能中斷正在進行的清理與報告寫入。 + +`main.py` 在 run 的 try 區塊捕捉 KeyboardInterrupt 後回傳 130，並在 finally 還原原本 SIGINT handler。一般結果的 CLI exit code 為 PASSED → 0、FAILED → 1、CANCELLED → 130。 + +要區分兩個方向：Terminator 會送 SIGTERM 給 attempt 群組；但外部送 SIGTERM 給 runner 本身，目前沒有接成 token cancellation。 + +## Evidence and limits + +[Test Matrix](test_matrix/test_matrix_v1.6.1.md) 記錄 parent／child／grandchild、忽略 SIGTERM、reader 收尾、log 保存與 retry 前清理的證據。Fixture、monkeypatch 與模擬方式見 [Test Guide](test_guide.md#v1-6-1)。 + +目前限制： + +- 使用 POSIX process-group API；已有本機 macOS 測試紀錄，不能推論 Windows 支援或 Linux CI 已通過。 +- 自行建立新 session／脫離群組的後代不在清理範圍。 +- Terminator 開始時直接 process 已退出，會直接返回；這不保證其後代也已退出。 +- 正常完成後留下背景後代的情境，沒有主動清理保證。 +- 已退出但尚未回收的 zombie 可能仍保有 PID；群組消失時間受 OS 回收行為影響。 +- SIGINT 測試直接呼叫 handler，尚未驗證真實終端訊號、handler 還原與 CLI exit code 的完整流程。 +- 第二次 Ctrl+C、未處理例外與重複清理失敗，不保證完成 teardown 或保存完整報告。 + +本文件描述實作與既有測試證據，不新增測試執行紀錄；實際命令與結果見 [Definition of Done](definition_of_done/definition_of_done_v1.6.1.md)。 diff --git a/docs/roadmap.md b/docs/roadmap.md index 03c25be..8d11872 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -1,6 +1,6 @@ # Device Test Runner Roadmap -目前已實作 v1.6.0 的基本取消功能。接下來依序處理程序終止、run 逾時與取消後的清理，再加入 YAML 變數與 recorder 管理。 +目前已實作 v1.6.1 的程序群組終止、reader 收尾與 SIGINT handler，並通過本機測試。套件版本同步、平台驗證與發佈仍待完成。接下來處理 run 逾時與取消後的清理，再加入 YAML 變數與 recorder 管理。 本文件保留各版本的功能規劃。`Completed` 表示該節記錄的功能已完成；`Planned` 與 `Future` 表示尚未實作。實際發佈條件與驗證結果請見各版本的完成條件文件。 @@ -98,13 +98,7 @@ Device Test Runner 負責流程控制，但不應承擔所有硬體測試細節 * DeviceTestRunner orchestration * 基本 PASSED／FAILED 判定 -#### 相關技術 - -* Configuration loading -* Data model design -* subprocess -* orchestration basics -* input／output flow +相關技術：Configuration loading、Data model design、subprocess、orchestration basics、input／output flow。 #### 狀態 @@ -125,12 +119,7 @@ Completed * 更新 unit tests * 更新 integration tests -#### 相關技術 - -* Naming consistency -* Refactoring -* backward compatibility -* test maintenance +相關技術：Naming consistency、Refactoring、backward compatibility、test maintenance。 #### 狀態 @@ -164,13 +153,7 @@ artifacts/ └── stderr/ ``` -#### 相關技術 - -* Artifact ownership -* Output organization -* metadata design -* serialization -* separation of concerns +相關技術：Artifact ownership、Output organization、metadata design、serialization、separation of concerns。 #### 狀態 @@ -209,13 +192,7 @@ global_teardown * teardown execution foundation * lifecycle report structure -#### 相關技術 - -* Test lifecycle -* stage orchestration -* execution ordering -* state transitions -* status aggregation +相關技術：Test lifecycle、stage orchestration、execution ordering、state transitions、status aggregation。 #### 狀態 @@ -284,13 +261,7 @@ artifact: } ``` -#### 相關技術 - -* Validation abstraction -* policy separation -* domain-independent validation -* status aggregation -* post-execution verification +相關技術：Validation abstraction、policy separation、domain-independent validation、status aggregation、post-execution verification。 #### 狀態 @@ -336,13 +307,7 @@ retry: * validation failure * non-retryable configuration error -#### 相關技術 - -* Policy objects -* attempt tracking -* transient failure -* error classification -* idempotency +相關技術：Policy objects、attempt tracking、transient failure、error classification、idempotency。 #### 狀態 @@ -478,7 +443,7 @@ Foundation implemented and locally tested; release pending. 不將完整 Timeout v1.6.0 已有直接 `Popen` 程序的 terminate → 固定兩秒等待 → kill，以及正常路徑的 stdout／stderr thread join。本版本擴充到 process group、後代程序與各種結束路徑的收尾保證，不重做 cancellation token。 -目前使用 `shell=True`，沒有建立獨立 process group；reader join 沒有等待上限。Runner 雖然等待 executor 返回才 retry，但不代表前一次的後代程序已清乾淨；既有 retry cleanup 主要處理 required artifact targets。`main.py` 尚未接上 SIGINT。 +目前使用 `shell=True, start_new_session=True`，注入 `ProcessTerminator` 管理同群組的後代程序。SIGTERM 後確認整個群組退出，必要時 SIGKILL；每個 reader 的 join 上限為 2 秒。真實 retry 測試在第二次 attempt 開始時檢查前次 parent／child PID。`main.py` 已接上第一次 SIGINT 取消、第二次 KeyboardInterrupt。 #### 驗收重點 @@ -489,7 +454,11 @@ v1.6.0 已有直接 `Popen` 程序的 terminate → 固定兩秒等待 → kill #### 狀態 -Planned +Implemented; locally verified. Release pending. + +已驗證 timeout／cancel 下的 parent、child、grandchild 清理、忽略 SIGTERM 的後代、reader 結束與 log 落盤。SIGINT 目前是直接呼叫 handler 的單元測試。尚未驗證真實 CLI signal 路徑、Linux CI、脫離群組的後代，以及正常完成後留下背景程序的處理。 + +Runtime 為 `1.6.1`，套件仍為 `1.6.0`。完整結果與發佈待辦見 [v1.6.1 完成條件](definition_of_done/definition_of_done_v1.6.1.md)。 ### v1.6.2 — Run-level Timeout @@ -741,14 +710,7 @@ collect recorder artifacts * health command * process running check -#### 相關技術 - -* Background process -* process synchronization -* readiness -* start／stop lifecycle -* concurrent execution -* resource ownership +相關技術：Background process、process synchronization、readiness、start／stop lifecycle、concurrent execution、resource ownership。 #### 狀態 @@ -798,13 +760,7 @@ cleanup_errors: teardown command failed ``` -#### 相關技術 - -* try／finally -* failure preservation -* cleanup guarantees -* hooks -* multi-error reporting +相關技術：try／finally、failure preservation、cleanup guarantees、hooks、multi-error reporting。 #### 狀態 @@ -861,13 +817,7 @@ First Failure: - Exit Code: 1 ``` -#### 相關技術 - -* Aggregation -* reporting model -* observability -* diagnostics -* exit code design +相關技術：Aggregation、reporting model、observability、diagnostics、exit code design。 #### 狀態 @@ -996,15 +946,7 @@ Device + Recorder + Test Scripts * centralized report storage * execution history -#### 相關技術 - -* Distributed systems -* controller／worker architecture -* dispatch -* remote execution -* state machines -* failure recovery -* resource scheduling +相關技術：Distributed systems、controller／worker architecture、dispatch、remote execution、state machines、failure recovery、resource scheduling。 #### 狀態 @@ -1296,10 +1238,10 @@ Done ## 開發優先順序 -目前已實作並本機驗證 v1.6.0 cancellation foundation；完整取消保證與發佈仍待完成。接下來的開發優先順序： +目前已實作並本機驗證 v1.6.1 process-group cleanup；完整取消保證與發佈仍待完成。接下來的開發優先順序： ```text -1. v1.6.1 Safe Process Termination +1. v1.6.1 平台驗證、套件版本同步與發佈確認 2. v1.6.2 Run-level Timeout 3. v1.6.3 Cancellation-aware Cleanup 4. v1.7.x YAML Variables, Environment and Runtime Context @@ -1320,7 +1262,7 @@ Done * 過早的 microservices 拆分 * 完整 distributed scheduling -目前最重要的是先建立可靠的單機 execution lifecycle。 +先完成單機 execution lifecycle，再擴充遠端執行。 ## 長期方向 @@ -1345,18 +1287,3 @@ Controller ↓ Device Test Platform ``` - -這個專案同時也是以下能力的實作練習： - -* Python software design -* Test architecture -* Test lifecycle -* Orchestration -* Process management -* Artifact management -* Error handling -* Reliability -* Distributed systems -* Developer productivity -* Test infrastructure -* Platform engineering diff --git a/docs/test_guide.md b/docs/test_guide.md new file mode 100644 index 0000000..9a970ca --- /dev/null +++ b/docs/test_guide.md @@ -0,0 +1,361 @@ +# Pytest 測試指南 + +本文件說明 Device Test Runner 測試使用的 pytest 工具、fixture 與模擬方式。先從用法大綱找到需要的工具，再看各版本的實際案例。測試覆蓋哪些需求、重要修正與驗證結果，仍記錄在各版 [test matrix](test_matrix/)。 + +## 閱讀大綱 + +1. [工具與用法](#tools)：fixture、函式、decorator、Mock／Fake 與執行指令。 +2. [Fixture 與輔助腳本](#fixtures)：區分 pytest 提供的資源與測試自行啟動的腳本。 +3. [常用寫法](#usage)：共用語法與執行方式。 +4. [版本索引](#versions)：對照各版實際使用方式與來源。 + + + +## 工具與用法 + +下表提供閱讀入口，不表示每一版都使用全部工具。版本細節見後面的案例；教學範例不算新增測試。 + +| 工具／寫法 | 類型 | 用途 | 本專案的閱讀入口 | +| --- | --- | --- | --- | +| `assert` | Python 語法 | 比較實際與預期結果；pytest 顯示失敗位置與差異 | [v1.0.0](#v1-0-0) 起的案例 | +| `tmp_path` | Pytest 內建 fixture | 提供每個案例自己的暫存目錄 | [v1.0.0](#v1-0-0) | +| `monkeypatch`、`setattr()` | 內建 fixture 與方法 | 暫時替換函式／屬性，結束後還原 | [v1.2.0](#v1-2-0) | +| `unittest.mock.Mock` | Python 標準函式庫 | 建立可設定結果與記錄呼叫的假物件；不是 pytest fixture | [v1.2.0](#v1-2-0) | +| Fake／測試 helper | 專案自訂類別或函式 | 用簡單實作模擬依賴，或準備共用資料 | [v1.6.1](#v1-6-1) 的 FakeProcess | +| `@pytest.mark.parametrize` | Decorator | 同一段測試用不同資料執行 | [v1.0.0](#v1-0-0) | +| `capsys`、`readouterr()` | 內建 fixture 與方法 | 擷取 console 的 stdout／stderr | [v1.3.5](#v1-3-5) | +| `pytest.raises()` | 例外檢查工具 | 驗證操作拋出指定例外 | [v1.2.0](#v1-2-0)；[v1.6.1](#v1-6-1) 的 KeyboardInterrupt | +| 模擬時鐘 | 測試技巧 | 由測試推進時間，不必真的等待 | [v1.6.0](#v1-6-0) | +| `pytest.approx()` | 比較工具 | 容許浮點數計算的小誤差 | [v1.6.0](#v1-6-0) | +| `@pytest.mark.retry` 等 | 自訂 marker | 將測試分類，方便選擇執行 | [v1.5.0](#v1-5-0) | +| `-m`、`-k`、`檔案::函式` | 執行選項 | 分別依 marker、名稱或指定函式選擇測試 | 各版本的「執行與定位案例」 | +| `-q`、`-v` | 輸出選項 | 顯示精簡結果或逐一列出案例 | 各版本的執行範例 | + + + +## Fixture 與輔助腳本 + +Pytest fixture 是 pytest 準備後交給測試的資源。測試參數中的 `tmp_path`、`monkeypatch` 與 `capsys` 都是內建 fixture；不需要自己宣告 `@pytest.fixture`。 + +`@pytest.fixture` 用來定義自訂 fixture。本次核對的版本沒有這種自訂定義，因此不將它列為已使用功能。一般 helper 函式需要由程式呼叫，不會因為名稱有 mock 就自動注入測試。 + +`tests/fixtures/` 則是專案放置輔助腳本的資料夾。v1.6.1 的 `process_tree.py` 與 `retry_process.py` 由 subprocess 執行，並不是 pytest 自動注入的 fixture。詳細流程見 [v1.6.1](#v1-6-1)。 + + + +## 常用寫法 + +以下是語法示例，不是新增的專案測試。各版實際案例見後面的版本章節。 + +### 暫存目錄與參數化 + +Pytest 依參數名稱提供 `tmp_path`。`tmp_path: Path` 的型別標註只幫助閱讀，不負責提供資源。 + +```python +def test_write_output(tmp_path): + output_file = tmp_path / "output.txt" + output_file.write_text("done", encoding="utf-8") + assert output_file.read_text(encoding="utf-8") == "done" +``` + +`parametrize` 每列資料執行一次案例。資料參數不是內建 fixture；函式數也不一定等於案例數。 + +```python +import pytest + +@pytest.mark.parametrize( + argnames="exit_code, expected_success", + argvalues=[(0, True), (1, False)], +) +def test_exit_code_example(exit_code, expected_success): + assert (exit_code == 0) is expected_success +``` + +### 替換函式與模擬依賴 + +`monkeypatch` 是 fixture，`setattr()` 是它的方法。替換的是程式查找函式的位置，測試結束後會自動還原。 + +```python +monkeypatch.setattr(module_object, "function_name", fake_function) +monkeypatch.setattr("package.module.function_name", fake_function) +``` + +以上名稱需換成實際物件或路徑。假函式必須接受正式呼叫使用的參數。 + +Mock 建立可設定回傳值、例外與呼叫紀錄的假物件；monkeypatch 將程式使用的位置換成它。也可以用一般函式、簡單類別與 list，像 v1.6.1 的 `FakeProcess`。 + +若替換函式先記錄物件，再呼叫原本函式，測試仍會執行真實操作。因此使用 monkeypatch 不等於完全模擬 OS 行為。 + +### 輸出、例外與時間 + +| 工具 | 寫法與檢查方式 | +| --- | --- | +| `capsys` | `captured = capsys.readouterr()` 後檢查 `captured.out`／`captured.err`；log 檔案另以 `read_text()` 驗證 | +| `pytest.raises` | `with pytest.raises(ExpectedError):` 的區塊必須拋出指定例外，沒有拋出就失敗 | +| 模擬 sleep | 假函式記錄等待秒數，不真的睡眠；適合檢查單次等待請求 | +| 模擬時鐘 | 同時替換 sleep 與 clock，讓假 sleep 推進時間，clock 回傳新值；適合 deadline／polling 測試 | +| `pytest.approx` | 比較浮點值是否足夠接近，避免把多次 0.1 秒累加的精度誤差當成失敗 | + +### 選擇測試與閱讀結果 + +在對應版本的 checkout、已安裝依賴的環境執行： + +```bash +python -m pytest -q +python -m pytest path/to/test_file.py::test_function -v +python -m pytest -m retry -v +python -m pytest -k timeout -v +``` + +第二行的路徑與函式需換成實際案例。`-m` 依 marker 分類篩選，`-k` 依名稱等關鍵字篩選，`-q` 精簡輸出，`-v` 列出各案例。指定參數化函式時，會執行其所有資料列。 + +不是每個 retry 測試都有 `retry` marker。完整功能驗證應依 test matrix 選擇相關檔案或跑完整 suite。 + + + +## 版本索引 + +舊版依對應 Git tag 的測試說明；開發中版本依該節記錄的 source 基準。路徑或介面可能隨版本改變，請勿直接用目前程式推論舊版行為。本次搬移文件沒有重新執行歷史測試。 + +| 版本 | 閱讀重點 | 測試覆蓋與證據 | +| --- | --- | --- | +| [v1.0.0](#v1-0-0) | tmp_path、參數化與基本斷言 | [Test matrix](test_matrix/test_matrix_v1.0.0.md) | +| [v1.1.0](#v1-1-0) | 模型更名後的案例與參數化 | [Test matrix](test_matrix/test_matrix_v1.1.0.md) | +| [v1.2.0](#v1-2-0) | monkeypatch.setattr、Mock、TimeoutExpired | [Test matrix](test_matrix/test_matrix_v1.2.0.md) | +| [v1.3.0](#v1-3-0) | Lifecycle 與 artifact 測試的準備資料 | [Test matrix](test_matrix/test_matrix_v1.3.0.md) | +| [v1.3.5](#v1-3-5) | capsys 與輸出串流測試 | [Test matrix](test_matrix/test_matrix_v1.3.5.md) | +| [v1.4.0](#v1-4-0) | Artifact validation 的檔案隔離 | [Test matrix](test_matrix/test_matrix_v1.4.0.md) | +| [v1.4.1](#v1-4-1) | CSV／JSON 驗證案例與既有 pytest 工具 | [Test matrix](test_matrix/test_matrix_v1.4.1.md) | +| [v1.5.0](#v1-5-0) | Retry marker、pytest.raises 與替換 sleep | [Test matrix](test_matrix/test_matrix_v1.5.0.md) | +| [v1.5.1](#v1-5-1) | Artifact-aware retry 的分類與案例路徑 | [Test matrix](test_matrix/test_matrix_v1.5.1.md) | +| [v1.5.2](#v1-5-2) | Failure classification 的測試用法 | [Test matrix](test_matrix/test_matrix_v1.5.2.md) | +| [v1.5.3](#v1-5-3) | Selective retry 與設定錯誤測試 | [Test matrix](test_matrix/test_matrix_v1.5.3.md) | +| [v1.6.0](#v1-6-0) | 模擬時鐘、pytest.approx 與取消測試 | [Test matrix](test_matrix/test_matrix_v1.6.0.md) | +| [v1.6.1](#v1-6-1) | Process fixture 腳本、FakeProcess、四組參數與 SIGINT | [Test matrix](test_matrix/test_matrix_v1.6.1.md) | + + + + +## v1.0.0 + +依據 Git tag `v1.0.0`；覆蓋範圍見 [Test matrix](test_matrix/test_matrix_v1.0.0.md)。 + +本版沿用暫存目錄與參數化，使用方式見上方「常用寫法」。 + +| 實際案例 | 使用方式 | +| --- | --- | +| `tests/test_config_loader.py::test_config_loader_loads_yaml` | tmp_path | +| `tests/test_runner.py::test_runner_runs_single_scenario` | 參數化／測試資料準備 | + + + +## v1.1.0 + +依據 Git tag `v1.1.0`；覆蓋範圍見 [Test matrix](test_matrix/test_matrix_v1.1.0.md)。 + +本版沿用暫存目錄與參數化，使用方式見上方「常用寫法」。 + +| 實際案例 | 使用方式 | +| --- | --- | +| `tests/test_contract_config_yaml.py::test_config_yaml_contract` | tmp_path | +| `tests/test_command_executor.py::test_command_step_executor_success` | 參數化／測試資料準備 | + + + +## v1.2.0 + +依據 Git tag `v1.2.0`；覆蓋範圍見 [Test matrix](test_matrix/test_matrix_v1.2.0.md)。 + +本版 executor 測試使用 `monkeypatch.setattr(subprocess, "run", ...)`，替換 `subprocess.run`。成功案例回傳準備好的 `CompletedProcess`；逾時案例由假函式拋出 `TimeoutExpired`，不用真的等命令逾時。 + +本版 timeout 測試預期 `subprocess.TimeoutExpired` 往外拋。後續版本改成回傳失敗結果，因此不要把這個舊版預期直接套到新版 executor。 + +| 實際案例 | 使用方式 | +| --- | --- | +| `tests/test_config_loader.py::test_config_loader_loads_device_test_config` | tmp_path | +| `tests/test_executor.py::test_subprocess_executor_passes_timeout_to_subprocess` | monkeypatch、Mock | +| `tests/test_executor.py::test_subprocess_executor_return_success` | 參數化／測試資料準備 | +| `tests/test_executor.py::test_subprocess_executor_raised_timeout_error` | 參數化／測試資料準備 | + + + +## v1.3.0 + +依據 Git tag `v1.3.0`；覆蓋範圍見 [Test matrix](test_matrix/test_matrix_v1.3.0.md)。 + +本版 executor 測試使用 `monkeypatch.setattr(subprocess, "run", ...)`，替換 `subprocess.run`。成功案例回傳準備好的 `CompletedProcess`；逾時案例由假函式拋出 `TimeoutExpired`，不用真的等命令逾時。 + +| 實際案例 | 使用方式 | +| --- | --- | +| `tests/test_artifact.py::test_save_step_output` | 參數化／測試資料準備 | +| `tests/test_executor.py::test_subprocess_executor_passes_timeout_to_subprocess` | monkeypatch、Mock | + + + +## v1.3.5 + +依據 Git tag `v1.3.5`；覆蓋範圍見 [Test matrix](test_matrix/test_matrix_v1.3.5.md)。 + +本版 executor 測試替換 `subprocess.Popen`，用假 process 與串流模擬命令結果。這能檢查 executor 如何使用 subprocess，但不能證明 OS 真的建立或停止了程序。 + +| 實際案例 | 使用方式 | +| --- | --- | +| `tests/test_artifact.py::test_save_log_writer_step_stdout_and_stderr` | 參數化／測試資料準備 | +| `tests/test_executor.py::test_subprocess_executor_passes_timeout_to_subprocess` | monkeypatch、Mock | +| `tests/test_console_output.py::test_writer_displays_stdout_on_console` | tmp_path、capsys、parametrize | + + + +## v1.4.0 + +依據 Git tag `v1.4.0`；覆蓋範圍見 [Test matrix](test_matrix/test_matrix_v1.4.0.md)。 + +本版 executor 測試替換 `subprocess.Popen`，用假 process 與串流模擬命令結果。這能檢查 executor 如何使用 subprocess，但不能證明 OS 真的建立或停止了程序。 + +| 實際案例 | 使用方式 | +| --- | --- | +| `tests/test_artifact.py::test_save_log_writer_step_stdout_and_stderr` | 參數化／測試資料準備 | +| `tests/test_executor.py::test_subprocess_executor_passes_timeout_to_subprocess` | monkeypatch、Mock | +| `tests/test_console_output.py::test_writer_displays_stdout_on_console` | tmp_path、capsys、parametrize | + + + +## v1.4.1 + +依據 Git tag `v1.4.1`；覆蓋範圍見 [Test matrix](test_matrix/test_matrix_v1.4.1.md)。 + +本版 executor 測試替換 `subprocess.Popen`，用假 process 與串流模擬命令結果。這能檢查 executor 如何使用 subprocess，但不能證明 OS 真的建立或停止了程序。 + +| 實際案例 | 使用方式 | +| --- | --- | +| `tests/test_artifact.py::test_save_log_writer_step_stdout_and_stderr` | 參數化／測試資料準備 | +| `tests/test_executor.py::test_subprocess_executor_passes_timeout_to_subprocess` | monkeypatch、Mock | +| `tests/test_console_output.py::test_writer_displays_stdout_on_console` | tmp_path、capsys、parametrize | + + + +## v1.5.0 + +依據 Git tag `v1.5.0`；覆蓋範圍見 [Test matrix](test_matrix/test_matrix_v1.5.0.md)。 + +本版 executor 測試替換 `subprocess.Popen`，用假 process 與串流模擬命令結果。這能檢查 executor 如何使用 subprocess，但不能證明 OS 真的建立或停止了程序。 + +測試把 `runner.runner.time.sleep` 換成 `fake_sleep(seconds)`，假函式只把秒數放進 list。最後檢查 `sleep_calls == [2]`，證明 runner 要求等待兩秒，而不需要真的暫停兩秒。這是該版本的單次 sleep 實作；後續 polling 版本的斷言不同。 + +| 實際案例 | 使用方式 | +| --- | --- | +| `tests/test_artifact.py::test_save_log_writer_step_stdout_and_stderr` | 參數化／測試資料準備 | +| `tests/test_executor.py::test_subprocess_executor_passes_timeout_to_subprocess` | monkeypatch、Mock | +| `tests/test_console_output.py::test_writer_displays_stdout_on_console` | tmp_path、capsys、parametrize | +| `tests/test_config_loader.py::test_retry_max_attempts_must_be_positive` | pytest.raises | +| `tests/test_runner.py::test_retry_waits_between_attempts` | monkeypatch 替換 sleep | + +自訂 marker：`test_lifecycle`, `artifact`, `retry`。 + + + +## v1.5.1 + +依據 Git tag `v1.5.1`；覆蓋範圍見 [Test matrix](test_matrix/test_matrix_v1.5.1.md)。 + +本版 executor 測試替換 `subprocess.Popen`，用假 process 與串流模擬命令結果。這能檢查 executor 如何使用 subprocess，但不能證明 OS 真的建立或停止了程序。 + +測試把 `runner.runner.time.sleep` 換成 `fake_sleep(seconds)`，假函式只把秒數放進 list。最後檢查 `sleep_calls == [2]`，證明 runner 要求等待兩秒，而不需要真的暫停兩秒。這是該版本的單次 sleep 實作；後續 polling 版本的斷言不同。 + +| 實際案例 | 使用方式 | +| --- | --- | +| `tests/test_integration/test_integration.py::test_yaml_to_result_json` | tmp_path | +| `tests/test_unit/test_executor.py::test_subprocess_executor_passes_timeout_to_subprocess` | monkeypatch、Mock | +| `tests/test_unit/test_console_output.py::test_writer_displays_stdout_on_console` | tmp_path、capsys、parametrize | +| `tests/test_unit/test_artifact.py::test_save_log_writer_step_stdout_and_stderr` | 參數化／測試資料準備 | +| `tests/test_unit/test_config_loader.py::test_retry_max_attempts_must_be_positive` | pytest.raises | +| `tests/test_unit/test_runner.py::test_retry_waits_between_attempts` | monkeypatch 替換 sleep | + +自訂 marker：`test_lifecycle`, `artifact`, `retry`。 + + + +## v1.5.2 + +依據 Git tag `v1.5.2`；覆蓋範圍見 [Test matrix](test_matrix/test_matrix_v1.5.2.md)。 + +本版 executor 測試替換 `subprocess.Popen`，用假 process 與串流模擬命令結果。這能檢查 executor 如何使用 subprocess，但不能證明 OS 真的建立或停止了程序。 + +測試把 `runner.runner.time.sleep` 換成 `fake_sleep(seconds)`，假函式只把秒數放進 list。最後檢查 `sleep_calls == [2]`，證明 runner 要求等待兩秒，而不需要真的暫停兩秒。這是該版本的單次 sleep 實作；後續 polling 版本的斷言不同。 + +| 實際案例 | 使用方式 | +| --- | --- | +| `tests/test_integration/test_integration.py::test_yaml_to_result_json` | tmp_path | +| `tests/test_unit/test_executor.py::test_subprocess_executor_passes_timeout_to_subprocess` | monkeypatch、Mock | +| `tests/test_unit/test_console_output.py::test_writer_displays_stdout_on_console` | tmp_path、capsys、parametrize | +| `tests/test_unit/test_artifact.py::test_save_log_writer_step_stdout_and_stderr` | 參數化／測試資料準備 | +| `tests/test_unit/test_config_loader.py::test_retry_max_attempts_must_be_positive` | pytest.raises | +| `tests/test_unit/test_runner.py::test_retry_waits_between_attempts` | monkeypatch 替換 sleep | + +自訂 marker：`test_lifecycle`, `artifact`, `retry`。 + + + +## v1.5.3 + +依據 Git tag `v1.5.3`；覆蓋範圍見 [Test matrix](test_matrix/test_matrix_v1.5.3.md)。 + +本版 executor 測試替換 `subprocess.Popen`，用假 process 與串流模擬命令結果。這能檢查 executor 如何使用 subprocess，但不能證明 OS 真的建立或停止了程序。 + +測試把 `runner.runner.time.sleep` 換成 `fake_sleep(seconds)`，假函式只把秒數放進 list。最後檢查 `sleep_calls == [2]`，證明 runner 要求等待兩秒，而不需要真的暫停兩秒。這是該版本的單次 sleep 實作；後續 polling 版本的斷言不同。 + +| 實際案例 | 使用方式 | +| --- | --- | +| `tests/test_integration/test_integration.py::test_yaml_to_result_json` | tmp_path | +| `tests/test_unit/test_executor.py::test_subprocess_executor_passes_timeout_to_subprocess` | monkeypatch、Mock | +| `tests/test_unit/test_console_output.py::test_writer_displays_stdout_on_console` | tmp_path、capsys、parametrize | +| `tests/test_unit/test_artifact.py::test_save_log_writer_step_stdout_and_stderr` | 參數化／測試資料準備 | +| `tests/test_unit/test_config_loader.py::test_retry_max_attempts_must_be_positive` | pytest.raises | +| `tests/test_unit/test_runner.py::test_retry_waits_between_attempts` | monkeypatch 替換 sleep | + +自訂 marker：`test_lifecycle`, `artifact`, `retry`。 + + + +## v1.6.0 + +依據 Git tag `v1.6.0`；覆蓋範圍見 [Test matrix](test_matrix/test_matrix_v1.6.0.md)。 + +本版 executor 測試替換 `runner.executor.subprocess.Popen` 與 `time.perf_counter`，模擬程序仍在執行、時間已超過 deadline 的情境。測試不必真的等完整 timeout。 + +此版 delay 會反覆檢查時間與取消狀態，因此測試同時替換 `time.sleep` 與 `time.monotonic`。模擬的 sleep 只推進測試時間；模擬時鐘回報推進後的值，不必真的等待。只替換 sleep 卻不推進時鐘，可能讓等待迴圈無法按預期結束。 + +| 實際案例 | 使用方式 | +| --- | --- | +| `tests/test_integration/test_integration.py::test_yaml_to_result_json` | tmp_path | +| `tests/test_unit/test_executor.py::test_subprocess_executor_polls_completed_subprocess` | monkeypatch、Mock | +| `tests/test_unit/test_console_output.py::test_writer_displays_stdout_on_console` | tmp_path、capsys、parametrize | +| `tests/test_unit/test_artifact.py::test_save_log_writer_step_stdout_and_stderr` | 參數化／測試資料準備 | +| `tests/test_unit/test_cancellation.py::test_raise_if_cancelled_raises_after_cancel` | pytest.raises | +| `tests/test_unit/test_runner.py::test_retry_waits_between_attempts` | 模擬時鐘與 `pytest.approx` | + +自訂 marker：`cancelled`, `test_lifecycle`, `artifact`, `retry`。 + + + +## v1.6.1 + +依據 v1.6.1 工作版本；覆蓋範圍見 [Test matrix](test_matrix/test_matrix_v1.6.1.md)。 + +本版 executor 測試替換 `runner.executor.subprocess.Popen` 與 `time.perf_counter`，模擬程序仍在執行、時間已超過 deadline 的情境。測試不必真的等完整 timeout。 + +`test_second_sigint_raises_keyboard_interrupt` 使用 `pytest.raises(KeyboardInterrupt)` 驗證第二次 SIGINT handler 呼叫。這是直接呼叫函式，沒有真的向 pytest 發送 Ctrl+C。 + +此版 delay 會反覆檢查時間與取消狀態，因此測試同時替換 `time.sleep` 與 `time.monotonic`。模擬的 sleep 只推進測試時間；模擬時鐘回報推進後的值，不必真的等待。只替換 sleep 卻不推進時鐘，可能讓等待迴圈無法按預期結束。 + +`test_executor_cleans_entire_tree_and_drains_readers` 的四列參數是 timeout/cancel × default/ignore。它使用 monkeypatch 包住真正的 `Popen` 與 reader join，先記錄物件，再呼叫原函式；因此仍會啟動真實程序。相對地，`test_group_probe_permission_error_does_not_abort_cleanup` 替換 OS 呼叫並使用 FakeProcess，完全模擬群組探測。**有使用 monkeypatch，不代表整個測試都是假的。** + +| 實際案例 | 使用方式 | +| --- | --- | +| `tests/test_integration/test_integration.py::test_yaml_to_result_json` | tmp_path | +| `tests/test_integration/test_integration_cancellation.py::test_executor_cleans_entire_tree_and_drains_readers` | tmp_path、monkeypatch、四列參數化資料 | +| `tests/test_unit/test_console_output.py::test_writer_displays_stdout_on_console` | tmp_path、capsys、parametrize | +| `tests/test_unit/test_cancellation.py::test_raise_if_cancelled_raises_after_cancel` | pytest.raises | +| `tests/test_unit/test_runner.py::test_retry_waits_between_attempts` | 模擬時鐘與 `pytest.approx` | + +自訂 marker：`cancelled`, `test_lifecycle`, `artifact`, `retry`。 diff --git a/docs/test_matrix/test_matrix_v1.0.0.md b/docs/test_matrix/test_matrix_v1.0.0.md index 8b2b6d2..3649f28 100644 --- a/docs/test_matrix/test_matrix_v1.0.0.md +++ b/docs/test_matrix/test_matrix_v1.0.0.md @@ -11,3 +11,7 @@ Version baseline: Git tag `v1.0.0` | End-to-end execution | Integration | Sample YAML runs through executor and produces JSON output | `tests/test_integration.py` | Out of scope: artifact directories, full lifecycle stages, validation, retry, and timeout cancellation. + +## Pytest 用法 + +Fixture、mock、參數化與執行方式請見 [測試指南 v1.0.0](../test_guide.md#v1-0-0)。 diff --git a/docs/test_matrix/test_matrix_v1.1.0.md b/docs/test_matrix/test_matrix_v1.1.0.md index 9658d03..9084d1f 100644 --- a/docs/test_matrix/test_matrix_v1.1.0.md +++ b/docs/test_matrix/test_matrix_v1.1.0.md @@ -11,3 +11,7 @@ Version baseline: Git tag `v1.1.0` | End-to-end compatibility | Integration | The renamed model pipeline still executes a sample configuration | `tests/test_integration.py` | Regression focus: v1.0 behavior remains intact after naming and model refactoring. + +## Pytest 用法 + +Fixture、mock、參數化與執行方式請見 [測試指南 v1.1.0](../test_guide.md#v1-1-0)。 diff --git a/docs/test_matrix/test_matrix_v1.2.0.md b/docs/test_matrix/test_matrix_v1.2.0.md index 75e2d08..f0cae77 100644 --- a/docs/test_matrix/test_matrix_v1.2.0.md +++ b/docs/test_matrix/test_matrix_v1.2.0.md @@ -11,3 +11,7 @@ Version baseline: Git tag `v1.2.0` | JSON report | Integration | A run produces persisted output and report data | `tests/test_integration.py` | Out of scope: lifecycle failure routing, streaming console output, artifact validation, and retry. + +## Pytest 用法 + +Fixture、mock、參數化與執行方式請見 [測試指南 v1.2.0](../test_guide.md#v1-2-0)。 diff --git a/docs/test_matrix/test_matrix_v1.3.0.md b/docs/test_matrix/test_matrix_v1.3.0.md index 214303a..87a520c 100644 --- a/docs/test_matrix/test_matrix_v1.3.0.md +++ b/docs/test_matrix/test_matrix_v1.3.0.md @@ -12,3 +12,7 @@ Version baseline: Git tag `v1.3.0` | Lifecycle execution | Integration | YAML executes across the complete lifecycle | `tests/test_integration.py` | Regression focus: v1.2 artifact and executor behavior remains compatible with lifecycle orchestration. + +## Pytest 用法 + +Fixture、mock、參數化與執行方式請見 [測試指南 v1.3.0](../test_guide.md#v1-3-0)。 diff --git a/docs/test_matrix/test_matrix_v1.3.5.md b/docs/test_matrix/test_matrix_v1.3.5.md index 05adbb7..d945cd4 100644 --- a/docs/test_matrix/test_matrix_v1.3.5.md +++ b/docs/test_matrix/test_matrix_v1.3.5.md @@ -11,3 +11,7 @@ Version baseline: Git tag `v1.3.5` | End-to-end log pipeline | Integration | Console/executor/artifact output remains synchronized | `tests/test_integration.py` | Regression focus: lifecycle routing from v1.3.0 remains unchanged. + +## Pytest 用法 + +Fixture、mock、參數化與執行方式請見 [測試指南 v1.3.5](../test_guide.md#v1-3-5)。 diff --git a/docs/test_matrix/test_matrix_v1.4.0.md b/docs/test_matrix/test_matrix_v1.4.0.md index 1934cb4..2c79d83 100644 --- a/docs/test_matrix/test_matrix_v1.4.0.md +++ b/docs/test_matrix/test_matrix_v1.4.0.md @@ -12,3 +12,7 @@ Version baseline: Git tag `v1.4.0` | Validation report | Integration | Validation results are persisted in `result.json` | `tests/test_integration_artifact_validation.py` | Out of scope: CSV/JSON semantic validation and retry. + +## Pytest 用法 + +Fixture、mock、參數化與執行方式請見 [測試指南 v1.4.0](../test_guide.md#v1-4-0)。 diff --git a/docs/test_matrix/test_matrix_v1.4.1.md b/docs/test_matrix/test_matrix_v1.4.1.md index ec33031..65f1ad4 100644 --- a/docs/test_matrix/test_matrix_v1.4.1.md +++ b/docs/test_matrix/test_matrix_v1.4.1.md @@ -12,3 +12,7 @@ Version baseline: Git tag `v1.4.1` | Content validation | Integration | Valid content passes; invalid CSV/JSON makes the run fail | `tests/test_integration_artifact_validation.py` | Regression focus: all v1.4.0 file and directory rules remain supported. + +## Pytest 用法 + +Fixture、mock、參數化與執行方式請見 [測試指南 v1.4.1](../test_guide.md#v1-4-1)。 diff --git a/docs/test_matrix/test_matrix_v1.5.0.md b/docs/test_matrix/test_matrix_v1.5.0.md index a761352..8429207 100644 --- a/docs/test_matrix/test_matrix_v1.5.0.md +++ b/docs/test_matrix/test_matrix_v1.5.0.md @@ -13,3 +13,7 @@ Version baseline: Git tag `v1.5.0` | Real subprocess retry | Integration | A transient command can pass on a later attempt; exhaustion fails | `tests/test_integration.py` | Out of scope: artifact validation as an attempt-level retry trigger. + +## Pytest 用法 + +Fixture、mock、參數化與執行方式請見 [測試指南 v1.5.0](../test_guide.md#v1-5-0)。 diff --git a/docs/test_matrix/test_matrix_v1.5.1.md b/docs/test_matrix/test_matrix_v1.5.1.md index f1baeb1..c6cb17d 100644 --- a/docs/test_matrix/test_matrix_v1.5.1.md +++ b/docs/test_matrix/test_matrix_v1.5.1.md @@ -14,3 +14,7 @@ Historical baseline: the working tree after tag `v1.5.0` (release candidate; no | Real artifact-aware retry | Integration | Invalid first artifact retries and valid second artifact passes | `tests/test_integration/test_integration_retry.py` — currently blocked by invalid YAML fixture | Targeted test result recorded for this candidate: 24 passed, 2 failed. The two failures above must be resolved before release acceptance. + +## Pytest 用法 + +Fixture、mock、參數化與執行方式請見 [測試指南 v1.5.1](../test_guide.md#v1-5-1)。 diff --git a/docs/test_matrix/test_matrix_v1.5.2.md b/docs/test_matrix/test_matrix_v1.5.2.md index 8bffcfb..01fa388 100644 --- a/docs/test_matrix/test_matrix_v1.5.2.md +++ b/docs/test_matrix/test_matrix_v1.5.2.md @@ -25,3 +25,7 @@ Version scope: Failure Classification - Full command: `poetry run pytest -q` - Result: 112 passed - BDD description audit: 112 of 112 tests contain Given／When／Then + +## Pytest 用法 + +Fixture、mock、參數化與執行方式請見 [測試指南 v1.5.2](../test_guide.md#v1-5-2)。 diff --git a/docs/test_matrix/test_matrix_v1.5.3.md b/docs/test_matrix/test_matrix_v1.5.3.md index ab16126..b015a54 100644 --- a/docs/test_matrix/test_matrix_v1.5.3.md +++ b/docs/test_matrix/test_matrix_v1.5.3.md @@ -23,3 +23,7 @@ Version scope: Selective Retry and Artifact Criticality * Command: `poetry run pytest -q` * Result: **134 passed in 25.45s** (verified 2026-09-04) + +## Pytest 用法 + +Fixture、mock、參數化與執行方式請見 [測試指南 v1.5.3](../test_guide.md#v1-5-3)。 diff --git a/docs/test_matrix/test_matrix_v1.6.0.md b/docs/test_matrix/test_matrix_v1.6.0.md index 96dce33..3caf389 100644 --- a/docs/test_matrix/test_matrix_v1.6.0.md +++ b/docs/test_matrix/test_matrix_v1.6.0.md @@ -38,3 +38,7 @@ All 150 Python test functions under `tests/` were audited for meaningful Given * `test_retry_policy_retries_failure_before_max_attempts` actually supplies `FailureType.NONE` and asserts false for attempts one and two. Its docstring now describes those assertions; it is not evidence for failure retry. Other policy and runner tests cover actual failure retry. * `test_artifact_retry_defaults_to_false` checks an explicit optional rule and absent `after_step`, not a removed `retry_on_failure` default. * `configs/sample.yaml` is a demonstration configuration, not a proven all-pass acceptance run; see Definition of Done for the observed sample result. + +## Pytest 用法 + +Fixture、mock、參數化與執行方式請見 [測試指南 v1.6.0](../test_guide.md#v1-6-0)。 diff --git a/docs/test_matrix/test_matrix_v1.6.1.md b/docs/test_matrix/test_matrix_v1.6.1.md new file mode 100644 index 0000000..b9eca32 --- /dev/null +++ b/docs/test_matrix/test_matrix_v1.6.1.md @@ -0,0 +1,231 @@ +# Device Test Runner v1.6.1 Test Matrix + +Version scope: Safe Process Termination. Baseline: Git tag `v1.6.0`; source reviewed at `0dbdc51`. + +## Process lifecycle coverage + +| Area | Scenario | Level | Expected result | Evidence | +| --- | --- | --- | --- | --- | +| Process | Already exited | Unit, real process + monkeypatch | No group signal; terminated/killed false | `test_process.py::test_terminated_process_does_not_need_cleanup` | +| Process | SIGTERM graceful stop | Unit, real process | Exits without SIGKILL | `test_process.py::test_process_group_terminates_gracefully` | +| Process | SIGTERM ignored | Unit, real process | SIGKILL after grace period | `test_process.py::test_process_is_killed_when_sigterm_is_ignored` | +| Process | PermissionError during group probe | Unit, fake process | Retry probe instead of declaring exit | `test_process.py::test_group_probe_permission_error_does_not_abort_cleanup` | +| Process Group | Child and grandchild stop | Integration | All three recorded PIDs disappear | `test_integration_cancellation.py::test_executor_cleans_entire_tree_and_drains_readers` | +| Executor | Normal completion | Unit, real command | Success, code zero, expected output | `test_executor.py::test_subprocess_executor_return_success` | +| Executor | Timeout cleans group | Integration | TIMEOUT and no surviving tree members | Tree test, timeout cases | +| Executor | Cancel cleans group | Integration | CANCELLED and no surviving tree members | Tree test, cancel cases | +| Executor | Cancel differs from timeout | Integration | Mutually exclusive flags and expected failure type | Tree test, all four cases | +| stdout | Output before cancellation saved | Integration | Each process output appears in result and log file | Tree test, cancel cases | +| stderr | Error before cancellation saved | Integration | Each process error appears in result and log file | Tree test, cancel cases | +| Threads | stdout reader stops | Integration | Recorded stdout thread is not alive | Tree test | +| Threads | stderr reader stops | Integration | Recorded stderr thread is not alive | Tree test | +| Retry | Timeout may retry | Unit + Integration | Configured timeout retries and next attempt passes | `test_runner.py::test_runner_retries_configured_timeout`; `test_integration_retry.py::test_retry_does_not_leave_previous_attempt_process` | +| Retry | Previous cleanup before next start | Unit + Integration | Mock event order matches; real attempt two immediately checks previous parent and child PIDs | `test_runner.py::test_retry_starts_only_after_previous_attempt_cleanup`; real retry test above | +| Retry | Cancel never retries | Unit | One cancelled attempt; policy refuses even explicit CANCELLED allow-list | `test_runner.py::test_cancelled_step_is_not_retried`; `test_retry.py::test_cancelled_is_never_retried` | +| Cancellation | Pre-cancel skips normal lifecycle | Unit, mock executor | Only global_teardown runs | `test_runner.py::test_cancel_before_run_only_runs_global_teardown` | +| Lifecycle | Teardown after cancel | Unit, mock executor | Setup/scenario cancellation reaches teardown with active token | `test_runner.py::test_cancellation_lifecycle_and_summary` | +| Lifecycle | Global teardown after cancel | Unit, mock executor | All parameterized cancellation routes reach global_teardown | Same lifecycle test | +| SIGINT | First Ctrl+C | Unit, direct handler call | Token cancelled without KeyboardInterrupt | `test_signals.py::test_first_sigint_cancels_token` | +| SIGINT | Second Ctrl+C | Unit, direct handler call | KeyboardInterrupt; token remains cancelled | `test_signals.py::test_second_sigint_raises_keyboard_interrupt` | + +Unit files are under `tests/test_unit/`; integration files are under `tests/test_integration/`. The tree test has four cases: timeout/cancel × descendants accepting/ignoring SIGTERM. It waits for readiness before cancelling and cleans up even if an assertion fails. + +## 閱讀指南：這一版為什麼要測 Process？ + +一次測試命令可能啟動不只一個程序。例如 Python 測試腳本啟動量測工具，量測工具又啟動另一個背景程式。Timeout 或取消後，如果只停止最外層腳本，裡面的程式可能仍在量測、寫檔或占用裝置。下一次 retry 就可能和前一次互相干擾。 + +這一版主要確認三件事：**該停止的程序已停止、已產生的 log 有保存、清理完成後才開始 retry。** 上方表格是需求總覽，下方說明各測試如何提供證據。 + +### 先認識測試會用到的名詞 + +| 名詞 | 在本專案中的意思 | 讀測試時要注意什麼 | +| --- | --- | --- | +| Process | OS 中正在執行的程式，例如一個 Python 腳本 | Python 的 `Popen` 物件是控制外部程序的物件，不是外部程序本身 | +| PID | 一個程序的識別編號 | 測試把 PID 寫入檔案，清理後再檢查它是否還存在 | +| Parent／child／grandchild | 程序啟動另一個程序，形成父、子、孫的關係 | Parent 結束，不表示 child 也會一起結束 | +| Process group／PGID | 可以一起接收 signal 的程序群組／群組編號 | `killpg` 對群組發訊號；它不會自動搜尋所有父子關係 | +| Session | OS 用來組織程序群組的範圍 | `start_new_session=True` 讓 attempt 建立自己的 session 與 process group | +| Signal | 通知程序採取動作的 OS 機制 | SIGTERM、SIGKILL、SIGINT 的用途不同，見下表 | +| stdout／stderr | 一般輸出／錯誤輸出兩條串流 | 程序終止後，已寫出的內容仍應保留在結果與 log | +| Pipe | 把外部程序輸出送給 runner 的通道 | Child 若仍持有輸出端，parent 結束後 reader 也可能還在等資料 | +| Reader thread | Runner 內負責讀取 stdout 或 stderr 的執行緒 | 本專案各有一個；thread 和外部 process 是不同的東西 | +| Reap／回收 | 父程序取得直接子程序的退出狀態 | `poll()`／`wait()` 可以回收直接子程序，不能因此推論所有後代都結束 | + +PID 存在檢查有 OS 層級限制：已退出但尚未回收的 zombie 也可能仍有 PID。因此「查得到 PID」不一定表示它還在做工作；目前測試要求的是 PID 最後消失，跨平台回收行為仍需另外驗證。 + +| 操作 | 用途 | 本版的使用位置 | +| --- | --- | --- | +| SIGTERM | 請程序結束；程序可以安裝 handler 或忽略它 | 群組清理的第一步。預設動作可直接結束程序，不代表應用程式一定完成自己的清理 | +| SIGKILL | 強制停止，程序無法自行捕捉或忽略 | SIGTERM 寬限時間過後，群組仍存在時使用 | +| SIGINT | 終端按 Ctrl+C 通常產生的訊號 | 第一次要求取消；第二次 handler 呼叫拋出 `KeyboardInterrupt` | +| `os.kill(pid, 0)` | 檢查指定 PID 的存在與權限，不發出終止訊號 | 整合測試檢查舊程序是否消失 | +| `os.killpg(pgid, 0)` | 檢查指定群組的存在與權限，不發出終止訊號 | Terminator 等待群組退出 | + +### 測試中的程序與輸出關係 + +```mermaid +flowchart TD + E[SubprocessExecutor] --> P[Parent: level 0] + subgraph G[同一次 attempt 的 process group] + P --> C[Child: level 1] + C --> GC[Grandchild: level 2] + end + T[ProcessTerminator] -->|SIGTERM / SIGKILL| G + P -. 輸出 .-> L[stdout / stderr pipes] + C -. 輸出 .-> L + GC -. 輸出 .-> L + L --> R[兩個 reader threads] + R --> F[結果內容與 log 檔案] +``` + +[process_tree.py](../../tests/fixtures/process_tree.py) 使用同一支腳本建立三層程序。每一層記錄 PID、印出 stdout／stderr，再等待停止。`0.ready` 代表三層都完成初始化；cancel 測試等到它出現才取消，避免測到「child 還沒啟動就結束」的情況。 + +## Unit tests：先確認每個元件的責任 + +這裡的 Unit 是專案的檔案分類，**不代表全部使用假程序**。`test_process.py` 前三個測試會真的啟動 OS process；權限錯誤測試才用假物件，讓不容易穩定重現的狀況可重複測試。 + +### ProcessTerminator：怎麼停止程序？ + +檔案：[test_process.py](../../tests/test_unit/test_process.py) + +| Test case | 測試安排 | 關鍵檢查 | 能防止的問題 | +| --- | --- | --- | --- | +| `test_terminated_process_does_not_need_cleanup` | 啟動 `true` 並等它結束，再呼叫 terminator | `terminated=False`、`killed=False`、code 為 0，沒有呼叫 `killpg` | 對已結束的直接程序重複發送訊號 | +| `test_process_group_terminates_gracefully` | 啟動一個會等待的 Python process，再要求終止 | Process 結束，`terminated=True`、`killed=False` | 正常接受 SIGTERM 的程序也被不必要地強制終止 | +| `test_process_is_killed_when_sigterm_is_ignored` | 程序安裝忽略 SIGTERM 的 handler，印出 ready 後才開始清理 | `killed=True`，process 已結束 | 程序忽略停止通知，runner 卻一直等待或留下背景工作 | +| `test_group_probe_permission_error_does_not_abort_cleanup` | 模擬 SIGTERM 成功、群組探測沒有權限、下一次探測群組已消失 | 呼叫順序為 SIGTERM → probe → probe，沒有 SIGKILL | 把一次探測的權限錯誤當成程序已結束，或立刻中斷清理 | + +最後一個測試使用 pytest `monkeypatch` 暫時替換 `getpgid`／`killpg`，搭配 `FakeProcess` 與 list 記錄呼叫。PID `123` 是測試資料，不會拿去終止真實 OS 程序。測試結束後 pytest 會還原替換。 + +### Executor：逾時後有沒有交給 terminator？ + +檔案：[test_executor.py](../../tests/test_unit/test_executor.py) + +`test_subprocess_executor_raised_timeout_error` 把時間與 process 都模擬出來，讓測試不用真的等待 timeout。它確認： + +1. Executor 呼叫一次 `terminate_process_group(process)`。 +2. 模擬終止後的 return code 是 `-SIGTERM`；負值代表被該 signal 終止。 +3. 結果為 `TIMEOUT`，`timed_out=True`、`cancelled=False`、`success=False`。 + +這個測試驗證的是 **Executor 有委派清理並保留正確分類**。真正的 OS 群組是否停止，由 integration tests 驗證。正常完成則由 `test_subprocess_executor_return_success` 驗證 exit code 與輸出。 + +### Runner 與 SIGINT：流程順序是否正確？ + +| 檔案／案例 | 驗證方式 | 證據範圍 | +| --- | --- | --- | +| [test_runner.py](../../tests/test_unit/test_runner.py)：`test_retry_starts_only_after_previous_attempt_cleanup` | 假 executor 記錄 start、timeout、cleanup、return 等事件，確認第一次 cleanup 在第二次 start 之前 | 驗證 runner 的呼叫順序；沒有啟動真實 process | +| [test_signals.py](../../tests/test_unit/test_signals.py)：`test_first_sigint_cancels_token` | 直接呼叫一次 handler | Token 變成 cancelled，沒有拋出 KeyboardInterrupt | +| 同檔：`test_second_sigint_raises_keyboard_interrupt` | 先呼叫一次，再用 `pytest.raises` 檢查第二次 | 第二次確實拋出 KeyboardInterrupt，token 仍為 cancelled | + +SIGINT 單元測試不會真的對 pytest 按 Ctrl+C，也不驗證終端訊號傳遞。這讓 handler 規則可以穩定測試，但不能代替 CLI 端到端測試。 + +## Integration tests：真的啟動程序，確認整段流程 + +### 從單一 child 到三層程序 + +檔案：[test_integration_cancellation.py](../../tests/test_integration/test_integration_cancellation.py) + +| Test case | 主要檢查 | 與三層測試的關係 | +| --- | --- | --- | +| `test_executor_cancels_running_process` | 真實 command 被取消，分類為 CANCELLED，保留 started 輸出 | 基本取消流程 | +| `test_stdout_reader_thread_finishes_after_process_termination` | Executor 返回 cancelled，stdout 保留 start | 間接確認執行能收尾；沒有直接記錄 reader 狀態 | +| `test_stderr_is_drained_after_cancellation` | 取消前的 error-message 留在 result.stderr | 基本 stderr 保存 | +| `test_process_group_termination_cleans_child_processes` | Shell 啟動 child，清理後檢查 child PID 消失 | 直接驗證一層 child 清理 | +| `test_executor_cleans_entire_tree_and_drains_readers` | 三個 PID 消失、兩個 reader 已停止、輸出與 log 一致 | 補足 grandchild、忽略 SIGTERM、reader 存活狀態與 log 落盤證據 | + +三層測試是一個函式，透過 `pytest.mark.parametrize` 執行四種情境： + +| stop_reason | descendant_behavior | 實際情境 | 預期 | +| --- | --- | --- | --- | +| timeout | default | 到達 step timeout，後代正常接受 SIGTERM | TIMEOUT；全部停止 | +| timeout | ignore | 到達 step timeout，child／grandchild 忽略 SIGTERM | TIMEOUT；後代仍需被強制停止 | +| cancel | default | 三層準備好後取消，後代正常接受 SIGTERM | CANCELLED；全部停止 | +| cancel | ignore | 三層準備好後取消，child／grandchild 忽略 SIGTERM | CANCELLED；後代仍需被強制停止 | + +每種情境都確認三層各自寫出的 stdout／stderr 保留在結果中，且檔案內容與結果一致；記錄下來的兩個 reader threads 都已不再執行。測試最後也會清理程序，避免測試失敗時留下背景工作。 + +### Retry：檢查的時間點很重要 + +檔案：[test_integration_retry.py](../../tests/test_integration/test_integration_retry.py)，fixture：[retry_process.py](../../tests/fixtures/retry_process.py) + +`test_retry_does_not_leave_previous_attempt_process` 的流程： + +1. Attempt 1 記錄自己的 PID，啟動 child 並記錄 child PID，接著等待到 timeout。 +2. Runner 進入清理與 retry 流程。 +3. Attempt 2 **一開始就檢查兩個舊 PID**，不先 sleep 等它們消失。 +4. 兩個 PID 都不存在，才印出成功訊息並回傳 0；只要還有一個存在，fixture 就回傳失敗。 +5. 測試確認第一次為 TIMEOUT、第二次成功，而且執行次數正好是兩次。 + +如果只在整個 run 結束後檢查 PID，舊程序可能在 Attempt 2 開始後才退出，測試仍會通過。把檢查移到 Attempt 2 開始時，才能攔住兩次 attempt 重疊的問題。 + +## 本次 Process 修正與回歸測試 + +以下區分產品清理問題與測試本身的問題，方便日後看到相同錯誤時查找。 + +| 問題 | 原因與修正 | 對應證據 | +| --- | --- | --- | +| Parent 結束，但 child／grandchild 還活著 | 原本只看直接 process 的 `poll()` 就返回。改成繼續探測整個群組，寬限時間後仍存在就 SIGKILL | 三層測試的兩個 `ignore` 情境；補強時曾重現 timeout／cancel 都因 reader 無法結束而失敗 | +| Reader 一直等不到輸出結束 | 存活後代可能仍持有 pipe。清理群組後 join 每個 reader，並檢查 thread 已停止 | 三層測試同時檢查 PID、thread、result 與 log；reader timeout 例外路徑本身尚無專用測試 | +| 探測群組出現 PermissionError | 「無權檢查」不等於「不存在」。目前繼續等待，直到群組消失或 deadline 到期 | `test_group_probe_permission_error_does_not_abort_cleanup` | +| Retry 測試太晚檢查舊 PID | Run 結束後才檢查，無法證明下一次開始前已清乾淨。改由第二次 fixture 立即檢查 parent 與 child | 真實 retry 整合測試與 runner 事件順序測試 | +| `Mock object cannot be interpreted as an integer` | Mock Popen 的 PID 被傳入真實 `os.getpgid`。Executor 單元測試改為同時注入假 terminator，驗證委派呼叫 | Executor timeout 單元測試；不是在產品程式中把 Mock 轉成整數 | +| `PosixPath is not JSON serializable` | Runner 測試的假 executor 把 Path 放進要求字串的 log path 欄位。改為 `str(path)` | Runner cleanup 順序測試可完成 JSON 寫入；並修正 `retrun` 事件名稱拼字 | +| Retry 測試尚未跑到清理就失敗 | 修正 `fixtures` 路徑、command 結尾引號、`LifecycleSteps`、計數器先轉 int 再加一，以及 `result.artifact_dir` | 真實 retry 測試現在能走完 timeout → cleanup → retry | +| 測試依賴系統存在 `python` 命令 | Process 測試改用 `sys.executable`，沿用執行 pytest 的 Python | 真實 SIGTERM／SIGKILL 單元測試 | + +本次修正沒有把正常完成時的背景後代清理、任意 daemon 追蹤或所有例外下的報告保存一併實作。這些仍是下方 Coverage limits 的限制。 + +### 本次 AI 協作新增與補強的案例 + +以下記錄 AI 協作新增與修改的案例；驗證範圍見各案例說明。 + +| 類型 | 案例 | 新增或補強內容 | +| --- | --- | --- | +| 新增 | `test_executor_cleans_entire_tree_and_drains_readers` | 四種真實程序情境；新增 `process_tree.py` fixture；同時檢查三層 PID、兩個 reader 與兩份 log | +| 新增 | `test_group_probe_permission_error_does_not_abort_cleanup` | 可重複的權限探測回歸測試；後續改用 pytest monkeypatch、FakeProcess 與呼叫紀錄 | +| 新增 | `test_first_sigint_cancels_token` | 第一次 SIGINT handler 呼叫取消 token | +| 新增 | `test_second_sigint_raises_keyboard_interrupt` | 第二次 handler 呼叫拋出 KeyboardInterrupt | +| 補強 | `test_terminated_process_does_not_need_cleanup` | 除了結果旗標，也確認沒有送出群組訊號 | +| 補強 | `test_retry_does_not_leave_previous_attempt_process` | 修正測試設定；在第二次開始時檢查前次 parent／child，並驗證成功標記 | +| 修正 | `test_subprocess_executor_raised_timeout_error` | 假 terminator 配合假 process，驗證清理委派、退出狀態與 TIMEOUT 分類 | +| 修正 | `test_retry_starts_only_after_previous_attempt_cleanup` | 修正 log path 字串與事件拼字，讓既有順序斷言能執行完成 | + +測試寫法另外補上中文分段註解與 Given／When／Then docstring。註解說明用途；判斷功能有沒有被驗證時，請以 `assert` 與真實／模擬的測試邊界為準。 + +## 如何執行這些測試 + +在專案根目錄執行 Process 與 SIGINT 單元測試： + +```bash +poetry run pytest tests/test_unit/test_process.py tests/test_unit/test_signals.py -q +``` + +執行程序清理與 retry 整合測試： + +```bash +poetry run pytest tests/test_integration/test_integration_cancellation.py tests/test_integration/test_integration_retry.py -q +``` + +只看三層程序的四種情境，`-v` 會列出各參數案例： + +```bash +poetry run pytest tests/test_integration/test_integration_cancellation.py -k entire_tree -v +``` + +若使用現有 `.venv`，可將 `poetry run pytest` 換成 `.venv/bin/python -m pytest`。以上是執行方式，不是本次文件補充重新執行測試的紀錄；已觀察到的結果請見下方 Verification。 + +## Verification + +See [Definition of Done](../definition_of_done/definition_of_done_v1.6.1.md) for this update's exact commands, environment and results. There are 162 test functions and 168 parameterized cases. This update adds Given／When／Then docstrings to 12 functions; executable test AST is unchanged. + +## Coverage limits + +* The tree test checks readers and log files through the executor. Runner lifecycle cancellation still uses mocks; it is not a full CLI cancellation test. +* The real retry test checks the previous parent and child before attempt two does work. It does not create a grandchild; the separate tree test covers three levels. +* SIGINT tests call the handler directly. Real signal delivery, handler restoration and CLI exit codes lack dedicated end-to-end assertions. +* No guarantee is established for detached descendants, normal completion with background children, zombie reaping on Linux, repeated cleanup errors or second-interrupt report finalization. +* Local passing tests do not establish Windows support, successful Ubuntu CI, a total shutdown deadline or GitHub Release publication. + +## Pytest 用法 + +Fixture、mock、參數化與執行方式請見 [測試指南 v1.6.1](../test_guide.md#v1-6-1)。 diff --git a/tests/test_integration/test_integration_cancellation.py b/tests/test_integration/test_integration_cancellation.py index 8a02634..a083b3b 100644 --- a/tests/test_integration/test_integration_cancellation.py +++ b/tests/test_integration/test_integration_cancellation.py @@ -84,6 +84,12 @@ def cancel_soon(): def test_stdout_reader_thread_finishes_after_process_termination(tmp_path: Path): + """Acceptance scenario. + + Given a real command prints start and waits. + When the token is cancelled during execution. + Then execution returns cancelled and retains the start output. + """ token = CancellationToken() executor = SubprocessExecutor( @@ -138,6 +144,12 @@ def cancel_soon(): def test_stderr_is_drained_after_cancellation(tmp_path: Path): + """Acceptance scenario. + + Given a real command prints an error to stderr and waits. + When the token is cancelled during execution. + Then execution returns cancelled and retains the error output. + """ token = CancellationToken() executor = SubprocessExecutor( @@ -192,6 +204,12 @@ def cancel_soon(): def test_process_group_termination_cleans_child_processes(tmp_path: Path): + """Acceptance scenario. + + Given a shell has started a child whose PID is recorded. + When the terminator stops its process group. + Then the child PID no longer exists. + """ child_pid_file = tmp_path / "child.pid" command = f""" @@ -271,6 +289,12 @@ def test_executor_cleans_entire_tree_and_drains_readers( # default: child / grandchild 正常接受 SIGTERM。 # ignore: child / grandchild 忽略 SIGTERM，必須靠 SIGKILL 結束。 # + """Acceptance scenario. + + Given a parent, child and grandchild run with descendants accepting or ignoring SIGTERM. + When the executor observes timeout or cancellation. + Then all three PIDs disappear, both readers finish, and both output logs match the result. + """ fixture_script = Path(__file__).resolve().parents[1] / "fixtures" / "process_tree.py" # diff --git a/tests/test_integration/test_integration_retry.py b/tests/test_integration/test_integration_retry.py index 119adb3..53e936b 100644 --- a/tests/test_integration/test_integration_retry.py +++ b/tests/test_integration/test_integration_retry.py @@ -173,6 +173,12 @@ def test_retry_does_not_leave_previous_attempt_process(tmp_path: Path): # ------------------------------------------------ # + """Acceptance scenario. + + Given the first real attempt starts a child and exceeds its timeout. + When the runner retries the step. + Then attempt two verifies both previous PIDs are absent before succeeding. + """ project_root = Path(__file__).resolve().parents[2] fixture_script = project_root / "tests" / "fixtures" / "retry_process.py" diff --git a/tests/test_unit/test_process.py b/tests/test_unit/test_process.py index a0c8b60..7832013 100644 --- a/tests/test_unit/test_process.py +++ b/tests/test_unit/test_process.py @@ -7,6 +7,12 @@ def test_terminated_process_does_not_need_cleanup(monkeypatch): + """Acceptance scenario. + + Given the direct process has already exited with code zero. + When the terminator is called. + Then no group signal is sent and the result reports no termination or kill. + """ process = subprocess.Popen( ["true"], start_new_session=True, @@ -33,6 +39,12 @@ def fake_killpg(process_group_id, signal_number): def test_process_group_terminates_gracefully(): + """Acceptance scenario. + + Given a real process runs in a separate session. + When the terminator sends SIGTERM. + Then the process exits without SIGKILL. + """ process = subprocess.Popen( [ sys.executable, @@ -53,6 +65,12 @@ def test_process_group_terminates_gracefully(): def test_process_is_killed_when_sigterm_is_ignored(): + """Acceptance scenario. + + Given a real process confirms its SIGTERM ignore handler is ready. + When the termination grace period expires. + Then the terminator uses SIGKILL and the process exits. + """ command = ( "import signal, time;" "signal.signal(" @@ -94,6 +112,12 @@ def test_group_probe_permission_error_does_not_abort_cleanup(monkeypatch): # 檢查 process group 時，暫時沒有權限不代表 process 已經結束。 # 應該繼續等待，直到確認群組不存在。 # + """Acceptance scenario. + + Given a fake group probe raises PermissionError before reporting that the group is gone. + When the terminator sends SIGTERM and polls the group. + Then the probe is retried and cleanup completes without SIGKILL. + """ class FakeProcess: def __init__(self): self.pid = 123 diff --git a/tests/test_unit/test_runner.py b/tests/test_unit/test_runner.py index 3df3c83..3a8d6d7 100644 --- a/tests/test_unit/test_runner.py +++ b/tests/test_unit/test_runner.py @@ -2470,6 +2470,12 @@ def fake_monotonic(): def test_retry_starts_only_after_previous_attempt_cleanup(tmp_path: Path): + """Acceptance scenario. + + Given a tracking executor records cleanup before returning a timed-out first attempt. + When the runner retries and the second attempt succeeds. + Then the recorded first cleanup precedes the second start and the full event order matches. + """ config = RunnerConfig( test_case=DeviceTestCase( id="power_001", diff --git a/tests/test_unit/test_signals.py b/tests/test_unit/test_signals.py index bfa59de..bbbe85e 100644 --- a/tests/test_unit/test_signals.py +++ b/tests/test_unit/test_signals.py @@ -8,6 +8,12 @@ def test_first_sigint_cancels_token(): + """Acceptance scenario. + + Given a signal handler has an active token. + When its SIGINT handler is called once. + Then the token is cancelled without interrupting the test. + """ token = CancellationToken() handler = SignalCancellationHandler(token) @@ -20,6 +26,12 @@ def test_first_sigint_cancels_token(): def test_second_sigint_raises_keyboard_interrupt(): + """Acceptance scenario. + + Given a signal handler has received its first SIGINT. + When its SIGINT handler is called again. + Then KeyboardInterrupt is raised and the token remains cancelled. + """ token = CancellationToken() handler = SignalCancellationHandler(token) From 75c47e1417f03e9ab2fc63332fa8440f4c8bd35d Mon Sep 17 00:00:00 2001 From: Andrew Wang Date: Thu, 17 Sep 2026 23:24:55 +0800 Subject: [PATCH 6/9] add ci v0.2 --- .github/workflows/ci.yml | 21 ++++++++- .gitignore | 1 + docs/ci.md | 96 ++++++++++++++++++++++++++++++++++++++++ poetry.lock | 28 ++++++++++++ pyproject.toml | 12 +++++ 5 files changed, 157 insertions(+), 1 deletion(-) create mode 100644 docs/ci.md diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c2a3724..e6abc33 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -8,29 +8,48 @@ on: branches: - main +permissions: + contents: read + jobs: test: + name: Code Quality and Tests runs-on: ubuntu-latest steps: + # Step 1: Get source code - name: Checkout respository uses: actions/checkout@v7 + # Step 2: Setup Python - name: Set up Python uses: actions/setup-python@v7 with: - python-version: "3.12" + python-version: "3.14" + # Step 3: Install Poetry - name: Install Poetry run: pip install poetry + # Step 4: Install Python version - name: Show Python version run: python --version + # Step 5: Check Poetry version - name: Show Poetry version run: poetry --version + # Step 6: Install dependencies - name: Install dependencies run: poetry install + # Step 7: Ruff lint + - name: Ruff Lint + run: poetry run ruff check . + + # Step 8: Ruff formatting + - name: Ruff Format Checkt + run: poetry run ruff check format --check . + + # Step 9: Pytest - name: Run tests run: poetry run pytest diff --git a/.gitignore b/.gitignore index e03fa00..df833f1 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,7 @@ __pycache__/ .DS_Store .pytest_cache/ +.ruff_cache/ # Virtual environments .venv/ diff --git a/docs/ci.md b/docs/ci.md new file mode 100644 index 0000000..2629741 --- /dev/null +++ b/docs/ci.md @@ -0,0 +1,96 @@ +# Continuous Integration + +## Overview + +Device Test Runner uses GitHub Actions to automate code quality checks and testing. + +The CI workflow runs on: + +* Pushes to `main` +* Pull requests targeting `main` + +## CI Pipeline + +```text +Checkout + ↓ +Setup Python + ↓ +Install Dependencies + ↓ +Ruff Lint + ↓ +Ruff Format Check + ↓ +pytest + ↓ +Success / Failure +``` + +## Local Development + +Install project dependencies: + +```bash +poetry install +``` + +Run lint: + +```bash +poetry run ruff check . +``` + +Check formatting: + +```bash +poetry run ruff format --check . +``` + +Run tests: + +```bash +poetry run pytest +``` + +## Fixing Code Quality Issues + +Automatically fix supported lint violations: + +```bash +poetry run ruff check --fix . +``` + +Format the codebase: + +```bash +poetry run ruff format . +``` + +Review all changes before committing. + +## Quality Gates + +| Check | Purpose | +| ----------- | ------------------------------------ | +| Ruff Lint | Detect code quality issues | +| Ruff Format | Enforce consistent formatting | +| pytest | Verify expected application behavior | + +A failing check causes the CI job to fail. Subsequent steps are skipped by default. + +## Scope + +Version 0.2 focuses on Python code quality and existing automated tests. + +Not included: + +* Coverage reporting +* Test artifacts +* Matrix testing +* Deployment +* Hardware-dependent test execution + +## Workflow + +`.github/workflows/ci.yml` diff --git a/poetry.lock b/poetry.lock index 2fdc3f3..59fe0d4 100644 --- a/poetry.lock +++ b/poetry.lock @@ -465,6 +465,34 @@ files = [ {file = "pyyaml-6.0.3.tar.gz", hash = "sha256:d76623373421df22fb4cf8817020cbb7ef15c725b9d5e45f17e189bfc384190f"}, ] +[[package]] +name = "ruff" +version = "0.16.8" +description = "An extremely fast Python linter and code formatter, written in Rust." +optional = false +python-versions = ">=3.7" +groups = ["dev"] +files = [ + {file = "ruff-0.16.8-py3-none-linux_armv6l.whl", hash = "sha256:6ffbd6d87383c1edf5f6fa890f10200950240d7c1a16052a19a09d3a2307dd38"}, + {file = "ruff-0.16.8-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:42ed6b878ed61e3acca92f2730a17acff39286944ea82398544696366a6f925e"}, + {file = "ruff-0.16.8-py3-none-macosx_11_0_arm64.whl", hash = "sha256:7ea781c7f2afba8c6a505ea0fb3f994020249e0c450635f5381286fea6b46170"}, + {file = "ruff-0.16.8-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8efeae3bbe414a5efefda11a792dfb51ef90ac48d50c4830de2f644caf3e8659"}, + {file = "ruff-0.16.8-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:3a79b795469fef7fc6e908b218eed2eb17332afd85031db6480dc864560e69b2"}, + {file = "ruff-0.16.8-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3fdc5563cdc50555e6fba39322850860e9267c1b3d12c26a74729d8604c3c812"}, + {file = "ruff-0.16.8-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:34508983c70665578dab88f5223d8e6228307e1135398ca8bfc8b7e9501e282b"}, + {file = "ruff-0.16.8-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:644bb578569e0ffc575741232bd385dacdd6fbe123f1a729e7a225f54aa3957f"}, + {file = "ruff-0.16.8-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:15e7d226246961db9235098333caa13063906d3851136b84c2900b82f5daa1df"}, + {file = "ruff-0.16.8-py3-none-manylinux_2_31_riscv64.whl", hash = "sha256:a2bf6bc3e9ebdd4449abc6f06cf64b98051a2c61cf94d2fe9596518c881f1a1e"}, + {file = "ruff-0.16.8-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:6ca111ba0849539165e9e59d2b442542f3c1e8060ebbdea82494f1ffbccb1e1f"}, + {file = "ruff-0.16.8-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:359a1e5b495448ee1e91018064382ebc86f90e8aac2fed222c7d0e4e8df85fd2"}, + {file = "ruff-0.16.8-py3-none-musllinux_1_2_i686.whl", hash = "sha256:59e8f5681349474110b24d62e93cfda6593f5fa3473446ca3705200cac1a08b9"}, + {file = "ruff-0.16.8-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:efa3e7a16d1baaa79957888dfdf8be9ef2e44db81cb032af06d76632ab59e773"}, + {file = "ruff-0.16.8-py3-none-win32.whl", hash = "sha256:55793ba85c69921e89be061426d91a78652d6e50317c962240922747a4eb713f"}, + {file = "ruff-0.16.8-py3-none-win_amd64.whl", hash = "sha256:a6b85621fd3c81e31fc5f5add09c9c078b430db3595ca632efafdec9e64ebfaa"}, + {file = "ruff-0.16.8-py3-none-win_arm64.whl", hash = "sha256:d075e820af612102ce217f07cc93e69f9490b10ec13ea85fa87bd03d996cef8a"}, + {file = "ruff-0.16.8.tar.gz", hash = "sha256:9247bf92b5f04d825c8639a4fe423ec2e4222acd9222e58412b0dab7e442798b"}, +] + [[package]] name = "tomli" version = "2.4.1" diff --git a/pyproject.toml b/pyproject.toml index 8a71c89..4f6654b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -20,3 +20,15 @@ packages = [{ include = "runner" }] [build-system] requires = ["poetry-core>=2.0.0,<3.0.0"] build-backend = "poetry.core.masonry.api" + +[dependency-groups] +dev = [ + "ruff (>=0.16.8,<0.17.0)" +] + +[tool.ruff] +target-version = "py314" +line-length = 88 + +[tool.ruff.lint] +select = ["E4", "E7", "E9" , "F" ,"I"] \ No newline at end of file From 2ca94f07ea63829488067bd464545e6bb86c9f78 Mon Sep 17 00:00:00 2001 From: Andrew Wang Date: Sat, 19 Sep 2026 21:19:10 +0800 Subject: [PATCH 7/9] add ci (test) --- .github/workflows/ci.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e6abc33..518ee35 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -3,7 +3,8 @@ name: Device Test Runner CI on: push: branches: - - main + # - main + - '*' pull_request: branches: - main @@ -20,6 +21,9 @@ jobs: - name: Checkout respository uses: actions/checkout@v7 + - name: Display Branch Name + run: 'echo "Current Branch: ${{ github.ref }}"' + # Step 2: Setup Python - name: Set up Python uses: actions/setup-python@v7 From da5e3f000dd8a1fbb767351a559c6812ea1b1590 Mon Sep 17 00:00:00 2001 From: Andrew Wang Date: Sat, 19 Sep 2026 21:25:03 +0800 Subject: [PATCH 8/9] update ci (test) --- .github/workflows/ci.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 518ee35..56d2b5f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -3,8 +3,7 @@ name: Device Test Runner CI on: push: branches: - # - main - - '*' + - '**' pull_request: branches: - main From d0ff1a319670d1b5b307d698445b77efb334b116 Mon Sep 17 00:00:00 2001 From: Andrew Wang Date: Sat, 19 Sep 2026 21:50:27 +0800 Subject: [PATCH 9/9] add ci --- .github/workflows/ci.yml | 30 +++++--------- docs/ci.md | 71 +++++++++++++++++++++++++--------- poetry.lock | 28 -------------- pyproject.toml | 12 ------ tests/test_unit/test_runner.py | 36 ++--------------- 5 files changed, 65 insertions(+), 112 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 56d2b5f..4139db3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -3,7 +3,7 @@ name: Device Test Runner CI on: push: branches: - - '**' + - main pull_request: branches: - main @@ -16,43 +16,31 @@ jobs: name: Code Quality and Tests runs-on: ubuntu-latest steps: - # Step 1: Get source code - - name: Checkout respository + - name: Checkout repository uses: actions/checkout@v7 - - name: Display Branch Name - run: 'echo "Current Branch: ${{ github.ref }}"' - - # Step 2: Setup Python - name: Set up Python uses: actions/setup-python@v7 with: python-version: "3.14" - # Step 3: Install Poetry - name: Install Poetry - run: pip install poetry + run: python -m pip install "poetry>=2,<3" - # Step 4: Install Python version - name: Show Python version run: python --version - # Step 5: Check Poetry version - name: Show Poetry version run: poetry --version - # Step 6: Install dependencies - - name: Install dependencies - run: poetry install + - name: Check dependency lock file + run: poetry check --lock - # Step 7: Ruff lint - - name: Ruff Lint - run: poetry run ruff check . + - name: Install dependencies + run: poetry install --no-interaction - # Step 8: Ruff formatting - - name: Ruff Format Checkt - run: poetry run ruff check format --check . + - name: Code Quality + run: poetry run pre-commit run --all-files --show-diff-on-failure - # Step 9: Pytest - name: Run tests run: poetry run pytest diff --git a/docs/ci.md b/docs/ci.md index 2629741..29dcb17 100644 --- a/docs/ci.md +++ b/docs/ci.md @@ -16,11 +16,11 @@ Checkout ↓ Setup Python ↓ -Install Dependencies +Install Poetry and Check Lock File ↓ -Ruff Lint +Install Dependencies ↓ -Ruff Format Check +Pre-commit Hooks ↓ pytest ↓ @@ -29,22 +29,26 @@ Success / Failure ## Local Development -Install project dependencies: +The workflow uses Ubuntu and Python 3.14, matching the Python version in +`.pre-commit-config.yaml`. Poetry 2.x installs dependencies from `poetry.lock`, +including pre-commit and pytest. + +Check that project metadata and the lock file are consistent: ```bash -poetry install +poetry check --lock ``` -Run lint: +Install project dependencies: ```bash -poetry run ruff check . +poetry install ``` -Check formatting: +Run all code quality checks: ```bash -poetry run ruff format --check . +poetry run pre-commit run --all-files --show-diff-on-failure ``` Run tests: @@ -55,33 +59,64 @@ poetry run pytest ## Fixing Code Quality Issues -Automatically fix supported lint violations: +Run the hooks locally to apply supported whitespace, import sorting, and +formatting fixes: ```bash -poetry run ruff check --fix . +poetry run pre-commit run --all-files ``` -Format the codebase: +Review all changes and rerun the command before committing. Hooks are configured +in `.pre-commit-config.yaml` and include pre-commit-hooks, isort, Black, and Flake8. +The first run downloads the hook environments and requires network access. + +## Alternative Tool: Ruff + +Ruff is an alternative for Python linting, import sorting, and formatting. +It is not currently installed by the project or used in CI. To try it as a +development dependency: ```bash -poetry run ruff format . +poetry add --group dev ruff +``` + +This updates `pyproject.toml` and `poetry.lock`. Run lint and formatting checks: + +```bash +poetry run ruff check . +poetry run ruff check --select I . +poetry run ruff format --check --line-length 100 . +``` + +Apply supported fixes and formatting: + +```bash +poetry run ruff check --fix . +poetry run ruff check --select I --fix . +poetry run ruff format --line-length 100 . ``` -Review all changes before committing. +The line length matches the current formatting hooks. Ruff's default lint rules +and formatting are not identical to the existing Flake8, isort, and Black +configuration, and it does not replace all file hygiene checks in pre-commit. +Review any fixes and continue running the existing pre-commit checks before +committing. Adopting Ruff in CI would require updating the hook configuration +and agreeing on the lint rules first. ## Quality Gates | Check | Purpose | | ----------- | ------------------------------------ | -| Ruff Lint | Detect code quality issues | -| Ruff Format | Enforce consistent formatting | +| Poetry lock | Verify dependency metadata consistency | +| Pre-commit | Check syntax, file hygiene, imports, formatting, and lint | | pytest | Verify expected application behavior | A failing check causes the CI job to fail. Subsequent steps are skipped by default. ## Scope -Version 0.2 focuses on Python code quality and existing automated tests. +CI checks code quality and runs the full unit and integration test suite with +`poetry run pytest`, including process cleanup, retry, and cancellation tests. Not included: @@ -93,4 +128,4 @@ Not included: ## Workflow -`.github/workflows/ci.yml` +[CI workflow](../.github/workflows/ci.yml) diff --git a/poetry.lock b/poetry.lock index 59fe0d4..2fdc3f3 100644 --- a/poetry.lock +++ b/poetry.lock @@ -465,34 +465,6 @@ files = [ {file = "pyyaml-6.0.3.tar.gz", hash = "sha256:d76623373421df22fb4cf8817020cbb7ef15c725b9d5e45f17e189bfc384190f"}, ] -[[package]] -name = "ruff" -version = "0.16.8" -description = "An extremely fast Python linter and code formatter, written in Rust." -optional = false -python-versions = ">=3.7" -groups = ["dev"] -files = [ - {file = "ruff-0.16.8-py3-none-linux_armv6l.whl", hash = "sha256:6ffbd6d87383c1edf5f6fa890f10200950240d7c1a16052a19a09d3a2307dd38"}, - {file = "ruff-0.16.8-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:42ed6b878ed61e3acca92f2730a17acff39286944ea82398544696366a6f925e"}, - {file = "ruff-0.16.8-py3-none-macosx_11_0_arm64.whl", hash = "sha256:7ea781c7f2afba8c6a505ea0fb3f994020249e0c450635f5381286fea6b46170"}, - {file = "ruff-0.16.8-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8efeae3bbe414a5efefda11a792dfb51ef90ac48d50c4830de2f644caf3e8659"}, - {file = "ruff-0.16.8-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:3a79b795469fef7fc6e908b218eed2eb17332afd85031db6480dc864560e69b2"}, - {file = "ruff-0.16.8-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3fdc5563cdc50555e6fba39322850860e9267c1b3d12c26a74729d8604c3c812"}, - {file = "ruff-0.16.8-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:34508983c70665578dab88f5223d8e6228307e1135398ca8bfc8b7e9501e282b"}, - {file = "ruff-0.16.8-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:644bb578569e0ffc575741232bd385dacdd6fbe123f1a729e7a225f54aa3957f"}, - {file = "ruff-0.16.8-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:15e7d226246961db9235098333caa13063906d3851136b84c2900b82f5daa1df"}, - {file = "ruff-0.16.8-py3-none-manylinux_2_31_riscv64.whl", hash = "sha256:a2bf6bc3e9ebdd4449abc6f06cf64b98051a2c61cf94d2fe9596518c881f1a1e"}, - {file = "ruff-0.16.8-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:6ca111ba0849539165e9e59d2b442542f3c1e8060ebbdea82494f1ffbccb1e1f"}, - {file = "ruff-0.16.8-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:359a1e5b495448ee1e91018064382ebc86f90e8aac2fed222c7d0e4e8df85fd2"}, - {file = "ruff-0.16.8-py3-none-musllinux_1_2_i686.whl", hash = "sha256:59e8f5681349474110b24d62e93cfda6593f5fa3473446ca3705200cac1a08b9"}, - {file = "ruff-0.16.8-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:efa3e7a16d1baaa79957888dfdf8be9ef2e44db81cb032af06d76632ab59e773"}, - {file = "ruff-0.16.8-py3-none-win32.whl", hash = "sha256:55793ba85c69921e89be061426d91a78652d6e50317c962240922747a4eb713f"}, - {file = "ruff-0.16.8-py3-none-win_amd64.whl", hash = "sha256:a6b85621fd3c81e31fc5f5add09c9c078b430db3595ca632efafdec9e64ebfaa"}, - {file = "ruff-0.16.8-py3-none-win_arm64.whl", hash = "sha256:d075e820af612102ce217f07cc93e69f9490b10ec13ea85fa87bd03d996cef8a"}, - {file = "ruff-0.16.8.tar.gz", hash = "sha256:9247bf92b5f04d825c8639a4fe423ec2e4222acd9222e58412b0dab7e442798b"}, -] - [[package]] name = "tomli" version = "2.4.1" diff --git a/pyproject.toml b/pyproject.toml index 4f6654b..8a71c89 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -20,15 +20,3 @@ packages = [{ include = "runner" }] [build-system] requires = ["poetry-core>=2.0.0,<3.0.0"] build-backend = "poetry.core.masonry.api" - -[dependency-groups] -dev = [ - "ruff (>=0.16.8,<0.17.0)" -] - -[tool.ruff] -target-version = "py314" -line-length = 88 - -[tool.ruff.lint] -select = ["E4", "E7", "E9" , "F" ,"I"] \ No newline at end of file diff --git a/tests/test_unit/test_runner.py b/tests/test_unit/test_runner.py index 3a8d6d7..8e2dbfe 100644 --- a/tests/test_unit/test_runner.py +++ b/tests/test_unit/test_runner.py @@ -2108,7 +2108,7 @@ def test_cancel_stops_remaining_scenario_steps(tmp_path: Path): show_console_output=False, ) - result = runner.run(config=config, cancellation_token=token) + runner.run(config=config, cancellation_token=token) assert "scenario" in executor.executed_steps assert "scenario_2" not in executor.executed_steps @@ -2124,6 +2124,7 @@ def test_cancel_before_run_only_runs_global_teardown(tmp_path: Path): When the runner receives that token. Then only global_teardown is executed. """ + config = RunnerConfig( test_case=DeviceTestCase( id="power_001", @@ -2172,7 +2173,7 @@ def test_cancel_before_run_only_runs_global_teardown(tmp_path: Path): show_console_output=False, ) - result = runner.run(config=config, cancellation_token=token) + runner.run(config=config, cancellation_token=token) assert "global_setup" not in executor.executed_steps assert "setup" not in executor.executed_steps @@ -2188,36 +2189,6 @@ def test_cancel_during_retry_delay(tmp_path: Path, monkeypatch): When the token is cancelled during the first polling sleep. Then the wait returns true after one 0.1-second sleep. """ - config = RunnerConfig( - test_case=DeviceTestCase( - id="power_001", - name="power_001", - description="Description", - ), - device=DeviceInfo( - serial="device_001", - product="pixel", - build="build_001", - ), - retry=RetryConfig( - max_attempts=3, - delay_seconds=1, - ), - lifecycle=LifecycleConfig( - global_setup=LifecycleSteps(steps=[mock_step("global_setup")]), - setup=LifecycleSteps(steps=[mock_step("setup")]), - scenario=LifecycleSteps( - steps=[ - mock_step("scenario"), - ] - ), - teardown=LifecycleSteps(steps=[mock_step("teardown")]), - global_teardown=LifecycleSteps(steps=[mock_step("global_teardown")]), - ), - artifact=ArtifactConfig( - output_dir=str(tmp_path), - ), - ) token = CancellationToken() @@ -2469,7 +2440,6 @@ def fake_monotonic(): def test_retry_starts_only_after_previous_attempt_cleanup(tmp_path: Path): - """Acceptance scenario. Given a tracking executor records cleanup before returning a timed-out first attempt.