diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4139db3..4b5d91b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -3,7 +3,7 @@ name: Device Test Runner CI on: push: branches: - - main + - "**" pull_request: branches: - main @@ -12,8 +12,8 @@ permissions: contents: read jobs: - test: - name: Code Quality and Tests + code-quailiry: + name: Code Quality runs-on: ubuntu-latest steps: - name: Checkout repository @@ -27,20 +27,48 @@ jobs: - name: Install Poetry run: python -m pip install "poetry>=2,<3" - - name: Show Python version - run: python --version + - name: Install dependencies + run: poetry install --no-interaction - - name: Show Poetry version - run: poetry --version + - name: Run pre-commit + run: poetry run pre-commit run --all-files + unit-tests: + name: Unit Tests + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v7 - - name: Check dependency lock file - run: poetry check --lock + - name: Set up Python + uses: actions/setup-python@v7 + with: + python-version: "3.14" - - name: Install dependencies - run: poetry install --no-interaction + - name: Install Poetry + run: python -m pip install "poetry>=2,<3" + + - name: Install dependencies + run: poetry install --no-interaction + + - name: Run Unit Test + run: poetry run pytest tests/test_unit + integration-tests: + name: Integration Tests + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v7 + + - name: Set up Python + uses: actions/setup-python@v7 + with: + python-version: "3.14" + + - name: Install Poetry + run: python -m pip install "poetry>=2,<3" - - name: Code Quality - run: poetry run pre-commit run --all-files --show-diff-on-failure + - name: Install dependencies + run: poetry install --no-interaction - - name: Run tests - run: poetry run pytest + - name: Run Integration Test + run: poetry run pytest tests/test_integration diff --git a/.gitignore b/.gitignore index df833f1..eead2d3 100644 --- a/.gitignore +++ b/.gitignore @@ -12,4 +12,4 @@ env/ *.pyc artifact/ -.coverage \ No newline at end of file +.coverage diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 190e5d3..7914cea 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -22,6 +22,7 @@ repos: - id: isort exclude: migrations/ args: + - --profile=black - --line-length=100 - repo: https://github.com/psf/black diff --git a/.vscode/launch.json b/.vscode/launch.json index c2cd0b8..0123508 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -1,7 +1,4 @@ { - // Use IntelliSense to learn about possible attributes. - // Hover to view descriptions of existing attributes. - // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 "version": "0.2.0", "configurations": [ @@ -27,7 +24,7 @@ "args": [ "tests/test_unit/test_runner.py", "-k", - "test_artifact_missing_retries" + "test_cancelled_step_is_not_retried" ] } diff --git a/CHANGELOG.md b/CHANGELOG.md index c3b21b8..6d9e87e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,12 +1,38 @@ # Changelog -## [Unreleased] +## [1.6.2] + +Version scope: Run-level Timeout. Distribution, runtime and report versions are synchronized to 1.6.2. This entry describes the working tree, not a verified release. + +### Added + +- Optional top-level `run_timeout_seconds`, validated as a finite positive number; omitted/null means unlimited run time. +- Monotonic watchdog shared across normal steps and retries; cancellation reasons preserve the first request. +- `TIMED_OUT` run status and metadata `cancel_reason`, `run_timeout_seconds`, `run_timed_out`. +- Configuration, cancellation reason, watchdog, status, setup timeout, retry delay, serialized attempt history and orphan process regression tests. +- v1.6.2 architecture, test matrix, acceptance criteria, completion checklist and testing-guide cases. ### Changed -- Reorganized the README around setup, execution, configuration and reports. -- Edited versioned documentation for clearer headings, shorter explanations and readable acceptance conditions. -- Expanded the commit checklist with concrete documentation and release checks. +- Consolidated README, lifecycle documentation, testing guidance and release checks for v1.6.2. + +- `RunResult.passed` follows summary status; `StepAttemptResult.passed` is removed in favor of `success`. +- Run timeout interrupts normal work without retrying cancelled attempts; reachable cleanup uses fresh tokens and final validation still runs. + +### Fixed + +- Standardized run timeout report metadata on `run_timed_out`, with explicit JSON regression assertions. + +- Omitted timeout returns None; watchdog uses Event.wait instead of the misspelled method. +- Terminator checks the original process group even after the direct child is reaped. +- Missing existence-rule targets report ARTIFACT_MISSING; size diagnostics no longer contain indentation spaces. +- Tests correct empty-directory classification, fixture paths, imports, report access and unordered status parameters. + +### Verification and limitations + +- Current commands and results: [v1.6.2 Definition of Done](docs/definition_of_done/definition_of_done_v1.6.2.md). +- CLI TIMED_OUT currently returns 0; cleanup has no independent total deadline. Exception finalization, platform checks and detached-process handling remain open. +- Package version is synchronized; tag and GitHub Release remain pending. ## [1.6.1] diff --git a/CommitManual.md b/CommitManual.md index 29adae1..687c47a 100644 --- a/CommitManual.md +++ b/CommitManual.md @@ -4,11 +4,11 @@ ## 提交前 -- [ ] 確認版本號；需要升版時，同步套件、runtime 與報告版本。 +- [ ] 確認版本號；需要升版時，依下方「版本號更新步驟」同步套件、runtime 與報告版本。 - [ ] 檢查程式變更是否符合需求，移除暫存程式與除錯輸出。 - [ ] 更新受影響的測試，執行相關測試並記錄結果。 - [ ] 檢查 `tests/` 下每個 Python 測試函式的 docstring，使用具體的 `Given`、`When`、`Then` 行說明前提、操作與預期結果。 -- [ ] 更新 [README](README.md) 的使用方式、範例與限制。 +- [ ] 依下方「README 撰寫條件」更新 [README](README.md)，確認外部使用者無須閱讀舊版或 diff 即可理解專案並開始使用。 - [ ] 更新對應版本的架構文件，確認介面與資料流。 - [ ] 在版本架構文件中加入或更新 Mermaid UML：使用 `classDiagram` 說明主要類別與關係，使用 `sequenceDiagram` 說明關鍵執行流程。圖中的名稱、呼叫順序與分支須符合該版本實作；規劃中的設計須另外標示。 - [ ] 在 [CHANGELOG](CHANGELOG.md) 記錄使用者可觀察的變更。 @@ -23,6 +23,30 @@ - [ ] 檢查文件：先說用途，使用短句與具體動詞；保留必要術語，刪除空泛宣傳與重複結論。 - [ ] 執行 `git diff --check`，檢查連結、命令與範例。 +## README 撰寫條件 + +README 面向第一次接觸專案的外部使用者，必須能獨立閱讀。依目前實作說明用途與操作，不要求讀者先了解上一版、Git diff 或內部開發進度。 + +- [ ] 內容依序涵蓋專案簡介、解決的痛點、專案架構、安裝、使用手冊（User Manual）與文件導覽；測試、CI、限制與後續規劃等補充內容保持精簡。 +- [ ] 簡介說明專案用途、適用情境與主要能力；痛點對應實際功能，不將規劃中的功能寫成已支援。 +- [ ] 架構提供主要元件、資料流與目錄用途，詳細介面與 UML 連結至對應版本架構文件。 +- [ ] 安裝列出環境需求、取得專案與安裝命令。使用手冊提供可操作的入門範例，說明設定儲存位置、執行方式、輸出位置與結果判讀。 +- [ ] 範例交代外部工具、裝置需求與需替換的值；區分可直接執行、示意資料與刻意失敗的範例。重要限制放在相關操作附近。 +- [ ] 使用現在式描述目前行為。版本號可用於標示目前版本與文件連結，不以「本版新增」、「相較上一版」、「已移除」或逐版差異作為功能教學。 +- [ ] 版本差異記錄於 CHANGELOG；升級與介面相容性細節放在對應版本文件；歷史測試結果、驗收與發佈狀態保留於測試矩陣、驗收或完成條件；未來計畫放在 Roadmap。README 提供入口，避免複製維護紀錄。 +- [ ] 保留使用者需要知道的現有限制與替代操作，不因移除版本比較而刪除必要資訊。 +- [ ] 文件導覽連結有效，並清楚區分使用教學、架構、測試、歷史變更及維護流程。 +- [ ] 以首次使用者角度通讀：能回答「這是什麼、解決什麼問題、如何安裝與使用、去哪裡看結果與更多文件」，且不需對照上一版。 + +## 版本號更新步驟 + +1. 修改 `pyproject.toml` 的 `[project].version`，使用不含 `v` 的版本號，例如 `version = "1.6.2"`。 +2. 核對 `runner/runner.py` 的 `DeviceTestRunner.VERSION` 與套件版本一致。報告的 `metadata.runner_version` 由此值產生。 +3. 同步 README、CHANGELOG、Roadmap 及該版本架構、驗收與完成條件中的版本狀態；保留舊版文件的歷史紀錄。 +4. 執行 `poetry check --lock` 確認設定與 lock file 一致。僅修改專案版本時，不需手動改寫依賴版本或 lock file；若另有依賴設定變更，重新產生並檢查 lock diff。 +5. 執行 `poetry version --short` 核對套件版本，並確認 `DeviceTestRunner.VERSION` 與報告建構程式使用的版本一致。更新已安裝套件時，另執行 `poetry install --no-interaction`。 +6. 執行相關檢查及 `git diff --check`，提交時納入 `pyproject.toml`、必要的 runtime 修改與文件。版本同步不代表已建立 tag 或發布 GitHub Release。 + ## 發佈前 - [ ] 確認該版本的完成條件與驗收條件已滿足。 @@ -48,7 +72,7 @@ Codex 必須先完整閱讀本清單，並核對下列文件。`` 代 | 文件 | 核對內容 | | --- | --- | -| `README.md` | 使用方式、範例與限制 | +| `README.md` | 依「README 撰寫條件」核對專案簡介、痛點、架構、安裝、使用手冊、文件導覽與必要限制；以目前行為說明 | | `docs/architecture/architecture_v.md` | 架構、介面、資料流，以及符合該版本實作的 UML 類別圖與循序圖 | | `docs/test_matrix/test_matrix_v.md` | 測試覆蓋、證據與 AI 協作新增或補強的案例，連結至測試指南的對應版本 | | `docs/test_guide.md` | Pytest 用法大綱、工具分類、版本索引與各版實際案例 | diff --git a/Makefile b/Makefile index 67678df..d84b141 100644 --- a/Makefile +++ b/Makefile @@ -8,4 +8,4 @@ test: test_cov: @pytest -v ./tests \ --cov=runner \ - --cov-report=term-missing \ No newline at end of file + --cov-report=term-missing diff --git a/README.md b/README.md index ece6592..21f266a 100644 --- a/README.md +++ b/README.md @@ -6,31 +6,124 @@ Device Test Runner 使用 YAML 定義裝置測試流程，執行既有的 Bash、Python、ADB 等命令，並保存每次執行的輸出與 JSON 報告。它負責安排測試步驟、驗證輸出檔案，以及依設定重試失敗步驟；裝置操作仍由你的腳本處理。 -目前 runtime／報告版本為 **v1.6.1**，支援五階段測試流程、步驟逾時、選擇性重試、輸出檔案驗證、程序群組清理與 Ctrl+C 取消。`pyproject.toml` 仍為 `1.6.0`，套件版本同步與發佈確認列於 [完成條件](docs/definition_of_done/definition_of_done_v1.6.1.md)。Recorder 管理與遠端執行仍在規劃中。 +目前版本為 **v1.6.2**。支援五階段測試流程、整次執行與步驟逾時、選擇性重試、輸出檔案驗證、程序群組清理與 Ctrl+C 取消。 + +## 解決的痛點 + +- 測試命令散落在不同腳本：用 YAML 統一描述準備、測試與清理順序，沿用既有 Bash、Python 或 ADB 命令。 +- 命令成功不代表產物正確：檢查檔案是否存在、大小、副檔名、目錄內容，以及 CSV／JSON 內容。 +- 偶發失敗難以重現：依失敗類型重試，為每次嘗試保留獨立 stdout／stderr。 +- 測試卡住或中途停止：設定逾時、提出取消請求，並依生命週期規則執行清理。 +- 人工整理結果費時：每次執行產生獨立目錄與 JSON 報告，供檢查與自動化處理。 + +## 專案架構 + +```text +YAML Configuration + ↓ +Config Loader + ↓ +RunnerConfig + ↓ +DeviceTestRunner + ├── Lifecycle Orchestration + ├── CancellationToken + ├── RetryPolicy + ├── SubprocessExecutor + ├── ArtifactManager + ├── ArtifactValidator + └── JsonReporter + ↓ + result.json / per-attempt logs / validation results +``` + +| 路徑 | 用途 | +| --- | --- | +| `main.py` | CLI 入口，載入設定並啟動測試 | +| `runner/` | 流程控制、命令執行、重試、取消、產物驗證與報告 | +| `configs/` | YAML 設定範例 | +| `scripts/` | 供測試流程呼叫的腳本 | +| `tests/` | 專案自動化測試 | +| `docs/` | 架構、測試、驗收與版本文件 | ## 安裝 需要 Python 3.10+ 與 Poetry 2.x。程序群組清理使用 POSIX signal API；目前本機驗證為 macOS，沒有 Windows 支援驗證。在專案目錄執行： ```bash -git clone git@github.com:andrew771027/DeviceTestRunner.git +git clone https://github.com/andrew771027/DeviceTestRunner.git cd DeviceTestRunner poetry install ``` Poetry 會建立虛擬環境，依 `poetry.lock` 安裝專案與開發依賴。後續指令都透過 `poetry run` 執行。 -## 執行測試流程 +## 使用手冊（User Manual） + +### 建立並執行第一個流程 + +將以下內容存為 `configs/hello.yaml`。此範例不需要連接裝置，會產生一份 CSV 並檢查內容。五個 lifecycle 階段都需提供 `steps`；不使用的階段填入 `[]`。 + +```yaml +test_case: + id: hello + name: Hello Runner + description: Create and validate a CSV file. +device: + serial: demo + product: demo + build: demo +lifecycle: + global_setup: + steps: [] + setup: + steps: [] + scenario: + steps: + - name: write_csv + type: command + command: 'printf "timestamp,power\n1,110\n" > result.csv' + timeout_second: 10 + teardown: + steps: [] + global_teardown: + steps: [] +artifact: + output_dir: artifacts + validation: + rules: + - name: check_csv + type: csv_content + path: result.csv + required_columns: [timestamp, power] + min_rows: 1 +``` + +在專案根目錄執行： ```bash -poetry run python main.py --config configs/sample.yaml +poetry run python main.py --config configs/hello.yaml +``` + +CLI 會顯示最終狀態。到 `artifacts/` 下該次執行目錄查看 `result.json`、`result.csv` 與步驟的 stdout／stderr log。以報告的 `summary.status` 判斷結果：`PASSED`、`FAILED`、`CANCELLED` 或 `TIMED_OUT`。 + +接著可將 `command` 替換成自己的腳本或裝置命令。命令的工作目錄是該次 run directory；引用專案內腳本時使用 `$DEVICE_TEST_RUNNER_ROOT`，取得輸出目錄則使用 `$RUN_ARTIFACT_DIR`。例如： + +```yaml +command: bash "$DEVICE_TEST_RUNNER_ROOT/scripts/run_scenario.sh" ``` -範例設定會示範失敗情境。2026-09-12 的本機驗證結果為 `FAILED`：`run_unstable_command` 失敗後未重試，下一步被跳過，另有五項必要檔案驗證失敗。完整紀錄見 [v1.6.0 完成條件](docs/definition_of_done/definition_of_done_v1.6.0.md)。 +使用 ADB 等外部命令前，需自行安裝工具並確認裝置連線。`device` 欄位用於報告資料，裝置選擇仍需在命令中指定。 + +專案另提供 [sample.yaml](configs/sample.yaml)，展示多個階段、重試與驗證規則： + +```bash +poetry run python main.py --config configs/sample.yaml +``` -每次執行會建立獨立的輸出目錄，保存 `result.json` 與各次嘗試的 stdout、stderr。請讀取報告中的 `summary.status` 判斷結果：`PASSED`、`FAILED` 或 `CANCELLED`。 +此設定包含失敗命令與缺少產物的情境，適合觀察失敗處理，不應以全部通過作為安裝成功的條件。 -## 常用詞彙 +### 常用詞彙 | 詞彙 | 意義 | | --- | --- | @@ -41,7 +134,7 @@ poetry run python main.py --config configs/sample.yaml | Artifact | 測試產生的檔案，例如 log、CSV 或 JSON | | Cleanup | 清理作業，包括 `teardown` 與 `global_teardown` | -## 測試生命週期 +### 測試生命週期 測試依序執行五個階段： @@ -79,11 +172,11 @@ global_teardown 上述為一般失敗路由。進入 setup 區塊後，即使 setup／scenario 取消，仍執行兩個 cleanup stages；run 開始前或 global_setup 取消時只執行 global_teardown。如果 global_setup 成功後、進入 setup 區塊前已觀察到取消，也會跳過 teardown。Cleanup attempt 使用新的 token；這些是受控流程的 best effort，不涵蓋未處理 Python exception 或 KeyboardInterrupt。 -最終 `summary.status` 以 `CANCELLED` 優先；沒有取消時，failed step、skipped step 或 required artifact failure 任一存在即為 `FAILED`，其餘為 `PASSED`。 +最終 `summary.status` 先依取消原因判定：RUN_TIMEOUT 為 `TIMED_OUT`，USER_REQUEST 為 `CANCELLED`；沒有取消原因但有 cancelled step 也為 `CANCELLED`。沒有取消時，failed step、skipped step 或 required artifact failure 任一存在即為 `FAILED`，其餘為 `PASSED`。 -## 設定測試流程 +### 設定測試流程 -以下設定示範裝置命令、重試與 CSV 驗證： +以下設定示範裝置命令、整次執行逾時、重試與 CSV 驗證。可另存為 YAML 後透過 `--config` 指定；請先將裝置序號 `ABC123` 換成自己的序號： ```yaml test_case: @@ -96,6 +189,8 @@ device: product: pixel build: build_12345 +run_timeout_seconds: 3600 + retry: max_attempts: 3 delay_seconds: 1 @@ -160,9 +255,9 @@ artifact: min_rows: 1 ``` -`after_step` 將 validation rule 綁定到指定 step，runner 會在該 step 每次 command 成功後立即驗證。`required` 預設為 `true`：required rule 失敗會使 attempt 失敗，且只有 failure type 出現在 `retry.retry_on`、尚未達 `max_attempts` 時才重試；`required: false` 的失敗仍寫入 report，但不影響 step 或 run 狀態。Lifecycle 結束後會再次對所有規則執行 final validation，包括有 `after_step` 的規則。YAML 未設定 `retry_on` 時預設為空清單，因此不重試；`none`、`cancelled` 與未知值會被拒絕。直接使用 Python `RetryConfig()` 的預設清單不同，詳見 [v1.6.0 的設定說明](docs/architecture/architecture_v1.6.0.md)。 +`after_step` 將 validation rule 綁定到指定 step，runner 會在該 step 每次 command 成功後立即驗證。`required` 預設為 `true`：required rule 失敗會使 attempt 失敗，且只有 failure type 出現在 `retry.retry_on`、尚未達 `max_attempts` 時才重試；`required: false` 的失敗仍寫入 report，但不影響 step 或 run 狀態。Lifecycle 結束後會再次對所有規則執行 final validation，包括有 `after_step` 的規則。YAML 未設定 `retry_on` 時預設為空清單，因此不重試；`none`、`cancelled` 與未知值會被拒絕。直接使用 Python `RetryConfig()` 的預設清單不同，使用 Python API 時請明確指定重試條件。 -## 透過 Python API 取消 +### 透過 Python API 取消 CLI 第一次 Ctrl+C／SIGINT 會呼叫 `token.cancel()`；第二次會拋出 `KeyboardInterrupt`，可能中斷 cleanup 與報告寫入。CLI 正常完成回傳 0、FAILED 回傳 1、CANCELLED 或 run 期間的 KeyboardInterrupt 回傳 130。SIGTERM 尚未接到 token。 @@ -211,9 +306,21 @@ Executor 每 0.1 秒檢查取消與 timeout。每次 attempt 建立獨立 sessio 同群組的 child／grandchild 已有清理測試；自行脫離群組的程序，以及直接 process 正常退出後留下的背景程序，不在目前保證內。詳見 [Process Lifecycle](docs/process_lifecycle.md)。Cancelled attempt 不做 attempt validation 或 retry，但 run 最後仍驗證所有 artifacts。 -從 v1.6.0 升級時，`SubprocessExecutor` 建構子需新增 `process_terminator`。從更舊版本升級時，Python 呼叫端也需調整 `SubprocessExecutor.execute(..., cancellation_token=...)`，以及手動建構 result dataclasses 時的新欄位。`run(config)` 仍可不傳 token。完整差異見 [Architecture](docs/architecture/architecture_v1.6.1.md)。 +### 設定整次執行逾時 + +在 YAML 頂層設定： + +```yaml +run_timeout_seconds: 3600 +``` + +省略或 null 代表不設 run deadline；step 的 `timeout_second` 仍有效。只接受有限正數，拒絕 bool、字串、零、負數、NaN 與 Infinity。Deadline 在 global_setup 前啟動，不隨 retry 重設。 -## 輸出檔案 +Run timeout 透過 cancellation 停止一般工作，report 為 `TIMED_OUT`、`cancel_reason: run_timeout`。被中斷的 attempt 仍是 CANCELLED，`timed_out` 專指 step timeout。Retry delay 中逾時會保留前一次失敗，不新增 attempt。第一個取消原因不會被後續請求覆寫。 + +Cleanup 使用新 token 與自己的 step timeout；final validation 不會被取消中斷。Watchdog 在上述工作結束後停止，因此不保證 CLI 在設定秒數內返回。`metadata.run_timed_out` 在取消原因為 RUN_TIMEOUT 時是 `true`，其他情況為 `false`。CLI 目前沒有 TIMED_OUT 專用非零 exit code，會回傳 0；自動化應檢查 `summary.status`，不能只依 exit code 判定成功。 + +### 輸出檔案 每次測試執行會建立獨立的 run directory。 @@ -254,7 +361,7 @@ artifacts/ 相對路徑的 artifact validation rule 會以該次 run directory 為基準解析。每一次 retry 都有獨立的 stdout／stderr log，避免後一次 attempt 覆蓋先前的診斷資訊。 -## 報告範例 +### 報告範例 以下是單一步驟、無 artifact rules 的示意資料，並非 sample.yaml 的實際執行報告。 @@ -267,10 +374,13 @@ artifacts/ "device_serial": "demo", "device_product": "demo", "device_build": "demo", - "runner_version": "1.6.1", + "runner_version": "1.6.2", "started_at": "2026-09-12T00:00:00+00:00", "finished_at": "2026-09-12T00:00:01+00:00", - "cancel_requested": false + "cancel_requested": false, + "cancel_reason": null, + "run_timeout_seconds": null, + "run_timed_out": false }, "summary": { "status": "PASSED", @@ -319,28 +429,7 @@ artifacts/ } ``` -結果消費端應以 `summary.status` 判斷 run；`RunResult.passed` 目前只檢查 step success，無法完整反映取消請求或 final artifact failure。`StepAttemptResult.passed` 也只檢查 exit code，請使用 `success` 與 failure flags。 - -## 架構 - -```text -YAML Configuration - ↓ -Config Loader - ↓ -RunnerConfig - ↓ -DeviceTestRunner - ├── Lifecycle Orchestration - ├── CancellationToken - ├── RetryPolicy - ├── SubprocessExecutor - ├── ArtifactManager - ├── ArtifactValidator - └── JsonReporter - ↓ - result.json / per-attempt logs / validation results -``` +結果消費端應以 `summary.status` 判斷 run；Python API 的 `RunResult.passed` 只於 status 為 `PASSED` 時回傳 true。單次 attempt 請讀取 `success`、`failure_type`、`timed_out` 與 `cancelled`。 ## 執行專案測試 @@ -374,9 +463,7 @@ poetry run pytest -m cancelled poetry run pytest -m artifact ``` -歷史驗證紀錄（2026-09-12，Python 3.14）：`.venv/bin/python -m pytest -q` → **153 passed in 39.39s**。150 個測試函式皆有 Given／When／Then 說明；參數化後共 153 個案例。 - -本次 v1.6.1 驗證與 162 個測試函式／168 個案例的對照，見 [完成條件](docs/definition_of_done/definition_of_done_v1.6.1.md) 與 [測試矩陣](docs/test_matrix/test_matrix_v1.6.1.md)。 +測試工具與撰寫方式見 [測試指南](docs/test_guide.md)；覆蓋範圍與驗證紀錄見 [測試矩陣](docs/test_matrix/test_matrix_v1.6.2.md) 與 [完成條件](docs/definition_of_done/definition_of_done_v1.6.2.md)。 ## 持續整合 @@ -389,35 +476,27 @@ poetry install poetry run pytest ``` -## 手動更新版本文件 - -`.github/workflows/manual.yml` 的 `workflow_dispatch` 接受 `release_version`（如 `1.6.0`，不含 v）、`target_branch`（實際 checkout 與 push 的既有分支）及 `model`。GitHub Actions 的 Use workflow from 選項決定讀取哪個分支的 workflow 定義。 - -流程先安裝專案、跑 baseline tests，再請 Codex 更新文件與測試說明；驗證 `pytest -q`、`git diff --check` 與 Given／When／Then 行數後，有變更才 commit 並直接 push 到目標分支。這個流程不建立 PR、tag 或 GitHub Release。 - -Repository secret 名稱為 `OPENAI_API_KEY`，在 CLI invocation 映射成 `CODEX_API_KEY`；`--approve-for-me` 與 `--sandbox` 不同時使用。目標分支需允許這次 push，API 帳戶需有可用額度及模型存取權。 - -## 開發與版本規劃 +## 限制與後續規劃 -目前先補齊單機執行與取消流程，再加入可重用設定與 recorder 管理： +目前以單機命令流程為主。Recorder 管理、批次與並行執行、controller／worker 遠端執行仍在規劃中；詳細範圍與進度見 [Roadmap](docs/roadmap.md)。 -1. v1.6.1：已實作程序群組終止、輸出串流收尾與 SIGINT handler；平台驗證、版本同步與發佈待完成。 -2. v1.6.2：整次 run 的逾時設定。 -3. v1.6.3：取消後的清理範圍、時間限制與部分結果保存。 -4. v1.7.0～v1.7.2：YAML 靜態變數、環境變數與執行資訊。 -5. v1.8 之後：recorder、hooks、執行摘要、批次與並行執行。 -6. v2.0：controller／worker 遠端執行。 +程序清理與平台限制見 [Process Lifecycle](docs/process_lifecycle.md)。執行逾時後的 cleanup 與最終產物驗證可能延長返回時間；自動化整合請讀取報告狀態，`TIMED_OUT` 目前仍會回傳 exit code 0。 -詳細範圍與 keyword-driven 設計見 [Roadmap](docs/roadmap.md)。歷史變更見 [CHANGELOG](CHANGELOG.md)。提交與發佈前請使用 [提交檢查清單](CommitManual.md)。 +維護者提交、發佈與操作手動文件更新 workflow 時，請參閱 [提交與發佈檢查清單](CommitManual.md)。 ## 文件 | 文件 | 用途 | | --- | --- | -| [架構 v1.6.1](docs/architecture/architecture_v1.6.1.md) | 元件、取消路由、報告欄位與相容性 | +| [使用手冊](#使用手冊user-manual) | 建立設定、執行流程與閱讀報告 | +| [CHANGELOG](CHANGELOG.md) | 歷史變更與版本差異 | +| [Roadmap](docs/roadmap.md) | 後續功能與版本規劃 | +| [Process Lifecycle](docs/process_lifecycle.md) | 程序清理行為與限制 | +| [提交檢查清單](CommitManual.md) | 提交、發佈與文件維護流程 | +| [架構 v1.6.2](docs/architecture/architecture_v1.6.2.md) | 元件、取消路由、報告欄位與相容性 | | [測試指南](docs/test_guide.md) | Pytest 工具、fixture、Mock 與各版用法 | -| [測試矩陣 v1.6.1](docs/test_matrix/test_matrix_v1.6.1.md) | 功能對應的測試與覆蓋限制 | -| [驗收條件 v1.6.1](docs/acceptance_criteria/acceptance_criteria_v1.6.1.md) | 可觀察的預期行為 | -| [完成條件 v1.6.1](docs/definition_of_done/definition_of_done_v1.6.1.md) | 驗證紀錄與待完成的發佈項目 | +| [測試矩陣 v1.6.2](docs/test_matrix/test_matrix_v1.6.2.md) | 功能對應的測試與覆蓋限制 | +| [驗收條件 v1.6.2](docs/acceptance_criteria/acceptance_criteria_v1.6.2.md) | 可觀察的預期行為 | +| [完成條件 v1.6.2](docs/definition_of_done/definition_of_done_v1.6.2.md) | 驗證紀錄與待完成的發佈項目 | `docs/` 內的舊版文件保留當時的設計與介面，使用時請確認版本。 diff --git a/configs/sample.yaml b/configs/sample.yaml index e67400d..9d5c29c 100644 --- a/configs/sample.yaml +++ b/configs/sample.yaml @@ -8,6 +8,8 @@ device: product: pixel build: test_build +run_timeout_seconds: 3600 + retry: max_attempts: 3 delay_seconds: 1 diff --git a/docs/acceptance_criteria/acceptance_criteria_v1.6.2.md b/docs/acceptance_criteria/acceptance_criteria_v1.6.2.md new file mode 100644 index 0000000..c225545 --- /dev/null +++ b/docs/acceptance_criteria/acceptance_criteria_v1.6.2.md @@ -0,0 +1,22 @@ +# Device Test Runner v1.6.2 Acceptance Criteria + +Version scope: Run-level Timeout. These criteria describe the inspected working tree, not release approval. + +| ID | Given | When | Then | Evidence / status | +| --- | --- | --- | --- | --- | +| AC-1 | YAML omits run timeout | Configuration loads | Timeout is None; step timeout remains available | Verified by configuration test and source | +| AC-2 | Zero, negative, non-finite or boolean timeout | Parser validates | ValueError rejects the value | Verified, parameterized tests | +| AC-3 | A token already has a reason | Another cancellation arrives | First reason is preserved | Verified in both orders | +| AC-4 | A running watchdog | Deadline expires or stop joins it | Deadline requests RUN_TIMEOUT; stopped thread does not remain running | Verified, watchdog tests | +| AC-5 | A long command with a longer step timeout | Run deadline expires | Command is cancelled, no retry, remaining scenario steps do not run | Verified, real subprocess | +| AC-6 | A failed attempt waiting to retry | Run deadline expires | No new attempt; original PROCESS_ERROR remains | Verified, JSON assertions | +| AC-7 | Setup is interrupted after global setup succeeded | Cleanup begins | Teardown and global_teardown execute; scenario is skipped | Verified, file markers | +| AC-8 | Run timeout cancellation | Result is built and saved | TIMED_OUT with run_timeout reason; attempt flags preserve step/run distinction | Verified, status and integration tests | +| AC-9 | A step reaches its own timeout | Executor returns | FailureType.TIMEOUT and timed_out=true | Verified, mocked executor timing | +| AC-10 | Direct child is reaped but its group survives | Terminator is invoked | Descendant exits and terminated=true | Verified, real orphan fixture | + +Tests and boundaries are mapped in the [test matrix](../test_matrix/test_matrix_v1.6.2.md). Exact local results are in the [Definition of Done](../definition_of_done/definition_of_done_v1.6.2.md). + +## Acceptance limits + +Run timeout is not a hard upper bound on cleanup, artifact validation or report writing. CLI currently returns 0 for TIMED_OUT; nonzero CLI timeout handling is not accepted as complete. Unexpected-exception finalization, Linux CI, detached descendants, publication remain open. Local tests do not establish release readiness for these boundaries. diff --git a/docs/architecture/architecture_v1.6.2.md b/docs/architecture/architecture_v1.6.2.md new file mode 100644 index 0000000..68a0c35 --- /dev/null +++ b/docs/architecture/architecture_v1.6.2.md @@ -0,0 +1,104 @@ +# Device Test Runner Architecture v1.6.2 — Run-level Timeout + +本文件說明整次 run 的截止時間、取消原因與報告。基準為 `eea8507` 加上本次工作目錄 source／test 修正，對照 `v1.6.1` tag；不是已發佈版本的聲明。 + +## 設定與介面 + +YAML 頂層 `run_timeout_seconds` 可省略或設為 null，代表不設 run deadline；step 的 `timeout_second` 仍有效。數字必須有限且大於零；bool、字串、零、負數與非有限值被 ConfigLoader 拒絕。直接建構 RunnerConfig 不經 loader，watchdog 建構子僅檢查 `<= 0`，不提供相同的完整驗證。 + +`RunTimeoutWatchdog(timeout_seconds, cancellation_token)` 只可 start 一次，使用 monotonic 時鐘與 Event.wait。stop 設定事件並 join 執行緒。`CancellationToken.cancel(reason=USER_REQUEST)` 保留第一個原因；實際回傳是否接受的 bool，但目前型別標註仍是 `None`。 + +## 類別與依賴 + +```mermaid +classDiagram + class RunnerConfig { + run_timeout_seconds + } + class DeviceTestRunner { + run(config, cancellation_token) + } + class RunTimeoutWatchdog { + deadline + start() + stop() + } + class CancellationToken { + reason + is_cancelled + cancel(reason) + } + class SubprocessExecutor { + execute() + } + class ProcessTerminator { + terminate_process_group(process) + } + class JsonReporter { + save(result, output_dir) + } + DeviceTestRunner --> RunnerConfig + DeviceTestRunner --> RunTimeoutWatchdog + RunTimeoutWatchdog --> CancellationToken + DeviceTestRunner --> SubprocessExecutor + SubprocessExecutor --> CancellationToken + SubprocessExecutor --> ProcessTerminator + DeviceTestRunner --> JsonReporter +``` + +## 執行流程 + +```mermaid +sequenceDiagram + participant R as DeviceTestRunner + participant W as RunTimeoutWatchdog + participant T as CancellationToken + participant E as SubprocessExecutor + participant P as ProcessTerminator + participant V as ArtifactValidator + participant J as JsonReporter + opt run_timeout_seconds is configured + R->>W: start() + end + R->>E: execute(normal step, shared token) + opt deadline expires while process runs + W->>T: cancel(RUN_TIMEOUT) + E->>T: inspect is_cancelled + E->>P: terminate_process_group(process) + P-->>E: group cleanup complete + end + E-->>R: attempt result + Note over R: cancelled attempt skips validation and retry + opt global_setup succeeded + R->>E: execute(teardown, fresh token) + end + R->>E: execute(global_teardown, fresh token) + R->>V: validate_all(final rules) + R->>W: stop() in finally if watchdog exists + Note over R: build result using cancellation reason + R->>J: save(result, output_dir) +``` + +Watchdog 在 run directory 建立後、global_setup 前開始；retry 不會重設 deadline。一般 stage 邊界、executor polling 與 retry delay 都會讀取 token。於 retry delay 逾時不新增 attempt，也不將前一次 PROCESS_ERROR 改成 CANCELLED。Setup 逾時後，只要 global_setup 成功，就會執行 teardown；正常控制流程也會到達 global_teardown。 + +Watchdog 持續到 cleanup 與 final validation 結束，因此這些階段也可能設定 run 的取消原因；cleanup 使用新 token，依自己的 step timeout 執行。Artifact validation 本身不會被 token 中斷。finally 只保證 stop watchdog；非預期例外仍可能跳過 cleanup、final validation 或 report。Report 建構與寫入在 watchdog 停止之後。 + +## 報告與相容性 + +| 欄位／結果 | v1.6.2 行為 | +| --- | --- | +| metadata.runner_version | `1.6.2`，與 pyproject.toml 的 distribution version 一致 | +| metadata.cancel_reason | `user_request`、`run_timeout` 或 null | +| metadata.run_timeout_seconds | 設定數值或 null | +| metadata.run_timed_out | RUN_TIMEOUT 時為 true，其餘為 false | +| summary.status | RUN_TIMEOUT → TIMED_OUT；USER_REQUEST 或 cancelled steps → CANCELLED；失敗／跳過／required artifact 失敗 → FAILED；其餘 PASSED | +| attempt | Run timeout 中斷 command 為 CANCELLED、cancelled=true、timed_out=false；step timeout 仍是 TIMEOUT | +| RunResult.passed | 只在 summary.status 為 PASSED 時為 true | + +`StepAttemptResult.passed` 已移除，使用 `success`。JsonReporter 透過 dataclasses.asdict 保存原始欄位與 attempt history。CLI 目前只對 CANCELLED 回傳 130、FAILED 回傳 1；TIMED_OUT 會落入回傳 0 的分支。這是現有缺口，不代表逾時成功，也尚無 CLI 整合驗證。 + +## Process group 與限制 + +Terminator 不再因 direct child 已退出就返回；getpgid 找不到 child 時，以原 PID 探測 group。此回退依賴 executor 的 start_new_session=True。已有 orphan fixture 驗證同群組後代被清理，但不涵蓋 detached descendants，也不保證 executor 在所有正常結束路徑主動呼叫清理。 + +這是停止一般工作的 deadline，不是整個 CLI 的硬性時間上限。Cleanup scope／獨立總預算、例外後部分報告、Linux 平台與真實 CLI signals 仍待驗證或後續版本實作。詳見 [完成條件](../definition_of_done/definition_of_done_v1.6.2.md)。 diff --git a/docs/definition_of_done/definition_of_done_v1.6.2.md b/docs/definition_of_done/definition_of_done_v1.6.2.md new file mode 100644 index 0000000..4e879f7 --- /dev/null +++ b/docs/definition_of_done/definition_of_done_v1.6.2.md @@ -0,0 +1,56 @@ +# Device Test Runner v1.6.2 Definition of Done + +Version scope: Run-level Timeout. Source baseline: `eea8507` plus the inspected working tree, compared with `v1.6.1`. Local history was reviewed through the initial commit; no v1.6.2 tag was found locally. + +## Product and architecture + +- [x] Optional run timeout; loader rejects non-positive, non-finite and boolean values. +- [x] Monotonic watchdog shares one deadline across normal steps and retry delays. +- [x] Cancellation preserves the first reason; RUN_TIMEOUT maps to TIMED_OUT. +- [x] Interrupted commands do not retry; saved attempt history distinguishes cancellation from step timeout. +- [x] Setup timeout reaches teardown after successful global setup; global_teardown runs on the tested normal control path. +- [x] Terminator cleans an existing group after its direct child exits. +- [x] Report metadata and compatibility changes match the source, including the `run_timed_out` boolean. +- [x] Mermaid class and sequence diagrams describe current control flow and limits. + +## Quality and documentation + +- [x] README, CHANGELOG, Roadmap, Process Lifecycle, testing guide and four v1.6.2 documents updated. +- [x] Older versioned documents and historical test results retained. +- [x] 185 test functions have concrete Given/When/Then docstrings; parametrization collects 205 cases. +- [x] Added 19 missing descriptions and merged a duplicate Then line; executable test AST is unchanged. +- [x] Test matrix records AI collaboration evidence and remaining coverage gaps. + +### Local verification — 2026-09-30 + +Environment: macOS 15.5, Darwin x86_64, Python 3.14.0, repository `.venv`. Results describe this working tree, not Linux CI or a published release. + +| Check | Command / method | Observed result | +| --- | --- | --- | +| Before documentation edits | `.venv/bin/python -m pytest -q` | 205 passed in 41.17s | +| After initial documentation edits | `.venv/bin/python -m pytest -q` | 205 passed in 42.13s | +| After metadata field rename | `.venv/bin/python -m pytest -q` | 205 passed in 42.58s | +| Test descriptions | AST audit plus workflow-style line counts | 185 functions; one Given/When/Then per function | +| Executable test behavior | Compare AST after removing docstrings against pre-edit snapshot | Unchanged | +| Test lint | `.venv/bin/python -m pre_commit run flake8 --files` with the eight modified test files | Passed | + +Documentation-stage verification: `.venv/bin/python -m pytest -q` — **205 passed in 40.49s**. Relative-link validation checked 65 local links; matrix test references resolve; README JSON examples parse and Python examples compile as AST. `git diff --check` passed. A final docstring wording clarification preserves the same executable AST. Sample configuration and remote release checks were not run. Earlier sample results remain historical evidence only. + +## Release and follow-up + +- [x] Distribution version in pyproject.toml is `1.6.2`, matching runtime/report version. +- [ ] Handle TIMED_OUT with a nonzero CLI exit code and add CLI integration coverage; current code falls through to 0. +- [x] Standardize source and report metadata on `run_timed_out`; assert true for timeout and false by default. +- [ ] Correct CancellationToken.cancel return annotation. +- [ ] Define validation guarantees for directly constructed Python configurations. +- [ ] Verify multi-step real deadline accumulation, boundary races and timeout during cleanup/final validation. +- [ ] Implement or validate independent cleanup budgets and exception-safe partial report finalization. +- [ ] Verify Linux CI, detached descendants, normal completion with surviving background processes and real CLI signals. +- [ ] Verify a passing sample configuration and GitHub issue/milestone state. +- [ ] Create or verify tag v1.6.2 and GitHub Release publication; remote publication was not checked. + +Release readiness: source behavior is locally tested, but unchecked release and boundary items remain open. No commit, push, tag or GitHub Release was performed. + +## Documentation consolidation + +README timeout instructions, process lifecycle behavior and CHANGELOG changes are consolidated under v1.6.2. Earlier versioned documents remain historical records. This final consolidation changes documentation only; it does not rerun or replace the observed test results above. diff --git a/docs/images/3-1.PNG b/docs/images/3-1.PNG new file mode 100644 index 0000000..ecc4e20 Binary files /dev/null and b/docs/images/3-1.PNG differ diff --git a/docs/images/3-2.PNG b/docs/images/3-2.PNG new file mode 100644 index 0000000..6c0e15f Binary files /dev/null and b/docs/images/3-2.PNG differ diff --git a/docs/images/3-3.PNG b/docs/images/3-3.PNG new file mode 100644 index 0000000..368b698 Binary files /dev/null and b/docs/images/3-3.PNG differ diff --git a/docs/images/3-4.PNG b/docs/images/3-4.PNG new file mode 100644 index 0000000..502c54b Binary files /dev/null and b/docs/images/3-4.PNG differ diff --git a/docs/images/3-5.PNG b/docs/images/3-5.PNG new file mode 100644 index 0000000..bdde2a6 Binary files /dev/null and b/docs/images/3-5.PNG differ diff --git a/docs/images/3-6.PNG b/docs/images/3-6.PNG new file mode 100644 index 0000000..a4a2892 Binary files /dev/null and b/docs/images/3-6.PNG differ diff --git a/docs/images/3-7.PNG b/docs/images/3-7.PNG new file mode 100644 index 0000000..5b18287 Binary files /dev/null and b/docs/images/3-7.PNG differ diff --git a/docs/images/3-8.PNG b/docs/images/3-8.PNG new file mode 100644 index 0000000..2941cac Binary files /dev/null and b/docs/images/3-8.PNG differ diff --git a/docs/process_lifecycle.md b/docs/process_lifecycle.md index e98eabb..5c00fb3 100644 --- a/docs/process_lifecycle.md +++ b/docs/process_lifecycle.md @@ -1,10 +1,10 @@ # Device Test Runner Process Lifecycle -Version: v1.6.1 +Version: v1.6.2 ## Purpose -本文件說明一次 attempt 如何啟動、停止與收尾，以及清理和 retry 的關係。閱讀順序是「程序 → 群組 → 輸出 → 下一次 attempt」。類別介面與 UML 見 [Architecture](architecture/architecture_v1.6.1.md)，測試案例見 [Test Matrix](test_matrix/test_matrix_v1.6.1.md)。 +本文件說明一次 attempt 如何啟動、停止與收尾，以及清理和 retry 的關係。閱讀順序是「程序 → 群組 → 輸出 → 下一次 attempt」。類別介面與 UML 見 [Architecture](architecture/architecture_v1.6.2.md)，本版測試案例見 [Test Matrix](test_matrix/test_matrix_v1.6.2.md)。 ## Process model @@ -39,13 +39,25 @@ Executor 每輪依序檢查 process 完成、token 取消、step timeout。正 已觀察到完成的程序優先。如果程序仍執行中，取消與 timeout 同時可見，取消優先。以下流程描述清理成功的路徑；清理例外見後面的錯誤處理說明。 +## Run-level timeout + +YAML 的 `run_timeout_seconds` 控制整次 run 的 deadline，省略時不啟動 watchdog。Watchdog 在 global_setup 前開始，與 steps、retry delay 共用同一截止時間；逾時以 RUN_TIMEOUT 取消 token。第一個取消原因不會被覆寫。 + +被中斷的 attempt 記錄 CANCELLED，run 記錄 TIMED_OUT，metadata.run_timed_out 為 true。Step timeout 仍記錄 TIMEOUT。Retry delay 期間逾時不新增 attempt，也不改寫前一次失敗。 + +Cleanup 使用新 token 與自己的 step timeout。Watchdog 到 cleanup 與 final validation 後才停止；report 在停止後建立。這不保證整個程序於 deadline 內退出，非預期例外也不保證 cleanup 或 report 完成。 + ## Cleanup sequence ```mermaid flowchart TD A[觀察到 cancel 或 timeout] --> B{清理開始時直接 process 已退出?} - B -- Yes --> C[Terminator 直接返回，不送訊號] - B -- No --> D[取得 PGID，對群組送 SIGTERM] + B -- Yes --> P[以原 PID 探測 process group] + P --> Q{群組仍存在?} + Q -- No --> C[返回，無需終止] + Q -- Yes --> D[對群組送 SIGTERM] + B -- No --> O[取得 PGID] + O --> D D --> E[回收直接子程序並檢查群組] E --> F{寬限時間內群組消失?} F -- Yes --> G[Terminator 返回] @@ -58,7 +70,7 @@ flowchart TD K --> L[返回 attempt 結果，Runner 判斷後續流程] ``` -取得 PGID 或送出 signal 時，如果程序／群組已不存在，程式會處理 `ProcessLookupError`；不要求每次檢查之間程序狀態都保持不變。 +getpgid 找不到 direct child 時，改用原 PID 探測 group；此回退依賴 start_new_session=True。群組不存在才直接返回。送出 signal 時也處理 ProcessLookupError；不要求每次檢查之間程序狀態保持不變。 ### 為什麼不能只看 parent？ @@ -112,22 +124,22 @@ flowchart TD 第一次 Ctrl+C／SIGINT 呼叫 token.cancel，讓 executor 與 runner 走受控取消流程。第二次 handler 呼叫拋出 KeyboardInterrupt，可能中斷正在進行的清理與報告寫入。 -`main.py` 在 run 的 try 區塊捕捉 KeyboardInterrupt 後回傳 130，並在 finally 還原原本 SIGINT handler。一般結果的 CLI exit code 為 PASSED → 0、FAILED → 1、CANCELLED → 130。 +`main.py` 在 run 的 try 區塊捕捉 KeyboardInterrupt 後回傳 130，並在 finally 還原原本 SIGINT handler。一般結果的 CLI exit code 為 PASSED → 0、FAILED → 1、CANCELLED → 130。TIMED_OUT 目前落入回傳 0 的分支，是待修正缺口。 要區分兩個方向：Terminator 會送 SIGTERM 給 attempt 群組；但外部送 SIGTERM 給 runner 本身，目前沒有接成 token cancellation。 ## Evidence and limits -[Test Matrix](test_matrix/test_matrix_v1.6.1.md) 記錄 parent／child／grandchild、忽略 SIGTERM、reader 收尾、log 保存與 retry 前清理的證據。Fixture、monkeypatch 與模擬方式見 [Test Guide](test_guide.md#v1-6-1)。 +[v1.6.1 Test Matrix](test_matrix/test_matrix_v1.6.1.md) 保留 parent／child／grandchild、忽略 SIGTERM、reader 收尾、log 保存與 retry 前清理的證據。本版新增 run timeout 與 orphan process 證據見 [v1.6.2 Test Matrix](test_matrix/test_matrix_v1.6.2.md)。Fixture、monkeypatch 與模擬方式見 [Test Guide](test_guide.md#v1-6-2)。 目前限制： - 使用 POSIX process-group API；已有本機 macOS 測試紀錄，不能推論 Windows 支援或 Linux CI 已通過。 - 自行建立新 session／脫離群組的後代不在清理範圍。 -- Terminator 開始時直接 process 已退出，會直接返回；這不保證其後代也已退出。 +- Direct child 已退出時仍可清理原 group；但需要呼叫端實際進入 terminator，不代表所有正常完成路徑皆會清理。 - 正常完成後留下背景後代的情境，沒有主動清理保證。 - 已退出但尚未回收的 zombie 可能仍保有 PID；群組消失時間受 OS 回收行為影響。 - SIGINT 測試直接呼叫 handler，尚未驗證真實終端訊號、handler 還原與 CLI exit code 的完整流程。 - 第二次 Ctrl+C、未處理例外與重複清理失敗，不保證完成 teardown 或保存完整報告。 -本文件描述實作與既有測試證據，不新增測試執行紀錄；實際命令與結果見 [Definition of Done](definition_of_done/definition_of_done_v1.6.1.md)。 +本文件描述實作與既有測試證據，不新增測試執行紀錄；實際命令與結果見 [Definition of Done](definition_of_done/definition_of_done_v1.6.2.md)。 diff --git a/docs/roadmap.md b/docs/roadmap.md index 8d11872..0d37ba4 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -1,6 +1,6 @@ # Device Test Runner Roadmap -目前已實作 v1.6.1 的程序群組終止、reader 收尾與 SIGINT handler，並通過本機測試。套件版本同步、平台驗證與發佈仍待完成。接下來處理 run 逾時與取消後的清理，再加入 YAML 變數與 recorder 管理。 +目前已實作 v1.6.2 run-level timeout、程序群組終止、reader 收尾與 SIGINT handler，並通過本機測試。套件、runtime 與報告版本已同步為 1.6.2；平台驗證與發佈仍待完成。接下來處理取消後的清理，再加入 YAML 變數與 recorder 管理。 本文件保留各版本的功能規劃。`Completed` 表示該節記錄的功能已完成；`Planned` 與 `Future` 表示尚未實作。實際發佈條件與驗證結果請見各版本的完成條件文件。 @@ -462,33 +462,17 @@ Runtime 為 `1.6.1`，套件仍為 `1.6.0`。完整結果與發佈待辦見 [v1. ### v1.6.2 — Run-level Timeout -#### 目標 - -在既有 per-step timeout 之外，限制一般 run 工作的總執行時間，並透過統一 cancellation 流程停止工作。 - -#### 功能範圍 - -* 新增 `run_timeout_seconds` 設定與驗證，未設定時維持既有行為。 -* Run timeout 轉為 cancellation request，停止一般 stages、執行中的 command 與 retry delay。 -* 明確區分 step timeout 與 run timeout，報告保留取消原因。 -* Step timeout 可依 `retry_on` 重試；run timeout 不應因下一次 attempt 而重設 deadline 或繼續一般工作。 - -#### Relationship to v1.6.0 - -v1.6.0 只有 `timeout_second` 的 step deadline，沒有 run deadline。現有 token 只有取消狀態，`cancel_requested` 也不記錄原因，因此需要擴充原因資訊，避免將 run timeout 和使用者取消混為一談。 +#### 目標與實作 -相容性方向：沿用 cancellation 執行路徑；保留既有 attempt `timed_out` 對 step timeout 的意義。Run-level 原因欄位與最終 status 的 schema 在實作時明確定義，不能僅將 run timeout 冒充為某一步的 TIMEOUT。 +已新增可省略的 `run_timeout_seconds`、monotonic watchdog、first-reason-wins cancellation 與 TIMED_OUT status。一般 steps 與 retry delay 共用同一 deadline；step timeout 的 TIMEOUT 分類保持不變。Setup timeout 後可執行 teardown 與 global_teardown，報告保留原始 attempt history。 -#### 驗收重點 +#### 狀態 -* 多個未超時的 steps 累計仍可觸發 run timeout。 -* Retry delay 與 attempts 共用同一個 run deadline。 -* Run deadline 停止一般工作後仍進入 cleanup；cleanup 使用 v1.6.3 的獨立 scope／timeout。 -* 明確定義計時起點、涵蓋階段與 report finalization 邊界；run timeout 不等同整個程序必須立即退出。 +Implemented; locally tested working tree. Release pending. -#### 狀態 +Watchdog 在 global_setup 前啟動，到 cleanup 與 final validation 後才停止；cleanup 使用新 token，尚無獨立總預算。真實短 steps 累計 deadline、cleanup 期間逾時與例外後 report 仍缺少驗證。CLI TIMED_OUT 目前回傳 0，需修正與增加 CLI 測試。套件、runtime 與報告版本已同步為 1.6.2；Linux CI 與發佈另行完成。 -Planned +介面與邊界見 [架構](architecture/architecture_v1.6.2.md)，證據見 [完成條件](definition_of_done/definition_of_done_v1.6.2.md)。v1.6.3 的 cleanup scope／總預算仍是規劃，不能視為本版保證。 ### v1.6.3 — Cancellation-aware Cleanup @@ -1238,11 +1222,11 @@ Done ## 開發優先順序 -目前已實作並本機驗證 v1.6.1 process-group cleanup；完整取消保證與發佈仍待完成。接下來的開發優先順序： +目前已實作 v1.6.2 run-level timeout 與 process-group cleanup；完整取消保證與發佈仍待完成。接下來的開發優先順序： ```text -1. v1.6.1 平台驗證、套件版本同步與發佈確認 -2. v1.6.2 Run-level Timeout +1. 程序清理的平台驗證與發佈確認 +2. v1.6.2 CLI timeout exit code、平台驗證與發佈 3. v1.6.3 Cancellation-aware Cleanup 4. v1.7.x YAML Variables, Environment and Runtime Context 5. v1.8 Recorder Lifecycle diff --git a/docs/test_guide.md b/docs/test_guide.md index 9a970ca..82a5cb9 100644 --- a/docs/test_guide.md +++ b/docs/test_guide.md @@ -25,19 +25,22 @@ | `@pytest.mark.parametrize` | Decorator | 同一段測試用不同資料執行 | [v1.0.0](#v1-0-0) | | `capsys`、`readouterr()` | 內建 fixture 與方法 | 擷取 console 的 stdout／stderr | [v1.3.5](#v1-3-5) | | `pytest.raises()` | 例外檢查工具 | 驗證操作拋出指定例外 | [v1.2.0](#v1-2-0)；[v1.6.1](#v1-6-1) 的 KeyboardInterrupt | +| `shlex.quote()`、`repr()` | 標準函式庫／內建函式 | 分別處理 shell 參數與 Python 字串常值 | [v1.6.2 命令引用](#v1-6-2-command-quoting) | | 模擬時鐘 | 測試技巧 | 由測試推進時間，不必真的等待 | [v1.6.0](#v1-6-0) | | `pytest.approx()` | 比較工具 | 容許浮點數計算的小誤差 | [v1.6.0](#v1-6-0) | | `@pytest.mark.retry` 等 | 自訂 marker | 將測試分類，方便選擇執行 | [v1.5.0](#v1-5-0) | | `-m`、`-k`、`檔案::函式` | 執行選項 | 分別依 marker、名稱或指定函式選擇測試 | 各版本的「執行與定位案例」 | | `-q`、`-v` | 輸出選項 | 顯示精簡結果或逐一列出案例 | 各版本的執行範例 | +本版新增案例也使用 `json.loads` 讀取落盤報告，以及 `time.monotonic` 搭配有上限的輪詢；兩者是標準函式庫，不是 pytest fixture。見 [v1.6.2](#v1-6-2)。 + ## Fixture 與輔助腳本 Pytest fixture 是 pytest 準備後交給測試的資源。測試參數中的 `tmp_path`、`monkeypatch` 與 `capsys` 都是內建 fixture；不需要自己宣告 `@pytest.fixture`。 -`@pytest.fixture` 用來定義自訂 fixture。本次核對的版本沒有這種自訂定義，因此不將它列為已使用功能。一般 helper 函式需要由程式呼叫，不會因為名稱有 mock 就自動注入測試。 +`@pytest.fixture` 用來定義自訂 fixture。目前 executor、reporter 等測試使用此 decorator 準備輸入與預期資料。一般 helper 函式需要由程式呼叫，不會因為名稱有 mock 就自動注入測試。 `tests/fixtures/` 則是專案放置輔助腳本的資料夾。v1.6.1 的 `process_tree.py` 與 `retry_process.py` 由 subprocess 執行，並不是 pytest 自動注入的 fixture。詳細流程見 [v1.6.1](#v1-6-1)。 @@ -119,6 +122,7 @@ python -m pytest -k timeout -v | 版本 | 閱讀重點 | 測試覆蓋與證據 | | --- | --- | --- | +| [v1.6.2](#v1-6-2) | Watchdog、設定邊界、JSON attempt history、orphan process | [Test matrix](test_matrix/test_matrix_v1.6.2.md) | | [v1.0.0](#v1-0-0) | tmp_path、參數化與基本斷言 | [Test matrix](test_matrix/test_matrix_v1.0.0.md) | | [v1.1.0](#v1-1-0) | 模型更名後的案例與參數化 | [Test matrix](test_matrix/test_matrix_v1.1.0.md) | | [v1.2.0](#v1-2-0) | monkeypatch.setattr、Mock、TimeoutExpired | [Test matrix](test_matrix/test_matrix_v1.2.0.md) | @@ -359,3 +363,117 @@ python -m pytest -k timeout -v | `tests/test_unit/test_runner.py::test_retry_waits_between_attempts` | 模擬時鐘與 `pytest.approx` | 自訂 marker：`cancelled`, `test_lifecycle`, `artifact`, `retry`。 + + + +## v1.6.2 — Run-level Timeout + +基準為 `eea8507` 加工作目錄修正，對照 v1.6.1 tag。本節不重述舊版測試結果。 + +- `test_config_loader.py` 使用 tmp_path 寫入省略 timeout 的 YAML，走完整 loader；參數化 NaN、±Infinity、零、負數與 bool 則直接測試 timeout parser。parser 測試不能證明所有 YAML 型別轉換。 +- `test_run_timeout.py` 使用真實 thread、短 deadline 與最長兩秒輪詢，在 finally stop watchdog；不斷言精確排程時間。Stop 測試確認 join 完成與 token 未取消，未另等原期限。 +- `test_run_status.py` 以 tuple 參數列驗證狀態優先順序；set 無序，不適合表示 reason／expected 的配對。 +- `test_run_timeout_between_steps_stops_next_step` 使用專案自訂 fake executor 注入 RUN_TIMEOUT。它驗證 stage routing，不證明真實多步累計時間。 +- `long_running.py` 與 `orphan_process.py` 是 subprocess 輔助腳本，不是 pytest fixtures。Fixture 路徑從巢狀測試檔的 `parents[2]` 找到專案根目錄。 +- Runner integration tests 讀取 `result.json`，比對取消原因、狀態、attempt 數量與 flags；retry delay 情境保留 PROCESS_ERROR。這比僅檢查記憶體結果多驗證了序列化邊界，但仍未涵蓋 CLI exit code。 +- Orphan test 先等待 direct child 結束，確認 descendant 仍活著，再呼叫 terminator；finally 防禦性清理測試程序。這只驗證同一 process group。 + + + +### 組合 shell 命令：shlex.quote 與 repr + +`test_integration_runner_timeout.py` 的 cleanup command 會經過兩層解析：先由 POSIX shell 拆解命令，再由 `python -c` 執行 Python 程式碼。因此，shell 參數與 Python 字串常值需要分別處理。 + +| 寫法 | 輸出用途 | 本次測試中的位置 | +| --- | --- | --- | +| `str(path)` | 將 Path 轉為路徑文字；不加引號或跳脫 | 傳給 quote 或 repr 前 | +| `shlex.quote(text)` | 將文字表示成一個 POSIX shell 參數 | Python executable、fixture 路徑、完整 `-c` 程式碼 | +| `repr(text)` | 將字串表示成可放入 Python 原始碼的字串常值，含引號與必要跳脫 | `Path(...)` 的參數 | + +這些是 Python 標準函式庫／內建函式，不是 pytest 工具。`sys.executable` 指向目前執行測試的 Python，可讓 subprocess 使用同一個虛擬環境。 + +#### 1. shlex.quote：保留一個 shell 參數 + +```python +import shlex + +fixture_path = "/tmp/test project/long_running.py" +quoted = shlex.quote(fixture_path) + +assert quoted == "'/tmp/test project/long_running.py'" +assert shlex.split(quoted) == [fixture_path] +``` + +路徑包含空白時，直接拼接會被 shell 拆成多個參數。`quote()` 讓它保留為一個參數，也處理單引號、`$` 等 shell 特殊字元。它不一定每次都加引號：沒有特殊字元的字串可能原樣返回。 + +測試中的用法是分別引用每個參數，再組成命令： + +```python +python = shlex.quote(sys.executable) +fixture = shlex.quote(str(fixture_script)) +command = f"{python} {fixture}" +``` + +此片段接續測試中的 import 與 `fixture_script` 定義。不要對整條 `python fixture.py` 命令只呼叫一次 quote，否則 shell 會將整條文字視為單一參數。 + +#### 2. repr：產生 Python 字串常值 + +```python +import ast + +path_text = "/tmp/test project/Andrew's cleanup.txt" +marker_path = repr(path_text) + +assert ast.literal_eval(marker_path) == path_text +``` + +`repr()` 在這裡接收的是字串，因此結果可以嵌入 Python 程式碼。例如 `f"Path({marker_path})"` 會產生帶有正確引號的 `Path(...)` 呼叫。直接用 `str(marker)` 插入會缺少字串引號；手動包單引號又可能被路徑中的單引號截斷。 + +`repr()` 不是 shell quoting，不能代替 `shlex.quote()`。一般物件的 repr 也不保證是可執行的 Python 表達式；此技巧限定於本例的 `repr(str(marker))`。`f"{str(marker)!r}"` 是同樣的字串 repr 寫法。 + +#### 3. 組合兩層引用：完整可執行範例 + +以下示範與 cleanup 測試相同的組法，使用含空白與單引號的暫存路徑。範例會建立 marker、檢查內容，離開區塊後移除暫存目錄；它是教學範例，不是新增的 pytest 案例。 + +```python +import shlex +import subprocess +import sys +from pathlib import Path +from tempfile import TemporaryDirectory + +with TemporaryDirectory(prefix="runner example ") as directory: + marker = Path(directory) / "Andrew's cleanup.txt" + + python = shlex.quote(sys.executable) + marker_path = repr(str(marker)) + + code = ( + "from pathlib import Path; " + f"Path({marker_path}).write_text(" + "'cleanup completed', encoding='utf-8')" + ) + cleanup_command = f"{python} -c " + shlex.quote(code) + + subprocess.run(cleanup_command, shell=True, check=True) + + assert marker.read_text(encoding="utf-8") == "cleanup completed" +``` + +處理順序是：`repr(str(marker))` 先保護 Python 程式碼中的路徑常值，再用 `shlex.quote(code)` 將完整程式碼保留成 `-c` 的單一參數。Shell 移除外層引用後，Python 才解析內層字串常值。 + +本專案 executor 使用 `shell=True`，因此需要 shell quoting。若直接使用 subprocess 的參數清單且不經 shell，則可寫成 `subprocess.run([sys.executable, "-c", code], check=True)`，不要再對清單元素呼叫 `shlex.quote()`；`code` 內嵌路徑仍需要 repr。`shlex.quote()` 適用於 POSIX shell，不應視為 Windows cmd.exe 或 PowerShell 的 quoting 方法。 + +現有 integration tests 驗證 cleanup marker 與報告結果，沒有專門參數化所有特殊字元路徑；本節範例也不代表該覆蓋已加入測試矩陣。 + +### 執行與定位案例 + +以下是可重跑的命令，實際觀察結果另見 [完成條件](definition_of_done/definition_of_done_v1.6.2.md)。 + +```bash +.venv/bin/python -m pytest -q +.venv/bin/python -m pytest tests/test_unit/test_run_timeout.py tests/test_unit/test_run_status.py -q +.venv/bin/python -m pytest tests/test_integration/test_integration_runner_timeout.py -q +``` + +AI 協作新增與修正案例、原始缺口及驗證限制集中於 [v1.6.2 matrix](test_matrix/test_matrix_v1.6.2.md)。本次文件補充以 AST 移除 docstring 後比較，確認可執行測試行為不變。 diff --git a/docs/test_matrix/test_matrix_v1.6.2.md b/docs/test_matrix/test_matrix_v1.6.2.md new file mode 100644 index 0000000..e8bc02c --- /dev/null +++ b/docs/test_matrix/test_matrix_v1.6.2.md @@ -0,0 +1,58 @@ +# Device Test Runner v1.6.2 Test Matrix + +Version scope: Run-level Timeout. Evidence is the working tree based on `eea8507`, compared with tag `v1.6.1`; release publication is unverified. + +## Requirement coverage + +Unit paths are under `tests/test_unit/`; integration paths under `tests/test_integration/`. A test in the unit directory can still launch a real process. + +| Area | Scenario | Level | Test | Assertion / boundary | +| --- | --- | --- | --- | --- | +| Configuration | Missing timeout → unlimited | Unit / YAML | `test_config_loader.py::test_missing_run_timeout_defaults_to_unlimited` | None；不啟動 watchdog 的 runner 行為由程式核對 | +| Configuration | Zero / negative / non-finite rejected | Unit / parser | `test_config_loader.py::test_run_timeout_must_be_finite_and_positive` | 0、負數、NaN、±Infinity 拋 ValueError | +| Configuration | Boolean rejected | Unit / parser | `test_config_loader.py::test_run_timeout_rejects_boolean` | 拒絕 bool | +| Cancellation | First reason preserved | Unit | `test_cancellation.py::test_first_cancellation_reason_wins` | USER_REQUEST 不被 RUN_TIMEOUT 覆寫 | +| Cancellation | RUN_TIMEOUT recorded | Unit | `test_cancellation.py::test_run_timeout_reason_cannot_be_overwritten_by_user_cancel` | 反方向也保留第一個原因 | +| Watchdog | Deadline cancellation | Unit / real thread | `test_run_timeout.py::test_watchdog_requests_cancellation_on_timeout` | 限時輪詢確認 RUN_TIMEOUT | +| Watchdog | Stop prevents cancellation | Unit / real thread | `test_run_timeout.py::test_watchdog_stop_prevents_timeout` | stop 返回後 token 未取消且 thread 已停止；未另外等待超過原期限 | +| Watchdog | User reason survives deadline | Unit / real thread | `test_run_timeout.py::test_watchdog_does_not_overwrite_existing_cancellation_reason` | deadline 後仍為 USER_REQUEST | +| Executor | Interrupt long-running process | Integration | `test_integration_run_timeout_executor.py::test_run_timeout_cancels_running_executor` | CANCELLED、timed_out=false、保留 stdout，耗時小於 10 秒 | +| Executor | Step timeout remains TIMEOUT | Unit / Mock | `test_executor.py::test_subprocess_executor_raised_timeout_error` | TIMEOUT、timed_out=true、cancelled=false | +| Runner / Lifecycle | No retry; skip remaining steps; cleanup | Integration | `test_integration_runner_timeout.py::test_run_timout_stops_retry_and_runs_cleanup` | 一次 attempt、後續 step 未執行、cleanup marker 存在 | +| Runner / Reporting | Timeout during retry delay | Integration | `test_integration_runner_timeout.py::test_run_timeout_during_retry_delay` | 只留一筆 PROCESS_ERROR；step cancelled、run TIMED_OUT | +| Lifecycle | Timeout during setup | Integration | `test_integration_runner_timeout.py::test_run_timeout_during_setup_runs_cleanup` | 不跑 scenario，teardown 與 global_teardown 執行 | +| Runner | Timeout between steps | Unit / fake executor | `test_runner.py::test_run_timeout_between_steps_stops_next_step` | 第一步成功仍保存，下一步不執行；未使用真實 watchdog | +| Reporting | Status precedence | Unit / parameterized | `test_run_status.py::test_run_status_matrix` | 無取消原因時，failed／cancelled／skipped／artifact counts 的狀態；原因優先順序另由同檔測試驗證 | +| Process | Reaped child with live descendant | Unit directory / real process | `test_process.py::test_terminator_cleans_group_even_when_direct_child_has_exited` | 原 process group 被清理 | + +The two runner timeout/retry integration tests also read `result.json`: status, cancellation reason, configured timeout, one attempt, original failure type and flags are asserted. They also assert `metadata.run_timed_out` is true; the reporter unit test asserts its default is false. General reporter tests compare serialized dataclasses, but do not replace run-timeout-specific assertions. + +## AI 協作案例與修正 + +以下來源依本次對話紀錄確認；其他案例不推測作者。 + +| 原本缺口／失敗 | 修改 | 防止回歸的測試 | 限制 | +| --- | --- | --- | --- | +| 省略 timeout 被數字型別檢查拒絕 | loader 遇 None 直接回傳 | `test_missing_run_timeout_defaults_to_unlimited` | 未單獨驗證 null YAML | +| 非有限值缺少測試證據 | 加入 NaN、±Infinity、零與負數參數化 | `test_run_timeout_must_be_finite_and_positive` | 直接 RunnerConfig 不經 loader | +| 報告只檢查記憶體結果 | 兩個 runner integration tests 讀取 JSON 與 attempt flags | timeout during command／retry delay tests above | 尚未驗證 CLI exit code | +| `Event.eait` 拼字使 thread 崩潰 | 改用 Event.wait | `test_watchdog_requests_cancellation_on_timeout` | 排程時間不是精確即時保證 | +| 不存在 artifact 被分類 INVALID | exists 改為 MISSING | `test_exists_rule_fails_when_file_missing`、`test_validate_all_returns_all_results` | 空目錄仍是 INVALID | +| 大小訊息包含縮排空白 | 相鄰字串拼接 | `test_file_size_rule_fails_below_minimum`／`test_file_size_rule_fails_above_maximum` | 僅修正診斷格式 | +| 空目錄測試誤認 MISSING | 改為 INVALID 預期 | `test_directory_not_empty_fails_when_empty` | 路徑存在但內容不合規 | +| fixture root 少上一層、import／屬性／步驟名稱錯誤 | 修正測試準備與斷言 | setup timeout、orphan、executor 與 runner timeout tests | 測試錯誤與產品問題分開處理 | +| direct child 結束即跳過 group cleanup | 探測原 group 並清理 descendants | orphan test above；`test_terminated_process_does_not_need_cleanup` | 不涵蓋 detached child | +| status 參數用無序 set | 改用 tuple | `test_run_status_from_cancellation_reason` | 避免資料順序不穩定 | + +本次文件更新只補測試 docstring，不修改可執行行為。寫法見 [測試指南 v1.6.2](../test_guide.md#v1-6-2)，實際命令與結果見 [完成條件](../definition_of_done/definition_of_done_v1.6.2.md)。 + +## 尚未覆蓋 + +- 多個真實短 steps 累計觸發 deadline；目前 between-steps 用 fake executor 注入原因。 +- deadline 正好與 stop、process completion、user cancellation 同時發生的競態。 +- cleanup 或 final validation 期間首次觸發 timeout、例外後 report finalization。 +- CLI TIMED_OUT exit code、Linux CI、detached descendants 與所有正常退出後殘留背景程序的路徑。 + +## 命令組合的閱讀入口 + +Timeout cleanup 測試以 `shlex.quote()` 保留 shell 參數，以 `repr(str(marker))` 產生內嵌 Python 路徑常值。兩層解析、逐步範例與不經 shell 的替代寫法見 [測試指南：命令引用](../test_guide.md#v1-6-2-command-quoting)。現有測試未專門參數化含特殊字元的路徑；教學範例不計入測試案例數。 diff --git a/pyproject.toml b/pyproject.toml index 8a71c89..114fee5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "devicetestrunner" -version = "1.6.0" +version = "1.6.2" description = "" authors = [ {name = "Andrew Wang",email = "andew771027@gmail.com"} @@ -20,3 +20,6 @@ packages = [{ include = "runner" }] [build-system] requires = ["poetry-core>=2.0.0,<3.0.0"] build-backend = "poetry.core.masonry.api" + +[tool.pytest.ini_options] +testpaths = ["tests"] diff --git a/pytest.ini b/pytest.ini index 74e7456..f5caca5 100644 --- a/pytest.ini +++ b/pytest.ini @@ -1,4 +1,5 @@ [pytest] +testpaths = ["tests"] markers = test_lifecycle artifact diff --git a/runner/artifact_validator.py b/runner/artifact_validator.py index a8feabe..9a2e12b 100644 --- a/runner/artifact_validator.py +++ b/runner/artifact_validator.py @@ -123,7 +123,10 @@ def _validate_file_size(rule: ArtifactValidationRule, path: Path) -> ArtifactVal passed=False, required=rule.required, failure_type=FailureType.ARTIFACT_INVALID, - message=f"File size {actual_size} bytes is smaller than the minimum required size of {rule.min_size_bytes} bytes.", + message=( + f"File size {actual_size} bytes is smaller than the minimum " + f"required size of {rule.min_size_bytes} bytes." + ), actual_size_bytes=actual_size, ) @@ -135,7 +138,10 @@ def _validate_file_size(rule: ArtifactValidationRule, path: Path) -> ArtifactVal passed=False, required=rule.required, failure_type=FailureType.ARTIFACT_INVALID, - message=f"File size {actual_size} bytes exceeds the maximum allowed size of {rule.max_size_bytes} bytes.", + message=( + f"File size {actual_size} bytes exceeds the maximum allowed " + f"size of {rule.max_size_bytes} bytes." + ), actual_size_bytes=actual_size, ) @@ -253,7 +259,7 @@ def _validate_directory_not_empty( path=str(path), passed=has_contents, required=rule.required, - failure_type=(FailureType.NONE if has_contents else FailureType.ARTIFACT_MISSING), + failure_type=(FailureType.NONE if has_contents else FailureType.ARTIFACT_INVALID), message=message, ) diff --git a/runner/cancellation.py b/runner/cancellation.py index 5fb5335..f6a04f0 100644 --- a/runner/cancellation.py +++ b/runner/cancellation.py @@ -1,17 +1,46 @@ import threading +from enum import Enum + + +class CancellationReason(str, Enum): + USER_REQUEST = "user_request" + RUN_TIMEOUT = "run_timeout" class CancellationToken: def __init__(self) -> None: + self._lock = threading.Lock() self._event = threading.Event() + self._reason: CancellationReason | None = None + + def cancel(self, reason: CancellationReason = CancellationReason.USER_REQUEST) -> None: + """ + Request cancellation. + + Returns: + True: this call established cancellation. + False: cancellation was already requested. - def cancel(self) -> None: - self._event.set() + The first cancellation reason wins. + """ + with self._lock: + if self._event.is_set(): + return False + + self._reason = reason + self._event.set() + + return True @property def is_cancelled(self) -> bool: return self._event.is_set() + @property + def reason(self) -> CancellationReason | None: + with self._lock: + return self._reason + def raise_if_cancelled(self) -> None: if self.is_cancelled: raise CancellationRequested() diff --git a/runner/config.py b/runner/config.py index fe449ef..93baa5e 100644 --- a/runner/config.py +++ b/runner/config.py @@ -1,3 +1,4 @@ +import math from typing import Any import yaml @@ -47,9 +48,12 @@ def load(self, path: str) -> RunnerConfig: retry = self._load_retry(raw) + run_timeout_seconds = self._load_run_timeout_seconds(raw) + return RunnerConfig( test_case=test_case, device=device, + run_timeout_seconds=run_timeout_seconds, retry=retry, lifecycle=lifecycle, artifact=artifact, @@ -155,9 +159,26 @@ def _parse_retry_on(values: list[str]) -> list[FailureType]: if failure_type == FailureType.CANCELLED: raise ValueError("retry.retry_on cannot contain 'cancelled'") - if failure_type not in retry_on: retry_on.append(failure_type) return retry_on + + @staticmethod + def _load_run_timeout_seconds(raw: dict[str, Any]) -> float | None: + + raw_run_timeout = raw.get("run_timeout_seconds") + + if raw_run_timeout is None: + return None + + if isinstance(raw_run_timeout, bool) or not isinstance(raw_run_timeout, (int, float)): + raise ValueError("run_timeout_seconds must be a number") + + run_timeout_seconds = float(raw_run_timeout) + + if not math.isfinite(run_timeout_seconds) or run_timeout_seconds <= 0: + raise ValueError("run_timeout_seconds must be finite and > 0") + + return run_timeout_seconds diff --git a/runner/executor.py b/runner/executor.py index d7cd2d2..5c66edc 100644 --- a/runner/executor.py +++ b/runner/executor.py @@ -37,9 +37,7 @@ def execute( ) -> StepAttemptResult: if log_writer is None: - log_writer = self._create_default_log_writer( - stage=stage, step_name=step.name - ) + log_writer = self._create_default_log_writer(stage=stage, step_name=step.name) environment = os.environ.copy() @@ -141,6 +139,11 @@ def execute( break + return_code = process.poll() + + if return_code is not None: + break + # # 3. timeout # diff --git a/runner/models.py b/runner/models.py index cf5cd34..85ec81d 100644 --- a/runner/models.py +++ b/runner/models.py @@ -121,6 +121,7 @@ class RunnerConfig: lifecycle: LifecycleConfig retry: RetryConfig artifact: ArtifactConfig + run_timeout_seconds: float | None = None @dataclass(frozen=True) @@ -164,10 +165,6 @@ class StepAttemptResult: # v1.5.1 artifact_validation_results: list[ArtifactValidationResult] = field(default_factory=list) - @property - def passed(self) -> bool: - return self.exit_code == 0 - @dataclass(frozen=True) class StepResult: @@ -193,6 +190,9 @@ class RunMetadata: started_at: str finished_at: str cancel_requested: bool + cancel_reason: str | None = None + run_timeout_seconds: float | None = None + run_timed_out: bool = False @dataclass(frozen=True) @@ -224,4 +224,4 @@ class RunResult: @property def passed(self) -> bool: - return all(step_result.success for step_result in self.step_results) + return self.summary.status == "PASSED" diff --git a/runner/process.py b/runner/process.py index c918009..9596fb0 100644 --- a/runner/process.py +++ b/runner/process.py @@ -26,26 +26,24 @@ def __init__(self, grace_period_seconds: float = 2.0, poll_interval_seconds: flo def terminate_process_group(self, process: subprocess.Popen) -> ProcessTerminationResult: - # - # Process 已經結束 - # - if process.poll() is not None: - return ProcessTerminationResult( - terminated=False, - killed=False, - return_code=process.returncode, - ) + process.poll() try: - process_group_id = os.getpgid(process.pid) except ProcessLookupError: - return ProcessTerminationResult( - terminated=False, - killed=False, - return_code=process.poll(), - ) + # Executor 使用 start_new_session=True，group ID 就是 direct child PID。 + # Direct child 已被 reap 時，仍需檢查留在原群組中的 descendants。 + process_group_id = process.pid + + try: + os.killpg(process_group_id, 0) + except ProcessLookupError: + return ProcessTerminationResult( + terminated=False, + killed=False, + return_code=process.poll(), + ) # # -------------------------------------------------- diff --git a/runner/retry.py b/runner/retry.py index 4468b4d..94d133a 100644 --- a/runner/retry.py +++ b/runner/retry.py @@ -1,4 +1,4 @@ -from runner.models import ArtifactValidationResult, FailureType, RetryConfig +from runner.models import FailureType, RetryConfig class RetryPolicy: diff --git a/runner/run_status.py b/runner/run_status.py new file mode 100644 index 0000000..3d25e62 --- /dev/null +++ b/runner/run_status.py @@ -0,0 +1,31 @@ +from runner.cancellation import CancellationReason + + +def calculate_run_status( + *, + cancellation_reason: CancellationReason | None, + failed_steps: int, + cancelled_steps: int, + skipped_steps: int, + failed_required_artifact_rules: int, +) -> str: + + if cancellation_reason == CancellationReason.RUN_TIMEOUT: + return "TIMED_OUT" + + if cancellation_reason == CancellationReason.USER_REQUEST: + return "CANCELLED" + + if cancelled_steps > 0: + return "CANCELLED" + + if failed_steps > 0: + return "FAILED" + + if skipped_steps > 0: + return "FAILED" + + if failed_required_artifact_rules > 0: + return "FAILED" + + return "PASSED" diff --git a/runner/run_timeout.py b/runner/run_timeout.py new file mode 100644 index 0000000..29d6c17 --- /dev/null +++ b/runner/run_timeout.py @@ -0,0 +1,72 @@ +import threading +import time + +from runner.cancellation import CancellationReason, CancellationToken + + +class RunTimeoutWatchdog: + def __init__(self, timeout_seconds: float, cancellation_token: CancellationToken) -> None: + + if timeout_seconds <= 0: + raise ValueError("run_timeout_seconds must be > 0") + + self.timeout_seconds = timeout_seconds + self.cancellation_token = cancellation_token + + self._stop_event = threading.Event() + self._thread: threading.Thread | None = None + self._started_at: float | None = None + self._deadline: float | None = None + + @property + def deadline(self) -> float | None: + return self._deadline + + @property + def started_at(self) -> float | None: + return self._started_at + + @property + def is_running(self) -> bool: + return self._thread is not None and self._thread.is_alive() + + def start(self) -> None: + if self._thread is not None: + raise RuntimeError("RunTimeoutWatchdog can only start once") + + self._started_at = time.monotonic() + + self._deadline = self._started_at + self.timeout_seconds + + self._thread = threading.Thread( + target=self._watch, name="run-timeout-watchdog", daemon=True + ) + + self._thread.start() + + def _watch(self) -> None: + assert self._deadline is not None + + while True: + + remaining = self._deadline - time.monotonic() + + if remaining <= 0: + + if not self._stop_event.is_set(): + + self.cancellation_token.cancel(CancellationReason.RUN_TIMEOUT) + + return + + stopped = self._stop_event.wait(timeout=remaining) + + if stopped: + return + + def stop(self) -> None: + self._stop_event.set() + + if self._thread is not None: + + self._thread.join() diff --git a/runner/runner.py b/runner/runner.py index d69c563..e2d4708 100644 --- a/runner/runner.py +++ b/runner/runner.py @@ -5,7 +5,7 @@ from runner.artifact import ArtifactManager from runner.artifact_validator import ArtifactValidator -from runner.cancellation import CancellationToken +from runner.cancellation import CancellationReason, CancellationToken from runner.executor import SubprocessExecutor from runner.failure import FailureClassifier from runner.models import ( @@ -23,10 +23,12 @@ ) from runner.reporter import JsonReporter from runner.retry import RetryPolicy +from runner.run_status import calculate_run_status +from runner.run_timeout import RunTimeoutWatchdog class DeviceTestRunner: - VERSION = "1.6.1" + VERSION = "1.6.2" def __init__( self, @@ -66,37 +68,40 @@ def run( setup_success = False - if not cancellation_token.is_cancelled: + watchdog: RunTimeoutWatchdog | None = None - global_setup_success = self._run_stage( - stage="global_setup", - steps=config.lifecycle.global_setup.steps, - config=config, - run_dir=run_dir, - step_results=step_results, - stop_on_failure=True, + if config.run_timeout_seconds is not None: + + watchdog = RunTimeoutWatchdog( + timeout_seconds=config.run_timeout_seconds, cancellation_token=cancellation_token, - ignore_cancellation=False, ) - if global_setup_success and not cancellation_token.is_cancelled: + watchdog.start() - setup_success = self._run_stage( - stage="setup", - steps=config.lifecycle.setup.steps, - config=config, - run_dir=run_dir, - step_results=step_results, - stop_on_failure=True, - cancellation_token=cancellation_token, - ignore_cancellation=False, - ) + try: - if setup_success and not cancellation_token.is_cancelled: + # + # NORMAL LIFECYCLE + # + if not cancellation_token.is_cancelled: - self._run_stage( - stage="scenario", - steps=config.lifecycle.scenario.steps, + global_setup_success = self._run_stage( + stage="global_setup", + steps=config.lifecycle.global_setup.steps, + config=config, + run_dir=run_dir, + step_results=step_results, + stop_on_failure=True, + cancellation_token=cancellation_token, + ignore_cancellation=False, + ) + + if global_setup_success and not cancellation_token.is_cancelled: + + setup_success = self._run_stage( + stage="setup", + steps=config.lifecycle.setup.steps, config=config, run_dir=run_dir, step_results=step_results, @@ -105,8 +110,21 @@ def run( ignore_cancellation=False, ) + if setup_success and not cancellation_token.is_cancelled: + + self._run_stage( + stage="scenario", + steps=config.lifecycle.scenario.steps, + config=config, + run_dir=run_dir, + step_results=step_results, + stop_on_failure=True, + cancellation_token=cancellation_token, + ignore_cancellation=False, + ) + # - # teardown 是 cleanup。 + # CLEANUP LIFECYCLE # # 即使 cancellation token 已經是 cancelled， # teardown 還是要執行。 @@ -124,24 +142,31 @@ def run( ignore_cancellation=True, ) - # - # global teardown 永遠 best effort。 - # - self._run_stage( - stage="global_teardown", - steps=config.lifecycle.global_teardown.steps, - config=config, - run_dir=run_dir, - step_results=step_results, - stop_on_failure=False, - cancellation_token=cancellation_token, - ignore_cancellation=True, - ) + # + # global teardown 永遠 best effort。 + # + self._run_stage( + stage="global_teardown", + steps=config.lifecycle.global_teardown.steps, + config=config, + run_dir=run_dir, + step_results=step_results, + stop_on_failure=False, + cancellation_token=cancellation_token, + ignore_cancellation=True, + ) - # 最終 Run-level Artifact Validation - artifact_results = self.artifact_validator.validate_all( - rules=config.artifact.validation.rules, base_dir=run_dir - ) + # + # FINAL ARTIFACT VALIDATION + # + artifact_results = self.artifact_validator.validate_all( + rules=config.artifact.validation.rules, base_dir=run_dir + ) + + finally: + + if watchdog is not None: + watchdog.stop() finished_at = datetime.now(timezone.utc) @@ -155,7 +180,7 @@ def run( started_at=started_at, finished_at=finished_at, duration_seconds=duration_deconds, - cancel_requested=cancellation_token.is_cancelled, + cancellation_token=cancellation_token, ) self.reporter.save(result=run_result, output_dir=str(run_dir)) @@ -531,7 +556,7 @@ def _build_run_result( started_at: datetime, finished_at: datetime, duration_seconds: float, - cancel_requested: bool, + cancellation_token: CancellationToken, ) -> RunResult: configured_steps = self._count_configured_steps(config) @@ -550,18 +575,16 @@ def _build_run_result( configured_artifact_rules = len(artifact_results) - passed_artifact_rules = sum(result.passed for result in artifact_results) + passed_artifact_rules = sum(1 for result in artifact_results if result.passed) - failed_artifact_rules = sum( - not result.passed for result in artifact_results if not result.passed - ) + failed_artifact_rules = sum(1 for result in artifact_results if not result.passed) failed_required_artifact_rules = sum( 1 for result in artifact_results if (result.required and not result.passed) ) - status = self._calculate_status( - cancel_requested=cancel_requested, + status = calculate_run_status( + cancellation_reason=cancellation_token.reason, failed_steps=failed_steps, cancelled_steps=cancelled_steps, skipped_steps=skipped_steps, @@ -578,7 +601,12 @@ def _build_run_result( runner_version=self.VERSION, started_at=started_at.isoformat(), finished_at=finished_at.isoformat(), - cancel_requested=cancel_requested, + cancel_requested=cancellation_token.is_cancelled, + cancel_reason=( + cancellation_token.reason.value if cancellation_token.reason is not None else None + ), + run_timeout_seconds=config.run_timeout_seconds, + run_timed_out=(cancellation_token.reason == CancellationReason.RUN_TIMEOUT), ) summary = ExecutionSummary( @@ -621,29 +649,6 @@ def _count_configured_steps(config: RunnerConfig) -> int: ) ) - @staticmethod - def _calculate_status( - cancel_requested: bool, - failed_steps: int, - cancelled_steps: int, - skipped_steps: int, - failed_required_artifact_rules: int, - ) -> str: - - if cancel_requested or cancelled_steps > 0: - return "CANCELLED" - - if failed_steps > 0: - return "FAILED" - - if skipped_steps > 0: - return "FAILED" - - if failed_required_artifact_rules > 0: - return "FAILED" - - return "PASSED" - @staticmethod def _get_rules_for_step( step_name: str, diff --git a/scripts/artifact_retry.sh b/scripts/artifact_retry.sh index 8ad06b7..0e0622b 100644 --- a/scripts/artifact_retry.sh +++ b/scripts/artifact_retry.sh @@ -38,4 +38,4 @@ echo "Generating valid artifact" printf \ "timestamp,power\n1,100\n2,120\n" \ > "$OUTPUT_FILE" -exit 0 \ No newline at end of file +exit 0 diff --git a/tests/fixtures/long_running.py b/tests/fixtures/long_running.py new file mode 100644 index 0000000..19d5454 --- /dev/null +++ b/tests/fixtures/long_running.py @@ -0,0 +1,18 @@ +import os +import time + + +def main() -> None: + print(f"started pid={os.getpid()}", flush=True) + + while True: + print( + "still running", + flush=True, + ) + + time.sleep(0.1) + + +if __name__ == "__main__": + main() diff --git a/tests/fixtures/orphan_process.py b/tests/fixtures/orphan_process.py new file mode 100644 index 0000000..0ed7063 --- /dev/null +++ b/tests/fixtures/orphan_process.py @@ -0,0 +1,45 @@ +import subprocess +import sys +from pathlib import Path + + +def main() -> int: + if len(sys.argv) != 2: + print( + "Usage: orphan_process.py ", + file=sys.stderr, + ) + return 1 + + child_pid_file = Path(sys.argv[1]) + + child = subprocess.Popen( + [ + sys.executable, + "-c", + ("import time; " "time.sleep(30)"), + ] + ) + + child_pid_file.write_text( + str(child.pid), + encoding="utf-8", + ) + + print( + f"child pid={child.pid}", + flush=True, + ) + + # + # Important: + # parent exits immediately. + # + # Child remains in the same process group + # because it does NOT call setsid(). + # + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/test_integration/test_integration.py b/tests/test_integration/test_integration.py index 70e6a99..8f79f1c 100644 --- a/tests/test_integration/test_integration.py +++ b/tests/test_integration/test_integration.py @@ -18,7 +18,8 @@ def test_yaml_to_result_json(tmp_path): Given a complete YAML-defined device-test workflow is available. When the workflow is executed through the real runner boundary. - Then the end-to-end run produces a result JSON matching the YAML workflow and observed command outputs. + Then the end-to-end run produces a result JSON matching the YAML workflow and observed + command outputs. """ config_file = tmp_path / "intergration.yaml" config_file.write_text( @@ -32,7 +33,7 @@ def test_yaml_to_result_json(tmp_path): serial: xxx_003 product: product_003 build: test_001 - +run_timeout_seconds: 300 lifecycle: global_setup: steps: @@ -148,7 +149,7 @@ def test_yaml_to_result_json(tmp_path): report = json.loads((run_dir / "result.json").read_text(encoding="utf-8")) - assert report["metadata"]["runner_version"] == "1.6.1" + assert report["metadata"]["runner_version"] == "1.6.2" assert report["summary"]["status"] == "PASSED" @@ -175,7 +176,7 @@ def test_integration_stops_after_failed_step(tmp_path): serial: xxx_003 product: product_003 build: test_003 - +run_timeout_seconds: 300 lifecycle: global_setup: steps: @@ -293,7 +294,7 @@ def test_integration_stops_after_failed_step(tmp_path): report = json.loads((run_dir / "result.json").read_text(encoding="utf-8")) - assert report["metadata"]["runner_version"] == "1.6.1" + assert report["metadata"]["runner_version"] == "1.6.2" assert report["summary"]["status"] == "FAILED" @@ -305,7 +306,8 @@ def test_integration_steps_succeeds_after_party(tmp_path: Path): Given a complete YAML-defined device-test workflow is available. When the workflow is executed through the real runner boundary. - Then a transient subprocess failure is retried and the later successful attempt completes the workflow. + Then a transient subprocess failure is retried and the later successful attempt completes + the workflow. """ output_dir = tmp_path / "artifacts" config_file = tmp_path / "config.yaml" @@ -320,6 +322,7 @@ def test_integration_steps_succeeds_after_party(tmp_path: Path): serial: fake_serial product: fake_product build: fake_build + run_timeout_seconds: 300 retry: max_attempts: 3 delay_seconds: 1 @@ -409,6 +412,7 @@ def test_real_subprocess_fails_after_retry_exhausted(tmp_path: Path): serial: fake_serial product: fake_product build: fake_build + run_timeout_seconds: 300 retry: max_attempts: 3 delay_seconds: 1 diff --git a/tests/test_integration/test_integration_artifact_validation.py b/tests/test_integration/test_integration_artifact_validation.py index 28c9a67..8e99ca6 100644 --- a/tests/test_integration/test_integration_artifact_validation.py +++ b/tests/test_integration/test_integration_artifact_validation.py @@ -33,6 +33,7 @@ def test_command_creates_and_validates_artifact(tmp_path: Path): serial: fake_device product: fake_product build: fake_build + run_timeout_seconds: 300 lifecycle: global_setup: steps: [] @@ -135,6 +136,7 @@ def test_run_fails_when_command_does_not_create_artifact(tmp_path: Path): serial: fake_device product: fake_product build: fake_build + run_timeout_seconds: 300 lifecycle: global_setup: steps: [] @@ -210,6 +212,7 @@ def test_command_runs_inside_run_directory(tmp_path: Path): serial: fake_device product: fake_product build: fake_build + run_timeout_seconds: 300 lifecycle: global_setup: steps: [] @@ -279,6 +282,7 @@ def test_csv_and_json_content_validation(tmp_path: Path): serial: fake device product: fake_pixel build: fake_build + run_timeout_seconds: 300 lifecycle: global_setup: steps: [] @@ -297,8 +301,9 @@ def test_csv_and_json_content_validation(tmp_path: Path): timeout_second: 5 - name: create_json type: command - command: | - printf '{{"status":"PASSED","metrics":{{"average_power":110.0,"sample_count":2}}}}' > results/test_json_file.json + command: > + printf '{{"status":"PASSED","metrics":{{"average_power":110.0,"sample_count":2}}}}' + > results/test_json_file.json timeout_second: 5 teardown: steps: [] @@ -361,7 +366,7 @@ def test_csv_and_json_content_validation(tmp_path: Path): (Path(result.artifact_dir) / "result.json").read_text(encoding="utf-8") ) - assert saved_report["metadata"]["runner_version"] == "1.6.1" + assert saved_report["metadata"]["runner_version"] == "1.6.2" assert saved_report["summary"]["status"] == "PASSED" @@ -385,6 +390,7 @@ def test_run_fails_when_csv_content_invalid(tmp_path: Path): serial: fake_device product: fake_pixel build: fake_build + run_timeout_seconds: 300 lifecycle: global_setup: steps: [] @@ -469,6 +475,7 @@ def test_run_fails_when_json_status_invalid(tmp_path: Path): serial: fake_device product: fake_pixel build: fake_build + run_timeout_seconds: 300 lifecycle: global_setup: steps: [] diff --git a/tests/test_integration/test_integration_cancellation.py b/tests/test_integration/test_integration_cancellation.py index a083b3b..2973960 100644 --- a/tests/test_integration/test_integration_cancellation.py +++ b/tests/test_integration/test_integration_cancellation.py @@ -83,7 +83,6 @@ def cancel_soon(): def test_stdout_reader_thread_finishes_after_process_termination(tmp_path: Path): - """Acceptance scenario. Given a real command prints start and waits. @@ -143,7 +142,6 @@ def cancel_soon(): def test_stderr_is_drained_after_cancellation(tmp_path: Path): - """Acceptance scenario. Given a real command prints an error to stderr and waits. @@ -203,7 +201,6 @@ def cancel_soon(): def test_process_group_termination_cleans_child_processes(tmp_path: Path): - """Acceptance scenario. Given a shell has started a child whose PID is recorded. diff --git a/tests/test_integration/test_integration_retry.py b/tests/test_integration/test_integration_retry.py index 53e936b..f67fc03 100644 --- a/tests/test_integration/test_integration_retry.py +++ b/tests/test_integration/test_integration_retry.py @@ -48,7 +48,8 @@ def test_real_artifact_aware_retry(tmp_path: Path): Given a real command initially produces an unacceptable artifact and retry is enabled. When the runner validates the artifact and performs the configured retry flow. - Then the invalid first artifact triggers one retry and the corrected artifact allows the run to pass. + Then the invalid first artifact triggers one retry and the corrected artifact allows the run + to pass. """ output_dir = tmp_path / "artifacts" @@ -64,6 +65,7 @@ def test_real_artifact_aware_retry(tmp_path: Path): serial: fake_serial, product: fake_pixel, build: fake_build, + run_timeout_seconds: 300 retry: max_attempts: 3 delay_seconds: 0 diff --git a/tests/test_integration/test_integration_run_timeout_executor.py b/tests/test_integration/test_integration_run_timeout_executor.py new file mode 100644 index 0000000..8b3365f --- /dev/null +++ b/tests/test_integration/test_integration_run_timeout_executor.py @@ -0,0 +1,90 @@ +import sys +import time +from pathlib import Path + +from runner.artifact import ArtifactManager +from runner.cancellation import CancellationReason, CancellationToken +from runner.executor import SubprocessExecutor +from runner.failure import FailureClassifier +from runner.models import FailureType, LifecycleStepContent +from runner.process import ProcessTerminator +from runner.run_timeout import RunTimeoutWatchdog + + +def test_run_timeout_cancels_running_executor(tmp_path: Path): + """Acceptance scenario. + + Given a real long-running process has a longer step timeout than the run deadline. + When the watchdog cancels the executor token. + Then the process stops with CANCELLED, not step TIMEOUT, and preserves startup output. + """ + project_root = Path(__file__).resolve().parents[2] + + fixture_script = project_root / "tests" / "fixtures" / "long_running.py" + + assert fixture_script.exists() + + token = CancellationToken() + + classifier = FailureClassifier() + + executor = SubprocessExecutor( + project_directory=project_root, + failure_classifier=classifier, + process_terminator=ProcessTerminator(grace_period_seconds=0.2), + ) + + artifact_manager = ArtifactManager(tmp_path) + + run_dir = tmp_path / "run" + + run_dir.mkdir() + + log_writer = artifact_manager.create_step_log_writer( + run_dir=run_dir, + stage="scenario", + step_name="long_running", + attempt=1, + show_console=False, + ) + + step = LifecycleStepContent( + name="long_running", + type="command", + command=(f'"{sys.executable}" {fixture_script}'), + timeout_second=30, + ) + + watchdog = RunTimeoutWatchdog( + timeout_seconds=0.3, + cancellation_token=token, + ) + + started = time.monotonic() + + try: + + watchdog.start() + + with log_writer: + result = executor.execute( + step=step, + stage="scenario", + attempt=1, + log_writer=log_writer, + working_directory=run_dir, + cancellation_token=token, + ) + finally: + watchdog.stop() + + elapsed = time.monotonic() - started + + assert token.is_cancelled is True + assert token.reason == CancellationReason.RUN_TIMEOUT + assert result.success is False + assert result.cancelled is True + assert result.timed_out is False + assert result.failure_type == FailureType.CANCELLED + assert "started pid=" in result.stdout + assert elapsed < 10 diff --git a/tests/test_integration/test_integration_runner_timeout.py b/tests/test_integration/test_integration_runner_timeout.py new file mode 100644 index 0000000..2b9cc7e --- /dev/null +++ b/tests/test_integration/test_integration_runner_timeout.py @@ -0,0 +1,483 @@ +import json +import shlex +import sys +from pathlib import Path + +from runner.artifact import ArtifactManager +from runner.artifact_validator import ArtifactValidator +from runner.cancellation import CancellationReason, CancellationToken +from runner.executor import SubprocessExecutor +from runner.failure import FailureClassifier +from runner.models import ( + ArtifactConfig, + DeviceInfo, + DeviceTestCase, + FailureType, + LifecycleConfig, + LifecycleStepContent, + LifecycleSteps, + RetryConfig, + RunnerConfig, +) +from runner.process import ProcessTerminator +from runner.reporter import JsonReporter +from runner.runner import DeviceTestRunner + + +def test_run_timout_stops_retry_and_runs_cleanup(tmp_path: Path): + """Acceptance scenario. + + Given a long-running scenario has retries and a cleanup step. + When the run deadline cancels the running process. + Then cleanup runs and the report preserves one cancelled attempt with TIMED_OUT status. + """ + + project_root = Path(__file__).resolve().parents[2] + + fixture_script = project_root / "tests" / "fixtures" / "long_running.py" + + assert fixture_script.exists() + + marker = tmp_path / "cleanup.txt" + + python = shlex.quote(sys.executable) + + fixture = shlex.quote(str(fixture_script)) + + marker_path = repr(str(marker)) + + cleanup_command = f"{python} -c " + shlex.quote( + "from pathlib import Path; " + f"Path({marker_path}).write_text(" + "'cleanup completed', " + "encoding='utf-8')" + ) + + config = RunnerConfig( + test_case=DeviceTestCase( + id="run_timeout_001", + name="run_timeout_001", + description="Run-level timeout test", + ), + device=DeviceInfo( + serial="device_001", + product="pixel", + build="build_001", + ), + run_timeout_seconds=0.3, + retry=RetryConfig( + max_attempts=3, + delay_seconds=0.1, + retry_on=[FailureType.TIMEOUT, FailureType.DEVICE_OFFLINE], + ), + lifecycle=LifecycleConfig( + global_setup=LifecycleSteps(steps=[]), + setup=LifecycleSteps(steps=[]), + scenario=LifecycleSteps( + steps=[ + LifecycleStepContent( + name="long_running", + type="command", + command=f"{python} {fixture}", + timeout_second=30, + ), + LifecycleStepContent( + name="should_not_run", + type="command", + command="echo unexpected", + timeout_second=10, + ), + ] + ), + teardown=LifecycleSteps(steps=[]), + global_teardown=LifecycleSteps( + steps=[ + LifecycleStepContent( + name="cleanup", + type="command", + command=cleanup_command, + timeout_second=10, + ) + ] + ), + ), + artifact=ArtifactConfig(output_dir=str(tmp_path)), + ) + + classifier = FailureClassifier() + + runner = DeviceTestRunner( + executor=SubprocessExecutor( + project_directory=project_root, + failure_classifier=classifier, + process_terminator=ProcessTerminator(grace_period_seconds=0.2), + ), + artifact_manager=ArtifactManager(tmp_path), + artifact_validator=ArtifactValidator(), + failure_classifier=classifier, + reporter=JsonReporter(), + show_console_output=False, + ) + + token = CancellationToken() + + result = runner.run(config=config, cancellation_token=token) + + scenario_result = next( + step_result for step_result in result.step_results if step_result.name == "long_running" + ) + + assert scenario_result.attempts == 1 + assert scenario_result.cancelled is True + assert scenario_result.attempt_results[0].failure_type == FailureType.CANCELLED + + assert result.summary.status == "TIMED_OUT" + + assert token.is_cancelled is True + + assert marker.exists() + + assert marker.read_text(encoding="utf-8") == "cleanup completed" + + assert not any( + step_result.name == "should_not_run" and step_result.attempts > 0 + for step_result in result.step_results + ) + + report_path = Path(result.artifact_dir) / "result.json" + saved = json.loads(report_path.read_text(encoding="utf-8")) + + assert saved["summary"]["status"] == "TIMED_OUT" + assert saved["metadata"]["cancel_requested"] is True + assert saved["metadata"]["cancel_reason"] == "run_timeout" + assert saved["metadata"]["run_timed_out"] is True + assert result.metadata.run_timed_out is True + assert saved["metadata"]["run_timeout_seconds"] == config.run_timeout_seconds + + saved_scenario = next( + step_result + for step_result in saved["step_results"] + if step_result["name"] == "long_running" + ) + + assert saved_scenario["attempts"] == 1 + assert saved_scenario["cancelled"] is True + assert len(saved_scenario["attempt_results"]) == 1 + + saved_attempt = saved_scenario["attempt_results"][0] + + assert saved_attempt["attempt"] == 1 + assert saved_attempt["success"] is False + assert saved_attempt["cancelled"] is True + assert saved_attempt["timed_out"] is False + assert saved_attempt["failure_type"] == "cancelled" + + +def test_run_timeout_during_retry_delay(tmp_path: Path): + """Acceptance scenario. + + Given a failed command is waiting to retry. + When the run deadline expires during the retry delay. + Then the report preserves the original failed attempt without adding a cancelled attempt. + """ + + project_root = Path(__file__).resolve().parents[2] + + fixture_script = project_root / "tests" / "fixtures" / "long_running.py" + + assert fixture_script.exists() + + marker = tmp_path / "cleanup.txt" + + python = shlex.quote(sys.executable) + + marker_path = repr(str(marker)) + + cleanup_command = f"{python} -c " + shlex.quote( + "from pathlib import Path; " + f"Path({marker_path}).write_text(" + "'cleanup completed', " + "encoding='utf-8')" + ) + + config = RunnerConfig( + test_case=DeviceTestCase( + id="run_timeout_001", + name="run_timeout_001", + description="Run-level timeout test", + ), + device=DeviceInfo( + serial="device_001", + product="pixel", + build="build_001", + ), + run_timeout_seconds=0.3, + retry=RetryConfig( + max_attempts=3, + delay_seconds=2, + retry_on=[FailureType.PROCESS_ERROR], + ), + lifecycle=LifecycleConfig( + global_setup=LifecycleSteps(steps=[]), + setup=LifecycleSteps(steps=[]), + scenario=LifecycleSteps( + steps=[ + LifecycleStepContent( + name="exit_command", + type="command", + command="exit 1", + timeout_second=30, + ), + ] + ), + teardown=LifecycleSteps(steps=[]), + global_teardown=LifecycleSteps( + steps=[ + LifecycleStepContent( + name="cleanup", + type="command", + command=cleanup_command, + timeout_second=10, + ) + ] + ), + ), + artifact=ArtifactConfig(output_dir=str(tmp_path)), + ) + + classifier = FailureClassifier() + + runner = DeviceTestRunner( + executor=SubprocessExecutor( + project_directory=project_root, + failure_classifier=classifier, + process_terminator=ProcessTerminator(grace_period_seconds=0.2), + ), + artifact_manager=ArtifactManager(tmp_path), + artifact_validator=ArtifactValidator(), + failure_classifier=classifier, + reporter=JsonReporter(), + show_console_output=False, + ) + + result = runner.run(config=config, cancellation_token=CancellationToken()) + + scenario_result = next( + step_result for step_result in result.step_results if step_result.name == "exit_command" + ) + + assert scenario_result.attempts == 1 + assert scenario_result.attempt_results[0].failure_type == FailureType.PROCESS_ERROR + assert scenario_result.cancelled is True + + assert result.summary.status == "TIMED_OUT" + + assert marker.exists() + + report_path = Path(result.artifact_dir) / "result.json" + saved = json.loads(report_path.read_text(encoding="utf-8")) + + assert saved["summary"]["status"] == "TIMED_OUT" + assert saved["metadata"]["cancel_requested"] is True + assert saved["metadata"]["cancel_reason"] == "run_timeout" + assert saved["metadata"]["run_timed_out"] is True + assert result.metadata.run_timed_out is True + assert saved["metadata"]["run_timeout_seconds"] == config.run_timeout_seconds + + saved_scenario = next( + step_result + for step_result in saved["step_results"] + if step_result["name"] == "exit_command" + ) + + assert saved_scenario["attempts"] == 1 + assert saved_scenario["cancelled"] is True + assert len(saved_scenario["attempt_results"]) == 1 + + saved_attempt = saved_scenario["attempt_results"][0] + + assert saved_attempt["attempt"] == 1 + assert saved_attempt["success"] is False + assert saved_attempt["cancelled"] is False + assert saved_attempt["timed_out"] is False + assert saved_attempt["failure_type"] == "process_error" + + +def test_run_timeout_during_setup_runs_cleanup( + tmp_path: Path, +): + """Acceptance scenario. + + Given setup runs a long command after successful global setup. + When the run deadline interrupts setup. + Then scenario is skipped and both teardown markers are written with TIMED_OUT status. + """ + project_root = Path(__file__).resolve().parents[2] + + fixture_script = project_root / "tests" / "fixtures" / "long_running.py" + + assert fixture_script.exists() + + python = shlex.quote(sys.executable) + + fixture = shlex.quote(str(fixture_script)) + + teardown_marker = tmp_path / "teardown.txt" + + global_teardown_marker = tmp_path / "global_teardown.txt" + + scenario_marker = tmp_path / "scenario.txt" + + def marker_command( + path: Path, + content: str, + ) -> str: + return f"{python} -c " + shlex.quote( + "from pathlib import Path; " + f"Path({str(path)!r}).write_text(" + f"{content!r}, " + "encoding='utf-8')" + ) + + config = RunnerConfig( + test_case=DeviceTestCase( + id="run_timeout_setup", + name="run_timeout_setup", + description=("Run timeout during setup"), + ), + device=DeviceInfo( + serial="device_001", + product="pixel", + build="build_001", + ), + run_timeout_seconds=0.3, + retry=RetryConfig( + max_attempts=1, + delay_seconds=0, + retry_on=[], + ), + lifecycle=LifecycleConfig( + global_setup=LifecycleSteps( + steps=[ + LifecycleStepContent( + name="global_setup", + type="command", + command="true", + timeout_second=10, + ) + ] + ), + setup=LifecycleSteps( + steps=[ + LifecycleStepContent( + name="long_setup", + type="command", + command=(f"{python} {fixture}"), + timeout_second=30, + ) + ] + ), + scenario=LifecycleSteps( + steps=[ + LifecycleStepContent( + name="scenario", + type="command", + command=marker_command( + scenario_marker, + "scenario ran", + ), + timeout_second=10, + ) + ] + ), + teardown=LifecycleSteps( + steps=[ + LifecycleStepContent( + name="teardown", + type="command", + command=marker_command( + teardown_marker, + "teardown completed", + ), + timeout_second=10, + ) + ] + ), + global_teardown=LifecycleSteps( + steps=[ + LifecycleStepContent( + name="global_teardown", + type="command", + command=marker_command( + global_teardown_marker, + "global teardown completed", + ), + timeout_second=10, + ) + ] + ), + ), + artifact=ArtifactConfig( + output_dir=str(tmp_path), + ), + ) + + classifier = FailureClassifier() + + runner = DeviceTestRunner( + executor=SubprocessExecutor( + project_directory=project_root, + failure_classifier=classifier, + process_terminator=( + ProcessTerminator( + grace_period_seconds=0.2, + ) + ), + ), + artifact_manager=ArtifactManager(tmp_path), + artifact_validator=ArtifactValidator(), + failure_classifier=classifier, + reporter=JsonReporter(), + show_console_output=False, + ) + + token = CancellationToken() + + result = runner.run( + config=config, + cancellation_token=token, + ) + + assert token.is_cancelled is True + + assert token.reason == CancellationReason.RUN_TIMEOUT + + assert result.summary.status == "TIMED_OUT" + + # + # Setup was actually interrupted. + # + setup_result = next( + step_result for step_result in result.step_results if step_result.name == "long_setup" + ) + + assert setup_result.attempts == 1 + assert setup_result.cancelled is True + + assert setup_result.attempt_results[0].failure_type == FailureType.CANCELLED + + # + # Scenario must never start. + # + assert scenario_marker.exists() is False + + # + # Cleanup must still happen. + # + assert teardown_marker.exists() is True + + assert teardown_marker.read_text(encoding="utf-8") == "teardown completed" + + assert global_teardown_marker.exists() is True + + assert global_teardown_marker.read_text(encoding="utf-8") == "global teardown completed" diff --git a/tests/test_unit/test_artifact_validator.py b/tests/test_unit/test_artifact_validator.py index 9d8976b..0594835 100644 --- a/tests/test_unit/test_artifact_validator.py +++ b/tests/test_unit/test_artifact_validator.py @@ -102,9 +102,9 @@ def test_file_size_rule_fails_below_minimum(tmp_path: Path): assert result.type == "file_size" assert result.path == str(target) assert result.actual_size_bytes == 4 - assert ( - result.message - == f"File size {result.actual_size_bytes} bytes is smaller than the minimum required size of {rule.min_size_bytes} bytes." + assert result.message == ( + f"File size {result.actual_size_bytes} bytes is smaller than the minimum " + f"required size of {rule.min_size_bytes} bytes." ) assert result.failure_type == FailureType.ARTIFACT_INVALID @@ -134,9 +134,9 @@ def test_file_size_rule_fails_above_maximum(tmp_path: Path): assert result.type == "file_size" assert result.path == str(target) assert result.actual_size_bytes == 30 - assert ( - result.message - == f"File size {result.actual_size_bytes} bytes exceeds the maximum allowed size of {rule.max_size_bytes} bytes." + assert result.message == ( + f"File size {result.actual_size_bytes} bytes exceeds the maximum allowed " + f"size of {rule.max_size_bytes} bytes." ) assert result.failure_type == FailureType.ARTIFACT_INVALID @@ -239,9 +239,9 @@ def test_file_extension_rule_fails(tmp_path: Path): assert result.name == "test_file.pdf" assert result.type == "file_extension" assert result.path == str(target) - assert ( - result.message - == f"File extension '{target.suffix}' is not allowed entensions {sorted(rule.allowed_extensions)}." + assert result.message == ( + f"File extension '{target.suffix}' is not allowed entensions " + f"{sorted(rule.allowed_extensions)}." ) assert result.failure_type == FailureType.ARTIFACT_INVALID @@ -300,7 +300,7 @@ def test_directory_not_empty_fails_when_empty(tmp_path: Path): Given a directory contains no entries. When the artifact validator evaluates the rule. - Then the result is ARTIFACT_MISSING with an empty-directory diagnostic. + Then the result is ARTIFACT_INVALID with an empty-directory diagnostic. """ directory = tmp_path / "test_directory" directory.mkdir() @@ -316,7 +316,7 @@ def test_directory_not_empty_fails_when_empty(tmp_path: Path): assert result.type == "directory_not_empty" assert result.path == str(directory) assert result.message == "Directory is empty." - assert result.failure_type == FailureType.ARTIFACT_MISSING + assert result.failure_type == FailureType.ARTIFACT_INVALID def test_directory_not_empty_fails_for_file(tmp_path: Path): @@ -739,9 +739,8 @@ def test_json_content_fails_when_value_mismatch(tmp_path: Path): result = ArtifactValidator().validate(rule=rule, base_dir=tmp_path) assert result.passed is False - assert ( - result.message - == "JSON value validation failed: [\"status: expected 'PASSED', actual 'FAILED'\"]" + assert result.message == ( + "JSON value validation failed: [\"status: expected 'PASSED', actual 'FAILED'\"]" ) assert result.failure_type == FailureType.ARTIFACT_INVALID diff --git a/tests/test_unit/test_cancellation.py b/tests/test_unit/test_cancellation.py index 26014ea..3450d2c 100644 --- a/tests/test_unit/test_cancellation.py +++ b/tests/test_unit/test_cancellation.py @@ -1,18 +1,19 @@ import pytest -from runner.cancellation import CancellationRequested, CancellationToken +from runner.cancellation import CancellationReason, CancellationRequested, CancellationToken -def test_token_not_cancelled_initially(): +def test_token_initial_state(): """Acceptance scenario. Given a newly constructed cancellation token. When its state is read. - Then is_cancelled is false. + Then is_cancelled is false and reason is None. """ token = CancellationToken() assert token.is_cancelled is False + assert token.reason is None def test_token_can_be_cancelled(): @@ -69,3 +70,73 @@ def test_raise_if_cancelled_raises_after_cancel(): with pytest.raises(CancellationRequested): token.raise_if_cancelled() + + +def test_deafult_cancellation_reason(): + """Acceptance scenario. + + Given a new token has no cancellation reason. + When cancel is called without an explicit reason. + Then the token records USER_REQUEST. + """ + token = CancellationToken() + + accepted = token.cancel() + + assert accepted is True + + assert token.reason == CancellationReason.USER_REQUEST + + +def test_run_timeout_cancellation_reason(): + """Acceptance scenario. + + Given a new token is active. + When cancel is called with RUN_TIMEOUT. + Then cancellation is accepted and its reason is RUN_TIMEOUT. + """ + token = CancellationToken() + + accepted = token.cancel(CancellationReason.RUN_TIMEOUT) + + assert accepted is True + + assert token.reason == CancellationReason.RUN_TIMEOUT + + +def test_first_cancellation_reason_wins(): + """Acceptance scenario. + + Given USER_REQUEST is the first cancellation. + When RUN_TIMEOUT is requested afterward. + Then only the first request is accepted and USER_REQUEST is preserved. + """ + token = CancellationToken() + + first = token.cancel(CancellationReason.USER_REQUEST) + + second = token.cancel(CancellationReason.RUN_TIMEOUT) + + assert first is True + + assert second is False + + assert token.reason == CancellationReason.USER_REQUEST + + +def test_run_timeout_reason_cannot_be_overwritten_by_user_cancel(): + """Acceptance scenario. + + Given a token first receives RUN_TIMEOUT. + When USER_REQUEST is submitted afterward. + Then the second cancellation is rejected and RUN_TIMEOUT remains recorded. + """ + token = CancellationToken() + + first = token.cancel(CancellationReason.RUN_TIMEOUT) + + second = token.cancel(CancellationReason.USER_REQUEST) + + assert first is True + assert second is False + assert token.reason == CancellationReason.RUN_TIMEOUT diff --git a/tests/test_unit/test_config_loader.py b/tests/test_unit/test_config_loader.py index c036d78..7d173d9 100644 --- a/tests/test_unit/test_config_loader.py +++ b/tests/test_unit/test_config_loader.py @@ -33,7 +33,7 @@ def test_config_loader_loads_device_test_config(tmp_path: Path): serial: xxx001 product: pixel build: 2026.xx.001 - + run_timeout_seconds: 300 lifecycle: global_setup: steps: @@ -90,6 +90,8 @@ def test_config_loader_loads_device_test_config(tmp_path: Path): assert config.device.product == "pixel" assert config.device.build == "2026.xx.001" + assert config.run_timeout_seconds == 300 + assert isinstance(config.lifecycle, LifecycleConfig) assert len(config.lifecycle.global_setup.steps) == 1 @@ -149,7 +151,7 @@ def test_load_artifact_validation_config(tmp_path: Path): serial: xxx001 product: pixel build: 2026.xx.001 - + run_timeout_seconds: 300 lifecycle: global_setup: steps: [] @@ -222,7 +224,7 @@ def test_artifact_validation_is_optional(tmp_path: Path): serial: xxx001 product: pixel build: 2026.xx.001 - + run_timeout_seconds: 300 lifecycle: global_setup: steps: @@ -288,6 +290,7 @@ def test_artifact_retry_defaults_to_false(tmp_path: Path): serial: fake_serial product: fake_pixel build: fake_build + run_timeout_seconds: 300 retry: max_attempts: 3 delay_seconds: 1 @@ -347,6 +350,7 @@ def test_load_csv_and_json_validation_rules(tmp_path: Path): serial: fake product: pixel build: build_001 + run_timeout_seconds: 300 lifecycle: global_setup: steps: [] @@ -361,8 +365,9 @@ def test_load_csv_and_json_validation_rules(tmp_path: Path): timeout_second: 5 - name: create_json type: command - command: | - printf '{{"status":"PASSED","metrics":{{"average_power":110.0,"sample_count":2}}}}' > test_json_file.json + command: > + printf '{{"status":"PASSED","metrics":{{"average_power":110.0,"sample_count":2}}}}' + > test_json_file.json timeout_second: 5 teardown: steps: [] @@ -426,6 +431,7 @@ def test_load_retry_config(tmp_path: Path): serial: fake_serial product: fake_pixel build: fake_build + run_timeout_seconds: 300 retry: max_attempts: 3 delay_seconds: 3 @@ -471,6 +477,7 @@ def test_load_artifact_aware_retry_rule(tmp_path: Path): serial: fake_serial product: fake_pixel build: fake_build + run_timeout_seconds: 300 retry: max_attempts: 3 delay_seconds: 1 @@ -533,6 +540,7 @@ def test_retry_config_uses_default_values(tmp_path: Path): serial: fake_serial product: fake_pixel build: fake_build + run_timeout_seconds: 300 lifecycle: global_setup: steps: [] @@ -574,6 +582,7 @@ def test_retry_max_attempts_must_be_positive(tmp_path: Path): serial: fake_serial product: fake_pixel build: fake_build + run_timeout_seconds: 300 retry: max_attempts: -10 lifecycle: @@ -615,6 +624,7 @@ def test_retry_delay_seconds_must_be_positive(tmp_path: Path): serial: fake_serial product: fake_pixel build: fake_build + run_timeout_seconds: 300 retry: delay_seconds: -10 lifecycle: @@ -785,3 +795,82 @@ def test_retry_on_cancelled_is_invalid(): with pytest.raises(ValueError, match=r"retry\.retry_on cannot contain 'cancelled'"): ConfigLoader()._load_retry(raw=raw) + + +def test_missing_run_timeout_defaults_to_unlimited(tmp_path: Path): + """Acceptance scenario. + + Given YAML omits run_timeout_seconds. + When ConfigLoader loads the file. + Then the run timeout is None, allowing unlimited execution time. + """ + config_file = tmp_path / "sample.yaml" + config_file.write_text( + """ + test_case: + id: timeout_001 + name: timeout + description: Run without a deadline. + device: + serial: device_001 + product: pixel + build: build_001 + lifecycle: + global_setup: + steps: [] + setup: + steps: [] + scenario: + steps: [] + teardown: + steps: [] + global_teardown: + steps: [] + artifact: + output_dir: results + """, + encoding="utf-8", + ) + + config = ConfigLoader().load(config_file) + + assert config.run_timeout_seconds is None + + +@pytest.mark.parametrize( + "timeout_seconds", + [0, -1, -0.1, float("nan"), float("inf"), float("-inf")], +) +def test_run_timeout_must_be_finite_and_positive(timeout_seconds): + """Acceptance scenario. + + Given the run timeout is zero, negative or non-finite. + When the run timeout configuration is loaded. + Then ValueError rejects the invalid timeout. + """ + raw = {"run_timeout_seconds": timeout_seconds} + + with pytest.raises(ValueError, match="run_timeout_seconds must be finite and > 0"): + ConfigLoader._load_run_timeout_seconds(raw) + + +@pytest.mark.parametrize( + "timeout_seconds", + [ + True, + False, + ], +) +def test_run_timeout_rejects_boolean( + timeout_seconds, +): + """Acceptance scenario. + + Given run_timeout_seconds is a boolean. + When the configuration timeout is parsed. + Then ValueError rejects a boolean as a numeric timeout. + """ + raw = {"run_timeout_seconds": (timeout_seconds)} + + with pytest.raises(ValueError, match=("run_timeout_seconds " "must be a number")): + ConfigLoader._load_run_timeout_seconds(raw) diff --git a/tests/test_unit/test_executor.py b/tests/test_unit/test_executor.py index a299c2d..1ceced1 100644 --- a/tests/test_unit/test_executor.py +++ b/tests/test_unit/test_executor.py @@ -74,7 +74,6 @@ def test_subprocess_executor_return_success(tmp_path, test_case_id, step, stage, assert result.attempt == attempt assert result.exit_code == 0 - assert result.passed is True assert result.success is True assert result.cancelled is False assert result.timed_out is False @@ -142,7 +141,6 @@ def test_subprocess_executor_failure(tmp_path, test_case_id, step, stage, attemp assert result.attempt == attempt assert result.exit_code == 1 assert result.success is False - assert result.passed is False assert log_writer.stdout_path.read_text(encoding="utf-8") == result.stdout assert result.stdout == "" @@ -336,7 +334,6 @@ def terminate_process_group(process): assert result.attempt == 1 assert result.success is False assert result.exit_code == -signal.SIGTERM - assert result.passed is False assert log_writer.stdout_path.read_text(encoding="utf-8") == result.stdout assert result.stdout == "" diff --git a/tests/test_unit/test_process.py b/tests/test_unit/test_process.py index 7832013..f58962a 100644 --- a/tests/test_unit/test_process.py +++ b/tests/test_unit/test_process.py @@ -1,17 +1,32 @@ +import os import signal import subprocess import sys +import time +from pathlib import Path from runner.process import ProcessTerminator -def test_terminated_process_does_not_need_cleanup(monkeypatch): +def is_process_alive(pid: int) -> bool: + try: + os.kill(pid, 0) + + except ProcessLookupError: + return False + + except PermissionError: + return True + return True + + +def test_terminated_process_does_not_need_cleanup(monkeypatch): """Acceptance scenario. - Given the direct process has already exited with code zero. + Given the direct process has exited with code zero and its group is gone. When the terminator is called. - Then no group signal is sent and the result reports no termination or kill. + Then only a group probe is sent and the result reports no termination or kill. """ process = subprocess.Popen( ["true"], @@ -24,6 +39,8 @@ def test_terminated_process_does_not_need_cleanup(monkeypatch): def fake_killpg(process_group_id, signal_number): signal_calls.append((process_group_id, signal_number)) + assert signal_number == 0 + raise ProcessLookupError() monkeypatch.setattr("runner.process.os.killpg", fake_killpg) @@ -34,11 +51,10 @@ def fake_killpg(process_group_id, signal_number): assert result.terminated is False assert result.killed is False assert result.return_code == 0 - assert signal_calls == [] + assert signal_calls == [(process.pid, 0)] def test_process_group_terminates_gracefully(): - """Acceptance scenario. Given a real process runs in a separate session. @@ -64,7 +80,6 @@ def test_process_group_terminates_gracefully(): def test_process_is_killed_when_sigterm_is_ignored(): - """Acceptance scenario. Given a real process confirms its SIGTERM ignore handler is ready. @@ -118,6 +133,7 @@ def test_group_probe_permission_error_does_not_abort_cleanup(monkeypatch): When the terminator sends SIGTERM and polls the group. Then the probe is retried and cleanup completes without SIGKILL. """ + class FakeProcess: def __init__(self): self.pid = 123 @@ -166,3 +182,83 @@ def fake_killpg(process_group_id, signal_number): (123, 0), (123, 0), ] + + +def test_terminator_cleans_group_even_when_direct_child_has_exited( + tmp_path: Path, +): + """Acceptance scenario. + + Given a reaped direct child leaves a live descendant in its process group. + When the terminator cleans the original group. + Then the descendant exits and the result records termination. + """ + project_root = Path(__file__).resolve().parents[2] + + fixture_script = project_root / "tests" / "fixtures" / "orphan_process.py" + + assert fixture_script.exists() + + child_pid_file = tmp_path / "child.pid" + + process = subprocess.Popen( + [ + sys.executable, + str(fixture_script), + str(child_pid_file), + ], + start_new_session=True, + ) + + # + # Wait for direct parent to exit. + # + process.wait(timeout=5) + + assert process.poll() is not None + + # + # Fixture must have created the child. + # + assert child_pid_file.exists() + + child_pid = int(child_pid_file.read_text(encoding="utf-8")) + + # + # Critical precondition: + # + # direct Popen process is dead, + # but descendant is still alive. + # + assert is_process_alive(child_pid) is True + + terminator = ProcessTerminator( + grace_period_seconds=0.2, + poll_interval_seconds=0.05, + ) + + try: + result = terminator.terminate_process_group(process) + + deadline = time.monotonic() + 2 + + while is_process_alive(child_pid) and time.monotonic() < deadline: + time.sleep(0.05) + + assert is_process_alive(child_pid) is False + + assert result.terminated is True + + finally: + # + # Defensive cleanup in case the + # implementation under test is broken. + # + if is_process_alive(child_pid): + try: + os.kill( + child_pid, + 9, + ) + except ProcessLookupError: + pass diff --git a/tests/test_unit/test_reporter.py b/tests/test_unit/test_reporter.py index 11403ca..70713c5 100644 --- a/tests/test_unit/test_reporter.py +++ b/tests/test_unit/test_reporter.py @@ -156,7 +156,7 @@ def test_save_result_json( Given a completed run contains metadata, summaries, steps, and artifact results. When the reporter serializes the run result. - Then the generated JSON preserves metadata, summary counts, step results, and artifact validation results. + Then JSON preserves all result fields and defaults run_timed_out to false. """ run_result = RunResult( @@ -175,6 +175,7 @@ def test_save_result_json( saved = json.loads(output_path.read_text(encoding="utf-8")) assert saved["metadata"] == asdict(metadata) + assert saved["metadata"]["run_timed_out"] is False assert saved["summary"] == asdict(summary) assert saved["step_results"] == [asdict(result) for result in step_results] assert saved["artifact_dir"] == str(tmp_path) diff --git a/tests/test_unit/test_run_status.py b/tests/test_unit/test_run_status.py new file mode 100644 index 0000000..ffc1d0e --- /dev/null +++ b/tests/test_unit/test_run_status.py @@ -0,0 +1,145 @@ +import pytest + +from runner.cancellation import CancellationReason +from runner.run_status import calculate_run_status + + +@pytest.mark.parametrize( + "reason, expected", + [ + ( + CancellationReason.RUN_TIMEOUT, + "TIMED_OUT", + ), + ( + CancellationReason.USER_REQUEST, + "CANCELLED", + ), + ( + None, + "PASSED", + ), + ], +) +def test_run_status_from_cancellation_reason(reason, expected): + """Acceptance scenario. + + Given a run has the parameterized cancellation reason and no failures. + When calculate_run_status evaluates its summary. + Then the status matches TIMED_OUT, CANCELLED or PASSED for that reason. + """ + status = calculate_run_status( + cancellation_reason=reason, + failed_steps=0, + cancelled_steps=0, + skipped_steps=0, + failed_required_artifact_rules=0, + ) + + assert status == expected + + +def test_run_timeout_takes_priority_over_step_failure(): + """Acceptance scenario. + + Given RUN_TIMEOUT accompanies failed, cancelled and skipped steps. + When the run status is calculated. + Then TIMED_OUT takes priority over the other outcomes. + """ + status = calculate_run_status( + cancellation_reason=CancellationReason.RUN_TIMEOUT, + failed_steps=1, + cancelled_steps=1, + skipped_steps=2, + failed_required_artifact_rules=0, + ) + + assert status == "TIMED_OUT" + + +def test_user_cancellation_takes_priority_over_failure(): + """Acceptance scenario. + + Given USER_REQUEST accompanies failed steps. + When the run status is calculated. + Then CANCELLED takes priority over failure. + """ + status = calculate_run_status( + cancellation_reason=CancellationReason.USER_REQUEST, + failed_steps=1, + cancelled_steps=1, + skipped_steps=2, + failed_required_artifact_rules=1, + ) + + assert status == "CANCELLED" + + +@pytest.mark.parametrize( + ( + "failed_steps", + "cancelled_steps", + "skipped_steps", + "failed_required_artifact_rules", + "expected_status", + ), + [ + ( + 0, + 0, + 0, + 0, + "PASSED", + ), + ( + 1, + 0, + 0, + 0, + "FAILED", + ), + ( + 0, + 1, + 0, + 0, + "CANCELLED", + ), + ( + 0, + 0, + 1, + 0, + "FAILED", + ), + ( + 0, + 0, + 0, + 1, + "FAILED", + ), + ], +) +def test_run_status_matrix( + failed_steps, + cancelled_steps, + skipped_steps, + failed_required_artifact_rules, + expected_status, +): + """Acceptance scenario. + + Given no cancellation reason and summary counts matching a parameterized row. + When calculate_run_status evaluates that row. + Then the result matches the expected status precedence. + """ + status = calculate_run_status( + cancellation_reason=None, + failed_steps=failed_steps, + cancelled_steps=cancelled_steps, + skipped_steps=skipped_steps, + failed_required_artifact_rules=(failed_required_artifact_rules), + ) + + assert status == expected_status diff --git a/tests/test_unit/test_run_timeout.py b/tests/test_unit/test_run_timeout.py new file mode 100644 index 0000000..3ab9350 --- /dev/null +++ b/tests/test_unit/test_run_timeout.py @@ -0,0 +1,144 @@ +import time + +import pytest + +from runner.cancellation import CancellationReason, CancellationToken +from runner.run_timeout import RunTimeoutWatchdog + + +def test_watchdog_requests_cancellation_on_timeout(): + """Acceptance scenario. + + Given an active token has a short watchdog deadline. + When the deadline expires while the test polls with a bounded wait. + Then the token records RUN_TIMEOUT cancellation. + """ + token = CancellationToken() + + watchdog = RunTimeoutWatchdog(timeout_seconds=0.1, cancellation_token=token) + + try: + + watchdog.start() + + deadline = time.monotonic() + 2 + + while not token.is_cancelled and time.monotonic() < deadline: + + time.sleep(0.01) + + assert token.is_cancelled is True + + assert token.reason == CancellationReason.RUN_TIMEOUT + + finally: + watchdog.stop() + + +def test_watchdog_stop_prevents_timeout(): + """Acceptance scenario. + + Given a watchdog has started with an active token. + When stop signals and joins the watchdog thread. + Then the token remains active and the watchdog is no longer running. + """ + token = CancellationToken() + + watchdog = RunTimeoutWatchdog(timeout_seconds=0.2, cancellation_token=token) + + watchdog.start() + + watchdog.stop() + + assert token.is_cancelled is False + + assert watchdog.is_running is False + + +def test_watchdog_cannot_start_twice(): + """Acceptance scenario. + + Given a watchdog has already started. + When start is called again. + Then RuntimeError rejects the second start and finally stops the watchdog. + """ + token = CancellationToken() + + watchdog = RunTimeoutWatchdog(timeout_seconds=1, cancellation_token=token) + + try: + + watchdog.start() + + with pytest.raises(RuntimeError): + watchdog.start() + + finally: + watchdog.stop() + + +def test_watchdog_does_not_overwrite_existing_cancellation_reason(): + """Acceptance scenario. + + Given a token already records USER_REQUEST. + When a watchdog deadline expires on that token. + Then the original USER_REQUEST reason remains unchanged. + """ + token = CancellationToken() + + token.cancel(CancellationReason.USER_REQUEST) + + watchdog = RunTimeoutWatchdog( + timeout_seconds=0.05, + cancellation_token=token, + ) + + try: + watchdog.start() + + time.sleep(0.1) + + finally: + watchdog.stop() + + assert token.is_cancelled is True + + assert token.reason == CancellationReason.USER_REQUEST + + +def test_run_timeout_reason_cannot_be_overwritten_by_user_cancel(): + """Acceptance scenario. + + Given a token first receives RUN_TIMEOUT. + When USER_REQUEST is submitted afterward. + Then the second cancellation is rejected and RUN_TIMEOUT remains recorded. + """ + token = CancellationToken() + + first = token.cancel(CancellationReason.RUN_TIMEOUT) + + second = token.cancel(CancellationReason.USER_REQUEST) + + assert first is True + assert second is False + + assert token.is_cancelled is True + + assert token.reason == CancellationReason.RUN_TIMEOUT + + +@pytest.mark.parametrize( + "timeout_seconds", + [0, -1, -0.1], +) +def test_watchdog_rejectes_invalid_timeout(timeout_seconds): + """Acceptance scenario. + + Given a timeout is zero or negative. + When a watchdog is constructed. + Then ValueError rejects the timeout. + """ + token = CancellationToken() + + with pytest.raises(ValueError): + RunTimeoutWatchdog(timeout_seconds=timeout_seconds, cancellation_token=token) diff --git a/tests/test_unit/test_runner.py b/tests/test_unit/test_runner.py index 8e2dbfe..51685f4 100644 --- a/tests/test_unit/test_runner.py +++ b/tests/test_unit/test_runner.py @@ -6,7 +6,7 @@ from runner.artifact import ArtifactManager, StepLogWriter from runner.artifact_validator import ArtifactValidator -from runner.cancellation import CancellationToken +from runner.cancellation import CancellationReason, CancellationToken from runner.failure import FailureClassifier from runner.models import ( ArtifactConfig, @@ -374,6 +374,47 @@ def execute( ) +class MockCancelAfterFirstStepExecutor: + def __init__( + self, + token: CancellationToken, + ): + self.token = token + self.executed_steps: list[str] = [] + + def execute( + self, + step, + stage, + attempt, + log_writer, + working_directory, + cancellation_token, + ): + self.executed_steps.append(step.name) + + result = StepAttemptResult( + attempt=attempt, + success=True, + failure_type=FailureType.NONE, + timed_out=False, + cancelled=False, + exit_code=0, + duration_seconds=0.01, + stdout="success", + stderr="", + stdout_log_path=str(log_writer.stdout_path), + stderr_log_path=str(log_writer.stderr_path), + error="", + artifact_validation_results=[], + ) + + if step.name == "step_1": + self.token.cancel(CancellationReason.RUN_TIMEOUT) + + return result + + class MockRecordingArtifactValidator: def __init__(self): self.call_count = 0 @@ -2321,7 +2362,7 @@ def test_cancellation_lifecycle_and_summary( report = json.loads(report_path.read_text(encoding="utf-8")) assert report["metadata"]["cancel_requested"] is True - assert report["metadata"]["runner_version"] == "1.6.1" + assert report["metadata"]["runner_version"] == "1.6.2" assert report["summary"]["status"] == "CANCELLED" assert report["summary"]["cancelled_steps"] == expected_cancelled_steps assert report["summary"]["failed_steps"] == 0 @@ -2524,3 +2565,101 @@ def test_retry_starts_only_after_previous_attempt_cleanup(tmp_path: Path): "attempt_2_process_cleanup", "attempt_2_return", ] + + +def test_run_timeout_between_steps_stops_next_step( + tmp_path: Path, +): + """Acceptance scenario. + + Given a fake executor requests RUN_TIMEOUT after its first successful step. + When the runner considers the next scenario step. + Then only the first step ran and its success is preserved while the run is TIMED_OUT. + """ + token = CancellationToken() + + executor = MockCancelAfterFirstStepExecutor(token) + + config = RunnerConfig( + test_case=DeviceTestCase( + id="between_steps", + name="between_steps", + description=("Run timeout between steps"), + ), + device=DeviceInfo( + serial="device_001", + product="pixel", + build="build_001", + ), + # + # Don't start a real watchdog. + # + # This test controls cancellation + # deterministically through Executor. + # + run_timeout_seconds=None, + retry=RetryConfig( + max_attempts=1, + delay_seconds=0, + retry_on=[], + ), + lifecycle=LifecycleConfig( + global_setup=LifecycleSteps(steps=[]), + setup=LifecycleSteps(steps=[]), + scenario=LifecycleSteps( + steps=[ + LifecycleStepContent( + name="step_1", + type="command", + command="ignored", + timeout_second=10, + ), + LifecycleStepContent( + name="step_2", + type="command", + command="ignored", + timeout_second=10, + ), + ] + ), + teardown=LifecycleSteps(steps=[]), + global_teardown=LifecycleSteps(steps=[]), + ), + artifact=ArtifactConfig( + output_dir=str(tmp_path), + ), + ) + + runner = DeviceTestRunner( + executor=executor, + artifact_manager=ArtifactManager(tmp_path), + artifact_validator=ArtifactValidator(), + failure_classifier=FailureClassifier(), + reporter=JsonReporter(), + show_console_output=False, + ) + + result = runner.run( + config=config, + cancellation_token=token, + ) + + assert executor.executed_steps == ["step_1"] + + assert token.is_cancelled is True + + assert token.reason == CancellationReason.RUN_TIMEOUT + + assert result.summary.status == "TIMED_OUT" + + step_1_result = next( + step_result for step_result in result.step_results if step_result.name == "step_1" + ) + + assert step_1_result.success is True + assert step_1_result.attempts == 1 + + assert not any( + step_result.name == "step_2" and step_result.attempts > 0 + for step_result in result.step_results + ) diff --git a/tests/test_unit/test_signals.py b/tests/test_unit/test_signals.py index bbbe85e..e316c0b 100644 --- a/tests/test_unit/test_signals.py +++ b/tests/test_unit/test_signals.py @@ -7,7 +7,6 @@ def test_first_sigint_cancels_token(): - """Acceptance scenario. Given a signal handler has an active token. @@ -25,7 +24,6 @@ def test_first_sigint_cancels_token(): def test_second_sigint_raises_keyboard_interrupt(): - """Acceptance scenario. Given a signal handler has received its first SIGINT.