diff --git a/.github/workflows/build-verify.yml b/.github/workflows/build-verify.yml index c617063..29fa93c 100644 --- a/.github/workflows/build-verify.yml +++ b/.github/workflows/build-verify.yml @@ -14,6 +14,7 @@ on: - 'settings-advisor/**' - 'go-lsp/**' - 'worktree-manager/**' + - 'test-runner/**' - 'build.sh' - 'build.ps1' - '.go-version' @@ -29,6 +30,7 @@ on: - 'settings-advisor/**' - 'go-lsp/**' - 'worktree-manager/**' + - 'test-runner/**' - 'build.sh' - 'build.ps1' - '.go-version' @@ -244,3 +246,36 @@ jobs: exit 1 fi working-directory: . + + test-runner: + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, macos-latest, windows-latest] + runs-on: ${{ matrix.os }} + defaults: + run: + working-directory: test-runner/src + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-go@v5 + with: + go-version-file: .go-version + - run: go vet ./... + - run: go test -count=1 ./... + - name: govulncheck + if: runner.os == 'Linux' + run: | + go install golang.org/x/vuln/cmd/govulncheck@latest + "$GITHUB_WORKSPACE/.github/scripts/govulncheck-gate.sh" "$GITHUB_WORKSPACE/.go-version" + - name: Deterministic rebuild + if: runner.os == 'Linux' + run: ./build.sh test-runner + working-directory: . + - name: Verify committed binaries are up to date + if: runner.os == 'Linux' + run: git diff --exit-code -- test-runner/bin/test-runner-linux-amd64 test-runner/bin/test-runner-darwin-arm64 test-runner/bin/test-runner.exe + working-directory: . + - name: MCP smoke test through the distributed dispatcher + if: runner.os == 'Linux' + run: TEST_RUNNER_BINARY="$GITHUB_WORKSPACE/test-runner/bin/test-runner" go test -count=1 -run '^TestMCPStdio$' ./... diff --git a/AGENTS.md b/AGENTS.md index 2ee9193..907ac07 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -13,6 +13,7 @@ - `retro-status/`: リポジトリ情報をレトロゲーム風に表示する Go 製 MCP サーバー - `settings-advisor/`: ワークスペースに適した agy 設定を提案する Go 製 MCP サーバー - `worktree-manager/`: Git worktree の作成・削除・一覧・prune を行う Go 製 MCP サーバー +- `test-runner/`: Go テストを実行し、失敗ログと再実行引数を返す Go 製 MCP サーバー - `agy-plugin-kit/`: プラグイン作成用の command、skill、template、および Go 製 validator Go 製プラグインは原則として `/src/` に独立した Go モジュール、`/bin/` に配布用バイナリを持ちます。validator のみ `agy-plugin-kit/validator/{src,bin}/` 配下です。 @@ -71,12 +72,13 @@ go-lsp/src retro-status/src settings-advisor/src worktree-manager/src +test-runner/src ``` Go ソースを変更したら、リポジトリルートで対象を決定論的に再ビルドします。 ```sh -./build.sh +./build.sh ``` Windows PowerShell では同じ target を `./build.ps1` に渡します。決定論ビルドが前提とする Go のパッチ版は `.go-version` に定義され、`build.sh`/`build.ps1` が `GOTOOLCHAIN` で強制します(未取得なら Go が自動ダウンロード)。素の `go build` で焼いたバイナリはローカル Go 版によって別物になるため、必ずビルドスクリプト経由で再ビルドしてください。`.go-version` を独断で書き換えないこと。引き上げ条件は CLAUDE.md「Go バージョンの引き上げ方針」に定義され、条件1(固定中のマイナー内のパッチで直る stdlib 脆弱性)は CI の govulncheck が fail で通知します。 diff --git a/LESSONS.md b/LESSONS.md index c73896b..3971bce 100644 --- a/LESSONS.md +++ b/LESSONS.md @@ -458,3 +458,8 @@ response = json.loads(proc.stdout.readline()) #### 判断 14: bot 誤指摘(`master`→`main`)の却下 - Codex GitHub bot が「既定ブランチは `main`」と P2 指摘したが、`gh repo view`=`master`・`main` ref 不在・PR #7 が master マージ済みで**前提が誤り**と検証。鵜呑みにすると実ブランチでゲートが無効化される。却下根拠を PR コメントとコミットに残した(#28)。 + +## 2026-09-19: test-runner はログの保持上限と結果イベントの集計を分離する +- 却下した案: `CombinedOutput` でテスト出力全体を保持してから解析する、またはログ上限に達した時点で出力の読み取りを止める。 +- 決め手: 回帰テストでは 64 KiB のログを40回(2.5 MiB)出力した後に成功・失敗イベントを送る。逐次解析とログ保持上限を分けることで、保持1 MiB以内・返却ログ64 KiB以内を守りながら末尾の成功1件・失敗1件を集計できた。読み取りを止めると末尾の結果を失い、子プロセスのパイプも詰まる。 +- 覆す条件: 完全なログをディスク上の成果物として保存・取得する機能が必要になった場合。応答サイズ制限と結果集計は引き続き独立させる。 diff --git a/PLAN.md b/PLAN.md index 02c0cb5..a7e433f 100644 --- a/PLAN.md +++ b/PLAN.md @@ -209,3 +209,10 @@ settings-advisor のスキャン精度(CI/CD、言語、除外ディレクト - [x] ステップ1: スキャン検知機能と除外ロジックの改善・テスト追加 - [x] ステップ2: models.json 更新と traits 駆動モデル選定ロジックの実装・テスト追加 - [x] ステップ3: ドキュメント同期(README.md / README_ja.md)とビルド・品質検証 + +## フェーズ13: Go テストランナー (test-runner) の追加 + +現在地: 実装・Linuxローカル検証完了。macOS/Windowsのネイティブ実行はCI確認待ち。次フェーズには進まない。 + +- [x] MCP実行・結果集計・失敗テストの再実行引数が実装され、境界とキャンセルを検証済み。 +- [x] 日英ドキュメント・skill・3環境の配布物・CIが揃い、ローカル検証が完了。 diff --git a/README.md b/README.md index 0331cc3..62e9648 100644 --- a/README.md +++ b/README.md @@ -14,6 +14,7 @@ A collection of Model Context Protocol (MCP) plugins designed for AI assistants, | **[retro-status](./retro-status/README.md)** | MCP server scanning repositories to render RPG-style retro status ASCII art | | **[settings-advisor](./settings-advisor/README.md)** | MCP server analyzing workspace scale, languages, and settings to recommend optimal models and sandbox configurations | | **[worktree-manager](./worktree-manager/README.md)** | MCP server providing safe git worktree operations (list, add, remove, prune) for isolated parallel workflows | +| **[test-runner](./test-runner/README.md)** | MCP server running Go tests with bounded failure logs and targeted rerun arguments | ## Installation @@ -41,6 +42,9 @@ agy plugin install https://github.com/kwrkb/agy-plugins/settings-advisor # worktree-manager plugin agy plugin install https://github.com/kwrkb/agy-plugins/worktree-manager + +# test-runner plugin +agy plugin install https://github.com/kwrkb/agy-plugins/test-runner ``` Please refer to the README in each plugin directory for specific prerequisites (required CLI binaries on PATH / authentication). @@ -61,12 +65,13 @@ Each plugin includes an agent guide in `skills//SKILL.md` (loaded upon too | retro-status | `git` (recommended), `rg` (optional) | None | | settings-advisor | None (prebuilt binaries included) | None | | worktree-manager | `git` (on PATH) | None | +| test-runner | `go` >= 1.24 (on PATH) | None | ### Bundled Platforms & Self-Build -✅ **Windows native execution verified across all plugins** +Existing plugins have been verified on native Windows. `test-runner` includes CI tests for Linux, macOS, and Windows; local verification is on Linux. -Go-based plugins (`github`, `ast-grep`, `go-lsp`, `retro-status`, `settings-advisor`, `worktree-manager`, and `agy-plugin-kit` validator) include native binaries for **`linux/amd64`**, **`darwin/arm64` (Apple Silicon)**, and **`windows/amd64`** in `bin/`. The extensionless `bin/` dispatcher script detects the platform via `uname` and executes `--` (on Windows, agy directly invokes `.exe`). +Go-based plugins (`github`, `ast-grep`, `go-lsp`, `retro-status`, `settings-advisor`, `worktree-manager`, `test-runner`, and `agy-plugin-kit` validator) include native binaries for **`linux/amd64`**, **`darwin/arm64` (Apple Silicon)**, and **`windows/amd64`** in `bin/`. The extensionless `bin/` dispatcher script detects the platform via `uname` and executes `--` (on Windows, agy directly invokes `.exe`). **Other platforms (e.g. `linux/arm64` / `darwin/amd64`) are not bundled by default**, but can be built on the target machine without modifying any scripts: @@ -89,5 +94,6 @@ Each plugin acts as a wrapper for existing tools/servers and complies with their | settings-advisor | Custom Go implementation | MIT | | agy-plugin-kit | Custom Go implementation | MIT | | worktree-manager | `git` CLI | MIT | +| test-runner | `go test` (Go toolchain) | BSD-3-Clause | Plugins delegate execution to user-installed binaries/CLIs on PATH (no redistribution of external binaries). diff --git a/README_ja.md b/README_ja.md index 4490b62..9a2c4db 100644 --- a/README_ja.md +++ b/README_ja.md @@ -14,6 +14,7 @@ | **[retro-status](./retro-status/README_ja.md)** | リポジトリをスキャンし、RPGのレトロステータス画面風のAA(アスキーアート)で出力する MCP サーバー | | **[settings-advisor](./settings-advisor/README_ja.md)** | ワークスペースの規模・言語・機密/CI/本番設定を解析し、最適なモデル・サンドボックス・許可モードを控えめに提案する MCP サーバー | | **[worktree-manager](./worktree-manager/README_ja.md)** | `git worktree` の安全な操作(一覧・作成・削除・クリーンアップ)を提供し、ブランチ隔離や並行作業を支援する MCP サーバー | +| **[test-runner](./test-runner/README_ja.md)** | Goテストの実行・失敗ログの整理・再実行引数の生成を行う MCP サーバー | ## インストール方法 @@ -41,6 +42,9 @@ agy plugin install https://github.com/kwrkb/agy-plugins/settings-advisor # worktree-manager プラグイン agy plugin install https://github.com/kwrkb/agy-plugins/worktree-manager + +# test-runner プラグイン +agy plugin install https://github.com/kwrkb/agy-plugins/test-runner ``` 各プラグインの前提条件(PATH に入れるバイナリ / 認証設定)については、各ディレクトリの README を参照してください。 @@ -61,12 +65,13 @@ agy plugin install https://github.com/kwrkb/agy-plugins/worktree-manager | retro-status | `git`(推奨)、`rg`(任意) | 不要 | | settings-advisor | 不要(バイナリ同梱) | 不要 | | worktree-manager | `git`(PATH 上) | 不要 | +| test-runner | `go` >= 1.24 (PATH 上) | 不要 | ### 同梱プラットフォームと self-build -✅ **全プラグインで Windows ネイティブ動作確認済み** +既存プラグインは Windows ネイティブ動作確認済み。`test-runner` は Linux・macOS・Windows のCIテストを追加し、ローカルではLinuxで検証しています。 -Go 製プラグイン(`github`、`ast-grep`、`go-lsp`、`retro-status`、`settings-advisor`、`worktree-manager`、および `agy-plugin-kit` の validator)は、**`linux/amd64`**、**`darwin/arm64`(Apple Silicon)**、**`windows/amd64`** のネイティブバイナリを `bin/` に同梱しています。拡張子なしの `bin/` ディスパッチャスクリプトが `uname` で実行環境を判別し、`--` を起動します(Windows では agy が `.exe` を直接起動)。 +Go 製プラグイン(`github`、`ast-grep`、`go-lsp`、`retro-status`、`settings-advisor`、`worktree-manager`、`test-runner`、および `agy-plugin-kit` の validator)は、**`linux/amd64`**、**`darwin/arm64`(Apple Silicon)**、**`windows/amd64`** のネイティブバイナリを `bin/` に同梱しています。拡張子なしの `bin/` ディスパッチャスクリプトが `uname` で実行環境を判別し、`--` を起動します(Windows では agy が `.exe` を直接起動)。 **それ以外の環境(`linux/arm64` や `darwin/amd64` など)は標準同梱していません**が、スクリプトの書き換えなしで手元でビルドして動かせます: @@ -89,5 +94,6 @@ cd /src && CGO_ENABLED=0 go build -o "../bin/-$(go env GOOS)-$(go | settings-advisor | 独自 Go 実装 | MIT | | agy-plugin-kit | 独自 Go 実装 | MIT | | worktree-manager | `git` CLI | MIT | +| test-runner | `go test` (Go toolchain) | BSD-3-Clause | 各プラグインはユーザーの PATH 上にあるインストール済みバイナリ/CLI に処理を委譲します(外部バイナリの再配布は行っていません)。 diff --git a/build.ps1 b/build.ps1 index 807ba10..b951048 100644 --- a/build.ps1 +++ b/build.ps1 @@ -75,6 +75,7 @@ switch ($Target) { 'settings-advisor' { Build 'settings-advisor' 'settings-advisor' } 'go-lsp' { Build 'go-lsp' 'go-lsp' } 'worktree-manager' { Build 'worktree-manager' 'worktree-manager' } + 'test-runner' { Build 'test-runner' 'test-runner' } 'all' { Build 'github' 'github' Build 'agy-plugin-kit/validator' 'validator' @@ -83,9 +84,10 @@ switch ($Target) { Build 'settings-advisor' 'settings-advisor' Build 'go-lsp' 'go-lsp' Build 'worktree-manager' 'worktree-manager' + Build 'test-runner' 'test-runner' } default { - Write-Error "unknown target: $Target (expected: github | validator | ast-grep | retro-status | settings-advisor | go-lsp | worktree-manager | all)" + Write-Error "unknown target: $Target (expected: github | validator | ast-grep | retro-status | settings-advisor | go-lsp | worktree-manager | test-runner | all)" exit 2 } } diff --git a/build.sh b/build.sh index 7ee50ec..8018226 100755 --- a/build.sh +++ b/build.sh @@ -57,6 +57,7 @@ case "$target" in settings-advisor) build settings-advisor settings-advisor ;; go-lsp) build go-lsp go-lsp ;; worktree-manager) build worktree-manager worktree-manager ;; + test-runner) build test-runner test-runner ;; all) build github github build agy-plugin-kit/validator validator @@ -65,9 +66,10 @@ case "$target" in build settings-advisor settings-advisor build go-lsp go-lsp build worktree-manager worktree-manager + build test-runner test-runner ;; *) - echo "unknown target: $target (expected: github | validator | ast-grep | retro-status | settings-advisor | go-lsp | worktree-manager | all)" >&2 + echo "unknown target: $target (expected: github | validator | ast-grep | retro-status | settings-advisor | go-lsp | worktree-manager | test-runner | all)" >&2 exit 2 ;; esac diff --git a/test-runner/README.md b/test-runner/README.md new file mode 100644 index 0000000..198c20a --- /dev/null +++ b/test-runner/README.md @@ -0,0 +1,57 @@ +# test-runner — Go Test Runner + +A Go-only MCP plugin for agy. Runs one module at a time, summarizes test/build failures, and returns arguments for explicitly rerunning failed tests. Calls wait for completion; there is no background job, persistent history, or automatic retry. + +## Requirements and installation + +- Go 1.24 or later on `PATH`, with the toolchain/dependencies required by the target module available. +- Prebuilt servers: Linux amd64, macOS arm64, Windows amd64. Windows uses the OS-provided `taskkill.exe` to stop a process tree. + +```sh +agy plugin install https://github.com/kwrkb/agy-plugins/test-runner +``` + +## Tool: `go_test` + +| Argument | Meaning | +| :-- | :-- | +| `module_path` | Required directory containing `go.mod`. Prefer an absolute path: relative paths use the MCP server's working directory. | +| `packages` | Nonempty string array of Go package patterns; default `["./..."]`. Local paths and import paths are accepted only when all resolved packages belong to the selected module and are physically inside it. | +| `run` | Optional Go `-run` expression, including slash-separated subtest expressions. | +| `timeout_seconds` | Integer 1–300, default 60. Covers discovery, compilation, and tests. Process cleanup can take a few additional seconds. | + +Example arguments: + +```json +{"module_path":"/code/agy-plugins/test-runner/src","packages":["./..."],"timeout_seconds":60} +``` + +The server resolves packages with `go list -e -json`, then runs `go test -json -count=1 -timeout=…` using an argument array, without a shell. It always disables successful-test caching. It inherits the Go environment (including `go.work`, `GOFLAGS`, build settings, and toolchain selection); it does not recursively discover other modules. A workspace containing several modules can be tested through separate calls with explicit module directories. + +No arbitrary command, extra flags, `.go` file list, race/coverage option, fuzzing session, or benchmark option is exposed. Standard `go test` behavior, including examples and fuzz seed cases, remains in effect. Tests **execute project code** with the server's permissions and may change files or access the network. Go may use/download caches, dependencies, or a toolchain according to the inherited environment. This plugin is not an execution sandbox; use the host's execution permissions. + +## Results and retry + +Each MCP result contains a JSON text object: + +- `status`: `passed`, `failed`, `no_tests`, `timeout`, `cancelled`, or `error`. Test/build failures are ordinary results; execution errors, timeouts, and cancellation set MCP `isError`. +- `exit_code`: Go process exit code, or `null` when no process exit code is available. Discovery errors can report the discovery process's exit code. +- `elapsed_seconds`: Total elapsed time; `packages[].elapsed_seconds` is Go's per-package duration. +- `tests`: `passed`, `failed`, `skipped` terminal-event counts, **including parent tests and subtests**. Package events are not counted as tests. All-skipped tests are distinguished by these counts; `no_tests` means no test terminal events and no failures. +- `packages`: Sorted package results with Go status `pass`, `fail`, `skip`, or `incomplete`; failing tests include their names, logs, and `rerun` arguments. Package-level failures such as a panic or `TestMain` exit may have no named failure; inspect the package log. +- `build_failures`: Build package IDs and diagnostics, separate from failed test counts. IDs may include a test variant suffix. +- `stderr`, `error`: Diagnostics and execution errors, when present. +- `logs_truncated`: Some log bytes were omitted. Retained event logs and stderr together are bounded to 1 MiB; returned log text is bounded to 64 KiB, prioritizing named failures. JSON escaping can make the serialized result larger. +- `incomplete`: Partial output, interrupted execution, or missing terminal events. Never interpret this as complete validation. + +Use a failure's `rerun` object as the next `go_test` arguments. It retains the absolute module path and timeout, selects one package, and anchors/escapes each subtest-name segment. Go still runs parent setup to discover a selected subtest. Parent failures can overlap child failures; choose the parent to rerun a group or the child to narrow the retry. There is no server-side retry state. + +Malformed/oversized JSON events (over 1 MiB), over 100,000 result records, and discovery output over 8 MiB produce explicit execution errors. Output continues to drain after a parsing error to avoid blocking the child; the overall timeout still applies. Timeout/cancellation stops the Unix process group or the Windows process tree and bounds pipe shutdown. Programs that deliberately detach from the process tree are outside this mechanism. + +## Development and validation + +In `test-runner/src`, run `gofmt`, `go vet ./...`, and `go test -count=1 ./...`. Tests use temporary modules without third-party dependencies and verify the real Go JSON stream, exact subtest retries, child-process termination, and MCP stdio calls. + +From the repository root, run `./build.sh test-runner` (or `./build.ps1 test-runner`). These use the pinned `.go-version`; do not replace distributed binaries with an unpinned build. CI runs tests on Linux, macOS, and Windows and checks vulnerability policy and deterministic artifacts on Linux. + +The bundled agent guide is [test-runner-gemini](skills/test-runner-gemini/SKILL.md). diff --git a/test-runner/README_ja.md b/test-runner/README_ja.md new file mode 100644 index 0000000..eb2a488 --- /dev/null +++ b/test-runner/README_ja.md @@ -0,0 +1,57 @@ +# test-runner — Go テストランナー + +Go専用の agy 向けMCPプラグインです。1モジュールずつテストし、テスト失敗・ビルド失敗・再実行引数を整理して返します。呼び出しは完了まで待機します。バックグラウンド実行、履歴保存、自動リトライはありません。 + +## 前提条件とインストール + +- `PATH` 上に Go 1.24 以降。対象モジュールが必要とするツールチェーン・依存関係も必要です。 +- 配布物は Linux amd64、macOS arm64、Windows amd64。Windowsのプロセスツリー停止にはOS同梱の `taskkill.exe` を使います。 + +```sh +agy plugin install https://github.com/kwrkb/agy-plugins/test-runner +``` + +## ツール: `go_test` + +| 引数 | 内容 | +| :-- | :-- | +| `module_path` | 必須。`go.mod` があるディレクトリ。相対パスはMCPサーバーの作業ディレクトリ基準なので、絶対パスを推奨します。 | +| `packages` | 空でない文字列配列。既定 `["./..."]`。ローカルパス・import pathを受け付け、解決した全パッケージが対象モジュールに所属し、そのディレクトリ内にあることを確認します。 | +| `run` | 任意。Goの `-run` 正規表現。`/` 区切りのサブテスト指定にも対応します。 | +| `timeout_seconds` | 1~300の整数。既定60秒。探索・ビルド・テスト全体の制限です。停止処理には追加で数秒かかる場合があります。 | + +引数例: + +```json +{"module_path":"/code/agy-plugins/test-runner/src","packages":["./..."],"timeout_seconds":60} +``` + +`go list -e -json` でパッケージを解決し、`go test -json -count=1 -timeout=…` を引数配列で起動します。シェルは経由しません。成功結果のキャッシュ利用を無効にし、毎回実行します。`go.work`、`GOFLAGS`、ビルド設定、ツールチェーン指定などのGo環境は継承します。複数モジュールの自動探索は行わず、対象ディレクトリを指定して個別に呼び出します。 + +任意コマンド、追加フラグ、`.go` ファイル一覧、race・カバレッジ・継続的fuzzing・ベンチマーク用のオプションは提供しません。Exampleやfuzz seedの実行を含む通常の `go test` の動作は維持します。テストは**プロジェクトのコードをサーバーの権限で実行**するため、ファイル変更や通信を行う場合があります。Goも継承した環境に従ってキャッシュ利用・依存取得・ツールチェーン取得を行う場合があります。このプラグイン自体はサンドボックスではなく、ホスト側の実行権限を利用します。 + +## 結果と再実行 + +MCPのテキストとしてJSONオブジェクトを返します。 + +- `status`: `passed`、`failed`、`no_tests`、`timeout`、`cancelled`、`error`。テスト・ビルド失敗は通常の結果、実行エラー・タイムアウト・キャンセルはMCPの `isError` が真になります。 +- `exit_code`: Goプロセスの終了コード。取得できない場合は `null`。探索失敗時には探索プロセスの終了コードを返す場合があります。 +- `elapsed_seconds`: 全体の所要時間。`packages[].elapsed_seconds` はGoが報告したパッケージ単位の時間です。 +- `tests`: `passed`・`failed`・`skipped` の終了イベント件数。**親テストとサブテストの両方を含み**、パッケージのイベントは含みません。全件skipは件数で判別できます。失敗もテスト終了イベントもない場合が `no_tests` です。 +- `packages`: 名前順のパッケージ結果。状態はGoの `pass`・`fail`・`skip`、または `incomplete`。失敗テストには名前・ログ・`rerun` 引数を付けます。panicや `TestMain` の終了など、名前付き失敗がない場合はパッケージログを確認します。 +- `build_failures`: テスト失敗件数とは別のビルド診断。パッケージIDにテスト用variantの接尾辞が含まれる場合があります。 +- `stderr`・`error`: 診断ログ・実行エラーがある場合に返します。 +- `logs_truncated`: ログを省略した場合に真。イベントログとstderrの保持は合計1 MiB、返却ログ本文は合計64 KiBを上限とし、名前付き失敗を優先します。JSONエスケープ後の応答サイズはこれより大きくなる場合があります。 +- `incomplete`: 中断・不正出力・終了イベント欠落による部分的な結果です。検証完了とは扱わないでください。 + +失敗の `rerun` オブジェクトを次の `go_test` の引数として使えます。絶対モジュールパスと制限時間を維持し、パッケージを1つに絞り、サブテスト名の各階層をエスケープして完全一致させます。Goはサブテストの探索に親のセットアップも実行します。親と子の失敗は重複するため、グループを再実行するなら親、対象を絞るなら子を選びます。サーバー内に前回の状態は保存しません。 + +不正JSON、1 MiBを超えるイベント、100,000件を超える結果レコード、8 MiBを超える探索結果は実行エラーです。解析エラー後も出力を読み捨ててパイプの詰まりを防ぎ、全体タイムアウトを適用します。タイムアウト・キャンセル時はUnixのプロセスグループまたはWindowsのプロセスツリーを停止し、パイプ終了待ちにも上限を設けます。意図的にプロセスツリーから離脱したプログラムはこの停止機構の対象外です。 + +## 開発と検証 + +`test-runner/src` で `gofmt`、`go vet ./...`、`go test -count=1 ./...` を実行します。テストは第三者依存のない一時モジュールを作成し、実GoのJSON出力・サブテスト再実行・子プロセス停止・MCP stdioを検証します。 + +リポジトリルートで `./build.sh test-runner`(Windowsでは `./build.ps1 test-runner`)を実行します。配布物は `.go-version` の固定版でビルドし、素の `go build` の成果物では置き換えません。CIではLinux・macOS・Windowsのテストと、Linuxでの脆弱性ポリシー・決定論ビルドの検査を行います。 + +利用ガイドは [test-runner-gemini](skills/test-runner-gemini/SKILL.md) を同梱します。 diff --git a/test-runner/bin/test-runner b/test-runner/bin/test-runner new file mode 100755 index 0000000..b33823c --- /dev/null +++ b/test-runner/bin/test-runner @@ -0,0 +1,25 @@ +#!/bin/sh +# Platform dispatcher: exec the native binary for this OS/arch. +# This file is plain text (OS-independent) and is committed to git so that +# `agy plugin install` (which only copies files, no build) ships a single +# `command` target that works on every Unix platform. Windows is unaffected: +# agy appends `.exe` and runs `.exe` directly, bypassing this script. +# Keep stdout clean — MCP speaks NDJSON over stdout; errors go to stderr only. +dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) +base=test-runner +# Compute Go GOOS/GOARCH from uname; the binary name matches build.sh output. +case "$(uname -s)" in + Linux) goos=linux ;; + Darwin) goos=darwin ;; + *) echo "agy-plugins: unsupported OS $(uname -s)" >&2; exit 1 ;; +esac +case "$(uname -m)" in + x86_64|amd64) goarch=amd64 ;; + aarch64|arm64) goarch=arm64 ;; + *) echo "agy-plugins: unsupported arch $(uname -m)" >&2; exit 1 ;; +esac +# Platforms whose binary is not committed (e.g. linux/arm64, darwin/amd64) +# self-serve: build natively and the dispatcher picks it up — no edit needed. +bin="$dir/${base}-${goos}-${goarch}" +[ -x "$bin" ] || { echo "agy-plugins: no prebuilt binary for ${goos}/${goarch}; build natively with ./build.sh (see README)" >&2; exit 1; } +exec "$bin" "$@" diff --git a/test-runner/bin/test-runner-darwin-arm64 b/test-runner/bin/test-runner-darwin-arm64 new file mode 100755 index 0000000..7932da2 Binary files /dev/null and b/test-runner/bin/test-runner-darwin-arm64 differ diff --git a/test-runner/bin/test-runner-linux-amd64 b/test-runner/bin/test-runner-linux-amd64 new file mode 100755 index 0000000..4cbb3e8 Binary files /dev/null and b/test-runner/bin/test-runner-linux-amd64 differ diff --git a/test-runner/bin/test-runner.exe b/test-runner/bin/test-runner.exe new file mode 100755 index 0000000..fee5531 Binary files /dev/null and b/test-runner/bin/test-runner.exe differ diff --git a/test-runner/gemini-extension.json b/test-runner/gemini-extension.json new file mode 100644 index 0000000..3656647 --- /dev/null +++ b/test-runner/gemini-extension.json @@ -0,0 +1,10 @@ +{ + "name": "test-runner", + "version": "1.0.0", + "description": "Run Go tests and summarize failures with rerun arguments.", + "mcpServers": { + "test-runner": { + "command": "${extensionPath}${/}bin${/}test-runner" + } + } +} diff --git a/test-runner/skills/test-runner-gemini/SKILL.md b/test-runner/skills/test-runner-gemini/SKILL.md new file mode 100644 index 0000000..bd1326d --- /dev/null +++ b/test-runner/skills/test-runner-gemini/SKILL.md @@ -0,0 +1,54 @@ +--- +name: test-runner-gemini +description: go_test MCP ツールを使用して Go テストを実行し、失敗内容の把握と対象を絞った再実行を行うスキル。test-runner プラグイン利用時に参照する。 +--- + +# Go test runner スキル + +このスキルは、`test-runner` プラグインの `go_test` MCP ツールを使用して、Go モジュールのテスト実行、失敗原因の特定、および修正後の効率的な再実行を行うためのガイドです。 + +## 基本的な使い方 + +`go_test` を呼び出す際は、`go.mod` が存在するディレクトリの絶対パスを `module_path` に指定します。MCP サーバーのカレントワーキングディレクトリはリポジトリルートと異なる場合があるため、相対パスではなく絶対パスを推奨します。 + +- **`module_path`** (string, 必須): `go.mod` が置かれているディレクトリの絶対パス。マルチモジュール構成のリポジトリでは、対象モジュールごとに個別に呼び出します。 +- **`packages`** (array of string, 任意): パッケージパターン。既定値は `["./..."]`。フラグや `.go` ファイル一覧の指定は不可。 +- **`run`** (string, 任意): Go の `-run` 正規表現(サブテストの `/` 区切り構文に対応)。 +- **`timeout_seconds`** (integer, 任意): パッケージ探索・コンパイル・テスト実行全体の上限時間(1〜300秒、既定60秒)。 + +## 実行結果の確認ポイント + +返却される JSON オブジェクトから以下のフィールドを確認します: + +1. **`status`**: + - `passed`: 失敗なし(全件スキップの場合も含む。下記 `tests` を参照) + - `failed`: テスト失敗またはビルド失敗(MCP ツールとしては正常終了扱い) + - `no_tests`: テストの終了イベントが 0 件(実行対象のテストが無かった)。**関連テストが通った証拠にはならない** + - `error` / `timeout` / `cancelled`: ツールエラー、タイムアウト、キャンセル中断(MCP の `isError` が true) + これらをテスト成功として報告してはいけません。 +2. **`tests`**: + - `passed`・`failed`・`skipped` の件数。**親テストとサブテストの両方を含みます**。 + - 全件スキップは `no_tests` ではなく `passed` になります。`passed` でも `tests.skipped` だけが計上されていれば、実際には何も実行されていません。 +3. **`build_failures`**: + - ビルド・コンパイルエラーの一覧とログ。テスト失敗件数とは別に報告されます。 +4. **`packages`**: + - パッケージ単位の結果一覧。`failures` 配列に失敗したテスト名とログが含まれます。 + - 個別のテスト名がつかない失敗(panic、セットアップ失敗、`TestMain` の異常終了など)は、パッケージログ(`log`)および `stderr` を確認します。 +5. **`logs_truncated`**: + - 返却ログ上限(64 KiB)に達してログが切り詰められた場合に `true` となります。必要な診断情報を得るには、対象パッケージやテストを絞り込んで再実行してください。 +6. **`incomplete`**: + - プロセス中断やタイムアウト等により結果が途中で切れている場合に `true` となります。 + +## 修正後の再実行(rerun オブジェクトの活用) + +テストが失敗した場合、各失敗情報に `rerun` オブジェクトが含まれます。 + +- コード修正後は、失敗したテストの `rerun` オブジェクトをそのまま `go_test` の引数として渡すことで、該当テストのみをピンポイントで再実行できます。 +- 親テストとサブテストの両方が失敗として報告されている場合、自動で両方を再実行するのではなく、目的に応じて適切な粒度(親全体か特定のサブテストか)を選択してください。 +- テスト範囲を広げるのは、変更内容がそれを必要とする場合だけにします。 +- プラグインは自動リトライも履歴保存も行いません。実行エラーやタイムアウトが続く場合は無限に再試行せず、実際に確認できた範囲とその限界を報告してください。 + +## 権限と実行環境に関する注意 + +- テストはサーバーホストの権限と Go 環境(`go.work`、`GOFLAGS`、ツールチェーン等)を継承してプロジェクトのコードを直接実行します。 +- ファイルの書き込みや外部通信が発生する可能性があります。このツール自体はサンドボックスを提供しないため、現在のタスクの許可範囲内で実行してください。 diff --git a/test-runner/src/events.go b/test-runner/src/events.go new file mode 100644 index 0000000..e5ba964 --- /dev/null +++ b/test-runner/src/events.go @@ -0,0 +1,301 @@ +package main + +import ( + "bytes" + "encoding/json" + "fmt" + "regexp" + "sort" + "strings" + "unicode/utf8" +) + +const retainedLogLimit = 1 << 20 +const returnedLogLimit = 64 << 10 +const eventLimit = 1 << 20 +const recordLimit = 100000 + +type Counts struct { + Passed int `json:"passed"` + Failed int `json:"failed"` + Skipped int `json:"skipped"` +} +type Failure struct { + Test string `json:"test,omitempty"` + Log string `json:"log,omitempty"` + Rerun Options `json:"rerun"` +} +type PackageResult struct { + Package string `json:"package"` + Status string `json:"status"` + Elapsed float64 `json:"elapsed_seconds"` + Counts Counts `json:"tests"` + Failures []Failure `json:"failures,omitempty"` + Log string `json:"log,omitempty"` +} +type BuildFailure struct { + Package string `json:"package"` + Log string `json:"log,omitempty"` +} +type Result struct { + Status string `json:"status"` + ExitCode *int `json:"exit_code"` + Elapsed float64 `json:"elapsed_seconds"` + Counts Counts `json:"tests"` + Packages []PackageResult `json:"packages,omitempty"` + BuildFailures []BuildFailure `json:"build_failures,omitempty"` + Stderr string `json:"stderr,omitempty"` + Error string `json:"error,omitempty"` + LogsTruncated bool `json:"logs_truncated"` + Incomplete bool `json:"incomplete"` +} +type event struct { + Action string + Package string + Test string + Elapsed float64 + Output string + ImportPath string + FailedBuild string +} +type testState struct { + status string + log []byte +} +type packageState struct { + status string + elapsed float64 + tests map[string]*testState + log []byte +} +type eventStream struct { + pending []byte + err error + packages map[string]*packageState + buildLogs map[string][]byte + buildFailed map[string]bool + retained int + records int + truncated bool +} + +func newEventStream() *eventStream { + return &eventStream{packages: map[string]*packageState{}, buildLogs: map[string][]byte{}, buildFailed: map[string]bool{}} +} + +// Write parses complete lines without retaining the full stdout. On a malformed +// event, continue draining so that a child cannot deadlock on a full pipe. +func (s *eventStream) Write(p []byte) (int, error) { + n := len(p) + for len(p) > 0 && s.err == nil { + i := bytes.IndexByte(p, '\n') + if i < 0 { + i = len(p) + } + if len(s.pending)+i > eventLimit { + s.err = fmt.Errorf("JSON event exceeds %d bytes", eventLimit) + s.pending = nil + break + } + s.pending = append(s.pending, p[:i]...) + p = p[i:] + if len(p) == 0 { + break + } + s.consume(s.pending) + s.pending = s.pending[:0] + p = p[1:] + } + return n, nil +} +func (s *eventStream) finish() { + if s.err == nil && len(bytes.TrimSpace(s.pending)) > 0 { + s.consume(s.pending) + } + s.pending = nil +} +func (s *eventStream) appendLog(dst []byte, v string) []byte { + // Reserve room for the independently drained stderr buffer. + n := min(len(v), retainedLogLimit-returnedLogLimit-s.retained) + s.retained += n + s.truncated = s.truncated || n < len(v) + return append(dst, v[:n]...) +} +func (s *eventStream) addRecord() bool { + s.records++ + if s.records > recordLimit { + s.err = fmt.Errorf("more than %d result records", recordLimit) + return false + } + return true +} +func (s *eventStream) markBuildFailure(name string) { + if !s.buildFailed[name] && s.addRecord() { + s.buildFailed[name] = true + } +} +func (s *eventStream) consume(line []byte) { + if len(bytes.TrimSpace(line)) == 0 { + return + } + var e event + if err := json.Unmarshal(line, &e); err != nil || e.Action == "" { + s.err = fmt.Errorf("invalid go test JSON event") + return + } + if e.Action == "build-output" || e.Action == "build-fail" { + if e.ImportPath == "" { + s.err = fmt.Errorf("build event missing ImportPath") + return + } + if _, ok := s.buildLogs[e.ImportPath]; !ok { + if !s.addRecord() { + return + } + s.buildLogs[e.ImportPath] = nil + } + s.buildLogs[e.ImportPath] = s.appendLog(s.buildLogs[e.ImportPath], e.Output) + if e.Action == "build-fail" { + s.markBuildFailure(e.ImportPath) + } + return + } + if e.Package == "" { + s.err = fmt.Errorf("test event missing Package") + return + } + p := s.packages[e.Package] + if p == nil { + if !s.addRecord() { + return + } + p = &packageState{status: "incomplete", tests: map[string]*testState{}} + s.packages[e.Package] = p + } + if e.FailedBuild != "" { + s.markBuildFailure(e.FailedBuild) + } + if e.Test == "" { + if e.Action == "output" { + p.log = s.appendLog(p.log, e.Output) + } + if terminal(e.Action) { + p.status, p.elapsed = e.Action, e.Elapsed + } + return + } + t := p.tests[e.Test] + if t == nil { + if !s.addRecord() { + return + } + t = &testState{status: "incomplete"} + p.tests[e.Test] = t + } + if e.Action == "output" { + t.log = s.appendLog(t.log, e.Output) + } + if terminal(e.Action) { + t.status = e.Action + if e.Action != "fail" { + s.retained -= len(t.log) + t.log = nil + } + } +} +func terminal(a string) bool { return a == "pass" || a == "fail" || a == "skip" } +func exactRun(name string) string { + parts := strings.Split(name, "/") + for i, p := range parts { + parts[i] = "^" + regexp.QuoteMeta(p) + "$" + } + return strings.Join(parts, "/") +} + +// Apply the byte budget after replacing invalid UTF-8, so JSON encoding cannot +// expand malformed stderr past the documented log-text limit. +func boundedLog(b []byte, limit int) (string, bool) { + v := strings.ToValidUTF8(string(b), "�") + n := min(len(v), limit) + for n > 0 && !utf8.ValidString(v[:n]) { + n-- + } + return v[:n], n < len(v) +} + +func (s *eventStream) result(o Options, stderr *cappedBuffer) Result { + r := Result{Status: "passed", LogsTruncated: s.truncated || stderr.truncated} + remaining := returnedLogLimit + log := func(b []byte) string { + v, truncated := boundedLog(b, remaining) + remaining -= len(v) + r.LogsTruncated = r.LogsTruncated || truncated + return v + } + // Failures have priority over package framing and diagnostic stderr. + for _, name := range sortedKeys(s.packages) { + p := s.packages[name] + pr := PackageResult{Package: name, Status: p.status, Elapsed: p.elapsed} + for _, test := range sortedKeys(p.tests) { + t := p.tests[test] + switch t.status { + case "pass": + pr.Counts.Passed++ + case "skip": + pr.Counts.Skipped++ + case "fail": + pr.Counts.Failed++ + rerun := o + rerun.Packages, rerun.Run = []string{name}, exactRun(test) + pr.Failures = append(pr.Failures, Failure{Test: test, Log: log(t.log), Rerun: rerun}) + default: + r.Incomplete = true + } + } + if p.status == "incomplete" { + r.Incomplete = true + } + if p.status == "fail" || pr.Counts.Failed > 0 { + r.Status = "failed" + } + r.Counts.Passed += pr.Counts.Passed + r.Counts.Failed += pr.Counts.Failed + r.Counts.Skipped += pr.Counts.Skipped + r.Packages = append(r.Packages, pr) + } + for _, name := range sortedKeys(s.buildFailed) { + r.BuildFailures = append(r.BuildFailures, BuildFailure{Package: name, Log: log(s.buildLogs[name])}) + r.Status = "failed" + } + for i := range r.Packages { + p := &r.Packages[i] + // A panic or cancellation can leave test-scoped output without a fail + // event. Keep this evidence without inventing a failed-test count. + state := s.packages[p.Package] + for _, name := range sortedKeys(state.tests) { + t := state.tests[name] + if t.status == "incomplete" && len(t.log) > 0 { + p.Log += log([]byte(name+":\n")) + log(t.log) + } + } + if p.Status != "pass" && p.Status != "skip" { + p.Log += log(state.log) + } + } + r.Stderr = log(stderr.data) + if r.Status == "passed" && r.Counts == (Counts{}) { + r.Status = "no_tests" + } + if s.err != nil { + r.Status, r.Error, r.Incomplete = "error", s.err.Error(), true + } + return r +} +func sortedKeys[V any](m map[string]V) []string { + keys := make([]string, 0, len(m)) + for k := range m { + keys = append(keys, k) + } + sort.Strings(keys) + return keys +} diff --git a/test-runner/src/go.mod b/test-runner/src/go.mod new file mode 100644 index 0000000..06c81d5 --- /dev/null +++ b/test-runner/src/go.mod @@ -0,0 +1,14 @@ +module github.com/kwrkb/agy-plugins/test-runner + +go 1.26.5 + +require github.com/mark3labs/mcp-go v0.54.1 + +require ( + github.com/google/jsonschema-go v0.4.2 // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/santhosh-tekuri/jsonschema/v6 v6.0.2 // indirect + github.com/spf13/cast v1.7.1 // indirect + github.com/yosida95/uritemplate/v3 v3.0.2 // indirect + golang.org/x/text v0.14.0 // indirect +) diff --git a/test-runner/src/go.sum b/test-runner/src/go.sum new file mode 100644 index 0000000..d7a17e9 --- /dev/null +++ b/test-runner/src/go.sum @@ -0,0 +1,34 @@ +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dlclark/regexp2 v1.11.0 h1:G/nrcoOa7ZXlpoa/91N3X7mM3r8eIlMBBJZvsz/mxKI= +github.com/dlclark/regexp2 v1.11.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8= +github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= +github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/google/jsonschema-go v0.4.2 h1:tmrUohrwoLZZS/P3x7ex0WAVknEkBZM46iALbcqoRA8= +github.com/google/jsonschema-go v0.4.2/go.mod h1:r5quNTdLOYEz95Ru18zA0ydNbBuYoo9tgaYcxEYhJVE= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/mark3labs/mcp-go v0.54.1 h1:Ap/ptEB9FtWzFKM8NDsTA7QDxerQOC06eZigrTldVj0= +github.com/mark3labs/mcp-go v0.54.1/go.mod h1:+8WclSK1ZUweCP3hvktSji8n8ABG/95QaEkeVE/Uwas= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= +github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= +github.com/santhosh-tekuri/jsonschema/v6 v6.0.2 h1:KRzFb2m7YtdldCEkzs6KqmJw4nqEVZGK7IN2kJkjTuQ= +github.com/santhosh-tekuri/jsonschema/v6 v6.0.2/go.mod h1:JXeL+ps8p7/KNMjDQk3TCwPpBy0wYklyWTfbkIzdIFU= +github.com/spf13/cast v1.7.1 h1:cuNEagBQEHWN1FnbGEjCXL2szYEXqfJPbP2HNUaca9Y= +github.com/spf13/cast v1.7.1/go.mod h1:ancEpBxwJDODSW/UG4rDrAqiKolqNNh2DX3mk86cAdo= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +github.com/yosida95/uritemplate/v3 v3.0.2 h1:Ed3Oyj9yrmi9087+NczuL5BwkIc4wvTb5zIM+UJPGz4= +github.com/yosida95/uritemplate/v3 v3.0.2/go.mod h1:ILOh0sOhIJR3+L/8afwt/kE++YT040gmv5BQTMR2HP4= +golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= +golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/test-runner/src/main.go b/test-runner/src/main.go new file mode 100644 index 0000000..c677854 --- /dev/null +++ b/test-runner/src/main.go @@ -0,0 +1,100 @@ +package main + +import ( + "context" + "encoding/json" + "fmt" + "math" + "os" + "strings" + + "github.com/mark3labs/mcp-go/mcp" + "github.com/mark3labs/mcp-go/server" +) + +type Options struct { + ModulePath string `json:"module_path"` + Packages []string `json:"packages"` + Run string `json:"run,omitempty"` + Timeout int `json:"timeout_seconds"` +} + +func parseOptions(args map[string]any) (Options, error) { + o := Options{Packages: []string{"./..."}, Timeout: 60} + for k := range args { + if k != "module_path" && k != "packages" && k != "run" && k != "timeout_seconds" { + return o, fmt.Errorf("unknown argument %q", k) + } + } + var ok bool + o.ModulePath, ok = args["module_path"].(string) + if !ok || strings.TrimSpace(o.ModulePath) == "" { + return o, fmt.Errorf("module_path must be a nonempty string") + } + if v, exists := args["packages"]; exists { + // Marshal/unmarshal accepts both []string and JSON-decoded []any, but + // rejects mixed types. Do not use the SDK's coercing argument getters. + b, err := json.Marshal(v) + if err != nil || json.Unmarshal(b, &o.Packages) != nil || len(o.Packages) == 0 { + return o, fmt.Errorf("packages must be a nonempty string array") + } + } + for _, p := range o.Packages { + if p == "" || strings.HasPrefix(p, "-") || strings.ContainsAny(p, "\x00\r\n") || strings.HasSuffix(p, ".go") { + return o, fmt.Errorf("invalid package pattern %q (file lists and flags are not supported)", p) + } + } + if v, exists := args["run"]; exists { + o.Run, ok = v.(string) + if !ok || strings.ContainsRune(o.Run, 0) { + return o, fmt.Errorf("run must be a string without NUL bytes") + } + } + if v, exists := args["timeout_seconds"]; exists { + b, err := json.Marshal(v) + var n float64 + if err != nil || json.Unmarshal(b, &n) != nil || math.IsNaN(n) || n < 1 || n > 300 || math.Trunc(n) != n { + return o, fmt.Errorf("timeout_seconds must be an integer from 1 to 300") + } + o.Timeout = int(n) + } + return o, nil +} + +func newServer() *server.MCPServer { + s := server.NewMCPServer("test-runner", "1.0.0") + s.AddTool(mcp.NewTool("go_test", + mcp.WithDescription("Run Go tests in one module, summarizing failures and rerun arguments. Executes project code with inherited permissions/environment; tests and Go may write files, use caches, or access the network. No automatic retries. Timeout includes package discovery and compilation."), + mcp.WithReadOnlyHintAnnotation(false), + mcp.WithIdempotentHintAnnotation(false), + mcp.WithOpenWorldHintAnnotation(true), + mcp.WithString("module_path", mcp.Required(), mcp.Description("Directory containing go.mod. Prefer an absolute path; relative paths use the MCP server's working directory.")), + mcp.WithArray("packages", mcp.Items(map[string]any{"type": "string"}), mcp.MinItems(1), mcp.DefaultArray([]string{"./..."}), mcp.Description("Package patterns within this module, default [\"./...\"]. No flags or .go file lists.")), + mcp.WithString("run", mcp.Description("Optional Go -run expression, including slash-separated subtest expressions.")), + mcp.WithInteger("timeout_seconds", mcp.Min(1), mcp.Max(300), mcp.DefaultNumber(60), mcp.Description("Whole execution timeout in integer seconds, 1–300, default 60.")), + ), func(ctx context.Context, req mcp.CallToolRequest) (*mcp.CallToolResult, error) { + o, err := parseOptions(req.GetArguments()) + if err != nil { + return resultTool(Result{Status: "error", Error: err.Error()}), nil + } + return resultTool(runTests(ctx, o)), nil + }) + return s +} + +func resultTool(r Result) *mcp.CallToolResult { + b, err := json.Marshal(r) + if err != nil { + return mcp.NewToolResultError(err.Error()) + } + res := mcp.NewToolResultText(string(b)) + res.IsError = r.Status == "error" || r.Status == "timeout" || r.Status == "cancelled" + return res +} + +func main() { + if err := server.ServeStdio(newServer()); err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } +} diff --git a/test-runner/src/main_test.go b/test-runner/src/main_test.go new file mode 100644 index 0000000..40c3391 --- /dev/null +++ b/test-runner/src/main_test.go @@ -0,0 +1,501 @@ +package main + +import ( + "context" + "encoding/json" + "fmt" + "io" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + "time" + "unicode/utf8" + + "github.com/mark3labs/mcp-go/client" + "github.com/mark3labs/mcp-go/client/transport" + "github.com/mark3labs/mcp-go/mcp" +) + +func TestOptions(t *testing.T) { + for _, args := range []map[string]any{ + {}, {"module_path": 1}, {"module_path": " "}, + {"module_path": ".", "packages": []any{".", 2}}, + {"module_path": ".", "packages": []string{}}, + {"module_path": ".", "packages": []string{"-exec=evil"}}, + {"module_path": ".", "packages": []string{"a.go"}}, + {"module_path": ".", "packages": nil}, + {"module_path": ".", "run": false}, + {"module_path": ".", "timeout_seconds": "60"}, + {"module_path": ".", "timeout_seconds": 0}, + {"module_path": ".", "timeout_seconds": 301}, + {"module_path": ".", "timeout_seconds": 1.5}, + {"module_path": ".", "timeout_seconds": nil}, + {"module_path": ".", "args": []string{"-race"}}, + } { + if _, err := parseOptions(args); err == nil { + t.Errorf("accepted invalid arguments: %#v", args) + } + } + for _, timeout := range []int{1, 300} { + o, err := parseOptions(map[string]any{"module_path": "日本語 space", "timeout_seconds": timeout}) + if err != nil || o.Timeout != timeout || o.Packages[0] != "./..." { + t.Fatalf("%+v %v", o, err) + } + } +} + +func emit(s *eventStream, es ...event) { + for _, e := range es { + b, _ := json.Marshal(e) + _, _ = s.Write(append(b, '\n')) + } +} +func TestInterleavedEventsAndRerun(t *testing.T) { + s := newEventStream() + emit(s, + event{Action: "start", Package: "example.com/a"}, + event{Action: "run", Package: "example.com/a", Test: "TestParent"}, + event{Action: "run", Package: "example.com/a", Test: "TestParent/a+b[1]"}, + event{Action: "pause", Package: "example.com/a", Test: "TestParent/a+b[1]"}, + event{Action: "start", Package: "example.com/b"}, + event{Action: "skip", Package: "example.com/b"}, + event{Action: "cont", Package: "example.com/a", Test: "TestParent/a+b[1]"}, + event{Action: "output", Package: "example.com/a", Test: "TestParent/a+b[1]", Output: "failure reason\n"}, + event{Action: "fail", Package: "example.com/a", Test: "TestParent/a+b[1]"}, + event{Action: "fail", Package: "example.com/a", Test: "TestParent"}, + event{Action: "skip", Package: "example.com/a", Test: "TestSkip"}, + event{Action: "fail", Package: "example.com/a", Elapsed: 0.1}, + ) + s.finish() + r := s.result(Options{ModulePath: "/module", Timeout: 60}, &cappedBuffer{}) + if r.Status != "failed" || r.Incomplete || r.Counts != (Counts{Failed: 2, Skipped: 1}) { + t.Fatalf("%+v", r) + } + f := r.Packages[0].Failures[1] + if f.Log != "failure reason\n" || f.Rerun.Run != `^TestParent$/^a\+b\[1\]$` || f.Rerun.Packages[0] != "example.com/a" { + t.Fatalf("%+v", f) + } +} + +func TestBuildEventsAndMalformedOutput(t *testing.T) { + s := newEventStream() + emit(s, event{Action: "build-output", ImportPath: "example.com/a [example.com/a.test]", Output: "compile error"}, + event{Action: "build-fail", ImportPath: "example.com/a [example.com/a.test]"}, + event{Action: "fail", Package: "example.com/a", FailedBuild: "example.com/a [example.com/a.test]"}) + r := s.result(Options{}, &cappedBuffer{}) + if r.Status != "failed" || len(r.BuildFailures) != 1 || r.BuildFailures[0].Log != "compile error" || r.Counts.Failed != 0 { + t.Fatalf("%+v", r) + } + for _, bad := range []string{"not json\n", "{}\n", `{"Action":"pass"}`, `{"Action":"start","Package":`, strings.Repeat("x", eventLimit+1)} { + s = newEventStream() + _, _ = s.Write([]byte(bad)) + s.finish() + if s.result(Options{}, &cappedBuffer{}).Status != "error" { + t.Fatalf("accepted malformed output %.50q", bad) + } + } + // Complete JSON without a trailing newline is valid, including chunk splits. + s = newEventStream() + for _, chunk := range []string{`{"Action":"pa`, `ss","Package":"p"}`} { + _, _ = s.Write([]byte(chunk)) + } + s.finish() + if s.result(Options{}, &cappedBuffer{}).Status != "no_tests" { + t.Fatal(s.err) + } +} + +func TestInterruptedTestKeepsDiagnostic(t *testing.T) { + s := newEventStream() + emit(s, event{Action: "run", Package: "p", Test: "TestInterrupted"}, + event{Action: "output", Package: "p", Test: "TestInterrupted", Output: "last diagnostic before interruption\n"}) + s.finish() + r := s.result(Options{}, &cappedBuffer{}) + if !r.Incomplete || r.Counts != (Counts{}) || !strings.Contains(r.Packages[0].Log, "last diagnostic") { + t.Fatalf("lost partial diagnostic: %+v", r) + } +} + +func TestLogsAreBoundedWithoutLosingResults(t *testing.T) { + s := newEventStream() + for i := 0; i < 40; i++ { + emit(s, event{Action: "output", Package: "p", Test: "TestFail", Output: strings.Repeat("x", 65536)}) + } + emit(s, event{Action: "fail", Package: "p", Test: "TestFail"}, event{Action: "pass", Package: "p", Test: "TestPass"}, event{Action: "fail", Package: "p"}) + r := s.result(Options{}, &cappedBuffer{}) + if !r.LogsTruncated || r.Counts != (Counts{Passed: 1, Failed: 1}) || len(r.Packages[0].Failures[0].Log) != returnedLogLimit || s.retained > retainedLogLimit { + t.Fatalf("%+v retained=%d", r.Counts, s.retained) + } + // >64 KiB JSON lines are accepted; do not regress to Scanner's default limit. + s = newEventStream() + emit(s, event{Action: "output", Package: "p", Output: strings.Repeat("x", 128<<10)}, event{Action: "pass", Package: "p"}) + if s.err != nil { + t.Fatal(s.err) + } +} + +func TestLogBudgetAfterUTF8Replacement(t *testing.T) { + text, truncated := boundedLog([]byte(strings.Repeat("\xffx", returnedLogLimit)), returnedLogLimit) + if len(text) > returnedLogLimit || !utf8.ValidString(text) || !truncated { + t.Fatal("invalid UTF-8 exceeded log budget") + } + text, truncated = boundedLog([]byte("あい"), 4) + if text != "あ" || !truncated { + t.Fatalf("split multibyte character: %q", text) + } +} + +func writeFile(t *testing.T, name, data string) { + t.Helper() + if err := os.MkdirAll(filepath.Dir(name), 0755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(name, []byte(data), 0644); err != nil { + t.Fatal(err) + } +} +func moduleFixture(t *testing.T) string { + t.Helper() + dir := filepath.Join(t.TempDir(), "日本語 module space") + writeFile(t, filepath.Join(dir, "go.mod"), "module example.com/fixture\n\ngo 1.24.0\n") + writeFile(t, filepath.Join(dir, "fixture_test.go"), `package fixture +import "testing" +func TestPass(t *testing.T) { t.Parallel() } +func TestSkip(t *testing.T) { t.Skip("skip") } +func TestFail(t *testing.T) { + t.Run("a+b[1]", func(t *testing.T) { t.Fatal("expected failure") }) + t.Run("unrelated", func(t *testing.T) { t.Log("must not run in child retry") }) +} +func TestPanic(t *testing.T) { panic("panic marker") } +`) + return dir +} +func offlineGo(t *testing.T) { + t.Helper() + t.Setenv("GOPROXY", "off") + t.Setenv("GOSUMDB", "off") + t.Setenv("GOTOOLCHAIN", "local") + t.Setenv("GOWORK", "off") + t.Setenv("GOFLAGS", "") +} +func TestRealGoResultsAndRetry(t *testing.T) { + offlineGo(t) + dir := moduleFixture(t) + o := Options{ModulePath: dir, Packages: []string{"./..."}, Timeout: 60} + for _, tt := range []struct { + run, status string + passed, failed, skipped int + }{ + {"^TestPass$", "passed", 1, 0, 0}, + {"^TestSkip$", "passed", 0, 0, 1}, + {"^DoesNotExist$", "no_tests", 0, 0, 0}, + {"^TestFail$", "failed", 1, 2, 0}, + {"^TestPanic$", "failed", 0, 0, 0}, + } { + t.Run(tt.run, func(t *testing.T) { + o.Run = tt.run + r := runTests(t.Context(), o) + if r.Status != tt.status { + t.Fatalf("%+v", r) + } + if tt.run == "^TestPanic$" { + b, _ := json.Marshal(r) + if !strings.Contains(string(b), "panic marker") { + t.Fatalf("lost panic diagnostic: %s", b) + } + } + if tt.run != "^TestPanic$" && r.Counts != (Counts{tt.passed, tt.failed, tt.skipped}) { + t.Fatalf("%+v", r) + } + if tt.run == "^TestFail$" { + f := r.Packages[0].Failures[1] + rr := runTests(t.Context(), f.Rerun) + if rr.Status != "failed" || rr.Counts.Passed != 0 || rr.Counts.Failed != 2 || !strings.Contains(rr.Packages[0].Failures[1].Log, "expected failure") { + t.Fatalf("retry: %+v", rr) + } + } + }) + } + writeFile(t, filepath.Join(dir, "broken.go"), "package fixture\nvar x = undefinedSymbol\n") + r := runTests(t.Context(), o) + if r.Status != "failed" || len(r.BuildFailures) == 0 || r.Counts.Failed != 0 { + t.Fatalf("build failure: %+v", r) + } +} + +func TestModuleBoundariesAndEmptyModule(t *testing.T) { + offlineGo(t) + dir := moduleFixture(t) + o := Options{ModulePath: dir, Packages: []string{"fmt"}, Timeout: 60} + if r := runTests(t.Context(), o); r.Status != "error" { + t.Fatalf("external package allowed: %+v", r) + } + o.Packages = []string{"./does-not-exist"} + if r := runTests(t.Context(), o); r.Status != "error" { + t.Fatalf("missing package: %+v", r) + } + empty := t.TempDir() + writeFile(t, filepath.Join(empty, "go.mod"), "module example.com/empty\n\ngo 1.24.0\n") + o.ModulePath, o.Packages = empty, []string{"./..."} + if r := runTests(t.Context(), o); r.Status != "no_tests" { + t.Fatalf("empty module: %+v", r) + } + writeFile(t, filepath.Join(empty, "empty.go"), "package empty\n") + if r := runTests(t.Context(), o); r.Status != "no_tests" || len(r.Packages) != 1 { + t.Fatalf("package without tests: %+v", r) + } + writeFile(t, filepath.Join(empty, "empty_test.go"), "package empty\nimport (\"os\";\"testing\")\nfunc TestMain(m *testing.M) { os.Exit(2) }\n") + if r := runTests(t.Context(), o); r.Status != "failed" || r.Counts != (Counts{}) { + t.Fatalf("TestMain failure: %+v", r) + } + workspace := filepath.Join(t.TempDir(), "go.work") + writeFile(t, workspace, fmt.Sprintf("go 1.24.0\nuse (\n%q\n%q\n)\n", dir, empty)) + t.Setenv("GOWORK", workspace) + o.ModulePath, o.Packages = dir, []string{"example.com/empty"} + if r := runTests(t.Context(), o); r.Status != "error" || !strings.Contains(r.Error, "outside") { + t.Fatalf("other workspace module allowed: %+v", r) + } + t.Setenv("GOWORK", "off") + o.ModulePath = t.TempDir() + if r := runTests(t.Context(), o); r.Status != "error" { + t.Fatal(r) + } + o.ModulePath = dir + t.Setenv("PATH", t.TempDir()) + if r := runTests(t.Context(), o); r.Status != "error" || r.ExitCode != nil { + t.Fatalf("missing go: %+v", r) + } +} + +func TestPackageValidation(t *testing.T) { + root, _ := canonical(t.TempDir()) + outside, _ := canonical(t.TempDir()) + for _, p := range []listedPackage{ + {Dir: outside, ImportPath: "p", Module: &struct{ Dir string }{root}}, + {Dir: root, ImportPath: "p", Module: &struct{ Dir string }{outside}}, + {Dir: root, ImportPath: "-bad", Module: &struct{ Dir string }{root}}, + } { + b, _ := json.Marshal(p) + if _, err := validatePackages(b, root); err == nil { + t.Fatalf("accepted %+v", p) + } + } + link := filepath.Join(root, "link") + if err := os.Symlink(outside, link); err == nil { + p := listedPackage{Dir: link, ImportPath: "p", Module: &struct{ Dir string }{root}} + b, _ := json.Marshal(p) + if _, err := validatePackages(b, root); err == nil { + t.Fatal("symlink escaped module") + } + } +} + +// The helper's child appends a heartbeat, allowing cancellation tests to verify +// descendants are gone on every supported OS without platform-specific probes. +func TestProcessHelper(t *testing.T) { + mode := os.Getenv("TEST_RUNNER_HELPER") + if mode == "" { + return + } + if mode == "stderr" { + fmt.Fprint(os.Stderr, "setup failed") + os.Exit(2) + } + if mode == "child" { + for { + f, err := os.OpenFile(os.Getenv("TEST_RUNNER_HEARTBEAT"), os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0600) + if err != nil { + os.Exit(3) + } + _, _ = f.WriteString("x") + _ = f.Close() + time.Sleep(20 * time.Millisecond) + } + } + cmd := exec.Command(os.Args[0], "-test.run=^TestProcessHelper$") + cmd.Env = append(os.Environ(), "TEST_RUNNER_HELPER=child") + cmd.Stdout, cmd.Stderr = os.Stdout, os.Stderr + if err := cmd.Run(); err != nil { + os.Exit(4) + } + os.Exit(0) +} +func TestCancellationStopsDescendants(t *testing.T) { + for _, deadline := range []bool{false, true} { + t.Run(fmt.Sprint(deadline), func(t *testing.T) { + ctx, cancel := context.WithCancel(t.Context()) + if deadline { + ctx, cancel = context.WithTimeout(t.Context(), 2*time.Second) + } + defer cancel() + path := filepath.Join(t.TempDir(), "heartbeat") + cmd := exec.CommandContext(ctx, os.Args[0], "-test.run=^TestProcessHelper$") + cmd.Env = append(os.Environ(), "TEST_RUNNER_HELPER=parent", "TEST_RUNNER_HEARTBEAT="+path) + done := make(chan error, 1) + go func() { _, err := execute(cmd, io.Discard, io.Discard); done <- err }() + until := time.Now().Add(5 * time.Second) + for { + if st, err := os.Stat(path); err == nil && st.Size() > 0 { + break + } + if time.Now().After(until) { + t.Fatal("no child heartbeat") + } + time.Sleep(20 * time.Millisecond) + } + if !deadline { + cancel() + } + select { + case err := <-done: + if err == nil { + t.Fatal("expected cancellation") + } + case <-time.After(8 * time.Second): + t.Fatal("process did not stop") + } + before, _ := os.ReadFile(path) + time.Sleep(150 * time.Millisecond) + after, _ := os.ReadFile(path) + if len(before) != len(after) { + t.Fatal("child survived cancellation") + } + }) + } +} +func TestStderrAndCancelledContext(t *testing.T) { + cmd := exec.CommandContext(t.Context(), os.Args[0], "-test.run=^TestProcessHelper$") + cmd.Env = append(os.Environ(), "TEST_RUNNER_HELPER=stderr") + errout := &cappedBuffer{limit: 100} + code, err := execute(cmd, io.Discard, errout) + if err == nil || code == nil || *code != 2 || string(errout.data) != "setup failed" { + t.Fatalf("%v %v %s", code, err, errout.data) + } + ctx, cancel := context.WithCancel(t.Context()) + cancel() + r := runTests(ctx, Options{ModulePath: t.TempDir(), Timeout: 60}) + if r.Status != "cancelled" || !r.Incomplete { + t.Fatal(r) + } +} + +func TestMCPServerProcess(t *testing.T) { + if os.Getenv("TEST_RUNNER_MCP") != "1" { + return + } + main() + os.Exit(0) +} +func TestMCPStdio(t *testing.T) { + offlineGo(t) + dir := moduleFixture(t) + binary, args := os.Args[0], []string{"-test.run=^TestMCPServerProcess$"} + if distributed := os.Getenv("TEST_RUNNER_BINARY"); distributed != "" { + binary, args = distributed, nil + } + // Installation smoke tests may supply the copied dispatcher/native binary. + // All variants start outside both the plugin and the module under test. + t.Chdir(t.TempDir()) + c, err := client.NewStdioMCPClient(binary, append(os.Environ(), "TEST_RUNNER_MCP=1"), args...) + if err != nil { + t.Fatal(err) + } + defer c.Close() + ctx, cancel := context.WithTimeout(t.Context(), 90*time.Second) + defer cancel() + init := mcp.InitializeRequest{} + init.Params.ProtocolVersion = mcp.LATEST_PROTOCOL_VERSION + init.Params.ClientInfo = mcp.Implementation{Name: "test", Version: "1"} + if _, err = c.Initialize(ctx, init); err != nil { + t.Fatal(err) + } + list, err := c.ListTools(ctx, mcp.ListToolsRequest{}) + if err != nil || len(list.Tools) != 1 || list.Tools[0].Name != "go_test" { + t.Fatalf("%+v %v", list, err) + } + for _, tt := range []struct { + args map[string]any + status string + isError bool + }{ + {map[string]any{"module_path": dir, "run": "^TestPass$"}, "passed", false}, + {map[string]any{"module_path": dir, "run": "^TestFail$"}, "failed", false}, + {map[string]any{"module_path": dir, "timeout_seconds": "60"}, "error", true}, + } { + req := mcp.CallToolRequest{} + req.Params.Name = "go_test" + req.Params.Arguments = tt.args + res, err := c.CallTool(ctx, req) + if err != nil { + t.Fatal(err) + } + var r Result + if err = json.Unmarshal([]byte(res.Content[0].(mcp.TextContent).Text), &r); err != nil { + t.Fatal(err) + } + if r.Status != tt.status || res.IsError != tt.isError { + t.Fatalf("%+v isError=%v", r, res.IsError) + } + } + marker := filepath.Join(t.TempDir(), "test-started") + writeFile(t, filepath.Join(dir, "slow_test.go"), fmt.Sprintf(`package fixture +import ("os"; "testing"; "time") +func TestSlow(t *testing.T) { if err := os.WriteFile(%q, []byte("started"), 0600); err != nil { t.Fatal(err) }; time.Sleep(time.Minute) } +`, marker)) + // Use an explicit request ID so the test can send the real MCP cancellation + // notification while the tool is running, not just cancel its local waiter. + done := make(chan *transport.JSONRPCResponse, 1) + errors := make(chan error, 1) + go func() { + res, err := c.GetTransport().SendRequest(ctx, transport.JSONRPCRequest{ + JSONRPC: "2.0", ID: mcp.NewRequestId(int64(777)), Method: "tools/call", + Params: map[string]any{"name": "go_test", "arguments": map[string]any{"module_path": dir, "run": "^TestSlow$"}}, + }) + if err != nil { + errors <- err + return + } + done <- res + }() + until := time.Now().Add(30 * time.Second) + for { + if _, err := os.Stat(marker); err == nil { + break + } + if time.Now().After(until) { + t.Fatal("MCP test did not start") + } + time.Sleep(20 * time.Millisecond) + } + err = c.GetTransport().SendNotification(ctx, mcp.JSONRPCNotification{ + JSONRPC: "2.0", Notification: mcp.Notification{Method: "notifications/cancelled", + Params: mcp.NotificationParams{AdditionalFields: map[string]any{"requestId": 777}}}, + }) + if err != nil { + t.Fatal(err) + } + select { + case raw := <-done: + res, err := mcp.ParseCallToolResult(&raw.Result) + if err != nil { + t.Fatal(err) + } + var r Result + if err := json.Unmarshal([]byte(res.Content[0].(mcp.TextContent).Text), &r); err != nil { + t.Fatal(err) + } + if r.Status != "cancelled" || !r.Incomplete || !res.IsError { + t.Fatalf("cancellation: %+v", r) + } + case err := <-errors: + t.Fatal(err) + case <-time.After(10 * time.Second): + t.Fatal("MCP cancellation did not reach the running test") + } + r := runTests(ctx, Options{ModulePath: dir, Packages: []string{"."}, Run: "^TestSlow$", Timeout: 1}) + if r.Status != "timeout" || !r.Incomplete { + t.Fatalf("timeout: %+v", r) + } +} diff --git a/test-runner/src/process.go b/test-runner/src/process.go new file mode 100644 index 0000000..c434418 --- /dev/null +++ b/test-runner/src/process.go @@ -0,0 +1,50 @@ +package main + +import ( + "errors" + "fmt" + "io" + "os" + "os/exec" + "time" +) + +// cappedBuffer drains the pipe even after its retention limit is reached. +type cappedBuffer struct { + data []byte + limit int + truncated bool +} + +func (b *cappedBuffer) Write(p []byte) (int, error) { + n := len(p) + keep := min(n, max(0, b.limit-len(b.data))) + b.data = append(b.data, p[:keep]...) + b.truncated = b.truncated || keep < n + return n, nil +} + +func execute(cmd *exec.Cmd, out, stderr io.Writer) (*int, error) { + cmd.Stdout, cmd.Stderr = out, stderr + cmd.WaitDelay = 2 * time.Second + var stopErr error + configureProcess(cmd) + cmd.Cancel = func() error { + err := stopProcess(cmd) + if err != nil && !errors.Is(err, os.ErrProcessDone) { + stopErr = err + } + return err + } + err := cmd.Run() + // Cmd.Wait joins its cancellation goroutine before returning. + if stopErr != nil { + err = fmt.Errorf("%w; process-tree cleanup failed: %v", err, stopErr) + } + var code *int + if cmd.ProcessState != nil { + n := cmd.ProcessState.ExitCode() + code = &n + } + return code, err +} diff --git a/test-runner/src/process_unix.go b/test-runner/src/process_unix.go new file mode 100644 index 0000000..fcbed41 --- /dev/null +++ b/test-runner/src/process_unix.go @@ -0,0 +1,19 @@ +//go:build linux || darwin + +package main + +import ( + "errors" + "os" + "os/exec" + "syscall" +) + +func configureProcess(cmd *exec.Cmd) { cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true} } +func stopProcess(cmd *exec.Cmd) error { + err := syscall.Kill(-cmd.Process.Pid, syscall.SIGKILL) + if errors.Is(err, syscall.ESRCH) { + return os.ErrProcessDone + } + return err +} diff --git a/test-runner/src/process_windows.go b/test-runner/src/process_windows.go new file mode 100644 index 0000000..974c9f6 --- /dev/null +++ b/test-runner/src/process_windows.go @@ -0,0 +1,25 @@ +package main + +import ( + "context" + "fmt" + "os" + "os/exec" + "path/filepath" + "strconv" + "time" +) + +func configureProcess(cmd *exec.Cmd) {} +func stopProcess(cmd *exec.Cmd) error { + ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) + defer cancel() + // Resolve the OS utility directly, rather than a taskkill in the project PATH. + kill := exec.CommandContext(ctx, filepath.Join(os.Getenv("SystemRoot"), "System32", "taskkill.exe"), "/PID", strconv.Itoa(cmd.Process.Pid), "/T", "/F") + kill.WaitDelay = time.Second + if out, err := kill.CombinedOutput(); err != nil { + _ = cmd.Process.Kill() + return fmt.Errorf("taskkill: %w: %s", err, out) + } + return nil +} diff --git a/test-runner/src/runner.go b/test-runner/src/runner.go new file mode 100644 index 0000000..1bcf2a6 --- /dev/null +++ b/test-runner/src/runner.go @@ -0,0 +1,177 @@ +package main + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "os" + "os/exec" + "path/filepath" + "runtime" + "strings" + "time" +) + +type listedPackage struct { + Dir string + ImportPath string + Module *struct{ Dir string } + Error *struct{ Err string } +} + +func samePath(a, b string) bool { + if runtime.GOOS == "windows" { + return strings.EqualFold(a, b) + } + return a == b +} +func canonical(path string) (string, error) { + p, err := filepath.Abs(path) + if err != nil { + return "", err + } + return filepath.EvalSymlinks(p) +} +func validatePackages(data []byte, root string) ([]string, error) { + d := json.NewDecoder(bytes.NewReader(data)) + var packages []string + seen := map[string]bool{} + for { + var p listedPackage + if err := d.Decode(&p); err != nil { + if errors.Is(err, io.EOF) { + break + } + return nil, fmt.Errorf("invalid go list output: %w", err) + } + if p.Module == nil || p.Dir == "" { + if p.Error != nil { + return nil, fmt.Errorf("package discovery: %s", p.Error.Err) + } + return nil, fmt.Errorf("package %q is not in the selected module", p.ImportPath) + } + moduleDir, err := canonical(p.Module.Dir) + if err != nil || !samePath(moduleDir, root) { + return nil, fmt.Errorf("package %q is outside the selected module", p.ImportPath) + } + dir, err := canonical(p.Dir) + if err != nil { + return nil, err + } + rel, err := filepath.Rel(root, dir) + if err != nil || rel == ".." || strings.HasPrefix(rel, ".."+string(filepath.Separator)) || filepath.IsAbs(rel) { + return nil, fmt.Errorf("package %q resolves outside the module directory", p.ImportPath) + } + if p.ImportPath == "" || strings.HasPrefix(p.ImportPath, "-") || strings.ContainsAny(p.ImportPath, "\x00\r\n") { + return nil, fmt.Errorf("invalid resolved package name %q", p.ImportPath) + } + if !seen[p.ImportPath] { + packages = append(packages, p.ImportPath) + seen[p.ImportPath] = true + } + } + return packages, nil +} + +func runTests(parent context.Context, o Options) (r Result) { + start := time.Now() + ctx, cancel := context.WithTimeout(parent, time.Duration(o.Timeout)*time.Second) + defer cancel() + defer func() { + r.Elapsed = time.Since(start).Seconds() + if ctx.Err() != nil { + r.Status, r.Incomplete = "cancelled", true + if errors.Is(ctx.Err(), context.DeadlineExceeded) { + r.Status = "timeout" + } + if r.Error == "" { + r.Error = ctx.Err().Error() + } + } + }() + r.Status = "error" + root, err := canonical(o.ModulePath) + if err != nil { + r.Error = err.Error() + return + } + if st, err := os.Stat(filepath.Join(root, "go.mod")); err != nil || !st.Mode().IsRegular() { + r.Error = "module_path must contain a regular go.mod file" + return + } + o.ModulePath = root + goBinary, err := exec.LookPath("go") + if err != nil { + r.Error = err.Error() + return + } + // An absolute executable remains valid after cmd.Dir changes. + goBinary, err = filepath.Abs(goBinary) + if err != nil { + r.Error = err.Error() + return + } + args := append([]string{"list", "-e", "-json=Dir,ImportPath,Module,Error", "--"}, o.Packages...) + cmd := exec.CommandContext(ctx, goBinary, args...) + cmd.Dir = root + listed := &cappedBuffer{limit: 8 << 20} + stderr := &cappedBuffer{limit: returnedLogLimit} + code, err := execute(cmd, listed, stderr) + var truncated bool + r.Stderr, truncated = boundedLog(stderr.data, returnedLogLimit) + r.LogsTruncated = stderr.truncated || truncated + if err != nil || listed.truncated { + r.ExitCode = code + r.Error = fmt.Sprintf("package discovery failed: %v", err) + if listed.truncated { + r.Error = "package discovery exceeds 8 MiB" + } + return + } + packages, err := validatePackages(listed.data, root) + if err != nil { + r.Error = err.Error() + return + } + if len(packages) == 0 { + r.Status, r.ExitCode = "no_tests", code + return + } + args = []string{"test", "-json", "-count=1", fmt.Sprintf("-timeout=%ds", o.Timeout)} + if o.Run != "" { + args = append(args, "-run="+o.Run) + } + args = append(args, packages...) + cmd = exec.CommandContext(ctx, goBinary, args...) + cmd.Dir = root + stream := newEventStream() + code, err = execute(cmd, stream, stderr) + stream.finish() + r = stream.result(o, stderr) + r.ExitCode = code + if err != nil { + var exit *exec.ExitError + // Test/build failures are normal tool results. Infrastructure errors and + // unexplained nonzero exits are tool errors, even with partial events. + if !errors.As(err, &exit) || r.Status != "failed" { + r.Status = "error" + } + if r.Status == "error" || ctx.Err() != nil { + if r.Error == "" { + r.Error = err.Error() + } else { + r.Error += "; " + err.Error() + } + } + } + if r.Incomplete && r.Status != "failed" && r.Error == "" { + r.Status, r.Error = "error", "test output ended before terminal events" + } + if len(stream.packages) == 0 && r.Status == "no_tests" { + r.Status, r.Error, r.Incomplete = "error", "go test returned no package events", true + } + return +}