ci: self-discovering per-tool test matrix + run-tests.sh harness

.github/workflows/ci.yml

@@ -0,0 +1,66 @@
+name: ci
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+
+permissions:
+  contents: read
+
+jobs:
+  # Discover every tool that ships a test suite, so the matrix is self-maintaining:
+  # add a tool + its test_<tool>.sh and CI picks it up with no workflow edit.
+  discover:
+    runs-on: ubuntu-latest
+    outputs:
+      tools: ${{ steps.list.outputs.tools }}
+    steps:
+      - uses: actions/checkout@v4
+      - id: list
+        run: |
+          tools=$(for d in */; do
+            d="${d%/}"
+            [ "$d" = "scripts" ] && continue
+            if compgen -G "$d/test_*.sh" >/dev/null || compgen -G "$d/test-*.sh" >/dev/null; then
+              printf '%s\n' "$d"
+            fi
+          done | sort | jq -R . | jq -cs .)
+          echo "tools=$tools" >> "$GITHUB_OUTPUT"
+          echo "Discovered tools: $tools"
+
+  # One job per tool — a green check per tool inside one shared workflow.
+  test:
+    needs: discover
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        tool: ${{ fromJson(needs.discover.outputs.tools) }}
+    name: test ${{ matrix.tool }}
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: '3.x'
+      - name: Configure git identity (a few suites build temp repos)
+        run: |
+          git config --global user.email "ci@agent-ops.local"
+          git config --global user.name "agent-ops ci"
+          git config --global init.defaultBranch main
+      - name: Test ${{ matrix.tool }}
+        run: bash scripts/run-tests.sh "${{ matrix.tool }}"
+
+  # Single required status: green only if every tool's job passed.
+  ci-ok:
+    needs: test
+    if: always()
+    runs-on: ubuntu-latest
+    steps:
+      - name: Gate on the matrix result
+        run: |
+          if [ "${{ needs.test.result }}" != "success" ]; then
+            echo "One or more tool suites failed."
+            exit 1
+          fi
+          echo "All tool suites passed ✅"

scripts/run-tests.sh

@@ -0,0 +1,69 @@
+#!/usr/bin/env bash
+# Standardized test harness for the agent-ops fleet.
+#
+#   scripts/run-tests.sh              # run every tool's test suite, print a summary
+#   scripts/run-tests.sh agent-frisk  # run just one tool's suite
+#
+# Contract (every tool follows it): a tool lives in its own top-level directory and
+# ships a bash test script named test_<tool>.sh (or test-<tool>.sh). The script tallies
+# its own checks and EXITS NON-ZERO if any check failed. This harness simply discovers,
+# runs, and aggregates them — pass/fail is the script's exit code, nothing is parsed.
+set -uo pipefail
+
+ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
+cd "$ROOT"
+
+# Find the test script inside a tool directory, tolerant of the test_ / test- split.
+find_test() {
+  local dir="$1" t
+  for t in "$dir"/test_*.sh "$dir"/test-*.sh; do
+    [ -f "$t" ] && { printf '%s\n' "$t"; return 0; }
+  done
+  return 1
+}
+
+# Every top-level directory that ships a test script is a tool.
+list_tools() {
+  local dir
+  for dir in */; do
+    dir="${dir%/}"
+    [ "$dir" = ".git" ] || [ "$dir" = "scripts" ] && continue
+    find_test "$dir" >/dev/null 2>&1 && printf '%s\n' "$dir"
+  done
+}
+
+run_one() {
+  local tool="$1" test_script
+  if ! test_script="$(find_test "$tool")"; then
+    echo "::ERROR:: $tool — no test script found"
+    return 2
+  fi
+  echo "── $tool ($(basename "$test_script")) ──────────────────────────────"
+  ( cd "$tool" && bash "$(basename "$test_script")" )
+}
+
+main() {
+  if [ "$#" -ge 1 ]; then
+    run_one "$1"
+    exit $?
+  fi
+
+  local failed=() tool rc total=0
+  while IFS= read -r tool; do
+    total=$((total + 1))
+    if ! run_one "$tool"; then
+      failed+=("$tool")
+    fi
+    echo
+  done < <(list_tools)
+
+  echo "==================== SUMMARY ===================="
+  echo "tools tested: $total   passed: $((total - ${#failed[@]}))   failed: ${#failed[@]}"
+  if [ "${#failed[@]}" -gt 0 ]; then
+    printf 'FAILED: %s\n' "${failed[*]}"
+    exit 1
+  fi
+  echo "all green ✅"
+}
+
+main "$@"