bench: compare RepoDoctor with codebase-health-score

.github/workflows/benchmark-competitive.yml

@@ -0,0 +1,157 @@
+name: Competitive benchmark
+
+on:
+  pull_request:
+    paths:
+      - 'action.yml'
+      - 'README.md'
+      - '.github/workflows/benchmark-competitive.yml'
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+jobs:
+  repository-health:
+    name: RepoDoctor vs codebase-health-score
+    runs-on: ubuntu-24.04
+    timeout-minutes: 15
+    steps:
+      - name: Checkout
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
+
+      - name: Install benchmark tools
+        run: |
+          sudo apt-get update
+          sudo apt-get install --yes hyperfine time python3-venv curl git
+
+      - name: Install verified RepoDoctor v0.1.1 archive
+        shell: bash
+        run: |
+          set -euo pipefail
+          rm -rf /tmp/repodoctor-v011
+          mkdir -p /tmp/repodoctor-v011
+          curl -fL https://github.com/BLCCoreStudio/RepoDoctor/releases/download/v0.1.1/RepoDoctor-v0.1.1-linux-x86_64.tar.gz -o /tmp/repodoctor-v0.1.1.tar.gz
+          echo '9861e7b759e35da7a6fb913b7c6ee665f3e69b783129733c4d35d465dd1ec064  /tmp/repodoctor-v0.1.1.tar.gz' | sha256sum -c -
+          tar -xzf /tmp/repodoctor-v0.1.1.tar.gz -C /tmp/repodoctor-v011
+          REPODOCTOR_BIN='/tmp/repodoctor-v011/RepoDoctor-v0.1.1-linux-x86_64/repodoctor'
+          test -x "$REPODOCTOR_BIN"
+          "$REPODOCTOR_BIN" version
+          echo "REPODOCTOR_BIN=$REPODOCTOR_BIN" >> "$GITHUB_ENV"
+
+      - name: Install pinned codebase-health-score competitor
+        shell: bash
+        run: |
+          set -euo pipefail
+          python3 -m venv /tmp/codebase-health-venv
+          /tmp/codebase-health-venv/bin/pip install --quiet 'git+https://github.com/glue-tools-ai/codebase-health-score.git@a400f07001349a18a888c72a7c1bf60dbb4f41a8'
+          /tmp/codebase-health-venv/bin/codebase-health --help >/dev/null
+
+      - name: Create shared repository fixture
+        shell: bash
+        run: |
+          set -euo pipefail
+          rm -rf /tmp/repo-health-fixture
+          mkdir -p /tmp/repo-health-fixture/src /tmp/repo-health-fixture/tests /tmp/repo-health-fixture/.github/workflows
+          cd /tmp/repo-health-fixture
+          git init -q
+          git config user.name benchmark
+          git config user.email benchmark@example.invalid
+          cat > README.md <<'EOF'
+          # Benchmark Fixture
+          Synthetic Python repository used only for repository-health benchmarking.
+          EOF
+          cat > pyproject.toml <<'EOF'
+          [project]
+          name = "benchmark-fixture"
+          version = "0.1.0"
+          requires-python = ">=3.11"
+          EOF
+          cat > .gitignore <<'EOF'
+          __pycache__/
+          .venv/
+          EOF
+          cat > .github/workflows/ci.yml <<'EOF'
+          name: CI
+          on: [push]
+          jobs:
+            test:
+              runs-on: ubuntu-latest
+              steps:
+                - run: echo test
+          EOF
+          for i in $(seq -w 1 500); do
+            cat > "src/module_$i.py" <<EOF
+          def function_$i(value: int) -> int:
+              return value + $((10#$i))
+          EOF
+          done
+          for i in $(seq -w 1 100); do
+            printf 'def test_%s():\n    assert True\n' "$i" > "tests/test_$i.py"
+          done
+          git add .
+          git commit -qm 'benchmark fixture'
+
+      - name: Verify same task class
+        shell: bash
+        run: |
+          set -euo pipefail
+          mkdir -p benchmark-results
+          "$REPODOCTOR_BIN" scan /tmp/repo-health-fixture --format json >benchmark-results/repodoctor-output.json
+          /tmp/codebase-health-venv/bin/codebase-health analyze /tmp/repo-health-fixture >benchmark-results/codebase-health-output.txt
+          python3 - <<'PY'
+          import json
+          report=json.load(open('benchmark-results/repodoctor-output.json'))
+          text=open('benchmark-results/codebase-health-output.txt').read().lower()
+          assert report
+          assert 'score' in json.dumps(report).lower()
+          assert 'score' in text or '/100' in text
+          print('Both tools produced repository-health scoring output for the same fixture.')
+          PY
+
+      - name: Benchmark repository health scan
+        shell: bash
+        run: |
+          set -euo pipefail
+          {
+            echo "workload=repository-health analysis of the same synthetic Python repo (500 source + 100 test files)"
+            echo "repodoctor=0.1.1 extracted verified release archive"
+            echo "competitor=codebase-health-score@a400f07001349a18a888c72a7c1bf60dbb4f41a8"
+            echo "runner=ubuntu-24.04"
+            echo "kernel=$(uname -srmo)"
+            echo "cpu=$(lscpu | awk -F: '/Model name/{gsub(/^[ \t]+/,"",$2); print $2; exit}')"
+            echo "hyperfine=$(hyperfine --version | head -n1)"
+          } > benchmark-results/environment.txt
+
+          hyperfine --warmup 3 --runs 15 \
+            --export-json benchmark-results/repo-health.json \
+            --export-markdown benchmark-results/repo-health.md \
+            --command-name 'RepoDoctor 0.1.1' "$REPODOCTOR_BIN scan /tmp/repo-health-fixture --format json >/dev/null" \
+            --command-name 'codebase-health-score 1.0.0' '/tmp/codebase-health-venv/bin/codebase-health analyze /tmp/repo-health-fixture >/dev/null'
+
+          /usr/bin/time -v "$REPODOCTOR_BIN" scan /tmp/repo-health-fixture --format json >/dev/null 2>benchmark-results/repodoctor-time-v.txt
+          /usr/bin/time -v /tmp/codebase-health-venv/bin/codebase-health analyze /tmp/repo-health-fixture >/dev/null 2>benchmark-results/codebase-health-time-v.txt
+          cat benchmark-results/repo-health.md
+
+      - name: Publish summary
+        if: always()
+        run: |
+          if [[ -f benchmark-results/environment.txt ]]; then
+            echo '## Environment' >> "$GITHUB_STEP_SUMMARY"
+            echo '```text' >> "$GITHUB_STEP_SUMMARY"
+            cat benchmark-results/environment.txt >> "$GITHUB_STEP_SUMMARY"
+            echo '```' >> "$GITHUB_STEP_SUMMARY"
+          fi
+          if [[ -f benchmark-results/repo-health.md ]]; then
+            echo '## Repository-health scan' >> "$GITHUB_STEP_SUMMARY"
+            cat benchmark-results/repo-health.md >> "$GITHUB_STEP_SUMMARY"
+          fi
+
+      - name: Upload raw evidence
+        if: always()
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
+        with:
+          name: repodoctor-competitive-benchmark-${{ github.sha }}
+          path: benchmark-results/
+          if-no-files-found: warn
+          retention-days: 90