From 90bee0594b306e6d8aecfee47f2cc1af215e50e3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?BLC=20Core=20Studio=20=E2=80=94=20Developer=20Tools?= <207100624+BLCCoreStudio@users.noreply.github.com> Date: Thu, 3 Sep 2026 05:16:28 +0300 Subject: [PATCH 1/3] bench: add repository-health competitive benchmark --- .github/workflows/benchmark-competitive.yml | 156 ++++++++++++++++++++ 1 file changed, 156 insertions(+) create mode 100644 .github/workflows/benchmark-competitive.yml diff --git a/.github/workflows/benchmark-competitive.yml b/.github/workflows/benchmark-competitive.yml new file mode 100644 index 0000000..b13e512 --- /dev/null +++ b/.github/workflows/benchmark-competitive.yml @@ -0,0 +1,156 @@ +name: Competitive benchmark + +on: + pull_request: + paths: + - 'action.yml' + - 'README.md' + - '.github/workflows/benchmark-competitive.yml' + workflow_dispatch: + +permissions: + contents: read + +jobs: + repository-health: + name: RepoDoctor vs codebase-health-score + runs-on: ubuntu-24.04 + timeout-minutes: 15 + steps: + - name: Checkout + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + + - name: Install benchmark tools + run: | + sudo apt-get update + sudo apt-get install --yes hyperfine time python3-venv curl git + + - name: Install verified RepoDoctor v0.1.1 release + shell: bash + run: | + set -euo pipefail + cd /tmp + curl -fL https://github.com/BLCCoreStudio/RepoDoctor/releases/download/v0.1.1/repodoctor-linux-x86_64 -o repodoctor-linux-x86_64 + curl -fL https://github.com/BLCCoreStudio/RepoDoctor/releases/download/v0.1.1/repodoctor-linux-x86_64.sha256 -o repodoctor-linux-x86_64.sha256 + sha256sum -c repodoctor-linux-x86_64.sha256 + chmod +x repodoctor-linux-x86_64 + ./repodoctor-linux-x86_64 --version + + - name: Install pinned codebase-health-score competitor + shell: bash + run: | + set -euo pipefail + python3 -m venv /tmp/codebase-health-venv + /tmp/codebase-health-venv/bin/pip install --quiet 'git+https://github.com/glue-tools-ai/codebase-health-score.git@a400f07001349a18a888c72a7c1bf60dbb4f41a8' + /tmp/codebase-health-venv/bin/codebase-health --help >/dev/null + + - name: Create shared repository fixture + shell: bash + run: | + set -euo pipefail + rm -rf /tmp/repo-health-fixture + mkdir -p /tmp/repo-health-fixture/src /tmp/repo-health-fixture/tests /tmp/repo-health-fixture/.github/workflows + cd /tmp/repo-health-fixture + git init -q + git config user.name benchmark + git config user.email benchmark@example.invalid + cat > README.md <<'EOF' + # Benchmark Fixture + Synthetic Python repository used only for repository-health benchmarking. + EOF + cat > pyproject.toml <<'EOF' + [project] + name = "benchmark-fixture" + version = "0.1.0" + requires-python = ">=3.11" + EOF + cat > .gitignore <<'EOF' + __pycache__/ + .venv/ + EOF + cat > .github/workflows/ci.yml <<'EOF' + name: CI + on: [push] + jobs: + test: + runs-on: ubuntu-latest + steps: + - run: echo test + EOF + for i in $(seq -w 1 500); do + cat > "src/module_$i.py" < int: + return value + $((10#$i)) + EOF + done + for i in $(seq -w 1 100); do + printf 'def test_%s():\n assert True\n' "$i" > "tests/test_$i.py" + done + git add . + git commit -qm 'benchmark fixture' + + - name: Verify same task class + shell: bash + run: | + set -euo pipefail + /tmp/repodoctor-linux-x86_64 scan /tmp/repo-health-fixture --format json >/tmp/repodoctor.json + /tmp/codebase-health-venv/bin/codebase-health analyze /tmp/repo-health-fixture >/tmp/codebase-health.txt + python3 - <<'PY' + import json + report=json.load(open('/tmp/repodoctor.json')) + text=open('/tmp/codebase-health.txt').read().lower() + assert report + assert 'score' in json.dumps(report).lower() + assert 'score' in text or '/100' in text + print('Both tools produced repository-health scoring output for the same fixture.') + PY + + - name: Benchmark repository health scan + shell: bash + run: | + set -euo pipefail + mkdir -p benchmark-results + { + echo "workload=repository-health analysis of the same synthetic Python repo (500 source + 100 test files)" + echo "repodoctor=0.1.1" + echo "competitor=codebase-health-score@a400f07001349a18a888c72a7c1bf60dbb4f41a8" + echo "runner=ubuntu-24.04" + echo "kernel=$(uname -srmo)" + echo "cpu=$(lscpu | awk -F: '/Model name/{gsub(/^[ \t]+/,"",$2); print $2; exit}')" + echo "hyperfine=$(hyperfine --version | head -n1)" + } > benchmark-results/environment.txt + + hyperfine --warmup 3 --runs 15 \ + --export-json benchmark-results/repo-health.json \ + --export-markdown benchmark-results/repo-health.md \ + --command-name 'RepoDoctor 0.1.1' '/tmp/repodoctor-linux-x86_64 scan /tmp/repo-health-fixture --format json >/dev/null' \ + --command-name 'codebase-health-score 1.0.0' '/tmp/codebase-health-venv/bin/codebase-health analyze /tmp/repo-health-fixture >/dev/null' + + /usr/bin/time -v /tmp/repodoctor-linux-x86_64 scan /tmp/repo-health-fixture --format json >/dev/null 2>benchmark-results/repodoctor-time-v.txt + /usr/bin/time -v /tmp/codebase-health-venv/bin/codebase-health analyze /tmp/repo-health-fixture >/dev/null 2>benchmark-results/codebase-health-time-v.txt + cp /tmp/repodoctor.json benchmark-results/repodoctor-output.json + cp /tmp/codebase-health.txt benchmark-results/codebase-health-output.txt + cat benchmark-results/repo-health.md + + - name: Publish summary + if: always() + run: | + if [[ -f benchmark-results/environment.txt ]]; then + echo '## Environment' >> "$GITHUB_STEP_SUMMARY" + echo '```text' >> "$GITHUB_STEP_SUMMARY" + cat benchmark-results/environment.txt >> "$GITHUB_STEP_SUMMARY" + echo '```' >> "$GITHUB_STEP_SUMMARY" + fi + if [[ -f benchmark-results/repo-health.md ]]; then + echo '## Repository-health scan' >> "$GITHUB_STEP_SUMMARY" + cat benchmark-results/repo-health.md >> "$GITHUB_STEP_SUMMARY" + fi + + - name: Upload raw evidence + if: always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: repodoctor-competitive-benchmark-${{ github.sha }} + path: benchmark-results/ + if-no-files-found: error + retention-days: 90 From 6308797b4a4872c249bf792f34c8dc39e12444e0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?BLC=20Core=20Studio=20=E2=80=94=20Developer=20Tools?= <207100624+BLCCoreStudio@users.noreply.github.com> Date: Thu, 3 Sep 2026 05:20:00 +0300 Subject: [PATCH 2/3] bench: measure extracted RepoDoctor engine fairly --- .github/workflows/benchmark-competitive.yml | 37 +++++++++++---------- 1 file changed, 19 insertions(+), 18 deletions(-) diff --git a/.github/workflows/benchmark-competitive.yml b/.github/workflows/benchmark-competitive.yml index b13e512..dbf701a 100644 --- a/.github/workflows/benchmark-competitive.yml +++ b/.github/workflows/benchmark-competitive.yml @@ -25,16 +25,19 @@ jobs: sudo apt-get update sudo apt-get install --yes hyperfine time python3-venv curl git - - name: Install verified RepoDoctor v0.1.1 release + - name: Install verified RepoDoctor v0.1.1 archive shell: bash run: | set -euo pipefail - cd /tmp - curl -fL https://github.com/BLCCoreStudio/RepoDoctor/releases/download/v0.1.1/repodoctor-linux-x86_64 -o repodoctor-linux-x86_64 - curl -fL https://github.com/BLCCoreStudio/RepoDoctor/releases/download/v0.1.1/repodoctor-linux-x86_64.sha256 -o repodoctor-linux-x86_64.sha256 - sha256sum -c repodoctor-linux-x86_64.sha256 - chmod +x repodoctor-linux-x86_64 - ./repodoctor-linux-x86_64 --version + rm -rf /tmp/repodoctor-v011 + mkdir -p /tmp/repodoctor-v011 + curl -fL https://github.com/BLCCoreStudio/RepoDoctor/releases/download/v0.1.1/RepoDoctor-v0.1.1-linux-x86_64.tar.gz -o /tmp/repodoctor-v0.1.1.tar.gz + echo '9861e7b759e35da7a6fb913b7c6ee665f3e69b783129733c4d35d465dd1ec064 /tmp/repodoctor-v0.1.1.tar.gz' | sha256sum -c - + tar -xzf /tmp/repodoctor-v0.1.1.tar.gz -C /tmp/repodoctor-v011 + repodoctor_bin="$(find /tmp/repodoctor-v011 -maxdepth 3 -type f -name repodoctor -perm -u+x -print -quit)" + test -n "$repodoctor_bin" + cp "$repodoctor_bin" /tmp/repodoctor + /tmp/repodoctor version - name: Install pinned codebase-health-score competitor shell: bash @@ -93,12 +96,13 @@ jobs: shell: bash run: | set -euo pipefail - /tmp/repodoctor-linux-x86_64 scan /tmp/repo-health-fixture --format json >/tmp/repodoctor.json - /tmp/codebase-health-venv/bin/codebase-health analyze /tmp/repo-health-fixture >/tmp/codebase-health.txt + mkdir -p benchmark-results + /tmp/repodoctor scan /tmp/repo-health-fixture --format json >benchmark-results/repodoctor-output.json + /tmp/codebase-health-venv/bin/codebase-health analyze /tmp/repo-health-fixture >benchmark-results/codebase-health-output.txt python3 - <<'PY' import json - report=json.load(open('/tmp/repodoctor.json')) - text=open('/tmp/codebase-health.txt').read().lower() + report=json.load(open('benchmark-results/repodoctor-output.json')) + text=open('benchmark-results/codebase-health-output.txt').read().lower() assert report assert 'score' in json.dumps(report).lower() assert 'score' in text or '/100' in text @@ -109,10 +113,9 @@ jobs: shell: bash run: | set -euo pipefail - mkdir -p benchmark-results { echo "workload=repository-health analysis of the same synthetic Python repo (500 source + 100 test files)" - echo "repodoctor=0.1.1" + echo "repodoctor=0.1.1 extracted verified release archive" echo "competitor=codebase-health-score@a400f07001349a18a888c72a7c1bf60dbb4f41a8" echo "runner=ubuntu-24.04" echo "kernel=$(uname -srmo)" @@ -123,13 +126,11 @@ jobs: hyperfine --warmup 3 --runs 15 \ --export-json benchmark-results/repo-health.json \ --export-markdown benchmark-results/repo-health.md \ - --command-name 'RepoDoctor 0.1.1' '/tmp/repodoctor-linux-x86_64 scan /tmp/repo-health-fixture --format json >/dev/null' \ + --command-name 'RepoDoctor 0.1.1' '/tmp/repodoctor scan /tmp/repo-health-fixture --format json >/dev/null' \ --command-name 'codebase-health-score 1.0.0' '/tmp/codebase-health-venv/bin/codebase-health analyze /tmp/repo-health-fixture >/dev/null' - /usr/bin/time -v /tmp/repodoctor-linux-x86_64 scan /tmp/repo-health-fixture --format json >/dev/null 2>benchmark-results/repodoctor-time-v.txt + /usr/bin/time -v /tmp/repodoctor scan /tmp/repo-health-fixture --format json >/dev/null 2>benchmark-results/repodoctor-time-v.txt /usr/bin/time -v /tmp/codebase-health-venv/bin/codebase-health analyze /tmp/repo-health-fixture >/dev/null 2>benchmark-results/codebase-health-time-v.txt - cp /tmp/repodoctor.json benchmark-results/repodoctor-output.json - cp /tmp/codebase-health.txt benchmark-results/codebase-health-output.txt cat benchmark-results/repo-health.md - name: Publish summary @@ -152,5 +153,5 @@ jobs: with: name: repodoctor-competitive-benchmark-${{ github.sha }} path: benchmark-results/ - if-no-files-found: error + if-no-files-found: warn retention-days: 90 From 59087a51b451cfa1e4a2935919c58fc27af128f3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?BLC=20Core=20Studio=20=E2=80=94=20Developer=20Tools?= <207100624+BLCCoreStudio@users.noreply.github.com> Date: Thu, 3 Sep 2026 05:23:04 +0300 Subject: [PATCH 3/3] bench: preserve RepoDoctor runtime directory --- .github/workflows/benchmark-competitive.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/benchmark-competitive.yml b/.github/workflows/benchmark-competitive.yml index dbf701a..92e29cb 100644 --- a/.github/workflows/benchmark-competitive.yml +++ b/.github/workflows/benchmark-competitive.yml @@ -34,10 +34,10 @@ jobs: curl -fL https://github.com/BLCCoreStudio/RepoDoctor/releases/download/v0.1.1/RepoDoctor-v0.1.1-linux-x86_64.tar.gz -o /tmp/repodoctor-v0.1.1.tar.gz echo '9861e7b759e35da7a6fb913b7c6ee665f3e69b783129733c4d35d465dd1ec064 /tmp/repodoctor-v0.1.1.tar.gz' | sha256sum -c - tar -xzf /tmp/repodoctor-v0.1.1.tar.gz -C /tmp/repodoctor-v011 - repodoctor_bin="$(find /tmp/repodoctor-v011 -maxdepth 3 -type f -name repodoctor -perm -u+x -print -quit)" - test -n "$repodoctor_bin" - cp "$repodoctor_bin" /tmp/repodoctor - /tmp/repodoctor version + REPODOCTOR_BIN='/tmp/repodoctor-v011/RepoDoctor-v0.1.1-linux-x86_64/repodoctor' + test -x "$REPODOCTOR_BIN" + "$REPODOCTOR_BIN" version + echo "REPODOCTOR_BIN=$REPODOCTOR_BIN" >> "$GITHUB_ENV" - name: Install pinned codebase-health-score competitor shell: bash @@ -97,7 +97,7 @@ jobs: run: | set -euo pipefail mkdir -p benchmark-results - /tmp/repodoctor scan /tmp/repo-health-fixture --format json >benchmark-results/repodoctor-output.json + "$REPODOCTOR_BIN" scan /tmp/repo-health-fixture --format json >benchmark-results/repodoctor-output.json /tmp/codebase-health-venv/bin/codebase-health analyze /tmp/repo-health-fixture >benchmark-results/codebase-health-output.txt python3 - <<'PY' import json @@ -126,10 +126,10 @@ jobs: hyperfine --warmup 3 --runs 15 \ --export-json benchmark-results/repo-health.json \ --export-markdown benchmark-results/repo-health.md \ - --command-name 'RepoDoctor 0.1.1' '/tmp/repodoctor scan /tmp/repo-health-fixture --format json >/dev/null' \ + --command-name 'RepoDoctor 0.1.1' "$REPODOCTOR_BIN scan /tmp/repo-health-fixture --format json >/dev/null" \ --command-name 'codebase-health-score 1.0.0' '/tmp/codebase-health-venv/bin/codebase-health analyze /tmp/repo-health-fixture >/dev/null' - /usr/bin/time -v /tmp/repodoctor scan /tmp/repo-health-fixture --format json >/dev/null 2>benchmark-results/repodoctor-time-v.txt + /usr/bin/time -v "$REPODOCTOR_BIN" scan /tmp/repo-health-fixture --format json >/dev/null 2>benchmark-results/repodoctor-time-v.txt /usr/bin/time -v /tmp/codebase-health-venv/bin/codebase-health analyze /tmp/repo-health-fixture >/dev/null 2>benchmark-results/codebase-health-time-v.txt cat benchmark-results/repo-health.md