From 0f76322394ee444f1f10cf3667459231eb3209a7 Mon Sep 17 00:00:00 2001 From: Corey Leath Date: Wed, 5 Aug 2026 23:04:57 -0400 Subject: [PATCH 1/6] build: add reproducibility artifact generator --- scripts/reproduce.py | 145 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 145 insertions(+) create mode 100644 scripts/reproduce.py diff --git a/scripts/reproduce.py b/scripts/reproduce.py new file mode 100644 index 0000000..9c3caa0 --- /dev/null +++ b/scripts/reproduce.py @@ -0,0 +1,145 @@ +"""Regenerate the benchmark, coverage, quality, and inventory artifacts cited in README.""" + +from __future__ import annotations + +import json +import re +import subprocess +import sys +import xml.etree.ElementTree as ET +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +BENCHMARK_ARTIFACT = ROOT / "benchmark-results.json" +COVERAGE_ARTIFACT = ROOT / "coverage.xml" +REPRODUCIBILITY_ARTIFACT = ROOT / "reproducibility-results.json" +TYPE_CHECK_TARGETS = [ + "models.py", + "utils.py", + "event_bus.py", + "location_store.py", + "matching_engine.py", + "pricing_engine.py", + "consumer.py", +] + + +def run_command(name: str, command: list[str]) -> dict[str, object]: + """Run one required validation command and retain its exact invocation.""" + completed = subprocess.run( + command, + cwd=ROOT, + capture_output=True, + text=True, + check=False, + ) + result = { + "command": " ".join(command), + "return_code": completed.returncode, + } + if completed.returncode != 0: + sys.stdout.write(completed.stdout) + sys.stderr.write(completed.stderr) + raise RuntimeError(f"{name} failed with exit code {completed.returncode}") + return result + + +def tracked_paths() -> list[str]: + completed = subprocess.run( + ["git", "ls-files"], + cwd=ROOT, + capture_output=True, + text=True, + check=True, + ) + return [path for path in completed.stdout.splitlines() if path] + + +def coverage_percent() -> float: + root = ET.parse(COVERAGE_ARTIFACT).getroot() + return round(float(root.attrib["line-rate"]) * 100, 2) + + +def main() -> None: + results: dict[str, object] = { + "schema_version": 1, + "artifacts": { + "benchmark": BENCHMARK_ARTIFACT.name, + "coverage": COVERAGE_ARTIFACT.name, + "reproducibility": REPRODUCIBILITY_ARTIFACT.name, + }, + "commands": {}, + } + commands = results["commands"] + + commands["format"] = run_command( + "format check", [sys.executable, "-m", "black", "--check", ".", "--line-length=100"] + ) + commands["lint"] = run_command("lint", [sys.executable, "-m", "ruff", "check", "."]) + commands["type_check"] = run_command( + "type check", + [sys.executable, "-m", "mypy", *TYPE_CHECK_TARGETS, "--ignore-missing-imports"], + ) + commands["tests"] = run_command( + "tests and coverage", + [ + sys.executable, + "-m", + "pytest", + "--cov=.", + "--cov-report=term-missing", + f"--cov-report=xml:{COVERAGE_ARTIFACT.name}", + ], + ) + commands["benchmark"] = run_command( + "benchmark", + [ + sys.executable, + "benchmarks/ride_sharing_benchmarks.py", + "--iterations", + "500", + "--driver-count", + "100", + "--output", + BENCHMARK_ARTIFACT.name, + ], + ) + commands["benchmark_json"] = run_command( + "benchmark JSON validation", + [sys.executable, "-m", "json.tool", BENCHMARK_ARTIFACT.name], + ) + + paths = tracked_paths() + test_output = str(commands["tests"].get("command", "")) + del test_output + benchmark = json.loads(BENCHMARK_ARTIFACT.read_text(encoding="utf-8")) + results["engineering"] = { + "tracked_repository_files": len(paths), + "python_files": sum(path.endswith(".py") for path in paths), + "test_files": sum(Path(path).name.startswith("test_") and path.endswith(".py") for path in paths), + "github_actions_workflows": sum(path.startswith(".github/workflows/") for path in paths), + "infrastructure_manifests": sum( + path == "Dockerfile" + or path == "docker-compose.yml" + or path.startswith("infra/kubernetes/") + for path in paths + ), + } + results["coverage"] = { + "line_coverage_percent": coverage_percent(), + "report": COVERAGE_ARTIFACT.name, + } + results["benchmark"] = benchmark + REPRODUCIBILITY_ARTIFACT.write_text( + json.dumps(results, indent=2, sort_keys=True) + "\n", + encoding="utf-8", + ) + print(json.dumps(results, indent=2, sort_keys=True)) + + +if __name__ == "__main__": + try: + main() + except RuntimeError as error: + print(error, file=sys.stderr) + raise SystemExit(1) from error From ca371d8be205e6a55a5391755784bd526499d87a Mon Sep 17 00:00:00 2001 From: Corey Leath Date: Wed, 5 Aug 2026 23:05:10 -0400 Subject: [PATCH 2/6] build: add make reproduce target --- Makefile | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/Makefile b/Makefile index a786082..438b565 100644 --- a/Makefile +++ b/Makefile @@ -1,3 +1,5 @@ +.PHONY: up down logs test install lock-dev sync-dev benchmark reproduce + up: docker-compose up --build @@ -21,3 +23,6 @@ sync-dev: benchmark: python benchmarks/ride_sharing_benchmarks.py --output benchmark-results.json + +reproduce: + python scripts/reproduce.py From d94332765cb83589f3326f5cddd89853027250bd Mon Sep 17 00:00:00 2001 From: Corey Leath Date: Wed, 5 Aug 2026 23:05:45 -0400 Subject: [PATCH 3/6] docs: link metrics to reproducible artifacts --- README.md | 72 ++++++++++++++++++++++--------------------------------- 1 file changed, 29 insertions(+), 43 deletions(-) diff --git a/README.md b/README.md index 7452c63..f8c8886 100644 --- a/README.md +++ b/README.md @@ -101,41 +101,32 @@ Core components: ## Research Benchmarks and Recorded Metrics -Benchmark environment: - -- Date recorded: 2026-07-12 -- Runtime: CPython 3.12.13 on Windows local workspace -- Command: `python benchmarks/ride_sharing_benchmarks.py --iterations 500 --driver-count 100 --output benchmark-results.json` -- Result artifact: `benchmark-results.json` - -### Measured Microbenchmarks - -| Area | Workload | Recorded Result | Research Interpretation | -| --- | --- | ---: | --- | -| Event bus publish | 500 in-memory `ride.requested` events | 0.023745 ms avg publish latency | Validates low-overhead async fanout for local simulation. | -| Event delivery | 500 published events | 500 delivered messages | Confirms no message loss in the in-memory event bus harness. | -| Matching engine | 500 matches over 100 candidate drivers | 0.074141 ms avg match latency | Candidate ranking remains sub-millisecond for small local pools. | -| Matching selection | Deterministic synthetic pickup near driver 10 | `driver-10` selected | Confirms nearest-candidate behavior under controlled coordinates. | -| Driver location store | 500 upserts | 0.00402 ms avg upsert latency | In-memory telemetry writes are suitable for unit-level simulation. | -| Pricing engine | 500 surge calculations | 0.004456 ms avg compute latency | Demand/supply pricing calculation is effectively negligible locally. | -| Surge output | Demand 50-59, supply 20 | Last multiplier 1.44x | Confirms high-demand zone pricing response. | - -### Engineering Quality Metrics - -| Metric | Current Recorded Value | Source | -| --- | ---: | --- | -| Tracked repository files | 57 | Repository inventory | -| Python files | 31 | Repository inventory | -| Test files | 5 | `test_*.py` inventory | -| Passing tests | 8 | `pytest -q --cov=. --cov-report=term-missing` | -| Local coverage | 54% | Current focused core test suite | -| GitHub Actions workflows | 3 | `.github/workflows` | -| Infrastructure manifests | 4 | Docker, compose, Kubernetes | -| Benchmark JSON validation | Passing | `python -m json.tool benchmark-results.json` | -| Formatting | Passing | `black --check . --line-length 100` | -| Linting | Passing | `ruff check .` | -| Static typing scope | Passing on core modules | `mypy ... --ignore-missing-imports` | - +Benchmark evidence is generated from the reviewed checkout rather than copied into the README. Run `make reproduce` to regenerate the published benchmark and coverage artifacts. + +The command writes `benchmark-results.json`, `coverage.xml`, and `reproducibility-results.json` in the repository root. The JSON artifact records the exact commands, tracked-file inventory, quality-check outcomes, line coverage, and the benchmark payload from that run. + +### Measured Microbenchmarks + +| Area | Workload | Generated evidence | +| --- | --- | --- | +| Event bus publish and delivery | In-memory `ride.requested` events | `benchmark.event_bus` in `benchmark-results.json` | +| Matching engine | Synthetic candidates and a deterministic pickup | `benchmark.matching` in `benchmark-results.json` | +| Driver location store | In-memory telemetry upserts | `benchmark.location_store` in `benchmark-results.json` | +| Pricing engine | Synthetic demand and supply inputs | `benchmark.pricing` in `benchmark-results.json` | + +### Engineering Quality Metrics + +| Metric | Reproduced by | Artifact | +| --- | --- | --- | +| Tracked repository files | `git ls-files` inventory | `engineering.tracked_repository_files` in `reproducibility-results.json` | +| Python files | `git ls-files` inventory | `engineering.python_files` in `reproducibility-results.json` | +| Test files | `git ls-files` inventory | `engineering.test_files` in `reproducibility-results.json` | +| Test and line coverage | `pytest --cov=.` | `coverage.xml` and `coverage.line_coverage_percent` | +| GitHub Actions workflows | `.github/workflows/` inventory | `engineering.github_actions_workflows` | +| Infrastructure manifests | Docker and Kubernetes inventory | `engineering.infrastructure_manifests` | +| Formatting, linting, and typing | Black, Ruff, and mypy | `commands.format`, `commands.lint`, and `commands.type_check` | +| Benchmark JSON validation | `python -m json.tool benchmark-results.json` | `commands.benchmark_json` | + ### Architecture Target Metrics These are design targets for a production deployment, not claims from the local benchmark harness. @@ -154,14 +145,9 @@ These are design targets for a production deployment, not claims from the local The repository now has an explicit validation path: -```bash -python -m pip install -r requirements.txt -r requirements-dev.txt -black --check . --line-length 100 -ruff check . -mypy models.py utils.py event_bus.py location_store.py matching_engine.py pricing_engine.py consumer.py --ignore-missing-imports -pytest --cov=. --cov-report=term-missing -python benchmarks/ride_sharing_benchmarks.py --iterations 500 --driver-count 100 --output benchmark-results.json -python -m json.tool benchmark-results.json +```bash +python -m pip install -r requirements.txt -r requirements-dev.txt +make reproduce ``` GitHub Actions now: From b3dc0d89f513246432a730348bdf396ace517d4c Mon Sep 17 00:00:00 2001 From: Corey Leath Date: Wed, 5 Aug 2026 23:06:03 -0400 Subject: [PATCH 4/6] style: remove unused reproduction variable --- scripts/reproduce.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/scripts/reproduce.py b/scripts/reproduce.py index 9c3caa0..37609a8 100644 --- a/scripts/reproduce.py +++ b/scripts/reproduce.py @@ -110,8 +110,6 @@ def main() -> None: ) paths = tracked_paths() - test_output = str(commands["tests"].get("command", "")) - del test_output benchmark = json.loads(BENCHMARK_ARTIFACT.read_text(encoding="utf-8")) results["engineering"] = { "tracked_repository_files": len(paths), From 0a8bd9c6c01bf05767c4d66a124fc3c7d6843bac Mon Sep 17 00:00:00 2001 From: Corey Leath Date: Wed, 5 Aug 2026 23:06:13 -0400 Subject: [PATCH 5/6] ci: verify reproducible README evidence --- .github/workflows/ci.yml | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d905975..388a3db 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -70,3 +70,38 @@ jobs: echo "- Checks: black, ruff, mypy, pytest" echo "- Benchmark artifact: benchmark-results.json" } >> "$GITHUB_STEP_SUMMARY" + + + reproduce: + name: Reproduce README evidence + runs-on: ubuntu-latest + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.11" + cache: pip + cache-dependency-path: | + requirements.txt + requirements-dev.txt + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + python -m pip install -r requirements.txt -r requirements-dev.txt + + - name: Regenerate published evidence + run: make reproduce + + - name: Upload reproducibility artifacts + uses: actions/upload-artifact@v4 + with: + name: ride-sharing-reproducibility-artifacts + path: | + benchmark-results.json + coverage.xml + reproducibility-results.json From 06194960406707a7022e2b395e6266c15c9d9b9d Mon Sep 17 00:00:00 2001 From: Corey Leath Date: Wed, 5 Aug 2026 23:09:26 -0400 Subject: [PATCH 6/6] style: format reproducibility script --- scripts/reproduce.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/scripts/reproduce.py b/scripts/reproduce.py index 37609a8..b33512d 100644 --- a/scripts/reproduce.py +++ b/scripts/reproduce.py @@ -3,7 +3,6 @@ from __future__ import annotations import json -import re import subprocess import sys import xml.etree.ElementTree as ET @@ -114,7 +113,9 @@ def main() -> None: results["engineering"] = { "tracked_repository_files": len(paths), "python_files": sum(path.endswith(".py") for path in paths), - "test_files": sum(Path(path).name.startswith("test_") and path.endswith(".py") for path in paths), + "test_files": sum( + Path(path).name.startswith("test_") and path.endswith(".py") for path in paths + ), "github_actions_workflows": sum(path.startswith(".github/workflows/") for path in paths), "infrastructure_manifests": sum( path == "Dockerfile"