diff --git a/Makefile b/Makefile index a786082..438b565 100644 --- a/Makefile +++ b/Makefile @@ -1,3 +1,5 @@ +.PHONY: up down logs test install lock-dev sync-dev benchmark reproduce + up: docker-compose up --build @@ -21,3 +23,6 @@ sync-dev: benchmark: python benchmarks/ride_sharing_benchmarks.py --output benchmark-results.json + +reproduce: + python scripts/reproduce.py diff --git a/README.md b/README.md index 022260e..355273f 100644 --- a/README.md +++ b/README.md @@ -97,41 +97,32 @@ Core components: ## Research Benchmarks and Recorded Metrics -Benchmark environment: - -- Date recorded: 2026-07-12 -- Runtime: CPython 3.12.13 on Windows local workspace -- Command: `python benchmarks/ride_sharing_benchmarks.py --iterations 500 --driver-count 100 --output benchmark-results.json` -- Result artifact: `benchmark-results.json` - -### Measured Microbenchmarks - -| Area | Workload | Recorded Result | Research Interpretation | -| --- | --- | ---: | --- | -| Event bus publish | 500 in-memory `ride.requested` events | 0.023745 ms avg publish latency | Validates low-overhead async fanout for local simulation. | -| Event delivery | 500 published events | 500 delivered messages | Confirms no message loss in the in-memory event bus harness. | -| Matching engine | 500 matches over 100 candidate drivers | 0.074141 ms avg match latency | Candidate ranking remains sub-millisecond for small local pools. | -| Matching selection | Deterministic synthetic pickup near driver 10 | `driver-10` selected | Confirms nearest-candidate behavior under controlled coordinates. | -| Driver location store | 500 upserts | 0.00402 ms avg upsert latency | In-memory telemetry writes are suitable for unit-level simulation. | -| Pricing engine | 500 surge calculations | 0.004456 ms avg compute latency | Demand/supply pricing calculation is effectively negligible locally. | -| Surge output | Demand 50-59, supply 20 | Last multiplier 1.44x | Confirms high-demand zone pricing response. | - -### Engineering Quality Metrics - -| Metric | Current Recorded Value | Source | -| --- | ---: | --- | -| Tracked repository files | 57 | Repository inventory | -| Python files | 31 | Repository inventory | -| Test files | 5 | `test_*.py` inventory | -| Passing tests | 8 | `pytest -q --cov=. --cov-report=term-missing` | -| Local coverage | 54% | Current focused core test suite | -| GitHub Actions workflows | 3 | `.github/workflows` | -| Infrastructure manifests | 4 | Docker, compose, Kubernetes | -| Benchmark JSON validation | Passing | `python -m json.tool benchmark-results.json` | -| Formatting | Passing | `black --check . --line-length 100` | -| Linting | Passing | `ruff check .` | -| Static typing scope | Passing on core modules | `mypy ... --ignore-missing-imports` | - +Benchmark evidence is generated from the reviewed checkout rather than copied into the README. Run `make reproduce` to regenerate the published benchmark and coverage artifacts. + +The command writes `benchmark-results.json`, `coverage.xml`, and `reproducibility-results.json` in the repository root. The JSON artifact records the exact commands, tracked-file inventory, quality-check outcomes, line coverage, and the benchmark payload from that run. + +### Measured Microbenchmarks + +| Area | Workload | Generated evidence | +| --- | --- | --- | +| Event bus publish and delivery | In-memory `ride.requested` events | `benchmark.event_bus` in `benchmark-results.json` | +| Matching engine | Synthetic candidates and a deterministic pickup | `benchmark.matching` in `benchmark-results.json` | +| Driver location store | In-memory telemetry upserts | `benchmark.location_store` in `benchmark-results.json` | +| Pricing engine | Synthetic demand and supply inputs | `benchmark.pricing` in `benchmark-results.json` | + +### Engineering Quality Metrics + +| Metric | Reproduced by | Artifact | +| --- | --- | --- | +| Tracked repository files | `git ls-files` inventory | `engineering.tracked_repository_files` in `reproducibility-results.json` | +| Python files | `git ls-files` inventory | `engineering.python_files` in `reproducibility-results.json` | +| Test files | `git ls-files` inventory | `engineering.test_files` in `reproducibility-results.json` | +| Test and line coverage | `pytest --cov=.` | `coverage.xml` and `coverage.line_coverage_percent` | +| GitHub Actions workflows | `.github/workflows/` inventory | `engineering.github_actions_workflows` | +| Infrastructure manifests | Docker and Kubernetes inventory | `engineering.infrastructure_manifests` | +| Formatting, linting, and typing | Black, Ruff, and mypy | `commands.format`, `commands.lint`, and `commands.type_check` | +| Benchmark JSON validation | `python -m json.tool benchmark-results.json` | `commands.benchmark_json` | + ### Architecture Target Metrics These are design targets for a production deployment, not claims from the local benchmark harness. @@ -150,14 +141,9 @@ These are design targets for a production deployment, not claims from the local The repository now has an explicit validation path: -```bash -python -m pip install -r requirements.txt -r requirements-dev.txt -black --check . --line-length 100 -ruff check . -mypy models.py utils.py event_bus.py location_store.py matching_engine.py pricing_engine.py consumer.py --ignore-missing-imports -pytest --cov=. --cov-report=term-missing -python benchmarks/ride_sharing_benchmarks.py --iterations 500 --driver-count 100 --output benchmark-results.json -python -m json.tool benchmark-results.json +```bash +python -m pip install -r requirements.txt -r requirements-dev.txt +make reproduce ``` GitHub Actions now: diff --git a/scripts/reproduce.py b/scripts/reproduce.py new file mode 100644 index 0000000..b33512d --- /dev/null +++ b/scripts/reproduce.py @@ -0,0 +1,144 @@ +"""Regenerate the benchmark, coverage, quality, and inventory artifacts cited in README.""" + +from __future__ import annotations + +import json +import subprocess +import sys +import xml.etree.ElementTree as ET +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +BENCHMARK_ARTIFACT = ROOT / "benchmark-results.json" +COVERAGE_ARTIFACT = ROOT / "coverage.xml" +REPRODUCIBILITY_ARTIFACT = ROOT / "reproducibility-results.json" +TYPE_CHECK_TARGETS = [ + "models.py", + "utils.py", + "event_bus.py", + "location_store.py", + "matching_engine.py", + "pricing_engine.py", + "consumer.py", +] + + +def run_command(name: str, command: list[str]) -> dict[str, object]: + """Run one required validation command and retain its exact invocation.""" + completed = subprocess.run( + command, + cwd=ROOT, + capture_output=True, + text=True, + check=False, + ) + result = { + "command": " ".join(command), + "return_code": completed.returncode, + } + if completed.returncode != 0: + sys.stdout.write(completed.stdout) + sys.stderr.write(completed.stderr) + raise RuntimeError(f"{name} failed with exit code {completed.returncode}") + return result + + +def tracked_paths() -> list[str]: + completed = subprocess.run( + ["git", "ls-files"], + cwd=ROOT, + capture_output=True, + text=True, + check=True, + ) + return [path for path in completed.stdout.splitlines() if path] + + +def coverage_percent() -> float: + root = ET.parse(COVERAGE_ARTIFACT).getroot() + return round(float(root.attrib["line-rate"]) * 100, 2) + + +def main() -> None: + results: dict[str, object] = { + "schema_version": 1, + "artifacts": { + "benchmark": BENCHMARK_ARTIFACT.name, + "coverage": COVERAGE_ARTIFACT.name, + "reproducibility": REPRODUCIBILITY_ARTIFACT.name, + }, + "commands": {}, + } + commands = results["commands"] + + commands["format"] = run_command( + "format check", [sys.executable, "-m", "black", "--check", ".", "--line-length=100"] + ) + commands["lint"] = run_command("lint", [sys.executable, "-m", "ruff", "check", "."]) + commands["type_check"] = run_command( + "type check", + [sys.executable, "-m", "mypy", *TYPE_CHECK_TARGETS, "--ignore-missing-imports"], + ) + commands["tests"] = run_command( + "tests and coverage", + [ + sys.executable, + "-m", + "pytest", + "--cov=.", + "--cov-report=term-missing", + f"--cov-report=xml:{COVERAGE_ARTIFACT.name}", + ], + ) + commands["benchmark"] = run_command( + "benchmark", + [ + sys.executable, + "benchmarks/ride_sharing_benchmarks.py", + "--iterations", + "500", + "--driver-count", + "100", + "--output", + BENCHMARK_ARTIFACT.name, + ], + ) + commands["benchmark_json"] = run_command( + "benchmark JSON validation", + [sys.executable, "-m", "json.tool", BENCHMARK_ARTIFACT.name], + ) + + paths = tracked_paths() + benchmark = json.loads(BENCHMARK_ARTIFACT.read_text(encoding="utf-8")) + results["engineering"] = { + "tracked_repository_files": len(paths), + "python_files": sum(path.endswith(".py") for path in paths), + "test_files": sum( + Path(path).name.startswith("test_") and path.endswith(".py") for path in paths + ), + "github_actions_workflows": sum(path.startswith(".github/workflows/") for path in paths), + "infrastructure_manifests": sum( + path == "Dockerfile" + or path == "docker-compose.yml" + or path.startswith("infra/kubernetes/") + for path in paths + ), + } + results["coverage"] = { + "line_coverage_percent": coverage_percent(), + "report": COVERAGE_ARTIFACT.name, + } + results["benchmark"] = benchmark + REPRODUCIBILITY_ARTIFACT.write_text( + json.dumps(results, indent=2, sort_keys=True) + "\n", + encoding="utf-8", + ) + print(json.dumps(results, indent=2, sort_keys=True)) + + +if __name__ == "__main__": + try: + main() + except RuntimeError as error: + print(error, file=sys.stderr) + raise SystemExit(1) from error