diff --git a/.github/scripts/benchmark_formatter.py b/.github/scripts/benchmark_formatter.py new file mode 100644 index 0000000..6932554 --- /dev/null +++ b/.github/scripts/benchmark_formatter.py @@ -0,0 +1,247 @@ +import pathlib, re, sys + +try: + p = pathlib.Path("comparison.md") + if not p.exists(): + print("comparison.md not found, skipping post-processing.") + sys.exit(0) + + lines = p.read_text(encoding="utf-8").splitlines() + processed_lines = [] + in_code = False + + def strip_worker_suffix(text: str) -> str: + return re.sub(r"(\S+?)-\d+(\s|$)", r"\1\2", text) + + def get_icon(diff_val: float) -> str: + if diff_val > 10: + return "🐌" + if diff_val < -10: + return "šŸš€" + return "āž”ļø" + + def clean_superscripts(text: str) -> str: + return re.sub(r"[¹²³⁓⁵⁶⁷⁸⁹⁰]", "", text) + + def parse_val(token: str): + if "%" in token or "=" in token: + return None + token = clean_superscripts(token) + token = token.split("±")[0].strip() + token = token.split("(")[0].strip() + if not token: + return None + + m = re.match(r"^([-+]?\d*\.?\d+)([a-zA-Zµ]+)?$", token) + if not m: + return None + try: + val = float(m.group(1)) + except ValueError: + return None + suffix = (m.group(2) or "").replace("µ", "u") + multipliers = { + "n": 1e-9, + "ns": 1e-9, + "u": 1e-6, + "us": 1e-6, + "m": 1e-3, + "ms": 1e-3, + "s": 1.0, + "k": 1e3, + "K": 1e3, + "M": 1e6, + "G": 1e9, + "Ki": 1024.0, + "Mi": 1024.0**2, + "Gi": 1024.0**3, + "Ti": 1024.0**4, + "B": 1.0, + "B/op": 1.0, + "C": 1.0, # tolerate degree/unit markers that don't affect ratio + } + return val * multipliers.get(suffix, 1.0) + + def extract_two_numbers(tokens): + found = [] + for t in tokens[1:]: # skip name + if t in {"±", "āˆž", "~", "│", "│"}: + continue + if "%" in t or "=" in t: + continue + val = parse_val(t) + if val is not None: + found.append(val) + if len(found) == 2: + break + return found + + # Pass 0: + # 1. find a header line with pipes to derive alignment hint + # 2. calculate max content width to ensure right-most alignment + max_content_width = 0 + + for line in lines: + if line.strip() == "```": + in_code = not in_code + continue + if not in_code: + continue + + # Skip footnotes/meta for width calculation + if re.match(r"^\s*[¹²³⁓⁵⁶⁷⁸⁹⁰]", line) or re.search(r"need\s*>?=\s*\d+\s+samples", line): + continue + if not line.strip() or line.strip().startswith(("goos:", "goarch:", "pkg:", "cpu:")): + continue + # Header lines are handled separately in Pass 1 + if "│" in line and ("vs base" in line or "old" in line or "new" in line): + continue + + # It's likely a data line + # Check if it has an existing percentage we might move/align + curr_line = strip_worker_suffix(line).rstrip() + pct_match = re.search(r"([+-]?\d+\.\d+)%", curr_line) + if pct_match: + # If we are going to realign this, we count width up to the percentage + w = len(curr_line[: pct_match.start()].rstrip()) + else: + w = len(curr_line) + + if w > max_content_width: + max_content_width = w + + # Calculate global alignment target for Diff column + # Ensure target column is beyond the longest line with some padding + diff_col_start = max_content_width + 4 + + # Calculate right boundary (pipe) position + # Diff column width ~12 chars (e.g. "+100.00% šŸš€") + right_boundary = diff_col_start + 14 + + for line in lines: + if line.strip() == "```": + in_code = not in_code + processed_lines.append(line) + continue + + if not in_code: + processed_lines.append(line) + continue + + # footnotes keep untouched + if re.match(r"^\s*[¹²³⁓⁵⁶⁷⁸⁹⁰]", line) or re.search(r"need\s*>?=\s*\d+\s+samples", line): + processed_lines.append(line) + continue + + # header lines: ensure last column labeled Diff and force alignment + if "│" in line and ("vs base" in line or "old" in line or "new" in line): + # Strip trailing pipe and whitespace + stripped_header = line.rstrip().rstrip("│").rstrip() + + # If "vs base" is present, ensure we don't duplicate "Diff" if it's already there + # But we want to enforce OUR alignment, so we might strip existing Diff + stripped_header = re.sub(r"\s+Diff\s*$", "", stripped_header, flags=re.IGNORECASE) + stripped_header = re.sub(r"\s+Delta\b", "", stripped_header, flags=re.IGNORECASE) + + # Pad to diff_col_start + padding = diff_col_start - len(stripped_header) + if padding < 2: + padding = 2 # minimum spacing + # If header is wider than data (unlikely but possible), adjust diff_col_start + if len(stripped_header) < diff_col_start: + new_header = stripped_header + " " * (diff_col_start - len(stripped_header)) + else: + new_header = stripped_header + " " + + # Add Diff column header if it's the second header row (vs base) + if "vs base" in line or "new pr.json" in line: + new_header += "Diff" + + # Add closing pipe at the right boundary + current_len = len(new_header) + if current_len < right_boundary: + new_header += " " * (right_boundary - current_len) + + new_header += "│" + processed_lines.append(new_header) + continue + + # non-data meta lines + if not line.strip() or line.strip().startswith(("goos:", "goarch:", "pkg:")): + processed_lines.append(line) + continue + + original_line = line + line = strip_worker_suffix(line) + tokens = line.split() + if not tokens: + processed_lines.append(line) + continue + + numbers = extract_two_numbers(tokens) + pct_match = re.search(r"([+-]?\d+\.\d+)%", line) + + # Helper to align and append + def append_aligned(left_part, content): + if len(left_part) < diff_col_start: + aligned = left_part + " " * (diff_col_start - len(left_part)) + else: + aligned = left_part + " " + + # Ensure content doesn't exceed right boundary (visual check only, we don't truncate) + return f"{aligned}{content}" + + # Special handling for geomean when values missing or zero + is_geomean = tokens[0] == "geomean" + if is_geomean and (len(numbers) < 2 or any(v == 0 for v in numbers)) and not pct_match: + leading = re.match(r"^\s*", line).group(0) + left = f"{leading}geomean" + processed_lines.append(append_aligned(left, "n/a (has zero)")) + continue + + # when both values are zero, force diff = 0 and align + if len(numbers) == 2 and numbers[0] == 0 and numbers[1] == 0: + diff_val = 0.0 + icon = get_icon(diff_val) + left = line.rstrip() + processed_lines.append(append_aligned(left, f"{diff_val:+.2f}% {icon}")) + continue + + # recompute diff when we have two numeric values + if len(numbers) == 2 and numbers[0] != 0: + diff_val = (numbers[1] - numbers[0]) / numbers[0] * 100 + icon = get_icon(diff_val) + + left = line + if pct_match: + left = line[: pct_match.start()].rstrip() + else: + left = line.rstrip() + + processed_lines.append(append_aligned(left, f"{diff_val:+.2f}% {icon}")) + continue + + # fallback: align existing percentage to Diff column and (re)append icon + if pct_match: + try: + pct_val = float(pct_match.group(1)) + icon = get_icon(pct_val) + + left = line[: pct_match.start()].rstrip() + suffix = line[pct_match.end() :] + # Remove any existing icon after the percentage to avoid duplicates + suffix = re.sub(r"\s*(🐌|šŸš€|āž”ļø)", "", suffix) + + processed_lines.append(append_aligned(left, f"{pct_val:+.2f}% {icon}{suffix}")) + except ValueError: + processed_lines.append(line) + continue + + # If we cannot parse numbers or percentages, keep the original (only worker suffix stripped) + processed_lines.append(line) + + p.write_text("\n".join(processed_lines) + "\n", encoding="utf-8") + +except Exception as e: + print(f"Error post-processing comparison.md: {e}") + sys.exit(1) diff --git a/.github/scripts/download_artifact.js b/.github/scripts/download_artifact.js new file mode 100644 index 0000000..a3fddde --- /dev/null +++ b/.github/scripts/download_artifact.js @@ -0,0 +1,32 @@ +module.exports = async ({github, context, core}) => { + try { + const artifacts = await github.rest.actions.listWorkflowRunArtifacts({ + owner: context.repo.owner, + repo: context.repo.repo, + run_id: context.payload.workflow_run.id, + }); + + const matchArtifact = artifacts.data.artifacts.find((artifact) => { + return artifact.name == "benchmark-results"; + }); + + if (!matchArtifact) { + core.setFailed("No artifact named 'benchmark-results' found."); + return; + } + + const download = await github.rest.actions.downloadArtifact({ + owner: context.repo.owner, + repo: context.repo.repo, + artifact_id: matchArtifact.id, + archive_format: 'zip', + }); + + const fs = require('fs'); + const path = require('path'); + const workspace = process.env.GITHUB_WORKSPACE; + fs.writeFileSync(path.join(workspace, 'benchmark-results.zip'), Buffer.from(download.data)); + } catch (error) { + core.setFailed(`Failed to download artifact: ${error.message}`); + } +}; diff --git a/.github/scripts/format_benchmark_data.py b/.github/scripts/format_benchmark_data.py new file mode 100644 index 0000000..a00e562 --- /dev/null +++ b/.github/scripts/format_benchmark_data.py @@ -0,0 +1,90 @@ +import json +import os +import sys +import datetime +import re + + +def normalize_name(name): + name = re.sub(r"^test_benchmark_", "", name) + parts = name.split("_") + new_parts = [] + for p in parts: + if p.lower() in ["rbac", "abac", "acl", "api", "rest"]: + new_parts.append(p.upper()) + else: + new_parts.append(p.capitalize()) + return "".join(new_parts) + + +def main(): + if len(sys.argv) < 3: + print("Usage: python format_benchmark_data.py input.json output.json") + sys.exit(1) + + input_path = sys.argv[1] + output_path = sys.argv[2] + + try: + with open(input_path, "r", encoding="utf-8") as f: + data = json.load(f) + except Exception as e: + print(f"Error loading {input_path}: {e}") + sys.exit(1) + + # Get commit info from environment variables + # These should be set in the GitHub Action + commit_info = { + "author": { + "email": os.environ.get("COMMIT_AUTHOR_EMAIL", ""), + "name": os.environ.get("COMMIT_AUTHOR_NAME", ""), + "username": os.environ.get("COMMIT_AUTHOR_USERNAME", ""), + }, + "committer": { + "email": os.environ.get("COMMIT_COMMITTER_EMAIL", ""), + "name": os.environ.get("COMMIT_COMMITTER_NAME", ""), + "username": os.environ.get("COMMIT_COMMITTER_USERNAME", ""), + }, + "distinct": True, # Assuming true for push to master + "id": os.environ.get("COMMIT_ID", ""), + "message": os.environ.get("COMMIT_MESSAGE", ""), + "timestamp": os.environ.get("COMMIT_TIMESTAMP", ""), + "tree_id": os.environ.get("COMMIT_TREE_ID", ""), + "url": os.environ.get("COMMIT_URL", ""), + } + + # Get CPU count + cpu_count = data.get("machine_info", {}).get("cpu", {}).get("count") + if not cpu_count: + cpu_count = os.cpu_count() or 1 + + benches = [] + for bench in data.get("benchmarks", []): + # Convert mean (seconds) to ns + val_ns = bench["stats"]["mean"] * 1e9 + + # Format extra info + total_ops = bench["stats"]["rounds"] * bench["stats"]["iterations"] + extra = f"{total_ops} times" + + # Create entry + benches.append( + {"name": normalize_name(bench["name"]), "value": round(val_ns, 2), "unit": "ns/op", "extra": extra} + ) + + output_data = { + "commit": commit_info, + "date": int(datetime.datetime.now().timestamp() * 1000), # Current timestamp in ms + "tool": "python", + "procs": cpu_count, + "benches": benches, + } + + with open(output_path, "w", encoding="utf-8") as f: + json.dump(output_data, f, indent=2) + + print(f"Successfully formatted benchmark data to {output_path}") + + +if __name__ == "__main__": + main() diff --git a/.github/scripts/merge_benchmarks.py b/.github/scripts/merge_benchmarks.py new file mode 100644 index 0000000..8b6d10d --- /dev/null +++ b/.github/scripts/merge_benchmarks.py @@ -0,0 +1,36 @@ +import json +import sys +import glob + + +def merge_jsons(output_file, input_files): + merged_data = None + all_benchmarks = [] + + for file_path in input_files: + with open(file_path, "r", encoding="utf-8") as f: + data = json.load(f) + if merged_data is None: + merged_data = data + all_benchmarks.extend(data.get("benchmarks", [])) + + if merged_data: + merged_data["benchmarks"] = all_benchmarks + with open(output_file, "w", encoding="utf-8") as f: + json.dump(merged_data, f, indent=4) + + +if __name__ == "__main__": + if len(sys.argv) < 3: + print("Usage: python merge_benchmarks.py output.json input1.json input2.json ...") + sys.exit(1) + + output_file = sys.argv[1] + input_files = sys.argv[2:] + + # Expand globs if shell didn't + expanded_inputs = [] + for p in input_files: + expanded_inputs.extend(glob.glob(p)) + + merge_jsons(output_file, expanded_inputs) diff --git a/.github/scripts/post_comment.js b/.github/scripts/post_comment.js new file mode 100644 index 0000000..e7c0a10 --- /dev/null +++ b/.github/scripts/post_comment.js @@ -0,0 +1,59 @@ +module.exports = async ({github, context, core}) => { + const fs = require('fs'); + + // Validate pr_number.txt + if (!fs.existsSync('pr_number.txt')) { + core.setFailed("Required artifact file 'pr_number.txt' was not found in the workspace."); + return; + } + const prNumberContent = fs.readFileSync('pr_number.txt', 'utf8').trim(); + const issue_number = parseInt(prNumberContent, 10); + if (!Number.isFinite(issue_number) || issue_number <= 0) { + core.setFailed('Invalid PR number in pr_number.txt: "' + prNumberContent + '"'); + return; + } + + // Validate comparison.md + if (!fs.existsSync('comparison.md')) { + core.setFailed("Required artifact file 'comparison.md' was not found in the workspace."); + return; + } + let comparison; + try { + comparison = fs.readFileSync('comparison.md', 'utf8'); + } catch (error) { + core.setFailed("Failed to read 'comparison.md': " + error.message); + return; + } + + // Find existing comment + const { data: comments } = await github.rest.issues.listComments({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: issue_number, + }); + + const botComment = comments.find(comment => + comment.user.type === 'Bot' && + comment.body.includes('Benchmark Comparison') + ); + + const footer = 'šŸ¤– This comment will be automatically updated with the latest benchmark results.'; + const commentBody = `${comparison}\n\n${footer}`; + + if (botComment) { + await github.rest.issues.updateComment({ + owner: context.repo.owner, + repo: context.repo.repo, + comment_id: botComment.id, + body: commentBody + }); + } else { + await github.rest.issues.createComment({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: issue_number, + body: commentBody + }); + } +}; diff --git a/.github/scripts/pytest_benchstat.py b/.github/scripts/pytest_benchstat.py new file mode 100644 index 0000000..f0804b5 --- /dev/null +++ b/.github/scripts/pytest_benchstat.py @@ -0,0 +1,186 @@ +import json +import sys +import math +import re +import platform +import subprocess + +# Force UTF-8 output for Windows +sys.stdout.reconfigure(encoding="utf-8") + + +def load_json(path): + try: + with open(path, "r", encoding="utf-8") as f: + return json.load(f) + except Exception as e: + print(f"Error loading {path}: {e}", file=sys.stderr) + return None + + +def format_val(val): + if val is None: + return "N/A" + if val < 1e-9: + return f"{val*1e9:.2f}ns" + if val < 1e-6: + return f"{val*1e9:.2f}ns" # Use ns for < 1us too? benchstat usually uses ns, us, ms, s + if val < 1e-3: + return f"{val*1e6:.2f}us" + if val < 1: + return f"{val*1e3:.2f}ms" + return f"{val:.2f}s" + + +def normalize_name(name): + name = re.sub(r"^test_benchmark_", "", name) + parts = name.split("_") + new_parts = [] + for p in parts: + if p.lower() in ["rbac", "abac", "acl", "api", "rest"]: + new_parts.append(p.upper()) + else: + new_parts.append(p.capitalize()) + return "".join(new_parts) + + +def main(): + if len(sys.argv) < 3: + print("Usage: python pytest_benchstat.py base.json pr.json") + sys.exit(1) + + base_data = load_json(sys.argv[1]) + pr_data = load_json(sys.argv[2]) + + if not base_data or not pr_data: + sys.exit(1) + + base_map = {b["name"]: b["stats"] for b in base_data["benchmarks"]} + pr_map = {b["name"]: b["stats"] for b in pr_data["benchmarks"]} + + all_names = sorted(set(base_map.keys()) | set(pr_map.keys())) + + # Print Header + print("goos: linux") + print("goarch: amd64") + print("pkg: github.com/casbin/pycasbin") + + # Get CPU info + cpu_info = "GitHub Actions Runner" + try: + if platform.system() == "Linux": + command = "cat /proc/cpuinfo | grep 'model name' | head -1" + output = subprocess.check_output(command, shell=True).decode().strip() + if output: + cpu_info = output.split(": ")[1] + except Exception: + pass + print(f"cpu: {cpu_info}") + print("") + + w_name = 50 + w_val = 20 + + # Header + print(f"{'':<{w_name}}│ old base.json │ new pr.json │") + print(f"{'':<{w_name}}│ sec/op │ sec/op │") + + base_means = [] + pr_means = [] + + # Footnote tracking + need_low_sample_note = False + need_insignificant_note = False + need_geomean_note = False + + for name in all_names: + base = base_map.get(name) + pr = pr_map.get(name) + + base_mean = base["mean"] if base else 0 + pr_mean = pr["mean"] if pr else 0 + + base_std = base["stddev"] if base else 0 + pr_std = pr["stddev"] if pr else 0 + + base_rounds = base["rounds"] if base else 0 + pr_rounds = pr["rounds"] if pr else 0 + + if base_mean > 0: + base_means.append(base_mean) + if pr_mean > 0: + pr_means.append(pr_mean) + + # Format Value with StdDev and Superscript + def format_cell(val, std, rounds): + if val == 0: + return "N/A" + + # StdDev formatting + if rounds < 2 or std == 0: + std_str = "± āˆž" + else: + pct = (std / val) * 100 + std_str = f"± {pct:.0f}%" + + # Superscript for low sample size + note = "" + if rounds < 6: + note = "¹" + nonlocal need_low_sample_note + need_low_sample_note = True + + return f"{format_val(val)} {std_str} {note}" + + base_str = format_cell(base_mean, base_std, base_rounds) if base else "N/A" + pr_str = format_cell(pr_mean, pr_std, pr_rounds) if pr else "N/A" + + # Delta column (Statistical Significance) + + delta_str = "" + if base and pr: + # Simple check: do intervals overlap? + + n = min(base_rounds, pr_rounds) + + # Mock p-value logic for display + # If n < 4, benchstat says it can't detect difference usually? + p_val = 1.000 # Placeholder + + if n < 4: + delta_str = f"~ (p={p_val:.3f} n={n}) ²" + need_insignificant_note = True + else: + delta_str = f"~ (p=? n={n})" + + display_name = normalize_name(name) + + print(f"{display_name:<{w_name}} {base_str:<{w_val}} {pr_str:<{w_val}}") + + if base_means and pr_means: + # Filter out zero values for geomean calculation to avoid math error + base_geo_input = [x for x in base_means if x > 0] + pr_geo_input = [x for x in pr_means if x > 0] + + g_base_str = "N/A" + g_pr_str = "N/A" + + if base_geo_input: + g_base = math.exp(sum(math.log(x) for x in base_geo_input) / len(base_geo_input)) + g_base_str = f"{format_val(g_base)}" + + if pr_geo_input: + g_pr = math.exp(sum(math.log(x) for x in pr_geo_input) / len(pr_geo_input)) + g_pr_str = f"{format_val(g_pr)}" + + print(f"{'geomean':<{w_name}} {g_base_str:<{w_val}} {g_pr_str:<{w_val}}") + + # Print Footnotes + if need_low_sample_note: + print("¹ need >= 6 samples for confidence interval at level 0.95") + if need_insignificant_note: + print("² need >= 4 samples to detect a difference at alpha level 0.05") + + +if __name__ == "__main__": + main() diff --git a/.github/scripts/update_data_js.py b/.github/scripts/update_data_js.py new file mode 100644 index 0000000..32a90dc --- /dev/null +++ b/.github/scripts/update_data_js.py @@ -0,0 +1,83 @@ +import json +import sys +import os +import re +import time + + +def load_data_js(filepath): + if not os.path.exists(filepath): + return {"lastUpdate": 0, "repoUrl": "https://github.com/casbin/pycasbin", "entries": {}} + + with open(filepath, "r", encoding="utf-8") as f: + content = f.read() + + # Strip window.BENCHMARK_DATA = + match = re.search(r"window\.BENCHMARK_DATA\s*=\s*({.*});?", content, re.DOTALL) + if match: + try: + return json.loads(match.group(1)) + except json.JSONDecodeError: + print("Error decoding JSON from data.js", file=sys.stderr) + sys.exit(1) + return {"lastUpdate": 0, "repoUrl": "https://github.com/casbin/pycasbin", "entries": {}} + + +def save_data_js(filepath, data): + content = f"window.BENCHMARK_DATA = {json.dumps(data, indent=4)};" + with open(filepath, "w", encoding="utf-8") as f: + f.write(content) + + +def main(): + if len(sys.argv) < 3: + print("Usage: python update_data_js.py benchmark_result.json data.js") + sys.exit(1) + + bench_file = sys.argv[1] + data_js_file = sys.argv[2] + + with open(bench_file, "r", encoding="utf-8") as f: + bench_data = json.load(f) + + js_data = load_data_js(data_js_file) + + # Construct new entry + commit_info = bench_data.get("commit_info", {}) + + entry = { + "commit": { + "time": commit_info.get("time"), + "id": commit_info.get("id", "unknown"), + "author": commit_info.get("author_name", "unknown"), + "message": commit_info.get("message", "unknown"), + }, + "date": int(time.time() * 1000), # Current time in ms + "tool": "pycasbin", + "benchmarks": [], + } + + for b in bench_data.get("benchmarks", []): + name = b["name"] + # Strip test_benchmark_ prefix + name = re.sub(r"^test_benchmark_", "", name) + + # value in ns/op (mean is in seconds) + value = b["stats"]["mean"] * 1e9 + + entry["benchmarks"].append({"name": name, "unit": "ns/op", "value": value}) + + # Append to entries + group_name = "PyCasbin" + if group_name not in js_data["entries"]: + js_data["entries"][group_name] = [] + + js_data["entries"][group_name].append(entry) + js_data["lastUpdate"] = int(time.time() * 1000) + + save_data_js(data_js_file, js_data) + print(f"Updated {data_js_file} with {len(entry['benchmarks'])} benchmarks.") + + +if __name__ == "__main__": + main() diff --git a/.github/workflows/benchmark-push.yml b/.github/workflows/benchmark-push.yml new file mode 100644 index 0000000..6f1038d --- /dev/null +++ b/.github/workflows/benchmark-push.yml @@ -0,0 +1,63 @@ +name: Push Benchmark Data + +on: + push: + branches: + - master + +jobs: + benchmark: + name: Run Benchmark & Push Data + runs-on: ubuntu-latest + steps: + - name: Checkout repo + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.12' + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -r requirements.txt -r requirements_dev.txt + + - name: Run Benchmark + run: | + pytest -o "python_files=benchmark_*.py" tests/benchmarks --benchmark-json=benchmark_result.json + + - name: Checkout Data Repo + uses: actions/checkout@v4 + with: + repository: casbin/casbin-benchmark-data + token: ${{ secrets.CASBIN_BENCHMARK_DATA_TOKEN }} + path: benchmark-data + + - name: Format Benchmark Data + env: + COMMIT_AUTHOR_EMAIL: ${{ github.event.head_commit.author.email }} + COMMIT_AUTHOR_NAME: ${{ github.event.head_commit.author.name }} + COMMIT_AUTHOR_USERNAME: ${{ github.event.head_commit.author.username }} + COMMIT_COMMITTER_EMAIL: ${{ github.event.head_commit.committer.email }} + COMMIT_COMMITTER_NAME: ${{ github.event.head_commit.committer.name }} + COMMIT_COMMITTER_USERNAME: ${{ github.event.head_commit.committer.username }} + COMMIT_MESSAGE: ${{ github.event.head_commit.message }} + COMMIT_TIMESTAMP: ${{ github.event.head_commit.timestamp }} + COMMIT_URL: ${{ github.event.head_commit.url }} + COMMIT_ID: ${{ github.event.head_commit.id }} + run: | + python .github/scripts/format_benchmark_data.py benchmark_result.json formatted_result.json + + - name: Push Benchmark Result + working-directory: benchmark-data + run: | + mkdir -p pycasbin + cp ../formatted_result.json pycasbin/benchmark-${{ github.sha }}.json + + git config user.name "github-actions[bot]" + git config user.email "github-actions[bot]@users.noreply.github.com" + + git add pycasbin/benchmark-${{ github.sha }}.json + git commit -m "Add benchmark result for pycasbin commit ${{ github.sha }}" + git push diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 99c03a5..b606955 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -82,6 +82,7 @@ jobs: release: name: Release + if: github.repository == 'casbin/pycasbin' runs-on: ubuntu-latest needs: [test, coveralls] steps: diff --git a/.github/workflows/comment.yml b/.github/workflows/comment.yml new file mode 100644 index 0000000..feb02e3 --- /dev/null +++ b/.github/workflows/comment.yml @@ -0,0 +1,37 @@ +name: Post Benchmark Comment + +on: + workflow_run: + workflows: ["Performance Comparison for Pull Requests"] + types: + - completed + +permissions: + pull-requests: write + +jobs: + comment: + runs-on: ubuntu-latest + if: > + github.event.workflow_run.event == 'pull_request' && + github.event.workflow_run.conclusion == 'success' + steps: + - name: Checkout repo + uses: actions/checkout@v4 + + - name: 'Download artifact' + uses: actions/github-script@v7 + with: + script: | + const script = require('./.github/scripts/download_artifact.js') + await script({github, context, core}) + + - name: 'Unzip artifact' + run: unzip benchmark-results.zip + + - name: 'Post comment' + uses: actions/github-script@v7 + with: + script: | + const script = require('./.github/scripts/post_comment.js') + await script({github, context, core}) diff --git a/.github/workflows/performance-pr.yml b/.github/workflows/performance-pr.yml new file mode 100644 index 0000000..313776c --- /dev/null +++ b/.github/workflows/performance-pr.yml @@ -0,0 +1,147 @@ +name: Performance Comparison for Pull Requests + +on: + pull_request: + branches: [master] + +jobs: + benchmark-pr: + name: Run Benchmark + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + include: + - name: benchmark_adapter + target: benchmark_adapter.py + extra_args: "" + - name: benchmark_management_api + target: benchmark_management_api.py + extra_args: "" + - name: benchmark_model_large + target: benchmark_model.py + extra_args: "-k large" + - name: benchmark_model_others + target: benchmark_model.py + extra_args: "-k 'not large'" + - name: benchmark_role_manager + target: benchmark_role_manager.py + extra_args: "" + steps: + - name: Checkout PR branch + uses: actions/checkout@v4 + with: + ref: ${{ github.event.pull_request.head.sha }} + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.12' + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -r requirements.txt -r requirements_dev.txt + + # Run benchmark on PR branch + - name: Run benchmark on PR branch + run: | + pytest -o "python_files=${{ matrix.target }}" ${{ matrix.extra_args }} tests/benchmarks --benchmark-json=pr-${{ matrix.name }}.json + + # Checkout base branch and run benchmark + - name: Checkout base branch + uses: actions/checkout@v4 + with: + ref: ${{ github.event.pull_request.base.sha }} + clean: false + path: base + + - name: Run benchmark on base branch + working-directory: base + run: | + pip install -r requirements.txt -r requirements_dev.txt + pytest -o "python_files=${{ matrix.target }}" ${{ matrix.extra_args }} tests/benchmarks --benchmark-json=../base-${{ matrix.name }}.json + + - name: Upload benchmark shard + uses: actions/upload-artifact@v4 + with: + name: benchmark-shard-${{ matrix.name }} + path: | + pr-${{ matrix.name }}.json + base-${{ matrix.name }}.json + + report: + name: Generate Report + needs: benchmark-pr + runs-on: ubuntu-latest + steps: + - name: Checkout repo + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.12' + + - name: Download all benchmark shards + uses: actions/download-artifact@v4 + with: + pattern: benchmark-shard-* + merge-multiple: true + path: benchmark_data + + - name: Merge Benchmark Results + run: | + python .github/scripts/merge_benchmarks.py base.json benchmark_data/base-*.json + python .github/scripts/merge_benchmarks.py pr.json benchmark_data/pr-*.json + + # Save commit SHAs for display + - name: Save commit info + id: commits + run: | + BASE_SHA="${{ github.event.pull_request.base.sha }}" + HEAD_SHA="${{ github.event.pull_request.head.sha }}" + echo "base_short=${BASE_SHA:0:7}" >> $GITHUB_OUTPUT + echo "head_short=${HEAD_SHA:0:7}" >> $GITHUB_OUTPUT + + # Compare benchmarks using script + - name: Compare benchmarks + id: benchstat + run: | + cat > comparison.md << 'EOF' + ## Benchmark Comparison + + Comparing base branch (`${{ steps.commits.outputs.base_short }}`) + vs PR branch (`${{ steps.commits.outputs.head_short }}`) + + ``` + EOF + python3 .github/scripts/pytest_benchstat.py base.json pr.json >> comparison.md || true + echo '```' >> comparison.md + + # Post-process to append percentage + emoji column (šŸš€ faster < -10%, 🐌 slower > +10%, otherwise āž”ļø) + if [ ! -f comparison.md ]; then + echo "comparison.md not found after benchstat." >&2 + exit 1 + fi + python3 .github/scripts/benchmark_formatter.py + + # Save PR number + - name: Save PR number + run: | + PR_NUMBER="${{ github.event.pull_request.number }}" + if [ -z "$PR_NUMBER" ]; then + echo "Error: Pull request number is not available in event payload." >&2 + exit 1 + fi + echo "$PR_NUMBER" > pr_number.txt + + # Upload benchmark results + - name: Upload benchmark results + uses: actions/upload-artifact@v4 + with: + name: benchmark-results + path: | + comparison.md + pr_number.txt +