From dc444a7cbc4c37240cd4a079b535eba7c61e3dac Mon Sep 17 00:00:00 2001 From: Yuya Asano <64895419+sukeya@users.noreply.github.com> Date: Thu, 28 May 2026 21:59:36 +0900 Subject: [PATCH] Support making bench stat from one result. --- benchmark/compare_benchmark_stats.py | 344 +++++++++++++++++++++++++-- 1 file changed, 321 insertions(+), 23 deletions(-) diff --git a/benchmark/compare_benchmark_stats.py b/benchmark/compare_benchmark_stats.py index d5d97ab..ee4023b 100644 --- a/benchmark/compare_benchmark_stats.py +++ b/benchmark/compare_benchmark_stats.py @@ -10,6 +10,17 @@ from collections import defaultdict from pathlib import Path +IMPLEMENTATION_ORDER = ("STL", "absl", "platanus(auto)") + +BENCHMARK_PATTERN = re.compile( + r"^(?PBM_[^<]+)" + r"<(?PSTL|Absl|BTree)" + r"(?PMultiMap|MultiSet|Map|Set)" + r"<(?Pstd::(?:int32_t|int64_t|string))" + r"(?:, (?P\d+|platanus::kAutoSize))?" + r">>$" +) + def load_benchmarks(path: Path) -> tuple[dict[str, float], str]: with path.open("r", encoding="utf-8") as file: @@ -37,15 +48,18 @@ def load_benchmarks(path: Path) -> tuple[dict[str, float], str]: def parse_args() -> argparse.Namespace: parser = argparse.ArgumentParser( - description="Compute statistics for two Google Benchmark JSON files." - ) - parser.add_argument( - "left", - help="First benchmark JSON file.", + description=( + "Compute statistics for either two Google Benchmark JSON files or " + "one JSON file containing multiple implementations such as STL, absl, and platanus." + ) ) parser.add_argument( - "right", - help="Second benchmark JSON file.", + "inputs", + nargs="+", + help=( + "One benchmark JSON file for implementation comparison, or two JSON files " + "for file-to-file comparison." + ), ) parser.add_argument("-o", "--output", default="compared_stats.txt") parser.add_argument("--top", type=int, default=10, help="Top improved/regressed rows to show.") @@ -102,6 +116,280 @@ def container_kind(name: str) -> str: return match.group(1) if match else "Unknown" +def parse_benchmark_name(name: str) -> dict[str, str] | None: + match = BENCHMARK_PATTERN.match(name) + if match is None: + return None + + parsed = match.groupdict() + impl = parsed["impl"] + node_size = parsed["node_size"] + + if impl == "STL": + parsed["label"] = "STL" + elif impl == "Absl": + parsed["label"] = "absl" + elif impl == "BTree" and node_size == "platanus::kAutoSize": + parsed["label"] = "platanus(auto)" + else: + return None + + return parsed + + +def build_slot_label(row: dict[str, str]) -> str: + return f"{row['benchmark']} / {row['data_type']} / {row['container']}" + + +def collect_implementation_rows( + values: dict[str, float], +) -> tuple[ + dict[tuple[str, str, str], dict[str, dict[str, str | float]]], + list[str], +]: + rows: dict[tuple[str, str, str], dict[str, dict[str, str | float]]] = defaultdict(dict) + skipped: list[str] = [] + + for name, cpu_time in values.items(): + parsed = parse_benchmark_name(name) + if parsed is None: + skipped.append(name) + continue + + key = (parsed["benchmark"], parsed["data_type"], parsed["container"]) + rows[key][parsed["label"]] = { + "name": name, + "label": build_slot_label(parsed), + "benchmark": parsed["benchmark"], + "data_type": parsed["data_type"], + "container": parsed["container"], + "value": cpu_time, + } + + return rows, skipped + + +def compare_row( + slot: dict[str, dict[str, str | float]], + left_label: str, + right_label: str, +) -> dict[str, float | str] | None: + left_entry = slot.get(left_label) + right_entry = slot.get(right_label) + if left_entry is None or right_entry is None: + return None + + left_value = float(left_entry["value"]) + right_value = float(right_entry["value"]) + delta = right_value - left_value + percent = (delta / left_value) * 100.0 + speedup = left_value / right_value if right_value != 0.0 else math.inf + + return { + "name": str(left_entry["label"]), + "benchmark": str(left_entry["benchmark"]), + "container": str(left_entry["container"]), + "left": left_value, + "right": right_value, + "delta": delta, + "percent": percent, + "speedup": speedup, + } + + +def build_pairwise_report( + title: str, + left_label: str, + right_label: str, + rows: list[dict[str, float | str]], + time_unit: str, + top_n: int, +) -> list[str]: + improved = 0 + regressed = 0 + unchanged = 0 + + grouped_rows: dict[str, list[dict[str, float | str]]] = defaultdict(list) + container_rows: dict[str, list[dict[str, float | str]]] = defaultdict(list) + + for row in rows: + left = float(row["left"]) + right = float(row["right"]) + if math.isclose(left, right, rel_tol=1e-12, abs_tol=1e-12): + unchanged += 1 + elif right < left: + improved += 1 + else: + regressed += 1 + + grouped_rows[str(row["benchmark"])].append(row) + container_rows[str(row["container"])].append(row) + + delta_values = [float(row["delta"]) for row in rows] + percent_values = [float(row["percent"]) for row in rows] + speedup_values = [float(row["speedup"]) for row in rows if float(row["speedup"]) > 0.0] + + report: list[str] = [] + report.append(title) + report.append(f" compared rows = {len(rows)}") + report.append(f" lower cpu_time is better") + report.append(f" delta% = ({right_label} - {left_label}) / {left_label} * 100") + report.append(f" speedup = {left_label} / {right_label} (> 1 means {right_label} is faster)") + report.append(f" improved = {improved}") + report.append(f" regressed = {regressed}") + report.append(f" unchanged = {unchanged}") + report.append("") + + report.extend(format_summary("cpu_time delta", delta_values, time_unit)) + report.append("") + report.extend(format_summary("cpu_time delta percent", percent_values, "%")) + report.append("") + report.extend(format_summary("speedup", speedup_values)) + report.append(f" geometric_mean = {geometric_mean(speedup_values):.6f}") + report.append("") + + report.append("Per benchmark kind:") + for kind in sorted(grouped_rows): + kind_rows = grouped_rows[kind] + kind_percents = [float(row["percent"]) for row in kind_rows] + kind_speedups = [float(row["speedup"]) for row in kind_rows if float(row["speedup"]) > 0.0] + kind_improved = sum(1 for row in kind_rows if float(row["right"]) < float(row["left"])) + kind_regressed = sum(1 for row in kind_rows if float(row["right"]) > float(row["left"])) + kind_unchanged = len(kind_rows) - kind_improved - kind_regressed + report.append( + " " + f"{kind}: count={len(kind_rows)}, improved={kind_improved}, " + f"regressed={kind_regressed}, unchanged={kind_unchanged}, " + f"mean_delta%={statistics.fmean(kind_percents):.3f}, " + f"median_delta%={statistics.median(kind_percents):.3f}, " + f"geomean_speedup={geometric_mean(kind_speedups):.6f}" + ) + report.append("") + + report.append("Per container kind:") + for kind in ("Set", "MultiSet", "Map", "MultiMap", "Unknown"): + if kind not in container_rows: + continue + kind_rows = container_rows[kind] + kind_percents = [float(row["percent"]) for row in kind_rows] + kind_speedups = [float(row["speedup"]) for row in kind_rows if float(row["speedup"]) > 0.0] + kind_improved = sum(1 for row in kind_rows if float(row["right"]) < float(row["left"])) + kind_regressed = sum(1 for row in kind_rows if float(row["right"]) > float(row["left"])) + kind_unchanged = len(kind_rows) - kind_improved - kind_regressed + report.append( + " " + f"{kind}: count={len(kind_rows)}, improved={kind_improved}, " + f"regressed={kind_regressed}, unchanged={kind_unchanged}, " + f"mean_delta%={statistics.fmean(kind_percents):.3f}, " + f"median_delta%={statistics.median(kind_percents):.3f}, " + f"geomean_speedup={geometric_mean(kind_speedups):.6f}" + ) + report.append("") + + improved_rows = sorted(rows, key=lambda row: float(row["percent"]))[:top_n] + regressed_rows = sorted(rows, key=lambda row: float(row["percent"]), reverse=True)[:top_n] + + report.append(f"Top {top_n} improvements:") + for row in improved_rows: + report.append( + " " + f"{float(row['percent']):+8.3f}% speedup={float(row['speedup']):.6f} " + f"{row['name']}" + ) + report.append("") + + report.append(f"Top {top_n} regressions:") + for row in regressed_rows: + report.append( + " " + f"{float(row['percent']):+8.3f}% speedup={float(row['speedup']):.6f} " + f"{row['name']}" + ) + report.append("") + return report + + +def build_single_file_report( + input_path: Path, + values: dict[str, float], + time_unit: str, + top_n: int, +) -> str: + slots, skipped = collect_implementation_rows(values) + label_counts: dict[str, int] = defaultdict(int) + for slot in slots.values(): + for label in slot: + label_counts[label] += 1 + + full_triplets = [ + slot + for slot in slots.values() + if all(label in slot for label in IMPLEMENTATION_ORDER) + ] + if not full_triplets: + raise ValueError( + "No benchmark groups found with STL, absl, and platanus(auto) all present." + ) + + report: list[str] = [] + report.append("Benchmark Implementation Comparison Statistics") + report.append("") + report.append(f"input = {input_path}") + report.append(f"parsed benchmark rows = {len(values)}") + report.append(f"recognized grouped rows = {len(slots)}") + report.append(f"complete STL/absl/platanus(auto) groups = {len(full_triplets)}") + report.append(f"skipped unrecognized rows = {len(skipped)}") + report.append("") + report.append("Implementation coverage:") + for label in IMPLEMENTATION_ORDER: + report.append(f" {label}: {label_counts.get(label, 0)}") + report.append("") + + winner_counts: dict[str, int] = defaultdict(int) + tie_count = 0 + for slot in full_triplets: + values_by_label = {label: float(slot[label]["value"]) for label in IMPLEMENTATION_ORDER} + best_value = min(values_by_label.values()) + winners = [ + label + for label in IMPLEMENTATION_ORDER + if math.isclose(values_by_label[label], best_value, rel_tol=1e-12, abs_tol=1e-12) + ] + if len(winners) == 1: + winner_counts[winners[0]] += 1 + else: + tie_count += 1 + + report.append("Fastest implementation counts:") + for label in IMPLEMENTATION_ORDER: + report.append(f" {label}: {winner_counts.get(label, 0)}") + report.append(f" ties: {tie_count}") + report.append("") + + pair_reports = [ + ("STL vs platanus(auto)", "STL", "platanus(auto)"), + ("absl vs platanus(auto)", "absl", "platanus(auto)"), + ] + for title, left_label, right_label in pair_reports: + pair_rows = [ + row + for slot in full_triplets + if (row := compare_row(slot, left_label, right_label)) is not None + ] + report.extend( + build_pairwise_report( + title, + left_label, + right_label, + pair_rows, + time_unit, + top_n, + ) + ) + + return "\n".join(report) + "\n" + + def build_report( left_path: Path, right_path: Path, @@ -246,25 +534,35 @@ def build_report( def main() -> None: args = parse_args() - left_path = Path(args.left) - right_path = Path(args.right) + if not 1 <= len(args.inputs) <= 2: + raise ValueError("Provide either one input JSON file or two input JSON files.") + output_path = Path(args.output) - left_values, left_unit = load_benchmarks(left_path) - right_values, right_unit = load_benchmarks(right_path) - if left_unit != right_unit: - raise ValueError( - f"Time units do not match: {left_path}={left_unit}, {right_path}={right_unit}" + if len(args.inputs) == 1: + input_path = Path(args.inputs[0]) + values, unit = load_benchmarks(input_path) + if args.output == "compared_stats.txt": + output_path = input_path.with_name(f"{input_path.stem}_stats.txt") + report = build_single_file_report(input_path, values, unit, args.top) + else: + left_path = Path(args.inputs[0]) + right_path = Path(args.inputs[1]) + left_values, left_unit = load_benchmarks(left_path) + right_values, right_unit = load_benchmarks(right_path) + if left_unit != right_unit: + raise ValueError( + f"Time units do not match: {left_path}={left_unit}, {right_path}={right_unit}" + ) + + report = build_report( + left_path, + right_path, + left_values, + right_values, + left_unit, + args.top, ) - - report = build_report( - left_path, - right_path, - left_values, - right_values, - left_unit, - args.top, - ) output_path.write_text(report, encoding="utf-8") print(report, end="") print(f"Wrote {output_path}")