From 571c4727a4c89a335ee95ea82992d877bbe2316a Mon Sep 17 00:00:00 2001 From: ritsukai Date: Mon, 17 Aug 2026 14:53:58 +0200 Subject: [PATCH] Add a content version for the golden comment set Scores are measured against the golden set, and the golden set changes, so a score is only comparable to another score if both name the same one. The version is a SHA-256 over golden_comments/*.json in sorted filename order, with each file's JSON re-serialized canonically, so reformatting a file leaves it alone and editing content changes it. It is computed at scoring time and carried into benchmark_dashboard.json; both analysis entry points print it, and `python -m analysis.golden_version` prints it on its own. --- offline/analysis/benchmark_dashboard.py | 27 ++++++- offline/analysis/golden_version.py | 94 +++++++++++++++++++++++++ offline/analysis/score_profiles.py | 14 ++++ offline/tests/test_golden_version.py | 85 ++++++++++++++++++++++ 4 files changed, 217 insertions(+), 3 deletions(-) create mode 100644 offline/analysis/golden_version.py create mode 100644 offline/tests/test_golden_version.py diff --git a/offline/analysis/benchmark_dashboard.py b/offline/analysis/benchmark_dashboard.py index 559ba8c..48cb2f4 100644 --- a/offline/analysis/benchmark_dashboard.py +++ b/offline/analysis/benchmark_dashboard.py @@ -13,6 +13,13 @@ import json from pathlib import Path +try: + from analysis.golden_version import golden_set_version + from analysis.golden_version import resolve_golden_dir +except ImportError: # running as a script: python analysis/benchmark_dashboard.py + from golden_version import golden_set_version + from golden_version import resolve_golden_dir + # Tools excluded from the dashboard (superseded versions, incomplete runs, etc.) # Anything matching these exact slugs or the "mra-" prefix is hidden. _HIDDEN_TOOLS: frozenset[str] = frozenset({ @@ -1797,13 +1804,19 @@ def generate_html(all_models_data: dict, default_model: str) -> str: return html -def generate_json_data(all_models_data: dict, default_model: str) -> dict: - """Generate JSON data structure for export.""" +def generate_json_data(all_models_data: dict, default_model: str, golden_set: dict | None = None) -> dict: + """Generate JSON data structure for export. + + golden_set identifies the golden comment set these scores were measured + against; see analysis/golden_version.py. It is null when no golden files + were found, so the export never implies an identity it doesn't have. + """ # Generate and enrich predefined filters predefined_filters = generate_predefined_filters(all_models_data) predefined_filters = enrich_predefined_filters(predefined_filters, all_models_data) return { + "golden_set": golden_set, "models": all_models_data, "predefined_filters": predefined_filters, "tool_display_names": TOOL_DISPLAY_NAMES, @@ -1844,6 +1857,14 @@ def main(): print(f"No model results found in {args.results_dir}") return + # Identify the golden set these scores are measured against + golden_dir = resolve_golden_dir(args.results_dir) + golden_set = golden_set_version(golden_dir) + if golden_set: + print(f"Golden set: {golden_set['short']} ({golden_set['comments']} comments)") + else: + print(f"Golden set: not identified (no golden comment files in {golden_dir})") + # Use the model with the most tools evaluated as default default_model = max(all_models_data.keys(), key=lambda m: len(all_models_data[m].get("tools", []))) @@ -1858,7 +1879,7 @@ def main(): # Generate JSON print("Generating JSON data...") - json_data = generate_json_data(all_models_data, default_model) + json_data = generate_json_data(all_models_data, default_model, golden_set) with open(args.json_output, "w") as f: json.dump(json_data, f, indent=2) diff --git a/offline/analysis/golden_version.py b/offline/analysis/golden_version.py new file mode 100644 index 0000000..fa1f4f9 --- /dev/null +++ b/offline/analysis/golden_version.py @@ -0,0 +1,94 @@ +#!/usr/bin/env python3 +"""Derive a content identifier for the golden comment set. + +Every score is measured against the golden set, and the golden set changes. +Two scores are comparable only if both were measured against the same one, so +this derives an identifier from the golden files themselves rather than from a +number someone has to remember to bump. + +The identifier is a SHA-256 over every golden_comments/*.json, in sorted +filename order. Each file contributes its name and its JSON re-serialized +canonically (sorted keys, compact separators), so reformatting a file leaves +the identifier alone, while editing a comment, category, severity, title or +URL changes it. Files are read as UTF-8 explicitly, so the identifier does +not depend on the platform's locale. + +Usage: + python -m analysis.golden_version + python -m analysis.golden_version --results-dir results +""" + +import argparse +import hashlib +import json +from pathlib import Path + +SHORT_LENGTH = 12 + + +def resolve_golden_dir(results_dir: Path) -> Path: + """Locate golden_comments/ the same way the scorers do.""" + golden_dir = results_dir.parent / "golden_comments" + if not golden_dir.exists(): + golden_dir = Path("golden_comments") + return golden_dir + + +def golden_set_version(golden_dir: Path) -> dict | None: + """Identify the golden set in golden_dir, or None if there isn't one. + + Returns the digest plus the counts a human can check at a glance: how many + files, how many PRs, how many golden comments. + """ + if not golden_dir.exists(): + return None + + paths = sorted(golden_dir.glob("*.json")) + if not paths: + return None + + digest = hashlib.sha256() + prs = 0 + comments = 0 + + for path in paths: + with open(path, encoding="utf-8") as f: + data = json.load(f) + + canonical = json.dumps(data, sort_keys=True, separators=(",", ":"), ensure_ascii=False) + digest.update(f"{path.name}\0{canonical}\0".encode()) + + entries = data if isinstance(data, list) else [] + prs += len(entries) + comments += sum(len(entry.get("comments", [])) for entry in entries) + + sha256 = digest.hexdigest() + return { + "sha256": sha256, + "short": sha256[:SHORT_LENGTH], + "files": len(paths), + "prs": prs, + "comments": comments, + } + + +def main() -> None: + """Print the identifier of the golden set that scoring would use.""" + parser = argparse.ArgumentParser(description="Print the content identifier of the golden comment set") + parser.add_argument("--results-dir", type=Path, default=Path("results")) + args = parser.parse_args() + + golden_dir = resolve_golden_dir(args.results_dir) + version = golden_set_version(golden_dir) + + if version is None: + print(f"No golden comment files found in {golden_dir}") + return + + print(f"Golden set: {golden_dir}") + print(f" version: {version['short']} ({version['sha256']})") + print(f" contents: {version['files']} files, {version['prs']} PRs, {version['comments']} comments") + + +if __name__ == "__main__": + main() diff --git a/offline/analysis/score_profiles.py b/offline/analysis/score_profiles.py index fa3f38c..59e063b 100644 --- a/offline/analysis/score_profiles.py +++ b/offline/analysis/score_profiles.py @@ -16,6 +16,13 @@ import json from pathlib import Path +try: + from analysis.golden_version import golden_set_version + from analysis.golden_version import resolve_golden_dir +except ImportError: # running as a script: python analysis/score_profiles.py + from golden_version import golden_set_version + from golden_version import resolve_golden_dir + PROFILE_CATEGORIES: dict[str, frozenset[str]] = { "strict": frozenset({"bug", "security", "concurrency", "data", "api"}), "core": frozenset({"bug", "security", "concurrency", "data", "api", "perf", "test_gap", "doc_defect"}), @@ -189,6 +196,13 @@ def main() -> None: golden_categories = _load_golden_categories(args.results_dir) print(f"Golden categories loaded: {len(golden_categories)}") + golden_dir = resolve_golden_dir(args.results_dir) + golden_set = golden_set_version(golden_dir) + if golden_set: + print(f"Golden set: {golden_set['short']} ({golden_set['comments']} comments)") + else: + print(f"Golden set: not identified (no golden comment files in {golden_dir})") + if args.all_profiles: for profile in PROFILE_CATEGORIES: results = score_tools(evaluations, golden_categories, profile, args.beta) diff --git a/offline/tests/test_golden_version.py b/offline/tests/test_golden_version.py new file mode 100644 index 0000000..d469c4e --- /dev/null +++ b/offline/tests/test_golden_version.py @@ -0,0 +1,85 @@ +"""Tests for analysis.golden_version module.""" + +from __future__ import annotations + +import json + +from analysis.golden_version import golden_set_version + +ENTRY = { + "url": "https://github.com/org/repo/pull/1", + "pr_title": "Fix the thing", + "comments": [ + {"comment": "Off by one", "category": "bug", "severity": "High"}, + {"comment": "Races on close", "category": "concurrency", "severity": "Medium"}, + ], +} + + +def _write(golden_dir, name, entries, **dump_kwargs): + path = golden_dir / name + path.write_text(json.dumps(entries, **dump_kwargs)) + return path + + +def test_version_is_stable_across_calls(tmp_path): + _write(tmp_path, "a.json", [ENTRY]) + + first = golden_set_version(tmp_path) + second = golden_set_version(tmp_path) + + assert first == second + assert len(first["sha256"]) == 64 + assert first["short"] == first["sha256"][:12] + + +def test_reformatting_does_not_change_version(tmp_path): + _write(tmp_path, "a.json", [ENTRY]) + compact = golden_set_version(tmp_path) + + _write(tmp_path, "a.json", [ENTRY], indent=4, sort_keys=True) + reformatted = golden_set_version(tmp_path) + + assert compact["sha256"] == reformatted["sha256"] + + +def test_editing_a_comment_changes_version(tmp_path): + _write(tmp_path, "a.json", [ENTRY]) + before = golden_set_version(tmp_path) + + edited = json.loads(json.dumps(ENTRY)) + edited["comments"][0]["category"] = "security" + _write(tmp_path, "a.json", [edited]) + after = golden_set_version(tmp_path) + + assert before["sha256"] != after["sha256"] + + +def test_adding_a_comment_changes_version_and_counts(tmp_path): + _write(tmp_path, "a.json", [ENTRY]) + before = golden_set_version(tmp_path) + + extended = json.loads(json.dumps(ENTRY)) + extended["comments"].append({"comment": "Leaks a handle", "category": "bug", "severity": "Low"}) + _write(tmp_path, "a.json", [extended]) + after = golden_set_version(tmp_path) + + assert before["sha256"] != after["sha256"] + assert before["comments"] == 2 + assert after["comments"] == 3 + + +def test_counts_cover_every_file(tmp_path): + _write(tmp_path, "a.json", [ENTRY, ENTRY]) + _write(tmp_path, "b.json", [ENTRY]) + + version = golden_set_version(tmp_path) + + assert version["files"] == 2 + assert version["prs"] == 3 + assert version["comments"] == 6 + + +def test_returns_none_when_there_is_no_golden_set(tmp_path): + assert golden_set_version(tmp_path / "missing") is None + assert golden_set_version(tmp_path) is None