Add a version for the golden set

offline/analysis/benchmark_dashboard.py

@@ -13,6 +13,13 @@
 import json
 from pathlib import Path
 
+try:
+    from analysis.golden_version import golden_set_version
+    from analysis.golden_version import resolve_golden_dir
+except ImportError:  # running as a script: python analysis/benchmark_dashboard.py
+    from golden_version import golden_set_version
+    from golden_version import resolve_golden_dir
+
 # Tools excluded from the dashboard (superseded versions, incomplete runs, etc.)
 # Anything matching these exact slugs or the "mra-" prefix is hidden.
 _HIDDEN_TOOLS: frozenset[str] = frozenset({
@@ -1797,13 +1804,19 @@ def generate_html(all_models_data: dict, default_model: str) -> str:
     return html
 
 
-def generate_json_data(all_models_data: dict, default_model: str) -> dict:
-    """Generate JSON data structure for export."""
+def generate_json_data(all_models_data: dict, default_model: str, golden_set: dict | None = None) -> dict:
+    """Generate JSON data structure for export.
+
+    golden_set identifies the golden comment set these scores were measured
+    against; see analysis/golden_version.py. It is null when no golden files
+    were found, so the export never implies an identity it doesn't have.
+    """
     # Generate and enrich predefined filters
     predefined_filters = generate_predefined_filters(all_models_data)
     predefined_filters = enrich_predefined_filters(predefined_filters, all_models_data)
 
     return {
+        "golden_set": golden_set,
         "models": all_models_data,
         "predefined_filters": predefined_filters,
         "tool_display_names": TOOL_DISPLAY_NAMES,
@@ -1844,6 +1857,14 @@ def main():
         print(f"No model results found in {args.results_dir}")
         return
 
+    # Identify the golden set these scores are measured against
+    golden_dir = resolve_golden_dir(args.results_dir)
+    golden_set = golden_set_version(golden_dir)
+    if golden_set:
+        print(f"Golden set: {golden_set['short']} ({golden_set['comments']} comments)")
+    else:
+        print(f"Golden set: not identified (no golden comment files in {golden_dir})")
+
     # Use the model with the most tools evaluated as default
     default_model = max(all_models_data.keys(), key=lambda m: len(all_models_data[m].get("tools", [])))
 
@@ -1858,7 +1879,7 @@ def main():
 
     # Generate JSON
     print("Generating JSON data...")
-    json_data = generate_json_data(all_models_data, default_model)
+    json_data = generate_json_data(all_models_data, default_model, golden_set)
 
     with open(args.json_output, "w") as f:
         json.dump(json_data, f, indent=2)

offline/analysis/golden_version.py

@@ -0,0 +1,94 @@
+#!/usr/bin/env python3
+"""Derive a content identifier for the golden comment set.
+
+Every score is measured against the golden set, and the golden set changes.
+Two scores are comparable only if both were measured against the same one, so
+this derives an identifier from the golden files themselves rather than from a
+number someone has to remember to bump.
+
+The identifier is a SHA-256 over every golden_comments/*.json, in sorted
+filename order. Each file contributes its name and its JSON re-serialized
+canonically (sorted keys, compact separators), so reformatting a file leaves
+the identifier alone, while editing a comment, category, severity, title or
+URL changes it. Files are read as UTF-8 explicitly, so the identifier does
+not depend on the platform's locale.
+
+Usage:
+    python -m analysis.golden_version
+    python -m analysis.golden_version --results-dir results
+"""
+
+import argparse
+import hashlib
+import json
+from pathlib import Path
+
+SHORT_LENGTH = 12
+
+
+def resolve_golden_dir(results_dir: Path) -> Path:
+    """Locate golden_comments/ the same way the scorers do."""
+    golden_dir = results_dir.parent / "golden_comments"
+    if not golden_dir.exists():
+        golden_dir = Path("golden_comments")
+    return golden_dir
+
+
+def golden_set_version(golden_dir: Path) -> dict | None:
+    """Identify the golden set in golden_dir, or None if there isn't one.
+
+    Returns the digest plus the counts a human can check at a glance: how many
+    files, how many PRs, how many golden comments.
+    """
+    if not golden_dir.exists():
+        return None
+
+    paths = sorted(golden_dir.glob("*.json"))
+    if not paths:
+        return None
+
+    digest = hashlib.sha256()
+    prs = 0
+    comments = 0
+
+    for path in paths:
+        with open(path, encoding="utf-8") as f:
+            data = json.load(f)
+
+        canonical = json.dumps(data, sort_keys=True, separators=(",", ":"), ensure_ascii=False)
+        digest.update(f"{path.name}\0{canonical}\0".encode())
+
+        entries = data if isinstance(data, list) else []
+        prs += len(entries)
+        comments += sum(len(entry.get("comments", [])) for entry in entries)
+
+    sha256 = digest.hexdigest()
+    return {
+        "sha256": sha256,
+        "short": sha256[:SHORT_LENGTH],
+        "files": len(paths),
+        "prs": prs,
+        "comments": comments,
+    }
+
+
+def main() -> None:
+    """Print the identifier of the golden set that scoring would use."""
+    parser = argparse.ArgumentParser(description="Print the content identifier of the golden comment set")
+    parser.add_argument("--results-dir", type=Path, default=Path("results"))
+    args = parser.parse_args()
+
+    golden_dir = resolve_golden_dir(args.results_dir)
+    version = golden_set_version(golden_dir)
+
+    if version is None:
+        print(f"No golden comment files found in {golden_dir}")
+        return
+
+    print(f"Golden set:  {golden_dir}")
+    print(f"  version:   {version['short']}  ({version['sha256']})")
+    print(f"  contents:  {version['files']} files, {version['prs']} PRs, {version['comments']} comments")
+
+
+if __name__ == "__main__":
+    main()

offline/analysis/score_profiles.py

@@ -16,6 +16,13 @@
 import json
 from pathlib import Path
 
+try:
+    from analysis.golden_version import golden_set_version
+    from analysis.golden_version import resolve_golden_dir
+except ImportError:  # running as a script: python analysis/score_profiles.py
+    from golden_version import golden_set_version
+    from golden_version import resolve_golden_dir
+
 PROFILE_CATEGORIES: dict[str, frozenset[str]] = {
     "strict": frozenset({"bug", "security", "concurrency", "data", "api"}),
     "core": frozenset({"bug", "security", "concurrency", "data", "api", "perf", "test_gap", "doc_defect"}),
@@ -189,6 +196,13 @@ def main() -> None:
     golden_categories = _load_golden_categories(args.results_dir)
     print(f"Golden categories loaded: {len(golden_categories)}")
 
+    golden_dir = resolve_golden_dir(args.results_dir)
+    golden_set = golden_set_version(golden_dir)
+    if golden_set:
+        print(f"Golden set: {golden_set['short']} ({golden_set['comments']} comments)")
+    else:
+        print(f"Golden set: not identified (no golden comment files in {golden_dir})")
+
     if args.all_profiles:
         for profile in PROFILE_CATEGORIES:
             results = score_tools(evaluations, golden_categories, profile, args.beta)

offline/tests/test_golden_version.py

@@ -0,0 +1,85 @@
+"""Tests for analysis.golden_version module."""
+
+from __future__ import annotations
+
+import json
+
+from analysis.golden_version import golden_set_version
+
+ENTRY = {
+    "url": "https://github.com/org/repo/pull/1",
+    "pr_title": "Fix the thing",
+    "comments": [
+        {"comment": "Off by one", "category": "bug", "severity": "High"},
+        {"comment": "Races on close", "category": "concurrency", "severity": "Medium"},
+    ],
+}
+
+
+def _write(golden_dir, name, entries, **dump_kwargs):
+    path = golden_dir / name
+    path.write_text(json.dumps(entries, **dump_kwargs))
+    return path
+
+
+def test_version_is_stable_across_calls(tmp_path):
+    _write(tmp_path, "a.json", [ENTRY])
+
+    first = golden_set_version(tmp_path)
+    second = golden_set_version(tmp_path)
+
+    assert first == second
+    assert len(first["sha256"]) == 64
+    assert first["short"] == first["sha256"][:12]
+
+
+def test_reformatting_does_not_change_version(tmp_path):
+    _write(tmp_path, "a.json", [ENTRY])
+    compact = golden_set_version(tmp_path)
+
+    _write(tmp_path, "a.json", [ENTRY], indent=4, sort_keys=True)
+    reformatted = golden_set_version(tmp_path)
+
+    assert compact["sha256"] == reformatted["sha256"]
+
+
+def test_editing_a_comment_changes_version(tmp_path):
+    _write(tmp_path, "a.json", [ENTRY])
+    before = golden_set_version(tmp_path)
+
+    edited = json.loads(json.dumps(ENTRY))
+    edited["comments"][0]["category"] = "security"
+    _write(tmp_path, "a.json", [edited])
+    after = golden_set_version(tmp_path)
+
+    assert before["sha256"] != after["sha256"]
+
+
+def test_adding_a_comment_changes_version_and_counts(tmp_path):
+    _write(tmp_path, "a.json", [ENTRY])
+    before = golden_set_version(tmp_path)
+
+    extended = json.loads(json.dumps(ENTRY))
+    extended["comments"].append({"comment": "Leaks a handle", "category": "bug", "severity": "Low"})
+    _write(tmp_path, "a.json", [extended])
+    after = golden_set_version(tmp_path)
+
+    assert before["sha256"] != after["sha256"]
+    assert before["comments"] == 2
+    assert after["comments"] == 3
+
+
+def test_counts_cover_every_file(tmp_path):
+    _write(tmp_path, "a.json", [ENTRY, ENTRY])
+    _write(tmp_path, "b.json", [ENTRY])
+
+    version = golden_set_version(tmp_path)
+
+    assert version["files"] == 2
+    assert version["prs"] == 3
+    assert version["comments"] == 6
+
+
+def test_returns_none_when_there_is_no_golden_set(tmp_path):
+    assert golden_set_version(tmp_path / "missing") is None
+    assert golden_set_version(tmp_path) is None