From 149e2ce1ddb6800f183b99f8e77bf9cea20c193e Mon Sep 17 00:00:00 2001
From: kpuru88 <227705750+kpuru88@users.noreply.github.com>
Date: Mon, 24 Aug 2026 13:07:30 -0700
Subject: [PATCH 1/2] Add Shipwright official Martian benchmark results

---
 offline/README.md                             |    1 +
 offline/analysis/benchmark_dashboard.html     |    6 +-
 offline/analysis/benchmark_dashboard.json     | 3183 +++++++++++------
 offline/analysis/benchmark_dashboard.py       |    2 +
 .../candidates.json                           | 1426 +++++++-
 .../dedup_groups.json                         |  509 +++
 .../evaluations.json                          | 2674 +++++++++++++-
 offline/results/benchmark_data.json           | 1229 +++++++
 8 files changed, 7702 insertions(+), 1328 deletions(-)

diff --git a/offline/README.md b/offline/README.md
index 78b2815..7432f58 100644
--- a/offline/README.md
+++ b/offline/README.md
@@ -24,6 +24,7 @@ Open replication of the code review benchmark used by companies like [Augment](h
 | [Kodus](https://kodus.io/) | AI code review |
 | [Macroscope](https://www.macroscope.com/) | AI code review |
 | [Qodo](https://www.qodo.ai/) | AI code review |
+| [Shipwright](https://github.com/kpuru88/shipwright-agent) | AI code review |
 | [Sourcery](https://sourcery.ai/) | AI code review |
 
 Adding a new tool requires forking the benchmark PRs and collecting the tool's reviews — see Steps 0 and 1 below.
diff --git a/offline/analysis/benchmark_dashboard.html b/offline/analysis/benchmark_dashboard.html
index 927d654..eea907f 100644
--- a/offline/analysis/benchmark_dashboard.html
+++ b/offline/analysis/benchmark_dashboard.html
@@ -428,9 +428,9 @@
 
     <script>
         // All models data embedded
-        const allModelsData = {"anthropic_claude-opus-4-5-20251101": {"prs": [{"url": "https://github.com/keycloak/keycloak/pull/37429", "language": "Java", "pr_size": "medium", "domain": "UI", "change_type": "feature", "complexity": "moderate", "difficulty": "moderate", "risk": "medium", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 4}}, "codeant-v2": {"strict": {"tp": 0, "fp": 1, "fn": 1}, "core": {"tp": 0, "fp": 1, "fn": 3}, "all": {"tp": 0, "fp": 1, "fn": 5}}, "coderabbit": {"strict": {"tp": 0, "fp": 5, "fn": 1}, "core": {"tp": 2, "fp": 5, "fn": 1}, "all": {"tp": 4, "fp": 5, "fn": 1}}, "qodo-extended-v2": {"strict": {"tp": 0, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 1}, "all": {"tp": 2, "fp": 1, "fn": 3}}, "greptile-v4-1": {"strict": {"tp": 0, "fp": 3, "fn": 1}, "core": {"tp": 0, "fp": 3, "fn": 3}, "all": {"tp": 2, "fp": 3, "fn": 3}}, "gemini-v2": {"strict": {"tp": 0, "fp": 3, "fn": 1}, "core": {"tp": 2, "fp": 3, "fn": 1}, "all": {"tp": 4, "fp": 3, "fn": 1}}, "gitlab": {"strict": {"tp": 0, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 2}, "all": {"tp": 2, "fp": 1, "fn": 3}}, "copilot-v2": {"strict": {"tp": 0, "fp": 6, "fn": 1}, "core": {"tp": 2, "fp": 6, "fn": 1}, "all": {"tp": 3, "fp": 6, "fn": 2}}, "augment": {"strict": {"tp": 0, "fp": 3, "fn": 1}, "core": {"tp": 2, "fp": 3, "fn": 1}, "all": {"tp": 2, "fp": 3, "fn": 3}}, "bugbot": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 2, "fp": 0, "fn": 1}, "all": {"tp": 2, "fp": 0, "fn": 3}}, "claude": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 1, "fp": 0, "fn": 4}}, "claude-code": {"strict": {"tp": 0, "fp": 3, "fn": 1}, "core": {"tp": 0, "fp": 3, "fn": 3}, "all": {"tp": 2, "fp": 3, "fn": 3}}, "cubic-v2": {"strict": {"tp": 0, "fp": 2, "fn": 1}, "core": {"tp": 1, "fp": 2, "fn": 2}, "all": {"tp": 1, "fp": 2, "fn": 4}}, "devin": {"strict": {"tp": 0, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 1}, "all": {"tp": 2, "fp": 1, "fn": 3}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 5}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 4}}, "kodus-v2": {"strict": {"tp": 0, "fp": 2, "fn": 1}, "core": {"tp": 1, "fp": 2, "fn": 2}, "all": {"tp": 1, "fp": 2, "fn": 4}}, "macroscope": {"strict": {"tp": 0, "fp": 1, "fn": 1}, "core": {"tp": 0, "fp": 1, "fn": 3}, "all": {"tp": 0, "fp": 1, "fn": 5}}, "qodo-v2": {"strict": {"tp": 0, "fp": 3, "fn": 1}, "core": {"tp": 0, "fp": 3, "fn": 3}, "all": {"tp": 0, "fp": 3, "fn": 5}}, "sourcery": {"strict": {"tp": 0, "fp": 5, "fn": 1}, "core": {"tp": 0, "fp": 5, "fn": 3}, "all": {"tp": 2, "fp": 5, "fn": 3}}}}, {"url": "https://github.com/keycloak/keycloak/pull/37634", "language": "Java", "pr_size": "medium", "domain": "authentication", "change_type": "feature", "complexity": "moderate", "difficulty": "subtle", "risk": "high", "context": "file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 2, "fp": 0, "fn": 1}, "core": {"tp": 2, "fp": 0, "fn": 2}, "all": {"tp": 2, "fp": 0, "fn": 2}}, "codeant-v2": {"strict": {"tp": 1, "fp": 2, "fn": 2}, "core": {"tp": 1, "fp": 2, "fn": 3}, "all": {"tp": 1, "fp": 2, "fn": 3}}, "coderabbit": {"strict": {"tp": 3, "fp": 4, "fn": 0}, "core": {"tp": 3, "fp": 4, "fn": 1}, "all": {"tp": 3, "fp": 4, "fn": 1}}, "qodo-extended-v2": {"strict": {"tp": 2, "fp": 0, "fn": 1}, "core": {"tp": 3, "fp": 0, "fn": 1}, "all": {"tp": 3, "fp": 0, "fn": 1}}, "greptile-v4-1": {"strict": {"tp": 2, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 2}, "all": {"tp": 2, "fp": 1, "fn": 2}}, "gemini-v2": {"strict": {"tp": 2, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 2}, "all": {"tp": 2, "fp": 1, "fn": 2}}, "gitlab": {"strict": {"tp": 1, "fp": 1, "fn": 2}, "core": {"tp": 2, "fp": 1, "fn": 2}, "all": {"tp": 2, "fp": 1, "fn": 2}}, "copilot-v2": {"strict": {"tp": 2, "fp": 3, "fn": 1}, "core": {"tp": 3, "fp": 3, "fn": 1}, "all": {"tp": 3, "fp": 3, "fn": 1}}, "augment": {"strict": {"tp": 2, "fp": 2, "fn": 1}, "core": {"tp": 2, "fp": 2, "fn": 2}, "all": {"tp": 2, "fp": 2, "fn": 2}}, "bugbot": {"strict": {"tp": 2, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 2}, "all": {"tp": 2, "fp": 1, "fn": 2}}, "claude": {"strict": {"tp": 1, "fp": 1, "fn": 2}, "core": {"tp": 2, "fp": 1, "fn": 2}, "all": {"tp": 2, "fp": 1, "fn": 2}}, "claude-code": {"strict": {"tp": 0, "fp": 2, "fn": 3}, "core": {"tp": 0, "fp": 2, "fn": 4}, "all": {"tp": 0, "fp": 2, "fn": 4}}, "cubic-v2": {"strict": {"tp": 2, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 2}, "all": {"tp": 2, "fp": 1, "fn": 2}}, "devin": {"strict": {"tp": 2, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 2}, "all": {"tp": 2, "fp": 1, "fn": 2}}, "graphite": {"strict": {"tp": 2, "fp": 0, "fn": 1}, "core": {"tp": 2, "fp": 0, "fn": 2}, "all": {"tp": 2, "fp": 0, "fn": 2}}, "kg": {"strict": {"tp": 1, "fp": 2, "fn": 2}, "core": {"tp": 1, "fp": 2, "fn": 3}, "all": {"tp": 1, "fp": 2, "fn": 3}}, "kodus-v2": {"strict": {"tp": 2, "fp": 0, "fn": 1}, "core": {"tp": 2, "fp": 0, "fn": 2}, "all": {"tp": 2, "fp": 0, "fn": 2}}, "macroscope": {"strict": {"tp": 2, "fp": 2, "fn": 1}, "core": {"tp": 2, "fp": 2, "fn": 2}, "all": {"tp": 2, "fp": 2, "fn": 2}}, "qodo-v2": {"strict": {"tp": 2, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 2}, "all": {"tp": 2, "fp": 1, "fn": 2}}, "sourcery": {"strict": {"tp": 2, "fp": 0, "fn": 1}, "core": {"tp": 2, "fp": 0, "fn": 2}, "all": {"tp": 2, "fp": 0, "fn": 2}}}}, {"url": "https://github.com/keycloak/keycloak/pull/38446", "language": "Java", "pr_size": "medium", "domain": "authentication", "change_type": "feature", "complexity": "moderate", "difficulty": "moderate", "risk": "medium", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 0, "fp": 1, "fn": 2}, "core": {"tp": 0, "fp": 1, "fn": 2}, "all": {"tp": 0, "fp": 1, "fn": 2}}, "codeant-v2": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "coderabbit": {"strict": {"tp": 2, "fp": 2, "fn": 0}, "core": {"tp": 2, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "qodo-extended-v2": {"strict": {"tp": 1, "fp": 3, "fn": 1}, "core": {"tp": 1, "fp": 3, "fn": 1}, "all": {"tp": 1, "fp": 3, "fn": 1}}, "greptile-v4-1": {"strict": {"tp": 2, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "gemini-v2": {"strict": {"tp": 1, "fp": 3, "fn": 1}, "core": {"tp": 1, "fp": 3, "fn": 1}, "all": {"tp": 1, "fp": 3, "fn": 1}}, "gitlab": {"strict": {"tp": 1, "fp": 5, "fn": 1}, "core": {"tp": 1, "fp": 5, "fn": 1}, "all": {"tp": 1, "fp": 5, "fn": 1}}, "copilot-v2": {"strict": {"tp": 1, "fp": 6, "fn": 1}, "core": {"tp": 1, "fp": 6, "fn": 1}, "all": {"tp": 1, "fp": 6, "fn": 1}}, "augment": {"strict": {"tp": 1, "fp": 3, "fn": 1}, "core": {"tp": 1, "fp": 3, "fn": 1}, "all": {"tp": 1, "fp": 3, "fn": 1}}, "bugbot": {"strict": {"tp": 0, "fp": 3, "fn": 2}, "core": {"tp": 0, "fp": 3, "fn": 2}, "all": {"tp": 0, "fp": 3, "fn": 2}}, "claude": {"strict": {"tp": 1, "fp": 3, "fn": 1}, "core": {"tp": 1, "fp": 3, "fn": 1}, "all": {"tp": 1, "fp": 3, "fn": 1}}, "claude-code": {"strict": {"tp": 1, "fp": 5, "fn": 1}, "core": {"tp": 1, "fp": 5, "fn": 1}, "all": {"tp": 1, "fp": 5, "fn": 1}}, "cubic-v2": {"strict": {"tp": 1, "fp": 3, "fn": 1}, "core": {"tp": 1, "fp": 3, "fn": 1}, "all": {"tp": 1, "fp": 3, "fn": 1}}, "devin": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "kodus-v2": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "macroscope": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "qodo-v2": {"strict": {"tp": 2, "fp": 3, "fn": 0}, "core": {"tp": 2, "fp": 3, "fn": 0}, "all": {"tp": 2, "fp": 3, "fn": 0}}, "sourcery": {"strict": {"tp": 1, "fp": 6, "fn": 1}, "core": {"tp": 1, "fp": 6, "fn": 1}, "all": {"tp": 1, "fp": 6, "fn": 1}}}}, {"url": "https://github.com/keycloak/keycloak/pull/36882", "language": "Java", "pr_size": "small", "domain": "configuration", "change_type": "feature", "complexity": "moderate", "difficulty": "moderate", "risk": "medium", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 0, "fp": 1, "fn": 1}, "core": {"tp": 0, "fp": 1, "fn": 2}, "all": {"tp": 0, "fp": 1, "fn": 2}}, "codeant-v2": {"strict": {"tp": 0, "fp": 1, "fn": 1}, "core": {"tp": 0, "fp": 1, "fn": 2}, "all": {"tp": 0, "fp": 1, "fn": 2}}, "coderabbit": {"strict": {"tp": 0, "fp": 1, "fn": 1}, "core": {"tp": 0, "fp": 1, "fn": 2}, "all": {"tp": 0, "fp": 1, "fn": 2}}, "qodo-extended-v2": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "greptile-v4-1": {"strict": {"tp": 0, "fp": 2, "fn": 1}, "core": {"tp": 0, "fp": 2, "fn": 2}, "all": {"tp": 0, "fp": 2, "fn": 2}}, "gemini-v2": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "gitlab": {"strict": {"tp": 0, "fp": 2, "fn": 1}, "core": {"tp": 0, "fp": 2, "fn": 2}, "all": {"tp": 0, "fp": 2, "fn": 2}}, "copilot-v2": {"strict": {"tp": 0, "fp": 3, "fn": 1}, "core": {"tp": 0, "fp": 3, "fn": 2}, "all": {"tp": 0, "fp": 3, "fn": 2}}, "augment": {"strict": {"tp": 0, "fp": 1, "fn": 1}, "core": {"tp": 0, "fp": 1, "fn": 2}, "all": {"tp": 0, "fp": 1, "fn": 2}}, "bugbot": {"strict": {"tp": 0, "fp": 1, "fn": 1}, "core": {"tp": 0, "fp": 1, "fn": 2}, "all": {"tp": 0, "fp": 1, "fn": 2}}, "claude": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "claude-code": {"strict": {"tp": 0, "fp": 1, "fn": 1}, "core": {"tp": 0, "fp": 1, "fn": 2}, "all": {"tp": 0, "fp": 1, "fn": 2}}, "cubic-v2": {"strict": {"tp": 1, "fp": 2, "fn": 0}, "core": {"tp": 1, "fp": 2, "fn": 1}, "all": {"tp": 1, "fp": 2, "fn": 1}}, "devin": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "kodus-v2": {"strict": {"tp": 0, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "macroscope": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "qodo-v2": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "sourcery": {"strict": {"tp": 0, "fp": 4, "fn": 1}, "core": {"tp": 0, "fp": 4, "fn": 2}, "all": {"tp": 0, "fp": 4, "fn": 2}}}}, {"url": "https://github.com/keycloak/keycloak/pull/36880", "language": "Java", "pr_size": "medium", "domain": "authentication", "change_type": "feature", "complexity": "complex", "difficulty": "very_subtle", "risk": "high", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 3, "fp": 0, "fn": 1}, "core": {"tp": 3, "fp": 0, "fn": 1}, "all": {"tp": 3, "fp": 0, "fn": 2}}, "codeant-v2": {"strict": {"tp": 2, "fp": 0, "fn": 2}, "core": {"tp": 2, "fp": 0, "fn": 2}, "all": {"tp": 2, "fp": 0, "fn": 3}}, "coderabbit": {"strict": {"tp": 1, "fp": 5, "fn": 3}, "core": {"tp": 1, "fp": 5, "fn": 3}, "all": {"tp": 2, "fp": 5, "fn": 3}}, "qodo-extended-v2": {"strict": {"tp": 3, "fp": 0, "fn": 1}, "core": {"tp": 3, "fp": 0, "fn": 1}, "all": {"tp": 3, "fp": 0, "fn": 2}}, "greptile-v4-1": {"strict": {"tp": 1, "fp": 1, "fn": 3}, "core": {"tp": 1, "fp": 1, "fn": 3}, "all": {"tp": 2, "fp": 1, "fn": 3}}, "gemini-v2": {"strict": {"tp": 1, "fp": 3, "fn": 3}, "core": {"tp": 1, "fp": 3, "fn": 3}, "all": {"tp": 2, "fp": 3, "fn": 3}}, "gitlab": {"strict": {"tp": 1, "fp": 6, "fn": 3}, "core": {"tp": 1, "fp": 6, "fn": 3}, "all": {"tp": 2, "fp": 6, "fn": 3}}, "copilot-v2": {"strict": {"tp": 2, "fp": 6, "fn": 2}, "core": {"tp": 2, "fp": 6, "fn": 2}, "all": {"tp": 2, "fp": 6, "fn": 3}}, "augment": {"strict": {"tp": 1, "fp": 1, "fn": 3}, "core": {"tp": 1, "fp": 1, "fn": 3}, "all": {"tp": 1, "fp": 1, "fn": 4}}, "bugbot": {"strict": {"tp": 3, "fp": 0, "fn": 1}, "core": {"tp": 3, "fp": 0, "fn": 1}, "all": {"tp": 4, "fp": 0, "fn": 1}}, "claude": {"strict": {"tp": 1, "fp": 2, "fn": 3}, "core": {"tp": 1, "fp": 2, "fn": 3}, "all": {"tp": 2, "fp": 2, "fn": 3}}, "claude-code": {"strict": {"tp": 0, "fp": 2, "fn": 4}, "core": {"tp": 0, "fp": 2, "fn": 4}, "all": {"tp": 0, "fp": 2, "fn": 5}}, "cubic-v2": {"strict": {"tp": 1, "fp": 1, "fn": 3}, "core": {"tp": 1, "fp": 1, "fn": 3}, "all": {"tp": 1, "fp": 1, "fn": 4}}, "devin": {"strict": {"tp": 1, "fp": 0, "fn": 3}, "core": {"tp": 1, "fp": 0, "fn": 3}, "all": {"tp": 1, "fp": 0, "fn": 4}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 4}, "core": {"tp": 0, "fp": 0, "fn": 4}, "all": {"tp": 0, "fp": 0, "fn": 5}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 4}, "core": {"tp": 0, "fp": 0, "fn": 4}, "all": {"tp": 0, "fp": 0, "fn": 5}}, "kodus-v2": {"strict": {"tp": 2, "fp": 1, "fn": 2}, "core": {"tp": 2, "fp": 1, "fn": 2}, "all": {"tp": 2, "fp": 1, "fn": 3}}, "macroscope": {"strict": {"tp": 2, "fp": 4, "fn": 2}, "core": {"tp": 2, "fp": 4, "fn": 2}, "all": {"tp": 2, "fp": 4, "fn": 3}}, "qodo-v2": {"strict": {"tp": 2, "fp": 1, "fn": 2}, "core": {"tp": 2, "fp": 1, "fn": 2}, "all": {"tp": 2, "fp": 1, "fn": 3}}, "sourcery": {"strict": {"tp": 1, "fp": 4, "fn": 3}, "core": {"tp": 1, "fp": 4, "fn": 3}, "all": {"tp": 2, "fp": 4, "fn": 3}}}}, {"url": "https://github.com/keycloak/keycloak/pull/37038", "language": "Java", "pr_size": "small", "domain": "authentication", "change_type": "feature", "complexity": "complex", "difficulty": "very_subtle", "risk": "high", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "codeant-v2": {"strict": {"tp": 1, "fp": 5, "fn": 1}, "core": {"tp": 1, "fp": 5, "fn": 1}, "all": {"tp": 1, "fp": 5, "fn": 1}}, "coderabbit": {"strict": {"tp": 2, "fp": 7, "fn": 0}, "core": {"tp": 2, "fp": 7, "fn": 0}, "all": {"tp": 2, "fp": 7, "fn": 0}}, "qodo-extended-v2": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "greptile-v4-1": {"strict": {"tp": 2, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "gemini-v2": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "gitlab": {"strict": {"tp": 1, "fp": 2, "fn": 1}, "core": {"tp": 1, "fp": 2, "fn": 1}, "all": {"tp": 1, "fp": 2, "fn": 1}}, "copilot-v2": {"strict": {"tp": 2, "fp": 3, "fn": 0}, "core": {"tp": 2, "fp": 3, "fn": 0}, "all": {"tp": 2, "fp": 3, "fn": 0}}, "augment": {"strict": {"tp": 2, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "bugbot": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "claude": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "claude-code": {"strict": {"tp": 1, "fp": 2, "fn": 1}, "core": {"tp": 1, "fp": 2, "fn": 1}, "all": {"tp": 1, "fp": 2, "fn": 1}}, "cubic-v2": {"strict": {"tp": 2, "fp": 2, "fn": 0}, "core": {"tp": 2, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "devin": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "kodus-v2": {"strict": {"tp": 2, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "macroscope": {"strict": {"tp": 2, "fp": 2, "fn": 0}, "core": {"tp": 2, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "qodo-v2": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "sourcery": {"strict": {"tp": 1, "fp": 4, "fn": 1}, "core": {"tp": 1, "fp": 4, "fn": 1}, "all": {"tp": 1, "fp": 4, "fn": 1}}}}, {"url": "https://github.com/keycloak/keycloak/pull/33832", "language": "Java", "pr_size": "medium", "domain": "authentication", "change_type": "feature", "complexity": "moderate", "difficulty": "subtle", "risk": "high", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 2, "fp": 0, "fn": 1}, "all": {"tp": 2, "fp": 0, "fn": 2}}, "codeant-v2": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 2}, "all": {"tp": 2, "fp": 1, "fn": 2}}, "coderabbit": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 1}, "all": {"tp": 2, "fp": 1, "fn": 2}}, "qodo-extended-v2": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 1}, "all": {"tp": 2, "fp": 1, "fn": 2}}, "greptile-v4-1": {"strict": {"tp": 0, "fp": 3, "fn": 2}, "core": {"tp": 1, "fp": 3, "fn": 2}, "all": {"tp": 1, "fp": 3, "fn": 3}}, "gemini-v2": {"strict": {"tp": 2, "fp": 1, "fn": 0}, "core": {"tp": 3, "fp": 1, "fn": 0}, "all": {"tp": 3, "fp": 1, "fn": 1}}, "gitlab": {"strict": {"tp": 0, "fp": 2, "fn": 2}, "core": {"tp": 1, "fp": 2, "fn": 2}, "all": {"tp": 1, "fp": 2, "fn": 3}}, "copilot-v2": {"strict": {"tp": 1, "fp": 4, "fn": 1}, "core": {"tp": 2, "fp": 4, "fn": 1}, "all": {"tp": 3, "fp": 4, "fn": 1}}, "augment": {"strict": {"tp": 1, "fp": 2, "fn": 1}, "core": {"tp": 1, "fp": 2, "fn": 2}, "all": {"tp": 1, "fp": 2, "fn": 3}}, "bugbot": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 3}}, "claude": {"strict": {"tp": 0, "fp": 5, "fn": 2}, "core": {"tp": 1, "fp": 5, "fn": 2}, "all": {"tp": 1, "fp": 5, "fn": 3}}, "claude-code": {"strict": {"tp": 0, "fp": 1, "fn": 2}, "core": {"tp": 1, "fp": 1, "fn": 2}, "all": {"tp": 2, "fp": 1, "fn": 2}}, "cubic-v2": {"strict": {"tp": 1, "fp": 2, "fn": 1}, "core": {"tp": 2, "fp": 2, "fn": 1}, "all": {"tp": 2, "fp": 2, "fn": 2}}, "devin": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 4}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 3}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 4}}, "kodus-v2": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 4}}, "macroscope": {"strict": {"tp": 2, "fp": 4, "fn": 0}, "core": {"tp": 2, "fp": 4, "fn": 1}, "all": {"tp": 2, "fp": 4, "fn": 2}}, "qodo-v2": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 2}, "all": {"tp": 1, "fp": 1, "fn": 3}}, "sourcery": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 1}, "all": {"tp": 2, "fp": 1, "fn": 2}}}}, {"url": "https://github.com/keycloak/keycloak/pull/40940", "language": "Java", "pr_size": "small", "domain": "concurrency", "change_type": "bug_fix", "complexity": "moderate", "difficulty": "subtle", "risk": "high", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "codeant-v2": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "coderabbit": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "qodo-extended-v2": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "greptile-v4-1": {"strict": {"tp": 1, "fp": 2, "fn": 1}, "core": {"tp": 1, "fp": 2, "fn": 1}, "all": {"tp": 1, "fp": 2, "fn": 1}}, "gemini-v2": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "gitlab": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "copilot-v2": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "augment": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "bugbot": {"strict": {"tp": 1, "fp": 2, "fn": 1}, "core": {"tp": 1, "fp": 2, "fn": 1}, "all": {"tp": 1, "fp": 2, "fn": 1}}, "claude": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "claude-code": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "cubic-v2": {"strict": {"tp": 2, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "devin": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "graphite": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "kodus-v2": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "macroscope": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "qodo-v2": {"strict": {"tp": 2, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "sourcery": {"strict": {"tp": 1, "fp": 2, "fn": 1}, "core": {"tp": 1, "fp": 2, "fn": 1}, "all": {"tp": 1, "fp": 2, "fn": 1}}}}, {"url": "https://github.com/ai-code-review-evaluation/keycloak-greptile/pull/1", "language": "Java", "pr_size": "medium", "domain": "authentication", "change_type": "bug_fix", "complexity": "moderate", "difficulty": "subtle", "risk": "medium", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "codeant-v2": {"strict": {"tp": 1, "fp": 2, "fn": 0}, "core": {"tp": 1, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "coderabbit": {"strict": {"tp": 1, "fp": 3, "fn": 0}, "core": {"tp": 1, "fp": 3, "fn": 0}, "all": {"tp": 1, "fp": 3, "fn": 1}}, "qodo-extended-v2": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 1, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "greptile-v4-1": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 1, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "gemini-v2": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 1, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "gitlab": {"strict": {"tp": 1, "fp": 3, "fn": 0}, "core": {"tp": 1, "fp": 3, "fn": 0}, "all": {"tp": 1, "fp": 3, "fn": 1}}, "copilot-v2": {"strict": {"tp": 1, "fp": 2, "fn": 0}, "core": {"tp": 1, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "augment": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "bugbot": {"strict": {"tp": 0, "fp": 1, "fn": 1}, "core": {"tp": 0, "fp": 1, "fn": 1}, "all": {"tp": 0, "fp": 1, "fn": 2}}, "claude": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 0}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "claude-code": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 0}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "cubic-v2": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 1, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "devin": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "kg": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 0}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "kodus-v2": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "macroscope": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 1, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "qodo-v2": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 1, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "sourcery": {"strict": {"tp": 0, "fp": 3, "fn": 1}, "core": {"tp": 0, "fp": 3, "fn": 1}, "all": {"tp": 1, "fp": 3, "fn": 1}}}}, {"url": "https://github.com/getsentry/sentry/pull/93824", "language": "Python", "pr_size": "large", "domain": "concurrency", "change_type": "feature", "complexity": "complex", "difficulty": "subtle", "risk": "high", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 0, "fp": 3, "fn": 3}, "core": {"tp": 0, "fp": 3, "fn": 5}, "all": {"tp": 0, "fp": 3, "fn": 5}}, "codeant-v2": {"strict": {"tp": 0, "fp": 0, "fn": 3}, "core": {"tp": 2, "fp": 0, "fn": 3}, "all": {"tp": 2, "fp": 0, "fn": 3}}, "coderabbit": {"strict": {"tp": 0, "fp": 1, "fn": 3}, "core": {"tp": 0, "fp": 1, "fn": 5}, "all": {"tp": 0, "fp": 1, "fn": 5}}, "qodo-extended-v2": {"strict": {"tp": 2, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 3}, "all": {"tp": 2, "fp": 1, "fn": 3}}, "greptile-v4-1": {"strict": {"tp": 1, "fp": 1, "fn": 2}, "core": {"tp": 1, "fp": 1, "fn": 4}, "all": {"tp": 1, "fp": 1, "fn": 4}}, "gemini-v2": {"strict": {"tp": 2, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 3}, "all": {"tp": 2, "fp": 1, "fn": 3}}, "gitlab": {"strict": {"tp": 0, "fp": 2, "fn": 3}, "core": {"tp": 0, "fp": 2, "fn": 5}, "all": {"tp": 0, "fp": 2, "fn": 5}}, "copilot-v2": {"strict": {"tp": 1, "fp": 4, "fn": 2}, "core": {"tp": 3, "fp": 4, "fn": 2}, "all": {"tp": 3, "fp": 4, "fn": 2}}, "augment": {"strict": {"tp": 3, "fp": 1, "fn": 0}, "core": {"tp": 5, "fp": 1, "fn": 0}, "all": {"tp": 5, "fp": 1, "fn": 0}}, "bugbot": {"strict": {"tp": 1, "fp": 0, "fn": 2}, "core": {"tp": 1, "fp": 0, "fn": 4}, "all": {"tp": 1, "fp": 0, "fn": 4}}, "claude": {"strict": {"tp": 0, "fp": 2, "fn": 3}, "core": {"tp": 0, "fp": 2, "fn": 5}, "all": {"tp": 0, "fp": 2, "fn": 5}}, "claude-code": {"strict": {"tp": 3, "fp": 3, "fn": 0}, "core": {"tp": 3, "fp": 3, "fn": 2}, "all": {"tp": 3, "fp": 3, "fn": 2}}, "cubic-v2": {"strict": {"tp": 3, "fp": 1, "fn": 0}, "core": {"tp": 5, "fp": 1, "fn": 0}, "all": {"tp": 5, "fp": 1, "fn": 0}}, "devin": {"strict": {"tp": 1, "fp": 0, "fn": 2}, "core": {"tp": 1, "fp": 0, "fn": 4}, "all": {"tp": 1, "fp": 0, "fn": 4}}, "graphite": {"strict": {"tp": 1, "fp": 0, "fn": 2}, "core": {"tp": 1, "fp": 0, "fn": 4}, "all": {"tp": 1, "fp": 0, "fn": 4}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 3}, "core": {"tp": 0, "fp": 0, "fn": 5}, "all": {"tp": 0, "fp": 0, "fn": 5}}, "kodus-v2": {"strict": {"tp": 0, "fp": 0, "fn": 3}, "core": {"tp": 0, "fp": 0, "fn": 5}, "all": {"tp": 0, "fp": 0, "fn": 5}}, "macroscope": {"strict": {"tp": 1, "fp": 1, "fn": 2}, "core": {"tp": 3, "fp": 1, "fn": 2}, "all": {"tp": 3, "fp": 1, "fn": 2}}, "qodo-v2": {"strict": {"tp": 2, "fp": 3, "fn": 1}, "core": {"tp": 4, "fp": 3, "fn": 1}, "all": {"tp": 4, "fp": 3, "fn": 1}}, "sourcery": {"strict": {"tp": 2, "fp": 2, "fn": 1}, "core": {"tp": 4, "fp": 2, "fn": 1}, "all": {"tp": 4, "fp": 2, "fn": 1}}}}, {"url": "https://github.com/ai-code-review-evaluation/sentry-greptile/pull/5", "language": "Python", "pr_size": "large", "domain": "API", "change_type": "feature", "complexity": "moderate", "difficulty": "subtle", "risk": "medium", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 3, "fp": 1, "fn": 3}, "core": {"tp": 3, "fp": 1, "fn": 3}, "all": {"tp": 3, "fp": 1, "fn": 3}}, "codeant-v2": {"strict": {"tp": 2, "fp": 5, "fn": 4}, "core": {"tp": 2, "fp": 5, "fn": 4}, "all": {"tp": 2, "fp": 5, "fn": 4}}, "coderabbit": {"strict": {"tp": 4, "fp": 13, "fn": 2}, "core": {"tp": 4, "fp": 13, "fn": 2}, "all": {"tp": 4, "fp": 13, "fn": 2}}, "qodo-extended-v2": {"strict": {"tp": 2, "fp": 0, "fn": 4}, "core": {"tp": 2, "fp": 0, "fn": 4}, "all": {"tp": 2, "fp": 0, "fn": 4}}, "greptile-v4-1": {"strict": {"tp": 1, "fp": 0, "fn": 5}, "core": {"tp": 1, "fp": 0, "fn": 5}, "all": {"tp": 1, "fp": 0, "fn": 5}}, "gemini-v2": {"strict": {"tp": 2, "fp": 1, "fn": 4}, "core": {"tp": 2, "fp": 1, "fn": 4}, "all": {"tp": 2, "fp": 1, "fn": 4}}, "gitlab": {"strict": {"tp": 2, "fp": 1, "fn": 4}, "core": {"tp": 2, "fp": 1, "fn": 4}, "all": {"tp": 2, "fp": 1, "fn": 4}}, "copilot-v2": {"strict": {"tp": 3, "fp": 5, "fn": 3}, "core": {"tp": 3, "fp": 5, "fn": 3}, "all": {"tp": 3, "fp": 5, "fn": 3}}, "augment": {"strict": {"tp": 3, "fp": 2, "fn": 3}, "core": {"tp": 3, "fp": 2, "fn": 3}, "all": {"tp": 3, "fp": 2, "fn": 3}}, "bugbot": {"strict": {"tp": 2, "fp": 0, "fn": 4}, "core": {"tp": 2, "fp": 0, "fn": 4}, "all": {"tp": 2, "fp": 0, "fn": 4}}, "claude": {"strict": {"tp": 0, "fp": 0, "fn": 6}, "core": {"tp": 0, "fp": 0, "fn": 6}, "all": {"tp": 0, "fp": 0, "fn": 6}}, "claude-code": {"strict": {"tp": 0, "fp": 2, "fn": 6}, "core": {"tp": 0, "fp": 2, "fn": 6}, "all": {"tp": 0, "fp": 2, "fn": 6}}, "cubic-v2": {"strict": {"tp": 2, "fp": 2, "fn": 4}, "core": {"tp": 2, "fp": 2, "fn": 4}, "all": {"tp": 2, "fp": 2, "fn": 4}}, "devin": {"strict": {"tp": 2, "fp": 1, "fn": 4}, "core": {"tp": 2, "fp": 1, "fn": 4}, "all": {"tp": 2, "fp": 1, "fn": 4}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 6}, "core": {"tp": 0, "fp": 0, "fn": 6}, "all": {"tp": 0, "fp": 0, "fn": 6}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 6}, "core": {"tp": 0, "fp": 0, "fn": 6}, "all": {"tp": 0, "fp": 0, "fn": 6}}, "kodus-v2": {"strict": {"tp": 2, "fp": 1, "fn": 4}, "core": {"tp": 2, "fp": 1, "fn": 4}, "all": {"tp": 2, "fp": 1, "fn": 4}}, "macroscope": {"strict": {"tp": 2, "fp": 3, "fn": 4}, "core": {"tp": 2, "fp": 3, "fn": 4}, "all": {"tp": 2, "fp": 3, "fn": 4}}, "qodo-v2": {"strict": {"tp": 2, "fp": 0, "fn": 4}, "core": {"tp": 2, "fp": 0, "fn": 4}, "all": {"tp": 2, "fp": 0, "fn": 4}}, "sourcery": {"strict": {"tp": 2, "fp": 2, "fn": 4}, "core": {"tp": 2, "fp": 2, "fn": 4}, "all": {"tp": 2, "fp": 2, "fn": 4}}}}, {"url": "https://github.com/ai-code-review-evaluation/sentry-greptile/pull/1", "language": "Python", "pr_size": "medium", "domain": "API", "change_type": "performance", "complexity": "moderate", "difficulty": "subtle", "risk": "high", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 1, "fp": 1, "fn": 3}, "core": {"tp": 1, "fp": 1, "fn": 3}, "all": {"tp": 1, "fp": 1, "fn": 3}}, "codeant-v2": {"strict": {"tp": 1, "fp": 0, "fn": 3}, "core": {"tp": 1, "fp": 0, "fn": 3}, "all": {"tp": 1, "fp": 0, "fn": 3}}, "coderabbit": {"strict": {"tp": 3, "fp": 1, "fn": 1}, "core": {"tp": 3, "fp": 1, "fn": 1}, "all": {"tp": 3, "fp": 1, "fn": 1}}, "qodo-extended-v2": {"strict": {"tp": 4, "fp": 0, "fn": 0}, "core": {"tp": 4, "fp": 0, "fn": 0}, "all": {"tp": 4, "fp": 0, "fn": 0}}, "greptile-v4-1": {"strict": {"tp": 3, "fp": 2, "fn": 1}, "core": {"tp": 3, "fp": 2, "fn": 1}, "all": {"tp": 3, "fp": 2, "fn": 1}}, "gemini-v2": {"strict": {"tp": 3, "fp": 1, "fn": 1}, "core": {"tp": 3, "fp": 1, "fn": 1}, "all": {"tp": 3, "fp": 1, "fn": 1}}, "gitlab": {"strict": {"tp": 3, "fp": 1, "fn": 1}, "core": {"tp": 3, "fp": 1, "fn": 1}, "all": {"tp": 3, "fp": 1, "fn": 1}}, "copilot-v2": {"strict": {"tp": 2, "fp": 1, "fn": 2}, "core": {"tp": 2, "fp": 1, "fn": 2}, "all": {"tp": 2, "fp": 1, "fn": 2}}, "augment": {"strict": {"tp": 3, "fp": 1, "fn": 1}, "core": {"tp": 3, "fp": 1, "fn": 1}, "all": {"tp": 3, "fp": 1, "fn": 1}}, "bugbot": {"strict": {"tp": 3, "fp": 1, "fn": 1}, "core": {"tp": 3, "fp": 1, "fn": 1}, "all": {"tp": 3, "fp": 1, "fn": 1}}, "claude": {"strict": {"tp": 2, "fp": 4, "fn": 2}, "core": {"tp": 2, "fp": 4, "fn": 2}, "all": {"tp": 2, "fp": 4, "fn": 2}}, "claude-code": {"strict": {"tp": 2, "fp": 1, "fn": 2}, "core": {"tp": 2, "fp": 1, "fn": 2}, "all": {"tp": 2, "fp": 1, "fn": 2}}, "cubic-v2": {"strict": {"tp": 3, "fp": 0, "fn": 1}, "core": {"tp": 3, "fp": 0, "fn": 1}, "all": {"tp": 3, "fp": 0, "fn": 1}}, "devin": {"strict": {"tp": 3, "fp": 1, "fn": 1}, "core": {"tp": 3, "fp": 1, "fn": 1}, "all": {"tp": 3, "fp": 1, "fn": 1}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 4}, "core": {"tp": 0, "fp": 0, "fn": 4}, "all": {"tp": 0, "fp": 0, "fn": 4}}, "kg": {"strict": {"tp": 1, "fp": 1, "fn": 3}, "core": {"tp": 1, "fp": 1, "fn": 3}, "all": {"tp": 1, "fp": 1, "fn": 3}}, "kodus-v2": {"strict": {"tp": 1, "fp": 4, "fn": 3}, "core": {"tp": 1, "fp": 4, "fn": 3}, "all": {"tp": 1, "fp": 4, "fn": 3}}, "macroscope": {"strict": {"tp": 2, "fp": 0, "fn": 2}, "core": {"tp": 2, "fp": 0, "fn": 2}, "all": {"tp": 2, "fp": 0, "fn": 2}}, "qodo-v2": {"strict": {"tp": 3, "fp": 0, "fn": 1}, "core": {"tp": 3, "fp": 0, "fn": 1}, "all": {"tp": 3, "fp": 0, "fn": 1}}, "sourcery": {"strict": {"tp": 2, "fp": 3, "fn": 2}, "core": {"tp": 2, "fp": 3, "fn": 2}, "all": {"tp": 2, "fp": 3, "fn": 2}}}}, {"url": "https://github.com/grafana/grafana/pull/97529", "language": "Go", "pr_size": "small", "domain": "concurrency", "change_type": "performance", "complexity": "complex", "difficulty": "very_subtle", "risk": "critical", "context": "cross_file", "concern": "reliability", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "codeant-v2": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "coderabbit": {"strict": {"tp": 1, "fp": 3, "fn": 1}, "core": {"tp": 1, "fp": 3, "fn": 1}, "all": {"tp": 1, "fp": 3, "fn": 1}}, "qodo-extended-v2": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "greptile-v4-1": {"strict": {"tp": 2, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "gemini-v2": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "gitlab": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "copilot-v2": {"strict": {"tp": 1, "fp": 3, "fn": 1}, "core": {"tp": 1, "fp": 3, "fn": 1}, "all": {"tp": 1, "fp": 3, "fn": 1}}, "augment": {"strict": {"tp": 2, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "bugbot": {"strict": {"tp": 1, "fp": 2, "fn": 1}, "core": {"tp": 1, "fp": 2, "fn": 1}, "all": {"tp": 1, "fp": 2, "fn": 1}}, "claude": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "claude-code": {"strict": {"tp": 1, "fp": 4, "fn": 1}, "core": {"tp": 1, "fp": 4, "fn": 1}, "all": {"tp": 1, "fp": 4, "fn": 1}}, "cubic-v2": {"strict": {"tp": 2, "fp": 2, "fn": 0}, "core": {"tp": 2, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "devin": {"strict": {"tp": 0, "fp": 2, "fn": 2}, "core": {"tp": 0, "fp": 2, "fn": 2}, "all": {"tp": 0, "fp": 2, "fn": 2}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "kg": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "kodus-v2": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "macroscope": {"strict": {"tp": 2, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "qodo-v2": {"strict": {"tp": 2, "fp": 3, "fn": 0}, "core": {"tp": 2, "fp": 3, "fn": 0}, "all": {"tp": 2, "fp": 3, "fn": 0}}, "sourcery": {"strict": {"tp": 2, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}}}, {"url": "https://github.com/getsentry/sentry/pull/80168", "language": "Python", "pr_size": "small", "domain": "data_processing", "change_type": "feature", "complexity": "moderate", "difficulty": "moderate", "risk": "high", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 0, "fp": 1, "fn": 1}, "core": {"tp": 0, "fp": 1, "fn": 3}, "all": {"tp": 0, "fp": 1, "fn": 3}}, "codeant-v2": {"strict": {"tp": 0, "fp": 1, "fn": 1}, "core": {"tp": 0, "fp": 1, "fn": 3}, "all": {"tp": 0, "fp": 1, "fn": 3}}, "coderabbit": {"strict": {"tp": 0, "fp": 2, "fn": 1}, "core": {"tp": 2, "fp": 2, "fn": 1}, "all": {"tp": 2, "fp": 2, "fn": 1}}, "qodo-extended-v2": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 1}, "all": {"tp": 2, "fp": 0, "fn": 1}}, "greptile-v4-1": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 1}, "all": {"tp": 2, "fp": 1, "fn": 1}}, "gemini-v2": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 2}}, "gitlab": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 2}}, "copilot-v2": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 1}, "all": {"tp": 2, "fp": 0, "fn": 1}}, "augment": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 3, "fp": 0, "fn": 0}, "all": {"tp": 3, "fp": 0, "fn": 0}}, "bugbot": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 2}}, "claude": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 3}}, "claude-code": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 3, "fp": 1, "fn": 0}, "all": {"tp": 3, "fp": 1, "fn": 0}}, "cubic-v2": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 3, "fp": 1, "fn": 0}, "all": {"tp": 3, "fp": 1, "fn": 0}}, "devin": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 1, "fp": 1, "fn": 2}, "all": {"tp": 1, "fp": 1, "fn": 2}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 3}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 3}}, "kodus-v2": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 3}}, "macroscope": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 3}}, "qodo-v2": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 2}}, "sourcery": {"strict": {"tp": 1, "fp": 2, "fn": 0}, "core": {"tp": 2, "fp": 2, "fn": 1}, "all": {"tp": 2, "fp": 2, "fn": 1}}}}, {"url": "https://github.com/getsentry/sentry/pull/80528", "language": "Python", "pr_size": "small", "domain": "scheduling", "change_type": "refactoring", "complexity": "moderate", "difficulty": "subtle", "risk": "medium", "context": "file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "codeant-v2": {"strict": {"tp": 0, "fp": 1, "fn": 1}, "core": {"tp": 0, "fp": 1, "fn": 2}, "all": {"tp": 0, "fp": 1, "fn": 2}}, "coderabbit": {"strict": {"tp": 1, "fp": 3, "fn": 0}, "core": {"tp": 1, "fp": 3, "fn": 1}, "all": {"tp": 1, "fp": 3, "fn": 1}}, "qodo-extended-v2": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "greptile-v4-1": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "gemini-v2": {"strict": {"tp": 1, "fp": 5, "fn": 0}, "core": {"tp": 1, "fp": 5, "fn": 1}, "all": {"tp": 1, "fp": 5, "fn": 1}}, "gitlab": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "copilot-v2": {"strict": {"tp": 1, "fp": 2, "fn": 0}, "core": {"tp": 1, "fp": 2, "fn": 1}, "all": {"tp": 1, "fp": 2, "fn": 1}}, "augment": {"strict": {"tp": 1, "fp": 2, "fn": 0}, "core": {"tp": 1, "fp": 2, "fn": 1}, "all": {"tp": 1, "fp": 2, "fn": 1}}, "bugbot": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "claude": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "claude-code": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "cubic-v2": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "devin": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "graphite": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "kg": {"strict": {"tp": 0, "fp": 1, "fn": 1}, "core": {"tp": 0, "fp": 1, "fn": 2}, "all": {"tp": 0, "fp": 1, "fn": 2}}, "kodus-v2": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "macroscope": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "qodo-v2": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "sourcery": {"strict": {"tp": 1, "fp": 2, "fn": 0}, "core": {"tp": 1, "fp": 2, "fn": 1}, "all": {"tp": 1, "fp": 2, "fn": 1}}}}, {"url": "https://github.com/getsentry/sentry/pull/77754", "language": "Python", "pr_size": "medium", "domain": "serialization", "change_type": "feature", "complexity": "moderate", "difficulty": "moderate", "risk": "medium", "context": "file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 3}}, "codeant-v2": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 4}}, "coderabbit": {"strict": {"tp": 1, "fp": 2, "fn": 1}, "core": {"tp": 1, "fp": 2, "fn": 1}, "all": {"tp": 3, "fp": 2, "fn": 1}}, "qodo-extended-v2": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 3}}, "greptile-v4-1": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 3}}, "gemini-v2": {"strict": {"tp": 1, "fp": 2, "fn": 1}, "core": {"tp": 1, "fp": 2, "fn": 1}, "all": {"tp": 1, "fp": 2, "fn": 3}}, "gitlab": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 3, "fp": 0, "fn": 1}}, "copilot-v2": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 2, "fp": 1, "fn": 2}}, "augment": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 3}}, "bugbot": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 3}}, "claude": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 2, "fp": 0, "fn": 2}}, "claude-code": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 3}}, "cubic-v2": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 2}}, "devin": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 3}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 4}}, "kg": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 3}}, "kodus-v2": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 3}}, "macroscope": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 3}}, "qodo-v2": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 3}}, "sourcery": {"strict": {"tp": 1, "fp": 3, "fn": 1}, "core": {"tp": 1, "fp": 3, "fn": 1}, "all": {"tp": 2, "fp": 3, "fn": 2}}}}, {"url": "https://github.com/getsentry/sentry/pull/95633", "language": "Python", "pr_size": "medium", "domain": "concurrency", "change_type": "feature", "complexity": "moderate", "difficulty": "moderate", "risk": "medium", "context": "file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 0, "fp": 2, "fn": 0}, "core": {"tp": 0, "fp": 2, "fn": 1}, "all": {"tp": 0, "fp": 2, "fn": 2}}, "codeant-v2": {"strict": {"tp": 0, "fp": 1, "fn": 0}, "core": {"tp": 0, "fp": 1, "fn": 1}, "all": {"tp": 0, "fp": 1, "fn": 2}}, "coderabbit": {"strict": {"tp": 0, "fp": 7, "fn": 0}, "core": {"tp": 0, "fp": 7, "fn": 1}, "all": {"tp": 0, "fp": 7, "fn": 2}}, "qodo-extended-v2": {"strict": {"tp": 0, "fp": 4, "fn": 0}, "core": {"tp": 1, "fp": 4, "fn": 0}, "all": {"tp": 1, "fp": 4, "fn": 1}}, "greptile-v4-1": {"strict": {"tp": 0, "fp": 3, "fn": 0}, "core": {"tp": 0, "fp": 3, "fn": 1}, "all": {"tp": 0, "fp": 3, "fn": 2}}, "gemini-v2": {"strict": {"tp": 0, "fp": 5, "fn": 0}, "core": {"tp": 0, "fp": 5, "fn": 1}, "all": {"tp": 0, "fp": 5, "fn": 2}}, "gitlab": {"strict": {"tp": 0, "fp": 5, "fn": 0}, "core": {"tp": 1, "fp": 5, "fn": 0}, "all": {"tp": 1, "fp": 5, "fn": 1}}, "copilot-v2": {"strict": {"tp": 0, "fp": 3, "fn": 0}, "core": {"tp": 0, "fp": 3, "fn": 1}, "all": {"tp": 0, "fp": 3, "fn": 2}}, "augment": {"strict": {"tp": 0, "fp": 3, "fn": 0}, "core": {"tp": 0, "fp": 3, "fn": 1}, "all": {"tp": 0, "fp": 3, "fn": 2}}, "bugbot": {"strict": {"tp": 0, "fp": 2, "fn": 0}, "core": {"tp": 0, "fp": 2, "fn": 1}, "all": {"tp": 0, "fp": 2, "fn": 2}}, "claude": {"strict": {"tp": 0, "fp": 0, "fn": 0}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "claude-code": {"strict": {"tp": 0, "fp": 2, "fn": 0}, "core": {"tp": 0, "fp": 2, "fn": 1}, "all": {"tp": 0, "fp": 2, "fn": 2}}, "cubic-v2": {"strict": {"tp": 0, "fp": 3, "fn": 0}, "core": {"tp": 0, "fp": 3, "fn": 1}, "all": {"tp": 0, "fp": 3, "fn": 2}}, "devin": {"strict": {"tp": 0, "fp": 2, "fn": 0}, "core": {"tp": 0, "fp": 2, "fn": 1}, "all": {"tp": 0, "fp": 2, "fn": 2}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 0}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "kg": {"strict": {"tp": 0, "fp": 2, "fn": 0}, "core": {"tp": 0, "fp": 2, "fn": 1}, "all": {"tp": 0, "fp": 2, "fn": 2}}, "kodus-v2": {"strict": {"tp": 0, "fp": 0, "fn": 0}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "macroscope": {"strict": {"tp": 0, "fp": 2, "fn": 0}, "core": {"tp": 0, "fp": 2, "fn": 1}, "all": {"tp": 0, "fp": 2, "fn": 2}}, "qodo-v2": {"strict": {"tp": 0, "fp": 4, "fn": 0}, "core": {"tp": 0, "fp": 4, "fn": 1}, "all": {"tp": 0, "fp": 4, "fn": 2}}, "sourcery": {"strict": {"tp": 0, "fp": 4, "fn": 0}, "core": {"tp": 1, "fp": 4, "fn": 0}, "all": {"tp": 1, "fp": 4, "fn": 1}}}}, {"url": "https://github.com/ai-code-review-evaluation/sentry-greptile/pull/2", "language": "Python", "pr_size": "medium", "domain": "data_processing", "change_type": "performance", "complexity": "moderate", "difficulty": "subtle", "risk": "high", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 3, "fp": 1, "fn": 1}, "core": {"tp": 3, "fp": 1, "fn": 1}, "all": {"tp": 3, "fp": 1, "fn": 1}}, "codeant-v2": {"strict": {"tp": 3, "fp": 1, "fn": 1}, "core": {"tp": 3, "fp": 1, "fn": 1}, "all": {"tp": 3, "fp": 1, "fn": 1}}, "coderabbit": {"strict": {"tp": 0, "fp": 0, "fn": 4}, "core": {"tp": 0, "fp": 0, "fn": 4}, "all": {"tp": 0, "fp": 0, "fn": 4}}, "qodo-extended-v2": {"strict": {"tp": 3, "fp": 3, "fn": 1}, "core": {"tp": 3, "fp": 3, "fn": 1}, "all": {"tp": 3, "fp": 3, "fn": 1}}, "greptile-v4-1": {"strict": {"tp": 3, "fp": 2, "fn": 1}, "core": {"tp": 3, "fp": 2, "fn": 1}, "all": {"tp": 3, "fp": 2, "fn": 1}}, "gemini-v2": {"strict": {"tp": 3, "fp": 3, "fn": 1}, "core": {"tp": 3, "fp": 3, "fn": 1}, "all": {"tp": 3, "fp": 3, "fn": 1}}, "gitlab": {"strict": {"tp": 4, "fp": 1, "fn": 0}, "core": {"tp": 4, "fp": 1, "fn": 0}, "all": {"tp": 4, "fp": 1, "fn": 0}}, "copilot-v2": {"strict": {"tp": 3, "fp": 8, "fn": 1}, "core": {"tp": 3, "fp": 8, "fn": 1}, "all": {"tp": 3, "fp": 8, "fn": 1}}, "augment": {"strict": {"tp": 3, "fp": 2, "fn": 1}, "core": {"tp": 3, "fp": 2, "fn": 1}, "all": {"tp": 3, "fp": 2, "fn": 1}}, "bugbot": {"strict": {"tp": 4, "fp": 0, "fn": 0}, "core": {"tp": 4, "fp": 0, "fn": 0}, "all": {"tp": 4, "fp": 0, "fn": 0}}, "claude": {"strict": {"tp": 3, "fp": 5, "fn": 1}, "core": {"tp": 3, "fp": 5, "fn": 1}, "all": {"tp": 3, "fp": 5, "fn": 1}}, "claude-code": {"strict": {"tp": 3, "fp": 3, "fn": 1}, "core": {"tp": 3, "fp": 3, "fn": 1}, "all": {"tp": 3, "fp": 3, "fn": 1}}, "cubic-v2": {"strict": {"tp": 3, "fp": 1, "fn": 1}, "core": {"tp": 3, "fp": 1, "fn": 1}, "all": {"tp": 3, "fp": 1, "fn": 1}}, "devin": {"strict": {"tp": 0, "fp": 0, "fn": 4}, "core": {"tp": 0, "fp": 0, "fn": 4}, "all": {"tp": 0, "fp": 0, "fn": 4}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 4}, "core": {"tp": 0, "fp": 0, "fn": 4}, "all": {"tp": 0, "fp": 0, "fn": 4}}, "kg": {"strict": {"tp": 2, "fp": 2, "fn": 2}, "core": {"tp": 2, "fp": 2, "fn": 2}, "all": {"tp": 2, "fp": 2, "fn": 2}}, "kodus-v2": {"strict": {"tp": 2, "fp": 3, "fn": 2}, "core": {"tp": 2, "fp": 3, "fn": 2}, "all": {"tp": 2, "fp": 3, "fn": 2}}, "macroscope": {"strict": {"tp": 3, "fp": 0, "fn": 1}, "core": {"tp": 3, "fp": 0, "fn": 1}, "all": {"tp": 3, "fp": 0, "fn": 1}}, "qodo-v2": {"strict": {"tp": 4, "fp": 0, "fn": 0}, "core": {"tp": 4, "fp": 0, "fn": 0}, "all": {"tp": 4, "fp": 0, "fn": 0}}, "sourcery": {"strict": {"tp": 3, "fp": 7, "fn": 1}, "core": {"tp": 3, "fp": 7, "fn": 1}, "all": {"tp": 3, "fp": 7, "fn": 1}}}}, {"url": "https://github.com/ai-code-review-evaluation/sentry-greptile/pull/3", "language": "Python", "pr_size": "medium", "domain": "caching", "change_type": "feature", "complexity": "complex", "difficulty": "subtle", "risk": "medium", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 1, "fp": 1, "fn": 2}, "core": {"tp": 1, "fp": 1, "fn": 2}, "all": {"tp": 1, "fp": 1, "fn": 2}}, "codeant-v2": {"strict": {"tp": 3, "fp": 1, "fn": 0}, "core": {"tp": 3, "fp": 1, "fn": 0}, "all": {"tp": 3, "fp": 1, "fn": 0}}, "coderabbit": {"strict": {"tp": 2, "fp": 5, "fn": 1}, "core": {"tp": 2, "fp": 5, "fn": 1}, "all": {"tp": 2, "fp": 5, "fn": 1}}, "qodo-extended-v2": {"strict": {"tp": 3, "fp": 0, "fn": 0}, "core": {"tp": 3, "fp": 0, "fn": 0}, "all": {"tp": 3, "fp": 0, "fn": 0}}, "greptile-v4-1": {"strict": {"tp": 2, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 1}, "all": {"tp": 2, "fp": 1, "fn": 1}}, "gemini-v2": {"strict": {"tp": 2, "fp": 2, "fn": 1}, "core": {"tp": 2, "fp": 2, "fn": 1}, "all": {"tp": 2, "fp": 2, "fn": 1}}, "gitlab": {"strict": {"tp": 2, "fp": 2, "fn": 1}, "core": {"tp": 2, "fp": 2, "fn": 1}, "all": {"tp": 2, "fp": 2, "fn": 1}}, "copilot-v2": {"strict": {"tp": 1, "fp": 6, "fn": 2}, "core": {"tp": 1, "fp": 6, "fn": 2}, "all": {"tp": 1, "fp": 6, "fn": 2}}, "augment": {"strict": {"tp": 2, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 1}, "all": {"tp": 2, "fp": 1, "fn": 1}}, "bugbot": {"strict": {"tp": 0, "fp": 3, "fn": 3}, "core": {"tp": 0, "fp": 3, "fn": 3}, "all": {"tp": 0, "fp": 3, "fn": 3}}, "claude": {"strict": {"tp": 1, "fp": 0, "fn": 2}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 2}}, "claude-code": {"strict": {"tp": 1, "fp": 2, "fn": 2}, "core": {"tp": 1, "fp": 2, "fn": 2}, "all": {"tp": 1, "fp": 2, "fn": 2}}, "cubic-v2": {"strict": {"tp": 2, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 1}, "all": {"tp": 2, "fp": 1, "fn": 1}}, "devin": {"strict": {"tp": 2, "fp": 0, "fn": 1}, "core": {"tp": 2, "fp": 0, "fn": 1}, "all": {"tp": 2, "fp": 0, "fn": 1}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 3}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 3}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 3}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 3}}, "kodus-v2": {"strict": {"tp": 2, "fp": 2, "fn": 1}, "core": {"tp": 2, "fp": 2, "fn": 1}, "all": {"tp": 2, "fp": 2, "fn": 1}}, "macroscope": {"strict": {"tp": 2, "fp": 0, "fn": 1}, "core": {"tp": 2, "fp": 0, "fn": 1}, "all": {"tp": 2, "fp": 0, "fn": 1}}, "qodo-v2": {"strict": {"tp": 2, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 1}, "all": {"tp": 2, "fp": 1, "fn": 1}}, "sourcery": {"strict": {"tp": 2, "fp": 6, "fn": 1}, "core": {"tp": 2, "fp": 6, "fn": 1}, "all": {"tp": 2, "fp": 6, "fn": 1}}}}, {"url": "https://github.com/grafana/grafana/pull/103633", "language": "Go", "pr_size": "small", "domain": "caching", "change_type": "performance", "complexity": "moderate", "difficulty": "subtle", "risk": "high", "context": "cross_file", "concern": "security", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 2}, "all": {"tp": 1, "fp": 1, "fn": 2}}, "codeant-v2": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 2, "fp": 0, "fn": 1}, "all": {"tp": 2, "fp": 0, "fn": 1}}, "coderabbit": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 2}}, "qodo-extended-v2": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 2, "fp": 0, "fn": 1}, "all": {"tp": 2, "fp": 0, "fn": 1}}, "greptile-v4-1": {"strict": {"tp": 1, "fp": 2, "fn": 1}, "core": {"tp": 1, "fp": 2, "fn": 2}, "all": {"tp": 1, "fp": 2, "fn": 2}}, "gemini-v2": {"strict": {"tp": 0, "fp": 1, "fn": 2}, "core": {"tp": 0, "fp": 1, "fn": 3}, "all": {"tp": 0, "fp": 1, "fn": 3}}, "gitlab": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 2}, "all": {"tp": 1, "fp": 1, "fn": 2}}, "copilot-v2": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 1}, "all": {"tp": 2, "fp": 1, "fn": 1}}, "augment": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 2, "fp": 0, "fn": 1}, "all": {"tp": 2, "fp": 0, "fn": 1}}, "bugbot": {"strict": {"tp": 0, "fp": 1, "fn": 2}, "core": {"tp": 0, "fp": 1, "fn": 3}, "all": {"tp": 0, "fp": 1, "fn": 3}}, "claude": {"strict": {"tp": 0, "fp": 6, "fn": 2}, "core": {"tp": 0, "fp": 6, "fn": 3}, "all": {"tp": 0, "fp": 6, "fn": 3}}, "claude-code": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 2}, "all": {"tp": 1, "fp": 1, "fn": 2}}, "cubic-v2": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 3, "fp": 0, "fn": 0}, "all": {"tp": 3, "fp": 0, "fn": 0}}, "devin": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 3, "fp": 0, "fn": 0}, "all": {"tp": 3, "fp": 0, "fn": 0}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 3}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 3}}, "kodus-v2": {"strict": {"tp": 0, "fp": 1, "fn": 2}, "core": {"tp": 0, "fp": 1, "fn": 3}, "all": {"tp": 0, "fp": 1, "fn": 3}}, "macroscope": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 2}}, "qodo-v2": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 1}, "all": {"tp": 2, "fp": 1, "fn": 1}}, "sourcery": {"strict": {"tp": 0, "fp": 2, "fn": 2}, "core": {"tp": 0, "fp": 2, "fn": 3}, "all": {"tp": 0, "fp": 2, "fn": 3}}}}, {"url": "https://github.com/getsentry/sentry/pull/67876", "language": "Python", "pr_size": "small", "domain": "authentication", "change_type": "security_patch", "complexity": "moderate", "difficulty": "subtle", "risk": "high", "context": "cross_file", "concern": "security", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 1, "fp": 1, "fn": 2}, "core": {"tp": 1, "fp": 1, "fn": 2}, "all": {"tp": 1, "fp": 1, "fn": 2}}, "codeant-v2": {"strict": {"tp": 0, "fp": 1, "fn": 3}, "core": {"tp": 0, "fp": 1, "fn": 3}, "all": {"tp": 0, "fp": 1, "fn": 3}}, "coderabbit": {"strict": {"tp": 2, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 1}, "all": {"tp": 2, "fp": 1, "fn": 1}}, "qodo-extended-v2": {"strict": {"tp": 2, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 1}, "all": {"tp": 2, "fp": 1, "fn": 1}}, "greptile-v4-1": {"strict": {"tp": 1, "fp": 1, "fn": 2}, "core": {"tp": 1, "fp": 1, "fn": 2}, "all": {"tp": 1, "fp": 1, "fn": 2}}, "gemini-v2": {"strict": {"tp": 1, "fp": 1, "fn": 2}, "core": {"tp": 1, "fp": 1, "fn": 2}, "all": {"tp": 1, "fp": 1, "fn": 2}}, "gitlab": {"strict": {"tp": 1, "fp": 1, "fn": 2}, "core": {"tp": 1, "fp": 1, "fn": 2}, "all": {"tp": 1, "fp": 1, "fn": 2}}, "copilot-v2": {"strict": {"tp": 2, "fp": 13, "fn": 1}, "core": {"tp": 2, "fp": 13, "fn": 1}, "all": {"tp": 2, "fp": 13, "fn": 1}}, "augment": {"strict": {"tp": 1, "fp": 2, "fn": 2}, "core": {"tp": 1, "fp": 2, "fn": 2}, "all": {"tp": 1, "fp": 2, "fn": 2}}, "bugbot": {"strict": {"tp": 1, "fp": 2, "fn": 2}, "core": {"tp": 1, "fp": 2, "fn": 2}, "all": {"tp": 1, "fp": 2, "fn": 2}}, "claude": {"strict": {"tp": 1, "fp": 1, "fn": 2}, "core": {"tp": 1, "fp": 1, "fn": 2}, "all": {"tp": 1, "fp": 1, "fn": 2}}, "claude-code": {"strict": {"tp": 2, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 1}, "all": {"tp": 2, "fp": 1, "fn": 1}}, "cubic-v2": {"strict": {"tp": 3, "fp": 2, "fn": 0}, "core": {"tp": 3, "fp": 2, "fn": 0}, "all": {"tp": 3, "fp": 2, "fn": 0}}, "devin": {"strict": {"tp": 1, "fp": 0, "fn": 2}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 2}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 3}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 3}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 3}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 3}}, "kodus-v2": {"strict": {"tp": 0, "fp": 0, "fn": 3}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 3}}, "macroscope": {"strict": {"tp": 2, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 1}, "all": {"tp": 2, "fp": 1, "fn": 1}}, "qodo-v2": {"strict": {"tp": 2, "fp": 3, "fn": 1}, "core": {"tp": 2, "fp": 3, "fn": 1}, "all": {"tp": 2, "fp": 3, "fn": 1}}, "sourcery": {"strict": {"tp": 2, "fp": 7, "fn": 1}, "core": {"tp": 2, "fp": 7, "fn": 1}, "all": {"tp": 2, "fp": 7, "fn": 1}}}}, {"url": "https://github.com/keycloak/keycloak/pull/32918", "language": "Java", "pr_size": "small", "domain": "caching", "change_type": "feature", "complexity": "moderate", "difficulty": "subtle", "risk": "high", "context": "file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "codeant-v2": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "coderabbit": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "qodo-extended-v2": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "greptile-v4-1": {"strict": {"tp": 0, "fp": 3, "fn": 2}, "core": {"tp": 0, "fp": 3, "fn": 2}, "all": {"tp": 0, "fp": 3, "fn": 2}}, "gemini-v2": {"strict": {"tp": 0, "fp": 2, "fn": 2}, "core": {"tp": 0, "fp": 2, "fn": 2}, "all": {"tp": 0, "fp": 2, "fn": 2}}, "gitlab": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "copilot-v2": {"strict": {"tp": 1, "fp": 3, "fn": 1}, "core": {"tp": 1, "fp": 3, "fn": 1}, "all": {"tp": 1, "fp": 3, "fn": 1}}, "augment": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "bugbot": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "claude": {"strict": {"tp": 0, "fp": 1, "fn": 2}, "core": {"tp": 0, "fp": 1, "fn": 2}, "all": {"tp": 0, "fp": 1, "fn": 2}}, "claude-code": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "cubic-v2": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "devin": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "graphite": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "kg": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "kodus-v2": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "macroscope": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "qodo-v2": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "sourcery": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}}}, {"url": "https://github.com/grafana/grafana/pull/94942", "language": "Go", "pr_size": "small", "domain": "data_processing", "change_type": "feature", "complexity": "moderate", "difficulty": "moderate", "risk": "high", "context": "file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "codeant-v2": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "coderabbit": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "qodo-extended-v2": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "greptile-v4-1": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "gemini-v2": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "gitlab": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "copilot-v2": {"strict": {"tp": 2, "fp": 2, "fn": 0}, "core": {"tp": 2, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "augment": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "bugbot": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "claude": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "claude-code": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "cubic-v2": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "devin": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "kg": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "kodus-v2": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "macroscope": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "qodo-v2": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "sourcery": {"strict": {"tp": 2, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}}}, {"url": "https://github.com/grafana/grafana/pull/90939", "language": "Go", "pr_size": "small", "domain": "caching", "change_type": "performance", "complexity": "moderate", "difficulty": "subtle", "risk": "high", "context": "file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "codeant-v2": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "coderabbit": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "qodo-extended-v2": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "greptile-v4-1": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "gemini-v2": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "gitlab": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "copilot-v2": {"strict": {"tp": 1, "fp": 4, "fn": 1}, "core": {"tp": 1, "fp": 4, "fn": 1}, "all": {"tp": 1, "fp": 4, "fn": 1}}, "augment": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "bugbot": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "claude": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "claude-code": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "cubic-v2": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "devin": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "kg": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "kodus-v2": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "macroscope": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "qodo-v2": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "sourcery": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}}}, {"url": "https://github.com/grafana/grafana/pull/80329", "language": "Go", "pr_size": "small", "domain": "logging", "change_type": "performance", "complexity": "simple", "difficulty": "obvious", "risk": "low", "context": "local", "concern": "maintainability", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 0, "fp": 0, "fn": 0}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "codeant-v2": {"strict": {"tp": 0, "fp": 0, "fn": 0}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "coderabbit": {"strict": {"tp": 0, "fp": 2, "fn": 0}, "core": {"tp": 1, "fp": 2, "fn": 1}, "all": {"tp": 1, "fp": 2, "fn": 1}}, "qodo-extended-v2": {"strict": {"tp": 0, "fp": 2, "fn": 0}, "core": {"tp": 2, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "greptile-v4-1": {"strict": {"tp": 0, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "gemini-v2": {"strict": {"tp": 0, "fp": 2, "fn": 0}, "core": {"tp": 2, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "gitlab": {"strict": {"tp": 0, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "copilot-v2": {"strict": {"tp": 0, "fp": 2, "fn": 0}, "core": {"tp": 2, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "augment": {"strict": {"tp": 0, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "bugbot": {"strict": {"tp": 0, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "claude": {"strict": {"tp": 0, "fp": 4, "fn": 0}, "core": {"tp": 2, "fp": 4, "fn": 0}, "all": {"tp": 2, "fp": 4, "fn": 0}}, "claude-code": {"strict": {"tp": 0, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "cubic-v2": {"strict": {"tp": 0, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "devin": {"strict": {"tp": 0, "fp": 2, "fn": 0}, "core": {"tp": 2, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 0}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 0}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "kodus-v2": {"strict": {"tp": 0, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "macroscope": {"strict": {"tp": 0, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "qodo-v2": {"strict": {"tp": 0, "fp": 2, "fn": 0}, "core": {"tp": 2, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "sourcery": {"strict": {"tp": 0, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}}}, {"url": "https://github.com/grafana/grafana/pull/90045", "language": "Go", "pr_size": "medium", "domain": "logging", "change_type": "feature", "complexity": "moderate", "difficulty": "moderate", "risk": "medium", "context": "file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 2, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 1}, "all": {"tp": 2, "fp": 1, "fn": 1}}, "codeant-v2": {"strict": {"tp": 2, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 1}, "all": {"tp": 2, "fp": 1, "fn": 1}}, "coderabbit": {"strict": {"tp": 3, "fp": 2, "fn": 0}, "core": {"tp": 3, "fp": 2, "fn": 0}, "all": {"tp": 3, "fp": 2, "fn": 0}}, "qodo-extended-v2": {"strict": {"tp": 3, "fp": 2, "fn": 0}, "core": {"tp": 3, "fp": 2, "fn": 0}, "all": {"tp": 3, "fp": 2, "fn": 0}}, "greptile-v4-1": {"strict": {"tp": 3, "fp": 3, "fn": 0}, "core": {"tp": 3, "fp": 3, "fn": 0}, "all": {"tp": 3, "fp": 3, "fn": 0}}, "gemini-v2": {"strict": {"tp": 3, "fp": 5, "fn": 0}, "core": {"tp": 3, "fp": 5, "fn": 0}, "all": {"tp": 3, "fp": 5, "fn": 0}}, "gitlab": {"strict": {"tp": 3, "fp": 1, "fn": 0}, "core": {"tp": 3, "fp": 1, "fn": 0}, "all": {"tp": 3, "fp": 1, "fn": 0}}, "copilot-v2": {"strict": {"tp": 3, "fp": 5, "fn": 0}, "core": {"tp": 3, "fp": 5, "fn": 0}, "all": {"tp": 3, "fp": 5, "fn": 0}}, "augment": {"strict": {"tp": 2, "fp": 6, "fn": 1}, "core": {"tp": 2, "fp": 6, "fn": 1}, "all": {"tp": 2, "fp": 6, "fn": 1}}, "bugbot": {"strict": {"tp": 2, "fp": 4, "fn": 1}, "core": {"tp": 2, "fp": 4, "fn": 1}, "all": {"tp": 2, "fp": 4, "fn": 1}}, "claude": {"strict": {"tp": 3, "fp": 3, "fn": 0}, "core": {"tp": 3, "fp": 3, "fn": 0}, "all": {"tp": 3, "fp": 3, "fn": 0}}, "claude-code": {"strict": {"tp": 3, "fp": 2, "fn": 0}, "core": {"tp": 3, "fp": 2, "fn": 0}, "all": {"tp": 3, "fp": 2, "fn": 0}}, "cubic-v2": {"strict": {"tp": 3, "fp": 2, "fn": 0}, "core": {"tp": 3, "fp": 2, "fn": 0}, "all": {"tp": 3, "fp": 2, "fn": 0}}, "devin": {"strict": {"tp": 3, "fp": 2, "fn": 0}, "core": {"tp": 3, "fp": 2, "fn": 0}, "all": {"tp": 3, "fp": 2, "fn": 0}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 3}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 3}}, "kg": {"strict": {"tp": 3, "fp": 1, "fn": 0}, "core": {"tp": 3, "fp": 1, "fn": 0}, "all": {"tp": 3, "fp": 1, "fn": 0}}, "kodus-v2": {"strict": {"tp": 1, "fp": 0, "fn": 2}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 2}}, "macroscope": {"strict": {"tp": 2, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 1}, "all": {"tp": 2, "fp": 1, "fn": 1}}, "qodo-v2": {"strict": {"tp": 2, "fp": 6, "fn": 1}, "core": {"tp": 2, "fp": 6, "fn": 1}, "all": {"tp": 2, "fp": 6, "fn": 1}}, "sourcery": {"strict": {"tp": 3, "fp": 1, "fn": 0}, "core": {"tp": 3, "fp": 1, "fn": 0}, "all": {"tp": 3, "fp": 1, "fn": 0}}}}, {"url": "https://github.com/grafana/grafana/pull/106778", "language": "Go", "pr_size": "medium", "domain": "UI", "change_type": "feature", "complexity": "moderate", "difficulty": "subtle", "risk": "medium", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 0, "fp": 2, "fn": 2}, "core": {"tp": 0, "fp": 2, "fn": 2}, "all": {"tp": 0, "fp": 2, "fn": 2}}, "codeant-v2": {"strict": {"tp": 1, "fp": 2, "fn": 1}, "core": {"tp": 1, "fp": 2, "fn": 1}, "all": {"tp": 1, "fp": 2, "fn": 1}}, "coderabbit": {"strict": {"tp": 1, "fp": 6, "fn": 1}, "core": {"tp": 1, "fp": 6, "fn": 1}, "all": {"tp": 1, "fp": 6, "fn": 1}}, "qodo-extended-v2": {"strict": {"tp": 2, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "greptile-v4-1": {"strict": {"tp": 1, "fp": 3, "fn": 1}, "core": {"tp": 1, "fp": 3, "fn": 1}, "all": {"tp": 1, "fp": 3, "fn": 1}}, "gemini-v2": {"strict": {"tp": 0, "fp": 3, "fn": 2}, "core": {"tp": 0, "fp": 3, "fn": 2}, "all": {"tp": 0, "fp": 3, "fn": 2}}, "gitlab": {"strict": {"tp": 2, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "copilot-v2": {"strict": {"tp": 2, "fp": 2, "fn": 0}, "core": {"tp": 2, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "augment": {"strict": {"tp": 1, "fp": 2, "fn": 1}, "core": {"tp": 1, "fp": 2, "fn": 1}, "all": {"tp": 1, "fp": 2, "fn": 1}}, "bugbot": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "claude": {"strict": {"tp": 0, "fp": 10, "fn": 2}, "core": {"tp": 0, "fp": 10, "fn": 2}, "all": {"tp": 0, "fp": 10, "fn": 2}}, "claude-code": {"strict": {"tp": 2, "fp": 2, "fn": 0}, "core": {"tp": 2, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "cubic-v2": {"strict": {"tp": 2, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "devin": {"strict": {"tp": 2, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "kg": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "kodus-v2": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "macroscope": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "qodo-v2": {"strict": {"tp": 0, "fp": 2, "fn": 2}, "core": {"tp": 0, "fp": 2, "fn": 2}, "all": {"tp": 0, "fp": 2, "fn": 2}}, "sourcery": {"strict": {"tp": 0, "fp": 2, "fn": 2}, "core": {"tp": 0, "fp": 2, "fn": 2}, "all": {"tp": 0, "fp": 2, "fn": 2}}}}, {"url": "https://github.com/grafana/grafana/pull/107534", "language": "Go", "pr_size": "small", "domain": "testing", "change_type": "feature", "complexity": "moderate", "difficulty": "subtle", "risk": "low", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 0, "fp": 2, "fn": 0}, "core": {"tp": 0, "fp": 2, "fn": 1}, "all": {"tp": 0, "fp": 2, "fn": 1}}, "codeant-v2": {"strict": {"tp": 0, "fp": 0, "fn": 0}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 0, "fp": 0, "fn": 1}}, "coderabbit": {"strict": {"tp": 0, "fp": 1, "fn": 0}, "core": {"tp": 0, "fp": 1, "fn": 1}, "all": {"tp": 0, "fp": 1, "fn": 1}}, "qodo-extended-v2": {"strict": {"tp": 0, "fp": 1, "fn": 0}, "core": {"tp": 0, "fp": 1, "fn": 1}, "all": {"tp": 0, "fp": 1, "fn": 1}}, "greptile-v4-1": {"strict": {"tp": 0, "fp": 3, "fn": 0}, "core": {"tp": 0, "fp": 3, "fn": 1}, "all": {"tp": 0, "fp": 3, "fn": 1}}, "gemini-v2": {"strict": {"tp": 0, "fp": 3, "fn": 0}, "core": {"tp": 1, "fp": 3, "fn": 0}, "all": {"tp": 1, "fp": 3, "fn": 0}}, "gitlab": {"strict": {"tp": 0, "fp": 2, "fn": 0}, "core": {"tp": 0, "fp": 2, "fn": 1}, "all": {"tp": 0, "fp": 2, "fn": 1}}, "copilot-v2": {"strict": {"tp": 0, "fp": 2, "fn": 0}, "core": {"tp": 0, "fp": 2, "fn": 1}, "all": {"tp": 0, "fp": 2, "fn": 1}}, "augment": {"strict": {"tp": 0, "fp": 0, "fn": 0}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 0, "fp": 0, "fn": 1}}, "bugbot": {"strict": {"tp": 0, "fp": 1, "fn": 0}, "core": {"tp": 0, "fp": 1, "fn": 1}, "all": {"tp": 0, "fp": 1, "fn": 1}}, "claude": {"strict": {"tp": 0, "fp": 0, "fn": 0}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 0, "fp": 0, "fn": 1}}, "claude-code": {"strict": {"tp": 0, "fp": 3, "fn": 0}, "core": {"tp": 1, "fp": 3, "fn": 0}, "all": {"tp": 1, "fp": 3, "fn": 0}}, "cubic-v2": {"strict": {"tp": 0, "fp": 2, "fn": 0}, "core": {"tp": 1, "fp": 2, "fn": 0}, "all": {"tp": 1, "fp": 2, "fn": 0}}, "devin": {"strict": {"tp": 0, "fp": 1, "fn": 0}, "core": {"tp": 0, "fp": 1, "fn": 1}, "all": {"tp": 0, "fp": 1, "fn": 1}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 0}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 0, "fp": 0, "fn": 1}}, "kg": {"strict": {"tp": 0, "fp": 2, "fn": 0}, "core": {"tp": 0, "fp": 2, "fn": 1}, "all": {"tp": 0, "fp": 2, "fn": 1}}, "kodus-v2": {"strict": {"tp": 0, "fp": 0, "fn": 0}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 0, "fp": 0, "fn": 1}}, "macroscope": {"strict": {"tp": 0, "fp": 0, "fn": 0}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 0, "fp": 0, "fn": 1}}, "qodo-v2": {"strict": {"tp": 0, "fp": 1, "fn": 0}, "core": {"tp": 0, "fp": 1, "fn": 1}, "all": {"tp": 0, "fp": 1, "fn": 1}}, "sourcery": {"strict": {"tp": 0, "fp": 3, "fn": 0}, "core": {"tp": 0, "fp": 3, "fn": 1}, "all": {"tp": 0, "fp": 3, "fn": 1}}}}, {"url": "https://github.com/grafana/grafana/pull/79265", "language": "Go", "pr_size": "large", "domain": "authentication", "change_type": "feature", "complexity": "moderate", "difficulty": "subtle", "risk": "high", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 1, "fp": 0, "fn": 3}, "core": {"tp": 2, "fp": 0, "fn": 3}, "all": {"tp": 2, "fp": 0, "fn": 3}}, "codeant-v2": {"strict": {"tp": 1, "fp": 1, "fn": 3}, "core": {"tp": 1, "fp": 1, "fn": 4}, "all": {"tp": 1, "fp": 1, "fn": 4}}, "coderabbit": {"strict": {"tp": 2, "fp": 4, "fn": 2}, "core": {"tp": 3, "fp": 4, "fn": 2}, "all": {"tp": 3, "fp": 4, "fn": 2}}, "qodo-extended-v2": {"strict": {"tp": 2, "fp": 1, "fn": 2}, "core": {"tp": 2, "fp": 1, "fn": 3}, "all": {"tp": 2, "fp": 1, "fn": 3}}, "greptile-v4-1": {"strict": {"tp": 3, "fp": 2, "fn": 1}, "core": {"tp": 4, "fp": 2, "fn": 1}, "all": {"tp": 4, "fp": 2, "fn": 1}}, "gemini-v2": {"strict": {"tp": 1, "fp": 3, "fn": 3}, "core": {"tp": 1, "fp": 3, "fn": 4}, "all": {"tp": 1, "fp": 3, "fn": 4}}, "gitlab": {"strict": {"tp": 1, "fp": 1, "fn": 3}, "core": {"tp": 2, "fp": 1, "fn": 3}, "all": {"tp": 2, "fp": 1, "fn": 3}}, "copilot-v2": {"strict": {"tp": 2, "fp": 3, "fn": 2}, "core": {"tp": 2, "fp": 3, "fn": 3}, "all": {"tp": 2, "fp": 3, "fn": 3}}, "augment": {"strict": {"tp": 1, "fp": 0, "fn": 3}, "core": {"tp": 2, "fp": 0, "fn": 3}, "all": {"tp": 2, "fp": 0, "fn": 3}}, "bugbot": {"strict": {"tp": 1, "fp": 1, "fn": 3}, "core": {"tp": 1, "fp": 1, "fn": 4}, "all": {"tp": 1, "fp": 1, "fn": 4}}, "claude": {"strict": {"tp": 1, "fp": 2, "fn": 3}, "core": {"tp": 1, "fp": 2, "fn": 4}, "all": {"tp": 1, "fp": 2, "fn": 4}}, "claude-code": {"strict": {"tp": 2, "fp": 1, "fn": 2}, "core": {"tp": 2, "fp": 1, "fn": 3}, "all": {"tp": 2, "fp": 1, "fn": 3}}, "cubic-v2": {"strict": {"tp": 3, "fp": 1, "fn": 1}, "core": {"tp": 4, "fp": 1, "fn": 1}, "all": {"tp": 4, "fp": 1, "fn": 1}}, "devin": {"strict": {"tp": 2, "fp": 0, "fn": 2}, "core": {"tp": 2, "fp": 0, "fn": 3}, "all": {"tp": 2, "fp": 0, "fn": 3}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 4}, "core": {"tp": 0, "fp": 0, "fn": 5}, "all": {"tp": 0, "fp": 0, "fn": 5}}, "kg": {"strict": {"tp": 1, "fp": 0, "fn": 3}, "core": {"tp": 1, "fp": 0, "fn": 4}, "all": {"tp": 1, "fp": 0, "fn": 4}}, "kodus-v2": {"strict": {"tp": 0, "fp": 1, "fn": 4}, "core": {"tp": 1, "fp": 1, "fn": 4}, "all": {"tp": 1, "fp": 1, "fn": 4}}, "macroscope": {"strict": {"tp": 1, "fp": 0, "fn": 3}, "core": {"tp": 1, "fp": 0, "fn": 4}, "all": {"tp": 1, "fp": 0, "fn": 4}}, "qodo-v2": {"strict": {"tp": 2, "fp": 1, "fn": 2}, "core": {"tp": 3, "fp": 1, "fn": 2}, "all": {"tp": 3, "fp": 1, "fn": 2}}, "sourcery": {"strict": {"tp": 2, "fp": 3, "fn": 2}, "core": {"tp": 3, "fp": 3, "fn": 2}, "all": {"tp": 3, "fp": 3, "fn": 2}}}}, {"url": "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/9", "language": "Ruby", "pr_size": "small", "domain": "configuration", "change_type": "feature", "complexity": "moderate", "difficulty": "subtle", "risk": "medium", "context": "file", "concern": "reliability", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "codeant-v2": {"strict": {"tp": 0, "fp": 1, "fn": 1}, "core": {"tp": 0, "fp": 1, "fn": 2}, "all": {"tp": 0, "fp": 1, "fn": 2}}, "coderabbit": {"strict": {"tp": 0, "fp": 4, "fn": 1}, "core": {"tp": 0, "fp": 4, "fn": 2}, "all": {"tp": 0, "fp": 4, "fn": 2}}, "qodo-extended-v2": {"strict": {"tp": 0, "fp": 1, "fn": 1}, "core": {"tp": 0, "fp": 1, "fn": 2}, "all": {"tp": 0, "fp": 1, "fn": 2}}, "greptile-v4-1": {"strict": {"tp": 0, "fp": 2, "fn": 1}, "core": {"tp": 0, "fp": 2, "fn": 2}, "all": {"tp": 0, "fp": 2, "fn": 2}}, "gemini-v2": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "gitlab": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "copilot-v2": {"strict": {"tp": 0, "fp": 2, "fn": 1}, "core": {"tp": 0, "fp": 2, "fn": 2}, "all": {"tp": 0, "fp": 2, "fn": 2}}, "augment": {"strict": {"tp": 0, "fp": 2, "fn": 1}, "core": {"tp": 1, "fp": 2, "fn": 1}, "all": {"tp": 1, "fp": 2, "fn": 1}}, "bugbot": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "claude": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "claude-code": {"strict": {"tp": 1, "fp": 4, "fn": 0}, "core": {"tp": 1, "fp": 4, "fn": 1}, "all": {"tp": 1, "fp": 4, "fn": 1}}, "cubic-v2": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "devin": {"strict": {"tp": 0, "fp": 1, "fn": 1}, "core": {"tp": 0, "fp": 1, "fn": 2}, "all": {"tp": 0, "fp": 1, "fn": 2}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "kodus-v2": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "macroscope": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "qodo-v2": {"strict": {"tp": 0, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "sourcery": {"strict": {"tp": 0, "fp": 2, "fn": 1}, "core": {"tp": 0, "fp": 2, "fn": 2}, "all": {"tp": 0, "fp": 2, "fn": 2}}}}, {"url": "https://github.com/grafana/grafana/pull/76186", "language": "Go", "pr_size": "small", "domain": "logging", "change_type": "refactoring", "complexity": "moderate", "difficulty": "subtle", "risk": "high", "context": "cross_file", "concern": "reliability", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 0, "fp": 0, "fn": 3}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 3}}, "codeant-v2": {"strict": {"tp": 0, "fp": 1, "fn": 3}, "core": {"tp": 0, "fp": 1, "fn": 3}, "all": {"tp": 0, "fp": 1, "fn": 3}}, "coderabbit": {"strict": {"tp": 1, "fp": 0, "fn": 2}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 2}}, "qodo-extended-v2": {"strict": {"tp": 1, "fp": 0, "fn": 2}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 2}}, "greptile-v4-1": {"strict": {"tp": 1, "fp": 0, "fn": 2}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 2}}, "gemini-v2": {"strict": {"tp": 1, "fp": 2, "fn": 2}, "core": {"tp": 1, "fp": 2, "fn": 2}, "all": {"tp": 1, "fp": 2, "fn": 2}}, "gitlab": {"strict": {"tp": 3, "fp": 0, "fn": 0}, "core": {"tp": 3, "fp": 0, "fn": 0}, "all": {"tp": 3, "fp": 0, "fn": 0}}, "copilot-v2": {"strict": {"tp": 0, "fp": 2, "fn": 3}, "core": {"tp": 0, "fp": 2, "fn": 3}, "all": {"tp": 0, "fp": 2, "fn": 3}}, "augment": {"strict": {"tp": 2, "fp": 0, "fn": 1}, "core": {"tp": 2, "fp": 0, "fn": 1}, "all": {"tp": 2, "fp": 0, "fn": 1}}, "bugbot": {"strict": {"tp": 1, "fp": 0, "fn": 2}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 2}}, "claude": {"strict": {"tp": 0, "fp": 0, "fn": 3}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 3}}, "claude-code": {"strict": {"tp": 1, "fp": 1, "fn": 2}, "core": {"tp": 1, "fp": 1, "fn": 2}, "all": {"tp": 1, "fp": 1, "fn": 2}}, "cubic-v2": {"strict": {"tp": 3, "fp": 0, "fn": 0}, "core": {"tp": 3, "fp": 0, "fn": 0}, "all": {"tp": 3, "fp": 0, "fn": 0}}, "devin": {"strict": {"tp": 1, "fp": 0, "fn": 2}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 2}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 3}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 3}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 3}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 3}}, "kodus-v2": {"strict": {"tp": 2, "fp": 0, "fn": 1}, "core": {"tp": 2, "fp": 0, "fn": 1}, "all": {"tp": 2, "fp": 0, "fn": 1}}, "macroscope": {"strict": {"tp": 1, "fp": 0, "fn": 2}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 2}}, "qodo-v2": {"strict": {"tp": 2, "fp": 0, "fn": 1}, "core": {"tp": 2, "fp": 0, "fn": 1}, "all": {"tp": 2, "fp": 0, "fn": 1}}, "sourcery": {"strict": {"tp": 0, "fp": 3, "fn": 3}, "core": {"tp": 0, "fp": 3, "fn": 3}, "all": {"tp": 0, "fp": 3, "fn": 3}}}}, {"url": "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/10", "language": "Ruby", "pr_size": "large", "domain": "configuration", "change_type": "feature", "complexity": "moderate", "difficulty": "subtle", "risk": "high", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 2, "fp": 0, "fn": 5}, "core": {"tp": 2, "fp": 0, "fn": 5}, "all": {"tp": 2, "fp": 0, "fn": 5}}, "codeant-v2": {"strict": {"tp": 4, "fp": 5, "fn": 3}, "core": {"tp": 4, "fp": 5, "fn": 3}, "all": {"tp": 4, "fp": 5, "fn": 3}}, "coderabbit": {"strict": {"tp": 3, "fp": 8, "fn": 4}, "core": {"tp": 3, "fp": 8, "fn": 4}, "all": {"tp": 3, "fp": 8, "fn": 4}}, "qodo-extended-v2": {"strict": {"tp": 5, "fp": 4, "fn": 2}, "core": {"tp": 5, "fp": 4, "fn": 2}, "all": {"tp": 5, "fp": 4, "fn": 2}}, "greptile-v4-1": {"strict": {"tp": 2, "fp": 2, "fn": 5}, "core": {"tp": 2, "fp": 2, "fn": 5}, "all": {"tp": 2, "fp": 2, "fn": 5}}, "gemini-v2": {"strict": {"tp": 4, "fp": 2, "fn": 3}, "core": {"tp": 4, "fp": 2, "fn": 3}, "all": {"tp": 4, "fp": 2, "fn": 3}}, "gitlab": {"strict": {"tp": 2, "fp": 3, "fn": 5}, "core": {"tp": 2, "fp": 3, "fn": 5}, "all": {"tp": 2, "fp": 3, "fn": 5}}, "copilot-v2": {"strict": {"tp": 6, "fp": 5, "fn": 1}, "core": {"tp": 6, "fp": 5, "fn": 1}, "all": {"tp": 6, "fp": 5, "fn": 1}}, "augment": {"strict": {"tp": 3, "fp": 1, "fn": 4}, "core": {"tp": 3, "fp": 1, "fn": 4}, "all": {"tp": 3, "fp": 1, "fn": 4}}, "bugbot": {"strict": {"tp": 5, "fp": 2, "fn": 2}, "core": {"tp": 5, "fp": 2, "fn": 2}, "all": {"tp": 5, "fp": 2, "fn": 2}}, "claude": {"strict": {"tp": 1, "fp": 2, "fn": 6}, "core": {"tp": 1, "fp": 2, "fn": 6}, "all": {"tp": 1, "fp": 2, "fn": 6}}, "claude-code": {"strict": {"tp": 2, "fp": 4, "fn": 5}, "core": {"tp": 2, "fp": 4, "fn": 5}, "all": {"tp": 2, "fp": 4, "fn": 5}}, "cubic-v2": {"strict": {"tp": 3, "fp": 1, "fn": 4}, "core": {"tp": 3, "fp": 1, "fn": 4}, "all": {"tp": 3, "fp": 1, "fn": 4}}, "devin": {"strict": {"tp": 1, "fp": 1, "fn": 6}, "core": {"tp": 1, "fp": 1, "fn": 6}, "all": {"tp": 1, "fp": 1, "fn": 6}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 7}, "core": {"tp": 0, "fp": 0, "fn": 7}, "all": {"tp": 0, "fp": 0, "fn": 7}}, "kg": {"strict": {"tp": 1, "fp": 1, "fn": 6}, "core": {"tp": 1, "fp": 1, "fn": 6}, "all": {"tp": 1, "fp": 1, "fn": 6}}, "kodus-v2": {"strict": {"tp": 5, "fp": 4, "fn": 2}, "core": {"tp": 5, "fp": 4, "fn": 2}, "all": {"tp": 5, "fp": 4, "fn": 2}}, "macroscope": {"strict": {"tp": 5, "fp": 2, "fn": 2}, "core": {"tp": 5, "fp": 2, "fn": 2}, "all": {"tp": 5, "fp": 2, "fn": 2}}, "qodo-v2": {"strict": {"tp": 3, "fp": 4, "fn": 4}, "core": {"tp": 3, "fp": 4, "fn": 4}, "all": {"tp": 3, "fp": 4, "fn": 4}}, "sourcery": {"strict": {"tp": 5, "fp": 7, "fn": 2}, "core": {"tp": 5, "fp": 7, "fn": 2}, "all": {"tp": 5, "fp": 7, "fn": 2}}}}, {"url": "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/7", "language": "Ruby", "pr_size": "medium", "domain": "UI", "change_type": "bug_fix", "complexity": "simple", "difficulty": "subtle", "risk": "medium", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 3, "fp": 1, "fn": 0}, "core": {"tp": 3, "fp": 1, "fn": 0}, "all": {"tp": 5, "fp": 1, "fn": 0}}, "codeant-v2": {"strict": {"tp": 0, "fp": 8, "fn": 3}, "core": {"tp": 0, "fp": 8, "fn": 3}, "all": {"tp": 1, "fp": 8, "fn": 4}}, "coderabbit": {"strict": {"tp": 1, "fp": 1, "fn": 2}, "core": {"tp": 1, "fp": 1, "fn": 2}, "all": {"tp": 1, "fp": 1, "fn": 4}}, "qodo-extended-v2": {"strict": {"tp": 3, "fp": 2, "fn": 0}, "core": {"tp": 3, "fp": 2, "fn": 0}, "all": {"tp": 5, "fp": 2, "fn": 0}}, "greptile-v4-1": {"strict": {"tp": 2, "fp": 0, "fn": 1}, "core": {"tp": 2, "fp": 0, "fn": 1}, "all": {"tp": 4, "fp": 0, "fn": 1}}, "gemini-v2": {"strict": {"tp": 3, "fp": 1, "fn": 0}, "core": {"tp": 3, "fp": 1, "fn": 0}, "all": {"tp": 5, "fp": 1, "fn": 0}}, "gitlab": {"strict": {"tp": 3, "fp": 0, "fn": 0}, "core": {"tp": 3, "fp": 0, "fn": 0}, "all": {"tp": 5, "fp": 0, "fn": 0}}, "copilot-v2": {"strict": {"tp": 3, "fp": 1, "fn": 0}, "core": {"tp": 3, "fp": 1, "fn": 0}, "all": {"tp": 5, "fp": 1, "fn": 0}}, "augment": {"strict": {"tp": 3, "fp": 2, "fn": 0}, "core": {"tp": 3, "fp": 2, "fn": 0}, "all": {"tp": 5, "fp": 2, "fn": 0}}, "bugbot": {"strict": {"tp": 3, "fp": 1, "fn": 0}, "core": {"tp": 3, "fp": 1, "fn": 0}, "all": {"tp": 5, "fp": 1, "fn": 0}}, "claude": {"strict": {"tp": 3, "fp": 4, "fn": 0}, "core": {"tp": 3, "fp": 4, "fn": 0}, "all": {"tp": 5, "fp": 4, "fn": 0}}, "claude-code": {"strict": {"tp": 3, "fp": 0, "fn": 0}, "core": {"tp": 3, "fp": 0, "fn": 0}, "all": {"tp": 5, "fp": 0, "fn": 0}}, "cubic-v2": {"strict": {"tp": 3, "fp": 0, "fn": 0}, "core": {"tp": 3, "fp": 0, "fn": 0}, "all": {"tp": 5, "fp": 0, "fn": 0}}, "devin": {"strict": {"tp": 3, "fp": 0, "fn": 0}, "core": {"tp": 3, "fp": 0, "fn": 0}, "all": {"tp": 5, "fp": 0, "fn": 0}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 3}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 5}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 3}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 5}}, "kodus-v2": {"strict": {"tp": 3, "fp": 0, "fn": 0}, "core": {"tp": 3, "fp": 0, "fn": 0}, "all": {"tp": 3, "fp": 0, "fn": 2}}, "macroscope": {"strict": {"tp": 3, "fp": 0, "fn": 0}, "core": {"tp": 3, "fp": 0, "fn": 0}, "all": {"tp": 5, "fp": 0, "fn": 0}}, "qodo-v2": {"strict": {"tp": 3, "fp": 2, "fn": 0}, "core": {"tp": 3, "fp": 2, "fn": 0}, "all": {"tp": 5, "fp": 2, "fn": 0}}, "sourcery": {"strict": {"tp": 3, "fp": 1, "fn": 0}, "core": {"tp": 3, "fp": 1, "fn": 0}, "all": {"tp": 5, "fp": 1, "fn": 0}}}}, {"url": "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/8", "language": "Ruby", "pr_size": "medium", "domain": "API", "change_type": "bug_fix", "complexity": "moderate", "difficulty": "subtle", "risk": "medium", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 2, "fp": 2, "fn": 4}, "core": {"tp": 2, "fp": 2, "fn": 4}, "all": {"tp": 2, "fp": 2, "fn": 4}}, "codeant-v2": {"strict": {"tp": 5, "fp": 5, "fn": 1}, "core": {"tp": 5, "fp": 5, "fn": 1}, "all": {"tp": 5, "fp": 5, "fn": 1}}, "coderabbit": {"strict": {"tp": 4, "fp": 6, "fn": 2}, "core": {"tp": 4, "fp": 6, "fn": 2}, "all": {"tp": 4, "fp": 6, "fn": 2}}, "qodo-extended-v2": {"strict": {"tp": 3, "fp": 1, "fn": 3}, "core": {"tp": 3, "fp": 1, "fn": 3}, "all": {"tp": 3, "fp": 1, "fn": 3}}, "greptile-v4-1": {"strict": {"tp": 4, "fp": 1, "fn": 2}, "core": {"tp": 4, "fp": 1, "fn": 2}, "all": {"tp": 4, "fp": 1, "fn": 2}}, "gemini-v2": {"strict": {"tp": 2, "fp": 2, "fn": 4}, "core": {"tp": 2, "fp": 2, "fn": 4}, "all": {"tp": 2, "fp": 2, "fn": 4}}, "gitlab": {"strict": {"tp": 4, "fp": 1, "fn": 2}, "core": {"tp": 4, "fp": 1, "fn": 2}, "all": {"tp": 4, "fp": 1, "fn": 2}}, "copilot-v2": {"strict": {"tp": 3, "fp": 9, "fn": 3}, "core": {"tp": 3, "fp": 9, "fn": 3}, "all": {"tp": 3, "fp": 9, "fn": 3}}, "augment": {"strict": {"tp": 3, "fp": 2, "fn": 3}, "core": {"tp": 3, "fp": 2, "fn": 3}, "all": {"tp": 3, "fp": 2, "fn": 3}}, "bugbot": {"strict": {"tp": 2, "fp": 1, "fn": 4}, "core": {"tp": 2, "fp": 1, "fn": 4}, "all": {"tp": 2, "fp": 1, "fn": 4}}, "claude": {"strict": {"tp": 4, "fp": 3, "fn": 2}, "core": {"tp": 4, "fp": 3, "fn": 2}, "all": {"tp": 4, "fp": 3, "fn": 2}}, "claude-code": {"strict": {"tp": 2, "fp": 1, "fn": 4}, "core": {"tp": 2, "fp": 1, "fn": 4}, "all": {"tp": 2, "fp": 1, "fn": 4}}, "cubic-v2": {"strict": {"tp": 3, "fp": 1, "fn": 3}, "core": {"tp": 3, "fp": 1, "fn": 3}, "all": {"tp": 3, "fp": 1, "fn": 3}}, "devin": {"strict": {"tp": 2, "fp": 0, "fn": 4}, "core": {"tp": 2, "fp": 0, "fn": 4}, "all": {"tp": 2, "fp": 0, "fn": 4}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 6}, "core": {"tp": 0, "fp": 0, "fn": 6}, "all": {"tp": 0, "fp": 0, "fn": 6}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 6}, "core": {"tp": 0, "fp": 0, "fn": 6}, "all": {"tp": 0, "fp": 0, "fn": 6}}, "kodus-v2": {"strict": {"tp": 1, "fp": 6, "fn": 5}, "core": {"tp": 1, "fp": 6, "fn": 5}, "all": {"tp": 1, "fp": 6, "fn": 5}}, "macroscope": {"strict": {"tp": 2, "fp": 2, "fn": 4}, "core": {"tp": 2, "fp": 2, "fn": 4}, "all": {"tp": 2, "fp": 2, "fn": 4}}, "qodo-v2": {"strict": {"tp": 2, "fp": 1, "fn": 4}, "core": {"tp": 2, "fp": 1, "fn": 4}, "all": {"tp": 2, "fp": 1, "fn": 4}}, "sourcery": {"strict": {"tp": 4, "fp": 1, "fn": 2}, "core": {"tp": 4, "fp": 1, "fn": 2}, "all": {"tp": 4, "fp": 1, "fn": 2}}}}, {"url": "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/3", "language": "Ruby", "pr_size": "medium", "domain": "authentication", "change_type": "feature", "complexity": "moderate", "difficulty": "subtle", "risk": "medium", "context": "file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 1, "fp": 0, "fn": 2}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 2}}, "codeant-v2": {"strict": {"tp": 2, "fp": 2, "fn": 1}, "core": {"tp": 2, "fp": 2, "fn": 1}, "all": {"tp": 2, "fp": 2, "fn": 1}}, "coderabbit": {"strict": {"tp": 2, "fp": 5, "fn": 1}, "core": {"tp": 2, "fp": 5, "fn": 1}, "all": {"tp": 2, "fp": 5, "fn": 1}}, "qodo-extended-v2": {"strict": {"tp": 2, "fp": 0, "fn": 1}, "core": {"tp": 2, "fp": 0, "fn": 1}, "all": {"tp": 2, "fp": 0, "fn": 1}}, "greptile-v4-1": {"strict": {"tp": 2, "fp": 2, "fn": 1}, "core": {"tp": 2, "fp": 2, "fn": 1}, "all": {"tp": 2, "fp": 2, "fn": 1}}, "gemini-v2": {"strict": {"tp": 2, "fp": 3, "fn": 1}, "core": {"tp": 2, "fp": 3, "fn": 1}, "all": {"tp": 2, "fp": 3, "fn": 1}}, "gitlab": {"strict": {"tp": 0, "fp": 2, "fn": 3}, "core": {"tp": 0, "fp": 2, "fn": 3}, "all": {"tp": 0, "fp": 2, "fn": 3}}, "copilot-v2": {"strict": {"tp": 2, "fp": 6, "fn": 1}, "core": {"tp": 2, "fp": 6, "fn": 1}, "all": {"tp": 2, "fp": 6, "fn": 1}}, "augment": {"strict": {"tp": 1, "fp": 4, "fn": 2}, "core": {"tp": 1, "fp": 4, "fn": 2}, "all": {"tp": 1, "fp": 4, "fn": 2}}, "bugbot": {"strict": {"tp": 1, "fp": 1, "fn": 2}, "core": {"tp": 1, "fp": 1, "fn": 2}, "all": {"tp": 1, "fp": 1, "fn": 2}}, "claude": {"strict": {"tp": 1, "fp": 3, "fn": 2}, "core": {"tp": 1, "fp": 3, "fn": 2}, "all": {"tp": 1, "fp": 3, "fn": 2}}, "claude-code": {"strict": {"tp": 1, "fp": 4, "fn": 2}, "core": {"tp": 1, "fp": 4, "fn": 2}, "all": {"tp": 1, "fp": 4, "fn": 2}}, "cubic-v2": {"strict": {"tp": 2, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 1}, "all": {"tp": 2, "fp": 1, "fn": 1}}, "devin": {"strict": {"tp": 1, "fp": 1, "fn": 2}, "core": {"tp": 1, "fp": 1, "fn": 2}, "all": {"tp": 1, "fp": 1, "fn": 2}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 3}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 3}}, "kg": {"strict": {"tp": 1, "fp": 0, "fn": 2}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 2}}, "kodus-v2": {"strict": {"tp": 1, "fp": 0, "fn": 2}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 2}}, "macroscope": {"strict": {"tp": 2, "fp": 0, "fn": 1}, "core": {"tp": 2, "fp": 0, "fn": 1}, "all": {"tp": 2, "fp": 0, "fn": 1}}, "qodo-v2": {"strict": {"tp": 2, "fp": 2, "fn": 1}, "core": {"tp": 2, "fp": 2, "fn": 1}, "all": {"tp": 2, "fp": 2, "fn": 1}}, "sourcery": {"strict": {"tp": 1, "fp": 3, "fn": 2}, "core": {"tp": 1, "fp": 3, "fn": 2}, "all": {"tp": 1, "fp": 3, "fn": 2}}}}, {"url": "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/5", "language": "Ruby", "pr_size": "small", "domain": "UI", "change_type": "performance", "complexity": "moderate", "difficulty": "subtle", "risk": "medium", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 1, "fp": 1, "fn": 2}, "core": {"tp": 1, "fp": 1, "fn": 2}, "all": {"tp": 1, "fp": 1, "fn": 2}}, "codeant-v2": {"strict": {"tp": 0, "fp": 0, "fn": 3}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 3}}, "coderabbit": {"strict": {"tp": 1, "fp": 0, "fn": 2}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 2}}, "qodo-extended-v2": {"strict": {"tp": 2, "fp": 0, "fn": 1}, "core": {"tp": 2, "fp": 0, "fn": 1}, "all": {"tp": 2, "fp": 0, "fn": 1}}, "greptile-v4-1": {"strict": {"tp": 1, "fp": 2, "fn": 2}, "core": {"tp": 1, "fp": 2, "fn": 2}, "all": {"tp": 1, "fp": 2, "fn": 2}}, "gemini-v2": {"strict": {"tp": 2, "fp": 7, "fn": 1}, "core": {"tp": 2, "fp": 7, "fn": 1}, "all": {"tp": 2, "fp": 7, "fn": 1}}, "gitlab": {"strict": {"tp": 1, "fp": 1, "fn": 2}, "core": {"tp": 1, "fp": 1, "fn": 2}, "all": {"tp": 1, "fp": 1, "fn": 2}}, "copilot-v2": {"strict": {"tp": 2, "fp": 2, "fn": 1}, "core": {"tp": 2, "fp": 2, "fn": 1}, "all": {"tp": 2, "fp": 2, "fn": 1}}, "augment": {"strict": {"tp": 2, "fp": 0, "fn": 1}, "core": {"tp": 2, "fp": 0, "fn": 1}, "all": {"tp": 2, "fp": 0, "fn": 1}}, "bugbot": {"strict": {"tp": 1, "fp": 0, "fn": 2}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 2}}, "claude": {"strict": {"tp": 1, "fp": 0, "fn": 2}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 2}}, "claude-code": {"strict": {"tp": 2, "fp": 3, "fn": 1}, "core": {"tp": 2, "fp": 3, "fn": 1}, "all": {"tp": 2, "fp": 3, "fn": 1}}, "cubic-v2": {"strict": {"tp": 2, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 1}, "all": {"tp": 2, "fp": 1, "fn": 1}}, "devin": {"strict": {"tp": 1, "fp": 1, "fn": 2}, "core": {"tp": 1, "fp": 1, "fn": 2}, "all": {"tp": 1, "fp": 1, "fn": 2}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 3}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 3}}, "kg": {"strict": {"tp": 1, "fp": 0, "fn": 2}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 2}}, "kodus-v2": {"strict": {"tp": 0, "fp": 0, "fn": 3}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 3}}, "macroscope": {"strict": {"tp": 0, "fp": 0, "fn": 3}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 3}}, "qodo-v2": {"strict": {"tp": 1, "fp": 1, "fn": 2}, "core": {"tp": 1, "fp": 1, "fn": 2}, "all": {"tp": 1, "fp": 1, "fn": 2}}, "sourcery": {"strict": {"tp": 2, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 1}, "all": {"tp": 2, "fp": 1, "fn": 1}}}}, {"url": "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/6", "language": "Ruby", "pr_size": "small", "domain": "serialization", "change_type": "feature", "complexity": "simple", "difficulty": "moderate", "risk": "medium", "context": "file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 1, "fp": 1, "fn": 0}, "all": {"tp": 1, "fp": 1, "fn": 0}}, "codeant-v2": {"strict": {"tp": 1, "fp": 3, "fn": 0}, "core": {"tp": 1, "fp": 3, "fn": 0}, "all": {"tp": 1, "fp": 3, "fn": 0}}, "coderabbit": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 0}, "all": {"tp": 1, "fp": 0, "fn": 0}}, "qodo-extended-v2": {"strict": {"tp": 0, "fp": 2, "fn": 1}, "core": {"tp": 0, "fp": 2, "fn": 1}, "all": {"tp": 0, "fp": 2, "fn": 1}}, "greptile-v4-1": {"strict": {"tp": 1, "fp": 3, "fn": 0}, "core": {"tp": 1, "fp": 3, "fn": 0}, "all": {"tp": 1, "fp": 3, "fn": 0}}, "gemini-v2": {"strict": {"tp": 1, "fp": 5, "fn": 0}, "core": {"tp": 1, "fp": 5, "fn": 0}, "all": {"tp": 1, "fp": 5, "fn": 0}}, "gitlab": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 1, "fp": 1, "fn": 0}, "all": {"tp": 1, "fp": 1, "fn": 0}}, "copilot-v2": {"strict": {"tp": 1, "fp": 3, "fn": 0}, "core": {"tp": 1, "fp": 3, "fn": 0}, "all": {"tp": 1, "fp": 3, "fn": 0}}, "augment": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 1, "fp": 1, "fn": 0}, "all": {"tp": 1, "fp": 1, "fn": 0}}, "bugbot": {"strict": {"tp": 1, "fp": 2, "fn": 0}, "core": {"tp": 1, "fp": 2, "fn": 0}, "all": {"tp": 1, "fp": 2, "fn": 0}}, "claude": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 0, "fp": 0, "fn": 1}}, "claude-code": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 1, "fp": 1, "fn": 0}, "all": {"tp": 1, "fp": 1, "fn": 0}}, "cubic-v2": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 1, "fp": 1, "fn": 0}, "all": {"tp": 1, "fp": 1, "fn": 0}}, "devin": {"strict": {"tp": 1, "fp": 2, "fn": 0}, "core": {"tp": 1, "fp": 2, "fn": 0}, "all": {"tp": 1, "fp": 2, "fn": 0}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 0, "fp": 0, "fn": 1}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 0, "fp": 0, "fn": 1}}, "kodus-v2": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 1, "fp": 1, "fn": 0}, "all": {"tp": 1, "fp": 1, "fn": 0}}, "macroscope": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 0}, "all": {"tp": 1, "fp": 0, "fn": 0}}, "qodo-v2": {"strict": {"tp": 1, "fp": 2, "fn": 0}, "core": {"tp": 1, "fp": 2, "fn": 0}, "all": {"tp": 1, "fp": 2, "fn": 0}}, "sourcery": {"strict": {"tp": 1, "fp": 4, "fn": 0}, "core": {"tp": 1, "fp": 4, "fn": 0}, "all": {"tp": 1, "fp": 4, "fn": 0}}}}, {"url": "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/4", "language": "Ruby", "pr_size": "large", "domain": "API", "change_type": "feature", "complexity": "complex", "difficulty": "subtle", "risk": "critical", "context": "cross_file", "concern": "security", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 2, "fp": 2, "fn": 6}, "core": {"tp": 2, "fp": 2, "fn": 6}, "all": {"tp": 2, "fp": 2, "fn": 6}}, "codeant-v2": {"strict": {"tp": 3, "fp": 16, "fn": 5}, "core": {"tp": 3, "fp": 16, "fn": 5}, "all": {"tp": 3, "fp": 16, "fn": 5}}, "coderabbit": {"strict": {"tp": 6, "fp": 19, "fn": 2}, "core": {"tp": 6, "fp": 19, "fn": 2}, "all": {"tp": 6, "fp": 19, "fn": 2}}, "qodo-extended-v2": {"strict": {"tp": 4, "fp": 1, "fn": 4}, "core": {"tp": 4, "fp": 1, "fn": 4}, "all": {"tp": 4, "fp": 1, "fn": 4}}, "greptile-v4-1": {"strict": {"tp": 4, "fp": 1, "fn": 4}, "core": {"tp": 4, "fp": 1, "fn": 4}, "all": {"tp": 4, "fp": 1, "fn": 4}}, "gemini-v2": {"strict": {"tp": 5, "fp": 4, "fn": 3}, "core": {"tp": 5, "fp": 4, "fn": 3}, "all": {"tp": 5, "fp": 4, "fn": 3}}, "gitlab": {"strict": {"tp": 5, "fp": 0, "fn": 3}, "core": {"tp": 5, "fp": 0, "fn": 3}, "all": {"tp": 5, "fp": 0, "fn": 3}}, "copilot-v2": {"strict": {"tp": 6, "fp": 2, "fn": 2}, "core": {"tp": 6, "fp": 2, "fn": 2}, "all": {"tp": 6, "fp": 2, "fn": 2}}, "augment": {"strict": {"tp": 5, "fp": 0, "fn": 3}, "core": {"tp": 5, "fp": 0, "fn": 3}, "all": {"tp": 5, "fp": 0, "fn": 3}}, "bugbot": {"strict": {"tp": 2, "fp": 3, "fn": 6}, "core": {"tp": 2, "fp": 3, "fn": 6}, "all": {"tp": 2, "fp": 3, "fn": 6}}, "claude": {"strict": {"tp": 4, "fp": 3, "fn": 4}, "core": {"tp": 4, "fp": 3, "fn": 4}, "all": {"tp": 4, "fp": 3, "fn": 4}}, "claude-code": {"strict": {"tp": 5, "fp": 0, "fn": 3}, "core": {"tp": 5, "fp": 0, "fn": 3}, "all": {"tp": 5, "fp": 0, "fn": 3}}, "cubic-v2": {"strict": {"tp": 3, "fp": 1, "fn": 5}, "core": {"tp": 3, "fp": 1, "fn": 5}, "all": {"tp": 3, "fp": 1, "fn": 5}}, "devin": {"strict": {"tp": 3, "fp": 1, "fn": 5}, "core": {"tp": 3, "fp": 1, "fn": 5}, "all": {"tp": 3, "fp": 1, "fn": 5}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 8}, "core": {"tp": 0, "fp": 0, "fn": 8}, "all": {"tp": 0, "fp": 0, "fn": 8}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 8}, "core": {"tp": 0, "fp": 0, "fn": 8}, "all": {"tp": 0, "fp": 0, "fn": 8}}, "kodus-v2": {"strict": {"tp": 4, "fp": 1, "fn": 4}, "core": {"tp": 4, "fp": 1, "fn": 4}, "all": {"tp": 4, "fp": 1, "fn": 4}}, "macroscope": {"strict": {"tp": 2, "fp": 4, "fn": 6}, "core": {"tp": 2, "fp": 4, "fn": 6}, "all": {"tp": 2, "fp": 4, "fn": 6}}, "qodo-v2": {"strict": {"tp": 6, "fp": 5, "fn": 2}, "core": {"tp": 6, "fp": 5, "fn": 2}, "all": {"tp": 6, "fp": 5, "fn": 2}}, "sourcery": {"strict": {"tp": 3, "fp": 2, "fn": 5}, "core": {"tp": 3, "fp": 2, "fn": 5}, "all": {"tp": 3, "fp": 2, "fn": 5}}}}, {"url": "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/1", "language": "Ruby", "pr_size": "medium", "domain": "file_io", "change_type": "feature", "complexity": "moderate", "difficulty": "subtle", "risk": "medium", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 2, "fp": 0, "fn": 2}, "core": {"tp": 2, "fp": 0, "fn": 2}, "all": {"tp": 2, "fp": 0, "fn": 2}}, "codeant-v2": {"strict": {"tp": 2, "fp": 1, "fn": 2}, "core": {"tp": 2, "fp": 1, "fn": 2}, "all": {"tp": 2, "fp": 1, "fn": 2}}, "coderabbit": {"strict": {"tp": 3, "fp": 3, "fn": 1}, "core": {"tp": 3, "fp": 3, "fn": 1}, "all": {"tp": 3, "fp": 3, "fn": 1}}, "qodo-extended-v2": {"strict": {"tp": 2, "fp": 1, "fn": 2}, "core": {"tp": 2, "fp": 1, "fn": 2}, "all": {"tp": 2, "fp": 1, "fn": 2}}, "greptile-v4-1": {"strict": {"tp": 3, "fp": 1, "fn": 1}, "core": {"tp": 3, "fp": 1, "fn": 1}, "all": {"tp": 3, "fp": 1, "fn": 1}}, "gemini-v2": {"strict": {"tp": 3, "fp": 0, "fn": 1}, "core": {"tp": 3, "fp": 0, "fn": 1}, "all": {"tp": 3, "fp": 0, "fn": 1}}, "gitlab": {"strict": {"tp": 2, "fp": 0, "fn": 2}, "core": {"tp": 2, "fp": 0, "fn": 2}, "all": {"tp": 2, "fp": 0, "fn": 2}}, "copilot-v2": {"strict": {"tp": 3, "fp": 1, "fn": 1}, "core": {"tp": 3, "fp": 1, "fn": 1}, "all": {"tp": 3, "fp": 1, "fn": 1}}, "augment": {"strict": {"tp": 2, "fp": 2, "fn": 2}, "core": {"tp": 2, "fp": 2, "fn": 2}, "all": {"tp": 2, "fp": 2, "fn": 2}}, "bugbot": {"strict": {"tp": 2, "fp": 1, "fn": 2}, "core": {"tp": 2, "fp": 1, "fn": 2}, "all": {"tp": 2, "fp": 1, "fn": 2}}, "claude": {"strict": {"tp": 2, "fp": 0, "fn": 2}, "core": {"tp": 2, "fp": 0, "fn": 2}, "all": {"tp": 2, "fp": 0, "fn": 2}}, "claude-code": {"strict": {"tp": 3, "fp": 2, "fn": 1}, "core": {"tp": 3, "fp": 2, "fn": 1}, "all": {"tp": 3, "fp": 2, "fn": 1}}, "cubic-v2": {"strict": {"tp": 3, "fp": 0, "fn": 1}, "core": {"tp": 3, "fp": 0, "fn": 1}, "all": {"tp": 3, "fp": 0, "fn": 1}}, "devin": {"strict": {"tp": 2, "fp": 0, "fn": 2}, "core": {"tp": 2, "fp": 0, "fn": 2}, "all": {"tp": 2, "fp": 0, "fn": 2}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 4}, "core": {"tp": 0, "fp": 0, "fn": 4}, "all": {"tp": 0, "fp": 0, "fn": 4}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 4}, "core": {"tp": 0, "fp": 0, "fn": 4}, "all": {"tp": 0, "fp": 0, "fn": 4}}, "kodus-v2": {"strict": {"tp": 3, "fp": 0, "fn": 1}, "core": {"tp": 3, "fp": 0, "fn": 1}, "all": {"tp": 3, "fp": 0, "fn": 1}}, "macroscope": {"strict": {"tp": 1, "fp": 0, "fn": 3}, "core": {"tp": 1, "fp": 0, "fn": 3}, "all": {"tp": 1, "fp": 0, "fn": 3}}, "qodo-v2": {"strict": {"tp": 2, "fp": 3, "fn": 2}, "core": {"tp": 2, "fp": 3, "fn": 2}, "all": {"tp": 2, "fp": 3, "fn": 2}}, "sourcery": {"strict": {"tp": 2, "fp": 1, "fn": 2}, "core": {"tp": 2, "fp": 1, "fn": 2}, "all": {"tp": 2, "fp": 1, "fn": 2}}}}, {"url": "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/2", "language": "Ruby", "pr_size": "medium", "domain": "UI", "change_type": "feature", "complexity": "moderate", "difficulty": "moderate", "risk": "medium", "context": "file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 0}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "codeant-v2": {"strict": {"tp": 1, "fp": 5, "fn": 0}, "core": {"tp": 1, "fp": 5, "fn": 0}, "all": {"tp": 1, "fp": 5, "fn": 1}}, "coderabbit": {"strict": {"tp": 1, "fp": 5, "fn": 0}, "core": {"tp": 1, "fp": 5, "fn": 0}, "all": {"tp": 2, "fp": 5, "fn": 0}}, "qodo-extended-v2": {"strict": {"tp": 1, "fp": 3, "fn": 0}, "core": {"tp": 1, "fp": 3, "fn": 0}, "all": {"tp": 1, "fp": 3, "fn": 1}}, "greptile-v4-1": {"strict": {"tp": 1, "fp": 2, "fn": 0}, "core": {"tp": 1, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "gemini-v2": {"strict": {"tp": 1, "fp": 2, "fn": 0}, "core": {"tp": 1, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "gitlab": {"strict": {"tp": 1, "fp": 2, "fn": 0}, "core": {"tp": 1, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "copilot-v2": {"strict": {"tp": 1, "fp": 4, "fn": 0}, "core": {"tp": 1, "fp": 4, "fn": 0}, "all": {"tp": 2, "fp": 4, "fn": 0}}, "augment": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 1, "fp": 1, "fn": 0}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "bugbot": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 1, "fp": 1, "fn": 0}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "claude": {"strict": {"tp": 0, "fp": 1, "fn": 1}, "core": {"tp": 0, "fp": 1, "fn": 1}, "all": {"tp": 0, "fp": 1, "fn": 2}}, "claude-code": {"strict": {"tp": 0, "fp": 2, "fn": 1}, "core": {"tp": 0, "fp": 2, "fn": 1}, "all": {"tp": 0, "fp": 2, "fn": 2}}, "cubic-v2": {"strict": {"tp": 0, "fp": 2, "fn": 1}, "core": {"tp": 0, "fp": 2, "fn": 1}, "all": {"tp": 0, "fp": 2, "fn": 2}}, "devin": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 0}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "kodus-v2": {"strict": {"tp": 1, "fp": 2, "fn": 0}, "core": {"tp": 1, "fp": 2, "fn": 0}, "all": {"tp": 1, "fp": 2, "fn": 1}}, "macroscope": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 0}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "qodo-v2": {"strict": {"tp": 1, "fp": 2, "fn": 0}, "core": {"tp": 1, "fp": 2, "fn": 0}, "all": {"tp": 1, "fp": 2, "fn": 1}}, "sourcery": {"strict": {"tp": 1, "fp": 5, "fn": 0}, "core": {"tp": 1, "fp": 5, "fn": 0}, "all": {"tp": 1, "fp": 5, "fn": 1}}}}, {"url": "https://github.com/calcom/cal.com/pull/22532", "language": "TypeScript", "pr_size": "medium", "domain": "caching", "change_type": "feature", "complexity": "moderate", "difficulty": "subtle", "risk": "medium", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 1, "fp": 2, "fn": 3}, "core": {"tp": 1, "fp": 2, "fn": 3}, "all": {"tp": 1, "fp": 2, "fn": 3}}, "codeant-v2": {"strict": {"tp": 3, "fp": 2, "fn": 1}, "core": {"tp": 3, "fp": 2, "fn": 1}, "all": {"tp": 3, "fp": 2, "fn": 1}}, "coderabbit": {"strict": {"tp": 2, "fp": 9, "fn": 2}, "core": {"tp": 2, "fp": 9, "fn": 2}, "all": {"tp": 2, "fp": 9, "fn": 2}}, "qodo-extended-v2": {"strict": {"tp": 2, "fp": 1, "fn": 2}, "core": {"tp": 2, "fp": 1, "fn": 2}, "all": {"tp": 2, "fp": 1, "fn": 2}}, "greptile-v4-1": {"strict": {"tp": 3, "fp": 2, "fn": 1}, "core": {"tp": 3, "fp": 2, "fn": 1}, "all": {"tp": 3, "fp": 2, "fn": 1}}, "gemini-v2": {"strict": {"tp": 4, "fp": 0, "fn": 0}, "core": {"tp": 4, "fp": 0, "fn": 0}, "all": {"tp": 4, "fp": 0, "fn": 0}}, "gitlab": {"strict": {"tp": 2, "fp": 3, "fn": 2}, "core": {"tp": 2, "fp": 3, "fn": 2}, "all": {"tp": 2, "fp": 3, "fn": 2}}, "copilot-v2": {"strict": {"tp": 4, "fp": 2, "fn": 0}, "core": {"tp": 4, "fp": 2, "fn": 0}, "all": {"tp": 4, "fp": 2, "fn": 0}}, "augment": {"strict": {"tp": 4, "fp": 2, "fn": 0}, "core": {"tp": 4, "fp": 2, "fn": 0}, "all": {"tp": 4, "fp": 2, "fn": 0}}, "bugbot": {"strict": {"tp": 0, "fp": 1, "fn": 4}, "core": {"tp": 0, "fp": 1, "fn": 4}, "all": {"tp": 0, "fp": 1, "fn": 4}}, "claude": {"strict": {"tp": 0, "fp": 1, "fn": 4}, "core": {"tp": 0, "fp": 1, "fn": 4}, "all": {"tp": 0, "fp": 1, "fn": 4}}, "claude-code": {"strict": {"tp": 1, "fp": 3, "fn": 3}, "core": {"tp": 1, "fp": 3, "fn": 3}, "all": {"tp": 1, "fp": 3, "fn": 3}}, "cubic-v2": {"strict": {"tp": 0, "fp": 2, "fn": 4}, "core": {"tp": 0, "fp": 2, "fn": 4}, "all": {"tp": 0, "fp": 2, "fn": 4}}, "devin": {"strict": {"tp": 1, "fp": 1, "fn": 3}, "core": {"tp": 1, "fp": 1, "fn": 3}, "all": {"tp": 1, "fp": 1, "fn": 3}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 4}, "core": {"tp": 0, "fp": 0, "fn": 4}, "all": {"tp": 0, "fp": 0, "fn": 4}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 4}, "core": {"tp": 0, "fp": 0, "fn": 4}, "all": {"tp": 0, "fp": 0, "fn": 4}}, "kodus-v2": {"strict": {"tp": 1, "fp": 0, "fn": 3}, "core": {"tp": 1, "fp": 0, "fn": 3}, "all": {"tp": 1, "fp": 0, "fn": 3}}, "macroscope": {"strict": {"tp": 1, "fp": 1, "fn": 3}, "core": {"tp": 1, "fp": 1, "fn": 3}, "all": {"tp": 1, "fp": 1, "fn": 3}}, "qodo-v2": {"strict": {"tp": 3, "fp": 2, "fn": 1}, "core": {"tp": 3, "fp": 2, "fn": 1}, "all": {"tp": 3, "fp": 2, "fn": 1}}, "sourcery": {"strict": {"tp": 3, "fp": 1, "fn": 1}, "core": {"tp": 3, "fp": 1, "fn": 1}, "all": {"tp": 3, "fp": 1, "fn": 1}}}}, {"url": "https://github.com/calcom/cal.com/pull/8330", "language": "TypeScript", "pr_size": "small", "domain": "scheduling", "change_type": "bug_fix", "complexity": "moderate", "difficulty": "subtle", "risk": "medium", "context": "file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 2, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "codeant-v2": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "coderabbit": {"strict": {"tp": 2, "fp": 2, "fn": 0}, "core": {"tp": 2, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "qodo-extended-v2": {"strict": {"tp": 2, "fp": 2, "fn": 0}, "core": {"tp": 2, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "greptile-v4-1": {"strict": {"tp": 2, "fp": 2, "fn": 0}, "core": {"tp": 2, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "gemini-v2": {"strict": {"tp": 2, "fp": 3, "fn": 0}, "core": {"tp": 2, "fp": 3, "fn": 0}, "all": {"tp": 2, "fp": 3, "fn": 0}}, "gitlab": {"strict": {"tp": 2, "fp": 2, "fn": 0}, "core": {"tp": 2, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "copilot-v2": {"strict": {"tp": 2, "fp": 4, "fn": 0}, "core": {"tp": 2, "fp": 4, "fn": 0}, "all": {"tp": 2, "fp": 4, "fn": 0}}, "augment": {"strict": {"tp": 2, "fp": 3, "fn": 0}, "core": {"tp": 2, "fp": 3, "fn": 0}, "all": {"tp": 2, "fp": 3, "fn": 0}}, "bugbot": {"strict": {"tp": 2, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "claude": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "claude-code": {"strict": {"tp": 2, "fp": 2, "fn": 0}, "core": {"tp": 2, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "cubic-v2": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "devin": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "graphite": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "kg": {"strict": {"tp": 1, "fp": 2, "fn": 1}, "core": {"tp": 1, "fp": 2, "fn": 1}, "all": {"tp": 1, "fp": 2, "fn": 1}}, "kodus-v2": {"strict": {"tp": 2, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "macroscope": {"strict": {"tp": 2, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "qodo-v2": {"strict": {"tp": 2, "fp": 2, "fn": 0}, "core": {"tp": 2, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "sourcery": {"strict": {"tp": 2, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}}}, {"url": "https://github.com/calcom/cal.com/pull/14943", "language": "TypeScript", "pr_size": "small", "domain": "scheduling", "change_type": "feature", "complexity": "moderate", "difficulty": "subtle", "risk": "high", "context": "file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "codeant-v2": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "coderabbit": {"strict": {"tp": 2, "fp": 2, "fn": 0}, "core": {"tp": 2, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "qodo-extended-v2": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "greptile-v4-1": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "gemini-v2": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "gitlab": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "copilot-v2": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "augment": {"strict": {"tp": 2, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "bugbot": {"strict": {"tp": 1, "fp": 2, "fn": 1}, "core": {"tp": 1, "fp": 2, "fn": 1}, "all": {"tp": 1, "fp": 2, "fn": 1}}, "claude": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "claude-code": {"strict": {"tp": 0, "fp": 1, "fn": 2}, "core": {"tp": 0, "fp": 1, "fn": 2}, "all": {"tp": 0, "fp": 1, "fn": 2}}, "cubic-v2": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "devin": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "graphite": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "kg": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "kodus-v2": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "macroscope": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "qodo-v2": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "sourcery": {"strict": {"tp": 2, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}}}, {"url": "https://github.com/calcom/cal.com/pull/22345", "language": "TypeScript", "pr_size": "small", "domain": "database", "change_type": "migration", "complexity": "moderate", "difficulty": "subtle", "risk": "medium", "context": "file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 0, "fp": 2, "fn": 1}, "core": {"tp": 0, "fp": 2, "fn": 1}, "all": {"tp": 0, "fp": 2, "fn": 2}}, "codeant-v2": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "coderabbit": {"strict": {"tp": 0, "fp": 3, "fn": 1}, "core": {"tp": 0, "fp": 3, "fn": 1}, "all": {"tp": 1, "fp": 3, "fn": 1}}, "qodo-extended-v2": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "greptile-v4-1": {"strict": {"tp": 0, "fp": 2, "fn": 1}, "core": {"tp": 0, "fp": 2, "fn": 1}, "all": {"tp": 0, "fp": 2, "fn": 2}}, "gemini-v2": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 0}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "gitlab": {"strict": {"tp": 1, "fp": 2, "fn": 0}, "core": {"tp": 1, "fp": 2, "fn": 0}, "all": {"tp": 1, "fp": 2, "fn": 1}}, "copilot-v2": {"strict": {"tp": 0, "fp": 3, "fn": 1}, "core": {"tp": 0, "fp": 3, "fn": 1}, "all": {"tp": 1, "fp": 3, "fn": 1}}, "augment": {"strict": {"tp": 0, "fp": 1, "fn": 1}, "core": {"tp": 0, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "bugbot": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 1, "fp": 1, "fn": 0}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "claude": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "claude-code": {"strict": {"tp": 1, "fp": 2, "fn": 0}, "core": {"tp": 1, "fp": 2, "fn": 0}, "all": {"tp": 1, "fp": 2, "fn": 1}}, "cubic-v2": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 1, "fp": 1, "fn": 0}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "devin": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "kg": {"strict": {"tp": 0, "fp": 1, "fn": 1}, "core": {"tp": 0, "fp": 1, "fn": 1}, "all": {"tp": 0, "fp": 1, "fn": 2}}, "kodus-v2": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "macroscope": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "qodo-v2": {"strict": {"tp": 0, "fp": 3, "fn": 1}, "core": {"tp": 0, "fp": 3, "fn": 1}, "all": {"tp": 1, "fp": 3, "fn": 1}}, "sourcery": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 0}, "all": {"tp": 1, "fp": 0, "fn": 1}}}}, {"url": "https://github.com/calcom/cal.com/pull/11059", "language": "TypeScript", "pr_size": "large", "domain": "authentication", "change_type": "feature", "complexity": "complex", "difficulty": "subtle", "risk": "critical", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 6, "fp": 0, "fn": 3}, "core": {"tp": 6, "fp": 0, "fn": 3}, "all": {"tp": 6, "fp": 0, "fn": 3}}, "codeant-v2": {"strict": {"tp": 7, "fp": 4, "fn": 2}, "core": {"tp": 7, "fp": 4, "fn": 2}, "all": {"tp": 7, "fp": 4, "fn": 2}}, "coderabbit": {"strict": {"tp": 9, "fp": 10, "fn": 0}, "core": {"tp": 9, "fp": 10, "fn": 0}, "all": {"tp": 9, "fp": 10, "fn": 0}}, "qodo-extended-v2": {"strict": {"tp": 7, "fp": 2, "fn": 2}, "core": {"tp": 7, "fp": 2, "fn": 2}, "all": {"tp": 7, "fp": 2, "fn": 2}}, "greptile-v4-1": {"strict": {"tp": 5, "fp": 2, "fn": 4}, "core": {"tp": 5, "fp": 2, "fn": 4}, "all": {"tp": 5, "fp": 2, "fn": 4}}, "gemini-v2": {"strict": {"tp": 5, "fp": 1, "fn": 4}, "core": {"tp": 5, "fp": 1, "fn": 4}, "all": {"tp": 5, "fp": 1, "fn": 4}}, "gitlab": {"strict": {"tp": 6, "fp": 2, "fn": 3}, "core": {"tp": 6, "fp": 2, "fn": 3}, "all": {"tp": 6, "fp": 2, "fn": 3}}, "copilot-v2": {"strict": {"tp": 6, "fp": 2, "fn": 3}, "core": {"tp": 6, "fp": 2, "fn": 3}, "all": {"tp": 6, "fp": 2, "fn": 3}}, "augment": {"strict": {"tp": 9, "fp": 1, "fn": 0}, "core": {"tp": 9, "fp": 1, "fn": 0}, "all": {"tp": 9, "fp": 1, "fn": 0}}, "bugbot": {"strict": {"tp": 6, "fp": 4, "fn": 3}, "core": {"tp": 6, "fp": 4, "fn": 3}, "all": {"tp": 6, "fp": 4, "fn": 3}}, "claude": {"strict": {"tp": 6, "fp": 0, "fn": 3}, "core": {"tp": 6, "fp": 0, "fn": 3}, "all": {"tp": 6, "fp": 0, "fn": 3}}, "claude-code": {"strict": {"tp": 7, "fp": 0, "fn": 2}, "core": {"tp": 7, "fp": 0, "fn": 2}, "all": {"tp": 7, "fp": 0, "fn": 2}}, "cubic-v2": {"strict": {"tp": 4, "fp": 0, "fn": 5}, "core": {"tp": 4, "fp": 0, "fn": 5}, "all": {"tp": 4, "fp": 0, "fn": 5}}, "devin": {"strict": {"tp": 5, "fp": 0, "fn": 4}, "core": {"tp": 5, "fp": 0, "fn": 4}, "all": {"tp": 5, "fp": 0, "fn": 4}}, "graphite": {"strict": {"tp": 1, "fp": 0, "fn": 8}, "core": {"tp": 1, "fp": 0, "fn": 8}, "all": {"tp": 1, "fp": 0, "fn": 8}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 9}, "core": {"tp": 0, "fp": 0, "fn": 9}, "all": {"tp": 0, "fp": 0, "fn": 9}}, "kodus-v2": {"strict": {"tp": 4, "fp": 3, "fn": 5}, "core": {"tp": 4, "fp": 3, "fn": 5}, "all": {"tp": 4, "fp": 3, "fn": 5}}, "macroscope": {"strict": {"tp": 7, "fp": 6, "fn": 2}, "core": {"tp": 7, "fp": 6, "fn": 2}, "all": {"tp": 7, "fp": 6, "fn": 2}}, "qodo-v2": {"strict": {"tp": 8, "fp": 0, "fn": 1}, "core": {"tp": 8, "fp": 0, "fn": 1}, "all": {"tp": 8, "fp": 0, "fn": 1}}, "sourcery": {"strict": {"tp": 5, "fp": 0, "fn": 4}, "core": {"tp": 5, "fp": 0, "fn": 4}, "all": {"tp": 5, "fp": 0, "fn": 4}}}}, {"url": "https://github.com/calcom/cal.com/pull/7232", "language": "TypeScript", "pr_size": "medium", "domain": "scheduling", "change_type": "feature", "complexity": "moderate", "difficulty": "subtle", "risk": "high", "context": "cross_file", "concern": "reliability", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 1, "fp": 0, "fn": 2}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 2}}, "codeant-v2": {"strict": {"tp": 1, "fp": 2, "fn": 2}, "core": {"tp": 1, "fp": 2, "fn": 2}, "all": {"tp": 1, "fp": 2, "fn": 2}}, "coderabbit": {"strict": {"tp": 2, "fp": 9, "fn": 1}, "core": {"tp": 2, "fp": 9, "fn": 1}, "all": {"tp": 2, "fp": 9, "fn": 1}}, "qodo-extended-v2": {"strict": {"tp": 2, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 1}, "all": {"tp": 2, "fp": 1, "fn": 1}}, "greptile-v4-1": {"strict": {"tp": 1, "fp": 2, "fn": 2}, "core": {"tp": 1, "fp": 2, "fn": 2}, "all": {"tp": 1, "fp": 2, "fn": 2}}, "gemini-v2": {"strict": {"tp": 2, "fp": 3, "fn": 1}, "core": {"tp": 2, "fp": 3, "fn": 1}, "all": {"tp": 2, "fp": 3, "fn": 1}}, "gitlab": {"strict": {"tp": 0, "fp": 2, "fn": 3}, "core": {"tp": 0, "fp": 2, "fn": 3}, "all": {"tp": 0, "fp": 2, "fn": 3}}, "copilot-v2": {"strict": {"tp": 0, "fp": 3, "fn": 3}, "core": {"tp": 0, "fp": 3, "fn": 3}, "all": {"tp": 0, "fp": 3, "fn": 3}}, "augment": {"strict": {"tp": 2, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 1}, "all": {"tp": 2, "fp": 1, "fn": 1}}, "bugbot": {"strict": {"tp": 1, "fp": 1, "fn": 2}, "core": {"tp": 1, "fp": 1, "fn": 2}, "all": {"tp": 1, "fp": 1, "fn": 2}}, "claude": {"strict": {"tp": 1, "fp": 3, "fn": 2}, "core": {"tp": 1, "fp": 3, "fn": 2}, "all": {"tp": 1, "fp": 3, "fn": 2}}, "claude-code": {"strict": {"tp": 1, "fp": 1, "fn": 2}, "core": {"tp": 1, "fp": 1, "fn": 2}, "all": {"tp": 1, "fp": 1, "fn": 2}}, "cubic-v2": {"strict": {"tp": 2, "fp": 2, "fn": 1}, "core": {"tp": 2, "fp": 2, "fn": 1}, "all": {"tp": 2, "fp": 2, "fn": 1}}, "devin": {"strict": {"tp": 1, "fp": 1, "fn": 2}, "core": {"tp": 1, "fp": 1, "fn": 2}, "all": {"tp": 1, "fp": 1, "fn": 2}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 3}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 3}}, "kg": {"strict": {"tp": 0, "fp": 1, "fn": 3}, "core": {"tp": 0, "fp": 1, "fn": 3}, "all": {"tp": 0, "fp": 1, "fn": 3}}, "kodus-v2": {"strict": {"tp": 2, "fp": 2, "fn": 1}, "core": {"tp": 2, "fp": 2, "fn": 1}, "all": {"tp": 2, "fp": 2, "fn": 1}}, "macroscope": {"strict": {"tp": 1, "fp": 2, "fn": 2}, "core": {"tp": 1, "fp": 2, "fn": 2}, "all": {"tp": 1, "fp": 2, "fn": 2}}, "qodo-v2": {"strict": {"tp": 2, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 1}, "all": {"tp": 2, "fp": 1, "fn": 1}}, "sourcery": {"strict": {"tp": 1, "fp": 2, "fn": 2}, "core": {"tp": 1, "fp": 2, "fn": 2}, "all": {"tp": 1, "fp": 2, "fn": 2}}}}, {"url": "https://github.com/calcom/cal.com/pull/14740", "language": "TypeScript", "pr_size": "large", "domain": "scheduling", "change_type": "feature", "complexity": "moderate", "difficulty": "subtle", "risk": "critical", "context": "cross_file", "concern": "security", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 1, "fp": 1, "fn": 5}, "core": {"tp": 1, "fp": 1, "fn": 5}, "all": {"tp": 1, "fp": 1, "fn": 5}}, "codeant-v2": {"strict": {"tp": 3, "fp": 4, "fn": 3}, "core": {"tp": 3, "fp": 4, "fn": 3}, "all": {"tp": 3, "fp": 4, "fn": 3}}, "coderabbit": {"strict": {"tp": 5, "fp": 5, "fn": 1}, "core": {"tp": 5, "fp": 5, "fn": 1}, "all": {"tp": 5, "fp": 5, "fn": 1}}, "qodo-extended-v2": {"strict": {"tp": 4, "fp": 1, "fn": 2}, "core": {"tp": 4, "fp": 1, "fn": 2}, "all": {"tp": 4, "fp": 1, "fn": 2}}, "greptile-v4-1": {"strict": {"tp": 3, "fp": 2, "fn": 3}, "core": {"tp": 3, "fp": 2, "fn": 3}, "all": {"tp": 3, "fp": 2, "fn": 3}}, "gemini-v2": {"strict": {"tp": 3, "fp": 4, "fn": 3}, "core": {"tp": 3, "fp": 4, "fn": 3}, "all": {"tp": 3, "fp": 4, "fn": 3}}, "gitlab": {"strict": {"tp": 3, "fp": 4, "fn": 3}, "core": {"tp": 3, "fp": 4, "fn": 3}, "all": {"tp": 3, "fp": 4, "fn": 3}}, "copilot-v2": {"strict": {"tp": 5, "fp": 3, "fn": 1}, "core": {"tp": 5, "fp": 3, "fn": 1}, "all": {"tp": 5, "fp": 3, "fn": 1}}, "augment": {"strict": {"tp": 4, "fp": 3, "fn": 2}, "core": {"tp": 4, "fp": 3, "fn": 2}, "all": {"tp": 4, "fp": 3, "fn": 2}}, "bugbot": {"strict": {"tp": 3, "fp": 3, "fn": 3}, "core": {"tp": 3, "fp": 3, "fn": 3}, "all": {"tp": 3, "fp": 3, "fn": 3}}, "claude": {"strict": {"tp": 2, "fp": 0, "fn": 4}, "core": {"tp": 2, "fp": 0, "fn": 4}, "all": {"tp": 2, "fp": 0, "fn": 4}}, "claude-code": {"strict": {"tp": 2, "fp": 1, "fn": 4}, "core": {"tp": 2, "fp": 1, "fn": 4}, "all": {"tp": 2, "fp": 1, "fn": 4}}, "cubic-v2": {"strict": {"tp": 4, "fp": 2, "fn": 2}, "core": {"tp": 4, "fp": 2, "fn": 2}, "all": {"tp": 4, "fp": 2, "fn": 2}}, "devin": {"strict": {"tp": 2, "fp": 0, "fn": 4}, "core": {"tp": 2, "fp": 0, "fn": 4}, "all": {"tp": 2, "fp": 0, "fn": 4}}, "graphite": {"strict": {"tp": 1, "fp": 0, "fn": 5}, "core": {"tp": 1, "fp": 0, "fn": 5}, "all": {"tp": 1, "fp": 0, "fn": 5}}, "kg": {"strict": {"tp": 2, "fp": 0, "fn": 4}, "core": {"tp": 2, "fp": 0, "fn": 4}, "all": {"tp": 2, "fp": 0, "fn": 4}}, "kodus-v2": {"strict": {"tp": 2, "fp": 0, "fn": 4}, "core": {"tp": 2, "fp": 0, "fn": 4}, "all": {"tp": 2, "fp": 0, "fn": 4}}, "macroscope": {"strict": {"tp": 3, "fp": 3, "fn": 3}, "core": {"tp": 3, "fp": 3, "fn": 3}, "all": {"tp": 3, "fp": 3, "fn": 3}}, "qodo-v2": {"strict": {"tp": 5, "fp": 2, "fn": 1}, "core": {"tp": 5, "fp": 2, "fn": 1}, "all": {"tp": 5, "fp": 2, "fn": 1}}, "sourcery": {"strict": {"tp": 3, "fp": 1, "fn": 3}, "core": {"tp": 3, "fp": 1, "fn": 3}, "all": {"tp": 3, "fp": 1, "fn": 3}}}}, {"url": "https://github.com/calcom/cal.com/pull/10600", "language": "TypeScript", "pr_size": "medium", "domain": "authentication", "change_type": "feature", "complexity": "moderate", "difficulty": "subtle", "risk": "high", "context": "cross_file", "concern": "security", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 0, "fp": 1, "fn": 2}, "core": {"tp": 0, "fp": 1, "fn": 4}, "all": {"tp": 0, "fp": 1, "fn": 5}}, "codeant-v2": {"strict": {"tp": 1, "fp": 2, "fn": 1}, "core": {"tp": 1, "fp": 2, "fn": 3}, "all": {"tp": 1, "fp": 2, "fn": 4}}, "coderabbit": {"strict": {"tp": 1, "fp": 6, "fn": 1}, "core": {"tp": 1, "fp": 6, "fn": 3}, "all": {"tp": 2, "fp": 6, "fn": 3}}, "qodo-extended-v2": {"strict": {"tp": 1, "fp": 2, "fn": 1}, "core": {"tp": 2, "fp": 2, "fn": 2}, "all": {"tp": 2, "fp": 2, "fn": 3}}, "greptile-v4-1": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 2, "fp": 0, "fn": 2}, "all": {"tp": 3, "fp": 0, "fn": 2}}, "gemini-v2": {"strict": {"tp": 0, "fp": 5, "fn": 2}, "core": {"tp": 0, "fp": 5, "fn": 4}, "all": {"tp": 1, "fp": 5, "fn": 4}}, "gitlab": {"strict": {"tp": 0, "fp": 2, "fn": 2}, "core": {"tp": 1, "fp": 2, "fn": 3}, "all": {"tp": 2, "fp": 2, "fn": 3}}, "copilot-v2": {"strict": {"tp": 2, "fp": 4, "fn": 0}, "core": {"tp": 4, "fp": 4, "fn": 0}, "all": {"tp": 5, "fp": 4, "fn": 0}}, "augment": {"strict": {"tp": 0, "fp": 3, "fn": 2}, "core": {"tp": 1, "fp": 3, "fn": 3}, "all": {"tp": 2, "fp": 3, "fn": 3}}, "bugbot": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 3}, "all": {"tp": 1, "fp": 0, "fn": 4}}, "claude": {"strict": {"tp": 0, "fp": 6, "fn": 2}, "core": {"tp": 1, "fp": 6, "fn": 3}, "all": {"tp": 2, "fp": 6, "fn": 3}}, "claude-code": {"strict": {"tp": 0, "fp": 4, "fn": 2}, "core": {"tp": 0, "fp": 4, "fn": 4}, "all": {"tp": 1, "fp": 4, "fn": 4}}, "cubic-v2": {"strict": {"tp": 2, "fp": 2, "fn": 0}, "core": {"tp": 2, "fp": 2, "fn": 2}, "all": {"tp": 2, "fp": 2, "fn": 3}}, "devin": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 3}, "all": {"tp": 1, "fp": 0, "fn": 4}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 4}, "all": {"tp": 0, "fp": 0, "fn": 5}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 4}, "all": {"tp": 0, "fp": 0, "fn": 5}}, "kodus-v2": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 3}, "all": {"tp": 1, "fp": 0, "fn": 4}}, "macroscope": {"strict": {"tp": 0, "fp": 2, "fn": 2}, "core": {"tp": 0, "fp": 2, "fn": 4}, "all": {"tp": 0, "fp": 2, "fn": 5}}, "qodo-v2": {"strict": {"tp": 1, "fp": 2, "fn": 1}, "core": {"tp": 2, "fp": 2, "fn": 2}, "all": {"tp": 2, "fp": 2, "fn": 3}}, "sourcery": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 2}, "all": {"tp": 3, "fp": 1, "fn": 2}}}}, {"url": "https://github.com/calcom/cal.com/pull/10967", "language": "TypeScript", "pr_size": "large", "domain": "scheduling", "change_type": "bug_fix", "complexity": "complex", "difficulty": "subtle", "risk": "high", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 1, "fp": 1, "fn": 4}, "core": {"tp": 1, "fp": 1, "fn": 4}, "all": {"tp": 1, "fp": 1, "fn": 5}}, "codeant-v2": {"strict": {"tp": 2, "fp": 5, "fn": 3}, "core": {"tp": 2, "fp": 5, "fn": 3}, "all": {"tp": 2, "fp": 5, "fn": 4}}, "coderabbit": {"strict": {"tp": 5, "fp": 7, "fn": 0}, "core": {"tp": 5, "fp": 7, "fn": 0}, "all": {"tp": 5, "fp": 7, "fn": 1}}, "qodo-extended-v2": {"strict": {"tp": 4, "fp": 2, "fn": 1}, "core": {"tp": 4, "fp": 2, "fn": 1}, "all": {"tp": 4, "fp": 2, "fn": 2}}, "greptile-v4-1": {"strict": {"tp": 2, "fp": 1, "fn": 3}, "core": {"tp": 2, "fp": 1, "fn": 3}, "all": {"tp": 2, "fp": 1, "fn": 4}}, "gemini-v2": {"strict": {"tp": 2, "fp": 4, "fn": 3}, "core": {"tp": 2, "fp": 4, "fn": 3}, "all": {"tp": 2, "fp": 4, "fn": 4}}, "gitlab": {"strict": {"tp": 4, "fp": 1, "fn": 1}, "core": {"tp": 4, "fp": 1, "fn": 1}, "all": {"tp": 4, "fp": 1, "fn": 2}}, "copilot-v2": {"strict": {"tp": 5, "fp": 6, "fn": 0}, "core": {"tp": 5, "fp": 6, "fn": 0}, "all": {"tp": 5, "fp": 6, "fn": 1}}, "augment": {"strict": {"tp": 4, "fp": 0, "fn": 1}, "core": {"tp": 4, "fp": 0, "fn": 1}, "all": {"tp": 4, "fp": 0, "fn": 2}}, "bugbot": {"strict": {"tp": 3, "fp": 3, "fn": 2}, "core": {"tp": 3, "fp": 3, "fn": 2}, "all": {"tp": 4, "fp": 3, "fn": 2}}, "claude": {"strict": {"tp": 4, "fp": 3, "fn": 1}, "core": {"tp": 4, "fp": 3, "fn": 1}, "all": {"tp": 4, "fp": 3, "fn": 2}}, "claude-code": {"strict": {"tp": 4, "fp": 3, "fn": 1}, "core": {"tp": 4, "fp": 3, "fn": 1}, "all": {"tp": 5, "fp": 3, "fn": 1}}, "cubic-v2": {"strict": {"tp": 1, "fp": 2, "fn": 4}, "core": {"tp": 1, "fp": 2, "fn": 4}, "all": {"tp": 1, "fp": 2, "fn": 5}}, "devin": {"strict": {"tp": 2, "fp": 3, "fn": 3}, "core": {"tp": 2, "fp": 3, "fn": 3}, "all": {"tp": 2, "fp": 3, "fn": 4}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 5}, "core": {"tp": 0, "fp": 0, "fn": 5}, "all": {"tp": 0, "fp": 0, "fn": 6}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 5}, "core": {"tp": 0, "fp": 0, "fn": 5}, "all": {"tp": 0, "fp": 0, "fn": 6}}, "kodus-v2": {"strict": {"tp": 2, "fp": 2, "fn": 3}, "core": {"tp": 2, "fp": 2, "fn": 3}, "all": {"tp": 2, "fp": 2, "fn": 4}}, "macroscope": {"strict": {"tp": 3, "fp": 2, "fn": 2}, "core": {"tp": 3, "fp": 2, "fn": 2}, "all": {"tp": 3, "fp": 2, "fn": 3}}, "qodo-v2": {"strict": {"tp": 2, "fp": 3, "fn": 3}, "core": {"tp": 2, "fp": 3, "fn": 3}, "all": {"tp": 2, "fp": 3, "fn": 4}}, "sourcery": {"strict": {"tp": 3, "fp": 2, "fn": 2}, "core": {"tp": 3, "fp": 2, "fn": 2}, "all": {"tp": 3, "fp": 2, "fn": 3}}}}, {"url": "https://github.com/calcom/cal.com/pull/8087", "language": "TypeScript", "pr_size": "medium", "domain": "concurrency", "change_type": "refactoring", "complexity": "moderate", "difficulty": "subtle", "risk": "critical", "context": "cross_file", "concern": "reliability", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 1, "fp": 3, "fn": 0}, "core": {"tp": 1, "fp": 3, "fn": 0}, "all": {"tp": 1, "fp": 3, "fn": 1}}, "codeant-v2": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 1, "fp": 1, "fn": 0}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "coderabbit": {"strict": {"tp": 1, "fp": 6, "fn": 0}, "core": {"tp": 1, "fp": 6, "fn": 0}, "all": {"tp": 2, "fp": 6, "fn": 0}}, "qodo-extended-v2": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 1, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "greptile-v4-1": {"strict": {"tp": 1, "fp": 2, "fn": 0}, "core": {"tp": 1, "fp": 2, "fn": 0}, "all": {"tp": 1, "fp": 2, "fn": 1}}, "gemini-v2": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 0}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "gitlab": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 1, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "copilot-v2": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 1, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "augment": {"strict": {"tp": 1, "fp": 4, "fn": 0}, "core": {"tp": 1, "fp": 4, "fn": 0}, "all": {"tp": 1, "fp": 4, "fn": 1}}, "bugbot": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 0}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "claude": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 1, "fp": 1, "fn": 0}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "claude-code": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 1, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "cubic-v2": {"strict": {"tp": 1, "fp": 10, "fn": 0}, "core": {"tp": 1, "fp": 10, "fn": 0}, "all": {"tp": 2, "fp": 10, "fn": 0}}, "devin": {"strict": {"tp": 1, "fp": 2, "fn": 0}, "core": {"tp": 1, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "kg": {"strict": {"tp": 1, "fp": 2, "fn": 0}, "core": {"tp": 1, "fp": 2, "fn": 0}, "all": {"tp": 1, "fp": 2, "fn": 1}}, "kodus-v2": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "macroscope": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 0}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "qodo-v2": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 0}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "sourcery": {"strict": {"tp": 1, "fp": 3, "fn": 0}, "core": {"tp": 1, "fp": 3, "fn": 0}, "all": {"tp": 1, "fp": 3, "fn": 1}}}}], "tools": ["augment", "baz", "bugbot", "claude", "claude-code", "codeant-v2", "coderabbit", "copilot-v2", "cubic-v2", "devin", "gemini-v2", "gitlab", "graphite", "greptile-v4-1", "kg", "kodus-v2", "macroscope", "qodo-extended-v2", "qodo-v2", "sourcery"], "dimensions": {"language": ["Go", "Java", "Python", "Ruby", "TypeScript"], "pr_size": ["small", "medium", "large"], "domain": ["API", "UI", "authentication", "caching", "concurrency", "configuration", "data_processing", "database", "file_io", "logging", "scheduling", "serialization", "testing"], "change_type": ["bug_fix", "feature", "migration", "performance", "refactoring", "security_patch"], "complexity": ["simple", "moderate", "complex"], "difficulty": ["obvious", "moderate", "subtle", "very_subtle"], "risk": ["low", "medium", "high", "critical"], "context": ["local", "file", "cross_file", "system"], "concern": ["correctness", "maintainability", "reliability", "security"]}, "overall_metrics": {"augment": {"strict": {"precision": 56.5, "recall": 65.5, "f1": 60.7, "f2": 63.5, "tp": 91, "fp": 70, "fn": 48, "num_prs": 50}, "core": {"precision": 59.5, "recall": 65.2, "f1": 62.2, "f2": 64.0, "tp": 103, "fp": 70, "fn": 55, "num_prs": 50}, "all": {"precision": 60.7, "recall": 62.4, "f1": 61.5, "f2": 62.1, "tp": 108, "fp": 70, "fn": 65, "num_prs": 50}}, "cubic-v2": {"strict": {"precision": 58.3, "recall": 65.5, "f1": 61.7, "f2": 63.9, "tp": 91, "fp": 65, "fn": 48, "num_prs": 50}, "core": {"precision": 61.5, "recall": 65.8, "f1": 63.6, "f2": 64.9, "tp": 104, "fp": 65, "fn": 54, "num_prs": 50}, "all": {"precision": 62.4, "recall": 62.4, "f1": 62.4, "f2": 62.4, "tp": 108, "fp": 65, "fn": 65, "num_prs": 50}}, "copilot-v2": {"strict": {"precision": 35.6, "recall": 66.9, "f1": 46.5, "f2": 56.9, "tp": 93, "fp": 168, "fn": 46, "num_prs": 50}, "core": {"precision": 38.5, "recall": 66.5, "f1": 48.7, "f2": 58.0, "tp": 105, "fp": 168, "fn": 53, "num_prs": 50}, "all": {"precision": 40.6, "recall": 66.5, "f1": 50.4, "f2": 59.0, "tp": 115, "fp": 168, "fn": 58, "num_prs": 50}}, "greptile-v4-1": {"strict": {"precision": 49.0, "recall": 54.0, "f1": 51.4, "f2": 52.9, "tp": 75, "fp": 78, "fn": 64, "num_prs": 50}, "core": {"precision": 50.9, "recall": 51.3, "f1": 51.1, "f2": 51.2, "tp": 81, "fp": 78, "fn": 77, "num_prs": 50}, "all": {"precision": 53.3, "recall": 51.4, "f1": 52.4, "f2": 51.8, "tp": 89, "fp": 78, "fn": 84, "num_prs": 50}}, "sourcery": {"strict": {"precision": 38.4, "recall": 56.1, "f1": 45.6, "f2": 51.4, "tp": 78, "fp": 125, "fn": 61, "num_prs": 50}, "core": {"precision": 41.0, "recall": 55.1, "f1": 47.0, "f2": 51.5, "tp": 87, "fp": 125, "fn": 71, "num_prs": 50}, "all": {"precision": 43.2, "recall": 54.9, "f1": 48.3, "f2": 52.1, "tp": 95, "fp": 125, "fn": 78, "num_prs": 50}}, "qodo-v2": {"strict": {"precision": 53.0, "recall": 64.0, "f1": 58.0, "f2": 61.5, "tp": 89, "fp": 79, "fn": 50, "num_prs": 50}, "core": {"precision": 55.4, "recall": 62.0, "f1": 58.5, "f2": 60.6, "tp": 98, "fp": 79, "fn": 60, "num_prs": 50}, "all": {"precision": 56.4, "recall": 59.0, "f1": 57.6, "f2": 58.4, "tp": 102, "fp": 79, "fn": 71, "num_prs": 50}}, "gitlab": {"strict": {"precision": 51.0, "recall": 54.0, "f1": 52.4, "f2": 53.3, "tp": 75, "fp": 72, "fn": 64, "num_prs": 50}, "core": {"precision": 53.5, "recall": 52.5, "f1": 53.0, "f2": 52.7, "tp": 83, "fp": 72, "fn": 75, "num_prs": 50}, "all": {"precision": 55.8, "recall": 52.6, "f1": 54.2, "f2": 53.2, "tp": 91, "fp": 72, "fn": 82, "num_prs": 50}}, "codeant-v2": {"strict": {"precision": 39.0, "recall": 44.6, "f1": 41.6, "f2": 43.4, "tp": 62, "fp": 97, "fn": 77, "num_prs": 50}, "core": {"precision": 40.1, "recall": 41.1, "f1": 40.6, "f2": 40.9, "tp": 65, "fp": 97, "fn": 93, "num_prs": 50}, "all": {"precision": 41.2, "recall": 39.3, "f1": 40.2, "f2": 39.7, "tp": 68, "fp": 97, "fn": 105, "num_prs": 50}}, "bugbot": {"strict": {"precision": 54.8, "recall": 48.9, "f1": 51.7, "f2": 50.0, "tp": 68, "fp": 56, "fn": 71, "num_prs": 50}, "core": {"precision": 56.9, "recall": 46.8, "f1": 51.4, "f2": 48.6, "tp": 74, "fp": 56, "fn": 84, "num_prs": 50}, "all": {"precision": 58.2, "recall": 45.1, "f1": 50.8, "f2": 47.2, "tp": 78, "fp": 56, "fn": 95, "num_prs": 50}}, "claude-code": {"strict": {"precision": 44.3, "recall": 50.4, "f1": 47.1, "f2": 49.0, "tp": 70, "fp": 88, "fn": 69, "num_prs": 50}, "core": {"precision": 46.3, "recall": 48.1, "f1": 47.2, "f2": 47.7, "tp": 76, "fp": 88, "fn": 82, "num_prs": 50}, "all": {"precision": 48.8, "recall": 48.6, "f1": 48.7, "f2": 48.6, "tp": 84, "fp": 88, "fn": 89, "num_prs": 50}}, "devin": {"strict": {"precision": 67.4, "recall": 43.2, "f1": 52.6, "f2": 46.5, "tp": 60, "fp": 29, "fn": 79, "num_prs": 50}, "core": {"precision": 69.1, "recall": 41.1, "f1": 51.6, "f2": 44.8, "tp": 65, "fp": 29, "fn": 93, "num_prs": 50}, "all": {"precision": 70.1, "recall": 39.3, "f1": 50.4, "f2": 43.1, "tp": 68, "fp": 29, "fn": 105, "num_prs": 50}}, "macroscope": {"strict": {"precision": 60.2, "recall": 53.2, "f1": 56.5, "f2": 54.5, "tp": 74, "fp": 49, "fn": 65, "num_prs": 50}, "core": {"precision": 61.1, "recall": 48.7, "f1": 54.2, "f2": 50.8, "tp": 77, "fp": 49, "fn": 81, "num_prs": 50}, "all": {"precision": 62.3, "recall": 46.8, "f1": 53.5, "f2": 49.3, "tp": 81, "fp": 49, "fn": 92, "num_prs": 50}}, "claude": {"strict": {"precision": 41.0, "recall": 39.6, "f1": 40.3, "f2": 39.9, "tp": 55, "fp": 79, "fn": 84, "num_prs": 50}, "core": {"precision": 43.2, "recall": 38.0, "f1": 40.4, "f2": 38.9, "tp": 60, "fp": 79, "fn": 98, "num_prs": 50}, "all": {"precision": 45.5, "recall": 38.2, "f1": 41.5, "f2": 39.4, "tp": 66, "fp": 79, "fn": 107, "num_prs": 50}}, "gemini-v2": {"strict": {"precision": 43.2, "recall": 57.6, "f1": 49.4, "f2": 54.0, "tp": 80, "fp": 105, "fn": 59, "num_prs": 50}, "core": {"precision": 45.3, "recall": 55.1, "f1": 49.7, "f2": 52.8, "tp": 87, "fp": 105, "fn": 71, "num_prs": 50}, "all": {"precision": 47.5, "recall": 54.9, "f1": 50.9, "f2": 53.3, "tp": 95, "fp": 105, "fn": 78, "num_prs": 50}}, "coderabbit": {"strict": {"precision": 31.0, "recall": 62.6, "f1": 41.4, "f2": 52.0, "tp": 87, "fp": 194, "fn": 52, "num_prs": 50}, "core": {"precision": 32.6, "recall": 59.5, "f1": 42.2, "f2": 51.1, "tp": 94, "fp": 194, "fn": 64, "num_prs": 50}, "all": {"precision": 34.7, "recall": 59.5, "f1": 43.8, "f2": 52.1, "tp": 103, "fp": 194, "fn": 70, "num_prs": 50}}, "baz": {"strict": {"precision": 60.6, "recall": 41.0, "f1": 48.9, "f2": 43.8, "tp": 57, "fp": 37, "fn": 82, "num_prs": 50}, "core": {"precision": 61.5, "recall": 37.3, "f1": 46.5, "f2": 40.5, "tp": 59, "fp": 37, "fn": 99, "num_prs": 50}, "all": {"precision": 62.6, "recall": 35.8, "f1": 45.6, "f2": 39.2, "tp": 62, "fp": 37, "fn": 111, "num_prs": 50}}, "kg": {"strict": {"precision": 53.3, "recall": 17.3, "f1": 26.1, "f2": 20.0, "tp": 24, "fp": 21, "fn": 115, "num_prs": 50}, "core": {"precision": 54.3, "recall": 15.8, "f1": 24.5, "f2": 18.4, "tp": 25, "fp": 21, "fn": 133, "num_prs": 50}, "all": {"precision": 55.3, "recall": 15.0, "f1": 23.6, "f2": 17.6, "tp": 26, "fp": 21, "fn": 147, "num_prs": 50}}, "qodo-extended-v2": {"strict": {"precision": 64.8, "recall": 66.2, "f1": 65.5, "f2": 65.9, "tp": 92, "fp": 50, "fn": 47, "num_prs": 50}, "core": {"precision": 67.1, "recall": 64.6, "f1": 65.8, "f2": 65.1, "tp": 102, "fp": 50, "fn": 56, "num_prs": 50}, "all": {"precision": 67.9, "recall": 61.3, "f1": 64.4, "f2": 62.5, "tp": 106, "fp": 50, "fn": 67, "num_prs": 50}}, "kodus-v2": {"strict": {"precision": 57.1, "recall": 40.3, "f1": 47.3, "f2": 42.8, "tp": 56, "fp": 42, "fn": 83, "num_prs": 50}, "core": {"precision": 59.2, "recall": 38.6, "f1": 46.7, "f2": 41.5, "tp": 61, "fp": 42, "fn": 97, "num_prs": 50}, "all": {"precision": 59.6, "recall": 35.8, "f1": 44.8, "f2": 38.9, "tp": 62, "fp": 42, "fn": 111, "num_prs": 50}}, "graphite": {"strict": {"precision": 100.0, "recall": 7.9, "f1": 14.7, "f2": 9.7, "tp": 11, "fp": 0, "fn": 128, "num_prs": 50}, "core": {"precision": 100.0, "recall": 7.6, "f1": 14.1, "f2": 9.3, "tp": 12, "fp": 0, "fn": 146, "num_prs": 50}, "all": {"precision": 100.0, "recall": 6.9, "f1": 13.0, "f2": 8.5, "tp": 12, "fp": 0, "fn": 161, "num_prs": 50}}}}, "anthropic_claude-sonnet-4-5-20250929": {"prs": [{"url": "https://github.com/keycloak/keycloak/pull/37429", "language": "Java", "pr_size": "medium", "domain": "UI", "change_type": "feature", "complexity": "moderate", "difficulty": "moderate", "risk": "medium", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"augment": {"strict": {"tp": 0, "fp": 3, "fn": 1}, "core": {"tp": 2, "fp": 3, "fn": 1}, "all": {"tp": 2, "fp": 3, "fn": 3}}, "baz": {"strict": {"tp": 0, "fp": 1, "fn": 1}, "core": {"tp": 0, "fp": 1, "fn": 3}, "all": {"tp": 0, "fp": 1, "fn": 5}}, "bugbot": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 2, "fp": 0, "fn": 1}, "all": {"tp": 2, "fp": 0, "fn": 3}}, "claude": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 1, "fp": 0, "fn": 4}}, "claude-code": {"strict": {"tp": 0, "fp": 4, "fn": 1}, "core": {"tp": 0, "fp": 4, "fn": 3}, "all": {"tp": 1, "fp": 4, "fn": 4}}, "codeant-v2": {"strict": {"tp": 0, "fp": 1, "fn": 1}, "core": {"tp": 0, "fp": 1, "fn": 3}, "all": {"tp": 0, "fp": 1, "fn": 5}}, "coderabbit": {"strict": {"tp": 0, "fp": 5, "fn": 1}, "core": {"tp": 2, "fp": 5, "fn": 1}, "all": {"tp": 4, "fp": 5, "fn": 1}}, "cubic-v2": {"strict": {"tp": 0, "fp": 2, "fn": 1}, "core": {"tp": 1, "fp": 2, "fn": 2}, "all": {"tp": 1, "fp": 2, "fn": 4}}, "devin": {"strict": {"tp": 0, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 1}, "all": {"tp": 2, "fp": 1, "fn": 3}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 5}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 4}}, "kodus-v2": {"strict": {"tp": 0, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 2}, "all": {"tp": 1, "fp": 1, "fn": 4}}, "macroscope": {"strict": {"tp": 0, "fp": 1, "fn": 1}, "core": {"tp": 0, "fp": 1, "fn": 3}, "all": {"tp": 0, "fp": 1, "fn": 5}}, "qodo-extended-v2": {"strict": {"tp": 0, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 1}, "all": {"tp": 2, "fp": 1, "fn": 3}}, "qodo-v2": {"strict": {"tp": 0, "fp": 3, "fn": 1}, "core": {"tp": 0, "fp": 3, "fn": 3}, "all": {"tp": 0, "fp": 3, "fn": 5}}, "sourcery": {"strict": {"tp": 0, "fp": 5, "fn": 1}, "core": {"tp": 0, "fp": 5, "fn": 3}, "all": {"tp": 1, "fp": 5, "fn": 4}}, "greptile-v4-1": {"strict": {"tp": 0, "fp": 3, "fn": 1}, "core": {"tp": 0, "fp": 3, "fn": 3}, "all": {"tp": 2, "fp": 3, "fn": 3}}, "gemini-v2": {"strict": {"tp": 0, "fp": 5, "fn": 1}, "core": {"tp": 1, "fp": 5, "fn": 2}, "all": {"tp": 2, "fp": 5, "fn": 3}}, "gitlab": {"strict": {"tp": 0, "fp": 2, "fn": 1}, "core": {"tp": 1, "fp": 2, "fn": 2}, "all": {"tp": 2, "fp": 2, "fn": 3}}, "copilot-v2": {"strict": {"tp": 0, "fp": 5, "fn": 1}, "core": {"tp": 2, "fp": 5, "fn": 1}, "all": {"tp": 3, "fp": 5, "fn": 2}}}}, {"url": "https://github.com/keycloak/keycloak/pull/37634", "language": "Java", "pr_size": "medium", "domain": "authentication", "change_type": "feature", "complexity": "moderate", "difficulty": "subtle", "risk": "high", "context": "file", "concern": "correctness", "summary": "", "tool_metrics": {"augment": {"strict": {"tp": 2, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 2}, "all": {"tp": 2, "fp": 1, "fn": 2}}, "baz": {"strict": {"tp": 2, "fp": 0, "fn": 1}, "core": {"tp": 2, "fp": 0, "fn": 2}, "all": {"tp": 2, "fp": 0, "fn": 2}}, "bugbot": {"strict": {"tp": 2, "fp": 2, "fn": 1}, "core": {"tp": 2, "fp": 2, "fn": 2}, "all": {"tp": 2, "fp": 2, "fn": 2}}, "claude": {"strict": {"tp": 1, "fp": 3, "fn": 2}, "core": {"tp": 2, "fp": 3, "fn": 2}, "all": {"tp": 2, "fp": 3, "fn": 2}}, "claude-code": {"strict": {"tp": 0, "fp": 2, "fn": 3}, "core": {"tp": 0, "fp": 2, "fn": 4}, "all": {"tp": 0, "fp": 2, "fn": 4}}, "codeant-v2": {"strict": {"tp": 1, "fp": 2, "fn": 2}, "core": {"tp": 1, "fp": 2, "fn": 3}, "all": {"tp": 1, "fp": 2, "fn": 3}}, "coderabbit": {"strict": {"tp": 3, "fp": 6, "fn": 0}, "core": {"tp": 4, "fp": 6, "fn": 0}, "all": {"tp": 4, "fp": 6, "fn": 0}}, "cubic-v2": {"strict": {"tp": 2, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 2}, "all": {"tp": 2, "fp": 1, "fn": 2}}, "devin": {"strict": {"tp": 2, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 2}, "all": {"tp": 2, "fp": 1, "fn": 2}}, "graphite": {"strict": {"tp": 2, "fp": 0, "fn": 1}, "core": {"tp": 2, "fp": 0, "fn": 2}, "all": {"tp": 2, "fp": 0, "fn": 2}}, "kg": {"strict": {"tp": 1, "fp": 2, "fn": 2}, "core": {"tp": 1, "fp": 2, "fn": 3}, "all": {"tp": 1, "fp": 2, "fn": 3}}, "kodus-v2": {"strict": {"tp": 2, "fp": 0, "fn": 1}, "core": {"tp": 2, "fp": 0, "fn": 2}, "all": {"tp": 2, "fp": 0, "fn": 2}}, "macroscope": {"strict": {"tp": 2, "fp": 2, "fn": 1}, "core": {"tp": 2, "fp": 2, "fn": 2}, "all": {"tp": 2, "fp": 2, "fn": 2}}, "qodo-extended-v2": {"strict": {"tp": 2, "fp": 1, "fn": 1}, "core": {"tp": 3, "fp": 1, "fn": 1}, "all": {"tp": 3, "fp": 1, "fn": 1}}, "qodo-v2": {"strict": {"tp": 2, "fp": 0, "fn": 1}, "core": {"tp": 2, "fp": 0, "fn": 2}, "all": {"tp": 2, "fp": 0, "fn": 2}}, "sourcery": {"strict": {"tp": 2, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 2}, "all": {"tp": 2, "fp": 1, "fn": 2}}, "greptile-v4-1": {"strict": {"tp": 2, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 2}, "all": {"tp": 2, "fp": 1, "fn": 2}}, "gemini-v2": {"strict": {"tp": 2, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 2}, "all": {"tp": 2, "fp": 1, "fn": 2}}, "gitlab": {"strict": {"tp": 1, "fp": 1, "fn": 2}, "core": {"tp": 2, "fp": 1, "fn": 2}, "all": {"tp": 2, "fp": 1, "fn": 2}}, "copilot-v2": {"strict": {"tp": 2, "fp": 3, "fn": 1}, "core": {"tp": 3, "fp": 3, "fn": 1}, "all": {"tp": 3, "fp": 3, "fn": 1}}}}, {"url": "https://github.com/keycloak/keycloak/pull/38446", "language": "Java", "pr_size": "medium", "domain": "authentication", "change_type": "feature", "complexity": "moderate", "difficulty": "moderate", "risk": "medium", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"augment": {"strict": {"tp": 1, "fp": 3, "fn": 1}, "core": {"tp": 1, "fp": 3, "fn": 1}, "all": {"tp": 1, "fp": 3, "fn": 1}}, "baz": {"strict": {"tp": 0, "fp": 1, "fn": 2}, "core": {"tp": 0, "fp": 1, "fn": 2}, "all": {"tp": 0, "fp": 1, "fn": 2}}, "bugbot": {"strict": {"tp": 0, "fp": 3, "fn": 2}, "core": {"tp": 0, "fp": 3, "fn": 2}, "all": {"tp": 0, "fp": 3, "fn": 2}}, "claude": {"strict": {"tp": 1, "fp": 3, "fn": 1}, "core": {"tp": 1, "fp": 3, "fn": 1}, "all": {"tp": 1, "fp": 3, "fn": 1}}, "claude-code": {"strict": {"tp": 1, "fp": 5, "fn": 1}, "core": {"tp": 1, "fp": 5, "fn": 1}, "all": {"tp": 1, "fp": 5, "fn": 1}}, "codeant-v2": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "coderabbit": {"strict": {"tp": 2, "fp": 2, "fn": 0}, "core": {"tp": 2, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "cubic-v2": {"strict": {"tp": 1, "fp": 3, "fn": 1}, "core": {"tp": 1, "fp": 3, "fn": 1}, "all": {"tp": 1, "fp": 3, "fn": 1}}, "devin": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "kodus-v2": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "macroscope": {"strict": {"tp": 0, "fp": 3, "fn": 2}, "core": {"tp": 0, "fp": 3, "fn": 2}, "all": {"tp": 0, "fp": 3, "fn": 2}}, "qodo-extended-v2": {"strict": {"tp": 1, "fp": 4, "fn": 1}, "core": {"tp": 1, "fp": 4, "fn": 1}, "all": {"tp": 1, "fp": 4, "fn": 1}}, "qodo-v2": {"strict": {"tp": 2, "fp": 3, "fn": 0}, "core": {"tp": 2, "fp": 3, "fn": 0}, "all": {"tp": 2, "fp": 3, "fn": 0}}, "sourcery": {"strict": {"tp": 1, "fp": 6, "fn": 1}, "core": {"tp": 1, "fp": 6, "fn": 1}, "all": {"tp": 1, "fp": 6, "fn": 1}}, "greptile-v4-1": {"strict": {"tp": 2, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "gemini-v2": {"strict": {"tp": 1, "fp": 3, "fn": 1}, "core": {"tp": 1, "fp": 3, "fn": 1}, "all": {"tp": 1, "fp": 3, "fn": 1}}, "gitlab": {"strict": {"tp": 1, "fp": 5, "fn": 1}, "core": {"tp": 1, "fp": 5, "fn": 1}, "all": {"tp": 1, "fp": 5, "fn": 1}}, "copilot-v2": {"strict": {"tp": 1, "fp": 6, "fn": 1}, "core": {"tp": 1, "fp": 6, "fn": 1}, "all": {"tp": 1, "fp": 6, "fn": 1}}}}, {"url": "https://github.com/keycloak/keycloak/pull/36882", "language": "Java", "pr_size": "small", "domain": "configuration", "change_type": "feature", "complexity": "moderate", "difficulty": "moderate", "risk": "medium", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"augment": {"strict": {"tp": 0, "fp": 1, "fn": 1}, "core": {"tp": 0, "fp": 1, "fn": 2}, "all": {"tp": 0, "fp": 1, "fn": 2}}, "baz": {"strict": {"tp": 0, "fp": 1, "fn": 1}, "core": {"tp": 0, "fp": 1, "fn": 2}, "all": {"tp": 0, "fp": 1, "fn": 2}}, "bugbot": {"strict": {"tp": 0, "fp": 1, "fn": 1}, "core": {"tp": 0, "fp": 1, "fn": 2}, "all": {"tp": 0, "fp": 1, "fn": 2}}, "claude": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "claude-code": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "codeant-v2": {"strict": {"tp": 0, "fp": 1, "fn": 1}, "core": {"tp": 0, "fp": 1, "fn": 2}, "all": {"tp": 0, "fp": 1, "fn": 2}}, "coderabbit": {"strict": {"tp": 0, "fp": 2, "fn": 1}, "core": {"tp": 0, "fp": 2, "fn": 2}, "all": {"tp": 0, "fp": 2, "fn": 2}}, "cubic-v2": {"strict": {"tp": 1, "fp": 2, "fn": 0}, "core": {"tp": 1, "fp": 2, "fn": 1}, "all": {"tp": 1, "fp": 2, "fn": 1}}, "devin": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "kodus-v2": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "macroscope": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "qodo-extended-v2": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "qodo-v2": {"strict": {"tp": 0, "fp": 1, "fn": 1}, "core": {"tp": 0, "fp": 1, "fn": 2}, "all": {"tp": 0, "fp": 1, "fn": 2}}, "sourcery": {"strict": {"tp": 0, "fp": 4, "fn": 1}, "core": {"tp": 0, "fp": 4, "fn": 2}, "all": {"tp": 0, "fp": 4, "fn": 2}}, "greptile-v4-1": {"strict": {"tp": 0, "fp": 2, "fn": 1}, "core": {"tp": 0, "fp": 2, "fn": 2}, "all": {"tp": 0, "fp": 2, "fn": 2}}, "gemini-v2": {"strict": {"tp": 0, "fp": 2, "fn": 1}, "core": {"tp": 1, "fp": 2, "fn": 1}, "all": {"tp": 1, "fp": 2, "fn": 1}}, "gitlab": {"strict": {"tp": 0, "fp": 2, "fn": 1}, "core": {"tp": 0, "fp": 2, "fn": 2}, "all": {"tp": 0, "fp": 2, "fn": 2}}, "copilot-v2": {"strict": {"tp": 0, "fp": 3, "fn": 1}, "core": {"tp": 0, "fp": 3, "fn": 2}, "all": {"tp": 0, "fp": 3, "fn": 2}}}}, {"url": "https://github.com/keycloak/keycloak/pull/36880", "language": "Java", "pr_size": "medium", "domain": "authentication", "change_type": "feature", "complexity": "complex", "difficulty": "very_subtle", "risk": "high", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"augment": {"strict": {"tp": 1, "fp": 1, "fn": 3}, "core": {"tp": 1, "fp": 1, "fn": 3}, "all": {"tp": 1, "fp": 1, "fn": 4}}, "baz": {"strict": {"tp": 2, "fp": 0, "fn": 2}, "core": {"tp": 2, "fp": 0, "fn": 2}, "all": {"tp": 2, "fp": 0, "fn": 3}}, "bugbot": {"strict": {"tp": 2, "fp": 1, "fn": 2}, "core": {"tp": 2, "fp": 1, "fn": 2}, "all": {"tp": 3, "fp": 1, "fn": 2}}, "claude": {"strict": {"tp": 1, "fp": 3, "fn": 3}, "core": {"tp": 1, "fp": 3, "fn": 3}, "all": {"tp": 2, "fp": 3, "fn": 3}}, "claude-code": {"strict": {"tp": 0, "fp": 2, "fn": 4}, "core": {"tp": 0, "fp": 2, "fn": 4}, "all": {"tp": 0, "fp": 2, "fn": 5}}, "codeant-v2": {"strict": {"tp": 2, "fp": 0, "fn": 2}, "core": {"tp": 2, "fp": 0, "fn": 2}, "all": {"tp": 2, "fp": 0, "fn": 3}}, "coderabbit": {"strict": {"tp": 1, "fp": 3, "fn": 3}, "core": {"tp": 1, "fp": 3, "fn": 3}, "all": {"tp": 1, "fp": 3, "fn": 4}}, "cubic-v2": {"strict": {"tp": 1, "fp": 1, "fn": 3}, "core": {"tp": 1, "fp": 1, "fn": 3}, "all": {"tp": 1, "fp": 1, "fn": 4}}, "devin": {"strict": {"tp": 1, "fp": 0, "fn": 3}, "core": {"tp": 1, "fp": 0, "fn": 3}, "all": {"tp": 1, "fp": 0, "fn": 4}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 4}, "core": {"tp": 0, "fp": 0, "fn": 4}, "all": {"tp": 0, "fp": 0, "fn": 5}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 4}, "core": {"tp": 0, "fp": 0, "fn": 4}, "all": {"tp": 0, "fp": 0, "fn": 5}}, "kodus-v2": {"strict": {"tp": 2, "fp": 2, "fn": 2}, "core": {"tp": 2, "fp": 2, "fn": 2}, "all": {"tp": 2, "fp": 2, "fn": 3}}, "macroscope": {"strict": {"tp": 2, "fp": 4, "fn": 2}, "core": {"tp": 2, "fp": 4, "fn": 2}, "all": {"tp": 2, "fp": 4, "fn": 3}}, "qodo-extended-v2": {"strict": {"tp": 2, "fp": 0, "fn": 2}, "core": {"tp": 2, "fp": 0, "fn": 2}, "all": {"tp": 2, "fp": 0, "fn": 3}}, "qodo-v2": {"strict": {"tp": 1, "fp": 2, "fn": 3}, "core": {"tp": 1, "fp": 2, "fn": 3}, "all": {"tp": 1, "fp": 2, "fn": 4}}, "sourcery": {"strict": {"tp": 1, "fp": 4, "fn": 3}, "core": {"tp": 1, "fp": 4, "fn": 3}, "all": {"tp": 2, "fp": 4, "fn": 3}}, "greptile-v4-1": {"strict": {"tp": 1, "fp": 0, "fn": 3}, "core": {"tp": 1, "fp": 0, "fn": 3}, "all": {"tp": 2, "fp": 0, "fn": 3}}, "gemini-v2": {"strict": {"tp": 1, "fp": 3, "fn": 3}, "core": {"tp": 1, "fp": 3, "fn": 3}, "all": {"tp": 2, "fp": 3, "fn": 3}}, "gitlab": {"strict": {"tp": 1, "fp": 6, "fn": 3}, "core": {"tp": 1, "fp": 6, "fn": 3}, "all": {"tp": 2, "fp": 6, "fn": 3}}, "copilot-v2": {"strict": {"tp": 1, "fp": 6, "fn": 3}, "core": {"tp": 1, "fp": 6, "fn": 3}, "all": {"tp": 1, "fp": 6, "fn": 4}}}}, {"url": "https://github.com/keycloak/keycloak/pull/37038", "language": "Java", "pr_size": "small", "domain": "authentication", "change_type": "feature", "complexity": "complex", "difficulty": "very_subtle", "risk": "high", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"augment": {"strict": {"tp": 2, "fp": 3, "fn": 0}, "core": {"tp": 2, "fp": 3, "fn": 0}, "all": {"tp": 2, "fp": 3, "fn": 0}}, "baz": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "bugbot": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "claude": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "claude-code": {"strict": {"tp": 1, "fp": 4, "fn": 1}, "core": {"tp": 1, "fp": 4, "fn": 1}, "all": {"tp": 1, "fp": 4, "fn": 1}}, "codeant-v2": {"strict": {"tp": 1, "fp": 5, "fn": 1}, "core": {"tp": 1, "fp": 5, "fn": 1}, "all": {"tp": 1, "fp": 5, "fn": 1}}, "coderabbit": {"strict": {"tp": 2, "fp": 8, "fn": 0}, "core": {"tp": 2, "fp": 8, "fn": 0}, "all": {"tp": 2, "fp": 8, "fn": 0}}, "cubic-v2": {"strict": {"tp": 2, "fp": 2, "fn": 0}, "core": {"tp": 2, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "devin": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "kodus-v2": {"strict": {"tp": 2, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "macroscope": {"strict": {"tp": 2, "fp": 2, "fn": 0}, "core": {"tp": 2, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "qodo-extended-v2": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "qodo-v2": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "sourcery": {"strict": {"tp": 1, "fp": 3, "fn": 1}, "core": {"tp": 1, "fp": 3, "fn": 1}, "all": {"tp": 1, "fp": 3, "fn": 1}}, "greptile-v4-1": {"strict": {"tp": 2, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "gemini-v2": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "gitlab": {"strict": {"tp": 1, "fp": 2, "fn": 1}, "core": {"tp": 1, "fp": 2, "fn": 1}, "all": {"tp": 1, "fp": 2, "fn": 1}}, "copilot-v2": {"strict": {"tp": 2, "fp": 3, "fn": 0}, "core": {"tp": 2, "fp": 3, "fn": 0}, "all": {"tp": 2, "fp": 3, "fn": 0}}}}, {"url": "https://github.com/keycloak/keycloak/pull/33832", "language": "Java", "pr_size": "medium", "domain": "authentication", "change_type": "feature", "complexity": "moderate", "difficulty": "subtle", "risk": "high", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"augment": {"strict": {"tp": 1, "fp": 2, "fn": 1}, "core": {"tp": 1, "fp": 2, "fn": 2}, "all": {"tp": 1, "fp": 2, "fn": 3}}, "baz": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 2, "fp": 0, "fn": 1}, "all": {"tp": 2, "fp": 0, "fn": 2}}, "bugbot": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 3}}, "claude": {"strict": {"tp": 0, "fp": 5, "fn": 2}, "core": {"tp": 1, "fp": 5, "fn": 2}, "all": {"tp": 1, "fp": 5, "fn": 3}}, "claude-code": {"strict": {"tp": 0, "fp": 1, "fn": 2}, "core": {"tp": 1, "fp": 1, "fn": 2}, "all": {"tp": 2, "fp": 1, "fn": 2}}, "codeant-v2": {"strict": {"tp": 0, "fp": 2, "fn": 2}, "core": {"tp": 0, "fp": 2, "fn": 3}, "all": {"tp": 1, "fp": 2, "fn": 3}}, "coderabbit": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 1}, "all": {"tp": 2, "fp": 1, "fn": 2}}, "cubic-v2": {"strict": {"tp": 1, "fp": 2, "fn": 1}, "core": {"tp": 2, "fp": 2, "fn": 1}, "all": {"tp": 2, "fp": 2, "fn": 2}}, "devin": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 4}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 3}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 4}}, "kodus-v2": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 4}}, "macroscope": {"strict": {"tp": 1, "fp": 5, "fn": 1}, "core": {"tp": 1, "fp": 5, "fn": 2}, "all": {"tp": 1, "fp": 5, "fn": 3}}, "qodo-extended-v2": {"strict": {"tp": 0, "fp": 2, "fn": 2}, "core": {"tp": 1, "fp": 2, "fn": 2}, "all": {"tp": 1, "fp": 2, "fn": 3}}, "qodo-v2": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 2}, "all": {"tp": 1, "fp": 1, "fn": 3}}, "sourcery": {"strict": {"tp": 0, "fp": 2, "fn": 2}, "core": {"tp": 1, "fp": 2, "fn": 2}, "all": {"tp": 1, "fp": 2, "fn": 3}}, "greptile-v4-1": {"strict": {"tp": 0, "fp": 3, "fn": 2}, "core": {"tp": 1, "fp": 3, "fn": 2}, "all": {"tp": 1, "fp": 3, "fn": 3}}, "gemini-v2": {"strict": {"tp": 1, "fp": 2, "fn": 1}, "core": {"tp": 2, "fp": 2, "fn": 1}, "all": {"tp": 2, "fp": 2, "fn": 2}}, "gitlab": {"strict": {"tp": 0, "fp": 2, "fn": 2}, "core": {"tp": 1, "fp": 2, "fn": 2}, "all": {"tp": 1, "fp": 2, "fn": 3}}, "copilot-v2": {"strict": {"tp": 1, "fp": 4, "fn": 1}, "core": {"tp": 2, "fp": 4, "fn": 1}, "all": {"tp": 3, "fp": 4, "fn": 1}}}}, {"url": "https://github.com/keycloak/keycloak/pull/40940", "language": "Java", "pr_size": "small", "domain": "concurrency", "change_type": "bug_fix", "complexity": "moderate", "difficulty": "subtle", "risk": "high", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"augment": {"strict": {"tp": 1, "fp": 2, "fn": 1}, "core": {"tp": 1, "fp": 2, "fn": 1}, "all": {"tp": 1, "fp": 2, "fn": 1}}, "baz": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "bugbot": {"strict": {"tp": 1, "fp": 2, "fn": 1}, "core": {"tp": 1, "fp": 2, "fn": 1}, "all": {"tp": 1, "fp": 2, "fn": 1}}, "claude": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "claude-code": {"strict": {"tp": 1, "fp": 3, "fn": 1}, "core": {"tp": 1, "fp": 3, "fn": 1}, "all": {"tp": 1, "fp": 3, "fn": 1}}, "codeant-v2": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "coderabbit": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "cubic-v2": {"strict": {"tp": 2, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "devin": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "graphite": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "kodus-v2": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "macroscope": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "qodo-extended-v2": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "qodo-v2": {"strict": {"tp": 2, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "sourcery": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "greptile-v4-1": {"strict": {"tp": 1, "fp": 3, "fn": 1}, "core": {"tp": 1, "fp": 3, "fn": 1}, "all": {"tp": 1, "fp": 3, "fn": 1}}, "gemini-v2": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "gitlab": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "copilot-v2": {"strict": {"tp": 2, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}}}, {"url": "https://github.com/ai-code-review-evaluation/keycloak-greptile/pull/1", "language": "Java", "pr_size": "medium", "domain": "authentication", "change_type": "bug_fix", "complexity": "moderate", "difficulty": "subtle", "risk": "medium", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"augment": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "baz": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "bugbot": {"strict": {"tp": 0, "fp": 1, "fn": 1}, "core": {"tp": 0, "fp": 1, "fn": 1}, "all": {"tp": 0, "fp": 1, "fn": 2}}, "claude": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 0}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "claude-code": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 0}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "codeant-v2": {"strict": {"tp": 1, "fp": 2, "fn": 0}, "core": {"tp": 1, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "coderabbit": {"strict": {"tp": 1, "fp": 2, "fn": 0}, "core": {"tp": 1, "fp": 2, "fn": 0}, "all": {"tp": 1, "fp": 2, "fn": 1}}, "cubic-v2": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 1, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "devin": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "kg": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 0}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "kodus-v2": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "macroscope": {"strict": {"tp": 0, "fp": 1, "fn": 1}, "core": {"tp": 0, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "qodo-extended-v2": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 1, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "qodo-v2": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 1, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "sourcery": {"strict": {"tp": 0, "fp": 3, "fn": 1}, "core": {"tp": 0, "fp": 3, "fn": 1}, "all": {"tp": 1, "fp": 3, "fn": 1}}, "greptile-v4-1": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "gemini-v2": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 1, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "gitlab": {"strict": {"tp": 1, "fp": 3, "fn": 0}, "core": {"tp": 1, "fp": 3, "fn": 0}, "all": {"tp": 1, "fp": 3, "fn": 1}}, "copilot-v2": {"strict": {"tp": 1, "fp": 2, "fn": 0}, "core": {"tp": 1, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}}}, {"url": "https://github.com/getsentry/sentry/pull/93824", "language": "Python", "pr_size": "large", "domain": "concurrency", "change_type": "feature", "complexity": "complex", "difficulty": "subtle", "risk": "high", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"augment": {"strict": {"tp": 3, "fp": 1, "fn": 0}, "core": {"tp": 5, "fp": 1, "fn": 0}, "all": {"tp": 5, "fp": 1, "fn": 0}}, "baz": {"strict": {"tp": 0, "fp": 3, "fn": 3}, "core": {"tp": 0, "fp": 3, "fn": 5}, "all": {"tp": 0, "fp": 3, "fn": 5}}, "bugbot": {"strict": {"tp": 1, "fp": 0, "fn": 2}, "core": {"tp": 1, "fp": 0, "fn": 4}, "all": {"tp": 1, "fp": 0, "fn": 4}}, "claude": {"strict": {"tp": 0, "fp": 2, "fn": 3}, "core": {"tp": 0, "fp": 2, "fn": 5}, "all": {"tp": 0, "fp": 2, "fn": 5}}, "claude-code": {"strict": {"tp": 3, "fp": 2, "fn": 0}, "core": {"tp": 3, "fp": 2, "fn": 2}, "all": {"tp": 3, "fp": 2, "fn": 2}}, "codeant-v2": {"strict": {"tp": 0, "fp": 0, "fn": 3}, "core": {"tp": 2, "fp": 0, "fn": 3}, "all": {"tp": 2, "fp": 0, "fn": 3}}, "coderabbit": {"strict": {"tp": 1, "fp": 3, "fn": 2}, "core": {"tp": 1, "fp": 3, "fn": 4}, "all": {"tp": 1, "fp": 3, "fn": 4}}, "cubic-v2": {"strict": {"tp": 3, "fp": 1, "fn": 0}, "core": {"tp": 5, "fp": 1, "fn": 0}, "all": {"tp": 5, "fp": 1, "fn": 0}}, "devin": {"strict": {"tp": 1, "fp": 1, "fn": 2}, "core": {"tp": 1, "fp": 1, "fn": 4}, "all": {"tp": 1, "fp": 1, "fn": 4}}, "graphite": {"strict": {"tp": 1, "fp": 0, "fn": 2}, "core": {"tp": 1, "fp": 0, "fn": 4}, "all": {"tp": 1, "fp": 0, "fn": 4}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 3}, "core": {"tp": 0, "fp": 0, "fn": 5}, "all": {"tp": 0, "fp": 0, "fn": 5}}, "kodus-v2": {"strict": {"tp": 0, "fp": 0, "fn": 3}, "core": {"tp": 0, "fp": 0, "fn": 5}, "all": {"tp": 0, "fp": 0, "fn": 5}}, "macroscope": {"strict": {"tp": 1, "fp": 1, "fn": 2}, "core": {"tp": 3, "fp": 1, "fn": 2}, "all": {"tp": 3, "fp": 1, "fn": 2}}, "qodo-extended-v2": {"strict": {"tp": 2, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 3}, "all": {"tp": 2, "fp": 1, "fn": 3}}, "qodo-v2": {"strict": {"tp": 2, "fp": 2, "fn": 1}, "core": {"tp": 4, "fp": 2, "fn": 1}, "all": {"tp": 4, "fp": 2, "fn": 1}}, "sourcery": {"strict": {"tp": 2, "fp": 2, "fn": 1}, "core": {"tp": 4, "fp": 2, "fn": 1}, "all": {"tp": 4, "fp": 2, "fn": 1}}, "greptile-v4-1": {"strict": {"tp": 1, "fp": 1, "fn": 2}, "core": {"tp": 1, "fp": 1, "fn": 4}, "all": {"tp": 1, "fp": 1, "fn": 4}}, "gemini-v2": {"strict": {"tp": 2, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 3}, "all": {"tp": 2, "fp": 1, "fn": 3}}, "gitlab": {"strict": {"tp": 0, "fp": 2, "fn": 3}, "core": {"tp": 0, "fp": 2, "fn": 5}, "all": {"tp": 0, "fp": 2, "fn": 5}}, "copilot-v2": {"strict": {"tp": 1, "fp": 5, "fn": 2}, "core": {"tp": 3, "fp": 5, "fn": 2}, "all": {"tp": 3, "fp": 5, "fn": 2}}}}, {"url": "https://github.com/ai-code-review-evaluation/sentry-greptile/pull/5", "language": "Python", "pr_size": "large", "domain": "API", "change_type": "feature", "complexity": "moderate", "difficulty": "subtle", "risk": "medium", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"augment": {"strict": {"tp": 3, "fp": 2, "fn": 3}, "core": {"tp": 3, "fp": 2, "fn": 3}, "all": {"tp": 3, "fp": 2, "fn": 3}}, "baz": {"strict": {"tp": 3, "fp": 1, "fn": 3}, "core": {"tp": 3, "fp": 1, "fn": 3}, "all": {"tp": 3, "fp": 1, "fn": 3}}, "bugbot": {"strict": {"tp": 2, "fp": 0, "fn": 4}, "core": {"tp": 2, "fp": 0, "fn": 4}, "all": {"tp": 2, "fp": 0, "fn": 4}}, "claude": {"strict": {"tp": 0, "fp": 0, "fn": 6}, "core": {"tp": 0, "fp": 0, "fn": 6}, "all": {"tp": 0, "fp": 0, "fn": 6}}, "claude-code": {"strict": {"tp": 0, "fp": 2, "fn": 6}, "core": {"tp": 0, "fp": 2, "fn": 6}, "all": {"tp": 0, "fp": 2, "fn": 6}}, "codeant-v2": {"strict": {"tp": 2, "fp": 5, "fn": 4}, "core": {"tp": 2, "fp": 5, "fn": 4}, "all": {"tp": 2, "fp": 5, "fn": 4}}, "coderabbit": {"strict": {"tp": 4, "fp": 13, "fn": 2}, "core": {"tp": 4, "fp": 13, "fn": 2}, "all": {"tp": 4, "fp": 13, "fn": 2}}, "cubic-v2": {"strict": {"tp": 1, "fp": 3, "fn": 5}, "core": {"tp": 1, "fp": 3, "fn": 5}, "all": {"tp": 1, "fp": 3, "fn": 5}}, "devin": {"strict": {"tp": 2, "fp": 1, "fn": 4}, "core": {"tp": 2, "fp": 1, "fn": 4}, "all": {"tp": 2, "fp": 1, "fn": 4}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 6}, "core": {"tp": 0, "fp": 0, "fn": 6}, "all": {"tp": 0, "fp": 0, "fn": 6}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 6}, "core": {"tp": 0, "fp": 0, "fn": 6}, "all": {"tp": 0, "fp": 0, "fn": 6}}, "kodus-v2": {"strict": {"tp": 2, "fp": 1, "fn": 4}, "core": {"tp": 2, "fp": 1, "fn": 4}, "all": {"tp": 2, "fp": 1, "fn": 4}}, "macroscope": {"strict": {"tp": 2, "fp": 3, "fn": 4}, "core": {"tp": 2, "fp": 3, "fn": 4}, "all": {"tp": 2, "fp": 3, "fn": 4}}, "qodo-extended-v2": {"strict": {"tp": 2, "fp": 0, "fn": 4}, "core": {"tp": 2, "fp": 0, "fn": 4}, "all": {"tp": 2, "fp": 0, "fn": 4}}, "qodo-v2": {"strict": {"tp": 2, "fp": 0, "fn": 4}, "core": {"tp": 2, "fp": 0, "fn": 4}, "all": {"tp": 2, "fp": 0, "fn": 4}}, "sourcery": {"strict": {"tp": 2, "fp": 2, "fn": 4}, "core": {"tp": 2, "fp": 2, "fn": 4}, "all": {"tp": 2, "fp": 2, "fn": 4}}, "greptile-v4-1": {"strict": {"tp": 1, "fp": 0, "fn": 5}, "core": {"tp": 1, "fp": 0, "fn": 5}, "all": {"tp": 1, "fp": 0, "fn": 5}}, "gemini-v2": {"strict": {"tp": 2, "fp": 1, "fn": 4}, "core": {"tp": 2, "fp": 1, "fn": 4}, "all": {"tp": 2, "fp": 1, "fn": 4}}, "gitlab": {"strict": {"tp": 2, "fp": 1, "fn": 4}, "core": {"tp": 2, "fp": 1, "fn": 4}, "all": {"tp": 2, "fp": 1, "fn": 4}}, "copilot-v2": {"strict": {"tp": 3, "fp": 5, "fn": 3}, "core": {"tp": 3, "fp": 5, "fn": 3}, "all": {"tp": 3, "fp": 5, "fn": 3}}}}, {"url": "https://github.com/ai-code-review-evaluation/sentry-greptile/pull/1", "language": "Python", "pr_size": "medium", "domain": "API", "change_type": "performance", "complexity": "moderate", "difficulty": "subtle", "risk": "high", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"augment": {"strict": {"tp": 3, "fp": 1, "fn": 1}, "core": {"tp": 3, "fp": 1, "fn": 1}, "all": {"tp": 3, "fp": 1, "fn": 1}}, "baz": {"strict": {"tp": 1, "fp": 1, "fn": 3}, "core": {"tp": 1, "fp": 1, "fn": 3}, "all": {"tp": 1, "fp": 1, "fn": 3}}, "bugbot": {"strict": {"tp": 3, "fp": 1, "fn": 1}, "core": {"tp": 3, "fp": 1, "fn": 1}, "all": {"tp": 3, "fp": 1, "fn": 1}}, "claude": {"strict": {"tp": 2, "fp": 7, "fn": 2}, "core": {"tp": 2, "fp": 7, "fn": 2}, "all": {"tp": 2, "fp": 7, "fn": 2}}, "claude-code": {"strict": {"tp": 2, "fp": 1, "fn": 2}, "core": {"tp": 2, "fp": 1, "fn": 2}, "all": {"tp": 2, "fp": 1, "fn": 2}}, "codeant-v2": {"strict": {"tp": 1, "fp": 0, "fn": 3}, "core": {"tp": 1, "fp": 0, "fn": 3}, "all": {"tp": 1, "fp": 0, "fn": 3}}, "coderabbit": {"strict": {"tp": 3, "fp": 3, "fn": 1}, "core": {"tp": 3, "fp": 3, "fn": 1}, "all": {"tp": 3, "fp": 3, "fn": 1}}, "cubic-v2": {"strict": {"tp": 3, "fp": 1, "fn": 1}, "core": {"tp": 3, "fp": 1, "fn": 1}, "all": {"tp": 3, "fp": 1, "fn": 1}}, "devin": {"strict": {"tp": 3, "fp": 1, "fn": 1}, "core": {"tp": 3, "fp": 1, "fn": 1}, "all": {"tp": 3, "fp": 1, "fn": 1}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 4}, "core": {"tp": 0, "fp": 0, "fn": 4}, "all": {"tp": 0, "fp": 0, "fn": 4}}, "kg": {"strict": {"tp": 1, "fp": 1, "fn": 3}, "core": {"tp": 1, "fp": 1, "fn": 3}, "all": {"tp": 1, "fp": 1, "fn": 3}}, "kodus-v2": {"strict": {"tp": 1, "fp": 2, "fn": 3}, "core": {"tp": 1, "fp": 2, "fn": 3}, "all": {"tp": 1, "fp": 2, "fn": 3}}, "macroscope": {"strict": {"tp": 2, "fp": 0, "fn": 2}, "core": {"tp": 2, "fp": 0, "fn": 2}, "all": {"tp": 2, "fp": 0, "fn": 2}}, "qodo-extended-v2": {"strict": {"tp": 3, "fp": 0, "fn": 1}, "core": {"tp": 3, "fp": 0, "fn": 1}, "all": {"tp": 3, "fp": 0, "fn": 1}}, "qodo-v2": {"strict": {"tp": 1, "fp": 2, "fn": 3}, "core": {"tp": 1, "fp": 2, "fn": 3}, "all": {"tp": 1, "fp": 2, "fn": 3}}, "sourcery": {"strict": {"tp": 2, "fp": 3, "fn": 2}, "core": {"tp": 2, "fp": 3, "fn": 2}, "all": {"tp": 2, "fp": 3, "fn": 2}}, "greptile-v4-1": {"strict": {"tp": 3, "fp": 2, "fn": 1}, "core": {"tp": 3, "fp": 2, "fn": 1}, "all": {"tp": 3, "fp": 2, "fn": 1}}, "gemini-v2": {"strict": {"tp": 2, "fp": 4, "fn": 2}, "core": {"tp": 2, "fp": 4, "fn": 2}, "all": {"tp": 2, "fp": 4, "fn": 2}}, "gitlab": {"strict": {"tp": 3, "fp": 1, "fn": 1}, "core": {"tp": 3, "fp": 1, "fn": 1}, "all": {"tp": 3, "fp": 1, "fn": 1}}, "copilot-v2": {"strict": {"tp": 2, "fp": 1, "fn": 2}, "core": {"tp": 2, "fp": 1, "fn": 2}, "all": {"tp": 2, "fp": 1, "fn": 2}}}}, {"url": "https://github.com/grafana/grafana/pull/97529", "language": "Go", "pr_size": "small", "domain": "concurrency", "change_type": "performance", "complexity": "complex", "difficulty": "very_subtle", "risk": "critical", "context": "cross_file", "concern": "reliability", "summary": "", "tool_metrics": {"augment": {"strict": {"tp": 2, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "baz": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "bugbot": {"strict": {"tp": 1, "fp": 2, "fn": 1}, "core": {"tp": 1, "fp": 2, "fn": 1}, "all": {"tp": 1, "fp": 2, "fn": 1}}, "claude": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "claude-code": {"strict": {"tp": 1, "fp": 4, "fn": 1}, "core": {"tp": 1, "fp": 4, "fn": 1}, "all": {"tp": 1, "fp": 4, "fn": 1}}, "codeant-v2": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "coderabbit": {"strict": {"tp": 1, "fp": 3, "fn": 1}, "core": {"tp": 1, "fp": 3, "fn": 1}, "all": {"tp": 1, "fp": 3, "fn": 1}}, "cubic-v2": {"strict": {"tp": 2, "fp": 2, "fn": 0}, "core": {"tp": 2, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "devin": {"strict": {"tp": 0, "fp": 2, "fn": 2}, "core": {"tp": 0, "fp": 2, "fn": 2}, "all": {"tp": 0, "fp": 2, "fn": 2}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "kg": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "kodus-v2": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "macroscope": {"strict": {"tp": 1, "fp": 2, "fn": 1}, "core": {"tp": 1, "fp": 2, "fn": 1}, "all": {"tp": 1, "fp": 2, "fn": 1}}, "qodo-extended-v2": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "qodo-v2": {"strict": {"tp": 2, "fp": 4, "fn": 0}, "core": {"tp": 2, "fp": 4, "fn": 0}, "all": {"tp": 2, "fp": 4, "fn": 0}}, "sourcery": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "greptile-v4-1": {"strict": {"tp": 2, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "gemini-v2": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "gitlab": {"strict": {"tp": 1, "fp": 2, "fn": 1}, "core": {"tp": 1, "fp": 2, "fn": 1}, "all": {"tp": 1, "fp": 2, "fn": 1}}, "copilot-v2": {"strict": {"tp": 1, "fp": 3, "fn": 1}, "core": {"tp": 1, "fp": 3, "fn": 1}, "all": {"tp": 1, "fp": 3, "fn": 1}}}}, {"url": "https://github.com/getsentry/sentry/pull/80168", "language": "Python", "pr_size": "small", "domain": "data_processing", "change_type": "feature", "complexity": "moderate", "difficulty": "moderate", "risk": "high", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"augment": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 3, "fp": 0, "fn": 0}, "all": {"tp": 3, "fp": 0, "fn": 0}}, "baz": {"strict": {"tp": 0, "fp": 1, "fn": 1}, "core": {"tp": 0, "fp": 1, "fn": 3}, "all": {"tp": 0, "fp": 1, "fn": 3}}, "bugbot": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 2}}, "claude": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 3}}, "claude-code": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 3, "fp": 1, "fn": 0}, "all": {"tp": 3, "fp": 1, "fn": 0}}, "codeant-v2": {"strict": {"tp": 0, "fp": 1, "fn": 1}, "core": {"tp": 0, "fp": 1, "fn": 3}, "all": {"tp": 0, "fp": 1, "fn": 3}}, "coderabbit": {"strict": {"tp": 0, "fp": 2, "fn": 1}, "core": {"tp": 2, "fp": 2, "fn": 1}, "all": {"tp": 2, "fp": 2, "fn": 1}}, "cubic-v2": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 3, "fp": 1, "fn": 0}, "all": {"tp": 3, "fp": 1, "fn": 0}}, "devin": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 1, "fp": 1, "fn": 2}, "all": {"tp": 1, "fp": 1, "fn": 2}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 3}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 3}}, "kodus-v2": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 3}}, "macroscope": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 3}}, "qodo-extended-v2": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 1}, "all": {"tp": 2, "fp": 0, "fn": 1}}, "qodo-v2": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 2}}, "sourcery": {"strict": {"tp": 1, "fp": 2, "fn": 0}, "core": {"tp": 2, "fp": 2, "fn": 1}, "all": {"tp": 2, "fp": 2, "fn": 1}}, "greptile-v4-1": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 1}, "all": {"tp": 2, "fp": 1, "fn": 1}}, "gemini-v2": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 2}}, "gitlab": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 2}}, "copilot-v2": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 1}, "all": {"tp": 2, "fp": 0, "fn": 1}}}}, {"url": "https://github.com/getsentry/sentry/pull/80528", "language": "Python", "pr_size": "small", "domain": "scheduling", "change_type": "refactoring", "complexity": "moderate", "difficulty": "subtle", "risk": "medium", "context": "file", "concern": "correctness", "summary": "", "tool_metrics": {"augment": {"strict": {"tp": 1, "fp": 2, "fn": 0}, "core": {"tp": 1, "fp": 2, "fn": 1}, "all": {"tp": 1, "fp": 2, "fn": 1}}, "baz": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "bugbot": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "claude": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "claude-code": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "codeant-v2": {"strict": {"tp": 0, "fp": 1, "fn": 1}, "core": {"tp": 0, "fp": 1, "fn": 2}, "all": {"tp": 0, "fp": 1, "fn": 2}}, "coderabbit": {"strict": {"tp": 1, "fp": 3, "fn": 0}, "core": {"tp": 1, "fp": 3, "fn": 1}, "all": {"tp": 1, "fp": 3, "fn": 1}}, "cubic-v2": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "devin": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "graphite": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "kg": {"strict": {"tp": 0, "fp": 1, "fn": 1}, "core": {"tp": 0, "fp": 1, "fn": 2}, "all": {"tp": 0, "fp": 1, "fn": 2}}, "kodus-v2": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "macroscope": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "qodo-extended-v2": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "qodo-v2": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "sourcery": {"strict": {"tp": 1, "fp": 2, "fn": 0}, "core": {"tp": 1, "fp": 2, "fn": 1}, "all": {"tp": 1, "fp": 2, "fn": 1}}, "greptile-v4-1": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "gemini-v2": {"strict": {"tp": 1, "fp": 4, "fn": 0}, "core": {"tp": 2, "fp": 4, "fn": 0}, "all": {"tp": 2, "fp": 4, "fn": 0}}, "gitlab": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "copilot-v2": {"strict": {"tp": 1, "fp": 2, "fn": 0}, "core": {"tp": 1, "fp": 2, "fn": 1}, "all": {"tp": 1, "fp": 2, "fn": 1}}}}, {"url": "https://github.com/getsentry/sentry/pull/77754", "language": "Python", "pr_size": "medium", "domain": "serialization", "change_type": "feature", "complexity": "moderate", "difficulty": "moderate", "risk": "medium", "context": "file", "concern": "correctness", "summary": "", "tool_metrics": {"augment": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 3}}, "baz": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 3}}, "bugbot": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 3}}, "claude": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 2, "fp": 0, "fn": 2}}, "claude-code": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 3}}, "codeant-v2": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 4}}, "coderabbit": {"strict": {"tp": 1, "fp": 2, "fn": 1}, "core": {"tp": 1, "fp": 2, "fn": 1}, "all": {"tp": 3, "fp": 2, "fn": 1}}, "cubic-v2": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 2}}, "devin": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 3}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 4}}, "kg": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 3}}, "kodus-v2": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 3}}, "macroscope": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 3}}, "qodo-extended-v2": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 3}}, "qodo-v2": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 3}}, "sourcery": {"strict": {"tp": 1, "fp": 3, "fn": 1}, "core": {"tp": 1, "fp": 3, "fn": 1}, "all": {"tp": 2, "fp": 3, "fn": 2}}, "greptile-v4-1": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 3}}, "gemini-v2": {"strict": {"tp": 1, "fp": 3, "fn": 1}, "core": {"tp": 1, "fp": 3, "fn": 1}, "all": {"tp": 1, "fp": 3, "fn": 3}}, "gitlab": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 2, "fp": 1, "fn": 2}}, "copilot-v2": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 2, "fp": 1, "fn": 2}}}}, {"url": "https://github.com/getsentry/sentry/pull/95633", "language": "Python", "pr_size": "medium", "domain": "concurrency", "change_type": "feature", "complexity": "moderate", "difficulty": "moderate", "risk": "medium", "context": "file", "concern": "correctness", "summary": "", "tool_metrics": {"augment": {"strict": {"tp": 0, "fp": 6, "fn": 0}, "core": {"tp": 0, "fp": 6, "fn": 1}, "all": {"tp": 0, "fp": 6, "fn": 2}}, "baz": {"strict": {"tp": 0, "fp": 1, "fn": 0}, "core": {"tp": 0, "fp": 1, "fn": 1}, "all": {"tp": 0, "fp": 1, "fn": 2}}, "bugbot": {"strict": {"tp": 0, "fp": 2, "fn": 0}, "core": {"tp": 0, "fp": 2, "fn": 1}, "all": {"tp": 0, "fp": 2, "fn": 2}}, "claude": {"strict": {"tp": 0, "fp": 0, "fn": 0}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "claude-code": {"strict": {"tp": 0, "fp": 2, "fn": 0}, "core": {"tp": 0, "fp": 2, "fn": 1}, "all": {"tp": 0, "fp": 2, "fn": 2}}, "codeant-v2": {"strict": {"tp": 0, "fp": 1, "fn": 0}, "core": {"tp": 0, "fp": 1, "fn": 1}, "all": {"tp": 0, "fp": 1, "fn": 2}}, "coderabbit": {"strict": {"tp": 0, "fp": 7, "fn": 0}, "core": {"tp": 0, "fp": 7, "fn": 1}, "all": {"tp": 0, "fp": 7, "fn": 2}}, "cubic-v2": {"strict": {"tp": 0, "fp": 3, "fn": 0}, "core": {"tp": 0, "fp": 3, "fn": 1}, "all": {"tp": 0, "fp": 3, "fn": 2}}, "devin": {"strict": {"tp": 0, "fp": 2, "fn": 0}, "core": {"tp": 0, "fp": 2, "fn": 1}, "all": {"tp": 0, "fp": 2, "fn": 2}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 0}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "kg": {"strict": {"tp": 0, "fp": 2, "fn": 0}, "core": {"tp": 0, "fp": 2, "fn": 1}, "all": {"tp": 0, "fp": 2, "fn": 2}}, "kodus-v2": {"strict": {"tp": 0, "fp": 0, "fn": 0}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "macroscope": {"strict": {"tp": 0, "fp": 2, "fn": 0}, "core": {"tp": 0, "fp": 2, "fn": 1}, "all": {"tp": 0, "fp": 2, "fn": 2}}, "qodo-extended-v2": {"strict": {"tp": 0, "fp": 6, "fn": 0}, "core": {"tp": 0, "fp": 6, "fn": 1}, "all": {"tp": 0, "fp": 6, "fn": 2}}, "qodo-v2": {"strict": {"tp": 0, "fp": 4, "fn": 0}, "core": {"tp": 0, "fp": 4, "fn": 1}, "all": {"tp": 0, "fp": 4, "fn": 2}}, "sourcery": {"strict": {"tp": 0, "fp": 3, "fn": 0}, "core": {"tp": 1, "fp": 3, "fn": 0}, "all": {"tp": 1, "fp": 3, "fn": 1}}, "greptile-v4-1": {"strict": {"tp": 0, "fp": 3, "fn": 0}, "core": {"tp": 0, "fp": 3, "fn": 1}, "all": {"tp": 0, "fp": 3, "fn": 2}}, "gemini-v2": {"strict": {"tp": 0, "fp": 7, "fn": 0}, "core": {"tp": 0, "fp": 7, "fn": 1}, "all": {"tp": 0, "fp": 7, "fn": 2}}, "gitlab": {"strict": {"tp": 0, "fp": 5, "fn": 0}, "core": {"tp": 1, "fp": 5, "fn": 0}, "all": {"tp": 1, "fp": 5, "fn": 1}}, "copilot-v2": {"strict": {"tp": 0, "fp": 3, "fn": 0}, "core": {"tp": 0, "fp": 3, "fn": 1}, "all": {"tp": 0, "fp": 3, "fn": 2}}}}, {"url": "https://github.com/ai-code-review-evaluation/sentry-greptile/pull/2", "language": "Python", "pr_size": "medium", "domain": "data_processing", "change_type": "performance", "complexity": "moderate", "difficulty": "subtle", "risk": "high", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"augment": {"strict": {"tp": 3, "fp": 2, "fn": 1}, "core": {"tp": 3, "fp": 2, "fn": 1}, "all": {"tp": 3, "fp": 2, "fn": 1}}, "baz": {"strict": {"tp": 3, "fp": 1, "fn": 1}, "core": {"tp": 3, "fp": 1, "fn": 1}, "all": {"tp": 3, "fp": 1, "fn": 1}}, "bugbot": {"strict": {"tp": 4, "fp": 0, "fn": 0}, "core": {"tp": 4, "fp": 0, "fn": 0}, "all": {"tp": 4, "fp": 0, "fn": 0}}, "claude": {"strict": {"tp": 1, "fp": 6, "fn": 3}, "core": {"tp": 1, "fp": 6, "fn": 3}, "all": {"tp": 1, "fp": 6, "fn": 3}}, "claude-code": {"strict": {"tp": 3, "fp": 3, "fn": 1}, "core": {"tp": 3, "fp": 3, "fn": 1}, "all": {"tp": 3, "fp": 3, "fn": 1}}, "codeant-v2": {"strict": {"tp": 3, "fp": 1, "fn": 1}, "core": {"tp": 3, "fp": 1, "fn": 1}, "all": {"tp": 3, "fp": 1, "fn": 1}}, "coderabbit": {"strict": {"tp": 0, "fp": 0, "fn": 4}, "core": {"tp": 0, "fp": 0, "fn": 4}, "all": {"tp": 0, "fp": 0, "fn": 4}}, "cubic-v2": {"strict": {"tp": 3, "fp": 0, "fn": 1}, "core": {"tp": 3, "fp": 0, "fn": 1}, "all": {"tp": 3, "fp": 0, "fn": 1}}, "devin": {"strict": {"tp": 0, "fp": 0, "fn": 4}, "core": {"tp": 0, "fp": 0, "fn": 4}, "all": {"tp": 0, "fp": 0, "fn": 4}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 4}, "core": {"tp": 0, "fp": 0, "fn": 4}, "all": {"tp": 0, "fp": 0, "fn": 4}}, "kg": {"strict": {"tp": 1, "fp": 2, "fn": 3}, "core": {"tp": 1, "fp": 2, "fn": 3}, "all": {"tp": 1, "fp": 2, "fn": 3}}, "kodus-v2": {"strict": {"tp": 2, "fp": 3, "fn": 2}, "core": {"tp": 2, "fp": 3, "fn": 2}, "all": {"tp": 2, "fp": 3, "fn": 2}}, "macroscope": {"strict": {"tp": 3, "fp": 0, "fn": 1}, "core": {"tp": 3, "fp": 0, "fn": 1}, "all": {"tp": 3, "fp": 0, "fn": 1}}, "qodo-extended-v2": {"strict": {"tp": 2, "fp": 4, "fn": 2}, "core": {"tp": 2, "fp": 4, "fn": 2}, "all": {"tp": 2, "fp": 4, "fn": 2}}, "qodo-v2": {"strict": {"tp": 3, "fp": 1, "fn": 1}, "core": {"tp": 3, "fp": 1, "fn": 1}, "all": {"tp": 3, "fp": 1, "fn": 1}}, "sourcery": {"strict": {"tp": 3, "fp": 5, "fn": 1}, "core": {"tp": 3, "fp": 5, "fn": 1}, "all": {"tp": 3, "fp": 5, "fn": 1}}, "greptile-v4-1": {"strict": {"tp": 2, "fp": 3, "fn": 2}, "core": {"tp": 2, "fp": 3, "fn": 2}, "all": {"tp": 2, "fp": 3, "fn": 2}}, "gemini-v2": {"strict": {"tp": 3, "fp": 4, "fn": 1}, "core": {"tp": 3, "fp": 4, "fn": 1}, "all": {"tp": 3, "fp": 4, "fn": 1}}, "gitlab": {"strict": {"tp": 4, "fp": 1, "fn": 0}, "core": {"tp": 4, "fp": 1, "fn": 0}, "all": {"tp": 4, "fp": 1, "fn": 0}}, "copilot-v2": {"strict": {"tp": 3, "fp": 10, "fn": 1}, "core": {"tp": 3, "fp": 10, "fn": 1}, "all": {"tp": 3, "fp": 10, "fn": 1}}}}, {"url": "https://github.com/ai-code-review-evaluation/sentry-greptile/pull/3", "language": "Python", "pr_size": "medium", "domain": "caching", "change_type": "feature", "complexity": "complex", "difficulty": "subtle", "risk": "medium", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"augment": {"strict": {"tp": 2, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 1}, "all": {"tp": 2, "fp": 1, "fn": 1}}, "baz": {"strict": {"tp": 0, "fp": 3, "fn": 3}, "core": {"tp": 0, "fp": 3, "fn": 3}, "all": {"tp": 0, "fp": 3, "fn": 3}}, "bugbot": {"strict": {"tp": 0, "fp": 3, "fn": 3}, "core": {"tp": 0, "fp": 3, "fn": 3}, "all": {"tp": 0, "fp": 3, "fn": 3}}, "claude": {"strict": {"tp": 1, "fp": 0, "fn": 2}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 2}}, "claude-code": {"strict": {"tp": 1, "fp": 2, "fn": 2}, "core": {"tp": 1, "fp": 2, "fn": 2}, "all": {"tp": 1, "fp": 2, "fn": 2}}, "codeant-v2": {"strict": {"tp": 3, "fp": 2, "fn": 0}, "core": {"tp": 3, "fp": 2, "fn": 0}, "all": {"tp": 3, "fp": 2, "fn": 0}}, "coderabbit": {"strict": {"tp": 2, "fp": 5, "fn": 1}, "core": {"tp": 2, "fp": 5, "fn": 1}, "all": {"tp": 2, "fp": 5, "fn": 1}}, "cubic-v2": {"strict": {"tp": 2, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 1}, "all": {"tp": 2, "fp": 1, "fn": 1}}, "devin": {"strict": {"tp": 2, "fp": 0, "fn": 1}, "core": {"tp": 2, "fp": 0, "fn": 1}, "all": {"tp": 2, "fp": 0, "fn": 1}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 3}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 3}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 3}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 3}}, "kodus-v2": {"strict": {"tp": 2, "fp": 2, "fn": 1}, "core": {"tp": 2, "fp": 2, "fn": 1}, "all": {"tp": 2, "fp": 2, "fn": 1}}, "macroscope": {"strict": {"tp": 2, "fp": 0, "fn": 1}, "core": {"tp": 2, "fp": 0, "fn": 1}, "all": {"tp": 2, "fp": 0, "fn": 1}}, "qodo-extended-v2": {"strict": {"tp": 3, "fp": 0, "fn": 0}, "core": {"tp": 3, "fp": 0, "fn": 0}, "all": {"tp": 3, "fp": 0, "fn": 0}}, "qodo-v2": {"strict": {"tp": 2, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 1}, "all": {"tp": 2, "fp": 1, "fn": 1}}, "sourcery": {"strict": {"tp": 2, "fp": 5, "fn": 1}, "core": {"tp": 2, "fp": 5, "fn": 1}, "all": {"tp": 2, "fp": 5, "fn": 1}}, "greptile-v4-1": {"strict": {"tp": 2, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 1}, "all": {"tp": 2, "fp": 1, "fn": 1}}, "gemini-v2": {"strict": {"tp": 2, "fp": 2, "fn": 1}, "core": {"tp": 2, "fp": 2, "fn": 1}, "all": {"tp": 2, "fp": 2, "fn": 1}}, "gitlab": {"strict": {"tp": 2, "fp": 3, "fn": 1}, "core": {"tp": 2, "fp": 3, "fn": 1}, "all": {"tp": 2, "fp": 3, "fn": 1}}, "copilot-v2": {"strict": {"tp": 1, "fp": 7, "fn": 2}, "core": {"tp": 1, "fp": 7, "fn": 2}, "all": {"tp": 1, "fp": 7, "fn": 2}}}}, {"url": "https://github.com/grafana/grafana/pull/103633", "language": "Go", "pr_size": "small", "domain": "caching", "change_type": "performance", "complexity": "moderate", "difficulty": "subtle", "risk": "high", "context": "cross_file", "concern": "security", "summary": "", "tool_metrics": {"augment": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 2, "fp": 0, "fn": 1}, "all": {"tp": 2, "fp": 0, "fn": 1}}, "baz": {"strict": {"tp": 1, "fp": 2, "fn": 1}, "core": {"tp": 1, "fp": 2, "fn": 2}, "all": {"tp": 1, "fp": 2, "fn": 2}}, "bugbot": {"strict": {"tp": 0, "fp": 1, "fn": 2}, "core": {"tp": 0, "fp": 1, "fn": 3}, "all": {"tp": 0, "fp": 1, "fn": 3}}, "claude": {"strict": {"tp": 0, "fp": 5, "fn": 2}, "core": {"tp": 0, "fp": 5, "fn": 3}, "all": {"tp": 0, "fp": 5, "fn": 3}}, "claude-code": {"strict": {"tp": 0, "fp": 2, "fn": 2}, "core": {"tp": 0, "fp": 2, "fn": 3}, "all": {"tp": 0, "fp": 2, "fn": 3}}, "codeant-v2": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 2}}, "coderabbit": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 2}}, "cubic-v2": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 1}, "all": {"tp": 2, "fp": 1, "fn": 1}}, "devin": {"strict": {"tp": 1, "fp": 2, "fn": 1}, "core": {"tp": 1, "fp": 2, "fn": 2}, "all": {"tp": 1, "fp": 2, "fn": 2}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 3}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 3}}, "kodus-v2": {"strict": {"tp": 0, "fp": 1, "fn": 2}, "core": {"tp": 0, "fp": 1, "fn": 3}, "all": {"tp": 0, "fp": 1, "fn": 3}}, "macroscope": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 2}}, "qodo-extended-v2": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 2, "fp": 0, "fn": 1}, "all": {"tp": 2, "fp": 0, "fn": 1}}, "qodo-v2": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 1}, "all": {"tp": 2, "fp": 1, "fn": 1}}, "sourcery": {"strict": {"tp": 0, "fp": 2, "fn": 2}, "core": {"tp": 0, "fp": 2, "fn": 3}, "all": {"tp": 0, "fp": 2, "fn": 3}}, "greptile-v4-1": {"strict": {"tp": 1, "fp": 2, "fn": 1}, "core": {"tp": 1, "fp": 2, "fn": 2}, "all": {"tp": 1, "fp": 2, "fn": 2}}, "gemini-v2": {"strict": {"tp": 0, "fp": 3, "fn": 2}, "core": {"tp": 0, "fp": 3, "fn": 3}, "all": {"tp": 0, "fp": 3, "fn": 3}}, "gitlab": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 2}, "all": {"tp": 1, "fp": 1, "fn": 2}}, "copilot-v2": {"strict": {"tp": 0, "fp": 1, "fn": 2}, "core": {"tp": 1, "fp": 1, "fn": 2}, "all": {"tp": 1, "fp": 1, "fn": 2}}}}, {"url": "https://github.com/getsentry/sentry/pull/67876", "language": "Python", "pr_size": "small", "domain": "authentication", "change_type": "security_patch", "complexity": "moderate", "difficulty": "subtle", "risk": "high", "context": "cross_file", "concern": "security", "summary": "", "tool_metrics": {"augment": {"strict": {"tp": 1, "fp": 2, "fn": 2}, "core": {"tp": 1, "fp": 2, "fn": 2}, "all": {"tp": 1, "fp": 2, "fn": 2}}, "baz": {"strict": {"tp": 2, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 1}, "all": {"tp": 2, "fp": 1, "fn": 1}}, "bugbot": {"strict": {"tp": 1, "fp": 2, "fn": 2}, "core": {"tp": 1, "fp": 2, "fn": 2}, "all": {"tp": 1, "fp": 2, "fn": 2}}, "claude": {"strict": {"tp": 1, "fp": 1, "fn": 2}, "core": {"tp": 1, "fp": 1, "fn": 2}, "all": {"tp": 1, "fp": 1, "fn": 2}}, "claude-code": {"strict": {"tp": 1, "fp": 2, "fn": 2}, "core": {"tp": 1, "fp": 2, "fn": 2}, "all": {"tp": 1, "fp": 2, "fn": 2}}, "codeant-v2": {"strict": {"tp": 0, "fp": 1, "fn": 3}, "core": {"tp": 0, "fp": 1, "fn": 3}, "all": {"tp": 0, "fp": 1, "fn": 3}}, "coderabbit": {"strict": {"tp": 1, "fp": 3, "fn": 2}, "core": {"tp": 1, "fp": 3, "fn": 2}, "all": {"tp": 1, "fp": 3, "fn": 2}}, "cubic-v2": {"strict": {"tp": 3, "fp": 2, "fn": 0}, "core": {"tp": 3, "fp": 2, "fn": 0}, "all": {"tp": 3, "fp": 2, "fn": 0}}, "devin": {"strict": {"tp": 2, "fp": 0, "fn": 1}, "core": {"tp": 2, "fp": 0, "fn": 1}, "all": {"tp": 2, "fp": 0, "fn": 1}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 3}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 3}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 3}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 3}}, "kodus-v2": {"strict": {"tp": 0, "fp": 0, "fn": 3}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 3}}, "macroscope": {"strict": {"tp": 1, "fp": 1, "fn": 2}, "core": {"tp": 1, "fp": 1, "fn": 2}, "all": {"tp": 1, "fp": 1, "fn": 2}}, "qodo-extended-v2": {"strict": {"tp": 2, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 1}, "all": {"tp": 2, "fp": 1, "fn": 1}}, "qodo-v2": {"strict": {"tp": 3, "fp": 5, "fn": 0}, "core": {"tp": 3, "fp": 5, "fn": 0}, "all": {"tp": 3, "fp": 5, "fn": 0}}, "sourcery": {"strict": {"tp": 1, "fp": 8, "fn": 2}, "core": {"tp": 1, "fp": 8, "fn": 2}, "all": {"tp": 1, "fp": 8, "fn": 2}}, "greptile-v4-1": {"strict": {"tp": 1, "fp": 1, "fn": 2}, "core": {"tp": 1, "fp": 1, "fn": 2}, "all": {"tp": 1, "fp": 1, "fn": 2}}, "gemini-v2": {"strict": {"tp": 1, "fp": 1, "fn": 2}, "core": {"tp": 1, "fp": 1, "fn": 2}, "all": {"tp": 1, "fp": 1, "fn": 2}}, "gitlab": {"strict": {"tp": 1, "fp": 1, "fn": 2}, "core": {"tp": 1, "fp": 1, "fn": 2}, "all": {"tp": 1, "fp": 1, "fn": 2}}, "copilot-v2": {"strict": {"tp": 2, "fp": 13, "fn": 1}, "core": {"tp": 2, "fp": 13, "fn": 1}, "all": {"tp": 2, "fp": 13, "fn": 1}}}}, {"url": "https://github.com/keycloak/keycloak/pull/32918", "language": "Java", "pr_size": "small", "domain": "caching", "change_type": "feature", "complexity": "moderate", "difficulty": "subtle", "risk": "high", "context": "file", "concern": "correctness", "summary": "", "tool_metrics": {"augment": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "baz": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "bugbot": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "claude": {"strict": {"tp": 0, "fp": 1, "fn": 2}, "core": {"tp": 0, "fp": 1, "fn": 2}, "all": {"tp": 0, "fp": 1, "fn": 2}}, "claude-code": {"strict": {"tp": 1, "fp": 2, "fn": 1}, "core": {"tp": 1, "fp": 2, "fn": 1}, "all": {"tp": 1, "fp": 2, "fn": 1}}, "codeant-v2": {"strict": {"tp": 1, "fp": 2, "fn": 1}, "core": {"tp": 1, "fp": 2, "fn": 1}, "all": {"tp": 1, "fp": 2, "fn": 1}}, "coderabbit": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "cubic-v2": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "devin": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "graphite": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "kg": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "kodus-v2": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "macroscope": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "qodo-extended-v2": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "qodo-v2": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "sourcery": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "greptile-v4-1": {"strict": {"tp": 0, "fp": 3, "fn": 2}, "core": {"tp": 0, "fp": 3, "fn": 2}, "all": {"tp": 0, "fp": 3, "fn": 2}}, "gemini-v2": {"strict": {"tp": 0, "fp": 2, "fn": 2}, "core": {"tp": 0, "fp": 2, "fn": 2}, "all": {"tp": 0, "fp": 2, "fn": 2}}, "gitlab": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "copilot-v2": {"strict": {"tp": 1, "fp": 3, "fn": 1}, "core": {"tp": 1, "fp": 3, "fn": 1}, "all": {"tp": 1, "fp": 3, "fn": 1}}}}, {"url": "https://github.com/grafana/grafana/pull/94942", "language": "Go", "pr_size": "small", "domain": "data_processing", "change_type": "feature", "complexity": "moderate", "difficulty": "moderate", "risk": "high", "context": "file", "concern": "correctness", "summary": "", "tool_metrics": {"augment": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "baz": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "bugbot": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "claude": {"strict": {"tp": 1, "fp": 2, "fn": 1}, "core": {"tp": 1, "fp": 2, "fn": 1}, "all": {"tp": 1, "fp": 2, "fn": 1}}, "claude-code": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "codeant-v2": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "coderabbit": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "cubic-v2": {"strict": {"tp": 1, "fp": 2, "fn": 1}, "core": {"tp": 1, "fp": 2, "fn": 1}, "all": {"tp": 1, "fp": 2, "fn": 1}}, "devin": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "kg": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "kodus-v2": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "macroscope": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "qodo-extended-v2": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "qodo-v2": {"strict": {"tp": 2, "fp": 2, "fn": 0}, "core": {"tp": 2, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "sourcery": {"strict": {"tp": 2, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "greptile-v4-1": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "gemini-v2": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "gitlab": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "copilot-v2": {"strict": {"tp": 2, "fp": 2, "fn": 0}, "core": {"tp": 2, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}}}, {"url": "https://github.com/grafana/grafana/pull/90939", "language": "Go", "pr_size": "small", "domain": "caching", "change_type": "performance", "complexity": "moderate", "difficulty": "subtle", "risk": "high", "context": "file", "concern": "correctness", "summary": "", "tool_metrics": {"augment": {"strict": {"tp": 2, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "baz": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "bugbot": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "claude": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "claude-code": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "codeant-v2": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "coderabbit": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "cubic-v2": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "devin": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "kg": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "kodus-v2": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "macroscope": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "qodo-extended-v2": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "qodo-v2": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "sourcery": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "greptile-v4-1": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "gemini-v2": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "gitlab": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "copilot-v2": {"strict": {"tp": 1, "fp": 2, "fn": 1}, "core": {"tp": 1, "fp": 2, "fn": 1}, "all": {"tp": 1, "fp": 2, "fn": 1}}}}, {"url": "https://github.com/grafana/grafana/pull/80329", "language": "Go", "pr_size": "small", "domain": "logging", "change_type": "performance", "complexity": "simple", "difficulty": "obvious", "risk": "low", "context": "local", "concern": "maintainability", "summary": "", "tool_metrics": {"augment": {"strict": {"tp": 0, "fp": 2, "fn": 0}, "core": {"tp": 2, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "baz": {"strict": {"tp": 0, "fp": 0, "fn": 0}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "bugbot": {"strict": {"tp": 0, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "claude": {"strict": {"tp": 0, "fp": 4, "fn": 0}, "core": {"tp": 2, "fp": 4, "fn": 0}, "all": {"tp": 2, "fp": 4, "fn": 0}}, "claude-code": {"strict": {"tp": 0, "fp": 5, "fn": 0}, "core": {"tp": 2, "fp": 5, "fn": 0}, "all": {"tp": 2, "fp": 5, "fn": 0}}, "codeant-v2": {"strict": {"tp": 0, "fp": 0, "fn": 0}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "coderabbit": {"strict": {"tp": 0, "fp": 3, "fn": 0}, "core": {"tp": 1, "fp": 3, "fn": 1}, "all": {"tp": 1, "fp": 3, "fn": 1}}, "cubic-v2": {"strict": {"tp": 0, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "devin": {"strict": {"tp": 0, "fp": 2, "fn": 0}, "core": {"tp": 2, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 0}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 0}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "kodus-v2": {"strict": {"tp": 0, "fp": 2, "fn": 0}, "core": {"tp": 2, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "macroscope": {"strict": {"tp": 0, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "qodo-extended-v2": {"strict": {"tp": 0, "fp": 3, "fn": 0}, "core": {"tp": 2, "fp": 3, "fn": 0}, "all": {"tp": 2, "fp": 3, "fn": 0}}, "qodo-v2": {"strict": {"tp": 0, "fp": 3, "fn": 0}, "core": {"tp": 2, "fp": 3, "fn": 0}, "all": {"tp": 2, "fp": 3, "fn": 0}}, "sourcery": {"strict": {"tp": 0, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "greptile-v4-1": {"strict": {"tp": 0, "fp": 2, "fn": 0}, "core": {"tp": 2, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "gemini-v2": {"strict": {"tp": 0, "fp": 2, "fn": 0}, "core": {"tp": 2, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "gitlab": {"strict": {"tp": 0, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "copilot-v2": {"strict": {"tp": 0, "fp": 2, "fn": 0}, "core": {"tp": 2, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}}}, {"url": "https://github.com/grafana/grafana/pull/90045", "language": "Go", "pr_size": "medium", "domain": "logging", "change_type": "feature", "complexity": "moderate", "difficulty": "moderate", "risk": "medium", "context": "file", "concern": "correctness", "summary": "", "tool_metrics": {"augment": {"strict": {"tp": 3, "fp": 5, "fn": 0}, "core": {"tp": 3, "fp": 5, "fn": 0}, "all": {"tp": 3, "fp": 5, "fn": 0}}, "baz": {"strict": {"tp": 2, "fp": 2, "fn": 1}, "core": {"tp": 2, "fp": 2, "fn": 1}, "all": {"tp": 2, "fp": 2, "fn": 1}}, "bugbot": {"strict": {"tp": 2, "fp": 4, "fn": 1}, "core": {"tp": 2, "fp": 4, "fn": 1}, "all": {"tp": 2, "fp": 4, "fn": 1}}, "claude": {"strict": {"tp": 2, "fp": 5, "fn": 1}, "core": {"tp": 2, "fp": 5, "fn": 1}, "all": {"tp": 2, "fp": 5, "fn": 1}}, "claude-code": {"strict": {"tp": 3, "fp": 4, "fn": 0}, "core": {"tp": 3, "fp": 4, "fn": 0}, "all": {"tp": 3, "fp": 4, "fn": 0}}, "codeant-v2": {"strict": {"tp": 2, "fp": 3, "fn": 1}, "core": {"tp": 2, "fp": 3, "fn": 1}, "all": {"tp": 2, "fp": 3, "fn": 1}}, "coderabbit": {"strict": {"tp": 3, "fp": 2, "fn": 0}, "core": {"tp": 3, "fp": 2, "fn": 0}, "all": {"tp": 3, "fp": 2, "fn": 0}}, "cubic-v2": {"strict": {"tp": 3, "fp": 2, "fn": 0}, "core": {"tp": 3, "fp": 2, "fn": 0}, "all": {"tp": 3, "fp": 2, "fn": 0}}, "devin": {"strict": {"tp": 3, "fp": 2, "fn": 0}, "core": {"tp": 3, "fp": 2, "fn": 0}, "all": {"tp": 3, "fp": 2, "fn": 0}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 3}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 3}}, "kg": {"strict": {"tp": 3, "fp": 1, "fn": 0}, "core": {"tp": 3, "fp": 1, "fn": 0}, "all": {"tp": 3, "fp": 1, "fn": 0}}, "kodus-v2": {"strict": {"tp": 1, "fp": 0, "fn": 2}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 2}}, "macroscope": {"strict": {"tp": 2, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 1}, "all": {"tp": 2, "fp": 1, "fn": 1}}, "qodo-extended-v2": {"strict": {"tp": 3, "fp": 6, "fn": 0}, "core": {"tp": 3, "fp": 6, "fn": 0}, "all": {"tp": 3, "fp": 6, "fn": 0}}, "qodo-v2": {"strict": {"tp": 1, "fp": 7, "fn": 2}, "core": {"tp": 1, "fp": 7, "fn": 2}, "all": {"tp": 1, "fp": 7, "fn": 2}}, "sourcery": {"strict": {"tp": 2, "fp": 2, "fn": 1}, "core": {"tp": 2, "fp": 2, "fn": 1}, "all": {"tp": 2, "fp": 2, "fn": 1}}, "greptile-v4-1": {"strict": {"tp": 3, "fp": 3, "fn": 0}, "core": {"tp": 3, "fp": 3, "fn": 0}, "all": {"tp": 3, "fp": 3, "fn": 0}}, "gemini-v2": {"strict": {"tp": 3, "fp": 5, "fn": 0}, "core": {"tp": 3, "fp": 5, "fn": 0}, "all": {"tp": 3, "fp": 5, "fn": 0}}, "gitlab": {"strict": {"tp": 3, "fp": 2, "fn": 0}, "core": {"tp": 3, "fp": 2, "fn": 0}, "all": {"tp": 3, "fp": 2, "fn": 0}}, "copilot-v2": {"strict": {"tp": 3, "fp": 6, "fn": 0}, "core": {"tp": 3, "fp": 6, "fn": 0}, "all": {"tp": 3, "fp": 6, "fn": 0}}}}, {"url": "https://github.com/grafana/grafana/pull/106778", "language": "Go", "pr_size": "medium", "domain": "UI", "change_type": "feature", "complexity": "moderate", "difficulty": "subtle", "risk": "medium", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"augment": {"strict": {"tp": 1, "fp": 2, "fn": 1}, "core": {"tp": 1, "fp": 2, "fn": 1}, "all": {"tp": 1, "fp": 2, "fn": 1}}, "baz": {"strict": {"tp": 0, "fp": 2, "fn": 2}, "core": {"tp": 0, "fp": 2, "fn": 2}, "all": {"tp": 0, "fp": 2, "fn": 2}}, "bugbot": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "claude": {"strict": {"tp": 0, "fp": 12, "fn": 2}, "core": {"tp": 0, "fp": 12, "fn": 2}, "all": {"tp": 0, "fp": 12, "fn": 2}}, "claude-code": {"strict": {"tp": 2, "fp": 2, "fn": 0}, "core": {"tp": 2, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "codeant-v2": {"strict": {"tp": 1, "fp": 2, "fn": 1}, "core": {"tp": 1, "fp": 2, "fn": 1}, "all": {"tp": 1, "fp": 2, "fn": 1}}, "coderabbit": {"strict": {"tp": 1, "fp": 5, "fn": 1}, "core": {"tp": 1, "fp": 5, "fn": 1}, "all": {"tp": 1, "fp": 5, "fn": 1}}, "cubic-v2": {"strict": {"tp": 2, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "devin": {"strict": {"tp": 2, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "kg": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "kodus-v2": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "macroscope": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "qodo-extended-v2": {"strict": {"tp": 2, "fp": 2, "fn": 0}, "core": {"tp": 2, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "qodo-v2": {"strict": {"tp": 1, "fp": 6, "fn": 1}, "core": {"tp": 1, "fp": 6, "fn": 1}, "all": {"tp": 1, "fp": 6, "fn": 1}}, "sourcery": {"strict": {"tp": 0, "fp": 2, "fn": 2}, "core": {"tp": 0, "fp": 2, "fn": 2}, "all": {"tp": 0, "fp": 2, "fn": 2}}, "greptile-v4-1": {"strict": {"tp": 1, "fp": 3, "fn": 1}, "core": {"tp": 1, "fp": 3, "fn": 1}, "all": {"tp": 1, "fp": 3, "fn": 1}}, "gemini-v2": {"strict": {"tp": 0, "fp": 3, "fn": 2}, "core": {"tp": 0, "fp": 3, "fn": 2}, "all": {"tp": 0, "fp": 3, "fn": 2}}, "gitlab": {"strict": {"tp": 2, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "copilot-v2": {"strict": {"tp": 2, "fp": 2, "fn": 0}, "core": {"tp": 2, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}}}, {"url": "https://github.com/grafana/grafana/pull/107534", "language": "Go", "pr_size": "small", "domain": "testing", "change_type": "feature", "complexity": "moderate", "difficulty": "subtle", "risk": "low", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"augment": {"strict": {"tp": 0, "fp": 0, "fn": 0}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 0, "fp": 0, "fn": 1}}, "baz": {"strict": {"tp": 0, "fp": 2, "fn": 0}, "core": {"tp": 0, "fp": 2, "fn": 1}, "all": {"tp": 0, "fp": 2, "fn": 1}}, "bugbot": {"strict": {"tp": 0, "fp": 1, "fn": 0}, "core": {"tp": 0, "fp": 1, "fn": 1}, "all": {"tp": 0, "fp": 1, "fn": 1}}, "claude": {"strict": {"tp": 0, "fp": 0, "fn": 0}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 0, "fp": 0, "fn": 1}}, "claude-code": {"strict": {"tp": 0, "fp": 2, "fn": 0}, "core": {"tp": 0, "fp": 2, "fn": 1}, "all": {"tp": 0, "fp": 2, "fn": 1}}, "codeant-v2": {"strict": {"tp": 0, "fp": 0, "fn": 0}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 0, "fp": 0, "fn": 1}}, "coderabbit": {"strict": {"tp": 0, "fp": 1, "fn": 0}, "core": {"tp": 0, "fp": 1, "fn": 1}, "all": {"tp": 0, "fp": 1, "fn": 1}}, "cubic-v2": {"strict": {"tp": 0, "fp": 2, "fn": 0}, "core": {"tp": 1, "fp": 2, "fn": 0}, "all": {"tp": 1, "fp": 2, "fn": 0}}, "devin": {"strict": {"tp": 0, "fp": 1, "fn": 0}, "core": {"tp": 0, "fp": 1, "fn": 1}, "all": {"tp": 0, "fp": 1, "fn": 1}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 0}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 0, "fp": 0, "fn": 1}}, "kg": {"strict": {"tp": 0, "fp": 2, "fn": 0}, "core": {"tp": 0, "fp": 2, "fn": 1}, "all": {"tp": 0, "fp": 2, "fn": 1}}, "kodus-v2": {"strict": {"tp": 0, "fp": 0, "fn": 0}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 0, "fp": 0, "fn": 1}}, "macroscope": {"strict": {"tp": 0, "fp": 0, "fn": 0}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 0, "fp": 0, "fn": 1}}, "qodo-extended-v2": {"strict": {"tp": 0, "fp": 1, "fn": 0}, "core": {"tp": 0, "fp": 1, "fn": 1}, "all": {"tp": 0, "fp": 1, "fn": 1}}, "qodo-v2": {"strict": {"tp": 0, "fp": 1, "fn": 0}, "core": {"tp": 0, "fp": 1, "fn": 1}, "all": {"tp": 0, "fp": 1, "fn": 1}}, "sourcery": {"strict": {"tp": 0, "fp": 3, "fn": 0}, "core": {"tp": 0, "fp": 3, "fn": 1}, "all": {"tp": 0, "fp": 3, "fn": 1}}, "greptile-v4-1": {"strict": {"tp": 0, "fp": 4, "fn": 0}, "core": {"tp": 0, "fp": 4, "fn": 1}, "all": {"tp": 0, "fp": 4, "fn": 1}}, "gemini-v2": {"strict": {"tp": 0, "fp": 3, "fn": 0}, "core": {"tp": 1, "fp": 3, "fn": 0}, "all": {"tp": 1, "fp": 3, "fn": 0}}, "gitlab": {"strict": {"tp": 0, "fp": 1, "fn": 0}, "core": {"tp": 0, "fp": 1, "fn": 1}, "all": {"tp": 0, "fp": 1, "fn": 1}}, "copilot-v2": {"strict": {"tp": 0, "fp": 3, "fn": 0}, "core": {"tp": 0, "fp": 3, "fn": 1}, "all": {"tp": 0, "fp": 3, "fn": 1}}}}, {"url": "https://github.com/grafana/grafana/pull/79265", "language": "Go", "pr_size": "large", "domain": "authentication", "change_type": "feature", "complexity": "moderate", "difficulty": "subtle", "risk": "high", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"augment": {"strict": {"tp": 1, "fp": 1, "fn": 3}, "core": {"tp": 1, "fp": 1, "fn": 4}, "all": {"tp": 1, "fp": 1, "fn": 4}}, "baz": {"strict": {"tp": 1, "fp": 0, "fn": 3}, "core": {"tp": 2, "fp": 0, "fn": 3}, "all": {"tp": 2, "fp": 0, "fn": 3}}, "bugbot": {"strict": {"tp": 1, "fp": 1, "fn": 3}, "core": {"tp": 1, "fp": 1, "fn": 4}, "all": {"tp": 1, "fp": 1, "fn": 4}}, "claude": {"strict": {"tp": 1, "fp": 2, "fn": 3}, "core": {"tp": 1, "fp": 2, "fn": 4}, "all": {"tp": 1, "fp": 2, "fn": 4}}, "claude-code": {"strict": {"tp": 1, "fp": 2, "fn": 3}, "core": {"tp": 1, "fp": 2, "fn": 4}, "all": {"tp": 1, "fp": 2, "fn": 4}}, "codeant-v2": {"strict": {"tp": 1, "fp": 1, "fn": 3}, "core": {"tp": 1, "fp": 1, "fn": 4}, "all": {"tp": 1, "fp": 1, "fn": 4}}, "coderabbit": {"strict": {"tp": 2, "fp": 4, "fn": 2}, "core": {"tp": 3, "fp": 4, "fn": 2}, "all": {"tp": 3, "fp": 4, "fn": 2}}, "cubic-v2": {"strict": {"tp": 3, "fp": 1, "fn": 1}, "core": {"tp": 4, "fp": 1, "fn": 1}, "all": {"tp": 4, "fp": 1, "fn": 1}}, "devin": {"strict": {"tp": 2, "fp": 0, "fn": 2}, "core": {"tp": 2, "fp": 0, "fn": 3}, "all": {"tp": 2, "fp": 0, "fn": 3}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 4}, "core": {"tp": 0, "fp": 0, "fn": 5}, "all": {"tp": 0, "fp": 0, "fn": 5}}, "kg": {"strict": {"tp": 1, "fp": 0, "fn": 3}, "core": {"tp": 1, "fp": 0, "fn": 4}, "all": {"tp": 1, "fp": 0, "fn": 4}}, "kodus-v2": {"strict": {"tp": 0, "fp": 1, "fn": 4}, "core": {"tp": 1, "fp": 1, "fn": 4}, "all": {"tp": 1, "fp": 1, "fn": 4}}, "macroscope": {"strict": {"tp": 1, "fp": 0, "fn": 3}, "core": {"tp": 1, "fp": 0, "fn": 4}, "all": {"tp": 1, "fp": 0, "fn": 4}}, "qodo-extended-v2": {"strict": {"tp": 2, "fp": 1, "fn": 2}, "core": {"tp": 2, "fp": 1, "fn": 3}, "all": {"tp": 2, "fp": 1, "fn": 3}}, "qodo-v2": {"strict": {"tp": 2, "fp": 2, "fn": 2}, "core": {"tp": 2, "fp": 2, "fn": 3}, "all": {"tp": 2, "fp": 2, "fn": 3}}, "sourcery": {"strict": {"tp": 1, "fp": 4, "fn": 3}, "core": {"tp": 2, "fp": 4, "fn": 3}, "all": {"tp": 2, "fp": 4, "fn": 3}}, "greptile-v4-1": {"strict": {"tp": 2, "fp": 2, "fn": 2}, "core": {"tp": 3, "fp": 2, "fn": 2}, "all": {"tp": 3, "fp": 2, "fn": 2}}, "gemini-v2": {"strict": {"tp": 0, "fp": 4, "fn": 4}, "core": {"tp": 0, "fp": 4, "fn": 5}, "all": {"tp": 0, "fp": 4, "fn": 5}}, "gitlab": {"strict": {"tp": 1, "fp": 1, "fn": 3}, "core": {"tp": 2, "fp": 1, "fn": 3}, "all": {"tp": 2, "fp": 1, "fn": 3}}, "copilot-v2": {"strict": {"tp": 1, "fp": 4, "fn": 3}, "core": {"tp": 1, "fp": 4, "fn": 4}, "all": {"tp": 1, "fp": 4, "fn": 4}}}}, {"url": "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/9", "language": "Ruby", "pr_size": "small", "domain": "configuration", "change_type": "feature", "complexity": "moderate", "difficulty": "subtle", "risk": "medium", "context": "file", "concern": "reliability", "summary": "", "tool_metrics": {"augment": {"strict": {"tp": 0, "fp": 2, "fn": 1}, "core": {"tp": 1, "fp": 2, "fn": 1}, "all": {"tp": 1, "fp": 2, "fn": 1}}, "baz": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "bugbot": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "claude": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "claude-code": {"strict": {"tp": 1, "fp": 5, "fn": 0}, "core": {"tp": 1, "fp": 5, "fn": 1}, "all": {"tp": 1, "fp": 5, "fn": 1}}, "codeant-v2": {"strict": {"tp": 0, "fp": 1, "fn": 1}, "core": {"tp": 0, "fp": 1, "fn": 2}, "all": {"tp": 0, "fp": 1, "fn": 2}}, "coderabbit": {"strict": {"tp": 0, "fp": 3, "fn": 1}, "core": {"tp": 0, "fp": 3, "fn": 2}, "all": {"tp": 0, "fp": 3, "fn": 2}}, "cubic-v2": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "devin": {"strict": {"tp": 0, "fp": 1, "fn": 1}, "core": {"tp": 0, "fp": 1, "fn": 2}, "all": {"tp": 0, "fp": 1, "fn": 2}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "kodus-v2": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "macroscope": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "qodo-extended-v2": {"strict": {"tp": 0, "fp": 1, "fn": 1}, "core": {"tp": 0, "fp": 1, "fn": 2}, "all": {"tp": 0, "fp": 1, "fn": 2}}, "qodo-v2": {"strict": {"tp": 0, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "sourcery": {"strict": {"tp": 0, "fp": 3, "fn": 1}, "core": {"tp": 0, "fp": 3, "fn": 2}, "all": {"tp": 0, "fp": 3, "fn": 2}}, "greptile-v4-1": {"strict": {"tp": 0, "fp": 2, "fn": 1}, "core": {"tp": 0, "fp": 2, "fn": 2}, "all": {"tp": 0, "fp": 2, "fn": 2}}, "gemini-v2": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "gitlab": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "copilot-v2": {"strict": {"tp": 0, "fp": 2, "fn": 1}, "core": {"tp": 0, "fp": 2, "fn": 2}, "all": {"tp": 0, "fp": 2, "fn": 2}}}}, {"url": "https://github.com/grafana/grafana/pull/76186", "language": "Go", "pr_size": "small", "domain": "logging", "change_type": "refactoring", "complexity": "moderate", "difficulty": "subtle", "risk": "high", "context": "cross_file", "concern": "reliability", "summary": "", "tool_metrics": {"augment": {"strict": {"tp": 2, "fp": 0, "fn": 1}, "core": {"tp": 2, "fp": 0, "fn": 1}, "all": {"tp": 2, "fp": 0, "fn": 1}}, "baz": {"strict": {"tp": 0, "fp": 0, "fn": 3}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 3}}, "bugbot": {"strict": {"tp": 1, "fp": 0, "fn": 2}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 2}}, "claude": {"strict": {"tp": 0, "fp": 0, "fn": 3}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 3}}, "claude-code": {"strict": {"tp": 1, "fp": 1, "fn": 2}, "core": {"tp": 1, "fp": 1, "fn": 2}, "all": {"tp": 1, "fp": 1, "fn": 2}}, "codeant-v2": {"strict": {"tp": 0, "fp": 1, "fn": 3}, "core": {"tp": 0, "fp": 1, "fn": 3}, "all": {"tp": 0, "fp": 1, "fn": 3}}, "coderabbit": {"strict": {"tp": 1, "fp": 3, "fn": 2}, "core": {"tp": 1, "fp": 3, "fn": 2}, "all": {"tp": 1, "fp": 3, "fn": 2}}, "cubic-v2": {"strict": {"tp": 3, "fp": 0, "fn": 0}, "core": {"tp": 3, "fp": 0, "fn": 0}, "all": {"tp": 3, "fp": 0, "fn": 0}}, "devin": {"strict": {"tp": 1, "fp": 0, "fn": 2}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 2}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 3}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 3}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 3}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 3}}, "kodus-v2": {"strict": {"tp": 2, "fp": 0, "fn": 1}, "core": {"tp": 2, "fp": 0, "fn": 1}, "all": {"tp": 2, "fp": 0, "fn": 1}}, "macroscope": {"strict": {"tp": 1, "fp": 0, "fn": 2}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 2}}, "qodo-extended-v2": {"strict": {"tp": 1, "fp": 0, "fn": 2}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 2}}, "qodo-v2": {"strict": {"tp": 1, "fp": 1, "fn": 2}, "core": {"tp": 1, "fp": 1, "fn": 2}, "all": {"tp": 1, "fp": 1, "fn": 2}}, "sourcery": {"strict": {"tp": 0, "fp": 3, "fn": 3}, "core": {"tp": 0, "fp": 3, "fn": 3}, "all": {"tp": 0, "fp": 3, "fn": 3}}, "greptile-v4-1": {"strict": {"tp": 1, "fp": 0, "fn": 2}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 2}}, "gemini-v2": {"strict": {"tp": 1, "fp": 2, "fn": 2}, "core": {"tp": 1, "fp": 2, "fn": 2}, "all": {"tp": 1, "fp": 2, "fn": 2}}, "gitlab": {"strict": {"tp": 3, "fp": 0, "fn": 0}, "core": {"tp": 3, "fp": 0, "fn": 0}, "all": {"tp": 3, "fp": 0, "fn": 0}}, "copilot-v2": {"strict": {"tp": 0, "fp": 2, "fn": 3}, "core": {"tp": 0, "fp": 2, "fn": 3}, "all": {"tp": 0, "fp": 2, "fn": 3}}}}, {"url": "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/10", "language": "Ruby", "pr_size": "large", "domain": "configuration", "change_type": "feature", "complexity": "moderate", "difficulty": "subtle", "risk": "high", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"augment": {"strict": {"tp": 3, "fp": 2, "fn": 4}, "core": {"tp": 3, "fp": 2, "fn": 4}, "all": {"tp": 3, "fp": 2, "fn": 4}}, "baz": {"strict": {"tp": 2, "fp": 0, "fn": 5}, "core": {"tp": 2, "fp": 0, "fn": 5}, "all": {"tp": 2, "fp": 0, "fn": 5}}, "bugbot": {"strict": {"tp": 5, "fp": 2, "fn": 2}, "core": {"tp": 5, "fp": 2, "fn": 2}, "all": {"tp": 5, "fp": 2, "fn": 2}}, "claude": {"strict": {"tp": 1, "fp": 2, "fn": 6}, "core": {"tp": 1, "fp": 2, "fn": 6}, "all": {"tp": 1, "fp": 2, "fn": 6}}, "claude-code": {"strict": {"tp": 1, "fp": 5, "fn": 6}, "core": {"tp": 1, "fp": 5, "fn": 6}, "all": {"tp": 1, "fp": 5, "fn": 6}}, "codeant-v2": {"strict": {"tp": 2, "fp": 7, "fn": 5}, "core": {"tp": 2, "fp": 7, "fn": 5}, "all": {"tp": 2, "fp": 7, "fn": 5}}, "coderabbit": {"strict": {"tp": 4, "fp": 10, "fn": 3}, "core": {"tp": 4, "fp": 10, "fn": 3}, "all": {"tp": 4, "fp": 10, "fn": 3}}, "cubic-v2": {"strict": {"tp": 3, "fp": 1, "fn": 4}, "core": {"tp": 3, "fp": 1, "fn": 4}, "all": {"tp": 3, "fp": 1, "fn": 4}}, "devin": {"strict": {"tp": 1, "fp": 1, "fn": 6}, "core": {"tp": 1, "fp": 1, "fn": 6}, "all": {"tp": 1, "fp": 1, "fn": 6}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 7}, "core": {"tp": 0, "fp": 0, "fn": 7}, "all": {"tp": 0, "fp": 0, "fn": 7}}, "kg": {"strict": {"tp": 1, "fp": 0, "fn": 6}, "core": {"tp": 1, "fp": 0, "fn": 6}, "all": {"tp": 1, "fp": 0, "fn": 6}}, "kodus-v2": {"strict": {"tp": 5, "fp": 3, "fn": 2}, "core": {"tp": 5, "fp": 3, "fn": 2}, "all": {"tp": 5, "fp": 3, "fn": 2}}, "macroscope": {"strict": {"tp": 3, "fp": 4, "fn": 4}, "core": {"tp": 3, "fp": 4, "fn": 4}, "all": {"tp": 3, "fp": 4, "fn": 4}}, "qodo-extended-v2": {"strict": {"tp": 5, "fp": 4, "fn": 2}, "core": {"tp": 5, "fp": 4, "fn": 2}, "all": {"tp": 5, "fp": 4, "fn": 2}}, "qodo-v2": {"strict": {"tp": 2, "fp": 5, "fn": 5}, "core": {"tp": 2, "fp": 5, "fn": 5}, "all": {"tp": 2, "fp": 5, "fn": 5}}, "sourcery": {"strict": {"tp": 4, "fp": 7, "fn": 3}, "core": {"tp": 4, "fp": 7, "fn": 3}, "all": {"tp": 4, "fp": 7, "fn": 3}}, "greptile-v4-1": {"strict": {"tp": 2, "fp": 2, "fn": 5}, "core": {"tp": 2, "fp": 2, "fn": 5}, "all": {"tp": 2, "fp": 2, "fn": 5}}, "gemini-v2": {"strict": {"tp": 4, "fp": 2, "fn": 3}, "core": {"tp": 4, "fp": 2, "fn": 3}, "all": {"tp": 4, "fp": 2, "fn": 3}}, "gitlab": {"strict": {"tp": 2, "fp": 3, "fn": 5}, "core": {"tp": 2, "fp": 3, "fn": 5}, "all": {"tp": 2, "fp": 3, "fn": 5}}, "copilot-v2": {"strict": {"tp": 6, "fp": 5, "fn": 1}, "core": {"tp": 6, "fp": 5, "fn": 1}, "all": {"tp": 6, "fp": 5, "fn": 1}}}}, {"url": "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/7", "language": "Ruby", "pr_size": "medium", "domain": "UI", "change_type": "bug_fix", "complexity": "simple", "difficulty": "subtle", "risk": "medium", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"augment": {"strict": {"tp": 3, "fp": 2, "fn": 0}, "core": {"tp": 3, "fp": 2, "fn": 0}, "all": {"tp": 5, "fp": 2, "fn": 0}}, "baz": {"strict": {"tp": 2, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 1}, "all": {"tp": 4, "fp": 1, "fn": 1}}, "bugbot": {"strict": {"tp": 3, "fp": 1, "fn": 0}, "core": {"tp": 3, "fp": 1, "fn": 0}, "all": {"tp": 5, "fp": 1, "fn": 0}}, "claude": {"strict": {"tp": 2, "fp": 2, "fn": 1}, "core": {"tp": 2, "fp": 2, "fn": 1}, "all": {"tp": 4, "fp": 2, "fn": 1}}, "claude-code": {"strict": {"tp": 2, "fp": 2, "fn": 1}, "core": {"tp": 2, "fp": 2, "fn": 1}, "all": {"tp": 4, "fp": 2, "fn": 1}}, "codeant-v2": {"strict": {"tp": 0, "fp": 9, "fn": 3}, "core": {"tp": 0, "fp": 9, "fn": 3}, "all": {"tp": 0, "fp": 9, "fn": 5}}, "coderabbit": {"strict": {"tp": 0, "fp": 2, "fn": 3}, "core": {"tp": 0, "fp": 2, "fn": 3}, "all": {"tp": 0, "fp": 2, "fn": 5}}, "cubic-v2": {"strict": {"tp": 2, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 1}, "all": {"tp": 3, "fp": 1, "fn": 2}}, "devin": {"strict": {"tp": 2, "fp": 2, "fn": 1}, "core": {"tp": 2, "fp": 2, "fn": 1}, "all": {"tp": 4, "fp": 2, "fn": 1}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 3}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 5}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 3}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 5}}, "kodus-v2": {"strict": {"tp": 1, "fp": 0, "fn": 2}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 4}}, "macroscope": {"strict": {"tp": 2, "fp": 0, "fn": 1}, "core": {"tp": 2, "fp": 0, "fn": 1}, "all": {"tp": 4, "fp": 0, "fn": 1}}, "qodo-extended-v2": {"strict": {"tp": 2, "fp": 3, "fn": 1}, "core": {"tp": 2, "fp": 3, "fn": 1}, "all": {"tp": 3, "fp": 3, "fn": 2}}, "qodo-v2": {"strict": {"tp": 1, "fp": 3, "fn": 2}, "core": {"tp": 1, "fp": 3, "fn": 2}, "all": {"tp": 2, "fp": 3, "fn": 3}}, "sourcery": {"strict": {"tp": 2, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 1}, "all": {"tp": 3, "fp": 1, "fn": 2}}, "greptile-v4-1": {"strict": {"tp": 1, "fp": 0, "fn": 2}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 2, "fp": 0, "fn": 3}}, "gemini-v2": {"strict": {"tp": 3, "fp": 1, "fn": 0}, "core": {"tp": 3, "fp": 1, "fn": 0}, "all": {"tp": 5, "fp": 1, "fn": 0}}, "gitlab": {"strict": {"tp": 2, "fp": 0, "fn": 1}, "core": {"tp": 2, "fp": 0, "fn": 1}, "all": {"tp": 3, "fp": 0, "fn": 2}}, "copilot-v2": {"strict": {"tp": 3, "fp": 1, "fn": 0}, "core": {"tp": 3, "fp": 1, "fn": 0}, "all": {"tp": 5, "fp": 1, "fn": 0}}}}, {"url": "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/8", "language": "Ruby", "pr_size": "medium", "domain": "API", "change_type": "bug_fix", "complexity": "moderate", "difficulty": "subtle", "risk": "medium", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"augment": {"strict": {"tp": 3, "fp": 2, "fn": 3}, "core": {"tp": 3, "fp": 2, "fn": 3}, "all": {"tp": 3, "fp": 2, "fn": 3}}, "baz": {"strict": {"tp": 2, "fp": 3, "fn": 4}, "core": {"tp": 2, "fp": 3, "fn": 4}, "all": {"tp": 2, "fp": 3, "fn": 4}}, "bugbot": {"strict": {"tp": 1, "fp": 1, "fn": 5}, "core": {"tp": 1, "fp": 1, "fn": 5}, "all": {"tp": 1, "fp": 1, "fn": 5}}, "claude": {"strict": {"tp": 4, "fp": 3, "fn": 2}, "core": {"tp": 4, "fp": 3, "fn": 2}, "all": {"tp": 4, "fp": 3, "fn": 2}}, "claude-code": {"strict": {"tp": 2, "fp": 1, "fn": 4}, "core": {"tp": 2, "fp": 1, "fn": 4}, "all": {"tp": 2, "fp": 1, "fn": 4}}, "codeant-v2": {"strict": {"tp": 4, "fp": 9, "fn": 2}, "core": {"tp": 4, "fp": 9, "fn": 2}, "all": {"tp": 4, "fp": 9, "fn": 2}}, "coderabbit": {"strict": {"tp": 4, "fp": 6, "fn": 2}, "core": {"tp": 4, "fp": 6, "fn": 2}, "all": {"tp": 4, "fp": 6, "fn": 2}}, "cubic-v2": {"strict": {"tp": 3, "fp": 1, "fn": 3}, "core": {"tp": 3, "fp": 1, "fn": 3}, "all": {"tp": 3, "fp": 1, "fn": 3}}, "devin": {"strict": {"tp": 2, "fp": 0, "fn": 4}, "core": {"tp": 2, "fp": 0, "fn": 4}, "all": {"tp": 2, "fp": 0, "fn": 4}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 6}, "core": {"tp": 0, "fp": 0, "fn": 6}, "all": {"tp": 0, "fp": 0, "fn": 6}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 6}, "core": {"tp": 0, "fp": 0, "fn": 6}, "all": {"tp": 0, "fp": 0, "fn": 6}}, "kodus-v2": {"strict": {"tp": 1, "fp": 6, "fn": 5}, "core": {"tp": 1, "fp": 6, "fn": 5}, "all": {"tp": 1, "fp": 6, "fn": 5}}, "macroscope": {"strict": {"tp": 1, "fp": 3, "fn": 5}, "core": {"tp": 1, "fp": 3, "fn": 5}, "all": {"tp": 1, "fp": 3, "fn": 5}}, "qodo-extended-v2": {"strict": {"tp": 3, "fp": 1, "fn": 3}, "core": {"tp": 3, "fp": 1, "fn": 3}, "all": {"tp": 3, "fp": 1, "fn": 3}}, "qodo-v2": {"strict": {"tp": 2, "fp": 1, "fn": 4}, "core": {"tp": 2, "fp": 1, "fn": 4}, "all": {"tp": 2, "fp": 1, "fn": 4}}, "sourcery": {"strict": {"tp": 3, "fp": 2, "fn": 3}, "core": {"tp": 3, "fp": 2, "fn": 3}, "all": {"tp": 3, "fp": 2, "fn": 3}}, "greptile-v4-1": {"strict": {"tp": 3, "fp": 1, "fn": 3}, "core": {"tp": 3, "fp": 1, "fn": 3}, "all": {"tp": 3, "fp": 1, "fn": 3}}, "gemini-v2": {"strict": {"tp": 2, "fp": 2, "fn": 4}, "core": {"tp": 2, "fp": 2, "fn": 4}, "all": {"tp": 2, "fp": 2, "fn": 4}}, "gitlab": {"strict": {"tp": 3, "fp": 2, "fn": 3}, "core": {"tp": 3, "fp": 2, "fn": 3}, "all": {"tp": 3, "fp": 2, "fn": 3}}, "copilot-v2": {"strict": {"tp": 3, "fp": 14, "fn": 3}, "core": {"tp": 3, "fp": 14, "fn": 3}, "all": {"tp": 3, "fp": 14, "fn": 3}}}}, {"url": "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/3", "language": "Ruby", "pr_size": "medium", "domain": "authentication", "change_type": "feature", "complexity": "moderate", "difficulty": "subtle", "risk": "medium", "context": "file", "concern": "correctness", "summary": "", "tool_metrics": {"augment": {"strict": {"tp": 1, "fp": 4, "fn": 2}, "core": {"tp": 1, "fp": 4, "fn": 2}, "all": {"tp": 1, "fp": 4, "fn": 2}}, "baz": {"strict": {"tp": 1, "fp": 0, "fn": 2}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 2}}, "bugbot": {"strict": {"tp": 1, "fp": 1, "fn": 2}, "core": {"tp": 1, "fp": 1, "fn": 2}, "all": {"tp": 1, "fp": 1, "fn": 2}}, "claude": {"strict": {"tp": 1, "fp": 6, "fn": 2}, "core": {"tp": 1, "fp": 6, "fn": 2}, "all": {"tp": 1, "fp": 6, "fn": 2}}, "claude-code": {"strict": {"tp": 1, "fp": 4, "fn": 2}, "core": {"tp": 1, "fp": 4, "fn": 2}, "all": {"tp": 1, "fp": 4, "fn": 2}}, "codeant-v2": {"strict": {"tp": 1, "fp": 3, "fn": 2}, "core": {"tp": 1, "fp": 3, "fn": 2}, "all": {"tp": 1, "fp": 3, "fn": 2}}, "coderabbit": {"strict": {"tp": 2, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 1}, "all": {"tp": 2, "fp": 1, "fn": 1}}, "cubic-v2": {"strict": {"tp": 2, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 1}, "all": {"tp": 2, "fp": 1, "fn": 1}}, "devin": {"strict": {"tp": 1, "fp": 1, "fn": 2}, "core": {"tp": 1, "fp": 1, "fn": 2}, "all": {"tp": 1, "fp": 1, "fn": 2}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 3}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 3}}, "kg": {"strict": {"tp": 1, "fp": 0, "fn": 2}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 2}}, "kodus-v2": {"strict": {"tp": 1, "fp": 0, "fn": 2}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 2}}, "macroscope": {"strict": {"tp": 2, "fp": 0, "fn": 1}, "core": {"tp": 2, "fp": 0, "fn": 1}, "all": {"tp": 2, "fp": 0, "fn": 1}}, "qodo-extended-v2": {"strict": {"tp": 1, "fp": 1, "fn": 2}, "core": {"tp": 1, "fp": 1, "fn": 2}, "all": {"tp": 1, "fp": 1, "fn": 2}}, "qodo-v2": {"strict": {"tp": 2, "fp": 2, "fn": 1}, "core": {"tp": 2, "fp": 2, "fn": 1}, "all": {"tp": 2, "fp": 2, "fn": 1}}, "sourcery": {"strict": {"tp": 1, "fp": 3, "fn": 2}, "core": {"tp": 1, "fp": 3, "fn": 2}, "all": {"tp": 1, "fp": 3, "fn": 2}}, "greptile-v4-1": {"strict": {"tp": 1, "fp": 3, "fn": 2}, "core": {"tp": 1, "fp": 3, "fn": 2}, "all": {"tp": 1, "fp": 3, "fn": 2}}, "gemini-v2": {"strict": {"tp": 2, "fp": 3, "fn": 1}, "core": {"tp": 2, "fp": 3, "fn": 1}, "all": {"tp": 2, "fp": 3, "fn": 1}}, "gitlab": {"strict": {"tp": 0, "fp": 2, "fn": 3}, "core": {"tp": 0, "fp": 2, "fn": 3}, "all": {"tp": 0, "fp": 2, "fn": 3}}, "copilot-v2": {"strict": {"tp": 2, "fp": 6, "fn": 1}, "core": {"tp": 2, "fp": 6, "fn": 1}, "all": {"tp": 2, "fp": 6, "fn": 1}}}}, {"url": "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/5", "language": "Ruby", "pr_size": "small", "domain": "UI", "change_type": "performance", "complexity": "moderate", "difficulty": "subtle", "risk": "medium", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"augment": {"strict": {"tp": 2, "fp": 0, "fn": 1}, "core": {"tp": 2, "fp": 0, "fn": 1}, "all": {"tp": 2, "fp": 0, "fn": 1}}, "baz": {"strict": {"tp": 1, "fp": 2, "fn": 2}, "core": {"tp": 1, "fp": 2, "fn": 2}, "all": {"tp": 1, "fp": 2, "fn": 2}}, "bugbot": {"strict": {"tp": 0, "fp": 1, "fn": 3}, "core": {"tp": 0, "fp": 1, "fn": 3}, "all": {"tp": 0, "fp": 1, "fn": 3}}, "claude": {"strict": {"tp": 1, "fp": 0, "fn": 2}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 2}}, "claude-code": {"strict": {"tp": 2, "fp": 3, "fn": 1}, "core": {"tp": 2, "fp": 3, "fn": 1}, "all": {"tp": 2, "fp": 3, "fn": 1}}, "codeant-v2": {"strict": {"tp": 0, "fp": 0, "fn": 3}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 3}}, "coderabbit": {"strict": {"tp": 1, "fp": 0, "fn": 2}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 2}}, "cubic-v2": {"strict": {"tp": 2, "fp": 2, "fn": 1}, "core": {"tp": 2, "fp": 2, "fn": 1}, "all": {"tp": 2, "fp": 2, "fn": 1}}, "devin": {"strict": {"tp": 1, "fp": 1, "fn": 2}, "core": {"tp": 1, "fp": 1, "fn": 2}, "all": {"tp": 1, "fp": 1, "fn": 2}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 3}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 3}}, "kg": {"strict": {"tp": 1, "fp": 0, "fn": 2}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 2}}, "kodus-v2": {"strict": {"tp": 0, "fp": 0, "fn": 3}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 3}}, "macroscope": {"strict": {"tp": 0, "fp": 0, "fn": 3}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 3}}, "qodo-extended-v2": {"strict": {"tp": 2, "fp": 0, "fn": 1}, "core": {"tp": 2, "fp": 0, "fn": 1}, "all": {"tp": 2, "fp": 0, "fn": 1}}, "qodo-v2": {"strict": {"tp": 1, "fp": 1, "fn": 2}, "core": {"tp": 1, "fp": 1, "fn": 2}, "all": {"tp": 1, "fp": 1, "fn": 2}}, "sourcery": {"strict": {"tp": 2, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 1}, "all": {"tp": 2, "fp": 1, "fn": 1}}, "greptile-v4-1": {"strict": {"tp": 1, "fp": 2, "fn": 2}, "core": {"tp": 1, "fp": 2, "fn": 2}, "all": {"tp": 1, "fp": 2, "fn": 2}}, "gemini-v2": {"strict": {"tp": 2, "fp": 7, "fn": 1}, "core": {"tp": 2, "fp": 7, "fn": 1}, "all": {"tp": 2, "fp": 7, "fn": 1}}, "gitlab": {"strict": {"tp": 1, "fp": 1, "fn": 2}, "core": {"tp": 1, "fp": 1, "fn": 2}, "all": {"tp": 1, "fp": 1, "fn": 2}}, "copilot-v2": {"strict": {"tp": 2, "fp": 2, "fn": 1}, "core": {"tp": 2, "fp": 2, "fn": 1}, "all": {"tp": 2, "fp": 2, "fn": 1}}}}, {"url": "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/6", "language": "Ruby", "pr_size": "small", "domain": "serialization", "change_type": "feature", "complexity": "simple", "difficulty": "moderate", "risk": "medium", "context": "file", "concern": "correctness", "summary": "", "tool_metrics": {"augment": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 1, "fp": 1, "fn": 0}, "all": {"tp": 1, "fp": 1, "fn": 0}}, "baz": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 1, "fp": 1, "fn": 0}, "all": {"tp": 1, "fp": 1, "fn": 0}}, "bugbot": {"strict": {"tp": 1, "fp": 4, "fn": 0}, "core": {"tp": 1, "fp": 4, "fn": 0}, "all": {"tp": 1, "fp": 4, "fn": 0}}, "claude": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 0, "fp": 0, "fn": 1}}, "claude-code": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 1, "fp": 1, "fn": 0}, "all": {"tp": 1, "fp": 1, "fn": 0}}, "codeant-v2": {"strict": {"tp": 1, "fp": 2, "fn": 0}, "core": {"tp": 1, "fp": 2, "fn": 0}, "all": {"tp": 1, "fp": 2, "fn": 0}}, "coderabbit": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 0}, "all": {"tp": 1, "fp": 0, "fn": 0}}, "cubic-v2": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 1, "fp": 1, "fn": 0}, "all": {"tp": 1, "fp": 1, "fn": 0}}, "devin": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 1, "fp": 1, "fn": 0}, "all": {"tp": 1, "fp": 1, "fn": 0}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 0, "fp": 0, "fn": 1}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 0, "fp": 0, "fn": 1}}, "kodus-v2": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 1, "fp": 1, "fn": 0}, "all": {"tp": 1, "fp": 1, "fn": 0}}, "macroscope": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 0}, "all": {"tp": 1, "fp": 0, "fn": 0}}, "qodo-extended-v2": {"strict": {"tp": 0, "fp": 4, "fn": 1}, "core": {"tp": 0, "fp": 4, "fn": 1}, "all": {"tp": 0, "fp": 4, "fn": 1}}, "qodo-v2": {"strict": {"tp": 1, "fp": 2, "fn": 0}, "core": {"tp": 1, "fp": 2, "fn": 0}, "all": {"tp": 1, "fp": 2, "fn": 0}}, "sourcery": {"strict": {"tp": 1, "fp": 3, "fn": 0}, "core": {"tp": 1, "fp": 3, "fn": 0}, "all": {"tp": 1, "fp": 3, "fn": 0}}, "greptile-v4-1": {"strict": {"tp": 1, "fp": 3, "fn": 0}, "core": {"tp": 1, "fp": 3, "fn": 0}, "all": {"tp": 1, "fp": 3, "fn": 0}}, "gemini-v2": {"strict": {"tp": 1, "fp": 5, "fn": 0}, "core": {"tp": 1, "fp": 5, "fn": 0}, "all": {"tp": 1, "fp": 5, "fn": 0}}, "gitlab": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 1, "fp": 1, "fn": 0}, "all": {"tp": 1, "fp": 1, "fn": 0}}, "copilot-v2": {"strict": {"tp": 1, "fp": 3, "fn": 0}, "core": {"tp": 1, "fp": 3, "fn": 0}, "all": {"tp": 1, "fp": 3, "fn": 0}}}}, {"url": "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/4", "language": "Ruby", "pr_size": "large", "domain": "API", "change_type": "feature", "complexity": "complex", "difficulty": "subtle", "risk": "critical", "context": "cross_file", "concern": "security", "summary": "", "tool_metrics": {"augment": {"strict": {"tp": 4, "fp": 1, "fn": 4}, "core": {"tp": 4, "fp": 1, "fn": 4}, "all": {"tp": 4, "fp": 1, "fn": 4}}, "baz": {"strict": {"tp": 2, "fp": 2, "fn": 6}, "core": {"tp": 2, "fp": 2, "fn": 6}, "all": {"tp": 2, "fp": 2, "fn": 6}}, "bugbot": {"strict": {"tp": 2, "fp": 3, "fn": 6}, "core": {"tp": 2, "fp": 3, "fn": 6}, "all": {"tp": 2, "fp": 3, "fn": 6}}, "claude": {"strict": {"tp": 6, "fp": 2, "fn": 2}, "core": {"tp": 6, "fp": 2, "fn": 2}, "all": {"tp": 6, "fp": 2, "fn": 2}}, "claude-code": {"strict": {"tp": 4, "fp": 0, "fn": 4}, "core": {"tp": 4, "fp": 0, "fn": 4}, "all": {"tp": 4, "fp": 0, "fn": 4}}, "codeant-v2": {"strict": {"tp": 3, "fp": 16, "fn": 5}, "core": {"tp": 3, "fp": 16, "fn": 5}, "all": {"tp": 3, "fp": 16, "fn": 5}}, "coderabbit": {"strict": {"tp": 5, "fp": 11, "fn": 3}, "core": {"tp": 5, "fp": 11, "fn": 3}, "all": {"tp": 5, "fp": 11, "fn": 3}}, "cubic-v2": {"strict": {"tp": 1, "fp": 4, "fn": 7}, "core": {"tp": 1, "fp": 4, "fn": 7}, "all": {"tp": 1, "fp": 4, "fn": 7}}, "devin": {"strict": {"tp": 4, "fp": 1, "fn": 4}, "core": {"tp": 4, "fp": 1, "fn": 4}, "all": {"tp": 4, "fp": 1, "fn": 4}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 8}, "core": {"tp": 0, "fp": 0, "fn": 8}, "all": {"tp": 0, "fp": 0, "fn": 8}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 8}, "core": {"tp": 0, "fp": 0, "fn": 8}, "all": {"tp": 0, "fp": 0, "fn": 8}}, "kodus-v2": {"strict": {"tp": 3, "fp": 2, "fn": 5}, "core": {"tp": 3, "fp": 2, "fn": 5}, "all": {"tp": 3, "fp": 2, "fn": 5}}, "macroscope": {"strict": {"tp": 2, "fp": 5, "fn": 6}, "core": {"tp": 2, "fp": 5, "fn": 6}, "all": {"tp": 2, "fp": 5, "fn": 6}}, "qodo-extended-v2": {"strict": {"tp": 4, "fp": 1, "fn": 4}, "core": {"tp": 4, "fp": 1, "fn": 4}, "all": {"tp": 4, "fp": 1, "fn": 4}}, "qodo-v2": {"strict": {"tp": 6, "fp": 3, "fn": 2}, "core": {"tp": 6, "fp": 3, "fn": 2}, "all": {"tp": 6, "fp": 3, "fn": 2}}, "sourcery": {"strict": {"tp": 3, "fp": 2, "fn": 5}, "core": {"tp": 3, "fp": 2, "fn": 5}, "all": {"tp": 3, "fp": 2, "fn": 5}}, "greptile-v4-1": {"strict": {"tp": 5, "fp": 1, "fn": 3}, "core": {"tp": 5, "fp": 1, "fn": 3}, "all": {"tp": 5, "fp": 1, "fn": 3}}, "gemini-v2": {"strict": {"tp": 4, "fp": 6, "fn": 4}, "core": {"tp": 4, "fp": 6, "fn": 4}, "all": {"tp": 4, "fp": 6, "fn": 4}}, "gitlab": {"strict": {"tp": 4, "fp": 1, "fn": 4}, "core": {"tp": 4, "fp": 1, "fn": 4}, "all": {"tp": 4, "fp": 1, "fn": 4}}, "copilot-v2": {"strict": {"tp": 5, "fp": 5, "fn": 3}, "core": {"tp": 5, "fp": 5, "fn": 3}, "all": {"tp": 5, "fp": 5, "fn": 3}}}}, {"url": "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/1", "language": "Ruby", "pr_size": "medium", "domain": "file_io", "change_type": "feature", "complexity": "moderate", "difficulty": "subtle", "risk": "medium", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"augment": {"strict": {"tp": 2, "fp": 3, "fn": 2}, "core": {"tp": 2, "fp": 3, "fn": 2}, "all": {"tp": 2, "fp": 3, "fn": 2}}, "baz": {"strict": {"tp": 2, "fp": 0, "fn": 2}, "core": {"tp": 2, "fp": 0, "fn": 2}, "all": {"tp": 2, "fp": 0, "fn": 2}}, "bugbot": {"strict": {"tp": 2, "fp": 1, "fn": 2}, "core": {"tp": 2, "fp": 1, "fn": 2}, "all": {"tp": 2, "fp": 1, "fn": 2}}, "claude": {"strict": {"tp": 2, "fp": 0, "fn": 2}, "core": {"tp": 2, "fp": 0, "fn": 2}, "all": {"tp": 2, "fp": 0, "fn": 2}}, "claude-code": {"strict": {"tp": 3, "fp": 1, "fn": 1}, "core": {"tp": 3, "fp": 1, "fn": 1}, "all": {"tp": 3, "fp": 1, "fn": 1}}, "codeant-v2": {"strict": {"tp": 2, "fp": 2, "fn": 2}, "core": {"tp": 2, "fp": 2, "fn": 2}, "all": {"tp": 2, "fp": 2, "fn": 2}}, "coderabbit": {"strict": {"tp": 2, "fp": 3, "fn": 2}, "core": {"tp": 2, "fp": 3, "fn": 2}, "all": {"tp": 2, "fp": 3, "fn": 2}}, "cubic-v2": {"strict": {"tp": 3, "fp": 0, "fn": 1}, "core": {"tp": 3, "fp": 0, "fn": 1}, "all": {"tp": 3, "fp": 0, "fn": 1}}, "devin": {"strict": {"tp": 2, "fp": 0, "fn": 2}, "core": {"tp": 2, "fp": 0, "fn": 2}, "all": {"tp": 2, "fp": 0, "fn": 2}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 4}, "core": {"tp": 0, "fp": 0, "fn": 4}, "all": {"tp": 0, "fp": 0, "fn": 4}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 4}, "core": {"tp": 0, "fp": 0, "fn": 4}, "all": {"tp": 0, "fp": 0, "fn": 4}}, "kodus-v2": {"strict": {"tp": 3, "fp": 3, "fn": 1}, "core": {"tp": 3, "fp": 3, "fn": 1}, "all": {"tp": 3, "fp": 3, "fn": 1}}, "macroscope": {"strict": {"tp": 1, "fp": 1, "fn": 3}, "core": {"tp": 1, "fp": 1, "fn": 3}, "all": {"tp": 1, "fp": 1, "fn": 3}}, "qodo-extended-v2": {"strict": {"tp": 2, "fp": 1, "fn": 2}, "core": {"tp": 2, "fp": 1, "fn": 2}, "all": {"tp": 2, "fp": 1, "fn": 2}}, "qodo-v2": {"strict": {"tp": 2, "fp": 3, "fn": 2}, "core": {"tp": 2, "fp": 3, "fn": 2}, "all": {"tp": 2, "fp": 3, "fn": 2}}, "sourcery": {"strict": {"tp": 2, "fp": 1, "fn": 2}, "core": {"tp": 2, "fp": 1, "fn": 2}, "all": {"tp": 2, "fp": 1, "fn": 2}}, "greptile-v4-1": {"strict": {"tp": 3, "fp": 1, "fn": 1}, "core": {"tp": 3, "fp": 1, "fn": 1}, "all": {"tp": 3, "fp": 1, "fn": 1}}, "gemini-v2": {"strict": {"tp": 3, "fp": 0, "fn": 1}, "core": {"tp": 3, "fp": 0, "fn": 1}, "all": {"tp": 3, "fp": 0, "fn": 1}}, "gitlab": {"strict": {"tp": 2, "fp": 0, "fn": 2}, "core": {"tp": 2, "fp": 0, "fn": 2}, "all": {"tp": 2, "fp": 0, "fn": 2}}, "copilot-v2": {"strict": {"tp": 2, "fp": 1, "fn": 2}, "core": {"tp": 2, "fp": 1, "fn": 2}, "all": {"tp": 2, "fp": 1, "fn": 2}}}}, {"url": "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/2", "language": "Ruby", "pr_size": "medium", "domain": "UI", "change_type": "feature", "complexity": "moderate", "difficulty": "moderate", "risk": "medium", "context": "file", "concern": "correctness", "summary": "", "tool_metrics": {"augment": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 1, "fp": 1, "fn": 0}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "baz": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 0}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "bugbot": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 1, "fp": 1, "fn": 0}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "claude": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 1, "fp": 1, "fn": 0}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "claude-code": {"strict": {"tp": 0, "fp": 2, "fn": 1}, "core": {"tp": 0, "fp": 2, "fn": 1}, "all": {"tp": 0, "fp": 2, "fn": 2}}, "codeant-v2": {"strict": {"tp": 1, "fp": 5, "fn": 0}, "core": {"tp": 1, "fp": 5, "fn": 0}, "all": {"tp": 1, "fp": 5, "fn": 1}}, "coderabbit": {"strict": {"tp": 1, "fp": 6, "fn": 0}, "core": {"tp": 1, "fp": 6, "fn": 0}, "all": {"tp": 1, "fp": 6, "fn": 1}}, "cubic-v2": {"strict": {"tp": 0, "fp": 2, "fn": 1}, "core": {"tp": 0, "fp": 2, "fn": 1}, "all": {"tp": 0, "fp": 2, "fn": 2}}, "devin": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 0}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "kodus-v2": {"strict": {"tp": 1, "fp": 2, "fn": 0}, "core": {"tp": 1, "fp": 2, "fn": 0}, "all": {"tp": 1, "fp": 2, "fn": 1}}, "macroscope": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 0}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "qodo-extended-v2": {"strict": {"tp": 1, "fp": 3, "fn": 0}, "core": {"tp": 1, "fp": 3, "fn": 0}, "all": {"tp": 1, "fp": 3, "fn": 1}}, "qodo-v2": {"strict": {"tp": 1, "fp": 2, "fn": 0}, "core": {"tp": 1, "fp": 2, "fn": 0}, "all": {"tp": 1, "fp": 2, "fn": 1}}, "sourcery": {"strict": {"tp": 1, "fp": 5, "fn": 0}, "core": {"tp": 1, "fp": 5, "fn": 0}, "all": {"tp": 1, "fp": 5, "fn": 1}}, "greptile-v4-1": {"strict": {"tp": 1, "fp": 2, "fn": 0}, "core": {"tp": 1, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "gemini-v2": {"strict": {"tp": 1, "fp": 2, "fn": 0}, "core": {"tp": 1, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "gitlab": {"strict": {"tp": 1, "fp": 2, "fn": 0}, "core": {"tp": 1, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "copilot-v2": {"strict": {"tp": 1, "fp": 5, "fn": 0}, "core": {"tp": 1, "fp": 5, "fn": 0}, "all": {"tp": 1, "fp": 5, "fn": 1}}}}, {"url": "https://github.com/calcom/cal.com/pull/22532", "language": "TypeScript", "pr_size": "medium", "domain": "caching", "change_type": "feature", "complexity": "moderate", "difficulty": "subtle", "risk": "medium", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"augment": {"strict": {"tp": 4, "fp": 2, "fn": 0}, "core": {"tp": 4, "fp": 2, "fn": 0}, "all": {"tp": 4, "fp": 2, "fn": 0}}, "baz": {"strict": {"tp": 1, "fp": 2, "fn": 3}, "core": {"tp": 1, "fp": 2, "fn": 3}, "all": {"tp": 1, "fp": 2, "fn": 3}}, "bugbot": {"strict": {"tp": 0, "fp": 1, "fn": 4}, "core": {"tp": 0, "fp": 1, "fn": 4}, "all": {"tp": 0, "fp": 1, "fn": 4}}, "claude": {"strict": {"tp": 0, "fp": 1, "fn": 4}, "core": {"tp": 0, "fp": 1, "fn": 4}, "all": {"tp": 0, "fp": 1, "fn": 4}}, "claude-code": {"strict": {"tp": 0, "fp": 5, "fn": 4}, "core": {"tp": 0, "fp": 5, "fn": 4}, "all": {"tp": 0, "fp": 5, "fn": 4}}, "codeant-v2": {"strict": {"tp": 3, "fp": 2, "fn": 1}, "core": {"tp": 3, "fp": 2, "fn": 1}, "all": {"tp": 3, "fp": 2, "fn": 1}}, "coderabbit": {"strict": {"tp": 2, "fp": 3, "fn": 2}, "core": {"tp": 2, "fp": 3, "fn": 2}, "all": {"tp": 2, "fp": 3, "fn": 2}}, "cubic-v2": {"strict": {"tp": 0, "fp": 2, "fn": 4}, "core": {"tp": 0, "fp": 2, "fn": 4}, "all": {"tp": 0, "fp": 2, "fn": 4}}, "devin": {"strict": {"tp": 1, "fp": 1, "fn": 3}, "core": {"tp": 1, "fp": 1, "fn": 3}, "all": {"tp": 1, "fp": 1, "fn": 3}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 4}, "core": {"tp": 0, "fp": 0, "fn": 4}, "all": {"tp": 0, "fp": 0, "fn": 4}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 4}, "core": {"tp": 0, "fp": 0, "fn": 4}, "all": {"tp": 0, "fp": 0, "fn": 4}}, "kodus-v2": {"strict": {"tp": 1, "fp": 0, "fn": 3}, "core": {"tp": 1, "fp": 0, "fn": 3}, "all": {"tp": 1, "fp": 0, "fn": 3}}, "macroscope": {"strict": {"tp": 1, "fp": 1, "fn": 3}, "core": {"tp": 1, "fp": 1, "fn": 3}, "all": {"tp": 1, "fp": 1, "fn": 3}}, "qodo-extended-v2": {"strict": {"tp": 2, "fp": 1, "fn": 2}, "core": {"tp": 2, "fp": 1, "fn": 2}, "all": {"tp": 2, "fp": 1, "fn": 2}}, "qodo-v2": {"strict": {"tp": 3, "fp": 2, "fn": 1}, "core": {"tp": 3, "fp": 2, "fn": 1}, "all": {"tp": 3, "fp": 2, "fn": 1}}, "sourcery": {"strict": {"tp": 2, "fp": 2, "fn": 2}, "core": {"tp": 2, "fp": 2, "fn": 2}, "all": {"tp": 2, "fp": 2, "fn": 2}}, "greptile-v4-1": {"strict": {"tp": 3, "fp": 2, "fn": 1}, "core": {"tp": 3, "fp": 2, "fn": 1}, "all": {"tp": 3, "fp": 2, "fn": 1}}, "gemini-v2": {"strict": {"tp": 3, "fp": 1, "fn": 1}, "core": {"tp": 3, "fp": 1, "fn": 1}, "all": {"tp": 3, "fp": 1, "fn": 1}}, "gitlab": {"strict": {"tp": 2, "fp": 3, "fn": 2}, "core": {"tp": 2, "fp": 3, "fn": 2}, "all": {"tp": 2, "fp": 3, "fn": 2}}, "copilot-v2": {"strict": {"tp": 3, "fp": 3, "fn": 1}, "core": {"tp": 3, "fp": 3, "fn": 1}, "all": {"tp": 3, "fp": 3, "fn": 1}}}}, {"url": "https://github.com/calcom/cal.com/pull/8330", "language": "TypeScript", "pr_size": "small", "domain": "scheduling", "change_type": "bug_fix", "complexity": "moderate", "difficulty": "subtle", "risk": "medium", "context": "file", "concern": "correctness", "summary": "", "tool_metrics": {"augment": {"strict": {"tp": 2, "fp": 3, "fn": 0}, "core": {"tp": 2, "fp": 3, "fn": 0}, "all": {"tp": 2, "fp": 3, "fn": 0}}, "baz": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "bugbot": {"strict": {"tp": 2, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "claude": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "claude-code": {"strict": {"tp": 2, "fp": 2, "fn": 0}, "core": {"tp": 2, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "codeant-v2": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "coderabbit": {"strict": {"tp": 2, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "cubic-v2": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "devin": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "graphite": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "kg": {"strict": {"tp": 0, "fp": 3, "fn": 2}, "core": {"tp": 0, "fp": 3, "fn": 2}, "all": {"tp": 0, "fp": 3, "fn": 2}}, "kodus-v2": {"strict": {"tp": 2, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "macroscope": {"strict": {"tp": 2, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "qodo-extended-v2": {"strict": {"tp": 2, "fp": 2, "fn": 0}, "core": {"tp": 2, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "qodo-v2": {"strict": {"tp": 2, "fp": 2, "fn": 0}, "core": {"tp": 2, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "sourcery": {"strict": {"tp": 2, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "greptile-v4-1": {"strict": {"tp": 2, "fp": 2, "fn": 0}, "core": {"tp": 2, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "gemini-v2": {"strict": {"tp": 2, "fp": 3, "fn": 0}, "core": {"tp": 2, "fp": 3, "fn": 0}, "all": {"tp": 2, "fp": 3, "fn": 0}}, "gitlab": {"strict": {"tp": 2, "fp": 2, "fn": 0}, "core": {"tp": 2, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "copilot-v2": {"strict": {"tp": 2, "fp": 4, "fn": 0}, "core": {"tp": 2, "fp": 4, "fn": 0}, "all": {"tp": 2, "fp": 4, "fn": 0}}}}, {"url": "https://github.com/calcom/cal.com/pull/14943", "language": "TypeScript", "pr_size": "small", "domain": "scheduling", "change_type": "feature", "complexity": "moderate", "difficulty": "subtle", "risk": "high", "context": "file", "concern": "correctness", "summary": "", "tool_metrics": {"augment": {"strict": {"tp": 2, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "baz": {"strict": {"tp": 1, "fp": 2, "fn": 1}, "core": {"tp": 1, "fp": 2, "fn": 1}, "all": {"tp": 1, "fp": 2, "fn": 1}}, "bugbot": {"strict": {"tp": 1, "fp": 2, "fn": 1}, "core": {"tp": 1, "fp": 2, "fn": 1}, "all": {"tp": 1, "fp": 2, "fn": 1}}, "claude": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "claude-code": {"strict": {"tp": 0, "fp": 1, "fn": 2}, "core": {"tp": 0, "fp": 1, "fn": 2}, "all": {"tp": 0, "fp": 1, "fn": 2}}, "codeant-v2": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "coderabbit": {"strict": {"tp": 2, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "cubic-v2": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "devin": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "graphite": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "kg": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "kodus-v2": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "macroscope": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "qodo-extended-v2": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "qodo-v2": {"strict": {"tp": 2, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "sourcery": {"strict": {"tp": 2, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "greptile-v4-1": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "gemini-v2": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "gitlab": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "copilot-v2": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}}}, {"url": "https://github.com/calcom/cal.com/pull/22345", "language": "TypeScript", "pr_size": "small", "domain": "database", "change_type": "migration", "complexity": "moderate", "difficulty": "subtle", "risk": "medium", "context": "file", "concern": "correctness", "summary": "", "tool_metrics": {"augment": {"strict": {"tp": 0, "fp": 1, "fn": 1}, "core": {"tp": 0, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "baz": {"strict": {"tp": 0, "fp": 2, "fn": 1}, "core": {"tp": 0, "fp": 2, "fn": 1}, "all": {"tp": 0, "fp": 2, "fn": 2}}, "bugbot": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 1, "fp": 1, "fn": 0}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "claude": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "claude-code": {"strict": {"tp": 1, "fp": 3, "fn": 0}, "core": {"tp": 1, "fp": 3, "fn": 0}, "all": {"tp": 1, "fp": 3, "fn": 1}}, "codeant-v2": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "coderabbit": {"strict": {"tp": 0, "fp": 3, "fn": 1}, "core": {"tp": 0, "fp": 3, "fn": 1}, "all": {"tp": 1, "fp": 3, "fn": 1}}, "cubic-v2": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 1, "fp": 1, "fn": 0}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "devin": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "kg": {"strict": {"tp": 0, "fp": 1, "fn": 1}, "core": {"tp": 0, "fp": 1, "fn": 1}, "all": {"tp": 0, "fp": 1, "fn": 2}}, "kodus-v2": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "macroscope": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "qodo-extended-v2": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "qodo-v2": {"strict": {"tp": 0, "fp": 3, "fn": 1}, "core": {"tp": 0, "fp": 3, "fn": 1}, "all": {"tp": 1, "fp": 3, "fn": 1}}, "sourcery": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 0}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "greptile-v4-1": {"strict": {"tp": 0, "fp": 5, "fn": 1}, "core": {"tp": 0, "fp": 5, "fn": 1}, "all": {"tp": 0, "fp": 5, "fn": 2}}, "gemini-v2": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 0}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "gitlab": {"strict": {"tp": 1, "fp": 2, "fn": 0}, "core": {"tp": 1, "fp": 2, "fn": 0}, "all": {"tp": 1, "fp": 2, "fn": 1}}, "copilot-v2": {"strict": {"tp": 0, "fp": 5, "fn": 1}, "core": {"tp": 0, "fp": 5, "fn": 1}, "all": {"tp": 1, "fp": 5, "fn": 1}}}}, {"url": "https://github.com/calcom/cal.com/pull/11059", "language": "TypeScript", "pr_size": "large", "domain": "authentication", "change_type": "feature", "complexity": "complex", "difficulty": "subtle", "risk": "critical", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"augment": {"strict": {"tp": 8, "fp": 2, "fn": 1}, "core": {"tp": 8, "fp": 2, "fn": 1}, "all": {"tp": 8, "fp": 2, "fn": 1}}, "baz": {"strict": {"tp": 5, "fp": 0, "fn": 4}, "core": {"tp": 5, "fp": 0, "fn": 4}, "all": {"tp": 5, "fp": 0, "fn": 4}}, "bugbot": {"strict": {"tp": 6, "fp": 1, "fn": 3}, "core": {"tp": 6, "fp": 1, "fn": 3}, "all": {"tp": 6, "fp": 1, "fn": 3}}, "claude": {"strict": {"tp": 5, "fp": 1, "fn": 4}, "core": {"tp": 5, "fp": 1, "fn": 4}, "all": {"tp": 5, "fp": 1, "fn": 4}}, "claude-code": {"strict": {"tp": 4, "fp": 0, "fn": 5}, "core": {"tp": 4, "fp": 0, "fn": 5}, "all": {"tp": 4, "fp": 0, "fn": 5}}, "codeant-v2": {"strict": {"tp": 4, "fp": 4, "fn": 5}, "core": {"tp": 4, "fp": 4, "fn": 5}, "all": {"tp": 4, "fp": 4, "fn": 5}}, "coderabbit": {"strict": {"tp": 9, "fp": 9, "fn": 0}, "core": {"tp": 9, "fp": 9, "fn": 0}, "all": {"tp": 9, "fp": 9, "fn": 0}}, "cubic-v2": {"strict": {"tp": 3, "fp": 0, "fn": 6}, "core": {"tp": 3, "fp": 0, "fn": 6}, "all": {"tp": 3, "fp": 0, "fn": 6}}, "devin": {"strict": {"tp": 5, "fp": 0, "fn": 4}, "core": {"tp": 5, "fp": 0, "fn": 4}, "all": {"tp": 5, "fp": 0, "fn": 4}}, "graphite": {"strict": {"tp": 1, "fp": 0, "fn": 8}, "core": {"tp": 1, "fp": 0, "fn": 8}, "all": {"tp": 1, "fp": 0, "fn": 8}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 9}, "core": {"tp": 0, "fp": 0, "fn": 9}, "all": {"tp": 0, "fp": 0, "fn": 9}}, "kodus-v2": {"strict": {"tp": 4, "fp": 3, "fn": 5}, "core": {"tp": 4, "fp": 3, "fn": 5}, "all": {"tp": 4, "fp": 3, "fn": 5}}, "macroscope": {"strict": {"tp": 5, "fp": 5, "fn": 4}, "core": {"tp": 5, "fp": 5, "fn": 4}, "all": {"tp": 5, "fp": 5, "fn": 4}}, "qodo-extended-v2": {"strict": {"tp": 7, "fp": 2, "fn": 2}, "core": {"tp": 7, "fp": 2, "fn": 2}, "all": {"tp": 7, "fp": 2, "fn": 2}}, "qodo-v2": {"strict": {"tp": 8, "fp": 3, "fn": 1}, "core": {"tp": 8, "fp": 3, "fn": 1}, "all": {"tp": 8, "fp": 3, "fn": 1}}, "sourcery": {"strict": {"tp": 5, "fp": 0, "fn": 4}, "core": {"tp": 5, "fp": 0, "fn": 4}, "all": {"tp": 5, "fp": 0, "fn": 4}}, "greptile-v4-1": {"strict": {"tp": 5, "fp": 3, "fn": 4}, "core": {"tp": 5, "fp": 3, "fn": 4}, "all": {"tp": 5, "fp": 3, "fn": 4}}, "gemini-v2": {"strict": {"tp": 4, "fp": 2, "fn": 5}, "core": {"tp": 4, "fp": 2, "fn": 5}, "all": {"tp": 4, "fp": 2, "fn": 5}}, "gitlab": {"strict": {"tp": 5, "fp": 2, "fn": 4}, "core": {"tp": 5, "fp": 2, "fn": 4}, "all": {"tp": 5, "fp": 2, "fn": 4}}, "copilot-v2": {"strict": {"tp": 6, "fp": 1, "fn": 3}, "core": {"tp": 6, "fp": 1, "fn": 3}, "all": {"tp": 6, "fp": 1, "fn": 3}}}}, {"url": "https://github.com/calcom/cal.com/pull/7232", "language": "TypeScript", "pr_size": "medium", "domain": "scheduling", "change_type": "feature", "complexity": "moderate", "difficulty": "subtle", "risk": "high", "context": "cross_file", "concern": "reliability", "summary": "", "tool_metrics": {"augment": {"strict": {"tp": 1, "fp": 2, "fn": 2}, "core": {"tp": 1, "fp": 2, "fn": 2}, "all": {"tp": 1, "fp": 2, "fn": 2}}, "baz": {"strict": {"tp": 1, "fp": 0, "fn": 2}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 2}}, "bugbot": {"strict": {"tp": 1, "fp": 1, "fn": 2}, "core": {"tp": 1, "fp": 1, "fn": 2}, "all": {"tp": 1, "fp": 1, "fn": 2}}, "claude": {"strict": {"tp": 1, "fp": 3, "fn": 2}, "core": {"tp": 1, "fp": 3, "fn": 2}, "all": {"tp": 1, "fp": 3, "fn": 2}}, "claude-code": {"strict": {"tp": 1, "fp": 1, "fn": 2}, "core": {"tp": 1, "fp": 1, "fn": 2}, "all": {"tp": 1, "fp": 1, "fn": 2}}, "codeant-v2": {"strict": {"tp": 1, "fp": 2, "fn": 2}, "core": {"tp": 1, "fp": 2, "fn": 2}, "all": {"tp": 1, "fp": 2, "fn": 2}}, "coderabbit": {"strict": {"tp": 2, "fp": 7, "fn": 1}, "core": {"tp": 2, "fp": 7, "fn": 1}, "all": {"tp": 2, "fp": 7, "fn": 1}}, "cubic-v2": {"strict": {"tp": 2, "fp": 3, "fn": 1}, "core": {"tp": 2, "fp": 3, "fn": 1}, "all": {"tp": 2, "fp": 3, "fn": 1}}, "devin": {"strict": {"tp": 1, "fp": 1, "fn": 2}, "core": {"tp": 1, "fp": 1, "fn": 2}, "all": {"tp": 1, "fp": 1, "fn": 2}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 3}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 3}}, "kg": {"strict": {"tp": 0, "fp": 1, "fn": 3}, "core": {"tp": 0, "fp": 1, "fn": 3}, "all": {"tp": 0, "fp": 1, "fn": 3}}, "kodus-v2": {"strict": {"tp": 2, "fp": 2, "fn": 1}, "core": {"tp": 2, "fp": 2, "fn": 1}, "all": {"tp": 2, "fp": 2, "fn": 1}}, "macroscope": {"strict": {"tp": 1, "fp": 2, "fn": 2}, "core": {"tp": 1, "fp": 2, "fn": 2}, "all": {"tp": 1, "fp": 2, "fn": 2}}, "qodo-extended-v2": {"strict": {"tp": 2, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 1}, "all": {"tp": 2, "fp": 1, "fn": 1}}, "qodo-v2": {"strict": {"tp": 2, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 1}, "all": {"tp": 2, "fp": 1, "fn": 1}}, "sourcery": {"strict": {"tp": 1, "fp": 2, "fn": 2}, "core": {"tp": 1, "fp": 2, "fn": 2}, "all": {"tp": 1, "fp": 2, "fn": 2}}, "greptile-v4-1": {"strict": {"tp": 1, "fp": 2, "fn": 2}, "core": {"tp": 1, "fp": 2, "fn": 2}, "all": {"tp": 1, "fp": 2, "fn": 2}}, "gemini-v2": {"strict": {"tp": 2, "fp": 3, "fn": 1}, "core": {"tp": 2, "fp": 3, "fn": 1}, "all": {"tp": 2, "fp": 3, "fn": 1}}, "gitlab": {"strict": {"tp": 0, "fp": 2, "fn": 3}, "core": {"tp": 0, "fp": 2, "fn": 3}, "all": {"tp": 0, "fp": 2, "fn": 3}}, "copilot-v2": {"strict": {"tp": 0, "fp": 3, "fn": 3}, "core": {"tp": 0, "fp": 3, "fn": 3}, "all": {"tp": 0, "fp": 3, "fn": 3}}}}, {"url": "https://github.com/calcom/cal.com/pull/14740", "language": "TypeScript", "pr_size": "large", "domain": "scheduling", "change_type": "feature", "complexity": "moderate", "difficulty": "subtle", "risk": "critical", "context": "cross_file", "concern": "security", "summary": "", "tool_metrics": {"augment": {"strict": {"tp": 4, "fp": 3, "fn": 2}, "core": {"tp": 4, "fp": 3, "fn": 2}, "all": {"tp": 4, "fp": 3, "fn": 2}}, "baz": {"strict": {"tp": 1, "fp": 1, "fn": 5}, "core": {"tp": 1, "fp": 1, "fn": 5}, "all": {"tp": 1, "fp": 1, "fn": 5}}, "bugbot": {"strict": {"tp": 3, "fp": 3, "fn": 3}, "core": {"tp": 3, "fp": 3, "fn": 3}, "all": {"tp": 3, "fp": 3, "fn": 3}}, "claude": {"strict": {"tp": 2, "fp": 0, "fn": 4}, "core": {"tp": 2, "fp": 0, "fn": 4}, "all": {"tp": 2, "fp": 0, "fn": 4}}, "claude-code": {"strict": {"tp": 2, "fp": 1, "fn": 4}, "core": {"tp": 2, "fp": 1, "fn": 4}, "all": {"tp": 2, "fp": 1, "fn": 4}}, "codeant-v2": {"strict": {"tp": 3, "fp": 4, "fn": 3}, "core": {"tp": 3, "fp": 4, "fn": 3}, "all": {"tp": 3, "fp": 4, "fn": 3}}, "coderabbit": {"strict": {"tp": 4, "fp": 4, "fn": 2}, "core": {"tp": 4, "fp": 4, "fn": 2}, "all": {"tp": 4, "fp": 4, "fn": 2}}, "cubic-v2": {"strict": {"tp": 3, "fp": 2, "fn": 3}, "core": {"tp": 3, "fp": 2, "fn": 3}, "all": {"tp": 3, "fp": 2, "fn": 3}}, "devin": {"strict": {"tp": 2, "fp": 0, "fn": 4}, "core": {"tp": 2, "fp": 0, "fn": 4}, "all": {"tp": 2, "fp": 0, "fn": 4}}, "graphite": {"strict": {"tp": 1, "fp": 0, "fn": 5}, "core": {"tp": 1, "fp": 0, "fn": 5}, "all": {"tp": 1, "fp": 0, "fn": 5}}, "kg": {"strict": {"tp": 1, "fp": 1, "fn": 5}, "core": {"tp": 1, "fp": 1, "fn": 5}, "all": {"tp": 1, "fp": 1, "fn": 5}}, "kodus-v2": {"strict": {"tp": 2, "fp": 0, "fn": 4}, "core": {"tp": 2, "fp": 0, "fn": 4}, "all": {"tp": 2, "fp": 0, "fn": 4}}, "macroscope": {"strict": {"tp": 3, "fp": 3, "fn": 3}, "core": {"tp": 3, "fp": 3, "fn": 3}, "all": {"tp": 3, "fp": 3, "fn": 3}}, "qodo-extended-v2": {"strict": {"tp": 3, "fp": 2, "fn": 3}, "core": {"tp": 3, "fp": 2, "fn": 3}, "all": {"tp": 3, "fp": 2, "fn": 3}}, "qodo-v2": {"strict": {"tp": 4, "fp": 2, "fn": 2}, "core": {"tp": 4, "fp": 2, "fn": 2}, "all": {"tp": 4, "fp": 2, "fn": 2}}, "sourcery": {"strict": {"tp": 3, "fp": 2, "fn": 3}, "core": {"tp": 3, "fp": 2, "fn": 3}, "all": {"tp": 3, "fp": 2, "fn": 3}}, "greptile-v4-1": {"strict": {"tp": 3, "fp": 2, "fn": 3}, "core": {"tp": 3, "fp": 2, "fn": 3}, "all": {"tp": 3, "fp": 2, "fn": 3}}, "gemini-v2": {"strict": {"tp": 3, "fp": 4, "fn": 3}, "core": {"tp": 3, "fp": 4, "fn": 3}, "all": {"tp": 3, "fp": 4, "fn": 3}}, "gitlab": {"strict": {"tp": 2, "fp": 5, "fn": 4}, "core": {"tp": 2, "fp": 5, "fn": 4}, "all": {"tp": 2, "fp": 5, "fn": 4}}, "copilot-v2": {"strict": {"tp": 5, "fp": 5, "fn": 1}, "core": {"tp": 5, "fp": 5, "fn": 1}, "all": {"tp": 5, "fp": 5, "fn": 1}}}}, {"url": "https://github.com/calcom/cal.com/pull/10600", "language": "TypeScript", "pr_size": "medium", "domain": "authentication", "change_type": "feature", "complexity": "moderate", "difficulty": "subtle", "risk": "high", "context": "cross_file", "concern": "security", "summary": "", "tool_metrics": {"augment": {"strict": {"tp": 0, "fp": 3, "fn": 2}, "core": {"tp": 1, "fp": 3, "fn": 3}, "all": {"tp": 2, "fp": 3, "fn": 3}}, "baz": {"strict": {"tp": 0, "fp": 1, "fn": 2}, "core": {"tp": 0, "fp": 1, "fn": 4}, "all": {"tp": 0, "fp": 1, "fn": 5}}, "bugbot": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 3}, "all": {"tp": 1, "fp": 0, "fn": 4}}, "claude": {"strict": {"tp": 0, "fp": 6, "fn": 2}, "core": {"tp": 1, "fp": 6, "fn": 3}, "all": {"tp": 2, "fp": 6, "fn": 3}}, "claude-code": {"strict": {"tp": 0, "fp": 4, "fn": 2}, "core": {"tp": 0, "fp": 4, "fn": 4}, "all": {"tp": 1, "fp": 4, "fn": 4}}, "codeant-v2": {"strict": {"tp": 1, "fp": 2, "fn": 1}, "core": {"tp": 1, "fp": 2, "fn": 3}, "all": {"tp": 1, "fp": 2, "fn": 4}}, "coderabbit": {"strict": {"tp": 1, "fp": 6, "fn": 1}, "core": {"tp": 1, "fp": 6, "fn": 3}, "all": {"tp": 2, "fp": 6, "fn": 3}}, "cubic-v2": {"strict": {"tp": 2, "fp": 2, "fn": 0}, "core": {"tp": 2, "fp": 2, "fn": 2}, "all": {"tp": 2, "fp": 2, "fn": 3}}, "devin": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 3}, "all": {"tp": 1, "fp": 0, "fn": 4}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 4}, "all": {"tp": 0, "fp": 0, "fn": 5}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 4}, "all": {"tp": 0, "fp": 0, "fn": 5}}, "kodus-v2": {"strict": {"tp": 0, "fp": 1, "fn": 2}, "core": {"tp": 0, "fp": 1, "fn": 4}, "all": {"tp": 0, "fp": 1, "fn": 5}}, "macroscope": {"strict": {"tp": 0, "fp": 2, "fn": 2}, "core": {"tp": 0, "fp": 2, "fn": 4}, "all": {"tp": 0, "fp": 2, "fn": 5}}, "qodo-extended-v2": {"strict": {"tp": 1, "fp": 2, "fn": 1}, "core": {"tp": 2, "fp": 2, "fn": 2}, "all": {"tp": 2, "fp": 2, "fn": 3}}, "qodo-v2": {"strict": {"tp": 1, "fp": 3, "fn": 1}, "core": {"tp": 2, "fp": 3, "fn": 2}, "all": {"tp": 2, "fp": 3, "fn": 3}}, "sourcery": {"strict": {"tp": 1, "fp": 2, "fn": 1}, "core": {"tp": 2, "fp": 2, "fn": 2}, "all": {"tp": 3, "fp": 2, "fn": 2}}, "greptile-v4-1": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 2}, "all": {"tp": 3, "fp": 1, "fn": 2}}, "gemini-v2": {"strict": {"tp": 0, "fp": 5, "fn": 2}, "core": {"tp": 0, "fp": 5, "fn": 4}, "all": {"tp": 1, "fp": 5, "fn": 4}}, "gitlab": {"strict": {"tp": 0, "fp": 2, "fn": 2}, "core": {"tp": 1, "fp": 2, "fn": 3}, "all": {"tp": 2, "fp": 2, "fn": 3}}, "copilot-v2": {"strict": {"tp": 2, "fp": 4, "fn": 0}, "core": {"tp": 4, "fp": 4, "fn": 0}, "all": {"tp": 5, "fp": 4, "fn": 0}}}}, {"url": "https://github.com/calcom/cal.com/pull/10967", "language": "TypeScript", "pr_size": "large", "domain": "scheduling", "change_type": "bug_fix", "complexity": "complex", "difficulty": "subtle", "risk": "high", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"augment": {"strict": {"tp": 3, "fp": 2, "fn": 2}, "core": {"tp": 3, "fp": 2, "fn": 2}, "all": {"tp": 3, "fp": 2, "fn": 3}}, "baz": {"strict": {"tp": 1, "fp": 1, "fn": 4}, "core": {"tp": 1, "fp": 1, "fn": 4}, "all": {"tp": 1, "fp": 1, "fn": 5}}, "bugbot": {"strict": {"tp": 2, "fp": 4, "fn": 3}, "core": {"tp": 2, "fp": 4, "fn": 3}, "all": {"tp": 2, "fp": 4, "fn": 4}}, "claude": {"strict": {"tp": 4, "fp": 3, "fn": 1}, "core": {"tp": 4, "fp": 3, "fn": 1}, "all": {"tp": 4, "fp": 3, "fn": 2}}, "claude-code": {"strict": {"tp": 3, "fp": 5, "fn": 2}, "core": {"tp": 3, "fp": 5, "fn": 2}, "all": {"tp": 3, "fp": 5, "fn": 3}}, "codeant-v2": {"strict": {"tp": 1, "fp": 7, "fn": 4}, "core": {"tp": 1, "fp": 7, "fn": 4}, "all": {"tp": 1, "fp": 7, "fn": 5}}, "coderabbit": {"strict": {"tp": 5, "fp": 5, "fn": 0}, "core": {"tp": 5, "fp": 5, "fn": 0}, "all": {"tp": 5, "fp": 5, "fn": 1}}, "cubic-v2": {"strict": {"tp": 1, "fp": 2, "fn": 4}, "core": {"tp": 1, "fp": 2, "fn": 4}, "all": {"tp": 1, "fp": 2, "fn": 5}}, "devin": {"strict": {"tp": 3, "fp": 1, "fn": 2}, "core": {"tp": 3, "fp": 1, "fn": 2}, "all": {"tp": 3, "fp": 1, "fn": 3}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 5}, "core": {"tp": 0, "fp": 0, "fn": 5}, "all": {"tp": 0, "fp": 0, "fn": 6}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 5}, "core": {"tp": 0, "fp": 0, "fn": 5}, "all": {"tp": 0, "fp": 0, "fn": 6}}, "kodus-v2": {"strict": {"tp": 3, "fp": 1, "fn": 2}, "core": {"tp": 3, "fp": 1, "fn": 2}, "all": {"tp": 3, "fp": 1, "fn": 3}}, "macroscope": {"strict": {"tp": 3, "fp": 2, "fn": 2}, "core": {"tp": 3, "fp": 2, "fn": 2}, "all": {"tp": 3, "fp": 2, "fn": 3}}, "qodo-extended-v2": {"strict": {"tp": 1, "fp": 4, "fn": 4}, "core": {"tp": 1, "fp": 4, "fn": 4}, "all": {"tp": 1, "fp": 4, "fn": 5}}, "qodo-v2": {"strict": {"tp": 2, "fp": 3, "fn": 3}, "core": {"tp": 2, "fp": 3, "fn": 3}, "all": {"tp": 2, "fp": 3, "fn": 4}}, "sourcery": {"strict": {"tp": 2, "fp": 5, "fn": 3}, "core": {"tp": 2, "fp": 5, "fn": 3}, "all": {"tp": 2, "fp": 5, "fn": 4}}, "greptile-v4-1": {"strict": {"tp": 1, "fp": 2, "fn": 4}, "core": {"tp": 1, "fp": 2, "fn": 4}, "all": {"tp": 1, "fp": 2, "fn": 5}}, "gemini-v2": {"strict": {"tp": 1, "fp": 6, "fn": 4}, "core": {"tp": 1, "fp": 6, "fn": 4}, "all": {"tp": 1, "fp": 6, "fn": 5}}, "gitlab": {"strict": {"tp": 3, "fp": 3, "fn": 2}, "core": {"tp": 3, "fp": 3, "fn": 2}, "all": {"tp": 3, "fp": 3, "fn": 3}}, "copilot-v2": {"strict": {"tp": 4, "fp": 7, "fn": 1}, "core": {"tp": 4, "fp": 7, "fn": 1}, "all": {"tp": 4, "fp": 7, "fn": 2}}}}, {"url": "https://github.com/calcom/cal.com/pull/8087", "language": "TypeScript", "pr_size": "medium", "domain": "concurrency", "change_type": "refactoring", "complexity": "moderate", "difficulty": "subtle", "risk": "critical", "context": "cross_file", "concern": "reliability", "summary": "", "tool_metrics": {"augment": {"strict": {"tp": 1, "fp": 6, "fn": 0}, "core": {"tp": 1, "fp": 6, "fn": 0}, "all": {"tp": 1, "fp": 6, "fn": 1}}, "baz": {"strict": {"tp": 1, "fp": 2, "fn": 0}, "core": {"tp": 1, "fp": 2, "fn": 0}, "all": {"tp": 1, "fp": 2, "fn": 1}}, "bugbot": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 0}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "claude": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 1, "fp": 1, "fn": 0}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "claude-code": {"strict": {"tp": 1, "fp": 6, "fn": 0}, "core": {"tp": 1, "fp": 6, "fn": 0}, "all": {"tp": 1, "fp": 6, "fn": 1}}, "codeant-v2": {"strict": {"tp": 1, "fp": 4, "fn": 0}, "core": {"tp": 1, "fp": 4, "fn": 0}, "all": {"tp": 1, "fp": 4, "fn": 1}}, "coderabbit": {"strict": {"tp": 1, "fp": 5, "fn": 0}, "core": {"tp": 1, "fp": 5, "fn": 0}, "all": {"tp": 1, "fp": 5, "fn": 1}}, "cubic-v2": {"strict": {"tp": 1, "fp": 9, "fn": 0}, "core": {"tp": 1, "fp": 9, "fn": 0}, "all": {"tp": 2, "fp": 9, "fn": 0}}, "devin": {"strict": {"tp": 1, "fp": 3, "fn": 0}, "core": {"tp": 1, "fp": 3, "fn": 0}, "all": {"tp": 2, "fp": 3, "fn": 0}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "kg": {"strict": {"tp": 1, "fp": 2, "fn": 0}, "core": {"tp": 1, "fp": 2, "fn": 0}, "all": {"tp": 1, "fp": 2, "fn": 1}}, "kodus-v2": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "macroscope": {"strict": {"tp": 1, "fp": 4, "fn": 0}, "core": {"tp": 1, "fp": 4, "fn": 0}, "all": {"tp": 1, "fp": 4, "fn": 1}}, "qodo-extended-v2": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 1, "fp": 1, "fn": 0}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "qodo-v2": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 0}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "sourcery": {"strict": {"tp": 1, "fp": 3, "fn": 0}, "core": {"tp": 1, "fp": 3, "fn": 0}, "all": {"tp": 1, "fp": 3, "fn": 1}}, "greptile-v4-1": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 1, "fp": 1, "fn": 0}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "gemini-v2": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 0}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "gitlab": {"strict": {"tp": 1, "fp": 2, "fn": 0}, "core": {"tp": 1, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "copilot-v2": {"strict": {"tp": 1, "fp": 2, "fn": 0}, "core": {"tp": 1, "fp": 2, "fn": 0}, "all": {"tp": 1, "fp": 2, "fn": 1}}}}], "tools": ["augment", "baz", "bugbot", "claude", "claude-code", "codeant-v2", "coderabbit", "copilot-v2", "cubic-v2", "devin", "gemini-v2", "gitlab", "graphite", "greptile-v4-1", "kg", "kodus-v2", "macroscope", "qodo-extended-v2", "qodo-v2", "sourcery"], "dimensions": {"language": ["Go", "Java", "Python", "Ruby", "TypeScript"], "pr_size": ["small", "medium", "large"], "domain": ["API", "UI", "authentication", "caching", "concurrency", "configuration", "data_processing", "database", "file_io", "logging", "scheduling", "serialization", "testing"], "change_type": ["bug_fix", "feature", "migration", "performance", "refactoring", "security_patch"], "complexity": ["simple", "moderate", "complex"], "difficulty": ["obvious", "moderate", "subtle", "very_subtle"], "risk": ["low", "medium", "high", "critical"], "context": ["local", "file", "cross_file", "system"], "concern": ["correctness", "maintainability", "reliability", "security"]}, "overall_metrics": {"augment": {"strict": {"precision": 49.4, "recall": 61.9, "f1": 55.0, "f2": 58.9, "tp": 86, "fp": 88, "fn": 53, "num_prs": 50}, "core": {"precision": 52.4, "recall": 61.4, "f1": 56.6, "f2": 59.4, "tp": 97, "fp": 88, "fn": 61, "num_prs": 50}, "all": {"precision": 53.7, "recall": 59.0, "f1": 56.2, "f2": 57.8, "tp": 102, "fp": 88, "fn": 71, "num_prs": 50}}, "cubic-v2": {"strict": {"precision": 52.9, "recall": 59.7, "f1": 56.1, "f2": 58.2, "tp": 83, "fp": 74, "fn": 56, "num_prs": 50}, "core": {"precision": 56.5, "recall": 60.8, "f1": 58.5, "f2": 59.9, "tp": 96, "fp": 74, "fn": 62, "num_prs": 50}, "all": {"precision": 57.2, "recall": 57.2, "f1": 57.2, "f2": 57.2, "tp": 99, "fp": 74, "fn": 74, "num_prs": 50}}, "copilot-v2": {"strict": {"precision": 31.4, "recall": 61.9, "f1": 41.6, "f2": 51.8, "tp": 86, "fp": 188, "fn": 53, "num_prs": 50}, "core": {"precision": 34.3, "recall": 62.0, "f1": 44.1, "f2": 53.4, "tp": 98, "fp": 188, "fn": 60, "num_prs": 50}, "all": {"precision": 36.1, "recall": 61.3, "f1": 45.4, "f2": 53.8, "tp": 106, "fp": 188, "fn": 67, "num_prs": 50}}, "greptile-v4-1": {"strict": {"precision": 44.9, "recall": 50.4, "f1": 47.5, "f2": 49.2, "tp": 70, "fp": 86, "fn": 69, "num_prs": 50}, "core": {"precision": 46.9, "recall": 48.1, "f1": 47.5, "f2": 47.9, "tp": 76, "fp": 86, "fn": 82, "num_prs": 50}, "all": {"precision": 49.1, "recall": 48.0, "f1": 48.5, "f2": 48.2, "tp": 83, "fp": 86, "fn": 90, "num_prs": 50}}, "sourcery": {"strict": {"precision": 34.2, "recall": 48.9, "f1": 40.2, "f2": 45.0, "tp": 68, "fp": 131, "fn": 71, "num_prs": 50}, "core": {"precision": 37.0, "recall": 48.7, "f1": 42.1, "f2": 45.8, "tp": 77, "fp": 131, "fn": 81, "num_prs": 50}, "all": {"precision": 38.8, "recall": 48.0, "f1": 42.9, "f2": 45.8, "tp": 83, "fp": 131, "fn": 90, "num_prs": 50}}, "qodo-v2": {"strict": {"precision": 45.1, "recall": 59.0, "f1": 51.1, "f2": 55.6, "tp": 82, "fp": 100, "fn": 57, "num_prs": 50}, "core": {"precision": 47.1, "recall": 56.3, "f1": 51.3, "f2": 54.2, "tp": 89, "fp": 100, "fn": 69, "num_prs": 50}, "all": {"precision": 47.9, "recall": 53.2, "f1": 50.4, "f2": 52.0, "tp": 92, "fp": 100, "fn": 81, "num_prs": 50}}, "gitlab": {"strict": {"precision": 44.7, "recall": 48.9, "f1": 46.7, "f2": 48.0, "tp": 68, "fp": 84, "fn": 71, "num_prs": 50}, "core": {"precision": 47.5, "recall": 48.1, "f1": 47.8, "f2": 48.0, "tp": 76, "fp": 84, "fn": 82, "num_prs": 50}, "all": {"precision": 49.7, "recall": 48.0, "f1": 48.8, "f2": 48.3, "tp": 83, "fp": 84, "fn": 90, "num_prs": 50}}, "codeant-v2": {"strict": {"precision": 31.1, "recall": 37.4, "f1": 34.0, "f2": 36.0, "tp": 52, "fp": 115, "fn": 87, "num_prs": 50}, "core": {"precision": 32.4, "recall": 34.8, "f1": 33.5, "f2": 34.3, "tp": 55, "fp": 115, "fn": 103, "num_prs": 50}, "all": {"precision": 33.1, "recall": 32.9, "f1": 33.0, "f2": 33.0, "tp": 57, "fp": 115, "fn": 116, "num_prs": 50}}, "bugbot": {"strict": {"precision": 52.0, "recall": 46.0, "f1": 48.9, "f2": 47.1, "tp": 64, "fp": 59, "fn": 75, "num_prs": 50}, "core": {"precision": 54.3, "recall": 44.3, "f1": 48.8, "f2": 46.0, "tp": 70, "fp": 59, "fn": 88, "num_prs": 50}, "all": {"precision": 55.3, "recall": 42.2, "f1": 47.9, "f2": 44.3, "tp": 73, "fp": 59, "fn": 100, "num_prs": 50}}, "claude-code": {"strict": {"precision": 34.3, "recall": 42.4, "f1": 37.9, "f2": 40.5, "tp": 59, "fp": 113, "fn": 80, "num_prs": 50}, "core": {"precision": 36.5, "recall": 41.1, "f1": 38.7, "f2": 40.1, "tp": 65, "fp": 113, "fn": 93, "num_prs": 50}, "all": {"precision": 38.3, "recall": 40.5, "f1": 39.3, "f2": 40.0, "tp": 70, "fp": 113, "fn": 103, "num_prs": 50}}, "devin": {"strict": {"precision": 66.0, "recall": 44.6, "f1": 53.2, "f2": 47.7, "tp": 62, "fp": 32, "fn": 77, "num_prs": 50}, "core": {"precision": 67.3, "recall": 41.8, "f1": 51.6, "f2": 45.2, "tp": 66, "fp": 32, "fn": 92, "num_prs": 50}, "all": {"precision": 68.3, "recall": 39.9, "f1": 50.4, "f2": 43.5, "tp": 69, "fp": 32, "fn": 104, "num_prs": 50}}, "macroscope": {"strict": {"precision": 50.4, "recall": 45.3, "f1": 47.7, "f2": 46.3, "tp": 63, "fp": 62, "fn": 76, "num_prs": 50}, "core": {"precision": 51.6, "recall": 41.8, "f1": 46.2, "f2": 43.4, "tp": 66, "fp": 62, "fn": 92, "num_prs": 50}, "all": {"precision": 53.0, "recall": 40.5, "f1": 45.9, "f2": 42.5, "tp": 70, "fp": 62, "fn": 103, "num_prs": 50}}, "claude": {"strict": {"precision": 36.1, "recall": 37.4, "f1": 36.7, "f2": 37.1, "tp": 52, "fp": 92, "fn": 87, "num_prs": 50}, "core": {"precision": 38.3, "recall": 36.1, "f1": 37.1, "f2": 36.5, "tp": 57, "fp": 92, "fn": 101, "num_prs": 50}, "all": {"precision": 40.6, "recall": 36.4, "f1": 38.4, "f2": 37.2, "tp": 63, "fp": 92, "fn": 110, "num_prs": 50}}, "gemini-v2": {"strict": {"precision": 36.7, "recall": 51.8, "f1": 43.0, "f2": 47.9, "tp": 72, "fp": 124, "fn": 67, "num_prs": 50}, "core": {"precision": 39.2, "recall": 50.6, "f1": 44.2, "f2": 47.8, "tp": 80, "fp": 124, "fn": 78, "num_prs": 50}, "all": {"precision": 41.2, "recall": 50.3, "f1": 45.3, "f2": 48.2, "tp": 87, "fp": 124, "fn": 86, "num_prs": 50}}, "coderabbit": {"strict": {"precision": 31.8, "recall": 60.4, "f1": 41.7, "f2": 51.2, "tp": 84, "fp": 180, "fn": 55, "num_prs": 50}, "core": {"precision": 33.8, "recall": 58.2, "f1": 42.8, "f2": 50.9, "tp": 92, "fp": 180, "fn": 66, "num_prs": 50}, "all": {"precision": 35.3, "recall": 56.6, "f1": 43.5, "f2": 50.5, "tp": 98, "fp": 180, "fn": 75, "num_prs": 50}}, "baz": {"strict": {"precision": 54.6, "recall": 38.1, "f1": 44.9, "f2": 40.6, "tp": 53, "fp": 44, "fn": 86, "num_prs": 50}, "core": {"precision": 55.6, "recall": 34.8, "f1": 42.8, "f2": 37.6, "tp": 55, "fp": 44, "fn": 103, "num_prs": 50}, "all": {"precision": 56.9, "recall": 33.5, "f1": 42.2, "f2": 36.5, "tp": 58, "fp": 44, "fn": 115, "num_prs": 50}}, "kg": {"strict": {"precision": 48.8, "recall": 15.1, "f1": 23.1, "f2": 17.5, "tp": 21, "fp": 22, "fn": 118, "num_prs": 50}, "core": {"precision": 50.0, "recall": 13.9, "f1": 21.8, "f2": 16.3, "tp": 22, "fp": 22, "fn": 136, "num_prs": 50}, "all": {"precision": 51.1, "recall": 13.3, "f1": 21.1, "f2": 15.6, "tp": 23, "fp": 22, "fn": 150, "num_prs": 50}}, "qodo-extended-v2": {"strict": {"precision": 53.6, "recall": 58.3, "f1": 55.9, "f2": 57.3, "tp": 81, "fp": 70, "fn": 58, "num_prs": 50}, "core": {"precision": 56.2, "recall": 57.0, "f1": 56.6, "f2": 56.8, "tp": 90, "fp": 70, "fn": 68, "num_prs": 50}, "all": {"precision": 56.8, "recall": 53.2, "f1": 54.9, "f2": 53.9, "tp": 92, "fp": 70, "fn": 81, "num_prs": 50}}, "kodus-v2": {"strict": {"precision": 55.2, "recall": 38.1, "f1": 45.1, "f2": 40.6, "tp": 53, "fp": 43, "fn": 86, "num_prs": 50}, "core": {"precision": 57.4, "recall": 36.7, "f1": 44.8, "f2": 39.6, "tp": 58, "fp": 43, "fn": 100, "num_prs": 50}, "all": {"precision": 57.8, "recall": 34.1, "f1": 42.9, "f2": 37.2, "tp": 59, "fp": 43, "fn": 114, "num_prs": 50}}, "graphite": {"strict": {"precision": 100.0, "recall": 7.9, "f1": 14.7, "f2": 9.7, "tp": 11, "fp": 0, "fn": 128, "num_prs": 50}, "core": {"precision": 100.0, "recall": 7.6, "f1": 14.1, "f2": 9.3, "tp": 12, "fp": 0, "fn": 146, "num_prs": 50}, "all": {"precision": 100.0, "recall": 6.9, "f1": 13.0, "f2": 8.5, "tp": 12, "fp": 0, "fn": 161, "num_prs": 50}}}}, "openai_gpt-5.2": {"prs": [{"url": "https://github.com/keycloak/keycloak/pull/37429", "language": "Java", "pr_size": "medium", "domain": "UI", "change_type": "feature", "complexity": "moderate", "difficulty": "moderate", "risk": "medium", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 4}}, "codeant-v2": {"strict": {"tp": 0, "fp": 2, "fn": 1}, "core": {"tp": 0, "fp": 2, "fn": 3}, "all": {"tp": 0, "fp": 2, "fn": 5}}, "coderabbit": {"strict": {"tp": 0, "fp": 6, "fn": 1}, "core": {"tp": 2, "fp": 6, "fn": 1}, "all": {"tp": 4, "fp": 6, "fn": 1}}, "qodo-extended-v2": {"strict": {"tp": 0, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 1}, "all": {"tp": 2, "fp": 1, "fn": 3}}, "greptile-v4-1": {"strict": {"tp": 0, "fp": 3, "fn": 1}, "core": {"tp": 0, "fp": 3, "fn": 3}, "all": {"tp": 2, "fp": 3, "fn": 3}}, "gemini-v2": {"strict": {"tp": 0, "fp": 5, "fn": 1}, "core": {"tp": 1, "fp": 5, "fn": 2}, "all": {"tp": 3, "fp": 5, "fn": 2}}, "gitlab": {"strict": {"tp": 0, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 2}, "all": {"tp": 2, "fp": 1, "fn": 3}}, "copilot-v2": {"strict": {"tp": 0, "fp": 6, "fn": 1}, "core": {"tp": 2, "fp": 6, "fn": 1}, "all": {"tp": 3, "fp": 6, "fn": 2}}, "augment": {"strict": {"tp": 0, "fp": 3, "fn": 1}, "core": {"tp": 2, "fp": 3, "fn": 1}, "all": {"tp": 2, "fp": 3, "fn": 3}}, "bugbot": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 2, "fp": 0, "fn": 1}, "all": {"tp": 2, "fp": 0, "fn": 3}}, "claude": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 1, "fp": 0, "fn": 4}}, "claude-code": {"strict": {"tp": 0, "fp": 4, "fn": 1}, "core": {"tp": 0, "fp": 4, "fn": 3}, "all": {"tp": 2, "fp": 4, "fn": 3}}, "cubic-v2": {"strict": {"tp": 0, "fp": 2, "fn": 1}, "core": {"tp": 1, "fp": 2, "fn": 2}, "all": {"tp": 1, "fp": 2, "fn": 4}}, "devin": {"strict": {"tp": 0, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 1}, "all": {"tp": 2, "fp": 1, "fn": 3}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 5}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 4}}, "kodus-v2": {"strict": {"tp": 0, "fp": 3, "fn": 1}, "core": {"tp": 1, "fp": 3, "fn": 2}, "all": {"tp": 1, "fp": 3, "fn": 4}}, "macroscope": {"strict": {"tp": 0, "fp": 2, "fn": 1}, "core": {"tp": 0, "fp": 2, "fn": 3}, "all": {"tp": 0, "fp": 2, "fn": 5}}, "qodo-v2": {"strict": {"tp": 0, "fp": 3, "fn": 1}, "core": {"tp": 0, "fp": 3, "fn": 3}, "all": {"tp": 0, "fp": 3, "fn": 5}}, "sourcery": {"strict": {"tp": 0, "fp": 13, "fn": 1}, "core": {"tp": 0, "fp": 13, "fn": 3}, "all": {"tp": 2, "fp": 13, "fn": 3}}}}, {"url": "https://github.com/keycloak/keycloak/pull/37634", "language": "Java", "pr_size": "medium", "domain": "authentication", "change_type": "feature", "complexity": "moderate", "difficulty": "subtle", "risk": "high", "context": "file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 2, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 2}, "all": {"tp": 2, "fp": 1, "fn": 2}}, "codeant-v2": {"strict": {"tp": 2, "fp": 5, "fn": 1}, "core": {"tp": 2, "fp": 5, "fn": 2}, "all": {"tp": 2, "fp": 5, "fn": 2}}, "coderabbit": {"strict": {"tp": 3, "fp": 7, "fn": 0}, "core": {"tp": 3, "fp": 7, "fn": 1}, "all": {"tp": 3, "fp": 7, "fn": 1}}, "qodo-extended-v2": {"strict": {"tp": 2, "fp": 0, "fn": 1}, "core": {"tp": 3, "fp": 0, "fn": 1}, "all": {"tp": 3, "fp": 0, "fn": 1}}, "greptile-v4-1": {"strict": {"tp": 2, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 2}, "all": {"tp": 2, "fp": 1, "fn": 2}}, "gemini-v2": {"strict": {"tp": 2, "fp": 2, "fn": 1}, "core": {"tp": 2, "fp": 2, "fn": 2}, "all": {"tp": 2, "fp": 2, "fn": 2}}, "gitlab": {"strict": {"tp": 1, "fp": 1, "fn": 2}, "core": {"tp": 2, "fp": 1, "fn": 2}, "all": {"tp": 2, "fp": 1, "fn": 2}}, "copilot-v2": {"strict": {"tp": 2, "fp": 3, "fn": 1}, "core": {"tp": 3, "fp": 3, "fn": 1}, "all": {"tp": 3, "fp": 3, "fn": 1}}, "augment": {"strict": {"tp": 1, "fp": 5, "fn": 2}, "core": {"tp": 1, "fp": 5, "fn": 3}, "all": {"tp": 1, "fp": 5, "fn": 3}}, "bugbot": {"strict": {"tp": 2, "fp": 2, "fn": 1}, "core": {"tp": 2, "fp": 2, "fn": 2}, "all": {"tp": 2, "fp": 2, "fn": 2}}, "claude": {"strict": {"tp": 1, "fp": 2, "fn": 2}, "core": {"tp": 2, "fp": 2, "fn": 2}, "all": {"tp": 2, "fp": 2, "fn": 2}}, "claude-code": {"strict": {"tp": 0, "fp": 2, "fn": 3}, "core": {"tp": 0, "fp": 2, "fn": 4}, "all": {"tp": 0, "fp": 2, "fn": 4}}, "cubic-v2": {"strict": {"tp": 2, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 2}, "all": {"tp": 2, "fp": 1, "fn": 2}}, "devin": {"strict": {"tp": 2, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 2}, "all": {"tp": 2, "fp": 1, "fn": 2}}, "graphite": {"strict": {"tp": 1, "fp": 1, "fn": 2}, "core": {"tp": 1, "fp": 1, "fn": 3}, "all": {"tp": 1, "fp": 1, "fn": 3}}, "kg": {"strict": {"tp": 1, "fp": 2, "fn": 2}, "core": {"tp": 1, "fp": 2, "fn": 3}, "all": {"tp": 1, "fp": 2, "fn": 3}}, "kodus-v2": {"strict": {"tp": 2, "fp": 0, "fn": 1}, "core": {"tp": 2, "fp": 0, "fn": 2}, "all": {"tp": 2, "fp": 0, "fn": 2}}, "macroscope": {"strict": {"tp": 2, "fp": 2, "fn": 1}, "core": {"tp": 2, "fp": 2, "fn": 2}, "all": {"tp": 2, "fp": 2, "fn": 2}}, "qodo-v2": {"strict": {"tp": 2, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 2}, "all": {"tp": 2, "fp": 1, "fn": 2}}, "sourcery": {"strict": {"tp": 2, "fp": 0, "fn": 1}, "core": {"tp": 2, "fp": 0, "fn": 2}, "all": {"tp": 2, "fp": 0, "fn": 2}}}}, {"url": "https://github.com/keycloak/keycloak/pull/38446", "language": "Java", "pr_size": "medium", "domain": "authentication", "change_type": "feature", "complexity": "moderate", "difficulty": "moderate", "risk": "medium", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 0, "fp": 1, "fn": 2}, "core": {"tp": 0, "fp": 1, "fn": 2}, "all": {"tp": 0, "fp": 1, "fn": 2}}, "codeant-v2": {"strict": {"tp": 0, "fp": 1, "fn": 2}, "core": {"tp": 0, "fp": 1, "fn": 2}, "all": {"tp": 0, "fp": 1, "fn": 2}}, "coderabbit": {"strict": {"tp": 2, "fp": 3, "fn": 0}, "core": {"tp": 2, "fp": 3, "fn": 0}, "all": {"tp": 2, "fp": 3, "fn": 0}}, "qodo-extended-v2": {"strict": {"tp": 1, "fp": 3, "fn": 1}, "core": {"tp": 1, "fp": 3, "fn": 1}, "all": {"tp": 1, "fp": 3, "fn": 1}}, "greptile-v4-1": {"strict": {"tp": 1, "fp": 2, "fn": 1}, "core": {"tp": 1, "fp": 2, "fn": 1}, "all": {"tp": 1, "fp": 2, "fn": 1}}, "gemini-v2": {"strict": {"tp": 0, "fp": 4, "fn": 2}, "core": {"tp": 0, "fp": 4, "fn": 2}, "all": {"tp": 0, "fp": 4, "fn": 2}}, "gitlab": {"strict": {"tp": 0, "fp": 8, "fn": 2}, "core": {"tp": 0, "fp": 8, "fn": 2}, "all": {"tp": 0, "fp": 8, "fn": 2}}, "copilot-v2": {"strict": {"tp": 1, "fp": 8, "fn": 1}, "core": {"tp": 1, "fp": 8, "fn": 1}, "all": {"tp": 1, "fp": 8, "fn": 1}}, "augment": {"strict": {"tp": 1, "fp": 3, "fn": 1}, "core": {"tp": 1, "fp": 3, "fn": 1}, "all": {"tp": 1, "fp": 3, "fn": 1}}, "bugbot": {"strict": {"tp": 0, "fp": 3, "fn": 2}, "core": {"tp": 0, "fp": 3, "fn": 2}, "all": {"tp": 0, "fp": 3, "fn": 2}}, "claude": {"strict": {"tp": 1, "fp": 3, "fn": 1}, "core": {"tp": 1, "fp": 3, "fn": 1}, "all": {"tp": 1, "fp": 3, "fn": 1}}, "claude-code": {"strict": {"tp": 1, "fp": 5, "fn": 1}, "core": {"tp": 1, "fp": 5, "fn": 1}, "all": {"tp": 1, "fp": 5, "fn": 1}}, "cubic-v2": {"strict": {"tp": 1, "fp": 3, "fn": 1}, "core": {"tp": 1, "fp": 3, "fn": 1}, "all": {"tp": 1, "fp": 3, "fn": 1}}, "devin": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "kodus-v2": {"strict": {"tp": 0, "fp": 2, "fn": 2}, "core": {"tp": 0, "fp": 2, "fn": 2}, "all": {"tp": 0, "fp": 2, "fn": 2}}, "macroscope": {"strict": {"tp": 0, "fp": 4, "fn": 2}, "core": {"tp": 0, "fp": 4, "fn": 2}, "all": {"tp": 0, "fp": 4, "fn": 2}}, "qodo-v2": {"strict": {"tp": 2, "fp": 3, "fn": 0}, "core": {"tp": 2, "fp": 3, "fn": 0}, "all": {"tp": 2, "fp": 3, "fn": 0}}, "sourcery": {"strict": {"tp": 0, "fp": 7, "fn": 2}, "core": {"tp": 0, "fp": 7, "fn": 2}, "all": {"tp": 0, "fp": 7, "fn": 2}}}}, {"url": "https://github.com/keycloak/keycloak/pull/36882", "language": "Java", "pr_size": "small", "domain": "configuration", "change_type": "feature", "complexity": "moderate", "difficulty": "moderate", "risk": "medium", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 0, "fp": 1, "fn": 1}, "core": {"tp": 0, "fp": 1, "fn": 2}, "all": {"tp": 0, "fp": 1, "fn": 2}}, "codeant-v2": {"strict": {"tp": 0, "fp": 2, "fn": 1}, "core": {"tp": 0, "fp": 2, "fn": 2}, "all": {"tp": 0, "fp": 2, "fn": 2}}, "coderabbit": {"strict": {"tp": 0, "fp": 2, "fn": 1}, "core": {"tp": 0, "fp": 2, "fn": 2}, "all": {"tp": 0, "fp": 2, "fn": 2}}, "qodo-extended-v2": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "greptile-v4-1": {"strict": {"tp": 0, "fp": 2, "fn": 1}, "core": {"tp": 0, "fp": 2, "fn": 2}, "all": {"tp": 0, "fp": 2, "fn": 2}}, "gemini-v2": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "gitlab": {"strict": {"tp": 0, "fp": 2, "fn": 1}, "core": {"tp": 0, "fp": 2, "fn": 2}, "all": {"tp": 0, "fp": 2, "fn": 2}}, "copilot-v2": {"strict": {"tp": 0, "fp": 3, "fn": 1}, "core": {"tp": 0, "fp": 3, "fn": 2}, "all": {"tp": 0, "fp": 3, "fn": 2}}, "augment": {"strict": {"tp": 0, "fp": 1, "fn": 1}, "core": {"tp": 0, "fp": 1, "fn": 2}, "all": {"tp": 0, "fp": 1, "fn": 2}}, "bugbot": {"strict": {"tp": 0, "fp": 1, "fn": 1}, "core": {"tp": 0, "fp": 1, "fn": 2}, "all": {"tp": 0, "fp": 1, "fn": 2}}, "claude": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "claude-code": {"strict": {"tp": 0, "fp": 1, "fn": 1}, "core": {"tp": 0, "fp": 1, "fn": 2}, "all": {"tp": 0, "fp": 1, "fn": 2}}, "cubic-v2": {"strict": {"tp": 1, "fp": 2, "fn": 0}, "core": {"tp": 1, "fp": 2, "fn": 1}, "all": {"tp": 1, "fp": 2, "fn": 1}}, "devin": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "kodus-v2": {"strict": {"tp": 0, "fp": 2, "fn": 1}, "core": {"tp": 0, "fp": 2, "fn": 2}, "all": {"tp": 0, "fp": 2, "fn": 2}}, "macroscope": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "qodo-v2": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "sourcery": {"strict": {"tp": 0, "fp": 4, "fn": 1}, "core": {"tp": 0, "fp": 4, "fn": 2}, "all": {"tp": 0, "fp": 4, "fn": 2}}}}, {"url": "https://github.com/keycloak/keycloak/pull/36880", "language": "Java", "pr_size": "medium", "domain": "authentication", "change_type": "feature", "complexity": "complex", "difficulty": "very_subtle", "risk": "high", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 2, "fp": 1, "fn": 2}, "core": {"tp": 2, "fp": 1, "fn": 2}, "all": {"tp": 2, "fp": 1, "fn": 3}}, "codeant-v2": {"strict": {"tp": 2, "fp": 0, "fn": 2}, "core": {"tp": 2, "fp": 0, "fn": 2}, "all": {"tp": 2, "fp": 0, "fn": 3}}, "coderabbit": {"strict": {"tp": 1, "fp": 5, "fn": 3}, "core": {"tp": 1, "fp": 5, "fn": 3}, "all": {"tp": 2, "fp": 5, "fn": 3}}, "qodo-extended-v2": {"strict": {"tp": 2, "fp": 0, "fn": 2}, "core": {"tp": 2, "fp": 0, "fn": 2}, "all": {"tp": 2, "fp": 0, "fn": 3}}, "greptile-v4-1": {"strict": {"tp": 1, "fp": 1, "fn": 3}, "core": {"tp": 1, "fp": 1, "fn": 3}, "all": {"tp": 2, "fp": 1, "fn": 3}}, "gemini-v2": {"strict": {"tp": 1, "fp": 3, "fn": 3}, "core": {"tp": 1, "fp": 3, "fn": 3}, "all": {"tp": 2, "fp": 3, "fn": 3}}, "gitlab": {"strict": {"tp": 1, "fp": 6, "fn": 3}, "core": {"tp": 1, "fp": 6, "fn": 3}, "all": {"tp": 2, "fp": 6, "fn": 3}}, "copilot-v2": {"strict": {"tp": 1, "fp": 7, "fn": 3}, "core": {"tp": 1, "fp": 7, "fn": 3}, "all": {"tp": 1, "fp": 7, "fn": 4}}, "augment": {"strict": {"tp": 1, "fp": 1, "fn": 3}, "core": {"tp": 1, "fp": 1, "fn": 3}, "all": {"tp": 1, "fp": 1, "fn": 4}}, "bugbot": {"strict": {"tp": 2, "fp": 1, "fn": 2}, "core": {"tp": 2, "fp": 1, "fn": 2}, "all": {"tp": 3, "fp": 1, "fn": 2}}, "claude": {"strict": {"tp": 1, "fp": 4, "fn": 3}, "core": {"tp": 1, "fp": 4, "fn": 3}, "all": {"tp": 2, "fp": 4, "fn": 3}}, "claude-code": {"strict": {"tp": 0, "fp": 2, "fn": 4}, "core": {"tp": 0, "fp": 2, "fn": 4}, "all": {"tp": 0, "fp": 2, "fn": 5}}, "cubic-v2": {"strict": {"tp": 1, "fp": 1, "fn": 3}, "core": {"tp": 1, "fp": 1, "fn": 3}, "all": {"tp": 1, "fp": 1, "fn": 4}}, "devin": {"strict": {"tp": 1, "fp": 0, "fn": 3}, "core": {"tp": 1, "fp": 0, "fn": 3}, "all": {"tp": 1, "fp": 0, "fn": 4}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 4}, "core": {"tp": 0, "fp": 0, "fn": 4}, "all": {"tp": 0, "fp": 0, "fn": 5}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 4}, "core": {"tp": 0, "fp": 0, "fn": 4}, "all": {"tp": 0, "fp": 0, "fn": 5}}, "kodus-v2": {"strict": {"tp": 2, "fp": 1, "fn": 2}, "core": {"tp": 2, "fp": 1, "fn": 2}, "all": {"tp": 2, "fp": 1, "fn": 3}}, "macroscope": {"strict": {"tp": 2, "fp": 4, "fn": 2}, "core": {"tp": 2, "fp": 4, "fn": 2}, "all": {"tp": 2, "fp": 4, "fn": 3}}, "qodo-v2": {"strict": {"tp": 1, "fp": 3, "fn": 3}, "core": {"tp": 1, "fp": 3, "fn": 3}, "all": {"tp": 1, "fp": 3, "fn": 4}}, "sourcery": {"strict": {"tp": 1, "fp": 4, "fn": 3}, "core": {"tp": 1, "fp": 4, "fn": 3}, "all": {"tp": 2, "fp": 4, "fn": 3}}}}, {"url": "https://github.com/keycloak/keycloak/pull/37038", "language": "Java", "pr_size": "small", "domain": "authentication", "change_type": "feature", "complexity": "complex", "difficulty": "very_subtle", "risk": "high", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "codeant-v2": {"strict": {"tp": 1, "fp": 5, "fn": 1}, "core": {"tp": 1, "fp": 5, "fn": 1}, "all": {"tp": 1, "fp": 5, "fn": 1}}, "coderabbit": {"strict": {"tp": 2, "fp": 8, "fn": 0}, "core": {"tp": 2, "fp": 8, "fn": 0}, "all": {"tp": 2, "fp": 8, "fn": 0}}, "qodo-extended-v2": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "greptile-v4-1": {"strict": {"tp": 2, "fp": 2, "fn": 0}, "core": {"tp": 2, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "gemini-v2": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "gitlab": {"strict": {"tp": 1, "fp": 2, "fn": 1}, "core": {"tp": 1, "fp": 2, "fn": 1}, "all": {"tp": 1, "fp": 2, "fn": 1}}, "copilot-v2": {"strict": {"tp": 2, "fp": 3, "fn": 0}, "core": {"tp": 2, "fp": 3, "fn": 0}, "all": {"tp": 2, "fp": 3, "fn": 0}}, "augment": {"strict": {"tp": 2, "fp": 2, "fn": 0}, "core": {"tp": 2, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "bugbot": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "claude": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "claude-code": {"strict": {"tp": 1, "fp": 2, "fn": 1}, "core": {"tp": 1, "fp": 2, "fn": 1}, "all": {"tp": 1, "fp": 2, "fn": 1}}, "cubic-v2": {"strict": {"tp": 2, "fp": 2, "fn": 0}, "core": {"tp": 2, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "devin": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "kodus-v2": {"strict": {"tp": 2, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "macroscope": {"strict": {"tp": 2, "fp": 2, "fn": 0}, "core": {"tp": 2, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "qodo-v2": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "sourcery": {"strict": {"tp": 1, "fp": 4, "fn": 1}, "core": {"tp": 1, "fp": 4, "fn": 1}, "all": {"tp": 1, "fp": 4, "fn": 1}}}}, {"url": "https://github.com/keycloak/keycloak/pull/33832", "language": "Java", "pr_size": "medium", "domain": "authentication", "change_type": "feature", "complexity": "moderate", "difficulty": "subtle", "risk": "high", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 0, "fp": 1, "fn": 2}, "core": {"tp": 1, "fp": 1, "fn": 2}, "all": {"tp": 1, "fp": 1, "fn": 3}}, "codeant-v2": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 2}, "all": {"tp": 2, "fp": 1, "fn": 2}}, "coderabbit": {"strict": {"tp": 2, "fp": 4, "fn": 0}, "core": {"tp": 3, "fp": 4, "fn": 0}, "all": {"tp": 4, "fp": 4, "fn": 0}}, "qodo-extended-v2": {"strict": {"tp": 0, "fp": 2, "fn": 2}, "core": {"tp": 1, "fp": 2, "fn": 2}, "all": {"tp": 1, "fp": 2, "fn": 3}}, "greptile-v4-1": {"strict": {"tp": 0, "fp": 3, "fn": 2}, "core": {"tp": 1, "fp": 3, "fn": 2}, "all": {"tp": 1, "fp": 3, "fn": 3}}, "gemini-v2": {"strict": {"tp": 2, "fp": 1, "fn": 0}, "core": {"tp": 3, "fp": 1, "fn": 0}, "all": {"tp": 3, "fp": 1, "fn": 1}}, "gitlab": {"strict": {"tp": 0, "fp": 2, "fn": 2}, "core": {"tp": 1, "fp": 2, "fn": 2}, "all": {"tp": 1, "fp": 2, "fn": 3}}, "copilot-v2": {"strict": {"tp": 1, "fp": 6, "fn": 1}, "core": {"tp": 2, "fp": 6, "fn": 1}, "all": {"tp": 3, "fp": 6, "fn": 1}}, "augment": {"strict": {"tp": 1, "fp": 2, "fn": 1}, "core": {"tp": 1, "fp": 2, "fn": 2}, "all": {"tp": 1, "fp": 2, "fn": 3}}, "bugbot": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 3}}, "claude": {"strict": {"tp": 1, "fp": 4, "fn": 1}, "core": {"tp": 2, "fp": 4, "fn": 1}, "all": {"tp": 2, "fp": 4, "fn": 2}}, "claude-code": {"strict": {"tp": 0, "fp": 1, "fn": 2}, "core": {"tp": 1, "fp": 1, "fn": 2}, "all": {"tp": 2, "fp": 1, "fn": 2}}, "cubic-v2": {"strict": {"tp": 1, "fp": 2, "fn": 1}, "core": {"tp": 2, "fp": 2, "fn": 1}, "all": {"tp": 2, "fp": 2, "fn": 2}}, "devin": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 4}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 3}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 4}}, "kodus-v2": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 4}}, "macroscope": {"strict": {"tp": 2, "fp": 4, "fn": 0}, "core": {"tp": 2, "fp": 4, "fn": 1}, "all": {"tp": 2, "fp": 4, "fn": 2}}, "qodo-v2": {"strict": {"tp": 1, "fp": 2, "fn": 1}, "core": {"tp": 1, "fp": 2, "fn": 2}, "all": {"tp": 1, "fp": 2, "fn": 3}}, "sourcery": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 1}, "all": {"tp": 2, "fp": 1, "fn": 2}}}}, {"url": "https://github.com/keycloak/keycloak/pull/40940", "language": "Java", "pr_size": "small", "domain": "concurrency", "change_type": "bug_fix", "complexity": "moderate", "difficulty": "subtle", "risk": "high", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "codeant-v2": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "coderabbit": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "qodo-extended-v2": {"strict": {"tp": 2, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "greptile-v4-1": {"strict": {"tp": 1, "fp": 2, "fn": 1}, "core": {"tp": 1, "fp": 2, "fn": 1}, "all": {"tp": 1, "fp": 2, "fn": 1}}, "gemini-v2": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "gitlab": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "copilot-v2": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "augment": {"strict": {"tp": 2, "fp": 2, "fn": 0}, "core": {"tp": 2, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "bugbot": {"strict": {"tp": 1, "fp": 2, "fn": 1}, "core": {"tp": 1, "fp": 2, "fn": 1}, "all": {"tp": 1, "fp": 2, "fn": 1}}, "claude": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "claude-code": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "cubic-v2": {"strict": {"tp": 2, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "devin": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "graphite": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "kodus-v2": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "macroscope": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "qodo-v2": {"strict": {"tp": 2, "fp": 2, "fn": 0}, "core": {"tp": 2, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "sourcery": {"strict": {"tp": 1, "fp": 2, "fn": 1}, "core": {"tp": 1, "fp": 2, "fn": 1}, "all": {"tp": 1, "fp": 2, "fn": 1}}}}, {"url": "https://github.com/ai-code-review-evaluation/keycloak-greptile/pull/1", "language": "Java", "pr_size": "medium", "domain": "authentication", "change_type": "bug_fix", "complexity": "moderate", "difficulty": "subtle", "risk": "medium", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 1, "fp": 2, "fn": 0}, "core": {"tp": 1, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "codeant-v2": {"strict": {"tp": 1, "fp": 2, "fn": 0}, "core": {"tp": 1, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "coderabbit": {"strict": {"tp": 1, "fp": 3, "fn": 0}, "core": {"tp": 1, "fp": 3, "fn": 0}, "all": {"tp": 1, "fp": 3, "fn": 1}}, "qodo-extended-v2": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 1, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "greptile-v4-1": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 1, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "gemini-v2": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 1, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "gitlab": {"strict": {"tp": 1, "fp": 2, "fn": 0}, "core": {"tp": 1, "fp": 2, "fn": 0}, "all": {"tp": 1, "fp": 2, "fn": 1}}, "copilot-v2": {"strict": {"tp": 1, "fp": 2, "fn": 0}, "core": {"tp": 1, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "augment": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "bugbot": {"strict": {"tp": 0, "fp": 2, "fn": 1}, "core": {"tp": 0, "fp": 2, "fn": 1}, "all": {"tp": 0, "fp": 2, "fn": 2}}, "claude": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 0}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "claude-code": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 0}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "cubic-v2": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 1, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "devin": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "kg": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 0}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "kodus-v2": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "macroscope": {"strict": {"tp": 0, "fp": 3, "fn": 1}, "core": {"tp": 0, "fp": 3, "fn": 1}, "all": {"tp": 1, "fp": 3, "fn": 1}}, "qodo-v2": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 1, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "sourcery": {"strict": {"tp": 0, "fp": 3, "fn": 1}, "core": {"tp": 0, "fp": 3, "fn": 1}, "all": {"tp": 1, "fp": 3, "fn": 1}}}}, {"url": "https://github.com/getsentry/sentry/pull/93824", "language": "Python", "pr_size": "large", "domain": "concurrency", "change_type": "feature", "complexity": "complex", "difficulty": "subtle", "risk": "high", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 0, "fp": 2, "fn": 3}, "core": {"tp": 0, "fp": 2, "fn": 5}, "all": {"tp": 0, "fp": 2, "fn": 5}}, "codeant-v2": {"strict": {"tp": 0, "fp": 1, "fn": 3}, "core": {"tp": 2, "fp": 1, "fn": 3}, "all": {"tp": 2, "fp": 1, "fn": 3}}, "coderabbit": {"strict": {"tp": 1, "fp": 3, "fn": 2}, "core": {"tp": 1, "fp": 3, "fn": 4}, "all": {"tp": 1, "fp": 3, "fn": 4}}, "qodo-extended-v2": {"strict": {"tp": 2, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 3}, "all": {"tp": 2, "fp": 1, "fn": 3}}, "greptile-v4-1": {"strict": {"tp": 1, "fp": 1, "fn": 2}, "core": {"tp": 1, "fp": 1, "fn": 4}, "all": {"tp": 1, "fp": 1, "fn": 4}}, "gemini-v2": {"strict": {"tp": 2, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 3}, "all": {"tp": 2, "fp": 1, "fn": 3}}, "gitlab": {"strict": {"tp": 0, "fp": 2, "fn": 3}, "core": {"tp": 0, "fp": 2, "fn": 5}, "all": {"tp": 0, "fp": 2, "fn": 5}}, "copilot-v2": {"strict": {"tp": 1, "fp": 4, "fn": 2}, "core": {"tp": 3, "fp": 4, "fn": 2}, "all": {"tp": 3, "fp": 4, "fn": 2}}, "augment": {"strict": {"tp": 3, "fp": 1, "fn": 0}, "core": {"tp": 5, "fp": 1, "fn": 0}, "all": {"tp": 5, "fp": 1, "fn": 0}}, "bugbot": {"strict": {"tp": 1, "fp": 0, "fn": 2}, "core": {"tp": 1, "fp": 0, "fn": 4}, "all": {"tp": 1, "fp": 0, "fn": 4}}, "claude": {"strict": {"tp": 0, "fp": 3, "fn": 3}, "core": {"tp": 0, "fp": 3, "fn": 5}, "all": {"tp": 0, "fp": 3, "fn": 5}}, "claude-code": {"strict": {"tp": 3, "fp": 3, "fn": 0}, "core": {"tp": 3, "fp": 3, "fn": 2}, "all": {"tp": 3, "fp": 3, "fn": 2}}, "cubic-v2": {"strict": {"tp": 3, "fp": 1, "fn": 0}, "core": {"tp": 5, "fp": 1, "fn": 0}, "all": {"tp": 5, "fp": 1, "fn": 0}}, "devin": {"strict": {"tp": 1, "fp": 0, "fn": 2}, "core": {"tp": 1, "fp": 0, "fn": 4}, "all": {"tp": 1, "fp": 0, "fn": 4}}, "graphite": {"strict": {"tp": 1, "fp": 0, "fn": 2}, "core": {"tp": 1, "fp": 0, "fn": 4}, "all": {"tp": 1, "fp": 0, "fn": 4}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 3}, "core": {"tp": 0, "fp": 0, "fn": 5}, "all": {"tp": 0, "fp": 0, "fn": 5}}, "kodus-v2": {"strict": {"tp": 0, "fp": 0, "fn": 3}, "core": {"tp": 0, "fp": 0, "fn": 5}, "all": {"tp": 0, "fp": 0, "fn": 5}}, "macroscope": {"strict": {"tp": 1, "fp": 1, "fn": 2}, "core": {"tp": 3, "fp": 1, "fn": 2}, "all": {"tp": 3, "fp": 1, "fn": 2}}, "qodo-v2": {"strict": {"tp": 2, "fp": 3, "fn": 1}, "core": {"tp": 4, "fp": 3, "fn": 1}, "all": {"tp": 4, "fp": 3, "fn": 1}}, "sourcery": {"strict": {"tp": 2, "fp": 2, "fn": 1}, "core": {"tp": 4, "fp": 2, "fn": 1}, "all": {"tp": 4, "fp": 2, "fn": 1}}}}, {"url": "https://github.com/ai-code-review-evaluation/sentry-greptile/pull/5", "language": "Python", "pr_size": "large", "domain": "API", "change_type": "feature", "complexity": "moderate", "difficulty": "subtle", "risk": "medium", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 3, "fp": 1, "fn": 3}, "core": {"tp": 3, "fp": 1, "fn": 3}, "all": {"tp": 3, "fp": 1, "fn": 3}}, "codeant-v2": {"strict": {"tp": 2, "fp": 5, "fn": 4}, "core": {"tp": 2, "fp": 5, "fn": 4}, "all": {"tp": 2, "fp": 5, "fn": 4}}, "coderabbit": {"strict": {"tp": 4, "fp": 13, "fn": 2}, "core": {"tp": 4, "fp": 13, "fn": 2}, "all": {"tp": 4, "fp": 13, "fn": 2}}, "qodo-extended-v2": {"strict": {"tp": 2, "fp": 0, "fn": 4}, "core": {"tp": 2, "fp": 0, "fn": 4}, "all": {"tp": 2, "fp": 0, "fn": 4}}, "greptile-v4-1": {"strict": {"tp": 1, "fp": 0, "fn": 5}, "core": {"tp": 1, "fp": 0, "fn": 5}, "all": {"tp": 1, "fp": 0, "fn": 5}}, "gemini-v2": {"strict": {"tp": 2, "fp": 1, "fn": 4}, "core": {"tp": 2, "fp": 1, "fn": 4}, "all": {"tp": 2, "fp": 1, "fn": 4}}, "gitlab": {"strict": {"tp": 2, "fp": 1, "fn": 4}, "core": {"tp": 2, "fp": 1, "fn": 4}, "all": {"tp": 2, "fp": 1, "fn": 4}}, "copilot-v2": {"strict": {"tp": 3, "fp": 5, "fn": 3}, "core": {"tp": 3, "fp": 5, "fn": 3}, "all": {"tp": 3, "fp": 5, "fn": 3}}, "augment": {"strict": {"tp": 3, "fp": 2, "fn": 3}, "core": {"tp": 3, "fp": 2, "fn": 3}, "all": {"tp": 3, "fp": 2, "fn": 3}}, "bugbot": {"strict": {"tp": 2, "fp": 0, "fn": 4}, "core": {"tp": 2, "fp": 0, "fn": 4}, "all": {"tp": 2, "fp": 0, "fn": 4}}, "claude": {"strict": {"tp": 0, "fp": 0, "fn": 6}, "core": {"tp": 0, "fp": 0, "fn": 6}, "all": {"tp": 0, "fp": 0, "fn": 6}}, "claude-code": {"strict": {"tp": 0, "fp": 2, "fn": 6}, "core": {"tp": 0, "fp": 2, "fn": 6}, "all": {"tp": 0, "fp": 2, "fn": 6}}, "cubic-v2": {"strict": {"tp": 2, "fp": 2, "fn": 4}, "core": {"tp": 2, "fp": 2, "fn": 4}, "all": {"tp": 2, "fp": 2, "fn": 4}}, "devin": {"strict": {"tp": 2, "fp": 1, "fn": 4}, "core": {"tp": 2, "fp": 1, "fn": 4}, "all": {"tp": 2, "fp": 1, "fn": 4}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 6}, "core": {"tp": 0, "fp": 0, "fn": 6}, "all": {"tp": 0, "fp": 0, "fn": 6}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 6}, "core": {"tp": 0, "fp": 0, "fn": 6}, "all": {"tp": 0, "fp": 0, "fn": 6}}, "kodus-v2": {"strict": {"tp": 2, "fp": 1, "fn": 4}, "core": {"tp": 2, "fp": 1, "fn": 4}, "all": {"tp": 2, "fp": 1, "fn": 4}}, "macroscope": {"strict": {"tp": 2, "fp": 3, "fn": 4}, "core": {"tp": 2, "fp": 3, "fn": 4}, "all": {"tp": 2, "fp": 3, "fn": 4}}, "qodo-v2": {"strict": {"tp": 2, "fp": 0, "fn": 4}, "core": {"tp": 2, "fp": 0, "fn": 4}, "all": {"tp": 2, "fp": 0, "fn": 4}}, "sourcery": {"strict": {"tp": 2, "fp": 2, "fn": 4}, "core": {"tp": 2, "fp": 2, "fn": 4}, "all": {"tp": 2, "fp": 2, "fn": 4}}}}, {"url": "https://github.com/ai-code-review-evaluation/sentry-greptile/pull/1", "language": "Python", "pr_size": "medium", "domain": "API", "change_type": "performance", "complexity": "moderate", "difficulty": "subtle", "risk": "high", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 1, "fp": 1, "fn": 3}, "core": {"tp": 1, "fp": 1, "fn": 3}, "all": {"tp": 1, "fp": 1, "fn": 3}}, "codeant-v2": {"strict": {"tp": 1, "fp": 1, "fn": 3}, "core": {"tp": 1, "fp": 1, "fn": 3}, "all": {"tp": 1, "fp": 1, "fn": 3}}, "coderabbit": {"strict": {"tp": 3, "fp": 2, "fn": 1}, "core": {"tp": 3, "fp": 2, "fn": 1}, "all": {"tp": 3, "fp": 2, "fn": 1}}, "qodo-extended-v2": {"strict": {"tp": 3, "fp": 0, "fn": 1}, "core": {"tp": 3, "fp": 0, "fn": 1}, "all": {"tp": 3, "fp": 0, "fn": 1}}, "greptile-v4-1": {"strict": {"tp": 3, "fp": 2, "fn": 1}, "core": {"tp": 3, "fp": 2, "fn": 1}, "all": {"tp": 3, "fp": 2, "fn": 1}}, "gemini-v2": {"strict": {"tp": 3, "fp": 2, "fn": 1}, "core": {"tp": 3, "fp": 2, "fn": 1}, "all": {"tp": 3, "fp": 2, "fn": 1}}, "gitlab": {"strict": {"tp": 3, "fp": 1, "fn": 1}, "core": {"tp": 3, "fp": 1, "fn": 1}, "all": {"tp": 3, "fp": 1, "fn": 1}}, "copilot-v2": {"strict": {"tp": 2, "fp": 2, "fn": 2}, "core": {"tp": 2, "fp": 2, "fn": 2}, "all": {"tp": 2, "fp": 2, "fn": 2}}, "augment": {"strict": {"tp": 3, "fp": 1, "fn": 1}, "core": {"tp": 3, "fp": 1, "fn": 1}, "all": {"tp": 3, "fp": 1, "fn": 1}}, "bugbot": {"strict": {"tp": 3, "fp": 1, "fn": 1}, "core": {"tp": 3, "fp": 1, "fn": 1}, "all": {"tp": 3, "fp": 1, "fn": 1}}, "claude": {"strict": {"tp": 2, "fp": 6, "fn": 2}, "core": {"tp": 2, "fp": 6, "fn": 2}, "all": {"tp": 2, "fp": 6, "fn": 2}}, "claude-code": {"strict": {"tp": 2, "fp": 1, "fn": 2}, "core": {"tp": 2, "fp": 1, "fn": 2}, "all": {"tp": 2, "fp": 1, "fn": 2}}, "cubic-v2": {"strict": {"tp": 3, "fp": 0, "fn": 1}, "core": {"tp": 3, "fp": 0, "fn": 1}, "all": {"tp": 3, "fp": 0, "fn": 1}}, "devin": {"strict": {"tp": 3, "fp": 2, "fn": 1}, "core": {"tp": 3, "fp": 2, "fn": 1}, "all": {"tp": 3, "fp": 2, "fn": 1}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 4}, "core": {"tp": 0, "fp": 0, "fn": 4}, "all": {"tp": 0, "fp": 0, "fn": 4}}, "kg": {"strict": {"tp": 1, "fp": 1, "fn": 3}, "core": {"tp": 1, "fp": 1, "fn": 3}, "all": {"tp": 1, "fp": 1, "fn": 3}}, "kodus-v2": {"strict": {"tp": 1, "fp": 4, "fn": 3}, "core": {"tp": 1, "fp": 4, "fn": 3}, "all": {"tp": 1, "fp": 4, "fn": 3}}, "macroscope": {"strict": {"tp": 2, "fp": 1, "fn": 2}, "core": {"tp": 2, "fp": 1, "fn": 2}, "all": {"tp": 2, "fp": 1, "fn": 2}}, "qodo-v2": {"strict": {"tp": 2, "fp": 1, "fn": 2}, "core": {"tp": 2, "fp": 1, "fn": 2}, "all": {"tp": 2, "fp": 1, "fn": 2}}, "sourcery": {"strict": {"tp": 2, "fp": 3, "fn": 2}, "core": {"tp": 2, "fp": 3, "fn": 2}, "all": {"tp": 2, "fp": 3, "fn": 2}}}}, {"url": "https://github.com/grafana/grafana/pull/97529", "language": "Go", "pr_size": "small", "domain": "concurrency", "change_type": "performance", "complexity": "complex", "difficulty": "very_subtle", "risk": "critical", "context": "cross_file", "concern": "reliability", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "codeant-v2": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "coderabbit": {"strict": {"tp": 1, "fp": 3, "fn": 1}, "core": {"tp": 1, "fp": 3, "fn": 1}, "all": {"tp": 1, "fp": 3, "fn": 1}}, "qodo-extended-v2": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "greptile-v4-1": {"strict": {"tp": 2, "fp": 2, "fn": 0}, "core": {"tp": 2, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "gemini-v2": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "gitlab": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "copilot-v2": {"strict": {"tp": 1, "fp": 7, "fn": 1}, "core": {"tp": 1, "fp": 7, "fn": 1}, "all": {"tp": 1, "fp": 7, "fn": 1}}, "augment": {"strict": {"tp": 2, "fp": 2, "fn": 0}, "core": {"tp": 2, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "bugbot": {"strict": {"tp": 1, "fp": 3, "fn": 1}, "core": {"tp": 1, "fp": 3, "fn": 1}, "all": {"tp": 1, "fp": 3, "fn": 1}}, "claude": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "claude-code": {"strict": {"tp": 1, "fp": 4, "fn": 1}, "core": {"tp": 1, "fp": 4, "fn": 1}, "all": {"tp": 1, "fp": 4, "fn": 1}}, "cubic-v2": {"strict": {"tp": 2, "fp": 2, "fn": 0}, "core": {"tp": 2, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "devin": {"strict": {"tp": 0, "fp": 2, "fn": 2}, "core": {"tp": 0, "fp": 2, "fn": 2}, "all": {"tp": 0, "fp": 2, "fn": 2}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "kg": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "kodus-v2": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "macroscope": {"strict": {"tp": 1, "fp": 2, "fn": 1}, "core": {"tp": 1, "fp": 2, "fn": 1}, "all": {"tp": 1, "fp": 2, "fn": 1}}, "qodo-v2": {"strict": {"tp": 2, "fp": 4, "fn": 0}, "core": {"tp": 2, "fp": 4, "fn": 0}, "all": {"tp": 2, "fp": 4, "fn": 0}}, "sourcery": {"strict": {"tp": 2, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}}}, {"url": "https://github.com/getsentry/sentry/pull/80168", "language": "Python", "pr_size": "small", "domain": "data_processing", "change_type": "feature", "complexity": "moderate", "difficulty": "moderate", "risk": "high", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 0, "fp": 1, "fn": 1}, "core": {"tp": 0, "fp": 1, "fn": 3}, "all": {"tp": 0, "fp": 1, "fn": 3}}, "codeant-v2": {"strict": {"tp": 0, "fp": 1, "fn": 1}, "core": {"tp": 0, "fp": 1, "fn": 3}, "all": {"tp": 0, "fp": 1, "fn": 3}}, "coderabbit": {"strict": {"tp": 0, "fp": 2, "fn": 1}, "core": {"tp": 2, "fp": 2, "fn": 1}, "all": {"tp": 2, "fp": 2, "fn": 1}}, "qodo-extended-v2": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 1}, "all": {"tp": 2, "fp": 0, "fn": 1}}, "greptile-v4-1": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 1}, "all": {"tp": 2, "fp": 1, "fn": 1}}, "gemini-v2": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 2}}, "gitlab": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 2}}, "copilot-v2": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 1}, "all": {"tp": 2, "fp": 0, "fn": 1}}, "augment": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 3, "fp": 1, "fn": 0}, "all": {"tp": 3, "fp": 1, "fn": 0}}, "bugbot": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 2}}, "claude": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 3}}, "claude-code": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 3, "fp": 1, "fn": 0}, "all": {"tp": 3, "fp": 1, "fn": 0}}, "cubic-v2": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 3, "fp": 1, "fn": 0}, "all": {"tp": 3, "fp": 1, "fn": 0}}, "devin": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 1, "fp": 1, "fn": 2}, "all": {"tp": 1, "fp": 1, "fn": 2}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 3}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 3}}, "kodus-v2": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 3}}, "macroscope": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 3}}, "qodo-v2": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 2}}, "sourcery": {"strict": {"tp": 1, "fp": 2, "fn": 0}, "core": {"tp": 2, "fp": 2, "fn": 1}, "all": {"tp": 2, "fp": 2, "fn": 1}}}}, {"url": "https://github.com/getsentry/sentry/pull/80528", "language": "Python", "pr_size": "small", "domain": "scheduling", "change_type": "refactoring", "complexity": "moderate", "difficulty": "subtle", "risk": "medium", "context": "file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "codeant-v2": {"strict": {"tp": 0, "fp": 4, "fn": 1}, "core": {"tp": 0, "fp": 4, "fn": 2}, "all": {"tp": 0, "fp": 4, "fn": 2}}, "coderabbit": {"strict": {"tp": 1, "fp": 3, "fn": 0}, "core": {"tp": 1, "fp": 3, "fn": 1}, "all": {"tp": 1, "fp": 3, "fn": 1}}, "qodo-extended-v2": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "greptile-v4-1": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "gemini-v2": {"strict": {"tp": 1, "fp": 5, "fn": 0}, "core": {"tp": 1, "fp": 5, "fn": 1}, "all": {"tp": 1, "fp": 5, "fn": 1}}, "gitlab": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "copilot-v2": {"strict": {"tp": 1, "fp": 2, "fn": 0}, "core": {"tp": 1, "fp": 2, "fn": 1}, "all": {"tp": 1, "fp": 2, "fn": 1}}, "augment": {"strict": {"tp": 1, "fp": 2, "fn": 0}, "core": {"tp": 1, "fp": 2, "fn": 1}, "all": {"tp": 1, "fp": 2, "fn": 1}}, "bugbot": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "claude": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "claude-code": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "cubic-v2": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "devin": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "graphite": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "kg": {"strict": {"tp": 0, "fp": 1, "fn": 1}, "core": {"tp": 0, "fp": 1, "fn": 2}, "all": {"tp": 0, "fp": 1, "fn": 2}}, "kodus-v2": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "macroscope": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "qodo-v2": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "sourcery": {"strict": {"tp": 1, "fp": 2, "fn": 0}, "core": {"tp": 1, "fp": 2, "fn": 1}, "all": {"tp": 1, "fp": 2, "fn": 1}}}}, {"url": "https://github.com/getsentry/sentry/pull/77754", "language": "Python", "pr_size": "medium", "domain": "serialization", "change_type": "feature", "complexity": "moderate", "difficulty": "moderate", "risk": "medium", "context": "file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 3}}, "codeant-v2": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 4}}, "coderabbit": {"strict": {"tp": 1, "fp": 2, "fn": 1}, "core": {"tp": 1, "fp": 2, "fn": 1}, "all": {"tp": 3, "fp": 2, "fn": 1}}, "qodo-extended-v2": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 3}}, "greptile-v4-1": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 3}}, "gemini-v2": {"strict": {"tp": 1, "fp": 3, "fn": 1}, "core": {"tp": 1, "fp": 3, "fn": 1}, "all": {"tp": 1, "fp": 3, "fn": 3}}, "gitlab": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 2, "fp": 1, "fn": 2}}, "copilot-v2": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 2, "fp": 1, "fn": 2}}, "augment": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 3}}, "bugbot": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 3}}, "claude": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 2, "fp": 0, "fn": 2}}, "claude-code": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 3}}, "cubic-v2": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 2}}, "devin": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 3}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 4}}, "kg": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 3}}, "kodus-v2": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 3}}, "macroscope": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 3}}, "qodo-v2": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 3}}, "sourcery": {"strict": {"tp": 1, "fp": 3, "fn": 1}, "core": {"tp": 1, "fp": 3, "fn": 1}, "all": {"tp": 2, "fp": 3, "fn": 2}}}}, {"url": "https://github.com/getsentry/sentry/pull/95633", "language": "Python", "pr_size": "medium", "domain": "concurrency", "change_type": "feature", "complexity": "moderate", "difficulty": "moderate", "risk": "medium", "context": "file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 0, "fp": 3, "fn": 0}, "core": {"tp": 0, "fp": 3, "fn": 1}, "all": {"tp": 0, "fp": 3, "fn": 2}}, "codeant-v2": {"strict": {"tp": 0, "fp": 1, "fn": 0}, "core": {"tp": 0, "fp": 1, "fn": 1}, "all": {"tp": 0, "fp": 1, "fn": 2}}, "coderabbit": {"strict": {"tp": 0, "fp": 7, "fn": 0}, "core": {"tp": 0, "fp": 7, "fn": 1}, "all": {"tp": 0, "fp": 7, "fn": 2}}, "qodo-extended-v2": {"strict": {"tp": 0, "fp": 8, "fn": 0}, "core": {"tp": 0, "fp": 8, "fn": 1}, "all": {"tp": 0, "fp": 8, "fn": 2}}, "greptile-v4-1": {"strict": {"tp": 0, "fp": 5, "fn": 0}, "core": {"tp": 0, "fp": 5, "fn": 1}, "all": {"tp": 0, "fp": 5, "fn": 2}}, "gemini-v2": {"strict": {"tp": 0, "fp": 11, "fn": 0}, "core": {"tp": 0, "fp": 11, "fn": 1}, "all": {"tp": 0, "fp": 11, "fn": 2}}, "gitlab": {"strict": {"tp": 0, "fp": 5, "fn": 0}, "core": {"tp": 1, "fp": 5, "fn": 0}, "all": {"tp": 1, "fp": 5, "fn": 1}}, "copilot-v2": {"strict": {"tp": 0, "fp": 6, "fn": 0}, "core": {"tp": 0, "fp": 6, "fn": 1}, "all": {"tp": 0, "fp": 6, "fn": 2}}, "augment": {"strict": {"tp": 0, "fp": 4, "fn": 0}, "core": {"tp": 1, "fp": 4, "fn": 0}, "all": {"tp": 1, "fp": 4, "fn": 1}}, "bugbot": {"strict": {"tp": 0, "fp": 2, "fn": 0}, "core": {"tp": 0, "fp": 2, "fn": 1}, "all": {"tp": 0, "fp": 2, "fn": 2}}, "claude": {"strict": {"tp": 0, "fp": 0, "fn": 0}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "claude-code": {"strict": {"tp": 0, "fp": 2, "fn": 0}, "core": {"tp": 0, "fp": 2, "fn": 1}, "all": {"tp": 0, "fp": 2, "fn": 2}}, "cubic-v2": {"strict": {"tp": 0, "fp": 3, "fn": 0}, "core": {"tp": 0, "fp": 3, "fn": 1}, "all": {"tp": 0, "fp": 3, "fn": 2}}, "devin": {"strict": {"tp": 0, "fp": 4, "fn": 0}, "core": {"tp": 0, "fp": 4, "fn": 1}, "all": {"tp": 0, "fp": 4, "fn": 2}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 0}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "kg": {"strict": {"tp": 0, "fp": 2, "fn": 0}, "core": {"tp": 0, "fp": 2, "fn": 1}, "all": {"tp": 0, "fp": 2, "fn": 2}}, "kodus-v2": {"strict": {"tp": 0, "fp": 0, "fn": 0}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "macroscope": {"strict": {"tp": 0, "fp": 2, "fn": 0}, "core": {"tp": 0, "fp": 2, "fn": 1}, "all": {"tp": 0, "fp": 2, "fn": 2}}, "qodo-v2": {"strict": {"tp": 0, "fp": 4, "fn": 0}, "core": {"tp": 0, "fp": 4, "fn": 1}, "all": {"tp": 0, "fp": 4, "fn": 2}}, "sourcery": {"strict": {"tp": 0, "fp": 4, "fn": 0}, "core": {"tp": 1, "fp": 4, "fn": 0}, "all": {"tp": 1, "fp": 4, "fn": 1}}}}, {"url": "https://github.com/ai-code-review-evaluation/sentry-greptile/pull/2", "language": "Python", "pr_size": "medium", "domain": "data_processing", "change_type": "performance", "complexity": "moderate", "difficulty": "subtle", "risk": "high", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 3, "fp": 2, "fn": 1}, "core": {"tp": 3, "fp": 2, "fn": 1}, "all": {"tp": 3, "fp": 2, "fn": 1}}, "codeant-v2": {"strict": {"tp": 3, "fp": 1, "fn": 1}, "core": {"tp": 3, "fp": 1, "fn": 1}, "all": {"tp": 3, "fp": 1, "fn": 1}}, "coderabbit": {"strict": {"tp": 0, "fp": 0, "fn": 4}, "core": {"tp": 0, "fp": 0, "fn": 4}, "all": {"tp": 0, "fp": 0, "fn": 4}}, "qodo-extended-v2": {"strict": {"tp": 3, "fp": 4, "fn": 1}, "core": {"tp": 3, "fp": 4, "fn": 1}, "all": {"tp": 3, "fp": 4, "fn": 1}}, "greptile-v4-1": {"strict": {"tp": 2, "fp": 4, "fn": 2}, "core": {"tp": 2, "fp": 4, "fn": 2}, "all": {"tp": 2, "fp": 4, "fn": 2}}, "gemini-v2": {"strict": {"tp": 3, "fp": 5, "fn": 1}, "core": {"tp": 3, "fp": 5, "fn": 1}, "all": {"tp": 3, "fp": 5, "fn": 1}}, "gitlab": {"strict": {"tp": 4, "fp": 1, "fn": 0}, "core": {"tp": 4, "fp": 1, "fn": 0}, "all": {"tp": 4, "fp": 1, "fn": 0}}, "copilot-v2": {"strict": {"tp": 3, "fp": 10, "fn": 1}, "core": {"tp": 3, "fp": 10, "fn": 1}, "all": {"tp": 3, "fp": 10, "fn": 1}}, "augment": {"strict": {"tp": 3, "fp": 2, "fn": 1}, "core": {"tp": 3, "fp": 2, "fn": 1}, "all": {"tp": 3, "fp": 2, "fn": 1}}, "bugbot": {"strict": {"tp": 4, "fp": 0, "fn": 0}, "core": {"tp": 4, "fp": 0, "fn": 0}, "all": {"tp": 4, "fp": 0, "fn": 0}}, "claude": {"strict": {"tp": 3, "fp": 5, "fn": 1}, "core": {"tp": 3, "fp": 5, "fn": 1}, "all": {"tp": 3, "fp": 5, "fn": 1}}, "claude-code": {"strict": {"tp": 3, "fp": 2, "fn": 1}, "core": {"tp": 3, "fp": 2, "fn": 1}, "all": {"tp": 3, "fp": 2, "fn": 1}}, "cubic-v2": {"strict": {"tp": 3, "fp": 0, "fn": 1}, "core": {"tp": 3, "fp": 0, "fn": 1}, "all": {"tp": 3, "fp": 0, "fn": 1}}, "devin": {"strict": {"tp": 0, "fp": 0, "fn": 4}, "core": {"tp": 0, "fp": 0, "fn": 4}, "all": {"tp": 0, "fp": 0, "fn": 4}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 4}, "core": {"tp": 0, "fp": 0, "fn": 4}, "all": {"tp": 0, "fp": 0, "fn": 4}}, "kg": {"strict": {"tp": 2, "fp": 2, "fn": 2}, "core": {"tp": 2, "fp": 2, "fn": 2}, "all": {"tp": 2, "fp": 2, "fn": 2}}, "kodus-v2": {"strict": {"tp": 2, "fp": 3, "fn": 2}, "core": {"tp": 2, "fp": 3, "fn": 2}, "all": {"tp": 2, "fp": 3, "fn": 2}}, "macroscope": {"strict": {"tp": 3, "fp": 0, "fn": 1}, "core": {"tp": 3, "fp": 0, "fn": 1}, "all": {"tp": 3, "fp": 0, "fn": 1}}, "qodo-v2": {"strict": {"tp": 4, "fp": 0, "fn": 0}, "core": {"tp": 4, "fp": 0, "fn": 0}, "all": {"tp": 4, "fp": 0, "fn": 0}}, "sourcery": {"strict": {"tp": 3, "fp": 7, "fn": 1}, "core": {"tp": 3, "fp": 7, "fn": 1}, "all": {"tp": 3, "fp": 7, "fn": 1}}}}, {"url": "https://github.com/ai-code-review-evaluation/sentry-greptile/pull/3", "language": "Python", "pr_size": "medium", "domain": "caching", "change_type": "feature", "complexity": "complex", "difficulty": "subtle", "risk": "medium", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 0, "fp": 6, "fn": 3}, "core": {"tp": 0, "fp": 6, "fn": 3}, "all": {"tp": 0, "fp": 6, "fn": 3}}, "codeant-v2": {"strict": {"tp": 3, "fp": 1, "fn": 0}, "core": {"tp": 3, "fp": 1, "fn": 0}, "all": {"tp": 3, "fp": 1, "fn": 0}}, "coderabbit": {"strict": {"tp": 2, "fp": 3, "fn": 1}, "core": {"tp": 2, "fp": 3, "fn": 1}, "all": {"tp": 2, "fp": 3, "fn": 1}}, "qodo-extended-v2": {"strict": {"tp": 3, "fp": 0, "fn": 0}, "core": {"tp": 3, "fp": 0, "fn": 0}, "all": {"tp": 3, "fp": 0, "fn": 0}}, "greptile-v4-1": {"strict": {"tp": 2, "fp": 2, "fn": 1}, "core": {"tp": 2, "fp": 2, "fn": 1}, "all": {"tp": 2, "fp": 2, "fn": 1}}, "gemini-v2": {"strict": {"tp": 2, "fp": 2, "fn": 1}, "core": {"tp": 2, "fp": 2, "fn": 1}, "all": {"tp": 2, "fp": 2, "fn": 1}}, "gitlab": {"strict": {"tp": 2, "fp": 4, "fn": 1}, "core": {"tp": 2, "fp": 4, "fn": 1}, "all": {"tp": 2, "fp": 4, "fn": 1}}, "copilot-v2": {"strict": {"tp": 1, "fp": 9, "fn": 2}, "core": {"tp": 1, "fp": 9, "fn": 2}, "all": {"tp": 1, "fp": 9, "fn": 2}}, "augment": {"strict": {"tp": 1, "fp": 2, "fn": 2}, "core": {"tp": 1, "fp": 2, "fn": 2}, "all": {"tp": 1, "fp": 2, "fn": 2}}, "bugbot": {"strict": {"tp": 0, "fp": 3, "fn": 3}, "core": {"tp": 0, "fp": 3, "fn": 3}, "all": {"tp": 0, "fp": 3, "fn": 3}}, "claude": {"strict": {"tp": 1, "fp": 0, "fn": 2}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 2}}, "claude-code": {"strict": {"tp": 1, "fp": 2, "fn": 2}, "core": {"tp": 1, "fp": 2, "fn": 2}, "all": {"tp": 1, "fp": 2, "fn": 2}}, "cubic-v2": {"strict": {"tp": 2, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 1}, "all": {"tp": 2, "fp": 1, "fn": 1}}, "devin": {"strict": {"tp": 2, "fp": 0, "fn": 1}, "core": {"tp": 2, "fp": 0, "fn": 1}, "all": {"tp": 2, "fp": 0, "fn": 1}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 3}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 3}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 3}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 3}}, "kodus-v2": {"strict": {"tp": 2, "fp": 2, "fn": 1}, "core": {"tp": 2, "fp": 2, "fn": 1}, "all": {"tp": 2, "fp": 2, "fn": 1}}, "macroscope": {"strict": {"tp": 2, "fp": 0, "fn": 1}, "core": {"tp": 2, "fp": 0, "fn": 1}, "all": {"tp": 2, "fp": 0, "fn": 1}}, "qodo-v2": {"strict": {"tp": 2, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 1}, "all": {"tp": 2, "fp": 1, "fn": 1}}, "sourcery": {"strict": {"tp": 2, "fp": 6, "fn": 1}, "core": {"tp": 2, "fp": 6, "fn": 1}, "all": {"tp": 2, "fp": 6, "fn": 1}}}}, {"url": "https://github.com/grafana/grafana/pull/103633", "language": "Go", "pr_size": "small", "domain": "caching", "change_type": "performance", "complexity": "moderate", "difficulty": "subtle", "risk": "high", "context": "cross_file", "concern": "security", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 1, "fp": 2, "fn": 1}, "core": {"tp": 1, "fp": 2, "fn": 2}, "all": {"tp": 1, "fp": 2, "fn": 2}}, "codeant-v2": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 2}}, "coderabbit": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 2}}, "qodo-extended-v2": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 2, "fp": 0, "fn": 1}, "all": {"tp": 2, "fp": 0, "fn": 1}}, "greptile-v4-1": {"strict": {"tp": 1, "fp": 2, "fn": 1}, "core": {"tp": 1, "fp": 2, "fn": 2}, "all": {"tp": 1, "fp": 2, "fn": 2}}, "gemini-v2": {"strict": {"tp": 0, "fp": 3, "fn": 2}, "core": {"tp": 0, "fp": 3, "fn": 3}, "all": {"tp": 0, "fp": 3, "fn": 3}}, "gitlab": {"strict": {"tp": 1, "fp": 2, "fn": 1}, "core": {"tp": 1, "fp": 2, "fn": 2}, "all": {"tp": 1, "fp": 2, "fn": 2}}, "copilot-v2": {"strict": {"tp": 0, "fp": 1, "fn": 2}, "core": {"tp": 1, "fp": 1, "fn": 2}, "all": {"tp": 1, "fp": 1, "fn": 2}}, "augment": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 2, "fp": 0, "fn": 1}, "all": {"tp": 2, "fp": 0, "fn": 1}}, "bugbot": {"strict": {"tp": 0, "fp": 1, "fn": 2}, "core": {"tp": 0, "fp": 1, "fn": 3}, "all": {"tp": 0, "fp": 1, "fn": 3}}, "claude": {"strict": {"tp": 0, "fp": 6, "fn": 2}, "core": {"tp": 0, "fp": 6, "fn": 3}, "all": {"tp": 0, "fp": 6, "fn": 3}}, "claude-code": {"strict": {"tp": 0, "fp": 2, "fn": 2}, "core": {"tp": 0, "fp": 2, "fn": 3}, "all": {"tp": 0, "fp": 2, "fn": 3}}, "cubic-v2": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 1}, "all": {"tp": 2, "fp": 1, "fn": 1}}, "devin": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 1}, "all": {"tp": 2, "fp": 1, "fn": 1}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 3}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 3}}, "kodus-v2": {"strict": {"tp": 0, "fp": 1, "fn": 2}, "core": {"tp": 0, "fp": 1, "fn": 3}, "all": {"tp": 0, "fp": 1, "fn": 3}}, "macroscope": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 2}, "all": {"tp": 1, "fp": 1, "fn": 2}}, "qodo-v2": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 1}, "all": {"tp": 2, "fp": 1, "fn": 1}}, "sourcery": {"strict": {"tp": 0, "fp": 2, "fn": 2}, "core": {"tp": 0, "fp": 2, "fn": 3}, "all": {"tp": 0, "fp": 2, "fn": 3}}}}, {"url": "https://github.com/getsentry/sentry/pull/67876", "language": "Python", "pr_size": "small", "domain": "authentication", "change_type": "security_patch", "complexity": "moderate", "difficulty": "subtle", "risk": "high", "context": "cross_file", "concern": "security", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 1, "fp": 1, "fn": 2}, "core": {"tp": 1, "fp": 1, "fn": 2}, "all": {"tp": 1, "fp": 1, "fn": 2}}, "codeant-v2": {"strict": {"tp": 0, "fp": 2, "fn": 3}, "core": {"tp": 0, "fp": 2, "fn": 3}, "all": {"tp": 0, "fp": 2, "fn": 3}}, "coderabbit": {"strict": {"tp": 1, "fp": 2, "fn": 2}, "core": {"tp": 1, "fp": 2, "fn": 2}, "all": {"tp": 1, "fp": 2, "fn": 2}}, "qodo-extended-v2": {"strict": {"tp": 2, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 1}, "all": {"tp": 2, "fp": 1, "fn": 1}}, "greptile-v4-1": {"strict": {"tp": 1, "fp": 1, "fn": 2}, "core": {"tp": 1, "fp": 1, "fn": 2}, "all": {"tp": 1, "fp": 1, "fn": 2}}, "gemini-v2": {"strict": {"tp": 1, "fp": 1, "fn": 2}, "core": {"tp": 1, "fp": 1, "fn": 2}, "all": {"tp": 1, "fp": 1, "fn": 2}}, "gitlab": {"strict": {"tp": 1, "fp": 1, "fn": 2}, "core": {"tp": 1, "fp": 1, "fn": 2}, "all": {"tp": 1, "fp": 1, "fn": 2}}, "copilot-v2": {"strict": {"tp": 2, "fp": 13, "fn": 1}, "core": {"tp": 2, "fp": 13, "fn": 1}, "all": {"tp": 2, "fp": 13, "fn": 1}}, "augment": {"strict": {"tp": 1, "fp": 2, "fn": 2}, "core": {"tp": 1, "fp": 2, "fn": 2}, "all": {"tp": 1, "fp": 2, "fn": 2}}, "bugbot": {"strict": {"tp": 1, "fp": 2, "fn": 2}, "core": {"tp": 1, "fp": 2, "fn": 2}, "all": {"tp": 1, "fp": 2, "fn": 2}}, "claude": {"strict": {"tp": 1, "fp": 1, "fn": 2}, "core": {"tp": 1, "fp": 1, "fn": 2}, "all": {"tp": 1, "fp": 1, "fn": 2}}, "claude-code": {"strict": {"tp": 2, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 1}, "all": {"tp": 2, "fp": 1, "fn": 1}}, "cubic-v2": {"strict": {"tp": 3, "fp": 2, "fn": 0}, "core": {"tp": 3, "fp": 2, "fn": 0}, "all": {"tp": 3, "fp": 2, "fn": 0}}, "devin": {"strict": {"tp": 1, "fp": 1, "fn": 2}, "core": {"tp": 1, "fp": 1, "fn": 2}, "all": {"tp": 1, "fp": 1, "fn": 2}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 3}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 3}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 3}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 3}}, "kodus-v2": {"strict": {"tp": 0, "fp": 0, "fn": 3}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 3}}, "macroscope": {"strict": {"tp": 1, "fp": 1, "fn": 2}, "core": {"tp": 1, "fp": 1, "fn": 2}, "all": {"tp": 1, "fp": 1, "fn": 2}}, "qodo-v2": {"strict": {"tp": 2, "fp": 5, "fn": 1}, "core": {"tp": 2, "fp": 5, "fn": 1}, "all": {"tp": 2, "fp": 5, "fn": 1}}, "sourcery": {"strict": {"tp": 1, "fp": 8, "fn": 2}, "core": {"tp": 1, "fp": 8, "fn": 2}, "all": {"tp": 1, "fp": 8, "fn": 2}}}}, {"url": "https://github.com/keycloak/keycloak/pull/32918", "language": "Java", "pr_size": "small", "domain": "caching", "change_type": "feature", "complexity": "moderate", "difficulty": "subtle", "risk": "high", "context": "file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "codeant-v2": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "coderabbit": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "qodo-extended-v2": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "greptile-v4-1": {"strict": {"tp": 0, "fp": 3, "fn": 2}, "core": {"tp": 0, "fp": 3, "fn": 2}, "all": {"tp": 0, "fp": 3, "fn": 2}}, "gemini-v2": {"strict": {"tp": 0, "fp": 2, "fn": 2}, "core": {"tp": 0, "fp": 2, "fn": 2}, "all": {"tp": 0, "fp": 2, "fn": 2}}, "gitlab": {"strict": {"tp": 1, "fp": 2, "fn": 1}, "core": {"tp": 1, "fp": 2, "fn": 1}, "all": {"tp": 1, "fp": 2, "fn": 1}}, "copilot-v2": {"strict": {"tp": 1, "fp": 3, "fn": 1}, "core": {"tp": 1, "fp": 3, "fn": 1}, "all": {"tp": 1, "fp": 3, "fn": 1}}, "augment": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "bugbot": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "claude": {"strict": {"tp": 0, "fp": 2, "fn": 2}, "core": {"tp": 0, "fp": 2, "fn": 2}, "all": {"tp": 0, "fp": 2, "fn": 2}}, "claude-code": {"strict": {"tp": 1, "fp": 2, "fn": 1}, "core": {"tp": 1, "fp": 2, "fn": 1}, "all": {"tp": 1, "fp": 2, "fn": 1}}, "cubic-v2": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "devin": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "graphite": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "kg": {"strict": {"tp": 1, "fp": 2, "fn": 1}, "core": {"tp": 1, "fp": 2, "fn": 1}, "all": {"tp": 1, "fp": 2, "fn": 1}}, "kodus-v2": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "macroscope": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "qodo-v2": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "sourcery": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}}}, {"url": "https://github.com/grafana/grafana/pull/94942", "language": "Go", "pr_size": "small", "domain": "data_processing", "change_type": "feature", "complexity": "moderate", "difficulty": "moderate", "risk": "high", "context": "file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "codeant-v2": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "coderabbit": {"strict": {"tp": 1, "fp": 2, "fn": 1}, "core": {"tp": 1, "fp": 2, "fn": 1}, "all": {"tp": 1, "fp": 2, "fn": 1}}, "qodo-extended-v2": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "greptile-v4-1": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "gemini-v2": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "gitlab": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "copilot-v2": {"strict": {"tp": 2, "fp": 2, "fn": 0}, "core": {"tp": 2, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "augment": {"strict": {"tp": 2, "fp": 2, "fn": 0}, "core": {"tp": 2, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "bugbot": {"strict": {"tp": 1, "fp": 2, "fn": 1}, "core": {"tp": 1, "fp": 2, "fn": 1}, "all": {"tp": 1, "fp": 2, "fn": 1}}, "claude": {"strict": {"tp": 2, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "claude-code": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "cubic-v2": {"strict": {"tp": 1, "fp": 2, "fn": 1}, "core": {"tp": 1, "fp": 2, "fn": 1}, "all": {"tp": 1, "fp": 2, "fn": 1}}, "devin": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "kg": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "kodus-v2": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "macroscope": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "qodo-v2": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "sourcery": {"strict": {"tp": 2, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}}}, {"url": "https://github.com/grafana/grafana/pull/90939", "language": "Go", "pr_size": "small", "domain": "caching", "change_type": "performance", "complexity": "moderate", "difficulty": "subtle", "risk": "high", "context": "file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "codeant-v2": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "coderabbit": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "qodo-extended-v2": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "greptile-v4-1": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "gemini-v2": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "gitlab": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "copilot-v2": {"strict": {"tp": 1, "fp": 3, "fn": 1}, "core": {"tp": 1, "fp": 3, "fn": 1}, "all": {"tp": 1, "fp": 3, "fn": 1}}, "augment": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "bugbot": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "claude": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "claude-code": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "cubic-v2": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "devin": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "kg": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "kodus-v2": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "macroscope": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "qodo-v2": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "sourcery": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}}}, {"url": "https://github.com/grafana/grafana/pull/80329", "language": "Go", "pr_size": "small", "domain": "logging", "change_type": "performance", "complexity": "simple", "difficulty": "obvious", "risk": "low", "context": "local", "concern": "maintainability", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 0, "fp": 0, "fn": 0}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "codeant-v2": {"strict": {"tp": 0, "fp": 0, "fn": 0}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "coderabbit": {"strict": {"tp": 0, "fp": 4, "fn": 0}, "core": {"tp": 2, "fp": 4, "fn": 0}, "all": {"tp": 2, "fp": 4, "fn": 0}}, "qodo-extended-v2": {"strict": {"tp": 0, "fp": 3, "fn": 0}, "core": {"tp": 2, "fp": 3, "fn": 0}, "all": {"tp": 2, "fp": 3, "fn": 0}}, "greptile-v4-1": {"strict": {"tp": 0, "fp": 3, "fn": 0}, "core": {"tp": 2, "fp": 3, "fn": 0}, "all": {"tp": 2, "fp": 3, "fn": 0}}, "gemini-v2": {"strict": {"tp": 0, "fp": 3, "fn": 0}, "core": {"tp": 2, "fp": 3, "fn": 0}, "all": {"tp": 2, "fp": 3, "fn": 0}}, "gitlab": {"strict": {"tp": 0, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "copilot-v2": {"strict": {"tp": 0, "fp": 4, "fn": 0}, "core": {"tp": 2, "fp": 4, "fn": 0}, "all": {"tp": 2, "fp": 4, "fn": 0}}, "augment": {"strict": {"tp": 0, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "bugbot": {"strict": {"tp": 0, "fp": 4, "fn": 0}, "core": {"tp": 2, "fp": 4, "fn": 0}, "all": {"tp": 2, "fp": 4, "fn": 0}}, "claude": {"strict": {"tp": 0, "fp": 4, "fn": 0}, "core": {"tp": 2, "fp": 4, "fn": 0}, "all": {"tp": 2, "fp": 4, "fn": 0}}, "claude-code": {"strict": {"tp": 0, "fp": 4, "fn": 0}, "core": {"tp": 2, "fp": 4, "fn": 0}, "all": {"tp": 2, "fp": 4, "fn": 0}}, "cubic-v2": {"strict": {"tp": 0, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "devin": {"strict": {"tp": 0, "fp": 3, "fn": 0}, "core": {"tp": 2, "fp": 3, "fn": 0}, "all": {"tp": 2, "fp": 3, "fn": 0}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 0}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 0}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "kodus-v2": {"strict": {"tp": 0, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "macroscope": {"strict": {"tp": 0, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "qodo-v2": {"strict": {"tp": 0, "fp": 5, "fn": 0}, "core": {"tp": 2, "fp": 5, "fn": 0}, "all": {"tp": 2, "fp": 5, "fn": 0}}, "sourcery": {"strict": {"tp": 0, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}}}, {"url": "https://github.com/grafana/grafana/pull/90045", "language": "Go", "pr_size": "medium", "domain": "logging", "change_type": "feature", "complexity": "moderate", "difficulty": "moderate", "risk": "medium", "context": "file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 2, "fp": 2, "fn": 1}, "core": {"tp": 2, "fp": 2, "fn": 1}, "all": {"tp": 2, "fp": 2, "fn": 1}}, "codeant-v2": {"strict": {"tp": 2, "fp": 3, "fn": 1}, "core": {"tp": 2, "fp": 3, "fn": 1}, "all": {"tp": 2, "fp": 3, "fn": 1}}, "coderabbit": {"strict": {"tp": 3, "fp": 2, "fn": 0}, "core": {"tp": 3, "fp": 2, "fn": 0}, "all": {"tp": 3, "fp": 2, "fn": 0}}, "qodo-extended-v2": {"strict": {"tp": 3, "fp": 4, "fn": 0}, "core": {"tp": 3, "fp": 4, "fn": 0}, "all": {"tp": 3, "fp": 4, "fn": 0}}, "greptile-v4-1": {"strict": {"tp": 3, "fp": 3, "fn": 0}, "core": {"tp": 3, "fp": 3, "fn": 0}, "all": {"tp": 3, "fp": 3, "fn": 0}}, "gemini-v2": {"strict": {"tp": 3, "fp": 5, "fn": 0}, "core": {"tp": 3, "fp": 5, "fn": 0}, "all": {"tp": 3, "fp": 5, "fn": 0}}, "gitlab": {"strict": {"tp": 3, "fp": 2, "fn": 0}, "core": {"tp": 3, "fp": 2, "fn": 0}, "all": {"tp": 3, "fp": 2, "fn": 0}}, "copilot-v2": {"strict": {"tp": 3, "fp": 5, "fn": 0}, "core": {"tp": 3, "fp": 5, "fn": 0}, "all": {"tp": 3, "fp": 5, "fn": 0}}, "augment": {"strict": {"tp": 2, "fp": 6, "fn": 1}, "core": {"tp": 2, "fp": 6, "fn": 1}, "all": {"tp": 2, "fp": 6, "fn": 1}}, "bugbot": {"strict": {"tp": 3, "fp": 3, "fn": 0}, "core": {"tp": 3, "fp": 3, "fn": 0}, "all": {"tp": 3, "fp": 3, "fn": 0}}, "claude": {"strict": {"tp": 3, "fp": 4, "fn": 0}, "core": {"tp": 3, "fp": 4, "fn": 0}, "all": {"tp": 3, "fp": 4, "fn": 0}}, "claude-code": {"strict": {"tp": 3, "fp": 2, "fn": 0}, "core": {"tp": 3, "fp": 2, "fn": 0}, "all": {"tp": 3, "fp": 2, "fn": 0}}, "cubic-v2": {"strict": {"tp": 3, "fp": 2, "fn": 0}, "core": {"tp": 3, "fp": 2, "fn": 0}, "all": {"tp": 3, "fp": 2, "fn": 0}}, "devin": {"strict": {"tp": 3, "fp": 2, "fn": 0}, "core": {"tp": 3, "fp": 2, "fn": 0}, "all": {"tp": 3, "fp": 2, "fn": 0}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 3}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 3}}, "kg": {"strict": {"tp": 2, "fp": 2, "fn": 1}, "core": {"tp": 2, "fp": 2, "fn": 1}, "all": {"tp": 2, "fp": 2, "fn": 1}}, "kodus-v2": {"strict": {"tp": 1, "fp": 0, "fn": 2}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 2}}, "macroscope": {"strict": {"tp": 2, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 1}, "all": {"tp": 2, "fp": 1, "fn": 1}}, "qodo-v2": {"strict": {"tp": 2, "fp": 6, "fn": 1}, "core": {"tp": 2, "fp": 6, "fn": 1}, "all": {"tp": 2, "fp": 6, "fn": 1}}, "sourcery": {"strict": {"tp": 3, "fp": 0, "fn": 0}, "core": {"tp": 3, "fp": 0, "fn": 0}, "all": {"tp": 3, "fp": 0, "fn": 0}}}}, {"url": "https://github.com/grafana/grafana/pull/106778", "language": "Go", "pr_size": "medium", "domain": "UI", "change_type": "feature", "complexity": "moderate", "difficulty": "subtle", "risk": "medium", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 0, "fp": 2, "fn": 2}, "core": {"tp": 0, "fp": 2, "fn": 2}, "all": {"tp": 0, "fp": 2, "fn": 2}}, "codeant-v2": {"strict": {"tp": 1, "fp": 2, "fn": 1}, "core": {"tp": 1, "fp": 2, "fn": 1}, "all": {"tp": 1, "fp": 2, "fn": 1}}, "coderabbit": {"strict": {"tp": 1, "fp": 7, "fn": 1}, "core": {"tp": 1, "fp": 7, "fn": 1}, "all": {"tp": 1, "fp": 7, "fn": 1}}, "qodo-extended-v2": {"strict": {"tp": 2, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "greptile-v4-1": {"strict": {"tp": 1, "fp": 3, "fn": 1}, "core": {"tp": 1, "fp": 3, "fn": 1}, "all": {"tp": 1, "fp": 3, "fn": 1}}, "gemini-v2": {"strict": {"tp": 0, "fp": 3, "fn": 2}, "core": {"tp": 0, "fp": 3, "fn": 2}, "all": {"tp": 0, "fp": 3, "fn": 2}}, "gitlab": {"strict": {"tp": 2, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "copilot-v2": {"strict": {"tp": 2, "fp": 2, "fn": 0}, "core": {"tp": 2, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "augment": {"strict": {"tp": 1, "fp": 2, "fn": 1}, "core": {"tp": 1, "fp": 2, "fn": 1}, "all": {"tp": 1, "fp": 2, "fn": 1}}, "bugbot": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "claude": {"strict": {"tp": 0, "fp": 9, "fn": 2}, "core": {"tp": 0, "fp": 9, "fn": 2}, "all": {"tp": 0, "fp": 9, "fn": 2}}, "claude-code": {"strict": {"tp": 2, "fp": 2, "fn": 0}, "core": {"tp": 2, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "cubic-v2": {"strict": {"tp": 2, "fp": 2, "fn": 0}, "core": {"tp": 2, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "devin": {"strict": {"tp": 2, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "kg": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "kodus-v2": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "macroscope": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "qodo-v2": {"strict": {"tp": 1, "fp": 2, "fn": 1}, "core": {"tp": 1, "fp": 2, "fn": 1}, "all": {"tp": 1, "fp": 2, "fn": 1}}, "sourcery": {"strict": {"tp": 0, "fp": 2, "fn": 2}, "core": {"tp": 0, "fp": 2, "fn": 2}, "all": {"tp": 0, "fp": 2, "fn": 2}}}}, {"url": "https://github.com/grafana/grafana/pull/107534", "language": "Go", "pr_size": "small", "domain": "testing", "change_type": "feature", "complexity": "moderate", "difficulty": "subtle", "risk": "low", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 0, "fp": 2, "fn": 0}, "core": {"tp": 0, "fp": 2, "fn": 1}, "all": {"tp": 0, "fp": 2, "fn": 1}}, "codeant-v2": {"strict": {"tp": 0, "fp": 0, "fn": 0}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 0, "fp": 0, "fn": 1}}, "coderabbit": {"strict": {"tp": 0, "fp": 1, "fn": 0}, "core": {"tp": 0, "fp": 1, "fn": 1}, "all": {"tp": 0, "fp": 1, "fn": 1}}, "qodo-extended-v2": {"strict": {"tp": 0, "fp": 3, "fn": 0}, "core": {"tp": 0, "fp": 3, "fn": 1}, "all": {"tp": 0, "fp": 3, "fn": 1}}, "greptile-v4-1": {"strict": {"tp": 0, "fp": 5, "fn": 0}, "core": {"tp": 0, "fp": 5, "fn": 1}, "all": {"tp": 0, "fp": 5, "fn": 1}}, "gemini-v2": {"strict": {"tp": 0, "fp": 6, "fn": 0}, "core": {"tp": 0, "fp": 6, "fn": 1}, "all": {"tp": 0, "fp": 6, "fn": 1}}, "gitlab": {"strict": {"tp": 0, "fp": 2, "fn": 0}, "core": {"tp": 0, "fp": 2, "fn": 1}, "all": {"tp": 0, "fp": 2, "fn": 1}}, "copilot-v2": {"strict": {"tp": 0, "fp": 4, "fn": 0}, "core": {"tp": 0, "fp": 4, "fn": 1}, "all": {"tp": 0, "fp": 4, "fn": 1}}, "augment": {"strict": {"tp": 0, "fp": 0, "fn": 0}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 0, "fp": 0, "fn": 1}}, "bugbot": {"strict": {"tp": 0, "fp": 2, "fn": 0}, "core": {"tp": 0, "fp": 2, "fn": 1}, "all": {"tp": 0, "fp": 2, "fn": 1}}, "claude": {"strict": {"tp": 0, "fp": 0, "fn": 0}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 0, "fp": 0, "fn": 1}}, "claude-code": {"strict": {"tp": 0, "fp": 4, "fn": 0}, "core": {"tp": 0, "fp": 4, "fn": 1}, "all": {"tp": 0, "fp": 4, "fn": 1}}, "cubic-v2": {"strict": {"tp": 0, "fp": 2, "fn": 0}, "core": {"tp": 1, "fp": 2, "fn": 0}, "all": {"tp": 1, "fp": 2, "fn": 0}}, "devin": {"strict": {"tp": 0, "fp": 2, "fn": 0}, "core": {"tp": 0, "fp": 2, "fn": 1}, "all": {"tp": 0, "fp": 2, "fn": 1}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 0}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 0, "fp": 0, "fn": 1}}, "kg": {"strict": {"tp": 0, "fp": 2, "fn": 0}, "core": {"tp": 0, "fp": 2, "fn": 1}, "all": {"tp": 0, "fp": 2, "fn": 1}}, "kodus-v2": {"strict": {"tp": 0, "fp": 0, "fn": 0}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 0, "fp": 0, "fn": 1}}, "macroscope": {"strict": {"tp": 0, "fp": 0, "fn": 0}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 0, "fp": 0, "fn": 1}}, "qodo-v2": {"strict": {"tp": 0, "fp": 2, "fn": 0}, "core": {"tp": 0, "fp": 2, "fn": 1}, "all": {"tp": 0, "fp": 2, "fn": 1}}, "sourcery": {"strict": {"tp": 0, "fp": 3, "fn": 0}, "core": {"tp": 0, "fp": 3, "fn": 1}, "all": {"tp": 0, "fp": 3, "fn": 1}}}}, {"url": "https://github.com/grafana/grafana/pull/79265", "language": "Go", "pr_size": "large", "domain": "authentication", "change_type": "feature", "complexity": "moderate", "difficulty": "subtle", "risk": "high", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 0, "fp": 0, "fn": 4}, "core": {"tp": 1, "fp": 0, "fn": 4}, "all": {"tp": 1, "fp": 0, "fn": 4}}, "codeant-v2": {"strict": {"tp": 1, "fp": 2, "fn": 3}, "core": {"tp": 1, "fp": 2, "fn": 4}, "all": {"tp": 1, "fp": 2, "fn": 4}}, "coderabbit": {"strict": {"tp": 3, "fp": 5, "fn": 1}, "core": {"tp": 4, "fp": 5, "fn": 1}, "all": {"tp": 4, "fp": 5, "fn": 1}}, "qodo-extended-v2": {"strict": {"tp": 2, "fp": 1, "fn": 2}, "core": {"tp": 2, "fp": 1, "fn": 3}, "all": {"tp": 2, "fp": 1, "fn": 3}}, "greptile-v4-1": {"strict": {"tp": 2, "fp": 2, "fn": 2}, "core": {"tp": 3, "fp": 2, "fn": 2}, "all": {"tp": 3, "fp": 2, "fn": 2}}, "gemini-v2": {"strict": {"tp": 0, "fp": 4, "fn": 4}, "core": {"tp": 0, "fp": 4, "fn": 5}, "all": {"tp": 0, "fp": 4, "fn": 5}}, "gitlab": {"strict": {"tp": 0, "fp": 2, "fn": 4}, "core": {"tp": 1, "fp": 2, "fn": 4}, "all": {"tp": 1, "fp": 2, "fn": 4}}, "copilot-v2": {"strict": {"tp": 1, "fp": 4, "fn": 3}, "core": {"tp": 1, "fp": 4, "fn": 4}, "all": {"tp": 1, "fp": 4, "fn": 4}}, "augment": {"strict": {"tp": 1, "fp": 0, "fn": 3}, "core": {"tp": 2, "fp": 0, "fn": 3}, "all": {"tp": 2, "fp": 0, "fn": 3}}, "bugbot": {"strict": {"tp": 1, "fp": 1, "fn": 3}, "core": {"tp": 1, "fp": 1, "fn": 4}, "all": {"tp": 1, "fp": 1, "fn": 4}}, "claude": {"strict": {"tp": 1, "fp": 2, "fn": 3}, "core": {"tp": 1, "fp": 2, "fn": 4}, "all": {"tp": 1, "fp": 2, "fn": 4}}, "claude-code": {"strict": {"tp": 1, "fp": 1, "fn": 3}, "core": {"tp": 1, "fp": 1, "fn": 4}, "all": {"tp": 1, "fp": 1, "fn": 4}}, "cubic-v2": {"strict": {"tp": 3, "fp": 1, "fn": 1}, "core": {"tp": 4, "fp": 1, "fn": 1}, "all": {"tp": 4, "fp": 1, "fn": 1}}, "devin": {"strict": {"tp": 2, "fp": 0, "fn": 2}, "core": {"tp": 2, "fp": 0, "fn": 3}, "all": {"tp": 2, "fp": 0, "fn": 3}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 4}, "core": {"tp": 0, "fp": 0, "fn": 5}, "all": {"tp": 0, "fp": 0, "fn": 5}}, "kg": {"strict": {"tp": 1, "fp": 0, "fn": 3}, "core": {"tp": 1, "fp": 0, "fn": 4}, "all": {"tp": 1, "fp": 0, "fn": 4}}, "kodus-v2": {"strict": {"tp": 0, "fp": 1, "fn": 4}, "core": {"tp": 1, "fp": 1, "fn": 4}, "all": {"tp": 1, "fp": 1, "fn": 4}}, "macroscope": {"strict": {"tp": 1, "fp": 0, "fn": 3}, "core": {"tp": 1, "fp": 0, "fn": 4}, "all": {"tp": 1, "fp": 0, "fn": 4}}, "qodo-v2": {"strict": {"tp": 1, "fp": 2, "fn": 3}, "core": {"tp": 2, "fp": 2, "fn": 3}, "all": {"tp": 2, "fp": 2, "fn": 3}}, "sourcery": {"strict": {"tp": 2, "fp": 3, "fn": 2}, "core": {"tp": 3, "fp": 3, "fn": 2}, "all": {"tp": 3, "fp": 3, "fn": 2}}}}, {"url": "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/9", "language": "Ruby", "pr_size": "small", "domain": "configuration", "change_type": "feature", "complexity": "moderate", "difficulty": "subtle", "risk": "medium", "context": "file", "concern": "reliability", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "codeant-v2": {"strict": {"tp": 0, "fp": 3, "fn": 1}, "core": {"tp": 0, "fp": 3, "fn": 2}, "all": {"tp": 0, "fp": 3, "fn": 2}}, "coderabbit": {"strict": {"tp": 0, "fp": 2, "fn": 1}, "core": {"tp": 0, "fp": 2, "fn": 2}, "all": {"tp": 0, "fp": 2, "fn": 2}}, "qodo-extended-v2": {"strict": {"tp": 0, "fp": 1, "fn": 1}, "core": {"tp": 0, "fp": 1, "fn": 2}, "all": {"tp": 0, "fp": 1, "fn": 2}}, "greptile-v4-1": {"strict": {"tp": 0, "fp": 2, "fn": 1}, "core": {"tp": 0, "fp": 2, "fn": 2}, "all": {"tp": 0, "fp": 2, "fn": 2}}, "gemini-v2": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "gitlab": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "copilot-v2": {"strict": {"tp": 0, "fp": 3, "fn": 1}, "core": {"tp": 0, "fp": 3, "fn": 2}, "all": {"tp": 0, "fp": 3, "fn": 2}}, "augment": {"strict": {"tp": 0, "fp": 2, "fn": 1}, "core": {"tp": 1, "fp": 2, "fn": 1}, "all": {"tp": 1, "fp": 2, "fn": 1}}, "bugbot": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "claude": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "claude-code": {"strict": {"tp": 0, "fp": 5, "fn": 1}, "core": {"tp": 0, "fp": 5, "fn": 2}, "all": {"tp": 0, "fp": 5, "fn": 2}}, "cubic-v2": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "devin": {"strict": {"tp": 0, "fp": 1, "fn": 1}, "core": {"tp": 0, "fp": 1, "fn": 2}, "all": {"tp": 0, "fp": 1, "fn": 2}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "kodus-v2": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "macroscope": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "qodo-v2": {"strict": {"tp": 0, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "sourcery": {"strict": {"tp": 0, "fp": 2, "fn": 1}, "core": {"tp": 0, "fp": 2, "fn": 2}, "all": {"tp": 0, "fp": 2, "fn": 2}}}}, {"url": "https://github.com/grafana/grafana/pull/76186", "language": "Go", "pr_size": "small", "domain": "logging", "change_type": "refactoring", "complexity": "moderate", "difficulty": "subtle", "risk": "high", "context": "cross_file", "concern": "reliability", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 0, "fp": 0, "fn": 3}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 3}}, "codeant-v2": {"strict": {"tp": 0, "fp": 3, "fn": 3}, "core": {"tp": 0, "fp": 3, "fn": 3}, "all": {"tp": 0, "fp": 3, "fn": 3}}, "coderabbit": {"strict": {"tp": 1, "fp": 0, "fn": 2}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 2}}, "qodo-extended-v2": {"strict": {"tp": 1, "fp": 0, "fn": 2}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 2}}, "greptile-v4-1": {"strict": {"tp": 1, "fp": 0, "fn": 2}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 2}}, "gemini-v2": {"strict": {"tp": 1, "fp": 2, "fn": 2}, "core": {"tp": 1, "fp": 2, "fn": 2}, "all": {"tp": 1, "fp": 2, "fn": 2}}, "gitlab": {"strict": {"tp": 3, "fp": 3, "fn": 0}, "core": {"tp": 3, "fp": 3, "fn": 0}, "all": {"tp": 3, "fp": 3, "fn": 0}}, "copilot-v2": {"strict": {"tp": 0, "fp": 2, "fn": 3}, "core": {"tp": 0, "fp": 2, "fn": 3}, "all": {"tp": 0, "fp": 2, "fn": 3}}, "augment": {"strict": {"tp": 2, "fp": 0, "fn": 1}, "core": {"tp": 2, "fp": 0, "fn": 1}, "all": {"tp": 2, "fp": 0, "fn": 1}}, "bugbot": {"strict": {"tp": 1, "fp": 0, "fn": 2}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 2}}, "claude": {"strict": {"tp": 0, "fp": 0, "fn": 3}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 3}}, "claude-code": {"strict": {"tp": 1, "fp": 1, "fn": 2}, "core": {"tp": 1, "fp": 1, "fn": 2}, "all": {"tp": 1, "fp": 1, "fn": 2}}, "cubic-v2": {"strict": {"tp": 3, "fp": 0, "fn": 0}, "core": {"tp": 3, "fp": 0, "fn": 0}, "all": {"tp": 3, "fp": 0, "fn": 0}}, "devin": {"strict": {"tp": 1, "fp": 0, "fn": 2}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 2}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 3}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 3}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 3}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 3}}, "kodus-v2": {"strict": {"tp": 2, "fp": 0, "fn": 1}, "core": {"tp": 2, "fp": 0, "fn": 1}, "all": {"tp": 2, "fp": 0, "fn": 1}}, "macroscope": {"strict": {"tp": 1, "fp": 0, "fn": 2}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 2}}, "qodo-v2": {"strict": {"tp": 2, "fp": 0, "fn": 1}, "core": {"tp": 2, "fp": 0, "fn": 1}, "all": {"tp": 2, "fp": 0, "fn": 1}}, "sourcery": {"strict": {"tp": 0, "fp": 3, "fn": 3}, "core": {"tp": 0, "fp": 3, "fn": 3}, "all": {"tp": 0, "fp": 3, "fn": 3}}}}, {"url": "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/10", "language": "Ruby", "pr_size": "large", "domain": "configuration", "change_type": "feature", "complexity": "moderate", "difficulty": "subtle", "risk": "high", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 2, "fp": 1, "fn": 5}, "core": {"tp": 2, "fp": 1, "fn": 5}, "all": {"tp": 2, "fp": 1, "fn": 5}}, "codeant-v2": {"strict": {"tp": 2, "fp": 7, "fn": 5}, "core": {"tp": 2, "fp": 7, "fn": 5}, "all": {"tp": 2, "fp": 7, "fn": 5}}, "coderabbit": {"strict": {"tp": 5, "fp": 14, "fn": 2}, "core": {"tp": 5, "fp": 14, "fn": 2}, "all": {"tp": 5, "fp": 14, "fn": 2}}, "qodo-extended-v2": {"strict": {"tp": 4, "fp": 6, "fn": 3}, "core": {"tp": 4, "fp": 6, "fn": 3}, "all": {"tp": 4, "fp": 6, "fn": 3}}, "greptile-v4-1": {"strict": {"tp": 2, "fp": 3, "fn": 5}, "core": {"tp": 2, "fp": 3, "fn": 5}, "all": {"tp": 2, "fp": 3, "fn": 5}}, "gemini-v2": {"strict": {"tp": 4, "fp": 2, "fn": 3}, "core": {"tp": 4, "fp": 2, "fn": 3}, "all": {"tp": 4, "fp": 2, "fn": 3}}, "gitlab": {"strict": {"tp": 2, "fp": 3, "fn": 5}, "core": {"tp": 2, "fp": 3, "fn": 5}, "all": {"tp": 2, "fp": 3, "fn": 5}}, "copilot-v2": {"strict": {"tp": 6, "fp": 5, "fn": 1}, "core": {"tp": 6, "fp": 5, "fn": 1}, "all": {"tp": 6, "fp": 5, "fn": 1}}, "augment": {"strict": {"tp": 4, "fp": 1, "fn": 3}, "core": {"tp": 4, "fp": 1, "fn": 3}, "all": {"tp": 4, "fp": 1, "fn": 3}}, "bugbot": {"strict": {"tp": 5, "fp": 2, "fn": 2}, "core": {"tp": 5, "fp": 2, "fn": 2}, "all": {"tp": 5, "fp": 2, "fn": 2}}, "claude": {"strict": {"tp": 1, "fp": 2, "fn": 6}, "core": {"tp": 1, "fp": 2, "fn": 6}, "all": {"tp": 1, "fp": 2, "fn": 6}}, "claude-code": {"strict": {"tp": 1, "fp": 5, "fn": 6}, "core": {"tp": 1, "fp": 5, "fn": 6}, "all": {"tp": 1, "fp": 5, "fn": 6}}, "cubic-v2": {"strict": {"tp": 3, "fp": 1, "fn": 4}, "core": {"tp": 3, "fp": 1, "fn": 4}, "all": {"tp": 3, "fp": 1, "fn": 4}}, "devin": {"strict": {"tp": 2, "fp": 0, "fn": 5}, "core": {"tp": 2, "fp": 0, "fn": 5}, "all": {"tp": 2, "fp": 0, "fn": 5}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 7}, "core": {"tp": 0, "fp": 0, "fn": 7}, "all": {"tp": 0, "fp": 0, "fn": 7}}, "kg": {"strict": {"tp": 1, "fp": 0, "fn": 6}, "core": {"tp": 1, "fp": 0, "fn": 6}, "all": {"tp": 1, "fp": 0, "fn": 6}}, "kodus-v2": {"strict": {"tp": 5, "fp": 4, "fn": 2}, "core": {"tp": 5, "fp": 4, "fn": 2}, "all": {"tp": 5, "fp": 4, "fn": 2}}, "macroscope": {"strict": {"tp": 4, "fp": 3, "fn": 3}, "core": {"tp": 4, "fp": 3, "fn": 3}, "all": {"tp": 4, "fp": 3, "fn": 3}}, "qodo-v2": {"strict": {"tp": 4, "fp": 6, "fn": 3}, "core": {"tp": 4, "fp": 6, "fn": 3}, "all": {"tp": 4, "fp": 6, "fn": 3}}, "sourcery": {"strict": {"tp": 5, "fp": 7, "fn": 2}, "core": {"tp": 5, "fp": 7, "fn": 2}, "all": {"tp": 5, "fp": 7, "fn": 2}}}}, {"url": "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/7", "language": "Ruby", "pr_size": "medium", "domain": "UI", "change_type": "bug_fix", "complexity": "simple", "difficulty": "subtle", "risk": "medium", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 3, "fp": 1, "fn": 0}, "core": {"tp": 3, "fp": 1, "fn": 0}, "all": {"tp": 5, "fp": 1, "fn": 0}}, "codeant-v2": {"strict": {"tp": 0, "fp": 9, "fn": 3}, "core": {"tp": 0, "fp": 9, "fn": 3}, "all": {"tp": 0, "fp": 9, "fn": 5}}, "coderabbit": {"strict": {"tp": 0, "fp": 2, "fn": 3}, "core": {"tp": 0, "fp": 2, "fn": 3}, "all": {"tp": 0, "fp": 2, "fn": 5}}, "qodo-extended-v2": {"strict": {"tp": 2, "fp": 2, "fn": 1}, "core": {"tp": 2, "fp": 2, "fn": 1}, "all": {"tp": 3, "fp": 2, "fn": 2}}, "greptile-v4-1": {"strict": {"tp": 3, "fp": 0, "fn": 0}, "core": {"tp": 3, "fp": 0, "fn": 0}, "all": {"tp": 5, "fp": 0, "fn": 0}}, "gemini-v2": {"strict": {"tp": 3, "fp": 1, "fn": 0}, "core": {"tp": 3, "fp": 1, "fn": 0}, "all": {"tp": 5, "fp": 1, "fn": 0}}, "gitlab": {"strict": {"tp": 3, "fp": 0, "fn": 0}, "core": {"tp": 3, "fp": 0, "fn": 0}, "all": {"tp": 4, "fp": 0, "fn": 1}}, "copilot-v2": {"strict": {"tp": 3, "fp": 1, "fn": 0}, "core": {"tp": 3, "fp": 1, "fn": 0}, "all": {"tp": 5, "fp": 1, "fn": 0}}, "augment": {"strict": {"tp": 3, "fp": 2, "fn": 0}, "core": {"tp": 3, "fp": 2, "fn": 0}, "all": {"tp": 5, "fp": 2, "fn": 0}}, "bugbot": {"strict": {"tp": 2, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 1}, "all": {"tp": 4, "fp": 1, "fn": 1}}, "claude": {"strict": {"tp": 3, "fp": 3, "fn": 0}, "core": {"tp": 3, "fp": 3, "fn": 0}, "all": {"tp": 5, "fp": 3, "fn": 0}}, "claude-code": {"strict": {"tp": 2, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 1}, "all": {"tp": 4, "fp": 1, "fn": 1}}, "cubic-v2": {"strict": {"tp": 2, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 1}, "all": {"tp": 4, "fp": 1, "fn": 1}}, "devin": {"strict": {"tp": 2, "fp": 3, "fn": 1}, "core": {"tp": 2, "fp": 3, "fn": 1}, "all": {"tp": 4, "fp": 3, "fn": 1}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 3}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 5}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 3}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 5}}, "kodus-v2": {"strict": {"tp": 1, "fp": 0, "fn": 2}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 3, "fp": 0, "fn": 2}}, "macroscope": {"strict": {"tp": 1, "fp": 0, "fn": 2}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 2, "fp": 0, "fn": 3}}, "qodo-v2": {"strict": {"tp": 3, "fp": 1, "fn": 0}, "core": {"tp": 3, "fp": 1, "fn": 0}, "all": {"tp": 5, "fp": 1, "fn": 0}}, "sourcery": {"strict": {"tp": 2, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 1}, "all": {"tp": 3, "fp": 1, "fn": 2}}}}, {"url": "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/8", "language": "Ruby", "pr_size": "medium", "domain": "API", "change_type": "bug_fix", "complexity": "moderate", "difficulty": "subtle", "risk": "medium", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 2, "fp": 2, "fn": 4}, "core": {"tp": 2, "fp": 2, "fn": 4}, "all": {"tp": 2, "fp": 2, "fn": 4}}, "codeant-v2": {"strict": {"tp": 5, "fp": 7, "fn": 1}, "core": {"tp": 5, "fp": 7, "fn": 1}, "all": {"tp": 5, "fp": 7, "fn": 1}}, "coderabbit": {"strict": {"tp": 4, "fp": 6, "fn": 2}, "core": {"tp": 4, "fp": 6, "fn": 2}, "all": {"tp": 4, "fp": 6, "fn": 2}}, "qodo-extended-v2": {"strict": {"tp": 3, "fp": 2, "fn": 3}, "core": {"tp": 3, "fp": 2, "fn": 3}, "all": {"tp": 3, "fp": 2, "fn": 3}}, "greptile-v4-1": {"strict": {"tp": 4, "fp": 1, "fn": 2}, "core": {"tp": 4, "fp": 1, "fn": 2}, "all": {"tp": 4, "fp": 1, "fn": 2}}, "gemini-v2": {"strict": {"tp": 3, "fp": 2, "fn": 3}, "core": {"tp": 3, "fp": 2, "fn": 3}, "all": {"tp": 3, "fp": 2, "fn": 3}}, "gitlab": {"strict": {"tp": 4, "fp": 1, "fn": 2}, "core": {"tp": 4, "fp": 1, "fn": 2}, "all": {"tp": 4, "fp": 1, "fn": 2}}, "copilot-v2": {"strict": {"tp": 3, "fp": 8, "fn": 3}, "core": {"tp": 3, "fp": 8, "fn": 3}, "all": {"tp": 3, "fp": 8, "fn": 3}}, "augment": {"strict": {"tp": 3, "fp": 3, "fn": 3}, "core": {"tp": 3, "fp": 3, "fn": 3}, "all": {"tp": 3, "fp": 3, "fn": 3}}, "bugbot": {"strict": {"tp": 2, "fp": 1, "fn": 4}, "core": {"tp": 2, "fp": 1, "fn": 4}, "all": {"tp": 2, "fp": 1, "fn": 4}}, "claude": {"strict": {"tp": 4, "fp": 4, "fn": 2}, "core": {"tp": 4, "fp": 4, "fn": 2}, "all": {"tp": 4, "fp": 4, "fn": 2}}, "claude-code": {"strict": {"tp": 2, "fp": 1, "fn": 4}, "core": {"tp": 2, "fp": 1, "fn": 4}, "all": {"tp": 2, "fp": 1, "fn": 4}}, "cubic-v2": {"strict": {"tp": 3, "fp": 1, "fn": 3}, "core": {"tp": 3, "fp": 1, "fn": 3}, "all": {"tp": 3, "fp": 1, "fn": 3}}, "devin": {"strict": {"tp": 2, "fp": 0, "fn": 4}, "core": {"tp": 2, "fp": 0, "fn": 4}, "all": {"tp": 2, "fp": 0, "fn": 4}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 6}, "core": {"tp": 0, "fp": 0, "fn": 6}, "all": {"tp": 0, "fp": 0, "fn": 6}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 6}, "core": {"tp": 0, "fp": 0, "fn": 6}, "all": {"tp": 0, "fp": 0, "fn": 6}}, "kodus-v2": {"strict": {"tp": 1, "fp": 6, "fn": 5}, "core": {"tp": 1, "fp": 6, "fn": 5}, "all": {"tp": 1, "fp": 6, "fn": 5}}, "macroscope": {"strict": {"tp": 2, "fp": 2, "fn": 4}, "core": {"tp": 2, "fp": 2, "fn": 4}, "all": {"tp": 2, "fp": 2, "fn": 4}}, "qodo-v2": {"strict": {"tp": 2, "fp": 2, "fn": 4}, "core": {"tp": 2, "fp": 2, "fn": 4}, "all": {"tp": 2, "fp": 2, "fn": 4}}, "sourcery": {"strict": {"tp": 3, "fp": 2, "fn": 3}, "core": {"tp": 3, "fp": 2, "fn": 3}, "all": {"tp": 3, "fp": 2, "fn": 3}}}}, {"url": "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/3", "language": "Ruby", "pr_size": "medium", "domain": "authentication", "change_type": "feature", "complexity": "moderate", "difficulty": "subtle", "risk": "medium", "context": "file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 1, "fp": 1, "fn": 2}, "core": {"tp": 1, "fp": 1, "fn": 2}, "all": {"tp": 1, "fp": 1, "fn": 2}}, "codeant-v2": {"strict": {"tp": 1, "fp": 3, "fn": 2}, "core": {"tp": 1, "fp": 3, "fn": 2}, "all": {"tp": 1, "fp": 3, "fn": 2}}, "coderabbit": {"strict": {"tp": 2, "fp": 5, "fn": 1}, "core": {"tp": 2, "fp": 5, "fn": 1}, "all": {"tp": 2, "fp": 5, "fn": 1}}, "qodo-extended-v2": {"strict": {"tp": 2, "fp": 0, "fn": 1}, "core": {"tp": 2, "fp": 0, "fn": 1}, "all": {"tp": 2, "fp": 0, "fn": 1}}, "greptile-v4-1": {"strict": {"tp": 1, "fp": 3, "fn": 2}, "core": {"tp": 1, "fp": 3, "fn": 2}, "all": {"tp": 1, "fp": 3, "fn": 2}}, "gemini-v2": {"strict": {"tp": 2, "fp": 3, "fn": 1}, "core": {"tp": 2, "fp": 3, "fn": 1}, "all": {"tp": 2, "fp": 3, "fn": 1}}, "gitlab": {"strict": {"tp": 0, "fp": 2, "fn": 3}, "core": {"tp": 0, "fp": 2, "fn": 3}, "all": {"tp": 0, "fp": 2, "fn": 3}}, "copilot-v2": {"strict": {"tp": 2, "fp": 6, "fn": 1}, "core": {"tp": 2, "fp": 6, "fn": 1}, "all": {"tp": 2, "fp": 6, "fn": 1}}, "augment": {"strict": {"tp": 1, "fp": 5, "fn": 2}, "core": {"tp": 1, "fp": 5, "fn": 2}, "all": {"tp": 1, "fp": 5, "fn": 2}}, "bugbot": {"strict": {"tp": 1, "fp": 1, "fn": 2}, "core": {"tp": 1, "fp": 1, "fn": 2}, "all": {"tp": 1, "fp": 1, "fn": 2}}, "claude": {"strict": {"tp": 1, "fp": 4, "fn": 2}, "core": {"tp": 1, "fp": 4, "fn": 2}, "all": {"tp": 1, "fp": 4, "fn": 2}}, "claude-code": {"strict": {"tp": 1, "fp": 4, "fn": 2}, "core": {"tp": 1, "fp": 4, "fn": 2}, "all": {"tp": 1, "fp": 4, "fn": 2}}, "cubic-v2": {"strict": {"tp": 2, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 1}, "all": {"tp": 2, "fp": 1, "fn": 1}}, "devin": {"strict": {"tp": 1, "fp": 1, "fn": 2}, "core": {"tp": 1, "fp": 1, "fn": 2}, "all": {"tp": 1, "fp": 1, "fn": 2}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 3}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 3}}, "kg": {"strict": {"tp": 1, "fp": 0, "fn": 2}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 2}}, "kodus-v2": {"strict": {"tp": 1, "fp": 0, "fn": 2}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 2}}, "macroscope": {"strict": {"tp": 2, "fp": 0, "fn": 1}, "core": {"tp": 2, "fp": 0, "fn": 1}, "all": {"tp": 2, "fp": 0, "fn": 1}}, "qodo-v2": {"strict": {"tp": 2, "fp": 3, "fn": 1}, "core": {"tp": 2, "fp": 3, "fn": 1}, "all": {"tp": 2, "fp": 3, "fn": 1}}, "sourcery": {"strict": {"tp": 1, "fp": 3, "fn": 2}, "core": {"tp": 1, "fp": 3, "fn": 2}, "all": {"tp": 1, "fp": 3, "fn": 2}}}}, {"url": "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/5", "language": "Ruby", "pr_size": "small", "domain": "UI", "change_type": "performance", "complexity": "moderate", "difficulty": "subtle", "risk": "medium", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 1, "fp": 1, "fn": 2}, "core": {"tp": 1, "fp": 1, "fn": 2}, "all": {"tp": 1, "fp": 1, "fn": 2}}, "codeant-v2": {"strict": {"tp": 0, "fp": 0, "fn": 3}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 3}}, "coderabbit": {"strict": {"tp": 1, "fp": 0, "fn": 2}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 2}}, "qodo-extended-v2": {"strict": {"tp": 2, "fp": 0, "fn": 1}, "core": {"tp": 2, "fp": 0, "fn": 1}, "all": {"tp": 2, "fp": 0, "fn": 1}}, "greptile-v4-1": {"strict": {"tp": 1, "fp": 2, "fn": 2}, "core": {"tp": 1, "fp": 2, "fn": 2}, "all": {"tp": 1, "fp": 2, "fn": 2}}, "gemini-v2": {"strict": {"tp": 2, "fp": 7, "fn": 1}, "core": {"tp": 2, "fp": 7, "fn": 1}, "all": {"tp": 2, "fp": 7, "fn": 1}}, "gitlab": {"strict": {"tp": 1, "fp": 1, "fn": 2}, "core": {"tp": 1, "fp": 1, "fn": 2}, "all": {"tp": 1, "fp": 1, "fn": 2}}, "copilot-v2": {"strict": {"tp": 2, "fp": 2, "fn": 1}, "core": {"tp": 2, "fp": 2, "fn": 1}, "all": {"tp": 2, "fp": 2, "fn": 1}}, "augment": {"strict": {"tp": 2, "fp": 0, "fn": 1}, "core": {"tp": 2, "fp": 0, "fn": 1}, "all": {"tp": 2, "fp": 0, "fn": 1}}, "bugbot": {"strict": {"tp": 1, "fp": 0, "fn": 2}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 2}}, "claude": {"strict": {"tp": 1, "fp": 0, "fn": 2}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 2}}, "claude-code": {"strict": {"tp": 2, "fp": 3, "fn": 1}, "core": {"tp": 2, "fp": 3, "fn": 1}, "all": {"tp": 2, "fp": 3, "fn": 1}}, "cubic-v2": {"strict": {"tp": 2, "fp": 2, "fn": 1}, "core": {"tp": 2, "fp": 2, "fn": 1}, "all": {"tp": 2, "fp": 2, "fn": 1}}, "devin": {"strict": {"tp": 1, "fp": 1, "fn": 2}, "core": {"tp": 1, "fp": 1, "fn": 2}, "all": {"tp": 1, "fp": 1, "fn": 2}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 3}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 3}}, "kg": {"strict": {"tp": 1, "fp": 0, "fn": 2}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 2}}, "kodus-v2": {"strict": {"tp": 0, "fp": 0, "fn": 3}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 3}}, "macroscope": {"strict": {"tp": 0, "fp": 0, "fn": 3}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 3}}, "qodo-v2": {"strict": {"tp": 1, "fp": 1, "fn": 2}, "core": {"tp": 1, "fp": 1, "fn": 2}, "all": {"tp": 1, "fp": 1, "fn": 2}}, "sourcery": {"strict": {"tp": 2, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 1}, "all": {"tp": 2, "fp": 1, "fn": 1}}}}, {"url": "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/6", "language": "Ruby", "pr_size": "small", "domain": "serialization", "change_type": "feature", "complexity": "simple", "difficulty": "moderate", "risk": "medium", "context": "file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 0, "fp": 5, "fn": 1}, "core": {"tp": 0, "fp": 5, "fn": 1}, "all": {"tp": 0, "fp": 5, "fn": 1}}, "codeant-v2": {"strict": {"tp": 1, "fp": 2, "fn": 0}, "core": {"tp": 1, "fp": 2, "fn": 0}, "all": {"tp": 1, "fp": 2, "fn": 0}}, "coderabbit": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 1, "fp": 1, "fn": 0}, "all": {"tp": 1, "fp": 1, "fn": 0}}, "qodo-extended-v2": {"strict": {"tp": 0, "fp": 2, "fn": 1}, "core": {"tp": 0, "fp": 2, "fn": 1}, "all": {"tp": 0, "fp": 2, "fn": 1}}, "greptile-v4-1": {"strict": {"tp": 0, "fp": 4, "fn": 1}, "core": {"tp": 0, "fp": 4, "fn": 1}, "all": {"tp": 0, "fp": 4, "fn": 1}}, "gemini-v2": {"strict": {"tp": 0, "fp": 6, "fn": 1}, "core": {"tp": 0, "fp": 6, "fn": 1}, "all": {"tp": 0, "fp": 6, "fn": 1}}, "gitlab": {"strict": {"tp": 0, "fp": 2, "fn": 1}, "core": {"tp": 0, "fp": 2, "fn": 1}, "all": {"tp": 0, "fp": 2, "fn": 1}}, "copilot-v2": {"strict": {"tp": 1, "fp": 3, "fn": 0}, "core": {"tp": 1, "fp": 3, "fn": 0}, "all": {"tp": 1, "fp": 3, "fn": 0}}, "augment": {"strict": {"tp": 0, "fp": 3, "fn": 1}, "core": {"tp": 0, "fp": 3, "fn": 1}, "all": {"tp": 0, "fp": 3, "fn": 1}}, "bugbot": {"strict": {"tp": 0, "fp": 3, "fn": 1}, "core": {"tp": 0, "fp": 3, "fn": 1}, "all": {"tp": 0, "fp": 3, "fn": 1}}, "claude": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 0, "fp": 0, "fn": 1}}, "claude-code": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 0}, "all": {"tp": 1, "fp": 0, "fn": 0}}, "cubic-v2": {"strict": {"tp": 0, "fp": 2, "fn": 1}, "core": {"tp": 0, "fp": 2, "fn": 1}, "all": {"tp": 0, "fp": 2, "fn": 1}}, "devin": {"strict": {"tp": 0, "fp": 3, "fn": 1}, "core": {"tp": 0, "fp": 3, "fn": 1}, "all": {"tp": 0, "fp": 3, "fn": 1}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 0, "fp": 0, "fn": 1}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 0, "fp": 0, "fn": 1}}, "kodus-v2": {"strict": {"tp": 0, "fp": 2, "fn": 1}, "core": {"tp": 0, "fp": 2, "fn": 1}, "all": {"tp": 0, "fp": 2, "fn": 1}}, "macroscope": {"strict": {"tp": 0, "fp": 2, "fn": 1}, "core": {"tp": 0, "fp": 2, "fn": 1}, "all": {"tp": 0, "fp": 2, "fn": 1}}, "qodo-v2": {"strict": {"tp": 1, "fp": 2, "fn": 0}, "core": {"tp": 1, "fp": 2, "fn": 0}, "all": {"tp": 1, "fp": 2, "fn": 0}}, "sourcery": {"strict": {"tp": 0, "fp": 5, "fn": 1}, "core": {"tp": 0, "fp": 5, "fn": 1}, "all": {"tp": 0, "fp": 5, "fn": 1}}}}, {"url": "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/4", "language": "Ruby", "pr_size": "large", "domain": "API", "change_type": "feature", "complexity": "complex", "difficulty": "subtle", "risk": "critical", "context": "cross_file", "concern": "security", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 2, "fp": 2, "fn": 6}, "core": {"tp": 2, "fp": 2, "fn": 6}, "all": {"tp": 2, "fp": 2, "fn": 6}}, "codeant-v2": {"strict": {"tp": 1, "fp": 17, "fn": 7}, "core": {"tp": 1, "fp": 17, "fn": 7}, "all": {"tp": 1, "fp": 17, "fn": 7}}, "coderabbit": {"strict": {"tp": 4, "fp": 23, "fn": 4}, "core": {"tp": 4, "fp": 23, "fn": 4}, "all": {"tp": 4, "fp": 23, "fn": 4}}, "qodo-extended-v2": {"strict": {"tp": 3, "fp": 3, "fn": 5}, "core": {"tp": 3, "fp": 3, "fn": 5}, "all": {"tp": 3, "fp": 3, "fn": 5}}, "greptile-v4-1": {"strict": {"tp": 4, "fp": 1, "fn": 4}, "core": {"tp": 4, "fp": 1, "fn": 4}, "all": {"tp": 4, "fp": 1, "fn": 4}}, "gemini-v2": {"strict": {"tp": 4, "fp": 5, "fn": 4}, "core": {"tp": 4, "fp": 5, "fn": 4}, "all": {"tp": 4, "fp": 5, "fn": 4}}, "gitlab": {"strict": {"tp": 3, "fp": 2, "fn": 5}, "core": {"tp": 3, "fp": 2, "fn": 5}, "all": {"tp": 3, "fp": 2, "fn": 5}}, "copilot-v2": {"strict": {"tp": 5, "fp": 6, "fn": 3}, "core": {"tp": 5, "fp": 6, "fn": 3}, "all": {"tp": 5, "fp": 6, "fn": 3}}, "augment": {"strict": {"tp": 3, "fp": 2, "fn": 5}, "core": {"tp": 3, "fp": 2, "fn": 5}, "all": {"tp": 3, "fp": 2, "fn": 5}}, "bugbot": {"strict": {"tp": 2, "fp": 3, "fn": 6}, "core": {"tp": 2, "fp": 3, "fn": 6}, "all": {"tp": 2, "fp": 3, "fn": 6}}, "claude": {"strict": {"tp": 3, "fp": 3, "fn": 5}, "core": {"tp": 3, "fp": 3, "fn": 5}, "all": {"tp": 3, "fp": 3, "fn": 5}}, "claude-code": {"strict": {"tp": 3, "fp": 1, "fn": 5}, "core": {"tp": 3, "fp": 1, "fn": 5}, "all": {"tp": 3, "fp": 1, "fn": 5}}, "cubic-v2": {"strict": {"tp": 1, "fp": 3, "fn": 7}, "core": {"tp": 1, "fp": 3, "fn": 7}, "all": {"tp": 1, "fp": 3, "fn": 7}}, "devin": {"strict": {"tp": 3, "fp": 1, "fn": 5}, "core": {"tp": 3, "fp": 1, "fn": 5}, "all": {"tp": 3, "fp": 1, "fn": 5}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 8}, "core": {"tp": 0, "fp": 0, "fn": 8}, "all": {"tp": 0, "fp": 0, "fn": 8}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 8}, "core": {"tp": 0, "fp": 0, "fn": 8}, "all": {"tp": 0, "fp": 0, "fn": 8}}, "kodus-v2": {"strict": {"tp": 3, "fp": 2, "fn": 5}, "core": {"tp": 3, "fp": 2, "fn": 5}, "all": {"tp": 3, "fp": 2, "fn": 5}}, "macroscope": {"strict": {"tp": 1, "fp": 5, "fn": 7}, "core": {"tp": 1, "fp": 5, "fn": 7}, "all": {"tp": 1, "fp": 5, "fn": 7}}, "qodo-v2": {"strict": {"tp": 5, "fp": 13, "fn": 3}, "core": {"tp": 5, "fp": 13, "fn": 3}, "all": {"tp": 5, "fp": 13, "fn": 3}}, "sourcery": {"strict": {"tp": 2, "fp": 3, "fn": 6}, "core": {"tp": 2, "fp": 3, "fn": 6}, "all": {"tp": 2, "fp": 3, "fn": 6}}}}, {"url": "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/1", "language": "Ruby", "pr_size": "medium", "domain": "file_io", "change_type": "feature", "complexity": "moderate", "difficulty": "subtle", "risk": "medium", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 2, "fp": 1, "fn": 2}, "core": {"tp": 2, "fp": 1, "fn": 2}, "all": {"tp": 2, "fp": 1, "fn": 2}}, "codeant-v2": {"strict": {"tp": 2, "fp": 1, "fn": 2}, "core": {"tp": 2, "fp": 1, "fn": 2}, "all": {"tp": 2, "fp": 1, "fn": 2}}, "coderabbit": {"strict": {"tp": 3, "fp": 4, "fn": 1}, "core": {"tp": 3, "fp": 4, "fn": 1}, "all": {"tp": 3, "fp": 4, "fn": 1}}, "qodo-extended-v2": {"strict": {"tp": 2, "fp": 1, "fn": 2}, "core": {"tp": 2, "fp": 1, "fn": 2}, "all": {"tp": 2, "fp": 1, "fn": 2}}, "greptile-v4-1": {"strict": {"tp": 3, "fp": 1, "fn": 1}, "core": {"tp": 3, "fp": 1, "fn": 1}, "all": {"tp": 3, "fp": 1, "fn": 1}}, "gemini-v2": {"strict": {"tp": 3, "fp": 0, "fn": 1}, "core": {"tp": 3, "fp": 0, "fn": 1}, "all": {"tp": 3, "fp": 0, "fn": 1}}, "gitlab": {"strict": {"tp": 2, "fp": 0, "fn": 2}, "core": {"tp": 2, "fp": 0, "fn": 2}, "all": {"tp": 2, "fp": 0, "fn": 2}}, "copilot-v2": {"strict": {"tp": 2, "fp": 2, "fn": 2}, "core": {"tp": 2, "fp": 2, "fn": 2}, "all": {"tp": 2, "fp": 2, "fn": 2}}, "augment": {"strict": {"tp": 2, "fp": 4, "fn": 2}, "core": {"tp": 2, "fp": 4, "fn": 2}, "all": {"tp": 2, "fp": 4, "fn": 2}}, "bugbot": {"strict": {"tp": 2, "fp": 2, "fn": 2}, "core": {"tp": 2, "fp": 2, "fn": 2}, "all": {"tp": 2, "fp": 2, "fn": 2}}, "claude": {"strict": {"tp": 2, "fp": 0, "fn": 2}, "core": {"tp": 2, "fp": 0, "fn": 2}, "all": {"tp": 2, "fp": 0, "fn": 2}}, "claude-code": {"strict": {"tp": 3, "fp": 2, "fn": 1}, "core": {"tp": 3, "fp": 2, "fn": 1}, "all": {"tp": 3, "fp": 2, "fn": 1}}, "cubic-v2": {"strict": {"tp": 3, "fp": 0, "fn": 1}, "core": {"tp": 3, "fp": 0, "fn": 1}, "all": {"tp": 3, "fp": 0, "fn": 1}}, "devin": {"strict": {"tp": 2, "fp": 0, "fn": 2}, "core": {"tp": 2, "fp": 0, "fn": 2}, "all": {"tp": 2, "fp": 0, "fn": 2}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 4}, "core": {"tp": 0, "fp": 0, "fn": 4}, "all": {"tp": 0, "fp": 0, "fn": 4}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 4}, "core": {"tp": 0, "fp": 0, "fn": 4}, "all": {"tp": 0, "fp": 0, "fn": 4}}, "kodus-v2": {"strict": {"tp": 3, "fp": 1, "fn": 1}, "core": {"tp": 3, "fp": 1, "fn": 1}, "all": {"tp": 3, "fp": 1, "fn": 1}}, "macroscope": {"strict": {"tp": 1, "fp": 2, "fn": 3}, "core": {"tp": 1, "fp": 2, "fn": 3}, "all": {"tp": 1, "fp": 2, "fn": 3}}, "qodo-v2": {"strict": {"tp": 2, "fp": 4, "fn": 2}, "core": {"tp": 2, "fp": 4, "fn": 2}, "all": {"tp": 2, "fp": 4, "fn": 2}}, "sourcery": {"strict": {"tp": 2, "fp": 1, "fn": 2}, "core": {"tp": 2, "fp": 1, "fn": 2}, "all": {"tp": 2, "fp": 1, "fn": 2}}}}, {"url": "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/2", "language": "Ruby", "pr_size": "medium", "domain": "UI", "change_type": "feature", "complexity": "moderate", "difficulty": "moderate", "risk": "medium", "context": "file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 0}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "codeant-v2": {"strict": {"tp": 1, "fp": 5, "fn": 0}, "core": {"tp": 1, "fp": 5, "fn": 0}, "all": {"tp": 1, "fp": 5, "fn": 1}}, "coderabbit": {"strict": {"tp": 1, "fp": 7, "fn": 0}, "core": {"tp": 1, "fp": 7, "fn": 0}, "all": {"tp": 2, "fp": 7, "fn": 0}}, "qodo-extended-v2": {"strict": {"tp": 1, "fp": 2, "fn": 0}, "core": {"tp": 1, "fp": 2, "fn": 0}, "all": {"tp": 1, "fp": 2, "fn": 1}}, "greptile-v4-1": {"strict": {"tp": 1, "fp": 2, "fn": 0}, "core": {"tp": 1, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "gemini-v2": {"strict": {"tp": 1, "fp": 2, "fn": 0}, "core": {"tp": 1, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "gitlab": {"strict": {"tp": 1, "fp": 2, "fn": 0}, "core": {"tp": 1, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "copilot-v2": {"strict": {"tp": 1, "fp": 4, "fn": 0}, "core": {"tp": 1, "fp": 4, "fn": 0}, "all": {"tp": 2, "fp": 4, "fn": 0}}, "augment": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 1, "fp": 1, "fn": 0}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "bugbot": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 1, "fp": 1, "fn": 0}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "claude": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 1, "fp": 1, "fn": 0}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "claude-code": {"strict": {"tp": 0, "fp": 2, "fn": 1}, "core": {"tp": 0, "fp": 2, "fn": 1}, "all": {"tp": 0, "fp": 2, "fn": 2}}, "cubic-v2": {"strict": {"tp": 0, "fp": 2, "fn": 1}, "core": {"tp": 0, "fp": 2, "fn": 1}, "all": {"tp": 0, "fp": 2, "fn": 2}}, "devin": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 0}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "kodus-v2": {"strict": {"tp": 1, "fp": 2, "fn": 0}, "core": {"tp": 1, "fp": 2, "fn": 0}, "all": {"tp": 1, "fp": 2, "fn": 1}}, "macroscope": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 0}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "qodo-v2": {"strict": {"tp": 1, "fp": 3, "fn": 0}, "core": {"tp": 1, "fp": 3, "fn": 0}, "all": {"tp": 1, "fp": 3, "fn": 1}}, "sourcery": {"strict": {"tp": 1, "fp": 5, "fn": 0}, "core": {"tp": 1, "fp": 5, "fn": 0}, "all": {"tp": 1, "fp": 5, "fn": 1}}}}, {"url": "https://github.com/calcom/cal.com/pull/22532", "language": "TypeScript", "pr_size": "medium", "domain": "caching", "change_type": "feature", "complexity": "moderate", "difficulty": "subtle", "risk": "medium", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 1, "fp": 2, "fn": 3}, "core": {"tp": 1, "fp": 2, "fn": 3}, "all": {"tp": 1, "fp": 2, "fn": 3}}, "codeant-v2": {"strict": {"tp": 3, "fp": 3, "fn": 1}, "core": {"tp": 3, "fp": 3, "fn": 1}, "all": {"tp": 3, "fp": 3, "fn": 1}}, "coderabbit": {"strict": {"tp": 2, "fp": 3, "fn": 2}, "core": {"tp": 2, "fp": 3, "fn": 2}, "all": {"tp": 2, "fp": 3, "fn": 2}}, "qodo-extended-v2": {"strict": {"tp": 2, "fp": 2, "fn": 2}, "core": {"tp": 2, "fp": 2, "fn": 2}, "all": {"tp": 2, "fp": 2, "fn": 2}}, "greptile-v4-1": {"strict": {"tp": 2, "fp": 3, "fn": 2}, "core": {"tp": 2, "fp": 3, "fn": 2}, "all": {"tp": 2, "fp": 3, "fn": 2}}, "gemini-v2": {"strict": {"tp": 4, "fp": 0, "fn": 0}, "core": {"tp": 4, "fp": 0, "fn": 0}, "all": {"tp": 4, "fp": 0, "fn": 0}}, "gitlab": {"strict": {"tp": 2, "fp": 3, "fn": 2}, "core": {"tp": 2, "fp": 3, "fn": 2}, "all": {"tp": 2, "fp": 3, "fn": 2}}, "copilot-v2": {"strict": {"tp": 4, "fp": 2, "fn": 0}, "core": {"tp": 4, "fp": 2, "fn": 0}, "all": {"tp": 4, "fp": 2, "fn": 0}}, "augment": {"strict": {"tp": 4, "fp": 2, "fn": 0}, "core": {"tp": 4, "fp": 2, "fn": 0}, "all": {"tp": 4, "fp": 2, "fn": 0}}, "bugbot": {"strict": {"tp": 0, "fp": 1, "fn": 4}, "core": {"tp": 0, "fp": 1, "fn": 4}, "all": {"tp": 0, "fp": 1, "fn": 4}}, "claude": {"strict": {"tp": 0, "fp": 1, "fn": 4}, "core": {"tp": 0, "fp": 1, "fn": 4}, "all": {"tp": 0, "fp": 1, "fn": 4}}, "claude-code": {"strict": {"tp": 1, "fp": 3, "fn": 3}, "core": {"tp": 1, "fp": 3, "fn": 3}, "all": {"tp": 1, "fp": 3, "fn": 3}}, "cubic-v2": {"strict": {"tp": 0, "fp": 2, "fn": 4}, "core": {"tp": 0, "fp": 2, "fn": 4}, "all": {"tp": 0, "fp": 2, "fn": 4}}, "devin": {"strict": {"tp": 1, "fp": 1, "fn": 3}, "core": {"tp": 1, "fp": 1, "fn": 3}, "all": {"tp": 1, "fp": 1, "fn": 3}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 4}, "core": {"tp": 0, "fp": 0, "fn": 4}, "all": {"tp": 0, "fp": 0, "fn": 4}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 4}, "core": {"tp": 0, "fp": 0, "fn": 4}, "all": {"tp": 0, "fp": 0, "fn": 4}}, "kodus-v2": {"strict": {"tp": 1, "fp": 0, "fn": 3}, "core": {"tp": 1, "fp": 0, "fn": 3}, "all": {"tp": 1, "fp": 0, "fn": 3}}, "macroscope": {"strict": {"tp": 1, "fp": 1, "fn": 3}, "core": {"tp": 1, "fp": 1, "fn": 3}, "all": {"tp": 1, "fp": 1, "fn": 3}}, "qodo-v2": {"strict": {"tp": 3, "fp": 2, "fn": 1}, "core": {"tp": 3, "fp": 2, "fn": 1}, "all": {"tp": 3, "fp": 2, "fn": 1}}, "sourcery": {"strict": {"tp": 3, "fp": 1, "fn": 1}, "core": {"tp": 3, "fp": 1, "fn": 1}, "all": {"tp": 3, "fp": 1, "fn": 1}}}}, {"url": "https://github.com/calcom/cal.com/pull/8330", "language": "TypeScript", "pr_size": "small", "domain": "scheduling", "change_type": "bug_fix", "complexity": "moderate", "difficulty": "subtle", "risk": "medium", "context": "file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 2, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "codeant-v2": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "coderabbit": {"strict": {"tp": 2, "fp": 2, "fn": 0}, "core": {"tp": 2, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "qodo-extended-v2": {"strict": {"tp": 2, "fp": 2, "fn": 0}, "core": {"tp": 2, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "greptile-v4-1": {"strict": {"tp": 2, "fp": 2, "fn": 0}, "core": {"tp": 2, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "gemini-v2": {"strict": {"tp": 2, "fp": 3, "fn": 0}, "core": {"tp": 2, "fp": 3, "fn": 0}, "all": {"tp": 2, "fp": 3, "fn": 0}}, "gitlab": {"strict": {"tp": 2, "fp": 2, "fn": 0}, "core": {"tp": 2, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "copilot-v2": {"strict": {"tp": 2, "fp": 5, "fn": 0}, "core": {"tp": 2, "fp": 5, "fn": 0}, "all": {"tp": 2, "fp": 5, "fn": 0}}, "augment": {"strict": {"tp": 2, "fp": 3, "fn": 0}, "core": {"tp": 2, "fp": 3, "fn": 0}, "all": {"tp": 2, "fp": 3, "fn": 0}}, "bugbot": {"strict": {"tp": 2, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "claude": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "claude-code": {"strict": {"tp": 2, "fp": 2, "fn": 0}, "core": {"tp": 2, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "cubic-v2": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "devin": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "graphite": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "kg": {"strict": {"tp": 1, "fp": 2, "fn": 1}, "core": {"tp": 1, "fp": 2, "fn": 1}, "all": {"tp": 1, "fp": 2, "fn": 1}}, "kodus-v2": {"strict": {"tp": 2, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "macroscope": {"strict": {"tp": 2, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "qodo-v2": {"strict": {"tp": 2, "fp": 2, "fn": 0}, "core": {"tp": 2, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "sourcery": {"strict": {"tp": 2, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}}}, {"url": "https://github.com/calcom/cal.com/pull/14943", "language": "TypeScript", "pr_size": "small", "domain": "scheduling", "change_type": "feature", "complexity": "moderate", "difficulty": "subtle", "risk": "high", "context": "file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 1, "fp": 3, "fn": 1}, "core": {"tp": 1, "fp": 3, "fn": 1}, "all": {"tp": 1, "fp": 3, "fn": 1}}, "codeant-v2": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "coderabbit": {"strict": {"tp": 2, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "qodo-extended-v2": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "greptile-v4-1": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "gemini-v2": {"strict": {"tp": 2, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "gitlab": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "copilot-v2": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "augment": {"strict": {"tp": 2, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "bugbot": {"strict": {"tp": 1, "fp": 2, "fn": 1}, "core": {"tp": 1, "fp": 2, "fn": 1}, "all": {"tp": 1, "fp": 2, "fn": 1}}, "claude": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "claude-code": {"strict": {"tp": 0, "fp": 1, "fn": 2}, "core": {"tp": 0, "fp": 1, "fn": 2}, "all": {"tp": 0, "fp": 1, "fn": 2}}, "cubic-v2": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "devin": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "graphite": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "kg": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "kodus-v2": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "macroscope": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "qodo-v2": {"strict": {"tp": 2, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "sourcery": {"strict": {"tp": 2, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}}}, {"url": "https://github.com/calcom/cal.com/pull/22345", "language": "TypeScript", "pr_size": "small", "domain": "database", "change_type": "migration", "complexity": "moderate", "difficulty": "subtle", "risk": "medium", "context": "file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 0, "fp": 2, "fn": 1}, "core": {"tp": 0, "fp": 2, "fn": 1}, "all": {"tp": 0, "fp": 2, "fn": 2}}, "codeant-v2": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "coderabbit": {"strict": {"tp": 0, "fp": 3, "fn": 1}, "core": {"tp": 0, "fp": 3, "fn": 1}, "all": {"tp": 1, "fp": 3, "fn": 1}}, "qodo-extended-v2": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "greptile-v4-1": {"strict": {"tp": 0, "fp": 3, "fn": 1}, "core": {"tp": 0, "fp": 3, "fn": 1}, "all": {"tp": 0, "fp": 3, "fn": 2}}, "gemini-v2": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 0}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "gitlab": {"strict": {"tp": 1, "fp": 3, "fn": 0}, "core": {"tp": 1, "fp": 3, "fn": 0}, "all": {"tp": 1, "fp": 3, "fn": 1}}, "copilot-v2": {"strict": {"tp": 0, "fp": 3, "fn": 1}, "core": {"tp": 0, "fp": 3, "fn": 1}, "all": {"tp": 1, "fp": 3, "fn": 1}}, "augment": {"strict": {"tp": 0, "fp": 3, "fn": 1}, "core": {"tp": 0, "fp": 3, "fn": 1}, "all": {"tp": 1, "fp": 3, "fn": 1}}, "bugbot": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 1, "fp": 1, "fn": 0}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "claude": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "claude-code": {"strict": {"tp": 1, "fp": 2, "fn": 0}, "core": {"tp": 1, "fp": 2, "fn": 0}, "all": {"tp": 1, "fp": 2, "fn": 1}}, "cubic-v2": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 1, "fp": 1, "fn": 0}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "devin": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "kg": {"strict": {"tp": 0, "fp": 1, "fn": 1}, "core": {"tp": 0, "fp": 1, "fn": 1}, "all": {"tp": 0, "fp": 1, "fn": 2}}, "kodus-v2": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "macroscope": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "qodo-v2": {"strict": {"tp": 0, "fp": 3, "fn": 1}, "core": {"tp": 0, "fp": 3, "fn": 1}, "all": {"tp": 1, "fp": 3, "fn": 1}}, "sourcery": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 0}, "all": {"tp": 1, "fp": 0, "fn": 1}}}}, {"url": "https://github.com/calcom/cal.com/pull/11059", "language": "TypeScript", "pr_size": "large", "domain": "authentication", "change_type": "feature", "complexity": "complex", "difficulty": "subtle", "risk": "critical", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 4, "fp": 1, "fn": 5}, "core": {"tp": 4, "fp": 1, "fn": 5}, "all": {"tp": 4, "fp": 1, "fn": 5}}, "codeant-v2": {"strict": {"tp": 4, "fp": 5, "fn": 5}, "core": {"tp": 4, "fp": 5, "fn": 5}, "all": {"tp": 4, "fp": 5, "fn": 5}}, "coderabbit": {"strict": {"tp": 8, "fp": 13, "fn": 1}, "core": {"tp": 8, "fp": 13, "fn": 1}, "all": {"tp": 8, "fp": 13, "fn": 1}}, "qodo-extended-v2": {"strict": {"tp": 7, "fp": 2, "fn": 2}, "core": {"tp": 7, "fp": 2, "fn": 2}, "all": {"tp": 7, "fp": 2, "fn": 2}}, "greptile-v4-1": {"strict": {"tp": 5, "fp": 4, "fn": 4}, "core": {"tp": 5, "fp": 4, "fn": 4}, "all": {"tp": 5, "fp": 4, "fn": 4}}, "gemini-v2": {"strict": {"tp": 4, "fp": 2, "fn": 5}, "core": {"tp": 4, "fp": 2, "fn": 5}, "all": {"tp": 4, "fp": 2, "fn": 5}}, "gitlab": {"strict": {"tp": 5, "fp": 5, "fn": 4}, "core": {"tp": 5, "fp": 5, "fn": 4}, "all": {"tp": 5, "fp": 5, "fn": 4}}, "copilot-v2": {"strict": {"tp": 7, "fp": 3, "fn": 2}, "core": {"tp": 7, "fp": 3, "fn": 2}, "all": {"tp": 7, "fp": 3, "fn": 2}}, "augment": {"strict": {"tp": 8, "fp": 5, "fn": 1}, "core": {"tp": 8, "fp": 5, "fn": 1}, "all": {"tp": 8, "fp": 5, "fn": 1}}, "bugbot": {"strict": {"tp": 6, "fp": 2, "fn": 3}, "core": {"tp": 6, "fp": 2, "fn": 3}, "all": {"tp": 6, "fp": 2, "fn": 3}}, "claude": {"strict": {"tp": 5, "fp": 1, "fn": 4}, "core": {"tp": 5, "fp": 1, "fn": 4}, "all": {"tp": 5, "fp": 1, "fn": 4}}, "claude-code": {"strict": {"tp": 4, "fp": 0, "fn": 5}, "core": {"tp": 4, "fp": 0, "fn": 5}, "all": {"tp": 4, "fp": 0, "fn": 5}}, "cubic-v2": {"strict": {"tp": 3, "fp": 0, "fn": 6}, "core": {"tp": 3, "fp": 0, "fn": 6}, "all": {"tp": 3, "fp": 0, "fn": 6}}, "devin": {"strict": {"tp": 5, "fp": 0, "fn": 4}, "core": {"tp": 5, "fp": 0, "fn": 4}, "all": {"tp": 5, "fp": 0, "fn": 4}}, "graphite": {"strict": {"tp": 1, "fp": 1, "fn": 8}, "core": {"tp": 1, "fp": 1, "fn": 8}, "all": {"tp": 1, "fp": 1, "fn": 8}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 9}, "core": {"tp": 0, "fp": 0, "fn": 9}, "all": {"tp": 0, "fp": 0, "fn": 9}}, "kodus-v2": {"strict": {"tp": 4, "fp": 3, "fn": 5}, "core": {"tp": 4, "fp": 3, "fn": 5}, "all": {"tp": 4, "fp": 3, "fn": 5}}, "macroscope": {"strict": {"tp": 5, "fp": 5, "fn": 4}, "core": {"tp": 5, "fp": 5, "fn": 4}, "all": {"tp": 5, "fp": 5, "fn": 4}}, "qodo-v2": {"strict": {"tp": 9, "fp": 2, "fn": 0}, "core": {"tp": 9, "fp": 2, "fn": 0}, "all": {"tp": 9, "fp": 2, "fn": 0}}, "sourcery": {"strict": {"tp": 5, "fp": 0, "fn": 4}, "core": {"tp": 5, "fp": 0, "fn": 4}, "all": {"tp": 5, "fp": 0, "fn": 4}}}}, {"url": "https://github.com/calcom/cal.com/pull/7232", "language": "TypeScript", "pr_size": "medium", "domain": "scheduling", "change_type": "feature", "complexity": "moderate", "difficulty": "subtle", "risk": "high", "context": "cross_file", "concern": "reliability", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 1, "fp": 1, "fn": 2}, "core": {"tp": 1, "fp": 1, "fn": 2}, "all": {"tp": 1, "fp": 1, "fn": 2}}, "codeant-v2": {"strict": {"tp": 1, "fp": 2, "fn": 2}, "core": {"tp": 1, "fp": 2, "fn": 2}, "all": {"tp": 1, "fp": 2, "fn": 2}}, "coderabbit": {"strict": {"tp": 2, "fp": 12, "fn": 1}, "core": {"tp": 2, "fp": 12, "fn": 1}, "all": {"tp": 2, "fp": 12, "fn": 1}}, "qodo-extended-v2": {"strict": {"tp": 2, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 1}, "all": {"tp": 2, "fp": 1, "fn": 1}}, "greptile-v4-1": {"strict": {"tp": 1, "fp": 2, "fn": 2}, "core": {"tp": 1, "fp": 2, "fn": 2}, "all": {"tp": 1, "fp": 2, "fn": 2}}, "gemini-v2": {"strict": {"tp": 2, "fp": 3, "fn": 1}, "core": {"tp": 2, "fp": 3, "fn": 1}, "all": {"tp": 2, "fp": 3, "fn": 1}}, "gitlab": {"strict": {"tp": 0, "fp": 3, "fn": 3}, "core": {"tp": 0, "fp": 3, "fn": 3}, "all": {"tp": 0, "fp": 3, "fn": 3}}, "copilot-v2": {"strict": {"tp": 0, "fp": 3, "fn": 3}, "core": {"tp": 0, "fp": 3, "fn": 3}, "all": {"tp": 0, "fp": 3, "fn": 3}}, "augment": {"strict": {"tp": 1, "fp": 2, "fn": 2}, "core": {"tp": 1, "fp": 2, "fn": 2}, "all": {"tp": 1, "fp": 2, "fn": 2}}, "bugbot": {"strict": {"tp": 1, "fp": 3, "fn": 2}, "core": {"tp": 1, "fp": 3, "fn": 2}, "all": {"tp": 1, "fp": 3, "fn": 2}}, "claude": {"strict": {"tp": 1, "fp": 3, "fn": 2}, "core": {"tp": 1, "fp": 3, "fn": 2}, "all": {"tp": 1, "fp": 3, "fn": 2}}, "claude-code": {"strict": {"tp": 1, "fp": 1, "fn": 2}, "core": {"tp": 1, "fp": 1, "fn": 2}, "all": {"tp": 1, "fp": 1, "fn": 2}}, "cubic-v2": {"strict": {"tp": 2, "fp": 3, "fn": 1}, "core": {"tp": 2, "fp": 3, "fn": 1}, "all": {"tp": 2, "fp": 3, "fn": 1}}, "devin": {"strict": {"tp": 2, "fp": 3, "fn": 1}, "core": {"tp": 2, "fp": 3, "fn": 1}, "all": {"tp": 2, "fp": 3, "fn": 1}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 3}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 3}}, "kg": {"strict": {"tp": 0, "fp": 1, "fn": 3}, "core": {"tp": 0, "fp": 1, "fn": 3}, "all": {"tp": 0, "fp": 1, "fn": 3}}, "kodus-v2": {"strict": {"tp": 1, "fp": 3, "fn": 2}, "core": {"tp": 1, "fp": 3, "fn": 2}, "all": {"tp": 1, "fp": 3, "fn": 2}}, "macroscope": {"strict": {"tp": 1, "fp": 2, "fn": 2}, "core": {"tp": 1, "fp": 2, "fn": 2}, "all": {"tp": 1, "fp": 2, "fn": 2}}, "qodo-v2": {"strict": {"tp": 2, "fp": 3, "fn": 1}, "core": {"tp": 2, "fp": 3, "fn": 1}, "all": {"tp": 2, "fp": 3, "fn": 1}}, "sourcery": {"strict": {"tp": 1, "fp": 2, "fn": 2}, "core": {"tp": 1, "fp": 2, "fn": 2}, "all": {"tp": 1, "fp": 2, "fn": 2}}}}, {"url": "https://github.com/calcom/cal.com/pull/14740", "language": "TypeScript", "pr_size": "large", "domain": "scheduling", "change_type": "feature", "complexity": "moderate", "difficulty": "subtle", "risk": "critical", "context": "cross_file", "concern": "security", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 1, "fp": 1, "fn": 5}, "core": {"tp": 1, "fp": 1, "fn": 5}, "all": {"tp": 1, "fp": 1, "fn": 5}}, "codeant-v2": {"strict": {"tp": 3, "fp": 4, "fn": 3}, "core": {"tp": 3, "fp": 4, "fn": 3}, "all": {"tp": 3, "fp": 4, "fn": 3}}, "coderabbit": {"strict": {"tp": 4, "fp": 7, "fn": 2}, "core": {"tp": 4, "fp": 7, "fn": 2}, "all": {"tp": 4, "fp": 7, "fn": 2}}, "qodo-extended-v2": {"strict": {"tp": 4, "fp": 1, "fn": 2}, "core": {"tp": 4, "fp": 1, "fn": 2}, "all": {"tp": 4, "fp": 1, "fn": 2}}, "greptile-v4-1": {"strict": {"tp": 3, "fp": 2, "fn": 3}, "core": {"tp": 3, "fp": 2, "fn": 3}, "all": {"tp": 3, "fp": 2, "fn": 3}}, "gemini-v2": {"strict": {"tp": 3, "fp": 4, "fn": 3}, "core": {"tp": 3, "fp": 4, "fn": 3}, "all": {"tp": 3, "fp": 4, "fn": 3}}, "gitlab": {"strict": {"tp": 3, "fp": 4, "fn": 3}, "core": {"tp": 3, "fp": 4, "fn": 3}, "all": {"tp": 3, "fp": 4, "fn": 3}}, "copilot-v2": {"strict": {"tp": 5, "fp": 5, "fn": 1}, "core": {"tp": 5, "fp": 5, "fn": 1}, "all": {"tp": 5, "fp": 5, "fn": 1}}, "augment": {"strict": {"tp": 4, "fp": 3, "fn": 2}, "core": {"tp": 4, "fp": 3, "fn": 2}, "all": {"tp": 4, "fp": 3, "fn": 2}}, "bugbot": {"strict": {"tp": 3, "fp": 3, "fn": 3}, "core": {"tp": 3, "fp": 3, "fn": 3}, "all": {"tp": 3, "fp": 3, "fn": 3}}, "claude": {"strict": {"tp": 2, "fp": 0, "fn": 4}, "core": {"tp": 2, "fp": 0, "fn": 4}, "all": {"tp": 2, "fp": 0, "fn": 4}}, "claude-code": {"strict": {"tp": 2, "fp": 1, "fn": 4}, "core": {"tp": 2, "fp": 1, "fn": 4}, "all": {"tp": 2, "fp": 1, "fn": 4}}, "cubic-v2": {"strict": {"tp": 3, "fp": 3, "fn": 3}, "core": {"tp": 3, "fp": 3, "fn": 3}, "all": {"tp": 3, "fp": 3, "fn": 3}}, "devin": {"strict": {"tp": 2, "fp": 0, "fn": 4}, "core": {"tp": 2, "fp": 0, "fn": 4}, "all": {"tp": 2, "fp": 0, "fn": 4}}, "graphite": {"strict": {"tp": 1, "fp": 1, "fn": 5}, "core": {"tp": 1, "fp": 1, "fn": 5}, "all": {"tp": 1, "fp": 1, "fn": 5}}, "kg": {"strict": {"tp": 2, "fp": 0, "fn": 4}, "core": {"tp": 2, "fp": 0, "fn": 4}, "all": {"tp": 2, "fp": 0, "fn": 4}}, "kodus-v2": {"strict": {"tp": 2, "fp": 0, "fn": 4}, "core": {"tp": 2, "fp": 0, "fn": 4}, "all": {"tp": 2, "fp": 0, "fn": 4}}, "macroscope": {"strict": {"tp": 3, "fp": 3, "fn": 3}, "core": {"tp": 3, "fp": 3, "fn": 3}, "all": {"tp": 3, "fp": 3, "fn": 3}}, "qodo-v2": {"strict": {"tp": 5, "fp": 3, "fn": 1}, "core": {"tp": 5, "fp": 3, "fn": 1}, "all": {"tp": 5, "fp": 3, "fn": 1}}, "sourcery": {"strict": {"tp": 2, "fp": 3, "fn": 4}, "core": {"tp": 2, "fp": 3, "fn": 4}, "all": {"tp": 2, "fp": 3, "fn": 4}}}}, {"url": "https://github.com/calcom/cal.com/pull/10600", "language": "TypeScript", "pr_size": "medium", "domain": "authentication", "change_type": "feature", "complexity": "moderate", "difficulty": "subtle", "risk": "high", "context": "cross_file", "concern": "security", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 0, "fp": 1, "fn": 2}, "core": {"tp": 0, "fp": 1, "fn": 4}, "all": {"tp": 0, "fp": 1, "fn": 5}}, "codeant-v2": {"strict": {"tp": 1, "fp": 4, "fn": 1}, "core": {"tp": 1, "fp": 4, "fn": 3}, "all": {"tp": 1, "fp": 4, "fn": 4}}, "coderabbit": {"strict": {"tp": 1, "fp": 7, "fn": 1}, "core": {"tp": 1, "fp": 7, "fn": 3}, "all": {"tp": 2, "fp": 7, "fn": 3}}, "qodo-extended-v2": {"strict": {"tp": 1, "fp": 2, "fn": 1}, "core": {"tp": 2, "fp": 2, "fn": 2}, "all": {"tp": 2, "fp": 2, "fn": 3}}, "greptile-v4-1": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 2}, "all": {"tp": 3, "fp": 1, "fn": 2}}, "gemini-v2": {"strict": {"tp": 0, "fp": 5, "fn": 2}, "core": {"tp": 0, "fp": 5, "fn": 4}, "all": {"tp": 1, "fp": 5, "fn": 4}}, "gitlab": {"strict": {"tp": 0, "fp": 2, "fn": 2}, "core": {"tp": 1, "fp": 2, "fn": 3}, "all": {"tp": 2, "fp": 2, "fn": 3}}, "copilot-v2": {"strict": {"tp": 2, "fp": 3, "fn": 0}, "core": {"tp": 4, "fp": 3, "fn": 0}, "all": {"tp": 5, "fp": 3, "fn": 0}}, "augment": {"strict": {"tp": 0, "fp": 3, "fn": 2}, "core": {"tp": 1, "fp": 3, "fn": 3}, "all": {"tp": 2, "fp": 3, "fn": 3}}, "bugbot": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 3}, "all": {"tp": 1, "fp": 0, "fn": 4}}, "claude": {"strict": {"tp": 0, "fp": 10, "fn": 2}, "core": {"tp": 1, "fp": 10, "fn": 3}, "all": {"tp": 2, "fp": 10, "fn": 3}}, "claude-code": {"strict": {"tp": 0, "fp": 4, "fn": 2}, "core": {"tp": 0, "fp": 4, "fn": 4}, "all": {"tp": 1, "fp": 4, "fn": 4}}, "cubic-v2": {"strict": {"tp": 2, "fp": 2, "fn": 0}, "core": {"tp": 2, "fp": 2, "fn": 2}, "all": {"tp": 2, "fp": 2, "fn": 3}}, "devin": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 3}, "all": {"tp": 1, "fp": 0, "fn": 4}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 4}, "all": {"tp": 0, "fp": 0, "fn": 5}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 4}, "all": {"tp": 0, "fp": 0, "fn": 5}}, "kodus-v2": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 3}, "all": {"tp": 1, "fp": 0, "fn": 4}}, "macroscope": {"strict": {"tp": 0, "fp": 3, "fn": 2}, "core": {"tp": 0, "fp": 3, "fn": 4}, "all": {"tp": 0, "fp": 3, "fn": 5}}, "qodo-v2": {"strict": {"tp": 1, "fp": 5, "fn": 1}, "core": {"tp": 2, "fp": 5, "fn": 2}, "all": {"tp": 2, "fp": 5, "fn": 3}}, "sourcery": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 2}, "all": {"tp": 3, "fp": 1, "fn": 2}}}}, {"url": "https://github.com/calcom/cal.com/pull/10967", "language": "TypeScript", "pr_size": "large", "domain": "scheduling", "change_type": "bug_fix", "complexity": "complex", "difficulty": "subtle", "risk": "high", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 1, "fp": 2, "fn": 4}, "core": {"tp": 1, "fp": 2, "fn": 4}, "all": {"tp": 1, "fp": 2, "fn": 5}}, "codeant-v2": {"strict": {"tp": 2, "fp": 6, "fn": 3}, "core": {"tp": 2, "fp": 6, "fn": 3}, "all": {"tp": 2, "fp": 6, "fn": 4}}, "coderabbit": {"strict": {"tp": 5, "fp": 15, "fn": 0}, "core": {"tp": 5, "fp": 15, "fn": 0}, "all": {"tp": 5, "fp": 15, "fn": 1}}, "qodo-extended-v2": {"strict": {"tp": 3, "fp": 2, "fn": 2}, "core": {"tp": 3, "fp": 2, "fn": 2}, "all": {"tp": 3, "fp": 2, "fn": 3}}, "greptile-v4-1": {"strict": {"tp": 2, "fp": 2, "fn": 3}, "core": {"tp": 2, "fp": 2, "fn": 3}, "all": {"tp": 2, "fp": 2, "fn": 4}}, "gemini-v2": {"strict": {"tp": 1, "fp": 6, "fn": 4}, "core": {"tp": 1, "fp": 6, "fn": 4}, "all": {"tp": 1, "fp": 6, "fn": 5}}, "gitlab": {"strict": {"tp": 4, "fp": 1, "fn": 1}, "core": {"tp": 4, "fp": 1, "fn": 1}, "all": {"tp": 4, "fp": 1, "fn": 2}}, "copilot-v2": {"strict": {"tp": 4, "fp": 8, "fn": 1}, "core": {"tp": 4, "fp": 8, "fn": 1}, "all": {"tp": 4, "fp": 8, "fn": 2}}, "augment": {"strict": {"tp": 3, "fp": 2, "fn": 2}, "core": {"tp": 3, "fp": 2, "fn": 2}, "all": {"tp": 3, "fp": 2, "fn": 3}}, "bugbot": {"strict": {"tp": 3, "fp": 3, "fn": 2}, "core": {"tp": 3, "fp": 3, "fn": 2}, "all": {"tp": 3, "fp": 3, "fn": 3}}, "claude": {"strict": {"tp": 3, "fp": 4, "fn": 2}, "core": {"tp": 3, "fp": 4, "fn": 2}, "all": {"tp": 3, "fp": 4, "fn": 3}}, "claude-code": {"strict": {"tp": 4, "fp": 3, "fn": 1}, "core": {"tp": 4, "fp": 3, "fn": 1}, "all": {"tp": 4, "fp": 3, "fn": 2}}, "cubic-v2": {"strict": {"tp": 1, "fp": 2, "fn": 4}, "core": {"tp": 1, "fp": 2, "fn": 4}, "all": {"tp": 1, "fp": 2, "fn": 5}}, "devin": {"strict": {"tp": 3, "fp": 2, "fn": 2}, "core": {"tp": 3, "fp": 2, "fn": 2}, "all": {"tp": 3, "fp": 2, "fn": 3}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 5}, "core": {"tp": 0, "fp": 0, "fn": 5}, "all": {"tp": 0, "fp": 0, "fn": 6}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 5}, "core": {"tp": 0, "fp": 0, "fn": 5}, "all": {"tp": 0, "fp": 0, "fn": 6}}, "kodus-v2": {"strict": {"tp": 2, "fp": 2, "fn": 3}, "core": {"tp": 2, "fp": 2, "fn": 3}, "all": {"tp": 2, "fp": 2, "fn": 4}}, "macroscope": {"strict": {"tp": 2, "fp": 3, "fn": 3}, "core": {"tp": 2, "fp": 3, "fn": 3}, "all": {"tp": 2, "fp": 3, "fn": 4}}, "qodo-v2": {"strict": {"tp": 3, "fp": 2, "fn": 2}, "core": {"tp": 3, "fp": 2, "fn": 2}, "all": {"tp": 3, "fp": 2, "fn": 3}}, "sourcery": {"strict": {"tp": 2, "fp": 3, "fn": 3}, "core": {"tp": 2, "fp": 3, "fn": 3}, "all": {"tp": 2, "fp": 3, "fn": 4}}}}, {"url": "https://github.com/calcom/cal.com/pull/8087", "language": "TypeScript", "pr_size": "medium", "domain": "concurrency", "change_type": "refactoring", "complexity": "moderate", "difficulty": "subtle", "risk": "critical", "context": "cross_file", "concern": "reliability", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 1, "fp": 3, "fn": 0}, "core": {"tp": 1, "fp": 3, "fn": 0}, "all": {"tp": 1, "fp": 3, "fn": 1}}, "codeant-v2": {"strict": {"tp": 1, "fp": 2, "fn": 0}, "core": {"tp": 1, "fp": 2, "fn": 0}, "all": {"tp": 1, "fp": 2, "fn": 1}}, "coderabbit": {"strict": {"tp": 1, "fp": 7, "fn": 0}, "core": {"tp": 1, "fp": 7, "fn": 0}, "all": {"tp": 2, "fp": 7, "fn": 0}}, "qodo-extended-v2": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 1, "fp": 1, "fn": 0}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "greptile-v4-1": {"strict": {"tp": 1, "fp": 6, "fn": 0}, "core": {"tp": 1, "fp": 6, "fn": 0}, "all": {"tp": 1, "fp": 6, "fn": 1}}, "gemini-v2": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 0}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "gitlab": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 1, "fp": 1, "fn": 0}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "copilot-v2": {"strict": {"tp": 1, "fp": 2, "fn": 0}, "core": {"tp": 1, "fp": 2, "fn": 0}, "all": {"tp": 1, "fp": 2, "fn": 1}}, "augment": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 1, "fp": 1, "fn": 0}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "bugbot": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 0}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "claude": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 1, "fp": 1, "fn": 0}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "claude-code": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 1, "fp": 1, "fn": 0}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "cubic-v2": {"strict": {"tp": 1, "fp": 7, "fn": 0}, "core": {"tp": 1, "fp": 7, "fn": 0}, "all": {"tp": 2, "fp": 7, "fn": 0}}, "devin": {"strict": {"tp": 1, "fp": 7, "fn": 0}, "core": {"tp": 1, "fp": 7, "fn": 0}, "all": {"tp": 1, "fp": 7, "fn": 1}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "kg": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 0}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "kodus-v2": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "macroscope": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 1, "fp": 1, "fn": 0}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "qodo-v2": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 1, "fp": 1, "fn": 0}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "sourcery": {"strict": {"tp": 1, "fp": 3, "fn": 0}, "core": {"tp": 1, "fp": 3, "fn": 0}, "all": {"tp": 1, "fp": 3, "fn": 1}}}}], "tools": ["augment", "baz", "bugbot", "claude", "claude-code", "codeant-v2", "coderabbit", "copilot-v2", "cubic-v2", "devin", "gemini-v2", "gitlab", "graphite", "greptile-v4-1", "kg", "kodus-v2", "macroscope", "qodo-extended-v2", "qodo-v2", "sourcery"], "dimensions": {"language": ["Go", "Java", "Python", "Ruby", "TypeScript"], "pr_size": ["small", "medium", "large"], "domain": ["API", "UI", "authentication", "caching", "concurrency", "configuration", "data_processing", "database", "file_io", "logging", "scheduling", "serialization", "testing"], "change_type": ["bug_fix", "feature", "migration", "performance", "refactoring", "security_patch"], "complexity": ["simple", "moderate", "complex"], "difficulty": ["obvious", "moderate", "subtle", "very_subtle"], "risk": ["low", "medium", "high", "critical"], "context": ["local", "file", "cross_file", "system"], "concern": ["correctness", "maintainability", "reliability", "security"]}, "overall_metrics": {"augment": {"strict": {"precision": 45.9, "recall": 59.7, "f1": 51.9, "f2": 56.3, "tp": 83, "fp": 98, "fn": 56, "num_prs": 50}, "core": {"precision": 49.5, "recall": 60.8, "f1": 54.5, "f2": 58.1, "tp": 96, "fp": 98, "fn": 62, "num_prs": 50}, "all": {"precision": 50.8, "recall": 58.4, "f1": 54.3, "f2": 56.7, "tp": 101, "fp": 98, "fn": 72, "num_prs": 50}}, "cubic-v2": {"strict": {"precision": 53.2, "recall": 59.7, "f1": 56.3, "f2": 58.3, "tp": 83, "fp": 73, "fn": 56, "num_prs": 50}, "core": {"precision": 56.8, "recall": 60.8, "f1": 58.7, "f2": 59.9, "tp": 96, "fp": 73, "fn": 62, "num_prs": 50}, "all": {"precision": 57.8, "recall": 57.8, "f1": 57.8, "f2": 57.8, "tp": 100, "fp": 73, "fn": 73, "num_prs": 50}}, "copilot-v2": {"strict": {"precision": 30.4, "recall": 63.3, "f1": 41.1, "f2": 52.1, "tp": 88, "fp": 201, "fn": 51, "num_prs": 50}, "core": {"precision": 33.2, "recall": 63.3, "f1": 43.6, "f2": 53.6, "tp": 100, "fp": 201, "fn": 58, "num_prs": 50}, "all": {"precision": 35.2, "recall": 63.0, "f1": 45.1, "f2": 54.4, "tp": 109, "fp": 201, "fn": 64, "num_prs": 50}}, "greptile-v4-1": {"strict": {"precision": 40.2, "recall": 50.4, "f1": 44.7, "f2": 47.9, "tp": 70, "fp": 104, "fn": 69, "num_prs": 50}, "core": {"precision": 42.2, "recall": 48.1, "f1": 45.0, "f2": 46.8, "tp": 76, "fp": 104, "fn": 82, "num_prs": 50}, "all": {"precision": 44.7, "recall": 48.6, "f1": 46.5, "f2": 47.7, "tp": 84, "fp": 104, "fn": 89, "num_prs": 50}}, "sourcery": {"strict": {"precision": 33.3, "recall": 50.4, "f1": 40.1, "f2": 45.7, "tp": 70, "fp": 140, "fn": 69, "num_prs": 50}, "core": {"precision": 36.1, "recall": 50.0, "f1": 41.9, "f2": 46.4, "tp": 79, "fp": 140, "fn": 79, "num_prs": 50}, "all": {"precision": 38.1, "recall": 49.7, "f1": 43.1, "f2": 46.8, "tp": 86, "fp": 140, "fn": 87, "num_prs": 50}}, "qodo-v2": {"strict": {"precision": 44.1, "recall": 64.7, "f1": 52.5, "f2": 59.2, "tp": 90, "fp": 114, "fn": 49, "num_prs": 50}, "core": {"precision": 46.5, "recall": 62.7, "f1": 53.4, "f2": 58.6, "tp": 99, "fp": 114, "fn": 59, "num_prs": 50}, "all": {"precision": 47.5, "recall": 59.5, "f1": 52.8, "f2": 56.7, "tp": 103, "fp": 114, "fn": 70, "num_prs": 50}}, "gitlab": {"strict": {"precision": 42.2, "recall": 48.9, "f1": 45.3, "f2": 47.4, "tp": 68, "fp": 93, "fn": 71, "num_prs": 50}, "core": {"precision": 45.0, "recall": 48.1, "f1": 46.5, "f2": 47.4, "tp": 76, "fp": 93, "fn": 82, "num_prs": 50}, "all": {"precision": 46.9, "recall": 47.4, "f1": 47.1, "f2": 47.3, "tp": 82, "fp": 93, "fn": 91, "num_prs": 50}}, "codeant-v2": {"strict": {"precision": 29.3, "recall": 38.1, "f1": 33.1, "f2": 36.0, "tp": 53, "fp": 128, "fn": 86, "num_prs": 50}, "core": {"precision": 30.4, "recall": 35.4, "f1": 32.7, "f2": 34.3, "tp": 56, "fp": 128, "fn": 102, "num_prs": 50}, "all": {"precision": 31.2, "recall": 33.5, "f1": 32.3, "f2": 33.0, "tp": 58, "fp": 128, "fn": 115, "num_prs": 50}}, "bugbot": {"strict": {"precision": 49.6, "recall": 47.5, "f1": 48.5, "f2": 47.9, "tp": 66, "fp": 67, "fn": 73, "num_prs": 50}, "core": {"precision": 51.8, "recall": 45.6, "f1": 48.5, "f2": 46.7, "tp": 72, "fp": 67, "fn": 86, "num_prs": 50}, "all": {"precision": 52.8, "recall": 43.4, "f1": 47.6, "f2": 45.0, "tp": 75, "fp": 67, "fn": 98, "num_prs": 50}}, "claude-code": {"strict": {"precision": 38.2, "recall": 43.2, "f1": 40.5, "f2": 42.1, "tp": 60, "fp": 97, "fn": 79, "num_prs": 50}, "core": {"precision": 40.1, "recall": 41.1, "f1": 40.6, "f2": 40.9, "tp": 65, "fp": 97, "fn": 93, "num_prs": 50}, "all": {"precision": 42.3, "recall": 41.0, "f1": 41.6, "f2": 41.3, "tp": 71, "fp": 97, "fn": 102, "num_prs": 50}}, "devin": {"strict": {"precision": 56.6, "recall": 43.2, "f1": 49.0, "f2": 45.3, "tp": 60, "fp": 46, "fn": 79, "num_prs": 50}, "core": {"precision": 58.6, "recall": 41.1, "f1": 48.3, "f2": 43.7, "tp": 65, "fp": 46, "fn": 93, "num_prs": 50}, "all": {"precision": 59.3, "recall": 38.7, "f1": 46.9, "f2": 41.6, "tp": 67, "fp": 46, "fn": 106, "num_prs": 50}}, "macroscope": {"strict": {"precision": 47.7, "recall": 44.6, "f1": 46.1, "f2": 45.2, "tp": 62, "fp": 68, "fn": 77, "num_prs": 50}, "core": {"precision": 48.9, "recall": 41.1, "f1": 44.7, "f2": 42.5, "tp": 65, "fp": 68, "fn": 93, "num_prs": 50}, "all": {"precision": 50.0, "recall": 39.3, "f1": 44.0, "f2": 41.1, "tp": 68, "fp": 68, "fn": 105, "num_prs": 50}}, "claude": {"strict": {"precision": 36.7, "recall": 38.8, "f1": 37.8, "f2": 38.4, "tp": 54, "fp": 93, "fn": 85, "num_prs": 50}, "core": {"precision": 38.8, "recall": 37.3, "f1": 38.1, "f2": 37.6, "tp": 59, "fp": 93, "fn": 99, "num_prs": 50}, "all": {"precision": 41.1, "recall": 37.6, "f1": 39.3, "f2": 38.2, "tp": 65, "fp": 93, "fn": 108, "num_prs": 50}}, "gemini-v2": {"strict": {"precision": 36.2, "recall": 54.0, "f1": 43.4, "f2": 49.1, "tp": 75, "fp": 132, "fn": 64, "num_prs": 50}, "core": {"precision": 38.0, "recall": 51.3, "f1": 43.7, "f2": 47.9, "tp": 81, "fp": 132, "fn": 77, "num_prs": 50}, "all": {"precision": 40.3, "recall": 51.4, "f1": 45.2, "f2": 48.7, "tp": 89, "fp": 132, "fn": 84, "num_prs": 50}}, "coderabbit": {"strict": {"precision": 26.8, "recall": 61.9, "f1": 37.4, "f2": 49.0, "tp": 86, "fp": 235, "fn": 53, "num_prs": 50}, "core": {"precision": 28.6, "recall": 59.5, "f1": 38.6, "f2": 48.9, "tp": 94, "fp": 235, "fn": 64, "num_prs": 50}, "all": {"precision": 30.7, "recall": 60.1, "f1": 40.6, "f2": 50.4, "tp": 104, "fp": 235, "fn": 69, "num_prs": 50}}, "baz": {"strict": {"precision": 43.9, "recall": 36.0, "f1": 39.5, "f2": 37.3, "tp": 50, "fp": 64, "fn": 89, "num_prs": 50}, "core": {"precision": 44.8, "recall": 32.9, "f1": 38.0, "f2": 34.8, "tp": 52, "fp": 64, "fn": 106, "num_prs": 50}, "all": {"precision": 46.2, "recall": 31.8, "f1": 37.7, "f2": 33.9, "tp": 55, "fp": 64, "fn": 118, "num_prs": 50}}, "kg": {"strict": {"precision": 53.5, "recall": 16.5, "f1": 25.3, "f2": 19.2, "tp": 23, "fp": 20, "fn": 116, "num_prs": 50}, "core": {"precision": 54.5, "recall": 15.2, "f1": 23.8, "f2": 17.8, "tp": 24, "fp": 20, "fn": 134, "num_prs": 50}, "all": {"precision": 55.6, "recall": 14.5, "f1": 22.9, "f2": 17.0, "tp": 25, "fp": 20, "fn": 148, "num_prs": 50}}, "qodo-extended-v2": {"strict": {"precision": 55.9, "recall": 61.2, "f1": 58.4, "f2": 60.0, "tp": 85, "fp": 67, "fn": 54, "num_prs": 50}, "core": {"precision": 58.4, "recall": 59.5, "f1": 58.9, "f2": 59.3, "tp": 94, "fp": 67, "fn": 64, "num_prs": 50}, "all": {"precision": 58.9, "recall": 55.5, "f1": 57.1, "f2": 56.1, "tp": 96, "fp": 67, "fn": 77, "num_prs": 50}}, "kodus-v2": {"strict": {"precision": 50.5, "recall": 36.0, "f1": 42.0, "f2": 38.2, "tp": 50, "fp": 49, "fn": 89, "num_prs": 50}, "core": {"precision": 52.4, "recall": 34.2, "f1": 41.4, "f2": 36.7, "tp": 54, "fp": 49, "fn": 104, "num_prs": 50}, "all": {"precision": 53.8, "recall": 32.9, "f1": 40.9, "f2": 35.7, "tp": 57, "fp": 49, "fn": 116, "num_prs": 50}}, "graphite": {"strict": {"precision": 71.4, "recall": 7.2, "f1": 13.1, "f2": 8.8, "tp": 10, "fp": 4, "fn": 129, "num_prs": 50}, "core": {"precision": 73.3, "recall": 7.0, "f1": 12.7, "f2": 8.5, "tp": 11, "fp": 4, "fn": 147, "num_prs": 50}, "all": {"precision": 73.3, "recall": 6.4, "f1": 11.7, "f2": 7.8, "tp": 11, "fp": 4, "fn": 162, "num_prs": 50}}}}};
-        const toolDisplayNames = {"graphite": "Graphite", "qodo": "Qodo", "gemini": "Gemini", "claude": "Claude Code", "augment": "Augment", "bugbot": "Cursor Bugbot", "coderabbit": "CodeRabbit", "propel": "Propel", "copilot": "GitHub Copilot", "baz": "Baz", "greptile": "Greptile", "kg": "KG", "entelligence": "Entelligence", "cubic-v2": "Cubic v2", "sourcery": "Sourcery", "mesa": "Mesa", "codeant-v2": "CodeAnt", "claude-code": "Claude Code (CLI)", "devin": "Devin", "kodus-v2": "Kodus", "greptile-v4-1": "Greptile v4", "gemini-v2": "Gemini", "gitlab": "GitLab Duo Code Review", "qodo-v2": "Qodo v2", "qodo-extended-v2": "Qodo Extended", "macroscope": "Macroscope"};
-        const toolColors = {"graphite": "#6366f1", "qodo": "#8b5cf6", "gemini": "#06b6d4", "claude": "#f59e0b", "augment": "#10b981", "bugbot": "#3b82f6", "coderabbit": "#ec4899", "propel": "#14b8a6", "propel-v2": "#0d9488", "copilot": "#6b7280", "baz": "#f97316", "greptile": "#22c55e", "kg": "#a855f7", "entelligence": "#0ea5e9", "cubic-dev": "#d946ef", "sourcery": "#84cc16", "mesa": "#f43f5e", "codeant": "#e11d48", "codeant-v2": "#fb7185", "claude-code": "#d97706", "devin": "#7c3aed", "kodus-v2": "#059669", "greptile-v4": "#16a34a", "qodo-v2": "#7c3aed", "qodo-extended-v2": "#6d28d9", "macroscope": "#0891b2", "cubic-v2": "#c026d3"};
+        const allModelsData = {"anthropic_claude-opus-4-5-20251101": {"prs": [{"url": "https://github.com/keycloak/keycloak/pull/37429", "language": "Java", "pr_size": "medium", "domain": "UI", "change_type": "feature", "complexity": "moderate", "difficulty": "moderate", "risk": "medium", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 4}}, "codeant-v2": {"strict": {"tp": 0, "fp": 1, "fn": 1}, "core": {"tp": 0, "fp": 1, "fn": 3}, "all": {"tp": 0, "fp": 1, "fn": 5}}, "coderabbit": {"strict": {"tp": 0, "fp": 5, "fn": 1}, "core": {"tp": 2, "fp": 5, "fn": 1}, "all": {"tp": 4, "fp": 5, "fn": 1}}, "qodo-extended-v2": {"strict": {"tp": 0, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 1}, "all": {"tp": 2, "fp": 1, "fn": 3}}, "greptile-v4-1": {"strict": {"tp": 0, "fp": 3, "fn": 1}, "core": {"tp": 0, "fp": 3, "fn": 3}, "all": {"tp": 2, "fp": 3, "fn": 3}}, "gemini-v2": {"strict": {"tp": 0, "fp": 3, "fn": 1}, "core": {"tp": 2, "fp": 3, "fn": 1}, "all": {"tp": 4, "fp": 3, "fn": 1}}, "gitlab": {"strict": {"tp": 0, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 2}, "all": {"tp": 2, "fp": 1, "fn": 3}}, "copilot-v2": {"strict": {"tp": 0, "fp": 6, "fn": 1}, "core": {"tp": 2, "fp": 6, "fn": 1}, "all": {"tp": 3, "fp": 6, "fn": 2}}, "augment": {"strict": {"tp": 0, "fp": 3, "fn": 1}, "core": {"tp": 2, "fp": 3, "fn": 1}, "all": {"tp": 2, "fp": 3, "fn": 3}}, "bugbot": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 2, "fp": 0, "fn": 1}, "all": {"tp": 2, "fp": 0, "fn": 3}}, "claude": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 1, "fp": 0, "fn": 4}}, "claude-code": {"strict": {"tp": 0, "fp": 3, "fn": 1}, "core": {"tp": 0, "fp": 3, "fn": 3}, "all": {"tp": 2, "fp": 3, "fn": 3}}, "cubic-v2": {"strict": {"tp": 0, "fp": 2, "fn": 1}, "core": {"tp": 1, "fp": 2, "fn": 2}, "all": {"tp": 1, "fp": 2, "fn": 4}}, "devin": {"strict": {"tp": 0, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 1}, "all": {"tp": 2, "fp": 1, "fn": 3}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 5}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 4}}, "kodus-v2": {"strict": {"tp": 0, "fp": 2, "fn": 1}, "core": {"tp": 1, "fp": 2, "fn": 2}, "all": {"tp": 1, "fp": 2, "fn": 4}}, "macroscope": {"strict": {"tp": 0, "fp": 1, "fn": 1}, "core": {"tp": 0, "fp": 1, "fn": 3}, "all": {"tp": 0, "fp": 1, "fn": 5}}, "qodo-v2": {"strict": {"tp": 0, "fp": 3, "fn": 1}, "core": {"tp": 0, "fp": 3, "fn": 3}, "all": {"tp": 0, "fp": 3, "fn": 5}}, "sourcery": {"strict": {"tp": 0, "fp": 5, "fn": 1}, "core": {"tp": 0, "fp": 5, "fn": 3}, "all": {"tp": 2, "fp": 5, "fn": 3}}, "shipwright": {"strict": {"tp": 0, "fp": 2, "fn": 1}, "core": {"tp": 2, "fp": 2, "fn": 1}, "all": {"tp": 3, "fp": 2, "fn": 2}}}}, {"url": "https://github.com/keycloak/keycloak/pull/37634", "language": "Java", "pr_size": "medium", "domain": "authentication", "change_type": "feature", "complexity": "moderate", "difficulty": "subtle", "risk": "high", "context": "file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 2, "fp": 0, "fn": 1}, "core": {"tp": 2, "fp": 0, "fn": 2}, "all": {"tp": 2, "fp": 0, "fn": 2}}, "codeant-v2": {"strict": {"tp": 1, "fp": 2, "fn": 2}, "core": {"tp": 1, "fp": 2, "fn": 3}, "all": {"tp": 1, "fp": 2, "fn": 3}}, "coderabbit": {"strict": {"tp": 3, "fp": 4, "fn": 0}, "core": {"tp": 3, "fp": 4, "fn": 1}, "all": {"tp": 3, "fp": 4, "fn": 1}}, "qodo-extended-v2": {"strict": {"tp": 2, "fp": 0, "fn": 1}, "core": {"tp": 3, "fp": 0, "fn": 1}, "all": {"tp": 3, "fp": 0, "fn": 1}}, "greptile-v4-1": {"strict": {"tp": 2, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 2}, "all": {"tp": 2, "fp": 1, "fn": 2}}, "gemini-v2": {"strict": {"tp": 2, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 2}, "all": {"tp": 2, "fp": 1, "fn": 2}}, "gitlab": {"strict": {"tp": 1, "fp": 1, "fn": 2}, "core": {"tp": 2, "fp": 1, "fn": 2}, "all": {"tp": 2, "fp": 1, "fn": 2}}, "copilot-v2": {"strict": {"tp": 2, "fp": 3, "fn": 1}, "core": {"tp": 3, "fp": 3, "fn": 1}, "all": {"tp": 3, "fp": 3, "fn": 1}}, "augment": {"strict": {"tp": 2, "fp": 2, "fn": 1}, "core": {"tp": 2, "fp": 2, "fn": 2}, "all": {"tp": 2, "fp": 2, "fn": 2}}, "bugbot": {"strict": {"tp": 2, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 2}, "all": {"tp": 2, "fp": 1, "fn": 2}}, "claude": {"strict": {"tp": 1, "fp": 1, "fn": 2}, "core": {"tp": 2, "fp": 1, "fn": 2}, "all": {"tp": 2, "fp": 1, "fn": 2}}, "claude-code": {"strict": {"tp": 0, "fp": 2, "fn": 3}, "core": {"tp": 0, "fp": 2, "fn": 4}, "all": {"tp": 0, "fp": 2, "fn": 4}}, "cubic-v2": {"strict": {"tp": 2, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 2}, "all": {"tp": 2, "fp": 1, "fn": 2}}, "devin": {"strict": {"tp": 2, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 2}, "all": {"tp": 2, "fp": 1, "fn": 2}}, "graphite": {"strict": {"tp": 2, "fp": 0, "fn": 1}, "core": {"tp": 2, "fp": 0, "fn": 2}, "all": {"tp": 2, "fp": 0, "fn": 2}}, "kg": {"strict": {"tp": 1, "fp": 2, "fn": 2}, "core": {"tp": 1, "fp": 2, "fn": 3}, "all": {"tp": 1, "fp": 2, "fn": 3}}, "kodus-v2": {"strict": {"tp": 2, "fp": 0, "fn": 1}, "core": {"tp": 2, "fp": 0, "fn": 2}, "all": {"tp": 2, "fp": 0, "fn": 2}}, "macroscope": {"strict": {"tp": 2, "fp": 2, "fn": 1}, "core": {"tp": 2, "fp": 2, "fn": 2}, "all": {"tp": 2, "fp": 2, "fn": 2}}, "qodo-v2": {"strict": {"tp": 2, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 2}, "all": {"tp": 2, "fp": 1, "fn": 2}}, "sourcery": {"strict": {"tp": 2, "fp": 0, "fn": 1}, "core": {"tp": 2, "fp": 0, "fn": 2}, "all": {"tp": 2, "fp": 0, "fn": 2}}, "shipwright": {"strict": {"tp": 3, "fp": 0, "fn": 0}, "core": {"tp": 3, "fp": 0, "fn": 1}, "all": {"tp": 3, "fp": 0, "fn": 1}}}}, {"url": "https://github.com/keycloak/keycloak/pull/38446", "language": "Java", "pr_size": "medium", "domain": "authentication", "change_type": "feature", "complexity": "moderate", "difficulty": "moderate", "risk": "medium", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 0, "fp": 1, "fn": 2}, "core": {"tp": 0, "fp": 1, "fn": 2}, "all": {"tp": 0, "fp": 1, "fn": 2}}, "codeant-v2": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "coderabbit": {"strict": {"tp": 2, "fp": 2, "fn": 0}, "core": {"tp": 2, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "qodo-extended-v2": {"strict": {"tp": 1, "fp": 3, "fn": 1}, "core": {"tp": 1, "fp": 3, "fn": 1}, "all": {"tp": 1, "fp": 3, "fn": 1}}, "greptile-v4-1": {"strict": {"tp": 2, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "gemini-v2": {"strict": {"tp": 1, "fp": 3, "fn": 1}, "core": {"tp": 1, "fp": 3, "fn": 1}, "all": {"tp": 1, "fp": 3, "fn": 1}}, "gitlab": {"strict": {"tp": 1, "fp": 5, "fn": 1}, "core": {"tp": 1, "fp": 5, "fn": 1}, "all": {"tp": 1, "fp": 5, "fn": 1}}, "copilot-v2": {"strict": {"tp": 1, "fp": 6, "fn": 1}, "core": {"tp": 1, "fp": 6, "fn": 1}, "all": {"tp": 1, "fp": 6, "fn": 1}}, "augment": {"strict": {"tp": 1, "fp": 3, "fn": 1}, "core": {"tp": 1, "fp": 3, "fn": 1}, "all": {"tp": 1, "fp": 3, "fn": 1}}, "bugbot": {"strict": {"tp": 0, "fp": 3, "fn": 2}, "core": {"tp": 0, "fp": 3, "fn": 2}, "all": {"tp": 0, "fp": 3, "fn": 2}}, "claude": {"strict": {"tp": 1, "fp": 3, "fn": 1}, "core": {"tp": 1, "fp": 3, "fn": 1}, "all": {"tp": 1, "fp": 3, "fn": 1}}, "claude-code": {"strict": {"tp": 1, "fp": 5, "fn": 1}, "core": {"tp": 1, "fp": 5, "fn": 1}, "all": {"tp": 1, "fp": 5, "fn": 1}}, "cubic-v2": {"strict": {"tp": 1, "fp": 3, "fn": 1}, "core": {"tp": 1, "fp": 3, "fn": 1}, "all": {"tp": 1, "fp": 3, "fn": 1}}, "devin": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "kodus-v2": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "macroscope": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "qodo-v2": {"strict": {"tp": 2, "fp": 3, "fn": 0}, "core": {"tp": 2, "fp": 3, "fn": 0}, "all": {"tp": 2, "fp": 3, "fn": 0}}, "sourcery": {"strict": {"tp": 1, "fp": 6, "fn": 1}, "core": {"tp": 1, "fp": 6, "fn": 1}, "all": {"tp": 1, "fp": 6, "fn": 1}}, "shipwright": {"strict": {"tp": 0, "fp": 1, "fn": 2}, "core": {"tp": 0, "fp": 1, "fn": 2}, "all": {"tp": 0, "fp": 1, "fn": 2}}}}, {"url": "https://github.com/keycloak/keycloak/pull/36882", "language": "Java", "pr_size": "small", "domain": "configuration", "change_type": "feature", "complexity": "moderate", "difficulty": "moderate", "risk": "medium", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 0, "fp": 1, "fn": 1}, "core": {"tp": 0, "fp": 1, "fn": 2}, "all": {"tp": 0, "fp": 1, "fn": 2}}, "codeant-v2": {"strict": {"tp": 0, "fp": 1, "fn": 1}, "core": {"tp": 0, "fp": 1, "fn": 2}, "all": {"tp": 0, "fp": 1, "fn": 2}}, "coderabbit": {"strict": {"tp": 0, "fp": 1, "fn": 1}, "core": {"tp": 0, "fp": 1, "fn": 2}, "all": {"tp": 0, "fp": 1, "fn": 2}}, "qodo-extended-v2": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "greptile-v4-1": {"strict": {"tp": 0, "fp": 2, "fn": 1}, "core": {"tp": 0, "fp": 2, "fn": 2}, "all": {"tp": 0, "fp": 2, "fn": 2}}, "gemini-v2": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "gitlab": {"strict": {"tp": 0, "fp": 2, "fn": 1}, "core": {"tp": 0, "fp": 2, "fn": 2}, "all": {"tp": 0, "fp": 2, "fn": 2}}, "copilot-v2": {"strict": {"tp": 0, "fp": 3, "fn": 1}, "core": {"tp": 0, "fp": 3, "fn": 2}, "all": {"tp": 0, "fp": 3, "fn": 2}}, "augment": {"strict": {"tp": 0, "fp": 1, "fn": 1}, "core": {"tp": 0, "fp": 1, "fn": 2}, "all": {"tp": 0, "fp": 1, "fn": 2}}, "bugbot": {"strict": {"tp": 0, "fp": 1, "fn": 1}, "core": {"tp": 0, "fp": 1, "fn": 2}, "all": {"tp": 0, "fp": 1, "fn": 2}}, "claude": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "claude-code": {"strict": {"tp": 0, "fp": 1, "fn": 1}, "core": {"tp": 0, "fp": 1, "fn": 2}, "all": {"tp": 0, "fp": 1, "fn": 2}}, "cubic-v2": {"strict": {"tp": 1, "fp": 2, "fn": 0}, "core": {"tp": 1, "fp": 2, "fn": 1}, "all": {"tp": 1, "fp": 2, "fn": 1}}, "devin": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "kodus-v2": {"strict": {"tp": 0, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "macroscope": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "qodo-v2": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "sourcery": {"strict": {"tp": 0, "fp": 4, "fn": 1}, "core": {"tp": 0, "fp": 4, "fn": 2}, "all": {"tp": 0, "fp": 4, "fn": 2}}, "shipwright": {"strict": {"tp": 0, "fp": 2, "fn": 1}, "core": {"tp": 0, "fp": 2, "fn": 2}, "all": {"tp": 0, "fp": 2, "fn": 2}}}}, {"url": "https://github.com/keycloak/keycloak/pull/36880", "language": "Java", "pr_size": "medium", "domain": "authentication", "change_type": "feature", "complexity": "complex", "difficulty": "very_subtle", "risk": "high", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 3, "fp": 0, "fn": 1}, "core": {"tp": 3, "fp": 0, "fn": 1}, "all": {"tp": 3, "fp": 0, "fn": 2}}, "codeant-v2": {"strict": {"tp": 2, "fp": 0, "fn": 2}, "core": {"tp": 2, "fp": 0, "fn": 2}, "all": {"tp": 2, "fp": 0, "fn": 3}}, "coderabbit": {"strict": {"tp": 1, "fp": 5, "fn": 3}, "core": {"tp": 1, "fp": 5, "fn": 3}, "all": {"tp": 2, "fp": 5, "fn": 3}}, "qodo-extended-v2": {"strict": {"tp": 3, "fp": 0, "fn": 1}, "core": {"tp": 3, "fp": 0, "fn": 1}, "all": {"tp": 3, "fp": 0, "fn": 2}}, "greptile-v4-1": {"strict": {"tp": 1, "fp": 1, "fn": 3}, "core": {"tp": 1, "fp": 1, "fn": 3}, "all": {"tp": 2, "fp": 1, "fn": 3}}, "gemini-v2": {"strict": {"tp": 1, "fp": 3, "fn": 3}, "core": {"tp": 1, "fp": 3, "fn": 3}, "all": {"tp": 2, "fp": 3, "fn": 3}}, "gitlab": {"strict": {"tp": 1, "fp": 6, "fn": 3}, "core": {"tp": 1, "fp": 6, "fn": 3}, "all": {"tp": 2, "fp": 6, "fn": 3}}, "copilot-v2": {"strict": {"tp": 2, "fp": 6, "fn": 2}, "core": {"tp": 2, "fp": 6, "fn": 2}, "all": {"tp": 2, "fp": 6, "fn": 3}}, "augment": {"strict": {"tp": 1, "fp": 1, "fn": 3}, "core": {"tp": 1, "fp": 1, "fn": 3}, "all": {"tp": 1, "fp": 1, "fn": 4}}, "bugbot": {"strict": {"tp": 3, "fp": 0, "fn": 1}, "core": {"tp": 3, "fp": 0, "fn": 1}, "all": {"tp": 4, "fp": 0, "fn": 1}}, "claude": {"strict": {"tp": 1, "fp": 2, "fn": 3}, "core": {"tp": 1, "fp": 2, "fn": 3}, "all": {"tp": 2, "fp": 2, "fn": 3}}, "claude-code": {"strict": {"tp": 0, "fp": 2, "fn": 4}, "core": {"tp": 0, "fp": 2, "fn": 4}, "all": {"tp": 0, "fp": 2, "fn": 5}}, "cubic-v2": {"strict": {"tp": 1, "fp": 1, "fn": 3}, "core": {"tp": 1, "fp": 1, "fn": 3}, "all": {"tp": 1, "fp": 1, "fn": 4}}, "devin": {"strict": {"tp": 1, "fp": 0, "fn": 3}, "core": {"tp": 1, "fp": 0, "fn": 3}, "all": {"tp": 1, "fp": 0, "fn": 4}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 4}, "core": {"tp": 0, "fp": 0, "fn": 4}, "all": {"tp": 0, "fp": 0, "fn": 5}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 4}, "core": {"tp": 0, "fp": 0, "fn": 4}, "all": {"tp": 0, "fp": 0, "fn": 5}}, "kodus-v2": {"strict": {"tp": 2, "fp": 1, "fn": 2}, "core": {"tp": 2, "fp": 1, "fn": 2}, "all": {"tp": 2, "fp": 1, "fn": 3}}, "macroscope": {"strict": {"tp": 2, "fp": 4, "fn": 2}, "core": {"tp": 2, "fp": 4, "fn": 2}, "all": {"tp": 2, "fp": 4, "fn": 3}}, "qodo-v2": {"strict": {"tp": 2, "fp": 1, "fn": 2}, "core": {"tp": 2, "fp": 1, "fn": 2}, "all": {"tp": 2, "fp": 1, "fn": 3}}, "sourcery": {"strict": {"tp": 1, "fp": 4, "fn": 3}, "core": {"tp": 1, "fp": 4, "fn": 3}, "all": {"tp": 2, "fp": 4, "fn": 3}}, "shipwright": {"strict": {"tp": 0, "fp": 2, "fn": 4}, "core": {"tp": 0, "fp": 2, "fn": 4}, "all": {"tp": 0, "fp": 2, "fn": 5}}}}, {"url": "https://github.com/keycloak/keycloak/pull/37038", "language": "Java", "pr_size": "small", "domain": "authentication", "change_type": "feature", "complexity": "complex", "difficulty": "very_subtle", "risk": "high", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "codeant-v2": {"strict": {"tp": 1, "fp": 5, "fn": 1}, "core": {"tp": 1, "fp": 5, "fn": 1}, "all": {"tp": 1, "fp": 5, "fn": 1}}, "coderabbit": {"strict": {"tp": 2, "fp": 7, "fn": 0}, "core": {"tp": 2, "fp": 7, "fn": 0}, "all": {"tp": 2, "fp": 7, "fn": 0}}, "qodo-extended-v2": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "greptile-v4-1": {"strict": {"tp": 2, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "gemini-v2": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "gitlab": {"strict": {"tp": 1, "fp": 2, "fn": 1}, "core": {"tp": 1, "fp": 2, "fn": 1}, "all": {"tp": 1, "fp": 2, "fn": 1}}, "copilot-v2": {"strict": {"tp": 2, "fp": 3, "fn": 0}, "core": {"tp": 2, "fp": 3, "fn": 0}, "all": {"tp": 2, "fp": 3, "fn": 0}}, "augment": {"strict": {"tp": 2, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "bugbot": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "claude": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "claude-code": {"strict": {"tp": 1, "fp": 2, "fn": 1}, "core": {"tp": 1, "fp": 2, "fn": 1}, "all": {"tp": 1, "fp": 2, "fn": 1}}, "cubic-v2": {"strict": {"tp": 2, "fp": 2, "fn": 0}, "core": {"tp": 2, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "devin": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "kodus-v2": {"strict": {"tp": 2, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "macroscope": {"strict": {"tp": 2, "fp": 2, "fn": 0}, "core": {"tp": 2, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "qodo-v2": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "sourcery": {"strict": {"tp": 1, "fp": 4, "fn": 1}, "core": {"tp": 1, "fp": 4, "fn": 1}, "all": {"tp": 1, "fp": 4, "fn": 1}}, "shipwright": {"strict": {"tp": 2, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}}}, {"url": "https://github.com/keycloak/keycloak/pull/33832", "language": "Java", "pr_size": "medium", "domain": "authentication", "change_type": "feature", "complexity": "moderate", "difficulty": "subtle", "risk": "high", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 2, "fp": 0, "fn": 1}, "all": {"tp": 2, "fp": 0, "fn": 2}}, "codeant-v2": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 2}, "all": {"tp": 2, "fp": 1, "fn": 2}}, "coderabbit": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 1}, "all": {"tp": 2, "fp": 1, "fn": 2}}, "qodo-extended-v2": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 1}, "all": {"tp": 2, "fp": 1, "fn": 2}}, "greptile-v4-1": {"strict": {"tp": 0, "fp": 3, "fn": 2}, "core": {"tp": 1, "fp": 3, "fn": 2}, "all": {"tp": 1, "fp": 3, "fn": 3}}, "gemini-v2": {"strict": {"tp": 2, "fp": 1, "fn": 0}, "core": {"tp": 3, "fp": 1, "fn": 0}, "all": {"tp": 3, "fp": 1, "fn": 1}}, "gitlab": {"strict": {"tp": 0, "fp": 2, "fn": 2}, "core": {"tp": 1, "fp": 2, "fn": 2}, "all": {"tp": 1, "fp": 2, "fn": 3}}, "copilot-v2": {"strict": {"tp": 1, "fp": 4, "fn": 1}, "core": {"tp": 2, "fp": 4, "fn": 1}, "all": {"tp": 3, "fp": 4, "fn": 1}}, "augment": {"strict": {"tp": 1, "fp": 2, "fn": 1}, "core": {"tp": 1, "fp": 2, "fn": 2}, "all": {"tp": 1, "fp": 2, "fn": 3}}, "bugbot": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 3}}, "claude": {"strict": {"tp": 0, "fp": 5, "fn": 2}, "core": {"tp": 1, "fp": 5, "fn": 2}, "all": {"tp": 1, "fp": 5, "fn": 3}}, "claude-code": {"strict": {"tp": 0, "fp": 1, "fn": 2}, "core": {"tp": 1, "fp": 1, "fn": 2}, "all": {"tp": 2, "fp": 1, "fn": 2}}, "cubic-v2": {"strict": {"tp": 1, "fp": 2, "fn": 1}, "core": {"tp": 2, "fp": 2, "fn": 1}, "all": {"tp": 2, "fp": 2, "fn": 2}}, "devin": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 4}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 3}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 4}}, "kodus-v2": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 4}}, "macroscope": {"strict": {"tp": 2, "fp": 4, "fn": 0}, "core": {"tp": 2, "fp": 4, "fn": 1}, "all": {"tp": 2, "fp": 4, "fn": 2}}, "qodo-v2": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 2}, "all": {"tp": 1, "fp": 1, "fn": 3}}, "sourcery": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 1}, "all": {"tp": 2, "fp": 1, "fn": 2}}, "shipwright": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 2}, "all": {"tp": 1, "fp": 1, "fn": 3}}}}, {"url": "https://github.com/keycloak/keycloak/pull/40940", "language": "Java", "pr_size": "small", "domain": "concurrency", "change_type": "bug_fix", "complexity": "moderate", "difficulty": "subtle", "risk": "high", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "codeant-v2": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "coderabbit": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "qodo-extended-v2": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "greptile-v4-1": {"strict": {"tp": 1, "fp": 2, "fn": 1}, "core": {"tp": 1, "fp": 2, "fn": 1}, "all": {"tp": 1, "fp": 2, "fn": 1}}, "gemini-v2": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "gitlab": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "copilot-v2": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "augment": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "bugbot": {"strict": {"tp": 1, "fp": 2, "fn": 1}, "core": {"tp": 1, "fp": 2, "fn": 1}, "all": {"tp": 1, "fp": 2, "fn": 1}}, "claude": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "claude-code": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "cubic-v2": {"strict": {"tp": 2, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "devin": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "graphite": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "kodus-v2": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "macroscope": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "qodo-v2": {"strict": {"tp": 2, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "sourcery": {"strict": {"tp": 1, "fp": 2, "fn": 1}, "core": {"tp": 1, "fp": 2, "fn": 1}, "all": {"tp": 1, "fp": 2, "fn": 1}}, "shipwright": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}}}, {"url": "https://github.com/ai-code-review-evaluation/keycloak-greptile/pull/1", "language": "Java", "pr_size": "medium", "domain": "authentication", "change_type": "bug_fix", "complexity": "moderate", "difficulty": "subtle", "risk": "medium", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "codeant-v2": {"strict": {"tp": 1, "fp": 2, "fn": 0}, "core": {"tp": 1, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "coderabbit": {"strict": {"tp": 1, "fp": 3, "fn": 0}, "core": {"tp": 1, "fp": 3, "fn": 0}, "all": {"tp": 1, "fp": 3, "fn": 1}}, "qodo-extended-v2": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 1, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "greptile-v4-1": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 1, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "gemini-v2": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 1, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "gitlab": {"strict": {"tp": 1, "fp": 3, "fn": 0}, "core": {"tp": 1, "fp": 3, "fn": 0}, "all": {"tp": 1, "fp": 3, "fn": 1}}, "copilot-v2": {"strict": {"tp": 1, "fp": 2, "fn": 0}, "core": {"tp": 1, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "augment": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "bugbot": {"strict": {"tp": 0, "fp": 1, "fn": 1}, "core": {"tp": 0, "fp": 1, "fn": 1}, "all": {"tp": 0, "fp": 1, "fn": 2}}, "claude": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 0}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "claude-code": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 0}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "cubic-v2": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 1, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "devin": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "kg": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 0}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "kodus-v2": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "macroscope": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 1, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "qodo-v2": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 1, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "sourcery": {"strict": {"tp": 0, "fp": 3, "fn": 1}, "core": {"tp": 0, "fp": 3, "fn": 1}, "all": {"tp": 1, "fp": 3, "fn": 1}}, "shipwright": {"strict": {"tp": 0, "fp": 2, "fn": 1}, "core": {"tp": 0, "fp": 2, "fn": 1}, "all": {"tp": 0, "fp": 2, "fn": 2}}}}, {"url": "https://github.com/getsentry/sentry/pull/93824", "language": "Python", "pr_size": "large", "domain": "concurrency", "change_type": "feature", "complexity": "complex", "difficulty": "subtle", "risk": "high", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 0, "fp": 3, "fn": 3}, "core": {"tp": 0, "fp": 3, "fn": 5}, "all": {"tp": 0, "fp": 3, "fn": 5}}, "codeant-v2": {"strict": {"tp": 0, "fp": 0, "fn": 3}, "core": {"tp": 2, "fp": 0, "fn": 3}, "all": {"tp": 2, "fp": 0, "fn": 3}}, "coderabbit": {"strict": {"tp": 0, "fp": 1, "fn": 3}, "core": {"tp": 0, "fp": 1, "fn": 5}, "all": {"tp": 0, "fp": 1, "fn": 5}}, "qodo-extended-v2": {"strict": {"tp": 2, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 3}, "all": {"tp": 2, "fp": 1, "fn": 3}}, "greptile-v4-1": {"strict": {"tp": 1, "fp": 1, "fn": 2}, "core": {"tp": 1, "fp": 1, "fn": 4}, "all": {"tp": 1, "fp": 1, "fn": 4}}, "gemini-v2": {"strict": {"tp": 2, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 3}, "all": {"tp": 2, "fp": 1, "fn": 3}}, "gitlab": {"strict": {"tp": 0, "fp": 2, "fn": 3}, "core": {"tp": 0, "fp": 2, "fn": 5}, "all": {"tp": 0, "fp": 2, "fn": 5}}, "copilot-v2": {"strict": {"tp": 1, "fp": 4, "fn": 2}, "core": {"tp": 3, "fp": 4, "fn": 2}, "all": {"tp": 3, "fp": 4, "fn": 2}}, "augment": {"strict": {"tp": 3, "fp": 1, "fn": 0}, "core": {"tp": 5, "fp": 1, "fn": 0}, "all": {"tp": 5, "fp": 1, "fn": 0}}, "bugbot": {"strict": {"tp": 1, "fp": 0, "fn": 2}, "core": {"tp": 1, "fp": 0, "fn": 4}, "all": {"tp": 1, "fp": 0, "fn": 4}}, "claude": {"strict": {"tp": 0, "fp": 2, "fn": 3}, "core": {"tp": 0, "fp": 2, "fn": 5}, "all": {"tp": 0, "fp": 2, "fn": 5}}, "claude-code": {"strict": {"tp": 3, "fp": 3, "fn": 0}, "core": {"tp": 3, "fp": 3, "fn": 2}, "all": {"tp": 3, "fp": 3, "fn": 2}}, "cubic-v2": {"strict": {"tp": 3, "fp": 1, "fn": 0}, "core": {"tp": 5, "fp": 1, "fn": 0}, "all": {"tp": 5, "fp": 1, "fn": 0}}, "devin": {"strict": {"tp": 1, "fp": 0, "fn": 2}, "core": {"tp": 1, "fp": 0, "fn": 4}, "all": {"tp": 1, "fp": 0, "fn": 4}}, "graphite": {"strict": {"tp": 1, "fp": 0, "fn": 2}, "core": {"tp": 1, "fp": 0, "fn": 4}, "all": {"tp": 1, "fp": 0, "fn": 4}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 3}, "core": {"tp": 0, "fp": 0, "fn": 5}, "all": {"tp": 0, "fp": 0, "fn": 5}}, "kodus-v2": {"strict": {"tp": 0, "fp": 0, "fn": 3}, "core": {"tp": 0, "fp": 0, "fn": 5}, "all": {"tp": 0, "fp": 0, "fn": 5}}, "macroscope": {"strict": {"tp": 1, "fp": 1, "fn": 2}, "core": {"tp": 3, "fp": 1, "fn": 2}, "all": {"tp": 3, "fp": 1, "fn": 2}}, "qodo-v2": {"strict": {"tp": 2, "fp": 3, "fn": 1}, "core": {"tp": 4, "fp": 3, "fn": 1}, "all": {"tp": 4, "fp": 3, "fn": 1}}, "sourcery": {"strict": {"tp": 2, "fp": 2, "fn": 1}, "core": {"tp": 4, "fp": 2, "fn": 1}, "all": {"tp": 4, "fp": 2, "fn": 1}}, "shipwright": {"strict": {"tp": 2, "fp": 1, "fn": 1}, "core": {"tp": 4, "fp": 1, "fn": 1}, "all": {"tp": 4, "fp": 1, "fn": 1}}}}, {"url": "https://github.com/ai-code-review-evaluation/sentry-greptile/pull/5", "language": "Python", "pr_size": "large", "domain": "API", "change_type": "feature", "complexity": "moderate", "difficulty": "subtle", "risk": "medium", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 3, "fp": 1, "fn": 3}, "core": {"tp": 3, "fp": 1, "fn": 3}, "all": {"tp": 3, "fp": 1, "fn": 3}}, "codeant-v2": {"strict": {"tp": 2, "fp": 5, "fn": 4}, "core": {"tp": 2, "fp": 5, "fn": 4}, "all": {"tp": 2, "fp": 5, "fn": 4}}, "coderabbit": {"strict": {"tp": 4, "fp": 13, "fn": 2}, "core": {"tp": 4, "fp": 13, "fn": 2}, "all": {"tp": 4, "fp": 13, "fn": 2}}, "qodo-extended-v2": {"strict": {"tp": 2, "fp": 0, "fn": 4}, "core": {"tp": 2, "fp": 0, "fn": 4}, "all": {"tp": 2, "fp": 0, "fn": 4}}, "greptile-v4-1": {"strict": {"tp": 1, "fp": 0, "fn": 5}, "core": {"tp": 1, "fp": 0, "fn": 5}, "all": {"tp": 1, "fp": 0, "fn": 5}}, "gemini-v2": {"strict": {"tp": 2, "fp": 1, "fn": 4}, "core": {"tp": 2, "fp": 1, "fn": 4}, "all": {"tp": 2, "fp": 1, "fn": 4}}, "gitlab": {"strict": {"tp": 2, "fp": 1, "fn": 4}, "core": {"tp": 2, "fp": 1, "fn": 4}, "all": {"tp": 2, "fp": 1, "fn": 4}}, "copilot-v2": {"strict": {"tp": 3, "fp": 5, "fn": 3}, "core": {"tp": 3, "fp": 5, "fn": 3}, "all": {"tp": 3, "fp": 5, "fn": 3}}, "augment": {"strict": {"tp": 3, "fp": 2, "fn": 3}, "core": {"tp": 3, "fp": 2, "fn": 3}, "all": {"tp": 3, "fp": 2, "fn": 3}}, "bugbot": {"strict": {"tp": 2, "fp": 0, "fn": 4}, "core": {"tp": 2, "fp": 0, "fn": 4}, "all": {"tp": 2, "fp": 0, "fn": 4}}, "claude": {"strict": {"tp": 0, "fp": 0, "fn": 6}, "core": {"tp": 0, "fp": 0, "fn": 6}, "all": {"tp": 0, "fp": 0, "fn": 6}}, "claude-code": {"strict": {"tp": 0, "fp": 2, "fn": 6}, "core": {"tp": 0, "fp": 2, "fn": 6}, "all": {"tp": 0, "fp": 2, "fn": 6}}, "cubic-v2": {"strict": {"tp": 2, "fp": 2, "fn": 4}, "core": {"tp": 2, "fp": 2, "fn": 4}, "all": {"tp": 2, "fp": 2, "fn": 4}}, "devin": {"strict": {"tp": 2, "fp": 1, "fn": 4}, "core": {"tp": 2, "fp": 1, "fn": 4}, "all": {"tp": 2, "fp": 1, "fn": 4}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 6}, "core": {"tp": 0, "fp": 0, "fn": 6}, "all": {"tp": 0, "fp": 0, "fn": 6}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 6}, "core": {"tp": 0, "fp": 0, "fn": 6}, "all": {"tp": 0, "fp": 0, "fn": 6}}, "kodus-v2": {"strict": {"tp": 2, "fp": 1, "fn": 4}, "core": {"tp": 2, "fp": 1, "fn": 4}, "all": {"tp": 2, "fp": 1, "fn": 4}}, "macroscope": {"strict": {"tp": 2, "fp": 3, "fn": 4}, "core": {"tp": 2, "fp": 3, "fn": 4}, "all": {"tp": 2, "fp": 3, "fn": 4}}, "qodo-v2": {"strict": {"tp": 2, "fp": 0, "fn": 4}, "core": {"tp": 2, "fp": 0, "fn": 4}, "all": {"tp": 2, "fp": 0, "fn": 4}}, "sourcery": {"strict": {"tp": 2, "fp": 2, "fn": 4}, "core": {"tp": 2, "fp": 2, "fn": 4}, "all": {"tp": 2, "fp": 2, "fn": 4}}, "shipwright": {"strict": {"tp": 3, "fp": 0, "fn": 3}, "core": {"tp": 3, "fp": 0, "fn": 3}, "all": {"tp": 3, "fp": 0, "fn": 3}}}}, {"url": "https://github.com/ai-code-review-evaluation/sentry-greptile/pull/1", "language": "Python", "pr_size": "medium", "domain": "API", "change_type": "performance", "complexity": "moderate", "difficulty": "subtle", "risk": "high", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 1, "fp": 1, "fn": 3}, "core": {"tp": 1, "fp": 1, "fn": 3}, "all": {"tp": 1, "fp": 1, "fn": 3}}, "codeant-v2": {"strict": {"tp": 1, "fp": 0, "fn": 3}, "core": {"tp": 1, "fp": 0, "fn": 3}, "all": {"tp": 1, "fp": 0, "fn": 3}}, "coderabbit": {"strict": {"tp": 3, "fp": 1, "fn": 1}, "core": {"tp": 3, "fp": 1, "fn": 1}, "all": {"tp": 3, "fp": 1, "fn": 1}}, "qodo-extended-v2": {"strict": {"tp": 4, "fp": 0, "fn": 0}, "core": {"tp": 4, "fp": 0, "fn": 0}, "all": {"tp": 4, "fp": 0, "fn": 0}}, "greptile-v4-1": {"strict": {"tp": 3, "fp": 2, "fn": 1}, "core": {"tp": 3, "fp": 2, "fn": 1}, "all": {"tp": 3, "fp": 2, "fn": 1}}, "gemini-v2": {"strict": {"tp": 3, "fp": 1, "fn": 1}, "core": {"tp": 3, "fp": 1, "fn": 1}, "all": {"tp": 3, "fp": 1, "fn": 1}}, "gitlab": {"strict": {"tp": 3, "fp": 1, "fn": 1}, "core": {"tp": 3, "fp": 1, "fn": 1}, "all": {"tp": 3, "fp": 1, "fn": 1}}, "copilot-v2": {"strict": {"tp": 2, "fp": 1, "fn": 2}, "core": {"tp": 2, "fp": 1, "fn": 2}, "all": {"tp": 2, "fp": 1, "fn": 2}}, "augment": {"strict": {"tp": 3, "fp": 1, "fn": 1}, "core": {"tp": 3, "fp": 1, "fn": 1}, "all": {"tp": 3, "fp": 1, "fn": 1}}, "bugbot": {"strict": {"tp": 3, "fp": 1, "fn": 1}, "core": {"tp": 3, "fp": 1, "fn": 1}, "all": {"tp": 3, "fp": 1, "fn": 1}}, "claude": {"strict": {"tp": 2, "fp": 4, "fn": 2}, "core": {"tp": 2, "fp": 4, "fn": 2}, "all": {"tp": 2, "fp": 4, "fn": 2}}, "claude-code": {"strict": {"tp": 2, "fp": 1, "fn": 2}, "core": {"tp": 2, "fp": 1, "fn": 2}, "all": {"tp": 2, "fp": 1, "fn": 2}}, "cubic-v2": {"strict": {"tp": 3, "fp": 0, "fn": 1}, "core": {"tp": 3, "fp": 0, "fn": 1}, "all": {"tp": 3, "fp": 0, "fn": 1}}, "devin": {"strict": {"tp": 3, "fp": 1, "fn": 1}, "core": {"tp": 3, "fp": 1, "fn": 1}, "all": {"tp": 3, "fp": 1, "fn": 1}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 4}, "core": {"tp": 0, "fp": 0, "fn": 4}, "all": {"tp": 0, "fp": 0, "fn": 4}}, "kg": {"strict": {"tp": 1, "fp": 1, "fn": 3}, "core": {"tp": 1, "fp": 1, "fn": 3}, "all": {"tp": 1, "fp": 1, "fn": 3}}, "kodus-v2": {"strict": {"tp": 1, "fp": 4, "fn": 3}, "core": {"tp": 1, "fp": 4, "fn": 3}, "all": {"tp": 1, "fp": 4, "fn": 3}}, "macroscope": {"strict": {"tp": 2, "fp": 0, "fn": 2}, "core": {"tp": 2, "fp": 0, "fn": 2}, "all": {"tp": 2, "fp": 0, "fn": 2}}, "qodo-v2": {"strict": {"tp": 3, "fp": 0, "fn": 1}, "core": {"tp": 3, "fp": 0, "fn": 1}, "all": {"tp": 3, "fp": 0, "fn": 1}}, "sourcery": {"strict": {"tp": 2, "fp": 3, "fn": 2}, "core": {"tp": 2, "fp": 3, "fn": 2}, "all": {"tp": 2, "fp": 3, "fn": 2}}, "shipwright": {"strict": {"tp": 2, "fp": 3, "fn": 2}, "core": {"tp": 2, "fp": 3, "fn": 2}, "all": {"tp": 2, "fp": 3, "fn": 2}}}}, {"url": "https://github.com/grafana/grafana/pull/97529", "language": "Go", "pr_size": "small", "domain": "concurrency", "change_type": "performance", "complexity": "complex", "difficulty": "very_subtle", "risk": "critical", "context": "cross_file", "concern": "reliability", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "codeant-v2": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "coderabbit": {"strict": {"tp": 1, "fp": 3, "fn": 1}, "core": {"tp": 1, "fp": 3, "fn": 1}, "all": {"tp": 1, "fp": 3, "fn": 1}}, "qodo-extended-v2": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "greptile-v4-1": {"strict": {"tp": 2, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "gemini-v2": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "gitlab": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "copilot-v2": {"strict": {"tp": 1, "fp": 3, "fn": 1}, "core": {"tp": 1, "fp": 3, "fn": 1}, "all": {"tp": 1, "fp": 3, "fn": 1}}, "augment": {"strict": {"tp": 2, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "bugbot": {"strict": {"tp": 1, "fp": 2, "fn": 1}, "core": {"tp": 1, "fp": 2, "fn": 1}, "all": {"tp": 1, "fp": 2, "fn": 1}}, "claude": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "claude-code": {"strict": {"tp": 1, "fp": 4, "fn": 1}, "core": {"tp": 1, "fp": 4, "fn": 1}, "all": {"tp": 1, "fp": 4, "fn": 1}}, "cubic-v2": {"strict": {"tp": 2, "fp": 2, "fn": 0}, "core": {"tp": 2, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "devin": {"strict": {"tp": 0, "fp": 2, "fn": 2}, "core": {"tp": 0, "fp": 2, "fn": 2}, "all": {"tp": 0, "fp": 2, "fn": 2}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "kg": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "kodus-v2": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "macroscope": {"strict": {"tp": 2, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "qodo-v2": {"strict": {"tp": 2, "fp": 3, "fn": 0}, "core": {"tp": 2, "fp": 3, "fn": 0}, "all": {"tp": 2, "fp": 3, "fn": 0}}, "sourcery": {"strict": {"tp": 2, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "shipwright": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}}}, {"url": "https://github.com/getsentry/sentry/pull/80168", "language": "Python", "pr_size": "small", "domain": "data_processing", "change_type": "feature", "complexity": "moderate", "difficulty": "moderate", "risk": "high", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 0, "fp": 1, "fn": 1}, "core": {"tp": 0, "fp": 1, "fn": 3}, "all": {"tp": 0, "fp": 1, "fn": 3}}, "codeant-v2": {"strict": {"tp": 0, "fp": 1, "fn": 1}, "core": {"tp": 0, "fp": 1, "fn": 3}, "all": {"tp": 0, "fp": 1, "fn": 3}}, "coderabbit": {"strict": {"tp": 0, "fp": 2, "fn": 1}, "core": {"tp": 2, "fp": 2, "fn": 1}, "all": {"tp": 2, "fp": 2, "fn": 1}}, "qodo-extended-v2": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 1}, "all": {"tp": 2, "fp": 0, "fn": 1}}, "greptile-v4-1": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 1}, "all": {"tp": 2, "fp": 1, "fn": 1}}, "gemini-v2": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 2}}, "gitlab": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 2}}, "copilot-v2": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 1}, "all": {"tp": 2, "fp": 0, "fn": 1}}, "augment": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 3, "fp": 0, "fn": 0}, "all": {"tp": 3, "fp": 0, "fn": 0}}, "bugbot": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 2}}, "claude": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 3}}, "claude-code": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 3, "fp": 1, "fn": 0}, "all": {"tp": 3, "fp": 1, "fn": 0}}, "cubic-v2": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 3, "fp": 1, "fn": 0}, "all": {"tp": 3, "fp": 1, "fn": 0}}, "devin": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 1, "fp": 1, "fn": 2}, "all": {"tp": 1, "fp": 1, "fn": 2}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 3}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 3}}, "kodus-v2": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 3}}, "macroscope": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 3}}, "qodo-v2": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 2}}, "sourcery": {"strict": {"tp": 1, "fp": 2, "fn": 0}, "core": {"tp": 2, "fp": 2, "fn": 1}, "all": {"tp": 2, "fp": 2, "fn": 1}}, "shipwright": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 1}, "all": {"tp": 2, "fp": 0, "fn": 1}}}}, {"url": "https://github.com/getsentry/sentry/pull/80528", "language": "Python", "pr_size": "small", "domain": "scheduling", "change_type": "refactoring", "complexity": "moderate", "difficulty": "subtle", "risk": "medium", "context": "file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "codeant-v2": {"strict": {"tp": 0, "fp": 1, "fn": 1}, "core": {"tp": 0, "fp": 1, "fn": 2}, "all": {"tp": 0, "fp": 1, "fn": 2}}, "coderabbit": {"strict": {"tp": 1, "fp": 3, "fn": 0}, "core": {"tp": 1, "fp": 3, "fn": 1}, "all": {"tp": 1, "fp": 3, "fn": 1}}, "qodo-extended-v2": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "greptile-v4-1": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "gemini-v2": {"strict": {"tp": 1, "fp": 5, "fn": 0}, "core": {"tp": 1, "fp": 5, "fn": 1}, "all": {"tp": 1, "fp": 5, "fn": 1}}, "gitlab": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "copilot-v2": {"strict": {"tp": 1, "fp": 2, "fn": 0}, "core": {"tp": 1, "fp": 2, "fn": 1}, "all": {"tp": 1, "fp": 2, "fn": 1}}, "augment": {"strict": {"tp": 1, "fp": 2, "fn": 0}, "core": {"tp": 1, "fp": 2, "fn": 1}, "all": {"tp": 1, "fp": 2, "fn": 1}}, "bugbot": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "claude": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "claude-code": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "cubic-v2": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "devin": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "graphite": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "kg": {"strict": {"tp": 0, "fp": 1, "fn": 1}, "core": {"tp": 0, "fp": 1, "fn": 2}, "all": {"tp": 0, "fp": 1, "fn": 2}}, "kodus-v2": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "macroscope": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "qodo-v2": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "sourcery": {"strict": {"tp": 1, "fp": 2, "fn": 0}, "core": {"tp": 1, "fp": 2, "fn": 1}, "all": {"tp": 1, "fp": 2, "fn": 1}}, "shipwright": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}}}, {"url": "https://github.com/getsentry/sentry/pull/77754", "language": "Python", "pr_size": "medium", "domain": "serialization", "change_type": "feature", "complexity": "moderate", "difficulty": "moderate", "risk": "medium", "context": "file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 3}}, "codeant-v2": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 4}}, "coderabbit": {"strict": {"tp": 1, "fp": 2, "fn": 1}, "core": {"tp": 1, "fp": 2, "fn": 1}, "all": {"tp": 3, "fp": 2, "fn": 1}}, "qodo-extended-v2": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 3}}, "greptile-v4-1": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 3}}, "gemini-v2": {"strict": {"tp": 1, "fp": 2, "fn": 1}, "core": {"tp": 1, "fp": 2, "fn": 1}, "all": {"tp": 1, "fp": 2, "fn": 3}}, "gitlab": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 3, "fp": 0, "fn": 1}}, "copilot-v2": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 2, "fp": 1, "fn": 2}}, "augment": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 3}}, "bugbot": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 3}}, "claude": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 2, "fp": 0, "fn": 2}}, "claude-code": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 3}}, "cubic-v2": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 2}}, "devin": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 3}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 4}}, "kg": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 3}}, "kodus-v2": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 3}}, "macroscope": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 3}}, "qodo-v2": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 3}}, "sourcery": {"strict": {"tp": 1, "fp": 3, "fn": 1}, "core": {"tp": 1, "fp": 3, "fn": 1}, "all": {"tp": 2, "fp": 3, "fn": 2}}, "shipwright": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 3, "fp": 0, "fn": 1}}}}, {"url": "https://github.com/getsentry/sentry/pull/95633", "language": "Python", "pr_size": "medium", "domain": "concurrency", "change_type": "feature", "complexity": "moderate", "difficulty": "moderate", "risk": "medium", "context": "file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 0, "fp": 2, "fn": 0}, "core": {"tp": 0, "fp": 2, "fn": 1}, "all": {"tp": 0, "fp": 2, "fn": 2}}, "codeant-v2": {"strict": {"tp": 0, "fp": 1, "fn": 0}, "core": {"tp": 0, "fp": 1, "fn": 1}, "all": {"tp": 0, "fp": 1, "fn": 2}}, "coderabbit": {"strict": {"tp": 0, "fp": 7, "fn": 0}, "core": {"tp": 0, "fp": 7, "fn": 1}, "all": {"tp": 0, "fp": 7, "fn": 2}}, "qodo-extended-v2": {"strict": {"tp": 0, "fp": 4, "fn": 0}, "core": {"tp": 1, "fp": 4, "fn": 0}, "all": {"tp": 1, "fp": 4, "fn": 1}}, "greptile-v4-1": {"strict": {"tp": 0, "fp": 3, "fn": 0}, "core": {"tp": 0, "fp": 3, "fn": 1}, "all": {"tp": 0, "fp": 3, "fn": 2}}, "gemini-v2": {"strict": {"tp": 0, "fp": 5, "fn": 0}, "core": {"tp": 0, "fp": 5, "fn": 1}, "all": {"tp": 0, "fp": 5, "fn": 2}}, "gitlab": {"strict": {"tp": 0, "fp": 5, "fn": 0}, "core": {"tp": 1, "fp": 5, "fn": 0}, "all": {"tp": 1, "fp": 5, "fn": 1}}, "copilot-v2": {"strict": {"tp": 0, "fp": 3, "fn": 0}, "core": {"tp": 0, "fp": 3, "fn": 1}, "all": {"tp": 0, "fp": 3, "fn": 2}}, "augment": {"strict": {"tp": 0, "fp": 3, "fn": 0}, "core": {"tp": 0, "fp": 3, "fn": 1}, "all": {"tp": 0, "fp": 3, "fn": 2}}, "bugbot": {"strict": {"tp": 0, "fp": 2, "fn": 0}, "core": {"tp": 0, "fp": 2, "fn": 1}, "all": {"tp": 0, "fp": 2, "fn": 2}}, "claude": {"strict": {"tp": 0, "fp": 0, "fn": 0}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "claude-code": {"strict": {"tp": 0, "fp": 2, "fn": 0}, "core": {"tp": 0, "fp": 2, "fn": 1}, "all": {"tp": 0, "fp": 2, "fn": 2}}, "cubic-v2": {"strict": {"tp": 0, "fp": 3, "fn": 0}, "core": {"tp": 0, "fp": 3, "fn": 1}, "all": {"tp": 0, "fp": 3, "fn": 2}}, "devin": {"strict": {"tp": 0, "fp": 2, "fn": 0}, "core": {"tp": 0, "fp": 2, "fn": 1}, "all": {"tp": 0, "fp": 2, "fn": 2}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 0}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "kg": {"strict": {"tp": 0, "fp": 2, "fn": 0}, "core": {"tp": 0, "fp": 2, "fn": 1}, "all": {"tp": 0, "fp": 2, "fn": 2}}, "kodus-v2": {"strict": {"tp": 0, "fp": 0, "fn": 0}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "macroscope": {"strict": {"tp": 0, "fp": 2, "fn": 0}, "core": {"tp": 0, "fp": 2, "fn": 1}, "all": {"tp": 0, "fp": 2, "fn": 2}}, "qodo-v2": {"strict": {"tp": 0, "fp": 4, "fn": 0}, "core": {"tp": 0, "fp": 4, "fn": 1}, "all": {"tp": 0, "fp": 4, "fn": 2}}, "sourcery": {"strict": {"tp": 0, "fp": 4, "fn": 0}, "core": {"tp": 1, "fp": 4, "fn": 0}, "all": {"tp": 1, "fp": 4, "fn": 1}}, "shipwright": {"strict": {"tp": 0, "fp": 2, "fn": 0}, "core": {"tp": 0, "fp": 2, "fn": 1}, "all": {"tp": 0, "fp": 2, "fn": 2}}}}, {"url": "https://github.com/ai-code-review-evaluation/sentry-greptile/pull/2", "language": "Python", "pr_size": "medium", "domain": "data_processing", "change_type": "performance", "complexity": "moderate", "difficulty": "subtle", "risk": "high", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 3, "fp": 1, "fn": 1}, "core": {"tp": 3, "fp": 1, "fn": 1}, "all": {"tp": 3, "fp": 1, "fn": 1}}, "codeant-v2": {"strict": {"tp": 3, "fp": 1, "fn": 1}, "core": {"tp": 3, "fp": 1, "fn": 1}, "all": {"tp": 3, "fp": 1, "fn": 1}}, "coderabbit": {"strict": {"tp": 0, "fp": 0, "fn": 4}, "core": {"tp": 0, "fp": 0, "fn": 4}, "all": {"tp": 0, "fp": 0, "fn": 4}}, "qodo-extended-v2": {"strict": {"tp": 3, "fp": 3, "fn": 1}, "core": {"tp": 3, "fp": 3, "fn": 1}, "all": {"tp": 3, "fp": 3, "fn": 1}}, "greptile-v4-1": {"strict": {"tp": 3, "fp": 2, "fn": 1}, "core": {"tp": 3, "fp": 2, "fn": 1}, "all": {"tp": 3, "fp": 2, "fn": 1}}, "gemini-v2": {"strict": {"tp": 3, "fp": 3, "fn": 1}, "core": {"tp": 3, "fp": 3, "fn": 1}, "all": {"tp": 3, "fp": 3, "fn": 1}}, "gitlab": {"strict": {"tp": 4, "fp": 1, "fn": 0}, "core": {"tp": 4, "fp": 1, "fn": 0}, "all": {"tp": 4, "fp": 1, "fn": 0}}, "copilot-v2": {"strict": {"tp": 3, "fp": 8, "fn": 1}, "core": {"tp": 3, "fp": 8, "fn": 1}, "all": {"tp": 3, "fp": 8, "fn": 1}}, "augment": {"strict": {"tp": 3, "fp": 2, "fn": 1}, "core": {"tp": 3, "fp": 2, "fn": 1}, "all": {"tp": 3, "fp": 2, "fn": 1}}, "bugbot": {"strict": {"tp": 4, "fp": 0, "fn": 0}, "core": {"tp": 4, "fp": 0, "fn": 0}, "all": {"tp": 4, "fp": 0, "fn": 0}}, "claude": {"strict": {"tp": 3, "fp": 5, "fn": 1}, "core": {"tp": 3, "fp": 5, "fn": 1}, "all": {"tp": 3, "fp": 5, "fn": 1}}, "claude-code": {"strict": {"tp": 3, "fp": 3, "fn": 1}, "core": {"tp": 3, "fp": 3, "fn": 1}, "all": {"tp": 3, "fp": 3, "fn": 1}}, "cubic-v2": {"strict": {"tp": 3, "fp": 1, "fn": 1}, "core": {"tp": 3, "fp": 1, "fn": 1}, "all": {"tp": 3, "fp": 1, "fn": 1}}, "devin": {"strict": {"tp": 0, "fp": 0, "fn": 4}, "core": {"tp": 0, "fp": 0, "fn": 4}, "all": {"tp": 0, "fp": 0, "fn": 4}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 4}, "core": {"tp": 0, "fp": 0, "fn": 4}, "all": {"tp": 0, "fp": 0, "fn": 4}}, "kg": {"strict": {"tp": 2, "fp": 2, "fn": 2}, "core": {"tp": 2, "fp": 2, "fn": 2}, "all": {"tp": 2, "fp": 2, "fn": 2}}, "kodus-v2": {"strict": {"tp": 2, "fp": 3, "fn": 2}, "core": {"tp": 2, "fp": 3, "fn": 2}, "all": {"tp": 2, "fp": 3, "fn": 2}}, "macroscope": {"strict": {"tp": 3, "fp": 0, "fn": 1}, "core": {"tp": 3, "fp": 0, "fn": 1}, "all": {"tp": 3, "fp": 0, "fn": 1}}, "qodo-v2": {"strict": {"tp": 4, "fp": 0, "fn": 0}, "core": {"tp": 4, "fp": 0, "fn": 0}, "all": {"tp": 4, "fp": 0, "fn": 0}}, "sourcery": {"strict": {"tp": 3, "fp": 7, "fn": 1}, "core": {"tp": 3, "fp": 7, "fn": 1}, "all": {"tp": 3, "fp": 7, "fn": 1}}, "shipwright": {"strict": {"tp": 3, "fp": 4, "fn": 1}, "core": {"tp": 3, "fp": 4, "fn": 1}, "all": {"tp": 3, "fp": 4, "fn": 1}}}}, {"url": "https://github.com/ai-code-review-evaluation/sentry-greptile/pull/3", "language": "Python", "pr_size": "medium", "domain": "caching", "change_type": "feature", "complexity": "complex", "difficulty": "subtle", "risk": "medium", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 1, "fp": 1, "fn": 2}, "core": {"tp": 1, "fp": 1, "fn": 2}, "all": {"tp": 1, "fp": 1, "fn": 2}}, "codeant-v2": {"strict": {"tp": 3, "fp": 1, "fn": 0}, "core": {"tp": 3, "fp": 1, "fn": 0}, "all": {"tp": 3, "fp": 1, "fn": 0}}, "coderabbit": {"strict": {"tp": 2, "fp": 5, "fn": 1}, "core": {"tp": 2, "fp": 5, "fn": 1}, "all": {"tp": 2, "fp": 5, "fn": 1}}, "qodo-extended-v2": {"strict": {"tp": 3, "fp": 0, "fn": 0}, "core": {"tp": 3, "fp": 0, "fn": 0}, "all": {"tp": 3, "fp": 0, "fn": 0}}, "greptile-v4-1": {"strict": {"tp": 2, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 1}, "all": {"tp": 2, "fp": 1, "fn": 1}}, "gemini-v2": {"strict": {"tp": 2, "fp": 2, "fn": 1}, "core": {"tp": 2, "fp": 2, "fn": 1}, "all": {"tp": 2, "fp": 2, "fn": 1}}, "gitlab": {"strict": {"tp": 2, "fp": 2, "fn": 1}, "core": {"tp": 2, "fp": 2, "fn": 1}, "all": {"tp": 2, "fp": 2, "fn": 1}}, "copilot-v2": {"strict": {"tp": 1, "fp": 6, "fn": 2}, "core": {"tp": 1, "fp": 6, "fn": 2}, "all": {"tp": 1, "fp": 6, "fn": 2}}, "augment": {"strict": {"tp": 2, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 1}, "all": {"tp": 2, "fp": 1, "fn": 1}}, "bugbot": {"strict": {"tp": 0, "fp": 3, "fn": 3}, "core": {"tp": 0, "fp": 3, "fn": 3}, "all": {"tp": 0, "fp": 3, "fn": 3}}, "claude": {"strict": {"tp": 1, "fp": 0, "fn": 2}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 2}}, "claude-code": {"strict": {"tp": 1, "fp": 2, "fn": 2}, "core": {"tp": 1, "fp": 2, "fn": 2}, "all": {"tp": 1, "fp": 2, "fn": 2}}, "cubic-v2": {"strict": {"tp": 2, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 1}, "all": {"tp": 2, "fp": 1, "fn": 1}}, "devin": {"strict": {"tp": 2, "fp": 0, "fn": 1}, "core": {"tp": 2, "fp": 0, "fn": 1}, "all": {"tp": 2, "fp": 0, "fn": 1}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 3}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 3}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 3}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 3}}, "kodus-v2": {"strict": {"tp": 2, "fp": 2, "fn": 1}, "core": {"tp": 2, "fp": 2, "fn": 1}, "all": {"tp": 2, "fp": 2, "fn": 1}}, "macroscope": {"strict": {"tp": 2, "fp": 0, "fn": 1}, "core": {"tp": 2, "fp": 0, "fn": 1}, "all": {"tp": 2, "fp": 0, "fn": 1}}, "qodo-v2": {"strict": {"tp": 2, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 1}, "all": {"tp": 2, "fp": 1, "fn": 1}}, "sourcery": {"strict": {"tp": 2, "fp": 6, "fn": 1}, "core": {"tp": 2, "fp": 6, "fn": 1}, "all": {"tp": 2, "fp": 6, "fn": 1}}, "shipwright": {"strict": {"tp": 1, "fp": 3, "fn": 2}, "core": {"tp": 1, "fp": 3, "fn": 2}, "all": {"tp": 1, "fp": 3, "fn": 2}}}}, {"url": "https://github.com/grafana/grafana/pull/103633", "language": "Go", "pr_size": "small", "domain": "caching", "change_type": "performance", "complexity": "moderate", "difficulty": "subtle", "risk": "high", "context": "cross_file", "concern": "security", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 2}, "all": {"tp": 1, "fp": 1, "fn": 2}}, "codeant-v2": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 2, "fp": 0, "fn": 1}, "all": {"tp": 2, "fp": 0, "fn": 1}}, "coderabbit": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 2}}, "qodo-extended-v2": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 2, "fp": 0, "fn": 1}, "all": {"tp": 2, "fp": 0, "fn": 1}}, "greptile-v4-1": {"strict": {"tp": 1, "fp": 2, "fn": 1}, "core": {"tp": 1, "fp": 2, "fn": 2}, "all": {"tp": 1, "fp": 2, "fn": 2}}, "gemini-v2": {"strict": {"tp": 0, "fp": 1, "fn": 2}, "core": {"tp": 0, "fp": 1, "fn": 3}, "all": {"tp": 0, "fp": 1, "fn": 3}}, "gitlab": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 2}, "all": {"tp": 1, "fp": 1, "fn": 2}}, "copilot-v2": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 1}, "all": {"tp": 2, "fp": 1, "fn": 1}}, "augment": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 2, "fp": 0, "fn": 1}, "all": {"tp": 2, "fp": 0, "fn": 1}}, "bugbot": {"strict": {"tp": 0, "fp": 1, "fn": 2}, "core": {"tp": 0, "fp": 1, "fn": 3}, "all": {"tp": 0, "fp": 1, "fn": 3}}, "claude": {"strict": {"tp": 0, "fp": 6, "fn": 2}, "core": {"tp": 0, "fp": 6, "fn": 3}, "all": {"tp": 0, "fp": 6, "fn": 3}}, "claude-code": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 2}, "all": {"tp": 1, "fp": 1, "fn": 2}}, "cubic-v2": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 3, "fp": 0, "fn": 0}, "all": {"tp": 3, "fp": 0, "fn": 0}}, "devin": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 3, "fp": 0, "fn": 0}, "all": {"tp": 3, "fp": 0, "fn": 0}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 3}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 3}}, "kodus-v2": {"strict": {"tp": 0, "fp": 1, "fn": 2}, "core": {"tp": 0, "fp": 1, "fn": 3}, "all": {"tp": 0, "fp": 1, "fn": 3}}, "macroscope": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 2}}, "qodo-v2": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 1}, "all": {"tp": 2, "fp": 1, "fn": 1}}, "sourcery": {"strict": {"tp": 0, "fp": 2, "fn": 2}, "core": {"tp": 0, "fp": 2, "fn": 3}, "all": {"tp": 0, "fp": 2, "fn": 3}}, "shipwright": {"strict": {"tp": 0, "fp": 1, "fn": 2}, "core": {"tp": 0, "fp": 1, "fn": 3}, "all": {"tp": 0, "fp": 1, "fn": 3}}}}, {"url": "https://github.com/getsentry/sentry/pull/67876", "language": "Python", "pr_size": "small", "domain": "authentication", "change_type": "security_patch", "complexity": "moderate", "difficulty": "subtle", "risk": "high", "context": "cross_file", "concern": "security", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 1, "fp": 1, "fn": 2}, "core": {"tp": 1, "fp": 1, "fn": 2}, "all": {"tp": 1, "fp": 1, "fn": 2}}, "codeant-v2": {"strict": {"tp": 0, "fp": 1, "fn": 3}, "core": {"tp": 0, "fp": 1, "fn": 3}, "all": {"tp": 0, "fp": 1, "fn": 3}}, "coderabbit": {"strict": {"tp": 2, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 1}, "all": {"tp": 2, "fp": 1, "fn": 1}}, "qodo-extended-v2": {"strict": {"tp": 2, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 1}, "all": {"tp": 2, "fp": 1, "fn": 1}}, "greptile-v4-1": {"strict": {"tp": 1, "fp": 1, "fn": 2}, "core": {"tp": 1, "fp": 1, "fn": 2}, "all": {"tp": 1, "fp": 1, "fn": 2}}, "gemini-v2": {"strict": {"tp": 1, "fp": 1, "fn": 2}, "core": {"tp": 1, "fp": 1, "fn": 2}, "all": {"tp": 1, "fp": 1, "fn": 2}}, "gitlab": {"strict": {"tp": 1, "fp": 1, "fn": 2}, "core": {"tp": 1, "fp": 1, "fn": 2}, "all": {"tp": 1, "fp": 1, "fn": 2}}, "copilot-v2": {"strict": {"tp": 2, "fp": 13, "fn": 1}, "core": {"tp": 2, "fp": 13, "fn": 1}, "all": {"tp": 2, "fp": 13, "fn": 1}}, "augment": {"strict": {"tp": 1, "fp": 2, "fn": 2}, "core": {"tp": 1, "fp": 2, "fn": 2}, "all": {"tp": 1, "fp": 2, "fn": 2}}, "bugbot": {"strict": {"tp": 1, "fp": 2, "fn": 2}, "core": {"tp": 1, "fp": 2, "fn": 2}, "all": {"tp": 1, "fp": 2, "fn": 2}}, "claude": {"strict": {"tp": 1, "fp": 1, "fn": 2}, "core": {"tp": 1, "fp": 1, "fn": 2}, "all": {"tp": 1, "fp": 1, "fn": 2}}, "claude-code": {"strict": {"tp": 2, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 1}, "all": {"tp": 2, "fp": 1, "fn": 1}}, "cubic-v2": {"strict": {"tp": 3, "fp": 2, "fn": 0}, "core": {"tp": 3, "fp": 2, "fn": 0}, "all": {"tp": 3, "fp": 2, "fn": 0}}, "devin": {"strict": {"tp": 1, "fp": 0, "fn": 2}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 2}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 3}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 3}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 3}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 3}}, "kodus-v2": {"strict": {"tp": 0, "fp": 0, "fn": 3}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 3}}, "macroscope": {"strict": {"tp": 2, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 1}, "all": {"tp": 2, "fp": 1, "fn": 1}}, "qodo-v2": {"strict": {"tp": 2, "fp": 3, "fn": 1}, "core": {"tp": 2, "fp": 3, "fn": 1}, "all": {"tp": 2, "fp": 3, "fn": 1}}, "sourcery": {"strict": {"tp": 2, "fp": 7, "fn": 1}, "core": {"tp": 2, "fp": 7, "fn": 1}, "all": {"tp": 2, "fp": 7, "fn": 1}}, "shipwright": {"strict": {"tp": 1, "fp": 1, "fn": 2}, "core": {"tp": 1, "fp": 1, "fn": 2}, "all": {"tp": 1, "fp": 1, "fn": 2}}}}, {"url": "https://github.com/keycloak/keycloak/pull/32918", "language": "Java", "pr_size": "small", "domain": "caching", "change_type": "feature", "complexity": "moderate", "difficulty": "subtle", "risk": "high", "context": "file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "codeant-v2": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "coderabbit": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "qodo-extended-v2": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "greptile-v4-1": {"strict": {"tp": 0, "fp": 3, "fn": 2}, "core": {"tp": 0, "fp": 3, "fn": 2}, "all": {"tp": 0, "fp": 3, "fn": 2}}, "gemini-v2": {"strict": {"tp": 0, "fp": 2, "fn": 2}, "core": {"tp": 0, "fp": 2, "fn": 2}, "all": {"tp": 0, "fp": 2, "fn": 2}}, "gitlab": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "copilot-v2": {"strict": {"tp": 1, "fp": 3, "fn": 1}, "core": {"tp": 1, "fp": 3, "fn": 1}, "all": {"tp": 1, "fp": 3, "fn": 1}}, "augment": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "bugbot": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "claude": {"strict": {"tp": 0, "fp": 1, "fn": 2}, "core": {"tp": 0, "fp": 1, "fn": 2}, "all": {"tp": 0, "fp": 1, "fn": 2}}, "claude-code": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "cubic-v2": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "devin": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "graphite": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "kg": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "kodus-v2": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "macroscope": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "qodo-v2": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "sourcery": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "shipwright": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}}}, {"url": "https://github.com/grafana/grafana/pull/94942", "language": "Go", "pr_size": "small", "domain": "data_processing", "change_type": "feature", "complexity": "moderate", "difficulty": "moderate", "risk": "high", "context": "file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "codeant-v2": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "coderabbit": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "qodo-extended-v2": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "greptile-v4-1": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "gemini-v2": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "gitlab": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "copilot-v2": {"strict": {"tp": 2, "fp": 2, "fn": 0}, "core": {"tp": 2, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "augment": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "bugbot": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "claude": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "claude-code": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "cubic-v2": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "devin": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "kg": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "kodus-v2": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "macroscope": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "qodo-v2": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "sourcery": {"strict": {"tp": 2, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "shipwright": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}}}, {"url": "https://github.com/grafana/grafana/pull/90939", "language": "Go", "pr_size": "small", "domain": "caching", "change_type": "performance", "complexity": "moderate", "difficulty": "subtle", "risk": "high", "context": "file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "codeant-v2": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "coderabbit": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "qodo-extended-v2": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "greptile-v4-1": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "gemini-v2": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "gitlab": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "copilot-v2": {"strict": {"tp": 1, "fp": 4, "fn": 1}, "core": {"tp": 1, "fp": 4, "fn": 1}, "all": {"tp": 1, "fp": 4, "fn": 1}}, "augment": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "bugbot": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "claude": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "claude-code": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "cubic-v2": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "devin": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "kg": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "kodus-v2": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "macroscope": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "qodo-v2": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "sourcery": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "shipwright": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}}}, {"url": "https://github.com/grafana/grafana/pull/80329", "language": "Go", "pr_size": "small", "domain": "logging", "change_type": "performance", "complexity": "simple", "difficulty": "obvious", "risk": "low", "context": "local", "concern": "maintainability", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 0, "fp": 0, "fn": 0}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "codeant-v2": {"strict": {"tp": 0, "fp": 0, "fn": 0}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "coderabbit": {"strict": {"tp": 0, "fp": 2, "fn": 0}, "core": {"tp": 1, "fp": 2, "fn": 1}, "all": {"tp": 1, "fp": 2, "fn": 1}}, "qodo-extended-v2": {"strict": {"tp": 0, "fp": 2, "fn": 0}, "core": {"tp": 2, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "greptile-v4-1": {"strict": {"tp": 0, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "gemini-v2": {"strict": {"tp": 0, "fp": 2, "fn": 0}, "core": {"tp": 2, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "gitlab": {"strict": {"tp": 0, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "copilot-v2": {"strict": {"tp": 0, "fp": 2, "fn": 0}, "core": {"tp": 2, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "augment": {"strict": {"tp": 0, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "bugbot": {"strict": {"tp": 0, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "claude": {"strict": {"tp": 0, "fp": 4, "fn": 0}, "core": {"tp": 2, "fp": 4, "fn": 0}, "all": {"tp": 2, "fp": 4, "fn": 0}}, "claude-code": {"strict": {"tp": 0, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "cubic-v2": {"strict": {"tp": 0, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "devin": {"strict": {"tp": 0, "fp": 2, "fn": 0}, "core": {"tp": 2, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 0}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 0}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "kodus-v2": {"strict": {"tp": 0, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "macroscope": {"strict": {"tp": 0, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "qodo-v2": {"strict": {"tp": 0, "fp": 2, "fn": 0}, "core": {"tp": 2, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "sourcery": {"strict": {"tp": 0, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "shipwright": {"strict": {"tp": 0, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}}}, {"url": "https://github.com/grafana/grafana/pull/90045", "language": "Go", "pr_size": "medium", "domain": "logging", "change_type": "feature", "complexity": "moderate", "difficulty": "moderate", "risk": "medium", "context": "file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 2, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 1}, "all": {"tp": 2, "fp": 1, "fn": 1}}, "codeant-v2": {"strict": {"tp": 2, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 1}, "all": {"tp": 2, "fp": 1, "fn": 1}}, "coderabbit": {"strict": {"tp": 3, "fp": 2, "fn": 0}, "core": {"tp": 3, "fp": 2, "fn": 0}, "all": {"tp": 3, "fp": 2, "fn": 0}}, "qodo-extended-v2": {"strict": {"tp": 3, "fp": 2, "fn": 0}, "core": {"tp": 3, "fp": 2, "fn": 0}, "all": {"tp": 3, "fp": 2, "fn": 0}}, "greptile-v4-1": {"strict": {"tp": 3, "fp": 3, "fn": 0}, "core": {"tp": 3, "fp": 3, "fn": 0}, "all": {"tp": 3, "fp": 3, "fn": 0}}, "gemini-v2": {"strict": {"tp": 3, "fp": 5, "fn": 0}, "core": {"tp": 3, "fp": 5, "fn": 0}, "all": {"tp": 3, "fp": 5, "fn": 0}}, "gitlab": {"strict": {"tp": 3, "fp": 1, "fn": 0}, "core": {"tp": 3, "fp": 1, "fn": 0}, "all": {"tp": 3, "fp": 1, "fn": 0}}, "copilot-v2": {"strict": {"tp": 3, "fp": 5, "fn": 0}, "core": {"tp": 3, "fp": 5, "fn": 0}, "all": {"tp": 3, "fp": 5, "fn": 0}}, "augment": {"strict": {"tp": 2, "fp": 6, "fn": 1}, "core": {"tp": 2, "fp": 6, "fn": 1}, "all": {"tp": 2, "fp": 6, "fn": 1}}, "bugbot": {"strict": {"tp": 2, "fp": 4, "fn": 1}, "core": {"tp": 2, "fp": 4, "fn": 1}, "all": {"tp": 2, "fp": 4, "fn": 1}}, "claude": {"strict": {"tp": 3, "fp": 3, "fn": 0}, "core": {"tp": 3, "fp": 3, "fn": 0}, "all": {"tp": 3, "fp": 3, "fn": 0}}, "claude-code": {"strict": {"tp": 3, "fp": 2, "fn": 0}, "core": {"tp": 3, "fp": 2, "fn": 0}, "all": {"tp": 3, "fp": 2, "fn": 0}}, "cubic-v2": {"strict": {"tp": 3, "fp": 2, "fn": 0}, "core": {"tp": 3, "fp": 2, "fn": 0}, "all": {"tp": 3, "fp": 2, "fn": 0}}, "devin": {"strict": {"tp": 3, "fp": 2, "fn": 0}, "core": {"tp": 3, "fp": 2, "fn": 0}, "all": {"tp": 3, "fp": 2, "fn": 0}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 3}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 3}}, "kg": {"strict": {"tp": 3, "fp": 1, "fn": 0}, "core": {"tp": 3, "fp": 1, "fn": 0}, "all": {"tp": 3, "fp": 1, "fn": 0}}, "kodus-v2": {"strict": {"tp": 1, "fp": 0, "fn": 2}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 2}}, "macroscope": {"strict": {"tp": 2, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 1}, "all": {"tp": 2, "fp": 1, "fn": 1}}, "qodo-v2": {"strict": {"tp": 2, "fp": 6, "fn": 1}, "core": {"tp": 2, "fp": 6, "fn": 1}, "all": {"tp": 2, "fp": 6, "fn": 1}}, "sourcery": {"strict": {"tp": 3, "fp": 1, "fn": 0}, "core": {"tp": 3, "fp": 1, "fn": 0}, "all": {"tp": 3, "fp": 1, "fn": 0}}, "shipwright": {"strict": {"tp": 2, "fp": 6, "fn": 1}, "core": {"tp": 2, "fp": 6, "fn": 1}, "all": {"tp": 2, "fp": 6, "fn": 1}}}}, {"url": "https://github.com/grafana/grafana/pull/106778", "language": "Go", "pr_size": "medium", "domain": "UI", "change_type": "feature", "complexity": "moderate", "difficulty": "subtle", "risk": "medium", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 0, "fp": 2, "fn": 2}, "core": {"tp": 0, "fp": 2, "fn": 2}, "all": {"tp": 0, "fp": 2, "fn": 2}}, "codeant-v2": {"strict": {"tp": 1, "fp": 2, "fn": 1}, "core": {"tp": 1, "fp": 2, "fn": 1}, "all": {"tp": 1, "fp": 2, "fn": 1}}, "coderabbit": {"strict": {"tp": 1, "fp": 6, "fn": 1}, "core": {"tp": 1, "fp": 6, "fn": 1}, "all": {"tp": 1, "fp": 6, "fn": 1}}, "qodo-extended-v2": {"strict": {"tp": 2, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "greptile-v4-1": {"strict": {"tp": 1, "fp": 3, "fn": 1}, "core": {"tp": 1, "fp": 3, "fn": 1}, "all": {"tp": 1, "fp": 3, "fn": 1}}, "gemini-v2": {"strict": {"tp": 0, "fp": 3, "fn": 2}, "core": {"tp": 0, "fp": 3, "fn": 2}, "all": {"tp": 0, "fp": 3, "fn": 2}}, "gitlab": {"strict": {"tp": 2, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "copilot-v2": {"strict": {"tp": 2, "fp": 2, "fn": 0}, "core": {"tp": 2, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "augment": {"strict": {"tp": 1, "fp": 2, "fn": 1}, "core": {"tp": 1, "fp": 2, "fn": 1}, "all": {"tp": 1, "fp": 2, "fn": 1}}, "bugbot": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "claude": {"strict": {"tp": 0, "fp": 10, "fn": 2}, "core": {"tp": 0, "fp": 10, "fn": 2}, "all": {"tp": 0, "fp": 10, "fn": 2}}, "claude-code": {"strict": {"tp": 2, "fp": 2, "fn": 0}, "core": {"tp": 2, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "cubic-v2": {"strict": {"tp": 2, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "devin": {"strict": {"tp": 2, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "kg": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "kodus-v2": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "macroscope": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "qodo-v2": {"strict": {"tp": 0, "fp": 2, "fn": 2}, "core": {"tp": 0, "fp": 2, "fn": 2}, "all": {"tp": 0, "fp": 2, "fn": 2}}, "sourcery": {"strict": {"tp": 0, "fp": 2, "fn": 2}, "core": {"tp": 0, "fp": 2, "fn": 2}, "all": {"tp": 0, "fp": 2, "fn": 2}}, "shipwright": {"strict": {"tp": 0, "fp": 4, "fn": 2}, "core": {"tp": 0, "fp": 4, "fn": 2}, "all": {"tp": 0, "fp": 4, "fn": 2}}}}, {"url": "https://github.com/grafana/grafana/pull/107534", "language": "Go", "pr_size": "small", "domain": "testing", "change_type": "feature", "complexity": "moderate", "difficulty": "subtle", "risk": "low", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 0, "fp": 2, "fn": 0}, "core": {"tp": 0, "fp": 2, "fn": 1}, "all": {"tp": 0, "fp": 2, "fn": 1}}, "codeant-v2": {"strict": {"tp": 0, "fp": 0, "fn": 0}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 0, "fp": 0, "fn": 1}}, "coderabbit": {"strict": {"tp": 0, "fp": 1, "fn": 0}, "core": {"tp": 0, "fp": 1, "fn": 1}, "all": {"tp": 0, "fp": 1, "fn": 1}}, "qodo-extended-v2": {"strict": {"tp": 0, "fp": 1, "fn": 0}, "core": {"tp": 0, "fp": 1, "fn": 1}, "all": {"tp": 0, "fp": 1, "fn": 1}}, "greptile-v4-1": {"strict": {"tp": 0, "fp": 3, "fn": 0}, "core": {"tp": 0, "fp": 3, "fn": 1}, "all": {"tp": 0, "fp": 3, "fn": 1}}, "gemini-v2": {"strict": {"tp": 0, "fp": 3, "fn": 0}, "core": {"tp": 1, "fp": 3, "fn": 0}, "all": {"tp": 1, "fp": 3, "fn": 0}}, "gitlab": {"strict": {"tp": 0, "fp": 2, "fn": 0}, "core": {"tp": 0, "fp": 2, "fn": 1}, "all": {"tp": 0, "fp": 2, "fn": 1}}, "copilot-v2": {"strict": {"tp": 0, "fp": 2, "fn": 0}, "core": {"tp": 0, "fp": 2, "fn": 1}, "all": {"tp": 0, "fp": 2, "fn": 1}}, "augment": {"strict": {"tp": 0, "fp": 0, "fn": 0}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 0, "fp": 0, "fn": 1}}, "bugbot": {"strict": {"tp": 0, "fp": 1, "fn": 0}, "core": {"tp": 0, "fp": 1, "fn": 1}, "all": {"tp": 0, "fp": 1, "fn": 1}}, "claude": {"strict": {"tp": 0, "fp": 0, "fn": 0}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 0, "fp": 0, "fn": 1}}, "claude-code": {"strict": {"tp": 0, "fp": 3, "fn": 0}, "core": {"tp": 1, "fp": 3, "fn": 0}, "all": {"tp": 1, "fp": 3, "fn": 0}}, "cubic-v2": {"strict": {"tp": 0, "fp": 2, "fn": 0}, "core": {"tp": 1, "fp": 2, "fn": 0}, "all": {"tp": 1, "fp": 2, "fn": 0}}, "devin": {"strict": {"tp": 0, "fp": 1, "fn": 0}, "core": {"tp": 0, "fp": 1, "fn": 1}, "all": {"tp": 0, "fp": 1, "fn": 1}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 0}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 0, "fp": 0, "fn": 1}}, "kg": {"strict": {"tp": 0, "fp": 2, "fn": 0}, "core": {"tp": 0, "fp": 2, "fn": 1}, "all": {"tp": 0, "fp": 2, "fn": 1}}, "kodus-v2": {"strict": {"tp": 0, "fp": 0, "fn": 0}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 0, "fp": 0, "fn": 1}}, "macroscope": {"strict": {"tp": 0, "fp": 0, "fn": 0}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 0, "fp": 0, "fn": 1}}, "qodo-v2": {"strict": {"tp": 0, "fp": 1, "fn": 0}, "core": {"tp": 0, "fp": 1, "fn": 1}, "all": {"tp": 0, "fp": 1, "fn": 1}}, "sourcery": {"strict": {"tp": 0, "fp": 3, "fn": 0}, "core": {"tp": 0, "fp": 3, "fn": 1}, "all": {"tp": 0, "fp": 3, "fn": 1}}, "shipwright": {"strict": {"tp": 0, "fp": 2, "fn": 0}, "core": {"tp": 0, "fp": 2, "fn": 1}, "all": {"tp": 0, "fp": 2, "fn": 1}}}}, {"url": "https://github.com/grafana/grafana/pull/79265", "language": "Go", "pr_size": "large", "domain": "authentication", "change_type": "feature", "complexity": "moderate", "difficulty": "subtle", "risk": "high", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 1, "fp": 0, "fn": 3}, "core": {"tp": 2, "fp": 0, "fn": 3}, "all": {"tp": 2, "fp": 0, "fn": 3}}, "codeant-v2": {"strict": {"tp": 1, "fp": 1, "fn": 3}, "core": {"tp": 1, "fp": 1, "fn": 4}, "all": {"tp": 1, "fp": 1, "fn": 4}}, "coderabbit": {"strict": {"tp": 2, "fp": 4, "fn": 2}, "core": {"tp": 3, "fp": 4, "fn": 2}, "all": {"tp": 3, "fp": 4, "fn": 2}}, "qodo-extended-v2": {"strict": {"tp": 2, "fp": 1, "fn": 2}, "core": {"tp": 2, "fp": 1, "fn": 3}, "all": {"tp": 2, "fp": 1, "fn": 3}}, "greptile-v4-1": {"strict": {"tp": 3, "fp": 2, "fn": 1}, "core": {"tp": 4, "fp": 2, "fn": 1}, "all": {"tp": 4, "fp": 2, "fn": 1}}, "gemini-v2": {"strict": {"tp": 1, "fp": 3, "fn": 3}, "core": {"tp": 1, "fp": 3, "fn": 4}, "all": {"tp": 1, "fp": 3, "fn": 4}}, "gitlab": {"strict": {"tp": 1, "fp": 1, "fn": 3}, "core": {"tp": 2, "fp": 1, "fn": 3}, "all": {"tp": 2, "fp": 1, "fn": 3}}, "copilot-v2": {"strict": {"tp": 2, "fp": 3, "fn": 2}, "core": {"tp": 2, "fp": 3, "fn": 3}, "all": {"tp": 2, "fp": 3, "fn": 3}}, "augment": {"strict": {"tp": 1, "fp": 0, "fn": 3}, "core": {"tp": 2, "fp": 0, "fn": 3}, "all": {"tp": 2, "fp": 0, "fn": 3}}, "bugbot": {"strict": {"tp": 1, "fp": 1, "fn": 3}, "core": {"tp": 1, "fp": 1, "fn": 4}, "all": {"tp": 1, "fp": 1, "fn": 4}}, "claude": {"strict": {"tp": 1, "fp": 2, "fn": 3}, "core": {"tp": 1, "fp": 2, "fn": 4}, "all": {"tp": 1, "fp": 2, "fn": 4}}, "claude-code": {"strict": {"tp": 2, "fp": 1, "fn": 2}, "core": {"tp": 2, "fp": 1, "fn": 3}, "all": {"tp": 2, "fp": 1, "fn": 3}}, "cubic-v2": {"strict": {"tp": 3, "fp": 1, "fn": 1}, "core": {"tp": 4, "fp": 1, "fn": 1}, "all": {"tp": 4, "fp": 1, "fn": 1}}, "devin": {"strict": {"tp": 2, "fp": 0, "fn": 2}, "core": {"tp": 2, "fp": 0, "fn": 3}, "all": {"tp": 2, "fp": 0, "fn": 3}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 4}, "core": {"tp": 0, "fp": 0, "fn": 5}, "all": {"tp": 0, "fp": 0, "fn": 5}}, "kg": {"strict": {"tp": 1, "fp": 0, "fn": 3}, "core": {"tp": 1, "fp": 0, "fn": 4}, "all": {"tp": 1, "fp": 0, "fn": 4}}, "kodus-v2": {"strict": {"tp": 0, "fp": 1, "fn": 4}, "core": {"tp": 1, "fp": 1, "fn": 4}, "all": {"tp": 1, "fp": 1, "fn": 4}}, "macroscope": {"strict": {"tp": 1, "fp": 0, "fn": 3}, "core": {"tp": 1, "fp": 0, "fn": 4}, "all": {"tp": 1, "fp": 0, "fn": 4}}, "qodo-v2": {"strict": {"tp": 2, "fp": 1, "fn": 2}, "core": {"tp": 3, "fp": 1, "fn": 2}, "all": {"tp": 3, "fp": 1, "fn": 2}}, "sourcery": {"strict": {"tp": 2, "fp": 3, "fn": 2}, "core": {"tp": 3, "fp": 3, "fn": 2}, "all": {"tp": 3, "fp": 3, "fn": 2}}, "shipwright": {"strict": {"tp": 1, "fp": 1, "fn": 3}, "core": {"tp": 1, "fp": 1, "fn": 4}, "all": {"tp": 1, "fp": 1, "fn": 4}}}}, {"url": "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/9", "language": "Ruby", "pr_size": "small", "domain": "configuration", "change_type": "feature", "complexity": "moderate", "difficulty": "subtle", "risk": "medium", "context": "file", "concern": "reliability", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "codeant-v2": {"strict": {"tp": 0, "fp": 1, "fn": 1}, "core": {"tp": 0, "fp": 1, "fn": 2}, "all": {"tp": 0, "fp": 1, "fn": 2}}, "coderabbit": {"strict": {"tp": 0, "fp": 4, "fn": 1}, "core": {"tp": 0, "fp": 4, "fn": 2}, "all": {"tp": 0, "fp": 4, "fn": 2}}, "qodo-extended-v2": {"strict": {"tp": 0, "fp": 1, "fn": 1}, "core": {"tp": 0, "fp": 1, "fn": 2}, "all": {"tp": 0, "fp": 1, "fn": 2}}, "greptile-v4-1": {"strict": {"tp": 0, "fp": 2, "fn": 1}, "core": {"tp": 0, "fp": 2, "fn": 2}, "all": {"tp": 0, "fp": 2, "fn": 2}}, "gemini-v2": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "gitlab": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "copilot-v2": {"strict": {"tp": 0, "fp": 2, "fn": 1}, "core": {"tp": 0, "fp": 2, "fn": 2}, "all": {"tp": 0, "fp": 2, "fn": 2}}, "augment": {"strict": {"tp": 0, "fp": 2, "fn": 1}, "core": {"tp": 1, "fp": 2, "fn": 1}, "all": {"tp": 1, "fp": 2, "fn": 1}}, "bugbot": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "claude": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "claude-code": {"strict": {"tp": 1, "fp": 4, "fn": 0}, "core": {"tp": 1, "fp": 4, "fn": 1}, "all": {"tp": 1, "fp": 4, "fn": 1}}, "cubic-v2": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "devin": {"strict": {"tp": 0, "fp": 1, "fn": 1}, "core": {"tp": 0, "fp": 1, "fn": 2}, "all": {"tp": 0, "fp": 1, "fn": 2}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "kodus-v2": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "macroscope": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "qodo-v2": {"strict": {"tp": 0, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "sourcery": {"strict": {"tp": 0, "fp": 2, "fn": 1}, "core": {"tp": 0, "fp": 2, "fn": 2}, "all": {"tp": 0, "fp": 2, "fn": 2}}, "shipwright": {"strict": {"tp": 0, "fp": 1, "fn": 1}, "core": {"tp": 0, "fp": 1, "fn": 2}, "all": {"tp": 0, "fp": 1, "fn": 2}}}}, {"url": "https://github.com/grafana/grafana/pull/76186", "language": "Go", "pr_size": "small", "domain": "logging", "change_type": "refactoring", "complexity": "moderate", "difficulty": "subtle", "risk": "high", "context": "cross_file", "concern": "reliability", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 0, "fp": 0, "fn": 3}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 3}}, "codeant-v2": {"strict": {"tp": 0, "fp": 1, "fn": 3}, "core": {"tp": 0, "fp": 1, "fn": 3}, "all": {"tp": 0, "fp": 1, "fn": 3}}, "coderabbit": {"strict": {"tp": 1, "fp": 0, "fn": 2}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 2}}, "qodo-extended-v2": {"strict": {"tp": 1, "fp": 0, "fn": 2}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 2}}, "greptile-v4-1": {"strict": {"tp": 1, "fp": 0, "fn": 2}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 2}}, "gemini-v2": {"strict": {"tp": 1, "fp": 2, "fn": 2}, "core": {"tp": 1, "fp": 2, "fn": 2}, "all": {"tp": 1, "fp": 2, "fn": 2}}, "gitlab": {"strict": {"tp": 3, "fp": 0, "fn": 0}, "core": {"tp": 3, "fp": 0, "fn": 0}, "all": {"tp": 3, "fp": 0, "fn": 0}}, "copilot-v2": {"strict": {"tp": 0, "fp": 2, "fn": 3}, "core": {"tp": 0, "fp": 2, "fn": 3}, "all": {"tp": 0, "fp": 2, "fn": 3}}, "augment": {"strict": {"tp": 2, "fp": 0, "fn": 1}, "core": {"tp": 2, "fp": 0, "fn": 1}, "all": {"tp": 2, "fp": 0, "fn": 1}}, "bugbot": {"strict": {"tp": 1, "fp": 0, "fn": 2}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 2}}, "claude": {"strict": {"tp": 0, "fp": 0, "fn": 3}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 3}}, "claude-code": {"strict": {"tp": 1, "fp": 1, "fn": 2}, "core": {"tp": 1, "fp": 1, "fn": 2}, "all": {"tp": 1, "fp": 1, "fn": 2}}, "cubic-v2": {"strict": {"tp": 3, "fp": 0, "fn": 0}, "core": {"tp": 3, "fp": 0, "fn": 0}, "all": {"tp": 3, "fp": 0, "fn": 0}}, "devin": {"strict": {"tp": 1, "fp": 0, "fn": 2}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 2}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 3}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 3}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 3}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 3}}, "kodus-v2": {"strict": {"tp": 2, "fp": 0, "fn": 1}, "core": {"tp": 2, "fp": 0, "fn": 1}, "all": {"tp": 2, "fp": 0, "fn": 1}}, "macroscope": {"strict": {"tp": 1, "fp": 0, "fn": 2}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 2}}, "qodo-v2": {"strict": {"tp": 2, "fp": 0, "fn": 1}, "core": {"tp": 2, "fp": 0, "fn": 1}, "all": {"tp": 2, "fp": 0, "fn": 1}}, "sourcery": {"strict": {"tp": 0, "fp": 3, "fn": 3}, "core": {"tp": 0, "fp": 3, "fn": 3}, "all": {"tp": 0, "fp": 3, "fn": 3}}, "shipwright": {"strict": {"tp": 1, "fp": 1, "fn": 2}, "core": {"tp": 1, "fp": 1, "fn": 2}, "all": {"tp": 1, "fp": 1, "fn": 2}}}}, {"url": "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/10", "language": "Ruby", "pr_size": "large", "domain": "configuration", "change_type": "feature", "complexity": "moderate", "difficulty": "subtle", "risk": "high", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 2, "fp": 0, "fn": 5}, "core": {"tp": 2, "fp": 0, "fn": 5}, "all": {"tp": 2, "fp": 0, "fn": 5}}, "codeant-v2": {"strict": {"tp": 4, "fp": 5, "fn": 3}, "core": {"tp": 4, "fp": 5, "fn": 3}, "all": {"tp": 4, "fp": 5, "fn": 3}}, "coderabbit": {"strict": {"tp": 3, "fp": 8, "fn": 4}, "core": {"tp": 3, "fp": 8, "fn": 4}, "all": {"tp": 3, "fp": 8, "fn": 4}}, "qodo-extended-v2": {"strict": {"tp": 5, "fp": 4, "fn": 2}, "core": {"tp": 5, "fp": 4, "fn": 2}, "all": {"tp": 5, "fp": 4, "fn": 2}}, "greptile-v4-1": {"strict": {"tp": 2, "fp": 2, "fn": 5}, "core": {"tp": 2, "fp": 2, "fn": 5}, "all": {"tp": 2, "fp": 2, "fn": 5}}, "gemini-v2": {"strict": {"tp": 4, "fp": 2, "fn": 3}, "core": {"tp": 4, "fp": 2, "fn": 3}, "all": {"tp": 4, "fp": 2, "fn": 3}}, "gitlab": {"strict": {"tp": 2, "fp": 3, "fn": 5}, "core": {"tp": 2, "fp": 3, "fn": 5}, "all": {"tp": 2, "fp": 3, "fn": 5}}, "copilot-v2": {"strict": {"tp": 6, "fp": 5, "fn": 1}, "core": {"tp": 6, "fp": 5, "fn": 1}, "all": {"tp": 6, "fp": 5, "fn": 1}}, "augment": {"strict": {"tp": 3, "fp": 1, "fn": 4}, "core": {"tp": 3, "fp": 1, "fn": 4}, "all": {"tp": 3, "fp": 1, "fn": 4}}, "bugbot": {"strict": {"tp": 5, "fp": 2, "fn": 2}, "core": {"tp": 5, "fp": 2, "fn": 2}, "all": {"tp": 5, "fp": 2, "fn": 2}}, "claude": {"strict": {"tp": 1, "fp": 2, "fn": 6}, "core": {"tp": 1, "fp": 2, "fn": 6}, "all": {"tp": 1, "fp": 2, "fn": 6}}, "claude-code": {"strict": {"tp": 2, "fp": 4, "fn": 5}, "core": {"tp": 2, "fp": 4, "fn": 5}, "all": {"tp": 2, "fp": 4, "fn": 5}}, "cubic-v2": {"strict": {"tp": 3, "fp": 1, "fn": 4}, "core": {"tp": 3, "fp": 1, "fn": 4}, "all": {"tp": 3, "fp": 1, "fn": 4}}, "devin": {"strict": {"tp": 1, "fp": 1, "fn": 6}, "core": {"tp": 1, "fp": 1, "fn": 6}, "all": {"tp": 1, "fp": 1, "fn": 6}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 7}, "core": {"tp": 0, "fp": 0, "fn": 7}, "all": {"tp": 0, "fp": 0, "fn": 7}}, "kg": {"strict": {"tp": 1, "fp": 1, "fn": 6}, "core": {"tp": 1, "fp": 1, "fn": 6}, "all": {"tp": 1, "fp": 1, "fn": 6}}, "kodus-v2": {"strict": {"tp": 5, "fp": 4, "fn": 2}, "core": {"tp": 5, "fp": 4, "fn": 2}, "all": {"tp": 5, "fp": 4, "fn": 2}}, "macroscope": {"strict": {"tp": 5, "fp": 2, "fn": 2}, "core": {"tp": 5, "fp": 2, "fn": 2}, "all": {"tp": 5, "fp": 2, "fn": 2}}, "qodo-v2": {"strict": {"tp": 3, "fp": 4, "fn": 4}, "core": {"tp": 3, "fp": 4, "fn": 4}, "all": {"tp": 3, "fp": 4, "fn": 4}}, "sourcery": {"strict": {"tp": 5, "fp": 7, "fn": 2}, "core": {"tp": 5, "fp": 7, "fn": 2}, "all": {"tp": 5, "fp": 7, "fn": 2}}, "shipwright": {"strict": {"tp": 5, "fp": 1, "fn": 2}, "core": {"tp": 5, "fp": 1, "fn": 2}, "all": {"tp": 5, "fp": 1, "fn": 2}}}}, {"url": "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/7", "language": "Ruby", "pr_size": "medium", "domain": "UI", "change_type": "bug_fix", "complexity": "simple", "difficulty": "subtle", "risk": "medium", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 3, "fp": 1, "fn": 0}, "core": {"tp": 3, "fp": 1, "fn": 0}, "all": {"tp": 5, "fp": 1, "fn": 0}}, "codeant-v2": {"strict": {"tp": 0, "fp": 8, "fn": 3}, "core": {"tp": 0, "fp": 8, "fn": 3}, "all": {"tp": 1, "fp": 8, "fn": 4}}, "coderabbit": {"strict": {"tp": 1, "fp": 1, "fn": 2}, "core": {"tp": 1, "fp": 1, "fn": 2}, "all": {"tp": 1, "fp": 1, "fn": 4}}, "qodo-extended-v2": {"strict": {"tp": 3, "fp": 2, "fn": 0}, "core": {"tp": 3, "fp": 2, "fn": 0}, "all": {"tp": 5, "fp": 2, "fn": 0}}, "greptile-v4-1": {"strict": {"tp": 2, "fp": 0, "fn": 1}, "core": {"tp": 2, "fp": 0, "fn": 1}, "all": {"tp": 4, "fp": 0, "fn": 1}}, "gemini-v2": {"strict": {"tp": 3, "fp": 1, "fn": 0}, "core": {"tp": 3, "fp": 1, "fn": 0}, "all": {"tp": 5, "fp": 1, "fn": 0}}, "gitlab": {"strict": {"tp": 3, "fp": 0, "fn": 0}, "core": {"tp": 3, "fp": 0, "fn": 0}, "all": {"tp": 5, "fp": 0, "fn": 0}}, "copilot-v2": {"strict": {"tp": 3, "fp": 1, "fn": 0}, "core": {"tp": 3, "fp": 1, "fn": 0}, "all": {"tp": 5, "fp": 1, "fn": 0}}, "augment": {"strict": {"tp": 3, "fp": 2, "fn": 0}, "core": {"tp": 3, "fp": 2, "fn": 0}, "all": {"tp": 5, "fp": 2, "fn": 0}}, "bugbot": {"strict": {"tp": 3, "fp": 1, "fn": 0}, "core": {"tp": 3, "fp": 1, "fn": 0}, "all": {"tp": 5, "fp": 1, "fn": 0}}, "claude": {"strict": {"tp": 3, "fp": 4, "fn": 0}, "core": {"tp": 3, "fp": 4, "fn": 0}, "all": {"tp": 5, "fp": 4, "fn": 0}}, "claude-code": {"strict": {"tp": 3, "fp": 0, "fn": 0}, "core": {"tp": 3, "fp": 0, "fn": 0}, "all": {"tp": 5, "fp": 0, "fn": 0}}, "cubic-v2": {"strict": {"tp": 3, "fp": 0, "fn": 0}, "core": {"tp": 3, "fp": 0, "fn": 0}, "all": {"tp": 5, "fp": 0, "fn": 0}}, "devin": {"strict": {"tp": 3, "fp": 0, "fn": 0}, "core": {"tp": 3, "fp": 0, "fn": 0}, "all": {"tp": 5, "fp": 0, "fn": 0}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 3}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 5}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 3}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 5}}, "kodus-v2": {"strict": {"tp": 3, "fp": 0, "fn": 0}, "core": {"tp": 3, "fp": 0, "fn": 0}, "all": {"tp": 3, "fp": 0, "fn": 2}}, "macroscope": {"strict": {"tp": 3, "fp": 0, "fn": 0}, "core": {"tp": 3, "fp": 0, "fn": 0}, "all": {"tp": 5, "fp": 0, "fn": 0}}, "qodo-v2": {"strict": {"tp": 3, "fp": 2, "fn": 0}, "core": {"tp": 3, "fp": 2, "fn": 0}, "all": {"tp": 5, "fp": 2, "fn": 0}}, "sourcery": {"strict": {"tp": 3, "fp": 1, "fn": 0}, "core": {"tp": 3, "fp": 1, "fn": 0}, "all": {"tp": 5, "fp": 1, "fn": 0}}, "shipwright": {"strict": {"tp": 3, "fp": 2, "fn": 0}, "core": {"tp": 3, "fp": 2, "fn": 0}, "all": {"tp": 5, "fp": 2, "fn": 0}}}}, {"url": "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/8", "language": "Ruby", "pr_size": "medium", "domain": "API", "change_type": "bug_fix", "complexity": "moderate", "difficulty": "subtle", "risk": "medium", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 2, "fp": 2, "fn": 4}, "core": {"tp": 2, "fp": 2, "fn": 4}, "all": {"tp": 2, "fp": 2, "fn": 4}}, "codeant-v2": {"strict": {"tp": 5, "fp": 5, "fn": 1}, "core": {"tp": 5, "fp": 5, "fn": 1}, "all": {"tp": 5, "fp": 5, "fn": 1}}, "coderabbit": {"strict": {"tp": 4, "fp": 6, "fn": 2}, "core": {"tp": 4, "fp": 6, "fn": 2}, "all": {"tp": 4, "fp": 6, "fn": 2}}, "qodo-extended-v2": {"strict": {"tp": 3, "fp": 1, "fn": 3}, "core": {"tp": 3, "fp": 1, "fn": 3}, "all": {"tp": 3, "fp": 1, "fn": 3}}, "greptile-v4-1": {"strict": {"tp": 4, "fp": 1, "fn": 2}, "core": {"tp": 4, "fp": 1, "fn": 2}, "all": {"tp": 4, "fp": 1, "fn": 2}}, "gemini-v2": {"strict": {"tp": 2, "fp": 2, "fn": 4}, "core": {"tp": 2, "fp": 2, "fn": 4}, "all": {"tp": 2, "fp": 2, "fn": 4}}, "gitlab": {"strict": {"tp": 4, "fp": 1, "fn": 2}, "core": {"tp": 4, "fp": 1, "fn": 2}, "all": {"tp": 4, "fp": 1, "fn": 2}}, "copilot-v2": {"strict": {"tp": 3, "fp": 9, "fn": 3}, "core": {"tp": 3, "fp": 9, "fn": 3}, "all": {"tp": 3, "fp": 9, "fn": 3}}, "augment": {"strict": {"tp": 3, "fp": 2, "fn": 3}, "core": {"tp": 3, "fp": 2, "fn": 3}, "all": {"tp": 3, "fp": 2, "fn": 3}}, "bugbot": {"strict": {"tp": 2, "fp": 1, "fn": 4}, "core": {"tp": 2, "fp": 1, "fn": 4}, "all": {"tp": 2, "fp": 1, "fn": 4}}, "claude": {"strict": {"tp": 4, "fp": 3, "fn": 2}, "core": {"tp": 4, "fp": 3, "fn": 2}, "all": {"tp": 4, "fp": 3, "fn": 2}}, "claude-code": {"strict": {"tp": 2, "fp": 1, "fn": 4}, "core": {"tp": 2, "fp": 1, "fn": 4}, "all": {"tp": 2, "fp": 1, "fn": 4}}, "cubic-v2": {"strict": {"tp": 3, "fp": 1, "fn": 3}, "core": {"tp": 3, "fp": 1, "fn": 3}, "all": {"tp": 3, "fp": 1, "fn": 3}}, "devin": {"strict": {"tp": 2, "fp": 0, "fn": 4}, "core": {"tp": 2, "fp": 0, "fn": 4}, "all": {"tp": 2, "fp": 0, "fn": 4}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 6}, "core": {"tp": 0, "fp": 0, "fn": 6}, "all": {"tp": 0, "fp": 0, "fn": 6}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 6}, "core": {"tp": 0, "fp": 0, "fn": 6}, "all": {"tp": 0, "fp": 0, "fn": 6}}, "kodus-v2": {"strict": {"tp": 1, "fp": 6, "fn": 5}, "core": {"tp": 1, "fp": 6, "fn": 5}, "all": {"tp": 1, "fp": 6, "fn": 5}}, "macroscope": {"strict": {"tp": 2, "fp": 2, "fn": 4}, "core": {"tp": 2, "fp": 2, "fn": 4}, "all": {"tp": 2, "fp": 2, "fn": 4}}, "qodo-v2": {"strict": {"tp": 2, "fp": 1, "fn": 4}, "core": {"tp": 2, "fp": 1, "fn": 4}, "all": {"tp": 2, "fp": 1, "fn": 4}}, "sourcery": {"strict": {"tp": 4, "fp": 1, "fn": 2}, "core": {"tp": 4, "fp": 1, "fn": 2}, "all": {"tp": 4, "fp": 1, "fn": 2}}, "shipwright": {"strict": {"tp": 3, "fp": 1, "fn": 3}, "core": {"tp": 3, "fp": 1, "fn": 3}, "all": {"tp": 3, "fp": 1, "fn": 3}}}}, {"url": "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/3", "language": "Ruby", "pr_size": "medium", "domain": "authentication", "change_type": "feature", "complexity": "moderate", "difficulty": "subtle", "risk": "medium", "context": "file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 1, "fp": 0, "fn": 2}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 2}}, "codeant-v2": {"strict": {"tp": 2, "fp": 2, "fn": 1}, "core": {"tp": 2, "fp": 2, "fn": 1}, "all": {"tp": 2, "fp": 2, "fn": 1}}, "coderabbit": {"strict": {"tp": 2, "fp": 5, "fn": 1}, "core": {"tp": 2, "fp": 5, "fn": 1}, "all": {"tp": 2, "fp": 5, "fn": 1}}, "qodo-extended-v2": {"strict": {"tp": 2, "fp": 0, "fn": 1}, "core": {"tp": 2, "fp": 0, "fn": 1}, "all": {"tp": 2, "fp": 0, "fn": 1}}, "greptile-v4-1": {"strict": {"tp": 2, "fp": 2, "fn": 1}, "core": {"tp": 2, "fp": 2, "fn": 1}, "all": {"tp": 2, "fp": 2, "fn": 1}}, "gemini-v2": {"strict": {"tp": 2, "fp": 3, "fn": 1}, "core": {"tp": 2, "fp": 3, "fn": 1}, "all": {"tp": 2, "fp": 3, "fn": 1}}, "gitlab": {"strict": {"tp": 0, "fp": 2, "fn": 3}, "core": {"tp": 0, "fp": 2, "fn": 3}, "all": {"tp": 0, "fp": 2, "fn": 3}}, "copilot-v2": {"strict": {"tp": 2, "fp": 6, "fn": 1}, "core": {"tp": 2, "fp": 6, "fn": 1}, "all": {"tp": 2, "fp": 6, "fn": 1}}, "augment": {"strict": {"tp": 1, "fp": 4, "fn": 2}, "core": {"tp": 1, "fp": 4, "fn": 2}, "all": {"tp": 1, "fp": 4, "fn": 2}}, "bugbot": {"strict": {"tp": 1, "fp": 1, "fn": 2}, "core": {"tp": 1, "fp": 1, "fn": 2}, "all": {"tp": 1, "fp": 1, "fn": 2}}, "claude": {"strict": {"tp": 1, "fp": 3, "fn": 2}, "core": {"tp": 1, "fp": 3, "fn": 2}, "all": {"tp": 1, "fp": 3, "fn": 2}}, "claude-code": {"strict": {"tp": 1, "fp": 4, "fn": 2}, "core": {"tp": 1, "fp": 4, "fn": 2}, "all": {"tp": 1, "fp": 4, "fn": 2}}, "cubic-v2": {"strict": {"tp": 2, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 1}, "all": {"tp": 2, "fp": 1, "fn": 1}}, "devin": {"strict": {"tp": 1, "fp": 1, "fn": 2}, "core": {"tp": 1, "fp": 1, "fn": 2}, "all": {"tp": 1, "fp": 1, "fn": 2}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 3}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 3}}, "kg": {"strict": {"tp": 1, "fp": 0, "fn": 2}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 2}}, "kodus-v2": {"strict": {"tp": 1, "fp": 0, "fn": 2}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 2}}, "macroscope": {"strict": {"tp": 2, "fp": 0, "fn": 1}, "core": {"tp": 2, "fp": 0, "fn": 1}, "all": {"tp": 2, "fp": 0, "fn": 1}}, "qodo-v2": {"strict": {"tp": 2, "fp": 2, "fn": 1}, "core": {"tp": 2, "fp": 2, "fn": 1}, "all": {"tp": 2, "fp": 2, "fn": 1}}, "sourcery": {"strict": {"tp": 1, "fp": 3, "fn": 2}, "core": {"tp": 1, "fp": 3, "fn": 2}, "all": {"tp": 1, "fp": 3, "fn": 2}}, "shipwright": {"strict": {"tp": 1, "fp": 1, "fn": 2}, "core": {"tp": 1, "fp": 1, "fn": 2}, "all": {"tp": 1, "fp": 1, "fn": 2}}}}, {"url": "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/5", "language": "Ruby", "pr_size": "small", "domain": "UI", "change_type": "performance", "complexity": "moderate", "difficulty": "subtle", "risk": "medium", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 1, "fp": 1, "fn": 2}, "core": {"tp": 1, "fp": 1, "fn": 2}, "all": {"tp": 1, "fp": 1, "fn": 2}}, "codeant-v2": {"strict": {"tp": 0, "fp": 0, "fn": 3}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 3}}, "coderabbit": {"strict": {"tp": 1, "fp": 0, "fn": 2}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 2}}, "qodo-extended-v2": {"strict": {"tp": 2, "fp": 0, "fn": 1}, "core": {"tp": 2, "fp": 0, "fn": 1}, "all": {"tp": 2, "fp": 0, "fn": 1}}, "greptile-v4-1": {"strict": {"tp": 1, "fp": 2, "fn": 2}, "core": {"tp": 1, "fp": 2, "fn": 2}, "all": {"tp": 1, "fp": 2, "fn": 2}}, "gemini-v2": {"strict": {"tp": 2, "fp": 7, "fn": 1}, "core": {"tp": 2, "fp": 7, "fn": 1}, "all": {"tp": 2, "fp": 7, "fn": 1}}, "gitlab": {"strict": {"tp": 1, "fp": 1, "fn": 2}, "core": {"tp": 1, "fp": 1, "fn": 2}, "all": {"tp": 1, "fp": 1, "fn": 2}}, "copilot-v2": {"strict": {"tp": 2, "fp": 2, "fn": 1}, "core": {"tp": 2, "fp": 2, "fn": 1}, "all": {"tp": 2, "fp": 2, "fn": 1}}, "augment": {"strict": {"tp": 2, "fp": 0, "fn": 1}, "core": {"tp": 2, "fp": 0, "fn": 1}, "all": {"tp": 2, "fp": 0, "fn": 1}}, "bugbot": {"strict": {"tp": 1, "fp": 0, "fn": 2}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 2}}, "claude": {"strict": {"tp": 1, "fp": 0, "fn": 2}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 2}}, "claude-code": {"strict": {"tp": 2, "fp": 3, "fn": 1}, "core": {"tp": 2, "fp": 3, "fn": 1}, "all": {"tp": 2, "fp": 3, "fn": 1}}, "cubic-v2": {"strict": {"tp": 2, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 1}, "all": {"tp": 2, "fp": 1, "fn": 1}}, "devin": {"strict": {"tp": 1, "fp": 1, "fn": 2}, "core": {"tp": 1, "fp": 1, "fn": 2}, "all": {"tp": 1, "fp": 1, "fn": 2}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 3}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 3}}, "kg": {"strict": {"tp": 1, "fp": 0, "fn": 2}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 2}}, "kodus-v2": {"strict": {"tp": 0, "fp": 0, "fn": 3}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 3}}, "macroscope": {"strict": {"tp": 0, "fp": 0, "fn": 3}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 3}}, "qodo-v2": {"strict": {"tp": 1, "fp": 1, "fn": 2}, "core": {"tp": 1, "fp": 1, "fn": 2}, "all": {"tp": 1, "fp": 1, "fn": 2}}, "sourcery": {"strict": {"tp": 2, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 1}, "all": {"tp": 2, "fp": 1, "fn": 1}}, "shipwright": {"strict": {"tp": 1, "fp": 0, "fn": 2}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 2}}}}, {"url": "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/6", "language": "Ruby", "pr_size": "small", "domain": "serialization", "change_type": "feature", "complexity": "simple", "difficulty": "moderate", "risk": "medium", "context": "file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 1, "fp": 1, "fn": 0}, "all": {"tp": 1, "fp": 1, "fn": 0}}, "codeant-v2": {"strict": {"tp": 1, "fp": 3, "fn": 0}, "core": {"tp": 1, "fp": 3, "fn": 0}, "all": {"tp": 1, "fp": 3, "fn": 0}}, "coderabbit": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 0}, "all": {"tp": 1, "fp": 0, "fn": 0}}, "qodo-extended-v2": {"strict": {"tp": 0, "fp": 2, "fn": 1}, "core": {"tp": 0, "fp": 2, "fn": 1}, "all": {"tp": 0, "fp": 2, "fn": 1}}, "greptile-v4-1": {"strict": {"tp": 1, "fp": 3, "fn": 0}, "core": {"tp": 1, "fp": 3, "fn": 0}, "all": {"tp": 1, "fp": 3, "fn": 0}}, "gemini-v2": {"strict": {"tp": 1, "fp": 5, "fn": 0}, "core": {"tp": 1, "fp": 5, "fn": 0}, "all": {"tp": 1, "fp": 5, "fn": 0}}, "gitlab": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 1, "fp": 1, "fn": 0}, "all": {"tp": 1, "fp": 1, "fn": 0}}, "copilot-v2": {"strict": {"tp": 1, "fp": 3, "fn": 0}, "core": {"tp": 1, "fp": 3, "fn": 0}, "all": {"tp": 1, "fp": 3, "fn": 0}}, "augment": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 1, "fp": 1, "fn": 0}, "all": {"tp": 1, "fp": 1, "fn": 0}}, "bugbot": {"strict": {"tp": 1, "fp": 2, "fn": 0}, "core": {"tp": 1, "fp": 2, "fn": 0}, "all": {"tp": 1, "fp": 2, "fn": 0}}, "claude": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 0, "fp": 0, "fn": 1}}, "claude-code": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 1, "fp": 1, "fn": 0}, "all": {"tp": 1, "fp": 1, "fn": 0}}, "cubic-v2": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 1, "fp": 1, "fn": 0}, "all": {"tp": 1, "fp": 1, "fn": 0}}, "devin": {"strict": {"tp": 1, "fp": 2, "fn": 0}, "core": {"tp": 1, "fp": 2, "fn": 0}, "all": {"tp": 1, "fp": 2, "fn": 0}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 0, "fp": 0, "fn": 1}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 0, "fp": 0, "fn": 1}}, "kodus-v2": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 1, "fp": 1, "fn": 0}, "all": {"tp": 1, "fp": 1, "fn": 0}}, "macroscope": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 0}, "all": {"tp": 1, "fp": 0, "fn": 0}}, "qodo-v2": {"strict": {"tp": 1, "fp": 2, "fn": 0}, "core": {"tp": 1, "fp": 2, "fn": 0}, "all": {"tp": 1, "fp": 2, "fn": 0}}, "sourcery": {"strict": {"tp": 1, "fp": 4, "fn": 0}, "core": {"tp": 1, "fp": 4, "fn": 0}, "all": {"tp": 1, "fp": 4, "fn": 0}}, "shipwright": {"strict": {"tp": 1, "fp": 2, "fn": 0}, "core": {"tp": 1, "fp": 2, "fn": 0}, "all": {"tp": 1, "fp": 2, "fn": 0}}}}, {"url": "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/4", "language": "Ruby", "pr_size": "large", "domain": "API", "change_type": "feature", "complexity": "complex", "difficulty": "subtle", "risk": "critical", "context": "cross_file", "concern": "security", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 2, "fp": 2, "fn": 6}, "core": {"tp": 2, "fp": 2, "fn": 6}, "all": {"tp": 2, "fp": 2, "fn": 6}}, "codeant-v2": {"strict": {"tp": 3, "fp": 16, "fn": 5}, "core": {"tp": 3, "fp": 16, "fn": 5}, "all": {"tp": 3, "fp": 16, "fn": 5}}, "coderabbit": {"strict": {"tp": 6, "fp": 19, "fn": 2}, "core": {"tp": 6, "fp": 19, "fn": 2}, "all": {"tp": 6, "fp": 19, "fn": 2}}, "qodo-extended-v2": {"strict": {"tp": 4, "fp": 1, "fn": 4}, "core": {"tp": 4, "fp": 1, "fn": 4}, "all": {"tp": 4, "fp": 1, "fn": 4}}, "greptile-v4-1": {"strict": {"tp": 4, "fp": 1, "fn": 4}, "core": {"tp": 4, "fp": 1, "fn": 4}, "all": {"tp": 4, "fp": 1, "fn": 4}}, "gemini-v2": {"strict": {"tp": 5, "fp": 4, "fn": 3}, "core": {"tp": 5, "fp": 4, "fn": 3}, "all": {"tp": 5, "fp": 4, "fn": 3}}, "gitlab": {"strict": {"tp": 5, "fp": 0, "fn": 3}, "core": {"tp": 5, "fp": 0, "fn": 3}, "all": {"tp": 5, "fp": 0, "fn": 3}}, "copilot-v2": {"strict": {"tp": 6, "fp": 2, "fn": 2}, "core": {"tp": 6, "fp": 2, "fn": 2}, "all": {"tp": 6, "fp": 2, "fn": 2}}, "augment": {"strict": {"tp": 5, "fp": 0, "fn": 3}, "core": {"tp": 5, "fp": 0, "fn": 3}, "all": {"tp": 5, "fp": 0, "fn": 3}}, "bugbot": {"strict": {"tp": 2, "fp": 3, "fn": 6}, "core": {"tp": 2, "fp": 3, "fn": 6}, "all": {"tp": 2, "fp": 3, "fn": 6}}, "claude": {"strict": {"tp": 4, "fp": 3, "fn": 4}, "core": {"tp": 4, "fp": 3, "fn": 4}, "all": {"tp": 4, "fp": 3, "fn": 4}}, "claude-code": {"strict": {"tp": 5, "fp": 0, "fn": 3}, "core": {"tp": 5, "fp": 0, "fn": 3}, "all": {"tp": 5, "fp": 0, "fn": 3}}, "cubic-v2": {"strict": {"tp": 3, "fp": 1, "fn": 5}, "core": {"tp": 3, "fp": 1, "fn": 5}, "all": {"tp": 3, "fp": 1, "fn": 5}}, "devin": {"strict": {"tp": 3, "fp": 1, "fn": 5}, "core": {"tp": 3, "fp": 1, "fn": 5}, "all": {"tp": 3, "fp": 1, "fn": 5}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 8}, "core": {"tp": 0, "fp": 0, "fn": 8}, "all": {"tp": 0, "fp": 0, "fn": 8}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 8}, "core": {"tp": 0, "fp": 0, "fn": 8}, "all": {"tp": 0, "fp": 0, "fn": 8}}, "kodus-v2": {"strict": {"tp": 4, "fp": 1, "fn": 4}, "core": {"tp": 4, "fp": 1, "fn": 4}, "all": {"tp": 4, "fp": 1, "fn": 4}}, "macroscope": {"strict": {"tp": 2, "fp": 4, "fn": 6}, "core": {"tp": 2, "fp": 4, "fn": 6}, "all": {"tp": 2, "fp": 4, "fn": 6}}, "qodo-v2": {"strict": {"tp": 6, "fp": 5, "fn": 2}, "core": {"tp": 6, "fp": 5, "fn": 2}, "all": {"tp": 6, "fp": 5, "fn": 2}}, "sourcery": {"strict": {"tp": 3, "fp": 2, "fn": 5}, "core": {"tp": 3, "fp": 2, "fn": 5}, "all": {"tp": 3, "fp": 2, "fn": 5}}, "shipwright": {"strict": {"tp": 1, "fp": 4, "fn": 7}, "core": {"tp": 1, "fp": 4, "fn": 7}, "all": {"tp": 1, "fp": 4, "fn": 7}}}}, {"url": "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/1", "language": "Ruby", "pr_size": "medium", "domain": "file_io", "change_type": "feature", "complexity": "moderate", "difficulty": "subtle", "risk": "medium", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 2, "fp": 0, "fn": 2}, "core": {"tp": 2, "fp": 0, "fn": 2}, "all": {"tp": 2, "fp": 0, "fn": 2}}, "codeant-v2": {"strict": {"tp": 2, "fp": 1, "fn": 2}, "core": {"tp": 2, "fp": 1, "fn": 2}, "all": {"tp": 2, "fp": 1, "fn": 2}}, "coderabbit": {"strict": {"tp": 3, "fp": 3, "fn": 1}, "core": {"tp": 3, "fp": 3, "fn": 1}, "all": {"tp": 3, "fp": 3, "fn": 1}}, "qodo-extended-v2": {"strict": {"tp": 2, "fp": 1, "fn": 2}, "core": {"tp": 2, "fp": 1, "fn": 2}, "all": {"tp": 2, "fp": 1, "fn": 2}}, "greptile-v4-1": {"strict": {"tp": 3, "fp": 1, "fn": 1}, "core": {"tp": 3, "fp": 1, "fn": 1}, "all": {"tp": 3, "fp": 1, "fn": 1}}, "gemini-v2": {"strict": {"tp": 3, "fp": 0, "fn": 1}, "core": {"tp": 3, "fp": 0, "fn": 1}, "all": {"tp": 3, "fp": 0, "fn": 1}}, "gitlab": {"strict": {"tp": 2, "fp": 0, "fn": 2}, "core": {"tp": 2, "fp": 0, "fn": 2}, "all": {"tp": 2, "fp": 0, "fn": 2}}, "copilot-v2": {"strict": {"tp": 3, "fp": 1, "fn": 1}, "core": {"tp": 3, "fp": 1, "fn": 1}, "all": {"tp": 3, "fp": 1, "fn": 1}}, "augment": {"strict": {"tp": 2, "fp": 2, "fn": 2}, "core": {"tp": 2, "fp": 2, "fn": 2}, "all": {"tp": 2, "fp": 2, "fn": 2}}, "bugbot": {"strict": {"tp": 2, "fp": 1, "fn": 2}, "core": {"tp": 2, "fp": 1, "fn": 2}, "all": {"tp": 2, "fp": 1, "fn": 2}}, "claude": {"strict": {"tp": 2, "fp": 0, "fn": 2}, "core": {"tp": 2, "fp": 0, "fn": 2}, "all": {"tp": 2, "fp": 0, "fn": 2}}, "claude-code": {"strict": {"tp": 3, "fp": 2, "fn": 1}, "core": {"tp": 3, "fp": 2, "fn": 1}, "all": {"tp": 3, "fp": 2, "fn": 1}}, "cubic-v2": {"strict": {"tp": 3, "fp": 0, "fn": 1}, "core": {"tp": 3, "fp": 0, "fn": 1}, "all": {"tp": 3, "fp": 0, "fn": 1}}, "devin": {"strict": {"tp": 2, "fp": 0, "fn": 2}, "core": {"tp": 2, "fp": 0, "fn": 2}, "all": {"tp": 2, "fp": 0, "fn": 2}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 4}, "core": {"tp": 0, "fp": 0, "fn": 4}, "all": {"tp": 0, "fp": 0, "fn": 4}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 4}, "core": {"tp": 0, "fp": 0, "fn": 4}, "all": {"tp": 0, "fp": 0, "fn": 4}}, "kodus-v2": {"strict": {"tp": 3, "fp": 0, "fn": 1}, "core": {"tp": 3, "fp": 0, "fn": 1}, "all": {"tp": 3, "fp": 0, "fn": 1}}, "macroscope": {"strict": {"tp": 1, "fp": 0, "fn": 3}, "core": {"tp": 1, "fp": 0, "fn": 3}, "all": {"tp": 1, "fp": 0, "fn": 3}}, "qodo-v2": {"strict": {"tp": 2, "fp": 3, "fn": 2}, "core": {"tp": 2, "fp": 3, "fn": 2}, "all": {"tp": 2, "fp": 3, "fn": 2}}, "sourcery": {"strict": {"tp": 2, "fp": 1, "fn": 2}, "core": {"tp": 2, "fp": 1, "fn": 2}, "all": {"tp": 2, "fp": 1, "fn": 2}}, "shipwright": {"strict": {"tp": 2, "fp": 2, "fn": 2}, "core": {"tp": 2, "fp": 2, "fn": 2}, "all": {"tp": 2, "fp": 2, "fn": 2}}}}, {"url": "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/2", "language": "Ruby", "pr_size": "medium", "domain": "UI", "change_type": "feature", "complexity": "moderate", "difficulty": "moderate", "risk": "medium", "context": "file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 0}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "codeant-v2": {"strict": {"tp": 1, "fp": 5, "fn": 0}, "core": {"tp": 1, "fp": 5, "fn": 0}, "all": {"tp": 1, "fp": 5, "fn": 1}}, "coderabbit": {"strict": {"tp": 1, "fp": 5, "fn": 0}, "core": {"tp": 1, "fp": 5, "fn": 0}, "all": {"tp": 2, "fp": 5, "fn": 0}}, "qodo-extended-v2": {"strict": {"tp": 1, "fp": 3, "fn": 0}, "core": {"tp": 1, "fp": 3, "fn": 0}, "all": {"tp": 1, "fp": 3, "fn": 1}}, "greptile-v4-1": {"strict": {"tp": 1, "fp": 2, "fn": 0}, "core": {"tp": 1, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "gemini-v2": {"strict": {"tp": 1, "fp": 2, "fn": 0}, "core": {"tp": 1, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "gitlab": {"strict": {"tp": 1, "fp": 2, "fn": 0}, "core": {"tp": 1, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "copilot-v2": {"strict": {"tp": 1, "fp": 4, "fn": 0}, "core": {"tp": 1, "fp": 4, "fn": 0}, "all": {"tp": 2, "fp": 4, "fn": 0}}, "augment": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 1, "fp": 1, "fn": 0}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "bugbot": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 1, "fp": 1, "fn": 0}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "claude": {"strict": {"tp": 0, "fp": 1, "fn": 1}, "core": {"tp": 0, "fp": 1, "fn": 1}, "all": {"tp": 0, "fp": 1, "fn": 2}}, "claude-code": {"strict": {"tp": 0, "fp": 2, "fn": 1}, "core": {"tp": 0, "fp": 2, "fn": 1}, "all": {"tp": 0, "fp": 2, "fn": 2}}, "cubic-v2": {"strict": {"tp": 0, "fp": 2, "fn": 1}, "core": {"tp": 0, "fp": 2, "fn": 1}, "all": {"tp": 0, "fp": 2, "fn": 2}}, "devin": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 0}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "kodus-v2": {"strict": {"tp": 1, "fp": 2, "fn": 0}, "core": {"tp": 1, "fp": 2, "fn": 0}, "all": {"tp": 1, "fp": 2, "fn": 1}}, "macroscope": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 0}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "qodo-v2": {"strict": {"tp": 1, "fp": 2, "fn": 0}, "core": {"tp": 1, "fp": 2, "fn": 0}, "all": {"tp": 1, "fp": 2, "fn": 1}}, "sourcery": {"strict": {"tp": 1, "fp": 5, "fn": 0}, "core": {"tp": 1, "fp": 5, "fn": 0}, "all": {"tp": 1, "fp": 5, "fn": 1}}, "shipwright": {"strict": {"tp": 1, "fp": 2, "fn": 0}, "core": {"tp": 1, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}}}, {"url": "https://github.com/calcom/cal.com/pull/22532", "language": "TypeScript", "pr_size": "medium", "domain": "caching", "change_type": "feature", "complexity": "moderate", "difficulty": "subtle", "risk": "medium", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 1, "fp": 2, "fn": 3}, "core": {"tp": 1, "fp": 2, "fn": 3}, "all": {"tp": 1, "fp": 2, "fn": 3}}, "codeant-v2": {"strict": {"tp": 3, "fp": 2, "fn": 1}, "core": {"tp": 3, "fp": 2, "fn": 1}, "all": {"tp": 3, "fp": 2, "fn": 1}}, "coderabbit": {"strict": {"tp": 2, "fp": 9, "fn": 2}, "core": {"tp": 2, "fp": 9, "fn": 2}, "all": {"tp": 2, "fp": 9, "fn": 2}}, "qodo-extended-v2": {"strict": {"tp": 2, "fp": 1, "fn": 2}, "core": {"tp": 2, "fp": 1, "fn": 2}, "all": {"tp": 2, "fp": 1, "fn": 2}}, "greptile-v4-1": {"strict": {"tp": 3, "fp": 2, "fn": 1}, "core": {"tp": 3, "fp": 2, "fn": 1}, "all": {"tp": 3, "fp": 2, "fn": 1}}, "gemini-v2": {"strict": {"tp": 4, "fp": 0, "fn": 0}, "core": {"tp": 4, "fp": 0, "fn": 0}, "all": {"tp": 4, "fp": 0, "fn": 0}}, "gitlab": {"strict": {"tp": 2, "fp": 3, "fn": 2}, "core": {"tp": 2, "fp": 3, "fn": 2}, "all": {"tp": 2, "fp": 3, "fn": 2}}, "copilot-v2": {"strict": {"tp": 4, "fp": 2, "fn": 0}, "core": {"tp": 4, "fp": 2, "fn": 0}, "all": {"tp": 4, "fp": 2, "fn": 0}}, "augment": {"strict": {"tp": 4, "fp": 2, "fn": 0}, "core": {"tp": 4, "fp": 2, "fn": 0}, "all": {"tp": 4, "fp": 2, "fn": 0}}, "bugbot": {"strict": {"tp": 0, "fp": 1, "fn": 4}, "core": {"tp": 0, "fp": 1, "fn": 4}, "all": {"tp": 0, "fp": 1, "fn": 4}}, "claude": {"strict": {"tp": 0, "fp": 1, "fn": 4}, "core": {"tp": 0, "fp": 1, "fn": 4}, "all": {"tp": 0, "fp": 1, "fn": 4}}, "claude-code": {"strict": {"tp": 1, "fp": 3, "fn": 3}, "core": {"tp": 1, "fp": 3, "fn": 3}, "all": {"tp": 1, "fp": 3, "fn": 3}}, "cubic-v2": {"strict": {"tp": 0, "fp": 2, "fn": 4}, "core": {"tp": 0, "fp": 2, "fn": 4}, "all": {"tp": 0, "fp": 2, "fn": 4}}, "devin": {"strict": {"tp": 1, "fp": 1, "fn": 3}, "core": {"tp": 1, "fp": 1, "fn": 3}, "all": {"tp": 1, "fp": 1, "fn": 3}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 4}, "core": {"tp": 0, "fp": 0, "fn": 4}, "all": {"tp": 0, "fp": 0, "fn": 4}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 4}, "core": {"tp": 0, "fp": 0, "fn": 4}, "all": {"tp": 0, "fp": 0, "fn": 4}}, "kodus-v2": {"strict": {"tp": 1, "fp": 0, "fn": 3}, "core": {"tp": 1, "fp": 0, "fn": 3}, "all": {"tp": 1, "fp": 0, "fn": 3}}, "macroscope": {"strict": {"tp": 1, "fp": 1, "fn": 3}, "core": {"tp": 1, "fp": 1, "fn": 3}, "all": {"tp": 1, "fp": 1, "fn": 3}}, "qodo-v2": {"strict": {"tp": 3, "fp": 2, "fn": 1}, "core": {"tp": 3, "fp": 2, "fn": 1}, "all": {"tp": 3, "fp": 2, "fn": 1}}, "sourcery": {"strict": {"tp": 3, "fp": 1, "fn": 1}, "core": {"tp": 3, "fp": 1, "fn": 1}, "all": {"tp": 3, "fp": 1, "fn": 1}}, "shipwright": {"strict": {"tp": 1, "fp": 1, "fn": 3}, "core": {"tp": 1, "fp": 1, "fn": 3}, "all": {"tp": 1, "fp": 1, "fn": 3}}}}, {"url": "https://github.com/calcom/cal.com/pull/8330", "language": "TypeScript", "pr_size": "small", "domain": "scheduling", "change_type": "bug_fix", "complexity": "moderate", "difficulty": "subtle", "risk": "medium", "context": "file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 2, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "codeant-v2": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "coderabbit": {"strict": {"tp": 2, "fp": 2, "fn": 0}, "core": {"tp": 2, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "qodo-extended-v2": {"strict": {"tp": 2, "fp": 2, "fn": 0}, "core": {"tp": 2, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "greptile-v4-1": {"strict": {"tp": 2, "fp": 2, "fn": 0}, "core": {"tp": 2, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "gemini-v2": {"strict": {"tp": 2, "fp": 3, "fn": 0}, "core": {"tp": 2, "fp": 3, "fn": 0}, "all": {"tp": 2, "fp": 3, "fn": 0}}, "gitlab": {"strict": {"tp": 2, "fp": 2, "fn": 0}, "core": {"tp": 2, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "copilot-v2": {"strict": {"tp": 2, "fp": 4, "fn": 0}, "core": {"tp": 2, "fp": 4, "fn": 0}, "all": {"tp": 2, "fp": 4, "fn": 0}}, "augment": {"strict": {"tp": 2, "fp": 3, "fn": 0}, "core": {"tp": 2, "fp": 3, "fn": 0}, "all": {"tp": 2, "fp": 3, "fn": 0}}, "bugbot": {"strict": {"tp": 2, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "claude": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "claude-code": {"strict": {"tp": 2, "fp": 2, "fn": 0}, "core": {"tp": 2, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "cubic-v2": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "devin": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "graphite": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "kg": {"strict": {"tp": 1, "fp": 2, "fn": 1}, "core": {"tp": 1, "fp": 2, "fn": 1}, "all": {"tp": 1, "fp": 2, "fn": 1}}, "kodus-v2": {"strict": {"tp": 2, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "macroscope": {"strict": {"tp": 2, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "qodo-v2": {"strict": {"tp": 2, "fp": 2, "fn": 0}, "core": {"tp": 2, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "sourcery": {"strict": {"tp": 2, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "shipwright": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}}}, {"url": "https://github.com/calcom/cal.com/pull/14943", "language": "TypeScript", "pr_size": "small", "domain": "scheduling", "change_type": "feature", "complexity": "moderate", "difficulty": "subtle", "risk": "high", "context": "file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "codeant-v2": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "coderabbit": {"strict": {"tp": 2, "fp": 2, "fn": 0}, "core": {"tp": 2, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "qodo-extended-v2": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "greptile-v4-1": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "gemini-v2": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "gitlab": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "copilot-v2": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "augment": {"strict": {"tp": 2, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "bugbot": {"strict": {"tp": 1, "fp": 2, "fn": 1}, "core": {"tp": 1, "fp": 2, "fn": 1}, "all": {"tp": 1, "fp": 2, "fn": 1}}, "claude": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "claude-code": {"strict": {"tp": 0, "fp": 1, "fn": 2}, "core": {"tp": 0, "fp": 1, "fn": 2}, "all": {"tp": 0, "fp": 1, "fn": 2}}, "cubic-v2": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "devin": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "graphite": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "kg": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "kodus-v2": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "macroscope": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "qodo-v2": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "sourcery": {"strict": {"tp": 2, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "shipwright": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}}}, {"url": "https://github.com/calcom/cal.com/pull/22345", "language": "TypeScript", "pr_size": "small", "domain": "database", "change_type": "migration", "complexity": "moderate", "difficulty": "subtle", "risk": "medium", "context": "file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 0, "fp": 2, "fn": 1}, "core": {"tp": 0, "fp": 2, "fn": 1}, "all": {"tp": 0, "fp": 2, "fn": 2}}, "codeant-v2": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "coderabbit": {"strict": {"tp": 0, "fp": 3, "fn": 1}, "core": {"tp": 0, "fp": 3, "fn": 1}, "all": {"tp": 1, "fp": 3, "fn": 1}}, "qodo-extended-v2": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "greptile-v4-1": {"strict": {"tp": 0, "fp": 2, "fn": 1}, "core": {"tp": 0, "fp": 2, "fn": 1}, "all": {"tp": 0, "fp": 2, "fn": 2}}, "gemini-v2": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 0}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "gitlab": {"strict": {"tp": 1, "fp": 2, "fn": 0}, "core": {"tp": 1, "fp": 2, "fn": 0}, "all": {"tp": 1, "fp": 2, "fn": 1}}, "copilot-v2": {"strict": {"tp": 0, "fp": 3, "fn": 1}, "core": {"tp": 0, "fp": 3, "fn": 1}, "all": {"tp": 1, "fp": 3, "fn": 1}}, "augment": {"strict": {"tp": 0, "fp": 1, "fn": 1}, "core": {"tp": 0, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "bugbot": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 1, "fp": 1, "fn": 0}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "claude": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "claude-code": {"strict": {"tp": 1, "fp": 2, "fn": 0}, "core": {"tp": 1, "fp": 2, "fn": 0}, "all": {"tp": 1, "fp": 2, "fn": 1}}, "cubic-v2": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 1, "fp": 1, "fn": 0}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "devin": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "kg": {"strict": {"tp": 0, "fp": 1, "fn": 1}, "core": {"tp": 0, "fp": 1, "fn": 1}, "all": {"tp": 0, "fp": 1, "fn": 2}}, "kodus-v2": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "macroscope": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "qodo-v2": {"strict": {"tp": 0, "fp": 3, "fn": 1}, "core": {"tp": 0, "fp": 3, "fn": 1}, "all": {"tp": 1, "fp": 3, "fn": 1}}, "sourcery": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 0}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "shipwright": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 0, "fp": 0, "fn": 2}}}}, {"url": "https://github.com/calcom/cal.com/pull/11059", "language": "TypeScript", "pr_size": "large", "domain": "authentication", "change_type": "feature", "complexity": "complex", "difficulty": "subtle", "risk": "critical", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 6, "fp": 0, "fn": 3}, "core": {"tp": 6, "fp": 0, "fn": 3}, "all": {"tp": 6, "fp": 0, "fn": 3}}, "codeant-v2": {"strict": {"tp": 7, "fp": 4, "fn": 2}, "core": {"tp": 7, "fp": 4, "fn": 2}, "all": {"tp": 7, "fp": 4, "fn": 2}}, "coderabbit": {"strict": {"tp": 9, "fp": 10, "fn": 0}, "core": {"tp": 9, "fp": 10, "fn": 0}, "all": {"tp": 9, "fp": 10, "fn": 0}}, "qodo-extended-v2": {"strict": {"tp": 7, "fp": 2, "fn": 2}, "core": {"tp": 7, "fp": 2, "fn": 2}, "all": {"tp": 7, "fp": 2, "fn": 2}}, "greptile-v4-1": {"strict": {"tp": 5, "fp": 2, "fn": 4}, "core": {"tp": 5, "fp": 2, "fn": 4}, "all": {"tp": 5, "fp": 2, "fn": 4}}, "gemini-v2": {"strict": {"tp": 5, "fp": 1, "fn": 4}, "core": {"tp": 5, "fp": 1, "fn": 4}, "all": {"tp": 5, "fp": 1, "fn": 4}}, "gitlab": {"strict": {"tp": 6, "fp": 2, "fn": 3}, "core": {"tp": 6, "fp": 2, "fn": 3}, "all": {"tp": 6, "fp": 2, "fn": 3}}, "copilot-v2": {"strict": {"tp": 6, "fp": 2, "fn": 3}, "core": {"tp": 6, "fp": 2, "fn": 3}, "all": {"tp": 6, "fp": 2, "fn": 3}}, "augment": {"strict": {"tp": 9, "fp": 1, "fn": 0}, "core": {"tp": 9, "fp": 1, "fn": 0}, "all": {"tp": 9, "fp": 1, "fn": 0}}, "bugbot": {"strict": {"tp": 6, "fp": 4, "fn": 3}, "core": {"tp": 6, "fp": 4, "fn": 3}, "all": {"tp": 6, "fp": 4, "fn": 3}}, "claude": {"strict": {"tp": 6, "fp": 0, "fn": 3}, "core": {"tp": 6, "fp": 0, "fn": 3}, "all": {"tp": 6, "fp": 0, "fn": 3}}, "claude-code": {"strict": {"tp": 7, "fp": 0, "fn": 2}, "core": {"tp": 7, "fp": 0, "fn": 2}, "all": {"tp": 7, "fp": 0, "fn": 2}}, "cubic-v2": {"strict": {"tp": 4, "fp": 0, "fn": 5}, "core": {"tp": 4, "fp": 0, "fn": 5}, "all": {"tp": 4, "fp": 0, "fn": 5}}, "devin": {"strict": {"tp": 5, "fp": 0, "fn": 4}, "core": {"tp": 5, "fp": 0, "fn": 4}, "all": {"tp": 5, "fp": 0, "fn": 4}}, "graphite": {"strict": {"tp": 1, "fp": 0, "fn": 8}, "core": {"tp": 1, "fp": 0, "fn": 8}, "all": {"tp": 1, "fp": 0, "fn": 8}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 9}, "core": {"tp": 0, "fp": 0, "fn": 9}, "all": {"tp": 0, "fp": 0, "fn": 9}}, "kodus-v2": {"strict": {"tp": 4, "fp": 3, "fn": 5}, "core": {"tp": 4, "fp": 3, "fn": 5}, "all": {"tp": 4, "fp": 3, "fn": 5}}, "macroscope": {"strict": {"tp": 7, "fp": 6, "fn": 2}, "core": {"tp": 7, "fp": 6, "fn": 2}, "all": {"tp": 7, "fp": 6, "fn": 2}}, "qodo-v2": {"strict": {"tp": 8, "fp": 0, "fn": 1}, "core": {"tp": 8, "fp": 0, "fn": 1}, "all": {"tp": 8, "fp": 0, "fn": 1}}, "sourcery": {"strict": {"tp": 5, "fp": 0, "fn": 4}, "core": {"tp": 5, "fp": 0, "fn": 4}, "all": {"tp": 5, "fp": 0, "fn": 4}}, "shipwright": {"strict": {"tp": 5, "fp": 1, "fn": 4}, "core": {"tp": 5, "fp": 1, "fn": 4}, "all": {"tp": 5, "fp": 1, "fn": 4}}}}, {"url": "https://github.com/calcom/cal.com/pull/7232", "language": "TypeScript", "pr_size": "medium", "domain": "scheduling", "change_type": "feature", "complexity": "moderate", "difficulty": "subtle", "risk": "high", "context": "cross_file", "concern": "reliability", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 1, "fp": 0, "fn": 2}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 2}}, "codeant-v2": {"strict": {"tp": 1, "fp": 2, "fn": 2}, "core": {"tp": 1, "fp": 2, "fn": 2}, "all": {"tp": 1, "fp": 2, "fn": 2}}, "coderabbit": {"strict": {"tp": 2, "fp": 9, "fn": 1}, "core": {"tp": 2, "fp": 9, "fn": 1}, "all": {"tp": 2, "fp": 9, "fn": 1}}, "qodo-extended-v2": {"strict": {"tp": 2, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 1}, "all": {"tp": 2, "fp": 1, "fn": 1}}, "greptile-v4-1": {"strict": {"tp": 1, "fp": 2, "fn": 2}, "core": {"tp": 1, "fp": 2, "fn": 2}, "all": {"tp": 1, "fp": 2, "fn": 2}}, "gemini-v2": {"strict": {"tp": 2, "fp": 3, "fn": 1}, "core": {"tp": 2, "fp": 3, "fn": 1}, "all": {"tp": 2, "fp": 3, "fn": 1}}, "gitlab": {"strict": {"tp": 0, "fp": 2, "fn": 3}, "core": {"tp": 0, "fp": 2, "fn": 3}, "all": {"tp": 0, "fp": 2, "fn": 3}}, "copilot-v2": {"strict": {"tp": 0, "fp": 3, "fn": 3}, "core": {"tp": 0, "fp": 3, "fn": 3}, "all": {"tp": 0, "fp": 3, "fn": 3}}, "augment": {"strict": {"tp": 2, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 1}, "all": {"tp": 2, "fp": 1, "fn": 1}}, "bugbot": {"strict": {"tp": 1, "fp": 1, "fn": 2}, "core": {"tp": 1, "fp": 1, "fn": 2}, "all": {"tp": 1, "fp": 1, "fn": 2}}, "claude": {"strict": {"tp": 1, "fp": 3, "fn": 2}, "core": {"tp": 1, "fp": 3, "fn": 2}, "all": {"tp": 1, "fp": 3, "fn": 2}}, "claude-code": {"strict": {"tp": 1, "fp": 1, "fn": 2}, "core": {"tp": 1, "fp": 1, "fn": 2}, "all": {"tp": 1, "fp": 1, "fn": 2}}, "cubic-v2": {"strict": {"tp": 2, "fp": 2, "fn": 1}, "core": {"tp": 2, "fp": 2, "fn": 1}, "all": {"tp": 2, "fp": 2, "fn": 1}}, "devin": {"strict": {"tp": 1, "fp": 1, "fn": 2}, "core": {"tp": 1, "fp": 1, "fn": 2}, "all": {"tp": 1, "fp": 1, "fn": 2}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 3}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 3}}, "kg": {"strict": {"tp": 0, "fp": 1, "fn": 3}, "core": {"tp": 0, "fp": 1, "fn": 3}, "all": {"tp": 0, "fp": 1, "fn": 3}}, "kodus-v2": {"strict": {"tp": 2, "fp": 2, "fn": 1}, "core": {"tp": 2, "fp": 2, "fn": 1}, "all": {"tp": 2, "fp": 2, "fn": 1}}, "macroscope": {"strict": {"tp": 1, "fp": 2, "fn": 2}, "core": {"tp": 1, "fp": 2, "fn": 2}, "all": {"tp": 1, "fp": 2, "fn": 2}}, "qodo-v2": {"strict": {"tp": 2, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 1}, "all": {"tp": 2, "fp": 1, "fn": 1}}, "sourcery": {"strict": {"tp": 1, "fp": 2, "fn": 2}, "core": {"tp": 1, "fp": 2, "fn": 2}, "all": {"tp": 1, "fp": 2, "fn": 2}}, "shipwright": {"strict": {"tp": 2, "fp": 2, "fn": 1}, "core": {"tp": 2, "fp": 2, "fn": 1}, "all": {"tp": 2, "fp": 2, "fn": 1}}}}, {"url": "https://github.com/calcom/cal.com/pull/14740", "language": "TypeScript", "pr_size": "large", "domain": "scheduling", "change_type": "feature", "complexity": "moderate", "difficulty": "subtle", "risk": "critical", "context": "cross_file", "concern": "security", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 1, "fp": 1, "fn": 5}, "core": {"tp": 1, "fp": 1, "fn": 5}, "all": {"tp": 1, "fp": 1, "fn": 5}}, "codeant-v2": {"strict": {"tp": 3, "fp": 4, "fn": 3}, "core": {"tp": 3, "fp": 4, "fn": 3}, "all": {"tp": 3, "fp": 4, "fn": 3}}, "coderabbit": {"strict": {"tp": 5, "fp": 5, "fn": 1}, "core": {"tp": 5, "fp": 5, "fn": 1}, "all": {"tp": 5, "fp": 5, "fn": 1}}, "qodo-extended-v2": {"strict": {"tp": 4, "fp": 1, "fn": 2}, "core": {"tp": 4, "fp": 1, "fn": 2}, "all": {"tp": 4, "fp": 1, "fn": 2}}, "greptile-v4-1": {"strict": {"tp": 3, "fp": 2, "fn": 3}, "core": {"tp": 3, "fp": 2, "fn": 3}, "all": {"tp": 3, "fp": 2, "fn": 3}}, "gemini-v2": {"strict": {"tp": 3, "fp": 4, "fn": 3}, "core": {"tp": 3, "fp": 4, "fn": 3}, "all": {"tp": 3, "fp": 4, "fn": 3}}, "gitlab": {"strict": {"tp": 3, "fp": 4, "fn": 3}, "core": {"tp": 3, "fp": 4, "fn": 3}, "all": {"tp": 3, "fp": 4, "fn": 3}}, "copilot-v2": {"strict": {"tp": 5, "fp": 3, "fn": 1}, "core": {"tp": 5, "fp": 3, "fn": 1}, "all": {"tp": 5, "fp": 3, "fn": 1}}, "augment": {"strict": {"tp": 4, "fp": 3, "fn": 2}, "core": {"tp": 4, "fp": 3, "fn": 2}, "all": {"tp": 4, "fp": 3, "fn": 2}}, "bugbot": {"strict": {"tp": 3, "fp": 3, "fn": 3}, "core": {"tp": 3, "fp": 3, "fn": 3}, "all": {"tp": 3, "fp": 3, "fn": 3}}, "claude": {"strict": {"tp": 2, "fp": 0, "fn": 4}, "core": {"tp": 2, "fp": 0, "fn": 4}, "all": {"tp": 2, "fp": 0, "fn": 4}}, "claude-code": {"strict": {"tp": 2, "fp": 1, "fn": 4}, "core": {"tp": 2, "fp": 1, "fn": 4}, "all": {"tp": 2, "fp": 1, "fn": 4}}, "cubic-v2": {"strict": {"tp": 4, "fp": 2, "fn": 2}, "core": {"tp": 4, "fp": 2, "fn": 2}, "all": {"tp": 4, "fp": 2, "fn": 2}}, "devin": {"strict": {"tp": 2, "fp": 0, "fn": 4}, "core": {"tp": 2, "fp": 0, "fn": 4}, "all": {"tp": 2, "fp": 0, "fn": 4}}, "graphite": {"strict": {"tp": 1, "fp": 0, "fn": 5}, "core": {"tp": 1, "fp": 0, "fn": 5}, "all": {"tp": 1, "fp": 0, "fn": 5}}, "kg": {"strict": {"tp": 2, "fp": 0, "fn": 4}, "core": {"tp": 2, "fp": 0, "fn": 4}, "all": {"tp": 2, "fp": 0, "fn": 4}}, "kodus-v2": {"strict": {"tp": 2, "fp": 0, "fn": 4}, "core": {"tp": 2, "fp": 0, "fn": 4}, "all": {"tp": 2, "fp": 0, "fn": 4}}, "macroscope": {"strict": {"tp": 3, "fp": 3, "fn": 3}, "core": {"tp": 3, "fp": 3, "fn": 3}, "all": {"tp": 3, "fp": 3, "fn": 3}}, "qodo-v2": {"strict": {"tp": 5, "fp": 2, "fn": 1}, "core": {"tp": 5, "fp": 2, "fn": 1}, "all": {"tp": 5, "fp": 2, "fn": 1}}, "sourcery": {"strict": {"tp": 3, "fp": 1, "fn": 3}, "core": {"tp": 3, "fp": 1, "fn": 3}, "all": {"tp": 3, "fp": 1, "fn": 3}}, "shipwright": {"strict": {"tp": 2, "fp": 3, "fn": 4}, "core": {"tp": 2, "fp": 3, "fn": 4}, "all": {"tp": 2, "fp": 3, "fn": 4}}}}, {"url": "https://github.com/calcom/cal.com/pull/10600", "language": "TypeScript", "pr_size": "medium", "domain": "authentication", "change_type": "feature", "complexity": "moderate", "difficulty": "subtle", "risk": "high", "context": "cross_file", "concern": "security", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 0, "fp": 1, "fn": 2}, "core": {"tp": 0, "fp": 1, "fn": 4}, "all": {"tp": 0, "fp": 1, "fn": 5}}, "codeant-v2": {"strict": {"tp": 1, "fp": 2, "fn": 1}, "core": {"tp": 1, "fp": 2, "fn": 3}, "all": {"tp": 1, "fp": 2, "fn": 4}}, "coderabbit": {"strict": {"tp": 1, "fp": 6, "fn": 1}, "core": {"tp": 1, "fp": 6, "fn": 3}, "all": {"tp": 2, "fp": 6, "fn": 3}}, "qodo-extended-v2": {"strict": {"tp": 1, "fp": 2, "fn": 1}, "core": {"tp": 2, "fp": 2, "fn": 2}, "all": {"tp": 2, "fp": 2, "fn": 3}}, "greptile-v4-1": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 2, "fp": 0, "fn": 2}, "all": {"tp": 3, "fp": 0, "fn": 2}}, "gemini-v2": {"strict": {"tp": 0, "fp": 5, "fn": 2}, "core": {"tp": 0, "fp": 5, "fn": 4}, "all": {"tp": 1, "fp": 5, "fn": 4}}, "gitlab": {"strict": {"tp": 0, "fp": 2, "fn": 2}, "core": {"tp": 1, "fp": 2, "fn": 3}, "all": {"tp": 2, "fp": 2, "fn": 3}}, "copilot-v2": {"strict": {"tp": 2, "fp": 4, "fn": 0}, "core": {"tp": 4, "fp": 4, "fn": 0}, "all": {"tp": 5, "fp": 4, "fn": 0}}, "augment": {"strict": {"tp": 0, "fp": 3, "fn": 2}, "core": {"tp": 1, "fp": 3, "fn": 3}, "all": {"tp": 2, "fp": 3, "fn": 3}}, "bugbot": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 3}, "all": {"tp": 1, "fp": 0, "fn": 4}}, "claude": {"strict": {"tp": 0, "fp": 6, "fn": 2}, "core": {"tp": 1, "fp": 6, "fn": 3}, "all": {"tp": 2, "fp": 6, "fn": 3}}, "claude-code": {"strict": {"tp": 0, "fp": 4, "fn": 2}, "core": {"tp": 0, "fp": 4, "fn": 4}, "all": {"tp": 1, "fp": 4, "fn": 4}}, "cubic-v2": {"strict": {"tp": 2, "fp": 2, "fn": 0}, "core": {"tp": 2, "fp": 2, "fn": 2}, "all": {"tp": 2, "fp": 2, "fn": 3}}, "devin": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 3}, "all": {"tp": 1, "fp": 0, "fn": 4}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 4}, "all": {"tp": 0, "fp": 0, "fn": 5}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 4}, "all": {"tp": 0, "fp": 0, "fn": 5}}, "kodus-v2": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 3}, "all": {"tp": 1, "fp": 0, "fn": 4}}, "macroscope": {"strict": {"tp": 0, "fp": 2, "fn": 2}, "core": {"tp": 0, "fp": 2, "fn": 4}, "all": {"tp": 0, "fp": 2, "fn": 5}}, "qodo-v2": {"strict": {"tp": 1, "fp": 2, "fn": 1}, "core": {"tp": 2, "fp": 2, "fn": 2}, "all": {"tp": 2, "fp": 2, "fn": 3}}, "sourcery": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 2}, "all": {"tp": 3, "fp": 1, "fn": 2}}, "shipwright": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 2, "fp": 0, "fn": 2}, "all": {"tp": 3, "fp": 0, "fn": 2}}}}, {"url": "https://github.com/calcom/cal.com/pull/10967", "language": "TypeScript", "pr_size": "large", "domain": "scheduling", "change_type": "bug_fix", "complexity": "complex", "difficulty": "subtle", "risk": "high", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 1, "fp": 1, "fn": 4}, "core": {"tp": 1, "fp": 1, "fn": 4}, "all": {"tp": 1, "fp": 1, "fn": 5}}, "codeant-v2": {"strict": {"tp": 2, "fp": 5, "fn": 3}, "core": {"tp": 2, "fp": 5, "fn": 3}, "all": {"tp": 2, "fp": 5, "fn": 4}}, "coderabbit": {"strict": {"tp": 5, "fp": 7, "fn": 0}, "core": {"tp": 5, "fp": 7, "fn": 0}, "all": {"tp": 5, "fp": 7, "fn": 1}}, "qodo-extended-v2": {"strict": {"tp": 4, "fp": 2, "fn": 1}, "core": {"tp": 4, "fp": 2, "fn": 1}, "all": {"tp": 4, "fp": 2, "fn": 2}}, "greptile-v4-1": {"strict": {"tp": 2, "fp": 1, "fn": 3}, "core": {"tp": 2, "fp": 1, "fn": 3}, "all": {"tp": 2, "fp": 1, "fn": 4}}, "gemini-v2": {"strict": {"tp": 2, "fp": 4, "fn": 3}, "core": {"tp": 2, "fp": 4, "fn": 3}, "all": {"tp": 2, "fp": 4, "fn": 4}}, "gitlab": {"strict": {"tp": 4, "fp": 1, "fn": 1}, "core": {"tp": 4, "fp": 1, "fn": 1}, "all": {"tp": 4, "fp": 1, "fn": 2}}, "copilot-v2": {"strict": {"tp": 5, "fp": 6, "fn": 0}, "core": {"tp": 5, "fp": 6, "fn": 0}, "all": {"tp": 5, "fp": 6, "fn": 1}}, "augment": {"strict": {"tp": 4, "fp": 0, "fn": 1}, "core": {"tp": 4, "fp": 0, "fn": 1}, "all": {"tp": 4, "fp": 0, "fn": 2}}, "bugbot": {"strict": {"tp": 3, "fp": 3, "fn": 2}, "core": {"tp": 3, "fp": 3, "fn": 2}, "all": {"tp": 4, "fp": 3, "fn": 2}}, "claude": {"strict": {"tp": 4, "fp": 3, "fn": 1}, "core": {"tp": 4, "fp": 3, "fn": 1}, "all": {"tp": 4, "fp": 3, "fn": 2}}, "claude-code": {"strict": {"tp": 4, "fp": 3, "fn": 1}, "core": {"tp": 4, "fp": 3, "fn": 1}, "all": {"tp": 5, "fp": 3, "fn": 1}}, "cubic-v2": {"strict": {"tp": 1, "fp": 2, "fn": 4}, "core": {"tp": 1, "fp": 2, "fn": 4}, "all": {"tp": 1, "fp": 2, "fn": 5}}, "devin": {"strict": {"tp": 2, "fp": 3, "fn": 3}, "core": {"tp": 2, "fp": 3, "fn": 3}, "all": {"tp": 2, "fp": 3, "fn": 4}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 5}, "core": {"tp": 0, "fp": 0, "fn": 5}, "all": {"tp": 0, "fp": 0, "fn": 6}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 5}, "core": {"tp": 0, "fp": 0, "fn": 5}, "all": {"tp": 0, "fp": 0, "fn": 6}}, "kodus-v2": {"strict": {"tp": 2, "fp": 2, "fn": 3}, "core": {"tp": 2, "fp": 2, "fn": 3}, "all": {"tp": 2, "fp": 2, "fn": 4}}, "macroscope": {"strict": {"tp": 3, "fp": 2, "fn": 2}, "core": {"tp": 3, "fp": 2, "fn": 2}, "all": {"tp": 3, "fp": 2, "fn": 3}}, "qodo-v2": {"strict": {"tp": 2, "fp": 3, "fn": 3}, "core": {"tp": 2, "fp": 3, "fn": 3}, "all": {"tp": 2, "fp": 3, "fn": 4}}, "sourcery": {"strict": {"tp": 3, "fp": 2, "fn": 2}, "core": {"tp": 3, "fp": 2, "fn": 2}, "all": {"tp": 3, "fp": 2, "fn": 3}}, "shipwright": {"strict": {"tp": 3, "fp": 1, "fn": 2}, "core": {"tp": 3, "fp": 1, "fn": 2}, "all": {"tp": 3, "fp": 1, "fn": 3}}}}, {"url": "https://github.com/calcom/cal.com/pull/8087", "language": "TypeScript", "pr_size": "medium", "domain": "concurrency", "change_type": "refactoring", "complexity": "moderate", "difficulty": "subtle", "risk": "critical", "context": "cross_file", "concern": "reliability", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 1, "fp": 3, "fn": 0}, "core": {"tp": 1, "fp": 3, "fn": 0}, "all": {"tp": 1, "fp": 3, "fn": 1}}, "codeant-v2": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 1, "fp": 1, "fn": 0}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "coderabbit": {"strict": {"tp": 1, "fp": 6, "fn": 0}, "core": {"tp": 1, "fp": 6, "fn": 0}, "all": {"tp": 2, "fp": 6, "fn": 0}}, "qodo-extended-v2": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 1, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "greptile-v4-1": {"strict": {"tp": 1, "fp": 2, "fn": 0}, "core": {"tp": 1, "fp": 2, "fn": 0}, "all": {"tp": 1, "fp": 2, "fn": 1}}, "gemini-v2": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 0}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "gitlab": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 1, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "copilot-v2": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 1, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "augment": {"strict": {"tp": 1, "fp": 4, "fn": 0}, "core": {"tp": 1, "fp": 4, "fn": 0}, "all": {"tp": 1, "fp": 4, "fn": 1}}, "bugbot": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 0}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "claude": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 1, "fp": 1, "fn": 0}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "claude-code": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 1, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "cubic-v2": {"strict": {"tp": 1, "fp": 10, "fn": 0}, "core": {"tp": 1, "fp": 10, "fn": 0}, "all": {"tp": 2, "fp": 10, "fn": 0}}, "devin": {"strict": {"tp": 1, "fp": 2, "fn": 0}, "core": {"tp": 1, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "kg": {"strict": {"tp": 1, "fp": 2, "fn": 0}, "core": {"tp": 1, "fp": 2, "fn": 0}, "all": {"tp": 1, "fp": 2, "fn": 1}}, "kodus-v2": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "macroscope": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 0}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "qodo-v2": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 0}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "sourcery": {"strict": {"tp": 1, "fp": 3, "fn": 0}, "core": {"tp": 1, "fp": 3, "fn": 0}, "all": {"tp": 1, "fp": 3, "fn": 1}}, "shipwright": {"strict": {"tp": 1, "fp": 2, "fn": 0}, "core": {"tp": 1, "fp": 2, "fn": 0}, "all": {"tp": 1, "fp": 2, "fn": 1}}}}], "tools": ["augment", "baz", "bugbot", "claude", "claude-code", "codeant-v2", "coderabbit", "copilot-v2", "cubic-v2", "devin", "gemini-v2", "gitlab", "graphite", "greptile-v4-1", "kg", "kodus-v2", "macroscope", "qodo-extended-v2", "qodo-v2", "shipwright", "sourcery"], "dimensions": {"language": ["Go", "Java", "Python", "Ruby", "TypeScript"], "pr_size": ["small", "medium", "large"], "domain": ["API", "UI", "authentication", "caching", "concurrency", "configuration", "data_processing", "database", "file_io", "logging", "scheduling", "serialization", "testing"], "change_type": ["bug_fix", "feature", "migration", "performance", "refactoring", "security_patch"], "complexity": ["simple", "moderate", "complex"], "difficulty": ["obvious", "moderate", "subtle", "very_subtle"], "risk": ["low", "medium", "high", "critical"], "context": ["local", "file", "cross_file", "system"], "concern": ["correctness", "maintainability", "reliability", "security"]}, "overall_metrics": {"sourcery": {"strict": {"precision": 38.4, "recall": 56.1, "f1": 45.6, "f2": 51.4, "tp": 78, "fp": 125, "fn": 61, "num_prs": 50}, "core": {"precision": 41.0, "recall": 55.1, "f1": 47.0, "f2": 51.5, "tp": 87, "fp": 125, "fn": 71, "num_prs": 50}, "all": {"precision": 43.2, "recall": 54.9, "f1": 48.3, "f2": 52.1, "tp": 95, "fp": 125, "fn": 78, "num_prs": 50}}, "macroscope": {"strict": {"precision": 60.2, "recall": 53.2, "f1": 56.5, "f2": 54.5, "tp": 74, "fp": 49, "fn": 65, "num_prs": 50}, "core": {"precision": 61.1, "recall": 48.7, "f1": 54.2, "f2": 50.8, "tp": 77, "fp": 49, "fn": 81, "num_prs": 50}, "all": {"precision": 62.3, "recall": 46.8, "f1": 53.5, "f2": 49.3, "tp": 81, "fp": 49, "fn": 92, "num_prs": 50}}, "graphite": {"strict": {"precision": 100.0, "recall": 7.9, "f1": 14.7, "f2": 9.7, "tp": 11, "fp": 0, "fn": 128, "num_prs": 50}, "core": {"precision": 100.0, "recall": 7.6, "f1": 14.1, "f2": 9.3, "tp": 12, "fp": 0, "fn": 146, "num_prs": 50}, "all": {"precision": 100.0, "recall": 6.9, "f1": 13.0, "f2": 8.5, "tp": 12, "fp": 0, "fn": 161, "num_prs": 50}}, "augment": {"strict": {"precision": 56.5, "recall": 65.5, "f1": 60.7, "f2": 63.5, "tp": 91, "fp": 70, "fn": 48, "num_prs": 50}, "core": {"precision": 59.5, "recall": 65.2, "f1": 62.2, "f2": 64.0, "tp": 103, "fp": 70, "fn": 55, "num_prs": 50}, "all": {"precision": 60.7, "recall": 62.4, "f1": 61.5, "f2": 62.1, "tp": 108, "fp": 70, "fn": 65, "num_prs": 50}}, "qodo-extended-v2": {"strict": {"precision": 64.8, "recall": 66.2, "f1": 65.5, "f2": 65.9, "tp": 92, "fp": 50, "fn": 47, "num_prs": 50}, "core": {"precision": 67.1, "recall": 64.6, "f1": 65.8, "f2": 65.1, "tp": 102, "fp": 50, "fn": 56, "num_prs": 50}, "all": {"precision": 67.9, "recall": 61.3, "f1": 64.4, "f2": 62.5, "tp": 106, "fp": 50, "fn": 67, "num_prs": 50}}, "kodus-v2": {"strict": {"precision": 57.1, "recall": 40.3, "f1": 47.3, "f2": 42.8, "tp": 56, "fp": 42, "fn": 83, "num_prs": 50}, "core": {"precision": 59.2, "recall": 38.6, "f1": 46.7, "f2": 41.5, "tp": 61, "fp": 42, "fn": 97, "num_prs": 50}, "all": {"precision": 59.6, "recall": 35.8, "f1": 44.8, "f2": 38.9, "tp": 62, "fp": 42, "fn": 111, "num_prs": 50}}, "kg": {"strict": {"precision": 53.3, "recall": 17.3, "f1": 26.1, "f2": 20.0, "tp": 24, "fp": 21, "fn": 115, "num_prs": 50}, "core": {"precision": 54.3, "recall": 15.8, "f1": 24.5, "f2": 18.4, "tp": 25, "fp": 21, "fn": 133, "num_prs": 50}, "all": {"precision": 55.3, "recall": 15.0, "f1": 23.6, "f2": 17.6, "tp": 26, "fp": 21, "fn": 147, "num_prs": 50}}, "cubic-v2": {"strict": {"precision": 58.3, "recall": 65.5, "f1": 61.7, "f2": 63.9, "tp": 91, "fp": 65, "fn": 48, "num_prs": 50}, "core": {"precision": 61.5, "recall": 65.8, "f1": 63.6, "f2": 64.9, "tp": 104, "fp": 65, "fn": 54, "num_prs": 50}, "all": {"precision": 62.4, "recall": 62.4, "f1": 62.4, "f2": 62.4, "tp": 108, "fp": 65, "fn": 65, "num_prs": 50}}, "gemini-v2": {"strict": {"precision": 43.2, "recall": 57.6, "f1": 49.4, "f2": 54.0, "tp": 80, "fp": 105, "fn": 59, "num_prs": 50}, "core": {"precision": 45.3, "recall": 55.1, "f1": 49.7, "f2": 52.8, "tp": 87, "fp": 105, "fn": 71, "num_prs": 50}, "all": {"precision": 47.5, "recall": 54.9, "f1": 50.9, "f2": 53.3, "tp": 95, "fp": 105, "fn": 78, "num_prs": 50}}, "greptile-v4-1": {"strict": {"precision": 49.0, "recall": 54.0, "f1": 51.4, "f2": 52.9, "tp": 75, "fp": 78, "fn": 64, "num_prs": 50}, "core": {"precision": 50.9, "recall": 51.3, "f1": 51.1, "f2": 51.2, "tp": 81, "fp": 78, "fn": 77, "num_prs": 50}, "all": {"precision": 53.3, "recall": 51.4, "f1": 52.4, "f2": 51.8, "tp": 89, "fp": 78, "fn": 84, "num_prs": 50}}, "bugbot": {"strict": {"precision": 54.8, "recall": 48.9, "f1": 51.7, "f2": 50.0, "tp": 68, "fp": 56, "fn": 71, "num_prs": 50}, "core": {"precision": 56.9, "recall": 46.8, "f1": 51.4, "f2": 48.6, "tp": 74, "fp": 56, "fn": 84, "num_prs": 50}, "all": {"precision": 58.2, "recall": 45.1, "f1": 50.8, "f2": 47.2, "tp": 78, "fp": 56, "fn": 95, "num_prs": 50}}, "claude": {"strict": {"precision": 41.0, "recall": 39.6, "f1": 40.3, "f2": 39.9, "tp": 55, "fp": 79, "fn": 84, "num_prs": 50}, "core": {"precision": 43.2, "recall": 38.0, "f1": 40.4, "f2": 38.9, "tp": 60, "fp": 79, "fn": 98, "num_prs": 50}, "all": {"precision": 45.5, "recall": 38.2, "f1": 41.5, "f2": 39.4, "tp": 66, "fp": 79, "fn": 107, "num_prs": 50}}, "devin": {"strict": {"precision": 67.4, "recall": 43.2, "f1": 52.6, "f2": 46.5, "tp": 60, "fp": 29, "fn": 79, "num_prs": 50}, "core": {"precision": 69.1, "recall": 41.1, "f1": 51.6, "f2": 44.8, "tp": 65, "fp": 29, "fn": 93, "num_prs": 50}, "all": {"precision": 70.1, "recall": 39.3, "f1": 50.4, "f2": 43.1, "tp": 68, "fp": 29, "fn": 105, "num_prs": 50}}, "baz": {"strict": {"precision": 60.6, "recall": 41.0, "f1": 48.9, "f2": 43.8, "tp": 57, "fp": 37, "fn": 82, "num_prs": 50}, "core": {"precision": 61.5, "recall": 37.3, "f1": 46.5, "f2": 40.5, "tp": 59, "fp": 37, "fn": 99, "num_prs": 50}, "all": {"precision": 62.6, "recall": 35.8, "f1": 45.6, "f2": 39.2, "tp": 62, "fp": 37, "fn": 111, "num_prs": 50}}, "claude-code": {"strict": {"precision": 44.3, "recall": 50.4, "f1": 47.1, "f2": 49.0, "tp": 70, "fp": 88, "fn": 69, "num_prs": 50}, "core": {"precision": 46.3, "recall": 48.1, "f1": 47.2, "f2": 47.7, "tp": 76, "fp": 88, "fn": 82, "num_prs": 50}, "all": {"precision": 48.8, "recall": 48.6, "f1": 48.7, "f2": 48.6, "tp": 84, "fp": 88, "fn": 89, "num_prs": 50}}, "gitlab": {"strict": {"precision": 51.0, "recall": 54.0, "f1": 52.4, "f2": 53.3, "tp": 75, "fp": 72, "fn": 64, "num_prs": 50}, "core": {"precision": 53.5, "recall": 52.5, "f1": 53.0, "f2": 52.7, "tp": 83, "fp": 72, "fn": 75, "num_prs": 50}, "all": {"precision": 55.8, "recall": 52.6, "f1": 54.2, "f2": 53.2, "tp": 91, "fp": 72, "fn": 82, "num_prs": 50}}, "qodo-v2": {"strict": {"precision": 53.0, "recall": 64.0, "f1": 58.0, "f2": 61.5, "tp": 89, "fp": 79, "fn": 50, "num_prs": 50}, "core": {"precision": 55.4, "recall": 62.0, "f1": 58.5, "f2": 60.6, "tp": 98, "fp": 79, "fn": 60, "num_prs": 50}, "all": {"precision": 56.4, "recall": 59.0, "f1": 57.6, "f2": 58.4, "tp": 102, "fp": 79, "fn": 71, "num_prs": 50}}, "coderabbit": {"strict": {"precision": 31.0, "recall": 62.6, "f1": 41.4, "f2": 52.0, "tp": 87, "fp": 194, "fn": 52, "num_prs": 50}, "core": {"precision": 32.6, "recall": 59.5, "f1": 42.2, "f2": 51.1, "tp": 94, "fp": 194, "fn": 64, "num_prs": 50}, "all": {"precision": 34.7, "recall": 59.5, "f1": 43.8, "f2": 52.1, "tp": 103, "fp": 194, "fn": 70, "num_prs": 50}}, "codeant-v2": {"strict": {"precision": 39.0, "recall": 44.6, "f1": 41.6, "f2": 43.4, "tp": 62, "fp": 97, "fn": 77, "num_prs": 50}, "core": {"precision": 40.1, "recall": 41.1, "f1": 40.6, "f2": 40.9, "tp": 65, "fp": 97, "fn": 93, "num_prs": 50}, "all": {"precision": 41.2, "recall": 39.3, "f1": 40.2, "f2": 39.7, "tp": 68, "fp": 97, "fn": 105, "num_prs": 50}}, "shipwright": {"strict": {"precision": 49.6, "recall": 48.2, "f1": 48.9, "f2": 48.5, "tp": 67, "fp": 68, "fn": 72, "num_prs": 50}, "core": {"precision": 52.4, "recall": 47.5, "f1": 49.8, "f2": 48.4, "tp": 75, "fp": 68, "fn": 83, "num_prs": 50}, "all": {"precision": 54.4, "recall": 46.8, "f1": 50.3, "f2": 48.2, "tp": 81, "fp": 68, "fn": 92, "num_prs": 50}}, "copilot-v2": {"strict": {"precision": 35.6, "recall": 66.9, "f1": 46.5, "f2": 56.9, "tp": 93, "fp": 168, "fn": 46, "num_prs": 50}, "core": {"precision": 38.5, "recall": 66.5, "f1": 48.7, "f2": 58.0, "tp": 105, "fp": 168, "fn": 53, "num_prs": 50}, "all": {"precision": 40.6, "recall": 66.5, "f1": 50.4, "f2": 59.0, "tp": 115, "fp": 168, "fn": 58, "num_prs": 50}}}}, "anthropic_claude-sonnet-4-5-20250929": {"prs": [{"url": "https://github.com/keycloak/keycloak/pull/37429", "language": "Java", "pr_size": "medium", "domain": "UI", "change_type": "feature", "complexity": "moderate", "difficulty": "moderate", "risk": "medium", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"augment": {"strict": {"tp": 0, "fp": 3, "fn": 1}, "core": {"tp": 2, "fp": 3, "fn": 1}, "all": {"tp": 2, "fp": 3, "fn": 3}}, "baz": {"strict": {"tp": 0, "fp": 1, "fn": 1}, "core": {"tp": 0, "fp": 1, "fn": 3}, "all": {"tp": 0, "fp": 1, "fn": 5}}, "bugbot": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 2, "fp": 0, "fn": 1}, "all": {"tp": 2, "fp": 0, "fn": 3}}, "claude": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 1, "fp": 0, "fn": 4}}, "claude-code": {"strict": {"tp": 0, "fp": 4, "fn": 1}, "core": {"tp": 0, "fp": 4, "fn": 3}, "all": {"tp": 1, "fp": 4, "fn": 4}}, "codeant-v2": {"strict": {"tp": 0, "fp": 1, "fn": 1}, "core": {"tp": 0, "fp": 1, "fn": 3}, "all": {"tp": 0, "fp": 1, "fn": 5}}, "coderabbit": {"strict": {"tp": 0, "fp": 5, "fn": 1}, "core": {"tp": 2, "fp": 5, "fn": 1}, "all": {"tp": 4, "fp": 5, "fn": 1}}, "cubic-v2": {"strict": {"tp": 0, "fp": 2, "fn": 1}, "core": {"tp": 1, "fp": 2, "fn": 2}, "all": {"tp": 1, "fp": 2, "fn": 4}}, "devin": {"strict": {"tp": 0, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 1}, "all": {"tp": 2, "fp": 1, "fn": 3}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 5}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 4}}, "kodus-v2": {"strict": {"tp": 0, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 2}, "all": {"tp": 1, "fp": 1, "fn": 4}}, "macroscope": {"strict": {"tp": 0, "fp": 1, "fn": 1}, "core": {"tp": 0, "fp": 1, "fn": 3}, "all": {"tp": 0, "fp": 1, "fn": 5}}, "qodo-extended-v2": {"strict": {"tp": 0, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 1}, "all": {"tp": 2, "fp": 1, "fn": 3}}, "qodo-v2": {"strict": {"tp": 0, "fp": 3, "fn": 1}, "core": {"tp": 0, "fp": 3, "fn": 3}, "all": {"tp": 0, "fp": 3, "fn": 5}}, "sourcery": {"strict": {"tp": 0, "fp": 5, "fn": 1}, "core": {"tp": 0, "fp": 5, "fn": 3}, "all": {"tp": 1, "fp": 5, "fn": 4}}, "greptile-v4-1": {"strict": {"tp": 0, "fp": 3, "fn": 1}, "core": {"tp": 0, "fp": 3, "fn": 3}, "all": {"tp": 2, "fp": 3, "fn": 3}}, "gemini-v2": {"strict": {"tp": 0, "fp": 5, "fn": 1}, "core": {"tp": 1, "fp": 5, "fn": 2}, "all": {"tp": 2, "fp": 5, "fn": 3}}, "gitlab": {"strict": {"tp": 0, "fp": 2, "fn": 1}, "core": {"tp": 1, "fp": 2, "fn": 2}, "all": {"tp": 2, "fp": 2, "fn": 3}}, "copilot-v2": {"strict": {"tp": 0, "fp": 5, "fn": 1}, "core": {"tp": 2, "fp": 5, "fn": 1}, "all": {"tp": 3, "fp": 5, "fn": 2}}}}, {"url": "https://github.com/keycloak/keycloak/pull/37634", "language": "Java", "pr_size": "medium", "domain": "authentication", "change_type": "feature", "complexity": "moderate", "difficulty": "subtle", "risk": "high", "context": "file", "concern": "correctness", "summary": "", "tool_metrics": {"augment": {"strict": {"tp": 2, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 2}, "all": {"tp": 2, "fp": 1, "fn": 2}}, "baz": {"strict": {"tp": 2, "fp": 0, "fn": 1}, "core": {"tp": 2, "fp": 0, "fn": 2}, "all": {"tp": 2, "fp": 0, "fn": 2}}, "bugbot": {"strict": {"tp": 2, "fp": 2, "fn": 1}, "core": {"tp": 2, "fp": 2, "fn": 2}, "all": {"tp": 2, "fp": 2, "fn": 2}}, "claude": {"strict": {"tp": 1, "fp": 3, "fn": 2}, "core": {"tp": 2, "fp": 3, "fn": 2}, "all": {"tp": 2, "fp": 3, "fn": 2}}, "claude-code": {"strict": {"tp": 0, "fp": 2, "fn": 3}, "core": {"tp": 0, "fp": 2, "fn": 4}, "all": {"tp": 0, "fp": 2, "fn": 4}}, "codeant-v2": {"strict": {"tp": 1, "fp": 2, "fn": 2}, "core": {"tp": 1, "fp": 2, "fn": 3}, "all": {"tp": 1, "fp": 2, "fn": 3}}, "coderabbit": {"strict": {"tp": 3, "fp": 6, "fn": 0}, "core": {"tp": 4, "fp": 6, "fn": 0}, "all": {"tp": 4, "fp": 6, "fn": 0}}, "cubic-v2": {"strict": {"tp": 2, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 2}, "all": {"tp": 2, "fp": 1, "fn": 2}}, "devin": {"strict": {"tp": 2, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 2}, "all": {"tp": 2, "fp": 1, "fn": 2}}, "graphite": {"strict": {"tp": 2, "fp": 0, "fn": 1}, "core": {"tp": 2, "fp": 0, "fn": 2}, "all": {"tp": 2, "fp": 0, "fn": 2}}, "kg": {"strict": {"tp": 1, "fp": 2, "fn": 2}, "core": {"tp": 1, "fp": 2, "fn": 3}, "all": {"tp": 1, "fp": 2, "fn": 3}}, "kodus-v2": {"strict": {"tp": 2, "fp": 0, "fn": 1}, "core": {"tp": 2, "fp": 0, "fn": 2}, "all": {"tp": 2, "fp": 0, "fn": 2}}, "macroscope": {"strict": {"tp": 2, "fp": 2, "fn": 1}, "core": {"tp": 2, "fp": 2, "fn": 2}, "all": {"tp": 2, "fp": 2, "fn": 2}}, "qodo-extended-v2": {"strict": {"tp": 2, "fp": 1, "fn": 1}, "core": {"tp": 3, "fp": 1, "fn": 1}, "all": {"tp": 3, "fp": 1, "fn": 1}}, "qodo-v2": {"strict": {"tp": 2, "fp": 0, "fn": 1}, "core": {"tp": 2, "fp": 0, "fn": 2}, "all": {"tp": 2, "fp": 0, "fn": 2}}, "sourcery": {"strict": {"tp": 2, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 2}, "all": {"tp": 2, "fp": 1, "fn": 2}}, "greptile-v4-1": {"strict": {"tp": 2, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 2}, "all": {"tp": 2, "fp": 1, "fn": 2}}, "gemini-v2": {"strict": {"tp": 2, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 2}, "all": {"tp": 2, "fp": 1, "fn": 2}}, "gitlab": {"strict": {"tp": 1, "fp": 1, "fn": 2}, "core": {"tp": 2, "fp": 1, "fn": 2}, "all": {"tp": 2, "fp": 1, "fn": 2}}, "copilot-v2": {"strict": {"tp": 2, "fp": 3, "fn": 1}, "core": {"tp": 3, "fp": 3, "fn": 1}, "all": {"tp": 3, "fp": 3, "fn": 1}}}}, {"url": "https://github.com/keycloak/keycloak/pull/38446", "language": "Java", "pr_size": "medium", "domain": "authentication", "change_type": "feature", "complexity": "moderate", "difficulty": "moderate", "risk": "medium", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"augment": {"strict": {"tp": 1, "fp": 3, "fn": 1}, "core": {"tp": 1, "fp": 3, "fn": 1}, "all": {"tp": 1, "fp": 3, "fn": 1}}, "baz": {"strict": {"tp": 0, "fp": 1, "fn": 2}, "core": {"tp": 0, "fp": 1, "fn": 2}, "all": {"tp": 0, "fp": 1, "fn": 2}}, "bugbot": {"strict": {"tp": 0, "fp": 3, "fn": 2}, "core": {"tp": 0, "fp": 3, "fn": 2}, "all": {"tp": 0, "fp": 3, "fn": 2}}, "claude": {"strict": {"tp": 1, "fp": 3, "fn": 1}, "core": {"tp": 1, "fp": 3, "fn": 1}, "all": {"tp": 1, "fp": 3, "fn": 1}}, "claude-code": {"strict": {"tp": 1, "fp": 5, "fn": 1}, "core": {"tp": 1, "fp": 5, "fn": 1}, "all": {"tp": 1, "fp": 5, "fn": 1}}, "codeant-v2": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "coderabbit": {"strict": {"tp": 2, "fp": 2, "fn": 0}, "core": {"tp": 2, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "cubic-v2": {"strict": {"tp": 1, "fp": 3, "fn": 1}, "core": {"tp": 1, "fp": 3, "fn": 1}, "all": {"tp": 1, "fp": 3, "fn": 1}}, "devin": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "kodus-v2": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "macroscope": {"strict": {"tp": 0, "fp": 3, "fn": 2}, "core": {"tp": 0, "fp": 3, "fn": 2}, "all": {"tp": 0, "fp": 3, "fn": 2}}, "qodo-extended-v2": {"strict": {"tp": 1, "fp": 4, "fn": 1}, "core": {"tp": 1, "fp": 4, "fn": 1}, "all": {"tp": 1, "fp": 4, "fn": 1}}, "qodo-v2": {"strict": {"tp": 2, "fp": 3, "fn": 0}, "core": {"tp": 2, "fp": 3, "fn": 0}, "all": {"tp": 2, "fp": 3, "fn": 0}}, "sourcery": {"strict": {"tp": 1, "fp": 6, "fn": 1}, "core": {"tp": 1, "fp": 6, "fn": 1}, "all": {"tp": 1, "fp": 6, "fn": 1}}, "greptile-v4-1": {"strict": {"tp": 2, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "gemini-v2": {"strict": {"tp": 1, "fp": 3, "fn": 1}, "core": {"tp": 1, "fp": 3, "fn": 1}, "all": {"tp": 1, "fp": 3, "fn": 1}}, "gitlab": {"strict": {"tp": 1, "fp": 5, "fn": 1}, "core": {"tp": 1, "fp": 5, "fn": 1}, "all": {"tp": 1, "fp": 5, "fn": 1}}, "copilot-v2": {"strict": {"tp": 1, "fp": 6, "fn": 1}, "core": {"tp": 1, "fp": 6, "fn": 1}, "all": {"tp": 1, "fp": 6, "fn": 1}}}}, {"url": "https://github.com/keycloak/keycloak/pull/36882", "language": "Java", "pr_size": "small", "domain": "configuration", "change_type": "feature", "complexity": "moderate", "difficulty": "moderate", "risk": "medium", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"augment": {"strict": {"tp": 0, "fp": 1, "fn": 1}, "core": {"tp": 0, "fp": 1, "fn": 2}, "all": {"tp": 0, "fp": 1, "fn": 2}}, "baz": {"strict": {"tp": 0, "fp": 1, "fn": 1}, "core": {"tp": 0, "fp": 1, "fn": 2}, "all": {"tp": 0, "fp": 1, "fn": 2}}, "bugbot": {"strict": {"tp": 0, "fp": 1, "fn": 1}, "core": {"tp": 0, "fp": 1, "fn": 2}, "all": {"tp": 0, "fp": 1, "fn": 2}}, "claude": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "claude-code": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "codeant-v2": {"strict": {"tp": 0, "fp": 1, "fn": 1}, "core": {"tp": 0, "fp": 1, "fn": 2}, "all": {"tp": 0, "fp": 1, "fn": 2}}, "coderabbit": {"strict": {"tp": 0, "fp": 2, "fn": 1}, "core": {"tp": 0, "fp": 2, "fn": 2}, "all": {"tp": 0, "fp": 2, "fn": 2}}, "cubic-v2": {"strict": {"tp": 1, "fp": 2, "fn": 0}, "core": {"tp": 1, "fp": 2, "fn": 1}, "all": {"tp": 1, "fp": 2, "fn": 1}}, "devin": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "kodus-v2": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "macroscope": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "qodo-extended-v2": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "qodo-v2": {"strict": {"tp": 0, "fp": 1, "fn": 1}, "core": {"tp": 0, "fp": 1, "fn": 2}, "all": {"tp": 0, "fp": 1, "fn": 2}}, "sourcery": {"strict": {"tp": 0, "fp": 4, "fn": 1}, "core": {"tp": 0, "fp": 4, "fn": 2}, "all": {"tp": 0, "fp": 4, "fn": 2}}, "greptile-v4-1": {"strict": {"tp": 0, "fp": 2, "fn": 1}, "core": {"tp": 0, "fp": 2, "fn": 2}, "all": {"tp": 0, "fp": 2, "fn": 2}}, "gemini-v2": {"strict": {"tp": 0, "fp": 2, "fn": 1}, "core": {"tp": 1, "fp": 2, "fn": 1}, "all": {"tp": 1, "fp": 2, "fn": 1}}, "gitlab": {"strict": {"tp": 0, "fp": 2, "fn": 1}, "core": {"tp": 0, "fp": 2, "fn": 2}, "all": {"tp": 0, "fp": 2, "fn": 2}}, "copilot-v2": {"strict": {"tp": 0, "fp": 3, "fn": 1}, "core": {"tp": 0, "fp": 3, "fn": 2}, "all": {"tp": 0, "fp": 3, "fn": 2}}}}, {"url": "https://github.com/keycloak/keycloak/pull/36880", "language": "Java", "pr_size": "medium", "domain": "authentication", "change_type": "feature", "complexity": "complex", "difficulty": "very_subtle", "risk": "high", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"augment": {"strict": {"tp": 1, "fp": 1, "fn": 3}, "core": {"tp": 1, "fp": 1, "fn": 3}, "all": {"tp": 1, "fp": 1, "fn": 4}}, "baz": {"strict": {"tp": 2, "fp": 0, "fn": 2}, "core": {"tp": 2, "fp": 0, "fn": 2}, "all": {"tp": 2, "fp": 0, "fn": 3}}, "bugbot": {"strict": {"tp": 2, "fp": 1, "fn": 2}, "core": {"tp": 2, "fp": 1, "fn": 2}, "all": {"tp": 3, "fp": 1, "fn": 2}}, "claude": {"strict": {"tp": 1, "fp": 3, "fn": 3}, "core": {"tp": 1, "fp": 3, "fn": 3}, "all": {"tp": 2, "fp": 3, "fn": 3}}, "claude-code": {"strict": {"tp": 0, "fp": 2, "fn": 4}, "core": {"tp": 0, "fp": 2, "fn": 4}, "all": {"tp": 0, "fp": 2, "fn": 5}}, "codeant-v2": {"strict": {"tp": 2, "fp": 0, "fn": 2}, "core": {"tp": 2, "fp": 0, "fn": 2}, "all": {"tp": 2, "fp": 0, "fn": 3}}, "coderabbit": {"strict": {"tp": 1, "fp": 3, "fn": 3}, "core": {"tp": 1, "fp": 3, "fn": 3}, "all": {"tp": 1, "fp": 3, "fn": 4}}, "cubic-v2": {"strict": {"tp": 1, "fp": 1, "fn": 3}, "core": {"tp": 1, "fp": 1, "fn": 3}, "all": {"tp": 1, "fp": 1, "fn": 4}}, "devin": {"strict": {"tp": 1, "fp": 0, "fn": 3}, "core": {"tp": 1, "fp": 0, "fn": 3}, "all": {"tp": 1, "fp": 0, "fn": 4}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 4}, "core": {"tp": 0, "fp": 0, "fn": 4}, "all": {"tp": 0, "fp": 0, "fn": 5}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 4}, "core": {"tp": 0, "fp": 0, "fn": 4}, "all": {"tp": 0, "fp": 0, "fn": 5}}, "kodus-v2": {"strict": {"tp": 2, "fp": 2, "fn": 2}, "core": {"tp": 2, "fp": 2, "fn": 2}, "all": {"tp": 2, "fp": 2, "fn": 3}}, "macroscope": {"strict": {"tp": 2, "fp": 4, "fn": 2}, "core": {"tp": 2, "fp": 4, "fn": 2}, "all": {"tp": 2, "fp": 4, "fn": 3}}, "qodo-extended-v2": {"strict": {"tp": 2, "fp": 0, "fn": 2}, "core": {"tp": 2, "fp": 0, "fn": 2}, "all": {"tp": 2, "fp": 0, "fn": 3}}, "qodo-v2": {"strict": {"tp": 1, "fp": 2, "fn": 3}, "core": {"tp": 1, "fp": 2, "fn": 3}, "all": {"tp": 1, "fp": 2, "fn": 4}}, "sourcery": {"strict": {"tp": 1, "fp": 4, "fn": 3}, "core": {"tp": 1, "fp": 4, "fn": 3}, "all": {"tp": 2, "fp": 4, "fn": 3}}, "greptile-v4-1": {"strict": {"tp": 1, "fp": 0, "fn": 3}, "core": {"tp": 1, "fp": 0, "fn": 3}, "all": {"tp": 2, "fp": 0, "fn": 3}}, "gemini-v2": {"strict": {"tp": 1, "fp": 3, "fn": 3}, "core": {"tp": 1, "fp": 3, "fn": 3}, "all": {"tp": 2, "fp": 3, "fn": 3}}, "gitlab": {"strict": {"tp": 1, "fp": 6, "fn": 3}, "core": {"tp": 1, "fp": 6, "fn": 3}, "all": {"tp": 2, "fp": 6, "fn": 3}}, "copilot-v2": {"strict": {"tp": 1, "fp": 6, "fn": 3}, "core": {"tp": 1, "fp": 6, "fn": 3}, "all": {"tp": 1, "fp": 6, "fn": 4}}}}, {"url": "https://github.com/keycloak/keycloak/pull/37038", "language": "Java", "pr_size": "small", "domain": "authentication", "change_type": "feature", "complexity": "complex", "difficulty": "very_subtle", "risk": "high", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"augment": {"strict": {"tp": 2, "fp": 3, "fn": 0}, "core": {"tp": 2, "fp": 3, "fn": 0}, "all": {"tp": 2, "fp": 3, "fn": 0}}, "baz": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "bugbot": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "claude": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "claude-code": {"strict": {"tp": 1, "fp": 4, "fn": 1}, "core": {"tp": 1, "fp": 4, "fn": 1}, "all": {"tp": 1, "fp": 4, "fn": 1}}, "codeant-v2": {"strict": {"tp": 1, "fp": 5, "fn": 1}, "core": {"tp": 1, "fp": 5, "fn": 1}, "all": {"tp": 1, "fp": 5, "fn": 1}}, "coderabbit": {"strict": {"tp": 2, "fp": 8, "fn": 0}, "core": {"tp": 2, "fp": 8, "fn": 0}, "all": {"tp": 2, "fp": 8, "fn": 0}}, "cubic-v2": {"strict": {"tp": 2, "fp": 2, "fn": 0}, "core": {"tp": 2, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "devin": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "kodus-v2": {"strict": {"tp": 2, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "macroscope": {"strict": {"tp": 2, "fp": 2, "fn": 0}, "core": {"tp": 2, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "qodo-extended-v2": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "qodo-v2": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "sourcery": {"strict": {"tp": 1, "fp": 3, "fn": 1}, "core": {"tp": 1, "fp": 3, "fn": 1}, "all": {"tp": 1, "fp": 3, "fn": 1}}, "greptile-v4-1": {"strict": {"tp": 2, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "gemini-v2": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "gitlab": {"strict": {"tp": 1, "fp": 2, "fn": 1}, "core": {"tp": 1, "fp": 2, "fn": 1}, "all": {"tp": 1, "fp": 2, "fn": 1}}, "copilot-v2": {"strict": {"tp": 2, "fp": 3, "fn": 0}, "core": {"tp": 2, "fp": 3, "fn": 0}, "all": {"tp": 2, "fp": 3, "fn": 0}}}}, {"url": "https://github.com/keycloak/keycloak/pull/33832", "language": "Java", "pr_size": "medium", "domain": "authentication", "change_type": "feature", "complexity": "moderate", "difficulty": "subtle", "risk": "high", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"augment": {"strict": {"tp": 1, "fp": 2, "fn": 1}, "core": {"tp": 1, "fp": 2, "fn": 2}, "all": {"tp": 1, "fp": 2, "fn": 3}}, "baz": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 2, "fp": 0, "fn": 1}, "all": {"tp": 2, "fp": 0, "fn": 2}}, "bugbot": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 3}}, "claude": {"strict": {"tp": 0, "fp": 5, "fn": 2}, "core": {"tp": 1, "fp": 5, "fn": 2}, "all": {"tp": 1, "fp": 5, "fn": 3}}, "claude-code": {"strict": {"tp": 0, "fp": 1, "fn": 2}, "core": {"tp": 1, "fp": 1, "fn": 2}, "all": {"tp": 2, "fp": 1, "fn": 2}}, "codeant-v2": {"strict": {"tp": 0, "fp": 2, "fn": 2}, "core": {"tp": 0, "fp": 2, "fn": 3}, "all": {"tp": 1, "fp": 2, "fn": 3}}, "coderabbit": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 1}, "all": {"tp": 2, "fp": 1, "fn": 2}}, "cubic-v2": {"strict": {"tp": 1, "fp": 2, "fn": 1}, "core": {"tp": 2, "fp": 2, "fn": 1}, "all": {"tp": 2, "fp": 2, "fn": 2}}, "devin": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 4}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 3}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 4}}, "kodus-v2": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 4}}, "macroscope": {"strict": {"tp": 1, "fp": 5, "fn": 1}, "core": {"tp": 1, "fp": 5, "fn": 2}, "all": {"tp": 1, "fp": 5, "fn": 3}}, "qodo-extended-v2": {"strict": {"tp": 0, "fp": 2, "fn": 2}, "core": {"tp": 1, "fp": 2, "fn": 2}, "all": {"tp": 1, "fp": 2, "fn": 3}}, "qodo-v2": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 2}, "all": {"tp": 1, "fp": 1, "fn": 3}}, "sourcery": {"strict": {"tp": 0, "fp": 2, "fn": 2}, "core": {"tp": 1, "fp": 2, "fn": 2}, "all": {"tp": 1, "fp": 2, "fn": 3}}, "greptile-v4-1": {"strict": {"tp": 0, "fp": 3, "fn": 2}, "core": {"tp": 1, "fp": 3, "fn": 2}, "all": {"tp": 1, "fp": 3, "fn": 3}}, "gemini-v2": {"strict": {"tp": 1, "fp": 2, "fn": 1}, "core": {"tp": 2, "fp": 2, "fn": 1}, "all": {"tp": 2, "fp": 2, "fn": 2}}, "gitlab": {"strict": {"tp": 0, "fp": 2, "fn": 2}, "core": {"tp": 1, "fp": 2, "fn": 2}, "all": {"tp": 1, "fp": 2, "fn": 3}}, "copilot-v2": {"strict": {"tp": 1, "fp": 4, "fn": 1}, "core": {"tp": 2, "fp": 4, "fn": 1}, "all": {"tp": 3, "fp": 4, "fn": 1}}}}, {"url": "https://github.com/keycloak/keycloak/pull/40940", "language": "Java", "pr_size": "small", "domain": "concurrency", "change_type": "bug_fix", "complexity": "moderate", "difficulty": "subtle", "risk": "high", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"augment": {"strict": {"tp": 1, "fp": 2, "fn": 1}, "core": {"tp": 1, "fp": 2, "fn": 1}, "all": {"tp": 1, "fp": 2, "fn": 1}}, "baz": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "bugbot": {"strict": {"tp": 1, "fp": 2, "fn": 1}, "core": {"tp": 1, "fp": 2, "fn": 1}, "all": {"tp": 1, "fp": 2, "fn": 1}}, "claude": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "claude-code": {"strict": {"tp": 1, "fp": 3, "fn": 1}, "core": {"tp": 1, "fp": 3, "fn": 1}, "all": {"tp": 1, "fp": 3, "fn": 1}}, "codeant-v2": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "coderabbit": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "cubic-v2": {"strict": {"tp": 2, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "devin": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "graphite": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "kodus-v2": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "macroscope": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "qodo-extended-v2": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "qodo-v2": {"strict": {"tp": 2, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "sourcery": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "greptile-v4-1": {"strict": {"tp": 1, "fp": 3, "fn": 1}, "core": {"tp": 1, "fp": 3, "fn": 1}, "all": {"tp": 1, "fp": 3, "fn": 1}}, "gemini-v2": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "gitlab": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "copilot-v2": {"strict": {"tp": 2, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}}}, {"url": "https://github.com/ai-code-review-evaluation/keycloak-greptile/pull/1", "language": "Java", "pr_size": "medium", "domain": "authentication", "change_type": "bug_fix", "complexity": "moderate", "difficulty": "subtle", "risk": "medium", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"augment": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "baz": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "bugbot": {"strict": {"tp": 0, "fp": 1, "fn": 1}, "core": {"tp": 0, "fp": 1, "fn": 1}, "all": {"tp": 0, "fp": 1, "fn": 2}}, "claude": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 0}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "claude-code": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 0}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "codeant-v2": {"strict": {"tp": 1, "fp": 2, "fn": 0}, "core": {"tp": 1, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "coderabbit": {"strict": {"tp": 1, "fp": 2, "fn": 0}, "core": {"tp": 1, "fp": 2, "fn": 0}, "all": {"tp": 1, "fp": 2, "fn": 1}}, "cubic-v2": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 1, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "devin": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "kg": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 0}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "kodus-v2": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "macroscope": {"strict": {"tp": 0, "fp": 1, "fn": 1}, "core": {"tp": 0, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "qodo-extended-v2": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 1, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "qodo-v2": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 1, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "sourcery": {"strict": {"tp": 0, "fp": 3, "fn": 1}, "core": {"tp": 0, "fp": 3, "fn": 1}, "all": {"tp": 1, "fp": 3, "fn": 1}}, "greptile-v4-1": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "gemini-v2": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 1, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "gitlab": {"strict": {"tp": 1, "fp": 3, "fn": 0}, "core": {"tp": 1, "fp": 3, "fn": 0}, "all": {"tp": 1, "fp": 3, "fn": 1}}, "copilot-v2": {"strict": {"tp": 1, "fp": 2, "fn": 0}, "core": {"tp": 1, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}}}, {"url": "https://github.com/getsentry/sentry/pull/93824", "language": "Python", "pr_size": "large", "domain": "concurrency", "change_type": "feature", "complexity": "complex", "difficulty": "subtle", "risk": "high", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"augment": {"strict": {"tp": 3, "fp": 1, "fn": 0}, "core": {"tp": 5, "fp": 1, "fn": 0}, "all": {"tp": 5, "fp": 1, "fn": 0}}, "baz": {"strict": {"tp": 0, "fp": 3, "fn": 3}, "core": {"tp": 0, "fp": 3, "fn": 5}, "all": {"tp": 0, "fp": 3, "fn": 5}}, "bugbot": {"strict": {"tp": 1, "fp": 0, "fn": 2}, "core": {"tp": 1, "fp": 0, "fn": 4}, "all": {"tp": 1, "fp": 0, "fn": 4}}, "claude": {"strict": {"tp": 0, "fp": 2, "fn": 3}, "core": {"tp": 0, "fp": 2, "fn": 5}, "all": {"tp": 0, "fp": 2, "fn": 5}}, "claude-code": {"strict": {"tp": 3, "fp": 2, "fn": 0}, "core": {"tp": 3, "fp": 2, "fn": 2}, "all": {"tp": 3, "fp": 2, "fn": 2}}, "codeant-v2": {"strict": {"tp": 0, "fp": 0, "fn": 3}, "core": {"tp": 2, "fp": 0, "fn": 3}, "all": {"tp": 2, "fp": 0, "fn": 3}}, "coderabbit": {"strict": {"tp": 1, "fp": 3, "fn": 2}, "core": {"tp": 1, "fp": 3, "fn": 4}, "all": {"tp": 1, "fp": 3, "fn": 4}}, "cubic-v2": {"strict": {"tp": 3, "fp": 1, "fn": 0}, "core": {"tp": 5, "fp": 1, "fn": 0}, "all": {"tp": 5, "fp": 1, "fn": 0}}, "devin": {"strict": {"tp": 1, "fp": 1, "fn": 2}, "core": {"tp": 1, "fp": 1, "fn": 4}, "all": {"tp": 1, "fp": 1, "fn": 4}}, "graphite": {"strict": {"tp": 1, "fp": 0, "fn": 2}, "core": {"tp": 1, "fp": 0, "fn": 4}, "all": {"tp": 1, "fp": 0, "fn": 4}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 3}, "core": {"tp": 0, "fp": 0, "fn": 5}, "all": {"tp": 0, "fp": 0, "fn": 5}}, "kodus-v2": {"strict": {"tp": 0, "fp": 0, "fn": 3}, "core": {"tp": 0, "fp": 0, "fn": 5}, "all": {"tp": 0, "fp": 0, "fn": 5}}, "macroscope": {"strict": {"tp": 1, "fp": 1, "fn": 2}, "core": {"tp": 3, "fp": 1, "fn": 2}, "all": {"tp": 3, "fp": 1, "fn": 2}}, "qodo-extended-v2": {"strict": {"tp": 2, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 3}, "all": {"tp": 2, "fp": 1, "fn": 3}}, "qodo-v2": {"strict": {"tp": 2, "fp": 2, "fn": 1}, "core": {"tp": 4, "fp": 2, "fn": 1}, "all": {"tp": 4, "fp": 2, "fn": 1}}, "sourcery": {"strict": {"tp": 2, "fp": 2, "fn": 1}, "core": {"tp": 4, "fp": 2, "fn": 1}, "all": {"tp": 4, "fp": 2, "fn": 1}}, "greptile-v4-1": {"strict": {"tp": 1, "fp": 1, "fn": 2}, "core": {"tp": 1, "fp": 1, "fn": 4}, "all": {"tp": 1, "fp": 1, "fn": 4}}, "gemini-v2": {"strict": {"tp": 2, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 3}, "all": {"tp": 2, "fp": 1, "fn": 3}}, "gitlab": {"strict": {"tp": 0, "fp": 2, "fn": 3}, "core": {"tp": 0, "fp": 2, "fn": 5}, "all": {"tp": 0, "fp": 2, "fn": 5}}, "copilot-v2": {"strict": {"tp": 1, "fp": 5, "fn": 2}, "core": {"tp": 3, "fp": 5, "fn": 2}, "all": {"tp": 3, "fp": 5, "fn": 2}}}}, {"url": "https://github.com/ai-code-review-evaluation/sentry-greptile/pull/5", "language": "Python", "pr_size": "large", "domain": "API", "change_type": "feature", "complexity": "moderate", "difficulty": "subtle", "risk": "medium", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"augment": {"strict": {"tp": 3, "fp": 2, "fn": 3}, "core": {"tp": 3, "fp": 2, "fn": 3}, "all": {"tp": 3, "fp": 2, "fn": 3}}, "baz": {"strict": {"tp": 3, "fp": 1, "fn": 3}, "core": {"tp": 3, "fp": 1, "fn": 3}, "all": {"tp": 3, "fp": 1, "fn": 3}}, "bugbot": {"strict": {"tp": 2, "fp": 0, "fn": 4}, "core": {"tp": 2, "fp": 0, "fn": 4}, "all": {"tp": 2, "fp": 0, "fn": 4}}, "claude": {"strict": {"tp": 0, "fp": 0, "fn": 6}, "core": {"tp": 0, "fp": 0, "fn": 6}, "all": {"tp": 0, "fp": 0, "fn": 6}}, "claude-code": {"strict": {"tp": 0, "fp": 2, "fn": 6}, "core": {"tp": 0, "fp": 2, "fn": 6}, "all": {"tp": 0, "fp": 2, "fn": 6}}, "codeant-v2": {"strict": {"tp": 2, "fp": 5, "fn": 4}, "core": {"tp": 2, "fp": 5, "fn": 4}, "all": {"tp": 2, "fp": 5, "fn": 4}}, "coderabbit": {"strict": {"tp": 4, "fp": 13, "fn": 2}, "core": {"tp": 4, "fp": 13, "fn": 2}, "all": {"tp": 4, "fp": 13, "fn": 2}}, "cubic-v2": {"strict": {"tp": 1, "fp": 3, "fn": 5}, "core": {"tp": 1, "fp": 3, "fn": 5}, "all": {"tp": 1, "fp": 3, "fn": 5}}, "devin": {"strict": {"tp": 2, "fp": 1, "fn": 4}, "core": {"tp": 2, "fp": 1, "fn": 4}, "all": {"tp": 2, "fp": 1, "fn": 4}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 6}, "core": {"tp": 0, "fp": 0, "fn": 6}, "all": {"tp": 0, "fp": 0, "fn": 6}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 6}, "core": {"tp": 0, "fp": 0, "fn": 6}, "all": {"tp": 0, "fp": 0, "fn": 6}}, "kodus-v2": {"strict": {"tp": 2, "fp": 1, "fn": 4}, "core": {"tp": 2, "fp": 1, "fn": 4}, "all": {"tp": 2, "fp": 1, "fn": 4}}, "macroscope": {"strict": {"tp": 2, "fp": 3, "fn": 4}, "core": {"tp": 2, "fp": 3, "fn": 4}, "all": {"tp": 2, "fp": 3, "fn": 4}}, "qodo-extended-v2": {"strict": {"tp": 2, "fp": 0, "fn": 4}, "core": {"tp": 2, "fp": 0, "fn": 4}, "all": {"tp": 2, "fp": 0, "fn": 4}}, "qodo-v2": {"strict": {"tp": 2, "fp": 0, "fn": 4}, "core": {"tp": 2, "fp": 0, "fn": 4}, "all": {"tp": 2, "fp": 0, "fn": 4}}, "sourcery": {"strict": {"tp": 2, "fp": 2, "fn": 4}, "core": {"tp": 2, "fp": 2, "fn": 4}, "all": {"tp": 2, "fp": 2, "fn": 4}}, "greptile-v4-1": {"strict": {"tp": 1, "fp": 0, "fn": 5}, "core": {"tp": 1, "fp": 0, "fn": 5}, "all": {"tp": 1, "fp": 0, "fn": 5}}, "gemini-v2": {"strict": {"tp": 2, "fp": 1, "fn": 4}, "core": {"tp": 2, "fp": 1, "fn": 4}, "all": {"tp": 2, "fp": 1, "fn": 4}}, "gitlab": {"strict": {"tp": 2, "fp": 1, "fn": 4}, "core": {"tp": 2, "fp": 1, "fn": 4}, "all": {"tp": 2, "fp": 1, "fn": 4}}, "copilot-v2": {"strict": {"tp": 3, "fp": 5, "fn": 3}, "core": {"tp": 3, "fp": 5, "fn": 3}, "all": {"tp": 3, "fp": 5, "fn": 3}}}}, {"url": "https://github.com/ai-code-review-evaluation/sentry-greptile/pull/1", "language": "Python", "pr_size": "medium", "domain": "API", "change_type": "performance", "complexity": "moderate", "difficulty": "subtle", "risk": "high", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"augment": {"strict": {"tp": 3, "fp": 1, "fn": 1}, "core": {"tp": 3, "fp": 1, "fn": 1}, "all": {"tp": 3, "fp": 1, "fn": 1}}, "baz": {"strict": {"tp": 1, "fp": 1, "fn": 3}, "core": {"tp": 1, "fp": 1, "fn": 3}, "all": {"tp": 1, "fp": 1, "fn": 3}}, "bugbot": {"strict": {"tp": 3, "fp": 1, "fn": 1}, "core": {"tp": 3, "fp": 1, "fn": 1}, "all": {"tp": 3, "fp": 1, "fn": 1}}, "claude": {"strict": {"tp": 2, "fp": 7, "fn": 2}, "core": {"tp": 2, "fp": 7, "fn": 2}, "all": {"tp": 2, "fp": 7, "fn": 2}}, "claude-code": {"strict": {"tp": 2, "fp": 1, "fn": 2}, "core": {"tp": 2, "fp": 1, "fn": 2}, "all": {"tp": 2, "fp": 1, "fn": 2}}, "codeant-v2": {"strict": {"tp": 1, "fp": 0, "fn": 3}, "core": {"tp": 1, "fp": 0, "fn": 3}, "all": {"tp": 1, "fp": 0, "fn": 3}}, "coderabbit": {"strict": {"tp": 3, "fp": 3, "fn": 1}, "core": {"tp": 3, "fp": 3, "fn": 1}, "all": {"tp": 3, "fp": 3, "fn": 1}}, "cubic-v2": {"strict": {"tp": 3, "fp": 1, "fn": 1}, "core": {"tp": 3, "fp": 1, "fn": 1}, "all": {"tp": 3, "fp": 1, "fn": 1}}, "devin": {"strict": {"tp": 3, "fp": 1, "fn": 1}, "core": {"tp": 3, "fp": 1, "fn": 1}, "all": {"tp": 3, "fp": 1, "fn": 1}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 4}, "core": {"tp": 0, "fp": 0, "fn": 4}, "all": {"tp": 0, "fp": 0, "fn": 4}}, "kg": {"strict": {"tp": 1, "fp": 1, "fn": 3}, "core": {"tp": 1, "fp": 1, "fn": 3}, "all": {"tp": 1, "fp": 1, "fn": 3}}, "kodus-v2": {"strict": {"tp": 1, "fp": 2, "fn": 3}, "core": {"tp": 1, "fp": 2, "fn": 3}, "all": {"tp": 1, "fp": 2, "fn": 3}}, "macroscope": {"strict": {"tp": 2, "fp": 0, "fn": 2}, "core": {"tp": 2, "fp": 0, "fn": 2}, "all": {"tp": 2, "fp": 0, "fn": 2}}, "qodo-extended-v2": {"strict": {"tp": 3, "fp": 0, "fn": 1}, "core": {"tp": 3, "fp": 0, "fn": 1}, "all": {"tp": 3, "fp": 0, "fn": 1}}, "qodo-v2": {"strict": {"tp": 1, "fp": 2, "fn": 3}, "core": {"tp": 1, "fp": 2, "fn": 3}, "all": {"tp": 1, "fp": 2, "fn": 3}}, "sourcery": {"strict": {"tp": 2, "fp": 3, "fn": 2}, "core": {"tp": 2, "fp": 3, "fn": 2}, "all": {"tp": 2, "fp": 3, "fn": 2}}, "greptile-v4-1": {"strict": {"tp": 3, "fp": 2, "fn": 1}, "core": {"tp": 3, "fp": 2, "fn": 1}, "all": {"tp": 3, "fp": 2, "fn": 1}}, "gemini-v2": {"strict": {"tp": 2, "fp": 4, "fn": 2}, "core": {"tp": 2, "fp": 4, "fn": 2}, "all": {"tp": 2, "fp": 4, "fn": 2}}, "gitlab": {"strict": {"tp": 3, "fp": 1, "fn": 1}, "core": {"tp": 3, "fp": 1, "fn": 1}, "all": {"tp": 3, "fp": 1, "fn": 1}}, "copilot-v2": {"strict": {"tp": 2, "fp": 1, "fn": 2}, "core": {"tp": 2, "fp": 1, "fn": 2}, "all": {"tp": 2, "fp": 1, "fn": 2}}}}, {"url": "https://github.com/grafana/grafana/pull/97529", "language": "Go", "pr_size": "small", "domain": "concurrency", "change_type": "performance", "complexity": "complex", "difficulty": "very_subtle", "risk": "critical", "context": "cross_file", "concern": "reliability", "summary": "", "tool_metrics": {"augment": {"strict": {"tp": 2, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "baz": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "bugbot": {"strict": {"tp": 1, "fp": 2, "fn": 1}, "core": {"tp": 1, "fp": 2, "fn": 1}, "all": {"tp": 1, "fp": 2, "fn": 1}}, "claude": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "claude-code": {"strict": {"tp": 1, "fp": 4, "fn": 1}, "core": {"tp": 1, "fp": 4, "fn": 1}, "all": {"tp": 1, "fp": 4, "fn": 1}}, "codeant-v2": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "coderabbit": {"strict": {"tp": 1, "fp": 3, "fn": 1}, "core": {"tp": 1, "fp": 3, "fn": 1}, "all": {"tp": 1, "fp": 3, "fn": 1}}, "cubic-v2": {"strict": {"tp": 2, "fp": 2, "fn": 0}, "core": {"tp": 2, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "devin": {"strict": {"tp": 0, "fp": 2, "fn": 2}, "core": {"tp": 0, "fp": 2, "fn": 2}, "all": {"tp": 0, "fp": 2, "fn": 2}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "kg": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "kodus-v2": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "macroscope": {"strict": {"tp": 1, "fp": 2, "fn": 1}, "core": {"tp": 1, "fp": 2, "fn": 1}, "all": {"tp": 1, "fp": 2, "fn": 1}}, "qodo-extended-v2": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "qodo-v2": {"strict": {"tp": 2, "fp": 4, "fn": 0}, "core": {"tp": 2, "fp": 4, "fn": 0}, "all": {"tp": 2, "fp": 4, "fn": 0}}, "sourcery": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "greptile-v4-1": {"strict": {"tp": 2, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "gemini-v2": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "gitlab": {"strict": {"tp": 1, "fp": 2, "fn": 1}, "core": {"tp": 1, "fp": 2, "fn": 1}, "all": {"tp": 1, "fp": 2, "fn": 1}}, "copilot-v2": {"strict": {"tp": 1, "fp": 3, "fn": 1}, "core": {"tp": 1, "fp": 3, "fn": 1}, "all": {"tp": 1, "fp": 3, "fn": 1}}}}, {"url": "https://github.com/getsentry/sentry/pull/80168", "language": "Python", "pr_size": "small", "domain": "data_processing", "change_type": "feature", "complexity": "moderate", "difficulty": "moderate", "risk": "high", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"augment": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 3, "fp": 0, "fn": 0}, "all": {"tp": 3, "fp": 0, "fn": 0}}, "baz": {"strict": {"tp": 0, "fp": 1, "fn": 1}, "core": {"tp": 0, "fp": 1, "fn": 3}, "all": {"tp": 0, "fp": 1, "fn": 3}}, "bugbot": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 2}}, "claude": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 3}}, "claude-code": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 3, "fp": 1, "fn": 0}, "all": {"tp": 3, "fp": 1, "fn": 0}}, "codeant-v2": {"strict": {"tp": 0, "fp": 1, "fn": 1}, "core": {"tp": 0, "fp": 1, "fn": 3}, "all": {"tp": 0, "fp": 1, "fn": 3}}, "coderabbit": {"strict": {"tp": 0, "fp": 2, "fn": 1}, "core": {"tp": 2, "fp": 2, "fn": 1}, "all": {"tp": 2, "fp": 2, "fn": 1}}, "cubic-v2": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 3, "fp": 1, "fn": 0}, "all": {"tp": 3, "fp": 1, "fn": 0}}, "devin": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 1, "fp": 1, "fn": 2}, "all": {"tp": 1, "fp": 1, "fn": 2}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 3}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 3}}, "kodus-v2": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 3}}, "macroscope": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 3}}, "qodo-extended-v2": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 1}, "all": {"tp": 2, "fp": 0, "fn": 1}}, "qodo-v2": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 2}}, "sourcery": {"strict": {"tp": 1, "fp": 2, "fn": 0}, "core": {"tp": 2, "fp": 2, "fn": 1}, "all": {"tp": 2, "fp": 2, "fn": 1}}, "greptile-v4-1": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 1}, "all": {"tp": 2, "fp": 1, "fn": 1}}, "gemini-v2": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 2}}, "gitlab": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 2}}, "copilot-v2": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 1}, "all": {"tp": 2, "fp": 0, "fn": 1}}}}, {"url": "https://github.com/getsentry/sentry/pull/80528", "language": "Python", "pr_size": "small", "domain": "scheduling", "change_type": "refactoring", "complexity": "moderate", "difficulty": "subtle", "risk": "medium", "context": "file", "concern": "correctness", "summary": "", "tool_metrics": {"augment": {"strict": {"tp": 1, "fp": 2, "fn": 0}, "core": {"tp": 1, "fp": 2, "fn": 1}, "all": {"tp": 1, "fp": 2, "fn": 1}}, "baz": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "bugbot": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "claude": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "claude-code": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "codeant-v2": {"strict": {"tp": 0, "fp": 1, "fn": 1}, "core": {"tp": 0, "fp": 1, "fn": 2}, "all": {"tp": 0, "fp": 1, "fn": 2}}, "coderabbit": {"strict": {"tp": 1, "fp": 3, "fn": 0}, "core": {"tp": 1, "fp": 3, "fn": 1}, "all": {"tp": 1, "fp": 3, "fn": 1}}, "cubic-v2": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "devin": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "graphite": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "kg": {"strict": {"tp": 0, "fp": 1, "fn": 1}, "core": {"tp": 0, "fp": 1, "fn": 2}, "all": {"tp": 0, "fp": 1, "fn": 2}}, "kodus-v2": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "macroscope": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "qodo-extended-v2": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "qodo-v2": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "sourcery": {"strict": {"tp": 1, "fp": 2, "fn": 0}, "core": {"tp": 1, "fp": 2, "fn": 1}, "all": {"tp": 1, "fp": 2, "fn": 1}}, "greptile-v4-1": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "gemini-v2": {"strict": {"tp": 1, "fp": 4, "fn": 0}, "core": {"tp": 2, "fp": 4, "fn": 0}, "all": {"tp": 2, "fp": 4, "fn": 0}}, "gitlab": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "copilot-v2": {"strict": {"tp": 1, "fp": 2, "fn": 0}, "core": {"tp": 1, "fp": 2, "fn": 1}, "all": {"tp": 1, "fp": 2, "fn": 1}}}}, {"url": "https://github.com/getsentry/sentry/pull/77754", "language": "Python", "pr_size": "medium", "domain": "serialization", "change_type": "feature", "complexity": "moderate", "difficulty": "moderate", "risk": "medium", "context": "file", "concern": "correctness", "summary": "", "tool_metrics": {"augment": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 3}}, "baz": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 3}}, "bugbot": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 3}}, "claude": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 2, "fp": 0, "fn": 2}}, "claude-code": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 3}}, "codeant-v2": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 4}}, "coderabbit": {"strict": {"tp": 1, "fp": 2, "fn": 1}, "core": {"tp": 1, "fp": 2, "fn": 1}, "all": {"tp": 3, "fp": 2, "fn": 1}}, "cubic-v2": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 2}}, "devin": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 3}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 4}}, "kg": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 3}}, "kodus-v2": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 3}}, "macroscope": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 3}}, "qodo-extended-v2": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 3}}, "qodo-v2": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 3}}, "sourcery": {"strict": {"tp": 1, "fp": 3, "fn": 1}, "core": {"tp": 1, "fp": 3, "fn": 1}, "all": {"tp": 2, "fp": 3, "fn": 2}}, "greptile-v4-1": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 3}}, "gemini-v2": {"strict": {"tp": 1, "fp": 3, "fn": 1}, "core": {"tp": 1, "fp": 3, "fn": 1}, "all": {"tp": 1, "fp": 3, "fn": 3}}, "gitlab": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 2, "fp": 1, "fn": 2}}, "copilot-v2": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 2, "fp": 1, "fn": 2}}}}, {"url": "https://github.com/getsentry/sentry/pull/95633", "language": "Python", "pr_size": "medium", "domain": "concurrency", "change_type": "feature", "complexity": "moderate", "difficulty": "moderate", "risk": "medium", "context": "file", "concern": "correctness", "summary": "", "tool_metrics": {"augment": {"strict": {"tp": 0, "fp": 6, "fn": 0}, "core": {"tp": 0, "fp": 6, "fn": 1}, "all": {"tp": 0, "fp": 6, "fn": 2}}, "baz": {"strict": {"tp": 0, "fp": 1, "fn": 0}, "core": {"tp": 0, "fp": 1, "fn": 1}, "all": {"tp": 0, "fp": 1, "fn": 2}}, "bugbot": {"strict": {"tp": 0, "fp": 2, "fn": 0}, "core": {"tp": 0, "fp": 2, "fn": 1}, "all": {"tp": 0, "fp": 2, "fn": 2}}, "claude": {"strict": {"tp": 0, "fp": 0, "fn": 0}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "claude-code": {"strict": {"tp": 0, "fp": 2, "fn": 0}, "core": {"tp": 0, "fp": 2, "fn": 1}, "all": {"tp": 0, "fp": 2, "fn": 2}}, "codeant-v2": {"strict": {"tp": 0, "fp": 1, "fn": 0}, "core": {"tp": 0, "fp": 1, "fn": 1}, "all": {"tp": 0, "fp": 1, "fn": 2}}, "coderabbit": {"strict": {"tp": 0, "fp": 7, "fn": 0}, "core": {"tp": 0, "fp": 7, "fn": 1}, "all": {"tp": 0, "fp": 7, "fn": 2}}, "cubic-v2": {"strict": {"tp": 0, "fp": 3, "fn": 0}, "core": {"tp": 0, "fp": 3, "fn": 1}, "all": {"tp": 0, "fp": 3, "fn": 2}}, "devin": {"strict": {"tp": 0, "fp": 2, "fn": 0}, "core": {"tp": 0, "fp": 2, "fn": 1}, "all": {"tp": 0, "fp": 2, "fn": 2}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 0}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "kg": {"strict": {"tp": 0, "fp": 2, "fn": 0}, "core": {"tp": 0, "fp": 2, "fn": 1}, "all": {"tp": 0, "fp": 2, "fn": 2}}, "kodus-v2": {"strict": {"tp": 0, "fp": 0, "fn": 0}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "macroscope": {"strict": {"tp": 0, "fp": 2, "fn": 0}, "core": {"tp": 0, "fp": 2, "fn": 1}, "all": {"tp": 0, "fp": 2, "fn": 2}}, "qodo-extended-v2": {"strict": {"tp": 0, "fp": 6, "fn": 0}, "core": {"tp": 0, "fp": 6, "fn": 1}, "all": {"tp": 0, "fp": 6, "fn": 2}}, "qodo-v2": {"strict": {"tp": 0, "fp": 4, "fn": 0}, "core": {"tp": 0, "fp": 4, "fn": 1}, "all": {"tp": 0, "fp": 4, "fn": 2}}, "sourcery": {"strict": {"tp": 0, "fp": 3, "fn": 0}, "core": {"tp": 1, "fp": 3, "fn": 0}, "all": {"tp": 1, "fp": 3, "fn": 1}}, "greptile-v4-1": {"strict": {"tp": 0, "fp": 3, "fn": 0}, "core": {"tp": 0, "fp": 3, "fn": 1}, "all": {"tp": 0, "fp": 3, "fn": 2}}, "gemini-v2": {"strict": {"tp": 0, "fp": 7, "fn": 0}, "core": {"tp": 0, "fp": 7, "fn": 1}, "all": {"tp": 0, "fp": 7, "fn": 2}}, "gitlab": {"strict": {"tp": 0, "fp": 5, "fn": 0}, "core": {"tp": 1, "fp": 5, "fn": 0}, "all": {"tp": 1, "fp": 5, "fn": 1}}, "copilot-v2": {"strict": {"tp": 0, "fp": 3, "fn": 0}, "core": {"tp": 0, "fp": 3, "fn": 1}, "all": {"tp": 0, "fp": 3, "fn": 2}}}}, {"url": "https://github.com/ai-code-review-evaluation/sentry-greptile/pull/2", "language": "Python", "pr_size": "medium", "domain": "data_processing", "change_type": "performance", "complexity": "moderate", "difficulty": "subtle", "risk": "high", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"augment": {"strict": {"tp": 3, "fp": 2, "fn": 1}, "core": {"tp": 3, "fp": 2, "fn": 1}, "all": {"tp": 3, "fp": 2, "fn": 1}}, "baz": {"strict": {"tp": 3, "fp": 1, "fn": 1}, "core": {"tp": 3, "fp": 1, "fn": 1}, "all": {"tp": 3, "fp": 1, "fn": 1}}, "bugbot": {"strict": {"tp": 4, "fp": 0, "fn": 0}, "core": {"tp": 4, "fp": 0, "fn": 0}, "all": {"tp": 4, "fp": 0, "fn": 0}}, "claude": {"strict": {"tp": 1, "fp": 6, "fn": 3}, "core": {"tp": 1, "fp": 6, "fn": 3}, "all": {"tp": 1, "fp": 6, "fn": 3}}, "claude-code": {"strict": {"tp": 3, "fp": 3, "fn": 1}, "core": {"tp": 3, "fp": 3, "fn": 1}, "all": {"tp": 3, "fp": 3, "fn": 1}}, "codeant-v2": {"strict": {"tp": 3, "fp": 1, "fn": 1}, "core": {"tp": 3, "fp": 1, "fn": 1}, "all": {"tp": 3, "fp": 1, "fn": 1}}, "coderabbit": {"strict": {"tp": 0, "fp": 0, "fn": 4}, "core": {"tp": 0, "fp": 0, "fn": 4}, "all": {"tp": 0, "fp": 0, "fn": 4}}, "cubic-v2": {"strict": {"tp": 3, "fp": 0, "fn": 1}, "core": {"tp": 3, "fp": 0, "fn": 1}, "all": {"tp": 3, "fp": 0, "fn": 1}}, "devin": {"strict": {"tp": 0, "fp": 0, "fn": 4}, "core": {"tp": 0, "fp": 0, "fn": 4}, "all": {"tp": 0, "fp": 0, "fn": 4}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 4}, "core": {"tp": 0, "fp": 0, "fn": 4}, "all": {"tp": 0, "fp": 0, "fn": 4}}, "kg": {"strict": {"tp": 1, "fp": 2, "fn": 3}, "core": {"tp": 1, "fp": 2, "fn": 3}, "all": {"tp": 1, "fp": 2, "fn": 3}}, "kodus-v2": {"strict": {"tp": 2, "fp": 3, "fn": 2}, "core": {"tp": 2, "fp": 3, "fn": 2}, "all": {"tp": 2, "fp": 3, "fn": 2}}, "macroscope": {"strict": {"tp": 3, "fp": 0, "fn": 1}, "core": {"tp": 3, "fp": 0, "fn": 1}, "all": {"tp": 3, "fp": 0, "fn": 1}}, "qodo-extended-v2": {"strict": {"tp": 2, "fp": 4, "fn": 2}, "core": {"tp": 2, "fp": 4, "fn": 2}, "all": {"tp": 2, "fp": 4, "fn": 2}}, "qodo-v2": {"strict": {"tp": 3, "fp": 1, "fn": 1}, "core": {"tp": 3, "fp": 1, "fn": 1}, "all": {"tp": 3, "fp": 1, "fn": 1}}, "sourcery": {"strict": {"tp": 3, "fp": 5, "fn": 1}, "core": {"tp": 3, "fp": 5, "fn": 1}, "all": {"tp": 3, "fp": 5, "fn": 1}}, "greptile-v4-1": {"strict": {"tp": 2, "fp": 3, "fn": 2}, "core": {"tp": 2, "fp": 3, "fn": 2}, "all": {"tp": 2, "fp": 3, "fn": 2}}, "gemini-v2": {"strict": {"tp": 3, "fp": 4, "fn": 1}, "core": {"tp": 3, "fp": 4, "fn": 1}, "all": {"tp": 3, "fp": 4, "fn": 1}}, "gitlab": {"strict": {"tp": 4, "fp": 1, "fn": 0}, "core": {"tp": 4, "fp": 1, "fn": 0}, "all": {"tp": 4, "fp": 1, "fn": 0}}, "copilot-v2": {"strict": {"tp": 3, "fp": 10, "fn": 1}, "core": {"tp": 3, "fp": 10, "fn": 1}, "all": {"tp": 3, "fp": 10, "fn": 1}}}}, {"url": "https://github.com/ai-code-review-evaluation/sentry-greptile/pull/3", "language": "Python", "pr_size": "medium", "domain": "caching", "change_type": "feature", "complexity": "complex", "difficulty": "subtle", "risk": "medium", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"augment": {"strict": {"tp": 2, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 1}, "all": {"tp": 2, "fp": 1, "fn": 1}}, "baz": {"strict": {"tp": 0, "fp": 3, "fn": 3}, "core": {"tp": 0, "fp": 3, "fn": 3}, "all": {"tp": 0, "fp": 3, "fn": 3}}, "bugbot": {"strict": {"tp": 0, "fp": 3, "fn": 3}, "core": {"tp": 0, "fp": 3, "fn": 3}, "all": {"tp": 0, "fp": 3, "fn": 3}}, "claude": {"strict": {"tp": 1, "fp": 0, "fn": 2}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 2}}, "claude-code": {"strict": {"tp": 1, "fp": 2, "fn": 2}, "core": {"tp": 1, "fp": 2, "fn": 2}, "all": {"tp": 1, "fp": 2, "fn": 2}}, "codeant-v2": {"strict": {"tp": 3, "fp": 2, "fn": 0}, "core": {"tp": 3, "fp": 2, "fn": 0}, "all": {"tp": 3, "fp": 2, "fn": 0}}, "coderabbit": {"strict": {"tp": 2, "fp": 5, "fn": 1}, "core": {"tp": 2, "fp": 5, "fn": 1}, "all": {"tp": 2, "fp": 5, "fn": 1}}, "cubic-v2": {"strict": {"tp": 2, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 1}, "all": {"tp": 2, "fp": 1, "fn": 1}}, "devin": {"strict": {"tp": 2, "fp": 0, "fn": 1}, "core": {"tp": 2, "fp": 0, "fn": 1}, "all": {"tp": 2, "fp": 0, "fn": 1}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 3}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 3}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 3}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 3}}, "kodus-v2": {"strict": {"tp": 2, "fp": 2, "fn": 1}, "core": {"tp": 2, "fp": 2, "fn": 1}, "all": {"tp": 2, "fp": 2, "fn": 1}}, "macroscope": {"strict": {"tp": 2, "fp": 0, "fn": 1}, "core": {"tp": 2, "fp": 0, "fn": 1}, "all": {"tp": 2, "fp": 0, "fn": 1}}, "qodo-extended-v2": {"strict": {"tp": 3, "fp": 0, "fn": 0}, "core": {"tp": 3, "fp": 0, "fn": 0}, "all": {"tp": 3, "fp": 0, "fn": 0}}, "qodo-v2": {"strict": {"tp": 2, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 1}, "all": {"tp": 2, "fp": 1, "fn": 1}}, "sourcery": {"strict": {"tp": 2, "fp": 5, "fn": 1}, "core": {"tp": 2, "fp": 5, "fn": 1}, "all": {"tp": 2, "fp": 5, "fn": 1}}, "greptile-v4-1": {"strict": {"tp": 2, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 1}, "all": {"tp": 2, "fp": 1, "fn": 1}}, "gemini-v2": {"strict": {"tp": 2, "fp": 2, "fn": 1}, "core": {"tp": 2, "fp": 2, "fn": 1}, "all": {"tp": 2, "fp": 2, "fn": 1}}, "gitlab": {"strict": {"tp": 2, "fp": 3, "fn": 1}, "core": {"tp": 2, "fp": 3, "fn": 1}, "all": {"tp": 2, "fp": 3, "fn": 1}}, "copilot-v2": {"strict": {"tp": 1, "fp": 7, "fn": 2}, "core": {"tp": 1, "fp": 7, "fn": 2}, "all": {"tp": 1, "fp": 7, "fn": 2}}}}, {"url": "https://github.com/grafana/grafana/pull/103633", "language": "Go", "pr_size": "small", "domain": "caching", "change_type": "performance", "complexity": "moderate", "difficulty": "subtle", "risk": "high", "context": "cross_file", "concern": "security", "summary": "", "tool_metrics": {"augment": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 2, "fp": 0, "fn": 1}, "all": {"tp": 2, "fp": 0, "fn": 1}}, "baz": {"strict": {"tp": 1, "fp": 2, "fn": 1}, "core": {"tp": 1, "fp": 2, "fn": 2}, "all": {"tp": 1, "fp": 2, "fn": 2}}, "bugbot": {"strict": {"tp": 0, "fp": 1, "fn": 2}, "core": {"tp": 0, "fp": 1, "fn": 3}, "all": {"tp": 0, "fp": 1, "fn": 3}}, "claude": {"strict": {"tp": 0, "fp": 5, "fn": 2}, "core": {"tp": 0, "fp": 5, "fn": 3}, "all": {"tp": 0, "fp": 5, "fn": 3}}, "claude-code": {"strict": {"tp": 0, "fp": 2, "fn": 2}, "core": {"tp": 0, "fp": 2, "fn": 3}, "all": {"tp": 0, "fp": 2, "fn": 3}}, "codeant-v2": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 2}}, "coderabbit": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 2}}, "cubic-v2": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 1}, "all": {"tp": 2, "fp": 1, "fn": 1}}, "devin": {"strict": {"tp": 1, "fp": 2, "fn": 1}, "core": {"tp": 1, "fp": 2, "fn": 2}, "all": {"tp": 1, "fp": 2, "fn": 2}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 3}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 3}}, "kodus-v2": {"strict": {"tp": 0, "fp": 1, "fn": 2}, "core": {"tp": 0, "fp": 1, "fn": 3}, "all": {"tp": 0, "fp": 1, "fn": 3}}, "macroscope": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 2}}, "qodo-extended-v2": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 2, "fp": 0, "fn": 1}, "all": {"tp": 2, "fp": 0, "fn": 1}}, "qodo-v2": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 1}, "all": {"tp": 2, "fp": 1, "fn": 1}}, "sourcery": {"strict": {"tp": 0, "fp": 2, "fn": 2}, "core": {"tp": 0, "fp": 2, "fn": 3}, "all": {"tp": 0, "fp": 2, "fn": 3}}, "greptile-v4-1": {"strict": {"tp": 1, "fp": 2, "fn": 1}, "core": {"tp": 1, "fp": 2, "fn": 2}, "all": {"tp": 1, "fp": 2, "fn": 2}}, "gemini-v2": {"strict": {"tp": 0, "fp": 3, "fn": 2}, "core": {"tp": 0, "fp": 3, "fn": 3}, "all": {"tp": 0, "fp": 3, "fn": 3}}, "gitlab": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 2}, "all": {"tp": 1, "fp": 1, "fn": 2}}, "copilot-v2": {"strict": {"tp": 0, "fp": 1, "fn": 2}, "core": {"tp": 1, "fp": 1, "fn": 2}, "all": {"tp": 1, "fp": 1, "fn": 2}}}}, {"url": "https://github.com/getsentry/sentry/pull/67876", "language": "Python", "pr_size": "small", "domain": "authentication", "change_type": "security_patch", "complexity": "moderate", "difficulty": "subtle", "risk": "high", "context": "cross_file", "concern": "security", "summary": "", "tool_metrics": {"augment": {"strict": {"tp": 1, "fp": 2, "fn": 2}, "core": {"tp": 1, "fp": 2, "fn": 2}, "all": {"tp": 1, "fp": 2, "fn": 2}}, "baz": {"strict": {"tp": 2, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 1}, "all": {"tp": 2, "fp": 1, "fn": 1}}, "bugbot": {"strict": {"tp": 1, "fp": 2, "fn": 2}, "core": {"tp": 1, "fp": 2, "fn": 2}, "all": {"tp": 1, "fp": 2, "fn": 2}}, "claude": {"strict": {"tp": 1, "fp": 1, "fn": 2}, "core": {"tp": 1, "fp": 1, "fn": 2}, "all": {"tp": 1, "fp": 1, "fn": 2}}, "claude-code": {"strict": {"tp": 1, "fp": 2, "fn": 2}, "core": {"tp": 1, "fp": 2, "fn": 2}, "all": {"tp": 1, "fp": 2, "fn": 2}}, "codeant-v2": {"strict": {"tp": 0, "fp": 1, "fn": 3}, "core": {"tp": 0, "fp": 1, "fn": 3}, "all": {"tp": 0, "fp": 1, "fn": 3}}, "coderabbit": {"strict": {"tp": 1, "fp": 3, "fn": 2}, "core": {"tp": 1, "fp": 3, "fn": 2}, "all": {"tp": 1, "fp": 3, "fn": 2}}, "cubic-v2": {"strict": {"tp": 3, "fp": 2, "fn": 0}, "core": {"tp": 3, "fp": 2, "fn": 0}, "all": {"tp": 3, "fp": 2, "fn": 0}}, "devin": {"strict": {"tp": 2, "fp": 0, "fn": 1}, "core": {"tp": 2, "fp": 0, "fn": 1}, "all": {"tp": 2, "fp": 0, "fn": 1}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 3}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 3}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 3}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 3}}, "kodus-v2": {"strict": {"tp": 0, "fp": 0, "fn": 3}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 3}}, "macroscope": {"strict": {"tp": 1, "fp": 1, "fn": 2}, "core": {"tp": 1, "fp": 1, "fn": 2}, "all": {"tp": 1, "fp": 1, "fn": 2}}, "qodo-extended-v2": {"strict": {"tp": 2, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 1}, "all": {"tp": 2, "fp": 1, "fn": 1}}, "qodo-v2": {"strict": {"tp": 3, "fp": 5, "fn": 0}, "core": {"tp": 3, "fp": 5, "fn": 0}, "all": {"tp": 3, "fp": 5, "fn": 0}}, "sourcery": {"strict": {"tp": 1, "fp": 8, "fn": 2}, "core": {"tp": 1, "fp": 8, "fn": 2}, "all": {"tp": 1, "fp": 8, "fn": 2}}, "greptile-v4-1": {"strict": {"tp": 1, "fp": 1, "fn": 2}, "core": {"tp": 1, "fp": 1, "fn": 2}, "all": {"tp": 1, "fp": 1, "fn": 2}}, "gemini-v2": {"strict": {"tp": 1, "fp": 1, "fn": 2}, "core": {"tp": 1, "fp": 1, "fn": 2}, "all": {"tp": 1, "fp": 1, "fn": 2}}, "gitlab": {"strict": {"tp": 1, "fp": 1, "fn": 2}, "core": {"tp": 1, "fp": 1, "fn": 2}, "all": {"tp": 1, "fp": 1, "fn": 2}}, "copilot-v2": {"strict": {"tp": 2, "fp": 13, "fn": 1}, "core": {"tp": 2, "fp": 13, "fn": 1}, "all": {"tp": 2, "fp": 13, "fn": 1}}}}, {"url": "https://github.com/keycloak/keycloak/pull/32918", "language": "Java", "pr_size": "small", "domain": "caching", "change_type": "feature", "complexity": "moderate", "difficulty": "subtle", "risk": "high", "context": "file", "concern": "correctness", "summary": "", "tool_metrics": {"augment": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "baz": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "bugbot": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "claude": {"strict": {"tp": 0, "fp": 1, "fn": 2}, "core": {"tp": 0, "fp": 1, "fn": 2}, "all": {"tp": 0, "fp": 1, "fn": 2}}, "claude-code": {"strict": {"tp": 1, "fp": 2, "fn": 1}, "core": {"tp": 1, "fp": 2, "fn": 1}, "all": {"tp": 1, "fp": 2, "fn": 1}}, "codeant-v2": {"strict": {"tp": 1, "fp": 2, "fn": 1}, "core": {"tp": 1, "fp": 2, "fn": 1}, "all": {"tp": 1, "fp": 2, "fn": 1}}, "coderabbit": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "cubic-v2": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "devin": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "graphite": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "kg": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "kodus-v2": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "macroscope": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "qodo-extended-v2": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "qodo-v2": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "sourcery": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "greptile-v4-1": {"strict": {"tp": 0, "fp": 3, "fn": 2}, "core": {"tp": 0, "fp": 3, "fn": 2}, "all": {"tp": 0, "fp": 3, "fn": 2}}, "gemini-v2": {"strict": {"tp": 0, "fp": 2, "fn": 2}, "core": {"tp": 0, "fp": 2, "fn": 2}, "all": {"tp": 0, "fp": 2, "fn": 2}}, "gitlab": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "copilot-v2": {"strict": {"tp": 1, "fp": 3, "fn": 1}, "core": {"tp": 1, "fp": 3, "fn": 1}, "all": {"tp": 1, "fp": 3, "fn": 1}}}}, {"url": "https://github.com/grafana/grafana/pull/94942", "language": "Go", "pr_size": "small", "domain": "data_processing", "change_type": "feature", "complexity": "moderate", "difficulty": "moderate", "risk": "high", "context": "file", "concern": "correctness", "summary": "", "tool_metrics": {"augment": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "baz": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "bugbot": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "claude": {"strict": {"tp": 1, "fp": 2, "fn": 1}, "core": {"tp": 1, "fp": 2, "fn": 1}, "all": {"tp": 1, "fp": 2, "fn": 1}}, "claude-code": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "codeant-v2": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "coderabbit": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "cubic-v2": {"strict": {"tp": 1, "fp": 2, "fn": 1}, "core": {"tp": 1, "fp": 2, "fn": 1}, "all": {"tp": 1, "fp": 2, "fn": 1}}, "devin": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "kg": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "kodus-v2": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "macroscope": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "qodo-extended-v2": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "qodo-v2": {"strict": {"tp": 2, "fp": 2, "fn": 0}, "core": {"tp": 2, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "sourcery": {"strict": {"tp": 2, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "greptile-v4-1": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "gemini-v2": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "gitlab": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "copilot-v2": {"strict": {"tp": 2, "fp": 2, "fn": 0}, "core": {"tp": 2, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}}}, {"url": "https://github.com/grafana/grafana/pull/90939", "language": "Go", "pr_size": "small", "domain": "caching", "change_type": "performance", "complexity": "moderate", "difficulty": "subtle", "risk": "high", "context": "file", "concern": "correctness", "summary": "", "tool_metrics": {"augment": {"strict": {"tp": 2, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "baz": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "bugbot": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "claude": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "claude-code": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "codeant-v2": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "coderabbit": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "cubic-v2": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "devin": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "kg": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "kodus-v2": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "macroscope": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "qodo-extended-v2": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "qodo-v2": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "sourcery": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "greptile-v4-1": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "gemini-v2": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "gitlab": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "copilot-v2": {"strict": {"tp": 1, "fp": 2, "fn": 1}, "core": {"tp": 1, "fp": 2, "fn": 1}, "all": {"tp": 1, "fp": 2, "fn": 1}}}}, {"url": "https://github.com/grafana/grafana/pull/80329", "language": "Go", "pr_size": "small", "domain": "logging", "change_type": "performance", "complexity": "simple", "difficulty": "obvious", "risk": "low", "context": "local", "concern": "maintainability", "summary": "", "tool_metrics": {"augment": {"strict": {"tp": 0, "fp": 2, "fn": 0}, "core": {"tp": 2, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "baz": {"strict": {"tp": 0, "fp": 0, "fn": 0}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "bugbot": {"strict": {"tp": 0, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "claude": {"strict": {"tp": 0, "fp": 4, "fn": 0}, "core": {"tp": 2, "fp": 4, "fn": 0}, "all": {"tp": 2, "fp": 4, "fn": 0}}, "claude-code": {"strict": {"tp": 0, "fp": 5, "fn": 0}, "core": {"tp": 2, "fp": 5, "fn": 0}, "all": {"tp": 2, "fp": 5, "fn": 0}}, "codeant-v2": {"strict": {"tp": 0, "fp": 0, "fn": 0}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "coderabbit": {"strict": {"tp": 0, "fp": 3, "fn": 0}, "core": {"tp": 1, "fp": 3, "fn": 1}, "all": {"tp": 1, "fp": 3, "fn": 1}}, "cubic-v2": {"strict": {"tp": 0, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "devin": {"strict": {"tp": 0, "fp": 2, "fn": 0}, "core": {"tp": 2, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 0}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 0}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "kodus-v2": {"strict": {"tp": 0, "fp": 2, "fn": 0}, "core": {"tp": 2, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "macroscope": {"strict": {"tp": 0, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "qodo-extended-v2": {"strict": {"tp": 0, "fp": 3, "fn": 0}, "core": {"tp": 2, "fp": 3, "fn": 0}, "all": {"tp": 2, "fp": 3, "fn": 0}}, "qodo-v2": {"strict": {"tp": 0, "fp": 3, "fn": 0}, "core": {"tp": 2, "fp": 3, "fn": 0}, "all": {"tp": 2, "fp": 3, "fn": 0}}, "sourcery": {"strict": {"tp": 0, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "greptile-v4-1": {"strict": {"tp": 0, "fp": 2, "fn": 0}, "core": {"tp": 2, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "gemini-v2": {"strict": {"tp": 0, "fp": 2, "fn": 0}, "core": {"tp": 2, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "gitlab": {"strict": {"tp": 0, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "copilot-v2": {"strict": {"tp": 0, "fp": 2, "fn": 0}, "core": {"tp": 2, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}}}, {"url": "https://github.com/grafana/grafana/pull/90045", "language": "Go", "pr_size": "medium", "domain": "logging", "change_type": "feature", "complexity": "moderate", "difficulty": "moderate", "risk": "medium", "context": "file", "concern": "correctness", "summary": "", "tool_metrics": {"augment": {"strict": {"tp": 3, "fp": 5, "fn": 0}, "core": {"tp": 3, "fp": 5, "fn": 0}, "all": {"tp": 3, "fp": 5, "fn": 0}}, "baz": {"strict": {"tp": 2, "fp": 2, "fn": 1}, "core": {"tp": 2, "fp": 2, "fn": 1}, "all": {"tp": 2, "fp": 2, "fn": 1}}, "bugbot": {"strict": {"tp": 2, "fp": 4, "fn": 1}, "core": {"tp": 2, "fp": 4, "fn": 1}, "all": {"tp": 2, "fp": 4, "fn": 1}}, "claude": {"strict": {"tp": 2, "fp": 5, "fn": 1}, "core": {"tp": 2, "fp": 5, "fn": 1}, "all": {"tp": 2, "fp": 5, "fn": 1}}, "claude-code": {"strict": {"tp": 3, "fp": 4, "fn": 0}, "core": {"tp": 3, "fp": 4, "fn": 0}, "all": {"tp": 3, "fp": 4, "fn": 0}}, "codeant-v2": {"strict": {"tp": 2, "fp": 3, "fn": 1}, "core": {"tp": 2, "fp": 3, "fn": 1}, "all": {"tp": 2, "fp": 3, "fn": 1}}, "coderabbit": {"strict": {"tp": 3, "fp": 2, "fn": 0}, "core": {"tp": 3, "fp": 2, "fn": 0}, "all": {"tp": 3, "fp": 2, "fn": 0}}, "cubic-v2": {"strict": {"tp": 3, "fp": 2, "fn": 0}, "core": {"tp": 3, "fp": 2, "fn": 0}, "all": {"tp": 3, "fp": 2, "fn": 0}}, "devin": {"strict": {"tp": 3, "fp": 2, "fn": 0}, "core": {"tp": 3, "fp": 2, "fn": 0}, "all": {"tp": 3, "fp": 2, "fn": 0}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 3}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 3}}, "kg": {"strict": {"tp": 3, "fp": 1, "fn": 0}, "core": {"tp": 3, "fp": 1, "fn": 0}, "all": {"tp": 3, "fp": 1, "fn": 0}}, "kodus-v2": {"strict": {"tp": 1, "fp": 0, "fn": 2}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 2}}, "macroscope": {"strict": {"tp": 2, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 1}, "all": {"tp": 2, "fp": 1, "fn": 1}}, "qodo-extended-v2": {"strict": {"tp": 3, "fp": 6, "fn": 0}, "core": {"tp": 3, "fp": 6, "fn": 0}, "all": {"tp": 3, "fp": 6, "fn": 0}}, "qodo-v2": {"strict": {"tp": 1, "fp": 7, "fn": 2}, "core": {"tp": 1, "fp": 7, "fn": 2}, "all": {"tp": 1, "fp": 7, "fn": 2}}, "sourcery": {"strict": {"tp": 2, "fp": 2, "fn": 1}, "core": {"tp": 2, "fp": 2, "fn": 1}, "all": {"tp": 2, "fp": 2, "fn": 1}}, "greptile-v4-1": {"strict": {"tp": 3, "fp": 3, "fn": 0}, "core": {"tp": 3, "fp": 3, "fn": 0}, "all": {"tp": 3, "fp": 3, "fn": 0}}, "gemini-v2": {"strict": {"tp": 3, "fp": 5, "fn": 0}, "core": {"tp": 3, "fp": 5, "fn": 0}, "all": {"tp": 3, "fp": 5, "fn": 0}}, "gitlab": {"strict": {"tp": 3, "fp": 2, "fn": 0}, "core": {"tp": 3, "fp": 2, "fn": 0}, "all": {"tp": 3, "fp": 2, "fn": 0}}, "copilot-v2": {"strict": {"tp": 3, "fp": 6, "fn": 0}, "core": {"tp": 3, "fp": 6, "fn": 0}, "all": {"tp": 3, "fp": 6, "fn": 0}}}}, {"url": "https://github.com/grafana/grafana/pull/106778", "language": "Go", "pr_size": "medium", "domain": "UI", "change_type": "feature", "complexity": "moderate", "difficulty": "subtle", "risk": "medium", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"augment": {"strict": {"tp": 1, "fp": 2, "fn": 1}, "core": {"tp": 1, "fp": 2, "fn": 1}, "all": {"tp": 1, "fp": 2, "fn": 1}}, "baz": {"strict": {"tp": 0, "fp": 2, "fn": 2}, "core": {"tp": 0, "fp": 2, "fn": 2}, "all": {"tp": 0, "fp": 2, "fn": 2}}, "bugbot": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "claude": {"strict": {"tp": 0, "fp": 12, "fn": 2}, "core": {"tp": 0, "fp": 12, "fn": 2}, "all": {"tp": 0, "fp": 12, "fn": 2}}, "claude-code": {"strict": {"tp": 2, "fp": 2, "fn": 0}, "core": {"tp": 2, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "codeant-v2": {"strict": {"tp": 1, "fp": 2, "fn": 1}, "core": {"tp": 1, "fp": 2, "fn": 1}, "all": {"tp": 1, "fp": 2, "fn": 1}}, "coderabbit": {"strict": {"tp": 1, "fp": 5, "fn": 1}, "core": {"tp": 1, "fp": 5, "fn": 1}, "all": {"tp": 1, "fp": 5, "fn": 1}}, "cubic-v2": {"strict": {"tp": 2, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "devin": {"strict": {"tp": 2, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "kg": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "kodus-v2": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "macroscope": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "qodo-extended-v2": {"strict": {"tp": 2, "fp": 2, "fn": 0}, "core": {"tp": 2, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "qodo-v2": {"strict": {"tp": 1, "fp": 6, "fn": 1}, "core": {"tp": 1, "fp": 6, "fn": 1}, "all": {"tp": 1, "fp": 6, "fn": 1}}, "sourcery": {"strict": {"tp": 0, "fp": 2, "fn": 2}, "core": {"tp": 0, "fp": 2, "fn": 2}, "all": {"tp": 0, "fp": 2, "fn": 2}}, "greptile-v4-1": {"strict": {"tp": 1, "fp": 3, "fn": 1}, "core": {"tp": 1, "fp": 3, "fn": 1}, "all": {"tp": 1, "fp": 3, "fn": 1}}, "gemini-v2": {"strict": {"tp": 0, "fp": 3, "fn": 2}, "core": {"tp": 0, "fp": 3, "fn": 2}, "all": {"tp": 0, "fp": 3, "fn": 2}}, "gitlab": {"strict": {"tp": 2, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "copilot-v2": {"strict": {"tp": 2, "fp": 2, "fn": 0}, "core": {"tp": 2, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}}}, {"url": "https://github.com/grafana/grafana/pull/107534", "language": "Go", "pr_size": "small", "domain": "testing", "change_type": "feature", "complexity": "moderate", "difficulty": "subtle", "risk": "low", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"augment": {"strict": {"tp": 0, "fp": 0, "fn": 0}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 0, "fp": 0, "fn": 1}}, "baz": {"strict": {"tp": 0, "fp": 2, "fn": 0}, "core": {"tp": 0, "fp": 2, "fn": 1}, "all": {"tp": 0, "fp": 2, "fn": 1}}, "bugbot": {"strict": {"tp": 0, "fp": 1, "fn": 0}, "core": {"tp": 0, "fp": 1, "fn": 1}, "all": {"tp": 0, "fp": 1, "fn": 1}}, "claude": {"strict": {"tp": 0, "fp": 0, "fn": 0}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 0, "fp": 0, "fn": 1}}, "claude-code": {"strict": {"tp": 0, "fp": 2, "fn": 0}, "core": {"tp": 0, "fp": 2, "fn": 1}, "all": {"tp": 0, "fp": 2, "fn": 1}}, "codeant-v2": {"strict": {"tp": 0, "fp": 0, "fn": 0}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 0, "fp": 0, "fn": 1}}, "coderabbit": {"strict": {"tp": 0, "fp": 1, "fn": 0}, "core": {"tp": 0, "fp": 1, "fn": 1}, "all": {"tp": 0, "fp": 1, "fn": 1}}, "cubic-v2": {"strict": {"tp": 0, "fp": 2, "fn": 0}, "core": {"tp": 1, "fp": 2, "fn": 0}, "all": {"tp": 1, "fp": 2, "fn": 0}}, "devin": {"strict": {"tp": 0, "fp": 1, "fn": 0}, "core": {"tp": 0, "fp": 1, "fn": 1}, "all": {"tp": 0, "fp": 1, "fn": 1}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 0}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 0, "fp": 0, "fn": 1}}, "kg": {"strict": {"tp": 0, "fp": 2, "fn": 0}, "core": {"tp": 0, "fp": 2, "fn": 1}, "all": {"tp": 0, "fp": 2, "fn": 1}}, "kodus-v2": {"strict": {"tp": 0, "fp": 0, "fn": 0}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 0, "fp": 0, "fn": 1}}, "macroscope": {"strict": {"tp": 0, "fp": 0, "fn": 0}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 0, "fp": 0, "fn": 1}}, "qodo-extended-v2": {"strict": {"tp": 0, "fp": 1, "fn": 0}, "core": {"tp": 0, "fp": 1, "fn": 1}, "all": {"tp": 0, "fp": 1, "fn": 1}}, "qodo-v2": {"strict": {"tp": 0, "fp": 1, "fn": 0}, "core": {"tp": 0, "fp": 1, "fn": 1}, "all": {"tp": 0, "fp": 1, "fn": 1}}, "sourcery": {"strict": {"tp": 0, "fp": 3, "fn": 0}, "core": {"tp": 0, "fp": 3, "fn": 1}, "all": {"tp": 0, "fp": 3, "fn": 1}}, "greptile-v4-1": {"strict": {"tp": 0, "fp": 4, "fn": 0}, "core": {"tp": 0, "fp": 4, "fn": 1}, "all": {"tp": 0, "fp": 4, "fn": 1}}, "gemini-v2": {"strict": {"tp": 0, "fp": 3, "fn": 0}, "core": {"tp": 1, "fp": 3, "fn": 0}, "all": {"tp": 1, "fp": 3, "fn": 0}}, "gitlab": {"strict": {"tp": 0, "fp": 1, "fn": 0}, "core": {"tp": 0, "fp": 1, "fn": 1}, "all": {"tp": 0, "fp": 1, "fn": 1}}, "copilot-v2": {"strict": {"tp": 0, "fp": 3, "fn": 0}, "core": {"tp": 0, "fp": 3, "fn": 1}, "all": {"tp": 0, "fp": 3, "fn": 1}}}}, {"url": "https://github.com/grafana/grafana/pull/79265", "language": "Go", "pr_size": "large", "domain": "authentication", "change_type": "feature", "complexity": "moderate", "difficulty": "subtle", "risk": "high", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"augment": {"strict": {"tp": 1, "fp": 1, "fn": 3}, "core": {"tp": 1, "fp": 1, "fn": 4}, "all": {"tp": 1, "fp": 1, "fn": 4}}, "baz": {"strict": {"tp": 1, "fp": 0, "fn": 3}, "core": {"tp": 2, "fp": 0, "fn": 3}, "all": {"tp": 2, "fp": 0, "fn": 3}}, "bugbot": {"strict": {"tp": 1, "fp": 1, "fn": 3}, "core": {"tp": 1, "fp": 1, "fn": 4}, "all": {"tp": 1, "fp": 1, "fn": 4}}, "claude": {"strict": {"tp": 1, "fp": 2, "fn": 3}, "core": {"tp": 1, "fp": 2, "fn": 4}, "all": {"tp": 1, "fp": 2, "fn": 4}}, "claude-code": {"strict": {"tp": 1, "fp": 2, "fn": 3}, "core": {"tp": 1, "fp": 2, "fn": 4}, "all": {"tp": 1, "fp": 2, "fn": 4}}, "codeant-v2": {"strict": {"tp": 1, "fp": 1, "fn": 3}, "core": {"tp": 1, "fp": 1, "fn": 4}, "all": {"tp": 1, "fp": 1, "fn": 4}}, "coderabbit": {"strict": {"tp": 2, "fp": 4, "fn": 2}, "core": {"tp": 3, "fp": 4, "fn": 2}, "all": {"tp": 3, "fp": 4, "fn": 2}}, "cubic-v2": {"strict": {"tp": 3, "fp": 1, "fn": 1}, "core": {"tp": 4, "fp": 1, "fn": 1}, "all": {"tp": 4, "fp": 1, "fn": 1}}, "devin": {"strict": {"tp": 2, "fp": 0, "fn": 2}, "core": {"tp": 2, "fp": 0, "fn": 3}, "all": {"tp": 2, "fp": 0, "fn": 3}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 4}, "core": {"tp": 0, "fp": 0, "fn": 5}, "all": {"tp": 0, "fp": 0, "fn": 5}}, "kg": {"strict": {"tp": 1, "fp": 0, "fn": 3}, "core": {"tp": 1, "fp": 0, "fn": 4}, "all": {"tp": 1, "fp": 0, "fn": 4}}, "kodus-v2": {"strict": {"tp": 0, "fp": 1, "fn": 4}, "core": {"tp": 1, "fp": 1, "fn": 4}, "all": {"tp": 1, "fp": 1, "fn": 4}}, "macroscope": {"strict": {"tp": 1, "fp": 0, "fn": 3}, "core": {"tp": 1, "fp": 0, "fn": 4}, "all": {"tp": 1, "fp": 0, "fn": 4}}, "qodo-extended-v2": {"strict": {"tp": 2, "fp": 1, "fn": 2}, "core": {"tp": 2, "fp": 1, "fn": 3}, "all": {"tp": 2, "fp": 1, "fn": 3}}, "qodo-v2": {"strict": {"tp": 2, "fp": 2, "fn": 2}, "core": {"tp": 2, "fp": 2, "fn": 3}, "all": {"tp": 2, "fp": 2, "fn": 3}}, "sourcery": {"strict": {"tp": 1, "fp": 4, "fn": 3}, "core": {"tp": 2, "fp": 4, "fn": 3}, "all": {"tp": 2, "fp": 4, "fn": 3}}, "greptile-v4-1": {"strict": {"tp": 2, "fp": 2, "fn": 2}, "core": {"tp": 3, "fp": 2, "fn": 2}, "all": {"tp": 3, "fp": 2, "fn": 2}}, "gemini-v2": {"strict": {"tp": 0, "fp": 4, "fn": 4}, "core": {"tp": 0, "fp": 4, "fn": 5}, "all": {"tp": 0, "fp": 4, "fn": 5}}, "gitlab": {"strict": {"tp": 1, "fp": 1, "fn": 3}, "core": {"tp": 2, "fp": 1, "fn": 3}, "all": {"tp": 2, "fp": 1, "fn": 3}}, "copilot-v2": {"strict": {"tp": 1, "fp": 4, "fn": 3}, "core": {"tp": 1, "fp": 4, "fn": 4}, "all": {"tp": 1, "fp": 4, "fn": 4}}}}, {"url": "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/9", "language": "Ruby", "pr_size": "small", "domain": "configuration", "change_type": "feature", "complexity": "moderate", "difficulty": "subtle", "risk": "medium", "context": "file", "concern": "reliability", "summary": "", "tool_metrics": {"augment": {"strict": {"tp": 0, "fp": 2, "fn": 1}, "core": {"tp": 1, "fp": 2, "fn": 1}, "all": {"tp": 1, "fp": 2, "fn": 1}}, "baz": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "bugbot": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "claude": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "claude-code": {"strict": {"tp": 1, "fp": 5, "fn": 0}, "core": {"tp": 1, "fp": 5, "fn": 1}, "all": {"tp": 1, "fp": 5, "fn": 1}}, "codeant-v2": {"strict": {"tp": 0, "fp": 1, "fn": 1}, "core": {"tp": 0, "fp": 1, "fn": 2}, "all": {"tp": 0, "fp": 1, "fn": 2}}, "coderabbit": {"strict": {"tp": 0, "fp": 3, "fn": 1}, "core": {"tp": 0, "fp": 3, "fn": 2}, "all": {"tp": 0, "fp": 3, "fn": 2}}, "cubic-v2": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "devin": {"strict": {"tp": 0, "fp": 1, "fn": 1}, "core": {"tp": 0, "fp": 1, "fn": 2}, "all": {"tp": 0, "fp": 1, "fn": 2}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "kodus-v2": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "macroscope": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "qodo-extended-v2": {"strict": {"tp": 0, "fp": 1, "fn": 1}, "core": {"tp": 0, "fp": 1, "fn": 2}, "all": {"tp": 0, "fp": 1, "fn": 2}}, "qodo-v2": {"strict": {"tp": 0, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "sourcery": {"strict": {"tp": 0, "fp": 3, "fn": 1}, "core": {"tp": 0, "fp": 3, "fn": 2}, "all": {"tp": 0, "fp": 3, "fn": 2}}, "greptile-v4-1": {"strict": {"tp": 0, "fp": 2, "fn": 1}, "core": {"tp": 0, "fp": 2, "fn": 2}, "all": {"tp": 0, "fp": 2, "fn": 2}}, "gemini-v2": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "gitlab": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "copilot-v2": {"strict": {"tp": 0, "fp": 2, "fn": 1}, "core": {"tp": 0, "fp": 2, "fn": 2}, "all": {"tp": 0, "fp": 2, "fn": 2}}}}, {"url": "https://github.com/grafana/grafana/pull/76186", "language": "Go", "pr_size": "small", "domain": "logging", "change_type": "refactoring", "complexity": "moderate", "difficulty": "subtle", "risk": "high", "context": "cross_file", "concern": "reliability", "summary": "", "tool_metrics": {"augment": {"strict": {"tp": 2, "fp": 0, "fn": 1}, "core": {"tp": 2, "fp": 0, "fn": 1}, "all": {"tp": 2, "fp": 0, "fn": 1}}, "baz": {"strict": {"tp": 0, "fp": 0, "fn": 3}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 3}}, "bugbot": {"strict": {"tp": 1, "fp": 0, "fn": 2}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 2}}, "claude": {"strict": {"tp": 0, "fp": 0, "fn": 3}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 3}}, "claude-code": {"strict": {"tp": 1, "fp": 1, "fn": 2}, "core": {"tp": 1, "fp": 1, "fn": 2}, "all": {"tp": 1, "fp": 1, "fn": 2}}, "codeant-v2": {"strict": {"tp": 0, "fp": 1, "fn": 3}, "core": {"tp": 0, "fp": 1, "fn": 3}, "all": {"tp": 0, "fp": 1, "fn": 3}}, "coderabbit": {"strict": {"tp": 1, "fp": 3, "fn": 2}, "core": {"tp": 1, "fp": 3, "fn": 2}, "all": {"tp": 1, "fp": 3, "fn": 2}}, "cubic-v2": {"strict": {"tp": 3, "fp": 0, "fn": 0}, "core": {"tp": 3, "fp": 0, "fn": 0}, "all": {"tp": 3, "fp": 0, "fn": 0}}, "devin": {"strict": {"tp": 1, "fp": 0, "fn": 2}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 2}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 3}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 3}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 3}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 3}}, "kodus-v2": {"strict": {"tp": 2, "fp": 0, "fn": 1}, "core": {"tp": 2, "fp": 0, "fn": 1}, "all": {"tp": 2, "fp": 0, "fn": 1}}, "macroscope": {"strict": {"tp": 1, "fp": 0, "fn": 2}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 2}}, "qodo-extended-v2": {"strict": {"tp": 1, "fp": 0, "fn": 2}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 2}}, "qodo-v2": {"strict": {"tp": 1, "fp": 1, "fn": 2}, "core": {"tp": 1, "fp": 1, "fn": 2}, "all": {"tp": 1, "fp": 1, "fn": 2}}, "sourcery": {"strict": {"tp": 0, "fp": 3, "fn": 3}, "core": {"tp": 0, "fp": 3, "fn": 3}, "all": {"tp": 0, "fp": 3, "fn": 3}}, "greptile-v4-1": {"strict": {"tp": 1, "fp": 0, "fn": 2}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 2}}, "gemini-v2": {"strict": {"tp": 1, "fp": 2, "fn": 2}, "core": {"tp": 1, "fp": 2, "fn": 2}, "all": {"tp": 1, "fp": 2, "fn": 2}}, "gitlab": {"strict": {"tp": 3, "fp": 0, "fn": 0}, "core": {"tp": 3, "fp": 0, "fn": 0}, "all": {"tp": 3, "fp": 0, "fn": 0}}, "copilot-v2": {"strict": {"tp": 0, "fp": 2, "fn": 3}, "core": {"tp": 0, "fp": 2, "fn": 3}, "all": {"tp": 0, "fp": 2, "fn": 3}}}}, {"url": "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/10", "language": "Ruby", "pr_size": "large", "domain": "configuration", "change_type": "feature", "complexity": "moderate", "difficulty": "subtle", "risk": "high", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"augment": {"strict": {"tp": 3, "fp": 2, "fn": 4}, "core": {"tp": 3, "fp": 2, "fn": 4}, "all": {"tp": 3, "fp": 2, "fn": 4}}, "baz": {"strict": {"tp": 2, "fp": 0, "fn": 5}, "core": {"tp": 2, "fp": 0, "fn": 5}, "all": {"tp": 2, "fp": 0, "fn": 5}}, "bugbot": {"strict": {"tp": 5, "fp": 2, "fn": 2}, "core": {"tp": 5, "fp": 2, "fn": 2}, "all": {"tp": 5, "fp": 2, "fn": 2}}, "claude": {"strict": {"tp": 1, "fp": 2, "fn": 6}, "core": {"tp": 1, "fp": 2, "fn": 6}, "all": {"tp": 1, "fp": 2, "fn": 6}}, "claude-code": {"strict": {"tp": 1, "fp": 5, "fn": 6}, "core": {"tp": 1, "fp": 5, "fn": 6}, "all": {"tp": 1, "fp": 5, "fn": 6}}, "codeant-v2": {"strict": {"tp": 2, "fp": 7, "fn": 5}, "core": {"tp": 2, "fp": 7, "fn": 5}, "all": {"tp": 2, "fp": 7, "fn": 5}}, "coderabbit": {"strict": {"tp": 4, "fp": 10, "fn": 3}, "core": {"tp": 4, "fp": 10, "fn": 3}, "all": {"tp": 4, "fp": 10, "fn": 3}}, "cubic-v2": {"strict": {"tp": 3, "fp": 1, "fn": 4}, "core": {"tp": 3, "fp": 1, "fn": 4}, "all": {"tp": 3, "fp": 1, "fn": 4}}, "devin": {"strict": {"tp": 1, "fp": 1, "fn": 6}, "core": {"tp": 1, "fp": 1, "fn": 6}, "all": {"tp": 1, "fp": 1, "fn": 6}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 7}, "core": {"tp": 0, "fp": 0, "fn": 7}, "all": {"tp": 0, "fp": 0, "fn": 7}}, "kg": {"strict": {"tp": 1, "fp": 0, "fn": 6}, "core": {"tp": 1, "fp": 0, "fn": 6}, "all": {"tp": 1, "fp": 0, "fn": 6}}, "kodus-v2": {"strict": {"tp": 5, "fp": 3, "fn": 2}, "core": {"tp": 5, "fp": 3, "fn": 2}, "all": {"tp": 5, "fp": 3, "fn": 2}}, "macroscope": {"strict": {"tp": 3, "fp": 4, "fn": 4}, "core": {"tp": 3, "fp": 4, "fn": 4}, "all": {"tp": 3, "fp": 4, "fn": 4}}, "qodo-extended-v2": {"strict": {"tp": 5, "fp": 4, "fn": 2}, "core": {"tp": 5, "fp": 4, "fn": 2}, "all": {"tp": 5, "fp": 4, "fn": 2}}, "qodo-v2": {"strict": {"tp": 2, "fp": 5, "fn": 5}, "core": {"tp": 2, "fp": 5, "fn": 5}, "all": {"tp": 2, "fp": 5, "fn": 5}}, "sourcery": {"strict": {"tp": 4, "fp": 7, "fn": 3}, "core": {"tp": 4, "fp": 7, "fn": 3}, "all": {"tp": 4, "fp": 7, "fn": 3}}, "greptile-v4-1": {"strict": {"tp": 2, "fp": 2, "fn": 5}, "core": {"tp": 2, "fp": 2, "fn": 5}, "all": {"tp": 2, "fp": 2, "fn": 5}}, "gemini-v2": {"strict": {"tp": 4, "fp": 2, "fn": 3}, "core": {"tp": 4, "fp": 2, "fn": 3}, "all": {"tp": 4, "fp": 2, "fn": 3}}, "gitlab": {"strict": {"tp": 2, "fp": 3, "fn": 5}, "core": {"tp": 2, "fp": 3, "fn": 5}, "all": {"tp": 2, "fp": 3, "fn": 5}}, "copilot-v2": {"strict": {"tp": 6, "fp": 5, "fn": 1}, "core": {"tp": 6, "fp": 5, "fn": 1}, "all": {"tp": 6, "fp": 5, "fn": 1}}}}, {"url": "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/7", "language": "Ruby", "pr_size": "medium", "domain": "UI", "change_type": "bug_fix", "complexity": "simple", "difficulty": "subtle", "risk": "medium", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"augment": {"strict": {"tp": 3, "fp": 2, "fn": 0}, "core": {"tp": 3, "fp": 2, "fn": 0}, "all": {"tp": 5, "fp": 2, "fn": 0}}, "baz": {"strict": {"tp": 2, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 1}, "all": {"tp": 4, "fp": 1, "fn": 1}}, "bugbot": {"strict": {"tp": 3, "fp": 1, "fn": 0}, "core": {"tp": 3, "fp": 1, "fn": 0}, "all": {"tp": 5, "fp": 1, "fn": 0}}, "claude": {"strict": {"tp": 2, "fp": 2, "fn": 1}, "core": {"tp": 2, "fp": 2, "fn": 1}, "all": {"tp": 4, "fp": 2, "fn": 1}}, "claude-code": {"strict": {"tp": 2, "fp": 2, "fn": 1}, "core": {"tp": 2, "fp": 2, "fn": 1}, "all": {"tp": 4, "fp": 2, "fn": 1}}, "codeant-v2": {"strict": {"tp": 0, "fp": 9, "fn": 3}, "core": {"tp": 0, "fp": 9, "fn": 3}, "all": {"tp": 0, "fp": 9, "fn": 5}}, "coderabbit": {"strict": {"tp": 0, "fp": 2, "fn": 3}, "core": {"tp": 0, "fp": 2, "fn": 3}, "all": {"tp": 0, "fp": 2, "fn": 5}}, "cubic-v2": {"strict": {"tp": 2, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 1}, "all": {"tp": 3, "fp": 1, "fn": 2}}, "devin": {"strict": {"tp": 2, "fp": 2, "fn": 1}, "core": {"tp": 2, "fp": 2, "fn": 1}, "all": {"tp": 4, "fp": 2, "fn": 1}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 3}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 5}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 3}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 5}}, "kodus-v2": {"strict": {"tp": 1, "fp": 0, "fn": 2}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 4}}, "macroscope": {"strict": {"tp": 2, "fp": 0, "fn": 1}, "core": {"tp": 2, "fp": 0, "fn": 1}, "all": {"tp": 4, "fp": 0, "fn": 1}}, "qodo-extended-v2": {"strict": {"tp": 2, "fp": 3, "fn": 1}, "core": {"tp": 2, "fp": 3, "fn": 1}, "all": {"tp": 3, "fp": 3, "fn": 2}}, "qodo-v2": {"strict": {"tp": 1, "fp": 3, "fn": 2}, "core": {"tp": 1, "fp": 3, "fn": 2}, "all": {"tp": 2, "fp": 3, "fn": 3}}, "sourcery": {"strict": {"tp": 2, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 1}, "all": {"tp": 3, "fp": 1, "fn": 2}}, "greptile-v4-1": {"strict": {"tp": 1, "fp": 0, "fn": 2}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 2, "fp": 0, "fn": 3}}, "gemini-v2": {"strict": {"tp": 3, "fp": 1, "fn": 0}, "core": {"tp": 3, "fp": 1, "fn": 0}, "all": {"tp": 5, "fp": 1, "fn": 0}}, "gitlab": {"strict": {"tp": 2, "fp": 0, "fn": 1}, "core": {"tp": 2, "fp": 0, "fn": 1}, "all": {"tp": 3, "fp": 0, "fn": 2}}, "copilot-v2": {"strict": {"tp": 3, "fp": 1, "fn": 0}, "core": {"tp": 3, "fp": 1, "fn": 0}, "all": {"tp": 5, "fp": 1, "fn": 0}}}}, {"url": "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/8", "language": "Ruby", "pr_size": "medium", "domain": "API", "change_type": "bug_fix", "complexity": "moderate", "difficulty": "subtle", "risk": "medium", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"augment": {"strict": {"tp": 3, "fp": 2, "fn": 3}, "core": {"tp": 3, "fp": 2, "fn": 3}, "all": {"tp": 3, "fp": 2, "fn": 3}}, "baz": {"strict": {"tp": 2, "fp": 3, "fn": 4}, "core": {"tp": 2, "fp": 3, "fn": 4}, "all": {"tp": 2, "fp": 3, "fn": 4}}, "bugbot": {"strict": {"tp": 1, "fp": 1, "fn": 5}, "core": {"tp": 1, "fp": 1, "fn": 5}, "all": {"tp": 1, "fp": 1, "fn": 5}}, "claude": {"strict": {"tp": 4, "fp": 3, "fn": 2}, "core": {"tp": 4, "fp": 3, "fn": 2}, "all": {"tp": 4, "fp": 3, "fn": 2}}, "claude-code": {"strict": {"tp": 2, "fp": 1, "fn": 4}, "core": {"tp": 2, "fp": 1, "fn": 4}, "all": {"tp": 2, "fp": 1, "fn": 4}}, "codeant-v2": {"strict": {"tp": 4, "fp": 9, "fn": 2}, "core": {"tp": 4, "fp": 9, "fn": 2}, "all": {"tp": 4, "fp": 9, "fn": 2}}, "coderabbit": {"strict": {"tp": 4, "fp": 6, "fn": 2}, "core": {"tp": 4, "fp": 6, "fn": 2}, "all": {"tp": 4, "fp": 6, "fn": 2}}, "cubic-v2": {"strict": {"tp": 3, "fp": 1, "fn": 3}, "core": {"tp": 3, "fp": 1, "fn": 3}, "all": {"tp": 3, "fp": 1, "fn": 3}}, "devin": {"strict": {"tp": 2, "fp": 0, "fn": 4}, "core": {"tp": 2, "fp": 0, "fn": 4}, "all": {"tp": 2, "fp": 0, "fn": 4}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 6}, "core": {"tp": 0, "fp": 0, "fn": 6}, "all": {"tp": 0, "fp": 0, "fn": 6}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 6}, "core": {"tp": 0, "fp": 0, "fn": 6}, "all": {"tp": 0, "fp": 0, "fn": 6}}, "kodus-v2": {"strict": {"tp": 1, "fp": 6, "fn": 5}, "core": {"tp": 1, "fp": 6, "fn": 5}, "all": {"tp": 1, "fp": 6, "fn": 5}}, "macroscope": {"strict": {"tp": 1, "fp": 3, "fn": 5}, "core": {"tp": 1, "fp": 3, "fn": 5}, "all": {"tp": 1, "fp": 3, "fn": 5}}, "qodo-extended-v2": {"strict": {"tp": 3, "fp": 1, "fn": 3}, "core": {"tp": 3, "fp": 1, "fn": 3}, "all": {"tp": 3, "fp": 1, "fn": 3}}, "qodo-v2": {"strict": {"tp": 2, "fp": 1, "fn": 4}, "core": {"tp": 2, "fp": 1, "fn": 4}, "all": {"tp": 2, "fp": 1, "fn": 4}}, "sourcery": {"strict": {"tp": 3, "fp": 2, "fn": 3}, "core": {"tp": 3, "fp": 2, "fn": 3}, "all": {"tp": 3, "fp": 2, "fn": 3}}, "greptile-v4-1": {"strict": {"tp": 3, "fp": 1, "fn": 3}, "core": {"tp": 3, "fp": 1, "fn": 3}, "all": {"tp": 3, "fp": 1, "fn": 3}}, "gemini-v2": {"strict": {"tp": 2, "fp": 2, "fn": 4}, "core": {"tp": 2, "fp": 2, "fn": 4}, "all": {"tp": 2, "fp": 2, "fn": 4}}, "gitlab": {"strict": {"tp": 3, "fp": 2, "fn": 3}, "core": {"tp": 3, "fp": 2, "fn": 3}, "all": {"tp": 3, "fp": 2, "fn": 3}}, "copilot-v2": {"strict": {"tp": 3, "fp": 14, "fn": 3}, "core": {"tp": 3, "fp": 14, "fn": 3}, "all": {"tp": 3, "fp": 14, "fn": 3}}}}, {"url": "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/3", "language": "Ruby", "pr_size": "medium", "domain": "authentication", "change_type": "feature", "complexity": "moderate", "difficulty": "subtle", "risk": "medium", "context": "file", "concern": "correctness", "summary": "", "tool_metrics": {"augment": {"strict": {"tp": 1, "fp": 4, "fn": 2}, "core": {"tp": 1, "fp": 4, "fn": 2}, "all": {"tp": 1, "fp": 4, "fn": 2}}, "baz": {"strict": {"tp": 1, "fp": 0, "fn": 2}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 2}}, "bugbot": {"strict": {"tp": 1, "fp": 1, "fn": 2}, "core": {"tp": 1, "fp": 1, "fn": 2}, "all": {"tp": 1, "fp": 1, "fn": 2}}, "claude": {"strict": {"tp": 1, "fp": 6, "fn": 2}, "core": {"tp": 1, "fp": 6, "fn": 2}, "all": {"tp": 1, "fp": 6, "fn": 2}}, "claude-code": {"strict": {"tp": 1, "fp": 4, "fn": 2}, "core": {"tp": 1, "fp": 4, "fn": 2}, "all": {"tp": 1, "fp": 4, "fn": 2}}, "codeant-v2": {"strict": {"tp": 1, "fp": 3, "fn": 2}, "core": {"tp": 1, "fp": 3, "fn": 2}, "all": {"tp": 1, "fp": 3, "fn": 2}}, "coderabbit": {"strict": {"tp": 2, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 1}, "all": {"tp": 2, "fp": 1, "fn": 1}}, "cubic-v2": {"strict": {"tp": 2, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 1}, "all": {"tp": 2, "fp": 1, "fn": 1}}, "devin": {"strict": {"tp": 1, "fp": 1, "fn": 2}, "core": {"tp": 1, "fp": 1, "fn": 2}, "all": {"tp": 1, "fp": 1, "fn": 2}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 3}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 3}}, "kg": {"strict": {"tp": 1, "fp": 0, "fn": 2}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 2}}, "kodus-v2": {"strict": {"tp": 1, "fp": 0, "fn": 2}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 2}}, "macroscope": {"strict": {"tp": 2, "fp": 0, "fn": 1}, "core": {"tp": 2, "fp": 0, "fn": 1}, "all": {"tp": 2, "fp": 0, "fn": 1}}, "qodo-extended-v2": {"strict": {"tp": 1, "fp": 1, "fn": 2}, "core": {"tp": 1, "fp": 1, "fn": 2}, "all": {"tp": 1, "fp": 1, "fn": 2}}, "qodo-v2": {"strict": {"tp": 2, "fp": 2, "fn": 1}, "core": {"tp": 2, "fp": 2, "fn": 1}, "all": {"tp": 2, "fp": 2, "fn": 1}}, "sourcery": {"strict": {"tp": 1, "fp": 3, "fn": 2}, "core": {"tp": 1, "fp": 3, "fn": 2}, "all": {"tp": 1, "fp": 3, "fn": 2}}, "greptile-v4-1": {"strict": {"tp": 1, "fp": 3, "fn": 2}, "core": {"tp": 1, "fp": 3, "fn": 2}, "all": {"tp": 1, "fp": 3, "fn": 2}}, "gemini-v2": {"strict": {"tp": 2, "fp": 3, "fn": 1}, "core": {"tp": 2, "fp": 3, "fn": 1}, "all": {"tp": 2, "fp": 3, "fn": 1}}, "gitlab": {"strict": {"tp": 0, "fp": 2, "fn": 3}, "core": {"tp": 0, "fp": 2, "fn": 3}, "all": {"tp": 0, "fp": 2, "fn": 3}}, "copilot-v2": {"strict": {"tp": 2, "fp": 6, "fn": 1}, "core": {"tp": 2, "fp": 6, "fn": 1}, "all": {"tp": 2, "fp": 6, "fn": 1}}}}, {"url": "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/5", "language": "Ruby", "pr_size": "small", "domain": "UI", "change_type": "performance", "complexity": "moderate", "difficulty": "subtle", "risk": "medium", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"augment": {"strict": {"tp": 2, "fp": 0, "fn": 1}, "core": {"tp": 2, "fp": 0, "fn": 1}, "all": {"tp": 2, "fp": 0, "fn": 1}}, "baz": {"strict": {"tp": 1, "fp": 2, "fn": 2}, "core": {"tp": 1, "fp": 2, "fn": 2}, "all": {"tp": 1, "fp": 2, "fn": 2}}, "bugbot": {"strict": {"tp": 0, "fp": 1, "fn": 3}, "core": {"tp": 0, "fp": 1, "fn": 3}, "all": {"tp": 0, "fp": 1, "fn": 3}}, "claude": {"strict": {"tp": 1, "fp": 0, "fn": 2}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 2}}, "claude-code": {"strict": {"tp": 2, "fp": 3, "fn": 1}, "core": {"tp": 2, "fp": 3, "fn": 1}, "all": {"tp": 2, "fp": 3, "fn": 1}}, "codeant-v2": {"strict": {"tp": 0, "fp": 0, "fn": 3}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 3}}, "coderabbit": {"strict": {"tp": 1, "fp": 0, "fn": 2}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 2}}, "cubic-v2": {"strict": {"tp": 2, "fp": 2, "fn": 1}, "core": {"tp": 2, "fp": 2, "fn": 1}, "all": {"tp": 2, "fp": 2, "fn": 1}}, "devin": {"strict": {"tp": 1, "fp": 1, "fn": 2}, "core": {"tp": 1, "fp": 1, "fn": 2}, "all": {"tp": 1, "fp": 1, "fn": 2}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 3}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 3}}, "kg": {"strict": {"tp": 1, "fp": 0, "fn": 2}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 2}}, "kodus-v2": {"strict": {"tp": 0, "fp": 0, "fn": 3}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 3}}, "macroscope": {"strict": {"tp": 0, "fp": 0, "fn": 3}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 3}}, "qodo-extended-v2": {"strict": {"tp": 2, "fp": 0, "fn": 1}, "core": {"tp": 2, "fp": 0, "fn": 1}, "all": {"tp": 2, "fp": 0, "fn": 1}}, "qodo-v2": {"strict": {"tp": 1, "fp": 1, "fn": 2}, "core": {"tp": 1, "fp": 1, "fn": 2}, "all": {"tp": 1, "fp": 1, "fn": 2}}, "sourcery": {"strict": {"tp": 2, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 1}, "all": {"tp": 2, "fp": 1, "fn": 1}}, "greptile-v4-1": {"strict": {"tp": 1, "fp": 2, "fn": 2}, "core": {"tp": 1, "fp": 2, "fn": 2}, "all": {"tp": 1, "fp": 2, "fn": 2}}, "gemini-v2": {"strict": {"tp": 2, "fp": 7, "fn": 1}, "core": {"tp": 2, "fp": 7, "fn": 1}, "all": {"tp": 2, "fp": 7, "fn": 1}}, "gitlab": {"strict": {"tp": 1, "fp": 1, "fn": 2}, "core": {"tp": 1, "fp": 1, "fn": 2}, "all": {"tp": 1, "fp": 1, "fn": 2}}, "copilot-v2": {"strict": {"tp": 2, "fp": 2, "fn": 1}, "core": {"tp": 2, "fp": 2, "fn": 1}, "all": {"tp": 2, "fp": 2, "fn": 1}}}}, {"url": "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/6", "language": "Ruby", "pr_size": "small", "domain": "serialization", "change_type": "feature", "complexity": "simple", "difficulty": "moderate", "risk": "medium", "context": "file", "concern": "correctness", "summary": "", "tool_metrics": {"augment": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 1, "fp": 1, "fn": 0}, "all": {"tp": 1, "fp": 1, "fn": 0}}, "baz": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 1, "fp": 1, "fn": 0}, "all": {"tp": 1, "fp": 1, "fn": 0}}, "bugbot": {"strict": {"tp": 1, "fp": 4, "fn": 0}, "core": {"tp": 1, "fp": 4, "fn": 0}, "all": {"tp": 1, "fp": 4, "fn": 0}}, "claude": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 0, "fp": 0, "fn": 1}}, "claude-code": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 1, "fp": 1, "fn": 0}, "all": {"tp": 1, "fp": 1, "fn": 0}}, "codeant-v2": {"strict": {"tp": 1, "fp": 2, "fn": 0}, "core": {"tp": 1, "fp": 2, "fn": 0}, "all": {"tp": 1, "fp": 2, "fn": 0}}, "coderabbit": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 0}, "all": {"tp": 1, "fp": 0, "fn": 0}}, "cubic-v2": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 1, "fp": 1, "fn": 0}, "all": {"tp": 1, "fp": 1, "fn": 0}}, "devin": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 1, "fp": 1, "fn": 0}, "all": {"tp": 1, "fp": 1, "fn": 0}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 0, "fp": 0, "fn": 1}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 0, "fp": 0, "fn": 1}}, "kodus-v2": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 1, "fp": 1, "fn": 0}, "all": {"tp": 1, "fp": 1, "fn": 0}}, "macroscope": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 0}, "all": {"tp": 1, "fp": 0, "fn": 0}}, "qodo-extended-v2": {"strict": {"tp": 0, "fp": 4, "fn": 1}, "core": {"tp": 0, "fp": 4, "fn": 1}, "all": {"tp": 0, "fp": 4, "fn": 1}}, "qodo-v2": {"strict": {"tp": 1, "fp": 2, "fn": 0}, "core": {"tp": 1, "fp": 2, "fn": 0}, "all": {"tp": 1, "fp": 2, "fn": 0}}, "sourcery": {"strict": {"tp": 1, "fp": 3, "fn": 0}, "core": {"tp": 1, "fp": 3, "fn": 0}, "all": {"tp": 1, "fp": 3, "fn": 0}}, "greptile-v4-1": {"strict": {"tp": 1, "fp": 3, "fn": 0}, "core": {"tp": 1, "fp": 3, "fn": 0}, "all": {"tp": 1, "fp": 3, "fn": 0}}, "gemini-v2": {"strict": {"tp": 1, "fp": 5, "fn": 0}, "core": {"tp": 1, "fp": 5, "fn": 0}, "all": {"tp": 1, "fp": 5, "fn": 0}}, "gitlab": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 1, "fp": 1, "fn": 0}, "all": {"tp": 1, "fp": 1, "fn": 0}}, "copilot-v2": {"strict": {"tp": 1, "fp": 3, "fn": 0}, "core": {"tp": 1, "fp": 3, "fn": 0}, "all": {"tp": 1, "fp": 3, "fn": 0}}}}, {"url": "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/4", "language": "Ruby", "pr_size": "large", "domain": "API", "change_type": "feature", "complexity": "complex", "difficulty": "subtle", "risk": "critical", "context": "cross_file", "concern": "security", "summary": "", "tool_metrics": {"augment": {"strict": {"tp": 4, "fp": 1, "fn": 4}, "core": {"tp": 4, "fp": 1, "fn": 4}, "all": {"tp": 4, "fp": 1, "fn": 4}}, "baz": {"strict": {"tp": 2, "fp": 2, "fn": 6}, "core": {"tp": 2, "fp": 2, "fn": 6}, "all": {"tp": 2, "fp": 2, "fn": 6}}, "bugbot": {"strict": {"tp": 2, "fp": 3, "fn": 6}, "core": {"tp": 2, "fp": 3, "fn": 6}, "all": {"tp": 2, "fp": 3, "fn": 6}}, "claude": {"strict": {"tp": 6, "fp": 2, "fn": 2}, "core": {"tp": 6, "fp": 2, "fn": 2}, "all": {"tp": 6, "fp": 2, "fn": 2}}, "claude-code": {"strict": {"tp": 4, "fp": 0, "fn": 4}, "core": {"tp": 4, "fp": 0, "fn": 4}, "all": {"tp": 4, "fp": 0, "fn": 4}}, "codeant-v2": {"strict": {"tp": 3, "fp": 16, "fn": 5}, "core": {"tp": 3, "fp": 16, "fn": 5}, "all": {"tp": 3, "fp": 16, "fn": 5}}, "coderabbit": {"strict": {"tp": 5, "fp": 11, "fn": 3}, "core": {"tp": 5, "fp": 11, "fn": 3}, "all": {"tp": 5, "fp": 11, "fn": 3}}, "cubic-v2": {"strict": {"tp": 1, "fp": 4, "fn": 7}, "core": {"tp": 1, "fp": 4, "fn": 7}, "all": {"tp": 1, "fp": 4, "fn": 7}}, "devin": {"strict": {"tp": 4, "fp": 1, "fn": 4}, "core": {"tp": 4, "fp": 1, "fn": 4}, "all": {"tp": 4, "fp": 1, "fn": 4}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 8}, "core": {"tp": 0, "fp": 0, "fn": 8}, "all": {"tp": 0, "fp": 0, "fn": 8}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 8}, "core": {"tp": 0, "fp": 0, "fn": 8}, "all": {"tp": 0, "fp": 0, "fn": 8}}, "kodus-v2": {"strict": {"tp": 3, "fp": 2, "fn": 5}, "core": {"tp": 3, "fp": 2, "fn": 5}, "all": {"tp": 3, "fp": 2, "fn": 5}}, "macroscope": {"strict": {"tp": 2, "fp": 5, "fn": 6}, "core": {"tp": 2, "fp": 5, "fn": 6}, "all": {"tp": 2, "fp": 5, "fn": 6}}, "qodo-extended-v2": {"strict": {"tp": 4, "fp": 1, "fn": 4}, "core": {"tp": 4, "fp": 1, "fn": 4}, "all": {"tp": 4, "fp": 1, "fn": 4}}, "qodo-v2": {"strict": {"tp": 6, "fp": 3, "fn": 2}, "core": {"tp": 6, "fp": 3, "fn": 2}, "all": {"tp": 6, "fp": 3, "fn": 2}}, "sourcery": {"strict": {"tp": 3, "fp": 2, "fn": 5}, "core": {"tp": 3, "fp": 2, "fn": 5}, "all": {"tp": 3, "fp": 2, "fn": 5}}, "greptile-v4-1": {"strict": {"tp": 5, "fp": 1, "fn": 3}, "core": {"tp": 5, "fp": 1, "fn": 3}, "all": {"tp": 5, "fp": 1, "fn": 3}}, "gemini-v2": {"strict": {"tp": 4, "fp": 6, "fn": 4}, "core": {"tp": 4, "fp": 6, "fn": 4}, "all": {"tp": 4, "fp": 6, "fn": 4}}, "gitlab": {"strict": {"tp": 4, "fp": 1, "fn": 4}, "core": {"tp": 4, "fp": 1, "fn": 4}, "all": {"tp": 4, "fp": 1, "fn": 4}}, "copilot-v2": {"strict": {"tp": 5, "fp": 5, "fn": 3}, "core": {"tp": 5, "fp": 5, "fn": 3}, "all": {"tp": 5, "fp": 5, "fn": 3}}}}, {"url": "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/1", "language": "Ruby", "pr_size": "medium", "domain": "file_io", "change_type": "feature", "complexity": "moderate", "difficulty": "subtle", "risk": "medium", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"augment": {"strict": {"tp": 2, "fp": 3, "fn": 2}, "core": {"tp": 2, "fp": 3, "fn": 2}, "all": {"tp": 2, "fp": 3, "fn": 2}}, "baz": {"strict": {"tp": 2, "fp": 0, "fn": 2}, "core": {"tp": 2, "fp": 0, "fn": 2}, "all": {"tp": 2, "fp": 0, "fn": 2}}, "bugbot": {"strict": {"tp": 2, "fp": 1, "fn": 2}, "core": {"tp": 2, "fp": 1, "fn": 2}, "all": {"tp": 2, "fp": 1, "fn": 2}}, "claude": {"strict": {"tp": 2, "fp": 0, "fn": 2}, "core": {"tp": 2, "fp": 0, "fn": 2}, "all": {"tp": 2, "fp": 0, "fn": 2}}, "claude-code": {"strict": {"tp": 3, "fp": 1, "fn": 1}, "core": {"tp": 3, "fp": 1, "fn": 1}, "all": {"tp": 3, "fp": 1, "fn": 1}}, "codeant-v2": {"strict": {"tp": 2, "fp": 2, "fn": 2}, "core": {"tp": 2, "fp": 2, "fn": 2}, "all": {"tp": 2, "fp": 2, "fn": 2}}, "coderabbit": {"strict": {"tp": 2, "fp": 3, "fn": 2}, "core": {"tp": 2, "fp": 3, "fn": 2}, "all": {"tp": 2, "fp": 3, "fn": 2}}, "cubic-v2": {"strict": {"tp": 3, "fp": 0, "fn": 1}, "core": {"tp": 3, "fp": 0, "fn": 1}, "all": {"tp": 3, "fp": 0, "fn": 1}}, "devin": {"strict": {"tp": 2, "fp": 0, "fn": 2}, "core": {"tp": 2, "fp": 0, "fn": 2}, "all": {"tp": 2, "fp": 0, "fn": 2}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 4}, "core": {"tp": 0, "fp": 0, "fn": 4}, "all": {"tp": 0, "fp": 0, "fn": 4}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 4}, "core": {"tp": 0, "fp": 0, "fn": 4}, "all": {"tp": 0, "fp": 0, "fn": 4}}, "kodus-v2": {"strict": {"tp": 3, "fp": 3, "fn": 1}, "core": {"tp": 3, "fp": 3, "fn": 1}, "all": {"tp": 3, "fp": 3, "fn": 1}}, "macroscope": {"strict": {"tp": 1, "fp": 1, "fn": 3}, "core": {"tp": 1, "fp": 1, "fn": 3}, "all": {"tp": 1, "fp": 1, "fn": 3}}, "qodo-extended-v2": {"strict": {"tp": 2, "fp": 1, "fn": 2}, "core": {"tp": 2, "fp": 1, "fn": 2}, "all": {"tp": 2, "fp": 1, "fn": 2}}, "qodo-v2": {"strict": {"tp": 2, "fp": 3, "fn": 2}, "core": {"tp": 2, "fp": 3, "fn": 2}, "all": {"tp": 2, "fp": 3, "fn": 2}}, "sourcery": {"strict": {"tp": 2, "fp": 1, "fn": 2}, "core": {"tp": 2, "fp": 1, "fn": 2}, "all": {"tp": 2, "fp": 1, "fn": 2}}, "greptile-v4-1": {"strict": {"tp": 3, "fp": 1, "fn": 1}, "core": {"tp": 3, "fp": 1, "fn": 1}, "all": {"tp": 3, "fp": 1, "fn": 1}}, "gemini-v2": {"strict": {"tp": 3, "fp": 0, "fn": 1}, "core": {"tp": 3, "fp": 0, "fn": 1}, "all": {"tp": 3, "fp": 0, "fn": 1}}, "gitlab": {"strict": {"tp": 2, "fp": 0, "fn": 2}, "core": {"tp": 2, "fp": 0, "fn": 2}, "all": {"tp": 2, "fp": 0, "fn": 2}}, "copilot-v2": {"strict": {"tp": 2, "fp": 1, "fn": 2}, "core": {"tp": 2, "fp": 1, "fn": 2}, "all": {"tp": 2, "fp": 1, "fn": 2}}}}, {"url": "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/2", "language": "Ruby", "pr_size": "medium", "domain": "UI", "change_type": "feature", "complexity": "moderate", "difficulty": "moderate", "risk": "medium", "context": "file", "concern": "correctness", "summary": "", "tool_metrics": {"augment": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 1, "fp": 1, "fn": 0}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "baz": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 0}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "bugbot": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 1, "fp": 1, "fn": 0}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "claude": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 1, "fp": 1, "fn": 0}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "claude-code": {"strict": {"tp": 0, "fp": 2, "fn": 1}, "core": {"tp": 0, "fp": 2, "fn": 1}, "all": {"tp": 0, "fp": 2, "fn": 2}}, "codeant-v2": {"strict": {"tp": 1, "fp": 5, "fn": 0}, "core": {"tp": 1, "fp": 5, "fn": 0}, "all": {"tp": 1, "fp": 5, "fn": 1}}, "coderabbit": {"strict": {"tp": 1, "fp": 6, "fn": 0}, "core": {"tp": 1, "fp": 6, "fn": 0}, "all": {"tp": 1, "fp": 6, "fn": 1}}, "cubic-v2": {"strict": {"tp": 0, "fp": 2, "fn": 1}, "core": {"tp": 0, "fp": 2, "fn": 1}, "all": {"tp": 0, "fp": 2, "fn": 2}}, "devin": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 0}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "kodus-v2": {"strict": {"tp": 1, "fp": 2, "fn": 0}, "core": {"tp": 1, "fp": 2, "fn": 0}, "all": {"tp": 1, "fp": 2, "fn": 1}}, "macroscope": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 0}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "qodo-extended-v2": {"strict": {"tp": 1, "fp": 3, "fn": 0}, "core": {"tp": 1, "fp": 3, "fn": 0}, "all": {"tp": 1, "fp": 3, "fn": 1}}, "qodo-v2": {"strict": {"tp": 1, "fp": 2, "fn": 0}, "core": {"tp": 1, "fp": 2, "fn": 0}, "all": {"tp": 1, "fp": 2, "fn": 1}}, "sourcery": {"strict": {"tp": 1, "fp": 5, "fn": 0}, "core": {"tp": 1, "fp": 5, "fn": 0}, "all": {"tp": 1, "fp": 5, "fn": 1}}, "greptile-v4-1": {"strict": {"tp": 1, "fp": 2, "fn": 0}, "core": {"tp": 1, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "gemini-v2": {"strict": {"tp": 1, "fp": 2, "fn": 0}, "core": {"tp": 1, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "gitlab": {"strict": {"tp": 1, "fp": 2, "fn": 0}, "core": {"tp": 1, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "copilot-v2": {"strict": {"tp": 1, "fp": 5, "fn": 0}, "core": {"tp": 1, "fp": 5, "fn": 0}, "all": {"tp": 1, "fp": 5, "fn": 1}}}}, {"url": "https://github.com/calcom/cal.com/pull/22532", "language": "TypeScript", "pr_size": "medium", "domain": "caching", "change_type": "feature", "complexity": "moderate", "difficulty": "subtle", "risk": "medium", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"augment": {"strict": {"tp": 4, "fp": 2, "fn": 0}, "core": {"tp": 4, "fp": 2, "fn": 0}, "all": {"tp": 4, "fp": 2, "fn": 0}}, "baz": {"strict": {"tp": 1, "fp": 2, "fn": 3}, "core": {"tp": 1, "fp": 2, "fn": 3}, "all": {"tp": 1, "fp": 2, "fn": 3}}, "bugbot": {"strict": {"tp": 0, "fp": 1, "fn": 4}, "core": {"tp": 0, "fp": 1, "fn": 4}, "all": {"tp": 0, "fp": 1, "fn": 4}}, "claude": {"strict": {"tp": 0, "fp": 1, "fn": 4}, "core": {"tp": 0, "fp": 1, "fn": 4}, "all": {"tp": 0, "fp": 1, "fn": 4}}, "claude-code": {"strict": {"tp": 0, "fp": 5, "fn": 4}, "core": {"tp": 0, "fp": 5, "fn": 4}, "all": {"tp": 0, "fp": 5, "fn": 4}}, "codeant-v2": {"strict": {"tp": 3, "fp": 2, "fn": 1}, "core": {"tp": 3, "fp": 2, "fn": 1}, "all": {"tp": 3, "fp": 2, "fn": 1}}, "coderabbit": {"strict": {"tp": 2, "fp": 3, "fn": 2}, "core": {"tp": 2, "fp": 3, "fn": 2}, "all": {"tp": 2, "fp": 3, "fn": 2}}, "cubic-v2": {"strict": {"tp": 0, "fp": 2, "fn": 4}, "core": {"tp": 0, "fp": 2, "fn": 4}, "all": {"tp": 0, "fp": 2, "fn": 4}}, "devin": {"strict": {"tp": 1, "fp": 1, "fn": 3}, "core": {"tp": 1, "fp": 1, "fn": 3}, "all": {"tp": 1, "fp": 1, "fn": 3}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 4}, "core": {"tp": 0, "fp": 0, "fn": 4}, "all": {"tp": 0, "fp": 0, "fn": 4}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 4}, "core": {"tp": 0, "fp": 0, "fn": 4}, "all": {"tp": 0, "fp": 0, "fn": 4}}, "kodus-v2": {"strict": {"tp": 1, "fp": 0, "fn": 3}, "core": {"tp": 1, "fp": 0, "fn": 3}, "all": {"tp": 1, "fp": 0, "fn": 3}}, "macroscope": {"strict": {"tp": 1, "fp": 1, "fn": 3}, "core": {"tp": 1, "fp": 1, "fn": 3}, "all": {"tp": 1, "fp": 1, "fn": 3}}, "qodo-extended-v2": {"strict": {"tp": 2, "fp": 1, "fn": 2}, "core": {"tp": 2, "fp": 1, "fn": 2}, "all": {"tp": 2, "fp": 1, "fn": 2}}, "qodo-v2": {"strict": {"tp": 3, "fp": 2, "fn": 1}, "core": {"tp": 3, "fp": 2, "fn": 1}, "all": {"tp": 3, "fp": 2, "fn": 1}}, "sourcery": {"strict": {"tp": 2, "fp": 2, "fn": 2}, "core": {"tp": 2, "fp": 2, "fn": 2}, "all": {"tp": 2, "fp": 2, "fn": 2}}, "greptile-v4-1": {"strict": {"tp": 3, "fp": 2, "fn": 1}, "core": {"tp": 3, "fp": 2, "fn": 1}, "all": {"tp": 3, "fp": 2, "fn": 1}}, "gemini-v2": {"strict": {"tp": 3, "fp": 1, "fn": 1}, "core": {"tp": 3, "fp": 1, "fn": 1}, "all": {"tp": 3, "fp": 1, "fn": 1}}, "gitlab": {"strict": {"tp": 2, "fp": 3, "fn": 2}, "core": {"tp": 2, "fp": 3, "fn": 2}, "all": {"tp": 2, "fp": 3, "fn": 2}}, "copilot-v2": {"strict": {"tp": 3, "fp": 3, "fn": 1}, "core": {"tp": 3, "fp": 3, "fn": 1}, "all": {"tp": 3, "fp": 3, "fn": 1}}}}, {"url": "https://github.com/calcom/cal.com/pull/8330", "language": "TypeScript", "pr_size": "small", "domain": "scheduling", "change_type": "bug_fix", "complexity": "moderate", "difficulty": "subtle", "risk": "medium", "context": "file", "concern": "correctness", "summary": "", "tool_metrics": {"augment": {"strict": {"tp": 2, "fp": 3, "fn": 0}, "core": {"tp": 2, "fp": 3, "fn": 0}, "all": {"tp": 2, "fp": 3, "fn": 0}}, "baz": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "bugbot": {"strict": {"tp": 2, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "claude": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "claude-code": {"strict": {"tp": 2, "fp": 2, "fn": 0}, "core": {"tp": 2, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "codeant-v2": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "coderabbit": {"strict": {"tp": 2, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "cubic-v2": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "devin": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "graphite": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "kg": {"strict": {"tp": 0, "fp": 3, "fn": 2}, "core": {"tp": 0, "fp": 3, "fn": 2}, "all": {"tp": 0, "fp": 3, "fn": 2}}, "kodus-v2": {"strict": {"tp": 2, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "macroscope": {"strict": {"tp": 2, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "qodo-extended-v2": {"strict": {"tp": 2, "fp": 2, "fn": 0}, "core": {"tp": 2, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "qodo-v2": {"strict": {"tp": 2, "fp": 2, "fn": 0}, "core": {"tp": 2, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "sourcery": {"strict": {"tp": 2, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "greptile-v4-1": {"strict": {"tp": 2, "fp": 2, "fn": 0}, "core": {"tp": 2, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "gemini-v2": {"strict": {"tp": 2, "fp": 3, "fn": 0}, "core": {"tp": 2, "fp": 3, "fn": 0}, "all": {"tp": 2, "fp": 3, "fn": 0}}, "gitlab": {"strict": {"tp": 2, "fp": 2, "fn": 0}, "core": {"tp": 2, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "copilot-v2": {"strict": {"tp": 2, "fp": 4, "fn": 0}, "core": {"tp": 2, "fp": 4, "fn": 0}, "all": {"tp": 2, "fp": 4, "fn": 0}}}}, {"url": "https://github.com/calcom/cal.com/pull/14943", "language": "TypeScript", "pr_size": "small", "domain": "scheduling", "change_type": "feature", "complexity": "moderate", "difficulty": "subtle", "risk": "high", "context": "file", "concern": "correctness", "summary": "", "tool_metrics": {"augment": {"strict": {"tp": 2, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "baz": {"strict": {"tp": 1, "fp": 2, "fn": 1}, "core": {"tp": 1, "fp": 2, "fn": 1}, "all": {"tp": 1, "fp": 2, "fn": 1}}, "bugbot": {"strict": {"tp": 1, "fp": 2, "fn": 1}, "core": {"tp": 1, "fp": 2, "fn": 1}, "all": {"tp": 1, "fp": 2, "fn": 1}}, "claude": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "claude-code": {"strict": {"tp": 0, "fp": 1, "fn": 2}, "core": {"tp": 0, "fp": 1, "fn": 2}, "all": {"tp": 0, "fp": 1, "fn": 2}}, "codeant-v2": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "coderabbit": {"strict": {"tp": 2, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "cubic-v2": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "devin": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "graphite": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "kg": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "kodus-v2": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "macroscope": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "qodo-extended-v2": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "qodo-v2": {"strict": {"tp": 2, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "sourcery": {"strict": {"tp": 2, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "greptile-v4-1": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "gemini-v2": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "gitlab": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "copilot-v2": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}}}, {"url": "https://github.com/calcom/cal.com/pull/22345", "language": "TypeScript", "pr_size": "small", "domain": "database", "change_type": "migration", "complexity": "moderate", "difficulty": "subtle", "risk": "medium", "context": "file", "concern": "correctness", "summary": "", "tool_metrics": {"augment": {"strict": {"tp": 0, "fp": 1, "fn": 1}, "core": {"tp": 0, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "baz": {"strict": {"tp": 0, "fp": 2, "fn": 1}, "core": {"tp": 0, "fp": 2, "fn": 1}, "all": {"tp": 0, "fp": 2, "fn": 2}}, "bugbot": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 1, "fp": 1, "fn": 0}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "claude": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "claude-code": {"strict": {"tp": 1, "fp": 3, "fn": 0}, "core": {"tp": 1, "fp": 3, "fn": 0}, "all": {"tp": 1, "fp": 3, "fn": 1}}, "codeant-v2": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "coderabbit": {"strict": {"tp": 0, "fp": 3, "fn": 1}, "core": {"tp": 0, "fp": 3, "fn": 1}, "all": {"tp": 1, "fp": 3, "fn": 1}}, "cubic-v2": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 1, "fp": 1, "fn": 0}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "devin": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "kg": {"strict": {"tp": 0, "fp": 1, "fn": 1}, "core": {"tp": 0, "fp": 1, "fn": 1}, "all": {"tp": 0, "fp": 1, "fn": 2}}, "kodus-v2": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "macroscope": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "qodo-extended-v2": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "qodo-v2": {"strict": {"tp": 0, "fp": 3, "fn": 1}, "core": {"tp": 0, "fp": 3, "fn": 1}, "all": {"tp": 1, "fp": 3, "fn": 1}}, "sourcery": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 0}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "greptile-v4-1": {"strict": {"tp": 0, "fp": 5, "fn": 1}, "core": {"tp": 0, "fp": 5, "fn": 1}, "all": {"tp": 0, "fp": 5, "fn": 2}}, "gemini-v2": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 0}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "gitlab": {"strict": {"tp": 1, "fp": 2, "fn": 0}, "core": {"tp": 1, "fp": 2, "fn": 0}, "all": {"tp": 1, "fp": 2, "fn": 1}}, "copilot-v2": {"strict": {"tp": 0, "fp": 5, "fn": 1}, "core": {"tp": 0, "fp": 5, "fn": 1}, "all": {"tp": 1, "fp": 5, "fn": 1}}}}, {"url": "https://github.com/calcom/cal.com/pull/11059", "language": "TypeScript", "pr_size": "large", "domain": "authentication", "change_type": "feature", "complexity": "complex", "difficulty": "subtle", "risk": "critical", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"augment": {"strict": {"tp": 8, "fp": 2, "fn": 1}, "core": {"tp": 8, "fp": 2, "fn": 1}, "all": {"tp": 8, "fp": 2, "fn": 1}}, "baz": {"strict": {"tp": 5, "fp": 0, "fn": 4}, "core": {"tp": 5, "fp": 0, "fn": 4}, "all": {"tp": 5, "fp": 0, "fn": 4}}, "bugbot": {"strict": {"tp": 6, "fp": 1, "fn": 3}, "core": {"tp": 6, "fp": 1, "fn": 3}, "all": {"tp": 6, "fp": 1, "fn": 3}}, "claude": {"strict": {"tp": 5, "fp": 1, "fn": 4}, "core": {"tp": 5, "fp": 1, "fn": 4}, "all": {"tp": 5, "fp": 1, "fn": 4}}, "claude-code": {"strict": {"tp": 4, "fp": 0, "fn": 5}, "core": {"tp": 4, "fp": 0, "fn": 5}, "all": {"tp": 4, "fp": 0, "fn": 5}}, "codeant-v2": {"strict": {"tp": 4, "fp": 4, "fn": 5}, "core": {"tp": 4, "fp": 4, "fn": 5}, "all": {"tp": 4, "fp": 4, "fn": 5}}, "coderabbit": {"strict": {"tp": 9, "fp": 9, "fn": 0}, "core": {"tp": 9, "fp": 9, "fn": 0}, "all": {"tp": 9, "fp": 9, "fn": 0}}, "cubic-v2": {"strict": {"tp": 3, "fp": 0, "fn": 6}, "core": {"tp": 3, "fp": 0, "fn": 6}, "all": {"tp": 3, "fp": 0, "fn": 6}}, "devin": {"strict": {"tp": 5, "fp": 0, "fn": 4}, "core": {"tp": 5, "fp": 0, "fn": 4}, "all": {"tp": 5, "fp": 0, "fn": 4}}, "graphite": {"strict": {"tp": 1, "fp": 0, "fn": 8}, "core": {"tp": 1, "fp": 0, "fn": 8}, "all": {"tp": 1, "fp": 0, "fn": 8}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 9}, "core": {"tp": 0, "fp": 0, "fn": 9}, "all": {"tp": 0, "fp": 0, "fn": 9}}, "kodus-v2": {"strict": {"tp": 4, "fp": 3, "fn": 5}, "core": {"tp": 4, "fp": 3, "fn": 5}, "all": {"tp": 4, "fp": 3, "fn": 5}}, "macroscope": {"strict": {"tp": 5, "fp": 5, "fn": 4}, "core": {"tp": 5, "fp": 5, "fn": 4}, "all": {"tp": 5, "fp": 5, "fn": 4}}, "qodo-extended-v2": {"strict": {"tp": 7, "fp": 2, "fn": 2}, "core": {"tp": 7, "fp": 2, "fn": 2}, "all": {"tp": 7, "fp": 2, "fn": 2}}, "qodo-v2": {"strict": {"tp": 8, "fp": 3, "fn": 1}, "core": {"tp": 8, "fp": 3, "fn": 1}, "all": {"tp": 8, "fp": 3, "fn": 1}}, "sourcery": {"strict": {"tp": 5, "fp": 0, "fn": 4}, "core": {"tp": 5, "fp": 0, "fn": 4}, "all": {"tp": 5, "fp": 0, "fn": 4}}, "greptile-v4-1": {"strict": {"tp": 5, "fp": 3, "fn": 4}, "core": {"tp": 5, "fp": 3, "fn": 4}, "all": {"tp": 5, "fp": 3, "fn": 4}}, "gemini-v2": {"strict": {"tp": 4, "fp": 2, "fn": 5}, "core": {"tp": 4, "fp": 2, "fn": 5}, "all": {"tp": 4, "fp": 2, "fn": 5}}, "gitlab": {"strict": {"tp": 5, "fp": 2, "fn": 4}, "core": {"tp": 5, "fp": 2, "fn": 4}, "all": {"tp": 5, "fp": 2, "fn": 4}}, "copilot-v2": {"strict": {"tp": 6, "fp": 1, "fn": 3}, "core": {"tp": 6, "fp": 1, "fn": 3}, "all": {"tp": 6, "fp": 1, "fn": 3}}}}, {"url": "https://github.com/calcom/cal.com/pull/7232", "language": "TypeScript", "pr_size": "medium", "domain": "scheduling", "change_type": "feature", "complexity": "moderate", "difficulty": "subtle", "risk": "high", "context": "cross_file", "concern": "reliability", "summary": "", "tool_metrics": {"augment": {"strict": {"tp": 1, "fp": 2, "fn": 2}, "core": {"tp": 1, "fp": 2, "fn": 2}, "all": {"tp": 1, "fp": 2, "fn": 2}}, "baz": {"strict": {"tp": 1, "fp": 0, "fn": 2}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 2}}, "bugbot": {"strict": {"tp": 1, "fp": 1, "fn": 2}, "core": {"tp": 1, "fp": 1, "fn": 2}, "all": {"tp": 1, "fp": 1, "fn": 2}}, "claude": {"strict": {"tp": 1, "fp": 3, "fn": 2}, "core": {"tp": 1, "fp": 3, "fn": 2}, "all": {"tp": 1, "fp": 3, "fn": 2}}, "claude-code": {"strict": {"tp": 1, "fp": 1, "fn": 2}, "core": {"tp": 1, "fp": 1, "fn": 2}, "all": {"tp": 1, "fp": 1, "fn": 2}}, "codeant-v2": {"strict": {"tp": 1, "fp": 2, "fn": 2}, "core": {"tp": 1, "fp": 2, "fn": 2}, "all": {"tp": 1, "fp": 2, "fn": 2}}, "coderabbit": {"strict": {"tp": 2, "fp": 7, "fn": 1}, "core": {"tp": 2, "fp": 7, "fn": 1}, "all": {"tp": 2, "fp": 7, "fn": 1}}, "cubic-v2": {"strict": {"tp": 2, "fp": 3, "fn": 1}, "core": {"tp": 2, "fp": 3, "fn": 1}, "all": {"tp": 2, "fp": 3, "fn": 1}}, "devin": {"strict": {"tp": 1, "fp": 1, "fn": 2}, "core": {"tp": 1, "fp": 1, "fn": 2}, "all": {"tp": 1, "fp": 1, "fn": 2}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 3}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 3}}, "kg": {"strict": {"tp": 0, "fp": 1, "fn": 3}, "core": {"tp": 0, "fp": 1, "fn": 3}, "all": {"tp": 0, "fp": 1, "fn": 3}}, "kodus-v2": {"strict": {"tp": 2, "fp": 2, "fn": 1}, "core": {"tp": 2, "fp": 2, "fn": 1}, "all": {"tp": 2, "fp": 2, "fn": 1}}, "macroscope": {"strict": {"tp": 1, "fp": 2, "fn": 2}, "core": {"tp": 1, "fp": 2, "fn": 2}, "all": {"tp": 1, "fp": 2, "fn": 2}}, "qodo-extended-v2": {"strict": {"tp": 2, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 1}, "all": {"tp": 2, "fp": 1, "fn": 1}}, "qodo-v2": {"strict": {"tp": 2, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 1}, "all": {"tp": 2, "fp": 1, "fn": 1}}, "sourcery": {"strict": {"tp": 1, "fp": 2, "fn": 2}, "core": {"tp": 1, "fp": 2, "fn": 2}, "all": {"tp": 1, "fp": 2, "fn": 2}}, "greptile-v4-1": {"strict": {"tp": 1, "fp": 2, "fn": 2}, "core": {"tp": 1, "fp": 2, "fn": 2}, "all": {"tp": 1, "fp": 2, "fn": 2}}, "gemini-v2": {"strict": {"tp": 2, "fp": 3, "fn": 1}, "core": {"tp": 2, "fp": 3, "fn": 1}, "all": {"tp": 2, "fp": 3, "fn": 1}}, "gitlab": {"strict": {"tp": 0, "fp": 2, "fn": 3}, "core": {"tp": 0, "fp": 2, "fn": 3}, "all": {"tp": 0, "fp": 2, "fn": 3}}, "copilot-v2": {"strict": {"tp": 0, "fp": 3, "fn": 3}, "core": {"tp": 0, "fp": 3, "fn": 3}, "all": {"tp": 0, "fp": 3, "fn": 3}}}}, {"url": "https://github.com/calcom/cal.com/pull/14740", "language": "TypeScript", "pr_size": "large", "domain": "scheduling", "change_type": "feature", "complexity": "moderate", "difficulty": "subtle", "risk": "critical", "context": "cross_file", "concern": "security", "summary": "", "tool_metrics": {"augment": {"strict": {"tp": 4, "fp": 3, "fn": 2}, "core": {"tp": 4, "fp": 3, "fn": 2}, "all": {"tp": 4, "fp": 3, "fn": 2}}, "baz": {"strict": {"tp": 1, "fp": 1, "fn": 5}, "core": {"tp": 1, "fp": 1, "fn": 5}, "all": {"tp": 1, "fp": 1, "fn": 5}}, "bugbot": {"strict": {"tp": 3, "fp": 3, "fn": 3}, "core": {"tp": 3, "fp": 3, "fn": 3}, "all": {"tp": 3, "fp": 3, "fn": 3}}, "claude": {"strict": {"tp": 2, "fp": 0, "fn": 4}, "core": {"tp": 2, "fp": 0, "fn": 4}, "all": {"tp": 2, "fp": 0, "fn": 4}}, "claude-code": {"strict": {"tp": 2, "fp": 1, "fn": 4}, "core": {"tp": 2, "fp": 1, "fn": 4}, "all": {"tp": 2, "fp": 1, "fn": 4}}, "codeant-v2": {"strict": {"tp": 3, "fp": 4, "fn": 3}, "core": {"tp": 3, "fp": 4, "fn": 3}, "all": {"tp": 3, "fp": 4, "fn": 3}}, "coderabbit": {"strict": {"tp": 4, "fp": 4, "fn": 2}, "core": {"tp": 4, "fp": 4, "fn": 2}, "all": {"tp": 4, "fp": 4, "fn": 2}}, "cubic-v2": {"strict": {"tp": 3, "fp": 2, "fn": 3}, "core": {"tp": 3, "fp": 2, "fn": 3}, "all": {"tp": 3, "fp": 2, "fn": 3}}, "devin": {"strict": {"tp": 2, "fp": 0, "fn": 4}, "core": {"tp": 2, "fp": 0, "fn": 4}, "all": {"tp": 2, "fp": 0, "fn": 4}}, "graphite": {"strict": {"tp": 1, "fp": 0, "fn": 5}, "core": {"tp": 1, "fp": 0, "fn": 5}, "all": {"tp": 1, "fp": 0, "fn": 5}}, "kg": {"strict": {"tp": 1, "fp": 1, "fn": 5}, "core": {"tp": 1, "fp": 1, "fn": 5}, "all": {"tp": 1, "fp": 1, "fn": 5}}, "kodus-v2": {"strict": {"tp": 2, "fp": 0, "fn": 4}, "core": {"tp": 2, "fp": 0, "fn": 4}, "all": {"tp": 2, "fp": 0, "fn": 4}}, "macroscope": {"strict": {"tp": 3, "fp": 3, "fn": 3}, "core": {"tp": 3, "fp": 3, "fn": 3}, "all": {"tp": 3, "fp": 3, "fn": 3}}, "qodo-extended-v2": {"strict": {"tp": 3, "fp": 2, "fn": 3}, "core": {"tp": 3, "fp": 2, "fn": 3}, "all": {"tp": 3, "fp": 2, "fn": 3}}, "qodo-v2": {"strict": {"tp": 4, "fp": 2, "fn": 2}, "core": {"tp": 4, "fp": 2, "fn": 2}, "all": {"tp": 4, "fp": 2, "fn": 2}}, "sourcery": {"strict": {"tp": 3, "fp": 2, "fn": 3}, "core": {"tp": 3, "fp": 2, "fn": 3}, "all": {"tp": 3, "fp": 2, "fn": 3}}, "greptile-v4-1": {"strict": {"tp": 3, "fp": 2, "fn": 3}, "core": {"tp": 3, "fp": 2, "fn": 3}, "all": {"tp": 3, "fp": 2, "fn": 3}}, "gemini-v2": {"strict": {"tp": 3, "fp": 4, "fn": 3}, "core": {"tp": 3, "fp": 4, "fn": 3}, "all": {"tp": 3, "fp": 4, "fn": 3}}, "gitlab": {"strict": {"tp": 2, "fp": 5, "fn": 4}, "core": {"tp": 2, "fp": 5, "fn": 4}, "all": {"tp": 2, "fp": 5, "fn": 4}}, "copilot-v2": {"strict": {"tp": 5, "fp": 5, "fn": 1}, "core": {"tp": 5, "fp": 5, "fn": 1}, "all": {"tp": 5, "fp": 5, "fn": 1}}}}, {"url": "https://github.com/calcom/cal.com/pull/10600", "language": "TypeScript", "pr_size": "medium", "domain": "authentication", "change_type": "feature", "complexity": "moderate", "difficulty": "subtle", "risk": "high", "context": "cross_file", "concern": "security", "summary": "", "tool_metrics": {"augment": {"strict": {"tp": 0, "fp": 3, "fn": 2}, "core": {"tp": 1, "fp": 3, "fn": 3}, "all": {"tp": 2, "fp": 3, "fn": 3}}, "baz": {"strict": {"tp": 0, "fp": 1, "fn": 2}, "core": {"tp": 0, "fp": 1, "fn": 4}, "all": {"tp": 0, "fp": 1, "fn": 5}}, "bugbot": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 3}, "all": {"tp": 1, "fp": 0, "fn": 4}}, "claude": {"strict": {"tp": 0, "fp": 6, "fn": 2}, "core": {"tp": 1, "fp": 6, "fn": 3}, "all": {"tp": 2, "fp": 6, "fn": 3}}, "claude-code": {"strict": {"tp": 0, "fp": 4, "fn": 2}, "core": {"tp": 0, "fp": 4, "fn": 4}, "all": {"tp": 1, "fp": 4, "fn": 4}}, "codeant-v2": {"strict": {"tp": 1, "fp": 2, "fn": 1}, "core": {"tp": 1, "fp": 2, "fn": 3}, "all": {"tp": 1, "fp": 2, "fn": 4}}, "coderabbit": {"strict": {"tp": 1, "fp": 6, "fn": 1}, "core": {"tp": 1, "fp": 6, "fn": 3}, "all": {"tp": 2, "fp": 6, "fn": 3}}, "cubic-v2": {"strict": {"tp": 2, "fp": 2, "fn": 0}, "core": {"tp": 2, "fp": 2, "fn": 2}, "all": {"tp": 2, "fp": 2, "fn": 3}}, "devin": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 3}, "all": {"tp": 1, "fp": 0, "fn": 4}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 4}, "all": {"tp": 0, "fp": 0, "fn": 5}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 4}, "all": {"tp": 0, "fp": 0, "fn": 5}}, "kodus-v2": {"strict": {"tp": 0, "fp": 1, "fn": 2}, "core": {"tp": 0, "fp": 1, "fn": 4}, "all": {"tp": 0, "fp": 1, "fn": 5}}, "macroscope": {"strict": {"tp": 0, "fp": 2, "fn": 2}, "core": {"tp": 0, "fp": 2, "fn": 4}, "all": {"tp": 0, "fp": 2, "fn": 5}}, "qodo-extended-v2": {"strict": {"tp": 1, "fp": 2, "fn": 1}, "core": {"tp": 2, "fp": 2, "fn": 2}, "all": {"tp": 2, "fp": 2, "fn": 3}}, "qodo-v2": {"strict": {"tp": 1, "fp": 3, "fn": 1}, "core": {"tp": 2, "fp": 3, "fn": 2}, "all": {"tp": 2, "fp": 3, "fn": 3}}, "sourcery": {"strict": {"tp": 1, "fp": 2, "fn": 1}, "core": {"tp": 2, "fp": 2, "fn": 2}, "all": {"tp": 3, "fp": 2, "fn": 2}}, "greptile-v4-1": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 2}, "all": {"tp": 3, "fp": 1, "fn": 2}}, "gemini-v2": {"strict": {"tp": 0, "fp": 5, "fn": 2}, "core": {"tp": 0, "fp": 5, "fn": 4}, "all": {"tp": 1, "fp": 5, "fn": 4}}, "gitlab": {"strict": {"tp": 0, "fp": 2, "fn": 2}, "core": {"tp": 1, "fp": 2, "fn": 3}, "all": {"tp": 2, "fp": 2, "fn": 3}}, "copilot-v2": {"strict": {"tp": 2, "fp": 4, "fn": 0}, "core": {"tp": 4, "fp": 4, "fn": 0}, "all": {"tp": 5, "fp": 4, "fn": 0}}}}, {"url": "https://github.com/calcom/cal.com/pull/10967", "language": "TypeScript", "pr_size": "large", "domain": "scheduling", "change_type": "bug_fix", "complexity": "complex", "difficulty": "subtle", "risk": "high", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"augment": {"strict": {"tp": 3, "fp": 2, "fn": 2}, "core": {"tp": 3, "fp": 2, "fn": 2}, "all": {"tp": 3, "fp": 2, "fn": 3}}, "baz": {"strict": {"tp": 1, "fp": 1, "fn": 4}, "core": {"tp": 1, "fp": 1, "fn": 4}, "all": {"tp": 1, "fp": 1, "fn": 5}}, "bugbot": {"strict": {"tp": 2, "fp": 4, "fn": 3}, "core": {"tp": 2, "fp": 4, "fn": 3}, "all": {"tp": 2, "fp": 4, "fn": 4}}, "claude": {"strict": {"tp": 4, "fp": 3, "fn": 1}, "core": {"tp": 4, "fp": 3, "fn": 1}, "all": {"tp": 4, "fp": 3, "fn": 2}}, "claude-code": {"strict": {"tp": 3, "fp": 5, "fn": 2}, "core": {"tp": 3, "fp": 5, "fn": 2}, "all": {"tp": 3, "fp": 5, "fn": 3}}, "codeant-v2": {"strict": {"tp": 1, "fp": 7, "fn": 4}, "core": {"tp": 1, "fp": 7, "fn": 4}, "all": {"tp": 1, "fp": 7, "fn": 5}}, "coderabbit": {"strict": {"tp": 5, "fp": 5, "fn": 0}, "core": {"tp": 5, "fp": 5, "fn": 0}, "all": {"tp": 5, "fp": 5, "fn": 1}}, "cubic-v2": {"strict": {"tp": 1, "fp": 2, "fn": 4}, "core": {"tp": 1, "fp": 2, "fn": 4}, "all": {"tp": 1, "fp": 2, "fn": 5}}, "devin": {"strict": {"tp": 3, "fp": 1, "fn": 2}, "core": {"tp": 3, "fp": 1, "fn": 2}, "all": {"tp": 3, "fp": 1, "fn": 3}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 5}, "core": {"tp": 0, "fp": 0, "fn": 5}, "all": {"tp": 0, "fp": 0, "fn": 6}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 5}, "core": {"tp": 0, "fp": 0, "fn": 5}, "all": {"tp": 0, "fp": 0, "fn": 6}}, "kodus-v2": {"strict": {"tp": 3, "fp": 1, "fn": 2}, "core": {"tp": 3, "fp": 1, "fn": 2}, "all": {"tp": 3, "fp": 1, "fn": 3}}, "macroscope": {"strict": {"tp": 3, "fp": 2, "fn": 2}, "core": {"tp": 3, "fp": 2, "fn": 2}, "all": {"tp": 3, "fp": 2, "fn": 3}}, "qodo-extended-v2": {"strict": {"tp": 1, "fp": 4, "fn": 4}, "core": {"tp": 1, "fp": 4, "fn": 4}, "all": {"tp": 1, "fp": 4, "fn": 5}}, "qodo-v2": {"strict": {"tp": 2, "fp": 3, "fn": 3}, "core": {"tp": 2, "fp": 3, "fn": 3}, "all": {"tp": 2, "fp": 3, "fn": 4}}, "sourcery": {"strict": {"tp": 2, "fp": 5, "fn": 3}, "core": {"tp": 2, "fp": 5, "fn": 3}, "all": {"tp": 2, "fp": 5, "fn": 4}}, "greptile-v4-1": {"strict": {"tp": 1, "fp": 2, "fn": 4}, "core": {"tp": 1, "fp": 2, "fn": 4}, "all": {"tp": 1, "fp": 2, "fn": 5}}, "gemini-v2": {"strict": {"tp": 1, "fp": 6, "fn": 4}, "core": {"tp": 1, "fp": 6, "fn": 4}, "all": {"tp": 1, "fp": 6, "fn": 5}}, "gitlab": {"strict": {"tp": 3, "fp": 3, "fn": 2}, "core": {"tp": 3, "fp": 3, "fn": 2}, "all": {"tp": 3, "fp": 3, "fn": 3}}, "copilot-v2": {"strict": {"tp": 4, "fp": 7, "fn": 1}, "core": {"tp": 4, "fp": 7, "fn": 1}, "all": {"tp": 4, "fp": 7, "fn": 2}}}}, {"url": "https://github.com/calcom/cal.com/pull/8087", "language": "TypeScript", "pr_size": "medium", "domain": "concurrency", "change_type": "refactoring", "complexity": "moderate", "difficulty": "subtle", "risk": "critical", "context": "cross_file", "concern": "reliability", "summary": "", "tool_metrics": {"augment": {"strict": {"tp": 1, "fp": 6, "fn": 0}, "core": {"tp": 1, "fp": 6, "fn": 0}, "all": {"tp": 1, "fp": 6, "fn": 1}}, "baz": {"strict": {"tp": 1, "fp": 2, "fn": 0}, "core": {"tp": 1, "fp": 2, "fn": 0}, "all": {"tp": 1, "fp": 2, "fn": 1}}, "bugbot": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 0}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "claude": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 1, "fp": 1, "fn": 0}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "claude-code": {"strict": {"tp": 1, "fp": 6, "fn": 0}, "core": {"tp": 1, "fp": 6, "fn": 0}, "all": {"tp": 1, "fp": 6, "fn": 1}}, "codeant-v2": {"strict": {"tp": 1, "fp": 4, "fn": 0}, "core": {"tp": 1, "fp": 4, "fn": 0}, "all": {"tp": 1, "fp": 4, "fn": 1}}, "coderabbit": {"strict": {"tp": 1, "fp": 5, "fn": 0}, "core": {"tp": 1, "fp": 5, "fn": 0}, "all": {"tp": 1, "fp": 5, "fn": 1}}, "cubic-v2": {"strict": {"tp": 1, "fp": 9, "fn": 0}, "core": {"tp": 1, "fp": 9, "fn": 0}, "all": {"tp": 2, "fp": 9, "fn": 0}}, "devin": {"strict": {"tp": 1, "fp": 3, "fn": 0}, "core": {"tp": 1, "fp": 3, "fn": 0}, "all": {"tp": 2, "fp": 3, "fn": 0}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "kg": {"strict": {"tp": 1, "fp": 2, "fn": 0}, "core": {"tp": 1, "fp": 2, "fn": 0}, "all": {"tp": 1, "fp": 2, "fn": 1}}, "kodus-v2": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "macroscope": {"strict": {"tp": 1, "fp": 4, "fn": 0}, "core": {"tp": 1, "fp": 4, "fn": 0}, "all": {"tp": 1, "fp": 4, "fn": 1}}, "qodo-extended-v2": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 1, "fp": 1, "fn": 0}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "qodo-v2": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 0}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "sourcery": {"strict": {"tp": 1, "fp": 3, "fn": 0}, "core": {"tp": 1, "fp": 3, "fn": 0}, "all": {"tp": 1, "fp": 3, "fn": 1}}, "greptile-v4-1": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 1, "fp": 1, "fn": 0}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "gemini-v2": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 0}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "gitlab": {"strict": {"tp": 1, "fp": 2, "fn": 0}, "core": {"tp": 1, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "copilot-v2": {"strict": {"tp": 1, "fp": 2, "fn": 0}, "core": {"tp": 1, "fp": 2, "fn": 0}, "all": {"tp": 1, "fp": 2, "fn": 1}}}}], "tools": ["augment", "baz", "bugbot", "claude", "claude-code", "codeant-v2", "coderabbit", "copilot-v2", "cubic-v2", "devin", "gemini-v2", "gitlab", "graphite", "greptile-v4-1", "kg", "kodus-v2", "macroscope", "qodo-extended-v2", "qodo-v2", "sourcery"], "dimensions": {"language": ["Go", "Java", "Python", "Ruby", "TypeScript"], "pr_size": ["small", "medium", "large"], "domain": ["API", "UI", "authentication", "caching", "concurrency", "configuration", "data_processing", "database", "file_io", "logging", "scheduling", "serialization", "testing"], "change_type": ["bug_fix", "feature", "migration", "performance", "refactoring", "security_patch"], "complexity": ["simple", "moderate", "complex"], "difficulty": ["obvious", "moderate", "subtle", "very_subtle"], "risk": ["low", "medium", "high", "critical"], "context": ["local", "file", "cross_file", "system"], "concern": ["correctness", "maintainability", "reliability", "security"]}, "overall_metrics": {"sourcery": {"strict": {"precision": 34.2, "recall": 48.9, "f1": 40.2, "f2": 45.0, "tp": 68, "fp": 131, "fn": 71, "num_prs": 50}, "core": {"precision": 37.0, "recall": 48.7, "f1": 42.1, "f2": 45.8, "tp": 77, "fp": 131, "fn": 81, "num_prs": 50}, "all": {"precision": 38.8, "recall": 48.0, "f1": 42.9, "f2": 45.8, "tp": 83, "fp": 131, "fn": 90, "num_prs": 50}}, "macroscope": {"strict": {"precision": 50.4, "recall": 45.3, "f1": 47.7, "f2": 46.3, "tp": 63, "fp": 62, "fn": 76, "num_prs": 50}, "core": {"precision": 51.6, "recall": 41.8, "f1": 46.2, "f2": 43.4, "tp": 66, "fp": 62, "fn": 92, "num_prs": 50}, "all": {"precision": 53.0, "recall": 40.5, "f1": 45.9, "f2": 42.5, "tp": 70, "fp": 62, "fn": 103, "num_prs": 50}}, "graphite": {"strict": {"precision": 100.0, "recall": 7.9, "f1": 14.7, "f2": 9.7, "tp": 11, "fp": 0, "fn": 128, "num_prs": 50}, "core": {"precision": 100.0, "recall": 7.6, "f1": 14.1, "f2": 9.3, "tp": 12, "fp": 0, "fn": 146, "num_prs": 50}, "all": {"precision": 100.0, "recall": 6.9, "f1": 13.0, "f2": 8.5, "tp": 12, "fp": 0, "fn": 161, "num_prs": 50}}, "augment": {"strict": {"precision": 49.4, "recall": 61.9, "f1": 55.0, "f2": 58.9, "tp": 86, "fp": 88, "fn": 53, "num_prs": 50}, "core": {"precision": 52.4, "recall": 61.4, "f1": 56.6, "f2": 59.4, "tp": 97, "fp": 88, "fn": 61, "num_prs": 50}, "all": {"precision": 53.7, "recall": 59.0, "f1": 56.2, "f2": 57.8, "tp": 102, "fp": 88, "fn": 71, "num_prs": 50}}, "kodus-v2": {"strict": {"precision": 55.2, "recall": 38.1, "f1": 45.1, "f2": 40.6, "tp": 53, "fp": 43, "fn": 86, "num_prs": 50}, "core": {"precision": 57.4, "recall": 36.7, "f1": 44.8, "f2": 39.6, "tp": 58, "fp": 43, "fn": 100, "num_prs": 50}, "all": {"precision": 57.8, "recall": 34.1, "f1": 42.9, "f2": 37.2, "tp": 59, "fp": 43, "fn": 114, "num_prs": 50}}, "qodo-extended-v2": {"strict": {"precision": 53.6, "recall": 58.3, "f1": 55.9, "f2": 57.3, "tp": 81, "fp": 70, "fn": 58, "num_prs": 50}, "core": {"precision": 56.2, "recall": 57.0, "f1": 56.6, "f2": 56.8, "tp": 90, "fp": 70, "fn": 68, "num_prs": 50}, "all": {"precision": 56.8, "recall": 53.2, "f1": 54.9, "f2": 53.9, "tp": 92, "fp": 70, "fn": 81, "num_prs": 50}}, "kg": {"strict": {"precision": 48.8, "recall": 15.1, "f1": 23.1, "f2": 17.5, "tp": 21, "fp": 22, "fn": 118, "num_prs": 50}, "core": {"precision": 50.0, "recall": 13.9, "f1": 21.8, "f2": 16.3, "tp": 22, "fp": 22, "fn": 136, "num_prs": 50}, "all": {"precision": 51.1, "recall": 13.3, "f1": 21.1, "f2": 15.6, "tp": 23, "fp": 22, "fn": 150, "num_prs": 50}}, "cubic-v2": {"strict": {"precision": 52.9, "recall": 59.7, "f1": 56.1, "f2": 58.2, "tp": 83, "fp": 74, "fn": 56, "num_prs": 50}, "core": {"precision": 56.5, "recall": 60.8, "f1": 58.5, "f2": 59.9, "tp": 96, "fp": 74, "fn": 62, "num_prs": 50}, "all": {"precision": 57.2, "recall": 57.2, "f1": 57.2, "f2": 57.2, "tp": 99, "fp": 74, "fn": 74, "num_prs": 50}}, "gemini-v2": {"strict": {"precision": 36.7, "recall": 51.8, "f1": 43.0, "f2": 47.9, "tp": 72, "fp": 124, "fn": 67, "num_prs": 50}, "core": {"precision": 39.2, "recall": 50.6, "f1": 44.2, "f2": 47.8, "tp": 80, "fp": 124, "fn": 78, "num_prs": 50}, "all": {"precision": 41.2, "recall": 50.3, "f1": 45.3, "f2": 48.2, "tp": 87, "fp": 124, "fn": 86, "num_prs": 50}}, "greptile-v4-1": {"strict": {"precision": 44.9, "recall": 50.4, "f1": 47.5, "f2": 49.2, "tp": 70, "fp": 86, "fn": 69, "num_prs": 50}, "core": {"precision": 46.9, "recall": 48.1, "f1": 47.5, "f2": 47.9, "tp": 76, "fp": 86, "fn": 82, "num_prs": 50}, "all": {"precision": 49.1, "recall": 48.0, "f1": 48.5, "f2": 48.2, "tp": 83, "fp": 86, "fn": 90, "num_prs": 50}}, "bugbot": {"strict": {"precision": 52.0, "recall": 46.0, "f1": 48.9, "f2": 47.1, "tp": 64, "fp": 59, "fn": 75, "num_prs": 50}, "core": {"precision": 54.3, "recall": 44.3, "f1": 48.8, "f2": 46.0, "tp": 70, "fp": 59, "fn": 88, "num_prs": 50}, "all": {"precision": 55.3, "recall": 42.2, "f1": 47.9, "f2": 44.3, "tp": 73, "fp": 59, "fn": 100, "num_prs": 50}}, "claude": {"strict": {"precision": 36.1, "recall": 37.4, "f1": 36.7, "f2": 37.1, "tp": 52, "fp": 92, "fn": 87, "num_prs": 50}, "core": {"precision": 38.3, "recall": 36.1, "f1": 37.1, "f2": 36.5, "tp": 57, "fp": 92, "fn": 101, "num_prs": 50}, "all": {"precision": 40.6, "recall": 36.4, "f1": 38.4, "f2": 37.2, "tp": 63, "fp": 92, "fn": 110, "num_prs": 50}}, "devin": {"strict": {"precision": 66.0, "recall": 44.6, "f1": 53.2, "f2": 47.7, "tp": 62, "fp": 32, "fn": 77, "num_prs": 50}, "core": {"precision": 67.3, "recall": 41.8, "f1": 51.6, "f2": 45.2, "tp": 66, "fp": 32, "fn": 92, "num_prs": 50}, "all": {"precision": 68.3, "recall": 39.9, "f1": 50.4, "f2": 43.5, "tp": 69, "fp": 32, "fn": 104, "num_prs": 50}}, "baz": {"strict": {"precision": 54.6, "recall": 38.1, "f1": 44.9, "f2": 40.6, "tp": 53, "fp": 44, "fn": 86, "num_prs": 50}, "core": {"precision": 55.6, "recall": 34.8, "f1": 42.8, "f2": 37.6, "tp": 55, "fp": 44, "fn": 103, "num_prs": 50}, "all": {"precision": 56.9, "recall": 33.5, "f1": 42.2, "f2": 36.5, "tp": 58, "fp": 44, "fn": 115, "num_prs": 50}}, "claude-code": {"strict": {"precision": 34.3, "recall": 42.4, "f1": 37.9, "f2": 40.5, "tp": 59, "fp": 113, "fn": 80, "num_prs": 50}, "core": {"precision": 36.5, "recall": 41.1, "f1": 38.7, "f2": 40.1, "tp": 65, "fp": 113, "fn": 93, "num_prs": 50}, "all": {"precision": 38.3, "recall": 40.5, "f1": 39.3, "f2": 40.0, "tp": 70, "fp": 113, "fn": 103, "num_prs": 50}}, "gitlab": {"strict": {"precision": 44.7, "recall": 48.9, "f1": 46.7, "f2": 48.0, "tp": 68, "fp": 84, "fn": 71, "num_prs": 50}, "core": {"precision": 47.5, "recall": 48.1, "f1": 47.8, "f2": 48.0, "tp": 76, "fp": 84, "fn": 82, "num_prs": 50}, "all": {"precision": 49.7, "recall": 48.0, "f1": 48.8, "f2": 48.3, "tp": 83, "fp": 84, "fn": 90, "num_prs": 50}}, "qodo-v2": {"strict": {"precision": 45.1, "recall": 59.0, "f1": 51.1, "f2": 55.6, "tp": 82, "fp": 100, "fn": 57, "num_prs": 50}, "core": {"precision": 47.1, "recall": 56.3, "f1": 51.3, "f2": 54.2, "tp": 89, "fp": 100, "fn": 69, "num_prs": 50}, "all": {"precision": 47.9, "recall": 53.2, "f1": 50.4, "f2": 52.0, "tp": 92, "fp": 100, "fn": 81, "num_prs": 50}}, "coderabbit": {"strict": {"precision": 31.8, "recall": 60.4, "f1": 41.7, "f2": 51.2, "tp": 84, "fp": 180, "fn": 55, "num_prs": 50}, "core": {"precision": 33.8, "recall": 58.2, "f1": 42.8, "f2": 50.9, "tp": 92, "fp": 180, "fn": 66, "num_prs": 50}, "all": {"precision": 35.3, "recall": 56.6, "f1": 43.5, "f2": 50.5, "tp": 98, "fp": 180, "fn": 75, "num_prs": 50}}, "codeant-v2": {"strict": {"precision": 31.1, "recall": 37.4, "f1": 34.0, "f2": 36.0, "tp": 52, "fp": 115, "fn": 87, "num_prs": 50}, "core": {"precision": 32.4, "recall": 34.8, "f1": 33.5, "f2": 34.3, "tp": 55, "fp": 115, "fn": 103, "num_prs": 50}, "all": {"precision": 33.1, "recall": 32.9, "f1": 33.0, "f2": 33.0, "tp": 57, "fp": 115, "fn": 116, "num_prs": 50}}, "copilot-v2": {"strict": {"precision": 31.4, "recall": 61.9, "f1": 41.6, "f2": 51.8, "tp": 86, "fp": 188, "fn": 53, "num_prs": 50}, "core": {"precision": 34.3, "recall": 62.0, "f1": 44.1, "f2": 53.4, "tp": 98, "fp": 188, "fn": 60, "num_prs": 50}, "all": {"precision": 36.1, "recall": 61.3, "f1": 45.4, "f2": 53.8, "tp": 106, "fp": 188, "fn": 67, "num_prs": 50}}}}, "openai_gpt-5.2": {"prs": [{"url": "https://github.com/keycloak/keycloak/pull/37429", "language": "Java", "pr_size": "medium", "domain": "UI", "change_type": "feature", "complexity": "moderate", "difficulty": "moderate", "risk": "medium", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 4}}, "codeant-v2": {"strict": {"tp": 0, "fp": 2, "fn": 1}, "core": {"tp": 0, "fp": 2, "fn": 3}, "all": {"tp": 0, "fp": 2, "fn": 5}}, "coderabbit": {"strict": {"tp": 0, "fp": 6, "fn": 1}, "core": {"tp": 2, "fp": 6, "fn": 1}, "all": {"tp": 4, "fp": 6, "fn": 1}}, "qodo-extended-v2": {"strict": {"tp": 0, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 1}, "all": {"tp": 2, "fp": 1, "fn": 3}}, "greptile-v4-1": {"strict": {"tp": 0, "fp": 3, "fn": 1}, "core": {"tp": 0, "fp": 3, "fn": 3}, "all": {"tp": 2, "fp": 3, "fn": 3}}, "gemini-v2": {"strict": {"tp": 0, "fp": 5, "fn": 1}, "core": {"tp": 1, "fp": 5, "fn": 2}, "all": {"tp": 3, "fp": 5, "fn": 2}}, "gitlab": {"strict": {"tp": 0, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 2}, "all": {"tp": 2, "fp": 1, "fn": 3}}, "copilot-v2": {"strict": {"tp": 0, "fp": 6, "fn": 1}, "core": {"tp": 2, "fp": 6, "fn": 1}, "all": {"tp": 3, "fp": 6, "fn": 2}}, "augment": {"strict": {"tp": 0, "fp": 3, "fn": 1}, "core": {"tp": 2, "fp": 3, "fn": 1}, "all": {"tp": 2, "fp": 3, "fn": 3}}, "bugbot": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 2, "fp": 0, "fn": 1}, "all": {"tp": 2, "fp": 0, "fn": 3}}, "claude": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 1, "fp": 0, "fn": 4}}, "claude-code": {"strict": {"tp": 0, "fp": 4, "fn": 1}, "core": {"tp": 0, "fp": 4, "fn": 3}, "all": {"tp": 2, "fp": 4, "fn": 3}}, "cubic-v2": {"strict": {"tp": 0, "fp": 2, "fn": 1}, "core": {"tp": 1, "fp": 2, "fn": 2}, "all": {"tp": 1, "fp": 2, "fn": 4}}, "devin": {"strict": {"tp": 0, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 1}, "all": {"tp": 2, "fp": 1, "fn": 3}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 5}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 4}}, "kodus-v2": {"strict": {"tp": 0, "fp": 3, "fn": 1}, "core": {"tp": 1, "fp": 3, "fn": 2}, "all": {"tp": 1, "fp": 3, "fn": 4}}, "macroscope": {"strict": {"tp": 0, "fp": 2, "fn": 1}, "core": {"tp": 0, "fp": 2, "fn": 3}, "all": {"tp": 0, "fp": 2, "fn": 5}}, "qodo-v2": {"strict": {"tp": 0, "fp": 3, "fn": 1}, "core": {"tp": 0, "fp": 3, "fn": 3}, "all": {"tp": 0, "fp": 3, "fn": 5}}, "sourcery": {"strict": {"tp": 0, "fp": 13, "fn": 1}, "core": {"tp": 0, "fp": 13, "fn": 3}, "all": {"tp": 2, "fp": 13, "fn": 3}}}}, {"url": "https://github.com/keycloak/keycloak/pull/37634", "language": "Java", "pr_size": "medium", "domain": "authentication", "change_type": "feature", "complexity": "moderate", "difficulty": "subtle", "risk": "high", "context": "file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 2, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 2}, "all": {"tp": 2, "fp": 1, "fn": 2}}, "codeant-v2": {"strict": {"tp": 2, "fp": 5, "fn": 1}, "core": {"tp": 2, "fp": 5, "fn": 2}, "all": {"tp": 2, "fp": 5, "fn": 2}}, "coderabbit": {"strict": {"tp": 3, "fp": 7, "fn": 0}, "core": {"tp": 3, "fp": 7, "fn": 1}, "all": {"tp": 3, "fp": 7, "fn": 1}}, "qodo-extended-v2": {"strict": {"tp": 2, "fp": 0, "fn": 1}, "core": {"tp": 3, "fp": 0, "fn": 1}, "all": {"tp": 3, "fp": 0, "fn": 1}}, "greptile-v4-1": {"strict": {"tp": 2, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 2}, "all": {"tp": 2, "fp": 1, "fn": 2}}, "gemini-v2": {"strict": {"tp": 2, "fp": 2, "fn": 1}, "core": {"tp": 2, "fp": 2, "fn": 2}, "all": {"tp": 2, "fp": 2, "fn": 2}}, "gitlab": {"strict": {"tp": 1, "fp": 1, "fn": 2}, "core": {"tp": 2, "fp": 1, "fn": 2}, "all": {"tp": 2, "fp": 1, "fn": 2}}, "copilot-v2": {"strict": {"tp": 2, "fp": 3, "fn": 1}, "core": {"tp": 3, "fp": 3, "fn": 1}, "all": {"tp": 3, "fp": 3, "fn": 1}}, "augment": {"strict": {"tp": 1, "fp": 5, "fn": 2}, "core": {"tp": 1, "fp": 5, "fn": 3}, "all": {"tp": 1, "fp": 5, "fn": 3}}, "bugbot": {"strict": {"tp": 2, "fp": 2, "fn": 1}, "core": {"tp": 2, "fp": 2, "fn": 2}, "all": {"tp": 2, "fp": 2, "fn": 2}}, "claude": {"strict": {"tp": 1, "fp": 2, "fn": 2}, "core": {"tp": 2, "fp": 2, "fn": 2}, "all": {"tp": 2, "fp": 2, "fn": 2}}, "claude-code": {"strict": {"tp": 0, "fp": 2, "fn": 3}, "core": {"tp": 0, "fp": 2, "fn": 4}, "all": {"tp": 0, "fp": 2, "fn": 4}}, "cubic-v2": {"strict": {"tp": 2, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 2}, "all": {"tp": 2, "fp": 1, "fn": 2}}, "devin": {"strict": {"tp": 2, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 2}, "all": {"tp": 2, "fp": 1, "fn": 2}}, "graphite": {"strict": {"tp": 1, "fp": 1, "fn": 2}, "core": {"tp": 1, "fp": 1, "fn": 3}, "all": {"tp": 1, "fp": 1, "fn": 3}}, "kg": {"strict": {"tp": 1, "fp": 2, "fn": 2}, "core": {"tp": 1, "fp": 2, "fn": 3}, "all": {"tp": 1, "fp": 2, "fn": 3}}, "kodus-v2": {"strict": {"tp": 2, "fp": 0, "fn": 1}, "core": {"tp": 2, "fp": 0, "fn": 2}, "all": {"tp": 2, "fp": 0, "fn": 2}}, "macroscope": {"strict": {"tp": 2, "fp": 2, "fn": 1}, "core": {"tp": 2, "fp": 2, "fn": 2}, "all": {"tp": 2, "fp": 2, "fn": 2}}, "qodo-v2": {"strict": {"tp": 2, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 2}, "all": {"tp": 2, "fp": 1, "fn": 2}}, "sourcery": {"strict": {"tp": 2, "fp": 0, "fn": 1}, "core": {"tp": 2, "fp": 0, "fn": 2}, "all": {"tp": 2, "fp": 0, "fn": 2}}}}, {"url": "https://github.com/keycloak/keycloak/pull/38446", "language": "Java", "pr_size": "medium", "domain": "authentication", "change_type": "feature", "complexity": "moderate", "difficulty": "moderate", "risk": "medium", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 0, "fp": 1, "fn": 2}, "core": {"tp": 0, "fp": 1, "fn": 2}, "all": {"tp": 0, "fp": 1, "fn": 2}}, "codeant-v2": {"strict": {"tp": 0, "fp": 1, "fn": 2}, "core": {"tp": 0, "fp": 1, "fn": 2}, "all": {"tp": 0, "fp": 1, "fn": 2}}, "coderabbit": {"strict": {"tp": 2, "fp": 3, "fn": 0}, "core": {"tp": 2, "fp": 3, "fn": 0}, "all": {"tp": 2, "fp": 3, "fn": 0}}, "qodo-extended-v2": {"strict": {"tp": 1, "fp": 3, "fn": 1}, "core": {"tp": 1, "fp": 3, "fn": 1}, "all": {"tp": 1, "fp": 3, "fn": 1}}, "greptile-v4-1": {"strict": {"tp": 1, "fp": 2, "fn": 1}, "core": {"tp": 1, "fp": 2, "fn": 1}, "all": {"tp": 1, "fp": 2, "fn": 1}}, "gemini-v2": {"strict": {"tp": 0, "fp": 4, "fn": 2}, "core": {"tp": 0, "fp": 4, "fn": 2}, "all": {"tp": 0, "fp": 4, "fn": 2}}, "gitlab": {"strict": {"tp": 0, "fp": 8, "fn": 2}, "core": {"tp": 0, "fp": 8, "fn": 2}, "all": {"tp": 0, "fp": 8, "fn": 2}}, "copilot-v2": {"strict": {"tp": 1, "fp": 8, "fn": 1}, "core": {"tp": 1, "fp": 8, "fn": 1}, "all": {"tp": 1, "fp": 8, "fn": 1}}, "augment": {"strict": {"tp": 1, "fp": 3, "fn": 1}, "core": {"tp": 1, "fp": 3, "fn": 1}, "all": {"tp": 1, "fp": 3, "fn": 1}}, "bugbot": {"strict": {"tp": 0, "fp": 3, "fn": 2}, "core": {"tp": 0, "fp": 3, "fn": 2}, "all": {"tp": 0, "fp": 3, "fn": 2}}, "claude": {"strict": {"tp": 1, "fp": 3, "fn": 1}, "core": {"tp": 1, "fp": 3, "fn": 1}, "all": {"tp": 1, "fp": 3, "fn": 1}}, "claude-code": {"strict": {"tp": 1, "fp": 5, "fn": 1}, "core": {"tp": 1, "fp": 5, "fn": 1}, "all": {"tp": 1, "fp": 5, "fn": 1}}, "cubic-v2": {"strict": {"tp": 1, "fp": 3, "fn": 1}, "core": {"tp": 1, "fp": 3, "fn": 1}, "all": {"tp": 1, "fp": 3, "fn": 1}}, "devin": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "kodus-v2": {"strict": {"tp": 0, "fp": 2, "fn": 2}, "core": {"tp": 0, "fp": 2, "fn": 2}, "all": {"tp": 0, "fp": 2, "fn": 2}}, "macroscope": {"strict": {"tp": 0, "fp": 4, "fn": 2}, "core": {"tp": 0, "fp": 4, "fn": 2}, "all": {"tp": 0, "fp": 4, "fn": 2}}, "qodo-v2": {"strict": {"tp": 2, "fp": 3, "fn": 0}, "core": {"tp": 2, "fp": 3, "fn": 0}, "all": {"tp": 2, "fp": 3, "fn": 0}}, "sourcery": {"strict": {"tp": 0, "fp": 7, "fn": 2}, "core": {"tp": 0, "fp": 7, "fn": 2}, "all": {"tp": 0, "fp": 7, "fn": 2}}}}, {"url": "https://github.com/keycloak/keycloak/pull/36882", "language": "Java", "pr_size": "small", "domain": "configuration", "change_type": "feature", "complexity": "moderate", "difficulty": "moderate", "risk": "medium", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 0, "fp": 1, "fn": 1}, "core": {"tp": 0, "fp": 1, "fn": 2}, "all": {"tp": 0, "fp": 1, "fn": 2}}, "codeant-v2": {"strict": {"tp": 0, "fp": 2, "fn": 1}, "core": {"tp": 0, "fp": 2, "fn": 2}, "all": {"tp": 0, "fp": 2, "fn": 2}}, "coderabbit": {"strict": {"tp": 0, "fp": 2, "fn": 1}, "core": {"tp": 0, "fp": 2, "fn": 2}, "all": {"tp": 0, "fp": 2, "fn": 2}}, "qodo-extended-v2": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "greptile-v4-1": {"strict": {"tp": 0, "fp": 2, "fn": 1}, "core": {"tp": 0, "fp": 2, "fn": 2}, "all": {"tp": 0, "fp": 2, "fn": 2}}, "gemini-v2": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "gitlab": {"strict": {"tp": 0, "fp": 2, "fn": 1}, "core": {"tp": 0, "fp": 2, "fn": 2}, "all": {"tp": 0, "fp": 2, "fn": 2}}, "copilot-v2": {"strict": {"tp": 0, "fp": 3, "fn": 1}, "core": {"tp": 0, "fp": 3, "fn": 2}, "all": {"tp": 0, "fp": 3, "fn": 2}}, "augment": {"strict": {"tp": 0, "fp": 1, "fn": 1}, "core": {"tp": 0, "fp": 1, "fn": 2}, "all": {"tp": 0, "fp": 1, "fn": 2}}, "bugbot": {"strict": {"tp": 0, "fp": 1, "fn": 1}, "core": {"tp": 0, "fp": 1, "fn": 2}, "all": {"tp": 0, "fp": 1, "fn": 2}}, "claude": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "claude-code": {"strict": {"tp": 0, "fp": 1, "fn": 1}, "core": {"tp": 0, "fp": 1, "fn": 2}, "all": {"tp": 0, "fp": 1, "fn": 2}}, "cubic-v2": {"strict": {"tp": 1, "fp": 2, "fn": 0}, "core": {"tp": 1, "fp": 2, "fn": 1}, "all": {"tp": 1, "fp": 2, "fn": 1}}, "devin": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "kodus-v2": {"strict": {"tp": 0, "fp": 2, "fn": 1}, "core": {"tp": 0, "fp": 2, "fn": 2}, "all": {"tp": 0, "fp": 2, "fn": 2}}, "macroscope": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "qodo-v2": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "sourcery": {"strict": {"tp": 0, "fp": 4, "fn": 1}, "core": {"tp": 0, "fp": 4, "fn": 2}, "all": {"tp": 0, "fp": 4, "fn": 2}}}}, {"url": "https://github.com/keycloak/keycloak/pull/36880", "language": "Java", "pr_size": "medium", "domain": "authentication", "change_type": "feature", "complexity": "complex", "difficulty": "very_subtle", "risk": "high", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 2, "fp": 1, "fn": 2}, "core": {"tp": 2, "fp": 1, "fn": 2}, "all": {"tp": 2, "fp": 1, "fn": 3}}, "codeant-v2": {"strict": {"tp": 2, "fp": 0, "fn": 2}, "core": {"tp": 2, "fp": 0, "fn": 2}, "all": {"tp": 2, "fp": 0, "fn": 3}}, "coderabbit": {"strict": {"tp": 1, "fp": 5, "fn": 3}, "core": {"tp": 1, "fp": 5, "fn": 3}, "all": {"tp": 2, "fp": 5, "fn": 3}}, "qodo-extended-v2": {"strict": {"tp": 2, "fp": 0, "fn": 2}, "core": {"tp": 2, "fp": 0, "fn": 2}, "all": {"tp": 2, "fp": 0, "fn": 3}}, "greptile-v4-1": {"strict": {"tp": 1, "fp": 1, "fn": 3}, "core": {"tp": 1, "fp": 1, "fn": 3}, "all": {"tp": 2, "fp": 1, "fn": 3}}, "gemini-v2": {"strict": {"tp": 1, "fp": 3, "fn": 3}, "core": {"tp": 1, "fp": 3, "fn": 3}, "all": {"tp": 2, "fp": 3, "fn": 3}}, "gitlab": {"strict": {"tp": 1, "fp": 6, "fn": 3}, "core": {"tp": 1, "fp": 6, "fn": 3}, "all": {"tp": 2, "fp": 6, "fn": 3}}, "copilot-v2": {"strict": {"tp": 1, "fp": 7, "fn": 3}, "core": {"tp": 1, "fp": 7, "fn": 3}, "all": {"tp": 1, "fp": 7, "fn": 4}}, "augment": {"strict": {"tp": 1, "fp": 1, "fn": 3}, "core": {"tp": 1, "fp": 1, "fn": 3}, "all": {"tp": 1, "fp": 1, "fn": 4}}, "bugbot": {"strict": {"tp": 2, "fp": 1, "fn": 2}, "core": {"tp": 2, "fp": 1, "fn": 2}, "all": {"tp": 3, "fp": 1, "fn": 2}}, "claude": {"strict": {"tp": 1, "fp": 4, "fn": 3}, "core": {"tp": 1, "fp": 4, "fn": 3}, "all": {"tp": 2, "fp": 4, "fn": 3}}, "claude-code": {"strict": {"tp": 0, "fp": 2, "fn": 4}, "core": {"tp": 0, "fp": 2, "fn": 4}, "all": {"tp": 0, "fp": 2, "fn": 5}}, "cubic-v2": {"strict": {"tp": 1, "fp": 1, "fn": 3}, "core": {"tp": 1, "fp": 1, "fn": 3}, "all": {"tp": 1, "fp": 1, "fn": 4}}, "devin": {"strict": {"tp": 1, "fp": 0, "fn": 3}, "core": {"tp": 1, "fp": 0, "fn": 3}, "all": {"tp": 1, "fp": 0, "fn": 4}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 4}, "core": {"tp": 0, "fp": 0, "fn": 4}, "all": {"tp": 0, "fp": 0, "fn": 5}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 4}, "core": {"tp": 0, "fp": 0, "fn": 4}, "all": {"tp": 0, "fp": 0, "fn": 5}}, "kodus-v2": {"strict": {"tp": 2, "fp": 1, "fn": 2}, "core": {"tp": 2, "fp": 1, "fn": 2}, "all": {"tp": 2, "fp": 1, "fn": 3}}, "macroscope": {"strict": {"tp": 2, "fp": 4, "fn": 2}, "core": {"tp": 2, "fp": 4, "fn": 2}, "all": {"tp": 2, "fp": 4, "fn": 3}}, "qodo-v2": {"strict": {"tp": 1, "fp": 3, "fn": 3}, "core": {"tp": 1, "fp": 3, "fn": 3}, "all": {"tp": 1, "fp": 3, "fn": 4}}, "sourcery": {"strict": {"tp": 1, "fp": 4, "fn": 3}, "core": {"tp": 1, "fp": 4, "fn": 3}, "all": {"tp": 2, "fp": 4, "fn": 3}}}}, {"url": "https://github.com/keycloak/keycloak/pull/37038", "language": "Java", "pr_size": "small", "domain": "authentication", "change_type": "feature", "complexity": "complex", "difficulty": "very_subtle", "risk": "high", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "codeant-v2": {"strict": {"tp": 1, "fp": 5, "fn": 1}, "core": {"tp": 1, "fp": 5, "fn": 1}, "all": {"tp": 1, "fp": 5, "fn": 1}}, "coderabbit": {"strict": {"tp": 2, "fp": 8, "fn": 0}, "core": {"tp": 2, "fp": 8, "fn": 0}, "all": {"tp": 2, "fp": 8, "fn": 0}}, "qodo-extended-v2": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "greptile-v4-1": {"strict": {"tp": 2, "fp": 2, "fn": 0}, "core": {"tp": 2, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "gemini-v2": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "gitlab": {"strict": {"tp": 1, "fp": 2, "fn": 1}, "core": {"tp": 1, "fp": 2, "fn": 1}, "all": {"tp": 1, "fp": 2, "fn": 1}}, "copilot-v2": {"strict": {"tp": 2, "fp": 3, "fn": 0}, "core": {"tp": 2, "fp": 3, "fn": 0}, "all": {"tp": 2, "fp": 3, "fn": 0}}, "augment": {"strict": {"tp": 2, "fp": 2, "fn": 0}, "core": {"tp": 2, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "bugbot": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "claude": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "claude-code": {"strict": {"tp": 1, "fp": 2, "fn": 1}, "core": {"tp": 1, "fp": 2, "fn": 1}, "all": {"tp": 1, "fp": 2, "fn": 1}}, "cubic-v2": {"strict": {"tp": 2, "fp": 2, "fn": 0}, "core": {"tp": 2, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "devin": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "kodus-v2": {"strict": {"tp": 2, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "macroscope": {"strict": {"tp": 2, "fp": 2, "fn": 0}, "core": {"tp": 2, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "qodo-v2": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "sourcery": {"strict": {"tp": 1, "fp": 4, "fn": 1}, "core": {"tp": 1, "fp": 4, "fn": 1}, "all": {"tp": 1, "fp": 4, "fn": 1}}}}, {"url": "https://github.com/keycloak/keycloak/pull/33832", "language": "Java", "pr_size": "medium", "domain": "authentication", "change_type": "feature", "complexity": "moderate", "difficulty": "subtle", "risk": "high", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 0, "fp": 1, "fn": 2}, "core": {"tp": 1, "fp": 1, "fn": 2}, "all": {"tp": 1, "fp": 1, "fn": 3}}, "codeant-v2": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 2}, "all": {"tp": 2, "fp": 1, "fn": 2}}, "coderabbit": {"strict": {"tp": 2, "fp": 4, "fn": 0}, "core": {"tp": 3, "fp": 4, "fn": 0}, "all": {"tp": 4, "fp": 4, "fn": 0}}, "qodo-extended-v2": {"strict": {"tp": 0, "fp": 2, "fn": 2}, "core": {"tp": 1, "fp": 2, "fn": 2}, "all": {"tp": 1, "fp": 2, "fn": 3}}, "greptile-v4-1": {"strict": {"tp": 0, "fp": 3, "fn": 2}, "core": {"tp": 1, "fp": 3, "fn": 2}, "all": {"tp": 1, "fp": 3, "fn": 3}}, "gemini-v2": {"strict": {"tp": 2, "fp": 1, "fn": 0}, "core": {"tp": 3, "fp": 1, "fn": 0}, "all": {"tp": 3, "fp": 1, "fn": 1}}, "gitlab": {"strict": {"tp": 0, "fp": 2, "fn": 2}, "core": {"tp": 1, "fp": 2, "fn": 2}, "all": {"tp": 1, "fp": 2, "fn": 3}}, "copilot-v2": {"strict": {"tp": 1, "fp": 6, "fn": 1}, "core": {"tp": 2, "fp": 6, "fn": 1}, "all": {"tp": 3, "fp": 6, "fn": 1}}, "augment": {"strict": {"tp": 1, "fp": 2, "fn": 1}, "core": {"tp": 1, "fp": 2, "fn": 2}, "all": {"tp": 1, "fp": 2, "fn": 3}}, "bugbot": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 3}}, "claude": {"strict": {"tp": 1, "fp": 4, "fn": 1}, "core": {"tp": 2, "fp": 4, "fn": 1}, "all": {"tp": 2, "fp": 4, "fn": 2}}, "claude-code": {"strict": {"tp": 0, "fp": 1, "fn": 2}, "core": {"tp": 1, "fp": 1, "fn": 2}, "all": {"tp": 2, "fp": 1, "fn": 2}}, "cubic-v2": {"strict": {"tp": 1, "fp": 2, "fn": 1}, "core": {"tp": 2, "fp": 2, "fn": 1}, "all": {"tp": 2, "fp": 2, "fn": 2}}, "devin": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 4}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 3}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 4}}, "kodus-v2": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 4}}, "macroscope": {"strict": {"tp": 2, "fp": 4, "fn": 0}, "core": {"tp": 2, "fp": 4, "fn": 1}, "all": {"tp": 2, "fp": 4, "fn": 2}}, "qodo-v2": {"strict": {"tp": 1, "fp": 2, "fn": 1}, "core": {"tp": 1, "fp": 2, "fn": 2}, "all": {"tp": 1, "fp": 2, "fn": 3}}, "sourcery": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 1}, "all": {"tp": 2, "fp": 1, "fn": 2}}}}, {"url": "https://github.com/keycloak/keycloak/pull/40940", "language": "Java", "pr_size": "small", "domain": "concurrency", "change_type": "bug_fix", "complexity": "moderate", "difficulty": "subtle", "risk": "high", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "codeant-v2": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "coderabbit": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "qodo-extended-v2": {"strict": {"tp": 2, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "greptile-v4-1": {"strict": {"tp": 1, "fp": 2, "fn": 1}, "core": {"tp": 1, "fp": 2, "fn": 1}, "all": {"tp": 1, "fp": 2, "fn": 1}}, "gemini-v2": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "gitlab": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "copilot-v2": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "augment": {"strict": {"tp": 2, "fp": 2, "fn": 0}, "core": {"tp": 2, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "bugbot": {"strict": {"tp": 1, "fp": 2, "fn": 1}, "core": {"tp": 1, "fp": 2, "fn": 1}, "all": {"tp": 1, "fp": 2, "fn": 1}}, "claude": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "claude-code": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "cubic-v2": {"strict": {"tp": 2, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "devin": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "graphite": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "kodus-v2": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "macroscope": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "qodo-v2": {"strict": {"tp": 2, "fp": 2, "fn": 0}, "core": {"tp": 2, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "sourcery": {"strict": {"tp": 1, "fp": 2, "fn": 1}, "core": {"tp": 1, "fp": 2, "fn": 1}, "all": {"tp": 1, "fp": 2, "fn": 1}}}}, {"url": "https://github.com/ai-code-review-evaluation/keycloak-greptile/pull/1", "language": "Java", "pr_size": "medium", "domain": "authentication", "change_type": "bug_fix", "complexity": "moderate", "difficulty": "subtle", "risk": "medium", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 1, "fp": 2, "fn": 0}, "core": {"tp": 1, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "codeant-v2": {"strict": {"tp": 1, "fp": 2, "fn": 0}, "core": {"tp": 1, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "coderabbit": {"strict": {"tp": 1, "fp": 3, "fn": 0}, "core": {"tp": 1, "fp": 3, "fn": 0}, "all": {"tp": 1, "fp": 3, "fn": 1}}, "qodo-extended-v2": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 1, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "greptile-v4-1": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 1, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "gemini-v2": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 1, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "gitlab": {"strict": {"tp": 1, "fp": 2, "fn": 0}, "core": {"tp": 1, "fp": 2, "fn": 0}, "all": {"tp": 1, "fp": 2, "fn": 1}}, "copilot-v2": {"strict": {"tp": 1, "fp": 2, "fn": 0}, "core": {"tp": 1, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "augment": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "bugbot": {"strict": {"tp": 0, "fp": 2, "fn": 1}, "core": {"tp": 0, "fp": 2, "fn": 1}, "all": {"tp": 0, "fp": 2, "fn": 2}}, "claude": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 0}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "claude-code": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 0}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "cubic-v2": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 1, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "devin": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "kg": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 0}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "kodus-v2": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "macroscope": {"strict": {"tp": 0, "fp": 3, "fn": 1}, "core": {"tp": 0, "fp": 3, "fn": 1}, "all": {"tp": 1, "fp": 3, "fn": 1}}, "qodo-v2": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 1, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "sourcery": {"strict": {"tp": 0, "fp": 3, "fn": 1}, "core": {"tp": 0, "fp": 3, "fn": 1}, "all": {"tp": 1, "fp": 3, "fn": 1}}}}, {"url": "https://github.com/getsentry/sentry/pull/93824", "language": "Python", "pr_size": "large", "domain": "concurrency", "change_type": "feature", "complexity": "complex", "difficulty": "subtle", "risk": "high", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 0, "fp": 2, "fn": 3}, "core": {"tp": 0, "fp": 2, "fn": 5}, "all": {"tp": 0, "fp": 2, "fn": 5}}, "codeant-v2": {"strict": {"tp": 0, "fp": 1, "fn": 3}, "core": {"tp": 2, "fp": 1, "fn": 3}, "all": {"tp": 2, "fp": 1, "fn": 3}}, "coderabbit": {"strict": {"tp": 1, "fp": 3, "fn": 2}, "core": {"tp": 1, "fp": 3, "fn": 4}, "all": {"tp": 1, "fp": 3, "fn": 4}}, "qodo-extended-v2": {"strict": {"tp": 2, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 3}, "all": {"tp": 2, "fp": 1, "fn": 3}}, "greptile-v4-1": {"strict": {"tp": 1, "fp": 1, "fn": 2}, "core": {"tp": 1, "fp": 1, "fn": 4}, "all": {"tp": 1, "fp": 1, "fn": 4}}, "gemini-v2": {"strict": {"tp": 2, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 3}, "all": {"tp": 2, "fp": 1, "fn": 3}}, "gitlab": {"strict": {"tp": 0, "fp": 2, "fn": 3}, "core": {"tp": 0, "fp": 2, "fn": 5}, "all": {"tp": 0, "fp": 2, "fn": 5}}, "copilot-v2": {"strict": {"tp": 1, "fp": 4, "fn": 2}, "core": {"tp": 3, "fp": 4, "fn": 2}, "all": {"tp": 3, "fp": 4, "fn": 2}}, "augment": {"strict": {"tp": 3, "fp": 1, "fn": 0}, "core": {"tp": 5, "fp": 1, "fn": 0}, "all": {"tp": 5, "fp": 1, "fn": 0}}, "bugbot": {"strict": {"tp": 1, "fp": 0, "fn": 2}, "core": {"tp": 1, "fp": 0, "fn": 4}, "all": {"tp": 1, "fp": 0, "fn": 4}}, "claude": {"strict": {"tp": 0, "fp": 3, "fn": 3}, "core": {"tp": 0, "fp": 3, "fn": 5}, "all": {"tp": 0, "fp": 3, "fn": 5}}, "claude-code": {"strict": {"tp": 3, "fp": 3, "fn": 0}, "core": {"tp": 3, "fp": 3, "fn": 2}, "all": {"tp": 3, "fp": 3, "fn": 2}}, "cubic-v2": {"strict": {"tp": 3, "fp": 1, "fn": 0}, "core": {"tp": 5, "fp": 1, "fn": 0}, "all": {"tp": 5, "fp": 1, "fn": 0}}, "devin": {"strict": {"tp": 1, "fp": 0, "fn": 2}, "core": {"tp": 1, "fp": 0, "fn": 4}, "all": {"tp": 1, "fp": 0, "fn": 4}}, "graphite": {"strict": {"tp": 1, "fp": 0, "fn": 2}, "core": {"tp": 1, "fp": 0, "fn": 4}, "all": {"tp": 1, "fp": 0, "fn": 4}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 3}, "core": {"tp": 0, "fp": 0, "fn": 5}, "all": {"tp": 0, "fp": 0, "fn": 5}}, "kodus-v2": {"strict": {"tp": 0, "fp": 0, "fn": 3}, "core": {"tp": 0, "fp": 0, "fn": 5}, "all": {"tp": 0, "fp": 0, "fn": 5}}, "macroscope": {"strict": {"tp": 1, "fp": 1, "fn": 2}, "core": {"tp": 3, "fp": 1, "fn": 2}, "all": {"tp": 3, "fp": 1, "fn": 2}}, "qodo-v2": {"strict": {"tp": 2, "fp": 3, "fn": 1}, "core": {"tp": 4, "fp": 3, "fn": 1}, "all": {"tp": 4, "fp": 3, "fn": 1}}, "sourcery": {"strict": {"tp": 2, "fp": 2, "fn": 1}, "core": {"tp": 4, "fp": 2, "fn": 1}, "all": {"tp": 4, "fp": 2, "fn": 1}}}}, {"url": "https://github.com/ai-code-review-evaluation/sentry-greptile/pull/5", "language": "Python", "pr_size": "large", "domain": "API", "change_type": "feature", "complexity": "moderate", "difficulty": "subtle", "risk": "medium", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 3, "fp": 1, "fn": 3}, "core": {"tp": 3, "fp": 1, "fn": 3}, "all": {"tp": 3, "fp": 1, "fn": 3}}, "codeant-v2": {"strict": {"tp": 2, "fp": 5, "fn": 4}, "core": {"tp": 2, "fp": 5, "fn": 4}, "all": {"tp": 2, "fp": 5, "fn": 4}}, "coderabbit": {"strict": {"tp": 4, "fp": 13, "fn": 2}, "core": {"tp": 4, "fp": 13, "fn": 2}, "all": {"tp": 4, "fp": 13, "fn": 2}}, "qodo-extended-v2": {"strict": {"tp": 2, "fp": 0, "fn": 4}, "core": {"tp": 2, "fp": 0, "fn": 4}, "all": {"tp": 2, "fp": 0, "fn": 4}}, "greptile-v4-1": {"strict": {"tp": 1, "fp": 0, "fn": 5}, "core": {"tp": 1, "fp": 0, "fn": 5}, "all": {"tp": 1, "fp": 0, "fn": 5}}, "gemini-v2": {"strict": {"tp": 2, "fp": 1, "fn": 4}, "core": {"tp": 2, "fp": 1, "fn": 4}, "all": {"tp": 2, "fp": 1, "fn": 4}}, "gitlab": {"strict": {"tp": 2, "fp": 1, "fn": 4}, "core": {"tp": 2, "fp": 1, "fn": 4}, "all": {"tp": 2, "fp": 1, "fn": 4}}, "copilot-v2": {"strict": {"tp": 3, "fp": 5, "fn": 3}, "core": {"tp": 3, "fp": 5, "fn": 3}, "all": {"tp": 3, "fp": 5, "fn": 3}}, "augment": {"strict": {"tp": 3, "fp": 2, "fn": 3}, "core": {"tp": 3, "fp": 2, "fn": 3}, "all": {"tp": 3, "fp": 2, "fn": 3}}, "bugbot": {"strict": {"tp": 2, "fp": 0, "fn": 4}, "core": {"tp": 2, "fp": 0, "fn": 4}, "all": {"tp": 2, "fp": 0, "fn": 4}}, "claude": {"strict": {"tp": 0, "fp": 0, "fn": 6}, "core": {"tp": 0, "fp": 0, "fn": 6}, "all": {"tp": 0, "fp": 0, "fn": 6}}, "claude-code": {"strict": {"tp": 0, "fp": 2, "fn": 6}, "core": {"tp": 0, "fp": 2, "fn": 6}, "all": {"tp": 0, "fp": 2, "fn": 6}}, "cubic-v2": {"strict": {"tp": 2, "fp": 2, "fn": 4}, "core": {"tp": 2, "fp": 2, "fn": 4}, "all": {"tp": 2, "fp": 2, "fn": 4}}, "devin": {"strict": {"tp": 2, "fp": 1, "fn": 4}, "core": {"tp": 2, "fp": 1, "fn": 4}, "all": {"tp": 2, "fp": 1, "fn": 4}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 6}, "core": {"tp": 0, "fp": 0, "fn": 6}, "all": {"tp": 0, "fp": 0, "fn": 6}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 6}, "core": {"tp": 0, "fp": 0, "fn": 6}, "all": {"tp": 0, "fp": 0, "fn": 6}}, "kodus-v2": {"strict": {"tp": 2, "fp": 1, "fn": 4}, "core": {"tp": 2, "fp": 1, "fn": 4}, "all": {"tp": 2, "fp": 1, "fn": 4}}, "macroscope": {"strict": {"tp": 2, "fp": 3, "fn": 4}, "core": {"tp": 2, "fp": 3, "fn": 4}, "all": {"tp": 2, "fp": 3, "fn": 4}}, "qodo-v2": {"strict": {"tp": 2, "fp": 0, "fn": 4}, "core": {"tp": 2, "fp": 0, "fn": 4}, "all": {"tp": 2, "fp": 0, "fn": 4}}, "sourcery": {"strict": {"tp": 2, "fp": 2, "fn": 4}, "core": {"tp": 2, "fp": 2, "fn": 4}, "all": {"tp": 2, "fp": 2, "fn": 4}}}}, {"url": "https://github.com/ai-code-review-evaluation/sentry-greptile/pull/1", "language": "Python", "pr_size": "medium", "domain": "API", "change_type": "performance", "complexity": "moderate", "difficulty": "subtle", "risk": "high", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 1, "fp": 1, "fn": 3}, "core": {"tp": 1, "fp": 1, "fn": 3}, "all": {"tp": 1, "fp": 1, "fn": 3}}, "codeant-v2": {"strict": {"tp": 1, "fp": 1, "fn": 3}, "core": {"tp": 1, "fp": 1, "fn": 3}, "all": {"tp": 1, "fp": 1, "fn": 3}}, "coderabbit": {"strict": {"tp": 3, "fp": 2, "fn": 1}, "core": {"tp": 3, "fp": 2, "fn": 1}, "all": {"tp": 3, "fp": 2, "fn": 1}}, "qodo-extended-v2": {"strict": {"tp": 3, "fp": 0, "fn": 1}, "core": {"tp": 3, "fp": 0, "fn": 1}, "all": {"tp": 3, "fp": 0, "fn": 1}}, "greptile-v4-1": {"strict": {"tp": 3, "fp": 2, "fn": 1}, "core": {"tp": 3, "fp": 2, "fn": 1}, "all": {"tp": 3, "fp": 2, "fn": 1}}, "gemini-v2": {"strict": {"tp": 3, "fp": 2, "fn": 1}, "core": {"tp": 3, "fp": 2, "fn": 1}, "all": {"tp": 3, "fp": 2, "fn": 1}}, "gitlab": {"strict": {"tp": 3, "fp": 1, "fn": 1}, "core": {"tp": 3, "fp": 1, "fn": 1}, "all": {"tp": 3, "fp": 1, "fn": 1}}, "copilot-v2": {"strict": {"tp": 2, "fp": 2, "fn": 2}, "core": {"tp": 2, "fp": 2, "fn": 2}, "all": {"tp": 2, "fp": 2, "fn": 2}}, "augment": {"strict": {"tp": 3, "fp": 1, "fn": 1}, "core": {"tp": 3, "fp": 1, "fn": 1}, "all": {"tp": 3, "fp": 1, "fn": 1}}, "bugbot": {"strict": {"tp": 3, "fp": 1, "fn": 1}, "core": {"tp": 3, "fp": 1, "fn": 1}, "all": {"tp": 3, "fp": 1, "fn": 1}}, "claude": {"strict": {"tp": 2, "fp": 6, "fn": 2}, "core": {"tp": 2, "fp": 6, "fn": 2}, "all": {"tp": 2, "fp": 6, "fn": 2}}, "claude-code": {"strict": {"tp": 2, "fp": 1, "fn": 2}, "core": {"tp": 2, "fp": 1, "fn": 2}, "all": {"tp": 2, "fp": 1, "fn": 2}}, "cubic-v2": {"strict": {"tp": 3, "fp": 0, "fn": 1}, "core": {"tp": 3, "fp": 0, "fn": 1}, "all": {"tp": 3, "fp": 0, "fn": 1}}, "devin": {"strict": {"tp": 3, "fp": 2, "fn": 1}, "core": {"tp": 3, "fp": 2, "fn": 1}, "all": {"tp": 3, "fp": 2, "fn": 1}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 4}, "core": {"tp": 0, "fp": 0, "fn": 4}, "all": {"tp": 0, "fp": 0, "fn": 4}}, "kg": {"strict": {"tp": 1, "fp": 1, "fn": 3}, "core": {"tp": 1, "fp": 1, "fn": 3}, "all": {"tp": 1, "fp": 1, "fn": 3}}, "kodus-v2": {"strict": {"tp": 1, "fp": 4, "fn": 3}, "core": {"tp": 1, "fp": 4, "fn": 3}, "all": {"tp": 1, "fp": 4, "fn": 3}}, "macroscope": {"strict": {"tp": 2, "fp": 1, "fn": 2}, "core": {"tp": 2, "fp": 1, "fn": 2}, "all": {"tp": 2, "fp": 1, "fn": 2}}, "qodo-v2": {"strict": {"tp": 2, "fp": 1, "fn": 2}, "core": {"tp": 2, "fp": 1, "fn": 2}, "all": {"tp": 2, "fp": 1, "fn": 2}}, "sourcery": {"strict": {"tp": 2, "fp": 3, "fn": 2}, "core": {"tp": 2, "fp": 3, "fn": 2}, "all": {"tp": 2, "fp": 3, "fn": 2}}}}, {"url": "https://github.com/grafana/grafana/pull/97529", "language": "Go", "pr_size": "small", "domain": "concurrency", "change_type": "performance", "complexity": "complex", "difficulty": "very_subtle", "risk": "critical", "context": "cross_file", "concern": "reliability", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "codeant-v2": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "coderabbit": {"strict": {"tp": 1, "fp": 3, "fn": 1}, "core": {"tp": 1, "fp": 3, "fn": 1}, "all": {"tp": 1, "fp": 3, "fn": 1}}, "qodo-extended-v2": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "greptile-v4-1": {"strict": {"tp": 2, "fp": 2, "fn": 0}, "core": {"tp": 2, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "gemini-v2": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "gitlab": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "copilot-v2": {"strict": {"tp": 1, "fp": 7, "fn": 1}, "core": {"tp": 1, "fp": 7, "fn": 1}, "all": {"tp": 1, "fp": 7, "fn": 1}}, "augment": {"strict": {"tp": 2, "fp": 2, "fn": 0}, "core": {"tp": 2, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "bugbot": {"strict": {"tp": 1, "fp": 3, "fn": 1}, "core": {"tp": 1, "fp": 3, "fn": 1}, "all": {"tp": 1, "fp": 3, "fn": 1}}, "claude": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "claude-code": {"strict": {"tp": 1, "fp": 4, "fn": 1}, "core": {"tp": 1, "fp": 4, "fn": 1}, "all": {"tp": 1, "fp": 4, "fn": 1}}, "cubic-v2": {"strict": {"tp": 2, "fp": 2, "fn": 0}, "core": {"tp": 2, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "devin": {"strict": {"tp": 0, "fp": 2, "fn": 2}, "core": {"tp": 0, "fp": 2, "fn": 2}, "all": {"tp": 0, "fp": 2, "fn": 2}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "kg": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "kodus-v2": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "macroscope": {"strict": {"tp": 1, "fp": 2, "fn": 1}, "core": {"tp": 1, "fp": 2, "fn": 1}, "all": {"tp": 1, "fp": 2, "fn": 1}}, "qodo-v2": {"strict": {"tp": 2, "fp": 4, "fn": 0}, "core": {"tp": 2, "fp": 4, "fn": 0}, "all": {"tp": 2, "fp": 4, "fn": 0}}, "sourcery": {"strict": {"tp": 2, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}}}, {"url": "https://github.com/getsentry/sentry/pull/80168", "language": "Python", "pr_size": "small", "domain": "data_processing", "change_type": "feature", "complexity": "moderate", "difficulty": "moderate", "risk": "high", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 0, "fp": 1, "fn": 1}, "core": {"tp": 0, "fp": 1, "fn": 3}, "all": {"tp": 0, "fp": 1, "fn": 3}}, "codeant-v2": {"strict": {"tp": 0, "fp": 1, "fn": 1}, "core": {"tp": 0, "fp": 1, "fn": 3}, "all": {"tp": 0, "fp": 1, "fn": 3}}, "coderabbit": {"strict": {"tp": 0, "fp": 2, "fn": 1}, "core": {"tp": 2, "fp": 2, "fn": 1}, "all": {"tp": 2, "fp": 2, "fn": 1}}, "qodo-extended-v2": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 1}, "all": {"tp": 2, "fp": 0, "fn": 1}}, "greptile-v4-1": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 1}, "all": {"tp": 2, "fp": 1, "fn": 1}}, "gemini-v2": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 2}}, "gitlab": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 2}}, "copilot-v2": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 1}, "all": {"tp": 2, "fp": 0, "fn": 1}}, "augment": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 3, "fp": 1, "fn": 0}, "all": {"tp": 3, "fp": 1, "fn": 0}}, "bugbot": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 2}}, "claude": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 3}}, "claude-code": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 3, "fp": 1, "fn": 0}, "all": {"tp": 3, "fp": 1, "fn": 0}}, "cubic-v2": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 3, "fp": 1, "fn": 0}, "all": {"tp": 3, "fp": 1, "fn": 0}}, "devin": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 1, "fp": 1, "fn": 2}, "all": {"tp": 1, "fp": 1, "fn": 2}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 3}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 3}}, "kodus-v2": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 3}}, "macroscope": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 3}}, "qodo-v2": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 2}}, "sourcery": {"strict": {"tp": 1, "fp": 2, "fn": 0}, "core": {"tp": 2, "fp": 2, "fn": 1}, "all": {"tp": 2, "fp": 2, "fn": 1}}}}, {"url": "https://github.com/getsentry/sentry/pull/80528", "language": "Python", "pr_size": "small", "domain": "scheduling", "change_type": "refactoring", "complexity": "moderate", "difficulty": "subtle", "risk": "medium", "context": "file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "codeant-v2": {"strict": {"tp": 0, "fp": 4, "fn": 1}, "core": {"tp": 0, "fp": 4, "fn": 2}, "all": {"tp": 0, "fp": 4, "fn": 2}}, "coderabbit": {"strict": {"tp": 1, "fp": 3, "fn": 0}, "core": {"tp": 1, "fp": 3, "fn": 1}, "all": {"tp": 1, "fp": 3, "fn": 1}}, "qodo-extended-v2": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "greptile-v4-1": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "gemini-v2": {"strict": {"tp": 1, "fp": 5, "fn": 0}, "core": {"tp": 1, "fp": 5, "fn": 1}, "all": {"tp": 1, "fp": 5, "fn": 1}}, "gitlab": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "copilot-v2": {"strict": {"tp": 1, "fp": 2, "fn": 0}, "core": {"tp": 1, "fp": 2, "fn": 1}, "all": {"tp": 1, "fp": 2, "fn": 1}}, "augment": {"strict": {"tp": 1, "fp": 2, "fn": 0}, "core": {"tp": 1, "fp": 2, "fn": 1}, "all": {"tp": 1, "fp": 2, "fn": 1}}, "bugbot": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "claude": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "claude-code": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "cubic-v2": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "devin": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "graphite": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "kg": {"strict": {"tp": 0, "fp": 1, "fn": 1}, "core": {"tp": 0, "fp": 1, "fn": 2}, "all": {"tp": 0, "fp": 1, "fn": 2}}, "kodus-v2": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "macroscope": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "qodo-v2": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "sourcery": {"strict": {"tp": 1, "fp": 2, "fn": 0}, "core": {"tp": 1, "fp": 2, "fn": 1}, "all": {"tp": 1, "fp": 2, "fn": 1}}}}, {"url": "https://github.com/getsentry/sentry/pull/77754", "language": "Python", "pr_size": "medium", "domain": "serialization", "change_type": "feature", "complexity": "moderate", "difficulty": "moderate", "risk": "medium", "context": "file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 3}}, "codeant-v2": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 4}}, "coderabbit": {"strict": {"tp": 1, "fp": 2, "fn": 1}, "core": {"tp": 1, "fp": 2, "fn": 1}, "all": {"tp": 3, "fp": 2, "fn": 1}}, "qodo-extended-v2": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 3}}, "greptile-v4-1": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 3}}, "gemini-v2": {"strict": {"tp": 1, "fp": 3, "fn": 1}, "core": {"tp": 1, "fp": 3, "fn": 1}, "all": {"tp": 1, "fp": 3, "fn": 3}}, "gitlab": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 2, "fp": 1, "fn": 2}}, "copilot-v2": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 2, "fp": 1, "fn": 2}}, "augment": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 3}}, "bugbot": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 3}}, "claude": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 2, "fp": 0, "fn": 2}}, "claude-code": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 3}}, "cubic-v2": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 2}}, "devin": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 3}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 4}}, "kg": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 3}}, "kodus-v2": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 3}}, "macroscope": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 3}}, "qodo-v2": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 3}}, "sourcery": {"strict": {"tp": 1, "fp": 3, "fn": 1}, "core": {"tp": 1, "fp": 3, "fn": 1}, "all": {"tp": 2, "fp": 3, "fn": 2}}}}, {"url": "https://github.com/getsentry/sentry/pull/95633", "language": "Python", "pr_size": "medium", "domain": "concurrency", "change_type": "feature", "complexity": "moderate", "difficulty": "moderate", "risk": "medium", "context": "file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 0, "fp": 3, "fn": 0}, "core": {"tp": 0, "fp": 3, "fn": 1}, "all": {"tp": 0, "fp": 3, "fn": 2}}, "codeant-v2": {"strict": {"tp": 0, "fp": 1, "fn": 0}, "core": {"tp": 0, "fp": 1, "fn": 1}, "all": {"tp": 0, "fp": 1, "fn": 2}}, "coderabbit": {"strict": {"tp": 0, "fp": 7, "fn": 0}, "core": {"tp": 0, "fp": 7, "fn": 1}, "all": {"tp": 0, "fp": 7, "fn": 2}}, "qodo-extended-v2": {"strict": {"tp": 0, "fp": 8, "fn": 0}, "core": {"tp": 0, "fp": 8, "fn": 1}, "all": {"tp": 0, "fp": 8, "fn": 2}}, "greptile-v4-1": {"strict": {"tp": 0, "fp": 5, "fn": 0}, "core": {"tp": 0, "fp": 5, "fn": 1}, "all": {"tp": 0, "fp": 5, "fn": 2}}, "gemini-v2": {"strict": {"tp": 0, "fp": 11, "fn": 0}, "core": {"tp": 0, "fp": 11, "fn": 1}, "all": {"tp": 0, "fp": 11, "fn": 2}}, "gitlab": {"strict": {"tp": 0, "fp": 5, "fn": 0}, "core": {"tp": 1, "fp": 5, "fn": 0}, "all": {"tp": 1, "fp": 5, "fn": 1}}, "copilot-v2": {"strict": {"tp": 0, "fp": 6, "fn": 0}, "core": {"tp": 0, "fp": 6, "fn": 1}, "all": {"tp": 0, "fp": 6, "fn": 2}}, "augment": {"strict": {"tp": 0, "fp": 4, "fn": 0}, "core": {"tp": 1, "fp": 4, "fn": 0}, "all": {"tp": 1, "fp": 4, "fn": 1}}, "bugbot": {"strict": {"tp": 0, "fp": 2, "fn": 0}, "core": {"tp": 0, "fp": 2, "fn": 1}, "all": {"tp": 0, "fp": 2, "fn": 2}}, "claude": {"strict": {"tp": 0, "fp": 0, "fn": 0}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "claude-code": {"strict": {"tp": 0, "fp": 2, "fn": 0}, "core": {"tp": 0, "fp": 2, "fn": 1}, "all": {"tp": 0, "fp": 2, "fn": 2}}, "cubic-v2": {"strict": {"tp": 0, "fp": 3, "fn": 0}, "core": {"tp": 0, "fp": 3, "fn": 1}, "all": {"tp": 0, "fp": 3, "fn": 2}}, "devin": {"strict": {"tp": 0, "fp": 4, "fn": 0}, "core": {"tp": 0, "fp": 4, "fn": 1}, "all": {"tp": 0, "fp": 4, "fn": 2}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 0}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "kg": {"strict": {"tp": 0, "fp": 2, "fn": 0}, "core": {"tp": 0, "fp": 2, "fn": 1}, "all": {"tp": 0, "fp": 2, "fn": 2}}, "kodus-v2": {"strict": {"tp": 0, "fp": 0, "fn": 0}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "macroscope": {"strict": {"tp": 0, "fp": 2, "fn": 0}, "core": {"tp": 0, "fp": 2, "fn": 1}, "all": {"tp": 0, "fp": 2, "fn": 2}}, "qodo-v2": {"strict": {"tp": 0, "fp": 4, "fn": 0}, "core": {"tp": 0, "fp": 4, "fn": 1}, "all": {"tp": 0, "fp": 4, "fn": 2}}, "sourcery": {"strict": {"tp": 0, "fp": 4, "fn": 0}, "core": {"tp": 1, "fp": 4, "fn": 0}, "all": {"tp": 1, "fp": 4, "fn": 1}}}}, {"url": "https://github.com/ai-code-review-evaluation/sentry-greptile/pull/2", "language": "Python", "pr_size": "medium", "domain": "data_processing", "change_type": "performance", "complexity": "moderate", "difficulty": "subtle", "risk": "high", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 3, "fp": 2, "fn": 1}, "core": {"tp": 3, "fp": 2, "fn": 1}, "all": {"tp": 3, "fp": 2, "fn": 1}}, "codeant-v2": {"strict": {"tp": 3, "fp": 1, "fn": 1}, "core": {"tp": 3, "fp": 1, "fn": 1}, "all": {"tp": 3, "fp": 1, "fn": 1}}, "coderabbit": {"strict": {"tp": 0, "fp": 0, "fn": 4}, "core": {"tp": 0, "fp": 0, "fn": 4}, "all": {"tp": 0, "fp": 0, "fn": 4}}, "qodo-extended-v2": {"strict": {"tp": 3, "fp": 4, "fn": 1}, "core": {"tp": 3, "fp": 4, "fn": 1}, "all": {"tp": 3, "fp": 4, "fn": 1}}, "greptile-v4-1": {"strict": {"tp": 2, "fp": 4, "fn": 2}, "core": {"tp": 2, "fp": 4, "fn": 2}, "all": {"tp": 2, "fp": 4, "fn": 2}}, "gemini-v2": {"strict": {"tp": 3, "fp": 5, "fn": 1}, "core": {"tp": 3, "fp": 5, "fn": 1}, "all": {"tp": 3, "fp": 5, "fn": 1}}, "gitlab": {"strict": {"tp": 4, "fp": 1, "fn": 0}, "core": {"tp": 4, "fp": 1, "fn": 0}, "all": {"tp": 4, "fp": 1, "fn": 0}}, "copilot-v2": {"strict": {"tp": 3, "fp": 10, "fn": 1}, "core": {"tp": 3, "fp": 10, "fn": 1}, "all": {"tp": 3, "fp": 10, "fn": 1}}, "augment": {"strict": {"tp": 3, "fp": 2, "fn": 1}, "core": {"tp": 3, "fp": 2, "fn": 1}, "all": {"tp": 3, "fp": 2, "fn": 1}}, "bugbot": {"strict": {"tp": 4, "fp": 0, "fn": 0}, "core": {"tp": 4, "fp": 0, "fn": 0}, "all": {"tp": 4, "fp": 0, "fn": 0}}, "claude": {"strict": {"tp": 3, "fp": 5, "fn": 1}, "core": {"tp": 3, "fp": 5, "fn": 1}, "all": {"tp": 3, "fp": 5, "fn": 1}}, "claude-code": {"strict": {"tp": 3, "fp": 2, "fn": 1}, "core": {"tp": 3, "fp": 2, "fn": 1}, "all": {"tp": 3, "fp": 2, "fn": 1}}, "cubic-v2": {"strict": {"tp": 3, "fp": 0, "fn": 1}, "core": {"tp": 3, "fp": 0, "fn": 1}, "all": {"tp": 3, "fp": 0, "fn": 1}}, "devin": {"strict": {"tp": 0, "fp": 0, "fn": 4}, "core": {"tp": 0, "fp": 0, "fn": 4}, "all": {"tp": 0, "fp": 0, "fn": 4}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 4}, "core": {"tp": 0, "fp": 0, "fn": 4}, "all": {"tp": 0, "fp": 0, "fn": 4}}, "kg": {"strict": {"tp": 2, "fp": 2, "fn": 2}, "core": {"tp": 2, "fp": 2, "fn": 2}, "all": {"tp": 2, "fp": 2, "fn": 2}}, "kodus-v2": {"strict": {"tp": 2, "fp": 3, "fn": 2}, "core": {"tp": 2, "fp": 3, "fn": 2}, "all": {"tp": 2, "fp": 3, "fn": 2}}, "macroscope": {"strict": {"tp": 3, "fp": 0, "fn": 1}, "core": {"tp": 3, "fp": 0, "fn": 1}, "all": {"tp": 3, "fp": 0, "fn": 1}}, "qodo-v2": {"strict": {"tp": 4, "fp": 0, "fn": 0}, "core": {"tp": 4, "fp": 0, "fn": 0}, "all": {"tp": 4, "fp": 0, "fn": 0}}, "sourcery": {"strict": {"tp": 3, "fp": 7, "fn": 1}, "core": {"tp": 3, "fp": 7, "fn": 1}, "all": {"tp": 3, "fp": 7, "fn": 1}}}}, {"url": "https://github.com/ai-code-review-evaluation/sentry-greptile/pull/3", "language": "Python", "pr_size": "medium", "domain": "caching", "change_type": "feature", "complexity": "complex", "difficulty": "subtle", "risk": "medium", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 0, "fp": 6, "fn": 3}, "core": {"tp": 0, "fp": 6, "fn": 3}, "all": {"tp": 0, "fp": 6, "fn": 3}}, "codeant-v2": {"strict": {"tp": 3, "fp": 1, "fn": 0}, "core": {"tp": 3, "fp": 1, "fn": 0}, "all": {"tp": 3, "fp": 1, "fn": 0}}, "coderabbit": {"strict": {"tp": 2, "fp": 3, "fn": 1}, "core": {"tp": 2, "fp": 3, "fn": 1}, "all": {"tp": 2, "fp": 3, "fn": 1}}, "qodo-extended-v2": {"strict": {"tp": 3, "fp": 0, "fn": 0}, "core": {"tp": 3, "fp": 0, "fn": 0}, "all": {"tp": 3, "fp": 0, "fn": 0}}, "greptile-v4-1": {"strict": {"tp": 2, "fp": 2, "fn": 1}, "core": {"tp": 2, "fp": 2, "fn": 1}, "all": {"tp": 2, "fp": 2, "fn": 1}}, "gemini-v2": {"strict": {"tp": 2, "fp": 2, "fn": 1}, "core": {"tp": 2, "fp": 2, "fn": 1}, "all": {"tp": 2, "fp": 2, "fn": 1}}, "gitlab": {"strict": {"tp": 2, "fp": 4, "fn": 1}, "core": {"tp": 2, "fp": 4, "fn": 1}, "all": {"tp": 2, "fp": 4, "fn": 1}}, "copilot-v2": {"strict": {"tp": 1, "fp": 9, "fn": 2}, "core": {"tp": 1, "fp": 9, "fn": 2}, "all": {"tp": 1, "fp": 9, "fn": 2}}, "augment": {"strict": {"tp": 1, "fp": 2, "fn": 2}, "core": {"tp": 1, "fp": 2, "fn": 2}, "all": {"tp": 1, "fp": 2, "fn": 2}}, "bugbot": {"strict": {"tp": 0, "fp": 3, "fn": 3}, "core": {"tp": 0, "fp": 3, "fn": 3}, "all": {"tp": 0, "fp": 3, "fn": 3}}, "claude": {"strict": {"tp": 1, "fp": 0, "fn": 2}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 2}}, "claude-code": {"strict": {"tp": 1, "fp": 2, "fn": 2}, "core": {"tp": 1, "fp": 2, "fn": 2}, "all": {"tp": 1, "fp": 2, "fn": 2}}, "cubic-v2": {"strict": {"tp": 2, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 1}, "all": {"tp": 2, "fp": 1, "fn": 1}}, "devin": {"strict": {"tp": 2, "fp": 0, "fn": 1}, "core": {"tp": 2, "fp": 0, "fn": 1}, "all": {"tp": 2, "fp": 0, "fn": 1}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 3}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 3}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 3}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 3}}, "kodus-v2": {"strict": {"tp": 2, "fp": 2, "fn": 1}, "core": {"tp": 2, "fp": 2, "fn": 1}, "all": {"tp": 2, "fp": 2, "fn": 1}}, "macroscope": {"strict": {"tp": 2, "fp": 0, "fn": 1}, "core": {"tp": 2, "fp": 0, "fn": 1}, "all": {"tp": 2, "fp": 0, "fn": 1}}, "qodo-v2": {"strict": {"tp": 2, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 1}, "all": {"tp": 2, "fp": 1, "fn": 1}}, "sourcery": {"strict": {"tp": 2, "fp": 6, "fn": 1}, "core": {"tp": 2, "fp": 6, "fn": 1}, "all": {"tp": 2, "fp": 6, "fn": 1}}}}, {"url": "https://github.com/grafana/grafana/pull/103633", "language": "Go", "pr_size": "small", "domain": "caching", "change_type": "performance", "complexity": "moderate", "difficulty": "subtle", "risk": "high", "context": "cross_file", "concern": "security", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 1, "fp": 2, "fn": 1}, "core": {"tp": 1, "fp": 2, "fn": 2}, "all": {"tp": 1, "fp": 2, "fn": 2}}, "codeant-v2": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 2}}, "coderabbit": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 2}}, "qodo-extended-v2": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 2, "fp": 0, "fn": 1}, "all": {"tp": 2, "fp": 0, "fn": 1}}, "greptile-v4-1": {"strict": {"tp": 1, "fp": 2, "fn": 1}, "core": {"tp": 1, "fp": 2, "fn": 2}, "all": {"tp": 1, "fp": 2, "fn": 2}}, "gemini-v2": {"strict": {"tp": 0, "fp": 3, "fn": 2}, "core": {"tp": 0, "fp": 3, "fn": 3}, "all": {"tp": 0, "fp": 3, "fn": 3}}, "gitlab": {"strict": {"tp": 1, "fp": 2, "fn": 1}, "core": {"tp": 1, "fp": 2, "fn": 2}, "all": {"tp": 1, "fp": 2, "fn": 2}}, "copilot-v2": {"strict": {"tp": 0, "fp": 1, "fn": 2}, "core": {"tp": 1, "fp": 1, "fn": 2}, "all": {"tp": 1, "fp": 1, "fn": 2}}, "augment": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 2, "fp": 0, "fn": 1}, "all": {"tp": 2, "fp": 0, "fn": 1}}, "bugbot": {"strict": {"tp": 0, "fp": 1, "fn": 2}, "core": {"tp": 0, "fp": 1, "fn": 3}, "all": {"tp": 0, "fp": 1, "fn": 3}}, "claude": {"strict": {"tp": 0, "fp": 6, "fn": 2}, "core": {"tp": 0, "fp": 6, "fn": 3}, "all": {"tp": 0, "fp": 6, "fn": 3}}, "claude-code": {"strict": {"tp": 0, "fp": 2, "fn": 2}, "core": {"tp": 0, "fp": 2, "fn": 3}, "all": {"tp": 0, "fp": 2, "fn": 3}}, "cubic-v2": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 1}, "all": {"tp": 2, "fp": 1, "fn": 1}}, "devin": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 1}, "all": {"tp": 2, "fp": 1, "fn": 1}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 3}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 3}}, "kodus-v2": {"strict": {"tp": 0, "fp": 1, "fn": 2}, "core": {"tp": 0, "fp": 1, "fn": 3}, "all": {"tp": 0, "fp": 1, "fn": 3}}, "macroscope": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 2}, "all": {"tp": 1, "fp": 1, "fn": 2}}, "qodo-v2": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 1}, "all": {"tp": 2, "fp": 1, "fn": 1}}, "sourcery": {"strict": {"tp": 0, "fp": 2, "fn": 2}, "core": {"tp": 0, "fp": 2, "fn": 3}, "all": {"tp": 0, "fp": 2, "fn": 3}}}}, {"url": "https://github.com/getsentry/sentry/pull/67876", "language": "Python", "pr_size": "small", "domain": "authentication", "change_type": "security_patch", "complexity": "moderate", "difficulty": "subtle", "risk": "high", "context": "cross_file", "concern": "security", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 1, "fp": 1, "fn": 2}, "core": {"tp": 1, "fp": 1, "fn": 2}, "all": {"tp": 1, "fp": 1, "fn": 2}}, "codeant-v2": {"strict": {"tp": 0, "fp": 2, "fn": 3}, "core": {"tp": 0, "fp": 2, "fn": 3}, "all": {"tp": 0, "fp": 2, "fn": 3}}, "coderabbit": {"strict": {"tp": 1, "fp": 2, "fn": 2}, "core": {"tp": 1, "fp": 2, "fn": 2}, "all": {"tp": 1, "fp": 2, "fn": 2}}, "qodo-extended-v2": {"strict": {"tp": 2, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 1}, "all": {"tp": 2, "fp": 1, "fn": 1}}, "greptile-v4-1": {"strict": {"tp": 1, "fp": 1, "fn": 2}, "core": {"tp": 1, "fp": 1, "fn": 2}, "all": {"tp": 1, "fp": 1, "fn": 2}}, "gemini-v2": {"strict": {"tp": 1, "fp": 1, "fn": 2}, "core": {"tp": 1, "fp": 1, "fn": 2}, "all": {"tp": 1, "fp": 1, "fn": 2}}, "gitlab": {"strict": {"tp": 1, "fp": 1, "fn": 2}, "core": {"tp": 1, "fp": 1, "fn": 2}, "all": {"tp": 1, "fp": 1, "fn": 2}}, "copilot-v2": {"strict": {"tp": 2, "fp": 13, "fn": 1}, "core": {"tp": 2, "fp": 13, "fn": 1}, "all": {"tp": 2, "fp": 13, "fn": 1}}, "augment": {"strict": {"tp": 1, "fp": 2, "fn": 2}, "core": {"tp": 1, "fp": 2, "fn": 2}, "all": {"tp": 1, "fp": 2, "fn": 2}}, "bugbot": {"strict": {"tp": 1, "fp": 2, "fn": 2}, "core": {"tp": 1, "fp": 2, "fn": 2}, "all": {"tp": 1, "fp": 2, "fn": 2}}, "claude": {"strict": {"tp": 1, "fp": 1, "fn": 2}, "core": {"tp": 1, "fp": 1, "fn": 2}, "all": {"tp": 1, "fp": 1, "fn": 2}}, "claude-code": {"strict": {"tp": 2, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 1}, "all": {"tp": 2, "fp": 1, "fn": 1}}, "cubic-v2": {"strict": {"tp": 3, "fp": 2, "fn": 0}, "core": {"tp": 3, "fp": 2, "fn": 0}, "all": {"tp": 3, "fp": 2, "fn": 0}}, "devin": {"strict": {"tp": 1, "fp": 1, "fn": 2}, "core": {"tp": 1, "fp": 1, "fn": 2}, "all": {"tp": 1, "fp": 1, "fn": 2}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 3}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 3}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 3}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 3}}, "kodus-v2": {"strict": {"tp": 0, "fp": 0, "fn": 3}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 3}}, "macroscope": {"strict": {"tp": 1, "fp": 1, "fn": 2}, "core": {"tp": 1, "fp": 1, "fn": 2}, "all": {"tp": 1, "fp": 1, "fn": 2}}, "qodo-v2": {"strict": {"tp": 2, "fp": 5, "fn": 1}, "core": {"tp": 2, "fp": 5, "fn": 1}, "all": {"tp": 2, "fp": 5, "fn": 1}}, "sourcery": {"strict": {"tp": 1, "fp": 8, "fn": 2}, "core": {"tp": 1, "fp": 8, "fn": 2}, "all": {"tp": 1, "fp": 8, "fn": 2}}}}, {"url": "https://github.com/keycloak/keycloak/pull/32918", "language": "Java", "pr_size": "small", "domain": "caching", "change_type": "feature", "complexity": "moderate", "difficulty": "subtle", "risk": "high", "context": "file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "codeant-v2": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "coderabbit": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "qodo-extended-v2": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "greptile-v4-1": {"strict": {"tp": 0, "fp": 3, "fn": 2}, "core": {"tp": 0, "fp": 3, "fn": 2}, "all": {"tp": 0, "fp": 3, "fn": 2}}, "gemini-v2": {"strict": {"tp": 0, "fp": 2, "fn": 2}, "core": {"tp": 0, "fp": 2, "fn": 2}, "all": {"tp": 0, "fp": 2, "fn": 2}}, "gitlab": {"strict": {"tp": 1, "fp": 2, "fn": 1}, "core": {"tp": 1, "fp": 2, "fn": 1}, "all": {"tp": 1, "fp": 2, "fn": 1}}, "copilot-v2": {"strict": {"tp": 1, "fp": 3, "fn": 1}, "core": {"tp": 1, "fp": 3, "fn": 1}, "all": {"tp": 1, "fp": 3, "fn": 1}}, "augment": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "bugbot": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "claude": {"strict": {"tp": 0, "fp": 2, "fn": 2}, "core": {"tp": 0, "fp": 2, "fn": 2}, "all": {"tp": 0, "fp": 2, "fn": 2}}, "claude-code": {"strict": {"tp": 1, "fp": 2, "fn": 1}, "core": {"tp": 1, "fp": 2, "fn": 1}, "all": {"tp": 1, "fp": 2, "fn": 1}}, "cubic-v2": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "devin": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "graphite": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "kg": {"strict": {"tp": 1, "fp": 2, "fn": 1}, "core": {"tp": 1, "fp": 2, "fn": 1}, "all": {"tp": 1, "fp": 2, "fn": 1}}, "kodus-v2": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "macroscope": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "qodo-v2": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "sourcery": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}}}, {"url": "https://github.com/grafana/grafana/pull/94942", "language": "Go", "pr_size": "small", "domain": "data_processing", "change_type": "feature", "complexity": "moderate", "difficulty": "moderate", "risk": "high", "context": "file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "codeant-v2": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "coderabbit": {"strict": {"tp": 1, "fp": 2, "fn": 1}, "core": {"tp": 1, "fp": 2, "fn": 1}, "all": {"tp": 1, "fp": 2, "fn": 1}}, "qodo-extended-v2": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "greptile-v4-1": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "gemini-v2": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "gitlab": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "copilot-v2": {"strict": {"tp": 2, "fp": 2, "fn": 0}, "core": {"tp": 2, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "augment": {"strict": {"tp": 2, "fp": 2, "fn": 0}, "core": {"tp": 2, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "bugbot": {"strict": {"tp": 1, "fp": 2, "fn": 1}, "core": {"tp": 1, "fp": 2, "fn": 1}, "all": {"tp": 1, "fp": 2, "fn": 1}}, "claude": {"strict": {"tp": 2, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "claude-code": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "cubic-v2": {"strict": {"tp": 1, "fp": 2, "fn": 1}, "core": {"tp": 1, "fp": 2, "fn": 1}, "all": {"tp": 1, "fp": 2, "fn": 1}}, "devin": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "kg": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "kodus-v2": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "macroscope": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "qodo-v2": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "sourcery": {"strict": {"tp": 2, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}}}, {"url": "https://github.com/grafana/grafana/pull/90939", "language": "Go", "pr_size": "small", "domain": "caching", "change_type": "performance", "complexity": "moderate", "difficulty": "subtle", "risk": "high", "context": "file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "codeant-v2": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "coderabbit": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "qodo-extended-v2": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "greptile-v4-1": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "gemini-v2": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "gitlab": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "copilot-v2": {"strict": {"tp": 1, "fp": 3, "fn": 1}, "core": {"tp": 1, "fp": 3, "fn": 1}, "all": {"tp": 1, "fp": 3, "fn": 1}}, "augment": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "bugbot": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "claude": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "claude-code": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "cubic-v2": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "devin": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "kg": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "kodus-v2": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "macroscope": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "qodo-v2": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "sourcery": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}}}, {"url": "https://github.com/grafana/grafana/pull/80329", "language": "Go", "pr_size": "small", "domain": "logging", "change_type": "performance", "complexity": "simple", "difficulty": "obvious", "risk": "low", "context": "local", "concern": "maintainability", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 0, "fp": 0, "fn": 0}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "codeant-v2": {"strict": {"tp": 0, "fp": 0, "fn": 0}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "coderabbit": {"strict": {"tp": 0, "fp": 4, "fn": 0}, "core": {"tp": 2, "fp": 4, "fn": 0}, "all": {"tp": 2, "fp": 4, "fn": 0}}, "qodo-extended-v2": {"strict": {"tp": 0, "fp": 3, "fn": 0}, "core": {"tp": 2, "fp": 3, "fn": 0}, "all": {"tp": 2, "fp": 3, "fn": 0}}, "greptile-v4-1": {"strict": {"tp": 0, "fp": 3, "fn": 0}, "core": {"tp": 2, "fp": 3, "fn": 0}, "all": {"tp": 2, "fp": 3, "fn": 0}}, "gemini-v2": {"strict": {"tp": 0, "fp": 3, "fn": 0}, "core": {"tp": 2, "fp": 3, "fn": 0}, "all": {"tp": 2, "fp": 3, "fn": 0}}, "gitlab": {"strict": {"tp": 0, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "copilot-v2": {"strict": {"tp": 0, "fp": 4, "fn": 0}, "core": {"tp": 2, "fp": 4, "fn": 0}, "all": {"tp": 2, "fp": 4, "fn": 0}}, "augment": {"strict": {"tp": 0, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "bugbot": {"strict": {"tp": 0, "fp": 4, "fn": 0}, "core": {"tp": 2, "fp": 4, "fn": 0}, "all": {"tp": 2, "fp": 4, "fn": 0}}, "claude": {"strict": {"tp": 0, "fp": 4, "fn": 0}, "core": {"tp": 2, "fp": 4, "fn": 0}, "all": {"tp": 2, "fp": 4, "fn": 0}}, "claude-code": {"strict": {"tp": 0, "fp": 4, "fn": 0}, "core": {"tp": 2, "fp": 4, "fn": 0}, "all": {"tp": 2, "fp": 4, "fn": 0}}, "cubic-v2": {"strict": {"tp": 0, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "devin": {"strict": {"tp": 0, "fp": 3, "fn": 0}, "core": {"tp": 2, "fp": 3, "fn": 0}, "all": {"tp": 2, "fp": 3, "fn": 0}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 0}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 0}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "kodus-v2": {"strict": {"tp": 0, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "macroscope": {"strict": {"tp": 0, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "qodo-v2": {"strict": {"tp": 0, "fp": 5, "fn": 0}, "core": {"tp": 2, "fp": 5, "fn": 0}, "all": {"tp": 2, "fp": 5, "fn": 0}}, "sourcery": {"strict": {"tp": 0, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}}}, {"url": "https://github.com/grafana/grafana/pull/90045", "language": "Go", "pr_size": "medium", "domain": "logging", "change_type": "feature", "complexity": "moderate", "difficulty": "moderate", "risk": "medium", "context": "file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 2, "fp": 2, "fn": 1}, "core": {"tp": 2, "fp": 2, "fn": 1}, "all": {"tp": 2, "fp": 2, "fn": 1}}, "codeant-v2": {"strict": {"tp": 2, "fp": 3, "fn": 1}, "core": {"tp": 2, "fp": 3, "fn": 1}, "all": {"tp": 2, "fp": 3, "fn": 1}}, "coderabbit": {"strict": {"tp": 3, "fp": 2, "fn": 0}, "core": {"tp": 3, "fp": 2, "fn": 0}, "all": {"tp": 3, "fp": 2, "fn": 0}}, "qodo-extended-v2": {"strict": {"tp": 3, "fp": 4, "fn": 0}, "core": {"tp": 3, "fp": 4, "fn": 0}, "all": {"tp": 3, "fp": 4, "fn": 0}}, "greptile-v4-1": {"strict": {"tp": 3, "fp": 3, "fn": 0}, "core": {"tp": 3, "fp": 3, "fn": 0}, "all": {"tp": 3, "fp": 3, "fn": 0}}, "gemini-v2": {"strict": {"tp": 3, "fp": 5, "fn": 0}, "core": {"tp": 3, "fp": 5, "fn": 0}, "all": {"tp": 3, "fp": 5, "fn": 0}}, "gitlab": {"strict": {"tp": 3, "fp": 2, "fn": 0}, "core": {"tp": 3, "fp": 2, "fn": 0}, "all": {"tp": 3, "fp": 2, "fn": 0}}, "copilot-v2": {"strict": {"tp": 3, "fp": 5, "fn": 0}, "core": {"tp": 3, "fp": 5, "fn": 0}, "all": {"tp": 3, "fp": 5, "fn": 0}}, "augment": {"strict": {"tp": 2, "fp": 6, "fn": 1}, "core": {"tp": 2, "fp": 6, "fn": 1}, "all": {"tp": 2, "fp": 6, "fn": 1}}, "bugbot": {"strict": {"tp": 3, "fp": 3, "fn": 0}, "core": {"tp": 3, "fp": 3, "fn": 0}, "all": {"tp": 3, "fp": 3, "fn": 0}}, "claude": {"strict": {"tp": 3, "fp": 4, "fn": 0}, "core": {"tp": 3, "fp": 4, "fn": 0}, "all": {"tp": 3, "fp": 4, "fn": 0}}, "claude-code": {"strict": {"tp": 3, "fp": 2, "fn": 0}, "core": {"tp": 3, "fp": 2, "fn": 0}, "all": {"tp": 3, "fp": 2, "fn": 0}}, "cubic-v2": {"strict": {"tp": 3, "fp": 2, "fn": 0}, "core": {"tp": 3, "fp": 2, "fn": 0}, "all": {"tp": 3, "fp": 2, "fn": 0}}, "devin": {"strict": {"tp": 3, "fp": 2, "fn": 0}, "core": {"tp": 3, "fp": 2, "fn": 0}, "all": {"tp": 3, "fp": 2, "fn": 0}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 3}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 3}}, "kg": {"strict": {"tp": 2, "fp": 2, "fn": 1}, "core": {"tp": 2, "fp": 2, "fn": 1}, "all": {"tp": 2, "fp": 2, "fn": 1}}, "kodus-v2": {"strict": {"tp": 1, "fp": 0, "fn": 2}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 2}}, "macroscope": {"strict": {"tp": 2, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 1}, "all": {"tp": 2, "fp": 1, "fn": 1}}, "qodo-v2": {"strict": {"tp": 2, "fp": 6, "fn": 1}, "core": {"tp": 2, "fp": 6, "fn": 1}, "all": {"tp": 2, "fp": 6, "fn": 1}}, "sourcery": {"strict": {"tp": 3, "fp": 0, "fn": 0}, "core": {"tp": 3, "fp": 0, "fn": 0}, "all": {"tp": 3, "fp": 0, "fn": 0}}}}, {"url": "https://github.com/grafana/grafana/pull/106778", "language": "Go", "pr_size": "medium", "domain": "UI", "change_type": "feature", "complexity": "moderate", "difficulty": "subtle", "risk": "medium", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 0, "fp": 2, "fn": 2}, "core": {"tp": 0, "fp": 2, "fn": 2}, "all": {"tp": 0, "fp": 2, "fn": 2}}, "codeant-v2": {"strict": {"tp": 1, "fp": 2, "fn": 1}, "core": {"tp": 1, "fp": 2, "fn": 1}, "all": {"tp": 1, "fp": 2, "fn": 1}}, "coderabbit": {"strict": {"tp": 1, "fp": 7, "fn": 1}, "core": {"tp": 1, "fp": 7, "fn": 1}, "all": {"tp": 1, "fp": 7, "fn": 1}}, "qodo-extended-v2": {"strict": {"tp": 2, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "greptile-v4-1": {"strict": {"tp": 1, "fp": 3, "fn": 1}, "core": {"tp": 1, "fp": 3, "fn": 1}, "all": {"tp": 1, "fp": 3, "fn": 1}}, "gemini-v2": {"strict": {"tp": 0, "fp": 3, "fn": 2}, "core": {"tp": 0, "fp": 3, "fn": 2}, "all": {"tp": 0, "fp": 3, "fn": 2}}, "gitlab": {"strict": {"tp": 2, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "copilot-v2": {"strict": {"tp": 2, "fp": 2, "fn": 0}, "core": {"tp": 2, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "augment": {"strict": {"tp": 1, "fp": 2, "fn": 1}, "core": {"tp": 1, "fp": 2, "fn": 1}, "all": {"tp": 1, "fp": 2, "fn": 1}}, "bugbot": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "claude": {"strict": {"tp": 0, "fp": 9, "fn": 2}, "core": {"tp": 0, "fp": 9, "fn": 2}, "all": {"tp": 0, "fp": 9, "fn": 2}}, "claude-code": {"strict": {"tp": 2, "fp": 2, "fn": 0}, "core": {"tp": 2, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "cubic-v2": {"strict": {"tp": 2, "fp": 2, "fn": 0}, "core": {"tp": 2, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "devin": {"strict": {"tp": 2, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "kg": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "kodus-v2": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "macroscope": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "qodo-v2": {"strict": {"tp": 1, "fp": 2, "fn": 1}, "core": {"tp": 1, "fp": 2, "fn": 1}, "all": {"tp": 1, "fp": 2, "fn": 1}}, "sourcery": {"strict": {"tp": 0, "fp": 2, "fn": 2}, "core": {"tp": 0, "fp": 2, "fn": 2}, "all": {"tp": 0, "fp": 2, "fn": 2}}}}, {"url": "https://github.com/grafana/grafana/pull/107534", "language": "Go", "pr_size": "small", "domain": "testing", "change_type": "feature", "complexity": "moderate", "difficulty": "subtle", "risk": "low", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 0, "fp": 2, "fn": 0}, "core": {"tp": 0, "fp": 2, "fn": 1}, "all": {"tp": 0, "fp": 2, "fn": 1}}, "codeant-v2": {"strict": {"tp": 0, "fp": 0, "fn": 0}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 0, "fp": 0, "fn": 1}}, "coderabbit": {"strict": {"tp": 0, "fp": 1, "fn": 0}, "core": {"tp": 0, "fp": 1, "fn": 1}, "all": {"tp": 0, "fp": 1, "fn": 1}}, "qodo-extended-v2": {"strict": {"tp": 0, "fp": 3, "fn": 0}, "core": {"tp": 0, "fp": 3, "fn": 1}, "all": {"tp": 0, "fp": 3, "fn": 1}}, "greptile-v4-1": {"strict": {"tp": 0, "fp": 5, "fn": 0}, "core": {"tp": 0, "fp": 5, "fn": 1}, "all": {"tp": 0, "fp": 5, "fn": 1}}, "gemini-v2": {"strict": {"tp": 0, "fp": 6, "fn": 0}, "core": {"tp": 0, "fp": 6, "fn": 1}, "all": {"tp": 0, "fp": 6, "fn": 1}}, "gitlab": {"strict": {"tp": 0, "fp": 2, "fn": 0}, "core": {"tp": 0, "fp": 2, "fn": 1}, "all": {"tp": 0, "fp": 2, "fn": 1}}, "copilot-v2": {"strict": {"tp": 0, "fp": 4, "fn": 0}, "core": {"tp": 0, "fp": 4, "fn": 1}, "all": {"tp": 0, "fp": 4, "fn": 1}}, "augment": {"strict": {"tp": 0, "fp": 0, "fn": 0}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 0, "fp": 0, "fn": 1}}, "bugbot": {"strict": {"tp": 0, "fp": 2, "fn": 0}, "core": {"tp": 0, "fp": 2, "fn": 1}, "all": {"tp": 0, "fp": 2, "fn": 1}}, "claude": {"strict": {"tp": 0, "fp": 0, "fn": 0}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 0, "fp": 0, "fn": 1}}, "claude-code": {"strict": {"tp": 0, "fp": 4, "fn": 0}, "core": {"tp": 0, "fp": 4, "fn": 1}, "all": {"tp": 0, "fp": 4, "fn": 1}}, "cubic-v2": {"strict": {"tp": 0, "fp": 2, "fn": 0}, "core": {"tp": 1, "fp": 2, "fn": 0}, "all": {"tp": 1, "fp": 2, "fn": 0}}, "devin": {"strict": {"tp": 0, "fp": 2, "fn": 0}, "core": {"tp": 0, "fp": 2, "fn": 1}, "all": {"tp": 0, "fp": 2, "fn": 1}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 0}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 0, "fp": 0, "fn": 1}}, "kg": {"strict": {"tp": 0, "fp": 2, "fn": 0}, "core": {"tp": 0, "fp": 2, "fn": 1}, "all": {"tp": 0, "fp": 2, "fn": 1}}, "kodus-v2": {"strict": {"tp": 0, "fp": 0, "fn": 0}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 0, "fp": 0, "fn": 1}}, "macroscope": {"strict": {"tp": 0, "fp": 0, "fn": 0}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 0, "fp": 0, "fn": 1}}, "qodo-v2": {"strict": {"tp": 0, "fp": 2, "fn": 0}, "core": {"tp": 0, "fp": 2, "fn": 1}, "all": {"tp": 0, "fp": 2, "fn": 1}}, "sourcery": {"strict": {"tp": 0, "fp": 3, "fn": 0}, "core": {"tp": 0, "fp": 3, "fn": 1}, "all": {"tp": 0, "fp": 3, "fn": 1}}}}, {"url": "https://github.com/grafana/grafana/pull/79265", "language": "Go", "pr_size": "large", "domain": "authentication", "change_type": "feature", "complexity": "moderate", "difficulty": "subtle", "risk": "high", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 0, "fp": 0, "fn": 4}, "core": {"tp": 1, "fp": 0, "fn": 4}, "all": {"tp": 1, "fp": 0, "fn": 4}}, "codeant-v2": {"strict": {"tp": 1, "fp": 2, "fn": 3}, "core": {"tp": 1, "fp": 2, "fn": 4}, "all": {"tp": 1, "fp": 2, "fn": 4}}, "coderabbit": {"strict": {"tp": 3, "fp": 5, "fn": 1}, "core": {"tp": 4, "fp": 5, "fn": 1}, "all": {"tp": 4, "fp": 5, "fn": 1}}, "qodo-extended-v2": {"strict": {"tp": 2, "fp": 1, "fn": 2}, "core": {"tp": 2, "fp": 1, "fn": 3}, "all": {"tp": 2, "fp": 1, "fn": 3}}, "greptile-v4-1": {"strict": {"tp": 2, "fp": 2, "fn": 2}, "core": {"tp": 3, "fp": 2, "fn": 2}, "all": {"tp": 3, "fp": 2, "fn": 2}}, "gemini-v2": {"strict": {"tp": 0, "fp": 4, "fn": 4}, "core": {"tp": 0, "fp": 4, "fn": 5}, "all": {"tp": 0, "fp": 4, "fn": 5}}, "gitlab": {"strict": {"tp": 0, "fp": 2, "fn": 4}, "core": {"tp": 1, "fp": 2, "fn": 4}, "all": {"tp": 1, "fp": 2, "fn": 4}}, "copilot-v2": {"strict": {"tp": 1, "fp": 4, "fn": 3}, "core": {"tp": 1, "fp": 4, "fn": 4}, "all": {"tp": 1, "fp": 4, "fn": 4}}, "augment": {"strict": {"tp": 1, "fp": 0, "fn": 3}, "core": {"tp": 2, "fp": 0, "fn": 3}, "all": {"tp": 2, "fp": 0, "fn": 3}}, "bugbot": {"strict": {"tp": 1, "fp": 1, "fn": 3}, "core": {"tp": 1, "fp": 1, "fn": 4}, "all": {"tp": 1, "fp": 1, "fn": 4}}, "claude": {"strict": {"tp": 1, "fp": 2, "fn": 3}, "core": {"tp": 1, "fp": 2, "fn": 4}, "all": {"tp": 1, "fp": 2, "fn": 4}}, "claude-code": {"strict": {"tp": 1, "fp": 1, "fn": 3}, "core": {"tp": 1, "fp": 1, "fn": 4}, "all": {"tp": 1, "fp": 1, "fn": 4}}, "cubic-v2": {"strict": {"tp": 3, "fp": 1, "fn": 1}, "core": {"tp": 4, "fp": 1, "fn": 1}, "all": {"tp": 4, "fp": 1, "fn": 1}}, "devin": {"strict": {"tp": 2, "fp": 0, "fn": 2}, "core": {"tp": 2, "fp": 0, "fn": 3}, "all": {"tp": 2, "fp": 0, "fn": 3}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 4}, "core": {"tp": 0, "fp": 0, "fn": 5}, "all": {"tp": 0, "fp": 0, "fn": 5}}, "kg": {"strict": {"tp": 1, "fp": 0, "fn": 3}, "core": {"tp": 1, "fp": 0, "fn": 4}, "all": {"tp": 1, "fp": 0, "fn": 4}}, "kodus-v2": {"strict": {"tp": 0, "fp": 1, "fn": 4}, "core": {"tp": 1, "fp": 1, "fn": 4}, "all": {"tp": 1, "fp": 1, "fn": 4}}, "macroscope": {"strict": {"tp": 1, "fp": 0, "fn": 3}, "core": {"tp": 1, "fp": 0, "fn": 4}, "all": {"tp": 1, "fp": 0, "fn": 4}}, "qodo-v2": {"strict": {"tp": 1, "fp": 2, "fn": 3}, "core": {"tp": 2, "fp": 2, "fn": 3}, "all": {"tp": 2, "fp": 2, "fn": 3}}, "sourcery": {"strict": {"tp": 2, "fp": 3, "fn": 2}, "core": {"tp": 3, "fp": 3, "fn": 2}, "all": {"tp": 3, "fp": 3, "fn": 2}}}}, {"url": "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/9", "language": "Ruby", "pr_size": "small", "domain": "configuration", "change_type": "feature", "complexity": "moderate", "difficulty": "subtle", "risk": "medium", "context": "file", "concern": "reliability", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "codeant-v2": {"strict": {"tp": 0, "fp": 3, "fn": 1}, "core": {"tp": 0, "fp": 3, "fn": 2}, "all": {"tp": 0, "fp": 3, "fn": 2}}, "coderabbit": {"strict": {"tp": 0, "fp": 2, "fn": 1}, "core": {"tp": 0, "fp": 2, "fn": 2}, "all": {"tp": 0, "fp": 2, "fn": 2}}, "qodo-extended-v2": {"strict": {"tp": 0, "fp": 1, "fn": 1}, "core": {"tp": 0, "fp": 1, "fn": 2}, "all": {"tp": 0, "fp": 1, "fn": 2}}, "greptile-v4-1": {"strict": {"tp": 0, "fp": 2, "fn": 1}, "core": {"tp": 0, "fp": 2, "fn": 2}, "all": {"tp": 0, "fp": 2, "fn": 2}}, "gemini-v2": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "gitlab": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "copilot-v2": {"strict": {"tp": 0, "fp": 3, "fn": 1}, "core": {"tp": 0, "fp": 3, "fn": 2}, "all": {"tp": 0, "fp": 3, "fn": 2}}, "augment": {"strict": {"tp": 0, "fp": 2, "fn": 1}, "core": {"tp": 1, "fp": 2, "fn": 1}, "all": {"tp": 1, "fp": 2, "fn": 1}}, "bugbot": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "claude": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "claude-code": {"strict": {"tp": 0, "fp": 5, "fn": 1}, "core": {"tp": 0, "fp": 5, "fn": 2}, "all": {"tp": 0, "fp": 5, "fn": 2}}, "cubic-v2": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "devin": {"strict": {"tp": 0, "fp": 1, "fn": 1}, "core": {"tp": 0, "fp": 1, "fn": 2}, "all": {"tp": 0, "fp": 1, "fn": 2}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "kodus-v2": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "macroscope": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "qodo-v2": {"strict": {"tp": 0, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "sourcery": {"strict": {"tp": 0, "fp": 2, "fn": 1}, "core": {"tp": 0, "fp": 2, "fn": 2}, "all": {"tp": 0, "fp": 2, "fn": 2}}}}, {"url": "https://github.com/grafana/grafana/pull/76186", "language": "Go", "pr_size": "small", "domain": "logging", "change_type": "refactoring", "complexity": "moderate", "difficulty": "subtle", "risk": "high", "context": "cross_file", "concern": "reliability", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 0, "fp": 0, "fn": 3}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 3}}, "codeant-v2": {"strict": {"tp": 0, "fp": 3, "fn": 3}, "core": {"tp": 0, "fp": 3, "fn": 3}, "all": {"tp": 0, "fp": 3, "fn": 3}}, "coderabbit": {"strict": {"tp": 1, "fp": 0, "fn": 2}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 2}}, "qodo-extended-v2": {"strict": {"tp": 1, "fp": 0, "fn": 2}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 2}}, "greptile-v4-1": {"strict": {"tp": 1, "fp": 0, "fn": 2}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 2}}, "gemini-v2": {"strict": {"tp": 1, "fp": 2, "fn": 2}, "core": {"tp": 1, "fp": 2, "fn": 2}, "all": {"tp": 1, "fp": 2, "fn": 2}}, "gitlab": {"strict": {"tp": 3, "fp": 3, "fn": 0}, "core": {"tp": 3, "fp": 3, "fn": 0}, "all": {"tp": 3, "fp": 3, "fn": 0}}, "copilot-v2": {"strict": {"tp": 0, "fp": 2, "fn": 3}, "core": {"tp": 0, "fp": 2, "fn": 3}, "all": {"tp": 0, "fp": 2, "fn": 3}}, "augment": {"strict": {"tp": 2, "fp": 0, "fn": 1}, "core": {"tp": 2, "fp": 0, "fn": 1}, "all": {"tp": 2, "fp": 0, "fn": 1}}, "bugbot": {"strict": {"tp": 1, "fp": 0, "fn": 2}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 2}}, "claude": {"strict": {"tp": 0, "fp": 0, "fn": 3}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 3}}, "claude-code": {"strict": {"tp": 1, "fp": 1, "fn": 2}, "core": {"tp": 1, "fp": 1, "fn": 2}, "all": {"tp": 1, "fp": 1, "fn": 2}}, "cubic-v2": {"strict": {"tp": 3, "fp": 0, "fn": 0}, "core": {"tp": 3, "fp": 0, "fn": 0}, "all": {"tp": 3, "fp": 0, "fn": 0}}, "devin": {"strict": {"tp": 1, "fp": 0, "fn": 2}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 2}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 3}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 3}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 3}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 3}}, "kodus-v2": {"strict": {"tp": 2, "fp": 0, "fn": 1}, "core": {"tp": 2, "fp": 0, "fn": 1}, "all": {"tp": 2, "fp": 0, "fn": 1}}, "macroscope": {"strict": {"tp": 1, "fp": 0, "fn": 2}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 2}}, "qodo-v2": {"strict": {"tp": 2, "fp": 0, "fn": 1}, "core": {"tp": 2, "fp": 0, "fn": 1}, "all": {"tp": 2, "fp": 0, "fn": 1}}, "sourcery": {"strict": {"tp": 0, "fp": 3, "fn": 3}, "core": {"tp": 0, "fp": 3, "fn": 3}, "all": {"tp": 0, "fp": 3, "fn": 3}}}}, {"url": "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/10", "language": "Ruby", "pr_size": "large", "domain": "configuration", "change_type": "feature", "complexity": "moderate", "difficulty": "subtle", "risk": "high", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 2, "fp": 1, "fn": 5}, "core": {"tp": 2, "fp": 1, "fn": 5}, "all": {"tp": 2, "fp": 1, "fn": 5}}, "codeant-v2": {"strict": {"tp": 2, "fp": 7, "fn": 5}, "core": {"tp": 2, "fp": 7, "fn": 5}, "all": {"tp": 2, "fp": 7, "fn": 5}}, "coderabbit": {"strict": {"tp": 5, "fp": 14, "fn": 2}, "core": {"tp": 5, "fp": 14, "fn": 2}, "all": {"tp": 5, "fp": 14, "fn": 2}}, "qodo-extended-v2": {"strict": {"tp": 4, "fp": 6, "fn": 3}, "core": {"tp": 4, "fp": 6, "fn": 3}, "all": {"tp": 4, "fp": 6, "fn": 3}}, "greptile-v4-1": {"strict": {"tp": 2, "fp": 3, "fn": 5}, "core": {"tp": 2, "fp": 3, "fn": 5}, "all": {"tp": 2, "fp": 3, "fn": 5}}, "gemini-v2": {"strict": {"tp": 4, "fp": 2, "fn": 3}, "core": {"tp": 4, "fp": 2, "fn": 3}, "all": {"tp": 4, "fp": 2, "fn": 3}}, "gitlab": {"strict": {"tp": 2, "fp": 3, "fn": 5}, "core": {"tp": 2, "fp": 3, "fn": 5}, "all": {"tp": 2, "fp": 3, "fn": 5}}, "copilot-v2": {"strict": {"tp": 6, "fp": 5, "fn": 1}, "core": {"tp": 6, "fp": 5, "fn": 1}, "all": {"tp": 6, "fp": 5, "fn": 1}}, "augment": {"strict": {"tp": 4, "fp": 1, "fn": 3}, "core": {"tp": 4, "fp": 1, "fn": 3}, "all": {"tp": 4, "fp": 1, "fn": 3}}, "bugbot": {"strict": {"tp": 5, "fp": 2, "fn": 2}, "core": {"tp": 5, "fp": 2, "fn": 2}, "all": {"tp": 5, "fp": 2, "fn": 2}}, "claude": {"strict": {"tp": 1, "fp": 2, "fn": 6}, "core": {"tp": 1, "fp": 2, "fn": 6}, "all": {"tp": 1, "fp": 2, "fn": 6}}, "claude-code": {"strict": {"tp": 1, "fp": 5, "fn": 6}, "core": {"tp": 1, "fp": 5, "fn": 6}, "all": {"tp": 1, "fp": 5, "fn": 6}}, "cubic-v2": {"strict": {"tp": 3, "fp": 1, "fn": 4}, "core": {"tp": 3, "fp": 1, "fn": 4}, "all": {"tp": 3, "fp": 1, "fn": 4}}, "devin": {"strict": {"tp": 2, "fp": 0, "fn": 5}, "core": {"tp": 2, "fp": 0, "fn": 5}, "all": {"tp": 2, "fp": 0, "fn": 5}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 7}, "core": {"tp": 0, "fp": 0, "fn": 7}, "all": {"tp": 0, "fp": 0, "fn": 7}}, "kg": {"strict": {"tp": 1, "fp": 0, "fn": 6}, "core": {"tp": 1, "fp": 0, "fn": 6}, "all": {"tp": 1, "fp": 0, "fn": 6}}, "kodus-v2": {"strict": {"tp": 5, "fp": 4, "fn": 2}, "core": {"tp": 5, "fp": 4, "fn": 2}, "all": {"tp": 5, "fp": 4, "fn": 2}}, "macroscope": {"strict": {"tp": 4, "fp": 3, "fn": 3}, "core": {"tp": 4, "fp": 3, "fn": 3}, "all": {"tp": 4, "fp": 3, "fn": 3}}, "qodo-v2": {"strict": {"tp": 4, "fp": 6, "fn": 3}, "core": {"tp": 4, "fp": 6, "fn": 3}, "all": {"tp": 4, "fp": 6, "fn": 3}}, "sourcery": {"strict": {"tp": 5, "fp": 7, "fn": 2}, "core": {"tp": 5, "fp": 7, "fn": 2}, "all": {"tp": 5, "fp": 7, "fn": 2}}}}, {"url": "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/7", "language": "Ruby", "pr_size": "medium", "domain": "UI", "change_type": "bug_fix", "complexity": "simple", "difficulty": "subtle", "risk": "medium", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 3, "fp": 1, "fn": 0}, "core": {"tp": 3, "fp": 1, "fn": 0}, "all": {"tp": 5, "fp": 1, "fn": 0}}, "codeant-v2": {"strict": {"tp": 0, "fp": 9, "fn": 3}, "core": {"tp": 0, "fp": 9, "fn": 3}, "all": {"tp": 0, "fp": 9, "fn": 5}}, "coderabbit": {"strict": {"tp": 0, "fp": 2, "fn": 3}, "core": {"tp": 0, "fp": 2, "fn": 3}, "all": {"tp": 0, "fp": 2, "fn": 5}}, "qodo-extended-v2": {"strict": {"tp": 2, "fp": 2, "fn": 1}, "core": {"tp": 2, "fp": 2, "fn": 1}, "all": {"tp": 3, "fp": 2, "fn": 2}}, "greptile-v4-1": {"strict": {"tp": 3, "fp": 0, "fn": 0}, "core": {"tp": 3, "fp": 0, "fn": 0}, "all": {"tp": 5, "fp": 0, "fn": 0}}, "gemini-v2": {"strict": {"tp": 3, "fp": 1, "fn": 0}, "core": {"tp": 3, "fp": 1, "fn": 0}, "all": {"tp": 5, "fp": 1, "fn": 0}}, "gitlab": {"strict": {"tp": 3, "fp": 0, "fn": 0}, "core": {"tp": 3, "fp": 0, "fn": 0}, "all": {"tp": 4, "fp": 0, "fn": 1}}, "copilot-v2": {"strict": {"tp": 3, "fp": 1, "fn": 0}, "core": {"tp": 3, "fp": 1, "fn": 0}, "all": {"tp": 5, "fp": 1, "fn": 0}}, "augment": {"strict": {"tp": 3, "fp": 2, "fn": 0}, "core": {"tp": 3, "fp": 2, "fn": 0}, "all": {"tp": 5, "fp": 2, "fn": 0}}, "bugbot": {"strict": {"tp": 2, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 1}, "all": {"tp": 4, "fp": 1, "fn": 1}}, "claude": {"strict": {"tp": 3, "fp": 3, "fn": 0}, "core": {"tp": 3, "fp": 3, "fn": 0}, "all": {"tp": 5, "fp": 3, "fn": 0}}, "claude-code": {"strict": {"tp": 2, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 1}, "all": {"tp": 4, "fp": 1, "fn": 1}}, "cubic-v2": {"strict": {"tp": 2, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 1}, "all": {"tp": 4, "fp": 1, "fn": 1}}, "devin": {"strict": {"tp": 2, "fp": 3, "fn": 1}, "core": {"tp": 2, "fp": 3, "fn": 1}, "all": {"tp": 4, "fp": 3, "fn": 1}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 3}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 5}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 3}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 5}}, "kodus-v2": {"strict": {"tp": 1, "fp": 0, "fn": 2}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 3, "fp": 0, "fn": 2}}, "macroscope": {"strict": {"tp": 1, "fp": 0, "fn": 2}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 2, "fp": 0, "fn": 3}}, "qodo-v2": {"strict": {"tp": 3, "fp": 1, "fn": 0}, "core": {"tp": 3, "fp": 1, "fn": 0}, "all": {"tp": 5, "fp": 1, "fn": 0}}, "sourcery": {"strict": {"tp": 2, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 1}, "all": {"tp": 3, "fp": 1, "fn": 2}}}}, {"url": "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/8", "language": "Ruby", "pr_size": "medium", "domain": "API", "change_type": "bug_fix", "complexity": "moderate", "difficulty": "subtle", "risk": "medium", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 2, "fp": 2, "fn": 4}, "core": {"tp": 2, "fp": 2, "fn": 4}, "all": {"tp": 2, "fp": 2, "fn": 4}}, "codeant-v2": {"strict": {"tp": 5, "fp": 7, "fn": 1}, "core": {"tp": 5, "fp": 7, "fn": 1}, "all": {"tp": 5, "fp": 7, "fn": 1}}, "coderabbit": {"strict": {"tp": 4, "fp": 6, "fn": 2}, "core": {"tp": 4, "fp": 6, "fn": 2}, "all": {"tp": 4, "fp": 6, "fn": 2}}, "qodo-extended-v2": {"strict": {"tp": 3, "fp": 2, "fn": 3}, "core": {"tp": 3, "fp": 2, "fn": 3}, "all": {"tp": 3, "fp": 2, "fn": 3}}, "greptile-v4-1": {"strict": {"tp": 4, "fp": 1, "fn": 2}, "core": {"tp": 4, "fp": 1, "fn": 2}, "all": {"tp": 4, "fp": 1, "fn": 2}}, "gemini-v2": {"strict": {"tp": 3, "fp": 2, "fn": 3}, "core": {"tp": 3, "fp": 2, "fn": 3}, "all": {"tp": 3, "fp": 2, "fn": 3}}, "gitlab": {"strict": {"tp": 4, "fp": 1, "fn": 2}, "core": {"tp": 4, "fp": 1, "fn": 2}, "all": {"tp": 4, "fp": 1, "fn": 2}}, "copilot-v2": {"strict": {"tp": 3, "fp": 8, "fn": 3}, "core": {"tp": 3, "fp": 8, "fn": 3}, "all": {"tp": 3, "fp": 8, "fn": 3}}, "augment": {"strict": {"tp": 3, "fp": 3, "fn": 3}, "core": {"tp": 3, "fp": 3, "fn": 3}, "all": {"tp": 3, "fp": 3, "fn": 3}}, "bugbot": {"strict": {"tp": 2, "fp": 1, "fn": 4}, "core": {"tp": 2, "fp": 1, "fn": 4}, "all": {"tp": 2, "fp": 1, "fn": 4}}, "claude": {"strict": {"tp": 4, "fp": 4, "fn": 2}, "core": {"tp": 4, "fp": 4, "fn": 2}, "all": {"tp": 4, "fp": 4, "fn": 2}}, "claude-code": {"strict": {"tp": 2, "fp": 1, "fn": 4}, "core": {"tp": 2, "fp": 1, "fn": 4}, "all": {"tp": 2, "fp": 1, "fn": 4}}, "cubic-v2": {"strict": {"tp": 3, "fp": 1, "fn": 3}, "core": {"tp": 3, "fp": 1, "fn": 3}, "all": {"tp": 3, "fp": 1, "fn": 3}}, "devin": {"strict": {"tp": 2, "fp": 0, "fn": 4}, "core": {"tp": 2, "fp": 0, "fn": 4}, "all": {"tp": 2, "fp": 0, "fn": 4}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 6}, "core": {"tp": 0, "fp": 0, "fn": 6}, "all": {"tp": 0, "fp": 0, "fn": 6}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 6}, "core": {"tp": 0, "fp": 0, "fn": 6}, "all": {"tp": 0, "fp": 0, "fn": 6}}, "kodus-v2": {"strict": {"tp": 1, "fp": 6, "fn": 5}, "core": {"tp": 1, "fp": 6, "fn": 5}, "all": {"tp": 1, "fp": 6, "fn": 5}}, "macroscope": {"strict": {"tp": 2, "fp": 2, "fn": 4}, "core": {"tp": 2, "fp": 2, "fn": 4}, "all": {"tp": 2, "fp": 2, "fn": 4}}, "qodo-v2": {"strict": {"tp": 2, "fp": 2, "fn": 4}, "core": {"tp": 2, "fp": 2, "fn": 4}, "all": {"tp": 2, "fp": 2, "fn": 4}}, "sourcery": {"strict": {"tp": 3, "fp": 2, "fn": 3}, "core": {"tp": 3, "fp": 2, "fn": 3}, "all": {"tp": 3, "fp": 2, "fn": 3}}}}, {"url": "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/3", "language": "Ruby", "pr_size": "medium", "domain": "authentication", "change_type": "feature", "complexity": "moderate", "difficulty": "subtle", "risk": "medium", "context": "file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 1, "fp": 1, "fn": 2}, "core": {"tp": 1, "fp": 1, "fn": 2}, "all": {"tp": 1, "fp": 1, "fn": 2}}, "codeant-v2": {"strict": {"tp": 1, "fp": 3, "fn": 2}, "core": {"tp": 1, "fp": 3, "fn": 2}, "all": {"tp": 1, "fp": 3, "fn": 2}}, "coderabbit": {"strict": {"tp": 2, "fp": 5, "fn": 1}, "core": {"tp": 2, "fp": 5, "fn": 1}, "all": {"tp": 2, "fp": 5, "fn": 1}}, "qodo-extended-v2": {"strict": {"tp": 2, "fp": 0, "fn": 1}, "core": {"tp": 2, "fp": 0, "fn": 1}, "all": {"tp": 2, "fp": 0, "fn": 1}}, "greptile-v4-1": {"strict": {"tp": 1, "fp": 3, "fn": 2}, "core": {"tp": 1, "fp": 3, "fn": 2}, "all": {"tp": 1, "fp": 3, "fn": 2}}, "gemini-v2": {"strict": {"tp": 2, "fp": 3, "fn": 1}, "core": {"tp": 2, "fp": 3, "fn": 1}, "all": {"tp": 2, "fp": 3, "fn": 1}}, "gitlab": {"strict": {"tp": 0, "fp": 2, "fn": 3}, "core": {"tp": 0, "fp": 2, "fn": 3}, "all": {"tp": 0, "fp": 2, "fn": 3}}, "copilot-v2": {"strict": {"tp": 2, "fp": 6, "fn": 1}, "core": {"tp": 2, "fp": 6, "fn": 1}, "all": {"tp": 2, "fp": 6, "fn": 1}}, "augment": {"strict": {"tp": 1, "fp": 5, "fn": 2}, "core": {"tp": 1, "fp": 5, "fn": 2}, "all": {"tp": 1, "fp": 5, "fn": 2}}, "bugbot": {"strict": {"tp": 1, "fp": 1, "fn": 2}, "core": {"tp": 1, "fp": 1, "fn": 2}, "all": {"tp": 1, "fp": 1, "fn": 2}}, "claude": {"strict": {"tp": 1, "fp": 4, "fn": 2}, "core": {"tp": 1, "fp": 4, "fn": 2}, "all": {"tp": 1, "fp": 4, "fn": 2}}, "claude-code": {"strict": {"tp": 1, "fp": 4, "fn": 2}, "core": {"tp": 1, "fp": 4, "fn": 2}, "all": {"tp": 1, "fp": 4, "fn": 2}}, "cubic-v2": {"strict": {"tp": 2, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 1}, "all": {"tp": 2, "fp": 1, "fn": 1}}, "devin": {"strict": {"tp": 1, "fp": 1, "fn": 2}, "core": {"tp": 1, "fp": 1, "fn": 2}, "all": {"tp": 1, "fp": 1, "fn": 2}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 3}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 3}}, "kg": {"strict": {"tp": 1, "fp": 0, "fn": 2}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 2}}, "kodus-v2": {"strict": {"tp": 1, "fp": 0, "fn": 2}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 2}}, "macroscope": {"strict": {"tp": 2, "fp": 0, "fn": 1}, "core": {"tp": 2, "fp": 0, "fn": 1}, "all": {"tp": 2, "fp": 0, "fn": 1}}, "qodo-v2": {"strict": {"tp": 2, "fp": 3, "fn": 1}, "core": {"tp": 2, "fp": 3, "fn": 1}, "all": {"tp": 2, "fp": 3, "fn": 1}}, "sourcery": {"strict": {"tp": 1, "fp": 3, "fn": 2}, "core": {"tp": 1, "fp": 3, "fn": 2}, "all": {"tp": 1, "fp": 3, "fn": 2}}}}, {"url": "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/5", "language": "Ruby", "pr_size": "small", "domain": "UI", "change_type": "performance", "complexity": "moderate", "difficulty": "subtle", "risk": "medium", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 1, "fp": 1, "fn": 2}, "core": {"tp": 1, "fp": 1, "fn": 2}, "all": {"tp": 1, "fp": 1, "fn": 2}}, "codeant-v2": {"strict": {"tp": 0, "fp": 0, "fn": 3}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 3}}, "coderabbit": {"strict": {"tp": 1, "fp": 0, "fn": 2}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 2}}, "qodo-extended-v2": {"strict": {"tp": 2, "fp": 0, "fn": 1}, "core": {"tp": 2, "fp": 0, "fn": 1}, "all": {"tp": 2, "fp": 0, "fn": 1}}, "greptile-v4-1": {"strict": {"tp": 1, "fp": 2, "fn": 2}, "core": {"tp": 1, "fp": 2, "fn": 2}, "all": {"tp": 1, "fp": 2, "fn": 2}}, "gemini-v2": {"strict": {"tp": 2, "fp": 7, "fn": 1}, "core": {"tp": 2, "fp": 7, "fn": 1}, "all": {"tp": 2, "fp": 7, "fn": 1}}, "gitlab": {"strict": {"tp": 1, "fp": 1, "fn": 2}, "core": {"tp": 1, "fp": 1, "fn": 2}, "all": {"tp": 1, "fp": 1, "fn": 2}}, "copilot-v2": {"strict": {"tp": 2, "fp": 2, "fn": 1}, "core": {"tp": 2, "fp": 2, "fn": 1}, "all": {"tp": 2, "fp": 2, "fn": 1}}, "augment": {"strict": {"tp": 2, "fp": 0, "fn": 1}, "core": {"tp": 2, "fp": 0, "fn": 1}, "all": {"tp": 2, "fp": 0, "fn": 1}}, "bugbot": {"strict": {"tp": 1, "fp": 0, "fn": 2}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 2}}, "claude": {"strict": {"tp": 1, "fp": 0, "fn": 2}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 2}}, "claude-code": {"strict": {"tp": 2, "fp": 3, "fn": 1}, "core": {"tp": 2, "fp": 3, "fn": 1}, "all": {"tp": 2, "fp": 3, "fn": 1}}, "cubic-v2": {"strict": {"tp": 2, "fp": 2, "fn": 1}, "core": {"tp": 2, "fp": 2, "fn": 1}, "all": {"tp": 2, "fp": 2, "fn": 1}}, "devin": {"strict": {"tp": 1, "fp": 1, "fn": 2}, "core": {"tp": 1, "fp": 1, "fn": 2}, "all": {"tp": 1, "fp": 1, "fn": 2}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 3}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 3}}, "kg": {"strict": {"tp": 1, "fp": 0, "fn": 2}, "core": {"tp": 1, "fp": 0, "fn": 2}, "all": {"tp": 1, "fp": 0, "fn": 2}}, "kodus-v2": {"strict": {"tp": 0, "fp": 0, "fn": 3}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 3}}, "macroscope": {"strict": {"tp": 0, "fp": 0, "fn": 3}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 3}}, "qodo-v2": {"strict": {"tp": 1, "fp": 1, "fn": 2}, "core": {"tp": 1, "fp": 1, "fn": 2}, "all": {"tp": 1, "fp": 1, "fn": 2}}, "sourcery": {"strict": {"tp": 2, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 1}, "all": {"tp": 2, "fp": 1, "fn": 1}}}}, {"url": "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/6", "language": "Ruby", "pr_size": "small", "domain": "serialization", "change_type": "feature", "complexity": "simple", "difficulty": "moderate", "risk": "medium", "context": "file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 0, "fp": 5, "fn": 1}, "core": {"tp": 0, "fp": 5, "fn": 1}, "all": {"tp": 0, "fp": 5, "fn": 1}}, "codeant-v2": {"strict": {"tp": 1, "fp": 2, "fn": 0}, "core": {"tp": 1, "fp": 2, "fn": 0}, "all": {"tp": 1, "fp": 2, "fn": 0}}, "coderabbit": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 1, "fp": 1, "fn": 0}, "all": {"tp": 1, "fp": 1, "fn": 0}}, "qodo-extended-v2": {"strict": {"tp": 0, "fp": 2, "fn": 1}, "core": {"tp": 0, "fp": 2, "fn": 1}, "all": {"tp": 0, "fp": 2, "fn": 1}}, "greptile-v4-1": {"strict": {"tp": 0, "fp": 4, "fn": 1}, "core": {"tp": 0, "fp": 4, "fn": 1}, "all": {"tp": 0, "fp": 4, "fn": 1}}, "gemini-v2": {"strict": {"tp": 0, "fp": 6, "fn": 1}, "core": {"tp": 0, "fp": 6, "fn": 1}, "all": {"tp": 0, "fp": 6, "fn": 1}}, "gitlab": {"strict": {"tp": 0, "fp": 2, "fn": 1}, "core": {"tp": 0, "fp": 2, "fn": 1}, "all": {"tp": 0, "fp": 2, "fn": 1}}, "copilot-v2": {"strict": {"tp": 1, "fp": 3, "fn": 0}, "core": {"tp": 1, "fp": 3, "fn": 0}, "all": {"tp": 1, "fp": 3, "fn": 0}}, "augment": {"strict": {"tp": 0, "fp": 3, "fn": 1}, "core": {"tp": 0, "fp": 3, "fn": 1}, "all": {"tp": 0, "fp": 3, "fn": 1}}, "bugbot": {"strict": {"tp": 0, "fp": 3, "fn": 1}, "core": {"tp": 0, "fp": 3, "fn": 1}, "all": {"tp": 0, "fp": 3, "fn": 1}}, "claude": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 0, "fp": 0, "fn": 1}}, "claude-code": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 0}, "all": {"tp": 1, "fp": 0, "fn": 0}}, "cubic-v2": {"strict": {"tp": 0, "fp": 2, "fn": 1}, "core": {"tp": 0, "fp": 2, "fn": 1}, "all": {"tp": 0, "fp": 2, "fn": 1}}, "devin": {"strict": {"tp": 0, "fp": 3, "fn": 1}, "core": {"tp": 0, "fp": 3, "fn": 1}, "all": {"tp": 0, "fp": 3, "fn": 1}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 0, "fp": 0, "fn": 1}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 0, "fp": 0, "fn": 1}}, "kodus-v2": {"strict": {"tp": 0, "fp": 2, "fn": 1}, "core": {"tp": 0, "fp": 2, "fn": 1}, "all": {"tp": 0, "fp": 2, "fn": 1}}, "macroscope": {"strict": {"tp": 0, "fp": 2, "fn": 1}, "core": {"tp": 0, "fp": 2, "fn": 1}, "all": {"tp": 0, "fp": 2, "fn": 1}}, "qodo-v2": {"strict": {"tp": 1, "fp": 2, "fn": 0}, "core": {"tp": 1, "fp": 2, "fn": 0}, "all": {"tp": 1, "fp": 2, "fn": 0}}, "sourcery": {"strict": {"tp": 0, "fp": 5, "fn": 1}, "core": {"tp": 0, "fp": 5, "fn": 1}, "all": {"tp": 0, "fp": 5, "fn": 1}}}}, {"url": "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/4", "language": "Ruby", "pr_size": "large", "domain": "API", "change_type": "feature", "complexity": "complex", "difficulty": "subtle", "risk": "critical", "context": "cross_file", "concern": "security", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 2, "fp": 2, "fn": 6}, "core": {"tp": 2, "fp": 2, "fn": 6}, "all": {"tp": 2, "fp": 2, "fn": 6}}, "codeant-v2": {"strict": {"tp": 1, "fp": 17, "fn": 7}, "core": {"tp": 1, "fp": 17, "fn": 7}, "all": {"tp": 1, "fp": 17, "fn": 7}}, "coderabbit": {"strict": {"tp": 4, "fp": 23, "fn": 4}, "core": {"tp": 4, "fp": 23, "fn": 4}, "all": {"tp": 4, "fp": 23, "fn": 4}}, "qodo-extended-v2": {"strict": {"tp": 3, "fp": 3, "fn": 5}, "core": {"tp": 3, "fp": 3, "fn": 5}, "all": {"tp": 3, "fp": 3, "fn": 5}}, "greptile-v4-1": {"strict": {"tp": 4, "fp": 1, "fn": 4}, "core": {"tp": 4, "fp": 1, "fn": 4}, "all": {"tp": 4, "fp": 1, "fn": 4}}, "gemini-v2": {"strict": {"tp": 4, "fp": 5, "fn": 4}, "core": {"tp": 4, "fp": 5, "fn": 4}, "all": {"tp": 4, "fp": 5, "fn": 4}}, "gitlab": {"strict": {"tp": 3, "fp": 2, "fn": 5}, "core": {"tp": 3, "fp": 2, "fn": 5}, "all": {"tp": 3, "fp": 2, "fn": 5}}, "copilot-v2": {"strict": {"tp": 5, "fp": 6, "fn": 3}, "core": {"tp": 5, "fp": 6, "fn": 3}, "all": {"tp": 5, "fp": 6, "fn": 3}}, "augment": {"strict": {"tp": 3, "fp": 2, "fn": 5}, "core": {"tp": 3, "fp": 2, "fn": 5}, "all": {"tp": 3, "fp": 2, "fn": 5}}, "bugbot": {"strict": {"tp": 2, "fp": 3, "fn": 6}, "core": {"tp": 2, "fp": 3, "fn": 6}, "all": {"tp": 2, "fp": 3, "fn": 6}}, "claude": {"strict": {"tp": 3, "fp": 3, "fn": 5}, "core": {"tp": 3, "fp": 3, "fn": 5}, "all": {"tp": 3, "fp": 3, "fn": 5}}, "claude-code": {"strict": {"tp": 3, "fp": 1, "fn": 5}, "core": {"tp": 3, "fp": 1, "fn": 5}, "all": {"tp": 3, "fp": 1, "fn": 5}}, "cubic-v2": {"strict": {"tp": 1, "fp": 3, "fn": 7}, "core": {"tp": 1, "fp": 3, "fn": 7}, "all": {"tp": 1, "fp": 3, "fn": 7}}, "devin": {"strict": {"tp": 3, "fp": 1, "fn": 5}, "core": {"tp": 3, "fp": 1, "fn": 5}, "all": {"tp": 3, "fp": 1, "fn": 5}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 8}, "core": {"tp": 0, "fp": 0, "fn": 8}, "all": {"tp": 0, "fp": 0, "fn": 8}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 8}, "core": {"tp": 0, "fp": 0, "fn": 8}, "all": {"tp": 0, "fp": 0, "fn": 8}}, "kodus-v2": {"strict": {"tp": 3, "fp": 2, "fn": 5}, "core": {"tp": 3, "fp": 2, "fn": 5}, "all": {"tp": 3, "fp": 2, "fn": 5}}, "macroscope": {"strict": {"tp": 1, "fp": 5, "fn": 7}, "core": {"tp": 1, "fp": 5, "fn": 7}, "all": {"tp": 1, "fp": 5, "fn": 7}}, "qodo-v2": {"strict": {"tp": 5, "fp": 13, "fn": 3}, "core": {"tp": 5, "fp": 13, "fn": 3}, "all": {"tp": 5, "fp": 13, "fn": 3}}, "sourcery": {"strict": {"tp": 2, "fp": 3, "fn": 6}, "core": {"tp": 2, "fp": 3, "fn": 6}, "all": {"tp": 2, "fp": 3, "fn": 6}}}}, {"url": "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/1", "language": "Ruby", "pr_size": "medium", "domain": "file_io", "change_type": "feature", "complexity": "moderate", "difficulty": "subtle", "risk": "medium", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 2, "fp": 1, "fn": 2}, "core": {"tp": 2, "fp": 1, "fn": 2}, "all": {"tp": 2, "fp": 1, "fn": 2}}, "codeant-v2": {"strict": {"tp": 2, "fp": 1, "fn": 2}, "core": {"tp": 2, "fp": 1, "fn": 2}, "all": {"tp": 2, "fp": 1, "fn": 2}}, "coderabbit": {"strict": {"tp": 3, "fp": 4, "fn": 1}, "core": {"tp": 3, "fp": 4, "fn": 1}, "all": {"tp": 3, "fp": 4, "fn": 1}}, "qodo-extended-v2": {"strict": {"tp": 2, "fp": 1, "fn": 2}, "core": {"tp": 2, "fp": 1, "fn": 2}, "all": {"tp": 2, "fp": 1, "fn": 2}}, "greptile-v4-1": {"strict": {"tp": 3, "fp": 1, "fn": 1}, "core": {"tp": 3, "fp": 1, "fn": 1}, "all": {"tp": 3, "fp": 1, "fn": 1}}, "gemini-v2": {"strict": {"tp": 3, "fp": 0, "fn": 1}, "core": {"tp": 3, "fp": 0, "fn": 1}, "all": {"tp": 3, "fp": 0, "fn": 1}}, "gitlab": {"strict": {"tp": 2, "fp": 0, "fn": 2}, "core": {"tp": 2, "fp": 0, "fn": 2}, "all": {"tp": 2, "fp": 0, "fn": 2}}, "copilot-v2": {"strict": {"tp": 2, "fp": 2, "fn": 2}, "core": {"tp": 2, "fp": 2, "fn": 2}, "all": {"tp": 2, "fp": 2, "fn": 2}}, "augment": {"strict": {"tp": 2, "fp": 4, "fn": 2}, "core": {"tp": 2, "fp": 4, "fn": 2}, "all": {"tp": 2, "fp": 4, "fn": 2}}, "bugbot": {"strict": {"tp": 2, "fp": 2, "fn": 2}, "core": {"tp": 2, "fp": 2, "fn": 2}, "all": {"tp": 2, "fp": 2, "fn": 2}}, "claude": {"strict": {"tp": 2, "fp": 0, "fn": 2}, "core": {"tp": 2, "fp": 0, "fn": 2}, "all": {"tp": 2, "fp": 0, "fn": 2}}, "claude-code": {"strict": {"tp": 3, "fp": 2, "fn": 1}, "core": {"tp": 3, "fp": 2, "fn": 1}, "all": {"tp": 3, "fp": 2, "fn": 1}}, "cubic-v2": {"strict": {"tp": 3, "fp": 0, "fn": 1}, "core": {"tp": 3, "fp": 0, "fn": 1}, "all": {"tp": 3, "fp": 0, "fn": 1}}, "devin": {"strict": {"tp": 2, "fp": 0, "fn": 2}, "core": {"tp": 2, "fp": 0, "fn": 2}, "all": {"tp": 2, "fp": 0, "fn": 2}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 4}, "core": {"tp": 0, "fp": 0, "fn": 4}, "all": {"tp": 0, "fp": 0, "fn": 4}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 4}, "core": {"tp": 0, "fp": 0, "fn": 4}, "all": {"tp": 0, "fp": 0, "fn": 4}}, "kodus-v2": {"strict": {"tp": 3, "fp": 1, "fn": 1}, "core": {"tp": 3, "fp": 1, "fn": 1}, "all": {"tp": 3, "fp": 1, "fn": 1}}, "macroscope": {"strict": {"tp": 1, "fp": 2, "fn": 3}, "core": {"tp": 1, "fp": 2, "fn": 3}, "all": {"tp": 1, "fp": 2, "fn": 3}}, "qodo-v2": {"strict": {"tp": 2, "fp": 4, "fn": 2}, "core": {"tp": 2, "fp": 4, "fn": 2}, "all": {"tp": 2, "fp": 4, "fn": 2}}, "sourcery": {"strict": {"tp": 2, "fp": 1, "fn": 2}, "core": {"tp": 2, "fp": 1, "fn": 2}, "all": {"tp": 2, "fp": 1, "fn": 2}}}}, {"url": "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/2", "language": "Ruby", "pr_size": "medium", "domain": "UI", "change_type": "feature", "complexity": "moderate", "difficulty": "moderate", "risk": "medium", "context": "file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 0}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "codeant-v2": {"strict": {"tp": 1, "fp": 5, "fn": 0}, "core": {"tp": 1, "fp": 5, "fn": 0}, "all": {"tp": 1, "fp": 5, "fn": 1}}, "coderabbit": {"strict": {"tp": 1, "fp": 7, "fn": 0}, "core": {"tp": 1, "fp": 7, "fn": 0}, "all": {"tp": 2, "fp": 7, "fn": 0}}, "qodo-extended-v2": {"strict": {"tp": 1, "fp": 2, "fn": 0}, "core": {"tp": 1, "fp": 2, "fn": 0}, "all": {"tp": 1, "fp": 2, "fn": 1}}, "greptile-v4-1": {"strict": {"tp": 1, "fp": 2, "fn": 0}, "core": {"tp": 1, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "gemini-v2": {"strict": {"tp": 1, "fp": 2, "fn": 0}, "core": {"tp": 1, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "gitlab": {"strict": {"tp": 1, "fp": 2, "fn": 0}, "core": {"tp": 1, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "copilot-v2": {"strict": {"tp": 1, "fp": 4, "fn": 0}, "core": {"tp": 1, "fp": 4, "fn": 0}, "all": {"tp": 2, "fp": 4, "fn": 0}}, "augment": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 1, "fp": 1, "fn": 0}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "bugbot": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 1, "fp": 1, "fn": 0}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "claude": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 1, "fp": 1, "fn": 0}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "claude-code": {"strict": {"tp": 0, "fp": 2, "fn": 1}, "core": {"tp": 0, "fp": 2, "fn": 1}, "all": {"tp": 0, "fp": 2, "fn": 2}}, "cubic-v2": {"strict": {"tp": 0, "fp": 2, "fn": 1}, "core": {"tp": 0, "fp": 2, "fn": 1}, "all": {"tp": 0, "fp": 2, "fn": 2}}, "devin": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 0}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "kodus-v2": {"strict": {"tp": 1, "fp": 2, "fn": 0}, "core": {"tp": 1, "fp": 2, "fn": 0}, "all": {"tp": 1, "fp": 2, "fn": 1}}, "macroscope": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 0}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "qodo-v2": {"strict": {"tp": 1, "fp": 3, "fn": 0}, "core": {"tp": 1, "fp": 3, "fn": 0}, "all": {"tp": 1, "fp": 3, "fn": 1}}, "sourcery": {"strict": {"tp": 1, "fp": 5, "fn": 0}, "core": {"tp": 1, "fp": 5, "fn": 0}, "all": {"tp": 1, "fp": 5, "fn": 1}}}}, {"url": "https://github.com/calcom/cal.com/pull/22532", "language": "TypeScript", "pr_size": "medium", "domain": "caching", "change_type": "feature", "complexity": "moderate", "difficulty": "subtle", "risk": "medium", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 1, "fp": 2, "fn": 3}, "core": {"tp": 1, "fp": 2, "fn": 3}, "all": {"tp": 1, "fp": 2, "fn": 3}}, "codeant-v2": {"strict": {"tp": 3, "fp": 3, "fn": 1}, "core": {"tp": 3, "fp": 3, "fn": 1}, "all": {"tp": 3, "fp": 3, "fn": 1}}, "coderabbit": {"strict": {"tp": 2, "fp": 3, "fn": 2}, "core": {"tp": 2, "fp": 3, "fn": 2}, "all": {"tp": 2, "fp": 3, "fn": 2}}, "qodo-extended-v2": {"strict": {"tp": 2, "fp": 2, "fn": 2}, "core": {"tp": 2, "fp": 2, "fn": 2}, "all": {"tp": 2, "fp": 2, "fn": 2}}, "greptile-v4-1": {"strict": {"tp": 2, "fp": 3, "fn": 2}, "core": {"tp": 2, "fp": 3, "fn": 2}, "all": {"tp": 2, "fp": 3, "fn": 2}}, "gemini-v2": {"strict": {"tp": 4, "fp": 0, "fn": 0}, "core": {"tp": 4, "fp": 0, "fn": 0}, "all": {"tp": 4, "fp": 0, "fn": 0}}, "gitlab": {"strict": {"tp": 2, "fp": 3, "fn": 2}, "core": {"tp": 2, "fp": 3, "fn": 2}, "all": {"tp": 2, "fp": 3, "fn": 2}}, "copilot-v2": {"strict": {"tp": 4, "fp": 2, "fn": 0}, "core": {"tp": 4, "fp": 2, "fn": 0}, "all": {"tp": 4, "fp": 2, "fn": 0}}, "augment": {"strict": {"tp": 4, "fp": 2, "fn": 0}, "core": {"tp": 4, "fp": 2, "fn": 0}, "all": {"tp": 4, "fp": 2, "fn": 0}}, "bugbot": {"strict": {"tp": 0, "fp": 1, "fn": 4}, "core": {"tp": 0, "fp": 1, "fn": 4}, "all": {"tp": 0, "fp": 1, "fn": 4}}, "claude": {"strict": {"tp": 0, "fp": 1, "fn": 4}, "core": {"tp": 0, "fp": 1, "fn": 4}, "all": {"tp": 0, "fp": 1, "fn": 4}}, "claude-code": {"strict": {"tp": 1, "fp": 3, "fn": 3}, "core": {"tp": 1, "fp": 3, "fn": 3}, "all": {"tp": 1, "fp": 3, "fn": 3}}, "cubic-v2": {"strict": {"tp": 0, "fp": 2, "fn": 4}, "core": {"tp": 0, "fp": 2, "fn": 4}, "all": {"tp": 0, "fp": 2, "fn": 4}}, "devin": {"strict": {"tp": 1, "fp": 1, "fn": 3}, "core": {"tp": 1, "fp": 1, "fn": 3}, "all": {"tp": 1, "fp": 1, "fn": 3}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 4}, "core": {"tp": 0, "fp": 0, "fn": 4}, "all": {"tp": 0, "fp": 0, "fn": 4}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 4}, "core": {"tp": 0, "fp": 0, "fn": 4}, "all": {"tp": 0, "fp": 0, "fn": 4}}, "kodus-v2": {"strict": {"tp": 1, "fp": 0, "fn": 3}, "core": {"tp": 1, "fp": 0, "fn": 3}, "all": {"tp": 1, "fp": 0, "fn": 3}}, "macroscope": {"strict": {"tp": 1, "fp": 1, "fn": 3}, "core": {"tp": 1, "fp": 1, "fn": 3}, "all": {"tp": 1, "fp": 1, "fn": 3}}, "qodo-v2": {"strict": {"tp": 3, "fp": 2, "fn": 1}, "core": {"tp": 3, "fp": 2, "fn": 1}, "all": {"tp": 3, "fp": 2, "fn": 1}}, "sourcery": {"strict": {"tp": 3, "fp": 1, "fn": 1}, "core": {"tp": 3, "fp": 1, "fn": 1}, "all": {"tp": 3, "fp": 1, "fn": 1}}}}, {"url": "https://github.com/calcom/cal.com/pull/8330", "language": "TypeScript", "pr_size": "small", "domain": "scheduling", "change_type": "bug_fix", "complexity": "moderate", "difficulty": "subtle", "risk": "medium", "context": "file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 2, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "codeant-v2": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "coderabbit": {"strict": {"tp": 2, "fp": 2, "fn": 0}, "core": {"tp": 2, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "qodo-extended-v2": {"strict": {"tp": 2, "fp": 2, "fn": 0}, "core": {"tp": 2, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "greptile-v4-1": {"strict": {"tp": 2, "fp": 2, "fn": 0}, "core": {"tp": 2, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "gemini-v2": {"strict": {"tp": 2, "fp": 3, "fn": 0}, "core": {"tp": 2, "fp": 3, "fn": 0}, "all": {"tp": 2, "fp": 3, "fn": 0}}, "gitlab": {"strict": {"tp": 2, "fp": 2, "fn": 0}, "core": {"tp": 2, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "copilot-v2": {"strict": {"tp": 2, "fp": 5, "fn": 0}, "core": {"tp": 2, "fp": 5, "fn": 0}, "all": {"tp": 2, "fp": 5, "fn": 0}}, "augment": {"strict": {"tp": 2, "fp": 3, "fn": 0}, "core": {"tp": 2, "fp": 3, "fn": 0}, "all": {"tp": 2, "fp": 3, "fn": 0}}, "bugbot": {"strict": {"tp": 2, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "claude": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "claude-code": {"strict": {"tp": 2, "fp": 2, "fn": 0}, "core": {"tp": 2, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "cubic-v2": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "devin": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "graphite": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "kg": {"strict": {"tp": 1, "fp": 2, "fn": 1}, "core": {"tp": 1, "fp": 2, "fn": 1}, "all": {"tp": 1, "fp": 2, "fn": 1}}, "kodus-v2": {"strict": {"tp": 2, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "macroscope": {"strict": {"tp": 2, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "qodo-v2": {"strict": {"tp": 2, "fp": 2, "fn": 0}, "core": {"tp": 2, "fp": 2, "fn": 0}, "all": {"tp": 2, "fp": 2, "fn": 0}}, "sourcery": {"strict": {"tp": 2, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}}}, {"url": "https://github.com/calcom/cal.com/pull/14943", "language": "TypeScript", "pr_size": "small", "domain": "scheduling", "change_type": "feature", "complexity": "moderate", "difficulty": "subtle", "risk": "high", "context": "file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 1, "fp": 3, "fn": 1}, "core": {"tp": 1, "fp": 3, "fn": 1}, "all": {"tp": 1, "fp": 3, "fn": 1}}, "codeant-v2": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 2}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "coderabbit": {"strict": {"tp": 2, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "qodo-extended-v2": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "greptile-v4-1": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "gemini-v2": {"strict": {"tp": 2, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "gitlab": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "copilot-v2": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "augment": {"strict": {"tp": 2, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "bugbot": {"strict": {"tp": 1, "fp": 2, "fn": 1}, "core": {"tp": 1, "fp": 2, "fn": 1}, "all": {"tp": 1, "fp": 2, "fn": 1}}, "claude": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "claude-code": {"strict": {"tp": 0, "fp": 1, "fn": 2}, "core": {"tp": 0, "fp": 1, "fn": 2}, "all": {"tp": 0, "fp": 1, "fn": 2}}, "cubic-v2": {"strict": {"tp": 2, "fp": 0, "fn": 0}, "core": {"tp": 2, "fp": 0, "fn": 0}, "all": {"tp": 2, "fp": 0, "fn": 0}}, "devin": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "graphite": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "kg": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "kodus-v2": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 1, "fp": 1, "fn": 1}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "macroscope": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "qodo-v2": {"strict": {"tp": 2, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}, "sourcery": {"strict": {"tp": 2, "fp": 1, "fn": 0}, "core": {"tp": 2, "fp": 1, "fn": 0}, "all": {"tp": 2, "fp": 1, "fn": 0}}}}, {"url": "https://github.com/calcom/cal.com/pull/22345", "language": "TypeScript", "pr_size": "small", "domain": "database", "change_type": "migration", "complexity": "moderate", "difficulty": "subtle", "risk": "medium", "context": "file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 0, "fp": 2, "fn": 1}, "core": {"tp": 0, "fp": 2, "fn": 1}, "all": {"tp": 0, "fp": 2, "fn": 2}}, "codeant-v2": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "coderabbit": {"strict": {"tp": 0, "fp": 3, "fn": 1}, "core": {"tp": 0, "fp": 3, "fn": 1}, "all": {"tp": 1, "fp": 3, "fn": 1}}, "qodo-extended-v2": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "greptile-v4-1": {"strict": {"tp": 0, "fp": 3, "fn": 1}, "core": {"tp": 0, "fp": 3, "fn": 1}, "all": {"tp": 0, "fp": 3, "fn": 2}}, "gemini-v2": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 0}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "gitlab": {"strict": {"tp": 1, "fp": 3, "fn": 0}, "core": {"tp": 1, "fp": 3, "fn": 0}, "all": {"tp": 1, "fp": 3, "fn": 1}}, "copilot-v2": {"strict": {"tp": 0, "fp": 3, "fn": 1}, "core": {"tp": 0, "fp": 3, "fn": 1}, "all": {"tp": 1, "fp": 3, "fn": 1}}, "augment": {"strict": {"tp": 0, "fp": 3, "fn": 1}, "core": {"tp": 0, "fp": 3, "fn": 1}, "all": {"tp": 1, "fp": 3, "fn": 1}}, "bugbot": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 1, "fp": 1, "fn": 0}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "claude": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "claude-code": {"strict": {"tp": 1, "fp": 2, "fn": 0}, "core": {"tp": 1, "fp": 2, "fn": 0}, "all": {"tp": 1, "fp": 2, "fn": 1}}, "cubic-v2": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 1, "fp": 1, "fn": 0}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "devin": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "kg": {"strict": {"tp": 0, "fp": 1, "fn": 1}, "core": {"tp": 0, "fp": 1, "fn": 1}, "all": {"tp": 0, "fp": 1, "fn": 2}}, "kodus-v2": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "macroscope": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "qodo-v2": {"strict": {"tp": 0, "fp": 3, "fn": 1}, "core": {"tp": 0, "fp": 3, "fn": 1}, "all": {"tp": 1, "fp": 3, "fn": 1}}, "sourcery": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 0}, "all": {"tp": 1, "fp": 0, "fn": 1}}}}, {"url": "https://github.com/calcom/cal.com/pull/11059", "language": "TypeScript", "pr_size": "large", "domain": "authentication", "change_type": "feature", "complexity": "complex", "difficulty": "subtle", "risk": "critical", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 4, "fp": 1, "fn": 5}, "core": {"tp": 4, "fp": 1, "fn": 5}, "all": {"tp": 4, "fp": 1, "fn": 5}}, "codeant-v2": {"strict": {"tp": 4, "fp": 5, "fn": 5}, "core": {"tp": 4, "fp": 5, "fn": 5}, "all": {"tp": 4, "fp": 5, "fn": 5}}, "coderabbit": {"strict": {"tp": 8, "fp": 13, "fn": 1}, "core": {"tp": 8, "fp": 13, "fn": 1}, "all": {"tp": 8, "fp": 13, "fn": 1}}, "qodo-extended-v2": {"strict": {"tp": 7, "fp": 2, "fn": 2}, "core": {"tp": 7, "fp": 2, "fn": 2}, "all": {"tp": 7, "fp": 2, "fn": 2}}, "greptile-v4-1": {"strict": {"tp": 5, "fp": 4, "fn": 4}, "core": {"tp": 5, "fp": 4, "fn": 4}, "all": {"tp": 5, "fp": 4, "fn": 4}}, "gemini-v2": {"strict": {"tp": 4, "fp": 2, "fn": 5}, "core": {"tp": 4, "fp": 2, "fn": 5}, "all": {"tp": 4, "fp": 2, "fn": 5}}, "gitlab": {"strict": {"tp": 5, "fp": 5, "fn": 4}, "core": {"tp": 5, "fp": 5, "fn": 4}, "all": {"tp": 5, "fp": 5, "fn": 4}}, "copilot-v2": {"strict": {"tp": 7, "fp": 3, "fn": 2}, "core": {"tp": 7, "fp": 3, "fn": 2}, "all": {"tp": 7, "fp": 3, "fn": 2}}, "augment": {"strict": {"tp": 8, "fp": 5, "fn": 1}, "core": {"tp": 8, "fp": 5, "fn": 1}, "all": {"tp": 8, "fp": 5, "fn": 1}}, "bugbot": {"strict": {"tp": 6, "fp": 2, "fn": 3}, "core": {"tp": 6, "fp": 2, "fn": 3}, "all": {"tp": 6, "fp": 2, "fn": 3}}, "claude": {"strict": {"tp": 5, "fp": 1, "fn": 4}, "core": {"tp": 5, "fp": 1, "fn": 4}, "all": {"tp": 5, "fp": 1, "fn": 4}}, "claude-code": {"strict": {"tp": 4, "fp": 0, "fn": 5}, "core": {"tp": 4, "fp": 0, "fn": 5}, "all": {"tp": 4, "fp": 0, "fn": 5}}, "cubic-v2": {"strict": {"tp": 3, "fp": 0, "fn": 6}, "core": {"tp": 3, "fp": 0, "fn": 6}, "all": {"tp": 3, "fp": 0, "fn": 6}}, "devin": {"strict": {"tp": 5, "fp": 0, "fn": 4}, "core": {"tp": 5, "fp": 0, "fn": 4}, "all": {"tp": 5, "fp": 0, "fn": 4}}, "graphite": {"strict": {"tp": 1, "fp": 1, "fn": 8}, "core": {"tp": 1, "fp": 1, "fn": 8}, "all": {"tp": 1, "fp": 1, "fn": 8}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 9}, "core": {"tp": 0, "fp": 0, "fn": 9}, "all": {"tp": 0, "fp": 0, "fn": 9}}, "kodus-v2": {"strict": {"tp": 4, "fp": 3, "fn": 5}, "core": {"tp": 4, "fp": 3, "fn": 5}, "all": {"tp": 4, "fp": 3, "fn": 5}}, "macroscope": {"strict": {"tp": 5, "fp": 5, "fn": 4}, "core": {"tp": 5, "fp": 5, "fn": 4}, "all": {"tp": 5, "fp": 5, "fn": 4}}, "qodo-v2": {"strict": {"tp": 9, "fp": 2, "fn": 0}, "core": {"tp": 9, "fp": 2, "fn": 0}, "all": {"tp": 9, "fp": 2, "fn": 0}}, "sourcery": {"strict": {"tp": 5, "fp": 0, "fn": 4}, "core": {"tp": 5, "fp": 0, "fn": 4}, "all": {"tp": 5, "fp": 0, "fn": 4}}}}, {"url": "https://github.com/calcom/cal.com/pull/7232", "language": "TypeScript", "pr_size": "medium", "domain": "scheduling", "change_type": "feature", "complexity": "moderate", "difficulty": "subtle", "risk": "high", "context": "cross_file", "concern": "reliability", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 1, "fp": 1, "fn": 2}, "core": {"tp": 1, "fp": 1, "fn": 2}, "all": {"tp": 1, "fp": 1, "fn": 2}}, "codeant-v2": {"strict": {"tp": 1, "fp": 2, "fn": 2}, "core": {"tp": 1, "fp": 2, "fn": 2}, "all": {"tp": 1, "fp": 2, "fn": 2}}, "coderabbit": {"strict": {"tp": 2, "fp": 12, "fn": 1}, "core": {"tp": 2, "fp": 12, "fn": 1}, "all": {"tp": 2, "fp": 12, "fn": 1}}, "qodo-extended-v2": {"strict": {"tp": 2, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 1}, "all": {"tp": 2, "fp": 1, "fn": 1}}, "greptile-v4-1": {"strict": {"tp": 1, "fp": 2, "fn": 2}, "core": {"tp": 1, "fp": 2, "fn": 2}, "all": {"tp": 1, "fp": 2, "fn": 2}}, "gemini-v2": {"strict": {"tp": 2, "fp": 3, "fn": 1}, "core": {"tp": 2, "fp": 3, "fn": 1}, "all": {"tp": 2, "fp": 3, "fn": 1}}, "gitlab": {"strict": {"tp": 0, "fp": 3, "fn": 3}, "core": {"tp": 0, "fp": 3, "fn": 3}, "all": {"tp": 0, "fp": 3, "fn": 3}}, "copilot-v2": {"strict": {"tp": 0, "fp": 3, "fn": 3}, "core": {"tp": 0, "fp": 3, "fn": 3}, "all": {"tp": 0, "fp": 3, "fn": 3}}, "augment": {"strict": {"tp": 1, "fp": 2, "fn": 2}, "core": {"tp": 1, "fp": 2, "fn": 2}, "all": {"tp": 1, "fp": 2, "fn": 2}}, "bugbot": {"strict": {"tp": 1, "fp": 3, "fn": 2}, "core": {"tp": 1, "fp": 3, "fn": 2}, "all": {"tp": 1, "fp": 3, "fn": 2}}, "claude": {"strict": {"tp": 1, "fp": 3, "fn": 2}, "core": {"tp": 1, "fp": 3, "fn": 2}, "all": {"tp": 1, "fp": 3, "fn": 2}}, "claude-code": {"strict": {"tp": 1, "fp": 1, "fn": 2}, "core": {"tp": 1, "fp": 1, "fn": 2}, "all": {"tp": 1, "fp": 1, "fn": 2}}, "cubic-v2": {"strict": {"tp": 2, "fp": 3, "fn": 1}, "core": {"tp": 2, "fp": 3, "fn": 1}, "all": {"tp": 2, "fp": 3, "fn": 1}}, "devin": {"strict": {"tp": 2, "fp": 3, "fn": 1}, "core": {"tp": 2, "fp": 3, "fn": 1}, "all": {"tp": 2, "fp": 3, "fn": 1}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 3}, "core": {"tp": 0, "fp": 0, "fn": 3}, "all": {"tp": 0, "fp": 0, "fn": 3}}, "kg": {"strict": {"tp": 0, "fp": 1, "fn": 3}, "core": {"tp": 0, "fp": 1, "fn": 3}, "all": {"tp": 0, "fp": 1, "fn": 3}}, "kodus-v2": {"strict": {"tp": 1, "fp": 3, "fn": 2}, "core": {"tp": 1, "fp": 3, "fn": 2}, "all": {"tp": 1, "fp": 3, "fn": 2}}, "macroscope": {"strict": {"tp": 1, "fp": 2, "fn": 2}, "core": {"tp": 1, "fp": 2, "fn": 2}, "all": {"tp": 1, "fp": 2, "fn": 2}}, "qodo-v2": {"strict": {"tp": 2, "fp": 3, "fn": 1}, "core": {"tp": 2, "fp": 3, "fn": 1}, "all": {"tp": 2, "fp": 3, "fn": 1}}, "sourcery": {"strict": {"tp": 1, "fp": 2, "fn": 2}, "core": {"tp": 1, "fp": 2, "fn": 2}, "all": {"tp": 1, "fp": 2, "fn": 2}}}}, {"url": "https://github.com/calcom/cal.com/pull/14740", "language": "TypeScript", "pr_size": "large", "domain": "scheduling", "change_type": "feature", "complexity": "moderate", "difficulty": "subtle", "risk": "critical", "context": "cross_file", "concern": "security", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 1, "fp": 1, "fn": 5}, "core": {"tp": 1, "fp": 1, "fn": 5}, "all": {"tp": 1, "fp": 1, "fn": 5}}, "codeant-v2": {"strict": {"tp": 3, "fp": 4, "fn": 3}, "core": {"tp": 3, "fp": 4, "fn": 3}, "all": {"tp": 3, "fp": 4, "fn": 3}}, "coderabbit": {"strict": {"tp": 4, "fp": 7, "fn": 2}, "core": {"tp": 4, "fp": 7, "fn": 2}, "all": {"tp": 4, "fp": 7, "fn": 2}}, "qodo-extended-v2": {"strict": {"tp": 4, "fp": 1, "fn": 2}, "core": {"tp": 4, "fp": 1, "fn": 2}, "all": {"tp": 4, "fp": 1, "fn": 2}}, "greptile-v4-1": {"strict": {"tp": 3, "fp": 2, "fn": 3}, "core": {"tp": 3, "fp": 2, "fn": 3}, "all": {"tp": 3, "fp": 2, "fn": 3}}, "gemini-v2": {"strict": {"tp": 3, "fp": 4, "fn": 3}, "core": {"tp": 3, "fp": 4, "fn": 3}, "all": {"tp": 3, "fp": 4, "fn": 3}}, "gitlab": {"strict": {"tp": 3, "fp": 4, "fn": 3}, "core": {"tp": 3, "fp": 4, "fn": 3}, "all": {"tp": 3, "fp": 4, "fn": 3}}, "copilot-v2": {"strict": {"tp": 5, "fp": 5, "fn": 1}, "core": {"tp": 5, "fp": 5, "fn": 1}, "all": {"tp": 5, "fp": 5, "fn": 1}}, "augment": {"strict": {"tp": 4, "fp": 3, "fn": 2}, "core": {"tp": 4, "fp": 3, "fn": 2}, "all": {"tp": 4, "fp": 3, "fn": 2}}, "bugbot": {"strict": {"tp": 3, "fp": 3, "fn": 3}, "core": {"tp": 3, "fp": 3, "fn": 3}, "all": {"tp": 3, "fp": 3, "fn": 3}}, "claude": {"strict": {"tp": 2, "fp": 0, "fn": 4}, "core": {"tp": 2, "fp": 0, "fn": 4}, "all": {"tp": 2, "fp": 0, "fn": 4}}, "claude-code": {"strict": {"tp": 2, "fp": 1, "fn": 4}, "core": {"tp": 2, "fp": 1, "fn": 4}, "all": {"tp": 2, "fp": 1, "fn": 4}}, "cubic-v2": {"strict": {"tp": 3, "fp": 3, "fn": 3}, "core": {"tp": 3, "fp": 3, "fn": 3}, "all": {"tp": 3, "fp": 3, "fn": 3}}, "devin": {"strict": {"tp": 2, "fp": 0, "fn": 4}, "core": {"tp": 2, "fp": 0, "fn": 4}, "all": {"tp": 2, "fp": 0, "fn": 4}}, "graphite": {"strict": {"tp": 1, "fp": 1, "fn": 5}, "core": {"tp": 1, "fp": 1, "fn": 5}, "all": {"tp": 1, "fp": 1, "fn": 5}}, "kg": {"strict": {"tp": 2, "fp": 0, "fn": 4}, "core": {"tp": 2, "fp": 0, "fn": 4}, "all": {"tp": 2, "fp": 0, "fn": 4}}, "kodus-v2": {"strict": {"tp": 2, "fp": 0, "fn": 4}, "core": {"tp": 2, "fp": 0, "fn": 4}, "all": {"tp": 2, "fp": 0, "fn": 4}}, "macroscope": {"strict": {"tp": 3, "fp": 3, "fn": 3}, "core": {"tp": 3, "fp": 3, "fn": 3}, "all": {"tp": 3, "fp": 3, "fn": 3}}, "qodo-v2": {"strict": {"tp": 5, "fp": 3, "fn": 1}, "core": {"tp": 5, "fp": 3, "fn": 1}, "all": {"tp": 5, "fp": 3, "fn": 1}}, "sourcery": {"strict": {"tp": 2, "fp": 3, "fn": 4}, "core": {"tp": 2, "fp": 3, "fn": 4}, "all": {"tp": 2, "fp": 3, "fn": 4}}}}, {"url": "https://github.com/calcom/cal.com/pull/10600", "language": "TypeScript", "pr_size": "medium", "domain": "authentication", "change_type": "feature", "complexity": "moderate", "difficulty": "subtle", "risk": "high", "context": "cross_file", "concern": "security", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 0, "fp": 1, "fn": 2}, "core": {"tp": 0, "fp": 1, "fn": 4}, "all": {"tp": 0, "fp": 1, "fn": 5}}, "codeant-v2": {"strict": {"tp": 1, "fp": 4, "fn": 1}, "core": {"tp": 1, "fp": 4, "fn": 3}, "all": {"tp": 1, "fp": 4, "fn": 4}}, "coderabbit": {"strict": {"tp": 1, "fp": 7, "fn": 1}, "core": {"tp": 1, "fp": 7, "fn": 3}, "all": {"tp": 2, "fp": 7, "fn": 3}}, "qodo-extended-v2": {"strict": {"tp": 1, "fp": 2, "fn": 1}, "core": {"tp": 2, "fp": 2, "fn": 2}, "all": {"tp": 2, "fp": 2, "fn": 3}}, "greptile-v4-1": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 2}, "all": {"tp": 3, "fp": 1, "fn": 2}}, "gemini-v2": {"strict": {"tp": 0, "fp": 5, "fn": 2}, "core": {"tp": 0, "fp": 5, "fn": 4}, "all": {"tp": 1, "fp": 5, "fn": 4}}, "gitlab": {"strict": {"tp": 0, "fp": 2, "fn": 2}, "core": {"tp": 1, "fp": 2, "fn": 3}, "all": {"tp": 2, "fp": 2, "fn": 3}}, "copilot-v2": {"strict": {"tp": 2, "fp": 3, "fn": 0}, "core": {"tp": 4, "fp": 3, "fn": 0}, "all": {"tp": 5, "fp": 3, "fn": 0}}, "augment": {"strict": {"tp": 0, "fp": 3, "fn": 2}, "core": {"tp": 1, "fp": 3, "fn": 3}, "all": {"tp": 2, "fp": 3, "fn": 3}}, "bugbot": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 3}, "all": {"tp": 1, "fp": 0, "fn": 4}}, "claude": {"strict": {"tp": 0, "fp": 10, "fn": 2}, "core": {"tp": 1, "fp": 10, "fn": 3}, "all": {"tp": 2, "fp": 10, "fn": 3}}, "claude-code": {"strict": {"tp": 0, "fp": 4, "fn": 2}, "core": {"tp": 0, "fp": 4, "fn": 4}, "all": {"tp": 1, "fp": 4, "fn": 4}}, "cubic-v2": {"strict": {"tp": 2, "fp": 2, "fn": 0}, "core": {"tp": 2, "fp": 2, "fn": 2}, "all": {"tp": 2, "fp": 2, "fn": 3}}, "devin": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 3}, "all": {"tp": 1, "fp": 0, "fn": 4}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 4}, "all": {"tp": 0, "fp": 0, "fn": 5}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 2}, "core": {"tp": 0, "fp": 0, "fn": 4}, "all": {"tp": 0, "fp": 0, "fn": 5}}, "kodus-v2": {"strict": {"tp": 1, "fp": 0, "fn": 1}, "core": {"tp": 1, "fp": 0, "fn": 3}, "all": {"tp": 1, "fp": 0, "fn": 4}}, "macroscope": {"strict": {"tp": 0, "fp": 3, "fn": 2}, "core": {"tp": 0, "fp": 3, "fn": 4}, "all": {"tp": 0, "fp": 3, "fn": 5}}, "qodo-v2": {"strict": {"tp": 1, "fp": 5, "fn": 1}, "core": {"tp": 2, "fp": 5, "fn": 2}, "all": {"tp": 2, "fp": 5, "fn": 3}}, "sourcery": {"strict": {"tp": 1, "fp": 1, "fn": 1}, "core": {"tp": 2, "fp": 1, "fn": 2}, "all": {"tp": 3, "fp": 1, "fn": 2}}}}, {"url": "https://github.com/calcom/cal.com/pull/10967", "language": "TypeScript", "pr_size": "large", "domain": "scheduling", "change_type": "bug_fix", "complexity": "complex", "difficulty": "subtle", "risk": "high", "context": "cross_file", "concern": "correctness", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 1, "fp": 2, "fn": 4}, "core": {"tp": 1, "fp": 2, "fn": 4}, "all": {"tp": 1, "fp": 2, "fn": 5}}, "codeant-v2": {"strict": {"tp": 2, "fp": 6, "fn": 3}, "core": {"tp": 2, "fp": 6, "fn": 3}, "all": {"tp": 2, "fp": 6, "fn": 4}}, "coderabbit": {"strict": {"tp": 5, "fp": 15, "fn": 0}, "core": {"tp": 5, "fp": 15, "fn": 0}, "all": {"tp": 5, "fp": 15, "fn": 1}}, "qodo-extended-v2": {"strict": {"tp": 3, "fp": 2, "fn": 2}, "core": {"tp": 3, "fp": 2, "fn": 2}, "all": {"tp": 3, "fp": 2, "fn": 3}}, "greptile-v4-1": {"strict": {"tp": 2, "fp": 2, "fn": 3}, "core": {"tp": 2, "fp": 2, "fn": 3}, "all": {"tp": 2, "fp": 2, "fn": 4}}, "gemini-v2": {"strict": {"tp": 1, "fp": 6, "fn": 4}, "core": {"tp": 1, "fp": 6, "fn": 4}, "all": {"tp": 1, "fp": 6, "fn": 5}}, "gitlab": {"strict": {"tp": 4, "fp": 1, "fn": 1}, "core": {"tp": 4, "fp": 1, "fn": 1}, "all": {"tp": 4, "fp": 1, "fn": 2}}, "copilot-v2": {"strict": {"tp": 4, "fp": 8, "fn": 1}, "core": {"tp": 4, "fp": 8, "fn": 1}, "all": {"tp": 4, "fp": 8, "fn": 2}}, "augment": {"strict": {"tp": 3, "fp": 2, "fn": 2}, "core": {"tp": 3, "fp": 2, "fn": 2}, "all": {"tp": 3, "fp": 2, "fn": 3}}, "bugbot": {"strict": {"tp": 3, "fp": 3, "fn": 2}, "core": {"tp": 3, "fp": 3, "fn": 2}, "all": {"tp": 3, "fp": 3, "fn": 3}}, "claude": {"strict": {"tp": 3, "fp": 4, "fn": 2}, "core": {"tp": 3, "fp": 4, "fn": 2}, "all": {"tp": 3, "fp": 4, "fn": 3}}, "claude-code": {"strict": {"tp": 4, "fp": 3, "fn": 1}, "core": {"tp": 4, "fp": 3, "fn": 1}, "all": {"tp": 4, "fp": 3, "fn": 2}}, "cubic-v2": {"strict": {"tp": 1, "fp": 2, "fn": 4}, "core": {"tp": 1, "fp": 2, "fn": 4}, "all": {"tp": 1, "fp": 2, "fn": 5}}, "devin": {"strict": {"tp": 3, "fp": 2, "fn": 2}, "core": {"tp": 3, "fp": 2, "fn": 2}, "all": {"tp": 3, "fp": 2, "fn": 3}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 5}, "core": {"tp": 0, "fp": 0, "fn": 5}, "all": {"tp": 0, "fp": 0, "fn": 6}}, "kg": {"strict": {"tp": 0, "fp": 0, "fn": 5}, "core": {"tp": 0, "fp": 0, "fn": 5}, "all": {"tp": 0, "fp": 0, "fn": 6}}, "kodus-v2": {"strict": {"tp": 2, "fp": 2, "fn": 3}, "core": {"tp": 2, "fp": 2, "fn": 3}, "all": {"tp": 2, "fp": 2, "fn": 4}}, "macroscope": {"strict": {"tp": 2, "fp": 3, "fn": 3}, "core": {"tp": 2, "fp": 3, "fn": 3}, "all": {"tp": 2, "fp": 3, "fn": 4}}, "qodo-v2": {"strict": {"tp": 3, "fp": 2, "fn": 2}, "core": {"tp": 3, "fp": 2, "fn": 2}, "all": {"tp": 3, "fp": 2, "fn": 3}}, "sourcery": {"strict": {"tp": 2, "fp": 3, "fn": 3}, "core": {"tp": 2, "fp": 3, "fn": 3}, "all": {"tp": 2, "fp": 3, "fn": 4}}}}, {"url": "https://github.com/calcom/cal.com/pull/8087", "language": "TypeScript", "pr_size": "medium", "domain": "concurrency", "change_type": "refactoring", "complexity": "moderate", "difficulty": "subtle", "risk": "critical", "context": "cross_file", "concern": "reliability", "summary": "", "tool_metrics": {"baz": {"strict": {"tp": 1, "fp": 3, "fn": 0}, "core": {"tp": 1, "fp": 3, "fn": 0}, "all": {"tp": 1, "fp": 3, "fn": 1}}, "codeant-v2": {"strict": {"tp": 1, "fp": 2, "fn": 0}, "core": {"tp": 1, "fp": 2, "fn": 0}, "all": {"tp": 1, "fp": 2, "fn": 1}}, "coderabbit": {"strict": {"tp": 1, "fp": 7, "fn": 0}, "core": {"tp": 1, "fp": 7, "fn": 0}, "all": {"tp": 2, "fp": 7, "fn": 0}}, "qodo-extended-v2": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 1, "fp": 1, "fn": 0}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "greptile-v4-1": {"strict": {"tp": 1, "fp": 6, "fn": 0}, "core": {"tp": 1, "fp": 6, "fn": 0}, "all": {"tp": 1, "fp": 6, "fn": 1}}, "gemini-v2": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 0}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "gitlab": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 1, "fp": 1, "fn": 0}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "copilot-v2": {"strict": {"tp": 1, "fp": 2, "fn": 0}, "core": {"tp": 1, "fp": 2, "fn": 0}, "all": {"tp": 1, "fp": 2, "fn": 1}}, "augment": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 1, "fp": 1, "fn": 0}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "bugbot": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 0}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "claude": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 1, "fp": 1, "fn": 0}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "claude-code": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 1, "fp": 1, "fn": 0}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "cubic-v2": {"strict": {"tp": 1, "fp": 7, "fn": 0}, "core": {"tp": 1, "fp": 7, "fn": 0}, "all": {"tp": 2, "fp": 7, "fn": 0}}, "devin": {"strict": {"tp": 1, "fp": 7, "fn": 0}, "core": {"tp": 1, "fp": 7, "fn": 0}, "all": {"tp": 1, "fp": 7, "fn": 1}}, "graphite": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "kg": {"strict": {"tp": 1, "fp": 0, "fn": 0}, "core": {"tp": 1, "fp": 0, "fn": 0}, "all": {"tp": 1, "fp": 0, "fn": 1}}, "kodus-v2": {"strict": {"tp": 0, "fp": 0, "fn": 1}, "core": {"tp": 0, "fp": 0, "fn": 1}, "all": {"tp": 0, "fp": 0, "fn": 2}}, "macroscope": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 1, "fp": 1, "fn": 0}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "qodo-v2": {"strict": {"tp": 1, "fp": 1, "fn": 0}, "core": {"tp": 1, "fp": 1, "fn": 0}, "all": {"tp": 1, "fp": 1, "fn": 1}}, "sourcery": {"strict": {"tp": 1, "fp": 3, "fn": 0}, "core": {"tp": 1, "fp": 3, "fn": 0}, "all": {"tp": 1, "fp": 3, "fn": 1}}}}], "tools": ["augment", "baz", "bugbot", "claude", "claude-code", "codeant-v2", "coderabbit", "copilot-v2", "cubic-v2", "devin", "gemini-v2", "gitlab", "graphite", "greptile-v4-1", "kg", "kodus-v2", "macroscope", "qodo-extended-v2", "qodo-v2", "sourcery"], "dimensions": {"language": ["Go", "Java", "Python", "Ruby", "TypeScript"], "pr_size": ["small", "medium", "large"], "domain": ["API", "UI", "authentication", "caching", "concurrency", "configuration", "data_processing", "database", "file_io", "logging", "scheduling", "serialization", "testing"], "change_type": ["bug_fix", "feature", "migration", "performance", "refactoring", "security_patch"], "complexity": ["simple", "moderate", "complex"], "difficulty": ["obvious", "moderate", "subtle", "very_subtle"], "risk": ["low", "medium", "high", "critical"], "context": ["local", "file", "cross_file", "system"], "concern": ["correctness", "maintainability", "reliability", "security"]}, "overall_metrics": {"sourcery": {"strict": {"precision": 33.3, "recall": 50.4, "f1": 40.1, "f2": 45.7, "tp": 70, "fp": 140, "fn": 69, "num_prs": 50}, "core": {"precision": 36.1, "recall": 50.0, "f1": 41.9, "f2": 46.4, "tp": 79, "fp": 140, "fn": 79, "num_prs": 50}, "all": {"precision": 38.1, "recall": 49.7, "f1": 43.1, "f2": 46.8, "tp": 86, "fp": 140, "fn": 87, "num_prs": 50}}, "macroscope": {"strict": {"precision": 47.7, "recall": 44.6, "f1": 46.1, "f2": 45.2, "tp": 62, "fp": 68, "fn": 77, "num_prs": 50}, "core": {"precision": 48.9, "recall": 41.1, "f1": 44.7, "f2": 42.5, "tp": 65, "fp": 68, "fn": 93, "num_prs": 50}, "all": {"precision": 50.0, "recall": 39.3, "f1": 44.0, "f2": 41.1, "tp": 68, "fp": 68, "fn": 105, "num_prs": 50}}, "graphite": {"strict": {"precision": 71.4, "recall": 7.2, "f1": 13.1, "f2": 8.8, "tp": 10, "fp": 4, "fn": 129, "num_prs": 50}, "core": {"precision": 73.3, "recall": 7.0, "f1": 12.7, "f2": 8.5, "tp": 11, "fp": 4, "fn": 147, "num_prs": 50}, "all": {"precision": 73.3, "recall": 6.4, "f1": 11.7, "f2": 7.8, "tp": 11, "fp": 4, "fn": 162, "num_prs": 50}}, "augment": {"strict": {"precision": 45.9, "recall": 59.7, "f1": 51.9, "f2": 56.3, "tp": 83, "fp": 98, "fn": 56, "num_prs": 50}, "core": {"precision": 49.5, "recall": 60.8, "f1": 54.5, "f2": 58.1, "tp": 96, "fp": 98, "fn": 62, "num_prs": 50}, "all": {"precision": 50.8, "recall": 58.4, "f1": 54.3, "f2": 56.7, "tp": 101, "fp": 98, "fn": 72, "num_prs": 50}}, "qodo-extended-v2": {"strict": {"precision": 55.9, "recall": 61.2, "f1": 58.4, "f2": 60.0, "tp": 85, "fp": 67, "fn": 54, "num_prs": 50}, "core": {"precision": 58.4, "recall": 59.5, "f1": 58.9, "f2": 59.3, "tp": 94, "fp": 67, "fn": 64, "num_prs": 50}, "all": {"precision": 58.9, "recall": 55.5, "f1": 57.1, "f2": 56.1, "tp": 96, "fp": 67, "fn": 77, "num_prs": 50}}, "kodus-v2": {"strict": {"precision": 50.5, "recall": 36.0, "f1": 42.0, "f2": 38.2, "tp": 50, "fp": 49, "fn": 89, "num_prs": 50}, "core": {"precision": 52.4, "recall": 34.2, "f1": 41.4, "f2": 36.7, "tp": 54, "fp": 49, "fn": 104, "num_prs": 50}, "all": {"precision": 53.8, "recall": 32.9, "f1": 40.9, "f2": 35.7, "tp": 57, "fp": 49, "fn": 116, "num_prs": 50}}, "kg": {"strict": {"precision": 53.5, "recall": 16.5, "f1": 25.3, "f2": 19.2, "tp": 23, "fp": 20, "fn": 116, "num_prs": 50}, "core": {"precision": 54.5, "recall": 15.2, "f1": 23.8, "f2": 17.8, "tp": 24, "fp": 20, "fn": 134, "num_prs": 50}, "all": {"precision": 55.6, "recall": 14.5, "f1": 22.9, "f2": 17.0, "tp": 25, "fp": 20, "fn": 148, "num_prs": 50}}, "cubic-v2": {"strict": {"precision": 53.2, "recall": 59.7, "f1": 56.3, "f2": 58.3, "tp": 83, "fp": 73, "fn": 56, "num_prs": 50}, "core": {"precision": 56.8, "recall": 60.8, "f1": 58.7, "f2": 59.9, "tp": 96, "fp": 73, "fn": 62, "num_prs": 50}, "all": {"precision": 57.8, "recall": 57.8, "f1": 57.8, "f2": 57.8, "tp": 100, "fp": 73, "fn": 73, "num_prs": 50}}, "gemini-v2": {"strict": {"precision": 36.2, "recall": 54.0, "f1": 43.4, "f2": 49.1, "tp": 75, "fp": 132, "fn": 64, "num_prs": 50}, "core": {"precision": 38.0, "recall": 51.3, "f1": 43.7, "f2": 47.9, "tp": 81, "fp": 132, "fn": 77, "num_prs": 50}, "all": {"precision": 40.3, "recall": 51.4, "f1": 45.2, "f2": 48.7, "tp": 89, "fp": 132, "fn": 84, "num_prs": 50}}, "greptile-v4-1": {"strict": {"precision": 40.2, "recall": 50.4, "f1": 44.7, "f2": 47.9, "tp": 70, "fp": 104, "fn": 69, "num_prs": 50}, "core": {"precision": 42.2, "recall": 48.1, "f1": 45.0, "f2": 46.8, "tp": 76, "fp": 104, "fn": 82, "num_prs": 50}, "all": {"precision": 44.7, "recall": 48.6, "f1": 46.5, "f2": 47.7, "tp": 84, "fp": 104, "fn": 89, "num_prs": 50}}, "bugbot": {"strict": {"precision": 49.6, "recall": 47.5, "f1": 48.5, "f2": 47.9, "tp": 66, "fp": 67, "fn": 73, "num_prs": 50}, "core": {"precision": 51.8, "recall": 45.6, "f1": 48.5, "f2": 46.7, "tp": 72, "fp": 67, "fn": 86, "num_prs": 50}, "all": {"precision": 52.8, "recall": 43.4, "f1": 47.6, "f2": 45.0, "tp": 75, "fp": 67, "fn": 98, "num_prs": 50}}, "claude": {"strict": {"precision": 36.7, "recall": 38.8, "f1": 37.8, "f2": 38.4, "tp": 54, "fp": 93, "fn": 85, "num_prs": 50}, "core": {"precision": 38.8, "recall": 37.3, "f1": 38.1, "f2": 37.6, "tp": 59, "fp": 93, "fn": 99, "num_prs": 50}, "all": {"precision": 41.1, "recall": 37.6, "f1": 39.3, "f2": 38.2, "tp": 65, "fp": 93, "fn": 108, "num_prs": 50}}, "devin": {"strict": {"precision": 56.6, "recall": 43.2, "f1": 49.0, "f2": 45.3, "tp": 60, "fp": 46, "fn": 79, "num_prs": 50}, "core": {"precision": 58.6, "recall": 41.1, "f1": 48.3, "f2": 43.7, "tp": 65, "fp": 46, "fn": 93, "num_prs": 50}, "all": {"precision": 59.3, "recall": 38.7, "f1": 46.9, "f2": 41.6, "tp": 67, "fp": 46, "fn": 106, "num_prs": 50}}, "baz": {"strict": {"precision": 43.9, "recall": 36.0, "f1": 39.5, "f2": 37.3, "tp": 50, "fp": 64, "fn": 89, "num_prs": 50}, "core": {"precision": 44.8, "recall": 32.9, "f1": 38.0, "f2": 34.8, "tp": 52, "fp": 64, "fn": 106, "num_prs": 50}, "all": {"precision": 46.2, "recall": 31.8, "f1": 37.7, "f2": 33.9, "tp": 55, "fp": 64, "fn": 118, "num_prs": 50}}, "claude-code": {"strict": {"precision": 38.2, "recall": 43.2, "f1": 40.5, "f2": 42.1, "tp": 60, "fp": 97, "fn": 79, "num_prs": 50}, "core": {"precision": 40.1, "recall": 41.1, "f1": 40.6, "f2": 40.9, "tp": 65, "fp": 97, "fn": 93, "num_prs": 50}, "all": {"precision": 42.3, "recall": 41.0, "f1": 41.6, "f2": 41.3, "tp": 71, "fp": 97, "fn": 102, "num_prs": 50}}, "gitlab": {"strict": {"precision": 42.2, "recall": 48.9, "f1": 45.3, "f2": 47.4, "tp": 68, "fp": 93, "fn": 71, "num_prs": 50}, "core": {"precision": 45.0, "recall": 48.1, "f1": 46.5, "f2": 47.4, "tp": 76, "fp": 93, "fn": 82, "num_prs": 50}, "all": {"precision": 46.9, "recall": 47.4, "f1": 47.1, "f2": 47.3, "tp": 82, "fp": 93, "fn": 91, "num_prs": 50}}, "qodo-v2": {"strict": {"precision": 44.1, "recall": 64.7, "f1": 52.5, "f2": 59.2, "tp": 90, "fp": 114, "fn": 49, "num_prs": 50}, "core": {"precision": 46.5, "recall": 62.7, "f1": 53.4, "f2": 58.6, "tp": 99, "fp": 114, "fn": 59, "num_prs": 50}, "all": {"precision": 47.5, "recall": 59.5, "f1": 52.8, "f2": 56.7, "tp": 103, "fp": 114, "fn": 70, "num_prs": 50}}, "coderabbit": {"strict": {"precision": 26.8, "recall": 61.9, "f1": 37.4, "f2": 49.0, "tp": 86, "fp": 235, "fn": 53, "num_prs": 50}, "core": {"precision": 28.6, "recall": 59.5, "f1": 38.6, "f2": 48.9, "tp": 94, "fp": 235, "fn": 64, "num_prs": 50}, "all": {"precision": 30.7, "recall": 60.1, "f1": 40.6, "f2": 50.4, "tp": 104, "fp": 235, "fn": 69, "num_prs": 50}}, "codeant-v2": {"strict": {"precision": 29.3, "recall": 38.1, "f1": 33.1, "f2": 36.0, "tp": 53, "fp": 128, "fn": 86, "num_prs": 50}, "core": {"precision": 30.4, "recall": 35.4, "f1": 32.7, "f2": 34.3, "tp": 56, "fp": 128, "fn": 102, "num_prs": 50}, "all": {"precision": 31.2, "recall": 33.5, "f1": 32.3, "f2": 33.0, "tp": 58, "fp": 128, "fn": 115, "num_prs": 50}}, "copilot-v2": {"strict": {"precision": 30.4, "recall": 63.3, "f1": 41.1, "f2": 52.1, "tp": 88, "fp": 201, "fn": 51, "num_prs": 50}, "core": {"precision": 33.2, "recall": 63.3, "f1": 43.6, "f2": 53.6, "tp": 100, "fp": 201, "fn": 58, "num_prs": 50}, "all": {"precision": 35.2, "recall": 63.0, "f1": 45.1, "f2": 54.4, "tp": 109, "fp": 201, "fn": 64, "num_prs": 50}}}}};
+        const toolDisplayNames = {"graphite": "Graphite", "qodo": "Qodo", "gemini": "Gemini", "claude": "Claude Code", "augment": "Augment", "bugbot": "Cursor Bugbot", "coderabbit": "CodeRabbit", "propel": "Propel", "copilot": "GitHub Copilot", "baz": "Baz", "greptile": "Greptile", "kg": "KG", "entelligence": "Entelligence", "cubic-v2": "Cubic v2", "sourcery": "Sourcery", "mesa": "Mesa", "codeant-v2": "CodeAnt", "claude-code": "Claude Code (CLI)", "devin": "Devin", "kodus-v2": "Kodus", "greptile-v4-1": "Greptile v4", "gemini-v2": "Gemini", "gitlab": "GitLab Duo Code Review", "qodo-v2": "Qodo v2", "qodo-extended-v2": "Qodo Extended", "macroscope": "Macroscope", "shipwright": "Shipwright"};
+        const toolColors = {"graphite": "#6366f1", "qodo": "#8b5cf6", "gemini": "#06b6d4", "claude": "#f59e0b", "augment": "#10b981", "bugbot": "#3b82f6", "coderabbit": "#ec4899", "propel": "#14b8a6", "propel-v2": "#0d9488", "copilot": "#6b7280", "baz": "#f97316", "greptile": "#22c55e", "kg": "#a855f7", "entelligence": "#0ea5e9", "cubic-dev": "#d946ef", "sourcery": "#84cc16", "mesa": "#f43f5e", "codeant": "#e11d48", "codeant-v2": "#fb7185", "claude-code": "#d97706", "devin": "#7c3aed", "kodus-v2": "#059669", "greptile-v4": "#16a34a", "qodo-v2": "#7c3aed", "qodo-extended-v2": "#6d28d9", "macroscope": "#0891b2", "cubic-v2": "#c026d3", "shipwright": "#2563eb"};
         const predefinedFilters = [{"id": "high_precision", "label": "Highest Precision", "filters": {}, "sort": "precision", "description": "Tools ranked by precision - fewer false positives, more reliable findings", "best_model": "anthropic_claude-opus-4-5-20251101", "best_tool": "graphite", "best_score": 100.0}, {"id": "tool_kg_language_ruby", "label": "Best for Ruby (Precision)", "filters": {"language": ["Ruby"]}, "description": "Ruby codebases with Rails patterns.", "best_model": "anthropic_claude-sonnet-4-5-20250929", "best_tool": "kg", "best_score": 100.0, "sort": "precision"}, {"id": "tool_sourcery_difficulty_moderate_context_file", "label": "Moderate Bugs + File Context (Recall)", "filters": {"difficulty": ["moderate"], "context": ["file"]}, "description": "Requires careful reading. Requires full file understanding.", "best_model": "anthropic_claude-opus-4-5-20251101", "best_tool": "sourcery", "best_score": 90.0, "sort": "recall"}, {"id": "tool_coderabbit_language_typescript_domain_scheduling", "label": "Typescript + Scheduling (Recall)", "filters": {"language": ["TypeScript"], "domain": ["scheduling"]}, "description": "TypeScript codebases with frontend patterns. Task scheduling.", "best_model": "anthropic_claude-opus-4-5-20251101", "best_tool": "coderabbit", "best_score": 88.9, "sort": "recall"}, {"id": "go_small", "label": "Best for Small Go PRs", "filters": {"language": ["Go"], "pr_size": ["small"]}, "description": "Go codebases with concurrency patterns. Small PRs with 1-2 files, easier to review thoroughly", "best_model": "anthropic_claude-opus-4-5-20251101", "best_tool": "cubic-v2", "best_score": 88.2}, {"id": "complexity_complex", "label": "Best for Complex Code", "filters": {"complexity": ["complex"]}, "description": "Deep logic and dependencies.", "best_model": "anthropic_claude-opus-4-5-20251101", "best_tool": "augment", "best_score": 81.1}, {"id": "domain_ui", "label": "Best for Ui", "filters": {"domain": ["UI"]}, "description": "User interface and frontend.", "best_model": "anthropic_claude-opus-4-5-20251101", "best_tool": "bugbot", "best_score": 78.3}, {"id": "risk_critical", "label": "Best for Critical Risk", "filters": {"risk": ["critical"]}, "description": "Critical security or data corruption risk.", "best_model": "anthropic_claude-opus-4-5-20251101", "best_tool": "qodo-v2", "best_score": 75.9}, {"id": "tool_codeant-v2_pr_size_small_change_type_performance", "label": "Small PRs + Performance Optimization (Precision)", "filters": {"pr_size": ["small"], "change_type": ["performance"]}, "description": "Small PRs with 1-2 files, easier to review thoroughly Performance optimization changes", "best_model": "anthropic_claude-sonnet-4-5-20250929", "best_tool": "codeant-v2", "best_score": 75.0, "sort": "precision"}, {"id": "tool_gitlab_change_type_bug_fix_context_cross_file", "label": "Bug Fixes + Cross-File", "filters": {"change_type": ["bug_fix"], "context": ["cross_file"]}, "description": "Bug fixes and issue resolution Spans multiple files.", "best_model": "openai_gpt-5.2", "best_tool": "gitlab", "best_score": 74.3}, {"id": "tool_baz_language_java_domain_authentication", "label": "Java + Authentication", "filters": {"language": ["Java"], "domain": ["authentication"]}, "description": "Java codebases with OOP patterns. Auth and access control.", "best_model": "anthropic_claude-opus-4-5-20251101", "best_tool": "baz", "best_score": 74.1}, {"id": "python_medium", "label": "Best for Medium Python PRs", "filters": {"language": ["Python"], "pr_size": ["medium"]}, "description": "Python codebases with dynamic typing. Medium PRs with 3-5 files, typical feature development", "best_model": "anthropic_claude-opus-4-5-20251101", "best_tool": "qodo-extended-v2", "best_score": 72.7}, {"id": "domain_concurrency", "label": "Best for Concurrency", "filters": {"domain": ["concurrency"]}, "description": "Threading and async operations.", "best_model": "anthropic_claude-opus-4-5-20251101", "best_tool": "macroscope", "best_score": 69.6}, {"id": "tool_claude_change_type_bug_fix", "label": "Best for Bug Fixes", "filters": {"change_type": ["bug_fix"]}, "description": "Bug fixes and issue resolution", "best_model": "anthropic_claude-sonnet-4-5-20250929", "best_tool": "claude", "best_score": 68.3}, {"id": "tool_greptile-v4-1_language_ruby_pr_size_medium", "label": "Ruby + Medium PRs", "filters": {"language": ["Ruby"], "pr_size": ["medium"]}, "description": "Ruby codebases with Rails patterns. Medium PRs with 3-5 files, typical feature development", "best_model": "openai_gpt-5.2", "best_tool": "greptile-v4-1", "best_score": 66.7}, {"id": "high_recall", "label": "Highest Recall", "filters": {}, "sort": "recall", "description": "Tools ranked by recall - catches more issues, may have more noise", "best_model": "anthropic_claude-opus-4-5-20251101", "best_tool": "copilot-v2", "best_score": 66.5}, {"id": "tool_kodus-v2_language_ruby_change_type_feature", "label": "Ruby + Features", "filters": {"language": ["Ruby"], "change_type": ["feature"]}, "description": "Ruby codebases with Rails patterns. New feature implementation", "best_model": "anthropic_claude-opus-4-5-20251101", "best_tool": "kodus-v2", "best_score": 61.2}, {"id": "difficulty_moderate", "label": "Best for Moderate Bugs", "filters": {"difficulty": ["moderate"]}, "description": "Requires careful reading.", "best_model": "anthropic_claude-sonnet-4-5-20250929", "best_tool": "devin", "best_score": 57.9}, {"id": "tool_gemini-v2_language_ruby_difficulty_subtle", "label": "Ruby + Subtle Bugs", "filters": {"language": ["Ruby"], "difficulty": ["subtle"]}, "description": "Ruby codebases with Rails patterns. Non-obvious, needs domain knowledge.", "best_model": "openai_gpt-5.2", "best_tool": "gemini-v2", "best_score": 56.4}, {"id": "language_go", "label": "Best for Go", "filters": {"language": ["Go"]}, "description": "Go codebases with concurrency patterns.", "best_model": "anthropic_claude-opus-4-5-20251101", "best_tool": "cubic-v2", "best_score": 84.2}, {"id": "change_type_performance", "label": "Best for Performance Optimization", "filters": {"change_type": ["performance"]}, "description": "Performance optimization changes", "best_model": "anthropic_claude-opus-4-5-20251101", "best_tool": "cubic-v2", "best_score": 82.9}, {"id": "complex_subtle", "label": "Complex & Subtle", "filters": {"complexity": ["complex"], "difficulty": ["subtle", "very_subtle"]}, "description": "Deep logic and dependencies. Non-obvious, needs domain knowledge.", "best_model": "anthropic_claude-opus-4-5-20251101", "best_tool": "augment", "best_score": 81.1}, {"id": "pr_size_small", "label": "Best for Small PRs", "filters": {"pr_size": ["small"]}, "description": "Small PRs with 1-2 files, easier to review thoroughly", "best_model": "anthropic_claude-opus-4-5-20251101", "best_tool": "cubic-v2", "best_score": 79.6}, {"id": "domain_caching", "label": "Best for Caching", "filters": {"domain": ["caching"]}, "description": "Cache and memoization.", "best_model": "anthropic_claude-opus-4-5-20251101", "best_tool": "augment", "best_score": 78.6}, {"id": "pr_size_large", "label": "Best for Large PRs", "filters": {"pr_size": ["large"]}, "description": "Large PRs with 6+ files, complex changes requiring careful review", "best_model": "anthropic_claude-opus-4-5-20251101", "best_tool": "augment", "best_score": 74.5}, {"id": "change_type_bug_fix", "label": "Best for Bug Fixes", "filters": {"change_type": ["bug_fix"]}, "description": "Bug fixes and issue resolution", "best_model": "anthropic_claude-opus-4-5-20251101", "best_tool": "augment", "best_score": 73.2}, {"id": "language_python", "label": "Best for Python", "filters": {"language": ["Python"]}, "description": "Python codebases with dynamic typing.", "best_model": "openai_gpt-5.2", "best_tool": "cubic-v2", "best_score": 72.5}, {"id": "high_risk_auth", "label": "High Risk Auth", "filters": {"risk": ["high", "critical"], "domain": ["authentication"]}, "description": "Significant impact, potential data loss. Auth and access control.", "best_model": "anthropic_claude-opus-4-5-20251101", "best_tool": "qodo-extended-v2", "best_score": 71.9}, {"id": "context_file", "label": "Best for File Context", "filters": {"context": ["file"]}, "description": "Requires full file understanding.", "best_model": "anthropic_claude-opus-4-5-20251101", "best_tool": "cubic-v2", "best_score": 71.6}, {"id": "risk_high", "label": "Best for High Risk", "filters": {"risk": ["high"]}, "description": "Significant impact, potential data loss.", "best_model": "anthropic_claude-opus-4-5-20251101", "best_tool": "cubic-v2", "best_score": 71.3}, {"id": "language_java", "label": "Best for Java", "filters": {"language": ["Java"]}, "description": "Java codebases with OOP patterns.", "best_model": "anthropic_claude-opus-4-5-20251101", "best_tool": "qodo-extended-v2", "best_score": 70.8}, {"id": "domain_authentication", "label": "Best for Authentication", "filters": {"domain": ["authentication"]}, "description": "Auth and access control.", "best_model": "anthropic_claude-opus-4-5-20251101", "best_tool": "qodo-extended-v2", "best_score": 69.2}, {"id": "ruby_medium", "label": "Best for Medium Ruby PRs", "filters": {"language": ["Ruby"], "pr_size": ["medium"]}, "description": "Ruby codebases with Rails patterns. Medium PRs with 3-5 files, typical feature development", "best_model": "anthropic_claude-opus-4-5-20251101", "best_tool": "cubic-v2", "best_score": 68.8}, {"id": "java_medium", "label": "Best for Medium Java PRs", "filters": {"language": ["Java"], "pr_size": ["medium"]}, "description": "Java codebases with OOP patterns. Medium PRs with 3-5 files, typical feature development", "best_model": "anthropic_claude-opus-4-5-20251101", "best_tool": "qodo-extended-v2", "best_score": 68.6}, {"id": "domain_scheduling", "label": "Best for Scheduling", "filters": {"domain": ["scheduling"]}, "description": "Task scheduling.", "best_model": "anthropic_claude-opus-4-5-20251101", "best_tool": "qodo-extended-v2", "best_score": 68.3}, {"id": "language_typescript", "label": "Best for Typescript", "filters": {"language": ["TypeScript"]}, "description": "TypeScript codebases with frontend patterns.", "best_model": "anthropic_claude-opus-4-5-20251101", "best_tool": "augment", "best_score": 68.2}, {"id": "context_cross_file", "label": "Best for Cross-File", "filters": {"context": ["cross_file"]}, "description": "Spans multiple files.", "best_model": "anthropic_claude-opus-4-5-20251101", "best_tool": "qodo-extended-v2", "best_score": 67.5}, {"id": "pr_size_medium", "label": "Best for Medium PRs", "filters": {"pr_size": ["medium"]}, "description": "Medium PRs with 3-5 files, typical feature development", "best_model": "anthropic_claude-opus-4-5-20251101", "best_tool": "qodo-extended-v2", "best_score": 67.1}, {"id": "concern_reliability", "label": "Best for Reliability", "filters": {"concern": ["reliability"]}, "description": "Error handling and system stability", "best_model": "anthropic_claude-opus-4-5-20251101", "best_tool": "qodo-v2", "best_score": 66.7}, {"id": "difficulty_subtle", "label": "Best for Subtle Bugs", "filters": {"difficulty": ["subtle"]}, "description": "Non-obvious, needs domain knowledge.", "best_model": "anthropic_claude-opus-4-5-20251101", "best_tool": "qodo-extended-v2", "best_score": 66.4}, {"id": "concern_correctness", "label": "Best for Correctness", "filters": {"concern": ["correctness"]}, "description": "Logical correctness and expected behavior", "best_model": "anthropic_claude-opus-4-5-20251101", "best_tool": "qodo-extended-v2", "best_score": 66.4}, {"id": "high_f1", "label": "Highest Score", "filters": {}, "sort": "f1", "description": "Tools ranked by F-beta score (default F2, recall-weighted)", "best_model": "anthropic_claude-opus-4-5-20251101", "best_tool": "qodo-extended-v2", "best_score": 65.8}, {"id": "concern_security", "label": "Best for Security", "filters": {"concern": ["security"]}, "description": "Security vulnerabilities and attack vectors", "best_model": "anthropic_claude-opus-4-5-20251101", "best_tool": "cubic-v2", "best_score": 65.2}, {"id": "security_critical", "label": "Security Critical", "filters": {"concern": ["security"], "risk": ["high", "critical"]}, "description": "Security vulnerabilities and attack vectors Significant impact, potential data loss.", "best_model": "anthropic_claude-opus-4-5-20251101", "best_tool": "cubic-v2", "best_score": 65.2}, {"id": "language_ruby", "label": "Best for Ruby", "filters": {"language": ["Ruby"]}, "description": "Ruby codebases with Rails patterns.", "best_model": "anthropic_claude-opus-4-5-20251101", "best_tool": "cubic-v2", "best_score": 63.8}, {"id": "complexity_moderate", "label": "Best for Moderate Code", "filters": {"complexity": ["moderate"]}, "description": "Some abstraction.", "best_model": "anthropic_claude-opus-4-5-20251101", "best_tool": "cubic-v2", "best_score": 63.4}, {"id": "change_type_feature", "label": "Best for Features", "filters": {"change_type": ["feature"]}, "description": "New feature implementation", "best_model": "anthropic_claude-opus-4-5-20251101", "best_tool": "qodo-extended-v2", "best_score": 62.9}, {"id": "risk_medium", "label": "Best for Medium Risk", "filters": {"risk": ["medium"]}, "description": "Moderate user impact.", "best_model": "anthropic_claude-opus-4-5-20251101", "best_tool": "cubic-v2", "best_score": 59.5}];
 
         let currentModel = 'anthropic_claude-opus-4-5-20251101';
diff --git a/offline/analysis/benchmark_dashboard.json b/offline/analysis/benchmark_dashboard.json
index 95e8ac7..42a8129 100644
--- a/offline/analysis/benchmark_dashboard.json
+++ b/offline/analysis/benchmark_dashboard.json
@@ -354,6 +354,23 @@
                 "fp": 5,
                 "fn": 3
               }
+            },
+            "shipwright": {
+              "strict": {
+                "tp": 0,
+                "fp": 2,
+                "fn": 1
+              },
+              "core": {
+                "tp": 2,
+                "fp": 2,
+                "fn": 1
+              },
+              "all": {
+                "tp": 3,
+                "fp": 2,
+                "fn": 2
+              }
             }
           }
         },
@@ -709,6 +726,23 @@
                 "fp": 0,
                 "fn": 2
               }
+            },
+            "shipwright": {
+              "strict": {
+                "tp": 3,
+                "fp": 0,
+                "fn": 0
+              },
+              "core": {
+                "tp": 3,
+                "fp": 0,
+                "fn": 1
+              },
+              "all": {
+                "tp": 3,
+                "fp": 0,
+                "fn": 1
+              }
             }
           }
         },
@@ -1064,6 +1098,23 @@
                 "fp": 6,
                 "fn": 1
               }
+            },
+            "shipwright": {
+              "strict": {
+                "tp": 0,
+                "fp": 1,
+                "fn": 2
+              },
+              "core": {
+                "tp": 0,
+                "fp": 1,
+                "fn": 2
+              },
+              "all": {
+                "tp": 0,
+                "fp": 1,
+                "fn": 2
+              }
             }
           }
         },
@@ -1419,6 +1470,23 @@
                 "fp": 4,
                 "fn": 2
               }
+            },
+            "shipwright": {
+              "strict": {
+                "tp": 0,
+                "fp": 2,
+                "fn": 1
+              },
+              "core": {
+                "tp": 0,
+                "fp": 2,
+                "fn": 2
+              },
+              "all": {
+                "tp": 0,
+                "fp": 2,
+                "fn": 2
+              }
             }
           }
         },
@@ -1774,6 +1842,23 @@
                 "fp": 4,
                 "fn": 3
               }
+            },
+            "shipwright": {
+              "strict": {
+                "tp": 0,
+                "fp": 2,
+                "fn": 4
+              },
+              "core": {
+                "tp": 0,
+                "fp": 2,
+                "fn": 4
+              },
+              "all": {
+                "tp": 0,
+                "fp": 2,
+                "fn": 5
+              }
             }
           }
         },
@@ -2129,6 +2214,23 @@
                 "fp": 4,
                 "fn": 1
               }
+            },
+            "shipwright": {
+              "strict": {
+                "tp": 2,
+                "fp": 1,
+                "fn": 0
+              },
+              "core": {
+                "tp": 2,
+                "fp": 1,
+                "fn": 0
+              },
+              "all": {
+                "tp": 2,
+                "fp": 1,
+                "fn": 0
+              }
             }
           }
         },
@@ -2484,6 +2586,23 @@
                 "fp": 1,
                 "fn": 2
               }
+            },
+            "shipwright": {
+              "strict": {
+                "tp": 1,
+                "fp": 1,
+                "fn": 1
+              },
+              "core": {
+                "tp": 1,
+                "fp": 1,
+                "fn": 2
+              },
+              "all": {
+                "tp": 1,
+                "fp": 1,
+                "fn": 3
+              }
             }
           }
         },
@@ -2839,6 +2958,23 @@
                 "fp": 2,
                 "fn": 1
               }
+            },
+            "shipwright": {
+              "strict": {
+                "tp": 1,
+                "fp": 1,
+                "fn": 1
+              },
+              "core": {
+                "tp": 1,
+                "fp": 1,
+                "fn": 1
+              },
+              "all": {
+                "tp": 1,
+                "fp": 1,
+                "fn": 1
+              }
             }
           }
         },
@@ -3194,6 +3330,23 @@
                 "fp": 3,
                 "fn": 1
               }
+            },
+            "shipwright": {
+              "strict": {
+                "tp": 0,
+                "fp": 2,
+                "fn": 1
+              },
+              "core": {
+                "tp": 0,
+                "fp": 2,
+                "fn": 1
+              },
+              "all": {
+                "tp": 0,
+                "fp": 2,
+                "fn": 2
+              }
             }
           }
         },
@@ -3549,6 +3702,23 @@
                 "fp": 2,
                 "fn": 1
               }
+            },
+            "shipwright": {
+              "strict": {
+                "tp": 2,
+                "fp": 1,
+                "fn": 1
+              },
+              "core": {
+                "tp": 4,
+                "fp": 1,
+                "fn": 1
+              },
+              "all": {
+                "tp": 4,
+                "fp": 1,
+                "fn": 1
+              }
             }
           }
         },
@@ -3904,6 +4074,23 @@
                 "fp": 2,
                 "fn": 4
               }
+            },
+            "shipwright": {
+              "strict": {
+                "tp": 3,
+                "fp": 0,
+                "fn": 3
+              },
+              "core": {
+                "tp": 3,
+                "fp": 0,
+                "fn": 3
+              },
+              "all": {
+                "tp": 3,
+                "fp": 0,
+                "fn": 3
+              }
             }
           }
         },
@@ -4259,6 +4446,23 @@
                 "fp": 3,
                 "fn": 2
               }
+            },
+            "shipwright": {
+              "strict": {
+                "tp": 2,
+                "fp": 3,
+                "fn": 2
+              },
+              "core": {
+                "tp": 2,
+                "fp": 3,
+                "fn": 2
+              },
+              "all": {
+                "tp": 2,
+                "fp": 3,
+                "fn": 2
+              }
             }
           }
         },
@@ -4614,6 +4818,23 @@
                 "fp": 1,
                 "fn": 0
               }
+            },
+            "shipwright": {
+              "strict": {
+                "tp": 1,
+                "fp": 0,
+                "fn": 1
+              },
+              "core": {
+                "tp": 1,
+                "fp": 0,
+                "fn": 1
+              },
+              "all": {
+                "tp": 1,
+                "fp": 0,
+                "fn": 1
+              }
             }
           }
         },
@@ -4969,6 +5190,23 @@
                 "fp": 2,
                 "fn": 1
               }
+            },
+            "shipwright": {
+              "strict": {
+                "tp": 1,
+                "fp": 0,
+                "fn": 0
+              },
+              "core": {
+                "tp": 2,
+                "fp": 0,
+                "fn": 1
+              },
+              "all": {
+                "tp": 2,
+                "fp": 0,
+                "fn": 1
+              }
             }
           }
         },
@@ -5324,6 +5562,23 @@
                 "fp": 2,
                 "fn": 1
               }
+            },
+            "shipwright": {
+              "strict": {
+                "tp": 0,
+                "fp": 0,
+                "fn": 1
+              },
+              "core": {
+                "tp": 0,
+                "fp": 0,
+                "fn": 2
+              },
+              "all": {
+                "tp": 0,
+                "fp": 0,
+                "fn": 2
+              }
             }
           }
         },
@@ -5679,6 +5934,23 @@
                 "fp": 3,
                 "fn": 2
               }
+            },
+            "shipwright": {
+              "strict": {
+                "tp": 2,
+                "fp": 0,
+                "fn": 0
+              },
+              "core": {
+                "tp": 2,
+                "fp": 0,
+                "fn": 0
+              },
+              "all": {
+                "tp": 3,
+                "fp": 0,
+                "fn": 1
+              }
             }
           }
         },
@@ -6034,6 +6306,23 @@
                 "fp": 4,
                 "fn": 1
               }
+            },
+            "shipwright": {
+              "strict": {
+                "tp": 0,
+                "fp": 2,
+                "fn": 0
+              },
+              "core": {
+                "tp": 0,
+                "fp": 2,
+                "fn": 1
+              },
+              "all": {
+                "tp": 0,
+                "fp": 2,
+                "fn": 2
+              }
             }
           }
         },
@@ -6389,6 +6678,23 @@
                 "fp": 7,
                 "fn": 1
               }
+            },
+            "shipwright": {
+              "strict": {
+                "tp": 3,
+                "fp": 4,
+                "fn": 1
+              },
+              "core": {
+                "tp": 3,
+                "fp": 4,
+                "fn": 1
+              },
+              "all": {
+                "tp": 3,
+                "fp": 4,
+                "fn": 1
+              }
             }
           }
         },
@@ -6744,6 +7050,23 @@
                 "fp": 6,
                 "fn": 1
               }
+            },
+            "shipwright": {
+              "strict": {
+                "tp": 1,
+                "fp": 3,
+                "fn": 2
+              },
+              "core": {
+                "tp": 1,
+                "fp": 3,
+                "fn": 2
+              },
+              "all": {
+                "tp": 1,
+                "fp": 3,
+                "fn": 2
+              }
             }
           }
         },
@@ -7099,6 +7422,23 @@
                 "fp": 2,
                 "fn": 3
               }
+            },
+            "shipwright": {
+              "strict": {
+                "tp": 0,
+                "fp": 1,
+                "fn": 2
+              },
+              "core": {
+                "tp": 0,
+                "fp": 1,
+                "fn": 3
+              },
+              "all": {
+                "tp": 0,
+                "fp": 1,
+                "fn": 3
+              }
             }
           }
         },
@@ -7454,6 +7794,23 @@
                 "fp": 7,
                 "fn": 1
               }
+            },
+            "shipwright": {
+              "strict": {
+                "tp": 1,
+                "fp": 1,
+                "fn": 2
+              },
+              "core": {
+                "tp": 1,
+                "fp": 1,
+                "fn": 2
+              },
+              "all": {
+                "tp": 1,
+                "fp": 1,
+                "fn": 2
+              }
             }
           }
         },
@@ -7809,6 +8166,23 @@
                 "fp": 1,
                 "fn": 1
               }
+            },
+            "shipwright": {
+              "strict": {
+                "tp": 1,
+                "fp": 1,
+                "fn": 1
+              },
+              "core": {
+                "tp": 1,
+                "fp": 1,
+                "fn": 1
+              },
+              "all": {
+                "tp": 1,
+                "fp": 1,
+                "fn": 1
+              }
             }
           }
         },
@@ -8164,6 +8538,23 @@
                 "fp": 1,
                 "fn": 0
               }
+            },
+            "shipwright": {
+              "strict": {
+                "tp": 2,
+                "fp": 0,
+                "fn": 0
+              },
+              "core": {
+                "tp": 2,
+                "fp": 0,
+                "fn": 0
+              },
+              "all": {
+                "tp": 2,
+                "fp": 0,
+                "fn": 0
+              }
             }
           }
         },
@@ -8519,6 +8910,23 @@
                 "fp": 1,
                 "fn": 1
               }
+            },
+            "shipwright": {
+              "strict": {
+                "tp": 0,
+                "fp": 0,
+                "fn": 2
+              },
+              "core": {
+                "tp": 0,
+                "fp": 0,
+                "fn": 2
+              },
+              "all": {
+                "tp": 0,
+                "fp": 0,
+                "fn": 2
+              }
             }
           }
         },
@@ -8874,6 +9282,23 @@
                 "fp": 1,
                 "fn": 0
               }
+            },
+            "shipwright": {
+              "strict": {
+                "tp": 0,
+                "fp": 0,
+                "fn": 0
+              },
+              "core": {
+                "tp": 2,
+                "fp": 0,
+                "fn": 0
+              },
+              "all": {
+                "tp": 2,
+                "fp": 0,
+                "fn": 0
+              }
             }
           }
         },
@@ -9229,6 +9654,23 @@
                 "fp": 1,
                 "fn": 0
               }
+            },
+            "shipwright": {
+              "strict": {
+                "tp": 2,
+                "fp": 6,
+                "fn": 1
+              },
+              "core": {
+                "tp": 2,
+                "fp": 6,
+                "fn": 1
+              },
+              "all": {
+                "tp": 2,
+                "fp": 6,
+                "fn": 1
+              }
             }
           }
         },
@@ -9584,6 +10026,23 @@
                 "fp": 2,
                 "fn": 2
               }
+            },
+            "shipwright": {
+              "strict": {
+                "tp": 0,
+                "fp": 4,
+                "fn": 2
+              },
+              "core": {
+                "tp": 0,
+                "fp": 4,
+                "fn": 2
+              },
+              "all": {
+                "tp": 0,
+                "fp": 4,
+                "fn": 2
+              }
             }
           }
         },
@@ -9939,6 +10398,23 @@
                 "fp": 3,
                 "fn": 1
               }
+            },
+            "shipwright": {
+              "strict": {
+                "tp": 0,
+                "fp": 2,
+                "fn": 0
+              },
+              "core": {
+                "tp": 0,
+                "fp": 2,
+                "fn": 1
+              },
+              "all": {
+                "tp": 0,
+                "fp": 2,
+                "fn": 1
+              }
             }
           }
         },
@@ -10294,6 +10770,23 @@
                 "fp": 3,
                 "fn": 2
               }
+            },
+            "shipwright": {
+              "strict": {
+                "tp": 1,
+                "fp": 1,
+                "fn": 3
+              },
+              "core": {
+                "tp": 1,
+                "fp": 1,
+                "fn": 4
+              },
+              "all": {
+                "tp": 1,
+                "fp": 1,
+                "fn": 4
+              }
             }
           }
         },
@@ -10649,6 +11142,23 @@
                 "fp": 2,
                 "fn": 2
               }
+            },
+            "shipwright": {
+              "strict": {
+                "tp": 0,
+                "fp": 1,
+                "fn": 1
+              },
+              "core": {
+                "tp": 0,
+                "fp": 1,
+                "fn": 2
+              },
+              "all": {
+                "tp": 0,
+                "fp": 1,
+                "fn": 2
+              }
             }
           }
         },
@@ -11004,6 +11514,23 @@
                 "fp": 3,
                 "fn": 3
               }
+            },
+            "shipwright": {
+              "strict": {
+                "tp": 1,
+                "fp": 1,
+                "fn": 2
+              },
+              "core": {
+                "tp": 1,
+                "fp": 1,
+                "fn": 2
+              },
+              "all": {
+                "tp": 1,
+                "fp": 1,
+                "fn": 2
+              }
             }
           }
         },
@@ -11359,6 +11886,23 @@
                 "fp": 7,
                 "fn": 2
               }
+            },
+            "shipwright": {
+              "strict": {
+                "tp": 5,
+                "fp": 1,
+                "fn": 2
+              },
+              "core": {
+                "tp": 5,
+                "fp": 1,
+                "fn": 2
+              },
+              "all": {
+                "tp": 5,
+                "fp": 1,
+                "fn": 2
+              }
             }
           }
         },
@@ -11714,6 +12258,23 @@
                 "fp": 1,
                 "fn": 0
               }
+            },
+            "shipwright": {
+              "strict": {
+                "tp": 3,
+                "fp": 2,
+                "fn": 0
+              },
+              "core": {
+                "tp": 3,
+                "fp": 2,
+                "fn": 0
+              },
+              "all": {
+                "tp": 5,
+                "fp": 2,
+                "fn": 0
+              }
             }
           }
         },
@@ -12069,6 +12630,23 @@
                 "fp": 1,
                 "fn": 2
               }
+            },
+            "shipwright": {
+              "strict": {
+                "tp": 3,
+                "fp": 1,
+                "fn": 3
+              },
+              "core": {
+                "tp": 3,
+                "fp": 1,
+                "fn": 3
+              },
+              "all": {
+                "tp": 3,
+                "fp": 1,
+                "fn": 3
+              }
             }
           }
         },
@@ -12424,6 +13002,23 @@
                 "fp": 3,
                 "fn": 2
               }
+            },
+            "shipwright": {
+              "strict": {
+                "tp": 1,
+                "fp": 1,
+                "fn": 2
+              },
+              "core": {
+                "tp": 1,
+                "fp": 1,
+                "fn": 2
+              },
+              "all": {
+                "tp": 1,
+                "fp": 1,
+                "fn": 2
+              }
             }
           }
         },
@@ -12779,6 +13374,23 @@
                 "fp": 1,
                 "fn": 1
               }
+            },
+            "shipwright": {
+              "strict": {
+                "tp": 1,
+                "fp": 0,
+                "fn": 2
+              },
+              "core": {
+                "tp": 1,
+                "fp": 0,
+                "fn": 2
+              },
+              "all": {
+                "tp": 1,
+                "fp": 0,
+                "fn": 2
+              }
             }
           }
         },
@@ -13134,6 +13746,23 @@
                 "fp": 4,
                 "fn": 0
               }
+            },
+            "shipwright": {
+              "strict": {
+                "tp": 1,
+                "fp": 2,
+                "fn": 0
+              },
+              "core": {
+                "tp": 1,
+                "fp": 2,
+                "fn": 0
+              },
+              "all": {
+                "tp": 1,
+                "fp": 2,
+                "fn": 0
+              }
             }
           }
         },
@@ -13489,6 +14118,23 @@
                 "fp": 2,
                 "fn": 5
               }
+            },
+            "shipwright": {
+              "strict": {
+                "tp": 1,
+                "fp": 4,
+                "fn": 7
+              },
+              "core": {
+                "tp": 1,
+                "fp": 4,
+                "fn": 7
+              },
+              "all": {
+                "tp": 1,
+                "fp": 4,
+                "fn": 7
+              }
             }
           }
         },
@@ -13844,6 +14490,23 @@
                 "fp": 1,
                 "fn": 2
               }
+            },
+            "shipwright": {
+              "strict": {
+                "tp": 2,
+                "fp": 2,
+                "fn": 2
+              },
+              "core": {
+                "tp": 2,
+                "fp": 2,
+                "fn": 2
+              },
+              "all": {
+                "tp": 2,
+                "fp": 2,
+                "fn": 2
+              }
             }
           }
         },
@@ -14199,6 +14862,23 @@
                 "fp": 5,
                 "fn": 1
               }
+            },
+            "shipwright": {
+              "strict": {
+                "tp": 1,
+                "fp": 2,
+                "fn": 0
+              },
+              "core": {
+                "tp": 1,
+                "fp": 2,
+                "fn": 0
+              },
+              "all": {
+                "tp": 2,
+                "fp": 2,
+                "fn": 0
+              }
             }
           }
         },
@@ -14554,6 +15234,23 @@
                 "fp": 1,
                 "fn": 1
               }
+            },
+            "shipwright": {
+              "strict": {
+                "tp": 1,
+                "fp": 1,
+                "fn": 3
+              },
+              "core": {
+                "tp": 1,
+                "fp": 1,
+                "fn": 3
+              },
+              "all": {
+                "tp": 1,
+                "fp": 1,
+                "fn": 3
+              }
             }
           }
         },
@@ -14909,6 +15606,23 @@
                 "fp": 1,
                 "fn": 0
               }
+            },
+            "shipwright": {
+              "strict": {
+                "tp": 2,
+                "fp": 0,
+                "fn": 0
+              },
+              "core": {
+                "tp": 2,
+                "fp": 0,
+                "fn": 0
+              },
+              "all": {
+                "tp": 2,
+                "fp": 0,
+                "fn": 0
+              }
             }
           }
         },
@@ -15264,6 +15978,23 @@
                 "fp": 1,
                 "fn": 0
               }
+            },
+            "shipwright": {
+              "strict": {
+                "tp": 2,
+                "fp": 0,
+                "fn": 0
+              },
+              "core": {
+                "tp": 2,
+                "fp": 0,
+                "fn": 0
+              },
+              "all": {
+                "tp": 2,
+                "fp": 0,
+                "fn": 0
+              }
             }
           }
         },
@@ -15619,6 +16350,23 @@
                 "fp": 0,
                 "fn": 1
               }
+            },
+            "shipwright": {
+              "strict": {
+                "tp": 0,
+                "fp": 0,
+                "fn": 1
+              },
+              "core": {
+                "tp": 0,
+                "fp": 0,
+                "fn": 1
+              },
+              "all": {
+                "tp": 0,
+                "fp": 0,
+                "fn": 2
+              }
             }
           }
         },
@@ -15974,6 +16722,23 @@
                 "fp": 0,
                 "fn": 4
               }
+            },
+            "shipwright": {
+              "strict": {
+                "tp": 5,
+                "fp": 1,
+                "fn": 4
+              },
+              "core": {
+                "tp": 5,
+                "fp": 1,
+                "fn": 4
+              },
+              "all": {
+                "tp": 5,
+                "fp": 1,
+                "fn": 4
+              }
             }
           }
         },
@@ -16329,6 +17094,23 @@
                 "fp": 2,
                 "fn": 2
               }
+            },
+            "shipwright": {
+              "strict": {
+                "tp": 2,
+                "fp": 2,
+                "fn": 1
+              },
+              "core": {
+                "tp": 2,
+                "fp": 2,
+                "fn": 1
+              },
+              "all": {
+                "tp": 2,
+                "fp": 2,
+                "fn": 1
+              }
             }
           }
         },
@@ -16684,6 +17466,23 @@
                 "fp": 1,
                 "fn": 3
               }
+            },
+            "shipwright": {
+              "strict": {
+                "tp": 2,
+                "fp": 3,
+                "fn": 4
+              },
+              "core": {
+                "tp": 2,
+                "fp": 3,
+                "fn": 4
+              },
+              "all": {
+                "tp": 2,
+                "fp": 3,
+                "fn": 4
+              }
             }
           }
         },
@@ -17039,6 +17838,23 @@
                 "fp": 1,
                 "fn": 2
               }
+            },
+            "shipwright": {
+              "strict": {
+                "tp": 1,
+                "fp": 0,
+                "fn": 1
+              },
+              "core": {
+                "tp": 2,
+                "fp": 0,
+                "fn": 2
+              },
+              "all": {
+                "tp": 3,
+                "fp": 0,
+                "fn": 2
+              }
             }
           }
         },
@@ -17394,6 +18210,23 @@
                 "fp": 2,
                 "fn": 3
               }
+            },
+            "shipwright": {
+              "strict": {
+                "tp": 3,
+                "fp": 1,
+                "fn": 2
+              },
+              "core": {
+                "tp": 3,
+                "fp": 1,
+                "fn": 2
+              },
+              "all": {
+                "tp": 3,
+                "fp": 1,
+                "fn": 3
+              }
             }
           }
         },
@@ -17749,6 +18582,23 @@
                 "fp": 3,
                 "fn": 1
               }
+            },
+            "shipwright": {
+              "strict": {
+                "tp": 1,
+                "fp": 2,
+                "fn": 0
+              },
+              "core": {
+                "tp": 1,
+                "fp": 2,
+                "fn": 0
+              },
+              "all": {
+                "tp": 1,
+                "fp": 2,
+                "fn": 1
+              }
             }
           }
         }
@@ -17773,6 +18623,7 @@
         "macroscope",
         "qodo-extended-v2",
         "qodo-v2",
+        "shipwright",
         "sourcery"
       ],
       "dimensions": {
@@ -17842,6 +18693,102 @@
         ]
       },
       "overall_metrics": {
+        "sourcery": {
+          "strict": {
+            "precision": 38.4,
+            "recall": 56.1,
+            "f1": 45.6,
+            "f2": 51.4,
+            "tp": 78,
+            "fp": 125,
+            "fn": 61,
+            "num_prs": 50
+          },
+          "core": {
+            "precision": 41.0,
+            "recall": 55.1,
+            "f1": 47.0,
+            "f2": 51.5,
+            "tp": 87,
+            "fp": 125,
+            "fn": 71,
+            "num_prs": 50
+          },
+          "all": {
+            "precision": 43.2,
+            "recall": 54.9,
+            "f1": 48.3,
+            "f2": 52.1,
+            "tp": 95,
+            "fp": 125,
+            "fn": 78,
+            "num_prs": 50
+          }
+        },
+        "macroscope": {
+          "strict": {
+            "precision": 60.2,
+            "recall": 53.2,
+            "f1": 56.5,
+            "f2": 54.5,
+            "tp": 74,
+            "fp": 49,
+            "fn": 65,
+            "num_prs": 50
+          },
+          "core": {
+            "precision": 61.1,
+            "recall": 48.7,
+            "f1": 54.2,
+            "f2": 50.8,
+            "tp": 77,
+            "fp": 49,
+            "fn": 81,
+            "num_prs": 50
+          },
+          "all": {
+            "precision": 62.3,
+            "recall": 46.8,
+            "f1": 53.5,
+            "f2": 49.3,
+            "tp": 81,
+            "fp": 49,
+            "fn": 92,
+            "num_prs": 50
+          }
+        },
+        "graphite": {
+          "strict": {
+            "precision": 100.0,
+            "recall": 7.9,
+            "f1": 14.7,
+            "f2": 9.7,
+            "tp": 11,
+            "fp": 0,
+            "fn": 128,
+            "num_prs": 50
+          },
+          "core": {
+            "precision": 100.0,
+            "recall": 7.6,
+            "f1": 14.1,
+            "f2": 9.3,
+            "tp": 12,
+            "fp": 0,
+            "fn": 146,
+            "num_prs": 50
+          },
+          "all": {
+            "precision": 100.0,
+            "recall": 6.9,
+            "f1": 13.0,
+            "f2": 8.5,
+            "tp": 12,
+            "fp": 0,
+            "fn": 161,
+            "num_prs": 50
+          }
+        },
         "augment": {
           "strict": {
             "precision": 56.5,
@@ -17874,6 +18821,102 @@
             "num_prs": 50
           }
         },
+        "qodo-extended-v2": {
+          "strict": {
+            "precision": 64.8,
+            "recall": 66.2,
+            "f1": 65.5,
+            "f2": 65.9,
+            "tp": 92,
+            "fp": 50,
+            "fn": 47,
+            "num_prs": 50
+          },
+          "core": {
+            "precision": 67.1,
+            "recall": 64.6,
+            "f1": 65.8,
+            "f2": 65.1,
+            "tp": 102,
+            "fp": 50,
+            "fn": 56,
+            "num_prs": 50
+          },
+          "all": {
+            "precision": 67.9,
+            "recall": 61.3,
+            "f1": 64.4,
+            "f2": 62.5,
+            "tp": 106,
+            "fp": 50,
+            "fn": 67,
+            "num_prs": 50
+          }
+        },
+        "kodus-v2": {
+          "strict": {
+            "precision": 57.1,
+            "recall": 40.3,
+            "f1": 47.3,
+            "f2": 42.8,
+            "tp": 56,
+            "fp": 42,
+            "fn": 83,
+            "num_prs": 50
+          },
+          "core": {
+            "precision": 59.2,
+            "recall": 38.6,
+            "f1": 46.7,
+            "f2": 41.5,
+            "tp": 61,
+            "fp": 42,
+            "fn": 97,
+            "num_prs": 50
+          },
+          "all": {
+            "precision": 59.6,
+            "recall": 35.8,
+            "f1": 44.8,
+            "f2": 38.9,
+            "tp": 62,
+            "fp": 42,
+            "fn": 111,
+            "num_prs": 50
+          }
+        },
+        "kg": {
+          "strict": {
+            "precision": 53.3,
+            "recall": 17.3,
+            "f1": 26.1,
+            "f2": 20.0,
+            "tp": 24,
+            "fp": 21,
+            "fn": 115,
+            "num_prs": 50
+          },
+          "core": {
+            "precision": 54.3,
+            "recall": 15.8,
+            "f1": 24.5,
+            "f2": 18.4,
+            "tp": 25,
+            "fp": 21,
+            "fn": 133,
+            "num_prs": 50
+          },
+          "all": {
+            "precision": 55.3,
+            "recall": 15.0,
+            "f1": 23.6,
+            "f2": 17.6,
+            "tp": 26,
+            "fp": 21,
+            "fn": 147,
+            "num_prs": 50
+          }
+        },
         "cubic-v2": {
           "strict": {
             "precision": 58.3,
@@ -17906,35 +18949,35 @@
             "num_prs": 50
           }
         },
-        "copilot-v2": {
+        "gemini-v2": {
           "strict": {
-            "precision": 35.6,
-            "recall": 66.9,
-            "f1": 46.5,
-            "f2": 56.9,
-            "tp": 93,
-            "fp": 168,
-            "fn": 46,
+            "precision": 43.2,
+            "recall": 57.6,
+            "f1": 49.4,
+            "f2": 54.0,
+            "tp": 80,
+            "fp": 105,
+            "fn": 59,
             "num_prs": 50
           },
           "core": {
-            "precision": 38.5,
-            "recall": 66.5,
-            "f1": 48.7,
-            "f2": 58.0,
-            "tp": 105,
-            "fp": 168,
-            "fn": 53,
+            "precision": 45.3,
+            "recall": 55.1,
+            "f1": 49.7,
+            "f2": 52.8,
+            "tp": 87,
+            "fp": 105,
+            "fn": 71,
             "num_prs": 50
           },
           "all": {
-            "precision": 40.6,
-            "recall": 66.5,
-            "f1": 50.4,
-            "f2": 59.0,
-            "tp": 115,
-            "fp": 168,
-            "fn": 58,
+            "precision": 47.5,
+            "recall": 54.9,
+            "f1": 50.9,
+            "f2": 53.3,
+            "tp": 95,
+            "fp": 105,
+            "fn": 78,
             "num_prs": 50
           }
         },
@@ -17970,163 +19013,131 @@
             "num_prs": 50
           }
         },
-        "sourcery": {
+        "bugbot": {
           "strict": {
-            "precision": 38.4,
-            "recall": 56.1,
-            "f1": 45.6,
-            "f2": 51.4,
-            "tp": 78,
-            "fp": 125,
-            "fn": 61,
+            "precision": 54.8,
+            "recall": 48.9,
+            "f1": 51.7,
+            "f2": 50.0,
+            "tp": 68,
+            "fp": 56,
+            "fn": 71,
             "num_prs": 50
           },
           "core": {
-            "precision": 41.0,
-            "recall": 55.1,
-            "f1": 47.0,
-            "f2": 51.5,
-            "tp": 87,
-            "fp": 125,
-            "fn": 71,
+            "precision": 56.9,
+            "recall": 46.8,
+            "f1": 51.4,
+            "f2": 48.6,
+            "tp": 74,
+            "fp": 56,
+            "fn": 84,
             "num_prs": 50
           },
           "all": {
-            "precision": 43.2,
-            "recall": 54.9,
-            "f1": 48.3,
-            "f2": 52.1,
-            "tp": 95,
-            "fp": 125,
-            "fn": 78,
+            "precision": 58.2,
+            "recall": 45.1,
+            "f1": 50.8,
+            "f2": 47.2,
+            "tp": 78,
+            "fp": 56,
+            "fn": 95,
             "num_prs": 50
           }
         },
-        "qodo-v2": {
+        "claude": {
           "strict": {
-            "precision": 53.0,
-            "recall": 64.0,
-            "f1": 58.0,
-            "f2": 61.5,
-            "tp": 89,
+            "precision": 41.0,
+            "recall": 39.6,
+            "f1": 40.3,
+            "f2": 39.9,
+            "tp": 55,
             "fp": 79,
-            "fn": 50,
+            "fn": 84,
             "num_prs": 50
           },
           "core": {
-            "precision": 55.4,
-            "recall": 62.0,
-            "f1": 58.5,
-            "f2": 60.6,
-            "tp": 98,
+            "precision": 43.2,
+            "recall": 38.0,
+            "f1": 40.4,
+            "f2": 38.9,
+            "tp": 60,
             "fp": 79,
-            "fn": 60,
+            "fn": 98,
             "num_prs": 50
           },
           "all": {
-            "precision": 56.4,
-            "recall": 59.0,
-            "f1": 57.6,
-            "f2": 58.4,
-            "tp": 102,
+            "precision": 45.5,
+            "recall": 38.2,
+            "f1": 41.5,
+            "f2": 39.4,
+            "tp": 66,
             "fp": 79,
-            "fn": 71,
-            "num_prs": 50
-          }
-        },
-        "gitlab": {
-          "strict": {
-            "precision": 51.0,
-            "recall": 54.0,
-            "f1": 52.4,
-            "f2": 53.3,
-            "tp": 75,
-            "fp": 72,
-            "fn": 64,
-            "num_prs": 50
-          },
-          "core": {
-            "precision": 53.5,
-            "recall": 52.5,
-            "f1": 53.0,
-            "f2": 52.7,
-            "tp": 83,
-            "fp": 72,
-            "fn": 75,
-            "num_prs": 50
-          },
-          "all": {
-            "precision": 55.8,
-            "recall": 52.6,
-            "f1": 54.2,
-            "f2": 53.2,
-            "tp": 91,
-            "fp": 72,
-            "fn": 82,
+            "fn": 107,
             "num_prs": 50
           }
         },
-        "codeant-v2": {
+        "devin": {
           "strict": {
-            "precision": 39.0,
-            "recall": 44.6,
-            "f1": 41.6,
-            "f2": 43.4,
-            "tp": 62,
-            "fp": 97,
-            "fn": 77,
+            "precision": 67.4,
+            "recall": 43.2,
+            "f1": 52.6,
+            "f2": 46.5,
+            "tp": 60,
+            "fp": 29,
+            "fn": 79,
             "num_prs": 50
           },
           "core": {
-            "precision": 40.1,
+            "precision": 69.1,
             "recall": 41.1,
-            "f1": 40.6,
-            "f2": 40.9,
+            "f1": 51.6,
+            "f2": 44.8,
             "tp": 65,
-            "fp": 97,
+            "fp": 29,
             "fn": 93,
             "num_prs": 50
           },
           "all": {
-            "precision": 41.2,
+            "precision": 70.1,
             "recall": 39.3,
-            "f1": 40.2,
-            "f2": 39.7,
+            "f1": 50.4,
+            "f2": 43.1,
             "tp": 68,
-            "fp": 97,
+            "fp": 29,
             "fn": 105,
             "num_prs": 50
           }
         },
-        "bugbot": {
+        "baz": {
           "strict": {
-            "precision": 54.8,
-            "recall": 48.9,
-            "f1": 51.7,
-            "f2": 50.0,
-            "tp": 68,
-            "fp": 56,
-            "fn": 71,
+            "precision": 60.6,
+            "recall": 41.0,
+            "f1": 48.9,
+            "f2": 43.8,
+            "tp": 57,
+            "fp": 37,
+            "fn": 82,
             "num_prs": 50
           },
           "core": {
-            "precision": 56.9,
-            "recall": 46.8,
-            "f1": 51.4,
-            "f2": 48.6,
-            "tp": 74,
-            "fp": 56,
-            "fn": 84,
+            "precision": 61.5,
+            "recall": 37.3,
+            "f1": 46.5,
+            "f2": 40.5,
+            "tp": 59,
+            "fp": 37,
+            "fn": 99,
             "num_prs": 50
           },
           "all": {
-            "precision": 58.2,
-            "recall": 45.1,
-            "f1": 50.8,
-            "f2": 47.2,
-            "tp": 78,
-            "fp": 56,
-            "fn": 95,
+            "precision": 62.6,
+            "recall": 35.8,
+            "f1": 45.6,
+            "f2": 39.2,
+            "tp": 62,
+            "fp": 37,
+            "fn": 111,
             "num_prs": 50
           }
         },
@@ -18162,132 +19173,68 @@
             "num_prs": 50
           }
         },
-        "devin": {
+        "gitlab": {
           "strict": {
-            "precision": 67.4,
-            "recall": 43.2,
-            "f1": 52.6,
-            "f2": 46.5,
-            "tp": 60,
-            "fp": 29,
-            "fn": 79,
+            "precision": 51.0,
+            "recall": 54.0,
+            "f1": 52.4,
+            "f2": 53.3,
+            "tp": 75,
+            "fp": 72,
+            "fn": 64,
             "num_prs": 50
           },
           "core": {
-            "precision": 69.1,
-            "recall": 41.1,
-            "f1": 51.6,
-            "f2": 44.8,
-            "tp": 65,
-            "fp": 29,
-            "fn": 93,
+            "precision": 53.5,
+            "recall": 52.5,
+            "f1": 53.0,
+            "f2": 52.7,
+            "tp": 83,
+            "fp": 72,
+            "fn": 75,
             "num_prs": 50
           },
           "all": {
-            "precision": 70.1,
-            "recall": 39.3,
-            "f1": 50.4,
-            "f2": 43.1,
-            "tp": 68,
-            "fp": 29,
-            "fn": 105,
-            "num_prs": 50
-          }
-        },
-        "macroscope": {
-          "strict": {
-            "precision": 60.2,
-            "recall": 53.2,
-            "f1": 56.5,
-            "f2": 54.5,
-            "tp": 74,
-            "fp": 49,
-            "fn": 65,
-            "num_prs": 50
-          },
-          "core": {
-            "precision": 61.1,
-            "recall": 48.7,
+            "precision": 55.8,
+            "recall": 52.6,
             "f1": 54.2,
-            "f2": 50.8,
-            "tp": 77,
-            "fp": 49,
-            "fn": 81,
-            "num_prs": 50
-          },
-          "all": {
-            "precision": 62.3,
-            "recall": 46.8,
-            "f1": 53.5,
-            "f2": 49.3,
-            "tp": 81,
-            "fp": 49,
-            "fn": 92,
+            "f2": 53.2,
+            "tp": 91,
+            "fp": 72,
+            "fn": 82,
             "num_prs": 50
           }
         },
-        "claude": {
+        "qodo-v2": {
           "strict": {
-            "precision": 41.0,
-            "recall": 39.6,
-            "f1": 40.3,
-            "f2": 39.9,
-            "tp": 55,
+            "precision": 53.0,
+            "recall": 64.0,
+            "f1": 58.0,
+            "f2": 61.5,
+            "tp": 89,
             "fp": 79,
-            "fn": 84,
+            "fn": 50,
             "num_prs": 50
           },
           "core": {
-            "precision": 43.2,
-            "recall": 38.0,
-            "f1": 40.4,
-            "f2": 38.9,
-            "tp": 60,
+            "precision": 55.4,
+            "recall": 62.0,
+            "f1": 58.5,
+            "f2": 60.6,
+            "tp": 98,
             "fp": 79,
-            "fn": 98,
+            "fn": 60,
             "num_prs": 50
           },
           "all": {
-            "precision": 45.5,
-            "recall": 38.2,
-            "f1": 41.5,
-            "f2": 39.4,
-            "tp": 66,
+            "precision": 56.4,
+            "recall": 59.0,
+            "f1": 57.6,
+            "f2": 58.4,
+            "tp": 102,
             "fp": 79,
-            "fn": 107,
-            "num_prs": 50
-          }
-        },
-        "gemini-v2": {
-          "strict": {
-            "precision": 43.2,
-            "recall": 57.6,
-            "f1": 49.4,
-            "f2": 54.0,
-            "tp": 80,
-            "fp": 105,
-            "fn": 59,
-            "num_prs": 50
-          },
-          "core": {
-            "precision": 45.3,
-            "recall": 55.1,
-            "f1": 49.7,
-            "f2": 52.8,
-            "tp": 87,
-            "fp": 105,
             "fn": 71,
             "num_prs": 50
-          },
-          "all": {
-            "precision": 47.5,
-            "recall": 54.9,
-            "f1": 50.9,
-            "f2": 53.3,
-            "tp": 95,
-            "fp": 105,
-            "fn": 78,
-            "num_prs": 50
           }
         },
         "coderabbit": {
@@ -18322,163 +19269,99 @@
             "num_prs": 50
           }
         },
-        "baz": {
+        "codeant-v2": {
           "strict": {
-            "precision": 60.6,
-            "recall": 41.0,
-            "f1": 48.9,
-            "f2": 43.8,
-            "tp": 57,
-            "fp": 37,
-            "fn": 82,
+            "precision": 39.0,
+            "recall": 44.6,
+            "f1": 41.6,
+            "f2": 43.4,
+            "tp": 62,
+            "fp": 97,
+            "fn": 77,
             "num_prs": 50
           },
           "core": {
-            "precision": 61.5,
-            "recall": 37.3,
-            "f1": 46.5,
-            "f2": 40.5,
-            "tp": 59,
-            "fp": 37,
-            "fn": 99,
+            "precision": 40.1,
+            "recall": 41.1,
+            "f1": 40.6,
+            "f2": 40.9,
+            "tp": 65,
+            "fp": 97,
+            "fn": 93,
             "num_prs": 50
           },
           "all": {
-            "precision": 62.6,
-            "recall": 35.8,
-            "f1": 45.6,
-            "f2": 39.2,
-            "tp": 62,
-            "fp": 37,
-            "fn": 111,
-            "num_prs": 50
-          }
-        },
-        "kg": {
-          "strict": {
-            "precision": 53.3,
-            "recall": 17.3,
-            "f1": 26.1,
-            "f2": 20.0,
-            "tp": 24,
-            "fp": 21,
-            "fn": 115,
-            "num_prs": 50
-          },
-          "core": {
-            "precision": 54.3,
-            "recall": 15.8,
-            "f1": 24.5,
-            "f2": 18.4,
-            "tp": 25,
-            "fp": 21,
-            "fn": 133,
-            "num_prs": 50
-          },
-          "all": {
-            "precision": 55.3,
-            "recall": 15.0,
-            "f1": 23.6,
-            "f2": 17.6,
-            "tp": 26,
-            "fp": 21,
-            "fn": 147,
+            "precision": 41.2,
+            "recall": 39.3,
+            "f1": 40.2,
+            "f2": 39.7,
+            "tp": 68,
+            "fp": 97,
+            "fn": 105,
             "num_prs": 50
           }
         },
-        "qodo-extended-v2": {
+        "shipwright": {
           "strict": {
-            "precision": 64.8,
-            "recall": 66.2,
-            "f1": 65.5,
-            "f2": 65.9,
-            "tp": 92,
-            "fp": 50,
-            "fn": 47,
+            "precision": 49.6,
+            "recall": 48.2,
+            "f1": 48.9,
+            "f2": 48.5,
+            "tp": 67,
+            "fp": 68,
+            "fn": 72,
             "num_prs": 50
           },
           "core": {
-            "precision": 67.1,
-            "recall": 64.6,
-            "f1": 65.8,
-            "f2": 65.1,
-            "tp": 102,
-            "fp": 50,
-            "fn": 56,
-            "num_prs": 50
-          },
-          "all": {
-            "precision": 67.9,
-            "recall": 61.3,
-            "f1": 64.4,
-            "f2": 62.5,
-            "tp": 106,
-            "fp": 50,
-            "fn": 67,
-            "num_prs": 50
-          }
-        },
-        "kodus-v2": {
-          "strict": {
-            "precision": 57.1,
-            "recall": 40.3,
-            "f1": 47.3,
-            "f2": 42.8,
-            "tp": 56,
-            "fp": 42,
+            "precision": 52.4,
+            "recall": 47.5,
+            "f1": 49.8,
+            "f2": 48.4,
+            "tp": 75,
+            "fp": 68,
             "fn": 83,
             "num_prs": 50
           },
-          "core": {
-            "precision": 59.2,
-            "recall": 38.6,
-            "f1": 46.7,
-            "f2": 41.5,
-            "tp": 61,
-            "fp": 42,
-            "fn": 97,
-            "num_prs": 50
-          },
           "all": {
-            "precision": 59.6,
-            "recall": 35.8,
-            "f1": 44.8,
-            "f2": 38.9,
-            "tp": 62,
-            "fp": 42,
-            "fn": 111,
+            "precision": 54.4,
+            "recall": 46.8,
+            "f1": 50.3,
+            "f2": 48.2,
+            "tp": 81,
+            "fp": 68,
+            "fn": 92,
             "num_prs": 50
           }
         },
-        "graphite": {
+        "copilot-v2": {
           "strict": {
-            "precision": 100.0,
-            "recall": 7.9,
-            "f1": 14.7,
-            "f2": 9.7,
-            "tp": 11,
-            "fp": 0,
-            "fn": 128,
+            "precision": 35.6,
+            "recall": 66.9,
+            "f1": 46.5,
+            "f2": 56.9,
+            "tp": 93,
+            "fp": 168,
+            "fn": 46,
             "num_prs": 50
           },
           "core": {
-            "precision": 100.0,
-            "recall": 7.6,
-            "f1": 14.1,
-            "f2": 9.3,
-            "tp": 12,
-            "fp": 0,
-            "fn": 146,
+            "precision": 38.5,
+            "recall": 66.5,
+            "f1": 48.7,
+            "f2": 58.0,
+            "tp": 105,
+            "fp": 168,
+            "fn": 53,
             "num_prs": 50
           },
           "all": {
-            "precision": 100.0,
-            "recall": 6.9,
-            "f1": 13.0,
-            "f2": 8.5,
-            "tp": 12,
-            "fp": 0,
-            "fn": 161,
+            "precision": 40.6,
+            "recall": 66.5,
+            "f1": 50.4,
+            "f2": 59.0,
+            "tp": 115,
+            "fp": 168,
+            "fn": 58,
             "num_prs": 50
           }
         }
@@ -36326,134 +37209,6 @@
         ]
       },
       "overall_metrics": {
-        "augment": {
-          "strict": {
-            "precision": 49.4,
-            "recall": 61.9,
-            "f1": 55.0,
-            "f2": 58.9,
-            "tp": 86,
-            "fp": 88,
-            "fn": 53,
-            "num_prs": 50
-          },
-          "core": {
-            "precision": 52.4,
-            "recall": 61.4,
-            "f1": 56.6,
-            "f2": 59.4,
-            "tp": 97,
-            "fp": 88,
-            "fn": 61,
-            "num_prs": 50
-          },
-          "all": {
-            "precision": 53.7,
-            "recall": 59.0,
-            "f1": 56.2,
-            "f2": 57.8,
-            "tp": 102,
-            "fp": 88,
-            "fn": 71,
-            "num_prs": 50
-          }
-        },
-        "cubic-v2": {
-          "strict": {
-            "precision": 52.9,
-            "recall": 59.7,
-            "f1": 56.1,
-            "f2": 58.2,
-            "tp": 83,
-            "fp": 74,
-            "fn": 56,
-            "num_prs": 50
-          },
-          "core": {
-            "precision": 56.5,
-            "recall": 60.8,
-            "f1": 58.5,
-            "f2": 59.9,
-            "tp": 96,
-            "fp": 74,
-            "fn": 62,
-            "num_prs": 50
-          },
-          "all": {
-            "precision": 57.2,
-            "recall": 57.2,
-            "f1": 57.2,
-            "f2": 57.2,
-            "tp": 99,
-            "fp": 74,
-            "fn": 74,
-            "num_prs": 50
-          }
-        },
-        "copilot-v2": {
-          "strict": {
-            "precision": 31.4,
-            "recall": 61.9,
-            "f1": 41.6,
-            "f2": 51.8,
-            "tp": 86,
-            "fp": 188,
-            "fn": 53,
-            "num_prs": 50
-          },
-          "core": {
-            "precision": 34.3,
-            "recall": 62.0,
-            "f1": 44.1,
-            "f2": 53.4,
-            "tp": 98,
-            "fp": 188,
-            "fn": 60,
-            "num_prs": 50
-          },
-          "all": {
-            "precision": 36.1,
-            "recall": 61.3,
-            "f1": 45.4,
-            "f2": 53.8,
-            "tp": 106,
-            "fp": 188,
-            "fn": 67,
-            "num_prs": 50
-          }
-        },
-        "greptile-v4-1": {
-          "strict": {
-            "precision": 44.9,
-            "recall": 50.4,
-            "f1": 47.5,
-            "f2": 49.2,
-            "tp": 70,
-            "fp": 86,
-            "fn": 69,
-            "num_prs": 50
-          },
-          "core": {
-            "precision": 46.9,
-            "recall": 48.1,
-            "f1": 47.5,
-            "f2": 47.9,
-            "tp": 76,
-            "fp": 86,
-            "fn": 82,
-            "num_prs": 50
-          },
-          "all": {
-            "precision": 49.1,
-            "recall": 48.0,
-            "f1": 48.5,
-            "f2": 48.2,
-            "tp": 83,
-            "fp": 86,
-            "fn": 90,
-            "num_prs": 50
-          }
-        },
         "sourcery": {
           "strict": {
             "precision": 34.2,
@@ -36486,99 +37241,291 @@
             "num_prs": 50
           }
         },
-        "qodo-v2": {
+        "macroscope": {
           "strict": {
-            "precision": 45.1,
+            "precision": 50.4,
+            "recall": 45.3,
+            "f1": 47.7,
+            "f2": 46.3,
+            "tp": 63,
+            "fp": 62,
+            "fn": 76,
+            "num_prs": 50
+          },
+          "core": {
+            "precision": 51.6,
+            "recall": 41.8,
+            "f1": 46.2,
+            "f2": 43.4,
+            "tp": 66,
+            "fp": 62,
+            "fn": 92,
+            "num_prs": 50
+          },
+          "all": {
+            "precision": 53.0,
+            "recall": 40.5,
+            "f1": 45.9,
+            "f2": 42.5,
+            "tp": 70,
+            "fp": 62,
+            "fn": 103,
+            "num_prs": 50
+          }
+        },
+        "graphite": {
+          "strict": {
+            "precision": 100.0,
+            "recall": 7.9,
+            "f1": 14.7,
+            "f2": 9.7,
+            "tp": 11,
+            "fp": 0,
+            "fn": 128,
+            "num_prs": 50
+          },
+          "core": {
+            "precision": 100.0,
+            "recall": 7.6,
+            "f1": 14.1,
+            "f2": 9.3,
+            "tp": 12,
+            "fp": 0,
+            "fn": 146,
+            "num_prs": 50
+          },
+          "all": {
+            "precision": 100.0,
+            "recall": 6.9,
+            "f1": 13.0,
+            "f2": 8.5,
+            "tp": 12,
+            "fp": 0,
+            "fn": 161,
+            "num_prs": 50
+          }
+        },
+        "augment": {
+          "strict": {
+            "precision": 49.4,
+            "recall": 61.9,
+            "f1": 55.0,
+            "f2": 58.9,
+            "tp": 86,
+            "fp": 88,
+            "fn": 53,
+            "num_prs": 50
+          },
+          "core": {
+            "precision": 52.4,
+            "recall": 61.4,
+            "f1": 56.6,
+            "f2": 59.4,
+            "tp": 97,
+            "fp": 88,
+            "fn": 61,
+            "num_prs": 50
+          },
+          "all": {
+            "precision": 53.7,
             "recall": 59.0,
-            "f1": 51.1,
-            "f2": 55.6,
-            "tp": 82,
-            "fp": 100,
-            "fn": 57,
+            "f1": 56.2,
+            "f2": 57.8,
+            "tp": 102,
+            "fp": 88,
+            "fn": 71,
+            "num_prs": 50
+          }
+        },
+        "kodus-v2": {
+          "strict": {
+            "precision": 55.2,
+            "recall": 38.1,
+            "f1": 45.1,
+            "f2": 40.6,
+            "tp": 53,
+            "fp": 43,
+            "fn": 86,
             "num_prs": 50
           },
           "core": {
-            "precision": 47.1,
-            "recall": 56.3,
-            "f1": 51.3,
-            "f2": 54.2,
-            "tp": 89,
-            "fp": 100,
-            "fn": 69,
+            "precision": 57.4,
+            "recall": 36.7,
+            "f1": 44.8,
+            "f2": 39.6,
+            "tp": 58,
+            "fp": 43,
+            "fn": 100,
             "num_prs": 50
           },
           "all": {
-            "precision": 47.9,
+            "precision": 57.8,
+            "recall": 34.1,
+            "f1": 42.9,
+            "f2": 37.2,
+            "tp": 59,
+            "fp": 43,
+            "fn": 114,
+            "num_prs": 50
+          }
+        },
+        "qodo-extended-v2": {
+          "strict": {
+            "precision": 53.6,
+            "recall": 58.3,
+            "f1": 55.9,
+            "f2": 57.3,
+            "tp": 81,
+            "fp": 70,
+            "fn": 58,
+            "num_prs": 50
+          },
+          "core": {
+            "precision": 56.2,
+            "recall": 57.0,
+            "f1": 56.6,
+            "f2": 56.8,
+            "tp": 90,
+            "fp": 70,
+            "fn": 68,
+            "num_prs": 50
+          },
+          "all": {
+            "precision": 56.8,
             "recall": 53.2,
-            "f1": 50.4,
-            "f2": 52.0,
+            "f1": 54.9,
+            "f2": 53.9,
             "tp": 92,
-            "fp": 100,
+            "fp": 70,
             "fn": 81,
             "num_prs": 50
           }
         },
-        "gitlab": {
+        "kg": {
           "strict": {
-            "precision": 44.7,
-            "recall": 48.9,
-            "f1": 46.7,
-            "f2": 48.0,
-            "tp": 68,
-            "fp": 84,
-            "fn": 71,
+            "precision": 48.8,
+            "recall": 15.1,
+            "f1": 23.1,
+            "f2": 17.5,
+            "tp": 21,
+            "fp": 22,
+            "fn": 118,
             "num_prs": 50
           },
           "core": {
-            "precision": 47.5,
-            "recall": 48.1,
-            "f1": 47.8,
-            "f2": 48.0,
-            "tp": 76,
-            "fp": 84,
-            "fn": 82,
+            "precision": 50.0,
+            "recall": 13.9,
+            "f1": 21.8,
+            "f2": 16.3,
+            "tp": 22,
+            "fp": 22,
+            "fn": 136,
             "num_prs": 50
           },
           "all": {
-            "precision": 49.7,
-            "recall": 48.0,
-            "f1": 48.8,
-            "f2": 48.3,
+            "precision": 51.1,
+            "recall": 13.3,
+            "f1": 21.1,
+            "f2": 15.6,
+            "tp": 23,
+            "fp": 22,
+            "fn": 150,
+            "num_prs": 50
+          }
+        },
+        "cubic-v2": {
+          "strict": {
+            "precision": 52.9,
+            "recall": 59.7,
+            "f1": 56.1,
+            "f2": 58.2,
             "tp": 83,
-            "fp": 84,
-            "fn": 90,
+            "fp": 74,
+            "fn": 56,
+            "num_prs": 50
+          },
+          "core": {
+            "precision": 56.5,
+            "recall": 60.8,
+            "f1": 58.5,
+            "f2": 59.9,
+            "tp": 96,
+            "fp": 74,
+            "fn": 62,
+            "num_prs": 50
+          },
+          "all": {
+            "precision": 57.2,
+            "recall": 57.2,
+            "f1": 57.2,
+            "f2": 57.2,
+            "tp": 99,
+            "fp": 74,
+            "fn": 74,
             "num_prs": 50
           }
         },
-        "codeant-v2": {
+        "gemini-v2": {
           "strict": {
-            "precision": 31.1,
-            "recall": 37.4,
-            "f1": 34.0,
-            "f2": 36.0,
-            "tp": 52,
-            "fp": 115,
-            "fn": 87,
+            "precision": 36.7,
+            "recall": 51.8,
+            "f1": 43.0,
+            "f2": 47.9,
+            "tp": 72,
+            "fp": 124,
+            "fn": 67,
             "num_prs": 50
           },
           "core": {
-            "precision": 32.4,
-            "recall": 34.8,
-            "f1": 33.5,
-            "f2": 34.3,
-            "tp": 55,
-            "fp": 115,
-            "fn": 103,
+            "precision": 39.2,
+            "recall": 50.6,
+            "f1": 44.2,
+            "f2": 47.8,
+            "tp": 80,
+            "fp": 124,
+            "fn": 78,
             "num_prs": 50
           },
           "all": {
-            "precision": 33.1,
-            "recall": 32.9,
-            "f1": 33.0,
-            "f2": 33.0,
-            "tp": 57,
-            "fp": 115,
-            "fn": 116,
+            "precision": 41.2,
+            "recall": 50.3,
+            "f1": 45.3,
+            "f2": 48.2,
+            "tp": 87,
+            "fp": 124,
+            "fn": 86,
+            "num_prs": 50
+          }
+        },
+        "greptile-v4-1": {
+          "strict": {
+            "precision": 44.9,
+            "recall": 50.4,
+            "f1": 47.5,
+            "f2": 49.2,
+            "tp": 70,
+            "fp": 86,
+            "fn": 69,
+            "num_prs": 50
+          },
+          "core": {
+            "precision": 46.9,
+            "recall": 48.1,
+            "f1": 47.5,
+            "f2": 47.9,
+            "tp": 76,
+            "fp": 86,
+            "fn": 82,
+            "num_prs": 50
+          },
+          "all": {
+            "precision": 49.1,
+            "recall": 48.0,
+            "f1": 48.5,
+            "f2": 48.2,
+            "tp": 83,
+            "fp": 86,
+            "fn": 90,
             "num_prs": 50
           }
         },
@@ -36614,35 +37561,35 @@
             "num_prs": 50
           }
         },
-        "claude-code": {
+        "claude": {
           "strict": {
-            "precision": 34.3,
-            "recall": 42.4,
-            "f1": 37.9,
-            "f2": 40.5,
-            "tp": 59,
-            "fp": 113,
-            "fn": 80,
+            "precision": 36.1,
+            "recall": 37.4,
+            "f1": 36.7,
+            "f2": 37.1,
+            "tp": 52,
+            "fp": 92,
+            "fn": 87,
             "num_prs": 50
           },
           "core": {
-            "precision": 36.5,
-            "recall": 41.1,
-            "f1": 38.7,
-            "f2": 40.1,
-            "tp": 65,
-            "fp": 113,
-            "fn": 93,
+            "precision": 38.3,
+            "recall": 36.1,
+            "f1": 37.1,
+            "f2": 36.5,
+            "tp": 57,
+            "fp": 92,
+            "fn": 101,
             "num_prs": 50
           },
           "all": {
-            "precision": 38.3,
-            "recall": 40.5,
-            "f1": 39.3,
-            "f2": 40.0,
-            "tp": 70,
-            "fp": 113,
-            "fn": 103,
+            "precision": 40.6,
+            "recall": 36.4,
+            "f1": 38.4,
+            "f2": 37.2,
+            "tp": 63,
+            "fp": 92,
+            "fn": 110,
             "num_prs": 50
           }
         },
@@ -36678,99 +37625,131 @@
             "num_prs": 50
           }
         },
-        "macroscope": {
+        "baz": {
           "strict": {
-            "precision": 50.4,
-            "recall": 45.3,
-            "f1": 47.7,
-            "f2": 46.3,
-            "tp": 63,
-            "fp": 62,
-            "fn": 76,
+            "precision": 54.6,
+            "recall": 38.1,
+            "f1": 44.9,
+            "f2": 40.6,
+            "tp": 53,
+            "fp": 44,
+            "fn": 86,
             "num_prs": 50
           },
           "core": {
-            "precision": 51.6,
-            "recall": 41.8,
-            "f1": 46.2,
-            "f2": 43.4,
-            "tp": 66,
-            "fp": 62,
-            "fn": 92,
+            "precision": 55.6,
+            "recall": 34.8,
+            "f1": 42.8,
+            "f2": 37.6,
+            "tp": 55,
+            "fp": 44,
+            "fn": 103,
             "num_prs": 50
           },
           "all": {
-            "precision": 53.0,
+            "precision": 56.9,
+            "recall": 33.5,
+            "f1": 42.2,
+            "f2": 36.5,
+            "tp": 58,
+            "fp": 44,
+            "fn": 115,
+            "num_prs": 50
+          }
+        },
+        "claude-code": {
+          "strict": {
+            "precision": 34.3,
+            "recall": 42.4,
+            "f1": 37.9,
+            "f2": 40.5,
+            "tp": 59,
+            "fp": 113,
+            "fn": 80,
+            "num_prs": 50
+          },
+          "core": {
+            "precision": 36.5,
+            "recall": 41.1,
+            "f1": 38.7,
+            "f2": 40.1,
+            "tp": 65,
+            "fp": 113,
+            "fn": 93,
+            "num_prs": 50
+          },
+          "all": {
+            "precision": 38.3,
             "recall": 40.5,
-            "f1": 45.9,
-            "f2": 42.5,
+            "f1": 39.3,
+            "f2": 40.0,
             "tp": 70,
-            "fp": 62,
+            "fp": 113,
             "fn": 103,
             "num_prs": 50
           }
         },
-        "claude": {
+        "gitlab": {
           "strict": {
-            "precision": 36.1,
-            "recall": 37.4,
-            "f1": 36.7,
-            "f2": 37.1,
-            "tp": 52,
-            "fp": 92,
-            "fn": 87,
+            "precision": 44.7,
+            "recall": 48.9,
+            "f1": 46.7,
+            "f2": 48.0,
+            "tp": 68,
+            "fp": 84,
+            "fn": 71,
             "num_prs": 50
           },
           "core": {
-            "precision": 38.3,
-            "recall": 36.1,
-            "f1": 37.1,
-            "f2": 36.5,
-            "tp": 57,
-            "fp": 92,
-            "fn": 101,
+            "precision": 47.5,
+            "recall": 48.1,
+            "f1": 47.8,
+            "f2": 48.0,
+            "tp": 76,
+            "fp": 84,
+            "fn": 82,
             "num_prs": 50
           },
           "all": {
-            "precision": 40.6,
-            "recall": 36.4,
-            "f1": 38.4,
-            "f2": 37.2,
-            "tp": 63,
-            "fp": 92,
-            "fn": 110,
+            "precision": 49.7,
+            "recall": 48.0,
+            "f1": 48.8,
+            "f2": 48.3,
+            "tp": 83,
+            "fp": 84,
+            "fn": 90,
             "num_prs": 50
           }
         },
-        "gemini-v2": {
+        "qodo-v2": {
           "strict": {
-            "precision": 36.7,
-            "recall": 51.8,
-            "f1": 43.0,
-            "f2": 47.9,
-            "tp": 72,
-            "fp": 124,
-            "fn": 67,
+            "precision": 45.1,
+            "recall": 59.0,
+            "f1": 51.1,
+            "f2": 55.6,
+            "tp": 82,
+            "fp": 100,
+            "fn": 57,
             "num_prs": 50
           },
           "core": {
-            "precision": 39.2,
-            "recall": 50.6,
-            "f1": 44.2,
-            "f2": 47.8,
-            "tp": 80,
-            "fp": 124,
-            "fn": 78,
+            "precision": 47.1,
+            "recall": 56.3,
+            "f1": 51.3,
+            "f2": 54.2,
+            "tp": 89,
+            "fp": 100,
+            "fn": 69,
             "num_prs": 50
           },
           "all": {
-            "precision": 41.2,
-            "recall": 50.3,
-            "f1": 45.3,
-            "f2": 48.2,
-            "tp": 87,
-            "fp": 124,
-            "fn": 86,
+            "precision": 47.9,
+            "recall": 53.2,
+            "f1": 50.4,
+            "f2": 52.0,
+            "tp": 92,
+            "fp": 100,
+            "fn": 81,
             "num_prs": 50
           }
         },
@@ -36806,163 +37785,67 @@
             "num_prs": 50
           }
         },
-        "baz": {
+        "codeant-v2": {
           "strict": {
-            "precision": 54.6,
-            "recall": 38.1,
-            "f1": 44.9,
-            "f2": 40.6,
-            "tp": 53,
-            "fp": 44,
-            "fn": 86,
+            "precision": 31.1,
+            "recall": 37.4,
+            "f1": 34.0,
+            "f2": 36.0,
+            "tp": 52,
+            "fp": 115,
+            "fn": 87,
             "num_prs": 50
           },
           "core": {
-            "precision": 55.6,
+            "precision": 32.4,
             "recall": 34.8,
-            "f1": 42.8,
-            "f2": 37.6,
+            "f1": 33.5,
+            "f2": 34.3,
             "tp": 55,
-            "fp": 44,
+            "fp": 115,
             "fn": 103,
             "num_prs": 50
           },
           "all": {
-            "precision": 56.9,
-            "recall": 33.5,
-            "f1": 42.2,
-            "f2": 36.5,
-            "tp": 58,
-            "fp": 44,
-            "fn": 115,
-            "num_prs": 50
-          }
-        },
-        "kg": {
-          "strict": {
-            "precision": 48.8,
-            "recall": 15.1,
-            "f1": 23.1,
-            "f2": 17.5,
-            "tp": 21,
-            "fp": 22,
-            "fn": 118,
-            "num_prs": 50
-          },
-          "core": {
-            "precision": 50.0,
-            "recall": 13.9,
-            "f1": 21.8,
-            "f2": 16.3,
-            "tp": 22,
-            "fp": 22,
-            "fn": 136,
-            "num_prs": 50
-          },
-          "all": {
-            "precision": 51.1,
-            "recall": 13.3,
-            "f1": 21.1,
-            "f2": 15.6,
-            "tp": 23,
-            "fp": 22,
-            "fn": 150,
-            "num_prs": 50
-          }
-        },
-        "qodo-extended-v2": {
-          "strict": {
-            "precision": 53.6,
-            "recall": 58.3,
-            "f1": 55.9,
-            "f2": 57.3,
-            "tp": 81,
-            "fp": 70,
-            "fn": 58,
-            "num_prs": 50
-          },
-          "core": {
-            "precision": 56.2,
-            "recall": 57.0,
-            "f1": 56.6,
-            "f2": 56.8,
-            "tp": 90,
-            "fp": 70,
-            "fn": 68,
-            "num_prs": 50
-          },
-          "all": {
-            "precision": 56.8,
-            "recall": 53.2,
-            "f1": 54.9,
-            "f2": 53.9,
-            "tp": 92,
-            "fp": 70,
-            "fn": 81,
-            "num_prs": 50
-          }
-        },
-        "kodus-v2": {
-          "strict": {
-            "precision": 55.2,
-            "recall": 38.1,
-            "f1": 45.1,
-            "f2": 40.6,
-            "tp": 53,
-            "fp": 43,
-            "fn": 86,
-            "num_prs": 50
-          },
-          "core": {
-            "precision": 57.4,
-            "recall": 36.7,
-            "f1": 44.8,
-            "f2": 39.6,
-            "tp": 58,
-            "fp": 43,
-            "fn": 100,
-            "num_prs": 50
-          },
-          "all": {
-            "precision": 57.8,
-            "recall": 34.1,
-            "f1": 42.9,
-            "f2": 37.2,
-            "tp": 59,
-            "fp": 43,
-            "fn": 114,
+            "precision": 33.1,
+            "recall": 32.9,
+            "f1": 33.0,
+            "f2": 33.0,
+            "tp": 57,
+            "fp": 115,
+            "fn": 116,
             "num_prs": 50
           }
         },
-        "graphite": {
+        "copilot-v2": {
           "strict": {
-            "precision": 100.0,
-            "recall": 7.9,
-            "f1": 14.7,
-            "f2": 9.7,
-            "tp": 11,
-            "fp": 0,
-            "fn": 128,
+            "precision": 31.4,
+            "recall": 61.9,
+            "f1": 41.6,
+            "f2": 51.8,
+            "tp": 86,
+            "fp": 188,
+            "fn": 53,
             "num_prs": 50
           },
           "core": {
-            "precision": 100.0,
-            "recall": 7.6,
-            "f1": 14.1,
-            "f2": 9.3,
-            "tp": 12,
-            "fp": 0,
-            "fn": 146,
+            "precision": 34.3,
+            "recall": 62.0,
+            "f1": 44.1,
+            "f2": 53.4,
+            "tp": 98,
+            "fp": 188,
+            "fn": 60,
             "num_prs": 50
           },
           "all": {
-            "precision": 100.0,
-            "recall": 6.9,
-            "f1": 13.0,
-            "f2": 8.5,
-            "tp": 12,
-            "fp": 0,
-            "fn": 161,
+            "precision": 36.1,
+            "recall": 61.3,
+            "f1": 45.4,
+            "f2": 53.8,
+            "tp": 106,
+            "fp": 188,
+            "fn": 67,
             "num_prs": 50
           }
         }
@@ -54810,6 +55693,102 @@
         ]
       },
       "overall_metrics": {
+        "sourcery": {
+          "strict": {
+            "precision": 33.3,
+            "recall": 50.4,
+            "f1": 40.1,
+            "f2": 45.7,
+            "tp": 70,
+            "fp": 140,
+            "fn": 69,
+            "num_prs": 50
+          },
+          "core": {
+            "precision": 36.1,
+            "recall": 50.0,
+            "f1": 41.9,
+            "f2": 46.4,
+            "tp": 79,
+            "fp": 140,
+            "fn": 79,
+            "num_prs": 50
+          },
+          "all": {
+            "precision": 38.1,
+            "recall": 49.7,
+            "f1": 43.1,
+            "f2": 46.8,
+            "tp": 86,
+            "fp": 140,
+            "fn": 87,
+            "num_prs": 50
+          }
+        },
+        "macroscope": {
+          "strict": {
+            "precision": 47.7,
+            "recall": 44.6,
+            "f1": 46.1,
+            "f2": 45.2,
+            "tp": 62,
+            "fp": 68,
+            "fn": 77,
+            "num_prs": 50
+          },
+          "core": {
+            "precision": 48.9,
+            "recall": 41.1,
+            "f1": 44.7,
+            "f2": 42.5,
+            "tp": 65,
+            "fp": 68,
+            "fn": 93,
+            "num_prs": 50
+          },
+          "all": {
+            "precision": 50.0,
+            "recall": 39.3,
+            "f1": 44.0,
+            "f2": 41.1,
+            "tp": 68,
+            "fp": 68,
+            "fn": 105,
+            "num_prs": 50
+          }
+        },
+        "graphite": {
+          "strict": {
+            "precision": 71.4,
+            "recall": 7.2,
+            "f1": 13.1,
+            "f2": 8.8,
+            "tp": 10,
+            "fp": 4,
+            "fn": 129,
+            "num_prs": 50
+          },
+          "core": {
+            "precision": 73.3,
+            "recall": 7.0,
+            "f1": 12.7,
+            "f2": 8.5,
+            "tp": 11,
+            "fp": 4,
+            "fn": 147,
+            "num_prs": 50
+          },
+          "all": {
+            "precision": 73.3,
+            "recall": 6.4,
+            "f1": 11.7,
+            "f2": 7.8,
+            "tp": 11,
+            "fp": 4,
+            "fn": 162,
+            "num_prs": 50
+          }
+        },
         "augment": {
           "strict": {
             "precision": 45.9,
@@ -54842,6 +55821,102 @@
             "num_prs": 50
           }
         },
+        "qodo-extended-v2": {
+          "strict": {
+            "precision": 55.9,
+            "recall": 61.2,
+            "f1": 58.4,
+            "f2": 60.0,
+            "tp": 85,
+            "fp": 67,
+            "fn": 54,
+            "num_prs": 50
+          },
+          "core": {
+            "precision": 58.4,
+            "recall": 59.5,
+            "f1": 58.9,
+            "f2": 59.3,
+            "tp": 94,
+            "fp": 67,
+            "fn": 64,
+            "num_prs": 50
+          },
+          "all": {
+            "precision": 58.9,
+            "recall": 55.5,
+            "f1": 57.1,
+            "f2": 56.1,
+            "tp": 96,
+            "fp": 67,
+            "fn": 77,
+            "num_prs": 50
+          }
+        },
+        "kodus-v2": {
+          "strict": {
+            "precision": 50.5,
+            "recall": 36.0,
+            "f1": 42.0,
+            "f2": 38.2,
+            "tp": 50,
+            "fp": 49,
+            "fn": 89,
+            "num_prs": 50
+          },
+          "core": {
+            "precision": 52.4,
+            "recall": 34.2,
+            "f1": 41.4,
+            "f2": 36.7,
+            "tp": 54,
+            "fp": 49,
+            "fn": 104,
+            "num_prs": 50
+          },
+          "all": {
+            "precision": 53.8,
+            "recall": 32.9,
+            "f1": 40.9,
+            "f2": 35.7,
+            "tp": 57,
+            "fp": 49,
+            "fn": 116,
+            "num_prs": 50
+          }
+        },
+        "kg": {
+          "strict": {
+            "precision": 53.5,
+            "recall": 16.5,
+            "f1": 25.3,
+            "f2": 19.2,
+            "tp": 23,
+            "fp": 20,
+            "fn": 116,
+            "num_prs": 50
+          },
+          "core": {
+            "precision": 54.5,
+            "recall": 15.2,
+            "f1": 23.8,
+            "f2": 17.8,
+            "tp": 24,
+            "fp": 20,
+            "fn": 134,
+            "num_prs": 50
+          },
+          "all": {
+            "precision": 55.6,
+            "recall": 14.5,
+            "f1": 22.9,
+            "f2": 17.0,
+            "tp": 25,
+            "fp": 20,
+            "fn": 148,
+            "num_prs": 50
+          }
+        },
         "cubic-v2": {
           "strict": {
             "precision": 53.2,
@@ -54874,35 +55949,35 @@
             "num_prs": 50
           }
         },
-        "copilot-v2": {
+        "gemini-v2": {
           "strict": {
-            "precision": 30.4,
-            "recall": 63.3,
-            "f1": 41.1,
-            "f2": 52.1,
-            "tp": 88,
-            "fp": 201,
-            "fn": 51,
+            "precision": 36.2,
+            "recall": 54.0,
+            "f1": 43.4,
+            "f2": 49.1,
+            "tp": 75,
+            "fp": 132,
+            "fn": 64,
             "num_prs": 50
           },
           "core": {
-            "precision": 33.2,
-            "recall": 63.3,
-            "f1": 43.6,
-            "f2": 53.6,
-            "tp": 100,
-            "fp": 201,
-            "fn": 58,
+            "precision": 38.0,
+            "recall": 51.3,
+            "f1": 43.7,
+            "f2": 47.9,
+            "tp": 81,
+            "fp": 132,
+            "fn": 77,
             "num_prs": 50
           },
           "all": {
-            "precision": 35.2,
-            "recall": 63.0,
-            "f1": 45.1,
-            "f2": 54.4,
-            "tp": 109,
-            "fp": 201,
-            "fn": 64,
+            "precision": 40.3,
+            "recall": 51.4,
+            "f1": 45.2,
+            "f2": 48.7,
+            "tp": 89,
+            "fp": 132,
+            "fn": 84,
             "num_prs": 50
           }
         },
@@ -54938,134 +56013,6 @@
             "num_prs": 50
           }
         },
-        "sourcery": {
-          "strict": {
-            "precision": 33.3,
-            "recall": 50.4,
-            "f1": 40.1,
-            "f2": 45.7,
-            "tp": 70,
-            "fp": 140,
-            "fn": 69,
-            "num_prs": 50
-          },
-          "core": {
-            "precision": 36.1,
-            "recall": 50.0,
-            "f1": 41.9,
-            "f2": 46.4,
-            "tp": 79,
-            "fp": 140,
-            "fn": 79,
-            "num_prs": 50
-          },
-          "all": {
-            "precision": 38.1,
-            "recall": 49.7,
-            "f1": 43.1,
-            "f2": 46.8,
-            "tp": 86,
-            "fp": 140,
-            "fn": 87,
-            "num_prs": 50
-          }
-        },
-        "qodo-v2": {
-          "strict": {
-            "precision": 44.1,
-            "recall": 64.7,
-            "f1": 52.5,
-            "f2": 59.2,
-            "tp": 90,
-            "fp": 114,
-            "fn": 49,
-            "num_prs": 50
-          },
-          "core": {
-            "precision": 46.5,
-            "recall": 62.7,
-            "f1": 53.4,
-            "f2": 58.6,
-            "tp": 99,
-            "fp": 114,
-            "fn": 59,
-            "num_prs": 50
-          },
-          "all": {
-            "precision": 47.5,
-            "recall": 59.5,
-            "f1": 52.8,
-            "f2": 56.7,
-            "tp": 103,
-            "fp": 114,
-            "fn": 70,
-            "num_prs": 50
-          }
-        },
-        "gitlab": {
-          "strict": {
-            "precision": 42.2,
-            "recall": 48.9,
-            "f1": 45.3,
-            "f2": 47.4,
-            "tp": 68,
-            "fp": 93,
-            "fn": 71,
-            "num_prs": 50
-          },
-          "core": {
-            "precision": 45.0,
-            "recall": 48.1,
-            "f1": 46.5,
-            "f2": 47.4,
-            "tp": 76,
-            "fp": 93,
-            "fn": 82,
-            "num_prs": 50
-          },
-          "all": {
-            "precision": 46.9,
-            "recall": 47.4,
-            "f1": 47.1,
-            "f2": 47.3,
-            "tp": 82,
-            "fp": 93,
-            "fn": 91,
-            "num_prs": 50
-          }
-        },
-        "codeant-v2": {
-          "strict": {
-            "precision": 29.3,
-            "recall": 38.1,
-            "f1": 33.1,
-            "f2": 36.0,
-            "tp": 53,
-            "fp": 128,
-            "fn": 86,
-            "num_prs": 50
-          },
-          "core": {
-            "precision": 30.4,
-            "recall": 35.4,
-            "f1": 32.7,
-            "f2": 34.3,
-            "tp": 56,
-            "fp": 128,
-            "fn": 102,
-            "num_prs": 50
-          },
-          "all": {
-            "precision": 31.2,
-            "recall": 33.5,
-            "f1": 32.3,
-            "f2": 33.0,
-            "tp": 58,
-            "fp": 128,
-            "fn": 115,
-            "num_prs": 50
-          }
-        },
         "bugbot": {
           "strict": {
             "precision": 49.6,
@@ -55098,35 +56045,35 @@
             "num_prs": 50
           }
         },
-        "claude-code": {
+        "claude": {
           "strict": {
-            "precision": 38.2,
-            "recall": 43.2,
-            "f1": 40.5,
-            "f2": 42.1,
-            "tp": 60,
-            "fp": 97,
-            "fn": 79,
+            "precision": 36.7,
+            "recall": 38.8,
+            "f1": 37.8,
+            "f2": 38.4,
+            "tp": 54,
+            "fp": 93,
+            "fn": 85,
             "num_prs": 50
           },
           "core": {
-            "precision": 40.1,
-            "recall": 41.1,
-            "f1": 40.6,
-            "f2": 40.9,
-            "tp": 65,
-            "fp": 97,
-            "fn": 93,
+            "precision": 38.8,
+            "recall": 37.3,
+            "f1": 38.1,
+            "f2": 37.6,
+            "tp": 59,
+            "fp": 93,
+            "fn": 99,
             "num_prs": 50
           },
           "all": {
-            "precision": 42.3,
-            "recall": 41.0,
-            "f1": 41.6,
-            "f2": 41.3,
-            "tp": 71,
-            "fp": 97,
-            "fn": 102,
+            "precision": 41.1,
+            "recall": 37.6,
+            "f1": 39.3,
+            "f2": 38.2,
+            "tp": 65,
+            "fp": 93,
+            "fn": 108,
             "num_prs": 50
           }
         },
@@ -55162,99 +56109,131 @@
             "num_prs": 50
           }
         },
-        "macroscope": {
+        "baz": {
           "strict": {
-            "precision": 47.7,
-            "recall": 44.6,
-            "f1": 46.1,
-            "f2": 45.2,
-            "tp": 62,
-            "fp": 68,
-            "fn": 77,
+            "precision": 43.9,
+            "recall": 36.0,
+            "f1": 39.5,
+            "f2": 37.3,
+            "tp": 50,
+            "fp": 64,
+            "fn": 89,
             "num_prs": 50
           },
           "core": {
-            "precision": 48.9,
+            "precision": 44.8,
+            "recall": 32.9,
+            "f1": 38.0,
+            "f2": 34.8,
+            "tp": 52,
+            "fp": 64,
+            "fn": 106,
+            "num_prs": 50
+          },
+          "all": {
+            "precision": 46.2,
+            "recall": 31.8,
+            "f1": 37.7,
+            "f2": 33.9,
+            "tp": 55,
+            "fp": 64,
+            "fn": 118,
+            "num_prs": 50
+          }
+        },
+        "claude-code": {
+          "strict": {
+            "precision": 38.2,
+            "recall": 43.2,
+            "f1": 40.5,
+            "f2": 42.1,
+            "tp": 60,
+            "fp": 97,
+            "fn": 79,
+            "num_prs": 50
+          },
+          "core": {
+            "precision": 40.1,
             "recall": 41.1,
-            "f1": 44.7,
-            "f2": 42.5,
+            "f1": 40.6,
+            "f2": 40.9,
             "tp": 65,
-            "fp": 68,
+            "fp": 97,
             "fn": 93,
             "num_prs": 50
           },
           "all": {
-            "precision": 50.0,
-            "recall": 39.3,
-            "f1": 44.0,
-            "f2": 41.1,
-            "tp": 68,
-            "fp": 68,
-            "fn": 105,
+            "precision": 42.3,
+            "recall": 41.0,
+            "f1": 41.6,
+            "f2": 41.3,
+            "tp": 71,
+            "fp": 97,
+            "fn": 102,
             "num_prs": 50
           }
         },
-        "claude": {
+        "gitlab": {
           "strict": {
-            "precision": 36.7,
-            "recall": 38.8,
-            "f1": 37.8,
-            "f2": 38.4,
-            "tp": 54,
+            "precision": 42.2,
+            "recall": 48.9,
+            "f1": 45.3,
+            "f2": 47.4,
+            "tp": 68,
             "fp": 93,
-            "fn": 85,
+            "fn": 71,
             "num_prs": 50
           },
           "core": {
-            "precision": 38.8,
-            "recall": 37.3,
-            "f1": 38.1,
-            "f2": 37.6,
-            "tp": 59,
+            "precision": 45.0,
+            "recall": 48.1,
+            "f1": 46.5,
+            "f2": 47.4,
+            "tp": 76,
             "fp": 93,
-            "fn": 99,
+            "fn": 82,
             "num_prs": 50
           },
           "all": {
-            "precision": 41.1,
-            "recall": 37.6,
-            "f1": 39.3,
-            "f2": 38.2,
-            "tp": 65,
+            "precision": 46.9,
+            "recall": 47.4,
+            "f1": 47.1,
+            "f2": 47.3,
+            "tp": 82,
             "fp": 93,
-            "fn": 108,
+            "fn": 91,
             "num_prs": 50
           }
         },
-        "gemini-v2": {
+        "qodo-v2": {
           "strict": {
-            "precision": 36.2,
-            "recall": 54.0,
-            "f1": 43.4,
-            "f2": 49.1,
-            "tp": 75,
-            "fp": 132,
-            "fn": 64,
+            "precision": 44.1,
+            "recall": 64.7,
+            "f1": 52.5,
+            "f2": 59.2,
+            "tp": 90,
+            "fp": 114,
+            "fn": 49,
             "num_prs": 50
           },
           "core": {
-            "precision": 38.0,
-            "recall": 51.3,
-            "f1": 43.7,
-            "f2": 47.9,
-            "tp": 81,
-            "fp": 132,
-            "fn": 77,
+            "precision": 46.5,
+            "recall": 62.7,
+            "f1": 53.4,
+            "f2": 58.6,
+            "tp": 99,
+            "fp": 114,
+            "fn": 59,
             "num_prs": 50
           },
           "all": {
-            "precision": 40.3,
-            "recall": 51.4,
-            "f1": 45.2,
-            "f2": 48.7,
-            "tp": 89,
-            "fp": 132,
-            "fn": 84,
+            "precision": 47.5,
+            "recall": 59.5,
+            "f1": 52.8,
+            "f2": 56.7,
+            "tp": 103,
+            "fp": 114,
+            "fn": 70,
             "num_prs": 50
           }
         },
@@ -55290,164 +56269,68 @@
             "num_prs": 50
           }
         },
-        "baz": {
+        "codeant-v2": {
           "strict": {
-            "precision": 43.9,
-            "recall": 36.0,
-            "f1": 39.5,
-            "f2": 37.3,
-            "tp": 50,
-            "fp": 64,
-            "fn": 89,
+            "precision": 29.3,
+            "recall": 38.1,
+            "f1": 33.1,
+            "f2": 36.0,
+            "tp": 53,
+            "fp": 128,
+            "fn": 86,
             "num_prs": 50
           },
           "core": {
-            "precision": 44.8,
-            "recall": 32.9,
-            "f1": 38.0,
-            "f2": 34.8,
-            "tp": 52,
-            "fp": 64,
-            "fn": 106,
+            "precision": 30.4,
+            "recall": 35.4,
+            "f1": 32.7,
+            "f2": 34.3,
+            "tp": 56,
+            "fp": 128,
+            "fn": 102,
             "num_prs": 50
           },
           "all": {
-            "precision": 46.2,
-            "recall": 31.8,
-            "f1": 37.7,
-            "f2": 33.9,
-            "tp": 55,
-            "fp": 64,
-            "fn": 118,
+            "precision": 31.2,
+            "recall": 33.5,
+            "f1": 32.3,
+            "f2": 33.0,
+            "tp": 58,
+            "fp": 128,
+            "fn": 115,
             "num_prs": 50
           }
         },
-        "kg": {
+        "copilot-v2": {
           "strict": {
-            "precision": 53.5,
-            "recall": 16.5,
-            "f1": 25.3,
-            "f2": 19.2,
-            "tp": 23,
-            "fp": 20,
-            "fn": 116,
+            "precision": 30.4,
+            "recall": 63.3,
+            "f1": 41.1,
+            "f2": 52.1,
+            "tp": 88,
+            "fp": 201,
+            "fn": 51,
             "num_prs": 50
           },
           "core": {
-            "precision": 54.5,
-            "recall": 15.2,
-            "f1": 23.8,
-            "f2": 17.8,
-            "tp": 24,
-            "fp": 20,
-            "fn": 134,
+            "precision": 33.2,
+            "recall": 63.3,
+            "f1": 43.6,
+            "f2": 53.6,
+            "tp": 100,
+            "fp": 201,
+            "fn": 58,
             "num_prs": 50
           },
           "all": {
-            "precision": 55.6,
-            "recall": 14.5,
-            "f1": 22.9,
-            "f2": 17.0,
-            "tp": 25,
-            "fp": 20,
-            "fn": 148,
-            "num_prs": 50
-          }
-        },
-        "qodo-extended-v2": {
-          "strict": {
-            "precision": 55.9,
-            "recall": 61.2,
-            "f1": 58.4,
-            "f2": 60.0,
-            "tp": 85,
-            "fp": 67,
-            "fn": 54,
-            "num_prs": 50
-          },
-          "core": {
-            "precision": 58.4,
-            "recall": 59.5,
-            "f1": 58.9,
-            "f2": 59.3,
-            "tp": 94,
-            "fp": 67,
+            "precision": 35.2,
+            "recall": 63.0,
+            "f1": 45.1,
+            "f2": 54.4,
+            "tp": 109,
+            "fp": 201,
             "fn": 64,
             "num_prs": 50
-          },
-          "all": {
-            "precision": 58.9,
-            "recall": 55.5,
-            "f1": 57.1,
-            "f2": 56.1,
-            "tp": 96,
-            "fp": 67,
-            "fn": 77,
-            "num_prs": 50
-          }
-        },
-        "kodus-v2": {
-          "strict": {
-            "precision": 50.5,
-            "recall": 36.0,
-            "f1": 42.0,
-            "f2": 38.2,
-            "tp": 50,
-            "fp": 49,
-            "fn": 89,
-            "num_prs": 50
-          },
-          "core": {
-            "precision": 52.4,
-            "recall": 34.2,
-            "f1": 41.4,
-            "f2": 36.7,
-            "tp": 54,
-            "fp": 49,
-            "fn": 104,
-            "num_prs": 50
-          },
-          "all": {
-            "precision": 53.8,
-            "recall": 32.9,
-            "f1": 40.9,
-            "f2": 35.7,
-            "tp": 57,
-            "fp": 49,
-            "fn": 116,
-            "num_prs": 50
-          }
-        },
-        "graphite": {
-          "strict": {
-            "precision": 71.4,
-            "recall": 7.2,
-            "f1": 13.1,
-            "f2": 8.8,
-            "tp": 10,
-            "fp": 4,
-            "fn": 129,
-            "num_prs": 50
-          },
-          "core": {
-            "precision": 73.3,
-            "recall": 7.0,
-            "f1": 12.7,
-            "f2": 8.5,
-            "tp": 11,
-            "fp": 4,
-            "fn": 147,
-            "num_prs": 50
-          },
-          "all": {
-            "precision": 73.3,
-            "recall": 6.4,
-            "f1": 11.7,
-            "f2": 7.8,
-            "tp": 11,
-            "fp": 4,
-            "fn": 162,
-            "num_prs": 50
           }
         }
       }
@@ -56148,7 +57031,8 @@
     "gitlab": "GitLab Duo Code Review",
     "qodo-v2": "Qodo v2",
     "qodo-extended-v2": "Qodo Extended",
-    "macroscope": "Macroscope"
+    "macroscope": "Macroscope",
+    "shipwright": "Shipwright"
   },
   "tool_colors": {
     "graphite": "#6366f1",
@@ -56177,7 +57061,8 @@
     "qodo-v2": "#7c3aed",
     "qodo-extended-v2": "#6d28d9",
     "macroscope": "#0891b2",
-    "cubic-v2": "#c026d3"
+    "cubic-v2": "#c026d3",
+    "shipwright": "#2563eb"
   },
   "default_model": "anthropic_claude-opus-4-5-20251101",
   "min_prs_threshold": 5,
diff --git a/offline/analysis/benchmark_dashboard.py b/offline/analysis/benchmark_dashboard.py
index 559ba8c..b891a30 100644
--- a/offline/analysis/benchmark_dashboard.py
+++ b/offline/analysis/benchmark_dashboard.py
@@ -162,6 +162,7 @@ def _compute_profile_metrics(
     "qodo-extended-v2": "Qodo Extended",
     "macroscope": "Macroscope",
     "cubic-v2": "Cubic v2",
+    "shipwright": "Shipwright",
 }
 
 TOOL_COLORS = {
@@ -192,6 +193,7 @@ def _compute_profile_metrics(
     "qodo-extended-v2": "#6d28d9",
     "macroscope": "#0891b2",
     "cubic-v2": "#c026d3",
+    "shipwright": "#2563eb",
 }
 
 
diff --git a/offline/results/anthropic_claude-opus-4-5-20251101/candidates.json b/offline/results/anthropic_claude-opus-4-5-20251101/candidates.json
index 6a6aea9..1c97d76 100644
--- a/offline/results/anthropic_claude-opus-4-5-20251101/candidates.json
+++ b/offline/results/anthropic_claude-opus-4-5-20251101/candidates.json
@@ -1256,6 +1256,35 @@
         "line": null,
         "source": "extracted"
       }
+    ],
+    "shipwright": [
+      {
+        "check_id": "benchmark.review",
+        "kind": "security",
+        "line": 448,
+        "path": "services/src/main/java/org/keycloak/services/resources/admin/UsersResource.java",
+        "severity": "critical",
+        "source": "shipwright.review",
+        "text": "In UsersResource.searchForUser(), the previous `.filter(usersEvaluator::canView)` post-filter on the searched user stream was removed. Now, if an admin has no `canView()` (users) and no group-based view permission (getGroupIdsWithViewPermission() returns an empty set, which happens whenever the admin has zero group-level view/manage-members grants, e.g. when access is granted via a user-specific permission instead of a group permission), the `session.setAttribute(UserModel.GROUPS, ...)` restriction is never applied and the raw, unfiltered `searchForUserStream` result is returned to the caller with no per-user authorization check at all. This is an authorization bypass: such an admin will see every user in the realm instead of only the users they are permitted to view. Location: services/src/main/java/org/keycloak/services/resources/admin/UsersResource.java:448"
+      },
+      {
+        "check_id": "benchmark.review",
+        "kind": "security",
+        "line": 56,
+        "path": "services/src/main/java/org/keycloak/services/resources/admin/permissions/GroupPermissionsV2.java",
+        "severity": "critical",
+        "source": "shipwright.review",
+        "text": "In the new GroupPermissionsV2, the global (no-arg) canManage() incorrectly evaluates `hasPermission(null, AdminPermissionsSchema.VIEW, AdminPermissionsSchema.MANAGE)` \u2014 identical to canView(). This means an admin who only holds the VIEW scope permission on the 'all groups' resource (and not MANAGE) will have canManage() return true, which is used by requireManage()/GroupsResource create-group and other manage-gated endpoints, granting create/update/delete capability on groups to a user who was only granted read access. The per-group overload canManage(GroupModel) correctly checks only AdminPermissionsSchema.MANAGE, confirming the global variant is an introduced copy/paste bug. Location: services/src/main/java/org/keycloak/services/resources/admin/permissions/GroupPermissionsV2.java:56"
+      },
+      {
+        "check_id": "benchmark.review",
+        "kind": "bug",
+        "line": 102,
+        "path": "services/src/main/java/org/keycloak/services/resources/admin/permissions/GroupPermissionsV2.java",
+        "severity": "critical",
+        "source": "shipwright.review",
+        "text": "In GroupPermissionsV2.getGroupIdsWithViewPermission(), the code iterates Groups-type Resources and adds `groupResource.getId()` to the granted set. However, per AdminPermissionsSchema.getOrCreateResource()/resolveGroup() and this same class's hasPermission() (which looks up group resources via `resourceStore.findByName(server, groupId)`), the actual group ID is stored as the Resource's *name*, not its internal store id. Collecting `.getId()` instead of `.getName()` returns internal authorization-resource UUIDs that never match any real GroupModel id. The resulting set is later used as `session.setAttribute(UserModel.GROUPS, groupIds)` to restrict user search results by group membership, so this bug makes the entire group-based partial-view feature non-functional (it will filter against nonexistent group ids, effectively returning no users instead of the intended restricted set). Location: services/src/main/java/org/keycloak/services/resources/admin/permissions/GroupPermissionsV2.java:102"
+      }
     ]
   },
   "https://github.com/keycloak/keycloak/pull/33832": {
@@ -2521,6 +2550,35 @@
         "line": null,
         "source": "extracted"
       }
+    ],
+    "shipwright": [
+      {
+        "check_id": "benchmark.review",
+        "kind": "security",
+        "line": 58,
+        "path": "common/src/main/java/org/keycloak/common/crypto/CryptoIntegration.java",
+        "severity": "high",
+        "source": "shipwright.review",
+        "text": "CryptoIntegration.detectProvider() previously threw an IllegalStateException whenever more than one CryptoProvider was found on the classpath, which acted as a fail-fast guard against classpath misconfiguration (e.g., both FIPS and non-FIPS crypto providers present). The new code sorts by order() and silently picks the first, only logging at debug level. Since DefaultCryptoProvider, WildFlyElytronProvider and FIPS1402Provider are all now assigned the same order() value (200), if more than one of these 'real' providers ends up on the classpath together (the exact misconfiguration the old check protected against), the selection between them becomes non-deterministic and only surfaces via a debug log line, silently risking selection of the wrong (e.g. non-FIPS) crypto provider instead of failing loudly as before. Location: common/src/main/java/org/keycloak/common/crypto/CryptoIntegration.java:58"
+      },
+      {
+        "check_id": "benchmark.review",
+        "kind": "bug",
+        "line": 68,
+        "path": "common/src/main/java/org/keycloak/common/crypto/CryptoIntegration.java",
+        "severity": "low",
+        "source": "shipwright.review",
+        "text": "The ignored-providers message is built dynamically from provider class names and passed directly as the format string to logger.debugf(String), which interprets its argument as a printf-style format string. If any loaded provider's class/package name were to contain a '%' character, this would throw an IllegalFormatException at runtime instead of just logging a message. Location: common/src/main/java/org/keycloak/common/crypto/CryptoIntegration.java:68"
+      },
+      {
+        "check_id": "benchmark.review",
+        "kind": "doc_defect",
+        "line": 45,
+        "path": "authz/client/src/main/java/org/keycloak/authorization/client/util/crypto/AuthzClientCryptoProvider.java",
+        "severity": "medium",
+        "source": "shipwright.review",
+        "text": "AuthzClientCryptoProvider.getBouncyCastleProvider() is documented/named to return the Bouncy Castle security Provider (per the CryptoProvider interface contract used by other implementations, which return BouncyIntegration.PROVIDER), but this implementation instead returns the Provider of the default KeyStore type (e.g. JKS/PKCS12's provider), which is unrelated to Bouncy Castle. This is misleading and functionally incorrect relative to the method's name/contract. Location: authz/client/src/main/java/org/keycloak/authorization/client/util/crypto/AuthzClientCryptoProvider.java:45"
+      }
     ]
   },
   "https://github.com/keycloak/keycloak/pull/40940": {
@@ -3519,6 +3577,26 @@
         "line": null,
         "source": "extracted"
       }
+    ],
+    "shipwright": [
+      {
+        "check_id": "benchmark.review",
+        "kind": "test_gap",
+        "line": 154,
+        "path": "tests/base/src/test/java/org/keycloak/tests/admin/group/GroupTest.java",
+        "severity": "medium",
+        "source": "shipwright.review",
+        "text": "The background reader thread loop `while (!deletedAll.get())` is never joined before the test asserts `caughtExceptions` is empty. Since `deletedAll.set(true)` is called immediately followed by the assertion, the thread may still be mid-iteration and catch (or fail to catch and report) an exception after the check has already passed, making the test racy and able to silently miss the very NPE it is meant to detect. Location: tests/base/src/test/java/org/keycloak/tests/admin/group/GroupTest.java:154"
+      },
+      {
+        "check_id": "benchmark.review",
+        "kind": "concurrency",
+        "line": 149,
+        "path": "tests/base/src/test/java/org/keycloak/tests/admin/group/GroupTest.java",
+        "severity": "medium",
+        "source": "shipwright.review",
+        "text": "The reader thread's termination flag `deletedAll` is only set to true after the `groupUuuids.forEach(...).remove()` loop completes without exception. If any group deletion throws (e.g. network error, 404, etc.), `deletedAll.set(true)` is never reached, leaving the background thread in an infinite polling loop for the lifetime of the test JVM (no try/finally to guarantee thread shutdown), causing a thread/resource leak and continued unwanted server calls after the test fails. Location: tests/base/src/test/java/org/keycloak/tests/admin/group/GroupTest.java:149"
+      }
     ]
   },
   "https://github.com/ai-code-review-evaluation/keycloak-greptile/pull/1": {
@@ -4521,6 +4599,26 @@
         "line": null,
         "source": "extracted"
       }
+    ],
+    "shipwright": [
+      {
+        "check_id": "benchmark.review",
+        "kind": "style",
+        "line": 204,
+        "path": "testsuite/integration-arquillian/tests/base/src/test/java/org/keycloak/testsuite/webauthn/passwordless/PasskeysOrganizationAuthenticationTest.java",
+        "severity": "low",
+        "source": "shipwright.review",
+        "text": "Both usages of `Assert.assertThrows(NoSuchElementException.class, ...)` were removed in this diff (replaced with `assertThat(...).notNullValue()`), but the `NoSuchElementException` import is not removed. If this was the only usage of that class in the file, the import becomes unused, which will fail Keycloak's strict checkstyle build (no-unused-imports rule). Location: testsuite/integration-arquillian/tests/base/src/test/java/org/keycloak/testsuite/webauthn/passwordless/PasskeysOrganizationAuthenticationTest.java:204"
+      },
+      {
+        "check_id": "benchmark.review",
+        "kind": "style",
+        "line": 36,
+        "path": "testsuite/integration-arquillian/tests/base/src/test/java/org/keycloak/testsuite/webauthn/passwordless/PasskeysUsernamePasswordFormTest.java",
+        "severity": "low",
+        "source": "shipwright.review",
+        "text": "`import org.keycloak.models.credential.PasswordCredentialModel;` was added to this test file, but the diff does not show any usage of `PasswordCredentialModel` anywhere in the file's added/changed code (unlike the sibling test `PasskeysUsernameFormTest`, where the same import is actually used to filter credentials). This looks like an unused import copied from the other test class, which would fail an unused-import checkstyle check. Location: testsuite/integration-arquillian/tests/base/src/test/java/org/keycloak/testsuite/webauthn/passwordless/PasskeysUsernamePasswordFormTest.java:36"
+      }
     ]
   },
   "https://github.com/getsentry/sentry/pull/93824": {
@@ -5583,6 +5681,44 @@
         "line": null,
         "source": "extracted"
       }
+    ],
+    "shipwright": [
+      {
+        "check_id": "benchmark.review",
+        "kind": "concurrency",
+        "line": 345,
+        "path": "src/sentry/spans/consumers/process/flusher.py",
+        "severity": "high",
+        "source": "shipwright.review",
+        "text": "In SpanFlusher.join(), the per-process wait/terminate loop now does `break` as soon as the deadline is exceeded before entering the wait-loop for a given process. This exits the entire `for process_index, process in self.processes.items()` loop, so `process.terminate()` is never called for the current process nor for any remaining processes in the dict. The original single-process implementation always called `self.process.terminate()` unconditionally after the wait loop regardless of deadline status. This change can leave orphaned multiprocessing child processes running after `join()` returns when a timeout is supplied and multiple flusher processes exist. Location: src/sentry/spans/consumers/process/flusher.py:345"
+      },
+      {
+        "check_id": "benchmark.review",
+        "kind": "bug",
+        "line": 57,
+        "path": "src/sentry/spans/consumers/process/flusher.py",
+        "severity": "low",
+        "source": "shipwright.review",
+        "text": "`self.max_processes = max_processes or len(buffer.assigned_shards)` treats an explicitly passed `max_processes=0` the same as `None` (falls back to shard count) instead of honoring the literal value 0, due to using `or` instead of an explicit `is None` check. Location: src/sentry/spans/consumers/process/flusher.py:57"
+      },
+      {
+        "check_id": "benchmark.review",
+        "kind": "data",
+        "line": 190,
+        "path": "src/sentry/spans/consumers/process/flusher.py",
+        "severity": "low",
+        "source": "shipwright.review",
+        "text": "Metric tag keys for shard grouping are inconsistent within the same `main()` method: `spans.buffer.flusher.produce` and `spans.buffer.segment_size_bytes` use tag key `shard` (holding a comma-joined multi-shard string), while `spans.buffer.flusher.wait_produce` uses tag key `shards` for the same comma-joined value, and `spans.buffer.flusher_unhealthy` uses `shard` with a single integer value per shard. This inconsistency produces mismatched/incompatible metric labels for what should be the same dimension across the same code path. Location: src/sentry/spans/consumers/process/flusher.py:190"
+      },
+      {
+        "check_id": "benchmark.review",
+        "kind": "test_gap",
+        "line": 63,
+        "path": "tests/sentry/spans/consumers/process/test_consumer.py",
+        "severity": "low",
+        "source": "shipwright.review",
+        "text": "The added `time.sleep(0.1)` call intended to \"Give flusher threads time to process after drift change\" is a no-op because the test earlier does `monkeypatch.setattr(\"time.sleep\", lambda _: None)`, which patches the global `time.sleep` used by the newly added `import time` in this test file as well. The comment is therefore misleading and the wait does not actually occur, making the test rely on timing that isn't actually being introduced. Location: tests/sentry/spans/consumers/process/test_consumer.py:63"
+      }
     ]
   },
   "https://github.com/ai-code-review-evaluation/sentry-greptile/pull/5": {
@@ -7304,6 +7440,35 @@
         "line": null,
         "source": "extracted"
       }
+    ],
+    "shipwright": [
+      {
+        "check_id": "benchmark.review",
+        "kind": "bug",
+        "line": 135,
+        "path": "src/sentry/replays/endpoints/project_replay_summarize_breadcrumbs.py",
+        "severity": "high",
+        "source": "shipwright.review",
+        "text": "In fetch_error_details, error_ids are paired with nodestore results via `zip(error_ids, events.values())`. `nodestore.backend.get_multi(node_ids)` is not guaranteed to return a dict whose values() are in the same order as the input error_ids (it may omit missing entries or reorder), so this positional zip can associate the wrong error_id/title/message/timestamp with an event, corrupting the error context fed into the AI summarizer. Location: src/sentry/replays/endpoints/project_replay_summarize_breadcrumbs.py:135"
+      },
+      {
+        "check_id": "benchmark.review",
+        "kind": "bug",
+        "line": 52,
+        "path": "src/sentry/issues/endpoints/browser_reporting_collector.py",
+        "severity": "medium",
+        "source": "shipwright.review",
+        "text": "validate_timestamp/validate_age use truthy checks (`self.initial_data.get(\"age\")` / `self.initial_data.get(\"timestamp\")`) to detect whether the other field is present. Since 0 is a valid timestamp/age value (timestamp has min_value=0), a report where both age and timestamp are 0 will bypass the mutual-exclusion validation entirely (both checks evaluate falsy), letting an invalid mixed report through undetected. Location: src/sentry/issues/endpoints/browser_reporting_collector.py:52"
+      },
+      {
+        "check_id": "benchmark.review",
+        "kind": "bug",
+        "line": 80,
+        "path": "src/sentry/preprod/api/endpoints/organization_preprod_artifact_assemble.py",
+        "severity": "low",
+        "source": "shipwright.review",
+        "text": "analytics.record(\"preprod_artifact.api.assemble\", ...) is emitted before the `features.has(\"organizations:preprod-artifact-assemble\", ...)` guard, so the analytics event fires even for organizations without the feature enabled (or otherwise rejected requests), inflating/skewing analytics with events for requests that never actually assemble an artifact. Location: src/sentry/preprod/api/endpoints/organization_preprod_artifact_assemble.py:80"
+      }
     ]
   },
   "https://github.com/ai-code-review-evaluation/sentry-greptile/pull/1": {
@@ -8758,6 +8923,53 @@
         "line": null,
         "source": "extracted"
       }
+    ],
+    "shipwright": [
+      {
+        "check_id": "benchmark.review",
+        "kind": "bug",
+        "line": 875,
+        "path": "src/sentry/api/paginator.py",
+        "severity": "high",
+        "source": "shipwright.review",
+        "text": "OptimizedCursorPaginator's negative-offset branch (triggered when enable_advanced_features and cursor.offset<0) slices a Django QuerySet with a negative start index (`queryset[start_offset:stop]`). Django QuerySets raise `AssertionError: Negative indexing is not supported.` for negative slice bounds (unlike Python lists), so this 'advanced pagination' code path will always crash at runtime rather than working as documented. Location: src/sentry/api/paginator.py:875"
+      },
+      {
+        "check_id": "benchmark.review",
+        "kind": "doc_defect",
+        "line": 872,
+        "path": "src/sentry/api/paginator.py",
+        "severity": "medium",
+        "source": "shipwright.review",
+        "text": "The comment/docstring claims 'The underlying Django ORM properly handles negative slicing automatically' and 'This is safe because permissions are checked at the queryset level', but no such queryset-level restriction is added and Django querysets do not support negative slicing at all \u2014 the documented safety/behavior is false and misleading for anyone maintaining this code. Location: src/sentry/api/paginator.py:872"
+      },
+      {
+        "check_id": "benchmark.review",
+        "kind": "security",
+        "line": 73,
+        "path": "src/sentry/api/endpoints/organization_auditlogs.py",
+        "severity": "medium",
+        "source": "shipwright.review",
+        "text": "New endpoint branch gates an 'advanced' pagination mode (which purportedly allows negative offsets/out-of-bounds cursor navigation on audit logs) behind only `request.user.is_superuser or organization_context.member.has_global_access`, with no additional queryset scoping added to prevent the negative-offset path from returning records outside the intended audit-log window for the organization. Combined with the broken/undocumented negative-offset semantics, this is a risky, under-verified authorization surface for a security-sensitive (audit log) endpoint. Location: src/sentry/api/endpoints/organization_auditlogs.py:73"
+      },
+      {
+        "check_id": "benchmark.review",
+        "kind": "bug",
+        "line": 74,
+        "path": "src/sentry/api/endpoints/organization_auditlogs.py",
+        "severity": "medium",
+        "source": "shipwright.review",
+        "text": "`organization_context.member` can be None for a superuser accessing the organization without being an actual member; because Python evaluates `or` short-circuit only when the left operand is falsy, for a non-superuser request this expression dereferences `.has_global_access` on `organization_context.member`, which will raise AttributeError if `member` is None, crashing the endpoint instead of falling back to normal pagination. Location: src/sentry/api/endpoints/organization_auditlogs.py:74"
+      },
+      {
+        "check_id": "benchmark.review",
+        "kind": "bug",
+        "line": 179,
+        "path": "src/sentry/api/paginator.py",
+        "severity": "medium",
+        "source": "shipwright.review",
+        "text": "In BasePaginator.get_result, offset clamping is only applied for the non-prev branch (`start_offset = max(0, offset) if not cursor.is_prev else offset`); when `cursor.is_prev` is True the original (potentially negative) offset is passed straight into `queryset[start_offset:stop]` unclamped, producing inconsistent behavior \u2014 forward pagination silently clamps a negative offset to 0, while backward (prev) pagination with the same negative offset will hit the same negative-slice failure that this change intended to avoid. Location: src/sentry/api/paginator.py:179"
+      }
     ]
   },
   "https://github.com/grafana/grafana/pull/97529": {
@@ -9707,6 +9919,17 @@
         "line": null,
         "source": "extracted"
       }
+    ],
+    "shipwright": [
+      {
+        "check_id": "benchmark.review",
+        "kind": "concurrency",
+        "line": 134,
+        "path": "pkg/storage/unified/search/bleve.go",
+        "severity": "high",
+        "source": "shipwright.review",
+        "text": "BuildIndex no longer holds cacheMu for the duration of building an index; the lock is only acquired right before writing to b.cache at the end. This allows two concurrent BuildIndex calls for the same key to both build a full bleve index (including a file-backed bleve.New(dir, mapper) call) in parallel; whichever finishes last overwrites b.cache[key], silently orphaning the other freshly-built index (and its open file handles for file-based indexes) since nothing closes the discarded index. This introduces a race condition and a resource leak that did not exist when the whole function was serialized by the mutex. Location: pkg/storage/unified/search/bleve.go:134"
+      }
     ]
   },
   "https://github.com/getsentry/sentry/pull/80168": {
@@ -10608,6 +10831,26 @@
         "line": null,
         "source": "extracted"
       }
+    ],
+    "shipwright": [
+      {
+        "check_id": "benchmark.review",
+        "kind": "bug",
+        "line": 9,
+        "path": "src/sentry/incidents/grouptype.py",
+        "severity": "high",
+        "source": "shipwright.review",
+        "text": "`MetricAlertDetectorHandler` was changed to subclass `StatefulDetectorHandler`, which now declares three abstract methods (`get_dedupe_value`, `get_group_key_values`, and the newly-added `build_occurrence_and_event_data`), but the class body is just `pass`. Since these abstract methods are not implemented, instantiating `MetricAlertDetectorHandler` (e.g. via `Detector.detector_handler` which does `group_type.detector_handler(self)`) will raise `TypeError: Can't instantiate abstract class MetricAlertDetectorHandler with abstract methods ...`. Previously this class was a concrete no-op handler; now it is broken and unusable at runtime. Location: src/sentry/incidents/grouptype.py:9"
+      },
+      {
+        "check_id": "benchmark.review",
+        "kind": "doc_defect",
+        "line": 229,
+        "path": "src/sentry/workflow_engine/processors/detector.py",
+        "severity": "low",
+        "source": "shipwright.review",
+        "text": "The `evaluate` method's return type was changed from `list[DetectorEvaluationResult]` to `dict[DetectorGroupKey, DetectorEvaluationResult]`, but the docstring was not updated and still states 'Evaluates a given data packet and returns a list of `DetectorEvaluationResult`', which now inaccurately describes the actual return type. Location: src/sentry/workflow_engine/processors/detector.py:229"
+      }
     ]
   },
   "https://github.com/getsentry/sentry/pull/80528": {
@@ -11273,7 +11516,8 @@
         "line": null,
         "source": "extracted"
       }
-    ]
+    ],
+    "shipwright": []
   },
   "https://github.com/getsentry/sentry/pull/77754": {
     "gemini": [
@@ -12075,6 +12319,35 @@
         "line": null,
         "source": "extracted"
       }
+    ],
+    "shipwright": [
+      {
+        "check_id": "benchmark.review",
+        "kind": "bug",
+        "line": 16,
+        "path": "src/sentry/integrations/services/assignment_source.py",
+        "severity": "medium",
+        "source": "shipwright.review",
+        "text": "`AssignmentSource.queued: datetime = timezone.now()` is a dataclass field default that is evaluated exactly once, at class/module import time, not per-instance construction. Every `AssignmentSource` created without explicitly passing `queued` (e.g. via `AssignmentSource.from_integration()`) will get the same stale timestamp (process start time) instead of the actual time the assignment was queued, defeating the purpose of the field. Location: src/sentry/integrations/services/assignment_source.py:16"
+      },
+      {
+        "check_id": "benchmark.review",
+        "kind": "data",
+        "line": 140,
+        "path": "src/sentry/integrations/utils/sync.py",
+        "severity": "medium",
+        "source": "shipwright.review",
+        "text": "`AssignmentSource.to_dict()` uses `asdict()`, which serializes the `queued` field as a raw `datetime` object (not an ISO string). This dict is then passed directly as celery task kwargs in `sync_group_assignee_outbound` (`assignment_source.to_dict()` -> `apply_async(kwargs={...})`), which will fail to serialize with the default JSON task serializer since `datetime` objects are not JSON serializable. Location: src/sentry/integrations/utils/sync.py:140"
+      },
+      {
+        "check_id": "benchmark.review",
+        "kind": "doc_defect",
+        "line": 7,
+        "path": "tests/sentry/integrations/services/test_assignment_source.py",
+        "severity": "low",
+        "source": "shipwright.review",
+        "text": "Test method `test_from_dict_empty_array` actually tests an empty dict (`{}`), not an array; the name is misleading relative to what the test body exercises. Location: tests/sentry/integrations/services/test_assignment_source.py:7"
+      }
     ]
   },
   "https://github.com/getsentry/sentry/pull/95633": {
@@ -13108,6 +13381,26 @@
         "line": null,
         "source": "extracted"
       }
+    ],
+    "shipwright": [
+      {
+        "check_id": "benchmark.review",
+        "kind": "concurrency",
+        "line": 330,
+        "path": "src/sentry/remote_subscriptions/consumers/queue_consumer.py",
+        "severity": "critical",
+        "source": "shipwright.review",
+        "text": "SimpleQueueProcessingStrategy.close() and terminate() both call self.queue_pool.shutdown(), which is the FixedQueuePool instance shared and persisted at the factory level (self.queue_pool is created once in ResultsStrategyFactory.__init__ and reused across every create_with_partitions() call). Arroyo calls close()/terminate() on the strategy each time partitions are revoked/rebalanced, not just on final consumer shutdown. This permanently kills all queue worker threads and shuts down the queues (queue.shutdown()) after the very first rebalance, but create_thread_queue_parallel_worker() will still hand out the same (now-dead) queue_pool to any subsequently created strategy, since only ResultsStrategyFactory.shutdown() sets queue_pool to None. Later messages submitted to the dead pool will raise on work_queue.put(), be caught by the broad except in submit(), and have their offsets immediately marked complete/committed even though they were never actually processed by result_processor \u2014 causing silent message/data loss after any partition reassignment. Location: src/sentry/remote_subscriptions/consumers/queue_consumer.py:330"
+      },
+      {
+        "check_id": "benchmark.review",
+        "kind": "bug",
+        "line": 128,
+        "path": "src/sentry/remote_subscriptions/consumers/queue_consumer.py",
+        "severity": "high",
+        "source": "shipwright.review",
+        "text": "OrderedQueueWorker.run() and FixedQueuePool.shutdown() rely on queue.ShutDown and queue.Queue.shutdown(), which were only added to the Python standard library in Python 3.13. If the codebase targets an earlier Python version (as is typical for this backend), referencing queue.ShutDown will raise AttributeError at runtime the first time a worker thread's get() call or the pool's shutdown() method executes, breaking the entire thread-queue-parallel mode. Location: src/sentry/remote_subscriptions/consumers/queue_consumer.py:128"
+      }
     ]
   },
   "https://github.com/ai-code-review-evaluation/sentry-greptile/pull/2": {
@@ -14767,6 +15060,62 @@
         "line": null,
         "source": "extracted"
       }
+    ],
+    "shipwright": [
+      {
+        "check_id": "benchmark.review",
+        "kind": "bug",
+        "line": 75,
+        "path": "src/sentry/api/endpoints/organization_auditlogs.py",
+        "severity": "critical",
+        "source": "shipwright.review",
+        "text": "`enable_advanced = request.user.is_superuser or organization_context.member.has_global_access` is executed unconditionally for every request (not only when `use_optimized` is true), and `organization_context` is never defined/imported anywhere in this diff or method. For any non-superuser request this raises a NameError, breaking the entire audit log endpoint. Location: src/sentry/api/endpoints/organization_auditlogs.py:75"
+      },
+      {
+        "check_id": "benchmark.review",
+        "kind": "bug",
+        "line": 895,
+        "path": "src/sentry/api/paginator.py",
+        "severity": "high",
+        "source": "shipwright.review",
+        "text": "When `enable_advanced_features` is set and `cursor.offset < 0`, the code does `queryset[start_offset:stop]` with a negative `start_offset` directly on a Django QuerySet. Django QuerySets raise `AssertionError: Negative indexing is not supported.` for negative slice indices, so this branch will always crash at runtime, directly contradicting the comment claiming 'the underlying Django ORM properly handles negative slicing automatically'. Location: src/sentry/api/paginator.py:895"
+      },
+      {
+        "check_id": "benchmark.review",
+        "kind": "data",
+        "line": 446,
+        "path": "src/sentry/spans/buffer.py",
+        "severity": "medium",
+        "source": "shipwright.review",
+        "text": "The per-key `len(payloads[key]) > self.max_segment_spans` guard that enforced a configurable maximum segment span count was removed entirely from the Python flush path, with enforcement now relying solely on the hardcoded `1000` eviction threshold in add-buffer.lua. If `self.max_segment_spans` is configured to a different value elsewhere in the class, the effective cap silently diverges from the configured value and the config becomes dead/ineffective. Location: src/sentry/spans/buffer.py:446"
+      },
+      {
+        "check_id": "benchmark.review",
+        "kind": "doc_defect",
+        "line": 30,
+        "path": "src/sentry/scripts/spans/add-buffer.lua",
+        "severity": "medium",
+        "source": "shipwright.review",
+        "text": "The redirect-chasing loop limit was reduced from 10000 to 1000 iterations, and the explanatory comment describing the correctness risk of hitting this limit ('segment trees of depth 10k may not be joined together correctly') was deleted rather than updated, obscuring that deep segment trees are now 10x more likely to fail to join correctly. Location: src/sentry/scripts/spans/add-buffer.lua:30"
+      },
+      {
+        "check_id": "benchmark.review",
+        "kind": "bug",
+        "line": 142,
+        "path": "src/sentry/spans/consumers/process/factory.py",
+        "severity": "medium",
+        "source": "shipwright.review",
+        "text": "`val[\"end_timestamp_precise\"]` uses direct dict indexing (unlike the other optional-like fields such as `parent_span_id` which use `.get()`), so any incoming span message that lacks this new field (e.g. during schema rollout or from producers not yet updated) will raise a KeyError and crash batch processing instead of being handled gracefully. Location: src/sentry/spans/consumers/process/factory.py:142"
+      },
+      {
+        "check_id": "benchmark.review",
+        "kind": "doc_defect",
+        "line": 179,
+        "path": "src/sentry/api/paginator.py",
+        "severity": "low",
+        "source": "shipwright.review",
+        "text": "The comment claims this change 'allow[s] negative offsets to enable efficient bidirectional pagination', but the code actually clamps offset to `max(0, offset)` for the non-prev branch, meaning negative offsets are neutralized rather than enabled; the comment misdescribes the actual behavior. Location: src/sentry/api/paginator.py:179"
+      }
     ]
   },
   "https://github.com/ai-code-review-evaluation/sentry-greptile/pull/3": {
@@ -16208,6 +16557,44 @@
         "line": null,
         "source": "extracted"
       }
+    ],
+    "shipwright": [
+      {
+        "check_id": "benchmark.review",
+        "kind": "security",
+        "line": 1041,
+        "path": "src/sentry/search/events/datasets/discover.py",
+        "severity": "high",
+        "source": "shipwright.review",
+        "text": "The new `upsampled_count` SnQLFunction lacks `private=True`, unlike the immediately preceding SnQLFunction definition in the same list. This means any user query can call `upsampled_count()` directly (e.g. via yAxis=upsampled_count()), completely bypassing the `is_errors_query_for_error_upsampled_projects` allowlist/eligibility gating that the rest of this PR relies on to restrict sample_weight-based aggregation to allowlisted error-upsampled projects. Location: src/sentry/search/events/datasets/discover.py:1041"
+      },
+      {
+        "check_id": "benchmark.review",
+        "kind": "doc_defect",
+        "line": 34,
+        "path": "src/sentry/api/helpers/error_upsampling.py",
+        "severity": "low",
+        "source": "shipwright.review",
+        "text": "The docstring in `_are_all_projects_error_upsampled` states the function 'reads the allowlist configuration fresh each time... to ensure we always have the latest configuration state,' but this is contradicted by the caller `is_errors_query_for_error_upsampled_projects`, which wraps this exact check in a 60-second cache and skips calling it entirely on cache hits. The documentation is misleading about actual staleness/consistency behavior introduced in the same file. Location: src/sentry/api/helpers/error_upsampling.py:34"
+      },
+      {
+        "check_id": "benchmark.review",
+        "kind": "bug",
+        "line": 356,
+        "path": "src/sentry/testutils/factories.py",
+        "severity": "medium",
+        "source": "shipwright.review",
+        "text": "`if client_sample_rate:` treats a legitimate falsy sample rate value of `0` (or `0.0`) as absent, so `sample_rate` will never be set on the normalized event data when client_sample_rate is exactly 0, silently dropping a valid (if edge-case) value instead of using an explicit `is not None` check. Location: src/sentry/testutils/factories.py:356"
+      },
+      {
+        "check_id": "benchmark.review",
+        "kind": "bug",
+        "line": 346,
+        "path": "src/sentry/testutils/factories.py",
+        "severity": "low",
+        "source": "shipwright.review",
+        "text": "Two bare `except Exception: pass` blocks silently swallow any error while reading nested context data or converting to float, masking programming errors (e.g. malformed contexts dict) in this newly added test-data helper without any logging, making failures in event construction hard to diagnose. Location: src/sentry/testutils/factories.py:346"
+      }
     ]
   },
   "https://github.com/grafana/grafana/pull/103633": {
@@ -17179,6 +17566,17 @@
         "line": null,
         "source": "extracted"
       }
+    ],
+    "shipwright": [
+      {
+        "check_id": "benchmark.review",
+        "kind": "data",
+        "line": 135,
+        "path": "pkg/services/authz/rbac/service.go",
+        "severity": "low",
+        "source": "shipwright.review",
+        "text": "In Service.Check, when getCachedIdentityPermissions returns a valid (cache-hit) permission map but checkPermission evaluates to false, the code falls through and unconditionally increments s.metrics.permissionCacheUsage.WithLabelValues(\"false\", ...) even though the permission cache was actually hit (err == nil). This mislabels a cache-hit-but-denied case as a cache miss, corrupting the permissionCacheUsage metric used to measure cache effectiveness. Location: pkg/services/authz/rbac/service.go:135"
+      }
     ]
   },
   "https://github.com/getsentry/sentry/pull/67876": {
@@ -18293,6 +18691,26 @@
         "line": null,
         "source": "extracted"
       }
+    ],
+    "shipwright": [
+      {
+        "check_id": "benchmark.review",
+        "kind": "bug",
+        "line": 505,
+        "path": "src/sentry/integrations/github/integration.py",
+        "severity": "high",
+        "source": "shipwright.review",
+        "text": "In GitHubInstallation.dispatch, the new authentication check does `integration.metadata[\"sender\"][\"login\"]` with direct dict indexing and no guard for missing keys. The `sender` key in metadata is only populated once the GitHub installation webhook has been processed; the OAuth callback (which drives this pipeline step) can arrive before that webhook completes. If the key is missing, this raises an unhandled KeyError/500 instead of the intended graceful 'Invalid installation request' error page, defeating the purpose of the new security check. Location: src/sentry/integrations/github/integration.py:505"
+      },
+      {
+        "check_id": "benchmark.review",
+        "kind": "concurrency",
+        "line": 438,
+        "path": "src/sentry/integrations/github/integration.py",
+        "severity": "medium",
+        "source": "shipwright.review",
+        "text": "OAuthLoginView.dispatch wraps the token-exchange response parsing in a bare `except Exception: payload = {}`, silently swallowing all errors (network failures, malformed responses, decode errors) with no logging. This masks legitimate failures behind a generic 'Invalid installation request' message, making it impossible to diagnose real integration/token-exchange problems from logs. Location: src/sentry/integrations/github/integration.py:438"
+      }
     ]
   },
   "https://github.com/keycloak/keycloak/pull/32918": {
@@ -19033,6 +19451,26 @@
         "line": null,
         "source": "extracted"
       }
+    ],
+    "shipwright": [
+      {
+        "check_id": "benchmark.review",
+        "kind": "api",
+        "line": 245,
+        "path": "model/infinispan/src/main/java/org/keycloak/models/cache/infinispan/idp/InfinispanIdentityProviderStorageProvider.java",
+        "severity": "high",
+        "source": "shipwright.review",
+        "text": "In the new getForLogin() cache-hit path, IDs are resolved via `session.identityProviders().getById(id)` and added directly to the result set, without the `createOrganizationAwareIdentityProviderModel` wrapping that is applied on every other exit path of the same method (cache-invalid branch, first cache-miss branch, and the stale-id fallback branch). This means that once the login cache is populated (the common case after the first call), returned IdentityProviderModel instances are of a different, unwrapped shape than what callers previously always received, silently skipping whatever organization-aware behavior that wrapper provides. Location: model/infinispan/src/main/java/org/keycloak/models/cache/infinispan/idp/InfinispanIdentityProviderStorageProvider.java:245"
+      },
+      {
+        "check_id": "benchmark.review",
+        "kind": "bug",
+        "line": 382,
+        "path": "testsuite/integration-arquillian/tests/base/src/test/java/org/keycloak/testsuite/organization/cache/OrganizationCacheTest.java",
+        "severity": "medium",
+        "source": "shipwright.review",
+        "text": "Test cleanup registers removal using the hardcoded literal alias \"alias\" instead of the actual created alias (\"idp-alias-\" + i / \"idp-alias-20\"), so the created identity providers are never actually removed by cleanup. This leaks 20+ identity providers (and their organization associations) across test executions instead of tearing them down as intended. Location: testsuite/integration-arquillian/tests/base/src/test/java/org/keycloak/testsuite/organization/cache/OrganizationCacheTest.java:382"
+      }
     ]
   },
   "https://github.com/grafana/grafana/pull/94942": {
@@ -19894,6 +20332,35 @@
         "line": null,
         "source": "extracted"
       }
+    ],
+    "shipwright": [
+      {
+        "check_id": "benchmark.review",
+        "kind": "bug",
+        "line": 193,
+        "path": "pkg/expr/reader.go",
+        "severity": "critical",
+        "source": "shipwright.review",
+        "text": "enableSqlExpressions() always returns false regardless of the feature flag state: it computes `enabled := !IsEnabledGlobally(...)`, and both the `if enabled` branch and the fallthrough path return `false`. As a result, ReadQuery's new guard `if !enabled { return error }` unconditionally rejects SQL expression queries even when FlagSqlExpressions is enabled, permanently breaking the feature this PR is meant to introduce. Location: pkg/expr/reader.go:193"
+      },
+      {
+        "check_id": "benchmark.review",
+        "kind": "bug",
+        "line": 11,
+        "path": "pkg/expr/sql/db.go",
+        "severity": "critical",
+        "source": "shipwright.review",
+        "text": "The new sql.DB type (replacing the removed go-duck dependency) has all three methods (TablesList, RunCommands, QueryFramesInto) unconditionally return `errors.New(\"not implemented\")`. Since sql_command.go's Execute() now calls db.QueryFramesInto directly with no fallback, every SQL expression execution will fail at runtime with a generic 'not implemented' error instead of performing the query. Location: pkg/expr/sql/db.go:11"
+      },
+      {
+        "check_id": "benchmark.review",
+        "kind": "bug",
+        "line": 23,
+        "path": "pkg/expr/sql/parser.go",
+        "severity": "high",
+        "source": "shipwright.review",
+        "text": "parser.go's TablesList now calls the stub NewInMemoryDB()/RunCommands() which always returns \"not implemented\", so any consumer that resolves table lists for SQL expressions will always fail, silently breaking table-list resolution introduced/relied upon by this package. Location: pkg/expr/sql/parser.go:23"
+      }
     ]
   },
   "https://github.com/grafana/grafana/pull/90939": {
@@ -20444,7 +20911,8 @@
         "line": null,
         "source": "extracted"
       }
-    ]
+    ],
+    "shipwright": []
   },
   "https://github.com/grafana/grafana/pull/80329": {
     "gemini": [
@@ -21584,6 +22052,26 @@
         "line": null,
         "source": "extracted"
       }
+    ],
+    "shipwright": [
+      {
+        "check_id": "benchmark.review",
+        "kind": "bug",
+        "line": 528,
+        "path": "pkg/services/annotations/annotationsimpl/xorm_store.go",
+        "severity": "medium",
+        "source": "shipwright.review",
+        "text": "CleanAnnotations/CleanOrphanedAnnotationTags now log every batch cycle via r.log.Error even when there is no error (e.g. `r.log.Error(\"Annotations to clean by time\", \"count\", len(ids), \"ids\", ids, \"cond\", cond, \"err\", err)` when err is nil). This mislabels normal, successful cleanup operations as errors, which will pollute error-level logs/alerts, and additionally logs the full slice of IDs (which can be tens of thousands per batch, as demonstrated by the new 40000-row test case), producing excessively large log lines on every batch iteration. Location: pkg/services/annotations/annotationsimpl/xorm_store.go:528"
+      },
+      {
+        "check_id": "benchmark.review",
+        "kind": "perf",
+        "line": 77,
+        "path": "pkg/services/cleanup/cleanup.go",
+        "severity": "medium",
+        "source": "shipwright.review",
+        "text": "The cleanup service ticker interval was reduced from 10 minutes to 1 minute with no accompanying explanation or config change, causing the full cleanup job (annotations, tmp files, etc.) to run 10x more frequently. This is a significant behavioral change for a PR framed as 'database performance optimization' and could increase DB load rather than reduce it, especially combined with the smaller per-batch loop introduced in this PR. Location: pkg/services/cleanup/cleanup.go:77"
+      }
     ]
   },
   "https://github.com/grafana/grafana/pull/90045": {
@@ -23272,6 +23760,80 @@
         "line": null,
         "source": "extracted"
       }
+    ],
+    "shipwright": [
+      {
+        "check_id": "benchmark.review",
+        "kind": "concurrency",
+        "line": 47,
+        "path": "pkg/apiserver/rest/dualwriter_mode3.go",
+        "severity": "high",
+        "source": "shipwright.review",
+        "text": "In Create, Delete, Update, and DeleteCollection, the async write to Legacy storage is launched in a goroutine using `context.WithTimeoutCause(ctx, ...)` where `ctx` is the original request-scoped context. Since these goroutines run after the primary Storage operation has already returned to the caller (and the HTTP request context is commonly canceled once the response is written), the derived context is likely to be canceled before or shortly after the goroutine starts, causing legacy writes to fail intermittently/consistently. In addition, errors returned from `d.Legacy.Create/Delete/Update/DeleteCollection` inside these goroutines are only fed into a duration metric (as a boolean) and are never logged via `log.Error` as they were previously, silently swallowing legacy write failures. Location: pkg/apiserver/rest/dualwriter_mode3.go:47"
+      },
+      {
+        "check_id": "benchmark.review",
+        "kind": "bug",
+        "line": 41,
+        "path": "pkg/apiserver/rest/dualwriter_mode3.go",
+        "severity": "medium",
+        "source": "shipwright.review",
+        "text": "In Create(), when `d.Storage.Create` fails, the code calls `d.recordLegacyDuration(true, mode3Str, options.Kind, method, startStorage)` to record a Storage failure, mislabeling it as a legacy-duration metric instead of `d.recordStorageDuration`. This corrupts the storage vs legacy metrics. Location: pkg/apiserver/rest/dualwriter_mode3.go:41"
+      },
+      {
+        "check_id": "benchmark.review",
+        "kind": "bug",
+        "line": 121,
+        "path": "pkg/apiserver/rest/dualwriter_mode3.go",
+        "severity": "medium",
+        "source": "shipwright.review",
+        "text": "In Update(), when `d.Storage.Update` fails, the code calls `d.recordLegacyDuration(true, mode3Str, options.Kind, method, startStorage)` instead of `d.recordStorageDuration`, mislabeling a storage failure as a legacy metric, inconsistent with the equivalent success branch a few lines below that correctly calls `recordStorageDuration`. Location: pkg/apiserver/rest/dualwriter_mode3.go:121"
+      },
+      {
+        "check_id": "benchmark.review",
+        "kind": "bug",
+        "line": 161,
+        "path": "pkg/apiserver/rest/dualwriter_mode3.go",
+        "severity": "medium",
+        "source": "shipwright.review",
+        "text": "In DeleteCollection()'s async goroutine, the result of `d.Legacy.DeleteCollection` is recorded via `d.recordStorageDuration(err != nil, mode3Str, options.Kind, method, startLegacy)` instead of `d.recordLegacyDuration`, mislabeling a legacy-store duration/error as a storage metric. Location: pkg/apiserver/rest/dualwriter_mode3.go:161"
+      },
+      {
+        "check_id": "benchmark.review",
+        "kind": "bug",
+        "line": 100,
+        "path": "pkg/apiserver/rest/dualwriter_mode3.go",
+        "severity": "low",
+        "source": "shipwright.review",
+        "text": "In Delete(), the success path calls `d.recordStorageDuration(false, mode3Str, name, method, startStorage)`, passing the resource `name` as the metric's kind label instead of `options.Kind`, which is used everywhere else (including the error branch just above it in the same function). Location: pkg/apiserver/rest/dualwriter_mode3.go:100"
+      },
+      {
+        "check_id": "benchmark.review",
+        "kind": "bug",
+        "line": 93,
+        "path": "pkg/apiserver/rest/dualwriter_mode3.go",
+        "severity": "low",
+        "source": "shipwright.review",
+        "text": "In Delete(), an enriched logger `log` (with name/kind/method fields) is created, but the context is populated with the un-enriched `d.Log` instead: `ctx = klog.NewContext(ctx, d.Log)`. Every other method in this file (Create, Get, List, Update, DeleteCollection) correctly injects the enriched `log` variable into the context, so downstream code using `klog.FromContext(ctx)` in Delete's call path loses the name/kind/method fields. Location: pkg/apiserver/rest/dualwriter_mode3.go:93"
+      },
+      {
+        "check_id": "benchmark.review",
+        "kind": "data",
+        "line": 133,
+        "path": "pkg/apiserver/rest/dualwriter_mode3.go",
+        "severity": "medium",
+        "source": "shipwright.review",
+        "text": "Update() now passes the original `objInfo` (unwrapped) directly to `d.Legacy.Update`, instead of wrapping it with the object actually returned by `d.Storage.Update` (as the previous implementation did via `updateWrapper{upstream: objInfo, updated: obj}`). This means Legacy will independently recompute the updated object from its own current state rather than applying the same final object written to Storage, risking data divergence between the two stores. Location: pkg/apiserver/rest/dualwriter_mode3.go:133"
+      },
+      {
+        "check_id": "benchmark.review",
+        "kind": "doc_defect",
+        "line": 150,
+        "path": "pkg/apiserver/rest/dualwriter_mode3_test.go",
+        "severity": "low",
+        "source": "shipwright.review",
+        "text": "The test case named \"error when listing an object in the unified store is not implemented\" in TestMode3_List no longer exercises a not-implemented scenario (List is now fully implemented in this PR); it merely mocks a generic error return from Storage.List, so the test name is misleading relative to what it actually verifies. Location: pkg/apiserver/rest/dualwriter_mode3_test.go:150"
+      }
     ]
   },
   "https://github.com/grafana/grafana/pull/106778": {
@@ -24647,6 +25209,44 @@
         "line": null,
         "source": "extracted"
       }
+    ],
+    "shipwright": [
+      {
+        "check_id": "benchmark.review",
+        "kind": "bug",
+        "line": 300,
+        "path": "public/app/features/alerting/unified/hooks/useAbilities.ts",
+        "severity": "critical",
+        "source": "shipwright.review",
+        "text": "In useIsGrafanaPromRuleEditable (new function), the code calls `ctx.hasPermissionInMetadata(rulesPermissions.update, folder)` and `ctx.hasPermissionInMetadata(rulesPermissions.delete, folder)`, but `ctx` is never imported or defined anywhere in this file's diff. This will fail to compile (TS2304: Cannot find name 'ctx') or throw a ReferenceError at runtime, breaking all Grafana Prometheus-rule editability/removability checks (used by useAllGrafanaPromRuleAbilities, which gates Edit/Delete/Pause/Restore actions across the new rule-list UI). Location: public/app/features/alerting/unified/hooks/useAbilities.ts:300"
+      },
+      {
+        "check_id": "benchmark.review",
+        "kind": "security",
+        "line": 247,
+        "path": "public/app/features/alerting/unified/hooks/useAbilities.ts",
+        "severity": "high",
+        "source": "shipwright.review",
+        "text": "In useAllRulerRuleAbilities (used for cloud/Mimir/Loki ruler rules via useRulerRuleAbility/useRulerRuleAbilities, which power AlertRuleMenu and RuleActionsButtons for all data sources, not just Grafana), the previous federated-rule-group check `isFederated = isFederatedRuleGroup(rule.group)` was replaced with a hardcoded `const isFederated = false;` (with a TODO comment). This removes the immutability guard for federated rule groups on cloud/Mimir rulers, so rules in federated groups that were previously blocked from Edit/Delete/Duplicate will now incorrectly be treated as mutable. Location: public/app/features/alerting/unified/hooks/useAbilities.ts:247"
+      },
+      {
+        "check_id": "benchmark.review",
+        "kind": "bug",
+        "line": 29,
+        "path": "public/app/features/alerting/unified/rule-list/GrafanaRuleListItem.tsx",
+        "severity": "medium",
+        "source": "shipwright.review",
+        "text": "The new GrafanaRuleListItem builds the rule view link as `createRelativeUrl('/alerting/grafana/${uid}/view')` without the `returnTo` query param that the deleted GrafanaRuleLoader.tsx previously included (`createRelativeUrl(..., { returnTo })` using `createReturnTo()`). This is a functional regression: users clicking a Grafana rule from the list view will lose the 'return to' navigation context that was previously preserved. Location: public/app/features/alerting/unified/rule-list/GrafanaRuleListItem.tsx:29"
+      },
+      {
+        "check_id": "benchmark.review",
+        "kind": "test_gap",
+        "line": 205,
+        "path": "public/app/features/alerting/unified/rule-list/GrafanaGroupLoader.test.tsx",
+        "severity": "medium",
+        "source": "shipwright.review",
+        "text": "The new test 'should render correct menu actions when More button is clicked' grants AccessControlAction.AlertingRuleDelete (both globally and via setFolderAccessControl) in the shared beforeEach, but the test only asserts on silence/copyLink/duplicate/export and then asserts `menuItems.length` equals exactly 4, never checking or accounting for the Delete menu item that the granted delete permission should surface. This makes the test's assertions inconsistent with its own setup and likely to mask a missing/incorrect Delete action or produce a brittle/incorrect pass. Location: public/app/features/alerting/unified/rule-list/GrafanaGroupLoader.test.tsx:205"
+      }
     ]
   },
   "https://github.com/grafana/grafana/pull/107534": {
@@ -25398,6 +25998,26 @@
         "line": null,
         "source": "extracted"
       }
+    ],
+    "shipwright": [
+      {
+        "check_id": "benchmark.review",
+        "kind": "perf",
+        "line": 49,
+        "path": "public/app/plugins/datasource/loki/shardQuerySplitting.ts",
+        "severity": "medium",
+        "source": "shipwright.review",
+        "text": "runShardSplitQuery now interpolates each query up-front via `.map((query) => datasource.applyTemplateVariables(query, request.scopedVars, request.filters))`, replacing the previous single batched `interpolateVariablesInQueries` call. The accompanying test change confirms this causes `applyTemplateVariables` to be invoked 5 times instead of once (`expect(datasource.applyTemplateVariables).toHaveBeenCalledTimes(5)`), meaning the same query is now redundantly re-interpolated once at the top level and again per shard group downstream. This duplicate interpolation wastes computation and risks inconsistent results if the per-shard-group interpolation call uses different scopedVars/filters than the top-level call, since both write into the same query object. Location: public/app/plugins/datasource/loki/shardQuerySplitting.ts:49"
+      },
+      {
+        "check_id": "benchmark.review",
+        "kind": "bug",
+        "line": 296,
+        "path": "public/app/plugins/datasource/loki/querySplitting.ts",
+        "severity": "low",
+        "source": "shipwright.review",
+        "text": "The new top-level interpolation (`applyTemplateVariables`) is applied to every split-eligible query before the time range or shards are computed, resolving macros/scoped vars against the request's overall scopedVars. Since query splitting exists specifically to divide a single request into multiple sub-requests over time/shard partitions, resolving variables once up-front (rather than per generated sub-query) risks baking a single global interpolation into every emitted sub-query, whereas the previous batch call semantics and downstream per-chunk processing assumed interpolation could still be adjusted per partition. Location: public/app/plugins/datasource/loki/querySplitting.ts:296"
+      }
     ]
   },
   "https://github.com/grafana/grafana/pull/79265": {
@@ -26677,6 +27297,26 @@
         "line": null,
         "source": "extracted"
       }
+    ],
+    "shipwright": [
+      {
+        "check_id": "benchmark.review",
+        "kind": "concurrency",
+        "line": 90,
+        "path": "pkg/services/anonymous/anonimpl/anonstore/database.go",
+        "severity": "medium",
+        "source": "shipwright.review",
+        "text": "CreateOrUpdateDevice now does a check-then-act (CountDevices followed by a separate insert/update) with no transaction or locking. Concurrent anonymous requests can all read count < deviceLimit before any of them inserts, allowing the device count to exceed the configured deviceLimit (TOCTOU race), defeating the new device-limit enforcement under load. Location: pkg/services/anonymous/anonimpl/anonstore/database.go:90"
+      },
+      {
+        "check_id": "benchmark.review",
+        "kind": "concurrency",
+        "line": 44,
+        "path": "pkg/services/anonymous/anonimpl/client.go",
+        "severity": "medium",
+        "source": "shipwright.review",
+        "text": "Anonymous.Authenticate previously tagged devices in a background goroutine with its own context.Background()+2min timeout and a recover() to swallow panics, decoupling tagging from the request lifecycle. The diff removes the goroutine, timeout, and panic recovery, making TagDevice run synchronously inline using the request's ctx. This means (1) if the client cancels/the request context is done before the DB write completes, the device tag/update can be aborted mid-flight instead of completing in the background as before, and (2) any panic inside TagDevice/tagDeviceUI now propagates up through Authenticate with no recover(), instead of being caught and logged as previously. Location: pkg/services/anonymous/anonimpl/client.go:44"
+      }
     ]
   },
   "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/9": {
@@ -27502,6 +28142,17 @@
         "line": null,
         "source": "extracted"
       }
+    ],
+    "shipwright": [
+      {
+        "check_id": "benchmark.review",
+        "kind": "bug",
+        "line": 22,
+        "path": "config/initializers/i18n.rb",
+        "severity": "critical",
+        "source": "shipwright.review",
+        "text": "In config/initializers/i18n.rb, FallbackLocaleList#ensure_loaded! calls `I18n.ensure_loaded! l`, but the new `ensure_loaded!(locale)` method is defined on the I18n backend instance (lib/freedom_patches/translate_accelerator.rb), not on the top-level I18n module. The I18n module does not expose or delegate an `ensure_loaded!` class method, so this call raises NoMethodError. Since ApplicationController now calls `I18n.fallbacks.ensure_loaded!` on every request (in the locale-setting method), this breaks every request that goes through that code path. The call should be `I18n.backend.ensure_loaded! l`. Location: config/initializers/i18n.rb:22"
+      }
     ]
   },
   "https://github.com/grafana/grafana/pull/76186": {
@@ -28220,6 +28871,26 @@
         "line": null,
         "source": "extracted"
       }
+    ],
+    "shipwright": [
+      {
+        "check_id": "benchmark.review",
+        "kind": "test_gap",
+        "line": 46,
+        "path": "pkg/plugins/log/fake.go",
+        "severity": "high",
+        "source": "shipwright.review",
+        "text": "TestLogger.FromContext (pkg/plugins/log/fake.go) returns a brand-new NewTestLogger() instance instead of the receiver, discarding all call-tracking state. Since LoggerMiddleware.logRequest now calls `m.logger.FromContext(ctx).Info(...)` instead of `m.logger.Info(...)`, any test using the fake TestLogger to assert on InfoLogs/DebugLogs/etc. after invoking the middleware will silently see no recorded calls, since the logs are recorded on a throwaway logger rather than the original fake instance. Location: pkg/plugins/log/fake.go:46"
+      },
+      {
+        "check_id": "benchmark.review",
+        "kind": "style",
+        "line": 75,
+        "path": "pkg/services/pluginsintegration/clientmiddleware/metrics_middleware_test.go",
+        "severity": "low",
+        "source": "shipwright.review",
+        "text": "The middleware file was renamed from instrumentation_middleware_test.go to metrics_middleware_test.go and the internal call was updated to newMetricsMiddleware, but the test function itself is still named TestInstrumentationMiddleware, leaving a stale/misleading test name that no longer matches the renamed middleware or file. Location: pkg/services/pluginsintegration/clientmiddleware/metrics_middleware_test.go:75"
+      }
     ]
   },
   "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/10": {
@@ -30325,6 +30996,62 @@
         "line": null,
         "source": "extracted"
       }
+    ],
+    "shipwright": [
+      {
+        "check_id": "benchmark.review",
+        "kind": "bug",
+        "line": 10,
+        "path": "db/migrate/20150818190757_create_embeddable_hosts.rb",
+        "severity": "critical",
+        "source": "shipwright.review",
+        "text": "The migration runs `execute(\"SELECT c.id FROM categories AS c INNER JOIN site_settings AS s ON s.value = c.name WHERE s.name = 'embed_category'\")[0]['id'].to_i`. If no site_settings row exists for 'embed_category' (the common case, since it defaults to '' and is often never overridden), the query returns an empty result set, `[0]` is `nil`, and calling `['id']` on `nil` raises `NoMethodError`, crashing the migration for most existing installs. Location: db/migrate/20150818190757_create_embeddable_hosts.rb:10"
+      },
+      {
+        "check_id": "benchmark.review",
+        "kind": "security",
+        "line": 25,
+        "path": "db/migrate/20150818190757_create_embeddable_hosts.rb",
+        "severity": "medium",
+        "source": "shipwright.review",
+        "text": "Migration inserts host values into SQL via string interpolation (`VALUES ('#{h}', ...)`) instead of parameter binding/quoting, which is unsafe if `h` (pulled from the old `embeddable_hosts` site setting) contains a single quote or other SQL metacharacters. Location: db/migrate/20150818190757_create_embeddable_hosts.rb:25"
+      },
+      {
+        "check_id": "benchmark.review",
+        "kind": "bug",
+        "line": 5,
+        "path": "app/models/embeddable_host.rb",
+        "severity": "high",
+        "source": "shipwright.review",
+        "text": "`before_validation` unconditionally calls `self.host.sub!(...)` twice. If `host` is nil (e.g. missing/blank param), this raises `NoMethodError` instead of surfacing a normal ActiveRecord validation error, crashing create/update instead of returning a friendly error. Location: app/models/embeddable_host.rb:5"
+      },
+      {
+        "check_id": "benchmark.review",
+        "kind": "bug",
+        "line": 16,
+        "path": "app/models/embeddable_host.rb",
+        "severity": "medium",
+        "source": "shipwright.review",
+        "text": "`record_for_host` compares `lower(host) = ?` against `host = uri.host`, which is not downcased before being passed as the bind parameter. A referrer/URL with mixed-case hostname (e.g. `http://EvilTrout.com`) will fail to match the lowercased DB column, causing legitimate allowed hosts to be rejected by `host_allowed?`. Location: app/models/embeddable_host.rb:16"
+      },
+      {
+        "check_id": "benchmark.review",
+        "kind": "bug",
+        "line": 8,
+        "path": "app/controllers/admin/embeddable_hosts_controller.rb",
+        "severity": "high",
+        "source": "shipwright.review",
+        "text": "`update` and `destroy` fetch the record with `EmbeddableHost.where(id: params[:id]).first` (which returns nil if not found) and then unconditionally call `save_host(host)` / `host.destroy` on it, causing a `NoMethodError` instead of a proper 404/error response when the id doesn't exist. Location: app/controllers/admin/embeddable_hosts_controller.rb:8"
+      },
+      {
+        "check_id": "benchmark.review",
+        "kind": "style",
+        "line": 1,
+        "path": "spec/fabricators/category_fabricator.rb",
+        "severity": "medium",
+        "source": "shipwright.review",
+        "text": "The contents of `spec/fabricators/category_fabricator.rb` and the new `spec/fabricators/embeddable_host_fabricator.rb` were swapped: `category_fabricator.rb` now only defines `Fabricator(:embeddable_host)`, while the new `embeddable_host_fabricator.rb` contains the `:category`/`:diff_category`/`:happy_category`/`:private_category` fabricator definitions that used to live in `category_fabricator.rb`. Location: spec/fabricators/category_fabricator.rb:1"
+      }
     ]
   },
   "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/7": {
@@ -31564,6 +32291,53 @@
         "line": null,
         "source": "extracted"
       }
+    ],
+    "shipwright": [
+      {
+        "check_id": "benchmark.review",
+        "kind": "bug",
+        "line": 291,
+        "path": "app/assets/stylesheets/desktop/topic-post.scss",
+        "severity": "medium",
+        "source": "shipwright.review",
+        "text": "When converting to dark-light-choose, the light-theme (primary) lightness value was changed from 30% to 70% instead of being preserved, unlike every other conversion in this diff which keeps the original primary lightness for the light theme and uses (100-x)% for the secondary/dark value. This alters the light-theme link color unintentionally. Location: app/assets/stylesheets/desktop/topic-post.scss:291"
+      },
+      {
+        "check_id": "benchmark.review",
+        "kind": "bug",
+        "line": 101,
+        "path": "app/assets/stylesheets/mobile/modal.scss",
+        "severity": "medium",
+        "source": "shipwright.review",
+        "text": "The primary/secondary lightness values were swapped when converting to dark-light-choose: light theme now uses 30% instead of the original 70%, and dark theme uses 70% instead of the expected complementary 30%. This inverts the intended light/dark appearance instead of preserving the light theme and only adding a dark-theme override. Location: app/assets/stylesheets/mobile/modal.scss:101"
+      },
+      {
+        "check_id": "benchmark.review",
+        "kind": "bug",
+        "line": 522,
+        "path": "app/assets/stylesheets/desktop/user.scss",
+        "severity": "medium",
+        "source": "shipwright.review",
+        "text": "The original primary lightness of 30% was replaced with 50% for both the light and dark arguments of dark-light-choose, instead of preserving 30% for light and using the complementary 70% for dark (the pattern used everywhere else in this diff). This changes the light-theme .name color unintentionally. Location: app/assets/stylesheets/desktop/user.scss:522"
+      },
+      {
+        "check_id": "benchmark.review",
+        "kind": "bug",
+        "line": 182,
+        "path": "app/assets/stylesheets/mobile/topic-post.scss",
+        "severity": "medium",
+        "source": "shipwright.review",
+        "text": "The original primary lightness of 20% for h3 was replaced with 50% for both light and dark arguments instead of preserving 20% for light and 80% for dark, deviating from the conversion pattern used elsewhere and unintentionally altering the light-theme heading color. Location: app/assets/stylesheets/mobile/topic-post.scss:182"
+      },
+      {
+        "check_id": "benchmark.review",
+        "kind": "bug",
+        "line": 497,
+        "path": "app/assets/stylesheets/mobile/user.scss",
+        "severity": "medium",
+        "source": "shipwright.review",
+        "text": "The original primary lightness of 30% for .name was replaced with 50% for both light and dark arguments of dark-light-choose instead of preserving 30% for light and 70% for dark, unintentionally changing the light-theme appearance. Location: app/assets/stylesheets/mobile/user.scss:497"
+      }
     ]
   },
   "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/8": {
@@ -33299,6 +34073,35 @@
         "line": null,
         "source": "extracted"
       }
+    ],
+    "shipwright": [
+      {
+        "check_id": "benchmark.review",
+        "kind": "api",
+        "line": 66,
+        "path": "app/controllers/admin/groups_controller.rb",
+        "severity": "critical",
+        "source": "shipwright.review",
+        "text": "routes.rb defines `delete \"members\" => \"groups#remove_member\"` and `put \"members\" => \"groups#add_members\"` as member routes nested under `resources :groups` (matching the URL pattern `/admin/groups/:id/members`, same style as the removed `get \"users\"`), so Rails will populate `params[:id]` with the group id. However `Admin::GroupsController#add_members` and `#remove_member` both call `params.require(:group_id)` to look up the group, which will never be present on a real request generated by the JS client (`Discourse.ajax('/admin/groups/' + id + '/members.json', ...)` in `group.js`). This raises `ActionController::ParameterMissing` and breaks the add/remove-member feature the PR claims to fix. The specs mask this by invoking the actions directly with an explicit `group_id:` param instead of going through routing, so the bug isn't caught by the added tests. Location: app/controllers/admin/groups_controller.rb:66"
+      },
+      {
+        "check_id": "benchmark.review",
+        "kind": "bug",
+        "line": 13,
+        "path": "app/assets/javascripts/admin/controllers/admin-group.js.es6",
+        "severity": "medium",
+        "source": "shipwright.review",
+        "text": "`totalPages` is computed as `Math.floor(user_count / limit) + 1`, which is off-by-one whenever `user_count` is an exact multiple of `limit` (e.g. user_count=50, limit=50 gives totalPages=2 instead of 1). This causes `showingLast` to stay false one page too early, so the \"next\" pagination control remains enabled past the real last page; clicking it sets `offset` to `user_count` and fetches an empty page, showing an extra blank page in the member list. Location: app/assets/javascripts/admin/controllers/admin-group.js.es6:13"
+      },
+      {
+        "check_id": "benchmark.review",
+        "kind": "bug",
+        "line": 1,
+        "path": "app/assets/javascripts/admin/templates/group_member.hbs",
+        "severity": "medium",
+        "source": "shipwright.review",
+        "text": "In the new `group_member.hbs` itemView template, `{{#unless automatic}}` is evaluated against the item view's context, which is the individual `member` (a Discourse.User) because it's rendered via `{{each member in members itemView=\"group-member\"}}` in `group.hbs`, not the parent Group model's `automatic` property. Since `member.automatic` is undefined, the `unless` block always renders, so the remove-member link is shown even for members of automatic groups (server-side still rejects via `can_not_modify_automatic`, but the UI incorrectly offers an action that will always fail). Location: app/assets/javascripts/admin/templates/group_member.hbs:1"
+      }
     ]
   },
   "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/3": {
@@ -34429,6 +35232,26 @@
         "line": null,
         "source": "extracted"
       }
+    ],
+    "shipwright": [
+      {
+        "check_id": "benchmark.review",
+        "kind": "security",
+        "line": 11,
+        "path": "app/models/blocked_email.rb",
+        "severity": "high",
+        "source": "shipwright.review",
+        "text": "BlockedEmail.should_block? performs an exact string match (`BlockedEmail.where(email: email).first`) without normalizing case. Since emails are case-insensitive by RFC and neither BlockedEmail nor the validator lowercases the value before lookup, a blocked email like 'bad@spamclub.com' can bypass the block simply by submitting 'Bad@SpamClub.com', defeating the entire purpose of this new blocking feature. Location: app/models/blocked_email.rb:11"
+      },
+      {
+        "check_id": "benchmark.review",
+        "kind": "bug",
+        "line": 17,
+        "path": "app/assets/javascripts/discourse/controllers/create_account_controller.js",
+        "severity": "medium",
+        "source": "shipwright.review",
+        "text": "`rejectedEmails: Em.A([])` is declared directly as a property value on the Ember.Object.extend() class definition rather than being initialized in an `init` hook. In Ember's classic object model, array/object literals defined this way are shared across all instances created from this class definition, so state from one instance (e.g. a previously rejected email) can leak into or persist across other instances of CreateAccountController. Location: app/assets/javascripts/discourse/controllers/create_account_controller.js:17"
+      }
     ]
   },
   "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/5": {
@@ -35421,6 +36244,26 @@
         "line": null,
         "source": "extracted"
       }
+    ],
+    "shipwright": [
+      {
+        "check_id": "benchmark.review",
+        "kind": "bug",
+        "line": 16,
+        "path": "app/assets/stylesheets/common/base/header.scss",
+        "severity": "high",
+        "source": "shipwright.review",
+        "text": "The `.title` rule (previously `float: left;`) was deleted entirely, and flexbox properties (`display: flex; align-items: center;`) were instead added to `.contents`. Based on the original indentation, `.title` was a sibling selector of `.contents` (same 4-space indent level), not a child of it. Removing `.title`'s `float: left` while making `.contents` (a different, sibling element) a flex container does nothing for `.title`'s layout \u2014 the title element loses its horizontal positioning entirely, breaking the header layout. Location: app/assets/stylesheets/common/base/header.scss:16"
+      },
+      {
+        "check_id": "benchmark.review",
+        "kind": "bug",
+        "line": 36,
+        "path": "app/assets/stylesheets/common/base/header.scss",
+        "severity": "high",
+        "source": "shipwright.review",
+        "text": "`.panel` had `float: right;` removed and replaced with `margin-left: auto;` plus `@include order(3);`. These properties only have an effect when the element is a flex *item* inside a flex container. `.panel` is a sibling of `.contents` (same indentation level as `.contents`/`.valign-helper`), not nested inside it, and no flex container is declared for their shared parent in this diff. As a result `.panel` loses its previous right-floating alignment with no functional replacement, breaking the header's right-side layout (login/sign-up buttons, panel icons, etc.). Location: app/assets/stylesheets/common/base/header.scss:36"
+      }
     ]
   },
   "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/6": {
@@ -36393,6 +37236,35 @@
         "line": null,
         "source": "extracted"
       }
+    ],
+    "shipwright": [
+      {
+        "check_id": "benchmark.review",
+        "kind": "api",
+        "line": 152,
+        "path": "app/serializers/user_serializer.rb",
+        "severity": "high",
+        "source": "shipwright.review",
+        "text": "`include_website_name` is missing the trailing `?` required by Discourse's ActiveModel::Serializer convention for conditional-attribute inclusion (see how other `include_<attr>?` methods work). Because the method is not recognized as a presence guard, `website_name` will always be serialized regardless of whether the user has a website set, contradicting the intended `website.present?` check and returning a bogus `website_name: nil` for users without a website. Location: app/serializers/user_serializer.rb:152"
+      },
+      {
+        "check_id": "benchmark.review",
+        "kind": "bug",
+        "line": 140,
+        "path": "app/serializers/user_serializer.rb",
+        "severity": "medium",
+        "source": "shipwright.review",
+        "text": "The new `website_name` domain-comparison logic compares hostnames (`website_host`, `discourse_host`) directly without normalizing case (no `.downcase`). Hostnames are case-insensitive per RFC, so a website URL with a mixed-case host (e.g. `http://Example.com/user`) will fail the `website_host == discourse_host` and subdomain-suffix comparisons even when it is actually the same domain as `Discourse.current_hostname`, causing the full path to not be shown when it should be. Location: app/serializers/user_serializer.rb:140"
+      },
+      {
+        "check_id": "benchmark.review",
+        "kind": "security",
+        "line": 143,
+        "path": "app/serializers/user_serializer.rb",
+        "severity": "medium",
+        "source": "shipwright.review",
+        "text": "The subdomain-matching branch strips only the first label of each host and compares the remaining suffix (`split('.')[1..-1].join('.')`) without validating that this remainder is a real registrable domain. For hosts under shared multi-label public suffixes (e.g. `foo.co.uk` vs `bar.co.uk`), the stripped suffix `co.uk` will match even though the two hosts are unrelated organizations, incorrectly causing the full website path to be exposed as if the website and instance shared the same domain. Location: app/serializers/user_serializer.rb:143"
+      }
     ]
   },
   "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/4": {
@@ -38806,6 +39678,53 @@
         "line": null,
         "source": "extracted"
       }
+    ],
+    "shipwright": [
+      {
+        "check_id": "benchmark.review",
+        "kind": "security",
+        "line": 131,
+        "path": "app/models/post.rb",
+        "severity": "critical",
+        "source": "shipwright.review",
+        "text": "When cook_method is raw_html, Post#cook now returns `raw` directly and skips post_analyzer.cook/Plugin::Filter.apply entirely (no sanitization pipeline runs). Jobs::PollFeed.poll_feed feeds raw external RSS/Atom HTML content (`CGI.unescapeHTML(i.content.scrub)`) straight into TopicEmbed.import -> PostCreator with cook_method: raw_html, and the embed view (app/views/embed/best.html.erb) renders `post.cooked` with `raw`. This creates a stored XSS vector: arbitrary/untrusted RSS feed content is rendered unsanitized as raw HTML in the embed iframe. Location: app/models/post.rb:131"
+      },
+      {
+        "check_id": "benchmark.review",
+        "kind": "security",
+        "line": 15,
+        "path": "app/assets/javascripts/embed.js",
+        "severity": "medium",
+        "source": "shipwright.review",
+        "text": "postMessage origin validation uses substring containment (`discourseUrl.indexOf(e.origin) === -1`) instead of a strict origin equality/prefix check. This is a fragile/incorrect security check: it tests whether the message's origin string appears anywhere inside discourseUrl rather than validating that the message actually originates from discourseUrl, which is the wrong direction for origin validation and can silently accept or reject messages incorrectly compared to a proper equality check. Location: app/assets/javascripts/embed.js:15"
+      },
+      {
+        "check_id": "benchmark.review",
+        "kind": "bug",
+        "line": 140,
+        "path": "lib/tasks/disqus.thor",
+        "severity": "high",
+        "source": "shipwright.review",
+        "text": "The Disqus import thor task was refactored to call `TopicEmbed.import_remote(user, t[:link], title: t[:title])` instead of `PostCreator.new(user, title:, raw:, created_at:, category:)`. This drops the previously supported `--category`/`-c` CLI option (and its `category_id` lookup) entirely, and no longer passes `created_at: Date.parse(t[:created_at])`, so imported topics lose both category placement and their original creation date \u2014 a functional regression compared to the prior implementation. Location: lib/tasks/disqus.thor:140"
+      },
+      {
+        "check_id": "benchmark.review",
+        "kind": "data",
+        "line": 3,
+        "path": "db/migrate/20131223171005_create_top_topics.rb",
+        "severity": "medium",
+        "source": "shipwright.review",
+        "text": "An already-existing migration (`create_table :top_topics`) was modified to add `force: true`. Since this file's timestamp/version predates the newly added migrations in this PR, any environment that already applied this migration and has data in `top_topics` risks that table being dropped and recreated if the migration is ever re-run (e.g., via db:migrate:redo or manual re-application), causing data loss. Existing migrations that have already run should not be edited to add destructive options. Location: db/migrate/20131223171005_create_top_topics.rb:3"
+      },
+      {
+        "check_id": "benchmark.review",
+        "kind": "bug",
+        "line": 29,
+        "path": "app/controllers/embed_controller.rb",
+        "severity": "low",
+        "source": "shipwright.review",
+        "text": "EmbedController#ensure_embeddable compares `URI(request.referer || '').host` to `SiteSetting.embeddable_host` with a case-sensitive `!=` check and no normalization on either side. If an admin configures embeddable_host with different casing than the browser-supplied referer host (or vice versa), legitimate embed requests will be rejected as InvalidAccess, and conversely there is no guarantee both sides are normalized consistently for comparison. Location: app/controllers/embed_controller.rb:29"
+      }
     ]
   },
   "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/1": {
@@ -40259,6 +41178,44 @@
         "line": null,
         "source": "extracted"
       }
+    ],
+    "shipwright": [
+      {
+        "check_id": "benchmark.review",
+        "kind": "bug",
+        "line": 182,
+        "path": "app/assets/javascripts/discourse/lib/utilities.js",
+        "severity": "medium",
+        "source": "shipwright.review",
+        "text": "Client-side file size validation now uses a hardcoded 10*1024 KB (10MB) constant instead of the configurable `Discourse.SiteSettings['max_' + type + '_size_kb']` / `Discourse.SiteSettings.max_image_size_kb`. This breaks per-type size limits (e.g. avatars, attachments) and will diverge from the server-side `SiteSetting.max_image_size_kb` used elsewhere in this same PR, causing inconsistent validation between client and server. Location: app/assets/javascripts/discourse/lib/utilities.js:182"
+      },
+      {
+        "check_id": "benchmark.review",
+        "kind": "bug",
+        "line": 55,
+        "path": "app/controllers/uploads_controller.rb",
+        "severity": "medium",
+        "source": "shipwright.review",
+        "text": "The API upload URL download cap was changed from `SiteSetting.max_image_size_kb.kilobytes` to a hardcoded `10.megabytes`. This contradicts the PR's stated goal of allowing larger images to be downloaded and then downsized: any image larger than 10MB fetched via URL will now fail to download entirely (silently rescued to nil) before ever reaching the new downsize logic, and the cap no longer respects site configuration. Location: app/controllers/uploads_controller.rb:55"
+      },
+      {
+        "check_id": "benchmark.review",
+        "kind": "api",
+        "line": 146,
+        "path": "app/models/optimized_image.rb",
+        "severity": "high",
+        "source": "shipwright.review",
+        "text": "`OptimizedImage.downsize` is now defined twice with different arities (`downsize(from, to, max_width, max_height, opts={})` and `downsize(from, to, dimensions, opts={})`). Ruby silently keeps only the second definition, making the first a dead/unreachable method whose original 4-positional-argument signature (from the pre-diff API) is gone. Any code still calling `downsize` with separate width/height arguments will now raise an ArgumentError. Location: app/models/optimized_image.rb:146"
+      },
+      {
+        "check_id": "benchmark.review",
+        "kind": "bug",
+        "line": 63,
+        "path": "app/controllers/uploads_controller.rb",
+        "severity": "medium",
+        "source": "shipwright.review",
+        "text": "The new downsize retry loop in the upload flow gives up silently after 5 attempts with no check on whether the image was actually reduced below `SiteSetting.max_image_size_kb`. If `OptimizedImage.downsize` fails or converges slowly, the oversized tempfile is passed straight to `Upload.create_for` anyway, defeating the max image size validation instead of enforcing it. Location: app/controllers/uploads_controller.rb:63"
+      }
     ]
   },
   "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/2": {
@@ -41557,6 +42514,44 @@
         "line": null,
         "source": "extracted"
       }
+    ],
+    "shipwright": [
+      {
+        "check_id": "benchmark.review",
+        "kind": "bug",
+        "line": 27,
+        "path": "app/controllers/topics_controller.rb",
+        "severity": "high",
+        "source": "shipwright.review",
+        "text": "`:unsubscribe` was added to the `except:` list of `before_filter :ensure_logged_in`, so the new `unsubscribe` action can now be reached without a logged-in user. However, the action body unconditionally calls `TopicUser.find_by(user_id: current_user.id, ...)`, which raises `NoMethodError: undefined method 'id' for nil:NilClass` when `current_user` is nil (e.g. when the emailed unsubscribe link is opened in a logged-out browser). Location: app/controllers/topics_controller.rb:27"
+      },
+      {
+        "check_id": "benchmark.review",
+        "kind": "bug",
+        "line": 106,
+        "path": "app/controllers/topics_controller.rb",
+        "severity": "high",
+        "source": "shipwright.review",
+        "text": "`tu = TopicUser.find_by(user_id: current_user.id, topic_id: params[:topic_id])` can return nil (no TopicUser row exists for that user/topic), but the following code immediately calls `tu.notification_level` and `tu.save!` without a nil guard, causing a NoMethodError crash instead of gracefully creating/handling the missing preference row. Location: app/controllers/topics_controller.rb:106"
+      },
+      {
+        "check_id": "benchmark.review",
+        "kind": "bug",
+        "line": 108,
+        "path": "app/controllers/topics_controller.rb",
+        "severity": "medium",
+        "source": "shipwright.review",
+        "text": "The unsubscribe toggle logic only downgrades a user's notification_level to `regular` when it is above `regular` (e.g. tracking/watching), and only sets it to `muted` when it is already at or below `regular`. This contradicts the new copy shown to the user (\"You will stop receiving notifications for <title>\", config/locales/client.en.yml `topic.unsubscribe.stop_notifications`), since a watching/tracking user clicking the unsubscribe link will still be left at `regular` and can continue to receive notifications, not actually unsubscribed as promised. Location: app/controllers/topics_controller.rb:108"
+      },
+      {
+        "check_id": "benchmark.review",
+        "kind": "style",
+        "line": 5,
+        "path": "app/assets/javascripts/discourse/controllers/topic-unsubscribe.js.es6",
+        "severity": "low",
+        "source": "shipwright.review",
+        "text": "The new controller property is misspelled `stopNotificiationsText` (should be `stopNotificationsText`); while it still works because the template references the same misspelled name, it is a misleading/incorrect identifier introduced by this PR. Location: app/assets/javascripts/discourse/controllers/topic-unsubscribe.js.es6:5"
+      }
     ]
   },
   "https://github.com/calcom/cal.com/pull/22532": {
@@ -42941,6 +43936,26 @@
         "line": null,
         "source": "extracted"
       }
+    ],
+    "shipwright": [
+      {
+        "check_id": "benchmark.review",
+        "kind": "bug",
+        "line": 70,
+        "path": "packages/platform/atoms/selected-calendars/wrappers/SelectedCalendarsSettingsWebWrapper.tsx",
+        "severity": "low",
+        "source": "shipwright.review",
+        "text": "The `actions` prop in both AppListCard renders now unconditionally wrap `CredentialActionsDropdown` in a `<div className=\"flex w-32 justify-end\">`, whereas previously the whole block (including the wrapping div) was only rendered when `!delegationCredentialId && !disableConnectionModification`. Since `CredentialActionsDropdown` returns `null` when `!canDisconnect && !hasCache` (e.g., delegation credentials or calendars without cache), the surrounding div still renders, leaving an empty 8rem-wide, right-aligned box where nothing used to appear at all \u2014 a visible UI regression introduced by this diff. Location: packages/platform/atoms/selected-calendars/wrappers/SelectedCalendarsSettingsWebWrapper.tsx:70"
+      },
+      {
+        "check_id": "benchmark.review",
+        "kind": "api",
+        "line": 21,
+        "path": "packages/trpc/server/routers/viewer/calendars/deleteCache.handler.ts",
+        "severity": "medium",
+        "source": "shipwright.review",
+        "text": "deleteCacheHandler throws a plain `Error(\"Credential not found or access denied\")` instead of a `TRPCError` with an appropriate code (e.g. NOT_FOUND/FORBIDDEN). In a tRPC mutation this will surface as a generic INTERNAL_SERVER_ERROR rather than a distinguishable client error, making it impossible for callers/monitoring to differentiate an authorization failure from an actual server fault. Location: packages/trpc/server/routers/viewer/calendars/deleteCache.handler.ts:21"
+      }
     ]
   },
   "https://github.com/calcom/cal.com/pull/8330": {
@@ -44199,6 +45214,26 @@
         "line": null,
         "source": "extracted"
       }
+    ],
+    "shipwright": [
+      {
+        "check_id": "benchmark.review",
+        "kind": "bug",
+        "line": 140,
+        "path": "packages/trpc/server/routers/viewer/slots.ts",
+        "severity": "high",
+        "source": "shipwright.review",
+        "text": "In the new working-hours check, `end` is computed using `slotStartTime` instead of `slotEndTime` (`const end = slotStartTime.hour() * 60 + slotStartTime.minute();`), so the code never actually validates that the slot's end time falls within the organizer's working hours. A slot that starts inside working hours but extends past closing time will incorrectly be treated as available. Location: packages/trpc/server/routers/viewer/slots.ts:140"
+      },
+      {
+        "check_id": "benchmark.review",
+        "kind": "bug",
+        "line": 121,
+        "path": "packages/trpc/server/routers/viewer/slots.ts",
+        "severity": "medium",
+        "source": "shipwright.review",
+        "text": "The date-override equality check `dayjs(date.start).add(utcOffset, \"minutes\") === dayjs(date.end).add(utcOffset, \"minutes\")` compares two freshly-created dayjs object instances with `===`, which is always false since they are different object references (dayjs objects are never referentially equal even for identical values). This branch, meant to detect zero-duration/whole-day overrides, can never execute, silently breaking the intended handling of that case. Location: packages/trpc/server/routers/viewer/slots.ts:121"
+      }
     ]
   },
   "https://github.com/calcom/cal.com/pull/14943": {
@@ -45012,6 +46047,26 @@
         "line": null,
         "source": "extracted"
       }
+    ],
+    "shipwright": [
+      {
+        "check_id": "benchmark.review",
+        "kind": "bug",
+        "line": 34,
+        "path": "packages/features/ee/workflows/api/scheduleSMSReminders.ts",
+        "severity": "high",
+        "source": "shipwright.review",
+        "text": "The new OR clause in deleteMany omits the `method: WorkflowMethods.SMS` (and scheduledDate) constraint that the first clause has, so any WorkflowReminder row (not just SMS reminders, and regardless of scheduledDate) with retryCount > 1 will be deleted. Since retryCount is a generic column added to WorkflowReminder, this can delete non-SMS reminders (email/whatsapp) that happen to have retryCount incremented elsewhere, unintentionally purging reminders unrelated to the SMS retry logic this PR is meant to implement. Location: packages/features/ee/workflows/api/scheduleSMSReminders.ts:34"
+      },
+      {
+        "check_id": "benchmark.review",
+        "kind": "concurrency",
+        "line": 181,
+        "path": "packages/features/ee/workflows/api/scheduleSMSReminders.ts",
+        "severity": "medium",
+        "source": "shipwright.review",
+        "text": "retryCount is incremented via a non-atomic read-modify-write (`retryCount: reminder.retryCount + 1`) using a value fetched earlier in the same handler invocation, rather than Prisma's atomic `{ increment: 1 }`. If this cron/handler runs concurrently (e.g., overlapping invocations processing the same reminder), updates can race and increments can be lost, undermining the retry-count-based cleanup this PR introduces. Location: packages/features/ee/workflows/api/scheduleSMSReminders.ts:181"
+      }
     ]
   },
   "https://github.com/calcom/cal.com/pull/22345": {
@@ -45810,7 +46865,8 @@
         "line": null,
         "source": "extracted"
       }
-    ]
+    ],
+    "shipwright": []
   },
   "https://github.com/calcom/cal.com/pull/11059": {
     "gemini": [
@@ -48219,6 +49275,53 @@
         "line": null,
         "source": "extracted"
       }
+    ],
+    "shipwright": [
+      {
+        "check_id": "benchmark.review",
+        "kind": "bug",
+        "line": 97,
+        "path": "packages/app-store/googlecalendar/lib/CalendarService.ts",
+        "severity": "critical",
+        "source": "shipwright.review",
+        "text": "parseRefreshTokenResponse's return value (a safeParse-style {success, data} object) is now stored directly as the credential `key` instead of the actual parsed token data: `const key = parseRefreshTokenResponse(googleCredentials, googleCredentialSchema); ... data: { key }`. Previously `googleCredentialSchema.parse(...)` returned the flat token object; now the credential.key JSON will be `{success, data}` wrapper, corrupting the stored Google Calendar OAuth credentials and breaking subsequent calendar API calls. Location: packages/app-store/googlecalendar/lib/CalendarService.ts:97"
+      },
+      {
+        "check_id": "benchmark.review",
+        "kind": "bug",
+        "line": 24,
+        "path": "packages/app-store/_utils/oauth/parseRefreshTokenResponse.ts",
+        "severity": "high",
+        "source": "shipwright.review",
+        "text": "When the refreshed token response lacks a `refresh_token` field, the code sets it to the literal string \"refresh_token\" instead of preserving the previous refresh token or leaving it undefined: `refreshTokenResponse.data.refresh_token = \"refresh_token\";`. This overwrites/corrupts the stored refresh token with an unusable placeholder value, breaking future token refresh attempts for any provider that omits refresh_token on refresh (a common OAuth behavior). Location: packages/app-store/_utils/oauth/parseRefreshTokenResponse.ts:24"
+      },
+      {
+        "check_id": "benchmark.review",
+        "kind": "bug",
+        "line": 9,
+        "path": "packages/app-store/_utils/oauth/parseRefreshTokenResponse.ts",
+        "severity": "high",
+        "source": "shipwright.review",
+        "text": "The computed property keys `[z.string().toString()]` and `[z.string().optional().toString()]` do not create dynamic/wildcard validators as the comments claim; `.toString()` is called on the Zod schema object (not a string), producing a fixed literal key (e.g. \"[object Object]\") for both entries, with the second silently overwriting the first. As a result, `minimumTokenResponseSchema` only actually validates `access_token`, and all other real response fields (expiry, scope, refresh_token, etc.) are stripped out of `.data` by zod's default unknown-key stripping when APP_CREDENTIAL_SHARING_ENABLED is on, causing loss of critical token metadata. Location: packages/app-store/_utils/oauth/parseRefreshTokenResponse.ts:9"
+      },
+      {
+        "check_id": "benchmark.review",
+        "kind": "api",
+        "line": 6,
+        "path": "packages/app-store/_utils/oauth/refreshOAuthTokens.ts",
+        "severity": "high",
+        "source": "shipwright.review",
+        "text": "`refreshOAuthTokens` returns a raw, unparsed `fetch` Response when credential sync is enabled, but returns provider-specific parsed shapes in the local-refresh branch (e.g. `fetchTokens.res` with a `.data` property for Google, or the `HubspotToken` object for Hubspot). Callers such as GoogleCalendarService (`const token = res?.data;`) and HubspotCalendarService (typed as `HubspotToken`) assume a single shape and will fail/throw at runtime (e.g. `token.access_token` on undefined) when APP_CREDENTIAL_SHARING_ENABLED and CALCOM_CREDENTIAL_SYNC_ENDPOINT are set. Location: packages/app-store/_utils/oauth/refreshOAuthTokens.ts:6"
+      },
+      {
+        "check_id": "benchmark.review",
+        "kind": "security",
+        "line": 21,
+        "path": "apps/web/pages/api/webhook/app-credential.ts",
+        "severity": "medium",
+        "source": "shipwright.review",
+        "text": "The webhook secret is compared using strict inequality (`!==`) rather than a constant-time comparison, making the credential-sync webhook endpoint vulnerable to timing attacks that could help an attacker guess CALCOM_WEBHOOK_SECRET. Location: apps/web/pages/api/webhook/app-credential.ts:21"
+      }
     ]
   },
   "https://github.com/calcom/cal.com/pull/7232": {
@@ -49751,6 +50854,44 @@
         "line": null,
         "source": "extracted"
       }
+    ],
+    "shipwright": [
+      {
+        "check_id": "benchmark.review",
+        "kind": "data",
+        "line": 213,
+        "path": "packages/features/ee/workflows/lib/reminders/emailReminderManager.ts",
+        "severity": "high",
+        "source": "shipwright.review",
+        "text": "When `immediateDelete` is true and a `referenceId` exists, `deleteScheduledEmailReminder` only calls the SendGrid cancel API and returns \u2014 it never deletes or updates (sets `cancelled: true`) the `WorkflowReminder` row. This function is called with `immediateDelete=true` from `handleNewBooking.ts` (reschedule cleanup) and `workflows.tsx` (step/workflow deletion), so those WorkflowReminder rows become orphaned in the DB with `scheduled: true`/`cancelled: null` forever, even though the underlying email was actually canceled, which can cause `scheduleEmailReminders.ts`'s cancellation cron (which filters on `cancelled: true`) to never clean them up and can confuse any code that treats these rows as still-active/scheduled reminders. Location: packages/features/ee/workflows/lib/reminders/emailReminderManager.ts:213"
+      },
+      {
+        "check_id": "benchmark.review",
+        "kind": "concurrency",
+        "line": 497,
+        "path": "packages/trpc/server/routers/viewer/bookings.tsx",
+        "severity": "high",
+        "source": "shipwright.review",
+        "text": "The reschedule mutation dropped the `await Promise.all(remindersToDelete)` call that previously awaited the actual deletion of old WorkflowReminder rows. The new code only fires `deleteScheduledEmailReminder`/`deleteScheduledSMSReminder` inside a `forEach` without awaiting them, so the mutation proceeds to create the new booking and schedule new workflow reminders before the old reminders are guaranteed to be cleaned up/canceled, introducing a race condition and swallowing any errors from the cleanup. Location: packages/trpc/server/routers/viewer/bookings.tsx:497"
+      },
+      {
+        "check_id": "benchmark.review",
+        "kind": "security",
+        "line": 378,
+        "path": "packages/trpc/server/routers/viewer/workflows.tsx",
+        "severity": "medium",
+        "source": "shipwright.review",
+        "text": "The previous `deleteMany` for reminders belonging to disabled event types explicitly scoped deletion to `booking: { userId: ctx.user.id }}`, providing an ownership check. The refactor removes this scoping entirely and instead calls `deleteScheduledEmailReminder`/`deleteScheduledSMSReminder`, whose underlying Prisma calls only filter by `id: reminderId` with no ownership/ownership-booking check, weakening the authorization guard that was present before this diff. Location: packages/trpc/server/routers/viewer/workflows.tsx:378"
+      },
+      {
+        "check_id": "benchmark.review",
+        "kind": "concurrency",
+        "line": 484,
+        "path": "packages/features/bookings/lib/handleCancelBooking.ts",
+        "severity": "medium",
+        "source": "shipwright.review",
+        "text": "The guard `if (reminder.scheduled && reminder.referenceId)` and the batched `prisma.workflowReminder.deleteMany` (previously part of the awaited `prismaPromises`) were removed. Reminder cleanup now only happens via unawaited, fire-and-forget calls to `deleteScheduledEmailReminder`/`deleteScheduledSMSReminder` inside the `forEach`, so the cancel-booking handler's response can return to the client before the WorkflowReminder rows are actually canceled/deleted, and any errors from that cleanup are silently swallowed instead of surfacing via `Promise.all`. Location: packages/features/bookings/lib/handleCancelBooking.ts:484"
+      }
     ]
   },
   "https://github.com/calcom/cal.com/pull/14740": {
@@ -51435,6 +52576,53 @@
         "line": null,
         "source": "extracted"
       }
+    ],
+    "shipwright": [
+      {
+        "check_id": "benchmark.review",
+        "kind": "security",
+        "line": 58,
+        "path": "packages/trpc/server/routers/viewer/bookings/addGuests.handler.ts",
+        "severity": "high",
+        "source": "shipwright.review",
+        "text": "Blacklisted guest emails are lowercased when read from env, but the incoming `guest` email is compared against the blacklist without normalization (`!blacklistedGuestEmails.includes(guest)`). A blacklisted email submitted with different casing (e.g. Foo@Bar.com) will bypass the blacklist filter entirely. Location: packages/trpc/server/routers/viewer/bookings/addGuests.handler.ts:58"
+      },
+      {
+        "check_id": "benchmark.review",
+        "kind": "bug",
+        "line": 33,
+        "path": "packages/trpc/server/routers/viewer/bookings/addGuests.handler.ts",
+        "severity": "high",
+        "source": "shipwright.review",
+        "text": "Authorization check uses `&&` instead of `||` between isTeamAdmin and isTeamOwner, requiring a user to be BOTH a team admin AND the team owner to be granted access via this path, rather than either role. This incorrectly denies legitimate team admins (who are not also the owner) permission to add guests. Location: packages/trpc/server/routers/viewer/bookings/addGuests.handler.ts:33"
+      },
+      {
+        "check_id": "benchmark.review",
+        "kind": "bug",
+        "line": 27,
+        "path": "packages/emails/templates/organizer-add-guests-email.ts",
+        "severity": "medium",
+        "source": "shipwright.review",
+        "text": "`this.calEvent.attendees[0].name` is accessed without verifying the attendees array is non-empty, which will throw if a booking somehow has no attendees when this template is rendered. Location: packages/emails/templates/organizer-add-guests-email.ts:27"
+      },
+      {
+        "check_id": "benchmark.review",
+        "kind": "concurrency",
+        "line": 168,
+        "path": "packages/trpc/server/routers/viewer/bookings/addGuests.handler.ts",
+        "severity": "low",
+        "source": "shipwright.review",
+        "text": "Errors from sendAddGuestsEmails are silently swallowed with a bare console.log instead of using the application logger, hiding email delivery failures from monitoring/observability after guests are added. Location: packages/trpc/server/routers/viewer/bookings/addGuests.handler.ts:168"
+      },
+      {
+        "check_id": "benchmark.review",
+        "kind": "bug",
+        "line": 29,
+        "path": "apps/web/components/dialog/AddGuestsDialog.tsx",
+        "severity": "low",
+        "source": "shipwright.review",
+        "text": "The onError handler builds `message` as a template string (\"${err.data?.code}: ${t(err.message)}\") which is always truthy, so the fallback `t('unable_to_add_guests')` in `showToast(message || t('unable_to_add_guests'), 'error')` is unreachable dead code and never displays a translated fallback message. Location: apps/web/components/dialog/AddGuestsDialog.tsx:29"
+      }
     ]
   },
   "https://github.com/calcom/cal.com/pull/10600": {
@@ -53019,6 +54207,35 @@
         "line": null,
         "source": "extracted"
       }
+    ],
+    "shipwright": [
+      {
+        "check_id": "benchmark.review",
+        "kind": "style",
+        "line": 6,
+        "path": "apps/web/components/auth/BackupCode.tsx",
+        "severity": "low",
+        "source": "shipwright.review",
+        "text": "BackupCode.tsx exports a function named `TwoFactor` (identical to the actual TwoFactor.tsx component) instead of `BackupCode`. This is misleading and could easily be confused with the real TwoFactor component when imported elsewhere, especially since both default-export the same identifier name. Location: apps/web/components/auth/BackupCode.tsx:6"
+      },
+      {
+        "check_id": "benchmark.review",
+        "kind": "concurrency",
+        "line": 151,
+        "path": "packages/features/auth/lib/next-auth-options.ts",
+        "severity": "high",
+        "source": "shipwright.review",
+        "text": "Backup code verification in the credentials authorize() flow reads the user's encrypted backupCodes, validates the supplied code against an in-memory array, mutates that array (`backupCodes[index] = null`) and only then persists the change via `prisma.user.update`. Two concurrent login attempts using the same backup code will both read the original (unmodified) encrypted array before either write completes, so both requests will find the code valid and succeed, defeating the intended one-time-use semantics of backup codes and allowing bypass/detection avoidance. Location: packages/features/auth/lib/next-auth-options.ts:151"
+      },
+      {
+        "check_id": "benchmark.review",
+        "kind": "perf",
+        "line": 100,
+        "path": "apps/web/components/settings/EnableTwoFactorModal.tsx",
+        "severity": "medium",
+        "source": "shipwright.review",
+        "text": "A Blob URL is created via `URL.createObjectURL(textBlob)` for the backup-codes download link and stored in `backupCodesUrl`. It is only revoked the next time `handleSetup` runs (`if (backupCodesUrl) URL.revokeObjectURL(backupCodesUrl)`), but `resetState()` \u2014 called both when the user cancels the flow and when the user clicks \"Close\" after viewing/downloading the backup codes \u2014 never revokes this URL. As a result, every completed 2FA setup leaks one Blob URL for the lifetime of the page. Location: apps/web/components/settings/EnableTwoFactorModal.tsx:100"
+      }
     ]
   },
   "https://github.com/calcom/cal.com/pull/10967": {
@@ -55137,6 +56354,53 @@
         "line": null,
         "source": "extracted"
       }
+    ],
+    "shipwright": [
+      {
+        "check_id": "benchmark.review",
+        "kind": "bug",
+        "line": 118,
+        "path": "packages/core/EventManager.ts",
+        "severity": "critical",
+        "source": "shipwright.review",
+        "text": "After destructuring `const [mainHostDestinationCalendar] = evt.destinationCalendar ?? [];`, the code calls `mainHostDestinationCalendar.integration` without any null-check. Since many callers now pass `destinationCalendar: []` (e.g. handleCancelBooking.ts, webhook.ts, paypal-webhook.ts) when there is no destination calendar, `mainHostDestinationCalendar` will be `undefined` and this line throws `TypeError: Cannot read properties of undefined (reading 'integration')` whenever `evt.location === MeetLocationType` and no destination calendar exists. Location: packages/core/EventManager.ts:118"
+      },
+      {
+        "check_id": "benchmark.review",
+        "kind": "bug",
+        "line": 255,
+        "path": "packages/app-store/googlecalendar/lib/CalendarService.ts",
+        "severity": "high",
+        "source": "shipwright.review",
+        "text": "In `updateEvent` the fallback branch computes `event.destinationCalendar?.find((cal) => cal.externalId === externalCalendarId)?.externalId` but this code only runs when `externalCalendarId` is falsy (it's the else-branch of `externalCalendarId ? externalCalendarId : ...`). Comparing `cal.externalId === externalCalendarId` against a falsy value will essentially never match a real calendar entry, so `selectedCalendar` will always resolve to `undefined` instead of falling back to the destination calendar's externalId, breaking calendar update routing. Location: packages/app-store/googlecalendar/lib/CalendarService.ts:255"
+      },
+      {
+        "check_id": "benchmark.review",
+        "kind": "bug",
+        "line": 316,
+        "path": "packages/app-store/googlecalendar/lib/CalendarService.ts",
+        "severity": "high",
+        "source": "shipwright.review",
+        "text": "Same broken fallback pattern as updateEvent: `deleteEvent` computes `calendarId` via `event.destinationCalendar?.find((cal) => cal.externalId === externalCalendarId)?.externalId` inside the branch where `externalCalendarId` is falsy, so the find predicate compares against a falsy value and will never match, making the destination-calendar fallback for delete always resolve to `undefined`. Location: packages/app-store/googlecalendar/lib/CalendarService.ts:316"
+      },
+      {
+        "check_id": "benchmark.review",
+        "kind": "bug",
+        "line": 1083,
+        "path": "packages/features/bookings/lib/handleNewBooking.ts",
+        "severity": "high",
+        "source": "shipwright.review",
+        "text": "`evt.destinationCalendar` is typed to be `null` when neither `eventType.destinationCalendar` nor `organizerUser.destinationCalendar` exist. The subsequent `evt.destinationCalendar?.push(...teamDestinationCalendars)` uses optional chaining, so when `evt.destinationCalendar` is `null` the push silently no-ops and all collected team member destination calendars are dropped, defeating the PR's stated purpose of supporting multiple hosts' destination calendars for collective events. Location: packages/features/bookings/lib/handleNewBooking.ts:1083"
+      },
+      {
+        "check_id": "benchmark.review",
+        "kind": "api",
+        "line": 221,
+        "path": "packages/types/Calendar.d.ts",
+        "severity": "medium",
+        "source": "shipwright.review",
+        "text": "The `Calendar` interface's `createEvent` signature was changed to require a second `credentialId: number` parameter, but only `GoogleCalendarService.createEvent` was updated to accept/use it. `LarkCalendarService.createEvent`, `Office365CalendarService.createEvent`, and the abstract `BaseCalendarService` in packages/lib/CalendarService.ts were not updated, so those providers cannot use the new per-credential destinationCalendar selection this PR introduces, leaving the multi-host destinationCalendar feature only partially implemented across calendar providers. Location: packages/types/Calendar.d.ts:221"
+      }
     ]
   },
   "https://github.com/calcom/cal.com/pull/8087": {
@@ -56505,6 +57769,35 @@
         "line": null,
         "source": "extracted"
       }
+    ],
+    "shipwright": [
+      {
+        "check_id": "benchmark.review",
+        "kind": "concurrency",
+        "line": 125,
+        "path": "packages/app-store/vital/lib/reschedule.ts",
+        "severity": "high",
+        "source": "shipwright.review",
+        "text": "The bookingRefsFiltered.forEach() callback was converted to `async (bookingRef) => {...}` to allow `await getCalendar(...)`, but Array.prototype.forEach does not await its callback. The enclosing try/catch block therefore no longer catches errors thrown inside the async callback (they become unhandled promise rejections), and the surrounding function returns before the calendar deleteEvent/deleteMeeting operations actually complete, creating a race condition when rescheduling. Location: packages/app-store/vital/lib/reschedule.ts:125"
+      },
+      {
+        "check_id": "benchmark.review",
+        "kind": "concurrency",
+        "line": 125,
+        "path": "packages/app-store/wipemycalother/lib/reschedule.ts",
+        "severity": "high",
+        "source": "shipwright.review",
+        "text": "Same fire-and-forget bug as in vital/reschedule.ts: forEach was made async to support `await getCalendar(...)`, but forEach ignores the returned promises, so the try/catch around it no longer catches rejected promises and the function proceeds without waiting for the calendar/meeting deletions to finish. Location: packages/app-store/wipemycalother/lib/reschedule.ts:125"
+      },
+      {
+        "check_id": "benchmark.review",
+        "kind": "concurrency",
+        "line": 553,
+        "path": "packages/trpc/server/routers/viewer/bookings.tsx",
+        "severity": "high",
+        "source": "shipwright.review",
+        "text": "bookingRefsFiltered.forEach() was changed to take an async callback to await getCalendar(...), but forEach does not await async callbacks. This makes calendar/meeting deletion during reschedule fire-and-forget: the handler can proceed (and respond) before old bookings' calendar events are actually deleted, and any thrown errors become unhandled promise rejections instead of being surfaced/caught. Location: packages/trpc/server/routers/viewer/bookings.tsx:553"
+      }
     ]
   },
   "https://github.com/keycloak/keycloak/pull/37429": {
@@ -57997,6 +59290,53 @@
         "line": null,
         "source": "extracted"
       }
+    ],
+    "shipwright": [
+      {
+        "check_id": "benchmark.review",
+        "kind": "bug",
+        "line": 101,
+        "path": "themes/src/main/resources-community/theme/base/account/messages/messages_lt.properties",
+        "severity": "medium",
+        "source": "shipwright.review",
+        "text": "totpStep1 value was replaced with Italian text (\"Installa una delle seguenti applicazioni sul tuo cellulare:\") in the Lithuanian (lt) properties file, so Lithuanian users will see Italian text instead of a Lithuanian translation. Location: themes/src/main/resources-community/theme/base/account/messages/messages_lt.properties:101"
+      },
+      {
+        "check_id": "benchmark.review",
+        "kind": "bug",
+        "line": 71,
+        "path": "themes/src/main/resources-community/theme/base/login/messages/messages_lt.properties",
+        "severity": "medium",
+        "source": "shipwright.review",
+        "text": "loginTotpStep1 value was replaced with Italian text (\"Installa una delle seguenti applicazioni sul tuo cellulare:\") in the Lithuanian (lt) login properties file, producing incorrect/foreign-language UI text for Lithuanian users. Location: themes/src/main/resources-community/theme/base/login/messages/messages_lt.properties:71"
+      },
+      {
+        "check_id": "benchmark.review",
+        "kind": "bug",
+        "line": 112,
+        "path": "themes/src/main/resources-community/theme/base/account/messages/messages_zh_CN.properties",
+        "severity": "medium",
+        "source": "shipwright.review",
+        "text": "totpStep1 was rewritten using Traditional Chinese characters (\u624b\u6a5f, \u61c9\u7528\u7a0b\u5f0f) in the Simplified Chinese (zh_CN) properties file, which should use Simplified Chinese forms (\u624b\u673a, \u5e94\u7528\u7a0b\u5e8f), causing an inconsistent/incorrect translation for zh_CN users. Location: themes/src/main/resources-community/theme/base/account/messages/messages_zh_CN.properties:112"
+      },
+      {
+        "check_id": "benchmark.review",
+        "kind": "bug",
+        "line": 140,
+        "path": "misc/theme-verifier/src/main/java/org/keycloak/themeverifier/VerifyMessageProperties.java",
+        "severity": "medium",
+        "source": "shipwright.review",
+        "text": "ANCHOR_PATTERN regex `</?a[^>]*>` is not properly boundary-anchored to the tag name 'a'; it will also match any tag whose name starts with 'a' (e.g. <abbr>, <article>, <aside>), causing santizeAnchors to misidentify unrelated tags as anchors and potentially emit false \"Didn't find anchor tag\" verifier errors or incorrectly strip unrelated tags. Location: misc/theme-verifier/src/main/java/org/keycloak/themeverifier/VerifyMessageProperties.java:140"
+      },
+      {
+        "check_id": "benchmark.review",
+        "kind": "style",
+        "line": 144,
+        "path": "misc/theme-verifier/src/main/java/org/keycloak/themeverifier/VerifyMessageProperties.java",
+        "severity": "low",
+        "source": "shipwright.review",
+        "text": "Method name `santizeAnchors` is a typo for `sanitizeAnchors`, a misleading identifier introduced by this diff for the new anchor-sanitization logic. Location: misc/theme-verifier/src/main/java/org/keycloak/themeverifier/VerifyMessageProperties.java:144"
+      }
     ]
   },
   "https://github.com/keycloak/keycloak/pull/37634": {
@@ -59218,6 +60558,35 @@
         "line": null,
         "source": "extracted"
       }
+    ],
+    "shipwright": [
+      {
+        "check_id": "benchmark.review",
+        "kind": "bug",
+        "line": 73,
+        "path": "services/src/main/java/org/keycloak/protocol/oidc/encode/AccessTokenContext.java",
+        "severity": "medium",
+        "source": "shipwright.review",
+        "text": "In AccessTokenContext's constructor, the null-check meant for `rawTokenId` mistakenly re-checks `grantType` twice: `Objects.requireNonNull(grantType, \"Null grantType not allowed\"); Objects.requireNonNull(grantType, \"Null rawTokenId not allowed\");`. As a result, a null `rawTokenId` is never rejected (NPE deferred to later usage, e.g. when concatenated in encodeTokenId) and the error message for the missing rawTokenId check is wrong (it will never fire, and if it did it would report the wrong field). Location: services/src/main/java/org/keycloak/protocol/oidc/encode/AccessTokenContext.java:73"
+      },
+      {
+        "check_id": "benchmark.review",
+        "kind": "test_gap",
+        "line": 476,
+        "path": "testsuite/integration-arquillian/tests/base/src/test/java/org/keycloak/testsuite/AssertEvents.java",
+        "severity": "critical",
+        "source": "shipwright.review",
+        "text": "isAccessTokenId() has two bugs that make it not test what it claims: (1) it extracts the grant-type shortcut with `substring(3, 5)` but per the actual encoding format (sessionType[0-2] + tokenType[2-4] + grantType[4-6], as verified by DefaultTokenContextEncoderProviderTest, e.g. \"trltcc:1234\"), the grant shortcut is at index 4-6, not 3-5 \u2014 this reads one character from tokenType and one from grantType. (2) The matching condition is inverted: `if (items[0].substring(3, 5).equals(expectedGrantShortcut)) return false;` returns false (no match) when the substring equals the expected shortcut, and falls through to the UUID check (ignoring the grant check) when it does not equal \u2014 the exact opposite of the intended assertion. This will cause the updated event assertions (expectCodeToToken, expectRefresh, expectDeviceCodeToToken, expectAuthReqIdToToken) to fail for correctly-encoded tokens and to silently ignore the grant type check for incorrect ones. Location: testsuite/integration-arquillian/tests/base/src/test/java/org/keycloak/testsuite/AssertEvents.java:476"
+      },
+      {
+        "check_id": "benchmark.review",
+        "kind": "test_gap",
+        "line": 66,
+        "path": "services/src/test/java/org/keycloak/protocol/oidc/encode/DefaultTokenContextEncoderProviderTest.java",
+        "severity": "medium",
+        "source": "shipwright.review",
+        "text": "testIncorrectGrantType() only manually sets `grantsByShortcuts`/`grantsToShortcuts` in @Before and never calls `factory.postInit(...)`, leaving `sessionFactory` null. When decoding the unregistered shortcut \"ac\", `getGrantTypeByShortcut` falls into the refresh branch and calls `sessionFactory.getProviderFactoriesStream(...)` on a null `sessionFactory`, throwing a NullPointerException rather than the intended `IllegalArgumentException` from the \"Unknown value for grant type\" validation path. The test passes only because it catches the overly broad `RuntimeException`, so it does not actually exercise the described \"incorrect grant type\" validation logic. Location: services/src/test/java/org/keycloak/protocol/oidc/encode/DefaultTokenContextEncoderProviderTest.java:66"
+      }
     ]
   },
   "https://github.com/keycloak/keycloak/pull/38446": {
@@ -60348,6 +61717,17 @@
         "line": null,
         "source": "extracted"
       }
+    ],
+    "shipwright": [
+      {
+        "check_id": "benchmark.review",
+        "kind": "security",
+        "line": 333,
+        "path": "testsuite/integration-arquillian/servers/auth-server/services/testsuite-providers/src/main/java/org/keycloak/testsuite/federation/BackwardsCompatibilityUserStorage.java",
+        "severity": "medium",
+        "source": "shipwright.review",
+        "text": "The new RecoveryAuthnCodesCredentialModel.TYPE branch added to BackwardsCompatibilityUserStorage.isValid() checks whether the submitted code exists in the stored code list via generatedKeys.stream().anyMatch(...), but never removes/marks the matched code as used and never persists an updated credential. This breaks the fundamental one-time-use guarantee of recovery/backup codes: a user (or attacker who intercepted a code) can replay the same recovery code indefinitely to authenticate, since the stored code list is never mutated after a successful match. This file is the PR's reference example of how a custom UserStorageProvider should implement recovery-code support, so the missing invalidation is a concrete, security-relevant logic gap introduced by this diff. Location: testsuite/integration-arquillian/servers/auth-server/services/testsuite-providers/src/main/java/org/keycloak/testsuite/federation/BackwardsCompatibilityUserStorage.java:333"
+      }
     ]
   },
   "https://github.com/keycloak/keycloak/pull/36882": {
@@ -61028,6 +62408,26 @@
         "line": null,
         "source": "extracted"
       }
+    ],
+    "shipwright": [
+      {
+        "check_id": "benchmark.review",
+        "kind": "data",
+        "line": 117,
+        "path": "operator/src/test/java/org/keycloak/operator/testsuite/integration/UpgradeTest.java",
+        "severity": "medium",
+        "source": "shipwright.review",
+        "text": "In createKeycloak's update-strategy helper, `kc.getSpec().getFeatureSpec().setEnabledFeatures(List.of(Profile.Feature.ROLLING_UPDATES.getKey()))` is called unconditionally, even when `getFeatureSpec()` was already non-null (i.e., pre-existing). This unconditionally overwrites any previously configured enabled features on the CR with a list containing only `rolling-updates`, silently discarding other features that may have been enabled earlier by the same or reused Keycloak spec object (the old code only touched the spec when `getUnsupported()` was null and never replaced its contents). Location: operator/src/test/java/org/keycloak/operator/testsuite/integration/UpgradeTest.java:117"
+      },
+      {
+        "check_id": "benchmark.review",
+        "kind": "style",
+        "line": 33,
+        "path": "quarkus/runtime/src/main/java/org/keycloak/quarkus/runtime/compatibility/CompatibilityResult.java",
+        "severity": "low",
+        "source": "shipwright.review",
+        "text": "New constant `FEATURE_DISABLED` breaks the naming convention of its sibling constants `ROLLING_UPGRADE_EXIT_CODE` and `RECREATE_UPGRADE_EXIT_CODE` (both suffixed with `_EXIT_CODE`), making it unclear from the name alone that this value is also used as a CLI exit code (it's passed to `picocli.exit(...)`). Location: quarkus/runtime/src/main/java/org/keycloak/quarkus/runtime/compatibility/CompatibilityResult.java:33"
+      }
     ]
   },
   "https://github.com/keycloak/keycloak/pull/36880": {
@@ -62172,6 +63572,26 @@
         "line": null,
         "source": "extracted"
       }
+    ],
+    "shipwright": [
+      {
+        "check_id": "benchmark.review",
+        "kind": "doc_defect",
+        "line": 76,
+        "path": "services/src/main/java/org/keycloak/services/resources/admin/permissions/ClientPermissionEvaluator.java",
+        "severity": "low",
+        "source": "shipwright.review",
+        "text": "The Javadoc added for the void methods `requireView()` and `requireViewClientScopes()` incorrectly describes them as returning a boolean (\"Returns {@code true} if {@link #canView()} returns {@code true}.\"), copy-pasted from the boolean-returning `canView()`/`canViewClientScopes()` docs. Every other `require*` method in this interface correctly documents \"Throws ForbiddenException if ... returns false\", so this is an inconsistent/misleading doc for these two methods. Location: services/src/main/java/org/keycloak/services/resources/admin/permissions/ClientPermissionEvaluator.java:76"
+      },
+      {
+        "check_id": "benchmark.review",
+        "kind": "doc_defect",
+        "line": 57,
+        "path": "services/src/main/java/org/keycloak/services/resources/admin/permissions/ClientPermissionEvaluator.java",
+        "severity": "low",
+        "source": "shipwright.review",
+        "text": "The newly added Javadoc for `canList()` on `ClientPermissionEvaluator` states the caller can list clients if they have the `AdminRoles#QUERY_USERS` role (in addition to QUERY_CLIENTS). QUERY_USERS is a user-management role and has no documented relation to client listing; this looks like a copy-paste error from the user permission evaluator documentation and misleads reviewers/consumers about the actual authorization requirement for listing clients. Location: services/src/main/java/org/keycloak/services/resources/admin/permissions/ClientPermissionEvaluator.java:57"
+      }
     ]
   }
 }
\ No newline at end of file
diff --git a/offline/results/anthropic_claude-opus-4-5-20251101/dedup_groups.json b/offline/results/anthropic_claude-opus-4-5-20251101/dedup_groups.json
index b2e31ad..251c0a5 100644
--- a/offline/results/anthropic_claude-opus-4-5-20251101/dedup_groups.json
+++ b/offline/results/anthropic_claude-opus-4-5-20251101/dedup_groups.json
@@ -645,6 +645,17 @@
       [
         8
       ]
+    ],
+    "shipwright": [
+      [
+        0
+      ],
+      [
+        1
+      ],
+      [
+        2
+      ]
     ]
   },
   "https://github.com/keycloak/keycloak/pull/33832": {
@@ -1285,6 +1296,17 @@
       [
         6
       ]
+    ],
+    "shipwright": [
+      [
+        0
+      ],
+      [
+        1
+      ],
+      [
+        2
+      ]
     ]
   },
   "https://github.com/ai-code-review-evaluation/keycloak-greptile/pull/1": {
@@ -1807,6 +1829,14 @@
       [
         2
       ]
+    ],
+    "shipwright": [
+      [
+        0
+      ],
+      [
+        1
+      ]
     ]
   },
   "https://github.com/getsentry/sentry/pull/93824": {
@@ -2357,6 +2387,20 @@
       [
         6
       ]
+    ],
+    "shipwright": [
+      [
+        0
+      ],
+      [
+        1
+      ],
+      [
+        2
+      ],
+      [
+        3
+      ]
     ]
   },
   "https://github.com/ai-code-review-evaluation/sentry-greptile/pull/5": {
@@ -3242,6 +3286,17 @@
       [
         15
       ]
+    ],
+    "shipwright": [
+      [
+        0
+      ],
+      [
+        1
+      ],
+      [
+        2
+      ]
     ]
   },
   "https://github.com/ai-code-review-evaluation/sentry-greptile/pull/1": {
@@ -3966,6 +4021,23 @@
       [
         2
       ]
+    ],
+    "shipwright": [
+      [
+        0
+      ],
+      [
+        1
+      ],
+      [
+        2
+      ],
+      [
+        3
+      ],
+      [
+        4
+      ]
     ]
   },
   "https://github.com/grafana/grafana/pull/97529": {
@@ -5252,6 +5324,14 @@
       [
         4
       ]
+    ],
+    "shipwright": [
+      [
+        0
+      ],
+      [
+        1
+      ]
     ]
   },
   "https://github.com/ai-code-review-evaluation/sentry-greptile/pull/2": {
@@ -6125,6 +6205,26 @@
       [
         8
       ]
+    ],
+    "shipwright": [
+      [
+        0
+      ],
+      [
+        1
+      ],
+      [
+        2
+      ],
+      [
+        3
+      ],
+      [
+        4
+      ],
+      [
+        5
+      ]
     ]
   },
   "https://github.com/ai-code-review-evaluation/sentry-greptile/pull/3": {
@@ -6895,6 +6995,20 @@
       [
         5
       ]
+    ],
+    "shipwright": [
+      [
+        0
+      ],
+      [
+        1
+      ],
+      [
+        2
+      ],
+      [
+        3
+      ]
     ]
   },
   "https://github.com/grafana/grafana/pull/103633": {
@@ -7947,6 +8061,14 @@
       [
         5
       ]
+    ],
+    "shipwright": [
+      [
+        0
+      ],
+      [
+        1
+      ]
     ]
   },
   "https://github.com/grafana/grafana/pull/94942": {
@@ -8331,6 +8453,15 @@
         3,
         4
       ]
+    ],
+    "shipwright": [
+      [
+        0
+      ],
+      [
+        1,
+        2
+      ]
     ]
   },
   "https://github.com/grafana/grafana/pull/90045": {
@@ -9182,6 +9313,30 @@
       [
         4
       ]
+    ],
+    "shipwright": [
+      [
+        0
+      ],
+      [
+        1,
+        2
+      ],
+      [
+        3
+      ],
+      [
+        4
+      ],
+      [
+        5
+      ],
+      [
+        6
+      ],
+      [
+        7
+      ]
     ]
   },
   "https://github.com/grafana/grafana/pull/106778": {
@@ -9902,6 +10057,20 @@
       [
         5
       ]
+    ],
+    "shipwright": [
+      [
+        0
+      ],
+      [
+        1
+      ],
+      [
+        2
+      ],
+      [
+        3
+      ]
     ]
   },
   "https://github.com/grafana/grafana/pull/107534": {
@@ -10252,6 +10421,14 @@
       [
         6
       ]
+    ],
+    "shipwright": [
+      [
+        0
+      ],
+      [
+        1
+      ]
     ]
   },
   "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/10": {
@@ -11370,6 +11547,26 @@
       [
         7
       ]
+    ],
+    "shipwright": [
+      [
+        0
+      ],
+      [
+        1
+      ],
+      [
+        2
+      ],
+      [
+        3
+      ],
+      [
+        4
+      ],
+      [
+        5
+      ]
     ]
   },
   "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/7": {
@@ -11997,6 +12194,23 @@
       [
         7
       ]
+    ],
+    "shipwright": [
+      [
+        0
+      ],
+      [
+        1
+      ],
+      [
+        2
+      ],
+      [
+        3
+      ],
+      [
+        4
+      ]
     ]
   },
   "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/8": {
@@ -12910,6 +13124,17 @@
       [
         4
       ]
+    ],
+    "shipwright": [
+      [
+        0
+      ],
+      [
+        1
+      ],
+      [
+        2
+      ]
     ]
   },
   "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/3": {
@@ -13502,6 +13727,14 @@
       [
         6
       ]
+    ],
+    "shipwright": [
+      [
+        0
+      ],
+      [
+        1
+      ]
     ]
   },
   "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/5": {
@@ -13972,6 +14205,14 @@
       [
         3
       ]
+    ],
+    "shipwright": [
+      [
+        0
+      ],
+      [
+        1
+      ]
     ]
   },
   "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/6": {
@@ -14463,6 +14704,17 @@
       [
         1
       ]
+    ],
+    "shipwright": [
+      [
+        0
+      ],
+      [
+        1
+      ],
+      [
+        2
+      ]
     ]
   },
   "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/4": {
@@ -15734,6 +15986,23 @@
       [
         13
       ]
+    ],
+    "shipwright": [
+      [
+        0
+      ],
+      [
+        1
+      ],
+      [
+        2
+      ],
+      [
+        3
+      ],
+      [
+        4
+      ]
     ]
   },
   "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/1": {
@@ -16415,6 +16684,20 @@
       [
         3
       ]
+    ],
+    "shipwright": [
+      [
+        0
+      ],
+      [
+        1
+      ],
+      [
+        2
+      ],
+      [
+        3
+      ]
     ]
   },
   "https://github.com/calcom/cal.com/pull/22532": {
@@ -17152,6 +17435,14 @@
       [
         4
       ]
+    ],
+    "shipwright": [
+      [
+        0
+      ],
+      [
+        1
+      ]
     ]
   },
   "https://github.com/calcom/cal.com/pull/8330": {
@@ -17821,6 +18112,14 @@
       [
         5
       ]
+    ],
+    "shipwright": [
+      [
+        0
+      ],
+      [
+        1
+      ]
     ]
   },
   "https://github.com/calcom/cal.com/pull/14943": {
@@ -18204,6 +18503,14 @@
       [
         1
       ]
+    ],
+    "shipwright": [
+      [
+        0
+      ],
+      [
+        1
+      ]
     ]
   },
   "https://github.com/calcom/cal.com/pull/22345": {
@@ -19793,6 +20100,23 @@
       [
         4
       ]
+    ],
+    "shipwright": [
+      [
+        0
+      ],
+      [
+        1
+      ],
+      [
+        2
+      ],
+      [
+        3
+      ],
+      [
+        4
+      ]
     ]
   },
   "https://github.com/calcom/cal.com/pull/7232": {
@@ -20549,6 +20873,20 @@
       [
         4
       ]
+    ],
+    "shipwright": [
+      [
+        0
+      ],
+      [
+        1
+      ],
+      [
+        2
+      ],
+      [
+        3
+      ]
     ]
   },
   "https://github.com/calcom/cal.com/pull/14740": {
@@ -21468,6 +21806,23 @@
       [
         4
       ]
+    ],
+    "shipwright": [
+      [
+        0
+      ],
+      [
+        1
+      ],
+      [
+        2
+      ],
+      [
+        3
+      ],
+      [
+        4
+      ]
     ]
   },
   "https://github.com/calcom/cal.com/pull/10967": {
@@ -22537,6 +22892,21 @@
       [
         11
       ]
+    ],
+    "shipwright": [
+      [
+        0
+      ],
+      [
+        1,
+        2
+      ],
+      [
+        3
+      ],
+      [
+        4
+      ]
     ]
   },
   "https://github.com/calcom/cal.com/pull/8087": {
@@ -23162,6 +23532,17 @@
       [
         4
       ]
+    ],
+    "shipwright": [
+      [
+        0
+      ],
+      [
+        1
+      ],
+      [
+        2
+      ]
     ]
   },
   "https://github.com/keycloak/keycloak/pull/37634": {
@@ -23810,6 +24191,17 @@
       [
         6
       ]
+    ],
+    "shipwright": [
+      [
+        0
+      ],
+      [
+        1
+      ],
+      [
+        2
+      ]
     ]
   },
   "https://github.com/keycloak/keycloak/pull/36880": {
@@ -24401,6 +24793,14 @@
       [
         4
       ]
+    ],
+    "shipwright": [
+      [
+        0
+      ],
+      [
+        1
+      ]
     ]
   },
   "https://github.com/keycloak/keycloak/pull/40940": {
@@ -24841,6 +25241,14 @@
       [
         7
       ]
+    ],
+    "shipwright": [
+      [
+        0
+      ],
+      [
+        1
+      ]
     ]
   },
   "https://github.com/keycloak/keycloak/pull/32918": {
@@ -25211,6 +25619,14 @@
       [
         3
       ]
+    ],
+    "shipwright": [
+      [
+        0
+      ],
+      [
+        1
+      ]
     ]
   },
   "https://github.com/grafana/grafana/pull/79265": {
@@ -25873,6 +26289,14 @@
       [
         4
       ]
+    ],
+    "shipwright": [
+      [
+        0
+      ],
+      [
+        1
+      ]
     ]
   },
   "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/2": {
@@ -26535,6 +26959,20 @@
       [
         5
       ]
+    ],
+    "shipwright": [
+      [
+        0
+      ],
+      [
+        1
+      ],
+      [
+        2
+      ],
+      [
+        3
+      ]
     ]
   },
   "https://github.com/calcom/cal.com/pull/10600": {
@@ -27367,6 +27805,17 @@
       [
         7
       ]
+    ],
+    "shipwright": [
+      [
+        0
+      ],
+      [
+        1
+      ],
+      [
+        2
+      ]
     ]
   },
   "https://github.com/grafana/grafana/pull/80329": {
@@ -27945,6 +28394,14 @@
       [
         4
       ]
+    ],
+    "shipwright": [
+      [
+        0
+      ],
+      [
+        1
+      ]
     ]
   },
   "https://github.com/keycloak/keycloak/pull/37429": {
@@ -28665,6 +29122,23 @@
       [
         3
       ]
+    ],
+    "shipwright": [
+      [
+        0
+      ],
+      [
+        1
+      ],
+      [
+        2
+      ],
+      [
+        3
+      ],
+      [
+        4
+      ]
     ]
   },
   "https://github.com/keycloak/keycloak/pull/38446": {
@@ -29675,6 +30149,14 @@
       [
         5
       ]
+    ],
+    "shipwright": [
+      [
+        0
+      ],
+      [
+        1
+      ]
     ]
   },
   "https://github.com/getsentry/sentry/pull/77754": {
@@ -30053,6 +30535,17 @@
       [
         5
       ]
+    ],
+    "shipwright": [
+      [
+        0
+      ],
+      [
+        1
+      ],
+      [
+        2
+      ]
     ]
   },
   "https://github.com/grafana/grafana/pull/90939": {
@@ -30952,6 +31445,14 @@
       [
         4
       ]
+    ],
+    "shipwright": [
+      [
+        0
+      ],
+      [
+        1
+      ]
     ]
   },
   "https://github.com/grafana/grafana/pull/76186": {
@@ -31299,6 +31800,14 @@
       [
         2
       ]
+    ],
+    "shipwright": [
+      [
+        0
+      ],
+      [
+        1
+      ]
     ]
   }
 }
\ No newline at end of file
diff --git a/offline/results/anthropic_claude-opus-4-5-20251101/evaluations.json b/offline/results/anthropic_claude-opus-4-5-20251101/evaluations.json
index 709a9b7..c01c22a 100644
--- a/offline/results/anthropic_claude-opus-4-5-20251101/evaluations.json
+++ b/offline/results/anthropic_claude-opus-4-5-20251101/evaluations.json
@@ -2655,6 +2655,67 @@
       "tool": "sourcery",
       "repo_name": "keycloak__keycloak__sourcery__PR37429__20260307",
       "pr_url": "https://github.com/code-review-benchmark/keycloak__keycloak__sourcery__PR37429__20260307/pull/1"
+    },
+    "shipwright": {
+      "skipped": false,
+      "true_positives": [
+        {
+          "golden_comment": "The translation is in Italian instead of Lithuanian. This should be translated to Lithuanian to match the file's locale (messages_lt.properties).",
+          "severity": "Medium",
+          "category": "doc_defect",
+          "matched_candidate": "totpStep1 value was replaced with Italian text (\"Installa una delle seguenti applicazioni sul tuo cellulare:\") in the Lithuanian (lt) properties file, so Lithuanian users will see Italian text instead of a Lithuanian translation. Location: themes/src/main/resources-community/theme/base/account/messages/messages_lt.properties:101",
+          "confidence": 1.0,
+          "reasoning": "The candidate issue identifies exactly the same problem as the golden comment - that Italian text was incorrectly placed in the Lithuanian (lt) properties file instead of a proper Lithuanian translation. Both comments point to the same bug: wrong language (Italian) in the Lithuanian locale file."
+        },
+        {
+          "golden_comment": "The totpStep1 value uses Traditional Chinese terms in the Simplified Chinese file (zh_CN), which is likely incorrect for this locale. Please verify the locale\u2011appropriate translation.",
+          "severity": "Medium",
+          "category": "doc_defect",
+          "matched_candidate": "totpStep1 was rewritten using Traditional Chinese characters (\u624b\u6a5f, \u61c9\u7528\u7a0b\u5f0f) in the Simplified Chinese (zh_CN) properties file, which should use Simplified Chinese forms (\u624b\u673a, \u5e94\u7528\u7a0b\u5e8f), causing an inconsistent/incorrect translation for zh_CN users. Location: themes/src/main/resources-community/theme/base/account/messages/messages_zh_CN.properties:112",
+          "confidence": 0.98,
+          "reasoning": "Both comments identify the same issue: the totpStep1 value in the zh_CN (Simplified Chinese) properties file incorrectly uses Traditional Chinese characters instead of Simplified Chinese. The candidate issue provides more specific details (mentioning the exact characters like \u624b\u6a5f vs \u624b\u673a) and the file location, but it's addressing the exact same translation locale mismatch problem as the golden comment."
+        },
+        {
+          "golden_comment": "The method name 'santizeAnchors' should be 'sanitizeAnchors' (missing 'i').",
+          "severity": "Low",
+          "category": "style",
+          "matched_candidate": "Method name `santizeAnchors` is a typo for `sanitizeAnchors`, a misleading identifier introduced by this diff for the new anchor-sanitization logic. Location: misc/theme-verifier/src/main/java/org/keycloak/themeverifier/VerifyMessageProperties.java:144",
+          "confidence": 1.0,
+          "reasoning": "The candidate issue identifies exactly the same problem as the golden comment - the method name 'santizeAnchors' is misspelled and should be 'sanitizeAnchors' (missing the letter 'i'). Both comments point to the same typo in the same method name."
+        }
+      ],
+      "false_positives": [
+        {
+          "candidate": "loginTotpStep1 value was replaced with Italian text (\"Installa una delle seguenti applicazioni sul tuo cellulare:\") in the Lithuanian (lt) login properties file, producing incorrect/foreign-language UI text for Lithuanian users. Location: themes/src/main/resources-community/theme/base/login/messages/messages_lt.properties:71"
+        },
+        {
+          "candidate": "ANCHOR_PATTERN regex `</?a[^>]*>` is not properly boundary-anchored to the tag name 'a'; it will also match any tag whose name starts with 'a' (e.g. <abbr>, <article>, <aside>), causing santizeAnchors to misidentify unrelated tags as anchors and potentially emit false \"Didn't find anchor tag\" verifier errors or incorrectly strip unrelated tags. Location: misc/theme-verifier/src/main/java/org/keycloak/themeverifier/VerifyMessageProperties.java:140"
+        }
+      ],
+      "false_negatives": [
+        {
+          "golden_comment": "The anchor sanitization logic has a potential issue where it consumes English matcher groups without proper validation. If the translated text has more anchor tags than the English text, this could lead to incorrect validation results.",
+          "severity": "Low",
+          "category": "security"
+        },
+        {
+          "golden_comment": "Field HTML_TAGS should be declared as 'private static final' following Java naming conventions for constants",
+          "severity": "Low",
+          "category": "style"
+        }
+      ],
+      "errors": [],
+      "total_candidates": 5,
+      "total_golden": 5,
+      "tp": 3,
+      "fp": 2,
+      "fn": 2,
+      "errors_count": 0,
+      "precision": 0.6,
+      "recall": 0.6,
+      "tool": "shipwright",
+      "repo_name": "shipwright__https://github.com/keycloak/keycloak/pull/37429",
+      "pr_url": "https://github.com/keycloak/keycloak/pull/37429"
     }
   },
   "https://github.com/keycloak/keycloak/pull/37634": {
@@ -5079,6 +5140,55 @@
       "tool": "sourcery",
       "repo_name": "keycloak__keycloak__sourcery__PR37634__20260307",
       "pr_url": "https://github.com/code-review-benchmark/keycloak__keycloak__sourcery__PR37634__20260307/pull/1"
+    },
+    "shipwright": {
+      "skipped": false,
+      "true_positives": [
+        {
+          "golden_comment": "In AccessTokenContext constructor, the fourth Objects.requireNonNull validates grantType twice instead of validating rawTokenId, allowing null rawTokenId values to pass validation.",
+          "severity": "Critical",
+          "category": "bug",
+          "matched_candidate": "In AccessTokenContext's constructor, the null-check meant for `rawTokenId` mistakenly re-checks `grantType` twice: `Objects.requireNonNull(grantType, \"Null grantType not allowed\"); Objects.requireNonNull(grantType, \"Null rawTokenId not allowed\");`. As a result, a null `rawTokenId` is never rejected (NPE deferred to later usage, e.g. when concatenated in encodeTokenId) and the error message for the missing rawTokenId check is wrong (it will never fire, and if it did it would report the wrong field). Location: services/src/main/java/org/keycloak/protocol/oidc/encode/AccessTokenContext.java:73",
+          "confidence": 1.0,
+          "reasoning": "The candidate issue identifies exactly the same bug as the golden comment: in the AccessTokenContext constructor, the fourth Objects.requireNonNull call validates grantType twice instead of validating rawTokenId, which allows null rawTokenId values to pass validation. The candidate even provides the specific code showing the duplicate grantType check and explains the consequence (null rawTokenId not being rejected)."
+        },
+        {
+          "golden_comment": "In isAccessTokenId, the substring for the grant shortcut and the equality check look inverted: the grant shortcut occupies indices 4\u20135 (substring(4,6)), and a match should return true (combined with UUID check), not false.",
+          "severity": "High",
+          "category": "bug",
+          "matched_candidate": "isAccessTokenId() has two bugs that make it not test what it claims: (1) it extracts the grant-type shortcut with `substring(3, 5)` but per the actual encoding format (sessionType[0-2] + tokenType[2-4] + grantType[4-6], as verified by DefaultTokenContextEncoderProviderTest, e.g. \"trltcc:1234\"), the grant shortcut is at index 4-6, not 3-5 \u2014 this reads one character from tokenType and one from grantType. (2) The matching condition is inverted: `if (items[0].substring(3, 5).equals(expectedGrantShortcut)) return false;` returns false (no match) when the substring equals the expected shortcut, and falls through to the UUID check (ignoring the grant check) when it does not equal \u2014 the exact opposite of the intended assertion. This will cause the updated event assertions (expectCodeToToken, expectRefresh, expectDeviceCodeToToken, expectAuthReqIdToToken) to fail for correctly-encoded tokens and to silently ignore the grant type check for incorrect ones. Location: testsuite/integration-arquillian/tests/base/src/test/java/org/keycloak/testsuite/AssertEvents.java:476",
+          "confidence": 0.95,
+          "reasoning": "The candidate issue identifies the same two bugs as the golden comment: (1) the substring indices are wrong - should be substring(4,6) for the grant shortcut, not substring(3,5), and (2) the equality check logic is inverted - returning false when it should return true (match). Both comments point to the exact same problems in the isAccessTokenId method, just with different levels of detail in the explanation."
+        },
+        {
+          "golden_comment": " Catching generic RuntimeException is too broad. The implementation throws IllegalArgumentException specifically - catch that instead for more precise testing.",
+          "severity": "Low",
+          "category": "bug",
+          "matched_candidate": "testIncorrectGrantType() only manually sets `grantsByShortcuts`/`grantsToShortcuts` in @Before and never calls `factory.postInit(...)`, leaving `sessionFactory` null. When decoding the unregistered shortcut \"ac\", `getGrantTypeByShortcut` falls into the refresh branch and calls `sessionFactory.getProviderFactoriesStream(...)` on a null `sessionFactory`, throwing a NullPointerException rather than the intended `IllegalArgumentException` from the \"Unknown value for grant type\" validation path. The test passes only because it catches the overly broad `RuntimeException`, so it does not actually exercise the described \"incorrect grant type\" validation logic. Location: services/src/test/java/org/keycloak/protocol/oidc/encode/DefaultTokenContextEncoderProviderTest.java:66",
+          "confidence": 0.9,
+          "reasoning": "The candidate issue identifies the same core problem as the golden comment - that the test catches RuntimeException which is too broad. The candidate goes further to explain WHY this is problematic (the test actually throws NullPointerException instead of the intended IllegalArgumentException), but both comments point to the same fundamental issue: catching RuntimeException is too broad and masks the actual behavior. The golden comment says to catch IllegalArgumentException specifically, and the candidate explains that the current broad catch allows the test to pass even though it's not testing what it claims to test."
+        }
+      ],
+      "false_positives": [],
+      "false_negatives": [
+        {
+          "golden_comment": "Javadoc mentions \"usually like 3-letters shortcut\" but some implementations use 2-letter shortcuts (\"ac\", \"cc\", \"rt\", \"te\", \"pc\", \"ci\", \"ro\"). Consider updating documentation to reflect actual usage pattern.",
+          "severity": "Low",
+          "category": "doc_defect"
+        }
+      ],
+      "errors": [],
+      "total_candidates": 3,
+      "total_golden": 4,
+      "tp": 3,
+      "fp": 0,
+      "fn": 1,
+      "errors_count": 0,
+      "precision": 1.0,
+      "recall": 0.75,
+      "tool": "shipwright",
+      "repo_name": "shipwright__https://github.com/keycloak/keycloak/pull/37634",
+      "pr_url": "https://github.com/keycloak/keycloak/pull/37634"
     }
   },
   "https://github.com/keycloak/keycloak/pull/38446": {
@@ -7036,6 +7146,39 @@
       "tool": "sourcery",
       "repo_name": "keycloak__keycloak__sourcery__PR38446__20260307",
       "pr_url": "https://github.com/code-review-benchmark/keycloak__keycloak__sourcery__PR38446__20260307/pull/1"
+    },
+    "shipwright": {
+      "skipped": false,
+      "true_positives": [],
+      "false_positives": [
+        {
+          "candidate": "The new RecoveryAuthnCodesCredentialModel.TYPE branch added to BackwardsCompatibilityUserStorage.isValid() checks whether the submitted code exists in the stored code list via generatedKeys.stream().anyMatch(...), but never removes/marks the matched code as used and never persists an updated credential. This breaks the fundamental one-time-use guarantee of recovery/backup codes: a user (or attacker who intercepted a code) can replay the same recovery code indefinitely to authenticate, since the stored code list is never mutated after a successful match. This file is the PR's reference example of how a custom UserStorageProvider should implement recovery-code support, so the missing invalidation is a concrete, security-relevant logic gap introduced by this diff. Location: testsuite/integration-arquillian/servers/auth-server/services/testsuite-providers/src/main/java/org/keycloak/testsuite/federation/BackwardsCompatibilityUserStorage.java:333"
+        }
+      ],
+      "false_negatives": [
+        {
+          "golden_comment": "Unsafe raw List deserialization without type safety. Calling Optional.get() directly on the Optional returned by RecoveryAuthnCodesUtils.getCredential(user) without checking isPresent() can lead to a NoSuchElementException if the Optional is empty.",
+          "severity": "Medium",
+          "category": "bug"
+        },
+        {
+          "golden_comment": "After creating the RecoveryAuthnCodesCredentialModel, consider setting its id from the stored credential (e.g., myUser.recoveryCodes.getId()); otherwise getId() will be null and downstream removal by id (e.g., removeStoredCredentialById in the authenticator flow) may not work.",
+          "severity": "Low",
+          "category": "bug"
+        }
+      ],
+      "errors": [],
+      "total_candidates": 1,
+      "total_golden": 2,
+      "tp": 0,
+      "fp": 1,
+      "fn": 2,
+      "errors_count": 0,
+      "precision": 0.0,
+      "recall": 0.0,
+      "tool": "shipwright",
+      "repo_name": "shipwright__https://github.com/keycloak/keycloak/pull/38446",
+      "pr_url": "https://github.com/keycloak/keycloak/pull/38446"
     }
   },
   "https://github.com/keycloak/keycloak/pull/36882": {
@@ -8632,6 +8775,42 @@
       "tool": "sourcery",
       "repo_name": "keycloak__keycloak__sourcery__PR36882__20260307",
       "pr_url": "https://github.com/code-review-benchmark/keycloak__keycloak__sourcery__PR36882__20260307/pull/1"
+    },
+    "shipwright": {
+      "skipped": false,
+      "true_positives": [],
+      "false_positives": [
+        {
+          "candidate": "In createKeycloak's update-strategy helper, `kc.getSpec().getFeatureSpec().setEnabledFeatures(List.of(Profile.Feature.ROLLING_UPDATES.getKey()))` is called unconditionally, even when `getFeatureSpec()` was already non-null (i.e., pre-existing). This unconditionally overwrites any previously configured enabled features on the CR with a list containing only `rolling-updates`, silently discarding other features that may have been enabled earlier by the same or reused Keycloak spec object (the old code only touched the spec when `getUnsupported()` was null and never replaced its contents). Location: operator/src/test/java/org/keycloak/operator/testsuite/integration/UpgradeTest.java:117"
+        },
+        {
+          "candidate": "New constant `FEATURE_DISABLED` breaks the naming convention of its sibling constants `ROLLING_UPGRADE_EXIT_CODE` and `RECREATE_UPGRADE_EXIT_CODE` (both suffixed with `_EXIT_CODE`), making it unclear from the name alone that this value is also used as a CLI exit code (it's passed to `picocli.exit(...)`). Location: quarkus/runtime/src/main/java/org/keycloak/quarkus/runtime/compatibility/CompatibilityResult.java:33"
+        }
+      ],
+      "false_negatives": [
+        {
+          "golden_comment": "Incorrect method call for exit codes. The picocli.exit() method calls System.exit() directly, which is problematic:",
+          "severity": "Medium",
+          "category": "bug"
+        },
+        {
+          "golden_comment": "Need to add changelog/release notes documenting the exit code change and migration guidance",
+          "severity": "Low",
+          "category": "doc_defect"
+        }
+      ],
+      "errors": [],
+      "total_candidates": 2,
+      "total_golden": 2,
+      "tp": 0,
+      "fp": 2,
+      "fn": 2,
+      "errors_count": 0,
+      "precision": 0.0,
+      "recall": 0.0,
+      "tool": "shipwright",
+      "repo_name": "shipwright__https://github.com/keycloak/keycloak/pull/36882",
+      "pr_url": "https://github.com/keycloak/keycloak/pull/36882"
     }
   },
   "https://github.com/keycloak/keycloak/pull/36880": {
@@ -11029,6 +11208,57 @@
       "tool": "sourcery",
       "repo_name": "keycloak__keycloak__sourcery__PR36880__20260307",
       "pr_url": "https://github.com/code-review-benchmark/keycloak__keycloak__sourcery__PR36880__20260307/pull/1"
+    },
+    "shipwright": {
+      "skipped": false,
+      "true_positives": [],
+      "false_positives": [
+        {
+          "candidate": "The Javadoc added for the void methods `requireView()` and `requireViewClientScopes()` incorrectly describes them as returning a boolean (\"Returns {@code true} if {@link #canView()} returns {@code true}.\"), copy-pasted from the boolean-returning `canView()`/`canViewClientScopes()` docs. Every other `require*` method in this interface correctly documents \"Throws ForbiddenException if ... returns false\", so this is an inconsistent/misleading doc for these two methods. Location: services/src/main/java/org/keycloak/services/resources/admin/permissions/ClientPermissionEvaluator.java:76"
+        },
+        {
+          "candidate": "The newly added Javadoc for `canList()` on `ClientPermissionEvaluator` states the caller can list clients if they have the `AdminRoles#QUERY_USERS` role (in addition to QUERY_CLIENTS). QUERY_USERS is a user-management role and has no documented relation to client listing; this looks like a copy-paste error from the user permission evaluator documentation and misleads reviewers/consumers about the actual authorization requirement for listing clients. Location: services/src/main/java/org/keycloak/services/resources/admin/permissions/ClientPermissionEvaluator.java:57"
+        }
+      ],
+      "false_negatives": [
+        {
+          "golden_comment": "Inconsistent feature flag bug causing orphaned permissions. The AdminPermissions event listener, responsible for cleaning up permissions upon role, client, or group removal, is incorrectly guarded by the ADMIN_FINE_GRAINED_AUTHZ (V1) feature flag. This is inconsistent with other methods in the class that use ADMIN_FINE_GRAINED_AUTHZ_V2. Consequently, if ADMIN_FINE_GRAINED_AUTHZ_V2 is enabled but V1 is not, the permission cleanup logic will not execute, leading to orphaned permission data. Cleanup should occur regardless of which fine-grained authorization version is enabled.",
+          "severity": "High",
+          "category": "bug"
+        },
+        {
+          "golden_comment": "In hasPermission(ClientModel client, String scope), the resource lookup uses findByName(server, client.getId(), server.getId()), but AdminPermissionsSchema.getOrCreateResource creates per-client resources with the owner set to resourceServer.getClientId(), so this lookup will never find those resources and will always fall back to the 'all-clients' resource, effectively ignoring client-specific permissions.",
+          "severity": "High",
+          "category": "bug"
+        },
+        {
+          "golden_comment": "In getClientsWithPermission(String scope), iterating resourceStore.findByType(server, AdminPermissionsSchema.CLIENTS_RESOURCE_TYPE) and returning resource.getName() will only ever consider the type-level 'Clients' resource (per-client resources have no type) and return its name, while AvailableRoleMappingResource#getRoleIdsWithPermissions expects actual client IDs to pass to realm.getClientById, which can lead to incorrect behavior or a null client and subsequent failures.",
+          "severity": "High",
+          "category": "bug"
+        },
+        {
+          "golden_comment": "In ClientPermissionsV2.hasPermission, the method getResourceTypeResource could return null, leading to a NullPointerException in findByResource. Defensive null checks are needed to prevent this issue.",
+          "severity": "High",
+          "category": "bug"
+        },
+        {
+          "golden_comment": "Remove unused getEvaluationContext private method in ClientPermissionsV2 - it is defined but never called",
+          "severity": "Low",
+          "category": "style"
+        }
+      ],
+      "errors": [],
+      "total_candidates": 2,
+      "total_golden": 5,
+      "tp": 0,
+      "fp": 2,
+      "fn": 5,
+      "errors_count": 0,
+      "precision": 0.0,
+      "recall": 0.0,
+      "tool": "shipwright",
+      "repo_name": "shipwright__https://github.com/keycloak/keycloak/pull/36880",
+      "pr_url": "https://github.com/keycloak/keycloak/pull/36880"
     }
   },
   "https://github.com/keycloak/keycloak/pull/37038": {
@@ -13020,6 +13250,45 @@
       "tool": "sourcery",
       "repo_name": "keycloak__keycloak__sourcery__PR37038__20260307",
       "pr_url": "https://github.com/code-review-benchmark/keycloak__keycloak__sourcery__PR37038__20260307/pull/1"
+    },
+    "shipwright": {
+      "skipped": false,
+      "true_positives": [
+        {
+          "golden_comment": "In GroupPermissionsV2.canManage(), the permission fallback checks for both VIEW and MANAGE scopes (hasPermission(null, AdminPermissionsSchema.VIEW, AdminPermissionsSchema.MANAGE)), but canManage() should only grant access for MANAGE scope, not VIEW. This means users with only VIEW permission on groups can pass the canManage() check, granting them write access they should not have. The canView() method correctly checks both VIEW and MANAGE, but canManage() should only check MANAGE.",
+          "severity": "High",
+          "category": "bug",
+          "matched_candidate": "In the new GroupPermissionsV2, the global (no-arg) canManage() incorrectly evaluates `hasPermission(null, AdminPermissionsSchema.VIEW, AdminPermissionsSchema.MANAGE)` \u2014 identical to canView(). This means an admin who only holds the VIEW scope permission on the 'all groups' resource (and not MANAGE) will have canManage() return true, which is used by requireManage()/GroupsResource create-group and other manage-gated endpoints, granting create/update/delete capability on groups to a user who was only granted read access. The per-group overload canManage(GroupModel) correctly checks only AdminPermissionsSchema.MANAGE, confirming the global variant is an introduced copy/paste bug. Location: services/src/main/java/org/keycloak/services/resources/admin/permissions/GroupPermissionsV2.java:56",
+          "confidence": 1.0,
+          "reasoning": "The candidate issue identifies exactly the same bug as the golden comment: in GroupPermissionsV2.canManage(), the permission check incorrectly includes both VIEW and MANAGE scopes (hasPermission(null, AdminPermissionsSchema.VIEW, AdminPermissionsSchema.MANAGE)), when it should only check MANAGE. Both comments explain that this allows users with only VIEW permission to pass the canManage() check, granting them write access they shouldn't have. The candidate even provides additional context about the per-group overload being correct and identifies the specific line number."
+        },
+        {
+          "golden_comment": "In getGroupIdsWithViewPermission, hasPermission is called with groupResource.getId() and the same groupResource.getId() is added to granted, but hasPermission resolves resources by name (treating the argument as a group id) and the GroupPermissionEvaluator contract says this method returns group IDs that are later used as UserModel.GROUPS and in getUsersCount group filters. This mismatch means per-group VIEW_MEMBERS/MANAGE_MEMBERS permissions may not yield the expected group IDs for filtering and counts, and evaluation may effectively only look at the type-level 'all-groups' resource; consider revisiting whether this should operate on the underlying group ids (resource names) instead so it aligns with the JPA queries and the interface contract.",
+          "severity": "High",
+          "category": "api",
+          "matched_candidate": "In GroupPermissionsV2.getGroupIdsWithViewPermission(), the code iterates Groups-type Resources and adds `groupResource.getId()` to the granted set. However, per AdminPermissionsSchema.getOrCreateResource()/resolveGroup() and this same class's hasPermission() (which looks up group resources via `resourceStore.findByName(server, groupId)`), the actual group ID is stored as the Resource's *name*, not its internal store id. Collecting `.getId()` instead of `.getName()` returns internal authorization-resource UUIDs that never match any real GroupModel id. The resulting set is later used as `session.setAttribute(UserModel.GROUPS, groupIds)` to restrict user search results by group membership, so this bug makes the entire group-based partial-view feature non-functional (it will filter against nonexistent group ids, effectively returning no users instead of the intended restricted set). Location: services/src/main/java/org/keycloak/services/resources/admin/permissions/GroupPermissionsV2.java:102",
+          "confidence": 0.95,
+          "reasoning": "Both comments identify the same core issue: in getGroupIdsWithViewPermission(), the code uses groupResource.getId() (the internal authorization resource UUID) when it should be using the group's actual ID (stored as the resource's name). Both explain that hasPermission() resolves resources by name (treating the argument as a group id), and that this mismatch causes the group-based filtering to fail because the collected IDs don't match real GroupModel IDs. The golden comment is more nuanced about the implications, while the candidate is more direct about calling it a bug, but they're pointing to the exact same underlying problem."
+        }
+      ],
+      "false_positives": [
+        {
+          "candidate": "In UsersResource.searchForUser(), the previous `.filter(usersEvaluator::canView)` post-filter on the searched user stream was removed. Now, if an admin has no `canView()` (users) and no group-based view permission (getGroupIdsWithViewPermission() returns an empty set, which happens whenever the admin has zero group-level view/manage-members grants, e.g. when access is granted via a user-specific permission instead of a group permission), the `session.setAttribute(UserModel.GROUPS, ...)` restriction is never applied and the raw, unfiltered `searchForUserStream` result is returned to the caller with no per-user authorization check at all. This is an authorization bypass: such an admin will see every user in the realm instead of only the users they are permitted to view. Location: services/src/main/java/org/keycloak/services/resources/admin/UsersResource.java:448"
+        }
+      ],
+      "false_negatives": [],
+      "errors": [],
+      "total_candidates": 3,
+      "total_golden": 2,
+      "tp": 2,
+      "fp": 1,
+      "fn": 0,
+      "errors_count": 0,
+      "precision": 0.6666666666666666,
+      "recall": 1.0,
+      "tool": "shipwright",
+      "repo_name": "shipwright__https://github.com/keycloak/keycloak/pull/37038",
+      "pr_url": "https://github.com/keycloak/keycloak/pull/37038"
     }
   },
   "https://github.com/keycloak/keycloak/pull/33832": {
@@ -15234,6 +15503,53 @@
       "tool": "sourcery",
       "repo_name": "keycloak__keycloak__sourcery__PR33832__20260307",
       "pr_url": "https://github.com/code-review-benchmark/keycloak__keycloak__sourcery__PR33832__20260307/pull/1"
+    },
+    "shipwright": {
+      "skipped": false,
+      "true_positives": [
+        {
+          "golden_comment": "Returns wrong provider (default keystore instead of BouncyCastle)",
+          "severity": "High",
+          "category": "bug",
+          "matched_candidate": "AuthzClientCryptoProvider.getBouncyCastleProvider() is documented/named to return the Bouncy Castle security Provider (per the CryptoProvider interface contract used by other implementations, which return BouncyIntegration.PROVIDER), but this implementation instead returns the Provider of the default KeyStore type (e.g. JKS/PKCS12's provider), which is unrelated to Bouncy Castle. This is misleading and functionally incorrect relative to the method's name/contract. Location: authz/client/src/main/java/org/keycloak/authorization/client/util/crypto/AuthzClientCryptoProvider.java:45",
+          "confidence": 0.98,
+          "reasoning": "The candidate issue identifies exactly the same problem as the golden comment. Both point out that the getBouncyCastleProvider() method incorrectly returns the default keystore provider instead of the BouncyCastle provider. The candidate provides more detail about the location and context, but it's describing the identical bug - wrong provider being returned."
+        }
+      ],
+      "false_positives": [
+        {
+          "candidate": "The ignored-providers message is built dynamically from provider class names and passed directly as the format string to logger.debugf(String), which interprets its argument as a printf-style format string. If any loaded provider's class/package name were to contain a '%' character, this would throw an IllegalFormatException at runtime instead of just logging a message. Location: common/src/main/java/org/keycloak/common/crypto/CryptoIntegration.java:68"
+        }
+      ],
+      "false_negatives": [
+        {
+          "golden_comment": "Dead code exists where ASN1Encoder instances are created and written to, but their results are immediately discarded. The actual encoding is performed by new ASN1Encoder instances created in the subsequent return statement, rendering the earlier operations useless.",
+          "severity": "Low",
+          "category": "perf"
+        },
+        {
+          "golden_comment": "When length starts at -1 from indefinite-length encoding, the arithmetic 'length = length - bytes.length' will behave unexpectedly, potentially leading to incorrect processing of encoded data.",
+          "severity": "Medium",
+          "category": "bug"
+        },
+        {
+          "golden_comment": "Abstract order() method on CryptoProvider may break backward compatibility with existing implementations",
+          "severity": "Low",
+          "category": "speculative"
+        }
+      ],
+      "errors": [],
+      "total_candidates": 3,
+      "total_golden": 4,
+      "tp": 1,
+      "fp": 1,
+      "fn": 3,
+      "errors_count": 0,
+      "precision": 0.3333333333333333,
+      "recall": 0.25,
+      "tool": "shipwright",
+      "repo_name": "shipwright__https://github.com/keycloak/keycloak/pull/33832",
+      "pr_url": "https://github.com/keycloak/keycloak/pull/33832"
     }
   },
   "https://github.com/keycloak/keycloak/pull/40940": {
@@ -17022,6 +17338,43 @@
       "tool": "sourcery",
       "repo_name": "keycloak__keycloak__sourcery__PR40940__20260307",
       "pr_url": "https://github.com/code-review-benchmark/keycloak__keycloak__sourcery__PR40940__20260307/pull/1"
+    },
+    "shipwright": {
+      "skipped": false,
+      "true_positives": [
+        {
+          "golden_comment": "The reader thread isn\u2019t waited for; flipping deletedAll to true and asserting immediately can race and miss exceptions added just after the flag change, making this test flaky.",
+          "severity": "Medium",
+          "category": "concurrency",
+          "matched_candidate": "The background reader thread loop `while (!deletedAll.get())` is never joined before the test asserts `caughtExceptions` is empty. Since `deletedAll.set(true)` is called immediately followed by the assertion, the thread may still be mid-iteration and catch (or fail to catch and report) an exception after the check has already passed, making the test racy and able to silently miss the very NPE it is meant to detect. Location: tests/base/src/test/java/org/keycloak/tests/admin/group/GroupTest.java:154",
+          "confidence": 0.98,
+          "reasoning": "The candidate issue identifies exactly the same problem as the golden comment: the reader thread is not waited for (joined) before the assertion, and setting deletedAll to true followed immediately by the assertion creates a race condition where exceptions added after the flag change could be missed, making the test flaky. The candidate provides more detail but describes the identical underlying issue."
+        }
+      ],
+      "false_positives": [
+        {
+          "candidate": "The reader thread's termination flag `deletedAll` is only set to true after the `groupUuuids.forEach(...).remove()` loop completes without exception. If any group deletion throws (e.g. network error, 404, etc.), `deletedAll.set(true)` is never reached, leaving the background thread in an infinite polling loop for the lifetime of the test JVM (no try/finally to guarantee thread shutdown), causing a thread/resource leak and continued unwanted server calls after the test fails. Location: tests/base/src/test/java/org/keycloak/tests/admin/group/GroupTest.java:149"
+        }
+      ],
+      "false_negatives": [
+        {
+          "golden_comment": "Returning null from getSubGroupsCount() violates the GroupModel contract (Javadoc says it never returns null) and may lead to NPEs in callers that expect a non-null count.",
+          "severity": "Critical",
+          "category": "api"
+        }
+      ],
+      "errors": [],
+      "total_candidates": 2,
+      "total_golden": 2,
+      "tp": 1,
+      "fp": 1,
+      "fn": 1,
+      "errors_count": 0,
+      "precision": 0.5,
+      "recall": 0.5,
+      "tool": "shipwright",
+      "repo_name": "shipwright__https://github.com/keycloak/keycloak/pull/40940",
+      "pr_url": "https://github.com/keycloak/keycloak/pull/40940"
     }
   },
   "https://github.com/ai-code-review-evaluation/keycloak-greptile/pull/1": {
@@ -18868,6 +19221,42 @@
       "tool": "sourcery",
       "repo_name": "keycloak__keycloak-greptile__sourcery__PR1__20260307",
       "pr_url": "https://github.com/code-review-benchmark/keycloak__keycloak-greptile__sourcery__PR1__20260307/pull/1"
+    },
+    "shipwright": {
+      "skipped": false,
+      "true_positives": [],
+      "false_positives": [
+        {
+          "candidate": "Both usages of `Assert.assertThrows(NoSuchElementException.class, ...)` were removed in this diff (replaced with `assertThat(...).notNullValue()`), but the `NoSuchElementException` import is not removed. If this was the only usage of that class in the file, the import becomes unused, which will fail Keycloak's strict checkstyle build (no-unused-imports rule). Location: testsuite/integration-arquillian/tests/base/src/test/java/org/keycloak/testsuite/webauthn/passwordless/PasskeysOrganizationAuthenticationTest.java:204"
+        },
+        {
+          "candidate": "`import org.keycloak.models.credential.PasswordCredentialModel;` was added to this test file, but the diff does not show any usage of `PasswordCredentialModel` anywhere in the file's added/changed code (unlike the sibling test `PasskeysUsernameFormTest`, where the same import is actually used to filter credentials). This looks like an unused import copied from the other test class, which would fail an unused-import checkstyle check. Location: testsuite/integration-arquillian/tests/base/src/test/java/org/keycloak/testsuite/webauthn/passwordless/PasskeysUsernamePasswordFormTest.java:36"
+        }
+      ],
+      "false_negatives": [
+        {
+          "golden_comment": "In UsernameForm.authenticate(), isConditionalPasskeysEnabled() is called without passing the required UserModel parameter.",
+          "severity": "Medium",
+          "category": "api"
+        },
+        {
+          "golden_comment": "With isConditionalPasskeysEnabled(UserModel user) requiring user != null, authenticate(...) will not call webauthnAuth.fillContextForm(context) on the initial login page where context.getUser() is still null, so conditional passkey UI will not be set up for first-time passkey login. Consider whether this should also be enabled when no user has been selected yet so ID-less passkey authentication on the initial login form continues to work.",
+          "severity": "Medium",
+          "category": "speculative"
+        }
+      ],
+      "errors": [],
+      "total_candidates": 2,
+      "total_golden": 2,
+      "tp": 0,
+      "fp": 2,
+      "fn": 2,
+      "errors_count": 0,
+      "precision": 0.0,
+      "recall": 0.0,
+      "tool": "shipwright",
+      "repo_name": "shipwright__https://github.com/keycloak/keycloak/pull/41249",
+      "pr_url": "https://github.com/keycloak/keycloak/pull/41249"
     }
   },
   "https://github.com/getsentry/sentry/pull/93824": {
@@ -21478,6 +21867,67 @@
       "tool": "sourcery",
       "repo_name": "sentry__sentry__sourcery__PR93824__20260307",
       "pr_url": "https://github.com/code-review-benchmark/sentry__sentry__sourcery__PR93824__20260307/pull/1"
+    },
+    "shipwright": {
+      "skipped": false,
+      "true_positives": [
+        {
+          "golden_comment": "In the SpanFlusher flush loop, metric tags inconsistently use 'shard' (e.g., in spans.buffer.flusher.produce and spans.buffer.segment_size_bytes) and 'shards' (in spans.buffer.flusher.wait_produce) for the same shard identifier value. This prevents correct metric aggregation across flush stages \u2014 dashboards and alerts filtering by tag 'shard' will miss data tagged as 'shards' and vice versa.",
+          "severity": "Medium",
+          "category": "bug",
+          "matched_candidate": "Metric tag keys for shard grouping are inconsistent within the same `main()` method: `spans.buffer.flusher.produce` and `spans.buffer.segment_size_bytes` use tag key `shard` (holding a comma-joined multi-shard string), while `spans.buffer.flusher.wait_produce` uses tag key `shards` for the same comma-joined value, and `spans.buffer.flusher_unhealthy` uses `shard` with a single integer value per shard. This inconsistency produces mismatched/incompatible metric labels for what should be the same dimension across the same code path. Location: src/sentry/spans/consumers/process/flusher.py:190",
+          "confidence": 0.98,
+          "reasoning": "The candidate issue identifies exactly the same problem as the golden comment: inconsistent metric tag keys using 'shard' vs 'shards' for the same shard identifier value in the SpanFlusher flush loop. Both comments point out that 'spans.buffer.flusher.produce' and 'spans.buffer.segment_size_bytes' use 'shard' while 'spans.buffer.flusher.wait_produce' uses 'shards', and both explain that this inconsistency prevents proper metric aggregation. The candidate even provides additional detail about 'spans.buffer.flusher_unhealthy' and the specific file location."
+        },
+        {
+          "golden_comment": "The test_basic function in test_consumer.py uses a monkeypatched time.sleep that returns immediately, but then calls time.sleep(0.1) expecting it to actually wait.",
+          "severity": "Low",
+          "category": "test_gap",
+          "matched_candidate": "The added `time.sleep(0.1)` call intended to \"Give flusher threads time to process after drift change\" is a no-op because the test earlier does `monkeypatch.setattr(\"time.sleep\", lambda _: None)`, which patches the global `time.sleep` used by the newly added `import time` in this test file as well. The comment is therefore misleading and the wait does not actually occur, making the test rely on timing that isn't actually being introduced. Location: tests/sentry/spans/consumers/process/test_consumer.py:63",
+          "confidence": 0.98,
+          "reasoning": "The candidate issue identifies exactly the same problem as the golden comment: the test_basic function uses a monkeypatched time.sleep that returns immediately (lambda _: None), but then calls time.sleep(0.1) expecting it to actually wait. The candidate correctly identifies the location (test_consumer.py), explains that the monkeypatch makes the sleep a no-op, and notes that the intended wait does not actually occur. This is the same underlying issue."
+        },
+        {
+          "golden_comment": "Sleep in test_consumer.py won\u2019t actually wait because time.sleep was monkeypatched above; consider restoring sleep or using a different sync to ensure the flusher has time to process.",
+          "severity": "Medium",
+          "category": "test_gap",
+          "matched_candidate": "The added `time.sleep(0.1)` call intended to \"Give flusher threads time to process after drift change\" is a no-op because the test earlier does `monkeypatch.setattr(\"time.sleep\", lambda _: None)`, which patches the global `time.sleep` used by the newly added `import time` in this test file as well. The comment is therefore misleading and the wait does not actually occur, making the test rely on timing that isn't actually being introduced. Location: tests/sentry/spans/consumers/process/test_consumer.py:63",
+          "confidence": 0.98,
+          "reasoning": "The candidate issue identifies the exact same problem as the golden comment: the time.sleep(0.1) call won't actually wait because time.sleep was monkeypatched earlier in the test to be a no-op (lambda _: None). Both comments point out that this makes the intended delay ineffective and suggest the test timing won't work as expected. The candidate even provides more specific details about the location and the exact monkeypatch that causes the issue."
+        },
+        {
+          "golden_comment": "Breaking out of the loop when the deadline has elapsed can skip terminating remaining flusher processes, potentially leaving them running after shutdown; consider ensuring termination is attempted even if the deadline is exceeded.",
+          "severity": "Medium",
+          "category": "concurrency",
+          "matched_candidate": "In SpanFlusher.join(), the per-process wait/terminate loop now does `break` as soon as the deadline is exceeded before entering the wait-loop for a given process. This exits the entire `for process_index, process in self.processes.items()` loop, so `process.terminate()` is never called for the current process nor for any remaining processes in the dict. The original single-process implementation always called `self.process.terminate()` unconditionally after the wait loop regardless of deadline status. This change can leave orphaned multiprocessing child processes running after `join()` returns when a timeout is supplied and multiple flusher processes exist. Location: src/sentry/spans/consumers/process/flusher.py:345",
+          "confidence": 0.98,
+          "reasoning": "The candidate issue identifies the exact same problem as the golden comment: breaking out of the loop when the deadline is exceeded causes remaining flusher processes to not have terminate() called on them, potentially leaving them running after shutdown. Both comments point to the same bug in the same location (the join() method's loop behavior) and express the same concern about orphaned/running processes after the deadline is exceeded."
+        }
+      ],
+      "false_positives": [
+        {
+          "candidate": "`self.max_processes = max_processes or len(buffer.assigned_shards)` treats an explicitly passed `max_processes=0` the same as `None` (falls back to shard count) instead of honoring the literal value 0, due to using `or` instead of an explicit `is None` check. Location: src/sentry/spans/consumers/process/flusher.py:57"
+        }
+      ],
+      "false_negatives": [
+        {
+          "golden_comment": "Because flusher processes are created via multiprocessing.get_context('spawn').Process, they are instances of multiprocessing.context.SpawnProcess, which on POSIX is not a subclass of multiprocessing.Process, so this isinstance check will always be false and hung processes won't be killed here.",
+          "severity": "High",
+          "category": "bug"
+        }
+      ],
+      "errors": [],
+      "total_candidates": 4,
+      "total_golden": 5,
+      "tp": 4,
+      "fp": 1,
+      "fn": 1,
+      "errors_count": 0,
+      "precision": 1.0,
+      "recall": 0.8,
+      "tool": "shipwright",
+      "repo_name": "shipwright__https://github.com/getsentry/sentry/pull/93824",
+      "pr_url": "https://github.com/getsentry/sentry/pull/93824"
     }
   },
   "https://github.com/ai-code-review-evaluation/sentry-greptile/pull/5": {
@@ -24263,6 +24713,65 @@
       "tool": "sourcery",
       "repo_name": "sentry__sentry-greptile__sourcery__PR5__20260307",
       "pr_url": "https://github.com/code-review-benchmark/sentry__sentry-greptile__sourcery__PR5__20260307/pull/1"
+    },
+    "shipwright": {
+      "skipped": false,
+      "true_positives": [
+        {
+          "golden_comment": "Using zip(error_ids, events.values()) assumes the get_multi result preserves the input order; dict value order is not guaranteed to match error_ids, so event data can be paired with the wrong ID (missing nodes also shift alignment).",
+          "severity": "Low",
+          "category": "bug",
+          "matched_candidate": "In fetch_error_details, error_ids are paired with nodestore results via `zip(error_ids, events.values())`. `nodestore.backend.get_multi(node_ids)` is not guaranteed to return a dict whose values() are in the same order as the input error_ids (it may omit missing entries or reorder), so this positional zip can associate the wrong error_id/title/message/timestamp with an event, corrupting the error context fed into the AI summarizer. Location: src/sentry/replays/endpoints/project_replay_summarize_breadcrumbs.py:135",
+          "confidence": 0.98,
+          "reasoning": "The candidate issue identifies the exact same problem as the golden comment: using zip(error_ids, events.values()) assumes the get_multi result preserves input order, but dict value order is not guaranteed to match error_ids. Both comments point out that this can cause event data to be paired with the wrong ID, and both mention that missing entries can affect alignment. The candidate even provides additional context about the consequences (corrupting error context for AI summarizer) and the specific file location."
+        },
+        {
+          "golden_comment": "Validation logic incorrectly uses .get('timestamp'), which treats a timestamp of 0 as absent. It should directly check 'timestamp' in self.initial_data to ensure proper field presence detection, preventing both 'age=0' and 'timestamp' from passing mutual-exclusion validation.",
+          "severity": "High",
+          "category": "bug",
+          "matched_candidate": "validate_timestamp/validate_age use truthy checks (`self.initial_data.get(\"age\")` / `self.initial_data.get(\"timestamp\")`) to detect whether the other field is present. Since 0 is a valid timestamp/age value (timestamp has min_value=0), a report where both age and timestamp are 0 will bypass the mutual-exclusion validation entirely (both checks evaluate falsy), letting an invalid mixed report through undetected. Location: src/sentry/issues/endpoints/browser_reporting_collector.py:52",
+          "confidence": 0.98,
+          "reasoning": "The candidate issue identifies the exact same problem as the golden comment: using truthy checks (.get()) for timestamp/age validation, which incorrectly treats a value of 0 as absent/falsy. Both comments point out that this breaks the mutual-exclusion validation when values are 0, and both suggest the same underlying fix (checking for field presence rather than truthy values). The candidate even provides the specific location and explains the same edge case scenario."
+        },
+        {
+          "golden_comment": "Analytics events are recorded before the feature flag check, resulting in analytics being logged for all requests, including those that return 404 due to the feature being disabled. This can lead to misleading analytics data.",
+          "severity": "Medium",
+          "category": "bug",
+          "matched_candidate": "analytics.record(\"preprod_artifact.api.assemble\", ...) is emitted before the `features.has(\"organizations:preprod-artifact-assemble\", ...)` guard, so the analytics event fires even for organizations without the feature enabled (or otherwise rejected requests), inflating/skewing analytics with events for requests that never actually assemble an artifact. Location: src/sentry/preprod/api/endpoints/organization_preprod_artifact_assemble.py:80",
+          "confidence": 0.98,
+          "reasoning": "The candidate issue identifies the exact same problem as the golden comment: analytics events (analytics.record) are being recorded before the feature flag check (features.has), which means analytics are logged for all requests including those that will be rejected due to the feature being disabled. Both comments point to the same code location and the same consequence of misleading/skewed analytics data."
+        }
+      ],
+      "false_positives": [],
+      "false_negatives": [
+        {
+          "golden_comment": "The BrowserReportingCollectorEndpoint returns HttpResponse objects but the endpoint inherits from Endpoint which expects rest_framework.response.Response objects.",
+          "severity": "Medium",
+          "category": "api"
+        },
+        {
+          "golden_comment": "In BaseDetectorTypeValidator.update(), the validator incorrectly updates instance.type using validated_data.get('detector_type') instead of validated_data.get('type').",
+          "severity": "Medium",
+          "category": "bug"
+        },
+        {
+          "golden_comment": "Hardcoded empty data passed to TableWidgetVisualization causes it to always render an empty table instead of displaying actual results when the feature flag is enabled.",
+          "severity": "Medium",
+          "category": "bug"
+        }
+      ],
+      "errors": [],
+      "total_candidates": 3,
+      "total_golden": 6,
+      "tp": 3,
+      "fp": 0,
+      "fn": 3,
+      "errors_count": 0,
+      "precision": 1.0,
+      "recall": 0.5,
+      "tool": "shipwright",
+      "repo_name": "shipwright__https://github.com/ai-code-review-evaluation/sentry-greptile/pull/5",
+      "pr_url": "https://github.com/ai-code-review-evaluation/sentry-greptile/pull/5"
     }
   },
   "https://github.com/ai-code-review-evaluation/sentry-greptile/pull/1": {
@@ -26755,6 +27264,62 @@
       "tool": "sourcery",
       "repo_name": "sentry__sentry-greptile__sourcery__PR1__20260307",
       "pr_url": "https://github.com/code-review-benchmark/sentry__sentry-greptile__sourcery__PR1__20260307/pull/1"
+    },
+    "shipwright": {
+      "skipped": false,
+      "true_positives": [
+        {
+          "golden_comment": "The OptimizedCursorPaginator in organization_auditlogs.py attempts to use negative slice offsets on Django querysets, but Django querysets do not support negative slicing and will raise an AssertionError.",
+          "severity": "High",
+          "category": "bug",
+          "matched_candidate": "OptimizedCursorPaginator's negative-offset branch (triggered when enable_advanced_features and cursor.offset<0) slices a Django QuerySet with a negative start index (`queryset[start_offset:stop]`). Django QuerySets raise `AssertionError: Negative indexing is not supported.` for negative slice bounds (unlike Python lists), so this 'advanced pagination' code path will always crash at runtime rather than working as documented. Location: src/sentry/api/paginator.py:875",
+          "confidence": 0.95,
+          "reasoning": "The candidate issue identifies the exact same problem as the golden comment: the OptimizedCursorPaginator uses negative slice offsets on Django querysets, which Django does not support and will raise an AssertionError. The candidate provides more specific details (the exact condition and line number), but it's describing the same underlying bug. The only minor difference is the golden comment mentions 'organization_auditlogs.py' while the candidate mentions 'src/sentry/api/paginator.py:875', but this is likely because the paginator class is defined in paginator.py and used in organization_auditlogs.py - they're referring to the same code issue."
+        },
+        {
+          "golden_comment": "When requests are authenticated with API keys or org auth tokens (which have user_id=None), organization_context.member is None. Line 71 attempts to access organization_context.member.has_global_access without checking if member is None, causing an AttributeError crash when optimized_pagination=true is used, even though the request passed all permission checks with valid org:write scope.",
+          "severity": "High",
+          "category": "bug",
+          "matched_candidate": "`organization_context.member` can be None for a superuser accessing the organization without being an actual member; because Python evaluates `or` short-circuit only when the left operand is falsy, for a non-superuser request this expression dereferences `.has_global_access` on `organization_context.member`, which will raise AttributeError if `member` is None, crashing the endpoint instead of falling back to normal pagination. Location: src/sentry/api/endpoints/organization_auditlogs.py:74",
+          "confidence": 0.92,
+          "reasoning": "Both comments identify the same issue: organization_context.member can be None, and accessing .has_global_access on it will cause an AttributeError. The golden comment specifies this happens with API keys/org auth tokens (user_id=None), while the candidate mentions superuser access without membership. Both point to the same bug at essentially the same location (line 71 vs 74, likely due to different line counting) where member.has_global_access is accessed without a None check, causing a crash. The core issue is identical: missing None check on organization_context.member before accessing has_global_access."
+        }
+      ],
+      "false_positives": [
+        {
+          "candidate": "The comment/docstring claims 'The underlying Django ORM properly handles negative slicing automatically' and 'This is safe because permissions are checked at the queryset level', but no such queryset-level restriction is added and Django querysets do not support negative slicing at all \u2014 the documented safety/behavior is false and misleading for anyone maintaining this code. Location: src/sentry/api/paginator.py:872"
+        },
+        {
+          "candidate": "New endpoint branch gates an 'advanced' pagination mode (which purportedly allows negative offsets/out-of-bounds cursor navigation on audit logs) behind only `request.user.is_superuser or organization_context.member.has_global_access`, with no additional queryset scoping added to prevent the negative-offset path from returning records outside the intended audit-log window for the organization. Combined with the broken/undocumented negative-offset semantics, this is a risky, under-verified authorization surface for a security-sensitive (audit log) endpoint. Location: src/sentry/api/endpoints/organization_auditlogs.py:73"
+        },
+        {
+          "candidate": "In BasePaginator.get_result, offset clamping is only applied for the non-prev branch (`start_offset = max(0, offset) if not cursor.is_prev else offset`); when `cursor.is_prev` is True the original (potentially negative) offset is passed straight into `queryset[start_offset:stop]` unclamped, producing inconsistent behavior \u2014 forward pagination silently clamps a negative offset to 0, while backward (prev) pagination with the same negative offset will hit the same negative-slice failure that this change intended to avoid. Location: src/sentry/api/paginator.py:179"
+        }
+      ],
+      "false_negatives": [
+        {
+          "golden_comment": "get_item_key assumes a numeric key, but the paginator is used with order_by=-datetime in the audit logs endpoint; calling math.floor/ceil on a datetime will raise a TypeError.",
+          "severity": "High",
+          "category": "bug"
+        },
+        {
+          "golden_comment": "Importing non-existent OptimizedCursorPaginator",
+          "severity": "Low",
+          "category": "bug"
+        }
+      ],
+      "errors": [],
+      "total_candidates": 5,
+      "total_golden": 4,
+      "tp": 2,
+      "fp": 3,
+      "fn": 2,
+      "errors_count": 0,
+      "precision": 0.4,
+      "recall": 0.5,
+      "tool": "shipwright",
+      "repo_name": "shipwright__https://github.com/ai-code-review-evaluation/sentry-greptile/pull/1",
+      "pr_url": "https://github.com/ai-code-review-evaluation/sentry-greptile/pull/1"
     }
   },
   "https://github.com/grafana/grafana/pull/97529": {
@@ -28594,6 +29159,39 @@
       "tool": "sourcery",
       "repo_name": "grafana__grafana__sourcery__PR97529__20260308",
       "pr_url": "https://github.com/code-review-benchmark/grafana__grafana__sourcery__PR97529__20260308/pull/1"
+    },
+    "shipwright": {
+      "skipped": false,
+      "true_positives": [
+        {
+          "golden_comment": "A race condition in BuildIndex allows multiple goroutines to concurrently build the same expensive index for the same key. This is caused by moving the b.cacheMu lock from protecting the entire function to only protecting the final cache assignment. ",
+          "severity": "High",
+          "category": "concurrency",
+          "matched_candidate": "BuildIndex no longer holds cacheMu for the duration of building an index; the lock is only acquired right before writing to b.cache at the end. This allows two concurrent BuildIndex calls for the same key to both build a full bleve index (including a file-backed bleve.New(dir, mapper) call) in parallel; whichever finishes last overwrites b.cache[key], silently orphaning the other freshly-built index (and its open file handles for file-based indexes) since nothing closes the discarded index. This introduces a race condition and a resource leak that did not exist when the whole function was serialized by the mutex. Location: pkg/storage/unified/search/bleve.go:134",
+          "confidence": 0.98,
+          "reasoning": "The candidate issue identifies the exact same race condition problem as the golden comment. Both describe how BuildIndex no longer holds cacheMu for the entire function duration, only acquiring it at the end for the cache assignment. Both explain that this allows multiple goroutines to concurrently build the same expensive index for the same key. The candidate provides additional detail about resource leaks (orphaned indexes and file handles), but the core issue identified is identical - the race condition caused by moving the lock from protecting the entire function to only protecting the final cache assignment."
+        }
+      ],
+      "false_positives": [],
+      "false_negatives": [
+        {
+          "golden_comment": "Calling s.search.TotalDocs() here may race with concurrent index creation: TotalDocs iterates b.cache without synchronization, and the event watcher goroutine started just above could trigger BuildIndex writes concurrently, potentially causing a concurrent map read/write panic.",
+          "severity": "High",
+          "category": "concurrency"
+        }
+      ],
+      "errors": [],
+      "total_candidates": 1,
+      "total_golden": 2,
+      "tp": 1,
+      "fp": 0,
+      "fn": 1,
+      "errors_count": 0,
+      "precision": 1.0,
+      "recall": 0.5,
+      "tool": "shipwright",
+      "repo_name": "shipwright__https://github.com/grafana/grafana/pull/97529",
+      "pr_url": "https://github.com/grafana/grafana/pull/97529"
     }
   },
   "https://github.com/getsentry/sentry/pull/80168": {
@@ -30511,6 +31109,47 @@
       "tool": "sourcery",
       "repo_name": "sentry__sentry__sourcery__PR80168__20260307",
       "pr_url": "https://github.com/code-review-benchmark/sentry__sentry__sourcery__PR80168__20260307/pull/1"
+    },
+    "shipwright": {
+      "skipped": false,
+      "true_positives": [
+        {
+          "golden_comment": "MetricAlertDetectorHandler inherits from StatefulDetectorHandler but only contains pass, failing to implement its required abstract methods: counter_names (property), get_dedupe_value(), get_group_key_values(), and build_occurrence_and_event_data(). This will cause a TypeError at runtime when the class is instantiated.",
+          "severity": "High",
+          "category": "api",
+          "matched_candidate": "`MetricAlertDetectorHandler` was changed to subclass `StatefulDetectorHandler`, which now declares three abstract methods (`get_dedupe_value`, `get_group_key_values`, and the newly-added `build_occurrence_and_event_data`), but the class body is just `pass`. Since these abstract methods are not implemented, instantiating `MetricAlertDetectorHandler` (e.g. via `Detector.detector_handler` which does `group_type.detector_handler(self)`) will raise `TypeError: Can't instantiate abstract class MetricAlertDetectorHandler with abstract methods ...`. Previously this class was a concrete no-op handler; now it is broken and unusable at runtime. Location: src/sentry/incidents/grouptype.py:9",
+          "confidence": 0.95,
+          "reasoning": "The candidate issue identifies the same core problem as the golden comment: MetricAlertDetectorHandler inherits from StatefulDetectorHandler but only contains 'pass', failing to implement the required abstract methods. Both mention get_dedupe_value, get_group_key_values, and build_occurrence_and_event_data as unimplemented abstract methods. The candidate mentions 3 abstract methods while the golden mentions 4 (including counter_names property), but this is a minor difference - the core issue about TypeError at runtime due to unimplemented abstract methods is the same."
+        },
+        {
+          "golden_comment": "Docstring says this returns a list of DetectorEvaluationResult, but the method now returns a dict keyed by DetectorGroupKey. Consider updating the docstring to match the new return type.",
+          "severity": "Low",
+          "category": "doc_defect",
+          "matched_candidate": "The `evaluate` method's return type was changed from `list[DetectorEvaluationResult]` to `dict[DetectorGroupKey, DetectorEvaluationResult]`, but the docstring was not updated and still states 'Evaluates a given data packet and returns a list of `DetectorEvaluationResult`', which now inaccurately describes the actual return type. Location: src/sentry/workflow_engine/processors/detector.py:229",
+          "confidence": 1.0,
+          "reasoning": "The candidate issue identifies exactly the same problem as the golden comment: the docstring says the method returns a list of DetectorEvaluationResult, but the actual return type is now a dict keyed by DetectorGroupKey. Both comments point to the same documentation inconsistency and suggest updating the docstring to match the new return type."
+        }
+      ],
+      "false_positives": [],
+      "false_negatives": [
+        {
+          "golden_comment": "Wrong value `6` passed for `group_2` occurrence \u2014 the data packet specifies `\"group_2\": 10`. Should be `10` to accurately reflect the test scenario, which will ensure the test produces the correct occurrence data.",
+          "severity": "Medium",
+          "category": "test_gap"
+        }
+      ],
+      "errors": [],
+      "total_candidates": 2,
+      "total_golden": 3,
+      "tp": 2,
+      "fp": 0,
+      "fn": 1,
+      "errors_count": 0,
+      "precision": 1.0,
+      "recall": 0.6666666666666666,
+      "tool": "shipwright",
+      "repo_name": "shipwright__https://github.com/getsentry/sentry/pull/80168",
+      "pr_url": "https://github.com/getsentry/sentry/pull/80168"
     }
   },
   "https://github.com/getsentry/sentry/pull/80528": {
@@ -32219,6 +32858,35 @@
       "tool": "sourcery",
       "repo_name": "sentry__sentry__sourcery__PR80528__20260307",
       "pr_url": "https://github.com/code-review-benchmark/sentry__sentry__sourcery__PR80528__20260307/pull/1"
+    },
+    "shipwright": {
+      "skipped": false,
+      "true_positives": [],
+      "false_positives": [],
+      "false_negatives": [
+        {
+          "golden_comment": "The function modifies the config variable to include display values but then returns the original monitor.config instead of the modified version.",
+          "severity": "High",
+          "category": "bug"
+        },
+        {
+          "golden_comment": "The code fetches MonitorCheckIn objects by ID when the required data already exists in previous_checkins. This creates an unnecessary database query.",
+          "severity": "Low",
+          "category": "perf"
+        }
+      ],
+      "errors": [],
+      "total_candidates": 0,
+      "total_golden": 2,
+      "tp": 0,
+      "fp": 0,
+      "fn": 2,
+      "errors_count": 0,
+      "precision": 0.0,
+      "recall": 0.0,
+      "tool": "shipwright",
+      "repo_name": "shipwright__https://github.com/getsentry/sentry/pull/80528",
+      "pr_url": "https://github.com/getsentry/sentry/pull/80528"
     }
   },
   "https://github.com/getsentry/sentry/pull/77754": {
@@ -34438,6 +35106,55 @@
       "tool": "sourcery",
       "repo_name": "sentry__sentry__sourcery__PR77754__20260307",
       "pr_url": "https://github.com/code-review-benchmark/sentry__sentry__sourcery__PR77754__20260307/pull/1"
+    },
+    "shipwright": {
+      "skipped": false,
+      "true_positives": [
+        {
+          "golden_comment": "The AssignmentSource dataclass uses queued: datetime = timezone.now() as a default field value, which evaluates timezone.now() once at class definition time and shares that single datetime instance across all objects.",
+          "severity": "Medium",
+          "category": "concurrency",
+          "matched_candidate": "`AssignmentSource.queued: datetime = timezone.now()` is a dataclass field default that is evaluated exactly once, at class/module import time, not per-instance construction. Every `AssignmentSource` created without explicitly passing `queued` (e.g. via `AssignmentSource.from_integration()`) will get the same stale timestamp (process start time) instead of the actual time the assignment was queued, defeating the purpose of the field. Location: src/sentry/integrations/services/assignment_source.py:16",
+          "confidence": 1.0,
+          "reasoning": "The candidate issue identifies exactly the same problem as the golden comment: the AssignmentSource dataclass has a default field value `queued: datetime = timezone.now()` that is evaluated once at class definition time rather than per-instance, causing all instances to share the same stale timestamp. The candidate even provides the same file location and line number, and explains the same consequence (shared datetime instance across all objects)."
+        },
+        {
+          "golden_comment": "Method name says 'empty_array' but tests empty dict - consider renaming to 'test_from_dict_empty_dict' for clarity.",
+          "severity": "Low",
+          "category": "style",
+          "matched_candidate": "Test method `test_from_dict_empty_array` actually tests an empty dict (`{}`), not an array; the name is misleading relative to what the test body exercises. Location: tests/sentry/integrations/services/test_assignment_source.py:7",
+          "confidence": 0.98,
+          "reasoning": "The candidate issue identifies the exact same problem as the golden comment - that the test method named 'test_from_dict_empty_array' is actually testing an empty dict ({}), not an array, making the name misleading. Both comments point to the same naming inconsistency issue in the same file and suggest the name should reflect what's actually being tested."
+        },
+        {
+          "golden_comment": "to_dict() returns a datetime for queued; if this dict is passed in task kwargs (e.g., via apply_async), JSON serialization may fail depending on the serializer, which can cause enqueue errors.",
+          "severity": "Medium",
+          "category": "bug",
+          "matched_candidate": "`AssignmentSource.to_dict()` uses `asdict()`, which serializes the `queued` field as a raw `datetime` object (not an ISO string). This dict is then passed directly as celery task kwargs in `sync_group_assignee_outbound` (`assignment_source.to_dict()` -> `apply_async(kwargs={...})`), which will fail to serialize with the default JSON task serializer since `datetime` objects are not JSON serializable. Location: src/sentry/integrations/utils/sync.py:140",
+          "confidence": 0.98,
+          "reasoning": "The candidate issue identifies the exact same problem as the golden comment: the to_dict() method returns a datetime object for the 'queued' field, and when this dict is passed to apply_async as task kwargs, JSON serialization will fail because datetime objects are not JSON serializable. The candidate even provides the specific location and explains the flow through sync_group_assignee_outbound, which adds more detail but describes the same underlying issue."
+        }
+      ],
+      "false_positives": [],
+      "false_negatives": [
+        {
+          "golden_comment": "The method name has a typo: test_from_dict_inalid_data should be test_from_dict_invalid_data.",
+          "severity": "Low",
+          "category": "style"
+        }
+      ],
+      "errors": [],
+      "total_candidates": 3,
+      "total_golden": 4,
+      "tp": 3,
+      "fp": 0,
+      "fn": 1,
+      "errors_count": 0,
+      "precision": 1.0,
+      "recall": 0.75,
+      "tool": "shipwright",
+      "repo_name": "shipwright__https://github.com/getsentry/sentry/pull/77754",
+      "pr_url": "https://github.com/getsentry/sentry/pull/77754"
     }
   },
   "https://github.com/getsentry/sentry/pull/95633": {
@@ -36434,6 +37151,42 @@
       "tool": "sourcery",
       "repo_name": "sentry__sentry__sourcery__PR95633__20260307",
       "pr_url": "https://github.com/code-review-benchmark/sentry__sentry__sourcery__PR95633__20260307/pull/1"
+    },
+    "shipwright": {
+      "skipped": false,
+      "true_positives": [],
+      "false_positives": [
+        {
+          "candidate": "SimpleQueueProcessingStrategy.close() and terminate() both call self.queue_pool.shutdown(), which is the FixedQueuePool instance shared and persisted at the factory level (self.queue_pool is created once in ResultsStrategyFactory.__init__ and reused across every create_with_partitions() call). Arroyo calls close()/terminate() on the strategy each time partitions are revoked/rebalanced, not just on final consumer shutdown. This permanently kills all queue worker threads and shuts down the queues (queue.shutdown()) after the very first rebalance, but create_thread_queue_parallel_worker() will still hand out the same (now-dead) queue_pool to any subsequently created strategy, since only ResultsStrategyFactory.shutdown() sets queue_pool to None. Later messages submitted to the dead pool will raise on work_queue.put(), be caught by the broad except in submit(), and have their offsets immediately marked complete/committed even though they were never actually processed by result_processor \u2014 causing silent message/data loss after any partition reassignment. Location: src/sentry/remote_subscriptions/consumers/queue_consumer.py:330"
+        },
+        {
+          "candidate": "OrderedQueueWorker.run() and FixedQueuePool.shutdown() rely on queue.ShutDown and queue.Queue.shutdown(), which were only added to the Python standard library in Python 3.13. If the codebase targets an earlier Python version (as is typical for this backend), referencing queue.ShutDown will raise AttributeError at runtime the first time a worker thread's get() call or the pool's shutdown() method executes, breaking the entire thread-queue-parallel mode. Location: src/sentry/remote_subscriptions/consumers/queue_consumer.py:128"
+        }
+      ],
+      "false_negatives": [
+        {
+          "golden_comment": "The magic number 50 for max_wait is used repeatedly throughout the tests. Consider extracting this as a named constant to improve maintainability.",
+          "severity": "Low",
+          "category": "style"
+        },
+        {
+          "golden_comment": "The test test_thread_queue_parallel_error_handling has a docstring that doesn't match the test implementation.",
+          "severity": "Low",
+          "category": "doc_defect"
+        }
+      ],
+      "errors": [],
+      "total_candidates": 2,
+      "total_golden": 2,
+      "tp": 0,
+      "fp": 2,
+      "fn": 2,
+      "errors_count": 0,
+      "precision": 0.0,
+      "recall": 0.0,
+      "tool": "shipwright",
+      "repo_name": "shipwright__https://github.com/getsentry/sentry/pull/95633",
+      "pr_url": "https://github.com/getsentry/sentry/pull/95633"
     }
   },
   "https://github.com/ai-code-review-evaluation/sentry-greptile/pull/2": {
@@ -39034,6 +39787,68 @@
       "tool": "sourcery",
       "repo_name": "sentry__sentry-greptile__sourcery__PR2__20260307",
       "pr_url": "https://github.com/code-review-benchmark/sentry__sentry-greptile__sourcery__PR2__20260307/pull/1"
+    },
+    "shipwright": {
+      "skipped": false,
+      "true_positives": [
+        {
+          "golden_comment": "OptimizedCursorPaginator's negative-offset branch in get_result() passes cursor.offset directly as start_offset to queryset slicing when enable_advanced_features is True and cursor.offset < 0.",
+          "severity": "Critical",
+          "category": "bug",
+          "matched_candidate": "When `enable_advanced_features` is set and `cursor.offset < 0`, the code does `queryset[start_offset:stop]` with a negative `start_offset` directly on a Django QuerySet. Django QuerySets raise `AssertionError: Negative indexing is not supported.` for negative slice indices, so this branch will always crash at runtime, directly contradicting the comment claiming 'the underlying Django ORM properly handles negative slicing automatically'. Location: src/sentry/api/paginator.py:895",
+          "confidence": 0.98,
+          "reasoning": "The candidate issue identifies the exact same problem as the golden comment: in OptimizedCursorPaginator's get_result() method, when enable_advanced_features is True and cursor.offset < 0, the code passes the negative cursor.offset directly as start_offset to queryset slicing. The candidate goes further to explain why this is problematic (Django QuerySets don't support negative indexing), but it's identifying the same underlying bug."
+        },
+        {
+          "golden_comment": "In BasePaginator.get_result(), when cursor.is_prev is True, it sets start_offset = offset without clamping, allowing negative offset values to be passed to queryset slicing.",
+          "severity": "High",
+          "category": "bug",
+          "matched_candidate": "When `enable_advanced_features` is set and `cursor.offset < 0`, the code does `queryset[start_offset:stop]` with a negative `start_offset` directly on a Django QuerySet. Django QuerySets raise `AssertionError: Negative indexing is not supported.` for negative slice indices, so this branch will always crash at runtime, directly contradicting the comment claiming 'the underlying Django ORM properly handles negative slicing automatically'. Location: src/sentry/api/paginator.py:895",
+          "confidence": 0.9,
+          "reasoning": "Both comments identify the same core issue: negative offset values being passed to Django QuerySet slicing, which will cause an error. The golden comment describes this as happening in BasePaginator.get_result() when cursor.is_prev is True and start_offset = offset without clamping. The candidate issue describes the same problem - negative start_offset being used in queryset slicing, noting Django raises AssertionError for negative indexing. While the candidate mentions 'enable_advanced_features' and a specific line number (895), both are pointing to the same underlying bug: negative offset values not being properly clamped before being used in queryset slicing."
+        },
+        {
+          "golden_comment": "Accessing .has_global_access on organization_context.member can raise an AttributeError when organization_context is None, particularly for non-superuser non-member users.",
+          "severity": "High",
+          "category": "bug",
+          "matched_candidate": "`enable_advanced = request.user.is_superuser or organization_context.member.has_global_access` is executed unconditionally for every request (not only when `use_optimized` is true), and `organization_context` is never defined/imported anywhere in this diff or method. For any non-superuser request this raises a NameError, breaking the entire audit log endpoint. Location: src/sentry/api/endpoints/organization_auditlogs.py:75",
+          "confidence": 0.9,
+          "reasoning": "Both comments identify the same underlying issue: accessing organization_context.member.has_global_access can fail when organization_context is None/undefined. The golden comment specifically mentions AttributeError when organization_context is None, while the candidate mentions NameError because organization_context is never defined. Both are pointing to the same problematic line of code and the same root cause - that organization_context may not be available/valid, causing the access to .member.has_global_access to fail for non-superuser users."
+        }
+      ],
+      "false_positives": [
+        {
+          "candidate": "The per-key `len(payloads[key]) > self.max_segment_spans` guard that enforced a configurable maximum segment span count was removed entirely from the Python flush path, with enforcement now relying solely on the hardcoded `1000` eviction threshold in add-buffer.lua. If `self.max_segment_spans` is configured to a different value elsewhere in the class, the effective cap silently diverges from the configured value and the config becomes dead/ineffective. Location: src/sentry/spans/buffer.py:446"
+        },
+        {
+          "candidate": "The redirect-chasing loop limit was reduced from 10000 to 1000 iterations, and the explanatory comment describing the correctness risk of hitting this limit ('segment trees of depth 10k may not be joined together correctly') was deleted rather than updated, obscuring that deep segment trees are now 10x more likely to fail to join correctly. Location: src/sentry/scripts/spans/add-buffer.lua:30"
+        },
+        {
+          "candidate": "`val[\"end_timestamp_precise\"]` uses direct dict indexing (unlike the other optional-like fields such as `parent_span_id` which use `.get()`), so any incoming span message that lacks this new field (e.g. during schema rollout or from producers not yet updated) will raise a KeyError and crash batch processing instead of being handled gracefully. Location: src/sentry/spans/consumers/process/factory.py:142"
+        },
+        {
+          "candidate": "The comment claims this change 'allow[s] negative offsets to enable efficient bidirectional pagination', but the code actually clamps offset to `max(0, offset)` for the non-prev branch, meaning negative offsets are neutralized rather than enabled; the comment misdescribes the actual behavior. Location: src/sentry/api/paginator.py:179"
+        }
+      ],
+      "false_negatives": [
+        {
+          "golden_comment": "OptimizedCursorPaginator.get_item_key uses floor/ceil on a datetime key (order_by='-datetime'), causing TypeError.",
+          "severity": "High",
+          "category": "bug"
+        }
+      ],
+      "errors": [],
+      "total_candidates": 6,
+      "total_golden": 4,
+      "tp": 3,
+      "fp": 4,
+      "fn": 1,
+      "errors_count": 0,
+      "precision": 0.5,
+      "recall": 0.75,
+      "tool": "shipwright",
+      "repo_name": "shipwright__https://github.com/getsentry/sentry/pull/92393",
+      "pr_url": "https://github.com/getsentry/sentry/pull/92393"
     }
   },
   "https://github.com/ai-code-review-evaluation/sentry-greptile/pull/3": {
@@ -41345,6 +42160,54 @@
       "tool": "sourcery",
       "repo_name": "sentry__sentry-greptile__sourcery__PR3__20260307",
       "pr_url": "https://github.com/code-review-benchmark/sentry__sentry-greptile__sourcery__PR3__20260307/pull/1"
+    },
+    "shipwright": {
+      "skipped": false,
+      "true_positives": [
+        {
+          "golden_comment": "In _set_sample_rate_from_error_sampling(), the check `if client_sample_rate:` treats 0.0 as falsy, so a sample_rate of 0.0 is silently skipped and never set on normalized_data.",
+          "severity": "Low",
+          "category": "bug",
+          "matched_candidate": "`if client_sample_rate:` treats a legitimate falsy sample rate value of `0` (or `0.0`) as absent, so `sample_rate` will never be set on the normalized event data when client_sample_rate is exactly 0, silently dropping a valid (if edge-case) value instead of using an explicit `is not None` check. Location: src/sentry/testutils/factories.py:356",
+          "confidence": 0.98,
+          "reasoning": "The candidate issue identifies the exact same problem as the golden comment: the `if client_sample_rate:` check treats 0.0 as falsy, causing a sample_rate of 0.0 to be silently skipped and never set on normalized_data. Both comments point to the same bug in the same function (_set_sample_rate_from_error_sampling), suggest the same fix (using `is not None` check), and describe the same consequence (valid 0.0 sample rate being dropped). The candidate even provides the specific file location."
+        }
+      ],
+      "false_positives": [
+        {
+          "candidate": "The new `upsampled_count` SnQLFunction lacks `private=True`, unlike the immediately preceding SnQLFunction definition in the same list. This means any user query can call `upsampled_count()` directly (e.g. via yAxis=upsampled_count()), completely bypassing the `is_errors_query_for_error_upsampled_projects` allowlist/eligibility gating that the rest of this PR relies on to restrict sample_weight-based aggregation to allowlisted error-upsampled projects. Location: src/sentry/search/events/datasets/discover.py:1041"
+        },
+        {
+          "candidate": "The docstring in `_are_all_projects_error_upsampled` states the function 'reads the allowlist configuration fresh each time... to ensure we always have the latest configuration state,' but this is contradicted by the caller `is_errors_query_for_error_upsampled_projects`, which wraps this exact check in a 60-second cache and skips calling it entirely on cache hits. The documentation is misleading about actual staleness/consistency behavior introduced in the same file. Location: src/sentry/api/helpers/error_upsampling.py:34"
+        },
+        {
+          "candidate": "Two bare `except Exception: pass` blocks silently swallow any error while reading nested context data or converting to float, masking programming errors (e.g. malformed contexts dict) in this newly added test-data helper without any logging, making failures in event construction hard to diagnose. Location: src/sentry/testutils/factories.py:346"
+        }
+      ],
+      "false_negatives": [
+        {
+          "golden_comment": "Using Python\u2019s built-in hash() to build cache keys is non-deterministic across processes (hash randomization), so keys won\u2019t match across workers and invalidate_upsampling_cache may fail to delete them. Use a deterministic serialization of project_ids for the cache key.",
+          "severity": "Low",
+          "category": "data"
+        },
+        {
+          "golden_comment": "The upsampling eligibility check passes the outer dataset instead of the actual dataset used by scoped_dataset. In paths where the query ultimately runs against discover (e.g., dashboard split) while the original dataset is metrics, upsampling may be skipped even when all projects are allowlisted.",
+          "severity": "Medium",
+          "category": "bug"
+        }
+      ],
+      "errors": [],
+      "total_candidates": 4,
+      "total_golden": 3,
+      "tp": 1,
+      "fp": 3,
+      "fn": 2,
+      "errors_count": 0,
+      "precision": 0.25,
+      "recall": 0.3333333333333333,
+      "tool": "shipwright",
+      "repo_name": "shipwright__https://github.com/getsentry/sentry/pull/94376",
+      "pr_url": "https://github.com/getsentry/sentry/pull/94376"
     }
   },
   "https://github.com/grafana/grafana/pull/103633": {
@@ -43321,6 +44184,44 @@
       "tool": "sourcery",
       "repo_name": "grafana__grafana__sourcery__PR103633__20260308",
       "pr_url": "https://github.com/code-review-benchmark/grafana__grafana__sourcery__PR103633__20260308/pull/1"
+    },
+    "shipwright": {
+      "skipped": false,
+      "true_positives": [],
+      "false_positives": [
+        {
+          "candidate": "In Service.Check, when getCachedIdentityPermissions returns a valid (cache-hit) permission map but checkPermission evaluates to false, the code falls through and unconditionally increments s.metrics.permissionCacheUsage.WithLabelValues(\"false\", ...) even though the permission cache was actually hit (err == nil). This mislabels a cache-hit-but-denied case as a cache miss, corrupting the permissionCacheUsage metric used to measure cache effectiveness. Location: pkg/services/authz/rbac/service.go:135"
+        }
+      ],
+      "false_negatives": [
+        {
+          "golden_comment": "The Check operation exhibits asymmetric cache trust logic: cached permission grants are trusted and returned immediately, but cached denials from the same permission cache are ignored, leading to a fresh database lookup. This allows stale cached grants to provide access to revoked resources, posing a security risk. ",
+          "severity": "High",
+          "category": "security"
+        },
+        {
+          "golden_comment": "The test comment says the cached permissions 'allow access', but the map stores false for dashboards:uid:dash1, so checkPermission will still treat this scope as not allowed.",
+          "severity": "Low",
+          "category": "doc_defect"
+        },
+        {
+          "golden_comment": "The cache key construction in `userPermDenialCacheKey` uses simple string concatenation with underscores, which can lead to ambiguous keys and collision risks when any component contains underscores or is empty. This could result in incorrect denial cache hits, affecting user permissions.",
+          "severity": "Medium",
+          "category": "data"
+        }
+      ],
+      "errors": [],
+      "total_candidates": 1,
+      "total_golden": 3,
+      "tp": 0,
+      "fp": 1,
+      "fn": 3,
+      "errors_count": 0,
+      "precision": 0.0,
+      "recall": 0.0,
+      "tool": "shipwright",
+      "repo_name": "shipwright__https://github.com/grafana/grafana/pull/103633",
+      "pr_url": "https://github.com/grafana/grafana/pull/103633"
     }
   },
   "https://github.com/getsentry/sentry/pull/67876": {
@@ -45506,6 +46407,48 @@
       "tool": "sourcery",
       "repo_name": "sentry__sentry__sourcery__PR67876__20260307",
       "pr_url": "https://github.com/code-review-benchmark/sentry__sentry__sourcery__PR67876__20260307/pull/1"
+    },
+    "shipwright": {
+      "skipped": false,
+      "true_positives": [
+        {
+          "golden_comment": "The code attempts to access integration.metadata[sender][login] without checking for the existence of the sender key. This causes a KeyError for integrations where the sender metadata was not set during creation",
+          "severity": "High",
+          "category": "bug",
+          "matched_candidate": "In GitHubInstallation.dispatch, the new authentication check does `integration.metadata[\"sender\"][\"login\"]` with direct dict indexing and no guard for missing keys. The `sender` key in metadata is only populated once the GitHub installation webhook has been processed; the OAuth callback (which drives this pipeline step) can arrive before that webhook completes. If the key is missing, this raises an unhandled KeyError/500 instead of the intended graceful 'Invalid installation request' error page, defeating the purpose of the new security check. Location: src/sentry/integrations/github/integration.py:505",
+          "confidence": 0.98,
+          "reasoning": "The candidate issue identifies the exact same problem as the golden comment: accessing integration.metadata['sender']['login'] without checking if the 'sender' key exists, which will cause a KeyError. The candidate provides additional context about why this happens (OAuth callback arriving before webhook completes) and the consequences (500 error instead of graceful error page), but it's fundamentally the same underlying issue - missing key validation before accessing nested dictionary keys."
+        }
+      ],
+      "false_positives": [
+        {
+          "candidate": "OAuthLoginView.dispatch wraps the token-exchange response parsing in a bare `except Exception: payload = {}`, silently swallowing all errors (network failures, malformed responses, decode errors) with no logging. This masks legitimate failures behind a generic 'Invalid installation request' message, making it impossible to diagnose real integration/token-exchange problems from logs. Location: src/sentry/integrations/github/integration.py:438"
+        }
+      ],
+      "false_negatives": [
+        {
+          "golden_comment": "Null reference if github_authenticated_user state is missing",
+          "severity": "Medium",
+          "category": "bug"
+        },
+        {
+          "golden_comment": "OAuth state uses pipeline.signature (static) instead of a per-request random value",
+          "severity": "Medium",
+          "category": "security"
+        }
+      ],
+      "errors": [],
+      "total_candidates": 2,
+      "total_golden": 3,
+      "tp": 1,
+      "fp": 1,
+      "fn": 2,
+      "errors_count": 0,
+      "precision": 0.5,
+      "recall": 0.3333333333333333,
+      "tool": "shipwright",
+      "repo_name": "shipwright__https://github.com/getsentry/sentry/pull/67876",
+      "pr_url": "https://github.com/getsentry/sentry/pull/67876"
     }
   },
   "https://github.com/keycloak/keycloak/pull/32918": {
@@ -47242,6 +48185,43 @@
       "tool": "sourcery",
       "repo_name": "keycloak__keycloak__sourcery__PR32918__20260307",
       "pr_url": "https://github.com/code-review-benchmark/keycloak__keycloak__sourcery__PR32918__20260307/pull/1"
+    },
+    "shipwright": {
+      "skipped": false,
+      "true_positives": [
+        {
+          "golden_comment": "Cleanup reference uses incorrect alias - should be 'idp-alias-' + i instead of 'alias'.",
+          "severity": "Medium",
+          "category": "bug",
+          "matched_candidate": "Test cleanup registers removal using the hardcoded literal alias \"alias\" instead of the actual created alias (\"idp-alias-\" + i / \"idp-alias-20\"), so the created identity providers are never actually removed by cleanup. This leaks 20+ identity providers (and their organization associations) across test executions instead of tearing them down as intended. Location: testsuite/integration-arquillian/tests/base/src/test/java/org/keycloak/testsuite/organization/cache/OrganizationCacheTest.java:382",
+          "confidence": 1.0,
+          "reasoning": "The candidate issue identifies the exact same problem as the golden comment - that the cleanup code uses the hardcoded literal 'alias' instead of the dynamically generated alias ('idp-alias-' + i). Both comments point to the same bug where identity providers won't be properly cleaned up because the wrong alias is being used in the cleanup reference."
+        }
+      ],
+      "false_positives": [
+        {
+          "candidate": "In the new getForLogin() cache-hit path, IDs are resolved via `session.identityProviders().getById(id)` and added directly to the result set, without the `createOrganizationAwareIdentityProviderModel` wrapping that is applied on every other exit path of the same method (cache-invalid branch, first cache-miss branch, and the stale-id fallback branch). This means that once the login cache is populated (the common case after the first call), returned IdentityProviderModel instances are of a different, unwrapped shape than what callers previously always received, silently skipping whatever organization-aware behavior that wrapper provides. Location: model/infinispan/src/main/java/org/keycloak/models/cache/infinispan/idp/InfinispanIdentityProviderStorageProvider.java:245"
+        }
+      ],
+      "false_negatives": [
+        {
+          "golden_comment": "InfinispanIdentityProviderStorageProvider.getForLogin() recursively calls session.identityProviders().getById() instead of idpDelegate.getById(), causing the cache lookup to re-enter the same caching layer.",
+          "severity": "Critical",
+          "category": "bug"
+        }
+      ],
+      "errors": [],
+      "total_candidates": 2,
+      "total_golden": 2,
+      "tp": 1,
+      "fp": 1,
+      "fn": 1,
+      "errors_count": 0,
+      "precision": 0.5,
+      "recall": 0.5,
+      "tool": "shipwright",
+      "repo_name": "shipwright__https://github.com/keycloak/keycloak/pull/32918",
+      "pr_url": "https://github.com/keycloak/keycloak/pull/32918"
     }
   },
   "https://github.com/grafana/grafana/pull/94942": {
@@ -48980,6 +49960,41 @@
       "tool": "sourcery",
       "repo_name": "grafana__grafana__sourcery__PR94942__20260308",
       "pr_url": "https://github.com/code-review-benchmark/grafana__grafana__sourcery__PR94942__20260308/pull/1"
+    },
+    "shipwright": {
+      "skipped": false,
+      "true_positives": [
+        {
+          "golden_comment": "The enableSqlExpressions function has flawed logic that always returns false, effectively disabling SQL expressions unconditionally:",
+          "severity": "Critical",
+          "category": "bug",
+          "matched_candidate": "enableSqlExpressions() always returns false regardless of the feature flag state: it computes `enabled := !IsEnabledGlobally(...)`, and both the `if enabled` branch and the fallthrough path return `false`. As a result, ReadQuery's new guard `if !enabled { return error }` unconditionally rejects SQL expression queries even when FlagSqlExpressions is enabled, permanently breaking the feature this PR is meant to introduce. Location: pkg/expr/reader.go:193",
+          "confidence": 0.98,
+          "reasoning": "The candidate issue identifies exactly the same problem as the golden comment: the enableSqlExpressions function has flawed logic that always returns false regardless of the feature flag state. The candidate provides more detail about the specific code flow (the enabled := !IsEnabledGlobally(...) computation and both branches returning false), but it's pointing to the identical underlying bug - the function unconditionally returns false, effectively disabling SQL expressions."
+        },
+        {
+          "golden_comment": "The DB methods NewInMemoryDB().RunCommands, DB.QueryFramesInto, and DB.TablesList in pkg/expr/sql/db.go return hardcoded 'not implemented' errors after removing the go-duck dependency.",
+          "severity": "High",
+          "category": "api",
+          "matched_candidate": "The new sql.DB type (replacing the removed go-duck dependency) has all three methods (TablesList, RunCommands, QueryFramesInto) unconditionally return `errors.New(\"not implemented\")`. Since sql_command.go's Execute() now calls db.QueryFramesInto directly with no fallback, every SQL expression execution will fail at runtime with a generic 'not implemented' error instead of performing the query. Location: pkg/expr/sql/db.go:11",
+          "confidence": 0.98,
+          "reasoning": "The candidate issue identifies the exact same problem as the golden comment - that the DB methods (TablesList, RunCommands, QueryFramesInto) in pkg/expr/sql/db.go return hardcoded 'not implemented' errors after removing the go-duck dependency. The candidate even goes further to explain the runtime impact, but the core issue identified is identical."
+        }
+      ],
+      "false_positives": [],
+      "false_negatives": [],
+      "errors": [],
+      "total_candidates": 3,
+      "total_golden": 2,
+      "tp": 2,
+      "fp": 0,
+      "fn": 0,
+      "errors_count": 0,
+      "precision": 0.6666666666666666,
+      "recall": 1.0,
+      "tool": "shipwright",
+      "repo_name": "shipwright__https://github.com/grafana/grafana/pull/94942",
+      "pr_url": "https://github.com/grafana/grafana/pull/94942"
     }
   },
   "https://github.com/grafana/grafana/pull/90939": {
@@ -50640,6 +51655,35 @@
       "tool": "sourcery",
       "repo_name": "grafana__grafana__sourcery__PR90939__20260308",
       "pr_url": "https://github.com/code-review-benchmark/grafana__grafana__sourcery__PR90939__20260308/pull/1"
+    },
+    "shipwright": {
+      "skipped": false,
+      "true_positives": [],
+      "false_positives": [],
+      "false_negatives": [
+        {
+          "golden_comment": "The GetWebAssets function implements an incomplete double-checked locking pattern for caching web assets. The function first checks if the cache is populated using a read lock (RLock), and if the cache is empty, it acquires a write lock to populate it. However, it fails to re-check whether the cache was populated by another goroutine while waiting to acquire the write lock.",
+          "severity": "Medium",
+          "category": "concurrency"
+        },
+        {
+          "golden_comment": "In addition to the missing double-check, the function has a critical flaw in its error handling: it unconditionally assigns the fetch result to the cache (line 69: entryPointAssetsCache = result) regardless of whether the fetch succeeded or failed. When an error occurs during asset fetching, result is nil, and this nil value overwrites any previously valid cache entry.",
+          "severity": "High",
+          "category": "bug"
+        }
+      ],
+      "errors": [],
+      "total_candidates": 0,
+      "total_golden": 2,
+      "tp": 0,
+      "fp": 0,
+      "fn": 2,
+      "errors_count": 0,
+      "precision": 0.0,
+      "recall": 0.0,
+      "tool": "shipwright",
+      "repo_name": "shipwright__https://github.com/grafana/grafana/pull/90939",
+      "pr_url": "https://github.com/grafana/grafana/pull/90939"
     }
   },
   "https://github.com/grafana/grafana/pull/80329": {
@@ -52439,6 +53483,41 @@
       "tool": "sourcery",
       "repo_name": "grafana__grafana__sourcery__PR80329__20260308",
       "pr_url": "https://github.com/code-review-benchmark/grafana__grafana__sourcery__PR80329__20260308/pull/1"
+    },
+    "shipwright": {
+      "skipped": false,
+      "true_positives": [
+        {
+          "golden_comment": "The code uses Error log level for what appears to be debugging information. This will pollute error logs in production. Consider using Debug or Info level instead.",
+          "severity": "Low",
+          "category": "perf",
+          "matched_candidate": "CleanAnnotations/CleanOrphanedAnnotationTags now log every batch cycle via r.log.Error even when there is no error (e.g. `r.log.Error(\"Annotations to clean by time\", \"count\", len(ids), \"ids\", ids, \"cond\", cond, \"err\", err)` when err is nil). This mislabels normal, successful cleanup operations as errors, which will pollute error-level logs/alerts, and additionally logs the full slice of IDs (which can be tens of thousands per batch, as demonstrated by the new 40000-row test case), producing excessively large log lines on every batch iteration. Location: pkg/services/annotations/annotationsimpl/xorm_store.go:528",
+          "confidence": 0.95,
+          "reasoning": "The candidate issue identifies the same core problem as the golden comment: using Error log level for non-error/debugging information, which will pollute error logs. The candidate provides more specific details about the location and additional concerns (logging large ID slices), but the fundamental issue - using Error level logging for what should be Debug/Info level - is the same."
+        },
+        {
+          "golden_comment": "The cleanup job ticker was changed from 10 minutes to 1 minute, which can lead to increased database load and contention due to more frequent execution of cleanup tasks. This change should be reviewed to ensure it aligns with performance expectations.",
+          "severity": "Low",
+          "category": "perf",
+          "matched_candidate": "The cleanup service ticker interval was reduced from 10 minutes to 1 minute with no accompanying explanation or config change, causing the full cleanup job (annotations, tmp files, etc.) to run 10x more frequently. This is a significant behavioral change for a PR framed as 'database performance optimization' and could increase DB load rather than reduce it, especially combined with the smaller per-batch loop introduced in this PR. Location: pkg/services/cleanup/cleanup.go:77",
+          "confidence": 0.98,
+          "reasoning": "The candidate issue identifies the exact same problem as the golden comment - the cleanup job ticker interval being changed from 10 minutes to 1 minute, with concerns about increased database load and more frequent execution of cleanup tasks. The candidate even provides the specific file location (pkg/services/cleanup/cleanup.go:77) and elaborates on the same performance concerns mentioned in the golden comment."
+        }
+      ],
+      "false_positives": [],
+      "false_negatives": [],
+      "errors": [],
+      "total_candidates": 2,
+      "total_golden": 2,
+      "tp": 2,
+      "fp": 0,
+      "fn": 0,
+      "errors_count": 0,
+      "precision": 1.0,
+      "recall": 1.0,
+      "tool": "shipwright",
+      "repo_name": "shipwright__https://github.com/grafana/grafana/pull/80329",
+      "pr_url": "https://github.com/grafana/grafana/pull/80329"
     }
   },
   "https://github.com/grafana/grafana/pull/90045": {
@@ -54831,6 +55910,66 @@
       "tool": "sourcery",
       "repo_name": "grafana__grafana__sourcery__PR90045__20260308",
       "pr_url": "https://github.com/code-review-benchmark/grafana__grafana__sourcery__PR90045__20260308/pull/1"
+    },
+    "shipwright": {
+      "skipped": false,
+      "true_positives": [
+        {
+          "golden_comment": "The context is being created with d.Log instead of the log variable that was initialized with additional context values (name, kind, method). This means those values won't be propagated to the logging context.",
+          "severity": "Medium",
+          "category": "bug",
+          "matched_candidate": "In Delete(), an enriched logger `log` (with name/kind/method fields) is created, but the context is populated with the un-enriched `d.Log` instead: `ctx = klog.NewContext(ctx, d.Log)`. Every other method in this file (Create, Get, List, Update, DeleteCollection) correctly injects the enriched `log` variable into the context, so downstream code using `klog.FromContext(ctx)` in Delete's call path loses the name/kind/method fields. Location: pkg/apiserver/rest/dualwriter_mode3.go:93",
+          "confidence": 1.0,
+          "reasoning": "The candidate issue identifies exactly the same problem as the golden comment: in the Delete() method, the context is being created with d.Log instead of the enriched 'log' variable that contains additional context values (name, kind, method). The candidate even specifies the exact location (line 93) and correctly notes that other methods in the file handle this correctly."
+        },
+        {
+          "golden_comment": "Inconsistency: using name instead of options.Kind for metrics recording differs from other methods.",
+          "severity": "Medium",
+          "category": "api",
+          "matched_candidate": "In Delete(), the success path calls `d.recordStorageDuration(false, mode3Str, name, method, startStorage)`, passing the resource `name` as the metric's kind label instead of `options.Kind`, which is used everywhere else (including the error branch just above it in the same function). Location: pkg/apiserver/rest/dualwriter_mode3.go:100",
+          "confidence": 0.98,
+          "reasoning": "The candidate issue identifies exactly the same problem as the golden comment: in the Delete() method, the success path uses 'name' instead of 'options.Kind' for metrics recording, which is inconsistent with other methods and even with the error branch in the same function. The candidate provides more specific details (file location, line number, function name) but describes the identical inconsistency issue."
+        }
+      ],
+      "false_positives": [
+        {
+          "candidate": "In Create, Delete, Update, and DeleteCollection, the async write to Legacy storage is launched in a goroutine using `context.WithTimeoutCause(ctx, ...)` where `ctx` is the original request-scoped context. Since these goroutines run after the primary Storage operation has already returned to the caller (and the HTTP request context is commonly canceled once the response is written), the derived context is likely to be canceled before or shortly after the goroutine starts, causing legacy writes to fail intermittently/consistently. In addition, errors returned from `d.Legacy.Create/Delete/Update/DeleteCollection` inside these goroutines are only fed into a duration metric (as a boolean) and are never logged via `log.Error` as they were previously, silently swallowing legacy write failures. Location: pkg/apiserver/rest/dualwriter_mode3.go:47"
+        },
+        {
+          "candidate": "In Create(), when `d.Storage.Create` fails, the code calls `d.recordLegacyDuration(true, mode3Str, options.Kind, method, startStorage)` to record a Storage failure, mislabeling it as a legacy-duration metric instead of `d.recordStorageDuration`. This corrupts the storage vs legacy metrics. Location: pkg/apiserver/rest/dualwriter_mode3.go:41"
+        },
+        {
+          "candidate": "In Update(), when `d.Storage.Update` fails, the code calls `d.recordLegacyDuration(true, mode3Str, options.Kind, method, startStorage)` instead of `d.recordStorageDuration`, mislabeling a storage failure as a legacy metric, inconsistent with the equivalent success branch a few lines below that correctly calls `recordStorageDuration`. Location: pkg/apiserver/rest/dualwriter_mode3.go:121"
+        },
+        {
+          "candidate": "In DeleteCollection()'s async goroutine, the result of `d.Legacy.DeleteCollection` is recorded via `d.recordStorageDuration(err != nil, mode3Str, options.Kind, method, startLegacy)` instead of `d.recordLegacyDuration`, mislabeling a legacy-store duration/error as a storage metric. Location: pkg/apiserver/rest/dualwriter_mode3.go:161"
+        },
+        {
+          "candidate": "Update() now passes the original `objInfo` (unwrapped) directly to `d.Legacy.Update`, instead of wrapping it with the object actually returned by `d.Storage.Update` (as the previous implementation did via `updateWrapper{upstream: objInfo, updated: obj}`). This means Legacy will independently recompute the updated object from its own current state rather than applying the same final object written to Storage, risking data divergence between the two stores. Location: pkg/apiserver/rest/dualwriter_mode3.go:133"
+        },
+        {
+          "candidate": "The test case named \"error when listing an object in the unified store is not implemented\" in TestMode3_List no longer exercises a not-implemented scenario (List is now fully implemented in this PR); it merely mocks a generic error return from Storage.List, so the test name is misleading relative to what it actually verifies. Location: pkg/apiserver/rest/dualwriter_mode3_test.go:150"
+        }
+      ],
+      "false_negatives": [
+        {
+          "golden_comment": "In DualWriterMode3's DeleteCollection method, when a storage operation fails, recordStorageDuration is incorrectly called with recordLegacyDuration instead.",
+          "severity": "High",
+          "category": "bug"
+        }
+      ],
+      "errors": [],
+      "total_candidates": 8,
+      "total_golden": 3,
+      "tp": 2,
+      "fp": 6,
+      "fn": 1,
+      "errors_count": 0,
+      "precision": 0.25,
+      "recall": 0.6666666666666666,
+      "tool": "shipwright",
+      "repo_name": "shipwright__https://github.com/grafana/grafana/pull/90045",
+      "pr_url": "https://github.com/grafana/grafana/pull/90045"
     }
   },
   "https://github.com/grafana/grafana/pull/106778": {
@@ -56899,6 +58038,48 @@
       "tool": "sourcery",
       "repo_name": "grafana__grafana__sourcery__PR106778__20260308",
       "pr_url": "https://github.com/code-review-benchmark/grafana__grafana__sourcery__PR106778__20260308/pull/1"
+    },
+    "shipwright": {
+      "skipped": false,
+      "true_positives": [],
+      "false_positives": [
+        {
+          "candidate": "In useIsGrafanaPromRuleEditable (new function), the code calls `ctx.hasPermissionInMetadata(rulesPermissions.update, folder)` and `ctx.hasPermissionInMetadata(rulesPermissions.delete, folder)`, but `ctx` is never imported or defined anywhere in this file's diff. This will fail to compile (TS2304: Cannot find name 'ctx') or throw a ReferenceError at runtime, breaking all Grafana Prometheus-rule editability/removability checks (used by useAllGrafanaPromRuleAbilities, which gates Edit/Delete/Pause/Restore actions across the new rule-list UI). Location: public/app/features/alerting/unified/hooks/useAbilities.ts:300"
+        },
+        {
+          "candidate": "In useAllRulerRuleAbilities (used for cloud/Mimir/Loki ruler rules via useRulerRuleAbility/useRulerRuleAbilities, which power AlertRuleMenu and RuleActionsButtons for all data sources, not just Grafana), the previous federated-rule-group check `isFederated = isFederatedRuleGroup(rule.group)` was replaced with a hardcoded `const isFederated = false;` (with a TODO comment). This removes the immutability guard for federated rule groups on cloud/Mimir rulers, so rules in federated groups that were previously blocked from Edit/Delete/Duplicate will now incorrectly be treated as mutable. Location: public/app/features/alerting/unified/hooks/useAbilities.ts:247"
+        },
+        {
+          "candidate": "The new GrafanaRuleListItem builds the rule view link as `createRelativeUrl('/alerting/grafana/${uid}/view')` without the `returnTo` query param that the deleted GrafanaRuleLoader.tsx previously included (`createRelativeUrl(..., { returnTo })` using `createReturnTo()`). This is a functional regression: users clicking a Grafana rule from the list view will lose the 'return to' navigation context that was previously preserved. Location: public/app/features/alerting/unified/rule-list/GrafanaRuleListItem.tsx:29"
+        },
+        {
+          "candidate": "The new test 'should render correct menu actions when More button is clicked' grants AccessControlAction.AlertingRuleDelete (both globally and via setFolderAccessControl) in the shared beforeEach, but the test only asserts on silence/copyLink/duplicate/export and then asserts `menuItems.length` equals exactly 4, never checking or accounting for the Delete menu item that the granted delete permission should surface. This makes the test's assertions inconsistent with its own setup and likely to mask a missing/incorrect Delete action or produce a brittle/incorrect pass. Location: public/app/features/alerting/unified/rule-list/GrafanaGroupLoader.test.tsx:205"
+        }
+      ],
+      "false_negatives": [
+        {
+          "golden_comment": "The rendered GrafanaRuleListItem is missing the required key prop for React list items. This can cause rendering issues when the list order changes.",
+          "severity": "Medium",
+          "category": "bug"
+        },
+        {
+          "golden_comment": "RuleActionsButtons is invoked with only promRule, but SilenceGrafanaRuleDrawer inside RuleActionsButtons still depends on a Grafana Ruler rule being present, so for Grafana rules coming from list views the 'Silence notifications' menu entry (now driven by Grafana Prom abilities) will toggle showSilenceDrawer without ever rendering the drawer. This means clicking 'Silence notifications' for these rules has no visible effect, even when abilities indicate silencing is allowed.",
+          "severity": "High",
+          "category": "bug"
+        }
+      ],
+      "errors": [],
+      "total_candidates": 4,
+      "total_golden": 2,
+      "tp": 0,
+      "fp": 4,
+      "fn": 2,
+      "errors_count": 0,
+      "precision": 0.0,
+      "recall": 0.0,
+      "tool": "shipwright",
+      "repo_name": "shipwright__https://github.com/grafana/grafana/pull/106778",
+      "pr_url": "https://github.com/grafana/grafana/pull/106778"
     }
   },
   "https://github.com/grafana/grafana/pull/107534": {
@@ -58424,6 +59605,37 @@
       "tool": "sourcery",
       "repo_name": "grafana__grafana__sourcery__PR107534__20260308",
       "pr_url": "https://github.com/code-review-benchmark/grafana__grafana__sourcery__PR107534__20260308/pull/1"
+    },
+    "shipwright": {
+      "skipped": false,
+      "true_positives": [],
+      "false_positives": [
+        {
+          "candidate": "runShardSplitQuery now interpolates each query up-front via `.map((query) => datasource.applyTemplateVariables(query, request.scopedVars, request.filters))`, replacing the previous single batched `interpolateVariablesInQueries` call. The accompanying test change confirms this causes `applyTemplateVariables` to be invoked 5 times instead of once (`expect(datasource.applyTemplateVariables).toHaveBeenCalledTimes(5)`), meaning the same query is now redundantly re-interpolated once at the top level and again per shard group downstream. This duplicate interpolation wastes computation and risks inconsistent results if the per-shard-group interpolation call uses different scopedVars/filters than the top-level call, since both write into the same query object. Location: public/app/plugins/datasource/loki/shardQuerySplitting.ts:49"
+        },
+        {
+          "candidate": "The new top-level interpolation (`applyTemplateVariables`) is applied to every split-eligible query before the time range or shards are computed, resolving macros/scoped vars against the request's overall scopedVars. Since query splitting exists specifically to divide a single request into multiple sub-requests over time/shard partitions, resolving variables once up-front (rather than per generated sub-query) risks baking a single global interpolation into every emitted sub-query, whereas the previous batch call semantics and downstream per-chunk processing assumed interpolation could still be adjusted per partition. Location: public/app/plugins/datasource/loki/querySplitting.ts:296"
+        }
+      ],
+      "false_negatives": [
+        {
+          "golden_comment": "The applyTemplateVariables method is called with request.filters as the third parameter, but this parameter is not used in the corresponding test setup.",
+          "severity": "Low",
+          "category": "test_gap"
+        }
+      ],
+      "errors": [],
+      "total_candidates": 2,
+      "total_golden": 1,
+      "tp": 0,
+      "fp": 2,
+      "fn": 1,
+      "errors_count": 0,
+      "precision": 0.0,
+      "recall": 0.0,
+      "tool": "shipwright",
+      "repo_name": "shipwright__https://github.com/grafana/grafana/pull/107534",
+      "pr_url": "https://github.com/grafana/grafana/pull/107534"
     }
   },
   "https://github.com/grafana/grafana/pull/79265": {
@@ -61122,6 +62334,58 @@
       "tool": "sourcery",
       "repo_name": "grafana__grafana__sourcery__PR79265__20260308",
       "pr_url": "https://github.com/code-review-benchmark/grafana__grafana__sourcery__PR79265__20260308/pull/1"
+    },
+    "shipwright": {
+      "skipped": false,
+      "true_positives": [
+        {
+          "golden_comment": "Race condition: Multiple concurrent requests could pass the device count check simultaneously and create devices beyond the limit. Consider using a database transaction or lock.",
+          "severity": "High",
+          "category": "concurrency",
+          "matched_candidate": "CreateOrUpdateDevice now does a check-then-act (CountDevices followed by a separate insert/update) with no transaction or locking. Concurrent anonymous requests can all read count < deviceLimit before any of them inserts, allowing the device count to exceed the configured deviceLimit (TOCTOU race), defeating the new device-limit enforcement under load. Location: pkg/services/anonymous/anonimpl/anonstore/database.go:90",
+          "confidence": 0.98,
+          "reasoning": "The candidate issue identifies exactly the same problem as the golden comment: a race condition/TOCTOU vulnerability where multiple concurrent requests can pass the device count check simultaneously and create devices beyond the limit. Both comments point to the same underlying issue - the check-then-act pattern without proper transaction or locking, allowing the device limit to be exceeded under concurrent load. The candidate even provides more specific details (file location, technical term TOCTOU) but describes the identical bug."
+        }
+      ],
+      "false_positives": [
+        {
+          "candidate": "Anonymous.Authenticate previously tagged devices in a background goroutine with its own context.Background()+2min timeout and a recover() to swallow panics, decoupling tagging from the request lifecycle. The diff removes the goroutine, timeout, and panic recovery, making TagDevice run synchronously inline using the request's ctx. This means (1) if the client cancels/the request context is done before the DB write completes, the device tag/update can be aborted mid-flight instead of completing in the background as before, and (2) any panic inside TagDevice/tagDeviceUI now propagates up through Authenticate with no recover(), instead of being caught and logged as previously. Location: pkg/services/anonymous/anonimpl/client.go:44"
+        }
+      ],
+      "false_negatives": [
+        {
+          "golden_comment": "Anonymous authentication now fails entirely if anonDeviceService.TagDevice returns ErrDeviceLimitReached. Previously, device tagging was asynchronous and non-blocking. This change prevents anonymous users from authenticating when the device limit is reached.",
+          "severity": "Medium",
+          "category": "bug"
+        },
+        {
+          "golden_comment": "The updateDevice function returns ErrDeviceLimitReached when rowsAffected == 0, but zero affected rows means the device doesn't exist or falls outside the time window, not that a limit was reached.",
+          "severity": "Low",
+          "category": "doc_defect"
+        },
+        {
+          "golden_comment": "Time window calculation inconsistency: Using device.UpdatedAt.UTC().Add(-anonymousDeviceExpiration) as the lower bound but device.UpdatedAt as the current time may not match the intended logic. Consider using time.Now().UTC() consistently.",
+          "severity": "Low",
+          "category": "bug"
+        },
+        {
+          "golden_comment": "The `tagDeviceUI` function populates the local cache with the device ID before the `CreateOrUpdateDevice` call. If `ErrDeviceLimitReached` is returned, the cache entry remains set, allowing subsequent requests to bypass the device limit until the cache expires after 29 minutes.",
+          "severity": "High",
+          "category": "bug"
+        }
+      ],
+      "errors": [],
+      "total_candidates": 2,
+      "total_golden": 5,
+      "tp": 1,
+      "fp": 1,
+      "fn": 4,
+      "errors_count": 0,
+      "precision": 0.5,
+      "recall": 0.2,
+      "tool": "shipwright",
+      "repo_name": "shipwright__https://github.com/grafana/grafana/pull/79265",
+      "pr_url": "https://github.com/grafana/grafana/pull/79265"
     }
   },
   "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/9": {
@@ -62905,6 +64169,39 @@
       "tool": "sourcery",
       "repo_name": "discourse__discourse-graphite__sourcery__PR9__20260308",
       "pr_url": "https://github.com/code-review-benchmark/discourse__discourse-graphite__sourcery__PR9__20260308/pull/1"
+    },
+    "shipwright": {
+      "skipped": false,
+      "true_positives": [],
+      "false_positives": [
+        {
+          "candidate": "In config/initializers/i18n.rb, FallbackLocaleList#ensure_loaded! calls `I18n.ensure_loaded! l`, but the new `ensure_loaded!(locale)` method is defined on the I18n backend instance (lib/freedom_patches/translate_accelerator.rb), not on the top-level I18n module. The I18n module does not expose or delegate an `ensure_loaded!` class method, so this call raises NoMethodError. Since ApplicationController now calls `I18n.fallbacks.ensure_loaded!` on every request (in the locale-setting method), this breaks every request that goes through that code path. The call should be `I18n.backend.ensure_loaded! l`. Location: config/initializers/i18n.rb:22"
+        }
+      ],
+      "false_negatives": [
+        {
+          "golden_comment": "The @loaded_locales instance variable in translate_accelerator.rb is lazily initialized and accessed by multiple concurrent threads without synchronization.",
+          "severity": "Low",
+          "category": "concurrency"
+        },
+        {
+          "golden_comment": "Consider normalizing the input locale (e.g., to a symbol) when checking/loading here to avoid double-loading if the same locale is passed as a String vs Symbol (also applies to other locations in the PR).",
+          "severity": "Low",
+          "category": "perf"
+        }
+      ],
+      "errors": [],
+      "total_candidates": 1,
+      "total_golden": 2,
+      "tp": 0,
+      "fp": 1,
+      "fn": 2,
+      "errors_count": 0,
+      "precision": 0.0,
+      "recall": 0.0,
+      "tool": "shipwright",
+      "repo_name": "shipwright__https://github.com/discourse/discourse/commit/ecfa17b5a79dfdc91e7a4d50b42ae78a35d0a293",
+      "pr_url": "https://github.com/discourse/discourse/commit/ecfa17b5a79dfdc91e7a4d50b42ae78a35d0a293"
     }
   },
   "https://github.com/grafana/grafana/pull/76186": {
@@ -64745,6 +66042,48 @@
       "tool": "sourcery",
       "repo_name": "grafana__grafana__sourcery__PR76186__20260308",
       "pr_url": "https://github.com/code-review-benchmark/grafana__grafana__sourcery__PR76186__20260308/pull/1"
+    },
+    "shipwright": {
+      "skipped": false,
+      "true_positives": [
+        {
+          "golden_comment": "The `TestLogger.FromContext` method returns a new `TestLogger` instance via `NewTestLogger()` instead of returning the receiver (f). This causes logs to be recorded on an un-referenced instance, leading to broken test assertions that expect logs to be recorded on the original logger.",
+          "severity": "High",
+          "category": "bug",
+          "matched_candidate": "TestLogger.FromContext (pkg/plugins/log/fake.go) returns a brand-new NewTestLogger() instance instead of the receiver, discarding all call-tracking state. Since LoggerMiddleware.logRequest now calls `m.logger.FromContext(ctx).Info(...)` instead of `m.logger.Info(...)`, any test using the fake TestLogger to assert on InfoLogs/DebugLogs/etc. after invoking the middleware will silently see no recorded calls, since the logs are recorded on a throwaway logger rather than the original fake instance. Location: pkg/plugins/log/fake.go:46",
+          "confidence": 1.0,
+          "reasoning": "The candidate issue identifies exactly the same problem as the golden comment: TestLogger.FromContext returns a new TestLogger instance via NewTestLogger() instead of returning the receiver, which causes logs to be recorded on an unreferenced/throwaway instance rather than the original logger. This breaks test assertions that expect logs to be recorded on the original logger. The candidate even provides additional context about how LoggerMiddleware.logRequest uses this method, but the core issue is identical."
+        }
+      ],
+      "false_positives": [
+        {
+          "candidate": "The middleware file was renamed from instrumentation_middleware_test.go to metrics_middleware_test.go and the internal call was updated to newMetricsMiddleware, but the test function itself is still named TestInstrumentationMiddleware, leaving a stale/misleading test name that no longer matches the renamed middleware or file. Location: pkg/services/pluginsintegration/clientmiddleware/metrics_middleware_test.go:75"
+        }
+      ],
+      "false_negatives": [
+        {
+          "golden_comment": "The ContextualLoggerMiddleware methods (QueryData, CallResource, CheckHealth, CollectMetrics) panic when a nil request is received. This occurs because they directly access req.PluginContext (via the instrumentContext function) without first checking if req is nil. This is a regression, as previous middleware layers gracefully handled nil requests.",
+          "severity": "High",
+          "category": "bug"
+        },
+        {
+          "golden_comment": "The traceID is no longer logged for plugin requests. During a refactoring, the tracing import and the logic to extract and add traceID from the context to log parameters were removed from the LoggerMiddleware. The newly introduced ContextualLoggerMiddleware does not add this information, resulting in missing traceID in plugin request logs and impacting debugging and request tracing capabilities.",
+          "severity": "Low",
+          "category": "bug"
+        }
+      ],
+      "errors": [],
+      "total_candidates": 2,
+      "total_golden": 3,
+      "tp": 1,
+      "fp": 1,
+      "fn": 2,
+      "errors_count": 0,
+      "precision": 0.5,
+      "recall": 0.3333333333333333,
+      "tool": "shipwright",
+      "repo_name": "shipwright__https://github.com/grafana/grafana/pull/76186",
+      "pr_url": "https://github.com/grafana/grafana/pull/76186"
     }
   },
   "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/10": {
@@ -67963,6 +69302,80 @@
       "tool": "sourcery",
       "repo_name": "discourse__discourse-graphite__sourcery__PR10__20260308",
       "pr_url": "https://github.com/code-review-benchmark/discourse__discourse-graphite__sourcery__PR10__20260308/pull/1"
+    },
+    "shipwright": {
+      "skipped": false,
+      "true_positives": [
+        {
+          "golden_comment": "The EmbeddableHost model's before_validation callback calls self.host.sub! to strip http://, https://, and path segments, but if self.host is nil, this raises NoMethodError.",
+          "severity": "Critical",
+          "category": "bug",
+          "matched_candidate": "`before_validation` unconditionally calls `self.host.sub!(...)` twice. If `host` is nil (e.g. missing/blank param), this raises `NoMethodError` instead of surfacing a normal ActiveRecord validation error, crashing create/update instead of returning a friendly error. Location: app/models/embeddable_host.rb:5",
+          "confidence": 0.98,
+          "reasoning": "The candidate issue identifies the exact same problem as the golden comment: the before_validation callback calls self.host.sub!() which will raise NoMethodError if self.host is nil. Both comments point to the same bug in the same location (app/models/embeddable_host.rb) and describe the same consequence (NoMethodError when host is nil)."
+        },
+        {
+          "golden_comment": "The update and destroy methods in Admin::EmbeddableHostsController do not validate the existence of the EmbeddableHost record retrieved by ID. If EmbeddableHost.where(id: params[:id]).first returns nil (i.e., the host does not exist), attempting to call methods on the nil object (e.g., save_host or destroy) will result in a NoMethodError.",
+          "severity": "Medium",
+          "category": "bug",
+          "matched_candidate": "`update` and `destroy` fetch the record with `EmbeddableHost.where(id: params[:id]).first` (which returns nil if not found) and then unconditionally call `save_host(host)` / `host.destroy` on it, causing a `NoMethodError` instead of a proper 404/error response when the id doesn't exist. Location: app/controllers/admin/embeddable_hosts_controller.rb:8",
+          "confidence": 1.0,
+          "reasoning": "The candidate issue identifies exactly the same problem as the golden comment: both point out that the update and destroy methods in Admin::EmbeddableHostsController use EmbeddableHost.where(id: params[:id]).first which can return nil, and then unconditionally call methods (save_host or destroy) on the potentially nil object, resulting in a NoMethodError. The candidate even provides the specific file location and line number."
+        },
+        {
+          "golden_comment": "record_for_host compares lower(host) = ? but does not normalize the parameter\u2019s case, so mixed\u2011case referer hosts may fail to match even though comparison intends to be case\u2011insensitive.",
+          "severity": "Medium",
+          "category": "bug",
+          "matched_candidate": "`record_for_host` compares `lower(host) = ?` against `host = uri.host`, which is not downcased before being passed as the bind parameter. A referrer/URL with mixed-case hostname (e.g. `http://EvilTrout.com`) will fail to match the lowercased DB column, causing legitimate allowed hosts to be rejected by `host_allowed?`. Location: app/models/embeddable_host.rb:16",
+          "confidence": 1.0,
+          "reasoning": "The candidate issue identifies exactly the same problem as the golden comment: the `record_for_host` method uses `lower(host) = ?` in the SQL query but doesn't normalize/downcase the `uri.host` parameter before passing it as the bind parameter. Both comments point out that this causes mixed-case hostnames to fail matching even though the comparison is intended to be case-insensitive. The candidate even provides a specific example and the exact file location."
+        },
+        {
+          "golden_comment": "The contents of `category_fabricator.rb` and `embeddable_host_fabricator.rb` have been swapped, resulting in the loss of original category fabricators and potentially causing errors in fabricator usage.",
+          "severity": "High",
+          "category": "bug",
+          "matched_candidate": "The contents of `spec/fabricators/category_fabricator.rb` and the new `spec/fabricators/embeddable_host_fabricator.rb` were swapped: `category_fabricator.rb` now only defines `Fabricator(:embeddable_host)`, while the new `embeddable_host_fabricator.rb` contains the `:category`/`:diff_category`/`:happy_category`/`:private_category` fabricator definitions that used to live in `category_fabricator.rb`. Location: spec/fabricators/category_fabricator.rb:1",
+          "confidence": 0.98,
+          "reasoning": "The candidate issue identifies exactly the same problem as the golden comment - that the contents of category_fabricator.rb and embeddable_host_fabricator.rb have been swapped. The candidate provides more detail about what specific fabricators are affected, but it's describing the identical underlying issue: the file contents were accidentally swapped, which would cause errors in fabricator usage."
+        },
+        {
+          "golden_comment": "The migration accesses execute(...)[0]['id'] without checking if the query returned any rows, which causes a crash when no embed_category setting exists. This can lead to a broken feature when attempting to edit category/host relationships.",
+          "severity": "High",
+          "category": "bug",
+          "matched_candidate": "The migration runs `execute(\"SELECT c.id FROM categories AS c INNER JOIN site_settings AS s ON s.value = c.name WHERE s.name = 'embed_category'\")[0]['id'].to_i`. If no site_settings row exists for 'embed_category' (the common case, since it defaults to '' and is often never overridden), the query returns an empty result set, `[0]` is `nil`, and calling `['id']` on `nil` raises `NoMethodError`, crashing the migration for most existing installs. Location: db/migrate/20150818190757_create_embeddable_hosts.rb:10",
+          "confidence": 1.0,
+          "reasoning": "The candidate issue identifies the exact same problem as the golden comment: the migration accesses execute(...)[0]['id'] without checking if the query returned any rows, which will crash when no embed_category setting exists. Both comments point to the same bug at the same location (line 10 of the migration file), describe the same root cause (empty result set from the query), and the same consequence (crash/broken migration)."
+        }
+      ],
+      "false_positives": [
+        {
+          "candidate": "Migration inserts host values into SQL via string interpolation (`VALUES ('#{h}', ...)`) instead of parameter binding/quoting, which is unsafe if `h` (pulled from the old `embeddable_hosts` site setting) contains a single quote or other SQL metacharacters. Location: db/migrate/20150818190757_create_embeddable_hosts.rb:25"
+        }
+      ],
+      "false_negatives": [
+        {
+          "golden_comment": "Because this migration inserts embeddable_hosts rows with raw SQL, any existing embeddable_hosts values that include http:// or /https:// or path segments won\u2019t go through the EmbeddableHost model\u2019s normalization, so the new host lookup (which compares only the bare host) may fail for migrated data. Consider ensuring that migrated hosts are normalized to the same format as newly created EmbeddableHost records so existing embedding configurations keep working.",
+          "severity": "High",
+          "category": "data"
+        },
+        {
+          "golden_comment": "The SQL query in the `app/controllers/admin/embeddable_hosts_controller.rb` file uses string interpolation (`'#{h}'`) to insert values, which is vulnerable to SQL injection. It should use parameter binding or proper quoting to prevent this security vulnerability.",
+          "severity": "Critical",
+          "category": "security"
+        }
+      ],
+      "errors": [],
+      "total_candidates": 6,
+      "total_golden": 7,
+      "tp": 5,
+      "fp": 1,
+      "fn": 2,
+      "errors_count": 0,
+      "precision": 0.8333333333333334,
+      "recall": 0.7142857142857143,
+      "tool": "shipwright",
+      "repo_name": "shipwright__https://github.com/discourse/discourse/commit/d1c69189f3c90ecf56013a8da904da9bff9a8e19",
+      "pr_url": "https://github.com/discourse/discourse/commit/d1c69189f3c90ecf56013a8da904da9bff9a8e19"
     }
   },
   "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/7": {
@@ -70529,6 +71942,72 @@
       "tool": "sourcery",
       "repo_name": "discourse__discourse-graphite__sourcery__PR7__20260308",
       "pr_url": "https://github.com/code-review-benchmark/discourse__discourse-graphite__sourcery__PR7__20260308/pull/1"
+    },
+    "shipwright": {
+      "skipped": false,
+      "true_positives": [
+        {
+          "golden_comment": "In .topic-meta-data h5 a, the original code had color: scale-color($primary, $lightness: 30%) but was changed to dark-light-choose(scale-color($primary, $lightness: 70%), scale-color($secondary, $lightness: 30%)). The lightness for the light theme changed from 30% to 70%, which is a dramatic inversion",
+          "severity": "Low",
+          "category": "style",
+          "matched_candidate": "When converting to dark-light-choose, the light-theme (primary) lightness value was changed from 30% to 70% instead of being preserved, unlike every other conversion in this diff which keeps the original primary lightness for the light theme and uses (100-x)% for the secondary/dark value. This alters the light-theme link color unintentionally. Location: app/assets/stylesheets/desktop/topic-post.scss:291",
+          "confidence": 0.98,
+          "reasoning": "The candidate issue identifies the exact same problem as the golden comment: in the .topic-meta-data h5 a selector, the lightness value for the light theme was changed from 30% to 70% when converting to dark-light-choose, which is inconsistent with other conversions in the diff and represents an unintentional change to the light-theme link color. Both comments point to the same bug with the same specific values (30% to 70% lightness change)."
+        },
+        {
+          "golden_comment": "This change for desktop/user.css changes $primary from 30% to 50% for the light theme; most other changes preserve the original $primary value and move the complement to $secondary for dark. Consider reviewing this (also applies to a similar .name change in the mobile variant).",
+          "severity": "Low",
+          "category": "speculative",
+          "matched_candidate": "The original primary lightness of 30% was replaced with 50% for both the light and dark arguments of dark-light-choose, instead of preserving 30% for light and using the complementary 70% for dark (the pattern used everywhere else in this diff). This changes the light-theme .name color unintentionally. Location: app/assets/stylesheets/desktop/user.scss:522",
+          "confidence": 0.95,
+          "reasoning": "The candidate issue identifies the exact same problem as the golden comment: the change from 30% to 50% for the primary lightness in the light theme for the .name selector in desktop/user.scss. Both comments point out that this deviates from the pattern used elsewhere (preserving original primary value for light, using complement for dark). The candidate even specifies the exact location (line 522) and notes this changes the light-theme color unintentionally. The golden comment also mentions a similar issue in the mobile variant, which the candidate doesn't mention, but the core issue identified is the same."
+        },
+        {
+          "golden_comment": "In topic-post.css the original code used $lightness: 70% but the replacement uses $lightness: 30% for the light theme. This makes the text significantly darker than intended.",
+          "severity": "Low",
+          "category": "bug",
+          "matched_candidate": "When converting to dark-light-choose, the light-theme (primary) lightness value was changed from 30% to 70% instead of being preserved, unlike every other conversion in this diff which keeps the original primary lightness for the light theme and uses (100-x)% for the secondary/dark value. This alters the light-theme link color unintentionally. Location: app/assets/stylesheets/desktop/topic-post.scss:291",
+          "confidence": 0.85,
+          "reasoning": "The candidate issue identifies the same underlying problem - a lightness value mismatch in the dark-light-choose conversion. However, the candidate describes it as '30% to 70%' while the golden comment says '70% to 30%'. Both are pointing to the same issue: the light theme lightness value is incorrect after the conversion. The candidate correctly identifies this is in topic-post.scss and that it alters the light-theme color unintentionally. The direction of the error description is inverted, but they're identifying the same bug - the lightness values are swapped/incorrect for the light theme."
+        },
+        {
+          "golden_comment": "The light theme color for `.topic-meta-data h5 a` was changed from `scale-color($primary, $lightness: 30%)` to `scale-color($primary, $lightness: 70%)`, which may unintentionally reduce contrast and affect readability. This change should be reviewed to ensure it aligns with design intentions.",
+          "severity": "Medium",
+          "category": "bug",
+          "matched_candidate": "When converting to dark-light-choose, the light-theme (primary) lightness value was changed from 30% to 70% instead of being preserved, unlike every other conversion in this diff which keeps the original primary lightness for the light theme and uses (100-x)% for the secondary/dark value. This alters the light-theme link color unintentionally. Location: app/assets/stylesheets/desktop/topic-post.scss:291",
+          "confidence": 0.98,
+          "reasoning": "The candidate issue identifies the exact same problem as the golden comment: the light theme color for `.topic-meta-data h5 a` was changed from 30% lightness to 70% lightness, which is an unintentional change that differs from the pattern used elsewhere in the diff. Both comments point out this specific value change and flag it as potentially problematic/unintentional. The candidate even provides the specific file location."
+        },
+        {
+          "golden_comment": "In mobile/user.scss line 497, the lightness value for dark themes should be inverted to 70% instead of being changed to 50%. This change may lead to unintended visual regressions in light themes.",
+          "severity": "Medium",
+          "category": "bug",
+          "matched_candidate": "The original primary lightness of 30% for .name was replaced with 50% for both light and dark arguments of dark-light-choose instead of preserving 30% for light and 70% for dark, unintentionally changing the light-theme appearance. Location: app/assets/stylesheets/mobile/user.scss:497",
+          "confidence": 0.95,
+          "reasoning": "The candidate issue identifies the same problem as the golden comment - both point to line 497 in mobile/user.scss where the dark-light-choose function was incorrectly modified. Both note that: 1) the lightness value for dark themes should be 70% (not 50%), and 2) this change affects light theme appearance unintentionally. The candidate provides slightly more detail by mentioning the original 30% value should be preserved for light themes, but it's fundamentally the same issue."
+        }
+      ],
+      "false_positives": [
+        {
+          "candidate": "The primary/secondary lightness values were swapped when converting to dark-light-choose: light theme now uses 30% instead of the original 70%, and dark theme uses 70% instead of the expected complementary 30%. This inverts the intended light/dark appearance instead of preserving the light theme and only adding a dark-theme override. Location: app/assets/stylesheets/mobile/modal.scss:101"
+        },
+        {
+          "candidate": "The original primary lightness of 20% for h3 was replaced with 50% for both light and dark arguments instead of preserving 20% for light and 80% for dark, deviating from the conversion pattern used elsewhere and unintentionally altering the light-theme heading color. Location: app/assets/stylesheets/mobile/topic-post.scss:182"
+        }
+      ],
+      "false_negatives": [],
+      "errors": [],
+      "total_candidates": 5,
+      "total_golden": 5,
+      "tp": 5,
+      "fp": 2,
+      "fn": 0,
+      "errors_count": 0,
+      "precision": 1.0,
+      "recall": 1.0,
+      "tool": "shipwright",
+      "repo_name": "shipwright__https://github.com/discourse/discourse/commit/d38c4d5f7443223c81529c22470293771baf9f38",
+      "pr_url": "https://github.com/discourse/discourse/commit/d38c4d5f7443223c81529c22470293771baf9f38"
     }
   },
   "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/8": {
@@ -73343,6 +74822,69 @@
       "tool": "sourcery",
       "repo_name": "discourse__discourse-graphite__sourcery__PR8__20260308",
       "pr_url": "https://github.com/code-review-benchmark/discourse__discourse-graphite__sourcery__PR8__20260308/pull/1"
+    },
+    "shipwright": {
+      "skipped": false,
+      "true_positives": [
+        {
+          "golden_comment": "In the next action, capping the next offset at user_count can produce an empty page (e.g., total equal to limit results in offset == total, showing 2/2 with no members). This can cause confusing UX on the last page.",
+          "severity": "Medium",
+          "category": "bug",
+          "matched_candidate": "`totalPages` is computed as `Math.floor(user_count / limit) + 1`, which is off-by-one whenever `user_count` is an exact multiple of `limit` (e.g. user_count=50, limit=50 gives totalPages=2 instead of 1). This causes `showingLast` to stay false one page too early, so the \"next\" pagination control remains enabled past the real last page; clicking it sets `offset` to `user_count` and fetches an empty page, showing an extra blank page in the member list. Location: app/assets/javascripts/admin/controllers/admin-group.js.es6:13",
+          "confidence": 0.9,
+          "reasoning": "Both comments identify the same underlying issue: the pagination logic can result in an empty page being shown. The golden comment focuses on how capping the next offset at user_count can produce an empty page (offset == total). The candidate comment identifies the same problem from a different angle - the totalPages calculation being off-by-one when user_count is an exact multiple of limit, which causes the 'next' control to remain enabled past the real last page, resulting in fetching an empty page. Both are describing the same bug where users can navigate to an empty/blank page in the member list pagination."
+        },
+        {
+          "golden_comment": "The `totalPages` calculation using `Math.floor(user_count / limit) + 1` produces an extra page when `user_count` is an exact multiple of `limit`, leading to incorrect pagination. For example, with 100 users and a limit of 50, it reports 3 pages instead of 2, allowing navigation to an empty page.",
+          "severity": "Medium",
+          "category": "bug",
+          "matched_candidate": "`totalPages` is computed as `Math.floor(user_count / limit) + 1`, which is off-by-one whenever `user_count` is an exact multiple of `limit` (e.g. user_count=50, limit=50 gives totalPages=2 instead of 1). This causes `showingLast` to stay false one page too early, so the \"next\" pagination control remains enabled past the real last page; clicking it sets `offset` to `user_count` and fetches an empty page, showing an extra blank page in the member list. Location: app/assets/javascripts/admin/controllers/admin-group.js.es6:13",
+          "confidence": 0.98,
+          "reasoning": "The candidate issue identifies the exact same bug as the golden comment: the totalPages calculation using Math.floor(user_count / limit) + 1 produces an off-by-one error when user_count is an exact multiple of limit. Both describe the same consequence - an extra page being shown/accessible when it shouldn't be. The candidate even provides a similar example (50/50 giving 2 instead of 1, while golden uses 100/50 giving 3 instead of 2). They point to the same underlying mathematical error in the same calculation."
+        },
+        {
+          "golden_comment": "The route provides the group's ID in params[:id], but the code attempts to read it from params[:group_id], causing the action to fail. This results in a broken feature when trying to access group memberships.",
+          "severity": "High",
+          "category": "bug",
+          "matched_candidate": "routes.rb defines `delete \"members\" => \"groups#remove_member\"` and `put \"members\" => \"groups#add_members\"` as member routes nested under `resources :groups` (matching the URL pattern `/admin/groups/:id/members`, same style as the removed `get \"users\"`), so Rails will populate `params[:id]` with the group id. However `Admin::GroupsController#add_members` and `#remove_member` both call `params.require(:group_id)` to look up the group, which will never be present on a real request generated by the JS client (`Discourse.ajax('/admin/groups/' + id + '/members.json', ...)` in `group.js`). This raises `ActionController::ParameterMissing` and breaks the add/remove-member feature the PR claims to fix. The specs mask this by invoking the actions directly with an explicit `group_id:` param instead of going through routing, so the bug isn't caught by the added tests. Location: app/controllers/admin/groups_controller.rb:66",
+          "confidence": 0.98,
+          "reasoning": "The candidate issue identifies the exact same underlying problem as the golden comment: the route provides the group's ID in params[:id], but the controller code attempts to read it from params[:group_id], causing the action to fail. The candidate provides more detail about the specific methods affected (add_members and remove_member) and explains why the tests don't catch the bug, but it's fundamentally the same issue about the parameter name mismatch between routing and controller code."
+        }
+      ],
+      "false_positives": [
+        {
+          "candidate": "In the new `group_member.hbs` itemView template, `{{#unless automatic}}` is evaluated against the item view's context, which is the individual `member` (a Discourse.User) because it's rendered via `{{each member in members itemView=\"group-member\"}}` in `group.hbs`, not the parent Group model's `automatic` property. Since `member.automatic` is undefined, the `unless` block always renders, so the remove-member link is shown even for members of automatic groups (server-side still rejects via `can_not_modify_automatic`, but the UI incorrectly offers an action that will always fail). Location: app/assets/javascripts/admin/templates/group_member.hbs:1"
+        }
+      ],
+      "false_negatives": [
+        {
+          "golden_comment": " The findMembers() call is now asynchronous and unhandled. The controller may not have member data immediately available, creating a race condition.",
+          "severity": "High",
+          "category": "concurrency"
+        },
+        {
+          "golden_comment": "The remove_member test uses xhr :put to call the remove_member action, but the route defines delete 'members' => 'groups#remove_member' requiring DELETE method.",
+          "severity": "Medium",
+          "category": "api"
+        },
+        {
+          "golden_comment": "Expected to return a value at the end of function - code paths without explicit returns will return undefined",
+          "severity": "Medium",
+          "category": "bug"
+        }
+      ],
+      "errors": [],
+      "total_candidates": 3,
+      "total_golden": 6,
+      "tp": 3,
+      "fp": 1,
+      "fn": 3,
+      "errors_count": 0,
+      "precision": 1.0,
+      "recall": 0.5,
+      "tool": "shipwright",
+      "repo_name": "shipwright__https://github.com/discourse/discourse/commit/060cda77729cb1c4a827560e09e89a7b22078ba9",
+      "pr_url": "https://github.com/discourse/discourse/commit/060cda77729cb1c4a827560e09e89a7b22078ba9"
     }
   },
   "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/3": {
@@ -75358,6 +76900,48 @@
       "tool": "sourcery",
       "repo_name": "discourse__discourse-graphite__sourcery__PR3__20260308",
       "pr_url": "https://github.com/code-review-benchmark/discourse__discourse-graphite__sourcery__PR3__20260308/pull/1"
+    },
+    "shipwright": {
+      "skipped": false,
+      "true_positives": [
+        {
+          "golden_comment": "The case-sensitive email lookup in `BlockedEmail.should_block?` creates a bypass risk, allowing 'User@Example.com' to bypass a block on 'user@example.com'. This can lead to unauthorized access to blocked users.",
+          "severity": "Critical",
+          "category": "security",
+          "matched_candidate": "BlockedEmail.should_block? performs an exact string match (`BlockedEmail.where(email: email).first`) without normalizing case. Since emails are case-insensitive by RFC and neither BlockedEmail nor the validator lowercases the value before lookup, a blocked email like 'bad@spamclub.com' can bypass the block simply by submitting 'Bad@SpamClub.com', defeating the entire purpose of this new blocking feature. Location: app/models/blocked_email.rb:11",
+          "confidence": 0.98,
+          "reasoning": "Both comments identify the exact same issue: the BlockedEmail.should_block? method performs case-sensitive email matching, which allows users to bypass email blocks by using different letter casing (e.g., 'User@Example.com' vs 'user@example.com'). Both point to the same location (app/models/blocked_email.rb) and describe the same security vulnerability where the blocking feature can be defeated through case variation."
+        }
+      ],
+      "false_positives": [
+        {
+          "candidate": "`rejectedEmails: Em.A([])` is declared directly as a property value on the Ember.Object.extend() class definition rather than being initialized in an `init` hook. In Ember's classic object model, array/object literals defined this way are shared across all instances created from this class definition, so state from one instance (e.g. a previously rejected email) can leak into or persist across other instances of CreateAccountController. Location: app/assets/javascripts/discourse/controllers/create_account_controller.js:17"
+        }
+      ],
+      "false_negatives": [
+        {
+          "golden_comment": "BlockedEmail.should_block_email? method has side effects during a read operation - it updates statistics even when just checking if an email should be blocked. This could cause race conditions in concurrent environments and makes the method name misleading.",
+          "severity": "Medium",
+          "category": "concurrency"
+        },
+        {
+          "golden_comment": "The regex pattern @(#{domains}) in email_in_restriction_setting? method only matches domain suffixes, not full domains with proper boundary anchoring. An attacker can register evil.example.com to bypass a whitelist entry for example.com.",
+          "severity": "Medium",
+          "category": "bug"
+        }
+      ],
+      "errors": [],
+      "total_candidates": 2,
+      "total_golden": 3,
+      "tp": 1,
+      "fp": 1,
+      "fn": 2,
+      "errors_count": 0,
+      "precision": 0.5,
+      "recall": 0.3333333333333333,
+      "tool": "shipwright",
+      "repo_name": "shipwright__https://github.com/discourse/discourse/commit/5f8a130277dbddc95d133cd2832be639baf89213",
+      "pr_url": "https://github.com/discourse/discourse/commit/5f8a130277dbddc95d133cd2832be639baf89213"
     }
   },
   "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/5": {
@@ -77327,6 +78911,44 @@
       "tool": "sourcery",
       "repo_name": "discourse__discourse-graphite__sourcery__PR5__20260308",
       "pr_url": "https://github.com/code-review-benchmark/discourse__discourse-graphite__sourcery__PR5__20260308/pull/1"
+    },
+    "shipwright": {
+      "skipped": false,
+      "true_positives": [
+        {
+          "golden_comment": "Mixing float: left with flexbox causes layout issues. Further this PR removes the float-based right alignment for .d-header .panel, which may cause the login panel in the non-Ember/noscript header (where .panel is nested inside .row and not a flex item) to stack under the title instead of remaining right-aligned.",
+          "severity": "Low",
+          "category": "bug",
+          "matched_candidate": "`.panel` had `float: right;` removed and replaced with `margin-left: auto;` plus `@include order(3);`. These properties only have an effect when the element is a flex *item* inside a flex container. `.panel` is a sibling of `.contents` (same indentation level as `.contents`/`.valign-helper`), not nested inside it, and no flex container is declared for their shared parent in this diff. As a result `.panel` loses its previous right-floating alignment with no functional replacement, breaking the header's right-side layout (login/sign-up buttons, panel icons, etc.). Location: app/assets/stylesheets/common/base/header.scss:36",
+          "confidence": 0.92,
+          "reasoning": "Both comments identify the same core issue: the removal of `float: right` from `.panel` and its replacement with flexbox properties (`margin-left: auto`, `order`) that won't work because `.panel` is not inside a flex container. Both point out this will break the right-side alignment of the header panel (login buttons, etc.). The golden comment specifically mentions the non-Ember/noscript header scenario, while the candidate describes the same structural problem (`.panel` not being a flex item). They're identifying the same underlying bug."
+        }
+      ],
+      "false_positives": [],
+      "false_negatives": [
+        {
+          "golden_comment": "-ms-align-items never existed in any version of IE/Edge; the correct legacy property is -ms-flex-align.",
+          "severity": "Low",
+          "category": "bug"
+        },
+        {
+          "golden_comment": "-webkit-box-ordinal-group and -moz-box-ordinal-group require values starting from 1, not 0. If $val is 0, this will cause incorrect rendering in older browsers, leading to layout issues.",
+          "severity": "Medium",
+          "category": "bug"
+        }
+      ],
+      "errors": [],
+      "total_candidates": 2,
+      "total_golden": 3,
+      "tp": 1,
+      "fp": 0,
+      "fn": 2,
+      "errors_count": 0,
+      "precision": 0.5,
+      "recall": 0.3333333333333333,
+      "tool": "shipwright",
+      "repo_name": "shipwright__https://github.com/discourse/discourse/commit/5b229316ee4c661836ed1161139692a3e8527444",
+      "pr_url": "https://github.com/discourse/discourse/commit/5b229316ee4c661836ed1161139692a3e8527444"
     }
   },
   "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/6": {
@@ -78952,6 +80574,40 @@
       "tool": "sourcery",
       "repo_name": "discourse__discourse-graphite__sourcery__PR6__20260308",
       "pr_url": "https://github.com/code-review-benchmark/discourse__discourse-graphite__sourcery__PR6__20260308/pull/1"
+    },
+    "shipwright": {
+      "skipped": false,
+      "true_positives": [
+        {
+          "golden_comment": "The include_website_name method is missing the required ? suffix. Rails serializers expect include_ methods to end with ? for conditional attribute inclusion, a convention followed by other methods in this serializer. Without it, the website_name attribute may not be conditionally included as intended. Additionally, the '.' << website_host string concatenation should be replaced with '.' + website_host or '.#{website_host}' to avoid mutating string literals, which can lead to issues.",
+          "severity": "Medium",
+          "category": "api",
+          "matched_candidate": "`include_website_name` is missing the trailing `?` required by Discourse's ActiveModel::Serializer convention for conditional-attribute inclusion (see how other `include_<attr>?` methods work). Because the method is not recognized as a presence guard, `website_name` will always be serialized regardless of whether the user has a website set, contradicting the intended `website.present?` check and returning a bogus `website_name: nil` for users without a website. Location: app/serializers/user_serializer.rb:152",
+          "confidence": 0.9,
+          "reasoning": "The candidate issue correctly identifies the missing '?' suffix on the include_website_name method, which is the primary issue mentioned in the golden comment. Both point out that Rails/Discourse serializers expect include_ methods to end with '?' for conditional attribute inclusion, and that without it, the website_name attribute won't be conditionally included as intended. However, the candidate issue does not mention the secondary concern about string concatenation ('.' << website_host) that the golden comment raises. Despite missing this secondary issue, the main issue (missing '?' suffix) is correctly identified."
+        }
+      ],
+      "false_positives": [
+        {
+          "candidate": "The new `website_name` domain-comparison logic compares hostnames (`website_host`, `discourse_host`) directly without normalizing case (no `.downcase`). Hostnames are case-insensitive per RFC, so a website URL with a mixed-case host (e.g. `http://Example.com/user`) will fail the `website_host == discourse_host` and subdomain-suffix comparisons even when it is actually the same domain as `Discourse.current_hostname`, causing the full path to not be shown when it should be. Location: app/serializers/user_serializer.rb:140"
+        },
+        {
+          "candidate": "The subdomain-matching branch strips only the first label of each host and compares the remaining suffix (`split('.')[1..-1].join('.')`) without validating that this remainder is a real registrable domain. For hosts under shared multi-label public suffixes (e.g. `foo.co.uk` vs `bar.co.uk`), the stripped suffix `co.uk` will match even though the two hosts are unrelated organizations, incorrectly causing the full website path to be exposed as if the website and instance shared the same domain. Location: app/serializers/user_serializer.rb:143"
+        }
+      ],
+      "false_negatives": [],
+      "errors": [],
+      "total_candidates": 3,
+      "total_golden": 1,
+      "tp": 1,
+      "fp": 2,
+      "fn": 0,
+      "errors_count": 0,
+      "precision": 0.3333333333333333,
+      "recall": 1.0,
+      "tool": "shipwright",
+      "repo_name": "shipwright__https://github.com/discourse/discourse/commit/267d8be1f556ed59639ced396c885bb44586da19",
+      "pr_url": "https://github.com/discourse/discourse/commit/267d8be1f556ed59639ced396c885bb44586da19"
     }
   },
   "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/4": {
@@ -82633,6 +84289,82 @@
       "tool": "sourcery",
       "repo_name": "discourse__discourse-graphite__sourcery__PR4__20260308",
       "pr_url": "https://github.com/code-review-benchmark/discourse__discourse-graphite__sourcery__PR4__20260308/pull/1"
+    },
+    "shipwright": {
+      "skipped": false,
+      "true_positives": [
+        {
+          "golden_comment": "The current origin validation using indexOf is insufficient and can be bypassed. An attacker could use a malicious domain like evil-discourseUrl.com to pass this check.",
+          "severity": "Medium",
+          "category": "security",
+          "matched_candidate": "postMessage origin validation uses substring containment (`discourseUrl.indexOf(e.origin) === -1`) instead of a strict origin equality/prefix check. This is a fragile/incorrect security check: it tests whether the message's origin string appears anywhere inside discourseUrl rather than validating that the message actually originates from discourseUrl, which is the wrong direction for origin validation and can silently accept or reject messages incorrectly compared to a proper equality check. Location: app/assets/javascripts/embed.js:15",
+          "confidence": 0.95,
+          "reasoning": "The candidate issue identifies the same security vulnerability as the golden comment - the flawed origin validation using indexOf/substring containment. The golden comment specifically mentions that an attacker could bypass this check with a malicious domain like 'evil-discourseUrl.com'. The candidate issue describes the same problem: using substring containment instead of strict origin equality, which is 'fragile/incorrect security check' that can 'silently accept or reject messages incorrectly'. Both are pointing to the same security flaw in the postMessage origin validation at the same location."
+        }
+      ],
+      "false_positives": [
+        {
+          "candidate": "When cook_method is raw_html, Post#cook now returns `raw` directly and skips post_analyzer.cook/Plugin::Filter.apply entirely (no sanitization pipeline runs). Jobs::PollFeed.poll_feed feeds raw external RSS/Atom HTML content (`CGI.unescapeHTML(i.content.scrub)`) straight into TopicEmbed.import -> PostCreator with cook_method: raw_html, and the embed view (app/views/embed/best.html.erb) renders `post.cooked` with `raw`. This creates a stored XSS vector: arbitrary/untrusted RSS feed content is rendered unsanitized as raw HTML in the embed iframe. Location: app/models/post.rb:131"
+        },
+        {
+          "candidate": "The Disqus import thor task was refactored to call `TopicEmbed.import_remote(user, t[:link], title: t[:title])` instead of `PostCreator.new(user, title:, raw:, created_at:, category:)`. This drops the previously supported `--category`/`-c` CLI option (and its `category_id` lookup) entirely, and no longer passes `created_at: Date.parse(t[:created_at])`, so imported topics lose both category placement and their original creation date \u2014 a functional regression compared to the prior implementation. Location: lib/tasks/disqus.thor:140"
+        },
+        {
+          "candidate": "An already-existing migration (`create_table :top_topics`) was modified to add `force: true`. Since this file's timestamp/version predates the newly added migrations in this PR, any environment that already applied this migration and has data in `top_topics` risks that table being dropped and recreated if the migration is ever re-run (e.g., via db:migrate:redo or manual re-application), causing data loss. Existing migrations that have already run should not be edited to add destructive options. Location: db/migrate/20131223171005_create_top_topics.rb:3"
+        },
+        {
+          "candidate": "EmbedController#ensure_embeddable compares `URI(request.referer || '').host` to `SiteSetting.embeddable_host` with a case-sensitive `!=` check and no normalization on either side. If an admin configures embeddable_host with different casing than the browser-supplied referer host (or vice versa), legitimate embed requests will be rejected as InvalidAccess, and conversely there is no guarantee both sides are normalized consistently for comparison. Location: app/controllers/embed_controller.rb:29"
+        }
+      ],
+      "false_negatives": [
+        {
+          "golden_comment": "The PollFeed job calls open(SiteSetting.feed_polling_url) without validating the URL, allowing an attacker who controls the feed_polling_url site setting to make the server fetch arbitrary internal or external URLs.",
+          "severity": "Critical",
+          "category": "security"
+        },
+        {
+          "golden_comment": "postMessage targetOrigin should be the origin (scheme+host+port), not the full referrer URL; using the full URL will cause the message to be dropped and prevent resizing.",
+          "severity": "Medium",
+          "category": "security"
+        },
+        {
+          "golden_comment": "The code sets X-Frame-Options: ALLOWALL which completely disables clickjacking protection. The referer validation can be bypassed (referer headers are easily spoofed), and the fallback to empty string for nil referer masks validation failures.",
+          "severity": "Medium",
+          "category": "security"
+        },
+        {
+          "golden_comment": "The TopicEmbed.import method is susceptible to a NoMethodError if the contents parameter is nil when attempting to append a string, and an XSS vulnerability due to unescaped url interpolation in the generated HTML.",
+          "severity": "Medium",
+          "category": "bug"
+        },
+        {
+          "golden_comment": "The ERB block closes with end if, which is invalid Ruby/ERB and will raise at render; it should just be end to close the if block.",
+          "severity": "Medium",
+          "category": "bug"
+        },
+        {
+          "golden_comment": "Calling `i.content.scrub` assumes that `i.content` is always populated. However, many RSS feeds do not provide a `content` field, leading to a potential nil pointer exception when `i.content` is nil. A fallback to `description` should be implemented to prevent crashes.",
+          "severity": "High",
+          "category": "bug"
+        },
+        {
+          "golden_comment": "An unescaped request header (request.referer) is interpolated into JavaScript without proper escaping, which can lead to XSS vulnerabilities if the referer contains malicious content. This issue can allow attackers to execute arbitrary scripts in the context of the application.",
+          "severity": "Critical",
+          "category": "security"
+        }
+      ],
+      "errors": [],
+      "total_candidates": 5,
+      "total_golden": 8,
+      "tp": 1,
+      "fp": 4,
+      "fn": 7,
+      "errors_count": 0,
+      "precision": 0.2,
+      "recall": 0.125,
+      "tool": "shipwright",
+      "repo_name": "shipwright__https://github.com/discourse/discourse/commit/4f8aed295a29954023b2849c060ef4fb299d1b5d",
+      "pr_url": "https://github.com/discourse/discourse/commit/4f8aed295a29954023b2849c060ef4fb299d1b5d"
     }
   },
   "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/1": {
@@ -84944,6 +86676,59 @@
       "tool": "sourcery",
       "repo_name": "discourse__discourse-graphite__sourcery__PR1__20260308",
       "pr_url": "https://github.com/code-review-benchmark/discourse__discourse-graphite__sourcery__PR1__20260308/pull/1"
+    },
+    "shipwright": {
+      "skipped": false,
+      "true_positives": [
+        {
+          "golden_comment": "The downsize method is defined twice. The second definition, which expects a single dimensions string parameter, overrides the first, which expected separate max_width and max_height parameters. This makes the original method unreachable and breaks existing code that calls it with separate width and height arguments.",
+          "severity": "Medium",
+          "category": "bug",
+          "matched_candidate": "`OptimizedImage.downsize` is now defined twice with different arities (`downsize(from, to, max_width, max_height, opts={})` and `downsize(from, to, dimensions, opts={})`). Ruby silently keeps only the second definition, making the first a dead/unreachable method whose original 4-positional-argument signature (from the pre-diff API) is gone. Any code still calling `downsize` with separate width/height arguments will now raise an ArgumentError. Location: app/models/optimized_image.rb:146",
+          "confidence": 1.0,
+          "reasoning": "The candidate issue identifies exactly the same problem as the golden comment: the downsize method is defined twice with different parameter signatures (one with separate max_width/max_height, one with a single dimensions string), Ruby keeps only the second definition, making the first unreachable, and existing code calling with separate width/height arguments will break. The candidate even provides the specific line number and method signatures."
+        },
+        {
+          "golden_comment": "Hardcoding maxSizeKB = 10 * 1024 ignores Discourse.SiteSettings['max_' + type + '_size_kb'], so the client-side limit can diverge from server-side and per-type settings (also applies to the 413 handler below).",
+          "severity": "Low",
+          "category": "bug",
+          "matched_candidate": "Client-side file size validation now uses a hardcoded 10*1024 KB (10MB) constant instead of the configurable `Discourse.SiteSettings['max_' + type + '_size_kb']` / `Discourse.SiteSettings.max_image_size_kb`. This breaks per-type size limits (e.g. avatars, attachments) and will diverge from the server-side `SiteSetting.max_image_size_kb` used elsewhere in this same PR, causing inconsistent validation between client and server. Location: app/assets/javascripts/discourse/lib/utilities.js:182",
+          "confidence": 0.98,
+          "reasoning": "The candidate issue identifies the exact same problem as the golden comment: the hardcoded maxSizeKB = 10 * 1024 value ignores the configurable Discourse.SiteSettings['max_' + type + '_size_kb'] setting, causing client-side limits to diverge from server-side and per-type settings. The candidate even provides more detail about the specific implications (avatars, attachments) and mentions the same concern about client-server inconsistency."
+        }
+      ],
+      "false_positives": [
+        {
+          "candidate": "The API upload URL download cap was changed from `SiteSetting.max_image_size_kb.kilobytes` to a hardcoded `10.megabytes`. This contradicts the PR's stated goal of allowing larger images to be downloaded and then downsized: any image larger than 10MB fetched via URL will now fail to download entirely (silently rescued to nil) before ever reaching the new downsize logic, and the cap no longer respects site configuration. Location: app/controllers/uploads_controller.rb:55"
+        },
+        {
+          "candidate": "The new downsize retry loop in the upload flow gives up silently after 5 attempts with no check on whether the image was actually reduced below `SiteSetting.max_image_size_kb`. If `OptimizedImage.downsize` fails or converges slowly, the oversized tempfile is passed straight to `Upload.create_for` anyway, defeating the max image size validation instead of enforcing it. Location: app/controllers/uploads_controller.rb:63"
+        }
+      ],
+      "false_negatives": [
+        {
+          "golden_comment": "Passing 80% as the dimensions can fail for animated GIFs when allow_animated_thumbnails is true, since the animated path uses gifsicle --resize-fit which expects WxH geometry, not a percentage; downsizing would then silently fail.",
+          "severity": "Medium",
+          "category": "data"
+        },
+        {
+          "golden_comment": "The loop checking `tempfile.size` may never terminate correctly due to stale cached size, leading to potential infinite loops or incorrect file handling after modifications.",
+          "severity": "Medium",
+          "category": "bug"
+        }
+      ],
+      "errors": [],
+      "total_candidates": 4,
+      "total_golden": 4,
+      "tp": 2,
+      "fp": 2,
+      "fn": 2,
+      "errors_count": 0,
+      "precision": 0.5,
+      "recall": 0.5,
+      "tool": "shipwright",
+      "repo_name": "shipwright__https://github.com/discourse/discourse/commit/ffbaf8c54269df2ce510de91245760fddce09896",
+      "pr_url": "https://github.com/discourse/discourse/commit/ffbaf8c54269df2ce510de91245760fddce09896"
     }
   },
   "https://github.com/ai-code-review-evaluation/discourse-graphite/pull/2": {
@@ -86974,6 +88759,48 @@
       "tool": "sourcery",
       "repo_name": "discourse__discourse-graphite__sourcery__PR2__20260308",
       "pr_url": "https://github.com/code-review-benchmark/discourse__discourse-graphite__sourcery__PR2__20260308/pull/1"
+    },
+    "shipwright": {
+      "skipped": false,
+      "true_positives": [
+        {
+          "golden_comment": "logic: Potential nil pointer exception - if no TopicUser record exists, tu will be nil and calling methods on it will crash",
+          "severity": "High",
+          "category": "bug",
+          "matched_candidate": "`tu = TopicUser.find_by(user_id: current_user.id, topic_id: params[:topic_id])` can return nil (no TopicUser row exists for that user/topic), but the following code immediately calls `tu.notification_level` and `tu.save!` without a nil guard, causing a NoMethodError crash instead of gracefully creating/handling the missing preference row. Location: app/controllers/topics_controller.rb:106",
+          "confidence": 0.98,
+          "reasoning": "The candidate issue identifies the exact same problem as the golden comment: TopicUser.find_by can return nil, and the code then calls methods on tu (notification_level, save!) without checking for nil first, which will cause a crash (NoMethodError/nil pointer exception). Both comments point to the same bug - missing nil guard for the TopicUser record."
+        },
+        {
+          "golden_comment": "Typo in property name: 'stopNotificiationsText' should be 'stopNotificationsText' (missing 'n' in 'Notifications')",
+          "severity": "Low",
+          "category": "style",
+          "matched_candidate": "The new controller property is misspelled `stopNotificiationsText` (should be `stopNotificationsText`); while it still works because the template references the same misspelled name, it is a misleading/incorrect identifier introduced by this PR. Location: app/assets/javascripts/discourse/controllers/topic-unsubscribe.js.es6:5",
+          "confidence": 1.0,
+          "reasoning": "The candidate issue identifies the exact same typo problem as the golden comment: 'stopNotificiationsText' should be 'stopNotificationsText' (extra 'i' making it 'Notificiation' instead of 'Notification'). Both comments point to the same misspelled property name in the same file."
+        }
+      ],
+      "false_positives": [
+        {
+          "candidate": "`:unsubscribe` was added to the `except:` list of `before_filter :ensure_logged_in`, so the new `unsubscribe` action can now be reached without a logged-in user. However, the action body unconditionally calls `TopicUser.find_by(user_id: current_user.id, ...)`, which raises `NoMethodError: undefined method 'id' for nil:NilClass` when `current_user` is nil (e.g. when the emailed unsubscribe link is opened in a logged-out browser). Location: app/controllers/topics_controller.rb:27"
+        },
+        {
+          "candidate": "The unsubscribe toggle logic only downgrades a user's notification_level to `regular` when it is above `regular` (e.g. tracking/watching), and only sets it to `muted` when it is already at or below `regular`. This contradicts the new copy shown to the user (\"You will stop receiving notifications for <title>\", config/locales/client.en.yml `topic.unsubscribe.stop_notifications`), since a watching/tracking user clicking the unsubscribe link will still be left at `regular` and can continue to receive notifications, not actually unsubscribed as promised. Location: app/controllers/topics_controller.rb:108"
+        }
+      ],
+      "false_negatives": [],
+      "errors": [],
+      "total_candidates": 4,
+      "total_golden": 2,
+      "tp": 2,
+      "fp": 2,
+      "fn": 0,
+      "errors_count": 0,
+      "precision": 0.5,
+      "recall": 1.0,
+      "tool": "shipwright",
+      "repo_name": "shipwright__https://github.com/discourse/discourse/commit/6669a2d94d76eea3b99b8c476d12b1eb66726b07",
+      "pr_url": "https://github.com/discourse/discourse/commit/6669a2d94d76eea3b99b8c476d12b1eb66726b07"
     }
   },
   "https://github.com/calcom/cal.com/pull/22532": {
@@ -89274,6 +91101,53 @@
       "tool": "sourcery",
       "repo_name": "cal_dot_com__cal.com__sourcery__PR22532__20260307",
       "pr_url": "https://github.com/code-review-benchmark/cal_dot_com__cal.com__sourcery__PR22532__20260307/pull/1"
+    },
+    "shipwright": {
+      "skipped": false,
+      "true_positives": [
+        {
+          "golden_comment": "Throwing a generic Error for authentication failures results in a 500 response in tRPC. Instead, it should return a structured TRPCError with an appropriate code, such as FORBIDDEN, to accurately reflect the error type.",
+          "severity": "High",
+          "category": "api",
+          "matched_candidate": "deleteCacheHandler throws a plain `Error(\"Credential not found or access denied\")` instead of a `TRPCError` with an appropriate code (e.g. NOT_FOUND/FORBIDDEN). In a tRPC mutation this will surface as a generic INTERNAL_SERVER_ERROR rather than a distinguishable client error, making it impossible for callers/monitoring to differentiate an authorization failure from an actual server fault. Location: packages/trpc/server/routers/viewer/calendars/deleteCache.handler.ts:21",
+          "confidence": 0.98,
+          "reasoning": "The candidate issue identifies the exact same problem as the golden comment: throwing a generic Error instead of a TRPCError with an appropriate code (like FORBIDDEN) for authentication/authorization failures. Both point out that this results in incorrect error handling - the golden says '500 response' while the candidate says 'INTERNAL_SERVER_ERROR', which are the same thing. Both recommend using TRPCError with appropriate codes like FORBIDDEN. The candidate even specifies the exact file location where this occurs."
+        }
+      ],
+      "false_positives": [
+        {
+          "candidate": "The `actions` prop in both AppListCard renders now unconditionally wrap `CredentialActionsDropdown` in a `<div className=\"flex w-32 justify-end\">`, whereas previously the whole block (including the wrapping div) was only rendered when `!delegationCredentialId && !disableConnectionModification`. Since `CredentialActionsDropdown` returns `null` when `!canDisconnect && !hasCache` (e.g., delegation credentials or calendars without cache), the surrounding div still renders, leaving an empty 8rem-wide, right-aligned box where nothing used to appear at all \u2014 a visible UI regression introduced by this diff. Location: packages/platform/atoms/selected-calendars/wrappers/SelectedCalendarsSettingsWebWrapper.tsx:70"
+        }
+      ],
+      "false_negatives": [
+        {
+          "golden_comment": "The updateManyByCredentialId call uses an empty data object, which prevents Prisma's @updatedAt decorator from updating the updatedAt timestamp. This results in inaccurate cache status tracking, as the timestamp isn't updated when the cache is refreshed. To fix this, explicitly set the updatedAt field.",
+          "severity": "Medium",
+          "category": "bug"
+        },
+        {
+          "golden_comment": "logic: macOS-specific sed syntax with empty string after -i flag will fail on Linux systems",
+          "severity": "Low",
+          "category": "bug"
+        },
+        {
+          "golden_comment": "Date is formatted with hardcoded 'en-US' locale instead of using the user's locale from useLocale, which can lead to incorrect date representation for users in different locales.",
+          "severity": "Medium",
+          "category": "bug"
+        }
+      ],
+      "errors": [],
+      "total_candidates": 2,
+      "total_golden": 4,
+      "tp": 1,
+      "fp": 1,
+      "fn": 3,
+      "errors_count": 0,
+      "precision": 0.5,
+      "recall": 0.25,
+      "tool": "shipwright",
+      "repo_name": "shipwright__https://github.com/calcom/cal.com/pull/22532",
+      "pr_url": "https://github.com/calcom/cal.com/pull/22532"
     }
   },
   "https://github.com/calcom/cal.com/pull/8330": {
@@ -91283,6 +93157,41 @@
       "tool": "sourcery",
       "repo_name": "cal_dot_com__cal.com__sourcery__PR8330__20260307",
       "pr_url": "https://github.com/code-review-benchmark/cal_dot_com__cal.com__sourcery__PR8330__20260307/pull/1"
+    },
+    "shipwright": {
+      "skipped": false,
+      "true_positives": [
+        {
+          "golden_comment": "Incorrect end time calculation using slotStartTime instead of slotEndTime",
+          "severity": "Medium",
+          "category": "bug",
+          "matched_candidate": "In the new working-hours check, `end` is computed using `slotStartTime` instead of `slotEndTime` (`const end = slotStartTime.hour() * 60 + slotStartTime.minute();`), so the code never actually validates that the slot's end time falls within the organizer's working hours. A slot that starts inside working hours but extends past closing time will incorrectly be treated as available. Location: packages/trpc/server/routers/viewer/slots.ts:140",
+          "confidence": 1.0,
+          "reasoning": "The candidate issue identifies exactly the same problem as the golden comment - that the end time calculation incorrectly uses slotStartTime instead of slotEndTime. The candidate provides additional context about the consequences (slots extending past closing time being incorrectly treated as available) and the specific location, but the core issue is identical."
+        },
+        {
+          "golden_comment": "Using === for dayjs object comparison will always return false as it compares object references, not values. Use .isSame() method instead: dayjs(date.start).add(utcOffset, 'minutes').isSame(dayjs(date.end).add(utcOffset, minutes))",
+          "severity": "Medium",
+          "category": "bug",
+          "matched_candidate": "The date-override equality check `dayjs(date.start).add(utcOffset, \"minutes\") === dayjs(date.end).add(utcOffset, \"minutes\")` compares two freshly-created dayjs object instances with `===`, which is always false since they are different object references (dayjs objects are never referentially equal even for identical values). This branch, meant to detect zero-duration/whole-day overrides, can never execute, silently breaking the intended handling of that case. Location: packages/trpc/server/routers/viewer/slots.ts:121",
+          "confidence": 0.98,
+          "reasoning": "The candidate issue identifies the exact same problem as the golden comment: using === to compare two dayjs objects will always return false because it compares object references rather than values. Both comments point to the same code location (the date-override equality check in slots.ts) and identify that this comparison will never work as intended. The candidate also correctly identifies the consequence (the branch can never execute) and implicitly suggests the same fix (using proper dayjs comparison methods)."
+        }
+      ],
+      "false_positives": [],
+      "false_negatives": [],
+      "errors": [],
+      "total_candidates": 2,
+      "total_golden": 2,
+      "tp": 2,
+      "fp": 0,
+      "fn": 0,
+      "errors_count": 0,
+      "precision": 1.0,
+      "recall": 1.0,
+      "tool": "shipwright",
+      "repo_name": "shipwright__https://github.com/calcom/cal.com/pull/8330",
+      "pr_url": "https://github.com/calcom/cal.com/pull/8330"
     }
   },
   "https://github.com/calcom/cal.com/pull/14943": {
@@ -93054,6 +94963,41 @@
       "tool": "sourcery",
       "repo_name": "cal_dot_com__cal.com__sourcery__PR14943__20260307",
       "pr_url": "https://github.com/code-review-benchmark/cal_dot_com__cal.com__sourcery__PR14943__20260307/pull/1"
+    },
+    "shipwright": {
+      "skipped": false,
+      "true_positives": [
+        {
+          "golden_comment": "Using retryCount: reminder.retryCount + 1 reads a possibly stale value and can lose increments under concurrency; consider an atomic increment via Prisma (increment: 1) to avoid race conditions (also applies to the similar update in the catch block).",
+          "severity": "High",
+          "category": "concurrency",
+          "matched_candidate": "retryCount is incremented via a non-atomic read-modify-write (`retryCount: reminder.retryCount + 1`) using a value fetched earlier in the same handler invocation, rather than Prisma's atomic `{ increment: 1 }`. If this cron/handler runs concurrently (e.g., overlapping invocations processing the same reminder), updates can race and increments can be lost, undermining the retry-count-based cleanup this PR introduces. Location: packages/features/ee/workflows/api/scheduleSMSReminders.ts:181",
+          "confidence": 0.98,
+          "reasoning": "The candidate issue identifies exactly the same problem as the golden comment: using `retryCount: reminder.retryCount + 1` is a non-atomic read-modify-write operation that can lose increments under concurrent execution, and recommends using Prisma's atomic `{ increment: 1 }` instead. The candidate even specifies the exact file and line location, and mentions the same concern about race conditions affecting the retry-count-based cleanup logic."
+        },
+        {
+          "golden_comment": "The deletion logic in scheduleSMSReminders.ts incorrectly deletes non-SMS workflow reminders (e.g., Email, WhatsApp) that have retryCount > 1. This occurs because the retryCount condition within the OR clause for deletion lacks a method: WorkflowMethods.SMS filter, causing it to apply to all reminder types instead of only SMS reminders, which is the intended scope of this function.",
+          "severity": "High",
+          "category": "bug",
+          "matched_candidate": "The new OR clause in deleteMany omits the `method: WorkflowMethods.SMS` (and scheduledDate) constraint that the first clause has, so any WorkflowReminder row (not just SMS reminders, and regardless of scheduledDate) with retryCount > 1 will be deleted. Since retryCount is a generic column added to WorkflowReminder, this can delete non-SMS reminders (email/whatsapp) that happen to have retryCount incremented elsewhere, unintentionally purging reminders unrelated to the SMS retry logic this PR is meant to implement. Location: packages/features/ee/workflows/api/scheduleSMSReminders.ts:34",
+          "confidence": 0.98,
+          "reasoning": "The candidate issue identifies the exact same problem as the golden comment: the deletion logic in scheduleSMSReminders.ts incorrectly deletes non-SMS workflow reminders (Email, WhatsApp) because the retryCount > 1 condition in the OR clause lacks the `method: WorkflowMethods.SMS` filter. Both comments point to the same bug where the retryCount condition applies to all reminder types instead of only SMS reminders, which is the intended scope of the function."
+        }
+      ],
+      "false_positives": [],
+      "false_negatives": [],
+      "errors": [],
+      "total_candidates": 2,
+      "total_golden": 2,
+      "tp": 2,
+      "fp": 0,
+      "fn": 0,
+      "errors_count": 0,
+      "precision": 1.0,
+      "recall": 1.0,
+      "tool": "shipwright",
+      "repo_name": "shipwright__https://github.com/calcom/cal.com/pull/14943",
+      "pr_url": "https://github.com/calcom/cal.com/pull/14943"
     }
   },
   "https://github.com/calcom/cal.com/pull/22345": {
@@ -94708,118 +96652,147 @@
       "errors_count": 0,
       "precision": 0.0,
       "recall": 0.0,
-      "tool": "kodus-v2",
-      "repo_name": "cal_dot_com__cal.com__kodus-v2__PR22345__20260312",
-      "pr_url": "https://github.com/code-review-benchmark/cal_dot_com__cal.com__kodus-v2__PR22345__20260312/pull/1"
-    },
-    "macroscope": {
-      "skipped": false,
-      "true_positives": [
-        {
-          "golden_comment": "Fetching userIdsFromOrg only when teamsFromOrg.length > 0 can exclude org-level members for orgs without child teams; consider deriving from teamIds (which includes orgId) or removing the guard so org-only orgs still include member user bookings.",
-          "severity": "Medium",
-          "category": "speculative",
-          "matched_candidate": "When an organization has no child teams, `userIdsFromOrg` is set to an empty array and the query for org members is skipped entirely. This causes personal bookings (`isTeamBooking = false`) from members who are directly in the org to be missing from org-scope insights. The condition should query members of `options.orgId` regardless of whether there are child teams.",
-          "confidence": 0.95,
-          "reasoning": "Both comments identify the same issue: when an organization has no child teams, the userIdsFromOrg query is skipped (set to empty array), which causes org-level members' bookings to be excluded from the results. Both suggest that org members should be queried regardless of whether child teams exist."
-        }
-      ],
-      "false_positives": [],
-      "false_negatives": [
-        {
-          "golden_comment": "In getBaseConditions(), the else if (filterConditions) and final else branches are unreachable. This is because getAuthorizationConditions() always returns a non-null Prisma.Sql object, making authConditions always truthy, which means only the first two if/else if conditions are ever evaluated.",
-          "severity": "Low",
-          "category": "bug"
-        }
-      ],
-      "errors": [],
-      "total_candidates": 1,
-      "total_golden": 2,
-      "tp": 1,
-      "fp": 0,
-      "fn": 1,
-      "errors_count": 0,
-      "precision": 1.0,
-      "recall": 0.5,
-      "tool": "macroscope",
-      "repo_name": "cal_dot_com__cal.com__macroscope__PR22345__20260309",
-      "pr_url": "https://github.com/code-review-benchmark/cal_dot_com__cal.com__macroscope__PR22345__20260309/pull/1"
-    },
-    "qodo-v2": {
-      "skipped": false,
-      "true_positives": [
-        {
-          "golden_comment": "Fetching userIdsFromOrg only when teamsFromOrg.length > 0 can exclude org-level members for orgs without child teams; consider deriving from teamIds (which includes orgId) or removing the guard so org-only orgs still include member user bookings.",
-          "severity": "Medium",
-          "category": "speculative",
-          "matched_candidate": "buildOrgAuthorizationCondition only fetches org user IDs when teamsFromOrg.length > 0, even though org membership is represented via Membership on the org team ID itself. If an org has no child teams, the SQL will include only team bookings and will omit the user-booking branch for org members, leading to incomplete org-scope visibility.",
-          "confidence": 0.95,
-          "reasoning": "Both comments identify the same issue: the condition that only fetches org user IDs when teamsFromOrg.length > 0 will cause problems for orgs without child teams, leading to incomplete visibility of org member bookings. The candidate correctly identifies the same bug about org-level members being excluded when there are no child teams."
-        }
-      ],
-      "false_positives": [
-        {
-          "candidate": "Tests deep-compare Prisma.sql objects, but org/team authorization SQL embeds teamIds/userIds arrays derived from findMany queries without orderBy. That ordering is not guaranteed, so the produced Prisma.sql.values can vary between runs even if the SQL is semantically equivalent, causing flaky tests."
-        },
-        {
-          "candidate": "The constructor accepts a looser InsightsBookingServicePublicOptions where teamId is optional even when scope is 'team', so invalid option shapes compile and are only caught by runtime safeParse. On validation failure the service silently sets options=null and returns 1=0 authorization, which can turn a caller bug into confusing 'no data' behavior."
-        },
-        {
-          "candidate": "The integration test suite removed the caching tests but the service still relies on cachedAuthConditions/cachedFilterConditions. This reduces regression protection around caching semantics."
-        }
-      ],
-      "false_negatives": [
-        {
-          "golden_comment": "In getBaseConditions(), the else if (filterConditions) and final else branches are unreachable. This is because getAuthorizationConditions() always returns a non-null Prisma.Sql object, making authConditions always truthy, which means only the first two if/else if conditions are ever evaluated.",
-          "severity": "Low",
-          "category": "bug"
-        }
-      ],
-      "errors": [],
-      "total_candidates": 4,
-      "total_golden": 2,
-      "tp": 1,
-      "fp": 3,
-      "fn": 1,
-      "errors_count": 0,
-      "precision": 0.25,
-      "recall": 0.5,
-      "tool": "qodo-v2",
-      "repo_name": "cal_dot_com__cal.com__qodo-v2__PR22345__20260227",
-      "pr_url": "https://github.com/code-review-benchmark/cal_dot_com__cal.com__qodo-v2__PR22345__20260227/pull/1"
-    },
-    "sourcery": {
-      "skipped": false,
-      "true_positives": [
-        {
-          "golden_comment": "In getBaseConditions(), the else if (filterConditions) and final else branches are unreachable. This is because getAuthorizationConditions() always returns a non-null Prisma.Sql object, making authConditions always truthy, which means only the first two if/else if conditions are ever evaluated.",
-          "severity": "Low",
-          "category": "bug",
-          "matched_candidate": "getBaseConditions contains unreachable else branches (else if (filterConditions) and final else) because getAuthorizationConditions always returns a Prisma.Sql (defaulting to NOTHING_CONDITION), so authConditions is never falsy. Either allow getAuthorizationConditions to return null to make these branches meaningful, or simplify getBaseConditions to assume authConditions is always present.",
-          "confidence": 1.0,
-          "reasoning": "The candidate issue identifies exactly the same problem as the golden comment: the else if (filterConditions) and final else branches in getBaseConditions() are unreachable because getAuthorizationConditions() always returns a non-null Prisma.Sql object (defaulting to NOTHING_CONDITION), making authConditions always truthy. The candidate even provides the same suggested fixes."
-        }
-      ],
-      "false_positives": [],
-      "false_negatives": [
-        {
-          "golden_comment": "Fetching userIdsFromOrg only when teamsFromOrg.length > 0 can exclude org-level members for orgs without child teams; consider deriving from teamIds (which includes orgId) or removing the guard so org-only orgs still include member user bookings.",
-          "severity": "Medium",
-          "category": "speculative"
-        }
-      ],
-      "errors": [],
-      "total_candidates": 1,
-      "total_golden": 2,
-      "tp": 1,
-      "fp": 0,
-      "fn": 1,
-      "errors_count": 0,
-      "precision": 1.0,
-      "recall": 0.5,
-      "tool": "sourcery",
-      "repo_name": "cal_dot_com__cal.com__sourcery__PR22345__20260307",
-      "pr_url": "https://github.com/code-review-benchmark/cal_dot_com__cal.com__sourcery__PR22345__20260307/pull/1"
+      "tool": "kodus-v2",
+      "repo_name": "cal_dot_com__cal.com__kodus-v2__PR22345__20260312",
+      "pr_url": "https://github.com/code-review-benchmark/cal_dot_com__cal.com__kodus-v2__PR22345__20260312/pull/1"
+    },
+    "macroscope": {
+      "skipped": false,
+      "true_positives": [
+        {
+          "golden_comment": "Fetching userIdsFromOrg only when teamsFromOrg.length > 0 can exclude org-level members for orgs without child teams; consider deriving from teamIds (which includes orgId) or removing the guard so org-only orgs still include member user bookings.",
+          "severity": "Medium",
+          "category": "speculative",
+          "matched_candidate": "When an organization has no child teams, `userIdsFromOrg` is set to an empty array and the query for org members is skipped entirely. This causes personal bookings (`isTeamBooking = false`) from members who are directly in the org to be missing from org-scope insights. The condition should query members of `options.orgId` regardless of whether there are child teams.",
+          "confidence": 0.95,
+          "reasoning": "Both comments identify the same issue: when an organization has no child teams, the userIdsFromOrg query is skipped (set to empty array), which causes org-level members' bookings to be excluded from the results. Both suggest that org members should be queried regardless of whether child teams exist."
+        }
+      ],
+      "false_positives": [],
+      "false_negatives": [
+        {
+          "golden_comment": "In getBaseConditions(), the else if (filterConditions) and final else branches are unreachable. This is because getAuthorizationConditions() always returns a non-null Prisma.Sql object, making authConditions always truthy, which means only the first two if/else if conditions are ever evaluated.",
+          "severity": "Low",
+          "category": "bug"
+        }
+      ],
+      "errors": [],
+      "total_candidates": 1,
+      "total_golden": 2,
+      "tp": 1,
+      "fp": 0,
+      "fn": 1,
+      "errors_count": 0,
+      "precision": 1.0,
+      "recall": 0.5,
+      "tool": "macroscope",
+      "repo_name": "cal_dot_com__cal.com__macroscope__PR22345__20260309",
+      "pr_url": "https://github.com/code-review-benchmark/cal_dot_com__cal.com__macroscope__PR22345__20260309/pull/1"
+    },
+    "qodo-v2": {
+      "skipped": false,
+      "true_positives": [
+        {
+          "golden_comment": "Fetching userIdsFromOrg only when teamsFromOrg.length > 0 can exclude org-level members for orgs without child teams; consider deriving from teamIds (which includes orgId) or removing the guard so org-only orgs still include member user bookings.",
+          "severity": "Medium",
+          "category": "speculative",
+          "matched_candidate": "buildOrgAuthorizationCondition only fetches org user IDs when teamsFromOrg.length > 0, even though org membership is represented via Membership on the org team ID itself. If an org has no child teams, the SQL will include only team bookings and will omit the user-booking branch for org members, leading to incomplete org-scope visibility.",
+          "confidence": 0.95,
+          "reasoning": "Both comments identify the same issue: the condition that only fetches org user IDs when teamsFromOrg.length > 0 will cause problems for orgs without child teams, leading to incomplete visibility of org member bookings. The candidate correctly identifies the same bug about org-level members being excluded when there are no child teams."
+        }
+      ],
+      "false_positives": [
+        {
+          "candidate": "Tests deep-compare Prisma.sql objects, but org/team authorization SQL embeds teamIds/userIds arrays derived from findMany queries without orderBy. That ordering is not guaranteed, so the produced Prisma.sql.values can vary between runs even if the SQL is semantically equivalent, causing flaky tests."
+        },
+        {
+          "candidate": "The constructor accepts a looser InsightsBookingServicePublicOptions where teamId is optional even when scope is 'team', so invalid option shapes compile and are only caught by runtime safeParse. On validation failure the service silently sets options=null and returns 1=0 authorization, which can turn a caller bug into confusing 'no data' behavior."
+        },
+        {
+          "candidate": "The integration test suite removed the caching tests but the service still relies on cachedAuthConditions/cachedFilterConditions. This reduces regression protection around caching semantics."
+        }
+      ],
+      "false_negatives": [
+        {
+          "golden_comment": "In getBaseConditions(), the else if (filterConditions) and final else branches are unreachable. This is because getAuthorizationConditions() always returns a non-null Prisma.Sql object, making authConditions always truthy, which means only the first two if/else if conditions are ever evaluated.",
+          "severity": "Low",
+          "category": "bug"
+        }
+      ],
+      "errors": [],
+      "total_candidates": 4,
+      "total_golden": 2,
+      "tp": 1,
+      "fp": 3,
+      "fn": 1,
+      "errors_count": 0,
+      "precision": 0.25,
+      "recall": 0.5,
+      "tool": "qodo-v2",
+      "repo_name": "cal_dot_com__cal.com__qodo-v2__PR22345__20260227",
+      "pr_url": "https://github.com/code-review-benchmark/cal_dot_com__cal.com__qodo-v2__PR22345__20260227/pull/1"
+    },
+    "sourcery": {
+      "skipped": false,
+      "true_positives": [
+        {
+          "golden_comment": "In getBaseConditions(), the else if (filterConditions) and final else branches are unreachable. This is because getAuthorizationConditions() always returns a non-null Prisma.Sql object, making authConditions always truthy, which means only the first two if/else if conditions are ever evaluated.",
+          "severity": "Low",
+          "category": "bug",
+          "matched_candidate": "getBaseConditions contains unreachable else branches (else if (filterConditions) and final else) because getAuthorizationConditions always returns a Prisma.Sql (defaulting to NOTHING_CONDITION), so authConditions is never falsy. Either allow getAuthorizationConditions to return null to make these branches meaningful, or simplify getBaseConditions to assume authConditions is always present.",
+          "confidence": 1.0,
+          "reasoning": "The candidate issue identifies exactly the same problem as the golden comment: the else if (filterConditions) and final else branches in getBaseConditions() are unreachable because getAuthorizationConditions() always returns a non-null Prisma.Sql object (defaulting to NOTHING_CONDITION), making authConditions always truthy. The candidate even provides the same suggested fixes."
+        }
+      ],
+      "false_positives": [],
+      "false_negatives": [
+        {
+          "golden_comment": "Fetching userIdsFromOrg only when teamsFromOrg.length > 0 can exclude org-level members for orgs without child teams; consider deriving from teamIds (which includes orgId) or removing the guard so org-only orgs still include member user bookings.",
+          "severity": "Medium",
+          "category": "speculative"
+        }
+      ],
+      "errors": [],
+      "total_candidates": 1,
+      "total_golden": 2,
+      "tp": 1,
+      "fp": 0,
+      "fn": 1,
+      "errors_count": 0,
+      "precision": 1.0,
+      "recall": 0.5,
+      "tool": "sourcery",
+      "repo_name": "cal_dot_com__cal.com__sourcery__PR22345__20260307",
+      "pr_url": "https://github.com/code-review-benchmark/cal_dot_com__cal.com__sourcery__PR22345__20260307/pull/1"
+    },
+    "shipwright": {
+      "skipped": false,
+      "true_positives": [],
+      "false_positives": [],
+      "false_negatives": [
+        {
+          "golden_comment": "In getBaseConditions(), the else if (filterConditions) and final else branches are unreachable. This is because getAuthorizationConditions() always returns a non-null Prisma.Sql object, making authConditions always truthy, which means only the first two if/else if conditions are ever evaluated.",
+          "severity": "Low",
+          "category": "bug"
+        },
+        {
+          "golden_comment": "Fetching userIdsFromOrg only when teamsFromOrg.length > 0 can exclude org-level members for orgs without child teams; consider deriving from teamIds (which includes orgId) or removing the guard so org-only orgs still include member user bookings.",
+          "severity": "Medium",
+          "category": "speculative"
+        }
+      ],
+      "errors": [],
+      "total_candidates": 0,
+      "total_golden": 2,
+      "tp": 0,
+      "fp": 0,
+      "fn": 2,
+      "errors_count": 0,
+      "precision": 0.0,
+      "recall": 0.0,
+      "tool": "shipwright",
+      "repo_name": "shipwright__https://github.com/calcom/cal.com/pull/22345",
+      "pr_url": "https://github.com/calcom/cal.com/pull/22345"
     }
   },
   "https://github.com/calcom/cal.com/pull/11059": {
@@ -98554,6 +100527,90 @@
       "tool": "sourcery",
       "repo_name": "cal_dot_com__cal.com__sourcery__PR11059__20260307",
       "pr_url": "https://github.com/code-review-benchmark/cal_dot_com__cal.com__sourcery__PR11059__20260307/pull/1"
+    },
+    "shipwright": {
+      "skipped": false,
+      "true_positives": [
+        {
+          "golden_comment": "The parseRefreshTokenResponse function incorrectly sets refresh_token to the hardcoded string 'refresh_token' when it's missing from the OAuth refresh token response. This invalidates the token, breaking subsequent token refreshes and causing authentication failures.",
+          "severity": "High",
+          "category": "bug",
+          "matched_candidate": "When the refreshed token response lacks a `refresh_token` field, the code sets it to the literal string \"refresh_token\" instead of preserving the previous refresh token or leaving it undefined: `refreshTokenResponse.data.refresh_token = \"refresh_token\";`. This overwrites/corrupts the stored refresh token with an unusable placeholder value, breaking future token refresh attempts for any provider that omits refresh_token on refresh (a common OAuth behavior). Location: packages/app-store/_utils/oauth/parseRefreshTokenResponse.ts:24",
+          "confidence": 1.0,
+          "reasoning": "The candidate issue identifies exactly the same problem as the golden comment: the parseRefreshTokenResponse function incorrectly sets refresh_token to the hardcoded string 'refresh_token' when it's missing from the OAuth refresh token response. Both comments point out that this corrupts/invalidates the token and breaks subsequent token refresh attempts. The candidate even provides the specific file location and line number, and explains the same consequence (authentication failures/breaking future token refresh attempts)."
+        },
+        {
+          "golden_comment": "The Zod schema in parseRefreshTokenResponse uses computed property keys [z.string().toString()] and [z.string().optional().toString()] in minimumTokenResponseSchema, which is invalid Zod syntax and will throw a runtime error.",
+          "severity": "High",
+          "category": "bug",
+          "matched_candidate": "The computed property keys `[z.string().toString()]` and `[z.string().optional().toString()]` do not create dynamic/wildcard validators as the comments claim; `.toString()` is called on the Zod schema object (not a string), producing a fixed literal key (e.g. \"[object Object]\") for both entries, with the second silently overwriting the first. As a result, `minimumTokenResponseSchema` only actually validates `access_token`, and all other real response fields (expiry, scope, refresh_token, etc.) are stripped out of `.data` by zod's default unknown-key stripping when APP_CREDENTIAL_SHARING_ENABLED is on, causing loss of critical token metadata. Location: packages/app-store/_utils/oauth/parseRefreshTokenResponse.ts:9",
+          "confidence": 0.9,
+          "reasoning": "Both comments identify the same fundamental issue: the computed property keys using [z.string().toString()] and [z.string().optional().toString()] in minimumTokenResponseSchema are invalid/problematic Zod syntax. The golden comment says it will throw a runtime error, while the candidate provides more detail about what actually happens (toString() on Zod schema produces '[object Object]' causing the second to overwrite the first, leading to data loss). Both are pointing to the same bug with the computed property key syntax in the Zod schema, just with different explanations of the consequences."
+        },
+        {
+          "golden_comment": "parseRefreshTokenResponse returns a Zod safeParse result ({ success, data, error }), not the credential key object. Persisting that as key stores the wrapper instead of the token payload; we should store the parsed data or use schema parse.",
+          "severity": "High",
+          "category": "data",
+          "matched_candidate": "parseRefreshTokenResponse's return value (a safeParse-style {success, data} object) is now stored directly as the credential `key` instead of the actual parsed token data: `const key = parseRefreshTokenResponse(googleCredentials, googleCredentialSchema); ... data: { key }`. Previously `googleCredentialSchema.parse(...)` returned the flat token object; now the credential.key JSON will be `{success, data}` wrapper, corrupting the stored Google Calendar OAuth credentials and breaking subsequent calendar API calls. Location: packages/app-store/googlecalendar/lib/CalendarService.ts:97",
+          "confidence": 0.98,
+          "reasoning": "The candidate issue identifies exactly the same problem as the golden comment: parseRefreshTokenResponse returns a safeParse result object ({success, data} wrapper) instead of the actual token data, and this wrapper object is being stored as the credential key instead of the parsed token payload. Both comments point to the same bug at the same location (CalendarService.ts around line 97) and explain the same consequence - corrupted stored credentials."
+        },
+        {
+          "golden_comment": "When APP_CREDENTIAL_SHARING_ENABLED and CALCOM_CREDENTIAL_SYNC_ENDPOINT are set, the refreshFunction helper returns the fetch Response, but several callers (for example GoogleCalendarService.refreshAccessToken expecting res.data, and HubspotCalendarService.refreshAccessToken expecting a HubspotToken) assume it returns the integration-specific token object. That mismatch will cause runtime errors in the sync-enabled path unless the return type or those call sites are adjusted.",
+          "severity": "High",
+          "category": "api",
+          "matched_candidate": "`refreshOAuthTokens` returns a raw, unparsed `fetch` Response when credential sync is enabled, but returns provider-specific parsed shapes in the local-refresh branch (e.g. `fetchTokens.res` with a `.data` property for Google, or the `HubspotToken` object for Hubspot). Callers such as GoogleCalendarService (`const token = res?.data;`) and HubspotCalendarService (typed as `HubspotToken`) assume a single shape and will fail/throw at runtime (e.g. `token.access_token` on undefined) when APP_CREDENTIAL_SHARING_ENABLED and CALCOM_CREDENTIAL_SYNC_ENDPOINT are set. Location: packages/app-store/_utils/oauth/refreshOAuthTokens.ts:6",
+          "confidence": 0.98,
+          "reasoning": "The candidate issue identifies the exact same problem as the golden comment: when APP_CREDENTIAL_SHARING_ENABLED and CALCOM_CREDENTIAL_SYNC_ENDPOINT are set, the refreshOAuthTokens function returns a raw fetch Response, but callers like GoogleCalendarService (expecting res.data) and HubspotCalendarService (expecting HubspotToken) assume integration-specific token objects. This mismatch will cause runtime errors. The candidate even mentions the same specific examples (GoogleCalendarService with res.data, HubspotCalendarService with HubspotToken) and the same root cause."
+        },
+        {
+          "golden_comment": "When the sync endpoint path is used, res is a fetch Response and has no .data; res?.data will be undefined and token.access_token will throw at runtime. This relies on a consistent return shape from refreshOAuthTokens, which isn\u2019t guaranteed currently.",
+          "severity": "High",
+          "category": "bug",
+          "matched_candidate": "`refreshOAuthTokens` returns a raw, unparsed `fetch` Response when credential sync is enabled, but returns provider-specific parsed shapes in the local-refresh branch (e.g. `fetchTokens.res` with a `.data` property for Google, or the `HubspotToken` object for Hubspot). Callers such as GoogleCalendarService (`const token = res?.data;`) and HubspotCalendarService (typed as `HubspotToken`) assume a single shape and will fail/throw at runtime (e.g. `token.access_token` on undefined) when APP_CREDENTIAL_SHARING_ENABLED and CALCOM_CREDENTIAL_SYNC_ENDPOINT are set. Location: packages/app-store/_utils/oauth/refreshOAuthTokens.ts:6",
+          "confidence": 0.98,
+          "reasoning": "The candidate issue identifies the exact same problem as the golden comment: when the sync endpoint path is used, refreshOAuthTokens returns a raw fetch Response, but callers like GoogleCalendarService expect a .data property (res?.data). The candidate explicitly mentions that 'token.access_token' will fail/throw at runtime when res?.data is undefined, which matches the golden comment's concern about 'res?.data will be undefined and token.access_token will throw at runtime'. Both issues point to the inconsistent return shape from refreshOAuthTokens causing runtime failures."
+        }
+      ],
+      "false_positives": [
+        {
+          "candidate": "The webhook secret is compared using strict inequality (`!==`) rather than a constant-time comparison, making the credential-sync webhook endpoint vulnerable to timing attacks that could help an attacker guess CALCOM_WEBHOOK_SECRET. Location: apps/web/pages/api/webhook/app-credential.ts:21"
+        }
+      ],
+      "false_negatives": [
+        {
+          "golden_comment": "The `refreshOAuthTokens` function is passed `credentialId` instead of `userId`, which breaks the credential sync functionality by causing incorrect user lookups. This issue can lead to failures in syncing user credentials, impacting user experience.",
+          "severity": "High",
+          "category": "bug"
+        },
+        {
+          "golden_comment": "The Salesforce integration checks `response.statusText` instead of `response.ok` when handling OAuth token refresh responses. This can lead to unreliable behavior, as a successful response may not be correctly identified, resulting in the use of stale credentials and potential authentication failures.",
+          "severity": "High",
+          "category": "bug"
+        },
+        {
+          "golden_comment": "The `jsforce.Connection` is created using stale `credentialKey.access_token` and `credentialKey.instance_url` after a token refresh, which can lead to authentication failures and incorrect data access. Ensure that the connection is established using the refreshed token values instead.",
+          "severity": "High",
+          "category": "bug"
+        },
+        {
+          "golden_comment": "The `appCredentialWebhookRequestBodySchema.parse` function can throw on invalid payloads, resulting in 500 Internal Server Errors instead of the expected 400 Bad Request responses. This can lead to confusion for clients and hinder proper error handling.",
+          "severity": "Medium",
+          "category": "api"
+        }
+      ],
+      "errors": [],
+      "total_candidates": 5,
+      "total_golden": 9,
+      "tp": 5,
+      "fp": 1,
+      "fn": 4,
+      "errors_count": 0,
+      "precision": 1.0,
+      "recall": 0.5555555555555556,
+      "tool": "shipwright",
+      "repo_name": "shipwright__https://github.com/calcom/cal.com/pull/11059",
+      "pr_url": "https://github.com/calcom/cal.com/pull/11059"
     }
   },
   "https://github.com/calcom/cal.com/pull/7232": {
@@ -100647,81 +102704,129 @@
       "errors_count": 0,
       "precision": 0.3333333333333333,
       "recall": 0.3333333333333333,
-      "tool": "macroscope",
-      "repo_name": "cal_dot_com__cal.com__macroscope__PR7232__20260309",
-      "pr_url": "https://github.com/code-review-benchmark/cal_dot_com__cal.com__macroscope__PR7232__20260309/pull/1"
+      "tool": "macroscope",
+      "repo_name": "cal_dot_com__cal.com__macroscope__PR7232__20260309",
+      "pr_url": "https://github.com/code-review-benchmark/cal_dot_com__cal.com__macroscope__PR7232__20260309/pull/1"
+    },
+    "qodo-v2": {
+      "skipped": false,
+      "true_positives": [
+        {
+          "golden_comment": "Asynchronous functions deleteScheduledEmailReminder and deleteScheduledSMSReminder are called without await inside forEach loops. This occurs during booking rescheduling/cancellation, and workflow/workflow step deletion/updates. Consequently, scheduled workflow reminders may not be reliably cancelled, potentially leaving them active.",
+          "severity": "Medium",
+          "category": "concurrency",
+          "matched_candidate": "Several handlers call async deleteScheduledEmailReminder/deleteScheduledSMSReminder inside forEach without awaiting, so cancellations can be dropped when the request finishes (especially in serverless) and promise rejections won't be caught, resulting in reminders still being sent for cancelled/rescheduled bookings",
+          "confidence": 0.95,
+          "reasoning": "The candidate identifies the exact same issue: async functions deleteScheduledEmailReminder/deleteScheduledSMSReminder are called inside forEach without await, leading to unreliable cancellation of scheduled reminders. Both comments highlight the same root cause (missing await in forEach loops) and the same consequence (reminders may still be sent for cancelled/rescheduled bookings)."
+        },
+        {
+          "golden_comment": "When immediateDelete is true, the deleteScheduledEmailReminder function cancels the SendGrid email but fails to delete the corresponding WorkflowReminder record from the database. This creates orphaned database entries and is inconsistent with the immediateDelete: false path, which marks the record as cancelled. The SendGrid DELETE API call is also omitted in this path.",
+          "severity": "High",
+          "category": "data",
+          "matched_candidate": "Reschedule flow cancels previous booking email reminders with immediateDelete=true, which bypasses the new deferred-cancellation (cancelled flag + cron) approach and is inconsistent with other reschedule/cancel paths, potentially leaving reminders active or reintroducing the cancellation-pressure pattern the PR aims to avoid",
+          "confidence": 0.75,
+          "reasoning": "Both comments identify the same core issue: when immediateDelete=true is used, the code path is inconsistent with the deferred cancellation approach. The golden comment specifically notes the database record isn't deleted/marked as cancelled, while the candidate notes it bypasses the deferred-cancellation approach. They're pointing to the same inconsistency problem, though the golden comment is more specific about the technical details (orphaned records, missing SendGrid DELETE call)."
+        }
+      ],
+      "false_positives": [
+        {
+          "candidate": "The new cron cancellation query doesn't restrict to EMAIL reminders or non-null referenceId and then posts batch_id from reminder.referenceId which can be null, causing failed cancellations or attempts to cancel non-email reminders, with errors only being console.logged"
+        }
+      ],
+      "false_negatives": [
+        {
+          "golden_comment": "The 'cancelled' field is nullable, which can lead to existing records having null values instead of a default of false. This can cause incorrect behavior in the application when checking the cancellation status of reminders.",
+          "severity": "Medium",
+          "category": "data"
+        }
+      ],
+      "errors": [],
+      "total_candidates": 3,
+      "total_golden": 3,
+      "tp": 2,
+      "fp": 1,
+      "fn": 1,
+      "errors_count": 0,
+      "precision": 0.6666666666666666,
+      "recall": 0.6666666666666666,
+      "tool": "qodo-v2",
+      "repo_name": "cal_dot_com__cal.com__qodo-v2__PR7232__20260227",
+      "pr_url": "https://github.com/code-review-benchmark/cal_dot_com__cal.com__qodo-v2__PR7232__20260227/pull/1"
+    },
+    "sourcery": {
+      "skipped": false,
+      "true_positives": [
+        {
+          "golden_comment": "When immediateDelete is true, the deleteScheduledEmailReminder function cancels the SendGrid email but fails to delete the corresponding WorkflowReminder record from the database. This creates orphaned database entries and is inconsistent with the immediateDelete: false path, which marks the record as cancelled. The SendGrid DELETE API call is also omitted in this path.",
+          "severity": "High",
+          "category": "data",
+          "matched_candidate": "In `deleteScheduledEmailReminder`, the `immediateDelete` branch posts the SendGrid cancel but never deletes or flags the corresponding `workflowReminder` row, which can leave orphan reminders in the DB for cases like deleted workflows and rescheduled bookings",
+          "confidence": 0.95,
+          "reasoning": "The candidate identifies the same core issue: when immediateDelete is true, the deleteScheduledEmailReminder function cancels the SendGrid email but fails to delete or update the workflowReminder database record, leaving orphaned entries. Both comments point to the same bug of inconsistent database handling in the immediateDelete branch."
+        }
+      ],
+      "false_positives": [
+        {
+          "candidate": "The cancellation loop in `scheduleEmailReminders` (over `remindersToCancel`) awaits each SendGrid request sequentially inside a `for` loop; if many reminders are due this could be slow, consider using `Promise.all` with per-reminder try/catch or small batches to parallelize these calls"
+        },
+        {
+          "candidate": "The new `deleteScheduledSMSReminder` always deletes the `workflowReminder` row (even when `referenceId` is null), while email reminders are now soft-cancelled via a `cancelled` flag and cleaned up by the cron; inconsistent behavior between SMS and email paths may cause issues for later introspection"
+        }
+      ],
+      "false_negatives": [
+        {
+          "golden_comment": "Asynchronous functions deleteScheduledEmailReminder and deleteScheduledSMSReminder are called without await inside forEach loops. This occurs during booking rescheduling/cancellation, and workflow/workflow step deletion/updates. Consequently, scheduled workflow reminders may not be reliably cancelled, potentially leaving them active.",
+          "severity": "Medium",
+          "category": "concurrency"
+        },
+        {
+          "golden_comment": "The 'cancelled' field is nullable, which can lead to existing records having null values instead of a default of false. This can cause incorrect behavior in the application when checking the cancellation status of reminders.",
+          "severity": "Medium",
+          "category": "data"
+        }
+      ],
+      "errors": [],
+      "total_candidates": 3,
+      "total_golden": 3,
+      "tp": 1,
+      "fp": 2,
+      "fn": 2,
+      "errors_count": 0,
+      "precision": 0.3333333333333333,
+      "recall": 0.3333333333333333,
+      "tool": "sourcery",
+      "repo_name": "cal_dot_com__cal.com__sourcery__PR7232__20260307",
+      "pr_url": "https://github.com/code-review-benchmark/cal_dot_com__cal.com__sourcery__PR7232__20260307/pull/1"
     },
-    "qodo-v2": {
+    "shipwright": {
       "skipped": false,
       "true_positives": [
         {
           "golden_comment": "Asynchronous functions deleteScheduledEmailReminder and deleteScheduledSMSReminder are called without await inside forEach loops. This occurs during booking rescheduling/cancellation, and workflow/workflow step deletion/updates. Consequently, scheduled workflow reminders may not be reliably cancelled, potentially leaving them active.",
           "severity": "Medium",
           "category": "concurrency",
-          "matched_candidate": "Several handlers call async deleteScheduledEmailReminder/deleteScheduledSMSReminder inside forEach without awaiting, so cancellations can be dropped when the request finishes (especially in serverless) and promise rejections won't be caught, resulting in reminders still being sent for cancelled/rescheduled bookings",
+          "matched_candidate": "The reschedule mutation dropped the `await Promise.all(remindersToDelete)` call that previously awaited the actual deletion of old WorkflowReminder rows. The new code only fires `deleteScheduledEmailReminder`/`deleteScheduledSMSReminder` inside a `forEach` without awaiting them, so the mutation proceeds to create the new booking and schedule new workflow reminders before the old reminders are guaranteed to be cleaned up/canceled, introducing a race condition and swallowing any errors from the cleanup. Location: packages/trpc/server/routers/viewer/bookings.tsx:497",
           "confidence": 0.95,
-          "reasoning": "The candidate identifies the exact same issue: async functions deleteScheduledEmailReminder/deleteScheduledSMSReminder are called inside forEach without await, leading to unreliable cancellation of scheduled reminders. Both comments highlight the same root cause (missing await in forEach loops) and the same consequence (reminders may still be sent for cancelled/rescheduled bookings)."
+          "reasoning": "The candidate issue identifies the exact same problem as the golden comment: async functions deleteScheduledEmailReminder and deleteScheduledSMSReminder are called inside forEach loops without await, causing scheduled workflow reminders to not be reliably cancelled. The candidate specifically mentions the reschedule mutation context and the race condition this creates, which aligns with the golden comment's mention of 'booking rescheduling/cancellation'. Both identify the core issue of unawaited async deletion calls in forEach loops."
         },
         {
           "golden_comment": "When immediateDelete is true, the deleteScheduledEmailReminder function cancels the SendGrid email but fails to delete the corresponding WorkflowReminder record from the database. This creates orphaned database entries and is inconsistent with the immediateDelete: false path, which marks the record as cancelled. The SendGrid DELETE API call is also omitted in this path.",
           "severity": "High",
           "category": "data",
-          "matched_candidate": "Reschedule flow cancels previous booking email reminders with immediateDelete=true, which bypasses the new deferred-cancellation (cancelled flag + cron) approach and is inconsistent with other reschedule/cancel paths, potentially leaving reminders active or reintroducing the cancellation-pressure pattern the PR aims to avoid",
-          "confidence": 0.75,
-          "reasoning": "Both comments identify the same core issue: when immediateDelete=true is used, the code path is inconsistent with the deferred cancellation approach. The golden comment specifically notes the database record isn't deleted/marked as cancelled, while the candidate notes it bypasses the deferred-cancellation approach. They're pointing to the same inconsistency problem, though the golden comment is more specific about the technical details (orphaned records, missing SendGrid DELETE call)."
-        }
-      ],
-      "false_positives": [
-        {
-          "candidate": "The new cron cancellation query doesn't restrict to EMAIL reminders or non-null referenceId and then posts batch_id from reminder.referenceId which can be null, causing failed cancellations or attempts to cancel non-email reminders, with errors only being console.logged"
-        }
-      ],
-      "false_negatives": [
-        {
-          "golden_comment": "The 'cancelled' field is nullable, which can lead to existing records having null values instead of a default of false. This can cause incorrect behavior in the application when checking the cancellation status of reminders.",
-          "severity": "Medium",
-          "category": "data"
-        }
-      ],
-      "errors": [],
-      "total_candidates": 3,
-      "total_golden": 3,
-      "tp": 2,
-      "fp": 1,
-      "fn": 1,
-      "errors_count": 0,
-      "precision": 0.6666666666666666,
-      "recall": 0.6666666666666666,
-      "tool": "qodo-v2",
-      "repo_name": "cal_dot_com__cal.com__qodo-v2__PR7232__20260227",
-      "pr_url": "https://github.com/code-review-benchmark/cal_dot_com__cal.com__qodo-v2__PR7232__20260227/pull/1"
-    },
-    "sourcery": {
-      "skipped": false,
-      "true_positives": [
-        {
-          "golden_comment": "When immediateDelete is true, the deleteScheduledEmailReminder function cancels the SendGrid email but fails to delete the corresponding WorkflowReminder record from the database. This creates orphaned database entries and is inconsistent with the immediateDelete: false path, which marks the record as cancelled. The SendGrid DELETE API call is also omitted in this path.",
-          "severity": "High",
-          "category": "data",
-          "matched_candidate": "In `deleteScheduledEmailReminder`, the `immediateDelete` branch posts the SendGrid cancel but never deletes or flags the corresponding `workflowReminder` row, which can leave orphan reminders in the DB for cases like deleted workflows and rescheduled bookings",
-          "confidence": 0.95,
-          "reasoning": "The candidate identifies the same core issue: when immediateDelete is true, the deleteScheduledEmailReminder function cancels the SendGrid email but fails to delete or update the workflowReminder database record, leaving orphaned entries. Both comments point to the same bug of inconsistent database handling in the immediateDelete branch."
+          "matched_candidate": "When `immediateDelete` is true and a `referenceId` exists, `deleteScheduledEmailReminder` only calls the SendGrid cancel API and returns \u2014 it never deletes or updates (sets `cancelled: true`) the `WorkflowReminder` row. This function is called with `immediateDelete=true` from `handleNewBooking.ts` (reschedule cleanup) and `workflows.tsx` (step/workflow deletion), so those WorkflowReminder rows become orphaned in the DB with `scheduled: true`/`cancelled: null` forever, even though the underlying email was actually canceled, which can cause `scheduleEmailReminders.ts`'s cancellation cron (which filters on `cancelled: true`) to never clean them up and can confuse any code that treats these rows as still-active/scheduled reminders. Location: packages/features/ee/workflows/lib/reminders/emailReminderManager.ts:213",
+          "confidence": 0.98,
+          "reasoning": "The candidate issue identifies the exact same problem as the golden comment: when immediateDelete is true and a referenceId exists, the deleteScheduledEmailReminder function only calls the SendGrid cancel API but fails to delete or update the WorkflowReminder database record. Both comments point out that this creates orphaned database entries that remain with scheduled=true/cancelled=null, which is inconsistent with the immediateDelete=false path. The candidate provides additional context about where this is called from and the downstream effects, but it's fundamentally the same bug."
         }
       ],
       "false_positives": [
         {
-          "candidate": "The cancellation loop in `scheduleEmailReminders` (over `remindersToCancel`) awaits each SendGrid request sequentially inside a `for` loop; if many reminders are due this could be slow, consider using `Promise.all` with per-reminder try/catch or small batches to parallelize these calls"
+          "candidate": "The previous `deleteMany` for reminders belonging to disabled event types explicitly scoped deletion to `booking: { userId: ctx.user.id }}`, providing an ownership check. The refactor removes this scoping entirely and instead calls `deleteScheduledEmailReminder`/`deleteScheduledSMSReminder`, whose underlying Prisma calls only filter by `id: reminderId` with no ownership/ownership-booking check, weakening the authorization guard that was present before this diff. Location: packages/trpc/server/routers/viewer/workflows.tsx:378"
         },
         {
-          "candidate": "The new `deleteScheduledSMSReminder` always deletes the `workflowReminder` row (even when `referenceId` is null), while email reminders are now soft-cancelled via a `cancelled` flag and cleaned up by the cron; inconsistent behavior between SMS and email paths may cause issues for later introspection"
+          "candidate": "The guard `if (reminder.scheduled && reminder.referenceId)` and the batched `prisma.workflowReminder.deleteMany` (previously part of the awaited `prismaPromises`) were removed. Reminder cleanup now only happens via unawaited, fire-and-forget calls to `deleteScheduledEmailReminder`/`deleteScheduledSMSReminder` inside the `forEach`, so the cancel-booking handler's response can return to the client before the WorkflowReminder rows are actually canceled/deleted, and any errors from that cleanup are silently swallowed instead of surfacing via `Promise.all`. Location: packages/features/bookings/lib/handleCancelBooking.ts:484"
         }
       ],
       "false_negatives": [
-        {
-          "golden_comment": "Asynchronous functions deleteScheduledEmailReminder and deleteScheduledSMSReminder are called without await inside forEach loops. This occurs during booking rescheduling/cancellation, and workflow/workflow step deletion/updates. Consequently, scheduled workflow reminders may not be reliably cancelled, potentially leaving them active.",
-          "severity": "Medium",
-          "category": "concurrency"
-        },
         {
           "golden_comment": "The 'cancelled' field is nullable, which can lead to existing records having null values instead of a default of false. This can cause incorrect behavior in the application when checking the cancellation status of reminders.",
           "severity": "Medium",
@@ -100729,17 +102834,17 @@
         }
       ],
       "errors": [],
-      "total_candidates": 3,
+      "total_candidates": 4,
       "total_golden": 3,
-      "tp": 1,
+      "tp": 2,
       "fp": 2,
-      "fn": 2,
+      "fn": 1,
       "errors_count": 0,
-      "precision": 0.3333333333333333,
-      "recall": 0.3333333333333333,
-      "tool": "sourcery",
-      "repo_name": "cal_dot_com__cal.com__sourcery__PR7232__20260307",
-      "pr_url": "https://github.com/code-review-benchmark/cal_dot_com__cal.com__sourcery__PR7232__20260307/pull/1"
+      "precision": 0.5,
+      "recall": 0.6666666666666666,
+      "tool": "shipwright",
+      "repo_name": "shipwright__https://github.com/calcom/cal.com/pull/7232",
+      "pr_url": "https://github.com/calcom/cal.com/pull/7232"
     }
   },
   "https://github.com/calcom/cal.com/pull/14740": {
@@ -103776,6 +105881,72 @@
       "tool": "sourcery",
       "repo_name": "cal_dot_com__cal.com__sourcery__PR14740__20260307",
       "pr_url": "https://github.com/code-review-benchmark/cal_dot_com__cal.com__sourcery__PR14740__20260307/pull/1"
+    },
+    "shipwright": {
+      "skipped": false,
+      "true_positives": [
+        {
+          "golden_comment": "The addGuests.handler.ts blacklist check uses blacklistedGuestEmails.includes(guest) after lowercasing the blacklist but not the input guest emails, allowing attackers to bypass BLACKLISTED_GUEST_EMAILS restrictions by submitting emails with uppercase characters.",
+          "severity": "High",
+          "category": "security",
+          "matched_candidate": "Blacklisted guest emails are lowercased when read from env, but the incoming `guest` email is compared against the blacklist without normalization (`!blacklistedGuestEmails.includes(guest)`). A blacklisted email submitted with different casing (e.g. Foo@Bar.com) will bypass the blacklist filter entirely. Location: packages/trpc/server/routers/viewer/bookings/addGuests.handler.ts:58",
+          "confidence": 1.0,
+          "reasoning": "The candidate issue identifies exactly the same problem as the golden comment: the blacklist check in addGuests.handler.ts lowercases the blacklisted emails but does not lowercase/normalize the incoming guest emails before comparison, allowing bypass of the blacklist by using uppercase characters. The candidate even provides the same file location and explains the same attack vector."
+        },
+        {
+          "golden_comment": "The logic for checking team admin/owner permissions is incorrect. This condition uses AND (&&) which requires both isTeamAdmin AND isTeamOwner to be true, but it should use OR (||) since a user needs to be either an admin OR an owner to have permission.",
+          "severity": "Critical",
+          "category": "bug",
+          "matched_candidate": "Authorization check uses `&&` instead of `||` between isTeamAdmin and isTeamOwner, requiring a user to be BOTH a team admin AND the team owner to be granted access via this path, rather than either role. This incorrectly denies legitimate team admins (who are not also the owner) permission to add guests. Location: packages/trpc/server/routers/viewer/bookings/addGuests.handler.ts:33",
+          "confidence": 1.0,
+          "reasoning": "The candidate issue identifies exactly the same problem as the golden comment: the authorization check incorrectly uses `&&` (AND) instead of `||` (OR) between isTeamAdmin and isTeamOwner. Both comments explain that this requires a user to be BOTH admin AND owner instead of EITHER, which incorrectly denies access to legitimate team admins. The candidate even provides the specific file location."
+        }
+      ],
+      "false_positives": [
+        {
+          "candidate": "`this.calEvent.attendees[0].name` is accessed without verifying the attendees array is non-empty, which will throw if a booking somehow has no attendees when this template is rendered. Location: packages/emails/templates/organizer-add-guests-email.ts:27"
+        },
+        {
+          "candidate": "Errors from sendAddGuestsEmails are silently swallowed with a bare console.log instead of using the application logger, hiding email delivery failures from monitoring/observability after guests are added. Location: packages/trpc/server/routers/viewer/bookings/addGuests.handler.ts:168"
+        },
+        {
+          "candidate": "The onError handler builds `message` as a template string (\"${err.data?.code}: ${t(err.message)}\") which is always truthy, so the fallback `t('unable_to_add_guests')` in `showToast(message || t('unable_to_add_guests'), 'error')` is unreachable dead code and never displays a translated fallback message. Location: apps/web/components/dialog/AddGuestsDialog.tsx:29"
+        }
+      ],
+      "false_negatives": [
+        {
+          "golden_comment": "The sendAddGuestsEmails function in email-manager.ts passes the full calendarEvent.attendees array when determining which attendees receive AttendeeScheduledEmail vs AttendeeAddGuestsEmail, but checks if newGuests.includes(attendee.email) where newGuests is the input guests array.",
+          "severity": "Medium",
+          "category": "bug"
+        },
+        {
+          "golden_comment": "uniqueGuests filters out existing attendees and blacklisted emails but does not deduplicate duplicates within the input; createMany can insert duplicate attendee rows if the client submits repeated emails.",
+          "severity": "Medium",
+          "category": "bug"
+        },
+        {
+          "golden_comment": "Starting with an array containing an empty string may cause validation issues. Consider starting with an empty array [] and handling the empty state in the MultiEmail component instead.",
+          "severity": "Low",
+          "category": "bug"
+        },
+        {
+          "golden_comment": "The `sendAddGuestsEmails` function does not check the `disableStandardEmails` flags in the `EventType` metadata, which may result in sending emails to hosts/attendees even when standard emails are disabled for the event type.",
+          "severity": "Medium",
+          "category": "bug"
+        }
+      ],
+      "errors": [],
+      "total_candidates": 5,
+      "total_golden": 6,
+      "tp": 2,
+      "fp": 3,
+      "fn": 4,
+      "errors_count": 0,
+      "precision": 0.4,
+      "recall": 0.3333333333333333,
+      "tool": "shipwright",
+      "repo_name": "shipwright__https://github.com/calcom/cal.com/pull/14740",
+      "pr_url": "https://github.com/calcom/cal.com/pull/14740"
     }
   },
   "https://github.com/calcom/cal.com/pull/10600": {
@@ -106494,6 +108665,60 @@
       "tool": "sourcery",
       "repo_name": "cal_dot_com__cal.com__sourcery__PR10600__20260307",
       "pr_url": "https://github.com/code-review-benchmark/cal_dot_com__cal.com__sourcery__PR10600__20260307/pull/1"
+    },
+    "shipwright": {
+      "skipped": false,
+      "true_positives": [
+        {
+          "golden_comment": "The exported function TwoFactor handles backup codes and is in BackupCode.tsx. Inconsistent naming.",
+          "severity": "Low",
+          "category": "style",
+          "matched_candidate": "BackupCode.tsx exports a function named `TwoFactor` (identical to the actual TwoFactor.tsx component) instead of `BackupCode`. This is misleading and could easily be confused with the real TwoFactor component when imported elsewhere, especially since both default-export the same identifier name. Location: apps/web/components/auth/BackupCode.tsx:6",
+          "confidence": 0.95,
+          "reasoning": "The candidate issue identifies the same problem as the golden comment - that BackupCode.tsx exports a function named 'TwoFactor' instead of 'BackupCode', which is inconsistent/misleading naming. Both comments point to the same naming inconsistency issue in the same file."
+        },
+        {
+          "golden_comment": "Because backupCodes are decrypted and mutated in memory before being written back, two concurrent login requests using the same backupCode could both pass this check and update, so a single backup code may effectively be accepted more than once if used concurrently, weakening the intended one-time-use semantics.",
+          "severity": "High",
+          "category": "concurrency",
+          "matched_candidate": "Backup code verification in the credentials authorize() flow reads the user's encrypted backupCodes, validates the supplied code against an in-memory array, mutates that array (`backupCodes[index] = null`) and only then persists the change via `prisma.user.update`. Two concurrent login attempts using the same backup code will both read the original (unmodified) encrypted array before either write completes, so both requests will find the code valid and succeed, defeating the intended one-time-use semantics of backup codes and allowing bypass/detection avoidance. Location: packages/features/auth/lib/next-auth-options.ts:151",
+          "confidence": 1.0,
+          "reasoning": "The candidate issue identifies exactly the same race condition vulnerability as the golden comment. Both describe: 1) backup codes being decrypted and validated in memory, 2) the array being mutated locally before persistence, 3) concurrent login requests both reading the original unmodified array, 4) both requests succeeding with the same backup code, 5) this defeating the one-time-use semantics. The candidate even provides the specific file location and line number. This is a perfect semantic match."
+        },
+        {
+          "golden_comment": "Object URL for backup codes is not revoked on component unmount, leading to a potential memory leak. This can cause increased memory usage over time, especially if the component is mounted and unmounted frequently.",
+          "severity": "Low",
+          "category": "perf",
+          "matched_candidate": "A Blob URL is created via `URL.createObjectURL(textBlob)` for the backup-codes download link and stored in `backupCodesUrl`. It is only revoked the next time `handleSetup` runs (`if (backupCodesUrl) URL.revokeObjectURL(backupCodesUrl)`), but `resetState()` \u2014 called both when the user cancels the flow and when the user clicks \"Close\" after viewing/downloading the backup codes \u2014 never revokes this URL. As a result, every completed 2FA setup leaks one Blob URL for the lifetime of the page. Location: apps/web/components/settings/EnableTwoFactorModal.tsx:100",
+          "confidence": 0.9,
+          "reasoning": "Both comments identify the same underlying issue: the Blob URL created via URL.createObjectURL is not being properly revoked/cleaned up. The golden comment specifically mentions 'component unmount' as the cleanup point, while the candidate mentions that resetState() doesn't revoke the URL and it's only revoked on the next handleSetup call. Both are pointing to the same memory leak problem - the Blob URL persists longer than it should and causes memory leaks. The candidate provides more specific details about when the leak occurs (during resetState/cancel/close flows) but it's fundamentally the same issue about improper cleanup of the Object URL."
+        }
+      ],
+      "false_positives": [],
+      "false_negatives": [
+        {
+          "golden_comment": "Error message mentions 'backup code login' but this is a disable endpoint, not login",
+          "severity": "Low",
+          "category": "doc_defect"
+        },
+        {
+          "golden_comment": "Backup code validation is case-sensitive due to the use of indexOf(). This causes validation to fail if a user enters uppercase hex characters, as backup codes should be case-insensitive for a better user experience.",
+          "severity": "Medium",
+          "category": "bug"
+        }
+      ],
+      "errors": [],
+      "total_candidates": 3,
+      "total_golden": 5,
+      "tp": 3,
+      "fp": 0,
+      "fn": 2,
+      "errors_count": 0,
+      "precision": 1.0,
+      "recall": 0.6,
+      "tool": "shipwright",
+      "repo_name": "shipwright__https://github.com/calcom/cal.com/pull/10600",
+      "pr_url": "https://github.com/calcom/cal.com/pull/10600"
     }
   },
   "https://github.com/calcom/cal.com/pull/10967": {
@@ -109667,6 +111892,69 @@
       "tool": "sourcery",
       "repo_name": "cal_dot_com__cal.com__sourcery__PR10967__20260307",
       "pr_url": "https://github.com/code-review-benchmark/cal_dot_com__cal.com__sourcery__PR10967__20260307/pull/1"
+    },
+    "shipwright": {
+      "skipped": false,
+      "true_positives": [
+        {
+          "golden_comment": "A null reference occurs in GoogleCalendarService.createEvent() and updateEvent() when mainHostDestinationCalendar is destructured from evt.destinationCalendar but evt.destinationCalendar is null or an empty array, causing mainHostDestinationCalendar to be undefined.",
+          "severity": "High",
+          "category": "bug",
+          "matched_candidate": "After destructuring `const [mainHostDestinationCalendar] = evt.destinationCalendar ?? [];`, the code calls `mainHostDestinationCalendar.integration` without any null-check. Since many callers now pass `destinationCalendar: []` (e.g. handleCancelBooking.ts, webhook.ts, paypal-webhook.ts) when there is no destination calendar, `mainHostDestinationCalendar` will be `undefined` and this line throws `TypeError: Cannot read properties of undefined (reading 'integration')` whenever `evt.location === MeetLocationType` and no destination calendar exists. Location: packages/core/EventManager.ts:118",
+          "confidence": 0.85,
+          "reasoning": "Both comments identify the same underlying issue: destructuring `mainHostDestinationCalendar` from `evt.destinationCalendar` when it's null or an empty array causes `mainHostDestinationCalendar` to be undefined, leading to a null reference error when trying to access properties on it. The golden comment mentions this occurs in GoogleCalendarService.createEvent() and updateEvent(), while the candidate mentions EventManager.ts:118. These could be different locations where the same pattern exists, but the core bug pattern is identical - the same destructuring issue with the same variable name and same root cause (empty array or null destinationCalendar). The candidate correctly identifies the exact same bug pattern."
+        },
+        {
+          "golden_comment": "Logic error: when externalCalendarId is provided, you're searching for a calendar where externalId === externalCalendarId, but this will always fail since you're looking for a calendar that matches itself. Should likely find by credentialId or use different logic.",
+          "severity": "High",
+          "category": "bug",
+          "matched_candidate": "In `updateEvent` the fallback branch computes `event.destinationCalendar?.find((cal) => cal.externalId === externalCalendarId)?.externalId` but this code only runs when `externalCalendarId` is falsy (it's the else-branch of `externalCalendarId ? externalCalendarId : ...`). Comparing `cal.externalId === externalCalendarId` against a falsy value will essentially never match a real calendar entry, so `selectedCalendar` will always resolve to `undefined` instead of falling back to the destination calendar's externalId, breaking calendar update routing. Location: packages/app-store/googlecalendar/lib/CalendarService.ts:255",
+          "confidence": 0.85,
+          "reasoning": "Both comments identify the same fundamental issue with the calendar lookup logic. The golden comment points out that searching for a calendar where externalId === externalCalendarId will always fail because you're looking for a calendar that matches itself. The candidate comment identifies the same problematic code pattern - the find operation with `cal.externalId === externalCalendarId` comparison that will never match properly, resulting in `selectedCalendar` always being undefined. Both are pointing to the same broken calendar selection logic in the updateEvent function, just explaining the bug from slightly different angles."
+        },
+        {
+          "golden_comment": "The Calendar interface now requires createEvent(event, credentialId), but some implementations (e.g., Lark/Office365) still declare createEvent(event) only\u2014this breaks the interface contract (also applies to other locations in the PR).",
+          "severity": "Low",
+          "category": "api",
+          "matched_candidate": "The `Calendar` interface's `createEvent` signature was changed to require a second `credentialId: number` parameter, but only `GoogleCalendarService.createEvent` was updated to accept/use it. `LarkCalendarService.createEvent`, `Office365CalendarService.createEvent`, and the abstract `BaseCalendarService` in packages/lib/CalendarService.ts were not updated, so those providers cannot use the new per-credential destinationCalendar selection this PR introduces, leaving the multi-host destinationCalendar feature only partially implemented across calendar providers. Location: packages/types/Calendar.d.ts:221",
+          "confidence": 0.98,
+          "reasoning": "The candidate issue identifies the exact same problem as the golden comment: the Calendar interface's createEvent method was changed to require a credentialId parameter, but implementations like LarkCalendarService and Office365CalendarService were not updated to match this new signature, breaking the interface contract. The candidate even mentions the same specific implementations (Lark, Office365) and points to the same location (Calendar.d.ts). The candidate provides additional context about the feature implications but fundamentally identifies the same interface contract violation."
+        }
+      ],
+      "false_positives": [
+        {
+          "candidate": "`evt.destinationCalendar` is typed to be `null` when neither `eventType.destinationCalendar` nor `organizerUser.destinationCalendar` exist. The subsequent `evt.destinationCalendar?.push(...teamDestinationCalendars)` uses optional chaining, so when `evt.destinationCalendar` is `null` the push silently no-ops and all collected team member destination calendars are dropped, defeating the PR's stated purpose of supporting multiple hosts' destination calendars for collective events. Location: packages/features/bookings/lib/handleNewBooking.ts:1083"
+        }
+      ],
+      "false_negatives": [
+        {
+          "golden_comment": "The optional chaining on mainHostDestinationCalendar?.integration is redundant since you already check mainHostDestinationCalendar in the ternary condition.",
+          "severity": "Low",
+          "category": "style"
+        },
+        {
+          "golden_comment": "Logic inversion in organization creation: The slug property is now conditionally set when IS_TEAM_BILLING_ENABLED is true, instead of when it's false as originally intended. This change, combined with requestedSlug still being set when IS_TEAM_BILLING_ENABLED is true, results in both properties being set when billing is enabled, and neither when disabled",
+          "severity": "Medium",
+          "category": "bug"
+        },
+        {
+          "golden_comment": "The `credential` may be undefined when passed to `updateEvent` if `reference.credentialId` is set but the credential is not found. This can lead to a runtime error when `updateEvent` is called with an undefined credential, which should be handled to prevent crashes.",
+          "severity": "High",
+          "category": "bug"
+        }
+      ],
+      "errors": [],
+      "total_candidates": 5,
+      "total_golden": 6,
+      "tp": 3,
+      "fp": 1,
+      "fn": 3,
+      "errors_count": 0,
+      "precision": 0.6,
+      "recall": 0.5,
+      "tool": "shipwright",
+      "repo_name": "shipwright__https://github.com/calcom/cal.com/pull/10967",
+      "pr_url": "https://github.com/calcom/cal.com/pull/10967"
     }
   },
   "https://github.com/calcom/cal.com/pull/8087": {
@@ -111642,6 +113930,46 @@
       "tool": "sourcery",
       "repo_name": "cal_dot_com__cal.com__sourcery__PR8087__20260307",
       "pr_url": "https://github.com/code-review-benchmark/cal_dot_com__cal.com__sourcery__PR8087__20260307/pull/1"
+    },
+    "shipwright": {
+      "skipped": false,
+      "true_positives": [
+        {
+          "golden_comment": "The code uses forEach with async callbacks, which causes asynchronous operations (e.g., calendar/video event deletions, payment refunds) to run concurrently without being awaited. This 'fire-and-forget' behavior leads to unhandled promise rejections, race conditions, and incomplete cleanup, as surrounding try-catch blocks cannot properly handle errors from these unawaited promises. Replace forEach with for...of loops or Promise.all() with map() to ensure proper sequential execution and error handling.",
+          "severity": "Critical",
+          "category": "concurrency",
+          "matched_candidate": "The bookingRefsFiltered.forEach() callback was converted to `async (bookingRef) => {...}` to allow `await getCalendar(...)`, but Array.prototype.forEach does not await its callback. The enclosing try/catch block therefore no longer catches errors thrown inside the async callback (they become unhandled promise rejections), and the surrounding function returns before the calendar deleteEvent/deleteMeeting operations actually complete, creating a race condition when rescheduling. Location: packages/app-store/vital/lib/reschedule.ts:125",
+          "confidence": 0.95,
+          "reasoning": "The candidate issue identifies the exact same underlying problem as the golden comment: using forEach with async callbacks causes the async operations to not be properly awaited. The candidate specifically mentions: (1) forEach doesn't await its callback, (2) try/catch won't catch errors (unhandled promise rejections), (3) race conditions occur because the function returns before operations complete. This matches all the key points in the golden comment about fire-and-forget behavior, unhandled promise rejections, race conditions, and incomplete cleanup. The candidate even points to a specific location in the codebase where this occurs."
+        }
+      ],
+      "false_positives": [
+        {
+          "candidate": "Same fire-and-forget bug as in vital/reschedule.ts: forEach was made async to support `await getCalendar(...)`, but forEach ignores the returned promises, so the try/catch around it no longer catches rejected promises and the function proceeds without waiting for the calendar/meeting deletions to finish. Location: packages/app-store/wipemycalother/lib/reschedule.ts:125"
+        },
+        {
+          "candidate": "bookingRefsFiltered.forEach() was changed to take an async callback to await getCalendar(...), but forEach does not await async callbacks. This makes calendar/meeting deletion during reschedule fire-and-forget: the handler can proceed (and respond) before old bookings' calendar events are actually deleted, and any thrown errors become unhandled promise rejections instead of being surfaced/caught. Location: packages/trpc/server/routers/viewer/bookings.tsx:553"
+        }
+      ],
+      "false_negatives": [
+        {
+          "golden_comment": "Consider adding try-catch around the await to handle import failures gracefully",
+          "severity": "Low",
+          "category": "speculative"
+        }
+      ],
+      "errors": [],
+      "total_candidates": 3,
+      "total_golden": 2,
+      "tp": 1,
+      "fp": 2,
+      "fn": 1,
+      "errors_count": 0,
+      "precision": 0.3333333333333333,
+      "recall": 0.5,
+      "tool": "shipwright",
+      "repo_name": "shipwright__https://github.com/calcom/cal.com/pull/8087",
+      "pr_url": "https://github.com/calcom/cal.com/pull/8087"
     }
   }
 }
\ No newline at end of file
diff --git a/offline/results/benchmark_data.json b/offline/results/benchmark_data.json
index f6a1028..1ea5ba1 100644
--- a/offline/results/benchmark_data.json
+++ b/offline/results/benchmark_data.json
@@ -1789,6 +1789,43 @@
             "created_at": "2026-06-28T23:12:51Z"
           }
         ]
+      },
+      {
+        "pr_url": "https://github.com/keycloak/keycloak/pull/37429",
+        "repo_name": "shipwright__https://github.com/keycloak/keycloak/pull/37429",
+        "review_comments": [
+          {
+            "body": "totpStep1 value was replaced with Italian text (\"Installa una delle seguenti applicazioni sul tuo cellulare:\") in the Lithuanian (lt) properties file, so Lithuanian users will see Italian text instead of a Lithuanian translation. Location: themes/src/main/resources-community/theme/base/account/messages/messages_lt.properties:101",
+            "created_at": null,
+            "line": 101,
+            "path": "themes/src/main/resources-community/theme/base/account/messages/messages_lt.properties"
+          },
+          {
+            "body": "loginTotpStep1 value was replaced with Italian text (\"Installa una delle seguenti applicazioni sul tuo cellulare:\") in the Lithuanian (lt) login properties file, producing incorrect/foreign-language UI text for Lithuanian users. Location: themes/src/main/resources-community/theme/base/login/messages/messages_lt.properties:71",
+            "created_at": null,
+            "line": 71,
+            "path": "themes/src/main/resources-community/theme/base/login/messages/messages_lt.properties"
+          },
+          {
+            "body": "totpStep1 was rewritten using Traditional Chinese characters (手機, 應用程式) in the Simplified Chinese (zh_CN) properties file, which should use Simplified Chinese forms (手机, 应用程序), causing an inconsistent/incorrect translation for zh_CN users. Location: themes/src/main/resources-community/theme/base/account/messages/messages_zh_CN.properties:112",
+            "created_at": null,
+            "line": 112,
+            "path": "themes/src/main/resources-community/theme/base/account/messages/messages_zh_CN.properties"
+          },
+          {
+            "body": "ANCHOR_PATTERN regex `</?a[^>]*>` is not properly boundary-anchored to the tag name 'a'; it will also match any tag whose name starts with 'a' (e.g. <abbr>, <article>, <aside>), causing santizeAnchors to misidentify unrelated tags as anchors and potentially emit false \"Didn't find anchor tag\" verifier errors or incorrectly strip unrelated tags. Location: misc/theme-verifier/src/main/java/org/keycloak/themeverifier/VerifyMessageProperties.java:140",
+            "created_at": null,
+            "line": 140,
+            "path": "misc/theme-verifier/src/main/java/org/keycloak/themeverifier/VerifyMessageProperties.java"
+          },
+          {
+            "body": "Method name `santizeAnchors` is a typo for `sanitizeAnchors`, a misleading identifier introduced by this diff for the new anchor-sanitization logic. Location: misc/theme-verifier/src/main/java/org/keycloak/themeverifier/VerifyMessageProperties.java:144",
+            "created_at": null,
+            "line": 144,
+            "path": "misc/theme-verifier/src/main/java/org/keycloak/themeverifier/VerifyMessageProperties.java"
+          }
+        ],
+        "tool": "shipwright"
       }
     ]
   },
@@ -3247,6 +3284,31 @@
             "created_at": "2026-06-28T21:20:12Z"
           }
         ]
+      },
+      {
+        "pr_url": "https://github.com/keycloak/keycloak/pull/37634",
+        "repo_name": "shipwright__https://github.com/keycloak/keycloak/pull/37634",
+        "review_comments": [
+          {
+            "body": "In AccessTokenContext's constructor, the null-check meant for `rawTokenId` mistakenly re-checks `grantType` twice: `Objects.requireNonNull(grantType, \"Null grantType not allowed\"); Objects.requireNonNull(grantType, \"Null rawTokenId not allowed\");`. As a result, a null `rawTokenId` is never rejected (NPE deferred to later usage, e.g. when concatenated in encodeTokenId) and the error message for the missing rawTokenId check is wrong (it will never fire, and if it did it would report the wrong field). Location: services/src/main/java/org/keycloak/protocol/oidc/encode/AccessTokenContext.java:73",
+            "created_at": null,
+            "line": 73,
+            "path": "services/src/main/java/org/keycloak/protocol/oidc/encode/AccessTokenContext.java"
+          },
+          {
+            "body": "isAccessTokenId() has two bugs that make it not test what it claims: (1) it extracts the grant-type shortcut with `substring(3, 5)` but per the actual encoding format (sessionType[0-2] + tokenType[2-4] + grantType[4-6], as verified by DefaultTokenContextEncoderProviderTest, e.g. \"trltcc:1234\"), the grant shortcut is at index 4-6, not 3-5 — this reads one character from tokenType and one from grantType. (2) The matching condition is inverted: `if (items[0].substring(3, 5).equals(expectedGrantShortcut)) return false;` returns false (no match) when the substring equals the expected shortcut, and falls through to the UUID check (ignoring the grant check) when it does not equal — the exact opposite of the intended assertion. This will cause the updated event assertions (expectCodeToToken, expectRefresh, expectDeviceCodeToToken, expectAuthReqIdToToken) to fail for correctly-encoded tokens and to silently ignore the grant type check for incorrect ones. Location: testsuite/integration-arquillian/tests/base/src/test/java/org/keycloak/testsuite/AssertEvents.java:476",
+            "created_at": null,
+            "line": 476,
+            "path": "testsuite/integration-arquillian/tests/base/src/test/java/org/keycloak/testsuite/AssertEvents.java"
+          },
+          {
+            "body": "testIncorrectGrantType() only manually sets `grantsByShortcuts`/`grantsToShortcuts` in @Before and never calls `factory.postInit(...)`, leaving `sessionFactory` null. When decoding the unregistered shortcut \"ac\", `getGrantTypeByShortcut` falls into the refresh branch and calls `sessionFactory.getProviderFactoriesStream(...)` on a null `sessionFactory`, throwing a NullPointerException rather than the intended `IllegalArgumentException` from the \"Unknown value for grant type\" validation path. The test passes only because it catches the overly broad `RuntimeException`, so it does not actually exercise the described \"incorrect grant type\" validation logic. Location: services/src/test/java/org/keycloak/protocol/oidc/encode/DefaultTokenContextEncoderProviderTest.java:66",
+            "created_at": null,
+            "line": 66,
+            "path": "services/src/test/java/org/keycloak/protocol/oidc/encode/DefaultTokenContextEncoderProviderTest.java"
+          }
+        ],
+        "tool": "shipwright"
       }
     ]
   },
@@ -4705,6 +4767,19 @@
             "created_at": "2026-06-28T23:26:03Z"
           }
         ]
+      },
+      {
+        "pr_url": "https://github.com/keycloak/keycloak/pull/38446",
+        "repo_name": "shipwright__https://github.com/keycloak/keycloak/pull/38446",
+        "review_comments": [
+          {
+            "body": "The new RecoveryAuthnCodesCredentialModel.TYPE branch added to BackwardsCompatibilityUserStorage.isValid() checks whether the submitted code exists in the stored code list via generatedKeys.stream().anyMatch(...), but never removes/marks the matched code as used and never persists an updated credential. This breaks the fundamental one-time-use guarantee of recovery/backup codes: a user (or attacker who intercepted a code) can replay the same recovery code indefinitely to authenticate, since the stored code list is never mutated after a successful match. This file is the PR's reference example of how a custom UserStorageProvider should implement recovery-code support, so the missing invalidation is a concrete, security-relevant logic gap introduced by this diff. Location: testsuite/integration-arquillian/servers/auth-server/services/testsuite-providers/src/main/java/org/keycloak/testsuite/federation/BackwardsCompatibilityUserStorage.java:333",
+            "created_at": null,
+            "line": 333,
+            "path": "testsuite/integration-arquillian/servers/auth-server/services/testsuite-providers/src/main/java/org/keycloak/testsuite/federation/BackwardsCompatibilityUserStorage.java"
+          }
+        ],
+        "tool": "shipwright"
       }
     ]
   },
@@ -5863,6 +5938,25 @@
             "created_at": "2026-06-28T22:40:43Z"
           }
         ]
+      },
+      {
+        "pr_url": "https://github.com/keycloak/keycloak/pull/36882",
+        "repo_name": "shipwright__https://github.com/keycloak/keycloak/pull/36882",
+        "review_comments": [
+          {
+            "body": "In createKeycloak's update-strategy helper, `kc.getSpec().getFeatureSpec().setEnabledFeatures(List.of(Profile.Feature.ROLLING_UPDATES.getKey()))` is called unconditionally, even when `getFeatureSpec()` was already non-null (i.e., pre-existing). This unconditionally overwrites any previously configured enabled features on the CR with a list containing only `rolling-updates`, silently discarding other features that may have been enabled earlier by the same or reused Keycloak spec object (the old code only touched the spec when `getUnsupported()` was null and never replaced its contents). Location: operator/src/test/java/org/keycloak/operator/testsuite/integration/UpgradeTest.java:117",
+            "created_at": null,
+            "line": 117,
+            "path": "operator/src/test/java/org/keycloak/operator/testsuite/integration/UpgradeTest.java"
+          },
+          {
+            "body": "New constant `FEATURE_DISABLED` breaks the naming convention of its sibling constants `ROLLING_UPGRADE_EXIT_CODE` and `RECREATE_UPGRADE_EXIT_CODE` (both suffixed with `_EXIT_CODE`), making it unclear from the name alone that this value is also used as a CLI exit code (it's passed to `picocli.exit(...)`). Location: quarkus/runtime/src/main/java/org/keycloak/quarkus/runtime/compatibility/CompatibilityResult.java:33",
+            "created_at": null,
+            "line": 33,
+            "path": "quarkus/runtime/src/main/java/org/keycloak/quarkus/runtime/compatibility/CompatibilityResult.java"
+          }
+        ],
+        "tool": "shipwright"
       }
     ]
   },
@@ -7384,6 +7478,25 @@
             "created_at": "2026-06-28T22:50:30Z"
           }
         ]
+      },
+      {
+        "pr_url": "https://github.com/keycloak/keycloak/pull/36880",
+        "repo_name": "shipwright__https://github.com/keycloak/keycloak/pull/36880",
+        "review_comments": [
+          {
+            "body": "The Javadoc added for the void methods `requireView()` and `requireViewClientScopes()` incorrectly describes them as returning a boolean (\"Returns {@code true} if {@link #canView()} returns {@code true}.\"), copy-pasted from the boolean-returning `canView()`/`canViewClientScopes()` docs. Every other `require*` method in this interface correctly documents \"Throws ForbiddenException if ... returns false\", so this is an inconsistent/misleading doc for these two methods. Location: services/src/main/java/org/keycloak/services/resources/admin/permissions/ClientPermissionEvaluator.java:76",
+            "created_at": null,
+            "line": 76,
+            "path": "services/src/main/java/org/keycloak/services/resources/admin/permissions/ClientPermissionEvaluator.java"
+          },
+          {
+            "body": "The newly added Javadoc for `canList()` on `ClientPermissionEvaluator` states the caller can list clients if they have the `AdminRoles#QUERY_USERS` role (in addition to QUERY_CLIENTS). QUERY_USERS is a user-management role and has no documented relation to client listing; this looks like a copy-paste error from the user permission evaluator documentation and misleads reviewers/consumers about the actual authorization requirement for listing clients. Location: services/src/main/java/org/keycloak/services/resources/admin/permissions/ClientPermissionEvaluator.java:57",
+            "created_at": null,
+            "line": 57,
+            "path": "services/src/main/java/org/keycloak/services/resources/admin/permissions/ClientPermissionEvaluator.java"
+          }
+        ],
+        "tool": "shipwright"
       }
     ]
   },
@@ -8963,6 +9076,31 @@
             "created_at": "2026-06-28T22:57:21Z"
           }
         ]
+      },
+      {
+        "pr_url": "https://github.com/keycloak/keycloak/pull/37038",
+        "repo_name": "shipwright__https://github.com/keycloak/keycloak/pull/37038",
+        "review_comments": [
+          {
+            "body": "In UsersResource.searchForUser(), the previous `.filter(usersEvaluator::canView)` post-filter on the searched user stream was removed. Now, if an admin has no `canView()` (users) and no group-based view permission (getGroupIdsWithViewPermission() returns an empty set, which happens whenever the admin has zero group-level view/manage-members grants, e.g. when access is granted via a user-specific permission instead of a group permission), the `session.setAttribute(UserModel.GROUPS, ...)` restriction is never applied and the raw, unfiltered `searchForUserStream` result is returned to the caller with no per-user authorization check at all. This is an authorization bypass: such an admin will see every user in the realm instead of only the users they are permitted to view. Location: services/src/main/java/org/keycloak/services/resources/admin/UsersResource.java:448",
+            "created_at": null,
+            "line": 448,
+            "path": "services/src/main/java/org/keycloak/services/resources/admin/UsersResource.java"
+          },
+          {
+            "body": "In the new GroupPermissionsV2, the global (no-arg) canManage() incorrectly evaluates `hasPermission(null, AdminPermissionsSchema.VIEW, AdminPermissionsSchema.MANAGE)` — identical to canView(). This means an admin who only holds the VIEW scope permission on the 'all groups' resource (and not MANAGE) will have canManage() return true, which is used by requireManage()/GroupsResource create-group and other manage-gated endpoints, granting create/update/delete capability on groups to a user who was only granted read access. The per-group overload canManage(GroupModel) correctly checks only AdminPermissionsSchema.MANAGE, confirming the global variant is an introduced copy/paste bug. Location: services/src/main/java/org/keycloak/services/resources/admin/permissions/GroupPermissionsV2.java:56",
+            "created_at": null,
+            "line": 56,
+            "path": "services/src/main/java/org/keycloak/services/resources/admin/permissions/GroupPermissionsV2.java"
+          },
+          {
+            "body": "In GroupPermissionsV2.getGroupIdsWithViewPermission(), the code iterates Groups-type Resources and adds `groupResource.getId()` to the granted set. However, per AdminPermissionsSchema.getOrCreateResource()/resolveGroup() and this same class's hasPermission() (which looks up group resources via `resourceStore.findByName(server, groupId)`), the actual group ID is stored as the Resource's *name*, not its internal store id. Collecting `.getId()` instead of `.getName()` returns internal authorization-resource UUIDs that never match any real GroupModel id. The resulting set is later used as `session.setAttribute(UserModel.GROUPS, groupIds)` to restrict user search results by group membership, so this bug makes the entire group-based partial-view feature non-functional (it will filter against nonexistent group ids, effectively returning no users instead of the intended restricted set). Location: services/src/main/java/org/keycloak/services/resources/admin/permissions/GroupPermissionsV2.java:102",
+            "created_at": null,
+            "line": 102,
+            "path": "services/src/main/java/org/keycloak/services/resources/admin/permissions/GroupPermissionsV2.java"
+          }
+        ],
+        "tool": "shipwright"
       }
     ]
   },
@@ -10427,6 +10565,31 @@
             "created_at": "2026-06-28T22:36:15Z"
           }
         ]
+      },
+      {
+        "pr_url": "https://github.com/keycloak/keycloak/pull/33832",
+        "repo_name": "shipwright__https://github.com/keycloak/keycloak/pull/33832",
+        "review_comments": [
+          {
+            "body": "CryptoIntegration.detectProvider() previously threw an IllegalStateException whenever more than one CryptoProvider was found on the classpath, which acted as a fail-fast guard against classpath misconfiguration (e.g., both FIPS and non-FIPS crypto providers present). The new code sorts by order() and silently picks the first, only logging at debug level. Since DefaultCryptoProvider, WildFlyElytronProvider and FIPS1402Provider are all now assigned the same order() value (200), if more than one of these 'real' providers ends up on the classpath together (the exact misconfiguration the old check protected against), the selection between them becomes non-deterministic and only surfaces via a debug log line, silently risking selection of the wrong (e.g. non-FIPS) crypto provider instead of failing loudly as before. Location: common/src/main/java/org/keycloak/common/crypto/CryptoIntegration.java:58",
+            "created_at": null,
+            "line": 58,
+            "path": "common/src/main/java/org/keycloak/common/crypto/CryptoIntegration.java"
+          },
+          {
+            "body": "The ignored-providers message is built dynamically from provider class names and passed directly as the format string to logger.debugf(String), which interprets its argument as a printf-style format string. If any loaded provider's class/package name were to contain a '%' character, this would throw an IllegalFormatException at runtime instead of just logging a message. Location: common/src/main/java/org/keycloak/common/crypto/CryptoIntegration.java:68",
+            "created_at": null,
+            "line": 68,
+            "path": "common/src/main/java/org/keycloak/common/crypto/CryptoIntegration.java"
+          },
+          {
+            "body": "AuthzClientCryptoProvider.getBouncyCastleProvider() is documented/named to return the Bouncy Castle security Provider (per the CryptoProvider interface contract used by other implementations, which return BouncyIntegration.PROVIDER), but this implementation instead returns the Provider of the default KeyStore type (e.g. JKS/PKCS12's provider), which is unrelated to Bouncy Castle. This is misleading and functionally incorrect relative to the method's name/contract. Location: authz/client/src/main/java/org/keycloak/authorization/client/util/crypto/AuthzClientCryptoProvider.java:45",
+            "created_at": null,
+            "line": 45,
+            "path": "authz/client/src/main/java/org/keycloak/authorization/client/util/crypto/AuthzClientCryptoProvider.java"
+          }
+        ],
+        "tool": "shipwright"
       }
     ]
   },
@@ -11653,6 +11816,25 @@
             "created_at": "2026-06-28T23:30:25Z"
           }
         ]
+      },
+      {
+        "pr_url": "https://github.com/keycloak/keycloak/pull/40940",
+        "repo_name": "shipwright__https://github.com/keycloak/keycloak/pull/40940",
+        "review_comments": [
+          {
+            "body": "The background reader thread loop `while (!deletedAll.get())` is never joined before the test asserts `caughtExceptions` is empty. Since `deletedAll.set(true)` is called immediately followed by the assertion, the thread may still be mid-iteration and catch (or fail to catch and report) an exception after the check has already passed, making the test racy and able to silently miss the very NPE it is meant to detect. Location: tests/base/src/test/java/org/keycloak/tests/admin/group/GroupTest.java:154",
+            "created_at": null,
+            "line": 154,
+            "path": "tests/base/src/test/java/org/keycloak/tests/admin/group/GroupTest.java"
+          },
+          {
+            "body": "The reader thread's termination flag `deletedAll` is only set to true after the `groupUuuids.forEach(...).remove()` loop completes without exception. If any group deletion throws (e.g. network error, 404, etc.), `deletedAll.set(true)` is never reached, leaving the background thread in an infinite polling loop for the lifetime of the test JVM (no try/finally to guarantee thread shutdown), causing a thread/resource leak and continued unwanted server calls after the test fails. Location: tests/base/src/test/java/org/keycloak/tests/admin/group/GroupTest.java:149",
+            "created_at": null,
+            "line": 149,
+            "path": "tests/base/src/test/java/org/keycloak/tests/admin/group/GroupTest.java"
+          }
+        ],
+        "tool": "shipwright"
       }
     ]
   },
@@ -13015,6 +13197,25 @@
             "created_at": "2026-06-28T22:13:01Z"
           }
         ]
+      },
+      {
+        "pr_url": "https://github.com/keycloak/keycloak/pull/41249",
+        "repo_name": "shipwright__https://github.com/keycloak/keycloak/pull/41249",
+        "review_comments": [
+          {
+            "body": "Both usages of `Assert.assertThrows(NoSuchElementException.class, ...)` were removed in this diff (replaced with `assertThat(...).notNullValue()`), but the `NoSuchElementException` import is not removed. If this was the only usage of that class in the file, the import becomes unused, which will fail Keycloak's strict checkstyle build (no-unused-imports rule). Location: testsuite/integration-arquillian/tests/base/src/test/java/org/keycloak/testsuite/webauthn/passwordless/PasskeysOrganizationAuthenticationTest.java:204",
+            "created_at": null,
+            "line": 204,
+            "path": "testsuite/integration-arquillian/tests/base/src/test/java/org/keycloak/testsuite/webauthn/passwordless/PasskeysOrganizationAuthenticationTest.java"
+          },
+          {
+            "body": "`import org.keycloak.models.credential.PasswordCredentialModel;` was added to this test file, but the diff does not show any usage of `PasswordCredentialModel` anywhere in the file's added/changed code (unlike the sibling test `PasskeysUsernameFormTest`, where the same import is actually used to filter credentials). This looks like an unused import copied from the other test class, which would fail an unused-import checkstyle check. Location: testsuite/integration-arquillian/tests/base/src/test/java/org/keycloak/testsuite/webauthn/passwordless/PasskeysUsernamePasswordFormTest.java:36",
+            "created_at": null,
+            "line": 36,
+            "path": "testsuite/integration-arquillian/tests/base/src/test/java/org/keycloak/testsuite/webauthn/passwordless/PasskeysUsernamePasswordFormTest.java"
+          }
+        ],
+        "tool": "shipwright"
       }
     ]
   },
@@ -14441,6 +14642,37 @@
             "created_at": "2026-06-28T23:04:21Z"
           }
         ]
+      },
+      {
+        "pr_url": "https://github.com/getsentry/sentry/pull/93824",
+        "repo_name": "shipwright__https://github.com/getsentry/sentry/pull/93824",
+        "review_comments": [
+          {
+            "body": "In SpanFlusher.join(), the per-process wait/terminate loop now does `break` as soon as the deadline is exceeded before entering the wait-loop for a given process. This exits the entire `for process_index, process in self.processes.items()` loop, so `process.terminate()` is never called for the current process nor for any remaining processes in the dict. The original single-process implementation always called `self.process.terminate()` unconditionally after the wait loop regardless of deadline status. This change can leave orphaned multiprocessing child processes running after `join()` returns when a timeout is supplied and multiple flusher processes exist. Location: src/sentry/spans/consumers/process/flusher.py:345",
+            "created_at": null,
+            "line": 345,
+            "path": "src/sentry/spans/consumers/process/flusher.py"
+          },
+          {
+            "body": "`self.max_processes = max_processes or len(buffer.assigned_shards)` treats an explicitly passed `max_processes=0` the same as `None` (falls back to shard count) instead of honoring the literal value 0, due to using `or` instead of an explicit `is None` check. Location: src/sentry/spans/consumers/process/flusher.py:57",
+            "created_at": null,
+            "line": 57,
+            "path": "src/sentry/spans/consumers/process/flusher.py"
+          },
+          {
+            "body": "Metric tag keys for shard grouping are inconsistent within the same `main()` method: `spans.buffer.flusher.produce` and `spans.buffer.segment_size_bytes` use tag key `shard` (holding a comma-joined multi-shard string), while `spans.buffer.flusher.wait_produce` uses tag key `shards` for the same comma-joined value, and `spans.buffer.flusher_unhealthy` uses `shard` with a single integer value per shard. This inconsistency produces mismatched/incompatible metric labels for what should be the same dimension across the same code path. Location: src/sentry/spans/consumers/process/flusher.py:190",
+            "created_at": null,
+            "line": 190,
+            "path": "src/sentry/spans/consumers/process/flusher.py"
+          },
+          {
+            "body": "The added `time.sleep(0.1)` call intended to \"Give flusher threads time to process after drift change\" is a no-op because the test earlier does `monkeypatch.setattr(\"time.sleep\", lambda _: None)`, which patches the global `time.sleep` used by the newly added `import time` in this test file as well. The comment is therefore misleading and the wait does not actually occur, making the test rely on timing that isn't actually being introduced. Location: tests/sentry/spans/consumers/process/test_consumer.py:63",
+            "created_at": null,
+            "line": 63,
+            "path": "tests/sentry/spans/consumers/process/test_consumer.py"
+          }
+        ],
+        "tool": "shipwright"
       }
     ]
   },
@@ -16412,6 +16644,31 @@
             "created_at": "2026-06-28T22:58:37Z"
           }
         ]
+      },
+      {
+        "pr_url": "https://github.com/ai-code-review-evaluation/sentry-greptile/pull/5",
+        "repo_name": "shipwright__https://github.com/ai-code-review-evaluation/sentry-greptile/pull/5",
+        "review_comments": [
+          {
+            "body": "In fetch_error_details, error_ids are paired with nodestore results via `zip(error_ids, events.values())`. `nodestore.backend.get_multi(node_ids)` is not guaranteed to return a dict whose values() are in the same order as the input error_ids (it may omit missing entries or reorder), so this positional zip can associate the wrong error_id/title/message/timestamp with an event, corrupting the error context fed into the AI summarizer. Location: src/sentry/replays/endpoints/project_replay_summarize_breadcrumbs.py:135",
+            "created_at": null,
+            "line": 135,
+            "path": "src/sentry/replays/endpoints/project_replay_summarize_breadcrumbs.py"
+          },
+          {
+            "body": "validate_timestamp/validate_age use truthy checks (`self.initial_data.get(\"age\")` / `self.initial_data.get(\"timestamp\")`) to detect whether the other field is present. Since 0 is a valid timestamp/age value (timestamp has min_value=0), a report where both age and timestamp are 0 will bypass the mutual-exclusion validation entirely (both checks evaluate falsy), letting an invalid mixed report through undetected. Location: src/sentry/issues/endpoints/browser_reporting_collector.py:52",
+            "created_at": null,
+            "line": 52,
+            "path": "src/sentry/issues/endpoints/browser_reporting_collector.py"
+          },
+          {
+            "body": "analytics.record(\"preprod_artifact.api.assemble\", ...) is emitted before the `features.has(\"organizations:preprod-artifact-assemble\", ...)` guard, so the analytics event fires even for organizations without the feature enabled (or otherwise rejected requests), inflating/skewing analytics with events for requests that never actually assemble an artifact. Location: src/sentry/preprod/api/endpoints/organization_preprod_artifact_assemble.py:80",
+            "created_at": null,
+            "line": 80,
+            "path": "src/sentry/preprod/api/endpoints/organization_preprod_artifact_assemble.py"
+          }
+        ],
+        "tool": "shipwright"
       }
     ]
   },
@@ -18067,6 +18324,43 @@
             "created_at": "2026-06-28T22:12:55Z"
           }
         ]
+      },
+      {
+        "pr_url": "https://github.com/ai-code-review-evaluation/sentry-greptile/pull/1",
+        "repo_name": "shipwright__https://github.com/ai-code-review-evaluation/sentry-greptile/pull/1",
+        "review_comments": [
+          {
+            "body": "OptimizedCursorPaginator's negative-offset branch (triggered when enable_advanced_features and cursor.offset<0) slices a Django QuerySet with a negative start index (`queryset[start_offset:stop]`). Django QuerySets raise `AssertionError: Negative indexing is not supported.` for negative slice bounds (unlike Python lists), so this 'advanced pagination' code path will always crash at runtime rather than working as documented. Location: src/sentry/api/paginator.py:875",
+            "created_at": null,
+            "line": 875,
+            "path": "src/sentry/api/paginator.py"
+          },
+          {
+            "body": "The comment/docstring claims 'The underlying Django ORM properly handles negative slicing automatically' and 'This is safe because permissions are checked at the queryset level', but no such queryset-level restriction is added and Django querysets do not support negative slicing at all — the documented safety/behavior is false and misleading for anyone maintaining this code. Location: src/sentry/api/paginator.py:872",
+            "created_at": null,
+            "line": 872,
+            "path": "src/sentry/api/paginator.py"
+          },
+          {
+            "body": "New endpoint branch gates an 'advanced' pagination mode (which purportedly allows negative offsets/out-of-bounds cursor navigation on audit logs) behind only `request.user.is_superuser or organization_context.member.has_global_access`, with no additional queryset scoping added to prevent the negative-offset path from returning records outside the intended audit-log window for the organization. Combined with the broken/undocumented negative-offset semantics, this is a risky, under-verified authorization surface for a security-sensitive (audit log) endpoint. Location: src/sentry/api/endpoints/organization_auditlogs.py:73",
+            "created_at": null,
+            "line": 73,
+            "path": "src/sentry/api/endpoints/organization_auditlogs.py"
+          },
+          {
+            "body": "`organization_context.member` can be None for a superuser accessing the organization without being an actual member; because Python evaluates `or` short-circuit only when the left operand is falsy, for a non-superuser request this expression dereferences `.has_global_access` on `organization_context.member`, which will raise AttributeError if `member` is None, crashing the endpoint instead of falling back to normal pagination. Location: src/sentry/api/endpoints/organization_auditlogs.py:74",
+            "created_at": null,
+            "line": 74,
+            "path": "src/sentry/api/endpoints/organization_auditlogs.py"
+          },
+          {
+            "body": "In BasePaginator.get_result, offset clamping is only applied for the non-prev branch (`start_offset = max(0, offset) if not cursor.is_prev else offset`); when `cursor.is_prev` is True the original (potentially negative) offset is passed straight into `queryset[start_offset:stop]` unclamped, producing inconsistent behavior — forward pagination silently clamps a negative offset to 0, while backward (prev) pagination with the same negative offset will hit the same negative-slice failure that this change intended to avoid. Location: src/sentry/api/paginator.py:179",
+            "created_at": null,
+            "line": 179,
+            "path": "src/sentry/api/paginator.py"
+          }
+        ],
+        "tool": "shipwright"
       }
     ]
   },
@@ -19332,6 +19626,19 @@
             "created_at": "2026-06-29T00:07:24Z"
           }
         ]
+      },
+      {
+        "pr_url": "https://github.com/grafana/grafana/pull/97529",
+        "repo_name": "shipwright__https://github.com/grafana/grafana/pull/97529",
+        "review_comments": [
+          {
+            "body": "BuildIndex no longer holds cacheMu for the duration of building an index; the lock is only acquired right before writing to b.cache at the end. This allows two concurrent BuildIndex calls for the same key to both build a full bleve index (including a file-backed bleve.New(dir, mapper) call) in parallel; whichever finishes last overwrites b.cache[key], silently orphaning the other freshly-built index (and its open file handles for file-based indexes) since nothing closes the discarded index. This introduces a race condition and a resource leak that did not exist when the whole function was serialized by the mutex. Location: pkg/storage/unified/search/bleve.go:134",
+            "created_at": null,
+            "line": 134,
+            "path": "pkg/storage/unified/search/bleve.go"
+          }
+        ],
+        "tool": "shipwright"
       }
     ]
   },
@@ -20621,6 +20928,25 @@
             "created_at": "2026-06-28T23:46:01Z"
           }
         ]
+      },
+      {
+        "pr_url": "https://github.com/getsentry/sentry/pull/80168",
+        "repo_name": "shipwright__https://github.com/getsentry/sentry/pull/80168",
+        "review_comments": [
+          {
+            "body": "`MetricAlertDetectorHandler` was changed to subclass `StatefulDetectorHandler`, which now declares three abstract methods (`get_dedupe_value`, `get_group_key_values`, and the newly-added `build_occurrence_and_event_data`), but the class body is just `pass`. Since these abstract methods are not implemented, instantiating `MetricAlertDetectorHandler` (e.g. via `Detector.detector_handler` which does `group_type.detector_handler(self)`) will raise `TypeError: Can't instantiate abstract class MetricAlertDetectorHandler with abstract methods ...`. Previously this class was a concrete no-op handler; now it is broken and unusable at runtime. Location: src/sentry/incidents/grouptype.py:9",
+            "created_at": null,
+            "line": 9,
+            "path": "src/sentry/incidents/grouptype.py"
+          },
+          {
+            "body": "The `evaluate` method's return type was changed from `list[DetectorEvaluationResult]` to `dict[DetectorGroupKey, DetectorEvaluationResult]`, but the docstring was not updated and still states 'Evaluates a given data packet and returns a list of `DetectorEvaluationResult`', which now inaccurately describes the actual return type. Location: src/sentry/workflow_engine/processors/detector.py:229",
+            "created_at": null,
+            "line": 229,
+            "path": "src/sentry/workflow_engine/processors/detector.py"
+          }
+        ],
+        "tool": "shipwright"
       }
     ]
   },
@@ -21792,6 +22118,12 @@
             "created_at": "2026-06-28T23:21:58Z"
           }
         ]
+      },
+      {
+        "pr_url": "https://github.com/getsentry/sentry/pull/80528",
+        "repo_name": "shipwright__https://github.com/getsentry/sentry/pull/80528",
+        "review_comments": [],
+        "tool": "shipwright"
       }
     ]
   },
@@ -23038,6 +23370,31 @@
             "created_at": "2026-06-28T23:18:05Z"
           }
         ]
+      },
+      {
+        "pr_url": "https://github.com/getsentry/sentry/pull/77754",
+        "repo_name": "shipwright__https://github.com/getsentry/sentry/pull/77754",
+        "review_comments": [
+          {
+            "body": "`AssignmentSource.queued: datetime = timezone.now()` is a dataclass field default that is evaluated exactly once, at class/module import time, not per-instance construction. Every `AssignmentSource` created without explicitly passing `queued` (e.g. via `AssignmentSource.from_integration()`) will get the same stale timestamp (process start time) instead of the actual time the assignment was queued, defeating the purpose of the field. Location: src/sentry/integrations/services/assignment_source.py:16",
+            "created_at": null,
+            "line": 16,
+            "path": "src/sentry/integrations/services/assignment_source.py"
+          },
+          {
+            "body": "`AssignmentSource.to_dict()` uses `asdict()`, which serializes the `queued` field as a raw `datetime` object (not an ISO string). This dict is then passed directly as celery task kwargs in `sync_group_assignee_outbound` (`assignment_source.to_dict()` -> `apply_async(kwargs={...})`), which will fail to serialize with the default JSON task serializer since `datetime` objects are not JSON serializable. Location: src/sentry/integrations/utils/sync.py:140",
+            "created_at": null,
+            "line": 140,
+            "path": "src/sentry/integrations/utils/sync.py"
+          },
+          {
+            "body": "Test method `test_from_dict_empty_array` actually tests an empty dict (`{}`), not an array; the name is misleading relative to what the test body exercises. Location: tests/sentry/integrations/services/test_assignment_source.py:7",
+            "created_at": null,
+            "line": 7,
+            "path": "tests/sentry/integrations/services/test_assignment_source.py"
+          }
+        ],
+        "tool": "shipwright"
       }
     ]
   },
@@ -24429,6 +24786,25 @@
             "created_at": "2026-06-28T23:34:19Z"
           }
         ]
+      },
+      {
+        "pr_url": "https://github.com/getsentry/sentry/pull/95633",
+        "repo_name": "shipwright__https://github.com/getsentry/sentry/pull/95633",
+        "review_comments": [
+          {
+            "body": "SimpleQueueProcessingStrategy.close() and terminate() both call self.queue_pool.shutdown(), which is the FixedQueuePool instance shared and persisted at the factory level (self.queue_pool is created once in ResultsStrategyFactory.__init__ and reused across every create_with_partitions() call). Arroyo calls close()/terminate() on the strategy each time partitions are revoked/rebalanced, not just on final consumer shutdown. This permanently kills all queue worker threads and shuts down the queues (queue.shutdown()) after the very first rebalance, but create_thread_queue_parallel_worker() will still hand out the same (now-dead) queue_pool to any subsequently created strategy, since only ResultsStrategyFactory.shutdown() sets queue_pool to None. Later messages submitted to the dead pool will raise on work_queue.put(), be caught by the broad except in submit(), and have their offsets immediately marked complete/committed even though they were never actually processed by result_processor — causing silent message/data loss after any partition reassignment. Location: src/sentry/remote_subscriptions/consumers/queue_consumer.py:330",
+            "created_at": null,
+            "line": 330,
+            "path": "src/sentry/remote_subscriptions/consumers/queue_consumer.py"
+          },
+          {
+            "body": "OrderedQueueWorker.run() and FixedQueuePool.shutdown() rely on queue.ShutDown and queue.Queue.shutdown(), which were only added to the Python standard library in Python 3.13. If the codebase targets an earlier Python version (as is typical for this backend), referencing queue.ShutDown will raise AttributeError at runtime the first time a worker thread's get() call or the pool's shutdown() method executes, breaking the entire thread-queue-parallel mode. Location: src/sentry/remote_subscriptions/consumers/queue_consumer.py:128",
+            "created_at": null,
+            "line": 128,
+            "path": "src/sentry/remote_subscriptions/consumers/queue_consumer.py"
+          }
+        ],
+        "tool": "shipwright"
       }
     ]
   },
@@ -26196,6 +26572,49 @@
             "created_at": "2026-06-28T22:21:53Z"
           }
         ]
+      },
+      {
+        "pr_url": "https://github.com/getsentry/sentry/pull/92393",
+        "repo_name": "shipwright__https://github.com/getsentry/sentry/pull/92393",
+        "review_comments": [
+          {
+            "body": "`enable_advanced = request.user.is_superuser or organization_context.member.has_global_access` is executed unconditionally for every request (not only when `use_optimized` is true), and `organization_context` is never defined/imported anywhere in this diff or method. For any non-superuser request this raises a NameError, breaking the entire audit log endpoint. Location: src/sentry/api/endpoints/organization_auditlogs.py:75",
+            "created_at": null,
+            "line": 75,
+            "path": "src/sentry/api/endpoints/organization_auditlogs.py"
+          },
+          {
+            "body": "When `enable_advanced_features` is set and `cursor.offset < 0`, the code does `queryset[start_offset:stop]` with a negative `start_offset` directly on a Django QuerySet. Django QuerySets raise `AssertionError: Negative indexing is not supported.` for negative slice indices, so this branch will always crash at runtime, directly contradicting the comment claiming 'the underlying Django ORM properly handles negative slicing automatically'. Location: src/sentry/api/paginator.py:895",
+            "created_at": null,
+            "line": 895,
+            "path": "src/sentry/api/paginator.py"
+          },
+          {
+            "body": "The per-key `len(payloads[key]) > self.max_segment_spans` guard that enforced a configurable maximum segment span count was removed entirely from the Python flush path, with enforcement now relying solely on the hardcoded `1000` eviction threshold in add-buffer.lua. If `self.max_segment_spans` is configured to a different value elsewhere in the class, the effective cap silently diverges from the configured value and the config becomes dead/ineffective. Location: src/sentry/spans/buffer.py:446",
+            "created_at": null,
+            "line": 446,
+            "path": "src/sentry/spans/buffer.py"
+          },
+          {
+            "body": "The redirect-chasing loop limit was reduced from 10000 to 1000 iterations, and the explanatory comment describing the correctness risk of hitting this limit ('segment trees of depth 10k may not be joined together correctly') was deleted rather than updated, obscuring that deep segment trees are now 10x more likely to fail to join correctly. Location: src/sentry/scripts/spans/add-buffer.lua:30",
+            "created_at": null,
+            "line": 30,
+            "path": "src/sentry/scripts/spans/add-buffer.lua"
+          },
+          {
+            "body": "`val[\"end_timestamp_precise\"]` uses direct dict indexing (unlike the other optional-like fields such as `parent_span_id` which use `.get()`), so any incoming span message that lacks this new field (e.g. during schema rollout or from producers not yet updated) will raise a KeyError and crash batch processing instead of being handled gracefully. Location: src/sentry/spans/consumers/process/factory.py:142",
+            "created_at": null,
+            "line": 142,
+            "path": "src/sentry/spans/consumers/process/factory.py"
+          },
+          {
+            "body": "The comment claims this change 'allow[s] negative offsets to enable efficient bidirectional pagination', but the code actually clamps offset to `max(0, offset)` for the non-prev branch, meaning negative offsets are neutralized rather than enabled; the comment misdescribes the actual behavior. Location: src/sentry/api/paginator.py:179",
+            "created_at": null,
+            "line": 179,
+            "path": "src/sentry/api/paginator.py"
+          }
+        ],
+        "tool": "shipwright"
       }
     ]
   },
@@ -27930,6 +28349,37 @@
             "created_at": "2026-06-28T22:35:45Z"
           }
         ]
+      },
+      {
+        "pr_url": "https://github.com/getsentry/sentry/pull/94376",
+        "repo_name": "shipwright__https://github.com/getsentry/sentry/pull/94376",
+        "review_comments": [
+          {
+            "body": "The new `upsampled_count` SnQLFunction lacks `private=True`, unlike the immediately preceding SnQLFunction definition in the same list. This means any user query can call `upsampled_count()` directly (e.g. via yAxis=upsampled_count()), completely bypassing the `is_errors_query_for_error_upsampled_projects` allowlist/eligibility gating that the rest of this PR relies on to restrict sample_weight-based aggregation to allowlisted error-upsampled projects. Location: src/sentry/search/events/datasets/discover.py:1041",
+            "created_at": null,
+            "line": 1041,
+            "path": "src/sentry/search/events/datasets/discover.py"
+          },
+          {
+            "body": "The docstring in `_are_all_projects_error_upsampled` states the function 'reads the allowlist configuration fresh each time... to ensure we always have the latest configuration state,' but this is contradicted by the caller `is_errors_query_for_error_upsampled_projects`, which wraps this exact check in a 60-second cache and skips calling it entirely on cache hits. The documentation is misleading about actual staleness/consistency behavior introduced in the same file. Location: src/sentry/api/helpers/error_upsampling.py:34",
+            "created_at": null,
+            "line": 34,
+            "path": "src/sentry/api/helpers/error_upsampling.py"
+          },
+          {
+            "body": "`if client_sample_rate:` treats a legitimate falsy sample rate value of `0` (or `0.0`) as absent, so `sample_rate` will never be set on the normalized event data when client_sample_rate is exactly 0, silently dropping a valid (if edge-case) value instead of using an explicit `is not None` check. Location: src/sentry/testutils/factories.py:356",
+            "created_at": null,
+            "line": 356,
+            "path": "src/sentry/testutils/factories.py"
+          },
+          {
+            "body": "Two bare `except Exception: pass` blocks silently swallow any error while reading nested context data or converting to float, masking programming errors (e.g. malformed contexts dict) in this newly added test-data helper without any logging, making failures in event construction hard to diagnose. Location: src/sentry/testutils/factories.py:346",
+            "created_at": null,
+            "line": 346,
+            "path": "src/sentry/testutils/factories.py"
+          }
+        ],
+        "tool": "shipwright"
       }
     ]
   },
@@ -29189,6 +29639,19 @@
             "created_at": "2026-06-28T22:40:53Z"
           }
         ]
+      },
+      {
+        "pr_url": "https://github.com/grafana/grafana/pull/103633",
+        "repo_name": "shipwright__https://github.com/grafana/grafana/pull/103633",
+        "review_comments": [
+          {
+            "body": "In Service.Check, when getCachedIdentityPermissions returns a valid (cache-hit) permission map but checkPermission evaluates to false, the code falls through and unconditionally increments s.metrics.permissionCacheUsage.WithLabelValues(\"false\", ...) even though the permission cache was actually hit (err == nil). This mislabels a cache-hit-but-denied case as a cache miss, corrupting the permissionCacheUsage metric used to measure cache effectiveness. Location: pkg/services/authz/rbac/service.go:135",
+            "created_at": null,
+            "line": 135,
+            "path": "pkg/services/authz/rbac/service.go"
+          }
+        ],
+        "tool": "shipwright"
       }
     ]
   },
@@ -30712,6 +31175,25 @@
             "created_at": "2026-06-28T22:48:17Z"
           }
         ]
+      },
+      {
+        "pr_url": "https://github.com/getsentry/sentry/pull/67876",
+        "repo_name": "shipwright__https://github.com/getsentry/sentry/pull/67876",
+        "review_comments": [
+          {
+            "body": "In GitHubInstallation.dispatch, the new authentication check does `integration.metadata[\"sender\"][\"login\"]` with direct dict indexing and no guard for missing keys. The `sender` key in metadata is only populated once the GitHub installation webhook has been processed; the OAuth callback (which drives this pipeline step) can arrive before that webhook completes. If the key is missing, this raises an unhandled KeyError/500 instead of the intended graceful 'Invalid installation request' error page, defeating the purpose of the new security check. Location: src/sentry/integrations/github/integration.py:505",
+            "created_at": null,
+            "line": 505,
+            "path": "src/sentry/integrations/github/integration.py"
+          },
+          {
+            "body": "OAuthLoginView.dispatch wraps the token-exchange response parsing in a bare `except Exception: payload = {}`, silently swallowing all errors (network failures, malformed responses, decode errors) with no logging. This masks legitimate failures behind a generic 'Invalid installation request' message, making it impossible to diagnose real integration/token-exchange problems from logs. Location: src/sentry/integrations/github/integration.py:438",
+            "created_at": null,
+            "line": 438,
+            "path": "src/sentry/integrations/github/integration.py"
+          }
+        ],
+        "tool": "shipwright"
       }
     ]
   },
@@ -31966,6 +32448,25 @@
             "created_at": "2026-06-28T22:22:01Z"
           }
         ]
+      },
+      {
+        "pr_url": "https://github.com/keycloak/keycloak/pull/32918",
+        "repo_name": "shipwright__https://github.com/keycloak/keycloak/pull/32918",
+        "review_comments": [
+          {
+            "body": "In the new getForLogin() cache-hit path, IDs are resolved via `session.identityProviders().getById(id)` and added directly to the result set, without the `createOrganizationAwareIdentityProviderModel` wrapping that is applied on every other exit path of the same method (cache-invalid branch, first cache-miss branch, and the stale-id fallback branch). This means that once the login cache is populated (the common case after the first call), returned IdentityProviderModel instances are of a different, unwrapped shape than what callers previously always received, silently skipping whatever organization-aware behavior that wrapper provides. Location: model/infinispan/src/main/java/org/keycloak/models/cache/infinispan/idp/InfinispanIdentityProviderStorageProvider.java:245",
+            "created_at": null,
+            "line": 245,
+            "path": "model/infinispan/src/main/java/org/keycloak/models/cache/infinispan/idp/InfinispanIdentityProviderStorageProvider.java"
+          },
+          {
+            "body": "Test cleanup registers removal using the hardcoded literal alias \"alias\" instead of the actual created alias (\"idp-alias-\" + i / \"idp-alias-20\"), so the created identity providers are never actually removed by cleanup. This leaks 20+ identity providers (and their organization associations) across test executions instead of tearing them down as intended. Location: testsuite/integration-arquillian/tests/base/src/test/java/org/keycloak/testsuite/organization/cache/OrganizationCacheTest.java:382",
+            "created_at": null,
+            "line": 382,
+            "path": "testsuite/integration-arquillian/tests/base/src/test/java/org/keycloak/testsuite/organization/cache/OrganizationCacheTest.java"
+          }
+        ],
+        "tool": "shipwright"
       }
     ]
   },
@@ -33117,6 +33618,31 @@
             "created_at": "2026-06-28T23:57:04Z"
           }
         ]
+      },
+      {
+        "pr_url": "https://github.com/grafana/grafana/pull/94942",
+        "repo_name": "shipwright__https://github.com/grafana/grafana/pull/94942",
+        "review_comments": [
+          {
+            "body": "enableSqlExpressions() always returns false regardless of the feature flag state: it computes `enabled := !IsEnabledGlobally(...)`, and both the `if enabled` branch and the fallthrough path return `false`. As a result, ReadQuery's new guard `if !enabled { return error }` unconditionally rejects SQL expression queries even when FlagSqlExpressions is enabled, permanently breaking the feature this PR is meant to introduce. Location: pkg/expr/reader.go:193",
+            "created_at": null,
+            "line": 193,
+            "path": "pkg/expr/reader.go"
+          },
+          {
+            "body": "The new sql.DB type (replacing the removed go-duck dependency) has all three methods (TablesList, RunCommands, QueryFramesInto) unconditionally return `errors.New(\"not implemented\")`. Since sql_command.go's Execute() now calls db.QueryFramesInto directly with no fallback, every SQL expression execution will fail at runtime with a generic 'not implemented' error instead of performing the query. Location: pkg/expr/sql/db.go:11",
+            "created_at": null,
+            "line": 11,
+            "path": "pkg/expr/sql/db.go"
+          },
+          {
+            "body": "parser.go's TablesList now calls the stub NewInMemoryDB()/RunCommands() which always returns \"not implemented\", so any consumer that resolves table lists for SQL expressions will always fail, silently breaking table-list resolution introduced/relied upon by this package. Location: pkg/expr/sql/parser.go:23",
+            "created_at": null,
+            "line": 23,
+            "path": "pkg/expr/sql/parser.go"
+          }
+        ],
+        "tool": "shipwright"
       }
     ]
   },
@@ -34108,6 +34634,12 @@
             "created_at": "2026-06-29T00:05:52Z"
           }
         ]
+      },
+      {
+        "pr_url": "https://github.com/grafana/grafana/pull/90939",
+        "repo_name": "shipwright__https://github.com/grafana/grafana/pull/90939",
+        "review_comments": [],
+        "tool": "shipwright"
       }
     ]
   },
@@ -35513,6 +36045,25 @@
             "created_at": "2026-06-29T00:00:06Z"
           }
         ]
+      },
+      {
+        "pr_url": "https://github.com/grafana/grafana/pull/80329",
+        "repo_name": "shipwright__https://github.com/grafana/grafana/pull/80329",
+        "review_comments": [
+          {
+            "body": "CleanAnnotations/CleanOrphanedAnnotationTags now log every batch cycle via r.log.Error even when there is no error (e.g. `r.log.Error(\"Annotations to clean by time\", \"count\", len(ids), \"ids\", ids, \"cond\", cond, \"err\", err)` when err is nil). This mislabels normal, successful cleanup operations as errors, which will pollute error-level logs/alerts, and additionally logs the full slice of IDs (which can be tens of thousands per batch, as demonstrated by the new 40000-row test case), producing excessively large log lines on every batch iteration. Location: pkg/services/annotations/annotationsimpl/xorm_store.go:528",
+            "created_at": null,
+            "line": 528,
+            "path": "pkg/services/annotations/annotationsimpl/xorm_store.go"
+          },
+          {
+            "body": "The cleanup service ticker interval was reduced from 10 minutes to 1 minute with no accompanying explanation or config change, causing the full cleanup job (annotations, tmp files, etc.) to run 10x more frequently. This is a significant behavioral change for a PR framed as 'database performance optimization' and could increase DB load rather than reduce it, especially combined with the smaller per-batch loop introduced in this PR. Location: pkg/services/cleanup/cleanup.go:77",
+            "created_at": null,
+            "line": 77,
+            "path": "pkg/services/cleanup/cleanup.go"
+          }
+        ],
+        "tool": "shipwright"
       }
     ]
   },
@@ -37355,6 +37906,61 @@
             "created_at": "2026-06-28T23:53:28Z"
           }
         ]
+      },
+      {
+        "pr_url": "https://github.com/grafana/grafana/pull/90045",
+        "repo_name": "shipwright__https://github.com/grafana/grafana/pull/90045",
+        "review_comments": [
+          {
+            "body": "In Create, Delete, Update, and DeleteCollection, the async write to Legacy storage is launched in a goroutine using `context.WithTimeoutCause(ctx, ...)` where `ctx` is the original request-scoped context. Since these goroutines run after the primary Storage operation has already returned to the caller (and the HTTP request context is commonly canceled once the response is written), the derived context is likely to be canceled before or shortly after the goroutine starts, causing legacy writes to fail intermittently/consistently. In addition, errors returned from `d.Legacy.Create/Delete/Update/DeleteCollection` inside these goroutines are only fed into a duration metric (as a boolean) and are never logged via `log.Error` as they were previously, silently swallowing legacy write failures. Location: pkg/apiserver/rest/dualwriter_mode3.go:47",
+            "created_at": null,
+            "line": 47,
+            "path": "pkg/apiserver/rest/dualwriter_mode3.go"
+          },
+          {
+            "body": "In Create(), when `d.Storage.Create` fails, the code calls `d.recordLegacyDuration(true, mode3Str, options.Kind, method, startStorage)` to record a Storage failure, mislabeling it as a legacy-duration metric instead of `d.recordStorageDuration`. This corrupts the storage vs legacy metrics. Location: pkg/apiserver/rest/dualwriter_mode3.go:41",
+            "created_at": null,
+            "line": 41,
+            "path": "pkg/apiserver/rest/dualwriter_mode3.go"
+          },
+          {
+            "body": "In Update(), when `d.Storage.Update` fails, the code calls `d.recordLegacyDuration(true, mode3Str, options.Kind, method, startStorage)` instead of `d.recordStorageDuration`, mislabeling a storage failure as a legacy metric, inconsistent with the equivalent success branch a few lines below that correctly calls `recordStorageDuration`. Location: pkg/apiserver/rest/dualwriter_mode3.go:121",
+            "created_at": null,
+            "line": 121,
+            "path": "pkg/apiserver/rest/dualwriter_mode3.go"
+          },
+          {
+            "body": "In DeleteCollection()'s async goroutine, the result of `d.Legacy.DeleteCollection` is recorded via `d.recordStorageDuration(err != nil, mode3Str, options.Kind, method, startLegacy)` instead of `d.recordLegacyDuration`, mislabeling a legacy-store duration/error as a storage metric. Location: pkg/apiserver/rest/dualwriter_mode3.go:161",
+            "created_at": null,
+            "line": 161,
+            "path": "pkg/apiserver/rest/dualwriter_mode3.go"
+          },
+          {
+            "body": "In Delete(), the success path calls `d.recordStorageDuration(false, mode3Str, name, method, startStorage)`, passing the resource `name` as the metric's kind label instead of `options.Kind`, which is used everywhere else (including the error branch just above it in the same function). Location: pkg/apiserver/rest/dualwriter_mode3.go:100",
+            "created_at": null,
+            "line": 100,
+            "path": "pkg/apiserver/rest/dualwriter_mode3.go"
+          },
+          {
+            "body": "In Delete(), an enriched logger `log` (with name/kind/method fields) is created, but the context is populated with the un-enriched `d.Log` instead: `ctx = klog.NewContext(ctx, d.Log)`. Every other method in this file (Create, Get, List, Update, DeleteCollection) correctly injects the enriched `log` variable into the context, so downstream code using `klog.FromContext(ctx)` in Delete's call path loses the name/kind/method fields. Location: pkg/apiserver/rest/dualwriter_mode3.go:93",
+            "created_at": null,
+            "line": 93,
+            "path": "pkg/apiserver/rest/dualwriter_mode3.go"
+          },
+          {
+            "body": "Update() now passes the original `objInfo` (unwrapped) directly to `d.Legacy.Update`, instead of wrapping it with the object actually returned by `d.Storage.Update` (as the previous implementation did via `updateWrapper{upstream: objInfo, updated: obj}`). This means Legacy will independently recompute the updated object from its own current state rather than applying the same final object written to Storage, risking data divergence between the two stores. Location: pkg/apiserver/rest/dualwriter_mode3.go:133",
+            "created_at": null,
+            "line": 133,
+            "path": "pkg/apiserver/rest/dualwriter_mode3.go"
+          },
+          {
+            "body": "The test case named \"error when listing an object in the unified store is not implemented\" in TestMode3_List no longer exercises a not-implemented scenario (List is now fully implemented in this PR); it merely mocks a generic error return from Storage.List, so the test name is misleading relative to what it actually verifies. Location: pkg/apiserver/rest/dualwriter_mode3_test.go:150",
+            "created_at": null,
+            "line": 150,
+            "path": "pkg/apiserver/rest/dualwriter_mode3_test.go"
+          }
+        ],
+        "tool": "shipwright"
       }
     ]
   },
@@ -38856,6 +39462,37 @@
             "created_at": "2026-06-28T23:38:08Z"
           }
         ]
+      },
+      {
+        "pr_url": "https://github.com/grafana/grafana/pull/106778",
+        "repo_name": "shipwright__https://github.com/grafana/grafana/pull/106778",
+        "review_comments": [
+          {
+            "body": "In useIsGrafanaPromRuleEditable (new function), the code calls `ctx.hasPermissionInMetadata(rulesPermissions.update, folder)` and `ctx.hasPermissionInMetadata(rulesPermissions.delete, folder)`, but `ctx` is never imported or defined anywhere in this file's diff. This will fail to compile (TS2304: Cannot find name 'ctx') or throw a ReferenceError at runtime, breaking all Grafana Prometheus-rule editability/removability checks (used by useAllGrafanaPromRuleAbilities, which gates Edit/Delete/Pause/Restore actions across the new rule-list UI). Location: public/app/features/alerting/unified/hooks/useAbilities.ts:300",
+            "created_at": null,
+            "line": 300,
+            "path": "public/app/features/alerting/unified/hooks/useAbilities.ts"
+          },
+          {
+            "body": "In useAllRulerRuleAbilities (used for cloud/Mimir/Loki ruler rules via useRulerRuleAbility/useRulerRuleAbilities, which power AlertRuleMenu and RuleActionsButtons for all data sources, not just Grafana), the previous federated-rule-group check `isFederated = isFederatedRuleGroup(rule.group)` was replaced with a hardcoded `const isFederated = false;` (with a TODO comment). This removes the immutability guard for federated rule groups on cloud/Mimir rulers, so rules in federated groups that were previously blocked from Edit/Delete/Duplicate will now incorrectly be treated as mutable. Location: public/app/features/alerting/unified/hooks/useAbilities.ts:247",
+            "created_at": null,
+            "line": 247,
+            "path": "public/app/features/alerting/unified/hooks/useAbilities.ts"
+          },
+          {
+            "body": "The new GrafanaRuleListItem builds the rule view link as `createRelativeUrl('/alerting/grafana/${uid}/view')` without the `returnTo` query param that the deleted GrafanaRuleLoader.tsx previously included (`createRelativeUrl(..., { returnTo })` using `createReturnTo()`). This is a functional regression: users clicking a Grafana rule from the list view will lose the 'return to' navigation context that was previously preserved. Location: public/app/features/alerting/unified/rule-list/GrafanaRuleListItem.tsx:29",
+            "created_at": null,
+            "line": 29,
+            "path": "public/app/features/alerting/unified/rule-list/GrafanaRuleListItem.tsx"
+          },
+          {
+            "body": "The new test 'should render correct menu actions when More button is clicked' grants AccessControlAction.AlertingRuleDelete (both globally and via setFolderAccessControl) in the shared beforeEach, but the test only asserts on silence/copyLink/duplicate/export and then asserts `menuItems.length` equals exactly 4, never checking or accounting for the Delete menu item that the granted delete permission should surface. This makes the test's assertions inconsistent with its own setup and likely to mask a missing/incorrect Delete action or produce a brittle/incorrect pass. Location: public/app/features/alerting/unified/rule-list/GrafanaGroupLoader.test.tsx:205",
+            "created_at": null,
+            "line": 205,
+            "path": "public/app/features/alerting/unified/rule-list/GrafanaGroupLoader.test.tsx"
+          }
+        ],
+        "tool": "shipwright"
       }
     ]
   },
@@ -39992,6 +40629,25 @@
             "created_at": "2026-06-28T23:15:44Z"
           }
         ]
+      },
+      {
+        "pr_url": "https://github.com/grafana/grafana/pull/107534",
+        "repo_name": "shipwright__https://github.com/grafana/grafana/pull/107534",
+        "review_comments": [
+          {
+            "body": "runShardSplitQuery now interpolates each query up-front via `.map((query) => datasource.applyTemplateVariables(query, request.scopedVars, request.filters))`, replacing the previous single batched `interpolateVariablesInQueries` call. The accompanying test change confirms this causes `applyTemplateVariables` to be invoked 5 times instead of once (`expect(datasource.applyTemplateVariables).toHaveBeenCalledTimes(5)`), meaning the same query is now redundantly re-interpolated once at the top level and again per shard group downstream. This duplicate interpolation wastes computation and risks inconsistent results if the per-shard-group interpolation call uses different scopedVars/filters than the top-level call, since both write into the same query object. Location: public/app/plugins/datasource/loki/shardQuerySplitting.ts:49",
+            "created_at": null,
+            "line": 49,
+            "path": "public/app/plugins/datasource/loki/shardQuerySplitting.ts"
+          },
+          {
+            "body": "The new top-level interpolation (`applyTemplateVariables`) is applied to every split-eligible query before the time range or shards are computed, resolving macros/scoped vars against the request's overall scopedVars. Since query splitting exists specifically to divide a single request into multiple sub-requests over time/shard partitions, resolving variables once up-front (rather than per generated sub-query) risks baking a single global interpolation into every emitted sub-query, whereas the previous batch call semantics and downstream per-chunk processing assumed interpolation could still be adjusted per partition. Location: public/app/plugins/datasource/loki/querySplitting.ts:296",
+            "created_at": null,
+            "line": 296,
+            "path": "public/app/plugins/datasource/loki/querySplitting.ts"
+          }
+        ],
+        "tool": "shipwright"
       }
     ]
   },
@@ -41466,6 +42122,25 @@
             "created_at": "2026-06-28T22:20:35Z"
           }
         ]
+      },
+      {
+        "pr_url": "https://github.com/grafana/grafana/pull/79265",
+        "repo_name": "shipwright__https://github.com/grafana/grafana/pull/79265",
+        "review_comments": [
+          {
+            "body": "CreateOrUpdateDevice now does a check-then-act (CountDevices followed by a separate insert/update) with no transaction or locking. Concurrent anonymous requests can all read count < deviceLimit before any of them inserts, allowing the device count to exceed the configured deviceLimit (TOCTOU race), defeating the new device-limit enforcement under load. Location: pkg/services/anonymous/anonimpl/anonstore/database.go:90",
+            "created_at": null,
+            "line": 90,
+            "path": "pkg/services/anonymous/anonimpl/anonstore/database.go"
+          },
+          {
+            "body": "Anonymous.Authenticate previously tagged devices in a background goroutine with its own context.Background()+2min timeout and a recover() to swallow panics, decoupling tagging from the request lifecycle. The diff removes the goroutine, timeout, and panic recovery, making TagDevice run synchronously inline using the request's ctx. This means (1) if the client cancels/the request context is done before the DB write completes, the device tag/update can be aborted mid-flight instead of completing in the background as before, and (2) any panic inside TagDevice/tagDeviceUI now propagates up through Authenticate with no recover(), instead of being caught and logged as previously. Location: pkg/services/anonymous/anonimpl/client.go:44",
+            "created_at": null,
+            "line": 44,
+            "path": "pkg/services/anonymous/anonimpl/client.go"
+          }
+        ],
+        "tool": "shipwright"
       }
     ]
   },
@@ -42570,6 +43245,19 @@
             "created_at": "2026-06-28T23:57:17Z"
           }
         ]
+      },
+      {
+        "pr_url": "https://github.com/discourse/discourse/commit/ecfa17b5a79dfdc91e7a4d50b42ae78a35d0a293",
+        "repo_name": "shipwright__https://github.com/discourse/discourse/commit/ecfa17b5a79dfdc91e7a4d50b42ae78a35d0a293",
+        "review_comments": [
+          {
+            "body": "In config/initializers/i18n.rb, FallbackLocaleList#ensure_loaded! calls `I18n.ensure_loaded! l`, but the new `ensure_loaded!(locale)` method is defined on the I18n backend instance (lib/freedom_patches/translate_accelerator.rb), not on the top-level I18n module. The I18n module does not expose or delegate an `ensure_loaded!` class method, so this call raises NoMethodError. Since ApplicationController now calls `I18n.fallbacks.ensure_loaded!` on every request (in the locale-setting method), this breaks every request that goes through that code path. The call should be `I18n.backend.ensure_loaded! l`. Location: config/initializers/i18n.rb:22",
+            "created_at": null,
+            "line": 22,
+            "path": "config/initializers/i18n.rb"
+          }
+        ],
+        "tool": "shipwright"
       }
     ]
   },
@@ -43733,6 +44421,25 @@
             "created_at": "2026-06-28T22:55:56Z"
           }
         ]
+      },
+      {
+        "pr_url": "https://github.com/grafana/grafana/pull/76186",
+        "repo_name": "shipwright__https://github.com/grafana/grafana/pull/76186",
+        "review_comments": [
+          {
+            "body": "TestLogger.FromContext (pkg/plugins/log/fake.go) returns a brand-new NewTestLogger() instance instead of the receiver, discarding all call-tracking state. Since LoggerMiddleware.logRequest now calls `m.logger.FromContext(ctx).Info(...)` instead of `m.logger.Info(...)`, any test using the fake TestLogger to assert on InfoLogs/DebugLogs/etc. after invoking the middleware will silently see no recorded calls, since the logs are recorded on a throwaway logger rather than the original fake instance. Location: pkg/plugins/log/fake.go:46",
+            "created_at": null,
+            "line": 46,
+            "path": "pkg/plugins/log/fake.go"
+          },
+          {
+            "body": "The middleware file was renamed from instrumentation_middleware_test.go to metrics_middleware_test.go and the internal call was updated to newMetricsMiddleware, but the test function itself is still named TestInstrumentationMiddleware, leaving a stale/misleading test name that no longer matches the renamed middleware or file. Location: pkg/services/pluginsintegration/clientmiddleware/metrics_middleware_test.go:75",
+            "created_at": null,
+            "line": 75,
+            "path": "pkg/services/pluginsintegration/clientmiddleware/metrics_middleware_test.go"
+          }
+        ],
+        "tool": "shipwright"
       }
     ]
   },
@@ -45852,6 +46559,49 @@
             "created_at": "2026-06-28T23:54:24Z"
           }
         ]
+      },
+      {
+        "pr_url": "https://github.com/discourse/discourse/commit/d1c69189f3c90ecf56013a8da904da9bff9a8e19",
+        "repo_name": "shipwright__https://github.com/discourse/discourse/commit/d1c69189f3c90ecf56013a8da904da9bff9a8e19",
+        "review_comments": [
+          {
+            "body": "The migration runs `execute(\"SELECT c.id FROM categories AS c INNER JOIN site_settings AS s ON s.value = c.name WHERE s.name = 'embed_category'\")[0]['id'].to_i`. If no site_settings row exists for 'embed_category' (the common case, since it defaults to '' and is often never overridden), the query returns an empty result set, `[0]` is `nil`, and calling `['id']` on `nil` raises `NoMethodError`, crashing the migration for most existing installs. Location: db/migrate/20150818190757_create_embeddable_hosts.rb:10",
+            "created_at": null,
+            "line": 10,
+            "path": "db/migrate/20150818190757_create_embeddable_hosts.rb"
+          },
+          {
+            "body": "Migration inserts host values into SQL via string interpolation (`VALUES ('#{h}', ...)`) instead of parameter binding/quoting, which is unsafe if `h` (pulled from the old `embeddable_hosts` site setting) contains a single quote or other SQL metacharacters. Location: db/migrate/20150818190757_create_embeddable_hosts.rb:25",
+            "created_at": null,
+            "line": 25,
+            "path": "db/migrate/20150818190757_create_embeddable_hosts.rb"
+          },
+          {
+            "body": "`before_validation` unconditionally calls `self.host.sub!(...)` twice. If `host` is nil (e.g. missing/blank param), this raises `NoMethodError` instead of surfacing a normal ActiveRecord validation error, crashing create/update instead of returning a friendly error. Location: app/models/embeddable_host.rb:5",
+            "created_at": null,
+            "line": 5,
+            "path": "app/models/embeddable_host.rb"
+          },
+          {
+            "body": "`record_for_host` compares `lower(host) = ?` against `host = uri.host`, which is not downcased before being passed as the bind parameter. A referrer/URL with mixed-case hostname (e.g. `http://EvilTrout.com`) will fail to match the lowercased DB column, causing legitimate allowed hosts to be rejected by `host_allowed?`. Location: app/models/embeddable_host.rb:16",
+            "created_at": null,
+            "line": 16,
+            "path": "app/models/embeddable_host.rb"
+          },
+          {
+            "body": "`update` and `destroy` fetch the record with `EmbeddableHost.where(id: params[:id]).first` (which returns nil if not found) and then unconditionally call `save_host(host)` / `host.destroy` on it, causing a `NoMethodError` instead of a proper 404/error response when the id doesn't exist. Location: app/controllers/admin/embeddable_hosts_controller.rb:8",
+            "created_at": null,
+            "line": 8,
+            "path": "app/controllers/admin/embeddable_hosts_controller.rb"
+          },
+          {
+            "body": "The contents of `spec/fabricators/category_fabricator.rb` and the new `spec/fabricators/embeddable_host_fabricator.rb` were swapped: `category_fabricator.rb` now only defines `Fabricator(:embeddable_host)`, while the new `embeddable_host_fabricator.rb` contains the `:category`/`:diff_category`/`:happy_category`/`:private_category` fabricator definitions that used to live in `category_fabricator.rb`. Location: spec/fabricators/category_fabricator.rb:1",
+            "created_at": null,
+            "line": 1,
+            "path": "spec/fabricators/category_fabricator.rb"
+          }
+        ],
+        "tool": "shipwright"
       }
     ]
   },
@@ -47379,6 +48129,43 @@
             "created_at": "2026-06-28T23:37:06Z"
           }
         ]
+      },
+      {
+        "pr_url": "https://github.com/discourse/discourse/commit/d38c4d5f7443223c81529c22470293771baf9f38",
+        "repo_name": "shipwright__https://github.com/discourse/discourse/commit/d38c4d5f7443223c81529c22470293771baf9f38",
+        "review_comments": [
+          {
+            "body": "When converting to dark-light-choose, the light-theme (primary) lightness value was changed from 30% to 70% instead of being preserved, unlike every other conversion in this diff which keeps the original primary lightness for the light theme and uses (100-x)% for the secondary/dark value. This alters the light-theme link color unintentionally. Location: app/assets/stylesheets/desktop/topic-post.scss:291",
+            "created_at": null,
+            "line": 291,
+            "path": "app/assets/stylesheets/desktop/topic-post.scss"
+          },
+          {
+            "body": "The primary/secondary lightness values were swapped when converting to dark-light-choose: light theme now uses 30% instead of the original 70%, and dark theme uses 70% instead of the expected complementary 30%. This inverts the intended light/dark appearance instead of preserving the light theme and only adding a dark-theme override. Location: app/assets/stylesheets/mobile/modal.scss:101",
+            "created_at": null,
+            "line": 101,
+            "path": "app/assets/stylesheets/mobile/modal.scss"
+          },
+          {
+            "body": "The original primary lightness of 30% was replaced with 50% for both the light and dark arguments of dark-light-choose, instead of preserving 30% for light and using the complementary 70% for dark (the pattern used everywhere else in this diff). This changes the light-theme .name color unintentionally. Location: app/assets/stylesheets/desktop/user.scss:522",
+            "created_at": null,
+            "line": 522,
+            "path": "app/assets/stylesheets/desktop/user.scss"
+          },
+          {
+            "body": "The original primary lightness of 20% for h3 was replaced with 50% for both light and dark arguments instead of preserving 20% for light and 80% for dark, deviating from the conversion pattern used elsewhere and unintentionally altering the light-theme heading color. Location: app/assets/stylesheets/mobile/topic-post.scss:182",
+            "created_at": null,
+            "line": 182,
+            "path": "app/assets/stylesheets/mobile/topic-post.scss"
+          },
+          {
+            "body": "The original primary lightness of 30% for .name was replaced with 50% for both light and dark arguments of dark-light-choose instead of preserving 30% for light and 70% for dark, unintentionally changing the light-theme appearance. Location: app/assets/stylesheets/mobile/user.scss:497",
+            "created_at": null,
+            "line": 497,
+            "path": "app/assets/stylesheets/mobile/user.scss"
+          }
+        ],
+        "tool": "shipwright"
       }
     ]
   },
@@ -49374,6 +50161,31 @@
             "created_at": "2026-06-28T23:44:35Z"
           }
         ]
+      },
+      {
+        "pr_url": "https://github.com/discourse/discourse/commit/060cda77729cb1c4a827560e09e89a7b22078ba9",
+        "repo_name": "shipwright__https://github.com/discourse/discourse/commit/060cda77729cb1c4a827560e09e89a7b22078ba9",
+        "review_comments": [
+          {
+            "body": "routes.rb defines `delete \"members\" => \"groups#remove_member\"` and `put \"members\" => \"groups#add_members\"` as member routes nested under `resources :groups` (matching the URL pattern `/admin/groups/:id/members`, same style as the removed `get \"users\"`), so Rails will populate `params[:id]` with the group id. However `Admin::GroupsController#add_members` and `#remove_member` both call `params.require(:group_id)` to look up the group, which will never be present on a real request generated by the JS client (`Discourse.ajax('/admin/groups/' + id + '/members.json', ...)` in `group.js`). This raises `ActionController::ParameterMissing` and breaks the add/remove-member feature the PR claims to fix. The specs mask this by invoking the actions directly with an explicit `group_id:` param instead of going through routing, so the bug isn't caught by the added tests. Location: app/controllers/admin/groups_controller.rb:66",
+            "created_at": null,
+            "line": 66,
+            "path": "app/controllers/admin/groups_controller.rb"
+          },
+          {
+            "body": "`totalPages` is computed as `Math.floor(user_count / limit) + 1`, which is off-by-one whenever `user_count` is an exact multiple of `limit` (e.g. user_count=50, limit=50 gives totalPages=2 instead of 1). This causes `showingLast` to stay false one page too early, so the \"next\" pagination control remains enabled past the real last page; clicking it sets `offset` to `user_count` and fetches an empty page, showing an extra blank page in the member list. Location: app/assets/javascripts/admin/controllers/admin-group.js.es6:13",
+            "created_at": null,
+            "line": 13,
+            "path": "app/assets/javascripts/admin/controllers/admin-group.js.es6"
+          },
+          {
+            "body": "In the new `group_member.hbs` itemView template, `{{#unless automatic}}` is evaluated against the item view's context, which is the individual `member` (a Discourse.User) because it's rendered via `{{each member in members itemView=\"group-member\"}}` in `group.hbs`, not the parent Group model's `automatic` property. Since `member.automatic` is undefined, the `unless` block always renders, so the remove-member link is shown even for members of automatic groups (server-side still rejects via `can_not_modify_automatic`, but the UI incorrectly offers an action that will always fail). Location: app/assets/javascripts/admin/templates/group_member.hbs:1",
+            "created_at": null,
+            "line": 1,
+            "path": "app/assets/javascripts/admin/templates/group_member.hbs"
+          }
+        ],
+        "tool": "shipwright"
       }
     ]
   },
@@ -50865,6 +51677,25 @@
             "created_at": "2026-06-28T22:47:11Z"
           }
         ]
+      },
+      {
+        "pr_url": "https://github.com/discourse/discourse/commit/5f8a130277dbddc95d133cd2832be639baf89213",
+        "repo_name": "shipwright__https://github.com/discourse/discourse/commit/5f8a130277dbddc95d133cd2832be639baf89213",
+        "review_comments": [
+          {
+            "body": "BlockedEmail.should_block? performs an exact string match (`BlockedEmail.where(email: email).first`) without normalizing case. Since emails are case-insensitive by RFC and neither BlockedEmail nor the validator lowercases the value before lookup, a blocked email like 'bad@spamclub.com' can bypass the block simply by submitting 'Bad@SpamClub.com', defeating the entire purpose of this new blocking feature. Location: app/models/blocked_email.rb:11",
+            "created_at": null,
+            "line": 11,
+            "path": "app/models/blocked_email.rb"
+          },
+          {
+            "body": "`rejectedEmails: Em.A([])` is declared directly as a property value on the Ember.Object.extend() class definition rather than being initialized in an `init` hook. In Ember's classic object model, array/object literals defined this way are shared across all instances created from this class definition, so state from one instance (e.g. a previously rejected email) can leak into or persist across other instances of CreateAccountController. Location: app/assets/javascripts/discourse/controllers/create_account_controller.js:17",
+            "created_at": null,
+            "line": 17,
+            "path": "app/assets/javascripts/discourse/controllers/create_account_controller.js"
+          }
+        ],
+        "tool": "shipwright"
       }
     ]
   },
@@ -52089,6 +52920,25 @@
             "created_at": "2026-06-28T23:08:26Z"
           }
         ]
+      },
+      {
+        "pr_url": "https://github.com/discourse/discourse/commit/5b229316ee4c661836ed1161139692a3e8527444",
+        "repo_name": "shipwright__https://github.com/discourse/discourse/commit/5b229316ee4c661836ed1161139692a3e8527444",
+        "review_comments": [
+          {
+            "body": "The `.title` rule (previously `float: left;`) was deleted entirely, and flexbox properties (`display: flex; align-items: center;`) were instead added to `.contents`. Based on the original indentation, `.title` was a sibling selector of `.contents` (same 4-space indent level), not a child of it. Removing `.title`'s `float: left` while making `.contents` (a different, sibling element) a flex container does nothing for `.title`'s layout — the title element loses its horizontal positioning entirely, breaking the header layout. Location: app/assets/stylesheets/common/base/header.scss:16",
+            "created_at": null,
+            "line": 16,
+            "path": "app/assets/stylesheets/common/base/header.scss"
+          },
+          {
+            "body": "`.panel` had `float: right;` removed and replaced with `margin-left: auto;` plus `@include order(3);`. These properties only have an effect when the element is a flex *item* inside a flex container. `.panel` is a sibling of `.contents` (same indentation level as `.contents`/`.valign-helper`), not nested inside it, and no flex container is declared for their shared parent in this diff. As a result `.panel` loses its previous right-floating alignment with no functional replacement, breaking the header's right-side layout (login/sign-up buttons, panel icons, etc.). Location: app/assets/stylesheets/common/base/header.scss:36",
+            "created_at": null,
+            "line": 36,
+            "path": "app/assets/stylesheets/common/base/header.scss"
+          }
+        ],
+        "tool": "shipwright"
       }
     ]
   },
@@ -53315,6 +54165,31 @@
             "created_at": "2026-06-28T23:23:17Z"
           }
         ]
+      },
+      {
+        "pr_url": "https://github.com/discourse/discourse/commit/267d8be1f556ed59639ced396c885bb44586da19",
+        "repo_name": "shipwright__https://github.com/discourse/discourse/commit/267d8be1f556ed59639ced396c885bb44586da19",
+        "review_comments": [
+          {
+            "body": "`include_website_name` is missing the trailing `?` required by Discourse's ActiveModel::Serializer convention for conditional-attribute inclusion (see how other `include_<attr>?` methods work). Because the method is not recognized as a presence guard, `website_name` will always be serialized regardless of whether the user has a website set, contradicting the intended `website.present?` check and returning a bogus `website_name: nil` for users without a website. Location: app/serializers/user_serializer.rb:152",
+            "created_at": null,
+            "line": 152,
+            "path": "app/serializers/user_serializer.rb"
+          },
+          {
+            "body": "The new `website_name` domain-comparison logic compares hostnames (`website_host`, `discourse_host`) directly without normalizing case (no `.downcase`). Hostnames are case-insensitive per RFC, so a website URL with a mixed-case host (e.g. `http://Example.com/user`) will fail the `website_host == discourse_host` and subdomain-suffix comparisons even when it is actually the same domain as `Discourse.current_hostname`, causing the full path to not be shown when it should be. Location: app/serializers/user_serializer.rb:140",
+            "created_at": null,
+            "line": 140,
+            "path": "app/serializers/user_serializer.rb"
+          },
+          {
+            "body": "The subdomain-matching branch strips only the first label of each host and compares the remaining suffix (`split('.')[1..-1].join('.')`) without validating that this remainder is a real registrable domain. For hosts under shared multi-label public suffixes (e.g. `foo.co.uk` vs `bar.co.uk`), the stripped suffix `co.uk` will match even though the two hosts are unrelated organizations, incorrectly causing the full website path to be exposed as if the website and instance shared the same domain. Location: app/serializers/user_serializer.rb:143",
+            "created_at": null,
+            "line": 143,
+            "path": "app/serializers/user_serializer.rb"
+          }
+        ],
+        "tool": "shipwright"
       }
     ]
   },
@@ -55740,6 +56615,43 @@
             "created_at": "2026-06-28T23:03:36Z"
           }
         ]
+      },
+      {
+        "pr_url": "https://github.com/discourse/discourse/commit/4f8aed295a29954023b2849c060ef4fb299d1b5d",
+        "repo_name": "shipwright__https://github.com/discourse/discourse/commit/4f8aed295a29954023b2849c060ef4fb299d1b5d",
+        "review_comments": [
+          {
+            "body": "When cook_method is raw_html, Post#cook now returns `raw` directly and skips post_analyzer.cook/Plugin::Filter.apply entirely (no sanitization pipeline runs). Jobs::PollFeed.poll_feed feeds raw external RSS/Atom HTML content (`CGI.unescapeHTML(i.content.scrub)`) straight into TopicEmbed.import -> PostCreator with cook_method: raw_html, and the embed view (app/views/embed/best.html.erb) renders `post.cooked` with `raw`. This creates a stored XSS vector: arbitrary/untrusted RSS feed content is rendered unsanitized as raw HTML in the embed iframe. Location: app/models/post.rb:131",
+            "created_at": null,
+            "line": 131,
+            "path": "app/models/post.rb"
+          },
+          {
+            "body": "postMessage origin validation uses substring containment (`discourseUrl.indexOf(e.origin) === -1`) instead of a strict origin equality/prefix check. This is a fragile/incorrect security check: it tests whether the message's origin string appears anywhere inside discourseUrl rather than validating that the message actually originates from discourseUrl, which is the wrong direction for origin validation and can silently accept or reject messages incorrectly compared to a proper equality check. Location: app/assets/javascripts/embed.js:15",
+            "created_at": null,
+            "line": 15,
+            "path": "app/assets/javascripts/embed.js"
+          },
+          {
+            "body": "The Disqus import thor task was refactored to call `TopicEmbed.import_remote(user, t[:link], title: t[:title])` instead of `PostCreator.new(user, title:, raw:, created_at:, category:)`. This drops the previously supported `--category`/`-c` CLI option (and its `category_id` lookup) entirely, and no longer passes `created_at: Date.parse(t[:created_at])`, so imported topics lose both category placement and their original creation date — a functional regression compared to the prior implementation. Location: lib/tasks/disqus.thor:140",
+            "created_at": null,
+            "line": 140,
+            "path": "lib/tasks/disqus.thor"
+          },
+          {
+            "body": "An already-existing migration (`create_table :top_topics`) was modified to add `force: true`. Since this file's timestamp/version predates the newly added migrations in this PR, any environment that already applied this migration and has data in `top_topics` risks that table being dropped and recreated if the migration is ever re-run (e.g., via db:migrate:redo or manual re-application), causing data loss. Existing migrations that have already run should not be edited to add destructive options. Location: db/migrate/20131223171005_create_top_topics.rb:3",
+            "created_at": null,
+            "line": 3,
+            "path": "db/migrate/20131223171005_create_top_topics.rb"
+          },
+          {
+            "body": "EmbedController#ensure_embeddable compares `URI(request.referer || '').host` to `SiteSetting.embeddable_host` with a case-sensitive `!=` check and no normalization on either side. If an admin configures embeddable_host with different casing than the browser-supplied referer host (or vice versa), legitimate embed requests will be rejected as InvalidAccess, and conversely there is no guarantee both sides are normalized consistently for comparison. Location: app/controllers/embed_controller.rb:29",
+            "created_at": null,
+            "line": 29,
+            "path": "app/controllers/embed_controller.rb"
+          }
+        ],
+        "tool": "shipwright"
       }
     ]
   },
@@ -57329,6 +58241,37 @@
             "created_at": "2026-06-28T22:28:25Z"
           }
         ]
+      },
+      {
+        "pr_url": "https://github.com/discourse/discourse/commit/ffbaf8c54269df2ce510de91245760fddce09896",
+        "repo_name": "shipwright__https://github.com/discourse/discourse/commit/ffbaf8c54269df2ce510de91245760fddce09896",
+        "review_comments": [
+          {
+            "body": "Client-side file size validation now uses a hardcoded 10*1024 KB (10MB) constant instead of the configurable `Discourse.SiteSettings['max_' + type + '_size_kb']` / `Discourse.SiteSettings.max_image_size_kb`. This breaks per-type size limits (e.g. avatars, attachments) and will diverge from the server-side `SiteSetting.max_image_size_kb` used elsewhere in this same PR, causing inconsistent validation between client and server. Location: app/assets/javascripts/discourse/lib/utilities.js:182",
+            "created_at": null,
+            "line": 182,
+            "path": "app/assets/javascripts/discourse/lib/utilities.js"
+          },
+          {
+            "body": "The API upload URL download cap was changed from `SiteSetting.max_image_size_kb.kilobytes` to a hardcoded `10.megabytes`. This contradicts the PR's stated goal of allowing larger images to be downloaded and then downsized: any image larger than 10MB fetched via URL will now fail to download entirely (silently rescued to nil) before ever reaching the new downsize logic, and the cap no longer respects site configuration. Location: app/controllers/uploads_controller.rb:55",
+            "created_at": null,
+            "line": 55,
+            "path": "app/controllers/uploads_controller.rb"
+          },
+          {
+            "body": "`OptimizedImage.downsize` is now defined twice with different arities (`downsize(from, to, max_width, max_height, opts={})` and `downsize(from, to, dimensions, opts={})`). Ruby silently keeps only the second definition, making the first a dead/unreachable method whose original 4-positional-argument signature (from the pre-diff API) is gone. Any code still calling `downsize` with separate width/height arguments will now raise an ArgumentError. Location: app/models/optimized_image.rb:146",
+            "created_at": null,
+            "line": 146,
+            "path": "app/models/optimized_image.rb"
+          },
+          {
+            "body": "The new downsize retry loop in the upload flow gives up silently after 5 attempts with no check on whether the image was actually reduced below `SiteSetting.max_image_size_kb`. If `OptimizedImage.downsize` fails or converges slowly, the oversized tempfile is passed straight to `Upload.create_for` anyway, defeating the max image size validation instead of enforcing it. Location: app/controllers/uploads_controller.rb:63",
+            "created_at": null,
+            "line": 63,
+            "path": "app/controllers/uploads_controller.rb"
+          }
+        ],
+        "tool": "shipwright"
       }
     ]
   },
@@ -58860,6 +59803,37 @@
             "created_at": "2026-06-28T22:28:38Z"
           }
         ]
+      },
+      {
+        "pr_url": "https://github.com/discourse/discourse/commit/6669a2d94d76eea3b99b8c476d12b1eb66726b07",
+        "repo_name": "shipwright__https://github.com/discourse/discourse/commit/6669a2d94d76eea3b99b8c476d12b1eb66726b07",
+        "review_comments": [
+          {
+            "body": "`:unsubscribe` was added to the `except:` list of `before_filter :ensure_logged_in`, so the new `unsubscribe` action can now be reached without a logged-in user. However, the action body unconditionally calls `TopicUser.find_by(user_id: current_user.id, ...)`, which raises `NoMethodError: undefined method 'id' for nil:NilClass` when `current_user` is nil (e.g. when the emailed unsubscribe link is opened in a logged-out browser). Location: app/controllers/topics_controller.rb:27",
+            "created_at": null,
+            "line": 27,
+            "path": "app/controllers/topics_controller.rb"
+          },
+          {
+            "body": "`tu = TopicUser.find_by(user_id: current_user.id, topic_id: params[:topic_id])` can return nil (no TopicUser row exists for that user/topic), but the following code immediately calls `tu.notification_level` and `tu.save!` without a nil guard, causing a NoMethodError crash instead of gracefully creating/handling the missing preference row. Location: app/controllers/topics_controller.rb:106",
+            "created_at": null,
+            "line": 106,
+            "path": "app/controllers/topics_controller.rb"
+          },
+          {
+            "body": "The unsubscribe toggle logic only downgrades a user's notification_level to `regular` when it is above `regular` (e.g. tracking/watching), and only sets it to `muted` when it is already at or below `regular`. This contradicts the new copy shown to the user (\"You will stop receiving notifications for <title>\", config/locales/client.en.yml `topic.unsubscribe.stop_notifications`), since a watching/tracking user clicking the unsubscribe link will still be left at `regular` and can continue to receive notifications, not actually unsubscribed as promised. Location: app/controllers/topics_controller.rb:108",
+            "created_at": null,
+            "line": 108,
+            "path": "app/controllers/topics_controller.rb"
+          },
+          {
+            "body": "The new controller property is misspelled `stopNotificiationsText` (should be `stopNotificationsText`); while it still works because the template references the same misspelled name, it is a misleading/incorrect identifier introduced by this PR. Location: app/assets/javascripts/discourse/controllers/topic-unsubscribe.js.es6:5",
+            "created_at": null,
+            "line": 5,
+            "path": "app/assets/javascripts/discourse/controllers/topic-unsubscribe.js.es6"
+          }
+        ],
+        "tool": "shipwright"
       }
     ]
   },
@@ -60473,6 +61447,25 @@
             "created_at": "2026-06-28T23:46:15Z"
           }
         ]
+      },
+      {
+        "pr_url": "https://github.com/calcom/cal.com/pull/22532",
+        "repo_name": "shipwright__https://github.com/calcom/cal.com/pull/22532",
+        "review_comments": [
+          {
+            "body": "The `actions` prop in both AppListCard renders now unconditionally wrap `CredentialActionsDropdown` in a `<div className=\"flex w-32 justify-end\">`, whereas previously the whole block (including the wrapping div) was only rendered when `!delegationCredentialId && !disableConnectionModification`. Since `CredentialActionsDropdown` returns `null` when `!canDisconnect && !hasCache` (e.g., delegation credentials or calendars without cache), the surrounding div still renders, leaving an empty 8rem-wide, right-aligned box where nothing used to appear at all — a visible UI regression introduced by this diff. Location: packages/platform/atoms/selected-calendars/wrappers/SelectedCalendarsSettingsWebWrapper.tsx:70",
+            "created_at": null,
+            "line": 70,
+            "path": "packages/platform/atoms/selected-calendars/wrappers/SelectedCalendarsSettingsWebWrapper.tsx"
+          },
+          {
+            "body": "deleteCacheHandler throws a plain `Error(\"Credential not found or access denied\")` instead of a `TRPCError` with an appropriate code (e.g. NOT_FOUND/FORBIDDEN). In a tRPC mutation this will surface as a generic INTERNAL_SERVER_ERROR rather than a distinguishable client error, making it impossible for callers/monitoring to differentiate an authorization failure from an actual server fault. Location: packages/trpc/server/routers/viewer/calendars/deleteCache.handler.ts:21",
+            "created_at": null,
+            "line": 21,
+            "path": "packages/trpc/server/routers/viewer/calendars/deleteCache.handler.ts"
+          }
+        ],
+        "tool": "shipwright"
       }
     ]
   },
@@ -61993,6 +62986,25 @@
             "created_at": "2026-06-28T23:11:51Z"
           }
         ]
+      },
+      {
+        "pr_url": "https://github.com/calcom/cal.com/pull/8330",
+        "repo_name": "shipwright__https://github.com/calcom/cal.com/pull/8330",
+        "review_comments": [
+          {
+            "body": "In the new working-hours check, `end` is computed using `slotStartTime` instead of `slotEndTime` (`const end = slotStartTime.hour() * 60 + slotStartTime.minute();`), so the code never actually validates that the slot's end time falls within the organizer's working hours. A slot that starts inside working hours but extends past closing time will incorrectly be treated as available. Location: packages/trpc/server/routers/viewer/slots.ts:140",
+            "created_at": null,
+            "line": 140,
+            "path": "packages/trpc/server/routers/viewer/slots.ts"
+          },
+          {
+            "body": "The date-override equality check `dayjs(date.start).add(utcOffset, \"minutes\") === dayjs(date.end).add(utcOffset, \"minutes\")` compares two freshly-created dayjs object instances with `===`, which is always false since they are different object references (dayjs objects are never referentially equal even for identical values). This branch, meant to detect zero-duration/whole-day overrides, can never execute, silently breaking the intended handling of that case. Location: packages/trpc/server/routers/viewer/slots.ts:121",
+            "created_at": null,
+            "line": 121,
+            "path": "packages/trpc/server/routers/viewer/slots.ts"
+          }
+        ],
+        "tool": "shipwright"
       }
     ]
   },
@@ -63231,6 +64243,25 @@
             "created_at": "2026-06-28T23:31:37Z"
           }
         ]
+      },
+      {
+        "pr_url": "https://github.com/calcom/cal.com/pull/14943",
+        "repo_name": "shipwright__https://github.com/calcom/cal.com/pull/14943",
+        "review_comments": [
+          {
+            "body": "The new OR clause in deleteMany omits the `method: WorkflowMethods.SMS` (and scheduledDate) constraint that the first clause has, so any WorkflowReminder row (not just SMS reminders, and regardless of scheduledDate) with retryCount > 1 will be deleted. Since retryCount is a generic column added to WorkflowReminder, this can delete non-SMS reminders (email/whatsapp) that happen to have retryCount incremented elsewhere, unintentionally purging reminders unrelated to the SMS retry logic this PR is meant to implement. Location: packages/features/ee/workflows/api/scheduleSMSReminders.ts:34",
+            "created_at": null,
+            "line": 34,
+            "path": "packages/features/ee/workflows/api/scheduleSMSReminders.ts"
+          },
+          {
+            "body": "retryCount is incremented via a non-atomic read-modify-write (`retryCount: reminder.retryCount + 1`) using a value fetched earlier in the same handler invocation, rather than Prisma's atomic `{ increment: 1 }`. If this cron/handler runs concurrently (e.g., overlapping invocations processing the same reminder), updates can race and increments can be lost, undermining the retry-count-based cleanup this PR introduces. Location: packages/features/ee/workflows/api/scheduleSMSReminders.ts:181",
+            "created_at": null,
+            "line": 181,
+            "path": "packages/features/ee/workflows/api/scheduleSMSReminders.ts"
+          }
+        ],
+        "tool": "shipwright"
       }
     ]
   },
@@ -64407,6 +65438,12 @@
             "created_at": "2026-06-28T22:40:20Z"
           }
         ]
+      },
+      {
+        "pr_url": "https://github.com/calcom/cal.com/pull/22345",
+        "repo_name": "shipwright__https://github.com/calcom/cal.com/pull/22345",
+        "review_comments": [],
+        "tool": "shipwright"
       }
     ]
   },
@@ -66706,6 +67743,43 @@
             "created_at": "2026-06-28T23:17:49Z"
           }
         ]
+      },
+      {
+        "pr_url": "https://github.com/calcom/cal.com/pull/11059",
+        "repo_name": "shipwright__https://github.com/calcom/cal.com/pull/11059",
+        "review_comments": [
+          {
+            "body": "parseRefreshTokenResponse's return value (a safeParse-style {success, data} object) is now stored directly as the credential `key` instead of the actual parsed token data: `const key = parseRefreshTokenResponse(googleCredentials, googleCredentialSchema); ... data: { key }`. Previously `googleCredentialSchema.parse(...)` returned the flat token object; now the credential.key JSON will be `{success, data}` wrapper, corrupting the stored Google Calendar OAuth credentials and breaking subsequent calendar API calls. Location: packages/app-store/googlecalendar/lib/CalendarService.ts:97",
+            "created_at": null,
+            "line": 97,
+            "path": "packages/app-store/googlecalendar/lib/CalendarService.ts"
+          },
+          {
+            "body": "When the refreshed token response lacks a `refresh_token` field, the code sets it to the literal string \"refresh_token\" instead of preserving the previous refresh token or leaving it undefined: `refreshTokenResponse.data.refresh_token = \"refresh_token\";`. This overwrites/corrupts the stored refresh token with an unusable placeholder value, breaking future token refresh attempts for any provider that omits refresh_token on refresh (a common OAuth behavior). Location: packages/app-store/_utils/oauth/parseRefreshTokenResponse.ts:24",
+            "created_at": null,
+            "line": 24,
+            "path": "packages/app-store/_utils/oauth/parseRefreshTokenResponse.ts"
+          },
+          {
+            "body": "The computed property keys `[z.string().toString()]` and `[z.string().optional().toString()]` do not create dynamic/wildcard validators as the comments claim; `.toString()` is called on the Zod schema object (not a string), producing a fixed literal key (e.g. \"[object Object]\") for both entries, with the second silently overwriting the first. As a result, `minimumTokenResponseSchema` only actually validates `access_token`, and all other real response fields (expiry, scope, refresh_token, etc.) are stripped out of `.data` by zod's default unknown-key stripping when APP_CREDENTIAL_SHARING_ENABLED is on, causing loss of critical token metadata. Location: packages/app-store/_utils/oauth/parseRefreshTokenResponse.ts:9",
+            "created_at": null,
+            "line": 9,
+            "path": "packages/app-store/_utils/oauth/parseRefreshTokenResponse.ts"
+          },
+          {
+            "body": "`refreshOAuthTokens` returns a raw, unparsed `fetch` Response when credential sync is enabled, but returns provider-specific parsed shapes in the local-refresh branch (e.g. `fetchTokens.res` with a `.data` property for Google, or the `HubspotToken` object for Hubspot). Callers such as GoogleCalendarService (`const token = res?.data;`) and HubspotCalendarService (typed as `HubspotToken`) assume a single shape and will fail/throw at runtime (e.g. `token.access_token` on undefined) when APP_CREDENTIAL_SHARING_ENABLED and CALCOM_CREDENTIAL_SYNC_ENDPOINT are set. Location: packages/app-store/_utils/oauth/refreshOAuthTokens.ts:6",
+            "created_at": null,
+            "line": 6,
+            "path": "packages/app-store/_utils/oauth/refreshOAuthTokens.ts"
+          },
+          {
+            "body": "The webhook secret is compared using strict inequality (`!==`) rather than a constant-time comparison, making the credential-sync webhook endpoint vulnerable to timing attacks that could help an attacker guess CALCOM_WEBHOOK_SECRET. Location: apps/web/pages/api/webhook/app-credential.ts:21",
+            "created_at": null,
+            "line": 21,
+            "path": "apps/web/pages/api/webhook/app-credential.ts"
+          }
+        ],
+        "tool": "shipwright"
       }
     ]
   },
@@ -68422,6 +69496,37 @@
             "created_at": "2026-06-28T23:04:20Z"
           }
         ]
+      },
+      {
+        "pr_url": "https://github.com/calcom/cal.com/pull/7232",
+        "repo_name": "shipwright__https://github.com/calcom/cal.com/pull/7232",
+        "review_comments": [
+          {
+            "body": "When `immediateDelete` is true and a `referenceId` exists, `deleteScheduledEmailReminder` only calls the SendGrid cancel API and returns — it never deletes or updates (sets `cancelled: true`) the `WorkflowReminder` row. This function is called with `immediateDelete=true` from `handleNewBooking.ts` (reschedule cleanup) and `workflows.tsx` (step/workflow deletion), so those WorkflowReminder rows become orphaned in the DB with `scheduled: true`/`cancelled: null` forever, even though the underlying email was actually canceled, which can cause `scheduleEmailReminders.ts`'s cancellation cron (which filters on `cancelled: true`) to never clean them up and can confuse any code that treats these rows as still-active/scheduled reminders. Location: packages/features/ee/workflows/lib/reminders/emailReminderManager.ts:213",
+            "created_at": null,
+            "line": 213,
+            "path": "packages/features/ee/workflows/lib/reminders/emailReminderManager.ts"
+          },
+          {
+            "body": "The reschedule mutation dropped the `await Promise.all(remindersToDelete)` call that previously awaited the actual deletion of old WorkflowReminder rows. The new code only fires `deleteScheduledEmailReminder`/`deleteScheduledSMSReminder` inside a `forEach` without awaiting them, so the mutation proceeds to create the new booking and schedule new workflow reminders before the old reminders are guaranteed to be cleaned up/canceled, introducing a race condition and swallowing any errors from the cleanup. Location: packages/trpc/server/routers/viewer/bookings.tsx:497",
+            "created_at": null,
+            "line": 497,
+            "path": "packages/trpc/server/routers/viewer/bookings.tsx"
+          },
+          {
+            "body": "The previous `deleteMany` for reminders belonging to disabled event types explicitly scoped deletion to `booking: { userId: ctx.user.id }}`, providing an ownership check. The refactor removes this scoping entirely and instead calls `deleteScheduledEmailReminder`/`deleteScheduledSMSReminder`, whose underlying Prisma calls only filter by `id: reminderId` with no ownership/ownership-booking check, weakening the authorization guard that was present before this diff. Location: packages/trpc/server/routers/viewer/workflows.tsx:378",
+            "created_at": null,
+            "line": 378,
+            "path": "packages/trpc/server/routers/viewer/workflows.tsx"
+          },
+          {
+            "body": "The guard `if (reminder.scheduled && reminder.referenceId)` and the batched `prisma.workflowReminder.deleteMany` (previously part of the awaited `prismaPromises`) were removed. Reminder cleanup now only happens via unawaited, fire-and-forget calls to `deleteScheduledEmailReminder`/`deleteScheduledSMSReminder` inside the `forEach`, so the cancel-booking handler's response can return to the client before the WorkflowReminder rows are actually canceled/deleted, and any errors from that cleanup are silently swallowed instead of surfacing via `Promise.all`. Location: packages/features/bookings/lib/handleCancelBooking.ts:484",
+            "created_at": null,
+            "line": 484,
+            "path": "packages/features/bookings/lib/handleCancelBooking.ts"
+          }
+        ],
+        "tool": "shipwright"
       }
     ]
   },
@@ -70394,6 +71499,43 @@
             "created_at": "2026-06-28T23:39:36Z"
           }
         ]
+      },
+      {
+        "pr_url": "https://github.com/calcom/cal.com/pull/14740",
+        "repo_name": "shipwright__https://github.com/calcom/cal.com/pull/14740",
+        "review_comments": [
+          {
+            "body": "Blacklisted guest emails are lowercased when read from env, but the incoming `guest` email is compared against the blacklist without normalization (`!blacklistedGuestEmails.includes(guest)`). A blacklisted email submitted with different casing (e.g. Foo@Bar.com) will bypass the blacklist filter entirely. Location: packages/trpc/server/routers/viewer/bookings/addGuests.handler.ts:58",
+            "created_at": null,
+            "line": 58,
+            "path": "packages/trpc/server/routers/viewer/bookings/addGuests.handler.ts"
+          },
+          {
+            "body": "Authorization check uses `&&` instead of `||` between isTeamAdmin and isTeamOwner, requiring a user to be BOTH a team admin AND the team owner to be granted access via this path, rather than either role. This incorrectly denies legitimate team admins (who are not also the owner) permission to add guests. Location: packages/trpc/server/routers/viewer/bookings/addGuests.handler.ts:33",
+            "created_at": null,
+            "line": 33,
+            "path": "packages/trpc/server/routers/viewer/bookings/addGuests.handler.ts"
+          },
+          {
+            "body": "`this.calEvent.attendees[0].name` is accessed without verifying the attendees array is non-empty, which will throw if a booking somehow has no attendees when this template is rendered. Location: packages/emails/templates/organizer-add-guests-email.ts:27",
+            "created_at": null,
+            "line": 27,
+            "path": "packages/emails/templates/organizer-add-guests-email.ts"
+          },
+          {
+            "body": "Errors from sendAddGuestsEmails are silently swallowed with a bare console.log instead of using the application logger, hiding email delivery failures from monitoring/observability after guests are added. Location: packages/trpc/server/routers/viewer/bookings/addGuests.handler.ts:168",
+            "created_at": null,
+            "line": 168,
+            "path": "packages/trpc/server/routers/viewer/bookings/addGuests.handler.ts"
+          },
+          {
+            "body": "The onError handler builds `message` as a template string (\"${err.data?.code}: ${t(err.message)}\") which is always truthy, so the fallback `t('unable_to_add_guests')` in `showToast(message || t('unable_to_add_guests'), 'error')` is unreachable dead code and never displays a translated fallback message. Location: apps/web/components/dialog/AddGuestsDialog.tsx:29",
+            "created_at": null,
+            "line": 29,
+            "path": "apps/web/components/dialog/AddGuestsDialog.tsx"
+          }
+        ],
+        "tool": "shipwright"
       }
     ]
   },
@@ -72114,6 +73256,31 @@
             "created_at": "2026-06-28T22:29:53Z"
           }
         ]
+      },
+      {
+        "pr_url": "https://github.com/calcom/cal.com/pull/10600",
+        "repo_name": "shipwright__https://github.com/calcom/cal.com/pull/10600",
+        "review_comments": [
+          {
+            "body": "BackupCode.tsx exports a function named `TwoFactor` (identical to the actual TwoFactor.tsx component) instead of `BackupCode`. This is misleading and could easily be confused with the real TwoFactor component when imported elsewhere, especially since both default-export the same identifier name. Location: apps/web/components/auth/BackupCode.tsx:6",
+            "created_at": null,
+            "line": 6,
+            "path": "apps/web/components/auth/BackupCode.tsx"
+          },
+          {
+            "body": "Backup code verification in the credentials authorize() flow reads the user's encrypted backupCodes, validates the supplied code against an in-memory array, mutates that array (`backupCodes[index] = null`) and only then persists the change via `prisma.user.update`. Two concurrent login attempts using the same backup code will both read the original (unmodified) encrypted array before either write completes, so both requests will find the code valid and succeed, defeating the intended one-time-use semantics of backup codes and allowing bypass/detection avoidance. Location: packages/features/auth/lib/next-auth-options.ts:151",
+            "created_at": null,
+            "line": 151,
+            "path": "packages/features/auth/lib/next-auth-options.ts"
+          },
+          {
+            "body": "A Blob URL is created via `URL.createObjectURL(textBlob)` for the backup-codes download link and stored in `backupCodesUrl`. It is only revoked the next time `handleSetup` runs (`if (backupCodesUrl) URL.revokeObjectURL(backupCodesUrl)`), but `resetState()` — called both when the user cancels the flow and when the user clicks \"Close\" after viewing/downloading the backup codes — never revokes this URL. As a result, every completed 2FA setup leaks one Blob URL for the lifetime of the page. Location: apps/web/components/settings/EnableTwoFactorModal.tsx:100",
+            "created_at": null,
+            "line": 100,
+            "path": "apps/web/components/settings/EnableTwoFactorModal.tsx"
+          }
+        ],
+        "tool": "shipwright"
       }
     ]
   },
@@ -74331,6 +75498,43 @@
             "created_at": "2026-06-28T22:35:43Z"
           }
         ]
+      },
+      {
+        "pr_url": "https://github.com/calcom/cal.com/pull/10967",
+        "repo_name": "shipwright__https://github.com/calcom/cal.com/pull/10967",
+        "review_comments": [
+          {
+            "body": "After destructuring `const [mainHostDestinationCalendar] = evt.destinationCalendar ?? [];`, the code calls `mainHostDestinationCalendar.integration` without any null-check. Since many callers now pass `destinationCalendar: []` (e.g. handleCancelBooking.ts, webhook.ts, paypal-webhook.ts) when there is no destination calendar, `mainHostDestinationCalendar` will be `undefined` and this line throws `TypeError: Cannot read properties of undefined (reading 'integration')` whenever `evt.location === MeetLocationType` and no destination calendar exists. Location: packages/core/EventManager.ts:118",
+            "created_at": null,
+            "line": 118,
+            "path": "packages/core/EventManager.ts"
+          },
+          {
+            "body": "In `updateEvent` the fallback branch computes `event.destinationCalendar?.find((cal) => cal.externalId === externalCalendarId)?.externalId` but this code only runs when `externalCalendarId` is falsy (it's the else-branch of `externalCalendarId ? externalCalendarId : ...`). Comparing `cal.externalId === externalCalendarId` against a falsy value will essentially never match a real calendar entry, so `selectedCalendar` will always resolve to `undefined` instead of falling back to the destination calendar's externalId, breaking calendar update routing. Location: packages/app-store/googlecalendar/lib/CalendarService.ts:255",
+            "created_at": null,
+            "line": 255,
+            "path": "packages/app-store/googlecalendar/lib/CalendarService.ts"
+          },
+          {
+            "body": "Same broken fallback pattern as updateEvent: `deleteEvent` computes `calendarId` via `event.destinationCalendar?.find((cal) => cal.externalId === externalCalendarId)?.externalId` inside the branch where `externalCalendarId` is falsy, so the find predicate compares against a falsy value and will never match, making the destination-calendar fallback for delete always resolve to `undefined`. Location: packages/app-store/googlecalendar/lib/CalendarService.ts:316",
+            "created_at": null,
+            "line": 316,
+            "path": "packages/app-store/googlecalendar/lib/CalendarService.ts"
+          },
+          {
+            "body": "`evt.destinationCalendar` is typed to be `null` when neither `eventType.destinationCalendar` nor `organizerUser.destinationCalendar` exist. The subsequent `evt.destinationCalendar?.push(...teamDestinationCalendars)` uses optional chaining, so when `evt.destinationCalendar` is `null` the push silently no-ops and all collected team member destination calendars are dropped, defeating the PR's stated purpose of supporting multiple hosts' destination calendars for collective events. Location: packages/features/bookings/lib/handleNewBooking.ts:1083",
+            "created_at": null,
+            "line": 1083,
+            "path": "packages/features/bookings/lib/handleNewBooking.ts"
+          },
+          {
+            "body": "The `Calendar` interface's `createEvent` signature was changed to require a second `credentialId: number` parameter, but only `GoogleCalendarService.createEvent` was updated to accept/use it. `LarkCalendarService.createEvent`, `Office365CalendarService.createEvent`, and the abstract `BaseCalendarService` in packages/lib/CalendarService.ts were not updated, so those providers cannot use the new per-credential destinationCalendar selection this PR introduces, leaving the multi-host destinationCalendar feature only partially implemented across calendar providers. Location: packages/types/Calendar.d.ts:221",
+            "created_at": null,
+            "line": 221,
+            "path": "packages/types/Calendar.d.ts"
+          }
+        ],
+        "tool": "shipwright"
       }
     ]
   },
@@ -75831,6 +77035,31 @@
             "created_at": "2026-06-28T22:13:21Z"
           }
         ]
+      },
+      {
+        "pr_url": "https://github.com/calcom/cal.com/pull/8087",
+        "repo_name": "shipwright__https://github.com/calcom/cal.com/pull/8087",
+        "review_comments": [
+          {
+            "body": "The bookingRefsFiltered.forEach() callback was converted to `async (bookingRef) => {...}` to allow `await getCalendar(...)`, but Array.prototype.forEach does not await its callback. The enclosing try/catch block therefore no longer catches errors thrown inside the async callback (they become unhandled promise rejections), and the surrounding function returns before the calendar deleteEvent/deleteMeeting operations actually complete, creating a race condition when rescheduling. Location: packages/app-store/vital/lib/reschedule.ts:125",
+            "created_at": null,
+            "line": 125,
+            "path": "packages/app-store/vital/lib/reschedule.ts"
+          },
+          {
+            "body": "Same fire-and-forget bug as in vital/reschedule.ts: forEach was made async to support `await getCalendar(...)`, but forEach ignores the returned promises, so the try/catch around it no longer catches rejected promises and the function proceeds without waiting for the calendar/meeting deletions to finish. Location: packages/app-store/wipemycalother/lib/reschedule.ts:125",
+            "created_at": null,
+            "line": 125,
+            "path": "packages/app-store/wipemycalother/lib/reschedule.ts"
+          },
+          {
+            "body": "bookingRefsFiltered.forEach() was changed to take an async callback to await getCalendar(...), but forEach does not await async callbacks. This makes calendar/meeting deletion during reschedule fire-and-forget: the handler can proceed (and respond) before old bookings' calendar events are actually deleted, and any thrown errors become unhandled promise rejections instead of being surfaced/caught. Location: packages/trpc/server/routers/viewer/bookings.tsx:553",
+            "created_at": null,
+            "line": 553,
+            "path": "packages/trpc/server/routers/viewer/bookings.tsx"
+          }
+        ],
+        "tool": "shipwright"
       }
     ]
   }

From a20652ef478f931eb6610f71410baf0b26329967 Mon Sep 17 00:00:00 2001
From: kpuru88 <227705750+kpuru88@users.noreply.github.com>
Date: Mon, 24 Aug 2026 13:07:30 -0700
Subject: [PATCH 2/2] Configure Shipwright review downloads

---
 .../step1_download_prs.py                     | 15 ++++++++-----
 offline/tests/test_step1.py                   | 22 +++++++++++++++++++
 2 files changed, 31 insertions(+), 6 deletions(-)

diff --git a/offline/code_review_benchmark/step1_download_prs.py b/offline/code_review_benchmark/step1_download_prs.py
index cedf882..f81f84f 100644
--- a/offline/code_review_benchmark/step1_download_prs.py
+++ b/offline/code_review_benchmark/step1_download_prs.py
@@ -106,9 +106,12 @@ def _is_bot(user: dict | None) -> bool:
     return login.endswith("[bot]")
 
 
-# Tools that post their review comments as a human GitHub account rather than a bot.
-# For these, bot filtering is skipped and all human comments are collected.
-_NON_BOT_TOOLS: frozenset[str] = frozenset({"claude"})
+# Per-tool author filtering used when downloading reviews from benchmark forks.
+# Unlisted tools retain the benchmark's bot-only default.
+_TOOL_DOWNLOAD_CONFIG: dict[str, dict[str, bool]] = {
+    "claude": {"require_bot": False},
+    "shipwright": {"require_bot": True},
+}
 
 
 def fetch_review_comments(org: str, repo: str, pr: int, tool: str = "") -> list[dict]:
@@ -116,10 +119,10 @@ def fetch_review_comments(org: str, repo: str, pr: int, tool: str = "") -> list[
 
     For most tools only bot-authored comments are collected, preventing human
     trigger commands (e.g. '/propel review') from polluting candidate extraction.
-    Tools listed in _NON_BOT_TOOLS post reviews under a human account; for those
-    all human comments are collected as-is.
+    Tools configured with ``require_bot=False`` post reviews under a human
+    account; for those all human comments are collected as-is.
     """
-    require_bot = tool not in _NON_BOT_TOOLS
+    require_bot = _TOOL_DOWNLOAD_CONFIG.get(tool, {"require_bot": True})["require_bot"]
     comments = []
 
     def _include(user: dict | None) -> bool:
diff --git a/offline/tests/test_step1.py b/offline/tests/test_step1.py
index e877a45..de4a966 100644
--- a/offline/tests/test_step1.py
+++ b/offline/tests/test_step1.py
@@ -86,6 +86,28 @@ def fake_gh(_args):
     ]
 
 
+def test_shipwright_download_config_collects_only_bot_reviews(monkeypatch):
+    shipwright_bot = {"login": "shipwright-agent[bot]", "type": "Bot"}
+    human = {"login": "benchmark-operator", "type": "User"}
+    responses = [
+        [
+            {"path": "app.py", "line": 7, "body": "finding", "created_at": "now", "user": shipwright_bot},
+            {"path": "app.py", "line": 8, "body": "trigger", "created_at": "now", "user": human},
+        ],
+        [],
+        [],
+    ]
+
+    monkeypatch.setattr(step1, "gh", lambda _args: responses.pop(0))
+
+    comments = step1.fetch_review_comments("org", "repo", 1, tool="shipwright")
+
+    assert step1._TOOL_DOWNLOAD_CONFIG["shipwright"] == {"require_bot": True}
+    assert comments == [
+        {"path": "app.py", "line": 7, "body": "finding", "created_at": "now"}
+    ]
+
+
 def test_fetch_repo_data(monkeypatch):
     def stub_pr_metadata(_org, _repo, pr):
         return {"title": f"PR {pr}", "url": "https://github.com/org/repo/pull/1"}