diff --git a/offline/analysis/score_profiles.py b/offline/analysis/score_profiles.py index 617a9bb..fa3f38c 100644 --- a/offline/analysis/score_profiles.py +++ b/offline/analysis/score_profiles.py @@ -19,7 +19,10 @@ PROFILE_CATEGORIES: dict[str, frozenset[str]] = { "strict": frozenset({"bug", "security", "concurrency", "data", "api"}), "core": frozenset({"bug", "security", "concurrency", "data", "api", "perf", "test_gap", "doc_defect"}), - "all": frozenset({"bug", "security", "concurrency", "data", "api", "perf", "test_gap", "doc_defect", "style", "speculative"}), + "all": frozenset({ + "bug", "security", "concurrency", "data", "api", + "perf", "test_gap", "doc_defect", "style", "speculative", + }), } # Tools excluded from the dashboard (superseded versions, incomplete runs, etc.) @@ -136,7 +139,8 @@ def print_table(results: dict[str, dict], profile: str, beta: float) -> None: sorted_tools = sorted(results.keys(), key=lambda t: results[t]["fbeta"], reverse=True) print(f"\n{'=' * 75}") - print(f" Profile: {profile.upper()} | {beta_label} | Categories: {', '.join(sorted(PROFILE_CATEGORIES[profile]))}") + categories = ", ".join(sorted(PROFILE_CATEGORIES[profile])) + print(f" Profile: {profile.upper()} | {beta_label} | Categories: {categories}") print(f"{'=' * 75}") print(f" {'Tool':<22s} {'Prec':>6s} {'Recall':>7s} {'F1':>6s} {beta_label:>6s} {'TP':>4s} {'FP':>4s} {'FN':>4s}") print(f" {'-' * 70}") diff --git a/offline/tests/test_step3.py b/offline/tests/test_step3.py index 8e026d2..b16205f 100644 --- a/offline/tests/test_step3.py +++ b/offline/tests/test_step3.py @@ -151,7 +151,7 @@ def add_argument(self, *_, **__): def parse_args(self): return SimpleNamespace( tool=None, limit=None, force=False, structured=False, - evaluations_file=None, dedup_groups=None, + evaluations_file=None, dedup_groups=None, no_dedup=False, ) monkeypatch.setattr(argparse, "ArgumentParser", DummyParser) diff --git a/online/etl/llm/prompts.py b/online/etl/llm/prompts.py index a503bc1..c5875b5 100644 --- a/online/etl/llm/prompts.py +++ b/online/etl/llm/prompts.py @@ -1,32 +1,47 @@ """Prompt templates for the LLM analysis pipeline.""" -EXTRACT_BOT_SUGGESTIONS = """You are analyzing a pull request to extract all actionable suggestions made by a code review bot. +EXTRACT_BOT_SUGGESTIONS = """You are analyzing a pull request to extract concrete code-review findings flagged by a code review bot. The bot's username is: {bot_username} Below you will see: -1. The commits that were under review (the code state the bot saw), including full diffs -2. The bot's review comments on those commits +1. The bot's review comments on the commits +2. The commits that were under review (the code state the bot saw), including full diffs -For each actionable suggestion the bot made, extract: +The Bot Review Comments section is a numbered list. Each entry is labeled as REVIEW_BODY, +INLINE_REVIEW_COMMENT, or ISSUE_COMMENT. Process all numbered comments. Top-level inline +review comments generally contain substantive findings; review bodies and issue comments may +contain findings or summaries. + +Your job is to identify every distinct substantive finding the bot flagged. A finding can be a +bug, defect, risk, incorrect behavior, missing validation, security concern, performance concern, +maintainability issue, documentation problem, test gap, or other concrete code-review concern. + +Only extract findings grounded in the bot's comments. Do not invent findings solely from the +commit diff. Use the commit diff only to understand code context for a comment. + +For each finding, extract: - A unique ID (S1, S2, ...) -- A description of what was suggested +- A concise description of the issue or concern the bot flagged - The category (bug, style, performance, security, refactor, documentation, other) - The file path and line number if available - Severity (low, medium, high, critical) -Only include ACTIONABLE suggestions — skip generic praise, summaries, or "looks good" comments. -Skip bot comments that are purely informational without suggesting any change. +Skip generic praise, status/progress messages, quota/limit messages, pure summaries without a +concrete finding, and "looks good" / "no issues found" comments. Also skip follow-up comments +that merely confirm a fix, acknowledge a clarification, say a parent comment was wrong, too +broad, pre-existing, already addressed, or out of scope, unless that same comment introduces a +new distinct code issue. PR Title: {pr_title} PR Author: {pr_author} Repository: {repo_name} -=== Commits Under Review (code the bot reviewed) === -{commits_under_review} - === Bot Review Comments === {bot_comments} + +=== Commits Under Review (code the bot reviewed) === +{commits_under_review} """ EXTRACT_HUMAN_ACTIONS = """You are analyzing post-review commit diffs to extract every concrete code issue that was fixed or improved AFTER the bot reviewed the PR. diff --git a/online/etl/llm/schemas.py b/online/etl/llm/schemas.py index 543307c..dd65c1e 100644 --- a/online/etl/llm/schemas.py +++ b/online/etl/llm/schemas.py @@ -7,8 +7,8 @@ class BotSuggestion(BaseModel): - issue_id: str = Field(description="Unique identifier for this suggestion (e.g. 'S1', 'S2')") - description: str = Field(description="What the bot suggested or flagged") + issue_id: str = Field(description="Unique identifier for this finding (e.g. 'S1', 'S2')") + description: str = Field(description="The issue or concern the bot flagged") category: str = Field( description="Category: 'bug', 'style', 'performance', 'security', 'refactor', 'documentation', 'other'" ) @@ -18,7 +18,7 @@ class BotSuggestion(BaseModel): class BotSuggestionsResponse(BaseModel): - suggestions: list[BotSuggestion] = Field(description="All actionable suggestions made by the bot") + suggestions: list[BotSuggestion] = Field(description="All concrete code-review findings flagged by the bot") class HumanAction(BaseModel): diff --git a/online/etl/pipeline/analyze.py b/online/etl/pipeline/analyze.py index 953a6aa..3dfc56b 100644 --- a/online/etl/pipeline/analyze.py +++ b/online/etl/pipeline/analyze.py @@ -5,6 +5,7 @@ import asyncio import json import logging +import re from openai import BadRequestError @@ -22,6 +23,8 @@ logger = logging.getLogger(__name__) +_HTML_COMMENT_RE = re.compile(r"", re.DOTALL) + def _find_bot_review_commit( reviews: list[dict], @@ -142,6 +145,11 @@ def _format_commits_with_diffs(commits: list[dict], details_by_sha: dict[str, di return "\n".join(lines) +def _clean_bot_comment_body(body: str) -> str: + """Remove hidden HTML comments that are not visible review content.""" + return _HTML_COMMENT_RE.sub("", body).strip() + + def _format_bot_comments(events: list[dict], chatbot_username: str) -> str: """Format bot's review/review_comment/issue_comment events with full context. @@ -149,6 +157,7 @@ def _format_bot_comments(events: list[dict], chatbot_username: str) -> str: to other commenters' threads, not original review suggestions. """ lines = [] + comment_num = 1 for e in events: if not same_github_actor(e.get("actor"), chatbot_username): continue @@ -164,23 +173,29 @@ def _format_bot_comments(events: list[dict], chatbot_username: str) -> str: if etype == "review": state = data.get("state", "") - body = data.get("body") or "" - lines.append(f"[{ts}] REVIEW ({state}):") + body = _clean_bot_comment_body(data.get("body") or "") + lines.append(f"COMMENT C{comment_num} [REVIEW_BODY state={state} timestamp={ts}]") if body: - lines.append(f" {body}") + lines.append(body) elif etype in ("review_comment", "issue_comment"): - body = data.get("body") or "" + body = _clean_bot_comment_body(data.get("body") or "") path = data.get("path") or "" line = data.get("line") or "" - loc = f" ({path}:{line})" if path else "" diff_hunk = data.get("diff_hunk") or "" resolved = " [RESOLVED]" if data.get("is_resolved") else "" - lines.append(f"[{ts}] {etype.upper()}{loc}{resolved}:") + if etype == "review_comment": + label = "INLINE_REVIEW_COMMENT" + location = f" path={path}:{line}" if path else "" + else: + label = "ISSUE_COMMENT" + location = "" + lines.append(f"COMMENT C{comment_num} [{label}{location}{resolved} timestamp={ts}]") if diff_hunk: - lines.append(f" Code context:\n ```\n{diff_hunk}\n ```") + lines.append(f"Code context:\n```diff\n{diff_hunk}\n```") if body: - lines.append(f" {body}") + lines.append(body) lines.append("") + comment_num += 1 return "\n".join(lines) if lines else "(no bot comments)" diff --git a/online/etl/tests/test_analyze_formatting.py b/online/etl/tests/test_analyze_formatting.py new file mode 100644 index 0000000..8445c0d --- /dev/null +++ b/online/etl/tests/test_analyze_formatting.py @@ -0,0 +1,91 @@ +"""Tests for bot comment formatting used by LLM extraction.""" + +from __future__ import annotations + +from pipeline.analyze import _clean_bot_comment_body +from pipeline.analyze import _format_bot_comments + + +def test_clean_bot_comment_body_removes_hidden_html_comments() -> None: + body = """ +Actual review finding. +""" + + assert _clean_bot_comment_body(body) == "Actual review finding." + + +def test_clean_bot_comment_body_preserves_visible_markdown_blocks() -> None: + body = """Actual review finding. + +
+Debug trace +Internal prompt text +
+ +generated metadata +""" + + assert _clean_bot_comment_body(body) == body.strip() + + +def test_clean_bot_comment_body_preserves_footer_text() -> None: + body = """Potential null dereference here. + +--- +If you found this review helpful, react to this comment. +""" + + assert _clean_bot_comment_body(body) == body.strip() + + +def test_clean_bot_comment_body_preserves_configured_findings_after_separator() -> None: + body = """Initial context. + +--- +P2: This option is configured incorrectly and can fail at runtime. +""" + + assert ( + _clean_bot_comment_body(body) + == "Initial context.\n\n---\nP2: This option is configured incorrectly and can fail at runtime." + ) + + +def test_format_bot_comments_labels_and_numbers_comments() -> None: + events = [ + { + "actor": "reviewer[bot]", + "event_type": "review_comment", + "timestamp": "2026-07-01T12:00:00Z", + "data": { + "path": "src/app.py", + "line": 42, + "diff_hunk": "@@ -1 +1 @@\n-old\n+new", + "body": "This can throw when value is null.", + }, + }, + { + "actor": "reviewer[bot]", + "event_type": "review_comment", + "timestamp": "2026-07-01T12:01:00Z", + "data": { + "in_reply_to_id": 123, + "path": "src/app.py", + "line": 42, + "body": "Fixed now.", + }, + }, + { + "actor": "reviewer[bot]", + "event_type": "review", + "timestamp": "2026-07-01T12:02:00Z", + "data": {"state": "commented", "body": "Summary finding."}, + }, + ] + + formatted = _format_bot_comments(events, "reviewer[bot]") + + assert "COMMENT C1 [INLINE_REVIEW_COMMENT path=src/app.py:42 timestamp=2026-07-01T12:00:00Z]" in formatted + assert "Code context:\n```diff\n@@ -1 +1 @@\n-old\n+new\n```" in formatted + assert "COMMENT C2 [REVIEW_BODY state=commented timestamp=2026-07-01T12:02:00Z]" in formatted + assert "Fixed now." not in formatted