Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
23 changes: 21 additions & 2 deletions .github/workflows/ai-review-reusable.yml
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,9 @@
# Secrets:
# ANTHROPIC_API_KEY - Claude API key (required)
# OPENAI_API_KEY - Codex / OpenAI API key (required)
# SONAR_TOKEN - SonarQube token that can browse the project (optional; only a private
# project needs it). The gate collects the PR's SonarQube findings from
# the SonarQube Cloud check run on the PR head for the reviewers to fix.
# AI_REVIEW_PUSH_TOKEN - Token that pushes fixes: a GitHub App token (or fine-grained PAT) with
# contents:write on this repository and no workflows permission, so
# GitHub itself refuses a push that changes a workflow. Without it the
Expand All @@ -25,8 +28,8 @@
# the required scan status would never report on the new head and the
# PR could not merge until someone pushed again.
#
# The caller must grant the job actions: read, contents: read, pull-requests: write and
# statuses: read. Add the `skip-ai-review` label to a PR to opt out. A manual run
# The caller must grant the job actions: read, checks: read, contents: read, pull-requests: write
# and statuses: read (without checks: read, SonarQube findings are left out). Add the `skip-ai-review` label to a PR to opt out. A manual run
# fails when it cannot review, so it is not mistaken for a review that passed.
#
# Third party actions are pinned to a full commit SHA, because a tag can be moved
Expand Down Expand Up @@ -97,6 +100,16 @@ on:
required: false
type: string
default: security/malicious-code-scan
sonar_project_key:
description: SonarQube project key whose PR findings the reviewers fix (default is taken from the SonarQube Cloud check run)
required: false
type: string
default: ""
sonar_wait_minutes:
description: How long the gate waits for a SonarQube analysis still running on the PR head (default 10)
required: false
type: string
default: ""
shared_ref:
description: Ref of OpenC3/.github to take the scripts and prompt from; match the ref in `uses:` (ai-review-run.yml is always taken from main)
required: false
Expand All @@ -109,6 +122,8 @@ on:
required: true
AI_REVIEW_PUSH_TOKEN:
required: false
SONAR_TOKEN:
required: false

permissions:
contents: read
Expand Down Expand Up @@ -170,6 +185,7 @@ jobs:
# The most any job of the review gets; each job takes only what it needs
permissions:
actions: read
checks: read
contents: read
pull-requests: write
statuses: read
Expand All @@ -188,8 +204,11 @@ jobs:
scan_workflow_name: ${{ inputs.scan_workflow_name }}
scan_wait_minutes: ${{ inputs.scan_wait_minutes }}
scan_status_context: ${{ inputs.scan_status_context }}
sonar_project_key: ${{ inputs.sonar_project_key }}
sonar_wait_minutes: ${{ inputs.sonar_wait_minutes }}
shared_ref: ${{ inputs.shared_ref }}
secrets:
ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
AI_REVIEW_PUSH_TOKEN: ${{ secrets.AI_REVIEW_PUSH_TOKEN }}
SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}
24 changes: 22 additions & 2 deletions .github/workflows/ai-review-run.yml
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,8 @@
# and only the permissions it needs, so what the agents can reach holds nothing
# worth escaping their sandbox for:
#
# gate - decides whether to review and collects failed CI logs (read-only token)
# gate - decides whether to review and collects failed CI logs and SonarQube
# findings (read-only token)
# review - Claude and Codex take turns in throwaway containers with no network
# but an API proxy holding the turn's key (ai-review/ai_review_loop.sh).
# The job's token can only read the repository, and the runner's own
Expand Down Expand Up @@ -69,6 +70,14 @@ on:
required: false
type: string
default: security/malicious-code-scan
sonar_project_key:
required: false
type: string
default: ""
sonar_wait_minutes:
required: false
type: string
default: ""
shared_ref:
required: false
type: string
Expand All @@ -80,6 +89,8 @@ on:
required: true
AI_REVIEW_PUSH_TOKEN:
required: false
SONAR_TOKEN:
required: false

permissions:
contents: read
Expand All @@ -96,6 +107,7 @@ jobs:
timeout-minutes: 30
permissions:
actions: read
checks: read
contents: read
pull-requests: read
statuses: read
Expand All @@ -106,6 +118,7 @@ jobs:
head_ref: ${{ steps.gate.outputs.head_ref }}
base_ref: ${{ steps.gate.outputs.base_ref }}
ci_failures: ${{ steps.gate.outputs.ci_failures }}
sonar_findings: ${{ steps.gate.outputs.sonar_findings }}
steps:
- name: Harden the runner (Audit all outbound calls)
uses: step-security/harden-runner@e14015d583714f6e62063499dc959a02595150a1 # v2.21.1
Expand Down Expand Up @@ -152,6 +165,9 @@ jobs:
SCAN_CONTEXT: ${{ inputs.scan_status_context }}
SCAN_WAIT_MINUTES: ${{ inputs.scan_wait_minutes || '10' }}
MAX_CI_ROUNDS: ${{ inputs.max_ci_rounds || '3' }}
SONAR_PROJECT_KEY: ${{ inputs.sonar_project_key }}
SONAR_WAIT_MINUTES: ${{ inputs.sonar_wait_minutes || '10' }}
SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}
OUT_DIR: ${{ runner.temp }}/ai-review
run: bash shared/ai-review/ai_review_gate.sh

Expand All @@ -160,7 +176,9 @@ jobs:
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
name: ai-review-ci-failures
path: ${{ runner.temp }}/ai-review/ci_failures.md
path: |
${{ runner.temp }}/ai-review/ci_failures.md
${{ runner.temp }}/ai-review/sonar_findings.md
retention-days: 1
# A re-run of the job replaces the earlier attempt's
overwrite: true
Expand Down Expand Up @@ -253,6 +271,8 @@ jobs:
RESULT_DIR: ${{ runner.temp }}/ai-review-result
CI_FAILURES_FILE: ${{ runner.temp }}/ai-review-ci/ci_failures.md
CI_FAILURE_COUNT: ${{ needs.gate.outputs.ci_failures }}
SONAR_FINDINGS_FILE: ${{ runner.temp }}/ai-review-ci/sonar_findings.md
SONAR_FINDING_COUNT: ${{ needs.gate.outputs.sonar_findings }}
run: |
git config user.name "github-actions[bot]"
git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
Expand Down
109 changes: 102 additions & 7 deletions ai-review/ai_review_gate.sh
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@
# if purchased from OpenC3, Inc.

# Decides whether the AI review loop should run for a PR and collects failed
# CI job logs for the reviewers. Every CI workflow completion triggers the AI
# CI job logs and SonarQube findings for the reviewers. Every CI workflow completion triggers the AI
# Review workflow, so this lets only the run that sees all CI finished proceed.
# The Malicious Code Scan does not trigger the review, so that run waits for a
# scan still running on the PR head. A manual (workflow_dispatch) run fails
Expand All @@ -21,9 +21,12 @@
# One of: PR_NUMBER, HEAD_SHA
# Optional env: FORCE (review even if this commit was already reviewed), REVIEW_WORKFLOW,
# SCAN_WORKFLOW, SCAN_CONTEXT, MAX_CI_ROUNDS, LOG_LINES,
# SCAN_WAIT_MINUTES (how long to wait for a running scan), SCAN_POLL_SECONDS
# SCAN_WAIT_MINUTES (how long to wait for a running scan), SCAN_POLL_SECONDS,
# SONAR_PROJECT_KEY (default: from the SonarQube check run), SONAR_TOKEN (for a private
# project), SONAR_HOST_URL, SONAR_APP (the check run's app slug),
# SONAR_WAIT_MINUTES (how long to wait for a running analysis)
#
# Step outputs: skip, reason, pr, head_sha, head_ref, base_ref, ci_failures
# Step outputs: skip, reason, pr, head_sha, head_ref, base_ref, ci_failures, sonar_findings

set -euo pipefail

Expand All @@ -39,14 +42,20 @@ MAX_CI_ROUNDS="${MAX_CI_ROUNDS:-3}"
SCAN_WAIT_MINUTES="${SCAN_WAIT_MINUTES:-10}"
SCAN_POLL_SECONDS="${SCAN_POLL_SECONDS:-30}"
LOG_LINES="${LOG_LINES:-150}"
SONAR_PROJECT_KEY="${SONAR_PROJECT_KEY:-}"
SONAR_HOST_URL="${SONAR_HOST_URL:-https://sonarcloud.io}"
SONAR_APP="${SONAR_APP:-sonarqubecloud}"
SONAR_WAIT_MINUTES="${SONAR_WAIT_MINUTES:-10}"
GITHUB_OUTPUT="${GITHUB_OUTPUT:-/dev/null}"
repo="$GITHUB_REPOSITORY"
PR_NUMBER="${PR_NUMBER:-}"
HEAD_SHA="${HEAD_SHA:-}"

mkdir -p "$OUT_DIR"
CI_FILE="$OUT_DIR/ci_failures.md"
SONAR_FILE="$OUT_DIR/sonar_findings.md"
: > "$CI_FILE"
: > "$SONAR_FILE"

output() { echo "$1=$2" >> "$GITHUB_OUTPUT"; }
skip() {
Expand Down Expand Up @@ -187,25 +196,111 @@ while IFS=$'\t' read -r run_id run_name run_conclusion run_url; do
done < <(jq -r '.[] | select(.conclusion == "failure" or .conclusion == "timed_out" or .conclusion == "startup_failure")
| [.id, .name, .conclusion, .html_url] | @tsv' <<< "$runs")

# When the head commit is the loop's own fix, only go again to fix CI, and only a few times
# SonarQube reports through its GitHub App as a check run, not an Actions run, so the runs above
# never include it. Wait for its analysis of this commit, then collect what it found on the PR.
# Sonar trouble (no check run, an outage, no checks: read) only leaves its findings out.
sonar_findings=0
sonar_check() {
gh api "repos/$repo/commits/$HEAD_SHA/check-runs?per_page=100" --paginate \
--jq ".check_runs[] | select(.app.slug == \"$SONAR_APP\")" | jq -s '.[0] // {}'
}
# The token goes through stdin rather than the command line. Anything but a JSON object fails, so
# the callers' jq cannot stop the gate on an error page.
sonar_api() {
if [[ -n "${SONAR_TOKEN:-}" ]]; then
printf 'header = "Authorization: Bearer %s"\n' "$SONAR_TOKEN"
fi | curl -fsS --max-time 30 -K - "$SONAR_HOST_URL/api/$1" | jq -ce 'objects'
}
if ! sonar="$(sonar_check)"; then
echo "::warning::Could not read the check runs on $HEAD_SHA (does the workflow have checks: read?); SonarQube findings are left out"
sonar='{}'
fi
deadline=$((SECONDS + SONAR_WAIT_MINUTES * 60))
while [[ "$(jq -r '.status // "completed"' <<< "$sonar")" != "completed" ]] && (( SECONDS < deadline )); do
[[ "$(gh api "repos/$repo/pulls/$PR_NUMBER" --jq .head.sha)" == "$HEAD_SHA" ]] ||
skip "the PR head moved past $HEAD_SHA while waiting for the SonarQube analysis"
echo "Waiting for the SonarQube analysis on $HEAD_SHA ($(jq -r .status <<< "$sonar"))"
sleep "$SCAN_POLL_SECONDS"
sonar="$(sonar_check)" || sonar='{}'
done
sonar_status="$(jq -r '.status // ""' <<< "$sonar")"
if [[ -n "$sonar_status" && "$sonar_status" != "completed" ]]; then
echo "::warning::The SonarQube analysis of $HEAD_SHA was still $sonar_status after ${SONAR_WAIT_MINUTES} minute(s); its findings are left out"
elif [[ -n "$sonar_status" ]]; then
project="$SONAR_PROJECT_KEY"
if [[ -z "$project" ]]; then
project="$(python3 -c 'import sys, urllib.parse as u; print(u.parse_qs(u.urlsplit(sys.argv[1]).query).get("id", [""])[0])' \
"$(jq -r '.details_url // ""' <<< "$sonar")")"
fi
if [[ ! "$project" =~ ^[A-Za-z0-9_.:-]+$ ]]; then
echo "::warning::No usable SonarQube project key ('$project'); set sonar_project_key. SonarQube findings are left out"
else
query="$(jq -rn --arg k "$project" --arg pr "$PR_NUMBER" '"projectKey=\($k | @uri)&pullRequest=\($pr | @uri)"')"
# Paths come back as <project>:<path>; messages are kept to one line. $project is jq's.
# shellcheck disable=SC2016
defs='def loc: (.component | ltrimstr($project + ":")) + (if .line then ":\(.line)" else "" end);
def text: gsub("[\r\n]+"; " ");'
if gate="$(sonar_api "qualitygates/project_status?$query")"; then
failed="$(jq -r '.projectStatus.conditions // [] | .[] | select(.status == "ERROR")
| "- \(.metricKey) is \(.actualValue) (fails when \(.comparator) \(.errorThreshold))"' <<< "$gate")"
if [[ -n "$failed" ]]; then
sonar_findings=$((sonar_findings + $(wc -l <<< "$failed")))
printf '### Quality gate failed\n\n%s\n\n' "$failed" >> "$SONAR_FILE"
fi
else
echo "::warning::Could not read the SonarQube quality gate for $project PR #$PR_NUMBER"
fi
if issues="$(sonar_api "issues/search?${query/projectKey=/componentKeys=}&issueStatuses=OPEN,CONFIRMED&ps=500")"; then
count="$(jq '.issues | length' <<< "$issues")"
if (( count > 0 )); then
sonar_findings=$((sonar_findings + count))
{
echo "### Open issues ($(jq '.paging.total // .total // (.issues | length)' <<< "$issues"))"
echo
jq -r --arg project "$project" "$defs"' .issues[] | "- **\(.severity // "?")** \(loc): \(.message | text) (rule \(.rule))"' <<< "$issues"
echo
} >> "$SONAR_FILE"
fi
else
echo "::warning::Could not read the SonarQube issues for $project PR #$PR_NUMBER"
fi
# Listed but not counted: one that is safe needs a human to mark it in SonarQube, which no fix
# commit can do, so counting it would send every fix round back for another
if hotspots="$(sonar_api "hotspots/search?$query&status=TO_REVIEW&ps=500")"; then
if (( $(jq '.hotspots | length' <<< "$hotspots") > 0 )); then
{
echo "### Security hotspots to review"
echo
jq -r --arg project "$project" "$defs"' .hotspots[] | "- **\(.vulnerabilityProbability // "?")** \(loc): \(.message | text) (rule \(.ruleKey))"' <<< "$hotspots"
echo
} >> "$SONAR_FILE"
fi
else
echo "::warning::Could not read the SonarQube security hotspots for $project PR #$PR_NUMBER"
fi
fi
fi

# When the head commit is the loop's own fix, only go again to fix CI or SonarQube findings, and only a few times
head_message="$(gh api "repos/$repo/commits/$HEAD_SHA" --jq .commit.message)"
if grep -q '^AI-Review-Bot: true$' <<< "$head_message"; then
(( failures > 0 )) || skip "head commit is an AI review fix and CI passed"
(( failures + sonar_findings > 0 )) || skip "head commit is an AI review fix and CI passed"
# Count distinct loop runs among the consecutive AI review commits at the tip of the PR
rounds="$(gh api "repos/$repo/pulls/$PR_NUMBER/commits" --paginate --jq '[.[].commit.message]' | jq -s '
add | reverse
| (map(test("(?m)^AI-Review-Bot: true$") | not) | index(true)) as $human
| (if $human == null then . else .[:$human] end)
| map(capture("(?m)^AI-Review-Run: (?<id>\\S+)$").id) | unique | length')"
if (( rounds >= MAX_CI_ROUNDS )); then
skip "CI still failing after $rounds AI fix round(s) (max $MAX_CI_ROUNDS)"
skip "CI or SonarQube still failing after $rounds AI fix round(s) (max $MAX_CI_ROUNDS)"
fi
fi

echo "PR #$PR_NUMBER at $HEAD_SHA: $total CI run(s) complete, $failures CI failure(s)"
echo "PR #$PR_NUMBER at $HEAD_SHA: $total CI run(s) complete, $failures CI failure(s), $sonar_findings SonarQube finding(s)"
output skip false
output pr "$PR_NUMBER"
output head_sha "$HEAD_SHA"
output head_ref "$(pr_field .head.ref)"
output base_ref "$(pr_field .base.ref)"
output ci_failures "$failures"
output sonar_findings "$sonar_findings"
16 changes: 16 additions & 0 deletions ai-review/ai_review_loop.sh
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@
# Required env: BASE_REF, CLAUDE_API_KEY, CODEX_API_KEY, SANDBOX_IMAGE (built from sandbox/)
# Optional env: MAX_TURNS, TIME_LIMIT_MINUTES, CLAUDE_MODEL, CODEX_MODEL, CLAUDE_MAX_BUDGET_USD, CODEX_SANDBOX,
# CI_FAILURES_FILE (failed CI job logs from ai_review_gate.sh), CI_FAILURE_COUNT,
# SONAR_FINDINGS_FILE (SonarQube findings from ai_review_gate.sh), SONAR_FINDING_COUNT,
# REVIEW_INSTRUCTIONS (repository-specific guidance for the prompt), GITHUB_RUN_ID,
# RESULT_DIR, ANTHROPIC_UPSTREAM and OPENAI_UPSTREAM (where the proxy sends each API's calls)
#
Expand Down Expand Up @@ -63,6 +64,7 @@ SCHEMA="$SCRIPT_DIR/schema.json"
POLICY="$SCRIPT_DIR/patch_policy.py"
HISTORY="$OUT_DIR/history.md"
CI_FAILURES_FILE="${CI_FAILURES_FILE:-}"
SONAR_FINDINGS_FILE="${SONAR_FINDINGS_FILE:-}"
RUN_ID="${GITHUB_RUN_ID:-local}"

# Keep the runner's system and user git config (such as its LFS filter) out of the harness's git
Expand Down Expand Up @@ -233,6 +235,16 @@ build_prompt() {
echo "All CI checks passed."
fi
echo
echo "## SonarQube findings for the commit under review"
echo
if [[ -n "$SONAR_FINDINGS_FILE" && -s "$SONAR_FINDINGS_FILE" ]]; then
echo "SonarQube reported these on the PR. Fix them as your job describes (unless a previous turn already did):"
echo
cat "$SONAR_FINDINGS_FILE"
else
echo "None reported."
fi
echo
echo "## Previous turns"
echo
if [[ -s "$HISTORY" ]]; then
Expand Down Expand Up @@ -331,6 +343,10 @@ write_result() {
echo "CI had ${CI_FAILURE_COUNT:-some} failure(s) on the reviewed commit; the reviewers were asked to fix them."
echo
fi
if [[ -n "$SONAR_FINDINGS_FILE" && -s "$SONAR_FINDINGS_FILE" ]]; then
echo "SonarQube had ${SONAR_FINDING_COUNT:-some} finding(s) on the reviewed commit; the reviewers were asked to fix them."
echo
fi
case "$state" in
converged) echo "✅ Claude and Codex converged after $turn turn(s) with $commits fix commit(s)." ;;
max_turns) echo "⚠️ Stopped after the maximum of $MAX_TURNS turns without converging ($commits fix commit(s)). A human should look at the last few turns." ;;
Expand Down
Loading
Loading