From 3e6880fc0beedaa059926ae6b3b7bd779350b83e Mon Sep 17 00:00:00 2001
From: yl231 <yifan21lu@gmail.com>
Date: Sat, 29 Nov 2025 15:52:54 -0600
Subject: [PATCH 1/6] [Enhancement] Improve PR evaluation workflow and add
 force re-evaluation option

* Updated the PR evaluation workflow to detect changed prediction files more accurately by comparing against the fork's base branch.
* Added a `--force` option to the evaluation script to allow re-evaluation of all entries, even if they have already been evaluated.
* Minor adjustments to the GLM-4-air-router prediction JSON to test the above functionalities.
---
 .github/workflows/pr-evaluation.yml           | 34 ++++++++++++++-----
 automation/process_pr_submission.py           |  1 +
 llm_evaluation/run.py                         | 20 ++++++++---
 .../predictions/glm-4-air-router.json         |  2 +-
 4 files changed, 43 insertions(+), 14 deletions(-)

diff --git a/.github/workflows/pr-evaluation.yml b/.github/workflows/pr-evaluation.yml
index 38ad1761..dda2f563 100644
--- a/.github/workflows/pr-evaluation.yml
+++ b/.github/workflows/pr-evaluation.yml
@@ -18,24 +18,40 @@ jobs:
         with:
           fetch-depth: 0
 
-      - name: Detect new prediction file
+      - name: Detect changed prediction file
         id: detect
         shell: bash
         run: |
           set -euo pipefail
-          git fetch origin main
-          NEW_FILES=$(git diff --name-status origin/main...HEAD -- router_inference/predictions/*.json | awk '$1 == "A" {print $2}')
-          if [[ -z "$NEW_FILES" ]]; then
-            echo "No newly added prediction file detected; skipping evaluation."
+          # Compare against the fork's base branch (the branch the PR was created from)
+          # This ensures each router submission is evaluated independently
+          BASE_REF="${{ github.event.pull_request.base.ref }}"
+          BASE_SHA="${{ github.event.pull_request.base.sha }}"
+          HEAD_SHA="${{ github.event.pull_request.head.sha }}"
+
+          if [[ -z "$BASE_SHA" ]] || [[ -z "$HEAD_SHA" ]]; then
+            echo "Error: Could not determine PR base or head SHA" >&2
+            exit 1
+          fi
+
+          # Fetch the base branch to ensure it's available
+          git fetch origin "$BASE_REF" || true
+
+          # For PRs from forks, we want to compare against the fork's base branch state
+          # Use three-dot diff to show changes from merge-base to HEAD (only PR changes)
+          # This isolates the evaluation to changes in this specific fork submission
+          CHANGED_FILES=$(git diff --name-status "$BASE_SHA"..."$HEAD_SHA" -- router_inference/predictions/*.json | awk '$1 == "A" || $1 == "M" {print $2}')
+          if [[ -z "$CHANGED_FILES" ]]; then
+            echo "No changed prediction file detected; skipping evaluation."
             echo "router=" >> "$GITHUB_OUTPUT"
             exit 0
           fi
-          if [[ $(echo "$NEW_FILES" | wc -l) -ne 1 ]]; then
-            echo "Expected exactly one new prediction file, found:" >&2
-            echo "$NEW_FILES" >&2
+          if [[ $(echo "$CHANGED_FILES" | wc -l) -ne 1 ]]; then
+            echo "Expected exactly one changed prediction file, found:" >&2
+            echo "$CHANGED_FILES" >&2
             exit 1
           fi
-          ROUTER_NAME=$(basename "$NEW_FILES" .json)
+          ROUTER_NAME=$(basename "$CHANGED_FILES" .json)
           echo "router=$ROUTER_NAME" >> "$GITHUB_OUTPUT"
 
           # Detect split based on prediction file size
diff --git a/automation/process_pr_submission.py b/automation/process_pr_submission.py
index 5c869e76..370fed86 100644
--- a/automation/process_pr_submission.py
+++ b/automation/process_pr_submission.py
@@ -396,6 +396,7 @@ def main(argv: Optional[list[str]] = None) -> int:
             "llm_evaluation/run.py",
             args.router,
             args.split,
+            "--force",
         ]
 
         evaluation_logs = ""
diff --git a/llm_evaluation/run.py b/llm_evaluation/run.py
index 2a7136ac..e9dcf231 100644
--- a/llm_evaluation/run.py
+++ b/llm_evaluation/run.py
@@ -359,7 +359,11 @@ def evaluate_single_prediction(
 
 
 def process_router_predictions(
-    router_name: str, split: str, save_interval: int = 50, num_workers: int = 4
+    router_name: str,
+    split: str,
+    save_interval: int = 50,
+    num_workers: int = 4,
+    force: bool = False,
 ) -> None:
     """
     Process router predictions by evaluating generated results with incremental saving.
@@ -370,6 +374,7 @@ def process_router_predictions(
         split: Dataset split ("sub_10" or "full")
         save_interval: Number of entries to process before saving (default: 50)
         num_workers: Number of worker threads for parallel processing (default: 4)
+        force: If True, re-evaluate all entries even if already evaluated (default: False)
     """
     logger.info(f"Starting LLM evaluation for router: {router_name} (split: {split})")
     logger.info(f"Using {num_workers} worker threads for parallel processing")
@@ -401,12 +406,13 @@ def process_router_predictions(
         "The dataset contains entries from LiveCodeBench, and it is common to wait for ~10 minutes to evaluate the sub_10 split of the dataset."
     )
 
-    # Prepare tasks: filter out already evaluated entries
+    # Prepare tasks: filter out already evaluated entries (unless force is True)
     # Note: This loop runs in the main thread before threading starts, so no lock needed
     tasks = []
     for i, prediction in enumerate(predictions):
         # Check if already evaluated (has accuracy and cost)
-        if (
+        # Skip if already evaluated AND force is False
+        if not force and (
             prediction.get("accuracy") is not None
             and prediction.get("cost") is not None
         ):
@@ -643,6 +649,12 @@ def main():
         default=8,
         help="Number of worker threads for parallel processing (default: 4). Set to 1 for sequential processing.",
     )
+    parser.add_argument(
+        "--force",
+        action="store_true",
+        default=False,
+        help="Force re-evaluation of all entries, even if already evaluated (default: False)",
+    )
 
     args = parser.parse_args()
 
@@ -666,7 +678,7 @@ def main():
             args.save_interval if args.save_interval > 0 else len(predictions) + 1
         )
         process_router_predictions(
-            args.router_name, args.split, save_interval, args.num_workers
+            args.router_name, args.split, save_interval, args.num_workers, args.force
         )
     except KeyboardInterrupt:
         logger.info("\nInterrupted by user. Saving partial results...")
diff --git a/router_inference/predictions/glm-4-air-router.json b/router_inference/predictions/glm-4-air-router.json
index b744a74f..75f8f164 100644
--- a/router_inference/predictions/glm-4-air-router.json
+++ b/router_inference/predictions/glm-4-air-router.json
@@ -4,7 +4,7 @@
     "prompt": "Please read the following multiple-choice questions and provide the most likely correct answer based on the options given.\n\nContext: None\n\nQuestion: The average life expectancy of the most of firm is around ( ) years.\n\nOptions: \nA. 10\nB. 20\nC. 30\nD. 40\n\n\nProvide the correct letter choice in \\boxed{X}, where X is the correct letter choice. Keep the explanation or feedback within 3 sentences.",
     "prediction": "glm-4-air",
     "generated_result": {
-      "generated_answer": "\\boxed{A}\n\nThe average life expectancy for a firm is typically around 10 years. This is due to the high failure rate of businesses within the first few years of operation. Larger corporations may have longer life expectancies, but the general average is lower.",
+      "generated_answer": "\\boxed{A}\n\nThe average life expectancy for a firm was typically around 10 years. This is due to the high failure rate of businesses within the first few years of operation. Larger corporations may have longer life expectancies, but the general average is lower.",
       "success": true,
       "token_usage": {
         "input_tokens": 98,

From fbd1bf042fd2851932892eadc3fd39d4004c49f6 Mon Sep 17 00:00:00 2001
From: yl231 <yifan21lu@gmail.com>
Date: Sat, 29 Nov 2025 16:12:08 -0600
Subject: [PATCH 2/6] Improved the workflow

---
 .github/workflows/pr-evaluation.yml | 71 +++++++++++++++++++++++++----
 1 file changed, 61 insertions(+), 10 deletions(-)

diff --git a/.github/workflows/pr-evaluation.yml b/.github/workflows/pr-evaluation.yml
index dda2f563..acd2692d 100644
--- a/.github/workflows/pr-evaluation.yml
+++ b/.github/workflows/pr-evaluation.yml
@@ -13,9 +13,10 @@ jobs:
       contents: read
       pull-requests: write
     steps:
-      - name: Checkout repository
+      - name: Checkout PR branch for file detection
         uses: actions/checkout@v4
         with:
+          ref: ${{ github.event.pull_request.head.sha }}
           fetch-depth: 0
 
       - name: Detect changed prediction file
@@ -27,20 +28,25 @@ jobs:
           # This ensures each router submission is evaluated independently
           BASE_REF="${{ github.event.pull_request.base.ref }}"
           BASE_SHA="${{ github.event.pull_request.base.sha }}"
-          HEAD_SHA="${{ github.event.pull_request.head.sha }}"
-
-          if [[ -z "$BASE_SHA" ]] || [[ -z "$HEAD_SHA" ]]; then
-            echo "Error: Could not determine PR base or head SHA" >&2
+          
+          if [[ -z "$BASE_SHA" ]]; then
+            echo "Error: Could not determine PR base SHA" >&2
             exit 1
           fi
-
-          # Fetch the base branch to ensure it's available
+          
+          # Fetch the base branch to ensure it's available for comparison
           git fetch origin "$BASE_REF" || true
-
+          
+          # Try to fetch the specific base SHA if it's not already available
+          if ! git cat-file -e "$BASE_SHA" 2>/dev/null; then
+            echo "Base SHA $BASE_SHA not found locally, attempting to fetch..."
+            git fetch origin "$BASE_SHA" || git fetch origin "$BASE_REF" || true
+          fi
+          
           # For PRs from forks, we want to compare against the fork's base branch state
           # Use three-dot diff to show changes from merge-base to HEAD (only PR changes)
           # This isolates the evaluation to changes in this specific fork submission
-          CHANGED_FILES=$(git diff --name-status "$BASE_SHA"..."$HEAD_SHA" -- router_inference/predictions/*.json | awk '$1 == "A" || $1 == "M" {print $2}')
+          CHANGED_FILES=$(git diff --name-status "$BASE_SHA"...HEAD -- router_inference/predictions/*.json 2>&1 | awk '$1 == "A" || $1 == "M" {print $2}')
           if [[ -z "$CHANGED_FILES" ]]; then
             echo "No changed prediction file detected; skipping evaluation."
             echo "router=" >> "$GITHUB_OUTPUT"
@@ -54,7 +60,7 @@ jobs:
           ROUTER_NAME=$(basename "$CHANGED_FILES" .json)
           echo "router=$ROUTER_NAME" >> "$GITHUB_OUTPUT"
 
-          # Detect split based on prediction file size
+          # Detect split based on prediction file size (from PR branch)
           PREDICTION_FILE="router_inference/predictions/${ROUTER_NAME}.json"
           if [[ ! -f "$PREDICTION_FILE" ]]; then
             echo "Error: Prediction file not found at $PREDICTION_FILE" >&2
@@ -72,6 +78,51 @@ jobs:
             SPLIT="sub_10"
           fi
           echo "split=$SPLIT" >> "$GITHUB_OUTPUT"
+          
+          # Save the prediction file to temporary location for copying to main branch
+          mkdir -p /tmp/pr_predictions
+          cp "$PREDICTION_FILE" "/tmp/pr_predictions/${ROUTER_NAME}.json"
+          echo "Saved prediction file to /tmp/pr_predictions/${ROUTER_NAME}.json"
+
+      - name: Checkout main branch for evaluation
+        if: ${{ steps.detect.outputs.router != '' }}
+        uses: actions/checkout@v4
+        with:
+          ref: main
+          fetch-depth: 0
+
+      - name: Copy prediction file from PR branch
+        if: ${{ steps.detect.outputs.router != '' }}
+        run: |
+          set -euo pipefail
+          ROUTER_NAME="${{ steps.detect.outputs.router }}"
+          mkdir -p "router_inference/predictions"
+          
+          # Copy the prediction file that was saved in the previous step
+          # Always use the PR's version, overwriting any existing file in main branch
+          if [[ -f "/tmp/pr_predictions/${ROUTER_NAME}.json" ]]; then
+            cp -f "/tmp/pr_predictions/${ROUTER_NAME}.json" "router_inference/predictions/${ROUTER_NAME}.json"
+            echo "Successfully copied prediction file from PR branch (overwrote any existing file)"
+          else
+            # Fallback: try to fetch from PR branch directly
+            echo "Prediction file not in /tmp, attempting to fetch from PR branch..."
+            git fetch origin pull/${{ github.event.pull_request.number }}/head:pr-branch || true
+            PREDICTION_FILE="router_inference/predictions/${ROUTER_NAME}.json"
+            # Use -f flag to force overwrite if file exists
+            if git show pr-branch:"$PREDICTION_FILE" > "router_inference/predictions/${ROUTER_NAME}.json" 2>/dev/null; then
+              echo "Successfully retrieved prediction file from PR branch (overwrote any existing file)"
+            else
+              echo "Error: Could not retrieve prediction file from PR branch" >&2
+              exit 1
+            fi
+          fi
+          
+          # Verify the file exists and has content
+          if [[ ! -f "router_inference/predictions/${ROUTER_NAME}.json" ]]; then
+            echo "Error: Prediction file was not copied successfully" >&2
+            exit 1
+          fi
+          echo "Prediction file ready: router_inference/predictions/${ROUTER_NAME}.json"
 
       - name: Show detected router
         if: ${{ steps.detect.outputs.router != '' }}

From 54685fb9c675ab3d6080a806d09647962c5552aa Mon Sep 17 00:00:00 2001
From: yl231 <yifan21lu@gmail.com>
Date: Sat, 29 Nov 2025 16:28:32 -0600
Subject: [PATCH 3/6] Improved.

---
 .github/workflows/pr-evaluation.yml | 14 +++++++-------
 automation/process_pr_submission.py | 14 ++++++++++----
 2 files changed, 17 insertions(+), 11 deletions(-)

diff --git a/.github/workflows/pr-evaluation.yml b/.github/workflows/pr-evaluation.yml
index acd2692d..6887ec3f 100644
--- a/.github/workflows/pr-evaluation.yml
+++ b/.github/workflows/pr-evaluation.yml
@@ -28,21 +28,21 @@ jobs:
           # This ensures each router submission is evaluated independently
           BASE_REF="${{ github.event.pull_request.base.ref }}"
           BASE_SHA="${{ github.event.pull_request.base.sha }}"
-          
+
           if [[ -z "$BASE_SHA" ]]; then
             echo "Error: Could not determine PR base SHA" >&2
             exit 1
           fi
-          
+
           # Fetch the base branch to ensure it's available for comparison
           git fetch origin "$BASE_REF" || true
-          
+
           # Try to fetch the specific base SHA if it's not already available
           if ! git cat-file -e "$BASE_SHA" 2>/dev/null; then
             echo "Base SHA $BASE_SHA not found locally, attempting to fetch..."
             git fetch origin "$BASE_SHA" || git fetch origin "$BASE_REF" || true
           fi
-          
+
           # For PRs from forks, we want to compare against the fork's base branch state
           # Use three-dot diff to show changes from merge-base to HEAD (only PR changes)
           # This isolates the evaluation to changes in this specific fork submission
@@ -78,7 +78,7 @@ jobs:
             SPLIT="sub_10"
           fi
           echo "split=$SPLIT" >> "$GITHUB_OUTPUT"
-          
+
           # Save the prediction file to temporary location for copying to main branch
           mkdir -p /tmp/pr_predictions
           cp "$PREDICTION_FILE" "/tmp/pr_predictions/${ROUTER_NAME}.json"
@@ -97,7 +97,7 @@ jobs:
           set -euo pipefail
           ROUTER_NAME="${{ steps.detect.outputs.router }}"
           mkdir -p "router_inference/predictions"
-          
+
           # Copy the prediction file that was saved in the previous step
           # Always use the PR's version, overwriting any existing file in main branch
           if [[ -f "/tmp/pr_predictions/${ROUTER_NAME}.json" ]]; then
@@ -116,7 +116,7 @@ jobs:
               exit 1
             fi
           fi
-          
+
           # Verify the file exists and has content
           if [[ ! -f "router_inference/predictions/${ROUTER_NAME}.json" ]]; then
             echo "Error: Prediction file was not copied successfully" >&2
diff --git a/automation/process_pr_submission.py b/automation/process_pr_submission.py
index 370fed86..9cfd04e8 100644
--- a/automation/process_pr_submission.py
+++ b/automation/process_pr_submission.py
@@ -151,7 +151,7 @@ def cleanup_worktree(worktree_path: Path, branch_name: str, *, keep: bool) -> No
 def ensure_prediction_file_added(
     worktree_path: Path, base_ref: str, router_name: str
 ) -> None:
-    """Verify the PR adds a new prediction file for the specified router."""
+    """Verify the PR adds or modifies a prediction file for the specified router."""
 
     target_path = Path("router_inference") / "predictions" / f"{router_name}.json"
 
@@ -174,14 +174,20 @@ def ensure_prediction_file_added(
 
     lines = [line.strip() for line in completed.stdout.splitlines() if line.strip()]
     for line in lines:
-        if line.startswith("A\t") or line.startswith("A "):
+        # Allow both added (A) and modified (M) files
+        if (
+            line.startswith("A\t")
+            or line.startswith("A ")
+            or line.startswith("M\t")
+            or line.startswith("M ")
+        ):
             return
 
     raise RuntimeError(
         textwrap.dedent(
             f"""
-            Expected pull request to add a new prediction file {target_path}.
-            Diff against {base_ref} did not show a newly added file.
+            Expected pull request to add or modify a prediction file {target_path}.
+            Diff against {base_ref} did not show a newly added or modified file.
             """
         ).strip()
     )

From f52e84643044dbe82534e38be26daf9aaeea689f Mon Sep 17 00:00:00 2001
From: yl231 <yifan21lu@gmail.com>
Date: Sat, 29 Nov 2025 16:45:03 -0600
Subject: [PATCH 4/6] fixed comparing with the pr branch rather than main
 branch

---
 .github/workflows/pr-evaluation.yml | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/pr-evaluation.yml b/.github/workflows/pr-evaluation.yml
index 6887ec3f..9cf7afc9 100644
--- a/.github/workflows/pr-evaluation.yml
+++ b/.github/workflows/pr-evaluation.yml
@@ -147,10 +147,14 @@ jobs:
           ROUTERARENA_DATASET_DIR: ${{ github.workspace }}/dataset
         run: |
           set -euo pipefail
+          # Use the PR's base branch for comparison (fork's base, not upstream main)
+          BASE_REF="${{ github.event.pull_request.base.ref }}"
+          BASE_SHA="${{ github.event.pull_request.base.sha }}"
           uv run python automation/process_pr_submission.py \
             --pr "${{ github.event.pull_request.number }}" \
             --router "${{ steps.detect.outputs.router }}" \
-            --split "${{ steps.detect.outputs.split }}" > evaluation_output.txt 2>&1
+            --split "${{ steps.detect.outputs.split }}" \
+            --base-ref "$BASE_SHA" > evaluation_output.txt 2>&1
           # Extract metrics from output
           if grep -q "Metrics:" evaluation_output.txt; then
             python3 automation/extract_metrics.py evaluation_output.txt

From d9022fccec7885696cb8042214a9822ecab3cc97 Mon Sep 17 00:00:00 2001
From: yl231 <yifan21lu@gmail.com>
Date: Sun, 30 Nov 2025 01:47:22 -0600
Subject: [PATCH 5/6] re-run

---
 automation/process_pr_submission.py | 147 +++++++++++++++++++++++-----
 1 file changed, 121 insertions(+), 26 deletions(-)

diff --git a/automation/process_pr_submission.py b/automation/process_pr_submission.py
index 9cfd04e8..dfdf03ef 100644
--- a/automation/process_pr_submission.py
+++ b/automation/process_pr_submission.py
@@ -155,39 +155,134 @@ def ensure_prediction_file_added(
 
     target_path = Path("router_inference") / "predictions" / f"{router_name}.json"
 
-    diff_cmd = [
-        "git",
-        "diff",
-        "--name-status",
-        f"{base_ref}...HEAD",
-        "--",
-        str(target_path),
-    ]
+    # Check if the file exists in the PR branch (HEAD)
+    full_path = worktree_path / target_path
+    if not full_path.exists():
+        raise RuntimeError(
+            f"Prediction file {target_path} does not exist in PR branch."
+        )
 
-    completed = subprocess.run(
-        diff_cmd,
-        cwd=worktree_path,
-        check=True,
-        text=True,
-        capture_output=True,
-    )
+    # Try to check if base_ref exists in the worktree's git repository
+    base_ref_exists = False
+    try:
+        subprocess.run(
+            ["git", "cat-file", "-e", base_ref],
+            cwd=worktree_path,
+            check=True,
+            capture_output=True,
+        )
+        base_ref_exists = True
+    except subprocess.CalledProcessError:
+        # Base ref not found, try to fetch it from origin
+        try:
+            subprocess.run(
+                ["git", "fetch", "origin", base_ref],
+                cwd=worktree_path,
+                check=True,
+                capture_output=True,
+            )
+            # Verify it exists now
+            try:
+                subprocess.run(
+                    ["git", "cat-file", "-e", base_ref],
+                    cwd=worktree_path,
+                    check=True,
+                    capture_output=True,
+                )
+                base_ref_exists = True
+            except subprocess.CalledProcessError:
+                pass
+        except subprocess.CalledProcessError:
+            pass
+
+    # If base_ref exists, use diff to check if file was added/modified
+    if base_ref_exists:
+        # Try three-dot diff first (shows changes from merge-base)
+        diff_cmd = [
+            "git",
+            "diff",
+            "--name-status",
+            f"{base_ref}...HEAD",
+            "--",
+            str(target_path),
+        ]
+
+        completed = subprocess.run(
+            diff_cmd,
+            cwd=worktree_path,
+            check=False,  # Don't fail if diff fails
+            text=True,
+            capture_output=True,
+        )
+
+        lines = [line.strip() for line in completed.stdout.splitlines() if line.strip()]
+        for line in lines:
+            # Allow both added (A) and modified (M) files
+            if (
+                line.startswith("A\t")
+                or line.startswith("A ")
+                or line.startswith("M\t")
+                or line.startswith("M ")
+            ):
+                return
+
+        # If three-dot diff didn't work, try two-dot diff
+        diff_cmd = [
+            "git",
+            "diff",
+            "--name-status",
+            f"{base_ref}..HEAD",
+            "--",
+            str(target_path),
+        ]
 
-    lines = [line.strip() for line in completed.stdout.splitlines() if line.strip()]
-    for line in lines:
-        # Allow both added (A) and modified (M) files
-        if (
-            line.startswith("A\t")
-            or line.startswith("A ")
-            or line.startswith("M\t")
-            or line.startswith("M ")
-        ):
+        completed = subprocess.run(
+            diff_cmd,
+            cwd=worktree_path,
+            check=False,
+            text=True,
+            capture_output=True,
+        )
+
+        lines = [line.strip() for line in completed.stdout.splitlines() if line.strip()]
+        for line in lines:
+            if (
+                line.startswith("A\t")
+                or line.startswith("A ")
+                or line.startswith("M\t")
+                or line.startswith("M ")
+            ):
+                return
+
+        # If both diffs failed, check if file exists in base_ref
+        try:
+            subprocess.run(
+                ["git", "cat-file", "-e", f"{base_ref}:{target_path}"],
+                cwd=worktree_path,
+                check=True,
+                capture_output=True,
+            )
+            # File exists in base_ref, so it's a modification
             return
+        except subprocess.CalledProcessError:
+            # File doesn't exist in base_ref, so it's an addition
+            # Since file exists in HEAD, this is valid
+            return
+
+    # If base_ref doesn't exist, we can't verify via diff
+    # But since the workflow already detected the file change, we'll accept it
+    # Just verify the file exists in the PR branch
+    if full_path.exists():
+        return
 
+    # Final fallback: if we can't verify via diff but file exists, accept it
+    # The workflow has already verified the file change
     raise RuntimeError(
         textwrap.dedent(
             f"""
-            Expected pull request to add or modify a prediction file {target_path}.
-            Diff against {base_ref} did not show a newly added or modified file.
+            Could not verify prediction file {target_path} was added or modified.
+            Base ref {base_ref} is not available in PR branch history.
+            File exists in PR branch: {full_path.exists()}
             """
         ).strip()
     )

From ae5068693286912e0937f12d8d342ff3ac2134b9 Mon Sep 17 00:00:00 2001
From: yl231 <yifan21lu@gmail.com>
Date: Sun, 30 Nov 2025 02:05:21 -0600
Subject: [PATCH 6/6] re-test

---
 .github/workflows/pr-evaluation.yml |  45 ++-------
 automation/process_pr_submission.py | 147 +++++-----------------------
 2 files changed, 35 insertions(+), 157 deletions(-)

diff --git a/.github/workflows/pr-evaluation.yml b/.github/workflows/pr-evaluation.yml
index 9cf7afc9..d13f81c2 100644
--- a/.github/workflows/pr-evaluation.yml
+++ b/.github/workflows/pr-evaluation.yml
@@ -79,50 +79,23 @@ jobs:
           fi
           echo "split=$SPLIT" >> "$GITHUB_OUTPUT"
 
-          # Save the prediction file to temporary location for copying to main branch
-          mkdir -p /tmp/pr_predictions
-          cp "$PREDICTION_FILE" "/tmp/pr_predictions/${ROUTER_NAME}.json"
-          echo "Saved prediction file to /tmp/pr_predictions/${ROUTER_NAME}.json"
-
-      - name: Checkout main branch for evaluation
-        if: ${{ steps.detect.outputs.router != '' }}
-        uses: actions/checkout@v4
-        with:
-          ref: main
-          fetch-depth: 0
-
-      - name: Copy prediction file from PR branch
+      - name: Continue using PR branch for evaluation
         if: ${{ steps.detect.outputs.router != '' }}
         run: |
           set -euo pipefail
+          # We stay on the PR branch to use the code from the PR
+          # This allows the PR to include both router submissions AND code improvements
+          # The prediction file is already available from the detection step
           ROUTER_NAME="${{ steps.detect.outputs.router }}"
-          mkdir -p "router_inference/predictions"
-
-          # Copy the prediction file that was saved in the previous step
-          # Always use the PR's version, overwriting any existing file in main branch
-          if [[ -f "/tmp/pr_predictions/${ROUTER_NAME}.json" ]]; then
-            cp -f "/tmp/pr_predictions/${ROUTER_NAME}.json" "router_inference/predictions/${ROUTER_NAME}.json"
-            echo "Successfully copied prediction file from PR branch (overwrote any existing file)"
-          else
-            # Fallback: try to fetch from PR branch directly
-            echo "Prediction file not in /tmp, attempting to fetch from PR branch..."
-            git fetch origin pull/${{ github.event.pull_request.number }}/head:pr-branch || true
-            PREDICTION_FILE="router_inference/predictions/${ROUTER_NAME}.json"
-            # Use -f flag to force overwrite if file exists
-            if git show pr-branch:"$PREDICTION_FILE" > "router_inference/predictions/${ROUTER_NAME}.json" 2>/dev/null; then
-              echo "Successfully retrieved prediction file from PR branch (overwrote any existing file)"
-            else
-              echo "Error: Could not retrieve prediction file from PR branch" >&2
-              exit 1
-            fi
-          fi
+          PREDICTION_FILE="router_inference/predictions/${ROUTER_NAME}.json"
 
           # Verify the file exists and has content
-          if [[ ! -f "router_inference/predictions/${ROUTER_NAME}.json" ]]; then
-            echo "Error: Prediction file was not copied successfully" >&2
+          if [[ ! -f "$PREDICTION_FILE" ]]; then
+            echo "Error: Prediction file not found at $PREDICTION_FILE" >&2
             exit 1
           fi
-          echo "Prediction file ready: router_inference/predictions/${ROUTER_NAME}.json"
+          echo "Using PR branch for evaluation (includes both router submission and code changes)"
+          echo "Prediction file ready: $PREDICTION_FILE"
 
       - name: Show detected router
         if: ${{ steps.detect.outputs.router != '' }}
diff --git a/automation/process_pr_submission.py b/automation/process_pr_submission.py
index dfdf03ef..9cfd04e8 100644
--- a/automation/process_pr_submission.py
+++ b/automation/process_pr_submission.py
@@ -155,134 +155,39 @@ def ensure_prediction_file_added(
 
     target_path = Path("router_inference") / "predictions" / f"{router_name}.json"
 
-    # Check if the file exists in the PR branch (HEAD)
-    full_path = worktree_path / target_path
-    if not full_path.exists():
-        raise RuntimeError(
-            f"Prediction file {target_path} does not exist in PR branch."
-        )
-
-    # Try to check if base_ref exists in the worktree's git repository
-    base_ref_exists = False
-    try:
-        subprocess.run(
-            ["git", "cat-file", "-e", base_ref],
-            cwd=worktree_path,
-            check=True,
-            capture_output=True,
-        )
-        base_ref_exists = True
-    except subprocess.CalledProcessError:
-        # Base ref not found, try to fetch it from origin
-        try:
-            subprocess.run(
-                ["git", "fetch", "origin", base_ref],
-                cwd=worktree_path,
-                check=True,
-                capture_output=True,
-            )
-            # Verify it exists now
-            try:
-                subprocess.run(
-                    ["git", "cat-file", "-e", base_ref],
-                    cwd=worktree_path,
-                    check=True,
-                    capture_output=True,
-                )
-                base_ref_exists = True
-            except subprocess.CalledProcessError:
-                pass
-        except subprocess.CalledProcessError:
-            pass
-
-    # If base_ref exists, use diff to check if file was added/modified
-    if base_ref_exists:
-        # Try three-dot diff first (shows changes from merge-base)
-        diff_cmd = [
-            "git",
-            "diff",
-            "--name-status",
-            f"{base_ref}...HEAD",
-            "--",
-            str(target_path),
-        ]
-
-        completed = subprocess.run(
-            diff_cmd,
-            cwd=worktree_path,
-            check=False,  # Don't fail if diff fails
-            text=True,
-            capture_output=True,
-        )
-
-        lines = [line.strip() for line in completed.stdout.splitlines() if line.strip()]
-        for line in lines:
-            # Allow both added (A) and modified (M) files
-            if (
-                line.startswith("A\t")
-                or line.startswith("A ")
-                or line.startswith("M\t")
-                or line.startswith("M ")
-            ):
-                return
-
-        # If three-dot diff didn't work, try two-dot diff
-        diff_cmd = [
-            "git",
-            "diff",
-            "--name-status",
-            f"{base_ref}..HEAD",
-            "--",
-            str(target_path),
-        ]
+    diff_cmd = [
+        "git",
+        "diff",
+        "--name-status",
+        f"{base_ref}...HEAD",
+        "--",
+        str(target_path),
+    ]
 
-        completed = subprocess.run(
-            diff_cmd,
-            cwd=worktree_path,
-            check=False,
-            text=True,
-            capture_output=True,
-        )
+    completed = subprocess.run(
+        diff_cmd,
+        cwd=worktree_path,
+        check=True,
+        text=True,
+        capture_output=True,
+    )
 
-        lines = [line.strip() for line in completed.stdout.splitlines() if line.strip()]
-        for line in lines:
-            if (
-                line.startswith("A\t")
-                or line.startswith("A ")
-                or line.startswith("M\t")
-                or line.startswith("M ")
-            ):
-                return
-
-        # If both diffs failed, check if file exists in base_ref
-        try:
-            subprocess.run(
-                ["git", "cat-file", "-e", f"{base_ref}:{target_path}"],
-                cwd=worktree_path,
-                check=True,
-                capture_output=True,
-            )
-            # File exists in base_ref, so it's a modification
+    lines = [line.strip() for line in completed.stdout.splitlines() if line.strip()]
+    for line in lines:
+        # Allow both added (A) and modified (M) files
+        if (
+            line.startswith("A\t")
+            or line.startswith("A ")
+            or line.startswith("M\t")
+            or line.startswith("M ")
+        ):
             return
-        except subprocess.CalledProcessError:
-            # File doesn't exist in base_ref, so it's an addition
-            # Since file exists in HEAD, this is valid
-            return
-
-    # If base_ref doesn't exist, we can't verify via diff
-    # But since the workflow already detected the file change, we'll accept it
-    # Just verify the file exists in the PR branch
-    if full_path.exists():
-        return
 
-    # Final fallback: if we can't verify via diff but file exists, accept it
-    # The workflow has already verified the file change
     raise RuntimeError(
         textwrap.dedent(
             f"""
-            Could not verify prediction file {target_path} was added or modified.
-            Base ref {base_ref} is not available in PR branch history.
-            File exists in PR branch: {full_path.exists()}
+            Expected pull request to add or modify a prediction file {target_path}.
+            Diff against {base_ref} did not show a newly added or modified file.
             """
         ).strip()
     )