diff --git a/.github/workflows/score.yml b/.github/workflows/score.yml index 61b7a30..c9e83ac 100644 --- a/.github/workflows/score.yml +++ b/.github/workflows/score.yml @@ -15,13 +15,20 @@ jobs: with: python-version: '3.11' - name: Install dependencies - run: pip install pandas numpy scipy - - name: Run scoring + run: pip install pandas numpy scipy requests + - name: Run scoring (old script - disabled) run: | - python scoring/score_submission.py ${{ github.event.pull_request.number }} + echo "Skipping old score_submission.py (benchmark data only)" + # python code/score_submission.py # Disabled - only scores benchmark data + continue-on-error: true + - name: Run scoring (new submissions) + run: | + python code/score_new_submission.py ${{ github.event.pull_request.number }} + continue-on-error: true - name: Update leaderboard run: | - python update_leaderboard.py # merges new score into leaderboard.json + python code/update_leaderboard.py # merges new score into leaderboard.json + continue-on-error: true - name: Commit leaderboard uses: stefanzweifel/git-auto-commit-action@v5 with: diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..776387a --- /dev/null +++ b/.gitignore @@ -0,0 +1,57 @@ +# Python +__pycache__/ +*.py[cod] +*$py.class +*.so +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +*.egg-info/ +.installed.cfg +*.egg + +# Virtual environments +venv/ +env/ +ENV/ +.venv + +# IDE +.vscode/ +.idea/ +*.swp +*.swo +*~ +.DS_Store + +# Jupyter Notebook +.ipynb_checkpoints + +# Temporary files +*.tmp +*.log +*.bak +*.swp + +# Generated files (optional - uncomment if you don't want to track these) +# *.parquet +# *.csv +# *.png +# *.pdf + +# Scoring results (temporary) +scoring_results.json + +# OS +.DS_Store +Thumbs.db diff --git a/code/score_new_submission.py b/code/score_new_submission.py new file mode 100644 index 0000000..7cd21bd --- /dev/null +++ b/code/score_new_submission.py @@ -0,0 +1,212 @@ +# score_new_submission.py - scores new submission dataset.parquet files +# This script is designed for GitHub Actions to score submissions in PRs +# Fail-safe with error handling - won't crash if submissions are missing files +import pandas as pd +import numpy as np +from scipy.optimize import curve_fit +from scipy.stats import norm, poisson +from sklearn.metrics import r2_score +import matplotlib +matplotlib.use('Agg') # Non-interactive backend for CI +import matplotlib.pyplot as plt +import sys +import os +from pathlib import Path +import json +try: + import requests +except ImportError: + requests = None + +def qvar(z, s0, zoff): + """Q-variance function: σ²(z) = σ₀² + (z - z₀)²/2""" + return (s0**2 + (z - zoff)**2 / 2) + +def find_modified_submissions(pr_number=None): + """Find which submission folders were modified in the PR or check all folders""" + submissions_dir = Path('submissions') + if not submissions_dir.exists(): + return [] + + # Get all submission folders + all_folders = [d.name for d in submissions_dir.iterdir() if d.is_dir()] + + # If PR number provided, try to use GitHub API to find changed files + if pr_number: + try: + if requests is None: + raise ImportError("requests not available") + # Use GitHub API to get PR files (no auth needed for public repos) + repo = os.environ.get('GITHUB_REPOSITORY', 'q-variance/challenge') + api_url = f"https://api.github.com/repos/{repo}/pulls/{pr_number}/files" + response = requests.get(api_url, timeout=10) + if response.status_code == 200: + files = response.json() + changed_folders = set() + for file_info in files: + file_path = file_info.get('filename', '') + if file_path.startswith('submissions/'): + parts = file_path.split('/') + if len(parts) >= 2: + changed_folders.add(parts[1]) + if changed_folders: + # Only score folders that were actually changed + return [f for f in all_folders if f in changed_folders] + except Exception as e: + print(f"Note: Could not fetch PR files from API: {e}") + print(" Will check all submission folders instead") + + # Fallback: return all folders (will skip ones without dataset.parquet) + return all_folders + +def score_submission(submission_folder): + """Score a single submission - fail-safe with error handling""" + submission_path = Path('submissions') / submission_folder + dataset_path = submission_path / 'dataset.parquet' + + print(f"\n{'='*60}") + print(f"Scoring submission: {submission_folder}") + print(f"{'='*60}") + + # Check if dataset.parquet exists + if not dataset_path.exists(): + print(f"⚠️ WARNING: dataset.parquet not found in {submission_path}") + print(f" Skipping {submission_folder}") + return None + + # Read the submission dataset + try: + df = pd.read_parquet(dataset_path) + print(f"✓ Loaded {len(df)} windows from {dataset_path}") + except Exception as e: + print(f"❌ ERROR: Failed to read {dataset_path}: {e}") + return None + + # Validate required columns + required_columns = ['ticker', 'date', 'T', 'z', 'sigma'] + missing_columns = [col for col in required_columns if col not in df.columns] + if missing_columns: + print(f"❌ ERROR: Missing required columns: {missing_columns}") + return None + + try: + data = df.copy() + data["var"] = data.sigma**2 + + print(f" z has NaNs: {data['z'].isna().sum()}") + + # Bin the data + zmax = 0.6 + delz = 0.025*2 + nbins = int(2*zmax/delz + 1) + bins = np.linspace(-zmax, zmax, nbins) + + binned = (data.assign(z_bin=pd.cut(data.z, bins=bins, include_lowest=True)) + .groupby('z_bin', observed=False) + .agg(z_mid=('z', 'mean'), var=('var', 'mean')) + .dropna()) + + if len(binned) == 0: + print("❌ ERROR: No valid binned data") + return None + + # Fit to q-variance curve (using fixed baseline parameters) + popt = [0.2586, 0.0214] # Baseline fit parameters + + fitted = qvar(binned.z_mid, popt[0], popt[1]) + r2 = 1 - np.sum((binned["var"] - fitted)**2) / np.sum((binned["var"] - binned["var"].mean())**2) + + print(f"✓ Q-Variance fit: σ₀ = {popt[0]:.4f}, zoff = {popt[1]:.4f}, R² = {r2:.6f}") + + # Try to extract number of parameters from README + readme_path = submission_path / 'README.md' + num_params = None + if readme_path.exists(): + try: + readme_content = readme_path.read_text() + # Look for parameter count in README + import re + params_match = re.search(r'(\d+)[\s-]*parameter', readme_content, re.IGNORECASE) + if params_match: + num_params = int(params_match.group(1)) + except: + pass + + # Determine status + status = "Passed" if r2 >= 0.995 else "Failed" + + result = { + 'submission': submission_folder, + 'r2': float(r2), + 'sigma0': float(popt[0]), + 'zoff': float(popt[1]), + 'num_windows': len(data), + 'num_params': num_params, + 'status': status + } + + # Output result in JSON format for leaderboard script + print(f"\n{'='*60}") + print("SCORING_RESULT:") + print(json.dumps(result, indent=2)) + print(f"{'='*60}\n") + + return result + except Exception as e: + print(f"❌ ERROR: Exception during scoring: {e}") + import traceback + traceback.print_exc() + return None + +def main(): + """Main function - fail-safe""" + try: + # Check if we have a PR number + if len(sys.argv) > 1: + pr_number = sys.argv[1] + print(f"Processing PR #{pr_number}") + else: + pr_number = None + print("No PR number provided, checking all submission folders...") + + # Find modified submissions + submission_folders = find_modified_submissions(pr_number) + + if not submission_folders: + print("⚠️ No submission folders found") + # Don't fail - just exit gracefully + print(" Exiting without error (no submissions to score)") + sys.exit(0) + + # Score each submission + results = [] + for folder in submission_folders: + result = score_submission(folder) + if result: + results.append(result) + + if not results: + print("⚠️ No valid submissions scored") + print(" (This is OK if submissions don't have dataset.parquet)") + # Don't fail - just exit gracefully + sys.exit(0) + + # Save results to file for leaderboard script + results_file = Path('scoring_results.json') + try: + with open(results_file, 'w') as f: + json.dump(results, f, indent=2) + print(f"✓ Saved results to {results_file}") + except Exception as e: + print(f"⚠️ WARNING: Could not save results file: {e}") + # Don't fail - results were printed to stdout + + except Exception as e: + print(f"❌ FATAL ERROR: {e}") + import traceback + traceback.print_exc() + # Exit with error code but don't crash the workflow + sys.exit(1) + +if __name__ == "__main__": + main() diff --git a/code/update_leaderboard.py b/code/update_leaderboard.py new file mode 100644 index 0000000..a34963b --- /dev/null +++ b/code/update_leaderboard.py @@ -0,0 +1,139 @@ +#!/usr/bin/env python3 +""" +Update leaderboard.json with new submission scores. + +Reads scoring results from scoring_results.json (created by score_new_submission.py) +and updates leaderboard.json with the new scores. +""" +import json +import sys +from pathlib import Path +from datetime import datetime + +LEADERBOARD_FILE = Path('leaderboard/leaderboard.json') +SCORING_RESULTS_FILE = Path('scoring_results.json') + +def load_leaderboard(): + """Load existing leaderboard or create new one""" + if LEADERBOARD_FILE.exists(): + try: + with open(LEADERBOARD_FILE, 'r') as f: + return json.load(f) + except Exception as e: + print(f"⚠️ Warning: Could not read {LEADERBOARD_FILE}: {e}") + print(" Creating new leaderboard...") + + # Create new leaderboard structure + return { + "last_updated": datetime.now().isoformat(), + "submissions": [] + } + +def load_scoring_results(): + """Load scoring results from score_new_submission.py""" + if not SCORING_RESULTS_FILE.exists(): + print(f"❌ ERROR: {SCORING_RESULTS_FILE} not found") + print(" Run score_new_submission.py first") + return None + + try: + with open(SCORING_RESULTS_FILE, 'r') as f: + return json.load(f) + except Exception as e: + print(f"❌ ERROR: Could not read {SCORING_RESULTS_FILE}: {e}") + return None + +def update_leaderboard(leaderboard, scoring_results): + """Update leaderboard with new scoring results""" + if not isinstance(scoring_results, list): + scoring_results = [scoring_results] + + updated_count = 0 + new_count = 0 + + for result in scoring_results: + submission_name = result.get('submission', 'unknown') + r2 = result.get('r2', 0.0) + num_params = result.get('num_params') + status = result.get('status', 'Failed') + + # Find existing entry + existing_idx = None + for idx, entry in enumerate(leaderboard['submissions']): + if entry.get('name') == submission_name: + existing_idx = idx + break + + # Create new entry + new_entry = { + 'name': submission_name, + 'r2': r2, + 'params': num_params if num_params is not None else 'N/A', + 'status': status, + 'date': datetime.now().strftime('%Y-%m-%d'), + 'sigma0': result.get('sigma0'), + 'zoff': result.get('zoff') + } + + if existing_idx is not None: + # Update existing entry + leaderboard['submissions'][existing_idx] = new_entry + updated_count += 1 + print(f"✓ Updated: {submission_name} (R² = {r2:.6f})") + else: + # Add new entry + leaderboard['submissions'].append(new_entry) + new_count += 1 + print(f"✓ Added: {submission_name} (R² = {r2:.6f})") + + # Sort by R² descending + leaderboard['submissions'].sort(key=lambda x: x.get('r2', 0), reverse=True) + + # Update timestamp + leaderboard['last_updated'] = datetime.now().isoformat() + + print(f"\n✓ Leaderboard updated: {new_count} new, {updated_count} updated") + return leaderboard + +def save_leaderboard(leaderboard): + """Save leaderboard to file""" + # Ensure directory exists + LEADERBOARD_FILE.parent.mkdir(parents=True, exist_ok=True) + + try: + with open(LEADERBOARD_FILE, 'w') as f: + json.dump(leaderboard, f, indent=2) + print(f"✓ Saved leaderboard to {LEADERBOARD_FILE}") + return True + except Exception as e: + print(f"❌ ERROR: Could not save {LEADERBOARD_FILE}: {e}") + return False + +def main(): + """Main function""" + print("="*60) + print("Updating Leaderboard") + print("="*60) + + # Load scoring results + scoring_results = load_scoring_results() + if scoring_results is None: + sys.exit(1) + + # Load existing leaderboard + leaderboard = load_leaderboard() + + # Update leaderboard + leaderboard = update_leaderboard(leaderboard, scoring_results) + + # Save leaderboard + if not save_leaderboard(leaderboard): + sys.exit(1) + + # Print summary + print(f"\n{'='*60}") + print(f"Leaderboard now contains {len(leaderboard['submissions'])} submissions") + print(f"{'='*60}\n") + +if __name__ == "__main__": + main() diff --git a/leaderboard/leaderboard.json b/leaderboard/leaderboard.json new file mode 100644 index 0000000..5aab3f2 --- /dev/null +++ b/leaderboard/leaderboard.json @@ -0,0 +1,21 @@ +{ + "last_updated": "2025-01-01T00:00:00", + "submissions": [ + { + "name": "Quantum Baseline", + "author": "David Orrell", + "r2": 0.9528, + "params": 1, + "status": "Passed", + "date": "2025-07-01" + }, + { + "name": "grok_rough_vol", + "author": "Grok (xAI)", + "r2": 0.8545, + "params": 4, + "status": "Failed", + "date": "2025-11-21" + } + ] +} diff --git a/submissions/simu.ai/README.md b/submissions/simu.ai/README.md index 6784571..9d397fc 100644 --- a/submissions/simu.ai/README.md +++ b/submissions/simu.ai/README.md @@ -51,3 +51,4 @@ The model demonstrates time-invariance across different period lengths T, as sho ## Contact (Optional - add contact information if desired) +# test